pax_global_header00006660000000000000000000000064151752323550014522gustar00rootroot0000000000000052 comment=6255b3956027aaa6c2092ed42869f58e8d1942f5 incus-7.0.0/000077500000000000000000000000001517523235500126475ustar00rootroot00000000000000incus-7.0.0/.codespell-ignore000066400000000000000000000000641517523235500161030ustar00rootroot00000000000000AtLeast attachs destOp ECT inport renderD requestor incus-7.0.0/.deepsource.toml000066400000000000000000000004061517523235500157600ustar00rootroot00000000000000version = 1 test_patterns = [ "test/**", "*_test.go" ] [[analyzers]] name = "python" enabled = true [analyzers.meta] runtime_version = "3.x.x" [[analyzers]] name = "go" enabled = true [analyzers.meta] import_paths = ["github.com/lxc/incus"] incus-7.0.0/.devcontainer/000077500000000000000000000000001517523235500154065ustar00rootroot00000000000000incus-7.0.0/.devcontainer/Dockerfile000066400000000000000000000061031517523235500174000ustar00rootroot00000000000000ARG GO_VERSION=1.25 ARG DEBIAN_VERSION=trixie # Go development container FROM golang:${GO_VERSION}-${DEBIAN_VERSION} ARG USERNAME=vscode ARG USER_UID=1000 ARG USER_GID=1000 # Install necessary tools. RUN sed -r -i 's/^Components: main$/Components: main contrib/g' /etc/apt/sources.list.d/debian.sources && \ apt update && \ apt install -y \ acl \ aspell \ aspell-en \ attr \ autoconf \ automake \ bind9-dnsutils \ btrfs-progs \ busybox-static \ ceph-common \ curl \ dnsmasq-base \ ebtables \ flake8 \ gettext \ git \ jq \ less \ libacl1-dev \ libcap-dev \ # libcowsql-dev libdbus-1-dev \ # liblxc-dev \ liblxc1 \ liblz4-dev \ libseccomp-dev \ libselinux1-dev \ libsqlite3-dev \ libtool \ libudev-dev \ libusb-1.0-0-dev \ libuv1-dev \ locales \ locales-all \ lvm2 \ lxc-dev \ lxc-templates \ make \ man-db \ pipx \ pkg-config \ protoc-gen-go \ python3-matplotlib \ python3.13-venv \ rsync \ ruby-mdl \ shellcheck \ socat \ sqlite3 \ squashfs-tools \ sudo \ tar \ tcl \ thin-provisioning-tools \ vim \ # Disabled for now, very slow to install. # zfsutils-linux xz-utils # Globally install codespell RUN pipx install --global codespell # Add vscode user and add it to sudoers. RUN groupadd -g 1000 $USERNAME && \ useradd -s /bin/bash -u $USER_UID -g $USER_GID -m $USERNAME && \ mkdir -p /etc/sudoers.d && \ echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME && \ chmod 0440 /etc/sudoers.d/$USERNAME # Setup for vscode user. USER $USERNAME ENV EDITOR=vi \ LANG=en_US.UTF-8 \ CGO_CFLAGS="-I/home/vscode/vendor/raft/include/ -I/home/vscode/vendor/cowsql/include/" \ CGO_LDFLAGS="-L/home/vscode/vendor/raft/.libs -L/home/vscode/vendor/cowsql/.libs/" \ LD_LIBRARY_PATH="/home/vscode/vendor/raft/.libs/:/home/vscode/vendor/cowsql/.libs/" \ CGO_LDFLAGS_ALLOW="(-Wl,-wrap,pthread_create)|(-Wl,-z,now)" # Build Go tools with user vscode to ensure correct file and directory permissions for the build artifacts. RUN go install -v github.com/google/go-licenses@latest && \ go install -v github.com/766b/go-outliner@latest && \ GOTOOLCHAIN="" go install -v golang.org/x/tools/gopls@latest && \ go install -v github.com/go-delve/delve/cmd/dlv@latest && \ go install -v golang.org/x/tools/cmd/goimports@latest && \ go install -v golang.org/x/vuln/cmd/govulncheck@latest && \ go install -v mvdan.cc/gofumpt@latest && \ curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin # Make dependencies COPY Makefile /home/vscode RUN cd /home/vscode && \ mkdir /home/vscode/vendor && \ make deps USER root # Since we use a volume for /go to persist the content between executions, we need to preserve the binaries. RUN mv /go/bin/* /usr/local/bin incus-7.0.0/.devcontainer/devcontainer.json000066400000000000000000000034461517523235500207710ustar00rootroot00000000000000{ "name": "Incus", "build": { "dockerfile": "Dockerfile", "context": ".." }, "customizations": { "vscode": { "extensions": [ "golang.go", "766b.go-outliner", "ms-azuretools.vscode-docker", "ms-vscode.makefile-tools", "github.vscode-github-actions", "davidanson.vscode-markdownlint", "shardulm94.trailing-spaces", "Gruntfuggly.todo-tree" ], "settings": { "files.insertFinalNewline": true, "go.goroot": "/usr/local/go", "go.gopath": "/go", "go.lintTool": "golangci-lint", "go.lintOnSave": "package", "go.lintFlags": [ "--fast" ], "go.useLanguageServer": true, "goOutliner.extendExplorerTab": true, "gopls": { "formatting.gofumpt": true, "formatting.local": "github.com/lxc/incus", "ui.diagnostic.staticcheck": false }, "[go]": { "editor.formatOnSave": true, "editor.codeActionsOnSave": { "source.organizeImports": "explicit" } }, "[go.mod]": { "editor.formatOnSave": true, "editor.codeActionsOnSave": { "source.organizeImports": "explicit" } }, "search.exclude": { "**/.git": true } } } }, "postCreateCommand": "go mod download", "mounts": [ "source=incus_devcontainer_cache,target=/home/vscode/.cache,type=volume", "source=incus_devcontainer_goroot,target=/go,type=volume" ], "runArgs": [ "--privileged", "-u", "vscode", "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined", "-v", "${env:HOME}/.ssh:/home/vscode/.ssh:ro", "--name", "${localEnv:USER}_incus_devcontainer" ], "remoteUser": "vscode" } incus-7.0.0/.github/000077500000000000000000000000001517523235500142075ustar00rootroot00000000000000incus-7.0.0/.github/CODEOWNERS000066400000000000000000000000141517523235500155750ustar00rootroot00000000000000* @stgraber incus-7.0.0/.github/FUNDING.yml000066400000000000000000000002551517523235500160260ustar00rootroot00000000000000# Frequent committers who contribute to Incus on their own time can add # themselves to the list here so users who feel like sponsoring can find # them. github: - stgraber incus-7.0.0/.github/ISSUE_TEMPLATE/000077500000000000000000000000001517523235500163725ustar00rootroot00000000000000incus-7.0.0/.github/ISSUE_TEMPLATE/bug-reports.yml000066400000000000000000000041661517523235500213750ustar00rootroot00000000000000name: Bug report description: File a bug report. type: bug body: - type: markdown attributes: value: | > [!NOTE] > Thank you for taking the time to fill out this bug report. As this issue will be read and debugged by humans, we kindly ask you to refrain from using AI tools to try to interpret your problem or suggest patches, as per our [contribution guidelines](https://github.com/lxc/incus/blob/main/CONTRIBUTING.md). - type: checkboxes attributes: label: Is there an existing issue for this? description: Please search to see if an issue already exists for the bug you encountered. options: - label: There is no existing issue for this bug required: true - type: checkboxes attributes: label: Is this happening on an up to date version of Incus? description: Please make sure that your system has all updates applied and is running a current version of Incus or Incus LTS. options: - label: This is happening on a supported version of Incus required: true - type: textarea attributes: label: Incus system details description: Output of `incus info`. render: yaml validations: required: true - type: textarea attributes: label: Instance details description: If the issue affects an instance, please include the output of `incus config show NAME`. validations: required: false - type: textarea attributes: label: Instance log description: If the issue is related to an instance startup failure, please include `incus info --show-log NAME`. validations: required: false - type: textarea attributes: label: Current behavior description: A concise description of what you're experiencing. validations: required: false - type: textarea attributes: label: Expected behavior description: A concise description of what you expected to happen. validations: required: false - type: textarea attributes: label: Steps to reproduce description: Step by step instructions to reproduce the behavior. placeholder: | 1. Step one 2. Step two 3. Step three validations: required: true incus-7.0.0/.github/ISSUE_TEMPLATE/config.yml000066400000000000000000000002441517523235500203620ustar00rootroot00000000000000blank_issues_enabled: false contact_links: - name: Support question url: https://discuss.linuxcontainers.org about: Please ask and answer questions here. incus-7.0.0/.github/ISSUE_TEMPLATE/feature-requests.yml000066400000000000000000000013561517523235500224260ustar00rootroot00000000000000name: Feature request description: File a feature request. type: feature body: - type: checkboxes attributes: label: Is there an existing issue for this? description: Please search to see if an issue already exists for the feature you'd like to see added. options: - label: There is no existing issue for this feature required: true - type: textarea attributes: label: What are you currently unable to do description: A concise description of the problem you're trying to solve. validations: required: true - type: textarea attributes: label: What do you think would need to be added description: A concise description of what you think should be added to Incus. validations: required: false incus-7.0.0/.github/SUPPORT.md000066400000000000000000000002751517523235500157110ustar00rootroot00000000000000The Incus team uses GitHub for issue and feature tracking, not for user support. For information on how to get support, see [Support](https://linuxcontainers.org/incus/docs/main/support/). incus-7.0.0/.github/dependabot.yml000066400000000000000000000002051517523235500170340ustar00rootroot00000000000000version: 2 updates: - package-ecosystem: "github-actions" directory: "/" labels: [] schedule: interval: "weekly" incus-7.0.0/.github/labeler.yml000066400000000000000000000003031517523235500163340ustar00rootroot00000000000000API: - changed-files: - any-glob-to-any-file: - doc/api-extensions.md - doc/rest-api.yaml - shared/api/**/* Documentation: - changed-files: - any-glob-to-any-file: - doc/**/* incus-7.0.0/.github/workflows/000077500000000000000000000000001517523235500162445ustar00rootroot00000000000000incus-7.0.0/.github/workflows/commits.yml000066400000000000000000000024231517523235500204430ustar00rootroot00000000000000name: Commits on: - pull_request permissions: contents: read jobs: dco-check: permissions: pull-requests: read # for tim-actions/get-pr-commits to get list of commits from the PR name: Signed-off-by (DCO) runs-on: ubuntu-24.04 steps: - name: Get PR Commits id: 'get-pr-commits' uses: tim-actions/get-pr-commits@master with: token: ${{ secrets.GITHUB_TOKEN }} - name: Check that all commits are signed-off uses: tim-actions/dco@master with: commits: ${{ steps.get-pr-commits.outputs.commits }} target-branch: permissions: contents: none name: Branch target runs-on: ubuntu-24.04 steps: - name: Check branch target env: TARGET: ${{ github.event.pull_request.base.ref }} TITLE: ${{ github.event.pull_request.title }} run: | set -eux TARGET_FROM_PR_TITLE="$(echo "${TITLE}" | sed -n 's/.*(\(stable-[0-9]\.[0-9]\))$/\1/p')" if [ -z "${TARGET_FROM_PR_TITLE}" ]; then TARGET_FROM_PR_TITLE="main" else echo "Branch target overridden from PR title" fi [ "${TARGET}" = "${TARGET_FROM_PR_TITLE}" ] && exit 0 echo "Invalid branch target: ${TARGET} != ${TARGET_FROM_PR_TITLE}" exit 1 incus-7.0.0/.github/workflows/release.yml000066400000000000000000000024061517523235500204110ustar00rootroot00000000000000name: Release on: push: tags: - '*' permissions: contents: write issues: write id-token: write attestations: write jobs: goreleaser: name: Release runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v6 with: fetch-depth: 0 - name: Install Go uses: actions/setup-go@v6 with: go-version: stable - name: Install syft uses: anchore/sbom-action/download-syft@v0 - name: Format version id: version run: | raw="${GITHUB_REF_NAME#v}" IFS='.' read -r major minor patch <<< "$raw" if [ "${patch}" = "0" ]; then echo "value=${major}.${minor}" >> $GITHUB_OUTPUT else echo "value=${major}.${minor}.${patch}" >> $GITHUB_OUTPUT fi - name: Run GoReleaser uses: goreleaser/goreleaser-action@v7 with: distribution: goreleaser version: latest args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} INCUS_VERSION: ${{ steps.version.outputs.value }} - name: Handle attestation uses: actions/attest@v4 with: subject-checksums: ./dist/checksums.txt incus-7.0.0/.github/workflows/slop.yml000066400000000000000000000020271517523235500177450ustar00rootroot00000000000000name: Cleanup slop on: issues: types: - opened permissions: issues: write jobs: close-untyped: name: Close issue if type is missing if: ${{ !github.event.issue.pull_request && !github.event.issue.type && github.event.issue.author_association != 'MEMBER' && github.event.issue.author_association != 'OWNER' }} runs-on: ubuntu-latest steps: - name: Close issue uses: actions/github-script@v9 with: script: | await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, body: "Issues must be created through the [GitHub web interface](https://github.com/lxc/incus/issues/new/choose). Closing automatically." }); await github.rest.issues.update({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, state: "closed", }); incus-7.0.0/.github/workflows/tests.yml000066400000000000000000000476341517523235500201470ustar00rootroot00000000000000name: Tests on: push: branches: - main - stable-* pull_request: permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: code-tests: name: Code runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: go: - oldstable - stable - tip steps: - name: Checkout uses: actions/checkout@v6 with: # Differential ShellCheck requires full git history fetch-depth: 0 - name: Dependency Review uses: actions/dependency-review-action@v4 if: github.event_name == 'pull_request' with: allow-ghsas: GHSA-4p9m-8gc4-rw2h - id: ShellCheck name: Differential ShellCheck uses: redhat-plumbers-in-action/differential-shellcheck@v5 env: SHELLCHECK_OPTS: --shell sh with: token: ${{ secrets.GITHUB_TOKEN }} exclude-path: internal/server/instance/drivers/agent-loader/rc.d/incus-agent if: github.event_name == 'pull_request' && matrix.go == 'stable' - name: Upload artifact with ShellCheck defects in SARIF format uses: actions/upload-artifact@v7 with: name: Differential ShellCheck SARIF path: ${{ steps.ShellCheck.outputs.sarif }} if: github.event_name == 'pull_request' && matrix.go == 'stable' - name: Install Go (${{ matrix.go }}) uses: actions/setup-go@v6 with: go-version: ${{ matrix.go }} if: matrix.go != 'tip' - name: Install Go (stable) uses: actions/setup-go@v6 with: go-version: stable if: matrix.go == 'tip' - name: Install Go (tip) run: | go install golang.org/dl/gotip@latest gotip download ~/sdk/gotip/bin/go version echo "PATH=$HOME/go/bin:$HOME/sdk/gotip/bin/:$PATH" >> $GITHUB_ENV if: matrix.go == 'tip' - name: Install dependencies run: | sudo apt-get update sudo apt-get install --no-install-recommends -y \ curl \ gettext \ git \ libacl1-dev \ libcap-dev \ libdbus-1-dev \ libcowsql-dev \ liblxc-dev \ lxc-templates \ libseccomp-dev \ libselinux-dev \ libsqlite3-dev \ libtool \ libudev-dev \ make \ pipx \ pkg-config \ shellcheck # With pipx >= 1.5.0, we could use pipx --global instead. PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin \ pipx install codespell flake8 - name: Fix repository permissions run: | sudo chown -R runner:docker . - name: Check compatible min Go version run: | go mod tidy - name: Download go dependencies run: | go mod download - name: Run Incus build run: | make - name: Run static analysis env: GITHUB_BEFORE: ${{ github.event.before }} run: | make static-analysis - name: Unit tests (all) run: | sudo --preserve-env=CGO_CFLAGS,CGO_LDFLAGS,CGO_LDFLAGS_ALLOW,LD_LIBRARY_PATH LD_LIBRARY_PATH=${LD_LIBRARY_PATH} env "PATH=${PATH}" go test ./... system-tests: name: System strategy: fail-fast: false matrix: go: - oldstable - stable - tip suite: - cluster - standalone_core - standalone_container - standalone_network - standalone_storage backend: - dir os: - ubuntu-24.04 - ubuntu-24.04-arm include: # Run standalone storage tests on all storage drivers but only on Ubuntu 24.04 with stable Go - os: ubuntu-24.04 backend: btrfs go: stable suite: standalone_storage - os: ubuntu-24.04 backend: ceph go: stable suite: standalone_storage - os: ubuntu-24.04 backend: linstor go: stable suite: standalone_storage - os: ubuntu-24.04 backend: lvm go: stable suite: standalone_storage - os: ubuntu-24.04 backend: random go: stable suite: standalone_storage - os: ubuntu-24.04 backend: zfs go: stable suite: standalone_storage # Run cluster tests on all storage drivers but only on Ubuntu 24.04 with stable Go - os: ubuntu-24.04 backend: btrfs go: stable suite: cluster - os: ubuntu-24.04 backend: ceph go: stable suite: cluster - os: ubuntu-24.04 backend: linstor go: stable suite: cluster - os: ubuntu-24.04 backend: lvm go: stable suite: cluster - os: ubuntu-24.04 backend: random go: stable suite: cluster - os: ubuntu-24.04 backend: zfs go: stable suite: cluster runs-on: ${{ matrix.os }} steps: - name: Performance tuning run: | set -eux # optimize ext4 FSes for performance, not reliability for fs in $(findmnt --noheading --type ext4 --list --uniq | awk '{print $1}'); do # nombcache and data=writeback cannot be changed on remount sudo mount -o remount,noatime,barrier=0,commit=6000 "${fs}" || true done # disable dpkg from calling sync() echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io - name: Reclaim some space run: | set -eux sudo snap remove lxd --purge # Purge older snap revisions that are disabled/superseded by newer revisions of the same snap snap list --all | while read -r name _ rev _ _ notes _; do [ "${notes}" = "disabled" ] && snap remove "${name}" --revision "${rev}" --purge done || true # This was inspired from https://github.com/easimon/maximize-build-space df -h / # dotnet sudo rm -rf /usr/share/dotnet # android sudo rm -rf /usr/local/lib/android # haskell sudo rm -rf /opt/ghc df -h / - name: Remove docker run: | set -eux sudo apt-get autopurge -y moby-containerd docker uidmap sudo ip link delete docker0 sudo nft flush ruleset - name: Checkout uses: actions/checkout@v6 - name: Install Go (${{ matrix.go }}) uses: actions/setup-go@v6 with: go-version: ${{ matrix.go }} if: matrix.go != 'tip' - name: Install Go (stable) uses: actions/setup-go@v6 with: go-version: stable if: matrix.go == 'tip' - name: Install Go (tip) run: | go install golang.org/dl/gotip@latest gotip download ~/sdk/gotip/bin/go version echo "PATH=$HOME/go/bin:$HOME/sdk/gotip/bin/:$PATH" >> $GITHUB_ENV if: matrix.go == 'tip' - name: Install dependencies run: | set -x # Configure ppa:ubuntu-lxc/daily directly # (apt-add-repository relies on the Launchpad API which is unreliable). sudo install -d -m 0755 /etc/apt/keyrings codename="$(lsb_release -cs)" curl -fsSL "https://keyserver.ubuntu.com/pks/lookup?op=get&options=mr&search=0xE9C00C1B1B59A86C2CFEE6990CE27B8C4122B4B7" | sudo tee /etc/apt/keyrings/ubuntu-lxc-daily.asc > /dev/null sudo tee /etc/apt/sources.list.d/ubuntu-lxc-daily.sources > /dev/null <> "$GITHUB_ENV" - name: Run Incus build run: | make - name: Setup scratch space if: "matrix.backend == 'ceph' || matrix.backend == 'linstor'" run: | set -eux if mountpoint -q /mnt; then [ -e /mnt/swapfile ] && sudo swapoff /mnt/swapfile block_path="$(findmnt --noheadings --output SOURCE --target /mnt | sed 's/[0-9]\+$//')" sudo umount /mnt sudo wipefs -a "${block_path}" sudo ln -s "${block_path}" "/dev/scratch" else sudo truncate -s 20G /scratch.img block_path="$(sudo losetup --show -f /scratch.img)" sudo ln -s "${block_path}" "/dev/scratch" fi - name: Setup MicroCeph if: matrix.backend == 'ceph' run: | set -x sudo apt-get install --no-install-recommends -y snapd sudo snap install microceph --channel=quincy/stable sudo apt-get install --no-install-recommends -y ceph-common sudo microceph cluster bootstrap sudo microceph.ceph config set global osd_pool_default_size 1 sudo microceph.ceph config set global mon_allow_pool_delete true sudo microceph.ceph config set global osd_memory_target 939524096 sudo microceph.ceph osd crush rule rm replicated_rule sudo microceph.ceph osd crush rule create-replicated replicated default osd for flag in nosnaptrim noscrub nobackfill norebalance norecover noscrub nodeep-scrub; do sudo microceph.ceph osd set $flag done # Repurpose the ephemeral disk for ceph OSD. sudo microceph disk add --wipe "$(readlink -f /dev/scratch)" sudo rm -rf /etc/ceph sudo ln -s /var/snap/microceph/current/conf/ /etc/ceph sudo microceph enable rgw sudo microceph.ceph osd pool create cephfs_meta 32 sudo microceph.ceph osd pool create cephfs_data 32 sudo microceph.ceph fs new cephfs cephfs_meta cephfs_data sudo microceph.ceph fs ls sleep 30 sudo microceph.ceph status # Wait until there are no more "unkowns" pgs for _ in $(seq 60); do if sudo microceph.ceph pg stat | grep -wF unknown; then sleep 1 else break fi done sudo microceph.ceph status sudo rm -f /snap/bin/rbd - name: Setup LINSTOR if: matrix.backend == 'linstor' run: | set -x # Configure ppa:linbit/linbit-drbd9-stack directly # (apt-add-repository relies on the Launchpad API which is unreliable). sudo install -d -m 0755 /etc/apt/keyrings curl -fsSL "https://keyserver.ubuntu.com/pks/lookup?op=get&options=mr&search=0xCC1B5A793C04BB3905AD837734893610CEAA9512" | sudo tee /etc/apt/keyrings/linbit-drbd9-stack.asc > /dev/null sudo tee /etc/apt/sources.list.d/linbit-drbd9-stack.sources > /dev/null <> "$GITHUB_ENV" - name: "Ensure offline mode (block image server)" run: | sudo nft add table inet filter sudo nft add chain 'inet filter output { type filter hook output priority 10 ; }' sudo nft add rule inet filter output ip daddr 45.45.148.8 reject sudo nft add rule inet filter output ip6 daddr 2602:fc62:a:1::8 reject - name: "Run system tests (${{ matrix.go }}, ${{ matrix.suite }}, ${{ matrix.backend }})" env: CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)" INCUS_CEPH_CLUSTER: "ceph" INCUS_CEPH_CEPHFS: "cephfs" INCUS_CEPH_CEPHOBJECT_RADOSGW: "http://127.0.0.1" INCUS_LINSTOR_LOCAL_SATELLITE: "local" INCUS_CONCURRENT: "1" INCUS_VERBOSE: "1" INCUS_OFFLINE: "1" INCUS_TMPFS: "1" INCUS_REQUIRED_TESTS: "test_storage_buckets" run: | chmod +x ~ echo "root:1000000:1000000000" | sudo tee /etc/subuid /etc/subgid cd test sudo --preserve-env=PATH,GOPATH,GITHUB_ACTIONS,INCUS_VERBOSE,INCUS_BACKEND,INCUS_CEPH_CLUSTER,INCUS_CEPH_CEPHFS,INCUS_CEPH_CEPHOBJECT_RADOSGW,INCUS_LINSTOR_LOCAL_SATELLITE,INCUS_LINSTOR_CLUSTER,INCUS_OFFLINE,INCUS_SKIP_TESTS,INCUS_REQUIRED_TESTS, INCUS_BACKEND=${{ matrix.backend }} env LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./main.sh ${{ matrix.suite }} client: name: Client strategy: fail-fast: false matrix: go: - oldstable - stable os: - ubuntu-latest - macos-latest - windows-latest runs-on: ${{ matrix.os }} steps: - name: Checkout code uses: actions/checkout@v6 - name: Install Go uses: actions/setup-go@v6 with: go-version: ${{ matrix.go }} - name: Create build directory run: | mkdir bin - name: Build static incus (x86_64) env: CGO_ENABLED: 0 GOARCH: amd64 run: | go build -o bin/incus.x86_64 ./cmd/incus - name: Build static incus (aarch64) env: CGO_ENABLED: 0 GOARCH: arm64 run: | go build -o bin/incus.aarch64 ./cmd/incus - name: Build static incus-agent (x86_64) env: CGO_ENABLED: 0 GOARCH: amd64 run: | go build -o bin/incus-agent.x86_64 ./cmd/incus-agent - name: Build static incus-agent (aarch64) env: CGO_ENABLED: 0 GOARCH: arm64 run: | go build -o bin/incus-agent.aarch64 ./cmd/incus-agent - name: Build static incus-migrate if: runner.os == 'Linux' env: CGO_ENABLED: 0 run: | GOARCH=amd64 go build -o bin/incus-migrate.x86_64 ./cmd/incus-migrate GOARCH=arm64 go build -o bin/incus-migrate.aarch64 ./cmd/incus-migrate - name: Build static lxd-to-incus if: runner.os == 'Linux' env: CGO_ENABLED: 0 run: | GOARCH=amd64 go build -o bin/lxd-to-incus.x86_64 ./cmd/lxd-to-incus GOARCH=arm64 go build -o bin/lxd-to-incus.aarch64 ./cmd/lxd-to-incus - name: Unit tests (client) env: CGO_ENABLED: 0 run: go test -v ./client/... - name: Unit tests (incus) env: CGO_ENABLED: 0 run: go test -v ./cmd/incus/... - name: Unit tests (shared) env: CGO_ENABLED: 0 run: go test -v ./shared/... - name: Upload incus client artifacts if: matrix.go == 'stable' uses: actions/upload-artifact@v7 continue-on-error: true with: name: ${{ runner.os }} path: bin/ documentation: name: Documentation runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v6 - name: Install Go uses: actions/setup-go@v6 with: go-version: stable - name: Install dependencies run: | sudo apt-get install -y aspell aspell-en ruby sudo gem install --no-document mdl - name: Run markdown linter run: | make doc-lint - name: Run spell checker run: | make doc-spellcheck - name: Run inclusive naming checker uses: get-woke/woke-action@v0 with: fail-on-error: true woke-args: "*.md **/*.md -c https://github.com/canonical/Inclusive-naming/raw/main/config.yml" - name: Run link checker run: | make doc-linkcheck - name: Build docs (Sphinx) run: make doc - name: Print warnings (Sphinx) run: if [ -s doc/.sphinx/warnings.txt ]; then cat doc/.sphinx/warnings.txt; exit 1; fi - name: Upload documentation artifacts if: always() uses: actions/upload-artifact@v7 with: name: documentation path: doc/html incus-7.0.0/.github/workflows/triage.yml000066400000000000000000000006501517523235500202430ustar00rootroot00000000000000name: Triaging on: - pull_request_target permissions: contents: read jobs: label: permissions: contents: read # for actions/labeler to determine modified files pull-requests: write # for actions/labeler to add labels to PRs name: PR labels runs-on: ubuntu-24.04 steps: - uses: actions/labeler@v6 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" sync-labels: true incus-7.0.0/.gitignore000066400000000000000000000013621517523235500146410ustar00rootroot00000000000000*.swp po/*.mo po/*.po~ incus-*.tar.xz .vagrant *~ tags # Potential binaries cmd/fuidshift/fuidshift cmd/incus/incus cmd/lxc-to-incus/lxc-to-incus cmd/lxd-to-incus/lxd-to-incus cmd/incus-agent/incus-agent cmd/incus-benchmark/incus-benchmark cmd/incus-migrate/incus-migrate cmd/incus-user/incus-user test/dev_incus-client/dev_incus-client test/syscall/sysinfo/sysinfo test/mini-oidc/mini-oidc test/mini-oidc/user.data test/tls2jwt/tls2jwt # Sphinx doc/html/ doc/reference/manpages/**/*.md doc/.sphinx/deps/ doc/.sphinx/.doctrees/ doc/.sphinx/themes/ doc/.sphinx/venv/ doc/.sphinx/warnings.txt doc/.sphinx/.wordlist.dic doc/.sphinx/_static/swagger-ui doc/.sphinx/_static/download doc/__pycache__ # For Atom ctags .tags .tags1 # For JetBrains IDEs .idea incus-7.0.0/.golangci.yml000066400000000000000000000052311517523235500152340ustar00rootroot00000000000000version: "2" linters: enable: - godot - misspell - revive - whitespace settings: errcheck: exclude-functions: - (io.ReadCloser).Close - (io.WriteCloser).Close - (io.ReadWriteCloser).Close - (*os.File).Close - (*github.com/gorilla/websocket.Conn).Close - (*github.com/mdlayher/vsock.Listener).Close - os.Remove - (*compress/gzip.Writer).Close - (*github.com/fatih/color.Color).Printf - (*github.com/fatih/color.Color).Println revive: rules: - name: exported arguments: - checkPrivateReceivers - disableStutteringCheck - name: import-shadowing - name: unchecked-type-assertion - name: var-naming arguments: - [] - [] - - upperCaseConst: true - name: early-return - name: redundant-import-alias - name: redefines-builtin-id - name: struct-tag - name: receiver-naming - name: deep-exit - name: defer - name: bool-literal-in-expr - name: comment-spacings - name: use-any - name: bare-return - name: empty-block - name: range-val-address - name: range-val-in-closure - name: var-declaration - name: useless-break - name: error-naming - name: indent-error-flow - name: datarace - name: modifies-value-receiver - name: empty-lines - name: duplicated-imports - name: error-return exclusions: generated: lax rules: - linters: - revive source: '^//generate-database:mapper ' - linters: - revive text: "avoid package names that conflict with Go standard library package names" path: "^internal/io/" - linters: - revive - godot path: "^test/mini-oidc/storage/" - linters: - staticcheck text: "ST1005:" - linters: - godot text: "Comment should end in a period" source: '^// Example:' - path: internal/util/ text: "avoid meaningless package names" - path: internal/server/util/ text: "avoid meaningless package names" paths: - third_party$ - builtin$ - examples$ formatters: enable: - gci - gofumpt - goimports settings: gci: sections: - standard - default - prefix(github.com/lxc/incus) goimports: local-prefixes: - github.com/lxc/incus exclusions: generated: lax paths: - third_party$ - builtin$ - examples$ incus-7.0.0/.goreleaser.yaml000066400000000000000000000100431517523235500157370ustar00rootroot00000000000000version: 2 archives: - id: incus ids: - incus formats: - binary name_template: >- bin. {{- if eq .Os "darwin" }}macos {{- else }}{{ .Os }}{{ end }}.incus. {{- if eq .Arch "amd64" }}x86_64 {{- else if eq .Arch "arm64" }}aarch64 {{- else }}{{ .Arch }}{{ end }} - id: incus-agent ids: - incus-agent formats: - binary name_template: >- bin. {{- if eq .Os "darwin" }}macos {{- else }}{{ .Os }}{{ end }}.incus-agent. {{- if eq .Arch "amd64" }}x86_64 {{- else if eq .Arch "arm64" }}aarch64 {{- else }}{{ .Arch }}{{ end }} - id: incus-benchmark ids: - incus-benchmark formats: - binary name_template: >- bin. {{- if eq .Os "darwin" }}macos {{- else }}{{ .Os }}{{ end }}.incus-benchmark. {{- if eq .Arch "amd64" }}x86_64 {{- else if eq .Arch "arm64" }}aarch64 {{- else }}{{ .Arch }}{{ end }} - id: incus-migrate ids: - incus-migrate formats: - binary name_template: >- bin. {{- if eq .Os "darwin" }}macos {{- else }}{{ .Os }}{{ end }}.incus-migrate. {{- if eq .Arch "amd64" }}x86_64 {{- else if eq .Arch "arm64" }}aarch64 {{- else }}{{ .Arch }}{{ end }} - id: incus-simplestreams ids: - incus-simplestreams formats: - binary name_template: >- bin. {{- if eq .Os "darwin" }}macos {{- else }}{{ .Os }}{{ end }}.incus-simplestreams. {{- if eq .Arch "amd64" }}x86_64 {{- else if eq .Arch "arm64" }}aarch64 {{- else }}{{ .Arch }}{{ end }} - id: lxd-to-incus ids: - lxd-to-incus formats: - binary name_template: >- bin. {{- if eq .Os "darwin" }}macos {{- else }}{{ .Os }}{{ end }}.lxd-to-incus. {{- if eq .Arch "amd64" }}x86_64 {{- else if eq .Arch "arm64" }}aarch64 {{- else }}{{ .Arch }}{{ end }} before: hooks: - go mod download - go mod vendor - sh -c "git show-ref HEAD | cut -d' ' -f1 > .gitref" - git clone --depth=1 https://github.com/cowsql/cowsql vendor/cowsql - sh -c "cd vendor/cowsql && git show-ref HEAD | cut -d' ' -f1 > .gitref" - rm -Rf vendor/cowsql/.git - git clone --depth=1 https://github.com/cowsql/raft vendor/raft - sh -c "cd vendor/raft && git show-ref HEAD | cut -d' ' -f1 > .gitref" - rm -Rf vendor/raft/.git - make doc builds: - id: incus main: ./cmd/incus env: - CGO_ENABLED=0 goos: - darwin - freebsd - linux - windows goarch: - amd64 - arm64 - id: incus-agent main: ./cmd/incus-agent env: - CGO_ENABLED=0 goos: - darwin - freebsd - linux - windows goarch: - amd64 - arm64 - id: incus-benchmark main: ./cmd/incus-benchmark env: - CGO_ENABLED=0 goos: - darwin - freebsd - linux - windows goarch: - amd64 - arm64 - id: incus-migrate main: ./cmd/incus-migrate env: - CGO_ENABLED=0 goos: - linux goarch: - amd64 - arm64 - id: incus-simplestreams main: ./cmd/incus-simplestreams env: - CGO_ENABLED=0 goos: - linux goarch: - amd64 - arm64 - id: lxd-to-incus main: ./cmd/lxd-to-incus env: - CGO_ENABLED=0 goos: - linux goarch: - amd64 - arm64 changelog: use: "github-native" checksum: name_template: "checksums.txt" gomod: proxy: true mod: mod milestones: - repo: owner: lxc name: incus close: true fail_on_error: false name_template: "incus-{{ .Env.INCUS_VERSION }}" release: github: owner: lxc name: incus name_template: "Incus {{ .Env.INCUS_VERSION }}" sboms: - id: archive artifacts: archive ids: - incus - incus-agent - incus-benchmark - incus-migrate - incus-simplestreams - lxd-to-incus - id: source artifacts: source source: enabled: true format: "tar.gz" prefix_template: "{{ .ProjectName }}-{{ .Env.INCUS_VERSION }}/" files: - ".gitref" - "doc/html" - "vendor/*" incus-7.0.0/.vscode/000077500000000000000000000000001517523235500142105ustar00rootroot00000000000000incus-7.0.0/.vscode/launch.json000066400000000000000000000037161517523235500163640ustar00rootroot00000000000000{ // Use IntelliSense to learn about possible attributes. // Hover to view descriptions of existing attributes. // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ { // if the incusd is running, this will attach to it. "name": "Attach to Incusd", "type": "go", "request": "attach", "mode": "local", "processId": "incusd", "asRoot":true, "console": "integratedTerminal" }, { // after running `make` to install incusd, assuming that your go/bin is in your home directory, this should launch incusd if its not a service. // if it is an active service, you actually need to restart the service, and then attach to it. "name": "Launch Incusd", "type":"go", "request": "launch", "mode": "exec", "asRoot": true, "program": "${userHome}/go/bin/incusd", "env": { "PATH": "${env:PATH}:${userHome}/go/bin/", "LD_LIBRARY_PATH": "${userHome}/go/deps/raft/.libs/:${userHome}/go/deps/cowsql/.libs/" }, "args": [ "--group", "sudo" ], "console": "integratedTerminal", }, { "name": "Launch Incusd --debug", "type":"go", "request": "launch", "mode": "exec", "asRoot": true, "program": "${userHome}/go/bin/incusd", "env": { "PATH": "${env:PATH}:${userHome}/go/bin/", "LD_LIBRARY_PATH": "${userHome}/go/deps/raft/.libs/:${userHome}/go/deps/cowsql/.libs/" }, "args": [ "--group", "sudo", "--debug" ], "console": "integratedTerminal", } ] }incus-7.0.0/AUTHORS000066400000000000000000000003631517523235500137210ustar00rootroot00000000000000Unless mentioned otherwise in a specific file's header, all code in this project is released under the Apache 2.0 license. The list of authors and contributors can be retrieved from the git commit history and in some cases, the file headers. incus-7.0.0/CODE_OF_CONDUCT.md000066400000000000000000000064361517523235500154570ustar00rootroot00000000000000# Contributor Covenant Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at coc@linuxcontainers.org. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see incus-7.0.0/CONTRIBUTING.md000066400000000000000000000123111517523235500150760ustar00rootroot00000000000000# Contributing The Incus team appreciates contributions to the project, through pull requests, issues on the [GitHub repository](https://github.com/lxc/incus/issues), or discussions or questions on the [forum](https://discuss.linuxcontainers.org). Check the following guidelines before contributing to the project. ## Code of Conduct When contributing, you must adhere to the Code of Conduct, which is available at: [`https://github.com/lxc/incus/blob/main/CODE_OF_CONDUCT.md`](https://github.com/lxc/incus/blob/main/CODE_OF_CONDUCT.md) ## License and copyright By default, any contribution to this project is made under the Apache 2.0 license. The author of a change remains the copyright holder of their code (no copyright assignment). ## No Large Language Models (LLMs) or similar AI tools All contributions to this project are expected to be done by human beings or through standard predictable tooling (e.g. scripts, formatters, ...). We expect all contributors to be able to reason about the code that they contribute and explain why they're taking a particular approach. LLMs and similar predictive tools have the annoying tendency of producing large amount of low quality code with subtle issues which end up taking the maintainers more time to debug than it would have taken to write the code by hand in the first place. Any attempt at hiding the use of LLMs or similar tools in Incus contributions will result in a revert of the affected changes and a ban from the project. ## Pull requests Changes to this project should be proposed as pull requests on GitHub at: [`https://github.com/lxc/incus`](https://github.com/lxc/incus) Proposed changes will then go through review there and once approved, be merged in the main branch. ### Commit structure Separate commits should be used for: - API extension (`api: Add XYZ extension`, contains `doc/api-extensions.md` and `internal/version/api.go`) - Documentation (`doc: Update XYZ` for files in `doc/`) - API structure (`shared/api: Add XYZ` for changes to `shared/api/`) - Go client package (`client: Add XYZ` for changes to `client/`) - CLI (`cmd/: Change XYZ` for changes to `cmd/`) - Incus daemon (`incus/: Add support for XYZ` for changes to `incus/`) - Tests (`tests: Add test for XYZ` for changes to `tests/`) The same kind of pattern extends to the other tools in the Incus code tree and depending on complexity, things may be split into even smaller chunks. When updating strings in the CLI tool (`cmd/`), you may need a commit to update the templates: make i18n git commit -a -s -m "i18n: Update translation templates" po/ When updating API (`shared/api`), you may need a commit to update the swagger YAML: make update-api git commit -s -m "doc/rest-api: Refresh swagger YAML" doc/rest-api.yaml This structure makes it easier for contributions to be reviewed and also greatly simplifies the process of back-porting fixes to stable branches. ### Developer Certificate of Origin To improve tracking of contributions to this project we use the DCO 1.1 and use a "sign-off" procedure for all changes going into the branch. The sign-off is a simple line at the end of the explanation for the commit which certifies that you wrote it or otherwise have the right to pass it on as an open-source contribution. ``` Developer Certificate of Origin Version 1.1 Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 660 York Street, Suite 102, San Francisco, CA 94110 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. ``` An example of a valid sign-off line is: ``` Signed-off-by: Random J Developer ``` Use a known identity and a valid e-mail address. Sorry, no anonymous contributions are allowed. We also require each commit be individually signed-off by their author, even when part of a larger set. You may find `git commit -s` useful. ## More information For more information, see [Contributing](https://linuxcontainers.org/incus/docs/main/contributing/) in the documentation. incus-7.0.0/COPYING000066400000000000000000000261361517523235500137120ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. incus-7.0.0/Makefile000066400000000000000000000337651517523235500143250ustar00rootroot00000000000000GO ?= go DOMAIN=incus POFILES=$(wildcard po/*.po) MOFILES=$(patsubst %.po,%.mo,$(POFILES)) LINGUAS=$(basename $(POFILES)) POTFILE=po/$(DOMAIN).pot VERSION=$(or ${CUSTOM_VERSION},$(shell grep "var Version" internal/version/flex.go | cut -d'"' -f2)) ARCHIVE=incus-$(VERSION).tar HASH := \# TAG_SQLITE3=$(shell printf "$(HASH)include \nvoid main(){cowsql_node_id n = 1;}" | $(CC) ${CGO_CFLAGS} -o /dev/null -xc - >/dev/null 2>&1 && echo "libsqlite3") GOPATH ?= $(shell $(GO) env GOPATH) CGO_LDFLAGS_ALLOW ?= (-Wl,-wrap,pthread_create)|(-Wl,-z,now) SPHINXENV=doc/.sphinx/venv/bin/activate SPHINXPIPPATH=doc/.sphinx/venv/bin/pip OVN_MINVER=23.03.0 OVS_MINVER=2.15.0 ifneq "$(wildcard vendor)" "" RAFT_PATH=$(CURDIR)/vendor/raft COWSQL_PATH=$(CURDIR)/vendor/cowsql else RAFT_PATH=$(GOPATH)/deps/raft COWSQL_PATH=$(GOPATH)/deps/cowsql endif # section(Build): Build Incus .PHONY: default default: build .PHONY: build # doc: Build all Incus binaries (same as make and make default) build: ifeq "$(TAG_SQLITE3)" "" @echo "Missing cowsql, run \"make deps\" to setup." exit 1 endif CC="$(CC)" CGO_LDFLAGS_ALLOW="$(CGO_LDFLAGS_ALLOW)" $(GO) install -v -tags "$(TAG_SQLITE3)" $(DEBUG) ./... CGO_ENABLED=0 $(GO) install -v -tags netgo ./cmd/incus-migrate CGO_ENABLED=0 $(GO) install -v -tags agent,netgo ./cmd/incus-agent @echo "Incus built successfully" .PHONY: client # doc: Build the Incus client client: $(GO) install -v -tags "$(TAG_SQLITE3)" $(DEBUG) ./cmd/incus @echo "Incus client built successfully" .PHONY: incus-agent # doc: Build the Incus agent incus-agent: CGO_ENABLED=0 $(GO) install -v -tags agent,netgo ./cmd/incus-agent @echo "Incus agent built successfully" .PHONY: incus-migrate # doc: Build the Incus migration tool incus-migrate: CGO_ENABLED=0 $(GO) install -v -tags netgo ./cmd/incus-migrate @echo "Incus migration tool built successfully" .PHONY: debug # doc: Build Incus in debug mode debug: ifeq "$(TAG_SQLITE3)" "" @echo "Missing custom libsqlite3, run \"make deps\" to setup." exit 1 endif CC="$(CC)" CGO_LDFLAGS_ALLOW="$(CGO_LDFLAGS_ALLOW)" $(GO) install -v -tags "$(TAG_SQLITE3) logdebug" $(DEBUG) ./... CGO_ENABLED=0 $(GO) install -v -tags "netgo,logdebug" ./cmd/incus-migrate CGO_ENABLED=0 $(GO) install -v -tags "agent,netgo,logdebug" ./cmd/incus-agent @echo "Incus built successfully" .PHONY: nocache # doc: Build Incus ignoring the local Go cache nocache: ifeq "$(TAG_SQLITE3)" "" @echo "Missing custom libsqlite3, run \"make deps\" to setup." exit 1 endif CC="$(CC)" CGO_LDFLAGS_ALLOW="$(CGO_LDFLAGS_ALLOW)" $(GO) install -a -v -tags "$(TAG_SQLITE3)" $(DEBUG) ./... CGO_ENABLED=0 $(GO) install -a -v -tags netgo ./cmd/incus-migrate CGO_ENABLED=0 $(GO) install -a -v -tags agent,netgo ./cmd/incus-agent @echo "Incus built successfully" .PHONY: race # doc: Build Incus in race condition detection mode race: ifeq "$(TAG_SQLITE3)" "" @echo "Missing custom libsqlite3, run \"make deps\" to setup." exit 1 endif CC="$(CC)" CGO_LDFLAGS_ALLOW="$(CGO_LDFLAGS_ALLOW)" $(GO) install -race -v -tags "$(TAG_SQLITE3)" $(DEBUG) ./... CGO_ENABLED=0 $(GO) install -v -tags netgo ./cmd/incus-migrate CGO_ENABLED=0 $(GO) install -v -tags agent,netgo ./cmd/incus-agent @echo "Incus built successfully" # section(Dependencies): Manage Incus dependencies .PHONY: deps # doc: Build Incus dependencies deps: @if [ ! -e "$(RAFT_PATH)" ]; then \ git clone --depth=1 "https://github.com/cowsql/raft" "$(RAFT_PATH)"; \ elif [ -e "$(RAFT_PATH)/.git" ]; then \ cd "$(RAFT_PATH)"; git pull; \ fi cd "$(RAFT_PATH)" && \ autoreconf -i && \ ./configure && \ make # cowsql @if [ ! -e "$(COWSQL_PATH)" ]; then \ git clone --depth=1 "https://github.com/cowsql/cowsql" "$(COWSQL_PATH)"; \ elif [ -e "$(COWSQL_PATH)/.git" ]; then \ cd "$(COWSQL_PATH)"; git pull; \ fi cd "$(COWSQL_PATH)" && \ autoreconf -i && \ PKG_CONFIG_PATH="$(RAFT_PATH)" ./configure && \ make CFLAGS="-I$(RAFT_PATH)/include/" LDFLAGS="-L$(RAFT_PATH)/.libs/" # environment @echo "" @echo "Please set the following in your environment (possibly ~/.bashrc)" @echo "export CGO_CFLAGS=\"-I$(RAFT_PATH)/include/ -I$(COWSQL_PATH)/include/\"" @echo "export CGO_LDFLAGS=\"-L$(RAFT_PATH)/.libs -L$(COWSQL_PATH)/.libs/\"" @echo "export LD_LIBRARY_PATH=\"$(RAFT_PATH)/.libs/:$(COWSQL_PATH)/.libs/\"" @echo "export CGO_LDFLAGS_ALLOW=\"(-Wl,-wrap,pthread_create)|(-Wl,-z,now)\"" .PHONY: update-gomod # doc: Update Go dependencies update-gomod: ifneq "$(INCUS_OFFLINE)" "" @echo "The update-gomod target cannot be run in offline mode." exit 1 endif $(GO) get -t -v -u ./... $(GO) mod tidy --go=1.25.6 $(GO) get toolchain@none @echo "Dependencies updated" # section(Schemas): Update Incus data schemas .PHONY: update-ovsdb # doc: Update OVSDB schema update-ovsdb: go install github.com/ovn-kubernetes/libovsdb/cmd/modelgen@main rm -Rf internal/server/network/ovs/schema mkdir internal/server/network/ovs/schema curl -s https://raw.githubusercontent.com/openvswitch/ovs/v$(OVS_MINVER)/vswitchd/vswitch.ovsschema -o internal/server/network/ovs/schema/ovs.json modelgen -o internal/server/network/ovs/schema/ovs internal/server/network/ovs/schema/ovs.json rm internal/server/network/ovs/schema/*.json rm -Rf internal/server/network/ovn/schema mkdir internal/server/network/ovn/schema curl -s https://raw.githubusercontent.com/ovn-org/ovn/v$(OVN_MINVER)/ovn-nb.ovsschema -o internal/server/network/ovn/schema/ovn-nb.json curl -s https://raw.githubusercontent.com/ovn-org/ovn/v$(OVN_MINVER)/ovn-sb.ovsschema -o internal/server/network/ovn/schema/ovn-sb.json curl -s https://raw.githubusercontent.com/ovn-org/ovn/v$(OVN_MINVER)/ovn-ic-nb.ovsschema -o internal/server/network/ovn/schema/ovn-ic-nb.json curl -s https://raw.githubusercontent.com/ovn-org/ovn/v$(OVN_MINVER)/ovn-ic-sb.ovsschema -o internal/server/network/ovn/schema/ovn-ic-sb.json modelgen -o internal/server/network/ovn/schema/ovn-nb internal/server/network/ovn/schema/ovn-nb.json modelgen -o internal/server/network/ovn/schema/ovn-sb internal/server/network/ovn/schema/ovn-sb.json modelgen -o internal/server/network/ovn/schema/ovn-ic-nb internal/server/network/ovn/schema/ovn-ic-nb.json modelgen -o internal/server/network/ovn/schema/ovn-ic-sb internal/server/network/ovn/schema/ovn-ic-sb.json rm internal/server/network/ovn/schema/*.json .PHONY: update-protobuf # doc: Update Protobuf schema update-protobuf: protoc --go_out=. ./internal/migration/migrate.proto .PHONY: update-schema # doc: Update database schema update-schema: cd cmd/generate-database && $(GO) build -o $(GOPATH)/bin/generate-database -tags "$(TAG_SQLITE3)" $(DEBUG) && cd - $(GO) generate ./... gofumpt -w ./internal/server/db/ goimports -w ./internal/server/db/ @echo "Code generation completed" .PHONY: update-api # doc: Update API schema update-api: ifeq "$(INCUS_OFFLINE)" "" (cd / ; $(GO) install -v -x github.com/go-swagger/go-swagger/cmd/swagger@master) endif swagger generate spec -o doc/rest-api.yaml -w ./cmd/incusd -m .PHONY: update-metadata # doc: Update configuration metadata update-metadata: build @echo "Generating golang documentation metadata" cd cmd/generate-config && CGO_ENABLED=0 $(GO) build -o $(GOPATH)/bin/generate-config $(GOPATH)/bin/generate-config . --json ./internal/server/metadata/configuration.json --txt ./doc/config_options.txt # OpenFGA Syntax Transformer: https://github.com/openfga/syntax-transformer .PHONY: update-openfga # doc: Update OpenFGA schema update-openfga: ifeq ($(shell command -v fga),) (cd / ; $(GO) install -v -x github.com/openfga/cli/cmd/fga@latest) endif @printf 'package auth\n\n// Code generated by Makefile; DO NOT EDIT.\n\nvar authModel = `%s`\n' '$(shell fga model transform --file=./internal/server/auth/driver_openfga_model.openfga | jq -c)' > ./internal/server/auth/driver_openfga_model.go # section(Documentation): Build Incus documentation .PHONY: doc # doc: Setup the build environment and build the documentation doc: doc-setup doc-incremental .PHONY: doc-setup # doc: Setup a documentation build environment doc-setup: client @echo "Setting up documentation build environment" python3 -m venv doc/.sphinx/venv . $(SPHINXENV) ; pip install --require-virtualenv --upgrade -r doc/.sphinx/requirements.txt --log doc/.sphinx/venv/pip_install.log @test ! -f doc/.sphinx/venv/pip_list.txt || \ mv doc/.sphinx/venv/pip_list.txt doc/.sphinx/venv/pip_list.txt.bak $(SPHINXPIPPATH) list --local --format=freeze > doc/.sphinx/venv/pip_list.txt find doc/reference/manpages/ -name "*.md" -type f -delete rm -Rf doc/html rm -Rf doc/.sphinx/.doctrees .PHONY: doc-incremental # doc: Build the documentation doc-incremental: @echo "Build the documentation" . $(SPHINXENV) ; sphinx-build -c doc/ -b dirhtml doc/ doc/html/ -d doc/.sphinx/.doctrees -w doc/.sphinx/warnings.txt .PHONY: doc-serve # doc: Serve the documentation on localhost:8001 doc-serve: cd doc/html; python3 -m http.server 8001 .PHONY: doc-spellcheck # doc: Check spelling errors on the documentation doc-spellcheck: doc . $(SPHINXENV) ; python3 -m pyspelling -c doc/.sphinx/spellingcheck.yaml .PHONY: doc-linkcheck # doc: Check broken links on the documentation doc-linkcheck: doc-setup . $(SPHINXENV) ; LOCAL_SPHINX_BUILD=True sphinx-build -c doc/ -b linkcheck doc/ doc/html/ -d doc/.sphinx/.doctrees .PHONY: doc-lint # doc: Lint the documentation doc-lint: doc/.sphinx/.markdownlint/doc-lint.sh .PHONY: woke-install # doc: Install the inclusive checker woke-install: @type woke >/dev/null 2>&1 || \ { echo "Installing \"woke\" snap... \n"; sudo snap install woke; } .PHONY: doc-woke # doc: Check for non-inclusive phrasing doc-woke: woke-install woke *.md **/*.md -c https://github.com/canonical/Inclusive-naming/raw/main/config.yml # section(Tests): Run the tests .PHONY: check # doc: Run the test suite check: default ifeq "$(INCUS_OFFLINE)" "" (cd / ; $(GO) install -v -x github.com/rogpeppe/godeps@latest) (cd / ; $(GO) install -v -x github.com/tsenart/deadcode@latest) (cd / ; $(GO) install -v -x golang.org/x/lint/golint@latest) endif CGO_LDFLAGS_ALLOW="$(CGO_LDFLAGS_ALLOW)" $(GO) test -v -tags "$(TAG_SQLITE3)" $(DEBUG) ./... cd test && ./main.sh .PHONY: static-analysis # doc: Run static analysis static-analysis: ifeq ($(shell command -v go-licenses),) (cd / ; $(GO) install -v -x github.com/google/go-licenses@latest) endif ifeq ($(shell command -v govulncheck),) go install golang.org/x/vuln/cmd/govulncheck@latest endif ifeq ($(shell command -v golangci-lint),) curl -sSfL https://golangci-lint.run/install.sh | sh -s -- -b $$($(GO) env GOPATH)/bin endif ifeq ($(shell command -v shellcheck),) echo "Please install shellcheck" exit 1 endif ifeq ($(shell command -v flake8),) echo "Please install flake8" exit 1 endif ifeq ($(shell command -v codespell),) echo "Please install codespell" exit 1 endif ifeq ($(shell command -v run-parts),) echo "Please install run-parts" exit 1 endif flake8 test/deps/import-busybox shellcheck --shell sh test/*.sh test/includes/*.sh test/suites/*.sh test/backends/*.sh test/lint/*.sh shellcheck test/extras/*.sh run-parts $(shell run-parts -V >/dev/null 2>&1 && echo -n "--verbose --exit-on-error --regex '.sh'") test/lint .PHONY: staticcheck # doc: Run static checks staticcheck: ifeq ($(shell command -v staticcheck),) (cd / ; $(GO) install -v -x honnef.co/go/tools/cmd/staticcheck@latest) endif # To get advance notice of deprecated function usage, consider running: # sed -i 's/^go 1\.[0-9]\+$/go 1.18/' go.mod # before 'make staticcheck'. # Run staticcheck against all the dirs containing Go files. staticcheck $$(git ls-files *.go | sed 's|^|./|; s|/[^/]\+\.go$$||' | sort -u) .PHONY: unit-test # doc: Run unit tests unit-test: sudo --preserve-env=CGO_CFLAGS,CGO_LDFLAGS,CGO_LDFLAGS_ALLOW,LD_LIBRARY_PATH LD_LIBRARY_PATH=${LD_LIBRARY_PATH} env "PATH=${PATH}" $(GO) test ./... # section(Internationalization): Generate internationalization files .PHONY: i18n # doc: Generate internationalization files i18n: update-pot update-po po/%.mo: po/%.po msgfmt --statistics -o $@ $< po/%.po: po/$(DOMAIN).pot msgmerge -U po/$*.po po/$(DOMAIN).pot .PHONY: update-po # doc: Update PO files update-po: set -eu; \ for lang in $(LINGUAS); do\ msgmerge --backup=none -U $$lang.po po/$(DOMAIN).pot; \ done .PHONY: update-pot # doc: Update POT file update-pot: ifeq "$(INCUS_OFFLINE)" "" (cd / ; $(GO) install -v -x github.com/snapcore/snapd/i18n/xgettext-go@2.57.1) endif xgettext-go -o po/$(DOMAIN).pot --add-comments-tag=TRANSLATORS: --sort-output --package-name=$(DOMAIN) --msgid-bugs-address=lxc-devel@lists.linuxcontainers.org --keyword=i18n.G --keyword-plural=i18n.NG cmd/incus/*.go cmd/incus/color/*.go cmd/incus/usage/*.go shared/cliconfig/*.go sed -i s/CHARSET/UTF-8/ po/$(DOMAIN).pot .PHONY: build-mo # doc! Build MO files build-mo: $(MOFILES) # section(Miscellaneous): Targets that don’t fit in any category .PHONY: dist # doc: Prepare a release tarball dist: doc # Cleanup rm -Rf $(ARCHIVE).xz # Create build dir $(eval TMP := $(shell mktemp -d)) git archive --prefix=incus-$(VERSION)/ HEAD | tar -x -C $(TMP) git show-ref HEAD | cut -d' ' -f1 > $(TMP)/incus-$(VERSION)/.gitref # Download dependencies (cd $(TMP)/incus-$(VERSION) ; $(GO) mod vendor) # Download the cowsql libraries git clone --depth=1 https://github.com/cowsql/cowsql $(TMP)/incus-$(VERSION)/vendor/cowsql (cd $(TMP)/incus-$(VERSION)/vendor/cowsql ; git show-ref HEAD | cut -d' ' -f1 > .gitref) git clone --depth=1 https://github.com/cowsql/raft $(TMP)/incus-$(VERSION)/vendor/raft (cd $(TMP)/incus-$(VERSION)/vendor/raft ; git show-ref HEAD | cut -d' ' -f1 > .gitref) # Copy doc output cp -r doc/html $(TMP)/incus-$(VERSION)/doc/html/ # Assemble tarball tar --exclude-vcs -C $(TMP) -Jcf $(ARCHIVE).xz incus-$(VERSION)/ # Cleanup rm -Rf $(TMP) .PHONY: help # doc: Show this help help: @echo The following targets are supported: @sed -En 's/^#\s*section\(([^)]*)\):\s*(.*)$$/\n\x1b[1m\1:\x1b[0m \2/p;/^\.PHONY:/{N;N;s/^\.PHONY:\s*([^[:space:]]+)\n#\s*doc(:\s*(.*)\n\1:\s*$$|!\s*(.*)\n\1:[^\n]*)/ \1!\3\4/p;s/^\.PHONY:\s*([^[:space:]]+)\s*\n#\s*doc:\s*(.*)\n\1:\s*(.+)$$/ \1!\2 (runs \3)/p}' Makefile | awk -F! '{printf "%-20s%s\n", $$1, $$2}' incus-7.0.0/README.md000066400000000000000000000124771517523235500141410ustar00rootroot00000000000000# Incus Incus is a modern, secure and powerful system container and virtual machine manager. It provides a unified experience for running and managing full Linux systems inside containers or virtual machines. Incus supports images for a large number of Linux distributions (official Ubuntu images and images provided by the community) and is built around a very powerful, yet pretty simple, REST API. Incus scales from one instance on a single machine to a cluster in a full data center rack, making it suitable for running workloads both for development and in production. Incus allows you to easily set up a system that feels like a small private cloud. You can run any type of workload in an efficient way while keeping your resources optimized. You should consider using Incus if you want to containerize different environments or run virtual machines, or in general run and manage your infrastructure in a cost-effective way. You can try Incus online at: [`https://linuxcontainers.org/incus/try-it/`](https://linuxcontainers.org/incus/try-it/) ## Project history Incus, which is named after the [Cumulonimbus incus](https://en.wikipedia.org/wiki/Cumulonimbus_incus) or anvil cloud started as a community fork of Canonical's LXD following [Canonical's takeover](https://linuxcontainers.org/lxd/) of the LXD project from the Linux Containers community. The project was then adopted by the Linux Containers community, taking back the spot left empty by LXD's departure. Incus is a true open source community project, free of any [CLA](https://en.wikipedia.org/wiki/Contributor_License_Agreement) and remains released under the [Apache 2.0 license](https://www.apache.org/licenses/LICENSE-2.0). It's maintained by the same team of developers that first created LXD. LXD users wishing to migrate to Incus can easily do so through a migration tool called [`lxd-to-incus`](https://linuxcontainers.org/incus/docs/main/howto/server_migrate_lxd/). ## Get started See [Getting started](https://linuxcontainers.org/incus/docs/main/tutorial/first_steps/) in the Incus documentation for installation instructions and first steps. - Release announcements: [`https://discuss.linuxcontainers.org/c/news/`](https://discuss.linuxcontainers.org/c/news/) - Release tarballs: [`https://github.com/lxc/incus/releases/`](https://github.com/lxc/incus/releases/) - Documentation: [`https://linuxcontainers.org/incus/docs/main/`](https://linuxcontainers.org/incus/docs/main/) ## Status Type | Service | Status --- | --- | --- Tests | GitHub | [![Build Status](https://github.com/lxc/incus/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/lxc/incus/actions?query=event%3Apush+branch%3Amain) Go documentation | Godoc | [![GoDoc](https://godoc.org/github.com/lxc/incus/v7/client?status.svg)](https://godoc.org/github.com/lxc/incus/v7/client) Static analysis | GoReport | [![Go Report Card](https://goreportcard.com/badge/github.com/lxc/incus)](https://goreportcard.com/report/github.com/lxc/incus) Translations | Weblate | [![Translation status](https://hosted.weblate.org/widget/incus/svg-badge.svg)](https://hosted.weblate.org/projects/incus/) ## Security Consider the following aspects to ensure that your Incus installation is secure: - Keep your operating system up-to-date and install all available security patches. - Use only supported Incus versions. - Restrict access to the Incus daemon and the remote API. - Do not use privileged containers unless required. If you use privileged containers, put appropriate security measures in place. See the [LXC security page](https://linuxcontainers.org/lxc/security/) for more information. - Configure your network interfaces to be secure. See [Security](https://github.com/lxc/incus/blob/main/doc/explanation/security.md) for detailed information. **IMPORTANT:** Local access to Incus through the Unix socket always grants full access to Incus. This includes the ability to attach file system paths or devices to any instance as well as tweak the security features on any instance. Therefore, you should only give such access to users who you'd trust with root access to your system. ## Support and community The following channels are available for you to interact with the Incus community. ### Bug reports You can file bug reports and feature requests at: [`https://github.com/lxc/incus/issues/new`](https://github.com/lxc/incus/issues/new) ### Community support Community support is handled at: [`https://discuss.linuxcontainers.org`](https://discuss.linuxcontainers.org) ### Commercial support Commercial support is currently available from [Zabbly](https://zabbly.com) for users of their [Debian or Ubuntu packages](https://github.com/zabbly/incus). ## Documentation The official documentation is available at: [`https://github.com/lxc/incus/tree/main/doc`](https://github.com/lxc/incus/tree/main/doc) ## Contributing Fixes and new features are greatly appreciated. Make sure to read our [contributing guidelines](CONTRIBUTING.md) first! incus-7.0.0/SECURITY.md000066400000000000000000000022541517523235500144430ustar00rootroot00000000000000# Security policy ## Supported versions Incus has two types of releases: - Feature releases - LTS releases For feature releases, only the latest one is supported, and we usually don't do point releases. Instead, users are expected to wait until the next release. For LTS releases, we do periodic bugfix releases that include an accumulation of bugfixes from the feature releases. Such bugfix releases do not include new features. ## What qualifies as a security issue We don't consider privileged containers to be root safe, so any exploit allowing someone to escape them will not qualify as a security issue. This doesn't mean that we're not interested in preventing such escapes, but we simply do not consider such containers to be root safe. Unprivileged container escapes are certainly something we'd consider a security issue, especially if somehow facilitated by Incus. ## Reporting security issues Security issues can be reported by e-mail to security@linuxcontainers.org. Alternatively security issues can also be reported through Github at: https://github.com/lxc/incus/security/advisories/new incus-7.0.0/client/000077500000000000000000000000001517523235500141255ustar00rootroot00000000000000incus-7.0.0/client/connection.go000066400000000000000000000312311517523235500166130ustar00rootroot00000000000000package incus import ( "context" "crypto/sha256" "fmt" "net/http" "net/url" "os" "path/filepath" "slices" "strings" "time" "github.com/gorilla/websocket" "github.com/zitadel/oidc/v3/pkg/oidc" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/simplestreams" "github.com/lxc/incus/v7/shared/util" ) // ConnectionArgs represents a set of common connection properties. type ConnectionArgs struct { // TLS certificate of the remote server. If not specified, the system CA is used. TLSServerCert string // TLS certificate to use for client authentication. TLSClientCert string // TLS key to use for client authentication. TLSClientKey string // TLS CA to validate against when in PKI mode. TLSCA string // User agent string UserAgent string // Authentication type AuthType string // Custom proxy Proxy func(*http.Request) (*url.URL, error) // Custom HTTP Client (used as base for the connection) HTTPClient *http.Client // TransportWrapper wraps the *http.Transport set by Incus TransportWrapper func(*http.Transport) HTTPTransporter // Controls whether a client verifies the server's certificate chain and host name. InsecureSkipVerify bool // Controls whether to perform an exact certificate match (will ignore expiry). IdenticalCertificate bool // Cookie jar CookieJar http.CookieJar // OpenID Connect tokens OIDCTokens *oidc.Tokens[*oidc.IDTokenClaims] // Do not block for OIDC authentication OIDCNonInteractive bool // Skip the event listener endpoint SkipGetEvents bool // Skip automatic GetServer request upon connection SkipGetServer bool // Caching support for image servers CachePath string CacheExpiry time.Duration // Temp storage. TempPath string } // ConnectIncus lets you connect to a remote Incus daemon over HTTPs. // // A client certificate (TLSClientCert) and key (TLSClientKey) must be provided. // // If connecting to an Incus daemon running in PKI mode, the PKI CA (TLSCA) must also be provided. // // Unless the remote server is trusted by the system CA, the remote certificate must be provided (TLSServerCert). func ConnectIncus(uri string, args *ConnectionArgs) (InstanceServer, error) { return ConnectIncusWithContext(context.Background(), uri, args) } // ConnectIncusWithContext lets you connect to a remote Incus daemon over HTTPs with context.Context. // // A client certificate (TLSClientCert) and key (TLSClientKey) must be provided. // // If connecting to an Incus daemon running in PKI mode, the PKI CA (TLSCA) must also be provided. // // Unless the remote server is trusted by the system CA, the remote certificate must be provided (TLSServerCert). func ConnectIncusWithContext(ctx context.Context, uri string, args *ConnectionArgs) (InstanceServer, error) { // Cleanup URL uri = strings.TrimSuffix(uri, "/") logger.Debug("Connecting to a remote Incus over HTTPS", logger.Ctx{"url": uri}) return httpsIncus(ctx, uri, args) } // ConnectIncusHTTP lets you connect to a VM agent over a VM socket. func ConnectIncusHTTP(args *ConnectionArgs, client *http.Client) (InstanceServer, error) { return ConnectIncusHTTPWithContext(context.Background(), args, client) } // ConnectIncusHTTPWithContext lets you connect to a VM agent over a VM socket with context.Context. func ConnectIncusHTTPWithContext(ctx context.Context, args *ConnectionArgs, client *http.Client) (InstanceServer, error) { logger.Debug("Connecting to a VM agent over a VM socket") // Use empty args if not specified if args == nil { args = &ConnectionArgs{} } httpBaseURL, err := url.Parse("https://custom.socket") if err != nil { return nil, err } ctxConnected, ctxConnectedCancel := context.WithCancel(context.Background()) // Initialize the client struct server := ProtocolIncus{ ctx: ctx, httpBaseURL: *httpBaseURL, httpProtocol: "custom", httpUserAgent: args.UserAgent, ctxConnected: ctxConnected, ctxConnectedCancel: ctxConnectedCancel, eventConns: make(map[string]*websocket.Conn), eventListeners: make(map[string][]*EventListener), skipEvents: args.SkipGetEvents, tempPath: args.TempPath, } // Setup the HTTP client server.http = client // Test the connection and seed the server information if !args.SkipGetServer { serverStatus, _, err := server.GetServer() if err != nil { return nil, err } // Record the server certificate server.httpCertificate = serverStatus.Environment.Certificate } return &server, nil } // ConnectIncusUnix lets you connect to a remote Incus daemon over a local unix socket. // // If the path argument is empty, then $INCUS_SOCKET will be used, if // unset $INCUS_DIR/unix.socket will be used and if that one isn't set // either, then the path will default to /var/lib/incus/unix.socket or /run/incus/unix.socket. func ConnectIncusUnix(path string, args *ConnectionArgs) (InstanceServer, error) { return ConnectIncusUnixWithContext(context.Background(), path, args) } // ConnectIncusUnixWithContext lets you connect to a remote Incus daemon over a local unix socket with context.Context. // // If the path argument is empty, then $INCUS_SOCKET will be used, if // unset $INCUS_DIR/unix.socket will be used and if that one isn't set // either, then the path will default to /var/lib/incus/unix.socket or /run/incus/unix.socket. func ConnectIncusUnixWithContext(ctx context.Context, path string, args *ConnectionArgs) (InstanceServer, error) { logger.Debug("Connecting to a local Incus over a Unix socket") // Use empty args if not specified if args == nil { args = &ConnectionArgs{} } httpBaseURL, err := url.Parse("http://unix.socket") if err != nil { return nil, err } ctxConnected, ctxConnectedCancel := context.WithCancel(context.Background()) // Determine the socket path var projectName string if path == "" { path = os.Getenv("INCUS_SOCKET") if path == "" { incusDir := os.Getenv("INCUS_DIR") if incusDir == "" { _, err := os.Lstat("/run/incus/unix.socket") if err == nil { incusDir = "/run/incus" } else { incusDir = "/var/lib/incus" } } path = filepath.Join(incusDir, "unix.socket") userPath := filepath.Join(incusDir, "unix.socket.user") if !util.PathIsWritable(path) && util.PathIsWritable(userPath) { // Handle the use of incus-user. path = userPath // When using incus-user, the project list is typically restricted. // So let's try to be smart about the project we're using. projectName = fmt.Sprintf("user-%d", os.Geteuid()) } } } // Initialize the client struct server := ProtocolIncus{ ctx: ctx, httpBaseURL: *httpBaseURL, httpUnixPath: path, httpProtocol: "unix", httpUserAgent: args.UserAgent, ctxConnected: ctxConnected, ctxConnectedCancel: ctxConnectedCancel, eventConns: make(map[string]*websocket.Conn), eventListeners: make(map[string][]*EventListener), skipEvents: args.SkipGetEvents, project: projectName, tempPath: args.TempPath, } // Setup the HTTP client httpClient, err := unixHTTPClient(args, path) if err != nil { return nil, err } server.http = httpClient // Test the connection and seed the server information if !args.SkipGetServer { serverStatus, _, err := server.GetServer() if err != nil { return nil, err } // Record the server certificate server.httpCertificate = serverStatus.Environment.Certificate } return &server, nil } // ConnectPublicIncus lets you connect to a remote public Incus daemon over HTTPs. // // Unless the remote server is trusted by the system CA, the remote certificate must be provided (TLSServerCert). func ConnectPublicIncus(uri string, args *ConnectionArgs) (ImageServer, error) { return ConnectPublicIncusWithContext(context.Background(), uri, args) } // ConnectPublicIncusWithContext lets you connect to a remote public Incus daemon over HTTPs with context.Context. // // Unless the remote server is trusted by the system CA, the remote certificate must be provided (TLSServerCert). func ConnectPublicIncusWithContext(ctx context.Context, uri string, args *ConnectionArgs) (ImageServer, error) { logger.Debug("Connecting to a remote public Incus over HTTPS") // Cleanup URL uri = strings.TrimSuffix(uri, "/") return httpsIncus(ctx, uri, args) } // ConnectSimpleStreams lets you connect to a remote SimpleStreams image server over HTTPs. // // Unless the remote server is trusted by the system CA, the remote certificate must be provided (TLSServerCert). func ConnectSimpleStreams(uri string, args *ConnectionArgs) (ImageServer, error) { logger.Debug("Connecting to a remote simplestreams server", logger.Ctx{"URL": uri}) // Cleanup URL uri = strings.TrimSuffix(uri, "/") // Use empty args if not specified if args == nil { args = &ConnectionArgs{} } // Initialize the client struct server := ProtocolSimpleStreams{ httpHost: uri, httpUserAgent: args.UserAgent, httpCertificate: args.TLSServerCert, tempPath: args.TempPath, } // Setup the HTTP client httpClient, err := tlsHTTPClient(args.HTTPClient, args.TLSClientCert, args.TLSClientKey, args.TLSCA, args.TLSServerCert, args.InsecureSkipVerify, args.IdenticalCertificate, args.Proxy, args.TransportWrapper) if err != nil { return nil, err } server.http = httpClient // Get simplestreams client ssClient := simplestreams.NewClient(uri, *httpClient, args.UserAgent) server.ssClient = ssClient // Setup the cache if args.CachePath != "" { if !util.PathExists(args.CachePath) { return nil, fmt.Errorf("Cache directory %q doesn't exist", args.CachePath) } hashedURL := fmt.Sprintf("%x", sha256.Sum256([]byte(uri))) cachePath := filepath.Join(args.CachePath, hashedURL) cacheExpiry := args.CacheExpiry if cacheExpiry == 0 { cacheExpiry = time.Hour } if !util.PathExists(cachePath) { err := os.Mkdir(cachePath, 0o755) if err != nil { return nil, err } } ssClient.SetCache(cachePath, cacheExpiry) } return &server, nil } // ConnectOCI lets you connect to a remote OCI image registry over HTTPs. // // Unless the remote server is trusted by the system CA, the remote certificate must be provided (TLSServerCert). func ConnectOCI(uri string, args *ConnectionArgs) (ImageServer, error) { logger.Debug("Connecting to a remote OCI server", logger.Ctx{"URL": uri}) // Cleanup URL uri = strings.TrimSuffix(uri, "/") // Use empty args if not specified if args == nil { args = &ConnectionArgs{} } // Initialize the client struct server := ProtocolOCI{ httpHost: uri, httpUserAgent: args.UserAgent, httpCertificate: args.TLSServerCert, cache: map[string]ociInfo{}, errors: map[string]error{}, tempPath: args.TempPath, } // Setup the HTTP client httpClient, err := tlsHTTPClient(args.HTTPClient, args.TLSClientCert, args.TLSClientKey, args.TLSCA, args.TLSServerCert, args.InsecureSkipVerify, args.IdenticalCertificate, args.Proxy, args.TransportWrapper) if err != nil { return nil, err } server.http = httpClient return &server, nil } // Internal function called by ConnectIncus and ConnectPublicIncus. func httpsIncus(ctx context.Context, requestURL string, args *ConnectionArgs) (InstanceServer, error) { // Use empty args if not specified if args == nil { args = &ConnectionArgs{} } httpBaseURL, err := url.Parse(requestURL) if err != nil { return nil, err } ctxConnected, ctxConnectedCancel := context.WithCancel(context.Background()) // Initialize the client struct server := ProtocolIncus{ ctx: ctx, httpCertificate: args.TLSServerCert, httpBaseURL: *httpBaseURL, httpProtocol: "https", httpUserAgent: args.UserAgent, ctxConnected: ctxConnected, ctxConnectedCancel: ctxConnectedCancel, eventConns: make(map[string]*websocket.Conn), eventListeners: make(map[string][]*EventListener), skipEvents: args.SkipGetEvents, tempPath: args.TempPath, } if slices.Contains([]string{api.AuthenticationMethodOIDC}, args.AuthType) { server.RequireAuthenticated(true) } // Setup the HTTP client httpClient, err := tlsHTTPClient(args.HTTPClient, args.TLSClientCert, args.TLSClientKey, args.TLSCA, args.TLSServerCert, args.InsecureSkipVerify, args.IdenticalCertificate, args.Proxy, args.TransportWrapper) if err != nil { return nil, err } if args.CookieJar != nil { httpClient.Jar = args.CookieJar } server.http = httpClient if args.AuthType == api.AuthenticationMethodOIDC { server.setupOIDCClient(args.OIDCTokens, args.OIDCNonInteractive) } // Test the connection and seed the server information if !args.SkipGetServer { _, _, err := server.GetServer() if err != nil { return nil, err } } return &server, nil } incus-7.0.0/client/doc.go000066400000000000000000000067361517523235500152350ustar00rootroot00000000000000// Package incus implements a client for the Incus API // // # Overview // // This package lets you connect to Incus daemons or SimpleStream image // servers over a Unix socket or HTTPs. You can then interact with those // remote servers, creating instances, images, moving them around, ... // // The following examples make use of several imports: // // import ( // "github.com/lxc/incus/client" // "github.com/lxc/incus/shared/api" // "github.com/lxc/incus/shared/termios" // ) // // # Example - instance creation // // This creates a container on a local Incus daemon and then starts it. // // // Connect to Incus over the Unix socket // c, err := incus.ConnectIncusUnix("", nil) // if err != nil { // return err // } // // // Instance creation request // name := "my-container" // req := api.InstancesPost{ // Name: name, // Source: api.InstanceSource{ // Type: "image", // Alias: "my-image", # e.g. alpine/3.20 // Server: "https://images.linuxcontainers.org", // Protocol: "simplestreams", // }, // Type: "container" // } // // // Get Incus to create the instance (background operation) // op, err := c.CreateInstance(req) // if err != nil { // return err // } // // // Wait for the operation to complete // err = op.Wait() // if err != nil { // return err // } // // // Get Incus to start the instance (background operation) // reqState := api.InstanceStatePut{ // Action: "start", // Timeout: -1, // } // // op, err = c.UpdateInstanceState(name, reqState, "") // if err != nil { // return err // } // // // Wait for the operation to complete // err = op.Wait() // if err != nil { // return err // } // // # Example - command execution // // This executes an interactive bash terminal // // // Connect to Incus over the Unix socket // c, err := incus.ConnectIncusUnix("", nil) // if err != nil { // return err // } // // // Setup the exec request // req := api.InstanceExecPost{ // Command: []string{"bash"}, // WaitForWS: true, // Interactive: true, // Width: 80, // Height: 15, // } // // // Setup the exec arguments (fds) // args := incus.InstanceExecArgs{ // Stdin: os.Stdin, // Stdout: os.Stdout, // Stderr: os.Stderr, // } // // // Setup the terminal (set to raw mode) // if req.Interactive { // cfd := int(syscall.Stdin) // oldttystate, err := termios.MakeRaw(cfd) // if err != nil { // return err // } // // defer termios.Restore(cfd, oldttystate) // } // // // Get the current state // op, err := c.ExecInstance(name, req, &args) // if err != nil { // return err // } // // // Wait for it to complete // err = op.Wait() // if err != nil { // return err // } // // # Example - image copy // // This copies an image from a simplestreams server to a local Incus daemon // // // Connect to Incus over the Unix socket // c, err := incus.ConnectIncusUnix("", nil) // if err != nil { // return err // } // // // Connect to the remote SimpleStreams server // d, err = incus.ConnectSimpleStreams("https://images.linuxcontainers.org", nil) // if err != nil { // return err // } // // // Resolve the alias // alias, _, err := d.GetImageAlias("centos/7") // if err != nil { // return err // } // // // Get the image information // image, _, err := d.GetImage(alias.Target) // if err != nil { // return err // } // // // Ask Incus to copy the image from the remote server // op, err := d.CopyImage(*image, c, nil) // if err != nil { // return err // } // // // And wait for it to finish // err = op.Wait() // if err != nil { // return err // } package incus incus-7.0.0/client/events.go000066400000000000000000000054131517523235500157630ustar00rootroot00000000000000package incus import ( "context" "errors" "sync" "github.com/lxc/incus/v7/shared/api" ) // The EventListener struct is used to interact with an Incus event stream. type EventListener struct { r *ProtocolIncus ctx context.Context ctxCancel context.CancelFunc err error // projectName stores which project this event listener is associated with (empty for all projects). projectName string targets []*EventTarget targetsLock sync.Mutex } // The EventTarget struct is returned to the caller of AddHandler and used in RemoveHandler. type EventTarget struct { function func(api.Event) types []string } // AddHandler adds a function to be called whenever an event is received. func (e *EventListener) AddHandler(types []string, function func(api.Event)) (*EventTarget, error) { if function == nil { return nil, errors.New("A valid function must be provided") } // Handle locking e.targetsLock.Lock() defer e.targetsLock.Unlock() // Create a new target target := EventTarget{ function: function, types: types, } // And add it to the targets e.targets = append(e.targets, &target) return &target, nil } // RemoveHandler removes a function to be called whenever an event is received. func (e *EventListener) RemoveHandler(target *EventTarget) error { if target == nil { return errors.New("A valid event target must be provided") } // Handle locking e.targetsLock.Lock() defer e.targetsLock.Unlock() // Locate and remove the function from the list for i, entry := range e.targets { if entry == target { copy(e.targets[i:], e.targets[i+1:]) e.targets[len(e.targets)-1] = nil e.targets = e.targets[:len(e.targets)-1] return nil } } return errors.New("Couldn't find this function and event types combination") } // Disconnect must be used once done listening for events. func (e *EventListener) Disconnect() { // Handle locking e.r.eventListenersLock.Lock() defer e.r.eventListenersLock.Unlock() if e.ctx.Err() != nil { return } // Locate and remove it from the global list for i, listener := range e.r.eventListeners[e.projectName] { if listener == e { copy(e.r.eventListeners[e.projectName][i:], e.r.eventListeners[e.projectName][i+1:]) e.r.eventListeners[e.projectName][len(e.r.eventListeners[e.projectName])-1] = nil e.r.eventListeners[e.projectName] = e.r.eventListeners[e.projectName][:len(e.r.eventListeners[e.projectName])-1] break } } // Turn off the handler e.err = nil e.ctxCancel() } // Wait blocks until the server disconnects the connection or Disconnect() is called. func (e *EventListener) Wait() error { <-e.ctx.Done() return e.err } // IsActive returns true if this listener is still connected, false otherwise. func (e *EventListener) IsActive() bool { return e.ctx.Err() == nil } incus-7.0.0/client/incus.go000066400000000000000000000401471517523235500156030ustar00rootroot00000000000000package incus import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" neturl "net/url" "slices" "strings" "sync" "time" "github.com/gorilla/websocket" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/tcp" ) // ProtocolIncus represents an Incus API server. type ProtocolIncus struct { ctx context.Context server *api.Server ctxConnected context.Context ctxConnectedCancel context.CancelFunc // eventConns contains event listener connections associated to a project name (or empty for all projects). eventConns map[string]*websocket.Conn // eventConnsLock controls write access to the eventConns. eventConnsLock sync.Mutex // eventListeners is a slice of event listeners associated to a project name (or empty for all projects). eventListeners map[string][]*EventListener eventListenersLock sync.Mutex // skipEvents tracks whether we were configured not to connect to the events endpoint skipEvents bool http *http.Client httpCertificate string httpBaseURL neturl.URL httpUnixPath string httpProtocol string httpUserAgent string requireAuthenticated bool clusterTarget string project string oidcClient *oidcClient tempPath string } // Disconnect gets rid of any background goroutines. func (r *ProtocolIncus) Disconnect() { if r.ctxConnected.Err() != nil { r.ctxConnectedCancel() } } // GetConnectionInfo returns the basic connection information used to interact with the server. func (r *ProtocolIncus) GetConnectionInfo() (*ConnectionInfo, error) { info := ConnectionInfo{} info.Certificate = r.httpCertificate info.Protocol = "incus" info.URL = r.httpBaseURL.String() info.SocketPath = r.httpUnixPath info.Project = r.project if info.Project == "" { info.Project = api.ProjectDefaultName } info.Target = r.clusterTarget if info.Target == "" && r.server != nil { info.Target = r.server.Environment.ServerName } urls := []string{} if r.httpProtocol == "https" { urls = append(urls, r.httpBaseURL.String()) } if r.server != nil && len(r.server.Environment.Addresses) > 0 { for _, addr := range r.server.Environment.Addresses { if strings.HasPrefix(addr, ":") { continue } url := fmt.Sprintf("https://%s", addr) if !slices.Contains(urls, url) { urls = append(urls, url) } } } info.Addresses = urls return &info, nil } // isSameServer compares the calling ProtocolIncus object with the provided server object to check if they are the same server. // It verifies the equality based on their connection information (Protocol, Certificate, Project, and Target). func (r *ProtocolIncus) isSameServer(server Server) bool { // Short path checking if the two structs are identical. if r == server { return true } // Short path if either of the structs are nil. if r == nil || server == nil { return false } // When dealing with uninitialized servers, we can't safely compare. if r.server == nil { return false } // Get the connection info from both servers. srcInfo, err := r.GetConnectionInfo() if err != nil { return false } dstInfo, err := server.GetConnectionInfo() if err != nil { return false } // Check whether we're dealing with the same server. return srcInfo.Protocol == dstInfo.Protocol && srcInfo.Certificate == dstInfo.Certificate && srcInfo.Project == dstInfo.Project && srcInfo.Target == dstInfo.Target } // GetHTTPClient returns the http client used for the connection. This can be used to set custom http options. func (r *ProtocolIncus) GetHTTPClient() (*http.Client, error) { if r.http == nil { return nil, errors.New("HTTP client isn't set, bad connection") } return r.http, nil } // DoHTTP performs a Request, using OIDC authentication if set. func (r *ProtocolIncus) DoHTTP(req *http.Request) (*http.Response, error) { r.addClientHeaders(req) if r.oidcClient != nil { return r.oidcClient.do(req) } resp, err := r.http.Do(req) if resp != nil && resp.StatusCode == http.StatusUseProxy && req.GetBody != nil { // Reset the request body. body, err := req.GetBody() if err != nil { return nil, err } req.Body = body // Retry the request. return r.http.Do(req) } return resp, err } // DoWebsocket performs a websocket connection, using OIDC authentication if set. func (r *ProtocolIncus) DoWebsocket(dialer websocket.Dialer, uri string, req *http.Request) (*websocket.Conn, *http.Response, error) { r.addClientHeaders(req) if r.oidcClient != nil { return r.oidcClient.dial(dialer, uri, req) } return dialer.Dial(uri, req.Header) } // addClientHeaders sets headers from client settings. // User-Agent (if r.httpUserAgent is set). // X-Incus-authenticated (if r.requireAuthenticated is set). // OIDC Authorization header (if r.oidcClient is set). func (r *ProtocolIncus) addClientHeaders(req *http.Request) { if r.httpUserAgent != "" { req.Header.Set("User-Agent", r.httpUserAgent) } if r.requireAuthenticated { req.Header.Set("X-Incus-authenticated", "true") } if r.oidcClient != nil { req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.oidcClient.getAccessToken())) } } // RequireAuthenticated sets whether we expect to be authenticated with the server. func (r *ProtocolIncus) RequireAuthenticated(authenticated bool) { r.requireAuthenticated = authenticated } // RawQuery allows directly querying the Incus API // // This should only be used by internal Incus tools. func (r *ProtocolIncus) RawQuery(method string, path string, data any, ETag string) (*api.Response, string, error) { // Generate the URL url := fmt.Sprintf("%s%s", r.httpBaseURL.String(), path) return r.rawQuery(method, url, data, ETag) } // RawWebsocket allows directly connection to Incus API websockets // // This should only be used by internal Incus tools. func (r *ProtocolIncus) RawWebsocket(path string) (*websocket.Conn, error) { return r.websocket(path) } // RawOperation allows direct querying of an Incus API endpoint returning // background operations. func (r *ProtocolIncus) RawOperation(method string, path string, data any, ETag string) (Operation, string, error) { return r.queryOperation(method, path, data, ETag) } // Internal functions. func incusParseResponse(resp *http.Response) (*api.Response, string, error) { // Get the ETag etag := resp.Header.Get("ETag") // Decode the response decoder := json.NewDecoder(resp.Body) response := api.Response{} err := decoder.Decode(&response) if err != nil { // Check the return value for a cleaner error if resp.StatusCode != http.StatusOK { return nil, "", fmt.Errorf("Failed to fetch %s: %s", resp.Request.URL.String(), resp.Status) } return nil, "", err } // Handle errors if response.Type == api.ErrorResponse { return &response, "", api.StatusErrorf(resp.StatusCode, "%v", response.Error) } return &response, etag, nil } // rawQuery is a method that sends an HTTP request to the Incus server with the provided method, URL, data, and ETag. // It processes the request based on the data's type and handles the HTTP response, returning parsed results or an error if it occurs. func (r *ProtocolIncus) rawQuery(method string, url string, data any, ETag string) (*api.Response, string, error) { var req *http.Request var err error // Log the request logger.Debug("Sending request to Incus", logger.Ctx{ "method": method, "url": url, "etag": ETag, }) // Get a new HTTP request setup if data != nil { switch data := data.(type) { case io.Reader: // Some data to be sent along with the request req, err = http.NewRequestWithContext(r.ctx, method, url, io.NopCloser(data)) if err != nil { return nil, "", err } req.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(data), nil } // Set the encoding accordingly req.Header.Set("Content-Type", "application/octet-stream") default: // Encode the provided data buf := bytes.Buffer{} err := json.NewEncoder(&buf).Encode(data) if err != nil { return nil, "", err } // Some data to be sent along with the request // Use a reader since the request body needs to be seekable req, err = http.NewRequestWithContext(r.ctx, method, url, bytes.NewReader(buf.Bytes())) if err != nil { return nil, "", err } req.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(buf.Bytes())), nil } // Set the encoding accordingly req.Header.Set("Content-Type", "application/json") // Log the data logger.Debugf("%s", logger.Pretty(data)) } } else { // No data to be sent along with the request req, err = http.NewRequestWithContext(r.ctx, method, url, nil) if err != nil { return nil, "", err } } // Set the ETag if ETag != "" { req.Header.Set("If-Match", ETag) } // Send the request resp, err := r.DoHTTP(req) if err != nil { return nil, "", err } defer func() { _ = resp.Body.Close() }() return incusParseResponse(resp) } // setURLQueryAttributes modifies the supplied URL's query string with the client's current target and project. func (r *ProtocolIncus) setURLQueryAttributes(apiURL *neturl.URL) { // Extract query fields and update for cluster targeting or project values := apiURL.Query() if r.clusterTarget != "" { if values.Get("target") == "" { values.Set("target", r.clusterTarget) } } if r.project != "" { if values.Get("project") == "" && values.Get("all-projects") == "" { values.Set("project", r.project) } } apiURL.RawQuery = values.Encode() } func (r *ProtocolIncus) setQueryAttributes(uri string) (string, error) { // Parse the full URI fields, err := neturl.Parse(uri) if err != nil { return "", err } r.setURLQueryAttributes(fields) return fields.String(), nil } func (r *ProtocolIncus) query(method string, path string, data any, ETag string) (*api.Response, string, error) { // Generate the URL url := fmt.Sprintf("%s/1.0%s", r.httpBaseURL.String(), path) // Add project/target url, err := r.setQueryAttributes(url) if err != nil { return nil, "", err } // Run the actual query return r.rawQuery(method, url, data, ETag) } // queryStruct sends a query to the Incus server, then converts the response metadata into the specified target struct. // The function logs the retrieved data, returns the etag of the response, and handles any errors during this process. func (r *ProtocolIncus) queryStruct(method string, path string, data any, ETag string, target any) (string, error) { resp, etag, err := r.query(method, path, data, ETag) if err != nil { return "", err } err = resp.MetadataAsStruct(&target) if err != nil { return "", err } // Log the data logger.Debugf("Got response struct from Incus") logger.Debugf("%s", logger.Pretty(target)) return etag, nil } // queryOperation sends a query to the Incus server and then converts the response metadata into an Operation object. // It sets up an early event listener, performs the query, processes the response, and manages the lifecycle of the event listener. func (r *ProtocolIncus) queryOperation(method string, path string, data any, ETag string) (Operation, string, error) { // Attempt to setup an early event listener var listener *EventListener skipListener := r.skipEvents if !skipListener { var err error listener, err = r.GetEvents() if err != nil { if api.StatusErrorCheck(err, http.StatusForbidden) { skipListener = true } listener = nil } } // Send the query resp, etag, err := r.query(method, path, data, ETag) if err != nil { if listener != nil { listener.Disconnect() } return nil, "", err } // Get to the operation respOperation, err := resp.MetadataAsOperation() if err != nil { if listener != nil { listener.Disconnect() } return nil, "", err } // Setup an Operation wrapper op := operation{ Operation: *respOperation, r: r, listener: listener, skipListener: skipListener, chActive: make(chan bool), } // Log the data logger.Debugf("Got operation from Incus") logger.Debugf("%s", logger.Pretty(op.Operation)) return &op, etag, nil } // rawWebsocket creates a websocket connection to the provided URL using the underlying HTTP transport of the ProtocolIncus receiver. // It sets up the request headers, manages the connection handshake, sets TCP timeouts, and handles any errors that may occur during these operations. func (r *ProtocolIncus) rawWebsocket(url string) (*websocket.Conn, error) { // Grab the http transport handler httpTransport, err := r.getUnderlyingHTTPTransport() if err != nil { return nil, err } // Setup a new websocket dialer based on it dialer := websocket.Dialer{ NetDialTLSContext: httpTransport.DialTLSContext, NetDialContext: httpTransport.DialContext, TLSClientConfig: httpTransport.TLSClientConfig, Proxy: httpTransport.Proxy, HandshakeTimeout: time.Second * 5, } // Create temporary http.Request using the http url, not the ws one, so that we can add the client headers // for the websocket request. req := &http.Request{URL: &r.httpBaseURL, Header: http.Header{}} // Establish the connection conn, resp, err := r.DoWebsocket(dialer, url, req) if err != nil { if resp != nil { apiResp, _, parseErr := incusParseResponse(resp) if parseErr != nil { err = errors.Join(err, parseErr) } if apiResp != nil && apiResp.Error != "" { err = errors.Join(err, errors.New(apiResp.Error)) } } return nil, err } // Set TCP timeout options. remoteTCP, _ := tcp.ExtractConn(conn.UnderlyingConn()) if remoteTCP != nil { err = tcp.SetTimeouts(remoteTCP, 0) if err != nil { logger.Warn("Failed setting TCP timeouts on remote connection", logger.Ctx{"err": err}) } } // Log the data logger.Debugf("Connected to the websocket: %v", url) return conn, nil } // websocket generates a websocket URL based on the provided path and the base URL of the ProtocolIncus receiver. // It then leverages the rawWebsocket method to establish and return a websocket connection to the generated URL. func (r *ProtocolIncus) websocket(path string) (*websocket.Conn, error) { // Generate the URL var url string if r.httpBaseURL.Scheme == "https" { url = fmt.Sprintf("wss://%s/1.0%s", r.httpBaseURL.Host, path) } else { url = fmt.Sprintf("ws://%s/1.0%s", r.httpBaseURL.Host, path) } return r.rawWebsocket(url) } // WithContext returns a client that will add context.Context. func (r *ProtocolIncus) WithContext(ctx context.Context) InstanceServer { rr := r rr.ctx = ctx return rr } // getUnderlyingHTTPTransport returns the *http.Transport used by the http client. If the http // client was initialized with a HTTPTransporter, it returns the wrapped *http.Transport. func (r *ProtocolIncus) getUnderlyingHTTPTransport() (*http.Transport, error) { switch t := r.http.Transport.(type) { case *http.Transport: return t, nil case HTTPTransporter: return t.Transport(), nil default: return nil, fmt.Errorf("Unexpected http.Transport type, %T", r) } } // getSourceImageConnectionInfo returns the connection information for the source image. // The returned `info` is nil if the source image is local. In this process, the `instSrc` // is also updated with the minimal source fields. func (r *ProtocolIncus) getSourceImageConnectionInfo(source ImageServer, image api.Image, instSrc *api.InstanceSource) (info *ConnectionInfo, err error) { // Set the minimal source fields instSrc.Type = "image" // Optimization for the local image case if r.isSameServer(source) { // Always use fingerprints for local case instSrc.Fingerprint = image.Fingerprint instSrc.Alias = "" return nil, nil } // Minimal source fields for remote image instSrc.Mode = "pull" // If we have an alias and the image is public, use that if instSrc.Alias != "" && image.Public { instSrc.Fingerprint = "" } else { instSrc.Fingerprint = image.Fingerprint instSrc.Alias = "" } // Get source server connection information info, err = source.GetConnectionInfo() if err != nil { return nil, err } instSrc.Protocol = info.Protocol instSrc.Certificate = info.Certificate // Generate secret token if needed if !image.Public { secret, err := source.GetImageSecret(image.Fingerprint) if err != nil { return nil, err } instSrc.Secret = secret } return info, nil } incus-7.0.0/client/incus_certificates.go000066400000000000000000000065361517523235500203340ustar00rootroot00000000000000package incus import ( "errors" "fmt" "net/url" "github.com/lxc/incus/v7/shared/api" ) // Certificate handling functions // GetCertificateFingerprints returns a list of certificate fingerprints. func (r *ProtocolIncus) GetCertificateFingerprints() ([]string, error) { // Fetch the raw URL values. urls := []string{} baseURL := "/certificates" _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetCertificates returns a list of certificates. func (r *ProtocolIncus) GetCertificates() ([]api.Certificate, error) { certificates := []api.Certificate{} // Fetch the raw value _, err := r.queryStruct("GET", "/certificates?recursion=1", nil, "", &certificates) if err != nil { return nil, err } return certificates, nil } // GetCertificatesWithFilter returns a filtered list of certificates. func (r *ProtocolIncus) GetCertificatesWithFilter(filters []string) ([]api.Certificate, error) { certificates := []api.Certificate{} v := url.Values{} v.Set("recursion", "1") v.Set("filter", parseFilters(filters)) // Fetch the raw value _, err := r.queryStruct("GET", fmt.Sprintf("/certificates?%s", v.Encode()), nil, "", &certificates) if err != nil { return nil, err } return certificates, nil } // GetCertificate returns the certificate entry for the provided fingerprint. func (r *ProtocolIncus) GetCertificate(fingerprint string) (*api.Certificate, string, error) { certificate := api.Certificate{} // Fetch the raw value etag, err := r.queryStruct("GET", fmt.Sprintf("/certificates/%s", url.PathEscape(fingerprint)), nil, "", &certificate) if err != nil { return nil, "", err } return &certificate, etag, nil } // CreateCertificate adds a new certificate to the Incus trust store. func (r *ProtocolIncus) CreateCertificate(certificate api.CertificatesPost) error { // Send the request _, _, err := r.query("POST", "/certificates", certificate, "") if err != nil { return err } return nil } // UpdateCertificate updates the certificate definition. func (r *ProtocolIncus) UpdateCertificate(fingerprint string, certificate api.CertificatePut, ETag string) error { if !r.HasExtension("certificate_update") { return errors.New("The server is missing the required \"certificate_update\" API extension") } // Send the request _, _, err := r.query("PUT", fmt.Sprintf("/certificates/%s", url.PathEscape(fingerprint)), certificate, ETag) if err != nil { return err } return nil } // DeleteCertificate removes a certificate from the Incus trust store. func (r *ProtocolIncus) DeleteCertificate(fingerprint string) error { // Send the request _, _, err := r.query("DELETE", fmt.Sprintf("/certificates/%s", url.PathEscape(fingerprint)), nil, "") if err != nil { return err } return nil } // CreateCertificateToken requests a certificate add token. func (r *ProtocolIncus) CreateCertificateToken(certificate api.CertificatesPost) (Operation, error) { if !r.HasExtension("certificate_token") { return nil, errors.New("The server is missing the required \"certificate_token\" API extension") } if !certificate.Token { return nil, errors.New("Token needs to be true if requesting a token") } // Send the request op, _, err := r.queryOperation("POST", "/certificates", certificate, "") if err != nil { return nil, err } return op, nil } incus-7.0.0/client/incus_cluster.go000066400000000000000000000242771517523235500173520ustar00rootroot00000000000000package incus import ( "errors" "fmt" "net/url" "github.com/lxc/incus/v7/shared/api" ) // GetCluster returns information about a cluster. func (r *ProtocolIncus) GetCluster() (*api.Cluster, string, error) { if !r.HasExtension("clustering") { return nil, "", errors.New("The server is missing the required \"clustering\" API extension") } cluster := &api.Cluster{} etag, err := r.queryStruct("GET", "/cluster", nil, "", &cluster) if err != nil { return nil, "", err } return cluster, etag, nil } // UpdateCluster requests to bootstrap a new cluster or join an existing one. func (r *ProtocolIncus) UpdateCluster(cluster api.ClusterPut, ETag string) (Operation, error) { if !r.HasExtension("clustering") { return nil, errors.New("The server is missing the required \"clustering\" API extension") } if cluster.ServerAddress != "" || cluster.ClusterToken != "" || len(cluster.MemberConfig) > 0 { if !r.HasExtension("clustering_join") { return nil, errors.New("The server is missing the required \"clustering_join\" API extension") } } op, _, err := r.queryOperation("PUT", "/cluster", cluster, ETag) if err != nil { return nil, err } return op, nil } // DeleteClusterMember makes the given member leave the cluster (gracefully or not, // depending on the force flag). func (r *ProtocolIncus) DeleteClusterMember(name string, force bool) error { if !r.HasExtension("clustering") { return errors.New("The server is missing the required \"clustering\" API extension") } params := "" if force { params += "?force=1" } _, _, err := r.query("DELETE", fmt.Sprintf("/cluster/members/%s%s", name, params), nil, "") if err != nil { return err } return nil } // DeletePendingClusterMember makes the given pending member leave the cluster (gracefully or not, // depending on the force flag). func (r *ProtocolIncus) DeletePendingClusterMember(name string, force bool) error { if !r.HasExtension("clustering") { return errors.New("The server is missing the required \"clustering\" API extension") } params := "?pending=1" if force { params += "&force=1" } _, _, err := r.query("DELETE", fmt.Sprintf("/cluster/members/%s%s", name, params), nil, "") if err != nil { return err } return nil } // GetClusterMemberNames returns the URLs of the current members in the cluster. func (r *ProtocolIncus) GetClusterMemberNames() ([]string, error) { if !r.HasExtension("clustering") { return nil, errors.New("The server is missing the required \"clustering\" API extension") } // Fetch the raw URL values. urls := []string{} baseURL := "/cluster/members" _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetClusterMembersWithFilter returns a filtered list of cluster members as ClusterMember structs. func (r *ProtocolIncus) GetClusterMembersWithFilter(filters []string) ([]api.ClusterMember, error) { if !r.HasExtension("clustering") { return nil, errors.New("The server is missing the required \"clustering\" API extension") } members := []api.ClusterMember{} v := url.Values{} v.Set("recursion", "1") v.Set("filter", parseFilters(filters)) _, err := r.queryStruct("GET", fmt.Sprintf("/cluster/members?%s", v.Encode()), nil, "", &members) if err != nil { return nil, err } return members, nil } // GetClusterMembers returns the current members of the cluster. func (r *ProtocolIncus) GetClusterMembers() ([]api.ClusterMember, error) { if !r.HasExtension("clustering") { return nil, errors.New("The server is missing the required \"clustering\" API extension") } members := []api.ClusterMember{} _, err := r.queryStruct("GET", "/cluster/members?recursion=1", nil, "", &members) if err != nil { return nil, err } return members, nil } // GetClusterMember returns information about the given member. func (r *ProtocolIncus) GetClusterMember(name string) (*api.ClusterMember, string, error) { if !r.HasExtension("clustering") { return nil, "", errors.New("The server is missing the required \"clustering\" API extension") } member := api.ClusterMember{} etag, err := r.queryStruct("GET", fmt.Sprintf("/cluster/members/%s", name), nil, "", &member) if err != nil { return nil, "", err } return &member, etag, nil } // UpdateClusterMember updates information about the given member. func (r *ProtocolIncus) UpdateClusterMember(name string, member api.ClusterMemberPut, ETag string) error { if !r.HasExtension("clustering_edit_roles") { return errors.New("The server is missing the required \"clustering_edit_roles\" API extension") } if member.FailureDomain != "" { if !r.HasExtension("clustering_failure_domains") { return errors.New("The server is missing the required \"clustering_failure_domains\" API extension") } } // Send the request _, _, err := r.query("PUT", fmt.Sprintf("/cluster/members/%s", name), member, ETag) if err != nil { return err } return nil } // RenameClusterMember changes the name of an existing member. func (r *ProtocolIncus) RenameClusterMember(name string, member api.ClusterMemberPost) error { if !r.HasExtension("clustering") { return errors.New("The server is missing the required \"clustering\" API extension") } _, _, err := r.query("POST", fmt.Sprintf("/cluster/members/%s", name), member, "") if err != nil { return err } return nil } // CreateClusterMember generates a join token to add a cluster member. func (r *ProtocolIncus) CreateClusterMember(member api.ClusterMembersPost) (Operation, error) { if !r.HasExtension("clustering_join_token") { return nil, errors.New("The server is missing the required \"clustering_join_token\" API extension") } op, _, err := r.queryOperation("POST", "/cluster/members", member, "") if err != nil { return nil, err } return op, nil } // UpdateClusterCertificate updates the cluster certificate for every node in the cluster. func (r *ProtocolIncus) UpdateClusterCertificate(certs api.ClusterCertificatePut, ETag string) error { if !r.HasExtension("clustering_update_cert") { return errors.New("The server is missing the required \"clustering_update_cert\" API extension") } _, _, err := r.query("PUT", "/cluster/certificate", certs, ETag) if err != nil { return err } return nil } // GetClusterMemberState gets state information about a cluster member. func (r *ProtocolIncus) GetClusterMemberState(name string) (*api.ClusterMemberState, string, error) { err := r.CheckExtension("cluster_member_state") if err != nil { return nil, "", err } state := api.ClusterMemberState{} u := api.NewURL().Path("cluster", "members", name, "state") etag, err := r.queryStruct("GET", u.String(), nil, "", &state) if err != nil { return nil, "", err } return &state, etag, err } // UpdateClusterMemberState evacuates or restores a cluster member. func (r *ProtocolIncus) UpdateClusterMemberState(name string, state api.ClusterMemberStatePost) (Operation, error) { if !r.HasExtension("clustering_evacuation") { return nil, errors.New("The server is missing the required \"clustering_evacuation\" API extension") } op, _, err := r.queryOperation("POST", fmt.Sprintf("/cluster/members/%s/state", name), state, "") if err != nil { return nil, err } return op, nil } // GetClusterGroups returns the cluster groups. func (r *ProtocolIncus) GetClusterGroups() ([]api.ClusterGroup, error) { if !r.HasExtension("clustering_groups") { return nil, errors.New("The server is missing the required \"clustering_groups\" API extension") } groups := []api.ClusterGroup{} _, err := r.queryStruct("GET", "/cluster/groups?recursion=1", nil, "", &groups) if err != nil { return nil, err } return groups, nil } // GetClusterGroupNames returns the cluster group names. func (r *ProtocolIncus) GetClusterGroupNames() ([]string, error) { if !r.HasExtension("clustering_groups") { return nil, errors.New("The server is missing the required \"clustering_groups\" API extension") } urls := []string{} _, err := r.queryStruct("GET", "/cluster/groups", nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames("/1.0/cluster/groups", urls...) } // RenameClusterGroup changes the name of an existing cluster group. func (r *ProtocolIncus) RenameClusterGroup(name string, group api.ClusterGroupPost) error { if !r.HasExtension("clustering_groups") { return errors.New("The server is missing the required \"clustering_groups\" API extension") } _, _, err := r.query("POST", fmt.Sprintf("/cluster/groups/%s", name), group, "") if err != nil { return err } return nil } // CreateClusterGroup creates a new cluster group. func (r *ProtocolIncus) CreateClusterGroup(group api.ClusterGroupsPost) error { if !r.HasExtension("clustering_groups") { return errors.New("The server is missing the required \"clustering_groups\" API extension") } _, _, err := r.query("POST", "/cluster/groups", group, "") if err != nil { return err } return nil } // DeleteClusterGroup deletes an existing cluster group. func (r *ProtocolIncus) DeleteClusterGroup(name string) error { if !r.HasExtension("clustering_groups") { return errors.New("The server is missing the required \"clustering_groups\" API extension") } _, _, err := r.query("DELETE", fmt.Sprintf("/cluster/groups/%s", name), nil, "") if err != nil { return err } return nil } // UpdateClusterGroup updates information about the given cluster group. func (r *ProtocolIncus) UpdateClusterGroup(name string, group api.ClusterGroupPut, ETag string) error { if !r.HasExtension("clustering_groups") { return errors.New("The server is missing the required \"clustering_groups\" API extension") } // Send the request _, _, err := r.query("PUT", fmt.Sprintf("/cluster/groups/%s", name), group, ETag) if err != nil { return err } return nil } // GetClusterGroup returns information about the given cluster group. func (r *ProtocolIncus) GetClusterGroup(name string) (*api.ClusterGroup, string, error) { if !r.HasExtension("clustering_groups") { return nil, "", errors.New("The server is missing the required \"clustering_groups\" API extension") } group := api.ClusterGroup{} etag, err := r.queryStruct("GET", fmt.Sprintf("/cluster/groups/%s", name), nil, "", &group) if err != nil { return nil, "", err } return &group, etag, nil } incus-7.0.0/client/incus_events.go000066400000000000000000000134461517523235500171710ustar00rootroot00000000000000package incus import ( "context" "encoding/json" "errors" "net/url" "slices" "strings" "time" "github.com/gorilla/websocket" "github.com/lxc/incus/v7/shared/api" ) // Event handling functions // getEvents connects to the Incus monitoring interface. func (r *ProtocolIncus) getEvents(allProjects bool, eventTypes []string) (*EventListener, error) { // Prevent anything else from interacting with the listeners r.eventListenersLock.Lock() defer r.eventListenersLock.Unlock() ctx, cancel := context.WithCancel(context.Background()) // Clear skipGetEvents once we've been directly called. r.skipEvents = false // Setup a new listener listener := EventListener{ r: r, ctx: ctx, ctxCancel: cancel, } connInfo, _ := r.GetConnectionInfo() if connInfo.Project == "" { return nil, errors.New("Unexpected empty project in connection info") } if !allProjects { listener.projectName = connInfo.Project } // There is an existing Go routine for the required project filter, so just add another target. if r.eventListeners[listener.projectName] != nil { r.eventListeners[listener.projectName] = append(r.eventListeners[listener.projectName], &listener) return &listener, nil } // Setup a new connection with Incus var queryParams []string if allProjects { queryParams = append(queryParams, "all-projects=true") } if len(eventTypes) > 0 { for i := range len(eventTypes) { eventTypes[i] = url.QueryEscape(eventTypes[i]) } queryParams = append(queryParams, "type="+strings.Join(eventTypes, ",")) } eventsURL := "/events" if len(queryParams) > 0 { eventsURL += "?" + strings.Join(queryParams, "&") } eventsURL, err := r.setQueryAttributes(eventsURL) if err != nil { return nil, err } // Connect websocket and save. wsConn, err := r.websocket(eventsURL) if err != nil { return nil, err } r.eventConnsLock.Lock() r.eventConns[listener.projectName] = wsConn // Save for others to use. r.eventConnsLock.Unlock() // Initialize the event listener list if we were able to connect to the events websocket. r.eventListeners[listener.projectName] = []*EventListener{&listener} // Spawn a watcher that will close the websocket connection after all // listeners are gone. stopCh := make(chan struct{}) go func() { for { select { case <-time.After(time.Minute): case <-r.ctxConnected.Done(): case <-stopCh: } r.eventListenersLock.Lock() r.eventConnsLock.Lock() if len(r.eventListeners[listener.projectName]) == 0 { // We don't need the connection anymore, disconnect and clear. if r.eventListeners[listener.projectName] != nil { _ = r.eventConns[listener.projectName].Close() delete(r.eventConns, listener.projectName) } r.eventListeners[listener.projectName] = nil r.eventListenersLock.Unlock() r.eventConnsLock.Unlock() return } r.eventListenersLock.Unlock() r.eventConnsLock.Unlock() } }() // Spawn the listener go func() { for { _, data, err := wsConn.ReadMessage() if err != nil { // Prevent anything else from interacting with the listeners r.eventListenersLock.Lock() defer r.eventListenersLock.Unlock() // Tell all the current listeners about the failure for _, listener := range r.eventListeners[listener.projectName] { listener.err = err listener.ctxCancel() } // And remove them all from the list so that when watcher routine runs it will // close the websocket connection. r.eventListeners[listener.projectName] = nil close(stopCh) // Instruct watcher go routine to cleanup. return } // Attempt to unpack the message event := api.Event{} err = json.Unmarshal(data, &event) if err != nil { continue } // Extract the message type if event.Type == "" { continue } // Send the message to all handlers r.eventListenersLock.Lock() for _, listener := range r.eventListeners[listener.projectName] { listener.targetsLock.Lock() for _, target := range listener.targets { if target.types != nil && !slices.Contains(target.types, event.Type) { continue } go target.function(event) } listener.targetsLock.Unlock() } r.eventListenersLock.Unlock() } }() return &listener, nil } // GetEvents gets the events for the project defined on the client. func (r *ProtocolIncus) GetEvents() (*EventListener, error) { return r.getEvents(false, nil) } // GetEventsByType gets the events filtered by the provided list of types // for the project defined on the client. func (r *ProtocolIncus) GetEventsByType(eventTypes []string) (listener *EventListener, err error) { return r.getEvents(false, eventTypes) } // GetEventsAllProjects gets events for all projects. func (r *ProtocolIncus) GetEventsAllProjects() (*EventListener, error) { return r.getEvents(true, nil) } // GetEventsAllProjectsByType gets the events filtered by the provided list of // types for all projects. func (r *ProtocolIncus) GetEventsAllProjectsByType(eventTypes []string) (listener *EventListener, err error) { return r.getEvents(true, eventTypes) } // SendEvent send an event to the server via the client's event listener connection. func (r *ProtocolIncus) SendEvent(event api.Event) error { r.eventConnsLock.Lock() defer r.eventConnsLock.Unlock() // Find an available event listener connection. // It doesn't matter which project the event listener connection is using, as this only affects which // events are received from the server, not which events we can send to it. var eventConn *websocket.Conn for _, eventConn = range r.eventConns { break } if eventConn == nil { return errors.New("No available event listener connection") } deadline, ok := r.ctx.Deadline() if !ok { deadline = time.Now().Add(5 * time.Second) } _ = eventConn.SetWriteDeadline(deadline) return eventConn.WriteJSON(event) } incus-7.0.0/client/incus_images.go000066400000000000000000000652251517523235500171340ustar00rootroot00000000000000package incus import ( "crypto/sha256" "errors" "fmt" "io" "mime" "mime/multipart" "net/http" "net/url" "os" "slices" "strings" "time" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/cancel" "github.com/lxc/incus/v7/shared/ioprogress" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // Image handling functions // GetImages returns a list of available images as Image structs. func (r *ProtocolIncus) GetImages() ([]api.Image, error) { images := []api.Image{} _, err := r.queryStruct("GET", "/images?recursion=1", nil, "", &images) if err != nil { return nil, err } return images, nil } // GetImagesAllProjects returns a list of images across all projects as Image structs. func (r *ProtocolIncus) GetImagesAllProjects() ([]api.Image, error) { images := []api.Image{} v := url.Values{} v.Set("recursion", "1") v.Set("all-projects", "true") if !r.HasExtension("images_all_projects") { return nil, errors.New("The server is missing the required \"images_all_projects\" API extension") } _, err := r.queryStruct("GET", fmt.Sprintf("/images?%s", v.Encode()), nil, "", &images) if err != nil { return nil, err } return images, nil } // GetImagesAllProjectsWithFilter returns a filtered list of images across all projects as Image structs. func (r *ProtocolIncus) GetImagesAllProjectsWithFilter(filters []string) ([]api.Image, error) { images := []api.Image{} v := url.Values{} v.Set("recursion", "1") v.Set("all-projects", "true") v.Set("filter", parseFilters(filters)) if !r.HasExtension("images_all_projects") { return nil, errors.New("The server is missing the required \"images_all_projects\" API extension") } _, err := r.queryStruct("GET", fmt.Sprintf("/images?%s", v.Encode()), nil, "", &images) if err != nil { return nil, err } return images, nil } // GetImagesWithFilter returns a filtered list of available images as Image structs. func (r *ProtocolIncus) GetImagesWithFilter(filters []string) ([]api.Image, error) { if !r.HasExtension("api_filtering") { return nil, errors.New("The server is missing the required \"api_filtering\" API extension") } images := []api.Image{} v := url.Values{} v.Set("recursion", "1") v.Set("filter", parseFilters(filters)) _, err := r.queryStruct("GET", fmt.Sprintf("/images?%s", v.Encode()), nil, "", &images) if err != nil { return nil, err } return images, nil } // GetImageFingerprints returns a list of available image fingerprints. func (r *ProtocolIncus) GetImageFingerprints() ([]string, error) { // Fetch the raw URL values. urls := []string{} baseURL := "/images" _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetImage returns an Image struct for the provided fingerprint. func (r *ProtocolIncus) GetImage(fingerprint string) (*api.Image, string, error) { return r.GetPrivateImage(fingerprint, "") } // GetImageFile downloads an image from the server, returning an ImageFileRequest struct. func (r *ProtocolIncus) GetImageFile(fingerprint string, req ImageFileRequest) (*ImageFileResponse, error) { return r.GetPrivateImageFile(fingerprint, "", req) } // GetImageSecret is a helper around CreateImageSecret that returns a secret for the image. func (r *ProtocolIncus) GetImageSecret(fingerprint string) (string, error) { op, err := r.CreateImageSecret(fingerprint) if err != nil { return "", err } opAPI := op.Get() secret, ok := opAPI.Metadata["secret"].(string) if !ok { return "", errors.New("Bad secret type") } return secret, nil } // GetPrivateImage is similar to GetImage but allows passing a secret download token. func (r *ProtocolIncus) GetPrivateImage(fingerprint string, secret string) (*api.Image, string, error) { image := api.Image{} // Build the API path path := fmt.Sprintf("/images/%s", url.PathEscape(fingerprint)) var err error path, err = r.setQueryAttributes(path) if err != nil { return nil, "", err } if secret != "" { path, err = setQueryParam(path, "secret", secret) if err != nil { return nil, "", err } } // Fetch the raw value etag, err := r.queryStruct("GET", path, nil, "", &image) if err != nil { return nil, "", err } return &image, etag, nil } // GetPrivateImageFile is similar to GetImageFile but allows passing a secret download token. func (r *ProtocolIncus) GetPrivateImageFile(fingerprint string, secret string, req ImageFileRequest) (*ImageFileResponse, error) { // Quick checks. if req.MetaFile == nil && req.RootfsFile == nil { return nil, errors.New("No file requested") } uri := fmt.Sprintf("/1.0/images/%s/export", url.PathEscape(fingerprint)) var err error uri, err = r.setQueryAttributes(uri) if err != nil { return nil, err } // Attempt to download from host if secret == "" && util.PathExists("/dev/incus/sock") && os.Geteuid() == 0 { unixURI := fmt.Sprintf("http://unix.socket%s", uri) // Setup the HTTP client devIncusHTTP, err := unixHTTPClient(nil, "/dev/incus/sock") if err == nil { resp, err := incusDownloadImage(fingerprint, unixURI, r.httpUserAgent, devIncusHTTP.Do, req) if err == nil { return resp, nil } } } // Build the URL uri = fmt.Sprintf("%s%s", r.httpBaseURL.String(), uri) if secret != "" { uri, err = setQueryParam(uri, "secret", secret) if err != nil { return nil, err } } // Use relatively short response header timeout so as not to hold the image lock open too long. // Deference client and transport in order to clone them so as to not modify timeout of base client. httpClient := *r.http httpTransport := httpClient.Transport.(*http.Transport).Clone() httpTransport.ResponseHeaderTimeout = 30 * time.Second httpClient.Transport = httpTransport return incusDownloadImage(fingerprint, uri, r.httpUserAgent, r.DoHTTP, req) } func incusDownloadImage(fingerprint string, uri string, userAgent string, do func(*http.Request) (*http.Response, error), req ImageFileRequest) (*ImageFileResponse, error) { // Prepare the response resp := ImageFileResponse{} // Prepare the download request request, err := http.NewRequest("GET", uri, nil) if err != nil { return nil, err } if userAgent != "" { request.Header.Set("User-Agent", userAgent) } // Start the request response, doneCh, err := cancel.CancelableDownload(req.Canceler, do, request) if err != nil { return nil, err } defer func() { _ = response.Body.Close() }() defer close(doneCh) if response.StatusCode != http.StatusOK { _, _, err := incusParseResponse(response) if err != nil { return nil, err } } ctype, ctypeParams, err := mime.ParseMediaType(response.Header.Get("Content-Type")) if err != nil { ctype = "application/octet-stream" } // Check the image type. imageType := response.Header.Get("X-Incus-Type") if imageType == "" { imageType = "incus" } // Handle the data body := response.Body if req.ProgressHandler != nil { reader := &ioprogress.ProgressReader{ ReadCloser: response.Body, Tracker: &ioprogress.ProgressTracker{ Length: response.ContentLength, }, } if response.ContentLength > 0 { reader.Tracker.Handler = func(percent int64, speed int64) { req.ProgressHandler(ioprogress.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, units.GetByteSizeString(speed, 2))}) } } else { reader.Tracker.Handler = func(received int64, speed int64) { req.ProgressHandler(ioprogress.ProgressData{Text: fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(received, 2), units.GetByteSizeString(speed, 2))}) } } body = reader } // Hashing hash256 := sha256.New() // Deal with split images if ctype == "multipart/form-data" { if req.MetaFile == nil || req.RootfsFile == nil { return nil, errors.New("Multi-part image but only one target file provided") } // Parse the POST data mr := multipart.NewReader(body, ctypeParams["boundary"]) // Get the metadata tarball part, err := mr.NextPart() if err != nil { return nil, err } if part.FormName() != "metadata" { return nil, errors.New("Invalid multipart image") } size, err := util.SafeCopy(io.MultiWriter(req.MetaFile, hash256), part) if err != nil { return nil, err } resp.MetaSize = size resp.MetaName = part.FileName() // Get the rootfs tarball part, err = mr.NextPart() if err != nil { return nil, err } if !slices.Contains([]string{"rootfs", "rootfs.img"}, part.FormName()) { return nil, errors.New("Invalid multipart image") } size, err = util.SafeCopy(io.MultiWriter(req.RootfsFile, hash256), part) if err != nil { return nil, err } resp.RootfsSize = size resp.RootfsName = part.FileName() // Check the hash hash := fmt.Sprintf("%x", hash256.Sum(nil)) if imageType != "oci" && !strings.HasPrefix(hash, fingerprint) { return nil, fmt.Errorf("Image fingerprint doesn't match. Got %s expected %s", hash, fingerprint) } return &resp, nil } // Deal with unified images _, cdParams, err := mime.ParseMediaType(response.Header.Get("Content-Disposition")) if err != nil { return nil, err } filename, ok := cdParams["filename"] if !ok { return nil, errors.New("No filename in Content-Disposition header") } size, err := util.SafeCopy(io.MultiWriter(req.MetaFile, hash256), body) if err != nil { return nil, err } resp.MetaSize = size resp.MetaName = filename // Check the hash hash := fmt.Sprintf("%x", hash256.Sum(nil)) if imageType != "oci" && !strings.HasPrefix(hash, fingerprint) { return nil, fmt.Errorf("Image fingerprint doesn't match. Got %s expected %s", hash, fingerprint) } return &resp, nil } // GetImageAliases returns the list of available aliases as ImageAliasesEntry structs. func (r *ProtocolIncus) GetImageAliases() ([]api.ImageAliasesEntry, error) { aliases := []api.ImageAliasesEntry{} // Fetch the raw value _, err := r.queryStruct("GET", "/images/aliases?recursion=1", nil, "", &aliases) if err != nil { return nil, err } return aliases, nil } // GetImageAliasNames returns the list of available alias names. func (r *ProtocolIncus) GetImageAliasNames() ([]string, error) { // Fetch the raw URL values. urls := []string{} baseURL := "/images/aliases" _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetImageAlias returns an existing alias as an ImageAliasesEntry struct. func (r *ProtocolIncus) GetImageAlias(name string) (*api.ImageAliasesEntry, string, error) { alias := api.ImageAliasesEntry{} // Fetch the raw value etag, err := r.queryStruct("GET", fmt.Sprintf("/images/aliases/%s", url.PathEscape(name)), nil, "", &alias) if err != nil { return nil, "", err } return &alias, etag, nil } // GetImageAliasType returns an existing alias as an ImageAliasesEntry struct. func (r *ProtocolIncus) GetImageAliasType(imageType string, name string) (*api.ImageAliasesEntry, string, error) { alias, etag, err := r.GetImageAlias(name) if err != nil { return nil, "", err } if imageType != "" { if alias.Type == "" { alias.Type = "container" } if alias.Type != imageType { return nil, "", errors.New("Alias doesn't exist for the specified type") } } return alias, etag, nil } // GetImageAliasArchitectures returns a map of architectures / targets. func (r *ProtocolIncus) GetImageAliasArchitectures(imageType string, name string) (map[string]*api.ImageAliasesEntry, error) { alias, _, err := r.GetImageAliasType(imageType, name) if err != nil { return nil, err } img, _, err := r.GetImage(alias.Target) if err != nil { return nil, err } return map[string]*api.ImageAliasesEntry{img.Architecture: alias}, nil } // CreateImage requests that Incus creates, copies or import a new image. func (r *ProtocolIncus) CreateImage(image api.ImagesPost, args *ImageCreateArgs) (Operation, error) { if image.CompressionAlgorithm != "" { if !r.HasExtension("image_compression_algorithm") { return nil, errors.New("The server is missing the required \"image_compression_algorithm\" API extension") } } // Send the JSON based request if args == nil { op, _, err := r.queryOperation("POST", "/images", image, "") if err != nil { return nil, err } return op, nil } // Prepare an image upload if args.MetaFile == nil { return nil, errors.New("Metadata file is required") } // Prepare the body var body io.Reader var contentType string if args.RootfsFile == nil { // If unified image, just pass it through body = args.MetaFile contentType = "application/octet-stream" } else { pr, pw := io.Pipe() // Setup the multipart writer w := multipart.NewWriter(pw) go func() { var ioErr error defer func() { cerr := w.Close() if ioErr == nil && cerr != nil { ioErr = cerr } _ = pw.CloseWithError(ioErr) }() // Metadata file fw, ioErr := w.CreateFormFile("metadata", args.MetaName) if ioErr != nil { return } _, ioErr = util.SafeCopy(fw, args.MetaFile) if ioErr != nil { return } // Rootfs file if args.Type == "virtual-machine" { fw, ioErr = w.CreateFormFile("rootfs.img", args.RootfsName) } else { fw, ioErr = w.CreateFormFile("rootfs", args.RootfsName) } if ioErr != nil { return } _, ioErr = util.SafeCopy(fw, args.RootfsFile) if ioErr != nil { return } // Done writing to multipart ioErr = w.Close() if ioErr != nil { return } ioErr = pw.Close() if ioErr != nil { return } }() // Setup progress handler if args.ProgressHandler != nil { body = &ioprogress.ProgressReader{ ReadCloser: pr, Tracker: &ioprogress.ProgressTracker{ Handler: func(received int64, speed int64) { args.ProgressHandler(ioprogress.ProgressData{Text: fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(received, 2), units.GetByteSizeString(speed, 2))}) }, }, } } else { body = pr } contentType = w.FormDataContentType() } // Prepare the HTTP request reqURL, err := r.setQueryAttributes(fmt.Sprintf("%s/1.0/images", r.httpBaseURL.String())) if err != nil { return nil, err } req, err := http.NewRequest("POST", reqURL, body) if err != nil { return nil, err } // Setup the headers req.Header.Set("Content-Type", contentType) if image.Public { req.Header.Set("X-Incus-public", "true") } if image.Filename != "" { req.Header.Set("X-Incus-filename", image.Filename) } if len(image.Properties) > 0 { imgProps := url.Values{} for k, v := range image.Properties { imgProps.Set(k, v) } req.Header.Set("X-Incus-properties", imgProps.Encode()) } if len(image.Profiles) > 0 { imgProfiles := url.Values{} for _, v := range image.Profiles { imgProfiles.Add("profile", v) } req.Header.Set("X-Incus-profiles", imgProfiles.Encode()) } if len(image.Aliases) > 0 { imgProfiles := url.Values{} for _, v := range image.Aliases { imgProfiles.Add("alias", v.Name) } req.Header.Set("X-Incus-aliases", imgProfiles.Encode()) } // Set the user agent if image.Source != nil && image.Source.Fingerprint != "" && image.Source.Secret != "" && image.Source.Mode == "push" { // Set fingerprint req.Header.Set("X-Incus-fingerprint", image.Source.Fingerprint) // Set secret req.Header.Set("X-Incus-secret", image.Source.Secret) } // Send the request resp, err := r.DoHTTP(req) if err != nil { return nil, err } defer func() { _ = resp.Body.Close() }() // Handle errors response, _, err := incusParseResponse(resp) if err != nil { return nil, err } // Get to the operation respOperation, err := response.MetadataAsOperation() if err != nil { return nil, err } // Setup an Operation wrapper op := operation{ Operation: *respOperation, r: r, chActive: make(chan bool), } return &op, nil } // tryCopyImage iterates through the source server URLs until one lets it download the image. func (r *ProtocolIncus) tryCopyImage(req api.ImagesPost, urls []string) (RemoteOperation, error) { if len(urls) == 0 { return nil, errors.New("The source server isn't listening on the network") } rop := remoteOperation{ chDone: make(chan bool), } // For older servers, apply the aliases after copy if !r.HasExtension("image_create_aliases") && req.Aliases != nil { rop.chPost = make(chan bool) go func() { defer close(rop.chPost) // Wait for the main operation to finish <-rop.chDone if rop.err != nil { return } var errs []remoteOperationResult // Get the operation data op, err := rop.GetTarget() if err != nil { errs = append(errs, remoteOperationResult{Error: err}) rop.err = remoteOperationError("Failed to get operation data", errs) return } // Extract the fingerprint fingerprint, ok := op.Metadata["fingerprint"].(string) if !ok { errs = append(errs, remoteOperationResult{Error: errors.New("Bad fingerprint")}) rop.err = remoteOperationError("Failed to get operation data", errs) return } // Add the aliases for _, entry := range req.Aliases { alias := api.ImageAliasesPost{} alias.Name = entry.Name alias.Target = fingerprint err := r.CreateImageAlias(alias) if err != nil { errs = append(errs, remoteOperationResult{Error: err}) rop.err = remoteOperationError("Failed to create image alias", errs) return } } }() } // Forward targetOp to remote op go func() { success := false var errs []remoteOperationResult for _, serverURL := range urls { req.Source.Server = serverURL op, err := r.CreateImage(req, nil) if err != nil { errs = append(errs, remoteOperationResult{URL: serverURL, Error: err}) continue } rop.handlerLock.Lock() rop.targetOp = op rop.handlerLock.Unlock() for _, handler := range rop.handlers { _, _ = rop.targetOp.AddHandler(handler) } err = rop.targetOp.Wait() if err != nil { errs = append(errs, remoteOperationResult{URL: serverURL, Error: err}) if localtls.IsConnectionError(err) { continue } break } success = true break } if !success { rop.err = remoteOperationError("Failed remote image download", errs) } close(rop.chDone) }() return &rop, nil } // CopyImage copies an image from a remote server. Additional options can be passed using ImageCopyArgs. func (r *ProtocolIncus) CopyImage(source ImageServer, image api.Image, args *ImageCopyArgs) (RemoteOperation, error) { // Quick checks. if r.isSameServer(source) { return nil, errors.New("The source and target servers must be different") } // Handle profile list overrides. if args != nil && args.Profiles != nil { if !r.HasExtension("image_copy_profile") { return nil, errors.New("The server is missing the required \"image_copy_profile\" API extension") } image.Profiles = args.Profiles } else { // If profiles aren't provided, clear the list on the source to // avoid requiring the destination to have them all. image.Profiles = nil } // Get source server connection information info, err := source.GetConnectionInfo() if err != nil { return nil, err } // Push mode if args != nil && args.Mode == "push" { // Get certificate and URL info, err := r.GetConnectionInfo() if err != nil { return nil, err } imagesPost := api.ImagesPost{ Source: &api.ImagesPostSource{ Fingerprint: image.Fingerprint, Mode: args.Mode, }, } imagesPost.Aliases = args.Aliases if args.CopyAliases { imagesPost.Aliases = image.Aliases if args.Aliases != nil { imagesPost.Aliases = append(imagesPost.Aliases, args.Aliases...) } } imagesPost.ExpiresAt = image.ExpiresAt imagesPost.Properties = image.Properties imagesPost.Public = args.Public // Receive token from target server. This token is later passed to the source which will use // it, together with the URL and certificate, to connect to the target. tokenOp, err := r.CreateImage(imagesPost, nil) if err != nil { return nil, err } opAPI := tokenOp.Get() secret, ok := opAPI.Metadata["secret"] if !ok { return nil, errors.New("No token provided") } req := api.ImageExportPost{ Target: info.URL, Certificate: info.Certificate, Secret: secret.(string), Project: info.Project, Profiles: image.Profiles, } exportOp, err := source.ExportImage(image.Fingerprint, req) if err != nil { _ = tokenOp.Cancel() return nil, err } rop := remoteOperation{ targetOp: exportOp, chDone: make(chan bool), } // Forward targetOp to remote op go func() { rop.err = rop.targetOp.Wait() _ = tokenOp.Cancel() close(rop.chDone) }() return &rop, nil } // Relay mode if args != nil && args.Mode == "relay" { metaFile, err := os.CreateTemp(r.tempPath, "incus_image_") if err != nil { return nil, err } defer func() { _ = os.Remove(metaFile.Name()) }() rootfsFile, err := os.CreateTemp(r.tempPath, "incus_image_") if err != nil { return nil, err } defer func() { _ = os.Remove(rootfsFile.Name()) }() // Import image req := ImageFileRequest{ MetaFile: metaFile, RootfsFile: rootfsFile, } resp, err := source.GetImageFile(image.Fingerprint, req) if err != nil { return nil, err } // Export image _, err = metaFile.Seek(0, io.SeekStart) if err != nil { return nil, err } _, err = rootfsFile.Seek(0, io.SeekStart) if err != nil { return nil, err } imagePost := api.ImagesPost{} imagePost.Public = args.Public imagePost.Profiles = image.Profiles imagePost.Aliases = args.Aliases if args.CopyAliases { imagePost.Aliases = image.Aliases if args.Aliases != nil { imagePost.Aliases = append(imagePost.Aliases, args.Aliases...) } } createArgs := &ImageCreateArgs{ MetaFile: metaFile, MetaName: image.Filename, Type: image.Type, } if resp.RootfsName != "" { // Deal with split images createArgs.RootfsFile = rootfsFile createArgs.RootfsName = image.Filename } rop := remoteOperation{ chDone: make(chan bool), } go func() { defer close(rop.chDone) op, err := r.CreateImage(imagePost, createArgs) if err != nil { rop.err = remoteOperationError("Failed to copy image", nil) return } rop.handlerLock.Lock() rop.targetOp = op rop.handlerLock.Unlock() for _, handler := range rop.handlers { _, _ = rop.targetOp.AddHandler(handler) } err = rop.targetOp.Wait() if err != nil { rop.err = remoteOperationError("Failed to copy image", nil) return } // Apply the aliases. for _, entry := range imagePost.Aliases { alias := api.ImageAliasesPost{} alias.Name = entry.Name alias.Target = image.Fingerprint err := r.CreateImageAlias(alias) if err != nil { rop.err = remoteOperationError("Failed to add alias", nil) return } } }() return &rop, nil } // Prepare the copy request req := api.ImagesPost{ Source: &api.ImagesPostSource{ ImageSource: api.ImageSource{ Certificate: info.Certificate, Protocol: info.Protocol, }, Fingerprint: image.Fingerprint, Mode: "pull", Type: "image", Project: info.Project, }, ImagePut: api.ImagePut{ Profiles: image.Profiles, }, } if args != nil { req.Source.ImageType = args.Type } // Generate secret token if needed if !image.Public { secret, err := source.GetImageSecret(image.Fingerprint) if err != nil { return nil, err } req.Source.Secret = secret } // Process the arguments if args != nil { req.Aliases = args.Aliases req.AutoUpdate = args.AutoUpdate req.Public = args.Public if args.CopyAliases { req.Aliases = image.Aliases if args.Aliases != nil { req.Aliases = append(req.Aliases, args.Aliases...) } } } return r.tryCopyImage(req, info.Addresses) } // UpdateImage updates the image definition. func (r *ProtocolIncus) UpdateImage(fingerprint string, image api.ImagePut, ETag string) error { // Send the request _, _, err := r.query("PUT", fmt.Sprintf("/images/%s", url.PathEscape(fingerprint)), image, ETag) if err != nil { return err } return nil } // DeleteImage requests that Incus removes an image from the store. func (r *ProtocolIncus) DeleteImage(fingerprint string) (Operation, error) { // Send the request op, _, err := r.queryOperation("DELETE", fmt.Sprintf("/images/%s", url.PathEscape(fingerprint)), nil, "") if err != nil { return nil, err } return op, nil } // RefreshImage requests that Incus issues an image refresh. func (r *ProtocolIncus) RefreshImage(fingerprint string) (Operation, error) { if !r.HasExtension("image_force_refresh") { return nil, errors.New("The server is missing the required \"image_force_refresh\" API extension") } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("/images/%s/refresh", url.PathEscape(fingerprint)), nil, "") if err != nil { return nil, err } return op, nil } // CreateImageSecret requests that Incus issues a temporary image secret. func (r *ProtocolIncus) CreateImageSecret(fingerprint string) (Operation, error) { // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("/images/%s/secret", url.PathEscape(fingerprint)), nil, "") if err != nil { return nil, err } return op, nil } // CreateImageAlias sets up a new image alias. func (r *ProtocolIncus) CreateImageAlias(alias api.ImageAliasesPost) error { // Send the request _, _, err := r.query("POST", "/images/aliases", alias, "") if err != nil { return err } return nil } // UpdateImageAlias updates the image alias definition. func (r *ProtocolIncus) UpdateImageAlias(name string, alias api.ImageAliasesEntryPut, ETag string) error { // Send the request _, _, err := r.query("PUT", fmt.Sprintf("/images/aliases/%s", url.PathEscape(name)), alias, ETag) if err != nil { return err } return nil } // RenameImageAlias renames an existing image alias. func (r *ProtocolIncus) RenameImageAlias(name string, alias api.ImageAliasesEntryPost) error { // Send the request _, _, err := r.query("POST", fmt.Sprintf("/images/aliases/%s", url.PathEscape(name)), alias, "") if err != nil { return err } return nil } // DeleteImageAlias removes an alias from the Incus image store. func (r *ProtocolIncus) DeleteImageAlias(name string) error { // Send the request _, _, err := r.query("DELETE", fmt.Sprintf("/images/aliases/%s", url.PathEscape(name)), nil, "") if err != nil { return err } return nil } // ExportImage exports (copies) an image to a remote server. func (r *ProtocolIncus) ExportImage(fingerprint string, image api.ImageExportPost) (Operation, error) { if !r.HasExtension("images_push_relay") { return nil, errors.New("The server is missing the required \"images_push_relay\" API extension") } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("/images/%s/export", url.PathEscape(fingerprint)), &image, "") if err != nil { return nil, err } return op, nil } incus-7.0.0/client/incus_instances.go000066400000000000000000002465321517523235500176600ustar00rootroot00000000000000package incus import ( "bufio" "bytes" "context" "encoding/json" "errors" "fmt" "io" "net" "net/http" "net/url" "path/filepath" "slices" "strings" "github.com/gorilla/websocket" "github.com/pkg/sftp" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/cancel" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/tcp" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/ws" ) // Instance handling functions. // instanceTypeToPath converts the instance type to a URL path prefix and query string values. func (r *ProtocolIncus) instanceTypeToPath(instanceType api.InstanceType) (string, url.Values, error) { v := url.Values{} // If a specific instance type has been requested, add the instance-type filter parameter // to the returned URL values so that it can be used in the final URL if needed to filter // the result set being returned. if instanceType != api.InstanceTypeAny { v.Set("instance-type", string(instanceType)) } return "/instances", v, nil } // GetInstanceNames returns a list of instance names. func (r *ProtocolIncus) GetInstanceNames(instanceType api.InstanceType) ([]string, error) { baseURL, v, err := r.instanceTypeToPath(instanceType) if err != nil { return nil, err } // Fetch the raw URL values. urls := []string{} _, err = r.queryStruct("GET", fmt.Sprintf("%s?%s", baseURL, v.Encode()), nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetInstanceNamesAllProjects returns a list of instance names from all projects. func (r *ProtocolIncus) GetInstanceNamesAllProjects(instanceType api.InstanceType) (map[string][]string, error) { instances := []api.Instance{} path, v, err := r.instanceTypeToPath(instanceType) if err != nil { return nil, err } v.Set("recursion", "1") v.Set("all-projects", "true") // Fetch the raw URL values. _, err = r.queryStruct("GET", fmt.Sprintf("%s?%s", path, v.Encode()), nil, "", &instances) if err != nil { return nil, err } names := map[string][]string{} for _, instance := range instances { names[instance.Project] = append(names[instance.Project], instance.Name) } return names, nil } // GetInstances returns a list of instances. func (r *ProtocolIncus) GetInstances(instanceType api.InstanceType) ([]api.Instance, error) { instances := []api.Instance{} path, v, err := r.instanceTypeToPath(instanceType) if err != nil { return nil, err } v.Set("recursion", "1") // Fetch the raw value _, err = r.queryStruct("GET", fmt.Sprintf("%s?%s", path, v.Encode()), nil, "", &instances) if err != nil { return nil, err } return instances, nil } // GetInstancesWithFilter returns a filtered list of instances. func (r *ProtocolIncus) GetInstancesWithFilter(instanceType api.InstanceType, filters []string) ([]api.Instance, error) { if !r.HasExtension("api_filtering") { return nil, errors.New("The server is missing the required \"api_filtering\" API extension") } instances := []api.Instance{} path, v, err := r.instanceTypeToPath(instanceType) if err != nil { return nil, err } v.Set("recursion", "1") v.Set("filter", parseFilters(filters)) // Fetch the raw value _, err = r.queryStruct("GET", fmt.Sprintf("%s?%s", path, v.Encode()), nil, "", &instances) if err != nil { return nil, err } return instances, nil } // GetInstancesAllProjects returns a list of instances from all projects. func (r *ProtocolIncus) GetInstancesAllProjects(instanceType api.InstanceType) ([]api.Instance, error) { instances := []api.Instance{} path, v, err := r.instanceTypeToPath(instanceType) if err != nil { return nil, err } v.Set("recursion", "1") v.Set("all-projects", "true") if !r.HasExtension("instance_all_projects") { return nil, errors.New("The server is missing the required \"instance_all_projects\" API extension") } // Fetch the raw value _, err = r.queryStruct("GET", fmt.Sprintf("%s?%s", path, v.Encode()), nil, "", &instances) if err != nil { return nil, err } return instances, nil } // GetInstancesAllProjectsWithFilter returns a filtered list of instances from all projects. func (r *ProtocolIncus) GetInstancesAllProjectsWithFilter(instanceType api.InstanceType, filters []string) ([]api.Instance, error) { if !r.HasExtension("api_filtering") { return nil, errors.New("The server is missing the required \"api_filtering\" API extension") } instances := []api.Instance{} path, v, err := r.instanceTypeToPath(instanceType) if err != nil { return nil, err } v.Set("recursion", "1") v.Set("all-projects", "true") v.Set("filter", parseFilters(filters)) if !r.HasExtension("instance_all_projects") { return nil, errors.New("The server is missing the required \"instance_all_projects\" API extension") } // Fetch the raw value _, err = r.queryStruct("GET", fmt.Sprintf("%s?%s", path, v.Encode()), nil, "", &instances) if err != nil { return nil, err } return instances, nil } // UpdateInstances updates all instances to match the requested state. func (r *ProtocolIncus) UpdateInstances(state api.InstancesPut, ETag string) (Operation, error) { path, v, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } // Send the request op, _, err := r.queryOperation("PUT", fmt.Sprintf("%s?%s", path, v.Encode()), state, ETag) if err != nil { return nil, err } return op, nil } // rebuildInstance initiates a rebuild of a given instance on the Incus Protocol server and returns the corresponding operation or an error. func (r *ProtocolIncus) rebuildInstance(instanceName string, instance api.InstanceRebuildPost) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("%s/%s/rebuild", path, url.PathEscape(instanceName)), instance, "") if err != nil { return nil, err } return op, nil } // tryRebuildInstance attempts to rebuild a specific instance on multiple target servers identified by their URLs. // It runs the rebuild process asynchronously and returns a RemoteOperation to monitor the progress and any errors. func (r *ProtocolIncus) tryRebuildInstance(instanceName string, req api.InstanceRebuildPost, urls []string, op Operation) (RemoteOperation, error) { if len(urls) == 0 { return nil, errors.New("The source server isn't listening on the network") } rop := remoteOperation{ chDone: make(chan bool), } operation := req.Source.Operation // Forward targetOp to remote op go func() { success := false var errors []remoteOperationResult for _, serverURL := range urls { if operation == "" { req.Source.Server = serverURL } else { req.Source.Operation = fmt.Sprintf("%s/1.0/operations/%s", serverURL, url.PathEscape(operation)) } op, err := r.rebuildInstance(instanceName, req) if err != nil { errors = append(errors, remoteOperationResult{URL: serverURL, Error: err}) continue } rop.handlerLock.Lock() rop.targetOp = op rop.handlerLock.Unlock() for _, handler := range rop.handlers { _, _ = rop.targetOp.AddHandler(handler) } err = rop.targetOp.Wait() if err != nil { errors = append(errors, remoteOperationResult{URL: serverURL, Error: err}) if localtls.IsConnectionError(err) { continue } break } success = true break } if !success { rop.err = remoteOperationError("Failed instance rebuild", errors) if op != nil { _ = op.Cancel() } } close(rop.chDone) }() return &rop, nil } // RebuildInstanceFromImage rebuilds an instance from an image. func (r *ProtocolIncus) RebuildInstanceFromImage(source ImageServer, image api.Image, instanceName string, req api.InstanceRebuildPost) (RemoteOperation, error) { err := r.CheckExtension("instances_rebuild") if err != nil { return nil, err } info, err := r.getSourceImageConnectionInfo(source, image, &req.Source) if err != nil { return nil, err } if info == nil { op, err := r.rebuildInstance(instanceName, req) if err != nil { return nil, err } rop := remoteOperation{ targetOp: op, chDone: make(chan bool), } // Forward targetOp to remote op go func() { rop.err = rop.targetOp.Wait() close(rop.chDone) }() return &rop, nil } return r.tryRebuildInstance(instanceName, req, info.Addresses, nil) } // RebuildInstance rebuilds an instance as empty. func (r *ProtocolIncus) RebuildInstance(instanceName string, instance api.InstanceRebuildPost) (op Operation, err error) { err = r.CheckExtension("instances_rebuild") if err != nil { return nil, err } return r.rebuildInstance(instanceName, instance) } // GetInstancesFull returns a list of instances including snapshots, backups and state. func (r *ProtocolIncus) GetInstancesFull(instanceType api.InstanceType) ([]api.InstanceFull, error) { instances := []api.InstanceFull{} path, v, err := r.instanceTypeToPath(instanceType) if err != nil { return nil, err } v.Set("recursion", "2") if !r.HasExtension("container_full") { return nil, errors.New("The server is missing the required \"container_full\" API extension") } // Fetch the raw value _, err = r.queryStruct("GET", fmt.Sprintf("%s?%s", path, v.Encode()), nil, "", &instances) if err != nil { return nil, err } return instances, nil } // GetInstancesFullWithFilter returns a filtered list of instances including snapshots, backups and state. func (r *ProtocolIncus) GetInstancesFullWithFilter(instanceType api.InstanceType, filters []string) ([]api.InstanceFull, error) { if !r.HasExtension("api_filtering") { return nil, errors.New("The server is missing the required \"api_filtering\" API extension") } instances := []api.InstanceFull{} path, v, err := r.instanceTypeToPath(instanceType) if err != nil { return nil, err } v.Set("recursion", "2") v.Set("filter", parseFilters(filters)) if !r.HasExtension("container_full") { return nil, errors.New("The server is missing the required \"container_full\" API extension") } // Fetch the raw value _, err = r.queryStruct("GET", fmt.Sprintf("%s?%s", path, v.Encode()), nil, "", &instances) if err != nil { return nil, err } return instances, nil } // GetInstancesFullAllProjects returns a list of instances including snapshots, backups and state from all projects. func (r *ProtocolIncus) GetInstancesFullAllProjects(instanceType api.InstanceType) ([]api.InstanceFull, error) { instances := []api.InstanceFull{} path, v, err := r.instanceTypeToPath(instanceType) if err != nil { return nil, err } v.Set("recursion", "2") v.Set("all-projects", "true") if !r.HasExtension("container_full") { return nil, errors.New("The server is missing the required \"container_full\" API extension") } if !r.HasExtension("instance_all_projects") { return nil, errors.New("The server is missing the required \"instance_all_projects\" API extension") } // Fetch the raw value _, err = r.queryStruct("GET", fmt.Sprintf("%s?%s", path, v.Encode()), nil, "", &instances) if err != nil { return nil, err } return instances, nil } // GetInstancesFullAllProjectsWithFilter returns a filtered list of instances including snapshots, backups and state from all projects. func (r *ProtocolIncus) GetInstancesFullAllProjectsWithFilter(instanceType api.InstanceType, filters []string) ([]api.InstanceFull, error) { if !r.HasExtension("api_filtering") { return nil, errors.New("The server is missing the required \"api_filtering\" API extension") } instances := []api.InstanceFull{} path, v, err := r.instanceTypeToPath(instanceType) if err != nil { return nil, err } v.Set("recursion", "2") v.Set("all-projects", "true") v.Set("filter", parseFilters(filters)) if !r.HasExtension("container_full") { return nil, errors.New("The server is missing the required \"container_full\" API extension") } if !r.HasExtension("instance_all_projects") { return nil, errors.New("The server is missing the required \"instance_all_projects\" API extension") } // Fetch the raw value _, err = r.queryStruct("GET", fmt.Sprintf("%s?%s", path, v.Encode()), nil, "", &instances) if err != nil { return nil, err } return instances, nil } // GetInstance returns the instance entry for the provided name. func (r *ProtocolIncus) GetInstance(name string) (*api.Instance, string, error) { instance := api.Instance{} path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, "", err } // Fetch the raw value etag, err := r.queryStruct("GET", fmt.Sprintf("%s/%s", path, url.PathEscape(name)), nil, "", &instance) if err != nil { return nil, "", err } return &instance, etag, nil } // GetInstanceFull returns the instance entry for the provided name along with snapshot information. func (r *ProtocolIncus) GetInstanceFull(name string) (*api.InstanceFull, string, error) { instance := api.InstanceFull{} if !r.HasExtension("instance_get_full") { // Backward compatibility. ct, _, err := r.GetInstance(name) if err != nil { return nil, "", err } cs, _, err := r.GetInstanceState(name) if err != nil { return nil, "", err } snaps, err := r.GetInstanceSnapshots(name) if err != nil { return nil, "", err } backups, err := r.GetInstanceBackups(name) if err != nil { return nil, "", err } instance.Instance = *ct instance.State = cs instance.Snapshots = snaps instance.Backups = backups return &instance, "", nil } path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, "", err } // Fetch the raw value etag, err := r.queryStruct("GET", fmt.Sprintf("%s/%s?recursion=1", path, url.PathEscape(name)), nil, "", &instance) if err != nil { return nil, "", err } return &instance, etag, nil } // CreateInstanceFromBackup is a convenience function to make it easier to // create a instance from a backup. func (r *ProtocolIncus) CreateInstanceFromBackup(args InstanceBackupArgs) (Operation, error) { if !r.HasExtension("container_backup") { return nil, errors.New("The server is missing the required \"container_backup\" API extension") } path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } if args.PoolName == "" && args.Name == "" && args.Config == nil && args.Devices == nil { // Send the request op, _, err := r.queryOperation("POST", path, args.BackupFile, "") if err != nil { return nil, err } return op, nil } if args.PoolName != "" && !r.HasExtension("container_backup_override_pool") { return nil, errors.New(`The server is missing the required "container_backup_override_pool" API extension`) } if args.Name != "" && !r.HasExtension("backup_override_name") { return nil, errors.New(`The server is missing the required "backup_override_name" API extension`) } if (args.Config != nil || args.Devices != nil) && !r.HasExtension("backup_override_config") { return nil, errors.New(`The server is missing the required "backup_override_config" API extension`) } // Prepare the HTTP request reqURL, err := r.setQueryAttributes(fmt.Sprintf("%s/1.0%s", r.httpBaseURL.String(), path)) if err != nil { return nil, err } req, err := http.NewRequest("POST", reqURL, args.BackupFile) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/octet-stream") if args.PoolName != "" { req.Header.Set("X-Incus-pool", args.PoolName) } if args.Name != "" { req.Header.Set("X-Incus-name", args.Name) } if args.Config != nil { configOverride := strings.Join(args.Config, " ") req.Header.Set("X-Incus-config", configOverride) } if args.Devices != nil { devicesOverride := strings.Join(args.Devices, " ") req.Header.Set("X-Incus-devices", devicesOverride) } // Send the request resp, err := r.DoHTTP(req) if err != nil { return nil, err } defer func() { _ = resp.Body.Close() }() // Handle errors response, _, err := incusParseResponse(resp) if err != nil { return nil, err } // Get to the operation respOperation, err := response.MetadataAsOperation() if err != nil { return nil, err } // Setup an Operation wrapper op := operation{ Operation: *respOperation, r: r, chActive: make(chan bool), } return &op, nil } // CreateInstance requests that Incus creates a new instance. func (r *ProtocolIncus) CreateInstance(instance api.InstancesPost) (Operation, error) { path, _, err := r.instanceTypeToPath(instance.Type) if err != nil { return nil, err } if instance.Source.InstanceOnly { if !r.HasExtension("container_only_migration") { return nil, errors.New("The server is missing the required \"container_only_migration\" API extension") } } // Send the request op, _, err := r.queryOperation("POST", path, instance, "") if err != nil { return nil, err } return op, nil } // tryCreateInstance attempts to create a new instance on multiple target servers specified by their URLs. // It runs the instance creation asynchronously and returns a RemoteOperation to monitor the progress and any errors. func (r *ProtocolIncus) tryCreateInstance(req api.InstancesPost, urls []string, op Operation) (RemoteOperation, error) { if len(urls) == 0 { return nil, errors.New("The source server isn't listening on the network") } rop := remoteOperation{ chDone: make(chan bool), } operation := req.Source.Operation // Forward targetOp to remote op chConnect := make(chan error, 1) chWait := make(chan error, 1) go func() { success := false var errors []remoteOperationResult for _, serverURL := range urls { if operation == "" { req.Source.Server = serverURL } else { req.Source.Operation = fmt.Sprintf("%s/1.0/operations/%s", serverURL, url.PathEscape(operation)) } op, err := r.CreateInstance(req) if err != nil { errors = append(errors, remoteOperationResult{URL: serverURL, Error: err}) continue } rop.handlerLock.Lock() rop.targetOp = op rop.handlerLock.Unlock() for _, handler := range rop.handlers { _, _ = rop.targetOp.AddHandler(handler) } err = rop.targetOp.Wait() if err != nil { errors = append(errors, remoteOperationResult{URL: serverURL, Error: err}) if localtls.IsConnectionError(err) { continue } break } success = true break } if success { chConnect <- nil close(chConnect) } else { chConnect <- remoteOperationError("Failed instance creation", errors) close(chConnect) if op != nil { _ = op.Cancel() } } }() if op != nil { go func() { chWait <- op.Wait() close(chWait) }() } go func() { var err error select { case err = <-chConnect: case err = <-chWait: } rop.err = err close(rop.chDone) }() return &rop, nil } // CreateInstanceFromImage is a convenience function to make it easier to create a instance from an existing image. func (r *ProtocolIncus) CreateInstanceFromImage(source ImageServer, image api.Image, req api.InstancesPost) (RemoteOperation, error) { info, err := r.getSourceImageConnectionInfo(source, image, &req.Source) if err != nil { return nil, err } // If the source server is the same as the target server, create the instance directly. if info == nil { op, err := r.CreateInstance(req) if err != nil { return nil, err } rop := remoteOperation{ targetOp: op, chDone: make(chan bool), } // Forward targetOp to remote op go func() { rop.err = rop.targetOp.Wait() close(rop.chDone) }() return &rop, nil } return r.tryCreateInstance(req, info.Addresses, nil) } // CopyInstance copies a instance from a remote server. Additional options can be passed using InstanceCopyArgs. func (r *ProtocolIncus) CopyInstance(source InstanceServer, instance api.Instance, args *InstanceCopyArgs) (RemoteOperation, error) { // Base request req := api.InstancesPost{ Name: instance.Name, InstancePut: instance.Writable(), Type: api.InstanceType(instance.Type), } req.Source.BaseImage = instance.Config["volatile.base_image"] // Process the copy arguments if args != nil { // Quick checks. if args.InstanceOnly { if !r.HasExtension("container_only_migration") { return nil, errors.New("The target server is missing the required \"container_only_migration\" API extension") } if !source.HasExtension("container_only_migration") { return nil, errors.New("The source server is missing the required \"container_only_migration\" API extension") } } if slices.Contains([]string{"push", "relay"}, args.Mode) { if !r.HasExtension("container_push") { return nil, errors.New("The target server is missing the required \"container_push\" API extension") } if !source.HasExtension("container_push") { return nil, errors.New("The source server is missing the required \"container_push\" API extension") } } if args.Mode == "push" && !source.HasExtension("container_push_target") { return nil, errors.New("The source server is missing the required \"container_push_target\" API extension") } if args.Refresh { if !r.HasExtension("container_incremental_copy") { return nil, errors.New("The target server is missing the required \"container_incremental_copy\" API extension") } if !source.HasExtension("container_incremental_copy") { return nil, errors.New("The source server is missing the required \"container_incremental_copy\" API extension") } } if args.RefreshExcludeOlder && !source.HasExtension("custom_volume_refresh_exclude_older_snapshots") { return nil, errors.New("The source server is missing the required \"custom_volume_refresh_exclude_older_snapshots\" API extension") } if args.AllowInconsistent { if !r.HasExtension("instance_allow_inconsistent_copy") { return nil, errors.New("The source server is missing the required \"instance_allow_inconsistent_copy\" API extension") } } // Allow overriding the target name if args.Name != "" { req.Name = args.Name } req.Source.Live = args.Live req.Source.InstanceOnly = args.InstanceOnly req.Source.Refresh = args.Refresh req.Source.RefreshExcludeOlder = args.RefreshExcludeOlder req.Source.AllowInconsistent = args.AllowInconsistent } if req.Source.Live { req.Source.Live = instance.StatusCode == api.Running } sourceInfo, err := source.GetConnectionInfo() if err != nil { return nil, fmt.Errorf("Failed to get source connection info: %w", err) } destInfo, err := r.GetConnectionInfo() if err != nil { return nil, fmt.Errorf("Failed to get destination connection info: %w", err) } // Optimization for the local copy case if destInfo.URL == sourceInfo.URL && destInfo.SocketPath == sourceInfo.SocketPath && (!r.IsClustered() || instance.Location == r.clusterTarget || r.HasExtension("cluster_internal_copy")) { // Project handling if destInfo.Project != sourceInfo.Project { if !r.HasExtension("container_copy_project") { return nil, errors.New("The server is missing the required \"container_copy_project\" API extension") } req.Source.Project = sourceInfo.Project } // Local copy source fields req.Source.Type = "copy" req.Source.Source = instance.Name // Copy the instance op, err := r.CreateInstance(req) if err != nil { return nil, err } rop := remoteOperation{ targetOp: op, chDone: make(chan bool), } // Forward targetOp to remote op go func() { rop.err = rop.targetOp.Wait() close(rop.chDone) }() return &rop, nil } // Source request sourceReq := api.InstancePost{ Migration: true, Live: req.Source.Live, InstanceOnly: req.Source.InstanceOnly, AllowInconsistent: req.Source.AllowInconsistent, Devices: req.Devices, } // Push mode migration if args != nil && args.Mode == "push" { // Get target server connection information info, err := r.GetConnectionInfo() if err != nil { return nil, err } // Create the instance req.Source.Type = "migration" req.Source.Mode = "push" req.Source.Refresh = args.Refresh req.Source.RefreshExcludeOlder = args.RefreshExcludeOlder op, err := r.CreateInstance(req) if err != nil { return nil, err } opAPI := op.Get() targetSecrets := map[string]string{} for k, v := range opAPI.Metadata { val, ok := v.(string) if ok { targetSecrets[k] = val } } // Prepare the source request target := api.InstancePostTarget{} target.Operation = opAPI.ID target.Websockets = targetSecrets target.Certificate = info.Certificate sourceReq.Target = &target return r.tryMigrateInstance(source, instance.Name, sourceReq, info.Addresses, op) } // Get source server connection information info, err := source.GetConnectionInfo() if err != nil { return nil, err } op, err := source.MigrateInstance(instance.Name, sourceReq) if err != nil { return nil, err } opAPI := op.Get() sourceSecrets := map[string]string{} for k, v := range opAPI.Metadata { val, ok := v.(string) if ok { sourceSecrets[k] = val } } // Relay mode migration if args != nil && args.Mode == "relay" { // Push copy source fields req.Source.Type = "migration" req.Source.Mode = "push" // Start the process targetOp, err := r.CreateInstance(req) if err != nil { return nil, err } targetOpAPI := targetOp.Get() // Extract the websockets targetSecrets := map[string]string{} for k, v := range targetOpAPI.Metadata { val, ok := v.(string) if ok { targetSecrets[k] = val } } // Launch the relay err = r.proxyMigration(targetOp.(*operation), targetSecrets, source, op.(*operation), sourceSecrets) if err != nil { return nil, err } // Prepare a tracking operation rop := remoteOperation{ targetOp: targetOp, chDone: make(chan bool), } // Forward targetOp to remote op go func() { rop.err = rop.targetOp.Wait() close(rop.chDone) }() return &rop, nil } // Pull mode migration req.Source.Type = "migration" req.Source.Mode = "pull" req.Source.Operation = opAPI.ID req.Source.Websockets = sourceSecrets req.Source.Certificate = info.Certificate return r.tryCreateInstance(req, info.Addresses, op) } // UpdateInstance updates the instance definition. func (r *ProtocolIncus) UpdateInstance(name string, instance api.InstancePut, ETag string) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } if instance.DiskOnly { err = r.CheckExtension("instance_snapshot_disk_only_restore") if err != nil { return nil, errors.New("The server is missing the required \"instance_snapshot_disk_only_restore\" API extension") } } // Send the request op, _, err := r.queryOperation("PUT", fmt.Sprintf("%s/%s", path, url.PathEscape(name)), instance, ETag) if err != nil { return nil, err } return op, nil } // RenameInstance requests that Incus renames the instance. func (r *ProtocolIncus) RenameInstance(name string, instance api.InstancePost) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } // Quick check. if instance.Migration { return nil, errors.New("Can't ask for a migration through RenameInstance") } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("%s/%s", path, url.PathEscape(name)), instance, "") if err != nil { return nil, err } return op, nil } // tryMigrateInstance attempts to migrate a specific instance from a source server to one of the target URLs. // The function runs the migration operation asynchronously and returns a RemoteOperation to track the progress and handle any errors. func (r *ProtocolIncus) tryMigrateInstance(source InstanceServer, name string, req api.InstancePost, urls []string, op Operation) (RemoteOperation, error) { if len(urls) == 0 { return nil, errors.New("The target server isn't listening on the network") } rop := remoteOperation{ chDone: make(chan bool), } operation := req.Target.Operation // Forward targetOp to remote op chConnect := make(chan error, 1) chWait := make(chan error, 1) go func() { success := false var errors []remoteOperationResult for _, serverURL := range urls { req.Target.Operation = fmt.Sprintf("%s/1.0/operations/%s", serverURL, url.PathEscape(operation)) op, err := source.MigrateInstance(name, req) if err != nil { errors = append(errors, remoteOperationResult{URL: serverURL, Error: err}) continue } rop.targetOp = op for _, handler := range rop.handlers { _, _ = rop.targetOp.AddHandler(handler) } err = rop.targetOp.Wait() if err != nil { errors = append(errors, remoteOperationResult{URL: serverURL, Error: err}) if localtls.IsConnectionError(err) { continue } break } success = true break } if success { chConnect <- nil close(chConnect) } else { chConnect <- remoteOperationError("Failed instance migration", errors) close(chConnect) if op != nil { _ = op.Cancel() } } }() if op != nil { go func() { chWait <- op.Wait() close(chWait) }() } go func() { var err error select { case err = <-chConnect: case err = <-chWait: } rop.err = err close(rop.chDone) }() return &rop, nil } // MigrateInstance requests that Incus prepares for a instance migration. func (r *ProtocolIncus) MigrateInstance(name string, instance api.InstancePost) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } if instance.InstanceOnly { if !r.HasExtension("container_only_migration") { return nil, errors.New("The server is missing the required \"container_only_migration\" API extension") } } if instance.Pool != "" && !r.HasExtension("instance_pool_move") { return nil, errors.New("The server is missing the required \"instance_pool_move\" API extension") } if instance.Project != "" && !r.HasExtension("instance_project_move") { return nil, errors.New("The server is missing the required \"instance_project_move\" API extension") } if instance.AllowInconsistent && !r.HasExtension("cluster_migration_inconsistent_copy") { return nil, errors.New("The server is missing the required \"cluster_migration_inconsistent_copy\" API extension") } // Quick check. if !instance.Migration { return nil, errors.New("Can't ask for a rename through MigrateInstance") } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("%s/%s", path, url.PathEscape(name)), instance, "") if err != nil { return nil, err } return op, nil } // DeleteInstance requests that Incus deletes the instance. func (r *ProtocolIncus) DeleteInstance(name string) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } // Send the request op, _, err := r.queryOperation("DELETE", fmt.Sprintf("%s/%s", path, url.PathEscape(name)), nil, "") if err != nil { return nil, err } return op, nil } // ExecInstance requests that Incus spawns a command inside the instance. func (r *ProtocolIncus) ExecInstance(instanceName string, exec api.InstanceExecPost, args *InstanceExecArgs) (Operation, error) { // Ensure args are equivalent to empty InstanceExecArgs. if args == nil { args = &InstanceExecArgs{} } if exec.RecordOutput { if !r.HasExtension("container_exec_recording") { return nil, errors.New("The server is missing the required \"container_exec_recording\" API extension") } } if exec.User > 0 || exec.Group > 0 || exec.Cwd != "" { if !r.HasExtension("container_exec_user_group_cwd") { return nil, errors.New("The server is missing the required \"container_exec_user_group_cwd\" API extension") } } var uri string if r.IsAgent() { uri = "/exec" } else { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } uri = fmt.Sprintf("%s/%s/exec", path, url.PathEscape(instanceName)) } // Send the request op, _, err := r.queryOperation("POST", uri, exec, "") if err != nil { return nil, err } opAPI := op.Get() // Process additional arguments // Parse the fds fds := map[string]string{} value, ok := opAPI.Metadata["fds"] if ok { values, ok := value.(map[string]any) if ok { for k, v := range values { val, ok := v.(string) if ok { fds[k] = val } } } } if exec.RecordOutput && (args.Stdout != nil || args.Stderr != nil) { err = op.Wait() if err != nil { return nil, err } opAPI = op.Get() outputFiles := map[string]string{} outputs, ok := opAPI.Metadata["output"].(map[string]any) if ok { for k, v := range outputs { val, ok := v.(string) if ok { outputFiles[k] = val } } } if outputFiles["1"] != "" { reader, _ := r.getInstanceExecOutputLogFile(instanceName, filepath.Base(outputFiles["1"])) if args.Stdout != nil { _, errCopy := util.SafeCopy(args.Stdout, reader) // Regardless of errCopy value, we want to delete the file after a copy operation errDelete := r.deleteInstanceExecOutputLogFile(instanceName, filepath.Base(outputFiles["1"])) if errDelete != nil { return nil, errDelete } if errCopy != nil { return nil, fmt.Errorf("Could not copy the content of the exec output log file to stdout: %w", err) } } err = r.deleteInstanceExecOutputLogFile(instanceName, filepath.Base(outputFiles["1"])) if err != nil { return nil, err } } if outputFiles["2"] != "" { reader, _ := r.getInstanceExecOutputLogFile(instanceName, filepath.Base(outputFiles["2"])) if args.Stderr != nil { _, errCopy := util.SafeCopy(args.Stderr, reader) errDelete := r.deleteInstanceExecOutputLogFile(instanceName, filepath.Base(outputFiles["1"])) if errDelete != nil { return nil, errDelete } if errCopy != nil { return nil, fmt.Errorf("Could not copy the content of the exec output log file to stderr: %w", err) } } err = r.deleteInstanceExecOutputLogFile(instanceName, filepath.Base(outputFiles["2"])) if err != nil { return nil, err } } } if fds[api.SecretNameControl] != "" { conn, err := r.GetOperationWebsocket(opAPI.ID, fds[api.SecretNameControl]) if err != nil { return nil, err } go func() { _, _, _ = conn.ReadMessage() // Consume pings from server. }() if args.Control != nil { // Call the control handler with a connection to the control socket go args.Control(conn) } } if exec.Interactive { // Handle interactive sections if args.Stdin != nil && args.Stdout != nil { // Connect to the websocket conn, err := r.GetOperationWebsocket(opAPI.ID, fds["0"]) if err != nil { return nil, err } // And attach stdin and stdout to it go func() { ws.MirrorRead(conn, args.Stdin) <-ws.MirrorWrite(conn, args.Stdout) _ = conn.Close() if args.DataDone != nil { close(args.DataDone) } }() } else { if args.DataDone != nil { close(args.DataDone) } } } else { // Handle non-interactive sessions dones := make(map[int]chan error) conns := []*websocket.Conn{} // Handle stdin if fds["0"] != "" { conn, err := r.GetOperationWebsocket(opAPI.ID, fds["0"]) if err != nil { return nil, err } go func() { _, _, _ = conn.ReadMessage() // Consume pings from server. }() conns = append(conns, conn) dones[0] = ws.MirrorRead(conn, args.Stdin) } waitConns := 0 // Used for keeping track of when stdout and stderr have finished. // Handle stdout if fds["1"] != "" { conn, err := r.GetOperationWebsocket(opAPI.ID, fds["1"]) if err != nil { return nil, err } // Discard Stdout from remote command if output writer not supplied. if args.Stdout == nil { args.Stdout = io.Discard } conns = append(conns, conn) dones[1] = ws.MirrorWrite(conn, args.Stdout) waitConns++ } // Handle stderr if fds["2"] != "" { conn, err := r.GetOperationWebsocket(opAPI.ID, fds["2"]) if err != nil { return nil, err } // Discard Stderr from remote command if output writer not supplied. if args.Stderr == nil { args.Stderr = io.Discard } conns = append(conns, conn) dones[2] = ws.MirrorWrite(conn, args.Stderr) waitConns++ } // Wait for everything to be done go func() { for { select { case <-dones[0]: // Handle stdin finish, but don't wait for it if output channels // have all finished. dones[0] = nil _ = conns[0].Close() case <-dones[1]: dones[1] = nil _ = conns[1].Close() waitConns-- case <-dones[2]: dones[2] = nil _ = conns[2].Close() waitConns-- } if waitConns <= 0 { // Close stdin websocket if defined and not already closed. if dones[0] != nil { conns[0].Close() } break } } if args.DataDone != nil { close(args.DataDone) } }() } return op, nil } // GetInstanceFile retrieves the provided path from the instance. func (r *ProtocolIncus) GetInstanceFile(instanceName string, filePath string) (io.ReadCloser, *InstanceFileResponse, error) { var err error var requestURL string urlEncode := func(path string, query map[string]string) (string, error) { u, err := url.Parse(path) if err != nil { return "", err } params := url.Values{} for key, value := range query { params.Add(key, value) } u.RawQuery = params.Encode() return u.String(), nil } if r.IsAgent() { requestURL, err = urlEncode( fmt.Sprintf("%s/1.0/files", r.httpBaseURL.String()), map[string]string{"path": filePath}) } else { var path string path, _, err = r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, nil, err } // Prepare the HTTP request requestURL, err = urlEncode( fmt.Sprintf("%s/1.0%s/%s/files", r.httpBaseURL.String(), path, url.PathEscape(instanceName)), map[string]string{"path": filePath}) } if err != nil { return nil, nil, err } requestURL, err = r.setQueryAttributes(requestURL) if err != nil { return nil, nil, err } req, err := http.NewRequest("GET", requestURL, nil) if err != nil { return nil, nil, err } // Send the request resp, err := r.DoHTTP(req) if err != nil { return nil, nil, err } // Check the return value for a cleaner error if resp.StatusCode != http.StatusOK { _, _, err := incusParseResponse(resp) if err != nil { return nil, nil, err } } // Parse the headers uid, gid, mode, fileType, _ := api.ParseFileHeaders(resp.Header) fileResp := InstanceFileResponse{ UID: uid, GID: gid, Mode: mode, Type: fileType, } if fileResp.Type == "directory" { // Decode the response response := api.Response{} decoder := json.NewDecoder(resp.Body) err = decoder.Decode(&response) if err != nil { return nil, nil, err } // Get the file list entries := []string{} err = response.MetadataAsStruct(&entries) if err != nil { return nil, nil, err } fileResp.Entries = entries return nil, &fileResp, err } return resp.Body, &fileResp, err } // CreateInstanceFile tells Incus to create a file in the instance. func (r *ProtocolIncus) CreateInstanceFile(instanceName string, filePath string, args InstanceFileArgs) error { if args.Type == "directory" { if !r.HasExtension("directory_manipulation") { return errors.New("The server is missing the required \"directory_manipulation\" API extension") } } if args.Type == "symlink" { if !r.HasExtension("file_symlinks") { return errors.New("The server is missing the required \"file_symlinks\" API extension") } } if args.WriteMode == "append" { if !r.HasExtension("file_append") { return errors.New("The server is missing the required \"file_append\" API extension") } } var requestURL string if r.IsAgent() { requestURL = fmt.Sprintf("%s/1.0/files?path=%s", r.httpBaseURL.String(), url.QueryEscape(filePath)) } else { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return err } // Prepare the HTTP request requestURL = fmt.Sprintf("%s/1.0%s/%s/files?path=%s", r.httpBaseURL.String(), path, url.PathEscape(instanceName), url.QueryEscape(filePath)) } requestURL, err := r.setQueryAttributes(requestURL) if err != nil { return err } req, err := http.NewRequest("POST", requestURL, args.Content) if err != nil { return err } req.GetBody = func() (io.ReadCloser, error) { _, err := args.Content.Seek(0, 0) if err != nil { return nil, err } return io.NopCloser(args.Content), nil } // Set the various headers if args.UID > -1 { req.Header.Set("X-Incus-uid", fmt.Sprintf("%d", args.UID)) } if args.GID > -1 { req.Header.Set("X-Incus-gid", fmt.Sprintf("%d", args.GID)) } if args.Mode > -1 { req.Header.Set("X-Incus-mode", fmt.Sprintf("%04o", args.Mode)) } if args.Type != "" { req.Header.Set("X-Incus-type", args.Type) } if args.WriteMode != "" { req.Header.Set("X-Incus-write", args.WriteMode) } // Send the request resp, err := r.DoHTTP(req) if err != nil { return err } // Check the return value for a cleaner error _, _, err = incusParseResponse(resp) if err != nil { return err } return nil } // DeleteInstanceFile deletes a file in the instance. func (r *ProtocolIncus) DeleteInstanceFile(instanceName string, filePath string) error { if !r.HasExtension("file_delete") { return errors.New("The server is missing the required \"file_delete\" API extension") } var requestURL string if r.IsAgent() { requestURL = fmt.Sprintf("/files?path=%s", url.QueryEscape(filePath)) } else { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return err } // Prepare the HTTP request requestURL = fmt.Sprintf("%s/%s/files?path=%s", path, url.PathEscape(instanceName), url.QueryEscape(filePath)) } requestURL, err := r.setQueryAttributes(requestURL) if err != nil { return err } // Send the request _, _, err = r.query("DELETE", requestURL, nil, "") if err != nil { return err } return nil } // rawConn connects to the apiURL, upgrades to the requested protocol and returns it. func (r *ProtocolIncus) rawConn(apiURL *url.URL, protocol string) (net.Conn, error) { // Get the HTTP transport. httpTransport, err := r.getUnderlyingHTTPTransport() if err != nil { return nil, err } req := &http.Request{ Method: http.MethodGet, URL: apiURL, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, Header: make(http.Header), Host: apiURL.Host, } req.Header["Upgrade"] = []string{protocol} req.Header["Connection"] = []string{"Upgrade"} r.addClientHeaders(req) // Establish the connection. var conn net.Conn if httpTransport.TLSClientConfig != nil { conn, err = httpTransport.DialTLSContext(context.Background(), "tcp", apiURL.Host) } else { conn, err = httpTransport.DialContext(context.Background(), "tcp", apiURL.Host) } if err != nil { return nil, err } remoteTCP, _ := tcp.ExtractConn(conn) if remoteTCP != nil { err = tcp.SetTimeouts(remoteTCP, 0) if err != nil { return nil, err } } err = req.Write(conn) if err != nil { return nil, err } resp, err := http.ReadResponse(bufio.NewReader(conn), req) if err != nil { return nil, err } if resp.StatusCode != http.StatusSwitchingProtocols { _, _, err := incusParseResponse(resp) if err != nil { return nil, err } } if resp.Header.Get("Upgrade") != protocol { return nil, errors.New("Missing or unexpected Upgrade header in response") } return conn, nil } // GetInstanceFileSFTPConn returns a connection to the instance's SFTP endpoint. func (r *ProtocolIncus) GetInstanceFileSFTPConn(instanceName string) (net.Conn, error) { apiURL := api.NewURL() apiURL.URL = r.httpBaseURL // Preload the URL with the client base URL. apiURL.Path("1.0", "instances", instanceName, "sftp") r.setURLQueryAttributes(&apiURL.URL) return r.rawConn(&apiURL.URL, "sftp") } // GetInstanceFileSFTP returns an SFTP connection to the instance. func (r *ProtocolIncus) GetInstanceFileSFTP(instanceName string) (*sftp.Client, error) { conn, err := r.GetInstanceFileSFTPConn(instanceName) if err != nil { return nil, err } // Get a SFTP client. client, err := sftp.NewClientPipe(conn, conn, sftp.MaxPacketUnchecked(128*1024)) if err != nil { _ = conn.Close() return nil, err } go func() { // Wait for the client to be done before closing the connection. _ = client.Wait() _ = conn.Close() }() return client, nil } // GetInstanceSnapshotNames returns a list of snapshot names for the instance. func (r *ProtocolIncus) GetInstanceSnapshotNames(instanceName string) ([]string, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } // Fetch the raw URL values. urls := []string{} baseURL := fmt.Sprintf("%s/%s/snapshots", path, url.PathEscape(instanceName)) _, err = r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetInstanceSnapshots returns a list of snapshots for the instance. func (r *ProtocolIncus) GetInstanceSnapshots(instanceName string) ([]api.InstanceSnapshot, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } snapshots := []api.InstanceSnapshot{} // Fetch the raw value _, err = r.queryStruct("GET", fmt.Sprintf("%s/%s/snapshots?recursion=1", path, url.PathEscape(instanceName)), nil, "", &snapshots) if err != nil { return nil, err } return snapshots, nil } // GetInstanceSnapshot returns a Snapshot struct for the provided instance and snapshot names. func (r *ProtocolIncus) GetInstanceSnapshot(instanceName string, name string) (*api.InstanceSnapshot, string, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, "", err } snapshot := api.InstanceSnapshot{} // Fetch the raw value etag, err := r.queryStruct("GET", fmt.Sprintf("%s/%s/snapshots/%s", path, url.PathEscape(instanceName), url.PathEscape(name)), nil, "", &snapshot) if err != nil { return nil, "", err } return &snapshot, etag, nil } // CreateInstanceSnapshot requests that Incus creates a new snapshot for the instance. func (r *ProtocolIncus) CreateInstanceSnapshot(instanceName string, snapshot api.InstanceSnapshotsPost) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } // Validate the request if snapshot.ExpiresAt != nil && !r.HasExtension("snapshot_expiry_creation") { return nil, errors.New("The server is missing the required \"snapshot_expiry_creation\" API extension") } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("%s/%s/snapshots", path, url.PathEscape(instanceName)), snapshot, "") if err != nil { return nil, err } return op, nil } // CopyInstanceSnapshot copies a snapshot from a remote server into a new instance. Additional options can be passed using InstanceCopyArgs. func (r *ProtocolIncus) CopyInstanceSnapshot(source InstanceServer, instanceName string, snapshot api.InstanceSnapshot, args *InstanceSnapshotCopyArgs) (RemoteOperation, error) { // Backward compatibility (with broken Name field) fields := strings.Split(snapshot.Name, "/") cName := instanceName sName := fields[len(fields)-1] // Base request req := api.InstancesPost{ Name: cName, InstancePut: api.InstancePut{ Architecture: snapshot.Architecture, Config: snapshot.Config, Devices: snapshot.Devices, Ephemeral: snapshot.Ephemeral, Profiles: snapshot.Profiles, }, } if snapshot.Stateful && args.Live { if !r.HasExtension("container_snapshot_stateful_migration") { return nil, errors.New("The server is missing the required \"container_snapshot_stateful_migration\" API extension") } req.Stateful = snapshot.Stateful req.Source.Live = false // Snapshots are never running and so we don't need live migration. } req.Source.BaseImage = snapshot.Config["volatile.base_image"] // Process the copy arguments if args != nil { // Quick checks. if slices.Contains([]string{"push", "relay"}, args.Mode) { if !r.HasExtension("container_push") { return nil, errors.New("The target server is missing the required \"container_push\" API extension") } if !source.HasExtension("container_push") { return nil, errors.New("The source server is missing the required \"container_push\" API extension") } } if args.Mode == "push" && !source.HasExtension("container_push_target") { return nil, errors.New("The source server is missing the required \"container_push_target\" API extension") } // Allow overriding the target name if args.Name != "" { req.Name = args.Name } } sourceInfo, err := source.GetConnectionInfo() if err != nil { return nil, fmt.Errorf("Failed to get source connection info: %w", err) } destInfo, err := r.GetConnectionInfo() if err != nil { return nil, fmt.Errorf("Failed to get destination connection info: %w", err) } instance, _, err := source.GetInstance(cName) if err != nil { return nil, fmt.Errorf("Failed to get instance info: %w", err) } // Optimization for the local copy case if destInfo.URL == sourceInfo.URL && destInfo.SocketPath == sourceInfo.SocketPath && (!r.IsClustered() || instance.Location == r.clusterTarget || r.HasExtension("cluster_internal_copy")) { // Project handling if destInfo.Project != sourceInfo.Project { if !r.HasExtension("container_copy_project") { return nil, errors.New("The server is missing the required \"container_copy_project\" API extension") } req.Source.Project = sourceInfo.Project } // Local copy source fields req.Source.Type = "copy" req.Source.Source = fmt.Sprintf("%s/%s", cName, sName) // Copy the instance op, err := r.CreateInstance(req) if err != nil { return nil, err } rop := remoteOperation{ targetOp: op, chDone: make(chan bool), } // Forward targetOp to remote op go func() { rop.err = rop.targetOp.Wait() close(rop.chDone) }() return &rop, nil } // If deadling with migration, we need to set the type. if source.HasExtension("virtual-machines") { inst, _, err := source.GetInstance(instanceName) if err != nil { return nil, err } req.Type = api.InstanceType(inst.Type) } // Source request sourceReq := api.InstanceSnapshotPost{ Migration: true, Name: args.Name, } if snapshot.Stateful && args.Live { sourceReq.Live = args.Live } // Push mode migration if args != nil && args.Mode == "push" { // Get target server connection information info, err := r.GetConnectionInfo() if err != nil { return nil, err } // Create the instance req.Source.Type = "migration" req.Source.Mode = "push" op, err := r.CreateInstance(req) if err != nil { return nil, err } opAPI := op.Get() targetSecrets := map[string]string{} for k, v := range opAPI.Metadata { val, ok := v.(string) if ok { targetSecrets[k] = val } } // Prepare the source request target := api.InstancePostTarget{} target.Operation = opAPI.ID target.Websockets = targetSecrets target.Certificate = info.Certificate sourceReq.Target = &target return r.tryMigrateInstanceSnapshot(source, cName, sName, sourceReq, info.Addresses) } // Get source server connection information info, err := source.GetConnectionInfo() if err != nil { return nil, err } op, err := source.MigrateInstanceSnapshot(cName, sName, sourceReq) if err != nil { return nil, err } opAPI := op.Get() sourceSecrets := map[string]string{} for k, v := range opAPI.Metadata { val, ok := v.(string) if ok { sourceSecrets[k] = val } } // Relay mode migration if args != nil && args.Mode == "relay" { // Push copy source fields req.Source.Type = "migration" req.Source.Mode = "push" // Start the process targetOp, err := r.CreateInstance(req) if err != nil { return nil, err } targetOpAPI := targetOp.Get() // Extract the websockets targetSecrets := map[string]string{} for k, v := range targetOpAPI.Metadata { val, ok := v.(string) if ok { targetSecrets[k] = val } } // Launch the relay err = r.proxyMigration(targetOp.(*operation), targetSecrets, source, op.(*operation), sourceSecrets) if err != nil { return nil, err } // Prepare a tracking operation rop := remoteOperation{ targetOp: targetOp, chDone: make(chan bool), } // Forward targetOp to remote op go func() { rop.err = rop.targetOp.Wait() close(rop.chDone) }() return &rop, nil } // Pull mode migration req.Source.Type = "migration" req.Source.Mode = "pull" req.Source.Operation = opAPI.ID req.Source.Websockets = sourceSecrets req.Source.Certificate = info.Certificate return r.tryCreateInstance(req, info.Addresses, op) } // RenameInstanceSnapshot requests that Incus renames the snapshot. func (r *ProtocolIncus) RenameInstanceSnapshot(instanceName string, name string, instance api.InstanceSnapshotPost) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } // Quick check. if instance.Migration { return nil, errors.New("Can't ask for a migration through RenameInstanceSnapshot") } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("%s/%s/snapshots/%s", path, url.PathEscape(instanceName), url.PathEscape(name)), instance, "") if err != nil { return nil, err } return op, nil } func (r *ProtocolIncus) tryMigrateInstanceSnapshot(source InstanceServer, instanceName string, name string, req api.InstanceSnapshotPost, urls []string) (RemoteOperation, error) { if len(urls) == 0 { return nil, errors.New("The target server isn't listening on the network") } rop := remoteOperation{ chDone: make(chan bool), } operation := req.Target.Operation // Forward targetOp to remote op go func() { success := false var errors []remoteOperationResult for _, serverURL := range urls { req.Target.Operation = fmt.Sprintf("%s/1.0/operations/%s", serverURL, url.PathEscape(operation)) op, err := source.MigrateInstanceSnapshot(instanceName, name, req) if err != nil { errors = append(errors, remoteOperationResult{URL: serverURL, Error: err}) continue } rop.targetOp = op for _, handler := range rop.handlers { _, _ = rop.targetOp.AddHandler(handler) } err = rop.targetOp.Wait() if err != nil { errors = append(errors, remoteOperationResult{URL: serverURL, Error: err}) if localtls.IsConnectionError(err) { continue } break } success = true break } if !success { rop.err = remoteOperationError("Failed instance migration", errors) } close(rop.chDone) }() return &rop, nil } // MigrateInstanceSnapshot requests that Incus prepares for a snapshot migration. func (r *ProtocolIncus) MigrateInstanceSnapshot(instanceName string, name string, instance api.InstanceSnapshotPost) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } // Quick check. if !instance.Migration { return nil, errors.New("Can't ask for a rename through MigrateInstanceSnapshot") } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("%s/%s/snapshots/%s", path, url.PathEscape(instanceName), url.PathEscape(name)), instance, "") if err != nil { return nil, err } return op, nil } // DeleteInstanceSnapshot requests that Incus deletes the instance snapshot. func (r *ProtocolIncus) DeleteInstanceSnapshot(instanceName string, name string) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } // Send the request op, _, err := r.queryOperation("DELETE", fmt.Sprintf("%s/%s/snapshots/%s", path, url.PathEscape(instanceName), url.PathEscape(name)), nil, "") if err != nil { return nil, err } return op, nil } // UpdateInstanceSnapshot requests that Incus updates the instance snapshot. func (r *ProtocolIncus) UpdateInstanceSnapshot(instanceName string, name string, instance api.InstanceSnapshotPut, ETag string) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } if !r.HasExtension("snapshot_expiry") { return nil, errors.New("The server is missing the required \"snapshot_expiry\" API extension") } // Send the request op, _, err := r.queryOperation("PUT", fmt.Sprintf("%s/%s/snapshots/%s", path, url.PathEscape(instanceName), url.PathEscape(name)), instance, ETag) if err != nil { return nil, err } return op, nil } // GetInstanceState returns a InstanceState entry for the provided instance name. func (r *ProtocolIncus) GetInstanceState(name string) (*api.InstanceState, string, error) { var uri string if r.IsAgent() { uri = "/state" } else { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, "", err } uri = fmt.Sprintf("%s/%s/state", path, url.PathEscape(name)) } state := api.InstanceState{} // Fetch the raw value etag, err := r.queryStruct("GET", uri, nil, "", &state) if err != nil { return nil, "", err } return &state, etag, nil } // UpdateInstanceState updates the instance to match the requested state. func (r *ProtocolIncus) UpdateInstanceState(name string, state api.InstanceStatePut, ETag string) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } // Send the request op, _, err := r.queryOperation("PUT", fmt.Sprintf("%s/%s/state", path, url.PathEscape(name)), state, ETag) if err != nil { return nil, err } return op, nil } // GetInstanceAccess returns an Access entry for the provided instance name. func (r *ProtocolIncus) GetInstanceAccess(name string) (api.Access, error) { access := api.Access{} if !r.HasExtension("instance_access") { return nil, errors.New("The server is missing the required \"instance_access\" API extension") } // Fetch the raw value _, err := r.queryStruct("GET", fmt.Sprintf("/instances/%s/access", url.PathEscape(name)), nil, "", &access) if err != nil { return nil, err } return access, nil } // GetInstanceLogfiles returns a list of logfiles for the instance. func (r *ProtocolIncus) GetInstanceLogfiles(name string) ([]string, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } // Fetch the raw URL values. urls := []string{} baseURL := fmt.Sprintf("%s/%s/logs", path, url.PathEscape(name)) _, err = r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetInstanceLogfile returns the content of the requested logfile. // // Note that it's the caller's responsibility to close the returned ReadCloser. func (r *ProtocolIncus) GetInstanceLogfile(name string, filename string) (io.ReadCloser, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } // Prepare the HTTP request uri := fmt.Sprintf("%s/1.0%s/%s/logs/%s", r.httpBaseURL.String(), path, url.PathEscape(name), url.PathEscape(filename)) uri, err = r.setQueryAttributes(uri) if err != nil { return nil, err } req, err := http.NewRequest("GET", uri, nil) if err != nil { return nil, err } // Send the request resp, err := r.DoHTTP(req) if err != nil { return nil, err } // Check the return value for a cleaner error if resp.StatusCode != http.StatusOK { _, _, err := incusParseResponse(resp) if err != nil { return nil, err } } return resp.Body, err } // DeleteInstanceLogfile deletes the requested logfile. func (r *ProtocolIncus) DeleteInstanceLogfile(name string, filename string) error { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return err } // Send the request _, _, err = r.query("DELETE", fmt.Sprintf("%s/%s/logs/%s", path, url.PathEscape(name), url.PathEscape(filename)), nil, "") if err != nil { return err } return nil } // getInstanceExecOutputLogFile returns the content of the requested exec logfile. // // Note that it's the caller's responsibility to close the returned ReadCloser. func (r *ProtocolIncus) getInstanceExecOutputLogFile(name string, filename string) (io.ReadCloser, error) { err := r.CheckExtension("container_exec_recording") if err != nil { return nil, err } path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } // Prepare the HTTP request uri := fmt.Sprintf("%s/1.0%s/%s/logs/exec-output/%s", r.httpBaseURL.String(), path, url.PathEscape(name), url.PathEscape(filename)) uri, err = r.setQueryAttributes(uri) if err != nil { return nil, err } req, err := http.NewRequest("GET", uri, nil) if err != nil { return nil, err } // Send the request resp, err := r.DoHTTP(req) if err != nil { return nil, err } // Check the return value for a cleaner error if resp.StatusCode != http.StatusOK { _, _, err := incusParseResponse(resp) if err != nil { return nil, err } } return resp.Body, nil } // deleteInstanceExecOutputLogFiles deletes the requested exec logfile. func (r *ProtocolIncus) deleteInstanceExecOutputLogFile(instanceName string, filename string) error { err := r.CheckExtension("container_exec_recording") if err != nil { return err } path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return err } // Send the request _, _, err = r.query("DELETE", fmt.Sprintf("%s/%s/logs/exec-output/%s", path, url.PathEscape(instanceName), url.PathEscape(filename)), nil, "") if err != nil { return err } return nil } // GetInstanceMetadata returns instance metadata. func (r *ProtocolIncus) GetInstanceMetadata(name string) (*api.ImageMetadata, string, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, "", err } if !r.HasExtension("container_edit_metadata") { return nil, "", errors.New("The server is missing the required \"container_edit_metadata\" API extension") } metadata := api.ImageMetadata{} uri := fmt.Sprintf("%s/%s/metadata", path, url.PathEscape(name)) etag, err := r.queryStruct("GET", uri, nil, "", &metadata) if err != nil { return nil, "", err } return &metadata, etag, err } // UpdateInstanceMetadata sets the content of the instance metadata file. func (r *ProtocolIncus) UpdateInstanceMetadata(name string, metadata api.ImageMetadata, ETag string) error { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return err } if !r.HasExtension("container_edit_metadata") { return errors.New("The server is missing the required \"container_edit_metadata\" API extension") } uri := fmt.Sprintf("%s/%s/metadata", path, url.PathEscape(name)) _, _, err = r.query("PUT", uri, metadata, ETag) if err != nil { return err } return nil } // GetInstanceTemplateFiles returns the list of names of template files for a instance. func (r *ProtocolIncus) GetInstanceTemplateFiles(instanceName string) ([]string, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } if !r.HasExtension("container_edit_metadata") { return nil, errors.New("The server is missing the required \"container_edit_metadata\" API extension") } templates := []string{} uri := fmt.Sprintf("%s/%s/metadata/templates", path, url.PathEscape(instanceName)) _, err = r.queryStruct("GET", uri, nil, "", &templates) if err != nil { return nil, err } return templates, nil } // GetInstanceTemplateFile returns the content of a template file for a instance. func (r *ProtocolIncus) GetInstanceTemplateFile(instanceName string, templateName string) (io.ReadCloser, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } if !r.HasExtension("container_edit_metadata") { return nil, errors.New("The server is missing the required \"container_edit_metadata\" API extension") } uri := fmt.Sprintf("%s/1.0%s/%s/metadata/templates?path=%s", r.httpBaseURL.String(), path, url.PathEscape(instanceName), url.QueryEscape(templateName)) uri, err = r.setQueryAttributes(uri) if err != nil { return nil, err } req, err := http.NewRequest("GET", uri, nil) if err != nil { return nil, err } // Send the request resp, err := r.DoHTTP(req) if err != nil { return nil, err } // Check the return value for a cleaner error if resp.StatusCode != http.StatusOK { _, _, err := incusParseResponse(resp) if err != nil { return nil, err } } return resp.Body, err } // CreateInstanceTemplateFile creates an a template for a instance. func (r *ProtocolIncus) CreateInstanceTemplateFile(instanceName string, templateName string, content io.ReadSeeker) error { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return err } if !r.HasExtension("container_edit_metadata") { return errors.New("The server is missing the required \"container_edit_metadata\" API extension") } uri := fmt.Sprintf("%s/1.0%s/%s/metadata/templates?path=%s", r.httpBaseURL.String(), path, url.PathEscape(instanceName), url.QueryEscape(templateName)) uri, err = r.setQueryAttributes(uri) if err != nil { return err } req, err := http.NewRequest("POST", uri, content) if err != nil { return err } req.GetBody = func() (io.ReadCloser, error) { _, err := content.Seek(0, 0) if err != nil { return nil, err } return io.NopCloser(content), nil } req.Header.Set("Content-Type", "application/octet-stream") // Send the request resp, err := r.DoHTTP(req) // Check the return value for a cleaner error if resp.StatusCode != http.StatusOK { _, _, err := incusParseResponse(resp) if err != nil { return err } } return err } // DeleteInstanceTemplateFile deletes a template file for a instance. func (r *ProtocolIncus) DeleteInstanceTemplateFile(name string, templateName string) error { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return err } if !r.HasExtension("container_edit_metadata") { return errors.New("The server is missing the required \"container_edit_metadata\" API extension") } _, _, err = r.query("DELETE", fmt.Sprintf("%s/%s/metadata/templates?path=%s", path, url.PathEscape(name), url.QueryEscape(templateName)), nil, "") return err } // ConsoleInstance requests that Incus attaches to the console device of a instance. func (r *ProtocolIncus) ConsoleInstance(instanceName string, console api.InstanceConsolePost, args *InstanceConsoleArgs) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } if !r.HasExtension("console") { return nil, errors.New("The server is missing the required \"console\" API extension") } if console.Type == "" { console.Type = "console" } if console.Type == "vga" && !r.HasExtension("console_vga_type") { return nil, errors.New("The server is missing the required \"console_vga_type\" API extension") } if console.Force && !r.HasExtension("console_force") { return nil, errors.New(`The server is missing the required "console_force" API extension`) } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("%s/%s/console", path, url.PathEscape(instanceName)), console, "") if err != nil { return nil, err } opAPI := op.Get() if args == nil || args.Terminal == nil { return nil, errors.New("A terminal must be set") } if args.Control == nil { return nil, errors.New("A control channel must be set") } // Parse the fds fds := map[string]string{} value, ok := opAPI.Metadata["fds"] if ok { values, ok := value.(map[string]any) if ok { for k, v := range values { val, ok := v.(string) if ok { fds[k] = val } } } } var controlConn *websocket.Conn // Call the control handler with a connection to the control socket if fds[api.SecretNameControl] == "" { return nil, errors.New("Did not receive a file descriptor for the control channel") } controlConn, err = r.GetOperationWebsocket(opAPI.ID, fds[api.SecretNameControl]) if err != nil { return nil, err } go args.Control(controlConn) // Connect to the websocket conn, err := r.GetOperationWebsocket(opAPI.ID, fds["0"]) if err != nil { return nil, err } // Detach from console. go func(consoleDisconnect <-chan bool) { <-consoleDisconnect msg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "Detaching from console") // We don't care if this fails. This is just for convenience. _ = controlConn.WriteMessage(websocket.CloseMessage, msg) _ = controlConn.Close() }(args.ConsoleDisconnect) // And attach stdin and stdout to it go func() { _, writeDone := ws.Mirror(conn, args.Terminal) <-writeDone _ = conn.Close() }() return op, nil } // ConsoleInstanceDynamic requests that Incus attaches to the console device of a // instance with the possibility of opening multiple connections to it. // // Every time the returned 'console' function is called, a new connection will // be established and proxied to the given io.ReadWriteCloser. func (r *ProtocolIncus) ConsoleInstanceDynamic(instanceName string, console api.InstanceConsolePost, args *InstanceConsoleArgs) (Operation, func(io.ReadWriteCloser) error, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, nil, err } if !r.HasExtension("console") { return nil, nil, errors.New("The server is missing the required \"console\" API extension") } if console.Type == "" { console.Type = "console" } if console.Type == "vga" && !r.HasExtension("console_vga_type") { return nil, nil, errors.New("The server is missing the required \"console_vga_type\" API extension") } if console.Force && !r.HasExtension("console_force") { return nil, nil, errors.New(`The server is missing the required "console_force" API extension`) } // Send the request. op, _, err := r.queryOperation("POST", fmt.Sprintf("%s/%s/console", path, url.PathEscape(instanceName)), console, "") if err != nil { return nil, nil, err } opAPI := op.Get() if args == nil { return nil, nil, errors.New("No arguments provided") } if args.Control == nil { return nil, nil, errors.New("A control channel must be set") } // Parse the fds. fds := map[string]string{} value, ok := opAPI.Metadata["fds"] if ok { values, ok := value.(map[string]any) if ok { for k, v := range values { val, ok := v.(string) if ok { fds[k] = val } } } } // Call the control handler with a connection to the control socket. if fds[api.SecretNameControl] == "" { return nil, nil, errors.New("Did not receive a file descriptor for the control channel") } controlConn, err := r.GetOperationWebsocket(opAPI.ID, fds[api.SecretNameControl]) if err != nil { return nil, nil, err } go args.Control(controlConn) // Handle main disconnect. go func(consoleDisconnect <-chan bool) { <-consoleDisconnect msg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "Detaching from console") // We don't care if this fails. This is just for convenience. _ = controlConn.WriteMessage(websocket.CloseMessage, msg) _ = controlConn.Close() }(args.ConsoleDisconnect) f := func(rwc io.ReadWriteCloser) error { // Connect to the websocket. conn, err := r.GetOperationWebsocket(opAPI.ID, fds["0"]) if err != nil { return err } // Attach reader/writer. _, writeDone := ws.Mirror(conn, rwc) <-writeDone _ = conn.Close() return nil } return op, f, nil } // GetInstanceConsoleLog requests that Incus attaches to the console device of a instance. // // Note that it's the caller's responsibility to close the returned ReadCloser. func (r *ProtocolIncus) GetInstanceConsoleLog(instanceName string, _ *InstanceConsoleLogArgs) (io.ReadCloser, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } if !r.HasExtension("console") { return nil, errors.New("The server is missing the required \"console\" API extension") } // Prepare the HTTP request uri := fmt.Sprintf("%s/1.0%s/%s/console", r.httpBaseURL.String(), path, url.PathEscape(instanceName)) uri, err = r.setQueryAttributes(uri) if err != nil { return nil, err } req, err := http.NewRequest("GET", uri, nil) if err != nil { return nil, err } // Send the request resp, err := r.DoHTTP(req) if err != nil { return nil, err } // Check the return value for a cleaner error if resp.StatusCode != http.StatusOK { _, _, err := incusParseResponse(resp) if err != nil { return nil, err } } return resp.Body, err } // DeleteInstanceConsoleLog deletes the requested instance's console log. func (r *ProtocolIncus) DeleteInstanceConsoleLog(instanceName string, _ *InstanceConsoleLogArgs) error { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return err } if !r.HasExtension("console") { return errors.New("The server is missing the required \"console\" API extension") } // Send the request _, _, err = r.query("DELETE", fmt.Sprintf("%s/%s/console", path, url.PathEscape(instanceName)), nil, "") if err != nil { return err } return nil } // GetInstanceBackupNames returns a list of backup names for the instance. func (r *ProtocolIncus) GetInstanceBackupNames(instanceName string) ([]string, error) { if !r.HasExtension("container_backup") { return nil, errors.New("The server is missing the required \"container_backup\" API extension") } path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } // Fetch the raw URL values. urls := []string{} baseURL := fmt.Sprintf("%s/%s/backups", path, url.PathEscape(instanceName)) _, err = r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetInstanceBackups returns a list of backups for the instance. func (r *ProtocolIncus) GetInstanceBackups(instanceName string) ([]api.InstanceBackup, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } if !r.HasExtension("container_backup") { return nil, errors.New("The server is missing the required \"container_backup\" API extension") } // Fetch the raw value backups := []api.InstanceBackup{} _, err = r.queryStruct("GET", fmt.Sprintf("%s/%s/backups?recursion=1", path, url.PathEscape(instanceName)), nil, "", &backups) if err != nil { return nil, err } return backups, nil } // GetInstanceBackup returns a Backup struct for the provided instance and backup names. func (r *ProtocolIncus) GetInstanceBackup(instanceName string, name string) (*api.InstanceBackup, string, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, "", err } if !r.HasExtension("container_backup") { return nil, "", errors.New("The server is missing the required \"container_backup\" API extension") } // Fetch the raw value backup := api.InstanceBackup{} etag, err := r.queryStruct("GET", fmt.Sprintf("%s/%s/backups/%s", path, url.PathEscape(instanceName), url.PathEscape(name)), nil, "", &backup) if err != nil { return nil, "", err } return &backup, etag, nil } // CreateInstanceBackup requests that Incus creates a new backup for the instance. func (r *ProtocolIncus) CreateInstanceBackup(instanceName string, backup api.InstanceBackupsPost) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } if !r.HasExtension("container_backup") { return nil, errors.New("The server is missing the required \"container_backup\" API extension") } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("%s/%s/backups", path, url.PathEscape(instanceName)), backup, "") if err != nil { return nil, err } return op, nil } // RenameInstanceBackup requests that Incus renames the backup. func (r *ProtocolIncus) RenameInstanceBackup(instanceName string, name string, backup api.InstanceBackupPost) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } if !r.HasExtension("container_backup") { return nil, errors.New("The server is missing the required \"container_backup\" API extension") } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("%s/%s/backups/%s", path, url.PathEscape(instanceName), url.PathEscape(name)), backup, "") if err != nil { return nil, err } return op, nil } // DeleteInstanceBackup requests that Incus deletes the instance backup. func (r *ProtocolIncus) DeleteInstanceBackup(instanceName string, name string) (Operation, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } if !r.HasExtension("container_backup") { return nil, errors.New("The server is missing the required \"container_backup\" API extension") } // Send the request op, _, err := r.queryOperation("DELETE", fmt.Sprintf("%s/%s/backups/%s", path, url.PathEscape(instanceName), url.PathEscape(name)), nil, "") if err != nil { return nil, err } return op, nil } // GetInstanceBackupFile requests the instance backup content. func (r *ProtocolIncus) GetInstanceBackupFile(instanceName string, name string, req *BackupFileRequest) (*BackupFileResponse, error) { path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return nil, err } if !r.HasExtension("container_backup") { return nil, errors.New("The server is missing the required \"container_backup\" API extension") } // Build the URL uri := fmt.Sprintf("%s/1.0%s/%s/backups/%s/export", r.httpBaseURL.String(), path, url.PathEscape(instanceName), url.PathEscape(name)) if r.project != "" { uri += fmt.Sprintf("?project=%s", url.QueryEscape(r.project)) } // Prepare the download request request, err := http.NewRequest("GET", uri, nil) if err != nil { return nil, err } if r.httpUserAgent != "" { request.Header.Set("User-Agent", r.httpUserAgent) } // Start the request response, doneCh, err := cancel.CancelableDownload(req.Canceler, r.DoHTTP, request) if err != nil { return nil, err } defer func() { _ = response.Body.Close() }() defer close(doneCh) if response.StatusCode != http.StatusOK { _, _, err := incusParseResponse(response) if err != nil { return nil, err } } // Handle the data body := response.Body if req.ProgressHandler != nil { body = &ioprogress.ProgressReader{ ReadCloser: response.Body, Tracker: &ioprogress.ProgressTracker{ Length: response.ContentLength, Handler: func(percent int64, speed int64) { req.ProgressHandler(ioprogress.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, units.GetByteSizeString(speed, 2))}) }, }, } } size, err := util.SafeCopy(req.BackupFile, body) if err != nil { return nil, err } resp := BackupFileResponse{} resp.Size = size return &resp, nil } // CreateInstanceBackupStream requests that Incus creates and returns new direct backup for the // instance. func (r *ProtocolIncus) CreateInstanceBackupStream(instanceName string, backup api.InstanceBackupsPost, req *BackupFileRequest) error { if !r.HasExtension("direct_backup") { return errors.New("The server is missing the required \"direct_backup\" API extension") } path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return err } // Build the URL uri := fmt.Sprintf("%s/1.0%s/%s/backups", r.httpBaseURL.String(), path, url.PathEscape(instanceName)) if r.project != "" { uri += fmt.Sprintf("?project=%s", url.QueryEscape(r.project)) } // Encode the backup data buf := bytes.Buffer{} err = json.NewEncoder(&buf).Encode(backup) if err != nil { return err } // Prepare the download request request, err := http.NewRequest("POST", uri, bytes.NewReader(buf.Bytes())) if err != nil { return err } request.Header.Set("Accept", "application/octet-stream") if r.httpUserAgent != "" { request.Header.Set("User-Agent", r.httpUserAgent) } // Start the request response, doneCh, err := cancel.CancelableDownload(req.Canceler, r.DoHTTP, request) if err != nil { return err } defer func() { _ = response.Body.Close() }() defer close(doneCh) if response.StatusCode != http.StatusOK { _, _, err = incusParseResponse(response) if err != nil { return err } } // Handle the data body := response.Body if req.ProgressHandler != nil { body = &ioprogress.ProgressReader{ ReadCloser: response.Body, Tracker: &ioprogress.ProgressTracker{ Handler: func(received int64, speed int64) { req.ProgressHandler(ioprogress.ProgressData{Text: fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(received, 2), units.GetByteSizeString(speed, 2))}) }, }, } } _, err = util.SafeCopy(req.BackupFile, body) return err } func (r *ProtocolIncus) proxyMigration(targetOp *operation, targetSecrets map[string]string, source InstanceServer, sourceOp *operation, sourceSecrets map[string]string) error { // Quick checks. for n := range targetSecrets { _, ok := sourceSecrets[n] if !ok { return fmt.Errorf("Migration target expects the \"%s\" socket but source isn't providing it", n) } } if targetSecrets[api.SecretNameControl] == "" { return errors.New("Migration target didn't setup the required \"control\" socket") } // Struct used to hold everything together type proxy struct { done chan struct{} sourceConn *websocket.Conn targetConn *websocket.Conn } proxies := map[string]*proxy{} // Connect the control socket sourceConn, err := source.GetOperationWebsocket(sourceOp.ID, sourceSecrets[api.SecretNameControl]) if err != nil { return err } targetConn, err := r.GetOperationWebsocket(targetOp.ID, targetSecrets[api.SecretNameControl]) if err != nil { return err } proxies[api.SecretNameControl] = &proxy{ done: ws.Proxy(sourceConn, targetConn), sourceConn: sourceConn, targetConn: targetConn, } // Connect the data sockets for name := range sourceSecrets { if name == api.SecretNameControl { continue } // Handle resets (used for multiple objects) sourceConn, err := source.GetOperationWebsocket(sourceOp.ID, sourceSecrets[name]) if err != nil { break } targetConn, err := r.GetOperationWebsocket(targetOp.ID, targetSecrets[name]) if err != nil { break } proxies[name] = &proxy{ sourceConn: sourceConn, targetConn: targetConn, done: ws.Proxy(sourceConn, targetConn), } } // Cleanup once everything is done go func() { // Wait for control socket <-proxies[api.SecretNameControl].done _ = proxies[api.SecretNameControl].sourceConn.Close() _ = proxies[api.SecretNameControl].targetConn.Close() // Then deal with the others for name, proxy := range proxies { if name == api.SecretNameControl { continue } <-proxy.done _ = proxy.sourceConn.Close() _ = proxy.targetConn.Close() } }() return nil } // GetInstanceDebugMemory retrieves memory debug information for a given instance and saves it to the specified file path. func (r *ProtocolIncus) GetInstanceDebugMemory(name string, format string) (io.ReadCloser, error) { path, v, err := r.instanceTypeToPath(api.InstanceTypeVM) if err != nil { return nil, err } v.Set("format", format) // Prepare the HTTP request requestURL := fmt.Sprintf("%s/1.0%s/%s/debug/memory?%s", r.httpBaseURL.String(), path, url.PathEscape(name), v.Encode()) requestURL, err = r.setQueryAttributes(requestURL) if err != nil { return nil, err } req, err := http.NewRequest("GET", requestURL, nil) if err != nil { return nil, err } // Send the request resp, err := r.DoHTTP(req) if err != nil { return nil, err } // Check the return value for a cleaner error if resp.StatusCode != http.StatusOK { _, _, err := incusParseResponse(resp) if err != nil { return nil, err } } return resp.Body, nil } // CreateInstanceBitmap requests that Incus creates a new bitmap for the instance. func (r *ProtocolIncus) CreateInstanceBitmap(name string, bitmap api.StorageVolumeBitmapsPost) error { if !r.HasExtension("storage_volume_nbd") { return errors.New("The server is missing the required \"storage_volume_nbd\" API extension") } path, _, err := r.instanceTypeToPath(api.InstanceTypeAny) if err != nil { return err } // Send the request _, _, err = r.query("POST", fmt.Sprintf("%s/%s/bitmaps", path, url.PathEscape(name)), bitmap, "") if err != nil { return err } return nil } incus-7.0.0/client/incus_metadata.go000066400000000000000000000011461517523235500174370ustar00rootroot00000000000000package incus import ( "errors" "github.com/lxc/incus/v7/shared/api" ) // GetMetadataConfiguration returns a configuration metadata struct. func (r *ProtocolIncus) GetMetadataConfiguration() (*api.MetadataConfiguration, error) { metadataConfiguration := api.MetadataConfiguration{} if !r.HasExtension("metadata_configuration") { return nil, errors.New("The server is missing the required \"metadata_configuration\" API extension") } _, err := r.queryStruct("GET", "/metadata/configuration", nil, "", &metadataConfiguration) if err != nil { return nil, err } return &metadataConfiguration, nil } incus-7.0.0/client/incus_network_acls.go000066400000000000000000000112001517523235500203420ustar00rootroot00000000000000package incus import ( "errors" "fmt" "io" "net/http" "net/url" "github.com/lxc/incus/v7/shared/api" ) // GetNetworkACLNames returns a list of network ACL names. func (r *ProtocolIncus) GetNetworkACLNames() ([]string, error) { if !r.HasExtension("network_acl") { return nil, errors.New(`The server is missing the required "network_acl" API extension`) } // Fetch the raw URL values. urls := []string{} baseURL := "/network-acls" _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetNetworkACLs returns a list of Network ACL structs. func (r *ProtocolIncus) GetNetworkACLs() ([]api.NetworkACL, error) { if !r.HasExtension("network_acl") { return nil, errors.New(`The server is missing the required "network_acl" API extension`) } acls := []api.NetworkACL{} // Fetch the raw value. _, err := r.queryStruct("GET", "/network-acls?recursion=1", nil, "", &acls) if err != nil { return nil, err } return acls, nil } // GetNetworkACLsAllProjects returns all list of Network ACL structs across all projects. func (r *ProtocolIncus) GetNetworkACLsAllProjects() ([]api.NetworkACL, error) { if !r.HasExtension("network_acls_all_projects") { return nil, errors.New(`The server is missing the required "network_acls_all_projects" API extension`) } acls := []api.NetworkACL{} _, err := r.queryStruct("GET", "/network-acls?recursion=1&all-projects=true", nil, "", &acls) if err != nil { return nil, err } return acls, nil } // GetNetworkACL returns a Network ACL entry for the provided name. func (r *ProtocolIncus) GetNetworkACL(name string) (*api.NetworkACL, string, error) { if !r.HasExtension("network_acl") { return nil, "", errors.New(`The server is missing the required "network_acl" API extension`) } acl := api.NetworkACL{} // Fetch the raw value. etag, err := r.queryStruct("GET", fmt.Sprintf("/network-acls/%s", url.PathEscape(name)), nil, "", &acl) if err != nil { return nil, "", err } return &acl, etag, nil } // GetNetworkACLLogfile returns a reader for the ACL log file. // // Note that it's the caller's responsibility to close the returned ReadCloser. func (r *ProtocolIncus) GetNetworkACLLogfile(name string) (io.ReadCloser, error) { if !r.HasExtension("network_acl_log") { return nil, errors.New(`The server is missing the required "network_acl_log" API extension`) } // Prepare the HTTP request uri := fmt.Sprintf("%s/1.0/network-acls/%s/log", r.httpBaseURL.String(), url.PathEscape(name)) uri, err := r.setQueryAttributes(uri) if err != nil { return nil, err } req, err := http.NewRequest("GET", uri, nil) if err != nil { return nil, err } // Send the request resp, err := r.DoHTTP(req) if err != nil { return nil, err } // Check the return value for a cleaner error if resp.StatusCode != http.StatusOK { _, _, err := incusParseResponse(resp) if err != nil { return nil, err } } return resp.Body, err } // CreateNetworkACL defines a new network ACL using the provided struct. func (r *ProtocolIncus) CreateNetworkACL(acl api.NetworkACLsPost) error { if !r.HasExtension("network_acl") { return errors.New(`The server is missing the required "network_acl" API extension`) } // Send the request. _, _, err := r.query("POST", "/network-acls", acl, "") if err != nil { return err } return nil } // UpdateNetworkACL updates the network ACL to match the provided struct. func (r *ProtocolIncus) UpdateNetworkACL(name string, acl api.NetworkACLPut, ETag string) error { if !r.HasExtension("network_acl") { return errors.New(`The server is missing the required "network_acl" API extension`) } // Send the request. _, _, err := r.query("PUT", fmt.Sprintf("/network-acls/%s", url.PathEscape(name)), acl, ETag) if err != nil { return err } return nil } // RenameNetworkACL renames an existing network ACL entry. func (r *ProtocolIncus) RenameNetworkACL(name string, acl api.NetworkACLPost) error { if !r.HasExtension("network_acl") { return errors.New(`The server is missing the required "network_acl" API extension`) } // Send the request. _, _, err := r.query("POST", fmt.Sprintf("/network-acls/%s", url.PathEscape(name)), acl, "") if err != nil { return err } return nil } // DeleteNetworkACL deletes an existing network ACL. func (r *ProtocolIncus) DeleteNetworkACL(name string) error { if !r.HasExtension("network_acl") { return errors.New(`The server is missing the required "network_acl" API extension`) } // Send the request. _, _, err := r.query("DELETE", fmt.Sprintf("/network-acls/%s", url.PathEscape(name)), nil, "") if err != nil { return err } return nil } incus-7.0.0/client/incus_network_address_sets.go000066400000000000000000000102011517523235500221030ustar00rootroot00000000000000package incus import ( "errors" "fmt" "net/url" "github.com/lxc/incus/v7/shared/api" ) // GetNetworkAddressSetNames returns a list of network address set names. func (r *ProtocolIncus) GetNetworkAddressSetNames() ([]string, error) { if !r.HasExtension("network_address_set") { return nil, errors.New(`The server is missing the required "network_address_set" API extension`) } // Fetch the raw URL values. urls := []string{} baseURL := "/network-address-sets" _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetNetworkAddressSets returns a list of network address set structs. func (r *ProtocolIncus) GetNetworkAddressSets() ([]api.NetworkAddressSet, error) { if !r.HasExtension("network_address_set") { return nil, errors.New(`The server is missing the required "network_address_set" API extension`) } addressSets := []api.NetworkAddressSet{} // Fetch the raw value. _, err := r.queryStruct("GET", "/network-address-sets?recursion=1", nil, "", &addressSets) if err != nil { return nil, err } return addressSets, nil } // GetNetworkAddressSetsAllProjects returns a list of network address set structs across all projects. func (r *ProtocolIncus) GetNetworkAddressSetsAllProjects() ([]api.NetworkAddressSet, error) { if !r.HasExtension("network_address_set") { return nil, errors.New(`The server is missing the required "network_address_set" API extension`) } addressSets := []api.NetworkAddressSet{} _, err := r.queryStruct("GET", "/network-address-sets?recursion=1&all-projects=true", nil, "", &addressSets) if err != nil { return nil, err } return addressSets, nil } // GetNetworkAddressSet returns a network address set entry for the provided name. func (r *ProtocolIncus) GetNetworkAddressSet(name string) (*api.NetworkAddressSet, string, error) { if !r.HasExtension("network_address_set") { return nil, "", errors.New(`The server is missing the required "network_address_set" API extension`) } addrSet := api.NetworkAddressSet{} // Fetch the raw value. etag, err := r.queryStruct("GET", fmt.Sprintf("/network-address-sets/%s", url.PathEscape(name)), nil, "", &addrSet) if err != nil { return nil, "", err } return &addrSet, etag, nil } // CreateNetworkAddressSet defines a new network address set using the provided struct. func (r *ProtocolIncus) CreateNetworkAddressSet(as api.NetworkAddressSetsPost) error { if !r.HasExtension("network_address_set") { return errors.New(`The server is missing the required "network_address_set" API extension`) } // Send the request. _, _, err := r.query("POST", "/network-address-sets", as, "") if err != nil { return err } return nil } // UpdateNetworkAddressSet updates the network address set to match the provided struct. func (r *ProtocolIncus) UpdateNetworkAddressSet(name string, as api.NetworkAddressSetPut, ETag string) error { if !r.HasExtension("network_address_set") { return errors.New(`The server is missing the required "network_address_set" API extension`) } // Send the request. _, _, err := r.query("PUT", fmt.Sprintf("/network-address-sets/%s", url.PathEscape(name)), as, ETag) if err != nil { return err } return nil } // RenameNetworkAddressSet renames an existing network address set entry. func (r *ProtocolIncus) RenameNetworkAddressSet(name string, as api.NetworkAddressSetPost) error { if !r.HasExtension("network_address_set") { return errors.New(`The server is missing the required "network_address_set" API extension`) } // Send the request. _, _, err := r.query("POST", fmt.Sprintf("/network-address-sets/%s", url.PathEscape(name)), as, "") if err != nil { return err } return nil } // DeleteNetworkAddressSet deletes an existing network address set. func (r *ProtocolIncus) DeleteNetworkAddressSet(name string) error { if !r.HasExtension("network_address_set") { return errors.New(`The server is missing the required "network_address_set" API extension`) } // Send the request. _, _, err := r.query("DELETE", fmt.Sprintf("/network-address-sets/%s", url.PathEscape(name)), nil, "") if err != nil { return err } return nil } incus-7.0.0/client/incus_network_allocations.go000066400000000000000000000020451517523235500217370ustar00rootroot00000000000000package incus import ( "github.com/lxc/incus/v7/shared/api" ) // GetNetworkAllocations returns a list of Network allocations for a specific project. func (r *ProtocolIncus) GetNetworkAllocations() ([]api.NetworkAllocations, error) { err := r.CheckExtension("network_allocations") if err != nil { return nil, err } // Fetch the raw value. netAllocations := []api.NetworkAllocations{} _, err = r.queryStruct("GET", "/network-allocations", nil, "", &netAllocations) if err != nil { return nil, err } return netAllocations, nil } // GetNetworkAllocationsAllProjects returns a list of Network allocations across all projects. func (r *ProtocolIncus) GetNetworkAllocationsAllProjects() ([]api.NetworkAllocations, error) { err := r.CheckExtension("network_allocations") if err != nil { return nil, err } // Fetch the raw value. netAllocations := []api.NetworkAllocations{} _, err = r.queryStruct("GET", "/network-allocations?all-projects=true", nil, "", &netAllocations) if err != nil { return nil, err } return netAllocations, nil } incus-7.0.0/client/incus_network_forwards.go000066400000000000000000000066321517523235500212640ustar00rootroot00000000000000package incus import ( "errors" "fmt" "net/url" "github.com/lxc/incus/v7/shared/api" ) // GetNetworkForwardAddresses returns a list of network forward listen addresses. func (r *ProtocolIncus) GetNetworkForwardAddresses(networkName string) ([]string, error) { if !r.HasExtension("network_forward") { return nil, errors.New(`The server is missing the required "network_forward" API extension`) } // Fetch the raw URL values. urls := []string{} baseURL := fmt.Sprintf("/networks/%s/forwards", url.PathEscape(networkName)) _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetNetworkForwards returns a list of Network forward structs. func (r *ProtocolIncus) GetNetworkForwards(networkName string) ([]api.NetworkForward, error) { if !r.HasExtension("network_forward") { return nil, errors.New(`The server is missing the required "network_forward" API extension`) } forwards := []api.NetworkForward{} // Fetch the raw value. _, err := r.queryStruct("GET", fmt.Sprintf("/networks/%s/forwards?recursion=1", url.PathEscape(networkName)), nil, "", &forwards) if err != nil { return nil, err } return forwards, nil } // GetNetworkForward returns a Network forward entry for the provided network and listen address. func (r *ProtocolIncus) GetNetworkForward(networkName string, listenAddress string) (*api.NetworkForward, string, error) { if !r.HasExtension("network_forward") { return nil, "", errors.New(`The server is missing the required "network_forward" API extension`) } forward := api.NetworkForward{} // Fetch the raw value. etag, err := r.queryStruct("GET", fmt.Sprintf("/networks/%s/forwards/%s", url.PathEscape(networkName), url.PathEscape(listenAddress)), nil, "", &forward) if err != nil { return nil, "", err } return &forward, etag, nil } // CreateNetworkForward defines a new network forward using the provided struct. func (r *ProtocolIncus) CreateNetworkForward(networkName string, forward api.NetworkForwardsPost) error { if !r.HasExtension("network_forward") { return errors.New(`The server is missing the required "network_forward" API extension`) } // Send the request. _, _, err := r.query("POST", fmt.Sprintf("/networks/%s/forwards", url.PathEscape(networkName)), forward, "") if err != nil { return err } return nil } // UpdateNetworkForward updates the network forward to match the provided struct. func (r *ProtocolIncus) UpdateNetworkForward(networkName string, listenAddress string, forward api.NetworkForwardPut, ETag string) error { if !r.HasExtension("network_forward") { return errors.New(`The server is missing the required "network_forward" API extension`) } // Send the request. _, _, err := r.query("PUT", fmt.Sprintf("/networks/%s/forwards/%s", url.PathEscape(networkName), url.PathEscape(listenAddress)), forward, ETag) if err != nil { return err } return nil } // DeleteNetworkForward deletes an existing network forward. func (r *ProtocolIncus) DeleteNetworkForward(networkName string, listenAddress string) error { if !r.HasExtension("network_forward") { return errors.New(`The server is missing the required "network_forward" API extension`) } // Send the request. _, _, err := r.query("DELETE", fmt.Sprintf("/networks/%s/forwards/%s", url.PathEscape(networkName), url.PathEscape(listenAddress)), nil, "") if err != nil { return err } return nil } incus-7.0.0/client/incus_network_integrations.go000066400000000000000000000074321517523235500221420ustar00rootroot00000000000000package incus import ( "errors" "fmt" "net/url" "github.com/lxc/incus/v7/shared/api" ) // GetNetworkIntegrationNames returns a list of network integration names. func (r *ProtocolIncus) GetNetworkIntegrationNames() ([]string, error) { if !r.HasExtension("network_integrations") { return nil, errors.New(`The server is missing the required "network_integrations" API extension`) } // Fetch the raw URL values. urls := []string{} baseURL := "/network-integrations" _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetNetworkIntegrations returns a list of network integration structs. func (r *ProtocolIncus) GetNetworkIntegrations() ([]api.NetworkIntegration, error) { if !r.HasExtension("network_integrations") { return nil, errors.New(`The server is missing the required "network_integrations" API extension`) } integrations := []api.NetworkIntegration{} // Fetch the raw value. _, err := r.queryStruct("GET", "/network-integrations?recursion=1", nil, "", &integrations) if err != nil { return nil, err } return integrations, nil } // GetNetworkIntegration returns a network integration entry. func (r *ProtocolIncus) GetNetworkIntegration(name string) (*api.NetworkIntegration, string, error) { if !r.HasExtension("network_integrations") { return nil, "", errors.New(`The server is missing the required "network_integrations" API extension`) } integration := api.NetworkIntegration{} // Fetch the raw value. etag, err := r.queryStruct("GET", fmt.Sprintf("/network-integrations/%s", url.PathEscape(name)), nil, "", &integration) if err != nil { return nil, "", err } return &integration, etag, nil } // CreateNetworkIntegration defines a new network integration using the provided struct. // Returns true if the integration connection has been mutually created. Returns false if integrationing has been only initiated. func (r *ProtocolIncus) CreateNetworkIntegration(integration api.NetworkIntegrationsPost) error { if !r.HasExtension("network_integrations") { return errors.New(`The server is missing the required "network_integrations" API extension`) } // Send the request. _, _, err := r.query("POST", "/network-integrations", integration, "") if err != nil { return err } return nil } // UpdateNetworkIntegration updates the network integration to match the provided struct. func (r *ProtocolIncus) UpdateNetworkIntegration(name string, integration api.NetworkIntegrationPut, ETag string) error { if !r.HasExtension("network_integrations") { return errors.New(`The server is missing the required "network_integrations" API extension`) } // Send the request. _, _, err := r.query("PUT", fmt.Sprintf("/network-integrations/%s", url.PathEscape(name)), integration, ETag) if err != nil { return err } return nil } // RenameNetworkIntegration renames an existing network integration entry. func (r *ProtocolIncus) RenameNetworkIntegration(name string, network api.NetworkIntegrationPost) error { if !r.HasExtension("network_integrations") { return errors.New("The server is missing the required \"network_integrations\" API extension") } // Send the request _, _, err := r.query("POST", fmt.Sprintf("/network-integrations/%s", url.PathEscape(name)), network, "") if err != nil { return err } return nil } // DeleteNetworkIntegration deletes an existing network integration. func (r *ProtocolIncus) DeleteNetworkIntegration(name string) error { if !r.HasExtension("network_integrations") { return errors.New(`The server is missing the required "network_integrations" API extension`) } // Send the request. _, _, err := r.query("DELETE", fmt.Sprintf("/network-integrations/%s", url.PathEscape(name)), nil, "") if err != nil { return err } return nil } incus-7.0.0/client/incus_network_load_balancers.go000066400000000000000000000075721517523235500223720ustar00rootroot00000000000000package incus import ( "github.com/lxc/incus/v7/shared/api" ) // GetNetworkLoadBalancerAddresses returns a list of network load balancer listen addresses. func (r *ProtocolIncus) GetNetworkLoadBalancerAddresses(networkName string) ([]string, error) { err := r.CheckExtension("network_load_balancer") if err != nil { return nil, err } // Fetch the raw URL values. urls := []string{} u := api.NewURL().Path("networks", networkName, "load-balancers") _, err = r.queryStruct("GET", u.String(), nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(u.String(), urls...) } // GetNetworkLoadBalancers returns a list of Network load balancer structs. func (r *ProtocolIncus) GetNetworkLoadBalancers(networkName string) ([]api.NetworkLoadBalancer, error) { err := r.CheckExtension("network_load_balancer") if err != nil { return nil, err } loadBalancers := []api.NetworkLoadBalancer{} // Fetch the raw value. u := api.NewURL().Path("networks", networkName, "load-balancers").WithQuery("recursion", "1") _, err = r.queryStruct("GET", u.String(), nil, "", &loadBalancers) if err != nil { return nil, err } return loadBalancers, nil } // GetNetworkLoadBalancer returns a Network load balancer entry for the provided network and listen address. func (r *ProtocolIncus) GetNetworkLoadBalancer(networkName string, listenAddress string) (*api.NetworkLoadBalancer, string, error) { err := r.CheckExtension("network_load_balancer") if err != nil { return nil, "", err } loadBalancer := api.NetworkLoadBalancer{} // Fetch the raw value. u := api.NewURL().Path("networks", networkName, "load-balancers", listenAddress) etag, err := r.queryStruct("GET", u.String(), nil, "", &loadBalancer) if err != nil { return nil, "", err } return &loadBalancer, etag, nil } // CreateNetworkLoadBalancer defines a new network load balancer using the provided struct. func (r *ProtocolIncus) CreateNetworkLoadBalancer(networkName string, loadBalancer api.NetworkLoadBalancersPost) error { err := r.CheckExtension("network_load_balancer") if err != nil { return err } // Send the request. u := api.NewURL().Path("networks", networkName, "load-balancers") _, _, err = r.query("POST", u.String(), loadBalancer, "") if err != nil { return err } return nil } // UpdateNetworkLoadBalancer updates the network load balancer to match the provided struct. func (r *ProtocolIncus) UpdateNetworkLoadBalancer(networkName string, listenAddress string, loadBalancer api.NetworkLoadBalancerPut, ETag string) error { err := r.CheckExtension("network_load_balancer") if err != nil { return err } // Send the request. u := api.NewURL().Path("networks", networkName, "load-balancers", listenAddress) _, _, err = r.query("PUT", u.String(), loadBalancer, ETag) if err != nil { return err } return nil } // DeleteNetworkLoadBalancer deletes an existing network load balancer. func (r *ProtocolIncus) DeleteNetworkLoadBalancer(networkName string, listenAddress string) error { err := r.CheckExtension("network_load_balancer") if err != nil { return err } // Send the request. u := api.NewURL().Path("networks", networkName, "load-balancers", listenAddress) _, _, err = r.query("DELETE", u.String(), nil, "") if err != nil { return err } return nil } // GetNetworkLoadBalancerState returns a Network load balancer state for the provided network and listen address. func (r *ProtocolIncus) GetNetworkLoadBalancerState(networkName string, listenAddress string) (*api.NetworkLoadBalancerState, error) { err := r.CheckExtension("network_load_balancer_state") if err != nil { return nil, err } lbState := api.NetworkLoadBalancerState{} // Fetch the raw value. u := api.NewURL().Path("networks", networkName, "load-balancers", listenAddress, "state") _, err = r.queryStruct("GET", u.String(), nil, "", &lbState) if err != nil { return nil, err } return &lbState, nil } incus-7.0.0/client/incus_network_peers.go000066400000000000000000000067701517523235500205560ustar00rootroot00000000000000package incus import ( "errors" "fmt" "net/url" "github.com/lxc/incus/v7/shared/api" ) // GetNetworkPeerNames returns a list of network peer names. func (r *ProtocolIncus) GetNetworkPeerNames(networkName string) ([]string, error) { if !r.HasExtension("network_peer") { return nil, errors.New(`The server is missing the required "network_peer" API extension`) } // Fetch the raw URL values. urls := []string{} baseURL := fmt.Sprintf("/networks/%s/peers", url.PathEscape(networkName)) _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetNetworkPeers returns a list of network peer structs. func (r *ProtocolIncus) GetNetworkPeers(networkName string) ([]api.NetworkPeer, error) { if !r.HasExtension("network_peer") { return nil, errors.New(`The server is missing the required "network_peer" API extension`) } peers := []api.NetworkPeer{} // Fetch the raw value. _, err := r.queryStruct("GET", fmt.Sprintf("/networks/%s/peers?recursion=1", url.PathEscape(networkName)), nil, "", &peers) if err != nil { return nil, err } return peers, nil } // GetNetworkPeer returns a network peer entry for the provided network and peer name. func (r *ProtocolIncus) GetNetworkPeer(networkName string, peerName string) (*api.NetworkPeer, string, error) { if !r.HasExtension("network_peer") { return nil, "", errors.New(`The server is missing the required "network_peer" API extension`) } peer := api.NetworkPeer{} // Fetch the raw value. etag, err := r.queryStruct("GET", fmt.Sprintf("/networks/%s/peers/%s", url.PathEscape(networkName), url.PathEscape(peerName)), nil, "", &peer) if err != nil { return nil, "", err } return &peer, etag, nil } // CreateNetworkPeer defines a new network peer using the provided struct. // Returns true if the peer connection has been mutually created. Returns false if peering has been only initiated. func (r *ProtocolIncus) CreateNetworkPeer(networkName string, peer api.NetworkPeersPost) error { if !r.HasExtension("network_peer") { return errors.New(`The server is missing the required "network_peer" API extension`) } if peer.Type != "" && peer.Type != "local" && !r.HasExtension("network_integrations") { return errors.New(`The server is missing the required "network_integrations" API extension`) } // Send the request. _, _, err := r.query("POST", fmt.Sprintf("/networks/%s/peers", url.PathEscape(networkName)), peer, "") if err != nil { return err } return nil } // UpdateNetworkPeer updates the network peer to match the provided struct. func (r *ProtocolIncus) UpdateNetworkPeer(networkName string, peerName string, peer api.NetworkPeerPut, ETag string) error { if !r.HasExtension("network_peer") { return errors.New(`The server is missing the required "network_peer" API extension`) } // Send the request. _, _, err := r.query("PUT", fmt.Sprintf("/networks/%s/peers/%s", url.PathEscape(networkName), url.PathEscape(peerName)), peer, ETag) if err != nil { return err } return nil } // DeleteNetworkPeer deletes an existing network peer. func (r *ProtocolIncus) DeleteNetworkPeer(networkName string, peerName string) error { if !r.HasExtension("network_peer") { return errors.New(`The server is missing the required "network_peer" API extension`) } // Send the request. _, _, err := r.query("DELETE", fmt.Sprintf("/networks/%s/peers/%s", url.PathEscape(networkName), url.PathEscape(peerName)), nil, "") if err != nil { return err } return nil } incus-7.0.0/client/incus_network_zones.go000066400000000000000000000151761517523235500205760ustar00rootroot00000000000000package incus import ( "errors" "fmt" "net/url" "github.com/lxc/incus/v7/shared/api" ) // GetNetworkZoneNames returns a list of network zone names. func (r *ProtocolIncus) GetNetworkZoneNames() ([]string, error) { if !r.HasExtension("network_dns") { return nil, errors.New(`The server is missing the required "network_dns" API extension`) } // Fetch the raw URL values. urls := []string{} baseURL := "/network-zones" _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetNetworkZones returns a list of Network zone structs. func (r *ProtocolIncus) GetNetworkZones() ([]api.NetworkZone, error) { if !r.HasExtension("network_dns") { return nil, errors.New(`The server is missing the required "network_dns" API extension`) } zones := []api.NetworkZone{} // Fetch the raw value. _, err := r.queryStruct("GET", "/network-zones?recursion=1", nil, "", &zones) if err != nil { return nil, err } return zones, nil } // GetNetworkZonesAllProjects returns a list of network zones across all projects as NetworkZone structs. func (r *ProtocolIncus) GetNetworkZonesAllProjects() ([]api.NetworkZone, error) { err := r.CheckExtension("network_zones_all_projects") if err != nil { return nil, errors.New(`The server is missing the required "network_zones_all_projects" API extension`) } zones := []api.NetworkZone{} _, err = r.queryStruct("GET", "/network-zones?recursion=1&all-projects=true", nil, "", &zones) if err != nil { return nil, err } return zones, nil } // GetNetworkZone returns a Network zone entry for the provided name. func (r *ProtocolIncus) GetNetworkZone(name string) (*api.NetworkZone, string, error) { if !r.HasExtension("network_dns") { return nil, "", errors.New(`The server is missing the required "network_dns" API extension`) } zone := api.NetworkZone{} // Fetch the raw value. etag, err := r.queryStruct("GET", fmt.Sprintf("/network-zones/%s", url.PathEscape(name)), nil, "", &zone) if err != nil { return nil, "", err } return &zone, etag, nil } // CreateNetworkZone defines a new Network zone using the provided struct. func (r *ProtocolIncus) CreateNetworkZone(zone api.NetworkZonesPost) error { if !r.HasExtension("network_dns") { return errors.New(`The server is missing the required "network_dns" API extension`) } // Send the request. _, _, err := r.query("POST", "/network-zones", zone, "") if err != nil { return err } return nil } // UpdateNetworkZone updates the network zone to match the provided struct. func (r *ProtocolIncus) UpdateNetworkZone(name string, zone api.NetworkZonePut, ETag string) error { if !r.HasExtension("network_dns") { return errors.New(`The server is missing the required "network_dns" API extension`) } // Send the request. _, _, err := r.query("PUT", fmt.Sprintf("/network-zones/%s", url.PathEscape(name)), zone, ETag) if err != nil { return err } return nil } // DeleteNetworkZone deletes an existing network zone. func (r *ProtocolIncus) DeleteNetworkZone(name string) error { if !r.HasExtension("network_dns") { return errors.New(`The server is missing the required "network_dns" API extension`) } // Send the request. _, _, err := r.query("DELETE", fmt.Sprintf("/network-zones/%s", url.PathEscape(name)), nil, "") if err != nil { return err } return nil } // GetNetworkZoneRecordNames returns a list of network zone record names. func (r *ProtocolIncus) GetNetworkZoneRecordNames(zone string) ([]string, error) { if !r.HasExtension("network_dns_records") { return nil, errors.New(`The server is missing the required "network_dns_records" API extension`) } // Fetch the raw URL values. urls := []string{} baseURL := fmt.Sprintf("/network-zones/%s/records", url.PathEscape(zone)) _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetNetworkZoneRecords returns a list of Network zone record structs. func (r *ProtocolIncus) GetNetworkZoneRecords(zone string) ([]api.NetworkZoneRecord, error) { if !r.HasExtension("network_dns_records") { return nil, errors.New(`The server is missing the required "network_dns_records" API extension`) } records := []api.NetworkZoneRecord{} // Fetch the raw value. _, err := r.queryStruct("GET", fmt.Sprintf("/network-zones/%s/records?recursion=1", url.PathEscape(zone)), nil, "", &records) if err != nil { return nil, err } return records, nil } // GetNetworkZoneRecord returns a Network zone record entry for the provided zone and name. func (r *ProtocolIncus) GetNetworkZoneRecord(zone string, name string) (*api.NetworkZoneRecord, string, error) { if !r.HasExtension("network_dns_records") { return nil, "", errors.New(`The server is missing the required "network_dns_records" API extension`) } record := api.NetworkZoneRecord{} // Fetch the raw value. etag, err := r.queryStruct("GET", fmt.Sprintf("/network-zones/%s/records/%s", url.PathEscape(zone), url.PathEscape(name)), nil, "", &record) if err != nil { return nil, "", err } return &record, etag, nil } // CreateNetworkZoneRecord defines a new Network zone record using the provided struct. func (r *ProtocolIncus) CreateNetworkZoneRecord(zone string, record api.NetworkZoneRecordsPost) error { if !r.HasExtension("network_dns_records") { return errors.New(`The server is missing the required "network_dns_records" API extension`) } // Send the request. _, _, err := r.query("POST", fmt.Sprintf("/network-zones/%s/records", url.PathEscape(zone)), record, "") if err != nil { return err } return nil } // UpdateNetworkZoneRecord updates the network zone record to match the provided struct. func (r *ProtocolIncus) UpdateNetworkZoneRecord(zone string, name string, record api.NetworkZoneRecordPut, ETag string) error { if !r.HasExtension("network_dns_records") { return errors.New(`The server is missing the required "network_dns_records" API extension`) } // Send the request. _, _, err := r.query("PUT", fmt.Sprintf("/network-zones/%s/records/%s", url.PathEscape(zone), url.PathEscape(name)), record, ETag) if err != nil { return err } return nil } // DeleteNetworkZoneRecord deletes an existing network zone record. func (r *ProtocolIncus) DeleteNetworkZoneRecord(zone string, name string) error { if !r.HasExtension("network_dns_records") { return errors.New(`The server is missing the required "network_dns_records" API extension`) } // Send the request. _, _, err := r.query("DELETE", fmt.Sprintf("/network-zones/%s/records/%s", url.PathEscape(zone), url.PathEscape(name)), nil, "") if err != nil { return err } return nil } incus-7.0.0/client/incus_networks.go000066400000000000000000000133741517523235500175410ustar00rootroot00000000000000package incus import ( "errors" "fmt" "net/url" "github.com/lxc/incus/v7/shared/api" ) // GetNetworkNames returns a list of network names. func (r *ProtocolIncus) GetNetworkNames() ([]string, error) { if !r.HasExtension("network") { return nil, errors.New("The server is missing the required \"network\" API extension") } // Fetch the raw values. urls := []string{} baseURL := "/networks" _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetNetworks returns a list of Network struct. func (r *ProtocolIncus) GetNetworks() ([]api.Network, error) { if !r.HasExtension("network") { return nil, errors.New("The server is missing the required \"network\" API extension") } networks := []api.Network{} // Fetch the raw value _, err := r.queryStruct("GET", "/networks?recursion=1", nil, "", &networks) if err != nil { return nil, err } return networks, nil } // GetNetworksWithFilter returns a list of filtered Network struct. func (r *ProtocolIncus) GetNetworksWithFilter(filters []string) ([]api.Network, error) { if !r.HasExtension("network") { return nil, errors.New("The server is missing the required \"network\" API extension") } networks := []api.Network{} v := url.Values{} v.Set("recursion", "1") v.Set("filter", parseFilters(filters)) // Fetch the raw value _, err := r.queryStruct("GET", fmt.Sprintf("/networks?%s", v.Encode()), nil, "", &networks) if err != nil { return nil, err } return networks, nil } // GetNetworksAllProjects gets all networks across all projects. func (r *ProtocolIncus) GetNetworksAllProjects() ([]api.Network, error) { if !r.HasExtension("networks_all_projects") { return nil, errors.New(`The server is missing the required "networks_all_projects" API extension`) } networks := []api.Network{} _, err := r.queryStruct("GET", "/networks?recursion=1&all-projects=true", nil, "", &networks) if err != nil { return nil, err } return networks, nil } // GetNetworksAllProjectsWithFilter gets a filtered list of all networks across all projects. func (r *ProtocolIncus) GetNetworksAllProjectsWithFilter(filters []string) ([]api.Network, error) { if !r.HasExtension("networks_all_projects") { return nil, errors.New(`The server is missing the required "networks_all_projects" API extension`) } networks := []api.Network{} v := url.Values{} v.Set("recursion", "1") v.Set("all-projects", "true") v.Set("filter", parseFilters(filters)) _, err := r.queryStruct("GET", fmt.Sprintf("/networks?%s", v.Encode()), nil, "", &networks) if err != nil { return nil, err } return networks, nil } // GetNetwork returns a Network entry for the provided name. func (r *ProtocolIncus) GetNetwork(name string) (*api.Network, string, error) { if !r.HasExtension("network") { return nil, "", errors.New("The server is missing the required \"network\" API extension") } network := api.Network{} // Fetch the raw value etag, err := r.queryStruct("GET", fmt.Sprintf("/networks/%s", url.PathEscape(name)), nil, "", &network) if err != nil { return nil, "", err } return &network, etag, nil } // GetNetworkLeases returns a list of Network struct. func (r *ProtocolIncus) GetNetworkLeases(name string) ([]api.NetworkLease, error) { if !r.HasExtension("network_leases") { return nil, errors.New("The server is missing the required \"network_leases\" API extension") } leases := []api.NetworkLease{} // Fetch the raw value _, err := r.queryStruct("GET", fmt.Sprintf("/networks/%s/leases", url.PathEscape(name)), nil, "", &leases) if err != nil { return nil, err } return leases, nil } // GetNetworkState returns metrics and information on the running network. func (r *ProtocolIncus) GetNetworkState(name string) (*api.NetworkState, error) { if !r.HasExtension("network_state") { return nil, errors.New("The server is missing the required \"network_state\" API extension") } state := api.NetworkState{} // Fetch the raw value _, err := r.queryStruct("GET", fmt.Sprintf("/networks/%s/state", url.PathEscape(name)), nil, "", &state) if err != nil { return nil, err } return &state, nil } // CreateNetwork defines a new network using the provided Network struct. func (r *ProtocolIncus) CreateNetwork(network api.NetworksPost) error { if !r.HasExtension("network") { return errors.New("The server is missing the required \"network\" API extension") } // Send the request _, _, err := r.query("POST", "/networks", network, "") if err != nil { return err } return nil } // UpdateNetwork updates the network to match the provided Network struct. func (r *ProtocolIncus) UpdateNetwork(name string, network api.NetworkPut, ETag string) error { if !r.HasExtension("network") { return errors.New("The server is missing the required \"network\" API extension") } // Send the request _, _, err := r.query("PUT", fmt.Sprintf("/networks/%s", url.PathEscape(name)), network, ETag) if err != nil { return err } return nil } // RenameNetwork renames an existing network entry. func (r *ProtocolIncus) RenameNetwork(name string, network api.NetworkPost) error { if !r.HasExtension("network") { return errors.New("The server is missing the required \"network\" API extension") } // Send the request _, _, err := r.query("POST", fmt.Sprintf("/networks/%s", url.PathEscape(name)), network, "") if err != nil { return err } return nil } // DeleteNetwork deletes an existing network. func (r *ProtocolIncus) DeleteNetwork(name string) error { if !r.HasExtension("network") { return errors.New("The server is missing the required \"network\" API extension") } // Send the request _, _, err := r.query("DELETE", fmt.Sprintf("/networks/%s", url.PathEscape(name)), nil, "") if err != nil { return err } return nil } incus-7.0.0/client/incus_oidc.go000066400000000000000000000237501517523235500166020ustar00rootroot00000000000000package incus import ( "context" "crypto/rand" "errors" "fmt" "io" "net/http" "net/url" "os" "os/signal" "strings" "syscall" "time" "github.com/gorilla/websocket" "github.com/zitadel/oidc/v3/pkg/client/rp" httphelper "github.com/zitadel/oidc/v3/pkg/http" "github.com/zitadel/oidc/v3/pkg/oidc" "golang.org/x/oauth2" "github.com/lxc/incus/v7/shared/util" ) // ErrOIDCExpired is returned when the token is expired and we can't retry the request ourselves. var ErrOIDCExpired = errors.New("OIDC token expired, please re-try the request") // setupOIDCClient initializes the OIDC (OpenID Connect) client with given tokens if it hasn't been set up already. // It also assigns the protocol's http client to the oidcClient's httpClient. func (r *ProtocolIncus) setupOIDCClient(token *oidc.Tokens[*oidc.IDTokenClaims], skipAuthenticate bool) { if r.oidcClient != nil { return } r.oidcClient = newOIDCClient(token) r.oidcClient.skipAuthenticate = skipAuthenticate r.oidcClient.httpClient = r.http } // GetOIDCTokens returns the current OIDC tokens (if any) from the OIDC client. // // This should only be used by internal Incus tools when it's not possible to get the tokens from a Config struct. func (r *ProtocolIncus) GetOIDCTokens() *oidc.Tokens[*oidc.IDTokenClaims] { if r.oidcClient == nil { return nil } return r.oidcClient.tokens } // oidcTransport is a custom HTTP transport that injects the audience field into requests directed at the device authorization endpoint. type oidcTransport struct { deviceAuthorizationEndpoint string audience string } // RoundTrip is a method of oidcTransport that modifies the request, adds the audience parameter if appropriate, and sends it along. func (o *oidcTransport) RoundTrip(r *http.Request) (*http.Response, error) { // Don't modify the request if it's not to the device authorization endpoint, or there are no // URL parameters which need to be set. if r.URL.String() != o.deviceAuthorizationEndpoint || len(o.audience) == 0 { return http.DefaultTransport.RoundTrip(r) } err := r.ParseForm() if err != nil { return nil, err } if o.audience != "" { r.Form.Add("audience", o.audience) } // Update the body with the new URL parameters. body := r.Form.Encode() r.Body = io.NopCloser(strings.NewReader(body)) r.ContentLength = int64(len(body)) return http.DefaultTransport.RoundTrip(r) } var errRefreshAccessToken = errors.New("Failed refreshing access token") type oidcClient struct { httpClient *http.Client oidcTransport *oidcTransport tokens *oidc.Tokens[*oidc.IDTokenClaims] skipAuthenticate bool } // oidcClient is a structure encapsulating an HTTP client, OIDC transport, and a token for OpenID Connect (OIDC) operations. // newOIDCClient constructs a new oidcClient, ensuring the token field is non-nil to prevent panics during authentication. func newOIDCClient(tokens *oidc.Tokens[*oidc.IDTokenClaims]) *oidcClient { client := oidcClient{ tokens: tokens, httpClient: &http.Client{}, oidcTransport: &oidcTransport{}, } // Ensure client.tokens is never nil otherwise authenticate() will panic. if client.tokens == nil { client.tokens = &oidc.Tokens[*oidc.IDTokenClaims]{} } return &client } // getAccessToken returns the Access Token from the oidcClient's tokens, or an empty string if no tokens are present. func (o *oidcClient) getAccessToken() string { if o.tokens == nil || o.tokens.Token == nil { return "" } return o.tokens.AccessToken } // do function executes an HTTP request using the oidcClient's http client, and manages authorization by refreshing or authenticating as needed. // If the request fails with an HTTP Unauthorized status, it attempts to refresh the access token, or perform an OIDC authentication if refresh fails. func (o *oidcClient) do(req *http.Request) (*http.Response, error) { resp, err := o.httpClient.Do(req) if err != nil { return nil, err } // Return immediately if the error is not HTTP status unauthorized. if resp.StatusCode != http.StatusUnauthorized { return resp, nil } issuer := resp.Header.Get("X-Incus-OIDC-issuer") clientID := resp.Header.Get("X-Incus-OIDC-clientid") audience := resp.Header.Get("X-Incus-OIDC-audience") scopes := resp.Header.Get("X-Incus-OIDC-scopes") if scopes == "" { scopes = "openid,offline_access" } if issuer == "" || clientID == "" { return resp, nil } // Refresh the token. err = o.refresh(issuer, clientID, scopes) if err != nil { if o.skipAuthenticate { return nil, fmt.Errorf("Authentication not found or expired: %w", err) } err = o.authenticate(issuer, clientID, audience, scopes) if err != nil { return nil, err } } // If not dealing with something we can retry, return a clear error. if req.Method != "GET" && req.GetBody == nil { return resp, ErrOIDCExpired } // Set the new access token in the header. req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", o.tokens.AccessToken)) // Reset the request body. if req.GetBody != nil { body, err := req.GetBody() if err != nil { return nil, err } req.Body = body } resp, err = o.httpClient.Do(req) if err != nil { return nil, err } return resp, nil } // dial function executes a websocket request and handles OIDC authentication and refresh. func (o *oidcClient) dial(dialer websocket.Dialer, uri string, req *http.Request) (*websocket.Conn, *http.Response, error) { conn, resp, err := dialer.Dial(uri, req.Header) if err != nil && resp == nil { return nil, nil, err } // Return immediately if the error is not HTTP status unauthorized. if conn != nil && resp.StatusCode != http.StatusUnauthorized { return conn, resp, nil } issuer := resp.Header.Get("X-Incus-OIDC-issuer") clientID := resp.Header.Get("X-Incus-OIDC-clientid") audience := resp.Header.Get("X-Incus-OIDC-audience") scopes := resp.Header.Get("X-Incus-OIDC-scopes") if scopes == "" { scopes = "openid,offline_access" } if issuer == "" || clientID == "" { return nil, resp, err } err = o.refresh(issuer, clientID, scopes) if err != nil { if o.skipAuthenticate { return nil, resp, fmt.Errorf("Authentication not found or expired: %w", err) } err = o.authenticate(issuer, clientID, audience, scopes) if err != nil { return nil, resp, err } } // Set the new access token in the header. req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", o.tokens.AccessToken)) return dialer.Dial(uri, req.Header) } // getProvider initializes a new OpenID Connect Relying Party for a given issuer and clientID. // The function also creates a secure CookieHandler with random encryption and hash keys, and applies a series of configurations on the Relying Party. func (o *oidcClient) getProvider(issuer string, clientID string, scopes string) (rp.RelyingParty, error) { hashKey := make([]byte, 16) encryptKey := make([]byte, 16) _, err := rand.Read(hashKey) if err != nil { return nil, err } _, err = rand.Read(encryptKey) if err != nil { return nil, err } cookieHandler := httphelper.NewCookieHandler(hashKey, encryptKey, httphelper.WithUnsecure()) options := []rp.Option{ rp.WithCookieHandler(cookieHandler), rp.WithVerifierOpts(rp.WithIssuedAtOffset(5 * time.Second)), rp.WithPKCE(cookieHandler), rp.WithHTTPClient(o.httpClient), } provider, err := rp.NewRelyingPartyOIDC(context.TODO(), issuer, clientID, "", "", strings.Split(scopes, ","), options...) if err != nil { return nil, err } return provider, nil } // refresh attempts to refresh the OpenID Connect access token for the client using the refresh token. // If no token is present or the refresh token is empty, it returns an error. If successful, it updates the access token and other relevant token fields. func (o *oidcClient) refresh(issuer string, clientID string, scopes string) error { if o.tokens.Token == nil || o.tokens.RefreshToken == "" { return errRefreshAccessToken } provider, err := o.getProvider(issuer, clientID, scopes) if err != nil { return errRefreshAccessToken } oauthTokens, err := rp.RefreshTokens[*oidc.IDTokenClaims](context.TODO(), provider, o.tokens.RefreshToken, "", "") if err != nil { return errRefreshAccessToken } o.tokens.AccessToken = oauthTokens.AccessToken o.tokens.TokenType = oauthTokens.TokenType o.tokens.Expiry = oauthTokens.Expiry if oauthTokens.RefreshToken != "" { o.tokens.RefreshToken = oauthTokens.RefreshToken } return nil } // authenticate initiates the OpenID Connect device flow authentication process for the client. // It presents a user code for the end user to input in the device that has web access and waits for them to complete the authentication, // subsequently updating the client's tokens upon successful authentication. func (o *oidcClient) authenticate(issuer string, clientID string, audience string, scopes string) error { // Store the old transport and restore it in the end. oldTransport := o.httpClient.Transport o.oidcTransport.audience = audience o.httpClient.Transport = o.oidcTransport defer func() { o.httpClient.Transport = oldTransport }() provider, err := o.getProvider(issuer, clientID, scopes) if err != nil { return err } o.oidcTransport.deviceAuthorizationEndpoint = provider.GetDeviceAuthorizationEndpoint() resp, err := rp.DeviceAuthorization(context.TODO(), strings.Split(scopes, ","), provider, nil) if err != nil { return err } u, _ := url.Parse(resp.VerificationURIComplete) fmt.Printf("URL: %s\n", u.String()) fmt.Printf("Code: %s\n\n", resp.UserCode) _ = util.OpenBrowser(u.String()) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGINT) defer stop() token, err := rp.DeviceAccessToken(ctx, resp.DeviceCode, time.Duration(resp.Interval)*time.Second, provider) if err != nil { return err } if o.tokens.Token == nil { o.tokens.Token = &oauth2.Token{} } o.tokens.Expiry = time.Now().Add(time.Duration(token.ExpiresIn)) o.tokens.IDToken = token.IDToken o.tokens.AccessToken = token.AccessToken o.tokens.TokenType = token.TokenType if token.RefreshToken != "" { o.tokens.RefreshToken = token.RefreshToken } return nil } incus-7.0.0/client/incus_operations.go000066400000000000000000000075221517523235500200460ustar00rootroot00000000000000package incus import ( "fmt" "net/url" "github.com/gorilla/websocket" "github.com/lxc/incus/v7/shared/api" ) // GetOperationUUIDs returns a list of operation uuids. func (r *ProtocolIncus) GetOperationUUIDs() ([]string, error) { // Fetch the raw URL values. urls := []string{} baseURL := "/operations" _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetOperations returns a list of Operation struct. func (r *ProtocolIncus) GetOperations() ([]api.Operation, error) { apiOperations := map[string][]api.Operation{} // Fetch the raw value. _, err := r.queryStruct("GET", "/operations?recursion=1", nil, "", &apiOperations) if err != nil { return nil, err } // Turn it into a list of operations. operations := []api.Operation{} for _, v := range apiOperations { operations = append(operations, v...) } return operations, nil } // GetOperationsAllProjects returns a list of operations from all projects. func (r *ProtocolIncus) GetOperationsAllProjects() ([]api.Operation, error) { err := r.CheckExtension("operations_get_query_all_projects") if err != nil { return nil, err } apiOperations := map[string][]api.Operation{} path := "/operations" v := url.Values{} v.Set("recursion", "1") v.Set("all-projects", "true") // Fetch the raw value. _, err = r.queryStruct("GET", fmt.Sprintf("%s?%s", path, v.Encode()), nil, "", &apiOperations) if err != nil { return nil, err } // Turn it into a list of operations. operations := []api.Operation{} for _, v := range apiOperations { operations = append(operations, v...) } return operations, nil } // GetOperation returns an Operation entry for the provided uuid. func (r *ProtocolIncus) GetOperation(uuid string) (*api.Operation, string, error) { op := api.Operation{} // Fetch the raw value etag, err := r.queryStruct("GET", fmt.Sprintf("/operations/%s", url.PathEscape(uuid)), nil, "", &op) if err != nil { return nil, "", err } return &op, etag, nil } // GetOperationWait returns an Operation entry for the provided uuid once it's complete or hits the timeout. func (r *ProtocolIncus) GetOperationWait(uuid string, timeout int) (*api.Operation, string, error) { op := api.Operation{} // Unset the response header timeout so that the request does not time out. transport, err := r.getUnderlyingHTTPTransport() if err != nil { return nil, "", err } transport.ResponseHeaderTimeout = 0 // Fetch the raw value etag, err := r.queryStruct("GET", fmt.Sprintf("/operations/%s/wait?timeout=%d", url.PathEscape(uuid), timeout), nil, "", &op) if err != nil { return nil, "", err } return &op, etag, nil } // GetOperationWaitSecret returns an Operation entry for the provided uuid and secret once it's complete or hits the timeout. func (r *ProtocolIncus) GetOperationWaitSecret(uuid string, secret string, timeout int) (*api.Operation, string, error) { op := api.Operation{} // Fetch the raw value etag, err := r.queryStruct("GET", fmt.Sprintf("/operations/%s/wait?secret=%s&timeout=%d", url.PathEscape(uuid), url.PathEscape(secret), timeout), nil, "", &op) if err != nil { return nil, "", err } return &op, etag, nil } // GetOperationWebsocket returns a websocket connection for the provided operation. func (r *ProtocolIncus) GetOperationWebsocket(uuid string, secret string) (*websocket.Conn, error) { path := fmt.Sprintf("/operations/%s/websocket", url.PathEscape(uuid)) if secret != "" { path = fmt.Sprintf("%s?secret=%s", path, url.QueryEscape(secret)) } return r.websocket(path) } // DeleteOperation deletes (cancels) a running operation. func (r *ProtocolIncus) DeleteOperation(uuid string) error { // Send the request _, _, err := r.query("DELETE", fmt.Sprintf("/operations/%s", url.PathEscape(uuid)), nil, "") if err != nil { return err } return nil } incus-7.0.0/client/incus_profiles.go000066400000000000000000000075721517523235500175130ustar00rootroot00000000000000package incus import ( "errors" "fmt" "net/url" "github.com/lxc/incus/v7/shared/api" ) // Profile handling functions // GetProfileNames returns a list of available profile names. func (r *ProtocolIncus) GetProfileNames() ([]string, error) { // Fetch the raw URL values. urls := []string{} baseURL := "/profiles" _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetProfiles returns a list of available Profile structs. func (r *ProtocolIncus) GetProfiles() ([]api.Profile, error) { profiles := []api.Profile{} // Fetch the raw value _, err := r.queryStruct("GET", "/profiles?recursion=1", nil, "", &profiles) if err != nil { return nil, err } return profiles, nil } // GetProfilesWithFilter returns a filtered list of available Profile structs. func (r *ProtocolIncus) GetProfilesWithFilter(filters []string) ([]api.Profile, error) { profiles := []api.Profile{} v := url.Values{} v.Set("recursion", "1") v.Set("filter", parseFilters(filters)) _, err := r.queryStruct("GET", fmt.Sprintf("/profiles?%s", v.Encode()), nil, "", &profiles) if err != nil { return nil, err } return profiles, nil } // GetProfilesAllProjects returns a list of profiles across all projects as Profile structs. func (r *ProtocolIncus) GetProfilesAllProjects() ([]api.Profile, error) { err := r.CheckExtension("profiles_all_projects") if err != nil { return nil, errors.New(`The server is missing the required "profiles_all_projects" API extension`) } profiles := []api.Profile{} _, err = r.queryStruct("GET", "/profiles?recursion=1&all-projects=true", nil, "", &profiles) if err != nil { return nil, err } return profiles, nil } // GetProfilesAllProjectsWithFilter returns a filtered list of profiles across all projects as Profile structs. func (r *ProtocolIncus) GetProfilesAllProjectsWithFilter(filters []string) ([]api.Profile, error) { err := r.CheckExtension("profiles_all_projects") if err != nil { return nil, errors.New(`The server is missing the required "profiles_all_projects" API extension`) } profiles := []api.Profile{} v := url.Values{} v.Set("recursion", "1") v.Set("all-projects", "true") v.Set("filter", parseFilters(filters)) _, err = r.queryStruct("GET", fmt.Sprintf("/profiles?%s", v.Encode()), nil, "", &profiles) if err != nil { return nil, err } return profiles, nil } // GetProfile returns a Profile entry for the provided name. func (r *ProtocolIncus) GetProfile(name string) (*api.Profile, string, error) { profile := api.Profile{} // Fetch the raw value etag, err := r.queryStruct("GET", fmt.Sprintf("/profiles/%s", url.PathEscape(name)), nil, "", &profile) if err != nil { return nil, "", err } return &profile, etag, nil } // CreateProfile defines a new instance profile. func (r *ProtocolIncus) CreateProfile(profile api.ProfilesPost) error { // Send the request _, _, err := r.query("POST", "/profiles", profile, "") if err != nil { return err } return nil } // UpdateProfile updates the profile to match the provided Profile struct. func (r *ProtocolIncus) UpdateProfile(name string, profile api.ProfilePut, ETag string) error { // Send the request _, _, err := r.query("PUT", fmt.Sprintf("/profiles/%s", url.PathEscape(name)), profile, ETag) if err != nil { return err } return nil } // RenameProfile renames an existing profile entry. func (r *ProtocolIncus) RenameProfile(name string, profile api.ProfilePost) error { // Send the request _, _, err := r.query("POST", fmt.Sprintf("/profiles/%s", url.PathEscape(name)), profile, "") if err != nil { return err } return nil } // DeleteProfile deletes a profile. func (r *ProtocolIncus) DeleteProfile(name string) error { // Send the request _, _, err := r.query("DELETE", fmt.Sprintf("/profiles/%s", url.PathEscape(name)), nil, "") if err != nil { return err } return nil } incus-7.0.0/client/incus_projects.go000066400000000000000000000122331517523235500175070ustar00rootroot00000000000000package incus import ( "errors" "fmt" "net/url" "github.com/lxc/incus/v7/shared/api" ) // Project handling functions // GetProjectNames returns a list of available project names. func (r *ProtocolIncus) GetProjectNames() ([]string, error) { if !r.HasExtension("projects") { return nil, errors.New("The server is missing the required \"projects\" API extension") } // Fetch the raw URL values. urls := []string{} baseURL := "/projects" _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetProjects returns a list of available Project structs. func (r *ProtocolIncus) GetProjects() ([]api.Project, error) { if !r.HasExtension("projects") { return nil, errors.New("The server is missing the required \"projects\" API extension") } projects := []api.Project{} // Fetch the raw value _, err := r.queryStruct("GET", "/projects?recursion=1", nil, "", &projects) if err != nil { return nil, err } return projects, nil } // GetProjectsWithFilter returns a filtered list of projects as Project structs. func (r *ProtocolIncus) GetProjectsWithFilter(filters []string) ([]api.Project, error) { if !r.HasExtension("projects") { return nil, errors.New("The server is missing the required \"projects\" API extension") } projects := []api.Project{} v := url.Values{} v.Set("recursion", "1") v.Set("filter", parseFilters(filters)) _, err := r.queryStruct("GET", fmt.Sprintf("/projects?%s", v.Encode()), nil, "", &projects) if err != nil { return nil, err } return projects, nil } // GetProject returns a Project entry for the provided name. func (r *ProtocolIncus) GetProject(name string) (*api.Project, string, error) { if !r.HasExtension("projects") { return nil, "", errors.New("The server is missing the required \"projects\" API extension") } project := api.Project{} // Fetch the raw value etag, err := r.queryStruct("GET", fmt.Sprintf("/projects/%s", url.PathEscape(name)), nil, "", &project) if err != nil { return nil, "", err } return &project, etag, nil } // GetProjectState returns a Project state for the provided name. func (r *ProtocolIncus) GetProjectState(name string) (*api.ProjectState, error) { if !r.HasExtension("project_usage") { return nil, errors.New("The server is missing the required \"project_usage\" API extension") } projectState := api.ProjectState{} // Fetch the raw value _, err := r.queryStruct("GET", fmt.Sprintf("/projects/%s/state", url.PathEscape(name)), nil, "", &projectState) if err != nil { return nil, err } return &projectState, nil } // GetProjectAccess returns an Access entry for the specified project. func (r *ProtocolIncus) GetProjectAccess(name string) (api.Access, error) { access := api.Access{} if !r.HasExtension("project_access") { return nil, errors.New("The server is missing the required \"project_access\" API extension") } // Fetch the raw value _, err := r.queryStruct("GET", fmt.Sprintf("/projects/%s/access", url.PathEscape(name)), nil, "", &access) if err != nil { return nil, err } return access, nil } // CreateProject defines a new project. func (r *ProtocolIncus) CreateProject(project api.ProjectsPost) error { if !r.HasExtension("projects") { return errors.New("The server is missing the required \"projects\" API extension") } // Send the request _, _, err := r.query("POST", "/projects", project, "") if err != nil { return err } return nil } // UpdateProject updates the project to match the provided Project struct. func (r *ProtocolIncus) UpdateProject(name string, project api.ProjectPut, ETag string) error { if !r.HasExtension("projects") { return errors.New("The server is missing the required \"projects\" API extension") } // Send the request _, _, err := r.query("PUT", fmt.Sprintf("/projects/%s", url.PathEscape(name)), project, ETag) if err != nil { return err } return nil } // RenameProject renames an existing project entry. func (r *ProtocolIncus) RenameProject(name string, project api.ProjectPost) (Operation, error) { if !r.HasExtension("projects") { return nil, errors.New("The server is missing the required \"projects\" API extension") } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("/projects/%s", url.PathEscape(name)), project, "") if err != nil { return nil, err } return op, nil } // DeleteProject deletes a project. func (r *ProtocolIncus) DeleteProject(name string) error { if !r.HasExtension("projects") { return errors.New("The server is missing the required \"projects\" API extension") } // Send the request _, _, err := r.query("DELETE", fmt.Sprintf("/projects/%s", url.PathEscape(name)), nil, "") if err != nil { return err } return nil } // DeleteProjectForce deletes a project and everything inside of it. func (r *ProtocolIncus) DeleteProjectForce(name string) error { if !r.HasExtension("projects_force_delete") { return errors.New("The server is missing the required \"projects_force_delete\" API extension") } // Send the request _, _, err := r.query("DELETE", fmt.Sprintf("/projects/%s?force=1", url.PathEscape(name)), nil, "") if err != nil { return err } return nil } incus-7.0.0/client/incus_server.go000066400000000000000000000425371517523235500171760ustar00rootroot00000000000000package incus import ( "errors" "fmt" "io" "net/http" "slices" "github.com/gorilla/websocket" "github.com/lxc/incus/v7/shared/api" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) // Server handling functions // GetServer returns the server status as a Server struct. func (r *ProtocolIncus) GetServer() (*api.Server, string, error) { server := api.Server{} // Fetch the raw value etag, err := r.queryStruct("GET", "", nil, "", &server) if err != nil { return nil, "", err } // Fill in certificate fingerprint if not provided if server.Environment.CertificateFingerprint == "" && server.Environment.Certificate != "" { var err error server.Environment.CertificateFingerprint, err = localtls.CertFingerprintStr(server.Environment.Certificate) if err != nil { return nil, "", err } } if !server.Public && len(server.AuthMethods) == 0 { // TLS is always available for Incus servers server.AuthMethods = []string{api.AuthenticationMethodTLS} } // Add the value to the cache r.server = &server return &server, etag, nil } // UpdateServer updates the server status to match the provided Server struct. func (r *ProtocolIncus) UpdateServer(server api.ServerPut, ETag string) error { // Send the request _, _, err := r.query("PUT", "", server, ETag) if err != nil { return err } return nil } // HasExtension returns true if the server supports a given API extension. // Deprecated: Use CheckExtension instead. func (r *ProtocolIncus) HasExtension(extension string) bool { // If no cached API information, just assume we're good // This is needed for those rare cases where we must avoid a GetServer call if r.server == nil { return true } return slices.Contains(r.server.APIExtensions, extension) } // CheckExtension checks if the server has the specified extension. func (r *ProtocolIncus) CheckExtension(extensionName string) error { if !r.HasExtension(extensionName) { return fmt.Errorf("The server is missing the required %q API extension", extensionName) } return nil } // IsClustered returns true if the server is part of an Incus cluster. func (r *ProtocolIncus) IsClustered() bool { return r.server.Environment.ServerClustered } // GetServerResources returns the resources available to a given Incus server. func (r *ProtocolIncus) GetServerResources() (*api.Resources, error) { if !r.HasExtension("resources") { return nil, errors.New("The server is missing the required \"resources\" API extension") } resources := api.Resources{} // Fetch the raw value _, err := r.queryStruct("GET", "/resources", nil, "", &resources) if err != nil { return nil, err } return &resources, nil } // UseProject returns a client that will use a specific project. func (r *ProtocolIncus) UseProject(name string) InstanceServer { return &ProtocolIncus{ ctx: r.ctx, ctxConnected: r.ctxConnected, ctxConnectedCancel: r.ctxConnectedCancel, server: r.server, http: r.http, httpCertificate: r.httpCertificate, httpBaseURL: r.httpBaseURL, httpProtocol: r.httpProtocol, httpUserAgent: r.httpUserAgent, httpUnixPath: r.httpUnixPath, requireAuthenticated: r.requireAuthenticated, clusterTarget: r.clusterTarget, project: name, eventConns: make(map[string]*websocket.Conn), // New project specific listener conns. eventListeners: make(map[string][]*EventListener), // New project specific listeners. skipEvents: r.skipEvents, oidcClient: r.oidcClient, } } // UseTarget returns a client that will target a specific cluster member. // Use this member-specific operations such as specific container // placement, preparing a new storage pool or network, ... func (r *ProtocolIncus) UseTarget(name string) InstanceServer { return &ProtocolIncus{ ctx: r.ctx, ctxConnected: r.ctxConnected, ctxConnectedCancel: r.ctxConnectedCancel, server: r.server, http: r.http, httpCertificate: r.httpCertificate, httpBaseURL: r.httpBaseURL, httpProtocol: r.httpProtocol, httpUserAgent: r.httpUserAgent, httpUnixPath: r.httpUnixPath, requireAuthenticated: r.requireAuthenticated, project: r.project, eventConns: make(map[string]*websocket.Conn), // New target specific listener conns. eventListeners: make(map[string][]*EventListener), // New target specific listeners. skipEvents: r.skipEvents, oidcClient: r.oidcClient, clusterTarget: name, } } // IsAgent returns true if the server is an Incus agent. func (r *ProtocolIncus) IsAgent() bool { return r.server != nil && r.server.Environment.Server == "incus-agent" } // GetMetrics returns the text OpenMetrics data. func (r *ProtocolIncus) GetMetrics() (string, error) { // Check that the server supports it. if !r.HasExtension("metrics") { return "", errors.New("The server is missing the required \"metrics\" API extension") } // Prepare the request. requestURL, err := r.setQueryAttributes(fmt.Sprintf("%s/1.0/metrics", r.httpBaseURL.String())) if err != nil { return "", err } req, err := http.NewRequest("GET", requestURL, nil) if err != nil { return "", err } // Send the request. resp, err := r.DoHTTP(req) if err != nil { return "", err } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("Bad HTTP status: %d", resp.StatusCode) } // Get the content. content, err := io.ReadAll(resp.Body) if err != nil { return "", err } return string(content), nil } // ApplyServerPreseed configures a target Incus server with the provided server and cluster configuration. func (r *ProtocolIncus) ApplyServerPreseed(config api.InitPreseed) error { // Apply server configuration. if len(config.Config) > 0 { // Get current config. server, etag, err := r.GetServer() if err != nil { return fmt.Errorf("Failed to retrieve current server configuration: %w", err) } for k, v := range config.Config { server.Config[k] = fmt.Sprintf("%v", v) } // Apply it. err = r.UpdateServer(server.Writable(), etag) if err != nil { return fmt.Errorf("Failed to update server configuration: %w", err) } } // Apply storage configuration. if len(config.StoragePools) > 0 { // Get the list of storagePools. storagePoolNames, err := r.GetStoragePoolNames() if err != nil { return fmt.Errorf("Failed to retrieve list of storage pools: %w", err) } // StoragePool creator createStoragePool := func(storagePool api.StoragePoolsPost) error { // Create the storagePool if doesn't exist. err := r.CreateStoragePool(storagePool) if err != nil { return fmt.Errorf("Failed to create storage pool %q: %w", storagePool.Name, err) } return nil } // StoragePool updater. updateStoragePool := func(target api.StoragePoolsPost) error { // Get the current storagePool. storagePool, etag, err := r.GetStoragePool(target.Name) if err != nil { return fmt.Errorf("Failed to retrieve current storage pool %q: %w", target.Name, err) } // Quick check. if storagePool.Driver != target.Driver { return fmt.Errorf("Storage pool %q is of type %q instead of %q", storagePool.Name, storagePool.Driver, target.Driver) } // Description override. if target.Description != "" { storagePool.Description = target.Description } // Config overrides. for k, v := range target.Config { storagePool.Config[k] = fmt.Sprintf("%v", v) } // Apply it. err = r.UpdateStoragePool(target.Name, storagePool.Writable(), etag) if err != nil { return fmt.Errorf("Failed to update storage pool %q: %w", target.Name, err) } return nil } for _, storagePool := range config.StoragePools { // New storagePool. if !slices.Contains(storagePoolNames, storagePool.Name) { err := createStoragePool(storagePool) if err != nil { return err } continue } // Existing storagePool. err := updateStoragePool(storagePool) if err != nil { return err } } } // Apply network configuration function. applyNetwork := func(target api.InitNetworksProjectPost) error { network, etag, err := r.UseProject(target.Project).GetNetwork(target.Name) if err != nil { // Create the network if doesn't exist. err := r.UseProject(target.Project).CreateNetwork(target.NetworksPost) if err != nil { return fmt.Errorf("Failed to create local member network %q in project %q: %w", target.Name, target.Project, err) } } else { // Description override. if target.Description != "" { network.Description = target.Description } // Config overrides. for k, v := range target.Config { network.Config[k] = fmt.Sprintf("%v", v) } // Apply it. err = r.UseProject(target.Project).UpdateNetwork(target.Name, network.Writable(), etag) if err != nil { return fmt.Errorf("Failed to update local member network %q in project %q: %w", target.Name, target.Project, err) } } return nil } // Apply networks in the default project before other projects config applied (so that if the projects // depend on a network in the default project they can have their config applied successfully). for i := range config.Networks { // Populate default project if not specified for backwards compatibility with earlier // preseed dump files. if config.Networks[i].Project == "" { config.Networks[i].Project = api.ProjectDefaultName } if config.Networks[i].Project != api.ProjectDefaultName { continue } err := applyNetwork(config.Networks[i]) if err != nil { return err } } // Apply project configuration. if len(config.Projects) > 0 { // Get the list of projects. projectNames, err := r.GetProjectNames() if err != nil { return fmt.Errorf("Failed to retrieve list of projects: %w", err) } // Project creator. createProject := func(project api.ProjectsPost) error { // Create the project if doesn't exist. err := r.CreateProject(project) if err != nil { return fmt.Errorf("Failed to create local member project %q: %w", project.Name, err) } return nil } // Project updater. updateProject := func(target api.ProjectsPost) error { // Get the current project. project, etag, err := r.GetProject(target.Name) if err != nil { return fmt.Errorf("Failed to retrieve current project %q: %w", target.Name, err) } // Description override. if target.Description != "" { project.Description = target.Description } // Config overrides. for k, v := range target.Config { project.Config[k] = fmt.Sprintf("%v", v) } // Apply it. err = r.UpdateProject(target.Name, project.Writable(), etag) if err != nil { return fmt.Errorf("Failed to update local member project %q: %w", target.Name, err) } return nil } for _, project := range config.Projects { // New project. if !slices.Contains(projectNames, project.Name) { err := createProject(project) if err != nil { return err } continue } // Existing project. err := updateProject(project) if err != nil { return err } } } // Apply networks in non-default projects after project config applied (so that their projects exist). for i := range config.Networks { if config.Networks[i].Project == api.ProjectDefaultName { continue } err := applyNetwork(config.Networks[i]) if err != nil { return err } } // Apply storage volumes configuration. applyStorageVolume := func(storageVolume api.InitStorageVolumesProjectPost) error { // Get the current storageVolume. currentStorageVolume, etag, err := r.UseProject(storageVolume.Project).GetStoragePoolVolume(storageVolume.Pool, storageVolume.Type, storageVolume.Name) if err != nil { // Create the storage volume if it doesn't exist. err := r.UseProject(storageVolume.Project).CreateStoragePoolVolume(storageVolume.Pool, storageVolume.StorageVolumesPost) if err != nil { return fmt.Errorf("Failed to create storage volume %q in project %q on pool %q: %w", storageVolume.Name, storageVolume.Project, storageVolume.Pool, err) } } else { // Quick check. if currentStorageVolume.Type != storageVolume.Type { return fmt.Errorf("Storage volume %q in project %q is of type %q instead of %q", currentStorageVolume.Name, storageVolume.Project, currentStorageVolume.Type, storageVolume.Type) } // Prepare the update. newStorageVolume := api.StorageVolumePut{} err = util.DeepCopy(currentStorageVolume.Writable(), &newStorageVolume) if err != nil { return fmt.Errorf("Failed to copy configuration of storage volume %q in project %q: %w", storageVolume.Name, storageVolume.Project, err) } // Description override. if storageVolume.Description != "" { newStorageVolume.Description = storageVolume.Description } // Config overrides. for k, v := range storageVolume.Config { newStorageVolume.Config[k] = fmt.Sprintf("%v", v) } // Apply it. err = r.UseProject(storageVolume.Project).UpdateStoragePoolVolume(storageVolume.Pool, storageVolume.Type, currentStorageVolume.Name, newStorageVolume, etag) if err != nil { return fmt.Errorf("Failed to update storage volume %q in project %q: %w", storageVolume.Name, storageVolume.Project, err) } } return nil } // Apply storage volumes in the default project before other projects config. for i := range config.StorageVolumes { // Populate default project if not specified. if config.StorageVolumes[i].Project == "" { config.StorageVolumes[i].Project = api.ProjectDefaultName } // Populate default type if not specified. if config.StorageVolumes[i].Type == "" { config.StorageVolumes[i].Type = "custom" } err := applyStorageVolume(config.StorageVolumes[i]) if err != nil { return err } } // Apply profile configuration. if len(config.Profiles) > 0 { // Apply profile configuration. applyProfile := func(profile api.InitProfileProjectPost) error { // Get the current profile. currentProfile, etag, err := r.UseProject(profile.Project).GetProfile(profile.Name) if err != nil { // // Create the profile if it doesn't exist. err := r.UseProject(profile.Project).CreateProfile(profile.ProfilesPost) if err != nil { return fmt.Errorf("Failed to create profile %q in project %q: %w", profile.Name, profile.Project, err) } } else { // Prepare the update. updatedProfile := api.ProfilePut{} err = util.DeepCopy(currentProfile.Writable(), &updatedProfile) if err != nil { return fmt.Errorf("Failed to copy configuration of profile %q in project %q: %w", profile.Name, profile.Project, err) } // Description override. if profile.Description != "" { updatedProfile.Description = profile.Description } // Config overrides. for k, v := range profile.Config { updatedProfile.Config[k] = fmt.Sprintf("%v", v) } // Device overrides. for k, v := range profile.Devices { // New device. _, ok := updatedProfile.Devices[k] if !ok { updatedProfile.Devices[k] = v continue } // Existing device. for configKey, configValue := range v { updatedProfile.Devices[k][configKey] = fmt.Sprintf("%v", configValue) } } // Apply it. err = r.UseProject(profile.Project).UpdateProfile(profile.Name, updatedProfile, etag) if err != nil { return fmt.Errorf("Failed to update profile %q in project %q: %w", profile.Name, profile.Project, err) } } return nil } for _, profile := range config.Profiles { if profile.Project == "" { profile.Project = api.ProjectDefaultName } err := applyProfile(profile) if err != nil { return err } } } // Apply certificate configuration. if len(config.Certificates) > 0 { for _, certificate := range config.Certificates { err := r.CreateCertificate(certificate) if err != nil { return fmt.Errorf("Failed to create certificate %q: %w", certificate.Name, err) } } } // Cluster configuration. if config.Cluster != nil && config.Cluster.Enabled { // Get the current cluster configuration currentCluster, etag, err := r.GetCluster() if err != nil { return fmt.Errorf("Failed to retrieve current cluster config: %w", err) } // Check if already enabled if !currentCluster.Enabled { // Configure the cluster op, err := r.UpdateCluster(config.Cluster.ClusterPut, etag) if err != nil { return fmt.Errorf("Failed to configure cluster: %w", err) } err = op.Wait() if err != nil { return fmt.Errorf("Failed to configure cluster: %w", err) } } } // Apply cluster group configurations. if len(config.ClusterGroups) > 0 { for _, clusterGroup := range config.ClusterGroups { // Check if it already exists. existing, etag, err := r.GetClusterGroup(clusterGroup.Name) if err == nil && existing != nil { // Keep existing members if none specified (set of empty slice to empty). if clusterGroup.Members == nil { clusterGroup.Members = existing.Members } // Update the existing group. err = r.UpdateClusterGroup(clusterGroup.Name, clusterGroup.ClusterGroupPut, etag) if err != nil { return fmt.Errorf("Failed to update cluster group") } continue } // Create the new group. err = r.CreateClusterGroup(clusterGroup) if err != nil { return fmt.Errorf("Failed to create cluster group") } } } return nil } incus-7.0.0/client/incus_storage_buckets.go000066400000000000000000000440671517523235500210540ustar00rootroot00000000000000package incus import ( "bytes" "encoding/json" "errors" "fmt" "net/http" "net/url" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/cancel" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // GetStoragePoolBucketNames returns a list of storage bucket names. func (r *ProtocolIncus) GetStoragePoolBucketNames(poolName string) ([]string, error) { err := r.CheckExtension("storage_buckets") if err != nil { return nil, err } // Fetch the raw URL values. urls := []string{} u := api.NewURL().Path("storage-pools", poolName, "buckets") _, err = r.queryStruct("GET", u.String(), nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(u.String(), urls...) } // GetStoragePoolBuckets returns a list of storage buckets for the provided pool. func (r *ProtocolIncus) GetStoragePoolBuckets(poolName string) ([]api.StorageBucket, error) { err := r.CheckExtension("storage_buckets") if err != nil { return nil, err } buckets := []api.StorageBucket{} // Fetch the raw value. u := api.NewURL().Path("storage-pools", poolName, "buckets").WithQuery("recursion", "1") _, err = r.queryStruct("GET", u.String(), nil, "", &buckets) if err != nil { return nil, err } return buckets, nil } // GetStoragePoolBucketsWithFilter returns a filtered list of storage buckets for the provided pool. func (r *ProtocolIncus) GetStoragePoolBucketsWithFilter(poolName string, filters []string) ([]api.StorageBucket, error) { err := r.CheckExtension("storage_buckets") if err != nil { return nil, err } buckets := []api.StorageBucket{} // Fetch the raw value u := api.NewURL().Path("storage-pools", poolName, "buckets"). WithQuery("recursion", "1"). WithQuery("filter", parseFilters(filters)) _, err = r.queryStruct("GET", u.String(), nil, "", &buckets) if err != nil { return nil, err } return buckets, nil } // GetStoragePoolBucketsAllProjects gets all storage pool buckets across all projects. func (r *ProtocolIncus) GetStoragePoolBucketsAllProjects(poolName string) ([]api.StorageBucket, error) { err := r.CheckExtension("storage_buckets_all_projects") if err != nil { return nil, errors.New(`The server is missing the required "storage_buckets_all_projects" API extension`) } buckets := []api.StorageBucket{} u := api.NewURL().Path("storage-pools", poolName, "buckets").WithQuery("recursion", "1").WithQuery("all-projects", "true") _, err = r.queryStruct("GET", u.String(), nil, "", &buckets) if err != nil { return nil, err } return buckets, nil } // GetStoragePoolBucketsWithFilterAllProjects gets a filtered list of storage pool buckets across all projects. func (r *ProtocolIncus) GetStoragePoolBucketsWithFilterAllProjects(poolName string, filters []string) ([]api.StorageBucket, error) { err := r.CheckExtension("storage_buckets") if err != nil { return nil, err } err = r.CheckExtension("storage_buckets_all_projects") if err != nil { return nil, errors.New(`The server is missing the required "storage_buckets_all_projects" API extension`) } buckets := []api.StorageBucket{} u := api.NewURL().Path("storage-pools", poolName, "buckets"). WithQuery("recursion", "1"). WithQuery("filter", parseFilters(filters)). WithQuery("all-projects", "true") _, err = r.queryStruct("GET", u.String(), nil, "", &buckets) if err != nil { return nil, err } return buckets, nil } // GetStoragePoolBucketsFull returns a list of storage buckets for the provided pool (full struct). func (r *ProtocolIncus) GetStoragePoolBucketsFull(poolName string) ([]api.StorageBucketFull, error) { err := r.CheckExtension("storage_bucket_full") if err != nil { return nil, err } buckets := []api.StorageBucketFull{} // Fetch the raw value. u := api.NewURL().Path("storage-pools", poolName, "buckets").WithQuery("recursion", "2") _, err = r.queryStruct("GET", u.String(), nil, "", &buckets) if err != nil { return nil, err } return buckets, nil } // GetStoragePoolBucketsFullWithFilter returns a filtered list of storage buckets for the provided pool (full struct). func (r *ProtocolIncus) GetStoragePoolBucketsFullWithFilter(poolName string, filters []string) ([]api.StorageBucketFull, error) { err := r.CheckExtension("storage_bucket_full") if err != nil { return nil, err } buckets := []api.StorageBucketFull{} // Fetch the raw value u := api.NewURL().Path("storage-pools", poolName, "buckets"). WithQuery("recursion", "2"). WithQuery("filter", parseFilters(filters)) _, err = r.queryStruct("GET", u.String(), nil, "", &buckets) if err != nil { return nil, err } return buckets, nil } // GetStoragePoolBucketsFullAllProjects gets all storage pool buckets across all projects (full struct). func (r *ProtocolIncus) GetStoragePoolBucketsFullAllProjects(poolName string) ([]api.StorageBucketFull, error) { err := r.CheckExtension("storage_bucket_full") if err != nil { return nil, errors.New(`The server is missing the required "storage_bucket_full" API extension`) } buckets := []api.StorageBucketFull{} u := api.NewURL().Path("storage-pools", poolName, "buckets").WithQuery("recursion", "2").WithQuery("all-projects", "true") _, err = r.queryStruct("GET", u.String(), nil, "", &buckets) if err != nil { return nil, err } return buckets, nil } // GetStoragePoolBucketsFullWithFilterAllProjects gets a filtered list of storage pool buckets across all projects (full struct). func (r *ProtocolIncus) GetStoragePoolBucketsFullWithFilterAllProjects(poolName string, filters []string) ([]api.StorageBucketFull, error) { err := r.CheckExtension("storage_bucket_full") if err != nil { return nil, err } buckets := []api.StorageBucketFull{} u := api.NewURL().Path("storage-pools", poolName, "buckets"). WithQuery("recursion", "2"). WithQuery("filter", parseFilters(filters)). WithQuery("all-projects", "true") _, err = r.queryStruct("GET", u.String(), nil, "", &buckets) if err != nil { return nil, err } return buckets, nil } // GetStoragePoolBucket returns a storage bucket entry for the provided pool and bucket name. func (r *ProtocolIncus) GetStoragePoolBucket(poolName string, bucketName string) (*api.StorageBucket, string, error) { err := r.CheckExtension("storage_buckets") if err != nil { return nil, "", err } bucket := api.StorageBucket{} // Fetch the raw value. u := api.NewURL().Path("storage-pools", poolName, "buckets", bucketName) etag, err := r.queryStruct("GET", u.String(), nil, "", &bucket) if err != nil { return nil, "", err } return &bucket, etag, nil } // GetStoragePoolBucketFull returns a full storage bucket entry for the provided pool and bucket name. func (r *ProtocolIncus) GetStoragePoolBucketFull(poolName string, bucketName string) (*api.StorageBucketFull, string, error) { err := r.CheckExtension("storage_bucket_full") if err != nil { return nil, "", err } bucket := api.StorageBucketFull{} // Fetch the raw value. u := api.NewURL().Path("storage-pools", poolName, "buckets", bucketName).WithQuery("recursion", "1") etag, err := r.queryStruct("GET", u.String(), nil, "", &bucket) if err != nil { return nil, "", err } return &bucket, etag, nil } // CreateStoragePoolBucket defines a new storage bucket using the provided struct. // If the server supports storage_buckets_create_credentials API extension, then this function will return the // initial admin credentials. Otherwise it will be nil. func (r *ProtocolIncus) CreateStoragePoolBucket(poolName string, bucket api.StorageBucketsPost) (*api.StorageBucketKey, error) { err := r.CheckExtension("storage_buckets") if err != nil { return nil, err } u := api.NewURL().Path("storage-pools", poolName, "buckets") // Send the request and get the resulting key info (including generated keys). if r.HasExtension("storage_buckets_create_credentials") { var newKey api.StorageBucketKey _, err = r.queryStruct("POST", u.String(), bucket, "", &newKey) if err != nil { return nil, err } return &newKey, nil } _, _, err = r.query("POST", u.String(), bucket, "") if err != nil { return nil, err } return nil, nil } // UpdateStoragePoolBucket updates the storage bucket to match the provided struct. func (r *ProtocolIncus) UpdateStoragePoolBucket(poolName string, bucketName string, bucket api.StorageBucketPut, ETag string) error { err := r.CheckExtension("storage_buckets") if err != nil { return err } // Send the request. u := api.NewURL().Path("storage-pools", poolName, "buckets", bucketName) _, _, err = r.query("PUT", u.String(), bucket, ETag) if err != nil { return err } return nil } // DeleteStoragePoolBucket deletes an existing storage bucket. func (r *ProtocolIncus) DeleteStoragePoolBucket(poolName string, bucketName string) error { err := r.CheckExtension("storage_buckets") if err != nil { return err } // Send the request. u := api.NewURL().Path("storage-pools", poolName, "buckets", bucketName) _, _, err = r.query("DELETE", u.String(), nil, "") if err != nil { return err } return nil } // GetStoragePoolBucketKeyNames returns a list of storage bucket key names. func (r *ProtocolIncus) GetStoragePoolBucketKeyNames(poolName string, bucketName string) ([]string, error) { err := r.CheckExtension("storage_buckets") if err != nil { return nil, err } // Fetch the raw URL values. urls := []string{} u := api.NewURL().Path("storage-pools", poolName, "buckets", bucketName, "keys") _, err = r.queryStruct("GET", u.String(), nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(u.String(), urls...) } // GetStoragePoolBucketKeys returns a list of storage bucket keys for the provided pool and bucket. func (r *ProtocolIncus) GetStoragePoolBucketKeys(poolName string, bucketName string) ([]api.StorageBucketKey, error) { err := r.CheckExtension("storage_buckets") if err != nil { return nil, err } bucketKeys := []api.StorageBucketKey{} // Fetch the raw value. u := api.NewURL().Path("storage-pools", poolName, "buckets", bucketName, "keys").WithQuery("recursion", "1") _, err = r.queryStruct("GET", u.String(), nil, "", &bucketKeys) if err != nil { return nil, err } return bucketKeys, nil } // GetStoragePoolBucketKey returns a storage bucket key entry for the provided pool, bucket and key name. func (r *ProtocolIncus) GetStoragePoolBucketKey(poolName string, bucketName string, keyName string) (*api.StorageBucketKey, string, error) { err := r.CheckExtension("storage_buckets") if err != nil { return nil, "", err } bucketKey := api.StorageBucketKey{} // Fetch the raw value. u := api.NewURL().Path("storage-pools", poolName, "buckets", bucketName, "keys", keyName) etag, err := r.queryStruct("GET", u.String(), nil, "", &bucketKey) if err != nil { return nil, "", err } return &bucketKey, etag, nil } // CreateStoragePoolBucketKey adds a key to a storage bucket. func (r *ProtocolIncus) CreateStoragePoolBucketKey(poolName string, bucketName string, key api.StorageBucketKeysPost) (*api.StorageBucketKey, error) { err := r.CheckExtension("storage_buckets") if err != nil { return nil, err } // Send the request and get the resulting key info (including generated keys). var newKey api.StorageBucketKey u := api.NewURL().Path("storage-pools", poolName, "buckets", bucketName, "keys") _, err = r.queryStruct("POST", u.String(), key, "", &newKey) if err != nil { return nil, err } return &newKey, err } // UpdateStoragePoolBucketKey updates an existing storage bucket key. func (r *ProtocolIncus) UpdateStoragePoolBucketKey(poolName string, bucketName string, keyName string, key api.StorageBucketKeyPut, ETag string) error { err := r.CheckExtension("storage_buckets") if err != nil { return err } // Send the request. u := api.NewURL().Path("storage-pools", poolName, "buckets", bucketName, "keys", keyName) _, _, err = r.query("PUT", u.String(), key, ETag) if err != nil { return err } return nil } // DeleteStoragePoolBucketKey removes a key from a storage bucket. func (r *ProtocolIncus) DeleteStoragePoolBucketKey(poolName string, bucketName string, keyName string) error { err := r.CheckExtension("storage_buckets") if err != nil { return err } // Send the request. u := api.NewURL().Path("storage-pools", poolName, "buckets", bucketName, "keys", keyName) _, _, err = r.query("DELETE", u.String(), nil, "") if err != nil { return err } return nil } // CreateStoragePoolBucketBackup creates a new storage bucket backup. func (r *ProtocolIncus) CreateStoragePoolBucketBackup(poolName string, bucketName string, backup api.StorageBucketBackupsPost) (Operation, error) { err := r.CheckExtension("storage_bucket_backup") if err != nil { return nil, err } op, _, err := r.queryOperation("POST", fmt.Sprintf("/storage-pools/%s/buckets/%s/backups", url.PathEscape(poolName), url.PathEscape(bucketName)), backup, "") if err != nil { return nil, err } return op, nil } // DeleteStoragePoolBucketBackup deletes an existing storage bucket backup. func (r *ProtocolIncus) DeleteStoragePoolBucketBackup(pool string, bucketName string, name string) (Operation, error) { err := r.CheckExtension("storage_bucket_backup") if err != nil { return nil, err } op, _, err := r.queryOperation("DELETE", fmt.Sprintf("/storage-pools/%s/buckets/%s/backups/%s", url.PathEscape(pool), url.PathEscape(bucketName), url.PathEscape(name)), nil, "") if err != nil { return nil, err } return op, nil } // GetStoragePoolBucketBackupFile returns the storage bucket file. func (r *ProtocolIncus) GetStoragePoolBucketBackupFile(pool string, bucketName string, name string, req *BackupFileRequest) (*BackupFileResponse, error) { err := r.CheckExtension("storage_bucket_backup") if err != nil { return nil, err } // Build the URL uri := fmt.Sprintf("%s/1.0/storage-pools/%s/buckets/%s/backups/%s/export", r.httpBaseURL.String(), url.PathEscape(pool), url.PathEscape(bucketName), url.PathEscape(name)) if r.project != "" { uri += fmt.Sprintf("?project=%s", url.QueryEscape(r.project)) } // Prepare the download request request, err := http.NewRequest("GET", uri, nil) if err != nil { return nil, err } if r.httpUserAgent != "" { request.Header.Set("User-Agent", r.httpUserAgent) } // Start the request response, doneCh, err := cancel.CancelableDownload(req.Canceler, r.DoHTTP, request) if err != nil { return nil, err } defer func() { _ = response.Body.Close() }() defer close(doneCh) if response.StatusCode != http.StatusOK { _, _, err := incusParseResponse(response) if err != nil { return nil, err } } // Handle the data body := response.Body if req.ProgressHandler != nil { body = &ioprogress.ProgressReader{ ReadCloser: response.Body, Tracker: &ioprogress.ProgressTracker{ Length: response.ContentLength, Handler: func(percent int64, speed int64) { req.ProgressHandler(ioprogress.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, units.GetByteSizeString(speed, 2))}) }, }, } } size, err := util.SafeCopy(req.BackupFile, body) if err != nil { return nil, err } resp := BackupFileResponse{} resp.Size = size return &resp, nil } // CreateStoragePoolBucketBackupStream requests that Incus creates and returns new direct backup // for the storage bucket. func (r *ProtocolIncus) CreateStoragePoolBucketBackupStream(poolName string, bucketName string, backup api.StorageBucketBackupsPost, req *BackupFileRequest) error { if !r.HasExtension("direct_backup") { return errors.New("The server is missing the required \"direct_backup\" API extension") } // Build the URL uri := fmt.Sprintf("%s/1.0/storage-pools/%s/buckets/%s/backups", r.httpBaseURL.String(), url.PathEscape(poolName), url.PathEscape(bucketName)) if r.project != "" { uri += fmt.Sprintf("?project=%s", url.QueryEscape(r.project)) } // Encode the backup data buf := bytes.Buffer{} err := json.NewEncoder(&buf).Encode(backup) if err != nil { return err } // Prepare the download request request, err := http.NewRequest("POST", uri, bytes.NewReader(buf.Bytes())) if err != nil { return err } request.Header.Set("Accept", "application/octet-stream") if r.httpUserAgent != "" { request.Header.Set("User-Agent", r.httpUserAgent) } // Start the request response, doneCh, err := cancel.CancelableDownload(req.Canceler, r.DoHTTP, request) if err != nil { return err } defer func() { _ = response.Body.Close() }() defer close(doneCh) if response.StatusCode != http.StatusOK { _, _, err = incusParseResponse(response) if err != nil { return err } } // Handle the data body := response.Body if req.ProgressHandler != nil { body = &ioprogress.ProgressReader{ ReadCloser: response.Body, Tracker: &ioprogress.ProgressTracker{ Handler: func(received int64, speed int64) { req.ProgressHandler(ioprogress.ProgressData{Text: fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(received, 2), units.GetByteSizeString(speed, 2))}) }, }, } } _, err = util.SafeCopy(req.BackupFile, body) return err } // CreateStoragePoolBucketFromBackup creates a new storage bucket from a backup. func (r *ProtocolIncus) CreateStoragePoolBucketFromBackup(pool string, args StoragePoolBucketBackupArgs) (Operation, error) { if !r.HasExtension("storage_bucket_backup") { return nil, errors.New(`The server is missing the required "custom_volume_backup" API extension`) } path := fmt.Sprintf("/storage-pools/%s/buckets", url.PathEscape(pool)) // Prepare the HTTP request. reqURL, err := r.setQueryAttributes(fmt.Sprintf("%s/1.0%s", r.httpBaseURL.String(), path)) if err != nil { return nil, err } req, err := http.NewRequest("POST", reqURL, args.BackupFile) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/octet-stream") if args.Name != "" { req.Header.Set("X-Incus-name", args.Name) } // Send the request. resp, err := r.DoHTTP(req) if err != nil { return nil, err } defer func() { _ = resp.Body.Close() }() // Handle errors. response, _, err := incusParseResponse(resp) if err != nil { return nil, err } respOperation, err := response.MetadataAsOperation() if err != nil { return nil, err } op := operation{ Operation: *respOperation, r: r, chActive: make(chan bool), } return &op, nil } incus-7.0.0/client/incus_storage_pools.go000066400000000000000000000076431517523235500205470ustar00rootroot00000000000000package incus import ( "errors" "fmt" "net/url" "github.com/lxc/incus/v7/shared/api" ) // Storage pool handling functions // GetStoragePoolNames returns the names of all storage pools. func (r *ProtocolIncus) GetStoragePoolNames() ([]string, error) { if !r.HasExtension("storage") { return nil, errors.New("The server is missing the required \"storage\" API extension") } // Fetch the raw URL values. urls := []string{} baseURL := "/storage-pools" _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetStoragePools returns a list of StoragePool entries. func (r *ProtocolIncus) GetStoragePools() ([]api.StoragePool, error) { if !r.HasExtension("storage") { return nil, errors.New("The server is missing the required \"storage\" API extension") } pools := []api.StoragePool{} // Fetch the raw value _, err := r.queryStruct("GET", "/storage-pools?recursion=1", nil, "", &pools) if err != nil { return nil, err } return pools, nil } // GetStoragePoolsWithFilter returns a filtered list of storage pools as StoragePool structs. func (r *ProtocolIncus) GetStoragePoolsWithFilter(filters []string) ([]api.StoragePool, error) { if !r.HasExtension("storage") { return nil, errors.New("The server is missing the required \"storage\" API extension") } pools := []api.StoragePool{} v := url.Values{} v.Set("recursion", "1") v.Set("filter", parseFilters(filters)) _, err := r.queryStruct("GET", fmt.Sprintf("/storage-pools?%s", v.Encode()), nil, "", &pools) if err != nil { return nil, err } return pools, nil } // GetStoragePool returns a StoragePool entry for the provided pool name. func (r *ProtocolIncus) GetStoragePool(name string) (*api.StoragePool, string, error) { if !r.HasExtension("storage") { return nil, "", errors.New("The server is missing the required \"storage\" API extension") } pool := api.StoragePool{} // Fetch the raw value etag, err := r.queryStruct("GET", fmt.Sprintf("/storage-pools/%s", url.PathEscape(name)), nil, "", &pool) if err != nil { return nil, "", err } return &pool, etag, nil } // CreateStoragePool defines a new storage pool using the provided StoragePool struct. func (r *ProtocolIncus) CreateStoragePool(pool api.StoragePoolsPost) error { if !r.HasExtension("storage") { return errors.New("The server is missing the required \"storage\" API extension") } // Send the request _, _, err := r.query("POST", "/storage-pools", pool, "") if err != nil { return err } return nil } // UpdateStoragePool updates the pool to match the provided StoragePool struct. func (r *ProtocolIncus) UpdateStoragePool(name string, pool api.StoragePoolPut, ETag string) error { if !r.HasExtension("storage") { return errors.New("The server is missing the required \"storage\" API extension") } // Send the request _, _, err := r.query("PUT", fmt.Sprintf("/storage-pools/%s", url.PathEscape(name)), pool, ETag) if err != nil { return err } return nil } // DeleteStoragePool deletes a storage pool. func (r *ProtocolIncus) DeleteStoragePool(name string) error { if !r.HasExtension("storage") { return errors.New("The server is missing the required \"storage\" API extension") } // Send the request _, _, err := r.query("DELETE", fmt.Sprintf("/storage-pools/%s", url.PathEscape(name)), nil, "") if err != nil { return err } return nil } // GetStoragePoolResources gets the resources available to a given storage pool. func (r *ProtocolIncus) GetStoragePoolResources(name string) (*api.ResourcesStoragePool, error) { if !r.HasExtension("resources") { return nil, errors.New("The server is missing the required \"resources\" API extension") } res := api.ResourcesStoragePool{} // Fetch the raw value _, err := r.queryStruct("GET", fmt.Sprintf("/storage-pools/%s/resources", url.PathEscape(name)), nil, "", &res) if err != nil { return nil, err } return &res, nil } incus-7.0.0/client/incus_storage_volumes.go000066400000000000000000001356511517523235500211060ustar00rootroot00000000000000package incus import ( "bytes" "encoding/json" "errors" "fmt" "io" "net" "net/http" "net/url" "strings" "github.com/pkg/sftp" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/cancel" "github.com/lxc/incus/v7/shared/ioprogress" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // Storage volumes handling function // GetStoragePoolVolumeNames returns the names of all volumes in a pool. func (r *ProtocolIncus) GetStoragePoolVolumeNames(pool string) ([]string, error) { if !r.HasExtension("storage") { return nil, errors.New("The server is missing the required \"storage\" API extension") } // Fetch the raw URL values. urls := []string{} baseURL := fmt.Sprintf("/storage-pools/%s/volumes", url.PathEscape(pool)) _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetStoragePoolVolumeNamesAllProjects returns the names of all volumes in a pool for all projects. func (r *ProtocolIncus) GetStoragePoolVolumeNamesAllProjects(pool string) (map[string][]string, error) { err := r.CheckExtension("storage") if err != nil { return nil, err } err = r.CheckExtension("storage_volumes_all_projects") if err != nil { return nil, err } // Fetch the raw URL values. urls := []string{} u := api.NewURL().Path("storage-pools", pool, "volumes").WithQuery("all-projects", "true") _, err = r.queryStruct("GET", u.String(), nil, "", &urls) if err != nil { return nil, err } names := make(map[string][]string) for _, urlString := range urls { resourceURL, err := url.Parse(urlString) if err != nil { return nil, fmt.Errorf("Could not parse unexpected URL %q: %w", urlString, err) } project := resourceURL.Query().Get("project") if project == "" { project = api.ProjectDefaultName } _, after, found := strings.Cut(resourceURL.Path, fmt.Sprintf("%s/", u.URL.Path)) if !found { return nil, fmt.Errorf("Unexpected URL path %q", resourceURL) } names[project] = append(names[project], after) } return names, nil } // GetStoragePoolVolumes returns a list of StorageVolume entries for the provided pool. func (r *ProtocolIncus) GetStoragePoolVolumes(pool string) ([]api.StorageVolume, error) { if !r.HasExtension("storage") { return nil, errors.New("The server is missing the required \"storage\" API extension") } volumes := []api.StorageVolume{} // Fetch the raw value _, err := r.queryStruct("GET", fmt.Sprintf("/storage-pools/%s/volumes?recursion=1", url.PathEscape(pool)), nil, "", &volumes) if err != nil { return nil, err } return volumes, nil } // GetStoragePoolVolumesAllProjects returns a list of StorageVolume entries for the provided pool for all projects. func (r *ProtocolIncus) GetStoragePoolVolumesAllProjects(pool string) ([]api.StorageVolume, error) { err := r.CheckExtension("storage") if err != nil { return nil, err } err = r.CheckExtension("storage_volumes_all_projects") if err != nil { return nil, err } volumes := []api.StorageVolume{} uri := api.NewURL().Path("storage-pools", pool, "volumes"). WithQuery("recursion", "1"). WithQuery("all-projects", "true") // Fetch the raw value. _, err = r.queryStruct("GET", uri.String(), nil, "", &volumes) if err != nil { return nil, err } return volumes, nil } // GetStoragePoolVolumesWithFilter returns a filtered list of StorageVolume entries for the provided pool. func (r *ProtocolIncus) GetStoragePoolVolumesWithFilter(pool string, filters []string) ([]api.StorageVolume, error) { if !r.HasExtension("storage") { return nil, errors.New("The server is missing the required \"storage\" API extension") } volumes := []api.StorageVolume{} v := url.Values{} v.Set("recursion", "1") v.Set("filter", parseFilters(filters)) // Fetch the raw value _, err := r.queryStruct("GET", fmt.Sprintf("/storage-pools/%s/volumes?%s", url.PathEscape(pool), v.Encode()), nil, "", &volumes) if err != nil { return nil, err } return volumes, nil } // GetStoragePoolVolumesWithFilterAllProjects returns a filtered list of StorageVolume entries for the provided pool for all projects. func (r *ProtocolIncus) GetStoragePoolVolumesWithFilterAllProjects(pool string, filters []string) ([]api.StorageVolume, error) { err := r.CheckExtension("storage") if err != nil { return nil, err } err = r.CheckExtension("storage_volumes_all_projects") if err != nil { return nil, err } volumes := []api.StorageVolume{} uri := api.NewURL().Path("storage-pools", pool, "volumes"). WithQuery("recursion", "1"). WithQuery("filter", parseFilters(filters)). WithQuery("all-projects", "true") // Fetch the raw value. _, err = r.queryStruct("GET", uri.String(), nil, "", &volumes) if err != nil { return nil, err } return volumes, nil } // GetStoragePoolVolumesFull returns a list of StorageVolume entries for the provided pool (full struct). func (r *ProtocolIncus) GetStoragePoolVolumesFull(pool string) ([]api.StorageVolumeFull, error) { if !r.HasExtension("storage_volume_full") { return nil, errors.New("The server is missing the required \"storage_volume_full\" API extension") } volumes := []api.StorageVolumeFull{} // Fetch the raw value _, err := r.queryStruct("GET", fmt.Sprintf("/storage-pools/%s/volumes?recursion=2", url.PathEscape(pool)), nil, "", &volumes) if err != nil { return nil, err } return volumes, nil } // GetStoragePoolVolumesFullAllProjects returns a list of StorageVolume entries for the provided pool for all projects (full struct). func (r *ProtocolIncus) GetStoragePoolVolumesFullAllProjects(pool string) ([]api.StorageVolumeFull, error) { err := r.CheckExtension("storage_volume_full") if err != nil { return nil, err } volumes := []api.StorageVolumeFull{} uri := api.NewURL().Path("storage-pools", pool, "volumes"). WithQuery("recursion", "2"). WithQuery("all-projects", "true") // Fetch the raw value. _, err = r.queryStruct("GET", uri.String(), nil, "", &volumes) if err != nil { return nil, err } return volumes, nil } // GetStoragePoolVolumesFullWithFilter returns a filtered list of StorageVolume entries for the provided pool (full struct). func (r *ProtocolIncus) GetStoragePoolVolumesFullWithFilter(pool string, filters []string) ([]api.StorageVolumeFull, error) { if !r.HasExtension("storage_volume_full") { return nil, errors.New("The server is missing the required \"storage_volume_full\" API extension") } volumes := []api.StorageVolumeFull{} v := url.Values{} v.Set("recursion", "2") v.Set("filter", parseFilters(filters)) // Fetch the raw value _, err := r.queryStruct("GET", fmt.Sprintf("/storage-pools/%s/volumes?%s", url.PathEscape(pool), v.Encode()), nil, "", &volumes) if err != nil { return nil, err } return volumes, nil } // GetStoragePoolVolumesFullWithFilterAllProjects returns a filtered list of StorageVolume entries for the provided pool for all projects (full struct). func (r *ProtocolIncus) GetStoragePoolVolumesFullWithFilterAllProjects(pool string, filters []string) ([]api.StorageVolumeFull, error) { err := r.CheckExtension("storage_volume_full") if err != nil { return nil, err } volumes := []api.StorageVolumeFull{} uri := api.NewURL().Path("storage-pools", pool, "volumes"). WithQuery("recursion", "2"). WithQuery("filter", parseFilters(filters)). WithQuery("all-projects", "true") // Fetch the raw value. _, err = r.queryStruct("GET", uri.String(), nil, "", &volumes) if err != nil { return nil, err } return volumes, nil } // GetStoragePoolVolume returns a StorageVolume entry for the provided pool and volume name. func (r *ProtocolIncus) GetStoragePoolVolume(pool string, volType string, name string) (*api.StorageVolume, string, error) { if !r.HasExtension("storage") { return nil, "", errors.New("The server is missing the required \"storage\" API extension") } volume := api.StorageVolume{} // Fetch the raw value path := fmt.Sprintf("/storage-pools/%s/volumes/%s/%s", url.PathEscape(pool), url.PathEscape(volType), url.PathEscape(name)) etag, err := r.queryStruct("GET", path, nil, "", &volume) if err != nil { return nil, "", err } return &volume, etag, nil } // GetStoragePoolVolumeFull returns a StorageVolumeFull entry for the provided pool and volume name. func (r *ProtocolIncus) GetStoragePoolVolumeFull(pool string, volType string, name string) (*api.StorageVolumeFull, string, error) { if !r.HasExtension("storage_volume_full") { return nil, "", errors.New("The server is missing the required \"storage_volume_full\" API extension") } volume := api.StorageVolumeFull{} // Fetch the raw value path := fmt.Sprintf("/storage-pools/%s/volumes/%s/%s?recursion=1", url.PathEscape(pool), url.PathEscape(volType), url.PathEscape(name)) etag, err := r.queryStruct("GET", path, nil, "", &volume) if err != nil { return nil, "", err } return &volume, etag, nil } // GetStoragePoolVolumeState returns a StorageVolumeState entry for the provided pool and volume name. func (r *ProtocolIncus) GetStoragePoolVolumeState(pool string, volType string, name string) (*api.StorageVolumeState, error) { if !r.HasExtension("storage_volume_state") { return nil, errors.New("The server is missing the required \"storage_volume_state\" API extension") } // Fetch the raw value state := api.StorageVolumeState{} path := fmt.Sprintf("/storage-pools/%s/volumes/%s/%s/state", url.PathEscape(pool), url.PathEscape(volType), url.PathEscape(name)) _, err := r.queryStruct("GET", path, nil, "", &state) if err != nil { return nil, err } return &state, nil } // CreateStoragePoolVolume defines a new storage volume. func (r *ProtocolIncus) CreateStoragePoolVolume(pool string, volume api.StorageVolumesPost) error { if !r.HasExtension("storage") { return errors.New("The server is missing the required \"storage\" API extension") } // Send the request path := fmt.Sprintf("/storage-pools/%s/volumes/%s", url.PathEscape(pool), url.PathEscape(volume.Type)) _, _, err := r.query("POST", path, volume, "") if err != nil { return err } return nil } // CreateStoragePoolVolumeSnapshot defines a new storage volume. func (r *ProtocolIncus) CreateStoragePoolVolumeSnapshot(pool string, volumeType string, volumeName string, snapshot api.StorageVolumeSnapshotsPost) (Operation, error) { if !r.HasExtension("storage_api_volume_snapshots") { return nil, errors.New("The server is missing the required \"storage_api_volume_snapshots\" API extension") } // Send the request path := fmt.Sprintf("/storage-pools/%s/volumes/%s/%s/snapshots", url.PathEscape(pool), url.PathEscape(volumeType), url.PathEscape(volumeName)) op, _, err := r.queryOperation("POST", path, snapshot, "") if err != nil { return nil, err } return op, nil } // GetStoragePoolVolumeSnapshotNames returns a list of snapshot names for the // storage volume. func (r *ProtocolIncus) GetStoragePoolVolumeSnapshotNames(pool string, volumeType string, volumeName string) ([]string, error) { if !r.HasExtension("storage_api_volume_snapshots") { return nil, errors.New("The server is missing the required \"storage_api_volume_snapshots\" API extension") } // Fetch the raw URL values. urls := []string{} baseURL := fmt.Sprintf("/storage-pools/%s/volumes/%s/%s/snapshots", url.PathEscape(pool), url.PathEscape(volumeType), url.PathEscape(volumeName)) _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetStoragePoolVolumeSnapshots returns a list of snapshots for the storage // volume. func (r *ProtocolIncus) GetStoragePoolVolumeSnapshots(pool string, volumeType string, volumeName string) ([]api.StorageVolumeSnapshot, error) { if !r.HasExtension("storage_api_volume_snapshots") { return nil, errors.New("The server is missing the required \"storage_api_volume_snapshots\" API extension") } snapshots := []api.StorageVolumeSnapshot{} path := fmt.Sprintf("/storage-pools/%s/volumes/%s/%s/snapshots?recursion=1", url.PathEscape(pool), url.PathEscape(volumeType), url.PathEscape(volumeName)) _, err := r.queryStruct("GET", path, nil, "", &snapshots) if err != nil { return nil, err } return snapshots, nil } // GetStoragePoolVolumeSnapshot returns a snapshots for the storage volume. func (r *ProtocolIncus) GetStoragePoolVolumeSnapshot(pool string, volumeType string, volumeName string, snapshotName string) (*api.StorageVolumeSnapshot, string, error) { if !r.HasExtension("storage_api_volume_snapshots") { return nil, "", errors.New("The server is missing the required \"storage_api_volume_snapshots\" API extension") } snapshot := api.StorageVolumeSnapshot{} path := fmt.Sprintf("/storage-pools/%s/volumes/%s/%s/snapshots/%s", url.PathEscape(pool), url.PathEscape(volumeType), url.PathEscape(volumeName), url.PathEscape(snapshotName)) etag, err := r.queryStruct("GET", path, nil, "", &snapshot) if err != nil { return nil, "", err } return &snapshot, etag, nil } // RenameStoragePoolVolumeSnapshot renames a storage volume snapshot. func (r *ProtocolIncus) RenameStoragePoolVolumeSnapshot(pool string, volumeType string, volumeName string, snapshotName string, snapshot api.StorageVolumeSnapshotPost) (Operation, error) { if !r.HasExtension("storage_api_volume_snapshots") { return nil, errors.New("The server is missing the required \"storage_api_volume_snapshots\" API extension") } path := fmt.Sprintf("/storage-pools/%s/volumes/%s/%s/snapshots/%s", url.PathEscape(pool), url.PathEscape(volumeType), url.PathEscape(volumeName), url.PathEscape(snapshotName)) // Send the request op, _, err := r.queryOperation("POST", path, snapshot, "") if err != nil { return nil, err } return op, nil } // DeleteStoragePoolVolumeSnapshot deletes a storage volume snapshot. func (r *ProtocolIncus) DeleteStoragePoolVolumeSnapshot(pool string, volumeType string, volumeName string, snapshotName string) (Operation, error) { if !r.HasExtension("storage_api_volume_snapshots") { return nil, errors.New("The server is missing the required \"storage_api_volume_snapshots\" API extension") } // Send the request path := fmt.Sprintf( "/storage-pools/%s/volumes/%s/%s/snapshots/%s", url.PathEscape(pool), url.PathEscape(volumeType), url.PathEscape(volumeName), url.PathEscape(snapshotName)) op, _, err := r.queryOperation("DELETE", path, nil, "") if err != nil { return nil, err } return op, nil } // UpdateStoragePoolVolumeSnapshot updates the volume to match the provided StoragePoolVolume struct. func (r *ProtocolIncus) UpdateStoragePoolVolumeSnapshot(pool string, volumeType string, volumeName string, snapshotName string, volume api.StorageVolumeSnapshotPut, ETag string) error { if !r.HasExtension("storage_api_volume_snapshots") { return errors.New("The server is missing the required \"storage_api_volume_snapshots\" API extension") } // Send the request path := fmt.Sprintf("/storage-pools/%s/volumes/%s/%s/snapshots/%s", url.PathEscape(pool), url.PathEscape(volumeType), url.PathEscape(volumeName), url.PathEscape(snapshotName)) _, _, err := r.queryOperation("PUT", path, volume, ETag) if err != nil { return err } return nil } // MigrateStoragePoolVolume requests that Incus prepares for a storage volume migration. func (r *ProtocolIncus) MigrateStoragePoolVolume(pool string, volume api.StorageVolumePost) (Operation, error) { if !r.HasExtension("storage_api_remote_volume_handling") { return nil, errors.New("The server is missing the required \"storage_api_remote_volume_handling\" API extension") } // Quick check. if !volume.Migration { return nil, errors.New("Can't ask for a rename through MigrateStoragePoolVolume") } var req any var path string srcVolParentName, srcVolSnapName, srcIsSnapshot := api.GetParentAndSnapshotName(volume.Name) if srcIsSnapshot { err := r.CheckExtension("storage_api_remote_volume_snapshot_copy") if err != nil { return nil, err } // Set the actual name of the snapshot without delimiter. req = api.StorageVolumeSnapshotPost{ Name: srcVolSnapName, Migration: volume.Migration, Target: volume.Target, } path = api.NewURL().Path("storage-pools", pool, "volumes", "custom", srcVolParentName, "snapshots", srcVolSnapName).String() } else { req = volume path = api.NewURL().Path("storage-pools", pool, "volumes", "custom", volume.Name).String() } // Send the request op, _, err := r.queryOperation("POST", path, req, "") if err != nil { return nil, err } return op, nil } func (r *ProtocolIncus) tryMigrateStoragePoolVolume(source InstanceServer, pool string, req api.StorageVolumePost, urls []string) (RemoteOperation, error) { if len(urls) == 0 { return nil, errors.New("The source server isn't listening on the network") } rop := remoteOperation{ chDone: make(chan bool), } operation := req.Target.Operation // Forward targetOp to remote op go func() { success := false var errors []remoteOperationResult for _, serverURL := range urls { req.Target.Operation = fmt.Sprintf("%s/1.0/operations/%s", serverURL, url.PathEscape(operation)) // Send the request top, err := source.MigrateStoragePoolVolume(pool, req) if err != nil { errors = append(errors, remoteOperationResult{URL: serverURL, Error: err}) continue } rop := remoteOperation{ targetOp: top, chDone: make(chan bool), } for _, handler := range rop.handlers { _, _ = rop.targetOp.AddHandler(handler) } err = rop.targetOp.Wait() if err != nil { errors = append(errors, remoteOperationResult{URL: serverURL, Error: err}) if localtls.IsConnectionError(err) { continue } break } success = true break } if !success { rop.err = remoteOperationError("Failed storage volume creation", errors) } close(rop.chDone) }() return &rop, nil } // tryCreateStoragePoolVolume attempts to create a storage volume in the specified storage pool. // It will try to do this on every server in the provided list of urls, and waits for the creation to be complete. func (r *ProtocolIncus) tryCreateStoragePoolVolume(pool string, req api.StorageVolumesPost, urls []string) (RemoteOperation, error) { if len(urls) == 0 { return nil, errors.New("The source server isn't listening on the network") } rop := remoteOperation{ chDone: make(chan bool), } operation := req.Source.Operation // Forward targetOp to remote op go func() { success := false var errors []remoteOperationResult for _, serverURL := range urls { req.Source.Operation = fmt.Sprintf("%s/1.0/operations/%s", serverURL, url.PathEscape(operation)) // Send the request path := fmt.Sprintf("/storage-pools/%s/volumes/%s", url.PathEscape(pool), url.PathEscape(req.Type)) top, _, err := r.queryOperation("POST", path, req, "") if err != nil { errors = append(errors, remoteOperationResult{URL: serverURL, Error: err}) continue } rop := remoteOperation{ targetOp: top, chDone: make(chan bool), } for _, handler := range rop.handlers { _, _ = rop.targetOp.AddHandler(handler) } err = rop.targetOp.Wait() if err != nil { errors = append(errors, remoteOperationResult{URL: serverURL, Error: err}) if localtls.IsConnectionError(err) { continue } break } success = true break } if !success { rop.err = remoteOperationError("Failed storage volume creation", errors) } close(rop.chDone) }() return &rop, nil } // CopyStoragePoolVolume copies an existing storage volume. func (r *ProtocolIncus) CopyStoragePoolVolume(pool string, source InstanceServer, sourcePool string, volume api.StorageVolume, args *StoragePoolVolumeCopyArgs) (RemoteOperation, error) { if !r.HasExtension("storage_api_local_volume_handling") { return nil, errors.New("The server is missing the required \"storage_api_local_volume_handling\" API extension") } if args != nil && args.VolumeOnly && !r.HasExtension("storage_api_volume_snapshots") { return nil, errors.New("The target server is missing the required \"storage_api_volume_snapshots\" API extension") } if args != nil && args.Refresh && !r.HasExtension("custom_volume_refresh") { return nil, errors.New("The target server is missing the required \"custom_volume_refresh\" API extension") } if args != nil && args.RefreshExcludeOlder && !r.HasExtension("custom_volume_refresh_exclude_older_snapshots") { return nil, errors.New("The target server is missing the required \"custom_volume_refresh_exclude_older_snapshots\" API extension") } req := api.StorageVolumesPost{ Name: args.Name, Type: volume.Type, Source: api.StorageVolumeSource{ Name: volume.Name, Type: "copy", Pool: sourcePool, VolumeOnly: args.VolumeOnly, Refresh: args.Refresh, RefreshExcludeOlder: args.RefreshExcludeOlder, }, } req.Config = volume.Config req.Description = volume.Description req.ContentType = volume.ContentType sourceInfo, err := source.GetConnectionInfo() if err != nil { return nil, fmt.Errorf("Failed to get source connection info: %w", err) } destInfo, err := r.GetConnectionInfo() if err != nil { return nil, fmt.Errorf("Failed to get destination connection info: %w", err) } clusterInternalVolumeCopy := r.CheckExtension("cluster_internal_custom_volume_copy") == nil // Copy the storage pool volume locally. if destInfo.URL == sourceInfo.URL && destInfo.SocketPath == sourceInfo.SocketPath && (volume.Location == r.clusterTarget || (volume.Location == "none" && r.clusterTarget == "") || clusterInternalVolumeCopy) { // Project handling if destInfo.Project != sourceInfo.Project { if !r.HasExtension("storage_api_project") { return nil, errors.New("The server is missing the required \"storage_api_project\" API extension") } req.Source.Project = sourceInfo.Project } if clusterInternalVolumeCopy { req.Source.Location = sourceInfo.Target } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("/storage-pools/%s/volumes/%s", url.PathEscape(pool), url.PathEscape(volume.Type)), req, "") if err != nil { return nil, err } rop := remoteOperation{ targetOp: op, chDone: make(chan bool), } // Forward targetOp to remote op go func() { rop.err = rop.targetOp.Wait() close(rop.chDone) }() return &rop, nil } if !r.HasExtension("storage_api_remote_volume_handling") { return nil, errors.New("The server is missing the required \"storage_api_remote_volume_handling\" API extension") } sourceReq := api.StorageVolumePost{ Migration: true, Name: volume.Name, Pool: sourcePool, } if args != nil { sourceReq.VolumeOnly = args.VolumeOnly } // Push mode migration if args != nil && args.Mode == "push" { // Get target server connection information info, err := r.GetConnectionInfo() if err != nil { return nil, err } // Set the source type and direction req.Source.Type = "migration" req.Source.Mode = "push" // Send the request path := fmt.Sprintf("/storage-pools/%s/volumes/%s", url.PathEscape(pool), url.PathEscape(volume.Type)) // Send the request op, _, err := r.queryOperation("POST", path, req, "") if err != nil { return nil, err } opAPI := op.Get() targetSecrets := map[string]string{} for k, v := range opAPI.Metadata { val, ok := v.(string) if ok { targetSecrets[k] = val } } // Prepare the source request target := api.StorageVolumePostTarget{} target.Operation = opAPI.ID target.Websockets = targetSecrets target.Certificate = info.Certificate sourceReq.Target = &target return r.tryMigrateStoragePoolVolume(source, sourcePool, sourceReq, info.Addresses) } // Get source server connection information info, err := source.GetConnectionInfo() if err != nil { return nil, err } // Get secrets from source server op, err := source.MigrateStoragePoolVolume(sourcePool, sourceReq) if err != nil { return nil, err } opAPI := op.Get() // Prepare source server secrets for remote sourceSecrets := map[string]string{} for k, v := range opAPI.Metadata { val, ok := v.(string) if ok { sourceSecrets[k] = val } } // Relay mode migration if args != nil && args.Mode == "relay" { // Push copy source fields req.Source.Type = "migration" req.Source.Mode = "push" // Send the request path := fmt.Sprintf("/storage-pools/%s/volumes/%s", url.PathEscape(pool), url.PathEscape(volume.Type)) // Send the request targetOp, _, err := r.queryOperation("POST", path, req, "") if err != nil { return nil, err } targetOpAPI := targetOp.Get() // Extract the websockets targetSecrets := map[string]string{} for k, v := range targetOpAPI.Metadata { val, ok := v.(string) if ok { targetSecrets[k] = val } } // Launch the relay err = r.proxyMigration(targetOp.(*operation), targetSecrets, source, op.(*operation), sourceSecrets) if err != nil { return nil, err } // Prepare a tracking operation rop := remoteOperation{ targetOp: targetOp, chDone: make(chan bool), } // Forward targetOp to remote op go func() { rop.err = rop.targetOp.Wait() close(rop.chDone) }() return &rop, nil } // Pull mode migration req.Source.Type = "migration" req.Source.Mode = "pull" req.Source.Operation = opAPI.ID req.Source.Websockets = sourceSecrets req.Source.Certificate = info.Certificate return r.tryCreateStoragePoolVolume(pool, req, info.Addresses) } // MoveStoragePoolVolume renames or moves an existing storage volume. func (r *ProtocolIncus) MoveStoragePoolVolume(pool string, source InstanceServer, sourcePool string, volume api.StorageVolume, args *StoragePoolVolumeMoveArgs) (RemoteOperation, error) { if !r.HasExtension("storage_api_local_volume_handling") { return nil, errors.New("The server is missing the required \"storage_api_local_volume_handling\" API extension") } if r != source { return nil, errors.New("Moving storage volumes between remotes is not implemented") } req := api.StorageVolumePost{ Name: args.Name, Pool: pool, } if args.Project != "" { if !r.HasExtension("storage_volume_project_move") { return nil, errors.New("The server is missing the required \"storage_volume_project_move\" API extension") } req.Project = args.Project } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("/storage-pools/%s/volumes/%s/%s", url.PathEscape(sourcePool), url.PathEscape(volume.Type), volume.Name), req, "") if err != nil { return nil, err } rop := remoteOperation{ targetOp: op, chDone: make(chan bool), } // Forward targetOp to remote op go func() { rop.err = rop.targetOp.Wait() close(rop.chDone) }() return &rop, nil } // UpdateStoragePoolVolume updates the volume to match the provided StoragePoolVolume struct. func (r *ProtocolIncus) UpdateStoragePoolVolume(pool string, volType string, name string, volume api.StorageVolumePut, ETag string) error { if !r.HasExtension("storage") { return errors.New("The server is missing the required \"storage\" API extension") } if volume.Restore != "" && !r.HasExtension("storage_api_volume_snapshots") { return errors.New("The server is missing the required \"storage_api_volume_snapshots\" API extension") } // Send the request path := fmt.Sprintf("/storage-pools/%s/volumes/%s/%s", url.PathEscape(pool), url.PathEscape(volType), url.PathEscape(name)) _, _, err := r.query("PUT", path, volume, ETag) if err != nil { return err } return nil } // DeleteStoragePoolVolume deletes a storage pool. func (r *ProtocolIncus) DeleteStoragePoolVolume(pool string, volType string, name string) error { if !r.HasExtension("storage") { return errors.New("The server is missing the required \"storage\" API extension") } // Send the request path := fmt.Sprintf("/storage-pools/%s/volumes/%s/%s", url.PathEscape(pool), url.PathEscape(volType), url.PathEscape(name)) _, _, err := r.query("DELETE", path, nil, "") if err != nil { return err } return nil } // RenameStoragePoolVolume renames a storage volume. func (r *ProtocolIncus) RenameStoragePoolVolume(pool string, volType string, name string, volume api.StorageVolumePost) error { if !r.HasExtension("storage_api_volume_rename") { return errors.New("The server is missing the required \"storage_api_volume_rename\" API extension") } path := fmt.Sprintf("/storage-pools/%s/volumes/%s/%s", url.PathEscape(pool), url.PathEscape(volType), url.PathEscape(name)) // Send the request _, _, err := r.query("POST", path, volume, "") if err != nil { return err } return nil } // GetStorageVolumeBackupNames returns a list of volume backup names. func (r *ProtocolIncus) GetStorageVolumeBackupNames(pool string, volName string) ([]string, error) { if !r.HasExtension("custom_volume_backup") { return nil, errors.New("The server is missing the required \"custom_volume_backup\" API extension") } // Fetch the raw URL values. urls := []string{} baseURL := fmt.Sprintf("/storage-pools/%s/volumes/custom/%s/backups", url.PathEscape(pool), url.PathEscape(volName)) _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetStorageVolumeBackups returns a list of custom volume backups. func (r *ProtocolIncus) GetStorageVolumeBackups(pool string, volName string) ([]api.StorageVolumeBackup, error) { if !r.HasExtension("custom_volume_backup") { return nil, errors.New("The server is missing the required \"custom_volume_backup\" API extension") } // Fetch the raw value backups := []api.StorageVolumeBackup{} _, err := r.queryStruct("GET", fmt.Sprintf("/storage-pools/%s/volumes/custom/%s/backups?recursion=1", url.PathEscape(pool), url.PathEscape(volName)), nil, "", &backups) if err != nil { return nil, err } return backups, nil } // GetStorageVolumeBackup returns a custom volume backup. func (r *ProtocolIncus) GetStorageVolumeBackup(pool string, volName string, name string) (*api.StorageVolumeBackup, string, error) { if !r.HasExtension("custom_volume_backup") { return nil, "", errors.New("The server is missing the required \"custom_volume_backup\" API extension") } // Fetch the raw value backup := api.StorageVolumeBackup{} etag, err := r.queryStruct("GET", fmt.Sprintf("/storage-pools/%s/volumes/custom/%s/backups/%s", url.PathEscape(pool), url.PathEscape(volName), url.PathEscape(name)), nil, "", &backup) if err != nil { return nil, "", err } return &backup, etag, nil } // CreateStorageVolumeBackup creates new custom volume backup. func (r *ProtocolIncus) CreateStorageVolumeBackup(pool string, volName string, backup api.StorageVolumeBackupsPost) (Operation, error) { if !r.HasExtension("custom_volume_backup") { return nil, errors.New("The server is missing the required \"custom_volume_backup\" API extension") } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("/storage-pools/%s/volumes/custom/%s/backups", url.PathEscape(pool), url.PathEscape(volName)), backup, "") if err != nil { return nil, err } return op, nil } // RenameStorageVolumeBackup renames a custom volume backup. func (r *ProtocolIncus) RenameStorageVolumeBackup(pool string, volName string, name string, backup api.StorageVolumeBackupPost) (Operation, error) { if !r.HasExtension("custom_volume_backup") { return nil, errors.New("The server is missing the required \"custom_volume_backup\" API extension") } // Send the request op, _, err := r.queryOperation("POST", fmt.Sprintf("/storage-pools/%s/volumes/custom/%s/backups/%s", url.PathEscape(pool), url.PathEscape(volName), url.PathEscape(name)), backup, "") if err != nil { return nil, err } return op, nil } // DeleteStorageVolumeBackup deletes a custom volume backup. func (r *ProtocolIncus) DeleteStorageVolumeBackup(pool string, volName string, name string) (Operation, error) { if !r.HasExtension("custom_volume_backup") { return nil, errors.New("The server is missing the required \"custom_volume_backup\" API extension") } // Send the request op, _, err := r.queryOperation("DELETE", fmt.Sprintf("/storage-pools/%s/volumes/custom/%s/backups/%s", url.PathEscape(pool), url.PathEscape(volName), url.PathEscape(name)), nil, "") if err != nil { return nil, err } return op, nil } // GetStorageVolumeBackupFile requests the custom volume backup content. func (r *ProtocolIncus) GetStorageVolumeBackupFile(pool string, volName string, name string, req *BackupFileRequest) (*BackupFileResponse, error) { if !r.HasExtension("custom_volume_backup") { return nil, errors.New("The server is missing the required \"custom_volume_backup\" API extension") } // Build the URL uri := fmt.Sprintf("%s/1.0/storage-pools/%s/volumes/custom/%s/backups/%s/export", r.httpBaseURL.String(), url.PathEscape(pool), url.PathEscape(volName), url.PathEscape(name)) // Add project/target uri, err := r.setQueryAttributes(uri) if err != nil { return nil, err } // Prepare the download request request, err := http.NewRequest("GET", uri, nil) if err != nil { return nil, err } if r.httpUserAgent != "" { request.Header.Set("User-Agent", r.httpUserAgent) } // Start the request response, doneCh, err := cancel.CancelableDownload(req.Canceler, r.DoHTTP, request) if err != nil { return nil, err } defer func() { _ = response.Body.Close() }() defer close(doneCh) if response.StatusCode != http.StatusOK { _, _, err := incusParseResponse(response) if err != nil { return nil, err } } // Handle the data body := response.Body if req.ProgressHandler != nil { body = &ioprogress.ProgressReader{ ReadCloser: response.Body, Tracker: &ioprogress.ProgressTracker{ Length: response.ContentLength, Handler: func(percent int64, speed int64) { req.ProgressHandler(ioprogress.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, units.GetByteSizeString(speed, 2))}) }, }, } } size, err := util.SafeCopy(req.BackupFile, body) if err != nil { return nil, err } resp := BackupFileResponse{} resp.Size = size return &resp, nil } // CreateStorageVolumeBackupStream requests that Incus creates and returns new direct backup for // the storage volume. func (r *ProtocolIncus) CreateStorageVolumeBackupStream(pool string, volName string, backup api.StorageVolumeBackupsPost, req *BackupFileRequest) error { if !r.HasExtension("direct_backup") { return errors.New("The server is missing the required \"direct_backup\" API extension") } // Build the URL uri := fmt.Sprintf("%s/1.0/storage-pools/%s/volumes/custom/%s/backups", r.httpBaseURL.String(), url.PathEscape(pool), url.PathEscape(volName)) if r.project != "" { uri += fmt.Sprintf("?project=%s", url.QueryEscape(r.project)) } // Encode the backup data buf := bytes.Buffer{} err := json.NewEncoder(&buf).Encode(backup) if err != nil { return err } // Prepare the download request request, err := http.NewRequest("POST", uri, bytes.NewReader(buf.Bytes())) if err != nil { return err } request.Header.Set("Accept", "application/octet-stream") if r.httpUserAgent != "" { request.Header.Set("User-Agent", r.httpUserAgent) } // Start the request response, doneCh, err := cancel.CancelableDownload(req.Canceler, r.DoHTTP, request) if err != nil { return err } defer func() { _ = response.Body.Close() }() defer close(doneCh) if response.StatusCode != http.StatusOK { _, _, err = incusParseResponse(response) if err != nil { return err } } // Handle the data body := response.Body if req.ProgressHandler != nil { body = &ioprogress.ProgressReader{ ReadCloser: response.Body, Tracker: &ioprogress.ProgressTracker{ Handler: func(received int64, speed int64) { req.ProgressHandler(ioprogress.ProgressData{Text: fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(received, 2), units.GetByteSizeString(speed, 2))}) }, }, } } _, err = util.SafeCopy(req.BackupFile, body) return err } // CreateStoragePoolVolumeFromMigration defines a new storage volume. // In contrast to CreateStoragePoolVolume, it also returns an operation object. func (r *ProtocolIncus) CreateStoragePoolVolumeFromMigration(pool string, volume api.StorageVolumesPost) (Operation, error) { // Send the request path := fmt.Sprintf("/storage-pools/%s/volumes/%s", url.PathEscape(pool), url.PathEscape(volume.Type)) op, _, err := r.queryOperation("POST", path, volume, "") if err != nil { return nil, err } return op, nil } // CreateStoragePoolVolumeFromISO creates a custom volume from an ISO file. func (r *ProtocolIncus) CreateStoragePoolVolumeFromISO(pool string, args StorageVolumeBackupArgs) (Operation, error) { err := r.CheckExtension("custom_volume_iso") if err != nil { return nil, err } if args.Name == "" { return nil, errors.New("Missing volume name") } path := fmt.Sprintf("/storage-pools/%s/volumes/custom", url.PathEscape(pool)) // Prepare the HTTP request. reqURL, err := r.setQueryAttributes(fmt.Sprintf("%s/1.0%s", r.httpBaseURL.String(), path)) if err != nil { return nil, err } req, err := http.NewRequest("POST", reqURL, args.BackupFile) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("X-Incus-name", args.Name) req.Header.Set("X-Incus-type", "iso") // Send the request. resp, err := r.DoHTTP(req) if err != nil { return nil, err } defer func() { _ = resp.Body.Close() }() // Handle errors. response, _, err := incusParseResponse(resp) if err != nil { return nil, err } // Get to the operation. respOperation, err := response.MetadataAsOperation() if err != nil { return nil, err } // Setup an Operation wrapper. op := operation{ Operation: *respOperation, r: r, chActive: make(chan bool), } return &op, nil } // CreateStoragePoolVolumeFromBackup creates a custom volume from a backup file. func (r *ProtocolIncus) CreateStoragePoolVolumeFromBackup(pool string, args StorageVolumeBackupArgs) (Operation, error) { if !r.HasExtension("custom_volume_backup") { return nil, errors.New(`The server is missing the required "custom_volume_backup" API extension`) } if args.Name != "" && !r.HasExtension("backup_override_name") { return nil, errors.New(`The server is missing the required "backup_override_name" API extension`) } path := fmt.Sprintf("/storage-pools/%s/volumes/custom", url.PathEscape(pool)) // Prepare the HTTP request. reqURL, err := r.setQueryAttributes(fmt.Sprintf("%s/1.0%s", r.httpBaseURL.String(), path)) if err != nil { return nil, err } req, err := http.NewRequest("POST", reqURL, args.BackupFile) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/octet-stream") if args.Name != "" { req.Header.Set("X-Incus-name", args.Name) } // Send the request. resp, err := r.DoHTTP(req) if err != nil { return nil, err } defer func() { _ = resp.Body.Close() }() // Handle errors. response, _, err := incusParseResponse(resp) if err != nil { return nil, err } // Get to the operation. respOperation, err := response.MetadataAsOperation() if err != nil { return nil, err } // Setup an Operation wrapper. op := operation{ Operation: *respOperation, r: r, chActive: make(chan bool), } return &op, nil } // GetStoragePoolVolumeBlockNBDConn returns a connection to the volume's NBD endpoint. func (r *ProtocolIncus) GetStoragePoolVolumeBlockNBDConn(pool string, volType string, volName string, args StorageVolumeNBDPost) (net.Conn, error) { if !r.HasExtension("storage_volume_nbd") { return nil, errors.New(`The server is missing the required "storage_volume_nbd" API extension`) } u := api.NewURL() u.URL = r.httpBaseURL // Preload the URL with the client base URL. u.Path("1.0", "storage-pools", pool, "volumes", volType, volName, "nbd") values := u.Query() if args.Writable { values.Set("writable", "1") } u.RawQuery = values.Encode() r.setURLQueryAttributes(&u.URL) return r.rawConn(&u.URL, "nbd") } // GetStoragePoolVolumeFileSFTPConn returns a connection to the volume's SFTP endpoint. func (r *ProtocolIncus) GetStoragePoolVolumeFileSFTPConn(pool string, volType string, volName string) (net.Conn, error) { if !r.HasExtension("custom_volume_sftp") { return nil, errors.New(`The server is missing the required "custom_volume_sftp" API extension`) } u := api.NewURL() u.URL = r.httpBaseURL // Preload the URL with the client base URL. u.Path("1.0", "storage-pools", pool, "volumes", volType, volName, "sftp") r.setURLQueryAttributes(&u.URL) return r.rawConn(&u.URL, "sftp") } // GetStoragePoolVolumeFileSFTP returns an SFTP connection to the volume. func (r *ProtocolIncus) GetStoragePoolVolumeFileSFTP(pool string, volType string, volName string) (*sftp.Client, error) { if !r.HasExtension("custom_volume_sftp") { return nil, errors.New(`The server is missing the required "custom_volume_sftp" API extension`) } conn, err := r.GetStoragePoolVolumeFileSFTPConn(pool, volType, volName) if err != nil { return nil, err } // Get a SFTP client. client, err := sftp.NewClientPipe(conn, conn, sftp.MaxPacketUnchecked(128*1024)) if err != nil { _ = conn.Close() return nil, err } go func() { // Wait for the client to be done before closing the connection. _ = client.Wait() _ = conn.Close() }() return client, nil } // GetStorageVolumeFile retrieves the provided path from the storage volume. func (r *ProtocolIncus) GetStorageVolumeFile(pool string, volumeType string, volumeName string, filePath string) (io.ReadCloser, *InstanceFileResponse, error) { // Send the request path := fmt.Sprintf( "/storage-pools/%s/volumes/%s/%s/files?path=%s", url.PathEscape(pool), url.PathEscape(volumeType), url.PathEscape(volumeName), url.QueryEscape(filePath), ) requestURL, err := r.setQueryAttributes(fmt.Sprintf("%s/1.0%s", r.httpBaseURL.String(), path)) if err != nil { return nil, nil, err } req, err := http.NewRequest("GET", requestURL, nil) if err != nil { return nil, nil, err } // Send the request resp, err := r.DoHTTP(req) if err != nil { return nil, nil, err } // Check the return value for a cleaner error if resp.StatusCode != http.StatusOK { _, _, err := incusParseResponse(resp) if err != nil { return nil, nil, err } } // Parse the headers uid, gid, mode, fileType, _ := api.ParseFileHeaders(resp.Header) fileResp := InstanceFileResponse{ UID: uid, GID: gid, Mode: mode, Type: fileType, } if fileResp.Type == "directory" { // Decode the response response := api.Response{} decoder := json.NewDecoder(resp.Body) err = decoder.Decode(&response) if err != nil { return nil, nil, err } // Get the file list entries := []string{} err = response.MetadataAsStruct(&entries) if err != nil { return nil, nil, err } fileResp.Entries = entries return nil, &fileResp, err } return resp.Body, &fileResp, err } // CreateStorageVolumeFile tells Incus to create a file in the storage volume. func (r *ProtocolIncus) CreateStorageVolumeFile(pool string, volumeType string, volumeName string, filePath string, args InstanceFileArgs) error { // Send the request path := fmt.Sprintf( "/storage-pools/%s/volumes/%s/%s/files?path=%s", url.PathEscape(pool), url.PathEscape(volumeType), url.PathEscape(volumeName), url.QueryEscape(filePath), ) requestURL, err := r.setQueryAttributes(fmt.Sprintf("%s/1.0%s", r.httpBaseURL.String(), path)) if err != nil { return err } req, err := http.NewRequest("POST", requestURL, args.Content) if err != nil { return err } req.GetBody = func() (io.ReadCloser, error) { _, err := args.Content.Seek(0, 0) if err != nil { return nil, err } return io.NopCloser(args.Content), nil } // Set the various headers if args.UID > -1 { req.Header.Set("X-Incus-uid", fmt.Sprintf("%d", args.UID)) } if args.GID > -1 { req.Header.Set("X-Incus-gid", fmt.Sprintf("%d", args.GID)) } if args.Mode > -1 { req.Header.Set("X-Incus-mode", fmt.Sprintf("%04o", args.Mode)) } if args.Type != "" { req.Header.Set("X-Incus-type", args.Type) } if args.WriteMode != "" { req.Header.Set("X-Incus-write", args.WriteMode) } // Send the request resp, err := r.DoHTTP(req) if err != nil { return err } // Check the return value for a cleaner error _, _, err = incusParseResponse(resp) if err != nil { return err } return nil } // DeleteStorageVolumeFile deletes a file in the storage volume. func (r *ProtocolIncus) DeleteStorageVolumeFile(pool string, volumeType string, volumeName string, filePath string) error { // Send the request path := fmt.Sprintf( "/storage-pools/%s/volumes/%s/%s/files?path=%s", url.PathEscape(pool), url.PathEscape(volumeType), url.PathEscape(volumeName), url.QueryEscape(filePath), ) requestURL, err := r.setQueryAttributes(path) if err != nil { return err } // Send the request _, _, err = r.query("DELETE", requestURL, nil, "") if err != nil { return err } return nil } // GetStorageVolumeBitmapNames returns a list of volume bitmap names. func (r *ProtocolIncus) GetStorageVolumeBitmapNames(pool string, volumeType string, volumeName string) ([]string, error) { if !r.HasExtension("storage_volume_nbd") { return nil, errors.New("The server is missing the required \"storage_volume_nbd\" API extension") } // Fetch the raw URL values. urls := []string{} baseURL := fmt.Sprintf( "/storage-pools/%s/volumes/%s/%s/bitmaps", url.PathEscape(pool), url.PathEscape(volumeType), url.PathEscape(volumeName)) _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetStorageVolumeBitmaps returns a list of volume bitmaps. func (r *ProtocolIncus) GetStorageVolumeBitmaps(pool string, volumeType string, volumeName string) ([]api.StorageVolumeBitmap, error) { if !r.HasExtension("storage_volume_nbd") { return nil, errors.New("The server is missing the required \"storage_volume_nbd\" API extension") } bitmaps := []api.StorageVolumeBitmap{} path := fmt.Sprintf("/storage-pools/%s/volumes/%s/%s/bitmaps?recursion=1", url.PathEscape(pool), url.PathEscape(volumeType), url.PathEscape(volumeName)) _, err := r.queryStruct("GET", path, nil, "", &bitmaps) if err != nil { return nil, err } return bitmaps, nil } // CreateStorageVolumeBitmap creates a new volume bitmap. func (r *ProtocolIncus) CreateStorageVolumeBitmap(pool string, volumeType string, volumeName string, bitmap api.StorageVolumeBitmapsPost) error { if !r.HasExtension("storage_volume_nbd") { return errors.New("The server is missing the required \"storage_volume_nbd\" API extension") } path := fmt.Sprintf("/storage-pools/%s/volumes/%s/%s/bitmaps", url.PathEscape(pool), url.PathEscape(volumeType), url.PathEscape(volumeName)) // Send the request _, _, err := r.query("POST", path, bitmap, "") if err != nil { return err } return nil } // DeleteStorageVolumeBitmap deletes a volume bitmap. func (r *ProtocolIncus) DeleteStorageVolumeBitmap(pool string, volumeType string, volumeName string, bitmapName string) error { if !r.HasExtension("storage_volume_nbd") { return errors.New("The server is missing the required \"storage_volume_nbd\" API extension") } path := fmt.Sprintf( "/storage-pools/%s/volumes/%s/%s/bitmaps/%s", url.PathEscape(pool), url.PathEscape(volumeType), url.PathEscape(volumeName), url.PathEscape(bitmapName)) _, _, err := r.query("DELETE", path, nil, "") if err != nil { return err } return nil } incus-7.0.0/client/incus_warnings.go000066400000000000000000000044041517523235500175070ustar00rootroot00000000000000package incus import ( "errors" "fmt" "net/url" "github.com/lxc/incus/v7/shared/api" ) // Warning handling functions // GetWarningUUIDs returns a list of operation uuids. func (r *ProtocolIncus) GetWarningUUIDs() ([]string, error) { if !r.HasExtension("warnings") { return nil, errors.New("The server is missing the required \"warnings\" API extension") } // Fetch the raw values. urls := []string{} baseURL := "/warnings" _, err := r.queryStruct("GET", baseURL, nil, "", &urls) if err != nil { return nil, err } // Parse it. return urlsToResourceNames(baseURL, urls...) } // GetWarnings returns a list of warnings. func (r *ProtocolIncus) GetWarnings() ([]api.Warning, error) { if !r.HasExtension("warnings") { return nil, errors.New("The server is missing the required \"warnings\" API extension") } warnings := []api.Warning{} _, err := r.queryStruct("GET", "/warnings?recursion=1", nil, "", &warnings) if err != nil { return nil, err } return warnings, nil } // GetWarning returns the warning with the given UUID. func (r *ProtocolIncus) GetWarning(UUID string) (*api.Warning, string, error) { if !r.HasExtension("warnings") { return nil, "", errors.New("The server is missing the required \"warnings\" API extension") } warning := api.Warning{} etag, err := r.queryStruct("GET", fmt.Sprintf("/warnings/%s", url.PathEscape(UUID)), nil, "", &warning) if err != nil { return nil, "", err } return &warning, etag, nil } // UpdateWarning updates the warning with the given UUID. func (r *ProtocolIncus) UpdateWarning(UUID string, warning api.WarningPut, ETag string) error { if !r.HasExtension("warnings") { return errors.New("The server is missing the required \"warnings\" API extension") } // Send the request _, _, err := r.query("PUT", fmt.Sprintf("/warnings/%s", url.PathEscape(UUID)), warning, ETag) if err != nil { return err } return nil } // DeleteWarning deletes the provided warning. func (r *ProtocolIncus) DeleteWarning(UUID string) error { if !r.HasExtension("warnings") { return errors.New("The server is missing the required \"warnings\" API extension") } // Send the request _, _, err := r.query("DELETE", fmt.Sprintf("/warnings/%s", url.PathEscape(UUID)), nil, "") if err != nil { return err } return nil } incus-7.0.0/client/interfaces.go000066400000000000000000001124161517523235500166040ustar00rootroot00000000000000package incus import ( "context" "io" "net" "net/http" "github.com/gorilla/websocket" "github.com/pkg/sftp" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/cancel" "github.com/lxc/incus/v7/shared/ioprogress" ) // The Operation type represents a currently running operation. type Operation interface { AddHandler(function func(api.Operation)) (target *EventTarget, err error) Cancel() (err error) Get() (op api.Operation) GetWebsocket(secret string) (conn *websocket.Conn, err error) RemoveHandler(target *EventTarget) (err error) Refresh() (err error) Wait() (err error) WaitContext(ctx context.Context) error } // The RemoteOperation type represents an Operation that may be using multiple servers. type RemoteOperation interface { AddHandler(function func(api.Operation)) (target *EventTarget, err error) CancelTarget() (err error) GetTarget() (op *api.Operation, err error) Wait() (err error) } // The Server type represents a generic read-only server. type Server interface { GetConnectionInfo() (info *ConnectionInfo, err error) GetHTTPClient() (client *http.Client, err error) DoHTTP(req *http.Request) (resp *http.Response, err error) Disconnect() } // The ImageServer type represents a read-only image server. type ImageServer interface { Server // Image handling functions GetImages() (images []api.Image, err error) GetImagesAllProjects() (images []api.Image, err error) GetImagesAllProjectsWithFilter(filters []string) (images []api.Image, err error) GetImageFingerprints() (fingerprints []string, err error) GetImagesWithFilter(filters []string) (images []api.Image, err error) GetImage(fingerprint string) (image *api.Image, ETag string, err error) GetImageFile(fingerprint string, req ImageFileRequest) (resp *ImageFileResponse, err error) GetImageSecret(fingerprint string) (secret string, err error) GetPrivateImage(fingerprint string, secret string) (image *api.Image, ETag string, err error) GetPrivateImageFile(fingerprint string, secret string, req ImageFileRequest) (resp *ImageFileResponse, err error) GetImageAliases() (aliases []api.ImageAliasesEntry, err error) GetImageAliasNames() (names []string, err error) GetImageAlias(name string) (alias *api.ImageAliasesEntry, ETag string, err error) GetImageAliasType(imageType string, name string) (alias *api.ImageAliasesEntry, ETag string, err error) GetImageAliasArchitectures(imageType string, name string) (entries map[string]*api.ImageAliasesEntry, err error) ExportImage(fingerprint string, image api.ImageExportPost) (Operation, error) } // The InstanceServer type represents a full featured Incus server. type InstanceServer interface { ImageServer // Server functions GetMetrics() (metrics string, err error) GetServer() (server *api.Server, ETag string, err error) GetServerResources() (resources *api.Resources, err error) UpdateServer(server api.ServerPut, ETag string) (err error) ApplyServerPreseed(config api.InitPreseed) error HasExtension(extension string) (exists bool) RequireAuthenticated(authenticated bool) IsClustered() (clustered bool) UseTarget(name string) (client InstanceServer) UseProject(name string) (client InstanceServer) // Certificate functions GetCertificateFingerprints() (fingerprints []string, err error) GetCertificates() (certificates []api.Certificate, err error) GetCertificatesWithFilter(filters []string) ([]api.Certificate, error) GetCertificate(fingerprint string) (certificate *api.Certificate, ETag string, err error) CreateCertificate(certificate api.CertificatesPost) (err error) UpdateCertificate(fingerprint string, certificate api.CertificatePut, ETag string) (err error) DeleteCertificate(fingerprint string) (err error) CreateCertificateToken(certificate api.CertificatesPost) (op Operation, err error) // Instance functions. GetInstanceNames(instanceType api.InstanceType) (names []string, err error) GetInstanceNamesAllProjects(instanceType api.InstanceType) (names map[string][]string, err error) GetInstances(instanceType api.InstanceType) (instances []api.Instance, err error) GetInstancesFull(instanceType api.InstanceType) (instances []api.InstanceFull, err error) GetInstancesAllProjects(instanceType api.InstanceType) (instances []api.Instance, err error) GetInstancesFullAllProjects(instanceType api.InstanceType) (instances []api.InstanceFull, err error) GetInstancesWithFilter(instanceType api.InstanceType, filters []string) (instances []api.Instance, err error) GetInstancesFullWithFilter(instanceType api.InstanceType, filters []string) (instances []api.InstanceFull, err error) GetInstancesAllProjectsWithFilter(instanceType api.InstanceType, filters []string) (instances []api.Instance, err error) GetInstancesFullAllProjectsWithFilter(instanceType api.InstanceType, filters []string) (instances []api.InstanceFull, err error) GetInstance(name string) (instance *api.Instance, ETag string, err error) GetInstanceFull(name string) (instance *api.InstanceFull, ETag string, err error) CreateInstance(instance api.InstancesPost) (op Operation, err error) CreateInstanceFromImage(source ImageServer, image api.Image, req api.InstancesPost) (op RemoteOperation, err error) CopyInstance(source InstanceServer, instance api.Instance, args *InstanceCopyArgs) (op RemoteOperation, err error) UpdateInstance(name string, instance api.InstancePut, ETag string) (op Operation, err error) RenameInstance(name string, instance api.InstancePost) (op Operation, err error) MigrateInstance(name string, instance api.InstancePost) (op Operation, err error) DeleteInstance(name string) (op Operation, err error) UpdateInstances(state api.InstancesPut, ETag string) (op Operation, err error) RebuildInstance(instanceName string, req api.InstanceRebuildPost) (op Operation, err error) RebuildInstanceFromImage(source ImageServer, image api.Image, instanceName string, req api.InstanceRebuildPost) (op RemoteOperation, err error) ExecInstance(instanceName string, exec api.InstanceExecPost, args *InstanceExecArgs) (op Operation, err error) ConsoleInstance(instanceName string, console api.InstanceConsolePost, args *InstanceConsoleArgs) (op Operation, err error) ConsoleInstanceDynamic(instanceName string, console api.InstanceConsolePost, args *InstanceConsoleArgs) (Operation, func(io.ReadWriteCloser) error, error) CreateInstanceBitmap(name string, bitmap api.StorageVolumeBitmapsPost) error GetInstanceConsoleLog(instanceName string, args *InstanceConsoleLogArgs) (content io.ReadCloser, err error) DeleteInstanceConsoleLog(instanceName string, args *InstanceConsoleLogArgs) (err error) GetInstanceFile(instanceName string, path string) (content io.ReadCloser, resp *InstanceFileResponse, err error) CreateInstanceFile(instanceName string, path string, args InstanceFileArgs) (err error) DeleteInstanceFile(instanceName string, path string) (err error) GetInstanceFileSFTPConn(instanceName string) (net.Conn, error) GetInstanceFileSFTP(instanceName string) (*sftp.Client, error) GetInstanceSnapshotNames(instanceName string) (names []string, err error) GetInstanceSnapshots(instanceName string) (snapshots []api.InstanceSnapshot, err error) GetInstanceSnapshot(instanceName string, name string) (snapshot *api.InstanceSnapshot, ETag string, err error) CreateInstanceSnapshot(instanceName string, snapshot api.InstanceSnapshotsPost) (op Operation, err error) CopyInstanceSnapshot(source InstanceServer, instanceName string, snapshot api.InstanceSnapshot, args *InstanceSnapshotCopyArgs) (op RemoteOperation, err error) RenameInstanceSnapshot(instanceName string, name string, instance api.InstanceSnapshotPost) (op Operation, err error) MigrateInstanceSnapshot(instanceName string, name string, instance api.InstanceSnapshotPost) (op Operation, err error) DeleteInstanceSnapshot(instanceName string, name string) (op Operation, err error) UpdateInstanceSnapshot(instanceName string, name string, instance api.InstanceSnapshotPut, ETag string) (op Operation, err error) GetInstanceBackupNames(instanceName string) (names []string, err error) GetInstanceBackups(instanceName string) (backups []api.InstanceBackup, err error) GetInstanceBackup(instanceName string, name string) (backup *api.InstanceBackup, ETag string, err error) CreateInstanceBackup(instanceName string, backup api.InstanceBackupsPost) (op Operation, err error) RenameInstanceBackup(instanceName string, name string, backup api.InstanceBackupPost) (op Operation, err error) DeleteInstanceBackup(instanceName string, name string) (op Operation, err error) GetInstanceBackupFile(instanceName string, name string, req *BackupFileRequest) (resp *BackupFileResponse, err error) CreateInstanceBackupStream(instanceName string, backup api.InstanceBackupsPost, req *BackupFileRequest) (err error) CreateInstanceFromBackup(args InstanceBackupArgs) (op Operation, err error) GetInstanceState(name string) (state *api.InstanceState, ETag string, err error) UpdateInstanceState(name string, state api.InstanceStatePut, ETag string) (op Operation, err error) GetInstanceAccess(name string) (access api.Access, err error) GetInstanceLogfiles(name string) (logfiles []string, err error) GetInstanceLogfile(name string, filename string) (content io.ReadCloser, err error) DeleteInstanceLogfile(name string, filename string) (err error) GetInstanceMetadata(name string) (metadata *api.ImageMetadata, ETag string, err error) UpdateInstanceMetadata(name string, metadata api.ImageMetadata, ETag string) (err error) GetInstanceTemplateFiles(instanceName string) (templates []string, err error) GetInstanceTemplateFile(instanceName string, templateName string) (content io.ReadCloser, err error) CreateInstanceTemplateFile(instanceName string, templateName string, content io.ReadSeeker) (err error) DeleteInstanceTemplateFile(name string, templateName string) (err error) GetInstanceDebugMemory(name string, format string) (rc io.ReadCloser, err error) // Event handling functions GetEvents() (listener *EventListener, err error) GetEventsByType(eventTypes []string) (listener *EventListener, err error) GetEventsAllProjects() (listener *EventListener, err error) GetEventsAllProjectsByType(eventTypes []string) (listener *EventListener, err error) SendEvent(event api.Event) error // Image functions CreateImage(image api.ImagesPost, args *ImageCreateArgs) (op Operation, err error) CopyImage(source ImageServer, image api.Image, args *ImageCopyArgs) (op RemoteOperation, err error) UpdateImage(fingerprint string, image api.ImagePut, ETag string) (err error) DeleteImage(fingerprint string) (op Operation, err error) RefreshImage(fingerprint string) (op Operation, err error) CreateImageSecret(fingerprint string) (op Operation, err error) CreateImageAlias(alias api.ImageAliasesPost) (err error) UpdateImageAlias(name string, alias api.ImageAliasesEntryPut, ETag string) (err error) RenameImageAlias(name string, alias api.ImageAliasesEntryPost) (err error) DeleteImageAlias(name string) (err error) // Configuration metadata functions GetMetadataConfiguration() (meta *api.MetadataConfiguration, err error) // Network functions ("network" API extension) GetNetworkNames() (names []string, err error) GetNetworks() (networks []api.Network, err error) GetNetworksWithFilter(filters []string) (networks []api.Network, err error) GetNetworksAllProjects() (networks []api.Network, err error) GetNetworksAllProjectsWithFilter(filters []string) (networks []api.Network, err error) GetNetwork(name string) (network *api.Network, ETag string, err error) GetNetworkLeases(name string) (leases []api.NetworkLease, err error) GetNetworkState(name string) (state *api.NetworkState, err error) CreateNetwork(network api.NetworksPost) (err error) UpdateNetwork(name string, network api.NetworkPut, ETag string) (err error) RenameNetwork(name string, network api.NetworkPost) (err error) DeleteNetwork(name string) (err error) // Network forward functions ("network_forward" API extension) GetNetworkForwardAddresses(networkName string) ([]string, error) GetNetworkForwards(networkName string) ([]api.NetworkForward, error) GetNetworkForward(networkName string, listenAddress string) (forward *api.NetworkForward, ETag string, err error) CreateNetworkForward(networkName string, forward api.NetworkForwardsPost) error UpdateNetworkForward(networkName string, listenAddress string, forward api.NetworkForwardPut, ETag string) (err error) DeleteNetworkForward(networkName string, listenAddress string) (err error) // Network load balancer functions ("network_load_balancer" API extension) GetNetworkLoadBalancerAddresses(networkName string) ([]string, error) GetNetworkLoadBalancers(networkName string) ([]api.NetworkLoadBalancer, error) GetNetworkLoadBalancer(networkName string, listenAddress string) (forward *api.NetworkLoadBalancer, ETag string, err error) CreateNetworkLoadBalancer(networkName string, forward api.NetworkLoadBalancersPost) error UpdateNetworkLoadBalancer(networkName string, listenAddress string, forward api.NetworkLoadBalancerPut, ETag string) (err error) DeleteNetworkLoadBalancer(networkName string, listenAddress string) (err error) GetNetworkLoadBalancerState(networkName string, listenAddress string) (lbState *api.NetworkLoadBalancerState, err error) // Network peer functions ("network_peer" API extension) GetNetworkPeerNames(networkName string) ([]string, error) GetNetworkPeers(networkName string) ([]api.NetworkPeer, error) GetNetworkPeer(networkName string, peerName string) (peer *api.NetworkPeer, ETag string, err error) CreateNetworkPeer(networkName string, peer api.NetworkPeersPost) error UpdateNetworkPeer(networkName string, peerName string, peer api.NetworkPeerPut, ETag string) (err error) DeleteNetworkPeer(networkName string, peerName string) (err error) // Network ACL functions ("network_acl" API extension) GetNetworkACLNames() (names []string, err error) GetNetworkACLs() (acls []api.NetworkACL, err error) GetNetworkACLsAllProjects() (acls []api.NetworkACL, err error) GetNetworkACL(name string) (acl *api.NetworkACL, ETag string, err error) GetNetworkACLLogfile(name string) (log io.ReadCloser, err error) CreateNetworkACL(acl api.NetworkACLsPost) (err error) UpdateNetworkACL(name string, acl api.NetworkACLPut, ETag string) (err error) RenameNetworkACL(name string, acl api.NetworkACLPost) (err error) DeleteNetworkACL(name string) (err error) // Network address set functions ("network_address_set" API extension) GetNetworkAddressSetNames() (names []string, err error) GetNetworkAddressSets() (AddressSets []api.NetworkAddressSet, err error) GetNetworkAddressSetsAllProjects() (AddressSets []api.NetworkAddressSet, err error) GetNetworkAddressSet(name string) (AddressSet *api.NetworkAddressSet, ETag string, err error) CreateNetworkAddressSet(AddressSet api.NetworkAddressSetsPost) (err error) UpdateNetworkAddressSet(name string, AddressSet api.NetworkAddressSetPut, ETag string) (err error) RenameNetworkAddressSet(name string, AddressSet api.NetworkAddressSetPost) (err error) DeleteNetworkAddressSet(name string) (err error) // Network allocations functions ("network_allocations" API extension) GetNetworkAllocations() (allocations []api.NetworkAllocations, err error) GetNetworkAllocationsAllProjects() (allocations []api.NetworkAllocations, err error) // Network zone functions ("network_dns" API extension) GetNetworkZonesAllProjects() (zones []api.NetworkZone, err error) GetNetworkZoneNames() (names []string, err error) GetNetworkZones() (zones []api.NetworkZone, err error) GetNetworkZone(name string) (zone *api.NetworkZone, ETag string, err error) CreateNetworkZone(zone api.NetworkZonesPost) (err error) UpdateNetworkZone(name string, zone api.NetworkZonePut, ETag string) (err error) DeleteNetworkZone(name string) (err error) GetNetworkZoneRecordNames(zone string) (names []string, err error) GetNetworkZoneRecords(zone string) (records []api.NetworkZoneRecord, err error) GetNetworkZoneRecord(zone string, name string) (record *api.NetworkZoneRecord, ETag string, err error) CreateNetworkZoneRecord(zone string, record api.NetworkZoneRecordsPost) (err error) UpdateNetworkZoneRecord(zone string, name string, record api.NetworkZoneRecordPut, ETag string) (err error) DeleteNetworkZoneRecord(zone string, name string) (err error) // Network integrations functions ("network_integrations" API extension) GetNetworkIntegrationNames() (names []string, err error) GetNetworkIntegrations() (integrations []api.NetworkIntegration, err error) GetNetworkIntegration(name string) (integration *api.NetworkIntegration, ETag string, err error) CreateNetworkIntegration(integration api.NetworkIntegrationsPost) (err error) UpdateNetworkIntegration(name string, integration api.NetworkIntegrationPut, ETag string) (err error) RenameNetworkIntegration(name string, integration api.NetworkIntegrationPost) (err error) DeleteNetworkIntegration(name string) (err error) // Operation functions GetOperationUUIDs() (uuids []string, err error) GetOperations() (operations []api.Operation, err error) GetOperationsAllProjects() (operations []api.Operation, err error) GetOperation(uuid string) (op *api.Operation, ETag string, err error) GetOperationWait(uuid string, timeout int) (op *api.Operation, ETag string, err error) GetOperationWaitSecret(uuid string, secret string, timeout int) (op *api.Operation, ETag string, err error) GetOperationWebsocket(uuid string, secret string) (conn *websocket.Conn, err error) DeleteOperation(uuid string) (err error) // Profile functions GetProfilesAllProjects() (profiles []api.Profile, err error) GetProfilesAllProjectsWithFilter(filters []string) ([]api.Profile, error) GetProfileNames() (names []string, err error) GetProfiles() (profiles []api.Profile, err error) GetProfilesWithFilter(filters []string) ([]api.Profile, error) GetProfile(name string) (profile *api.Profile, ETag string, err error) CreateProfile(profile api.ProfilesPost) (err error) UpdateProfile(name string, profile api.ProfilePut, ETag string) (err error) RenameProfile(name string, profile api.ProfilePost) (err error) DeleteProfile(name string) (err error) // Project functions GetProjectNames() (names []string, err error) GetProjects() (projects []api.Project, err error) GetProjectsWithFilter(filters []string) (projects []api.Project, err error) GetProject(name string) (project *api.Project, ETag string, err error) GetProjectState(name string) (project *api.ProjectState, err error) GetProjectAccess(name string) (access api.Access, err error) CreateProject(project api.ProjectsPost) (err error) UpdateProject(name string, project api.ProjectPut, ETag string) (err error) RenameProject(name string, project api.ProjectPost) (op Operation, err error) DeleteProject(name string) (err error) DeleteProjectForce(name string) (err error) // Storage pool functions ("storage" API extension) GetStoragePoolNames() (names []string, err error) GetStoragePools() (pools []api.StoragePool, err error) GetStoragePoolsWithFilter(filters []string) ([]api.StoragePool, error) GetStoragePool(name string) (pool *api.StoragePool, ETag string, err error) GetStoragePoolResources(name string) (resources *api.ResourcesStoragePool, err error) CreateStoragePool(pool api.StoragePoolsPost) (err error) UpdateStoragePool(name string, pool api.StoragePoolPut, ETag string) (err error) DeleteStoragePool(name string) (err error) // Storage bucket functions ("storage_buckets" API extension) GetStoragePoolBucketNames(poolName string) ([]string, error) GetStoragePoolBucketsAllProjects(poolName string) ([]api.StorageBucket, error) GetStoragePoolBucketsWithFilterAllProjects(poolName string, filters []string) (bucket []api.StorageBucket, err error) GetStoragePoolBuckets(poolName string) ([]api.StorageBucket, error) GetStoragePoolBucketsWithFilter(poolName string, filters []string) (bucket []api.StorageBucket, err error) GetStoragePoolBucketsFullAllProjects(poolName string) ([]api.StorageBucketFull, error) GetStoragePoolBucketsFullWithFilterAllProjects(poolName string, filters []string) (bucket []api.StorageBucketFull, err error) GetStoragePoolBucketsFull(poolName string) ([]api.StorageBucketFull, error) GetStoragePoolBucketsFullWithFilter(poolName string, filters []string) (bucket []api.StorageBucketFull, err error) GetStoragePoolBucket(poolName string, bucketName string) (bucket *api.StorageBucket, ETag string, err error) GetStoragePoolBucketFull(poolName string, bucketName string) (bucket *api.StorageBucketFull, ETag string, err error) CreateStoragePoolBucket(poolName string, bucket api.StorageBucketsPost) (*api.StorageBucketKey, error) UpdateStoragePoolBucket(poolName string, bucketName string, bucket api.StorageBucketPut, ETag string) (err error) DeleteStoragePoolBucket(poolName string, bucketName string) (err error) GetStoragePoolBucketKeyNames(poolName string, bucketName string) ([]string, error) GetStoragePoolBucketKeys(poolName string, bucketName string) ([]api.StorageBucketKey, error) GetStoragePoolBucketKey(poolName string, bucketName string, keyName string) (key *api.StorageBucketKey, ETag string, err error) CreateStoragePoolBucketKey(poolName string, bucketName string, key api.StorageBucketKeysPost) (newKey *api.StorageBucketKey, err error) UpdateStoragePoolBucketKey(poolName string, bucketName string, keyName string, key api.StorageBucketKeyPut, ETag string) (err error) DeleteStoragePoolBucketKey(poolName string, bucketName string, keyName string) (err error) // Storage bucket backup functions ("storage_bucket_backup" API extension) CreateStoragePoolBucketBackup(poolName string, bucketName string, backup api.StorageBucketBackupsPost) (op Operation, err error) DeleteStoragePoolBucketBackup(pool string, bucketName string, name string) (op Operation, err error) GetStoragePoolBucketBackupFile(pool string, bucketName string, name string, req *BackupFileRequest) (resp *BackupFileResponse, err error) CreateStoragePoolBucketBackupStream(pool string, bucketName string, backup api.StorageBucketBackupsPost, req *BackupFileRequest) (err error) CreateStoragePoolBucketFromBackup(pool string, args StoragePoolBucketBackupArgs) (op Operation, err error) // Storage volume functions ("storage" API extension) GetStoragePoolVolumeNames(pool string) (names []string, err error) GetStoragePoolVolumeNamesAllProjects(pool string) (names map[string][]string, err error) GetStoragePoolVolumes(pool string) (volumes []api.StorageVolume, err error) GetStoragePoolVolumesAllProjects(pool string) (volumes []api.StorageVolume, err error) GetStoragePoolVolumesWithFilter(pool string, filters []string) (volumes []api.StorageVolume, err error) GetStoragePoolVolumesWithFilterAllProjects(pool string, filters []string) (volumes []api.StorageVolume, err error) GetStoragePoolVolumesFull(pool string) (volumes []api.StorageVolumeFull, err error) GetStoragePoolVolumesFullAllProjects(pool string) (volumes []api.StorageVolumeFull, err error) GetStoragePoolVolumesFullWithFilter(pool string, filters []string) (volumes []api.StorageVolumeFull, err error) GetStoragePoolVolumesFullWithFilterAllProjects(pool string, filters []string) (volumes []api.StorageVolumeFull, err error) GetStoragePoolVolume(pool string, volType string, name string) (volume *api.StorageVolume, ETag string, err error) GetStoragePoolVolumeFull(pool string, volType string, name string) (volume *api.StorageVolumeFull, ETag string, err error) GetStoragePoolVolumeState(pool string, volType string, name string) (state *api.StorageVolumeState, err error) CreateStoragePoolVolume(pool string, volume api.StorageVolumesPost) (err error) UpdateStoragePoolVolume(pool string, volType string, name string, volume api.StorageVolumePut, ETag string) (err error) DeleteStoragePoolVolume(pool string, volType string, name string) (err error) RenameStoragePoolVolume(pool string, volType string, name string, volume api.StorageVolumePost) (err error) CopyStoragePoolVolume(pool string, source InstanceServer, sourcePool string, volume api.StorageVolume, args *StoragePoolVolumeCopyArgs) (op RemoteOperation, err error) MoveStoragePoolVolume(pool string, source InstanceServer, sourcePool string, volume api.StorageVolume, args *StoragePoolVolumeMoveArgs) (op RemoteOperation, err error) MigrateStoragePoolVolume(pool string, volume api.StorageVolumePost) (op Operation, err error) // Storage volume snapshot functions ("storage_api_volume_snapshots" API extension) CreateStoragePoolVolumeSnapshot(pool string, volumeType string, volumeName string, snapshot api.StorageVolumeSnapshotsPost) (op Operation, err error) DeleteStoragePoolVolumeSnapshot(pool string, volumeType string, volumeName string, snapshotName string) (op Operation, err error) GetStoragePoolVolumeSnapshotNames(pool string, volumeType string, volumeName string) (names []string, err error) GetStoragePoolVolumeSnapshots(pool string, volumeType string, volumeName string) (snapshots []api.StorageVolumeSnapshot, err error) GetStoragePoolVolumeSnapshot(pool string, volumeType string, volumeName string, snapshotName string) (snapshot *api.StorageVolumeSnapshot, ETag string, err error) RenameStoragePoolVolumeSnapshot(pool string, volumeType string, volumeName string, snapshotName string, snapshot api.StorageVolumeSnapshotPost) (op Operation, err error) UpdateStoragePoolVolumeSnapshot(pool string, volumeType string, volumeName string, snapshotName string, volume api.StorageVolumeSnapshotPut, ETag string) (err error) // Storage volume backup functions ("custom_volume_backup" API extension) GetStorageVolumeBackupNames(pool string, volName string) (names []string, err error) GetStorageVolumeBackups(pool string, volName string) (backups []api.StorageVolumeBackup, err error) GetStorageVolumeBackup(pool string, volName string, name string) (backup *api.StorageVolumeBackup, ETag string, err error) CreateStorageVolumeBackup(pool string, volName string, backup api.StorageVolumeBackupsPost) (op Operation, err error) RenameStorageVolumeBackup(pool string, volName string, name string, backup api.StorageVolumeBackupPost) (op Operation, err error) DeleteStorageVolumeBackup(pool string, volName string, name string) (op Operation, err error) GetStorageVolumeBackupFile(pool string, volName string, name string, req *BackupFileRequest) (resp *BackupFileResponse, err error) CreateStorageVolumeBackupStream(pool string, volName string, backup api.StorageVolumeBackupsPost, req *BackupFileRequest) (err error) CreateStoragePoolVolumeFromBackup(pool string, args StorageVolumeBackupArgs) (op Operation, err error) // Storage volume bitmaps manipulations functions ("storage_volume_nbd" API extension) GetStorageVolumeBitmapNames(pool string, volumeType string, volumeName string) ([]string, error) GetStorageVolumeBitmaps(pool string, volumeType string, volumeName string) ([]api.StorageVolumeBitmap, error) CreateStorageVolumeBitmap(pool string, volumeType string, volumeName string, bitmap api.StorageVolumeBitmapsPost) error DeleteStorageVolumeBitmap(pool string, volumeType string, volumeName string, bitmapName string) error // Storage volume ISO import function ("custom_volume_iso" API extension) CreateStoragePoolVolumeFromISO(pool string, args StorageVolumeBackupArgs) (op Operation, err error) CreateStoragePoolVolumeFromMigration(pool string, volume api.StorageVolumesPost) (op Operation, err error) // Storage volume file manipulations functions ("file_storage_volume" API extension) GetStorageVolumeFile(pool string, volumeType string, volumeName string, filePath string) (content io.ReadCloser, resp *InstanceFileResponse, err error) CreateStorageVolumeFile(pool string, volumeType string, volumeName string, filePath string, args InstanceFileArgs) (err error) DeleteStorageVolumeFile(pool string, volumeType string, volumeName string, filePath string) (err error) // Storage volume NBD functions ("storage_volume_nbd" API extension) GetStoragePoolVolumeBlockNBDConn(pool string, volType string, volName string, args StorageVolumeNBDPost) (net.Conn, error) // Storage volume SFTP functions ("custom_volume_sftp" API extension) GetStoragePoolVolumeFileSFTPConn(pool string, volType string, volName string) (net.Conn, error) GetStoragePoolVolumeFileSFTP(pool string, volType string, volName string) (*sftp.Client, error) // Cluster functions ("cluster" API extensions) GetCluster() (cluster *api.Cluster, ETag string, err error) UpdateCluster(cluster api.ClusterPut, ETag string) (op Operation, err error) DeleteClusterMember(name string, force bool) (err error) DeletePendingClusterMember(name string, force bool) (err error) GetClusterMemberNames() (names []string, err error) GetClusterMembers() (members []api.ClusterMember, err error) GetClusterMembersWithFilter(filters []string) ([]api.ClusterMember, error) GetClusterMember(name string) (member *api.ClusterMember, ETag string, err error) UpdateClusterMember(name string, member api.ClusterMemberPut, ETag string) (err error) RenameClusterMember(name string, member api.ClusterMemberPost) (err error) CreateClusterMember(member api.ClusterMembersPost) (op Operation, err error) UpdateClusterCertificate(certs api.ClusterCertificatePut, ETag string) (err error) GetClusterMemberState(name string) (*api.ClusterMemberState, string, error) UpdateClusterMemberState(name string, state api.ClusterMemberStatePost) (op Operation, err error) GetClusterGroups() ([]api.ClusterGroup, error) GetClusterGroupNames() ([]string, error) RenameClusterGroup(name string, group api.ClusterGroupPost) error CreateClusterGroup(group api.ClusterGroupsPost) error DeleteClusterGroup(name string) error UpdateClusterGroup(name string, group api.ClusterGroupPut, ETag string) error GetClusterGroup(name string) (*api.ClusterGroup, string, error) // Warning functions GetWarningUUIDs() (uuids []string, err error) GetWarnings() (warnings []api.Warning, err error) GetWarning(UUID string) (warning *api.Warning, ETag string, err error) UpdateWarning(UUID string, warning api.WarningPut, ETag string) (err error) DeleteWarning(UUID string) (err error) // Internal functions (for internal use) RawQuery(method string, path string, data any, queryETag string) (resp *api.Response, ETag string, err error) RawWebsocket(path string) (conn *websocket.Conn, err error) RawOperation(method string, path string, data any, queryETag string) (op Operation, ETag string, err error) } // The ConnectionInfo struct represents general information for a connection. type ConnectionInfo struct { Addresses []string Certificate string Protocol string URL string SocketPath string Project string Target string } // The BackupFileRequest struct is used for a backup download request. type BackupFileRequest struct { // Writer for the backup file BackupFile io.WriteSeeker // Progress handler (called whenever some progress is made) ProgressHandler func(progress ioprogress.ProgressData) // A canceler that can be used to interrupt some part of the image download request Canceler *cancel.HTTPRequestCanceller } // The BackupFileResponse struct is used as the response for backup downloads. type BackupFileResponse struct { // Size of backup file Size int64 } // The ImageCreateArgs struct is used for direct image upload. type ImageCreateArgs struct { // Reader for the meta file MetaFile io.Reader // Filename for the meta file MetaName string // Reader for the rootfs file RootfsFile io.Reader // Filename for the rootfs file RootfsName string // Progress handler (called with upload progress) ProgressHandler func(progress ioprogress.ProgressData) // Type of the image (container or virtual-machine) Type string } // The ImageFileRequest struct is used for an image download request. type ImageFileRequest struct { // Writer for the metadata file MetaFile io.ReadWriteSeeker // Writer for the rootfs file RootfsFile io.ReadWriteSeeker // Progress handler (called whenever some progress is made) ProgressHandler func(progress ioprogress.ProgressData) // A canceler that can be used to interrupt some part of the image download request Canceler *cancel.HTTPRequestCanceller // Path retriever for image delta downloads // If set, it must return the path to the image file or an empty string if not available DeltaSourceRetriever func(fingerprint string, file string) string } // The ImageFileResponse struct is used as the response for image downloads. type ImageFileResponse struct { // Filename for the metadata file MetaName string // Size of the metadata file MetaSize int64 // Filename for the rootfs file RootfsName string // Size of the rootfs file RootfsSize int64 } // The ImageCopyArgs struct is used to pass additional options during image copy. type ImageCopyArgs struct { // Aliases to add to the copied image. Aliases []api.ImageAlias // Whether to have Incus keep this image up to date AutoUpdate bool // Whether to copy the source image aliases to the target CopyAliases bool // Whether this image is to be made available to unauthenticated users Public bool // The image type to use for resolution Type string // The transfer mode, can be "pull" (default), "push" or "relay" Mode string // List of profiles to apply on the target. Profiles []string } // The StoragePoolVolumeCopyArgs struct is used to pass additional options // during storage volume copy. type StoragePoolVolumeCopyArgs struct { // New name for the target Name string // The transfer mode, can be "pull" (default), "push" or "relay" Mode string // API extension: storage_api_volume_snapshots VolumeOnly bool // API extension: custom_volume_refresh Refresh bool // API extension: custom_volume_refresh_exclude_older_snapshots RefreshExcludeOlder bool } // The StoragePoolVolumeMoveArgs struct is used to pass additional options // during storage volume move. type StoragePoolVolumeMoveArgs struct { StoragePoolVolumeCopyArgs // API extension: storage_volume_project_move Project string } // The StorageVolumeBackupArgs struct is used when creating a storage volume from a backup. // API extension: custom_volume_backup. type StorageVolumeBackupArgs struct { // The backup file BackupFile io.Reader // Name to import backup as Name string } // The InstanceBackupArgs struct is used when creating a instance from a backup. type InstanceBackupArgs struct { // The backup file BackupFile io.Reader // Storage pool to use PoolName string // Name to import backup as Name string // Config overrides. Config []string // Device overrides. Devices []string } // The InstanceCopyArgs struct is used to pass additional options during instance copy. type InstanceCopyArgs struct { // If set, the instance will be renamed on copy Name string // If set, the instance running state will be transferred (live migration) Live bool // If set, only the instance will copied, its snapshots won't InstanceOnly bool // The transfer mode, can be "pull" (default), "push" or "relay" Mode string // API extension: container_incremental_copy // Perform an incremental copy Refresh bool // API extension: custom_volume_refresh_exclude_older_snapshots RefreshExcludeOlder bool // API extension: instance_allow_inconsistent_copy AllowInconsistent bool } // The InstanceSnapshotCopyArgs struct is used to pass additional options during instance copy. type InstanceSnapshotCopyArgs struct { // If set, the instance will be renamed on copy Name string // The transfer mode, can be "pull" (default), "push" or "relay" Mode string // API extension: container_snapshot_stateful_migration // If set, the instance running state will be transferred (live migration) Live bool } // The InstanceConsoleArgs struct is used to pass additional options during a // instance console session. type InstanceConsoleArgs struct { // Bidirectional fd to pass to the instance Terminal io.ReadWriteCloser // Control message handler (window resize) Control func(conn *websocket.Conn) // Closing this Channel causes a disconnect from the instance's console ConsoleDisconnect chan bool } // The InstanceConsoleLogArgs struct is used to pass additional options during a // instance console log request. type InstanceConsoleLogArgs struct{} // The InstanceExecArgs struct is used to pass additional options during instance exec. type InstanceExecArgs struct { // Standard input Stdin io.Reader // Standard output Stdout io.Writer // Standard error Stderr io.Writer // Control message handler (window resize, signals, ...) Control func(conn *websocket.Conn) // Channel that will be closed when all data operations are done DataDone chan bool } // The InstanceFileArgs struct is used to pass the various options for a instance file upload. type InstanceFileArgs struct { // File content Content io.ReadSeeker // User id that owns the file UID int64 // Group id that owns the file GID int64 // File permissions Mode int // File type (file or directory) Type string // File write mode (overwrite or append) WriteMode string } // The InstanceFileResponse struct is used as part of the response for a instance file download. type InstanceFileResponse struct { // User id that owns the file UID int64 // Group id that owns the file GID int64 // File permissions Mode int // File type (file or directory) Type string // If a directory, the list of files inside it Entries []string } // The StoragePoolBucketBackupArgs struct is used when creating a storage volume from a backup. // API extension: storage_bucket_backup. type StoragePoolBucketBackupArgs struct { // The backup file BackupFile io.Reader // Name to import backup as Name string } // The StorageVolumeNBDPost struct is used when connecting to a storage volume over NBD. // API extension: storage_volume_nbd. type StorageVolumeNBDPost struct { // Writable Writable bool } incus-7.0.0/client/oci.go000066400000000000000000000024101517523235500152230ustar00rootroot00000000000000package incus import ( "errors" "net/http" ) // ProtocolOCI implements an OCI registry API client. type ProtocolOCI struct { http *http.Client httpHost string httpUserAgent string httpCertificate string // Cache for images. cache map[string]ociInfo // Error tracking for images. errors map[string]error tempPath string } // Disconnect is a no-op for OCI. func (r *ProtocolOCI) Disconnect() { } // GetConnectionInfo returns the basic connection information used to interact with the server. func (r *ProtocolOCI) GetConnectionInfo() (*ConnectionInfo, error) { info := ConnectionInfo{} info.Addresses = []string{r.httpHost} info.Certificate = r.httpCertificate info.Protocol = "oci" info.URL = r.httpHost return &info, nil } // GetHTTPClient returns the http client used for the connection. This can be used to set custom http options. func (r *ProtocolOCI) GetHTTPClient() (*http.Client, error) { if r.http == nil { return nil, errors.New("HTTP client isn't set, bad connection") } return r.http, nil } // DoHTTP performs a Request. func (r *ProtocolOCI) DoHTTP(req *http.Request) (*http.Response, error) { // Set the user agent. if r.httpUserAgent != "" { req.Header.Set("User-Agent", r.httpUserAgent) } return r.http.Do(req) } incus-7.0.0/client/oci_images.go000066400000000000000000000324031517523235500165550ustar00rootroot00000000000000package incus import ( "compress/gzip" "context" "crypto/sha256" "encoding/base64" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "os" "os/exec" "path/filepath" "strings" "time" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) type ociInfo struct { Alias string Name string `json:"Name"` Digest string `json:"Digest"` Created time.Time `json:"Created"` Architecture string `json:"Architecture"` Layers []string `json:"Layers"` LayersData []struct { Size int64 `json:"Size"` } `json:"LayersData"` } // Get the proxy host value. func (r *ProtocolOCI) getProxyHost() (*url.URL, error) { req, err := http.NewRequest("GET", r.httpHost, nil) if err != nil { return nil, err } proxy, err := r.http.Transport.(*http.Transport).Proxy(req) if err != nil { return nil, err } return proxy, nil } // Image handling functions // GetImages returns a list of available images as Image structs. func (r *ProtocolOCI) GetImages() ([]api.Image, error) { return nil, errors.New("Can't list images from OCI registry") } // GetImagesAllProjects returns a list of available images as Image structs. func (r *ProtocolOCI) GetImagesAllProjects() ([]api.Image, error) { return nil, errors.New("Can't list images from OCI registry") } // GetImagesAllProjectsWithFilter returns a filtered list of available images as Image structs. func (r *ProtocolOCI) GetImagesAllProjectsWithFilter(filters []string) ([]api.Image, error) { return nil, errors.New("Can't list images from OCI registry") } // GetImageFingerprints returns a list of available image fingerprints. func (r *ProtocolOCI) GetImageFingerprints() ([]string, error) { return nil, errors.New("Can't list images from OCI registry") } // GetImagesWithFilter returns a filtered list of available images as Image structs. func (r *ProtocolOCI) GetImagesWithFilter(_ []string) ([]api.Image, error) { return nil, errors.New("Can't list images from OCI registry") } // GetImage returns an Image struct for the provided fingerprint. func (r *ProtocolOCI) GetImage(fingerprint string) (*api.Image, string, error) { info, ok := r.cache[fingerprint] if !ok { _, err := exec.LookPath("skopeo") if err != nil { return nil, "", errors.New("OCI container handling requires \"skopeo\" be present on the system") } err, ok := r.errors[fingerprint] if ok { return nil, "", err } return nil, "", errors.New("Image not found") } img := api.Image{ ImagePut: api.ImagePut{ Public: true, Properties: map[string]string{ "architecture": info.Architecture, "type": "oci", "description": fmt.Sprintf("%s (OCI)", info.Name), "id": info.Alias, }, }, Aliases: []api.ImageAlias{{ Name: info.Alias, }}, Architecture: info.Architecture, Fingerprint: fingerprint, Type: string(api.InstanceTypeContainer), CreatedAt: info.Created, UploadedAt: info.Created, } var size int64 for _, layer := range info.LayersData { size += layer.Size } img.Size = size return &img, "", nil } // GetImageFile downloads an image from the server, returning an ImageFileResponse struct. func (r *ProtocolOCI) GetImageFile(fingerprint string, req ImageFileRequest) (*ImageFileResponse, error) { ctx := context.Background() // Get the cached entry. info, ok := r.cache[fingerprint] if !ok { _, err := exec.LookPath("skopeo") if err != nil { return nil, errors.New("OCI container handling requires \"skopeo\" be present on the system") } err, ok := r.errors[fingerprint] if ok { return nil, err } return nil, errors.New("Image not found") } // Quick checks. if req.MetaFile == nil && req.RootfsFile == nil { return nil, errors.New("No file requested") } if os.Geteuid() != 0 { return nil, errors.New("OCI image export currently requires root access") } // Get some temporary storage. ociPath, err := os.MkdirTemp(r.tempPath, "incus-oci-") if err != nil { return nil, err } defer func() { _ = os.RemoveAll(ociPath) }() err = os.Mkdir(filepath.Join(ociPath, "oci"), 0o700) if err != nil { return nil, err } err = os.Mkdir(filepath.Join(ociPath, "image"), 0o700) if err != nil { return nil, err } // Copy the image. if req.ProgressHandler != nil { req.ProgressHandler(ioprogress.ProgressData{Text: "Retrieving OCI image from registry"}) } imageTag := "latest" stdout, err := r.runSkopeo( "copy", info.Alias, "--remove-signatures", fmt.Sprintf("oci:%s:%s", filepath.Join(ociPath, "oci"), imageTag)) if err != nil { logger.Debug("Error copying remote image to local", logger.Ctx{"image": info.Alias, "stdout": stdout, "stderr": err}) return nil, err } // Convert to something usable. if req.ProgressHandler != nil { req.ProgressHandler(ioprogress.ProgressData{Text: "Unpacking the OCI image"}) } err = unpackOCIImage(filepath.Join(ociPath, "oci"), imageTag, filepath.Join(ociPath, "image")) if err != nil { logger.Debug("Error unpacking OCI image", logger.Ctx{"image": filepath.Join(ociPath, "oci"), "err": err}) return nil, err } // Generate a metadata.yaml. if req.ProgressHandler != nil { req.ProgressHandler(ioprogress.ProgressData{Text: "Generating image metadata"}) } metadata := api.ImageMetadata{ Architecture: info.Architecture, CreationDate: info.Created.Unix(), } data, err := json.Marshal(metadata) if err != nil { return nil, err } err = os.WriteFile(filepath.Join(ociPath, "image", "metadata.yaml"), data, 0o644) if err != nil { return nil, err } // Prepare response. resp := &ImageFileResponse{ MetaName: "metadata.tar.gz", RootfsName: "rootfs.tar.gz", } // Prepare to push the tarballs. var pipeRead io.ReadCloser var pipeWrite io.WriteCloser // Push the metadata tarball. pipeRead, pipeWrite = io.Pipe() defer pipeRead.Close() defer pipeWrite.Close() if req.ProgressHandler != nil { pipeRead = &ioprogress.ProgressReader{ ReadCloser: pipeRead, Tracker: &ioprogress.ProgressTracker{ Handler: func(received int64, speed int64) { req.ProgressHandler(ioprogress.ProgressData{Text: fmt.Sprintf("Generating metadata tarball: %s (%s/s)", units.GetByteSizeString(received, 2), units.GetByteSizeString(speed, 2))}) }, }, } } compressWrite := gzip.NewWriter(pipeWrite) metadataProcess := subprocess.NewProcessWithFds("tar", []string{"-cf", "-", "-C", filepath.Join(ociPath, "image"), "config.json", "metadata.yaml"}, nil, compressWrite, os.Stderr) err = metadataProcess.Start(ctx) if err != nil { return nil, err } go func() { _, _ = metadataProcess.Wait(ctx) compressWrite.Close() pipeWrite.Close() }() size, err := util.SafeCopy(req.MetaFile, pipeRead) if err != nil { return nil, err } resp.MetaSize = size // Push the rootfs tarball. pipeRead, pipeWrite = io.Pipe() defer pipeRead.Close() defer pipeWrite.Close() if req.ProgressHandler != nil { pipeRead = &ioprogress.ProgressReader{ ReadCloser: pipeRead, Tracker: &ioprogress.ProgressTracker{ Handler: func(received int64, speed int64) { req.ProgressHandler(ioprogress.ProgressData{Text: fmt.Sprintf("Generating rootfs tarball: %s (%s/s)", units.GetByteSizeString(received, 2), units.GetByteSizeString(speed, 2))}) }, }, } } compressWrite = gzip.NewWriter(pipeWrite) rootfsProcess := subprocess.NewProcessWithFds("tar", []string{"-cf", "-", "-C", filepath.Join(ociPath, "image", "rootfs"), "."}, nil, compressWrite, nil) err = rootfsProcess.Start(ctx) if err != nil { return nil, err } go func() { _, _ = rootfsProcess.Wait(ctx) compressWrite.Close() pipeWrite.Close() }() size, err = util.SafeCopy(req.RootfsFile, pipeRead) if err != nil { return nil, err } resp.RootfsSize = size return resp, nil } // GetImageSecret isn't relevant for the simplestreams protocol. func (r *ProtocolOCI) GetImageSecret(_ string) (string, error) { return "", errors.New("Private images aren't supported with OCI registry") } // GetPrivateImage isn't relevant for the simplestreams protocol. func (r *ProtocolOCI) GetPrivateImage(_ string, _ string) (*api.Image, string, error) { return nil, "", errors.New("Private images aren't supported with OCI registry") } // GetPrivateImageFile isn't relevant for the simplestreams protocol. func (r *ProtocolOCI) GetPrivateImageFile(_ string, _ string, _ ImageFileRequest) (*ImageFileResponse, error) { return nil, errors.New("Private images aren't supported with OCI registry") } // GetImageAliases returns the list of available aliases as ImageAliasesEntry structs. func (r *ProtocolOCI) GetImageAliases() ([]api.ImageAliasesEntry, error) { return nil, errors.New("Can't list image aliases from OCI registry") } // GetImageAliasNames returns the list of available alias names. func (r *ProtocolOCI) GetImageAliasNames() ([]string, error) { return nil, errors.New("Can't list image aliases from OCI registry") } func (r *ProtocolOCI) runSkopeo(action string, image string, args ...string) (string, error) { // Parse and mangle the server URL. uri, err := url.Parse(r.httpHost) if err != nil { return "", err } // Get proxy details. proxy, err := r.getProxyHost() if err != nil { return "", err } var env []string if proxy != nil { env = []string{ fmt.Sprintf("HTTPS_PROXY=%s", proxy), fmt.Sprintf("HTTP_PROXY=%s", proxy), } } // Handle authentication. if uri.User != nil { creds, err := json.Marshal(map[string]any{ "auths": map[string]any{ uri.Scheme + "://" + uri.Host: map[string]string{ "auth": base64.StdEncoding.EncodeToString([]byte(uri.User.String())), }, }, }) if err != nil { return "", err } authFile, err := os.CreateTemp(r.tempPath, "incus_client_auth_") if err != nil { return "", err } defer authFile.Close() defer os.Remove(authFile.Name()) err = authFile.Chmod(0o600) if err != nil { return "", err } _, err = fmt.Fprintf(authFile, "%s", creds) if err != nil { return "", err } uri.User = nil args = append(args, fmt.Sprintf("--authfile=%s", authFile.Name())) } // Prepare the arguments. uri.Scheme = "docker" args = append([]string{"--insecure-policy", action, fmt.Sprintf("%s/%s", uri.String(), image)}, args...) // Get the image information from skopeo. stdout, _, err := subprocess.RunCommandSplit( context.TODO(), env, nil, "skopeo", args...) if err != nil { return "", err } return stdout, nil } // GetImageAlias returns an existing alias as an ImageAliasesEntry struct. func (r *ProtocolOCI) GetImageAlias(name string) (*api.ImageAliasesEntry, string, error) { // If image name is "IMAGE:TAG@HASH", drop ":TAG" so that skopeo uses the pinned hash instead. imageWithoutHash, hash, hasHash := strings.Cut(name, "@") if hasHash { imageWithoutTag, _, _ := strings.Cut(imageWithoutHash, ":") name = fmt.Sprintf("%s@%s", imageWithoutTag, hash) } // Get the image information from skopeo. stdout, err := r.runSkopeo("inspect", name) if err != nil { logger.Debug("Error getting image alias", logger.Ctx{"name": name, "stdout": stdout, "stderr": err}) r.errors[name] = err return nil, "", err } // Parse the image info. var info ociInfo err = json.Unmarshal([]byte(stdout), &info) if err != nil { r.errors[name] = err return nil, "", err } info.Alias = name info.Digest = r.computeFingerprint(info.Layers) archID, err := osarch.ArchitectureID(info.Architecture) if err != nil { r.errors[name] = err return nil, "", err } archName, err := osarch.ArchitectureName(archID) if err != nil { r.errors[name] = err return nil, "", err } info.Architecture = archName // Store it in the cache. r.cache[info.Digest] = info // Prepare the alias entry. alias := api.ImageAliasesEntry{ ImageAliasesEntryPut: api.ImageAliasesEntryPut{ Target: info.Digest, }, Name: name, Type: string(api.InstanceTypeContainer), } return &alias, "", nil } // GetImageAliasType returns an existing alias as an ImageAliasesEntry struct. func (r *ProtocolOCI) GetImageAliasType(imageType string, name string) (*api.ImageAliasesEntry, string, error) { if api.InstanceType(imageType) == api.InstanceTypeVM { return nil, "", errors.New("OCI images are only supported for containers") } return r.GetImageAlias(name) } // GetImageAliasArchitectures returns a map of architectures / targets. func (r *ProtocolOCI) GetImageAliasArchitectures(imageType string, name string) (map[string]*api.ImageAliasesEntry, error) { if api.InstanceType(imageType) == api.InstanceTypeVM { return nil, errors.New("OCI images are only supported for containers") } alias, _, err := r.GetImageAlias(name) if err != nil { return nil, err } localArch, err := osarch.ArchitectureGetLocal() if err != nil { return nil, err } return map[string]*api.ImageAliasesEntry{localArch: alias}, nil } // ExportImage exports (copies) an image to a remote server. func (r *ProtocolOCI) ExportImage(_ string, _ api.ImageExportPost) (Operation, error) { return nil, errors.New("Exporting images is not supported with OCI registry") } func (r *ProtocolOCI) computeFingerprint(layers []string) string { h := sha256.New() for _, layer := range layers { h.Write([]byte(layer)) } return fmt.Sprintf("%x", h.Sum(nil)) } incus-7.0.0/client/oci_util.go000066400000000000000000000002701517523235500162620ustar00rootroot00000000000000//go:build !linux package incus import ( "fmt" ) func unpackOCIImage(imagePath string, imageTag string, bundlePath string) error { return fmt.Errorf("Platform isn't supported") } incus-7.0.0/client/oci_util_linux.go000066400000000000000000000032221517523235500175010ustar00rootroot00000000000000//go:build linux package incus import ( "fmt" "github.com/apex/log" "github.com/opencontainers/umoci" "github.com/opencontainers/umoci/oci/cas/dir" "github.com/opencontainers/umoci/oci/casext" "github.com/opencontainers/umoci/oci/layer" "github.com/lxc/incus/v7/shared/logger" ) func init() { // apex/log is only used by umoci within Incus. // So configure its logger to forward to our logger with the relevant prefix. // Set the custom handler. log.SetHandler(&umociLogHandler{Message: "Unpacking OCI image"}) } // Custom handler to intercept logs. type umociLogHandler struct { Message string } // HandleLog implements a proxy between apex/log and our logger. func (h *umociLogHandler) HandleLog(e *log.Entry) error { switch e.Level { case log.DebugLevel: logger.Debug(h.Message, logger.Ctx{"log": e.Message}) case log.InfoLevel: logger.Info(h.Message, logger.Ctx{"log": e.Message}) case log.WarnLevel: logger.Warn(h.Message, logger.Ctx{"log": e.Message}) case log.ErrorLevel: logger.Error(h.Message, logger.Ctx{"log": e.Message}) case log.FatalLevel: logger.Panic(h.Message, logger.Ctx{"log": e.Message}) default: logger.Error("Unknown umoci log level", logger.Ctx{"log": e.Message}) } return nil } func unpackOCIImage(imagePath string, imageTag string, bundlePath string) error { var unpackOptions layer.UnpackOptions unpackOptions.KeepDirlinks = true // Get a reference to the CAS. engine, err := dir.Open(imagePath) if err != nil { return fmt.Errorf("Open CAS: %w", err) } engineExt := casext.NewEngine(engine) defer func() { _ = engine.Close() }() return umoci.Unpack(engineExt, imageTag, bundlePath, unpackOptions) } incus-7.0.0/client/operations.go000066400000000000000000000176471517523235500166560ustar00rootroot00000000000000package incus import ( "context" "encoding/json" "errors" "sync" "time" "github.com/gorilla/websocket" "github.com/lxc/incus/v7/shared/api" ) // The Operation type represents an ongoing Incus operation (asynchronous processing). type operation struct { api.Operation r *ProtocolIncus listener *EventListener handlerReady bool handlerLock sync.Mutex skipListener bool chActive chan bool } // AddHandler adds a function to be called whenever an event is received. func (op *operation) AddHandler(function func(api.Operation)) (*EventTarget, error) { if op.skipListener { return nil, errors.New("Cannot add handler, client operation does not support event listeners") } // Make sure we have a listener setup err := op.setupListener() if err != nil { return nil, err } // Make sure we're not racing with ourselves op.handlerLock.Lock() defer op.handlerLock.Unlock() // If we're done already, just return if op.StatusCode.IsFinal() { return nil, nil } // Wrap the function to filter unwanted messages wrapped := func(event api.Event) { op.handlerLock.Lock() newOp := api.Operation{} err := json.Unmarshal(event.Metadata, &newOp) if err != nil || newOp.ID != op.ID { op.handlerLock.Unlock() return } op.handlerLock.Unlock() function(newOp) } return op.listener.AddHandler([]string{"operation"}, wrapped) } // Cancel will request that Incus cancels the operation (if supported). func (op *operation) Cancel() error { return op.r.DeleteOperation(op.ID) } // Get returns the API operation struct. func (op *operation) Get() api.Operation { return op.Operation } // GetWebsocket returns a raw websocket connection from the operation. func (op *operation) GetWebsocket(secret string) (*websocket.Conn, error) { return op.r.GetOperationWebsocket(op.ID, secret) } // RemoveHandler removes a function to be called whenever an event is received. func (op *operation) RemoveHandler(target *EventTarget) error { if op.skipListener { return errors.New("Cannot remove handler, client operation does not support event listeners") } // Make sure we're not racing with ourselves op.handlerLock.Lock() defer op.handlerLock.Unlock() // If the listener is gone, just return if op.listener == nil { return nil } return op.listener.RemoveHandler(target) } // Refresh pulls the current version of the operation and updates the struct. func (op *operation) Refresh() error { // Get the current version of the operation newOp, _, err := op.r.GetOperation(op.ID) if err != nil { return err } // Update the operation struct op.Operation = *newOp return nil } // Wait lets you wait until the operation reaches a final state. func (op *operation) Wait() error { return op.WaitContext(context.Background()) } // WaitContext lets you wait until the operation reaches a final state with context.Context. func (op *operation) WaitContext(ctx context.Context) error { if op.skipListener { timeout := -1 deadline, ok := ctx.Deadline() if ok { timeout = int(time.Until(deadline).Seconds()) } opAPI, _, err := op.r.GetOperationWait(op.ID, timeout) if err != nil { return err } op.Operation = *opAPI if opAPI.Err != "" { return errors.New(opAPI.Err) } return nil } op.handlerLock.Lock() // Check if not done already if op.StatusCode.IsFinal() { if op.Err != "" { op.handlerLock.Unlock() return errors.New(op.Err) } op.handlerLock.Unlock() return nil } op.handlerLock.Unlock() // Make sure we have a listener setup err := op.setupListener() if err != nil { return err } select { case <-ctx.Done(): return ctx.Err() case <-op.chActive: } // We're done, parse the result if op.Err != "" { return errors.New(op.Err) } return nil } // setupListener initiates an event listener for an operation and manages updates to the operation's state. // It adds handlers to process events, monitors the listener for completion or errors, // and triggers a manual refresh of the operation's state to prevent race conditions. func (op *operation) setupListener() error { if op.skipListener { return errors.New("Cannot set up event listener, client operation does not support event listeners") } // Make sure we're not racing with ourselves op.handlerLock.Lock() defer op.handlerLock.Unlock() // We already have a listener setup if op.handlerReady { return nil } op.handlerReady = true // Get a new listener if op.listener == nil { listener, err := op.r.GetEvents() if err != nil { return err } op.listener = listener } // Setup the handler chReady := make(chan bool) _, err := op.listener.AddHandler([]string{"operation"}, func(event api.Event) { <-chReady // We don't want concurrency while processing events op.handlerLock.Lock() defer op.handlerLock.Unlock() // Check if we're done already (because of another event) if op.listener == nil { return } // Get an operation struct out of this data newOp := api.Operation{} err := json.Unmarshal(event.Metadata, &newOp) if err != nil || newOp.ID != op.ID { return } // Update the struct op.Operation = newOp // And check if we're done if op.StatusCode.IsFinal() { op.listener.Disconnect() op.listener = nil close(op.chActive) return } }) if err != nil { op.listener.Disconnect() op.listener = nil close(op.chActive) close(chReady) return err } // Monitor event listener go func() { <-chReady // We don't want concurrency while accessing the listener op.handlerLock.Lock() // Check if we're done already (because of another event) listener := op.listener if listener == nil { op.handlerLock.Unlock() return } op.handlerLock.Unlock() // Wait for the listener or operation to be done select { case <-listener.ctx.Done(): op.handlerLock.Lock() if op.listener != nil { op.Err = listener.err.Error() close(op.chActive) } op.handlerLock.Unlock() case <-op.chActive: return } }() // And do a manual refresh to avoid races err = op.Refresh() if err != nil { op.listener.Disconnect() op.listener = nil close(op.chActive) close(chReady) return err } // Check if not done already if op.StatusCode.IsFinal() { op.listener.Disconnect() op.listener = nil close(op.chActive) close(chReady) if op.Err != "" { return errors.New(op.Err) } return nil } // Start processing background updates close(chReady) return nil } // The remoteOperation type represents an ongoing Incus operation between two servers. type remoteOperation struct { targetOp Operation handlers []func(api.Operation) handlerLock sync.Mutex chDone chan bool chPost chan bool err error } // AddHandler adds a function to be called whenever an event is received. func (op *remoteOperation) AddHandler(function func(api.Operation)) (*EventTarget, error) { var err error var target *EventTarget op.handlerLock.Lock() defer op.handlerLock.Unlock() // Attach to the existing target operation if op.targetOp != nil { target, err = op.targetOp.AddHandler(function) if err != nil { return nil, err } } else { // Generate a mock EventTarget target = &EventTarget{ function: func(api.Event) { function(api.Operation{}) }, types: []string{"operation"}, } } // Add the handler to our list op.handlers = append(op.handlers, function) return target, nil } // CancelTarget attempts to cancel the target operation. func (op *remoteOperation) CancelTarget() error { if op.targetOp == nil { return errors.New("No associated target operation") } return op.targetOp.Cancel() } // GetTarget returns the target operation. func (op *remoteOperation) GetTarget() (*api.Operation, error) { if op.targetOp == nil { return nil, errors.New("No associated target operation") } opAPI := op.targetOp.Get() return &opAPI, nil } // Wait lets you wait until the operation reaches a final state. func (op *remoteOperation) Wait() error { <-op.chDone if op.chPost != nil { <-op.chPost } return op.err } incus-7.0.0/client/simplestreams.go000066400000000000000000000025061517523235500173470ustar00rootroot00000000000000package incus import ( "errors" "net/http" "github.com/lxc/incus/v7/shared/simplestreams" ) // ProtocolSimpleStreams implements a SimpleStreams API client. type ProtocolSimpleStreams struct { ssClient *simplestreams.SimpleStreams http *http.Client httpHost string httpUserAgent string httpCertificate string tempPath string } // Disconnect is a no-op for simplestreams. func (r *ProtocolSimpleStreams) Disconnect() { } // GetConnectionInfo returns the basic connection information used to interact with the server. func (r *ProtocolSimpleStreams) GetConnectionInfo() (*ConnectionInfo, error) { info := ConnectionInfo{} info.Addresses = []string{r.httpHost} info.Certificate = r.httpCertificate info.Protocol = "simplestreams" info.URL = r.httpHost return &info, nil } // GetHTTPClient returns the http client used for the connection. This can be used to set custom http options. func (r *ProtocolSimpleStreams) GetHTTPClient() (*http.Client, error) { if r.http == nil { return nil, errors.New("HTTP client isn't set, bad connection") } return r.http, nil } // DoHTTP performs a Request. func (r *ProtocolSimpleStreams) DoHTTP(req *http.Request) (*http.Response, error) { // Set the user agent if r.httpUserAgent != "" { req.Header.Set("User-Agent", r.httpUserAgent) } return r.http.Do(req) } incus-7.0.0/client/simplestreams_images.go000066400000000000000000000257061517523235500207030ustar00rootroot00000000000000package incus import ( "context" "crypto/sha256" "errors" "fmt" "io" "net/http" "net/url" "os" "os/exec" "strings" "time" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/simplestreams" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) // Image handling functions // GetImages returns a list of available images as Image structs. func (r *ProtocolSimpleStreams) GetImages() ([]api.Image, error) { return r.ssClient.ListImages() } // GetImagesAllProjects returns a list of available images as Image structs. func (r *ProtocolSimpleStreams) GetImagesAllProjects() ([]api.Image, error) { return r.GetImages() } // GetImagesAllProjectsWithFilter returns a filtered list of available images as Image structs. func (r *ProtocolSimpleStreams) GetImagesAllProjectsWithFilter(filters []string) ([]api.Image, error) { return nil, errors.New("GetImagesWithFilter is not supported by the simplestreams protocol") } // GetImageFingerprints returns a list of available image fingerprints. func (r *ProtocolSimpleStreams) GetImageFingerprints() ([]string, error) { // Get all the images from simplestreams images, err := r.ssClient.ListImages() if err != nil { return nil, err } // And now extract just the fingerprints fingerprints := []string{} for _, img := range images { fingerprints = append(fingerprints, img.Fingerprint) } return fingerprints, nil } // GetImagesWithFilter returns a filtered list of available images as Image structs. func (r *ProtocolSimpleStreams) GetImagesWithFilter(_ []string) ([]api.Image, error) { return nil, errors.New("GetImagesWithFilter is not supported by the simplestreams protocol") } // GetImage returns an Image struct for the provided fingerprint. func (r *ProtocolSimpleStreams) GetImage(fingerprint string) (*api.Image, string, error) { image, err := r.ssClient.GetImage(fingerprint) if err != nil { return nil, "", fmt.Errorf("Failed getting image: %w", err) } return image, "", err } // GetImageFile downloads an image from the server, returning an ImageFileResponse struct. func (r *ProtocolSimpleStreams) GetImageFile(fingerprint string, req ImageFileRequest) (*ImageFileResponse, error) { // Quick checks. if req.MetaFile == nil && req.RootfsFile == nil { return nil, errors.New("No file requested") } // Attempt to download from host if util.PathExists("/dev/incus/sock") && os.Geteuid() == 0 { unixURI := fmt.Sprintf("http://unix.socket/1.0/images/%s/export", url.PathEscape(fingerprint)) // Setup the HTTP client devIncusHTTP, err := unixHTTPClient(nil, "/dev/incus/sock") if err == nil { resp, err := incusDownloadImage(fingerprint, unixURI, r.httpUserAgent, devIncusHTTP.Do, req) if err == nil { return resp, nil } } } // Use relatively short response header timeout so as not to hold the image lock open too long. // Deference client and transport in order to clone them so as to not modify timeout of base client. httpClient := *r.http httpTransport := httpClient.Transport.(*http.Transport).Clone() httpTransport.ResponseHeaderTimeout = 30 * time.Second httpClient.Transport = httpTransport // Get the image and expand the fingerprint. image, err := r.ssClient.GetImage(fingerprint) if err != nil { return nil, err } fingerprint = image.Fingerprint // Get the file list files, err := r.ssClient.GetFiles(fingerprint) if err != nil { return nil, err } // Prepare the response resp := ImageFileResponse{} // Download function download := func(path string, filename string, hash string, target io.WriteSeeker) (int64, error) { // Try over http uri, err := url.JoinPath(fmt.Sprintf("http://%s", strings.TrimPrefix(r.httpHost, "https://")), path) if err != nil { return -1, err } size, err := util.DownloadFileHash(context.TODO(), &httpClient, r.httpUserAgent, req.ProgressHandler, req.Canceler, filename, uri, hash, sha256.New(), target) if err != nil { // Handle cancellation if err.Error() == "net/http: request canceled" { return -1, err } // Try over https uri, err := url.JoinPath(r.httpHost, path) if err != nil { return -1, err } size, err = util.DownloadFileHash(context.TODO(), &httpClient, r.httpUserAgent, req.ProgressHandler, req.Canceler, filename, uri, hash, sha256.New(), target) if err != nil { if errors.Is(err, util.ErrNotFound) { logger.Info("Unable to download file by hash, invalidate potentially outdated cache", logger.Ctx{"filename": filename, "uri": uri, "hash": hash}) r.ssClient.InvalidateCache() } return -1, err } } return size, nil } // Download the Incus image file meta, ok := files["meta"] if ok && req.MetaFile != nil { size, err := download(meta.Path, "metadata", meta.Sha256, req.MetaFile) if err != nil { return nil, err } parts := strings.Split(meta.Path, "/") resp.MetaName = parts[len(parts)-1] resp.MetaSize = size } // Download the rootfs rootfs, ok := files["root"] if ok && req.RootfsFile != nil { // Look for deltas (requires xdelta3) downloaded := false _, err := exec.LookPath("xdelta3") if err == nil && req.DeltaSourceRetriever != nil { applyDelta := func(file simplestreams.DownloadableFile, srcPath string, target io.Writer) (int64, error) { // Create temporary file for the delta deltaFile, err := os.CreateTemp(r.tempPath, "incus_image_") if err != nil { return -1, err } defer func() { _ = deltaFile.Close() }() defer func() { _ = os.Remove(deltaFile.Name()) }() // Download the delta _, err = download(file.Path, "rootfs delta", file.Sha256, deltaFile) if err != nil { return -1, err } // Create temporary file for the delta patchedFile, err := os.CreateTemp(r.tempPath, "incus_image_") if err != nil { return -1, err } defer func() { _ = patchedFile.Close() }() defer func() { _ = os.Remove(patchedFile.Name()) }() // Apply it _, err = subprocess.RunCommand("xdelta3", "-f", "-d", "-s", srcPath, deltaFile.Name(), patchedFile.Name()) if err != nil { return -1, err } // Copy to the target size, err := util.SafeCopy(req.RootfsFile, patchedFile) if err != nil { return -1, err } return size, nil } for filename, file := range files { _, srcFingerprint, prefixFound := strings.Cut(filename, "root.delta-") if !prefixFound { continue } // Check if we have the source file for the delta srcPath := req.DeltaSourceRetriever(srcFingerprint, "rootfs") if srcPath == "" { continue } size, err := applyDelta(file, srcPath, req.RootfsFile) if err != nil { return nil, err } parts := strings.Split(rootfs.Path, "/") resp.RootfsName = parts[len(parts)-1] resp.RootfsSize = size downloaded = true } } // Download the whole file if !downloaded { size, err := download(rootfs.Path, "rootfs", rootfs.Sha256, req.RootfsFile) if err != nil { return nil, err } parts := strings.Split(rootfs.Path, "/") resp.RootfsName = parts[len(parts)-1] resp.RootfsSize = size } } // Validate the full image hash. // // Normally we'd do that as we download the image to avoid having to // re-read the data, but because the simplestreams allows retries (HTTP to HTTPS), // we don't have a clean reader that can be used for that. // // Another situation where we couldn't do a streaming hash anyway is when processing delta images. hash256 := sha256.New() if resp.MetaSize > 0 && req.MetaFile != nil { _, err = req.MetaFile.Seek(0, io.SeekStart) if err != nil { return nil, err } _, err := util.SafeCopy(hash256, req.MetaFile) if err != nil { return nil, err } } if resp.RootfsSize > 0 && req.RootfsFile != nil { _, err = req.RootfsFile.Seek(0, io.SeekStart) if err != nil { return nil, err } _, err := util.SafeCopy(hash256, req.RootfsFile) if err != nil { return nil, err } } hash := fmt.Sprintf("%x", hash256.Sum(nil)) if hash != fingerprint { return nil, fmt.Errorf("Image fingerprint doesn't match. Got %s expected %s", hash, fingerprint) } return &resp, nil } // GetImageSecret isn't relevant for the simplestreams protocol. func (r *ProtocolSimpleStreams) GetImageSecret(_ string) (string, error) { return "", errors.New("Private images aren't supported by the simplestreams protocol") } // GetPrivateImage isn't relevant for the simplestreams protocol. func (r *ProtocolSimpleStreams) GetPrivateImage(_ string, _ string) (*api.Image, string, error) { return nil, "", errors.New("Private images aren't supported by the simplestreams protocol") } // GetPrivateImageFile isn't relevant for the simplestreams protocol. func (r *ProtocolSimpleStreams) GetPrivateImageFile(_ string, _ string, _ ImageFileRequest) (*ImageFileResponse, error) { return nil, errors.New("Private images aren't supported by the simplestreams protocol") } // GetImageAliases returns the list of available aliases as ImageAliasesEntry structs. func (r *ProtocolSimpleStreams) GetImageAliases() ([]api.ImageAliasesEntry, error) { return r.ssClient.ListAliases() } // GetImageAliasNames returns the list of available alias names. func (r *ProtocolSimpleStreams) GetImageAliasNames() ([]string, error) { // Get all the images from simplestreams aliases, err := r.ssClient.ListAliases() if err != nil { return nil, err } // And now extract just the names names := []string{} for _, alias := range aliases { names = append(names, alias.Name) } return names, nil } // GetImageAlias returns an existing alias as an ImageAliasesEntry struct. func (r *ProtocolSimpleStreams) GetImageAlias(name string) (*api.ImageAliasesEntry, string, error) { alias, err := r.ssClient.GetAlias("container", name) if err != nil { alias, err = r.ssClient.GetAlias("virtual-machine", name) if err != nil { return nil, "", err } } return alias, "", err } // GetImageAliasType returns an existing alias as an ImageAliasesEntry struct. func (r *ProtocolSimpleStreams) GetImageAliasType(imageType string, name string) (*api.ImageAliasesEntry, string, error) { if imageType == "" { return r.GetImageAlias(name) } alias, err := r.ssClient.GetAlias(imageType, name) if err != nil { return nil, "", err } return alias, "", err } // GetImageAliasArchitectures returns a map of architectures / targets. func (r *ProtocolSimpleStreams) GetImageAliasArchitectures(imageType string, name string) (map[string]*api.ImageAliasesEntry, error) { if imageType == "" { aliases, err := r.ssClient.GetAliasArchitectures("container", name) if err != nil { aliases, err = r.ssClient.GetAliasArchitectures("virtual-machine", name) if err != nil { return nil, err } } return aliases, nil } return r.ssClient.GetAliasArchitectures(imageType, name) } // ExportImage exports (copies) an image to a remote server. func (r *ProtocolSimpleStreams) ExportImage(_ string, _ api.ImageExportPost) (Operation, error) { return nil, errors.New("Exporting images is not supported by the simplestreams protocol") } incus-7.0.0/client/util.go000066400000000000000000000176231517523235500154420ustar00rootroot00000000000000package incus import ( "context" "crypto/tls" "crypto/x509" "encoding/pem" "errors" "fmt" "net" "net/http" "net/url" "strings" "time" "github.com/lxc/incus/v7/shared/proxy" localtls "github.com/lxc/incus/v7/shared/tls" ) // tlsHTTPClient creates an HTTP client with a specified Transport Layer Security (TLS) configuration. // It takes in parameters for client certificates, keys, Certificate Authority, server certificates, // a boolean for skipping verification, a proxy function, and a transport wrapper function. // It returns the HTTP client with the provided configurations and handles any errors that might occur during the setup process. func tlsHTTPClient(client *http.Client, tlsClientCert string, tlsClientKey string, tlsCA string, tlsServerCert string, insecureSkipVerify bool, identicalCertificate bool, proxyFunc func(req *http.Request) (*url.URL, error), transportWrapper func(t *http.Transport) HTTPTransporter) (*http.Client, error) { // Get the TLS configuration tlsConfig, err := localtls.GetTLSConfigMem(tlsClientCert, tlsClientKey, tlsCA, tlsServerCert, insecureSkipVerify) if err != nil { return nil, err } // If asked for an exact match, skip normal validation. if identicalCertificate { tlsConfig.InsecureSkipVerify = true } // Define the http transport transport := &http.Transport{ TLSClientConfig: tlsConfig, Proxy: proxy.FromEnvironment, DisableKeepAlives: true, ExpectContinueTimeout: time.Second * 30, ResponseHeaderTimeout: time.Second * 3600, TLSHandshakeTimeout: time.Second * 5, } // Allow overriding the proxy if proxyFunc != nil { transport.Proxy = proxyFunc } // Special TLS handling transport.DialTLSContext = func(ctx context.Context, network string, addr string) (net.Conn, error) { tlsDial := func(network string, addr string, config *tls.Config, resetName bool) (net.Conn, error) { conn, err := localtls.RFC3493Dialer(ctx, network, addr) if err != nil { return nil, err } // Setup TLS if resetName { hostName, _, err := net.SplitHostPort(addr) if err != nil { hostName = addr } config = config.Clone() config.ServerName = hostName } tlsConn := tls.Client(conn, config) // Validate the connection err = tlsConn.Handshake() if err != nil { _ = conn.Close() return nil, err } if identicalCertificate { // Look for an exact match with the certificate provided. // But ignore any other issue (validity, scope, ...). cs := tlsConn.ConnectionState() if len(cs.PeerCertificates) < 1 { return nil, errors.New("Couldn't validate peer certificate") } if tlsServerCert == "" { return nil, errors.New("Peer certificate wasn't provided") } certBlock, _ := pem.Decode([]byte(tlsServerCert)) if certBlock == nil { return nil, errors.New("Invalid remote certificate") } expectedRemoteCert, err := x509.ParseCertificate(certBlock.Bytes) if err != nil { return nil, err } if !cs.PeerCertificates[0].Equal(expectedRemoteCert) { return nil, errors.New("Remote certificate differs from expected") } } if !config.InsecureSkipVerify { // Check certificate validity. err := tlsConn.VerifyHostname(config.ServerName) if err != nil { _ = conn.Close() return nil, err } } return tlsConn, nil } conn, err := tlsDial(network, addr, transport.TLSClientConfig, false) if err != nil { // We may have gotten redirected to a non-Incus machine return tlsDial(network, addr, transport.TLSClientConfig, true) } return conn, nil } // Define the http client if client == nil { client = &http.Client{} } if transportWrapper != nil { client.Transport = transportWrapper(transport) } else { client.Transport = transport } // Setup redirect policy client.CheckRedirect = func(req *http.Request, via []*http.Request) error { // Replicate the headers req.Header = via[len(via)-1].Header return nil } return client, nil } // unixHTTPClient creates an HTTP client that communicates over a Unix socket. // It takes in the connection arguments and the Unix socket path as parameters. // The function sets up a Unix socket dialer, configures the HTTP transport, and returns the HTTP client with the specified configurations. // Any errors encountered during the setup process are also handled by the function. func unixHTTPClient(args *ConnectionArgs, path string) (*http.Client, error) { // Setup a Unix socket dialer unixDial := func(_ context.Context, _ string, _ string) (net.Conn, error) { raddr, err := net.ResolveUnixAddr("unix", path) if err != nil { return nil, err } return net.DialUnix("unix", nil, raddr) } if args == nil { args = &ConnectionArgs{} } // Define the http transport transport := &http.Transport{ DialContext: unixDial, DisableKeepAlives: true, Proxy: args.Proxy, ExpectContinueTimeout: time.Second * 30, ResponseHeaderTimeout: time.Second * 3600, TLSHandshakeTimeout: time.Second * 5, } // Define the http client client := args.HTTPClient if client == nil { client = &http.Client{} } client.Transport = transport // Setup redirect policy client.CheckRedirect = func(req *http.Request, via []*http.Request) error { // Replicate the headers req.Header = via[len(via)-1].Header return nil } return client, nil } // remoteOperationResult used for storing the error that occurred for a particular remote URL. type remoteOperationResult struct { URL string Error error } func remoteOperationError(msg string, errorOperationResults []remoteOperationResult) error { // Check if empty if len(errorOperationResults) == 0 { return nil } // Check if all identical var err error for _, entry := range errorOperationResults { if err != nil && entry.Error.Error() != err.Error() { errorStrings := make([]string, 0, len(errorOperationResults)) for _, operationResult := range errorOperationResults { errorStrings = append(errorStrings, fmt.Sprintf("%s: %v", operationResult.URL, operationResult.Error)) } return fmt.Errorf("%s:\n - %s", msg, strings.Join(errorStrings, "\n - ")) } err = entry.Error } // Check if successful if err != nil { return fmt.Errorf("%s: %w", msg, err) } return nil } // Set the value of a query parameter in the given URI. func setQueryParam(uri, param, value string) (string, error) { fields, err := url.Parse(uri) if err != nil { return "", err } values := fields.Query() values.Set(param, url.QueryEscape(value)) fields.RawQuery = values.Encode() return fields.String(), nil } // urlsToResourceNames returns a list of resource names extracted from one or more URLs of the same resource type. // The resource type path prefix to match is provided by the matchPathPrefix argument. func urlsToResourceNames(matchPathPrefix string, urls ...string) ([]string, error) { resourceNames := make([]string, 0, len(urls)) for _, urlRaw := range urls { u, err := url.Parse(urlRaw) if err != nil { return nil, fmt.Errorf("Failed parsing URL %q: %w", urlRaw, err) } _, after, found := strings.Cut(u.Path, fmt.Sprintf("%s/", matchPathPrefix)) if !found { return nil, fmt.Errorf("Unexpected URL path %q", u) } resourceNames = append(resourceNames, after) } return resourceNames, nil } // parseFilters translates filters passed at client side to form acceptable by server-side API. func parseFilters(filters []string) string { var result []string for _, filter := range filters { if strings.Contains(filter, "=") { membs := strings.SplitN(filter, "=", 2) result = append(result, fmt.Sprintf("%s eq %s", membs[0], membs[1])) } } return strings.Join(result, " and ") } // HTTPTransporter represents a wrapper around *http.Transport. // It is used to add some pre and postprocessing logic to http requests / responses. type HTTPTransporter interface { http.RoundTripper // Transport what this struct wraps Transport() *http.Transport } incus-7.0.0/cmd/000077500000000000000000000000001517523235500134125ustar00rootroot00000000000000incus-7.0.0/cmd/fuidshift/000077500000000000000000000000001517523235500153775ustar00rootroot00000000000000incus-7.0.0/cmd/fuidshift/main.go000066400000000000000000000014521517523235500166540ustar00rootroot00000000000000package main import ( "os" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/version" ) type cmdGlobal struct { flagVersion bool flagHelp bool } func main() { // shift command (main) shiftCmd := cmdShift{} app := shiftCmd.command() app.SilenceUsage = true app.CompletionOptions = cobra.CompletionOptions{DisableDefaultCmd: true} // Global flags globalCmd := cmdGlobal{} shiftCmd.global = &globalCmd app.PersistentFlags().BoolVar(&globalCmd.flagVersion, "version", false, "Print version number") app.PersistentFlags().BoolVarP(&globalCmd.flagHelp, "help", "h", false, "Print help") // Version handling app.SetVersionTemplate("{{.Version}}\n") app.Version = version.Version // Run the main command and handle errors err := app.Execute() if err != nil { os.Exit(1) } } incus-7.0.0/cmd/fuidshift/main_shift.go000066400000000000000000000044471517523235500200600ustar00rootroot00000000000000package main import ( "errors" "fmt" "os" "github.com/spf13/cobra" "github.com/lxc/incus/v7/shared/idmap" ) type cmdShift struct { global *cmdGlobal flagReverse bool flagTestMode bool } func (c *cmdShift) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "fuidshift [...]" cmd.Short = "UID/GID shifter" cmd.Long = `Description: UID/GID shifter This tool lets you remap a filesystem tree, switching it from one set of UID/GID ranges to another. This is mostly useful when retrieving a wrongly shifted filesystem tree from a backup or broken system and having to remap everything either to the host UID/GID range (uid/gid 0 is root) or to an existing container's range. A range is represented as :::. Where "u" means shift uid, "g" means shift gid and "b" means shift uid and gid. ` cmd.Example = ` fuidshift my-dir/ b:0:100000:65536 u:10000:1000:1` cmd.RunE = c.run cmd.Flags().BoolVarP(&c.flagTestMode, "test", "t", false, "Test mode (no change to files)") cmd.Flags().BoolVarP(&c.flagReverse, "reverse", "r", false, "Perform a reverse mapping") return cmd } func (c *cmdShift) run(cmd *cobra.Command, args []string) error { // Help and usage if len(args) == 0 { return cmd.Help() } // Quick checks. if !c.flagTestMode && os.Geteuid() != 0 { return errors.New("This tool must be run as root") } // Handle mandatory arguments if len(args) < 2 { _ = cmd.Help() return errors.New("Missing required arguments") } directory := args[0] var skipper func(dir string, absPath string, fi os.FileInfo, newuid int64, newgid int64) error if c.flagTestMode { skipper = func(dir string, absPath string, fi os.FileInfo, newuid int64, newgid int64) error { fmt.Printf("I would shift %q to %d %d\n", absPath, newuid, newgid) return errors.New("dry run") } } // Parse the maps idmapSet := &idmap.Set{} for _, arg := range args[1:] { var err error idmapSet, err = idmapSet.Append(arg) if err != nil { return err } } // Reverse shifting if c.flagReverse { err := idmapSet.UnshiftPath(directory, skipper) if err != nil { return err } return nil } // Normal shifting err := idmapSet.ShiftPath(directory, skipper) if err != nil { return err } return nil } incus-7.0.0/cmd/generate-config/000077500000000000000000000000001517523235500164475ustar00rootroot00000000000000incus-7.0.0/cmd/generate-config/README.md000066400000000000000000000161771517523235500177420ustar00rootroot00000000000000# generate-config A small CLI to parse comments in a Golang codebase meant to be used for a documentation tool (like Sphinx for example). It parses the comments from the AST and extracts their documentation. ## Disclaimer `generate-config` is intended for internal use within the [Incus](https://github.com/lxc/incus) code base. There are no guarantees regarding backwards compatibility, API stability, or long-term availability. It may change or be removed at any time without prior notice. Use at your own discretion. ## Usage ```shell $ generate-config -h Usage of generate-config: -e value Path that will be excluded from the process ``` ## Formatting A comment is formatted this way: ```go // gendoc:generate(entity=cluster, group=cluster, key=scheduler.instance) // // // --- // shortdesc: Possible values are all, manual and group. See Automatic placement of instances for more information. // condition: container // defaultdesc: `all` // type: integer // liveupdate: `yes` // : clusterConfigKeys := map[string]func(value string) error{ "scheduler.instance": validate.Optional(validate.IsOneOf("all", "group", "manual")), } for k, v := range config { // gendoc:generate(entity=cluster, group=cluster, key=user.*) // // This is the real long desc. // // With two paragraphs. // // And a list: // // - Item // - Item // - Item // // example of a table: // // Key | Type | Scope | Default | Description // :-- | :--- | :---- | :------ | :---------- // `acme.agree_tos` | bool | global | `false` | Agree to ACME terms of service // `acme.ca_url` | string | global | `https://acme-v02.api.letsencrypt.org/directory` | URL to the directory resource of the ACME service // `acme.domain` | string | global | - | Domain for which the certificate is issued // `acme.email` | string | global | - | Email address used for the account registration // // --- // shortdesc: Free form user key/value storage (can be used in search). // condition: container // default: - // type: string // liveupdate: `yes` if strings.HasPrefix(k, "user.") { continue } validator, ok := clusterConfigKeys[k] if !ok { return fmt.Errorf("Invalid cluster configuration key %q", k) } err := validator(v) if err != nil { return fmt.Errorf("Invalid cluster configuration key %q value", k) } } return nil ``` The go-swagger spec from source generator can only handles `swagger:meta` (global file/package level documentation), `swagger:route` (API endpoints), `swagger:params` (function parameters), `swagger:operation` (method documentation), `swagger:response` (API response content documentation), `swagger:model` (struct documentation) generation. In our use case, we would want a config variable spec generator that can bundle any key-value data pairs alongside metadata to build a sense of hierarchy and identity (we want to associate a unique key to each gendoc comment group that will also be displayed in the generated documentation) In a swagger fashion, `generate-config` can associate metadata key-value pairs (here for example, `group` and `key`) to data key-value pairs. As a result, it can generate a YAML tree out of the code documentation and also a Markdown document. ### Output Here is the JSON output of the example shown above: ```json { "configs": { "cluster": [ { "scheduler.instance": { "condition": "container", "defaultdesc": "`all`", "liveupdate": "`yes`", "longdesc": "", "shortdesc": " Possible values are all, manual and group. See Automatic placement of instances for more", "type": "integer" } }, { "user.*": { "condition": "container", "defaultdesc": "-", "liveupdate": "`yes`", "longdesc": " This is the real long desc. With two paragraphs. And a list: - Item - Item - Item And a table: Key | Type | Scope | Default | Description :-- | :--- | :---- | :------ | :---------- `acme.agree_tos` | bool | global | `false` | Agree to ACME terms of service `acme.ca_url` | string | global | `https://acme-v02.api.letsencrypt.org/directory` | URL to the directory resource of the ACME service `acme.domain` | string | global | - | Domain for which the certificate is issued `acme.email` | string | global | - | Email address used for the account registration ", "shortdesc": "Free form user key/value storage (can be used in search).", "type": "string" } } ], } } ``` Here is the `.txt` output of the example shown above: ```plain \`\`\`{config:option} user.* cluster :type: string :liveupdate: `yes` :shortdesc: Free form user key/value storage (can be used in search). :condition: container :default: - This is the real long desc. With two paragraphs. And a list: - Item - Item - Item example of a table: Key | Type | Scope | Default | Description :-- | :--- | :---- | :------ | :---------- `acme.agree_tos` | bool | global | `false` | Agree to ACME terms of service `acme.ca_url` | string | global | `https://acme-v02.api.letsencrypt.org/directory` | URL to the directory resource of the ACME service `acme.domain` | string | global | - | Domain for which the certificate is issued `acme.email` | string | global | - | Email address used for the account registration \`\`\` \`\`\`{config:option} scheduler.instance cluster :liveupdate: `yes` :shortdesc: Possible values are all, manual and group. See Automatic placement of instances for more information. :condition: container :default: `all` :type: integer \`\`\` ``` incus-7.0.0/cmd/generate-config/incus_doc.go000066400000000000000000000265441517523235500207570ustar00rootroot00000000000000package main import ( "bytes" "encoding/json" "fmt" "go/ast" "go/parser" "go/token" "log" "os" "path/filepath" "regexp" "slices" "sort" "strings" "time" ) var ( globalGenDocRegex = regexp.MustCompile(`(?m)gendoc:generate\((.*)\)([\S\s]+)\s+---\n([\S\s]+)`) genDocMetadataRegex = regexp.MustCompile(`(?m)([^,\s]+)=([^,\s]+)`) genDocDataRegex = regexp.MustCompile(`(?m)([\S]+):[\s]+([\S \"\']+)`) ) var mdKeys = []string{"entity", "group", "key"} // IterableAny is a generic type that represents a type or an iterable container. type IterableAny interface { any | []any } // doc is the structure of the JSON file that contains the generated configuration metadata. type doc struct { Configs map[string]any `json:"configs"` } // sortConfigKeys alphabetically sorts the entries by key (config option key) within each config group in an entity. func sortConfigKeys(projectEntries map[string]any) { for _, entityValue := range projectEntries { groupValues, ok := entityValue.(map[string]any) if !ok { continue } for _, groupValue := range groupValues { configEntries, ok := groupValue.(map[string]any)["keys"].([]any) if !ok { continue } sort.Slice(configEntries, func(i, j int) bool { // Get the only key for each map element in the slice var keyI, keyJ string confI, confJ := configEntries[i].(map[string]any), configEntries[j].(map[string]any) for k := range confI { keyI = k break // There is only one key-value pair in each map } for k := range confJ { keyJ = k break // There is only one key-value pair in each map } // Compare the keys return keyI < keyJ }) } } } // getSortedKeysFromMap returns the keys of a map sorted alphabetically. func getSortedKeysFromMap[K string, V IterableAny](m map[K]V) []K { keys := make([]K, 0, len(m)) for k := range m { keys = append(keys, k) } slices.Sort(keys) return keys } func parse(path string, outputJSONPath string, excludedPaths []string) (*doc, error) { jsonDoc := &doc{} docKeys := make(map[string]struct{}) projectEntries := make(map[string]any) err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { if err != nil { return err } // Skip excluded paths if slices.Contains(excludedPaths, path) { if info.IsDir() { log.Printf("Skipping excluded directory: %v", path) return filepath.SkipDir } log.Printf("Skipping excluded file: %v", path) return nil } // Only process go files if !info.IsDir() && filepath.Ext(path) != ".go" { return nil } // Continue walking if directory if info.IsDir() { return nil } // Parse file and create the AST fset := token.NewFileSet() var f *ast.File f, err = parser.ParseFile(fset, path, nil, parser.ParseComments) if err != nil { return err } fileEntries := make([]map[string]any, 0) // Loop in comment groups for _, cg := range f.Comments { s := cg.Text() entry := make(map[string]any) groupKeyEntry := make(map[string]any) for _, match := range globalGenDocRegex.FindAllStringSubmatch(s, -1) { // check that the match contains the expected number of groups if len(match) != 4 { continue } log.Printf("Found gendoc at %s", fset.Position(cg.Pos()).String()) metadata := match[1] longdesc := match[2] data := match[3] // process metadata metadataMap := make(map[string]string) var entityKey string var groupKey string var simpleKey string for _, mdKVMatch := range genDocMetadataRegex.FindAllStringSubmatch(metadata, -1) { if len(mdKVMatch) != 3 { continue } mdKey := mdKVMatch[1] mdValue := mdKVMatch[2] // check that the metadata key is among the expected ones if !slices.Contains(mdKeys, mdKey) { continue } if mdKey == "entity" { entityKey = mdValue } if mdKey == "group" { groupKey = mdValue } if mdKey == "key" { simpleKey = mdValue } metadataMap[mdKey] = mdValue } // Check that this metadata is not already present mdKeyHash := fmt.Sprintf("%s/%s/%s", entityKey, groupKey, simpleKey) _, ok := docKeys[mdKeyHash] if ok { return fmt.Errorf("Duplicate key '%s' found at %s", mdKeyHash, fset.Position(cg.Pos()).String()) } docKeys[mdKeyHash] = struct{}{} configKeyEntry := make(map[string]any) configKeyEntry[metadataMap["key"]] = make(map[string]any) configKeyEntry[metadataMap["key"]].(map[string]any)["longdesc"] = strings.TrimLeft(longdesc, "\n\t\v\f\r") for _, dataKVMatch := range genDocDataRegex.FindAllStringSubmatch(data, -1) { if len(dataKVMatch) != 3 { continue } configKeyEntry[metadataMap["key"]].(map[string]any)[dataKVMatch[1]] = dataKVMatch[2] } _, ok = groupKeyEntry[metadataMap["group"]] if ok { _, ok = groupKeyEntry[metadataMap["group"]].(map[string]any)["keys"] if ok { groupKeyEntry[metadataMap["group"]].(map[string]any)["keys"] = append( groupKeyEntry[metadataMap["group"]].(map[string]any)["keys"].([]any), configKeyEntry, ) } else { groupKeyEntry[metadataMap["group"]].(map[string]any)["keys"] = []any{configKeyEntry} } } else { groupKeyEntry[metadataMap["group"]] = make(map[string]any) groupKeyEntry[metadataMap["group"]].(map[string]any)["keys"] = []any{configKeyEntry} } entry[metadataMap["entity"]] = groupKeyEntry } if len(entry) > 0 { fileEntries = append(fileEntries, entry) } } // Update projectEntries for _, entry := range fileEntries { for entityKey, entityValue := range entry { _, ok := projectEntries[entityKey] if !ok { projectEntries[entityKey] = entityValue } else { groupValues, ok := entityValue.(map[string]any) if !ok { continue } for groupKey, groupValue := range groupValues { _, ok := projectEntries[entityKey].(map[string]any)[groupKey] if !ok { projectEntries[entityKey].(map[string]any)[groupKey] = groupValue } else { // merge the config keys configKeys, ok := groupValue.(map[string]any)["keys"].([]any) if !ok { continue } projectEntries[entityKey].(map[string]any)[groupKey].(map[string]any)["keys"] = append( projectEntries[entityKey].(map[string]any)[groupKey].(map[string]any)["keys"].([]any), configKeys..., ) } } } } } return nil }) if err != nil { return nil, err } // sort the config keys alphabetically sortConfigKeys(projectEntries) jsonDoc.Configs = projectEntries data, err := json.MarshalIndent(jsonDoc, "", "\t") if err != nil { return nil, fmt.Errorf("Error while marshaling project documentation: %v", err) } if outputJSONPath != "" { buf := bytes.NewBufferString("") _, err = buf.Write(data) if err != nil { return nil, fmt.Errorf("Error while writing the JSON project documentation: %v", err) } err := os.WriteFile(outputJSONPath, buf.Bytes(), 0o644) if err != nil { return nil, fmt.Errorf("Error while writing the JSON project documentation: %v", err) } } return jsonDoc, nil } func writeDocFile(inputJSONPath, outputTxtPath string) error { countMaxBackTicks := func(s string) int { count, currCount := 0, 0 n := len(s) for i := range n { if s[i] == '`' { currCount++ continue } if currCount > count { count = currCount } currCount = 0 } return count } specialChars := []string{"", "*", "_", "#", "+", "-", ".", "!", "no", "yes"} // read the JSON file which is the source of truth for the generation of the .txt file jsonData, err := os.ReadFile(inputJSONPath) if err != nil { return err } var jsonDoc doc err = json.Unmarshal(jsonData, &jsonDoc) if err != nil { return err } sortedEntityKeys := getSortedKeysFromMap(jsonDoc.Configs) // create a string buffer buffer := bytes.NewBufferString("// Code generated by generate-config from the incus project; DO NOT EDIT.\n\n") for _, entityKey := range sortedEntityKeys { entityEntries := jsonDoc.Configs[entityKey] sortedGroupKeys := getSortedKeysFromMap(entityEntries.(map[string]any)) for _, groupKey := range sortedGroupKeys { groupEntries := entityEntries.(map[string]any)[groupKey] fmt.Fprintf(buffer, "\n", entityKey, groupKey) groupKeys, ok := groupEntries.(map[string]any)["keys"].([]any) if !ok { continue } for _, configEntry := range groupKeys { configEntry, ok := configEntry.(map[string]any) if !ok { continue } for configKey, configContent := range configEntry { // There is only one key-value pair in each map kvBuffer := bytes.NewBufferString("") var backticksCount int var longDescContent string sortedConfigContentKeys := getSortedKeysFromMap(configContent.(map[string]any)) for _, configEntryContentKey := range sortedConfigContentKeys { configContentValue := configContent.(map[string]any)[configEntryContentKey] if configEntryContentKey == "longdesc" { backticksCount = countMaxBackTicks(configContentValue.(string)) c, ok := configContentValue.(string) if ok { longDescContent = c } continue } configContentValueStr, ok := configContentValue.(string) if ok { if (strings.HasSuffix(configContentValueStr, "`") && strings.HasPrefix(configContentValueStr, "`")) || slices.Contains(specialChars, configContentValueStr) { configContentValueStr = fmt.Sprintf("\"%s\"", configContentValueStr) } } else { switch configEntryContentTyped := configContentValue.(type) { case int, float64, bool: configContentValueStr = fmt.Sprint(configEntryContentTyped) case time.Time: configContentValueStr = fmt.Sprint(configEntryContentTyped.Format(time.RFC3339)) } } var quoteFormattedValue string if strings.Contains(configContentValueStr, `"`) { if strings.HasPrefix(configContentValueStr, `"`) && strings.HasSuffix(configContentValueStr, `"`) { for i, s := range configContentValueStr[1 : len(configContentValueStr)-1] { if s == '"' { _ = strings.Replace(configContentValueStr, `"`, `\"`, i) } } quoteFormattedValue = configContentValueStr } else { quoteFormattedValue = strings.ReplaceAll(configContentValueStr, `"`, `\"`) } } else { quoteFormattedValue = fmt.Sprintf("\"%s\"", configContentValueStr) } fmt.Fprintf(kvBuffer, ":%s: %s\n", configEntryContentKey, quoteFormattedValue) } if backticksCount < 3 { fmt.Fprintf(buffer, "```{config:option} %s %s-%s\n%s%s\n```\n\n", configKey, entityKey, groupKey, kvBuffer.String(), strings.TrimLeft(longDescContent, "\n")) } else { configQuotes := strings.Repeat("`", backticksCount+1) fmt.Fprintf(buffer, "%s{config:option} %s %s-%s\n%s%s\n%s\n\n", configQuotes, configKey, entityKey, groupKey, kvBuffer.String(), strings.TrimLeft(longDescContent, "\n"), configQuotes) } } } fmt.Fprintf(buffer, "\n", entityKey, groupKey) } } err = os.WriteFile(outputTxtPath, buffer.Bytes(), 0o644) if err != nil { return fmt.Errorf("Error while writing the Markdown project documentation: %v", err) } return nil } incus-7.0.0/cmd/generate-config/incus_doc_test.go000066400000000000000000000044351517523235500220110ustar00rootroot00000000000000package main import ( "testing" "github.com/stretchr/testify/assert" ) // Test the alphabetical sorting of a `generate-config` JSON structure. func TestJSONSorted(t *testing.T) { projectEntries := make(map[string]any) projectEntries["entityKey1"] = map[string]any{ "groupKey1": map[string]any{ "keys": []any{ map[string]any{ "a.core.server.test.b": map[string]string{ "todo5": "stuff", "todo6": "stuff", }, }, map[string]any{ "a.core.server.test.c": map[string]string{ "todo3": "stuff", "todo4": "stuff", }, }, map[string]any{ "b.core.server.test.a": map[string]string{ "todo1": "stuff", "todo2": "stuff", }, }, }, }, } projectEntries["entityKey2"] = map[string]any{ "groupKey2": map[string]any{ "keys": []any{ map[string]any{ "000.111.222": map[string]string{ "todo9": "stuff", "todo10": "stuff", }, }, map[string]any{ "aaa.ccc.bbb": map[string]string{ "todo7": "stuff", "todo8": "stuff", }, }, map[string]any{ "zzz.*": map[string]string{ "todo11": "stuff", "todo12": "stuff", }, }, }, }, } sortedProjectEntries := make(map[string]any) sortedProjectEntries["entityKey1"] = map[string]any{ "groupKey1": map[string]any{ "keys": []any{ map[string]any{ "a.core.server.test.b": map[string]string{ "todo5": "stuff", "todo6": "stuff", }, }, map[string]any{ "a.core.server.test.c": map[string]string{ "todo3": "stuff", "todo4": "stuff", }, }, map[string]any{ "b.core.server.test.a": map[string]string{ "todo1": "stuff", "todo2": "stuff", }, }, }, }, } sortedProjectEntries["entityKey2"] = map[string]any{ "groupKey2": map[string]any{ "keys": []any{ map[string]any{ "000.111.222": map[string]string{ "todo9": "stuff", "todo10": "stuff", }, }, map[string]any{ "aaa.ccc.bbb": map[string]string{ "todo7": "stuff", "todo8": "stuff", }, }, map[string]any{ "zzz.*": map[string]string{ "todo11": "stuff", "todo12": "stuff", }, }, }, }, } sortConfigKeys(projectEntries) assert.Equal(t, sortedProjectEntries, projectEntries) } incus-7.0.0/cmd/generate-config/main.go000066400000000000000000000026541517523235500177310ustar00rootroot00000000000000package main import ( "errors" "fmt" "log" "os" "github.com/spf13/cobra" ) var ( exclude []string jsonOutput string txtOutput string rootCmd = &cobra.Command{ Use: "generate-config", Short: "generate-config - a simple tool to generate documentation for Incus", Long: "generate-config - a simple tool to generate documentation for Incus. It outputs a YAML and a Markdown file that contain the content of all `gendoc:generate` statements in the project.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { if len(args) != 1 { return errors.New("Please provide a path to the project") } path := args[0] _, err := parse(path, jsonOutput, exclude) if err != nil { return err } if txtOutput != "" { err = writeDocFile(jsonOutput, txtOutput) if err != nil { return err } } return nil }, } ) func main() { rootCmd.Flags().StringSliceVarP(&exclude, "exclude", "e", []string{}, "Path to exclude from the process") rootCmd.Flags().StringVarP(&jsonOutput, "json", "j", "configuration.json", "Output JSON file containing the generated configuration") rootCmd.Flags().StringVarP(&txtOutput, "txt", "t", "", "Output TXT file containing the generated documentation") err := rootCmd.Execute() if err != nil { fmt.Fprintf(os.Stderr, "generate-config failed: %v", err) os.Exit(1) } log.Println("generate-config finished successfully") } incus-7.0.0/cmd/generate-database/000077500000000000000000000000001517523235500167465ustar00rootroot00000000000000incus-7.0.0/cmd/generate-database/README.md000066400000000000000000000327341517523235500202360ustar00rootroot00000000000000# `generate-database` ## Introduction `generate-database` is a database statement and associated `go` function generator for Incus and related projects. `generate-database` utilizes `go`'s code generation directives (`//go:generate ...`) alongside go's [ast](https://pkg.go.dev/go/ast) and [types](https://pkg.go.dev/go/types) packages for parsing the syntax tree for go structs and variables. We use `generate-database` for the majority of our SQL statements and database interactions on the `go` side for consistency and predictability. ## Disclaimer `generate-database` is intended for internal use within the [Incus](https://github.com/lxc/incus) code base. There are no guarantees regarding backwards compatibility, API stability, or long-term availability. It may change or be removed at any time without prior notice. Use at your own discretion. ## Usage ### Initialization #### Package global Once per package, that uses `generate-database` for generation of database statements and associated `go` functions, `generate-database` needs to be invoked using the following `go:generate` instruction: ```go //go:generate generate-database db mapper generate ``` This will initiate a call to `generate-database db mapper generate`, which will then search for `//generate-database:mapper` directives in the same file and process those. The following flags are available: * `--package` / `-p`: Package import paths to search for structs to parse. Defaults to the caller package. Can be used more than once. #### File Generally the first thing we will want to do for any newly generated file is to ensure the file has been cleared of content: ```go //generate-database:mapper target instances.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" ``` ### Generation Directive Arguments The generation directive arguments have the following form: `//generate-database:mapper flags ` The following flags are available: * `--build` / `-b`: build comment to include (commands: `reset`) * `--interface` / `-i`: create interface files (commands: `reset`, `method`) * `--entity` / `-e`: database entity to generate the method or statement for (commands: `stmt`, `method`) Example: * `//generate-database:mapper stmt -e instance objects table=table_name` The `table` key can be used to override the generated table name for a specified one. * `//generate-database:mapper method -i -e instance Create references=Config,Device` For some tables (defined below under [Additional Information](#Additional-Information) as [EntityTable](#EntityTable), the `references=` key can be provided with the name of a [ReferenceTable](#ReferenceTable) or [MapTable](#MapTable) struct. This directive would produce `CreateInstance` in addition to `CreateInstanceConfig` and `CreateInstanceDevices`: * `//generate-database:mapper method -i -e instance_profile Create struct=Instance` * `//generate-database:mapper method -i -e instance_profile Create struct=Profile` For some tables (defined below under [Additional Information](#Additional-Information) as [AssociationTable](#AssociationTable), `method` declarations must include a `struct=` to indicate the directionality of the function. An invocation can be called for each direction. This would produce `CreateInstanceProfiles` and `CreateProfileInstances` respectively. ### SQL Statement Generation SQL generation supports the following SQL statement types: Type | Description :--- | :---- `objects` | Creates a basic SELECT statement of the form `SELECT FROM ORDER BY `. `objects-by--and-...` | Parses a pre-existing SELECT statement variable declaration of the form produced by`objects`, and appends a `WHERE` clause with the given fields located in the associated struct. Specifically looks for a variable declaration of the form `var Objects = RegisterStmt("SQL String")` `names` | Creates a basic SELECT statement of the form `SELECT FROM
ORDER BY `. `names-by--and-...` | Parses a pre-existing SELECT statement variable declaration of the form produced by`names`, and appends a `WHERE` clause with the given fields located in the associated struct. Specifically looks for a variable declaration of the form `var Objects = RegisterStmt("SQL String")` `create` | Creates a basic INSERT statement of the form `INSERT INTO
VALUES`. `create-or-replace` | Creates a basic INSERT statement of the form `INSERT OR REPLACE INTO
VALUES`. `delete-by--and-...` | Creates a DELETE statement of the form `DELETE FROM
WHERE ` where the constraint is based on the given fields of the associated struct. `id` | Creates a basic SELECT statement that returns just the internal ID of the table. `rename` | Creates an UPDATE statement that updates the primary key of a table: `UPDATE
SET WHERE `. `update` | Creates an UPDATE statement of the form `UPDATE
SET WHERE `. #### Examples ```go //generate-database:mapper stmt -e instance objects //generate-database:mapper stmt -e instance objects-by-Name-and-Project //generate-database:mapper stmt -e instance create //generate-database:mapper stmt -e instance update //generate-database:mapper stmt -e instance delete-by-Name-and-Project ``` #### Statement Related Go Tags There are several tags that can be added to fields of a struct that will be parsed by the `ast` package. Tag | Description :-- | :---- `sql=
.` | Supply an explicit table and column name to use for this struct field. `coalesce=` | Generates a SQL coalesce function with the given value `coalesce(, value)`. `order=yes` | Override the default `ORDER BY` columns with all fields specifying this tag. `join=` | Applies a `JOIN` of the form `JOIN ON
. = `. `leftjoin=` | Applies a `LEFT JOIN` of the same form as a `JOIN`. `joinon=
.` | Overrides the default `JOIN ON` clause with the given table and column, replacing `
.` above. `jointo=` | Overrides the default target column `id` with the given column, replacing the `id` in `` above. This is intended for "loose" foreign keys, not using the ID column. Therefore, this is intended to be used in conjunction with `joinon` and `omit=create,update` to get the expected behavior. `primary=yes` | Assigns column associated with the field to be sufficient for returning a row from the table. Will default to `Name` if unspecified. Fields with this key will be included in the default 'ORDER BY' clause. `omit=` | Omits a given field from consideration for the comma separated list of statement types (`create`, `objects-by-Name`, `update`). `ignore` | Outright ignore the struct field as though it does not exist. `ignore` needs to be the only tag value in order to be recognized. `marshal=` | Marshal/Unmarshal data into the field. The column must be a TEXT column. If `marshal=yes`, then the type must implement both `Marshal` and `Unmarshal`. If `marshal=json`, the type is marshaled to JSON using the standard library ([json.Marshal](https://pkg.go.dev/encoding/json#Marshal)). This works for entity tables only, and not for association or mapping tables. `create_timestamp` | Automatically set the value of this column to the current time (UTC) when the respective record is created, namely in `Create` and `CreateOrReplace` (regardless if the record is actually created or updated). `update_timestamp` | Automatically set the value of this column to the current time (UTC) for every operation altering the record, namely `Create`, `CreateOrReplace`, `Rename` and `Update`. ### Go Function Generation Go function generation supports the following types: Type | Description :--- | :---- `GetNames` | Return a slice of primary keys for all rows in a table matching the filter. Cannot be used with composite keys. `GetMany` | Return a slice of structs for all rows in a table matching the filter. `GetOne` | Return a single struct corresponding to a row with the given primary keys. Depends on `GetMany`. `ID` | Return the ID column from the table corresponding to the given primary keys. `Exists` | Returns whether there is an row in the table with the given primary keys. Depends on `ID.` `Create` | Insert a row from the given struct into the table if not already present. Depends on `Exists` `CreateOrReplace` | Insert a row from the given struct into the table, regardless of if an entry already exists. `Rename` | Update the primary key for a table row. `Update` | Update the columns at a given row, specified by primary key. `DeleteOne` | Delete exactly one row from the table. `DeleteMany` | Delete one or more rows from the table. ```go //generate-database:mapper method -i -e instance GetMany //generate-database:mapper method -i -e instance GetOne //generate-database:mapper method -i -e instance ID //generate-database:mapper method -i -e instance Exist //generate-database:mapper method -i -e instance Create //generate-database:mapper method -i -e instance Update //generate-database:mapper method -i -e instance DeleteOne-by-Project-and-Name //generate-database:mapper method -i -e instance DeleteMany-by-Name ``` ### Additional Information All structs should have an `ID` field, as well as an additional `Filter` struct prefixed with the original struct name. This should include any fields that should be considered for filtering in `WHERE` clauses. These fields should be pointers to facilitate omission and inclusion without setting default values. Example: ```go type Instance struct { ID int Name string Project string } type InstanceFilter struct { Name *string Project *string } ``` `generate-database` will handle parsing of structs differently based on the composition of the struct in four different ways. Non-`EntityType` structs will only support `GetMany`, `Create`, `Update`, and `Delete` functions. ### EntityTable Most structs will get treated this way, and represent a normal table. * If a table has an associated table for which a `ReferenceTable` or `MapTable` as defined below is applicable, functions specific to this entity can be generated by including a comma separated list to `references=` in the code generation directive for `GetMany`, `Create`, or `Update` directives. * The `Create` method directive for `EntityTable` will expect on the `ID` and `Exist` method directives to be present. * All `CREATE`, `UPDATE`, and `DELETE` statements that include a joined table will expect a `var ID = RegisterStmt('SQL String')` to exist for the joining table. ### ReferenceTable A struct that contains a field named `ReferenceID` will be parsed this way. `generate-database` will use this struct to generate more abstract SQL statements and functions of the form `_`. Real world invocation of these statements and functions should be done through an `EntityTable` `method` call with the tag `references=`. This `EntityTable` will replace the `` above. Example: ```go //generate-database:mapper stmt -e device create //generate-database:mapper method -e device Create type Device struct { ID int ReferenceID int Name string Type string } //... //generate-database:mapper method -e instance Create references=Device // This will produce a function called `CreateInstanceDevices`. ``` ### MapTable This is a special type of `ReferenceTable` with fields named `Key` and `Value`. On the SQL side, this is treated exactly like a `ReferenceTable`, but on the `go` side, the return values will be a map. Example: ```go //generate-database:mapper stmt -e config create //generate-database:mapper method -e config Create type Config struct { ID int ReferenceID int Key string Value string } //... //generate-database:mapper method -e instance Create references=Config // This will produce a function called `CreateInstanceConfig`, which will return a `map[string]string`. ``` ### AssociationTable This is a special type of table that contains two fields of the form `ID`, where `` corresponds to two other structs present in the same package. This will generate code for compound tables of the form `_` that are generally used to associate two tables together by their IDs. `method` generation declarations for these statements should include a `struct=` to indicate the directionality of the function. An invocation can be called for each direction. Example: ```go //generate-database:mapper method -i -e instance_profile Create struct=Instance //generate-database:mapper method -i -e instance_profile Create struct=Profile type InstanceProfile struct { InstanceID int ProfileID int } ``` incus-7.0.0/cmd/generate-database/db.go000066400000000000000000000166771517523235500177030ustar00rootroot00000000000000//go:build linux && cgo && !agent package main import ( "encoding/csv" "errors" "fmt" "go/build" "os" "strings" "github.com/spf13/cobra" "github.com/spf13/pflag" "golang.org/x/tools/go/packages" "github.com/lxc/incus/v7/cmd/generate-database/db" "github.com/lxc/incus/v7/cmd/generate-database/file" "github.com/lxc/incus/v7/cmd/generate-database/lex" ) // Return a new db command. func newDb() *cobra.Command { cmd := &cobra.Command{ Use: "db [sub-command]", Short: "Database-related code generation.", RunE: func(cmd *cobra.Command, args []string) error { return errors.New("Not implemented") }, } cmd.AddCommand(newDbSchema()) cmd.AddCommand(newDbMapper()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } return cmd } func newDbSchema() *cobra.Command { cmd := &cobra.Command{ Use: "schema", Short: "Generate database schema by applying updates.", RunE: func(cmd *cobra.Command, args []string) error { return db.UpdateSchema() }, } return cmd } func newDbMapper() *cobra.Command { cmd := &cobra.Command{ Use: "mapper [sub-command]", Short: "Generate code mapping database rows to Go structs.", RunE: func(cmd *cobra.Command, args []string) error { return errors.New("Not implemented") }, } cmd.AddCommand(newDbMapperGenerate()) return cmd } func newDbMapperGenerate() *cobra.Command { var pkgs *[]string var boilerplateFilename string cmd := &cobra.Command{ Use: "generate", Short: "Generate database statememnts and transaction method and interface signature.", RunE: func(cmd *cobra.Command, args []string) error { if os.Getenv("GOPACKAGE") == "" { return errors.New("GOPACKAGE environment variable is not set") } return generate(*pkgs, boilerplateFilename) }, } flags := cmd.Flags() pkgs = flags.StringArrayP("package", "p", []string{}, "Go package where the entity struct is declared") flags.StringVarP(&boilerplateFilename, "boilerplate-file", "b", "-", "Filename of the file where the mapper boilerplate is written to") return cmd } const prefix = "//generate-database:mapper " func generate(pkgs []string, boilerplateFilename string) error { localPath, err := os.Getwd() if err != nil { return err } localPkg, err := packages.Load(&packages.Config{Mode: packages.NeedName}, localPath) if err != nil { return err } localPkgPath := localPkg[0].PkgPath if len(pkgs) == 0 { pkgs = []string{localPkgPath} } parsedPkgs, err := packageLoad(pkgs) if err != nil { return err } err = file.Boilerplate(boilerplateFilename) if err != nil { return err } registeredSQLStmts := map[string]string{} for _, parsedPkg := range parsedPkgs { for _, goFile := range parsedPkg.CompiledGoFiles { body, err := os.ReadFile(goFile) if err != nil { return err } // Reset target to stdout target := "-" lines := strings.Split(string(body), "\n") for _, line := range lines { // Lazy matching for prefix, does not consider Go syntax and therefore // lines starting with prefix, that are part of e.g. multiline strings // match as well. This is highly unlikely to cause false positives. after, ok := strings.CutPrefix(line, prefix) if ok { line = after // Use csv parser to properly handle arguments surrounded by double quotes. r := csv.NewReader(strings.NewReader(line)) r.Comma = ' ' // space args, err := r.Read() if err != nil { return err } if len(args) == 0 { return errors.New("command missing") } command := args[0] switch command { case "target": if len(args) != 2 { return fmt.Errorf("invalid arguments for command target, one argument for the target filename: %s", line) } target = args[1] case "reset": err = commandReset(args[1:], parsedPkgs, target, localPkgPath) case "stmt": err = commandStmt(args[1:], target, parsedPkgs, registeredSQLStmts, localPkgPath) case "method": err = commandMethod(args[1:], target, parsedPkgs, registeredSQLStmts, localPkgPath) default: err = fmt.Errorf("unknown command: %s", command) } if err != nil { return err } } } } } return nil } func commandReset(commandLine []string, parsedPkgs []*packages.Package, target string, localPkgPath string) error { var err error flags := pflag.NewFlagSet("", pflag.ContinueOnError) iface := flags.BoolP("interface", "i", false, "create interface files") buildComment := flags.StringP("build", "b", "", "build comment to include") err = flags.Parse(commandLine) if err != nil { return err } imports := db.Imports for _, pkg := range parsedPkgs { if pkg.PkgPath == localPkgPath { continue } imports = append(imports, pkg.PkgPath) } err = file.Reset(target, imports, *buildComment, *iface) if err != nil { return err } return nil } func commandStmt(commandLine []string, target string, parsedPkgs []*packages.Package, registeredSQLStmts map[string]string, localPkgPath string) error { var err error flags := pflag.NewFlagSet("", pflag.ContinueOnError) entity := flags.StringP("entity", "e", "", "database entity to generate the statement for") err = flags.Parse(commandLine) if err != nil { return err } if len(flags.Args()) < 1 { return errors.New("argument missing for stmt command") } kind := flags.Arg(0) config, err := parseParams(flags.Args()[1:]) if err != nil { return err } stmt, err := db.NewStmt(localPkgPath, parsedPkgs, *entity, kind, config, registeredSQLStmts) if err != nil { return err } return file.Append(*entity, target, stmt, false) } func commandMethod(commandLine []string, target string, parsedPkgs []*packages.Package, registeredSQLStmts map[string]string, localPkgPath string) error { var err error flags := pflag.NewFlagSet("", pflag.ContinueOnError) iface := flags.BoolP("interface", "i", false, "create interface files") entity := flags.StringP("entity", "e", "", "database entity to generate the method for") err = flags.Parse(commandLine) if err != nil { return err } if len(flags.Args()) < 1 { return errors.New("argument missing for method command") } kind := flags.Arg(0) config, err := parseParams(flags.Args()[1:]) if err != nil { return err } method, err := db.NewMethod(localPkgPath, parsedPkgs, *entity, kind, config, registeredSQLStmts) if err != nil { return err } return file.Append(*entity, target, method, *iface) } func packageLoad(pkgs []string) ([]*packages.Package, error) { pkgPaths := []string{} for _, pkg := range pkgs { if pkg == "" { var err error localPath, err := os.Getwd() if err != nil { return nil, err } pkgPaths = append(pkgPaths, localPath) } else { importPkg, err := build.Import(pkg, "", build.FindOnly) if err != nil { return nil, fmt.Errorf("Invalid import path %q: %w", pkg, err) } pkgPaths = append(pkgPaths, importPkg.Dir) } } parsedPkgs, err := packages.Load(&packages.Config{ Mode: packages.LoadTypes | packages.NeedTypesInfo, }, pkgPaths...) if err != nil { return nil, err } return parsedPkgs, nil } func parseParams(args []string) (map[string]string, error) { config := map[string]string{} for _, arg := range args { key, value, err := lex.KeyValue(arg) if err != nil { return nil, fmt.Errorf("Invalid config parameter: %w", err) } config[key] = value } return config, nil } incus-7.0.0/cmd/generate-database/db/000077500000000000000000000000001517523235500173335ustar00rootroot00000000000000incus-7.0.0/cmd/generate-database/db/constants.go000066400000000000000000000003471517523235500217020ustar00rootroot00000000000000//go:build linux && cgo && !agent package db // Imports is a list of the package imports every generated source file has. var Imports = []string{ "context", "database/sql", "fmt", "strings", "github.com/mattn/go-sqlite3", } incus-7.0.0/cmd/generate-database/db/lex.go000066400000000000000000000073451517523235500204630ustar00rootroot00000000000000package db import ( "fmt" "strings" "github.com/lxc/incus/v7/cmd/generate-database/lex" "github.com/lxc/incus/v7/shared/util" ) // Return the table name for the given database entity. func entityTable(entity string, override string) string { if override != "" { return override } entityParts := strings.Split(lex.SnakeCase(entity), "_") tableParts := make([]string, len(entityParts)) for i, part := range entityParts { if strings.HasSuffix(part, "ty") || strings.HasSuffix(part, "ly") { tableParts[i] = part } else { tableParts[i] = lex.Plural(part) } } return strings.Join(tableParts, "_") } // Return the name of the Filter struct for the given database entity. func entityFilter(entity string) string { return fmt.Sprintf("%sFilter", lex.PascalCase(entity)) } // Return the name of the global variable holding the registration code for // the given kind of statement aganst the given entity. func stmtCodeVar(entity string, kind string, filters ...string) string { prefix := lex.CamelCase(entity) name := fmt.Sprintf("%s%s", prefix, lex.PascalCase(kind)) if len(filters) > 0 { name += "By" name += strings.Join(filters, "And") } return name } // operation returns the kind of operation being performed, without filter fields. func operation(kind string) string { return strings.Split(kind, "-by-")[0] } // activeFilters returns the filters mentioned in the command name. func activeFilters(kind string) []string { startIndex := strings.Index(kind, "-by-") + len("-by-") return strings.Split(kind[startIndex:], "-and-") } // Return an expression evaluating if a filter should be used (based on active // criteria). func activeCriteria(filter []string, ignoredFilter []string) string { expr := "" for i, name := range filter { if i > 0 { expr += " && " } expr += fmt.Sprintf("filter.%s != nil", name) } for _, name := range ignoredFilter { if len(expr) > 0 { expr += " && " } expr += fmt.Sprintf("filter.%s == nil", name) } return expr } // Return the code for a "dest" function, to be passed as parameter to // selectObjects in order to scan a single row. func destFunc(slice string, entity string, importType string, fields []*Field) string { var builder strings.Builder writeLine := func(line string) { fmt.Fprintf(&builder, "%s\n", line) } writeLine(`func(scan func(dest ...any) error) error {`) varName := lex.Minuscule(string(entity[0])) writeLine(fmt.Sprintf("%s := %s{}", varName, importType)) checkErr := func() { writeLine("if err != nil {\nreturn err\n}") writeLine("") } unmarshal := func(declVarName string, field *Field) { unmarshalFunc := "unmarshal" if field.Config.Get("marshal") == "json" { unmarshalFunc = "unmarshalJSON" } writeLine(fmt.Sprintf("err = %s(%s, &%s.%s)", unmarshalFunc, declVarName, varName, field.Name)) checkErr() } args := make([]string, len(fields)) declVars := make(map[string]*Field, len(fields)) declVarNames := make([]string, 0, len(fields)) for i, field := range fields { var arg string if util.IsNeitherFalseNorEmpty(field.Config.Get("marshal")) { declVarName := fmt.Sprintf("%sStr", lex.Minuscule(field.Name)) declVarNames = append(declVarNames, declVarName) declVars[declVarName] = field arg = fmt.Sprintf("&%s", declVarName) } else { arg = fmt.Sprintf("&%s.%s", varName, field.Name) } args[i] = arg } for _, declVarName := range declVarNames { writeLine(fmt.Sprintf("var %s string", declVarName)) } writeLine(fmt.Sprintf("err := scan(%s)", strings.Join(args, ", "))) checkErr() for _, declVarName := range declVarNames { unmarshal(declVarName, declVars[declVarName]) } writeLine(fmt.Sprintf("%s = append(%s, %s)\n", slice, slice, varName)) writeLine("return nil") writeLine("}") return builder.String() } incus-7.0.0/cmd/generate-database/db/mapping.go000066400000000000000000000424741517523235500213300ustar00rootroot00000000000000package db import ( "fmt" "go/ast" "go/types" "net/url" "slices" "strings" "github.com/lxc/incus/v7/cmd/generate-database/lex" "github.com/lxc/incus/v7/shared/util" ) // Mapping holds information for mapping database tables to a Go structure. type Mapping struct { Local bool // Whether the entity is in the same package as the generated code. FilterLocal bool // Whether the entity is in the same package as the generated code. Package string // Package of the Go struct Name string // Name of the Go struct. Fields []*Field // Metadata about the Go struct. Filterable bool // Whether the Go struct has a Filter companion struct for filtering queries. Filters []*Field // Metadata about the Go struct used for filter fields. Type TableType // Type of table structure for this Go struct. } // TableType represents the logical type of the table defined by the Go struct. type TableType int // EntityTable represents the type for any entity that maps to a Go struct. var EntityTable = TableType(0) // ReferenceTable represents the type for for any entity that contains an // 'entity_id' field mapping to a parent entity. var ReferenceTable = TableType(1) // AssociationTable represents the type for an entity that associates two // other entities. var AssociationTable = TableType(2) // MapTable represents the type for a table storing key/value pairs. var MapTable = TableType(3) // NaturalKey returns the struct fields that can be used as natural key for // uniquely identifying a row in the underlying table (==. // // By convention the natural key field is the one called "Name", unless // specified otherwise with the `db:natural_key` tags. func (m *Mapping) NaturalKey() []*Field { key := []*Field{} for _, field := range m.Fields { if field.Config.Get("primary") != "" { key = append(key, field) } } if len(key) == 0 { // Default primary key. key = append(key, m.FieldByName("Name")) } return key } // Identifier returns the field that uniquely identifies this entity. func (m *Mapping) Identifier() *Field { var fallback *Field for _, field := range m.NaturalKey() { if field.Config.Get("primary") != "" { return field } if field.Name == "Name" || field.Name == "Fingerprint" { fallback = field } } return fallback } // TableName determines the table associated to the struct. // - Individual fields may bypass this with their own `sql=
.` tags. // - The override `table=` directive key is checked first. // - The struct name itself is used to approximate the table name if none of the above apply. func (m *Mapping) TableName(entity string, override string) string { table := entityTable(entity, override) if m.Type == ReferenceTable || m.Type == MapTable { table = "%s_" + table } return table } // ContainsFields checks that the mapping contains fields with the same type // and name of given ones. func (m *Mapping) ContainsFields(fields []*Field) bool { matches := map[*Field]bool{} for _, field := range m.Fields { for _, other := range fields { if field.Name == other.Name && field.Type.Name == other.Type.Name { matches[field] = true } } } return len(matches) == len(fields) } // FieldByName returns the field with the given name, if any. func (m *Mapping) FieldByName(name string) *Field { for _, field := range m.Fields { if field.Name == name { return field } } return nil } // ActiveFilters returns the active filter fields for the kind of method. func (m *Mapping) ActiveFilters(kind string) []*Field { names := activeFilters(kind) fields := []*Field{} for _, name := range names { field := m.FieldByName(name) if field != nil { fields = append(fields, field) } } return fields } // FieldColumnName returns the column name of the field with the given name, // prefixed with the entity's table name. func (m *Mapping) FieldColumnName(name string, table string) string { field := m.FieldByName(name) return fmt.Sprintf("%s.%s", table, field.Column()) } // FilterFieldByName returns the field with the given name if that field can be // used as query filter, an error otherwise. func (m *Mapping) FilterFieldByName(name string) (*Field, error) { for _, filter := range m.Filters { if name == filter.Name { if filter.Type.Code != TypeColumn { return nil, fmt.Errorf("Unknown filter %q not a column", name) } return filter, nil } } return nil, fmt.Errorf("Unknown filter %q", name) } // ColumnFields returns the fields that map directly to a database column, // either on this table or on a joined one. func (m *Mapping) ColumnFields(exclude ...string) []*Field { fields := []*Field{} for _, field := range m.Fields { if slices.Contains(exclude, field.Name) { continue } if field.Type.Code == TypeColumn { fields = append(fields, field) } } return fields } // ScalarFields returns the fields that map directly to a single database // column on another table that can be joined to this one. func (m *Mapping) ScalarFields() []*Field { fields := []*Field{} for _, field := range m.Fields { if field.Config.Get("join") != "" || field.Config.Get("leftjoin") != "" { fields = append(fields, field) } } return fields } // RefFields returns the fields that are one-to-many references to other // tables. func (m *Mapping) RefFields() []*Field { fields := []*Field{} for _, field := range m.Fields { if field.Type.Code == TypeSlice || field.Type.Code == TypeMap { fields = append(fields, field) } } return fields } // FieldArgs converts the given fields to function arguments, rendering their // name and type. func (m *Mapping) FieldArgs(fields []*Field, extra ...string) string { args := []string{} for _, field := range fields { name := lex.Minuscule(field.Name) if name == "type" { name = lex.Minuscule(m.Name) + field.Name } arg := fmt.Sprintf("%s %s", name, field.Type.Name) args = append(args, arg) } args = append(args, extra...) return strings.Join(args, ", ") } // FieldParams converts the given fields to function parameters, rendering their // name. func (m *Mapping) FieldParams(fields []*Field) string { args := make([]string, len(fields)) for i, field := range fields { name := lex.Minuscule(field.Name) if name == "type" { name = lex.Minuscule(m.Name) + field.Name } args[i] = name } return strings.Join(args, ", ") } // FieldParamsMarshal converts the given fields to function parameters, rendering their // name. If the field is configured to marshal input/output, the name will be `marshaled{name}`. func (m *Mapping) FieldParamsMarshal(fields []*Field) string { args := make([]string, len(fields)) for i, field := range fields { name := lex.Minuscule(field.Name) if name == "type" { name = lex.Minuscule(m.Name) + field.Name } if util.IsNeitherFalseNorEmpty(field.Config.Get("marshal")) { name = fmt.Sprintf("marshaled%s", field.Name) } args[i] = name } return strings.Join(args, ", ") } // ImportType returns the type of the entity for the mapping, prefixing the import package if necessary. func (m *Mapping) ImportType() string { name := lex.PascalCase(m.Name) if m.Local { return name } return m.Package + "." + lex.PascalCase(name) } // ImportFilterType returns the Filter type of the entity for the mapping, prefixing the import package if necessary. func (m *Mapping) ImportFilterType() string { name := lex.PascalCase(entityFilter(m.Name)) if m.FilterLocal { return name } return m.Package + "." + name } // Field holds all information about a field in a Go struct that is relevant // for database code generation. type Field struct { Name string Type Type Primary bool // Whether this field is part of the natural primary key. Config url.Values } // Stmt must be used only on a non-columnar field. It returns the name of // statement that should be used to fetch this field. A statement with that // name must have been generated for the entity at hand. func (f *Field) Stmt() string { switch f.Name { case "UsedBy": return "used_by" default: return "" } } // IsScalar returns true if the field is a scalar column value from a joined table. func (f *Field) IsScalar() bool { return f.joinConfig() != "" } // IsIndirect returns true if the field is a scalar column value from a joined // table that in turn requires another join. func (f *Field) IsIndirect() bool { return f.IsScalar() && f.Config.Get("via") != "" } // IsPrimary returns true if the field part of the natural key. func (f *Field) IsPrimary() bool { return f.Config.Get("primary") != "" || f.Name == "Name" } // Column returns the name of the database column the field maps to. The type // code of the field must be TypeColumn. func (f *Field) Column() string { if f.Type.Code != TypeColumn { panic("attempt to get column name of non-column field") } column := lex.SnakeCase(f.Name) join := f.joinConfig() if join != "" { column = fmt.Sprintf("%s AS %s", join, column) } return column } // SelectColumn returns a column name suitable for use with 'SELECT' statements. // - Applies a `coalesce()` function if the 'coalesce' tag is present. // - Returns the column in the form '. AS ' if the `join` tag is present. func (f *Field) SelectColumn(mapping *Mapping, primaryTable string) (string, error) { // ReferenceTable and MapTable require specific fields, so parse those instead of checking tags. if mapping.Type == ReferenceTable || mapping.Type == MapTable { table := primaryTable column := fmt.Sprintf("%s.%s", table, lex.SnakeCase(f.Name)) column = strings.ReplaceAll(column, "reference", "%s") return column, nil } tableName, columnName, err := f.SQLConfig() if err != nil { return "", err } if tableName == "" { tableName = primaryTable } if columnName == "" { columnName = lex.SnakeCase(f.Name) } var column string join := f.joinConfig() if join != "" { column = join } else { column = fmt.Sprintf("%s.%s", tableName, columnName) } coalesce, ok := f.Config["coalesce"] if ok { column = fmt.Sprintf("coalesce(%s, %s)", column, coalesce[0]) } if join != "" { column = fmt.Sprintf("%s AS %s", column, columnName) } return column, nil } // OrderBy returns a column name suitable for use with the 'ORDER BY' clause. func (f *Field) OrderBy(mapping *Mapping, primaryTable string) (string, error) { // ReferenceTable and MapTable require specific fields, so parse those instead of checking tags. if mapping.Type == ReferenceTable || mapping.Type == MapTable { table := primaryTable column := fmt.Sprintf("%s.%s", table, lex.SnakeCase(f.Name)) column = strings.ReplaceAll(column, "reference", "%s") return column, nil } if f.IsScalar() { tableName, _, err := f.ScalarTableColumn() if err != nil { return "", err } return tableName + ".id", nil } tableName, columnName, err := f.SQLConfig() if err != nil { return "", nil } if columnName == "" { columnName = lex.SnakeCase(f.Name) } if tableName == "" { tableName = primaryTable } if tableName != "" { return fmt.Sprintf("%s.%s", tableName, columnName), nil } return fmt.Sprintf("%s.%s", entityTable(mapping.Name, tableName), columnName), nil } // JoinClause returns an SQL 'JOIN' clause using the 'join' and 'joinon' tags, if present. func (f *Field) JoinClause(mapping *Mapping, table string) (string, error) { joinTemplate := "\n JOIN %s ON %s = %s.%s" if f.Config.Get("join") != "" && f.Config.Get("leftjoin") != "" { return "", fmt.Errorf("Cannot join and leftjoin at the same time for field %q of struct %q", f.Name, mapping.Name) } join := f.joinConfig() if f.Config.Get("leftjoin") != "" { joinTemplate = strings.ReplaceAll(joinTemplate, "JOIN", "LEFT JOIN") } joinTable, _, ok := strings.Cut(join, ".") if !ok { return "", fmt.Errorf("'join' tag for field %q of struct %q must be of form
.", f.Name, mapping.Name) } joinOn := f.Config.Get("joinon") if joinOn == "" { tableName, columnName, err := f.SQLConfig() if err != nil { return "", err } if tableName != "" && columnName != "" { joinOn = fmt.Sprintf("%s.%s", tableName, columnName) } else { joinOn = fmt.Sprintf("%s.%s_id", table, lex.Singular(joinTable)) } } _, _, ok = strings.Cut(joinOn, ".") if !ok { return "", fmt.Errorf("'joinon' tag of field %q of struct %q must be of form '
.'", f.Name, mapping.Name) } joinTo := "id" if f.Config.Get("jointo") != "" { joinTo = f.Config.Get("jointo") } return fmt.Sprintf(joinTemplate, joinTable, joinOn, joinTable, joinTo), nil } // InsertColumn returns a column name and parameter value suitable for an 'INSERT', 'UPDATE', or 'DELETE' statement. // - If a 'join' tag is present, the package will be searched for the corresponding 'jointableID' registered statement // to select the ID to insert into this table. // - If a 'joinon' tag is present, but this table is not among the conditions, then the join will be considered indirect, // and an empty string will be returned. func (f *Field) InsertColumn(mapping *Mapping, primaryTable string, defs map[*ast.Ident]types.Object, registeredSQLStmts map[string]string, allFields []*Field) (string, string, error) { var column string var value string var err error if f.IsScalar() { tableName, columnName, err := f.SQLConfig() if err != nil { return "", "", err } if tableName == "" { tableName = primaryTable } // If there is a 'joinon' tag present without this table in the condition, then assume there is no column for this field. joinOn := f.Config.Get("joinon") if joinOn != "" { before, after, ok := strings.Cut(joinOn, ".") if !ok { return "", "", fmt.Errorf("'joinon' tag of field %q of struct %q must be of form '
.'", f.Name, mapping.Name) } columnName = after if tableName != before { return "", "", nil } } table, _, ok := strings.Cut(f.joinConfig(), ".") if !ok { return "", "", fmt.Errorf("'join' tag of field %q of struct %q must be of form
.", f.Name, mapping.Name) } if columnName != "" { column = columnName } else { column = lex.Singular(table) + "_id" } varName := stmtCodeVar(lex.Singular(table), "ID") joinStmt, err := ParseStmt(varName, defs, registeredSQLStmts) if err != nil { return "", "", fmt.Errorf("Failed to find registered statement %q for field %q of struct %q: %w", varName, f.Name, mapping.Name, err) } // Keep track of the join config of other fields that have a corresponding table column. otherJoins := map[string]string{} for _, otherField := range allFields { if f.Name == otherField.Name { continue } joinCfg := otherField.joinConfig() table, _, ok := strings.Cut(joinCfg, ".") if !ok { continue } // If 'joinon' points to a different table, then there is no column on the table for this field. joinOn := otherField.Config.Get("joinon") if joinOn == "" || strings.HasPrefix(joinOn, tableName+".") { otherJoins[table] = joinCfg } } // If the field maps to a column with a foreign key to table A, but table A has a composite key with table B, // then if we already have a field mapping to table B, just reuse its ID. if strings.Contains(joinStmt, "JOIN ") && strings.Contains(joinStmt, "WHERE ") { wheres := strings.Split(joinStmt, "WHERE ") for table, joinCfg := range otherJoins { wheres[1] = strings.ReplaceAll(wheres[1], joinCfg+" = ?", table+".id = "+lex.Singular(table)+"_id") } joinStmt = strings.Join(wheres, "WHERE ") } value = fmt.Sprintf("(%s)", strings.ReplaceAll(strings.ReplaceAll(joinStmt, "`", ""), "\n", "")) value = strings.ReplaceAll(value, " ", " ") } else { column, err = f.SelectColumn(mapping, primaryTable) if err != nil { return "", "", err } // Strip the table name and coalesce function if present. _, column, _ = strings.Cut(column, ".") column, _, _ = strings.Cut(column, ",") if mapping.Type == ReferenceTable || mapping.Type == MapTable { column = strings.ReplaceAll(column, "reference", "%s") } value = "?" } return column, value, nil } func (f *Field) joinConfig() string { join := f.Config.Get("join") if join == "" { join = f.Config.Get("leftjoin") } return join } // SQLConfig returns the table and column specified by the 'sql' config key, if present. func (f *Field) SQLConfig() (string, string, error) { where := f.Config.Get("sql") if where == "" { return "", "", nil } table, column, ok := strings.Cut(where, ".") if !ok { return "", "", fmt.Errorf("'sql' config for field %q should be of the form
.", f.Name) } return table, column, nil } // ScalarTableColumn gets the table and column from the join configuration. func (f *Field) ScalarTableColumn() (string, string, error) { join := f.joinConfig() if join == "" { return "", "", fmt.Errorf("Missing join config for field %q", f.Name) } joinFields := strings.Split(join, ".") if len(joinFields) != 2 { return "", "", fmt.Errorf("Join config must be of the format
. for field %q", f.Name) } return joinFields[0], joinFields[1], nil } // FieldNames returns the names of the given fields. func FieldNames(fields []*Field) []string { names := []string{} for _, f := range fields { names = append(names, f.Name) } return names } // Type holds all information about a field in a field type that is relevant // for database code generation. type Type struct { Name string Code int } // Possible type code. const ( TypeColumn = iota TypeSlice TypeMap ) incus-7.0.0/cmd/generate-database/db/method.go000066400000000000000000001730051517523235500211500ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "fmt" "go/types" "strings" "golang.org/x/tools/go/packages" "github.com/lxc/incus/v7/cmd/generate-database/file" "github.com/lxc/incus/v7/cmd/generate-database/lex" "github.com/lxc/incus/v7/shared/util" ) // Method generates a code snippet for a particular database query method. type Method struct { entity string // Name of the database entity kind string // Kind of statement to generate ref string // ref is the current reference method for the method kind config map[string]string // Configuration parameters localPath string pkgs []*types.Package // Package to perform for struct declaration lookup registeredSQLStmts map[string]string // Lookup for SQL statements registered during this execution, which are therefore not included in the parsed package information } // NewMethod returiiin a new method code snippet for executing a certain mapping. func NewMethod(localPath string, parsedPkgs []*packages.Package, entity, kind string, config map[string]string, registeredSQLStmts map[string]string) (*Method, error) { pkgTypes, err := parsePkgDecls(entity, kind, parsedPkgs) if err != nil { return nil, err } method := &Method{ entity: entity, kind: kind, config: config, localPath: localPath, pkgs: pkgTypes, registeredSQLStmts: registeredSQLStmts, } return method, nil } // Generate the desired method. func (m *Method) Generate(buf *file.Buffer) error { mapping, err := Parse(m.localPath, m.pkgs, lex.PascalCase(m.entity), m.kind) if err != nil { return fmt.Errorf("Unable to parse go struct %q: %w", lex.PascalCase(m.entity), err) } if mapping.Type != EntityTable { switch operation(m.kind) { case "GetMany": return m.getMany(buf) case "Create": return m.create(buf, false) case "Update": return m.update(buf) case "DeleteMany": return m.delete(buf, false) default: return fmt.Errorf("Unknown method kind '%s'", m.kind) } } switch operation(m.kind) { case "GetMany": return m.getMany(buf) case "GetNames": return m.getNames(buf) case "GetOne": return m.getOne(buf) case "ID": return m.id(buf) case "Exists": return m.exists(buf) case "Create": return m.create(buf, false) case "CreateOrReplace": return m.create(buf, true) case "Rename": return m.rename(buf) case "Update": return m.update(buf) case "DeleteOne": return m.delete(buf, true) case "DeleteMany": return m.delete(buf, false) default: return fmt.Errorf("Unknown method kind '%s'", m.kind) } } // GenerateSignature generates an interface signature for the method. func (m *Method) GenerateSignature(buf *file.Buffer) error { buf.N() buf.L("// %sGenerated is an interface of generated methods for %s.", lex.PascalCase(m.entity), lex.PascalCase(m.entity)) buf.L("type %sGenerated interface {", lex.PascalCase(m.entity)) defer m.end(buf) if m.config["references"] != "" { refFields := strings.Split(m.config["references"], ",") for _, fieldName := range refFields { m.ref = fieldName err := m.signature(buf, true) if err != nil { return err } m.ref = "" buf.N() } } return m.signature(buf, true) } func (m *Method) getNames(buf *file.Buffer) error { mapping, err := Parse(m.localPath, m.pkgs, lex.PascalCase(m.entity), m.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } // Go type name the objects to return (e.g. api.Foo). structField := mapping.NaturalKey()[0] err = m.signature(buf, false) if err != nil { return err } defer m.end(buf) buf.L("var err error") buf.N() buf.L("// Result slice.") buf.L("names := make(%s, 0)", lex.Slice(structField.Type.Name)) buf.N() filters, ignoredFilters := FiltersFromStmt(m.pkgs, "names", m.entity, mapping.Filters, m.registeredSQLStmts) buf.N() buf.L("// Pick the prepared statement and arguments to use based on active criteria.") buf.L("var sqlStmt *sql.Stmt") buf.L("args := []any{}") buf.L("queryParts := [2]string{}") buf.N() buf.L("if len(filters) == 0 {") buf.L("sqlStmt, err = Stmt(db, %s)", stmtCodeVar(m.entity, "names")) m.ifErrNotNil(buf, false, "nil", fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, "names"))) buf.L("}") buf.N() if len(filters) > 0 { buf.L("for i, filter := range filters {") } else { buf.L("for _, filter := range filters {") } for i, filter := range filters { branch := "if" if i > 0 { branch = "} else if" } buf.L("%s %s {", branch, activeCriteria(filter, ignoredFilters[i])) var args string for _, name := range filter { for _, field := range mapping.Fields { if name == field.Name && util.IsNeitherFalseNorEmpty(field.Config.Get("marshal")) { marshalFunc := "marshal" if strings.ToLower(field.Config.Get("marshal")) == "json" { marshalFunc = "marshalJSON" } buf.L("marshaledFilter%s, err := %s(filter.%s)", name, marshalFunc, name) m.ifErrNotNil(buf, true, "nil", "err") args += fmt.Sprintf("marshaledFilter%s,", name) } else if name == field.Name { args += fmt.Sprintf("filter.%s,", name) } } } buf.L("args = append(args, []any{%s}...)", args) buf.L("if len(filters) == 1 {") buf.L("sqlStmt, err = Stmt(db, %s)", stmtCodeVar(m.entity, "names", filter...)) m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, "names", filter...))) buf.L("break") buf.L("}") buf.N() buf.L("query, err := StmtString(%s)", stmtCodeVar(m.entity, "names", filter...)) m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, "names"))) buf.L("parts := strings.SplitN(query, \"ORDER BY\", 2)") buf.L("if i == 0 {") buf.L("copy(queryParts[:], parts)") buf.L("continue") buf.L("}") buf.N() buf.L("_, where, _ := strings.Cut(parts[0], \"WHERE\")") buf.L("queryParts[0] += \"OR\" + where") } branch := "if" if len(filters) > 0 { branch = "} else if" } buf.L("%s %s {", branch, activeCriteria([]string{}, FieldNames(mapping.Filters))) buf.L("return nil, fmt.Errorf(\"Cannot filter on empty %s\")", entityFilter(mapping.Name)) buf.L("} else {") buf.L("return nil, errors.New(\"No statement exists for the given Filter\")") buf.L("}") buf.L("}") buf.N() buf.L("// Select.") buf.L("var rows *sql.Rows") buf.L("if sqlStmt != nil {") buf.L("rows, err = sqlStmt.QueryContext(ctx, args...)") buf.L("} else {") buf.L("queryStr := strings.Join(queryParts[:], \"ORDER BY\")") buf.L("rows, err = db.QueryContext(ctx, queryStr, args...)") buf.L("}") buf.N() m.ifErrNotNil(buf, true, "nil", "err") buf.L("defer func() { _ = rows.Close() }()") buf.L("for rows.Next() {") buf.L("var identifier %s", structField.Type.Name) buf.L("err := rows.Scan(&identifier)") m.ifErrNotNil(buf, true, "nil", "err") buf.L("names = append(names, identifier)") buf.L("}") buf.N() buf.L("err = rows.Err()") m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to fetch from \"%s\" table: %%w", err)`, entityTable(m.entity, m.config["table"]))) buf.L("return names, nil") return nil } func (m *Method) getMany(buf *file.Buffer) error { mapping, err := Parse(m.localPath, m.pkgs, lex.PascalCase(m.entity), m.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } err = m.getManyTemplateFuncs(buf, mapping) if err != nil { return err } if m.config["references"] != "" { parentTable := mapping.TableName(m.entity, m.config["table"]) refFields := strings.Split(m.config["references"], ",") refs := make([]*Mapping, len(refFields)) for i, fieldName := range refFields { refMapping, err := Parse(m.localPath, m.pkgs, fieldName, m.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } refs[len(refs)-1-i] = refMapping } defer func() { for _, refMapping := range refs { err = m.getRefs(buf, parentTable, refMapping) if err != nil { return } } }() } // Go type name the objects to return (e.g. api.Foo). typ := mapping.ImportType() err = m.signature(buf, false) if err != nil { return err } defer m.end(buf) buf.L("var err error") buf.N() buf.L("// Result slice.") buf.L("objects := make(%s, 0)", lex.Slice(typ)) buf.N() if mapping.Type == ReferenceTable || mapping.Type == MapTable { stmtVar := stmtCodeVar(m.entity, "objects") stmtLocal := stmtVar + "Local" buf.L("%s := strings.ReplaceAll(%s, \"%%s_id\", fmt.Sprintf(\"%%s_id\", parentColumnPrefix))", stmtLocal, stmtVar) buf.L("fillParent := make([]any, strings.Count(%s, \"%%s\"))", stmtLocal) buf.L("for i := range fillParent {") buf.L("fillParent[i] = parentTablePrefix") buf.L("}") buf.N() buf.L("queryStr := fmt.Sprintf(%s, fillParent...)", stmtLocal) buf.L("queryParts := strings.SplitN(queryStr, \"ORDER BY\", 2)") buf.L("args := []any{}") buf.N() buf.L("for i, filter := range filters {") buf.L("var cond string") buf.L("if i == 0 {") buf.L("cond = \" WHERE ( %%s )\"") buf.L("} else {") buf.L("cond = \" OR ( %%s )\"") buf.L("}") buf.N() buf.L("entries := []string{}") for _, filter := range mapping.Filters { // Skip over filter fields that are themselves filters for a referenced table. found := false for _, refField := range mapping.RefFields() { if filter.Type.Name == entityFilter(refField.Name) { found = true break } } if found { continue } buf.L("if filter.%s != nil {", filter.Name) buf.L("entries = append(entries, \"%s = ?\")", lex.SnakeCase(filter.Name)) buf.L("args = append(args, filter.%s)", filter.Name) buf.L("}") buf.N() } buf.L("if len(entries) == 0 {") buf.L("return nil, fmt.Errorf(\"Cannot filter on empty %s\")", entityFilter(mapping.Name)) buf.L("}") buf.N() buf.L("queryParts[0] += fmt.Sprintf(cond, strings.Join(entries, \" AND \"))") buf.L("}") buf.N() buf.L("queryStr = strings.Join(queryParts, \" ORDER BY\")") } else if mapping.Type == AssociationTable { filter := m.config["struct"] + "ID" buf.L("sqlStmt, err := Stmt(db, %s)", stmtCodeVar(m.entity, "objects", filter)) m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, "objects", filter))) buf.L("args := []any{%sID}", lex.Minuscule(m.config["struct"])) } else { filters, ignoredFilters := FiltersFromStmt(m.pkgs, "objects", m.entity, mapping.Filters, m.registeredSQLStmts) buf.N() buf.L("// Pick the prepared statement and arguments to use based on active criteria.") buf.L("var sqlStmt *sql.Stmt") buf.L("args := []any{}") buf.L("queryParts := [2]string{}") buf.N() buf.L("if len(filters) == 0 {") buf.L("sqlStmt, err = Stmt(db, %s)", stmtCodeVar(m.entity, "objects")) m.ifErrNotNil(buf, false, "nil", fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, "objects"))) buf.L("}") buf.N() buf.L("for i, filter := range filters {") for i, filter := range filters { branch := "if" if i > 0 { branch = "} else if" } buf.L("%s %s {", branch, activeCriteria(filter, ignoredFilters[i])) var args string for _, name := range filter { for _, field := range mapping.Fields { if name == field.Name && util.IsNeitherFalseNorEmpty(field.Config.Get("marshal")) { marshalFunc := "marshal" if strings.ToLower(field.Config.Get("marshal")) == "json" { marshalFunc = "marshalJSON" } buf.L("marshaledFilter%s, err := %s(filter.%s)", name, marshalFunc, name) m.ifErrNotNil(buf, true, "nil", "err") args += fmt.Sprintf("marshaledFilter%s,", name) } else if name == field.Name { args += fmt.Sprintf("filter.%s,", name) } } } buf.L("args = append(args, []any{%s}...)", args) buf.L("if len(filters) == 1 {") buf.L("sqlStmt, err = Stmt(db, %s)", stmtCodeVar(m.entity, "objects", filter...)) m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, "objects", filter...))) buf.L("break") buf.L("}") buf.N() buf.L("query, err := StmtString(%s)", stmtCodeVar(m.entity, "objects", filter...)) m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, "objects"))) buf.L("parts := strings.SplitN(query, \"ORDER BY\", 2)") buf.L("if i == 0 {") buf.L("copy(queryParts[:], parts)") buf.L("continue") buf.L("}") buf.N() buf.L("_, where, _ := strings.Cut(parts[0], \"WHERE\")") buf.L("queryParts[0] += \"OR\" + where") } branch := "if" if len(filters) > 0 { branch = "} else if" } buf.L("%s %s {", branch, activeCriteria([]string{}, FieldNames(mapping.Filters))) buf.L("return nil, fmt.Errorf(\"Cannot filter on empty %s\")", entityFilter(mapping.Name)) buf.L("} else {") buf.L("return nil, errors.New(\"No statement exists for the given Filter\")") buf.L("}") buf.L("}") buf.N() } switch mapping.Type { case EntityTable: buf.L("// Select.") buf.L("if sqlStmt != nil {") buf.L("objects, err = get%s(ctx, sqlStmt, args...)", lex.Plural(mapping.Name)) buf.L("} else {") buf.L("queryStr := strings.Join(queryParts[:], \"ORDER BY\")") buf.L("objects, err = get%sRaw(ctx, db, queryStr, args...)", lex.Plural(mapping.Name)) buf.L("}") buf.N() m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to fetch from \"%s\" table: %%w", err)`, entityTable(m.entity, m.config["table"]))) case ReferenceTable, MapTable: buf.L("// Select.") buf.L("objects, err = get%sRaw(ctx, db, queryStr, parentTablePrefix, args...)", lex.Plural(mapping.Name)) m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to fetch from \"%%s_%s\" table: %%w", parentTablePrefix, err)`, entityTable(m.entity, m.config["table"]))) default: buf.N() buf.L("// Select.") buf.L("objects, err = get%s(ctx, sqlStmt, args...)", lex.Plural(mapping.Name)) m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to fetch from \"%s\" table: %%w", err)`, entityTable(m.entity, m.config["table"]))) } for _, field := range mapping.RefFields() { refStruct := lex.Singular(field.Name) refVar := lex.Minuscule(refStruct) refSlice := lex.Plural(refVar) refMapping, err := Parse(m.localPath, m.pkgs, refStruct, "") if err != nil { return fmt.Errorf("Could not find definition for reference struct %q: %w", refStruct, err) } switch refMapping.Type { case EntityTable: assocStruct := mapping.Name + field.Name buf.L("%s, err := Get%s()", lex.Minuscule(assocStruct), assocStruct) m.ifErrNotNil(buf, true, "nil", "err") buf.L("for i := range objects {") buf.L("objects[i].%s = make([]string, 0)", field.Name) buf.L("refIDs, ok := %s[objects[i].ID]", lex.Minuscule(assocStruct)) buf.L("if ok {") buf.L("for _, refID := range refIDs {") buf.L("%sURIs, err := Get%sURIs(%sFilter{ID: &refID})", refVar, refStruct, refStruct) m.ifErrNotNil(buf, true, "nil", "err") if field.Config.Get("uri") == "" { uriName := strings.ReplaceAll(lex.SnakeCase(refSlice), "_", "-") buf.L("uris, err := urlsToResourceNames(\"/%s\", %sURIs...)", uriName, refVar) m.ifErrNotNil(buf, true, "nil", "err") buf.L("%sURIs = uris", refVar) } buf.L("objects[i].%s = append(objects[i].%s, %sURIs...)", field.Name, field.Name, refVar) buf.L("}") buf.L("}") buf.L("}") case ReferenceTable: buf.L("%sFilters := []%s{}", refVar, entityFilter(refStruct)) buf.L("for _, f := range filters {") buf.L("filter := f.%s", refStruct) buf.L("if filter != nil {") buf.L("if %s {", activeCriteria(nil, FieldNames(refMapping.Filters))) buf.L("return nil, fmt.Errorf(\"Cannot filter on empty %s\")", entityFilter(refMapping.Name)) buf.L("}") buf.N() buf.L("%sFilters = append(%sFilters, *filter)", refVar, refVar) buf.L("}") buf.L("}") buf.N() if mapping.Type == ReferenceTable { // A reference table should let its child reference know about its parent. buf.L("%s, err := Get%s(ctx, db, parentTablePrefix+\"_%s\", parent_columnPrefix+\"_%s\", %sFilters...)", refSlice, lex.Plural(refStruct), lex.Plural(m.entity), m.entity, refVar) m.ifErrNotNil(buf, true, "nil", "err") } else { buf.L("%s, err := Get%s(ctx, db, \"%s\", %sFilters...)", refSlice, lex.Plural(refStruct), m.entity, refVar) m.ifErrNotNil(buf, true, "nil", "err") } buf.L("for i := range objects {") switch field.Type.Code { case TypeSlice: buf.L("objects[i].%s = %s[objects[i].ID]", lex.Plural(refStruct), refSlice) case TypeMap: buf.L("objects[i].%s = map[string]%s{}", lex.Plural(refStruct), refStruct) buf.L("for _, obj := range %s[objects[i].ID] {", refSlice) buf.L("_, ok := objects[i].%s[obj.%s]", lex.Plural(refStruct), refMapping.NaturalKey()[0].Name) buf.L("if !ok {") buf.L("objects[i].%s[obj.%s] = obj", lex.Plural(refStruct), refMapping.NaturalKey()[0].Name) buf.L("} else {") buf.L("return nil, fmt.Errorf(\"Found duplicate %s with name %%q\", obj.%s)", refStruct, refMapping.NaturalKey()[0].Name) buf.L("}") buf.L("}") } buf.L("}") case MapTable: buf.L("%sFilters := []%s{}", refVar, entityFilter(refStruct)) buf.L("for _, f := range filters {") buf.L("filter := f.%s", refStruct) buf.L("if filter != nil {") buf.L("if %s {", activeCriteria(nil, FieldNames(refMapping.Filters))) buf.L("return nil, fmt.Errorf(\"Cannot filter on empty %s\")", entityFilter(refMapping.Name)) buf.L("}") buf.N() buf.L("%sFilters = append(%sFilters, *filter)", refVar, refVar) buf.L("}") buf.L("}") buf.N() if mapping.Type == ReferenceTable { // A reference table should let its child reference know about its parent. buf.L("%s, err := Get%s(ctx, db, parentTablePrefix+\"_%s\", parentColumnPrefix+\"_%s\", %sFilters...)", refSlice, lex.Plural(refStruct), lex.Plural(m.entity), m.entity, refVar) m.ifErrNotNil(buf, true, "nil", "err") } else { buf.L("%s, err := Get%s(ctx, db, \"%s\", %sFilters...)", refSlice, lex.Plural(refStruct), m.entity, refVar) m.ifErrNotNil(buf, true, "nil", "err") } buf.L("for i := range objects {") buf.L("_, ok := %s[objects[i].ID]", refSlice) buf.L("if !ok {") buf.L("objects[i].%s = map[string]string{}", refStruct) buf.L("} else {") buf.L("objects[i].%s = %s[objects[i].ID]", lex.Plural(refStruct), refSlice) buf.L("}") buf.L("}") } buf.N() } switch mapping.Type { case AssociationTable: ref := strings.ReplaceAll(mapping.Name, m.config["struct"], "") refMapping, err := Parse(m.localPath, m.pkgs, ref, "") if err != nil { return fmt.Errorf("Could not find definition for reference struct %q: %w", ref, err) } buf.L("result := make([]%s, len(objects))", refMapping.ImportType()) buf.L("for i, object := range objects {") buf.L("%s, err := Get%s(ctx, db, %s{ID: &object.%sID})", lex.Minuscule(ref), lex.Plural(ref), refMapping.ImportFilterType(), ref) m.ifErrNotNil(buf, true, "nil", "err") buf.L("result[i] = %s[0]", lex.Minuscule(ref)) buf.L("}") buf.N() buf.L("return result, nil") case ReferenceTable: buf.L("resultMap := map[int][]%s{}", mapping.ImportType()) buf.L("for _, object := range objects {") buf.L("_, ok := resultMap[object.ReferenceID]") buf.L("if !ok {") buf.L("resultMap[object.ReferenceID] = []%s{}", mapping.ImportType()) buf.L("}") buf.N() buf.L("resultMap[object.ReferenceID] = append(resultMap[object.ReferenceID], object)") buf.L("}") buf.N() buf.L("return resultMap, nil") case MapTable: buf.L("resultMap := map[int]map[string]string{}") buf.L("for _, object := range objects {") buf.L("_, ok := resultMap[object.ReferenceID]") buf.L("if !ok {") buf.L("resultMap[object.ReferenceID] = map[string]string{}") buf.L("}") buf.N() buf.L("resultMap[object.ReferenceID][object.Key] = object.Value") buf.L("}") buf.N() buf.L("return resultMap, nil") case EntityTable: buf.L("return objects, nil") } return nil } func (m *Method) getRefs(buf *file.Buffer, parentTable string, refMapping *Mapping) error { m.ref = refMapping.Name err := m.signature(buf, false) if err != nil { return err } defer m.end(buf) // reset m.ref in case m.signature is called again. m.ref = "" refStruct := refMapping.Name refVar := lex.Minuscule(refStruct) refList := lex.Plural(refVar) refParent := lex.CamelCase(m.entity) refParentList := refParent + lex.PascalCase(refList) switch refMapping.Type { case ReferenceTable: buf.L("%s, err := Get%s(ctx, db, \"%s\", \"%s\", filters...)", refParentList, lex.Plural(refStruct), parentTable, lex.SnakeCase(m.entity)) m.ifErrNotNil(buf, true, "nil", "err") buf.L("%s := map[string]%s{}", refList, refMapping.ImportType()) buf.L("for _, ref := range %s[%sID] {", refParentList, refParent) buf.L("_, ok := %s[ref.%s]", refList, refMapping.Identifier().Name) buf.L("if !ok {") buf.L("%s[ref.%s] = ref", refList, refMapping.Identifier().Name) buf.L("} else {") buf.L("return nil, fmt.Errorf(\"Found duplicate %s with name %%q\", ref.%s)", refStruct, refMapping.Identifier().Name) buf.L("}") buf.L("}") buf.N() case MapTable: buf.L("%s, err := Get%s(ctx, db, \"%s\", \"%s\", filters...)", refParentList, lex.Plural(refStruct), parentTable, lex.SnakeCase(m.entity)) m.ifErrNotNil(buf, true, "nil", "err") buf.L("%s, ok := %s[%sID]", refList, refParentList, refParent) buf.L("if !ok {") buf.L("%s = map[string]string{}", refList) buf.L("}") buf.N() } buf.L("return %s, nil", refList) return nil } func (m *Method) getOne(buf *file.Buffer) error { mapping, err := Parse(m.localPath, m.pkgs, lex.PascalCase(m.entity), m.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } nk := mapping.NaturalKey() err = m.signature(buf, false) if err != nil { return err } defer m.end(buf) buf.L("filter := %s{}", mapping.ImportFilterType()) for _, field := range nk { name := lex.Minuscule(field.Name) if name == "type" { name = lex.Minuscule(m.entity) + field.Name } buf.L("filter.%s = &%s", field.Name, name) } buf.N() buf.L("objects, err := Get%s(ctx, db, filter)", lex.Plural(lex.PascalCase(m.entity))) if mapping.Type == ReferenceTable || mapping.Type == MapTable { m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to fetch from \"%%s_%s\" table: %%w", parentTablePrefix, err)`, entityTable(m.entity, m.config["table"]))) } else { m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to fetch from \"%s\" table: %%w", err)`, entityTable(m.entity, m.config["table"]))) } buf.L("switch len(objects) {") buf.L("case 0:") buf.L(` return nil, ErrNotFound`) buf.L("case 1:") buf.L(" return &objects[0], nil") buf.L("default:") buf.L(` return nil, fmt.Errorf("More than one \"%s\" entry matches")`, entityTable(m.entity, m.config["table"])) buf.L("}") return nil } func (m *Method) id(buf *file.Buffer) error { // Support using a different structure or package to pass arguments to Create. entityCreate, ok := m.config["struct"] if !ok { entityCreate = lex.PascalCase(m.entity) } mapping, err := Parse(m.localPath, m.pkgs, entityCreate, m.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } nk := mapping.NaturalKey() err = m.signature(buf, false) if err != nil { return err } defer m.end(buf) buf.L("stmt, err := Stmt(db, %s)", stmtCodeVar(m.entity, "ID")) m.ifErrNotNil(buf, true, "-1", fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, "ID"))) for _, field := range nk { if util.IsNeitherFalseNorEmpty(field.Config.Get("marshal")) { marshalFunc := "marshal" if strings.ToLower(field.Config.Get("marshal")) == "json" { marshalFunc = "marshalJSON" } buf.L("marshaled%s, err := %s(%s)", field.Name, marshalFunc, lex.Minuscule(field.Name)) m.ifErrNotNil(buf, true, "-1", "err") } } buf.L("row := stmt.QueryRowContext(ctx, %s)", mapping.FieldParamsMarshal(nk)) buf.L("var id int64") buf.L("err = row.Scan(&id)") buf.L("if errors.Is(err, sql.ErrNoRows) {") buf.L(`return -1, ErrNotFound`) buf.L("}") buf.N() m.ifErrNotNil(buf, true, "-1", fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" ID: %%w", err)`, entityTable(m.entity, m.config["table"]))) buf.L("return id, nil") return nil } func (m *Method) exists(buf *file.Buffer) error { // Support using a different structure or package to pass arguments to Create. entityCreate, ok := m.config["struct"] if !ok { entityCreate = lex.PascalCase(m.entity) } mapping, err := Parse(m.localPath, m.pkgs, entityCreate, m.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } nk := mapping.NaturalKey() err = m.signature(buf, false) if err != nil { return err } defer m.end(buf) buf.L("stmt, err := Stmt(db, %s)", stmtCodeVar(m.entity, "ID")) m.ifErrNotNil(buf, true, "false", fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, "ID"))) for _, field := range nk { if util.IsNeitherFalseNorEmpty(field.Config.Get("marshal")) { marshalFunc := "marshal" if strings.ToLower(field.Config.Get("marshal")) == "json" { marshalFunc = "marshalJSON" } buf.L("marshaled%s, err := %s(%s)", field.Name, marshalFunc, lex.Minuscule(field.Name)) m.ifErrNotNil(buf, true, "false", "err") } } buf.L("row := stmt.QueryRowContext(ctx, %s)", mapping.FieldParamsMarshal(nk)) buf.L("var id int64") buf.L("err = row.Scan(&id)") buf.L("if errors.Is(err, sql.ErrNoRows) {") buf.L(` return false, nil`) buf.L("}") buf.N() m.ifErrNotNil(buf, true, "false", fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" ID: %%w", err)`, entityTable(m.entity, m.config["table"]))) buf.L("return true, nil") return nil } func (m *Method) create(buf *file.Buffer, replace bool) error { mapping, err := Parse(m.localPath, m.pkgs, lex.PascalCase(m.entity), m.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } if m.config["references"] != "" { parentTable := mapping.TableName(m.entity, m.config["table"]) refFields := strings.Split(m.config["references"], ",") refs := make([]*Mapping, len(refFields)) for i, fieldName := range refFields { refMapping, err := Parse(m.localPath, m.pkgs, fieldName, m.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } refs[len(refs)-1-i] = refMapping } defer func() { for _, refMapping := range refs { err = m.createRefs(buf, parentTable, refMapping) if err != nil { return } } }() } err = m.signature(buf, false) if err != nil { return err } defer m.end(buf) if mapping.Type == MapTable { buf.L("// An empty value means we are unsetting this key, so just return.") buf.L("if object.Value == \"\" {") buf.L("return nil") buf.L("}") buf.N() } if mapping.Type == ReferenceTable || mapping.Type == MapTable { stmtVar := stmtCodeVar(m.entity, "create") stmtLocal := stmtVar + "Local" buf.L("%s := strings.ReplaceAll(%s, \"%%s_id\", fmt.Sprintf(\"%%s_id\", parentColumnPrefix))", stmtLocal, stmtVar) buf.L("fillParent := make([]any, strings.Count(%s, \"%%s\"))", stmtLocal) buf.L("for i := range fillParent {") buf.L("fillParent[i] = parentTablePrefix") buf.L("}") buf.N() buf.L("queryStr := fmt.Sprintf(%s, fillParent...)", stmtLocal) createParams := "" columnFields := mapping.ColumnFields("ID") if mapping.Type == ReferenceTable { buf.L("for _, object := range objects {") } for i, field := range columnFields { createParams += fmt.Sprintf("object.%s", field.Name) if i < len(columnFields) { createParams += ", " } } refFields := mapping.RefFields() if len(refFields) == 0 { buf.L("_, err := db.ExecContext(ctx, queryStr, %s)", createParams) m.ifErrNotNil(buf, true, fmt.Sprintf(`fmt.Errorf("Insert failed for \"%%s_%s\" table: %%w", parentTablePrefix, err)`, lex.Plural(m.entity))) } else { buf.L("result, err := db.ExecContext(ctx, queryStr, %s)", createParams) m.ifErrNotNil(buf, true, fmt.Sprintf(`fmt.Errorf("Insert failed for \"%%s_%s\" table: %%w", parentTablePrefix, err)`, lex.Plural(m.entity))) buf.L("id, err := result.LastInsertId()") m.ifErrNotNil(buf, true, "fmt.Errorf(\"Failed to fetch ID: %w\", err)") } } else { nk := mapping.NaturalKey() nkParams := make([]string, len(nk)) for i, field := range nk { nkParams[i] = fmt.Sprintf("object.%s", field.Name) } kind := "create" if mapping.Type != AssociationTable && replace { kind = "create_or_replace" } if mapping.Type == AssociationTable { buf.L("for _, object := range objects {") } fields := mapping.ColumnFields("ID") buf.L("args := make([]any, %d)", len(fields)) buf.N() buf.L("// Populate the statement arguments. ") for i, field := range fields { if field.Config.Has("create_timestamp") || field.Config.Has("update_timestamp") { buf.L("args[%d] = time.Now().UTC().Format(time.RFC3339)", i) continue } if util.IsNeitherFalseNorEmpty(field.Config.Get("marshal")) { marshalFunc := "marshal" if strings.ToLower(field.Config.Get("marshal")) == "json" { marshalFunc = "marshalJSON" } buf.L("marshaled%s, err := %s(object.%s)", field.Name, marshalFunc, field.Name) m.ifErrNotNil(buf, true, "-1", "err") buf.L("args[%d] = marshaled%s", i, field.Name) } else { buf.L("args[%d] = object.%s", i, field.Name) } } buf.N() buf.L("// Prepared statement to use. ") buf.L("stmt, err := Stmt(db, %s)", stmtCodeVar(m.entity, kind)) if mapping.Type == AssociationTable { m.ifErrNotNil(buf, true, fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, kind))) buf.L(`// Execute the statement.`) buf.L(`_, err = stmt.Exec(args...)`) buf.L(`if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") {`) buf.L(` return ErrConflict`) buf.L(`}`) buf.N() m.ifErrNotNil(buf, true, fmt.Sprintf(`fmt.Errorf("Failed to create \"%s\" entry: %%w", err)`, entityTable(m.entity, m.config["table"]))) } else { m.ifErrNotNil(buf, true, "-1", fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, kind))) buf.L(`// Execute the statement.`) buf.L(`result, err := stmt.Exec(args...)`) buf.L(`if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") {`) buf.L(` return -1, ErrConflict`) buf.L(`}`) buf.N() m.ifErrNotNil(buf, true, "-1", fmt.Sprintf(`fmt.Errorf("Failed to create \"%s\" entry: %%w", err)`, entityTable(m.entity, m.config["table"]))) buf.L(`id, err := result.LastInsertId()`) m.ifErrNotNil(buf, true, "-1", fmt.Sprintf(`fmt.Errorf("Failed to fetch \"%s\" entry ID: %%w", err)`, entityTable(m.entity, m.config["table"]))) } } for _, field := range mapping.RefFields() { // TODO: Remove all references to UsedBy. if field.Name == "UsedBy" { continue } refStruct := lex.Singular(field.Name) refMapping, err := Parse(m.localPath, m.pkgs, lex.Singular(field.Name), "") if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } switch refMapping.Type { case EntityTable: assocStruct := mapping.Name + refStruct buf.L("// Update association table.") buf.L("object.ID = int(id)") buf.L("err = Update%s(ctx, db, object)", lex.Plural(assocStruct)) m.ifErrNotNil(buf, true, "-1", fmt.Sprintf("fmt.Errorf(\"Could not update association table: %%w\", err)")) continue case ReferenceTable: buf.L("for _, insert := range object.%s {", field.Name) buf.L("insert.ReferenceID = int(id)") case MapTable: buf.L("referenceID := int(id)") buf.L("for key, value := range object.%s {", field.Name) buf.L("insert := %s{", field.Name) for _, ref := range refMapping.ColumnFields("ID") { buf.L("%s: %s,", ref.Name, lex.Minuscule(ref.Name)) } buf.L("}") buf.N() } if mapping.Type != EntityTable { buf.L("err = Create%s(ctx, db, parentTablePrefix + \"_%s\", parentColumnPrefix + \"_%s\", insert)", refStruct, lex.Plural(m.entity), m.entity) m.ifErrNotNil(buf, false, fmt.Sprintf("fmt.Errorf(\"Insert %s failed for %s: %%w\", err)", field.Name, mapping.Name)) } else { buf.L("err = Create%s(ctx, db, \"%s\", insert)", refStruct, m.entity) m.ifErrNotNil(buf, false, "-1", fmt.Sprintf("fmt.Errorf(\"Insert %s failed for %s: %%w\", err)", field.Name, mapping.Name)) } buf.L("}") } switch mapping.Type { case ReferenceTable, AssociationTable: buf.L("}") buf.N() buf.L("return nil") case MapTable: buf.L("return nil") default: buf.L("return id, nil") } return nil } func (m *Method) createRefs(buf *file.Buffer, parentTable string, refMapping *Mapping) error { m.ref = refMapping.Name err := m.signature(buf, false) if err != nil { return err } defer m.end(buf) // reset m.ref in case m.signature is called again. m.ref = "" refStruct := refMapping.Name refVar := lex.Minuscule(refStruct) refParent := lex.CamelCase(m.entity) switch refMapping.Type { case ReferenceTable: buf.L("for key, %s := range %s {", refVar, lex.Plural(refVar)) buf.L("%s.ReferenceID = int(%sID)", refVar, refParent) buf.L("%s[key] = %s", lex.Plural(refVar), refVar) buf.L("}") buf.N() buf.L("err := Create%s(ctx, db, \"%s\", \"%s\", %s)", lex.Plural(refStruct), parentTable, lex.SnakeCase(m.entity), lex.Plural(refVar)) m.ifErrNotNil(buf, false, fmt.Sprintf("fmt.Errorf(\"Insert %s failed for %s: %%w\", err)", refStruct, lex.PascalCase(m.entity))) case MapTable: buf.L("referenceID := int(%sID)", refParent) buf.L("for key, value := range %s {", refVar) buf.L("insert := %s{", refStruct) for _, ref := range refMapping.ColumnFields("ID") { buf.L("%s: %s,", ref.Name, lex.Minuscule(ref.Name)) } buf.L("}") buf.N() buf.L("err := Create%s(ctx, db, \"%s\", \"%s\", insert)", refStruct, parentTable, lex.SnakeCase(m.entity)) m.ifErrNotNil(buf, true, fmt.Sprintf("fmt.Errorf(\"Insert %s failed for %s: %%w\", err)", refStruct, lex.PascalCase(m.entity))) buf.L("}") } buf.N() buf.L("return nil") return nil } func (m *Method) rename(buf *file.Buffer) error { mapping, err := Parse(m.localPath, m.pkgs, lex.PascalCase(m.entity), m.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } nk := mapping.NaturalKey() err = m.signature(buf, false) if err != nil { return err } defer m.end(buf) buf.L("stmt, err := Stmt(db, %s)", stmtCodeVar(m.entity, "rename")) m.ifErrNotNil(buf, true, fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, "rename"))) for _, field := range nk { if util.IsNeitherFalseNorEmpty(field.Config.Get("marshal")) { marshalFunc := "marshal" if strings.ToLower(field.Config.Get("marshal")) == "json" { marshalFunc = "marshalJSON" } buf.L("marshaled%s, err := %s(%s)", field.Name, marshalFunc, lex.Minuscule(field.Name)) m.ifErrNotNil(buf, true, "err") } } updatedAt := false for _, field := range mapping.Fields { if field.Config.Has("update_timestamp") { updatedAt = true break } } if updatedAt { buf.L("result, err := stmt.Exec(to, %s, %s)", "time.Now().UTC().Format(time.RFC3339)", mapping.FieldParamsMarshal(nk)) } else { buf.L("result, err := stmt.Exec(to, %s)", mapping.FieldParamsMarshal(nk)) } m.ifErrNotNil(buf, true, fmt.Sprintf("fmt.Errorf(\"Rename %s failed: %%w\", err)", mapping.Name)) buf.L("n, err := result.RowsAffected()") m.ifErrNotNil(buf, true, "fmt.Errorf(\"Fetch affected rows failed: %w\", err)") buf.L("if n != 1 {") buf.L(" return fmt.Errorf(\"Query affected %%d rows instead of 1\", n)") buf.L("}") buf.N() buf.L("return nil") return nil } func (m *Method) update(buf *file.Buffer) error { mapping, err := Parse(m.localPath, m.pkgs, lex.PascalCase(m.entity), m.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } // Support using a different structure or package to pass arguments to Create. entityUpdate, ok := m.config["struct"] if !ok { entityUpdate = mapping.Name } if m.config["references"] != "" { refFields := strings.Split(m.config["references"], ",") parentTable := mapping.TableName(m.entity, m.config["table"]) refs := make([]*Mapping, len(refFields)) for i, fieldName := range refFields { refMapping, err := Parse(m.localPath, m.pkgs, fieldName, m.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } refs[len(refs)-1-i] = refMapping } defer func() { for _, refMapping := range refs { err = m.updateRefs(buf, parentTable, refMapping) if err != nil { return } } }() } nk := mapping.NaturalKey() err = m.signature(buf, false) if err != nil { return err } defer m.end(buf) switch mapping.Type { case AssociationTable: ref := strings.ReplaceAll(mapping.Name, m.config["struct"], "") refMapping, err := Parse(m.localPath, m.pkgs, ref, "") if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } refSlice := lex.Minuscule(lex.Plural(mapping.Name)) buf.L("// Delete current entry.") buf.L("err := Delete%s%s(ctx, db, %sID)", m.config["struct"], lex.Plural(ref), lex.Minuscule(m.config["struct"])) m.ifErrNotNil(buf, true, "err") buf.L("// Get new entry IDs.") buf.L("%s := make([]%s, 0, len(%s%s))", refSlice, mapping.ImportType(), lex.Minuscule(ref), lex.Plural(refMapping.Identifier().Name)) buf.L("for _, entry := range %s%s {", lex.Minuscule(ref), lex.Plural(refMapping.Identifier().Name)) buf.L("refID, err := Get%sID(ctx, db, entry)", ref) m.ifErrNotNil(buf, true, "err") fields := fmt.Sprintf("%sID: %sID, %sID: int(refID)", m.config["struct"], lex.Minuscule(m.config["struct"]), ref) buf.L("%s = append(%s, %s{%s})", refSlice, refSlice, mapping.ImportType(), fields) buf.L("}") buf.N() buf.L("err = Create%s%s(ctx, db, %s)", m.config["struct"], lex.Plural(ref), refSlice) m.ifErrNotNil(buf, true, "err") case ReferenceTable: buf.L("// Delete current entry.") buf.L("err := Delete%s(ctx, db, parentTablePrefix, parentColumnPrefix, referenceID)", lex.PascalCase(lex.Plural(m.entity))) m.ifErrNotNil(buf, true, "err") buf.L("// Insert new entries.") buf.L("for key, object := range %s {", lex.Plural(m.entity)) buf.L("object.ReferenceID = referenceID") buf.L("%s[key] = object", lex.Plural(m.entity)) buf.L("}") buf.N() buf.L("err = Create%s(ctx, db, parentTablePrefix, parentColumnPrefix, %s)", lex.PascalCase(lex.Plural(m.entity)), lex.Plural(m.entity)) m.ifErrNotNil(buf, true, "err") case MapTable: buf.L("// Delete current entry.") buf.L("err := Delete%s(ctx, db, parentTablePrefix, parentColumnPrefix, referenceID)", lex.PascalCase(lex.Plural(m.entity))) m.ifErrNotNil(buf, true, "err") buf.L("// Insert new entries.") buf.L("for key, value := range config {") buf.L("object := %s{", mapping.Name) for _, field := range mapping.ColumnFields("ID") { buf.L("%s: %s,", field.Name, lex.Minuscule(field.Name)) } buf.L("}") buf.N() buf.L("err = Create%s(ctx, db, parentTablePrefix, parentColumnPrefix, object)", lex.PascalCase(m.entity)) m.ifErrNotNil(buf, false, "err") buf.L("}") buf.N() case EntityTable: updateMapping, err := Parse(m.localPath, m.pkgs, entityUpdate, m.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } buf.L("id, err := Get%sID(ctx, db, %s)", lex.PascalCase(m.entity), mapping.FieldParams(nk)) m.ifErrNotNil(buf, true, "err") buf.L("stmt, err := Stmt(db, %s)", stmtCodeVar(m.entity, "update")) m.ifErrNotNil(buf, true, fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, "update"))) fields := updateMapping.ColumnFields("ID") // This exclude the ID column, which is autogenerated. params := make([]string, len(fields)) for i, field := range fields { if field.Config.Has("update_timestamp") { params[i] = "time.Now().UTC().Format(time.RFC3339)" continue } if util.IsNeitherFalseNorEmpty(field.Config.Get("marshal")) { marshalFunc := "marshal" if strings.ToLower(field.Config.Get("marshal")) == "json" { marshalFunc = "marshalJSON" } buf.L("marshaled%s, err := %s(object.%s)", field.Name, marshalFunc, field.Name) m.ifErrNotNil(buf, true, "err") params[i] = fmt.Sprintf("marshaled%s", field.Name) } else { params[i] = fmt.Sprintf("object.%s", field.Name) } } buf.L("result, err := stmt.Exec(%s)", strings.Join(params, ", ")+", id") m.ifErrNotNil(buf, true, fmt.Sprintf(`fmt.Errorf("Update \"%s\" entry failed: %%w", err)`, entityTable(m.entity, m.config["table"]))) buf.L("n, err := result.RowsAffected()") m.ifErrNotNil(buf, true, "fmt.Errorf(\"Fetch affected rows: %w\", err)") buf.L("if n != 1 {") buf.L(" return fmt.Errorf(\"Query updated %%d rows instead of 1\", n)") buf.L("}") buf.N() for _, field := range mapping.RefFields() { // TODO: Eliminate UsedBy fields and move to dedicated slices for entities. if field.Name == "UsedBy" { continue } refStruct := lex.Singular(field.Name) refMapping, err := Parse(m.localPath, m.pkgs, lex.Singular(field.Name), "") if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } switch refMapping.Type { case EntityTable: assocStruct := mapping.Name + refStruct buf.L("// Update association table.") buf.L("object.ID = int(id)") buf.L("err = Update%s(ctx, db, object)", lex.Plural(assocStruct)) m.ifErrNotNil(buf, true, "fmt.Errorf(\"Could not update association table: %w\", err)") case ReferenceTable: buf.L("err = Update%s(ctx, db, \"%s\", int(id), object.%s)", lex.Singular(field.Name), m.entity, field.Name) m.ifErrNotNil(buf, true, fmt.Sprintf("fmt.Errorf(\"Replace %s for %s failed: %%w\", err)", field.Name, mapping.Name)) case MapTable: buf.L("err = Update%s(ctx, db, \"%s\", int(id), object.%s)", lex.Singular(field.Name), m.entity, field.Name) m.ifErrNotNil(buf, true, fmt.Sprintf("fmt.Errorf(\"Replace %s for %s failed: %%w\", err)", field.Name, mapping.Name)) buf.N() } } } buf.L("return nil") return nil } func (m *Method) updateRefs(buf *file.Buffer, parentTable string, refMapping *Mapping) error { m.ref = refMapping.Name err := m.signature(buf, false) if err != nil { return err } defer m.end(buf) // reset m.ref in case m.signature is called again. m.ref = "" refStruct := refMapping.Name refVar := lex.Minuscule(refStruct) refList := lex.Plural(refVar) refParent := lex.CamelCase(m.entity) buf.L("err := Update%s(ctx, db, \"%s\", \"%s\", int(%sID), %s)", lex.Plural(refStruct), parentTable, lex.SnakeCase(m.entity), refParent, refList) m.ifErrNotNil(buf, true, fmt.Sprintf("fmt.Errorf(\"Replace %s for %s failed: %%w\", err)", refStruct, lex.PascalCase(m.entity))) buf.L("return nil") return nil } func (m *Method) delete(buf *file.Buffer, deleteOne bool) error { mapping, err := Parse(m.localPath, m.pkgs, lex.PascalCase(m.entity), m.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } err = m.signature(buf, false) if err != nil { return err } defer m.end(buf) switch mapping.Type { case AssociationTable: buf.L("stmt, err := Stmt(db, %s)", stmtCodeVar(m.entity, "delete", m.config["struct"]+"ID")) m.ifErrNotNil(buf, true, fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, "delete", m.config["struct"]+"ID"))) buf.L("result, err := stmt.Exec(int(%sID))", lex.Minuscule(m.config["struct"])) m.ifErrNotNil(buf, true, fmt.Sprintf(`fmt.Errorf("Delete \"%s\" entry failed: %%w", err)`, entityTable(m.entity, m.config["table"]))) case ReferenceTable, MapTable: stmtVar := stmtCodeVar(m.entity, "delete") stmtLocal := stmtVar + "Local" buf.L("%s := strings.ReplaceAll(%s, \"%%s_id\", fmt.Sprintf(\"%%s_id\", parentColumnPrefix))", stmtLocal, stmtVar) buf.L("fillParent := make([]any, strings.Count(%s, \"%%s\"))", stmtLocal) buf.L("for i := range fillParent {") buf.L("fillParent[i] = parentTablePrefix") buf.L("}") buf.N() buf.L("queryStr := fmt.Sprintf(%s, fillParent...)", stmtLocal) buf.L("result, err := db.ExecContext(ctx, queryStr, referenceID)") m.ifErrNotNil(buf, true, fmt.Sprintf(`fmt.Errorf("Delete entry for \"%%s_%s\" failed: %%w", parentTablePrefix, err)`, m.entity)) default: activeFilters := mapping.ActiveFilters(m.kind) buf.L("stmt, err := Stmt(db, %s)", stmtCodeVar(m.entity, "delete", FieldNames(activeFilters)...)) for _, field := range activeFilters { if util.IsNeitherFalseNorEmpty(field.Config.Get("marshal")) { marshalFunc := "marshal" if strings.ToLower(field.Config.Get("marshal")) == "json" { marshalFunc = "marshalJSON" } buf.L("marshaled%s, err := %s(%s)", field.Name, marshalFunc, lex.Minuscule(field.Name)) m.ifErrNotNil(buf, true, "err") } } m.ifErrNotNil(buf, true, fmt.Sprintf(`fmt.Errorf("Failed to get \"%s\" prepared statement: %%w", err)`, stmtCodeVar(m.entity, "delete", FieldNames(activeFilters)...))) buf.L("result, err := stmt.Exec(%s)", mapping.FieldParamsMarshal(activeFilters)) m.ifErrNotNil(buf, true, fmt.Sprintf(`fmt.Errorf("Delete \"%s\": %%w", err)`, entityTable(m.entity, m.config["table"]))) } if deleteOne { buf.L("n, err := result.RowsAffected()") } else { buf.L("_, err = result.RowsAffected()") } m.ifErrNotNil(buf, true, "fmt.Errorf(\"Fetch affected rows: %w\", err)") if deleteOne { buf.L("if n == 0 {") buf.L(` return ErrNotFound`) buf.L("} else if n > 1 {") buf.L(" return fmt.Errorf(\"Query deleted %%d %s rows instead of 1\", n)", lex.PascalCase(m.entity)) buf.L("}") } buf.N() buf.L("return nil") return nil } // signature generates a method or interface signature with comments, arguments, and return values. func (m *Method) signature(buf *file.Buffer, isInterface bool) error { mapping, err := Parse(m.localPath, m.pkgs, lex.PascalCase(m.entity), m.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } comment := "" args := "ctx context.Context, db dbtx, " rets := "" switch mapping.Type { case AssociationTable: ref := strings.ReplaceAll(mapping.Name, m.config["struct"], "") refMapping, err := Parse(m.localPath, m.pkgs, ref, "") if err != nil { return fmt.Errorf("Failed to parse struct %q", ref) } switch operation(m.kind) { case "GetMany": comment = fmt.Sprintf("returns all available %s for the %s.", lex.Plural(ref), m.config["struct"]) args += fmt.Sprintf("%sID int", lex.Minuscule(m.config["struct"])) rets = fmt.Sprintf("(_ []%s, _err error)", refMapping.ImportType()) case "Create": comment = fmt.Sprintf("adds a new %s to the database.", m.entity) args += fmt.Sprintf("objects []%s", mapping.ImportType()) rets = "(_err error)" case "Update": comment = fmt.Sprintf("updates the %s matching the given key parameters.", m.entity) if len(refMapping.NaturalKey()) > 1 { return fmt.Errorf("Cannot generate update method for associative table: Reference table struct %q has more than one natural key", ref) } else if refMapping.Identifier() == nil { return fmt.Errorf("Cannot generate update method for associative table: Identifier for reference table struct %q must be `Name` or `Fingerprint`", ref) } args += fmt.Sprintf("%sID int, %s%s []%s", lex.Minuscule(m.config["struct"]), lex.Minuscule(ref), lex.Plural(refMapping.Identifier().Name), refMapping.Identifier().Type.Name) rets = "(_err error)" case "DeleteMany": comment = fmt.Sprintf("deletes the %s matching the given key parameters.", m.entity) args += fmt.Sprintf("%sID int", lex.Minuscule(m.config["struct"])) rets = "(_err error)" default: return fmt.Errorf("Unknown method kind '%s'", m.kind) } case ReferenceTable: switch operation(m.kind) { case "GetMany": comment = fmt.Sprintf("returns all available %s for the parent entity.", lex.Plural(m.entity)) args += fmt.Sprintf("parentTablePrefix string, parentColumnPrefix string, filters ...%s", mapping.ImportFilterType()) rets = fmt.Sprintf("(_ map[int][]%s, _err error)", mapping.ImportType()) case "Create": comment = fmt.Sprintf("adds a new %s to the database.", m.entity) args += fmt.Sprintf("parentTablePrefix string, parentColumnPrefix string, objects map[string]%s", mapping.ImportType()) rets = "(_err error)" case "Update": comment = fmt.Sprintf("updates the %s matching the given key parameters.", m.entity) args += fmt.Sprintf("parentTablePrefix string, parentColumnPrefix string, referenceID int, %s map[string]%s", lex.Plural(m.entity), mapping.ImportType()) rets = "(_err error)" case "DeleteMany": comment = fmt.Sprintf("deletes the %s matching the given key parameters.", m.entity) args += "parentTablePrefix string, parentColumnPrefix string, referenceID int" rets = "(_err error)" default: return fmt.Errorf("Unknown method kind '%s'", m.kind) } case MapTable: switch operation(m.kind) { case "GetMany": comment = fmt.Sprintf("returns all available %s.", lex.Plural(m.entity)) args += fmt.Sprintf("parentTablePrefix string, parentColumnPrefix string, filters ...%s", entityFilter(lex.PascalCase(m.entity))) rets = "(_ map[int]map[string]string, _err error)" case "Create": comment = fmt.Sprintf("adds a new %s to the database.", m.entity) args += fmt.Sprintf("parentTablePrefix string, parentColumnPrefix string, object %s", mapping.Name) rets = "(_err error)" case "Update": comment = fmt.Sprintf("updates the %s matching the given key parameters.", m.entity) args += "parentTablePrefix string, parentColumnPrefix string, referenceID int, config map[string]string" rets = "(_err error)" case "DeleteMany": comment = fmt.Sprintf("deletes the %s matching the given key parameters.", m.entity) args += "parentTablePrefix string, parentColumnPrefix string, referenceID int" rets = "(_err error)" default: return fmt.Errorf("Unknown method kind '%s'", m.kind) } case EntityTable: switch operation(m.kind) { case "URIs": comment = fmt.Sprintf("returns all available %s URIs.", m.entity) args += fmt.Sprintf("filter %s", mapping.ImportFilterType()) rets = "(_ []string, _err error)" case "GetMany": if m.ref == "" { comment = fmt.Sprintf("returns all available %s.", lex.Plural(m.entity)) args += fmt.Sprintf("filters ...%s", mapping.ImportFilterType()) rets = fmt.Sprintf("(_ %s, _err error)", lex.Slice(mapping.ImportType())) } else { refMapping, err := Parse(m.localPath, m.pkgs, m.ref, "") if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } comment = fmt.Sprintf("returns all available %s %s", mapping.Name, lex.Plural(m.ref)) args += fmt.Sprintf("%sID int, filters ...%s", lex.Minuscule(mapping.Name), refMapping.ImportFilterType()) var retType string switch refMapping.Type { case ReferenceTable: retType = fmt.Sprintf("map[%s]%s", refMapping.Identifier().Type.Name, refMapping.ImportType()) case MapTable: retType = "map[string]string" } rets = fmt.Sprintf("(_ %s, _err error)", retType) } case "GetNames": comment = fmt.Sprintf("returns the identifying field of %s.", m.entity) args += fmt.Sprintf("filters ...%s", mapping.ImportFilterType()) rets = fmt.Sprintf("(_ %s, _err error)", lex.Slice(mapping.NaturalKey()[0].Type.Name)) case "GetOne": comment = fmt.Sprintf("returns the %s with the given key.", m.entity) args += mapping.FieldArgs(mapping.NaturalKey()) rets = fmt.Sprintf("(_ %s, _err error)", lex.Star(mapping.ImportType())) case "ID": comment = fmt.Sprintf("return the ID of the %s with the given key.", m.entity) args += mapping.FieldArgs(mapping.NaturalKey()) rets = "(_ int64, _err error)" case "Exists": comment = fmt.Sprintf("checks if a %s with the given key exists.", m.entity) args += mapping.FieldArgs(mapping.NaturalKey()) rets = "(_ bool, _err error)" case "Create": structMapping := mapping if m.ref != "" { structMapping, err = Parse(m.localPath, m.pkgs, m.ref, "") if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } } else if m.config["struct"] != "" { structMapping, err = Parse(m.localPath, m.pkgs, m.config["struct"], "") if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } } if m.ref == "" { comment = fmt.Sprintf("adds a new %s to the database.", m.entity) args += fmt.Sprintf("object %s", structMapping.ImportType()) rets = "(_ int64, _err error)" } else { comment = fmt.Sprintf("adds new %s %s to the database.", m.entity, lex.Plural(m.ref)) rets = "(_err error)" switch structMapping.Type { case ReferenceTable: args += fmt.Sprintf("%sID int64, %s map[%s]%s", lex.CamelCase(m.entity), lex.Plural(lex.Minuscule(m.ref)), structMapping.Identifier().Type.Name, structMapping.ImportType()) case MapTable: args += fmt.Sprintf("%sID int64, %s map[string]string", lex.CamelCase(m.entity), lex.Minuscule(m.ref)) } } case "CreateOrReplace": structMapping := mapping if m.config["struct"] != "" { structMapping, err = Parse(m.localPath, m.pkgs, m.config["struct"], "") if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } } comment = fmt.Sprintf("adds a new %s to the database.", m.entity) args += fmt.Sprintf("object %s", structMapping.ImportType()) rets = "(_ int64, _err error)" case "Rename": comment = fmt.Sprintf("renames the %s matching the given key parameters.", m.entity) args += mapping.FieldArgs(mapping.NaturalKey(), "to string") rets = "(_err error)" case "Update": structMapping := mapping if m.ref != "" { structMapping, err = Parse(m.localPath, m.pkgs, m.ref, "") if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } } else if m.config["struct"] != "" { structMapping, err = Parse(m.localPath, m.pkgs, m.config["struct"], "") if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } } if m.ref == "" { comment = fmt.Sprintf("updates the %s matching the given key parameters.", m.entity) args += mapping.FieldArgs(mapping.NaturalKey(), fmt.Sprintf("object %s", structMapping.ImportType())) rets = "(_err error)" } else { comment = fmt.Sprintf("updates the %s %s matching the given key parameters.", m.entity, m.ref) rets = "(_err error)" switch structMapping.Type { case ReferenceTable: args += fmt.Sprintf("%sID int64, %s map[%s]%s", lex.CamelCase(m.entity), lex.Minuscule(lex.Plural(m.ref)), structMapping.Identifier().Type.Name, structMapping.ImportType()) case MapTable: args += fmt.Sprintf("%sID int64, %s map[string]string", lex.CamelCase(m.entity), lex.Minuscule(lex.Plural(m.ref))) } } case "DeleteOne": comment = fmt.Sprintf("deletes the %s matching the given key parameters.", m.entity) args += mapping.FieldArgs(mapping.ActiveFilters(m.kind)) rets = "(_err error)" case "DeleteMany": comment = fmt.Sprintf("deletes the %s matching the given key parameters.", m.entity) args += mapping.FieldArgs(mapping.ActiveFilters(m.kind)) rets = "(_err error)" default: return fmt.Errorf("Unknown method kind '%s'", m.kind) } } args, err = m.sqlTxCheck(mapping, args) if err != nil { return err } m.begin(buf, mapping, comment, args, rets, isInterface) if isInterface { return nil } return nil } func (m *Method) begin(buf *file.Buffer, mapping *Mapping, comment string, args string, rets string, isInterface bool) { name := "" entity := lex.PascalCase(m.entity) if mapping.Type == AssociationTable { parent := m.config["struct"] ref := strings.ReplaceAll(entity, parent, "") switch operation(m.kind) { case "GetMany": name = fmt.Sprintf("Get%s%s", parent, lex.Plural(ref)) case "Create": name = fmt.Sprintf("Create%s%s", parent, lex.Plural(ref)) case "Update": name = fmt.Sprintf("Update%s%s", parent, lex.Plural(ref)) case "DeleteMany": name = fmt.Sprintf("Delete%s%s", parent, lex.Plural(ref)) } } else { entity = entity + m.ref switch operation(m.kind) { case "URIs": name = fmt.Sprintf("Get%sURIs", entity) case "GetMany": name = fmt.Sprintf("Get%s", lex.Plural(entity)) case "GetNames": name = fmt.Sprintf("Get%sNames", entity) case "GetOne": name = fmt.Sprintf("Get%s", entity) case "ID": name = fmt.Sprintf("Get%sID", entity) case "Exists": name = fmt.Sprintf("%sExists", entity) case "Create": if mapping.Type == ReferenceTable || m.ref != "" { entity = lex.Plural(entity) } name = fmt.Sprintf("Create%s", entity) case "CreateOrReplace": if mapping.Type == ReferenceTable || m.ref != "" { entity = lex.Plural(entity) } name = fmt.Sprintf("CreateOrReplace%s", entity) case "Rename": name = fmt.Sprintf("Rename%s", entity) case "Update": if mapping.Type == ReferenceTable || m.ref != "" { entity = lex.Plural(entity) } name = fmt.Sprintf("Update%s", entity) case "DeleteOne": name = fmt.Sprintf("Delete%s", entity) case "DeleteMany": name = fmt.Sprintf("Delete%s", lex.Plural(entity)) default: name = fmt.Sprintf("%s%s", entity, m.kind) } } buf.L("// %s %s", name, comment) buf.L("// generator: %s %s", m.entity, m.kind) if isInterface { // Named return values are not needed for the interface definition. rets = strings.ReplaceAll(rets, "_err ", "") rets = strings.ReplaceAll(rets, "_ ", "") buf.L("%s(%s) %s", name, args, rets) } else { buf.L("func %s(%s) %s {", name, args, rets) buf.L("defer func() {") buf.L("_err = mapErr(_err, %q)", lex.Capital(m.entity)) buf.L("}()") buf.N() } } func (m *Method) sqlTxCheck(mapping *Mapping, args string) (string, error) { txCheck := false switch mapping.Type { case EntityTable: if m.kind == "Update" || m.kind == "ID" { txCheck = true } else if m.ref != "" { refMapping, err := Parse(m.localPath, m.pkgs, m.ref, "") if err != nil { return "", fmt.Errorf("Parse entity struct: %w", err) } if refMapping.Type != MapTable || m.kind == "GetMany" { txCheck = true } } case AssociationTable: txCheck = true case ReferenceTable: txCheck = true } if txCheck { args = strings.ReplaceAll(args, "dbtx", "tx") } return args, nil } func (m *Method) ifErrNotNil(buf *file.Buffer, newLine bool, rets ...string) { buf.L("if err != nil {") buf.L("return %s", strings.Join(rets, ", ")) buf.L("}") if newLine { buf.N() } } func (m *Method) end(buf *file.Buffer) { buf.L("}") } // getManyTemplateFuncs returns two functions that can be used to perform generic queries without validation, and return // a slice of objects matching the entity. One function will accept pre-registered statements, and the other will accept // raw queries. func (m *Method) getManyTemplateFuncs(buf *file.Buffer, mapping *Mapping) error { if mapping.Type == AssociationTable { if m.config["struct"] != "" && strings.HasSuffix(mapping.Name, m.config["struct"]) { return nil } } tableName := mapping.TableName(m.entity, m.config["table"]) // Create a function to get the column names to use with SELECT statements for the entity. buf.L("// %sColumns returns a string of column names to be used with a SELECT statement for the entity.", lex.Minuscule(mapping.Name)) buf.L("// Use this function when building statements to retrieve database entries matching the %s entity.", mapping.Name) buf.L("func %sColumns() string {", lex.Minuscule(mapping.Name)) columns := make([]string, len(mapping.Fields)) for i, field := range mapping.Fields { column, err := field.SelectColumn(mapping, tableName) if err != nil { return err } columns[i] = column } buf.L("return \"%s\"", strings.Join(columns, ", ")) buf.L("}") buf.N() // Create a function supporting prepared statements. buf.L("// get%s can be used to run handwritten sql.Stmts to return a slice of objects.", lex.Plural(mapping.Name)) if mapping.Type != ReferenceTable && mapping.Type != MapTable { buf.L("func get%s(ctx context.Context, stmt *sql.Stmt, args ...any) ([]%s, error) {", lex.Plural(mapping.Name), mapping.ImportType()) } else { buf.L("func get%s(ctx context.Context, stmt *sql.Stmt, parent string, args ...any) ([]%s, error) {", lex.Plural(mapping.Name), mapping.ImportType()) } buf.L("objects := make([]%s, 0)", mapping.ImportType()) buf.N() buf.L("dest := %s", destFunc("objects", lex.PascalCase(m.entity), mapping.ImportType(), mapping.ColumnFields())) buf.N() buf.L("err := selectObjects(ctx, stmt, dest, args...)") if mapping.Type != ReferenceTable && mapping.Type != MapTable { m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to fetch from \"%s\" table: %%w", err)`, tableName)) } else { m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to fetch from \"%s\" table: %%w", parent, err)`, tableName)) } buf.L(" return objects, nil") buf.L("}") buf.N() // Create a function supporting raw queries. buf.L("// get%sRaw can be used to run handwritten query strings to return a slice of objects.", lex.Plural(mapping.Name)) if mapping.Type != ReferenceTable && mapping.Type != MapTable { buf.L("func get%sRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]%s, error) {", lex.Plural(mapping.Name), mapping.ImportType()) } else { buf.L("func get%sRaw(ctx context.Context, db dbtx, sql string, parent string, args ...any) ([]%s, error) {", lex.Plural(mapping.Name), mapping.ImportType()) } buf.L("objects := make([]%s, 0)", mapping.ImportType()) buf.N() buf.L("dest := %s", destFunc("objects", lex.PascalCase(m.entity), mapping.ImportType(), mapping.ColumnFields())) buf.N() buf.L("err := scan(ctx, db, sql, dest, args...)") if mapping.Type != ReferenceTable && mapping.Type != MapTable { m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to fetch from \"%s\" table: %%w", err)`, tableName)) } else { m.ifErrNotNil(buf, true, "nil", fmt.Sprintf(`fmt.Errorf("Failed to fetch from \"%s\" table: %%w", parent, err)`, tableName)) } buf.L(" return objects, nil") buf.L("}") buf.N() return nil } incus-7.0.0/cmd/generate-database/db/parse.go000066400000000000000000000344701517523235500210040ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "fmt" "go/ast" "go/types" "net/url" "reflect" "slices" "sort" "strconv" "strings" "golang.org/x/tools/go/packages" "github.com/lxc/incus/v7/cmd/generate-database/lex" "github.com/lxc/incus/v7/shared/util" ) // FiltersFromStmt parses all filtering statement defined for the given entity. It // returns all supported combinations of filters, sorted by number of criteria, and // the corresponding set of unused filters from the Filter struct. func FiltersFromStmt(pkgs []*types.Package, kind string, entity string, filters []*Field, registeredSQLStmts map[string]string) ([][]string, [][]string) { for _, pkg := range pkgs { objects := pkg.Scope().Names() stmtFilters := [][]string{} prefix := fmt.Sprintf("%s%sBy", lex.CamelCase(entity), lex.PascalCase(kind)) seenNames := make(map[string]struct{}, len(objects)) for _, name := range objects { if !strings.HasPrefix(name, prefix) { continue } rest := name[len(prefix):] stmtFilters = append(stmtFilters, strings.Split(rest, "And")) seenNames[rest] = struct{}{} } for name := range registeredSQLStmts { if !strings.HasPrefix(name, prefix) { continue } rest := name[len(prefix):] _, ok := seenNames[rest] if ok { continue } stmtFilters = append(stmtFilters, strings.Split(rest, "And")) } stmtFilters = sortFilters(stmtFilters) ignoredFilters := [][]string{} for _, filterGroup := range stmtFilters { ignoredFilterGroup := []string{} for _, filter := range filters { if !slices.Contains(filterGroup, filter.Name) { ignoredFilterGroup = append(ignoredFilterGroup, filter.Name) } } ignoredFilters = append(ignoredFilters, ignoredFilterGroup) } return stmtFilters, ignoredFilters } return nil, nil } // RefFiltersFromStmt parses all filtering statement defined for the given entity reference. func RefFiltersFromStmt(pkg *types.Package, entity string, ref string, filters []*Field, registeredSQLStmts map[string]string) ([][]string, [][]string) { objects := pkg.Scope().Names() stmtFilters := [][]string{} prefix := fmt.Sprintf("%s%sRefBy", lex.CamelCase(entity), lex.Capital(ref)) seenNames := make(map[string]struct{}, len(objects)) for _, name := range objects { if !strings.HasPrefix(name, prefix) { continue } rest := name[len(prefix):] stmtFilters = append(stmtFilters, strings.Split(rest, "And")) seenNames[rest] = struct{}{} } for name := range registeredSQLStmts { if !strings.HasPrefix(name, prefix) { continue } rest := name[len(prefix):] _, ok := seenNames[rest] if ok { continue } stmtFilters = append(stmtFilters, strings.Split(rest, "And")) } stmtFilters = sortFilters(stmtFilters) ignoredFilters := [][]string{} for _, filterGroup := range stmtFilters { ignoredFilterGroup := []string{} for _, filter := range filters { if !slices.Contains(filterGroup, filter.Name) { ignoredFilterGroup = append(ignoredFilterGroup, filter.Name) } } ignoredFilters = append(ignoredFilters, ignoredFilterGroup) } return stmtFilters, ignoredFilters } func sortFilters(filters [][]string) [][]string { sort.Slice(filters, func(i, j int) bool { n1 := len(filters[i]) n2 := len(filters[j]) if n1 != n2 { return n1 > n2 } f1 := sortFilter(filters[i]) f2 := sortFilter(filters[j]) for k := range f1 { if f1[k] == f2[k] { continue } return f1[k] > f2[k] } panic("duplicate filter") }) return filters } func sortFilter(filter []string) []string { f := make([]string, len(filter)) copy(f, filter) sort.Sort(sort.Reverse(sort.StringSlice(f))) return f } // Parse the structure declaration with the given name found in the given Go package. // Any 'Entity' struct should also have an 'EntityFilter' struct defined in the same file. func Parse(localPath string, pkgs []*types.Package, name string, kind string) (*Mapping, error) { m := &Mapping{} for _, pkg := range pkgs { // Find the package that has the main entity struct. str := findStruct(pkg.Scope(), name) if str == nil { continue } fields, err := parseStruct(str, kind, pkg.Name()) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", name, err) } m.Local = pkg.Path() == localPath m.Package = pkg.Name() m.Name = name m.Fields = fields m.Type = tableType(pkgs, fields) m.Filterable = true oldStructHasTags := false for _, f := range m.Fields { if len(f.Config) > 0 { oldStructHasTags = true break } } if oldStructHasTags { break } } if m.Package == "" { return nil, fmt.Errorf("No declaration found for %q", name) } if m.Filterable && m.Filters == nil { for _, pkg := range pkgs { filters, err := ParseFilter(m, kind, name, pkg) if err != nil { return nil, err } if filters != nil { m.Filters = filters m.FilterLocal = pkg.Path() == localPath break } } if m.Filters == nil { filterName := name + "Filter" return nil, fmt.Errorf("No declaration found for filter %q", filterName) } } return m, nil } // ParseFilter finds the Filter struct in the given package. func ParseFilter(m *Mapping, kind string, name string, pkg *types.Package) ([]*Field, error) { // The 'EntityFilter' struct. This is used for filtering on specific fields of the entity. filterName := name + "Filter" filterStr := findStruct(pkg.Scope(), filterName) if filterStr == nil { return nil, nil } filters, err := parseStruct(filterStr, kind, pkg.Name()) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", name, err) } for i, filter := range filters { // Any field in EntityFilter must be present in the original struct. field := m.FieldByName(filter.Name) if field == nil { return nil, fmt.Errorf("Filter field %q is not in struct %q", filter.Name, name) } // Assign the config tags from the main entity struct to the Filter struct. filters[i].Config = field.Config // A Filter field and its indirect references must all be in the Filter struct. if field.IsIndirect() { indirectField := lex.PascalCase(field.Config.Get("via")) for i, f := range filters { if f.Name == indirectField { break } if i == len(filters)-1 { return nil, fmt.Errorf("Field %q requires field %q in struct %q", field.Name, indirectField, name+"Filter") } } } } return filters, nil } // ParseStmt returns the SQL string passed as an argument to a variable declaration of a call to RegisterStmt with the given name. // e.g. the SELECT string from 'var instanceObjects = RegisterStmt(`SELECT * from instances...`)'. func ParseStmt(name string, defs map[*ast.Ident]types.Object, registeredSQLStmts map[string]string) (string, error) { sql, ok := registeredSQLStmts[name] if ok { return sql, nil } for stmtVar := range defs { if stmtVar.Name != name { continue } spec, ok := stmtVar.Obj.Decl.(*ast.ValueSpec) if !ok { continue } if len(spec.Values) != 1 { continue } expr, ok := spec.Values[0].(*ast.CallExpr) if !ok { continue } if len(expr.Args) != 1 { continue } lit, ok := expr.Args[0].(*ast.BasicLit) if !ok { continue } return lit.Value, nil } return "", fmt.Errorf("Declaration for %q not found", name) } // tableType determines the TableType for the given struct fields. func tableType(pkgs []*types.Package, fields []*Field) TableType { fieldNames := FieldNames(fields) idFields := []string{} for _, field := range fields { if field.Name == "ID" { idFields = nil break } if strings.HasSuffix(lex.SnakeCase(field.Name), "_id") { structName, ok := strings.CutSuffix(field.Name, "ID") if ok { idFields = append(idFields, structName) } } } if len(idFields) == 2 { var struct1 *types.Struct var struct2 *types.Struct for _, pkg := range pkgs { if struct1 == nil { struct1 = findStruct(pkg.Scope(), lex.PascalCase(lex.Singular(idFields[0]))) } if struct2 == nil { struct2 = findStruct(pkg.Scope(), lex.PascalCase(lex.Singular(idFields[1]))) } } if struct1 != nil && struct2 != nil { return AssociationTable } } if slices.Contains(fieldNames, "ReferenceID") { if slices.Contains(fieldNames, "Key") && slices.Contains(fieldNames, "Value") { return MapTable } return ReferenceTable } return EntityTable } func parsePkgDecls(entity string, kind string, pkgs []*packages.Package) ([]*types.Package, error) { structName := lex.PascalCase(entity) pkgTypes := make([]*types.Package, 0, len(pkgs)) numSeenDecls := 0 numTaggedDecls := 0 for _, pkg := range pkgs { for _, decl := range pkg.Types.Scope().Names() { // Don't validate any structs beyond the one we care about. if decl != structName { continue } if numTaggedDecls > 1 { return nil, fmt.Errorf("Entity declaration exists in more than one package %q: %q. Remove db tags from one definition", pkg.Name, decl) } // If we encountered a non-struct declaration, just ignore it. obj := pkg.Types.Scope().Lookup(decl) structDecl, ok := obj.Type().Underlying().(*types.Struct) if !ok { continue } numSeenDecls++ fields, err := parseStruct(structDecl, kind, pkg.Types.Name()) if err != nil { return nil, err } for _, f := range fields { if len(f.Config) > 0 { numTaggedDecls++ break } } } pkgTypes = append(pkgTypes, pkg.Types) } if numSeenDecls > 1 && numTaggedDecls != 1 { return nil, fmt.Errorf("Struct %q declaration exists in more than one package. Apply db tags to one definition", structName) } if numSeenDecls == 0 { return nil, fmt.Errorf("No declaration found for struct %q", structName) } return pkgTypes, nil } // Find the StructType node for the structure with the given name. func findStruct(scope *types.Scope, name string) *types.Struct { obj := scope.Lookup(name) if obj == nil { return nil } typ, ok := obj.(*types.TypeName) if !ok { return nil } str, ok := typ.Type().Underlying().(*types.Struct) if !ok { return nil } return str } // Extract field information from the given structure. func parseStruct(str *types.Struct, kind string, pkgName string) ([]*Field, error) { fields := make([]*Field, 0) for i := range str.NumFields() { f := str.Field(i) if f.Embedded() { // Check if this is a parent struct. parentStr, ok := f.Type().Underlying().(*types.Struct) if !ok { continue } parentFields, err := parseStruct(parentStr, kind, pkgName) if err != nil { return nil, fmt.Errorf("Failed to parse parent struct: %w", err) } fields = append(fields, parentFields...) continue } field, err := parseField(f, str.Tag(i), kind, pkgName) if err != nil { return nil, err } // Don't add field if it has been ignored. if field != nil { fields = append(fields, field) } } return fields, nil } func parseField(f *types.Var, structTag string, kind string, pkgName string) (*Field, error) { name := f.Name() if !f.Exported() { return nil, fmt.Errorf("Unexported field name %q", name) } // Ignore fields that are marked with a tag of `db:"ignore"` if structTag != "" { tagValue := reflect.StructTag(structTag).Get("db") if tagValue == "ignore" { return nil, nil } } typeName := parseType(f.Type(), pkgName) if typeName == "" { return nil, fmt.Errorf("Unsupported type for field %q", name) } typeObj := Type{ Name: typeName, } var config url.Values if structTag != "" { var err error config, err = url.ParseQuery(reflect.StructTag(structTag).Get("db")) if err != nil { return nil, fmt.Errorf("Parse 'db' structure tag: %w", err) } err = validateFieldConfig(config) if err != nil { return nil, fmt.Errorf("Invalid struct tag for field %q: %v", name, err) } } typeObj.Code = TypeColumn if config.Get("marshal") == "" { if strings.HasPrefix(typeName, "[]") { typeObj.Code = TypeSlice } else if strings.HasPrefix(typeName, "map[") { typeObj.Code = TypeMap } } // Ignore fields that are marked with `db:"omit"`. omit := config.Get("omit") if omit != "" { omitFields := strings.Split(omit, ",") stmtKind := strings.ReplaceAll(lex.SnakeCase(kind), "_", "-") switch kind { case "URIs": stmtKind = "names" case "GetMany": stmtKind = "objects" case "GetOne": stmtKind = "objects" case "DeleteMany": stmtKind = "delete" case "DeleteOne": stmtKind = "delete" } if slices.Contains(omitFields, kind) || slices.Contains(omitFields, stmtKind) { return nil, nil } else if kind == "exists" && slices.Contains(omitFields, "id") { // Exists checks ID, so if we are omitting the field from ID, also omit it from Exists. return nil, nil } } field := Field{ Name: name, Type: typeObj, Config: config, } return &field, nil } func parseType(x types.Type, pkgName string) string { switch t := x.(type) { case *types.Pointer: return parseType(t.Elem(), pkgName) case *types.Slice: return "[]" + parseType(t.Elem(), pkgName) case *types.Basic: s := t.String() if s == "byte" { return "uint8" } return s case *types.Array: return "[" + strconv.FormatInt(t.Len(), 10) + "]" + parseType(t.Elem(), pkgName) case *types.Map: return "map[" + t.Key().String() + "]" + parseType(t.Elem(), pkgName) case *types.Named: if pkgName == t.Obj().Pkg().Name() { return t.Obj().Name() } return t.Obj().Pkg().Name() + "." + t.Obj().Name() case nil: return "" default: return "" } } func validateFieldConfig(config url.Values) error { for tag, values := range config { switch tag { case "sql", "coalesce", "join", "leftjoin", "joinon", "omit": _, err := exactlyOneValue(tag, values) return err case "order", "primary", "ignore": value, err := exactlyOneValue(tag, values) if err != nil { return err } if !util.IsTrue(value) && !util.IsFalse(value) { return fmt.Errorf("Unexpected value %q for %q tag", value, tag) } case "marshal": value, err := exactlyOneValue(tag, values) if err != nil { return err } if !util.IsTrue(value) && !util.IsFalse(value) && strings.ToLower(value) != "json" { return fmt.Errorf("Unexpected value %q for %q tag", value, tag) } } } return nil } func exactlyOneValue(tag string, values []string) (string, error) { if len(values) == 0 { return "", fmt.Errorf("Missing value for %q tag", tag) } if len(values) > 1 { return "", fmt.Errorf("More than one value for %q tag", tag) } return values[0], nil } incus-7.0.0/cmd/generate-database/db/parse_test.go000066400000000000000000000026451517523235500220420ustar00rootroot00000000000000package db_test import ( "go/types" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/tools/go/packages" "github.com/lxc/incus/v7/cmd/generate-database/db" ) type Person struct { Name string } type Class struct { Time time.Time Room string } type Teacher struct { Person Subjects []string IsSubstitute bool Classes []Class } type TeacherFilter struct{} func TestParse(t *testing.T) { pkg, err := packages.Load(&packages.Config{ Mode: packages.LoadTypes | packages.NeedTypesInfo, Tests: true, }, "") require.NoError(t, err) m, err := db.Parse(pkg[1].PkgPath, []*types.Package{pkg[1].Types}, "Teacher", "objects") require.NoError(t, err) assert.Equal(t, "db_test", m.Package) assert.Equal(t, "Teacher", m.Name) fields := m.Fields assert.Len(t, fields, 4) assert.Equal(t, "Name", fields[0].Name) assert.Equal(t, "Subjects", fields[1].Name) assert.Equal(t, "IsSubstitute", fields[2].Name) assert.Equal(t, "Classes", fields[3].Name) assert.Equal(t, "string", fields[0].Type.Name) assert.Equal(t, "[]string", fields[1].Type.Name) assert.Equal(t, "bool", fields[2].Type.Name) assert.Equal(t, "[]Class", fields[3].Type.Name) assert.Equal(t, db.TypeColumn, fields[0].Type.Code) assert.Equal(t, db.TypeSlice, fields[1].Type.Code) assert.Equal(t, db.TypeColumn, fields[2].Type.Code) assert.Equal(t, db.TypeSlice, fields[3].Type.Code) } incus-7.0.0/cmd/generate-database/db/schema.go000066400000000000000000000007361517523235500211300ustar00rootroot00000000000000package db import ( "fmt" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/node" ) // UpdateSchema updates the schema.go file of the cluster and node databases. func UpdateSchema() error { err := cluster.SchemaDotGo() if err != nil { return fmt.Errorf("Update cluster database schema: %w", err) } err = node.SchemaDotGo() if err != nil { return fmt.Errorf("Update node database schema: %w", err) } return nil } incus-7.0.0/cmd/generate-database/db/stmt.go000066400000000000000000000361121517523235500206540ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "errors" "fmt" "go/ast" "go/types" "slices" "strings" "golang.org/x/tools/go/packages" "github.com/lxc/incus/v7/cmd/generate-database/file" "github.com/lxc/incus/v7/cmd/generate-database/lex" ) // Stmt generates a particular database query statement. type Stmt struct { entity string // Name of the database entity kind string // Kind of statement to generate config map[string]string // Configuration parameters localPath string pkgs []*types.Package // Package to perform for struct declaration lookups defs map[*ast.Ident]types.Object // Defs maps identifiers to the objects they define registeredSQLStmts map[string]string // Lookup for SQL statements registered during this execution, which are therefore not included in the parsed package information } // NewStmt return a new statement code snippet for running the given kind of // query against the given database entity. func NewStmt(localPath string, parsedPkgs []*packages.Package, entity, kind string, config map[string]string, registeredSQLStmts map[string]string) (*Stmt, error) { defs := map[*ast.Ident]types.Object{} for _, pkg := range parsedPkgs { for k, v := range pkg.TypesInfo.Defs { _, ok := defs[k] if ok { return nil, fmt.Errorf("Entity definition already exists: %q: %q", pkg.Name, v.Name()) } defs[k] = v } } pkgTypes, err := parsePkgDecls(entity, kind, parsedPkgs) if err != nil { return nil, err } stmt := &Stmt{ localPath: localPath, entity: entity, kind: kind, config: config, pkgs: pkgTypes, defs: defs, registeredSQLStmts: registeredSQLStmts, } return stmt, nil } // Generate plumbing and wiring code for the desired statement. func (s *Stmt) Generate(buf *file.Buffer) error { kind := strings.Split(s.kind, "-by-")[0] switch kind { case "objects": return s.objects(buf) case "names": return s.names(buf) case "delete": return s.delete(buf) case "create": return s.create(buf, false) case "create-or-replace": return s.create(buf, true) case "id": return s.id(buf) case "rename": return s.rename(buf) case "update": return s.update(buf) default: return fmt.Errorf("Unknown statement '%s'", s.kind) } } // GenerateSignature is not used for statements. func (s *Stmt) GenerateSignature(buf *file.Buffer) error { return nil } func (s *Stmt) objects(buf *file.Buffer) error { if strings.HasPrefix(s.kind, "objects-by") { return s.objectsBy(buf) } mapping, err := Parse(s.localPath, s.pkgs, lex.PascalCase(s.entity), s.kind) if err != nil { return err } table := mapping.TableName(s.entity, s.config["table"]) boiler := stmts["objects"] fields := mapping.ColumnFields() columns := make([]string, len(fields)) for i, field := range fields { column, err := field.SelectColumn(mapping, table) if err != nil { return err } columns[i] = column } orderBy := []string{} orderByFields := []*Field{} for _, field := range fields { if field.Config.Get("order") != "" { orderByFields = append(orderByFields, field) } } if len(orderByFields) < 1 { orderByFields = mapping.NaturalKey() } for _, field := range orderByFields { column, err := field.OrderBy(mapping, table) if err != nil { return err } orderBy = append(orderBy, column) } joinFields := mapping.ScalarFields() joins := make([]string, 0, len(joinFields)) for _, field := range joinFields { join, err := field.JoinClause(mapping, table) if err != nil { return err } if !slices.Contains(joins, join) { joins = append(joins, join) } } table += strings.Join(joins, "") sql := fmt.Sprintf(boiler, strings.Join(columns, ", "), table, strings.Join(orderBy, ", ")) kind := strings.ReplaceAll(s.kind, "-", "_") stmtName := stmtCodeVar(s.entity, kind) if mapping.Type == ReferenceTable || mapping.Type == MapTable { buf.L("const %s = `%s`", stmtName, sql) } else { s.register(buf, stmtName, sql) } return nil } // objectsBy parses the variable declaration produced by the 'objects' function, and appends a WHERE clause to its SQL // string using the objects-by- field suffixes, and then creates a new variable declaration. // Strictly, it will look for variables of the form 'var Objects = .RegisterStmt(`SQL String`)'. func (s *Stmt) objectsBy(buf *file.Buffer) error { mapping, err := Parse(s.localPath, s.pkgs, lex.PascalCase(s.entity), s.kind) if err != nil { return err } where := []string{} filters := strings.Split(s.kind[len("objects-by-"):], "-and-") sqlString, err := ParseStmt(stmtCodeVar(s.entity, "objects"), s.defs, s.registeredSQLStmts) if err != nil { return err } queryParts := strings.SplitN(sqlString, "ORDER BY", 2) joinStr := " JOIN" if strings.Contains(queryParts[0], " LEFT JOIN") { joinStr = " LEFT JOIN" } preJoin, _, _ := strings.Cut(queryParts[0], joinStr) _, tableName, _ := strings.Cut(preJoin, "FROM ") tableName, _, _ = strings.Cut(tableName, "\n") for _, filter := range filters { field, err := mapping.FilterFieldByName(filter) if err != nil { return err } table, columnName, err := field.SQLConfig() if err != nil { return err } var column string if table != "" && columnName != "" { if field.IsScalar() { column = columnName } else { column = table + "." + columnName } } else if field.IsScalar() { column = lex.SnakeCase(field.Name) } else { column = mapping.FieldColumnName(field.Name, tableName) } coalesce, ok := field.Config["coalesce"] if ok { // Ensure filters operate on the coalesced value for fields using coalesce setting. where = append(where, fmt.Sprintf("coalesce(%s, %s) = ? ", column, coalesce[0])) } else { where = append(where, fmt.Sprintf("%s = ? ", column)) } } queryParts[0] = fmt.Sprintf("%sWHERE ( %s)", queryParts[0], strings.Join(where, "AND ")) sqlString = strings.Join(queryParts, "\n ORDER BY") s.register(buf, stmtCodeVar(s.entity, "objects", filters...), sqlString) return nil } func (s *Stmt) names(buf *file.Buffer) error { if strings.HasPrefix(s.kind, "names-by") { return s.namesBy(buf) } mapping, err := Parse(s.localPath, s.pkgs, lex.PascalCase(s.entity), s.kind) if err != nil { return err } if len(mapping.NaturalKey()) > 1 { return errors.New("Can't return names for composite key objects") } table := mapping.TableName(s.entity, s.config["table"]) boiler := stmts["names"] field := mapping.NaturalKey()[0] column, err := field.SelectColumn(mapping, table) if err != nil { return err } orderByField := field if field.Config.Get("order") != "" { orderByField = field } orderBy, err := orderByField.OrderBy(mapping, table) if err != nil { return err } sql := fmt.Sprintf(boiler, column, table, orderBy) kind := strings.ReplaceAll(s.kind, "-", "_") stmtName := stmtCodeVar(s.entity, kind) s.register(buf, stmtName, sql) return nil } func (s *Stmt) namesBy(buf *file.Buffer) error { mapping, err := Parse(s.localPath, s.pkgs, lex.PascalCase(s.entity), s.kind) if err != nil { return err } if len(mapping.NaturalKey()) > 1 { return errors.New("Can't return names for composite key objects") } where := []string{} filters := strings.Split(s.kind[len("names-by-"):], "-and-") sqlString, err := ParseStmt(stmtCodeVar(s.entity, "names"), s.defs, s.registeredSQLStmts) if err != nil { return err } queryParts := strings.SplitN(sqlString, "ORDER BY", 2) _, tableName, _ := strings.Cut(queryParts[0], "FROM ") tableName, _, _ = strings.Cut(tableName, "\n") joins := []string{} for _, filter := range filters { field, err := mapping.FilterFieldByName(filter) if err != nil { return err } table, columnName, err := field.SQLConfig() if err != nil { return err } var column string if table != "" && columnName != "" { if field.IsScalar() { column = columnName } else { column = table + "." + columnName } } else if field.IsScalar() { join, err := field.JoinClause(mapping, tableName) if err != nil { return err } if !slices.Contains(joins, join) { joins = append(joins, join) } column = field.joinConfig() } else { column = mapping.FieldColumnName(field.Name, tableName) } coalesce, ok := field.Config["coalesce"] if ok { // Ensure filters operate on the coalesced value for fields using coalesce setting. where = append(where, fmt.Sprintf("coalesce(%s, %s) = ? ", column, coalesce[0])) } else { where = append(where, fmt.Sprintf("%s = ? ", column)) } } join := "" if len(joins) > 0 { join = strings.TrimLeftFunc(strings.Join(joins, ""), func(r rune) bool { return r == ' ' || r == '\n' }) join += "\n " } queryParts[0] = fmt.Sprintf("%s%sWHERE ( %s)", queryParts[0], join, strings.Join(where, "AND ")) sqlString = strings.Join(queryParts, "\n ORDER BY") s.register(buf, stmtCodeVar(s.entity, "names", filters...), sqlString) return nil } func (s *Stmt) create(buf *file.Buffer, replace bool) error { entityCreate := lex.PascalCase(s.entity) mapping, err := Parse(s.localPath, s.pkgs, entityCreate, s.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } table := mapping.TableName(s.entity, s.config["table"]) all := mapping.ColumnFields("ID") // This exclude the ID column, which is autogenerated. columns := make([]string, 0, len(all)) values := make([]string, 0, len(all)) for _, field := range all { column, value, err := field.InsertColumn(mapping, table, s.defs, s.registeredSQLStmts, all) if err != nil { return err } if column == "" && value == "" { continue } columns = append(columns, column) values = append(values, value) } tmpl := stmts[s.kind] if replace { tmpl = stmts["replace"] } sql := fmt.Sprintf(tmpl, table, strings.Join(columns, ", "), strings.Join(values, ", ")) kind := strings.Replace(s.kind, "-", "_", -2) stmtName := stmtCodeVar(s.entity, kind) if mapping.Type == ReferenceTable || mapping.Type == MapTable { buf.L("const %s = `%s`", stmtName, sql) } else { s.register(buf, stmtName, sql) } return nil } func (s *Stmt) id(buf *file.Buffer) error { mapping, err := Parse(s.localPath, s.pkgs, lex.PascalCase(s.entity), s.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } table := mapping.TableName(s.entity, s.config["table"]) nk := mapping.NaturalKey() where := make([]string, 0, len(nk)) joins := make([]string, 0, len(nk)) for _, field := range nk { tableName, columnName, err := field.SQLConfig() if err != nil { return err } var column string if field.IsScalar() { column = field.joinConfig() join, err := field.JoinClause(mapping, table) if !slices.Contains(joins, join) { joins = append(joins, join) } if err != nil { return err } } else if tableName != "" && columnName != "" { column = tableName + "." + columnName } else { column = mapping.FieldColumnName(field.Name, table) } where = append(where, fmt.Sprintf("%s = ?", column)) } sql := fmt.Sprintf(stmts[s.kind], table, table+strings.Join(joins, ""), strings.Join(where, " AND ")) stmtName := stmtCodeVar(s.entity, "ID") s.register(buf, stmtName, sql) return nil } func (s *Stmt) rename(buf *file.Buffer) error { mapping, err := Parse(s.localPath, s.pkgs, lex.PascalCase(s.entity), s.kind) if err != nil { return err } table := mapping.TableName(s.entity, s.config["table"]) nk := mapping.NaturalKey() updates := make([]string, 0, len(nk)) for _, field := range nk { column, value, err := field.InsertColumn(mapping, table, s.defs, s.registeredSQLStmts, nk) if err != nil { return err } if column == "" && value == "" { continue } updates = append(updates, fmt.Sprintf("%s = %s", column, value)) } updatedAt := "" for _, field := range mapping.Fields { if field.Config.Has("update_timestamp") { updatedAt = fmt.Sprintf(", %s = ?", field.Column()) break } } sql := fmt.Sprintf(stmts[s.kind], table, updatedAt, strings.Join(updates, " AND ")) kind := strings.ReplaceAll(s.kind, "-", "_") stmtName := stmtCodeVar(s.entity, kind) s.register(buf, stmtName, sql) return nil } func (s *Stmt) update(buf *file.Buffer) error { entityUpdate := lex.PascalCase(s.entity) mapping, err := Parse(s.localPath, s.pkgs, entityUpdate, s.kind) if err != nil { return fmt.Errorf("Parse entity struct: %w", err) } table := mapping.TableName(s.entity, s.config["table"]) all := mapping.ColumnFields("ID") // This exclude the ID column, which is autogenerated. updates := make([]string, 0, len(all)) for _, field := range all { column, value, err := field.InsertColumn(mapping, table, s.defs, s.registeredSQLStmts, all) if err != nil { return err } if column == "" && value == "" { continue } updates = append(updates, fmt.Sprintf("%s = %s", column, value)) } sql := fmt.Sprintf(stmts[s.kind], table, strings.Join(updates, ", "), "id = ?") kind := strings.ReplaceAll(s.kind, "-", "_") stmtName := stmtCodeVar(s.entity, kind) s.register(buf, stmtName, sql) return nil } func (s *Stmt) delete(buf *file.Buffer) error { mapping, err := Parse(s.localPath, s.pkgs, lex.PascalCase(s.entity), s.kind) if err != nil { return err } table := mapping.TableName(s.entity, s.config["table"]) var where string if mapping.Type == ReferenceTable || mapping.Type == MapTable { where = "%s_id = ?" } if strings.HasPrefix(s.kind, "delete-by") { filters := strings.Split(s.kind[len("delete-by-"):], "-and-") conditions := make([]string, 0, len(filters)) for _, filter := range filters { field, err := mapping.FilterFieldByName(filter) if err != nil { return err } column, value, err := field.InsertColumn(mapping, table, s.defs, s.registeredSQLStmts, nil) if err != nil { return err } if column == "" && value == "" { continue } conditions = append(conditions, fmt.Sprintf("%s = %s", column, value)) } where = strings.Join(conditions, " AND ") } sql := fmt.Sprintf(stmts["delete"], table, where) kind := strings.ReplaceAll(s.kind, "-", "_") stmtName := stmtCodeVar(s.entity, kind) if mapping.Type == ReferenceTable || mapping.Type == MapTable { buf.L("const %s = `%s`", stmtName, sql) } else { s.register(buf, stmtName, sql) } return nil } // Output a line of code that registers the given statement and declares the // associated statement code global variable. func (s *Stmt) register(buf *file.Buffer, stmtName, sql string) { s.registeredSQLStmts[stmtName] = sql if !strings.HasPrefix(sql, "`") || !strings.HasSuffix(sql, "`") { sql = fmt.Sprintf("`\n%s\n`", sql) } buf.L("var %s = RegisterStmt(%s)", stmtName, sql) } // Map of boilerplate statements. var stmts = map[string]string{ "names": "SELECT %s\n FROM %s\n ORDER BY %s", "objects": "SELECT %s\n FROM %s\n ORDER BY %s", "create": "INSERT INTO %s (%s)\n VALUES (%s)", "replace": "INSERT OR REPLACE INTO %s (%s)\n VALUES (%s)", "id": "SELECT %s.id FROM %s\n WHERE %s", "rename": "UPDATE %s SET name = ?%s WHERE %s", "update": "UPDATE %s\n SET %s\n WHERE %s", "delete": "DELETE FROM %s WHERE %s", } incus-7.0.0/cmd/generate-database/file/000077500000000000000000000000001517523235500176655ustar00rootroot00000000000000incus-7.0.0/cmd/generate-database/file/boilerplate/000077500000000000000000000000001517523235500221675ustar00rootroot00000000000000incus-7.0.0/cmd/generate-database/file/boilerplate/boilerplate.go000066400000000000000000000114301517523235500250170ustar00rootroot00000000000000package boilerplate import ( "context" "database/sql" "encoding/json" "errors" "fmt" ) type tx interface { //nolint:unused dbtx Commit() error Rollback() error } type dbtx interface { ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) PrepareContext(ctx context.Context, query string) (*sql.Stmt, error) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row } type preparer interface { Prepare(query string) (*sql.Stmt, error) } // RegisterStmt register a SQL statement. // // Registered statements will be prepared upfront and reused, to speed up // execution. // // Return a unique registration code. func RegisterStmt(sqlStmt string) int { code := len(stmts) stmts[code] = sqlStmt return code } // PrepareStmts prepares all registered statements and returns an index from // statement code to prepared statement object. func PrepareStmts(db preparer, skipErrors bool) (map[int]*sql.Stmt, error) { index := map[int]*sql.Stmt{} for code, sqlStmt := range stmts { stmt, err := db.Prepare(sqlStmt) if err != nil && !skipErrors { return nil, fmt.Errorf("%q: %w", sqlStmt, err) } index[code] = stmt } return index, nil } var stmts = map[int]string{} // Statement code to statement SQL text. // PreparedStmts is a placeholder for transitioning to package-scoped transaction functions. var PreparedStmts = map[int]*sql.Stmt{} // Stmt prepares the in-memory prepared statement for the transaction. func Stmt(db dbtx, code int) (*sql.Stmt, error) { stmt, ok := PreparedStmts[code] if !ok { return nil, fmt.Errorf("No prepared statement registered with code %d", code) } tx, ok := db.(*sql.Tx) if ok { return tx.Stmt(stmt), nil } return stmt, nil } // StmtString returns the in-memory query string with the given code. func StmtString(code int) (string, error) { stmt, ok := stmts[code] if !ok { return "", fmt.Errorf("No prepared statement registered with code %d", code) } return stmt, nil } var ( // ErrNotFound is the error returned, if the entity is not found in the DB. ErrNotFound = errors.New("Not found") // ErrConflict is the error returned, if the adding or updating an entity // causes a conflict with an existing entity. ErrConflict = errors.New("Conflict") ) var mapErr = defaultMapErr func defaultMapErr(err error, entity string) error { return err } // Marshaler is the interface that wraps the MarshalDB method, which converts // the underlying type into a string representation suitable for persistence in // the database. type Marshaler interface { MarshalDB() (string, error) } // Unmarshaler is the interface that wraps the UnmarshalDB method, which converts // a string representation retrieved from the database into the underlying type. type Unmarshaler interface { UnmarshalDB(string) error } func marshal(v any) (string, error) { marshaller, ok := v.(Marshaler) if !ok { return "", errors.New("Cannot marshal data, type does not implement DBMarshaler") } return marshaller.MarshalDB() } func unmarshal(data string, v any) error { if v == nil { return errors.New("Cannot unmarshal data into nil value") } unmarshaler, ok := v.(Unmarshaler) if !ok { return errors.New("Cannot marshal data, type does not implement DBUnmarshaler") } return unmarshaler.UnmarshalDB(data) } func marshalJSON(v any) (string, error) { marshalled, err := json.Marshal(v) if err != nil { return "", err } return string(marshalled), nil } func unmarshalJSON(data string, v any) error { return json.Unmarshal([]byte(data), v) } // dest is a function that is expected to return the objects to pass to the // 'dest' argument of sql.Rows.Scan(). It is invoked by SelectObjects once per // yielded row, and it will be passed the index of the row being scanned. type dest func(scan func(dest ...any) error) error // selectObjects executes a statement which must yield rows with a specific // columns schema. It invokes the given Dest hook for each yielded row. func selectObjects(ctx context.Context, stmt *sql.Stmt, rowFunc dest, args ...any) error { rows, err := stmt.QueryContext(ctx, args...) if err != nil { return err } defer func() { _ = rows.Close() }() for rows.Next() { err = rowFunc(rows.Scan) if err != nil { return err } } return rows.Err() } // scan runs a query with inArgs and provides the rowFunc with the scan function for each row. // It handles closing the rows and errors from the result set. func scan(ctx context.Context, db dbtx, sqlStmt string, rowFunc dest, inArgs ...any) error { rows, err := db.QueryContext(ctx, sqlStmt, inArgs...) if err != nil { return err } defer func() { _ = rows.Close() }() for rows.Next() { err = rowFunc(rows.Scan) if err != nil { return err } } return rows.Err() } incus-7.0.0/cmd/generate-database/file/boilerplate/boilerplate_test.go000066400000000000000000000004221517523235500260550ustar00rootroot00000000000000package boilerplate import ( "testing" ) func Test(t *testing.T) { // Fake the usage of the private variables and functions in the boilerplate. _ = mapErr _ = defaultMapErr _ = marshal _ = unmarshal _ = marshalJSON _ = unmarshalJSON _ = selectObjects _ = scan } incus-7.0.0/cmd/generate-database/file/buffer.go000066400000000000000000000013531517523235500214670ustar00rootroot00000000000000package file import ( "bytes" "fmt" "go/format" ) // Buffer for accumulating source code output. type Buffer struct { buf *bytes.Buffer } // Create a new source code text buffer. func newBuffer() *Buffer { return &Buffer{ buf: bytes.NewBuffer(nil), } } // L accumulates a single line of source code. func (b *Buffer) L(formatStr string, a ...any) { fmt.Fprintf(b.buf, formatStr, a...) b.N() } // N accumulates a single new line. func (b *Buffer) N() { fmt.Fprint(b.buf, "\n") } // Returns the source code to add to the target file. func (b *Buffer) code() ([]byte, error) { code, err := format.Source(b.buf.Bytes()) if err != nil { return nil, fmt.Errorf("Can't format generated source code: %w", err) } return code, nil } incus-7.0.0/cmd/generate-database/file/doc.go000066400000000000000000000001271517523235500207610ustar00rootroot00000000000000// Package file contains helpers to write auto-generated Go source files. package file incus-7.0.0/cmd/generate-database/file/snippet.go000066400000000000000000000002731517523235500217000ustar00rootroot00000000000000package file // Snippet generates a single code snippet of a target source file code. type Snippet interface { Generate(buffer *Buffer) error GenerateSignature(buffer *Buffer) error } incus-7.0.0/cmd/generate-database/file/write.go000066400000000000000000000114021517523235500213440ustar00rootroot00000000000000package file import ( _ "embed" "errors" "fmt" "os" "strings" "github.com/lxc/incus/v7/cmd/generate-database/lex" ) const codeGeneratedByLine = `// Code generated by generate-database from the incus project - DO NOT EDIT.` //go:embed boilerplate/boilerplate.go var boilerplate string // Boilerplate writes the general boilerplate code for mapper to the target package. func Boilerplate(path string) error { boilerplate = strings.Replace(boilerplate, "package boilerplate", fmt.Sprintf("package %s", os.Getenv("GOPACKAGE")), 1) content := codeGeneratedByLine + "\n\n" + boilerplate bytes := []byte(content) var err error if path == "-" { _, err = os.Stdout.Write(bytes) } else { err = os.WriteFile(path, []byte(content), 0o644) } if err != nil { return fmt.Errorf("Mapper boilerplate file %q: %w", path, err) } return nil } // Reset an auto-generated source file, writing a new empty file header. func Reset(path string, imports []string, buildComment string, iface bool) error { // A new line needs to be appended after the build comment. if buildComment != "" { buildComment = fmt.Sprintf(`%s `, buildComment) } if iface { err := resetInterface(path, buildComment) if err != nil { return err } } content := fmt.Sprintf(`%s%s package %s import ( `, buildComment, codeGeneratedByLine, os.Getenv("GOPACKAGE")) for _, uri := range imports { content += fmt.Sprintf("\t%q\n", uri) } content += ")\n\n" bytes := []byte(content) var err error if path == "-" { _, err = os.Stdout.Write(bytes) } else { err = os.WriteFile(path, []byte(content), 0o644) } if err != nil { return fmt.Errorf("Reset target source file %q: %w", path, err) } return nil } func resetInterface(path string, buildComment string) error { if strings.HasSuffix(path, "mapper.go") { parts := strings.Split(path, ".") interfacePath := strings.Join(parts[:len(parts)-2], ".") + ".interface.mapper.go" content := fmt.Sprintf("%spackage %s", buildComment, os.Getenv("GOPACKAGE")) err := os.WriteFile(interfacePath, []byte(content), 0o644) return err } return nil } // Append a code snippet to a file. func Append(entity string, path string, snippet Snippet, iface bool) error { if iface { err := appendInterface(entity, path, snippet) if err != nil { return err } } buffer := newBuffer() buffer.N() err := snippet.Generate(buffer) if err != nil { return fmt.Errorf("Generate code snippet: %w", err) } var file *os.File if path == "-" { file = os.Stdout } else { file, err = os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) if err != nil { return fmt.Errorf("Open target source code file %q: %w", path, err) } defer func() { _ = file.Close() }() } bytes, err := buffer.code() if err != nil { return err } _, err = file.Write(bytes) if err != nil { return fmt.Errorf("Append snippet to target source code file %q: %w", path, err) } // Return any errors on close if file is not stdout. if path != "-" { return file.Close() } return nil } func appendInterface(entity string, path string, snippet Snippet) error { if !strings.HasSuffix(path, ".mapper.go") { return nil } parts := strings.Split(path, ".") interfacePath := strings.Join(parts[:len(parts)-2], ".") + ".interface.mapper.go" stat, err := os.Stat(interfacePath) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil } return fmt.Errorf("could not get file info for path %q: %w", interfacePath, err) } buffer := newBuffer() file, err := os.OpenFile(interfacePath, os.O_RDWR, 0o644) if err != nil { return fmt.Errorf("Open target source code file %q: %w", interfacePath, err) } defer func() { _ = file.Close() }() err = snippet.GenerateSignature(buffer) if err != nil { return fmt.Errorf("Generate interface snippet: %w", err) } bytes, err := buffer.code() if err != nil { return err } declaration := fmt.Sprintf("type %sGenerated interface {", lex.PascalCase(entity)) content := make([]byte, stat.Size()) _, err = file.Read(content) if err != nil { return fmt.Errorf("Could not read interface file %q: %w", interfacePath, err) } firstWrite := !strings.Contains(string(content), declaration) if firstWrite { // If this is the first signature write to the file, append the whole thing. _, err = file.WriteAt(bytes, stat.Size()) } else { // If an interface already exists, just append the method, omitting everything before the first '{'. startIndex := 0 for i := range bytes { // type ObjectGenerated interface { if string(bytes[i]) == "{" { startIndex = i + 1 break } } // overwrite the closing brace. _, err = file.WriteAt(bytes[startIndex:], stat.Size()-2) } if err != nil { return fmt.Errorf("Append snippet to target source code file %q: %w", interfacePath, err) } return file.Close() } incus-7.0.0/cmd/generate-database/lex/000077500000000000000000000000001517523235500175365ustar00rootroot00000000000000incus-7.0.0/cmd/generate-database/lex/case.go000066400000000000000000000040631517523235500210030ustar00rootroot00000000000000package lex import ( "bytes" "strings" "unicode" "golang.org/x/text/cases" "golang.org/x/text/language" ) // Capital capitalizes the given string ("foo" -> "Foo"). func Capital(s string) string { return cases.Title(language.English, cases.NoLower).String(s) } // Minuscule turns the first character to lower case ("Foo" -> "foo") or the whole word if it is all uppercase ("UUID" -> "uuid"). func Minuscule(s string) string { if strings.ToUpper(s) == s { return strings.ToLower(s) } return strings.ToLower(s[:1]) + s[1:] } // CamelCase converts to camel case ("foo_bar" -> "fooBar"). func CamelCase(s string) string { return Minuscule(PascalCase(s)) } // PascalCase converts to pascal case ("foo_bar" -> "FooBar"). func PascalCase(s string) string { words := strings.Split(s, "_") for i := range words { words[i] = Capital(words[i]) } return strings.Join(words, "") } // SnakeCase converts to snake case ("FooBar" -> "foo_bar"). func SnakeCase(name string) string { var ret bytes.Buffer multipleUpper := false var lastUpper rune var beforeUpper rune for _, c := range name { // Non-lowercase character after uppercase is considered to be uppercase too. isUpper := (unicode.IsUpper(c) || (lastUpper != 0 && !unicode.IsLower(c))) if lastUpper != 0 { // Output a delimiter if last character was either the // first uppercase character in a row, or the last one // in a row (e.g. 'S' in "HTTPServer"). Do not output // a delimiter at the beginning of the name. firstInRow := !multipleUpper lastInRow := !isUpper if ret.Len() > 0 && (firstInRow || lastInRow) && beforeUpper != '_' { ret.WriteByte('_') } ret.WriteRune(unicode.ToLower(lastUpper)) } // Buffer uppercase char, do not output it yet as a delimiter // may be required if the next character is lowercase. if isUpper { multipleUpper = (lastUpper != 0) lastUpper = c continue } ret.WriteRune(c) lastUpper = 0 beforeUpper = c multipleUpper = false } if lastUpper != 0 { ret.WriteRune(unicode.ToLower(lastUpper)) } return ret.String() } incus-7.0.0/cmd/generate-database/lex/config.go000066400000000000000000000005631517523235500213360ustar00rootroot00000000000000package lex import ( "fmt" "strings" ) // KeyValue extracts the key and value encoded in the given string and // separated by '=' (foo=bar -> foo, bar). func KeyValue(s string) (string, string, error) { parts := strings.Split(s, "=") if len(parts) != 2 { return "", "", fmt.Errorf("The token %q is not a key/value pair", s) } return parts[0], parts[1], nil } incus-7.0.0/cmd/generate-database/lex/form.go000066400000000000000000000016061517523235500210330ustar00rootroot00000000000000package lex import ( "strings" ) // Plural converts to plural form ("foo" -> "foos"). func Plural(s string) string { // TODO: smarter algorithm? :) if strings.HasSuffix(strings.ToLower(s), "config") { return s } if strings.HasSuffix(s, "ch") || strings.HasSuffix(s, "sh") || strings.HasSuffix(s, "ss") { return s + "es" } if strings.HasSuffix(s, "y") { return s[:len(s)-1] + "ies" } if s[len(s)-1] != 's' { return s + "s" } return s } // Singular converts to singular form ("foos" -> "foo"). func Singular(s string) string { // TODO: smarter algorithm? :) before, ok := strings.CutSuffix(s, "ies") if ok { return before + "y" } before, ok = strings.CutSuffix(s, "es") if ok && (strings.HasSuffix(before, "ch") || strings.HasSuffix(before, "sh") || strings.HasSuffix(before, "ss")) { return before } if s[len(s)-1] == 's' { return s[:len(s)-1] } return s } incus-7.0.0/cmd/generate-database/lex/lang.go000066400000000000000000000016121517523235500210060ustar00rootroot00000000000000package lex import ( "fmt" ) // VarDecl holds information about a variable declaration. type VarDecl struct { Name string Expr string } func (d VarDecl) String() string { return fmt.Sprintf("%s %s", d.Name, d.Expr) } // MethodSignature holds information about a method signature. type MethodSignature struct { Comment string // Method comment Name string // Method name Receiver VarDecl // Receiver name and type Args []VarDecl // Method arguments Return []string // Return type } // Slice returns the type name of a slice of items of the given type. func Slice(typ string) string { return fmt.Sprintf("[]%s", typ) } // Element is the reverse of Slice, returning the element type name the slice // with given type. func Element(typ string) string { return typ[len("[]"):] } // Star adds a "*" prefix to the given string. func Star(s string) string { return "*" + s } incus-7.0.0/cmd/generate-database/main.go000066400000000000000000000001731517523235500202220ustar00rootroot00000000000000package main import ( "os" ) func main() { root := newRoot() err := root.Execute() if err != nil { os.Exit(1) } } incus-7.0.0/cmd/generate-database/root.go000066400000000000000000000013351517523235500202620ustar00rootroot00000000000000package main import ( "errors" "github.com/spf13/cobra" ) // Return a new root command. func newRoot() *cobra.Command { cmd := &cobra.Command{ Use: "generate-database", Short: "Code generation tool for Incus development", Long: `This is the entry point for all "go:generate" directives used in Incus' source code.`, RunE: func(cmd *cobra.Command, args []string) error { return errors.New("Not implemented") }, CompletionOptions: cobra.CompletionOptions{DisableDefaultCmd: true}, } cmd.AddCommand(newDb()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } return cmd } incus-7.0.0/cmd/incus-agent/000077500000000000000000000000001517523235500156275ustar00rootroot00000000000000incus-7.0.0/cmd/incus-agent/api.go000066400000000000000000000010361517523235500167270ustar00rootroot00000000000000package main import ( "net/http" "github.com/lxc/incus/v7/internal/server/response" ) // APIEndpoint represents a URL in our API. type APIEndpoint struct { Name string // Name for this endpoint. Path string // Path pattern for this endpoint. Get APIEndpointAction Put APIEndpointAction Post APIEndpointAction Delete APIEndpointAction Patch APIEndpointAction } // APIEndpointAction represents an action on an API endpoint. type APIEndpointAction struct { Handler func(d *Daemon, r *http.Request) response.Response } incus-7.0.0/cmd/incus-agent/api_1.0.go000066400000000000000000000116531517523235500173130ustar00rootroot00000000000000package main import ( "encoding/json" "errors" "fmt" "io" "net/http" "os" "path/filepath" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/ports" "github.com/lxc/incus/v7/internal/server/response" localvsock "github.com/lxc/incus/v7/internal/server/vsock" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" agentAPI "github.com/lxc/incus/v7/shared/api/agent" localtls "github.com/lxc/incus/v7/shared/tls" ) var api10Cmd = APIEndpoint{ Get: APIEndpointAction{Handler: api10Get}, Put: APIEndpointAction{Handler: api10Put}, } var api10 = []APIEndpoint{ api10Cmd, execCmd, eventsCmd, metricsCmd, operationsCmd, operationCmd, operationWebsocket, operationWait, sftpCmd, stateCmd, } func api10Get(d *Daemon, r *http.Request) response.Response { srv := api.ServerUntrusted{ APIExtensions: version.APIExtensions, APIStatus: "stable", APIVersion: version.APIVersion, Public: false, Auth: "trusted", AuthMethods: []string{api.AuthenticationMethodTLS}, } env, err := osGetEnvironment() if err != nil { return response.InternalError(err) } fullSrv := api.Server{ServerUntrusted: srv} fullSrv.Environment = *env return response.SyncResponseETag(true, fullSrv, fullSrv) } func setConnectionInfo(d *Daemon, rd io.Reader) error { var data agentAPI.API10Put err := json.NewDecoder(rd).Decode(&data) if err != nil { return err } d.DevIncusMu.Lock() d.serverCID = data.CID d.serverPort = data.Port d.serverCertificate = data.Certificate d.DevIncusEnabled = data.DevIncus d.DevIncusMu.Unlock() return nil } func api10Put(d *Daemon, r *http.Request) response.Response { err := setConnectionInfo(d, r.Body) if err != nil { return response.ErrorResponse(http.StatusInternalServerError, err.Error()) } // Try connecting to the host. client, err := getClient(d.serverCID, int(d.serverPort), d.serverCertificate, d.secretsLocation) if err != nil { return response.ErrorResponse(http.StatusInternalServerError, err.Error()) } server, err := incus.ConnectIncusHTTP(nil, client) if err != nil { return response.ErrorResponse(http.StatusInternalServerError, err.Error()) } defer server.Disconnect() // Let the host know, we were able to connect successfully. d.chConnected <- struct{}{} if d.DevIncusEnabled { err = startDevIncusServer(d) } else { err = stopDevIncusServer(d) } if err != nil { return response.ErrorResponse(http.StatusInternalServerError, err.Error()) } return response.EmptySyncResponse } func startDevIncusServer(d *Daemon) error { if d.Features != nil && !d.Features["guestapi"] { return nil } if !osGuestAPISupport { return nil } d.DevIncusMu.Lock() defer d.DevIncusMu.Unlock() // If a DevIncus server is already running, don't start a second one. if d.DevIncusRunning { return nil } servers["DevIncus"] = devIncusServer(d) // Prepare the DevIncus server. DevIncusListener, err := createDevIncuslListener("/dev") if err != nil { return err } d.DevIncusRunning = true // Start the DevIncus listener. go func() { err := servers["DevIncus"].Serve(DevIncusListener) if err != nil { d.DevIncusMu.Lock() d.DevIncusRunning = false d.DevIncusMu.Unlock() // http.ErrServerClosed can be ignored as this is returned when the server is closed intentionally. if !errors.Is(err, http.ErrServerClosed) { errChan <- err } } }() return nil } func stopDevIncusServer(d *Daemon) error { if !osGuestAPISupport { return nil } d.DevIncusMu.Lock() d.DevIncusRunning = false d.DevIncusMu.Unlock() if servers["DevIncus"] != nil { return servers["DevIncus"].Close() } return nil } func getClient(CID uint32, port int, serverCertificate string, secretsLocation string) (*http.Client, error) { agentCert, err := os.ReadFile(filepath.Join(secretsLocation, "agent.crt")) if err != nil { return nil, err } agentKey, err := os.ReadFile(filepath.Join(secretsLocation, "agent.key")) if err != nil { return nil, err } client, err := localvsock.HTTPClient(CID, port, string(agentCert), string(agentKey), serverCertificate) if err != nil { return nil, err } return client, nil } func startHTTPServer(d *Daemon, debug bool) error { l, err := osGetListener(ports.HTTPSDefaultPort) if err != nil { return fmt.Errorf("Failed to get listener: %w", err) } // Load the expected server certificate. cert, err := localtls.ReadCert(filepath.Join(d.secretsLocation, "server.crt")) if err != nil { return fmt.Errorf("Failed to read client certificate: %w", err) } tlsConfig, err := serverTLSConfig(d.secretsLocation) if err != nil { return fmt.Errorf("Failed to get TLS config: %w", err) } // Prepare the HTTP server. servers["http"] = restServer(tlsConfig, cert, debug, d) // Start the server. go func() { err := servers["http"].Serve(networkTLSListener(l, tlsConfig)) if !errors.Is(err, http.ErrServerClosed) { errChan <- err } _ = l.Close() }() return nil } incus-7.0.0/cmd/incus-agent/config.go000066400000000000000000000007021517523235500174220ustar00rootroot00000000000000package main import ( "errors" "io/fs" "os" "go.yaml.in/yaml/v4" ) type agentConfig struct { Features map[string]bool `yaml:"features"` } func loadAgentConfig(d *Daemon) error { data, err := os.ReadFile(osAgentConfigPath) if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil } return err } cfg := agentConfig{} err = yaml.Load(data, &cfg) if err != nil { return err } d.Features = cfg.Features return nil } incus-7.0.0/cmd/incus-agent/daemon.go000066400000000000000000000016461517523235500174300ustar00rootroot00000000000000package main import ( "sync" "github.com/lxc/incus/v7/internal/server/events" ) // A Daemon can respond to requests from a shared client. type Daemon struct { // Event servers. events *events.Server // Paths. secretsLocation string // Agent config. Features map[string]bool // ContextID and port of the host socket server. serverCID uint32 serverPort uint32 serverCertificate string // The channel which is used to indicate that the agent was able to connect to the host. chConnected chan struct{} DevIncusRunning bool DevIncusMu sync.Mutex DevIncusEnabled bool } // newDaemon returns a new Daemon object with the given configuration. func newDaemon(debug, verbose bool, secretsLocation string) *Daemon { hostEvents := events.NewServer(debug, verbose, nil) return &Daemon{ secretsLocation: secretsLocation, events: hostEvents, chConnected: make(chan struct{}), } } incus-7.0.0/cmd/incus-agent/dev_incus.go000066400000000000000000000222401517523235500201350ustar00rootroot00000000000000package main import ( "fmt" "net" "net/http" "os" "path/filepath" "strings" "time" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/server/daemon" "github.com/lxc/incus/v7/internal/server/device/config" localUtil "github.com/lxc/incus/v7/internal/server/util" api "github.com/lxc/incus/v7/shared/api/guest" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) // DevIncusServer creates an http.Server capable of handling requests against the // /dev/incus Unix socket endpoint created inside VMs. func devIncusServer(d *Daemon) *http.Server { return &http.Server{ Handler: devIncusAPI(d), IdleTimeout: 30 * time.Second, ReadHeaderTimeout: 3 * time.Second, ReadTimeout: 3 * time.Second, } } type devIncusHandler struct { path string /* * This API will have to be changed slightly when we decide to support * websocket events upgrading, but since we don't have events on the * server side right now either, I went the simple route to avoid * needless noise. */ f func(d *Daemon, w http.ResponseWriter, r *http.Request) *devIncusResponse } func getVsockClient(d *Daemon) (incus.InstanceServer, error) { // Try connecting to the host. client, err := getClient(d.serverCID, int(d.serverPort), d.serverCertificate, d.secretsLocation) if err != nil { return nil, err } server, err := incus.ConnectIncusHTTP(nil, client) if err != nil { return nil, err } return server, nil } var DevIncusConfigGet = devIncusHandler{"/1.0/config", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devIncusResponse { client, err := getVsockClient(d) if err != nil { return smartResponse(fmt.Errorf("Failed connecting to the host over vsock: %w", err)) } defer client.Disconnect() resp, _, err := client.RawQuery("GET", "/1.0/config", nil, "") if err != nil { return smartResponse(err) } var conf []string err = resp.MetadataAsStruct(&conf) if err != nil { return smartResponse(fmt.Errorf("Failed parsing response from host: %w", err)) } filtered := []string{} for _, k := range conf { if strings.HasPrefix(k, "/1.0/config/user.") || strings.HasPrefix(k, "/1.0/config/cloud-init.") { filtered = append(filtered, k) } } return okResponse(filtered, "json") }} var DevIncusConfigKeyGet = devIncusHandler{"/1.0/config/{key}", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devIncusResponse { key := r.PathValue("key") if key == "" { return &devIncusResponse{"bad request", http.StatusBadRequest, "raw"} } if !strings.HasPrefix(key, "user.") && !strings.HasPrefix(key, "cloud-init.") { return &devIncusResponse{"not authorized", http.StatusForbidden, "raw"} } client, err := getVsockClient(d) if err != nil { return smartResponse(fmt.Errorf("Failed connecting to host over vsock: %w", err)) } defer client.Disconnect() resp, _, err := client.RawQuery("GET", fmt.Sprintf("/1.0/config/%s", key), nil, "") if err != nil { return smartResponse(err) } var value string err = resp.MetadataAsStruct(&value) if err != nil { return smartResponse(fmt.Errorf("Failed parsing response from host: %w", err)) } return okResponse(value, "raw") }} var DevIncusMetadataGet = devIncusHandler{"/1.0/meta-data", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devIncusResponse { var client incus.InstanceServer var err error for range 10 { client, err = getVsockClient(d) if err == nil { break } time.Sleep(500 * time.Millisecond) } if err != nil { return smartResponse(fmt.Errorf("Failed connecting to host over vsock: %w", err)) } defer client.Disconnect() resp, _, err := client.RawQuery("GET", "/1.0/meta-data", nil, "") if err != nil { return smartResponse(err) } var metaData string err = resp.MetadataAsStruct(&metaData) if err != nil { return smartResponse(fmt.Errorf("Failed parsing response from host: %w", err)) } return okResponse(metaData, "raw") }} var devIncusEventsGet = devIncusHandler{"/1.0/events", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devIncusResponse { err := eventsGet(d, r).Render(w) if err != nil { return smartResponse(err) } return okResponse("", "raw") }} var DevIncusAPIGet = devIncusHandler{"/1.0", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devIncusResponse { client, err := getVsockClient(d) if err != nil { return smartResponse(fmt.Errorf("Failed connecting to host over vsock: %w", err)) } defer client.Disconnect() switch r.Method { case "GET": resp, _, err := client.RawQuery(r.Method, "/1.0", nil, "") if err != nil { return smartResponse(err) } var instanceData api.DevIncusGet err = resp.MetadataAsStruct(&instanceData) if err != nil { return smartResponse(fmt.Errorf("Failed parsing response from host: %w", err)) } return okResponse(instanceData, "json") case "PATCH": _, _, err := client.RawQuery(r.Method, "/1.0", r.Body, "") if err != nil { return smartResponse(err) } return okResponse("", "raw") default: return &devIncusResponse{fmt.Sprintf("method %q not allowed", r.Method), http.StatusBadRequest, "raw"} } }} var DevIncusDevicesGet = devIncusHandler{"/1.0/devices", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devIncusResponse { client, err := getVsockClient(d) if err != nil { return smartResponse(fmt.Errorf("Failed connecting to host over vsock: %w", err)) } defer client.Disconnect() resp, _, err := client.RawQuery("GET", "/1.0/devices", nil, "") if err != nil { return smartResponse(err) } var devices config.Devices err = resp.MetadataAsStruct(&devices) if err != nil { return smartResponse(fmt.Errorf("Failed parsing response from host: %w", err)) } return okResponse(devices, "json") }} var handlers = []devIncusHandler{ {"/{$}", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devIncusResponse { return okResponse([]string{"/1.0"}, "json") }}, DevIncusAPIGet, DevIncusConfigGet, DevIncusConfigKeyGet, DevIncusMetadataGet, devIncusEventsGet, DevIncusDevicesGet, } func hoistReq(f func(*Daemon, http.ResponseWriter, *http.Request) *devIncusResponse, d *Daemon) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { resp := f(d, w, r) if resp.code != http.StatusOK { http.Error(w, fmt.Sprintf("%s", resp.content), resp.code) } else if resp.ctype == "json" { w.Header().Set("Content-Type", "application/json") var debugLogger logger.Logger if daemon.Debug { debugLogger = logger.Logger(logger.Log) } _ = localUtil.WriteJSON(w, resp.content, debugLogger) } else if resp.ctype != "websocket" { w.Header().Set("Content-Type", "application/octet-stream") _, _ = fmt.Fprint(w, resp.content.(string)) } } } func devIncusAPI(d *Daemon) http.Handler { router := http.NewServeMux() for _, handler := range handlers { router.HandleFunc(handler.path, hoistReq(handler.f, d)) } return router } // Create a new net.Listener bound to the unix socket of the DevIncus endpoint. func createDevIncuslListener(dir string) (net.Listener, error) { path := filepath.Join(dir, "incus", "sock") err := os.MkdirAll(filepath.Dir(path), 0o755) if err != nil { return nil, err } // Add a symlink for legacy support. err = os.Symlink(filepath.Join(dir, "incus"), filepath.Join(dir, "lxd")) if err != nil && !os.IsExist(err) { return nil, err } // If this socket exists, that means a previous agent instance died and // didn't clean up. We assume that such agent instance is actually dead // if we get this far, since localCreateListener() tries to connect to // the actual incus socket to make sure that it is actually dead. So, it // is safe to remove it here without any checks. // // Also, it would be nice to SO_REUSEADDR here so we don't have to // delete the socket, but we can't: // http://stackoverflow.com/questions/15716302/so-reuseaddr-and-af-unix // // Note that this will force clients to reconnect when the daemon is restarted. err = socketUnixRemoveStale(path) if err != nil { return nil, err } listener, err := socketUnixListen(path) if err != nil { return nil, err } err = socketUnixSetPermissions(path, 0o600) if err != nil { _ = listener.Close() return nil, err } return listener, nil } // Remove any stale socket file at the given path. func socketUnixRemoveStale(path string) error { // If there's no socket file at all, there's nothing to do. if !util.PathExists(path) { return nil } logger.Debugf("Detected stale unix socket, deleting") err := os.Remove(path) if err != nil { return fmt.Errorf("could not delete stale local socket: %w", err) } return nil } // Change the file mode of the given unix socket file. func socketUnixSetPermissions(path string, mode os.FileMode) error { err := os.Chmod(path, mode) if err != nil { return fmt.Errorf("cannot set permissions on local socket: %w", err) } return nil } // Bind to the given unix socket path. func socketUnixListen(path string) (net.Listener, error) { addr, err := net.ResolveUnixAddr("unix", path) if err != nil { return nil, fmt.Errorf("cannot resolve socket address: %w", err) } listener, err := net.ListenUnix("unix", addr) if err != nil { return nil, fmt.Errorf("cannot bind socket: %w", err) } return listener, err } incus-7.0.0/cmd/incus-agent/events.go000066400000000000000000000076561517523235500175000ustar00rootroot00000000000000package main import ( "encoding/json" "errors" "net/http" "strings" "time" "github.com/lxc/incus/v7/internal/server/events" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/ws" ) var eventsCmd = APIEndpoint{ Path: "events", Get: APIEndpointAction{Handler: eventsGet}, Post: APIEndpointAction{Handler: eventsPost}, } type eventsServe struct { req *http.Request d *Daemon } func (r *eventsServe) Render(w http.ResponseWriter) error { return eventsSocket(r.d, r.req, w) } func (r *eventsServe) String() string { return "event handler" } // Code returns the HTTP code. func (r *eventsServe) Code() int { return http.StatusOK } func eventsSocket(d *Daemon, r *http.Request, w http.ResponseWriter) error { typeStr := r.FormValue("type") if typeStr == "" { // We add 'config' here to allow listeners on /dev/incus/sock to receive config changes. typeStr = "logging,operation,lifecycle,config,device" } var listenerConnection events.EventListenerConnection // If the client has not requested a websocket connection then fallback to long polling event stream mode. if r.Header.Get("Upgrade") == "websocket" { // Upgrade the connection to websocket conn, err := ws.Upgrader.Upgrade(w, r, nil) if err != nil { return err } defer func() { _ = conn.Close() }() // Ensure listener below ends when this function ends. listenerConnection = events.NewWebsocketListenerConnection(conn) } else { h, ok := w.(http.Hijacker) if !ok { return errors.New("Missing implemented http.Hijacker interface") } conn, _, err := h.Hijack() if err != nil { return err } defer func() { _ = conn.Close() }() // Ensure listener below ends when this function ends. listenerConnection, err = events.NewStreamListenerConnection(conn) if err != nil { return err } } // As we don't know which project we are in, subscribe to events from all projects. listener, err := d.events.AddListener("", true, nil, listenerConnection, strings.Split(typeStr, ","), nil, nil, nil) if err != nil { return err } listener.Wait(r.Context()) return nil } func eventsGet(d *Daemon, r *http.Request) response.Response { return &eventsServe{req: r, d: d} } func eventsPost(d *Daemon, r *http.Request) response.Response { var event api.Event err := json.NewDecoder(r.Body).Decode(&event) if err != nil { return response.InternalError(err) } err = d.events.Send("", event.Type, event.Metadata) if err != nil { return response.InternalError(err) } // Handle device related actions locally. go eventsProcess(d, event) return response.SyncResponse(true, nil) } func eventsProcess(d *Daemon, event api.Event) { // As we only handle mounts, skip if disabled. if d.Features != nil && !d.Features["mounts"] { return } // We currently only need to react to device events. if event.Type != "device" { return } type deviceEvent struct { Action string `json:"action"` Config map[string]string `json:"config"` Name string `json:"name"` } e := deviceEvent{} err := json.Unmarshal(event.Metadata, &e) if err != nil { return } // We only handle disk hotplug. if e.Config["type"] != "disk" { return } // And only for path based devices. if e.Config["path"] == "" { return } mntSource := "incus_" + e.Name if e.Action == "added" { // Attempt to perform the mount. for range 20 { time.Sleep(500 * time.Millisecond) err = osMountShared(mntSource, e.Config["path"], "virtiofs", nil) if err == nil { break } } if err != nil { logger.Infof("Failed to mount hotplug %q (Type: %q) to %q", mntSource, "virtiofs", e.Config["path"]) return } logger.Infof("Mounted hotplug %q (Type: %q) to %q", mntSource, "virtiofs", e.Config["path"]) } else if e.Action == "removed" { // Attempt to unmount the disk. _ = osUmount(mntSource, e.Config["path"], "virtiofs") } } incus-7.0.0/cmd/incus-agent/exec.go000066400000000000000000000235531517523235500171120ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "io" "io/fs" "maps" "net/http" "os" "os/exec" "strconv" "sync" "time" "github.com/gorilla/websocket" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/response" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/ws" ) const ( execWSControl = -1 execWSStdin = 0 execWSStdout = 1 execWSStderr = 2 ) var execCmd = APIEndpoint{ Name: "exec", Path: "exec", Post: APIEndpointAction{Handler: execPost}, } func execPost(d *Daemon, r *http.Request) response.Response { if d.Features != nil && !d.Features["exec"] { return response.Forbidden(errors.New("Command execution has been disabled by configuration")) } post := api.InstanceExecPost{} buf, err := io.ReadAll(r.Body) if err != nil { return response.BadRequest(err) } err = json.Unmarshal(buf, &post) if err != nil { return response.BadRequest(err) } if !post.WaitForWS { return response.BadRequest(errors.New("Websockets are required for VM exec")) } env := map[string]string{} if post.Environment != nil { maps.Copy(env, post.Environment) } osSetEnv(&post, env) webSocket := &execWs{} webSocket.fds = map[int]string{} webSocket.conns = map[int]*websocket.Conn{} webSocket.conns[execWSControl] = nil webSocket.conns[0] = nil // This is used for either TTY or Stdin. if !post.Interactive { webSocket.conns[execWSStdout] = nil webSocket.conns[execWSStderr] = nil } webSocket.requiredConnectedCtx, webSocket.requiredConnectedDone = context.WithCancel(context.Background()) webSocket.interactive = post.Interactive for i := range webSocket.conns { webSocket.fds[i], err = internalUtil.RandomHexString(32) if err != nil { return response.InternalError(err) } } webSocket.command = post.Command webSocket.env = env webSocket.width = post.Width webSocket.height = post.Height webSocket.cwd = post.Cwd webSocket.uid = post.User webSocket.gid = post.Group resources := map[string][]api.URL{} op, err := operations.OperationCreate(nil, "", operations.OperationClassWebsocket, operationtype.CommandExec, resources, webSocket.Metadata(), webSocket.Do, nil, webSocket.Connect, r) if err != nil { return response.InternalError(err) } // Link the operation to the agent's event server. op.SetEventServer(d.events) return operations.OperationResponse(op) } type execWs struct { command []string env map[string]string conns map[int]*websocket.Conn connsLock sync.Mutex requiredConnectedCtx context.Context requiredConnectedDone func() interactive bool fds map[int]string width int height int uid uint32 gid uint32 cwd string } func (s *execWs) Metadata() any { fds := jmap.Map{} for fd, secret := range s.fds { if fd == execWSControl { fds[api.SecretNameControl] = secret } else { fds[strconv.Itoa(fd)] = secret } } return jmap.Map{ "fds": fds, "command": s.command, "environment": s.env, "interactive": s.interactive, } } func (s *execWs) Connect(op *operations.Operation, r *http.Request, w http.ResponseWriter) error { secret := r.FormValue("secret") if secret == "" { return errors.New("missing secret") } for fd, fdSecret := range s.fds { if secret == fdSecret { conn, err := ws.Upgrader.Upgrade(w, r, nil) if err != nil { return err } s.connsLock.Lock() defer s.connsLock.Unlock() //nolint:revive val, found := s.conns[fd] if found && val == nil { s.conns[fd] = conn for _, c := range s.conns { if c == nil { return nil // Not all required connections connected yet. } } s.requiredConnectedDone() // All required connections now connected. return nil } else if !found { return errors.New("Unknown websocket number") } return errors.New("Websocket number already connected") } } /* If we didn't find the right secret, the user provided a bad one, * which 403, not 404, since this Operation actually exists */ return os.ErrPermission } func (s *execWs) Do(op *operations.Operation) error { // Once this function ends ensure that any connected websockets are closed. defer func() { s.connsLock.Lock() for i := range s.conns { if s.conns[i] != nil { _ = s.conns[i].Close() } } s.connsLock.Unlock() }() // As this function only gets called when the exec request has WaitForWS enabled, we expect the client to // connect to all of the required websockets within a short period of time and we won't proceed until then. logger.Debug("Waiting for exec websockets to connect") select { case <-s.requiredConnectedCtx.Done(): //nolint:revive //whyNoLint: this is intentional, the flow should continue if all websockets are connected break case <-time.After(time.Second * 5): return errors.New("Timed out waiting for websockets to connect") } var err error var ttys []io.ReadWriteCloser var ptys []io.ReadWriteCloser var stdin io.ReadCloser var stdout io.WriteCloser var stderr io.WriteCloser if s.interactive { ttys = make([]io.ReadWriteCloser, 1) ptys = make([]io.ReadWriteCloser, 1) ptys[0], ttys[0], err = osGetInteractiveConsole(s) if err != nil { return err } stdin = ttys[0] stdout = ttys[0] stderr = ttys[0] } else { ttys = make([]io.ReadWriteCloser, 3) ptys = make([]io.ReadWriteCloser, 3) for i := range ttys { ptys[i], ttys[i], err = os.Pipe() if err != nil { return err } } stdin = ptys[execWSStdin] stdout = ttys[execWSStdout] stderr = ttys[execWSStderr] } ctxCommand, cancel := context.WithCancel(context.Background()) waitAttachedChildIsDead, markAttachedChildIsDead := context.WithCancel(context.Background()) var wgEOF sync.WaitGroup finisher := func(cmdResult int, cmdErr error) error { // Cancel the context after we're done with cleanup. defer cancel() // Cancel this before closing the control connection so control handler can detect command ending. markAttachedChildIsDead() for _, tty := range ttys { _ = tty.Close() } s.connsLock.Lock() conn := s.conns[-1] s.connsLock.Unlock() if conn != nil { _ = conn.Close() // Close control connection (will cause control go routine to end). } wgEOF.Wait() for _, pty := range ptys { _ = pty.Close() } metadata := jmap.Map{"return": cmdResult} err = op.UpdateMetadata(metadata) if err != nil { return err } return cmdErr } var cmd *exec.Cmd if len(s.command) > 1 { cmd = exec.CommandContext(ctxCommand, s.command[0], s.command[1:]...) } else { cmd = exec.CommandContext(ctxCommand, s.command[0]) } // Prepare the environment for k, v := range s.env { cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) } cmd.Stdin = stdin cmd.Stdout = stdout cmd.Stderr = stderr cmd.Dir = s.cwd osPrepareExecCommand(s, cmd) err = cmd.Start() if err != nil { exitStatus := -1 if errors.Is(err, exec.ErrNotFound) || errors.Is(err, fs.ErrNotExist) { exitStatus = 127 } else if errors.Is(err, fs.ErrPermission) { exitStatus = 126 } return finisher(exitStatus, err) } l := logger.AddContext(logger.Ctx{"PID": cmd.Process.Pid, "interactive": s.interactive}) l.Debug("Instance process started") wgEOF.Add(1) go func() { defer wgEOF.Done() l.Debug("Exec control handler started") defer l.Debug("Exec control handler finished") s.connsLock.Lock() conn := s.conns[-1] s.connsLock.Unlock() for { mt, r, err := conn.NextReader() if err != nil || mt == websocket.CloseMessage { // Check if command process has finished normally, if so, no need to kill it. if waitAttachedChildIsDead.Err() != nil { return } if mt == websocket.CloseMessage { l.Warn("Got exec control websocket close message, killing command") } else { l.Warn("Failed getting exec control websocket reader, killing command", logger.Ctx{"err": err}) } cancel() return } buf, err := io.ReadAll(r) if err != nil { // Check if command process has finished normally, if so, no need to kill it. if waitAttachedChildIsDead.Err() != nil { return } l.Warn("Failed reading control websocket message, killing command", logger.Ctx{"err": err}) return } control := api.InstanceExecControl{} err = json.Unmarshal(buf, &control) if err != nil { l.Debug("Failed to unmarshal control socket command", logger.Ctx{"err": err}) continue } osHandleExecControl(control, s, ptys[0], cmd, l) } }() if s.interactive { wgEOF.Add(1) go func() { defer wgEOF.Done() l.Debug("Exec mirror websocket started", logger.Ctx{"number": 0}) defer l.Debug("Exec mirror websocket finished", logger.Ctx{"number": 0}) s.connsLock.Lock() conn := s.conns[0] s.connsLock.Unlock() readDone, writeDone := ws.Mirror(conn, osExecWrapper(waitAttachedChildIsDead, ptys[0])) <-readDone <-writeDone _ = conn.Close() }() } else { wgEOF.Add(len(ttys) - 1) for i := range ttys { go func(i int) { l.Debug("Exec mirror websocket started", logger.Ctx{"number": i}) defer l.Debug("Exec mirror websocket finished", logger.Ctx{"number": i}) if i == 0 { s.connsLock.Lock() conn := s.conns[i] s.connsLock.Unlock() <-ws.MirrorWrite(conn, ttys[i]) _ = ttys[i].Close() } else { s.connsLock.Lock() conn := s.conns[i] s.connsLock.Unlock() <-ws.MirrorRead(conn, ptys[i]) _ = ptys[i].Close() wgEOF.Done() } }(i) } } exitStatus, err := osExitStatus(cmd.Wait()) l.Debug("Instance process stopped", logger.Ctx{"err": err, "exitStatus": exitStatus}) return finisher(exitStatus, nil) } incus-7.0.0/cmd/incus-agent/main.go000066400000000000000000000030621517523235500171030ustar00rootroot00000000000000//go:debug httpmuxgo121=0 package main import ( "fmt" "os" "runtime" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/version" ) type cmdGlobal struct { flagVersion bool flagHelp bool flagService bool flagSecretsLocation string flagLogVerbose bool flagLogDebug bool } func main() { // agent command (main) agentCmd := cmdAgent{} app := agentCmd.command() app.SilenceUsage = true app.CompletionOptions = cobra.CompletionOptions{DisableDefaultCmd: true} // Workaround for main command app.Args = cobra.ArbitraryArgs // Global flags globalCmd := cmdGlobal{} agentCmd.global = &globalCmd app.PersistentFlags().BoolVar(&globalCmd.flagVersion, "version", false, "Print version number") app.PersistentFlags().BoolVarP(&globalCmd.flagHelp, "help", "h", false, "Print help") app.PersistentFlags().BoolVarP(&globalCmd.flagLogVerbose, "verbose", "v", false, "Show all information messages") app.PersistentFlags().BoolVarP(&globalCmd.flagLogDebug, "debug", "d", false, "Show all debug messages") app.PersistentFlags().StringVarP(&globalCmd.flagSecretsLocation, "secrets-location", "s", "", "Secrets location of the certificate and private key") if runtime.GOOS == "windows" { app.PersistentFlags().BoolVar(&globalCmd.flagService, "service", false, "Start as a system service") } // Version handling app.SetVersionTemplate("{{.Version}}\n") app.Version = version.Version // Run the main command and handle errors err := app.Execute() if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) os.Exit(1) } } incus-7.0.0/cmd/incus-agent/main_agent.go000066400000000000000000000167441517523235500202740ustar00rootroot00000000000000package main import ( "context" "encoding/json" "fmt" "net/http" "os" "os/signal" "slices" "sync" "time" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) var ( servers = make(map[string]*http.Server, 2) errChan = make(chan error) ) type cmdAgent struct { global *cmdGlobal } func (c *cmdAgent) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "incus-agent [--debug]" cmd.Short = "Incus virtual machine agent" cmd.Long = `Description: Incus virtual machine agent This daemon is to be run inside virtual machines managed by Incus. It will normally be started through init scripts present or injected into the virtual machine. ` cmd.RunE = c.run return cmd } func (c *cmdAgent) run(cmd *cobra.Command, args []string) error { if c.global.flagService { return runService("Incus-Agent", c) } // Setup logger. err := logger.InitLogger("", "", c.global.flagLogVerbose, c.global.flagLogDebug, nil) if err != nil { return fmt.Errorf("could not initialize logger: %w", err) } logger.Info("Starting") defer logger.Info("Stopped") // Apply the templated files. files, err := templatesApply("files/") if err != nil { return err } // Sync the hostname. if util.PathExists("/proc/sys/kernel/hostname") && slices.Contains(files, "/etc/hostname") { // Open the two files. src, err := os.Open("/etc/hostname") if err != nil { return err } dst, err := os.Create("/proc/sys/kernel/hostname") if err != nil { return err } // Copy the data. _, err = util.SafeCopy(dst, src) if err != nil { return err } // Close the files. _ = src.Close() err = dst.Close() if err != nil { return err } } else if util.PathExists("/etc/rc.conf.d/hostname") && slices.Contains(files, "/etc/rc.conf.d/hostname") { // Set the hostname. _, err := subprocess.RunCommand("service", "hostname", "restart") if err != nil { return err } } // Run cloud-init. if util.PathExists("/etc/cloud") && slices.Contains(files, "/var/lib/cloud/seed/nocloud-net/meta-data") { logger.Info("Seeding cloud-init") cloudInitPath := "/run/cloud-init" if util.PathExists(cloudInitPath) { logger.Info(fmt.Sprintf("Removing %q", cloudInitPath)) err = os.RemoveAll(cloudInitPath) if err != nil { return err } } logger.Info("Rebooting") _, _ = subprocess.RunCommand("reboot") // Wait up to 5min for the reboot to actually happen, if it doesn't, then move on to allowing connections. time.Sleep(300 * time.Second) } osReconfigureNetworkInterfaces() // Load the kernel driver. err = osLoadModules() if err != nil { return err } d := newDaemon(c.global.flagLogDebug, c.global.flagLogVerbose, c.global.flagSecretsLocation) // Load the agent configuration. err = loadAgentConfig(d) if err != nil { return err } // Mount shares from host. if d.Features == nil || d.Features["mounts"] { c.mountHostShares() } // Start the server. err = startHTTPServer(d, c.global.flagLogDebug) if err != nil { return fmt.Errorf("Failed to start HTTP server: %w", err) } // Check whether we should start the DevIncus server in the early setup. This way, /dev/incus/sock // will be available for any systemd services starting after the agent. if util.PathExists("agent.conf") { f, err := os.Open("agent.conf") if err != nil { return err } err = setConnectionInfo(d, f) if err != nil { _ = f.Close() return err } _ = f.Close() if d.DevIncusEnabled { err = startDevIncusServer(d) if err != nil { return err } } } // Create a cancellation context. ctx, cancelFunc := context.WithCancel(context.Background()) // Start status notifier in background. cancelStatusNotifier := c.startStatusNotifier(ctx, d.chConnected) // Done with early setup, tell systemd to continue boot. // Allows a service that needs a file that's generated by the agent to be able to declare After=incus-agent // and know the file will have been created by the time the service is started. if os.Getenv("NOTIFY_SOCKET") != "" { _, err := subprocess.RunCommand("systemd-notify", "READY=1") if err != nil { cancelStatusNotifier() // Ensure STOPPED status is written to QEMU status ringbuffer. cancelFunc() return fmt.Errorf("Failed to notify systemd of readiness: %w", err) } } // Cancel context on shutdown signal. chSignal := make(chan os.Signal, 1) signal.Notify(chSignal, osShutdownSignal) select { case <-chSignal: case err := <-errChan: cancelStatusNotifier() // Ensure STOPPED status is written to QEMU status ringbuffer. cancelFunc() return err } cancelStatusNotifier() // Ensure STOPPED status is written to QEMU status ringbuffer. cancelFunc() return nil } // startStatusNotifier sends status of agent to vserial ring buffer every 10s or when context is done. // Returns a function that can be used to update the running status to STOPPED in the ring buffer. func (c *cmdAgent) startStatusNotifier(ctx context.Context, chConnected <-chan struct{}) context.CancelFunc { // Write initial started status. _ = c.writeStatus("STARTED") wg := sync.WaitGroup{} exitCtx, exit := context.WithCancel(ctx) // Allows manual synchronous cancellation via cancel function. cancel := func() { exit() // Signal for the go routine to end. wg.Wait() // Wait for the go routine to actually finish. } wg.Add(1) go func() { defer wg.Done() // Signal to cancel function that we are done. ticker := time.NewTicker(time.Duration(time.Second) * 5) defer ticker.Stop() for { select { case <-chConnected: _ = c.writeStatus("CONNECTED") // Indicate we were able to connect. case <-ticker.C: _ = c.writeStatus("STARTED") // Re-populate status periodically in case the daemon restarts. case <-exitCtx.Done(): _ = c.writeStatus("STOPPED") // Indicate we are stopping and exit go routine. return } } }() return cancel } // writeStatus writes a status code to the vserial ring buffer used to detect agent status on host. func (c *cmdAgent) writeStatus(status string) error { if util.PathExists(osVioSerialPath) { vSerial, err := os.OpenFile(osVioSerialPath, os.O_RDWR, 0o600) if err != nil { return err } defer vSerial.Close() _, err = vSerial.Write(fmt.Appendf(nil, "%s\n", status)) if err != nil { return err } } return nil } // mountHostShares reads the agent-mounts.json file from config share and mounts the shares requested. func (c *cmdAgent) mountHostShares() { agentMountsFile := "./agent-mounts.json" if !util.PathExists(agentMountsFile) { return } b, err := os.ReadFile(agentMountsFile) if err != nil { logger.Errorf("Failed to load agent mounts file %q: %v", agentMountsFile, err) } var agentMounts []instancetype.VMAgentMount err = json.Unmarshal(b, &agentMounts) if err != nil { logger.Errorf("Failed to parse agent mounts file %q: %v", agentMountsFile, err) return } for _, mount := range agentMounts { if !slices.Contains([]string{"9p", "virtiofs"}, mount.FSType) { logger.Infof("Unsupported mount fstype %q", mount.FSType) continue } err = osMountShared(mount.Source, mount.Target, mount.FSType, mount.Options) if err != nil { logger.Infof("Failed to mount %q (Type: %q, Options: %v) to %q: %v", mount.Source, "virtiofs", mount.Options, mount.Target, err) continue } logger.Infof("Mounted %q (Type: %q, Options: %v) to %q", mount.Source, mount.FSType, mount.Options, mount.Target) } } incus-7.0.0/cmd/incus-agent/metrics.go000066400000000000000000000047621517523235500176350ustar00rootroot00000000000000package main import ( "errors" "net/http" "time" "github.com/shirou/gopsutil/v4/host" "github.com/lxc/incus/v7/internal/server/metrics" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/shared/logger" ) var metricsCmd = APIEndpoint{ Path: "metrics", Get: APIEndpointAction{Handler: metricsGet}, } func metricsGet(d *Daemon, r *http.Request) response.Response { if d.Features != nil && !d.Features["metrics"] { return response.Forbidden(errors.New("Guest metrics are disabled by configuration")) } if !osMetricsSupported { return response.NotFound(nil) } out := metrics.Metrics{} diskStats, err := osGetDiskMetrics(d) if err != nil { logger.Warn("Failed to get disk metrics", logger.Ctx{"err": err}) } else { out.Disk = diskStats } filesystemStats, err := osGetFilesystemMetrics(d) if err != nil { logger.Warn("Failed to get filesystem metrics", logger.Ctx{"err": err}) } else { out.Filesystem = filesystemStats } memStats, err := osGetMemoryMetrics(d) if err != nil { logger.Warn("Failed to get memory metrics", logger.Ctx{"err": err}) } else { out.Memory = memStats } netStats, err := getNetworkMetrics(d) if err != nil { logger.Warn("Failed to get network metrics", logger.Ctx{"err": err}) } else { out.Network = netStats } out.ProcessesTotal = uint64(osGetProcessesState()) out.BootTimeSeconds = uint64(osGetBootTime()) out.TimeSeconds = uint64(time.Now().Unix()) cpuStats, err := osGetCPUMetrics(d) if err != nil { logger.Warn("Failed to get CPU metrics", logger.Ctx{"err": err}) } else { out.CPU = cpuStats } return response.SyncResponse(true, &out) } func getNetworkMetrics(d *Daemon) ([]metrics.NetworkMetrics, error) { out := []metrics.NetworkMetrics{} for dev, state := range osGetNetworkState() { stats := metrics.NetworkMetrics{} stats.ReceiveBytes = uint64(state.Counters.BytesReceived) stats.ReceiveDrop = uint64(state.Counters.PacketsDroppedInbound) stats.ReceiveErrors = uint64(state.Counters.ErrorsReceived) stats.ReceivePackets = uint64(state.Counters.PacketsReceived) stats.TransmitBytes = uint64(state.Counters.BytesSent) stats.TransmitDrop = uint64(state.Counters.PacketsDroppedOutbound) stats.TransmitErrors = uint64(state.Counters.ErrorsSent) stats.TransmitPackets = uint64(state.Counters.PacketsSent) stats.Device = dev out = append(out, stats) } return out, nil } func osGetBootTime() int64 { bootTime, err := host.BootTime() if err != nil { return -1 } return int64(bootTime) } incus-7.0.0/cmd/incus-agent/network.go000066400000000000000000000022631517523235500176520ustar00rootroot00000000000000package main import ( "crypto/tls" "net" "sync" "github.com/lxc/incus/v7/internal/server/util" localtls "github.com/lxc/incus/v7/shared/tls" ) // A variation of the standard tls.Listener that supports atomically swapping // the underlying TLS configuration. Requests served before the swap will // continue using the old configuration. type networkListener struct { net.Listener mu sync.RWMutex config *tls.Config } func networkTLSListener(inner net.Listener, config *tls.Config) *networkListener { listener := &networkListener{ Listener: inner, config: config, } return listener } // Accept waits for and returns the next incoming TLS connection then use the // current TLS configuration to handle it. func (l *networkListener) Accept() (net.Conn, error) { c, err := l.Listener.Accept() if err != nil { return nil, err } l.mu.RLock() defer l.mu.RUnlock() return tls.Server(c, l.config), nil } func serverTLSConfig(secretsLocation string) (*tls.Config, error) { certInfo, err := localtls.KeyPairAndCA(secretsLocation, "agent", localtls.CertServer, false) if err != nil { return nil, err } tlsConfig := util.ServerTLSConfig(certInfo) return tlsConfig, nil } incus-7.0.0/cmd/incus-agent/operations.go000066400000000000000000000111531517523235500203420ustar00rootroot00000000000000package main import ( "context" "fmt" "log" "net/http" "strconv" "strings" "time" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/response" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/api" ) var operationCmd = APIEndpoint{ Path: "operations/{id}", Delete: APIEndpointAction{Handler: operationDelete}, Get: APIEndpointAction{Handler: operationGet}, } var operationsCmd = APIEndpoint{ Path: "operations", Get: APIEndpointAction{Handler: operationsGet}, } var operationWebsocket = APIEndpoint{ Path: "operations/{id}/websocket", Get: APIEndpointAction{Handler: operationWebsocketGet}, } var operationWait = APIEndpoint{ Path: "operations/{id}/wait", Get: APIEndpointAction{Handler: operationWaitGet}, } func operationDelete(d *Daemon, r *http.Request) response.Response { id := r.PathValue("id") if id == "" { return response.BadRequest(fmt.Errorf("Failed to extract operation ID from URL")) } // First check if the query is for a local operation from this node op, err := operations.OperationGetInternal(id) if err != nil { return response.SmartError(err) } _, err = op.Cancel() if err != nil { return response.BadRequest(err) } return response.EmptySyncResponse } func operationGet(d *Daemon, r *http.Request) response.Response { id := r.PathValue("id") if id == "" { return response.BadRequest(fmt.Errorf("Failed to extract operation ID from URL")) } var body *api.Operation // First check if the query is for a local operation from this node op, err := operations.OperationGetInternal(id) if err != nil { return response.SmartError(err) } _, body, err = op.Render() if err != nil { log.Println(fmt.Errorf("Failed to handle operations request: %w", err)) } return response.SyncResponse(true, body) } func operationsGet(d *Daemon, r *http.Request) response.Response { recursion := localUtil.IsRecursionRequest(r) localOperationURLs := func() (jmap.Map, error) { // Get all the operations ops := operations.Clone() // Build a list of URLs body := jmap.Map{} for _, v := range ops { status := strings.ToLower(v.Status().String()) _, ok := body[status] if !ok { body[status] = make([]string, 0) } body[status] = append(body[status].([]string), v.URL()) } return body, nil } localOperations := func() (jmap.Map, error) { // Get all the operations ops := operations.Clone() // Build a list of operations body := jmap.Map{} for _, v := range ops { status := strings.ToLower(v.Status().String()) _, ok := body[status] if !ok { body[status] = make([]*api.Operation, 0) } _, op, err := v.Render() if err != nil { return nil, err } body[status] = append(body[status].([]*api.Operation), op) } return body, nil } // Start with local operations var md jmap.Map var err error if recursion { md, err = localOperations() if err != nil { return response.InternalError(err) } } else { md, err = localOperationURLs() if err != nil { return response.InternalError(err) } } return response.SyncResponse(true, md) } func operationWebsocketGet(d *Daemon, r *http.Request) response.Response { id := r.PathValue("id") if id == "" { return response.BadRequest(fmt.Errorf("Failed to extract operation ID from URL")) } // First check if the query is for a local operation from this node op, err := operations.OperationGetInternal(id) if err != nil { return response.SmartError(err) } return operations.OperationWebSocket(r, op) } func operationWaitGet(d *Daemon, r *http.Request) response.Response { id := r.PathValue("id") if id == "" { return response.BadRequest(fmt.Errorf("Failed to extract operation ID from URL")) } var err error var timeoutSecs int timeout := r.FormValue("timeout") if timeout != "" { timeoutSecs, err = strconv.Atoi(timeout) if err != nil { return response.InternalError(fmt.Errorf("Failed to extract operation wait timeout from URL: %w", err)) } } else { timeoutSecs = -1 } var ctx context.Context var cancel context.CancelFunc if timeoutSecs > -1 { ctx, cancel = context.WithDeadline(r.Context(), time.Now().Add(time.Second*time.Duration(timeoutSecs))) } else { ctx, cancel = context.WithCancel(r.Context()) } defer cancel() op, err := operations.OperationGetInternal(id) if err != nil { return response.NotFound(err) } err = op.Wait(ctx) if err != nil { return response.SmartError(err) } _, opAPI, err := op.Render() if err != nil { return response.SmartError(err) } return response.SyncResponse(true, opAPI) } incus-7.0.0/cmd/incus-agent/os_bsd.go000066400000000000000000000110751517523235500174330ustar00rootroot00000000000000//go:build darwin || freebsd package main import ( "bytes" "errors" "fmt" "io" "net" "os" "os/exec" "strconv" "syscall" "unsafe" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) var ( osBaseWorkingDirectory = "/" osAgentConfigPath = "/usr/local/etc/incus-agent.yml" osVioSerialPath = "/dev/virtio-ports/org.linuxcontainers.incus" ) func runService(name string, agentCmd *cmdAgent) error { return errors.New("Not implemented.") } func parseBytes(b []byte) string { n := bytes.IndexByte(b, 0) if n < 0 { n = len(b) } return string(b[:n]) } func osGetEnvironment() (*api.ServerEnvironment, error) { uname := unix.Utsname{} err := unix.Uname(&uname) if err != nil { return nil, err } env := &api.ServerEnvironment{ Kernel: parseBytes(uname.Sysname[:]), KernelArchitecture: parseBytes(uname.Machine[:]), KernelVersion: parseBytes(uname.Release[:]), Server: "incus-agent", ServerPid: os.Getpid(), ServerVersion: version.Version, ServerName: parseBytes(uname.Nodename[:]), } return env, nil } // setPtySize is the same as linux.SetPtySize for BSD-likes. func setPtySize(fd int, width int, height int) (err error) { var dimensions [4]uint16 dimensions[0] = uint16(height) dimensions[1] = uint16(width) _, _, errno := unix.Syscall6(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.TIOCSWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0) if errno != 0 { return errno } return nil } func osGetInteractiveConsole(s *execWs) (*os.File, *os.File, error) { pty, tty, err := openPty(int64(s.uid), int64(s.gid)) if err != nil { return nil, nil, err } if s.width > 0 && s.height > 0 { _ = setPtySize(int(pty.Fd()), s.width, s.height) } return pty, tty, nil } func osPrepareExecCommand(s *execWs, cmd *exec.Cmd) { cmd.SysProcAttr = &syscall.SysProcAttr{ Credential: &syscall.Credential{ Uid: s.uid, Gid: s.gid, }, // Creates a new session if the calling process is not a process group leader. // The calling process is the leader of the new session, the process group leader of // the new process group, and has no controlling terminal. // This is important to allow remote shells to handle ctrl+c. Setsid: true, } // Make the given terminal the controlling terminal of the calling process. // The calling process must be a session leader and not have a controlling terminal already. // This is important as allows ctrl+c to work as expected for non-shell programs. if s.interactive { cmd.SysProcAttr.Setctty = true } } func osHandleExecControl(control api.InstanceExecControl, s *execWs, pty io.ReadWriteCloser, cmd *exec.Cmd, l logger.Logger) { if control.Command == "window-resize" && s.interactive { winchWidth, err := strconv.Atoi(control.Args["width"]) if err != nil { l.Debug("Unable to extract window width", logger.Ctx{"err": err}) return } winchHeight, err := strconv.Atoi(control.Args["height"]) if err != nil { l.Debug("Unable to extract window height", logger.Ctx{"err": err}) return } osFile, ok := pty.(*os.File) if ok { err = setPtySize(int(osFile.Fd()), winchWidth, winchHeight) if err != nil { l.Debug("Failed to set window size", logger.Ctx{"err": err, "width": winchWidth, "height": winchHeight}) return } } } else if control.Command == "signal" { err := unix.Kill(cmd.Process.Pid, unix.Signal(control.Signal)) if err != nil { l.Debug("Failed forwarding signal", logger.Ctx{"err": err, "signal": control.Signal}) return } l.Info("Forwarded signal", logger.Ctx{"signal": control.Signal}) } } // osExitStatus is is the same as linux.ExitStatus for BSD-likes. func osExitStatus(err error) (int, error) { if err == nil { return 0, err // No error exit status. } var exitErr *exec.ExitError // Detect and extract ExitError to check the embedded exit status. if errors.As(err, &exitErr) { // If the process was signaled, extract the signal. status, isWaitStatus := exitErr.Sys().(unix.WaitStatus) if isWaitStatus && status.Signaled() { return 128 + int(status.Signal()), nil // 128 + n == Fatal error signal "n" } // Otherwise capture the exit status from the command. return exitErr.ExitCode(), nil } return -1, err // Not able to extract an exit status. } func osGetListener(port int64) (net.Listener, error) { l, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) if err != nil { return nil, fmt.Errorf("Failed to listen on TCP: %w", err) } logger.Info("Started TCP listener") return l, nil } incus-7.0.0/cmd/incus-agent/os_common.go000066400000000000000000000147441517523235500201610ustar00rootroot00000000000000//go:build darwin || freebsd || windows package main import ( "context" "io" "math" "net" "os" "sort" "strconv" "github.com/shirou/gopsutil/v4/cpu" "github.com/shirou/gopsutil/v4/disk" "github.com/shirou/gopsutil/v4/mem" psUtilNet "github.com/shirou/gopsutil/v4/net" "github.com/shirou/gopsutil/v4/process" "github.com/lxc/incus/v7/internal/server/metrics" "github.com/lxc/incus/v7/shared/api" ) var ( osShutdownSignal = os.Interrupt osMetricsSupported = true osGuestAPISupport = false ) func osLoadModules() error { // No OS drivers to load by default. return nil } func osGetCPUMetrics(d *Daemon) ([]metrics.CPUMetrics, error) { cpuTimes, err := cpu.Times(true) if err != nil { return nil, err } cpuMetrics := make([]metrics.CPUMetrics, 0, len(cpuTimes)) for _, cpuTime := range cpuTimes { cpuMetrics = append(cpuMetrics, metrics.CPUMetrics{ CPU: cpuTime.CPU, SecondsUser: cpuTime.User, SecondsNice: cpuTime.Nice, SecondsSystem: cpuTime.System, SecondsIdle: cpuTime.Idle, SecondsIOWait: cpuTime.Iowait, SecondsIRQ: cpuTime.Irq, SecondsSoftIRQ: cpuTime.Softirq, SecondsSteal: cpuTime.Steal, }) } return cpuMetrics, nil } func osGetDiskMetrics(d *Daemon) ([]metrics.DiskMetrics, error) { counters, err := disk.IOCounters() if err != nil { return nil, err } devices := make([]string, 0, len(counters)) for device := range counters { devices = append(devices, device) } sort.Strings(devices) diskMetrics := make([]metrics.DiskMetrics, 0, len(devices)) for _, device := range devices { counter := counters[device] diskMetrics = append(diskMetrics, metrics.DiskMetrics{ Device: counter.Name, ReadBytes: counter.ReadBytes, ReadsCompleted: counter.ReadCount, WrittenBytes: counter.WriteBytes, WritesCompleted: counter.WriteCount, }) } return diskMetrics, nil } func osGetMemoryMetrics(d *Daemon) (metrics.MemoryMetrics, error) { virtualMemory, err := mem.VirtualMemory() if err != nil { return metrics.MemoryMetrics{}, err } swapMemory, err := mem.SwapMemory() if err != nil { return metrics.MemoryMetrics{}, err } return metrics.MemoryMetrics{ ActiveAnonBytes: 0, ActiveFileBytes: 0, ActiveBytes: virtualMemory.Active, CachedBytes: virtualMemory.Cached, DirtyBytes: virtualMemory.Dirty, HugepagesFreeBytes: virtualMemory.HugePagesFree * virtualMemory.HugePageSize, HugepagesTotalBytes: virtualMemory.HugePagesTotal * virtualMemory.HugePageSize, InactiveAnonBytes: 0, InactiveFileBytes: 0, InactiveBytes: virtualMemory.Inactive, MappedBytes: virtualMemory.Mapped, MemAvailableBytes: virtualMemory.Available, MemFreeBytes: virtualMemory.Free, MemTotalBytes: virtualMemory.Total, RSSBytes: 0, ShmemBytes: virtualMemory.Shared, SwapBytes: swapMemory.Total, UnevictableBytes: 0, WritebackBytes: virtualMemory.WriteBack, OOMKills: 0, }, nil } func osGetCPUState() api.InstanceStateCPU { cpuState := api.InstanceStateCPU{} cpuTimes, err := cpu.Times(false) if err != nil || len(cpuTimes) < 1 { cpuState.Usage = -1 } else { cpuTime := cpuTimes[0] cpuState.Usage = int64(math.Round((cpuTime.System + cpuTime.User) * 1e9)) } return cpuState } func osGetMemoryState() api.InstanceStateMemory { memory := api.InstanceStateMemory{} virtualMemory, err := mem.VirtualMemory() if err != nil { return memory } memory.Usage = int64(virtualMemory.Total - virtualMemory.Free) memory.Total = int64(virtualMemory.Total) return memory } func ipScope(ip net.IP) string { if ip.IsLoopback() { return "local" } if ip.To4() != nil { if ip[0] == 169 && ip[1] == 254 { return "link" } return "global" } if ip[0] == 0xfe && (ip[1]&0xc0) == 0x80 { return "link" } return "global" } func osGetNetworkState() map[string]api.InstanceStateNetwork { interfaces, err := psUtilNet.Interfaces() if err != nil { return map[string]api.InstanceStateNetwork{} } ioCounters, err := psUtilNet.IOCounters(true) if err != nil { return map[string]api.InstanceStateNetwork{} } // Create a map for fast lookup. counters := make(map[string]psUtilNet.IOCountersStat, len(ioCounters)) for _, c := range ioCounters { counters[c.Name] = c } sort.Slice(interfaces, func(i, j int) bool { return interfaces[i].Name < interfaces[j].Name }) network := make(map[string]api.InstanceStateNetwork, len(interfaces)) for _, intf := range interfaces { addrs := make([]api.InstanceStateNetworkAddress, 0, len(intf.Addrs)) for _, addr := range intf.Addrs { ip, ipnet, err := net.ParseCIDR(addr.Addr) if err != nil || ip == nil || ipnet == nil { continue } family := "inet" if ip.To4() == nil { family = "inet6" } ones, _ := ipnet.Mask.Size() addrs = append(addrs, api.InstanceStateNetworkAddress{ Family: family, Address: ip.String(), Netmask: strconv.Itoa(ones), Scope: ipScope(ip), }) } var cnt api.InstanceStateNetworkCounters counter, ok := counters[intf.Name] if ok { cnt = api.InstanceStateNetworkCounters{ BytesReceived: int64(counter.BytesRecv), BytesSent: int64(counter.BytesSent), PacketsReceived: int64(counter.PacketsRecv), PacketsSent: int64(counter.PacketsSent), ErrorsReceived: int64(counter.Errin), ErrorsSent: int64(counter.Errout), PacketsDroppedOutbound: int64(counter.Dropout), PacketsDroppedInbound: int64(counter.Dropin), } } interfaceState := "down" interfaceType := "unknown" for _, flag := range intf.Flags { if flag == "up" { interfaceState = "up" } else if flag == "broadcast" { interfaceType = "broadcast" } else if flag == "loopback" { interfaceType = "loopback" } else if flag == "pointtopoint" { interfaceType = "point-to-point" } } network[intf.Name] = api.InstanceStateNetwork{ Addresses: addrs, Counters: cnt, Hwaddr: intf.HardwareAddr, HostName: intf.Name, Mtu: intf.MTU, State: interfaceState, Type: interfaceType, } } return network } func osGetProcessesState() int64 { processes, err := process.Processes() if err != nil { return -1 } return int64(len(processes)) } func osReconfigureNetworkInterfaces() { // Agent assisted network reconfiguration isn't currently supported. return } func osExecWrapper(ctx context.Context, pty io.ReadWriteCloser) io.ReadWriteCloser { return pty } incus-7.0.0/cmd/incus-agent/os_darwin.go000066400000000000000000000206361517523235500201520ustar00rootroot00000000000000//go:build darwin package main import ( "errors" "fmt" "io/fs" "os" "sort" "strconv" "strings" "syscall" "unsafe" "github.com/shirou/gopsutil/v4/disk" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/server/metrics" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" ) func osMountShared(src string, dst string, fstype string, opts []string) error { if fstype != "9p" { return errors.New("Only 9p shares are supported on Darwin") } // 9p shares behave strangely on Darwin, as they don't get mounted in the file system, but rather // as volumes, and have their own subcommand that doesn't conform to the mount frontend. // Bind mounts are not natively supported, so we have to mount the 9p share, then symlink it. // Convert relative mounts to absolute from / otherwise dir creation fails or mount fails. if !strings.HasPrefix(dst, "/") { dst = fmt.Sprintf("/%s", dst) } // If the path exists and is neither an empty directory nor a broken symbolic link to a volume // (indicating with high probability a previous share mount which we didn't clean up properly), we // can't safely use it. stat, err := os.Lstat(dst) if err == nil { if stat.IsDir() { // Handle directories, failing if not empty. entries, err := os.ReadDir(dst) if err != nil { return fmt.Errorf("Failed to open directory %s: %w", dst, err) } if len(entries) > 0 { return errors.New("Unable to mount shares on non-empty directories") } } else if stat.Mode()&fs.ModeSymlink != 0 { // Handle symbolic links, fail if not broken. // Try to follow the link. _, err := os.Stat(dst) if err == nil { return fmt.Errorf("Unable to mount shares on working symbolic link %s", dst) } } else { return fmt.Errorf("Mount destination %s exists and is not empty", dst) } err = os.Remove(dst) if err != nil { return fmt.Errorf("Failed to prepare destination %s: %w", dst, err) } } else if !errors.Is(err, fs.ErrNotExist) { return err } _, err = subprocess.RunCommand("mount_9p", src) if err != nil { return err } // Volume naming is very predictable. If a disk has the same name as a 9p share, `mount_9p` fails. return os.Symlink("/Volumes/"+src, dst) } // osUmount is currently not used, but it is implemented just in case. func osUmount(src string, dst string, fstype string) error { if fstype != "9p" { return errors.New("Only 9p shares are supported on Darwin") } // First, remove the symlink. err := os.Remove(dst) if err != nil { return err } // Then, unmount the share. _, err = subprocess.RunCommand("umount", "/Volumes/"+src) return err } func osGetFilesystemMetrics(d *Daemon) ([]metrics.FilesystemMetrics, error) { partitions, err := disk.Partitions(true) if err != nil { return nil, err } sort.Slice(partitions, func(i, j int) bool { return partitions[i].Mountpoint < partitions[j].Mountpoint }) fsMetrics := make([]metrics.FilesystemMetrics, 0, len(partitions)) for _, partition := range partitions { var stat syscall.Statfs_t err = syscall.Statfs(partition.Mountpoint, &stat) if err != nil { continue } bsize := uint64(stat.Bsize) fsMetrics = append(fsMetrics, metrics.FilesystemMetrics{ Device: partition.Device, Mountpoint: partition.Mountpoint, FSType: partition.Fstype, AvailableBytes: stat.Bavail * bsize, FreeBytes: stat.Bfree * bsize, SizeBytes: stat.Blocks * bsize, }) } return fsMetrics, nil } func macOSVersionName(version string) (string, error) { parts := strings.Split(version, ".") var major, minor int var err error if len(parts) > 0 { major, err = strconv.Atoi(parts[0]) if err != nil { return "", err } } if len(parts) > 1 { minor, err = strconv.Atoi(parts[1]) if err != nil { return "", err } } switch major { case 26: return "Tahoe", nil case 15: return "Sequoia", nil case 14: return "Sonoma", nil case 13: return "Ventura", nil case 12: return "Monterey", nil case 11: return "Big Sur", nil case 10: switch minor { case 16: // Apparently, this one can happen. return "Big Sur", nil case 15: return "Catalina", nil case 14: return "Mojave", nil case 13: return "High Sierra", nil case 12: return "Sierra", nil case 11: return "El Capitan", nil case 10: return "Yosemite", nil case 9: return "Mavericks", nil case 8: return "Mountain Lion", nil case 7: return "Lion", nil case 6: return "Snow Leopard", nil case 5: return "Leopard", nil case 4: return "Tiger", nil case 3: return "Panther", nil case 2: return "Jaguar", nil case 1: return "Puma", nil case 0: return "Cheetah", nil } } return "", errors.New("Unknown macOS version") } func osGetOSState() *api.InstanceStateOSInfo { swVers, err := subprocess.RunCommand("sw_vers") if err != nil { return nil } var productName, productVersion string for _, line := range strings.Split(strings.TrimSpace(swVers), "\n") { key, after, found := strings.Cut(line, ":") if !found { continue } value := strings.TrimSpace(after) if key == "ProductName" { productName = value } else if key == "ProductVersion" { productVersion = value } } // Add the familiar version name if we are dealing with a known macOS version. if productName == "Mac OS X" || productName == "macOS" { versionName, err := macOSVersionName(productVersion) if err == nil { productVersion += " (" + versionName + ")" } } uname := unix.Utsname{} err = unix.Uname(&uname) if err != nil { return nil } serverName := parseBytes(uname.Nodename[:]) // Prepare OS struct. osInfo := &api.InstanceStateOSInfo{ OS: productName, OSVersion: productVersion, KernelVersion: parseBytes(uname.Release[:]), Hostname: serverName, FQDN: serverName, } return osInfo } // openPty is is the same as linux.OpenPty for Darwin. func openPty(uid, gid int64) (*os.File, *os.File, error) { reverter := revert.New() defer reverter.Fail() fd, err := unix.Open("/dev/ptmx", unix.O_RDWR|unix.O_CLOEXEC|unix.O_NOCTTY, 0) if err != nil { return nil, nil, err } ptx := os.NewFile(uintptr(fd), "") reverter.Add(func() { _ = ptx.Close() }) // Unlock the ptx and pty. _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(ptx.Fd()), unix.TIOCPTYUNLK, 0) if errno != 0 { return nil, nil, unix.Errno(errno) } var ptyName [256]byte _, _, errno = unix.Syscall(unix.SYS_IOCTL, uintptr(ptx.Fd()), unix.TIOCPTYGNAME, uintptr(unsafe.Pointer(&ptyName))) if errno != 0 { return nil, nil, unix.Errno(errno) } pty, err := os.OpenFile(parseBytes(ptyName[:]), unix.O_RDWR|unix.O_CLOEXEC|unix.O_NOCTTY, 0) if err != nil { return nil, nil, err } reverter.Add(func() { _ = pty.Close() }) // Configure both sides for _, entry := range []*os.File{ptx, pty} { // Get termios. t, err := unix.IoctlGetTermios(int(entry.Fd()), unix.TIOCGETA) if err != nil { return nil, nil, err } // Set flags. t.Cflag |= unix.IMAXBEL t.Cflag |= unix.IUTF8 t.Cflag |= unix.BRKINT t.Cflag |= unix.IXANY t.Cflag |= unix.HUPCL // Set termios. err = unix.IoctlSetTermios(int(entry.Fd()), unix.TIOCSETA, t) if err != nil { return nil, nil, err } // Set the default window size. sz := &unix.Winsize{ Col: 80, Row: 25, } err = unix.IoctlSetWinsize(int(entry.Fd()), unix.TIOCSWINSZ, sz) if err != nil { return nil, nil, err } // Set CLOEXEC. _, _, errno = unix.Syscall(unix.SYS_FCNTL, uintptr(entry.Fd()), unix.F_SETFD, unix.FD_CLOEXEC) if errno != 0 { return nil, nil, unix.Errno(errno) } } // Fix the ownership of the pty side. err = unix.Fchown(int(pty.Fd()), int(uid), int(gid)) if err != nil { return nil, nil, err } reverter.Success() return ptx, pty, nil } func osSetEnv(post *api.InstanceExecPost, env map[string]string) { env["PATH"] = "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" // If running as root, set some env variables. if post.User == 0 { // Set default value for HOME. Fix /root. home, ok := env["HOME"] if !ok || home == "/root" { env["HOME"] = "/var/root" } // Set default value for USER. _, ok = env["USER"] if !ok { env["USER"] = "root" } } // Set default value for LANG. _, ok := env["LANG"] if !ok { env["LANG"] = "C.UTF-8" } // Set the default working directory. if post.Cwd == "" { post.Cwd = env["HOME"] if post.Cwd == "" { post.Cwd = osBaseWorkingDirectory } } } incus-7.0.0/cmd/incus-agent/os_freebsd.go000066400000000000000000000142001517523235500202660ustar00rootroot00000000000000//go:build freebsd package main import ( "context" "errors" "fmt" "net" "os" "sort" "strings" "syscall" "time" "unsafe" "github.com/shirou/gopsutil/v4/disk" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/server/metrics" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) // isMountPoint returns true if path is a mount point. func isMountPoint(path string) bool { // Get the stat details. stat, err := os.Stat(path) if err != nil { return false } rootStat, err := os.Lstat(path + "/..") if err != nil { return false } return stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev } func osMountShared(src string, dst string, fstype string, opts []string) error { if fstype != "9p" { return errors.New("Only 9p shares are supported on FreeBSD") } // Convert relative mounts to absolute from / otherwise dir creation fails or mount fails. if !strings.HasPrefix(dst, "/") { dst = fmt.Sprintf("/%s", dst) } // Check mount path. if !util.PathExists(dst) { // Create the mount path. err := os.MkdirAll(dst, 0o755) if err != nil { return fmt.Errorf("Failed to create mount target %q", dst) } } else if isMountPoint(dst) { // Already mounted. return nil } args := []string{"-t", "p9fs", src, dst} for _, opt := range opts { args = append(args, "-o", opt) } _, err := subprocess.RunCommand("mount", args...) return err } // osUmount is currently not used, but it is implemented just in case. func osUmount(src string, dst string, fstype string) error { if fstype != "9p" { return errors.New("Only 9p shares are supported on FreeBSD") } _, err := subprocess.RunCommand("umount", src) return err } func osGetFilesystemMetrics(d *Daemon) ([]metrics.FilesystemMetrics, error) { partitions, err := disk.Partitions(true) if err != nil { return nil, err } sort.Slice(partitions, func(i, j int) bool { return partitions[i].Mountpoint < partitions[j].Mountpoint }) fsMetrics := make([]metrics.FilesystemMetrics, 0, len(partitions)) for _, partition := range partitions { var stat syscall.Statfs_t err = syscall.Statfs(partition.Mountpoint, &stat) if err != nil { continue } fsMetrics = append(fsMetrics, metrics.FilesystemMetrics{ Device: partition.Device, Mountpoint: partition.Mountpoint, FSType: partition.Fstype, AvailableBytes: uint64(stat.Bavail) * stat.Bsize, FreeBytes: stat.Bfree * stat.Bsize, SizeBytes: stat.Blocks * stat.Bsize, }) } return fsMetrics, nil } func osGetOSState() *api.InstanceStateOSInfo { osInfo := &api.InstanceStateOSInfo{} // Get information about the OS. lsbRelease, err := osarch.GetOSRelease() if err == nil { osInfo.OS = lsbRelease["NAME"] osInfo.OSVersion = lsbRelease["VERSION_ID"] } // Get information about the kernel version. uname := unix.Utsname{} err = unix.Uname(&uname) if err == nil { osInfo.KernelVersion = parseBytes(uname.Release[:]) } // Get the hostname. hostname, err := os.Hostname() if err == nil { osInfo.Hostname = hostname } // Get the FQDN. To avoid needing to run `hostname -f`, do a reverse host lookup for 127.0.1.1, and if found, return the first hostname as the FQDN. ctx, cancel := context.WithTimeout(context.TODO(), 100*time.Millisecond) defer cancel() var r net.Resolver fqdn, err := r.LookupAddr(ctx, "127.0.0.1") if err == nil && len(fqdn) > 0 { // Take the first returned hostname and trim the trailing dot. osInfo.FQDN = strings.TrimSuffix(fqdn[0], ".") } return osInfo } // openPty is is the same as linux.OpenPty for FreeBSD. func openPty(uid, gid int64) (*os.File, *os.File, error) { reverter := revert.New() defer reverter.Fail() fd, err := unix.Open("/dev/ptmx", unix.O_RDWR|unix.O_CLOEXEC|unix.O_NOCTTY, 0) if err != nil { return nil, nil, err } ptx := os.NewFile(uintptr(fd), "/dev/pts/ptmx") reverter.Add(func() { _ = ptx.Close() }) // Get the pty side. id := 0 _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(ptx.Fd()), unix.TIOCGPTN, uintptr(unsafe.Pointer(&id))) if errno != 0 { return nil, nil, unix.Errno(errno) } ptyPath := fmt.Sprintf("/dev/pts/%d", id) ptyFd, err := unix.Open(ptyPath, unix.O_NOCTTY|unix.O_CLOEXEC|os.O_RDWR, 0) if err != nil { return nil, nil, err } pty := os.NewFile(uintptr(ptyFd), ptyPath) reverter.Add(func() { _ = pty.Close() }) // Configure both sides for _, entry := range []*os.File{ptx, pty} { // Get termios. t, err := unix.IoctlGetTermios(int(entry.Fd()), unix.TIOCGETA) if err != nil { return nil, nil, err } // Set flags. t.Cflag |= unix.IMAXBEL t.Cflag |= unix.BRKINT t.Cflag |= unix.IXANY t.Cflag |= unix.HUPCL // Set termios. err = unix.IoctlSetTermios(int(entry.Fd()), unix.TIOCSETA, t) if err != nil { return nil, nil, err } // Set the default window size. sz := &unix.Winsize{ Col: 80, Row: 25, } err = unix.IoctlSetWinsize(int(entry.Fd()), unix.TIOCSWINSZ, sz) if err != nil { return nil, nil, err } // Set CLOEXEC. _, _, errno = unix.Syscall(unix.SYS_FCNTL, uintptr(entry.Fd()), unix.F_SETFD, unix.FD_CLOEXEC) if errno != 0 { return nil, nil, unix.Errno(errno) } } // Fix the ownership of the pty side. err = unix.Fchown(int(pty.Fd()), int(uid), int(gid)) if err != nil { return nil, nil, err } reverter.Success() return ptx, pty, nil } func osSetEnv(post *api.InstanceExecPost, env map[string]string) { // Set default value for PATH. _, ok := env["PATH"] if !ok { env["PATH"] = "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin" } // If running as root, set some env variables. if post.User == 0 { // Set default value for HOME. _, ok = env["HOME"] if !ok { env["HOME"] = "/root" } // Set default value for USER. _, ok = env["USER"] if !ok { env["USER"] = "root" } } // Set default value for LANG. _, ok = env["LANG"] if !ok { env["LANG"] = "C.UTF-8" } // Set the default working directory. if post.Cwd == "" { post.Cwd = env["HOME"] if post.Cwd == "" { post.Cwd = osBaseWorkingDirectory } } } incus-7.0.0/cmd/incus-agent/os_linux.go000066400000000000000000000545661517523235500200360ustar00rootroot00000000000000//go:build linux package main import ( "bufio" "bytes" "context" "encoding/json" "errors" "fmt" "io" "io/fs" "net" "os" "os/exec" "path/filepath" "regexp" "slices" "strconv" "strings" "syscall" "time" "github.com/mdlayher/vsock" "github.com/shirou/gopsutil/v4/process" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/ports" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/metrics" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) var ( // These mountpoints are excluded as they are irrelevant for metrics. // /var/lib/docker/* subdirectories are excluded for this reason: https://github.com/prometheus/node_exporter/pull/1003 osMetricsExcludeMountpoints = regexp.MustCompile(`^/(?:dev|proc|sys|var/lib/docker/.+)(?:$|/)`) osMetricsExcludeFilesystems = []string{"autofs", "binfmt_misc", "bpf", "cgroup", "cgroup2", "configfs", "debugfs", "devpts", "devtmpfs", "fusectl", "hugetlbfs", "iso9660", "mqueue", "nsfs", "overlay", "proc", "procfs", "pstore", "rpc_pipefs", "securityfs", "selinuxfs", "squashfs", "sysfs", "tracefs"} osShutdownSignal = unix.SIGTERM osExitStatus = linux.ExitStatus osBaseWorkingDirectory = "/" osMetricsSupported = true osGuestAPISupport = true osAgentConfigPath = "/etc/incus-agent.yml" osVioSerialPath = "/dev/virtio-ports/org.linuxcontainers.incus" ) func runService(name string, agentCmd *cmdAgent) error { return errors.New("Not implemented.") } func osGetEnvironment() (*api.ServerEnvironment, error) { uname, err := linux.Uname() if err != nil { return nil, err } serverName, err := os.Hostname() if err != nil { return nil, err } env := &api.ServerEnvironment{ Kernel: uname.Sysname, KernelArchitecture: uname.Machine, KernelVersion: uname.Release, Server: "incus-agent", ServerPid: os.Getpid(), ServerVersion: version.Version, ServerName: serverName, } return env, nil } func osLoadModules() error { // Attempt to load the virtio_net driver in case it's not be loaded yet. // This may be needed for later network configuration. _ = linux.LoadModule("virtio_net") // Load the vsock driver if not loaded yet, this is required for host communication. if !util.PathExists("/dev/vsock") { logger.Info("Loading vsock module") err := linux.LoadModule("vsock") if err != nil { return fmt.Errorf("Unable to load the vsock kernel module: %w", err) } // Wait for vsock device to appear. for range 5 { if !util.PathExists("/dev/vsock") { time.Sleep(1 * time.Second) } } } return nil } func osMountShared(src string, dst string, fstype string, opts []string) error { // Convert relative mounts to absolute from / otherwise dir creation fails or mount fails. if !strings.HasPrefix(dst, "/") { dst = fmt.Sprintf("/%s", dst) } // Check mount path. if !util.PathExists(dst) { // Create the mount path. err := os.MkdirAll(dst, 0o755) if err != nil { return fmt.Errorf("Failed to create mount target %q", dst) } } else if linux.IsMountPoint(dst) { // Already mounted. return nil } args := []string{"-t", fstype, src, dst} for _, opt := range opts { args = append(args, "-o", opt) } _, err := subprocess.RunCommand("mount", args...) return err } func osUmount(src string, dst string, fstype string) error { _, err := subprocess.RunCommand("umount", src) return err } func osGetCPUMetrics(d *Daemon) ([]metrics.CPUMetrics, error) { stats, err := os.ReadFile("/proc/stat") if err != nil { return nil, fmt.Errorf("Failed to read /proc/stat: %w", err) } out := []metrics.CPUMetrics{} scanner := bufio.NewScanner(bytes.NewReader(stats)) for scanner.Scan() { line := scanner.Text() fields := strings.Fields(line) // Only consider CPU info, skip everything else. Skip aggregated CPU stats since there will // be stats for each individual CPU. if !strings.HasPrefix(fields[0], "cpu") || fields[0] == "cpu" { continue } // Validate the number of fields only for lines starting with "cpu". if len(fields) < 9 { return nil, fmt.Errorf("Invalid /proc/stat content: %q", line) } stats := metrics.CPUMetrics{} stats.SecondsUser, err = strconv.ParseFloat(fields[1], 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[1], err) } stats.SecondsUser /= 100 stats.SecondsNice, err = strconv.ParseFloat(fields[2], 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[2], err) } stats.SecondsNice /= 100 stats.SecondsSystem, err = strconv.ParseFloat(fields[3], 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err) } stats.SecondsSystem /= 100 stats.SecondsIdle, err = strconv.ParseFloat(fields[4], 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[4], err) } stats.SecondsIdle /= 100 stats.SecondsIOWait, err = strconv.ParseFloat(fields[5], 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[5], err) } stats.SecondsIOWait /= 100 stats.SecondsIRQ, err = strconv.ParseFloat(fields[6], 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[6], err) } stats.SecondsIRQ /= 100 stats.SecondsSoftIRQ, err = strconv.ParseFloat(fields[7], 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[7], err) } stats.SecondsSoftIRQ /= 100 stats.SecondsSteal, err = strconv.ParseFloat(fields[8], 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[8], err) } stats.SecondsSteal /= 100 stats.CPU = fields[0] out = append(out, stats) } return out, nil } func osGetDiskMetrics(d *Daemon) ([]metrics.DiskMetrics, error) { diskStats, err := os.ReadFile("/proc/diskstats") if err != nil { return nil, fmt.Errorf("Failed to read /proc/diskstats: %w", err) } out := []metrics.DiskMetrics{} scanner := bufio.NewScanner(bytes.NewReader(diskStats)) for scanner.Scan() { line := scanner.Text() if line == "" { continue } fields := strings.Fields(line) if len(fields) < 10 { return nil, fmt.Errorf("Invalid /proc/diskstats content: %q", line) } stats := metrics.DiskMetrics{} stats.ReadsCompleted, err = strconv.ParseUint(fields[3], 10, 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err) } sectorsRead, err := strconv.ParseUint(fields[5], 10, 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err) } stats.ReadBytes = sectorsRead * 512 stats.WritesCompleted, err = strconv.ParseUint(fields[7], 10, 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err) } sectorsWritten, err := strconv.ParseUint(fields[9], 10, 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err) } stats.WrittenBytes = sectorsWritten * 512 stats.Device = fields[2] out = append(out, stats) } return out, nil } func osGetFilesystemMetrics(d *Daemon) ([]metrics.FilesystemMetrics, error) { mounts, err := os.ReadFile("/proc/mounts") if err != nil { return nil, fmt.Errorf("Failed to read /proc/mounts: %w", err) } out := []metrics.FilesystemMetrics{} scanner := bufio.NewScanner(bytes.NewReader(mounts)) for scanner.Scan() { line := scanner.Text() fields := strings.Fields(line) if len(fields) < 3 { return nil, fmt.Errorf("Invalid /proc/mounts content: %q", line) } // Skip uninteresting mounts if slices.Contains(osMetricsExcludeFilesystems, fields[2]) || osMetricsExcludeMountpoints.MatchString(fields[1]) { continue } stats := metrics.FilesystemMetrics{} stats.Mountpoint = fields[1] statfs, err := linux.StatVFS(stats.Mountpoint) if err != nil { return nil, fmt.Errorf("Failed to stat %s: %w", stats.Mountpoint, err) } fsType, err := linux.FSTypeToName(int32(statfs.Type)) if err == nil { stats.FSType = fsType } stats.AvailableBytes = statfs.Bavail * uint64(statfs.Bsize) stats.FreeBytes = statfs.Bfree * uint64(statfs.Bsize) stats.SizeBytes = statfs.Blocks * uint64(statfs.Bsize) stats.Device = fields[0] out = append(out, stats) } return out, nil } func osGetMemoryMetrics(d *Daemon) (metrics.MemoryMetrics, error) { content, err := os.ReadFile("/proc/meminfo") if err != nil { return metrics.MemoryMetrics{}, fmt.Errorf("Failed to read /proc/meminfo: %w", err) } out := metrics.MemoryMetrics{} scanner := bufio.NewScanner(bytes.NewReader(content)) for scanner.Scan() { line := scanner.Text() fields := strings.Fields(line) if len(fields) < 2 { return metrics.MemoryMetrics{}, fmt.Errorf("Invalid /proc/meminfo content: %q", line) } fields[0] = strings.TrimRight(fields[0], ":") value, err := strconv.ParseUint(fields[1], 10, 64) if err != nil { return metrics.MemoryMetrics{}, fmt.Errorf("Failed to parse %q: %w", fields[1], err) } // Multiply suffix (kB) if len(fields) == 3 { value *= 1024 } // FIXME: Missing RSS switch fields[0] { case "Active": out.ActiveBytes = value case "Active(anon)": out.ActiveAnonBytes = value case "Active(file)": out.ActiveFileBytes = value case "Cached": out.CachedBytes = value case "Dirty": out.DirtyBytes = value case "HugePages_Free": out.HugepagesFreeBytes = value case "HugePages_Total": out.HugepagesTotalBytes = value case "Inactive": out.InactiveBytes = value case "Inactive(anon)": out.InactiveAnonBytes = value case "Inactive(file)": out.InactiveFileBytes = value case "Mapped": out.MappedBytes = value case "MemAvailable": out.MemAvailableBytes = value case "MemFree": out.MemFreeBytes = value case "MemTotal": out.MemTotalBytes = value case "Shmem": out.ShmemBytes = value case "SwapCached": out.SwapBytes = value case "Unevictable": out.UnevictableBytes = value case "Writeback": out.WritebackBytes = value } } return out, nil } func osGetCPUState() api.InstanceStateCPU { var value []byte var err error cpu := api.InstanceStateCPU{} if util.PathExists("/sys/fs/cgroup/cpuacct/cpuacct.usage") { // CPU usage in seconds value, err = os.ReadFile("/sys/fs/cgroup/cpuacct/cpuacct.usage") if err != nil { cpu.Usage = -1 return cpu } valueInt, err := strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64) if err != nil { cpu.Usage = -1 return cpu } cpu.Usage = valueInt return cpu } else if util.PathExists("/sys/fs/cgroup/cpu.stat") { stats, err := os.ReadFile("/sys/fs/cgroup/cpu.stat") if err != nil { cpu.Usage = -1 return cpu } scanner := bufio.NewScanner(bytes.NewReader(stats)) for scanner.Scan() { fields := strings.Fields(scanner.Text()) if fields[0] == "usage_usec" { valueInt, err := strconv.ParseInt(fields[1], 10, 64) if err != nil { cpu.Usage = -1 return cpu } // usec -> nsec cpu.Usage = valueInt * 1000 return cpu } } } cpu.Usage = -1 return cpu } func osGetMemoryState() api.InstanceStateMemory { memory := api.InstanceStateMemory{} stats, err := osGetMemoryMetrics(nil) if err != nil { return memory } memory.Usage = int64(stats.MemTotalBytes) - int64(stats.MemFreeBytes) memory.Total = int64(stats.MemTotalBytes) // Memory peak in bytes value, err := os.ReadFile("/sys/fs/cgroup/memory/memory.max_usage_in_bytes") valueInt, err1 := strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64) if err == nil && err1 == nil { memory.UsagePeak = valueInt } return memory } func osGetNetworkState() map[string]api.InstanceStateNetwork { result := map[string]api.InstanceStateNetwork{} ifs, err := linux.NetlinkInterfaces() if err != nil { logger.Errorf("Failed to retrieve network interfaces: %v", err) return result } for _, iface := range ifs { network := api.InstanceStateNetwork{ Addresses: []api.InstanceStateNetworkAddress{}, Counters: api.InstanceStateNetworkCounters{}, } network.Hwaddr = iface.HardwareAddr.String() network.Mtu = iface.MTU if iface.Flags&net.FlagUp != 0 { network.State = "up" } else { network.State = "down" } if iface.Flags&net.FlagBroadcast != 0 { network.Type = "broadcast" } else if iface.Flags&net.FlagLoopback != 0 { network.Type = "loopback" } else if iface.Flags&net.FlagPointToPoint != 0 { network.Type = "point-to-point" } else { network.Type = "unknown" } // Counters value, err := os.ReadFile(fmt.Sprintf("/sys/class/net/%s/statistics/tx_bytes", iface.Name)) valueInt, err1 := strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64) if err == nil && err1 == nil { network.Counters.BytesSent = valueInt } value, err = os.ReadFile(fmt.Sprintf("/sys/class/net/%s/statistics/rx_bytes", iface.Name)) valueInt, err1 = strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64) if err == nil && err1 == nil { network.Counters.BytesReceived = valueInt } value, err = os.ReadFile(fmt.Sprintf("/sys/class/net/%s/statistics/tx_packets", iface.Name)) valueInt, err1 = strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64) if err == nil && err1 == nil { network.Counters.PacketsSent = valueInt } value, err = os.ReadFile(fmt.Sprintf("/sys/class/net/%s/statistics/rx_packets", iface.Name)) valueInt, err1 = strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64) if err == nil && err1 == nil { network.Counters.PacketsReceived = valueInt } // Addresses for _, addr := range iface.Addresses { addressFields := strings.Split(addr.String(), "/") networkAddress := api.InstanceStateNetworkAddress{ Address: addressFields[0], Netmask: addressFields[1], } scope := "global" if strings.HasPrefix(addressFields[0], "127") { scope = "local" } if addressFields[0] == "::1" { scope = "local" } if strings.HasPrefix(addressFields[0], "169.254") { scope = "link" } if strings.HasPrefix(addressFields[0], "fe80:") { scope = "link" } networkAddress.Scope = scope if strings.Contains(addressFields[0], ":") { networkAddress.Family = "inet6" } else { networkAddress.Family = "inet" } network.Addresses = append(network.Addresses, networkAddress) } result[iface.Name] = network } return result } func osGetProcessesState() int64 { pids, err := process.Pids() if err != nil { return -1 } return int64(len(pids)) } func osGetOSState() *api.InstanceStateOSInfo { osInfo := &api.InstanceStateOSInfo{} // Get information about the OS. lsbRelease, err := osarch.GetOSRelease() if err == nil { osInfo.OS = lsbRelease["NAME"] osInfo.OSVersion = lsbRelease["VERSION_ID"] } // Get information about the kernel version. uname, err := linux.Uname() if err == nil { osInfo.KernelVersion = uname.Release } // Get the hostname. hostname, err := os.Hostname() if err == nil { osInfo.Hostname = hostname } // Get the FQDN. To avoid needing to run `hostname -f`, do a reverse host lookup for 127.0.1.1, and if found, return the first hostname as the FQDN. ctx, cancel := context.WithTimeout(context.TODO(), 100*time.Millisecond) defer cancel() var r net.Resolver fqdn, err := r.LookupAddr(ctx, "127.0.0.1") if err == nil && len(fqdn) > 0 { // Take the first returned hostname and trim the trailing dot. osInfo.FQDN = strings.TrimSuffix(fqdn[0], ".") } return osInfo } // osReconfigureNetworkInterfaces checks for the existence of files under NICConfigDir in the config share. // Each file is named .json and contains the Device Name, NIC Name, MTU and MAC address. func osReconfigureNetworkInterfaces() { nicDirEntries, err := os.ReadDir(deviceConfig.NICConfigDir) if err != nil { // Abort if configuration folder does not exist (nothing to do), otherwise log and return. if errors.Is(err, fs.ErrNotExist) { return } logger.Error("Could not read network interface configuration directory", logger.Ctx{"err": err}) return } // Attempt to load the virtio_net driver in case it's not be loaded yet. _ = linux.LoadModule("virtio_net") // nicData is a map of MAC address to NICConfig. nicData := make(map[string]deviceConfig.NICConfig, len(nicDirEntries)) for _, f := range nicDirEntries { nicBytes, err := os.ReadFile(filepath.Join(deviceConfig.NICConfigDir, f.Name())) if err != nil { logger.Error("Could not read network interface configuration file", logger.Ctx{"err": err}) } var conf deviceConfig.NICConfig err = json.Unmarshal(nicBytes, &conf) if err != nil { logger.Error("Could not parse network interface configuration file", logger.Ctx{"err": err}) return } if conf.MACAddress != "" { nicData[conf.MACAddress] = conf } } // configureNIC applies any config specified for the interface based on its current MAC address. configureNIC := func(currentNIC net.Interface) error { reverter := revert.New() defer reverter.Fail() // Look for a NIC config entry for this interface based on its MAC address. nic, ok := nicData[currentNIC.HardwareAddr.String()] if !ok { return nil } var changeName, changeMTU bool if nic.NICName != "" && currentNIC.Name != nic.NICName { changeName = true } if nic.MTU > 0 && currentNIC.MTU != int(nic.MTU) { changeMTU = true } if !changeName && !changeMTU { return nil // Nothing to do. } link := ip.Link{ Name: currentNIC.Name, MTU: uint32(currentNIC.MTU), } err := link.SetDown() if err != nil { return err } reverter.Add(func() { _ = link.SetUp() }) // Apply the name from the NIC config if needed. if changeName { err = link.SetName(nic.NICName) if err != nil { return err } reverter.Add(func() { err := link.SetName(currentNIC.Name) if err != nil { return } link.Name = currentNIC.Name }) link.Name = nic.NICName } // Apply the MTU from the NIC config if needed. if changeMTU { err = link.SetMTU(nic.MTU) if err != nil { return err } link.MTU = nic.MTU reverter.Add(func() { err := link.SetMTU(uint32(currentNIC.MTU)) if err != nil { return } link.MTU = uint32(currentNIC.MTU) }) } err = link.SetUp() if err != nil { return err } reverter.Success() return nil } ifaces, err := net.Interfaces() if err != nil { logger.Error("Unable to read network interfaces", logger.Ctx{"err": err}) } for _, iface := range ifaces { err = configureNIC(iface) if err != nil { logger.Error("Unable to reconfigure network interface", logger.Ctx{"interface": iface.Name, "err": err}) } } } func osGetInteractiveConsole(s *execWs) (*os.File, *os.File, error) { pty, tty, err := linux.OpenPty(int64(s.uid), int64(s.gid)) if err != nil { return nil, nil, err } if s.width > 0 && s.height > 0 { _ = linux.SetPtySize(int(pty.Fd()), s.width, s.height) } return pty, tty, nil } func osPrepareExecCommand(s *execWs, cmd *exec.Cmd) { cmd.SysProcAttr = &syscall.SysProcAttr{ Credential: &syscall.Credential{ Uid: s.uid, Gid: s.gid, }, // Creates a new session if the calling process is not a process group leader. // The calling process is the leader of the new session, the process group leader of // the new process group, and has no controlling terminal. // This is important to allow remote shells to handle ctrl+c. Setsid: true, } // Make the given terminal the controlling terminal of the calling process. // The calling process must be a session leader and not have a controlling terminal already. // This is important as allows ctrl+c to work as expected for non-shell programs. if s.interactive { cmd.SysProcAttr.Setctty = true } } func osHandleExecControl(control api.InstanceExecControl, s *execWs, pty io.ReadWriteCloser, cmd *exec.Cmd, l logger.Logger) { if control.Command == "window-resize" && s.interactive { winchWidth, err := strconv.Atoi(control.Args["width"]) if err != nil { l.Debug("Unable to extract window width", logger.Ctx{"err": err}) return } winchHeight, err := strconv.Atoi(control.Args["height"]) if err != nil { l.Debug("Unable to extract window height", logger.Ctx{"err": err}) return } osFile, ok := pty.(*os.File) if ok { err = linux.SetPtySize(int(osFile.Fd()), winchWidth, winchHeight) if err != nil { l.Debug("Failed to set window size", logger.Ctx{"err": err, "width": winchWidth, "height": winchHeight}) return } } } else if control.Command == "signal" { err := unix.Kill(cmd.Process.Pid, unix.Signal(control.Signal)) if err != nil { l.Debug("Failed forwarding signal", logger.Ctx{"err": err, "signal": control.Signal}) return } l.Info("Forwarded signal", logger.Ctx{"signal": control.Signal}) } } func osExecWrapper(ctx context.Context, pty io.ReadWriteCloser) io.ReadWriteCloser { osFile, ok := pty.(*os.File) if !ok { return pty } return linux.NewExecWrapper(ctx, osFile) } func osGetListener(port int64) (net.Listener, error) { const CIDAny uint32 = 4294967295 // Equivalent to VMADDR_CID_ANY. // Setup the listener on wildcard CID for inbound connections from Incus. // We use the VMADDR_CID_ANY CID so that if the VM's CID changes in the future the listener still works. // A CID change can occur when restoring a stateful VM that was previously using one CID but is // subsequently restored using a different one. l, err := vsock.ListenContextID(CIDAny, ports.HTTPSDefaultPort, nil) if err != nil { return nil, fmt.Errorf("Failed to listen on vsock: %w", err) } logger.Info("Started vsock listener") return l, nil } func osSetEnv(post *api.InstanceExecPost, env map[string]string) { // Set default value for PATH. _, ok := env["PATH"] if !ok { env["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" } if util.PathExists("/snap/bin") { env["PATH"] = fmt.Sprintf("%s:/snap/bin", env["PATH"]) } // If running as root, set some env variables. if post.User == 0 { // Set default value for HOME. _, ok = env["HOME"] if !ok { env["HOME"] = "/root" } // Set default value for USER. _, ok = env["USER"] if !ok { env["USER"] = "root" } } // Set default value for LANG. _, ok = env["LANG"] if !ok { env["LANG"] = "C.UTF-8" } // Set the default working directory. if post.Cwd == "" { post.Cwd = env["HOME"] if post.Cwd == "" { post.Cwd = "/" } } } incus-7.0.0/cmd/incus-agent/os_windows.go000066400000000000000000000203611517523235500203530ustar00rootroot00000000000000//go:build windows package main import ( "context" "errors" "fmt" "io" "net" "os" "os/exec" "runtime" "sort" "strings" "time" "github.com/FuturFusion/vsock" "github.com/shirou/gopsutil/v4/disk" "golang.org/x/sys/windows" "golang.org/x/sys/windows/registry" "golang.org/x/sys/windows/svc" "golang.org/x/sys/windows/svc/debug" "golang.org/x/sys/windows/svc/eventlog" "github.com/lxc/incus/v7/internal/ports" "github.com/lxc/incus/v7/internal/server/metrics" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) var ( // https://dev.to/cosmic_predator/writing-a-windows-service-in-go-1d1m osBaseWorkingDirectory = "C:\\" osAgentConfigPath = "C:\\Program Files\\Incus-Agent\\incus-agent.yml" osVioSerialPath = `\\.\org.linuxcontainers.incus` ) func osGetListener(port int64) (net.Listener, error) { const CIDAny uint32 = 4294967295 // Equivalent to VMADDR_CID_ANY. // Setup the listener on wildcard CID for inbound connections from Incus. // We use the VMADDR_CID_ANY CID so that if the VM's CID changes in the future the listener still works. // A CID change can occur when restoring a stateful VM that was previously using one CID but is // subsequently restored using a different one. l, err := vsock.ListenContextID(CIDAny, ports.HTTPSDefaultPort, nil) if err != nil { return nil, fmt.Errorf("WINDOWS: Failed to listen on vsock: %w", err) } logger.Info("Started vsock listener") return l, nil } // Start of Windows service code block // Inspired of https://github.com/golang/sys/blob/master/windows/svc/example/service.go var elog debug.Log type incusAgentService struct { agentCmd *cmdAgent } func (m *incusAgentService) Execute(args []string, r <-chan svc.ChangeRequest, changes chan<- svc.Status) (ssec bool, errno uint32) { const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown tick := time.Tick(2 * time.Second) changes <- svc.Status{State: svc.StartPending} d := newDaemon(m.agentCmd.global.flagLogDebug, m.agentCmd.global.flagLogVerbose, m.agentCmd.global.flagSecretsLocation) // Start the server. err := startHTTPServer(d, m.agentCmd.global.flagLogDebug) if err != nil { changes <- svc.Status{State: svc.StopPending} elog.Error(1, fmt.Sprintf("Failed to start HTTP server: %s", err)) return } changes <- svc.Status{State: svc.Running, Accepts: cmdsAccepted} ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() // Start status notifier in background. cancelStatusNotifier := m.agentCmd.startStatusNotifier(ctx, d.chConnected) defer cancelStatusNotifier() loop: for { select { case <-tick: case c := <-r: switch c.Cmd { case svc.Interrogate: changes <- c.CurrentStatus case svc.Stop, svc.Shutdown: break loop default: elog.Error(1, fmt.Sprintf("Unexpected control request #%d", c)) } } } changes <- svc.Status{State: svc.StopPending} return } func runService(name string, agentCmd *cmdAgent) error { var err error if agentCmd.global.flagLogDebug { elog = debug.New(name) } else { elog, err = eventlog.Open(name) if err != nil { return err } } defer elog.Close() elog.Info(1, fmt.Sprintf("Starting %s service", name)) run := svc.Run if agentCmd.global.flagLogDebug { run = debug.Run } err = run(name, &incusAgentService{agentCmd: agentCmd}) if err != nil { elog.Error(1, fmt.Sprintf("%s service failed: %v", name, err)) return err } elog.Info(1, fmt.Sprintf("%s service stopped", name)) return nil } // End of Windows service code block func osGetEnvironment() (*api.ServerEnvironment, error) { serverName, err := os.Hostname() if err != nil { return nil, err } env := &api.ServerEnvironment{ Kernel: "Windows", KernelArchitecture: runtime.GOARCH, Server: "incus-agent", ServerPid: os.Getpid(), ServerVersion: version.Version, ServerName: serverName, } return env, nil } func osMountShared(src string, dst string, fstype string, opts []string) error { return errors.New("Dynamic mounts aren't supported on Windows") } func osUmount(src string, dst string, fstype string) error { return errors.New("Dynamic mounts aren't supported on Windows") } func osGetFilesystemMetrics(d *Daemon) ([]metrics.FilesystemMetrics, error) { partitions, err := disk.Partitions(true) if err != nil { return nil, err } sort.Slice(partitions, func(i, j int) bool { return partitions[i].Mountpoint < partitions[j].Mountpoint }) fsMetrics := make([]metrics.FilesystemMetrics, 0, len(partitions)) for _, partition := range partitions { usage, err := disk.Usage(partition.Mountpoint) if err != nil { continue } fsMetrics = append(fsMetrics, metrics.FilesystemMetrics{ Device: partition.Device, Mountpoint: partition.Mountpoint, FSType: partition.Fstype, AvailableBytes: usage.Free, FreeBytes: usage.Free, SizeBytes: usage.Total, }) } return fsMetrics, nil } func osGetOSState() *api.InstanceStateOSInfo { // Get Windows registry. k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE) if err != nil { return nil } defer k.Close() // Get local hostname. hostname, err := os.Hostname() if err != nil { return nil } // Get build info. v := *windows.RtlGetVersion() osVersion, _, err := k.GetStringValue("CurrentVersion") if err != nil { return nil } osName, _, err := k.GetStringValue("ProductName") if err != nil { return nil } osBuild, _, err := k.GetStringValue("CurrentBuild") if err != nil { return nil } // Windows 11 always self-reports as Windows 10. // The documented diferentiator is the build ID. if v.BuildNumber > 22000 { osName = strings.Replace(osName, "Windows 10", "Windows 11", 1) } // Prepare OS struct. osInfo := &api.InstanceStateOSInfo{ OS: osName, OSVersion: osBuild, KernelVersion: osVersion, Hostname: hostname, FQDN: hostname, } return osInfo } func osGetInteractiveConsole(s *execWs) (io.ReadWriteCloser, io.ReadWriteCloser, error) { return nil, nil, errors.New("Only non-interactive exec sessions are currently supported on Windows") } func osPrepareExecCommand(s *execWs, cmd *exec.Cmd) { if s.cwd == "" { cmd.Dir = osBaseWorkingDirectory } return } func osHandleExecControl(control api.InstanceExecControl, s *execWs, pty io.ReadWriteCloser, cmd *exec.Cmd, l logger.Logger) { // Ignore control messages. return } func osExitStatus(err error) (int, error) { return 0, err } func osSetEnv(post *api.InstanceExecPost, env map[string]string) { // SystemRoot is already set by default env["SystemDrive"] = "C:" // Program Files directories env["ProgramFiles"] = fmt.Sprintf("%s\\Program Files", env["SystemDrive"]) env["ProgramFiles(x86)"] = fmt.Sprintf("%s (x86)", env["ProgramFiles"]) env["ProgramW6432"] = fmt.Sprintf("%s", env["ProgramFiles"]) env["CommonProgramFiles"] = fmt.Sprintf("%s\\Common Files", env["ProgramFiles"]) env["CommonProgramFiles(x86)"] = fmt.Sprintf("%s (x86)\\Common Files", env["ProgramFiles"]) env["CommonProgramW6432"] = fmt.Sprintf("%s\\Common Files", env["ProgramFiles"]) // Windows directories env["WINDIR"] = fmt.Sprintf("%s\\WINDOWS", env["SystemDrive"]) env["TMP"] = fmt.Sprintf("%s\\Temp", env["WINDIR"]) env["TEMP"] = env["TMP"] // System32 directories system32 := fmt.Sprintf("%s\\System32", env["WINDIR"]) env["ComSpec"] = fmt.Sprintf("%s\\cmd.exe", system32) env["DriverData"] = fmt.Sprintf("%s\\Drivers\\DriverData", system32) // User profile directories env["USERPROFILE"] = fmt.Sprintf("%s\\config\\systemprofile", system32) env["LOCALAPPDATA"] = fmt.Sprintf("%s\\AppData\\Local", env["USERPROFILE"]) env["APPDATA"] = fmt.Sprintf("%s\\AppData\\Roaming", env["USERPROFILE"]) // Miscellaneous env["COMPUTERNAME"] = "" env["PATH"] = fmt.Sprintf("%s;%s;%s\\WindowsPowerShell\\v1.0", system32, env["WINDIR"], system32) env["PATHEXT"] = "COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL" env["ProgramData"] = fmt.Sprintf("%s\\ProgramData", env["SystemDrive"]) env["ALLUSERSPROFILE"] = env["ProgramData"] env["PUBLIC"] = fmt.Sprintf("%s\\Users\\Public", env["SystemDrive"]) // Set the default working directory. if post.Cwd == "" { post.Cwd = system32 } } incus-7.0.0/cmd/incus-agent/response.go000066400000000000000000000012171517523235500200150ustar00rootroot00000000000000package main import ( "net/http" "github.com/lxc/incus/v7/shared/api" ) type devIncusResponse struct { content any code int ctype string } func errorResponse(code int, msg string) *devIncusResponse { return &devIncusResponse{msg, code, "raw"} } func okResponse(ct any, ctype string) *devIncusResponse { return &devIncusResponse{ct, http.StatusOK, ctype} } func smartResponse(err error) *devIncusResponse { if err == nil { return okResponse(nil, "") } statusCode, found := api.StatusErrorMatch(err) if found { return errorResponse(statusCode, err.Error()) } return errorResponse(http.StatusInternalServerError, err.Error()) } incus-7.0.0/cmd/incus-agent/server.go000066400000000000000000000057631517523235500174770ustar00rootroot00000000000000package main import ( "bytes" "crypto/tls" "crypto/x509" "errors" "fmt" "io" "net/http" "time" internalIO "github.com/lxc/incus/v7/internal/io" "github.com/lxc/incus/v7/internal/server/response" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) func restServer(tlsConfig *tls.Config, cert *x509.Certificate, debug bool, d *Daemon) *http.Server { router := http.NewServeMux() router.HandleFunc("/{$}", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _ = response.SyncResponse(true, []string{"/1.0"}).Render(w) }) for _, c := range api10 { createCmd(router, "1.0", c, cert, debug, d) } return &http.Server{ Handler: router, TLSConfig: tlsConfig, IdleTimeout: 30 * time.Second, ReadHeaderTimeout: 3 * time.Second, } } func createCmd(restAPI *http.ServeMux, version string, c APIEndpoint, cert *x509.Certificate, debug bool, d *Daemon) { var uri string if c.Path == "" { uri = fmt.Sprintf("/%s", version) } else { uri = fmt.Sprintf("/%s/%s", version, c.Path) } restAPI.HandleFunc(uri, func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") if !authenticate(r, cert) { logger.Error("Not authorized") _ = response.InternalError(errors.New("Not authorized")).Render(w) return } // Dump full request JSON when in debug mode if r.Method != "GET" && localUtil.IsJSONRequest(r) { newBody := &bytes.Buffer{} captured := &bytes.Buffer{} multiW := io.MultiWriter(newBody, captured) _, err := util.SafeCopy(multiW, r.Body) if err != nil { _ = response.InternalError(err).Render(w) return } r.Body = internalIO.BytesReadCloser{Buf: newBody} localUtil.DebugJSON("API Request", captured, logger.Log) } // Actually process the request var resp response.Response handleRequest := func(action APIEndpointAction) response.Response { if action.Handler == nil { return response.NotImplemented(nil) } return action.Handler(d, r) } switch r.Method { case "GET": resp = handleRequest(c.Get) case "PUT": resp = handleRequest(c.Put) case "POST": resp = handleRequest(c.Post) case "DELETE": resp = handleRequest(c.Delete) case "PATCH": resp = handleRequest(c.Patch) default: resp = response.NotFound(fmt.Errorf("Method %q not found", r.Method)) } // Handle errors err := resp.Render(w) if err != nil { writeErr := response.InternalError(err).Render(w) if writeErr != nil { logger.Error("Failed writing error for HTTP response", logger.Ctx{"url": uri, "error": err, "writeErr": writeErr}) } } }) } func authenticate(r *http.Request, cert *x509.Certificate) bool { clientCerts := map[string]x509.Certificate{"0": *cert} for _, cert := range r.TLS.PeerCertificates { trusted, _ := localUtil.CheckTrustState(*cert, clientCerts, nil, false) if trusted { return true } } return false } incus-7.0.0/cmd/incus-agent/sftp.go000066400000000000000000000030711517523235500171330ustar00rootroot00000000000000package main import ( "errors" "fmt" "net/http" "github.com/pkg/sftp" "github.com/lxc/incus/v7/internal/server/response" ) var sftpCmd = APIEndpoint{ Name: "sftp", Path: "sftp", Get: APIEndpointAction{Handler: sftpHandler}, } func sftpHandler(d *Daemon, r *http.Request) response.Response { if d.Features != nil && !d.Features["files"] { return response.Forbidden(errors.New("File transfers are disabled by configuration")) } return &sftpServe{d, r} } type sftpServe struct { d *Daemon r *http.Request } func (r *sftpServe) String() string { return "sftp handler" } // Code returns the HTTP code. func (r *sftpServe) Code() int { return http.StatusOK } func (r *sftpServe) Render(w http.ResponseWriter) error { // Upgrade to sftp. if r.r.Header.Get("Upgrade") != "sftp" { http.Error(w, "Missing or invalid upgrade header", http.StatusBadRequest) return nil } hijacker, ok := w.(http.Hijacker) if !ok { http.Error(w, "Webserver doesn't support hijacking", http.StatusInternalServerError) return nil } conn, _, err := hijacker.Hijack() if err != nil { http.Error(w, fmt.Errorf("Failed to hijack connection: %w", err).Error(), http.StatusInternalServerError) return nil } defer func() { _ = conn.Close() }() err = response.Upgrade(conn, "sftp") if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return nil } // Start sftp server. server, err := sftp.NewServer(conn, sftp.WithAllocator(), sftp.WithServerWorkingDirectory(osBaseWorkingDirectory)) if err != nil { return nil } return server.Serve() } incus-7.0.0/cmd/incus-agent/state.go000066400000000000000000000013701517523235500172770ustar00rootroot00000000000000package main import ( "errors" "net/http" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/shared/api" ) var stateCmd = APIEndpoint{ Name: "state", Path: "state", Get: APIEndpointAction{Handler: stateGet}, } func stateGet(d *Daemon, r *http.Request) response.Response { if d.Features != nil && !d.Features["state"] { return response.Forbidden(errors.New("Guest state reporting is disabled by configuration")) } return response.SyncResponse(true, renderState()) } func renderState() *api.InstanceState { return &api.InstanceState{ CPU: osGetCPUState(), Memory: osGetMemoryState(), Network: osGetNetworkState(), Pid: 1, Processes: osGetProcessesState(), OSInfo: osGetOSState(), } } incus-7.0.0/cmd/incus-agent/templates.go000066400000000000000000000056061517523235500201630ustar00rootroot00000000000000package main import ( "fmt" "io/fs" "os" "path/filepath" "strconv" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) func templatesApply(path string) ([]string, error) { metaName := filepath.Join(path, "metadata.yaml") if !util.PathExists(metaName) { return nil, nil } // Parse the metadata. content, err := os.ReadFile(metaName) if err != nil { return nil, fmt.Errorf("Failed to read metadata: %w", err) } metadata := &api.ImageMetadata{} err = yaml.Load(content, metadata) if err != nil { return nil, fmt.Errorf("Could not parse metadata.yaml: %w", err) } // Go through the files and copy them into place. files := []string{} for tplPath, tpl := range metadata.Templates { err = func(tplPath string, tpl *api.ImageMetadataTemplate) error { filePath := filepath.Join(path, fmt.Sprintf("%s.out", tpl.Template)) if !util.PathExists(filePath) { return nil } var w *os.File if util.PathExists(tplPath) { if tpl.CreateOnly { return nil } // Open the existing file. w, err = os.Create(tplPath) if err != nil { return fmt.Errorf("Failed to create template file: %w", err) } } else { // UID and GID fileUID := int64(0) fileGID := int64(0) if tpl.UID != "" { id, err := strconv.ParseInt(tpl.UID, 10, 64) if err != nil { return fmt.Errorf("Bad file UID %q for %q: %w", tpl.UID, tplPath, err) } fileUID = id } if tpl.GID != "" { id, err := strconv.ParseInt(tpl.GID, 10, 64) if err != nil { return fmt.Errorf("Bad file GID %q for %q: %w", tpl.GID, tplPath, err) } fileGID = id } // Mode fileMode := fs.FileMode(0o644) if tpl.Mode != "" { if len(tpl.Mode) == 3 { tpl.Mode = fmt.Sprintf("0%s", tpl.Mode) } mode, err := strconv.ParseInt(tpl.Mode, 0, 0) if err != nil { return fmt.Errorf("Bad mode %q for %q: %w", tpl.Mode, tplPath, err) } fileMode = os.FileMode(mode) & os.ModePerm } // Create the directories leading to the file. err := os.MkdirAll(filepath.Dir(tplPath), 0o755) if err != nil { return err } // Create the file itself. w, err = os.Create(tplPath) if err != nil { return err } // Fix ownership. err = w.Chown(int(fileUID), int(fileGID)) if err != nil { return err } // Fix mode. err = w.Chmod(fileMode) if err != nil { return err } } defer func() { _ = w.Close() }() // Do the copy. src, err := os.Open(filePath) if err != nil { return err } defer func() { _ = src.Close() }() _, err = util.SafeCopy(w, src) if err != nil { return err } err = w.Close() if err != nil { return err } files = append(files, tplPath) return nil }(tplPath, tpl) if err != nil { return nil, err } } return files, nil } incus-7.0.0/cmd/incus-benchmark/000077500000000000000000000000001517523235500164635ustar00rootroot00000000000000incus-7.0.0/cmd/incus-benchmark/benchmark.go000066400000000000000000000140761517523235500207540ustar00rootroot00000000000000package main import ( "fmt" "strings" "sync" "time" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" config "github.com/lxc/incus/v7/shared/cliconfig" ) const userConfigKey = "user.incus-benchmark" // printServerInfo prints out information about the server. func printServerInfo(c incus.InstanceServer) error { server, _, err := c.GetServer() if err != nil { return err } env := server.Environment fmt.Println("Test environment:") fmt.Println(" Server backend:", env.Server) fmt.Println(" Server version:", env.ServerVersion) fmt.Println(" Kernel:", env.Kernel) fmt.Println(" Kernel tecture:", env.KernelArchitecture) fmt.Println(" Kernel version:", env.KernelVersion) fmt.Println(" Storage backend:", env.Storage) fmt.Println(" Storage version:", env.StorageVersion) fmt.Println(" Container backend:", env.Driver) fmt.Println(" Container version:", env.DriverVersion) fmt.Println("") return nil } // launchContainers launches a set of containers. func launchContainers(c incus.InstanceServer, count int, parallel int, image string, privileged bool, start bool, freeze bool) (time.Duration, error) { var duration time.Duration batchSize, err := getBatchSize(parallel) if err != nil { return duration, err } printTestConfig(count, batchSize, image, privileged, freeze) fingerprint, err := ensureImage(c, image) if err != nil { return duration, err } batchStart := func(index int, wg *sync.WaitGroup) { defer wg.Done() name := getContainerName(count, index) err := createContainer(c, fingerprint, name, privileged) if err != nil { logf("Failed to launch container '%s': %s", name, err) return } if start { err := startContainer(c, name) if err != nil { logf("Failed to start container '%s': %s", name, err) return } if freeze { err := freezeContainer(c, name) if err != nil { logf("Failed to freeze container '%s': %s", name, err) return } } } } duration = processBatch(count, batchSize, batchStart) return duration, nil } // getContainers returns containers created by the benchmark. func getContainers(c incus.InstanceServer) ([]api.Instance, error) { containers := []api.Instance{} allContainers, err := c.GetInstances(api.InstanceTypeContainer) if err != nil { return containers, err } for _, container := range allContainers { if container.Config[userConfigKey] == "true" { containers = append(containers, container) } } return containers, nil } // startContainers starts containers created by the benchmark. func startContainers(c incus.InstanceServer, containers []api.Instance, parallel int) (time.Duration, error) { var duration time.Duration batchSize, err := getBatchSize(parallel) if err != nil { return duration, err } count := len(containers) logf("Starting %d containers", count) batchStart := func(index int, wg *sync.WaitGroup) { defer wg.Done() container := containers[index] if !container.IsActive() { err := startContainer(c, container.Name) if err != nil { logf("Failed to start container '%s': %s", container.Name, err) return } } } duration = processBatch(count, batchSize, batchStart) return duration, nil } // stopContainers stops containers created by the benchmark. func stopContainers(c incus.InstanceServer, containers []api.Instance, parallel int) (time.Duration, error) { var duration time.Duration batchSize, err := getBatchSize(parallel) if err != nil { return duration, err } count := len(containers) logf("Stopping %d containers", count) batchStop := func(index int, wg *sync.WaitGroup) { defer wg.Done() container := containers[index] if container.IsActive() { err := stopContainer(c, container.Name) if err != nil { logf("Failed to stop container '%s': %s", container.Name, err) return } } } duration = processBatch(count, batchSize, batchStop) return duration, nil } // deleteContainers removes containers created by the benchmark. func deleteContainers(c incus.InstanceServer, containers []api.Instance, parallel int) (time.Duration, error) { var duration time.Duration batchSize, err := getBatchSize(parallel) if err != nil { return duration, err } count := len(containers) logf("Deleting %d containers", count) batchDelete := func(index int, wg *sync.WaitGroup) { defer wg.Done() container := containers[index] name := container.Name if container.IsActive() { err := stopContainer(c, name) if err != nil { logf("Failed to stop container '%s': %s", name, err) return } } err = deleteContainer(c, name) if err != nil { logf("Failed to delete container: %s", name) return } } duration = processBatch(count, batchSize, batchDelete) return duration, nil } func ensureImage(c incus.InstanceServer, image string) (string, error) { var fingerprint string if strings.Contains(image, ":") { defaultConfig := config.NewConfig("", true) defaultConfig.UserAgent = version.UserAgent remote, fp, err := defaultConfig.ParseRemote(image) if err != nil { return "", err } fingerprint = fp imageServer, err := defaultConfig.GetImageServer(remote) if err != nil { return "", err } if fingerprint == "" { fingerprint = "default" } alias, _, err := imageServer.GetImageAlias(fingerprint) if err == nil { fingerprint = alias.Target } _, _, err = c.GetImage(fingerprint) if err != nil { logf("Importing image into local store: %s", fingerprint) image, _, err := imageServer.GetImage(fingerprint) if err != nil { logf("Failed to import image: %s", err) return "", err } err = copyImage(c, imageServer, *image) if err != nil { logf("Failed to import image: %s", err) return "", err } } } else { fingerprint = image alias, _, err := c.GetImageAlias(image) if err == nil { fingerprint = alias.Target } else { _, _, err = c.GetImage(image) } if err != nil { logf("Image not found in local store: %s", image) return "", err } } logf("Found image in local store: %s", fingerprint) return fingerprint, nil } incus-7.0.0/cmd/incus-benchmark/benchmark_batch.go000066400000000000000000000022431517523235500221060ustar00rootroot00000000000000package main import ( "os" "sync" "time" ) func getBatchSize(parallel int) (int, error) { batchSize := parallel if batchSize < 1 { // Detect the number of parallel actions cpus, err := os.ReadDir("/sys/bus/cpu/devices") if err != nil { return -1, err } batchSize = len(cpus) } return batchSize, nil } func processBatch(count int, batchSize int, process func(index int, wg *sync.WaitGroup)) time.Duration { batches := count / batchSize remainder := count % batchSize processed := 0 wg := sync.WaitGroup{} nextStat := batchSize logf("Batch processing start") timeStart := time.Now() for range batches { for range batchSize { wg.Add(1) go process(processed, &wg) processed++ } wg.Wait() if processed >= nextStat { interval := time.Since(timeStart).Seconds() logf("Processed %d containers in %.3fs (%.3f/s)", processed, interval, float64(processed)/interval) nextStat = nextStat * 2 } } for range remainder { wg.Add(1) go process(processed, &wg) processed++ } wg.Wait() timeEnd := time.Now() duration := timeEnd.Sub(timeStart) logf("Batch processing completed in %.3fs", duration.Seconds()) return duration } incus-7.0.0/cmd/incus-benchmark/benchmark_operation.go000066400000000000000000000030561517523235500230300ustar00rootroot00000000000000package main import ( incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/api" ) func createContainer(c incus.InstanceServer, fingerprint string, name string, privileged bool) error { config := map[string]string{} if privileged { config["security.privileged"] = "true" } config[userConfigKey] = "true" req := api.InstancesPost{ Name: name, Source: api.InstanceSource{ Type: "image", Fingerprint: fingerprint, }, } req.Config = config op, err := c.CreateInstance(req) if err != nil { return err } return op.Wait() } func startContainer(c incus.InstanceServer, name string) error { op, err := c.UpdateInstanceState( name, api.InstanceStatePut{Action: "start", Timeout: -1}, "") if err != nil { return err } return op.Wait() } func stopContainer(c incus.InstanceServer, name string) error { op, err := c.UpdateInstanceState( name, api.InstanceStatePut{Action: "stop", Timeout: -1, Force: true}, "") if err != nil { return err } return op.Wait() } func freezeContainer(c incus.InstanceServer, name string) error { op, err := c.UpdateInstanceState( name, api.InstanceStatePut{Action: "freeze", Timeout: -1}, "") if err != nil { return err } return op.Wait() } func deleteContainer(c incus.InstanceServer, name string) error { op, err := c.DeleteInstance(name) if err != nil { return err } return op.Wait() } func copyImage(c incus.InstanceServer, s incus.ImageServer, image api.Image) error { op, err := c.CopyImage(s, image, nil) if err != nil { return err } return op.Wait() } incus-7.0.0/cmd/incus-benchmark/benchmark_report.go000066400000000000000000000040651517523235500223440ustar00rootroot00000000000000package main import ( "encoding/csv" "fmt" "io" "os" "time" ) // Subset of JMeter CSV log format that are required by Jenkins performance // plugin // (see http://jmeter.apache.org/usermanual/listeners.html#csvlogformat) var csvFields = []string{ "timeStamp", // in milliseconds since 1/1/1970 "elapsed", // in milliseconds "label", "responseCode", "success", // "true" or "false" } // CSVReport reads/writes a CSV report file. type CSVReport struct { Filename string records [][]string } // Load reads current content of the filename and loads records. func (r *CSVReport) load() error { file, err := os.Open(r.Filename) if err != nil { return err } defer func() { _ = file.Close() }() reader := csv.NewReader(file) for line := 1; err != io.EOF; line++ { record, err := reader.Read() if err == io.EOF { break } else if err != nil { return err } err = r.appendRecord(record) if err != nil { return err } } logf("Loaded report file %s", r.Filename) return nil } // Write writes current records to file. func (r *CSVReport) write() error { file, err := os.OpenFile(r.Filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640) if err != nil { return err } defer func() { _ = file.Close() }() writer := csv.NewWriter(file) err = writer.WriteAll(r.records) if err != nil { return err } logf("Written report file %s", r.Filename) return file.Close() } // AddRecord adds a record to the report. func (r *CSVReport) addRecord(label string, elapsed time.Duration) error { if len(r.records) == 0 { err := r.appendRecord(csvFields) if err != nil { return err } } record := []string{ fmt.Sprintf("%d", time.Now().UnixNano()/int64(time.Millisecond)), // timestamp fmt.Sprintf("%d", elapsed/time.Millisecond), label, "", // responseCode is not used "true", // success" } return r.appendRecord(record) } func (r *CSVReport) appendRecord(record []string) error { if len(record) != len(csvFields) { return fmt.Errorf("Invalid number of fields : %q", record) } r.records = append(r.records, record) return nil } incus-7.0.0/cmd/incus-benchmark/benchmark_util.go000066400000000000000000000017471517523235500220120ustar00rootroot00000000000000package main import ( "fmt" "time" ) func getContainerName(count int, index int) string { nameFormat := "benchmark-%." + fmt.Sprintf("%d", len(fmt.Sprintf("%d", count))) + "d" return fmt.Sprintf(nameFormat, index+1) } func logf(format string, args ...any) { fmt.Printf(fmt.Sprintf("[%s] %s\n", time.Now().Format(time.StampMilli), format), args...) } func printTestConfig(count int, batchSize int, image string, privileged bool, freeze bool) { privilegedStr := "unprivileged" if privileged { privilegedStr = "privileged" } mode := "normal startup" if freeze { mode = "start and freeze" } batches := count / batchSize remainder := count % batchSize fmt.Println("Test variables:") fmt.Println(" Container count:", count) fmt.Println(" Container mode:", privilegedStr) fmt.Println(" Startup mode:", mode) fmt.Println(" Image:", image) fmt.Println(" Batches:", batches) fmt.Println(" Batch size:", batchSize) fmt.Println(" Remainder:", remainder) fmt.Println("") } incus-7.0.0/cmd/incus-benchmark/main.go000066400000000000000000000073501517523235500177430ustar00rootroot00000000000000package main import ( "os" "time" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) type cmdGlobal struct { flagHelp bool flagParallel int flagProject string flagReportFile string flagReportLabel string flagVersion bool srv incus.InstanceServer report *CSVReport reportDuration time.Duration } func (c *cmdGlobal) run(cmd *cobra.Command, args []string) error { // Connect to the daemon srv, err := incus.ConnectIncusUnix("", nil) if err != nil { return err } c.srv = srv.UseProject(c.flagProject) // Print the initial header err = printServerInfo(srv) if err != nil { return err } // Setup report handling if c.flagReportFile != "" { c.report = &CSVReport{Filename: c.flagReportFile} if util.PathExists(c.flagReportFile) { err := c.report.load() if err != nil { return err } } } return nil } func (c *cmdGlobal) teardown(cmd *cobra.Command, args []string) error { // Nothing to do with not reporting if c.report == nil { return nil } label := cmd.Name() if c.flagReportLabel != "" { label = c.flagReportLabel } err := c.report.addRecord(label, c.reportDuration) if err != nil { return err } err = c.report.write() if err != nil { return err } return nil } func main() { app := &cobra.Command{} app.Use = "incus-benchmark" app.Short = "Benchmark performance of Incus" app.Long = `Description: Benchmark performance of Incus This tool lets you benchmark various actions on a local Incus daemon. It can be used just to check how fast a given host is, to compare performance on different servers or for performance tracking when doing changes to the codebase. A CSV report can be produced to be consumed by graphing software. ` app.Example = ` # Spawn 20 containers in batches of 4 incus-benchmark launch --count 20 --parallel 4 # Create 50 Alpine containers in batches of 10 incus-benchmark init --count 50 --parallel 10 images:alpine/edge # Delete all test containers using dynamic batch size incus-benchmark delete` app.SilenceUsage = true app.CompletionOptions = cobra.CompletionOptions{DisableDefaultCmd: true} // Global flags globalCmd := cmdGlobal{} app.PersistentPreRunE = globalCmd.run app.PersistentPostRunE = globalCmd.teardown app.PersistentFlags().BoolVar(&globalCmd.flagVersion, "version", false, "Print version number") app.PersistentFlags().BoolVarP(&globalCmd.flagHelp, "help", "h", false, "Print help") app.PersistentFlags().IntVarP(&globalCmd.flagParallel, "parallel", "P", -1, "Number of threads to use"+"``") app.PersistentFlags().StringVar(&globalCmd.flagReportFile, "report-file", "", "Path to the CSV report file"+"``") app.PersistentFlags().StringVar(&globalCmd.flagReportLabel, "report-label", "", "Label for the new entry in the report [default=ACTION]"+"``") app.PersistentFlags().StringVar(&globalCmd.flagProject, "project", api.ProjectDefaultName, "Project to use") // Version handling app.SetVersionTemplate("{{.Version}}\n") app.Version = version.Version // init sub-command initCmd := cmdInit{global: &globalCmd} app.AddCommand(initCmd.command()) // launch sub-command launchCmd := cmdLaunch{global: &globalCmd, init: &initCmd} app.AddCommand(launchCmd.command()) // start sub-command startCmd := cmdStart{global: &globalCmd} app.AddCommand(startCmd.command()) // stop sub-command stopCmd := cmdStop{global: &globalCmd} app.AddCommand(stopCmd.command()) // delete sub-command deleteCmd := cmdDelete{global: &globalCmd} app.AddCommand(deleteCmd.command()) // Run the main command and handle errors err := app.Execute() if err != nil { os.Exit(1) } } incus-7.0.0/cmd/incus-benchmark/main_delete.go000066400000000000000000000011471517523235500212630ustar00rootroot00000000000000package main import ( "github.com/spf13/cobra" ) type cmdDelete struct { global *cmdGlobal } func (c *cmdDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "delete" cmd.Short = "Delete containers" cmd.RunE = c.run return cmd } func (c *cmdDelete) run(cmd *cobra.Command, args []string) error { // Get the containers containers, err := getContainers(c.global.srv) if err != nil { return err } // Run the test duration, err := deleteContainers(c.global.srv, containers, c.global.flagParallel) if err != nil { return err } c.global.reportDuration = duration return nil } incus-7.0.0/cmd/incus-benchmark/main_init.go000066400000000000000000000015561517523235500207700ustar00rootroot00000000000000package main import ( "github.com/spf13/cobra" ) type cmdInit struct { global *cmdGlobal flagCount int flagPrivileged bool } func (c *cmdInit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "init [[:]]" cmd.Short = "Create containers" cmd.RunE = c.run cmd.Flags().IntVarP(&c.flagCount, "count", "C", 1, "Number of containers to create"+"``") cmd.Flags().BoolVar(&c.flagPrivileged, "privileged", false, "Use privileged containers") return cmd } func (c *cmdInit) run(cmd *cobra.Command, args []string) error { // Choose the image image := "images:debian/12" if len(args) > 0 { image = args[0] } // Run the test duration, err := launchContainers(c.global.srv, c.flagCount, c.global.flagParallel, image, c.flagPrivileged, false, false) if err != nil { return err } c.global.reportDuration = duration return nil } incus-7.0.0/cmd/incus-benchmark/main_launch.go000066400000000000000000000015531517523235500212740ustar00rootroot00000000000000package main import ( "github.com/spf13/cobra" ) type cmdLaunch struct { global *cmdGlobal init *cmdInit flagFreeze bool } func (c *cmdLaunch) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "launch [[:]]" cmd.Short = "Create and start containers" cmd.RunE = c.run cmd.Flags().AddFlagSet(c.init.command().Flags()) cmd.Flags().BoolVarP(&c.flagFreeze, "freeze", "F", false, "Freeze the container right after start") return cmd } func (c *cmdLaunch) run(cmd *cobra.Command, args []string) error { // Choose the image image := "images:debian/12" if len(args) > 0 { image = args[0] } // Run the test duration, err := launchContainers(c.global.srv, c.init.flagCount, c.global.flagParallel, image, c.init.flagPrivileged, true, c.flagFreeze) if err != nil { return err } c.global.reportDuration = duration return nil } incus-7.0.0/cmd/incus-benchmark/main_start.go000066400000000000000000000011411517523235500211500ustar00rootroot00000000000000package main import ( "github.com/spf13/cobra" ) type cmdStart struct { global *cmdGlobal } func (c *cmdStart) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "start" cmd.Short = "Start containers" cmd.RunE = c.run return cmd } func (c *cmdStart) run(cmd *cobra.Command, args []string) error { // Get the containers containers, err := getContainers(c.global.srv) if err != nil { return err } // Run the test duration, err := startContainers(c.global.srv, containers, c.global.flagParallel) if err != nil { return err } c.global.reportDuration = duration return nil } incus-7.0.0/cmd/incus-benchmark/main_stop.go000066400000000000000000000011331517523235500210010ustar00rootroot00000000000000package main import ( "github.com/spf13/cobra" ) type cmdStop struct { global *cmdGlobal } func (c *cmdStop) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "stop" cmd.Short = "Stop containers" cmd.RunE = c.run return cmd } func (c *cmdStop) run(cmd *cobra.Command, args []string) error { // Get the containers containers, err := getContainers(c.global.srv) if err != nil { return err } // Run the test duration, err := stopContainers(c.global.srv, containers, c.global.flagParallel) if err != nil { return err } c.global.reportDuration = duration return nil } incus-7.0.0/cmd/incus-migrate/000077500000000000000000000000001517523235500161615ustar00rootroot00000000000000incus-7.0.0/cmd/incus-migrate/cgo.go000066400000000000000000000006401517523235500172600ustar00rootroot00000000000000// build +linux,cgo package main // #cgo CFLAGS: -std=gnu11 -Wvla -Werror -fvisibility=hidden -Winit-self // #cgo CFLAGS: -Wformat=2 -Wshadow -Wendif-labels -fasynchronous-unwind-tables // #cgo CFLAGS: -pipe --param=ssp-buffer-size=4 -g -Wunused // #cgo CFLAGS: -Werror=implicit-function-declaration // #cgo CFLAGS: -Werror=return-type -Wendif-labels -Werror=overflow // #cgo CFLAGS: -Wnested-externs -fexceptions incus-7.0.0/cmd/incus-migrate/main.go000066400000000000000000000021151517523235500174330ustar00rootroot00000000000000package main import ( "bufio" "os" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/ask" ) type cmdGlobal struct { asker ask.Asker flagVersion bool flagHelp bool } func main() { // migrate command (main) migrateCmd := cmdMigrate{} app := migrateCmd.command() app.SilenceUsage = true app.CompletionOptions = cobra.CompletionOptions{DisableDefaultCmd: true} // Workaround for main command app.Args = cobra.ArbitraryArgs // Global flags globalCmd := cmdGlobal{asker: ask.NewAsker(bufio.NewReader(os.Stdin))} migrateCmd.global = &globalCmd app.PersistentFlags().BoolVar(&globalCmd.flagVersion, "version", false, "Print version number") app.PersistentFlags().BoolVarP(&globalCmd.flagHelp, "help", "h", false, "Print help") // Version handling app.SetVersionTemplate("{{.Version}}\n") app.Version = version.Version // netcat sub-command netcatCmd := cmdNetcat{global: &globalCmd} app.AddCommand(netcatCmd.command()) // Run the main command and handle errors err := app.Execute() if err != nil { os.Exit(1) } } incus-7.0.0/cmd/incus-migrate/main_migrate.go000066400000000000000000000322151517523235500211470ustar00rootroot00000000000000package main import ( "bufio" "context" "errors" "fmt" "net/http" "net/url" "os" "os/exec" "os/signal" "path/filepath" "runtime" "slices" "sort" "github.com/spf13/cobra" "golang.org/x/sys/unix" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" "github.com/lxc/incus/v7/shared/ask" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) // Migrator defines the methods required to perform a migration. type Migrator interface { gatherInfo() error migrate() error renderObject() error } // Migration is a base representation of a migration, which can be extended by more specific structs. type Migration struct { asker ask.Asker ctx context.Context migrationType MigrationType mounts []string pool string project string server incus.InstanceServer sourceFormat string sourcePath string target string } func (m *Migration) runMigration(migrationHandler func(path string) error) error { // Create the temporary directory to be used for the mounts path, err := os.MkdirTemp("", "incus-migrate_mount_") if err != nil { return err } var fullPath string if m.migrationType == MigrationTypeContainer || m.migrationType == MigrationTypeVolumeFilesystem { m.mounts = append(m.mounts, m.sourcePath) // Get and sort the mounts. sort.Strings(m.mounts) // Ensure we're not moved around. runtime.LockOSThread() defer runtime.UnlockOSThread() // Unshare a new mntns so our mounts don't leak. err := unix.Unshare(unix.CLONE_NEWNS) if err != nil { return fmt.Errorf("Failed to unshare mount namespace: %w", err) } // Prevent mount propagation back to initial namespace err = unix.Mount("", "/", "", unix.MS_REC|unix.MS_PRIVATE, "") if err != nil { return fmt.Errorf("Failed to disable mount propagation: %w", err) } // Automatically clean-up the temporary path on exit defer func(path string) { // Unmount the path if it's a mountpoint. _ = unix.Unmount(path, unix.MNT_DETACH) _ = unix.Unmount(filepath.Join(path, "root.img"), unix.MNT_DETACH) // Cleanup VM image files. _ = os.Remove(filepath.Join(path, "converted-raw-image.img")) _ = os.Remove(filepath.Join(path, "root.img")) // Remove the directory itself. _ = os.Remove(path) }(path) // Create the rootfs directory fullPath = fmt.Sprintf("%s/rootfs", path) err = os.Mkdir(fullPath, 0o755) if err != nil { return err } // Setup the source (mounts) err = setupSource(fullPath, m.mounts) if err != nil { return fmt.Errorf("Failed to setup the source: %w", err) } } else { _, ext, convCmd, _ := archive.DetectCompression(m.sourcePath) if ext == ".qcow2" || ext == ".vmdk" { // COnfirm the command is available. _, err := exec.LookPath(convCmd[0]) if err != nil { return fmt.Errorf("Unable to find required command %q", convCmd[0]) } destImg := filepath.Join(path, "converted-raw-image.img") cmd := []string{ "nice", "-n19", // Run with low priority to reduce CPU impact on other processes. } cmd = append(cmd, convCmd...) cmd = append(cmd, "-p", "-t", "writeback") // Check for Direct I/O support. from, err := os.OpenFile(m.sourcePath, unix.O_DIRECT|unix.O_RDONLY, 0) if err == nil { cmd = append(cmd, "-T", "none") _ = from.Close() } to, err := os.OpenFile(destImg, unix.O_DIRECT|unix.O_RDONLY, 0) if err == nil { cmd = append(cmd, "-t", "none") _ = to.Close() } cmd = append(cmd, m.sourcePath, destImg) fmt.Printf("Converting image %q to raw format before importing\n", m.sourcePath) c := exec.Command(cmd[0], cmd[1:]...) err = c.Run() if err != nil { return fmt.Errorf("Failed to convert image %q for importing: %w", m.sourcePath, err) } m.sourcePath = destImg } err = os.Symlink(m.sourcePath, filepath.Join(path, "root.img")) if err != nil { return err } fullPath = path } return migrationHandler(fullPath) } func (m *Migration) setSourceFormat() error { if m.sourcePath == "" { return errors.New("Missing source path") } if m.migrationType == "" { return errors.New("Missing migration type") } // When migrating a disk, report the detected source format if m.migrationType == MigrationTypeVM || m.migrationType == MigrationTypeVolumeBlock { if linux.IsBlockdevPath(m.sourcePath) { m.sourceFormat = "Block device" } else if _, ext, _, _ := archive.DetectCompression(m.sourcePath); ext == ".qcow2" { m.sourceFormat = "qcow2" } else if _, ext, _, _ := archive.DetectCompression(m.sourcePath); ext == ".vmdk" { m.sourceFormat = "vmdk" } else { // If the input isn't a block device or qcow2/vmdk image, assume it's raw. // Positively identifying a raw image depends on parsing MBR/GPT partition tables. m.sourceFormat = "raw" } } return nil } func (m *Migration) askTarget() error { if !m.server.IsClustered() { return nil } ok, err := m.asker.AskBool("Would you like to target a specific server or group in the cluster? [default=no]: ", "no") if err != nil { return err } if !ok { return nil } clusterTarget, err := m.asker.AskString("Target name: ", "", nil) if err != nil { return err } m.target = clusterTarget return nil } func (m *Migration) askPath(question string) (string, error) { isURL := false path, err := m.asker.AskString(question, "", func(s string) error { // Allow URLs. u, err := url.Parse(s) if err == nil && u.Scheme != "" && u.Host != "" { isURL = true return nil } // Check if a valid path. if !util.PathExists(s) { return errors.New("Path does not exist") } _, err = os.Stat(s) if err != nil { return err } return nil }) if err != nil { return "", err } // If a URL, download it. if isURL { // Create a temporary file. f, err := os.CreateTemp("", "") if err != nil { return "", err } defer func() { _ = f.Close() }() // Download the target. resp, err := http.Get(path) if err != nil { return "", err } defer resp.Body.Close() fmt.Printf("Downloading %q\n", path) _, err = util.SafeCopy(f, resp.Body) if err != nil { _ = os.Remove(f.Name()) return "", err } path = f.Name() } return path, nil } func (m *Migration) askProject(question string) error { projectNames, err := m.server.GetProjectNames() if err != nil { return err } if len(projectNames) > 1 { project, err := m.asker.AskChoice(question, projectNames, api.ProjectDefaultName) if err != nil { return err } m.project = project return nil } m.project = api.ProjectDefaultName return nil } type cmdMigrate struct { global *cmdGlobal flagRsyncArgs string } func (c *cmdMigrate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "incus-migrate" cmd.Short = "Physical to instance migration tool" cmd.Long = `Description: Physical to instance migration tool This tool lets you turn any Linux filesystem (including your current one) into an instance on a remote host. It will setup a clean mount tree made of the root filesystem and any additional mount you list, then transfer this through the migration API to create a new instance from it. The same set of options as ` + "`incus launch`" + ` are also supported. ` cmd.RunE = c.run cmd.Flags().StringVar(&c.flagRsyncArgs, "rsync-args", "", "Extra arguments to pass to rsync (for file transfers)"+"``") return cmd } func (c *cmdMigrate) askServer() (incus.InstanceServer, string, error) { // Detect local server. local, err := c.connectLocal() if err == nil { useLocal, err := c.global.asker.AskBool("The local Incus server is the target [default=yes]: ", "yes") if err != nil { return nil, "", err } if useLocal { return local, "", nil } } // Server address serverURL, err := c.global.asker.AskString("Please provide Incus server URL: ", "", nil) if err != nil { return nil, "", err } serverURL, err = parseURL(serverURL) if err != nil { return nil, "", err } args := incus.ConnectionArgs{ UserAgent: fmt.Sprintf("LXC-MIGRATE %s", version.Version), } // Attempt to connect server, err := incus.ConnectIncus(serverURL, &args) if err != nil { // Failed to connect using the system CA, so retrieve the remote certificate. certificate, err := localtls.GetRemoteCertificate(serverURL, args.UserAgent) if err != nil { return nil, "", fmt.Errorf("Failed to get remote certificate: %w", err) } digest := localtls.CertFingerprint(certificate) fmt.Println("Certificate fingerprint:", digest) fmt.Print("ok (y/n)? ") buf := bufio.NewReader(os.Stdin) line, _, err := buf.ReadLine() if err != nil { return nil, "", err } if len(line) < 1 || line[0] != 'y' && line[0] != 'Y' { return nil, "", errors.New("Server certificate rejected by user") } args.InsecureSkipVerify = true server, err = incus.ConnectIncus(serverURL, &args) if err != nil { return nil, "", fmt.Errorf("Failed to connect to server: %w", err) } } apiServer, _, err := server.GetServer() if err != nil { return nil, "", fmt.Errorf("Failed to get server: %w", err) } fmt.Println("") type AuthMethod int const ( authMethodTLSCertificate AuthMethod = iota authMethodTLSTemporaryCertificate authMethodTLSCertificateToken ) // TLS is always available var availableAuthMethods []AuthMethod var authMethod AuthMethod i := 1 if slices.Contains(apiServer.AuthMethods, api.AuthenticationMethodTLS) { fmt.Printf("%d) Use a certificate token\n", i) availableAuthMethods = append(availableAuthMethods, authMethodTLSCertificateToken) i++ fmt.Printf("%d) Use an existing TLS authentication certificate\n", i) availableAuthMethods = append(availableAuthMethods, authMethodTLSCertificate) i++ fmt.Printf("%d) Generate a temporary TLS authentication certificate\n", i) availableAuthMethods = append(availableAuthMethods, authMethodTLSTemporaryCertificate) } if len(apiServer.AuthMethods) > 1 || slices.Contains(apiServer.AuthMethods, api.AuthenticationMethodTLS) { authMethodInt, err := c.global.asker.AskInt("Please pick an authentication mechanism above: ", 1, int64(i), "", nil) if err != nil { return nil, "", err } authMethod = availableAuthMethods[authMethodInt-1] } var certPath string var keyPath string var token string switch authMethod { case authMethodTLSCertificate: certPath, err = c.global.asker.AskString("Please provide the certificate path: ", "", func(path string) error { if !util.PathExists(path) { return errors.New("File does not exist") } return nil }) if err != nil { return nil, "", err } keyPath, err = c.global.asker.AskString("Please provide the keyfile path: ", "", func(path string) error { if !util.PathExists(path) { return errors.New("File does not exist") } return nil }) if err != nil { return nil, "", err } case authMethodTLSCertificateToken: token, err = c.global.asker.AskString("Please provide the certificate token: ", "", func(token string) error { _, err := localtls.CertificateTokenDecode(token) if err != nil { return err } return nil }) if err != nil { return nil, "", err } case authMethodTLSTemporaryCertificate: // Intentionally ignored } var authType string switch authMethod { case authMethodTLSCertificate, authMethodTLSTemporaryCertificate, authMethodTLSCertificateToken: authType = api.AuthenticationMethodTLS } return c.connectTarget(serverURL, certPath, keyPath, authType, token) } func (c *cmdMigrate) run(_ *cobra.Command, _ []string) error { // Server server, clientFingerprint, err := c.askServer() if err != nil { return err } sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, os.Interrupt) ctx, cancel := context.WithCancel(context.Background()) go func() { <-sigChan if clientFingerprint != "" { _ = server.DeleteCertificate(clientFingerprint) } cancel() // The following nolint directive ignores the "deep-exit" rule of the revive linter. // We should be exiting cleanly by passing the above context into each invoked method and checking for // cancellation. Unfortunately our client methods do not accept a context argument. os.Exit(1) //nolint:revive }() if clientFingerprint != "" { defer func() { _ = server.DeleteCertificate(clientFingerprint) }() } // Provide migration type creationType, err := c.global.asker.AskInt(` What would you like to create? 1) Container 2) Virtual Machine 3) Virtual Machine (from .ova) 4) Custom Volume Please enter the number of your choice: `, 1, 4, "", nil) if err != nil { return err } var migrator Migrator switch creationType { case 1: migrator = newInstanceMigration(ctx, server, c.global.asker, c.flagRsyncArgs, MigrationTypeContainer) case 2: migrator = newInstanceMigration(ctx, server, c.global.asker, c.flagRsyncArgs, MigrationTypeVM) case 3: migrator = newOVAMigration(ctx, server, c.global.asker, c.flagRsyncArgs) case 4: migrator = newVolumeMigration(ctx, server, c.global.asker, c.flagRsyncArgs) } err = migrator.gatherInfo() if err != nil { return err } err = migrator.renderObject() if err != nil { return err } return migrator.migrate() } incus-7.0.0/cmd/incus-migrate/main_netcat.go000066400000000000000000000023451517523235500207760ustar00rootroot00000000000000package main import ( "errors" "net" "os" "sync" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/eagain" "github.com/lxc/incus/v7/shared/util" ) type cmdNetcat struct { global *cmdGlobal } func (c *cmdNetcat) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "netcat
" cmd.Short = "Sends stdin data to a unix socket" cmd.RunE = c.run cmd.Hidden = true return cmd } func (c *cmdNetcat) run(cmd *cobra.Command, args []string) error { // Help and usage if len(args) == 0 { _ = cmd.Help() return nil } // Handle mandatory arguments if len(args) != 1 { _ = cmd.Help() return errors.New("Missing required argument") } // Connect to the provided address uAddr, err := net.ResolveUnixAddr("unix", args[0]) if err != nil { return err } conn, err := net.DialUnix("unix", nil, uAddr) if err != nil { return err } // We'll wait until we're done reading from the socket wg := sync.WaitGroup{} wg.Add(1) go func() { _, err = util.SafeCopy(eagain.Writer{Writer: os.Stdout}, eagain.Reader{Reader: conn}) _ = conn.Close() wg.Done() }() go func() { _, _ = util.SafeCopy(eagain.Writer{Writer: conn}, eagain.Reader{Reader: os.Stdin}) }() // Wait wg.Wait() return err } incus-7.0.0/cmd/incus-migrate/migrate_instance.go000066400000000000000000000314571517523235500220360ustar00rootroot00000000000000package main import ( "bufio" "context" "errors" "fmt" "os" "os/exec" "slices" "strings" "go.yaml.in/yaml/v4" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/ask" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // InstanceMigration handles the migration logic for an instance. type InstanceMigration struct { *Migration flagRsyncArgs string instanceArgs api.InstancesPost volumes []*VolumeMigration } // newInstanceMigration returns a new InstanceMigration. func newInstanceMigration(ctx context.Context, server incus.InstanceServer, asker ask.Asker, flafRsyncArgs string, migraionType MigrationType) Migrator { return &InstanceMigration{ Migration: &Migration{ asker: asker, ctx: ctx, server: server, migrationType: migraionType, }, flagRsyncArgs: flafRsyncArgs, } } // gatherInfo collects information from the user about the instance to be created. func (m *InstanceMigration) gatherInfo() error { var err error // Quick checks. if m.migrationType == MigrationTypeContainer { if os.Geteuid() != 0 { return errors.New("This tool must be run as root for container migrations") } _, err := exec.LookPath("rsync") if err != nil { return errors.New("Unable to find required command \"rsync\"") } } m.instanceArgs = api.InstancesPost{ Source: api.InstanceSource{ Type: "migration", Mode: "push", }, } m.instanceArgs.Config = map[string]string{} m.instanceArgs.Devices = map[string]map[string]string{} if m.migrationType == MigrationTypeVM { m.instanceArgs.Type = api.InstanceTypeVM } else { m.instanceArgs.Type = api.InstanceTypeContainer } // Project err = m.askProject("Project to create the instance in [default=default]: ") if err != nil { return err } if m.project != "" { m.server = m.server.UseProject(m.project) } // Target err = m.askTarget() if err != nil { return err } m.server = m.server.UseTarget(m.target) // Instance name instanceNames, err := m.server.GetInstanceNames(api.InstanceTypeAny) if err != nil { return err } for { instanceName, err := m.asker.AskString("Name of the new instance: ", "", nil) if err != nil { return err } if slices.Contains(instanceNames, instanceName) { fmt.Printf("Instance %q already exists\n", instanceName) continue } m.instanceArgs.Name = instanceName break } var question string // Provide source path if m.migrationType == MigrationTypeVM || m.migrationType == MigrationTypeVolumeBlock { question = "Please provide the path or URL to a disk, partition, or qcow2/raw/vmdk image file: " } else { question = "Please provide the path to a root filesystem: " } // Provide source path m.sourcePath, err = m.askPath(question) if err != nil { return err } err = m.setSourceFormat() if err != nil { return err } err = m.askUEFISupport() if err != nil { return err } var mounts []string // Additional mounts for containers if m.instanceArgs.Type == api.InstanceTypeContainer { addMounts, err := m.asker.AskBool("Do you want to add additional filesystem mounts? [default=no]: ", "no") if err != nil { return err } if addMounts { for { path, err := m.asker.AskString("Please provide a path the filesystem mount path [empty value to continue]: ", "", func(s string) error { if s != "" { if util.PathExists(s) { return nil } return errors.New("Path does not exist") } return nil }) if err != nil { return err } if path == "" { break } mounts = append(mounts, path) } m.mounts = append(m.mounts, mounts...) } } return nil } // migrate performs the instance migration. func (m *InstanceMigration) migrate() error { if m.migrationType != MigrationTypeVM && m.migrationType != MigrationTypeContainer { return errors.New("Wrong migration type for migrate") } // Prioritize migrating all additional disks before the main instance. for _, vol := range m.volumes { err := vol.migrate() if err != nil { return err } } return m.runMigration(func(path string) error { // System architecture architectureName, err := osarch.ArchitectureGetLocal() if err != nil { return err } m.instanceArgs.Architecture = architectureName reverter := revert.New() defer reverter.Fail() // Create the instance op, err := m.server.CreateInstance(m.instanceArgs) if err != nil { return err } reverter.Add(func() { _, _ = m.server.DeleteInstance(m.instanceArgs.Name) }) progress := cli.ProgressRenderer{Format: "Transferring instance: %s"} _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } err = transferRootfs(m.ctx, op, path, m.flagRsyncArgs, m.migrationType) if err != nil { return err } progress.Done(fmt.Sprintf("Instance %s successfully created", m.instanceArgs.Name)) reverter.Success() return nil }) } // renderObject renders the state of the instance. func (m *InstanceMigration) renderObject() error { for { fmt.Println("\nInstance to be created:") scanner := bufio.NewScanner(strings.NewReader(m.render())) for scanner.Scan() { fmt.Printf(" %s\n", scanner.Text()) } fmt.Print(` Additional overrides can be applied at this stage: 1) Begin the migration with the above configuration 2) Override profile list 3) Set additional configuration options 4) Change instance storage pool or volume size 5) Change instance network 6) Add additional disk 7) Change additional disk storage pool `) choice, err := m.asker.AskInt("Please pick one of the options above [default=1]: ", 1, 6, "1", nil) if err != nil { return err } switch choice { case 1: return nil case 2: err = m.askProfiles() case 3: err = m.askConfig() case 4: err = m.askStorage() case 5: err = m.askNetwork() case 6: err = m.askDisk() case 7: err = m.askDiskStorage() } if err != nil { fmt.Println(err) } } } func (m *InstanceMigration) render() string { data := struct { Name string `yaml:"Name"` Project string `yaml:"Project"` Type api.InstanceType `yaml:"Type"` Source string `yaml:"Source"` SourceFormat string `yaml:"Source format,omitempty"` Mounts []string `yaml:"Mounts,omitempty"` Profiles []string `yaml:"Profiles,omitempty"` StoragePool string `yaml:"Storage pool,omitempty"` StorageSize string `yaml:"Storage pool size,omitempty"` Network string `yaml:"Network name,omitempty"` Config map[string]string `yaml:"Config,omitempty"` Disks map[string]map[string]string `yaml:"Disks,omitempty"` }{ m.instanceArgs.Name, m.project, m.instanceArgs.Type, m.sourcePath, m.sourceFormat, m.mounts, m.instanceArgs.Profiles, "", "", "", m.instanceArgs.Config, make(map[string]map[string]string), } disk, ok := m.instanceArgs.Devices["root"] if ok { data.StoragePool = disk["pool"] size, ok := disk["size"] if ok { data.StorageSize = size } } network, ok := m.instanceArgs.Devices["eth0"] if ok { data.Network = network["parent"] } for k, v := range m.instanceArgs.Devices { if v["type"] != "disk" || v["path"] == "/" { continue } data.Disks[k] = v } out, err := yaml.Dump(&data, yaml.V2) if err != nil { return "" } return string(out) } func (m *InstanceMigration) askProfiles() error { profileNames, err := m.server.GetProfileNames() if err != nil { return err } profiles, err := m.asker.AskString("Which profiles do you want to apply to the instance? (space separated) [default=default, \"-\" for none]: ", "default", func(s string) error { // This indicates that no profiles should be applied. if s == "-" { return nil } profiles := strings.Split(s, " ") for _, profile := range profiles { if !slices.Contains(profileNames, profile) { return fmt.Errorf("Unknown profile %q", profile) } } return nil }) if err != nil { return err } if profiles != "-" { m.instanceArgs.Profiles = strings.Split(profiles, " ") } return nil } func (m *InstanceMigration) askConfig() error { configs, err := m.asker.AskString("Please specify config keys and values (key=value ...): ", "", func(s string) error { if s == "" { return nil } for _, entry := range strings.Split(s, " ") { if !strings.Contains(entry, "=") { return fmt.Errorf("Bad key=value configuration: %v", entry) } } return nil }) if err != nil { return err } for _, entry := range strings.Split(configs, " ") { key, value, _ := strings.Cut(entry, "=") m.instanceArgs.Config[key] = value } return nil } func (m *InstanceMigration) askStorage() error { storagePools, err := m.server.GetStoragePoolNames() if err != nil { return err } if len(storagePools) == 0 { return errors.New("No storage pools available") } storagePool, err := m.asker.AskChoice("Please provide the storage pool to use: ", storagePools, "") if err != nil { return err } m.instanceArgs.Devices["root"] = map[string]string{ "type": "disk", "pool": storagePool, "path": "/", } changeStorageSize, err := m.asker.AskBool("Do you want to change the storage size? [default=no]: ", "no") if err != nil { return err } if changeStorageSize { size, err := m.asker.AskString("Please specify the storage size: ", "", func(s string) error { _, err := units.ParseByteSizeString(s) return err }) if err != nil { return err } m.instanceArgs.Devices["root"]["size"] = size } return nil } func (m *InstanceMigration) askDiskStorage() error { diskNames := []string{} for _, vol := range m.volumes { diskNames = append(diskNames, vol.customVolumeArgs.Name) } if len(diskNames) == 0 { return errors.New("No additional disks available") } diskName, err := m.asker.AskChoice("Please provide the disk name: ", diskNames, "") if err != nil { return err } storagePools, err := m.server.GetStoragePoolNames() if err != nil { return err } if len(storagePools) == 0 { return errors.New("No storage pools available") } storagePool, err := m.asker.AskChoice("Please provide the storage pool to use: ", storagePools, "") if err != nil { return err } m.instanceArgs.Devices[diskName]["pool"] = storagePool for _, vol := range m.volumes { if vol.customVolumeArgs.Name == diskName { vol.pool = storagePool break } } return nil } func (m *InstanceMigration) askNetwork() error { networks, err := m.server.GetNetworkNames() if err != nil { return err } network, err := m.asker.AskChoice("Please specify the network to use for the instance: ", networks, "") if err != nil { return err } m.instanceArgs.Devices["eth0"] = map[string]string{ "type": "nic", "nictype": "bridged", "parent": network, "name": "eth0", } return nil } func (m *InstanceMigration) askDisk() error { volMigrator, ok := newVolumeMigration(m.ctx, m.server, m.asker, m.flagRsyncArgs).(*VolumeMigration) if !ok { return errors.New("Migrator should be of type VolumeMigration") } volMigrator.project = m.project err := volMigrator.gatherInfo() if err != nil { return err } if m.migrationType == MigrationTypeContainer && volMigrator.migrationType == MigrationTypeVolumeBlock { return errors.New("Block disk is not supported by the container") } m.instanceArgs.Devices[volMigrator.customVolumeArgs.Name] = map[string]string{ "type": "disk", "pool": volMigrator.pool, "source": volMigrator.customVolumeArgs.Name, } if volMigrator.migrationType == MigrationTypeVolumeFilesystem { mountPath, err := m.asker.AskString("Provide mount path for this disk: ", "", nil) if err != nil { return err } m.instanceArgs.Devices[volMigrator.customVolumeArgs.Name]["path"] = mountPath } m.volumes = append(m.volumes, volMigrator) return nil } func (m *InstanceMigration) askUEFISupport() error { if m.instanceArgs.Type == api.InstanceTypeVM { architectureName, _ := osarch.ArchitectureGetLocal() if slices.Contains([]string{"x86_64", "aarch64"}, architectureName) { hasUEFI, err := m.asker.AskBool("Does the VM support UEFI booting? [default=yes]: ", "yes") if err != nil { return err } if hasUEFI { hasSecureBoot, err := m.asker.AskBool("Does the VM support UEFI Secure Boot? [default=yes]: ", "yes") if err != nil { return err } if !hasSecureBoot { m.instanceArgs.Config["security.secureboot"] = "false" } } else { m.instanceArgs.Config["security.csm"] = "true" m.instanceArgs.Config["security.secureboot"] = "false" } } } return nil } incus-7.0.0/cmd/incus-migrate/migrate_ova.go000066400000000000000000000237731517523235500210210ustar00rootroot00000000000000package main import ( "archive/tar" "context" "encoding/xml" "errors" "fmt" "io" "os" "path/filepath" "slices" "strings" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" "github.com/lxc/incus/v7/shared/ask" "github.com/lxc/incus/v7/shared/util" ) // VHResourceType defines what kind of resource this is (e.g., CPU, memory). type VHResourceType string const ( vhResourceTypeOther VHResourceType = "1" vhResourceTypeProcessor VHResourceType = "3" vhResourceTypeMemory VHResourceType = "4" ) // Envelope represents the root of the OVF file. // It typically wraps all metadata about the virtual appliance. type Envelope struct { XMLName xml.Name `xml:"Envelope"` References References `xml:"References"` DiskSection DiskSection `xml:"DiskSection"` VirtualSystem VirtualSystem `xml:"VirtualSystem"` } // References lists all external files used by the OVF (e.g., VMDK files). type References struct { Files []File `xml:"File"` } // File describes one file (usually a disk image) referenced by the OVF. type File struct { ID string `xml:"id,attr"` Href string `xml:"href,attr"` Size int64 `xml:"size,attr"` } // DiskSection contains one or more virtual disks definitions. type DiskSection struct { Disks []Disk `xml:"Disk"` } // Disk describes a virtual disk (size, backing file, format). type Disk struct { DiskID string `xml:"diskId,attr"` FileRef string `xml:"fileRef,attr"` Capacity string `xml:"capacity,attr"` Format string `xml:"format,attr"` } // VirtualSystem defines the configuration of a single virtual machine. type VirtualSystem struct { ID string `xml:"id,attr"` Name string `xml:"Name"` VirtualHardwareSection VirtualHardwareSection `xml:"VirtualHardwareSection"` } // VirtualHardwareSection lists all the hardware components for the VM. type VirtualHardwareSection struct { Items []Item `xml:"Item"` } // Item contains individual hardware definitions (CPU, memory, disk, etc.). type Item struct { Description string `xml:"Description"` ElementName string `xml:"ElementName"` InstanceID string `xml:"InstanceID"` ResourceType string `xml:"ResourceType"` VirtualQuantity string `xml:"VirtualQuantity"` Connection string `xml:"Connection"` HostResource string `xml:"HostResource"` } // OVAMigration handles the migration logic for an instance from .ova file. type OVAMigration struct { *Migration flagRsyncArgs string instance *InstanceMigration ovaPath string references map[string]string } // newOVAMigration returns a new OVAMigration. func newOVAMigration(ctx context.Context, server incus.InstanceServer, asker ask.Asker, flagRsyncArgs string) Migrator { return &OVAMigration{ Migration: &Migration{ asker: asker, ctx: ctx, server: server, migrationType: MigrationTypeVM, }, instance: newInstanceMigration(ctx, server, asker, flagRsyncArgs, MigrationTypeVM).(*InstanceMigration), flagRsyncArgs: flagRsyncArgs, references: map[string]string{}, } } // gatherInfo collects information from the user about the instance to be created. func (m *OVAMigration) gatherInfo() error { var err error m.ovaPath, err = m.askPath("Please provide the path or URL to a .ova file: ") if err != nil { return err } // Project err = m.askProject("Project to create the instance in [default=default]: ") if err != nil { return err } if m.project != "" { m.server = m.server.UseProject(m.project) } // Target err = m.askTarget() if err != nil { return err } m.server = m.server.UseTarget(m.target) // Pool pools, err := m.server.GetStoragePools() if err != nil { return err } poolNames := []string{} for _, p := range pools { poolNames = append(poolNames, p.Name) } for { poolName, err := m.asker.AskString("Name of the pool: ", "", nil) if err != nil { return err } if !slices.Contains(poolNames, poolName) { fmt.Printf("Pool %q doesn't exists\n", poolName) continue } m.pool = poolName break } m.instance.instanceArgs = api.InstancesPost{ Source: api.InstanceSource{ Type: "migration", Mode: "push", }, } m.instance.instanceArgs.Config = map[string]string{} m.instance.instanceArgs.Devices = map[string]map[string]string{} m.instance.instanceArgs.Devices["root"] = map[string]string{ "type": "disk", "pool": m.pool, "path": "/", } m.instance.instanceArgs.Type = api.InstanceTypeVM m.instance.project = m.project // Instance name instanceNames, err := m.server.GetInstanceNames(api.InstanceTypeAny) if err != nil { return err } for { instanceName, err := m.asker.AskString("Name of the new instance: ", "", nil) if err != nil { return err } if slices.Contains(instanceNames, instanceName) { fmt.Printf("Instance %q already exists\n", instanceName) continue } m.instance.instanceArgs.Name = instanceName break } err = m.instance.askUEFISupport() if err != nil { return err } err = m.readOVA() if err != nil { return err } return nil } // migrate performs the instance migration. func (m *OVAMigration) migrate() error { if m.migrationType != MigrationTypeVM { return errors.New("Wrong migration type for migrate") } // Create the temporary directory to be used for the ova files. outputPath, err := os.MkdirTemp("", "incus-migrate_ova_") if err != nil { return err } defer func() { _ = os.RemoveAll(outputPath) }() err = m.unpackOVA(outputPath) if err != nil { return err } // Update source paths for the instance and additional disks. // Currently, only filenames are kept, as the full paths become known after unpacking the OVA. m.instance.sourcePath = filepath.Join(outputPath, m.instance.sourcePath) err = m.validateDiskFormat(m.instance.sourcePath) if err != nil { return err } for _, v := range m.instance.volumes { // If the disk doesn't come from an OVA file, leave the source path unchanged. if !m.isDiskFromOVA(v.sourcePath) { continue } v.sourcePath = filepath.Join(outputPath, v.sourcePath) err = m.validateDiskFormat(m.instance.sourcePath) if err != nil { return err } } return m.instance.migrate() } // renderObject renders the state of the instance. func (m *OVAMigration) renderObject() error { return m.instance.renderObject() } // unpackOVA extracts the contents of the OVA file. func (m *OVAMigration) unpackOVA(outPath string) error { file, err := os.Open(m.ovaPath) if err != nil { return err } defer file.Close() tarReader := tar.NewReader(file) outPathRoot, err := os.OpenRoot(outPath) if err != nil { return err } defer func() { _ = outPathRoot.Close() }() for { header, err := tarReader.Next() if err == io.EOF { break // End of archive } if err != nil { return err } outFile, err := outPathRoot.Create(header.Name) if err != nil { return fmt.Errorf("Error creating file: %v", err) } _, err = util.SafeCopy(outFile, tarReader) if err != nil { outFile.Close() return fmt.Errorf("Error unpacking file: %v", err) } outFile.Close() } return nil } // readOVA reads the contents of the OVA file and parses the embedded OVF file. func (m *OVAMigration) readOVA() error { file, err := os.Open(m.ovaPath) if err != nil { return err } defer file.Close() tarReader := tar.NewReader(file) for { header, err := tarReader.Next() if err == io.EOF { break // End of archive } if err != nil { return err } // Look for the manifest (xml) file if strings.HasSuffix(header.Name, ".ovf") { var env Envelope decoder := xml.NewDecoder(tarReader) err := decoder.Decode(&env) if err != nil { panic(err) } return m.readOVFData(env) } } return nil } // readOVFData parses the OVF file and extracts information from it. func (m *OVAMigration) readOVFData(env Envelope) error { for _, f := range env.References.Files { m.references[f.ID] = f.Href } // Extract vCPUs and memory for _, item := range env.VirtualSystem.VirtualHardwareSection.Items { switch item.ResourceType { case string(vhResourceTypeProcessor): m.instance.instanceArgs.Config["limits.cpu"] = item.VirtualQuantity case string(vhResourceTypeMemory): m.instance.instanceArgs.Config["limits.memory"] = fmt.Sprintf("%sMB", item.VirtualQuantity) } } // Add disks for idx, disk := range env.DiskSection.Disks { if idx == 0 { m.instance.sourcePath = m.references[disk.FileRef] continue } err := m.addDisk(m.references[disk.FileRef], idx) if err != nil { return err } } return nil } // addDisk adds an additional disk to the VM instance. func (m *OVAMigration) addDisk(diskFileName string, index int) error { volMigrator, ok := newVolumeMigration(m.ctx, m.server, m.asker, m.flagRsyncArgs).(*VolumeMigration) if !ok { return errors.New("Migrator should be of type VolumeMigration") } diskName := fmt.Sprintf("%s-disk%d", m.instance.instanceArgs.Name, index) volMigrator.migrationType = MigrationTypeVolumeBlock volMigrator.project = m.project volMigrator.pool = m.pool volMigrator.sourcePath = diskFileName volMigrator.customVolumeArgs = api.StorageVolumesPost{ ContentType: "block", Name: diskName, Type: "custom", Source: api.StorageVolumeSource{ Type: "migration", Mode: "push", }, } m.instance.instanceArgs.Devices[volMigrator.customVolumeArgs.Name] = map[string]string{ "type": "disk", "pool": volMigrator.pool, "source": diskName, } m.instance.volumes = append(m.instance.volumes, volMigrator) return nil } // validateDiskFormat checks whether the provided disk format is supported. func (m *OVAMigration) validateDiskFormat(path string) error { _, ext, _, _ := archive.DetectCompression(path) if ext != ".vmdk" { return fmt.Errorf("%s disk format not supported", ext) } return nil } // isDiskFromOVA verifies whether the disk originates from an OVA file. func (m *OVAMigration) isDiskFromOVA(name string) bool { for _, v := range m.references { if v == name { return true } } return false } incus-7.0.0/cmd/incus-migrate/migrate_volume.go000066400000000000000000000130731517523235500215330ustar00rootroot00000000000000package main import ( "bufio" "context" "errors" "fmt" "os" "os/exec" "slices" "strings" "go.yaml.in/yaml/v4" incus "github.com/lxc/incus/v7/client" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/ask" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/revert" ) // VolumeMigration handles the migration logic for an custom volume. type VolumeMigration struct { *Migration customVolumeArgs api.StorageVolumesPost flagRsyncArgs string } // newVolumeMigration returns a new VolumeMigration. func newVolumeMigration(ctx context.Context, server incus.InstanceServer, asker ask.Asker, flagRsyncArgs string) Migrator { return &VolumeMigration{ Migration: &Migration{ asker: asker, ctx: ctx, server: server, }, flagRsyncArgs: flagRsyncArgs, } } // gatherInfo collects information from the user about the custom volume to be created. func (m *VolumeMigration) gatherInfo() error { var err error m.customVolumeArgs = api.StorageVolumesPost{ Type: "custom", Source: api.StorageVolumeSource{ Type: "migration", Mode: "push", }, } // Project if m.project == "" { err = m.askProject("Project to create the volume in [default=default]: ") if err != nil { return err } } if m.project != "" { m.server = m.server.UseProject(m.project) } // Target err = m.askTarget() if err != nil { return err } m.server = m.server.UseTarget(m.target) // Pool pools, err := m.server.GetStoragePools() if err != nil { return err } poolNames := []string{} for _, p := range pools { poolNames = append(poolNames, p.Name) } for { poolName, err := m.asker.AskString("Name of the pool: ", "", nil) if err != nil { return err } if !slices.Contains(poolNames, poolName) { fmt.Printf("Pool %q doesn't exists\n", poolName) continue } m.pool = poolName break } // Custom volume name volumes, err := m.server.GetStoragePoolVolumes(m.pool) if err != nil { return err } volumeNames := []string{} for _, v := range volumes { if v.Type != "custom" { continue } volumeNames = append(volumeNames, v.Name) } for { volumeName, err := m.asker.AskString("Name of the new custom volume: ", "", nil) if err != nil { return err } if slices.Contains(volumeNames, volumeName) { fmt.Printf("Storage volume %q already exists\n", volumeName) continue } m.customVolumeArgs.Name = volumeName break } m.sourcePath, err = m.askPath("Please provide the path to a disk or filesystem: ") if err != nil { return err } err = m.setMigrationType() if err != nil { return err } if m.migrationType == MigrationTypeVolumeFilesystem { m.customVolumeArgs.ContentType = "filesystem" } else { m.customVolumeArgs.ContentType = "block" } if m.migrationType == MigrationTypeVolumeFilesystem { if os.Geteuid() != 0 { return errors.New("This tool must be run as root for filesystem migrations") } _, err := exec.LookPath("rsync") if err != nil { return errors.New("Unable to find required command \"rsync\"") } } err = m.setSourceFormat() if err != nil { return err } return nil } // migrate performs the custom volume migration. func (m *VolumeMigration) migrate() error { if m.migrationType != MigrationTypeVolumeBlock && m.migrationType != MigrationTypeVolumeFilesystem { return errors.New("Wrong migration type for migrate") } // User decided not to migrate. if m.customVolumeArgs.Name == "" { return nil } return m.runMigration(func(path string) error { reverter := revert.New() defer reverter.Fail() // Create the custom volume op, err := m.server.CreateStoragePoolVolumeFromMigration(m.pool, m.customVolumeArgs) if err != nil { return err } reverter.Add(func() { _ = m.server.DeleteStoragePoolVolume(m.pool, "custom", m.customVolumeArgs.Name) }) progress := cli.ProgressRenderer{Format: "Transferring custom volume: %s"} _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } err = transferRootfs(m.ctx, op, path, m.flagRsyncArgs, m.migrationType) if err != nil { return err } progress.Done(fmt.Sprintf("Custom volume %s successfully created", m.customVolumeArgs.Name)) reverter.Success() return nil }) } // renderObject renders the state of the custom volume to be created. func (m *VolumeMigration) renderObject() error { fmt.Println("\nCustom volume to be created:") scanner := bufio.NewScanner(strings.NewReader(m.render())) for scanner.Scan() { fmt.Printf(" %s\n", scanner.Text()) } shouldMigrate, err := m.asker.AskBool("Do you want to continue? [default=yes]: ", "yes") if err != nil { return err } // Reset volume settings when user interrupts creation process if !shouldMigrate { m.customVolumeArgs = api.StorageVolumesPost{} } return nil } func (m *VolumeMigration) render() string { data := struct { Name string `yaml:"Name"` Project string `yaml:"Project"` Type string `yaml:"Type"` Source string `yaml:"Source"` SourceFormat string `yaml:"Source format,omitempty"` }{ m.customVolumeArgs.Name, m.project, m.customVolumeArgs.ContentType, m.sourcePath, m.sourceFormat, } out, err := yaml.Dump(&data, yaml.V2) if err != nil { return "" } return string(out) } func (m *VolumeMigration) setMigrationType() error { if m.sourcePath == "" { return errors.New("Missing source path") } if internalUtil.IsDir(m.sourcePath) { m.migrationType = MigrationTypeVolumeFilesystem } else { m.migrationType = MigrationTypeVolumeBlock } return nil } incus-7.0.0/cmd/incus-migrate/transfer.go000066400000000000000000000057501517523235500203430ustar00rootroot00000000000000package main import ( "context" "fmt" "io" "net" "os" "os/exec" "strings" "github.com/google/uuid" "github.com/gorilla/websocket" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/ws" ) // Send an rsync stream of a path over a websocket. func rsyncSend(ctx context.Context, conn *websocket.Conn, path string, rsyncArgs string, migrationType MigrationType) error { cmd, dataSocket, stderr, err := rsyncSendSetup(ctx, path, rsyncArgs, migrationType) if err != nil { return err } if dataSocket != nil { defer func() { _ = dataSocket.Close() }() } readDone, writeDone := ws.Mirror(conn, dataSocket) <-writeDone _ = dataSocket.Close() output, err := io.ReadAll(stderr) if err != nil { _ = cmd.Process.Kill() _ = cmd.Wait() return fmt.Errorf("Failed to rsync: %v\n%s", err, output) } err = cmd.Wait() <-readDone if err != nil { return fmt.Errorf("Failed to rsync: %v\n%s", err, output) } return nil } // Spawn the rsync process. func rsyncSendSetup(ctx context.Context, path string, rsyncArgs string, migrationType MigrationType) (*exec.Cmd, net.Conn, io.ReadCloser, error) { auds := fmt.Sprintf("@incus-migrate/%s", uuid.New().String()) if len(auds) > linux.ABSTRACT_UNIX_SOCK_LEN-1 { auds = auds[:linux.ABSTRACT_UNIX_SOCK_LEN-1] } l, err := net.Listen("unix", auds) if err != nil { return nil, nil, nil, err } execPath, err := os.Readlink("/proc/self/exe") if err != nil { return nil, nil, nil, err } if !util.PathExists(execPath) { execPath = os.Args[0] } rsyncCmd := fmt.Sprintf("sh -c \"%s netcat %s\"", execPath, auds) args := []string{ "-ar", "--devices", "--numeric-ids", "--partial", "--sparse", } if migrationType == MigrationTypeContainer || migrationType == MigrationTypeVolumeFilesystem { args = append(args, "--xattrs", "--delete", "--compress", "--compress-level=2") } if migrationType == MigrationTypeVM || migrationType == MigrationTypeVolumeBlock { args = append(args, "--exclude", "*.img") } args = append(args, "--filter=-x security.selinux", "--ignore-missing-args") if rsyncArgs != "" { args = append(args, strings.Split(rsyncArgs, " ")...) } args = append(args, []string{path, "localhost:/tmp/foo"}...) args = append(args, []string{"-e", rsyncCmd}...) cmd := exec.CommandContext(ctx, "rsync", args...) cmd.Stdout = os.Stderr stderr, err := cmd.StderrPipe() if err != nil { return nil, nil, nil, err } err = cmd.Start() if err != nil { return nil, nil, nil, err } conn, err := l.Accept() if err != nil { _ = cmd.Process.Kill() _ = cmd.Wait() return nil, nil, nil, err } _ = l.Close() return cmd, conn, stderr, nil } func protoSendError(conn *websocket.Conn, err error) { migration.ProtoSendControl(conn, err) if err != nil { closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") _ = conn.WriteMessage(websocket.CloseMessage, closeMsg) _ = conn.Close() } } incus-7.0.0/cmd/incus-migrate/utils.go000066400000000000000000000226441517523235500176600ustar00rootroot00000000000000package main import ( "bufio" "context" "crypto/x509" "encoding/pem" "errors" "fmt" "net/url" "os" "path/filepath" "reflect" "strings" "golang.org/x/sys/unix" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/ports" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/ws" ) // MigrationType represents the type of the migration. type MigrationType string // MigrationTypeContainer defines the migration type value for a container. const MigrationTypeContainer = MigrationType("container") // MigrationTypeVM defines the migration type value for a virtual-machine. const MigrationTypeVM = MigrationType("virtual-machine") // MigrationTypeVolumeFilesystem defines the migration type value for a custom volume of type filesystem. const MigrationTypeVolumeFilesystem = MigrationType("volume-filesystem") // MigrationTypeVolumeBlock defines the migration type value for a custom volume of type block. const MigrationTypeVolumeBlock = MigrationType("volume-block") func transferRootfs(ctx context.Context, op incus.Operation, rootfs string, rsyncArgs string, migrationType MigrationType) error { opAPI := op.Get() // Connect to the websockets wsControl, err := op.GetWebsocket(opAPI.Metadata[api.SecretNameControl].(string)) if err != nil { return err } abort := func(err error) error { protoSendError(wsControl, err) return err } wsFs, err := op.GetWebsocket(opAPI.Metadata[api.SecretNameFilesystem].(string)) if err != nil { return abort(err) } // Setup control struct var fs migration.MigrationFSType var rsyncHasFeature bool if migrationType == MigrationTypeVM || migrationType == MigrationTypeVolumeBlock { fs = migration.MigrationFSType_BLOCK_AND_RSYNC rsyncHasFeature = false } else { fs = migration.MigrationFSType_RSYNC rsyncHasFeature = true } offerHeader := migration.MigrationHeader{ RsyncFeatures: &migration.RsyncFeatures{ Xattrs: &rsyncHasFeature, Delete: &rsyncHasFeature, Compress: &rsyncHasFeature, }, Fs: &fs, } if migrationType == MigrationTypeVM || migrationType == MigrationTypeVolumeBlock { sourcePath := filepath.Join(rootfs, "root.img") size, err := blockDiskSizeBytes(sourcePath) if err != nil { return abort(err) } offerHeader.VolumeSize = &size rootfs = internalUtil.AddSlash(rootfs) } err = migration.ProtoSend(wsControl, &offerHeader) if err != nil { return abort(err) } var respHeader migration.MigrationHeader err = migration.ProtoRecv(wsControl, &respHeader) if err != nil { return abort(err) } rsyncFeaturesOffered := offerHeader.GetRsyncFeaturesSlice() rsyncFeaturesResponse := respHeader.GetRsyncFeaturesSlice() if !reflect.DeepEqual(rsyncFeaturesOffered, rsyncFeaturesResponse) { return abort(fmt.Errorf("Offered rsync features (%v) differ from those in the migration response (%v)", rsyncFeaturesOffered, rsyncFeaturesResponse)) } // Send the filesystem if migrationType != MigrationTypeVolumeBlock { err = rsyncSend(ctx, wsFs, rootfs, rsyncArgs, migrationType) if err != nil { return abort(fmt.Errorf("Failed sending filesystem volume: %w", err)) } } if migrationType == MigrationTypeVM || migrationType == MigrationTypeVolumeBlock { // Send block volume f, err := os.Open(filepath.Join(rootfs, "root.img")) if err != nil { return abort(err) } defer func() { _ = f.Close() }() conn := ws.NewWrapper(wsFs) go func() { <-ctx.Done() _ = conn.Close() _ = f.Close() }() _, err = util.SafeCopy(conn, f) if err != nil { return abort(err) } err = conn.Close() if err != nil { return abort(err) } } // Check the result msg := migration.MigrationControl{} err = migration.ProtoRecv(wsControl, &msg) if err != nil { _ = wsControl.Close() return err } if !msg.GetSuccess() { return errors.New(msg.GetMessage()) } return nil } func (m *cmdMigrate) connectLocal() (incus.InstanceServer, error) { args := incus.ConnectionArgs{} args.UserAgent = fmt.Sprintf("LXC-MIGRATE %s", version.Version) return incus.ConnectIncusUnix("", &args) } func (m *cmdMigrate) connectTarget(uri string, certPath string, keyPath string, authType string, token string) (incus.InstanceServer, string, error) { args := incus.ConnectionArgs{ AuthType: authType, } clientFingerprint := "" if authType == api.AuthenticationMethodTLS { var clientCrt []byte var clientKey []byte // Generate a new client certificate for this if certPath == "" || keyPath == "" { var err error clientCrt, clientKey, err = localtls.GenerateMemCert(true, false) if err != nil { return nil, "", err } clientFingerprint, err = localtls.CertFingerprintStr(string(clientCrt)) if err != nil { return nil, "", err } // When using certificate add tokens, there's no need to show the temporary certificate. if token == "" { fmt.Printf("\nYour temporary certificate is:\n%s\n", string(clientCrt)) } } else { var err error clientCrt, err = os.ReadFile(certPath) if err != nil { return nil, "", fmt.Errorf("Failed to read client certificate: %w", err) } clientKey, err = os.ReadFile(keyPath) if err != nil { return nil, "", fmt.Errorf("Failed to read client key: %w", err) } } args.TLSClientCert = string(clientCrt) args.TLSClientKey = string(clientKey) } // Attempt to connect using the system CA args.UserAgent = fmt.Sprintf("LXC-MIGRATE %s", version.Version) c, err := incus.ConnectIncus(uri, &args) var certificate *x509.Certificate if err != nil { // Failed to connect using the system CA, so retrieve the remote certificate certificate, err = localtls.GetRemoteCertificate(uri, args.UserAgent) if err != nil { return nil, "", err } } // Handle certificate prompt if certificate != nil { serverCrt := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificate.Raw}) args.TLSServerCert = string(serverCrt) // Setup a new connection, this time with the remote certificate c, err = incus.ConnectIncus(uri, &args) if err != nil { return nil, "", err } } // Get server information srv, _, err := c.GetServer() if err != nil { return nil, "", err } // Check if our cert is already trusted if srv.Auth == "trusted" { fmt.Printf("\nRemote server:\n Hostname: %s\n Version: %s\n\n", srv.Environment.ServerName, srv.Environment.ServerVersion) return c, "", nil } if authType == api.AuthenticationMethodTLS { if token != "" { req := api.CertificatesPost{ TrustToken: token, } err = c.CreateCertificate(req) if err != nil { return nil, "", fmt.Errorf("Failed to create certificate: %w", err) } } else { fmt.Println("A temporary client certificate was generated, use `incus config trust add` on the target server.") fmt.Println("") fmt.Print("Press ENTER after the certificate was added to the remote server: ") _, err = bufio.NewReader(os.Stdin).ReadString('\n') if err != nil { return nil, "", err } } } else { c.RequireAuthenticated(true) } // Get full server information srv, _, err = c.GetServer() if err != nil { if clientFingerprint != "" { _ = c.DeleteCertificate(clientFingerprint) } return nil, "", err } if srv.Auth == "untrusted" { return nil, "", errors.New("Server doesn't trust us after authentication") } fmt.Printf("\nRemote server:\n Hostname: %s\n Version: %s\n\n", srv.Environment.ServerName, srv.Environment.ServerVersion) return c, clientFingerprint, nil } func setupSource(path string, mounts []string) error { prefix := "/" if len(mounts) > 0 { prefix = mounts[0] } // Mount everything for _, mount := range mounts { target := fmt.Sprintf("%s/%s", path, strings.TrimPrefix(mount, prefix)) // Mount the path err := unix.Mount(mount, target, "none", unix.MS_BIND, "") if err != nil { return fmt.Errorf("Failed to mount %s: %w", mount, err) } // Make it read-only err = unix.Mount("", target, "none", unix.MS_BIND|unix.MS_RDONLY|unix.MS_REMOUNT, "") if err != nil { return fmt.Errorf("Failed to make %s read-only: %w", mount, err) } } return nil } func parseURL(URL string) (string, error) { uri, err := url.Parse(URL) if err != nil { return "", err } // Create a URL with scheme and hostname since it wasn't provided if uri.Scheme == "" && uri.Host == "" && uri.Path != "" { uri, err = url.Parse(fmt.Sprintf("https://%s", uri.Path)) if err != nil { return "", err } } // If no port was provided, use default port if uri.Port() == "" { uri.Host = fmt.Sprintf("%s:%d", uri.Hostname(), ports.HTTPSDefaultPort) } return uri.String(), nil } // blockDiskSizeBytes returns the size of a block disk (path can be either block device or raw file). func blockDiskSizeBytes(blockDiskPath string) (int64, error) { if linux.IsBlockdevPath(blockDiskPath) { // Attempt to open the device path. f, err := os.Open(blockDiskPath) if err != nil { return -1, err } defer func() { _ = f.Close() }() fd := int(f.Fd()) // Retrieve the block device size. res, err := unix.IoctlGetInt(fd, unix.BLKGETSIZE64) if err != nil { return -1, err } return int64(res), nil } // Block device is assumed to be a raw file. fi, err := os.Stat(blockDiskPath) if err != nil { return -1, err } return fi.Size(), nil } incus-7.0.0/cmd/incus-simplestreams/000077500000000000000000000000001517523235500174215ustar00rootroot00000000000000incus-7.0.0/cmd/incus-simplestreams/main.go000066400000000000000000000033511517523235500206760ustar00rootroot00000000000000package main import ( "os" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/version" ) type cmdGlobal struct { flagHelp bool flagVersion bool } func main() { app := &cobra.Command{} app.Use = "incus-simplestreams" app.Short = "Maintain and Incus-compatible simplestreams tree" app.Long = `Description: Maintain an Incus-compatible simplestreams tree This tool makes it easy to manage the files on a static image server using simplestreams index files as the publishing mechanism. ` app.SilenceUsage = true app.CompletionOptions = cobra.CompletionOptions{DisableDefaultCmd: true} // Global flags. globalCmd := cmdGlobal{} app.PersistentFlags().BoolVar(&globalCmd.flagVersion, "version", false, "Print version number") app.PersistentFlags().BoolVarP(&globalCmd.flagHelp, "help", "h", false, "Print help") // Help handling. app.SetHelpCommand(&cobra.Command{ Use: "no-help", Hidden: true, }) // Version handling. app.SetVersionTemplate("{{.Version}}\n") app.Version = version.Version // add sub-command. addCmd := cmdAdd{global: &globalCmd} app.AddCommand(addCmd.command()) // generate-metadata sub-command. generateMetadataCmd := cmdGenerateMetadata{global: &globalCmd} app.AddCommand(generateMetadataCmd.command()) // list sub-command. listCmd := cmdList{global: &globalCmd} app.AddCommand(listCmd.command()) // remove sub-command. removeCmd := cmdRemove{global: &globalCmd} app.AddCommand(removeCmd.command()) // verify sub-command. verifyCmd := cmdVerify{global: &globalCmd} app.AddCommand(verifyCmd.command()) pruneCmd := cmdPrune{global: &globalCmd} app.AddCommand(pruneCmd.command()) // Run the main command and handle errors. err := app.Execute() if err != nil { os.Exit(1) } } incus-7.0.0/cmd/incus-simplestreams/main_add.go000066400000000000000000000243671517523235500215200ustar00rootroot00000000000000package main import ( "archive/tar" "context" "crypto/sha256" "encoding/json" "errors" "fmt" "io" "io/fs" "os" "strings" "time" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/simplestreams" "github.com/lxc/incus/v7/shared/util" ) type cmdAdd struct { global *cmdGlobal flagAliases []string flagNoDefaultAlias bool flagProductName string } func (c *cmdAdd) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "add []" cmd.Aliases = []string{"import"} cmd.Short = "Add an image" cmd.Long = cli.FormatSection("Description:", `Add an image to the server This command parses the metadata tarball to retrieve the following fields from its metadata.yaml: - architecture - creation_date - properties["description"] - properties["os"] - properties["release"] - properties["variant"] - properties["architecture"] It then check computes the hash for the new image, confirm it's not already on the image server and finally adds it to the index. Unless "--no-default-alias" is specified, it generates a default "{os}/{release}/{variant}" alias. Unless "--product-name" is specified, the product name is generated as "{os}:{release}:{variant}:{architecture}". If one argument is specified, it is assumed to be a unified image, with both the metadata and rootfs in a single tarball. Otherwise, it is a split image (separate files for metadata and rootfs/disk). `) cmd.RunE = c.run cmd.Flags().StringArrayVar(&c.flagAliases, "alias", nil, "Add alias") cmd.Flags().BoolVar(&c.flagNoDefaultAlias, "no-default-alias", false, "Do not add the default alias") cmd.Flags().StringVar(&c.flagProductName, "product-name", "", "Set the product name") return cmd } // dataItem - holds information about the image data file. // used if different from the metadata file. type dataItem struct { Path string FileType string Size int64 Sha256 string Extension string combinedSha256 string } // parseImage parses the metadata and data, filling the dataItem struct. func (c *cmdAdd) parseImage(metaFile *os.File, dataFile *os.File) (*dataItem, error) { item := dataItem{ Path: dataFile.Name(), } // Read the header. _, extension, _, err := archive.DetectCompressionFile(dataFile) if err != nil { return nil, err } item.Extension = extension switch item.Extension { case ".squashfs": item.FileType = "squashfs" case ".qcow2": item.FileType = "disk-kvm.img" default: return nil, fmt.Errorf("Unsupported data type %q", item.Extension) } // Get the size. dataStat, err := dataFile.Stat() if err != nil { return nil, err } item.Size = dataStat.Size() // Get the sha256. _, err = dataFile.Seek(0, 0) if err != nil { return nil, err } hash256 := sha256.New() _, err = util.SafeCopy(hash256, dataFile) if err != nil { return nil, err } item.Sha256 = fmt.Sprintf("%x", hash256.Sum(nil)) // Get the combined sha256. _, err = metaFile.Seek(0, 0) if err != nil { return nil, err } _, err = dataFile.Seek(0, 0) if err != nil { return nil, err } hash256 = sha256.New() _, err = util.SafeCopy(hash256, metaFile) if err != nil { return nil, err } _, err = util.SafeCopy(hash256, dataFile) if err != nil { return nil, err } item.combinedSha256 = fmt.Sprintf("%x", hash256.Sum(nil)) return &item, nil } func (c *cmdAdd) run(cmd *cobra.Command, args []string) error { // Quick checks. exit, err := cli.CheckArgs(cmd, args, 1, 2) if exit { return err } isUnifiedTarball := (len(args) == 1) // Open the metadata. metaFile, err := os.Open(args[0]) if err != nil { return err } defer metaFile.Close() // Read the header. _, _, unpacker, err := archive.DetectCompressionFile(metaFile) if err != nil { return err } // Get the size. metaStat, err := metaFile.Stat() if err != nil { return err } metaSize := metaStat.Size() // Get the sha256. _, err = metaFile.Seek(0, 0) if err != nil { return err } hash256 := sha256.New() _, err = util.SafeCopy(hash256, metaFile) if err != nil { return err } metaSha256 := fmt.Sprintf("%x", hash256.Sum(nil)) // Set the metadata paths. metaPath := args[0] // Go through the tarball. _, err = metaFile.Seek(0, 0) if err != nil { return err } metaTar, metaTarCancel, err := archive.CompressedTarReader(context.Background(), metaFile, unpacker, "") if err != nil { return err } defer metaTarCancel() var hdr *tar.Header for { hdr, err = metaTar.Next() if err != nil { if err == io.EOF { break } return err } if hdr.Name == "metadata.yaml" { break } } if hdr == nil || hdr.Name != "metadata.yaml" { return errors.New("Couldn't find metadata.yaml in metadata tarball") } // Parse the metadata. metadata := api.ImageMetadata{} body, err := io.ReadAll(metaTar) if err != nil { return err } err = yaml.Load(body, &metadata) if err != nil { return err } // Validate the metadata. _, err = osarch.ArchitectureID(metadata.Architecture) if err != nil { return fmt.Errorf("Invalid architecture in metadata.yaml: %w", err) } if metadata.CreationDate == 0 { return errors.New("Missing creation date in metadata.yaml") } for _, prop := range []string{"os", "release", "variant", "architecture", "description"} { _, ok := metadata.Properties[prop] if !ok { return fmt.Errorf("Missing property %q in metadata.yaml", prop) } } var data *dataItem if !isUnifiedTarball { // Open the data. dataFile, err := os.Open(args[1]) if err != nil { return err } defer dataFile.Close() // Parse the content. data, err = c.parseImage(metaFile, dataFile) if err != nil { return err } } // Create the paths if missing. err = os.MkdirAll("images", 0o755) if err != nil && !os.IsExist(err) { return err } err = os.MkdirAll("streams/v1", 0o755) if err != nil && !os.IsExist(err) { return err } // Load the images file. products := simplestreams.Products{} body, err = os.ReadFile("streams/v1/images.json") if err != nil { if !errors.Is(err, fs.ErrNotExist) { return err } // Create a blank images file. products = simplestreams.Products{ ContentID: "images", DataType: "image-downloads", Format: "products:1.0", Products: map[string]simplestreams.Product{}, } } else { // Parse the existing images file. err = json.Unmarshal(body, &products) if err != nil { return err } } var productName string if c.flagProductName != "" { productName = c.flagProductName } else { productName = fmt.Sprintf("%s:%s:%s:%s", metadata.Properties["os"], metadata.Properties["release"], metadata.Properties["variant"], metadata.Properties["architecture"]) } // Check if the product already exists. product, ok := products.Products[productName] if !ok { var aliases []string if !c.flagNoDefaultAlias { // Generate a default alias aliases = append(aliases, fmt.Sprintf("%s/%s/%s", metadata.Properties["os"], metadata.Properties["release"], metadata.Properties["variant"])) } aliases = append(aliases, c.flagAliases...) // Create a new product. product = simplestreams.Product{ Aliases: strings.Join(aliases, ","), Architecture: metadata.Properties["architecture"], OperatingSystem: metadata.Properties["os"], Release: metadata.Properties["release"], ReleaseTitle: metadata.Properties["release"], Variant: metadata.Properties["variant"], Versions: map[string]simplestreams.ProductVersion{}, } } var fileType, fileKey, metaTargetPath string if !isUnifiedTarball { fileKey = "incus.tar.xz" fileType = "incus.tar.xz" metaTargetPath = fmt.Sprintf("images/%s.incus.tar.xz", metaSha256) } else { fileKey = "incus_combined.tar.gz" fileType = "incus_combined.tar.gz" metaTargetPath = fmt.Sprintf("images/%s.incus_combined.tar.gz", metaSha256) } // Check if a version already exists. versionName := time.Unix(metadata.CreationDate, 0).Format("200601021504") version, ok := product.Versions[versionName] if !ok { // Create a new version. version = simplestreams.ProductVersion{ Items: map[string]simplestreams.ProductVersionItem{ fileKey: { FileType: fileType, HashSha256: metaSha256, Size: metaSize, Path: metaTargetPath, }, }, } } else { // Check that we're dealing with the same metadata. _, ok := version.Items[fileKey] if !ok { // No fileKey found, add it. version.Items[fileKey] = simplestreams.ProductVersionItem{ FileType: fileType, HashSha256: metaSha256, Size: metaSize, Path: metaTargetPath, } } } // Copy the metadata file if missing. err = internalUtil.FileCopy(metaPath, metaTargetPath) if err != nil && !os.IsExist(err) { return err } if !isUnifiedTarball { // Check that the data file isn't already in. _, ok = version.Items[data.FileType] if ok { return fmt.Errorf("Already have a %q file for this image", data.FileType) } dataTargetPath := fmt.Sprintf("images/%s%s", metaSha256, data.Extension) // Add the file entry. version.Items[data.FileType] = simplestreams.ProductVersionItem{ FileType: data.FileType, HashSha256: data.Sha256, Size: data.Size, Path: dataTargetPath, } // Add the combined hash. metaItem := version.Items["incus.tar.xz"] switch data.FileType { case "squashfs": metaItem.CombinedSha256SquashFs = data.combinedSha256 case "disk-kvm.img": metaItem.CombinedSha256DiskKvmImg = data.combinedSha256 } version.Items["incus.tar.xz"] = metaItem // Copy the data file if missing. err = internalUtil.FileCopy(data.Path, dataTargetPath) if err != nil && !os.IsExist(err) { return err } } // Update the version. product.Versions[versionName] = version // Update the product. products.Products[productName] = product // Write back the images file. body, err = json.Marshal(&products) if err != nil { return err } err = os.WriteFile("streams/v1/images.json", body, 0o644) if err != nil { return err } // Re-generate the index. err = writeIndex(&products) if err != nil { return err } return nil } incus-7.0.0/cmd/incus-simplestreams/main_generate_metadata.go000066400000000000000000000071731517523235500244160ustar00rootroot00000000000000package main import ( "archive/tar" "bufio" "fmt" "io" "os" "os/exec" "time" "github.com/spf13/cobra" yaml "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/ask" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/osarch" ) type cmdGenerateMetadata struct { global *cmdGlobal } func (c *cmdGenerateMetadata) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "generate-metadata " cmd.Short = "Generate a metadata tarball" cmd.Long = cli.FormatSection("Description:", `Generate a metadata tarball This command produces an incus.tar.xz tarball for use with an existing QCOW2 or squashfs disk image. This command will prompt for all of the metadata tarball fields: - Operating system name - Release - Variant - Architecture - Description `) cmd.RunE = c.run return cmd } func (c *cmdGenerateMetadata) run(cmd *cobra.Command, args []string) error { // Quick checks. exit, err := cli.CheckArgs(cmd, args, 1, 1) if exit { return err } // Setup asker. asker := ask.NewAsker(bufio.NewReader(os.Stdin)) // Create the tarball. metaFile, err := os.Create(args[0]) if err != nil { return err } defer metaFile.Close() // Generate the metadata. timestamp := time.Now().UTC() metadata := api.ImageMetadata{ Properties: map[string]string{}, CreationDate: timestamp.Unix(), } // Question - os metaOS, err := asker.AskString("Operating system name: ", "", nil) if err != nil { return err } metadata.Properties["os"] = metaOS // Question - release metaRelease, err := asker.AskString("Release name: ", "", nil) if err != nil { return err } metadata.Properties["release"] = metaRelease // Question - variant metaVariant, err := asker.AskString("Variant name [default=\"default\"]: ", "default", nil) if err != nil { return err } metadata.Properties["variant"] = metaVariant // Question - architecture var incusArch string metaArchitecture, err := asker.AskString("Architecture name: ", "", func(value string) error { id, err := osarch.ArchitectureID(value) if err != nil { return err } incusArch, err = osarch.ArchitectureName(id) if err != nil { return err } return nil }) if err != nil { return err } metadata.Properties["architecture"] = metaArchitecture metadata.Architecture = incusArch // Question - description defaultDescription := fmt.Sprintf("%s %s (%s) (%s) (%s)", metaOS, metaRelease, metaVariant, metaArchitecture, timestamp.Format("200601021504")) metaDescription, err := asker.AskString(fmt.Sprintf("Description [default=\"%s\"]: ", defaultDescription), defaultDescription, nil) if err != nil { return err } metadata.Properties["description"] = metaDescription // Generate YAML. body, err := yaml.Dump(&metadata, yaml.V2) if err != nil { return err } // Prepare the tarball. tarPipeReader, tarPipeWriter := io.Pipe() tarWriter := tar.NewWriter(tarPipeWriter) // Compress the tarball. chDone := make(chan error) go func() { cmd := exec.Command("xz", "-9", "-c") cmd.Stdin = tarPipeReader cmd.Stdout = metaFile err := cmd.Run() chDone <- err }() // Add metadata.yaml. hdr := &tar.Header{ Name: "metadata.yaml", Size: int64(len(body)), Mode: 0o644, Uname: "root", Gname: "root", ModTime: time.Now(), } err = tarWriter.WriteHeader(hdr) if err != nil { return err } _, err = tarWriter.Write(body) if err != nil { return err } // Close the tarball. err = tarWriter.Close() if err != nil { return err } err = tarPipeWriter.Close() if err != nil { return err } err = <-chDone if err != nil { return err } return nil } incus-7.0.0/cmd/incus-simplestreams/main_list.go000066400000000000000000000034101517523235500217250ustar00rootroot00000000000000package main import ( "os" "sort" "github.com/spf13/cobra" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/simplestreams" ) type cmdList struct { global *cmdGlobal flagFormat string } func (c *cmdList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "list" cmd.Short = "List all images on the server" cmd.Long = cli.FormatSection("Description:", `List all image on the server This renders a table with all images currently published on the server. `) cmd.RunE = c.run cmd.Flags().StringVarP(&c.flagFormat, "format", "f", "table", `Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`+"``") cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } return cmd } func (c *cmdList) run(cmd *cobra.Command, args []string) error { // Quick checks. exit, err := cli.CheckArgs(cmd, args, 0, 0) if exit { return err } // Get a simplestreams client. ss := simplestreams.NewLocalClient("") // Get all the images. images, err := ss.ListImages() if err != nil { return err } // Generate the table. data := [][]string{} for _, image := range images { data = append(data, []string{image.Fingerprint, image.Properties["description"], image.Properties["os"], image.Properties["release"], image.Properties["variant"], image.Architecture, image.Type, image.CreatedAt.Format("2006/01/02 15:04 MST")}) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{ "FINGERPRINT", "DESCRIPTION", "OS", "RELEASE", "VARIANT", "ARCHITECTURE", "TYPE", "CREATED", } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, images) } incus-7.0.0/cmd/incus-simplestreams/main_prune.go000066400000000000000000000113341517523235500221070ustar00rootroot00000000000000package main import ( "encoding/json" "errors" "fmt" "io/fs" "os" "path/filepath" "slices" "sort" "github.com/spf13/cobra" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/simplestreams" ) type cmdPrune struct { global *cmdGlobal flagDryRun bool flagRetention int flagVerbose bool } func (c *cmdPrune) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "prune" cmd.Short = "Clean up obsolete files and data" cmd.Long = cli.FormatSection("Description:", `Cleans up obsolete tarball files and removes outdated versions of a product The prune command scans the project directory for tarball files that do not have corresponding references in the 'images.json' file. Any tarball file that is not listed in images.json is considered orphaned and will be deleted. Additionally this command will delete older images, keeping a configurable number of older images per product.`) cmd.RunE = c.run cmd.Flags().BoolVarP(&c.flagDryRun, "dry-run", "d", false, "Preview changes without executing actual operations") cmd.Flags().IntVarP(&c.flagRetention, "retention", "r", 2, "Number of older versions of the product to preserve"+"``") cmd.Flags().BoolVarP(&c.flagVerbose, "verbose", "v", false, "Show all information messages") return cmd } func (c *cmdPrune) run(cmd *cobra.Command, args []string) error { // Quick checks. exit, err := cli.CheckArgs(cmd, args, 0, 0) if exit { return err } if c.flagDryRun { c.flagVerbose = true } err = c.prune() if err != nil { return err } return nil } func (c *cmdPrune) pruneFiles(products *simplestreams.Products, filesToPreserve []string) error { deletedFiles := []string{} err := filepath.WalkDir("./images", func(path string, d fs.DirEntry, err error) error { if err != nil { return err } // Omit the path if it is a directory or if it exists in the images.json file. if d.IsDir() || slices.Contains(filesToPreserve, path) { return nil } if c.flagVerbose { deletedFiles = append(deletedFiles, path) } if !c.flagDryRun { e := os.Remove(path) if e != nil { return e } } return nil }) if err != nil { return err } if c.flagVerbose && len(deletedFiles) > 0 { fmt.Print("Following files were removed:\n") for _, file := range deletedFiles { fmt.Println(file) } } return nil } func (c *cmdPrune) prune() error { body, err := os.ReadFile("streams/v1/images.json") if err != nil { return err } products := simplestreams.Products{} err = json.Unmarshal(body, &products) if err != nil { return err } filesToPreserve := []string{} deletedItems := []string{} deletedVersions := []string{} for kProduct, product := range products.Products { versionNames := []string{} for kVersion, version := range product.Versions { for kItem, item := range version.Items { _, err := os.Stat(item.Path) if err != nil { if !errors.Is(err, os.ErrNotExist) { return err } if c.flagVerbose { deletedItems = append(deletedItems, fmt.Sprintf("%s:%s:%s", kProduct, kVersion, item.Path)) } // Corresponding file doesn't exist on disk. Remove item from products. delete(version.Items, kItem) } filesToPreserve = append(filesToPreserve, item.Path) } if len(version.Items) == 0 { delete(product.Versions, kVersion) continue } versionNames = append(versionNames, kVersion) } if len(product.Versions) == 0 { delete(products.Products, kProduct) continue } sort.Strings(versionNames) updatedVersions := map[string]simplestreams.ProductVersion{} iteration := 0 for i := len(versionNames) - 1; i >= 0; i-- { version := versionNames[i] if iteration <= c.flagRetention { updatedVersions[version] = product.Versions[version] } else if c.flagVerbose { deletedVersions = append(deletedVersions, fmt.Sprintf("%s:%s", kProduct, version)) } iteration += 1 } p := products.Products[kProduct] p.Versions = updatedVersions products.Products[kProduct] = p } if c.flagVerbose { if len(deletedItems) > 0 { fmt.Print("Following items were removed from images.json:\n") for _, item := range deletedItems { fmt.Println(item) } } if len(deletedVersions) > 0 { fmt.Print("Following versions were removed:\n") for _, version := range deletedVersions { fmt.Println(version) } } } if !c.flagDryRun { // Write back the images file. body, err = json.Marshal(&products) if err != nil { return err } err = os.WriteFile("streams/v1/images.json", body, 0o644) if err != nil { return err } // Re-generate the index. err = writeIndex(&products) if err != nil { return err } } err = c.pruneFiles(&products, filesToPreserve) if err != nil { return err } return nil } incus-7.0.0/cmd/incus-simplestreams/main_remove.go000066400000000000000000000071211517523235500222520ustar00rootroot00000000000000package main import ( "encoding/json" "errors" "fmt" "io/fs" "os" "github.com/spf13/cobra" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/simplestreams" ) type cmdRemove struct { global *cmdGlobal flagVerbose bool } func (c *cmdRemove) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "remove " cmd.Aliases = []string{"rm", "delete"} cmd.Short = "Remove an image" cmd.Long = cli.FormatSection("Description:", `Remove an image from the server This command locates the image from its fingerprint and removes it from the index. `) cmd.RunE = c.run cmd.Flags().BoolVarP(&c.flagVerbose, "verbose", "v", false, "Show all information messages") return cmd } func (c *cmdRemove) remove(path string) error { if c.flagVerbose { fmt.Printf("deleting: %s\n", path) } err := os.Remove(path) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } return nil } func (c *cmdRemove) run(cmd *cobra.Command, args []string) error { // Quick checks. exit, err := cli.CheckArgs(cmd, args, 1, 1) if exit { return err } // Get a simplestreams client. ss := simplestreams.NewLocalClient("") // Get the image. image, err := ss.GetImage(args[0]) if err != nil { return err } // Load the images file. body, err := os.ReadFile("streams/v1/images.json") if err != nil { return err } products := simplestreams.Products{} err = json.Unmarshal(body, &products) if err != nil { return err } // Delete the image entry. for kProduct, product := range products.Products { if product.OperatingSystem != image.Properties["os"] || product.Release != image.Properties["release"] || product.Variant != image.Properties["variant"] || product.Architecture != image.Properties["architecture"] { continue } for kVersion, version := range product.Versions { // Get the metadata entry. metaEntry, ok := version.Items["incus.tar.xz"] if ok { if metaEntry.CombinedSha256DiskKvmImg == image.Fingerprint { // Deleting a VM image. err = c.remove(version.Items["disk-kvm.img"].Path) if err != nil { return err } delete(version.Items, "disk-kvm.img") metaEntry.CombinedSha256DiskKvmImg = "" } else if metaEntry.CombinedSha256SquashFs == image.Fingerprint { // Deleting a container image. err = c.remove(version.Items["squashfs"].Path) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } delete(version.Items, "squashfs") metaEntry.CombinedSha256SquashFs = "" } else { continue } // Update the metadata entry. version.Items["incus.tar.xz"] = metaEntry // Delete the version if it's now empty. if len(version.Items) == 1 { err = c.remove(metaEntry.Path) if err != nil { return err } delete(product.Versions, kVersion) } } metaEntry, ok = version.Items["incus_combined.tar.gz"] if ok { if metaEntry.HashSha256 == image.Fingerprint { err = c.remove(metaEntry.Path) if err != nil { return err } delete(version.Items, "incus_combined.tar.gz") } // Delete the version if it's now empty. if len(version.Items) == 0 { delete(product.Versions, kVersion) } } } if len(product.Versions) == 0 { delete(products.Products, kProduct) } break } // Write back the images file. body, err = json.Marshal(&products) if err != nil { return err } err = os.WriteFile("streams/v1/images.json", body, 0o644) if err != nil { return err } // Re-generate the index. err = writeIndex(&products) if err != nil { return err } return nil } incus-7.0.0/cmd/incus-simplestreams/main_verify.go000066400000000000000000000043751517523235500222710ustar00rootroot00000000000000package main import ( "crypto/sha256" "encoding/json" "errors" "fmt" "io/fs" "os" "github.com/spf13/cobra" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/simplestreams" "github.com/lxc/incus/v7/shared/util" ) type cmdVerify struct { global *cmdGlobal } func (c *cmdVerify) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "verify" cmd.Short = "Verify the integrity of the server" cmd.Long = cli.FormatSection("Description:", `Verify the integrity of the server This command will analyze the image index and for every image and file in the index, will validate that the files on disk exist and are of the correct size and content. `) cmd.RunE = c.run return cmd } func (c *cmdVerify) run(cmd *cobra.Command, args []string) error { // Quick checks. exit, err := cli.CheckArgs(cmd, args, 0, 0) if exit { return err } // Load the images file. products := simplestreams.Products{} body, err := os.ReadFile("streams/v1/images.json") if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil } return err } // Parse the existing images file. err = json.Unmarshal(body, &products) if err != nil { return err } // Go over all the files. for _, product := range products.Products { for _, version := range product.Versions { for _, item := range version.Items { // Open the data. dataFile, err := os.Open(item.Path) if err != nil { if errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Missing image file %q", item.Path) } return err } // Get the size. dataStat, err := dataFile.Stat() if err != nil { return err } if item.Size != dataStat.Size() { return fmt.Errorf("File %q has a different size than listed in the index", item.Path) } // Get the sha256. _, err = dataFile.Seek(0, 0) if err != nil { return err } hash256 := sha256.New() _, err = util.SafeCopy(hash256, dataFile) if err != nil { return err } dataSha256 := fmt.Sprintf("%x", hash256.Sum(nil)) if item.HashSha256 != dataSha256 { return fmt.Errorf("File %q has a different SHA256 hash than listed in the index", item.Path) } // Done with this file. dataFile.Close() } } } return nil } incus-7.0.0/cmd/incus-simplestreams/utils.go000066400000000000000000000014371517523235500211150ustar00rootroot00000000000000package main import ( "encoding/json" "os" "github.com/lxc/incus/v7/shared/simplestreams" ) func writeIndex(products *simplestreams.Products) error { // Update the product list. productNames := make([]string, 0, len(products.Products)) for name := range products.Products { productNames = append(productNames, name) } // Write a new index file. stream := simplestreams.Stream{ Format: "index:1.0", Index: map[string]simplestreams.StreamIndex{ "images": { DataType: "image-downloads", Path: "streams/v1/images.json", Format: "products:1.0", Products: productNames, }, }, } body, err := json.Marshal(&stream) if err != nil { return err } err = os.WriteFile("streams/v1/index.json", body, 0o644) if err != nil { return err } return nil } incus-7.0.0/cmd/incus-user/000077500000000000000000000000001517523235500155075ustar00rootroot00000000000000incus-7.0.0/cmd/incus-user/main.go000066400000000000000000000031371517523235500167660ustar00rootroot00000000000000package main import ( "fmt" "os" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/logger" ) type cmdGlobal struct { flagHelp bool flagVersion bool flagLogVerbose bool flagLogDebug bool } func (c *cmdGlobal) preRun(cmd *cobra.Command, args []string) error { return logger.InitLogger("", "", c.flagLogVerbose, c.flagLogDebug, nil) } func run() error { // daemon command (main) daemonCmd := cmdDaemon{} app := daemonCmd.command() app.Use = "incus-user" app.Short = "Incus user project daemon" app.Long = `Description: Incus user project daemon This daemon is used to allow users that aren't considered to be Incus administrators access to a personal project with suitable restrictions. ` app.SilenceUsage = true app.CompletionOptions = cobra.CompletionOptions{DisableDefaultCmd: true} // Global flags globalCmd := cmdGlobal{} app.PersistentFlags().BoolVar(&globalCmd.flagVersion, "version", false, "Print version number") app.PersistentFlags().BoolVarP(&globalCmd.flagHelp, "help", "h", false, "Print help") app.PersistentFlags().BoolVarP(&globalCmd.flagLogVerbose, "verbose", "v", false, "Show all information messages") app.PersistentFlags().BoolVarP(&globalCmd.flagLogDebug, "debug", "d", false, "Show debug messages") app.PersistentPreRunE = globalCmd.preRun // Version handling app.SetVersionTemplate("{{.Version}}\n") app.Version = version.Version // Run the main command and handle errors return app.Execute() } func main() { err := run() if err != nil { fmt.Fprintf(os.Stderr, "Error: %v", err) os.Exit(1) } } incus-7.0.0/cmd/incus-user/main_daemon.go000066400000000000000000000110441517523235500203050ustar00rootroot00000000000000package main import ( "errors" "fmt" "net" "os" "os/user" "strconv" "sync" "time" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/linux" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" ) var ( mu sync.RWMutex connections uint64 transactions uint64 ) var projectNames []string type cmdDaemon struct { flagGroup string } func (c *cmdDaemon) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "incus-user" cmd.RunE = c.run cmd.Flags().StringVar(&c.flagGroup, "group", "", "The group of users that will be allowed to talk to incus-user"+"``") return cmd } func (c *cmdDaemon) run(cmd *cobra.Command, args []string) error { // Only root should run this. if os.Geteuid() != 0 { return errors.New("This must be run as root") } // Create storage. err := os.MkdirAll(internalUtil.VarPath("users"), 0o700) if err != nil && !os.IsExist(err) { return fmt.Errorf("Couldn't create storage: %w", err) } // Connect. logger.Debug("Connecting to the daemon") client, err := incus.ConnectIncusUnix("", nil) if err != nil { return fmt.Errorf("Unable to connect to the daemon: %w", err) } cinfo, err := client.GetConnectionInfo() if err != nil { return fmt.Errorf("Failed to obtain connection info: %w", err) } // Keep track of the socket path we used to successfully connect to the server serverUnixPath := cinfo.SocketPath // Validate the configuration. ok, err := serverIsConfigured(client) if err != nil { return fmt.Errorf("Failed to check the configuration: %w", err) } if !ok { logger.Info("Performing initial configuration") err = serverInitialConfiguration(client) if err != nil { return fmt.Errorf("Failed to apply initial configuration: %w", err) } } // Pull the list of projects. projectNames, err = client.GetProjectNames() if err != nil { return fmt.Errorf("Failed to pull project list: %w", err) } // Disconnect. client.Disconnect() // Setup the unix socket. listeners := linux.GetSystemdListeners(linux.SystemdListenFDsStart) if len(listeners) > 1 { return errors.New("More than one socket-activation FD received") } var listener *net.UnixListener if len(listeners) == 1 { // Handle socket activation. unixListener, ok := listeners[0].(*net.UnixListener) if !ok { return errors.New("Socket-activation FD isn't a unix socket") } listener = unixListener // Automatically shutdown after inactivity. go func(unixListener *net.UnixListener) { for { time.Sleep(30 * time.Second) // Check for active connections. mu.RLock() if connections > 0 { mu.RUnlock() continue } // Look for recent activity oldCount := transactions mu.RUnlock() time.Sleep(5 * time.Second) mu.RLock() if oldCount == transactions { mu.RUnlock() // Daemon has been inactive for 10s, exit. logger.Info("Daemon has been inactive, shutting down") err := unixListener.Close() if err != nil { logger.Errorf("Failed to close Unix listener: %v", err) } return } mu.RUnlock() } }(listener) } else { // Create our own socket. unixPath := internalUtil.VarPath("unix.socket.user") err := os.Remove(unixPath) if err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("Failed to delete pre-existing unix socket: %w", err) } unixAddr, err := net.ResolveUnixAddr("unix", unixPath) if err != nil { return fmt.Errorf("Unable to resolve unix socket: %w", err) } server, err := net.ListenUnix("unix", unixAddr) if err != nil { return fmt.Errorf("Unable to setup unix socket: %w", err) } err = os.Chmod(unixPath, 0o660) if err != nil { return fmt.Errorf("Unable to set socket permissions: %w", err) } if c.flagGroup != "" { g, err := user.LookupGroup(c.flagGroup) if err != nil { return fmt.Errorf("Cannot get group ID of '%s': %w", c.flagGroup, err) } gid, err := strconv.Atoi(g.Gid) if err != nil { return err } err = os.Chown(unixPath, os.Getuid(), gid) if err != nil { return fmt.Errorf("Cannot change ownership on local socket: %w", err) } } server.SetUnlinkOnClose(true) listener = server } // Start accepting requests. logger.Info("Starting up the server") for { // Accept new connection. conn, err := listener.AcceptUnix() if err != nil { if errors.Is(err, net.ErrClosed) { return nil } logger.Errorf("Failed to accept new connection: %v", err) continue } go proxyConnection(conn, serverUnixPath) } } incus-7.0.0/cmd/incus-user/proxy.go000066400000000000000000000062211517523235500172200ustar00rootroot00000000000000package main import ( "crypto/tls" "fmt" "net" "os" "slices" log "github.com/sirupsen/logrus" "github.com/lxc/incus/v7/internal/linux" internalUtil "github.com/lxc/incus/v7/internal/util" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) func tlsConfig(uid uint32) (*tls.Config, error) { // Load the client certificate. content, err := os.ReadFile(internalUtil.VarPath("users", fmt.Sprintf("%d", uid), "client.crt")) if err != nil { return nil, fmt.Errorf("Unable to open client certificate: %w", err) } tlsClientCert := string(content) // Load the client key. content, err = os.ReadFile(internalUtil.VarPath("users", fmt.Sprintf("%d", uid), "client.key")) if err != nil { return nil, fmt.Errorf("Unable to open client key: %w", err) } tlsClientKey := string(content) // Load the server certificate. certPath := internalUtil.VarPath("cluster.crt") if !util.PathExists(certPath) { certPath = internalUtil.VarPath("server.crt") } content, err = os.ReadFile(certPath) if err != nil { return nil, fmt.Errorf("Unable to open server certificate: %w", err) } tlsServerCert := string(content) return localtls.GetTLSConfigMem(tlsClientCert, tlsClientKey, "", tlsServerCert, false) } func proxyConnection(conn *net.UnixConn, serverUnixPath string) { defer func() { _ = conn.Close() mu.Lock() connections -= 1 mu.Unlock() }() // Increase counters. mu.Lock() transactions += 1 connections += 1 mu.Unlock() // Get credentials. creds, err := linux.GetUcred(conn) if err != nil { log.Errorf("Unable to get user credentials: %s", err) return } // Setup logging context. logger := log.WithFields(log.Fields{ "uid": creds.Uid, "gid": creds.Gid, "pid": creds.Pid, }) logger.Debug("Connected") defer logger.Debug("Disconnected") // Check if the user was setup. if !util.PathExists(internalUtil.VarPath("users", fmt.Sprintf("%d", creds.Uid))) || !slices.Contains(projectNames, fmt.Sprintf("user-%d", creds.Uid)) { log.Infof("Setting up for uid %d", creds.Uid) err := serverSetupUser(creds.Uid) if err != nil { log.Errorf("Failed to setup new user: %v", err) return } } // Connect to the daemon. unixAddr, err := net.ResolveUnixAddr("unix", serverUnixPath) if err != nil { log.Errorf("Unable to resolve the target server: %v", err) return } client, err := net.DialUnix("unix", nil, unixAddr) if err != nil { log.Errorf("Unable to connect to target server: %v", err) return } defer func() { _ = client.Close() }() // Get the TLS configuration tlsConfig, err := tlsConfig(creds.Uid) if err != nil { log.Errorf("Failed to load TLS connection settings: %v", err) return } // Setup TLS. _, err = client.Write([]byte("STARTTLS\n")) if err != nil { log.Errorf("Failed to setup TLS connection to target server: %v", err) return } tlsClient := tls.Client(client, tlsConfig) // Establish the TLS handshake. err = tlsClient.Handshake() if err != nil { _ = conn.Close() log.Errorf("Failed TLS handshake with target server: %v", err) return } // Start proxying. go func() { _, _ = util.SafeCopy(conn, tlsClient) }() _, _ = util.SafeCopy(tlsClient, conn) } incus-7.0.0/cmd/incus-user/server.go000066400000000000000000000223001517523235500173410ustar00rootroot00000000000000package main import ( "encoding/base64" "errors" "fmt" "net/http" "os" "path/filepath" "slices" "strconv" "strings" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/linux" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) func serverIsConfigured(client incus.InstanceServer) (bool, error) { // Look for networks. networks, err := client.GetNetworkNames() if err != nil { return false, fmt.Errorf("Failed to list networks: %w", err) } if !slices.Contains(networks, "incusbr0") { // Couldn't find incusbr0. return false, nil } // Look for storage pools. pools, err := client.GetStoragePoolNames() if err != nil { return false, fmt.Errorf("Failed to list storage pools: %w", err) } if !slices.Contains(pools, "default") { // No storage pool found. return false, nil } return true, nil } func serverInitialConfiguration(client incus.InstanceServer) error { // Load current server config. info, _, err := client.GetServer() if err != nil { return fmt.Errorf("Failed to get server info: %w", err) } availableBackends := linux.AvailableStorageDrivers(internalUtil.VarPath(), info.Environment.StorageSupportedDrivers, internalUtil.PoolTypeLocal) // Load the default profile. var profileNeedsUpdate bool profile, profileEtag, err := client.GetProfile("default") if err != nil { return fmt.Errorf("Failed to load default profile: %w", err) } // Look for storage pools. pools, err := client.GetStoragePools() if err != nil { return fmt.Errorf("Failed to list storage pools: %w", err) } if len(pools) == 0 { pool := api.StoragePoolsPost{} pool.Config = map[string]string{} pool.Name = "default" // Check if ZFS supported. if slices.Contains(availableBackends, "zfs") { pool.Driver = "zfs" // Check if zsys. poolName, _ := subprocess.RunCommand("zpool", "get", "-H", "-o", "value", "name", "rpool") if strings.TrimSpace(poolName) == "rpool" { pool.Config["source"] = "rpool/incus" } } else { // Fallback to dir backend. pool.Driver = "dir" } // Create the storage pool. err := client.CreateStoragePool(pool) if err != nil { return fmt.Errorf("Failed to create storage pool: %w", err) } // Add to default profile in default project. profile.Devices["root"] = map[string]string{ "type": "disk", "pool": "default", "path": "/", } profileNeedsUpdate = true } // Look for networks. networks, err := client.GetNetworks() if err != nil { return fmt.Errorf("Failed to list networks: %w", err) } found := false for _, network := range networks { if network.Managed { found = true break } } if !found { // Create incusbr0. network := api.NetworksPost{} network.Config = map[string]string{} network.Type = "bridge" network.Name = "incusbr0" err := client.CreateNetwork(network) if err != nil { return fmt.Errorf("Failed to create network: %w", err) } // Add to default profile in default project. profile.Devices["eth0"] = map[string]string{ "type": "nic", "network": "incusbr0", "name": "eth0", } profileNeedsUpdate = true } // Update the default profile. if profileNeedsUpdate { err = client.UpdateProfile("default", profile.Writable(), profileEtag) if err != nil { return fmt.Errorf("Failed to update default profile: %w", err) } } return nil } func serverSetupUser(uid uint32) error { projectName := fmt.Sprintf("user-%d", uid) networkName := fmt.Sprintf("incusbr-%d", uid) if len(networkName) > 15 { // For long UIDs, use a shorter slightly less descriptive interface name. networkName = fmt.Sprintf("user-%d", uid) } userPath := internalUtil.VarPath("users", fmt.Sprintf("%d", uid)) // User account. out, err := subprocess.RunCommand("getent", "passwd", fmt.Sprintf("%d", uid)) if err != nil { return fmt.Errorf("Failed to retrieve user information: %w", err) } pw := strings.Split(out, ":") if len(pw) != 7 { return errors.New("Invalid user entry") } // Setup reverter. reverter := revert.New() defer reverter.Fail() // Create certificate directory. err = os.MkdirAll(userPath, 0o700) if err != nil { return fmt.Errorf("Failed to create user directory: %w", err) } reverter.Add(func() { _ = os.RemoveAll(userPath) }) // Generate certificate. if !util.PathExists(filepath.Join(userPath, "client.crt")) || !util.PathExists(filepath.Join(userPath, "client.key")) { err = localtls.FindOrGenCert(filepath.Join(userPath, "client.crt"), filepath.Join(userPath, "client.key"), true, false) if err != nil { return fmt.Errorf("Failed to generate user certificate: %w", err) } } // Connect to the daemon. client, err := incus.ConnectIncusUnix("", nil) if err != nil { return fmt.Errorf("Unable to connect to the daemon: %w", err) } // Don't reset setup if the user already has a valid certificate and project access. checkCert, err := localtls.ReadCert(filepath.Join(userPath, "client.crt")) if err == nil { // Check if the certificate exists in the trust store. checkFingerprint := localtls.CertFingerprint(checkCert) existingCert, _, err := client.GetCertificate(checkFingerprint) if err == nil && len(existingCert.Projects) > 0 { return nil } } _, _, _ = client.GetServer() if !slices.Contains(projectNames, projectName) { // Create the project. err := client.CreateProject(api.ProjectsPost{ Name: projectName, ProjectPut: api.ProjectPut{ Description: fmt.Sprintf("User restricted project for %q (%s)", pw[0], pw[2]), Config: map[string]string{ "features.images": "true", "features.networks": "false", "features.networks.zones": "true", "features.profiles": "true", "features.storage.volumes": "true", "features.storage.buckets": "true", "restricted": "true", "restricted.containers.nesting": "allow", "restricted.devices.disk": "allow", "restricted.devices.disk.paths": pw[5], "restricted.devices.gpu": "allow", "restricted.idmap.uid": pw[2], "restricted.idmap.gid": pw[3], "restricted.networks.access": networkName, }, }, }) if err != nil { return fmt.Errorf("Unable to create project: %w", err) } reverter.Add(func() { _ = client.DeleteProject(projectName) }) // Create user-specific bridge. network := api.NetworksPost{} network.Config = map[string]string{} network.Type = "bridge" network.Name = networkName network.Description = fmt.Sprintf("Network for user restricted project %s", projectName) err = client.CreateNetwork(network) if err != nil && !api.StatusErrorCheck(err, http.StatusConflict) { return fmt.Errorf("Failed to create network: %w", err) } // Setup default profile. req := api.ProfilePut{ Description: "Default Incus profile", Devices: map[string]map[string]string{ "root": { "type": "disk", "path": "/", "pool": "default", }, "eth0": { "type": "nic", "name": "eth0", "network": networkName, }, }, } // Add uid/gid map if possible. pwUID, err := strconv.ParseInt(pw[2], 10, 64) if err != nil { return err } pwGID, err := strconv.ParseInt(pw[3], 10, 64) if err != nil { return err } idmapset, err := idmap.NewSetFromSystem("root") if err != nil && !errors.Is(err, idmap.ErrSubidUnsupported) { return fmt.Errorf("Failed to load system idmap: %w", err) } idmapAllowed := true if idmapset != nil { entries := []idmap.Entry{ {IsUID: true, HostID: pwUID, MapRange: 1}, {IsGID: true, HostID: pwGID, MapRange: 1}, } if !idmapset.Includes(&idmap.Set{Entries: entries}) { idmapAllowed = false } } if idmapAllowed { req.Config = map[string]string{ "raw.idmap": fmt.Sprintf("uid %d %d\ngid %d %d", pwUID, pwUID, pwGID, pwGID), } } err = client.UseProject(projectName).UpdateProfile("default", req, "") if err != nil { return fmt.Errorf("Unable to update the default profile: %w", err) } } // Parse the certificate. x509Cert, err := localtls.ReadCert(filepath.Join(userPath, "client.crt")) if err != nil { return fmt.Errorf("Unable to read user certificate: %w", err) } // Delete the certificate from the trust store if it already exists. fingerprint := localtls.CertFingerprint(x509Cert) _ = client.DeleteCertificate(fingerprint) // Add the certificate to the trust store. err = client.CreateCertificate(api.CertificatesPost{ CertificatePut: api.CertificatePut{ Name: fmt.Sprintf("incus-user-%d", uid), Type: "client", Restricted: true, Projects: []string{projectName}, Certificate: base64.StdEncoding.EncodeToString(x509Cert.Raw), }, }) if err != nil { return fmt.Errorf("Unable to add user certificate: %w", err) } reverter.Add(func() { _ = client.DeleteCertificate(localtls.CertFingerprint(x509Cert)) }) // Add the new project to our list. if !slices.Contains(projectNames, projectName) { projectNames = append(projectNames, projectName) } reverter.Success() return nil } incus-7.0.0/cmd/incus/000077500000000000000000000000001517523235500145335ustar00rootroot00000000000000incus-7.0.0/cmd/incus/action.go000066400000000000000000000253711517523235500163470ustar00rootroot00000000000000package main import ( "errors" "fmt" "os" "slices" "strings" "sync" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" config "github.com/lxc/incus/v7/shared/cliconfig" cli "github.com/lxc/incus/v7/shared/cmd" ) // Start. type cmdStart struct { global *cmdGlobal action *cmdAction } var cmdActionUsage = u.Usage{u.Either(u.Instance.Remote().List(1), u.Sequence(u.Flag("all"), u.RemoteColon.List(0)))} func (c *cmdStart) command() *cobra.Command { cmdAction := cmdAction{global: c.global} c.action = &cmdAction cmd := c.action.command("start") cmd.Use = cli.U("start", cmdActionUsage...) cmd.Short = i18n.G("Start instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Start instances`)) cmd.ValidArgsFunction = func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpInstances(toComplete) } return cmd } // Pause. type cmdPause struct { global *cmdGlobal action *cmdAction } func (c *cmdPause) command() *cobra.Command { cmdAction := cmdAction{global: c.global} c.action = &cmdAction cmd := c.action.command("pause") cmd.Use = cli.U("pause", cmdActionUsage...) cmd.Short = i18n.G("Pause instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Pause instances`)) cmd.Aliases = []string{"freeze"} cmd.ValidArgsFunction = func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpInstances(toComplete) } return cmd } // Resume. type cmdResume struct { global *cmdGlobal action *cmdAction } func (c *cmdResume) command() *cobra.Command { cmdAction := cmdAction{global: c.global} c.action = &cmdAction cmd := c.action.command("resume") cmd.Use = cli.U("resume", cmdActionUsage...) cmd.Short = i18n.G("Resume instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Resume instances`)) cmd.Aliases = []string{"unfreeze"} cmd.ValidArgsFunction = func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpInstances(toComplete) } return cmd } // Restart. type cmdRestart struct { global *cmdGlobal action *cmdAction } func (c *cmdRestart) command() *cobra.Command { cmdAction := cmdAction{global: c.global} c.action = &cmdAction cmd := c.action.command("restart") cmd.Use = cli.U("restart", cmdActionUsage...) cmd.Short = i18n.G("Restart instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Restart instances`)) cmd.ValidArgsFunction = func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpInstances(toComplete) } return cmd } // Stop. type cmdStop struct { global *cmdGlobal action *cmdAction } func (c *cmdStop) command() *cobra.Command { cmdAction := cmdAction{global: c.global} c.action = &cmdAction cmd := c.action.command("stop") cmd.Use = cli.U("stop", cmdActionUsage...) cmd.Short = i18n.G("Stop instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Stop instances`)) cmd.ValidArgsFunction = func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpInstances(toComplete) } return cmd } type cmdAction struct { global *cmdGlobal flagAll bool flagConsole string flagForce bool flagStateful bool flagStateless bool flagTimeout int } func (c *cmdAction) command(action string) *cobra.Command { cmd := &cobra.Command{} cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagAll, "all|a", i18n.G("Run against all instances")) switch action { case "stop": cli.AddBoolFlag(cmd.Flags(), &c.flagStateful, "stateful", i18n.G("Store the instance state")) case "start": cli.AddBoolFlag(cmd.Flags(), &c.flagStateless, "stateless", i18n.G("Ignore the instance state")) } if slices.Contains([]string{"start", "restart", "stop"}, action) { cli.AddStringFlag(cmd.Flags(), &c.flagConsole, "console", "", "console", i18n.G("Immediately attach to the console")) } if slices.Contains([]string{"restart", "stop"}, action) { cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G("Force the instance to stop")) cli.AddIntFlag(cmd.Flags(), &c.flagTimeout, "timeout", -1, i18n.G("Time to wait for the instance to shutdown cleanly")) } return cmd } // doActionAll is a method of the cmdAction structure. It performs a specified action on all instances of a remote resource. // It ensures that flags and parameters are appropriately set, and handles any errors that may occur during the process. func (c *cmdAction) doActionAll(action string, d incus.InstanceServer) error { // Pause is called freeze, resume is called unfreeze. switch action { case "pause": action = "freeze" case "resume": action = "unfreeze" } // Only store state if asked to. state := action == "stop" && c.flagStateful req := api.InstancesPut{ State: &api.InstanceStatePut{ Action: action, Timeout: c.flagTimeout, Force: c.flagForce, Stateful: state, }, } // Update all instances. op, err := d.UpdateInstances(req, "") if err != nil { return err } progress := cli.ProgressRenderer{ Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } progress.Done("") return nil } // doAction is a method of the cmdAction structure. It carries out a specified action on an instance, // using a given config and instance name. It manages state changes, flag checks, error handling and console attachment. func (c *cmdAction) doAction(action string, conf *config.Config, p *u.Parsed) error { d := p.RemoteServer instanceName := p.RemoteObject.String state := false // Pause is called freeze if action == "pause" { action = "freeze" } // Resume is called unfreeze if action == "resume" { action = "unfreeze" } // Only store state if asked to if action == "stop" && c.flagStateful { state = true } if action == "stop" && c.flagForce && c.flagConsole != "" { return errors.New(i18n.G("--console can't be used while forcing instance shutdown")) } if action == "start" { current, _, err := d.GetInstance(instanceName) if err != nil { return err } // "start" for a frozen instance means "unfreeze" if current.StatusCode == api.Frozen { action = "unfreeze" } // Always restore state (if present) unless asked not to if action == "start" && current.Stateful && !c.flagStateless { state = true } } req := api.InstanceStatePut{ Action: action, Timeout: c.flagTimeout, Force: c.flagForce, Stateful: state, } op, err := d.UpdateInstanceState(instanceName, req, "") if err != nil { return err } if action == "stop" && c.flagConsole != "" { // Handle console attach console := cmdConsole{} console.global = c.global console.flagType = c.flagConsole return console.console(d, instanceName) } progress := cli.ProgressRenderer{ Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } // Wait for operation to finish err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") projectArg := "" if conf.ProjectOverride != "" && conf.ProjectOverride != api.ProjectDefaultName { projectArg = " --project " + conf.ProjectOverride } return fmt.Errorf("%s\n"+i18n.G("Try `incus info --show-log %s%s` for more info"), err, formatRemote(conf, p), projectArg) } progress.Done("") // Handle console attach if c.flagConsole != "" { console := cmdConsole{} console.global = c.global console.flagType = c.flagConsole consoleErr := console.console(d, instanceName) if consoleErr != nil { // Check if still running. state, _, err := d.GetInstanceState(instanceName) if err != nil { return err } if state.StatusCode != api.Stopped { return consoleErr } console.flagShowLog = true return console.console(d, instanceName) } } return nil } // It handles actions on instances (single or all) and manages error handling, console flag restrictions, and batch operations. func (c *cmdAction) run(cmd *cobra.Command, args []string) error { parsed, err := cmdActionUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } action := cmd.Name() type batchEntry struct { parsed *u.Parsed err error } var batch []batchEntry if c.flagAll { if parsed[0].BranchID != 1 { return errors.New(i18n.G("Both --all and instance name given")) } parsedRemotes := parsed[0].List[1].List // If no remote passed, use current default. if len(parsedRemotes) == 0 { p, err := u.ParseDefault(u.RemoteColonOpt, c.global.conf) if err != nil { return err } parsedRemotes = append(parsedRemotes, p) } for _, p := range parsedRemotes { d := p.RemoteServer // See if we can use the bulk API. if d.HasExtension("instance_bulk_state_change") { err = c.doActionAll(action, d) if err != nil { return fmt.Errorf("%s: %w", p.RemoteName, err) } continue } instances, err := d.GetInstances(api.InstanceTypeAny) if err != nil { return err } for _, instance := range instances { switch action { case "start": if instance.StatusCode == api.Running { continue } case "stop": if instance.StatusCode == api.Stopped { continue } } reparsed := *p reparsed.RemoteObject = u.ParseString(instance.Name) batch = append(batch, batchEntry{&reparsed, nil}) } } } else { for _, p := range parsed[0].List { batch = append(batch, batchEntry{p, nil}) } } if c.flagConsole != "" { if c.flagAll { return errors.New(i18n.G("--console can't be used with --all")) } if len(batch) > 1 { return errors.New(i18n.G("--console only works with a single instance")) } } // Run the action for every listed instance var wg sync.WaitGroup wg.Add(len(batch)) for i := range batch { go func(entry *batchEntry) { defer wg.Done() entry.err = c.doAction(action, c.global.conf, entry.parsed) }(&batch[i]) } wg.Wait() // Single instance is easy if len(batch) == 1 { return batch[0].err } // Do fancier rendering for batches success := true for _, entry := range batch { if entry.err == nil { continue } success = false msg := fmt.Sprintf(i18n.G("error: %v"), entry.err) for _, line := range strings.Split(msg, "\n") { fmt.Fprintf(os.Stderr, "%s: %s\n", formatRemote(c.global.conf, entry.parsed), line) } } if !success { fmt.Fprintln(os.Stderr, "") return fmt.Errorf(i18n.G("Some instances failed to %s"), action) } return nil } incus-7.0.0/cmd/incus/admin.go000066400000000000000000000023351517523235500161550ustar00rootroot00000000000000//go:build linux package main import ( "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" "github.com/lxc/incus/v7/internal/i18n" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdAdmin struct { global *cmdGlobal } func (c *cmdAdmin) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("admin") cmd.Short = i18n.G("Manage incus daemon") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage incus daemon`)) // cluster adminClusterCmd := cmdAdminCluster{global: c.global} cmd.AddCommand(adminClusterCmd.command()) // init adminInitCmd := cmdAdminInit{global: c.global} cmd.AddCommand(adminInitCmd.command()) // os adminOSCmd := cmdAdminOS{global: c.global} cmd.AddCommand(adminOSCmd.command()) // recover sub-command adminRecoverCmd := cmdAdminRecover{global: c.global} cmd.AddCommand(adminRecoverCmd.command()) // shutdown sub-command shutdownCmd := cmdAdminShutdown{global: c.global} cmd.AddCommand(shutdownCmd.command()) // sql sub-command sqlCmd := cmdAdminSQL{global: c.global} cmd.AddCommand(sqlCmd.command()) // waitready sub-command adminWaitreadyCmd := cmdAdminWaitready{global: c.global} cmd.AddCommand(adminWaitreadyCmd.command()) return cmd } incus-7.0.0/cmd/incus/admin_cluster.go000066400000000000000000000026761517523235500177260ustar00rootroot00000000000000//go:build linux package main import ( "fmt" "os" "os/exec" "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" "github.com/lxc/incus/v7/internal/i18n" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/util" ) type cmdAdminCluster struct { global *cmdGlobal } func (c *cmdAdminCluster) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("cluster") cmd.Short = i18n.G("Low-level cluster administration commands") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Low level administration tools for inspecting and recovering clusters.`)) cmd.Run = c.run return cmd } func (c *cmdAdminCluster) run(_ *cobra.Command, args []string) { env := getEnviron() path, _ := exec.LookPath("incusd") if path == "" { if util.PathExists("/usr/libexec/incus/incusd") { path = "/usr/libexec/incus/incusd" } else if util.PathExists("/usr/lib/incus/incusd") { path = "/usr/lib/incus/incusd" } else if util.PathExists("/opt/incus/bin/incusd") { path = "/opt/incus/bin/incusd" env = append(env, "LD_LIBRARY_PATH=/opt/incus/lib/") } } if path == "" { fmt.Println(i18n.G(`The "cluster" subcommand requires access to internal server data. To do so, it's actually part of the "incusd" binary rather than "incus". You can invoke it through "incusd cluster".`)) os.Exit(1) // nolint:revive } _ = doExec(path, append([]string{"incusd", "admin", "cluster"}, args...), env) } incus-7.0.0/cmd/incus/admin_init.go000066400000000000000000000112371517523235500172010ustar00rootroot00000000000000//go:build linux package main import ( "errors" "fmt" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/internal/ports" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdAdminInit struct { global *cmdGlobal flagAuto bool flagMinimal bool flagPreseed bool flagDump bool flagNetworkAddress string flagNetworkPort int flagStorageBackend string flagStorageDevice string flagStorageLoopSize int flagStoragePool string } var cmdAdminInitUsage = u.Usage{u.Sequence(u.Flag("preseed"), u.Placeholder(i18n.G("preseed.yaml")).Optional()).Optional()} func (c *cmdAdminInit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("init", cmdAdminInitUsage...) cmd.Short = i18n.G("Configure the daemon") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Configure the daemon`)) cmd.Example = ` init --minimal init --auto [--network-address=IP] [--network-port=8443] [--storage-backend=dir] [--storage-create-device=DEVICE] [--storage-create-loop=SIZE] [--storage-pool=POOL] init --preseed [preseed.yaml] init --dump ` cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagAuto, "auto", i18n.G("Automatic (non-interactive) mode")) cli.AddBoolFlag(cmd.Flags(), &c.flagMinimal, "minimal", i18n.G("Minimal configuration (non-interactive)")) cli.AddBoolFlag(cmd.Flags(), &c.flagPreseed, "preseed", i18n.G("Pre-seed mode, expects YAML config from stdin")) cli.AddBoolFlag(cmd.Flags(), &c.flagDump, "dump", i18n.G("Dump YAML config to stdout")) cli.AddStringFlag(cmd.Flags(), &c.flagNetworkAddress, "network-address", "", "", i18n.G("Address to bind to (default: none)")) cli.AddIntFlag(cmd.Flags(), &c.flagNetworkPort, "network-port", -1, fmt.Sprintf(i18n.G("Port to bind to (default: %d)"), ports.HTTPSDefaultPort)) cli.AddStringFlag(cmd.Flags(), &c.flagStorageBackend, "storage-backend", "", "", i18n.G("Storage backend to use (btrfs, dir, lvm or zfs, default: dir)")) cli.AddStringFlag(cmd.Flags(), &c.flagStorageDevice, "storage-create-device", "", "", i18n.G("Setup device based storage using DEVICE")) cli.AddIntFlag(cmd.Flags(), &c.flagStorageLoopSize, "storage-create-loop", -1, i18n.G("Setup loop based storage with SIZE in GiB")) cli.AddStringFlag(cmd.Flags(), &c.flagStoragePool, "storage-pool", "", "", i18n.G("Storage pool to use or create")) return cmd } func (c *cmdAdminInit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdAdminInitUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } // Quick checks. if c.flagAuto && c.flagPreseed { return errors.New(i18n.G("Can't use --auto and --preseed together")) } if c.flagMinimal && c.flagPreseed { return errors.New(i18n.G("Can't use --minimal and --preseed together")) } if c.flagMinimal && c.flagAuto { return errors.New(i18n.G("Can't use --minimal and --auto together")) } if !c.flagAuto && (c.flagNetworkAddress != "" || c.flagNetworkPort != -1 || c.flagStorageBackend != "" || c.flagStorageDevice != "" || c.flagStorageLoopSize != -1 || c.flagStoragePool != "") { return errors.New(i18n.G("Configuration flags require --auto")) } if c.flagDump && (c.flagAuto || c.flagMinimal || c.flagPreseed || c.flagNetworkAddress != "" || c.flagNetworkPort != -1 || c.flagStorageBackend != "" || c.flagStorageDevice != "" || c.flagStorageLoopSize != -1 || c.flagStoragePool != "") { return errors.New(i18n.G("Can't use --dump with other flags")) } // Connect to the daemon d, err := incus.ConnectIncusUnix("", nil) if err != nil { return fmt.Errorf(i18n.G("Failed to connect to local daemon: %w"), err) } server, _, err := d.GetServer() if err != nil { return fmt.Errorf(i18n.G("Failed to connect to get server info: %w"), err) } // Dump mode if c.flagDump { err := c.runDump(d) if err != nil { return err } return nil } // Prepare the input data var config *api.InitPreseed switch { case c.flagPreseed: config, err = c.runPreseed(parsed[0].List[1]) if err != nil { return err } case c.flagAuto || c.flagMinimal: config, err = c.runAuto(d, server) if err != nil { return err } default: config, err = c.runInteractive(cmd, d, server) if err != nil { return err } } err = fillClusterConfig(config) if err != nil { return err } if config.Cluster != nil && config.Cluster.ClusterAddress != "" && config.Cluster.ServerAddress != "" { err = updateCluster(d, config) if err != nil { return err } return nil } return d.ApplyServerPreseed(*config) } incus-7.0.0/cmd/incus/admin_init_auto.go000066400000000000000000000135631517523235500202350ustar00rootroot00000000000000//go:build linux package main import ( "errors" "fmt" "slices" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/ports" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) func (c *cmdAdminInit) runAuto(d incus.InstanceServer, server *api.Server) (*api.InitPreseed, error) { // Quick checks. if c.flagStorageBackend != "" && !slices.Contains([]string{"dir", "btrfs", "lvm", "zfs"}, c.flagStorageBackend) { return nil, fmt.Errorf(i18n.G("The requested backend '%s' isn't supported by init"), c.flagStorageBackend) } if c.flagStorageBackend != "" && !slices.Contains(linux.AvailableStorageDrivers(internalUtil.VarPath(), server.Environment.StorageSupportedDrivers, internalUtil.PoolTypeAny), c.flagStorageBackend) { return nil, fmt.Errorf(i18n.G("The requested backend '%s' isn't available on your system (missing tools)"), c.flagStorageBackend) } if c.flagStorageBackend == "dir" || c.flagStorageBackend == "" { if c.flagStorageLoopSize != -1 || c.flagStorageDevice != "" || c.flagStoragePool != "" { return nil, errors.New(i18n.G("None of --storage-pool, --storage-create-device or --storage-create-loop may be used with the 'dir' backend")) } } else { if c.flagStorageLoopSize != -1 && c.flagStorageDevice != "" { return nil, errors.New(i18n.G("Only one of --storage-create-device or --storage-create-loop can be specified")) } } if c.flagNetworkAddress == "" { if c.flagNetworkPort != -1 { return nil, errors.New(i18n.G("--network-port can't be used without --network-address")) } } storagePools, err := d.GetStoragePoolNames() if err != nil { return nil, fmt.Errorf(i18n.G("Failed to retrieve list of storage pools: %w"), err) } if len(storagePools) > 0 && (c.flagStorageBackend != "" || c.flagStorageDevice != "" || c.flagStorageLoopSize != -1 || c.flagStoragePool != "") { return nil, errors.New(i18n.G("Storage has already been configured")) } // Detect the backing filesystem. backingFs, err := linux.DetectFilesystem(internalUtil.VarPath()) if err != nil { backingFs = "dir" } // Get the possible local storage drivers. storageDrivers := linux.AvailableStorageDrivers(internalUtil.VarPath(), server.Environment.StorageSupportedDrivers, internalUtil.PoolTypeLocal) // Defaults if c.flagNetworkPort == -1 { c.flagNetworkPort = ports.HTTPSDefaultPort } if c.flagStorageBackend == "" && c.flagStoragePool == "" && backingFs == "btrfs" && slices.Contains(storageDrivers, "btrfs") { // Use btrfs subvol if running on btrfs. c.flagStoragePool = internalUtil.VarPath("storage-pools", "default") c.flagStorageBackend = "btrfs" } else if c.flagStorageBackend == "" { c.flagStorageBackend = "dir" } // Fill in the node configuration config := api.InitLocalPreseed{} config.Config = map[string]string{} // Network listening if c.flagNetworkAddress != "" { config.Config["core.https_address"] = internalUtil.CanonicalNetworkAddressFromAddressAndPort(c.flagNetworkAddress, c.flagNetworkPort, ports.HTTPSDefaultPort) } // Storage configuration if len(storagePools) == 0 { // Storage pool pool := api.StoragePoolsPost{ Name: "default", Driver: c.flagStorageBackend, } pool.Config = map[string]string{} if c.flagStorageDevice != "" { pool.Config["source"] = c.flagStorageDevice } else if c.flagStorageLoopSize > 0 { pool.Config["size"] = fmt.Sprintf("%dGiB", c.flagStorageLoopSize) } else { pool.Config["source"] = c.flagStoragePool } // If using a device or loop, --storage-pool refers to the name of the new pool if c.flagStoragePool != "" && (c.flagStorageDevice != "" || c.flagStorageLoopSize != -1) { pool.Name = c.flagStoragePool } config.StoragePools = []api.StoragePoolsPost{pool} // Profile entry config.Profiles = []api.InitProfileProjectPost{{ ProfilesPost: api.ProfilesPost{ Name: "default", ProfilePut: api.ProfilePut{ Devices: map[string]map[string]string{ "root": { "type": "disk", "path": "/", "pool": pool.Name, }, }, }, }, Project: api.ProjectDefaultName, }} } // Network configuration networks, err := d.GetNetworks() if err != nil { return nil, fmt.Errorf(i18n.G("Failed to retrieve list of networks: %w"), err) } // Extract managed networks managedNetworks := []api.Network{} for _, network := range networks { if network.Managed { managedNetworks = append(managedNetworks, network) } } // Look for an existing network device in the profile defaultProfileNetwork := false defaultProfile, _, err := d.GetProfile("default") if err == nil { for _, dev := range defaultProfile.Devices { if dev["type"] == "nic" { defaultProfileNetwork = true break } } } // Define a new network if len(managedNetworks) == 0 && !defaultProfileNetwork { // Find a new name idx := 0 for { if util.PathExists(fmt.Sprintf("/sys/class/net/incusbr%d", idx)) { idx++ continue } break } // Define the new network network := api.InitNetworksProjectPost{} network.Name = fmt.Sprintf("incusbr%d", idx) network.Project = api.ProjectDefaultName config.Networks = append(config.Networks, network) // Add it to the profile if config.Profiles == nil { config.Profiles = []api.InitProfileProjectPost{{ ProfilesPost: api.ProfilesPost{ Name: "default", ProfilePut: api.ProfilePut{ Devices: map[string]map[string]string{ "eth0": { "type": "nic", "network": network.Name, "name": "eth0", }, }, }, }, Project: api.ProjectDefaultName, }} } else { config.Profiles[0].Devices["eth0"] = map[string]string{ "type": "nic", "network": network.Name, "name": "eth0", } } } return &api.InitPreseed{InitLocalPreseed: config}, nil } incus-7.0.0/cmd/incus/admin_init_dump.go000066400000000000000000000053741517523235500202330ustar00rootroot00000000000000//go:build linux package main import ( "fmt" yaml "go.yaml.in/yaml/v4" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" ) func (c *cmdAdminInit) runDump(d incus.InstanceServer) error { currentServer, _, err := d.GetServer() if err != nil { return fmt.Errorf(i18n.G("Failed to retrieve current server configuration: %w"), err) } var config api.InitLocalPreseed config.Config = currentServer.Config // Only retrieve networks in the default project as the preseed format doesn't support creating // projects at this time. networks, err := d.UseProject(api.ProjectDefaultName).GetNetworks() if err != nil { return fmt.Errorf(i18n.G("Failed to retrieve current server network configuration for project %q: %w"), api.ProjectDefaultName, err) } for _, network := range networks { // Only list managed networks. if !network.Managed { continue } networksPost := api.InitNetworksProjectPost{} networksPost.Config = network.Config networksPost.Description = network.Description networksPost.Name = network.Name networksPost.Type = network.Type networksPost.Project = api.ProjectDefaultName config.Networks = append(config.Networks, networksPost) } storagePools, err := d.GetStoragePools() if err != nil { return fmt.Errorf(i18n.G("Failed to retrieve current server configuration: %w"), err) } for _, storagePool := range storagePools { storagePoolsPost := api.StoragePoolsPost{} storagePoolsPost.Config = storagePool.Config storagePoolsPost.Description = storagePool.Description storagePoolsPost.Name = storagePool.Name storagePoolsPost.Driver = storagePool.Driver config.StoragePools = append(config.StoragePools, storagePoolsPost) } profiles, err := d.GetProfiles() if err != nil { return fmt.Errorf(i18n.G("Failed to retrieve current server configuration: %w"), err) } for _, profile := range profiles { profilesPost := api.InitProfileProjectPost{} profilesPost.Config = profile.Config profilesPost.Description = profile.Description profilesPost.Devices = profile.Devices profilesPost.Name = profile.Name config.Profiles = append(config.Profiles, profilesPost) } projects, err := d.GetProjects() if err != nil { return fmt.Errorf(i18n.G("Failed to retrieve current server configuration: %w"), err) } for _, project := range projects { projectsPost := api.ProjectsPost{} projectsPost.Config = project.Config projectsPost.Description = project.Description projectsPost.Name = project.Name config.Projects = append(config.Projects, projectsPost) } out, err := yaml.Dump(config, yaml.V2) if err != nil { return fmt.Errorf(i18n.G("Failed to retrieve current server configuration: %w"), err) } fmt.Printf("%s\n", out) return nil } incus-7.0.0/cmd/incus/admin_init_interactive.go000066400000000000000000000451261517523235500216020ustar00rootroot00000000000000//go:build linux package main import ( "errors" "fmt" "net" "os/exec" "slices" "strconv" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "golang.org/x/sys/unix" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/ports" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) func (c *cmdAdminInit) runInteractive(_ *cobra.Command, d incus.InstanceServer, server *api.Server) (*api.InitPreseed, error) { // Initialize config config := newInitPressed() // Clustering clustering, err := c.global.asker.AskBool(i18n.G("Would you like to use clustering?")+" (yes/no) [default=no]: ", "no") if err != nil { return nil, err } if clustering { err := askClustering(c.global.asker, config, nil, d, false) if err != nil { return nil, err } } // Ask all the other questions if config.Cluster == nil || config.Cluster.ClusterAddress == "" { // Storage err = c.askStorage(config, d, server) if err != nil { return nil, err } // Networking err = c.askNetworking(config, d) if err != nil { return nil, err } // Daemon config err = c.askDaemon(config, server) if err != nil { return nil, err } } // Print the YAML preSeedPrint, err := c.global.asker.AskBool(i18n.G("Would you like a YAML \"init\" preseed to be printed?")+" (yes/no) [default=no]: ", "no") if err != nil { return nil, err } if preSeedPrint { var object api.InitPreseed // If the user has chosen to join an existing cluster, print // only YAML for the cluster section, which is the only // relevant one. Otherwise print the regular config. if config.Cluster != nil && config.Cluster.ClusterAddress != "" { object = api.InitPreseed{} object.Cluster = config.Cluster } else { object = *config } out, err := yaml.Dump(object, yaml.V2) if err != nil { return nil, fmt.Errorf(i18n.G("Failed to render the config: %w"), err) } fmt.Printf("%s\n", out) } return config, nil } func (c *cmdAdminInit) askNetworking(config *api.InitPreseed, d incus.InstanceServer) error { var err error localBridgeCreate := false if config.Cluster == nil { localBridgeCreate, err = c.global.asker.AskBool(i18n.G("Would you like to create a new local network bridge?")+" (yes/no) [default=yes]: ", "yes") if err != nil { return err } } if !localBridgeCreate { useExistingInterface, err := c.global.asker.AskBool(i18n.G("Would you like to use an existing bridge or host interface?")+" (yes/no) [default=no]: ", "no") if err != nil { return err } if useExistingInterface { for { interfaceName, err := c.global.asker.AskString(i18n.G("Name of the existing bridge or host interface:")+" ", "", nil) if err != nil { return err } if !util.PathExists(fmt.Sprintf("/sys/class/net/%s", interfaceName)) { fmt.Println(i18n.G("The requested interface doesn't exist. Please choose another one.")) continue } // Add to the default profile config.Profiles[0].Devices["eth0"] = map[string]string{ "type": "nic", "nictype": "macvlan", "name": "eth0", "parent": interfaceName, } if util.PathExists(fmt.Sprintf("/sys/class/net/%s/bridge", interfaceName)) { config.Profiles[0].Devices["eth0"]["nictype"] = "bridged" } break } } return nil } for { // Define the network network := api.InitNetworksProjectPost{} network.Config = map[string]string{} network.Project = api.ProjectDefaultName // Network name network.Name, err = c.global.asker.AskString(i18n.G("What should the new bridge be called?")+" [default=incusbr0]: ", "incusbr0", validate.IsInterfaceName) if err != nil { return err } _, _, err = d.GetNetwork(network.Name) if err == nil { fmt.Printf(i18n.G("The requested network bridge \"%s\" already exists. Please choose another name.")+"\n", network.Name) continue } // Add to the default profile config.Profiles[0].Devices["eth0"] = map[string]string{ "type": "nic", "name": "eth0", "network": network.Name, } // IPv4 network.Config["ipv4.address"], err = c.global.asker.AskString(i18n.G("What IPv4 address should be used?")+" (CIDR subnet notation, “auto” or “none”) [default=auto]: ", "auto", func(value string) error { if slices.Contains([]string{"auto", "none"}, value) { return nil } return validate.Optional(validate.IsNetworkAddressCIDRV4)(value) }) if err != nil { return err } if !slices.Contains([]string{"auto", "none"}, network.Config["ipv4.address"]) { netIPv4UseNAT, err := c.global.asker.AskBool(i18n.G("Would you like to NAT IPv4 traffic on your bridge?")+" [default=yes]: ", "yes") if err != nil { return err } network.Config["ipv4.nat"] = fmt.Sprintf("%v", netIPv4UseNAT) } // IPv6 network.Config["ipv6.address"], err = c.global.asker.AskString(i18n.G("What IPv6 address should be used?")+" (CIDR subnet notation, “auto” or “none”) [default=auto]: ", "auto", func(value string) error { if slices.Contains([]string{"auto", "none"}, value) { return nil } return validate.Optional(validate.IsNetworkAddressCIDRV6)(value) }) if err != nil { return err } if !slices.Contains([]string{"auto", "none"}, network.Config["ipv6.address"]) { netIPv6UseNAT, err := c.global.asker.AskBool(i18n.G("Would you like to NAT IPv6 traffic on your bridge?")+" [default=yes]: ", "yes") if err != nil { return err } network.Config["ipv6.nat"] = fmt.Sprintf("%v", netIPv6UseNAT) } // Add the new network config.Networks = append(config.Networks, network) break } return nil } func (c *cmdAdminInit) askStorage(config *api.InitPreseed, d incus.InstanceServer, server *api.Server) error { if config.Cluster != nil { localStoragePool, err := c.global.asker.AskBool(i18n.G("Do you want to configure a new local storage pool?")+" (yes/no) [default=yes]: ", "yes") if err != nil { return err } if localStoragePool { err := c.askStoragePool(config, d, server, internalUtil.PoolTypeLocal) if err != nil { return err } } remoteStoragePool, err := c.global.asker.AskBool(i18n.G("Do you want to configure a new remote storage pool?")+" (yes/no) [default=no]: ", "no") if err != nil { return err } if remoteStoragePool { err := c.askStoragePool(config, d, server, internalUtil.PoolTypeRemote) if err != nil { return err } } return nil } storagePool, err := c.global.asker.AskBool(i18n.G("Do you want to configure a new storage pool?")+" (yes/no) [default=yes]: ", "yes") if err != nil { return err } if !storagePool { return nil } return c.askStoragePool(config, d, server, internalUtil.PoolTypeAny) } func (c *cmdAdminInit) askStoragePool(config *api.InitPreseed, d incus.InstanceServer, server *api.Server, poolType internalUtil.PoolType) error { // Figure out the preferred storage driver availableBackends := linux.AvailableStorageDrivers(internalUtil.VarPath(), server.Environment.StorageSupportedDrivers, poolType) if len(availableBackends) == 0 { if poolType != internalUtil.PoolTypeAny { return errors.New(i18n.G("No storage backends available")) } return fmt.Errorf(i18n.G("No %s storage backends available"), poolType) } backingFs, err := linux.DetectFilesystem(internalUtil.VarPath()) if err != nil { backingFs = "dir" } defaultStorage := "dir" if backingFs == "btrfs" && slices.Contains(availableBackends, "btrfs") { defaultStorage = "btrfs" } else if slices.Contains(availableBackends, "zfs") { defaultStorage = "zfs" } else if slices.Contains(availableBackends, "btrfs") { defaultStorage = "btrfs" } for { // Define the pool pool := api.StoragePoolsPost{} pool.Config = map[string]string{} if poolType == internalUtil.PoolTypeAny { pool.Name, err = c.global.asker.AskString(i18n.G("Name of the new storage pool")+" [default=default]: ", "default", nil) if err != nil { return err } } else { pool.Name = string(poolType) } _, _, err := d.GetStoragePool(pool.Name) if err == nil { if poolType == internalUtil.PoolTypeAny { fmt.Printf(i18n.G("The requested storage pool \"%s\" already exists. Please choose another name.")+"\n", pool.Name) continue } return fmt.Errorf(i18n.G("The %s storage pool already exists"), poolType) } // Add to the default profile if config.Profiles[0].Devices["root"] == nil { config.Profiles[0].Devices["root"] = map[string]string{ "type": "disk", "path": "/", "pool": pool.Name, } } // Storage backend if len(availableBackends) > 1 { defaultBackend := defaultStorage if poolType == internalUtil.PoolTypeRemote { if slices.Contains(availableBackends, "ceph") { defaultBackend = "ceph" } else { defaultBackend = availableBackends[0] // Default to first remote driver. } } pool.Driver, err = c.global.asker.AskChoice(fmt.Sprintf(i18n.G("Name of the storage backend to use (%s)")+" [default=%s]: ", strings.Join(availableBackends, ", "), defaultBackend), availableBackends, defaultBackend) if err != nil { return err } } else { pool.Driver = availableBackends[0] } // Optimization for dir if pool.Driver == "dir" { source, err := c.global.asker.AskString(fmt.Sprintf(i18n.G("Where should this storage pool store its data?")+" [default=%s]: ", internalUtil.VarPath("storage-pools", pool.Name)), "", validate.IsAny) if err != nil { return err } if source != "" { pool.Config["source"] = source } config.StoragePools = append(config.StoragePools, pool) break } // Optimization for btrfs on btrfs if pool.Driver == "btrfs" && backingFs == "btrfs" { btrfsSubvolume, err := c.global.asker.AskBool(fmt.Sprintf(i18n.G("Would you like to create a new btrfs subvolume under %s?")+" (yes/no) [default=yes]: ", internalUtil.VarPath("")), "yes") if err != nil { return err } if btrfsSubvolume { pool.Config["source"] = internalUtil.VarPath("storage-pools", pool.Name) config.StoragePools = append(config.StoragePools, pool) break } } // Optimization for zfs on zfs (when using Ubuntu's bpool/rpool) if pool.Driver == "zfs" && backingFs == "zfs" { poolName, _ := subprocess.RunCommand("zpool", "get", "-H", "-o", "value", "name", "rpool") if strings.TrimSpace(poolName) == "rpool" { zfsDataset, err := c.global.asker.AskBool(i18n.G("Would you like to create a new zfs dataset under rpool/incus?")+" (yes/no) [default=yes]: ", "yes") if err != nil { return err } if zfsDataset { pool.Config["source"] = "rpool/incus" config.StoragePools = append(config.StoragePools, pool) break } } } poolCreate, err := c.global.asker.AskBool(fmt.Sprintf(i18n.G("Create a new %s pool?")+" (yes/no) [default=yes]: ", strings.ToUpper(pool.Driver)), "yes") if err != nil { return err } if poolCreate { switch pool.Driver { case "ceph": // Ask for the name of the cluster pool.Config["ceph.cluster_name"], err = c.global.asker.AskString(i18n.G("Name of the existing CEPH cluster")+" [default=ceph]: ", "ceph", nil) if err != nil { return err } // Ask for the name of the osd pool pool.Config["ceph.osd.pool_name"], err = c.global.asker.AskString(i18n.G("Name of the OSD storage pool")+" [default=incus]: ", "incus", nil) if err != nil { return err } // Ask for the number of placement groups pool.Config["ceph.osd.pg_num"], err = c.global.asker.AskString(i18n.G("Number of placement groups")+" [default=32]: ", "32", nil) if err != nil { return err } case "cephfs": // Ask for the name of the cluster pool.Config["cephfs.cluster_name"], err = c.global.asker.AskString(i18n.G("Name of the existing CEPHfs cluster")+" [default=ceph]: ", "ceph", nil) if err != nil { return err } // Ask for the name of the cluster pool.Config["source"], err = c.global.asker.AskString(i18n.G("Name of the CEPHfs volume:")+" ", "", nil) if err != nil { return err } case "lvmcluster": // Ask for the volume group pool.Config["source"], err = c.global.asker.AskString(i18n.G("Name of the shared LVM volume group:")+" ", "", nil) if err != nil { return err } default: useEmptyBlockDev, err := c.global.asker.AskBool(i18n.G("Would you like to use an existing empty block device (e.g. a disk or partition)?")+" (yes/no) [default=no]: ", "no") if err != nil { return err } if useEmptyBlockDev { pool.Config["source"], err = c.global.asker.AskString(i18n.G("Path to the existing block device:")+" ", "", func(path string) error { if !linux.IsBlockdevPath(path) { return fmt.Errorf(i18n.G("%q is not a block device"), path) } return nil }) if err != nil { return err } } else { st := unix.Statfs_t{} err := unix.Statfs(internalUtil.VarPath(), &st) if err != nil { return fmt.Errorf(i18n.G("Couldn't statfs %s: %w"), internalUtil.VarPath(), err) } /* choose 5 GiB < x < 30GiB, where x is 20% of the disk size */ defaultSize := max(min(uint64(st.Frsize)*st.Blocks/(1024*1024*1024)/5, 30), 5) pool.Config["size"], err = c.global.asker.AskString( fmt.Sprintf(i18n.G("Size in GiB of the new loop device")+" (1GiB minimum) [default=%dGiB]: ", defaultSize), fmt.Sprintf("%dGiB", defaultSize), func(input string) error { input = strings.Split(input, "GiB")[0] result, err := strconv.ParseInt(input, 10, 64) if err != nil { return err } if result < 1 { return errors.New(i18n.G("Minimum size is 1GiB")) } return nil }, ) if err != nil { return err } if !strings.HasSuffix(pool.Config["size"], "GiB") { pool.Config["size"] = fmt.Sprintf("%sGiB", pool.Config["size"]) } } } } else { if pool.Driver == "ceph" { // ask for the name of the cluster pool.Config["ceph.cluster_name"], err = c.global.asker.AskString(i18n.G("Name of the existing CEPH cluster")+" [default=ceph]: ", "ceph", nil) if err != nil { return err } // ask for the name of the existing pool pool.Config["source"], err = c.global.asker.AskString(i18n.G("Name of the existing OSD storage pool")+" [default=incus]: ", "incus", nil) if err != nil { return err } pool.Config["ceph.osd.pool_name"] = pool.Config["source"] } else { question := fmt.Sprintf(i18n.G("Name of the existing %s pool or dataset:")+" ", strings.ToUpper(pool.Driver)) pool.Config["source"], err = c.global.asker.AskString(question, "", nil) if err != nil { return err } } } if pool.Driver == "lvm" { _, err := exec.LookPath("thin_check") if err != nil { fmt.Print("\n" + i18n.G(`The LVM thin provisioning tools couldn't be found. LVM can still be used without thin provisioning but this will disable over-provisioning, increase the space requirements and creation time of images, instances and snapshots. If you wish to use thin provisioning, abort now, install the tools from your Linux distribution and make sure that your user can see and run the "thin_check" command before running "init" again.`) + "\n\n") lvmContinueNoThin, err := c.global.asker.AskBool(i18n.G("Do you want to continue without thin provisioning?")+" (yes/no) [default=yes]: ", "yes") if err != nil { return err } if !lvmContinueNoThin { return errors.New(i18n.G("The LVM thin provisioning tools couldn't be found on the system")) } pool.Config["lvm.use_thinpool"] = "false" } } config.StoragePools = append(config.StoragePools, pool) break } return nil } func (c *cmdAdminInit) askDaemon(config *api.InitPreseed, server *api.Server) error { // Detect lack of uid/gid if linux.RunningInUserNS() { fmt.Print("\n" + i18n.G(`We detected that you are running inside an unprivileged container. This means that unless you manually configured your host otherwise, you will not have enough uids and gids to allocate to your containers. Your container's own allocation can be reused to avoid the problem. Doing so makes your nested containers slightly less safe as they could in theory attack their parent container and gain more privileges than they otherwise would.`) + "\n\n") shareParentAllocation, err := c.global.asker.AskBool(i18n.G("Would you like to have your containers share their parent's allocation?")+" (yes/no) [default=yes]: ", "yes") if err != nil { return err } if shareParentAllocation { config.Profiles[0].Config["security.privileged"] = "true" } } // Network listener if config.Cluster == nil { overNetwork, err := c.global.asker.AskBool(i18n.G("Would you like the server to be available over the network?")+" (yes/no) [default=no]: ", "no") if err != nil { return err } if overNetwork { isIPAddress := func(s string) error { if s != "all" && net.ParseIP(s) == nil { return fmt.Errorf(i18n.G("%q is not an IP address"), s) } return nil } netAddr, err := c.global.asker.AskString(i18n.G("Address to bind to (not including port)")+" [default=all]: ", "all", isIPAddress) if err != nil { return err } if netAddr == "all" { netAddr = "::" } if net.ParseIP(netAddr).To4() == nil { netAddr = fmt.Sprintf("[%s]", netAddr) } netPort, err := c.global.asker.AskInt(fmt.Sprintf(i18n.G("Port to bind to")+" [default=%d]: ", ports.HTTPSDefaultPort), 1, 65535, fmt.Sprintf("%d", ports.HTTPSDefaultPort), func(netPort int64) error { address := internalUtil.CanonicalNetworkAddressFromAddressAndPort(netAddr, int(netPort), ports.HTTPSDefaultPort) if err == nil { if server.Config["cluster.https_address"] == address || server.Config["core.https_address"] == address { // We already own the address, just move on. return nil } } listener, err := net.Listen("tcp", address) if err != nil { return fmt.Errorf(i18n.G("Can't bind address %q: %w"), address, err) } _ = listener.Close() return nil }) if err != nil { return err } config.Config["core.https_address"] = internalUtil.CanonicalNetworkAddressFromAddressAndPort(netAddr, int(netPort), ports.HTTPSDefaultPort) } } // Ask if the user wants images to be automatically refreshed imageStaleRefresh, err := c.global.asker.AskBool(i18n.G("Would you like stale cached images to be updated automatically?")+" (yes/no) [default=yes]: ", "yes") if err != nil { return err } if !imageStaleRefresh { config.Config["images.auto_update_interval"] = "0" } return nil } incus-7.0.0/cmd/incus/admin_init_preseed.go000066400000000000000000000016331517523235500207070ustar00rootroot00000000000000//go:build linux package main import ( "fmt" "io" "os" "go.yaml.in/yaml/v4" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" ) func (c *cmdAdminInit) runPreseed(p *u.Parsed) (*api.InitPreseed, error) { // Read the YAML var bytes []byte var err error if p.Skipped || p.String == "-" { bytes, err = io.ReadAll(os.Stdin) if err != nil { return nil, fmt.Errorf(i18n.G("Failed to read from stdin: %w"), err) } } else { bytes, err = os.ReadFile(p.String) if err != nil { return nil, fmt.Errorf(i18n.G("Failed to read from file: %w"), err) } } // Parse the YAML config := api.InitPreseed{} // Use strict checking to notify about unknown keys. err = yaml.Load(bytes, &config, yaml.WithKnownFields()) if err != nil { return nil, fmt.Errorf(i18n.G("Failed to parse the preseed: %w"), err) } return &config, nil } incus-7.0.0/cmd/incus/admin_os.go000066400000000000000000000020761517523235500166600ustar00rootroot00000000000000package main import ( "net/http" "net/url" "strings" "github.com/spf13/cobra" "github.com/lxc/incus-os/incus-osd/cli" ) // IncusOS management command. type cmdAdminOS struct { global *cmdGlobal } func (c *cmdAdminOS) command() *cobra.Command { args := &cli.Args{ SupportsTarget: true, SupportsRemote: true, DefaultListFormat: c.global.defaultListFormat(), DoHTTP: func(remoteName string, req *http.Request) (*http.Response, error) { if remoteName != "" && !strings.HasSuffix(remoteName, ":") { remoteName += ":" } // Parse the remote. remote, _, err := c.global.conf.ParseRemote(remoteName) if err != nil { return nil, err } // Attempt to connect. d, err := c.global.conf.GetInstanceServer(remote) if err != nil { return nil, err } // Get the URL prefix. httpInfo, err := d.GetConnectionInfo() if err != nil { return nil, err } req.URL, err = url.Parse(httpInfo.URL + req.URL.String()) if err != nil { return nil, err } return d.DoHTTP(req) }, } return cli.NewCommand(args) } incus-7.0.0/cmd/incus/admin_other.go000066400000000000000000000014331517523235500173540ustar00rootroot00000000000000//go:build !linux package main import ( "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" "github.com/lxc/incus/v7/internal/i18n" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdAdmin struct { global *cmdGlobal } func (c *cmdAdmin) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("admin") cmd.Short = i18n.G("Manage incus daemon") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Manage incus daemon`)) // os adminOSCmd := cmdAdminOS{global: c.global} cmd.AddCommand(adminOSCmd.command()) // recover sub-command adminRecoverCmd := cmdAdminRecover{global: c.global} cmd.AddCommand(adminRecoverCmd.command()) // sql sub-command sqlCmd := cmdAdminSQL{global: c.global} cmd.AddCommand(sqlCmd.command()) return cmd } incus-7.0.0/cmd/incus/admin_recover.go000066400000000000000000000175451517523235500177130ustar00rootroot00000000000000package main import ( "errors" "fmt" "strings" "github.com/spf13/cobra" "golang.org/x/text/cases" "golang.org/x/text/language" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/internal/recover" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/validate" ) type cmdAdminRecover struct { global *cmdGlobal } var cmdAdminRecoverUsage = u.Usage{u.RemoteColonOpt} func (c *cmdAdminRecover) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("recover", cmdAdminRecoverUsage...) cmd.Short = i18n.G("Recover missing instances and volumes from existing and unknown storage pools") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Recover missing instances and volumes from existing and unknown storage pools This command is mostly used for disaster recovery. It will ask you about unknown storage pools and attempt to access them, along with existing storage pools, and identify any missing instances and volumes that exist on the pools but are not in the database. It will then offer to recreate these database records.`)) cmd.RunE = c.run return cmd } func (c *cmdAdminRecover) run(cmd *cobra.Command, args []string) error { parsed, err := cmdAdminRecoverUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer server, _, err := d.GetServer() if err != nil { return err } isClustered := d.IsClustered() // Get list of existing storage pools to scan. existingPools, err := d.GetStoragePools() if err != nil { return fmt.Errorf(i18n.G("Failed getting existing storage pools: %w"), err) } fmt.Println(i18n.G("This server currently has the following storage pools:")) for _, existingPool := range existingPools { fmt.Printf(" - "+i18n.G("%s (backend=%q, source=%q)")+"\n", existingPool.Name, existingPool.Driver, existingPool.Config["source"]) } unknownPools := make([]api.StoragePoolsPost, 0, len(existingPools)) // Build up a list of unknown pools to scan. // We don't offer this option if the server is clustered because we don't allow creating storage pools on // an individual server when clustered. if !isClustered { var supportedDriverNames []string for { addUnknownPool, err := c.global.asker.AskBool(i18n.G("Would you like to recover another storage pool?")+" (yes/no) [default=no]: ", "no") if err != nil { return err } if !addUnknownPool { break } // Get available storage drivers if not done already. if supportedDriverNames == nil { for _, supportedDriver := range server.Environment.StorageSupportedDrivers { supportedDriverNames = append(supportedDriverNames, supportedDriver.Name) } } unknownPool := api.StoragePoolsPost{ StoragePoolPut: api.StoragePoolPut{ Config: make(map[string]string), }, } unknownPool.Name, err = c.global.asker.AskString(i18n.G("Name of the storage pool:")+" ", "", validate.Required(func(value string) error { if value == "" { return errors.New(i18n.G("Pool name cannot be empty")) } for _, p := range unknownPools { if value == p.Name { return fmt.Errorf(i18n.G("Storage pool %q is already on recover list"), value) } } return nil })) if err != nil { return err } unknownPool.Driver, err = c.global.asker.AskString(fmt.Sprintf(i18n.G("Name of the storage backend (%s):")+" ", strings.Join(supportedDriverNames, ", ")), "", validate.IsOneOf(supportedDriverNames...)) if err != nil { return err } unknownPool.Config["source"], err = c.global.asker.AskString(i18n.G("Source of the storage pool (block device, volume group, dataset, path, ... as applicable):")+" ", "", validate.IsNotEmpty) if err != nil { return err } for { var configKey, configValue string _, _ = c.global.asker.AskString(i18n.G("Additional storage pool configuration property (KEY=VALUE, empty when done):")+" ", "", validate.Optional(func(value string) error { configParts := strings.SplitN(value, "=", 2) if len(configParts) < 2 { return errors.New(i18n.G("Config option should be in the format KEY=VALUE")) } configKey = configParts[0] configValue = configParts[1] return nil })) if configKey == "" { break } unknownPool.Config[configKey] = configValue } unknownPools = append(unknownPools, unknownPool) } } fmt.Println(i18n.G("The recovery process will be scanning the following storage pools:")) for _, p := range existingPools { fmt.Printf(" - "+i18n.G("EXISTING: %q (backend=%q, source=%q)")+"\n", p.Name, p.Driver, p.Config["source"]) } for _, p := range unknownPools { fmt.Printf(" - "+i18n.G("NEW: %q (backend=%q, source=%q)")+"\n", p.Name, p.Driver, p.Config["source"]) } proceed, err := c.global.asker.AskBool(i18n.G("Would you like to continue with scanning for lost volumes?")+" (yes/no) [default=yes]: ", "yes") if err != nil { return err } if !proceed { return nil } fmt.Println(i18n.G("Scanning for unknown volumes...")) // Send /internal/recover/validate request to the daemon. reqValidate := recover.ValidatePost{ Pools: make([]api.StoragePoolsPost, 0, len(existingPools)+len(unknownPools)), } // Add existing pools to request. for _, p := range existingPools { reqValidate.Pools = append(reqValidate.Pools, api.StoragePoolsPost{ Name: p.Name, // Only send existing pool name, the rest will be looked up on server. }) } // Add unknown pools to request. reqValidate.Pools = append(reqValidate.Pools, unknownPools...) for { resp, _, err := d.RawQuery("POST", "/internal/recover/validate", reqValidate, "") if err != nil { return fmt.Errorf(i18n.G("Failed validation request: %w"), err) } var res recover.ValidateResult err = resp.MetadataAsStruct(&res) if err != nil { return fmt.Errorf(i18n.G("Failed parsing validation response: %w"), err) } if len(unknownPools) > 0 { fmt.Println(i18n.G("The following unknown storage pools have been found:")) for _, unknownPool := range unknownPools { fmt.Printf(" - "+i18n.G("Storage pool %q of type %q")+"\n", unknownPool.Name, unknownPool.Driver) } } if len(res.UnknownVolumes) > 0 { fmt.Println(i18n.G("The following unknown volumes have been found:")) for _, unknownVol := range res.UnknownVolumes { fmt.Printf(" - "+i18n.G("%s %q on pool %q in project %q (includes %d snapshots)")+"\n", cases.Title(language.English).String(unknownVol.Type), unknownVol.Name, unknownVol.Pool, unknownVol.Project, unknownVol.SnapshotCount) } } if len(res.DependencyErrors) == 0 { if len(unknownPools) == 0 && len(res.UnknownVolumes) == 0 { fmt.Println(i18n.G("No unknown storage pools or volumes found. Nothing to do.")) return nil } break // Dependencies met. } fmt.Println(i18n.G("You are currently missing the following:")) for _, depErr := range res.DependencyErrors { fmt.Printf(" - %s\n", depErr) } _, _ = c.global.asker.AskString(i18n.G("Please create those missing entries and then hit ENTER:")+" ", "", validate.Optional()) } proceed, err = c.global.asker.AskBool(i18n.G("Would you like those to be recovered?")+" (yes/no) [default=no]: ", "no") if err != nil { return err } if !proceed { return nil } fmt.Println(i18n.G("Starting recovery...")) // Send /internal/recover/import request to the daemon. // Don't lint next line with staticcheck. It says we should convert reqValidate directly to an RecoverImportPost // because their types are identical. This is less clear and will not work if either type changes in the future. reqImport := recover.ImportPost{ //nolint:staticcheck Pools: reqValidate.Pools, } _, _, err = d.RawQuery("POST", "/internal/recover/import", reqImport, "") if err != nil { return fmt.Errorf(i18n.G("Failed import request: %w"), err) } return nil } incus-7.0.0/cmd/incus/admin_shutdown.go000066400000000000000000000050141517523235500201050ustar00rootroot00000000000000//go:build linux package main import ( "errors" "fmt" "net/http" "net/url" "strconv" "time" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdAdminShutdown struct { global *cmdGlobal flagForce bool flagTimeout int } var cmdAdminShutdownUsage = u.Usage{} func (c *cmdAdminShutdown) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("shutdown", cmdAdminShutdownUsage...) cmd.Short = i18n.G("Tell the daemon to shutdown all instances and exit") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Tell the daemon to shutdown all instances and exit This will tell the daemon to start a clean shutdown of all instances, followed by having itself shutdown and exit. This can take quite a while as instances can take a long time to shutdown, especially if a non-standard timeout was configured for them.`)) cmd.RunE = c.run cli.AddIntFlag(cmd.Flags(), &c.flagTimeout, "timeout|t", 0, "Number of seconds to wait before giving up") cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", "Force shutdown instead of waiting for running operations to finish") return cmd } func (c *cmdAdminShutdown) run(cmd *cobra.Command, args []string) error { _, err := cmdAdminShutdownUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } connArgs := &incus.ConnectionArgs{ SkipGetServer: true, } d, err := incus.ConnectIncusUnix("", connArgs) if err != nil { return err } v := url.Values{} v.Set("force", strconv.FormatBool(c.flagForce)) chResult := make(chan error, 1) go func() { defer close(chResult) httpClient, err := d.GetHTTPClient() if err != nil { chResult <- err return } // Request shutdown, this shouldn't return until daemon has stopped so use a large request timeout. httpTransport, ok := httpClient.Transport.(*http.Transport) if !ok { chResult <- errors.New("Bad http transport") return } httpTransport.ResponseHeaderTimeout = 3600 * time.Second _, _, err = d.RawQuery("PUT", fmt.Sprintf("/internal/shutdown?%s", v.Encode()), nil, "") if err != nil { chResult <- err return } }() if c.flagTimeout > 0 { select { case err = <-chResult: return err case <-time.After(time.Second * time.Duration(c.flagTimeout)): return fmt.Errorf(i18n.G("Daemon still running after %ds timeout"), c.flagTimeout) } } return <-chResult } incus-7.0.0/cmd/incus/admin_sql.go000066400000000000000000000103071517523235500170320ustar00rootroot00000000000000package main import ( "encoding/json" "fmt" "io" "os" "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" internalSQL "github.com/lxc/incus/v7/internal/sql" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdAdminSQL struct { global *cmdGlobal flagFormat string } var cmdAdminSQLUsage = u.Usage{u.EitherVerbatim("local", "global").Remote(), u.Query} func (c *cmdAdminSQL) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("sql", cmdAdminSQLUsage...) cmd.Short = i18n.G("Execute a SQL query against the local or global database") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Execute a SQL query against the local or global database The local database is specific to the cluster member you target the command to, and contains member-specific data (such as the member network address). The global database is common to all members in the cluster, and contains cluster-specific data (such as profiles, containers, etc). Non-clustered servers still have both local and global databases. If is the special value "-", then the query is read from standard input. If is the special value ".dump", the command returns a SQL text dump of the given database. If is the special value ".schema", the command returns the SQL text schema of the given database. If is the special value ".tables", the command returns the SQL text tables of the given database. This internal command is mostly useful for debugging and disaster recovery. The development team will occasionally provide hotfixes to users as a set of database queries to fix some data inconsistency.`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } return cmd } func (c *cmdAdminSQL) run(cmd *cobra.Command, args []string) error { parsed, err := cmdAdminSQLUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer database := parsed[0].RemoteObject.String query := parsed[1].String if query == "-" { // Read from stdin bytes, err := io.ReadAll(os.Stdin) if err != nil { return fmt.Errorf(i18n.G("Failed to read from stdin: %w"), err) } query = string(bytes) } if query == ".dump" || query == ".schema" || query == ".tables" { url := fmt.Sprintf("/internal/sql?database=%s", database) switch query { case ".schema": url += "&dump=1" case ".tables": url += "&dump=2" } response, _, err := d.RawQuery("GET", url, nil, "") if err != nil { return fmt.Errorf(i18n.G("Failed to request dump: %w"), err) } dump := internalSQL.SQLDump{} err = json.Unmarshal(response.Metadata, &dump) if err != nil { return fmt.Errorf(i18n.G("Failed to parse dump response: %w"), err) } fmt.Print(dump.Text) return nil } data := internalSQL.SQLQuery{ Database: database, Query: query, } response, _, err := d.RawQuery("POST", "/internal/sql", data, "") if err != nil { return err } batch := internalSQL.SQLBatch{} err = json.Unmarshal(response.Metadata, &batch) if err != nil { return err } for i, result := range batch.Results { if len(batch.Results) > 1 { fmt.Printf(i18n.G("=> Query %d:")+"\n\n", i) } if result.Type == "select" { err := c.sqlPrintSelectResult(result) if err != nil { return err } } else { fmt.Printf(i18n.G("Rows affected: %d")+"\n", result.RowsAffected) } if len(batch.Results) > 1 { fmt.Println("") } } return nil } func (c *cmdAdminSQL) sqlPrintSelectResult(result internalSQL.SQLResult) error { data := [][]string{} for _, row := range result.Rows { rowData := []string{} for _, col := range row { rowData = append(rowData, fmt.Sprintf("%v", col)) } data = append(data, rowData) } return cli.RenderTable(os.Stdout, c.flagFormat, result.Columns, data, result) } incus-7.0.0/cmd/incus/admin_waitready.go000066400000000000000000000051461517523235500202310ustar00rootroot00000000000000//go:build linux package main import ( "fmt" "time" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/logger" ) type cmdAdminWaitready struct { global *cmdGlobal flagTimeout int } var cmdAdminWaitreadyUsage = u.Usage{} func (c *cmdAdminWaitready) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("waitready", cmdAdminWaitreadyUsage...) cmd.Short = i18n.G("Wait for the daemon to be ready to process requests") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Wait for the daemon to be ready to process requests This command will block until the daemon is reachable over its REST API and is done with early start tasks like re-starting previously started containers.`)) cmd.RunE = c.run cli.AddIntFlag(cmd.Flags(), &c.flagTimeout, "timeout|t", 0, "Number of seconds to wait before giving up") return cmd } func (c *cmdAdminWaitready) run(cmd *cobra.Command, args []string) error { _, err := cmdAdminWaitreadyUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } finger := make(chan error, 1) var errLast error go func() { for i := 0; ; i++ { // Start logging only after the 10'th attempt (about 5 // seconds). Then after the 30'th attempt (about 15 // seconds), log only only one attempt every 10 // attempts (about 5 seconds), to avoid being too // verbose. doLog := false if i > 10 { doLog = i < 30 || ((i % 10) == 0) } if doLog { logger.Debugf(i18n.G("Connecting to the daemon (attempt %d)"), i) } d, err := incus.ConnectIncusUnix("", nil) if err != nil { errLast = err if doLog { logger.Debugf(i18n.G("Failed connecting to the daemon (attempt %d): %v"), i, err) } time.Sleep(500 * time.Millisecond) continue } if doLog { logger.Debugf(i18n.G("Checking if the daemon is ready (attempt %d)"), i) } _, _, err = d.RawQuery("GET", "/internal/ready", nil, "") if err != nil { errLast = err if doLog { logger.Debugf(i18n.G("Failed to check if the daemon is ready (attempt %d): %v"), i, err) } time.Sleep(500 * time.Millisecond) continue } finger <- nil return } }() if c.flagTimeout > 0 { select { case <-finger: case <-time.After(time.Second * time.Duration(c.flagTimeout)): return fmt.Errorf(i18n.G("Daemon still not running after %ds timeout (%v)"), c.flagTimeout, errLast) } } else { <-finger } return nil } incus-7.0.0/cmd/incus/alias.go000066400000000000000000000137041517523235500161600ustar00rootroot00000000000000package main import ( "fmt" "os" "sort" "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdAlias struct { global *cmdGlobal } func (c *cmdAlias) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("alias") cmd.Short = i18n.G("Manage command aliases") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage command aliases`)) cmd.Hidden = true // Add aliasAddCmd := cmdAliasAdd{global: c.global, alias: c} cmd.AddCommand(aliasAddCmd.command()) // List aliasListCmd := cmdAliasList{global: c.global, alias: c} cmd.AddCommand(aliasListCmd.command()) // Rename aliasRenameCmd := cmdAliasRename{global: c.global, alias: c} cmd.AddCommand(aliasRenameCmd.command()) // Remove aliasRemoveCmd := cmdAliasRemove{global: c.global, alias: c} cmd.AddCommand(aliasRemoveCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Add. type cmdAliasAdd struct { global *cmdGlobal alias *cmdAlias } var cmdAliasAddUsage = u.Usage{u.NewName(u.Alias), u.Target(u.Placeholder(i18n.G("command")))} func (c *cmdAliasAdd) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("add", cmdAliasAddUsage...) cmd.Aliases = []string{"create"} cmd.Short = i18n.G("Add new aliases") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Add new aliases`)) cmd.Example = cli.FormatSection("", i18n.G( `incus alias add list "list -c ns46S" Overwrite the "list" command to pass -c ns46S.`)) cmd.RunE = c.run return cmd } func (c *cmdAliasAdd) run(cmd *cobra.Command, args []string) error { conf := c.global.conf parsed, err := cmdAliasAddUsage.Parse(conf, cmd, args) if err != nil { return err } alias := parsed[0].String // Look for an existing alias _, ok := conf.Aliases[alias] if ok { return fmt.Errorf(i18n.G("Alias %s already exists"), alias) } // Add the new alias conf.Aliases[alias] = parsed[1].String // Save the config return conf.SaveConfig(c.global.confPath) } // List. type cmdAliasList struct { global *cmdGlobal alias *cmdAlias flagFormat string } var cmdAliasListUsage = u.Usage{} func (c *cmdAliasList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdAliasListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List aliases") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`List aliases`)) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run return cmd } func (c *cmdAliasList) run(cmd *cobra.Command, args []string) error { conf := c.global.conf // Quick checks. _, err := cmdAliasListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } // List the aliases data := [][]string{} for k, v := range conf.Aliases { data = append(data, []string{k, v}) } // Apply default entries. for k, v := range defaultAliases { _, ok := conf.Aliases[k] if !ok { data = append(data, []string{k, v}) } } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{ i18n.G("ALIAS"), i18n.G("TARGET"), } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, conf.Aliases) } // Rename. type cmdAliasRename struct { global *cmdGlobal alias *cmdAlias } var cmdAliasRenameUsage = u.Usage{u.Alias, u.NewName(u.Alias)} func (c *cmdAliasRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdAliasRenameUsage...) cmd.Aliases = []string{"mv"} cmd.Short = i18n.G("Rename aliases") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Rename aliases`)) cmd.Example = cli.FormatSection("", i18n.G( `incus alias rename list my-list Rename existing alias "list" to "my-list".`)) cmd.RunE = c.run return cmd } func (c *cmdAliasRename) run(cmd *cobra.Command, args []string) error { conf := c.global.conf parsed, err := cmdAliasRenameUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } alias := parsed[0].String newAlias := parsed[1].String // Check for the existing alias target, ok := conf.Aliases[alias] if !ok { return fmt.Errorf(i18n.G("Alias %s doesn't exist"), alias) } // Check for the new alias _, ok = conf.Aliases[newAlias] if ok { return fmt.Errorf(i18n.G("Alias %s already exists"), newAlias) } // Rename the alias conf.Aliases[newAlias] = target delete(conf.Aliases, alias) // Save the config return conf.SaveConfig(c.global.confPath) } // Remove. type cmdAliasRemove struct { global *cmdGlobal alias *cmdAlias } var cmdAliasRemoveUsage = u.Usage{u.Alias} func (c *cmdAliasRemove) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remove", cmdAliasRemoveUsage...) cmd.Aliases = []string{"delete", "rm"} cmd.Short = i18n.G("Remove aliases") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Remove aliases`)) cmd.Example = cli.FormatSection("", i18n.G( `incus alias remove my-list Remove the "my-list" alias.`)) cmd.RunE = c.run return cmd } func (c *cmdAliasRemove) run(cmd *cobra.Command, args []string) error { conf := c.global.conf // Quick checks. parsed, err := cmdAliasRemoveUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } alias := parsed[0].String // Look for the alias _, ok := conf.Aliases[alias] if !ok { return fmt.Errorf(i18n.G("Alias %s doesn't exist"), alias) } // Delete the alias delete(conf.Aliases, alias) // Save the config return conf.SaveConfig(c.global.confPath) } incus-7.0.0/cmd/incus/cluster.go000066400000000000000000001515461517523235500165570ustar00rootroot00000000000000package main import ( "bufio" "encoding/pem" "errors" "fmt" "io" "maps" "net" "net/http" "os" "reflect" "slices" "sort" "strings" "github.com/spf13/cobra" yaml "go.yaml.in/yaml/v4" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/internal/ports" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/ask" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) type clusterColumn struct { Name string Data func(api.ClusterMember) string } type cmdCluster struct { global *cmdGlobal } func (c *cmdCluster) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("cluster") cmd.Short = i18n.G("Manage cluster members") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage cluster members`)) // List clusterListCmd := cmdClusterList{global: c.global, cluster: c} cmd.AddCommand(clusterListCmd.command()) // Rename clusterRenameCmd := cmdClusterRename{global: c.global, cluster: c} cmd.AddCommand(clusterRenameCmd.command()) // Remove clusterRemoveCmd := cmdClusterRemove{global: c.global, cluster: c} cmd.AddCommand(clusterRemoveCmd.command()) // Show clusterShowCmd := cmdClusterShow{global: c.global, cluster: c} cmd.AddCommand(clusterShowCmd.command()) // Info clusterInfoCmd := cmdClusterInfo{global: c.global, cluster: c} cmd.AddCommand(clusterInfoCmd.command()) // Get clusterGetCmd := cmdClusterGet{global: c.global, cluster: c} cmd.AddCommand(clusterGetCmd.command()) // Set clusterSetCmd := cmdClusterSet{global: c.global, cluster: c} cmd.AddCommand(clusterSetCmd.command()) // Unset clusterUnsetCmd := cmdClusterUnset{global: c.global, cluster: c, clusterSet: &clusterSetCmd} cmd.AddCommand(clusterUnsetCmd.command()) // Enable clusterEnableCmd := cmdClusterEnable{global: c.global, cluster: c} cmd.AddCommand(clusterEnableCmd.command()) // Edit clusterEditCmd := cmdClusterEdit{global: c.global, cluster: c} cmd.AddCommand(clusterEditCmd.command()) // Join cmdClusterJoin := cmdClusterJoin{global: c.global, cluster: c} cmd.AddCommand(cmdClusterJoin.command()) // Add token cmdClusterAdd := cmdClusterAdd{global: c.global, cluster: c} cmd.AddCommand(cmdClusterAdd.command()) // List tokens cmdClusterListTokens := cmdClusterListTokens{global: c.global, cluster: c} cmd.AddCommand(cmdClusterListTokens.command()) // Revoke tokens cmdClusterRevokeToken := cmdClusterRevokeToken{global: c.global, cluster: c} cmd.AddCommand(cmdClusterRevokeToken.command()) // Update certificate cmdClusterUpdateCertificate := cmdClusterUpdateCertificate{global: c.global, cluster: c} cmd.AddCommand(cmdClusterUpdateCertificate.command()) // Evacuate cluster member cmdClusterEvacuate := cmdClusterEvacuate{global: c.global, cluster: c} cmd.AddCommand(cmdClusterEvacuate.command()) // Restore cluster member cmdClusterRestore := cmdClusterRestore{global: c.global, cluster: c} cmd.AddCommand(cmdClusterRestore.command()) clusterGroupCmd := cmdClusterGroup{global: c.global, cluster: c} cmd.AddCommand(clusterGroupCmd.command()) clusterRoleCmd := cmdClusterRole{global: c.global, cluster: c} cmd.AddCommand(clusterRoleCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // List. type cmdClusterList struct { global *cmdGlobal cluster *cmdCluster flagColumns string flagFormat string flagAllProjects bool } var cmdClusterListUsage = u.Usage{u.RemoteColonOpt, u.Filter.List(0)} func (c *cmdClusterList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdClusterListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List all the cluster members") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List all the cluster members The -c option takes a (optionally comma-separated) list of arguments that control which cluster members attributes to output when displaying in table or csv format. Default column layout is: nurafdsm Column shorthand chars: n - Server name u - URL r - Roles a - Architecture f - Failure Domain d - Description s - Status m - Message`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultClusterColumns, "", i18n.G("Columns")) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("Display clusters from all projects")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } const defaultClusterColumns = "nurafdsm" func (c *cmdClusterList) parseColumns() ([]clusterColumn, error) { columnsShorthandMap := map[rune]clusterColumn{ 'n': {i18n.G("NAME"), c.serverColumnData}, 'u': {i18n.G("URL"), c.urlColumnData}, 'r': {i18n.G("ROLES"), c.rolesColumnData}, 'a': {i18n.G("ARCHITECTURE"), c.architectureColumnData}, 'f': {i18n.G("FAILURE DOMAIN"), c.failureDomainColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 's': {i18n.G("STATUS"), c.statusColumnData}, 'm': {i18n.G("MESSAGE"), c.messageColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []clusterColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdClusterList) serverColumnData(cluster api.ClusterMember) string { return cluster.ServerName } func (c *cmdClusterList) urlColumnData(cluster api.ClusterMember) string { return cluster.URL } func (c *cmdClusterList) rolesColumnData(cluster api.ClusterMember) string { roles := cluster.Roles rolesDelimiter := "\n" if c.flagFormat == "csv" { rolesDelimiter = "," } return strings.Join(roles, rolesDelimiter) } func (c *cmdClusterList) architectureColumnData(cluster api.ClusterMember) string { return cluster.Architecture } func (c *cmdClusterList) failureDomainColumnData(cluster api.ClusterMember) string { return cluster.FailureDomain } func (c *cmdClusterList) descriptionColumnData(cluster api.ClusterMember) string { return cluster.Description } func (c *cmdClusterList) statusColumnData(cluster api.ClusterMember) string { return strings.ToUpper(cluster.Status) } func (c *cmdClusterList) messageColumnData(cluster api.ClusterMember) string { return cluster.Message } func (c *cmdClusterList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer if c.global.flagProject != "" && c.flagAllProjects { return errors.New(i18n.G("Can't specify --project with --all-projects")) } filters := prepareClusterMemberServerFilters(parsed[1].StringList, api.ClusterMember{}) // Check if clustered cluster, _, err := d.GetCluster() if err != nil { return err } if !cluster.Enabled { return errors.New(i18n.G("Server isn't part of a cluster")) } // Get the cluster members members, err := d.GetClusterMembersWithFilter(filters) if err != nil { return err } // Process the columns columns, err := c.parseColumns() if err != nil { return err } // Render the table data := [][]string{} for _, member := range members { line := []string{} for _, column := range columns { line = append(line, column.Data(member)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, members) } // Show. type cmdClusterShow struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterShowUsage = u.Usage{u.Member.Remote()} func (c *cmdClusterShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdClusterShowUsage...) cmd.Short = i18n.G("Show details of a cluster member") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show details of a cluster member`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer memberName := parsed[0].RemoteObject.String // Get the member information member, _, err := d.GetClusterMember(memberName) if err != nil { return err } // Render as YAML data, err := yaml.Dump(&member, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Info. type cmdClusterInfo struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterInfoUsage = u.Usage{u.Member.Remote()} func (c *cmdClusterInfo) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("info", cmdClusterInfoUsage...) cmd.Short = i18n.G("Show useful information about a cluster member") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Show useful information about a cluster member`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterInfo) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterInfoUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer memberName := parsed[0].RemoteObject.String // Get the member state information. member, _, err := d.GetClusterMemberState(memberName) if err != nil { return err } // Render as YAML. data, err := yaml.Dump(&member, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Get. type cmdClusterGet struct { global *cmdGlobal cluster *cmdCluster flagIsProperty bool } var cmdClusterGetUsage = u.Usage{u.Member.Remote(), u.Key} func (c *cmdClusterGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdClusterGetUsage...) cmd.Short = i18n.G("Get values for cluster member configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, cmd.Short) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a cluster property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } if len(args) == 1 { return c.global.cmpClusterMemberConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer memberName := parsed[0].RemoteObject.String key := parsed[1].String // Get the member information member, _, err := d.GetClusterMember(memberName) if err != nil { return err } if c.flagIsProperty { w := member.Writable() res, err := getFieldByJSONTag(&w, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the cluster member %q: %v"), key, memberName, err) } fmt.Printf("%v\n", res) return nil } value, ok := member.Config[key] if !ok { return fmt.Errorf(i18n.G("The key %q does not exist on cluster member %q"), key, memberName) } fmt.Printf("%s\n", value) return nil } // Set. type cmdClusterSet struct { global *cmdGlobal cluster *cmdCluster flagIsProperty bool } var cmdClusterSetUsage = u.Usage{u.Member.Remote(), u.LegacyKV.List(1)} func (c *cmdClusterSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdClusterSetUsage...) cmd.Short = i18n.G("Set a cluster member's configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, cmd.Short) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a cluster property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdClusterSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer memberName := parsed[0].RemoteObject.String // Get the member information member, _, err := d.GetClusterMember(memberName) if err != nil { return err } // Get the new config keys keys, err := kvToMap(parsed[1]) if err != nil { return err } writable := member.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { maps.Copy(writable.Config, keys) } return d.UpdateClusterMember(memberName, writable, "") } func (c *cmdClusterSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Unset. type cmdClusterUnset struct { global *cmdGlobal cluster *cmdCluster clusterSet *cmdClusterSet flagIsProperty bool } var cmdClusterUnsetUsage = u.Usage{u.Member.Remote(), u.Key} func (c *cmdClusterUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdClusterUnsetUsage...) cmd.Short = i18n.G("Unset a cluster member's configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, cmd.Short) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a cluster property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } if len(args) == 1 { return c.global.cmpClusterMemberConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.clusterSet.flagIsProperty = c.flagIsProperty return unsetKey(c.clusterSet, cmd, parsed) } // Rename. type cmdClusterRename struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterRenameUsage = u.Usage{u.Member.Remote(), u.NewName(u.Member)} func (c *cmdClusterRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdClusterRenameUsage...) cmd.Aliases = []string{"mv"} cmd.Short = i18n.G("Rename a cluster member") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Rename a cluster member`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterRename) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterRenameUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer memberName := parsed[0].RemoteObject.String newMemberName := parsed[1].String // Perform the rename err = d.RenameClusterMember(memberName, api.ClusterMemberPost{ServerName: newMemberName}) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Member %s renamed to %s")+"\n", memberName, newMemberName) } return nil } // Remove. type cmdClusterRemove struct { global *cmdGlobal cluster *cmdCluster flagForce bool flagNonInteractive bool } var cmdClusterRemoveUsage = u.Usage{u.Member.Remote()} func (c *cmdClusterRemove) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remove", cmdClusterRemoveUsage...) cmd.Aliases = []string{"delete", "rm"} cmd.Short = i18n.G("Remove a member from the cluster") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Remove a member from the cluster`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G("Force removing a member, even if degraded")) cli.AddBoolFlag(cmd.Flags(), &c.flagNonInteractive, "yes", i18n.G("Don't require user confirmation for using --force")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterRemove) promptConfirmation(p *u.Parsed) error { reader := bufio.NewReader(os.Stdin) fmt.Printf(i18n.G(`Forcefully removing a server from the cluster should only be done as a last resort. The removed server will not be functional after this action and will require a full reset, losing any remaining instance, image or storage volume that the server may have held. When possible, a graceful removal should be preferred, this will require you to move any affected instance, image or storage volume to another server prior to the server being cleanly removed from the cluster. The --force flag should only be used if the server has died, been reinstalled or is otherwise never expected to come back up. Are you really sure you want to force removing %s? (yes/no): `), formatRemote(c.global.conf, p)) input, _ := reader.ReadString('\n') input = strings.TrimSuffix(input, "\n") if !slices.Contains([]string{i18n.G("yes")}, strings.ToLower(input)) { return errors.New(i18n.G("User aborted delete operation")) } return nil } func (c *cmdClusterRemove) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterRemoveUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer memberName := parsed[0].RemoteObject.String // Prompt for confirmation if --force is used. if !c.flagNonInteractive && c.flagForce { err := c.promptConfirmation(parsed[0]) if err != nil { return err } } // Delete the cluster member err = d.DeleteClusterMember(memberName, c.flagForce) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Member %s removed")+"\n", memberName) } return nil } // Enable. type cmdClusterEnable struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterEnableUsage = u.Usage{u.RemoteColonOpt, u.NewName(u.Member)} func (c *cmdClusterEnable) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("enable", cmdClusterEnableUsage...) cmd.Short = i18n.G("Enable clustering on a single non-clustered server") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Enable clustering on a single non-clustered server This command turns a non-clustered server into the first member of a new cluster, which will have the given name. It's required that the server is already available on the network. You can check that by running 'incus config get core.https_address', and possibly set a value for the address if not yet set.`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterEnable) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterEnableUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer memberName := parsed[1].String // Check if the server is available on the network. server, _, err := d.GetServer() if err != nil { return fmt.Errorf(i18n.G("Failed to retrieve current server configuration: %w"), err) } if server.Config["core.https_address"] == "" && server.Config["cluster.https_address"] == "" { return errors.New(i18n.G("This server is not available on the network")) } // Check if already enabled currentCluster, etag, err := d.GetCluster() if err != nil { return fmt.Errorf(i18n.G("Failed to retrieve current cluster config: %w"), err) } if currentCluster.Enabled { return errors.New(i18n.G("This server is already clustered")) } // Enable clustering. req := api.ClusterPut{} req.ServerName = memberName req.Enabled = true op, err := d.UpdateCluster(req, etag) if err != nil { return fmt.Errorf(i18n.G("Failed to configure cluster: %w"), err) } err = op.Wait() if err != nil { return fmt.Errorf(i18n.G("Failed to configure cluster: %w"), err) } fmt.Println(i18n.G("Clustering enabled")) return nil } // Edit. type cmdClusterEdit struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterEditUsage = u.Usage{u.Member.Remote()} func (c *cmdClusterEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdClusterEditUsage...) cmd.Short = i18n.G("Edit cluster member configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Edit cluster member configurations as YAML`)) cmd.Example = cli.FormatSection("", i18n.G( `incus cluster edit < member.yaml Update a cluster member using the content of member.yaml`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the cluster member. ### Any line starting with a '# will be ignored.`) } func (c *cmdClusterEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer memberName := parsed[0].RemoteObject.String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } newdata := api.ClusterMemberPut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateClusterMember(memberName, newdata, "") } // Extract the current value member, etag, err := d.GetClusterMember(memberName) if err != nil { return err } memberWritable := member.Writable() data, err := yaml.Dump(&memberWritable, yaml.V2) if err != nil { return err } // Spawn the editor content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor newdata := api.ClusterMemberPut{} err = yaml.Load(content, &newdata) if err == nil { err = d.UpdateClusterMember(memberName, newdata, etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Join. type cmdClusterJoin struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterJoinUsage = u.Usage{u.MakeRemote(u.Placeholder(i18n.G("cluster")), false), u.MakeRemote(u.Member, true)} func (c *cmdClusterJoin) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("join", cmdClusterJoinUsage...) cmd.Short = i18n.G("Join an existing server to a cluster") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Join an existing server to a cluster`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterJoin) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterJoinUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } cluster := parsed[0].RemoteServer member := parsed[1].RemoteServer config := newInitPressed() // Validate servers. if !cluster.IsClustered() { return errors.New(i18n.G("Target isn't a cluster")) } if member.IsClustered() { return errors.New(i18n.G("Target server is already clustered")) } // Ask the interactive questions. err = askClustering(c.global.asker, config, cluster, member, true) if err != nil { return err } err = fillClusterConfig(config) if err != nil { return err } err = updateCluster(member, config) if err != nil { return err } return nil } // Add. type cmdClusterAdd struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterAddUsage = u.Usage{u.NewName(u.Member).Remote()} func (c *cmdClusterAdd) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("add", cmdClusterAddUsage...) cmd.Short = i18n.G("Request a join token for adding a cluster member") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Request a join token for adding a cluster member`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterAdd) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterAddUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer memberName := parsed[0].RemoteObject.String // Request the join token. member := api.ClusterMembersPost{ ServerName: memberName, } op, err := d.CreateClusterMember(member) if err != nil { return err } opAPI := op.Get() joinToken, err := opAPI.ToClusterJoinToken() if err != nil { return fmt.Errorf(i18n.G("Failed converting token operation to join token: %w"), err) } if !c.global.flagQuiet { fmt.Printf(i18n.G("Member %s join token:")+"\n", memberName) } fmt.Println(joinToken.String()) return nil } // List Tokens. type cmdClusterListTokens struct { global *cmdGlobal cluster *cmdCluster flagFormat string flagColumns string } type clusterListTokenColumn struct { Name string Data func(*api.ClusterMemberJoinToken) string } var cmdClusterListTokensUsage = u.Usage{u.RemoteColonOpt} func (c *cmdClusterListTokens) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list-tokens", cmdClusterListTokensUsage...) cmd.Short = i18n.G("List all active cluster member join tokens") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List all active cluster member join tokens Default column layout: ntE == Columns == The -c option takes a comma separated list of arguments that control which network zone attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: n - Name t - Token E - Expires At`)) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable if demanded, e.g. csv,header`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultclusterTokensColumns, "", i18n.G("Columns")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } const defaultclusterTokensColumns = "ntE" func (c *cmdClusterListTokens) parseColumns() ([]clusterListTokenColumn, error) { columnsShorthandMap := map[rune]clusterListTokenColumn{ 'n': {i18n.G("NAME"), c.serverNameColumnData}, 't': {i18n.G("TOKEN"), c.tokenColumnData}, 'E': {i18n.G("EXPIRES AT"), c.expiresAtColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []clusterListTokenColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdClusterListTokens) serverNameColumnData(token *api.ClusterMemberJoinToken) string { return token.ServerName } func (c *cmdClusterListTokens) tokenColumnData(token *api.ClusterMemberJoinToken) string { return token.String() } func (c *cmdClusterListTokens) expiresAtColumnData(token *api.ClusterMemberJoinToken) string { return token.ExpiresAt.Local().Format(dateLayout) } func (c *cmdClusterListTokens) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterListTokensUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer // Check if clustered. cluster, _, err := d.GetCluster() if err != nil { return err } if !cluster.Enabled { return errors.New(i18n.G("Server isn't part of a cluster")) } // Get the cluster member join tokens. Use default project as join tokens are created in default project. ops, err := d.UseProject(api.ProjectDefaultName).GetOperations() if err != nil { return err } data := [][]string{} joinTokens := []*api.ClusterMemberJoinToken{} // Parse column flags. columns, err := c.parseColumns() if err != nil { return err } for _, op := range ops { if op.Class != api.OperationClassToken { continue } if op.StatusCode != api.Running { continue // Tokens are single use, so if cancelled but not deleted yet its not available. } joinToken, err := op.ToClusterJoinToken() if err != nil { continue // Operation is not a valid cluster member join token operation. } line := []string{} for _, column := range columns { line = append(line, column.Data(joinToken)) } joinTokens = append(joinTokens, joinToken) data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, joinTokens) } // Revoke Tokens. type cmdClusterRevokeToken struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterRevokeTokenUsage = u.Usage{u.Member.Remote()} func (c *cmdClusterRevokeToken) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("revoke-token", cmdClusterRevokeTokenUsage...) cmd.Short = i18n.G("Revoke cluster member join token") cmd.Long = cli.FormatSection(color.DescriptionPrefix, cmd.Short) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterRevokeToken) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterRevokeTokenUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer remoteName := parsed[0].RemoteName memberName := parsed[0].RemoteObject.String // Check if clustered. cluster, _, err := d.GetCluster() if err != nil { return err } if !cluster.Enabled { return errors.New(i18n.G("Server isn't part of a cluster")) } // Get the cluster member join tokens. Use default project as join tokens are created in default project. ops, err := d.UseProject(api.ProjectDefaultName).GetOperations() if err != nil { return err } for _, op := range ops { if op.Class != api.OperationClassToken { continue } if op.StatusCode != api.Running { continue // Tokens are single use, so if cancelled but not deleted yet its not available. } joinToken, err := op.ToClusterJoinToken() if err != nil { continue // Operation is not a valid cluster member join token operation. } if joinToken.ServerName == memberName { // Delete the operation err = d.DeleteOperation(op.ID) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Cluster join token for %s deleted")+"\n", formatRemote(c.global.conf, parsed[0])) } return nil } } return fmt.Errorf(i18n.G("No cluster join token for member %s on remote: %s"), memberName, remoteName) } // Update Certificates. type cmdClusterUpdateCertificate struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterUpdateCertificateUsage = u.Usage{u.RemoteColonOpt, u.Placeholder(i18n.G("cert.crt")), u.Placeholder(i18n.G("cert.key"))} func (c *cmdClusterUpdateCertificate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("update-certificate", cmdClusterUpdateCertificateUsage...) cmd.Aliases = []string{"update-cert"} cmd.Short = i18n.G("Update cluster certificate") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Update cluster certificate with PEM certificate and key read from input files.")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } if len(args) == 1 { return nil, cobra.ShellCompDirectiveDefault } if len(args) == 2 { return nil, cobra.ShellCompDirectiveDefault } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterUpdateCertificate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterUpdateCertificateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer remoteName := parsed[0].RemoteName certFile := parsed[1].String keyFile := parsed[2].String // Check if clustered. cluster, _, err := d.GetCluster() if err != nil { return err } if !cluster.Enabled { return errors.New(i18n.G("Server isn't part of a cluster")) } if !util.PathExists(certFile) { return fmt.Errorf(i18n.G("Could not find certificate file path: %s"), certFile) } if !util.PathExists(keyFile) { return fmt.Errorf(i18n.G("Could not find certificate key file path: %s"), keyFile) } cert, err := os.ReadFile(certFile) if err != nil { return fmt.Errorf(i18n.G("Could not read certificate file: %s with error: %v"), certFile, err) } key, err := os.ReadFile(keyFile) if err != nil { return fmt.Errorf(i18n.G("Could not read certificate key file: %s with error: %v"), keyFile, err) } certificates := api.ClusterCertificatePut{ ClusterCertificate: string(cert), ClusterCertificateKey: string(key), } err = d.UpdateClusterCertificate(certificates, "") if err != nil { return err } certf := c.global.conf.ServerCertPath(remoteName) if util.PathExists(certf) { err = os.WriteFile(certf, cert, 0o644) if err != nil { return fmt.Errorf(i18n.G("Could not write new remote certificate for remote '%s' with error: %v"), remoteName, err) } } if !c.global.flagQuiet { fmt.Println(i18n.G("Successfully updated cluster certificates")) } return nil } type cmdClusterEvacuateAction struct { global *cmdGlobal flagAction string flagForce bool } // Cluster member evacuation. type cmdClusterEvacuate struct { global *cmdGlobal cluster *cmdCluster action *cmdClusterEvacuateAction } var cmdClusterEvacuateRestoreUsage = u.Usage{u.Member.Remote()} func (c *cmdClusterEvacuate) command() *cobra.Command { cmdAction := cmdClusterEvacuateAction{global: c.global} c.action = &cmdAction cmd := c.action.command() cmd.Aliases = []string{"evac"} cmd.Use = cli.U("evacuate", cmdClusterEvacuateRestoreUsage...) cmd.Short = i18n.G("Evacuate cluster member") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Evacuate cluster member`)) cli.AddStringFlag(cmd.Flags(), &c.action.flagAction, "action", "", "", i18n.G(`Force a particular evacuation action`)) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // Cluster member restore. type cmdClusterRestore struct { global *cmdGlobal cluster *cmdCluster action *cmdClusterEvacuateAction } func (c *cmdClusterRestore) command() *cobra.Command { cmdAction := cmdClusterEvacuateAction{global: c.global} c.action = &cmdAction cmd := c.action.command() cmd.Use = cli.U("restore", cmdClusterEvacuateRestoreUsage...) cmd.Short = i18n.G("Restore cluster member") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Restore cluster member`)) cli.AddStringFlag(cmd.Flags(), &c.action.flagAction, "action", "", "", i18n.G(`Force a particular restoration action`)) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterEvacuateAction) command() *cobra.Command { cmd := &cobra.Command{} cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G(`Force evacuation without user confirmation`)) return cmd } func (c *cmdClusterEvacuateAction) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterEvacuateRestoreUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer memberName := parsed[0].RemoteObject.String if !c.flagForce { evacuate, err := c.global.asker.AskBool(fmt.Sprintf(i18n.G("Are you sure you want to %s cluster member %q? (yes/no) [default=no]: "), cmd.Name(), memberName), "no") if err != nil { return err } if !evacuate { return nil } } state := api.ClusterMemberStatePost{ Action: cmd.Name(), Mode: c.flagAction, } op, err := d.UpdateClusterMemberState(memberName, state) if err != nil { return fmt.Errorf(i18n.G("Failed to update cluster member state: %w"), err) } var format string if cmd.Name() == "restore" { format = i18n.G("Restoring cluster member: %s") } else { format = i18n.G("Evacuating cluster member: %s") } progress := cli.ProgressRenderer{ Format: format, Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } err = op.Wait() if err != nil { progress.Done("") return err } progress.Done("") return nil } // prepareClusterMemberServerFilters processes and formats filter criteria // for cluster members, ensuring they are in a format that the server can interpret. func prepareClusterMemberServerFilters(filters []string, i any) []string { formattedFilters := []string{} for _, filter := range filters { membs := strings.SplitN(filter, "=", 2) key := membs[0] if len(membs) == 1 { regexpValue := key if !strings.Contains(key, "^") && !strings.Contains(key, "$") { regexpValue = "^" + regexpValue + "$" } filter = fmt.Sprintf("server_name=(%s|^%s.*)", regexpValue, key) } else { firstPart := key if strings.Contains(key, ".") { firstPart = strings.Split(key, ".")[0] } if !structHasField(reflect.TypeOf(i), firstPart) { filter = fmt.Sprintf("config.%s", filter) } } formattedFilters = append(formattedFilters, filter) } return formattedFilters } func newInitPressed() *api.InitPreseed { // Initialize config config := api.InitPreseed{} config.Config = map[string]string{} config.Networks = []api.InitNetworksProjectPost{} config.StoragePools = []api.StoragePoolsPost{} config.Profiles = []api.InitProfileProjectPost{ { ProfilesPost: api.ProfilesPost{ Name: "default", ProfilePut: api.ProfilePut{ Config: map[string]string{}, Devices: map[string]map[string]string{}, }, }, Project: api.ProjectDefaultName, }, } return &config } func askClustering(asker ask.Asker, config *api.InitPreseed, cluster incus.InstanceServer, server incus.InstanceServer, forceJoinExisting bool) error { var err error // Setup the cluster seed data. config.Cluster = &api.InitClusterPreseed{} config.Cluster.Enabled = true // Get the current server's configuration. serverConfig, _, err := server.GetServer() if err != nil { return err } // Shared logic for server name. askForServerName := func() error { config.Cluster.ServerName, err = asker.AskString(fmt.Sprintf(i18n.G("What member name should be used to identify this server in the cluster?")+" [default=%s]: ", serverConfig.Environment.ServerName), serverConfig.Environment.ServerName, nil) if err != nil { return err } return nil } // Ask for the joining server's listen address. var address string if len(serverConfig.Environment.Addresses) > 0 { address, _, err = net.SplitHostPort(serverConfig.Environment.Addresses[0]) if err != nil { return err } } else { address = internalUtil.NetworkInterfaceAddress() } validateServerAddress := func(value string) error { address := internalUtil.CanonicalNetworkAddress(value, ports.HTTPSDefaultPort) host, _, _ := net.SplitHostPort(address) if slices.Contains([]string{"", "[::]", "0.0.0.0"}, host) { return errors.New(i18n.G("Invalid IP address or DNS name")) } return nil } serverAddress, err := asker.AskString(fmt.Sprintf(i18n.G("What IP address or DNS name should be used to reach this server?")+" [default=%s]: ", address), address, validateServerAddress) if err != nil { return err } serverAddress = internalUtil.CanonicalNetworkAddress(serverAddress, ports.HTTPSDefaultPort) config.Config["core.https_address"] = serverAddress // Check if joining a cluster or creating a new one. clusterJoin := false if !forceJoinExisting { clusterJoin, err = asker.AskBool(i18n.G("Are you joining an existing cluster?")+" (yes/no) [default=no]: ", "no") if err != nil { return err } } if clusterJoin || forceJoinExisting { // Handle joining an existing cluster. config.Cluster.ServerAddress = serverAddress // Check if we're joining a cluster during init time. if cluster == nil { // Root is required to access the certificate files if os.Geteuid() != 0 { return errors.New(i18n.G("Joining an existing cluster requires root privileges")) } // Get the join token from the user. var joinToken *api.ClusterMemberJoinToken validJoinToken := func(input string) error { j, err := internalUtil.JoinTokenDecode(input) if err != nil { return fmt.Errorf(i18n.G("Invalid join token: %w"), err) } joinToken = j // Store valid decoded join token return nil } clusterJoinToken, err := asker.AskString(i18n.G("Please provide join token:")+" ", "", validJoinToken) if err != nil { return err } // Set server name from the join token. config.Cluster.ServerName = joinToken.ServerName // Attempt to find a working cluster member to use for joining by retrieving the // cluster certificate from each address in the join token until we succeed. for _, clusterAddress := range joinToken.Addresses { config.Cluster.ClusterAddress = internalUtil.CanonicalNetworkAddress(clusterAddress, ports.HTTPSDefaultPort) // Cluster certificate cert, err := localtls.GetRemoteCertificate(fmt.Sprintf("https://%s", config.Cluster.ClusterAddress), version.UserAgent) if err != nil { fmt.Printf(i18n.G("Error connecting to existing cluster member %q: %v")+"\n", clusterAddress, err) continue } certDigest := localtls.CertFingerprint(cert) if joinToken.Fingerprint != certDigest { return fmt.Errorf(i18n.G("Certificate fingerprint mismatch between join token and cluster member %q"), clusterAddress) } config.Cluster.ClusterCertificate = string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})) break // We've found a working cluster member. } if config.Cluster.ClusterCertificate == "" { return errors.New(i18n.G("Unable to connect to any of the cluster members specified in join token")) } // Pass the raw join token. config.Cluster.ClusterToken = clusterJoinToken } else { // Ask for the server name. err = askForServerName() if err != nil { return err } // Get a token from the cluster. member := api.ClusterMembersPost{ ServerName: config.Cluster.ServerName, } op, err := cluster.CreateClusterMember(member) if err != nil { return err } opAPI := op.Get() joinToken, err := opAPI.ToClusterJoinToken() if err != nil { return fmt.Errorf(i18n.G("Failed converting token operation to join token: %w"), err) } // Get cluster connection info. connectInfo, err := cluster.GetConnectionInfo() if err != nil { return err } // Set the token. config.Cluster.ClusterAddress = connectInfo.URL config.Cluster.ClusterCertificate = connectInfo.Certificate config.Cluster.ClusterToken = joinToken.String() } // Confirm wiping. clusterWipeMember, err := asker.AskBool(i18n.G("All existing data is lost when joining a cluster, continue?")+" (yes/no) [default=no] ", "no") if err != nil { return err } if !clusterWipeMember { return errors.New(i18n.G("User aborted configuration")) } // Get a client for the cluster if we weren't provided one. if cluster == nil { // Connect to existing cluster serverCert, err := internalUtil.LoadServerCert(internalUtil.VarPath("")) if err != nil { return err } err = setupClusterTrust(serverCert, config.Cluster.ServerName, config.Cluster.ClusterAddress, config.Cluster.ClusterCertificate, config.Cluster.ClusterToken) if err != nil { return fmt.Errorf(i18n.G("Failed to setup trust relationship with cluster: %w"), err) } // Now we have setup trust, don't send to server, otherwise it will try and setup trust // again and if using a one-time join token, will fail. config.Cluster.ClusterToken = "" // Client parameters to connect to the target cluster member. args := &incus.ConnectionArgs{ TLSClientCert: string(serverCert.PublicKey()), TLSClientKey: string(serverCert.PrivateKey()), TLSServerCert: string(config.Cluster.ClusterCertificate), UserAgent: version.UserAgent, } client, err := incus.ConnectIncus(fmt.Sprintf("https://%s", config.Cluster.ClusterAddress), args) if err != nil { return err } cluster = client } // Get the list of required member config keys. clusterConfig, _, err := cluster.GetCluster() if err != nil { return fmt.Errorf(i18n.G("Failed to retrieve cluster information: %w"), err) } for i, config := range clusterConfig.MemberConfig { question := fmt.Sprintf(i18n.G("Choose %s:")+" ", config.Description) // Allow for empty values. configValue, err := asker.AskString(question, "", validate.Optional()) if err != nil { return err } clusterConfig.MemberConfig[i].Value = configValue } config.Cluster.MemberConfig = clusterConfig.MemberConfig } else { // Ask for server name since no token is provided. err = askForServerName() if err != nil { return err } } return nil } func setupClusterTrust(serverCert *localtls.CertInfo, serverName string, targetAddress string, targetCert string, targetToken string) error { // Connect to the target cluster node. args := &incus.ConnectionArgs{ TLSServerCert: targetCert, UserAgent: version.UserAgent, } target, err := incus.ConnectIncus(fmt.Sprintf("https://%s", targetAddress), args) if err != nil { return fmt.Errorf(i18n.G("Failed to connect to target cluster node %q: %w"), targetAddress, err) } cert, err := localtls.GenerateTrustCertificate(serverCert, serverName) if err != nil { return fmt.Errorf(i18n.G("Failed generating trust certificate: %w"), err) } post := api.CertificatesPost{ CertificatePut: cert.CertificatePut, TrustToken: targetToken, } err = target.CreateCertificate(post) if err != nil && !api.StatusErrorCheck(err, http.StatusConflict) { return fmt.Errorf(i18n.G("Failed to add server cert to cluster: %w"), err) } return nil } func fillClusterConfig(config *api.InitPreseed) error { // Check if the path to the cluster certificate is set // If yes then read cluster certificate from file if config.Cluster != nil && config.Cluster.ClusterCertificatePath != "" { if !util.PathExists(config.Cluster.ClusterCertificatePath) { return fmt.Errorf(i18n.G("Path %s doesn't exist"), config.Cluster.ClusterCertificatePath) } content, err := os.ReadFile(config.Cluster.ClusterCertificatePath) if err != nil { return err } config.Cluster.ClusterCertificate = string(content) } // Check if we got a cluster join token, if so, fill in the config with it. if config.Cluster != nil && config.Cluster.ClusterToken != "" { joinToken, err := internalUtil.JoinTokenDecode(config.Cluster.ClusterToken) if err != nil { return fmt.Errorf(i18n.G("Invalid cluster join token: %w"), err) } // Set server name from join token config.Cluster.ServerName = joinToken.ServerName if config.Cluster.ClusterCertificate == "" { // Attempt to find a working cluster member to use for joining by retrieving the // cluster certificate from each address in the join token until we succeed. for _, clusterAddress := range joinToken.Addresses { // Cluster URL config.Cluster.ClusterAddress = internalUtil.CanonicalNetworkAddress(clusterAddress, ports.HTTPSDefaultPort) // Cluster certificate cert, err := localtls.GetRemoteCertificate(fmt.Sprintf("https://%s", config.Cluster.ClusterAddress), version.UserAgent) if err != nil { fmt.Printf(i18n.G("Error connecting to existing cluster member %q: %v")+"\n", clusterAddress, err) continue } certDigest := localtls.CertFingerprint(cert) if joinToken.Fingerprint != certDigest { return fmt.Errorf(i18n.G("Certificate fingerprint mismatch between join token and cluster member %q"), clusterAddress) } config.Cluster.ClusterCertificate = string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})) break // We've found a working cluster member. } } if config.Cluster.ClusterCertificate == "" { return errors.New(i18n.G("Unable to connect to any of the cluster members specified in join token")) } } // If clustering is enabled, and no cluster.https_address network address // was specified, we fallback to core.https_address. if config.Cluster != nil && config.Config["core.https_address"] != "" && config.Config["cluster.https_address"] == "" { config.Config["cluster.https_address"] = config.Config["core.https_address"] } return nil } func updateCluster(d incus.InstanceServer, config *api.InitPreseed) error { // Ensure the server and cluster addresses are in canonical form. config.Cluster.ServerAddress = internalUtil.CanonicalNetworkAddress(config.Cluster.ServerAddress, ports.HTTPSDefaultPort) config.Cluster.ClusterAddress = internalUtil.CanonicalNetworkAddress(config.Cluster.ClusterAddress, ports.HTTPSDefaultPort) op, err := d.UpdateCluster(config.Cluster.ClusterPut, "") if err != nil { return fmt.Errorf(i18n.G("Failed to join cluster: %w"), err) } err = op.Wait() if err != nil { return fmt.Errorf(i18n.G("Failed to join cluster: %w"), err) } return nil } incus-7.0.0/cmd/incus/cluster_group.go000066400000000000000000000613101517523235500177600ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "maps" "os" "slices" "sort" "strings" "github.com/spf13/cobra" yaml "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" ) type cmdClusterGroup struct { global *cmdGlobal cluster *cmdCluster } type clusterGroupColumn struct { Name string Data func(api.ClusterGroup) string } func (c *cmdClusterGroup) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("group") cmd.Short = i18n.G("Manage cluster groups") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage cluster groups`)) // Assign clusterGroupAssignCmd := cmdClusterGroupAssign{global: c.global, cluster: c.cluster} cmd.AddCommand(clusterGroupAssignCmd.command()) // Create clusterGroupCreateCmd := cmdClusterGroupCreate{global: c.global, cluster: c.cluster} cmd.AddCommand(clusterGroupCreateCmd.command()) // Delete clusterGroupDeleteCmd := cmdClusterGroupDelete{global: c.global, cluster: c.cluster} cmd.AddCommand(clusterGroupDeleteCmd.command()) // Edit clusterGroupEditCmd := cmdClusterGroupEdit{global: c.global, cluster: c.cluster} cmd.AddCommand(clusterGroupEditCmd.command()) // List clusterGroupListCmd := cmdClusterGroupList{global: c.global, cluster: c.cluster} cmd.AddCommand(clusterGroupListCmd.command()) // Remove clusterGroupRemoveCmd := cmdClusterGroupRemove{global: c.global, cluster: c.cluster} cmd.AddCommand(clusterGroupRemoveCmd.command()) // Rename clusterGroupRenameCmd := cmdClusterGroupRename{global: c.global, cluster: c.cluster} cmd.AddCommand(clusterGroupRenameCmd.command()) // Get clusterGroupGetCmd := cmdClusterGroupGet{global: c.global, cluster: c.cluster} cmd.AddCommand(clusterGroupGetCmd.command()) // Set clusterGroupSetCmd := cmdClusterGroupSet{global: c.global, cluster: c.cluster} cmd.AddCommand(clusterGroupSetCmd.command()) // Unset clusterGroupUnsetCmd := cmdClusterGroupUnset{global: c.global, cluster: c.cluster, clusterGroupSet: &clusterGroupSetCmd} cmd.AddCommand(clusterGroupUnsetCmd.command()) // Show clusterGroupShowCmd := cmdClusterGroupShow{global: c.global, cluster: c.cluster} cmd.AddCommand(clusterGroupShowCmd.command()) // Add clusterGroupAddCmd := cmdClusterGroupAdd{global: c.global, cluster: c.cluster} cmd.AddCommand(clusterGroupAddCmd.command()) return cmd } // Assign. type cmdClusterGroupAssign struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterGroupAssignUsage = u.Usage{u.Member.Remote(), u.Group.List(1, ",")} func (c *cmdClusterGroupAssign) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("assign", cmdClusterGroupAssignUsage...) cmd.Aliases = []string{"apply"} cmd.Short = i18n.G("Assign sets of groups to cluster members") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Assign sets of groups to cluster members`)) cmd.Example = cli.FormatSection("", i18n.G( `incus cluster group assign foo default,bar Set the groups for "foo" to "default" and "bar". incus cluster group assign foo default Reset "foo" to only using the "default" cluster group.`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } if len(args) == 1 { return c.global.cmpClusterGroupNames(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterGroupAssign) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterGroupAssignUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer memberName := parsed[0].RemoteObject.String groups := parsed[1] member, etag, err := d.GetClusterMember(memberName) if err != nil { return err } member.Groups = groups.StringList err = d.UpdateClusterMember(memberName, member.Writable(), etag) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Cluster member %s added to cluster groups %v")+"\n", formatRemote(c.global.conf, parsed[0]), groups.StringList) } return nil } // Create. type cmdClusterGroupCreate struct { global *cmdGlobal cluster *cmdCluster flagDescription string } var cmdClusterGroupCreateUsage = u.Usage{u.NewName(u.Group).Remote()} func (c *cmdClusterGroupCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdClusterGroupCreateUsage...) cmd.Short = i18n.G("Create a cluster group") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Create a cluster group`)) cmd.Example = cli.FormatSection("", i18n.G(`incus cluster group create g1 Create a cluster group named g1 incus cluster group create g1 < config.yaml Create a cluster group named g1 with configuration from config.yaml`)) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Cluster group description")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterGroupCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterGroupCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer groupName := parsed[0].RemoteObject.String var stdinData api.ClusterGroupPut // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } err = loader.Load(&stdinData) if err != nil && !errors.Is(err, io.EOF) { return err } } // Create the cluster group group := api.ClusterGroupsPost{ Name: groupName, ClusterGroupPut: stdinData, } if c.flagDescription != "" { group.Description = c.flagDescription } err = d.CreateClusterGroup(group) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Cluster group %s created")+"\n", formatRemote(c.global.conf, parsed[0])) } return nil } // Delete. type cmdClusterGroupDelete struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterGroupDeleteUsage = u.Usage{u.Group.Remote().List(1)} func (c *cmdClusterGroupDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdClusterGroupDeleteUsage...) cmd.Aliases = []string{"rm"} cmd.Short = i18n.G("Delete cluster groups") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete cluster groups`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpClusterGroups(toComplete) } return cmd } func (c *cmdClusterGroupDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterGroupDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } var errs []error for _, p := range parsed[0].List { d := p.RemoteServer groupName := p.RemoteObject.String // Delete the cluster group err = d.DeleteClusterGroup(groupName) if err == nil { if !c.global.flagQuiet { fmt.Printf(i18n.G("Cluster group %s deleted")+"\n", formatRemote(c.global.conf, p)) } } else { errs = append(errs, err) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } // Edit. type cmdClusterGroupEdit struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterGroupEditUsage = u.Usage{u.Group.Remote()} func (c *cmdClusterGroupEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdClusterGroupEditUsage...) cmd.Short = i18n.G("Edit a cluster group") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Edit a cluster group`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterGroups(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterGroupEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterGroupEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer groupName := parsed[0].RemoteObject.String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } newdata := api.ClusterGroupPut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateClusterGroup(groupName, newdata, "") } // Extract the current value group, etag, err := d.GetClusterGroup(groupName) if err != nil { return err } data, err := yaml.Dump(group, yaml.V2) if err != nil { return err } // Spawn the editor content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor newdata := api.ClusterGroupPut{} err = yaml.Load(content, &newdata) if err == nil { err = d.UpdateClusterGroup(groupName, newdata, etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Returns a string explaining the expected YAML structure for a cluster group configuration. func (c *cmdClusterGroupEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the cluster group. ### Any line starting with a '# will be ignored.`) } // List. type cmdClusterGroupList struct { global *cmdGlobal cluster *cmdCluster flagFormat string flagColumns string } var cmdClusterGroupListUsage = u.Usage{u.RemoteColonOpt} func (c *cmdClusterGroupList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdClusterGroupListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List all the cluster groups") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List all the cluster groups Default column layout: ndm == Columns == The -c option takes a comma separated list of arguments that control which instance attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: n - Name d - Description m - Member`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultClusterGroupColumns, "", i18n.G("Columns")) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } const defaultClusterGroupColumns = "ndm" func (c *cmdClusterGroupList) parseColumns() ([]clusterGroupColumn, error) { columnsShorthandMap := map[rune]clusterGroupColumn{ 'n': {i18n.G("NAME"), c.clusterGroupNameColumnData}, 'm': {i18n.G("MEMBERS"), c.membersColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []clusterGroupColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdClusterGroupList) clusterGroupNameColumnData(group api.ClusterGroup) string { return group.Name } func (c *cmdClusterGroupList) descriptionColumnData(group api.ClusterGroup) string { return group.Description } func (c *cmdClusterGroupList) membersColumnData(group api.ClusterGroup) string { return fmt.Sprintf("%d", len(group.Members)) } func (c *cmdClusterGroupList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterGroupListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer // Check if clustered cluster, _, err := d.GetCluster() if err != nil { return err } if !cluster.Enabled { return errors.New(i18n.G("Server isn't part of a cluster")) } groups, err := d.GetClusterGroups() if err != nil { return err } // Parse column flags. columns, err := c.parseColumns() if err != nil { return err } // Render the table data := [][]string{} for _, group := range groups { line := []string{} for _, column := range columns { line = append(line, column.Data(group)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, groups) } // Remove. type cmdClusterGroupRemove struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterGroupRemoveUsage = u.Usage{u.Member.Remote(), u.Group} func (c *cmdClusterGroupRemove) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remove", cmdClusterGroupRemoveUsage...) cmd.Short = i18n.G("Remove member from group") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Remove a cluster member from a cluster group`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } if len(args) == 1 { return c.global.cmpClusterGroupNames(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterGroupRemove) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterGroupRemoveUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer memberName := parsed[0].RemoteObject.String group := parsed[1].String // Remove the cluster group member, etag, err := d.GetClusterMember(memberName) if err != nil { return err } if !slices.Contains(member.Groups, group) { return fmt.Errorf(i18n.G("Cluster group %s isn't currently applied to %s"), group, memberName) } groups := []string{} for _, g := range member.Groups { if g == group { continue } groups = append(groups, g) } member.Groups = groups err = d.UpdateClusterMember(memberName, member.Writable(), etag) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Cluster member %s removed from group %s")+"\n", formatRemote(c.global.conf, parsed[0]), group) } return nil } // Rename. type cmdClusterGroupRename struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterGroupRenameUsage = u.Usage{u.Group.Remote(), u.NewName(u.Group)} func (c *cmdClusterGroupRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdClusterGroupRenameUsage...) cmd.Aliases = []string{"mv"} cmd.Short = i18n.G("Rename a cluster group") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Rename a cluster group`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterGroups(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterGroupRename) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterGroupRenameUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer groupName := parsed[0].RemoteObject.String newGroupName := parsed[1].String // Perform the rename err = d.RenameClusterGroup(groupName, api.ClusterGroupPost{Name: newGroupName}) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Cluster group %s renamed to %s")+"\n", formatRemote(c.global.conf, parsed[0]), newGroupName) } return nil } // Show. type cmdClusterGroupShow struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterGroupShowUsage = u.Usage{u.Group.Remote()} func (c *cmdClusterGroupShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdClusterGroupShowUsage...) cmd.Short = i18n.G("Show cluster group configurations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show cluster group configurations`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterGroups(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterGroupShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterGroupShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer groupName := parsed[0].RemoteObject.String // Show the cluster group group, _, err := d.GetClusterGroup(groupName) if err != nil { return err } data, err := yaml.Dump(&group, yaml.V2) if err != nil { return err } fmt.Print(string(data)) return nil } // Add. type cmdClusterGroupAdd struct { global *cmdGlobal cluster *cmdCluster } var cmdClusterGroupAddUsage = u.Usage{u.Member.Remote(), u.Group} func (c *cmdClusterGroupAdd) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("add", cmdClusterGroupAddUsage...) cmd.Short = i18n.G("Add member to group") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Add a cluster member to a cluster group`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } if len(args) == 1 { return c.global.cmpClusterGroupNames(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterGroupAdd) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterGroupAddUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer memberName := parsed[0].RemoteObject.String groupName := parsed[1].String // Retrieve cluster member information. member, etag, err := d.GetClusterMember(memberName) if err != nil { return err } if slices.Contains(member.Groups, groupName) { return fmt.Errorf(i18n.G("Cluster member %s is already in group %s"), memberName, groupName) } member.Groups = append(member.Groups, groupName) err = d.UpdateClusterMember(memberName, member.Writable(), etag) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Cluster member %s added to group %s")+"\n", formatRemote(c.global.conf, parsed[0]), groupName) } return nil } // Get. type cmdClusterGroupGet struct { global *cmdGlobal cluster *cmdCluster flagIsProperty bool } var cmdClusterGroupGetUsage = u.Usage{u.Group.Remote(), u.Key} func (c *cmdClusterGroupGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdClusterGroupGetUsage...) cmd.Short = i18n.G("Get values for cluster group configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, cmd.Short) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a cluster group property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterGroups(toComplete) } if len(args) == 1 { return c.global.cmpClusterGroupConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterGroupGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterGroupGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer groupName := parsed[0].RemoteObject.String key := parsed[1].String // Get the group information group, _, err := d.GetClusterGroup(groupName) if err != nil { return err } if c.flagIsProperty { w := group.Writable() res, err := getFieldByJSONTag(&w, args[1]) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the cluster group %q: %v"), key, formatRemote(c.global.conf, parsed[0]), err) } fmt.Printf("%v\n", res) return nil } value, ok := group.Config[key] if !ok { return fmt.Errorf(i18n.G("The key %q does not exist on cluster group %q"), key, formatRemote(c.global.conf, parsed[0])) } fmt.Printf("%s\n", value) return nil } // Set. type cmdClusterGroupSet struct { global *cmdGlobal cluster *cmdCluster flagIsProperty bool } var cmdClusterGroupSetUsage = u.Usage{u.Group.Remote(), u.LegacyKV.List(1)} func (c *cmdClusterGroupSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdClusterGroupSetUsage...) cmd.Short = i18n.G("Set a cluster group's configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, cmd.Short) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a cluster group property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterGroups(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdClusterGroupSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer groupName := parsed[0].RemoteObject.String // Get the group information group, _, err := d.GetClusterGroup(groupName) if err != nil { return err } // Get the new config keys keys, err := kvToMap(parsed[1]) if err != nil { return err } writable := group.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { maps.Copy(writable.Config, keys) } return d.UpdateClusterGroup(groupName, writable, "") } func (c *cmdClusterGroupSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterGroupSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Unset. type cmdClusterGroupUnset struct { global *cmdGlobal cluster *cmdCluster clusterGroupSet *cmdClusterGroupSet flagIsProperty bool } var cmdClusterGroupUnsetUsage = u.Usage{u.Group.Remote(), u.Key} func (c *cmdClusterGroupUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdClusterGroupUnsetUsage...) cmd.Short = i18n.G("Unset a cluster group's configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, cmd.Short) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a cluster group property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterGroups(toComplete) } if len(args) == 1 { return c.global.cmpClusterGroupConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterGroupUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterGroupUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.clusterGroupSet.flagIsProperty = c.flagIsProperty return unsetKey(c.clusterGroupSet, cmd, parsed) } incus-7.0.0/cmd/incus/cluster_role.go000066400000000000000000000104261517523235500175670ustar00rootroot00000000000000package main import ( "fmt" "slices" "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdClusterRole struct { global *cmdGlobal cluster *cmdCluster } func (c *cmdClusterRole) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("role") cmd.Short = i18n.G("Manage cluster roles") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage cluster roles`)) // Add clusterRoleAddCmd := cmdClusterRoleAdd{global: c.global, cluster: c.cluster, clusterRole: c} cmd.AddCommand(clusterRoleAddCmd.command()) // Remove clusterRoleRemoveCmd := cmdClusterRoleRemove{global: c.global, cluster: c.cluster, clusterRole: c} cmd.AddCommand(clusterRoleRemoveCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } type cmdClusterRoleAdd struct { global *cmdGlobal cluster *cmdCluster clusterRole *cmdClusterRole } var cmdClusterRoleAddUsage = u.Usage{u.Member.Remote(), u.Role.List(1, ",")} func (c *cmdClusterRoleAdd) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("add", cmdClusterRoleAddUsage...) cmd.Aliases = []string{"create"} cmd.Short = i18n.G("Add roles to a cluster member") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Add roles to a cluster member`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterRoleAdd) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterRoleAddUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer memberName := parsed[0].RemoteObject.String newRoles := parsed[1].StringList // Extract the current value member, etag, err := d.GetClusterMember(memberName) if err != nil { return err } memberWritable := member.Writable() for _, newRole := range newRoles { if slices.Contains(memberWritable.Roles, newRole) { return fmt.Errorf(i18n.G("Member %q already has role %q"), formatRemote(c.global.conf, parsed[0]), newRole) } } memberWritable.Roles = append(memberWritable.Roles, newRoles...) return d.UpdateClusterMember(memberName, memberWritable, etag) } type cmdClusterRoleRemove struct { global *cmdGlobal cluster *cmdCluster clusterRole *cmdClusterRole } var cmdClusterRoleRemoteUsage = u.Usage{u.Member.Remote(), u.Role.List(1, ",")} func (c *cmdClusterRoleRemove) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remove", cmdClusterRoleRemoteUsage...) cmd.Aliases = []string{"delete", "rm"} cmd.Short = i18n.G("Remove roles from a cluster member") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Remove roles from a cluster member`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpClusterMembers(toComplete) } if len(args) == 1 { return c.global.cmpClusterMemberRoles(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdClusterRoleRemove) run(cmd *cobra.Command, args []string) error { parsed, err := cmdClusterRoleRemoteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer memberName := parsed[0].RemoteObject.String rolesToRemove := parsed[1].StringList // Extract the current value member, etag, err := d.GetClusterMember(memberName) if err != nil { return err } memberWritable := member.Writable() for _, roleToRemove := range rolesToRemove { if !slices.Contains(memberWritable.Roles, roleToRemove) { return fmt.Errorf(i18n.G("Member %q does not have role %q"), formatRemote(c.global.conf, parsed[0]), roleToRemove) } } memberWritable.Roles = removeElementsFromSlice(memberWritable.Roles, rolesToRemove...) return d.UpdateClusterMember(memberName, memberWritable, etag) } incus-7.0.0/cmd/incus/color/000077500000000000000000000000001517523235500156515ustar00rootroot00000000000000incus-7.0.0/cmd/incus/color/color.go000066400000000000000000000026371517523235500173260ustar00rootroot00000000000000package color import ( "github.com/fatih/color" "github.com/lxc/incus/v7/internal/i18n" ) func commandHeader(header string) string { return color.New(color.FgHiCyan, color.Bold, color.Underline).Sprint(header) } // A few prefixes used throughout the Incus client. var ( ErrorPrefix string WarningPrefix string DescriptionPrefix string RawUsagePrefix string UsagePrefix string AliasesPrefix string ExamplesPrefix string AvailableCommandsPrefix string FlagsPrefix string GlobalFlagsPrefix string AdditionalHelpTopicsPrefix string ) // Init initializes the global colored values. func Init(disable bool) { if disable { color.NoColor = true } ErrorPrefix = color.New(color.FgRed, color.Bold).Sprint(i18n.G("Error:")) WarningPrefix = color.New(color.FgYellow, color.Bold).Sprint(i18n.G("Warning:")) DescriptionPrefix = commandHeader(i18n.G("Description:")) RawUsagePrefix = i18n.G("Usage:") UsagePrefix = commandHeader(RawUsagePrefix) AliasesPrefix = commandHeader(i18n.G("Aliases:")) ExamplesPrefix = commandHeader(i18n.G("Examples:")) AvailableCommandsPrefix = commandHeader(i18n.G("Available Commands:")) FlagsPrefix = commandHeader(i18n.G("Flags:")) GlobalFlagsPrefix = commandHeader(i18n.G("Global Flags:")) AdditionalHelpTopicsPrefix = commandHeader(i18n.G("Additional Help Topics:")) } incus-7.0.0/cmd/incus/completion.go000066400000000000000000001041371517523235500172410ustar00rootroot00000000000000package main import ( "fmt" "io/fs" "os" "path/filepath" "regexp" "strings" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/shared/api" ) func (g *cmdGlobal) cmpClusterGroupNames(toComplete string) ([]string, cobra.ShellCompDirective) { var results []string cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(toComplete) if len(resources) <= 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] cluster, _, err := resource.server.GetCluster() if err != nil || !cluster.Enabled { return nil, cobra.ShellCompDirectiveError } results, err = resource.server.GetClusterGroupNames() if err != nil { return nil, cobra.ShellCompDirectiveError } return results, cmpDirectives } func (g *cmdGlobal) cmpClusterGroups(toComplete string) ([]string, cobra.ShellCompDirective) { results := []string{} cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(toComplete) if len(resources) <= 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] cluster, _, err := resource.server.GetCluster() if err != nil || !cluster.Enabled { return nil, cobra.ShellCompDirectiveError } groups, err := resource.server.GetClusterGroupNames() if err != nil { return nil, cobra.ShellCompDirectiveError } for _, group := range groups { var name string if resource.remote == g.conf.DefaultRemote && !strings.Contains(toComplete, g.conf.DefaultRemote) { name = group } else { name = fmt.Sprintf("%s:%s", resource.remote, group) } results = append(results, name) } if !strings.Contains(toComplete, ":") { remotes, directives := g.cmpRemotes(toComplete, false) results = append(results, remotes...) cmpDirectives |= directives } return results, cmpDirectives } func (g *cmdGlobal) cmpClusterGroupConfigs(groupName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(groupName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server cluster, _, err := client.GetCluster() if err != nil || !cluster.Enabled { return nil, cobra.ShellCompDirectiveError } group, _, err := client.GetClusterGroup(groupName) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for k := range group.Config { results = append(results, k) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpClusterMemberConfigs(memberName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(memberName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server cluster, _, err := client.GetCluster() if err != nil || !cluster.Enabled { return nil, cobra.ShellCompDirectiveError } member, _, err := client.GetClusterMember(memberName) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for k := range member.Config { results = append(results, k) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpClusterMemberRoles(memberName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(memberName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server cluster, _, err := client.GetCluster() if err != nil || !cluster.Enabled { return nil, cobra.ShellCompDirectiveError } member, _, err := client.GetClusterMember(memberName) if err != nil { return nil, cobra.ShellCompDirectiveError } return member.Roles, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpClusterMembers(toComplete string) ([]string, cobra.ShellCompDirective) { results := []string{} cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(toComplete) if len(resources) > 0 { resource := resources[0] cluster, _, err := resource.server.GetCluster() if err != nil || !cluster.Enabled { return nil, cobra.ShellCompDirectiveError } // Get the cluster members members, err := resource.server.GetClusterMembers() if err != nil { return nil, cobra.ShellCompDirectiveError } for _, member := range members { var name string if resource.remote == g.conf.DefaultRemote && !strings.Contains(toComplete, g.conf.DefaultRemote) { name = member.ServerName } else { name = fmt.Sprintf("%s:%s", resource.remote, member.ServerName) } results = append(results, name) } } if !strings.Contains(toComplete, ":") { remotes, directives := g.cmpRemotes(toComplete, false) results = append(results, remotes...) cmpDirectives |= directives } return results, cmpDirectives } func (g *cmdGlobal) cmpImages(toComplete string) ([]string, cobra.ShellCompDirective) { results := []string{} var remote string cmpDirectives := cobra.ShellCompDirectiveNoFileComp if strings.Contains(toComplete, ":") { remote = strings.Split(toComplete, ":")[0] } else { remote = g.conf.DefaultRemote } remoteServer, _ := g.conf.GetImageServer(remote) images, _ := remoteServer.GetImages() for _, image := range images { for _, alias := range image.Aliases { var name string if remote == g.conf.DefaultRemote && !strings.Contains(toComplete, g.conf.DefaultRemote) { name = alias.Name } else { name = fmt.Sprintf("%s:%s", remote, alias.Name) } results = append(results, name) } } if !strings.Contains(toComplete, ":") { remotes, directives := g.cmpRemotes(toComplete, true) results = append(results, remotes...) cmpDirectives |= directives } return results, cmpDirectives } func (g *cmdGlobal) cmpImageFingerprintsFromRemote(toComplete string, remote string) ([]string, cobra.ShellCompDirective) { results := []string{} if remote == "" { remote = g.conf.DefaultRemote } remoteServer, _ := g.conf.GetImageServer(remote) images, _ := remoteServer.GetImages() for _, image := range images { if !strings.HasPrefix(image.Fingerprint, toComplete) { continue } results = append(results, image.Fingerprint) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpInstanceAllKeys() ([]string, cobra.ShellCompDirective) { keys := []string{} for k := range instance.InstanceConfigKeysAny { keys = append(keys, k) } for k := range instance.InstanceConfigKeysContainer { keys = append(keys, k) } for k := range instance.InstanceConfigKeysVM { keys = append(keys, k) } return keys, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpInstanceConfigTemplates(instanceName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(instanceName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server instanceNameOnly := instanceName if strings.Contains(instanceName, ":") { instanceNameOnly = strings.Split(instanceName, ":")[1] } results, err := client.GetInstanceTemplateFiles(instanceNameOnly) if err != nil { cobra.CompDebug(fmt.Sprintf("%v", err), true) return nil, cobra.ShellCompDirectiveError } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpInstanceDeviceNames(instanceName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(instanceName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server instanceNameOnly, _, err := client.GetInstance(instanceName) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for k := range instanceNameOnly.Devices { results = append(results, k) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpInstanceSnapshots(instanceName string) ([]string, cobra.ShellCompDirective) { resources, err := g.parseServers(instanceName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server snapshots, err := client.GetInstanceSnapshotNames(instanceName) if err != nil { return nil, cobra.ShellCompDirectiveError } return snapshots, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpInstances(toComplete string) ([]string, cobra.ShellCompDirective) { results := []string{} cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(toComplete) if len(resources) > 0 { resource := resources[0] instances, _ := resource.server.GetInstanceNames(api.InstanceTypeAny) for _, instName := range instances { var name string if resource.remote == g.conf.DefaultRemote && !strings.Contains(toComplete, g.conf.DefaultRemote) { name = instName } else { name = fmt.Sprintf("%s:%s", resource.remote, instName) } if !strings.HasPrefix(name, toComplete) { continue } results = append(results, name) } } if !strings.Contains(toComplete, ":") { remotes, directives := g.cmpRemotes(toComplete, false) results = append(results, remotes...) cmpDirectives |= directives } return results, cmpDirectives } func (g *cmdGlobal) cmpInstancesAndSnapshots(toComplete string) ([]string, cobra.ShellCompDirective) { results := []string{} cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(toComplete) if len(resources) > 0 { resource := resources[0] if strings.Contains(resource.name, instance.SnapshotDelimiter) { instName := strings.SplitN(resource.name, instance.SnapshotDelimiter, 2)[0] snapshots, _ := resource.server.GetInstanceSnapshotNames(instName) for _, snapshot := range snapshots { results = append(results, fmt.Sprintf("%s/%s", instName, snapshot)) } } else { instances, _ := resource.server.GetInstanceNames(api.InstanceTypeAny) for _, instName := range instances { var name string if resource.remote == g.conf.DefaultRemote && !strings.Contains(toComplete, g.conf.DefaultRemote) { name = instName } else { name = fmt.Sprintf("%s:%s", resource.remote, instName) } results = append(results, name) } } } if !strings.Contains(toComplete, ":") { remotes, directives := g.cmpRemotes(toComplete, false) results = append(results, remotes...) cmpDirectives |= directives } return results, cmpDirectives } func (g *cmdGlobal) cmpInstanceNamesFromRemote(toComplete string) ([]string, cobra.ShellCompDirective) { results := []string{} resources, _ := g.parseServers(toComplete) if len(resources) > 0 { resource := resources[0] containers, _ := resource.server.GetInstanceNames("container") results = append(results, containers...) vms, _ := resource.server.GetInstanceNames("virtual-machine") results = append(results, vms...) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpNetworkACLConfigs(aclName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(aclName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server acl, _, err := client.GetNetworkACL(resource.name) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for k := range acl.Config { results = append(results, k) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpNetworkACLs(toComplete string) ([]string, cobra.ShellCompDirective) { results := []string{} cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(toComplete) if len(resources) <= 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] acls, err := resource.server.GetNetworkACLNames() if err != nil { return nil, cobra.ShellCompDirectiveError } for _, acl := range acls { var name string if resource.remote == g.conf.DefaultRemote && !strings.Contains(toComplete, g.conf.DefaultRemote) { name = acl } else { name = fmt.Sprintf("%s:%s", resource.remote, acl) } results = append(results, name) } if !strings.Contains(toComplete, ":") { remotes, directives := g.cmpRemotes(toComplete, false) results = append(results, remotes...) cmpDirectives |= directives } return results, cmpDirectives } func (g *cmdGlobal) cmpNetworkACLRuleProperties() ([]string, cobra.ShellCompDirective) { var results []string allowedKeys := networkACLRuleJSONStructFieldMap() for key := range allowedKeys { results = append(results, fmt.Sprintf("%s=", key)) } return results, cobra.ShellCompDirectiveNoSpace } func (g *cmdGlobal) cmpNetworkAddressSets(toComplete string) ([]string, cobra.ShellCompDirective) { results := []string{} cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(toComplete) if len(resources) <= 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] // Get the network address set names from the server. addrSets, err := resource.server.GetNetworkAddressSetNames() if err != nil { return nil, cobra.ShellCompDirectiveError } for _, addrSet := range addrSets { var name string if resource.remote == g.conf.DefaultRemote && !strings.Contains(toComplete, g.conf.DefaultRemote) { name = addrSet } else { name = fmt.Sprintf("%s:%s", resource.remote, addrSet) } results = append(results, name) } // Also suggest remotes if no ":" in toComplete. if !strings.Contains(toComplete, ":") { remotes, directives := g.cmpRemotes(toComplete, false) results = append(results, remotes...) cmpDirectives |= directives } return results, cmpDirectives } func (g *cmdGlobal) cmpNetworkAddressSetConfigs(addressSetName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(addressSetName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server // Get the network address set. addrSet, _, err := client.GetNetworkAddressSet(resource.name) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for k := range addrSet.Config { results = append(results, k) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpNetworkForwardConfigs(networkName string, listenAddress string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(networkName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server forward, _, err := client.GetNetworkForward(networkName, listenAddress) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for k := range forward.Config { results = append(results, k) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpNetworkForwards(networkName string) ([]string, cobra.ShellCompDirective) { cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(networkName) if len(resources) <= 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] results, err := resource.server.GetNetworkForwardAddresses(networkName) if err != nil { return nil, cobra.ShellCompDirectiveError } return results, cmpDirectives } func (g *cmdGlobal) cmpNetworkLoadBalancers(networkName string) ([]string, cobra.ShellCompDirective) { cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(networkName) if len(resources) <= 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] results, err := resource.server.GetNetworkForwardAddresses(networkName) if err != nil { return nil, cobra.ShellCompDirectiveError } return results, cmpDirectives } func (g *cmdGlobal) cmpNetworkPeerConfigs(networkName string, peerName string) ([]string, cobra.ShellCompDirective) { results := []string{} cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(networkName) if len(resources) <= 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] peer, _, err := resource.server.GetNetworkPeer(resource.name, peerName) if err != nil { return nil, cobra.ShellCompDirectiveError } for k := range peer.Config { results = append(results, k) } return results, cmpDirectives } func (g *cmdGlobal) cmpNetworkPeers(networkName string) ([]string, cobra.ShellCompDirective) { cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(networkName) if len(resources) <= 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] results, err := resource.server.GetNetworkPeerNames(networkName) if err != nil { return nil, cobra.ShellCompDirectiveError } return results, cmpDirectives } func (g *cmdGlobal) cmpNetworks(toComplete string) ([]string, cobra.ShellCompDirective) { results := []string{} cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(toComplete) if len(resources) > 0 { resource := resources[0] networks, err := resource.server.GetNetworkNames() if err != nil { return nil, cobra.ShellCompDirectiveError } for _, network := range networks { var name string if resource.remote == g.conf.DefaultRemote && !strings.Contains(toComplete, g.conf.DefaultRemote) { name = network } else { name = fmt.Sprintf("%s:%s", resource.remote, network) } results = append(results, name) } } if !strings.Contains(toComplete, ":") { remotes, directives := g.cmpRemotes(toComplete, false) results = append(results, remotes...) cmpDirectives |= directives } return results, cmpDirectives } func (g *cmdGlobal) cmpNetworkConfigs(networkName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(networkName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server network, _, err := client.GetNetwork(networkName) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for k := range network.Config { results = append(results, k) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpNetworkInstances(networkName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(networkName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server network, _, err := client.GetNetwork(networkName) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for _, i := range network.UsedBy { r := regexp.MustCompile(`/1.0/instances/(.*)`) match := r.FindStringSubmatch(i) if len(match) == 2 { results = append(results, match[1]) } } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpNetworkProfiles(networkName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(networkName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server network, _, err := client.GetNetwork(networkName) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for _, i := range network.UsedBy { r := regexp.MustCompile(`/1.0/profiles/(.*)`) match := r.FindStringSubmatch(i) if len(match) == 2 { results = append(results, match[1]) } } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpNetworkZoneConfigs(zoneName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(zoneName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server zone, _, err := client.GetNetworkZone(zoneName) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for k := range zone.Config { results = append(results, k) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpNetworkZoneRecordConfigs(zoneName string, recordName string) ([]string, cobra.ShellCompDirective) { results := []string{} cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(zoneName) if len(resources) <= 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] peer, _, err := resource.server.GetNetworkZoneRecord(resource.name, recordName) if err != nil { return nil, cobra.ShellCompDirectiveError } for k := range peer.Config { results = append(results, k) } return results, cmpDirectives } func (g *cmdGlobal) cmpNetworkZoneRecords(zoneName string) ([]string, cobra.ShellCompDirective) { cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(zoneName) if len(resources) <= 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] results, err := resource.server.GetNetworkZoneRecordNames(zoneName) if err != nil { return nil, cobra.ShellCompDirectiveError } return results, cmpDirectives } func (g *cmdGlobal) cmpNetworkZones(toComplete string) ([]string, cobra.ShellCompDirective) { results := []string{} cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(toComplete) if len(resources) > 0 { resource := resources[0] zones, err := resource.server.GetNetworkZoneNames() if err != nil { return nil, cobra.ShellCompDirectiveError } for _, project := range zones { var name string if resource.remote == g.conf.DefaultRemote && !strings.Contains(toComplete, g.conf.DefaultRemote) { name = project } else { name = fmt.Sprintf("%s:%s", resource.remote, project) } results = append(results, name) } } if !strings.Contains(toComplete, ":") { remotes, directives := g.cmpRemotes(toComplete, false) results = append(results, remotes...) cmpDirectives |= directives } return results, cmpDirectives } func (g *cmdGlobal) cmpProfileConfigs(profileName string) ([]string, cobra.ShellCompDirective) { resources, err := g.parseServers(profileName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server profile, _, err := client.GetProfile(resource.name) if err != nil { return nil, cobra.ShellCompDirectiveError } var configs []string for c := range profile.Config { configs = append(configs, c) } return configs, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpProfileDeviceNames(instanceName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(instanceName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server profile, _, err := client.GetProfile(resource.name) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for k := range profile.Devices { results = append(results, k) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpProfileNamesFromRemote(toComplete string) ([]string, cobra.ShellCompDirective) { results := []string{} resources, _ := g.parseServers(toComplete) if len(resources) > 0 { resource := resources[0] profiles, _ := resource.server.GetProfileNames() results = append(results, profiles...) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpProfiles(toComplete string, includeRemotes bool) ([]string, cobra.ShellCompDirective) { results := []string{} cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(toComplete) if len(resources) > 0 { resource := resources[0] profiles, _ := resource.server.GetProfileNames() for _, profile := range profiles { var name string if resource.remote == g.conf.DefaultRemote && !strings.Contains(toComplete, g.conf.DefaultRemote) { name = profile } else { name = fmt.Sprintf("%s:%s", resource.remote, profile) } results = append(results, name) } } if includeRemotes && !strings.Contains(toComplete, ":") { remotes, directives := g.cmpRemotes(toComplete, false) results = append(results, remotes...) cmpDirectives |= directives } return results, cmpDirectives } func (g *cmdGlobal) cmpProjectConfigs(projectName string) ([]string, cobra.ShellCompDirective) { resources, err := g.parseServers(projectName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server project, _, err := client.GetProject(resource.name) if err != nil { return nil, cobra.ShellCompDirectiveError } var configs []string for c := range project.Config { configs = append(configs, c) } return configs, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpProjects(toComplete string) ([]string, cobra.ShellCompDirective) { results := []string{} cmpDirectives := cobra.ShellCompDirectiveNoFileComp resources, _ := g.parseServers(toComplete) if len(resources) > 0 { resource := resources[0] projects, err := resource.server.GetProjectNames() if err != nil { return nil, cobra.ShellCompDirectiveError } for _, project := range projects { var name string if resource.remote == g.conf.DefaultRemote && !strings.Contains(toComplete, g.conf.DefaultRemote) { name = project } else { name = fmt.Sprintf("%s:%s", resource.remote, project) } results = append(results, name) } } if !strings.Contains(toComplete, ":") { remotes, directives := g.cmpRemotes(toComplete, false) results = append(results, remotes...) cmpDirectives |= directives } return results, cmpDirectives } func (g *cmdGlobal) cmpRemotes(toComplete string, includeAll bool) ([]string, cobra.ShellCompDirective) { results := []string{} for remoteName, rc := range g.conf.Remotes { if !includeAll && rc.Protocol != "incus" && rc.Protocol != "" { continue } if !strings.HasPrefix(remoteName, toComplete) { continue } results = append(results, fmt.Sprintf("%s:", remoteName)) } if len(results) > 0 { return results, cobra.ShellCompDirectiveNoSpace } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpRemoteNames() ([]string, cobra.ShellCompDirective) { results := []string{} for remoteName := range g.conf.Remotes { results = append(results, remoteName) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpStoragePoolConfigs(poolName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(poolName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server if strings.Contains(poolName, ":") { poolName = strings.Split(poolName, ":")[1] } pool, _, err := client.GetStoragePool(poolName) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for k := range pool.Config { results = append(results, k) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpStoragePoolWithVolume(toComplete string) ([]string, cobra.ShellCompDirective) { if !strings.Contains(toComplete, "/") { pools, compdir := g.cmpStoragePools(toComplete) if compdir == cobra.ShellCompDirectiveError { return nil, compdir } results := []string{} for _, pool := range pools { if strings.HasSuffix(pool, ":") { results = append(results, pool) } else { results = append(results, fmt.Sprintf("%s/", pool)) } } return results, cobra.ShellCompDirectiveNoSpace } pool := strings.Split(toComplete, "/")[0] volumes, compdir := g.cmpStoragePoolVolumes(pool) if compdir == cobra.ShellCompDirectiveError { return nil, compdir } results := []string{} for _, volume := range volumes { results = append(results, fmt.Sprintf("%s/%s", pool, volume)) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpStoragePools(toComplete string) ([]string, cobra.ShellCompDirective) { results := []string{} resources, _ := g.parseServers(toComplete) if len(resources) > 0 { resource := resources[0] storagePools, _ := resource.server.GetStoragePoolNames() for _, storage := range storagePools { var name string if resource.remote == g.conf.DefaultRemote && !strings.Contains(toComplete, g.conf.DefaultRemote) { name = storage } else { name = fmt.Sprintf("%s:%s", resource.remote, storage) } results = append(results, name) } } if !strings.Contains(toComplete, ":") { remotes, _ := g.cmpRemotes(toComplete, false) results = append(results, remotes...) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpStoragePoolVolumeConfigs(poolName string, volumeName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(poolName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server pool := poolName if strings.Contains(poolName, ":") { pool = strings.Split(poolName, ":")[1] } volName, volType := parseVolume("custom", volumeName) volume, _, err := client.GetStoragePoolVolume(pool, volType, volName) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for k := range volume.Config { results = append(results, k) } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpStoragePoolVolumeInstances(poolName string, volumeName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(poolName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server pool := poolName if strings.Contains(poolName, ":") { pool = strings.Split(poolName, ":")[1] } volName, volType := parseVolume("custom", volumeName) volume, _, err := client.GetStoragePoolVolume(pool, volType, volName) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for _, i := range volume.UsedBy { r := regexp.MustCompile(`/1.0/instances/(.*)`) match := r.FindStringSubmatch(i) if len(match) == 2 { results = append(results, match[1]) } } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpStoragePoolVolumeProfiles(poolName string, volumeName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(poolName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server pool := poolName if strings.Contains(poolName, ":") { pool = strings.Split(poolName, ":")[1] } volName, volType := parseVolume("custom", volumeName) volume, _, err := client.GetStoragePoolVolume(pool, volType, volName) if err != nil { return nil, cobra.ShellCompDirectiveError } var results []string for _, i := range volume.UsedBy { r := regexp.MustCompile(`/1.0/profiles/(.*)`) match := r.FindStringSubmatch(i) if len(match) == 2 { results = append(results, match[1]) } } return results, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpStoragePoolVolumeSnapshots(poolName string, volumeName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(poolName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server pool := poolName if strings.Contains(poolName, ":") { pool = strings.Split(poolName, ":")[1] } volName, volType := parseVolume("custom", volumeName) snapshots, err := client.GetStoragePoolVolumeSnapshotNames(pool, volType, volName) if err != nil { return nil, cobra.ShellCompDirectiveError } return snapshots, cobra.ShellCompDirectiveNoFileComp } func (g *cmdGlobal) cmpStoragePoolVolumes(poolName string) ([]string, cobra.ShellCompDirective) { // Parse remote resources, err := g.parseServers(poolName) if err != nil || len(resources) == 0 { return nil, cobra.ShellCompDirectiveError } resource := resources[0] client := resource.server pool := poolName if strings.Contains(poolName, ":") { pool = strings.Split(poolName, ":")[1] } volumes, err := client.GetStoragePoolVolumeNames(pool) if err != nil { return nil, cobra.ShellCompDirectiveError } return volumes, cobra.ShellCompDirectiveNoFileComp } func isSymlinkToDir(path string, d fs.DirEntry) bool { if d.Type()&fs.ModeSymlink == 0 { return false } info, err := os.Stat(path) if err != nil || !info.IsDir() { return false } return true } func (g *cmdGlobal) cmpFiles(toComplete string, includeLocalFiles bool) ([]string, cobra.ShellCompDirective) { instances, directives := g.cmpInstances(toComplete) for i := range instances { if strings.HasSuffix(instances[i], ":") { continue } instances[i] += "/" } if len(instances) == 0 { if includeLocalFiles { return nil, cobra.ShellCompDirectiveDefault } return instances, directives } directives |= cobra.ShellCompDirectiveNoSpace if !includeLocalFiles { return instances, directives } var files []string sep := string(filepath.Separator) dir, prefix := filepath.Split(toComplete) switch prefix { case ".": files = append(files, dir+"."+sep) fallthrough case "..": files = append(files, dir+".."+sep) directives |= cobra.ShellCompDirectiveNoSpace } root, err := filepath.EvalSymlinks(filepath.Dir(dir)) if err != nil { return append(instances, files...), directives } _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil || path == root { return err } base := filepath.Base(path) if strings.HasPrefix(base, prefix) { file := dir + base switch { case d.IsDir(): directives |= cobra.ShellCompDirectiveNoSpace file += sep case isSymlinkToDir(path, d): directives |= cobra.ShellCompDirectiveNoSpace if base == prefix { file += sep } } files = append(files, file) } if d.IsDir() { return fs.SkipDir } return nil }) return append(instances, files...), directives } incus-7.0.0/cmd/incus/config.go000066400000000000000000000507461517523235500163430ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "maps" "os" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" ) type cmdConfig struct { global *cmdGlobal flagTarget string } func (c *cmdConfig) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("config") cmd.Short = i18n.G("Manage instance and server configuration options") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Manage instance and server configuration options`)) // Device configDeviceCmd := cmdConfigDevice{global: c.global, config: c} cmd.AddCommand(configDeviceCmd.command()) // Edit configEditCmd := cmdConfigEdit{global: c.global, config: c} cmd.AddCommand(configEditCmd.command()) // Get configGetCmd := cmdConfigGet{global: c.global, config: c} cmd.AddCommand(configGetCmd.command()) // Metadata configMetadataCmd := cmdConfigMetadata{global: c.global, config: c} cmd.AddCommand(configMetadataCmd.command()) // Profile configProfileCmd := cmdProfile{global: c.global} profileCmd := configProfileCmd.command() profileCmd.Hidden = true profileCmd.Deprecated = i18n.G("please use `incus profile`") cmd.AddCommand(profileCmd) // Set configSetCmd := cmdConfigSet{global: c.global, config: c} cmd.AddCommand(configSetCmd.command()) // Show configShowCmd := cmdConfigShow{global: c.global, config: c} cmd.AddCommand(configShowCmd.command()) // Template configTemplateCmd := cmdConfigTemplate{global: c.global, config: c} cmd.AddCommand(configTemplateCmd.command()) // Trust configTrustCmd := cmdConfigTrust{global: c.global, config: c} cmd.AddCommand(configTrustCmd.command()) // Unset configUnsetCmd := cmdConfigUnset{global: c.global, config: c, configSet: &configSetCmd} cmd.AddCommand(configUnsetCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Edit. type cmdConfigEdit struct { global *cmdGlobal config *cmdConfig } var cmdConfigEditUsage = u.Usage{u.MakePath(u.Instance, u.Snapshot.Optional()).Optional().Remote()} func (c *cmdConfigEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdConfigEditUsage...) cmd.Short = i18n.G("Edit instance or server configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Edit instance or server configurations as YAML`)) cmd.Example = cli.FormatSection("", i18n.G( `incus config edit < instance.yaml Update the instance configuration from config.yaml.`)) cli.AddStringFlag(cmd.Flags(), &c.config.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // helpTemplate returns a sample YAML configuration and guidelines for editing instance configurations. func (c *cmdConfigEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the configuration. ### Any line starting with a '# will be ignored. ### ### A sample configuration looks like: ### name: instance1 ### profiles: ### - default ### config: ### volatile.eth0.hwaddr: 10:66:6a:e9:f8:7f ### devices: ### homedir: ### path: /extra ### source: /home/user ### type: disk ### ephemeral: false ### ### Note that the name is shown but cannot be changed`) } func (c *cmdConfigEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer parsedPath := parsed[0].RemoteObject // Edit the config if !parsedPath.Skipped { fields := parsedPath.StringList isSnapshot := !parsedPath.List[1].Skipped // Quick checks. if c.config.flagTarget != "" { return errors.New(i18n.G("--target cannot be used with instances")) } // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } var op incus.Operation if isSnapshot { newdata := api.InstanceSnapshotPut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } op, err = d.UpdateInstanceSnapshot(fields[0], fields[1], newdata, "") if err != nil { return err } } else { newdata := api.InstancePut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } op, err = d.UpdateInstance(parsedPath.String, newdata, "") if err != nil { return err } } return op.Wait() } var data []byte var etag string // Extract the current value if isSnapshot { var inst *api.InstanceSnapshot inst, etag, err = d.GetInstanceSnapshot(fields[0], fields[1]) if err != nil { return err } // Empty expanded config so it isn't shown in edit screen (relies on omitempty tag). inst.ExpandedConfig = nil inst.ExpandedDevices = nil data, err = yaml.Dump(&inst, yaml.V2) if err != nil { return err } } else { var inst *api.Instance inst, etag, err = d.GetInstance(parsedPath.String) if err != nil { return err } // Empty expanded config so it isn't shown in edit screen (relies on omitempty tag). inst.ExpandedConfig = nil inst.ExpandedDevices = nil data, err = yaml.Dump(&inst, yaml.V2) if err != nil { return err } } // Spawn the editor content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor if isSnapshot { newdata := api.InstanceSnapshotPut{} err = yaml.Load(content, &newdata) if err == nil { var op incus.Operation op, err = d.UpdateInstanceSnapshot(fields[0], fields[1], newdata, etag) if err == nil { err = op.Wait() } } } else { newdata := api.InstancePut{} err = yaml.Load(content, &newdata) if err == nil { var op incus.Operation op, err = d.UpdateInstance(parsedPath.String, newdata, etag) if err == nil { err = op.Wait() } } } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Targeting if c.config.flagTarget != "" { if !d.IsClustered() { return errors.New(i18n.G("To use --target, the destination remote must be a cluster")) } d = d.UseTarget(c.config.flagTarget) } // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } newdata := api.ServerPut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateServer(newdata, "") } // Extract the current value server, etag, err := d.GetServer() if err != nil { return err } brief := server.Writable() data, err := yaml.Dump(&brief, yaml.V2) if err != nil { return err } // Spawn the editor content, err := cli.TextEditor("", data) if err != nil { return err } for { // Parse the text received from the editor newdata := api.ServerPut{} err = yaml.Load(content, &newdata) if err == nil { err = d.UpdateServer(newdata, etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Get. type cmdConfigGet struct { global *cmdGlobal config *cmdConfig flagExpanded bool flagIsProperty bool } var cmdConfigGetUsage = u.Usage{u.MakePath(u.Instance, u.Snapshot.Optional()).Optional().Remote(), u.Key} func (c *cmdConfigGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdConfigGetUsage...) cmd.Short = i18n.G("Get values for instance or server configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Get values for instance or server configuration keys`)) cli.AddBoolFlag(cmd.Flags(), &c.flagExpanded, "expanded|e", i18n.G("Access the expanded configuration")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as an instance property")) cli.AddStringFlag(cmd.Flags(), &c.config.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } if len(args) == 1 { return c.global.cmpInstanceAllKeys() } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigGet) run(cmd *cobra.Command, args []string) error { // Do NOT blindly copy the following parsing line; it performs right-to-left parsing, which in // most cases is NOT what you want. parsed, err := cmdConfigGetUsage.Parse(c.global.conf, cmd, args, true) if err != nil { return err } d := parsed[0].RemoteServer parsedPath := parsed[0].RemoteObject key := parsed[1].String // Get the config key if !parsedPath.Skipped { fields := parsedPath.StringList isSnapshot := !parsedPath.List[1].Skipped // Quick checks. if c.config.flagTarget != "" { return errors.New(i18n.G("--target cannot be used with instances")) } if isSnapshot { inst, _, err := d.GetInstanceSnapshot(fields[0], fields[1]) if err != nil { return err } if c.flagIsProperty { res, err := getFieldByJSONTag(inst, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the instance snapshot %s: %v"), key, formatRemote(c.global.conf, parsed[0]), err) } fmt.Printf("%v\n", res) } else { if c.flagExpanded { fmt.Println(inst.ExpandedConfig[key]) } else { fmt.Println(inst.Config[key]) } } return nil } resp, _, err := d.GetInstance(parsedPath.String) if err != nil { return err } if c.flagIsProperty { w := resp.Writable() res, err := getFieldByJSONTag(&w, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the instance %q: %v"), key, formatRemote(c.global.conf, parsed[0]), err) } fmt.Printf("%v\n", res) } else { if c.flagExpanded { fmt.Println(resp.ExpandedConfig[key]) } else { fmt.Println(resp.Config[key]) } } } else { // Quick check. if c.flagExpanded { return errors.New(i18n.G("--expanded cannot be used with a server")) } // Targeting if c.config.flagTarget != "" { if !d.IsClustered() { return errors.New(i18n.G("To use --target, the destination remote must be a cluster")) } d = d.UseTarget(c.config.flagTarget) } resp, _, err := d.GetServer() if err != nil { return err } value := resp.Config[key] fmt.Println(value) } return nil } // Set. type cmdConfigSet struct { global *cmdGlobal config *cmdConfig flagIsProperty bool } var cmdConfigSetUsage = u.Usage{u.MakePath(u.Instance, u.Snapshot.Optional()).Optional().Remote(), u.LegacyKV.List(1)} func (c *cmdConfigSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdConfigSetUsage...) cmd.Short = i18n.G("Set instance or server configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set instance or server configuration keys For backward compatibility, a single configuration key may still be set with: incus config set [:][] `)) cmd.Example = cli.FormatSection("", i18n.G( `incus config set [:] limits.cpu=2 Will set a CPU limit of "2" for the instance. incus config set my-instance cloud-init.user-data - < cloud-init.yaml Sets the cloud-init user-data for instance "my-instance" by reading "cloud-init.yaml" through stdin. incus config set core.https_address=[::]:8443 Will have the server listen on IPv4 and IPv6 port 8443.`)) cli.AddStringFlag(cmd.Flags(), &c.config.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as an instance property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } if len(args) == 1 { return c.global.cmpInstanceAllKeys() } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdConfigSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer parsedPath := parsed[0].RemoteObject keys, err := kvToMap(parsed[1]) if err != nil { return err } // Set the config keys if !parsedPath.Skipped { fields := parsedPath.StringList isSnapshot := !parsedPath.List[1].Skipped // Quick checks. if c.config.flagTarget != "" { return errors.New(i18n.G("--target cannot be used with instances")) } if isSnapshot { inst, etag, err := d.GetInstanceSnapshot(fields[0], fields[1]) if err != nil { return err } writable := inst.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting properties: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } op, err := d.UpdateInstanceSnapshot(fields[0], fields[1], writable, etag) if err != nil { return err } return op.Wait() } return errors.New(i18n.G("The is no config key to set on an instance snapshot.")) } inst, etag, err := d.GetInstance(parsedPath.String) if err != nil { return err } writable := inst.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting properties: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { for k, v := range keys { if cmd.Name() == "unset" { _, ok := writable.Config[k] if !ok { return fmt.Errorf(i18n.G("Can't unset key '%s', it's not currently set"), k) } delete(writable.Config, k) } else { writable.Config[k] = v } } } op, err := d.UpdateInstance(parsedPath.String, writable, etag) if err != nil { return err } return op.Wait() } // Targeting if c.config.flagTarget != "" { if !d.IsClustered() { return errors.New(i18n.G("To use --target, the destination remote must be a cluster")) } d = d.UseTarget(c.config.flagTarget) } // Server keys server, etag, err := d.GetServer() if err != nil { return err } if server.Config == nil { server.Config = map[string]string{} } maps.Copy(server.Config, keys) return d.UpdateServer(server.Writable(), etag) } func (c *cmdConfigSet) run(cmd *cobra.Command, args []string) error { // Do NOT blindly copy the following parsing line; it performs right-to-left parsing, which in // most cases is NOT what you want. parsed, err := cmdConfigSetUsage.Parse(c.global.conf, cmd, args, true) if err != nil { return err } return c.set(cmd, parsed) } // Show. type cmdConfigShow struct { global *cmdGlobal config *cmdConfig flagExpanded bool } var cmdConfigShowUsage = u.Usage{u.MakePath(u.Instance, u.Snapshot.Optional()).Optional().Remote()} func (c *cmdConfigShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdConfigShowUsage...) cmd.Short = i18n.G("Show instance or server configurations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Show instance or server configurations`)) cli.AddBoolFlag(cmd.Flags(), &c.flagExpanded, "expanded|e", i18n.G("Show the expanded configuration")) cli.AddStringFlag(cmd.Flags(), &c.config.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) != 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return c.global.cmpInstances(toComplete) } return cmd } func (c *cmdConfigShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer parsedPath := parsed[0].RemoteObject // Show configuration var data []byte if parsedPath.Skipped { // Quick check. if c.flagExpanded { return errors.New(i18n.G("--expanded cannot be used with a server")) } // Targeting if c.config.flagTarget != "" { if !d.IsClustered() { return errors.New(i18n.G("To use --target, the destination remote must be a cluster")) } d = d.UseTarget(c.config.flagTarget) } // Server config server, _, err := d.GetServer() if err != nil { return err } brief := server.Writable() data, err = yaml.Dump(&brief, yaml.V2) if err != nil { return err } } else { // Quick checks. if c.config.flagTarget != "" { return errors.New(i18n.G("--target cannot be used with instances")) } // Instance or snapshot config var brief any if !parsedPath.List[1].Skipped { // Snapshot fields := parsedPath.StringList snap, _, err := d.GetInstanceSnapshot(fields[0], fields[1]) if err != nil { return err } brief = snap if c.flagExpanded { brief.(*api.InstanceSnapshot).Config = snap.ExpandedConfig brief.(*api.InstanceSnapshot).Devices = snap.ExpandedDevices } } else { // Instance inst, _, err := d.GetInstance(parsedPath.String) if err != nil { return err } writable := inst.Writable() brief = &writable if c.flagExpanded { brief.(*api.InstancePut).Config = inst.ExpandedConfig brief.(*api.InstancePut).Devices = inst.ExpandedDevices } } data, err = yaml.Dump(&brief, yaml.V2) if err != nil { return err } } fmt.Printf("%s", data) return nil } // Unset. type cmdConfigUnset struct { global *cmdGlobal config *cmdConfig configSet *cmdConfigSet flagIsProperty bool } var cmdConfigUnsetUsage = u.Usage{u.MakePath(u.Instance, u.Snapshot.Optional()).Optional().Remote(), u.Key} func (c *cmdConfigUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdConfigUnsetUsage...) cmd.Short = i18n.G("Unset instance or server configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Unset instance or server configuration keys`)) cli.AddStringFlag(cmd.Flags(), &c.config.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as an instance property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } if len(args) == 1 { return c.global.cmpInstanceAllKeys() } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigUnset) run(cmd *cobra.Command, args []string) error { // Do NOT blindly copy the following parsing line; it performs right-to-left parsing, which in // most cases is NOT what you want. parsed, err := cmdConfigUnsetUsage.Parse(c.global.conf, cmd, args, true) if err != nil { return err } c.configSet.flagIsProperty = c.flagIsProperty return unsetKey(c.configSet, cmd, parsed) } incus-7.0.0/cmd/incus/config_device.go000066400000000000000000000475571517523235500176700ustar00rootroot00000000000000package main import ( "errors" "fmt" "maps" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdConfigDevice struct { global *cmdGlobal config *cmdConfig profile *cmdProfile } func (c *cmdConfigDevice) formatUsage(usage u.Usage) u.Usage { if c.profile != nil { return append(u.Usage{u.Profile.Remote()}, usage...) } return append(u.Usage{u.Instance.Remote()}, usage...) } func (c *cmdConfigDevice) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("device") cmd.Short = i18n.G("Manage devices") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage devices`)) // Add configDeviceAddCmd := cmdConfigDeviceAdd{global: c.global, config: c.config, profile: c.profile, configDevice: c} cmd.AddCommand(configDeviceAddCmd.command()) // Get configDeviceGetCmd := cmdConfigDeviceGet{global: c.global, config: c.config, profile: c.profile, configDevice: c} cmd.AddCommand(configDeviceGetCmd.command()) // List configDeviceListCmd := cmdConfigDeviceList{global: c.global, config: c.config, profile: c.profile, configDevice: c} cmd.AddCommand(configDeviceListCmd.command()) // Override if c.config != nil { configDeviceOverrideCmd := cmdConfigDeviceOverride{global: c.global, config: c.config} cmd.AddCommand(configDeviceOverrideCmd.command()) } // Remove configDeviceRemoveCmd := cmdConfigDeviceRemove{global: c.global, config: c.config, profile: c.profile, configDevice: c} cmd.AddCommand(configDeviceRemoveCmd.command()) // Set configDeviceSetCmd := cmdConfigDeviceSet{global: c.global, config: c.config, profile: c.profile, configDevice: c} cmd.AddCommand(configDeviceSetCmd.command()) // Show configDeviceShowCmd := cmdConfigDeviceShow{global: c.global, config: c.config, profile: c.profile, configDevice: c} cmd.AddCommand(configDeviceShowCmd.command()) // Unset configDeviceUnsetCmd := cmdConfigDeviceUnset{global: c.global, config: c.config, profile: c.profile, configDevice: c, configDeviceSet: &configDeviceSetCmd} cmd.AddCommand(configDeviceUnsetCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Add. type cmdConfigDeviceAdd struct { global *cmdGlobal config *cmdConfig configDevice *cmdConfigDevice profile *cmdProfile } var cmdConfigDeviceAddUsage = u.Usage{u.NewName(u.Device), u.Type, u.LegacyKV.List(0)} func (c *cmdConfigDeviceAdd) command() *cobra.Command { cmd := &cobra.Command{} cmd.Aliases = []string{"create"} cmd.Use = cli.U("add", c.configDevice.formatUsage(cmdConfigDeviceAddUsage)...) cmd.Short = i18n.G("Add instance devices") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Add instance devices`)) if c.config != nil { cmd.Example = cli.FormatSection("", i18n.G( `incus config device add [:]instance1 disk source=/share/c1 path=/opt Will mount the host's /share/c1 onto /opt in the instance. incus config device add [:]instance1 disk pool=some-pool source=some-volume path=/opt Will mount the some-volume volume on some-pool onto /opt in the instance.`)) } else if c.profile != nil { cmd.Example = cli.FormatSection("", i18n.G( `incus profile device add [:]profile1 disk source=/share/c1 path=/opt Will mount the host's /share/c1 onto /opt in the instance. incus profile device add [:]profile1 disk pool=some-pool source=some-volume path=/opt Will mount the some-volume volume on some-pool onto /opt in the instance.`)) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { if c.config != nil { return c.global.cmpInstances(toComplete) } else if c.profile != nil { return c.global.cmpProfiles(toComplete, true) } } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigDeviceAdd) run(cmd *cobra.Command, args []string) error { parsed, err := c.configDevice.formatUsage(cmdConfigDeviceAddUsage).Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer objectName := parsed[0].RemoteObject.String // Add the device devName := parsed[1].String device, err := kvToMap(parsed[3]) if err != nil { return err } device["type"] = parsed[2].String if c.profile != nil { profile, etag, err := d.GetProfile(objectName) if err != nil { return err } if profile.Devices == nil { profile.Devices = make(map[string]map[string]string) } _, ok := profile.Devices[devName] if ok { return errors.New(i18n.G("The device already exists")) } profile.Devices[devName] = device err = d.UpdateProfile(objectName, profile.Writable(), etag) if err != nil { return err } } else { inst, etag, err := d.GetInstance(objectName) if err != nil { return err } _, ok := inst.Devices[devName] if ok { return errors.New(i18n.G("The device already exists")) } inst.Devices[devName] = device op, err := d.UpdateInstance(objectName, inst.Writable(), etag) if err != nil { return err } err = op.Wait() if err != nil { return err } } if !c.global.flagQuiet { fmt.Printf(i18n.G("Device %s added to %s")+"\n", devName, objectName) } return nil } // Get. type cmdConfigDeviceGet struct { global *cmdGlobal config *cmdConfig configDevice *cmdConfigDevice profile *cmdProfile } var cmdConfigDeviceGetUsage = u.Usage{u.Device, u.Key} func (c *cmdConfigDeviceGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", c.configDevice.formatUsage(cmdConfigDeviceGetUsage)...) cmd.Short = i18n.G("Get values for device configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Get values for device configuration keys`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { if c.config != nil { return c.global.cmpInstances(toComplete) } else if c.profile != nil { return c.global.cmpProfiles(toComplete, true) } } if len(args) == 1 { if c.config != nil { return c.global.cmpInstanceDeviceNames(args[0]) } else if c.profile != nil { return c.global.cmpProfileDeviceNames(args[0]) } } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigDeviceGet) run(cmd *cobra.Command, args []string) error { parsed, err := c.configDevice.formatUsage(cmdConfigDeviceGetUsage).Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer objectName := parsed[0].RemoteObject.String devName := parsed[1].String key := parsed[2].String if c.profile != nil { profile, _, err := d.GetProfile(objectName) if err != nil { return err } dev, ok := profile.Devices[devName] if !ok { return errors.New(i18n.G("Device doesn't exist")) } fmt.Println(dev[key]) } else { inst, _, err := d.GetInstance(objectName) if err != nil { return err } dev, ok := inst.Devices[devName] if !ok { _, ok = inst.ExpandedDevices[devName] if !ok { return errors.New(i18n.G("Device doesn't exist")) } return errors.New(i18n.G("Device from profile(s) cannot be retrieved for individual instance")) } fmt.Println(dev[key]) } return nil } // List. type cmdConfigDeviceList struct { global *cmdGlobal config *cmdConfig configDevice *cmdConfigDevice profile *cmdProfile } var cmdConfigDeviceListUsage = u.Usage{} func (c *cmdConfigDeviceList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", c.configDevice.formatUsage(cmdConfigDeviceListUsage)...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List instance devices") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`List instance devices`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { if c.config != nil { return c.global.cmpInstances(toComplete) } else if c.profile != nil { return c.global.cmpProfiles(toComplete, true) } } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigDeviceList) run(cmd *cobra.Command, args []string) error { parsed, err := c.configDevice.formatUsage(cmdConfigDeviceListUsage).Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer objectName := parsed[0].RemoteObject.String // List the devices var devices []string if c.profile != nil { profile, _, err := d.GetProfile(objectName) if err != nil { return err } for k := range profile.Devices { devices = append(devices, k) } } else { inst, _, err := d.GetInstance(objectName) if err != nil { return err } for k := range inst.Devices { devices = append(devices, k) } } fmt.Printf("%s\n", strings.Join(devices, "\n")) return nil } // Override. type cmdConfigDeviceOverride struct { global *cmdGlobal config *cmdConfig } var cmdConfigDeviceOverrideUsage = u.Usage{u.Instance.Remote(), u.Device, u.LegacyKV.List(0)} func (c *cmdConfigDeviceOverride) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("override", cmdConfigDeviceOverrideUsage...) cmd.Short = i18n.G("Copy profile inherited devices and override configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Copy profile inherited devices and override configuration keys`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigDeviceOverride) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigDeviceOverrideUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer objectName := parsed[0].RemoteObject.String devName := parsed[1].String // Override the device inst, etag, err := d.GetInstance(objectName) if err != nil { return err } _, ok := inst.Devices[devName] if ok { return errors.New(i18n.G("The device already exists")) } device, ok := inst.ExpandedDevices[devName] if !ok { return errors.New(i18n.G("The profile device doesn't exist")) } keys, err := kvToMap(parsed[2]) if err != nil { return err } maps.Copy(device, keys) inst.Devices[devName] = device op, err := d.UpdateInstance(objectName, inst.Writable(), etag) if err != nil { return err } err = op.Wait() if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Device %s overridden for %s")+"\n", devName, objectName) } return nil } // Remove. type cmdConfigDeviceRemove struct { global *cmdGlobal config *cmdConfig configDevice *cmdConfigDevice profile *cmdProfile } var cmdConfigDeviceRemoveUsage = u.Usage{u.Device.List(1)} func (c *cmdConfigDeviceRemove) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remove", c.configDevice.formatUsage(cmdConfigDeviceRemoveUsage)...) cmd.Aliases = []string{"delete", "rm"} cmd.Short = i18n.G("Remove instance devices") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Remove instance devices`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { if c.config != nil { return c.global.cmpInstances(toComplete) } else if c.profile != nil { return c.global.cmpProfiles(toComplete, true) } } if c.config != nil { return c.global.cmpInstanceDeviceNames(args[0]) } else if c.profile != nil { return c.global.cmpProfileDeviceNames(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigDeviceRemove) run(cmd *cobra.Command, args []string) error { parsed, err := c.configDevice.formatUsage(cmdConfigDeviceRemoveUsage).Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer objectName := parsed[0].RemoteObject.String // Remove the device if c.profile != nil { profile, etag, err := d.GetProfile(objectName) if err != nil { return err } var errs []error for _, p := range parsed[1].List { devName := p.String _, ok := profile.Devices[devName] if ok { delete(profile.Devices, devName) } else { errs = append(errs, fmt.Errorf(i18n.G("Device “%s” doesn't exist"), devName)) } } if len(errs) > 0 { return errors.Join(errs...) } err = d.UpdateProfile(objectName, profile.Writable(), etag) if err != nil { return err } } else { inst, etag, err := d.GetInstance(objectName) if err != nil { return err } var errs []error for _, p := range parsed[1].List { devName := p.String _, ok := inst.Devices[devName] if ok { delete(inst.Devices, devName) } else { _, ok := inst.ExpandedDevices[devName] if !ok { errs = append(errs, fmt.Errorf(i18n.G("Device “%s” doesn't exist"), devName)) } errs = append(errs, fmt.Errorf(i18n.G("Device from profile(s) cannot be removed from individual instance. Override device “%s” or modify profile instead"), devName)) } } if len(errs) > 0 { return errors.Join(errs...) } op, err := d.UpdateInstance(objectName, inst.Writable(), etag) if err != nil { return err } err = op.Wait() if err != nil { return err } } if !c.global.flagQuiet { fmt.Printf(i18n.G("Device %s removed from %s")+"\n", strings.Join(parsed[1].StringList, ", "), objectName) } return nil } // Set. type cmdConfigDeviceSet struct { global *cmdGlobal config *cmdConfig configDevice *cmdConfigDevice profile *cmdProfile } var cmdConfigDeviceSetUsage = u.Usage{u.Device, u.LegacyKV.List(1)} func (c *cmdConfigDeviceSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", c.configDevice.formatUsage(cmdConfigDeviceSetUsage)...) cmd.Short = i18n.G("Set device configuration keys") if c.config != nil { cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set device configuration keys For backward compatibility, a single configuration key may still be set with: incus config device set [:] `)) } else if c.profile != nil { cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set device configuration keys For backward compatibility, a single configuration key may still be set with: incus profile device set [:] `)) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { if c.config != nil { return c.global.cmpInstances(toComplete) } else if c.profile != nil { return c.global.cmpProfiles(toComplete, true) } } if len(args) == 1 { if c.config != nil { return c.global.cmpInstanceDeviceNames(args[0]) } else if c.profile != nil { return c.global.cmpProfileDeviceNames(args[0]) } } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdConfigDeviceSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer objectName := parsed[0].RemoteObject.String devName := parsed[1].String keys, err := kvToMap(parsed[2]) if err != nil { return err } if c.profile != nil { profile, etag, err := d.GetProfile(objectName) if err != nil { return err } dev, ok := profile.Devices[devName] if !ok { return errors.New(i18n.G("Device doesn't exist")) } maps.Copy(dev, keys) profile.Devices[devName] = dev err = d.UpdateProfile(objectName, profile.Writable(), etag) if err != nil { return err } } else { inst, etag, err := d.GetInstance(objectName) if err != nil { return err } dev, ok := inst.Devices[devName] if !ok { _, ok = inst.ExpandedDevices[devName] if !ok { return errors.New(i18n.G("Device doesn't exist")) } return errors.New(i18n.G("Device from profile(s) cannot be modified for individual instance. Override device or modify profile instead")) } maps.Copy(dev, keys) inst.Devices[devName] = dev op, err := d.UpdateInstance(objectName, inst.Writable(), etag) if err != nil { return err } err = op.Wait() if err != nil { return err } } return nil } func (c *cmdConfigDeviceSet) run(cmd *cobra.Command, args []string) error { parsed, err := c.configDevice.formatUsage(cmdConfigDeviceSetUsage).Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Show. type cmdConfigDeviceShow struct { global *cmdGlobal config *cmdConfig configDevice *cmdConfigDevice profile *cmdProfile } var cmdConfigDeviceShowUsage = u.Usage{} func (c *cmdConfigDeviceShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", c.configDevice.formatUsage(cmdConfigDeviceShowUsage)...) cmd.Short = i18n.G("Show full device configuration") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show full device configuration`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { if c.config != nil { return c.global.cmpInstances(toComplete) } else if c.profile != nil { return c.global.cmpProfiles(toComplete, true) } } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigDeviceShow) run(cmd *cobra.Command, args []string) error { parsed, err := c.configDevice.formatUsage(cmdConfigDeviceShowUsage).Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer objectName := parsed[0].RemoteObject.String // Show the devices var devices map[string]map[string]string if c.profile != nil { profile, _, err := d.GetProfile(objectName) if err != nil { return err } devices = profile.Devices } else { inst, _, err := d.GetInstance(objectName) if err != nil { return err } devices = inst.Devices } data, err := yaml.Dump(&devices, yaml.V2) if err != nil { return err } fmt.Print(string(data)) return nil } // Unset. type cmdConfigDeviceUnset struct { global *cmdGlobal config *cmdConfig configDevice *cmdConfigDevice configDeviceSet *cmdConfigDeviceSet profile *cmdProfile } var cmdConfigDeviceUnsetUsage = u.Usage{u.Device, u.Key} func (c *cmdConfigDeviceUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", c.configDevice.formatUsage(cmdConfigDeviceUnsetUsage)...) cmd.Short = i18n.G("Unset device configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Unset device configuration keys`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { if c.config != nil { return c.global.cmpInstances(toComplete) } else if c.profile != nil { return c.global.cmpProfiles(toComplete, true) } } if len(args) == 1 { if c.config != nil { return c.global.cmpInstanceDeviceNames(args[0]) } else if c.profile != nil { return c.global.cmpProfileDeviceNames(args[0]) } } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigDeviceUnset) run(cmd *cobra.Command, args []string) error { parsed, err := c.configDevice.formatUsage(cmdConfigDeviceUnsetUsage).Parse(c.global.conf, cmd, args) if err != nil { return err } return unsetKey(c.configDeviceSet, cmd, parsed) } incus-7.0.0/cmd/incus/config_metadata.go000066400000000000000000000123271517523235500201740ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "os" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" ) type cmdConfigMetadata struct { global *cmdGlobal config *cmdConfig } func (c *cmdConfigMetadata) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("metadata") cmd.Short = i18n.G("Manage instance metadata files") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage instance metadata files`)) // Edit configMetadataEditCmd := cmdConfigMetadataEdit{global: c.global, config: c.config, configMetadata: c} cmd.AddCommand(configMetadataEditCmd.command()) // Show configMetadataShowCmd := cmdConfigMetadataShow{global: c.global, config: c.config, configMetadata: c} cmd.AddCommand(configMetadataShowCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Edit. type cmdConfigMetadataEdit struct { global *cmdGlobal config *cmdConfig configMetadata *cmdConfigMetadata } var cmdConfigMetadataEditUsage = u.Usage{u.Instance.Remote()} func (c *cmdConfigMetadataEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdConfigMetadataEditUsage...) cmd.Short = i18n.G("Edit instance metadata files") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Edit instance metadata files`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigMetadataEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the instance metadata. ### Any line starting with a '# will be ignored. ### ### A sample configuration looks like: ### ### architecture: x86_64 ### creation_date: 1477146654 ### expiry_date: 0 ### properties: ### architecture: x86_64 ### description: BusyBox x86_64 ### name: busybox-x86_64 ### os: BusyBox ### templates: ### /template: ### when: ### - "" ### create_only: false ### template: template.tpl ### properties: {}`) } func (c *cmdConfigMetadataEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigMetadataEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String // Edit the metadata if !termios.IsTerminal(getStdinFd()) { metadata := api.ImageMetadata{} loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } err = loader.Load(&metadata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateInstanceMetadata(instanceName, metadata, "") } metadata, etag, err := d.GetInstanceMetadata(instanceName) if err != nil { return err } origContent, err := yaml.Dump(metadata, yaml.V2) if err != nil { return err } // Spawn the editor content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(origContent))) if err != nil { return err } for { metadata := api.ImageMetadata{} err = yaml.Load(content, &metadata) if err == nil { err = d.UpdateInstanceMetadata(instanceName, metadata, etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Show. type cmdConfigMetadataShow struct { global *cmdGlobal config *cmdConfig configMetadata *cmdConfigMetadata } var cmdConfigMetadataShowUsage = u.Usage{u.Instance.Remote()} func (c *cmdConfigMetadataShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdConfigMetadataShowUsage...) cmd.Short = i18n.G("Show instance metadata files") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show instance metadata files`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigMetadataShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigMetadataShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String // Show the instance metadata metadata, _, err := d.GetInstanceMetadata(instanceName) if err != nil { return err } content, err := yaml.Dump(metadata, yaml.V2) if err != nil { return err } fmt.Printf("%s", content) return nil } incus-7.0.0/cmd/incus/config_template.go000066400000000000000000000237101517523235500202250ustar00rootroot00000000000000package main import ( "bytes" "fmt" "io" "os" "sort" "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" ) type cmdConfigTemplate struct { global *cmdGlobal config *cmdConfig } func (c *cmdConfigTemplate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("template") cmd.Short = i18n.G("Manage instance file templates") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage instance file templates`)) // Create configTemplateCreateCmd := cmdConfigTemplateCreate{global: c.global, config: c.config, configTemplate: c} cmd.AddCommand(configTemplateCreateCmd.command()) // Delete configTemplateDeleteCmd := cmdConfigTemplateDelete{global: c.global, config: c.config, configTemplate: c} cmd.AddCommand(configTemplateDeleteCmd.command()) // Edit configTemplateEditCmd := cmdConfigTemplateEdit{global: c.global, config: c.config, configTemplate: c} cmd.AddCommand(configTemplateEditCmd.command()) // List configTemplateListCmd := cmdConfigTemplateList{global: c.global, config: c.config, configTemplate: c} cmd.AddCommand(configTemplateListCmd.command()) // Show configTemplateShowCmd := cmdConfigTemplateShow{global: c.global, config: c.config, configTemplate: c} cmd.AddCommand(configTemplateShowCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Create. type cmdConfigTemplateCreate struct { global *cmdGlobal config *cmdConfig configTemplate *cmdConfigTemplate } var cmdConfigTemplateCreateUsage = u.Usage{u.Instance.Remote(), u.NewName(u.Template)} func (c *cmdConfigTemplateCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdConfigTemplateCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create new instance file templates") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Create new instance file templates`)) cmd.Example = cli.FormatSection("", i18n.G(`incus config template create u1 t1 Create template t1 for instance u1 incus config template create u1 t1 < config.tpl Create template t1 for instance u1 from config.tpl`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigTemplateCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigTemplateCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } var stdinData io.ReadSeeker d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String templateName := parsed[1].String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { contents, err := io.ReadAll(os.Stdin) if err != nil { return err } // Reset the seek position stdinData = bytes.NewReader(contents) } // Create instance file template return d.CreateInstanceTemplateFile(instanceName, templateName, stdinData) } // Delete. type cmdConfigTemplateDelete struct { global *cmdGlobal config *cmdConfig configTemplate *cmdConfigTemplate } var cmdConfigTemplateDeleteUsage = u.Usage{u.Instance.Remote(), u.Template} func (c *cmdConfigTemplateDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdConfigTemplateDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete instance file templates") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete instance file templates`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } if len(args) == 1 { return c.global.cmpInstanceConfigTemplates(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigTemplateDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigTemplateDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String templateName := parsed[1].String // Delete instance file template return d.DeleteInstanceTemplateFile(instanceName, templateName) } // Edit. type cmdConfigTemplateEdit struct { global *cmdGlobal config *cmdConfig configTemplate *cmdConfigTemplate } var cmdConfigTemplateEditUsage = u.Usage{u.Instance.Remote(), u.Template} func (c *cmdConfigTemplateEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdConfigTemplateEditUsage...) cmd.Short = i18n.G("Edit instance file templates") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Edit instance file templates`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } if len(args) == 1 { return c.global.cmpInstanceConfigTemplates(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigTemplateEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigTemplateEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String templateName := parsed[1].String // Edit instance file template if !termios.IsTerminal(getStdinFd()) { return d.CreateInstanceTemplateFile(instanceName, templateName, os.Stdin) } reader, err := d.GetInstanceTemplateFile(instanceName, templateName) if err != nil { return err } content, err := io.ReadAll(reader) if err != nil { return err } // Spawn the editor content, err = cli.TextEditor("", content) if err != nil { return err } for { reader := bytes.NewReader(content) err := d.CreateInstanceTemplateFile(instanceName, templateName, reader) // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Error updating template file: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // List. type cmdConfigTemplateList struct { global *cmdGlobal config *cmdConfig configTemplate *cmdConfigTemplate flagFormat string } var cmdConfigTemplateListUsage = u.Usage{u.Instance.Remote()} func (c *cmdConfigTemplateList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdConfigTemplateListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List instance file templates") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`List instance file templates`)) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigTemplateList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigTemplateListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String // List the templates templates, err := d.GetInstanceTemplateFiles(instanceName) if err != nil { return err } // Render the table data := [][]string{} for _, template := range templates { data = append(data, []string{template}) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{ i18n.G("FILENAME"), } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, templates) } // Show. type cmdConfigTemplateShow struct { global *cmdGlobal config *cmdConfig configTemplate *cmdConfigTemplate } var cmdConfigTemplateShowUsage = u.Usage{u.Instance.Remote(), u.Template} func (c *cmdConfigTemplateShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdConfigTemplateShowUsage...) cmd.Short = i18n.G("Show content of instance file templates") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Show content of instance file templates`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } if len(args) == 1 { return c.global.cmpInstanceConfigTemplates(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdConfigTemplateShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigTemplateShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String templateName := parsed[1].String // Show the template template, err := d.GetInstanceTemplateFile(instanceName, templateName) if err != nil { return err } content, err := io.ReadAll(template) if err != nil { return err } fmt.Printf("%s", content) return nil } incus-7.0.0/cmd/incus/config_trust.go000066400000000000000000000552011517523235500175730ustar00rootroot00000000000000package main import ( "crypto/x509" "encoding/base64" "encoding/pem" "errors" "fmt" "io" "os" "path/filepath" "slices" "sort" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) type cmdConfigTrust struct { global *cmdGlobal config *cmdConfig } func (c *cmdConfigTrust) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("trust") cmd.Short = i18n.G("Manage trusted clients") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage trusted clients`)) // Add configTrustAddCmd := cmdConfigTrustAdd{global: c.global, config: c.config, configTrust: c} cmd.AddCommand(configTrustAddCmd.command()) // Add certificate configTrustAddCertificateCmd := cmdConfigTrustAddCertificate{global: c.global, config: c.config, configTrust: c} cmd.AddCommand(configTrustAddCertificateCmd.command()) // Edit configTrustEditCmd := cmdConfigTrustEdit{global: c.global, config: c.config, configTrust: c} cmd.AddCommand(configTrustEditCmd.command()) // List configTrustListCmd := cmdConfigTrustList{global: c.global, config: c.config, configTrust: c} cmd.AddCommand(configTrustListCmd.command()) // List tokens configTrustListTokensCmd := cmdConfigTrustListTokens{global: c.global, config: c.config, configTrust: c} cmd.AddCommand(configTrustListTokensCmd.command()) // Remove configTrustRemoveCmd := cmdConfigTrustRemove{global: c.global, config: c.config, configTrust: c} cmd.AddCommand(configTrustRemoveCmd.command()) // Revoke token configTrustRevokeTokenCmd := cmdConfigTrustRevokeToken{global: c.global, config: c.config, configTrust: c} cmd.AddCommand(configTrustRevokeTokenCmd.command()) // Show configTrustShowCmd := cmdConfigTrustShow{global: c.global, config: c.config, configTrust: c} cmd.AddCommand(configTrustShowCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Add. type cmdConfigTrustAdd struct { global *cmdGlobal config *cmdConfig configTrust *cmdConfigTrust flagProjects string flagRestricted bool } var cmdConfigTrustAddUsage = u.Usage{u.NewName(u.Client).Remote()} func (c *cmdConfigTrustAdd) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("add", cmdConfigTrustAddUsage...) cmd.Short = i18n.G("Add new trusted client") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Add new trusted client This will issue a trust token to be used by the client to add itself to the trust store. `)) cli.AddBoolFlag(cmd.Flags(), &c.flagRestricted, "restricted", i18n.G("Restrict the certificate to one or more projects")) cli.AddStringFlag(cmd.Flags(), &c.flagProjects, "projects", "", "", i18n.G("List of projects to restrict the certificate to")) cmd.RunE = c.run return cmd } func (c *cmdConfigTrustAdd) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigTrustAddUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer clientName := parsed[0].RemoteObject.String // Prepare the request. cert := api.CertificatesPost{} cert.Token = true cert.Name = clientName cert.Type = api.CertificateTypeClient cert.Restricted = c.flagRestricted if c.flagProjects != "" { cert.Projects = strings.Split(c.flagProjects, ",") } // Create the token. op, err := d.CreateCertificateToken(cert) if err != nil { return err } opAPI := op.Get() certificateToken, err := opAPI.ToCertificateAddToken() if err != nil { return fmt.Errorf(i18n.G("Failed converting token operation to certificate add token: %w"), err) } if !c.global.flagQuiet { fmt.Printf(i18n.G("Client %s certificate add token:")+"\n", cert.Name) } fmt.Println(certificateToken.String()) return nil } // Add certificate. type cmdConfigTrustAddCertificate struct { global *cmdGlobal config *cmdConfig configTrust *cmdConfigTrust flagProjects string flagRestricted bool flagName string flagType string flagDescription string } var cmdConfigTrustAddCertificateUsage = u.Usage{u.RemoteColonOpt, u.Placeholder(i18n.G("cert.crt"))} func (c *cmdConfigTrustAddCertificate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("add-certificate", cmdConfigTrustAddCertificateUsage...) cmd.Short = i18n.G("Add new trusted client certificate") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Add new trusted client certificate The following certificate types are supported: - client (default) - metrics `)) cli.AddBoolFlag(cmd.Flags(), &c.flagRestricted, "restricted", i18n.G("Restrict the certificate to one or more projects")) cli.AddStringFlag(cmd.Flags(), &c.flagProjects, "projects", "", "", i18n.G("List of projects to restrict the certificate to")) cli.AddStringFlag(cmd.Flags(), &c.flagName, "name", "", "", i18n.G("Alternative certificate name")) cli.AddStringFlag(cmd.Flags(), &c.flagType, "type|t", "client", "", i18n.G("Type of certificate")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Certificate description")) cmd.RunE = c.run return cmd } func (c *cmdConfigTrustAddCertificate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigTrustAddCertificateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer certPath := parsed[1].String // Validate flags. if !slices.Contains([]string{"client", "metrics"}, c.flagType) { return fmt.Errorf(i18n.G("Unknown certificate type %q"), c.flagType) } if certPath == "-" { certPath = "/dev/stdin" } // Check that the path exists. if !util.PathExists(certPath) { return fmt.Errorf(i18n.G("Provided certificate path doesn't exist: %s"), certPath) } // Validate server support for metrics. if c.flagType == "metrics" && !d.HasExtension("metrics") { return errors.New("The server doesn't implement metrics") } // Load the certificate. x509Cert, err := localtls.ReadCert(certPath) if err != nil { return err } var name string if c.flagName != "" { name = c.flagName } else { name = filepath.Base(certPath) } // Add trust relationship. cert := api.CertificatesPost{} cert.Certificate = base64.StdEncoding.EncodeToString(x509Cert.Raw) cert.Name = name cert.Description = c.flagDescription switch c.flagType { case "client": cert.Type = api.CertificateTypeClient case "metrics": cert.Type = api.CertificateTypeMetrics } cert.Restricted = c.flagRestricted if c.flagProjects != "" { cert.Projects = strings.Split(c.flagProjects, ",") } return d.CreateCertificate(cert) } // Edit. type cmdConfigTrustEdit struct { global *cmdGlobal config *cmdConfig configTrust *cmdConfigTrust } var cmdConfigTrustEditUsage = u.Usage{u.Fingerprint.Remote()} func (c *cmdConfigTrustEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdConfigTrustEditUsage...) cmd.Short = i18n.G("Edit trust configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Edit trust configurations as YAML`)) cmd.RunE = c.run return cmd } func (c *cmdConfigTrustEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the certificate. ### Any line starting with a '# will be ignored. ### ### Note that the fingerprint is shown but cannot be changed`) } func (c *cmdConfigTrustEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigTrustEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer fingerprint := parsed[0].RemoteObject.String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } newdata := api.CertificatePut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateCertificate(fingerprint, newdata, "") } // Extract the current value cert, etag, err := d.GetCertificate(fingerprint) if err != nil { return err } data, err := yaml.Dump(&cert, yaml.V2) if err != nil { return err } // Spawn the editor content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor newdata := api.CertificatePut{} err = yaml.Load(content, &newdata) if err == nil { err = d.UpdateCertificate(fingerprint, newdata, etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // List. type cmdConfigTrustList struct { global *cmdGlobal config *cmdConfig configTrust *cmdConfigTrust flagFormat string flagColumns string } type certificateColumn struct { Name string Data func(rowData rowData) string } type rowData struct { Cert api.Certificate TLSCert *x509.Certificate } var cmdConfigTrustListUsage = u.Usage{u.RemoteColonOpt, u.Filter.List(0)} func (c *cmdConfigTrustList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdConfigTrustListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List trusted clients") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List trusted clients The -c option takes a (optionally comma-separated) list of arguments that control which certificate attributes to output when displaying in table or csv format. Default column layout is: ntdfe Column shorthand chars: n - Name t - Type c - Common Name f - Fingerprint d - Description i - Issue date e - Expiry date r - Whether certificate is restricted p - Newline-separated list of projects`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", "ntdfe", "", i18n.G("Columns")) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run return cmd } func (c *cmdConfigTrustList) parseColumns() ([]certificateColumn, error) { columnsShorthandMap := map[rune]certificateColumn{ 'n': {i18n.G("NAME"), c.nameColumnData}, 't': {i18n.G("TYPE"), c.typeColumnData}, 'c': {i18n.G("COMMON NAME"), c.commonNameColumnData}, 'f': {i18n.G("FINGERPRINT"), c.fingerprintColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 'i': {i18n.G("ISSUE DATE"), c.issueDateColumnData}, 'e': {i18n.G("EXPIRY DATE"), c.expiryDateColumnData}, 'r': {i18n.G("RESTRICTED"), c.restrictedColumnData}, 'p': {i18n.G("PROJECTS"), c.projectColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []certificateColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdConfigTrustList) typeColumnData(rowData rowData) string { return rowData.Cert.Type } func (c *cmdConfigTrustList) nameColumnData(rowData rowData) string { return rowData.Cert.Name } func (c *cmdConfigTrustList) commonNameColumnData(rowData rowData) string { return rowData.TLSCert.Subject.CommonName } func (c *cmdConfigTrustList) fingerprintColumnData(rowData rowData) string { return rowData.Cert.Fingerprint[0:12] } func (c *cmdConfigTrustList) descriptionColumnData(rowData rowData) string { return rowData.Cert.Description } func (c *cmdConfigTrustList) issueDateColumnData(rowData rowData) string { return rowData.TLSCert.NotBefore.Local().Format(dateLayout) } func (c *cmdConfigTrustList) expiryDateColumnData(rowData rowData) string { return rowData.TLSCert.NotAfter.Local().Format(dateLayout) } func (c *cmdConfigTrustList) restrictedColumnData(rowData rowData) string { if rowData.Cert.Restricted { return i18n.G("yes") } return i18n.G("no") } func (c *cmdConfigTrustList) projectColumnData(rowData rowData) string { projects := []string{} projects = append(projects, rowData.Cert.Projects...) sort.Strings(projects) return strings.Join(projects, "\n") } func (c *cmdConfigTrustList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigTrustListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer filters := parsed[1].StringList // Process the columns columns, err := c.parseColumns() if err != nil { return err } filters = prepareCertificatesFilters(filters) // List trust relationships trust, err := d.GetCertificatesWithFilter(filters) if err != nil { return err } data := [][]string{} for _, cert := range trust { certBlock, _ := pem.Decode([]byte(cert.Certificate)) if certBlock == nil { return errors.New(i18n.G("Invalid certificate")) } tlsCert, err := x509.ParseCertificate(certBlock.Bytes) if err != nil { return err } rowData := rowData{cert, tlsCert} row := []string{} for _, column := range columns { row = append(row, column.Data(rowData)) } data = append(data, row) } sort.Sort(cli.StringList(data)) headers := []string{} for _, column := range columns { headers = append(headers, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, headers, data, trust) } // List tokens. type cmdConfigTrustListTokens struct { global *cmdGlobal config *cmdConfig configTrust *cmdConfigTrust flagFormat string flagColumns string } type configTrustListTokenColumn struct { Name string Data func(*api.CertificateAddToken) string } var cmdConfigTrustListTokensUsage = u.Usage{u.RemoteColonOpt} func (c *cmdConfigTrustListTokens) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list-tokens", cmdConfigTrustListTokensUsage...) cmd.Short = i18n.G("List all active certificate add tokens") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List all active certificate add tokens Default column layout: ntE == Columns == The -c option takes a comma separated list of arguments that control which network zone attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: n - Name t - Token E - Expires At`)) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultConfigTrustListTokenColumns, "", i18n.G("Columns")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run return cmd } const defaultConfigTrustListTokenColumns = "ntE" func (c *cmdConfigTrustListTokens) parseColumns() ([]configTrustListTokenColumn, error) { columnsShorthandMap := map[rune]configTrustListTokenColumn{ 'n': {i18n.G("NAME"), c.clientNameColumnData}, 't': {i18n.G("TOKEN"), c.tokenColumnData}, 'E': {i18n.G("EXPIRES AT"), c.expiresAtColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []configTrustListTokenColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdConfigTrustListTokens) clientNameColumnData(token *api.CertificateAddToken) string { return token.ClientName } func (c *cmdConfigTrustListTokens) tokenColumnData(token *api.CertificateAddToken) string { return token.String() } func (c *cmdConfigTrustListTokens) expiresAtColumnData(token *api.CertificateAddToken) string { if token.ExpiresAt.IsZero() { return " " } return token.ExpiresAt.Local().Format(dateLayout) } func (c *cmdConfigTrustListTokens) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigTrustListTokensUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer // Get the certificate add tokens. Use default project as join tokens are created in default project. ops, err := d.UseProject(api.ProjectDefaultName).GetOperations() if err != nil { return err } data := [][]string{} joinTokens := []*api.CertificateAddToken{} // Parse column flags. columns, err := c.parseColumns() if err != nil { return err } for _, op := range ops { if op.Class != api.OperationClassToken { continue } if op.StatusCode != api.Running { continue // Tokens are single use, so if cancelled but not deleted yet its not available. } joinToken, err := op.ToCertificateAddToken() if err != nil { continue // Operation is not a valid certificate add token operation. } line := []string{} for _, column := range columns { line = append(line, column.Data(joinToken)) } joinTokens = append(joinTokens, joinToken) data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, joinTokens) } // Remove. type cmdConfigTrustRemove struct { global *cmdGlobal config *cmdConfig configTrust *cmdConfigTrust } var cmdConfigTrustRemoveUsage = u.Usage{u.Fingerprint.Remote()} func (c *cmdConfigTrustRemove) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remove", cmdConfigTrustRemoveUsage...) cmd.Aliases = []string{"delete", "rm"} cmd.Short = i18n.G("Remove trusted client") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Remove trusted client`)) cmd.RunE = c.run return cmd } func (c *cmdConfigTrustRemove) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigTrustRemoveUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer fingerprint := parsed[0].RemoteObject.String // Remove trust relationship return d.DeleteCertificate(fingerprint) } // List tokens. type cmdConfigTrustRevokeToken struct { global *cmdGlobal config *cmdConfig configTrust *cmdConfigTrust } var cmdConfigTrustRevokeTokenUsage = u.Usage{u.Token.Remote()} func (c *cmdConfigTrustRevokeToken) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("revoke-token", cmdConfigTrustRevokeTokenUsage...) cmd.Short = i18n.G("Revoke certificate add token") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Revoke certificate add token`)) cmd.RunE = c.run return cmd } func (c *cmdConfigTrustRevokeToken) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigTrustRevokeTokenUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer remoteName := parsed[0].RemoteName token := parsed[0].RemoteObject.String // Get the certificate add tokens. Use default project as certificate add tokens are created in default project. ops, err := d.UseProject(api.ProjectDefaultName).GetOperations() if err != nil { return err } for _, op := range ops { if op.Class != api.OperationClassToken { continue } if op.StatusCode != api.Running { continue // Tokens are single use, so if cancelled but not deleted yet its not available. } joinToken, err := op.ToCertificateAddToken() if err != nil { continue // Operation is not a valid certificate add token operation. } if joinToken.ClientName == token { // Delete the operation err = d.DeleteOperation(op.ID) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Certificate add token for %s deleted")+"\n", token) } return nil } } return fmt.Errorf(i18n.G("No certificate add token for member %s on remote: %s"), token, remoteName) } // Show. type cmdConfigTrustShow struct { global *cmdGlobal config *cmdConfig configTrust *cmdConfigTrust } var cmdConfigTrustShowUsage = u.Usage{u.Fingerprint.Remote()} func (c *cmdConfigTrustShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdConfigTrustShowUsage...) cmd.Short = i18n.G("Show trust configurations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show trust configurations`)) cmd.RunE = c.run return cmd } func (c *cmdConfigTrustShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConfigTrustShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer fingerprint := parsed[0].RemoteObject.String // Show the certificate configuration cert, _, err := d.GetCertificate(fingerprint) if err != nil { return err } data, err := yaml.Dump(&cert, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // prepareCertificatesFilters processes and formats filter criteria // for storage buckets, ensuring they are in a format that the server can interpret. func prepareCertificatesFilters(filters []string) []string { formatedFilters := []string{} for _, filter := range filters { membs := strings.SplitN(filter, "=", 2) key := membs[0] if len(membs) == 1 { regexpValue := key if !strings.Contains(key, "^") && !strings.Contains(key, "$") { regexpValue = "^" + regexpValue + "$" } filter = fmt.Sprintf("name=(%s|^%s.*)", regexpValue, key) } formatedFilters = append(formatedFilters, filter) } return formatedFilters } incus-7.0.0/cmd/incus/console.go000066400000000000000000000227211517523235500165300ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "net" "os" "os/exec" "runtime" "slices" "strconv" "strings" "sync" "github.com/gorilla/websocket" "github.com/kballard/go-shellquote" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/termios" "github.com/lxc/incus/v7/shared/util" ) type cmdConsole struct { global *cmdGlobal flagForce bool flagShowLog bool flagType string } var cmdConsoleUsage = u.Usage{u.Instance.Remote()} func (c *cmdConsole) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("console", cmdConsoleUsage...) cmd.Short = i18n.G("Attach to instance consoles") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Attach to instance consoles This command allows you to interact with the boot console of an instance as well as retrieve past log entries from it.`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G("Forces a connection to the console, even if there is already an active session")) cli.AddBoolFlag(cmd.Flags(), &c.flagShowLog, "show-log", i18n.G("Retrieve the instance's console log")) cli.AddStringFlag(cmd.Flags(), &c.flagType, "type|t", c.global.defaultConsoleType(), "", i18n.G("Type of connection to establish: 'console' for serial console, 'vga' for SPICE graphical output")) cmd.ValidArgsFunction = func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpInstances(toComplete) } return cmd } func (c *cmdConsole) sendTermSize(control *websocket.Conn) error { width, height, err := termios.GetSize(int(os.Stdout.Fd())) if err != nil { return err } logger.Debugf("Window size is now: %dx%d", width, height) msg := api.InstanceExecControl{} msg.Command = "window-resize" msg.Args = make(map[string]string) msg.Args["width"] = strconv.Itoa(width) msg.Args["height"] = strconv.Itoa(height) return control.WriteJSON(msg) } type readWriteCloser struct { io.Reader io.WriteCloser } type stdinMirror struct { r io.Reader consoleDisconnect chan struct{} foundEscape *bool } // The pty has been switched to raw mode so we will only ever read a single // byte. The buffer size is therefore uninteresting to us. func (er stdinMirror) Read(p []byte) (int, error) { n, err := er.r.Read(p) v := rune(p[0]) if v == '\u0001' && !*er.foundEscape { *er.foundEscape = true return 0, err } if v == 'q' && *er.foundEscape { close(er.consoleDisconnect) return 0, err } *er.foundEscape = false return n, err } func (c *cmdConsole) run(cmd *cobra.Command, args []string) error { parsed, err := cmdConsoleUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String // Validate flags. if !slices.Contains([]string{"console", "vga"}, c.flagType) { return fmt.Errorf(i18n.G("Unknown output type %q"), c.flagType) } return c.console(d, instanceName) } func (c *cmdConsole) console(d incus.InstanceServer, name string) error { // Show the current log if requested. if c.flagShowLog { if c.flagType != "console" { return errors.New(i18n.G("The --show-log flag is only supported for by 'console' output type")) } console := &incus.InstanceConsoleLogArgs{} log, err := d.GetInstanceConsoleLog(name, console) if err != nil { return err } content, err := io.ReadAll(log) if err != nil { return err } fmt.Println(string(content)) return nil } // Handle running consoles. if c.flagType == "" { c.flagType = "console" } switch c.flagType { case "console": return c.text(d, name) case "vga": return c.vga(d, name) } return fmt.Errorf(i18n.G("Unknown console type %q"), c.flagType) } func (c *cmdConsole) text(d incus.InstanceServer, name string) error { // Configure the terminal cfd := int(os.Stdin.Fd()) oldTTYstate, err := termios.MakeRaw(cfd) if err != nil { return err } defer func() { _ = termios.Restore(cfd, oldTTYstate) }() handler := c.controlSocketHandler var width, height int width, height, err = termios.GetSize(int(os.Stdin.Fd())) if err != nil { return err } // Prepare the remote console req := api.InstanceConsolePost{ Width: width, Height: height, Type: "console", Force: c.flagForce, } consoleDisconnect := make(chan bool) manualDisconnect := make(chan struct{}) sendDisconnect := make(chan struct{}) defer close(sendDisconnect) consoleArgs := incus.InstanceConsoleArgs{ Terminal: &readWriteCloser{stdinMirror{ os.Stdin, manualDisconnect, new(bool), }, os.Stdout}, Control: handler, ConsoleDisconnect: consoleDisconnect, } go func() { select { case <-sendDisconnect: case <-manualDisconnect: } close(consoleDisconnect) // Make sure we leave the user back to a clean prompt. fmt.Print("\r\n") }() // Attach to the instance console op, err := d.ConsoleInstance(name, req, &consoleArgs) if err != nil { return err } fmt.Print(i18n.G("To detach from the console, press: +a q") + "\n\r") // Wait for the operation to complete err = op.Wait() if err != nil { return err } return nil } func (c *cmdConsole) vga(d incus.InstanceServer, name string) error { var err error conf := c.global.conf // We currently use the control websocket just to abort in case of errors. controlDone := make(chan struct{}, 1) handler := func(control *websocket.Conn) { <-controlDone closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") _ = control.WriteMessage(websocket.CloseMessage, closeMsg) } // Prepare the remote console. req := api.InstanceConsolePost{ Type: "vga", Force: c.flagForce, } chDisconnect := make(chan bool) chViewer := make(chan struct{}) consoleArgs := incus.InstanceConsoleArgs{ Control: handler, ConsoleDisconnect: chDisconnect, } // Setup local socket. var socket string var listener net.Listener if runtime.GOOS != "windows" { // Create a temporary unix socket mirroring the instance's spice socket. if !util.PathExists(conf.ConfigPath("sockets")) { err := os.MkdirAll(conf.ConfigPath("sockets"), 0o700) if err != nil { return err } } // Generate a random file name. path, err := os.CreateTemp(conf.ConfigPath("sockets"), "*.spice") if err != nil { return err } _ = path.Close() err = os.Remove(path.Name()) if err != nil { return err } // Listen on the socket. listener, err = net.Listen("unix", path.Name()) if err != nil { return err } defer func() { _ = os.Remove(path.Name()) }() socket = fmt.Sprintf("spice+unix://%s", path.Name()) } else { listener, err = net.Listen("tcp", "127.0.0.1:0") if err != nil { return err } addr, ok := listener.Addr().(*net.TCPAddr) if !ok { return errors.New("Bad TCP listener") } socket = fmt.Sprintf("spice://127.0.0.1:%d", addr.Port) } // Clean everything up when the viewer is done. go func() { <-chViewer _ = listener.Close() close(chDisconnect) }() // Spawn the remote console. op, connect, err := d.ConsoleInstanceDynamic(name, req, &consoleArgs) if err != nil { close(chViewer) return err } // Handle connections to the socket. wgConnections := sync.WaitGroup{} chConnected := make(chan struct{}) go func() { hasConnected := false for { conn, err := listener.Accept() if err != nil { return } if !hasConnected { hasConnected = true close(chConnected) } wgConnections.Add(1) go func(conn io.ReadWriteCloser) { defer wgConnections.Done() err = connect(conn) if err != nil { return } }(conn) } }() // Get the preferred SPICE command. preferredSpiceCmd := c.global.defaultConsoleSpiceCommand() // Use either spicy or remote-viewer if available. remoteViewer := c.findCommand("remote-viewer") spicy := c.findCommand("spicy") if preferredSpiceCmd != "" || remoteViewer != "" || spicy != "" { var cmd *exec.Cmd if preferredSpiceCmd != "" { // preferredSpiceCmd takes a string where the SOCKET keyword is replaced with the path to the SPICE socket. cmdSlice, err := shellquote.Split(strings.ReplaceAll(preferredSpiceCmd, "SOCKET", socket)) if err != nil { return err } cmd = exec.Command(cmdSlice[0], cmdSlice[1:]...) } else if remoteViewer != "" { cmd = exec.Command(remoteViewer, socket) } else { cmd = exec.Command(spicy, fmt.Sprintf("--uri=%s", socket)) } // Start the command. cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr err := cmd.Start() if err != nil { return fmt.Errorf(i18n.G("Failed starting command: %w"), err) } // Handle the command exiting. go func() { _ = cmd.Wait() close(chViewer) }() // Kill the viewer on remote disconnection. go func() { <-chConnected wgConnections.Wait() if cmd.Process == nil { return } _ = cmd.Process.Kill() }() } else { fmt.Println(i18n.G("The client automatically uses either spicy or remote-viewer when present.")) fmt.Println(i18n.G("As neither could be found, the raw SPICE socket can be found at:")) fmt.Printf(" %s\n", socket) // Wait for all connections to complete. <-chConnected wgConnections.Wait() close(chViewer) } // Wait for the operation to complete. err = op.Wait() if err != nil { return err } return nil } incus-7.0.0/cmd/incus/console_unix.go000066400000000000000000000013771517523235500175770ustar00rootroot00000000000000//go:build !windows package main import ( "os" "os/exec" "os/signal" "github.com/gorilla/websocket" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/shared/logger" ) func (c *cmdConsole) controlSocketHandler(control *websocket.Conn) { ch := make(chan os.Signal, 10) signal.Notify(ch, unix.SIGWINCH) for { sig := <-ch logger.Debugf("Received '%s signal', updating window geometry.", sig) err := c.sendTermSize(control) if err != nil { logger.Debugf("error setting term size %s", err) break } } closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") _ = control.WriteMessage(websocket.CloseMessage, closeMsg) } func (c *cmdConsole) findCommand(name string) string { path, _ := exec.LookPath(name) return path } incus-7.0.0/cmd/incus/console_windows.go000066400000000000000000000020201517523235500202700ustar00rootroot00000000000000//go:build windows package main import ( "fmt" "os" "os/exec" "path/filepath" "strings" "github.com/gorilla/websocket" "github.com/lxc/incus/v7/internal/i18n" ) func (c *cmdConsole) getTERM() (string, bool) { return "dumb", true } func (c *cmdConsole) controlSocketHandler(control *websocket.Conn) { // TODO: figure out what the equivalent of signal.SIGWINCH is on // windows and use that; for now if you resize your terminal it just // won't work quite correctly. err := c.sendTermSize(control) if err != nil { fmt.Printf(i18n.G("Error setting term size %s")+"\n", err) } } func (c *cmdConsole) findCommand(name string) string { path, _ := exec.LookPath(name) if path == "" { // Let's see if it's not in the usual location. programs, err := os.ReadDir("\\Program Files") if err != nil { return "" } for _, entry := range programs { if strings.HasPrefix(entry.Name(), "VirtViewer") { return filepath.Join("\\Program Files", entry.Name(), "bin", "remote-viewer.exe") } } } return path } incus-7.0.0/cmd/incus/copy.go000066400000000000000000000311461517523235500160410ustar00rootroot00000000000000package main import ( "errors" "fmt" "maps" "strings" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdCopy struct { global *cmdGlobal flagNoProfiles bool flagProfile []string flagConfig []string flagDevice []string flagEphemeral bool flagInstanceOnly bool flagMode string flagStateless bool flagStorage string flagTarget string flagTargetProject string flagRefresh bool flagRefreshExcludeOlder bool flagAllowInconsistent bool } var cmdCopyUsage = u.Usage{u.MakePath(u.Instance, u.Snapshot.Optional()).Remote(), u.NewName(u.Instance).Optional().Remote()} func (c *cmdCopy) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("copy", cmdCopyUsage...) cmd.Aliases = []string{"cp"} cmd.Short = i18n.G("Copy instances within or in between servers") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Copy instances within or in between servers Transfer modes (--mode): - pull: Target server pulls the data from the source server (source must listen on network) - push: Source server pushes the data to the target server (target must listen on network) - relay: The CLI connects to both source and server and proxies the data (both source and target must listen on network) The pull transfer mode is the default as it is compatible with all server versions. `)) cmd.RunE = c.run cli.AddStringArrayFlag(cmd.Flags(), &c.flagConfig, "config|c", i18n.G("Config key/value to apply to the new instance")) cli.AddStringArrayFlag(cmd.Flags(), &c.flagDevice, "device|d", i18n.G("New key/value to apply to a specific device")) cli.AddStringArrayFlag(cmd.Flags(), &c.flagProfile, "profile|p", i18n.G("Profile to apply to the new instance")) cli.AddBoolFlag(cmd.Flags(), &c.flagEphemeral, "ephemeral|e", i18n.G("Ephemeral instance")) cli.AddStringFlag(cmd.Flags(), &c.flagMode, "mode", "pull", "", i18n.G("Transfer mode. One of pull, push or relay")) cli.AddBoolFlag(cmd.Flags(), &c.flagInstanceOnly, "instance-only", i18n.G("Copy the instance without its snapshots")) cli.AddBoolFlag(cmd.Flags(), &c.flagStateless, "stateless", i18n.G("Copy a stateful instance stateless")) cli.AddStringFlag(cmd.Flags(), &c.flagStorage, "storage|s", "", "", i18n.G("Storage pool name")) cli.AddStringFlag(cmd.Flags(), &c.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.flagTargetProject, "target-project", "", "", i18n.G("Copy to a project different from the source")) cli.AddBoolFlag(cmd.Flags(), &c.flagNoProfiles, "no-profiles", i18n.G("Create the instance with no profiles applied")) cli.AddBoolFlag(cmd.Flags(), &c.flagRefresh, "refresh", i18n.G("Perform an incremental copy")) cli.AddBoolFlag(cmd.Flags(), &c.flagRefreshExcludeOlder, "refresh-exclude-older", i18n.G("During incremental copy, exclude source snapshots earlier than latest target snapshot")) cli.AddBoolFlag(cmd.Flags(), &c.flagAllowInconsistent, "allow-inconsistent", i18n.G("Ignore copy errors for volatile files")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } if len(args) == 1 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // copyOrMove runs the post-parsing command logic. func (c *cmdCopy) copyOrMove(cmd *cobra.Command, src *u.Parsed, dst *u.Parsed, keepVolatile bool, ephemeral int, stateful bool, instanceOnly bool, mode string, pool string, move bool) error { srcServer := src.RemoteServer srcInstanceName := src.RemoteObject.String // This function can be called from both the `copy` and `move` commands. As their first arguments // have a different grammar, additional care is taken here to normalize them. srcIsSnapshot := false srcSnapName := "" if cmd.Name() == "copy" { srcInstanceName = src.RemoteObject.List[0].String srcIsSnapshot = !src.RemoteObject.List[1].Skipped srcSnapName = src.RemoteObject.List[1].String } dstServer := dst.RemoteServer hasDstInstance := !dst.RemoteObject.Skipped dstInstanceName := dst.RemoteObject.String // Don't allow refreshing without profiles. if c.flagRefresh && c.flagNoProfiles { return errors.New(i18n.G("--no-profiles cannot be used with --refresh")) } // If the instance is being copied to a different remote and no destination name is // specified, use the source name. if !hasDstInstance { if srcServer == dstServer && c.flagTarget == "" { return errors.New(i18n.G("You must specify a destination instance name")) } dstInstanceName = srcInstanceName } // Project copies if c.flagTargetProject != "" { dstServer = dstServer.UseProject(c.flagTargetProject) } // Confirm that --target is only used with a cluster if c.flagTarget != "" && !dstServer.IsClustered() { return errors.New(i18n.G("To use --target, the destination remote must be a cluster")) } // Parse the config overrides configMap := map[string]string{} for _, entry := range c.flagConfig { key, value, found := strings.Cut(entry, "=") if !found { return fmt.Errorf(i18n.G("Bad key=value pair: %q"), entry) } configMap[key] = value } deviceMap, err := parseDeviceOverrides(c.flagDevice) if err != nil { return err } var op incus.RemoteOperation var writable api.InstancePut var start bool if srcIsSnapshot { if instanceOnly { return errors.New(i18n.G("--instance-only can't be passed when the source is a snapshot")) } // Prepare the instance creation request args := incus.InstanceSnapshotCopyArgs{ Name: dstInstanceName, Mode: mode, Live: stateful, } if c.flagRefresh { return errors.New(i18n.G("--refresh can only be used with instances")) } // Copy of a snapshot into a new instance entry, _, err := srcServer.GetInstanceSnapshot(srcInstanceName, srcSnapName) if err != nil { return err } // Overwrite profiles. if c.flagProfile != nil { entry.Profiles = c.flagProfile } else if c.flagNoProfiles { entry.Profiles = []string{} } // Allow setting additional config keys maps.Copy(entry.Config, configMap) // Allow setting device overrides for k, m := range deviceMap { if entry.Devices[k] == nil { entry.Devices[k] = m continue } if m["type"] == "none" { // When overriding with "none" type, clear the entire device. entry.Devices[k] = map[string]string{"type": "none"} continue } maps.Copy(entry.Devices[k], m) } // Allow overriding the ephemeral status switch ephemeral { case 1: entry.Ephemeral = true case 0: entry.Ephemeral = false } rootDiskDeviceKey, _, _ := instance.GetRootDiskDevice(entry.Devices) if rootDiskDeviceKey != "" && pool != "" { entry.Devices[rootDiskDeviceKey]["pool"] = pool } else if pool != "" { entry.Devices["root"] = map[string]string{ "type": "disk", "path": "/", "pool": pool, } } if entry.Config != nil { // Strip the last_state.power key in all cases delete(entry.Config, "volatile.last_state.power") if !keepVolatile { for k := range entry.Config { if !instance.InstanceIncludeWhenCopying(k, true) { delete(entry.Config, k) } } } } // Do the actual copy if c.flagTarget != "" { dstServer = dstServer.UseTarget(c.flagTarget) } op, err = dstServer.CopyInstanceSnapshot(srcServer, srcInstanceName, *entry, &args) if err != nil { return err } } else { // Prepare the instance creation request args := incus.InstanceCopyArgs{ Name: dstInstanceName, Live: stateful, InstanceOnly: instanceOnly, Mode: mode, Refresh: c.flagRefresh, RefreshExcludeOlder: c.flagRefreshExcludeOlder, AllowInconsistent: c.flagAllowInconsistent, } // Copy of an instance into a new instance entry, _, err := srcServer.GetInstance(srcInstanceName) if err != nil { return err } // Only start the instance back up if doing a stateless migration. // It's the server's job to start things back up when receiving a stateful migration. if entry.StatusCode == api.Running && move && !stateful { start = true } // Overwrite profiles. if c.flagProfile != nil { entry.Profiles = c.flagProfile } else if c.flagNoProfiles { entry.Profiles = []string{} } // Allow setting additional config keys maps.Copy(entry.Config, configMap) // Allow setting device overrides for k, m := range deviceMap { if entry.Devices[k] == nil { entry.Devices[k] = m continue } if m["type"] == "none" { // When overriding with "none" type, clear the entire device. entry.Devices[k] = map[string]string{"type": "none"} continue } maps.Copy(entry.Devices[k], m) } // Allow overriding the ephemeral status switch ephemeral { case 1: entry.Ephemeral = true case 0: entry.Ephemeral = false } rootDiskDeviceKey, _, _ := instance.GetRootDiskDevice(entry.Devices) if rootDiskDeviceKey != "" && pool != "" { entry.Devices[rootDiskDeviceKey]["pool"] = pool } else if pool != "" { entry.Devices["root"] = map[string]string{ "type": "disk", "path": "/", "pool": pool, } } // Strip the volatile keys if requested if !keepVolatile { for k := range entry.Config { if !instance.InstanceIncludeWhenCopying(k, true) { delete(entry.Config, k) } } } if entry.Config != nil { // Strip the last_state.power key in all cases delete(entry.Config, "volatile.last_state.power") } // Do the actual copy if c.flagTarget != "" { dstServer = dstServer.UseTarget(c.flagTarget) } op, err = dstServer.CopyInstance(srcServer, *entry, &args) if err != nil { return err } writable = entry.Writable() } // Watch the background operation progress := cli.ProgressRenderer{ Format: i18n.G("Transferring instance: %s"), Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } // Wait for the copy to complete err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } progress.Done("") if c.flagRefresh { inst, etag, err := dstServer.GetInstance(dstInstanceName) if err != nil { return fmt.Errorf(i18n.G("Failed to refresh target instance '%s': %v"), dstInstanceName, err) } // Ensure we don't change the target's volatile.idmap.next value. if inst.Config["volatile.idmap.next"] != writable.Config["volatile.idmap.next"] { writable.Config["volatile.idmap.next"] = inst.Config["volatile.idmap.next"] } // Ensure we don't change the target's root disk pool. srcRootDiskDeviceKey, _, _ := instance.GetRootDiskDevice(writable.Devices) destRootDiskDeviceKey, destRootDiskDevice, _ := instance.GetRootDiskDevice(inst.Devices) if srcRootDiskDeviceKey != "" && srcRootDiskDeviceKey == destRootDiskDeviceKey { writable.Devices[destRootDiskDeviceKey]["pool"] = destRootDiskDevice["pool"] } op, err := dstServer.UpdateInstance(dstInstanceName, writable, etag) if err != nil { return err } // Watch the background operation progress := cli.ProgressRenderer{ Format: i18n.G("Refreshing instance: %s"), Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } // Wait for the copy to complete err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } progress.Done("") } // Start the instance if needed if start { req := api.InstanceStatePut{ Action: string(instance.Start), } op, err := dstServer.UpdateInstanceState(dstInstanceName, req, "") if err != nil { return err } err = op.Wait() if err != nil { return err } } return nil } func (c *cmdCopy) run(cmd *cobra.Command, args []string) error { parsed, err := cmdCopyUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } // For copies, default to non-ephemeral and allow override (move uses -1) ephem := 0 if c.flagEphemeral { ephem = 1 } // Parse the mode mode := "pull" if c.flagMode != "" { mode = c.flagMode } stateful := !c.flagStateless && !c.flagRefresh keepVolatile := c.flagRefresh instanceOnly := c.flagInstanceOnly return c.copyOrMove(cmd, parsed[0], parsed[1], keepVolatile, ephem, stateful, instanceOnly, mode, c.flagStorage, false) } incus-7.0.0/cmd/incus/create.go000066400000000000000000000274571517523235500163440ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "maps" "net/url" "os" "path" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" config "github.com/lxc/incus/v7/shared/cliconfig" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" ) type cmdCreate struct { global *cmdGlobal flagConfig []string flagDevice []string flagEnvironmentFile string flagEphemeral bool flagNetwork string flagProfile []string flagStorage string flagTarget string flagType string flagNoProfiles bool flagEmpty bool flagVM bool flagDescription string } var cmdCreateUsage = u.Usage{u.Either(u.Flag("empty"), u.RemoteImage), u.NewName(u.Instance).Optional().Remote()} func (c *cmdCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdCreateUsage...) cmd.Short = i18n.G("Create instances from images") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Create instances from images`)) cmd.Example = cli.FormatSection("", i18n.G(`incus create images:debian/12 u1 Create the instance u1 incus create images:debian/12 u1 < config.yaml Create the instance with configuration from config.yaml incus launch images:debian/12 v2 --vm -d root,size=50GiB -d root,io.bus=nvme Create and start a virtual machine, overriding the disk size and bus`)) cmd.Aliases = []string{"init"} cmd.RunE = c.run cli.AddStringArrayFlag(cmd.Flags(), &c.flagConfig, "config|c", i18n.G("Config key/value to apply to the new instance")) cli.AddStringArrayFlag(cmd.Flags(), &c.flagProfile, "profile|p", i18n.G("Profile to apply to the new instance")) cli.AddStringArrayFlag(cmd.Flags(), &c.flagDevice, "device|d", i18n.G("New key/value to apply to a specific device")) cli.AddBoolFlag(cmd.Flags(), &c.flagEphemeral, "ephemeral|e", i18n.G("Ephemeral instance")) cli.AddStringFlag(cmd.Flags(), &c.flagEnvironmentFile, "environment-file", "", "", i18n.G("Include environment variables from file")) cli.AddStringFlag(cmd.Flags(), &c.flagNetwork, "network|n", "", "", i18n.G("Network name")) cli.AddStringFlag(cmd.Flags(), &c.flagStorage, "storage|s", "", "", i18n.G("Storage pool name")) cli.AddStringFlag(cmd.Flags(), &c.flagType, "type|t", "", "", i18n.G("Instance type")) cli.AddStringFlag(cmd.Flags(), &c.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagNoProfiles, "no-profiles", i18n.G("Create the instance with no profiles applied")) cli.AddBoolFlag(cmd.Flags(), &c.flagEmpty, "empty", i18n.G("Create an empty instance")) cli.AddBoolFlag(cmd.Flags(), &c.flagVM, "vm", i18n.G("Create a virtual machine")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Instance description")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) != 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return c.global.cmpImages(toComplete) } return cmd } func (c *cmdCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } _, err = c.create(c.global.conf, parsed, false) return err } func (c *cmdCreate) create(conf *config.Config, parsed []*u.Parsed, launch bool) (*u.Parsed, error) { p := parsed[1] d := p.RemoteServer remoteName := p.RemoteName hasInstance := !p.RemoteObject.Skipped instanceName := p.RemoteObject.String // If stdin isn't a terminal, read text from it var stdinData api.InstancePut if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return nil, err } err = loader.Load(&stdinData) if err != nil && !errors.Is(err, io.EOF) { return nil, err } } // Overwrite profiles. var profiles []string if c.flagProfile != nil { profiles = c.flagProfile } else if c.flagNoProfiles { profiles = []string{} } if !c.global.flagQuiet { if d.HasExtension("instance_create_start") && launch { if hasInstance { fmt.Printf(i18n.G("Launching %s")+"\n", formatRemote(conf, p)) } else { fmt.Print(i18n.G("Launching the instance") + "\n") } } else { if hasInstance { fmt.Printf(i18n.G("Creating %s")+"\n", formatRemote(conf, p)) } else { fmt.Print(i18n.G("Creating the instance") + "\n") } } } var devicesMap map[string]map[string]string if len(stdinData.Devices) > 0 { devicesMap = stdinData.Devices } else { devicesMap = map[string]map[string]string{} } if c.flagNetwork != "" { network, _, err := d.GetNetwork(c.flagNetwork) if err != nil { return nil, fmt.Errorf(i18n.G("Failed loading network %q: %w"), c.flagNetwork, err) } // Prepare the instance's NIC device entry. var device map[string]string if network.Managed && d.HasExtension("instance_nic_network") { // If network is managed, use the network property rather than nictype, so that the // network's inherited properties are loaded into the NIC when started. device = map[string]string{ "name": "eth0", "type": "nic", "network": network.Name, } } else { // If network is unmanaged default to using a macvlan connected to the specified interface. device = map[string]string{ "name": "eth0", "type": "nic", "nictype": "macvlan", "parent": c.flagNetwork, } if network.Type == "bridge" { // If the network type is an unmanaged bridge, use bridged NIC type. device["nictype"] = "bridged" } } devicesMap["eth0"] = device } var configMap map[string]string if len(stdinData.Config) > 0 { configMap = stdinData.Config } else { configMap = map[string]string{} } if c.flagEnvironmentFile != "" { envMap, err := readEnvironmentFile(c.flagEnvironmentFile) if err != nil { return nil, err } for k, v := range envMap { configMap["environment."+k] = v } } for _, entry := range c.flagConfig { key, value, found := strings.Cut(entry, "=") if !found { return nil, fmt.Errorf(i18n.G("Bad key=value pair: %q"), entry) } configMap[key] = value } // Check if the specified storage pool exists. if c.flagStorage != "" { _, _, err := d.GetStoragePool(c.flagStorage) if err != nil { return nil, fmt.Errorf(i18n.G("Failed loading storage pool %q: %w"), c.flagStorage, err) } devicesMap["root"] = map[string]string{ "type": "disk", "path": "/", "pool": c.flagStorage, } } // Decide whether we are creating a container or a virtual machine. instanceDBType := api.InstanceTypeContainer if c.flagVM { instanceDBType = api.InstanceTypeVM } // Set the target if provided. if c.flagTarget != "" { d = d.UseTarget(c.flagTarget) } // Setup instance creation request req := api.InstancesPost{ Name: instanceName, InstanceType: c.flagType, Type: instanceDBType, Start: launch, } req.Config = configMap if c.flagEphemeral { req.Ephemeral = c.flagEphemeral } if c.flagDescription != "" { req.Description = c.flagDescription } else { req.Description = stdinData.Description } if !c.flagNoProfiles && len(profiles) == 0 { if len(stdinData.Profiles) > 0 { req.Profiles = stdinData.Profiles } else { req.Profiles = nil } } else { req.Profiles = profiles } // Handle device overrides. deviceOverrides, err := parseDeviceOverrides(c.flagDevice) if err != nil { return nil, err } // Check to see if any of the overridden devices are for devices that are not yet defined in the // local devices (and thus maybe expected to be coming from profiles). profileDevices := make(map[string]map[string]string) needProfileExpansion := false for deviceName := range deviceOverrides { _, isLocalDevice := devicesMap[deviceName] if !isLocalDevice { needProfileExpansion = true break } } // If there are device overrides that are expected to be applied to profile devices then load the profiles // that would be applied server-side. if needProfileExpansion { // If the list of profiles is empty then the default profile would be applied on the server side. serverSideProfiles := req.Profiles if len(serverSideProfiles) == 0 { serverSideProfiles = []string{"default"} } // Get the effective expanded devices by overlaying each profile's devices in order. for _, profileName := range serverSideProfiles { profile, _, err := d.GetProfile(profileName) if err != nil { return nil, fmt.Errorf(i18n.G("Failed loading profile %q for device override: %w"), profileName, err) } maps.Copy(profileDevices, profile.Devices) } } // Apply device overrides. for deviceName := range deviceOverrides { _, isLocalDevice := devicesMap[deviceName] if isLocalDevice { // Apply overrides to local device. maps.Copy(devicesMap[deviceName], deviceOverrides[deviceName]) } else { // Check device exists in expanded profile devices. profileDeviceConfig, found := profileDevices[deviceName] if !found { return nil, fmt.Errorf(i18n.G("Cannot override config for device %q: Device not found in profile devices"), deviceName) } maps.Copy(profileDeviceConfig, deviceOverrides[deviceName]) // Add device to local devices. devicesMap[deviceName] = profileDeviceConfig } } req.Devices = devicesMap var opInfo api.Operation // If an image is provided, use it. if parsed[0].BranchID == 1 { imgRemoteName := parsed[0].List[0].Get(conf.DefaultRemote) imgServer, imgInfo, err := getImgInfo(d, conf, imgRemoteName, remoteName, parsed[0].List[1].String, &req.Source) if err != nil { return nil, err } if conf.Remotes[imgRemoteName].Protocol == "incus" { if imgInfo.Type != "virtual-machine" && c.flagVM { return nil, errors.New(i18n.G("Asked for a VM but image is of type container")) } req.Type = api.InstanceType(imgInfo.Type) } // Create the instance op, err := d.CreateInstanceFromImage(imgServer, *imgInfo, req) if err != nil { return nil, err } // Watch the background operation progress := cli.ProgressRenderer{ Format: i18n.G("Retrieving image: %s"), Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return nil, err } err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return nil, err } progress.Done("") // Extract the instance name info, err := op.GetTarget() if err != nil { return nil, err } opInfo = *info } else { req.Source.Type = "none" op, err := d.CreateInstance(req) if err != nil { return nil, err } err = op.Wait() if err != nil { return nil, err } opInfo = op.Get() } instances, ok := opInfo.Resources["instances"] if !ok || len(instances) == 0 { return nil, errors.New(i18n.G("Didn't get name of new instance from the server")) } if len(instances) == 1 && !hasInstance { uri, err := url.Parse(instances[0]) if err != nil { return nil, err } instanceName = path.Base(uri.Path) fmt.Printf(i18n.G("Instance name is: %s")+"\n", instanceName) } // Validate the network setup c.checkNetwork(d, instanceName) p.RemoteObject = u.ParseString(instanceName) return p, nil } func (c *cmdCreate) checkNetwork(d incus.InstanceServer, name string) { ct, _, err := d.GetInstance(name) if err != nil { return } for _, d := range ct.ExpandedDevices { if d["type"] == "nic" { return } } fmt.Fprint(os.Stderr, "\n"+i18n.G("The instance you are starting doesn't have any network attached to it.")+"\n") fmt.Fprint(os.Stderr, " "+i18n.G("To create a new network, use: incus network create")+"\n") fmt.Fprint(os.Stderr, " "+i18n.G("To attach a network to an instance, use: incus network attach")+"\n\n") } incus-7.0.0/cmd/incus/debug.go000066400000000000000000000042261517523235500161540ustar00rootroot00000000000000package main import ( "fmt" "os" "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/util" ) type cmdDebug struct { global *cmdGlobal } func (c *cmdDebug) command() *cobra.Command { cmd := &cobra.Command{} cmd.Hidden = true cmd.Use = cli.U("debug") cmd.Short = i18n.G("Debug commands") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Debug commands for instances`)) debugAttachCmd := cmdDebugMemory{global: c.global, debug: c} cmd.AddCommand(debugAttachCmd.command()) return cmd } type cmdDebugMemory struct { global *cmdGlobal debug *cmdDebug flagFormat string } var cmdDebugMemoryUsage = u.Usage{u.Instance.Remote(), u.Target(u.File)} func (c *cmdDebugMemory) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("dump-memory", cmdDebugMemoryUsage...) cmd.Short = i18n.G("Export a virtual machine's memory state") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Export the current memory state of a running virtual machine into a dump file. This can be useful for debugging or analysis purposes.`)) cmd.Example = cli.FormatSection("", i18n.G( `incus debug dump-memory vm1 memory-dump.elf --format=elf Creates an ELF format memory dump of the vm1 instance.`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", "elf", "", i18n.G("Format of memory dump (e.g. elf, win-dmp, kdump-zlib, kdump-raw-zlib, ...)")) return cmd } func (c *cmdDebugMemory) run(cmd *cobra.Command, args []string) error { parsed, err := cmdDebugMemoryUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String path := parsed[1].String target, err := os.Create(path) if err != nil { return err } rc, err := d.GetInstanceDebugMemory(instanceName, c.flagFormat) if err != nil { return fmt.Errorf(i18n.G("Failed to dump instance memory: %w"), err) } _, err = util.SafeCopy(target, rc) if err != nil { return err } return nil } incus-7.0.0/cmd/incus/delete.go000066400000000000000000000074361517523235500163360ustar00rootroot00000000000000package main import ( "bufio" "errors" "fmt" "os" "slices" "strings" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/util" ) type cmdDelete struct { global *cmdGlobal flagForce bool flagForceProtected bool flagInteractive bool } var cmdDeleteUsage = u.Usage{u.Instance.Remote().List(1)} func (c *cmdDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete instances`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G("Force the removal of running instances")) cli.AddBoolFlag(cmd.Flags(), &c.flagInteractive, "interactive|i", i18n.G("Require user confirmation")) cmd.ValidArgsFunction = func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpInstances(toComplete) } return cmd } func (c *cmdDelete) promptDelete(p *u.Parsed) error { reader := bufio.NewReader(os.Stdin) fmt.Printf(i18n.G("Remove %s (yes/no): "), formatRemote(c.global.conf, p)) input, _ := reader.ReadString('\n') input = strings.TrimSuffix(input, "\n") if !slices.Contains([]string{i18n.G("yes")}, strings.ToLower(input)) { return errors.New(i18n.G("User aborted delete operation")) } return nil } func (c *cmdDelete) doDelete(d incus.InstanceServer, name string) error { // Instance delete op, err := d.DeleteInstance(name) if err != nil { return err } return op.Wait() } func (c *cmdDelete) deleteOne(p *u.Parsed) error { conf := c.global.conf d := p.RemoteServer instanceName := p.RemoteObject.String connInfo, err := d.GetConnectionInfo() if err != nil { return err } if c.flagInteractive { err := c.promptDelete(p) if err != nil { return err } } ct, _, err := d.GetInstance(instanceName) if err != nil { return fmt.Errorf(i18n.G("Failed checking instance %s exists: %w"), formatRemote(conf, p), err) } if ct.StatusCode != 0 && ct.StatusCode != api.Stopped { if !c.flagForce { return fmt.Errorf(i18n.G("The instance %s is currently running, stop it first or pass --force"), formatRemote(conf, p)) } req := api.InstanceStatePut{ Action: "stop", Timeout: -1, Force: true, } op, err := d.UpdateInstanceState(instanceName, req, "") if err != nil { return err } err = op.Wait() if err != nil { return fmt.Errorf(i18n.G("Stopping the instance %s failed: %s"), formatRemote(conf, p), err) } if ct.Ephemeral { return nil } } if c.flagForceProtected && util.IsTrue(ct.ExpandedConfig["security.protection.delete"]) { // Refresh in case we had to stop it above. ct, etag, err := d.GetInstance(instanceName) if err != nil { return err } ct.Config["security.protection.delete"] = "false" op, err := d.UpdateInstance(instanceName, ct.Writable(), etag) if err != nil { return err } err = op.Wait() if err != nil { return err } } err = c.doDelete(d, instanceName) if err != nil { return fmt.Errorf(i18n.G("Failed deleting instance %s in project %q: %w"), formatRemote(conf, p), connInfo.Project, err) } return nil } func (c *cmdDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } var errs []error for _, p := range parsed[0].List { err := c.deleteOne(p) if err != nil { errs = append(errs, err) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } incus-7.0.0/cmd/incus/exec.go000066400000000000000000000142651517523235500160160ustar00rootroot00000000000000package main import ( "bytes" "errors" "io" "os" "strconv" "strings" "github.com/gorilla/websocket" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/termios" ) type cmdExec struct { global *cmdGlobal flagMode string flagEnvironment []string flagForceInteractive bool flagForceNonInteractive bool flagDisableStdin bool flagUser uint32 flagGroup uint32 flagCwd string interactive bool } var cmdExecUsage = u.Usage{u.Instance.Remote(), u.EndOfFlags, u.CommandLine} func (c *cmdExec) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("exec", cmdExecUsage...) cmd.Short = i18n.G("Execute commands in instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Execute commands in instances The command is executed directly using exec, so there is no shell and shell patterns (variables, file redirects, ...) won't be understood. If you need a shell environment you need to execute the shell executable, passing the shell commands as arguments, for example: incus exec -- sh -c "cd /tmp && pwd" Mode defaults to non-interactive, interactive mode is selected if both stdin AND stdout are terminals (stderr is ignored).`)) cmd.Example = cli.FormatSection("", i18n.G(`incus exec c1 bash Run the "bash" command in instance "c1" incus exec c1 -- ls -lh / Run the "ls -lh /" command in instance "c1"`)) cmd.RunE = c.run cli.AddStringArrayFlag(cmd.Flags(), &c.flagEnvironment, "env", i18n.G("Environment variable to set (e.g. HOME=/home/foo)")) cli.AddStringFlag(cmd.Flags(), &c.flagMode, "mode", "auto", "", i18n.G("Override the terminal mode (auto, interactive or non-interactive)")) cli.AddBoolFlag(cmd.Flags(), &c.flagForceInteractive, "force-interactive|t", i18n.G("Force pseudo-terminal allocation")) cli.AddBoolFlag(cmd.Flags(), &c.flagForceNonInteractive, "force-noninteractive|T", i18n.G("Disable pseudo-terminal allocation")) cli.AddBoolFlag(cmd.Flags(), &c.flagDisableStdin, "disable-stdin|n", i18n.G("Disable stdin (reads from /dev/null)")) cli.AddUint32Flag(cmd.Flags(), &c.flagUser, "user", i18n.G("User ID to run the command as (default 0)")) cli.AddUint32Flag(cmd.Flags(), &c.flagGroup, "group", i18n.G("Group ID to run the command as (default 0)")) cli.AddStringFlag(cmd.Flags(), &c.flagCwd, "cwd", "", "", i18n.G("Directory to run the command in (default /root)")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdExec) sendTermSize(control *websocket.Conn) error { width, height, err := termios.GetSize(getStdoutFd()) if err != nil { return err } logger.Debugf("Window size is now: %dx%d", width, height) msg := api.InstanceExecControl{} msg.Command = "window-resize" msg.Args = make(map[string]string) msg.Args["width"] = strconv.Itoa(width) msg.Args["height"] = strconv.Itoa(height) return control.WriteJSON(msg) } func (c *cmdExec) run(cmd *cobra.Command, args []string) error { conf := c.global.conf parsed, err := cmdExecUsage.Parse(conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String command := parsed[2].StringList if c.flagForceInteractive && c.flagForceNonInteractive { return errors.New(i18n.G("You can't pass -t and -T at the same time")) } if c.flagMode != "auto" && (c.flagForceInteractive || c.flagForceNonInteractive) { return errors.New(i18n.G("You can't pass -t or -T at the same time as --mode")) } // Set the environment env := map[string]string{} myTerm, ok := c.getTERM() if ok { env["TERM"] = myTerm } for _, arg := range c.flagEnvironment { pieces := strings.SplitN(arg, "=", 2) value := "" if len(pieces) > 1 { value = pieces[1] } env[pieces[0]] = value } // Configure the terminal stdinFd := getStdinFd() stdoutFd := getStdoutFd() stdinTerminal := termios.IsTerminal(stdinFd) stdoutTerminal := termios.IsTerminal(stdoutFd) // Determine interaction mode if c.flagDisableStdin { c.interactive = false } else if c.flagMode == "interactive" || c.flagForceInteractive { c.interactive = true } else if c.flagMode == "non-interactive" || c.flagForceNonInteractive { c.interactive = false } else { c.interactive = stdinTerminal && stdoutTerminal } // Record terminal state var oldttystate *termios.State if c.interactive && stdinTerminal { oldttystate, err = termios.MakeRaw(stdinFd) if err != nil { return err } defer func() { _ = termios.Restore(stdinFd, oldttystate) }() } // Setup interactive console handler handler := c.controlSocketHandler // Grab current terminal dimensions var width, height int if stdoutTerminal { width, height, err = termios.GetSize(getStdoutFd()) if err != nil { return err } } var stdin io.Reader stdin = os.Stdin if c.flagDisableStdin { stdin = bytes.NewReader(nil) } stdout := getStdout() // Prepare the command req := api.InstanceExecPost{ Command: command, WaitForWS: true, Interactive: c.interactive, Environment: env, Width: width, Height: height, User: c.flagUser, Group: c.flagGroup, Cwd: c.flagCwd, } execArgs := incus.InstanceExecArgs{ Stdin: stdin, Stdout: stdout, Stderr: os.Stderr, Control: handler, DataDone: make(chan bool), } // Run the command in the instance op, err := d.ExecInstance(instanceName, req, &execArgs) if err != nil { return err } // Wait for the operation to complete err = op.Wait() opAPI := op.Get() if opAPI.Metadata != nil { exitStatusRaw, ok := opAPI.Metadata["return"].(float64) if ok { c.global.ret = int(exitStatusRaw) } } if err != nil { return err } // Wait for any remaining I/O to be flushed <-execArgs.DataDone return nil } incus-7.0.0/cmd/incus/exec_unix.go000066400000000000000000000110501517523235500170460ustar00rootroot00000000000000//go:build !windows package main import ( "os" "os/signal" "github.com/gorilla/websocket" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) func (c *cmdExec) getTERM() (string, bool) { return os.LookupEnv("TERM") } func (c *cmdExec) controlSocketHandler(control *websocket.Conn) { ch := make(chan os.Signal, 10) signal.Notify(ch, unix.SIGWINCH, unix.SIGTERM, unix.SIGHUP, unix.SIGINT, unix.SIGQUIT, unix.SIGABRT, unix.SIGTSTP, unix.SIGTTIN, unix.SIGTTOU, unix.SIGUSR1, unix.SIGUSR2, unix.SIGSEGV, unix.SIGCONT) closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") defer func() { _ = control.WriteMessage(websocket.CloseMessage, closeMsg) }() for { sig := <-ch switch sig { case unix.SIGWINCH: if !c.interactive { // Don't send SIGWINCH to non-interactive, this can lead to console corruption/crashes. continue } logger.Debugf("Received '%s signal', updating window geometry.", sig) err := c.sendTermSize(control) if err != nil { logger.Debugf("error setting term size %s", err) return } case unix.SIGTERM: logger.Debugf("Received '%s signal', forwarding to executing program.", sig) err := c.forwardSignal(control, unix.SIGTERM) if err != nil { logger.Debugf("Failed to forward signal '%s'.", unix.SIGTERM) return } case unix.SIGHUP: file, err := os.OpenFile("/dev/tty", os.O_RDONLY|unix.O_NOCTTY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0o666) if err == nil { _ = file.Close() err = c.forwardSignal(control, unix.SIGHUP) } else { err = c.forwardSignal(control, unix.SIGTERM) sig = unix.SIGTERM } logger.Debugf("Received '%s signal', forwarding to executing program.", sig) if err != nil { logger.Debugf("Failed to forward signal '%s'.", sig) return } case unix.SIGINT: logger.Debugf("Received '%s signal', forwarding to executing program.", sig) err := c.forwardSignal(control, unix.SIGINT) if err != nil { logger.Debugf("Failed to forward signal '%s'.", unix.SIGINT) return } case unix.SIGQUIT: logger.Debugf("Received '%s signal', forwarding to executing program.", sig) err := c.forwardSignal(control, unix.SIGQUIT) if err != nil { logger.Debugf("Failed to forward signal '%s'.", unix.SIGQUIT) return } case unix.SIGABRT: logger.Debugf("Received '%s signal', forwarding to executing program.", sig) err := c.forwardSignal(control, unix.SIGABRT) if err != nil { logger.Debugf("Failed to forward signal '%s'.", unix.SIGABRT) return } case unix.SIGTSTP: logger.Debugf("Received '%s signal', forwarding to executing program.", sig) err := c.forwardSignal(control, unix.SIGTSTP) if err != nil { logger.Debugf("Failed to forward signal '%s'.", unix.SIGTSTP) return } case unix.SIGTTIN: logger.Debugf("Received '%s signal', forwarding to executing program.", sig) err := c.forwardSignal(control, unix.SIGTTIN) if err != nil { logger.Debugf("Failed to forward signal '%s'.", unix.SIGTTIN) return } case unix.SIGTTOU: logger.Debugf("Received '%s signal', forwarding to executing program.", sig) err := c.forwardSignal(control, unix.SIGTTOU) if err != nil { logger.Debugf("Failed to forward signal '%s'.", unix.SIGTTOU) return } case unix.SIGUSR1: logger.Debugf("Received '%s signal', forwarding to executing program.", sig) err := c.forwardSignal(control, unix.SIGUSR1) if err != nil { logger.Debugf("Failed to forward signal '%s'.", unix.SIGUSR1) return } case unix.SIGUSR2: logger.Debugf("Received '%s signal', forwarding to executing program.", sig) err := c.forwardSignal(control, unix.SIGUSR2) if err != nil { logger.Debugf("Failed to forward signal '%s'.", unix.SIGUSR2) return } case unix.SIGSEGV: logger.Debugf("Received '%s signal', forwarding to executing program.", sig) err := c.forwardSignal(control, unix.SIGSEGV) if err != nil { logger.Debugf("Failed to forward signal '%s'.", unix.SIGSEGV) return } case unix.SIGCONT: logger.Debugf("Received '%s signal', forwarding to executing program.", sig) err := c.forwardSignal(control, unix.SIGCONT) if err != nil { logger.Debugf("Failed to forward signal '%s'.", unix.SIGCONT) return } } } } func (c *cmdExec) forwardSignal(control *websocket.Conn, sig unix.Signal) error { logger.Debugf("Forwarding signal: %s", sig) msg := api.InstanceExecControl{} msg.Command = "signal" msg.Signal = int(sig) return control.WriteJSON(msg) } incus-7.0.0/cmd/incus/exec_windows.go000066400000000000000000000025741517523235500175700ustar00rootroot00000000000000//go:build windows package main import ( "io" "os" "os/signal" "github.com/gorilla/websocket" "golang.org/x/sys/windows" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) // Windows doesn't process ANSI sequences natively, so we wrap // os.Stdout for improved user experience for Windows client type WrappedWriteCloser struct { io.Closer wrapper io.Writer } func (wwc *WrappedWriteCloser) Write(p []byte) (int, error) { return wwc.wrapper.Write(p) } func (c *cmdExec) getTERM() (string, bool) { return "dumb", true } func (c *cmdExec) controlSocketHandler(control *websocket.Conn) { ch := make(chan os.Signal, 10) signal.Notify(ch, os.Interrupt) closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") defer control.WriteMessage(websocket.CloseMessage, closeMsg) for { sig := <-ch switch sig { case os.Interrupt: logger.Debugf("Received '%s signal', forwarding to executing program.", sig) err := c.forwardSignal(control, windows.SIGINT) if err != nil { logger.Debugf("Failed to forward signal '%s'.", windows.SIGINT) return } default: break } } } func (c *cmdExec) forwardSignal(control *websocket.Conn, sig windows.Signal) error { logger.Debugf("Forwarding signal: %s", sig) msg := api.InstanceExecControl{} msg.Command = "signal" msg.Signal = int(sig) return control.WriteJSON(msg) } incus-7.0.0/cmd/incus/export.go000066400000000000000000000131261517523235500164060ustar00rootroot00000000000000package main import ( "fmt" "io" "net/url" "os" "path" "time" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/util" ) type cmdExport struct { global *cmdGlobal flagInstanceOnly bool flagRootOnly bool flagOptimizedStorage bool flagCompressionAlgorithm string flagForce bool } var cmdExportUsage = u.Usage{u.Instance.Remote(), u.Target(u.File).Optional()} func (c *cmdExport) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("export", cmdExportUsage...) cmd.Short = i18n.G("Export instance backups") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Export instances as backup tarballs.`)) cmd.Example = cli.FormatSection("", i18n.G( `incus export u1 backup0.tar.gz Download a backup tarball of the u1 instance. incus export u1 - Download a backup tarball with it written to the standard output.`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagInstanceOnly, "instance-only", i18n.G("Whether or not to only backup the instance (without snapshots)")) cli.AddBoolFlag(cmd.Flags(), &c.flagRootOnly, "root-only", i18n.G("Whether or not to only backup the instance (without dependent volumes)")) cli.AddBoolFlag(cmd.Flags(), &c.flagOptimizedStorage, "optimized-storage", i18n.G("Use storage driver optimized format (can only be restored on a similar pool)")) cli.AddStringFlag(cmd.Flags(), &c.flagCompressionAlgorithm, "compression", "", "", i18n.G("Compression algorithm to use (none for uncompressed)")) cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G("Force overwriting existing backup file")) return cmd } func (c *cmdExport) run(cmd *cobra.Command, args []string) error { parsed, err := cmdExportUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String hasTarget := !parsed[1].Skipped targetName := parsed[1].Get("." + instanceName + ".backup") if isStdout(targetName) { // If outputting to stdout, quiesce the output. c.global.flagQuiet = true } else if hasTarget && !c.flagForce && util.PathExists(targetName) { // Check if the target path already exists. return fmt.Errorf(i18n.G("Target path %q already exists"), targetName) } instanceOnly := c.flagInstanceOnly req := api.InstanceBackupsPost{ Name: "", ExpiresAt: time.Now().Add(24 * time.Hour), InstanceOnly: instanceOnly, RootOnly: c.flagRootOnly, OptimizedStorage: c.flagOptimizedStorage, CompressionAlgorithm: c.flagCompressionAlgorithm, } var getter func(backupReq *incus.BackupFileRequest) error if d.HasExtension("direct_backup") { getter = func(backupReq *incus.BackupFileRequest) error { return d.CreateInstanceBackupStream(instanceName, req, backupReq) } } else { // Send the request. op, err := d.CreateInstanceBackup(instanceName, req) if err != nil { return fmt.Errorf(i18n.G("Create instance backup: %w"), err) } // Watch the background operation. progress := cli.ProgressRenderer{ Format: i18n.G("Backing up instance: %s"), Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } // Wait until backup is done. err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } progress.Done("") err = op.Wait() if err != nil { return err } // Get name of backup. uStr := op.Get().Resources["backups"][0] uri, err := url.Parse(uStr) if err != nil { return fmt.Errorf(i18n.G("Invalid URL %q: %w"), uStr, err) } backupName, err := url.PathUnescape(path.Base(uri.EscapedPath())) if err != nil { return fmt.Errorf(i18n.G("Invalid backup name segment in path %q: %w"), uri.EscapedPath(), err) } defer func() { // Delete backup after we're done. op, err = d.DeleteInstanceBackup(instanceName, backupName) if err == nil { _ = op.Wait() } }() getter = func(backupReq *incus.BackupFileRequest) error { _, err := d.GetInstanceBackupFile(instanceName, backupName, backupReq) return err } } var target *os.File if isStdout(targetName) { target = os.Stdout } else { target, err = os.Create(targetName) if err != nil { return err } defer func() { _ = target.Close() }() } // Prepare the download request. progress := cli.ProgressRenderer{ Format: i18n.G("Exporting the backup: %s"), Quiet: c.global.flagQuiet, } backupFileRequest := incus.BackupFileRequest{ BackupFile: io.WriteSeeker(target), ProgressHandler: progress.UpdateProgress, } // Export tarball. err = getter(&backupFileRequest) if err != nil { _ = os.Remove(targetName) progress.Done("") return fmt.Errorf(i18n.G("Fetch instance backup file: %w"), err) } // Detect backup file type and rename file accordingly. if !hasTarget { _, err := target.Seek(0, io.SeekStart) if err != nil { return err } _, ext, _, err := archive.DetectCompressionFile(target) if err != nil { return err } err = os.Rename(targetName, instanceName+ext) if err != nil { return fmt.Errorf(i18n.G("Failed to rename export file: %w"), err) } } err = target.Close() if err != nil { return fmt.Errorf(i18n.G("Failed to close export file: %w"), err) } progress.Done(i18n.G("Backup exported successfully!")) return nil } incus-7.0.0/cmd/incus/file.go000066400000000000000000000657231517523235500160160ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "io/fs" "net" "os" "path/filepath" "slices" "strconv" "strings" "github.com/pkg/sftp" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" internalIO "github.com/lxc/incus/v7/internal/io" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/termios" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) const ( // DirMode represents the file mode for creating dirs on `incus file pull/push`. DirMode = 0o755 // FileMode represents the file mode for creating files on `incus file create`. FileMode = 0o644 ) type cmdFile struct { global *cmdGlobal flagUID int flagGID int flagMode string flagMkdir bool } func (c *cmdFile) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("file") cmd.Short = i18n.G("Manage files in instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage files in instances`)) // Create fileCreateCmd := cmdFileCreate{global: c.global, file: c} cmd.AddCommand(fileCreateCmd.command()) // Delete fileDeleteCmd := cmdFileDelete{global: c.global, file: c} cmd.AddCommand(fileDeleteCmd.command()) // Mount fileMountCmd := cmdFileMount{global: c.global, file: c} cmd.AddCommand(fileMountCmd.command()) // Pull filePullCmd := cmdFilePull{global: c.global, file: c, puller: &pullable{}} cmd.AddCommand(filePullCmd.command()) // Push filePushCmd := cmdFilePush{global: c.global, file: c, pusher: &pushable{}} cmd.AddCommand(filePushCmd.command()) // Edit fileEditCmd := cmdFileEdit{global: c.global, file: c, filePull: &filePullCmd, filePush: &filePushCmd} cmd.AddCommand(fileEditCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Create. type cmdFileCreate struct { global *cmdGlobal file *cmdFile flagForce bool flagType string } var cmdFileCreateUsage = u.Usage{u.MakePath(u.Instance, u.Path).Remote(), u.SymlinkTargetPath.Optional()} func (c *cmdFileCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdFileCreateUsage...) cmd.Short = i18n.G("Create files and directories in instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Create files and directories in instances`)) cmd.Example = cli.FormatSection("", i18n.G( `incus file create foo/bar To create a file /bar in the foo instance. incus file create --type=symlink foo/bar baz To create a symlink /bar in instance foo whose target is baz.`)) cli.AddBoolFlag(cmd.Flags(), &c.file.flagMkdir, "create-dirs|p", i18n.G("Create any directories necessary")) cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G("Force creating files or directories")) cli.AddIntFlag(cmd.Flags(), &c.file.flagGID, "gid", -1, i18n.G("Set the file's gid on create")) cli.AddIntFlag(cmd.Flags(), &c.file.flagUID, "uid", -1, i18n.G("Set the file's uid on create")) cli.AddStringFlag(cmd.Flags(), &c.file.flagMode, "mode", "", "", i18n.G("Set the file's perms on create")) cli.AddStringFlag(cmd.Flags(), &c.flagType, "type|t", "file", "", i18n.G("The type to create (file, symlink, or directory)")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpFiles(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdFileCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdFileCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.List[0].String targetPath, isDir := normalizePath(parsed[0].RemoteObject.List[1].String) hasSymlink := !parsed[1].Skipped symlinkTargetPath := parsed[1].String if !slices.Contains([]string{"file", "symlink", "directory"}, c.flagType) { return fmt.Errorf(i18n.G("Invalid type %q"), c.flagType) } if hasSymlink { if c.flagType != "symlink" { return errors.New(i18n.G(`Symlink target path can only be used for type "symlink"`)) } symlinkTargetPath = filepath.Clean(symlinkTargetPath) } if isDir { c.flagType = "directory" } // Connect to SFTP. sftpConn, err := d.GetInstanceFileSFTP(instanceName) if err != nil { return err } defer func() { _ = sftpConn.Close() }() // Determine the target uid uid := max(c.file.flagUID, 0) // Determine the target gid gid := max(c.file.flagGID, 0) var mode os.FileMode // Determine the target mode switch c.flagType { case "directory": mode = os.FileMode(DirMode) case "file": mode = os.FileMode(FileMode) } if c.file.flagMode != "" { if len(c.file.flagMode) == 3 { c.file.flagMode = "0" + c.file.flagMode } m, err := strconv.ParseInt(c.file.flagMode, 0, 0) if err != nil { return err } mode = os.FileMode(m) } // Create needed paths if requested if c.file.flagMkdir { err = sftpRecursiveMkdir(sftpConn, filepath.Dir(targetPath), nil, int64(uid), int64(gid)) if err != nil { return err } } var content io.ReadSeeker var readCloser io.ReadCloser var contentLength int64 switch c.flagType { case "symlink": content = strings.NewReader(symlinkTargetPath) readCloser = io.NopCloser(content) contentLength = int64(len(symlinkTargetPath)) case "file": // Just creating an empty file. content = strings.NewReader("") readCloser = io.NopCloser(content) contentLength = 0 } fileArgs := incus.InstanceFileArgs{ Type: c.flagType, UID: int64(uid), GID: int64(gid), Mode: int(mode.Perm()), Content: content, } if c.flagForce { fileArgs.WriteMode = "overwrite" } progress := cli.ProgressRenderer{ Format: fmt.Sprintf(i18n.G("Creating %s: %%s"), targetPath), Quiet: c.global.flagQuiet, } if readCloser != nil { fileArgs.Content = internalIO.NewReadSeeker(&ioprogress.ProgressReader{ ReadCloser: readCloser, Tracker: &ioprogress.ProgressTracker{ Length: contentLength, Handler: func(percent int64, speed int64) { progress.UpdateProgress(ioprogress.ProgressData{ Text: fmt.Sprintf("%d%% (%s/s)", percent, units.GetByteSizeString(speed, 2)), }) }, }, }, fileArgs.Content) } err = sftpCreateFile(sftpConn, targetPath, fileArgs, false) if err != nil { progress.Done("") return err } progress.Done("") return nil } // Delete. type cmdFileDelete struct { global *cmdGlobal file *cmdFile flagForce bool } var cmdFileDeleteUsage = u.Usage{u.MakePath(u.Instance, u.Path).Remote().List(1)} func (c *cmdFileDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdFileDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete files in instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete files in instances`)) cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G("Force deleting files, directories, and subdirectories")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpFiles(toComplete, false) } return cmd } func (c *cmdFileDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdFileDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } // Store clients. sftpClients := map[string]*sftp.Client{} defer func() { for _, sftpClient := range sftpClients { _ = sftpClient.Close() } }() var errs []error for _, p := range parsed[0].List { err := func() error { d := p.RemoteServer instanceName := p.RemoteObject.List[0].String path, _ := normalizePath(p.RemoteObject.List[1].String) instanceID := p.RemoteName + ":" + instanceName sftpConn, ok := sftpClients[instanceID] if !ok { sftpConn, err = d.GetInstanceFileSFTP(instanceName) if err != nil { return err } sftpClients[instanceID] = sftpConn } if c.flagForce { err = sftpConn.RemoveAll(path) if err != nil { return err } return nil } return sftpConn.Remove(path) }() if err != nil { errs = append(errs, err) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } // Edit. type cmdFileEdit struct { global *cmdGlobal file *cmdFile filePull *cmdFilePull filePush *cmdFilePush } var cmdFileEditUsage = u.Usage{u.MakePath(u.Instance, u.Path).Remote()} func (c *cmdFileEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdFileEditUsage...) cmd.Short = i18n.G("Edit files in instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Edit files in instances`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpFiles(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdFileEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdFileEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } fileName := parsed[0].RemoteObject.List[1].String c.filePush.noModeChange = true // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { return c.filePush.push([]string{os.Stdin.Name()}, parsed[0]) } // Create temp file f, err := os.CreateTemp("", fmt.Sprintf("incus_file_edit_*%s", filepath.Ext(fileName))) if err != nil { return fmt.Errorf(i18n.G("Unable to create a temporary file: %v"), err) } fname := f.Name() _ = f.Close() _ = os.Remove(fname) // Tell pull/push that they're called from edit. c.filePull.edit = true c.filePush.edit = true // Extract current value defer func() { _ = os.Remove(fname) }() err = c.filePull.pull(parsed, fname) if err != nil { return err } // Spawn the editor _, err = cli.TextEditor(fname, []byte{}) if err != nil { return err } // Push the result err = c.filePush.push([]string{fname}, parsed[0]) if err != nil { return err } return nil } // Pull. type cmdFilePull struct { global *cmdGlobal file *cmdFile puller *pullable edit bool } var cmdFilePullUsage = u.Usage{u.MakePath(u.Instance, u.Path).Remote().List(1), u.Target(u.Path)} func (c *cmdFilePull) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("pull", cmdFilePullUsage...) cmd.Short = i18n.G("Pull files from instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Pull files from instances`)) cmd.Example = cli.FormatSection("", i18n.G( `incus file pull foo/etc/hosts . To pull /etc/hosts from the instance and write it to the current directory. incus file pull foo/etc/hosts - To pull /etc/hosts from the instance and write its output to standard output.`)) cli.AddBoolFlag(cmd.Flags(), &c.file.flagMkdir, "create-dirs|p", i18n.G("Create any directories necessary")) cli.AddBoolFlag(cmd.Flags(), &c.puller.flagRecursive, "recursive|r", i18n.G("Recursively transfer files")) cli.AddBoolFlag(cmd.Flags(), &c.puller.flagNoDereference, "no-dereference|P", i18n.G("Never follow symbolic links in source path")) cli.AddBoolFlag(cmd.Flags(), &c.puller.flagFollow, "follow|H", i18n.G("Follow command-line symbolic links in source path")) cli.AddBoolFlag(cmd.Flags(), &c.puller.flagDereference, "dereference|L", i18n.G("Always follow symbolic links in source path")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpFiles(toComplete, false) } return c.global.cmpFiles(toComplete, true) } return cmd } // pull runs the post-parsing command logic. func (c *cmdFilePull) pull(parsedFiles []*u.Parsed, target string) error { targetIsDir := strings.HasSuffix(target, "/") targetExists := true targetInfo, err := os.Stat(target) if err != nil { if !errors.Is(err, fs.ErrNotExist) { return err } targetExists = false } err = c.puller.preCheck(target) if err != nil { return err } /* * If the path exists, just use it. If it doesn't exist, it might be a * directory in one of three cases: * 1. Someone explicitly put "/" at the end * 2. Someone provided more than one source. In this case the target * should be a directory so we can save all the files into it. * 3. We are dealing with recursive copy */ if targetExists { targetIsDir = targetInfo.IsDir() if !targetIsDir && len(parsedFiles) > 1 { return errors.New(i18n.G("More than one file to download, but target is not a directory")) } } else if targetIsDir || len(parsedFiles) > 1 { err := os.MkdirAll(target, DirMode) if err != nil { return err } targetIsDir = true } else if c.file.flagMkdir { err := os.MkdirAll(filepath.Dir(target), DirMode) if err != nil { return err } } sftpClients := map[string]*sftp.Client{} defer func() { for _, sftpClient := range sftpClients { _ = sftpClient.Close() } }() var errs []error for _, p := range parsedFiles { err := func() error { d := p.RemoteServer instanceName := p.RemoteObject.List[0].String path := p.RemoteObject.List[1].String instanceID := p.RemoteName + ":" + instanceName sftpConn, ok := sftpClients[instanceID] if !ok { sftpConn, err = d.GetInstanceFileSFTP(instanceName) if err != nil { return err } sftpClients[instanceID] = sftpConn } srcInfo, normalizedPath, err := c.puller.statFile(sftpConn, path) if err != nil { return err } // Recursively copy directories. if srcInfo.IsDir() { return sftpRecursivePullFile(sftpConn, srcInfo, path, normalizedPath, target, c.global.flagQuiet, c.puller.flagDereference, len(parsedFiles) > 1 || util.PathExists(target)) } // Determine the target path. var targetPath string if targetIsDir { targetPath = filepath.Join(target, filepath.Base(normalizedPath)) } else { targetPath = target } // Prepare target. targetIsLink := srcInfo.Mode()&os.ModeSymlink != 0 var f *os.File var linkName string if isStdout(targetPath) { f = os.Stdout } else if targetIsLink { linkName, err = sftpConn.ReadLink(normalizedPath) if err != nil { return err } } else { f, err = os.Create(targetPath) if err != nil { return err } defer func() { _ = f.Close() }() // nolint:revive err = os.Chmod(targetPath, os.FileMode(srcInfo.Mode())) if err != nil { return err } } progress := cli.ProgressRenderer{ Format: fmt.Sprintf(i18n.G("Pulling %s from %s: %%s"), targetPath, normalizedPath), Quiet: c.global.flagQuiet, } writer := &ioprogress.ProgressWriter{ WriteCloser: f, Tracker: &ioprogress.ProgressTracker{ Handler: func(bytesReceived int64, speed int64) { if isStdout(targetPath) { return } progress.UpdateProgress(ioprogress.ProgressData{ Text: fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(bytesReceived, 2), units.GetByteSizeString(speed, 2)), }) }, }, } if targetIsLink { err = os.Symlink(linkName, targetPath) if err != nil { progress.Done("") return err } } else { src, err := sftpConn.Open(normalizedPath) if err != nil { return err } defer func() { _ = src.Close() }() _, err = util.SafeCopy(writer, src) if err != nil { progress.Done("") return err } } progress.Done("") return nil }() if err != nil { errs = append(errs, err) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } func (c *cmdFilePull) run(cmd *cobra.Command, args []string) error { // Do NOT blindly copy the following parsing line; it performs right-to-left parsing, which in // most cases is NOT what you want. parsed, err := cmdFilePullUsage.Parse(c.global.conf, cmd, args, true) if err != nil { return err } return c.pull(parsed[0].List, parsed[1].String) } // Push. type cmdFilePush struct { global *cmdGlobal file *cmdFile pusher *pushable edit bool noModeChange bool } var cmdFilePushUsage = u.Usage{u.Path.List(1), u.MakePath(u.Instance, u.Target(u.Path)).Remote()} func (c *cmdFilePush) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("push", cmdFilePushUsage...) cmd.Short = i18n.G("Push files into instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Push files into instances`)) cmd.Example = cli.FormatSection("", i18n.G( `incus file push /etc/hosts foo/etc/hosts To push /etc/hosts into the instance "foo". echo "Hello world" | incus file push - foo/root/test To read "Hello world" from standard input and write it into /root/test in instance "foo".`)) cli.AddBoolFlag(cmd.Flags(), &c.file.flagMkdir, "create-dirs|p", i18n.G("Create any directories necessary")) cli.AddIntFlag(cmd.Flags(), &c.file.flagUID, "uid", -1, i18n.G("Set the files' UIDs on push (in recursive mode, only sets the target directory's UID if it doesn't exist and -p is used)")) cli.AddIntFlag(cmd.Flags(), &c.file.flagGID, "gid", -1, i18n.G("Set the files' GIDs on push (in recursive mode, only sets the target directory's GID if it doesn't exist and -p is used)")) cli.AddStringFlag(cmd.Flags(), &c.file.flagMode, "mode", "", "", i18n.G("Set the file's perms on push (in recursive mode, sets the target directory's permissions if it doesn't exist)")) cli.AddBoolFlag(cmd.Flags(), &c.pusher.flagRecursive, "recursive|r", i18n.G("Recursively transfer files")) cli.AddBoolFlag(cmd.Flags(), &c.pusher.flagNoDereference, "no-dereference|P", i18n.G("Never follow symbolic links in source path")) cli.AddBoolFlag(cmd.Flags(), &c.pusher.flagFollow, "follow|H", i18n.G("Follow command-line symbolic links in source path")) cli.AddBoolFlag(cmd.Flags(), &c.pusher.flagDereference, "dereference|L", i18n.G("Always follow symbolic links in source path")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return nil, cobra.ShellCompDirectiveDefault } return c.global.cmpFiles(toComplete, true) } return cmd } // push runs the post-parsing command logic. func (c *cmdFilePush) push(srcFiles []string, parsedTarget *u.Parsed) error { d := parsedTarget.RemoteServer instanceName := parsedTarget.RemoteObject.List[0].String target, targetIsDir := normalizePath(parsedTarget.RemoteObject.List[1].String) targetExists := false err := c.pusher.preCheck() if err != nil { return err } // Connect to SFTP. sftpConn, err := d.GetInstanceFileSFTP(instanceName) if err != nil { return err } defer func() { _ = sftpConn.Close() }() targetInfo, err := sftpConn.Stat(target) if err == nil { targetExists = true if targetInfo.IsDir() { targetIsDir = true } else if len(srcFiles) > 1 || targetIsDir { // Let’s be extra careful and check that explicit requests for directories actually point to // directories. return fmt.Errorf(i18n.G("%s is not a directory"), target) } } else if len(srcFiles) > 1 && !c.file.flagMkdir { return errors.New(i18n.G("Missing target directory")) } var mode os.FileMode if c.file.flagMode != "" { if len(c.file.flagMode) == 3 { c.file.flagMode = "0" + c.file.flagMode } m, err := strconv.ParseInt(c.file.flagMode, 0, 0) if err != nil { return err } mode = os.FileMode(m) } var errs []error canProcessStdin := len(srcFiles) == 1 // Push the files for _, path := range srcFiles { err := func() error { var f *os.File var linkTarget string var size int64 m := mode uid := max(c.file.flagUID, 0) gid := max(c.file.flagGID, 0) if isStdin(path) { if !canProcessStdin { return errors.New(i18n.G("stdin can only be used once, with no other source arguments")) } if targetIsDir { return errors.New(i18n.G("A target file name must be specified when pushing from stdin; the target is a directory")) } canProcessStdin = false f = os.Stdin } else { srcInfo, wPath, err := c.pusher.statFile(path) if err != nil { return err } // Recursively copy directories. if srcInfo.IsDir() { return sftpRecursivePushFile(sftpConn, wPath, path, target, c.global.flagQuiet, c.pusher.flagDereference, len(srcFiles) > 1 || targetExists) } if srcInfo.Mode()&os.ModeSymlink != 0 { linkTarget, err = os.Readlink(path) if err != nil { return err } } else { f, err = os.Open(path) if err != nil { return fmt.Errorf(i18n.G("Failed to open source file %q: %v"), f, err) } size = srcInfo.Size() defer func() { _ = f.Close() }() } if c.file.flagUID == -1 || c.file.flagGID == -1 { dMode, dUID, dGID := internalIO.GetOwnerMode(srcInfo) if c.file.flagMode == "" { m = dMode } if c.file.flagUID == -1 { uid = dUID } if c.file.flagGID == -1 { gid = dGID } } } // Determine the target path. var targetPath string if targetIsDir { targetPath = filepath.Join(target, filepath.Base(path)) } else { targetPath = target } // Create needed paths if requested if c.file.flagMkdir { mode := os.FileMode(DirMode) err = sftpRecursiveMkdir(sftpConn, filepath.Dir(targetPath), &mode, int64(uid), int64(gid)) if err != nil { return err } } // Transfer the files. args := incus.InstanceFileArgs{ UID: -1, GID: -1, Mode: -1, } // Check if the path already exists. _, err := sftpConn.Stat(targetPath) fileExists := err == nil if !c.noModeChange { if !fileExists || c.file.flagUID != -1 { args.UID = int64(uid) } if !fileExists || c.file.flagGID != -1 { args.GID = int64(gid) } if !fileExists || c.file.flagMode != "" { args.Mode = int(m.Perm()) } } progress := cli.ProgressRenderer{ Format: fmt.Sprintf(i18n.G("Pushing %s to %s: %%s"), path, targetPath), Quiet: c.global.flagQuiet, } if f == nil { args.Type = "symlink" args.Content = strings.NewReader(linkTarget) } else { args.Type = "file" args.Content = internalIO.NewReadSeeker(&ioprogress.ProgressReader{ ReadCloser: f, Tracker: &ioprogress.ProgressTracker{ Length: size, Handler: func(percent int64, speed int64) { progress.UpdateProgress(ioprogress.ProgressData{ Text: fmt.Sprintf("%d%% (%s/s)", percent, units.GetByteSizeString(speed, 2)), }) }, }, }, f) } logger.Infof("Pushing %s to %s (%s)", path, targetPath, args.Type) err = sftpCreateFile(sftpConn, targetPath, args, true) progress.Done("") return err }() if err != nil { errs = append(errs, err) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } func (c *cmdFilePush) run(cmd *cobra.Command, args []string) error { // Do NOT blindly copy the following parsing line; it performs right-to-left parsing, which in // most cases is NOT what you want. parsed, err := cmdFilePushUsage.Parse(c.global.conf, cmd, args, true) if err != nil { return err } return c.push(parsed[0].StringList, parsed[1]) } // Mount. type cmdFileMount struct { global *cmdGlobal file *cmdFile flagListen string flagAuthNone bool flagAuthUser string } var cmdFileMountUsage = u.Usage{u.MakePath(u.Instance, u.Path.Optional()).Remote(), u.Target(u.Path).Optional()} func (c *cmdFileMount) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("mount", cmdFileMountUsage...) cmd.Short = i18n.G("Mount files from instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Mount files from instances. If no target path is provided, start an SSH SFTP listener instead.`)) cmd.Example = cli.FormatSection("", i18n.G( `incus file mount foo/root fooroot To mount /root from the instance foo onto the local fooroot directory. incus file mount foo To start an SSH SFTP listener for the root filesystem of instance foo.`)) cli.AddStringFlag(cmd.Flags(), &c.flagListen, "listen", "", "", i18n.G("Setup SSH SFTP listener on address:port instead of mounting")) cli.AddBoolFlag(cmd.Flags(), &c.flagAuthNone, "no-auth", i18n.G("Disable authentication when using SSH SFTP listener")) cli.AddStringFlag(cmd.Flags(), &c.flagAuthUser, "auth-user", "", "", i18n.G("Set authentication user when using SSH SFTP listener")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpFiles(toComplete, false) } if len(args) == 1 { return nil, cobra.ShellCompDirectiveDefault } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdFileMount) run(cmd *cobra.Command, args []string) error { parsed, err := cmdFileMountUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.List[0].String hasInstancePath := !parsed[0].RemoteObject.List[1].Skipped instancePath := parsed[0].RemoteObject.List[1].String hasTarget := !parsed[1].Skipped targetPath := filepath.Clean(parsed[1].String) // Determine the target if specified. if hasTarget { sb, err := os.Stat(targetPath) if err != nil { return err } if !sb.IsDir() { return errors.New(i18n.G("Target path must be a directory")) } } // Check which mode we should operate in. If target path is provided we use sshfs mode. if hasTarget && c.flagListen != "" { return errors.New(i18n.G("Target path and --listen flag cannot be used together")) } // Check instance path is provided in sshfs mode. if !hasInstancePath && hasTarget { return fmt.Errorf(i18n.G("An instance path is required for %s"), formatRemote(c.global.conf, parsed[0])) } // Check instance path isn't provided in listener mode. if hasInstancePath && !hasTarget { return errors.New(i18n.G("Instance path cannot be used in SSH SFTP listener mode")) } // Look for sshfs command if no SSH SFTP listener mode specified and a target mount path was specified. if c.flagListen == "" && hasTarget { // Setup sourcePath with leading / to ensure we reference the instance path from / location. if len(instancePath) == 0 || instancePath[0] != '/' { instancePath = "/" + instancePath } // Connect to SFTP. sftpConn, err := d.GetInstanceFileSFTPConn(instanceName) if err != nil { return fmt.Errorf(i18n.G("Failed connecting to instance SFTP: %w"), err) } defer func() { _ = sftpConn.Close() }() return sshfsMount(cmd.Context(), sftpConn, instanceName, instancePath, targetPath) } // Check the instance exists before starting the SFTP server. _, _, err = d.GetInstance(instanceName) if err != nil { return err } return sshSFTPServer(cmd.Context(), func() (net.Conn, error) { return d.GetInstanceFileSFTPConn(instanceName) }, c.flagAuthNone, c.flagAuthUser, c.flagListen) } incus-7.0.0/cmd/incus/image.go000066400000000000000000001340571517523235500161560ustar00rootroot00000000000000package main import ( "archive/tar" "bufio" "encoding/json" "errors" "fmt" "io" "maps" "os" "os/exec" "path/filepath" "reflect" "regexp" "sort" "strings" "time" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" internalFilter "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/i18n" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" "github.com/lxc/incus/v7/shared/ask" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/termios" "github.com/lxc/incus/v7/shared/util" ) type imageColumn struct { Name string Data func(api.Image) string } type cmdImage struct { global *cmdGlobal } func (c *cmdImage) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("image") cmd.Short = i18n.G("Manage images") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Manage images Instances are created from images. Those images were themselves either generated from an existing instance or downloaded from an image server. When using remote images, the server will automatically cache images for you and remove them upon expiration. The image unique identifier is the hash (sha-256) of its representation as a compressed tarball (or for split images, the concatenation of the metadata and rootfs tarballs). Images can be referenced by their full hash, shortest unique partial hash or alias name (if one is set).`)) // Alias imageAliasCmd := cmdImageAlias{global: c.global, image: c} cmd.AddCommand(imageAliasCmd.command()) // Copy imageCopyCmd := cmdImageCopy{global: c.global, image: c} cmd.AddCommand(imageCopyCmd.command()) // Delete imageDeleteCmd := cmdImageDelete{global: c.global, image: c} cmd.AddCommand(imageDeleteCmd.command()) // Edit imageEditCmd := cmdImageEdit{global: c.global, image: c} cmd.AddCommand(imageEditCmd.command()) // Generate metadata imageGenerateMetadataCmd := cmdImageGenerateMetadata{global: c.global, image: c} cmd.AddCommand(imageGenerateMetadataCmd.command()) // Export imageExportCmd := cmdImageExport{global: c.global, image: c} cmd.AddCommand(imageExportCmd.command()) // Import imageImportCmd := cmdImageImport{global: c.global, image: c} cmd.AddCommand(imageImportCmd.command()) // Info imageInfoCmd := cmdImageInfo{global: c.global, image: c} cmd.AddCommand(imageInfoCmd.command()) // List imageListCmd := cmdImageList{global: c.global, image: c} cmd.AddCommand(imageListCmd.command()) // Refresh imageRefreshCmd := cmdImageRefresh{global: c.global, image: c} cmd.AddCommand(imageRefreshCmd.command()) // Show imageShowCmd := cmdImageShow{global: c.global, image: c} cmd.AddCommand(imageShowCmd.command()) // Get-property imageGetPropCmd := cmdImageGetProp{global: c.global, image: c} cmd.AddCommand(imageGetPropCmd.command()) // Set-property imageSetPropCmd := cmdImageSetProp{global: c.global, image: c} cmd.AddCommand(imageSetPropCmd.command()) // Unset-property imageUnsetPropCmd := cmdImageUnsetProp{global: c.global, image: c, imageSetProp: &imageSetPropCmd} cmd.AddCommand(imageUnsetPropCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } func (c *cmdImage) dereferenceAlias(d incus.ImageServer, imageType string, inName string) string { if inName == "" { inName = "default" } result, _, _ := d.GetImageAliasType(imageType, inName) if result == nil { return inName } return result.Target } // Copy. type cmdImageCopy struct { global *cmdGlobal image *cmdImage flagAliases []string flagPublic bool flagCopyAliases bool flagReuse bool flagAutoUpdate bool flagVM bool flagMode string flagTargetProject string flagProfile []string } var cmdImageCopyUsage = u.Usage{u.RemoteImage, u.RemoteColonOpt} func (c *cmdImageCopy) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("copy", cmdImageCopyUsage...) cmd.Aliases = []string{"cp"} cmd.Short = i18n.G("Copy images between servers") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Copy images between servers The auto-update flag instructs the server to keep this image up to date. It requires the source to be an alias and for it to be public.`)) cli.AddBoolFlag(cmd.Flags(), &c.flagPublic, "public", i18n.G("Make image public")) cli.AddBoolFlag(cmd.Flags(), &c.flagCopyAliases, "copy-aliases", i18n.G("Copy aliases from source")) cli.AddBoolFlag(cmd.Flags(), &c.flagReuse, "reuse", i18n.G("If an alias already exists, delete and recreate it")) cli.AddBoolFlag(cmd.Flags(), &c.flagAutoUpdate, "auto-update", i18n.G("Keep the image up to date after initial copy")) cli.AddStringArrayFlag(cmd.Flags(), &c.flagAliases, "alias", i18n.G("New aliases to add to the image")) cli.AddBoolFlag(cmd.Flags(), &c.flagVM, "vm", i18n.G("Copy virtual machine images")) cli.AddStringFlag(cmd.Flags(), &c.flagMode, "mode", "pull", "", i18n.G("Transfer mode. One of pull (default), push or relay")) cli.AddStringFlag(cmd.Flags(), &c.flagTargetProject, "target-project", "", "", i18n.G("Copy to a project different from the source")) cli.AddStringArrayFlag(cmd.Flags(), &c.flagProfile, "profile|p", i18n.G("Profile to apply to the new image")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpImages(toComplete) } if len(args) == 1 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdImageCopy) run(cmd *cobra.Command, args []string) error { conf := c.global.conf parsed, err := cmdImageCopyUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } imgRemoteName := parsed[0].List[0].Get(conf.DefaultRemote) imageName := parsed[0].List[1].String d := parsed[1].RemoteServer remoteName := parsed[1].RemoteName remote := conf.Remotes[remoteName] if c.flagMode != "pull" && c.flagAutoUpdate { return errors.New(i18n.G("Auto update is only available in pull mode")) } if c.flagReuse && !c.flagCopyAliases { return errors.New(i18n.G("--reuse requires --copy-aliases")) } sourceServer, err := c.global.conf.GetImageServer(imgRemoteName) if err != nil { return err } // Resolve image type imageType := "" if c.flagVM { imageType = "virtual-machine" } // Set the correct project on target. if c.flagTargetProject != "" { d = d.UseProject(c.flagTargetProject) } else if remote.Protocol == "incus" { d = d.UseProject(remote.Project) } // Copy the image var imgInfo *api.Image if conf.Remotes[imgRemoteName].Protocol != "incus" && !c.flagCopyAliases && len(c.flagAliases) == 0 { // All image servers outside of other Incus servers are always public, so unless we // need the aliases list too or the real fingerprint, we can skip the otherwise very expensive // alias resolution and image info retrieval step. imgInfo = &api.Image{} imgInfo.Fingerprint = imageName imgInfo.Public = true } else { // Resolve any alias and then grab the image information from the source image := c.image.dereferenceAlias(sourceServer, imageType, imageName) imgInfo, _, err = sourceServer.GetImage(image) if err != nil { return err } } if imgInfo.Public && imgInfo.Fingerprint != imageName && !strings.HasPrefix(imgInfo.Fingerprint, imageName) { // If dealing with an alias, set the imgInfo fingerprint to match the provided alias (needed for auto-update) imgInfo.Fingerprint = imageName } aliases := make([]api.ImageAlias, len(c.flagAliases)) for i, entry := range c.flagAliases { aliases[i].Name = entry } copyArgs := incus.ImageCopyArgs{ Aliases: aliases, AutoUpdate: c.flagAutoUpdate, CopyAliases: c.flagCopyAliases, Public: c.flagPublic, Type: imageType, Mode: c.flagMode, Profiles: c.flagProfile, } // If --reuse was passed, delete any conflicting aliases on the target so they can be recreated. if c.flagReuse { conflicting := append([]api.ImageAlias{}, imgInfo.Aliases...) conflicting = append(conflicting, aliases...) existing, err := getCommonAliases(d, conflicting...) if err != nil { return fmt.Errorf(i18n.G("Failed to check for existing aliases: %w"), err) } for _, alias := range existing { err := d.DeleteImageAlias(alias.Name) if err != nil { return fmt.Errorf(i18n.G("Failed to remove alias %s: %w"), alias.Name, err) } } } // Do the copy op, err := d.CopyImage(sourceServer, *imgInfo, ©Args) if err != nil { return err } // Register progress handler progress := cli.ProgressRenderer{ Format: i18n.G("Copying the image: %s"), Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } // Wait for operation to finish err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } progress.Done(i18n.G("Image copied successfully!")) return nil } // Delete. type cmdImageDelete struct { global *cmdGlobal image *cmdImage } var cmdImageDeleteUsage = u.Usage{u.Image.Remote().List(1)} func (c *cmdImageDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdImageDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete images") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete images`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpImages(toComplete) } return cmd } func (c *cmdImageDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } var errs []error for _, p := range parsed[0].List { err := func() error { d := p.RemoteServer imageName := p.RemoteObject.String image := c.image.dereferenceAlias(d, "", imageName) op, err := d.DeleteImage(image) if err != nil { return err } return op.Wait() }() if err != nil { errs = append(errs, err) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } // Edit. type cmdImageEdit struct { global *cmdGlobal image *cmdImage } var cmdImageEditUsage = u.Usage{u.Image.Remote()} func (c *cmdImageEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdImageEditUsage...) cmd.Short = i18n.G("Edit image properties") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Edit image properties`)) cmd.Example = cli.FormatSection("", i18n.G( `incus image edit Launch a text editor to edit the properties incus image edit < image.yaml Load the image properties from a YAML file`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpImages(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdImageEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the image properties. ### Any line starting with a '# will be ignored. ### ### Each property is represented by a single line: ### An example would be: ### description: My custom image`) } func (c *cmdImageEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer imageName := parsed[0].RemoteObject.String // Resolve any aliases image := c.image.dereferenceAlias(d, "", imageName) if image == "" { image = imageName } // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } newdata := api.ImagePut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateImage(image, newdata, "") } // Extract the current value imgInfo, etag, err := d.GetImage(image) if err != nil { return err } brief := imgInfo.Writable() data, err := yaml.Dump(&brief, yaml.V2) if err != nil { return err } // Spawn the editor content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor newdata := api.ImagePut{} err = yaml.Load(content, &newdata) if err == nil { err = d.UpdateImage(image, newdata, etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Export. type cmdImageExport struct { global *cmdGlobal image *cmdImage flagVM bool } var cmdImageExportUsage = u.Usage{u.RemoteImage, u.Target(u.File).Optional()} func (c *cmdImageExport) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("export", cmdImageExportUsage...) cmd.Short = i18n.G("Export and download images") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Export and download images The output target is optional and defaults to the working directory.`)) cli.AddBoolFlag(cmd.Flags(), &c.flagVM, "vm", i18n.G("Query virtual machine images")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpImages(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdImageExport) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageExportUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } remoteName := parsed[0].List[0].Get(c.global.conf.DefaultRemote) imageName := parsed[0].List[1].String hasTarget := !parsed[1].Skipped target := parsed[1].Get(".") remoteServer, err := c.global.conf.GetImageServer(remoteName) if err != nil { return err } // Resolve aliases imageType := "" if c.flagVM { imageType = "virtual-machine" } fingerprint := c.image.dereferenceAlias(remoteServer, imageType, imageName) // Default target is current directory targetMeta := fingerprint if hasTarget { if internalUtil.IsDir(target) { targetMeta = filepath.Join(target, targetMeta) } else { targetMeta = target } } targetRootfs := targetMeta + ".root" // Prepare the files dest, err := os.Create(targetMeta) if err != nil { return err } defer func() { _ = dest.Close() }() destRootfs, err := os.Create(targetRootfs) if err != nil { return err } defer func() { _ = destRootfs.Close() }() // Prepare the download request progress := cli.ProgressRenderer{ Format: i18n.G("Exporting the image: %s"), Quiet: c.global.flagQuiet, } req := incus.ImageFileRequest{ MetaFile: io.ReadWriteSeeker(dest), RootfsFile: io.ReadWriteSeeker(destRootfs), ProgressHandler: progress.UpdateProgress, } // Download the image resp, err := remoteServer.GetImageFile(fingerprint, req) if err != nil { _ = os.Remove(targetMeta) _ = os.Remove(targetRootfs) progress.Done("") return err } // Truncate down to size if resp.RootfsSize > 0 { err = destRootfs.Truncate(resp.RootfsSize) if err != nil { return err } } err = dest.Truncate(resp.MetaSize) if err != nil { return err } // Cleanup if resp.RootfsSize == 0 { err := os.Remove(targetRootfs) if err != nil { _ = os.Remove(targetMeta) _ = os.Remove(targetRootfs) progress.Done("") return err } } // Rename files if internalUtil.IsDir(target) { if resp.MetaName != "" { err := os.Rename(targetMeta, filepath.Join(target, resp.MetaName)) if err != nil { _ = os.Remove(targetMeta) _ = os.Remove(targetRootfs) progress.Done("") return err } } if resp.RootfsSize > 0 && resp.RootfsName != "" { err := os.Rename(targetRootfs, filepath.Join(target, resp.RootfsName)) if err != nil { _ = os.Remove(targetMeta) _ = os.Remove(targetRootfs) progress.Done("") return err } } } else if resp.RootfsSize == 0 && hasTarget { if resp.MetaName != "" { extension := strings.SplitN(resp.MetaName, ".", 2)[1] err := os.Rename(targetMeta, fmt.Sprintf("%s.%s", targetMeta, extension)) if err != nil { _ = os.Remove(targetMeta) progress.Done("") return err } } } progress.Done(i18n.G("Image exported successfully!")) return nil } // Import. type cmdImageImport struct { global *cmdGlobal image *cmdImage flagPublic bool flagReuse bool flagAliases []string } // This is a hack of the parser to allow unambiguous parsing of the `incus image import` command in // a single pass. It is meant to be parsed right-to-left, allowing the parser to soft-fail when // strictly-formatted arguments (remotes and KV pairs) aren’t found. When reaching the leftmost // arguments, the compound atom’s smart handling of optional sub-atoms allows to bypass the // otherwise greedy parsing rule. var cmdImageImportUsage = u.Usage{u.Sequence(u.Either(u.Tarball, u.Directory, u.URL), u.Placeholder(i18n.G("rootfs tarball")).Optional()), u.RemoteColonOpt, u.KV.List(0)} func (c *cmdImageImport) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("import", cmdImageImportUsage...) cmd.Short = i18n.G("Import images into the image store") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Import image into the image store Directory import is only available on Linux and must be performed as root.`)) cli.AddBoolFlag(cmd.Flags(), &c.flagPublic, "public", i18n.G("Make image public")) cli.AddBoolFlag(cmd.Flags(), &c.flagReuse, "reuse", i18n.G("If the image alias already exists, delete and create a new one")) cli.AddStringArrayFlag(cmd.Flags(), &c.flagAliases, "alias", i18n.G("New aliases to add to the image")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return nil, cobra.ShellCompDirectiveDefault } if len(args) == 1 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdImageImport) packImageDir(path string) (string, error) { // Quick checks. if os.Geteuid() == -1 { return "", errors.New(i18n.G("Directory import is not available on this platform")) } else if os.Geteuid() != 0 { return "", errors.New(i18n.G("Must run as root to import from directory")) } outFile, err := os.CreateTemp("", "incus_image_") if err != nil { return "", err } defer func() { _ = outFile.Close() }() outFileName := outFile.Name() _, err = subprocess.RunCommand("tar", "-C", path, "--numeric-owner", "--restrict", "--force-local", "--xattrs", "-cJf", outFileName, "rootfs", "templates", "metadata.yaml") if err != nil { return "", err } return outFileName, outFile.Close() } func (c *cmdImageImport) run(cmd *cobra.Command, args []string) error { // Do NOT blindly copy the following parsing line; it performs right-to-left parsing, which in // most cases is NOT what you want. parsed, err := cmdImageImportUsage.Parse(c.global.conf, cmd, args, true) if err != nil { return err } // The first parsed atom contains both the image file and the optional rootfs tarball. imageFile := parsed[0].List[0].String hasRootfsFile := !parsed[0].List[1].Skipped rootfsFile := filepath.Clean(parsed[0].List[1].String) d := parsed[1].RemoteServer if util.PathExists(filepath.Clean(imageFile)) { imageFile = filepath.Clean(imageFile) } else if strings.HasPrefix(imageFile, "http://") { return errors.New(i18n.G("Only https:// is supported for remote image import")) } var createArgs *incus.ImageCreateArgs image := api.ImagesPost{} image.Public = c.flagPublic image.Properties, err = kvToMap(parsed[2]) if err != nil { return err } progress := cli.ProgressRenderer{ Format: i18n.G("Transferring image: %s"), Quiet: c.global.flagQuiet, } imageType := "container" if strings.HasPrefix(imageFile, "https://") { image.Source = &api.ImagesPostSource{} image.Source.Type = "url" image.Source.Mode = "pull" image.Source.Protocol = "direct" image.Source.URL = imageFile createArgs = nil } else { var meta io.ReadCloser var rootfs io.ReadCloser // Open meta if internalUtil.IsDir(imageFile) { imageFile, err = c.packImageDir(imageFile) if err != nil { return err } // remove temp file defer func() { _ = os.Remove(imageFile) }() } meta, err = os.Open(imageFile) if err != nil { return err } defer func() { _ = meta.Close() }() // Open rootfs if hasRootfsFile { rootfs, err = os.Open(rootfsFile) if err != nil { return err } defer func() { _ = rootfs.Close() }() _, ext, _, err := archive.DetectCompressionFile(rootfs) if err != nil { return err } _, err = rootfs.(*os.File).Seek(0, io.SeekStart) if err != nil { return err } if ext == ".qcow2" { imageType = "virtual-machine" } } createArgs = &incus.ImageCreateArgs{ MetaFile: meta, MetaName: filepath.Base(imageFile), RootfsFile: rootfs, RootfsName: filepath.Base(rootfsFile), ProgressHandler: progress.UpdateProgress, Type: imageType, } image.Filename = createArgs.MetaName } // Start the transfer op, err := d.CreateImage(image, createArgs) if err != nil { progress.Done("") return err } // Wait for operation to finish err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } opAPI := op.Get() // Get the fingerprint fingerprint, ok := opAPI.Metadata["fingerprint"].(string) if !ok { return errors.New("Bad fingerprint") } progress.Done(fmt.Sprintf(i18n.G("Image imported with fingerprint: %s"), fingerprint)) // Reformat aliases aliases := []api.ImageAlias{} for _, entry := range c.flagAliases { alias := api.ImageAlias{} alias.Name = entry aliases = append(aliases, alias) } // Delete images if necessary if c.flagReuse { err = deleteImagesByAliases(d, aliases) if err != nil { return err } } // Add the aliases if len(c.flagAliases) > 0 { err = ensureImageAliases(d, aliases, fingerprint) if err != nil { return err } } return nil } // Info. type cmdImageInfo struct { global *cmdGlobal image *cmdImage flagVM bool } var cmdImageInfoUsage = u.Usage{u.RemoteImage} func (c *cmdImageInfo) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("info", cmdImageInfoUsage...) cmd.Short = i18n.G("Show useful information about images") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Show useful information about images`)) cli.AddBoolFlag(cmd.Flags(), &c.flagVM, "vm", i18n.G("Query virtual machine images")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpImages(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdImageInfo) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageInfoUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } remoteName := parsed[0].List[0].Get(c.global.conf.DefaultRemote) imageName := parsed[0].List[1].String remoteServer, err := c.global.conf.GetImageServer(remoteName) if err != nil { return err } // Render info imageType := "" if c.flagVM { imageType = "virtual-machine" } image := c.image.dereferenceAlias(remoteServer, imageType, imageName) info, _, err := remoteServer.GetImage(image) if err != nil { return err } public := i18n.G("no") if info.Public { public = i18n.G("yes") } cached := i18n.G("no") if info.Cached { cached = i18n.G("yes") } autoUpdate := i18n.G("disabled") if info.AutoUpdate { autoUpdate = i18n.G("enabled") } imgType := "container" if info.Type != "" { imgType = info.Type } fmt.Printf(i18n.G("Fingerprint: %s")+"\n", info.Fingerprint) fmt.Printf(i18n.G("Size: %.2fMiB")+"\n", float64(info.Size)/1024.0/1024.0) fmt.Printf(i18n.G("Architecture: %s")+"\n", info.Architecture) fmt.Printf(i18n.G("Type: %s")+"\n", imgType) fmt.Printf(i18n.G("Public: %s")+"\n", public) fmt.Print(i18n.G("Timestamps:") + "\n") if !info.CreatedAt.IsZero() { fmt.Printf(" "+i18n.G("Created: %s")+"\n", info.CreatedAt.Local().Format(dateLayout)) } fmt.Printf(" "+i18n.G("Uploaded: %s")+"\n", info.UploadedAt.Local().Format(dateLayout)) if !info.ExpiresAt.IsZero() { fmt.Printf(" "+i18n.G("Expires: %s")+"\n", info.ExpiresAt.Local().Format(dateLayout)) } else { fmt.Print(" " + i18n.G("Expires: never") + "\n") } if !info.LastUsedAt.IsZero() { fmt.Printf(" "+i18n.G("Last used: %s")+"\n", info.LastUsedAt.Local().Format(dateLayout)) } else { fmt.Print(" " + i18n.G("Last used: never") + "\n") } fmt.Println(i18n.G("Properties:")) for key, value := range info.Properties { fmt.Printf(" %s: %s\n", key, value) } fmt.Println(i18n.G("Aliases:")) for _, alias := range info.Aliases { if alias.Description != "" { fmt.Printf(" - %s (%s)\n", alias.Name, alias.Description) } else { fmt.Printf(" - %s\n", alias.Name) } } fmt.Printf(i18n.G("Cached: %s")+"\n", cached) fmt.Printf(i18n.G("Auto update: %s")+"\n", autoUpdate) if info.UpdateSource != nil { fmt.Println(i18n.G("Source:")) fmt.Printf(" "+i18n.G("Server: %s")+"\n", info.UpdateSource.Server) fmt.Printf(" "+i18n.G("Protocol: %s")+"\n", info.UpdateSource.Protocol) fmt.Printf(" "+i18n.G("Alias: %s")+"\n", info.UpdateSource.Alias) } if len(info.Profiles) == 0 { fmt.Print(i18n.G("Profiles: ") + "[]\n") } else { fmt.Println(i18n.G("Profiles:")) for _, name := range info.Profiles { fmt.Printf(" - %s\n", name) } } return nil } // List. type cmdImageList struct { global *cmdGlobal image *cmdImage flagFormat string flagColumns string flagAllProjects bool } var cmdImageListUsage = u.Usage{u.Colon(u.Remote).Optional(), u.Filter.List(0)} func (c *cmdImageList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdImageListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List images") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List images Filters may be of the = form for property based filtering, or part of the image hash or part of the image alias name. The -c option takes a (optionally comma-separated) list of arguments that control which image attributes to output when displaying in table or csv format. Default column layout is: lfpdasu Column shorthand chars: l - Shortest image alias (and optionally number of other aliases) L - Newline-separated list of all image aliases f - Fingerprint (short) F - Fingerprint (long) p - Whether image is public d - Description e - Project a - Architecture s - Size u - Upload date t - Type`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultImagesColumns, "", i18n.G("Columns")) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("Display images from all projects")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) != 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return c.global.cmpImages(toComplete) } return cmd } const ( defaultImagesColumns = "lfpdatsu" defaultImagesColumnsAllProjects = "elfpdatsu" ) func (c *cmdImageList) parseColumns() ([]imageColumn, error) { columnsShorthandMap := map[rune]imageColumn{ 'a': {i18n.G("ARCHITECTURE"), c.architectureColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 'e': {i18n.G("PROJECT"), c.projectColumnData}, 'f': {i18n.G("FINGERPRINT"), c.fingerprintColumnData}, 'F': {i18n.G("FINGERPRINT"), c.fingerprintFullColumnData}, 'l': {i18n.G("ALIAS"), c.aliasColumnData}, 'L': {i18n.G("ALIASES"), c.aliasesColumnData}, 'p': {i18n.G("PUBLIC"), c.publicColumnData}, 's': {i18n.G("SIZE"), c.sizeColumnData}, 't': {i18n.G("TYPE"), c.typeColumnData}, 'u': {i18n.G("UPLOAD DATE"), c.uploadDateColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []imageColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdImageList) aliasColumnData(image api.Image) string { shortest := c.shortestAlias(image.Aliases) if len(image.Aliases) > 1 { shortest = fmt.Sprintf(i18n.G("%s (%d more)"), shortest, len(image.Aliases)-1) } return shortest } func (c *cmdImageList) aliasesColumnData(image api.Image) string { aliases := []string{} for _, alias := range image.Aliases { aliases = append(aliases, alias.Name) } sort.Strings(aliases) return strings.Join(aliases, "\n") } func (c *cmdImageList) fingerprintColumnData(image api.Image) string { return image.Fingerprint[0:12] } func (c *cmdImageList) fingerprintFullColumnData(image api.Image) string { return image.Fingerprint } func (c *cmdImageList) publicColumnData(image api.Image) string { if image.Public { return i18n.G("yes") } return i18n.G("no") } func (c *cmdImageList) descriptionColumnData(image api.Image) string { return c.findDescription(image.Properties) } func (c *cmdImageList) projectColumnData(image api.Image) string { return image.Project } func (c *cmdImageList) architectureColumnData(image api.Image) string { return image.Architecture } func (c *cmdImageList) sizeColumnData(image api.Image) string { return fmt.Sprintf("%.2fMiB", float64(image.Size)/1024.0/1024.0) } func (c *cmdImageList) typeColumnData(image api.Image) string { if image.Type == "" { return "CONTAINER" } return strings.ToUpper(image.Type) } func (c *cmdImageList) uploadDateColumnData(image api.Image) string { return image.UploadedAt.Local().Format(dateLayout) } func (c *cmdImageList) shortestAlias(list []api.ImageAlias) string { shortest := "" for _, l := range list { if shortest == "" { shortest = l.Name continue } if len(l.Name) != 0 && len(l.Name) < len(shortest) { shortest = l.Name } } return shortest } func (c *cmdImageList) findDescription(props map[string]string) string { for k, v := range props { if k == "description" { return v } } return "" } func (c *cmdImageList) imageShouldShow(filters []string, state *api.Image) bool { if len(filters) == 0 { return true } m := structToMap(state) for _, filter := range filters { found := false if strings.Contains(filter, "=") { membs := strings.SplitN(filter, "=", 2) key := membs[0] var value string if len(membs) < 2 { value = "" } else { value = membs[1] } for configKey, configValue := range state.Properties { if internalFilter.DotPrefixMatch(key, configKey) { // try to test filter value as a regexp regexpValue := value if !strings.Contains(value, "^") && !strings.Contains(value, "$") { regexpValue = "^" + regexpValue + "$" } r, err := regexp.Compile(regexpValue) // if not regexp compatible use original value if err != nil { if value == configValue { found = true break } } else if r.MatchString(configValue) { found = true break } } } val, ok := m[key] if ok && fmt.Sprintf("%v", val) == value { found = true } } else { for _, alias := range state.Aliases { if strings.Contains(alias.Name, filter) { found = true break } } if strings.Contains(state.Fingerprint, filter) { found = true } } if !found { return false } } return true } func (c *cmdImageList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } remoteName := c.global.conf.DefaultRemote if !parsed[0].Skipped { remoteName = parsed[0].List[0].String } filters := parsed[1].StringList remoteServer, err := c.global.conf.GetImageServer(remoteName) if err != nil { return err } // Add project column if --all-projects flag specified and no -c was passed. if c.flagAllProjects && c.flagColumns == defaultImagesColumns { c.flagColumns = defaultImagesColumnsAllProjects } // Process the columns columns, err := c.parseColumns() if err != nil { return err } serverFilters, clientFilters := getServerSupportedFilters(filters, []string{}, false) serverFilters = prepareImageServerFilters(serverFilters, api.Image{}) var allImages, images []api.Image if c.flagAllProjects { allImages, err = remoteServer.GetImagesAllProjectsWithFilter(serverFilters) if err != nil { allImages, err = remoteServer.GetImagesAllProjects() if err != nil { return err } clientFilters = filters } } else { allImages, err = remoteServer.GetImagesWithFilter(serverFilters) if err != nil { allImages, err = remoteServer.GetImages() if err != nil { return err } clientFilters = filters } } data := [][]string{} for _, image := range allImages { if !c.imageShouldShow(clientFilters, &image) { continue } images = append(images, image) row := []string{} for _, column := range columns { row = append(row, column.Data(image)) } data = append(data, row) } sort.Sort(cli.StringList(data)) rawData := make([]*api.Image, len(images)) for i := range images { rawData[i] = &images[i] } headers := []string{} for _, column := range columns { headers = append(headers, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, headers, data, rawData) } // Refresh. type cmdImageRefresh struct { global *cmdGlobal image *cmdImage } var cmdImageRefreshUsage = u.Usage{u.Image.Remote().List(1)} func (c *cmdImageRefresh) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("refresh", cmdImageRefreshUsage...) cmd.Short = i18n.G("Refresh images") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Refresh images`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpImages(toComplete) } return cmd } func (c *cmdImageRefresh) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageRefreshUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } for _, p := range parsed[0].List { d := p.RemoteServer imageName := p.RemoteObject.String image := c.image.dereferenceAlias(d, "", imageName) progress := cli.ProgressRenderer{ Format: i18n.G("Refreshing the image: %s"), Quiet: c.global.flagQuiet, } op, err := d.RefreshImage(image) if err != nil { return err } // Register progress handler _, err = op.AddHandler(progress.UpdateOp) if err != nil { return err } // Wait for the refresh to happen err = op.Wait() if err != nil { return err } opAPI := op.Get() // Check if refreshed refreshed := false flag, ok := opAPI.Metadata["refreshed"] if ok { refreshed = flag == true // nolint:revive } if refreshed { progress.Done(i18n.G("Image refreshed successfully!")) } else { progress.Done(i18n.G("Image already up to date.")) } } return nil } // Show. type cmdImageShow struct { global *cmdGlobal image *cmdImage flagVM bool } var cmdImageShowUsage = u.Usage{u.RemoteImage} func (c *cmdImageShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdImageShowUsage...) cmd.Short = i18n.G("Show image properties") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show image properties`)) cli.AddBoolFlag(cmd.Flags(), &c.flagVM, "vm", i18n.G("Query virtual machine images")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpImages(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdImageShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } remoteName := parsed[0].List[0].Get(c.global.conf.DefaultRemote) imageName := parsed[0].List[1].String remoteServer, err := c.global.conf.GetImageServer(remoteName) if err != nil { return err } // Show properties imageType := "" if c.flagVM { imageType = "virtual-machine" } image := c.image.dereferenceAlias(remoteServer, imageType, imageName) info, _, err := remoteServer.GetImage(image) if err != nil { return err } properties := info.Writable() data, err := yaml.Dump(&properties, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } type cmdImageGetProp struct { global *cmdGlobal image *cmdImage } var cmdImageGetPropUsage = u.Usage{u.RemoteImage, u.Key} func (c *cmdImageGetProp) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get-property", cmdImageGetPropUsage...) cmd.Short = i18n.G("Get image properties") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Get image properties`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpImages(toComplete) } if len(args) == 1 { // individual image prop could complete here return nil, cobra.ShellCompDirectiveNoFileComp } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdImageGetProp) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageGetPropUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } remoteName := parsed[0].List[0].Get(c.global.conf.DefaultRemote) imageName := parsed[0].List[1].String key := parsed[1].String remoteServer, err := c.global.conf.GetImageServer(remoteName) if err != nil { return err } // Get the corresponding property image := c.image.dereferenceAlias(remoteServer, "", imageName) info, _, err := remoteServer.GetImage(image) if err != nil { return err } prop, propFound := info.Properties[key] if !propFound { return errors.New(i18n.G("Property not found")) } fmt.Println(prop) return nil } type cmdImageSetProp struct { global *cmdGlobal image *cmdImage } var cmdImageSetPropUsage = u.Usage{u.Image.Remote(), u.LegacyKV.List(1)} func (c *cmdImageSetProp) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set-property", cmdImageSetPropUsage...) cmd.Short = i18n.G("Set image properties") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Set image properties`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpImages(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdImageSetProp) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer imageName := parsed[0].RemoteObject.String keys, err := kvToMap(parsed[1]) if err != nil { return err } // Get properties image := c.image.dereferenceAlias(d, "", imageName) info, etag, err := d.GetImage(image) if err != nil { return err } properties := info.Writable() maps.Copy(properties.Properties, keys) // Update image err = d.UpdateImage(image, properties, etag) if err != nil { return err } return nil } func (c *cmdImageSetProp) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageSetPropUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } type cmdImageUnsetProp struct { global *cmdGlobal image *cmdImage imageSetProp *cmdImageSetProp } var cmdImageUnsetPropUsage = u.Usage{u.Image.Remote(), u.Key} func (c *cmdImageUnsetProp) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset-property", cmdImageUnsetPropUsage...) cmd.Short = i18n.G("Unset image properties") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Unset image properties`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpImages(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdImageUnsetProp) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageUnsetPropUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return unsetKey(c.imageSetProp, cmd, parsed) } func structToMap(data any) map[string]any { dataBytes, err := json.Marshal(data) if err != nil { return nil } mapData := make(map[string]any) err = json.Unmarshal(dataBytes, &mapData) if err != nil { return nil } return mapData } // prepareImageServerFilter processes and formats filter criteria // for images, ensuring they are in a format that the server can interpret. func prepareImageServerFilters(filters []string, i any) []string { formatedFilters := []string{} for _, filter := range filters { membs := strings.SplitN(filter, "=", 2) if len(membs) == 1 { continue } firstPart := membs[0] if strings.Contains(membs[0], ".") { firstPart = strings.Split(membs[0], ".")[0] } if !structHasField(reflect.TypeOf(i), firstPart) { filter = fmt.Sprintf("properties.%s", filter) } formatedFilters = append(formatedFilters, filter) } return formatedFilters } type cmdImageGenerateMetadata struct { global *cmdGlobal image *cmdImage } var cmdImageGenerateMetadataUsage = u.Usage{u.Target(u.Path)} func (c *cmdImageGenerateMetadata) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("generate-metadata", cmdImageGenerateMetadataUsage...) cmd.Short = i18n.G("Generate a metadata tarball") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Generate a metadata tarball This command produces an incus.tar.xz tarball for use during import with an existing QCOW2 or squashfs disk image. This command will prompt for all of the metadata tarball fields: - Operating system name - Release - Variant - Architecture - Description `)) cmd.RunE = c.run return cmd } func (c *cmdImageGenerateMetadata) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageGenerateMetadataUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } target := parsed[0].String // Setup asker. asker := ask.NewAsker(bufio.NewReader(os.Stdin)) // Create the tarball. metaFile, err := os.Create(target) if err != nil { return err } defer metaFile.Close() // Generate the metadata. timestamp := time.Now().UTC() metadata := api.ImageMetadata{ Properties: map[string]string{}, CreationDate: timestamp.Unix(), } // Question - os metaOS, err := asker.AskString("Operating system name: ", "", nil) if err != nil { return err } metadata.Properties["os"] = metaOS // Question - release metaRelease, err := asker.AskString("Release name: ", "", nil) if err != nil { return err } metadata.Properties["release"] = metaRelease // Question - variant metaVariant, err := asker.AskString("Variant name [default=\"default\"]: ", "default", nil) if err != nil { return err } metadata.Properties["variant"] = metaVariant // Question - architecture var incusArch string metaArchitecture, err := asker.AskString("Architecture name: ", "", func(value string) error { id, err := osarch.ArchitectureID(value) if err != nil { return err } incusArch, err = osarch.ArchitectureName(id) if err != nil { return err } return nil }) if err != nil { return err } metadata.Properties["architecture"] = metaArchitecture metadata.Architecture = incusArch // Question - description defaultDescription := fmt.Sprintf("%s %s (%s) (%s) (%s)", metaOS, metaRelease, metaVariant, metaArchitecture, timestamp.Format("200601021504")) metaDescription, err := asker.AskString(fmt.Sprintf("Description [default=\"%s\"]: ", defaultDescription), defaultDescription, nil) if err != nil { return err } metadata.Properties["description"] = metaDescription // Generate YAML. body, err := yaml.Dump(&metadata, yaml.V2) if err != nil { return err } // Prepare the tarball. tarPipeReader, tarPipeWriter := io.Pipe() tarWriter := tar.NewWriter(tarPipeWriter) // Compress the tarball. chDone := make(chan error) go func() { cmd := exec.Command("xz", "-9", "-c") cmd.Stdin = tarPipeReader cmd.Stdout = metaFile err := cmd.Run() chDone <- err }() // Add metadata.yaml. hdr := &tar.Header{ Name: "metadata.yaml", Size: int64(len(body)), Mode: 0o644, Uname: "root", Gname: "root", ModTime: time.Now(), } err = tarWriter.WriteHeader(hdr) if err != nil { return err } _, err = tarWriter.Write(body) if err != nil { return err } // Close the tarball. err = tarWriter.Close() if err != nil { return err } err = tarPipeWriter.Close() if err != nil { return err } err = <-chDone if err != nil { return err } return nil } incus-7.0.0/cmd/incus/image_alias.go000066400000000000000000000236501517523235500173230ustar00rootroot00000000000000package main import ( "errors" "fmt" "os" "sort" "strings" "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) type imageAliasColumns struct { Name string Data func(api.ImageAliasesEntry) string } type cmdImageAlias struct { global *cmdGlobal image *cmdImage } func (c *cmdImageAlias) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("alias") cmd.Short = i18n.G("Manage image aliases") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage image aliases`)) // Create imageAliasCreateCmd := cmdImageAliasCreate{global: c.global, image: c.image, imageAlias: c} cmd.AddCommand(imageAliasCreateCmd.command()) // Delete imageAliasDeleteCmd := cmdImageAliasDelete{global: c.global, image: c.image, imageAlias: c} cmd.AddCommand(imageAliasDeleteCmd.command()) // List imageAliasListCmd := cmdImageAliasList{global: c.global, image: c.image, imageAlias: c} cmd.AddCommand(imageAliasListCmd.command()) // Rename imageAliasRenameCmd := cmdImageAliasRename{global: c.global, image: c.image, imageAlias: c} cmd.AddCommand(imageAliasRenameCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Create. type cmdImageAliasCreate struct { global *cmdGlobal image *cmdImage imageAlias *cmdImageAlias flagDescription string } var cmdImageAliasCreateUsage = u.Usage{u.NewName(u.Alias).Remote(), u.Fingerprint} func (c *cmdImageAliasCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdImageAliasCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create aliases for existing images") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Create aliases for existing images`)) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Image alias description")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) > 1 { return nil, cobra.ShellCompDirectiveNoFileComp } if len(args) == 0 { return c.global.cmpRemotes(toComplete, true) } remote, _, found := strings.Cut(args[0], ":") if !found { remote = "" } return c.global.cmpImageFingerprintsFromRemote(toComplete, remote) } return cmd } func (c *cmdImageAliasCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageAliasCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer aliasName := parsed[0].RemoteObject.String fingerprint := parsed[1].String // Create the alias alias := api.ImageAliasesPost{} alias.Name = aliasName alias.Target = fingerprint alias.Description = c.flagDescription return d.CreateImageAlias(alias) } // Delete. type cmdImageAliasDelete struct { global *cmdGlobal image *cmdImage imageAlias *cmdImageAlias } var cmdImageAliasDeleteUsage = u.Usage{u.Alias.Remote().List(1)} func (c *cmdImageAliasDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdImageAliasDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete image aliases") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete image aliases`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpImages(toComplete) } return cmd } func (c *cmdImageAliasDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageAliasDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } var errs []error for _, p := range parsed[0].List { d := p.RemoteServer aliasName := p.RemoteObject.String // Delete the alias err = d.DeleteImageAlias(aliasName) if err != nil { errs = append(errs, err) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } // List. type cmdImageAliasList struct { global *cmdGlobal image *cmdImage imageAlias *cmdImageAlias flagFormat string flagColumns string } var cmdImageAliasListUsage = u.Usage{u.Colon(u.Remote).Optional(), u.Filter.List(0)} func (c *cmdImageAliasList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdImageAliasListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List image aliases") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List image aliases Filters may be part of the image hash or part of the image alias name. Default column layout: aftd == Columns == The -c option takes a comma separated list of arguments that control which attributes of image aliases to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: a - Alias f - Fingerprint t - Type d - Description`)) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultImageAliasColumns, "", i18n.G("Columns")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) > 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return c.global.cmpRemotes(toComplete, true) } return cmd } const defaultImageAliasColumns = "aftd" func (c *cmdImageAliasList) aliasShouldShow(filters []string, state *api.ImageAliasesEntry) bool { if len(filters) == 0 { return true } for _, filter := range filters { if strings.Contains(state.Name, filter) || strings.Contains(state.Target, filter) { return true } } return false } func (c *cmdImageAliasList) parseColumns() ([]imageAliasColumns, error) { columnsShorthandMap := map[rune]imageAliasColumns{ 'a': {i18n.G("ALIAS"), c.imageAliasNameColumnData}, 'f': {i18n.G("FINGERPRINT"), c.targetColumnData}, 't': {i18n.G("TYPE"), c.typeColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumntData}, } columnList := strings.Split(c.flagColumns, ",") columns := []imageAliasColumns{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdImageAliasList) imageAliasNameColumnData(imageAlias api.ImageAliasesEntry) string { return imageAlias.Name } func (c *cmdImageAliasList) targetColumnData(imageAlias api.ImageAliasesEntry) string { return imageAlias.Target[0:12] } func (c *cmdImageAliasList) typeColumnData(imageAlias api.ImageAliasesEntry) string { return strings.ToUpper(imageAlias.Type) } func (c *cmdImageAliasList) descriptionColumntData(imageAlias api.ImageAliasesEntry) string { return imageAlias.Description } func (c *cmdImageAliasList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageAliasListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } remoteName := c.global.conf.DefaultRemote if !parsed[0].Skipped { remoteName = parsed[0].List[0].String } filters := parsed[1].StringList remoteServer, err := c.global.conf.GetImageServer(remoteName) if err != nil { return err } // List the aliases aliases, err := remoteServer.GetImageAliases() if err != nil { return err } columns, err := c.parseColumns() if err != nil { return err } // Render the table data := [][]string{} for _, alias := range aliases { if !c.aliasShouldShow(filters, &alias) { continue } if alias.Type == "" { alias.Type = "container" } line := []string{} for _, column := range columns { line = append(line, column.Data(alias)) } data = append(data, line) } sort.Sort(cli.StringList(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, aliases) } // Rename. type cmdImageAliasRename struct { global *cmdGlobal image *cmdImage imageAlias *cmdImageAlias } var cmdImageAliasRenameUsage = u.Usage{u.Alias.Remote(), u.NewName(u.Alias)} func (c *cmdImageAliasRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdImageAliasRenameUsage...) cmd.Aliases = []string{"mv"} cmd.Short = i18n.G("Rename aliases") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Rename aliases`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) > 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return c.global.cmpImages(toComplete) } return cmd } func (c *cmdImageAliasRename) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImageAliasRenameUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer aliasName := parsed[0].RemoteObject.String newAliasName := parsed[1].String // Rename the alias return d.RenameImageAlias(aliasName, api.ImageAliasesEntryPost{Name: newAliasName}) } incus-7.0.0/cmd/incus/image_test.go000066400000000000000000000006331517523235500172050ustar00rootroot00000000000000package main import ( "testing" "github.com/stretchr/testify/assert" "github.com/lxc/incus/v7/shared/api" ) func TestPrepareImageServerFilters(t *testing.T) { filters := []string{"foo", "requirements.secureboot=false", "type=container"} result := prepareImageServerFilters(filters, api.InstanceFull{}) assert.Equal(t, []string{"properties.requirements.secureboot=false", "type=container"}, result) } incus-7.0.0/cmd/incus/import.go000066400000000000000000000052631517523235500164020ustar00rootroot00000000000000package main import ( "fmt" "os" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/units" ) type cmdImport struct { global *cmdGlobal flagStorage string flagConfig []string flagDevice []string } var cmdImportUsage = u.Usage{u.RemoteColonOpt, u.BackupFile, u.NewName(u.Instance).Optional()} func (c *cmdImport) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("import", cmdImportUsage...) cmd.Short = i18n.G("Import instance backups") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Import backups of instances including their snapshots.`)) cmd.Example = cli.FormatSection("", i18n.G( `incus import backup0.tar.gz Create a new instance using backup0.tar.gz as the source.`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagStorage, "storage|s", "", "", i18n.G("Storage pool name")) cli.AddStringArrayFlag(cmd.Flags(), &c.flagConfig, "config|c", i18n.G("Config key/value to apply to the new instance")) cli.AddStringArrayFlag(cmd.Flags(), &c.flagDevice, "device|d", i18n.G("New key/value to apply to a specific device")) return cmd } func (c *cmdImport) run(cmd *cobra.Command, args []string) error { parsed, err := cmdImportUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer backupFile := parsed[1].String instanceName := parsed[2].String var file *os.File if isStdin(backupFile) { file = os.Stdin } else { file, err = os.Open(backupFile) if err != nil { return err } defer func() { _ = file.Close() }() } fstat, err := file.Stat() if err != nil { return err } progress := cli.ProgressRenderer{ Format: i18n.G("Importing instance: %s"), Quiet: c.global.flagQuiet, } createArgs := incus.InstanceBackupArgs{ BackupFile: &ioprogress.ProgressReader{ ReadCloser: file, Tracker: &ioprogress.ProgressTracker{ Length: fstat.Size(), Handler: func(percent int64, speed int64) { progress.UpdateProgress(ioprogress.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, units.GetByteSizeString(speed, 2))}) }, }, }, PoolName: c.flagStorage, Name: instanceName, Config: c.flagConfig, Devices: c.flagDevice, } op, err := d.CreateInstanceFromBackup(createArgs) if err != nil { progress.Done("") return err } // Wait for operation to finish. err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } progress.Done("") return nil } incus-7.0.0/cmd/incus/info.go000066400000000000000000000711241517523235500160220ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "os" "sort" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) type cmdInfo struct { global *cmdGlobal flagShowAccess bool flagShowLog string flagResources bool flagTarget string } var cmdInfoUsage = u.Usage{u.Instance.Optional().Remote()} func (c *cmdInfo) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("info", cmdInfoUsage...) cmd.Short = i18n.G("Show instance or server information") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Show instance or server information`)) cmd.Example = cli.FormatSection("", i18n.G( `incus info [:] [--show-log] For instance information. incus info [:] [--resources] For server information.`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagShowAccess, "show-access", i18n.G("Show the instance's access list")) cli.AddStringFlag(cmd.Flags(), &c.flagShowLog, "show-log", "", "default", i18n.G("Show the instance's recent log entries")) cli.AddBoolFlag(cmd.Flags(), &c.flagResources, "resources", i18n.G("Show the resources available to the server")) cli.AddStringFlag(cmd.Flags(), &c.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdInfo) run(cmd *cobra.Command, args []string) error { parsed, err := cmdInfoUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer hasInstance := !parsed[0].RemoteObject.Skipped instanceName := parsed[0].RemoteObject.String if !hasInstance { return c.remoteInfo(d) } if c.flagShowAccess { access, err := d.GetInstanceAccess(instanceName) if err != nil { return err } data, err := yaml.Dump(access, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } return c.instanceInfo(d, instanceName, c.flagShowLog) } func (c *cmdInfo) renderGPU(gpu api.ResourcesGPUCard, prefix string, initial bool) { if initial { fmt.Print(prefix) } fmt.Printf(i18n.G("NUMA node: %v")+"\n", gpu.NUMANode) if gpu.Vendor != "" { fmt.Printf(prefix+i18n.G("Vendor: %v (%v)")+"\n", gpu.Vendor, gpu.VendorID) } if gpu.Product != "" { fmt.Printf(prefix+i18n.G("Product: %v (%v)")+"\n", gpu.Product, gpu.ProductID) } if gpu.PCIAddress != "" { fmt.Printf(prefix+i18n.G("PCI address: %v")+"\n", gpu.PCIAddress) } if gpu.Driver != "" { fmt.Printf(prefix+i18n.G("Driver: %v (%v)")+"\n", gpu.Driver, gpu.DriverVersion) } if gpu.DRM != nil { fmt.Print(prefix + i18n.G("DRM:") + "\n") fmt.Printf(prefix+" "+i18n.G("ID: %d")+"\n", gpu.DRM.ID) if gpu.DRM.CardName != "" { fmt.Printf(prefix+" "+i18n.G("Card: %s (%s)")+"\n", gpu.DRM.CardName, gpu.DRM.CardDevice) } if gpu.DRM.ControlName != "" { fmt.Printf(prefix+" "+i18n.G("Control: %s (%s)")+"\n", gpu.DRM.ControlName, gpu.DRM.ControlDevice) } if gpu.DRM.RenderName != "" { fmt.Printf(prefix+" "+i18n.G("Render: %s (%s)")+"\n", gpu.DRM.RenderName, gpu.DRM.RenderDevice) } } if gpu.Nvidia != nil { fmt.Print(prefix + i18n.G("NVIDIA information:") + "\n") fmt.Printf(prefix+" "+i18n.G("Architecture: %v")+"\n", gpu.Nvidia.Architecture) fmt.Printf(prefix+" "+i18n.G("Brand: %v")+"\n", gpu.Nvidia.Brand) fmt.Printf(prefix+" "+i18n.G("Model: %v")+"\n", gpu.Nvidia.Model) fmt.Printf(prefix+" "+i18n.G("CUDA Version: %v")+"\n", gpu.Nvidia.CUDAVersion) fmt.Printf(prefix+" "+i18n.G("NVRM Version: %v")+"\n", gpu.Nvidia.NVRMVersion) fmt.Printf(prefix+" "+i18n.G("UUID: %v")+"\n", gpu.Nvidia.UUID) } if gpu.SRIOV != nil { fmt.Print(prefix + i18n.G("SR-IOV information:") + "\n") fmt.Printf(prefix+" "+i18n.G("Current number of VFs: %d")+"\n", gpu.SRIOV.CurrentVFs) fmt.Printf(prefix+" "+i18n.G("Maximum number of VFs: %d")+"\n", gpu.SRIOV.MaximumVFs) if len(gpu.SRIOV.VFs) > 0 { fmt.Printf(prefix+" "+i18n.G("VFs: %d")+"\n", gpu.SRIOV.MaximumVFs) for _, vf := range gpu.SRIOV.VFs { fmt.Print(prefix + " - ") c.renderGPU(vf, prefix+" ", false) } } } if gpu.Mdev != nil { fmt.Print(prefix + i18n.G("Mdev profiles:") + "\n") keys := make([]string, 0, len(gpu.Mdev)) for k := range gpu.Mdev { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { v := gpu.Mdev[k] fmt.Println(prefix + " - " + fmt.Sprintf(i18n.G("%s (%s) (%d available)"), k, v.Name, v.Available)) if v.Description != "" { for _, line := range strings.Split(v.Description, "\n") { fmt.Printf(prefix+" %s\n", line) } } } } } func (c *cmdInfo) renderNIC(nic api.ResourcesNetworkCard, prefix string, initial bool) { if initial { fmt.Print(prefix) } fmt.Printf(i18n.G("NUMA node: %v")+"\n", nic.NUMANode) if nic.Vendor != "" { fmt.Printf(prefix+i18n.G("Vendor: %v (%v)")+"\n", nic.Vendor, nic.VendorID) } if nic.Product != "" { fmt.Printf(prefix+i18n.G("Product: %v (%v)")+"\n", nic.Product, nic.ProductID) } if nic.PCIAddress != "" { fmt.Printf(prefix+i18n.G("PCI address: %v")+"\n", nic.PCIAddress) } if nic.Driver != "" { fmt.Printf(prefix+i18n.G("Driver: %v (%v)")+"\n", nic.Driver, nic.DriverVersion) } if len(nic.Ports) > 0 { fmt.Print(prefix + i18n.G("Ports:") + "\n") for _, port := range nic.Ports { fmt.Printf(prefix+" "+i18n.G("- Port %d (%s)")+"\n", port.Port, port.Protocol) fmt.Printf(prefix+" "+i18n.G("ID: %s")+"\n", port.ID) if port.Address != "" { fmt.Printf(prefix+" "+i18n.G("Address: %s")+"\n", port.Address) } if port.SupportedModes != nil { fmt.Printf(prefix+" "+i18n.G("Supported modes: %s")+"\n", strings.Join(port.SupportedModes, ", ")) } if port.SupportedPorts != nil { fmt.Printf(prefix+" "+i18n.G("Supported ports: %s")+"\n", strings.Join(port.SupportedPorts, ", ")) } if port.PortType != "" { fmt.Printf(prefix+" "+i18n.G("Port type: %s")+"\n", port.PortType) } if port.TransceiverType != "" { fmt.Printf(prefix+" "+i18n.G("Transceiver type: %s")+"\n", port.TransceiverType) } fmt.Printf(prefix+" "+i18n.G("Auto negotiation: %v")+"\n", port.AutoNegotiation) fmt.Printf(prefix+" "+i18n.G("Link detected: %v")+"\n", port.LinkDetected) if port.LinkSpeed > 0 { fmt.Printf(prefix+" "+i18n.G("Link speed: %dMbit/s (%s duplex)")+"\n", port.LinkSpeed, port.LinkDuplex) } if port.Infiniband != nil { fmt.Print(prefix + " " + i18n.G("Infiniband:") + "\n") if port.Infiniband.IsSMName != "" { fmt.Printf(prefix+" "+i18n.G("IsSM: %s (%s)")+"\n", port.Infiniband.IsSMName, port.Infiniband.IsSMDevice) } if port.Infiniband.MADName != "" { fmt.Printf(prefix+" "+i18n.G("MAD: %s (%s)")+"\n", port.Infiniband.MADName, port.Infiniband.MADDevice) } if port.Infiniband.VerbName != "" { fmt.Printf(prefix+" "+i18n.G("Verb: %s (%s)")+"\n", port.Infiniband.VerbName, port.Infiniband.VerbDevice) } } } } if nic.SRIOV != nil { fmt.Print(prefix + i18n.G("SR-IOV information:") + "\n") fmt.Printf(prefix+" "+i18n.G("Current number of VFs: %d")+"\n", nic.SRIOV.CurrentVFs) fmt.Printf(prefix+" "+i18n.G("Maximum number of VFs: %d")+"\n", nic.SRIOV.MaximumVFs) if len(nic.SRIOV.VFs) > 0 { fmt.Printf(prefix+" "+i18n.G("VFs: %d")+"\n", nic.SRIOV.MaximumVFs) for _, vf := range nic.SRIOV.VFs { fmt.Print(prefix + " - ") c.renderNIC(vf, prefix+" ", false) } } } } func (c *cmdInfo) renderDisk(disk api.ResourcesStorageDisk, prefix string, initial bool) { if initial { fmt.Print(prefix) } fmt.Printf(i18n.G("NUMA node: %v")+"\n", disk.NUMANode) fmt.Printf(prefix+i18n.G("ID: %s")+"\n", disk.ID) fmt.Printf(prefix+i18n.G("Device: %s")+"\n", disk.Device) if disk.Model != "" { fmt.Printf(prefix+i18n.G("Model: %s")+"\n", disk.Model) } if disk.Type != "" { fmt.Printf(prefix+i18n.G("Type: %s")+"\n", disk.Type) } fmt.Printf(prefix+i18n.G("Size: %s")+"\n", units.GetByteSizeStringIEC(int64(disk.Size), 2)) if disk.WWN != "" { fmt.Printf(prefix+i18n.G("WWN: %s")+"\n", disk.WWN) } fmt.Printf(prefix+i18n.G("Read-Only: %v")+"\n", disk.ReadOnly) fmt.Printf(prefix+i18n.G("Removable: %v")+"\n", disk.Removable) if len(disk.Partitions) != 0 { fmt.Print(prefix + i18n.G("Partitions:") + "\n") for _, partition := range disk.Partitions { fmt.Printf(prefix+" "+i18n.G("- Partition %d")+"\n", partition.Partition) fmt.Printf(prefix+" "+i18n.G("ID: %s")+"\n", partition.ID) fmt.Printf(prefix+" "+i18n.G("Device: %s")+"\n", partition.Device) fmt.Printf(prefix+" "+i18n.G("Read-Only: %v")+"\n", partition.ReadOnly) fmt.Printf(prefix+" "+i18n.G("Size: %s")+"\n", units.GetByteSizeStringIEC(int64(partition.Size), 2)) } } } func (c *cmdInfo) renderCPU(cpu api.ResourcesCPUSocket, prefix string) { if cpu.Vendor != "" { fmt.Printf(prefix+i18n.G("Vendor: %v")+"\n", cpu.Vendor) } if cpu.Name != "" { fmt.Printf(prefix+i18n.G("Name: %v")+"\n", cpu.Name) } if cpu.Cache != nil { fmt.Print(prefix + i18n.G("Caches:") + "\n") for _, cache := range cpu.Cache { fmt.Printf(prefix+" "+i18n.G("- Level %d (type: %s): %s")+"\n", cache.Level, cache.Type, units.GetByteSizeStringIEC(int64(cache.Size), 0)) } } fmt.Print(prefix + i18n.G("Cores:") + "\n") for _, core := range cpu.Cores { fmt.Printf(prefix+" - "+i18n.G("Core %d")+"\n", core.Core) fmt.Printf(prefix+" "+i18n.G("Frequency: %vMhz")+"\n", core.Frequency) fmt.Print(prefix + " " + i18n.G("Threads:") + "\n") for _, thread := range core.Threads { fmt.Printf(prefix+" - "+i18n.G("%d (id: %d, online: %v, NUMA node: %v)")+"\n", thread.Thread, thread.ID, thread.Online, thread.NUMANode) } } if cpu.Frequency > 0 { if cpu.FrequencyTurbo > 0 && cpu.FrequencyMinimum > 0 { fmt.Printf(prefix+i18n.G("Frequency: %vMhz (min: %vMhz, max: %vMhz)")+"\n", cpu.Frequency, cpu.FrequencyMinimum, cpu.FrequencyTurbo) } else { fmt.Printf(prefix+i18n.G("Frequency: %vMhz")+"\n", cpu.Frequency) } } } func (c *cmdInfo) renderUSB(usb api.ResourcesUSBDevice, prefix string) { fmt.Printf(prefix+i18n.G("Vendor: %v")+"\n", usb.Vendor) fmt.Printf(prefix+i18n.G("Vendor ID: %v")+"\n", usb.VendorID) fmt.Printf(prefix+i18n.G("Product: %v")+"\n", usb.Product) fmt.Printf(prefix+i18n.G("Product ID: %v")+"\n", usb.ProductID) fmt.Printf(prefix+i18n.G("Bus Address: %v")+"\n", usb.BusAddress) fmt.Printf(prefix+i18n.G("Device Address: %v")+"\n", usb.DeviceAddress) if len(usb.Serial) > 0 { fmt.Printf(prefix+i18n.G("Serial Number: %v")+"\n", usb.Serial) } } func (c *cmdInfo) renderPCI(pci api.ResourcesPCIDevice, prefix string) { fmt.Printf(prefix+i18n.G("Address: %v")+"\n", pci.PCIAddress) fmt.Printf(prefix+i18n.G("Vendor: %v")+"\n", pci.Vendor) fmt.Printf(prefix+i18n.G("Vendor ID: %v")+"\n", pci.VendorID) fmt.Printf(prefix+i18n.G("Product: %v")+"\n", pci.Product) fmt.Printf(prefix+i18n.G("Product ID: %v")+"\n", pci.ProductID) fmt.Printf(prefix+i18n.G("NUMA node: %v")+"\n", pci.NUMANode) fmt.Printf(prefix+i18n.G("IOMMU group: %v")+"\n", pci.IOMMUGroup) fmt.Printf(prefix+i18n.G("Driver: %v")+"\n", pci.Driver) } func (c *cmdInfo) renderSerial(serial api.ResourcesSerialDevice, prefix string) { fmt.Printf(prefix+i18n.G("Id: %v")+"\n", serial.ID) fmt.Printf(prefix+i18n.G("Device: %v")+"\n", serial.Device) fmt.Printf(prefix+i18n.G("DeviceID: %v")+"\n", serial.DeviceID) fmt.Printf(prefix+i18n.G("DevicePath: %v")+"\n", serial.DevicePath) fmt.Printf(prefix+i18n.G("Vendor: %v")+"\n", serial.Vendor) fmt.Printf(prefix+i18n.G("Vendor ID: %v")+"\n", serial.VendorID) fmt.Printf(prefix+i18n.G("Product: %v")+"\n", serial.Product) fmt.Printf(prefix+i18n.G("Product ID: %v")+"\n", serial.ProductID) fmt.Printf(prefix+i18n.G("Driver: %v")+"\n", serial.Driver) } func (c *cmdInfo) remoteInfo(d incus.InstanceServer) error { // Targeting if c.flagTarget != "" { if !d.IsClustered() { return errors.New(i18n.G("To use --target, the destination remote must be a cluster")) } d = d.UseTarget(c.flagTarget) } if c.flagResources { if !d.HasExtension("resources_v2") { return errors.New(i18n.G("The server doesn't implement the newer v2 resources API")) } resources, err := d.GetServerResources() if err != nil { return err } // System fmt.Print(i18n.G("System:") + "\n") if resources.System.UUID != "" { fmt.Printf(" "+i18n.G("UUID: %v")+"\n", resources.System.UUID) } if resources.System.Vendor != "" { fmt.Printf(" "+i18n.G("Vendor: %v")+"\n", resources.System.Vendor) } if resources.System.Product != "" { fmt.Printf(" "+i18n.G("Product: %v")+"\n", resources.System.Product) } if resources.System.Family != "" { fmt.Printf(" "+i18n.G("Family: %v")+"\n", resources.System.Family) } if resources.System.Version != "" { fmt.Printf(" "+i18n.G("Version: %v")+"\n", resources.System.Version) } if resources.System.Sku != "" { fmt.Printf(" "+i18n.G("SKU: %v")+"\n", resources.System.Sku) } if resources.System.Serial != "" { fmt.Printf(" "+i18n.G("Serial number: %v")+"\n", resources.System.Serial) } if resources.System.Type != "" { fmt.Printf(" "+i18n.G("Type: %s")+"\n", resources.System.Type) } // System: Chassis if resources.System.Chassis != nil { fmt.Print(i18n.G(" Chassis:") + "\n") if resources.System.Chassis.Vendor != "" { fmt.Printf(" "+i18n.G("Vendor: %s")+"\n", resources.System.Chassis.Vendor) } if resources.System.Chassis.Type != "" { fmt.Printf(" "+i18n.G("Type: %s")+"\n", resources.System.Chassis.Type) } if resources.System.Chassis.Version != "" { fmt.Printf(" "+i18n.G("Version: %s")+"\n", resources.System.Chassis.Version) } if resources.System.Chassis.Serial != "" { fmt.Printf(" "+i18n.G("Serial: %s")+"\n", resources.System.Chassis.Serial) } } // System: Motherboard if resources.System.Motherboard != nil { fmt.Print(i18n.G(" Motherboard:") + "\n") if resources.System.Motherboard.Vendor != "" { fmt.Printf(" "+i18n.G("Vendor: %s")+"\n", resources.System.Motherboard.Vendor) } if resources.System.Motherboard.Product != "" { fmt.Printf(" "+i18n.G("Product: %s")+"\n", resources.System.Motherboard.Product) } if resources.System.Motherboard.Serial != "" { fmt.Printf(" "+i18n.G("Serial: %s")+"\n", resources.System.Motherboard.Serial) } if resources.System.Motherboard.Version != "" { fmt.Printf(" "+i18n.G("Version: %s")+"\n", resources.System.Motherboard.Version) } } // System: Firmware if resources.System.Firmware != nil { fmt.Print(i18n.G(" Firmware:") + "\n") if resources.System.Firmware.Vendor != "" { fmt.Printf(" "+i18n.G("Vendor: %s")+"\n", resources.System.Firmware.Vendor) } if resources.System.Firmware.Version != "" { fmt.Printf(" "+i18n.G("Version: %s")+"\n", resources.System.Firmware.Version) } if resources.System.Firmware.Date != "" { fmt.Printf(" "+i18n.G("Date: %s")+"\n", resources.System.Firmware.Date) } } // Load fmt.Print("\n" + i18n.G("Load:") + "\n") if resources.Load.Processes > 0 { fmt.Printf(" "+i18n.G("Processes: %d")+"\n", resources.Load.Processes) fmt.Printf(" "+i18n.G("Average: %.2f %.2f %.2f")+"\n", resources.Load.Average1Min, resources.Load.Average5Min, resources.Load.Average10Min) } // CPU if len(resources.CPU.Sockets) == 1 { fmt.Print("\n" + i18n.G("CPU:") + "\n") fmt.Printf(" "+i18n.G("Architecture: %s")+"\n", resources.CPU.Architecture) c.renderCPU(resources.CPU.Sockets[0], " ") } else if len(resources.CPU.Sockets) > 1 { fmt.Print(i18n.G("CPUs:") + "\n") fmt.Printf(" "+i18n.G("Architecture: %s")+"\n", resources.CPU.Architecture) for _, cpu := range resources.CPU.Sockets { fmt.Printf(" "+i18n.G("Socket %d:")+"\n", cpu.Socket) c.renderCPU(cpu, " ") } } // Memory fmt.Print("\n" + i18n.G("Memory:") + "\n") if resources.Memory.HugepagesTotal > 0 { fmt.Print(" " + i18n.G("Hugepages:"+"\n")) fmt.Printf(" "+i18n.G("Free: %v")+"\n", units.GetByteSizeStringIEC(int64(resources.Memory.HugepagesTotal-resources.Memory.HugepagesUsed), 2)) fmt.Printf(" "+i18n.G("Used: %v")+"\n", units.GetByteSizeStringIEC(int64(resources.Memory.HugepagesUsed), 2)) fmt.Printf(" "+i18n.G("Total: %v")+"\n", units.GetByteSizeStringIEC(int64(resources.Memory.HugepagesTotal), 2)) } if len(resources.Memory.Nodes) > 1 { fmt.Print(" " + i18n.G("NUMA nodes:"+"\n")) for _, node := range resources.Memory.Nodes { fmt.Printf(" "+i18n.G("Node %d:"+"\n"), node.NUMANode) if node.HugepagesTotal > 0 { fmt.Print(" " + i18n.G("Hugepages:"+"\n")) fmt.Printf(" "+i18n.G("Free: %v")+"\n", units.GetByteSizeStringIEC(int64(node.HugepagesTotal-node.HugepagesUsed), 2)) fmt.Printf(" "+i18n.G("Used: %v")+"\n", units.GetByteSizeStringIEC(int64(node.HugepagesUsed), 2)) fmt.Printf(" "+i18n.G("Total: %v")+"\n", units.GetByteSizeStringIEC(int64(node.HugepagesTotal), 2)) } fmt.Printf(" "+i18n.G("Free: %v")+"\n", units.GetByteSizeStringIEC(int64(node.Total-node.Used), 2)) fmt.Printf(" "+i18n.G("Used: %v")+"\n", units.GetByteSizeStringIEC(int64(node.Used), 2)) fmt.Printf(" "+i18n.G("Total: %v")+"\n", units.GetByteSizeStringIEC(int64(node.Total), 2)) } } fmt.Printf(" "+i18n.G("Free: %v")+"\n", units.GetByteSizeStringIEC(int64(resources.Memory.Total-resources.Memory.Used), 2)) fmt.Printf(" "+i18n.G("Used: %v")+"\n", units.GetByteSizeStringIEC(int64(resources.Memory.Used), 2)) fmt.Printf(" "+i18n.G("Total: %v")+"\n", units.GetByteSizeStringIEC(int64(resources.Memory.Total), 2)) // GPUs if len(resources.GPU.Cards) == 1 { fmt.Print("\n" + i18n.G("GPU:") + "\n") c.renderGPU(resources.GPU.Cards[0], " ", true) } else if len(resources.GPU.Cards) > 1 { fmt.Print("\n" + i18n.G("GPUs:") + "\n") for id, gpu := range resources.GPU.Cards { fmt.Printf(" "+i18n.G("Card %d:")+"\n", id) c.renderGPU(gpu, " ", true) } } // Network interfaces if len(resources.Network.Cards) == 1 { fmt.Print("\n" + i18n.G("NIC:") + "\n") c.renderNIC(resources.Network.Cards[0], " ", true) } else if len(resources.Network.Cards) > 1 { fmt.Print("\n" + i18n.G("NICs:") + "\n") for id, nic := range resources.Network.Cards { fmt.Printf(" "+i18n.G("Card %d:")+"\n", id) c.renderNIC(nic, " ", true) } } // Storage if len(resources.Storage.Disks) == 1 { fmt.Print("\n" + i18n.G("Disk:") + "\n") c.renderDisk(resources.Storage.Disks[0], " ", true) } else if len(resources.Storage.Disks) > 1 { fmt.Print("\n" + i18n.G("Disks:") + "\n") for id, nic := range resources.Storage.Disks { fmt.Printf(" "+i18n.G("Disk %d:")+"\n", id) c.renderDisk(nic, " ", true) } } // USB if len(resources.USB.Devices) == 1 { fmt.Print("\n" + i18n.G("USB device:") + "\n") c.renderUSB(resources.USB.Devices[0], " ") } else if len(resources.USB.Devices) > 1 { fmt.Print("\n" + i18n.G("USB devices:") + "\n") for id, usb := range resources.USB.Devices { fmt.Printf(" "+i18n.G("Device %d:")+"\n", id) c.renderUSB(usb, " ") } } // PCI if len(resources.PCI.Devices) == 1 { fmt.Print("\n" + i18n.G("PCI device:") + "\n") c.renderPCI(resources.PCI.Devices[0], " ") } else if len(resources.PCI.Devices) > 1 { fmt.Print("\n" + i18n.G("PCI devices:") + "\n") for id, pci := range resources.PCI.Devices { fmt.Printf(" "+i18n.G("Device %d:")+"\n", id) c.renderPCI(pci, " ") } } // Serial if len(resources.Serial.Devices) == 1 { fmt.Print("\n" + i18n.G("Serial device:") + "\n") c.renderSerial(resources.Serial.Devices[0], " ") } else if len(resources.Serial.Devices) > 1 { fmt.Print("\n" + i18n.G("Serial devices:") + "\n") for id, serial := range resources.Serial.Devices { fmt.Printf(" "+i18n.G("Device %d:")+"\n", id) c.renderSerial(serial, " ") } } return nil } serverStatus, _, err := d.GetServer() if err != nil { return err } data, err := yaml.Dump(&serverStatus, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } func (c *cmdInfo) instanceInfo(d incus.InstanceServer, name string, showLog string) error { // Quick checks. if c.flagTarget != "" { return errors.New(i18n.G("--target cannot be used with instances")) } // Get the full instance data. inst, _, err := d.GetInstanceFull(name) if err != nil { return err } fmt.Printf(i18n.G("Name: %s")+"\n", inst.Name) fmt.Printf(i18n.G("Description: %s")+"\n", inst.Description) fmt.Printf(i18n.G("Status: %s")+"\n", strings.ToUpper(inst.Status)) instType := inst.Type if instType == "" { instType = "container" } if util.IsTrue(inst.ExpandedConfig["volatile.container.oci"]) { instType = fmt.Sprintf("%s (%s)", instType, i18n.G("application")) } if inst.Ephemeral { instType = fmt.Sprintf("%s (%s)", instType, i18n.G("ephemeral")) } fmt.Printf(i18n.G("Type: %s")+"\n", instType) fmt.Printf(i18n.G("Architecture: %s")+"\n", inst.Architecture) if inst.Location != "" && d.IsClustered() { fmt.Printf(i18n.G("Location: %s")+"\n", inst.Location) } if inst.State.Pid != 0 { fmt.Printf(i18n.G("PID: %d")+"\n", inst.State.Pid) } if !inst.CreatedAt.IsZero() { fmt.Printf(i18n.G("Created: %s")+"\n", inst.CreatedAt.Local().Format(dateLayout)) } if !inst.LastUsedAt.IsZero() { fmt.Printf(i18n.G("Last Used: %s")+"\n", inst.LastUsedAt.Local().Format(dateLayout)) } if inst.State.Pid != 0 { if !inst.State.StartedAt.IsZero() { fmt.Printf(i18n.G("Started: %s")+"\n", inst.State.StartedAt.Local().Format(dateLayout)) } // Operating System info if inst.State.OSInfo != nil { fmt.Println("\n" + i18n.G("Operating System:")) osInfo := fmt.Sprintf(" %s: %s\n", i18n.G("OS"), inst.State.OSInfo.OS) osInfo += fmt.Sprintf(" %s: %s\n", i18n.G("OS Version"), inst.State.OSInfo.OSVersion) osInfo += fmt.Sprintf(" %s: %s\n", i18n.G("Kernel Version"), inst.State.OSInfo.KernelVersion) osInfo += fmt.Sprintf(" %s: %s\n", i18n.G("Hostname"), inst.State.OSInfo.Hostname) osInfo += fmt.Sprintf(" %s: %s\n", i18n.G("FQDN"), inst.State.OSInfo.FQDN) fmt.Print(osInfo) } fmt.Println("\n" + i18n.G("Resources:")) // Processes fmt.Printf(" "+i18n.G("Processes: %d")+"\n", inst.State.Processes) // Disk usage diskInfo := "" if inst.State.Disk != nil { for entry, disk := range inst.State.Disk { if disk.Usage != 0 { diskInfo += fmt.Sprintf(" %s: %s\n", entry, units.GetByteSizeStringIEC(disk.Usage, 2)) } } } if diskInfo != "" { fmt.Printf(" %s\n", i18n.G("Disk usage:")) fmt.Print(diskInfo) } // CPU usage cpuInfo := "" if inst.State.CPU.Usage != 0 { cpuInfo += fmt.Sprintf(" %s: %v\n", i18n.G("CPU usage (in seconds)"), inst.State.CPU.Usage/1000000000) } if cpuInfo != "" { fmt.Printf(" %s\n", i18n.G("CPU usage:")) fmt.Print(cpuInfo) } // Memory usage memoryInfo := "" if inst.State.Memory.Usage != 0 { memoryInfo += fmt.Sprintf(" %s: %s\n", i18n.G("Memory (current)"), units.GetByteSizeStringIEC(inst.State.Memory.Usage, 2)) } if inst.State.Memory.UsagePeak != 0 { memoryInfo += fmt.Sprintf(" %s: %s\n", i18n.G("Memory (peak)"), units.GetByteSizeStringIEC(inst.State.Memory.UsagePeak, 2)) } if inst.State.Memory.SwapUsage != 0 { memoryInfo += fmt.Sprintf(" %s: %s\n", i18n.G("Swap (current)"), units.GetByteSizeStringIEC(inst.State.Memory.SwapUsage, 2)) } if inst.State.Memory.SwapUsagePeak != 0 { memoryInfo += fmt.Sprintf(" %s: %s\n", i18n.G("Swap (peak)"), units.GetByteSizeStringIEC(inst.State.Memory.SwapUsagePeak, 2)) } if memoryInfo != "" { fmt.Printf(" %s\n", i18n.G("Memory usage:")) fmt.Print(memoryInfo) } // Network usage and IP info networkInfo := "" if inst.State.Network != nil { network := inst.State.Network netNames := make([]string, 0, len(network)) for netName := range network { netNames = append(netNames, netName) } sort.Strings(netNames) for _, netName := range netNames { networkInfo += fmt.Sprintf(" %s:\n", netName) networkInfo += fmt.Sprintf(" %s: %s\n", i18n.G("Type"), network[netName].Type) networkInfo += fmt.Sprintf(" %s: %s\n", i18n.G("State"), strings.ToUpper(network[netName].State)) if network[netName].HostName != "" { networkInfo += fmt.Sprintf(" %s: %s\n", i18n.G("Host interface"), network[netName].HostName) } if network[netName].Hwaddr != "" { networkInfo += fmt.Sprintf(" %s: %s\n", i18n.G("MAC address"), network[netName].Hwaddr) } if network[netName].Mtu != 0 { networkInfo += fmt.Sprintf(" %s: %d\n", i18n.G("MTU"), network[netName].Mtu) } networkInfo += fmt.Sprintf(" %s: %s\n", i18n.G("Bytes received"), units.GetByteSizeString(network[netName].Counters.BytesReceived, 2)) networkInfo += fmt.Sprintf(" %s: %s\n", i18n.G("Bytes sent"), units.GetByteSizeString(network[netName].Counters.BytesSent, 2)) networkInfo += fmt.Sprintf(" %s: %d\n", i18n.G("Packets received"), network[netName].Counters.PacketsReceived) networkInfo += fmt.Sprintf(" %s: %d\n", i18n.G("Packets sent"), network[netName].Counters.PacketsSent) networkInfo += fmt.Sprintf(" %s:\n", i18n.G("IP addresses")) for _, addr := range network[netName].Addresses { if addr.Family == "inet" { networkInfo += fmt.Sprintf(" %s: %s/%s (%s)\n", addr.Family, addr.Address, addr.Netmask, addr.Scope) } else { networkInfo += fmt.Sprintf(" %s: %s/%s (%s)\n", addr.Family, addr.Address, addr.Netmask, addr.Scope) } } } } if networkInfo != "" { fmt.Printf(" %s\n", i18n.G("Network usage:")) fmt.Print(networkInfo) } } // List snapshots firstSnapshot := true if len(inst.Snapshots) > 0 { snapData := [][]string{} for _, snap := range inst.Snapshots { if firstSnapshot { fmt.Println("\n" + i18n.G("Snapshots:")) } var row []string fields := strings.Split(snap.Name, instance.SnapshotDelimiter) row = append(row, fields[len(fields)-1]) if !snap.CreatedAt.IsZero() { row = append(row, snap.CreatedAt.Local().Format(dateLayout)) } else { row = append(row, " ") } if !snap.ExpiresAt.IsZero() { row = append(row, snap.ExpiresAt.Local().Format(dateLayout)) } else { row = append(row, " ") } if snap.Stateful { row = append(row, "YES") } else { row = append(row, "NO") } firstSnapshot = false snapData = append(snapData, row) } snapHeader := []string{ i18n.G("Name"), i18n.G("Taken at"), i18n.G("Expires at"), i18n.G("Stateful"), } _ = cli.RenderTable(os.Stdout, cli.TableFormatTable, snapHeader, snapData, inst.Snapshots) } // List backups firstBackup := true if len(inst.Backups) > 0 { backupData := [][]string{} for _, backup := range inst.Backups { if firstBackup { fmt.Println("\n" + i18n.G("Backups:")) } var row []string row = append(row, backup.Name) if !backup.CreatedAt.IsZero() { row = append(row, backup.CreatedAt.Local().Format(dateLayout)) } else { row = append(row, " ") } if !backup.ExpiresAt.IsZero() { row = append(row, backup.ExpiresAt.Local().Format(dateLayout)) } else { row = append(row, " ") } if backup.InstanceOnly { row = append(row, "YES") } else { row = append(row, "NO") } if backup.OptimizedStorage { row = append(row, "YES") } else { row = append(row, "NO") } firstBackup = false backupData = append(backupData, row) } backupHeader := []string{ i18n.G("Name"), i18n.G("Taken at"), i18n.G("Expires at"), i18n.G("Instance Only"), i18n.G("Optimized Storage"), } _ = cli.RenderTable(os.Stdout, cli.TableFormatTable, backupHeader, backupData, inst.Backups) } if showLog != "" { var log io.Reader if showLog == "default" { switch inst.Type { case "container": showLog = "lxc.log" case "virtual-machine": showLog = "qemu.log" default: return fmt.Errorf(i18n.G("Unsupported instance type: %s"), inst.Type) } } log, err = d.GetInstanceLogfile(name, showLog) if err != nil { return err } stuff, err := io.ReadAll(log) if err != nil { return err } fmt.Printf("\n"+i18n.G("Log (%s):")+"\n\n%s\n", showLog, strings.TrimSpace(string(stuff))) } return nil } incus-7.0.0/cmd/incus/launch.go000066400000000000000000000077131517523235500163440ustar00rootroot00000000000000package main import ( "fmt" "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdLaunch struct { global *cmdGlobal init *cmdCreate flagConsole string } func (c *cmdLaunch) command() *cobra.Command { cmd := c.init.command() cmd.Use = cli.U("launch", cmdCreateUsage...) cmd.Short = i18n.G("Create and start instances from images") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Create and start instances from images`)) cmd.Example = cli.FormatSection("", i18n.G( `incus launch images:debian/12 u1 Create and start a container named u1 incus launch images:debian/12 u1 < config.yaml Create and start a container with configuration from config.yaml incus launch images:debian/12 u2 -t aws:t2.micro Create and start a container using the same size as an AWS t2.micro (1 vCPU, 1GiB of RAM) incus launch images:debian/12 v1 --vm -c limits.cpu=4 -c limits.memory=4GiB Create and start a virtual machine with 4 vCPUs and 4GiB of RAM incus launch images:debian/12 v2 --vm -d root,size=50GiB -d root,io.bus=nvme Create and start a virtual machine, overriding the disk size and bus`)) cmd.Hidden = false cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagConsole, "console", "", "console", i18n.G("Immediately attach to the console")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) != 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return c.global.cmpImages(toComplete) } return cmd } func (c *cmdLaunch) run(cmd *cobra.Command, args []string) error { conf := c.global.conf parsed, err := cmdCreateUsage.Parse(conf, cmd, args) if err != nil { return err } // Call the matching code from init p, err := c.init.create(conf, parsed, true) if err != nil { return err } d := p.RemoteServer instanceName := p.RemoteObject.String // Check if the instance was started by the server. if d.HasExtension("instance_create_start") { // Handle console attach if c.flagConsole != "" { console := cmdConsole{} console.global = c.global console.flagType = c.flagConsole consoleErr := console.console(d, instanceName) if consoleErr != nil { // Check if still running. state, _, err := d.GetInstanceState(instanceName) if err != nil { return err } if state.StatusCode != api.Stopped { return consoleErr } console.flagShowLog = true return console.console(d, instanceName) } } return nil } // Start the instance if !c.global.flagQuiet { fmt.Printf(i18n.G("Starting %s")+"\n", formatRemote(conf, p)) } req := api.InstanceStatePut{ Action: "start", Timeout: -1, } op, err := d.UpdateInstanceState(instanceName, req, "") if err != nil { return err } progress := cli.ProgressRenderer{ Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } // Wait for operation to finish err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") projectArg := "" if conf.ProjectOverride != "" && conf.ProjectOverride != api.ProjectDefaultName { projectArg = " --project " + conf.ProjectOverride } return fmt.Errorf("%s\n"+i18n.G("Try `incus info --show-log %s%s` for more info"), err, formatRemote(conf, p), projectArg) } progress.Done("") // Handle console attach if c.flagConsole != "" { console := cmdConsole{} console.global = c.global console.flagType = c.flagConsole consoleErr := console.console(d, instanceName) if consoleErr != nil { // Check if still running. state, _, err := d.GetInstanceState(instanceName) if err != nil { return err } if state.StatusCode != api.Stopped { return consoleErr } console.flagShowLog = true return console.console(d, instanceName) } } return nil } incus-7.0.0/cmd/incus/list.go000066400000000000000000000622721517523235500160460ustar00rootroot00000000000000package main import ( "errors" "fmt" "net" "os" "reflect" "slices" "sort" "strconv" "strings" "sync" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) type column struct { Name string Data columnData NeedsState bool NeedsSnapshots bool } type columnData func(api.InstanceFull) string type cmdList struct { global *cmdGlobal flagColumns string flagFast bool flagFormat string flagAllProjects bool rawFormat bool shorthandFilters map[string]func(*api.Instance, *api.InstanceState, string) bool } var cmdListUsage = u.Usage{u.RemoteColonOpt, u.Filter.List(0)} func (c *cmdList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List instances Default column layout: ns46tS Fast column layout: nsacPt A single keyword like "web" which will list any instance with a name starting by "web". A regular expression on the instance name. (e.g. .*web.*01$). A key/value pair referring to a configuration item. For those, the namespace can be abbreviated to the smallest unambiguous identifier. A key/value pair where the key is a shorthand. Multiple values must be delimited by ','. Available shorthands: - type={instance type} - status={instance current lifecycle status} - architecture={instance architecture} - location={location name} - ipv4={ip or CIDR} - ipv6={ip or CIDR} Examples: - "user.blah=abc" will list all instances with the "blah" user property set to "abc". - "u.blah=abc" will do the same - "security.privileged=true" will list all privileged instances - "s.privileged=true" will do the same - "type=container" will list all container instances - "type=container status=running" will list all running container instances A regular expression matching a configuration item or its value. (e.g. volatile.eth0.hwaddr=10:66:6a:.*). When multiple filters are passed, they are added one on top of the other, selecting instances which satisfy them all. == Columns == The -c option takes a comma separated list of arguments that control which instance attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: 4 - IPv4 address 6 - IPv6 address a - Architecture b - Storage pool c - Creation date d - Description D - disk usage e - Project name l - Last used date m - Memory usage M - Memory usage (%) n - Name N - Number of Processes p - PID of the instance's init process P - Profiles s - State S - Number of snapshots t - Type (persistent or ephemeral) u - CPU usage (in seconds) U - Started date L - Location of the instance (e.g. its cluster member) f - Base Image Fingerprint (short) F - Base Image Fingerprint (long) Custom columns are defined with "[config:|devices:]key[:name][:maxWidth]": KEY: The (extended) config or devices key to display. If [config:|devices:] is omitted then it defaults to config key. NAME: Name to display in the column header. Defaults to the key if not specified or empty. MAXWIDTH: Max width of the column (longer results are truncated). Defaults to -1 (unlimited). Use 0 to limit to the column header size.`)) cmd.Example = cli.FormatSection("", i18n.G( `incus list -c nFs46,volatile.eth0.hwaddr:MAC,config:image.os,devices:eth0.parent:ETHP Show instances using the "NAME", "BASE IMAGE", "STATE", "IPV4", "IPV6" and "MAC" columns. "BASE IMAGE", "MAC" and "IMAGE OS" are custom columns generated from instance configuration keys. "ETHP" is a custom column generated from a device key. incus list -c ns,user.comment:comment List instances with their running state and user comment.`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultColumns, "", i18n.G("Columns")) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddBoolFlag(cmd.Flags(), &c.flagFast, "fast", i18n.G("Fast mode (same as --columns=nsacPt)")) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("Display instances from all projects")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } const ( defaultColumns = "ns46tSL" defaultColumnsAllProjects = "ens46tSL" configColumnType = "config" deviceColumnType = "devices" ) func (c *cmdList) shouldShow(filters []string, inst *api.Instance, state *api.InstanceState) bool { c.mapShorthandFilters() for _, filter := range filters { membs := strings.SplitN(filter, "=", 2) key := membs[0] var value string if len(membs) < 2 { value = "" } else { value = membs[1] } if c.evaluateShorthandFilter(key, value, inst, state) { continue } return false } return true } func (c *cmdList) evaluateShorthandFilter(key string, value string, inst *api.Instance, state *api.InstanceState) bool { const shorthandValueDelimiter = "," shorthandFilterFunction, isShorthandFilter := c.shorthandFilters[strings.ToLower(key)] if !isShorthandFilter { return false } if !strings.Contains(value, shorthandValueDelimiter) { return shorthandFilterFunction(inst, state, value) } matched := false for _, curValue := range strings.Split(value, shorthandValueDelimiter) { if shorthandFilterFunction(inst, state, curValue) { matched = true } } return matched } func (c *cmdList) listInstances(d incus.InstanceServer, instances []api.Instance, filters []string, columns []column) error { threads := min(len(instances), 10) // Shortcut when needing state and snapshot info. hasSnapshots := false hasState := false for _, column := range columns { if column.NeedsSnapshots { hasSnapshots = true } if column.NeedsState { hasState = true } } if hasSnapshots && hasState { cInfo := []api.InstanceFull{} cInfoLock := sync.Mutex{} cInfoQueue := make(chan string, threads) cInfoWg := sync.WaitGroup{} for range threads { cInfoWg.Add(1) go func() { for { cName, more := <-cInfoQueue if !more { break } state, _, err := d.GetInstanceFull(cName) if err != nil { continue } cInfoLock.Lock() cInfo = append(cInfo, *state) cInfoLock.Unlock() } cInfoWg.Done() }() } for _, info := range instances { cInfoQueue <- info.Name } close(cInfoQueue) cInfoWg.Wait() return c.showInstances(cInfo, filters, columns) } cStates := map[string]*api.InstanceState{} cStatesLock := sync.Mutex{} cStatesQueue := make(chan string, threads) cStatesWg := sync.WaitGroup{} cSnapshots := map[string][]api.InstanceSnapshot{} cSnapshotsLock := sync.Mutex{} cSnapshotsQueue := make(chan string, threads) cSnapshotsWg := sync.WaitGroup{} for range threads { cStatesWg.Add(1) go func() { for { cName, more := <-cStatesQueue if !more { break } state, _, err := d.GetInstanceState(cName) if err != nil { continue } cStatesLock.Lock() cStates[cName] = state cStatesLock.Unlock() } cStatesWg.Done() }() cSnapshotsWg.Add(1) go func() { for { cName, more := <-cSnapshotsQueue if !more { break } snaps, err := d.GetInstanceSnapshots(cName) if err != nil { continue } cSnapshotsLock.Lock() cSnapshots[cName] = snaps cSnapshotsLock.Unlock() } cSnapshotsWg.Done() }() } for _, inst := range instances { for _, column := range columns { if column.NeedsState && inst.IsActive() { cStatesLock.Lock() _, ok := cStates[inst.Name] cStatesLock.Unlock() if ok { continue } cStatesLock.Lock() cStates[inst.Name] = nil cStatesLock.Unlock() cStatesQueue <- inst.Name } if column.NeedsSnapshots { cSnapshotsLock.Lock() _, ok := cSnapshots[inst.Name] cSnapshotsLock.Unlock() if ok { continue } cSnapshotsLock.Lock() cSnapshots[inst.Name] = nil cSnapshotsLock.Unlock() cSnapshotsQueue <- inst.Name } } } close(cStatesQueue) close(cSnapshotsQueue) cStatesWg.Wait() cSnapshotsWg.Wait() // Convert to Instance data := make([]api.InstanceFull, len(instances)) for i := range instances { data[i].Instance = instances[i] data[i].State = cStates[instances[i].Name] data[i].Snapshots = cSnapshots[instances[i].Name] } return c.showInstances(data, filters, columns) } func (c *cmdList) showInstances(instances []api.InstanceFull, filters []string, columns []column) error { // Generate the table data data := [][]string{} instancesFiltered := []api.InstanceFull{} for _, inst := range instances { if !c.shouldShow(filters, &inst.Instance, inst.State) { continue } instancesFiltered = append(instancesFiltered, inst) col := []string{} for _, column := range columns { col = append(col, column.Data(inst)) } data = append(data, col) } sort.Sort(cli.SortColumnsNaturally(data)) headers := []string{} for _, column := range columns { headers = append(headers, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, headers, data, instancesFiltered) } func (c *cmdList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer filters := parsed[1].StringList if c.global.flagProject != "" && c.flagAllProjects { return errors.New(i18n.G("Can't specify --project with --all-projects")) } // Check for raw unit formatting. formatFields := strings.SplitN(c.flagFormat, ",", 2) if len(formatFields) == 2 { formatOptions := strings.Split(formatFields[1], ",") if slices.Contains(formatOptions, "raw") { c.rawFormat = true } } // Get the list of columns columns, needsData, err := c.parseColumns(d.IsClustered()) if err != nil { return err } if needsData && d.HasExtension("container_full") { // Using the GetInstancesFull shortcut var instances []api.InstanceFull serverFilters, clientFilters := getServerSupportedFilters(filters, []string{"ipv4", "ipv6"}, true) serverFilters = prepareInstanceServerFilters(serverFilters, api.InstanceFull{}) if c.flagAllProjects { instances, err = d.GetInstancesFullAllProjectsWithFilter(api.InstanceTypeAny, serverFilters) } else { instances, err = d.GetInstancesFullWithFilter(api.InstanceTypeAny, serverFilters) } if err != nil { return err } return c.showInstances(instances, clientFilters, columns) } // Get the list of instances var instances []api.Instance serverFilters, clientFilters := getServerSupportedFilters(filters, []string{"ipv4", "ipv6"}, true) serverFilters = prepareInstanceServerFilters(serverFilters, api.Instance{}) if c.flagAllProjects { instances, err = d.GetInstancesAllProjectsWithFilter(api.InstanceTypeAny, serverFilters) } else { instances, err = d.GetInstancesWithFilter(api.InstanceTypeAny, serverFilters) } if err != nil { return err } // Fetch any remaining data and render the table return c.listInstances(d, instances, clientFilters, columns) } func (c *cmdList) parseColumns(clustered bool) ([]column, bool, error) { columnsShorthandMap := map[rune]column{ '4': {i18n.G("IPV4"), c.ip4ColumnData, true, false}, '6': {i18n.G("IPV6"), c.ip6ColumnData, true, false}, 'a': {i18n.G("ARCHITECTURE"), c.architectureColumnData, false, false}, 'b': {i18n.G("STORAGE POOL"), c.storagePoolColumnData, false, false}, 'c': {i18n.G("CREATED AT"), c.createdColumnData, false, false}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData, false, false}, 'D': {i18n.G("DISK USAGE"), c.diskUsageColumnData, true, false}, 'e': {i18n.G("PROJECT"), c.projectColumnData, false, false}, 'f': {i18n.G("BASE IMAGE"), c.baseImageColumnData, false, false}, 'F': {i18n.G("BASE IMAGE"), c.baseImageFullColumnData, false, false}, 'l': {i18n.G("LAST USED AT"), c.lastUsedColumnData, false, false}, 'm': {i18n.G("MEMORY USAGE"), c.memoryUsageColumnData, true, false}, 'M': {i18n.G("MEMORY USAGE%"), c.memoryUsagePercentColumnData, true, false}, 'n': {i18n.G("NAME"), c.nameColumnData, false, false}, 'N': {i18n.G("PROCESSES"), c.numberOfProcessesColumnData, true, false}, 'p': {i18n.G("PID"), c.pidColumnData, true, false}, 'P': {i18n.G("PROFILES"), c.profilesColumnData, false, false}, 'S': {i18n.G("SNAPSHOTS"), c.numberSnapshotsColumnData, false, true}, 's': {i18n.G("STATE"), c.statusColumnData, false, false}, 't': {i18n.G("TYPE"), c.typeColumnData, false, false}, 'u': {i18n.G("CPU USAGE"), c.cpuUsageSecondsColumnData, true, false}, 'U': {i18n.G("STARTED AT"), c.startedColumnData, true, false}, } // Add project column if --all-projects flag specified and // no one of --fast or --c was passed if c.flagAllProjects { if c.flagColumns == defaultColumns { c.flagColumns = defaultColumnsAllProjects } } if c.flagFast { if c.flagColumns != defaultColumns && c.flagColumns != defaultColumnsAllProjects { // --columns was specified too return nil, false, errors.New(i18n.G("Can't specify --fast with --columns")) } if c.flagColumns == defaultColumnsAllProjects { c.flagColumns = "ensacPt" } else { c.flagColumns = "nsacPt" } } if clustered { columnsShorthandMap['L'] = column{ i18n.G("LOCATION"), c.locationColumnData, false, false, } } columnList := strings.Split(c.flagColumns, ",") columns := []column{} needsData := false for _, columnEntry := range columnList { if columnEntry == "" { return nil, false, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } // Config keys always contain a period, parse anything without a // period as a series of shorthand runes. if !strings.Contains(columnEntry, ".") { if !clustered { if columnEntry != defaultColumns && columnEntry != defaultColumnsAllProjects { if strings.ContainsAny(columnEntry, "L") { return nil, false, errors.New(i18n.G("Can't specify column L when not clustered")) } } columnEntry = strings.ReplaceAll(columnEntry, "L", "") } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, false, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) if column.NeedsState || column.NeedsSnapshots { needsData = true } } } else { cc := strings.Split(columnEntry, ":") colType := configColumnType if (cc[0] == configColumnType || cc[0] == deviceColumnType) && len(cc) > 1 { colType = cc[0] cc = slices.Delete(cc, 0, 1) } if len(cc) > 3 { return nil, false, fmt.Errorf(i18n.G("Invalid config key column format (too many fields): '%s'"), columnEntry) } k := cc[0] if colType == configColumnType { _, err := instance.ConfigKeyChecker(k, api.InstanceTypeAny) if err != nil { return nil, false, fmt.Errorf(i18n.G("Invalid config key '%s' in '%s'"), k, columnEntry) } } column := column{Name: k} if len(cc) > 1 { if len(cc[1]) == 0 && len(cc) != 3 { return nil, false, fmt.Errorf(i18n.G("Invalid name in '%s', empty string is only allowed when defining maxWidth"), columnEntry) } column.Name = cc[1] } maxWidth := -1 if len(cc) > 2 { temp, err := strconv.ParseInt(cc[2], 10, 64) if err != nil { return nil, false, fmt.Errorf(i18n.G("Invalid max width (must be an integer) '%s' in '%s'"), cc[2], columnEntry) } if temp < -1 { return nil, false, fmt.Errorf(i18n.G("Invalid max width (must -1, 0 or a positive integer) '%s' in '%s'"), cc[2], columnEntry) } if temp == 0 { maxWidth = len(column.Name) } else { maxWidth = int(temp) } } if colType == configColumnType { column.Data = func(cInfo api.InstanceFull) string { v, ok := cInfo.Config[k] if !ok { v = cInfo.ExpandedConfig[k] } // Truncate the data according to the max width. A negative max width // indicates there is no effective limit. if maxWidth > 0 && len(v) > maxWidth { return v[:maxWidth] } return v } } if colType == deviceColumnType { column.Data = func(cInfo api.InstanceFull) string { d := strings.SplitN(k, ".", 2) if len(d) == 1 || len(d) > 2 { return "" } v, ok := cInfo.Devices[d[0]][d[1]] if !ok { v = cInfo.ExpandedDevices[d[0]][d[1]] } // Truncate the data according to the max width. A negative max width // indicates there is no effective limit. if maxWidth > 0 && len(v) > maxWidth { return v[:maxWidth] } return v } } columns = append(columns, column) if column.NeedsState || column.NeedsSnapshots { needsData = true } } } return columns, needsData, nil } func (c *cmdList) getBaseImage(cInfo api.InstanceFull, long bool) string { v, ok := cInfo.Config["volatile.base_image"] if !ok { return "" } if !long && len(v) >= 12 { v = v[:12] } return v } func (c *cmdList) baseImageColumnData(cInfo api.InstanceFull) string { return c.getBaseImage(cInfo, false) } func (c *cmdList) baseImageFullColumnData(cInfo api.InstanceFull) string { return c.getBaseImage(cInfo, true) } func (c *cmdList) nameColumnData(cInfo api.InstanceFull) string { return cInfo.Name } func (c *cmdList) descriptionColumnData(cInfo api.InstanceFull) string { return cInfo.Description } func (c *cmdList) statusColumnData(cInfo api.InstanceFull) string { return strings.ToUpper(cInfo.Status) } func (c *cmdList) ip4ColumnData(cInfo api.InstanceFull) string { if cInfo.IsActive() && cInfo.State != nil && cInfo.State.Network != nil { ipv4s := []string{} for netName, network := range cInfo.State.Network { if network.Type == "loopback" { continue } for _, addr := range network.Addresses { if slices.Contains([]string{"link", "local"}, addr.Scope) { continue } if addr.Family == "inet" { ipv4s = append(ipv4s, fmt.Sprintf("%s (%s)", addr.Address, netName)) } } } sort.Sort(sort.Reverse(sort.StringSlice(ipv4s))) return strings.Join(ipv4s, "\n") } return "" } func (c *cmdList) ip6ColumnData(cInfo api.InstanceFull) string { if cInfo.IsActive() && cInfo.State != nil && cInfo.State.Network != nil { ipv6s := []string{} for netName, network := range cInfo.State.Network { if network.Type == "loopback" { continue } for _, addr := range network.Addresses { if slices.Contains([]string{"link", "local"}, addr.Scope) { continue } if addr.Family == "inet6" { ipv6s = append(ipv6s, fmt.Sprintf("%s (%s)", addr.Address, netName)) } } } sort.Sort(sort.Reverse(sort.StringSlice(ipv6s))) return strings.Join(ipv6s, "\n") } return "" } func (c *cmdList) projectColumnData(cInfo api.InstanceFull) string { return cInfo.Project } func (c *cmdList) memoryUsageColumnData(cInfo api.InstanceFull) string { if cInfo.IsActive() && cInfo.State != nil && cInfo.State.Memory.Usage > 0 { if !c.rawFormat { return units.GetByteSizeStringIEC(cInfo.State.Memory.Usage, 2) } return strconv.FormatInt(cInfo.State.Memory.Usage, 10) } return "" } func (c *cmdList) memoryUsagePercentColumnData(cInfo api.InstanceFull) string { if cInfo.IsActive() && cInfo.State != nil && cInfo.State.Memory.Usage > 0 { if cInfo.ExpandedConfig["limits.memory"] != "" { memorylimit := cInfo.ExpandedConfig["limits.memory"] if strings.Contains(memorylimit, "%") { return "" } val, err := units.ParseByteSizeString(cInfo.ExpandedConfig["limits.memory"]) if err == nil && val > 0 { return fmt.Sprintf("%.1f%%", (float64(cInfo.State.Memory.Usage)/float64(val))*float64(100)) } } } return "" } func (c *cmdList) cpuUsageSecondsColumnData(cInfo api.InstanceFull) string { if cInfo.IsActive() && cInfo.State != nil && cInfo.State.CPU.Usage > 0 { return fmt.Sprintf("%ds", cInfo.State.CPU.Usage/1000000000) } return "" } func (c *cmdList) diskUsageColumnData(cInfo api.InstanceFull) string { rootDisk, _, _ := instance.GetRootDiskDevice(cInfo.ExpandedDevices) if cInfo.State != nil && cInfo.State.Disk != nil && cInfo.State.Disk[rootDisk].Usage > 0 { if !c.rawFormat { return units.GetByteSizeStringIEC(cInfo.State.Disk[rootDisk].Usage, 2) } return strconv.FormatInt(cInfo.State.Disk[rootDisk].Usage, 10) } return "" } func (c *cmdList) typeColumnData(cInfo api.InstanceFull) string { ret := strings.ToUpper(cInfo.Type) if ret == "" { ret = "CONTAINER" } if util.IsTrue(cInfo.ExpandedConfig["volatile.container.oci"]) { ret = fmt.Sprintf("%s (%s)", ret, i18n.G("APP")) } if cInfo.Ephemeral { ret = fmt.Sprintf("%s (%s)", ret, i18n.G("EPHEMERAL")) } return ret } func (c *cmdList) numberSnapshotsColumnData(cInfo api.InstanceFull) string { if cInfo.Snapshots != nil { return fmt.Sprintf("%d", len(cInfo.Snapshots)) } return "0" } func (c *cmdList) pidColumnData(cInfo api.InstanceFull) string { if cInfo.IsActive() && cInfo.State != nil { return fmt.Sprintf("%d", cInfo.State.Pid) } return "" } func (c *cmdList) architectureColumnData(cInfo api.InstanceFull) string { return cInfo.Architecture } func (c *cmdList) storagePoolColumnData(cInfo api.InstanceFull) string { for _, v := range cInfo.ExpandedDevices { if v["type"] == "disk" && v["path"] == "/" { return v["pool"] } } return "" } func (c *cmdList) profilesColumnData(cInfo api.InstanceFull) string { return strings.Join(cInfo.Profiles, "\n") } func (c *cmdList) createdColumnData(cInfo api.InstanceFull) string { if !cInfo.CreatedAt.IsZero() { return cInfo.CreatedAt.Local().Format(dateLayout) } return "" } func (c *cmdList) startedColumnData(cInfo api.InstanceFull) string { if cInfo.State != nil && !cInfo.State.StartedAt.IsZero() { return cInfo.State.StartedAt.Local().Format(dateLayout) } return "" } func (c *cmdList) lastUsedColumnData(cInfo api.InstanceFull) string { if !cInfo.LastUsedAt.IsZero() { return cInfo.LastUsedAt.Local().Format(dateLayout) } return "" } func (c *cmdList) numberOfProcessesColumnData(cInfo api.InstanceFull) string { if cInfo.IsActive() && cInfo.State != nil { return fmt.Sprintf("%d", cInfo.State.Processes) } return "" } func (c *cmdList) locationColumnData(cInfo api.InstanceFull) string { return cInfo.Location } func (c *cmdList) matchByNet(cState *api.InstanceState, query string, family string) bool { // Skip if no state. if cState == nil { return false } // Skip if no network data. if cState.Network == nil { return false } // Consider the filter as a CIDR. _, subnet, _ := net.ParseCIDR(query) // Go through interfaces. for _, network := range cState.Network { for _, addr := range network.Addresses { if family == "ipv6" && addr.Family != "inet6" { continue } if family == "ipv4" && addr.Family != "inet" { continue } if addr.Address == query { return true } if subnet != nil { ipAddr := net.ParseIP(addr.Address) if ipAddr != nil && subnet.Contains(ipAddr) { return true } } } } return false } func (c *cmdList) matchByIPV6(_ *api.Instance, cState *api.InstanceState, query string) bool { return c.matchByNet(cState, query, "ipv6") } func (c *cmdList) matchByIPV4(_ *api.Instance, cState *api.InstanceState, query string) bool { return c.matchByNet(cState, query, "ipv4") } func (c *cmdList) mapShorthandFilters() { c.shorthandFilters = map[string]func(*api.Instance, *api.InstanceState, string) bool{ "ipv4": c.matchByIPV4, "ipv6": c.matchByIPV6, } } // prepareInstanceServerFilters processes and formats filter criteria // for instances, ensuring they are in a format that the server can interpret. func prepareInstanceServerFilters(filters []string, i any) []string { formatedFilters := []string{} for _, filter := range filters { membs := strings.SplitN(filter, "=", 2) key := membs[0] if len(membs) == 1 { regexpValue := key if !strings.Contains(key, "^") && !strings.Contains(key, "$") { regexpValue = "^" + regexpValue + "$" } filter = fmt.Sprintf("name=(%s|^%s.*)", regexpValue, key) } else { firstPart := key if strings.Contains(key, ".") { firstPart = strings.Split(key, ".")[0] } if !structHasField(reflect.TypeOf(i), firstPart) { filter = fmt.Sprintf("expanded_config.%s", filter) } if key == "state" { filter = fmt.Sprintf("status=%s", membs[1]) } } formatedFilters = append(formatedFilters, filter) } return formatedFilters } incus-7.0.0/cmd/incus/list_test.go000066400000000000000000000154641517523235500171060ustar00rootroot00000000000000package main import ( "bytes" "math/rand" "strconv" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/shared/api" ) func TestShouldShow(t *testing.T) { list := cmdList{} inst := &api.Instance{ Name: "foo", ExpandedConfig: map[string]string{ "security.privileged": "1", "user.blah": "abc", "image.os": "Debian", "image.description": "Debian buster amd64 (20200429_05:24)", }, Status: "Running", Location: "mem-brain", Type: "Container", ExpandedDevices: map[string]map[string]string{ "eth0": { "name": "eth0", "type": "nic", "parent": "mybr0", "nictype": "bridged", }, }, InstancePut: api.InstancePut{ Architecture: "potato", Description: "Something which does something", }, } state := &api.InstanceState{ Network: map[string]api.InstanceStateNetwork{ "eth0": { Addresses: []api.InstanceStateNetworkAddress{ { Family: "inet", Address: "10.29.85.156", }, { Family: "inet6", Address: "fd42:72a:89ac:e457:1266:6aff:fe83:8301", }, }, }, }, } if !list.shouldShow([]string{"ipv4=10.29.85.0/24"}, inst, state) { t.Errorf("net=10.29.85.0/24 filter didn't work") } if list.shouldShow([]string{"ipv4=10.29.85.0/32"}, inst, state) { t.Errorf("net=10.29.85.0/32 filter did work but should not") } if !list.shouldShow([]string{"ipv4=10.29.85.156"}, inst, state) { t.Errorf("net=10.29.85.156 filter did not work") } if !list.shouldShow([]string{"ipv6=fd42:72a:89ac:e457:1266:6aff:fe83:8301"}, inst, state) { t.Errorf("net=fd42:72a:89ac:e457:1266:6aff:fe83:8301 filter didn't work") } if list.shouldShow([]string{"ipv6=fd42:072a:89ac:e457:1266:6aff:fe83:ffff/128"}, inst, state) { t.Errorf("net=1net=fd42:072a:89ac:e457:1266:6aff:fe83:ffff/128 filter did work but should not") } if !list.shouldShow([]string{"ipv6=fd42:72a:89ac:e457:1266:6aff:fe83:ffff/1"}, inst, state) { t.Errorf("net=fd42:72a:89ac:e457:1266:6aff:fe83:ffff/1 filter filter didn't work") } } // Used by TestColumns and TestInvalidColumns. const ( shorthand = "46abcdDefFlmMnNpPsStuUL" alphanum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" ) func TestColumns(t *testing.T) { keys := make([]string, 0, len(instance.InstanceConfigKeysAny)+len(instance.InstanceConfigKeysContainer)+len(instance.InstanceConfigKeysVM)) for k := range instance.InstanceConfigKeysAny { keys = append(keys, k) // Test compatibility with 'config:' prefix. keys = append(keys, "config:"+k) } for k := range instance.InstanceConfigKeysContainer { keys = append(keys, k) // Test compatibility with 'config:' prefix. keys = append(keys, "config:"+k) } for k := range instance.InstanceConfigKeysVM { keys = append(keys, k) // Test compatibility with 'config:' prefix. keys = append(keys, "config:"+k) } // Test with 'devices:'. keys = append(keys, "devices:eth0.parent.rand") keys = append(keys, "devices:root.path") randShorthand := func(buffer *bytes.Buffer) { buffer.WriteByte(shorthand[rand.Intn(len(shorthand))]) } randString := func(buffer *bytes.Buffer) { l := rand.Intn(20) if l == 0 { l = rand.Intn(20) + 20 } for range l { buffer.WriteByte(alphanum[rand.Intn(len(alphanum))]) } } randConfigKey := func(buffer *bytes.Buffer) { // Unconditionally prepend a comma so that we don't create an invalid // column string, redundant commas will be handled immediately prior // to parsing the string. buffer.WriteRune(',') switch rand.Intn(4) { case 0: buffer.WriteString(keys[rand.Intn(len(keys))]) case 1: buffer.WriteString("user.") randString(buffer) case 2: buffer.WriteString("environment.") randString(buffer) case 3: if rand.Intn(2) == 0 { buffer.WriteString(instance.ConfigVolatilePrefix) randString(buffer) buffer.WriteString(".hwaddr") } else { buffer.WriteString(instance.ConfigVolatilePrefix) randString(buffer) buffer.WriteString(".name") } } // Randomize the optional fields in a single shot. Empty names are legal // when specifying the max width, append an extra colon in this case. opt := rand.Intn(8) if opt&1 != 0 { buffer.WriteString(":") randString(buffer) } else if opt != 0 { buffer.WriteString(":") } switch opt { case 2, 3: buffer.WriteString(":-1") case 4, 5: buffer.WriteString(":0") case 6, 7: buffer.WriteRune(':') buffer.WriteString(strconv.FormatUint(uint64(rand.Uint32()), 10)) } // Unconditionally append a comma so that we don't create an invalid // column string, redundant commas will be handled immediately prior // to parsing the string. buffer.WriteRune(',') } for range 1000 { go func() { var buffer bytes.Buffer l := rand.Intn(10) if l == 0 { l = rand.Intn(10) + 10 } num := l for range l { switch rand.Intn(5) { case 0: if buffer.Len() > 0 { buffer.WriteRune(',') num-- } else { randShorthand(&buffer) } case 1, 2: randShorthand(&buffer) case 3, 4: randConfigKey(&buffer) } } // Generate the column string, removing any leading, trailing or duplicate commas. raw := removeDuplicatesFromString(strings.Trim(buffer.String(), ","), ",") list := cmdList{flagColumns: raw} clustered := strings.Contains(raw, "L") columns, _, err := list.parseColumns(clustered) if err != nil { t.Errorf("Failed to parse columns string. Input: %s, Error: %s", raw, err) } if len(columns) != num { t.Errorf("Did not generate correct number of columns. Expected: %d, Actual: %d, Input: %s", num, len(columns), raw) } }() } } func removeDuplicatesFromString(s string, sep string) string { dup := sep + sep for strings.Contains(s, dup) { s = strings.ReplaceAll(s, dup, sep) } return s } func TestInvalidColumns(t *testing.T) { run := func(raw string) { list := cmdList{flagColumns: raw} _, _, err := list.parseColumns(true) if err == nil { t.Errorf("Expected error from parseColumns, received nil. Input: %s", raw) } } for _, v := range alphanum { if !strings.ContainsRune(shorthand, v) { run(string(v)) } } run(",") run(",a") run("a,") run("4,,6") run(".") run(":") run("::") run(".key:") run("user.key:") run("user.key::") run(":user.key") run(":user.key:0") run("user.key::-2") run("user.key:name:-2") run("volatile") run("base_image") run("volatile.image") run("config:") run("config:image") run("devices:eth0") } func TestPrepareInstanceServerFilters(t *testing.T) { filters := []string{"foo", "user.a=blah", "name=v1", "state=running"} result := prepareInstanceServerFilters(filters, api.InstanceFull{}) assert.Equal(t, []string{"name=(^foo$|^foo.*)", "expanded_config.user.a=blah", "name=v1", "status=running"}, result) } incus-7.0.0/cmd/incus/main.go000066400000000000000000000444111517523235500160120ustar00rootroot00000000000000package main import ( "bufio" "errors" "fmt" "math/rand" "os" "os/user" "path" "slices" "github.com/kballard/go-shellquote" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/ask" config "github.com/lxc/incus/v7/shared/cliconfig" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/termios" "github.com/lxc/incus/v7/shared/util" ) type cmdGlobal struct { asker ask.Asker conf *config.Config confPath string cmd *cobra.Command ret int flagForceLocal bool flagHelp bool flagHelpAll bool flagLogDebug bool flagLogVerbose bool flagProject string flagQuiet bool flagVersion bool flagSubCmds bool } var commandFooter = i18n.G(`Use "{{.CommandPath}} [] --help" for more information about a command.`) func usageTemplate() string { return color.UsagePrefix + `{{if .Runnable}} {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}} {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}} ` + color.AliasesPrefix + ` {{.NameAndAliases}}{{end}}{{if .HasExample}} ` + color.ExamplesPrefix + ` {{.Example}}{{end}}{{if .HasAvailableSubCommands}} ` + color.AvailableCommandsPrefix + `{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} ` + color.FlagsPrefix + ` {{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} ` + color.GlobalFlagsPrefix + ` {{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}} ` + color.AdditionalHelpTopicsPrefix + `{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}} {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}} ` + commandFooter + `{{end}} ` } func usageTemplateSubCmds() string { return color.UsagePrefix + `{{if .Runnable}} {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}} {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}} ` + color.AliasesPrefix + ` {{.NameAndAliases}}{{end}}{{if .HasExample}} ` + color.ExamplesPrefix + ` {{.Example}}{{end}}{{if .HasAvailableSubCommands}} ` + color.AvailableCommandsPrefix + `{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} {{rpad .Name .NamePadding }} {{.Short}}{{if .HasSubCommands}}{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} {{rpad .Name .NamePadding }} {{.Short}}{{if .HasSubCommands}}{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} {{rpad .Name .NamePadding }} {{.Short}}{{if .HasSubCommands}}{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{end}}{{end}}{{end}}{{end}}{{end}}{{end}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} ` + color.FlagsPrefix + ` {{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} ` + color.GlobalFlagsPrefix + ` {{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}} ` + color.AdditionalHelpTopicsPrefix + `{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}} {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}} ` + commandFooter + `{{end}} ` } func aliases() []string { c, err := config.LoadConfig("") if err != nil { return nil } aliases := make([]string, 0, len(defaultAliases)+len(c.Aliases)) // Add default aliases for alias := range defaultAliases { aliases = append(aliases, alias) } // Add user-defined aliases for alias := range c.Aliases { aliases = append(aliases, alias) } return aliases } func createApp() (*cobra.Command, *cmdGlobal, error) { // Load config. conf, err := config.LoadConfig("") if err != nil { return nil, nil, fmt.Errorf(i18n.G("Failed to load configuration: %s"), err) } // Initialize colors. color.Init(conf.Defaults.NoColor) // Setup the parser. app := &cobra.Command{} app.Use = "incus" app.Short = i18n.G("Command line client for Incus") app.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Command line client for Incus All of Incus's features can be driven through the various commands below. For help with any of those, simply call them with --help. Custom commands can be defined through aliases, use "incus alias" to control those.`)) app.SilenceUsage = true app.SilenceErrors = true app.CompletionOptions = cobra.CompletionOptions{HiddenDefaultCmd: true} app.ValidArgs = aliases() // Global struct. globalCmd := cmdGlobal{cmd: app, asker: ask.NewAsker(bufio.NewReader(os.Stdin)), conf: conf} // Global flags. app.PersistentFlags().BoolVar(&globalCmd.flagVersion, "version", false, i18n.G("Print version number")) app.PersistentFlags().BoolVarP(&globalCmd.flagHelp, "help", "h", false, i18n.G("Print help")) app.PersistentFlags().BoolVar(&globalCmd.flagForceLocal, "force-local", false, i18n.G("Force using the local unix socket")) app.PersistentFlags().StringVar(&globalCmd.flagProject, "project", "", i18n.G("Override the source project")+"``") app.PersistentFlags().BoolVar(&globalCmd.flagLogDebug, "debug", false, i18n.G("Show all debug messages")) app.PersistentFlags().BoolVarP(&globalCmd.flagLogVerbose, "verbose", "v", false, i18n.G("Show all information messages")) app.PersistentFlags().BoolVarP(&globalCmd.flagQuiet, "quiet", "q", false, i18n.G("Don't show progress information")) app.PersistentFlags().BoolVar(&globalCmd.flagSubCmds, "sub-commands", false, i18n.G("Use with help or --help to view sub-commands")) app.PersistentFlags().BoolVar(&u.ExplainOnly, "explain", false, i18n.G("If the command is valid, explain its parsed arguments instead of running it")) // Wrappers app.PersistentPreRunE = globalCmd.preRun app.PersistentPostRunE = globalCmd.postRun // Version handling app.SetVersionTemplate("{{.Version}}\n") app.Version = version.Version // alias sub-command aliasCmd := cmdAlias{global: &globalCmd} app.AddCommand(aliasCmd.command()) // admin sub-command adminCmd := cmdAdmin{global: &globalCmd} app.AddCommand(adminCmd.command()) // cluster sub-command clusterCmd := cmdCluster{global: &globalCmd} app.AddCommand(clusterCmd.command()) // config sub-command configCmd := cmdConfig{global: &globalCmd} app.AddCommand(configCmd.command()) // console sub-command consoleCmd := cmdConsole{global: &globalCmd} app.AddCommand(consoleCmd.command()) // create sub-command createCmd := cmdCreate{global: &globalCmd} app.AddCommand(createCmd.command()) // copy sub-command copyCmd := cmdCopy{global: &globalCmd} app.AddCommand(copyCmd.command()) // delete sub-command deleteCmd := cmdDelete{global: &globalCmd} app.AddCommand(deleteCmd.command()) // exec sub-command execCmd := cmdExec{global: &globalCmd} app.AddCommand(execCmd.command()) // export sub-command exportCmd := cmdExport{global: &globalCmd} app.AddCommand(exportCmd.command()) // file sub-command fileCmd := cmdFile{global: &globalCmd} app.AddCommand(fileCmd.command()) // import sub-command importCmd := cmdImport{global: &globalCmd} app.AddCommand(importCmd.command()) // info sub-command infoCmd := cmdInfo{global: &globalCmd} app.AddCommand(infoCmd.command()) // image sub-command imageCmd := cmdImage{global: &globalCmd} app.AddCommand(imageCmd.command()) // launch sub-command launchCmd := cmdLaunch{global: &globalCmd, init: &createCmd} app.AddCommand(launchCmd.command()) // list sub-command listCmd := cmdList{global: &globalCmd} app.AddCommand(listCmd.command()) // manpage sub-command manpageCmd := cmdManpage{global: &globalCmd} app.AddCommand(manpageCmd.command()) // monitor sub-command monitorCmd := cmdMonitor{global: &globalCmd} app.AddCommand(monitorCmd.command()) // move sub-command moveCmd := cmdMove{global: &globalCmd} app.AddCommand(moveCmd.command()) // network sub-command networkCmd := cmdNetwork{global: &globalCmd} app.AddCommand(networkCmd.command()) // operation sub-command operationCmd := cmdOperation{global: &globalCmd} app.AddCommand(operationCmd.command()) // pause sub-command pauseCmd := cmdPause{global: &globalCmd} app.AddCommand(pauseCmd.command()) // publish sub-command publishCmd := cmdPublish{global: &globalCmd} app.AddCommand(publishCmd.command()) // profile sub-command profileCmd := cmdProfile{global: &globalCmd} app.AddCommand(profileCmd.command()) // project sub-command projectCmd := cmdProject{global: &globalCmd} app.AddCommand(projectCmd.command()) // query sub-command queryCmd := cmdQuery{global: &globalCmd} app.AddCommand(queryCmd.command()) // rebuild sub-command rebuildCmd := cmdRebuild{global: &globalCmd} app.AddCommand(rebuildCmd.command()) // rename sub-command renameCmd := cmdRename{global: &globalCmd} app.AddCommand(renameCmd.command()) // restart sub-command restartCmd := cmdRestart{global: &globalCmd} app.AddCommand(restartCmd.command()) // remote sub-command remoteCmd := cmdRemote{global: &globalCmd} app.AddCommand(remoteCmd.command()) // resume sub-command resumeCmd := cmdResume{global: &globalCmd} app.AddCommand(resumeCmd.command()) // snapshot sub-command snapshotCmd := cmdSnapshot{global: &globalCmd} app.AddCommand(snapshotCmd.command()) // storage sub-command storageCmd := cmdStorage{global: &globalCmd} app.AddCommand(storageCmd.command()) // start sub-command startCmd := cmdStart{global: &globalCmd} app.AddCommand(startCmd.command()) // stop sub-command stopCmd := cmdStop{global: &globalCmd} app.AddCommand(stopCmd.command()) // version sub-command versionCmd := cmdVersion{global: &globalCmd} app.AddCommand(versionCmd.command()) // top sub-command topCmd := cmdTop{global: &globalCmd} app.AddCommand(topCmd.command()) // warning sub-command warningCmd := cmdWarning{global: &globalCmd} app.AddCommand(warningCmd.command()) // webui sub-command webuiCmd := cmdWebui{global: &globalCmd} app.AddCommand(webuiCmd.command()) // debug sub-command debugCmd := cmdDebug{global: &globalCmd} app.AddCommand(debugCmd.command()) // wait sub-command waitCmd := cmdWait{global: &globalCmd} app.AddCommand(waitCmd.command()) // Get help command app.InitDefaultHelpCmd() var help *cobra.Command for _, cmd := range app.Commands() { if cmd.Name() == "help" { help = cmd break } } // Help flags cli.AddBoolFlag(app.Flags(), &globalCmd.flagHelpAll, "all|a", i18n.G("Show less common commands")) cli.AddBoolFlag(help.Flags(), &globalCmd.flagHelpAll, "all|a", i18n.G("Show less common commands")) return app, &globalCmd, nil } func main() { app, globalCmd, err := createApp() if err != nil { fmt.Fprintf(os.Stderr, "%s: %v\n", color.ErrorPrefix, err) os.Exit(1) } // Deal with --all and --sub-commands flags as well as process aliases. err = app.ParseFlags(os.Args[1:]) if err == nil { if globalCmd.flagHelpAll { // Show all commands for _, cmd := range app.Commands() { if cmd.Name() == "completion" { continue } cmd.Hidden = false } } if globalCmd.flagSubCmds { app.SetUsageTemplate(usageTemplateSubCmds()) } else { app.SetUsageTemplate(usageTemplate()) } } // Process aliases err = execIfAliases(app) if err != nil { fmt.Fprintf(os.Stderr, "%s: %v\n", color.ErrorPrefix, err) os.Exit(1) } // Run the main command and handle errors err = app.Execute() if err != nil { // Handle --explain. if errors.Is(err, u.ErrExplainOnly) { fmt.Println(err.Error()) os.Exit(0) } // Handle non-Linux systems if errors.Is(err, config.ErrNotLinux) { fmt.Fprintf(os.Stderr, "%s", i18n.G(`This client hasn't been configured to use a remote server yet. As your platform can't run native Linux instances, you must connect to a remote server. If you already added a remote server, make it the default with "incus remote switch NAME".`)+"\n") os.Exit(1) } // Default error handling if os.Getenv("INCUS_ALIASES") == "1" { fmt.Fprintf(os.Stderr, i18n.G("Error while executing alias expansion: %s\n"), shellquote.Join(os.Args...)) } fmt.Fprintf(os.Stderr, "%s %v\n", color.ErrorPrefix, err) // If custom exit status not set, use default error status. if globalCmd.ret == 0 { globalCmd.ret = 1 } } if globalCmd.ret != 0 { os.Exit(globalCmd.ret) } } func (c *cmdGlobal) preRun(cmd *cobra.Command, _ []string) error { var err error // If calling the help, skip pre-run if cmd.Name() == "help" { return nil } // Figure out a potential cache path. var cachePath string if os.Getenv("INCUS_CACHE") != "" { cachePath = os.Getenv("INCUS_CACHE") } else if os.Getenv("HOME") != "" && util.PathExists(os.Getenv("HOME")) { cachePath = path.Join(os.Getenv("HOME"), ".cache", "incus") } else { currentUser, err := user.Current() if err != nil { return err } if util.PathExists(currentUser.HomeDir) { cachePath = path.Join(currentUser.HomeDir, ".cache", "incus") } } if cachePath != "" { err := os.MkdirAll(cachePath, 0o700) if err != nil && !os.IsExist(err) { cachePath = "" } } // If no config dir could be found, treat as if --force-local was passed. if c.conf.ConfigDir == "" { c.flagForceLocal = true } c.confPath = c.conf.ConfigPath("config.yml") // Set cache directory in config. c.conf.CacheDir = cachePath // Override the project if c.flagProject != "" { c.conf.ProjectOverride = c.flagProject } else { c.conf.ProjectOverride = os.Getenv("INCUS_PROJECT") } // Setup password helper if termios.IsTerminal(getStdinFd()) { c.conf.PromptPassword = func(filename string) (string, error) { return c.asker.AskPasswordOnce(fmt.Sprintf(i18n.G("Password for %s: "), filename)), nil } } // If the user is running a command that may attempt to connect to the local daemon // and this is the first time the client has been run by the user, then check to see // if the server has been properly configured. Don't display the message if the var path // does not exist (server missing), as the user may be targeting a remote daemon. if !c.flagForceLocal && !util.PathExists(c.confPath) { // Create the config dir so that we don't get in here again for this user. err = os.MkdirAll(c.conf.ConfigDir, 0o750) if err != nil { return err } // Handle local servers. if util.PathExists(internalUtil.VarPath("")) { // Attempt to connect to the local server runInit := true d, err := incus.ConnectIncusUnix("", nil) if err == nil { // Check if server is initialized. info, _, err := d.GetServer() if err == nil && info.Environment.Storage != "" { runInit = false } // Detect usable project. names, err := d.GetProjectNames() if err == nil { if len(names) == 1 && names[0] != api.ProjectDefaultName { remote := c.conf.Remotes["local"] remote.Project = names[0] c.conf.Remotes["local"] = remote } } } flush := false if runInit && (cmd.Name() != "init" || cmd.Parent() == nil || cmd.Parent().Name() != "admin") { fmt.Fprint(os.Stderr, i18n.G("If this is your first time running Incus on this machine, you should also run: incus admin init")+"\n") flush = true } if !slices.Contains([]string{"admin", "create", "launch"}, cmd.Name()) && (cmd.Parent() == nil || cmd.Parent().Name() != "admin") { images := []string{"debian/12", "fedora/42", "opensuse/tumbleweed", "ubuntu/24.04"} image := images[rand.Intn(len(images))] fmt.Fprintf(os.Stderr, i18n.G(`To start your first container, try: incus launch images:%s Or for a virtual machine: incus launch images:%s --vm`)+"\n", image, image) flush = true } if flush { fmt.Fprint(os.Stderr, "\n") } } // And save the initial configuration err = c.conf.SaveConfig(c.confPath) if err != nil { return err } } // Set the user agent c.conf.UserAgent = version.UserAgent // Setup the logger err = logger.InitLogger("", "", c.flagLogVerbose, c.flagLogDebug, nil) if err != nil { return err } return nil } func (c *cmdGlobal) postRun(_ *cobra.Command, _ []string) error { if c.conf != nil && util.PathExists(c.confPath) { // Save OIDC tokens on exit c.conf.SaveOIDCTokens() } return nil } type remoteResource struct { remote string server incus.InstanceServer name string } func (c *cmdGlobal) parseServers(remotes ...string) ([]remoteResource, error) { servers := map[string]incus.InstanceServer{} resources := []remoteResource{} for _, remote := range remotes { // Parse the remote remoteName, name, err := c.conf.ParseRemote(remote) if err != nil { return nil, err } // Setup the struct resource := remoteResource{ remote: remoteName, name: name, } // Look at our cache _, ok := servers[remoteName] if ok { resource.server = servers[remoteName] resources = append(resources, resource) continue } // New connection d, err := c.conf.GetInstanceServer(remoteName) if err != nil { return nil, err } resource.server = d servers[remoteName] = d resources = append(resources, resource) } return resources, nil } func (c *cmdGlobal) checkArgs(cmd *cobra.Command, args []string, minArgs int, maxArgs int) (bool, error) { exit, err := cli.CheckArgs(cmd, args, minArgs, maxArgs) if err == cli.ErrBadArgs { // Use translated error message. return exit, errors.New(i18n.G("Invalid number of arguments")) } return exit, err } // Return the default list format if the user configured it, otherwise just return "table". func (c *cmdGlobal) defaultListFormat() string { if c.conf == nil || c.conf.Defaults.ListFormat == "" { return "table" } return c.conf.Defaults.ListFormat } // Return "vga" if preferred console type is VGA, otherwise return "console". func (c *cmdGlobal) defaultConsoleType() string { if c.conf == nil || c.conf.Defaults.ConsoleType == "" { return "console" } return c.conf.Defaults.ConsoleType } // Return the default console type if the user configured it, otherwise just return "console". func (c *cmdGlobal) defaultConsoleSpiceCommand() string { // Alternative SPICE command. if c.conf == nil || c.conf.Defaults.ConsoleSpiceCommand == "" { return "" } return c.conf.Defaults.ConsoleSpiceCommand } incus-7.0.0/cmd/incus/main_aliases.go000066400000000000000000000136151517523235500175150ustar00rootroot00000000000000package main import ( "fmt" "os" "os/exec" "regexp" "slices" "strconv" "strings" "github.com/kballard/go-shellquote" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/i18n" config "github.com/lxc/incus/v7/shared/cliconfig" ) var numberedArgRegex = regexp.MustCompile(`@ARG(\d+)@`) // defaultAliases contains LXC's built-in command line aliases. The built-in // aliases are checked only if no user-defined alias was found. var defaultAliases = map[string]string{ "shell": "exec @ARGS@ -- su -l", } func findAlias(aliases map[string]string, origArgs []string) ([]string, []string, bool) { foundAlias := false aliasKey := []string{} aliasValue := []string{} // Sort the aliases in a stable order, preferring the long multi-fields ones. aliasNames := make([]string, 0, len(aliases)) for k := range aliases { aliasNames = append(aliasNames, k) } slices.Sort(aliasNames) slices.SortStableFunc(aliasNames, func(a, b string) int { aFields := strings.Split(a, " ") bFields := strings.Split(b, " ") if len(aFields) == len(bFields) { return 0 } else if len(aFields) < len(bFields) { return 1 } return -1 }) for _, k := range aliasNames { v := aliases[k] foundAlias = true for i, key := range strings.Split(k, " ") { if len(origArgs) <= i+1 || origArgs[i+1] != key { foundAlias = false break } } if foundAlias { aliasKey = strings.Split(k, " ") fields, err := shellquote.Split(v) if err == nil { aliasValue = fields } else { aliasValue = strings.Split(v, " ") } break } } return aliasKey, aliasValue, foundAlias } func expandAlias(conf *config.Config, args []string, app *cobra.Command) ([]string, bool, error) { fset := app.Flags() nargs := fset.NArg() firstArgIndex := 1 firstPosArgIndex := 0 if fset.Arg(0) == "__complete" { nargs-- firstArgIndex++ firstPosArgIndex++ } if nargs == 0 { return nil, false, nil } if fset.Arg(0) == "incus" { return nil, false, nil } lastFlagIndex := slices.Index(args, fset.Arg(firstPosArgIndex)) // newArgs contains all the flags before the first positional argument newArgs := args[firstArgIndex:lastFlagIndex] // origArgs contains everything except the flags in newArgs origArgs := slices.Concat(args[:firstArgIndex], args[lastFlagIndex:]) // strip out completion subcommand and fragment from end completion := false completionFragment := "" if len(origArgs) >= 3 && origArgs[1] == "__complete" { completion = true completionFragment = origArgs[len(origArgs)-1] origArgs = append(origArgs[:1], origArgs[2:len(origArgs)-1]...) } aliasKey, aliasValue, foundAlias := findAlias(conf.Aliases, origArgs) if !foundAlias { aliasKey, aliasValue, foundAlias = findAlias(defaultAliases, origArgs) if !foundAlias { return []string{}, false, nil } } if !strings.HasPrefix(aliasValue[0], "/") { newArgs = append([]string{origArgs[0]}, newArgs...) } // The @ARGS@ are initially any arguments given after the alias key. var atArgs []string if len(origArgs) > len(aliasKey)+1 { atArgs = origArgs[len(aliasKey)+1:] } // Find the arguments that have been referenced directly e.g. @ARG1@. numberedArgsMap := map[int]string{} for _, aliasArg := range aliasValue { matches := numberedArgRegex.FindAllStringSubmatch(aliasArg, -1) if len(matches) == 0 { continue } for _, match := range matches { argNoStr := match[1] argNo, err := strconv.Atoi(argNoStr) if err != nil { return nil, false, fmt.Errorf(i18n.G("Invalid argument %q"), match[0]) } if argNo > len(atArgs) { return nil, false, fmt.Errorf(i18n.G("Found alias %q references an argument outside the given number"), strings.Join(aliasKey, " ")) } numberedArgsMap[argNo] = atArgs[argNo-1] } } // Remove directly referenced arguments from @ARGS@ for i := len(atArgs) - 1; i >= 0; i-- { _, ok := numberedArgsMap[i+1] if ok { atArgs = slices.Delete(atArgs, i, i+1) } } // Replace arguments hasReplacedArgsVar := false for _, aliasArg := range aliasValue { // Only replace all @ARGS@ when it is not part of another string if aliasArg == "@ARGS@" { // if completing we want to stop on @ARGS@ and append the completion below if completion { break } newArgs = append(newArgs, atArgs...) hasReplacedArgsVar = true continue } // Replace @ARG1@, @ARG2@ etc. as substrings matches := numberedArgRegex.FindAllStringSubmatch(aliasArg, -1) if len(matches) > 0 { newArg := aliasArg for _, match := range matches { argNoStr := match[1] argNo, err := strconv.Atoi(argNoStr) if err != nil { return nil, false, fmt.Errorf(i18n.G("Invalid argument %q"), match[0]) } replacement := numberedArgsMap[argNo] newArg = strings.ReplaceAll(newArg, match[0], replacement) } newArgs = append(newArgs, newArg) continue } newArgs = append(newArgs, aliasArg) } // add back in completion if it was stripped before if completion { newArgs = append([]string{newArgs[0], "__complete"}, newArgs[1:]...) newArgs = append(newArgs, completionFragment) } // Add the rest of the arguments only if @ARGS@ wasn't used. if !hasReplacedArgsVar { newArgs = append(newArgs, atArgs...) } return newArgs, true, nil } func execIfAliases(app *cobra.Command) error { // Avoid loops if os.Getenv("INCUS_ALIASES") == "1" { return nil } conf, err := config.LoadConfig("") if err != nil { return fmt.Errorf(i18n.G("Failed to load configuration: %s"), err) } // Expand the aliases newArgs, expanded, err := expandAlias(conf, os.Args, app) if err != nil { return err } else if !expanded { return nil } // Look for the executable path, err := exec.LookPath(newArgs[0]) if err != nil { return fmt.Errorf(i18n.G("Processing aliases failed: %s"), err) } // Re-exec environ := getEnviron() environ = append(environ, "INCUS_ALIASES=1") ret := doExec(path, newArgs, environ) return fmt.Errorf(i18n.G("Processing aliases failed: %s"), ret) } incus-7.0.0/cmd/incus/main_test.go000066400000000000000000000054721517523235500170550ustar00rootroot00000000000000package main import ( "testing" "github.com/stretchr/testify/assert" config "github.com/lxc/incus/v7/shared/cliconfig" ) type aliasTestcase struct { input []string expected []string expectErr bool } func slicesEqual(a, b []string) bool { if a == nil && b == nil { return true } if a == nil || b == nil { return false } if len(a) != len(b) { return false } for i := range a { if a[i] != b[i] { return false } } return true } func TestExpandAliases(t *testing.T) { aliases := map[string]string{ "tester 12": "list", "foo": "list @ARGS@ -c n", "ssh": "/usr/bin/ssh @ARGS@", "bar": "exec c1 -- @ARGS@", "fizz": "exec @ARG1@ -- echo @ARG2@", "snaps": "query /1.0/instances/@ARG1@/snapshots", "snapshots with recursion": "query /1.0/instances/@ARG1@/snapshots?recursion=@ARG2@", } testcases := []aliasTestcase{ { input: []string{"incus", "list"}, expected: []string{"incus", "list"}, }, { input: []string{"incus", "tester", "12"}, expected: []string{"incus", "list"}, }, { input: []string{"incus", "foo", "asdf"}, expected: []string{"incus", "list", "asdf", "-c", "n"}, }, { input: []string{"incus", "ssh", "c1"}, expected: []string{"/usr/bin/ssh", "c1"}, }, { input: []string{"incus", "bar", "ls", "/"}, expected: []string{"incus", "exec", "c1", "--", "ls", "/"}, }, { input: []string{"incus", "fizz", "c1", "buzz"}, expected: []string{"incus", "exec", "c1", "--", "echo", "buzz"}, }, { input: []string{"incus", "fizz", "c1"}, expectErr: true, }, { input: []string{"incus", "snaps", "c1"}, expected: []string{"incus", "query", "/1.0/instances/c1/snapshots"}, }, { input: []string{"incus", "snapshots", "with", "recursion", "c1", "2"}, expected: []string{"incus", "query", "/1.0/instances/c1/snapshots?recursion=2"}, }, { input: []string{"incus", "--project", "default", "fizz", "c1", "buzz"}, expected: []string{"incus", "--project", "default", "exec", "c1", "--", "echo", "buzz"}, }, { input: []string{"incus", "--project=default", "fizz", "c1", "buzz"}, expected: []string{"incus", "--project=default", "exec", "c1", "--", "echo", "buzz"}, }, } conf := &config.Config{Aliases: aliases} for _, tc := range testcases { app, _, _ := createApp() _ = app.ParseFlags(tc.input[1:]) result, expanded, err := expandAlias(conf, tc.input, app) if tc.expectErr { assert.Error(t, err) continue } if !expanded { if !slicesEqual(tc.input, tc.expected) { t.Errorf("didn't expand when expected to: %s", tc.input) } continue } if !slicesEqual(result, tc.expected) { t.Errorf("%s didn't match %s", result, tc.expected) } } } incus-7.0.0/cmd/incus/manpage.go000066400000000000000000000041011517523235500164660ustar00rootroot00000000000000package main import ( "fmt" "github.com/spf13/cobra" "github.com/spf13/cobra/doc" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdManpage struct { global *cmdGlobal flagFormat string flagAll bool } var cmdManpageUsage = u.Usage{u.Target(u.Directory)} func (c *cmdManpage) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("manpage", cmdManpageUsage...) cmd.Short = i18n.G("Generate manpages for all commands") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Generate manpages for all commands`)) cmd.Hidden = true cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", "man", "", i18n.G("Format (man|md|rest|yaml)")) cli.AddBoolFlag(cmd.Flags(), &c.flagAll, "all|a", i18n.G("Include less common commands")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { format := cmd.Flag("format").Value.String() switch format { case "man", "md", "rest", "yaml": return nil default: return fmt.Errorf(`Invalid value %q for flag "--format"`, format) } } cmd.RunE = c.run return cmd } func (c *cmdManpage) run(cmd *cobra.Command, args []string) error { parsed, err := cmdManpageUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } target := parsed[0].String // If asked to do all commands, mark them all visible. for _, c := range c.global.cmd.Commands() { if c.Name() == "completion" { continue } c.Hidden = false } // Generate the documentation. switch c.flagFormat { case "man": header := &doc.GenManHeader{ Title: i18n.G("Incus - Command line client"), Section: "1", } opts := doc.GenManTreeOptions{ Header: header, Path: target, CommandSeparator: ".", } err = doc.GenManTreeFromOpts(c.global.cmd, opts) case "md": err = doc.GenMarkdownTree(c.global.cmd, target) case "rest": err = doc.GenReSTTree(c.global.cmd, target) case "yaml": err = doc.GenYamlTree(c.global.cmd, target) } return err } incus-7.0.0/cmd/incus/monitor.go000066400000000000000000000114111517523235500165470ustar00rootroot00000000000000package main import ( "encoding/json" "errors" "fmt" "os" "slices" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdMonitor struct { global *cmdGlobal flagType []string flagPretty bool flagLogLevel string flagAllProjects bool flagFormat string } var cmdMonitorUsage = u.Usage{u.RemoteColonOpt} func (c *cmdMonitor) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("monitor", cmdMonitorUsage...) cmd.Short = i18n.G("Monitor a local or remote server") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Monitor a local or remote server By default the monitor will listen to all message types.`)) cmd.Example = cli.FormatSection("", i18n.G( `incus monitor --type=logging Only show log messages. incus monitor --pretty --type=logging --loglevel=info Show a pretty log of messages with info level or higher. incus monitor --type=lifecycle Only show lifecycle events.`)) cmd.Hidden = true cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagPretty, "pretty", i18n.G("Pretty rendering (short for --format=pretty)")) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("Show events from all projects")) cli.AddStringArrayFlag(cmd.Flags(), &c.flagType, "type|t", i18n.G("Event type to listen for")) cli.AddStringFlag(cmd.Flags(), &c.flagLogLevel, "loglevel", "", "", i18n.G("Minimum level for log messages (only available when using pretty format)")) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", "yaml", "", i18n.G("Format (json|pretty|yaml)")) return cmd } func (c *cmdMonitor) run(cmd *cobra.Command, args []string) error { parsed, err := cmdMonitorUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer if !slices.Contains([]string{"json", "pretty", "yaml"}, c.flagFormat) { return fmt.Errorf(i18n.G("Invalid format: %s"), c.flagFormat) } // Setup format. if c.flagPretty { c.flagFormat = "pretty" } if c.flagFormat != "pretty" && c.flagLogLevel != "" { return errors.New(i18n.G("Log level filtering can only be used with pretty formatting")) } var listener *incus.EventListener if c.flagAllProjects { listener, err = d.GetEventsAllProjects() } else { listener, err = d.GetEvents() } if err != nil { return err } logLevel := logrus.DebugLevel if c.flagLogLevel != "" { logLevel, err = logrus.ParseLevel(c.flagLogLevel) if err != nil { return err } } chError := make(chan error, 1) handler := func(event api.Event) { if c.flagFormat == "pretty" { // Parse the event. record, err := event.ToLogging() if err != nil { chError <- err return } if record.Lvl == "dbug" { record.Lvl = "debug" } // Get the log level. msgLevel, err := logrus.ParseLevel(record.Lvl) if err != nil { chError <- err return } // Check log level. if msgLevel > logLevel { return } // Setup logrus. logger := &logrus.Logger{ Out: os.Stdout, } entry := &logrus.Entry{Logger: logger} entry.Data = c.unpackCtx(record.Ctx) if event.Type == "logging" && d.IsClustered() { entry.Message = fmt.Sprintf("[%s] %s", event.Location, record.Msg) } else { entry.Message = record.Msg } entry.Time = record.Time entry.Level = msgLevel format := logrus.TextFormatter{FullTimestamp: true, PadLevelText: true} line, err := format.Format(entry) if err != nil { chError <- err return } fmt.Print(string(line)) return } // Render as JSON (to expand RawMessage) jsonRender, err := json.Marshal(&event) if err != nil { chError <- err return } // Read back to a clean interface var rawEvent any err = json.Unmarshal(jsonRender, &rawEvent) if err != nil { chError <- err return } // And now print the result. var render []byte switch c.flagFormat { case "yaml": render, err = yaml.Dump(&rawEvent, yaml.V2) if err != nil { chError <- err return } case "json": render, err = json.Marshal(&rawEvent) if err != nil { chError <- err return } } fmt.Printf("%s\n\n", render) } _, err = listener.AddHandler(c.flagType, handler) if err != nil { return err } go func() { chError <- listener.Wait() }() return <-chError } func (c *cmdMonitor) unpackCtx(ctx []any) logrus.Fields { out := logrus.Fields{} var key string for _, entry := range ctx { if key == "" { key = fmt.Sprintf("%v", entry) } else { out[key] = fmt.Sprintf("%v", entry) key = "" } } return out } incus-7.0.0/cmd/incus/move.go000066400000000000000000000220551517523235500160340ustar00rootroot00000000000000package main import ( "errors" "fmt" "maps" "strings" "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdMove struct { global *cmdGlobal flagNoProfiles bool flagProfile []string flagConfig []string flagInstanceOnly bool flagDevice []string flagMode string flagStateless bool flagStorage string flagTarget string flagTargetProject string flagAllowInconsistent bool } var cmdMoveUsage = u.Usage{u.Instance.Remote(), u.NewName(u.Instance).Optional().Remote()} func (c *cmdMove) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("move", cmdMoveUsage...) cmd.Aliases = []string{"mv"} cmd.Short = i18n.G("Move instances within or in between servers") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Move instances within or in between servers Transfer modes (--mode): - pull: Target server pulls the data from the source server (source must listen on network) - push: Source server pushes the data to the target server (target must listen on network) - relay: The CLI connects to both source and server and proxies the data (both source and target must listen on network) The pull transfer mode is the default as it is compatible with all server versions. `)) cmd.Example = cli.FormatSection("", i18n.G( `incus move [:] [:][] [--instance-only] Move an instance between two hosts, renaming it if destination name differs. incus move [--instance-only] Rename a local instance.`)) cmd.RunE = c.run cli.AddStringArrayFlag(cmd.Flags(), &c.flagConfig, "config|c", i18n.G("Config key/value to apply to the target instance")) cli.AddStringArrayFlag(cmd.Flags(), &c.flagDevice, "device|d", i18n.G("New key/value to apply to a specific device")) cli.AddStringArrayFlag(cmd.Flags(), &c.flagProfile, "profile|p", i18n.G("Profile to apply to the target instance")) cli.AddBoolFlag(cmd.Flags(), &c.flagNoProfiles, "no-profiles", i18n.G("Unset all profiles on the target instance")) cli.AddBoolFlag(cmd.Flags(), &c.flagInstanceOnly, "instance-only", i18n.G("Move the instance without its snapshots")) cli.AddStringFlag(cmd.Flags(), &c.flagMode, "mode", moveDefaultMode, "", i18n.G("Transfer mode. One of pull, push or relay.")) cli.AddBoolFlag(cmd.Flags(), &c.flagStateless, "stateless", i18n.G("Copy a stateful instance stateless")) cli.AddStringFlag(cmd.Flags(), &c.flagStorage, "storage|s", "", "", i18n.G("Storage pool name")) cli.AddStringFlag(cmd.Flags(), &c.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.flagTargetProject, "target-project", "", "", i18n.G("Copy to a project different from the source")) cli.AddBoolFlag(cmd.Flags(), &c.flagAllowInconsistent, "allow-inconsistent", i18n.G("Ignore copy errors for volatile files")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } if len(args) == 1 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdMove) run(cmd *cobra.Command, args []string) error { parsed, err := cmdMoveUsage.Parse(c.global.conf, cmd, args) if err != nil { return nil } srcServer := parsed[0].RemoteServer srcInstanceName := parsed[0].RemoteObject.String dstServer := parsed[1].RemoteServer hasDstInstance := !parsed[1].RemoteObject.Skipped dstInstanceName := parsed[1].RemoteObject.String // Parse the mode mode := moveDefaultMode if c.flagMode != "" { mode = c.flagMode } // As an optimization, if the source and destination are the same, do // this via a simple rename. This only works for instances that aren't // running, instances that are running should be live migrated (of // course, this changing of hostname isn't supported right now, so this // simply won't work). if srcServer == dstServer && c.flagTarget == "" && c.flagStorage == "" && c.flagTargetProject == "" { if !hasDstInstance { return errors.New(i18n.G("Can't perform local rename without a new instance name")) } if c.flagConfig != nil || c.flagDevice != nil || c.flagProfile != nil || c.flagNoProfiles { return errors.New(i18n.G("Can't override configuration or profiles in local rename")) } // Instance rename op, err := srcServer.RenameInstance(srcInstanceName, api.InstancePost{Name: dstInstanceName}) if err != nil { return err } return op.Wait() } stateful := !c.flagStateless isServerSide := func() bool { // Check if same source and destination. if srcServer != dstServer { return false } // Check if asked for specific client mode. if c.flagMode != moveDefaultMode { return false } // Check if override is requested with a server lacking support. if !srcServer.HasExtension("instance_move_config") { if len(c.flagConfig) > 0 { return false } if len(c.flagDevice) > 0 { return false } if len(c.flagProfile) > 0 { return false } } // Check if server supports moving pools. if c.flagStorage != "" && !srcServer.HasExtension("instance_pool_move") { return false } // Check if server supports moving projects. if c.flagTargetProject != "" && !srcServer.HasExtension("instance_project_move") { return false } return true }() // Support for server-side move in clusters. if isServerSide { return c.moveInstance(parsed[0], parsed[1], stateful) } cpy := cmdCopy{} cpy.global = c.global cpy.flagTarget = c.flagTarget cpy.flagTargetProject = c.flagTargetProject cpy.flagConfig = c.flagConfig cpy.flagDevice = c.flagDevice cpy.flagProfile = c.flagProfile cpy.flagNoProfiles = c.flagNoProfiles cpy.flagAllowInconsistent = c.flagAllowInconsistent instanceOnly := c.flagInstanceOnly // A move is just a copy followed by a delete; however, we want to // keep the volatile entries around since we are moving the instance. err = cpy.copyOrMove(cmd, parsed[0], parsed[1], true, -1, stateful, instanceOnly, mode, c.flagStorage, true) if err != nil { return err } del := cmdDelete{global: c.global} del.flagForce = true del.flagForceProtected = true err = del.deleteOne(parsed[0]) if err != nil { return fmt.Errorf(i18n.G("Failed to delete original instance after copying it: %w"), err) } return nil } // Move an instance between pools and projects using special POST /instances/ API. func (c *cmdMove) moveInstance(src *u.Parsed, dst *u.Parsed, stateful bool) error { srcServer := src.RemoteServer srcInstanceName := src.RemoteObject.String dstInstanceName := dst.RemoteObject.String if dst.RemoteObject.Skipped { dstInstanceName = srcInstanceName } if !srcServer.IsClustered() && c.flagTarget != "" { return errors.New(i18n.G("--target can only be used with clusters")) } // Set the target if specified. if c.flagTarget != "" { srcServer = srcServer.UseTarget(c.flagTarget) } // Pass the new pool to the migration API. req := api.InstancePost{ Name: dstInstanceName, Migration: true, InstanceOnly: c.flagInstanceOnly, Pool: c.flagStorage, Project: c.flagTargetProject, Live: stateful, } // Override profiles. var profiles *[]string if len(c.flagProfile) > 0 { profiles = &c.flagProfile } else if c.flagNoProfiles { profiles = &[]string{} } if profiles != nil { req.Profiles = *profiles } // Override config. if len(c.flagConfig) > 0 { req.Config = map[string]string{} for _, entry := range c.flagConfig { key, value, found := strings.Cut(entry, "=") if !found { return fmt.Errorf(i18n.G("Bad key=value pair: %q"), entry) } req.Config[key] = value } } // Override devices. if len(c.flagDevice) > 0 { req.Devices = map[string]map[string]string{} // Parse the overrides. deviceMap, err := parseDeviceOverrides(c.flagDevice) if err != nil { return err } // Fetch the current instance. inst, _, err := srcServer.GetInstance(srcInstanceName) if err != nil { return err } for devName, dev := range deviceMap { fullDev := inst.ExpandedDevices[devName] maps.Copy(fullDev, dev) req.Devices[devName] = fullDev } } // Move the instance. op, err := srcServer.MigrateInstance(srcInstanceName, req) if err != nil { return fmt.Errorf(i18n.G("Migration API failure: %w"), err) } // Watch the background operation progress := cli.ProgressRenderer{ Format: i18n.G("Transferring instance: %s"), Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } // Wait for the move to complete err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return fmt.Errorf(i18n.G("Migration operation failure: %w"), err) } progress.Done("") return nil } // Default migration mode when moving an instance. const moveDefaultMode = "pull" incus-7.0.0/cmd/incus/network.go000066400000000000000000001253541517523235500165650ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "maps" "os" "reflect" "regexp" "sort" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" "github.com/lxc/incus/v7/shared/units" ) type cmdNetwork struct { global *cmdGlobal flagTarget string flagType string } type networkColumn struct { Name string Data func(api.Network) string } func (c *cmdNetwork) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("network") cmd.Short = i18n.G("Manage and attach instances to networks") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Manage and attach instances to networks`)) // Attach networkAttachCmd := cmdNetworkAttach{global: c.global, network: c} cmd.AddCommand(networkAttachCmd.command()) // Attach profile networkAttachProfileCmd := cmdNetworkAttachProfile{global: c.global, network: c} cmd.AddCommand(networkAttachProfileCmd.command()) // Create networkCreateCmd := cmdNetworkCreate{global: c.global, network: c} cmd.AddCommand(networkCreateCmd.command()) // Delete networkDeleteCmd := cmdNetworkDelete{global: c.global, network: c} cmd.AddCommand(networkDeleteCmd.command()) // Detach networkDetachCmd := cmdNetworkDetach{global: c.global, network: c} cmd.AddCommand(networkDetachCmd.command()) // Detach profile networkDetachProfileCmd := cmdNetworkDetachProfile{global: c.global, network: c, networkDetach: &networkDetachCmd} cmd.AddCommand(networkDetachProfileCmd.command()) // Edit networkEditCmd := cmdNetworkEdit{global: c.global, network: c} cmd.AddCommand(networkEditCmd.command()) // Get networkGetCmd := cmdNetworkGet{global: c.global, network: c} cmd.AddCommand(networkGetCmd.command()) // Info networkInfoCmd := cmdNetworkInfo{global: c.global, network: c} cmd.AddCommand(networkInfoCmd.command()) // List networkListCmd := cmdNetworkList{global: c.global, network: c} cmd.AddCommand(networkListCmd.command()) // List allocations networkListAllocationsCmd := cmdNetworkListAllocations{global: c.global, network: c} cmd.AddCommand(networkListAllocationsCmd.command()) // List leases networkListLeasesCmd := cmdNetworkListLeases{global: c.global, network: c} cmd.AddCommand(networkListLeasesCmd.command()) // Rename networkRenameCmd := cmdNetworkRename{global: c.global, network: c} cmd.AddCommand(networkRenameCmd.command()) // Set networkSetCmd := cmdNetworkSet{global: c.global, network: c} cmd.AddCommand(networkSetCmd.command()) // Show networkShowCmd := cmdNetworkShow{global: c.global, network: c} cmd.AddCommand(networkShowCmd.command()) // Unset networkUnsetCmd := cmdNetworkUnset{global: c.global, network: c, networkSet: &networkSetCmd} cmd.AddCommand(networkUnsetCmd.command()) // ACL networkACLCmd := cmdNetworkACL{global: c.global} cmd.AddCommand(networkACLCmd.command()) // Address set networkAddressSetCmd := cmdNetworkAddressSet{global: c.global} cmd.AddCommand(networkAddressSetCmd.command()) // Forward networkForwardCmd := cmdNetworkForward{global: c.global} cmd.AddCommand(networkForwardCmd.command()) // Integration networkIntegrationCmd := cmdNetworkIntegration{global: c.global} cmd.AddCommand(networkIntegrationCmd.command()) // Load Balancer networkLoadBalancerCmd := cmdNetworkLoadBalancer{global: c.global} cmd.AddCommand(networkLoadBalancerCmd.command()) // Peer networkPeerCmd := cmdNetworkPeer{global: c.global} cmd.AddCommand(networkPeerCmd.command()) // Zone networkZoneCmd := cmdNetworkZone{global: c.global} cmd.AddCommand(networkZoneCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Attach. type cmdNetworkAttach struct { global *cmdGlobal network *cmdNetwork } var cmdNetworkAttachUsage = u.Usage{u.Network.Remote(), u.Instance, u.NewName(u.Device).Optional(u.NewName(u.Interface).Optional())} func (c *cmdNetworkAttach) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("attach", cmdNetworkAttachUsage...) cmd.Short = i18n.G("Attach network interfaces to instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Attach new network interfaces to instances`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpInstances(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkAttach) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkAttachUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String instanceName := parsed[1].String hasDevice := !parsed[2].Skipped deviceName := networkName interfaceName := "" if hasDevice { deviceName = parsed[2].List[0].String interfaceName = parsed[2].List[1].String } // Get the network entry network, _, err := d.GetNetwork(networkName) if err != nil { return err } // Prepare the instance's device entry var device map[string]string if network.Managed && d.HasExtension("instance_nic_network") { // If network is managed, use the network property rather than nictype, so that the network's // inherited properties are loaded into the NIC when started. device = map[string]string{ "type": "nic", "network": network.Name, } } else { // If network is unmanaged default to using a macvlan connected to the specified interface. device = map[string]string{ "type": "nic", "nictype": "macvlan", "parent": networkName, } if network.Type == "bridge" { // If the network type is an unmanaged bridge, use bridged NIC type. device["nictype"] = "bridged" } } device["name"] = interfaceName // Add the device to the instance err = instanceDeviceAdd(d, instanceName, deviceName, device) if err != nil { return err } return nil } // Attach profile. type cmdNetworkAttachProfile struct { global *cmdGlobal network *cmdNetwork } var cmdNetworkAttachProfileUsage = u.Usage{u.Network.Remote(), u.Profile, u.NewName(u.Device).Optional(u.NewName(u.Interface).Optional())} func (c *cmdNetworkAttachProfile) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("attach-profile", cmdNetworkAttachProfileUsage...) cmd.Short = i18n.G("Attach network interfaces to profiles") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Attach network interfaces to profiles`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpProfiles(args[0], false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkAttachProfile) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkAttachProfileUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String profileName := parsed[1].String hasDevice := !parsed[2].Skipped deviceName := networkName interfaceName := "" if hasDevice { deviceName = parsed[2].List[0].String interfaceName = parsed[2].List[1].String } // Get the network entry network, _, err := d.GetNetwork(networkName) if err != nil { return err } // Prepare the instance's device entry var device map[string]string if network.Managed && d.HasExtension("instance_nic_network") { // If network is managed, use the network property rather than nictype, so that the network's // inherited properties are loaded into the NIC when started. device = map[string]string{ "type": "nic", "network": network.Name, } } else { // If network is unmanaged default to using a macvlan connected to the specified interface. device = map[string]string{ "type": "nic", "nictype": "macvlan", "parent": networkName, } if network.Type == "bridge" { // If the network type is an unmanaged bridge, use bridged NIC type. device["nictype"] = "bridged" } } device["name"] = interfaceName // Add the device to the profile err = profileDeviceAdd(d, profileName, deviceName, device) if err != nil { return err } return nil } // Create. type cmdNetworkCreate struct { global *cmdGlobal network *cmdNetwork flagDescription string } var cmdNetworkCreateUsage = u.Usage{u.NewName(u.Network).Remote(), u.KV.List(0)} func (c *cmdNetworkCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdNetworkCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create new networks") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Create new networks`)) cmd.Example = cli.FormatSection("", i18n.G(`incus network create foo Create a new network called foo incus network create foo < config.yaml Create a new network called foo using the content of config.yaml. incus network create bar network=baz --type ovn Create a new OVN network called bar using baz as its uplink network`)) cli.AddStringFlag(cmd.Flags(), &c.network.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.network.flagType, "type|t", "", "", i18n.G("Network type")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Network description")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) != 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return c.global.cmpRemotes(toComplete, false) } return cmd } func (c *cmdNetworkCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String keys, err := kvToMap(parsed[1]) if err != nil { return err } var stdinData api.NetworkPut // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } err = loader.Load(&stdinData) if err != nil && !errors.Is(err, io.EOF) { return err } } // Create the network network := api.NetworksPost{ NetworkPut: stdinData, } network.Name = networkName network.Type = c.network.flagType if c.flagDescription != "" { network.Description = c.flagDescription } if network.Config == nil { network.Config = map[string]string{} } maps.Copy(network.Config, keys) // If a target member was specified the API won't actually create the // network, but only define it as pending in the database. if c.network.flagTarget != "" { d = d.UseTarget(c.network.flagTarget) } err = d.CreateNetwork(network) if err != nil { return err } if !c.global.flagQuiet { if c.network.flagTarget != "" { fmt.Printf(i18n.G("Network %s pending on member %s")+"\n", formatRemote(c.global.conf, parsed[0]), c.network.flagTarget) } else { fmt.Printf(i18n.G("Network %s created")+"\n", formatRemote(c.global.conf, parsed[0])) } } return nil } // Delete. type cmdNetworkDelete struct { global *cmdGlobal network *cmdNetwork } var cmdNetworkDeleteUsage = u.Usage{u.Network.Remote().List(1)} func (c *cmdNetworkDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdNetworkDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete networks") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete networks`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpNetworks(toComplete) } return cmd } func (c *cmdNetworkDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } var errs []error for _, p := range parsed[0].List { d := p.RemoteServer networkName := p.RemoteObject.String // Delete the network err = d.DeleteNetwork(networkName) if err != nil { errs = append(errs, err) continue } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network %s deleted")+"\n", formatRemote(c.global.conf, p)) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } // Detach. type cmdNetworkDetach struct { global *cmdGlobal network *cmdNetwork } var cmdNetworkDetachUsage = u.Usage{u.Network.Remote(), u.Instance, u.Device.Optional()} func (c *cmdNetworkDetach) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("detach", cmdNetworkDetachUsage...) cmd.Short = i18n.G("Detach network interfaces from instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Detach network interfaces from instances`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkInstances(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // Find a matching device. func (c *cmdNetworkDetach) findDevice(devices map[string]map[string]string, networkName string, dev *u.Parsed) (string, error) { hasDevice := !dev.Skipped devName := dev.String found := false for n, d := range devices { if hasDevice { if n == devName { if d["type"] != "nic" { return "", fmt.Errorf(i18n.G("The specified device is not a NIC (%s device)"), d["type"]) } if d["parent"] != networkName && d["network"] != networkName { return "", fmt.Errorf(i18n.G("The specified NIC does not point to the given network (found %s)"), d["parent"]+d["network"]) } found = true break } continue } if d["type"] == "nic" && (d["parent"] == networkName || d["network"] == networkName) { if found { return "", errors.New(i18n.G("More than one device matches, specify the device name")) } devName = n found = true } } if !found { return "", errors.New(i18n.G("No device found for this network")) } return devName, nil } func (c *cmdNetworkDetach) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkDetachUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String instanceName := parsed[1].String // Get the instance entry inst, etag, err := d.GetInstance(instanceName) if err != nil { return err } deviceName, err := c.findDevice(inst.Devices, networkName, parsed[2]) if err != nil { return err } // Remove the device delete(inst.Devices, deviceName) op, err := d.UpdateInstance(instanceName, inst.Writable(), etag) if err != nil { return err } return op.Wait() } // Detach profile. type cmdNetworkDetachProfile struct { global *cmdGlobal network *cmdNetwork networkDetach *cmdNetworkDetach } var cmdNetworkDetachProfileUsage = u.Usage{u.Network.Remote(), u.Profile, u.Device.Optional()} func (c *cmdNetworkDetachProfile) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("detach-profile", cmdNetworkDetachProfileUsage...) cmd.Short = i18n.G("Detach network interfaces from profiles") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Detach network interfaces from profiles`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkProfiles(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkDetachProfile) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkDetachProfileUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String profileName := parsed[1].String // Get the profile entry profile, etag, err := d.GetProfile(profileName) if err != nil { return err } deviceName, err := c.networkDetach.findDevice(profile.Devices, networkName, parsed[2]) if err != nil { return err } // Remove the device delete(profile.Devices, deviceName) err = d.UpdateProfile(profileName, profile.Writable(), etag) if err != nil { return err } return nil } // Edit. type cmdNetworkEdit struct { global *cmdGlobal network *cmdNetwork } var cmdNetworkEditUsage = u.Usage{u.Network.Remote()} func (c *cmdNetworkEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdNetworkEditUsage...) cmd.Short = i18n.G("Edit network configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Edit network configurations as YAML`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) != 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return c.global.cmpNetworks(toComplete) } return cmd } func (c *cmdNetworkEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the network. ### Any line starting with a '# will be ignored. ### ### A network consists of a set of configuration items. ### ### An example would look like: ### name: mybr0 ### config: ### ipv4.address: 10.62.42.1/24 ### ipv4.nat: true ### ipv6.address: fd00:56ad:9f7a:9800::1/64 ### ipv6.nat: true ### managed: true ### type: bridge ### ### Note that only the configuration can be changed.`) } func (c *cmdNetworkEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } newdata := api.NetworkPut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateNetwork(networkName, newdata, "") } // Extract the current value network, etag, err := d.GetNetwork(networkName) if err != nil { return err } if !network.Managed { return errors.New(i18n.G("Only managed networks can be modified")) } data, err := yaml.Dump(&network, yaml.V2) if err != nil { return err } // Spawn the editor content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor newdata := api.NetworkPut{} err = yaml.Load(content, &newdata) if err == nil { err = d.UpdateNetwork(networkName, newdata, etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Get. type cmdNetworkGet struct { global *cmdGlobal network *cmdNetwork flagIsProperty bool } var cmdNetworkGetUsage = u.Usage{u.Network.Remote(), u.Key} func (c *cmdNetworkGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdNetworkGetUsage...) cmd.Short = i18n.G("Get values for network configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Get values for network configuration keys`)) cli.AddStringFlag(cmd.Flags(), &c.network.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a network property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String key := parsed[1].String // Get the network key if c.network.flagTarget != "" { d = d.UseTarget(c.network.flagTarget) } resp, _, err := d.GetNetwork(networkName) if err != nil { return err } if c.flagIsProperty { w := resp.Writable() res, err := getFieldByJSONTag(&w, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the network %q: %v"), key, formatRemote(c.global.conf, parsed[0]), err) } fmt.Printf("%v\n", res) } else { for k, v := range resp.Config { if k == key { fmt.Printf("%s\n", v) } } } return nil } // Info. type cmdNetworkInfo struct { global *cmdGlobal network *cmdNetwork } var cmdNetworkInfoUsage = u.Usage{u.Network.Remote()} func (c *cmdNetworkInfo) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("info", cmdNetworkInfoUsage...) cmd.Short = i18n.G("Get runtime information on networks") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Get runtime information on networks`)) cli.AddStringFlag(cmd.Flags(), &c.network.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) != 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return c.global.cmpNetworks(toComplete) } return cmd } func (c *cmdNetworkInfo) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkInfoUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String // Targeting. if c.network.flagTarget != "" { if !d.IsClustered() { return errors.New(i18n.G("To use --target, the destination remote must be a cluster")) } d = d.UseTarget(c.network.flagTarget) } state, err := d.GetNetworkState(networkName) if err != nil { return err } // Interface information. fmt.Printf(i18n.G("Name: %s")+"\n", networkName) if state.Hwaddr != "" { fmt.Printf(i18n.G("MAC address: %s")+"\n", state.Hwaddr) } fmt.Printf(i18n.G("MTU: %d")+"\n", state.Mtu) fmt.Printf(i18n.G("State: %s")+"\n", state.State) fmt.Printf(i18n.G("Type: %s")+"\n", state.Type) // IP addresses. if len(state.Addresses) > 0 { fmt.Println("") fmt.Println(i18n.G("IP addresses:")) for _, addr := range state.Addresses { fmt.Printf(" %s\t%s/%s (%s)\n", addr.Family, addr.Address, addr.Netmask, addr.Scope) } } // Network usage. if state.Counters != nil { fmt.Println("") fmt.Println(i18n.G("Network usage:")) fmt.Printf(" %s: %s\n", i18n.G("Bytes received"), units.GetByteSizeString(state.Counters.BytesReceived, 2)) fmt.Printf(" %s: %s\n", i18n.G("Bytes sent"), units.GetByteSizeString(state.Counters.BytesSent, 2)) fmt.Printf(" %s: %d\n", i18n.G("Packets received"), state.Counters.PacketsReceived) fmt.Printf(" %s: %d\n", i18n.G("Packets sent"), state.Counters.PacketsSent) } // Bond information. if state.Bond != nil { fmt.Println("") fmt.Println(i18n.G("Bond:")) fmt.Printf(" %s: %s\n", i18n.G("Mode"), state.Bond.Mode) fmt.Printf(" %s: %s\n", i18n.G("Transmit policy"), state.Bond.TransmitPolicy) fmt.Printf(" %s: %d\n", i18n.G("Up delay"), state.Bond.UpDelay) fmt.Printf(" %s: %d\n", i18n.G("Down delay"), state.Bond.DownDelay) fmt.Printf(" %s: %d\n", i18n.G("MII Frequency"), state.Bond.MIIFrequency) fmt.Printf(" %s: %s\n", i18n.G("MII state"), state.Bond.MIIState) fmt.Printf(" %s: %s\n", i18n.G("Lower devices"), strings.Join(state.Bond.LowerDevices, ", ")) } // Bridge information. if state.Bridge != nil { fmt.Println("") fmt.Println(i18n.G("Bridge:")) fmt.Printf(" %s: %s\n", i18n.G("ID"), state.Bridge.ID) fmt.Printf(" %s: %v\n", i18n.G("STP"), state.Bridge.STP) fmt.Printf(" %s: %d\n", i18n.G("Forward delay"), state.Bridge.ForwardDelay) fmt.Printf(" %s: %d\n", i18n.G("Default VLAN ID"), state.Bridge.VLANDefault) fmt.Printf(" %s: %v\n", i18n.G("VLAN filtering"), state.Bridge.VLANFiltering) fmt.Printf(" %s: %s\n", i18n.G("Upper devices"), strings.Join(state.Bridge.UpperDevices, ", ")) } // VLAN information. if state.VLAN != nil { fmt.Println("") fmt.Println(i18n.G("VLAN:")) fmt.Printf(" %s: %s\n", i18n.G("Lower device"), state.VLAN.LowerDevice) fmt.Printf(" %s: %d\n", i18n.G("VLAN ID"), state.VLAN.VID) } // OVN information. if state.OVN != nil { fmt.Println("") fmt.Println(i18n.G("OVN:")) if state.OVN.Chassis != "" { fmt.Printf(" %s: %s\n", i18n.G("Chassis"), state.OVN.Chassis) } if state.OVN.LogicalRouter != "" { fmt.Printf(" %s: %s\n", i18n.G("Logical router"), state.OVN.LogicalRouter) } if state.OVN.LogicalSwitch != "" { fmt.Printf(" %s: %s\n", i18n.G("Logical switch"), state.OVN.LogicalSwitch) } if state.OVN.UplinkIPv4 != "" { fmt.Printf(" %s: %s\n", i18n.G("IPv4 uplink address"), state.OVN.UplinkIPv4) } if state.OVN.UplinkIPv6 != "" { fmt.Printf(" %s: %s\n", i18n.G("IPv6 uplink address"), state.OVN.UplinkIPv6) } } return nil } // List. type cmdNetworkList struct { global *cmdGlobal network *cmdNetwork flagFormat string flagColumns string flagAllProjects bool } var cmdNetworkListUsage = u.Usage{u.RemoteColonOpt, u.Filter.List(0)} func (c *cmdNetworkList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdNetworkListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List available networks") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List available networks Filters may be of the = form for property based filtering, or part of the network name. Filters must be delimited by a ','. Examples: - "foo" lists all networks that start with the name foo - "name=foo" lists all networks that exactly have the name foo - "type=bridge" lists all networks with the type bridge The -c option takes a (optionally comma-separated) list of arguments that control which image attributes to output when displaying in table or csv format. Default column layout is: ntm46dus Column shorthand chars: 4 - IPv4 address 6 - IPv6 address d - Description e - Project name m - Managed status n - Network Interface Name s - State t - Interface type u - Used by (count)`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultNetworkColumns, "", i18n.G("Columns")) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("List networks in all projects")) cli.AddStringFlag(cmd.Flags(), &c.network.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) != 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return c.global.cmpRemotes(toComplete, false) } return cmd } const defaultNetworkColumns = "ntm46dus" func (c *cmdNetworkList) parseColumns() ([]networkColumn, error) { columnsShorthandMap := map[rune]networkColumn{ 'e': {i18n.G("PROJECT"), c.projectColumnData}, 'n': {i18n.G("NAME"), c.networkNameColumnData}, 't': {i18n.G("TYPE"), c.typeColumnData}, 'm': {i18n.G("MANAGED"), c.managedColumnData}, '4': {i18n.G("IPV4"), c.ipv4ColumnData}, '6': {i18n.G("IPV6"), c.ipv6ColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 'u': {i18n.G("USED BY"), c.usedByColumnData}, 's': {i18n.G("STATE"), c.stateColumnData}, } if c.flagColumns == defaultNetworkColumns && c.flagAllProjects { c.flagColumns = "entm46dus" } columnList := strings.Split(c.flagColumns, ",") columns := []networkColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdNetworkList) networkNameColumnData(network api.Network) string { return network.Name } func (c *cmdNetworkList) typeColumnData(network api.Network) string { return network.Type } func (c *cmdNetworkList) managedColumnData(network api.Network) string { if network.Managed { return i18n.G("YES") } return i18n.G("NO") } func (c *cmdNetworkList) projectColumnData(network api.Network) string { return network.Project } func (c *cmdNetworkList) ipv4ColumnData(network api.Network) string { return network.Config["ipv4.address"] } func (c *cmdNetworkList) ipv6ColumnData(network api.Network) string { return network.Config["ipv6.address"] } func (c *cmdNetworkList) descriptionColumnData(network api.Network) string { return network.Description } func (c *cmdNetworkList) usedByColumnData(network api.Network) string { return fmt.Sprintf("%d", len(network.UsedBy)) } func (c *cmdNetworkList) stateColumnData(network api.Network) string { return strings.ToUpper(network.Status) } func (c *cmdNetworkList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer filters := parsed[1].StringList filters = prepareNetworkServerFilters(filters) serverFilters, _ := getServerSupportedFilters(filters, []string{}, false) if c.network.flagTarget != "" { d = d.UseTarget(c.network.flagTarget) } var networks []api.Network if c.flagAllProjects { networks, err = d.GetNetworksAllProjectsWithFilter(serverFilters) } else { networks, err = d.GetNetworksWithFilter(serverFilters) } if err != nil { return err } // Parse column flags. columns, err := c.parseColumns() if err != nil { return err } data := [][]string{} for _, network := range networks { line := []string{} for _, column := range columns { line = append(line, column.Data(network)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, networks) } // List leases. type cmdNetworkListLeases struct { global *cmdGlobal network *cmdNetwork flagFormat string flagColumns string } type networkLeasesColumn struct { Name string Data func(api.NetworkLease) string } var cmdNetworkListLeasesUsage = u.Usage{u.Network.Remote()} func (c *cmdNetworkListLeases) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list-leases", cmdNetworkListLeasesUsage...) cmd.Short = i18n.G("List DHCP leases") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List DHCP leases Default column layout: hmitL == Columns == The -c option takes a comma separated list of arguments that control which network zone attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: h - Hostname m - MAC Address i - IP Address t - Type L - Location of the DHCP Lease (e.g. its cluster member)`)) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultNetworkListLeasesColumns, "", i18n.G("Columns")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) != 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return c.global.cmpNetworks(toComplete) } return cmd } const defaultNetworkListLeasesColumns = "hmit" func (c *cmdNetworkListLeases) parseColumns(clustered bool) ([]networkLeasesColumn, error) { columnsShorthandMap := map[rune]networkLeasesColumn{ 'h': {i18n.G("HOSTNAME"), c.hostnameColumnData}, 'm': {i18n.G("MAC ADDRESS"), c.macAddressColumnData}, 'i': {i18n.G("IP ADDRESS"), c.ipAddressColumnData}, 't': {i18n.G("TYPE"), c.typeColumnData}, 'L': {i18n.G("LOCATION"), c.locationColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []networkLeasesColumn{} if c.flagColumns == defaultNetworkListLeasesColumns && clustered { columnList = append(columnList, "L") } for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdNetworkListLeases) hostnameColumnData(lease api.NetworkLease) string { return lease.Hostname } func (c *cmdNetworkListLeases) macAddressColumnData(lease api.NetworkLease) string { return lease.Hwaddr } func (c *cmdNetworkListLeases) ipAddressColumnData(lease api.NetworkLease) string { return lease.Address } func (c *cmdNetworkListLeases) typeColumnData(lease api.NetworkLease) string { return strings.ToUpper(lease.Type) } func (c *cmdNetworkListLeases) locationColumnData(lease api.NetworkLease) string { return lease.Location } func (c *cmdNetworkListLeases) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkListLeasesUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String // List DHCP leases leases, err := d.GetNetworkLeases(networkName) if err != nil { return err } // Parse column flags. columns, err := c.parseColumns(d.IsClustered()) if err != nil { return err } data := [][]string{} for _, lease := range leases { line := []string{} for _, column := range columns { line = append(line, column.Data(lease)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, leases) } // Rename. type cmdNetworkRename struct { global *cmdGlobal network *cmdNetwork } var cmdNetworkRenameUsage = u.Usage{u.Network.Remote(), u.NewName(u.Network)} func (c *cmdNetworkRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdNetworkRenameUsage...) cmd.Aliases = []string{"mv"} cmd.Short = i18n.G("Rename networks") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Rename networks`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) != 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return c.global.cmpNetworks(toComplete) } return cmd } func (c *cmdNetworkRename) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkRenameUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String newNetworkName := parsed[1].String // Rename the network err = d.RenameNetwork(networkName, api.NetworkPost{Name: newNetworkName}) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network %s renamed to %s")+"\n", formatRemote(c.global.conf, parsed[0]), newNetworkName) } return nil } // Set. type cmdNetworkSet struct { global *cmdGlobal network *cmdNetwork flagIsProperty bool } var cmdNetworkSetUsage = u.Usage{u.Network.Remote(), u.LegacyKV.List(1)} func (c *cmdNetworkSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdNetworkSetUsage...) cmd.Short = i18n.G("Set network configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set network configuration keys For backward compatibility, a single configuration key may still be set with: incus network set [:] `)) cli.AddStringFlag(cmd.Flags(), &c.network.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a network property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) != 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return c.global.cmpNetworks(toComplete) } return cmd } // set runs the post-parsing command logic. func (c *cmdNetworkSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String keys, err := kvToMap(parsed[1]) if err != nil { return err } // Handle targeting if c.network.flagTarget != "" { d = d.UseTarget(c.network.flagTarget) } // Get the network network, etag, err := d.GetNetwork(networkName) if err != nil { return err } if !network.Managed { return errors.New(i18n.G("Only managed networks can be modified")) } writable := network.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { maps.Copy(writable.Config, keys) } return d.UpdateNetwork(networkName, writable, etag) } func (c *cmdNetworkSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Show. type cmdNetworkShow struct { global *cmdGlobal network *cmdNetwork } var cmdNetworkShowUsage = u.Usage{u.Network.Remote()} func (c *cmdNetworkShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdNetworkShowUsage...) cmd.Short = i18n.G("Show network configurations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show network configurations`)) cli.AddStringFlag(cmd.Flags(), &c.network.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) != 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return c.global.cmpNetworks(toComplete) } return cmd } func (c *cmdNetworkShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String // Show the network config if c.network.flagTarget != "" { d = d.UseTarget(c.network.flagTarget) } network, _, err := d.GetNetwork(networkName) if err != nil { return err } sort.Strings(network.UsedBy) data, err := yaml.Dump(&network, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Unset. type cmdNetworkUnset struct { global *cmdGlobal network *cmdNetwork networkSet *cmdNetworkSet flagIsProperty bool } var cmdNetworkUnsetUsage = u.Usage{u.Network.Remote(), u.Key} func (c *cmdNetworkUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdNetworkUnsetUsage...) cmd.Short = i18n.G("Unset network configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Unset network configuration keys`)) cli.AddStringFlag(cmd.Flags(), &c.network.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a network property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.networkSet.flagIsProperty = c.flagIsProperty return unsetKey(c.networkSet, cmd, parsed) } // prepareNetworkServerFilter processes and formats filter criteria // for networks, ensuring they are in a format that the server can interpret. func prepareNetworkServerFilters(filters []string) []string { flattenedFilters := []string{} for _, filter := range filters { flattenedFilters = append(flattenedFilters, strings.Split(filter, ",")...) } formattedFilters := []string{} for _, filter := range flattenedFilters { membs := strings.SplitN(filter, "=", 2) key := membs[0] if len(membs) == 1 { filter = fmt.Sprintf("name=^%s($|.*)", regexp.QuoteMeta(key)) } else if len(membs) == 2 { firstPart := key if strings.Contains(key, ".") { firstPart = strings.Split(key, ".")[0] } if !structHasField(reflect.TypeOf(api.Network{}), firstPart) { filter = fmt.Sprintf("config.%s", filter) } if key == "state" { filter = fmt.Sprintf("status=%s", membs[1]) } } formattedFilters = append(formattedFilters, filter) } return formattedFilters } incus-7.0.0/cmd/incus/network_acl.go000066400000000000000000000670671517523235500174120ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "maps" "os" "reflect" "sort" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" "github.com/lxc/incus/v7/shared/util" ) type cmdNetworkACL struct { global *cmdGlobal } func (c *cmdNetworkACL) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("acl") cmd.Short = i18n.G("Manage network ACLs") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Manage network ACLs")) // List. networkACLListCmd := cmdNetworkACLList{global: c.global, networkACL: c} cmd.AddCommand(networkACLListCmd.command()) // Show. networkACLShowCmd := cmdNetworkACLShow{global: c.global, networkACL: c} cmd.AddCommand(networkACLShowCmd.command()) // Show log. networkACLShowLogCmd := cmdNetworkACLShowLog{global: c.global, networkACL: c} cmd.AddCommand(networkACLShowLogCmd.command()) // Get. networkACLGetCmd := cmdNetworkACLGet{global: c.global, networkACL: c} cmd.AddCommand(networkACLGetCmd.command()) // Create. networkACLCreateCmd := cmdNetworkACLCreate{global: c.global, networkACL: c} cmd.AddCommand(networkACLCreateCmd.command()) // Set. networkACLSetCmd := cmdNetworkACLSet{global: c.global, networkACL: c} cmd.AddCommand(networkACLSetCmd.command()) // Unset. networkACLUnsetCmd := cmdNetworkACLUnset{global: c.global, networkACL: c, networkACLSet: &networkACLSetCmd} cmd.AddCommand(networkACLUnsetCmd.command()) // Edit. networkACLEditCmd := cmdNetworkACLEdit{global: c.global, networkACL: c} cmd.AddCommand(networkACLEditCmd.command()) // Rename. networkACLRenameCmd := cmdNetworkACLRename{global: c.global, networkACL: c} cmd.AddCommand(networkACLRenameCmd.command()) // Delete. networkACLDeleteCmd := cmdNetworkACLDelete{global: c.global, networkACL: c} cmd.AddCommand(networkACLDeleteCmd.command()) // Rule. networkACLRuleCmd := cmdNetworkACLRule{global: c.global, networkACL: c} cmd.AddCommand(networkACLRuleCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // List. type cmdNetworkACLList struct { global *cmdGlobal networkACL *cmdNetworkACL flagFormat string flagAllProjects bool } var cmdNetworkACLListUsage = u.Usage{u.RemoteColonOpt} func (c *cmdNetworkACLList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdNetworkACLListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List available network ACLS") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("List available network ACL")) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("List network ACLs across all projects")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkACLList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkACLListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer var acls []api.NetworkACL if c.flagAllProjects { acls, err = d.GetNetworkACLsAllProjects() if err != nil { return err } } else { acls, err = d.GetNetworkACLs() if err != nil { return err } } data := [][]string{} for _, acl := range acls { strUsedBy := fmt.Sprintf("%d", len(acl.UsedBy)) details := []string{ acl.Name, acl.Description, strUsedBy, } if c.flagAllProjects { details = append([]string{acl.Project}, details...) } data = append(data, details) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{ i18n.G("NAME"), i18n.G("DESCRIPTION"), i18n.G("USED BY"), } if c.flagAllProjects { header = append([]string{i18n.G("PROJECT")}, header...) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, acls) } // Show. type cmdNetworkACLShow struct { global *cmdGlobal networkACL *cmdNetworkACL } var cmdNetworkACLShowUsage = u.Usage{u.ACL.Remote()} func (c *cmdNetworkACLShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdNetworkACLShowUsage...) cmd.Short = i18n.G("Show network ACL configurations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Show network ACL configurations")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkACLs(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkACLShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkACLShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer aclName := parsed[0].RemoteObject.String // Show the network ACL config. netACL, _, err := d.GetNetworkACL(aclName) if err != nil { return err } sort.Strings(netACL.UsedBy) data, err := yaml.Dump(&netACL, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Show log. type cmdNetworkACLShowLog struct { global *cmdGlobal networkACL *cmdNetworkACL } var cmdNetworkACLShowLogUsage = u.Usage{u.ACL.Remote()} func (c *cmdNetworkACLShowLog) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show-log", cmdNetworkACLShowLogUsage...) cmd.Short = i18n.G("Show network ACL log") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Show network ACL log")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkACLs(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkACLShowLog) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkACLShowLogUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer aclName := parsed[0].RemoteObject.String // Get the ACL log. log, err := d.GetNetworkACLLogfile(aclName) if err != nil { return err } _, err = util.SafeCopy(os.Stdout, log) _ = log.Close() return err } // Get. type cmdNetworkACLGet struct { global *cmdGlobal networkACL *cmdNetworkACL flagIsProperty bool } var cmdNetworkACLGetUsage = u.Usage{u.ACL.Remote(), u.Key} func (c *cmdNetworkACLGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdNetworkACLGetUsage...) cmd.Short = i18n.G("Get values for network ACL configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Get values for network ACL configuration keys")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a network ACL property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkACLs(toComplete) } if len(args) == 1 { return c.global.cmpNetworkACLConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkACLGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkACLGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer aclName := parsed[0].RemoteObject.String key := parsed[1].String resp, _, err := d.GetNetworkACL(aclName) if err != nil { return err } if c.flagIsProperty { w := resp.Writable() res, err := getFieldByJSONTag(&w, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the network ACL %q: %v"), key, formatRemote(c.global.conf, parsed[0]), err) } fmt.Printf("%v\n", res) } else { for k, v := range resp.Config { if k == key { fmt.Printf("%s\n", v) } } } return nil } // Create. type cmdNetworkACLCreate struct { global *cmdGlobal networkACL *cmdNetworkACL flagDescription string } var cmdNetworkACLCreateUsage = u.Usage{u.NewName(u.ACL).Remote(), u.KV.List(0)} func (c *cmdNetworkACLCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdNetworkACLCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create new network ACLs") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Create new network ACLs")) cmd.Example = cli.FormatSection("", i18n.G(`incus network acl create a1 Create network acl a1 incus network acl create a1 < config.yaml Create network acl with configuration from config.yaml`)) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Network ACL description")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkACLs(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkACLCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkACLCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer aclName := parsed[0].RemoteObject.String keys, err := kvToMap(parsed[1]) if err != nil { return err } // If stdin isn't a terminal, read yaml from it. var aclPut api.NetworkACLPut if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } err = loader.Load(&aclPut) if err != nil && !errors.Is(err, io.EOF) { return err } } // Create the network ACL. acl := api.NetworkACLsPost{ NetworkACLPost: api.NetworkACLPost{ Name: aclName, }, NetworkACLPut: aclPut, } if c.flagDescription != "" { acl.Description = c.flagDescription } if acl.Config == nil { acl.Config = map[string]string{} } maps.Copy(acl.Config, keys) err = d.CreateNetworkACL(acl) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network ACL %s created")+"\n", formatRemote(c.global.conf, parsed[0])) } return nil } // Set. type cmdNetworkACLSet struct { global *cmdGlobal networkACL *cmdNetworkACL flagIsProperty bool } var cmdNetworkACLSetUsage = u.Usage{u.ACL.Remote(), u.LegacyKV.List(1)} func (c *cmdNetworkACLSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdNetworkACLSetUsage...) cmd.Short = i18n.G("Set network ACL configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set network ACL configuration keys For backward compatibility, a single configuration key may still be set with: incus network set [:] `)) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a network ACL property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkACLs(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdNetworkACLSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer aclName := parsed[0].RemoteObject.String keys, err := kvToMap(parsed[1]) if err != nil { return err } // Get the network ACL. netACL, etag, err := d.GetNetworkACL(aclName) if err != nil { return err } writable := netACL.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { maps.Copy(writable.Config, keys) } return d.UpdateNetworkACL(aclName, writable, etag) } func (c *cmdNetworkACLSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkACLSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Unset. type cmdNetworkACLUnset struct { global *cmdGlobal networkACL *cmdNetworkACL networkACLSet *cmdNetworkACLSet flagIsProperty bool } var cmdNetworkACLUnsetUsage = u.Usage{u.ACL.Remote(), u.Key} func (c *cmdNetworkACLUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdNetworkACLUnsetUsage...) cmd.Short = i18n.G("Unset network ACL configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Unset network ACL configuration keys")) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a network ACL property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkACLs(toComplete) } if len(args) == 1 { return c.global.cmpNetworkACLConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkACLUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkACLUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.networkACLSet.flagIsProperty = c.flagIsProperty return unsetKey(c.networkACLSet, cmd, parsed) } // Edit. type cmdNetworkACLEdit struct { global *cmdGlobal networkACL *cmdNetworkACL } var cmdNetworkACLEditUsage = u.Usage{u.ACL.Remote()} func (c *cmdNetworkACLEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdNetworkACLEditUsage...) cmd.Short = i18n.G("Edit network ACL configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Edit network ACL configurations as YAML")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkACLs(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkACLEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the network ACL. ### Any line starting with a '# will be ignored. ### ### A network ACL consists of a set of rules and configuration items. ### ### An example would look like: ### name: allow-all-inbound ### description: test desc ### egress: [] ### ingress: ### - action: allow ### state: enabled ### protocol: "" ### source: "" ### source_port: "" ### destination: "" ### destination_port: "" ### icmp_type: "" ### icmp_code: "" ### config: ### user.foo: bah ### ### Note that only the ingress and egress rules, description and configuration keys can be changed.`) } func (c *cmdNetworkACLEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkACLEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer aclName := parsed[0].RemoteObject.String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } // Allow output of `incus network acl show` command to be passed in here, but only take the contents // of the NetworkACLPut fields when updating the ACL. The other fields are silently discarded. newdata := api.NetworkACL{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateNetworkACL(aclName, newdata.NetworkACLPut, "") } // Get the current config. netACL, etag, err := d.GetNetworkACL(aclName) if err != nil { return err } data, err := yaml.Dump(&netACL, yaml.V2) if err != nil { return err } // Spawn the editor. content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor. newdata := api.NetworkACL{} // We show the full ACL info, but only send the writable fields. err = yaml.Load(content, &newdata, yaml.WithKnownFields()) if err == nil { err = d.UpdateNetworkACL(aclName, newdata.Writable(), etag) } // Respawn the editor. if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Rename. type cmdNetworkACLRename struct { global *cmdGlobal networkACL *cmdNetworkACL } var cmdNetworkACLRenameUsage = u.Usage{u.ACL.Remote(), u.NewName(u.ACL)} func (c *cmdNetworkACLRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdNetworkACLRenameUsage...) cmd.Aliases = []string{"mv"} cmd.Short = i18n.G("Rename network ACLs") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Rename network ACLs")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkACLs(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkACLRename) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkACLRenameUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer aclName := parsed[0].RemoteObject.String newACLName := parsed[1].String // Rename the network. err = d.RenameNetworkACL(aclName, api.NetworkACLPost{Name: newACLName}) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network ACL %s renamed to %s")+"\n", formatRemote(c.global.conf, parsed[0]), newACLName) } return nil } // Delete. type cmdNetworkACLDelete struct { global *cmdGlobal networkACL *cmdNetworkACL } var cmdNetworkACLDeleteUsage = u.Usage{u.ACL.Remote().List(1)} func (c *cmdNetworkACLDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdNetworkACLDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete network ACLs") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Delete network ACLs")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpNetworkACLs(toComplete) } return cmd } func (c *cmdNetworkACLDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkACLDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } var errs []error for _, p := range parsed[0].List { d := p.RemoteServer aclName := p.RemoteObject.String // Delete the network ACL. err = d.DeleteNetworkACL(aclName) if err != nil { errs = append(errs, err) continue } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network ACL %s deleted")+"\n", formatRemote(c.global.conf, p)) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } // Add/Remove Rule. type cmdNetworkACLRule struct { global *cmdGlobal networkACL *cmdNetworkACL flagRemoveForce bool flagDescription string } func (c *cmdNetworkACLRule) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rule") cmd.Short = i18n.G("Manage network ACL rules") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Manage network ACL rules")) // Rule Add. cmd.AddCommand(c.commandAdd()) // Rule Remove. cmd.AddCommand(c.commandRemove()) return cmd } var cmdNetworkACLRuleAddUsage = u.Usage{u.ACL.Remote(), u.Direction, u.LegacyKV.List(1)} func (c *cmdNetworkACLRule) commandAdd() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("add", cmdNetworkACLRuleAddUsage...) cmd.Aliases = []string{"create"} cmd.Short = i18n.G("Add rules to an ACL") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Add rules to an ACL")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Rule description")) cmd.RunE = c.runAdd cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkACLs(toComplete) } if len(args) == 1 { return []string{"ingress", "egress"}, cobra.ShellCompDirectiveNoFileComp } if len(args) == 2 { return c.global.cmpNetworkACLRuleProperties() } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // networkACLRuleJSONStructFieldMap returns a map of JSON tag names to struct field indices for api.NetworkACLRule. func networkACLRuleJSONStructFieldMap() map[string]int { // Use reflect to get field names in rule from json tags. ruleType := reflect.TypeOf(api.NetworkACLRule{}) allowedKeys := make(map[string]int, ruleType.NumField()) for i := range ruleType.NumField() { field := ruleType.Field(i) if field.PkgPath != "" { continue // Skip unexported fields. It is empty for upper case (exported) field names. } if field.Type.Name() != "string" { continue // Skip non-string fields. } // Split the json tag into its name and options (e.g. json:"action,omitempty"). tagParts := strings.SplitN(string(field.Tag.Get(("json"))), ",", 2) fieldName := tagParts[0] if fieldName == "" { continue // Skip fields with no tagged field name. } allowedKeys[fieldName] = i // Add the name to allowed keys and record field index. } return allowedKeys } // parseConfigKeysToRule converts a map of key/value pairs into an api.NetworkACLRule using reflection. func (c *cmdNetworkACLRule) parseConfigToRule(config map[string]string) (*api.NetworkACLRule, error) { // Use reflect to get struct field indices in NetworkACLRule for json tags. allowedKeys := networkACLRuleJSONStructFieldMap() // Initialize new rule. rule := api.NetworkACLRule{} ruleValue := reflect.ValueOf(&rule).Elem() for k, v := range config { fieldIndex, found := allowedKeys[k] if !found { return nil, fmt.Errorf(i18n.G("Unknown key: %s"), k) } fieldValue := ruleValue.Field(fieldIndex) if !fieldValue.CanSet() { return nil, fmt.Errorf(i18n.G("Cannot set key: %s"), k) } fieldValue.SetString(v) // Set the value into the struct field. } return &rule, nil } func (c *cmdNetworkACLRule) runAdd(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkACLRuleAddUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer aclName := parsed[0].RemoteObject.String direction := parsed[1].String keys, err := kvToMap(parsed[2]) if err != nil { return err } // Get the network ACL. netACL, etag, err := d.GetNetworkACL(aclName) if err != nil { return err } rule, err := c.parseConfigToRule(keys) if err != nil { return err } if c.flagDescription != "" { rule.Description = c.flagDescription } rule.Normalise() // Strip space. // Default to enabled if not specified. if rule.State == "" { rule.State = "enabled" } // Add rule to the requested direction (if direction valid). switch direction { case "ingress": netACL.Ingress = append(netACL.Ingress, *rule) case "egress": netACL.Egress = append(netACL.Egress, *rule) } return d.UpdateNetworkACL(aclName, netACL.Writable(), etag) } var cmdNetworkACLRuleRemoveUsage = u.Usage{u.ACL.Remote(), u.Direction, u.LegacyKV.List(0)} func (c *cmdNetworkACLRule) commandRemove() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remove", cmdNetworkACLRuleRemoveUsage...) cmd.Aliases = []string{"delete", "rm"} cmd.Short = i18n.G("Remove rules from an ACL") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Remove rules from an ACL")) cli.AddBoolFlag(cmd.Flags(), &c.flagRemoveForce, "force|f", i18n.G("Remove all rules that match")) cmd.RunE = c.runRemove cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkACLs(toComplete) } if len(args) == 1 { return []string{"ingress", "egress"}, cobra.ShellCompDirectiveNoFileComp } if len(args) == 2 { return c.global.cmpNetworkACLRuleProperties() } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkACLRule) runRemove(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkACLRuleRemoveUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer aclName := parsed[0].RemoteObject.String direction := parsed[1].String filters, err := kvToMap(parsed[2]) if err != nil { return err } // Get the network ACL. netACL, etag, err := d.GetNetworkACL(aclName) if err != nil { return err } // Use reflect to get struct field indices in NetworkACLRule for json tags. allowedKeys := networkACLRuleJSONStructFieldMap() // Check the supplied filters match possible fields. for k := range filters { _, found := allowedKeys[k] if !found { return fmt.Errorf(i18n.G("Unknown key: %s"), k) } } // isFilterMatch returns whether the supplied rule has matching field values in the filters supplied. // If no filters are supplied, then the rule is considered to have matched. isFilterMatch := func(rule *api.NetworkACLRule, filters map[string]string) bool { ruleValue := reflect.ValueOf(rule).Elem() for k, v := range filters { fieldIndex, found := allowedKeys[k] if !found { return false } fieldValue := ruleValue.Field(fieldIndex) if fieldValue.String() != v { return false } } return true // Match found as all struct fields match the supplied filter values. } // removeFromRules removes a single rule that matches the filters supplied. If multiple rules match then // an error is returned unless c.flagRemoveForce is true, in which case all matching rules are removed. removeFromRules := func(rules []api.NetworkACLRule, filters map[string]string) ([]api.NetworkACLRule, error) { removed := false newRules := make([]api.NetworkACLRule, 0, len(rules)) for _, r := range rules { if isFilterMatch(&r, filters) { if removed && !c.flagRemoveForce { return nil, errors.New(i18n.G("Multiple rules match. Use --force to remove them all")) } removed = true continue // Don't add removed rule to newRules. } newRules = append(newRules, r) } if !removed { return nil, errors.New(i18n.G("No matching rule(s) found")) } return newRules, nil } // Remove matching rule(s) from the requested direction (if direction valid). switch direction { case "ingress": rules, err := removeFromRules(netACL.Ingress, filters) if err != nil { return err } netACL.Ingress = rules case "egress": rules, err := removeFromRules(netACL.Egress, filters) if err != nil { return err } netACL.Egress = rules } return d.UpdateNetworkACL(aclName, netACL.Writable(), etag) } incus-7.0.0/cmd/incus/network_address_set.go000066400000000000000000000521241517523235500211370ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "maps" "os" "slices" "sort" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" ) // cmdNetworkAddressSet represents the global network address set command. type cmdNetworkAddressSet struct { global *cmdGlobal } func (c *cmdNetworkAddressSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("address-set") cmd.Short = i18n.G("Manage network address sets") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Manage network address sets")) // List networkAddressSetListCmd := cmdNetworkAddressSetList{global: c.global, networkAddressSet: c} cmd.AddCommand(networkAddressSetListCmd.command()) // Show networkAddressSetShowCmd := cmdNetworkAddressSetShow{global: c.global, networkAddressSet: c} cmd.AddCommand(networkAddressSetShowCmd.command()) // Create networkAddressSetCreateCmd := cmdNetworkAddressSetCreate{global: c.global, networkAddressSet: c} cmd.AddCommand(networkAddressSetCreateCmd.command()) // Set networkAddressSetSetCmd := cmdNetworkAddressSetSet{global: c.global, networkAddressSet: c} cmd.AddCommand(networkAddressSetSetCmd.command()) // Unset networkAddressSetUnsetCmd := cmdNetworkAddressSetUnset{global: c.global, networkAddressSet: c, networkAddressSetSet: &networkAddressSetSetCmd} cmd.AddCommand(networkAddressSetUnsetCmd.command()) // Edit networkAddressSetEditCmd := cmdNetworkAddressSetEdit{global: c.global, networkAddressSet: c} cmd.AddCommand(networkAddressSetEditCmd.command()) // Rename networkAddressSetRenameCmd := cmdNetworkAddressSetRename{global: c.global, networkAddressSet: c} cmd.AddCommand(networkAddressSetRenameCmd.command()) // Delete networkAddressSetDeleteCmd := cmdNetworkAddressSetDelete{global: c.global, networkAddressSet: c} cmd.AddCommand(networkAddressSetDeleteCmd.command()) // Add networkAddressSetAddCmd := cmdNetworkAddressSetAdd{global: c.global, networkAddressSet: c} cmd.AddCommand(networkAddressSetAddCmd.command()) // Remove networkAddressSetRemoveCmd := cmdNetworkAddressSetRemove{global: c.global, networkAddressSet: c} cmd.AddCommand(networkAddressSetRemoveCmd.command()) // Workaround for subcommand usage errors cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } return cmd } // cmdNetworkAddressSetList defines the structure for listing network address sets. type cmdNetworkAddressSetList struct { global *cmdGlobal networkAddressSet *cmdNetworkAddressSet flagFormat string flagAllProjects bool } var cmdNetworkAddressSetListUsage = u.Usage{u.RemoteColonOpt} func (c *cmdNetworkAddressSetList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdNetworkAddressSetListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List available network address sets") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("List available network address sets")) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G("Format (csv|json|table|yaml|compact|markdown)")) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("List address sets across all projects")) cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkAddressSetList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkAddressSetListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer var sets []api.NetworkAddressSet if c.flagAllProjects { sets, err = d.GetNetworkAddressSetsAllProjects() if err != nil { return err } } else { sets, err = d.GetNetworkAddressSets() if err != nil { return err } } data := [][]string{} for _, as := range sets { strUsedBy := fmt.Sprintf("%d", len(as.UsedBy)) details := []string{ as.Name, as.Description, strings.Join(as.Addresses, "\n"), strUsedBy, } if c.flagAllProjects { details = append([]string{as.Project}, details...) } data = append(data, details) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{ i18n.G("NAME"), i18n.G("DESCRIPTION"), i18n.G("ADDRESSES"), i18n.G("USED BY"), } if c.flagAllProjects { header = append([]string{i18n.G("PROJECT")}, header...) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, sets) } // cmdNetworkAddressSetShow defines the structure for showing a network address set. type cmdNetworkAddressSetShow struct { global *cmdGlobal networkAddressSet *cmdNetworkAddressSet } var cmdNetworkAddressSetShowUsage = u.Usage{u.AddressSet.Remote()} func (c *cmdNetworkAddressSetShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdNetworkAddressSetShowUsage...) cmd.Short = i18n.G("Show network address set configuration") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Show network address set configuration")) cmd.RunE = c.run cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkAddressSets(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkAddressSetShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkAddressSetShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer addressSetName := parsed[0].RemoteObject.String addrSet, _, err := d.GetNetworkAddressSet(addressSetName) if err != nil { return err } sort.Strings(addrSet.UsedBy) data, err := yaml.Dump(&addrSet, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // cmdNetworkAddressSetCreate defines the structure for creating a network address set. type cmdNetworkAddressSetCreate struct { global *cmdGlobal networkAddressSet *cmdNetworkAddressSet flagDescription string } var cmdNetworkAddressSetCreateUsage = u.Usage{u.NewName(u.AddressSet).Remote(), u.KV.List(0)} func (c *cmdNetworkAddressSetCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdNetworkAddressSetCreateUsage...) cmd.Short = i18n.G("Create new network address sets") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Create new network address sets")) cmd.Example = cli.FormatSection("", i18n.G(`incus network address-set create as1 Create network address set as1 incus network address-set create as1 < config.yaml Create network address set with configuration from config.yaml`)) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Network address set description")) cmd.RunE = c.run cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkAddressSets(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkAddressSetCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkAddressSetCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer addressSetName := parsed[0].RemoteObject.String keys, err := kvToMap(parsed[1]) if err != nil { return err } var asPut api.NetworkAddressSetPut if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } err = loader.Load(&asPut) if err != nil && !errors.Is(err, io.EOF) { return err } } addrSet := api.NetworkAddressSetsPost{ NetworkAddressSetPost: api.NetworkAddressSetPost{ Name: addressSetName, }, NetworkAddressSetPut: asPut, } if c.flagDescription != "" { addrSet.Description = c.flagDescription } if addrSet.Config == nil { addrSet.Config = map[string]string{} } for k, v := range keys { if k == "addresses" { addresses := strings.Split(v, ",") // Split the comma-separated IPs addrSet.Addresses = append(addrSet.Addresses, addresses...) continue } addrSet.Config[k] = v } err = d.CreateNetworkAddressSet(addrSet) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network address set %s created")+"\n", formatRemote(c.global.conf, parsed[0])) } return nil } // cmdNetworkAddressSetSet defines the structure for setting network address set configuration. type cmdNetworkAddressSetSet struct { global *cmdGlobal networkAddressSet *cmdNetworkAddressSet flagIsProperty bool } var cmdNetworkAddressSetSetUsage = u.Usage{u.AddressSet.Remote(), u.LegacyKV.List(1)} func (c *cmdNetworkAddressSetSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdNetworkAddressSetSetUsage...) cmd.Short = i18n.G("Set network address set configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Set network address set configuration keys`)) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a network address set property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkAddressSets(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdNetworkAddressSetSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer addressSetName := parsed[0].RemoteObject.String keys, err := kvToMap(parsed[1]) if err != nil { return err } // Get current address set addrSet, etag, err := d.GetNetworkAddressSet(addressSetName) if err != nil { return err } writable := addrSet.Writable() if writable.Config == nil { writable.Config = make(map[string]string) } if c.flagIsProperty { // handle as properties err = unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } else { maps.Copy(writable.Config, keys) } return d.UpdateNetworkAddressSet(addressSetName, writable, etag) } func (c *cmdNetworkAddressSetSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkAddressSetSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // cmdNetworkAddressSetUnset defines the structure for unsetting network address set configuration keys. type cmdNetworkAddressSetUnset struct { global *cmdGlobal networkAddressSet *cmdNetworkAddressSet networkAddressSetSet *cmdNetworkAddressSetSet flagIsProperty bool } var cmdNetworkAddressSetUnsetUsage = u.Usage{u.AddressSet.Remote(), u.Key} func (c *cmdNetworkAddressSetUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdNetworkAddressSetUnsetUsage...) cmd.Short = i18n.G("Unset network address set configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Unset network address set configuration keys")) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a network address set property")) cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkAddressSets(toComplete) } if len(args) == 1 { return c.global.cmpNetworkAddressSetConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkAddressSetUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkAddressSetUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.networkAddressSetSet.flagIsProperty = c.flagIsProperty return unsetKey(c.networkAddressSetSet, cmd, parsed) } // cmdNetworkAddressSetEdit defines the structure for editing a network address set. type cmdNetworkAddressSetEdit struct { global *cmdGlobal networkAddressSet *cmdNetworkAddressSet } var cmdNetworkAddressSetEditUsage = u.Usage{u.AddressSet.Remote()} func (c *cmdNetworkAddressSetEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdNetworkAddressSetEditUsage...) cmd.Short = i18n.G("Edit network address set configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Edit network address set configurations as YAML")) cmd.RunE = c.run cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkAddressSets(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // helpTemplate provides a YAML template for editing address sets. func (c *cmdNetworkAddressSetEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the network address set. ### Any line starting with '#' will be ignored. ### ### For example: ### name: as1 ### description: "Test address set" ### addresses: ### - 10.0.0.1 ### - 2001:db8::1 ### external_ids: ### user.foo: bar `) } func (c *cmdNetworkAddressSetEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkAddressSetEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer addressSetName := parsed[0].RemoteObject.String // If stdin isn't terminal, read yaml from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } newdata := api.NetworkAddressSet{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateNetworkAddressSet(addressSetName, newdata.Writable(), "") } // Get current config addrSet, etag, err := d.GetNetworkAddressSet(addressSetName) if err != nil { return err } data, err := yaml.Dump(&addrSet, yaml.V2) if err != nil { return err } content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { newdata := api.NetworkAddressSet{} err = yaml.Load(content, &newdata, yaml.WithKnownFields()) if err == nil { err = d.UpdateNetworkAddressSet(addressSetName, newdata.Writable(), etag) } if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err2 := os.Stdin.Read(make([]byte, 1)) if err2 != nil { return err2 } content, err2 = cli.TextEditor("", content) if err2 != nil { return err2 } continue } break } return nil } // cmdNetworkAddressSetRename defines the structure for renaming a network address set. type cmdNetworkAddressSetRename struct { global *cmdGlobal networkAddressSet *cmdNetworkAddressSet } var cmdNetworkAddressSetRenameUsage = u.Usage{u.AddressSet.Remote(), u.NewName(u.AddressSet)} func (c *cmdNetworkAddressSetRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdNetworkAddressSetRenameUsage...) cmd.Aliases = []string{"mv"} cmd.Short = i18n.G("Rename network address sets") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Rename network address sets")) cmd.RunE = c.run cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkAddressSets(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkAddressSetRename) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkAddressSetRenameUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer addressSetName := parsed[0].RemoteObject.String newAddressSetName := parsed[1].String err = d.RenameNetworkAddressSet(addressSetName, api.NetworkAddressSetPost{Name: newAddressSetName}) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network address set %s renamed to %s")+"\n", formatRemote(c.global.conf, parsed[0]), newAddressSetName) } return nil } // cmdNetworkAddressSetDelete defines the structure for deleting a network address set. type cmdNetworkAddressSetDelete struct { global *cmdGlobal networkAddressSet *cmdNetworkAddressSet } var cmdNetworkAddressSetDeleteUsage = u.Usage{u.AddressSet.Remote().List(1)} func (c *cmdNetworkAddressSetDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdNetworkAddressSetDeleteUsage...) cmd.Aliases = []string{"rm"} cmd.Short = i18n.G("Delete network address sets") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Delete network address sets")) cmd.RunE = c.run cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpNetworkAddressSets(toComplete) } return cmd } func (c *cmdNetworkAddressSetDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkAddressSetDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } for _, p := range parsed[0].List { d := p.RemoteServer addressSetName := p.RemoteObject.String // Delete the address set. err = d.DeleteNetworkAddressSet(addressSetName) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network address set %s deleted")+"\n", formatRemote(c.global.conf, p)) } } return nil } // cmdNetworkAddressSetAdd defines the structure for adding addresses to a network address set. type cmdNetworkAddressSetAdd struct { global *cmdGlobal networkAddressSet *cmdNetworkAddressSet } var cmdNetworkAddressSetAddUsage = u.Usage{u.AddressSet.Remote(), u.Address.List(1)} func (c *cmdNetworkAddressSetAdd) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("add", cmdNetworkAddressSetAddUsage...) cmd.Short = i18n.G("Add addresses to a network address set") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Add addresses to a network address set")) cmd.RunE = c.run cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkAddressSets(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkAddressSetAdd) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkAddressSetAddUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer addressSetName := parsed[0].RemoteObject.String addresses := parsed[1].StringList addrSet, etag, err := d.GetNetworkAddressSet(addressSetName) if err != nil { return err } // Add addresses addrSet.Addresses = append(addrSet.Addresses, addresses...) return d.UpdateNetworkAddressSet(addressSetName, addrSet.Writable(), etag) } // cmdNetworkAddressSetRemove defines the structure for removing addresses from a network address set. type cmdNetworkAddressSetRemove struct { global *cmdGlobal networkAddressSet *cmdNetworkAddressSet } var cmdNetworkAddressSetRemoveUsage = u.Usage{u.AddressSet.Remote(), u.Address.List(1)} func (c *cmdNetworkAddressSetRemove) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remove", cmdNetworkAddressSetRemoveUsage...) cmd.Short = i18n.G("Remove addresses from a network address set") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Remove addresses from a network address set")) cmd.RunE = c.run cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkAddressSets(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkAddressSetRemove) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkAddressSetRemoveUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer addressSetName := parsed[0].RemoteObject.String addresses := parsed[1].StringList addrSet, etag, err := d.GetNetworkAddressSet(addressSetName) if err != nil { return err } newAddrs := make([]string, 0, len(addrSet.Addresses)) removedCount := 0 for _, addr := range addrSet.Addresses { match := false if slices.Contains(addresses, addr) { match = true removedCount++ } if !match { newAddrs = append(newAddrs, addr) } } if removedCount != len(addresses) { return errors.New(i18n.G("One or more provided address isn't currently in the set")) } addrSet.Addresses = newAddrs return d.UpdateNetworkAddressSet(addressSetName, addrSet.Writable(), etag) } incus-7.0.0/cmd/incus/network_allocations.go000066400000000000000000000114561517523235500211520ustar00rootroot00000000000000package main import ( "fmt" "os" "sort" "strings" "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdNetworkListAllocations struct { global *cmdGlobal network *cmdNetwork flagFormat string flagProject string flagAllProjects bool flagColumns string } type networkAllocationColumn struct { Name string Data func(api.NetworkAllocations) string } var cmdNetworkListAllocationsUsage = u.Usage{u.RemoteColonOpt} func (c *cmdNetworkListAllocations) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list-allocations", cmdNetworkListAllocationsUsage...) cmd.Short = i18n.G("List network allocations in use") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List network allocations in use Default column layout: uatnm == Columns == The -c option takes a comma separated list of arguments that control which network allocations attribute attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: u - Used by a - Address t - Type n - NAT m - Mac Address`)) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.MaximumNArgs(1) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddStringFlag(cmd.Flags(), &c.flagProject, "project|p", api.ProjectDefaultName, "", i18n.G("Run again a specific project")) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("Run against all projects")) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultNetworkAllocationColumns, "", i18n.G("Columns")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } return cmd } const defaultNetworkAllocationColumns = "uatnm" func (c *cmdNetworkListAllocations) parseColumns() ([]networkAllocationColumn, error) { columnsShorthandMap := map[rune]networkAllocationColumn{ 'u': {i18n.G("USED BY"), c.usedByColumnData}, 'a': {i18n.G("ADDRESS"), c.addressColumnData}, 't': {i18n.G("TYPE"), c.typeColumnData}, 'n': {i18n.G("NAT"), c.natColumnData}, 'm': {i18n.G("MAC ADDRESS"), c.macAddressColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []networkAllocationColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdNetworkListAllocations) usedByColumnData(alloc api.NetworkAllocations) string { return alloc.UsedBy } func (c *cmdNetworkListAllocations) addressColumnData(alloc api.NetworkAllocations) string { return alloc.Address } func (c *cmdNetworkListAllocations) typeColumnData(alloc api.NetworkAllocations) string { return alloc.Type } func (c *cmdNetworkListAllocations) natColumnData(alloc api.NetworkAllocations) string { strNat := "NO" if alloc.NAT { strNat = "YES" } return strNat } func (c *cmdNetworkListAllocations) macAddressColumnData(alloc api.NetworkAllocations) string { return alloc.Hwaddr } func (c *cmdNetworkListAllocations) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkListAllocationsUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer var addresses []api.NetworkAllocations if c.flagAllProjects { addresses, err = d.GetNetworkAllocationsAllProjects() if err != nil { return err } } else { addresses, err = d.GetNetworkAllocations() if err != nil { return err } } columns, err := c.parseColumns() if err != nil { return err } data := [][]string{} for _, address := range addresses { line := []string{} for _, column := range columns { line = append(line, column.Data(address)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, addresses) } incus-7.0.0/cmd/incus/network_forward.go000066400000000000000000000670301517523235500203050ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "maps" "os" "sort" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" ) type cmdNetworkForward struct { global *cmdGlobal flagTarget string } func (c *cmdNetworkForward) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("forward") cmd.Short = i18n.G("Manage network forwards") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Manage network forwards")) // List. networkForwardListCmd := cmdNetworkForwardList{global: c.global, networkForward: c} cmd.AddCommand(networkForwardListCmd.command()) // Show. networkForwardShowCmd := cmdNetworkForwardShow{global: c.global, networkForward: c} cmd.AddCommand(networkForwardShowCmd.command()) // Create. networkForwardCreateCmd := cmdNetworkForwardCreate{global: c.global, networkForward: c} cmd.AddCommand(networkForwardCreateCmd.command()) // Get. networkForwardGetCmd := cmdNetworkForwardGet{global: c.global, networkForward: c} cmd.AddCommand(networkForwardGetCmd.command()) // Set. networkForwardSetCmd := cmdNetworkForwardSet{global: c.global, networkForward: c} cmd.AddCommand(networkForwardSetCmd.command()) // Unset. networkForwardUnsetCmd := cmdNetworkForwardUnset{global: c.global, networkForward: c, networkForwardSet: &networkForwardSetCmd} cmd.AddCommand(networkForwardUnsetCmd.command()) // Edit. networkForwardEditCmd := cmdNetworkForwardEdit{global: c.global, networkForward: c} cmd.AddCommand(networkForwardEditCmd.command()) // Delete. networkForwardDeleteCmd := cmdNetworkForwardDelete{global: c.global, networkForward: c} cmd.AddCommand(networkForwardDeleteCmd.command()) // Port. networkForwardPortCmd := cmdNetworkForwardPort{global: c.global, networkForward: c} cmd.AddCommand(networkForwardPortCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // List. type cmdNetworkForwardList struct { global *cmdGlobal networkForward *cmdNetworkForward flagFormat string flagColumns string } type networkForwardColumn struct { Name string Data func(api.NetworkForward) string } var cmdNetworkForwardListUsage = u.Usage{u.Network.Remote()} func (c *cmdNetworkForwardList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdNetworkForwardListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List available network forwards") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List available network forwards Default column layout: ldDp == Columns == The -c option takes a comma separated list of arguments that control which instance attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: l - Listen Address d - Description D - Default Target Address p - Port L - Location of the network zone (e.g. its cluster member)`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultNetworkForwardColumns, "", i18n.G("Columns")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } const defaultNetworkForwardColumns = "ldDp" func (c *cmdNetworkForwardList) parseColumns(clustered bool) ([]networkForwardColumn, error) { columnsShorthandMap := map[rune]networkForwardColumn{ 'l': {i18n.G("LISTEN ADDRESS"), c.listenAddressColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 'D': {i18n.G("DEFAULT TARGET ADDRESS"), c.defaultTargetAddressColumnData}, 'p': {i18n.G("PORTS"), c.portsColumnData}, 'L': {i18n.G("LOCATION"), c.locationColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []networkForwardColumn{} if c.flagColumns == defaultNetworkForwardColumns && clustered { columnList = append(columnList, "L") } for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdNetworkForwardList) listenAddressColumnData(forward api.NetworkForward) string { return forward.ListenAddress } func (c *cmdNetworkForwardList) descriptionColumnData(forward api.NetworkForward) string { return forward.Description } func (c *cmdNetworkForwardList) defaultTargetAddressColumnData(forward api.NetworkForward) string { return forward.Config["target_address"] } func (c *cmdNetworkForwardList) portsColumnData(forward api.NetworkForward) string { return fmt.Sprintf("%d", len(forward.Ports)) } func (c *cmdNetworkForwardList) locationColumnData(forward api.NetworkForward) string { return forward.Location } func (c *cmdNetworkForwardList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkForwardListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String forwards, err := d.GetNetworkForwards(networkName) if err != nil { return err } // Parse column flags. columns, err := c.parseColumns(d.IsClustered()) if err != nil { return err } data := make([][]string, 0, len(forwards)) for _, forward := range forwards { line := []string{} for _, column := range columns { line = append(line, column.Data(forward)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, forwards) } // Show. type cmdNetworkForwardShow struct { global *cmdGlobal networkForward *cmdNetworkForward } var cmdNetworkForwardShowUsage = u.Usage{u.Network.Remote(), u.ListenAddress} func (c *cmdNetworkForwardShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdNetworkForwardShowUsage...) cmd.Short = i18n.G("Show network forward configurations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Show network forward configurations")) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.networkForward.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkForwards(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkForwardShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkForwardShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String // If a target was specified, create the forward on the given member. if c.networkForward.flagTarget != "" { d = d.UseTarget(c.networkForward.flagTarget) } // Show the network forward config. forward, _, err := d.GetNetworkForward(networkName, listenAddress) if err != nil { return err } data, err := yaml.Dump(&forward, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Create. type cmdNetworkForwardCreate struct { global *cmdGlobal networkForward *cmdNetworkForward flagDescription string } var cmdNetworkForwardCreateUsage = u.Usage{u.Network.Remote(), u.ListenAddress, u.KV.List(0)} func (c *cmdNetworkForwardCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdNetworkForwardCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create new network forwards") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Create new network forwards")) cmd.Example = cli.FormatSection("", i18n.G(`incus network forward create n1 127.0.0.1 incus network forward create n1 127.0.0.1 < config.yaml Create a new network forward for network n1 from config.yaml`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.networkForward.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Network forward description")) return cmd } func (c *cmdNetworkForwardCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkForwardCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String keys, err := kvToMap(parsed[2]) if err != nil { return err } // If stdin isn't a terminal, read yaml from it. var forwardPut api.NetworkForwardPut if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } err = loader.Load(&forwardPut) if err != nil && !errors.Is(err, io.EOF) { return err } } if forwardPut.Config == nil { forwardPut.Config = map[string]string{} } maps.Copy(forwardPut.Config, keys) // Create the network forward. forward := api.NetworkForwardsPost{ ListenAddress: listenAddress, NetworkForwardPut: forwardPut, } if c.flagDescription != "" { forward.Description = c.flagDescription } forward.Normalise() // If a target was specified, create the forward on the given member. if c.networkForward.flagTarget != "" { d = d.UseTarget(c.networkForward.flagTarget) } err = d.CreateNetworkForward(networkName, forward) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network forward %s created")+"\n", forward.ListenAddress) } return nil } // Get. type cmdNetworkForwardGet struct { global *cmdGlobal networkForward *cmdNetworkForward flagIsProperty bool } var cmdNetworkForwardGetUsage = u.Usage{u.Network.Remote(), u.ListenAddress, u.Key} func (c *cmdNetworkForwardGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdNetworkForwardGetUsage...) cmd.Short = i18n.G("Get values for network forward configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Get values for network forward configuration keys")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a network forward property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkForwards(args[0]) } if len(args) == 2 { return c.global.cmpNetworkForwardConfigs(args[0], args[1]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkForwardGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkForwardGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String key := parsed[2].String // Get the current config. forward, _, err := d.GetNetworkForward(networkName, listenAddress) if err != nil { return err } if c.flagIsProperty { w := forward.Writable() res, err := getFieldByJSONTag(&w, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the network forward %q: %v"), key, listenAddress, err) } fmt.Printf("%v\n", res) } else { for k, v := range forward.Config { if k == key { fmt.Printf("%s\n", v) } } } return nil } // Set. type cmdNetworkForwardSet struct { global *cmdGlobal networkForward *cmdNetworkForward flagIsProperty bool } var cmdNetworkForwardSetUsage = u.Usage{u.Network.Remote(), u.ListenAddress, u.LegacyKV.List(1)} func (c *cmdNetworkForwardSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdNetworkForwardSetUsage...) cmd.Short = i18n.G("Set network forward keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set network forward keys For backward compatibility, a single configuration key may still be set with: incus network set [:] `)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a network forward property")) cli.AddStringFlag(cmd.Flags(), &c.networkForward.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkForwards(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdNetworkForwardSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String keys, err := kvToMap(parsed[2]) if err != nil { return err } // If a target was specified, create the forward on the given member. if c.networkForward.flagTarget != "" { d = d.UseTarget(c.networkForward.flagTarget) } // Get the current config. forward, etag, err := d.GetNetworkForward(networkName, listenAddress) if err != nil { return err } if forward.Config == nil { forward.Config = map[string]string{} } writable := forward.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { maps.Copy(writable.Config, keys) } writable.Normalise() return d.UpdateNetworkForward(networkName, forward.ListenAddress, writable, etag) } func (c *cmdNetworkForwardSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkForwardSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Unset. type cmdNetworkForwardUnset struct { global *cmdGlobal networkForward *cmdNetworkForward networkForwardSet *cmdNetworkForwardSet flagIsProperty bool } var cmdNetworkForwardUnsetUsage = u.Usage{u.Network.Remote(), u.ListenAddress, u.Key} func (c *cmdNetworkForwardUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdNetworkForwardUnsetUsage...) cmd.Short = i18n.G("Unset network forward configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Unset network forward keys")) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a network forward property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkForwards(args[0]) } if len(args) == 2 { return c.global.cmpNetworkForwardConfigs(args[0], args[1]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkForwardUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkForwardUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.networkForwardSet.flagIsProperty = c.flagIsProperty return unsetKey(c.networkForwardSet, cmd, parsed) } // Edit. type cmdNetworkForwardEdit struct { global *cmdGlobal networkForward *cmdNetworkForward } var cmdNetworkForwardEditUsage = u.Usage{u.Network.Remote(), u.ListenAddress} func (c *cmdNetworkForwardEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdNetworkForwardEditUsage...) cmd.Short = i18n.G("Edit network forward configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Edit network forward configurations as YAML")) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.networkForward.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkForwards(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkForwardEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the network forward. ### Any line starting with a '# will be ignored. ### ### A network forward consists of a default target address and optional set of port forwards for a listen address. ### ### An example would look like: ### listen_address: 192.0.2.1 ### config: ### target_address: 198.51.100.2 ### description: test desc ### ports: ### - description: port forward ### protocol: tcp ### listen_port: 80,81,8080-8090 ### target_address: 198.51.100.3 ### target_port: 80,81,8080-8090 ### location: server01 ### ### Note that the listen_address and location cannot be changed.`) } func (c *cmdNetworkForwardEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkForwardEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String // If a target was specified, create the forward on the given member. if c.networkForward.flagTarget != "" { d = d.UseTarget(c.networkForward.flagTarget) } // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } // Allow output of `incus network forward show` command to be passed in here, but only take the // contents of the NetworkForwardPut fields when updating. The other fields are silently discarded. newData := api.NetworkForward{} err = loader.Load(&newData) if err != nil && !errors.Is(err, io.EOF) { return err } newData.Normalise() return d.UpdateNetworkForward(networkName, listenAddress, newData.NetworkForwardPut, "") } // Get the current config. forward, etag, err := d.GetNetworkForward(networkName, listenAddress) if err != nil { return err } data, err := yaml.Dump(&forward, yaml.V2) if err != nil { return err } // Spawn the editor. content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor. newData := api.NetworkForward{} // We show the full info, but only send the writable fields. err = yaml.Load(content, &newData, yaml.WithKnownFields()) if err == nil { newData.Normalise() err = d.UpdateNetworkForward(networkName, listenAddress, newData.Writable(), etag) } // Respawn the editor. if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Delete. type cmdNetworkForwardDelete struct { global *cmdGlobal networkForward *cmdNetworkForward } var cmdNetworkForwardDeleteUsage = u.Usage{u.Network.Remote(), u.ListenAddress} func (c *cmdNetworkForwardDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdNetworkForwardDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete network forwards") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Delete network forwards")) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.networkForward.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkForwards(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkForwardDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkForwardDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String // If a target was specified, create the forward on the given member. if c.networkForward.flagTarget != "" { d = d.UseTarget(c.networkForward.flagTarget) } // Delete the network forward. err = d.DeleteNetworkForward(networkName, listenAddress) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network forward %s deleted")+"\n", listenAddress) } return nil } // Add/Remove Port. type cmdNetworkForwardPort struct { global *cmdGlobal networkForward *cmdNetworkForward flagRemoveForce bool flagDescription string } func (c *cmdNetworkForwardPort) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("port") cmd.Short = i18n.G("Manage network forward ports") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Manage network forward ports")) // Port Add. cmd.AddCommand(c.commandAdd()) // Port Remove. cmd.AddCommand(c.commandRemove()) return cmd } var cmdNetworkForwardPortAddUsage = u.Usage{u.Network.Remote(), u.ListenAddress, u.Protocol, u.ListenPort.List(1, ","), u.Target(u.Address), u.Target(u.Port).List(0, ",")} func (c *cmdNetworkForwardPort) commandAdd() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("add", cmdNetworkForwardPortAddUsage...) cmd.Aliases = []string{"create"} cmd.Short = i18n.G("Add ports to a forward") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Add ports to a forward")) cmd.RunE = c.runAdd cli.AddStringFlag(cmd.Flags(), &c.networkForward.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Port description")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkForwards(args[0]) } if len(args) == 2 { return []string{"tcp", "udp"}, cobra.ShellCompDirectiveNoFileComp } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkForwardPort) runAdd(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkForwardPortAddUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String protocol := parsed[2].String // Only the list’s string representation is used. listenPorts := parsed[3].String targetAddress := parsed[4].String // Only the list’s string representation is used. targetPorts := parsed[5].String // If a target was specified, create the forward on the given member. if c.networkForward.flagTarget != "" { d = d.UseTarget(c.networkForward.flagTarget) } // Get the network forward. forward, etag, err := d.GetNetworkForward(networkName, listenAddress) if err != nil { return err } forward.Ports = append(forward.Ports, api.NetworkForwardPort{ Protocol: protocol, ListenPort: listenPorts, TargetAddress: targetAddress, TargetPort: targetPorts, Description: c.flagDescription, }) forward.Normalise() return d.UpdateNetworkForward(networkName, forward.ListenAddress, forward.Writable(), etag) } var cmdNetworkForwardPortRemoveUsage = u.Usage{u.Network.Remote(), u.ListenAddress, u.Protocol.Optional(u.ListenPort.List(0, ","))} func (c *cmdNetworkForwardPort) commandRemove() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remove", cmdNetworkForwardPortRemoveUsage...) cmd.Aliases = []string{"delete", "rm"} cmd.Short = i18n.G("Remove ports from a forward") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Remove ports from a forward")) cli.AddBoolFlag(cmd.Flags(), &c.flagRemoveForce, "force|f", i18n.G("Remove all ports that match")) cmd.RunE = c.runRemove cli.AddStringFlag(cmd.Flags(), &c.networkForward.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkForwards(args[0]) } if len(args) == 2 { return []string{"tcp", "udp"}, cobra.ShellCompDirectiveNoFileComp } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkForwardPort) runRemove(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkForwardPortRemoveUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String hasProtocol := !parsed[2].Skipped protocol := "" hasListenPorts := false listenPorts := "" if hasProtocol { protocol = parsed[2].List[0].String hasListenPorts = !parsed[2].List[1].Skipped // Only the list’s string representation is used. listenPorts = parsed[2].List[1].String } // If a target was specified, create the forward on the given member. if c.networkForward.flagTarget != "" { d = d.UseTarget(c.networkForward.flagTarget) } // Get the network forward. forward, etag, err := d.GetNetworkForward(networkName, listenAddress) if err != nil { return err } removed := false newPorts := make([]api.NetworkForwardPort, 0, len(forward.Ports)) for _, port := range forward.Ports { if hasProtocol && port.Protocol != protocol || hasListenPorts && port.ListenPort != listenPorts { newPorts = append(newPorts, port) } else { if removed && !c.flagRemoveForce { return errors.New(i18n.G("Multiple ports match. Use --force to remove them all")) } removed = true } } if !removed { return errors.New(i18n.G("No matching port(s) found")) } forward.Ports = newPorts forward.Normalise() return d.UpdateNetworkForward(networkName, forward.ListenAddress, forward.Writable(), etag) } incus-7.0.0/cmd/incus/network_integration.go000066400000000000000000000467131517523235500211710ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "maps" "os" "sort" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" ) type cmdNetworkIntegration struct { global *cmdGlobal } func (c *cmdNetworkIntegration) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("integration") cmd.Short = i18n.G("Manage network integrations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage network integrations`)) // Create networkIntegrationCreateCmd := cmdNetworkIntegrationCreate{global: c.global, networkIntegration: c} cmd.AddCommand(networkIntegrationCreateCmd.command()) // Delete networkIntegrationDeleteCmd := cmdNetworkIntegrationDelete{global: c.global, networkIntegration: c} cmd.AddCommand(networkIntegrationDeleteCmd.command()) // Edit networkIntegrationEditCmd := cmdNetworkIntegrationEdit{global: c.global, networkIntegration: c} cmd.AddCommand(networkIntegrationEditCmd.command()) // Get networkIntegrationGetCmd := cmdNetworkIntegrationGet{global: c.global, networkIntegration: c} cmd.AddCommand(networkIntegrationGetCmd.command()) // List networkIntegrationListCmd := cmdNetworkIntegrationList{global: c.global, networkIntegration: c} cmd.AddCommand(networkIntegrationListCmd.command()) // Rename networkIntegrationRenameCmd := cmdNetworkIntegrationRename{global: c.global, networkIntegration: c} cmd.AddCommand(networkIntegrationRenameCmd.command()) // Set networkIntegrationSetCmd := cmdNetworkIntegrationSet{global: c.global, networkIntegration: c} cmd.AddCommand(networkIntegrationSetCmd.command()) // Unset networkIntegrationUnsetCmd := cmdNetworkIntegrationUnset{global: c.global, networkIntegration: c, networkIntegrationSet: &networkIntegrationSetCmd} cmd.AddCommand(networkIntegrationUnsetCmd.command()) // Show networkIntegrationShowCmd := cmdNetworkIntegrationShow{global: c.global, networkIntegration: c} cmd.AddCommand(networkIntegrationShowCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Create. type cmdNetworkIntegrationCreate struct { global *cmdGlobal networkIntegration *cmdNetworkIntegration flagConfig []string } var cmdNetworkIntegrationCreateUsage = u.Usage{u.NewName(u.NetworkIntegration).Remote(), u.Type} func (c *cmdNetworkIntegrationCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdNetworkIntegrationCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create network integrations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Create network integrations`)) cmd.Example = cli.FormatSection("", i18n.G(`incus network integration create o1 ovn Create network integration o1 of type ovn incus network integration create o1 ovn < config.yaml Create network integration o1 of type ovn with configuration from config.yaml`)) cli.AddStringArrayFlag(cmd.Flags(), &c.flagConfig, "config|c", i18n.G("Config key/value to apply to the new network integration")) cmd.RunE = c.run return cmd } func (c *cmdNetworkIntegrationCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkIntegrationCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer integrationName := parsed[0].RemoteObject.String integrationType := parsed[1].String var stdinData api.NetworkIntegrationPut // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } err = loader.Load(&stdinData) if err != nil && !errors.Is(err, io.EOF) { return err } } // Create the network integration networkIntegration := api.NetworkIntegrationsPost{} networkIntegration.Name = integrationName networkIntegration.Type = integrationType networkIntegration.Description = stdinData.Description if stdinData.Config == nil { networkIntegration.Config = map[string]string{} for _, entry := range c.flagConfig { key, value, found := strings.Cut(entry, "=") if !found { return fmt.Errorf(i18n.G("Bad key=value pair: %q"), entry) } networkIntegration.Config[key] = value } } else { networkIntegration.Config = stdinData.Config } err = d.CreateNetworkIntegration(networkIntegration) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network integration %s created")+"\n", formatRemote(c.global.conf, parsed[0])) } return nil } // Delete. type cmdNetworkIntegrationDelete struct { global *cmdGlobal networkIntegration *cmdNetworkIntegration } var cmdNetworkIntegrationDeleteUsage = u.Usage{u.NetworkIntegration.Remote().List(1)} func (c *cmdNetworkIntegrationDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdNetworkIntegrationDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete network integrations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete network integrations`)) cmd.RunE = c.run return cmd } func (c *cmdNetworkIntegrationDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkIntegrationDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } var errs []error for _, p := range parsed[0].List { d := p.RemoteServer integrationName := p.RemoteObject.String // Delete the network integration err = d.DeleteNetworkIntegration(integrationName) if err != nil { errs = append(errs, err) continue } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network integration %s deleted")+"\n", formatRemote(c.global.conf, p)) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } // Edit. type cmdNetworkIntegrationEdit struct { global *cmdGlobal networkIntegration *cmdNetworkIntegration } var cmdNetworkIntegrationEditUsage = u.Usage{u.NetworkIntegration.Remote()} func (c *cmdNetworkIntegrationEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdNetworkIntegrationEditUsage...) cmd.Short = i18n.G("Edit network integration configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Edit network integration configurations as YAML`)) cmd.Example = cli.FormatSection("", i18n.G( `incus network integration edit < network-integration.yaml Update a network integration using the content of network-integration.yaml`)) cmd.RunE = c.run return cmd } func (c *cmdNetworkIntegrationEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the network integration. ### Any line starting with a '# will be ignored. ### ### Note that the name is shown but cannot be changed`) } func (c *cmdNetworkIntegrationEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkIntegrationEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer integrationName := parsed[0].RemoteObject.String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } newdata := api.NetworkIntegrationPut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateNetworkIntegration(integrationName, newdata, "") } // Extract the current value networkIntegration, etag, err := d.GetNetworkIntegration(integrationName) if err != nil { return err } data, err := yaml.Dump(&networkIntegration, yaml.V2) if err != nil { return err } // Spawn the editor content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor newdata := api.NetworkIntegrationPut{} err = yaml.Load(content, &newdata) if err == nil { err = d.UpdateNetworkIntegration(integrationName, newdata, etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Get. type cmdNetworkIntegrationGet struct { global *cmdGlobal networkIntegration *cmdNetworkIntegration flagIsProperty bool } type networkIntegrationColumn struct { Name string Data func(api.NetworkIntegration) string } var cmdNetworkIntegrationGetUsage = u.Usage{u.NetworkIntegration.Remote(), u.Key} func (c *cmdNetworkIntegrationGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdNetworkIntegrationGetUsage...) cmd.Short = i18n.G("Get values for network integration configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Get values for network integration configuration keys`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a network integration property")) return cmd } func (c *cmdNetworkIntegrationGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkIntegrationGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer integrationName := parsed[0].RemoteObject.String key := parsed[1].String // Get the configuration key networkIntegration, _, err := d.GetNetworkIntegration(integrationName) if err != nil { return err } if c.flagIsProperty { w := networkIntegration.Writable() res, err := getFieldByJSONTag(&w, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the network integration %q: %v"), key, formatRemote(c.global.conf, parsed[0]), err) } fmt.Printf("%v\n", res) } else { fmt.Printf("%s\n", networkIntegration.Config[key]) } return nil } // List. type cmdNetworkIntegrationList struct { global *cmdGlobal networkIntegration *cmdNetworkIntegration flagFormat string flagColumns string } var cmdNetworkIntegrationListUsage = u.Usage{u.RemoteColonOpt} func (c *cmdNetworkIntegrationList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdNetworkIntegrationListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List network integrations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List network integrations Default column layout: ndtu == Columns == The -c option takes a comma separated list of arguments that control which network integrations attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: n - Name d - Description t - Type u - Used by`)) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultNetworkIntegrationColumns, "", i18n.G("Columns")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run return cmd } const defaultNetworkIntegrationColumns = "ndtu" func (c *cmdNetworkIntegrationList) parseColumns() ([]networkIntegrationColumn, error) { columnsShorthandMap := map[rune]networkIntegrationColumn{ 'n': {i18n.G("NAME"), c.nameColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 't': {i18n.G("TYPE"), c.typeColumnData}, 'u': {i18n.G("USED BY"), c.usedByColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []networkIntegrationColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdNetworkIntegrationList) nameColumnData(integration api.NetworkIntegration) string { return integration.Name } func (c *cmdNetworkIntegrationList) descriptionColumnData(integration api.NetworkIntegration) string { return integration.Description } func (c *cmdNetworkIntegrationList) typeColumnData(integration api.NetworkIntegration) string { return integration.Type } func (c *cmdNetworkIntegrationList) usedByColumnData(integration api.NetworkIntegration) string { return fmt.Sprintf("%d", len(integration.UsedBy)) } func (c *cmdNetworkIntegrationList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkIntegrationListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer // List network integrations networkIntegrations, err := d.GetNetworkIntegrations() if err != nil { return err } // Parse column flags. columns, err := c.parseColumns() if err != nil { return err } data := [][]string{} for _, networkIntegration := range networkIntegrations { line := []string{} for _, column := range columns { line = append(line, column.Data(networkIntegration)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, networkIntegrations) } // Rename. type cmdNetworkIntegrationRename struct { global *cmdGlobal networkIntegration *cmdNetworkIntegration } var cmdNetworkIntegrationRenameUsage = u.Usage{u.NetworkIntegration.Remote(), u.NewName(u.NetworkIntegration)} func (c *cmdNetworkIntegrationRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdNetworkIntegrationRenameUsage...) cmd.Aliases = []string{"mv"} cmd.Short = i18n.G("Rename network integrations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Rename network integrations`)) cmd.RunE = c.run return cmd } func (c *cmdNetworkIntegrationRename) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkIntegrationRenameUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer integrationName := parsed[0].RemoteObject.String newIntegrationName := parsed[1].String // Rename the network integration err = d.RenameNetworkIntegration(integrationName, api.NetworkIntegrationPost{Name: newIntegrationName}) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network integration %s renamed to %s")+"\n", formatRemote(c.global.conf, parsed[0]), newIntegrationName) } return nil } // Set. type cmdNetworkIntegrationSet struct { global *cmdGlobal networkIntegration *cmdNetworkIntegration flagIsProperty bool } var cmdNetworkIntegrationSetUsage = u.Usage{u.NetworkIntegration.Remote(), u.LegacyKV.List(1)} func (c *cmdNetworkIntegrationSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdNetworkIntegrationSetUsage...) cmd.Short = i18n.G("Set network integration configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set network integration configuration keys For backward compatibility, a single configuration key may still be set with: incus network integration set [:] `)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a network integration property")) return cmd } // set runs the post-parsing command logic. func (c *cmdNetworkIntegrationSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer integrationName := parsed[0].RemoteObject.String keys, err := kvToMap(parsed[1]) if err != nil { return err } // Get the network integration networkIntegration, etag, err := d.GetNetworkIntegration(integrationName) if err != nil { return err } writable := networkIntegration.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { maps.Copy(writable.Config, keys) } return d.UpdateNetworkIntegration(integrationName, writable, etag) } func (c *cmdNetworkIntegrationSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkIntegrationSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Unset. type cmdNetworkIntegrationUnset struct { global *cmdGlobal networkIntegration *cmdNetworkIntegration networkIntegrationSet *cmdNetworkIntegrationSet flagIsProperty bool } var cmdNetworkIntegrationUnsetUsage = u.Usage{u.NetworkIntegration.Remote(), u.Key} func (c *cmdNetworkIntegrationUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdNetworkIntegrationUnsetUsage...) cmd.Short = i18n.G("Unset network integration configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Unset network integration configuration keys`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a network integration property")) return cmd } func (c *cmdNetworkIntegrationUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkIntegrationUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.networkIntegrationSet.flagIsProperty = c.flagIsProperty return unsetKey(c.networkIntegrationSet, cmd, parsed) } // Show. type cmdNetworkIntegrationShow struct { global *cmdGlobal networkIntegration *cmdNetworkIntegration } var cmdNetworkIntegrationShowUsage = u.Usage{u.NetworkIntegration.Remote()} func (c *cmdNetworkIntegrationShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdNetworkIntegrationShowUsage...) cmd.Short = i18n.G("Show network integration options") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show network integration options`)) cmd.RunE = c.run return cmd } func (c *cmdNetworkIntegrationShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkIntegrationShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer integrationName := parsed[0].RemoteObject.String // Show the network integration networkIntegration, _, err := d.GetNetworkIntegration(integrationName) if err != nil { return err } data, err := yaml.Dump(&networkIntegration, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } incus-7.0.0/cmd/incus/network_load_balancer.go000066400000000000000000001053411517523235500214050ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "maps" "os" "sort" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" ) type cmdNetworkLoadBalancer struct { global *cmdGlobal flagTarget string } func (c *cmdNetworkLoadBalancer) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("load-balancer") cmd.Short = i18n.G("Manage network load balancers") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Manage network load balancers")) // List. networkLoadBalancerListCmd := cmdNetworkLoadBalancerList{global: c.global, networkLoadBalancer: c} cmd.AddCommand(networkLoadBalancerListCmd.command()) // Show. networkLoadBalancerShowCmd := cmdNetworkLoadBalancerShow{global: c.global, networkLoadBalancer: c} cmd.AddCommand(networkLoadBalancerShowCmd.command()) // Create. networkLoadBalancerCreateCmd := cmdNetworkLoadBalancerCreate{global: c.global, networkLoadBalancer: c} cmd.AddCommand(networkLoadBalancerCreateCmd.command()) // Get. networkLoadBalancerGetCmd := cmdNetworkLoadBalancerGet{global: c.global, networkLoadBalancer: c} cmd.AddCommand(networkLoadBalancerGetCmd.command()) // Info. networkLoadBalancerInfoCmd := cmdNetworkLoadBalancerInfo{global: c.global, networkLoadBalancer: c} cmd.AddCommand(networkLoadBalancerInfoCmd.command()) // Set. networkLoadBalancerSetCmd := cmdNetworkLoadBalancerSet{global: c.global, networkLoadBalancer: c} cmd.AddCommand(networkLoadBalancerSetCmd.command()) // Unset. networkLoadBalancerUnsetCmd := cmdNetworkLoadBalancerUnset{global: c.global, networkLoadBalancer: c, networkLoadBalancerSet: &networkLoadBalancerSetCmd} cmd.AddCommand(networkLoadBalancerUnsetCmd.command()) // Edit. networkLoadBalancerEditCmd := cmdNetworkLoadBalancerEdit{global: c.global, networkLoadBalancer: c} cmd.AddCommand(networkLoadBalancerEditCmd.command()) // Delete. networkLoadBalancerDeleteCmd := cmdNetworkLoadBalancerDelete{global: c.global, networkLoadBalancer: c} cmd.AddCommand(networkLoadBalancerDeleteCmd.command()) // Backend. networkLoadBalancerBackendCmd := cmdNetworkLoadBalancerBackend{global: c.global, networkLoadBalancer: c} cmd.AddCommand(networkLoadBalancerBackendCmd.command()) // Port. networkLoadBalancerPortCmd := cmdNetworkLoadBalancerPort{global: c.global, networkLoadBalancer: c} cmd.AddCommand(networkLoadBalancerPortCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // List. type cmdNetworkLoadBalancerList struct { global *cmdGlobal networkLoadBalancer *cmdNetworkLoadBalancer flagFormat string flagColumns string } type networkLoadBalancerColumn struct { Name string Data func(api.NetworkLoadBalancer) string } var cmdNetworkLoadBalancerListUsage = u.Usage{u.Network.Remote()} func (c *cmdNetworkLoadBalancerList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdNetworkLoadBalancerListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List available network load balancers") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List available network load balancers Default column layout: ldp == Columns == The -c option takes a comma separated list of arguments that control which network load balancer attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: l - Listen Address d - Description p - Ports L - Location of the operation (e.g. its cluster member)`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultNetworkLoadBalancerColumns, "", i18n.G("Columns")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } const defaultNetworkLoadBalancerColumns = "ldp" func (c *cmdNetworkLoadBalancerList) parseColumns(clustered bool) ([]networkLoadBalancerColumn, error) { columnsShorthandMap := map[rune]networkLoadBalancerColumn{ 'l': {i18n.G("LISTEN ADDRESS"), c.listenAddressColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 'p': {i18n.G("PORTS"), c.portsColumnData}, 'L': {i18n.G("LOCATION"), c.locationColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []networkLoadBalancerColumn{} if c.flagColumns == defaultNetworkLoadBalancerColumns && clustered { columnList = append(columnList, "L") } for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdNetworkLoadBalancerList) listenAddressColumnData(loadBalancer api.NetworkLoadBalancer) string { return loadBalancer.ListenAddress } func (c *cmdNetworkLoadBalancerList) descriptionColumnData(loadBalancer api.NetworkLoadBalancer) string { return loadBalancer.Description } func (c *cmdNetworkLoadBalancerList) portsColumnData(loadBalancer api.NetworkLoadBalancer) string { return fmt.Sprintf("%d", len(loadBalancer.Ports)) } func (c *cmdNetworkLoadBalancerList) locationColumnData(loadBalancer api.NetworkLoadBalancer) string { return loadBalancer.Location } func (c *cmdNetworkLoadBalancerList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkLoadBalancerListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String loadBalancers, err := d.GetNetworkLoadBalancers(networkName) if err != nil { return err } // Parse column flags. columns, err := c.parseColumns(d.IsClustered()) if err != nil { return err } // Render the table data := [][]string{} for _, loadBalancer := range loadBalancers { line := []string{} for _, column := range columns { line = append(line, column.Data(loadBalancer)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, loadBalancers) } // Show. type cmdNetworkLoadBalancerShow struct { global *cmdGlobal networkLoadBalancer *cmdNetworkLoadBalancer } var cmdNetworkLoadBalancerShowUsage = u.Usage{u.Network.Remote(), u.ListenAddress} func (c *cmdNetworkLoadBalancerShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdNetworkLoadBalancerShowUsage...) cmd.Short = i18n.G("Show network load balancer configurations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Show network load balancer configurations")) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.networkLoadBalancer.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkLoadBalancers(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkLoadBalancerShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkLoadBalancerShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String // If a target was specified, use the load balancer on the given member. if c.networkLoadBalancer.flagTarget != "" { d = d.UseTarget(c.networkLoadBalancer.flagTarget) } // Show the network load balancer config. loadBalancer, _, err := d.GetNetworkLoadBalancer(networkName, listenAddress) if err != nil { return err } data, err := yaml.Dump(&loadBalancer, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Create. type cmdNetworkLoadBalancerCreate struct { global *cmdGlobal networkLoadBalancer *cmdNetworkLoadBalancer flagDescription string } var cmdNetworkLoadBalancerCreateUsage = u.Usage{u.Network.Remote(), u.ListenAddress, u.KV.List(0)} func (c *cmdNetworkLoadBalancerCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdNetworkLoadBalancerCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create new network load balancers") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Create new network load balancers")) cmd.Example = cli.FormatSection("", i18n.G(`incus network load-balancer create n1 127.0.0.1 Create network load-balancer for network n1 incus network load-balancer create n1 127.0.0.1 < config.yaml Create network load-balancer for network n1 with configuration from config.yaml`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.networkLoadBalancer.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Load balancer description")) return cmd } func (c *cmdNetworkLoadBalancerCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkLoadBalancerCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String keys, err := kvToMap(parsed[2]) if err != nil { return err } // If stdin isn't a terminal, read yaml from it. var loadBalancerPut api.NetworkLoadBalancerPut if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } err = loader.Load(&loadBalancerPut) if err != nil && !errors.Is(err, io.EOF) { return err } } if loadBalancerPut.Config == nil { loadBalancerPut.Config = map[string]string{} } maps.Copy(loadBalancerPut.Config, keys) // Create the network load balancer. loadBalancer := api.NetworkLoadBalancersPost{ ListenAddress: listenAddress, NetworkLoadBalancerPut: loadBalancerPut, } if c.flagDescription != "" { loadBalancer.Description = c.flagDescription } loadBalancer.Normalise() // If a target was specified, create the load balancer on the given member. if c.networkLoadBalancer.flagTarget != "" { d = d.UseTarget(c.networkLoadBalancer.flagTarget) } err = d.CreateNetworkLoadBalancer(networkName, loadBalancer) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network load balancer %s created")+"\n", loadBalancer.ListenAddress) } return nil } // Get. type cmdNetworkLoadBalancerGet struct { global *cmdGlobal networkLoadBalancer *cmdNetworkLoadBalancer flagIsProperty bool } var cmdNetworkLoadBalancerGetUsage = u.Usage{u.Network.Remote(), u.ListenAddress, u.Key} func (c *cmdNetworkLoadBalancerGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdNetworkLoadBalancerGetUsage...) cmd.Short = i18n.G("Get values for network load balancer configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Get values for network load balancer configuration keys")) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a network load balancer property")) return cmd } func (c *cmdNetworkLoadBalancerGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkLoadBalancerGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String key := parsed[2].String // Get the current config. loadBalancer, _, err := d.GetNetworkLoadBalancer(networkName, listenAddress) if err != nil { return err } if c.flagIsProperty { w := loadBalancer.Writable() res, err := getFieldByJSONTag(&w, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the load balancer %q: %v"), key, listenAddress, err) } fmt.Printf("%v\n", res) } else { for k, v := range loadBalancer.Config { if k == key { fmt.Printf("%s\n", v) } } } return nil } // Set. type cmdNetworkLoadBalancerSet struct { global *cmdGlobal networkLoadBalancer *cmdNetworkLoadBalancer flagIsProperty bool } var cmdNetworkLoadBalancerSetUsage = u.Usage{u.Network.Remote(), u.ListenAddress, u.LegacyKV.List(1)} func (c *cmdNetworkLoadBalancerSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdNetworkLoadBalancerSetUsage...) cmd.Short = i18n.G("Set network load balancer keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set network load balancer keys For backward compatibility, a single configuration key may still be set with: incus network set [:] `)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a network load balancer property")) cli.AddStringFlag(cmd.Flags(), &c.networkLoadBalancer.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkLoadBalancers(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdNetworkLoadBalancerSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String keys, err := kvToMap(parsed[2]) if err != nil { return err } // If a target was specified, use the load balancer on the given member. if c.networkLoadBalancer.flagTarget != "" { d = d.UseTarget(c.networkLoadBalancer.flagTarget) } // Get the current config. loadBalancer, etag, err := d.GetNetworkLoadBalancer(networkName, listenAddress) if err != nil { return err } if loadBalancer.Config == nil { loadBalancer.Config = map[string]string{} } writable := loadBalancer.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { maps.Copy(writable.Config, keys) } writable.Normalise() return d.UpdateNetworkLoadBalancer(networkName, loadBalancer.ListenAddress, writable, etag) } func (c *cmdNetworkLoadBalancerSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkLoadBalancerSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Unset. type cmdNetworkLoadBalancerUnset struct { global *cmdGlobal networkLoadBalancer *cmdNetworkLoadBalancer networkLoadBalancerSet *cmdNetworkLoadBalancerSet flagIsProperty bool } var cmdNetworkLoadBalancerUnsetUsage = u.Usage{u.Network.Remote(), u.ListenAddress, u.Key} func (c *cmdNetworkLoadBalancerUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdNetworkLoadBalancerUnsetUsage...) cmd.Short = i18n.G("Unset network load balancer configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Unset network load balancer keys")) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a network load balancer property")) return cmd } func (c *cmdNetworkLoadBalancerUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkLoadBalancerUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.networkLoadBalancerSet.flagIsProperty = c.flagIsProperty return unsetKey(c.networkLoadBalancerSet, cmd, parsed) } // Edit. type cmdNetworkLoadBalancerEdit struct { global *cmdGlobal networkLoadBalancer *cmdNetworkLoadBalancer } var cmdNetworkLoadBalancerEditUsage = u.Usage{u.Network.Remote(), u.ListenAddress} func (c *cmdNetworkLoadBalancerEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdNetworkLoadBalancerEditUsage...) cmd.Short = i18n.G("Edit network load balancer configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Edit network load balancer configurations as YAML")) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.networkLoadBalancer.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkLoadBalancers(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkLoadBalancerEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the network load balancer. ### Any line starting with a '# will be ignored. ### ### A network load balancer consists of a set of target backends and port forwards for a listen address. ### ### An example would look like: ### listen_address: 192.0.2.1 ### config: ### user.foo: bar ### description: test desc ### backends: ### - name: backend1 ### description: First backend server ### target_address: 192.0.3.1 ### target_port: 80 ### - name: backend2 ### description: Second backend server ### target_address: 192.0.3.2 ### target_port: 80 ### ports: ### - description: port forward ### protocol: tcp ### listen_port: 80,81,8080-8090 ### target_backend: ### - backend1 ### - backend2 ### location: server01 ### ### Note that the listen_address and location cannot be changed.`) } func (c *cmdNetworkLoadBalancerEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkLoadBalancerEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String // If a target was specified, use the load balancer on the given member. if c.networkLoadBalancer.flagTarget != "" { d = d.UseTarget(c.networkLoadBalancer.flagTarget) } // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } // Allow output of `incus network load-balancer show` command to be passed in here, but only take the // contents of the NetworkLoadBalancerPut fields when updating. // The other fields are silently discarded. newData := api.NetworkLoadBalancer{} err = loader.Load(&newData) if err != nil && !errors.Is(err, io.EOF) { return err } newData.Normalise() return d.UpdateNetworkLoadBalancer(networkName, listenAddress, newData.NetworkLoadBalancerPut, "") } // Get the current config. loadBalancer, etag, err := d.GetNetworkLoadBalancer(networkName, listenAddress) if err != nil { return err } data, err := yaml.Dump(&loadBalancer, yaml.V2) if err != nil { return err } // Spawn the editor. content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor. newData := api.NetworkLoadBalancer{} // We show the full info, but only send the writable fields. err = yaml.Load(content, &newData, yaml.WithKnownFields()) if err == nil { newData.Normalise() err = d.UpdateNetworkLoadBalancer(networkName, listenAddress, newData.Writable(), etag) } // Respawn the editor. if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Delete. type cmdNetworkLoadBalancerDelete struct { global *cmdGlobal networkLoadBalancer *cmdNetworkLoadBalancer } var cmdNetworkLoadBalancerDeleteUsage = u.Usage{u.Network.Remote(), u.ListenAddress} func (c *cmdNetworkLoadBalancerDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdNetworkLoadBalancerDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete network load balancers") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Delete network load balancers")) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.networkLoadBalancer.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkLoadBalancers(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkLoadBalancerDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkLoadBalancerDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String // If a target was specified, use the load balancer on the given member. if c.networkLoadBalancer.flagTarget != "" { d = d.UseTarget(c.networkLoadBalancer.flagTarget) } // Delete the network load balancer. err = d.DeleteNetworkLoadBalancer(networkName, listenAddress) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network load balancer %s deleted")+"\n", listenAddress) } return nil } // Add/Remove Backend. type cmdNetworkLoadBalancerBackend struct { global *cmdGlobal networkLoadBalancer *cmdNetworkLoadBalancer flagDescription string } func (c *cmdNetworkLoadBalancerBackend) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("backend") cmd.Short = i18n.G("Manage network load balancer backends") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Manage network load balancer backends")) // Backend Add. cmd.AddCommand(c.commandAdd()) // Backend Remove. cmd.AddCommand(c.commandRemove()) return cmd } var cmdNetworkLoadBalancerBackendAddUsage = u.Usage{u.Network.Remote(), u.ListenAddress, u.NewName(u.Backend), u.Target(u.Address), u.Target(u.Port).List(0, ",")} func (c *cmdNetworkLoadBalancerBackend) commandAdd() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("add", cmdNetworkLoadBalancerBackendAddUsage...) cmd.Aliases = []string{"create"} cmd.Short = i18n.G("Add backends to a load balancer") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Add backend to a load balancer")) cmd.RunE = c.runAdd cli.AddStringFlag(cmd.Flags(), &c.networkLoadBalancer.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Backend description")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkLoadBalancers(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkLoadBalancerBackend) runAdd(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkLoadBalancerBackendAddUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String backendName := parsed[2].String targetAddress := parsed[3].String // Only the list’s string representation is used. targetPorts := parsed[4].String // If a target was specified, use the load balancer on the given member. if c.networkLoadBalancer.flagTarget != "" { d = d.UseTarget(c.networkLoadBalancer.flagTarget) } // Get the network load balancer. loadBalancer, etag, err := d.GetNetworkLoadBalancer(networkName, listenAddress) if err != nil { return err } loadBalancer.Backends = append(loadBalancer.Backends, api.NetworkLoadBalancerBackend{ Name: backendName, TargetAddress: targetAddress, TargetPort: targetPorts, Description: c.flagDescription, }) loadBalancer.Normalise() return d.UpdateNetworkLoadBalancer(networkName, loadBalancer.ListenAddress, loadBalancer.Writable(), etag) } var cmdNetworkLoadBalancerBackendRemoveUsage = u.Usage{u.Network.Remote(), u.ListenAddress, u.Backend} func (c *cmdNetworkLoadBalancerBackend) commandRemove() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remove", cmdNetworkLoadBalancerBackendRemoveUsage...) cmd.Aliases = []string{"delete", "rm"} cmd.Short = i18n.G("Remove backends from a load balancer") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Remove backend from a load balancer")) cmd.RunE = c.runRemove cli.AddStringFlag(cmd.Flags(), &c.networkLoadBalancer.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkLoadBalancers(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkLoadBalancerBackend) runRemove(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkLoadBalancerBackendRemoveUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String backendName := parsed[2].String // If a target was specified, use the load balancer on the given member. if c.networkLoadBalancer.flagTarget != "" { d = d.UseTarget(c.networkLoadBalancer.flagTarget) } // Get the network load balancer. loadBalancer, etag, err := d.GetNetworkLoadBalancer(networkName, listenAddress) if err != nil { return err } removed := false newBackends := make([]api.NetworkLoadBalancerBackend, 0, len(loadBalancer.Backends)) for _, backend := range loadBalancer.Backends { if backend.Name == backendName { removed = true continue // Don't add removed backend to newBackends. } newBackends = append(newBackends, backend) } if !removed { return errors.New(i18n.G("No matching backend found")) } loadBalancer.Backends = newBackends loadBalancer.Normalise() return d.UpdateNetworkLoadBalancer(networkName, loadBalancer.ListenAddress, loadBalancer.Writable(), etag) } // Add/Remove Port. type cmdNetworkLoadBalancerPort struct { global *cmdGlobal networkLoadBalancer *cmdNetworkLoadBalancer flagRemoveForce bool flagDescription string } func (c *cmdNetworkLoadBalancerPort) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("port") cmd.Short = i18n.G("Manage network load balancer ports") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Manage network load balancer ports")) // Port Add. cmd.AddCommand(c.commandAdd()) // Port Remove. cmd.AddCommand(c.commandRemove()) return cmd } var cmdNetworkLoadBalancerPortAddUsage = u.Usage{u.Network.Remote(), u.ListenAddress, u.Protocol, u.ListenPort.List(1, ","), u.Backend.List(1, ",")} func (c *cmdNetworkLoadBalancerPort) commandAdd() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("add", cmdNetworkLoadBalancerPortAddUsage...) cmd.Aliases = []string{"create"} cmd.Short = i18n.G("Add ports to a load balancer") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Add ports to a load balancer")) cmd.RunE = c.runAdd cli.AddStringFlag(cmd.Flags(), &c.networkLoadBalancer.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Port description")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkLoadBalancers(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkLoadBalancerPort) runAdd(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkLoadBalancerPortAddUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String protocol := parsed[2].String // Only the list’s string representation is used. listenPorts := parsed[3].String backends := parsed[4].StringList // If a target was specified, use the load balancer on the given member. if c.networkLoadBalancer.flagTarget != "" { d = d.UseTarget(c.networkLoadBalancer.flagTarget) } // Get the network load balancer. loadBalancer, etag, err := d.GetNetworkLoadBalancer(networkName, listenAddress) if err != nil { return err } loadBalancer.Ports = append(loadBalancer.Ports, api.NetworkLoadBalancerPort{ Protocol: protocol, ListenPort: listenPorts, TargetBackend: backends, Description: c.flagDescription, }) loadBalancer.Normalise() return d.UpdateNetworkLoadBalancer(networkName, loadBalancer.ListenAddress, loadBalancer.Writable(), etag) } var cmdNetworkLoadBalancerPortRemoveUsage = u.Usage{u.Network.Remote(), u.ListenAddress, u.Protocol.Optional(u.ListenPort.List(0, ","))} func (c *cmdNetworkLoadBalancerPort) commandRemove() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remove", cmdNetworkLoadBalancerPortRemoveUsage...) cmd.Aliases = []string{"delete", "rm"} cmd.Short = i18n.G("Remove ports from a load balancer") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Remove ports from a load balancer")) cli.AddBoolFlag(cmd.Flags(), &c.flagRemoveForce, "force|f", i18n.G("Remove all ports that match")) cmd.RunE = c.runRemove cli.AddStringFlag(cmd.Flags(), &c.networkLoadBalancer.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkLoadBalancers(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkLoadBalancerPort) runRemove(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkLoadBalancerPortRemoveUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String hasProtocol := !parsed[2].Skipped protocol := "" hasListenPorts := false listenPorts := "" if hasProtocol { protocol = parsed[2].List[0].String hasListenPorts = !parsed[2].List[1].Skipped // Only the list’s string representation is used. listenPorts = parsed[2].List[1].String } // If a target was specified, use the load balancer on the given member. if c.networkLoadBalancer.flagTarget != "" { d = d.UseTarget(c.networkLoadBalancer.flagTarget) } // Get the network load balancer. loadBalancer, etag, err := d.GetNetworkLoadBalancer(networkName, listenAddress) if err != nil { return err } removed := false newPorts := make([]api.NetworkLoadBalancerPort, 0, len(loadBalancer.Ports)) for _, port := range loadBalancer.Ports { if hasProtocol && port.Protocol != protocol || hasListenPorts && port.ListenPort != listenPorts { newPorts = append(newPorts, port) } else { if removed && !c.flagRemoveForce { return errors.New(i18n.G("Multiple ports match. Use --force to remove them all")) } removed = true } } if !removed { return errors.New(i18n.G("No matching port(s) found")) } loadBalancer.Ports = newPorts loadBalancer.Normalise() return d.UpdateNetworkLoadBalancer(networkName, loadBalancer.ListenAddress, loadBalancer.Writable(), etag) } // Info. type cmdNetworkLoadBalancerInfo struct { global *cmdGlobal networkLoadBalancer *cmdNetworkLoadBalancer } var cmdNetworkLoadBalancerInfoUsage = u.Usage{u.Network.Remote(), u.ListenAddress} func (c *cmdNetworkLoadBalancerInfo) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("info", cmdNetworkLoadBalancerInfoUsage...) cmd.Short = i18n.G("Get current load balancer status") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Get current load-balancer status")) cmd.RunE = c.run return cmd } func (c *cmdNetworkLoadBalancerInfo) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkLoadBalancerInfoUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String listenAddress := parsed[1].String // Get the load-balancer state. lbState, err := d.GetNetworkLoadBalancerState(networkName, listenAddress) if err != nil { return err } // Render the state. if lbState.BackendHealth == nil { // Currently the only field in the state endpoint is the backend health, fail if it's missing. return errors.New(i18n.G("No load-balancer health information available")) } fmt.Println(i18n.G("Backend health:")) for backend, info := range lbState.BackendHealth { if len(info.Ports) == 0 { continue } fmt.Printf(" %s (%s):\n", backend, info.Address) for _, port := range info.Ports { fmt.Printf(" - %s/%d: %s\n", port.Protocol, port.Port, port.Status) } fmt.Println("") } return nil } incus-7.0.0/cmd/incus/network_peer.go000066400000000000000000000521421517523235500175720ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "maps" "os" "slices" "sort" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" ) type cmdNetworkPeer struct { global *cmdGlobal } func (c *cmdNetworkPeer) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("peer") cmd.Short = i18n.G("Manage network peerings") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Manage network peerings")) // List. networkPeerListCmd := cmdNetworkPeerList{global: c.global, networkPeer: c} cmd.AddCommand(networkPeerListCmd.command()) // Show. networkPeerShowCmd := cmdNetworkPeerShow{global: c.global, networkPeer: c} cmd.AddCommand(networkPeerShowCmd.command()) // Create. networkPeerCreateCmd := cmdNetworkPeerCreate{global: c.global, networkPeer: c} cmd.AddCommand(networkPeerCreateCmd.command()) // Get, networkPeerGetCmd := cmdNetworkPeerGet{global: c.global, networkPeer: c} cmd.AddCommand(networkPeerGetCmd.command()) // Set. networkPeerSetCmd := cmdNetworkPeerSet{global: c.global, networkPeer: c} cmd.AddCommand(networkPeerSetCmd.command()) // Unset. networkPeerUnsetCmd := cmdNetworkPeerUnset{global: c.global, networkPeer: c, networkPeerSet: &networkPeerSetCmd} cmd.AddCommand(networkPeerUnsetCmd.command()) // Edit. networkPeerEditCmd := cmdNetworkPeerEdit{global: c.global, networkPeer: c} cmd.AddCommand(networkPeerEditCmd.command()) // Delete. networkPeerDeleteCmd := cmdNetworkPeerDelete{global: c.global, networkPeer: c} cmd.AddCommand(networkPeerDeleteCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // List. type cmdNetworkPeerList struct { global *cmdGlobal networkPeer *cmdNetworkPeer flagFormat string flagColumns string } type networkPeerColumn struct { Name string Data func(api.NetworkPeer) string } var cmdNetworkPeerListUsage = u.Usage{u.Network.Remote()} func (c *cmdNetworkPeerList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdNetworkPeerListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List available network peers") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List available network peers Default column layout: ndpts == Columns == The -c option takes a comma separated list of arguments that control which network peer attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: n - Name d - description p - Peer t - Type s - State`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultNetworkPeerListColumns, "", i18n.G("Columns")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } const defaultNetworkPeerListColumns = "ndpts" func (c *cmdNetworkPeerList) parseColumns() ([]networkPeerColumn, error) { columnsShorthandMap := map[rune]networkPeerColumn{ 'n': {i18n.G("NAME"), c.nameColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 'p': {i18n.G("PEER"), c.peerColumnData}, 't': {i18n.G("TYPE"), c.typeColumnData}, 's': {i18n.G("STATE"), c.stateColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []networkPeerColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdNetworkPeerList) nameColumnData(peer api.NetworkPeer) string { return peer.Name } func (c *cmdNetworkPeerList) descriptionColumnData(peer api.NetworkPeer) string { return peer.Description } func (c *cmdNetworkPeerList) peerColumnData(peer api.NetworkPeer) string { target := "Unknown" if peer.TargetProject != "" && peer.TargetNetwork != "" { target = fmt.Sprintf("%s/%s", peer.TargetProject, peer.TargetNetwork) } else if peer.TargetIntegration != "" { target = peer.TargetIntegration } return target } func (c *cmdNetworkPeerList) typeColumnData(peer api.NetworkPeer) string { return peer.Type } func (c *cmdNetworkPeerList) stateColumnData(peer api.NetworkPeer) string { return strings.ToUpper(peer.Status) } func (c *cmdNetworkPeerList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkPeerListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String peers, err := d.GetNetworkPeers(networkName) if err != nil { return err } // Parse column flags. columns, err := c.parseColumns() if err != nil { return err } data := [][]string{} for _, peer := range peers { line := []string{} for _, column := range columns { line = append(line, column.Data(peer)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, peers) } // Show. type cmdNetworkPeerShow struct { global *cmdGlobal networkPeer *cmdNetworkPeer } var cmdNetworkPeerShowUsage = u.Usage{u.Network.Remote(), u.Peer} func (c *cmdNetworkPeerShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdNetworkPeerShowUsage...) cmd.Short = i18n.G("Show network peer configurations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Show network peer configurations")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkPeers(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkPeerShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkPeerShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String peerName := parsed[1].String // Show the network peer config. peer, _, err := d.GetNetworkPeer(networkName, peerName) if err != nil { return err } data, err := yaml.Dump(&peer, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Create. type cmdNetworkPeerCreate struct { global *cmdGlobal networkPeer *cmdNetworkPeer flagType string flagDescription string } var cmdNetworkPeerCreateUsage = u.Usage{u.Network.Remote(), u.NewName(u.Peer), u.MakePath(u.Target(u.Project).Optional(), u.Target(u.Placeholder(i18n.G("network or integration")))), u.KV.List(0)} func (c *cmdNetworkPeerCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdNetworkPeerCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create new network peering") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Create new network peering")) cmd.Example = cli.FormatSection("", i18n.G(`incus network peer create default peer1 web/default Create a new peering between network "default" in the current project and network "default" in the "web" project incus network peer create default peer2 ovn-ic --type=remote Create a new peering between network "default" in the current project and other remote networks through the "ovn-ic" integration incus network peer create default peer3 web/default < config.yaml Create a new peering between network default in the current project and network default in the web project using the configuration in the file config.yaml`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagType, "type|t", "local", "", i18n.G("Type of peer (local or remote)")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Peer description")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkPeerCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkPeerCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String peerName := parsed[1].String targetProject := parsed[2].List[0].String target := parsed[2].List[1].String keys, err := kvToMap(parsed[3]) if err != nil { return err } if !slices.Contains([]string{"local", "remote"}, c.flagType) { return errors.New(i18n.G("Invalid peer type")) } // If stdin isn't a terminal, read yaml from it. var peerPut api.NetworkPeerPut if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } err = loader.Load(&peerPut) if err != nil && !errors.Is(err, io.EOF) { return err } } if peerPut.Config == nil { peerPut.Config = map[string]string{} } maps.Copy(peerPut.Config, keys) // Create the network peer. peer := api.NetworkPeersPost{ Name: peerName, NetworkPeerPut: peerPut, Type: c.flagType, } switch c.flagType { case "local": peer.TargetProject = targetProject peer.TargetNetwork = target case "remote": peer.TargetIntegration = target } if c.flagDescription != "" { peer.Description = c.flagDescription } err = d.CreateNetworkPeer(networkName, peer) if err != nil { return err } if !c.global.flagQuiet { createdPeer, _, err := d.GetNetworkPeer(networkName, peer.Name) if err != nil { return fmt.Errorf(i18n.G("Failed getting peer's status: %w"), err) } switch createdPeer.Status { case api.NetworkStatusCreated: fmt.Printf(i18n.G("Network peer %s created")+"\n", peer.Name) case api.NetworkStatusPending: fmt.Printf(i18n.G("Network peer %s pending (please complete mutual peering on peer network)")+"\n", peer.Name) default: fmt.Printf(i18n.G("Network peer %s is in unexpected state %q")+"\n", peer.Name, createdPeer.Status) } } return nil } // Get. type cmdNetworkPeerGet struct { global *cmdGlobal networkPeer *cmdNetworkPeer flagIsProperty bool } var cmdNetworkPeerGetUsage = u.Usage{u.Network.Remote(), u.Peer, u.Key} func (c *cmdNetworkPeerGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdNetworkPeerGetUsage...) cmd.Short = i18n.G("Get values for network peer configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Get values for network peer configuration keys")) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a network peer property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkPeers(args[0]) } if len(args) == 2 { return c.global.cmpNetworkPeerConfigs(args[0], args[1]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkPeerGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkPeerGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String peerName := parsed[1].String key := parsed[1].String // Get the current config. peer, _, err := d.GetNetworkPeer(networkName, peerName) if err != nil { return err } if c.flagIsProperty { w := peer.Writable() res, err := getFieldByJSONTag(&w, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the network peer %q: %v"), key, peerName, err) } fmt.Printf("%v\n", res) } else { for k, v := range peer.Config { if k == key { fmt.Printf("%s\n", v) } } } return nil } // Set. type cmdNetworkPeerSet struct { global *cmdGlobal networkPeer *cmdNetworkPeer flagIsProperty bool } var cmdNetworkPeerSetUsage = u.Usage{u.Network.Remote(), u.Peer, u.LegacyKV.List(1)} func (c *cmdNetworkPeerSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdNetworkPeerSetUsage...) cmd.Short = i18n.G("Set network peer keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set network peer keys For backward compatibility, a single configuration key may still be set with: incus network set [:] `)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a network peer property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkPeers(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdNetworkPeerSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String peerName := parsed[1].String keys, err := kvToMap(parsed[2]) if err != nil { return err } // Get the current config. peer, etag, err := d.GetNetworkPeer(networkName, peerName) if err != nil { return err } if peer.Config == nil { peer.Config = map[string]string{} } writable := peer.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { maps.Copy(writable.Config, keys) } return d.UpdateNetworkPeer(networkName, peer.Name, writable, etag) } func (c *cmdNetworkPeerSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkPeerSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Unset. type cmdNetworkPeerUnset struct { global *cmdGlobal networkPeer *cmdNetworkPeer networkPeerSet *cmdNetworkPeerSet flagIsProperty bool } var cmdNetworkPeerUnsetUsage = u.Usage{u.Network.Remote(), u.Peer, u.Key} func (c *cmdNetworkPeerUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdNetworkPeerUnsetUsage...) cmd.Short = i18n.G("Unset network peer configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Unset network peer keys")) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a network peer property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkPeers(args[0]) } if len(args) == 2 { return c.global.cmpNetworkPeerConfigs(args[0], args[1]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkPeerUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkPeerUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.networkPeerSet.flagIsProperty = c.flagIsProperty return unsetKey(c.networkPeerSet, cmd, parsed) } // Edit. type cmdNetworkPeerEdit struct { global *cmdGlobal networkPeer *cmdNetworkPeer } var cmdNetworkPeerEditUsage = u.Usage{u.Network.Remote(), u.Peer} func (c *cmdNetworkPeerEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdNetworkPeerEditUsage...) cmd.Short = i18n.G("Edit network peer configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Edit network peer configurations as YAML")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkPeers(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkPeerEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the network peer. ### Any line starting with a '# will be ignored. ### ### An example would look like: ### description: A peering to mynet ### config: {} ### name: mypeer ### target_project: default ### target_network: mynet ### status: Pending ### ### Note that the name, target_project, target_network and status fields cannot be changed.`) } func (c *cmdNetworkPeerEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkPeerEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String peerName := parsed[1].String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } // Allow output of `incus network peer show` command to be passed in here, but only take the contents // of the NetworkPeerPut fields when updating. The other fields are silently discarded. newData := api.NetworkPeer{} err = loader.Load(&newData) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateNetworkPeer(networkName, peerName, newData.NetworkPeerPut, "") } // Get the current config. peer, etag, err := d.GetNetworkPeer(networkName, peerName) if err != nil { return err } data, err := yaml.Dump(&peer, yaml.V2) if err != nil { return err } // Spawn the editor. content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor. newData := api.NetworkPeer{} // We show the full info, but only send the writable fields. err = yaml.Load(content, &newData, yaml.WithKnownFields()) if err == nil { err = d.UpdateNetworkPeer(networkName, peerName, newData.Writable(), etag) } // Respawn the editor. if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Delete. type cmdNetworkPeerDelete struct { global *cmdGlobal networkPeer *cmdNetworkPeer } var cmdNetworkPeerDeleteUsage = u.Usage{u.Network.Remote(), u.Peer} func (c *cmdNetworkPeerDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdNetworkPeerDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete network peerings") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Delete network peerings")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworks(toComplete) } if len(args) == 1 { return c.global.cmpNetworkPeers(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkPeerDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkPeerDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer networkName := parsed[0].RemoteObject.String peerName := parsed[1].String // Delete the network peer. err = d.DeleteNetworkPeer(networkName, peerName) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network peer %s deleted")+"\n", peerName) } return nil } incus-7.0.0/cmd/incus/network_zone.go000066400000000000000000001216031517523235500176110ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "maps" "os" "slices" "sort" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" ) type cmdNetworkZone struct { global *cmdGlobal } type networkZoneColumn struct { Name string Data func(api.NetworkZone) string } func (c *cmdNetworkZone) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("zone") cmd.Short = i18n.G("Manage network zones") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Manage network zones")) // List. networkZoneListCmd := cmdNetworkZoneList{global: c.global, networkZone: c} cmd.AddCommand(networkZoneListCmd.command()) // Show. networkZoneShowCmd := cmdNetworkZoneShow{global: c.global, networkZone: c} cmd.AddCommand(networkZoneShowCmd.command()) // Get. networkZoneGetCmd := cmdNetworkZoneGet{global: c.global, networkZone: c} cmd.AddCommand(networkZoneGetCmd.command()) // Create. networkZoneCreateCmd := cmdNetworkZoneCreate{global: c.global, networkZone: c} cmd.AddCommand(networkZoneCreateCmd.command()) // Set. networkZoneSetCmd := cmdNetworkZoneSet{global: c.global, networkZone: c} cmd.AddCommand(networkZoneSetCmd.command()) // Unset. networkZoneUnsetCmd := cmdNetworkZoneUnset{global: c.global, networkZone: c, networkZoneSet: &networkZoneSetCmd} cmd.AddCommand(networkZoneUnsetCmd.command()) // Edit. networkZoneEditCmd := cmdNetworkZoneEdit{global: c.global, networkZone: c} cmd.AddCommand(networkZoneEditCmd.command()) // Delete. networkZoneDeleteCmd := cmdNetworkZoneDelete{global: c.global, networkZone: c} cmd.AddCommand(networkZoneDeleteCmd.command()) // Record. networkZoneRecordCmd := cmdNetworkZoneRecord{global: c.global, networkZone: c} cmd.AddCommand(networkZoneRecordCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // List. type cmdNetworkZoneList struct { global *cmdGlobal networkZone *cmdNetworkZone flagFormat string flagAllProjects bool flagColumns string } var cmdNetworkZoneListUsage = u.Usage{u.RemoteColonOpt} func (c *cmdNetworkZoneList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdNetworkZoneListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List available network zones") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List available network zone Default column layout: nDSdus == Columns == The -c option takes a comma separated list of arguments that control which network zone attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: d - Description e - Project name n - Name u - Used by`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("Display network zones from all projects")) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultNetworkZoneColumns, "", i18n.G("Columns")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } const defaultNetworkZoneColumns = "ndu" func (c *cmdNetworkZoneList) parseColumns() ([]networkZoneColumn, error) { columnsShorthandMap := map[rune]networkZoneColumn{ 'e': {i18n.G("PROJECT"), c.projectColumnData}, 'n': {i18n.G("NAME"), c.networkZoneNameColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 'u': {i18n.G("USED BY"), c.usedByColumnData}, } if c.flagColumns == defaultNetworkZoneColumns && c.flagAllProjects { c.flagColumns = "endu" } columnList := strings.Split(c.flagColumns, ",") columns := []networkZoneColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdNetworkZoneList) projectColumnData(networkZone api.NetworkZone) string { return networkZone.Project } func (c *cmdNetworkZoneList) networkZoneNameColumnData(networkZone api.NetworkZone) string { return networkZone.Name } func (c *cmdNetworkZoneList) descriptionColumnData(networkZone api.NetworkZone) string { return networkZone.Description } func (c *cmdNetworkZoneList) usedByColumnData(networkZone api.NetworkZone) string { return fmt.Sprintf("%d", len(networkZone.UsedBy)) } func (c *cmdNetworkZoneList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer var zones []api.NetworkZone if c.flagAllProjects { zones, err = d.GetNetworkZonesAllProjects() if err != nil { return err } } else { zones, err = d.GetNetworkZones() if err != nil { return err } } // Parse column flags. columns, err := c.parseColumns() if err != nil { return err } data := [][]string{} for _, zone := range zones { line := []string{} for _, column := range columns { line = append(line, column.Data(zone)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, zones) } // Show. type cmdNetworkZoneShow struct { global *cmdGlobal networkZone *cmdNetworkZone } var cmdNetworkZoneShowUsage = u.Usage{u.Zone.Remote()} func (c *cmdNetworkZoneShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdNetworkZoneShowUsage...) cmd.Short = i18n.G("Show network zone configurations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Show network zone configurations")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkZoneShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer zoneName := parsed[0].RemoteObject.String // Show the network zone config. netZone, _, err := d.GetNetworkZone(zoneName) if err != nil { return err } sort.Strings(netZone.UsedBy) data, err := yaml.Dump(&netZone, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Get. type cmdNetworkZoneGet struct { global *cmdGlobal networkZone *cmdNetworkZone flagIsProperty bool } var cmdNetworkZoneGetUsage = u.Usage{u.Zone.Remote(), u.Key} func (c *cmdNetworkZoneGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdNetworkZoneGetUsage...) cmd.Short = i18n.G("Get values for network zone configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Get values for network zone configuration keys")) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a network zone property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } if len(args) == 1 { return c.global.cmpNetworkZoneConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkZoneGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer zoneName := parsed[0].RemoteObject.String key := parsed[1].String resp, _, err := d.GetNetworkZone(zoneName) if err != nil { return err } if c.flagIsProperty { w := resp.Writable() res, err := getFieldByJSONTag(&w, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the network zone %q: %v"), key, formatRemote(c.global.conf, parsed[0]), err) } fmt.Printf("%v\n", res) } else { for k, v := range resp.Config { if k == key { fmt.Printf("%s\n", v) } } } return nil } // Create. type cmdNetworkZoneCreate struct { global *cmdGlobal networkZone *cmdNetworkZone flagDescription string } var cmdNetworkZoneCreateUsage = u.Usage{u.NewName(u.Zone).Remote(), u.KV.List(0)} func (c *cmdNetworkZoneCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdNetworkZoneCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create new network zones") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Create new network zones")) cmd.Example = cli.FormatSection("", i18n.G(`incus network zone create z1 Create network zone z1 incus network zone create z1 < config.yaml Create network zone z1 with configuration from config.yaml`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Zone description")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkZoneCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer zoneName := parsed[0].RemoteObject.String keys, err := kvToMap(parsed[1]) if err != nil { return err } // If stdin isn't a terminal, read yaml from it. var zonePut api.NetworkZonePut if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } err = loader.Load(&zonePut) if err != nil && !errors.Is(err, io.EOF) { return err } } // Create the network zone. zone := api.NetworkZonesPost{ Name: zoneName, NetworkZonePut: zonePut, } if zone.Config == nil { zone.Config = map[string]string{} } if c.flagDescription != "" { zone.Description = c.flagDescription } maps.Copy(zone.Config, keys) err = d.CreateNetworkZone(zone) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network zone %s created")+"\n", formatRemote(c.global.conf, parsed[0])) } return nil } // Set. type cmdNetworkZoneSet struct { global *cmdGlobal networkZone *cmdNetworkZone flagIsProperty bool } var cmdNetworkZoneSetUsage = u.Usage{u.Zone.Remote(), u.LegacyKV.List(1)} func (c *cmdNetworkZoneSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdNetworkZoneSetUsage...) cmd.Short = i18n.G("Set network zone configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set network zone configuration keys For backward compatibility, a single configuration key may still be set with: incus network set [:] `)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a network zone property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdNetworkZoneSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer zoneName := parsed[0].RemoteObject.String keys, err := kvToMap(parsed[1]) if err != nil { return err } // Get the network zone. netZone, etag, err := d.GetNetworkZone(zoneName) if err != nil { return err } writable := netZone.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { maps.Copy(writable.Config, keys) } return d.UpdateNetworkZone(zoneName, writable, etag) } func (c *cmdNetworkZoneSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Unset. type cmdNetworkZoneUnset struct { global *cmdGlobal networkZone *cmdNetworkZone networkZoneSet *cmdNetworkZoneSet flagIsProperty bool } var cmdNetworkZoneUnsetUsage = u.Usage{u.Zone.Remote(), u.Key} func (c *cmdNetworkZoneUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdNetworkZoneUnsetUsage...) cmd.Short = i18n.G("Unset network zone configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Unset network zone configuration keys")) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a network zone property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } if len(args) == 1 { return c.global.cmpNetworkZoneConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkZoneUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.networkZoneSet.flagIsProperty = c.flagIsProperty return unsetKey(c.networkZoneSet, cmd, parsed) } // Edit. type cmdNetworkZoneEdit struct { global *cmdGlobal networkZone *cmdNetworkZone } var cmdNetworkZoneEditUsage = u.Usage{u.Zone.Remote()} func (c *cmdNetworkZoneEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdNetworkZoneEditUsage...) cmd.Short = i18n.G("Edit network zone configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Edit network zone configurations as YAML")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkZoneEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the network zone. ### Any line starting with a '# will be ignored. ### ### A network zone consists of a set of rules and configuration items. ### ### An example would look like: ### name: example.net ### description: Internal domain ### config: ### user.foo: bah `) } func (c *cmdNetworkZoneEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer zoneName := parsed[0].RemoteObject.String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } // Allow output of `incus network zone show` command to be passed in here, but only take the contents // of the NetworkZonePut fields when updating the Zone. The other fields are silently discarded. newdata := api.NetworkZone{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateNetworkZone(zoneName, newdata.NetworkZonePut, "") } // Get the current config. netZone, etag, err := d.GetNetworkZone(zoneName) if err != nil { return err } data, err := yaml.Dump(&netZone, yaml.V2) if err != nil { return err } // Spawn the editor. content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor. newdata := api.NetworkZone{} // We show the full Zone info, but only send the writable fields. err = yaml.Load(content, &newdata, yaml.WithKnownFields()) if err == nil { err = d.UpdateNetworkZone(zoneName, newdata.Writable(), etag) } // Respawn the editor. if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Delete. type cmdNetworkZoneDelete struct { global *cmdGlobal networkZone *cmdNetworkZone } var cmdNetworkZoneDeleteUsage = u.Usage{u.Zone.Remote().List(1)} func (c *cmdNetworkZoneDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdNetworkZoneDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete network zones") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Delete network zones")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpNetworkZones(toComplete) } return cmd } func (c *cmdNetworkZoneDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } var errs []error for _, p := range parsed[0].List { d := p.RemoteServer zoneName := p.RemoteObject.String // Delete the network zone. err = d.DeleteNetworkZone(zoneName) if err != nil { errs = append(errs, err) continue } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network Zone %s deleted")+"\n", formatRemote(c.global.conf, p)) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } // Add/Remove Rule. type cmdNetworkZoneRecord struct { global *cmdGlobal networkZone *cmdNetworkZone } func (c *cmdNetworkZoneRecord) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("record") cmd.Short = i18n.G("Manage network zone records") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Manage network zone records")) // List. networkZoneRecordListCmd := cmdNetworkZoneRecordList{global: c.global, networkZoneRecord: c} cmd.AddCommand(networkZoneRecordListCmd.command()) // Show. networkZoneRecordShowCmd := cmdNetworkZoneRecordShow{global: c.global, networkZoneRecord: c} cmd.AddCommand(networkZoneRecordShowCmd.command()) // Get. networkZoneRecordGetCmd := cmdNetworkZoneRecordGet{global: c.global, networkZoneRecord: c} cmd.AddCommand(networkZoneRecordGetCmd.command()) // Create. networkZoneRecordCreateCmd := cmdNetworkZoneRecordCreate{global: c.global, networkZoneRecord: c} cmd.AddCommand(networkZoneRecordCreateCmd.command()) // Set. networkZoneRecordSetCmd := cmdNetworkZoneRecordSet{global: c.global, networkZoneRecord: c} cmd.AddCommand(networkZoneRecordSetCmd.command()) // Unset. networkZoneRecordUnsetCmd := cmdNetworkZoneRecordUnset{global: c.global, networkZoneRecord: c, networkZoneRecordSet: &networkZoneRecordSetCmd} cmd.AddCommand(networkZoneRecordUnsetCmd.command()) // Edit. networkZoneRecordEditCmd := cmdNetworkZoneRecordEdit{global: c.global, networkZoneRecord: c} cmd.AddCommand(networkZoneRecordEditCmd.command()) // Delete. networkZoneRecordDeleteCmd := cmdNetworkZoneRecordDelete{global: c.global, networkZoneRecord: c} cmd.AddCommand(networkZoneRecordDeleteCmd.command()) // Entry. networkZoneRecordEntryCmd := cmdNetworkZoneRecordEntry{global: c.global, networkZoneRecord: c} cmd.AddCommand(networkZoneRecordEntryCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // List. type cmdNetworkZoneRecordList struct { global *cmdGlobal networkZoneRecord *cmdNetworkZoneRecord flagFormat string } var cmdNetworkZoneRecordListUsage = u.Usage{u.Zone.Remote()} func (c *cmdNetworkZoneRecordList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdNetworkZoneRecordListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List available network zone records") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("List available network zone records")) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkZoneRecordList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneRecordListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer zoneName := parsed[0].RemoteObject.String // List the records. records, err := d.GetNetworkZoneRecords(zoneName) if err != nil { return err } data := [][]string{} for _, record := range records { entries := []string{} for _, entry := range record.Entries { entries = append(entries, fmt.Sprintf("%s %s", entry.Type, entry.Value)) } details := []string{ record.Name, record.Description, strings.Join(entries, "\n"), } data = append(data, details) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{ i18n.G("NAME"), i18n.G("DESCRIPTION"), i18n.G("ENTRIES"), } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, records) } // Show. type cmdNetworkZoneRecordShow struct { global *cmdGlobal networkZoneRecord *cmdNetworkZoneRecord } var cmdNetworkZoneRecordShowUsage = u.Usage{u.Zone.Remote(), u.Record} func (c *cmdNetworkZoneRecordShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdNetworkZoneRecordShowUsage...) cmd.Short = i18n.G("Show network zone record configuration") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Show network zone record configurations")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } if len(args) == 1 { return c.global.cmpNetworkZoneRecords(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkZoneRecordShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneRecordShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer zoneName := parsed[0].RemoteObject.String recordName := parsed[1].String // Show the network zone config. netRecord, _, err := d.GetNetworkZoneRecord(zoneName, recordName) if err != nil { return err } data, err := yaml.Dump(&netRecord, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Get. type cmdNetworkZoneRecordGet struct { global *cmdGlobal networkZoneRecord *cmdNetworkZoneRecord flagIsProperty bool } var cmdNetworkZoneRecordGetUsage = u.Usage{u.Zone.Remote(), u.Record, u.Key} func (c *cmdNetworkZoneRecordGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdNetworkZoneRecordGetUsage...) cmd.Short = i18n.G("Get values for network zone record configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Get values for network zone record configuration keys")) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a network zone record property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } if len(args) == 1 { return c.global.cmpNetworkZoneRecords(args[0]) } if len(args) == 2 { return c.global.cmpNetworkZoneRecordConfigs(args[0], args[1]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkZoneRecordGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneRecordGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer zoneName := parsed[0].RemoteObject.String recordName := parsed[1].String key := parsed[2].String resp, _, err := d.GetNetworkZoneRecord(zoneName, recordName) if err != nil { return err } if c.flagIsProperty { w := resp.Writable() res, err := getFieldByJSONTag(&w, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the network zone record %q: %v"), key, recordName, err) } fmt.Printf("%v\n", res) } else { for k, v := range resp.Config { if k == key { fmt.Printf("%s\n", v) } } } return nil } // Create. type cmdNetworkZoneRecordCreate struct { global *cmdGlobal networkZoneRecord *cmdNetworkZoneRecord flagDescription string } var cmdNetworkZoneRecordCreateUsage = u.Usage{u.Zone.Remote(), u.NewName(u.Record), u.KV.List(0)} func (c *cmdNetworkZoneRecordCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdNetworkZoneRecordCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create new network zone record") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Create new network zone record")) cmd.Example = cli.FormatSection("", i18n.G(`incus network zone record create z1 r1 Create record r1 for zone z1 incus network zone record create z1 r1 < config.yaml Create record r1 for zone z1 with configuration from config.yaml`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Record description")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } if len(args) == 1 { return c.global.cmpNetworkZoneRecords(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkZoneRecordCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneRecordCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer zoneName := parsed[0].RemoteObject.String recordName := parsed[1].String keys, err := kvToMap(parsed[2]) if err != nil { return err } // If stdin isn't a terminal, read yaml from it. var recordPut api.NetworkZoneRecordPut if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } err = loader.Load(&recordPut) if err != nil && !errors.Is(err, io.EOF) { return err } } // Create the network zone. record := api.NetworkZoneRecordsPost{ Name: recordName, NetworkZoneRecordPut: recordPut, } if record.Config == nil { record.Config = map[string]string{} } if c.flagDescription != "" { record.Description = c.flagDescription } maps.Copy(record.Config, keys) err = d.CreateNetworkZoneRecord(zoneName, record) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network zone record %s created")+"\n", recordName) } return nil } // Set. type cmdNetworkZoneRecordSet struct { global *cmdGlobal networkZoneRecord *cmdNetworkZoneRecord flagIsProperty bool } var cmdNetworkZoneRecordSetUsage = u.Usage{u.Zone.Remote(), u.Record, u.LegacyKV.List(1)} func (c *cmdNetworkZoneRecordSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdNetworkZoneRecordSetUsage...) cmd.Short = i18n.G("Set network zone record configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set network zone record configuration keys`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a network zone record property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } if len(args) == 1 { return c.global.cmpNetworkZoneRecords(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdNetworkZoneRecordSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer zoneName := parsed[0].RemoteObject.String recordName := parsed[1].String keys, err := kvToMap(parsed[2]) if err != nil { return err } // Get the network zone. netRecord, etag, err := d.GetNetworkZoneRecord(zoneName, recordName) if err != nil { return err } writable := netRecord.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { maps.Copy(writable.Config, keys) } return d.UpdateNetworkZoneRecord(zoneName, recordName, writable, etag) } func (c *cmdNetworkZoneRecordSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneRecordSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Unset. type cmdNetworkZoneRecordUnset struct { global *cmdGlobal networkZoneRecord *cmdNetworkZoneRecord networkZoneRecordSet *cmdNetworkZoneRecordSet flagIsProperty bool } var cmdNetworkZoneRecordUnsetUsage = u.Usage{u.Zone.Remote(), u.Record, u.Key} func (c *cmdNetworkZoneRecordUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdNetworkZoneRecordUnsetUsage...) cmd.Short = i18n.G("Unset network zone record configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Unset network zone record configuration keys")) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a network zone record property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } if len(args) == 1 { return c.global.cmpNetworkZoneRecords(args[0]) } if len(args) == 2 { return c.global.cmpNetworkZoneRecordConfigs(args[0], args[1]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkZoneRecordUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneRecordUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.networkZoneRecordSet.flagIsProperty = c.flagIsProperty return unsetKey(c.networkZoneRecordSet, cmd, parsed) } // Edit. type cmdNetworkZoneRecordEdit struct { global *cmdGlobal networkZoneRecord *cmdNetworkZoneRecord } var cmdNetworkZoneRecordEditUsage = u.Usage{u.Zone.Remote(), u.Record} func (c *cmdNetworkZoneRecordEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdNetworkZoneRecordEditUsage...) cmd.Short = i18n.G("Edit network zone record configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Edit network zone record configurations as YAML")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } if len(args) == 1 { return c.global.cmpNetworkZoneRecords(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkZoneRecordEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the network zone record. ### Any line starting with a '# will be ignored. ### ### A network zone consists of a set of rules and configuration items. ### ### An example would look like: ### name: foo ### description: SPF record ### config: ### user.foo: bah `) } func (c *cmdNetworkZoneRecordEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneRecordEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer zoneName := parsed[0].RemoteObject.String recordName := parsed[1].String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } // Allow output of `incus network zone show` command to be passed in here, but only take the contents // of the NetworkZonePut fields when updating the Zone. The other fields are silently discarded. newdata := api.NetworkZoneRecord{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateNetworkZoneRecord(zoneName, recordName, newdata.NetworkZoneRecordPut, "") } // Get the current config. netRecord, etag, err := d.GetNetworkZoneRecord(zoneName, recordName) if err != nil { return err } data, err := yaml.Dump(netRecord.Writable, yaml.V2) if err != nil { return err } // Spawn the editor. content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor. newdata := api.NetworkZoneRecord{} // We show the full Zone info, but only send the writable fields. err = yaml.Load(content, &newdata, yaml.WithKnownFields()) if err == nil { err = d.UpdateNetworkZoneRecord(zoneName, recordName, newdata.Writable(), etag) } // Respawn the editor. if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Delete. type cmdNetworkZoneRecordDelete struct { global *cmdGlobal networkZoneRecord *cmdNetworkZoneRecord } var cmdNetworkZoneRecordDeleteUsage = u.Usage{u.Zone.Remote(), u.Record} func (c *cmdNetworkZoneRecordDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdNetworkZoneRecordDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete network zone record") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Delete network zone record")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } if len(args) == 1 { return c.global.cmpNetworkZoneRecords(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkZoneRecordDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneRecordDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer zoneName := parsed[0].RemoteObject.String recordName := parsed[1].String // Delete the network zone. err = d.DeleteNetworkZoneRecord(zoneName, recordName) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Network zone record %s deleted")+"\n", recordName) } return nil } // Add/Remove Rule. type cmdNetworkZoneRecordEntry struct { global *cmdGlobal networkZoneRecord *cmdNetworkZoneRecord flagTTL uint64 } func (c *cmdNetworkZoneRecordEntry) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("entry") cmd.Short = i18n.G("Manage network zone record entries") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Manage network zone record entries")) // Rule Add. cmd.AddCommand(c.commandAdd()) // Rule Remove. cmd.AddCommand(c.commandRemove()) return cmd } var cmdNetworkZoneRecordEntryAddUsage = u.Usage{u.Zone.Remote(), u.Record, u.Type, u.Value} func (c *cmdNetworkZoneRecordEntry) commandAdd() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("add", cmdNetworkZoneRecordEntryAddUsage...) cmd.Aliases = []string{"create"} cmd.Short = i18n.G("Add a network zone record entry") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Add entries to a network zone record")) cmd.RunE = c.runAdd cli.AddUint64Flag(cmd.Flags(), &c.flagTTL, "ttl", i18n.G("Entry TTL")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } if len(args) == 1 { return c.global.cmpNetworkZoneRecords(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkZoneRecordEntry) runAdd(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneRecordEntryAddUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer zoneName := parsed[0].RemoteObject.String recordName := parsed[1].String entryType := parsed[2].String entryValue := parsed[3].String // Get the network record. netRecord, etag, err := d.GetNetworkZoneRecord(zoneName, recordName) if err != nil { return err } // Add the entry. netRecord.Entries = append(netRecord.Entries, api.NetworkZoneRecordEntry{ Type: entryType, TTL: c.flagTTL, Value: entryValue, }) return d.UpdateNetworkZoneRecord(zoneName, recordName, netRecord.Writable(), etag) } var cmdNetworkZoneRecordEntryRemoveUsage = u.Usage{u.Zone.Remote(), u.Record, u.Type, u.Value} func (c *cmdNetworkZoneRecordEntry) commandRemove() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remove", cmdNetworkZoneRecordEntryRemoveUsage...) cmd.Aliases = []string{"delete", "rm"} cmd.Short = i18n.G("Remove a network zone record entry") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Remove entries from a network zone record")) cmd.RunE = c.runRemove cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpNetworkZones(toComplete) } if len(args) == 1 { return c.global.cmpNetworkZoneRecords(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdNetworkZoneRecordEntry) runRemove(cmd *cobra.Command, args []string) error { parsed, err := cmdNetworkZoneRecordEntryRemoveUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer zoneName := parsed[0].RemoteObject.String recordName := parsed[1].String entryType := parsed[2].String entryValue := parsed[3].String // Get the network zone record. netRecord, etag, err := d.GetNetworkZoneRecord(zoneName, recordName) if err != nil { return err } found := false for i, entry := range netRecord.Entries { if entry.Type != entryType || entry.Value != entryValue { continue } found = true netRecord.Entries = slices.Delete(netRecord.Entries, i, i+1) break } if !found { return errors.New(i18n.G("Couldn't find a matching entry")) } return d.UpdateNetworkZoneRecord(zoneName, recordName, netRecord.Writable(), etag) } incus-7.0.0/cmd/incus/operation.go000066400000000000000000000202111517523235500170560ustar00rootroot00000000000000package main import ( "errors" "fmt" "os" "sort" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdOperation struct { global *cmdGlobal } type operationColumn struct { Name string Data func(api.Operation) string } func (c *cmdOperation) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("operation") cmd.Short = i18n.G("List, show and delete background operations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List, show and delete background operations`)) cmd.Hidden = true // Delete operationDeleteCmd := cmdOperationDelete{global: c.global, operation: c} cmd.AddCommand(operationDeleteCmd.command()) // List operationListCmd := cmdOperationList{global: c.global, operation: c} cmd.AddCommand(operationListCmd.command()) // Show operationShowCmd := cmdOperationShow{global: c.global, operation: c} cmd.AddCommand(operationShowCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Delete. type cmdOperationDelete struct { global *cmdGlobal operation *cmdOperation } var cmdOperationDeleteUsage = u.Usage{u.Operation.Remote().List(1)} func (c *cmdOperationDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdOperationDeleteUsage...) cmd.Aliases = []string{"cancel", "rm", "remove"} cmd.Short = i18n.G("Delete background operations (will attempt to cancel)") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Delete background operations (will attempt to cancel)`)) cmd.RunE = c.run return cmd } func (c *cmdOperationDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdOperationDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } var errs []error for _, p := range parsed[0].List { d := p.RemoteServer operationName := p.RemoteObject.String // Delete the operation err = d.DeleteOperation(operationName) if err != nil { errs = append(errs, err) continue } if !c.global.flagQuiet { fmt.Printf(i18n.G("Operation %s deleted")+"\n", formatRemote(c.global.conf, p)) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } // List. type cmdOperationList struct { global *cmdGlobal operation *cmdOperation flagFormat string flagColumns string flagAllProjects bool } var cmdOperationListUsage = u.Usage{u.RemoteColonOpt} func (c *cmdOperationList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdOperationListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List background operations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List background operations Default column layout: itdscCl == Columns == The -c option takes a comma separated list of arguments that control which attributes of background operations to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: i - ID t - Type d - Description s - State c - Cancelable C - Created L - Location of the operation (e.g. its cluster member)`)) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("List operations from all projects")) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultOperationColumns, "", i18n.G("Columns")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run return cmd } const defaultOperationColumns = "itdscC" func (c *cmdOperationList) parseColumns(clustered bool) ([]operationColumn, error) { columnsShorthandMap := map[rune]operationColumn{ 'i': {i18n.G("ID"), c.operationIDcolumnData}, 't': {i18n.G("TYPE"), c.typeColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 's': {i18n.G("STATE"), c.stateColumnData}, 'c': {i18n.G("CANCELABLE"), c.cancelableColumnData}, 'C': {i18n.G("CREATED"), c.createdColumnData}, 'L': {i18n.G("LOCATION"), c.locationColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []operationColumn{} if c.flagColumns == defaultOperationColumns && clustered { columnList = append(columnList, "L") } for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdOperationList) operationIDcolumnData(op api.Operation) string { return op.ID } func (c *cmdOperationList) typeColumnData(op api.Operation) string { return strings.ToUpper(op.Class) } func (c *cmdOperationList) descriptionColumnData(op api.Operation) string { return op.Description } func (c *cmdOperationList) stateColumnData(op api.Operation) string { return strings.ToUpper(op.Status) } func (c *cmdOperationList) cancelableColumnData(op api.Operation) string { strCancelable := i18n.G("NO") if op.MayCancel { strCancelable = i18n.G("YES") } return strCancelable } func (c *cmdOperationList) createdColumnData(op api.Operation) string { return op.CreatedAt.Local().Format(dateLayout) } func (c *cmdOperationList) locationColumnData(op api.Operation) string { return op.Location } func (c *cmdOperationList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdOperationListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer // Get operations var operations []api.Operation if c.flagAllProjects { operations, err = d.GetOperationsAllProjects() } else { operations, err = d.GetOperations() } if err != nil { return err } // Parse column flags. columns, err := c.parseColumns(d.IsClustered()) if err != nil { return err } // Render the table data := [][]string{} for _, op := range operations { line := []string{} for _, column := range columns { line = append(line, column.Data(op)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, operations) } // Show. type cmdOperationShow struct { global *cmdGlobal operation *cmdOperation } var cmdOperationShowUsage = u.Usage{u.Operation.Remote()} func (c *cmdOperationShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdOperationShowUsage...) cmd.Short = i18n.G("Show details on a background operation") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Show details on a background operation`)) cmd.Example = cli.FormatSection("", i18n.G( `incus operation show 344a79e4-d88a-45bf-9c39-c72c26f6ab8a Show details on that operation UUID`)) cmd.RunE = c.run return cmd } func (c *cmdOperationShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdOperationShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer operationName := parsed[0].RemoteObject.String // Get the operation op, _, err := d.GetOperation(operationName) if err != nil { return err } // Render as YAML data, err := yaml.Dump(&op, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } incus-7.0.0/cmd/incus/profile.go000066400000000000000000000706461517523235500165370ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "maps" "net/http" "os" "regexp" "slices" "sort" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" ) type profileColumn struct { Name string Data func(api.Profile) string } type cmdProfile struct { global *cmdGlobal } func (c *cmdProfile) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("profile") cmd.Short = i18n.G("Manage profiles") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage profiles`)) // Add profileAddCmd := cmdProfileAdd{global: c.global, profile: c} cmd.AddCommand(profileAddCmd.command()) // Assign profileAssignCmd := cmdProfileAssign{global: c.global, profile: c} cmd.AddCommand(profileAssignCmd.command()) // Copy profileCopyCmd := cmdProfileCopy{global: c.global, profile: c} cmd.AddCommand(profileCopyCmd.command()) // Create profileCreateCmd := cmdProfileCreate{global: c.global, profile: c} cmd.AddCommand(profileCreateCmd.command()) // Delete profileDeleteCmd := cmdProfileDelete{global: c.global, profile: c} cmd.AddCommand(profileDeleteCmd.command()) // Device profileDeviceCmd := cmdConfigDevice{global: c.global, profile: c} cmd.AddCommand(profileDeviceCmd.command()) // Edit profileEditCmd := cmdProfileEdit{global: c.global, profile: c} cmd.AddCommand(profileEditCmd.command()) // Get profileGetCmd := cmdProfileGet{global: c.global, profile: c} cmd.AddCommand(profileGetCmd.command()) // List profileListCmd := cmdProfileList{global: c.global, profile: c} cmd.AddCommand(profileListCmd.command()) // Remove profileRemoveCmd := cmdProfileRemove{global: c.global, profile: c} cmd.AddCommand(profileRemoveCmd.command()) // Rename profileRenameCmd := cmdProfileRename{global: c.global, profile: c} cmd.AddCommand(profileRenameCmd.command()) // Set profileSetCmd := cmdProfileSet{global: c.global, profile: c} cmd.AddCommand(profileSetCmd.command()) // Show profileShowCmd := cmdProfileShow{global: c.global, profile: c} cmd.AddCommand(profileShowCmd.command()) // Unset profileUnsetCmd := cmdProfileUnset{global: c.global, profile: c, profileSet: &profileSetCmd} cmd.AddCommand(profileUnsetCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Add. type cmdProfileAdd struct { global *cmdGlobal profile *cmdProfile } var cmdProfileAddUsage = u.Usage{u.Instance.Remote(), u.Profile} func (c *cmdProfileAdd) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("add", cmdProfileAddUsage...) cmd.Short = i18n.G("Add profiles to instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Add profiles to instances`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } if len(args) == 1 { return c.global.cmpProfiles(args[0], false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProfileAdd) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProfileAddUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String profileName := parsed[1].String // Add the profile inst, etag, err := d.GetInstance(instanceName) if err != nil { return err } inst.Profiles = append(inst.Profiles, profileName) op, err := d.UpdateInstance(instanceName, inst.Writable(), etag) if err != nil { return err } err = op.Wait() if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Profile %s added to %s")+"\n", profileName, formatRemote(c.global.conf, parsed[0])) } return nil } // Assign. type cmdProfileAssign struct { global *cmdGlobal profile *cmdProfile flagNoProfiles bool } var cmdProfileAssignUsage = u.Usage{u.Instance.Remote(), u.Either(u.Profile.List(1, ","), u.Flag("no-profiles"))} func (c *cmdProfileAssign) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("assign", cmdProfileAssignUsage...) cmd.Aliases = []string{"apply"} cmd.Short = i18n.G("Assign sets of profiles to instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Assign sets of profiles to instances`)) cmd.Example = cli.FormatSection("", i18n.G( `incus profile assign foo default,bar Set the profiles for "foo" to "default" and "bar". incus profile assign foo default Reset "foo" to only using the "default" profile. incus profile assign foo --no-profiles Remove all profile assigned to "foo"`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagNoProfiles, "no-profiles", i18n.G("Remove all profiles from the instance")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } return c.global.cmpProfiles(args[0], false) } return cmd } func (c *cmdProfileAssign) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProfileAssignUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String profiles := parsed[1].StringList inst, etag, err := d.GetInstance(instanceName) if err != nil { return err } if parsed[1].BranchID == 0 { if c.flagNoProfiles { return errors.New(i18n.G("--no-profiles cannot be used together with other arguments")) } inst.Profiles = profiles } else { inst.Profiles = []string{} } op, err := d.UpdateInstance(instanceName, inst.Writable(), etag) if err != nil { return err } err = op.Wait() if err != nil { return err } if !c.global.flagQuiet { if parsed[1].BranchID == 0 { fmt.Printf(i18n.G("Profiles %s applied to %s")+"\n", parsed[1].String, formatRemote(c.global.conf, parsed[0])) } else { fmt.Printf(i18n.G("All profiles removed from %s")+"\n", formatRemote(c.global.conf, parsed[0])) } } return nil } // Copy. type cmdProfileCopy struct { global *cmdGlobal profile *cmdProfile flagTargetProject string flagRefresh bool } var cmdProfileCopyUsage = u.Usage{u.Profile.Remote(), u.NewName(u.Profile).Remote()} func (c *cmdProfileCopy) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("copy", cmdProfileCopyUsage...) cmd.Aliases = []string{"cp"} cmd.Short = i18n.G("Copy profiles") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Copy profiles`)) cli.AddStringFlag(cmd.Flags(), &c.flagTargetProject, "target-project", "", "", i18n.G("Copy to a project different from the source")) cli.AddBoolFlag(cmd.Flags(), &c.flagRefresh, "refresh", i18n.G("Update the target profile from the source if it already exists")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProfiles(toComplete, true) } if len(args) == 1 { return c.global.cmpProfiles(toComplete, true) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProfileCopy) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProfileCopyUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } srcServer := parsed[0].RemoteServer srcProfile := parsed[0].RemoteObject.String dstServer := parsed[1].RemoteServer dstProfile := parsed[1].RemoteObject.String // Copy the profile profile, _, err := srcServer.GetProfile(srcProfile) if err != nil { return err } if c.flagTargetProject != "" { dstServer = dstServer.UseProject(c.flagTargetProject) } // Refresh the profile if requested. if c.flagRefresh { err := dstServer.UpdateProfile(dstProfile, profile.Writable(), "") if err == nil || !api.StatusErrorCheck(err, http.StatusNotFound) { return err } } newProfile := api.ProfilesPost{ ProfilePut: profile.Writable(), Name: dstProfile, } return dstServer.CreateProfile(newProfile) } // Create. type cmdProfileCreate struct { global *cmdGlobal profile *cmdProfile flagDescription string } var cmdProfileCreateUsage = u.Usage{u.NewName(u.Profile).Remote()} func (c *cmdProfileCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdProfileCreateUsage...) cmd.Short = i18n.G("Create profiles") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Create profiles`)) cmd.Example = cli.FormatSection("", i18n.G(`incus profile create p1 Create a profile named p1 incus profile create p1 < config.yaml Create a profile named p1 with configuration from config.yaml`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Profile description")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProfileCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProfileCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer profileName := parsed[0].RemoteObject.String var stdinData api.ProfilePut // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } err = loader.Load(&stdinData) if err != nil && !errors.Is(err, io.EOF) { return err } } // Create the profile profile := api.ProfilesPost{} profile.Name = profileName profile.ProfilePut = stdinData if c.flagDescription != "" { profile.Description = c.flagDescription } err = d.CreateProfile(profile) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Profile %s created")+"\n", formatRemote(c.global.conf, parsed[0])) } return nil } // Delete. type cmdProfileDelete struct { global *cmdGlobal profile *cmdProfile } var cmdProfileDeleteUsage = u.Usage{u.Profile.Remote().List(1)} func (c *cmdProfileDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdProfileDeleteUsage...) cmd.Aliases = []string{"rm"} cmd.Short = i18n.G("Delete profiles") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete profiles`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpProfiles(toComplete, true) } return cmd } func (c *cmdProfileDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProfileDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } var errs []error for _, p := range parsed[0].List { d := p.RemoteServer profileName := p.RemoteObject.String // Delete the profile err = d.DeleteProfile(profileName) if err != nil { errs = append(errs, err) continue } if !c.global.flagQuiet { fmt.Printf(i18n.G("Profile %s deleted")+"\n", formatRemote(c.global.conf, p)) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } // Edit. type cmdProfileEdit struct { global *cmdGlobal profile *cmdProfile } var cmdProfileEditUsage = u.Usage{u.Profile.Remote()} func (c *cmdProfileEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdProfileEditUsage...) cmd.Short = i18n.G("Edit profile configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Edit profile configurations as YAML`)) cmd.Example = cli.FormatSection("", i18n.G( `incus profile edit < profile.yaml Update a profile using the content of profile.yaml`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProfiles(toComplete, true) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProfileEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the profile. ### Any line starting with a '# will be ignored. ### ### A profile consists of a set of configuration items followed by a set of ### devices. ### ### An example would look like: ### name: onenic ### config: ### raw.lxc: lxc.aa_profile=unconfined ### devices: ### eth0: ### nictype: bridged ### parent: mybr0 ### type: nic ### ### Note that the name is shown but cannot be changed`) } func (c *cmdProfileEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProfileEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer profileName := parsed[0].RemoteObject.String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } newdata := api.ProfilePut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateProfile(profileName, newdata, "") } // Extract the current value profile, etag, err := d.GetProfile(profileName) if err != nil { return err } data, err := yaml.Dump(&profile, yaml.V2) if err != nil { return err } // Spawn the editor content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor newdata := api.ProfilePut{} err = yaml.Load(content, &newdata) if err == nil { err = d.UpdateProfile(profileName, newdata, etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Get. type cmdProfileGet struct { global *cmdGlobal profile *cmdProfile flagIsProperty bool } var cmdProfileGetUsage = u.Usage{u.Profile.Remote(), u.Key} func (c *cmdProfileGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdProfileGetUsage...) cmd.Short = i18n.G("Get values for profile configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Get values for profile configuration keys`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a profile property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProfiles(toComplete, true) } if len(args) == 1 { return c.global.cmpProfileConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProfileGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProfileGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer profileName := parsed[0].RemoteObject.String key := parsed[1].String // Get the configuration key profile, _, err := d.GetProfile(profileName) if err != nil { return err } if c.flagIsProperty { w := profile.Writable() res, err := getFieldByJSONTag(&w, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the profile %q: %v"), profileName, formatRemote(c.global.conf, parsed[0]), err) } fmt.Printf("%v\n", res) } else { fmt.Printf("%s\n", profile.Config[key]) } return nil } // List. type cmdProfileList struct { global *cmdGlobal profile *cmdProfile flagFormat string flagColumns string flagAllProjects bool } var cmdProfileListUsage = u.Usage{u.RemoteColonOpt, u.Filter.List(0)} func (c *cmdProfileList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdProfileListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List profiles") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List profiles Filters may be of the = form for property based filtering, or part of the profile name. Filters must be delimited by a ','. Examples: - "foo" lists all profiles that start with the name foo - "name=foo" lists all profiles that exactly have the name foo - "description=.*bar.*" lists all profiles with a description that contains "bar" The -c option takes a (optionally comma-separated) list of arguments that control which profile attributes to output when displaying in table or csv format. Default column layout is: ndu Column shorthand chars: n - Profile Name d - Description u - Used By`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultProfileColumns, "", i18n.G("Columns")) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("Display profiles from all projects")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } const ( defaultProfileColumns = "ndu" defaultProfileColumnsAllProjects = "endu" ) func (c *cmdProfileList) parseColumns() ([]profileColumn, error) { columnsShorthandMap := map[rune]profileColumn{ 'n': {i18n.G("NAME"), c.profileNameColumnData}, 'e': {i18n.G("PROJECT"), c.projectNameColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 'u': {i18n.G("USED BY"), c.usedByColumnData}, } // Add project column if --all-projects flag specified and no custom column was passed. if c.flagAllProjects { if c.flagColumns == defaultProfileColumns { c.flagColumns = defaultProfileColumnsAllProjects } } columnList := strings.Split(c.flagColumns, ",") columns := []profileColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdProfileList) profileNameColumnData(profile api.Profile) string { return profile.Name } func (c *cmdProfileList) descriptionColumnData(profile api.Profile) string { return profile.Description } func (c *cmdProfileList) projectNameColumnData(profile api.Profile) string { return profile.Project } func (c *cmdProfileList) usedByColumnData(profile api.Profile) string { return fmt.Sprintf("%d", len(profile.UsedBy)) } func (c *cmdProfileList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProfileListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer filters := parsed[1].StringList if c.global.flagProject != "" && c.flagAllProjects { return errors.New(i18n.G("Can't specify --project with --all-projects")) } flattenedFilters := []string{} for _, filter := range filters { flattenedFilters = append(flattenedFilters, strings.Split(filter, ",")...) } filters = flattenedFilters if len(filters) > 0 && !strings.Contains(filters[0], "=") { filters[0] = fmt.Sprintf("name=^%s($|.*)", regexp.QuoteMeta(filters[0])) } serverFilters, _ := getServerSupportedFilters(filters, []string{}, false) // List profiles var profiles []api.Profile if c.flagAllProjects { profiles, err = d.GetProfilesAllProjectsWithFilter(serverFilters) } else { profiles, err = d.GetProfilesWithFilter(serverFilters) } if err != nil { return err } columns, err := c.parseColumns() if err != nil { return err } data := [][]string{} for _, profile := range profiles { line := []string{} for _, column := range columns { line = append(line, column.Data(profile)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, profiles) } // Remove. type cmdProfileRemove struct { global *cmdGlobal profile *cmdProfile } var cmdProfileRemoveUsage = u.Usage{u.Instance.Remote(), u.Profile} func (c *cmdProfileRemove) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remove", cmdProfileRemoveUsage...) cmd.Short = i18n.G("Remove profiles from instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Remove profiles from instances`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } if len(args) == 1 { return c.global.cmpProfiles(args[0], false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProfileRemove) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProfileRemoveUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String profileName := parsed[1].String // Remove the profile inst, etag, err := d.GetInstance(instanceName) if err != nil { return err } if !slices.Contains(inst.Profiles, profileName) { return fmt.Errorf(i18n.G("Profile %s isn't currently applied to %s"), profileName, formatRemote(c.global.conf, parsed[0])) } profiles := []string{} for _, profile := range inst.Profiles { if profile == profileName { continue } profiles = append(profiles, profile) } inst.Profiles = profiles op, err := d.UpdateInstance(instanceName, inst.Writable(), etag) if err != nil { return err } err = op.Wait() if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Profile %s removed from %s")+"\n", profileName, formatRemote(c.global.conf, parsed[0])) } return nil } // Rename. type cmdProfileRename struct { global *cmdGlobal profile *cmdProfile } var cmdProfileRenameUsage = u.Usage{u.Profile.Remote(), u.NewName(u.Profile)} func (c *cmdProfileRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdProfileRenameUsage...) cmd.Aliases = []string{"mv"} cmd.Short = i18n.G("Rename profiles") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Rename profiles`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProfiles(toComplete, true) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProfileRename) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProfileRenameUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer profileName := parsed[0].RemoteObject.String newProfileName := parsed[1].String // Rename the profile err = d.RenameProfile(profileName, api.ProfilePost{Name: newProfileName}) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Profile %s renamed to %s")+"\n", formatRemote(c.global.conf, parsed[0]), newProfileName) } return nil } // Set. type cmdProfileSet struct { global *cmdGlobal profile *cmdProfile flagIsProperty bool } var cmdProfileSetUsage = u.Usage{u.Profile.Remote(), u.LegacyKV.List(1)} func (c *cmdProfileSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdProfileSetUsage...) cmd.Short = i18n.G("Set profile configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set profile configuration keys For backward compatibility, a single configuration key may still be set with: incus profile set [:] `)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a profile property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProfiles(toComplete, true) } if len(args) == 1 { return c.global.cmpInstanceAllKeys() } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdProfileSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer profileName := parsed[0].RemoteObject.String keys, err := kvToMap(parsed[1]) if err != nil { return err } // Get the profile profile, etag, err := d.GetProfile(profileName) if err != nil { return err } writable := profile.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { maps.Copy(writable.Config, keys) } return d.UpdateProfile(profileName, writable, etag) } func (c *cmdProfileSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProfileSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Show. type cmdProfileShow struct { global *cmdGlobal profile *cmdProfile } var cmdProfileShowUsage = u.Usage{u.Profile.Remote()} func (c *cmdProfileShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdProfileShowUsage...) cmd.Short = i18n.G("Show profile configurations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show profile configurations`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProfiles(toComplete, true) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProfileShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProfileShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer profileName := parsed[0].RemoteObject.String // Show the profile profile, _, err := d.GetProfile(profileName) if err != nil { return err } data, err := yaml.Dump(&profile, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Unset. type cmdProfileUnset struct { global *cmdGlobal profile *cmdProfile profileSet *cmdProfileSet flagIsProperty bool } var cmdProfileUnsetUsage = u.Usage{u.Profile.Remote(), u.Key} func (c *cmdProfileUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdProfileUnsetUsage...) cmd.Short = i18n.G("Unset profile configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Unset profile configuration keys`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a profile property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProfiles(toComplete, true) } if len(args) == 1 { return c.global.cmpProfileConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProfileUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProfileUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.profileSet.flagIsProperty = c.flagIsProperty return unsetKey(c.profileSet, cmd, parsed) } incus-7.0.0/cmd/incus/project.go000066400000000000000000000710131517523235500165320ustar00rootroot00000000000000package main import ( "bufio" "errors" "fmt" "io" "maps" "os" "reflect" "slices" "sort" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) type projectColumn struct { Name string Data func(api.Project) string } type cmdProject struct { global *cmdGlobal } func (c *cmdProject) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("project") cmd.Short = i18n.G("Manage projects") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage projects`)) // Create projectCreateCmd := cmdProjectCreate{global: c.global, project: c} cmd.AddCommand(projectCreateCmd.command()) // Delete projectDeleteCmd := cmdProjectDelete{global: c.global, project: c} cmd.AddCommand(projectDeleteCmd.command()) // Edit projectEditCmd := cmdProjectEdit{global: c.global, project: c} cmd.AddCommand(projectEditCmd.command()) // Get projectGetCmd := cmdProjectGet{global: c.global, project: c} cmd.AddCommand(projectGetCmd.command()) // List projectListCmd := cmdProjectList{global: c.global, project: c} cmd.AddCommand(projectListCmd.command()) // Rename projectRenameCmd := cmdProjectRename{global: c.global, project: c} cmd.AddCommand(projectRenameCmd.command()) // Set projectSetCmd := cmdProjectSet{global: c.global, project: c} cmd.AddCommand(projectSetCmd.command()) // Unset projectUnsetCmd := cmdProjectUnset{global: c.global, project: c, projectSet: &projectSetCmd} cmd.AddCommand(projectUnsetCmd.command()) // Show projectShowCmd := cmdProjectShow{global: c.global, project: c} cmd.AddCommand(projectShowCmd.command()) // Info projectGetInfo := cmdProjectInfo{global: c.global, project: c} cmd.AddCommand(projectGetInfo.command()) // Set default projectSwitchCmd := cmdProjectSwitch{global: c.global, project: c} cmd.AddCommand(projectSwitchCmd.command()) // Get current project projectGetCurrentCmd := cmdProjectGetCurrent{global: c.global, project: c} cmd.AddCommand(projectGetCurrentCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Create. type cmdProjectCreate struct { global *cmdGlobal project *cmdProject flagConfig []string flagDescription string } var cmdProjectCreateUsage = u.Usage{u.NewName(u.Project).Remote()} func (c *cmdProjectCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdProjectCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create projects") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Create projects`)) cmd.Example = cli.FormatSection("", i18n.G(`incus project create p1 Create a project named p1 incus project create p1 < config.yaml Create a project named p1 with configuration from config.yaml`)) cli.AddStringArrayFlag(cmd.Flags(), &c.flagConfig, "config|c", i18n.G("Config key/value to apply to the new project")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Project description")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProjectCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProjectCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer projectName := parsed[0].RemoteObject.String var stdinData api.ProjectPut // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } err = loader.Load(&stdinData) if err != nil && !errors.Is(err, io.EOF) { return err } } // Create the project project := api.ProjectsPost{} project.Name = projectName project.ProjectPut = stdinData if project.Config == nil { project.Config = map[string]string{} for _, entry := range c.flagConfig { key, value, found := strings.Cut(entry, "=") if !found { return fmt.Errorf(i18n.G("Bad key=value pair: %q"), entry) } project.Config[key] = value } } if c.flagDescription != "" { project.Description = c.flagDescription } err = d.CreateProject(project) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Project %s created")+"\n", formatRemote(c.global.conf, parsed[0])) } return nil } // Delete. type cmdProjectDelete struct { global *cmdGlobal project *cmdProject flagForce bool } var cmdProjectDeleteUsage = u.Usage{u.Project.Remote().List(1)} func (c *cmdProjectDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdProjectDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete projects") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete projects`)) cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G("Force delete the project and everything it contains.")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpProjects(toComplete) } return cmd } func (c *cmdProjectDelete) promptConfirmation(p *u.Parsed) error { reader := bufio.NewReader(os.Stdin) fmt.Printf(i18n.G("Remove %s and everything it contains (instances, images, volumes, networks, ...) (yes/no): "), formatRemote(c.global.conf, p)) input, _ := reader.ReadString('\n') input = strings.TrimSuffix(input, "\n") if !slices.Contains([]string{i18n.G("yes")}, strings.ToLower(input)) { return errors.New(i18n.G("User aborted delete operation")) } return nil } func (c *cmdProjectDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProjectDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } var errs []error for _, p := range parsed[0].List { d := p.RemoteServer remoteName := p.RemoteName projectName := p.RemoteObject.String err := func() error { if c.flagForce { err := c.promptConfirmation(p) if err != nil { return err } err = d.DeleteProjectForce(projectName) if err != nil { return err } } else { err = d.DeleteProject(projectName) if err != nil { return err } } if !c.global.flagQuiet { fmt.Printf(i18n.G("Project %s deleted")+"\n", formatRemote(c.global.conf, p)) } remote := c.global.conf.Remotes[remoteName] // Switch back to default project if remote.Project == projectName { remote.Project = "" c.global.conf.Remotes[remoteName] = remote return c.global.conf.SaveConfig(c.global.confPath) } return nil }() if err != nil { errs = append(errs, err) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } // Edit. type cmdProjectEdit struct { global *cmdGlobal project *cmdProject } var cmdProjectEditUsage = u.Usage{u.Project.Remote()} func (c *cmdProjectEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdProjectEditUsage...) cmd.Short = i18n.G("Edit project configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Edit project configurations as YAML`)) cmd.Example = cli.FormatSection("", i18n.G( `incus project edit < project.yaml Update a project using the content of project.yaml`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProjects(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProjectEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of the project. ### Any line starting with a '# will be ignored. ### ### A project consists of a set of features and a description. ### ### An example would look like: ### config: ### features.images: "true" ### features.networks: "true" ### features.networks.zones: "true" ### features.profiles: "true" ### features.storage.buckets: "true" ### features.storage.volumes: "true" ### description: My own project ### name: my-project ### ### Note that the name is shown but cannot be changed`) } func (c *cmdProjectEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProjectEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer projectName := parsed[0].RemoteObject.String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } newdata := api.ProjectPut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateProject(projectName, newdata, "") } // Extract the current value project, etag, err := d.GetProject(projectName) if err != nil { return err } data, err := yaml.Dump(&project, yaml.V2) if err != nil { return err } // Spawn the editor content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor newdata := api.ProjectPut{} err = yaml.Load(content, &newdata) if err == nil { err = d.UpdateProject(projectName, newdata, etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Get. type cmdProjectGet struct { global *cmdGlobal project *cmdProject flagIsProperty bool } var cmdProjectGetUsage = u.Usage{u.Project.Remote(), u.Key} func (c *cmdProjectGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdProjectGetUsage...) cmd.Short = i18n.G("Get values for project configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Get values for project configuration keys`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a project property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProjects(toComplete) } if len(args) == 1 { return c.global.cmpProjectConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProjectGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProjectGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer projectName := parsed[0].RemoteObject.String key := parsed[1].String // Get the configuration key project, _, err := d.GetProject(projectName) if err != nil { return err } if c.flagIsProperty { w := project.Writable() res, err := getFieldByJSONTag(&w, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the project %q: %v"), key, formatRemote(c.global.conf, parsed[0]), err) } fmt.Printf("%v\n", res) } else { fmt.Printf("%s\n", project.Config[key]) } return nil } // List. type cmdProjectList struct { global *cmdGlobal project *cmdProject flagFormat string flagColumns string } var cmdProjectListUsage = u.Usage{u.RemoteColonOpt, u.Filter.List(0)} func (c *cmdProjectList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdProjectListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List projects") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List projects The -c option takes a (optionally comma-separated) list of arguments that control which image attributes to output when displaying in table or csv format. Default column layout is: nipvbwzdu Column shorthand chars: n - Project Name i - Images p - Profiles v - Storage Volumes b - Storage Buckets w - Networks z - Network Zones d - Description u - Used By`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultProjectColumns, "", i18n.G("Columns")) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } const defaultProjectColumns = "nipvbwzdu" func (c *cmdProjectList) parseColumns() ([]projectColumn, error) { columnsShorthandMap := map[rune]projectColumn{ 'n': {i18n.G("NAME"), c.projectNameColumnData}, 'i': {i18n.G("IMAGES"), c.imagesColumnData}, 'p': {i18n.G("PROFILES"), c.profilesColumnData}, 'v': {i18n.G("STORAGE VOLUMES"), c.storageVolumesColumnData}, 'b': {i18n.G("STORAGE BUCKETS"), c.storageBucketsColumnData}, 'w': {i18n.G("NETWORKS"), c.networksColumnData}, 'z': {i18n.G("NETWORK ZONES"), c.networkZonesColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 'u': {i18n.G("USED BY"), c.usedByColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []projectColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdProjectList) projectNameColumnData(project api.Project) string { return project.Name } func (c *cmdProjectList) imagesColumnData(project api.Project) string { images := i18n.G("NO") if util.IsTrue(project.Config["features.images"]) { images = i18n.G("YES") } return images } func (c *cmdProjectList) profilesColumnData(project api.Project) string { profiles := i18n.G("NO") if util.IsTrue(project.Config["features.profiles"]) { profiles = i18n.G("YES") } return profiles } func (c *cmdProjectList) storageVolumesColumnData(project api.Project) string { storageVolumes := i18n.G("NO") if util.IsTrue(project.Config["features.storage.volumes"]) { storageVolumes = i18n.G("YES") } return storageVolumes } func (c *cmdProjectList) storageBucketsColumnData(project api.Project) string { storageBuckets := i18n.G("NO") if util.IsTrue(project.Config["features.storage.buckets"]) { storageBuckets = i18n.G("YES") } return storageBuckets } func (c *cmdProjectList) networksColumnData(project api.Project) string { networks := i18n.G("NO") if util.IsTrue(project.Config["features.networks"]) { networks = i18n.G("YES") } return networks } func (c *cmdProjectList) networkZonesColumnData(project api.Project) string { networkZones := i18n.G("NO") if util.IsTrue(project.Config["features.networks.zones"]) { networkZones = i18n.G("YES") } return networkZones } func (c *cmdProjectList) descriptionColumnData(project api.Project) string { return project.Description } func (c *cmdProjectList) usedByColumnData(project api.Project) string { return fmt.Sprintf("%d", len(project.UsedBy)) } func (c *cmdProjectList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProjectListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer filters := prepareProjectServerFilters(parsed[1].StringList, api.Project{}) // List projects projects, err := d.GetProjectsWithFilter(filters) if err != nil { return err } // Get the current project. info, err := d.GetConnectionInfo() if err != nil { return err } columns, err := c.parseColumns() if err != nil { return err } data := [][]string{} for _, project := range projects { line := []string{} for _, column := range columns { if column.Name == i18n.G("NAME") { if project.Name == info.Project { project.Name = fmt.Sprintf("%s (%s)", project.Name, i18n.G("current")) } } line = append(line, column.Data(project)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, projects) } // Rename. type cmdProjectRename struct { global *cmdGlobal project *cmdProject } var cmdProjectRenameUsage = u.Usage{u.Project.Remote(), u.NewName(u.Project)} func (c *cmdProjectRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdProjectRenameUsage...) cmd.Aliases = []string{"mv"} cmd.Short = i18n.G("Rename projects") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Rename projects`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProjects(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProjectRename) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProjectRenameUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer projectName := parsed[0].RemoteObject.String newProjectName := parsed[1].String // Rename the project op, err := d.RenameProject(projectName, api.ProjectPost{Name: newProjectName}) if err != nil { return err } err = op.Wait() if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Project %s renamed to %s")+"\n", formatRemote(c.global.conf, parsed[0]), newProjectName) } return nil } // Set. type cmdProjectSet struct { global *cmdGlobal project *cmdProject flagIsProperty bool } var cmdProjectSetUsage = u.Usage{u.Project.Remote(), u.LegacyKV.List(1)} func (c *cmdProjectSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdProjectSetUsage...) cmd.Short = i18n.G("Set project configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set project configuration keys For backward compatibility, a single configuration key may still be set with: incus project set [:] `)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a project property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProjects(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdProjectSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer projectName := parsed[0].RemoteObject.String keys, err := kvToMap(parsed[1]) if err != nil { return err } // Get the project project, etag, err := d.GetProject(projectName) if err != nil { return err } writable := project.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { maps.Copy(writable.Config, keys) } return d.UpdateProject(projectName, writable, etag) } func (c *cmdProjectSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProjectSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Unset. type cmdProjectUnset struct { global *cmdGlobal project *cmdProject projectSet *cmdProjectSet flagIsProperty bool } var cmdProjectUnsetUsage = u.Usage{u.Project.Remote(), u.Key} func (c *cmdProjectUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdProjectUnsetUsage...) cmd.Short = i18n.G("Unset project configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Unset project configuration keys`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a project property")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProjects(toComplete) } if len(args) == 1 { return c.global.cmpProjectConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProjectUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProjectUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.projectSet.flagIsProperty = c.flagIsProperty return unsetKey(c.projectSet, cmd, parsed) } // Show. type cmdProjectShow struct { global *cmdGlobal project *cmdProject } var cmdProjectShowUsage = u.Usage{u.Project.Remote()} func (c *cmdProjectShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdProjectShowUsage...) cmd.Short = i18n.G("Show project options") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show project options`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProjects(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProjectShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProjectShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer projectName := parsed[0].RemoteObject.String // Show the project project, _, err := d.GetProject(projectName) if err != nil { return err } data, err := yaml.Dump(&project, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Switch project. type cmdProjectSwitch struct { global *cmdGlobal project *cmdProject } var cmdProjectSwitchUsage = u.Usage{u.Project.Remote()} func (c *cmdProjectSwitch) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("switch", cmdProjectSwitchUsage...) cmd.Short = i18n.G("Switch the current project") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Switch the current project`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProjects(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProjectSwitch) run(cmd *cobra.Command, args []string) error { conf := c.global.conf parsed, err := cmdProjectSwitchUsage.Parse(conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer remoteName := parsed[0].RemoteName projectName := parsed[0].RemoteObject.String _, _, err = d.GetProject(projectName) if err != nil { return err } remote := conf.Remotes[remoteName] remote.Project = projectName conf.Remotes[remoteName] = remote return conf.SaveConfig(c.global.confPath) } // Info. type cmdProjectInfo struct { global *cmdGlobal project *cmdProject flagShowAccess bool flagFormat string } var cmdProjectInfoUsage = u.Usage{u.Project.Remote()} func (c *cmdProjectInfo) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("info", cmdProjectInfoUsage...) cmd.Short = i18n.G("Get a summary of resource allocations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Get a summary of resource allocations`)) cli.AddBoolFlag(cmd.Flags(), &c.flagShowAccess, "show-access", i18n.G("Show the instance's access list")) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpProjects(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdProjectInfo) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProjectInfoUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer projectName := parsed[0].RemoteObject.String if c.flagShowAccess { access, err := d.GetProjectAccess(projectName) if err != nil { return err } data, err := yaml.Dump(access, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Get the current allocations projectState, err := d.GetProjectState(projectName) if err != nil { return err } // Render the output byteLimits := []string{"disk", "memory"} data := [][]string{} for k, v := range projectState.Resources { shortKey := strings.SplitN(k, ".", 2)[0] limit := i18n.G("UNLIMITED") if v.Limit >= 0 { if slices.Contains(byteLimits, shortKey) { limit = units.GetByteSizeStringIEC(v.Limit, 2) } else { limit = fmt.Sprintf("%d", v.Limit) } } usage := "" if slices.Contains(byteLimits, shortKey) { usage = units.GetByteSizeStringIEC(v.Usage, 2) } else { usage = fmt.Sprintf("%d", v.Usage) } columnName := strings.ToUpper(k) fields := strings.SplitN(columnName, ".", 2) if len(fields) == 2 { columnName = fmt.Sprintf("%s (%s)", fields[0], fields[1]) } data = append(data, []string{columnName, limit, usage}) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{ i18n.G("RESOURCE"), i18n.G("LIMIT"), i18n.G("USAGE"), } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, projectState) } // Get current project. type cmdProjectGetCurrent struct { global *cmdGlobal project *cmdProject } var cmdProjectGetCurrentUsage = u.Usage{u.RemoteColonOpt} func (c *cmdProjectGetCurrent) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get-current", cmdProjectGetCurrentUsage...) cmd.Short = i18n.G("Show the current project") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show the current project`)) cmd.RunE = c.run return cmd } func (c *cmdProjectGetCurrent) run(cmd *cobra.Command, args []string) error { parsed, err := cmdProjectGetCurrentUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer serverInfo, _, err := d.GetServer() if err != nil { return err } // Print the project name. fmt.Println(serverInfo.Environment.Project) return nil } // prepareProjectServerFilter processes and formats filter criteria // for projects, ensuring they are in a format that the server can interpret. func prepareProjectServerFilters(filters []string, i any) []string { formattedFilters := []string{} for _, filter := range filters { membs := strings.SplitN(filter, "=", 2) key := membs[0] if len(membs) == 1 { regexpValue := key if !strings.Contains(key, "^") && !strings.Contains(key, "$") { regexpValue = "^" + regexpValue + "$" } filter = fmt.Sprintf("name=(%s|^%s.*)", regexpValue, key) } else { firstPart := key if strings.Contains(key, ".") { firstPart = strings.Split(key, ".")[0] } if !structHasField(reflect.TypeOf(i), firstPart) { filter = fmt.Sprintf("config.%s", filter) } } formattedFilters = append(formattedFilters, filter) } return formattedFilters } incus-7.0.0/cmd/incus/publish.go000066400000000000000000000161151517523235500165340ustar00rootroot00000000000000package main import ( "errors" "fmt" "strings" "time" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdPublish struct { global *cmdGlobal flagAliases []string flagCompressionAlgorithm string flagExpiresAt string flagMakePublic bool flagForce bool flagReuse bool flagFormat string } var cmdPublishUsage = u.Usage{u.MakePath(u.Instance, u.Snapshot.Optional()).Remote(), u.RemoteColonOpt, u.LegacyKV.List(0)} func (c *cmdPublish) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("publish", cmdPublishUsage...) cmd.Short = i18n.G("Publish instances as images") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Publish instances as images`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagMakePublic, "public", i18n.G("Make the image public")) cli.AddStringArrayFlag(cmd.Flags(), &c.flagAliases, "alias", i18n.G("New alias to define at target")) cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G("Stop the instance if currently running")) cli.AddStringFlag(cmd.Flags(), &c.flagCompressionAlgorithm, "compression", "", "", i18n.G("Compression algorithm to use (`none` for uncompressed)")) cli.AddStringFlag(cmd.Flags(), &c.flagExpiresAt, "expire", "", "", i18n.G("Image expiration date (format: rfc3339)")) cli.AddBoolFlag(cmd.Flags(), &c.flagReuse, "reuse", i18n.G("If the image alias already exists, delete and create a new one")) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format", "unified", "", i18n.G("Image format")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstancesAndSnapshots(toComplete) } if len(args) == 1 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdPublish) run(cmd *cobra.Command, args []string) error { parsed, err := cmdPublishUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } srcServer := parsed[0].RemoteServer isSnapshot := !parsed[0].RemoteObject.List[1].Skipped objectName := parsed[0].RemoteObject.String dstServer := parsed[1].RemoteServer keys, err := kvToMap(parsed[2]) if err != nil { return err } if !isSnapshot { inst, etag, err := srcServer.GetInstance(objectName) if err != nil { return err } wasRunning := inst.StatusCode != 0 && inst.StatusCode != api.Stopped wasEphemeral := inst.Ephemeral if wasRunning { if !c.flagForce { return errors.New(i18n.G("The instance is currently running. Use --force to have it stopped and restarted")) } if inst.Ephemeral { // Clear the ephemeral flag so the instance can be stopped without being destroyed. inst.Ephemeral = false op, err := srcServer.UpdateInstance(objectName, inst.Writable(), etag) if err != nil { return err } err = op.Wait() if err != nil { return err } } // Stop the instance. req := api.InstanceStatePut{ Action: string(instance.Stop), Timeout: -1, Force: true, } op, err := srcServer.UpdateInstanceState(objectName, req, "") if err != nil { return err } err = op.Wait() if err != nil { return errors.New(i18n.G("Stopping instance failed!")) } // Start the instance back up on exit. defer func() { req.Action = string(instance.Start) op, err = srcServer.UpdateInstanceState(objectName, req, "") if err != nil { return } _ = op.Wait() }() // If we had to clear the ephemeral flag, restore it now. if wasEphemeral { inst, etag, err := srcServer.GetInstance(objectName) if err != nil { return err } inst.Ephemeral = true op, err := srcServer.UpdateInstance(objectName, inst.Writable(), etag) if err != nil { return err } err = op.Wait() if err != nil { return err } } } } // Reformat aliases aliases := []api.ImageAlias{} for _, entry := range c.flagAliases { alias := api.ImageAlias{} alias.Name = entry aliases = append(aliases, alias) } // Create the image req := api.ImagesPost{ Source: &api.ImagesPostSource{ Type: "instance", Name: objectName, }, CompressionAlgorithm: c.flagCompressionAlgorithm, } // We should only set the properties field if there actually are any. // Otherwise we will only delete any existing properties on publish. // This is something which only direct callers of the API are allowed to // do. if len(keys) > 0 { req.Properties = keys } if isSnapshot { req.Source.Type = "snapshot" } else if !srcServer.HasExtension("instances") { req.Source.Type = "container" } if srcServer == dstServer { req.Public = c.flagMakePublic } if c.flagExpiresAt != "" { expiresAt, err := time.Parse(time.RFC3339, c.flagExpiresAt) if err != nil { return fmt.Errorf(i18n.G("Invalid expiration date: %w"), err) } req.ExpiresAt = expiresAt } existingAliases, err := getCommonAliases(dstServer, aliases...) if err != nil { return fmt.Errorf(i18n.G("Error retrieving aliases: %w"), err) } if !c.flagReuse && len(existingAliases) > 0 { names := []string{} for _, alias := range existingAliases { names = append(names, alias.Name) } return fmt.Errorf(i18n.G("Aliases already exists: %s"), strings.Join(names, ", ")) } req.Format = c.flagFormat op, err := srcServer.CreateImage(req, nil) if err != nil { return err } // Watch the background operation progress := cli.ProgressRenderer{ Format: i18n.G("Publishing instance: %s"), Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } // Wait for the copy to complete err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } progress.Done("") opAPI := op.Get() // Grab the fingerprint fingerprint, ok := opAPI.Metadata["fingerprint"].(string) if !ok { return errors.New("Bad fingerprint") } // For remote publish, copy to target now if srcServer != dstServer { defer func() { _, _ = srcServer.DeleteImage(fingerprint) }() // Get the source image image, _, err := srcServer.GetImage(fingerprint) if err != nil { return err } // Image copy arguments args := incus.ImageCopyArgs{ Public: c.flagMakePublic, } // Copy the image to the destination host op, err := dstServer.CopyImage(srcServer, *image, &args) if err != nil { return err } err = op.Wait() if err != nil { return err } } // Delete images if necessary if c.flagReuse { err = deleteImagesByAliases(dstServer, aliases) if err != nil { return err } } err = ensureImageAliases(dstServer, aliases, fingerprint) if err != nil { return err } fmt.Printf(i18n.G("Instance published with fingerprint: %s")+"\n", fingerprint) return nil } incus-7.0.0/cmd/incus/query.go000066400000000000000000000104141517523235500162270ustar00rootroot00000000000000package main import ( "bytes" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "slices" "strings" "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdQuery struct { global *cmdGlobal flagRespWait bool flagRespRaw bool flagAction string flagData string } var cmdQueryUsage = u.Usage{u.Placeholder(i18n.G("API path")).Remote()} func (c *cmdQuery) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("query", cmdQueryUsage...) cmd.Short = i18n.G("Send a raw query to the server") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Send a raw query to the server`)) cmd.Example = cli.FormatSection("", i18n.G( `incus query -X DELETE --wait /1.0/instances/c1 Delete local instance "c1".`)) cmd.Hidden = true cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagRespWait, "wait", i18n.G("Wait for the operation to complete")) cli.AddBoolFlag(cmd.Flags(), &c.flagRespRaw, "raw", i18n.G("Print the raw response")) cli.AddStringFlag(cmd.Flags(), &c.flagAction, "request|X", "GET", "", i18n.G("Action")) cli.AddStringFlag(cmd.Flags(), &c.flagData, "data|d", "", "", i18n.G("Input data")) return cmd } func (c *cmdQuery) pretty(input any) string { pretty := bytes.NewBufferString("") enc := json.NewEncoder(pretty) enc.SetEscapeHTML(false) enc.SetIndent("", "\t") err := enc.Encode(input) if err != nil { return fmt.Sprintf("%v", input) } return pretty.String() } func (c *cmdQuery) run(cmd *cobra.Command, args []string) error { parsed, err := cmdQueryUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer path := parsed[0].RemoteObject.String if c.global.flagProject != "" { return errors.New(i18n.G("--project cannot be used with the query command")) } if !slices.Contains([]string{"GET", "PUT", "POST", "PATCH", "DELETE"}, c.flagAction) { return fmt.Errorf(i18n.G("Action %q isn't supported by this tool"), c.flagAction) } // Validate path if !strings.HasPrefix(path, "/") { return errors.New(i18n.G("Query path must start with /")) } // Guess the encoding of the input var data any err = json.Unmarshal([]byte(c.flagData), &data) if err != nil { data = c.flagData } // Perform the query resp, _, err := d.RawQuery(c.flagAction, path, data, "") if err != nil { var jsonSyntaxError *json.SyntaxError var jsonUnmarshalTypeError *json.UnmarshalTypeError // If not JSON decoding error then fail immediately. if !errors.As(err, &jsonSyntaxError) && !errors.As(err, &jsonUnmarshalTypeError) && err.Error() != "EOF" { if c.flagRespRaw && resp != nil { fmt.Println(c.pretty(resp)) return nil } return err } // If JSON decoding error then try a plain request. cleanErr := err // Get the URL prefix httpInfo, err := d.GetConnectionInfo() if err != nil { return err } // Setup input. var rs io.ReadSeeker if c.flagData != "" { rs = bytes.NewReader([]byte(c.flagData)) } // Setup the request req, err := http.NewRequest(c.flagAction, fmt.Sprintf("%s%s", httpInfo.URL, path), rs) if err != nil { return err } // Set the encoding accordingly req.Header.Set("Content-Type", "plain/text") resp, err := d.DoHTTP(req) if err != nil { return err } if resp.StatusCode != http.StatusOK { return cleanErr } content, err := io.ReadAll(resp.Body) if err != nil { return err } fmt.Print(string(content)) return nil } if c.flagRespWait && resp.Operation != "" { uri, err := url.ParseRequestURI(resp.Operation) if err != nil { return err } resp, _, err = d.RawQuery("GET", fmt.Sprintf("%s/wait?%s", uri.Path, uri.RawQuery), "", "") if err != nil { return err } op := api.Operation{} err = json.Unmarshal(resp.Metadata, &op) if err == nil && op.Err != "" { return errors.New(op.Err) } } if c.flagRespRaw { fmt.Println(c.pretty(resp)) } else if resp.Metadata != nil && string(resp.Metadata) != "{}" { var content any err := json.Unmarshal(resp.Metadata, &content) if err != nil { return err } if content != nil { fmt.Println(c.pretty(content)) } } return nil } incus-7.0.0/cmd/incus/rebuild.go000066400000000000000000000072111517523235500165110ustar00rootroot00000000000000package main import ( "errors" "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) // Rebuild. type cmdRebuild struct { global *cmdGlobal flagEmpty bool flagForce bool } var cmdRebuildUsage = u.Usage{u.Either(u.Flag("empty"), u.RemoteImage), u.Instance.Remote()} func (c *cmdRebuild) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rebuild", cmdRebuildUsage...) cmd.Short = i18n.G("Rebuild instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Wipe the instance root disk and re-initialize with a new image (or empty volume).`)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagEmpty, "empty", i18n.G("Rebuild as an empty instance")) cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G("If an instance is running, stop it and then rebuild it")) return cmd } func (c *cmdRebuild) run(cmd *cobra.Command, args []string) error { conf := c.global.conf parsed, err := cmdRebuildUsage.Parse(conf, cmd, args) if err != nil { return err } d := parsed[1].RemoteServer remoteName := parsed[1].RemoteName instanceName := parsed[1].RemoteObject.String current, _, err := d.GetInstance(instanceName) if err != nil { return err } // If the instance is running, stop it first. if c.flagForce && current.StatusCode == api.Running { req := api.InstanceStatePut{ Action: "stop", Force: true, } // Update the instance. op, err := d.UpdateInstanceState(instanceName, req, "") if err != nil { return err } progress := cli.ProgressRenderer{ Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } } // Base request req := api.InstanceRebuildPost{ Source: api.InstanceSource{}, } if parsed[0].BranchID == 1 { imgRemoteName := parsed[0].List[0].Get(conf.DefaultRemote) imgServer, imgInfo, err := getImgInfo(d, conf, imgRemoteName, remoteName, parsed[0].List[1].String, &req.Source) if err != nil { return err } if conf.Remotes[imgRemoteName].Protocol == "incus" { if imgInfo.Type != "virtual-machine" && current.Type == "virtual-machine" { return errors.New(i18n.G("Asked for a VM but image is of type container")) } } op, err := d.RebuildInstanceFromImage(imgServer, *imgInfo, instanceName, req) if err != nil { return err } progress := cli.ProgressRenderer{ Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } // Wait for operation to finish err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } progress.Done("") } else { req.Source.Type = "none" op, err := d.RebuildInstance(instanceName, req) if err != nil { return err } err = op.Wait() if err != nil { return err } } // If the instance was stopped, start it back up. if c.flagForce && current.StatusCode == api.Running { req := api.InstanceStatePut{ Action: "start", } // Update the instance. op, err := d.UpdateInstanceState(instanceName, req, "") if err != nil { return err } progress := cli.ProgressRenderer{ Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } } return nil } incus-7.0.0/cmd/incus/remote.go000066400000000000000000001046151517523235500163640ustar00rootroot00000000000000package main import ( "bufio" "crypto/sha256" "crypto/tls" "crypto/x509" "encoding/pem" "errors" "fmt" "net" "net/http" "net/url" "os" "runtime" "slices" "sort" "strings" "time" "github.com/golang-jwt/jwt/v5" "github.com/spf13/cobra" "software.sslmate.com/src/go-pkcs12" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/internal/ports" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" config "github.com/lxc/incus/v7/shared/cliconfig" cli "github.com/lxc/incus/v7/shared/cmd" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) type cmdRemote struct { global *cmdGlobal } type remoteColumn struct { Name string Data func(string, config.Remote) string } func (c *cmdRemote) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remote") cmd.Short = i18n.G("Manage the list of remote servers") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage the list of remote servers`)) // Add remoteAddCmd := cmdRemoteAdd{global: c.global, remote: c} cmd.AddCommand(remoteAddCmd.command()) // Generate certificate remoteGenerateCertificateCmd := cmdRemoteGenerateCertificate{global: c.global, remote: c} cmd.AddCommand(remoteGenerateCertificateCmd.command()) // Get default remoteGetDefaultCmd := cmdRemoteGetDefault{global: c.global, remote: c} cmd.AddCommand(remoteGetDefaultCmd.command()) // List remoteListCmd := cmdRemoteList{global: c.global, remote: c} cmd.AddCommand(remoteListCmd.command()) if runtime.GOOS != "windows" { // Proxy remoteProxyCmd := cmdRemoteProxy{global: c.global, remote: c} cmd.AddCommand(remoteProxyCmd.command()) } // Rename remoteRenameCmd := cmdRemoteRename{global: c.global, remote: c} cmd.AddCommand(remoteRenameCmd.command()) // Remove remoteRemoveCmd := cmdRemoteRemove{global: c.global, remote: c} cmd.AddCommand(remoteRemoveCmd.command()) // Set default remoteSwitchCmd := cmdRemoteSwitch{global: c.global, remote: c} cmd.AddCommand(remoteSwitchCmd.command()) // Set URL remoteSetURLCmd := cmdRemoteSetURL{global: c.global, remote: c} cmd.AddCommand(remoteSetURLCmd.command()) // Get client certificate remoteGetClientCertificateCmd := cmdRemoteGetClientCertificate{global: c.global, remote: c} cmd.AddCommand(remoteGetClientCertificateCmd.command()) // Get client token remoteGetClientTokenCmd := cmdRemoteGetClientToken{global: c.global, remote: c} cmd.AddCommand(remoteGetClientTokenCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Add. type cmdRemoteAdd struct { global *cmdGlobal remote *cmdRemote flagAcceptCert bool flagToken string flagPublic bool flagProtocol string flagAuthType string flagProject string flagKeepAlive int flagCredHelper string } var cmdRemoteAddUsage = u.Usage{u.NewName(u.Remote).Optional(), u.EitherPlaceholder(i18n.G("IP"), i18n.G("FQDN"), i18n.G("URL"), i18n.G("token"))} func (c *cmdRemoteAdd) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("add", cmdRemoteAddUsage...) cmd.Short = i18n.G("Add new remote servers") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Add new remote servers URL for remote resources must be HTTPS (https://). Basic authentication can be used when combined with the "simplestreams" protocol: incus remote add some-name https://LOGIN:PASSWORD@example.com/some/path --protocol=simplestreams `)) cmd.RunE = c.run cli.AddBoolFlag(cmd.Flags(), &c.flagAcceptCert, "accept-certificate", i18n.G("Accept certificate")) cli.AddStringFlag(cmd.Flags(), &c.flagToken, "token", "", "", i18n.G("Remote trust token")) cli.AddStringFlag(cmd.Flags(), &c.flagProtocol, "protocol", "incus", "", i18n.G("Server protocol (incus, oci or simplestreams)")) cli.AddStringFlag(cmd.Flags(), &c.flagAuthType, "auth-type", "", "", i18n.G("Server authentication type (tls or oidc)")) cli.AddBoolFlag(cmd.Flags(), &c.flagPublic, "public", i18n.G("Public image server")) cli.AddStringFlag(cmd.Flags(), &c.flagProject, "project", "", "", i18n.G("Project to use for the remote")) cli.AddIntFlag(cmd.Flags(), &c.flagKeepAlive, "keepalive", 0, i18n.G("Maintain remote connection for faster commands")) cli.AddStringFlag(cmd.Flags(), &c.flagCredHelper, "credentials-helper", "", "", i18n.G("Binary helper for retrieving credentials")) return cmd } func (c *cmdRemoteAdd) findProject(d incus.InstanceServer, project string) (string, error) { if project == "" { // Check if we can pull a list of projects. if d.HasExtension("projects") { // Retrieve the allowed projects. names, err := d.GetProjectNames() if err != nil { return "", err } if len(names) == 0 { // If no allowed projects, just keep it to the default. return "", nil } else if len(names) == 1 { // If only a single project, use that. return names[0], nil } // Deal with multiple projects. if slices.Contains(names, api.ProjectDefaultName) { // If we have access to the default project, use it. return "", nil } // Let's ask the user. fmt.Println(i18n.G("Available projects:")) for _, name := range names { fmt.Println(" - " + name) } return c.global.asker.AskChoice(i18n.G("Name of the project to use for this remote:")+" ", names, "") } return "", nil } _, _, err := d.GetProject(project) if err != nil { return "", err } return project, nil } func (c *cmdRemoteAdd) runToken(server string, token string, rawToken *api.CertificateAddToken) error { conf := c.global.conf if !conf.HasClientCertificate() { fmt.Fprint(os.Stderr, i18n.G("Generating a client certificate. This may take a minute...")+"\n") err := conf.GenerateClientCertificate() if err != nil { return err } } for _, addr := range rawToken.Addresses { addr = fmt.Sprintf("https://%s", addr) err := c.addRemoteFromToken(addr, server, token, rawToken.Fingerprint) if err != nil { if api.StatusErrorCheck(err, http.StatusServiceUnavailable) { continue } return err } return nil } fmt.Println(i18n.G("All server addresses are unavailable")) fmt.Print(i18n.G("Please provide an alternate server address (empty to abort):") + " ") buf := bufio.NewReader(os.Stdin) line, _, err := buf.ReadLine() if err != nil { return err } if len(line) == 0 { return errors.New(i18n.G("Failed to add remote")) } err = c.addRemoteFromToken(string(line), server, token, rawToken.Fingerprint) if err != nil { return err } return nil } func (c *cmdRemoteAdd) addRemoteFromToken(addr string, server string, token string, fingerprint string) error { conf := c.global.conf var certificate *x509.Certificate var err error conf.Remotes[server] = config.Remote{ Addr: addr, Protocol: c.flagProtocol, AuthType: c.flagAuthType, KeepAlive: c.flagKeepAlive, } _, err = conf.GetInstanceServer(server) if err != nil { certificate, err = localtls.GetRemoteCertificate(addr, c.global.conf.UserAgent) if err != nil { return api.StatusErrorf(http.StatusServiceUnavailable, i18n.G("Unavailable remote server")+": %v", err) } certDigest := localtls.CertFingerprint(certificate) if fingerprint != certDigest { return fmt.Errorf(i18n.G("Certificate fingerprint mismatch between certificate token and server %q"), addr) } dnam := conf.ConfigPath("servercerts") err := os.MkdirAll(dnam, 0o750) if err != nil { return errors.New(i18n.G("Could not create server cert dir")) } certf := conf.ServerCertPath(server) certOut, err := os.Create(certf) if err != nil { return fmt.Errorf(i18n.G("Failed to create %q: %w"), certf, err) } err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: certificate.Raw}) if err != nil { return fmt.Errorf(i18n.G("Failed to write server cert file %q: %w"), certf, err) } err = certOut.Close() if err != nil { return fmt.Errorf(i18n.G("Failed to close server cert file %q: %w"), certf, err) } } d, err := conf.GetInstanceServer(server) if err != nil { return api.StatusErrorf(http.StatusServiceUnavailable, i18n.G("Unavailable remote server")+": %v", err) } req := api.CertificatesPost{ TrustToken: token, } err = d.CreateCertificate(req) if err != nil { return fmt.Errorf(i18n.G("Failed to create certificate: %w"), err) } // Handle project. remote := conf.Remotes[server] project, err := c.findProject(d, c.flagProject) if err != nil { return fmt.Errorf(i18n.G("Failed to find project: %w"), err) } remote.Project = project conf.Remotes[server] = remote return conf.SaveConfig(c.global.confPath) } func (c *cmdRemoteAdd) run(cmd *cobra.Command, args []string) error { conf := c.global.conf // Do NOT blindly copy the following parsing line; it performs right-to-left parsing, which in // most cases is NOT what you want. parsed, err := cmdRemoteAddUsage.Parse(conf, cmd, args, true) if err != nil { return err } addr := parsed[1].String server := parsed[0].Get(addr) // Validate the server name. if strings.Contains(server, ":") { return errors.New(i18n.G("Remote names may not contain colons")) } // Check for existing remote remote, ok := conf.Remotes[server] if ok { return fmt.Errorf(i18n.G("Remote %s exists as <%s>"), server, remote.Addr) } // Parse the URL var rScheme string var rHost string var rPort string // Initialize the remotes list if needed if conf.Remotes == nil { conf.Remotes = map[string]config.Remote{} } rawToken, err := localtls.CertificateTokenDecode(addr) if err == nil { return c.runToken(server, addr, rawToken) } // Complex remote URL parsing remoteURL, err := url.Parse(addr) if err != nil { remoteURL = &url.URL{Host: addr} } // Fast track image servers. if slices.Contains([]string{"oci", "simplestreams"}, c.flagProtocol) { if remoteURL.Scheme != "https" { return errors.New(i18n.G("Only https URLs are supported for oci and simplestreams")) } conf.Remotes[server] = config.Remote{ Addr: addr, Public: true, Protocol: c.flagProtocol, KeepAlive: c.flagKeepAlive, CredHelper: c.flagCredHelper, } return conf.SaveConfig(c.global.confPath) } else if c.flagProtocol != "incus" { return fmt.Errorf(i18n.G("Invalid protocol: %s"), c.flagProtocol) } // Fix broken URL parser if !strings.Contains(addr, "://") && remoteURL.Scheme != "" && remoteURL.Scheme != "unix" && remoteURL.Host == "" { remoteURL.Host = addr remoteURL.Scheme = "" } if remoteURL.Scheme != "" { if remoteURL.Scheme != "unix" && remoteURL.Scheme != "https" { return fmt.Errorf(i18n.G("Invalid URL scheme \"%s\" in \"%s\""), remoteURL.Scheme, addr) } rScheme = remoteURL.Scheme } else if addr[0] == '/' { rScheme = "unix" } else { if !internalUtil.IsUnixSocket(addr) { rScheme = "https" } else { rScheme = "unix" } } if remoteURL.Host != "" { rHost = remoteURL.Host } else { rHost = addr } host, port, err := net.SplitHostPort(rHost) if err == nil { rHost = host rPort = port } else { rPort = fmt.Sprintf("%d", ports.HTTPSDefaultPort) } if rScheme == "unix" { rHost = strings.TrimPrefix(strings.TrimPrefix(addr, "unix:"), "//") rPort = "" } if strings.Contains(rHost, ":") && !strings.HasPrefix(rHost, "[") { rHost = fmt.Sprintf("[%s]", rHost) } if rPort != "" { addr = rScheme + "://" + rHost + ":" + rPort } else { addr = rScheme + "://" + rHost } // Finally, actually add the remote, almost... If the remote is a private // HTTPS server then we need to ensure we have a client certificate before // adding the remote server. if rScheme != "unix" && !c.flagPublic && (c.flagAuthType == api.AuthenticationMethodTLS || c.flagAuthType == "") { if !conf.HasClientCertificate() { fmt.Fprint(os.Stderr, i18n.G("Generating a client certificate. This may take a minute...")+"\n") err = conf.GenerateClientCertificate() if err != nil { return err } } } conf.Remotes[server] = config.Remote{ Addr: addr, Protocol: c.flagProtocol, AuthType: c.flagAuthType, KeepAlive: c.flagKeepAlive, } // Attempt to connect var d incus.ImageServer if c.flagPublic { d, err = conf.GetImageServer(server) } else { d, err = conf.GetInstanceServer(server) } // Handle Unix socket connections if strings.HasPrefix(addr, "unix:") { if err != nil { return err } remote := conf.Remotes[server] remote.AuthType = api.AuthenticationMethodTLS // Handle project. project, err := c.findProject(d.(incus.InstanceServer), c.flagProject) if err != nil { return err } remote.Project = project conf.Remotes[server] = remote return conf.SaveConfig(c.global.confPath) } // Check if the system CA worked for the TLS connection var certificate *x509.Certificate if err != nil { // Failed to connect using the system CA, so retrieve the remote certificate certificate, err = localtls.GetRemoteCertificate(addr, c.global.conf.UserAgent) if err != nil { return err } } // Handle certificate prompt if certificate != nil { if !c.flagAcceptCert { digest := localtls.CertFingerprint(certificate) fmt.Printf(i18n.G("Certificate fingerprint: %s")+"\n", digest) fmt.Print(i18n.G("ok (y/n/[fingerprint])?") + " ") buf := bufio.NewReader(os.Stdin) line, _, err := buf.ReadLine() if err != nil { return err } if string(line) != digest { if len(line) < 1 || strings.ToLower(string(line[0])) == i18n.G("n") { return errors.New(i18n.G("Server certificate NACKed by user")) } else if strings.ToLower(string(line[0])) != i18n.G("y") { return errors.New(i18n.G("Please type 'y', 'n' or the fingerprint:")) } } } dnam := conf.ConfigPath("servercerts") err := os.MkdirAll(dnam, 0o750) if err != nil { return errors.New(i18n.G("Could not create server cert dir")) } certf := conf.ServerCertPath(server) certOut, err := os.Create(certf) if err != nil { return err } err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: certificate.Raw}) if err != nil { return fmt.Errorf(i18n.G("Could not write server cert file %q: %w"), certf, err) } err = certOut.Close() if err != nil { return fmt.Errorf(i18n.G("Could not close server cert file %q: %w"), certf, err) } // Setup a new connection, this time with the remote certificate if c.flagPublic { d, err = conf.GetImageServer(server) } else { d, err = conf.GetInstanceServer(server) } if err != nil { return err } } // Handle public remotes if c.flagPublic { conf.Remotes[server] = config.Remote{ Addr: addr, Public: true, KeepAlive: c.flagKeepAlive, } return conf.SaveConfig(c.global.confPath) } // Get server information srv, _, err := d.(incus.InstanceServer).GetServer() if err != nil { return err } // If not specified, the preferred order of authentication is 1) OIDC 2) TLS. if c.flagAuthType == "" { if !srv.Public && slices.Contains(srv.AuthMethods, api.AuthenticationMethodOIDC) { c.flagAuthType = api.AuthenticationMethodOIDC } else { c.flagAuthType = api.AuthenticationMethodTLS } if slices.Contains([]string{api.AuthenticationMethodOIDC}, c.flagAuthType) { // Update the remote configuration remote := conf.Remotes[server] remote.AuthType = c.flagAuthType conf.Remotes[server] = remote // Re-setup the client d, err = conf.GetInstanceServer(server) if err != nil { return err } d.(incus.InstanceServer).RequireAuthenticated(false) srv, _, err = d.(incus.InstanceServer).GetServer() if err != nil { return err } } else { // Update the remote configuration remote := conf.Remotes[server] remote.AuthType = c.flagAuthType conf.Remotes[server] = remote } } if !srv.Public && !slices.Contains(srv.AuthMethods, c.flagAuthType) { return fmt.Errorf(i18n.G("Authentication type '%s' not supported by server"), c.flagAuthType) } // Detect public remotes if srv.Public { conf.Remotes[server] = config.Remote{Addr: addr, Public: true, KeepAlive: c.flagKeepAlive} return conf.SaveConfig(c.global.confPath) } // Check if additional authentication is required. if srv.Auth != "trusted" { if c.flagAuthType == api.AuthenticationMethodTLS { // Prompt for trust token if c.flagToken == "" { c.flagToken, err = c.global.asker.AskString(fmt.Sprintf(i18n.G("Trust token for %s: "), server), "", nil) if err != nil { return err } } // Add client certificate to trust store req := api.CertificatesPost{ TrustToken: c.flagToken, } req.Type = api.CertificateTypeClient err = d.(incus.InstanceServer).CreateCertificate(req) if err != nil { return err } } else { d.(incus.InstanceServer).RequireAuthenticated(true) } // And check if trusted now srv, _, err = d.(incus.InstanceServer).GetServer() if err != nil { return err } if srv.Auth != "trusted" { return errors.New(i18n.G("Server doesn't trust us after authentication")) } if c.flagAuthType == api.AuthenticationMethodTLS { fmt.Println(i18n.G("Client certificate now trusted by server:"), server) } } // Handle project. remote = conf.Remotes[server] project, err := c.findProject(d.(incus.InstanceServer), c.flagProject) if err != nil { return err } remote.Project = project conf.Remotes[server] = remote return conf.SaveConfig(c.global.confPath) } // Generate certificate. type cmdRemoteGenerateCertificate struct { global *cmdGlobal remote *cmdRemote } var cmdRemoteGenerateCertificateUsage = u.Usage{} func (c *cmdRemoteGenerateCertificate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("generate-certificate", cmdRemoteGenerateCertificateUsage...) cmd.Short = i18n.G("Generate the client certificate") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Manually trigger the generation of a client certificate`)) cmd.RunE = c.run return cmd } func (c *cmdRemoteGenerateCertificate) run(cmd *cobra.Command, args []string) error { conf := c.global.conf _, err := cmdRemoteGenerateCertificateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } // Check if we already have a certificate. if conf.HasClientCertificate() { return errors.New(i18n.G("A client certificate is already present")) } // Generate the certificate. if !c.global.flagQuiet { fmt.Fprint(os.Stderr, i18n.G("Generating a client certificate. This may take a minute...")+"\n") } err = conf.GenerateClientCertificate() if err != nil { return err } return nil } // Get default. type cmdRemoteGetDefault struct { global *cmdGlobal remote *cmdRemote } var cmdRemoteGetDefaultUsage = u.Usage{} func (c *cmdRemoteGetDefault) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get-default", cmdRemoteGetDefaultUsage...) cmd.Short = i18n.G("Show the default remote") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show the default remote`)) cmd.RunE = c.run return cmd } func (c *cmdRemoteGetDefault) run(cmd *cobra.Command, args []string) error { _, err := cmdRemoteGetDefaultUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } // Show the default remote fmt.Println(c.global.conf.DefaultRemote) return nil } // Get client certificate. type cmdRemoteGetClientCertificate struct { global *cmdGlobal remote *cmdRemote flagFormat string } var cmdRemoteGetClientCertificateUsage = u.Usage{u.Target(u.File).Optional()} func (c *cmdRemoteGetClientCertificate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get-client-certificate", cmdRemoteGetClientCertificateUsage...) cmd.Short = i18n.G("Print or retrieve the client certificate used by this Incus client") cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", "pem", "", i18n.G("Format (pem|pfx)")) return cmd } func (c *cmdRemoteGetClientCertificate) run(cmd *cobra.Command, args []string) error { conf := c.global.conf parsed, err := cmdRemoteGetClientCertificateUsage.Parse(conf, cmd, args) if err != nil { return err } targetFile := parsed[0].String hasTargetFile := !parsed[0].Skipped if !slices.Contains([]string{"pem", "pfx"}, c.flagFormat) { return fmt.Errorf(i18n.G("Invalid certificate format %q"), c.flagFormat) } if c.flagFormat == "pfx" && !hasTargetFile { return errors.New("PFX export requires a filename") } // Check if we need to generate a new certificate. if !conf.HasClientCertificate() { if !c.global.flagQuiet { fmt.Fprint(os.Stderr, i18n.G("Generating a client certificate. This may take a minute...")+"\n") } err = conf.GenerateClientCertificate() if err != nil { return err } } // Read the certificate. tlsClientCert, tlsClientKey, _, err := conf.GetClientCertificate(conf.DefaultRemote) if err != nil { return fmt.Errorf("Failed to get certificate: %w", err) } if hasTargetFile { // Create the file. w, err := os.Create(targetFile) if err != nil { return err } defer func() { _ = w.Close() }() switch c.flagFormat { case "pem": _, err = fmt.Fprint(w, tlsClientCert) if err != nil { return err } case "pfx": // Restrict the permission as it includes the key. err = w.Chmod(0o600) if err != nil { return err } // Get a password. password := c.global.asker.AskPasswordOnce(fmt.Sprintf(i18n.G("Password for %s: "), targetFile)) if err != nil { return err } // Load the cert and key. cert, err := tls.X509KeyPair([]byte(tlsClientCert), []byte(tlsClientKey)) if err != nil { return err } // Get the PKCS12. pfx, err := pkcs12.Modern2023.Encode(cert.PrivateKey, cert.Leaf, nil, password) if err != nil { return err } _, err = fmt.Fprint(w, string(pfx)) if err != nil { return err } } return nil } fmt.Print(tlsClientCert) return nil } type cmdRemoteGetClientToken struct { global *cmdGlobal remote *cmdRemote } var cmdRemoteGetClientTokenUsage = u.Usage{u.Expiry} func (c *cmdRemoteGetClientToken) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get-client-token", cmdRemoteGetClientTokenUsage...) cmd.Short = i18n.G("Generate a client token derived from the client certificate") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Generate a client trust token derived from the existing client certificate and private key. This is useful for remote authentication workflows where a token is passed to another Incus server.`)) cmd.RunE = c.run return cmd } func (c *cmdRemoteGetClientToken) run(cmd *cobra.Command, args []string) error { conf := c.global.conf parsed, err := cmdRemoteGetClientTokenUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } // Parse the expiry. expiry, err := time.ParseDuration(parsed[0].String) if err != nil { return err } // Check if we need to generate a new certificate. if !conf.HasClientCertificate() { if !c.global.flagQuiet { fmt.Fprint(os.Stderr, i18n.G("Generating a client certificate. This may take a minute...")+"\n") } err = conf.GenerateClientCertificate() if err != nil { return err } } // Read the certificate. tlsClientCert, tlsClientKey, _, err := conf.GetClientCertificate("") if err != nil { return fmt.Errorf("Failed to get certificate: %w", err) } keypair, err := tls.X509KeyPair([]byte(tlsClientCert), []byte(tlsClientKey)) if err != nil { return err } // Use SHA-256 fingerprint of the first cert in the chain. fingerprint := sha256.Sum256(keypair.Certificate[0]) subject := fmt.Sprintf("%x", fingerprint) now := time.Now() claims := jwt.RegisteredClaims{ Subject: subject, IssuedAt: jwt.NewNumericDate(now), NotBefore: jwt.NewNumericDate(now), ExpiresAt: jwt.NewNumericDate(now.Add(expiry)), } // Trying signing with both ES384 and RS256. for _, alg := range []jwt.SigningMethod{jwt.SigningMethodES384, jwt.SigningMethodRS256} { token := jwt.NewWithClaims(alg, claims) tokenStr, err := token.SignedString(keypair.PrivateKey) if err == nil { fmt.Println(tokenStr) return nil } } return errors.New("Unable to sign JWT with available key algorithms") } // List. type cmdRemoteList struct { global *cmdGlobal remote *cmdRemote flagFormat string flagColumns string } var cmdRemoteListUsage = u.Usage{} func (c *cmdRemoteList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdRemoteListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List the available remotes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List the available remotes Default column layout: nupaPsg == Columns == The -c option takes a comma separated list of arguments that control which remote attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: n - Name u - URL p - Protocol a - Auth Type P - Public s - Static g - Global`)) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultRemoteColumns, "", i18n.G("Columns")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } return cmd } const defaultRemoteColumns = "nupaPsg" func (c *cmdRemoteList) parseColumns() ([]remoteColumn, error) { columnsShorthandMap := map[rune]remoteColumn{ 'n': {i18n.G("NAME"), c.remoteNameColumnData}, 'u': {i18n.G("URL"), c.addrColumnData}, 'p': {i18n.G("PROTOCOL"), c.protocolColumnData}, 'a': {i18n.G("AUTH TYPE"), c.authTypeColumnData}, 'P': {i18n.G("PUBLIC"), c.publicColumnData}, 's': {i18n.G("STATIC"), c.staticColumnData}, 'g': {i18n.G("GLOBAL"), c.globalColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []remoteColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdRemoteList) remoteNameColumnData(name string, _ config.Remote) string { conf := c.global.conf strName := name if name == conf.DefaultRemote { strName = fmt.Sprintf("%s (%s)", name, i18n.G("current")) } return strName } func (c *cmdRemoteList) addrColumnData(_ string, rc config.Remote) string { return rc.Addr } func (c *cmdRemoteList) protocolColumnData(_ string, rc config.Remote) string { return rc.Protocol } func (c *cmdRemoteList) authTypeColumnData(_ string, rc config.Remote) string { if rc.AuthType == "" { if strings.HasPrefix(rc.Addr, "unix:") { rc.AuthType = "file access" } else if rc.Protocol != "incus" { rc.AuthType = "none" } else { rc.AuthType = api.AuthenticationMethodTLS } } return rc.AuthType } func (c *cmdRemoteList) publicColumnData(_ string, rc config.Remote) string { strPublic := i18n.G("NO") if rc.Public { strPublic = i18n.G("YES") } return strPublic } func (c *cmdRemoteList) staticColumnData(_ string, rc config.Remote) string { strStatic := i18n.G("NO") if rc.Static { strStatic = i18n.G("YES") } return strStatic } func (c *cmdRemoteList) globalColumnData(_ string, rc config.Remote) string { strGlobal := i18n.G("NO") if rc.Global { strGlobal = i18n.G("YES") } return strGlobal } func (c *cmdRemoteList) run(cmd *cobra.Command, args []string) error { conf := c.global.conf _, err := cmdRemoteListUsage.Parse(conf, cmd, args) if err != nil { return err } columns, err := c.parseColumns() if err != nil { return err } // List the remotes data := [][]string{} for name, rc := range conf.Remotes { line := []string{} for _, column := range columns { line = append(line, column.Data(name, rc)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, conf.Remotes) } // Rename. type cmdRemoteRename struct { global *cmdGlobal remote *cmdRemote } var cmdRemoteRenameUsage = u.Usage{u.Remote, u.NewName(u.Remote)} func (c *cmdRemoteRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdRemoteRenameUsage...) cmd.Aliases = []string{"mv"} cmd.Short = i18n.G("Rename remotes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Rename remotes`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemoteNames() } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdRemoteRename) run(cmd *cobra.Command, args []string) error { conf := c.global.conf parsed, err := cmdRemoteRenameUsage.Parse(conf, cmd, args) if err != nil { return err } remoteName := parsed[0].String newRemoteName := parsed[1].String // Rename the remote rc, ok := conf.Remotes[remoteName] if !ok { return fmt.Errorf(i18n.G("Remote %s doesn't exist"), remoteName) } if rc.Static { return fmt.Errorf(i18n.G("Remote %s is static and cannot be modified"), remoteName) } _, ok = conf.Remotes[newRemoteName] if ok { return fmt.Errorf(i18n.G("Remote %s already exists"), newRemoteName) } // Rename the certificate file oldPath := conf.ServerCertPath(remoteName) newPath := conf.ServerCertPath(newRemoteName) if util.PathExists(oldPath) { if conf.Remotes[remoteName].Global { err := conf.CopyGlobalCert(remoteName, newRemoteName) if err != nil { return err } } else { err := os.Rename(oldPath, newPath) if err != nil { return err } } } rc.Global = false conf.Remotes[newRemoteName] = rc delete(conf.Remotes, remoteName) if conf.DefaultRemote == remoteName { conf.DefaultRemote = newRemoteName } return conf.SaveConfig(c.global.confPath) } // Remove. type cmdRemoteRemove struct { global *cmdGlobal remote *cmdRemote } var cmdRemoteRemoveUsage = u.Usage{u.Remote} func (c *cmdRemoteRemove) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("remove", cmdRemoteRemoveUsage...) cmd.Aliases = []string{"delete", "rm"} cmd.Short = i18n.G("Remove remotes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Remove remotes`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemoteNames() } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdRemoteRemove) run(cmd *cobra.Command, args []string) error { conf := c.global.conf parsed, err := cmdRemoteRemoveUsage.Parse(conf, cmd, args) if err != nil { return err } remoteName := parsed[0].String // Remove the remote rc, ok := conf.Remotes[remoteName] if !ok { return fmt.Errorf(i18n.G("Remote %s doesn't exist"), remoteName) } if rc.Static { return fmt.Errorf(i18n.G("Remote %s is static and cannot be modified"), remoteName) } if rc.Global { return fmt.Errorf(i18n.G("Remote %s is global and cannot be removed"), remoteName) } if conf.DefaultRemote == remoteName { return errors.New(i18n.G("Can't remove the default remote")) } delete(conf.Remotes, remoteName) _ = os.Remove(conf.ServerCertPath(remoteName)) _ = os.Remove(conf.CookiesPath(remoteName)) _ = os.Remove(conf.OIDCTokenPath(remoteName)) return conf.SaveConfig(c.global.confPath) } // Set default. type cmdRemoteSwitch struct { global *cmdGlobal remote *cmdRemote } var cmdRemoteSwitchUsage = u.Usage{u.Remote} func (c *cmdRemoteSwitch) command() *cobra.Command { cmd := &cobra.Command{} cmd.Aliases = []string{"set-default"} cmd.Use = cli.U("switch", cmdRemoteSwitchUsage...) cmd.Short = i18n.G("Switch the default remote") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Switch the default remote`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemoteNames() } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdRemoteSwitch) run(cmd *cobra.Command, args []string) error { conf := c.global.conf parsed, err := cmdRemoteSwitchUsage.Parse(conf, cmd, args) if err != nil { return err } remoteName := parsed[0].String // Set the default remote _, ok := conf.Remotes[remoteName] if !ok { return fmt.Errorf(i18n.G("Remote %s doesn't exist"), remoteName) } conf.DefaultRemote = remoteName return conf.SaveConfig(c.global.confPath) } // Set URL. type cmdRemoteSetURL struct { global *cmdGlobal remote *cmdRemote } var cmdRemoteSetURLUsage = u.Usage{u.Remote, u.URL} func (c *cmdRemoteSetURL) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set-url", cmdRemoteSetURLUsage...) cmd.Short = i18n.G("Set the URL for the remote") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Set the URL for the remote`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemoteNames() } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdRemoteSetURL) run(cmd *cobra.Command, args []string) error { conf := c.global.conf parsed, err := cmdRemoteSetURLUsage.Parse(conf, cmd, args) if err != nil { return err } remoteName := parsed[0].String remoteURL := parsed[1].String // Set the URL rc, ok := conf.Remotes[remoteName] if !ok { return fmt.Errorf(i18n.G("Remote %s doesn't exist"), remoteName) } if rc.Static { return fmt.Errorf(i18n.G("Remote %s is static and cannot be modified"), remoteName) } remote := conf.Remotes[remoteName] if remote.Global { err := conf.CopyGlobalCert(remoteName, remoteName) if err != nil { return err } remote.Global = false conf.Remotes[remoteName] = remote } remote.Addr = remoteURL conf.Remotes[remoteName] = remote return conf.SaveConfig(c.global.confPath) } incus-7.0.0/cmd/incus/remote_unix.go000066400000000000000000000155451517523235500174320ustar00rootroot00000000000000//go:build !windows package main import ( "encoding/json" "errors" "fmt" "net" "net/http" "net/http/httputil" "net/url" "os" "strings" "sync" "time" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdRemoteProxy struct { global *cmdGlobal remote *cmdRemote flagTimeout int } var cmdRemoteProxyUsage = u.Usage{u.Colon(u.Remote), u.Target(u.Placeholder(i18n.G("socket file")))} func (c *cmdRemoteProxy) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("proxy", cmdRemoteProxyUsage...) cmd.Short = i18n.G("Run a local API proxy") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Run a local API proxy for the remote`)) cmd.RunE = c.run cli.AddIntFlag(cmd.Flags(), &c.flagTimeout, "timeout", 0, i18n.G("Proxy timeout (exits when no connections)")) return cmd } func (c *cmdRemoteProxy) run(cmd *cobra.Command, args []string) error { parsed, err := cmdRemoteProxyUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } remoteName := strings.TrimSuffix(parsed[0].String, ":") path := parsed[1].String remote := c.global.conf.Remotes[remoteName] remote.KeepAlive = 0 // Attempt to read stdin for TLS connection details. _ = json.NewDecoder(os.Stdin).Decode(&remote.TLS) c.global.conf.Remotes[remoteName] = remote resources, err := c.global.parseServers(parsed[0].String) if err != nil { return err } s := resources[0].server // Create proxy socket. err = os.Remove(path) if err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("Failed to delete pre-existing unix socket: %w", err) } unixAddr, err := net.ResolveUnixAddr("unix", path) if err != nil { return fmt.Errorf("Unable to resolve unix socket: %w", err) } server, err := net.ListenUnix("unix", unixAddr) if err != nil { return fmt.Errorf("Unable to setup unix socket: %w", err) } err = os.Chmod(path, 0o600) if err != nil { return fmt.Errorf("Unable to set socket permissions: %w", err) } // Get the connection info. info, err := s.GetConnectionInfo() if err != nil { return err } uri, err := url.Parse(info.URL) if err != nil { return err } // Enable keep-alive for proxied connections. httpClient, err := s.GetHTTPClient() if err != nil { return err } httpTransport, ok := httpClient.Transport.(*http.Transport) if ok { httpTransport.DisableKeepAlives = false } // Get server info. api10, api10Etag, err := s.GetServer() if err != nil { return err } // Handle inbound connections. transport := remoteProxyTransport{ s: s, baseURL: uri, } connections := uint64(0) transactions := uint64(0) handler := remoteProxyHandler{ s: s, transport: transport, api10: api10, api10Etag: api10Etag, mu: &sync.RWMutex{}, connections: &connections, transactions: &transactions, } // Handle the timeout. if c.flagTimeout > 0 { go func() { for { time.Sleep(time.Duration(c.flagTimeout) * time.Second) // Check for active connections. handler.mu.RLock() if *handler.connections > 0 { handler.mu.RUnlock() continue } // Look for recent activity oldCount := uint64(*handler.transactions) handler.mu.RUnlock() time.Sleep(5 * time.Second) handler.mu.RLock() if oldCount == *handler.transactions { handler.mu.RUnlock() // Daemon has been inactive for 10s, exit. os.Exit(0) // nolint:revive } handler.mu.RUnlock() } }() } // Start the server. err = http.Serve(server, handler) if err != nil { return err } return nil } type remoteProxyTransport struct { s incus.InstanceServer baseURL *url.URL } // RoundTrip handles an HTTP request. func (t remoteProxyTransport) RoundTrip(r *http.Request) (*http.Response, error) { // Fix the request. r.URL.Scheme = t.baseURL.Scheme r.URL.Host = t.baseURL.Host r.RequestURI = "" resp, err := t.s.DoHTTP(r) if errors.Is(err, incus.ErrOIDCExpired) { // Override the response so the client knows to retry the request. resp.StatusCode = http.StatusUseProxy resp.Status = "Retry the request for OIDC refresh" return resp, nil } return resp, err } type remoteProxyHandler struct { s incus.InstanceServer transport http.RoundTripper url string mu *sync.RWMutex connections *uint64 transactions *uint64 api10 *api.Server api10Etag string token string } func (h remoteProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Increase counters. defer func() { h.mu.Lock() *h.connections-- h.mu.Unlock() }() h.mu.Lock() *h.transactions++ *h.connections++ h.mu.Unlock() // Don't allow cross-origin requests. origin := r.Header.Get("Origin") if origin != "" && origin != h.url { return } // Basic auth. if h.token != "" { // Parse query URL. values, err := url.ParseQuery(r.URL.RawQuery) if err != nil { return } token := values.Get("auth_token") if token != "" { // If a token was passed through the URL, persist it as a cookie. tokenCookie := http.Cookie{ Name: "auth_token", Value: token, Path: "/", Secure: false, HttpOnly: true, SameSite: http.SameSiteStrictMode, } http.SetCookie(w, &tokenCookie) } else { // If not, attempt to pull it from the cookie. cookie, err := r.Cookie("auth_token") if err != nil { // Fail authentication if no cookie can be found. w.WriteHeader(http.StatusUnauthorized) return } token = cookie.Value } // Check the user token against the expected value. if token != h.token { w.WriteHeader(http.StatusUnauthorized) return } } // Handle /1.0 internally (saves a round-trip). if r.RequestURI == "/1.0" || strings.HasPrefix(r.RequestURI, "/1.0?project=") { // Parse query URL. values, err := url.ParseQuery(r.URL.RawQuery) if err != nil { return } // Update project name to match. projectName := values.Get("project") if projectName == "" { projectName = api.ProjectDefaultName } api10 := api.Server(*h.api10) api10.Environment.Project = projectName // Set the request headers. w.Header().Set("Content-Type", "application/json") w.Header().Set("ETag", h.api10Etag) w.WriteHeader(http.StatusOK) // Generate a body from the cached data. serverBody, err := json.Marshal(api10) if err != nil { return } apiResponse := api.Response{ Type: "sync", Status: "success", StatusCode: 200, Metadata: serverBody, } body, err := json.Marshal(apiResponse) if err != nil { return } _, _ = w.Write(body) return } // Forward everything else. proxy := httputil.ReverseProxy{ Transport: h.transport, Director: func(*http.Request) {}, } proxy.ServeHTTP(w, r) } incus-7.0.0/cmd/incus/remote_windows.go000066400000000000000000000003201517523235500201220ustar00rootroot00000000000000//go:build windows package main import ( "github.com/spf13/cobra" ) type cmdRemoteProxy struct { global *cmdGlobal remote *cmdRemote } func (c *cmdRemoteProxy) command() *cobra.Command { return nil } incus-7.0.0/cmd/incus/rename.go000066400000000000000000000024641517523235500163370ustar00rootroot00000000000000package main import ( "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdRename struct { global *cmdGlobal } var cmdRenameUsage = u.Usage{u.Instance.Remote(), u.NewName(u.Instance)} func (c *cmdRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdRenameUsage...) cmd.Short = i18n.G("Rename instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Rename instances`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdRename) run(cmd *cobra.Command, args []string) error { conf := c.global.conf parsed, err := cmdRenameUsage.Parse(conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String newInstanceName := parsed[1].String op, err := d.RenameInstance(instanceName, api.InstancePost{Name: newInstanceName}) if err != nil { return err } return op.Wait() } incus-7.0.0/cmd/incus/snapshot.go000066400000000000000000000402101517523235500167160ustar00rootroot00000000000000package main import ( "bufio" "errors" "fmt" "io" "net/url" "os" "path" "slices" "strings" "time" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" ) type cmdSnapshot struct { global *cmdGlobal } func (c *cmdSnapshot) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("snapshot") cmd.Short = i18n.G("Manage instance snapshots") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage instance snapshots`)) // Create. snapshotCreateCmd := cmdSnapshotCreate{global: c.global, snapshot: c} cmd.AddCommand(snapshotCreateCmd.command()) // Delete. snapshotDeleteCmd := cmdSnapshotDelete{global: c.global, snapshot: c} cmd.AddCommand(snapshotDeleteCmd.command()) // List. snapshotListCmd := cmdSnapshotList{global: c.global, snapshot: c} cmd.AddCommand(snapshotListCmd.command()) // Rename. snapshotRenameCmd := cmdSnapshotRename{global: c.global, snapshot: c} cmd.AddCommand(snapshotRenameCmd.command()) // Restore. snapshotRestoreCmd := cmdSnapshotRestore{global: c.global, snapshot: c} cmd.AddCommand(snapshotRestoreCmd.command()) // Show. snapshotShowCmd := cmdSnapshotShow{global: c.global, snapshot: c} cmd.AddCommand(snapshotShowCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Create. type cmdSnapshotCreate struct { global *cmdGlobal snapshot *cmdSnapshot flagStateful bool flagNoExpiry bool flagExpiry string flagReuse bool } var cmdSnapshotCreateUsage = u.Usage{u.Instance.Remote(), u.NewName(u.Snapshot).Optional()} func (c *cmdSnapshotCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdSnapshotCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create instance snapshot") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Create instance snapshots When --stateful is used, attempt to checkpoint the instance's running state, including process memory state, TCP connections, ...`)) cmd.Example = cli.FormatSection("", i18n.G(`incus snapshot create u1 snap0 Create a snapshot of "u1" called "snap0". incus snapshot create u1 snap0 < config.yaml Create a snapshot of "u1" called "snap0" with the configuration from "config.yaml".`)) cli.AddBoolFlag(cmd.Flags(), &c.flagStateful, "stateful", i18n.G("Whether or not to snapshot the instance's running state")) cli.AddStringFlag(cmd.Flags(), &c.flagExpiry, "expiry", "", "", i18n.G("Expiry date or time span for the new snapshot")) cli.AddBoolFlag(cmd.Flags(), &c.flagNoExpiry, "no-expiry", i18n.G("Ignore any configured auto-expiry for the instance")) cli.AddBoolFlag(cmd.Flags(), &c.flagReuse, "reuse", i18n.G("If the snapshot name already exists, delete and create a new one")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdSnapshotCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdSnapshotCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String hasSnapName := !parsed[1].Skipped snapName := parsed[1].String if c.flagNoExpiry && c.flagExpiry != "" { return errors.New(i18n.G("Can't use both --no-expiry and --expiry")) } // If stdin isn't a terminal, read text from it var stdinData api.InstanceSnapshotPut if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } err = loader.Load(&stdinData) if err != nil && !errors.Is(err, io.EOF) { return err } } if c.flagReuse && hasSnapName { snap, _, _ := d.GetInstanceSnapshot(instanceName, snapName) if snap != nil { op, err := d.DeleteInstanceSnapshot(instanceName, snapName) if err != nil { return err } err = op.Wait() if err != nil { return err } } } req := api.InstanceSnapshotsPost{ Name: snapName, Stateful: c.flagStateful, } if c.flagNoExpiry { req.ExpiresAt = &time.Time{} } else if c.flagExpiry != "" { // Try to parse as a duration. expiry, err := instance.GetExpiry(time.Now(), c.flagExpiry) if err != nil { if !errors.Is(err, instance.ErrInvalidExpiry) { return err } // Fallback to date parsing. expiry, err = time.Parse(dateLayout, c.flagExpiry) if err != nil { return err } } req.ExpiresAt = &expiry } else if !stdinData.ExpiresAt.IsZero() { req.ExpiresAt = &stdinData.ExpiresAt } op, err := d.CreateInstanceSnapshot(instanceName, req) if err != nil { return err } err = op.Wait() if err != nil { return err } opInfo := op.Get() snapshots, ok := opInfo.Resources["instances_snapshots"] if !ok || len(snapshots) == 0 { return errors.New(i18n.G("Didn't get name of new instance snapshot from the server")) } if len(snapshots) == 1 && !hasSnapName { uri, err := url.Parse(snapshots[0]) if err != nil { return err } fmt.Printf(i18n.G("Instance snapshot name is: %s")+"\n", path.Base(uri.Path)) } return nil } // Delete. type cmdSnapshotDelete struct { global *cmdGlobal snapshot *cmdSnapshot flagInteractive bool } var cmdSnapshotDeleteUsage = u.Usage{u.Instance.Remote(), u.Snapshot} func (c *cmdSnapshotDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdSnapshotDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete instance snapshots") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete instance snapshots`)) cli.AddBoolFlag(cmd.Flags(), &c.flagInteractive, "interactive|i", i18n.G("Require user confirmation")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdSnapshotDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdSnapshotDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String snapName := parsed[1].String // Process with deletion. if c.flagInteractive { err := c.promptDelete(parsed[0], snapName) if err != nil { return err } } err = c.doDelete(d, instanceName, snapName) if err != nil { return err } return nil } func (c *cmdSnapshotDelete) promptDelete(p *u.Parsed, name string) error { reader := bufio.NewReader(os.Stdin) fmt.Printf(i18n.G("Remove snapshot %s from %s (yes/no): "), name, formatRemote(c.global.conf, p)) input, _ := reader.ReadString('\n') input = strings.TrimSuffix(input, "\n") if !slices.Contains([]string{i18n.G("yes")}, strings.ToLower(input)) { return errors.New(i18n.G("User aborted delete operation")) } return nil } func (c *cmdSnapshotDelete) doDelete(d incus.InstanceServer, instName string, name string) error { var op incus.Operation var err error // Snapshot delete op, err = d.DeleteInstanceSnapshot(instName, name) if err != nil { return err } return op.Wait() } // List. type cmdSnapshotList struct { global *cmdGlobal snapshot *cmdSnapshot flagFormat string flagColumns string } var cmdSnapshotListUsage = u.Usage{u.Instance.Remote()} type snapshotColumn struct { Name string Data func(api.InstanceSnapshot) string } func (c *cmdSnapshotList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdSnapshotListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List instance snapshots") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List instance snapshots Default column layout: nTEs == Columns == The -c option takes a comma separated list of arguments that control which snapshots attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: n - Name T - Taken At E - Expires At s - Stateful`)) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultSnapshotColumns, "", i18n.G("Columns")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } const defaultSnapshotColumns = "nTEs" func (c *cmdSnapshotList) parseColumns() ([]snapshotColumn, error) { columnsShorthandMap := map[rune]snapshotColumn{ 'n': {i18n.G("NAME"), c.nameColumnData}, 'T': {i18n.G("TAKEN AT"), c.takenAtColumnData}, 'E': {i18n.G("EXPIRES AT"), c.expiresAtColumnData}, 's': {i18n.G("STATEFUL"), c.statefulColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []snapshotColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdSnapshotList) nameColumnData(snapshot api.InstanceSnapshot) string { return snapshot.Name } func (c *cmdSnapshotList) takenAtColumnData(snapshot api.InstanceSnapshot) string { if snapshot.CreatedAt.IsZero() { return " " } return snapshot.CreatedAt.Local().Format(dateLayout) } func (c *cmdSnapshotList) expiresAtColumnData(snapshot api.InstanceSnapshot) string { if snapshot.ExpiresAt.IsZero() { return " " } return snapshot.ExpiresAt.Local().Format(dateLayout) } func (c *cmdSnapshotList) statefulColumnData(snapshot api.InstanceSnapshot) string { strStateful := "NO" if snapshot.Stateful { strStateful = "YES" } return strStateful } func (c *cmdSnapshotList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdSnapshotListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String snapshots, err := d.GetInstanceSnapshots(instanceName) if err != nil { return err } // Parse column flags. columns, err := c.parseColumns() if err != nil { return err } data := [][]string{} for _, snap := range snapshots { line := []string{} for _, column := range columns { line = append(line, column.Data(snap)) } data = append(data, line) } header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, snapshots) } // Rename. type cmdSnapshotRename struct { global *cmdGlobal snapshot *cmdSnapshot } var cmdSnapshotRenameUsage = u.Usage{u.Instance.Remote(), u.Snapshot, u.NewName(u.Snapshot)} func (c *cmdSnapshotRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdSnapshotRenameUsage...) cmd.Short = i18n.G("Rename instance snapshots") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Rename instance snapshots`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } if len(args) == 1 { return c.global.cmpInstanceSnapshots(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdSnapshotRename) run(cmd *cobra.Command, args []string) error { parsed, err := cmdSnapshotRenameUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String snapName := parsed[1].String newSnapName := parsed[2].String // Snapshot rename op, err := d.RenameInstanceSnapshot(instanceName, snapName, api.InstanceSnapshotPost{Name: newSnapName}) if err != nil { return err } return op.Wait() } // Restore. type cmdSnapshotRestore struct { global *cmdGlobal snapshot *cmdSnapshot flagStateful bool flagDiskOnly bool } var cmdSnapshotRestoreUsage = u.Usage{u.Instance.Remote(), u.Snapshot} func (c *cmdSnapshotRestore) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("restore", cmdSnapshotRestoreUsage...) cmd.Short = i18n.G("Restore instance snapshots") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Restore instance from snapshots If --stateful is passed, then the running state will be restored too. If --diskonly is passed, then only the disk will be restored.`)) cmd.Example = cli.FormatSection("", i18n.G( `incus snapshot restore u1 snap0 Restore instance u1 to snapshot snap0`)) cli.AddBoolFlag(cmd.Flags(), &c.flagStateful, "stateful", i18n.G("Whether or not to restore the instance's running state from snapshot (if available)")) cli.AddBoolFlag(cmd.Flags(), &c.flagDiskOnly, "diskonly", i18n.G("Whether or not to restore the instance's disk only")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } if len(args) == 1 { return c.global.cmpInstanceSnapshots(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdSnapshotRestore) run(cmd *cobra.Command, args []string) error { parsed, err := cmdSnapshotRestoreUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String snapName := parsed[1].String req := api.InstancePut{ Restore: instanceName + "/" + snapName, Stateful: c.flagStateful, DiskOnly: c.flagDiskOnly, } // Restore the snapshot op, err := d.UpdateInstance(instanceName, req, "") if err != nil { return err } return op.Wait() } // Show. type cmdSnapshotShow struct { global *cmdGlobal snapshot *cmdSnapshot } var cmdSnapshotShowUsage = u.Usage{u.Instance.Remote(), u.Snapshot} func (c *cmdSnapshotShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdSnapshotShowUsage...) cmd.Short = i18n.G("Show instance snapshot configuration") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Show instance snapshot configuration`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } if len(args) == 1 { return c.global.cmpInstanceSnapshots(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdSnapshotShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdSnapshotShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String snapName := parsed[1].String // Snapshot snap, _, err := d.GetInstanceSnapshot(instanceName, snapName) if err != nil { return err } data, err := yaml.Dump(&snap, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } incus-7.0.0/cmd/incus/storage.go000066400000000000000000000634661517523235500165450ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "maps" "net/url" "os" "reflect" "sort" "strconv" "strings" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" "github.com/lxc/incus/v7/shared/units" ) type cmdStorage struct { global *cmdGlobal flagTarget string } type storageColumn struct { Name string Data func(api.StoragePool) string } func (c *cmdStorage) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("storage") cmd.Short = i18n.G("Manage storage pools and volumes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage storage pools and volumes`)) // Create storageCreateCmd := cmdStorageCreate{global: c.global, storage: c} cmd.AddCommand(storageCreateCmd.command()) // Delete storageDeleteCmd := cmdStorageDelete{global: c.global, storage: c} cmd.AddCommand(storageDeleteCmd.command()) // Edit storageEditCmd := cmdStorageEdit{global: c.global, storage: c} cmd.AddCommand(storageEditCmd.command()) // Get storageGetCmd := cmdStorageGet{global: c.global, storage: c} cmd.AddCommand(storageGetCmd.command()) // Info storageInfoCmd := cmdStorageInfo{global: c.global, storage: c} cmd.AddCommand(storageInfoCmd.command()) // List storageListCmd := cmdStorageList{global: c.global, storage: c} cmd.AddCommand(storageListCmd.command()) // Set storageSetCmd := cmdStorageSet{global: c.global, storage: c} cmd.AddCommand(storageSetCmd.command()) // Show storageShowCmd := cmdStorageShow{global: c.global, storage: c} cmd.AddCommand(storageShowCmd.command()) // Unset storageUnsetCmd := cmdStorageUnset{global: c.global, storage: c, storageSet: &storageSetCmd} cmd.AddCommand(storageUnsetCmd.command()) // Bucket storageBucketCmd := cmdStorageBucket{global: c.global} cmd.AddCommand(storageBucketCmd.command()) // Volume storageVolumeCmd := cmdStorageVolume{global: c.global, storage: c} cmd.AddCommand(storageVolumeCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Create. type cmdStorageCreate struct { global *cmdGlobal storage *cmdStorage flagDescription string } var cmdStorageCreateUsage = u.Usage{u.NewName(u.Pool).Remote(), u.Driver, u.KV.List(0)} func (c *cmdStorageCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdStorageCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create storage pools") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Create storage pools`)) cmd.Example = cli.FormatSection("", i18n.G(`incus storage create s1 dir Create a storage pool s1 incus storage create s1 dir < config.yaml Create a storage pool s1 using the content of config.yaml `)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Storage pool description")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String driver := parsed[1].String keys, err := kvToMap(parsed[2]) if err != nil { return err } // If stdin isn't a terminal, read text from it var stdinData api.StoragePoolPut if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } err = loader.Load(&stdinData) if err != nil && !errors.Is(err, io.EOF) { return err } } // Create the new storage pool entry pool := api.StoragePoolsPost{StoragePoolPut: stdinData} pool.Name = poolName pool.Driver = driver if c.flagDescription != "" { pool.Description = c.flagDescription } if pool.Config == nil { pool.Config = map[string]string{} } maps.Copy(pool.Config, keys) // If a target member was specified the API won't actually create the // pool, but only define it as pending in the database. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } // Create the pool err = d.CreateStoragePool(pool) if err != nil { return err } if !c.global.flagQuiet { if c.storage.flagTarget != "" { fmt.Printf(i18n.G("Storage pool %s pending on member %s")+"\n", formatRemote(c.global.conf, parsed[0]), c.storage.flagTarget) } else { fmt.Printf(i18n.G("Storage pool %s created")+"\n", formatRemote(c.global.conf, parsed[0])) } } return nil } // Delete. type cmdStorageDelete struct { global *cmdGlobal storage *cmdStorage } var cmdStorageDeleteUsage = u.Usage{u.Pool.Remote().List(1)} func (c *cmdStorageDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdStorageDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete storage pools") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete storage pools`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpStoragePools(toComplete) } return cmd } func (c *cmdStorageDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } var errs []error for _, p := range parsed[0].List { d := p.RemoteServer poolName := p.RemoteObject.String // Delete the pool err = d.DeleteStoragePool(poolName) if err != nil { errs = append(errs, err) continue } if !c.global.flagQuiet { fmt.Printf(i18n.G("Storage pool %s deleted")+"\n", formatRemote(c.global.conf, p)) } } if len(errs) > 0 { return errors.Join(errs...) } return nil } // Edit. type cmdStorageEdit struct { global *cmdGlobal storage *cmdStorage } var cmdStorageEditUsage = u.Usage{u.Pool.Remote()} func (c *cmdStorageEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdStorageEditUsage...) cmd.Short = i18n.G("Edit storage pool configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Edit storage pool configurations as YAML`)) cmd.Example = cli.FormatSection("", i18n.G( `incus storage edit [:] < pool.yaml Update a storage pool using the content of pool.yaml.`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of a storage pool. ### Any line starting with a '#' will be ignored. ### ### A storage pool consists of a set of configuration items. ### ### An example would look like: ### name: default ### driver: zfs ### used_by: [] ### config: ### size: "61203283968" ### source: default ### zfs.pool_name: default`) } func (c *cmdStorageEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } newdata := api.StoragePoolPut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateStoragePool(poolName, newdata, "") } // Extract the current value pool, etag, err := d.GetStoragePool(poolName) if err != nil { return err } data, err := yaml.Dump(&pool, yaml.V2) if err != nil { return err } // Spawn the editor content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor newdata := api.StoragePoolPut{} err = yaml.Load(content, &newdata) if err == nil { err = d.UpdateStoragePool(poolName, newdata, etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Get. type cmdStorageGet struct { global *cmdGlobal storage *cmdStorage flagIsProperty bool } var cmdStorageGetUsage = u.Usage{u.Pool.Remote(), u.Key} func (c *cmdStorageGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdStorageGetUsage...) cmd.Short = i18n.G("Get values for storage pool configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Get values for storage pool configuration keys`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a storage property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String key := parsed[1].String // If a target member was specified, we return also member-specific config values. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } // Get the property resp, _, err := d.GetStoragePool(poolName) if err != nil { return err } if c.flagIsProperty { w := resp.Writable() res, err := getFieldByJSONTag(&w, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the storage pool %q: %v"), key, formatRemote(c.global.conf, parsed[0]), err) } fmt.Printf("%v\n", res) } else { v, ok := resp.Config[key] if ok { fmt.Println(v) } } return nil } // Info. type cmdStorageInfo struct { global *cmdGlobal storage *cmdStorage flagBytes bool } var cmdStorageInfoUsage = u.Usage{u.Pool.Remote()} func (c *cmdStorageInfo) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("info", cmdStorageInfoUsage...) cmd.Short = i18n.G("Show useful information about storage pools") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Show useful information about storage pools`)) cli.AddBoolFlag(cmd.Flags(), &c.flagBytes, "bytes", i18n.G("Show the used and free space in bytes")) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageInfo) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageInfoUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String // Targeting if c.storage.flagTarget != "" { if !d.IsClustered() { return errors.New(i18n.G("To use --target, the destination remote must be a cluster")) } d = d.UseTarget(c.storage.flagTarget) } // Get the pool information pool, _, err := d.GetStoragePool(poolName) if err != nil { return err } res, err := d.GetStoragePoolResources(poolName) if err != nil { return err } // Declare the poolinfo map of maps in order to build up the yaml poolinfo := make(map[string]map[string]string) poolusedby := make(map[string]map[string][]string) // Translations usedbystring := i18n.G("used by") infostring := i18n.G("info") namestring := i18n.G("name") driverstring := i18n.G("driver") descriptionstring := i18n.G("description") totalspacestring := i18n.G("total space") spaceusedstring := i18n.G("space used") // Initialize the usedby map poolusedby[usedbystring] = make(map[string][]string) // Build up the usedby map for _, v := range pool.UsedBy { uri, err := url.Parse(v) if err != nil { continue } fields := strings.Split(strings.TrimPrefix(uri.Path, "/1.0/"), "/") fieldsLen := len(fields) entityType := "unrecognized" entityName := uri.Path if fieldsLen > 1 { entityType = fields[0] entityName = fields[1] if fields[fieldsLen-2] == "snapshots" { continue // Skip snapshots as the parent entity will be included once in the list. } if fields[0] == "storage-pools" && fieldsLen > 3 { entityType = fields[2] entityName = fields[3] if entityType == "volumes" && fieldsLen > 4 { entityName = fields[4] } } } var sb strings.Builder var attribs []string sb.WriteString(entityName) // Show info regarding the project and location if present. values := uri.Query() projectName := values.Get("project") if projectName != "" { attribs = append(attribs, fmt.Sprintf("project %q", projectName)) } locationName := values.Get("target") if locationName != "" { attribs = append(attribs, fmt.Sprintf("location %q", locationName)) } if len(attribs) > 0 { sb.WriteString(" (") for i, attrib := range attribs { if i > 0 { sb.WriteString(", ") } sb.WriteString(attrib) } sb.WriteString(")") } poolusedby[usedbystring][entityType] = append(poolusedby[usedbystring][entityType], sb.String()) } // Initialize the info map poolinfo[infostring] = map[string]string{} // Build up the info map poolinfo[infostring][namestring] = pool.Name poolinfo[infostring][driverstring] = pool.Driver poolinfo[infostring][descriptionstring] = pool.Description if c.flagBytes { poolinfo[infostring][totalspacestring] = strconv.FormatUint(res.Space.Total, 10) poolinfo[infostring][spaceusedstring] = strconv.FormatUint(res.Space.Used, 10) } else { poolinfo[infostring][totalspacestring] = units.GetByteSizeStringIEC(int64(res.Space.Total), 2) poolinfo[infostring][spaceusedstring] = units.GetByteSizeStringIEC(int64(res.Space.Used), 2) } poolinfodata, err := yaml.Dump(poolinfo, yaml.V2) if err != nil { return err } poolusedbydata, err := yaml.Dump(poolusedby, yaml.V2) if err != nil { return err } fmt.Printf("%s", poolinfodata) fmt.Printf("%s", poolusedbydata) return nil } // List. type cmdStorageList struct { global *cmdGlobal storage *cmdStorage flagFormat string flagColumns string } var cmdStorageListUsage = u.Usage{u.RemoteColonOpt, u.Filter.List(0)} func (c *cmdStorageList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdStorageListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List available storage pools") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List available storage pools Default column layout: nDdus == Columns == The -c option takes a comma separated list of arguments that control which storage pools attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: n - Name D - Driver d - Description S - Source u - used by s - state`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultStorageColumns, "", i18n.G("Columns")) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpRemotes(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } const defaultStorageColumns = "nDdus" func (c *cmdStorageList) parseColumns() ([]storageColumn, error) { columnsShorthandMap := map[rune]storageColumn{ 'n': {i18n.G("NAME"), c.storageNameColumnData}, 'D': {i18n.G("DRIVER"), c.driverColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 'S': {i18n.G("SOURCE"), c.sourceColumnData}, 'u': {i18n.G("USED BY"), c.usedByColumnData}, 's': {i18n.G("STATE"), c.stateColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []storageColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdStorageList) storageNameColumnData(storage api.StoragePool) string { return storage.Name } func (c *cmdStorageList) driverColumnData(storage api.StoragePool) string { return storage.Driver } func (c *cmdStorageList) descriptionColumnData(storage api.StoragePool) string { return storage.Description } func (c *cmdStorageList) sourceColumnData(storage api.StoragePool) string { return storage.Config["source"] } func (c *cmdStorageList) usedByColumnData(storage api.StoragePool) string { return fmt.Sprintf("%d", len(storage.UsedBy)) } func (c *cmdStorageList) stateColumnData(storage api.StoragePool) string { return strings.ToUpper(storage.Status) } func (c *cmdStorageList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer filters := prepareStoragePoolsServerFilters(parsed[1].StringList, api.StoragePool{}) // Get the storage pools pools, err := d.GetStoragePoolsWithFilter(filters) if err != nil { return err } // Parse column flags. columns, err := c.parseColumns() if err != nil { return err } data := [][]string{} for _, pool := range pools { line := []string{} for _, column := range columns { line = append(line, column.Data(pool)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, pools) } // Set. type cmdStorageSet struct { global *cmdGlobal storage *cmdStorage flagIsProperty bool } var cmdStorageSetUsage = u.Usage{u.Pool.Remote(), u.LegacyKV.List(1)} func (c *cmdStorageSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdStorageSetUsage...) cmd.Short = i18n.G("Set storage pool configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set storage pool configuration keys For backward compatibility, a single configuration key may still be set with: incus storage set [:] `)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a storage property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdStorageSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String keys, err := kvToMap(parsed[1]) if err != nil { return err } if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } // Get the pool entry pool, etag, err := d.GetStoragePool(poolName) if err != nil { return err } writable := pool.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { if writable.Config == nil { writable.Config = make(map[string]string) } // Update the volume config keys. maps.Copy(writable.Config, keys) } err = d.UpdateStoragePool(poolName, writable, etag) if err != nil { return err } return nil } func (c *cmdStorageSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Show. type cmdStorageShow struct { global *cmdGlobal storage *cmdStorage flagResources bool } var cmdStorageShowUsage = u.Usage{u.Pool.Remote()} func (c *cmdStorageShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdStorageShowUsage...) cmd.Short = i18n.G("Show storage pool configurations and resources") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Show storage pool configurations and resources`)) cli.AddBoolFlag(cmd.Flags(), &c.flagResources, "resources", i18n.G("Show the resources available to the storage pool")) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String // If a target member was specified, we return also member-specific config values. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } if c.flagResources { res, err := d.GetStoragePoolResources(poolName) if err != nil { return err } data, err := yaml.Dump(&res, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } pool, _, err := d.GetStoragePool(poolName) if err != nil { return err } sort.Strings(pool.UsedBy) data, err := yaml.Dump(&pool, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Unset. type cmdStorageUnset struct { global *cmdGlobal storage *cmdStorage storageSet *cmdStorageSet flagIsProperty bool } var cmdStorageUnsetUsage = u.Usage{u.Pool.Remote(), u.Key} func (c *cmdStorageUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdStorageUnsetUsage...) cmd.Short = i18n.G("Unset storage pool configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Unset storage pool configuration keys`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a storage property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolConfigs(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.storageSet.flagIsProperty = c.flagIsProperty return unsetKey(c.storageSet, cmd, parsed) } // prepareStoragePoolsServerFilters processes and formats filter criteria // for storage pools, ensuring they are in a format that the server can interpret. func prepareStoragePoolsServerFilters(filters []string, i any) []string { formattedFilters := []string{} for _, filter := range filters { membs := strings.SplitN(filter, "=", 2) key := membs[0] if len(membs) == 1 { regexpValue := key if !strings.Contains(key, "^") && !strings.Contains(key, "$") { regexpValue = "^" + regexpValue + "$" } filter = fmt.Sprintf("name=(%s|^%s.*)", regexpValue, key) } else { firstPart := key if strings.Contains(key, ".") { firstPart = strings.Split(key, ".")[0] } if !structHasField(reflect.TypeOf(i), firstPart) { filter = fmt.Sprintf("config.%s", filter) } } formattedFilters = append(formattedFilters, filter) } return formattedFilters } incus-7.0.0/cmd/incus/storage_bucket.go000066400000000000000000001256741517523235500201020ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "maps" "net/url" "os" "path" "sort" "strings" "time" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/termios" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) type cmdStorageBucket struct { global *cmdGlobal flagTarget string } func (c *cmdStorageBucket) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("bucket") cmd.Short = i18n.G("Manage storage buckets") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage storage buckets.`)) // Create. storageBucketCreateCmd := cmdStorageBucketCreate{global: c.global, storageBucket: c} cmd.AddCommand(storageBucketCreateCmd.command()) // Delete. storageBucketDeleteCmd := cmdStorageBucketDelete{global: c.global, storageBucket: c} cmd.AddCommand(storageBucketDeleteCmd.command()) // Edit. storageBucketEditCmd := cmdStorageBucketEdit{global: c.global, storageBucket: c} cmd.AddCommand(storageBucketEditCmd.command()) // Get. storageBucketGetCmd := cmdStorageBucketGet{global: c.global, storageBucket: c} cmd.AddCommand(storageBucketGetCmd.command()) // List. storageBucketListCmd := cmdStorageBucketList{global: c.global, storageBucket: c} cmd.AddCommand(storageBucketListCmd.command()) // Set. storageBucketSetCmd := cmdStorageBucketSet{global: c.global, storageBucket: c} cmd.AddCommand(storageBucketSetCmd.command()) // Show. storageBucketShowCmd := cmdStorageBucketShow{global: c.global, storageBucket: c} cmd.AddCommand(storageBucketShowCmd.command()) // Unset. storageBucketUnsetCmd := cmdStorageBucketUnset{global: c.global, storageBucket: c, storageBucketSet: &storageBucketSetCmd} cmd.AddCommand(storageBucketUnsetCmd.command()) // Key. storageBucketKeyCmd := cmdStorageBucketKey{global: c.global, storageBucket: c} cmd.AddCommand(storageBucketKeyCmd.command()) // Export. storageBucketExportCmd := cmdStorageBucketExport{global: c.global, storageBucket: c} cmd.AddCommand(storageBucketExportCmd.command()) // Import. storageBucketImporttCmd := cmdStorageBucketImport{global: c.global, storageBucket: c} cmd.AddCommand(storageBucketImporttCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Create. type cmdStorageBucketCreate struct { global *cmdGlobal storageBucket *cmdStorageBucket flagDescription string } var cmdStorageBucketCreateUsage = u.Usage{u.Pool.Remote(), u.NewName(u.Bucket), u.KV.List(0)} func (c *cmdStorageBucketCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdStorageBucketCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create new custom storage buckets") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Create new custom storage buckets`)) cmd.Example = cli.FormatSection("", i18n.G(`incus storage bucket create p1 b01 Create a new storage bucket named b01 in storage pool p1 incus storage bucket create p1 b01 < config.yaml Create a new storage bucket named b01 in storage pool p1 using the content of config.yaml`)) cli.AddStringFlag(cmd.Flags(), &c.storageBucket.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Bucket description")) cmd.RunE = c.run return cmd } func (c *cmdStorageBucketCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String bucketName := parsed[1].String keys, err := kvToMap(parsed[2]) if err != nil { return err } // If stdin isn't a terminal, read yaml from it. var bucketPut api.StorageBucketPut if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } err = loader.Load(&bucketPut) if err != nil && !errors.Is(err, io.EOF) { return err } } if bucketPut.Config == nil { bucketPut.Config = map[string]string{} } maps.Copy(bucketPut.Config, keys) // Create the storage bucket. bucket := api.StorageBucketsPost{ Name: bucketName, StorageBucketPut: bucketPut, } if c.flagDescription != "" { bucket.Description = c.flagDescription } // If a target was specified, create the bucket on the given member. if c.storageBucket.flagTarget != "" { d = d.UseTarget(c.storageBucket.flagTarget) } adminKey, err := d.CreateStoragePoolBucket(poolName, bucket) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Storage bucket %q created")+"\n", bucketName) if adminKey != nil { fmt.Printf(i18n.G("Admin access key: %s")+"\n", adminKey.AccessKey) fmt.Printf(i18n.G("Admin secret key: %s")+"\n", adminKey.SecretKey) } } return nil } // Delete. type cmdStorageBucketDelete struct { global *cmdGlobal storageBucket *cmdStorageBucket } var cmdStorageBucketDeleteUsage = u.Usage{u.Pool.Remote(), u.Bucket} func (c *cmdStorageBucketDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdStorageBucketDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete storage buckets") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete storage buckets`)) cli.AddStringFlag(cmd.Flags(), &c.storageBucket.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run return cmd } func (c *cmdStorageBucketDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String bucketName := parsed[1].String // If a target was specified, delete the bucket on the given member. if c.storageBucket.flagTarget != "" { d = d.UseTarget(c.storageBucket.flagTarget) } // Delete the bucket. err = d.DeleteStoragePoolBucket(poolName, bucketName) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Storage bucket %q deleted")+"\n", bucketName) } return nil } // Edit. type cmdStorageBucketEdit struct { global *cmdGlobal storageBucket *cmdStorageBucket } var cmdStorageBucketEditUsage = u.Usage{u.Pool.Remote(), u.Bucket} func (c *cmdStorageBucketEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdStorageBucketEditUsage...) cmd.Short = i18n.G("Edit storage bucket configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Edit storage bucket configurations as YAML`)) cmd.Example = cli.FormatSection("", i18n.G(`incus storage bucket edit [:] < bucket.yaml Update a storage bucket using the content of bucket.yaml.`)) cli.AddStringFlag(cmd.Flags(), &c.storageBucket.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run return cmd } func (c *cmdStorageBucketEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of a storage bucket. ### Any line starting with a '# will be ignored. ### ### A storage bucket consists of a set of configuration items. ### ### name: bucket1 ### used_by: [] ### config: ### size: "61203283968"`) } func (c *cmdStorageBucketEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String bucketName := parsed[1].String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } // Allow output of `incus storage bucket show` command to be passed in here, but only take the // contents of the StorageBucketPut fields when updating. // The other fields are silently discarded. newdata := api.StorageBucketPut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateStoragePoolBucket(poolName, bucketName, newdata, "") } // If a target was specified, edit the bucket on the given member. if c.storageBucket.flagTarget != "" { d = d.UseTarget(c.storageBucket.flagTarget) } // Get the current config. bucket, etag, err := d.GetStoragePoolBucket(poolName, bucketName) if err != nil { return err } data, err := yaml.Dump(&bucket, yaml.V2) if err != nil { return err } // Spawn the editor. content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor newdata := api.StorageBucket{} err = yaml.Load(content, &newdata) if err == nil { err = d.UpdateStoragePoolBucket(poolName, bucketName, newdata.Writable(), etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Get. type cmdStorageBucketGet struct { global *cmdGlobal storageBucket *cmdStorageBucket flagIsProperty bool } var cmdStorageBucketGetUsage = u.Usage{u.Pool.Remote(), u.Bucket, u.Key} func (c *cmdStorageBucketGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdStorageBucketGetUsage...) cmd.Short = i18n.G("Get values for storage bucket configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Get values for storage bucket configuration keys`)) cli.AddStringFlag(cmd.Flags(), &c.storageBucket.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a storage bucket property")) cmd.RunE = c.run return cmd } func (c *cmdStorageBucketGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String bucketName := parsed[1].String key := parsed[2].String // If a target was specified, use the bucket on the given member. if c.storageBucket.flagTarget != "" { d = d.UseTarget(c.storageBucket.flagTarget) } // Get the storage bucket entry. resp, _, err := d.GetStoragePoolBucket(poolName, bucketName) if err != nil { return err } if c.flagIsProperty { w := resp.Writable() res, err := getFieldByJSONTag(&w, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the storage bucket %q: %v"), key, bucketName, err) } fmt.Printf("%v\n", res) } else { v, ok := resp.Config[key] if ok { fmt.Println(v) } } return nil } // List. type cmdStorageBucketList struct { global *cmdGlobal storageBucket *cmdStorageBucket flagFormat string flagAllProjects bool flagColumns string } var cmdStorageBucketListUsage = u.Usage{u.Pool.Remote(), u.Filter.List(0)} type storageBucketColumn struct { Name string Data func(api.StorageBucket) string } func (c *cmdStorageBucketList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdStorageBucketListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List storage buckets") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List storage buckets Default column layout: ndL == Columns == The -c option takes a comma separated list of arguments that control which storage bucket attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: e - Project name n - Name d - Description L - Location of the storage bucket (e.g. its cluster member)`)) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("Display storage pool buckets from all projects")) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultStorageBucketColumns, "", i18n.G("Columns")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run return cmd } const defaultStorageBucketColumns = "nd" // codespell:ignore nd func (c *cmdStorageBucketList) parseColumns(clustered bool) ([]storageBucketColumn, error) { columnsShorthandMap := map[rune]storageBucketColumn{ 'e': {i18n.G("PROJECT"), c.projectColumnData}, 'n': {i18n.G("NAME"), c.nameColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 'L': {i18n.G("LOCATION"), c.locationColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []storageBucketColumn{} if c.flagColumns == defaultStorageBucketColumns && clustered { columnList = append(columnList, "L") } if c.flagColumns == defaultStorageBucketColumns && c.flagAllProjects { columnList = append([]string{"e"}, columnList...) } for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdStorageBucketList) nameColumnData(bucket api.StorageBucket) string { return bucket.Name } func (c *cmdStorageBucketList) descriptionColumnData(bucket api.StorageBucket) string { return bucket.Description } func (c *cmdStorageBucketList) locationColumnData(bucket api.StorageBucket) string { return bucket.Location } func (c *cmdStorageBucketList) projectColumnData(bucket api.StorageBucket) string { return bucket.Project } func (c *cmdStorageBucketList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String filters := prepareStorageBucketFilters(parsed[1].StringList) var buckets []api.StorageBucket if c.flagAllProjects { buckets, err = d.GetStoragePoolBucketsWithFilterAllProjects(poolName, filters) if err != nil { return err } } else { buckets, err = d.GetStoragePoolBucketsWithFilter(poolName, filters) if err != nil { return err } } // Parse column flags. columns, err := c.parseColumns(d.IsClustered()) if err != nil { return err } data := make([][]string, 0, len(buckets)) for _, bucket := range buckets { line := []string{} for _, column := range columns { line = append(line, column.Data(bucket)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, buckets) } // Set. type cmdStorageBucketSet struct { global *cmdGlobal storageBucket *cmdStorageBucket flagIsProperty bool } var cmdStorageBucketSetUsage = u.Usage{u.Pool.Remote(), u.Bucket, u.LegacyKV.List(1)} func (c *cmdStorageBucketSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdStorageBucketSetUsage...) cmd.Short = i18n.G("Set storage bucket configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set storage bucket configuration keys For backward compatibility, a single configuration key may still be set with: incus storage bucket set [:] `)) cli.AddStringFlag(cmd.Flags(), &c.storageBucket.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a storage bucket property")) cmd.RunE = c.run return cmd } // prepareStorageBucketFilters processes and formats filter criteria // for storage buckets, ensuring they are in a format that the server can interpret. func prepareStorageBucketFilters(filters []string) []string { formatedFilters := []string{} for _, filter := range filters { membs := strings.SplitN(filter, "=", 2) key := membs[0] if len(membs) == 1 { regexpValue := key if !strings.Contains(key, "^") && !strings.Contains(key, "$") { regexpValue = "^" + regexpValue + "$" } filter = fmt.Sprintf("name=(%s|^%s.*)", regexpValue, key) } formatedFilters = append(formatedFilters, filter) } return formatedFilters } // set runs the post-parsing command logic. func (c *cmdStorageBucketSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String bucketName := parsed[1].String keys, err := kvToMap(parsed[2]) if err != nil { return err } // If a target was specified, use the bucket on the given member. if c.storageBucket.flagTarget != "" { d = d.UseTarget(c.storageBucket.flagTarget) } // Get the storage bucket entry. bucket, etag, err := d.GetStoragePoolBucket(poolName, bucketName) if err != nil { return err } writable := bucket.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { maps.Copy(writable.Config, keys) } err = d.UpdateStoragePoolBucket(poolName, bucketName, writable, etag) if err != nil { return err } return nil } func (c *cmdStorageBucketSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Show. type cmdStorageBucketShow struct { global *cmdGlobal storageBucket *cmdStorageBucket } var cmdStorageBucketShowUsage = u.Usage{u.Pool.Remote(), u.Bucket} func (c *cmdStorageBucketShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdStorageBucketShowUsage...) cmd.Short = i18n.G("Show storage bucket configurations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show storage bucket configurations`)) cmd.Example = cli.FormatSection("", i18n.G( `incus storage bucket show default data Will show the properties of a bucket called "data" in the "default" pool.`)) cli.AddStringFlag(cmd.Flags(), &c.storageBucket.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run return cmd } func (c *cmdStorageBucketShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String bucketName := parsed[1].String // If a target member was specified, get the bucket with the matching name on that member, if any. if c.storageBucket.flagTarget != "" { d = d.UseTarget(c.storageBucket.flagTarget) } bucket, _, err := d.GetStoragePoolBucket(poolName, bucketName) if err != nil { return err } data, err := yaml.Dump(&bucket, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Unset. type cmdStorageBucketUnset struct { global *cmdGlobal storageBucket *cmdStorageBucket storageBucketSet *cmdStorageBucketSet flagIsProperty bool } var cmdStorageBucketUnsetUsage = u.Usage{u.Pool.Remote(), u.Bucket, u.Key} func (c *cmdStorageBucketUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdStorageBucketUnsetUsage...) cmd.Short = i18n.G("Unset storage bucket configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Unset storage bucket configuration keys`)) cli.AddStringFlag(cmd.Flags(), &c.storageBucket.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a storage bucket property")) cmd.RunE = c.run return cmd } func (c *cmdStorageBucketUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.storageBucketSet.flagIsProperty = c.flagIsProperty return unsetKey(c.storageBucketSet, cmd, parsed) } // Key commands. type cmdStorageBucketKey struct { global *cmdGlobal storageBucket *cmdStorageBucket flagTarget string } func (c *cmdStorageBucketKey) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("key") cmd.Short = i18n.G("Manage storage bucket keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage storage bucket keys.`)) // Create. storageBucketKeyCreateCmd := cmdStorageBucketKeyCreate{global: c.global, storageBucketKey: c} cmd.AddCommand(storageBucketKeyCreateCmd.command()) // Delete. storageBucketKeyDeleteCmd := cmdStorageBucketKeyDelete{global: c.global, storageBucketKey: c} cmd.AddCommand(storageBucketKeyDeleteCmd.command()) // Edit. storageBucketKeyEditCmd := cmdStorageBucketKeyEdit{global: c.global, storageBucketKey: c} cmd.AddCommand(storageBucketKeyEditCmd.command()) // List. storageBucketKeyListCmd := cmdStorageBucketKeyList{global: c.global, storageBucketKey: c} cmd.AddCommand(storageBucketKeyListCmd.command()) // Show. storageBucketKeyShowCmd := cmdStorageBucketKeyShow{global: c.global, storageBucketKey: c} cmd.AddCommand(storageBucketKeyShowCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // List Keys. type cmdStorageBucketKeyList struct { global *cmdGlobal storageBucketKey *cmdStorageBucketKey flagFormat string flagColumns string } type storageBucketKeyListColumns struct { Name string Data func(api.StorageBucketKey) string } var cmdStorageBucketKeyListUsage = u.Usage{u.Pool.Remote(), u.Bucket} func (c *cmdStorageBucketKeyList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdStorageBucketKeyListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List storage bucket keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List storage bucket keys Default column layout: ndr == Columns == The -c option takes a comma separated list of arguments that control which storage bucket keys attributes to output when displaying in table or csv format. Column arguments are either pre-defined shorthand chars (see below), or (extended) config keys. Commas between consecutive shorthand chars are optional. Pre-defined column shorthand chars: n - Name d - Description r - Role`)) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddStringFlag(cmd.Flags(), &c.storageBucketKey.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultStorageBucketKeyColumns, "", i18n.G("Columns")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run return cmd } const defaultStorageBucketKeyColumns = "ndr" func (c *cmdStorageBucketKeyList) parseColumns() ([]storageBucketKeyListColumns, error) { columnsShorthandMap := map[rune]storageBucketKeyListColumns{ 'n': {i18n.G("NAME"), c.nameColumnData}, 'd': {i18n.G("DESCRIPTION"), c.descriptionColumnData}, 'r': {i18n.G("ROLE"), c.roleColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []storageBucketKeyListColumns{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdStorageBucketKeyList) nameColumnData(buckKey api.StorageBucketKey) string { return buckKey.Name } func (c *cmdStorageBucketKeyList) descriptionColumnData(buckKey api.StorageBucketKey) string { return buckKey.Description } func (c *cmdStorageBucketKeyList) roleColumnData(buckKey api.StorageBucketKey) string { return buckKey.Role } func (c *cmdStorageBucketKeyList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketKeyListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String bucketName := parsed[1].String // If a target member was specified, get the bucket with the matching name on that member, if any. if c.storageBucketKey.flagTarget != "" { d = d.UseTarget(c.storageBucketKey.flagTarget) } bucketKeys, err := d.GetStoragePoolBucketKeys(poolName, bucketName) if err != nil { return err } // Parse column flags. columns, err := c.parseColumns() if err != nil { return err } data := make([][]string, 0, len(bucketKeys)) for _, bucketKey := range bucketKeys { line := []string{} for _, column := range columns { line = append(line, column.Data(bucketKey)) } data = append(data, line) } sort.Sort(cli.SortColumnsNaturally(data)) header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, bucketKeys) } // Create Key. type cmdStorageBucketKeyCreate struct { global *cmdGlobal storageBucketKey *cmdStorageBucketKey flagRole string flagAccessKey string flagSecretKey string flagDescription string } var cmdStorageBucketKeyCreateUsage = u.Usage{u.Pool.Remote(), u.Bucket, u.NewName(u.Key)} func (c *cmdStorageBucketKeyCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdStorageBucketKeyCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create key for a storage bucket") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Create key for a storage bucket")) cmd.Example = cli.FormatSection("", i18n.G(`incus storage bucket key create p1 b01 k1 Create a key called k1 for the bucket b01 in the pool p1. incus storage bucket key create p1 b01 k1 < config.yaml Create a key called k1 for the bucket b01 in the pool p1 using the content of config.yaml.`)) cmd.RunE = c.runAdd cli.AddStringFlag(cmd.Flags(), &c.storageBucketKey.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.flagRole, "role", "read-only", "", i18n.G("Role (admin or read-only)")) cli.AddStringFlag(cmd.Flags(), &c.flagAccessKey, "access-key", "", "", i18n.G("Access key (auto-generated if empty)")) cli.AddStringFlag(cmd.Flags(), &c.flagSecretKey, "secret-key", "", "", i18n.G("Secret key (auto-generated if empty)")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Key description")) return cmd } func (c *cmdStorageBucketKeyCreate) runAdd(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketKeyCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String bucketName := parsed[1].String keyName := parsed[2].String // If a target member was specified, get the bucket with the matching name on that member, if any. if c.storageBucketKey.flagTarget != "" { d = d.UseTarget(c.storageBucketKey.flagTarget) } // If stdin isn't a terminal, read yaml from it. var bucketKeyPut api.StorageBucketKeyPut if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } err = loader.Load(&bucketKeyPut) if err != nil && !errors.Is(err, io.EOF) { return err } } req := api.StorageBucketKeysPost{ Name: keyName, StorageBucketKeyPut: bucketKeyPut, } if c.flagRole != "" { req.Role = c.flagRole } if c.flagAccessKey != "" { req.AccessKey = c.flagAccessKey } if c.flagSecretKey != "" { req.SecretKey = c.flagSecretKey } if c.flagDescription != "" { req.Description = c.flagDescription } key, err := d.CreateStoragePoolBucketKey(poolName, bucketName, req) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Storage bucket key %q added")+"\n", key.Name) fmt.Printf(i18n.G("Access key: %s")+"\n", key.AccessKey) fmt.Printf(i18n.G("Secret key: %s")+"\n", key.SecretKey) } return nil } // Delete Key. type cmdStorageBucketKeyDelete struct { global *cmdGlobal storageBucketKey *cmdStorageBucketKey } var cmdStorageBucketKeyDeleteUsage = u.Usage{u.Pool.Remote(), u.Bucket, u.Key} func (c *cmdStorageBucketKeyDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdStorageBucketKeyDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete key from a storage bucket") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G("Delete key from a storage bucket")) cmd.RunE = c.runRemove cli.AddStringFlag(cmd.Flags(), &c.storageBucketKey.flagTarget, "target", "", "", i18n.G("Cluster member name")) return cmd } func (c *cmdStorageBucketKeyDelete) runRemove(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketKeyDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String bucketName := parsed[1].String keyName := parsed[2].String // If a target member was specified, get the bucket with the matching name on that member, if any. if c.storageBucketKey.flagTarget != "" { d = d.UseTarget(c.storageBucketKey.flagTarget) } err = d.DeleteStoragePoolBucketKey(poolName, bucketName, keyName) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Storage bucket key %q removed")+"\n", keyName) } return nil } // Edit Key. type cmdStorageBucketKeyEdit struct { global *cmdGlobal storageBucketKey *cmdStorageBucketKey } var cmdStorageBucketKeyEditUsage = u.Usage{u.Pool.Remote(), u.Bucket, u.Key} func (c *cmdStorageBucketKeyEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdStorageBucketKeyEditUsage...) cmd.Short = i18n.G("Edit storage bucket key as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Edit storage bucket key as YAML`)) cmd.Example = cli.FormatSection("", i18n.G(`incus storage bucket edit [:] < key.yaml Update a storage bucket key using the content of key.yaml.`)) cli.AddStringFlag(cmd.Flags(), &c.storageBucketKey.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run return cmd } func (c *cmdStorageBucketKeyEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of a storage bucket. ### Any line starting with a '# will be ignored. ### ### A storage bucket consists of a set of configuration items. ### ### name: bucket1 ### used_by: [] ### config: ### size: "61203283968"`) } func (c *cmdStorageBucketKeyEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketKeyEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String bucketName := parsed[1].String keyName := parsed[2].String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } // Allow output of `incus storage bucket key show` command to be passed in here, but only take the // contents of the StorageBucketPut fields when updating. // The other fields are silently discarded. newdata := api.StorageBucketKeyPut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateStoragePoolBucketKey(poolName, bucketName, keyName, newdata, "") } // If a target was specified, edit the bucket on the given member. if c.storageBucketKey.flagTarget != "" { d = d.UseTarget(c.storageBucketKey.flagTarget) } // Get the current config. bucket, etag, err := d.GetStoragePoolBucketKey(poolName, bucketName, keyName) if err != nil { return err } data, err := yaml.Dump(&bucket, yaml.V2) if err != nil { return err } // Spawn the editor. content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } for { // Parse the text received from the editor newdata := api.StorageBucketKey{} err = yaml.Load(content, &newdata) if err == nil { err = d.UpdateStoragePoolBucketKey(poolName, bucketName, keyName, newdata.Writable(), etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Show Key. type cmdStorageBucketKeyShow struct { global *cmdGlobal storageBucketKey *cmdStorageBucketKey } var cmdStorageBucketKeyShowUsage = u.Usage{u.Pool.Remote(), u.Bucket, u.Key} func (c *cmdStorageBucketKeyShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdStorageBucketKeyShowUsage...) cmd.Short = i18n.G("Show storage bucket key configurations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show storage bucket key configurations`)) cmd.Example = cli.FormatSection("", i18n.G( `incus storage bucket key show default data foo Will show the properties of a bucket key called "foo" for a bucket called "data" in the "default" pool.`)) cli.AddStringFlag(cmd.Flags(), &c.storageBucketKey.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run return cmd } func (c *cmdStorageBucketKeyShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketKeyShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String bucketName := parsed[1].String keyName := parsed[2].String // If a target member was specified, get the bucket with the matching name on that member, if any. if c.storageBucketKey.flagTarget != "" { d = d.UseTarget(c.storageBucketKey.flagTarget) } bucket, _, err := d.GetStoragePoolBucketKey(poolName, bucketName, keyName) if err != nil { return err } data, err := yaml.Dump(&bucket, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } type cmdStorageBucketExport struct { global *cmdGlobal storageBucket *cmdStorageBucket flagCompressionAlgorithm string flagForce bool } var cmdStorageBucketExportUsage = u.Usage{u.Pool.Remote(), u.Bucket, u.Target(u.File).Optional()} func (c *cmdStorageBucketExport) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("export", cmdStorageBucketExportUsage...) cmd.Short = i18n.G("Export storage bucket") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Export storage buckets as tarball.`)) cmd.Example = cli.FormatSection("", i18n.G( `incus storage bucket export default b1 Download a backup tarball of the b1 storage bucket from the default pool.`)) cli.AddStringFlag(cmd.Flags(), &c.flagCompressionAlgorithm, "compression", "", "", i18n.G("Define a compression algorithm: for backup or none")) cli.AddStringFlag(cmd.Flags(), &c.storageBucket.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G("Force overwriting existing backup file")) cmd.RunE = c.run return cmd } func (c *cmdStorageBucketExport) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketExportUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String bucketName := parsed[1].String hasTarget := !parsed[2].Skipped targetName := parsed[2].Get("." + bucketName + ".backup") // If a target was specified, use the bucket on the given member. if c.storageBucket.flagTarget != "" { d = d.UseTarget(c.storageBucket.flagTarget) } if isStdout(targetName) { // If outputting to stdout, quiesce the output. c.global.flagQuiet = true } else if hasTarget && !c.flagForce && util.PathExists(targetName) { // Check if the target path already exists. return fmt.Errorf(i18n.G("Target path %q already exists"), targetName) } req := api.StorageBucketBackupsPost{ Name: "", ExpiresAt: time.Now().Add(23 * time.Hour), CompressionAlgorithm: c.flagCompressionAlgorithm, } var getter func(backupReq *incus.BackupFileRequest) error if d.HasExtension("direct_backup") { getter = func(backupReq *incus.BackupFileRequest) error { return d.CreateStoragePoolBucketBackupStream(poolName, bucketName, req, backupReq) } } else { op, err := d.CreateStoragePoolBucketBackup(poolName, bucketName, req) if err != nil { return fmt.Errorf(i18n.G("Failed to create backup: %v"), err) } // Watch the background operation. progress := cli.ProgressRenderer{ Format: i18n.G("Backing up storage bucket: %s"), Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } // Wait until backup is done. err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } progress.Done("") err = op.Wait() if err != nil { return err } // Get name of backup. utStr := op.Get().Resources["backups"][0] uri, err := url.Parse(utStr) if err != nil { return fmt.Errorf(i18n.G("Invalid URL %q: %w"), utStr, err) } backupName, err := url.PathUnescape(path.Base(uri.EscapedPath())) if err != nil { return fmt.Errorf(i18n.G("Invalid backup name segment in path %q: %w"), uri.EscapedPath(), err) } defer func() { // Delete backup after we're done. op, err := d.DeleteStoragePoolBucketBackup(poolName, bucketName, backupName) if err == nil { _ = op.Wait() } }() getter = func(backupReq *incus.BackupFileRequest) error { _, err := d.GetStoragePoolBucketBackupFile(poolName, bucketName, backupName, backupReq) return err } } var target *os.File if isStdout(targetName) { target = os.Stdout } else { target, err = os.Create(targetName) if err != nil { return err } defer func() { _ = target.Close() }() } // Prepare the download request. progress := cli.ProgressRenderer{ Format: i18n.G("Exporting backup of storage bucket: %s"), Quiet: c.global.flagForceLocal, } backupFileRequest := incus.BackupFileRequest{ BackupFile: io.WriteSeeker(target), ProgressHandler: progress.UpdateProgress, } // Export tarball. err = getter(&backupFileRequest) if err != nil { _ = os.Remove(targetName) progress.Done("") return fmt.Errorf(i18n.G("Failed to fetch storage bucket backup: %w"), err) } // Detect backup file type and rename file accordingly. if !hasTarget { _, err := target.Seek(0, io.SeekStart) if err != nil { return err } _, ext, _, err := archive.DetectCompressionFile(target) if err != nil { return err } err = os.Rename(targetName, bucketName+ext) if err != nil { return fmt.Errorf(i18n.G("Failed to rename export file: %w"), err) } } err = target.Close() if err != nil { return fmt.Errorf(i18n.G("Failed to close export file: %w"), err) } progress.Done(i18n.G("Backup exported successfully!")) return nil } // Import. type cmdStorageBucketImport struct { global *cmdGlobal storageBucket *cmdStorageBucket } var cmdStorageBucketImportUsage = u.Usage{u.Pool.Remote(), u.BackupFile, u.Bucket.Optional()} func (c *cmdStorageBucketImport) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("import", cmdStorageBucketImportUsage...) cmd.Short = i18n.G("Import storage bucket") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Import backups of storage buckets.`)) cmd.Example = cli.FormatSection("", i18n.G( `incus storage bucket import default backup0.tar.gz Create a new storage bucket using backup0.tar.gz as the source.`)) cli.AddStringFlag(cmd.Flags(), &c.storageBucket.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run return cmd } func (c *cmdStorageBucketImport) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageBucketImportUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String backupFile := parsed[1].String bucketName := parsed[2].String // Use the provided target. if c.storageBucket.flagTarget != "" { d = d.UseTarget(c.storageBucket.flagTarget) } var file *os.File if isStdin(backupFile) { file = os.Stdin } else { file, err = os.Open(backupFile) if err != nil { return err } defer func() { _ = file.Close() }() } fstat, err := file.Stat() if err != nil { return err } progress := cli.ProgressRenderer{ Format: i18n.G("Importing bucket: %s"), Quiet: c.global.flagQuiet, } createArgs := incus.StoragePoolBucketBackupArgs{ BackupFile: &ioprogress.ProgressReader{ ReadCloser: file, Tracker: &ioprogress.ProgressTracker{ Length: fstat.Size(), Handler: func(percent int64, speed int64) { progress.UpdateProgress(ioprogress.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, units.GetByteSizeString(speed, 2))}) }, }, }, Name: bucketName, } op, err := d.CreateStoragePoolBucketFromBackup(poolName, createArgs) if err != nil { return err } err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } progress.Done("") return nil } incus-7.0.0/cmd/incus/storage_volume.go000066400000000000000000003527661517523235500201400ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "io/fs" "maps" "net" "net/http" "net/url" "os" "path" "path/filepath" "slices" "sort" "strconv" "strings" "sync" "time" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/internal/instance" internalIO "github.com/lxc/incus/v7/internal/io" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/termios" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) type volumeColumn struct { Name string Data func(api.StorageVolume, api.StorageVolumeState) string NeedsState bool } type cmdStorageVolume struct { global *cmdGlobal storage *cmdStorage flagDestinationTarget string } func parseVolume(defaultType string, name string) (string, string) { fields := strings.SplitN(name, "/", 2) if len(fields) == 1 { return fields[0], defaultType } else if len(fields) == 2 && !slices.Contains([]string{"custom", "image", "container", "virtual-machine"}, fields[0]) { return name, defaultType } return fields[1], fields[0] } func (c *cmdStorageVolume) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("volume") cmd.Short = i18n.G("Manage storage volumes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Manage storage volumes Unless specified through a prefix, all volume operations affect "custom" (user created) volumes.`)) // Attach storageVolumeAttachCmd := cmdStorageVolumeAttach{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeAttachCmd.command()) // Attach profile storageVolumeAttachProfileCmd := cmdStorageVolumeAttachProfile{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeAttachProfileCmd.command()) // Copy storageVolumeCopyCmd := cmdStorageVolumeCopy{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeCopyCmd.command()) // Create storageVolumeCreateCmd := cmdStorageVolumeCreate{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeCreateCmd.command()) // Delete storageVolumeDeleteCmd := cmdStorageVolumeDelete{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeDeleteCmd.command()) // Detach storageVolumeDetachCmd := cmdStorageVolumeDetach{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeDetachCmd.command()) // Detach profile storageVolumeDetachProfileCmd := cmdStorageVolumeDetachProfile{global: c.global, storage: c.storage, storageVolume: c, storageVolumeDetach: &storageVolumeDetachCmd} cmd.AddCommand(storageVolumeDetachProfileCmd.command()) // Edit storageVolumeEditCmd := cmdStorageVolumeEdit{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeEditCmd.command()) // Export storageVolumeExportCmd := cmdStorageVolumeExport{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeExportCmd.command()) // Get storageVolumeGetCmd := cmdStorageVolumeGet{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeGetCmd.command()) // Import storageVolumeImportCmd := cmdStorageVolumeImport{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeImportCmd.command()) // Info storageVolumeInfoCmd := cmdStorageVolumeInfo{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeInfoCmd.command()) // List storageVolumeListCmd := cmdStorageVolumeList{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeListCmd.command()) // NBD storageVolumeNBDCmd := cmdStorageVolumeNBD{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeNBDCmd.Command()) // Rename storageVolumeRenameCmd := cmdStorageVolumeRename{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeRenameCmd.command()) // Move storageVolumeMoveCmd := cmdStorageVolumeMove{global: c.global, storage: c.storage, storageVolume: c, storageVolumeCopy: &storageVolumeCopyCmd, storageVolumeRename: &storageVolumeRenameCmd} cmd.AddCommand(storageVolumeMoveCmd.command()) // Set storageVolumeSetCmd := cmdStorageVolumeSet{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeSetCmd.command()) // Show storageVolumeShowCmd := cmdStorageVolumeShow{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeShowCmd.command()) // Snapshot storageVolumeSnapshotCmd := cmdStorageVolumeSnapshot{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeSnapshotCmd.command()) // Unset storageVolumeUnsetCmd := cmdStorageVolumeUnset{global: c.global, storage: c.storage, storageVolume: c, storageVolumeSet: &storageVolumeSetCmd} cmd.AddCommand(storageVolumeUnsetCmd.command()) // File storageVolumeFileCmd := cmdStorageVolumeFile{global: c.global, storage: c.storage, storageVolume: c} cmd.AddCommand(storageVolumeFileCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } func (c *cmdStorageVolume) parseVolumeWithPool(name string) (string, string) { fields := strings.SplitN(name, "/", 2) if len(fields) == 1 { return fields[0], "" } return fields[1], fields[0] } // Attach. type cmdStorageVolumeAttach struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume } var cmdStorageVolumeAttachUsage = u.Usage{u.Pool.Remote(), u.Volume, u.Instance, u.NewName(u.Device).Optional(u.Path.Optional())} func (c *cmdStorageVolumeAttach) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("attach", cmdStorageVolumeAttachUsage...) cmd.Short = i18n.G("Attach new custom storage volumes to instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Attach new custom storage volumes to instances`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } if len(args) == 2 { return c.global.cmpInstanceNamesFromRemote(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeAttach) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeAttachUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volName := parsed[1].String instanceName := parsed[2].String devName := volName devPath := "" if !parsed[3].Skipped { devName = parsed[3].List[0].String devPath = parsed[3].List[1].String } // Prepare the instance's device entry device := map[string]string{ "type": "disk", "pool": poolName, "source": volName, "path": devPath, } // Add the device to the instance err = instanceDeviceAdd(d, instanceName, devName, device) if err != nil { return err } return nil } // Attach profile. type cmdStorageVolumeAttachProfile struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume } var cmdStorageVolumeAttachProfileUsage = u.Usage{u.Pool.Remote(), u.Volume, u.Profile, u.NewName(u.Device).Optional(u.Path.Optional())} func (c *cmdStorageVolumeAttachProfile) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("attach-profile", cmdStorageVolumeAttachProfileUsage...) cmd.Short = i18n.G("Attach new custom storage volumes to profiles") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Attach new custom storage volumes to profiles`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } if len(args) == 2 { return c.global.cmpProfileNamesFromRemote(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeAttachProfile) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeAttachProfileUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volName := parsed[1].String profileName := parsed[2].String devName := volName devPath := "" if !parsed[3].Skipped { devName = parsed[3].List[0].String devPath = parsed[3].List[1].String } // Check if the requested storage volume actually exists vol, _, err := d.GetStoragePoolVolume(poolName, "custom", volName) if err != nil { return err } // Prepare the instance's device entry device := map[string]string{ "type": "disk", "pool": poolName, "source": vol.Name, } // Ignore path for block volumes if vol.ContentType != "block" { device["path"] = devPath } // Add the device to the instance err = profileDeviceAdd(d, profileName, devName, device) if err != nil { return err } return nil } // Copy. type cmdStorageVolumeCopy struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume flagMode string flagVolumeOnly bool flagTargetProject string flagRefresh bool flagRefreshExcludeOlder bool } var cmdStorageVolumeCopyUsage = u.Usage{u.MakePath(u.Pool, u.Volume, u.Snapshot.Optional()).Remote(), u.MakePath(u.Pool, u.NewName(u.Volume)).Remote()} func (c *cmdStorageVolumeCopy) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("copy", cmdStorageVolumeCopyUsage...) cmd.Aliases = []string{"cp"} cmd.Short = i18n.G("Copy custom storage volumes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Copy custom storage volumes`)) cli.AddStringFlag(cmd.Flags(), &c.flagMode, "mode", "pull", "", i18n.G("Transfer mode. One of pull (default), push or relay.")) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.storageVolume.flagDestinationTarget, "destination-target", "", "", i18n.G("Destination cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagVolumeOnly, "volume-only", i18n.G("Copy the volume without its snapshots")) cli.AddStringFlag(cmd.Flags(), &c.flagTargetProject, "target-project", "", "", i18n.G("Copy to a project different from the source")) cli.AddBoolFlag(cmd.Flags(), &c.flagRefresh, "refresh", i18n.G("Refresh and update the existing storage volume copies")) cli.AddBoolFlag(cmd.Flags(), &c.flagRefreshExcludeOlder, "refresh-exclude-older", i18n.G("During refresh, exclude source snapshots earlier than latest target snapshot")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePoolWithVolume(toComplete) } if len(args) == 1 { return c.global.cmpStoragePools(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // copyOrMove runs the post-parsing command logic. func (c *cmdStorageVolumeCopy) copyOrMove(cmd *cobra.Command, parsed []*u.Parsed) error { srcServer := parsed[0].RemoteServer srcPoolName := parsed[0].RemoteObject.List[0].String srcVolName := parsed[0].RemoteObject.List[1].String // This function can be called from both the `copy` and `move` commands. As their first arguments // have a different grammar, additional care is taken here to normalize them. srcIsSnapshot := false srcSnapName := "" if cmd.Name() == "copy" { srcIsSnapshot = !parsed[0].RemoteObject.List[2].Skipped srcSnapName = parsed[0].RemoteObject.List[2].String } dstServer := parsed[1].RemoteServer dstPoolName := parsed[1].RemoteObject.List[0].String dstVolName := parsed[1].RemoteObject.List[1].String // If the source server is standalone then --target cannot be provided. if c.storage.flagTarget != "" && !srcServer.IsClustered() { return errors.New(i18n.G("Cannot set --target when source server is not clustered")) } if c.storage.flagTarget != "" { srcServer = srcServer.UseTarget(c.storage.flagTarget) } // Check if requested storage volume exists. srcVol, _, err := srcServer.GetStoragePoolVolume(srcPoolName, "custom", srcVolName) if err != nil { return err } if srcIsSnapshot && c.flagVolumeOnly { return errors.New(i18n.G("Cannot set --volume-only when copying a snapshot")) } // If the volume is in local storage, set the target to its location (or provide a helpful error // message if the target is incorrect). If the volume is in remote storage (and the source server is clustered) we // can use any provided target. Note that for standalone servers, this will set the target to "none". if srcVol.Location != "" && srcVol.Location != "none" { if c.storage.flagTarget != "" && c.storage.flagTarget != srcVol.Location { return fmt.Errorf(i18n.G("Given target %q does not match source volume location %q"), c.storage.flagTarget, srcVol.Location) } srcServer = srcServer.UseTarget(srcVol.Location) } else if c.storage.flagTarget != "" && srcServer.IsClustered() { srcServer = srcServer.UseTarget(c.storage.flagTarget) } // We can always set the destination target if the destination server is clustered (for local storage volumes this // places the volume on the target member, for remote volumes this does nothing). if c.storageVolume.flagDestinationTarget != "" { if !dstServer.IsClustered() { return errors.New(i18n.G("Cannot set --destination-target when destination server is not clustered")) } dstServer = dstServer.UseTarget(c.storageVolume.flagDestinationTarget) } // Parse the mode mode := "pull" if c.flagMode != "" { mode = c.flagMode } var op incus.RemoteOperation // Messages opMsg := i18n.G("Copying the storage volume: %s") finalMsg := i18n.G("Storage volume copied successfully!") if cmd.Name() == "move" { opMsg = i18n.G("Moving the storage volume: %s") finalMsg = i18n.G("Storage volume moved successfully!") } // If source is a snapshot get source snapshot volume info and apply to the srcVol. if srcIsSnapshot { srcVolSnapshot, _, err := srcServer.GetStoragePoolVolumeSnapshot(srcPoolName, "custom", srcVolName, srcSnapName) if err != nil { return err } // Copy info from source snapshot into source volume used for new volume. srcVol.Name = srcVolName + "/" + srcSnapName srcVol.Config = srcVolSnapshot.Config srcVol.Description = srcVolSnapshot.Description } if cmd.Name() == "move" && srcServer == dstServer { args := &incus.StoragePoolVolumeMoveArgs{} args.Name = dstVolName args.Mode = mode args.VolumeOnly = false args.Project = c.flagTargetProject op, err = dstServer.MoveStoragePoolVolume(dstPoolName, srcServer, srcPoolName, *srcVol, args) if err != nil { return err } } else { args := &incus.StoragePoolVolumeCopyArgs{} args.Name = dstVolName args.Mode = mode args.VolumeOnly = c.flagVolumeOnly args.Refresh = c.flagRefresh args.RefreshExcludeOlder = c.flagRefreshExcludeOlder if c.flagTargetProject != "" { dstServer = dstServer.UseProject(c.flagTargetProject) } op, err = dstServer.CopyStoragePoolVolume(dstPoolName, srcServer, srcPoolName, *srcVol, args) if err != nil { return err } } // Register progress handler progress := cli.ProgressRenderer{ Format: opMsg, Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } // Wait for operation to finish err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } if cmd.Name() == "move" && srcServer != dstServer { err = srcServer.DeleteStoragePoolVolume(srcPoolName, srcVol.Type, srcVolName) if err != nil { progress.Done("") return fmt.Errorf(i18n.G("Failed deleting source volume after copy: %w"), err) } } progress.Done(finalMsg) return nil } func (c *cmdStorageVolumeCopy) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeCopyUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.copyOrMove(cmd, parsed) } // Create. type cmdStorageVolumeCreate struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume flagContentType string flagDescription string } var cmdStorageVolumeCreateUsage = u.Usage{u.Pool.Remote(), u.NewName(u.Volume), u.KV.List(0)} func (c *cmdStorageVolumeCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdStorageVolumeCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Create new custom storage volumes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Create new custom storage volumes`)) cmd.Example = cli.FormatSection("", i18n.G(`incus storage volume create default foo Create custom storage volume "foo" in pool "default" incus storage volume create default foo < config.yaml Create custom storage volume "foo" in pool "default" with configuration from config.yaml`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.flagContentType, "type|t", "filesystem", "", i18n.G("Content type, block or filesystem")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Volume description")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volName := parsed[1].String keys, err := kvToMap(parsed[2]) if err != nil { return err } var volumePut api.StorageVolumePut if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin, yaml.WithKnownFields()) if err != nil { return err } err = loader.Load(&volumePut) if err != nil && !errors.Is(err, io.EOF) { return err } } // Create the storage volume entry vol := api.StorageVolumesPost{ Name: volName, Type: "custom", ContentType: c.flagContentType, StorageVolumePut: volumePut, } if volumePut.Config == nil { vol.Config = map[string]string{} } maps.Copy(vol.Config, keys) if c.flagDescription != "" { vol.Description = c.flagDescription } // If a target was specified, create the volume on the given member. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } err = d.CreateStoragePoolVolume(poolName, vol) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Storage volume %s created")+"\n", volName) } return nil } // Delete. type cmdStorageVolumeDelete struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume } var cmdStorageVolumeDeleteUsage = u.Usage{u.Pool.Remote(), u.MakePath(u.StorageVolumeType.Optional(), u.Volume)} func (c *cmdStorageVolumeDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdStorageVolumeDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete custom storage volumes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete custom storage volumes`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volType := parsed[1].List[0].Get("custom") volName := parsed[1].List[1].String // If a target was specified, delete the volume on the given member. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } // Delete the volume err = d.DeleteStoragePoolVolume(poolName, volType, volName) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Storage volume %s deleted")+"\n", volName) } return nil } // Detach. type cmdStorageVolumeDetach struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume } var cmdStorageVolumeDetachUsage = u.Usage{u.Pool.Remote(), u.Volume, u.Instance, u.Device.Optional()} func (c *cmdStorageVolumeDetach) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("detach", cmdStorageVolumeDetachUsage...) cmd.Short = i18n.G("Detach custom storage volumes from instances") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Detach custom storage volumes from instances`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } if len(args) == 2 { return c.global.cmpStoragePoolVolumeInstances(args[0], args[1]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // Find a matching device. func (c *cmdStorageVolumeDetach) findDevice(devices map[string]map[string]string, poolName string, volName string, dev *u.Parsed) (string, error) { hasDevice := !dev.Skipped devName := dev.String found := false for n, d := range devices { if hasDevice { if n == devName { if d["type"] != "disk" { return "", fmt.Errorf(i18n.G("The specified device is not a disk (%s device)"), d["type"]) } if d["pool"] != poolName { return "", fmt.Errorf(i18n.G("The specified disk is not in the given pool (found %s)"), d["pool"]) } if d["source"] != volName { return "", fmt.Errorf(i18n.G("The specified disk does not point to the given storage volume (found %s)"), d["source"]) } found = true break } continue } if d["type"] == "disk" && d["pool"] == poolName && d["source"] == volName { if found { return "", errors.New(i18n.G("More than one device matches, specify the device name")) } devName = n found = true } } if !found { return "", errors.New(i18n.G("No device found for this storage volume")) } return devName, nil } func (c *cmdStorageVolumeDetach) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeDetachUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volName := parsed[1].String instanceName := parsed[2].String // Get the instance entry inst, etag, err := d.GetInstance(instanceName) if err != nil { return err } devName, err := c.findDevice(inst.Devices, poolName, volName, parsed[3]) if err != nil { return err } // Remove the device delete(inst.Devices, devName) op, err := d.UpdateInstance(instanceName, inst.Writable(), etag) if err != nil { return err } return op.Wait() } // Detach profile. type cmdStorageVolumeDetachProfile struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume storageVolumeDetach *cmdStorageVolumeDetach } var cmdStorageVolumeDetachProfileUsage = u.Usage{u.Pool.Remote(), u.Volume, u.Profile, u.Device.Optional()} func (c *cmdStorageVolumeDetachProfile) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("detach-profile", cmdStorageVolumeDetachProfileUsage...) cmd.Short = i18n.G("Detach custom storage volumes from profiles") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Detach custom storage volumes from profiles`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } if len(args) == 2 { return c.global.cmpStoragePoolVolumeProfiles(args[0], args[1]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeDetachProfile) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeDetachProfileUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volName := parsed[1].String profileName := parsed[2].String // Get the profile entry profile, etag, err := d.GetProfile(profileName) if err != nil { return err } devName, err := c.storageVolumeDetach.findDevice(profile.Devices, poolName, volName, parsed[3]) if err != nil { return err } // Remove the device delete(profile.Devices, devName) err = d.UpdateProfile(profileName, profile.Writable(), etag) if err != nil { return err } return nil } // Edit. type cmdStorageVolumeEdit struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume } // The parsing is ambiguous here, so we try to disambiguate by using a set of reserved names. var cmdStorageVolumeEditUsage = u.Usage{u.Pool.Remote(), u.MakePath(u.StorageVolumeType.Optional(), u.Volume, u.Snapshot.Optional())} func (c *cmdStorageVolumeEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdStorageVolumeEditUsage...) cmd.Short = i18n.G("Edit storage volume configurations as YAML") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Edit storage volume configurations as YAML If the type is not specified, incus assumes the type is "custom". Supported values for type are "custom", "container" and "virtual-machine".`)) cmd.Example = cli.FormatSection("", i18n.G( `incus storage volume edit default container/c1 Edit container storage volume "c1" in pool "default" incus storage volume edit default foo < volume.yaml Edit custom storage volume "foo" in pool "default" using the content of volume.yaml`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeEdit) helpTemplate() string { return i18n.G( `### This is a YAML representation of a storage volume. ### Any line starting with a '# will be ignored. ### ### A storage volume consists of a set of configuration items. ### ### name: foo ### type: custom ### used_by: [] ### config: ### size: "61203283968"`) } func (c *cmdStorageVolumeEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volType := parsed[1].List[0].Get("custom") volName := parsed[1].List[1].String isSnapshot := !parsed[1].List[2].Skipped snapName := parsed[1].List[2].String // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } if isSnapshot { newdata := api.StorageVolumeSnapshotPut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } err := d.UpdateStoragePoolVolumeSnapshot(poolName, volType, volName, snapName, newdata, "") if err != nil { return err } return nil } newdata := api.StorageVolumePut{} err = loader.Load(&newdata) if err != nil && !errors.Is(err, io.EOF) { return err } return d.UpdateStoragePoolVolume(poolName, volType, volName, newdata, "") } // If a target was specified, create the volume on the given member. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } var data []byte var snapVol *api.StorageVolumeSnapshot var vol *api.StorageVolume etag := "" if isSnapshot { // Extract the current value snapVol, etag, err = d.GetStoragePoolVolumeSnapshot(poolName, volType, volName, snapName) if err != nil { return err } data, err = yaml.Dump(&snapVol, yaml.V2) if err != nil { return err } } else { // Extract the current value vol, etag, err = d.GetStoragePoolVolume(poolName, volType, volName) if err != nil { return err } data, err = yaml.Dump(&vol, yaml.V2) if err != nil { return err } } // Spawn the editor content, err := cli.TextEditor("", []byte(c.helpTemplate()+"\n\n"+string(data))) if err != nil { return err } if isSnapshot { for { // Parse the text received from the editor newdata := api.StorageVolumeSnapshotPut{} err = yaml.Load(content, &newdata) if err == nil { err = d.UpdateStoragePoolVolumeSnapshot(poolName, volType, volName, snapName, newdata, etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } for { // Parse the text received from the editor newdata := api.StorageVolume{} err = yaml.Load(content, &newdata) if err == nil { err = d.UpdateStoragePoolVolume(poolName, volType, volName, newdata.Writable(), etag) } // Respawn the editor if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Config parsing error: %s")+"\n", err) fmt.Println(i18n.G("Press enter to open the editor again or ctrl+c to abort change")) _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } // Get. type cmdStorageVolumeGet struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume flagIsProperty bool } var cmdStorageVolumeGetUsage = u.Usage{u.Pool.Remote(), u.MakePath(u.StorageVolumeType.Optional(), u.Volume, u.Snapshot.Optional()), u.Key} func (c *cmdStorageVolumeGet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("get", cmdStorageVolumeGetUsage...) cmd.Short = i18n.G("Get values for storage volume configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Get values for storage volume configuration keys If the type is not specified, incus assumes the type is "custom". Supported values for type are "custom", "container" and "virtual-machine". For snapshots, add the snapshot name (only if type is one of custom, container or virtual-machine).`)) cmd.Example = cli.FormatSection("", i18n.G( `incus storage volume get default data size Returns the size of a custom volume "data" in pool "default" incus storage volume get default virtual-machine/data snapshots.expiry Returns the snapshot expiration period for a virtual machine "data" in pool "default"`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Get the key as a storage volume property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } if len(args) == 2 { return c.global.cmpStoragePoolVolumeConfigs(args[0], args[1]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeGet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeGetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volType := parsed[1].List[0].Get("custom") volName := parsed[1].List[1].String isSnapshot := !parsed[1].List[2].Skipped snapName := parsed[1].List[2].String key := parsed[2].String // If a target was specified, create the volume on the given member. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } if isSnapshot { resp, _, err := d.GetStoragePoolVolumeSnapshot(poolName, volType, volName, snapName) if err != nil { return err } if c.flagIsProperty { res, err := getFieldByJSONTag(resp, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the storage pool volume snapshot %s/%s: %v"), key, volName, snapName, err) } fmt.Printf("%v\n", res) } else { v, ok := resp.Config[key] if ok { fmt.Println(v) } } return nil } // Get the storage volume entry resp, _, err := d.GetStoragePoolVolume(poolName, volType, volName) if err != nil { // Give more context on missing volumes. if api.StatusErrorCheck(err, http.StatusNotFound) { return fmt.Errorf("Storage pool volume \"%s/%s\" not found", volType, volName) } return err } if c.flagIsProperty { res, err := getFieldByJSONTag(resp, key) if err != nil { return fmt.Errorf(i18n.G("The property %q does not exist on the storage pool volume %q: %v"), key, volName, err) } fmt.Printf("%v\n", res) } else { v, ok := resp.Config[key] if ok { fmt.Println(v) } } return nil } // Info. type cmdStorageVolumeInfo struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume } var cmdStorageVolumeInfoUsage = u.Usage{u.Pool.Remote(), u.MakePath(u.StorageVolumeType.Optional(), u.Volume)} func (c *cmdStorageVolumeInfo) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("info", cmdStorageVolumeInfoUsage...) cmd.Short = i18n.G("Show storage volume state information") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Show storage volume state information If the type is not specified, Incus assumes the type is "custom". Supported values for type are "custom", "container" and "virtual-machine".`)) cmd.Example = cli.FormatSection("", i18n.G(`incus storage volume info default foo Returns state information for a custom volume "foo" in pool "default" incus storage volume info default virtual-machine/v1 Returns state information for virtual machine "v1" in pool "default"`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeInfo) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeInfoUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volType := parsed[1].List[0].Get("custom") volName := parsed[1].List[1].String // If a target member was specified, get the volume with the matching // name on that member, if any. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } // Get the data. vol, _, err := d.GetStoragePoolVolume(poolName, volType, volName) if err != nil { // Give more context on missing volumes. if api.StatusErrorCheck(err, http.StatusNotFound) { return fmt.Errorf("Storage pool volume \"%s/%s\" not found", volType, volName) } return err } // Instead of failing here if the usage cannot be determined, it is just omitted. volState, _ := d.GetStoragePoolVolumeState(poolName, volType, volName) volSnapshots, err := d.GetStoragePoolVolumeSnapshots(poolName, volType, volName) if err != nil { return err } var volBackups []api.StorageVolumeBackup if d.HasExtension("custom_volume_backup") && volType == "custom" { volBackups, err = d.GetStorageVolumeBackups(poolName, volName) if err != nil { return err } } // Render the overview. fmt.Printf(i18n.G("Name: %s")+"\n", vol.Name) if vol.Description != "" { fmt.Printf(i18n.G("Description: %s")+"\n", vol.Description) } if vol.Type == "" { vol.Type = "custom" } fmt.Printf(i18n.G("Type: %s")+"\n", vol.Type) if vol.ContentType == "" { vol.ContentType = "filesystem" } fmt.Printf(i18n.G("Content type: %s")+"\n", vol.ContentType) if vol.Location != "" && d.IsClustered() { fmt.Printf(i18n.G("Location: %s")+"\n", vol.Location) } if volState != nil && volState.Usage != nil { fmt.Printf(i18n.G("Usage: %s")+"\n", units.GetByteSizeStringIEC(int64(volState.Usage.Used), 2)) if volState.Usage.Total > 0 { fmt.Printf(i18n.G("Total: %s")+"\n", units.GetByteSizeStringIEC(int64(volState.Usage.Total), 2)) } } if !vol.CreatedAt.IsZero() { fmt.Printf(i18n.G("Created: %s")+"\n", vol.CreatedAt.Local().Format(dateLayout)) } // List snapshots firstSnapshot := true if len(volSnapshots) > 0 { snapData := [][]string{} for _, snap := range volSnapshots { if firstSnapshot { fmt.Println("\n" + i18n.G("Snapshots:")) } var row []string fields := strings.Split(snap.Name, instance.SnapshotDelimiter) row = append(row, fields[len(fields)-1]) row = append(row, snap.Description) if snap.ExpiresAt != nil { row = append(row, snap.ExpiresAt.Local().Format(dateLayout)) } else { row = append(row, " ") } firstSnapshot = false snapData = append(snapData, row) } sort.Sort(cli.SortColumnsNaturally(snapData)) snapHeader := []string{ i18n.G("Name"), i18n.G("Description"), i18n.G("Expires at"), } _ = cli.RenderTable(os.Stdout, cli.TableFormatTable, snapHeader, snapData, volSnapshots) } // List backups firstBackup := true if len(volBackups) > 0 { backupData := [][]string{} for _, backup := range volBackups { if firstBackup { fmt.Println("\n" + i18n.G("Backups:")) } var row []string row = append(row, backup.Name) if !backup.CreatedAt.IsZero() { row = append(row, backup.CreatedAt.Local().Format(dateLayout)) } else { row = append(row, " ") } if !backup.ExpiresAt.IsZero() { row = append(row, backup.ExpiresAt.Local().Format(dateLayout)) } else { row = append(row, " ") } if backup.VolumeOnly { row = append(row, "YES") } else { row = append(row, "NO") } if backup.OptimizedStorage { row = append(row, "YES") } else { row = append(row, "NO") } firstBackup = false backupData = append(backupData, row) } backupHeader := []string{ i18n.G("Name"), i18n.G("Taken at"), i18n.G("Expires at"), i18n.G("Volume Only"), i18n.G("Optimized Storage"), } _ = cli.RenderTable(os.Stdout, cli.TableFormatTable, backupHeader, backupData, volBackups) } return nil } // List. type cmdStorageVolumeList struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume flagFormat string flagColumns string flagAllProjects bool defaultColumns string } var cmdStorageVolumeListUsage = u.Usage{u.Pool.Remote(), u.Filter.List(0)} func (c *cmdStorageVolumeList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdStorageVolumeListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List storage volumes") c.defaultColumns = "etndcuL" cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", c.defaultColumns, "", i18n.G("Columns")) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("All projects")) cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List storage volumes A single keyword like "vol" which will list any storage volume with a name starting by "vol". A regular expression on the storage volume name. (e.g. .*vol.*01$). A key/value pair where the key is a storage volume field name. Multiple values must be delimited by ','. Examples: - "type=custom" will list all custom storage volumes - "type=custom content_type=block" will list all custom block storage volumes == Columns == The -c option takes a (optionally comma-separated) list of arguments that control which image attributes to output when displaying in table or csv format. Column shorthand chars: c - Content type (filesystem or block) d - Description e - Project name L - Location of the instance (e.g. its cluster member) n - Name t - Type of volume (custom, image, container or virtual-machine) u - Number of references (used by) U - Current disk usage`)) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String // Process the filters filters := []string{} for _, filter := range parsed[1].StringList { membs := strings.SplitN(filter, "=", 2) key := membs[0] if len(membs) == 1 { regexpValue := key if !strings.Contains(key, "^") && !strings.Contains(key, "$") { regexpValue = "^" + regexpValue + "$" } filter = fmt.Sprintf("name=(%s|^%s.*)", regexpValue, key) } filters = append(filters, filter) } var volumes []api.StorageVolume if c.flagAllProjects { volumes, err = d.GetStoragePoolVolumesWithFilterAllProjects(poolName, filters) } else { volumes, err = d.GetStoragePoolVolumesWithFilter(poolName, filters) } if err != nil { return err } // Process the columns columns, err := c.parseColumns(d.IsClustered()) if err != nil { return err } // Render the table data := [][]string{} for _, vol := range volumes { row := []string{} for _, column := range columns { if column.NeedsState && !instance.IsSnapshot(vol.Name) && vol.Type != "image" { state, err := d.UseProject(vol.Project).GetStoragePoolVolumeState(poolName, vol.Type, vol.Name) if err != nil { return err } row = append(row, column.Data(vol, *state)) } else { row = append(row, column.Data(vol, api.StorageVolumeState{})) } } data = append(data, row) } if len(columns) >= 2 { sort.Sort(cli.ByNameAndType(data)) } rawData := make([]*api.StorageVolume, len(volumes)) for i := range volumes { rawData[i] = &volumes[i] } headers := []string{} for _, column := range columns { headers = append(headers, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, headers, data, rawData) } func (c *cmdStorageVolumeList) parseColumns(clustered bool) ([]volumeColumn, error) { columnsShorthandMap := map[rune]volumeColumn{ 't': {Name: i18n.G("TYPE"), Data: c.typeColumnData}, 'n': {Name: i18n.G("NAME"), Data: c.nameColumnData}, 'd': {Name: i18n.G("DESCRIPTION"), Data: c.descriptionColumnData}, 'c': {Name: i18n.G("CONTENT-TYPE"), Data: c.contentTypeColumnData}, 'u': {Name: i18n.G("USED BY"), Data: c.usedByColumnData}, 'U': {Name: i18n.G("USAGE"), Data: c.usageColumnData, NeedsState: true}, } if clustered { columnsShorthandMap['L'] = volumeColumn{Name: i18n.G("LOCATION"), Data: c.locationColumnData} } else { if c.flagColumns != c.defaultColumns { if strings.ContainsAny(c.flagColumns, "L") { return nil, errors.New(i18n.G("Can't specify column L when not clustered")) } } c.flagColumns = strings.ReplaceAll(c.flagColumns, "L", "") } if c.flagAllProjects { columnsShorthandMap['e'] = volumeColumn{Name: i18n.G("PROJECT"), Data: c.projectColumnData} } else { c.flagColumns = strings.ReplaceAll(c.flagColumns, "e", "") } columnList := strings.Split(c.flagColumns, ",") columns := []volumeColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdStorageVolumeList) typeColumnData(vol api.StorageVolume, _ api.StorageVolumeState) string { if instance.IsSnapshot(vol.Name) { return fmt.Sprintf("%s (snapshot)", vol.Type) } return vol.Type } func (c *cmdStorageVolumeList) nameColumnData(vol api.StorageVolume, _ api.StorageVolumeState) string { return vol.Name } func (c *cmdStorageVolumeList) descriptionColumnData(vol api.StorageVolume, _ api.StorageVolumeState) string { return vol.Description } func (c *cmdStorageVolumeList) contentTypeColumnData(vol api.StorageVolume, _ api.StorageVolumeState) string { if vol.ContentType == "" { return "filesystem" } return vol.ContentType } func (c *cmdStorageVolumeList) usedByColumnData(vol api.StorageVolume, _ api.StorageVolumeState) string { return strconv.Itoa(len(vol.UsedBy)) } func (c *cmdStorageVolumeList) locationColumnData(vol api.StorageVolume, _ api.StorageVolumeState) string { return vol.Location } func (c *cmdStorageVolumeList) usageColumnData(_ api.StorageVolume, state api.StorageVolumeState) string { if state.Usage != nil { return units.GetByteSizeStringIEC(int64(state.Usage.Used), 2) } return "" } func (c *cmdStorageVolumeList) projectColumnData(vol api.StorageVolume, _ api.StorageVolumeState) string { return vol.Project } // Move. type cmdStorageVolumeMove struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume storageVolumeCopy *cmdStorageVolumeCopy storageVolumeRename *cmdStorageVolumeRename } var cmdStorageVolumeMoveUsage = u.Usage{u.MakePath(u.Pool, u.Volume).Remote(), u.MakePath(u.Pool, u.NewName(u.Volume)).Remote()} func (c *cmdStorageVolumeMove) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("move", cmdStorageVolumeMoveUsage...) cmd.Aliases = []string{"mv"} cmd.Short = i18n.G("Move custom storage volumes between pools") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Move custom storage volumes between pools`)) cli.AddStringFlag(cmd.Flags(), &c.storageVolumeCopy.flagMode, "mode", "pull", "", i18n.G("Transfer mode, one of pull (default), push or relay")) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.storageVolume.flagDestinationTarget, "destination-target", "", "", i18n.G("Destination cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.storageVolumeCopy.flagTargetProject, "target-project", "", "", i18n.G("Move to a project different from the source")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePoolWithVolume(toComplete) } if len(args) == 1 { return c.global.cmpStoragePools(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeMove) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeMoveUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } // Source srcServer := parsed[0].RemoteServer srcPoolName := parsed[0].RemoteObject.List[0].String srcVolName := parsed[0].RemoteObject.List[1].String // Destination dstServer := parsed[1].RemoteServer dstPoolName := parsed[1].RemoteObject.List[0].String dstVolName := parsed[1].RemoteObject.List[1].String // Rename volume if both remotes and pools of source and target are equal // and neither destination cluster member name nor target project are set. if srcServer == dstServer && srcPoolName == dstPoolName && c.storageVolume.flagDestinationTarget == "" && c.storageVolumeCopy.flagTargetProject == "" { return c.storageVolumeRename.rename(srcServer, srcPoolName, srcVolName, dstVolName) } return c.storageVolumeCopy.copyOrMove(cmd, parsed) } // Rename. type cmdStorageVolumeRename struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume } var cmdStorageVolumeRenameUsage = u.Usage{u.Pool.Remote(), u.Volume, u.NewName(u.Volume)} func (c *cmdStorageVolumeRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdStorageVolumeRenameUsage...) cmd.Short = i18n.G("Rename custom storage volumes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Rename custom storage volumes`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // rename runs the post-parsing command logic. func (c *cmdStorageVolumeRename) rename(d incus.InstanceServer, poolName string, volName string, newVolName string) error { // If a target member was specified, get the volume with the matching // name on that member, if any. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } err := d.RenameStoragePoolVolume(poolName, "custom", volName, api.StorageVolumePost{Name: newVolName}) if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G(`Renamed storage volume from "%s" to "%s"`)+"\n", volName, newVolName) } return nil } func (c *cmdStorageVolumeRename) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeRenameUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volName := parsed[1].String newVolName := parsed[2].String return c.rename(d, poolName, volName, newVolName) } // Set. type cmdStorageVolumeSet struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume flagIsProperty bool } var cmdStorageVolumeSetUsage = u.Usage{u.Pool.Remote(), u.MakePath(u.StorageVolumeType.Optional(), u.Volume, u.Snapshot.Optional()), u.LegacyKV.List(1)} func (c *cmdStorageVolumeSet) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("set", cmdStorageVolumeSetUsage...) cmd.Short = i18n.G("Set storage volume configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Set storage volume configuration keys For backward compatibility, a single configuration key may still be set with: incus storage volume set [:] [/] If the type is not specified, Incus assumes the type is "custom". Supported values for type are "custom", "container" and "virtual-machine".`)) cmd.Example = cli.FormatSection("", i18n.G( `incus storage volume set default data size=1GiB Sets the size of a custom volume "data" in pool "default" to 1 GiB incus storage volume set default virtual-machine/data snapshots.expiry=7d Sets the snapshot expiration period for a virtual machine "data" in pool "default" to seven days`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Set the key as a storage volume property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } // TODO all volume config keys return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // set runs the post-parsing command logic. func (c *cmdStorageVolumeSet) set(cmd *cobra.Command, parsed []*u.Parsed) error { d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volType := parsed[1].List[0].Get("custom") volName := parsed[1].List[1].String isSnapshot := !parsed[1].List[2].Skipped snapName := parsed[1].List[2].String keys, err := kvToMap(parsed[2]) if err != nil { return err } if isSnapshot { if c.flagIsProperty { snapVol, etag, err := d.GetStoragePoolVolumeSnapshot(poolName, volType, volName, snapName) if err != nil { return err } writable := snapVol.Writable() if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } err = d.UpdateStoragePoolVolumeSnapshot(poolName, volType, volName, snapName, writable, etag) if err != nil { return err } return nil } return errors.New(i18n.G("Snapshots are read-only and can't have their configuration changed")) } // If a target was specified, create the volume on the given member. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } // Get the storage volume entry. vol, etag, err := d.GetStoragePoolVolume(poolName, volType, volName) if err != nil { // Give more context on missing volumes. if api.StatusErrorCheck(err, http.StatusNotFound) { return fmt.Errorf("Storage pool volume \"%s/%s\" not found", volType, volName) } return err } writable := vol.Writable() if c.flagIsProperty { if cmd.Name() == "unset" { for k := range keys { err := unsetFieldByJSONTag(&writable, k) if err != nil { return fmt.Errorf(i18n.G("Error unsetting property: %v"), err) } } } else { err := unpackKVToWritable(&writable, keys) if err != nil { return fmt.Errorf(i18n.G("Error setting properties: %v"), err) } } } else { // Update the volume config keys. maps.Copy(writable.Config, keys) } err = d.UpdateStoragePoolVolume(poolName, vol.Type, vol.Name, writable, etag) if err != nil { return err } return nil } func (c *cmdStorageVolumeSet) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeSetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.set(cmd, parsed) } // Show. type cmdStorageVolumeShow struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume } var cmdStorageVolumeShowUsage = u.Usage{u.Pool.Remote(), u.MakePath(u.StorageVolumeType.Optional(), u.Volume)} func (c *cmdStorageVolumeShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdStorageVolumeShowUsage...) cmd.Short = i18n.G("Show storage volume configurations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Show storage volume configurations If the type is not specified, Incus assumes the type is "custom". Supported values for type are "custom", "container" and "virtual-machine". For snapshots, add the snapshot name (only if type is one of custom, container or virtual-machine).`)) cmd.Example = cli.FormatSection("", i18n.G(`incus storage volume show default foo Will show the properties of custom volume "foo" in pool "default" incus storage volume show default virtual-machine/v1 Will show the properties of the virtual-machine volume "v1" in pool "default" incus storage volume show default container/c1 Will show the properties of the container volume "c1" in pool "default"`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volType := parsed[1].List[0].Get("custom") volName := parsed[1].List[1].String // If a target member was specified, get the volume with the matching // name on that member, if any. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } // Get the storage volume entry vol, _, err := d.GetStoragePoolVolume(poolName, volType, volName) if err != nil { // Give more context on missing volumes. if api.StatusErrorCheck(err, http.StatusNotFound) { if volType == "custom" { return fmt.Errorf("Storage pool volume \"%s/%s\" not found. Try virtual-machine or container for type", volType, volName) } return fmt.Errorf("Storage pool volume \"%s/%s\" not found", volType, volName) } return err } sort.Strings(vol.UsedBy) data, err := yaml.Dump(&vol, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Unset. type cmdStorageVolumeUnset struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume storageVolumeSet *cmdStorageVolumeSet flagIsProperty bool } var cmdStorageVolumeUnsetUsage = u.Usage{u.Pool.Remote(), u.MakePath(u.StorageVolumeType.Optional(), u.Volume, u.Snapshot.Optional()), u.Key} func (c *cmdStorageVolumeUnset) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("unset", cmdStorageVolumeUnsetUsage...) cmd.Short = i18n.G("Unset storage volume configuration keys") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Unset storage volume configuration keys If the type is not specified, Incus assumes the type is "custom". Supported values for type are "custom", "container" and "virtual-machine".`)) cmd.Example = cli.FormatSection("", i18n.G(`incus storage volume unset default foo size Removes the size/quota of custom volume "foo" in pool "default" incus storage volume unset default virtual-machine/v1 snapshots.expiry Removes the snapshot expiration period of virtual machine volume "v1" in pool "default"`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagIsProperty, "property|p", i18n.G("Unset the key as a storage volume property")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } if len(args) == 2 { return c.global.cmpStoragePoolVolumeConfigs(args[0], args[1]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeUnset) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeUnsetUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } c.storageVolumeSet.flagIsProperty = c.flagIsProperty return unsetKey(c.storageVolumeSet, cmd, parsed) } // File. type cmdStorageVolumeFile struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume flagUID int flagGID int flagMode string flagMkdir bool } func (c *cmdStorageVolumeFile) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("file") cmd.Short = i18n.G("Manage files in custom volumes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage files in custom volumes`)) // Create storageVolumeFileCreateCmd := cmdStorageVolumeFileCreate{global: c.global, storage: c.storage, storageVolume: c.storageVolume, storageVolumeFile: c} cmd.AddCommand(storageVolumeFileCreateCmd.command()) // Delete storageVolumeFileDeleteCmd := cmdStorageVolumeFileDelete{global: c.global, storage: c.storage, storageVolume: c.storageVolume, storageVolumeFile: c} cmd.AddCommand(storageVolumeFileDeleteCmd.command()) // Mount storageVolumeFileMountCmd := cmdStorageVolumeFileMount{global: c.global, storage: c.storage, storageVolume: c.storageVolume, storageVolumeFile: c} cmd.AddCommand(storageVolumeFileMountCmd.command()) // Pull storageVolumeFilePullCmd := cmdStorageVolumeFilePull{global: c.global, storage: c.storage, storageVolume: c.storageVolume, storageVolumeFile: c, puller: &pullable{}} cmd.AddCommand(storageVolumeFilePullCmd.command()) // Push storageVolumeFilePushCmd := cmdStorageVolumeFilePush{global: c.global, storage: c.storage, storageVolume: c.storageVolume, storageVolumeFile: c, pusher: &pushable{}} cmd.AddCommand(storageVolumeFilePushCmd.command()) // Edit storageVolumeFileEditCmd := cmdStorageVolumeFileEdit{global: c.global, filePull: &storageVolumeFilePullCmd, filePush: &storageVolumeFilePushCmd} cmd.AddCommand(storageVolumeFileEditCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Create. type cmdStorageVolumeFileCreate struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume storageVolumeFile *cmdStorageVolumeFile flagForce bool flagType string } var cmdStorageVolumeFileCreateUsage = u.Usage{u.Pool.Remote(), u.MakePath(u.Volume, u.Path), u.SymlinkTargetPath.Optional()} func (c *cmdStorageVolumeFileCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdStorageVolumeFileCreateUsage...) cmd.Short = i18n.G("Create files and directories in custom vollume") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Create files and directories in custom volume`)) cmd.Example = cli.FormatSection("", i18n.G( `incus storage volume file create foo bar/baz To create a file baz in the bar volume on the foo pool. incus file create --type=symlink foo bar/baz qux To create a symlink qux in bar storage volume on the foo pool whose target is baz.`)) cli.AddBoolFlag(cmd.Flags(), &c.storageVolumeFile.flagMkdir, "create-dirs|p", i18n.G("Create any directories necessary")) cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G("Force creating files or directories")) cli.AddIntFlag(cmd.Flags(), &c.storageVolumeFile.flagGID, "gid", -1, i18n.G("Set the file's gid on create")) cli.AddIntFlag(cmd.Flags(), &c.storageVolumeFile.flagUID, "uid", -1, i18n.G("Set the file's uid on create")) cli.AddStringFlag(cmd.Flags(), &c.storageVolumeFile.flagMode, "mode", "", "", i18n.G("Set the file's perms on create")) cli.AddStringFlag(cmd.Flags(), &c.flagType, "type|t", "file", "", i18n.G("The type to create (file, symlink, or directory)")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpFiles(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeFileCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeFileCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volName := parsed[1].List[0].String targetPath, isDir := normalizePath(parsed[1].List[1].String) isSymlink := !parsed[2].Skipped symlinkTargetPath := filepath.Clean(parsed[2].String) if !slices.Contains([]string{"file", "symlink", "directory"}, c.flagType) { return fmt.Errorf(i18n.G("Invalid type %q"), c.flagType) } if isSymlink && c.flagType != "symlink" { return errors.New(i18n.G(`Symlink target path can only be used for type "symlink"`)) } if isDir { c.flagType = "directory" } // Connect to SFTP. sftpConn, err := d.GetStoragePoolVolumeFileSFTP(poolName, "custom", volName) if err != nil { return err } defer func() { _ = sftpConn.Close() }() // Determine the target uid uid := max(c.storageVolumeFile.flagUID, 0) // Determine the target gid gid := max(c.storageVolumeFile.flagGID, 0) var mode os.FileMode // Determine the target mode switch c.flagType { case "directory": mode = os.FileMode(DirMode) case "file": mode = os.FileMode(FileMode) } if c.storageVolumeFile.flagMode != "" { if len(c.storageVolumeFile.flagMode) == 3 { c.storageVolumeFile.flagMode = "0" + c.storageVolumeFile.flagMode } m, err := strconv.ParseInt(c.storageVolumeFile.flagMode, 0, 0) if err != nil { return err } mode = os.FileMode(m) } // Create needed paths if requested if c.storageVolumeFile.flagMkdir { err := sftpRecursiveMkdir(sftpConn, filepath.Dir(targetPath), nil, int64(uid), int64(gid)) if err != nil { return err } } var content io.ReadSeeker var readCloser io.ReadCloser var contentLength int64 switch c.flagType { case "symlink": content = strings.NewReader(symlinkTargetPath) readCloser = io.NopCloser(content) contentLength = int64(len(symlinkTargetPath)) case "file": // Just creating an empty file. content = strings.NewReader("") readCloser = io.NopCloser(content) contentLength = 0 } fileArgs := incus.InstanceFileArgs{ Type: c.flagType, UID: int64(uid), GID: int64(gid), Mode: int(mode.Perm()), Content: content, } if c.flagForce { fileArgs.WriteMode = "overwrite" } progress := cli.ProgressRenderer{ Format: fmt.Sprintf(i18n.G("Creating %s: %%s"), targetPath), Quiet: c.global.flagQuiet, } if readCloser != nil { fileArgs.Content = internalIO.NewReadSeeker(&ioprogress.ProgressReader{ ReadCloser: readCloser, Tracker: &ioprogress.ProgressTracker{ Length: contentLength, Handler: func(percent int64, speed int64) { progress.UpdateProgress(ioprogress.ProgressData{ Text: fmt.Sprintf("%d%% (%s/s)", percent, units.GetByteSizeString(speed, 2)), }) }, }, }, fileArgs.Content) } err = sftpCreateFile(sftpConn, targetPath, fileArgs, true) if err != nil { progress.Done("") return err } progress.Done("") return nil } // Delete. type cmdStorageVolumeFileDelete struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume storageVolumeFile *cmdStorageVolumeFile flagForce bool } var cmdStorageVolumeFileDeleteUsage = u.Usage{u.Pool.Remote(), u.MakePath(u.Volume, u.Path)} func (c *cmdStorageVolumeFileDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdStorageVolumeFileDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete files in custom volume") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete files in custom volume`)) cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G("Force deleting files, directories, and subdirectories")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { return c.global.cmpFiles(toComplete, false) } return cmd } func (c *cmdStorageVolumeFileDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeFileDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volName := parsed[1].List[0].String fPath := parsed[1].List[1].String // Connect to SFTP. sftpConn, err := d.GetStoragePoolVolumeFileSFTP(poolName, "custom", volName) if err != nil { return err } defer func() { _ = sftpConn.Close() }() if c.flagForce { err = sftpConn.RemoveAll(fPath) if err != nil { return err } return nil } err = sftpConn.Remove(fPath) if err != nil { return err } return nil } // Mount. type cmdStorageVolumeFileMount struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume storageVolumeFile *cmdStorageVolumeFile flagListen string flagAuthNone bool flagAuthUser string } var cmdStorageVolumeFileMountUsage = u.Usage{u.Pool.Remote(), u.Volume, u.Target(u.Path).Optional()} func (c *cmdStorageVolumeFileMount) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("mount", cmdStorageVolumeFileMountUsage...) cmd.Short = i18n.G("Mount files from custom storage volumes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Mount files from custom storage volumes. If no target path is provided, start an SSH SFTP listener instead.`)) cmd.Example = cli.FormatSection("", i18n.G(`incus storage volume file mount mypool myvolume localdir To mount the storage volume myvolume from pool mypool onto the local directory localdir. incus storage volume file mount mypool myvolume To start an SSH SFTP listener for the storage volume myvolume from pool mypool.`)) cli.AddStringFlag(cmd.Flags(), &c.flagListen, "listen", "", "", i18n.G("Setup SSH SFTP listener on address:port instead of mounting")) cli.AddBoolFlag(cmd.Flags(), &c.flagAuthNone, "no-auth", i18n.G("Disable authentication when using SSH SFTP listener")) cli.AddStringFlag(cmd.Flags(), &c.flagAuthUser, "auth-user", "", "", i18n.G("Set authentication user when using SSH SFTP listener")) cmd.RunE = c.run // completion for pool, volume, host path cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } if len(args) == 2 { return nil, cobra.ShellCompDirectiveDefault } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeFileMount) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeFileMountUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volName := parsed[1].String hasTargetPath := !parsed[2].Skipped targetPath := filepath.Clean(parsed[2].String) entity := poolName + "/custom/" + volName // Determine the target if specified. if hasTargetPath { sb, err := os.Stat(targetPath) if err != nil { return err } if !sb.IsDir() { return errors.New(i18n.G("Target path must be a directory")) } // Check which mode we should operate in. If target path is provided we use sshfs mode. if c.flagListen != "" { return errors.New(i18n.G("Target path and --listen flag cannot be used together")) } // Connect to SFTP. sftpConn, err := d.GetStoragePoolVolumeFileSFTPConn(poolName, "custom", volName) if err != nil { return fmt.Errorf(i18n.G("Failed connecting to instance SFTP: %w"), err) } defer func() { _ = sftpConn.Close() }() return sshfsMount(cmd.Context(), sftpConn, entity, "", targetPath) } // Check if the pool and the volume exist before starting the SFTP server. _, _, err = d.GetStoragePoolVolume(poolName, "custom", volName) if err != nil { return err } return sshSFTPServer(cmd.Context(), func() (net.Conn, error) { return d.GetStoragePoolVolumeFileSFTPConn(poolName, "custom", volName) }, c.flagAuthNone, c.flagAuthUser, c.flagListen) } // Edit. type cmdStorageVolumeFileEdit struct { global *cmdGlobal filePull *cmdStorageVolumeFilePull filePush *cmdStorageVolumeFilePush } var cmdStorageVolumeFileEditUsage = u.Usage{u.Pool.Remote(), u.MakePath(u.Volume, u.Path)} func (c *cmdStorageVolumeFileEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("edit", cmdStorageVolumeFileEditUsage...) cmd.Short = i18n.G("Edit files in storage volumes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Edit files in storage volumes`)) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpFiles(toComplete, false) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeFileEdit) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeFileEditUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } fPath := parsed[1].List[1].String c.filePush.noModeChange = true // If stdin isn't a terminal, read text from it if !termios.IsTerminal(getStdinFd()) { return c.filePush.push(os.Stdin.Name(), parsed[0], parsed[1]) } // Create temp file f, err := os.CreateTemp("", fmt.Sprintf("incus_file_edit_*%s", filepath.Ext(fPath))) if err != nil { return fmt.Errorf(i18n.G("Unable to create a temporary file: %v"), err) } fname := f.Name() _ = f.Close() _ = os.Remove(fname) // Tell pull/push that they're called from edit. c.filePull.edit = true c.filePush.edit = true // Extract current value defer func() { _ = os.Remove(fname) }() err = c.filePull.pull(parsed[0], parsed[1], fname) if err != nil { return err } // Spawn the editor _, err = cli.TextEditor(fname, []byte{}) if err != nil { return err } // Push the result err = c.filePush.push(fname, parsed[0], parsed[1]) if err != nil { return err } return nil } // Pull. type cmdStorageVolumeFilePull struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume storageVolumeFile *cmdStorageVolumeFile puller *pullable edit bool } var cmdStorageVolumeFilePullUsage = u.Usage{u.Pool.Remote(), u.MakePath(u.Volume, u.Path), u.Target(u.Path)} func (c *cmdStorageVolumeFilePull) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("pull", cmdStorageVolumeFilePullUsage...) cmd.Short = i18n.G("Pull files from custom volumes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Pull files from custom volumes`)) cmd.Example = cli.FormatSection("", i18n.G( `incus custom volume file pull local v1/foo/etc/hosts . To pull /etc/hosts from the custom volume and write it to the current directory. incus file pull local v1 foo/etc/hosts - To pull /etc/hosts from the custom volume and write its output to standard output.`)) cli.AddBoolFlag(cmd.Flags(), &c.storageVolumeFile.flagMkdir, "create-dirs|p", i18n.G("Create any directories necessary")) cli.AddBoolFlag(cmd.Flags(), &c.puller.flagRecursive, "recursive|r", i18n.G("Recursively transfer files")) cli.AddBoolFlag(cmd.Flags(), &c.puller.flagNoDereference, "no-dereference|P", i18n.G("Never follow symbolic links in source path")) cli.AddBoolFlag(cmd.Flags(), &c.puller.flagFollow, "follow|H", i18n.G("Follow command-line symbolic links in source path")) cli.AddBoolFlag(cmd.Flags(), &c.puller.flagDereference, "dereference|L", i18n.G("Always follow symbolic links in source path")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpFiles(toComplete, false) } return c.global.cmpFiles(toComplete, true) } return cmd } // pull runs the post-parsing command logic. func (c *cmdStorageVolumeFilePull) pull(parsedPool *u.Parsed, parsedPath *u.Parsed, target string) error { d := parsedPool.RemoteServer poolName := parsedPool.RemoteObject.String volName := parsedPath.List[0].String fPath := "/" + parsedPath.List[1].String targetIsDir := strings.HasSuffix(target, "/") targetExists := true targetInfo, err := os.Stat(target) if err != nil { if !errors.Is(err, fs.ErrNotExist) { return err } targetExists = false } err = c.puller.preCheck(target) if err != nil { return err } /* * If the path exists, just use it. If it doesn't exist, it might be a * directory in one of two cases: * 1. Someone explicitly put "/" at the end * 2. We are dealing with recursive copy */ if targetExists { targetIsDir = targetInfo.IsDir() } else if targetIsDir { err := os.MkdirAll(target, DirMode) if err != nil { return err } } else if c.storageVolumeFile.flagMkdir { err := os.MkdirAll(filepath.Dir(target), DirMode) if err != nil { return err } } // Connect to SFTP. sftpConn, err := d.GetStoragePoolVolumeFileSFTP(poolName, "custom", volName) if err != nil { return fmt.Errorf(i18n.G("Failed connecting to instance SFTP: %w"), err) } defer func() { _ = sftpConn.Close() }() srcInfo, normalizedPath, err := c.puller.statFile(sftpConn, fPath) if err != nil { return err } // Recursively copy directories. if srcInfo.IsDir() { return sftpRecursivePullFile(sftpConn, srcInfo, fPath, normalizedPath, target, c.global.flagQuiet, c.puller.flagDereference, util.PathExists(target)) } var targetPath string if targetIsDir { targetPath = filepath.Join(target, filepath.Base(normalizedPath)) } else { targetPath = target } // Prepare target. targetIsLink := srcInfo.Mode()&os.ModeSymlink != 0 var f *os.File var linkName string if isStdout(targetPath) { f = os.Stdout } else if targetIsLink { linkName, err = sftpConn.ReadLink(normalizedPath) if err != nil { return err } } else { f, err = os.Create(targetPath) if err != nil { return err } defer func() { _ = f.Close() }() // nolint:revive err = os.Chmod(targetPath, os.FileMode(srcInfo.Mode())) if err != nil { return err } } progress := cli.ProgressRenderer{ Format: fmt.Sprintf(i18n.G("Pulling %s from %s: %%s"), targetPath, normalizedPath), Quiet: c.global.flagQuiet, } writer := &ioprogress.ProgressWriter{ WriteCloser: f, Tracker: &ioprogress.ProgressTracker{ Handler: func(bytesReceived int64, speed int64) { if isStdout(targetPath) { return } progress.UpdateProgress(ioprogress.ProgressData{ Text: fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(bytesReceived, 2), units.GetByteSizeString(speed, 2)), }) }, }, } if targetIsLink { err = os.Symlink(linkName, targetPath) if err != nil { progress.Done("") return err } } else { src, err := sftpConn.Open(normalizedPath) if err != nil { return err } defer func() { _ = src.Close() }() _, err = util.SafeCopy(writer, src) if err != nil { progress.Done("") return err } } progress.Done("") return nil } func (c *cmdStorageVolumeFilePull) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeFilePullUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.pull(parsed[0], parsed[1], parsed[2].String) } // Push. type cmdStorageVolumeFilePush struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume storageVolumeFile *cmdStorageVolumeFile pusher *pushable edit bool noModeChange bool } var cmdStorageVolumeFilePushUsage = u.Usage{u.Path, u.Pool.Remote(), u.MakePath(u.Volume, u.Target(u.Path))} func (c *cmdStorageVolumeFilePush) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("push", cmdStorageVolumeFilePushUsage...) cmd.Short = i18n.G("Push files into custom volumes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Push files into custom volumes`)) cmd.Example = cli.FormatSection("", i18n.G( `incus storage volume file push /etc/hosts local v1/etc/hosts To push /etc/hosts into the custom volume "v1". echo "Hello world" | incus storage volume file push - local v1 test To read "Hello world" from standard input and write it into test in volume "v1".`)) cli.AddBoolFlag(cmd.Flags(), &c.storageVolumeFile.flagMkdir, "create-dirs|p", i18n.G("Create any directories necessary")) cli.AddIntFlag(cmd.Flags(), &c.storageVolumeFile.flagUID, "uid", -1, i18n.G("Set the file's uid on push")) cli.AddIntFlag(cmd.Flags(), &c.storageVolumeFile.flagGID, "gid", -1, i18n.G("Set the file's gid on push")) cli.AddStringFlag(cmd.Flags(), &c.storageVolumeFile.flagMode, "mode", "", "", i18n.G("Set the file's perms on push")) cli.AddBoolFlag(cmd.Flags(), &c.pusher.flagRecursive, "recursive|r", i18n.G("Recursively transfer files")) cli.AddBoolFlag(cmd.Flags(), &c.pusher.flagNoDereference, "no-dereference|P", i18n.G("Never follow symbolic links in source path")) cli.AddBoolFlag(cmd.Flags(), &c.pusher.flagFollow, "follow|H", i18n.G("Follow command-line symbolic links in source path")) cli.AddBoolFlag(cmd.Flags(), &c.pusher.flagDereference, "dereference|L", i18n.G("Always follow symbolic links in source path")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return nil, cobra.ShellCompDirectiveDefault } return c.global.cmpFiles(toComplete, true) } return cmd } // push runs the post-parsing command logic. func (c *cmdStorageVolumeFilePush) push(srcFile string, parsedPool *u.Parsed, parsedTarget *u.Parsed) error { d := parsedPool.RemoteServer poolName := parsedPool.RemoteObject.String volName := parsedTarget.List[0].String target, targetIsDir := normalizePath(parsedTarget.List[1].String) targetExists := false // Connect to SFTP. sftpConn, err := d.GetStoragePoolVolumeFileSFTP(poolName, "custom", volName) if err != nil { return fmt.Errorf(i18n.G("Failed connecting to instance SFTP: %w"), err) } defer func() { _ = sftpConn.Close() }() targetInfo, err := sftpConn.Stat(target) if err == nil { targetExists = true if targetInfo.IsDir() { targetIsDir = true } else if targetIsDir { // Let’s be extra careful and check that explicit requests for directories actually point to // directories. return fmt.Errorf(i18n.G("%s is not a directory"), target) } } var mode os.FileMode if c.storageVolumeFile.flagMode != "" { if len(c.storageVolumeFile.flagMode) == 3 { c.storageVolumeFile.flagMode = "0" + c.storageVolumeFile.flagMode } m, err := strconv.ParseInt(c.storageVolumeFile.flagMode, 0, 0) if err != nil { return err } mode = os.FileMode(m) } // Push the files var f *os.File var linkTarget string var size int64 m := mode uid := max(c.storageVolumeFile.flagUID, 0) gid := max(c.storageVolumeFile.flagGID, 0) if isStdin(srcFile) { if targetIsDir { return errors.New(i18n.G("A target file name must be specified when pushing from stdin; the target is a directory")) } f = os.Stdin } else { srcInfo, wPath, err := c.pusher.statFile(srcFile) if err != nil { return err } // Recursively copy directories. if srcInfo.IsDir() { return sftpRecursivePushFile(sftpConn, wPath, srcFile, target, c.global.flagQuiet, c.pusher.flagDereference, targetExists) } if srcInfo.Mode()&os.ModeSymlink != 0 { linkTarget, err = os.Readlink(srcFile) if err != nil { return err } } else { f, err = os.Open(srcFile) if err != nil { return fmt.Errorf(i18n.G("Failed to open source file %q: %v"), f, err) } size = srcInfo.Size() defer func() { _ = f.Close() }() } if c.storageVolumeFile.flagUID == -1 || c.storageVolumeFile.flagGID == -1 { dMode, dUID, dGID := internalIO.GetOwnerMode(srcInfo) if c.storageVolumeFile.flagMode == "" { m = dMode } if c.storageVolumeFile.flagUID == -1 { uid = dUID } if c.storageVolumeFile.flagGID == -1 { gid = dGID } } } // Determine the target path. var targetPath string if targetIsDir { targetPath = filepath.Join(target, filepath.Base(srcFile)) } else { targetPath = target } // Create needed paths if requested if c.storageVolumeFile.flagMkdir { mode := os.FileMode(DirMode) err = sftpRecursiveMkdir(sftpConn, filepath.Dir(targetPath), &mode, int64(uid), int64(gid)) if err != nil { return err } } // Transfer the files. args := incus.InstanceFileArgs{ UID: -1, GID: -1, Mode: -1, } // Check if the path already exists. _, err = sftpConn.Stat(targetPath) fileExists := err == nil if !c.noModeChange { if !fileExists || c.storageVolumeFile.flagUID != -1 { args.UID = int64(uid) } if !fileExists || c.storageVolumeFile.flagGID != -1 { args.GID = int64(gid) } if !fileExists || c.storageVolumeFile.flagMode != "" { args.Mode = int(m.Perm()) } } progress := cli.ProgressRenderer{ Format: fmt.Sprintf(i18n.G("Pushing %s to %s: %%s"), srcFile, targetPath), Quiet: c.global.flagQuiet, } if f == nil { args.Type = "symlink" args.Content = strings.NewReader(linkTarget) } else { args.Type = "file" args.Content = internalIO.NewReadSeeker(&ioprogress.ProgressReader{ ReadCloser: f, Tracker: &ioprogress.ProgressTracker{ Length: size, Handler: func(percent int64, speed int64) { progress.UpdateProgress(ioprogress.ProgressData{ Text: fmt.Sprintf("%d%% (%s/s)", percent, units.GetByteSizeString(speed, 2)), }) }, }, }, f) } logger.Infof("Pushing %s to %s (%s)", srcFile, targetPath, args.Type) err = sftpCreateFile(sftpConn, targetPath, args, true) progress.Done("") return err } func (c *cmdStorageVolumeFilePush) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeFilePushUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } return c.push(parsed[0].String, parsed[1], parsed[2]) } // Snapshot. type cmdStorageVolumeSnapshot struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume } func (c *cmdStorageVolumeSnapshot) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("snapshot") cmd.Short = i18n.G("Manage storage volume snapshots") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage storage volume snapshots`)) // Create storageVolumeSnapshotCreateCmd := cmdStorageVolumeSnapshotCreate{global: c.global, storage: c.storage, storageVolume: c.storageVolume, storageVolumeSnapshot: c} cmd.AddCommand(storageVolumeSnapshotCreateCmd.command()) // Delete storageVolumeSnapshotDeleteCmd := cmdStorageVolumeSnapshotDelete{global: c.global, storage: c.storage, storageVolume: c.storageVolume, storageVolumeSnapshot: c} cmd.AddCommand(storageVolumeSnapshotDeleteCmd.command()) // List storageVolumeSnapshotListCmd := cmdStorageVolumeSnapshotList{global: c.global, storage: c.storage, storageVolume: c.storageVolume, storageVolumeSnapshot: c} cmd.AddCommand(storageVolumeSnapshotListCmd.command()) // Rename storageVolumeSnapshotRenameCmd := cmdStorageVolumeSnapshotRename{global: c.global, storage: c.storage, storageVolume: c.storageVolume, storageVolumeSnapshot: c} cmd.AddCommand(storageVolumeSnapshotRenameCmd.command()) // Restore storageVolumeSnapshotRestoreCmd := cmdStorageVolumeSnapshotRestore{global: c.global, storage: c.storage, storageVolume: c.storageVolume, storageVolumeSnapshot: c} cmd.AddCommand(storageVolumeSnapshotRestoreCmd.command()) // Restore storageVolumeSnapshotShowCmd := cmdStorageVolumeSnapshotShow{global: c.global, storage: c.storage, storageVolume: c.storageVolume, storageVolumeSnapshot: c} cmd.AddCommand(storageVolumeSnapshotShowCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // Snapshot create. type cmdStorageVolumeSnapshotCreate struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume storageVolumeSnapshot *cmdStorageVolumeSnapshot flagNoExpiry bool flagExpiry string flagReuse bool flagDescription string } var cmdStorageVolumeSnapshotCreateUsage = u.Usage{u.Pool.Remote(), u.Volume, u.NewName(u.Snapshot).Optional()} func (c *cmdStorageVolumeSnapshotCreate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("create", cmdStorageVolumeSnapshotCreateUsage...) cmd.Aliases = []string{"add"} cmd.Short = i18n.G("Snapshot storage volumes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Snapshot storage volumes`)) cmd.Example = cli.FormatSection("", i18n.G(`incus storage volume snapshot create default foo snap0 Create a snapshot of "foo" in pool "default" called "snap0" incus storage volume snapshot create default vol1 snap0 < config.yaml Create a snapshot of "foo" in pool "default" called "snap0" with the configuration from "config.yaml"`)) cli.AddStringFlag(cmd.Flags(), &c.flagExpiry, "expiry", "", "", i18n.G("Expiry date or time span for the new snapshot")) cli.AddBoolFlag(cmd.Flags(), &c.flagNoExpiry, "no-expiry", i18n.G("Ignore any configured auto-expiry for the storage volume")) cli.AddBoolFlag(cmd.Flags(), &c.flagReuse, "reuse", i18n.G("If the snapshot name already exists, delete and create a new one")) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddStringFlag(cmd.Flags(), &c.flagDescription, "description", "", "", i18n.G("Snapshot description")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeSnapshotCreate) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeSnapshotCreateUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volName := parsed[1].String hasSnapName := !parsed[2].Skipped snapName := parsed[2].String if c.flagNoExpiry && c.flagExpiry != "" { return errors.New(i18n.G("Can't use both --no-expiry and --expiry")) } // If stdin isn't a terminal, read text from it var stdinData api.StorageVolumeSnapshotPut if !termios.IsTerminal(getStdinFd()) { loader, err := yaml.NewLoader(os.Stdin) if err != nil { return err } err = loader.Load(&stdinData) if err != nil && !errors.Is(err, io.EOF) { return err } } // Use the provided target. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } // Check if the requested storage volume actually exists _, _, err = d.GetStoragePoolVolume(poolName, "custom", volName) if err != nil { return err } req := api.StorageVolumeSnapshotsPost{ Name: snapName, } if c.flagNoExpiry { req.ExpiresAt = &time.Time{} } else if c.flagExpiry != "" { // Try to parse as a duration. expiry, err := instance.GetExpiry(time.Now(), c.flagExpiry) if err != nil { if !errors.Is(err, instance.ErrInvalidExpiry) { return err } // Fallback to date parsing. expiry, err = time.Parse(dateLayout, c.flagExpiry) if err != nil { return err } } req.ExpiresAt = &expiry } else if stdinData.ExpiresAt != nil && !stdinData.ExpiresAt.IsZero() { req.ExpiresAt = stdinData.ExpiresAt } if c.flagReuse && hasSnapName { snap, _, _ := d.GetStoragePoolVolumeSnapshot(poolName, "custom", volName, snapName) if snap != nil { op, err := d.DeleteStoragePoolVolumeSnapshot(poolName, "custom", volName, snapName) if err != nil { return err } err = op.Wait() if err != nil { return err } } } op, err := d.CreateStoragePoolVolumeSnapshot(poolName, "custom", volName, req) if err != nil { return err } err = op.Wait() if err != nil { return err } opInfo := op.Get() snapshots, ok := opInfo.Resources["storage_volume_snapshots"] if !ok || len(snapshots) == 0 { return errors.New(i18n.G("Didn't get name of new volume snapshot from the server")) } if len(snapshots) == 1 && !hasSnapName { uri, err := url.Parse(snapshots[0]) if err != nil { return err } fmt.Printf(i18n.G("Volume snapshot name is: %s")+"\n", path.Base(uri.Path)) } return nil } // Snapshot delete. type cmdStorageVolumeSnapshotDelete struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume storageVolumeSnapshot *cmdStorageVolumeSnapshot } var cmdStorageVolumeSnapshotDeleteUsage = u.Usage{u.Pool.Remote(), u.Volume, u.Snapshot} func (c *cmdStorageVolumeSnapshotDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdStorageVolumeSnapshotDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete storage volume snapshots") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete storage volume snapshots`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } if len(args) == 2 { return c.global.cmpStoragePoolVolumeSnapshots(args[0], args[1]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeSnapshotDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeSnapshotDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volName := parsed[1].String snapName := parsed[2].String // If a target was specified, delete the volume on the given member. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } // Delete the snapshot op, err := d.DeleteStoragePoolVolumeSnapshot(poolName, "custom", volName, snapName) if err != nil { return err } err = op.Wait() if err != nil { return err } if !c.global.flagQuiet { fmt.Printf(i18n.G("Storage volume snapshot %s deleted from %s")+"\n", snapName, volName) } return nil } // Snapshot list. type cmdStorageVolumeSnapshotList struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume storageVolumeSnapshot *cmdStorageVolumeSnapshot flagFormat string flagColumns string flagAllProjects bool defaultColumns string } var cmdStorageVolumeSnapshotListUsage = u.Usage{u.Pool.Remote(), u.MakePath(u.StorageVolumeType.Optional(), u.Volume)} func (c *cmdStorageVolumeSnapshotList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdStorageVolumeSnapshotListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List storage volume snapshots") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`List storage volume snapshots`)) c.defaultColumns = "nTE" cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", c.defaultColumns, "", i18n.G("Columns")) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("All projects")) cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List storage volume snapshots The -c option takes a (optionally comma-separated) list of arguments that control which image attributes to output when displaying in table or csv format. Column shorthand chars: n - Name T - Taken at E - Expiry`)) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeSnapshotList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeSnapshotListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volType := parsed[1].List[0].Get("custom") volName := parsed[1].List[1].String // Check if the requested storage volume actually exists _, _, err = d.GetStoragePoolVolume(poolName, volType, volName) if err != nil { return err } return c.listSnapshots(d, poolName, volType, volName) } func (c *cmdStorageVolumeSnapshotList) listSnapshots(d incus.InstanceServer, poolName string, volumeType string, volumeName string) error { snapshots, err := d.GetStoragePoolVolumeSnapshots(poolName, volumeType, volumeName) if err != nil { return err } // Parse column flags. columns, err := c.parseColumns() if err != nil { return err } data := [][]string{} for _, snap := range snapshots { line := []string{} for _, column := range columns { line = append(line, column.Data(snap)) } data = append(data, line) } header := []string{} for _, column := range columns { header = append(header, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, header, data, snapshots) } type storageVolumeSnapshotColumn struct { Name string Data func(api.StorageVolumeSnapshot) string } func (c *cmdStorageVolumeSnapshotList) parseColumns() ([]storageVolumeSnapshotColumn, error) { columnsShorthandMap := map[rune]storageVolumeSnapshotColumn{ 'n': {i18n.G("NAME"), c.nameColumnData}, 'T': {i18n.G("TAKEN AT"), c.takenAtColumnData}, 'E': {i18n.G("EXPIRES AT"), c.expiresAtColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []storageVolumeSnapshotColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdStorageVolumeSnapshotList) nameColumnData(snapshot api.StorageVolumeSnapshot) string { _, snapName, _ := api.GetParentAndSnapshotName(snapshot.Name) return snapName } func (c *cmdStorageVolumeSnapshotList) takenAtColumnData(snapshot api.StorageVolumeSnapshot) string { if snapshot.CreatedAt.IsZero() { return " " } return snapshot.CreatedAt.Local().Format(dateLayout) } func (c *cmdStorageVolumeSnapshotList) expiresAtColumnData(snapshot api.StorageVolumeSnapshot) string { if snapshot.ExpiresAt == nil || snapshot.ExpiresAt.IsZero() { return " " } return snapshot.ExpiresAt.Local().Format(dateLayout) } // Snapshot rename. type cmdStorageVolumeSnapshotRename struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume storageVolumeSnapshot *cmdStorageVolumeSnapshot } var cmdStorageVolumeSnapshotRenameUsage = u.Usage{u.Pool.Remote(), u.Volume, u.Snapshot, u.NewName(u.Snapshot)} func (c *cmdStorageVolumeSnapshotRename) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("rename", cmdStorageVolumeSnapshotRenameUsage...) cmd.Short = i18n.G("Rename storage volume snapshots") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Rename storage volume snapshots`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } if len(args) == 2 { return c.global.cmpStoragePoolVolumeSnapshots(args[0], args[1]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeSnapshotRename) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeSnapshotRenameUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volName := parsed[1].String snapName := parsed[2].String newSnapName := parsed[3].String // If a target member was specified, get the volume with the matching // name on that member, if any. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } op, err := d.RenameStoragePoolVolumeSnapshot(poolName, "custom", volName, snapName, api.StorageVolumeSnapshotPost{Name: newSnapName}) if err != nil { return err } err = op.Wait() if err != nil { return err } fmt.Printf(i18n.G(`Renamed storage volume snapshot from "%s" to "%s"`)+"\n", snapName, newSnapName) return nil } // Snapshot restore. type cmdStorageVolumeSnapshotRestore struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume storageVolumeSnapshot *cmdStorageVolumeSnapshot } var cmdStorageVolumeSnapshotRestoreUsage = u.Usage{u.Pool.Remote(), u.Volume, u.Snapshot} func (c *cmdStorageVolumeSnapshotRestore) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("restore", cmdStorageVolumeSnapshotRestoreUsage...) cmd.Short = i18n.G("Restore storage volume snapshots") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Restore storage volume snapshots`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } if len(args) == 2 { return c.global.cmpStoragePoolVolumeSnapshots(args[0], args[1]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeSnapshotRestore) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeSnapshotRestoreUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volName := parsed[1].String snapName := parsed[2].String // Use the provided target. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } // Check if the requested storage volume actually exists _, etag, err := d.GetStoragePoolVolume(poolName, "custom", volName) if err != nil { return err } return d.UpdateStoragePoolVolume(poolName, "custom", volName, api.StorageVolumePut{Restore: snapName}, etag) } // Snapshot show. type cmdStorageVolumeSnapshotShow struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume storageVolumeSnapshot *cmdStorageVolumeSnapshot } var cmdStorageVolumeSnapshotShowUsage = u.Usage{u.Pool.Remote(), u.MakePath(u.StorageVolumeType.Optional(), u.Volume), u.Snapshot} func (c *cmdStorageVolumeSnapshotShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdStorageVolumeSnapshotShowUsage...) cmd.Short = i18n.G("Show storage volume snapshot configurations") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Show storage volume snapshhot configurations`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdStorageVolumeSnapshotShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeSnapshotShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volType := parsed[1].List[0].Get("custom") volName := parsed[1].List[1].String snapName := parsed[2].String // If a target member was specified, get the volume with the matching // name on that member, if any. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } // Get the storage volume entry vol, _, err := d.GetStoragePoolVolumeSnapshot(poolName, volType, volName, snapName) if err != nil { return err } data, err := yaml.Dump(&vol, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Export. type cmdStorageVolumeExport struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume flagVolumeOnly bool flagOptimizedStorage bool flagCompressionAlgorithm string flagForce bool } var cmdStorageVolumeExportUsage = u.Usage{u.Pool.Remote(), u.Volume, u.Target(u.File).Optional()} func (c *cmdStorageVolumeExport) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("export", cmdStorageVolumeExportUsage...) cmd.Short = i18n.G("Export custom storage volumes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Export custom storage volumes.`)) cli.AddBoolFlag(cmd.Flags(), &c.flagVolumeOnly, "volume-only", i18n.G("Export the volume without its snapshots (ignored for ISO storage volumes)")) cli.AddBoolFlag(cmd.Flags(), &c.flagOptimizedStorage, "optimized-storage", i18n.G("Use storage driver optimized format (can only be restored on a similar pool, ignored for ISO storage volumes)")) cli.AddStringFlag(cmd.Flags(), &c.flagCompressionAlgorithm, "compression", "", "", i18n.G("Compression algorithm to use (none for uncompressed, ignored for ISO storage volumes)")) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cli.AddBoolFlag(cmd.Flags(), &c.flagForce, "force|f", i18n.G("Force overwriting existing backup file")) cmd.RunE = c.run cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } return nil, cobra.ShellCompDirectiveDefault } return cmd } func (c *cmdStorageVolumeExport) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeExportUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volName := parsed[1].String hasTarget := !parsed[2].Skipped targetName := parsed[2].Get("." + volName + ".backup") // Use the provided target. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } volumeOnly := c.flagVolumeOnly // Get the storage volume entry. vol, _, err := d.GetStoragePoolVolume(poolName, "custom", volName) if err != nil { return fmt.Errorf("Storage pool volume \"custom/%s\" not found", volName) } if !hasTarget && vol.ContentType == "iso" { targetName = volName + ".iso" hasTarget = true } if isStdout(targetName) { // If outputting to stdout, quiesce the output. c.global.flagQuiet = true } else if hasTarget && !c.flagForce && util.PathExists(targetName) { // Check if the target path already exists. return fmt.Errorf(i18n.G("Target path %q already exists"), targetName) } req := api.StorageVolumeBackupsPost{ Name: "", ExpiresAt: time.Now().Add(24 * time.Hour), VolumeOnly: volumeOnly, OptimizedStorage: c.flagOptimizedStorage, CompressionAlgorithm: c.flagCompressionAlgorithm, } var getter func(backupReq *incus.BackupFileRequest) error if d.HasExtension("direct_backup") { getter = func(backupReq *incus.BackupFileRequest) error { return d.CreateStorageVolumeBackupStream(poolName, volName, req, backupReq) } } else { op, err := d.CreateStorageVolumeBackup(poolName, volName, req) if err != nil { return fmt.Errorf(i18n.G("Failed to create storage volume backup: %w"), err) } // Watch the background operation. progress := cli.ProgressRenderer{ Format: i18n.G("Backing up storage volume: %s"), Quiet: c.global.flagQuiet, } _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } // Wait until backup is done. err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } progress.Done("") err = op.Wait() if err != nil { return err } // Get name of backup. uStr := op.Get().Resources["backups"][0] uri, err := url.Parse(uStr) if err != nil { return fmt.Errorf(i18n.G("Invalid URL %q: %w"), uStr, err) } backupName, err := url.PathUnescape(path.Base(uri.EscapedPath())) if err != nil { return fmt.Errorf(i18n.G("Invalid backup name segment in path %q: %w"), uri.EscapedPath(), err) } defer func() { // Delete backup after we're done. op, err = d.DeleteStorageVolumeBackup(poolName, volName, backupName) if err == nil { _ = op.Wait() } }() getter = func(backupReq *incus.BackupFileRequest) error { _, err := d.GetStorageVolumeBackupFile(poolName, volName, backupName, backupReq) return err } } var target *os.File if isStdout(targetName) { target = os.Stdout } else { target, err = os.Create(targetName) if err != nil { return err } defer func() { _ = target.Close() }() } // Prepare the download request. progress := cli.ProgressRenderer{ Format: i18n.G("Exporting the backup: %s"), Quiet: c.global.flagQuiet, } backupFileRequest := incus.BackupFileRequest{ BackupFile: io.WriteSeeker(target), ProgressHandler: progress.UpdateProgress, } // Export tarball err = getter(&backupFileRequest) if err != nil { _ = os.Remove(targetName) progress.Done("") return fmt.Errorf(i18n.G("Failed to fetch storage volume backup file: %w"), err) } // Detect backup file type and rename file accordingly. if !hasTarget { _, err := target.Seek(0, io.SeekStart) if err != nil { return err } _, ext, _, err := archive.DetectCompressionFile(target) if err != nil { return err } err = os.Rename(targetName, volName+ext) if err != nil { return fmt.Errorf(i18n.G("Failed to rename export file: %w"), err) } } err = target.Close() if err != nil { return fmt.Errorf(i18n.G("Failed to close export file: %w"), err) } progress.Done(i18n.G("Backup exported successfully!")) return nil } // Import. type cmdStorageVolumeImport struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume flagType string } var cmdStorageVolumeImportUsage = u.Usage{u.Pool.Remote(), u.BackupFile, u.NewName(u.Volume).Optional()} func (c *cmdStorageVolumeImport) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("import", cmdStorageVolumeImportUsage...) cmd.Short = i18n.G("Import custom storage volumes") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Import custom storage volumes.`)) cmd.Example = cli.FormatSection("", i18n.G(`incus storage volume import default backup0.tar.gz Create a new custom volume using backup0.tar.gz as the source incus storage volume import default some-installer.iso installer --type=iso Create a new custom volume storing some-installer.iso for use as a CD-ROM image`)) cli.AddStringFlag(cmd.Flags(), &c.storage.flagTarget, "target", "", "", i18n.G("Cluster member name")) cmd.RunE = c.run cli.AddStringFlag(cmd.Flags(), &c.flagType, "type|t", "", "", i18n.G("Import type, backup or iso (default \"backup\")")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } return nil, cobra.ShellCompDirectiveDefault } return cmd } func (c *cmdStorageVolumeImport) run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeImportUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String backupFile := parsed[1].String hasVolName := !parsed[2].Skipped volName := parsed[2].String // Use the provided target. if c.storage.flagTarget != "" { d = d.UseTarget(c.storage.flagTarget) } var file *os.File if isStdin(backupFile) { file = os.Stdin } else { file, err = os.Open(backupFile) if err != nil { return err } defer func() { _ = file.Close() }() } fstat, err := file.Stat() if err != nil { return err } if c.flagType == "" { // Set type to iso if filename suffix is .iso. if strings.HasSuffix(strings.ToLower(backupFile), ".iso") { c.flagType = "iso" } else { c.flagType = "backup" } } else { // Validate type flag. if !slices.Contains([]string{"backup", "iso"}, c.flagType) { return errors.New(i18n.G("Import type needs to be \"backup\" or \"iso\"")) } } if c.flagType == "iso" && !hasVolName { volName = strings.TrimSuffix(filepath.Base(backupFile), filepath.Ext(backupFile)) } progress := cli.ProgressRenderer{ Format: i18n.G("Importing custom volume: %s"), Quiet: c.global.flagQuiet, } createArgs := incus.StorageVolumeBackupArgs{ BackupFile: &ioprogress.ProgressReader{ ReadCloser: file, Tracker: &ioprogress.ProgressTracker{ Length: fstat.Size(), Handler: func(percent int64, speed int64) { progress.UpdateProgress(ioprogress.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, units.GetByteSizeString(speed, 2))}) }, }, }, Name: volName, } var op incus.Operation if c.flagType == "iso" { op, err = d.CreateStoragePoolVolumeFromISO(poolName, createArgs) } else { op, err = d.CreateStoragePoolVolumeFromBackup(poolName, createArgs) } if err != nil { return err } // Wait for operation to finish. err = cli.CancelableWait(op, &progress) if err != nil { progress.Done("") return err } progress.Done("") return nil } // NBD. type cmdStorageVolumeNBD struct { global *cmdGlobal storage *cmdStorage storageVolume *cmdStorageVolume flagAddress string flagWritable bool } var cmdStorageVolumeNBDUsage = u.Usage{u.Pool.Remote(), u.MakePath(u.StorageVolumeType.Optional(), u.Volume)} // Command returns a cobra.Command for use with (*cobra.Command).AddCommand. func (c *cmdStorageVolumeNBD) Command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("nbd", cmdStorageVolumeNBDUsage...) cmd.Short = i18n.G("NBD access to a block storage volume") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `NBD access to a block storage volume.`)) cli.AddStringFlag(cmd.Flags(), &c.flagAddress, "address", "", "", i18n.G("Specific address to listen on")) cli.AddBoolFlag(cmd.Flags(), &c.flagWritable, "writable", i18n.G("Get write access to the disk")) cmd.RunE = c.Run // completion for pool, volume, host path cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpStoragePools(toComplete) } if len(args) == 1 { return c.global.cmpStoragePoolVolumes(args[0]) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } // Run runs the actual command logic. func (c *cmdStorageVolumeNBD) Run(cmd *cobra.Command, args []string) error { parsed, err := cmdStorageVolumeNBDUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer poolName := parsed[0].RemoteObject.String volType := parsed[1].List[0].Get("custom") volName := parsed[1].List[1].String // Check if the pool and the volume exist before starting the SFTP server. _, _, err = d.GetStoragePoolVolume(poolName, volType, volName) if err != nil { return err } // Proxy to a local listener. listenAddr := c.flagAddress if listenAddr == "" { listenAddr = "127.0.0.1:0" // Listen on a random local port if not specified. } listener, err := net.Listen("tcp", listenAddr) if err != nil { return fmt.Errorf(i18n.G("Failed to listen for connection: %w"), err) } fmt.Printf(i18n.G("NBD listening on %v")+"\n", listener.Addr()) // Wait for a connection. nConn, err := listener.Accept() if err != nil { return fmt.Errorf(i18n.G("Failed to accept incoming connection: %w"), err) } defer func() { _ = nConn.Close() }() fmt.Printf(i18n.G("NBD client connected %q")+"\n", nConn.RemoteAddr()) defer fmt.Printf(i18n.G("NBD client disconnected %q")+"\n", nConn.RemoteAddr()) // Connect to NBD. conn, err := d.GetStoragePoolVolumeBlockNBDConn(poolName, volType, volName, incus.StorageVolumeNBDPost{Writable: c.flagWritable}) if err != nil { return fmt.Errorf(i18n.G("NBD connection failed: %v")+"\n", err) } defer func() { _ = conn.Close() }() // Proxy the traffic. var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() _, _ = util.SafeCopy(conn, nConn) _ = conn.Close() _ = nConn.Close() }() go func() { defer wg.Done() _, _ = util.SafeCopy(nConn, conn) _ = conn.Close() _ = nConn.Close() }() wg.Wait() return nil } incus-7.0.0/cmd/incus/top.go000066400000000000000000000352531517523235500156740ustar00rootroot00000000000000package main import ( "bufio" "errors" "fmt" "os" "regexp" "slices" "sort" "strconv" "strings" "time" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/units" ) type topColumn struct { Name string Data func(displayData) string } type cmdTop struct { global *cmdGlobal targets []string flagAllProjects bool flagColumns string flagFormat string flagRefresh int } var cmdTopUsage = u.Usage{u.RemoteColonOpt} func (c *cmdTop) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("top", cmdTopUsage...) cmd.Short = i18n.G("Display resource usage info per instance") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Displays CPU usage, memory usage, and disk usage per instance Default column layout: numD == Columns == The -c option takes a comma separated list of arguments that control which instance attributes to output when displaying in table or compact format. Column arguments are pre-defined shorthand chars (see below). Commas between consecutive shorthand chars are optional. Column shorthand chars: D - disk usage e - Project name m - Memory usage n - Instance name u - CPU usage (in seconds)`)) cli.AddBoolFlag(cmd.Flags(), &c.flagAllProjects, "all-projects", i18n.G("Display instances from all projects")) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultTopColumns, "", i18n.G("Columns")) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G("Format (table|compact)")) cli.AddIntFlag(cmd.Flags(), &c.flagRefresh, "refresh", 10, i18n.G("Configure the refresh delay in seconds")) cmd.RunE = c.run return cmd } const ( defaultTopColumns = "numD" defaultTopColumnsAllProjects = "enumD" ) func (c *cmdTop) parseColumns() ([]topColumn, error) { columnsShorthandMap := map[rune]topColumn{ 'e': {i18n.G("PROJECT"), c.projectColumnData}, 'n': {i18n.G("INSTANCE NAME"), c.instanceNameColumnData}, 'u': {i18n.G("CPU TIME(s)"), c.cpuUsageColumnData}, 'm': {i18n.G("MEMORY"), c.memoryUsageColumnData}, 'D': {i18n.G("DISK"), c.diskUsageColumnData}, } columnList := strings.Split(c.flagColumns, ",") columns := []topColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } func (c *cmdTop) projectColumnData(dd displayData) string { return dd.project } func (c *cmdTop) instanceNameColumnData(dd displayData) string { return dd.instanceName } func (c *cmdTop) cpuUsageColumnData(dd displayData) string { return fmt.Sprintf("%.2f", dd.cpuUsage) } func (c *cmdTop) memoryUsageColumnData(dd displayData) string { if dd.memoryUsage > 0 { return units.GetByteSizeStringIEC(int64(dd.memoryUsage), 2) } return "" } func (c *cmdTop) diskUsageColumnData(dd displayData) string { if dd.diskUsage > 0 { return units.GetByteSizeStringIEC(int64(dd.diskUsage), 2) } return "" } // This function implements the `top` command. It queries the metrics API at (/1.0/metrics) and renders a list of // instances with their CPU, memory and disk usage columns. func (c *cmdTop) run(cmd *cobra.Command, args []string) error { parsed, err := cmdTopUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer // Add project column if --all-projects flag specified and no -c was passed. if c.flagAllProjects && c.flagColumns == defaultTopColumns { c.flagColumns = defaultTopColumnsAllProjects } // Validate flags. if !slices.Contains([]string{cli.TableFormatCompact, cli.TableFormatTable}, strings.SplitN(c.flagFormat, ",", 2)[0]) { return fmt.Errorf(i18n.G("Invalid format %q"), c.flagFormat) } if c.flagRefresh < 10 { return errors.New(i18n.G("The minimum refresh rate is 10s")) } // Get the current project. info, err := d.GetConnectionInfo() if err != nil { return err } if !c.flagAllProjects { d = d.UseProject(info.Project) } else { d = d.UseProject("") } // If clustered, get a list of targets. if d.IsClustered() { c.targets, err = d.GetClusterMemberNames() if err != nil { return err } } // These variables can be changed by the UI refreshInterval := time.Duration(c.flagRefresh) * time.Second sortingMethod := alphabetical // default is alphabetical, could change this to a flag // Start the ticker for periodic updates ticker := time.NewTicker(refreshInterval) defer ticker.Stop() // Call the update once before the loop err = c.updateDisplay(d, refreshInterval, sortingMethod) if err != nil { return err } durationChannel := make(chan time.Duration) sortingChannel := make(chan sortType) interruptChannel := make(chan bool) go handleKeystrokes(durationChannel, interruptChannel, sortingChannel) // Handles shortcuts on a separate Goroutine for { select { case shouldStop := <-interruptChannel: // This pauses the UI refresh loop if shouldStop { ticker.Stop() } else { err = c.updateDisplay(d, refreshInterval, sortingMethod) if err != nil { return err } ticker = time.NewTicker(refreshInterval) } case <-ticker.C: err = c.updateDisplay(d, refreshInterval, sortingMethod) if err != nil { return err } case sortType, ok := <-sortingChannel: if !ok { return nil // Exits if the channel is closed } sortingMethod = sortType case duration, ok := <-durationChannel: if !ok { return nil // Exits if the channel is closed } ticker.Stop() ticker = time.NewTicker(duration) refreshInterval = duration fmt.Printf(i18n.G("Updated interval to %v")+"\n", duration) // Update display err = c.updateDisplay(d, refreshInterval, sortingMethod) if err != nil { return err } } } } func handleKeystrokes(durationChannel chan time.Duration, interruptChannel chan bool, sortingChannel chan sortType) { reader := bufio.NewReader(os.Stdin) for { input, err := reader.ReadString('\n') if err != nil { fmt.Fprintf(os.Stderr, "Error reading from stdin: %v", err) return } input = input[:len(input)-1] // Strip newline character if input == "d" { interruptChannel <- true fmt.Print(i18n.G("Enter new delay in seconds:") + " ") delayInput, err := reader.ReadString('\n') if err != nil { fmt.Fprintf(os.Stderr, "Error reading new delay: %v", err) return } delayInput = delayInput[:len(delayInput)-1] // Strip newline character delaySec, err := strconv.ParseFloat(delayInput, 64) if err != nil || delaySec <= 0 { fmt.Println(i18n.G("Invalid input, please enter a positive number")) continue } // Send new duration back to the channel durationChannel <- time.Duration(delaySec * float64(time.Second)) } else if input == "s" { interruptChannel <- true fmt.Print(i18n.G("Enter a sorting type ('a' for alphabetical, 'c' for CPU, 'm' for memory, 'd' for disk):") + " ") sortingInput, err := reader.ReadString('\n') if err != nil { fmt.Fprintf(os.Stderr, "Error reading sorting type: %v", err) return } sortingInput = sortingInput[:len(sortingInput)-1] // Strip newline character // Send sorting type over sorting channel switch sortingInput { case "a": sortingChannel <- alphabetical case "c": sortingChannel <- cpuUsage case "m": sortingChannel <- memoryUsage case "d": sortingChannel <- diskUsage default: fmt.Println(i18n.G("Invalid sorting type provided")) } interruptChannel <- false } } } type sortType string const ( alphabetical sortType = "Alphabetical" cpuUsage sortType = "CPU Usage" memoryUsage sortType = "Memory Usage" diskUsage sortType = "Disk Usage" ) type displayData struct { project string instanceName string cpuUsage float64 memoryUsage float64 diskUsage float64 } func sortBySortingType(data []displayData, sortingType sortType) { sortFuncs := map[sortType]func(i, j int) bool{ alphabetical: func(i, j int) bool { if data[i].project != data[j].project { return data[i].project < data[j].project } return data[i].instanceName < data[j].instanceName }, cpuUsage: func(i, j int) bool { return data[i].cpuUsage > data[j].cpuUsage }, memoryUsage: func(i, j int) bool { return data[i].memoryUsage > data[j].memoryUsage }, diskUsage: func(i, j int) bool { return data[i].diskUsage > data[j].diskUsage }, } sortFunc, ok := sortFuncs[sortingType] if ok { sort.Slice(data, sortFunc) } else { fmt.Println(i18n.G("Invalid sorting type")) } } func (c *cmdTop) updateDisplay(d incus.InstanceServer, refreshInterval time.Duration, sortingType sortType) error { var metrics []string if c.targets == nil { rawMetrics, err := d.GetMetrics() if err != nil { return err } metrics = []string{rawMetrics} } else { metrics = make([]string, 0, len(c.targets)) for _, target := range c.targets { rawMetrics, err := d.UseTarget(target).GetMetrics() if err != nil { return err } metrics = append(metrics, rawMetrics) } } metricSet, entries, err := parseMetricsFromString(strings.Join(metrics, "\n")) if err != nil { return err } data := []displayData{} for projectName, names := range entries { for _, currentName := range names { cpuSeconds := metricSet.getMetricValue(cpuSecondsTotal, currentName) memoryFree := metricSet.getMetricValue(memoryMemAvailableBytes, currentName) memoryTotal := metricSet.getMetricValue(memoryMemTotalBytes, currentName) diskTotal := metricSet.getMetricValue(filesystemSizeBytes, currentName) diskFree := metricSet.getMetricValue(filesystemFreeBytes, currentName) data = append(data, displayData{ project: projectName, instanceName: currentName, cpuUsage: cpuSeconds, memoryUsage: memoryTotal - memoryFree, diskUsage: diskTotal - diskFree, }) } } // Perform sort operation sortBySortingType(data, sortingType) // Process the columns columns, err := c.parseColumns() if err != nil { return err } dataFormatted := [][]string{} for _, d := range data { row := []string{} for _, column := range columns { row = append(row, column.Data(d)) } dataFormatted = append(dataFormatted, row) } headers := []string{} for _, column := range columns { headers = append(headers, column.Name) } fmt.Print("\033[H\033[2J") // Clear the terminal on each tick err = cli.RenderTable(os.Stdout, c.flagFormat, headers, dataFormatted, nil) if err != nil { return err } fmt.Println(i18n.G("Press 'd' + ENTER to change delay")) fmt.Println(i18n.G("Press 's' + ENTER to change sorting method")) fmt.Println(i18n.G("Press CTRL-C to exit")) fmt.Println() fmt.Println(i18n.G("Delay:"), refreshInterval) fmt.Println(i18n.G("Sorting Method:"), sortingType) return nil } type sample struct { labels map[string]string value float64 } type metricType int type metricSet struct { set map[metricType][]sample labels map[string]string } const ( // CPUSecondsTotal represents the total CPU seconds used. cpuSecondsTotal metricType = iota // FilesystemAvailBytes represents the available bytes on a filesystem. filesystemFreeBytes // FilesystemSizeBytes represents the size in bytes of a filesystem. filesystemSizeBytes // MemoryMemAvailableBytes represents the amount of available memory. memoryMemAvailableBytes // MemoryMemTotalBytes represents the amount of used memory. memoryMemTotalBytes ) // MetricNames associates a metric type to its name. var metricNames = map[metricType]string{ cpuSecondsTotal: "incus_cpu_seconds_total", filesystemFreeBytes: "incus_filesystem_free_bytes", filesystemSizeBytes: "incus_filesystem_size_bytes", memoryMemAvailableBytes: "incus_memory_MemAvailable_bytes", memoryMemTotalBytes: "incus_memory_MemTotal_bytes", } func (ms *metricSet) getMetricValue(metricType metricType, instanceName string) float64 { value := 0.0 if samples, exists := ms.set[metricType]; exists { // Check if metricType exists for _, sample := range samples { if (metricType == filesystemFreeBytes || metricType == filesystemSizeBytes) && sample.labels["mountpoint"] != "/" { continue } if metricType == cpuSecondsTotal && sample.labels["mode"] == "idle" { continue } if sample.labels["name"] == instanceName { value += sample.value } } } return value } // ParseMetricsFromString parses OpenMetrics formatted logs from a string and converts them to a MetricSet. func parseMetricsFromString(input string) (*metricSet, map[string][]string, error) { scanner := bufio.NewScanner(strings.NewReader(input)) metricSet := &metricSet{ set: make(map[metricType][]sample), labels: make(map[string]string), } metricLineRegex := regexp.MustCompile(`^(\w+)\{(.+)\}\s+([\d\.]+e[+-]?\d+|[\d\.]+)$`) for scanner.Scan() { line := scanner.Text() matches := metricLineRegex.FindStringSubmatch(line) if matches == nil { continue } metricName, labelPart, valueStr := matches[1], matches[2], matches[3] value, err := strconv.ParseFloat(valueStr, 64) if err != nil { return nil, nil, fmt.Errorf("Invalid metric value: %v", err) } metricType, found := findMetricTypeByName(metricName) if !found { continue } labels := parseLabels(labelPart) sample := sample{ labels: labels, value: value, } metricSet.set[metricType] = append(metricSet.set[metricType], sample) } err := scanner.Err() if err != nil { return nil, nil, err } names := map[string][]string{} if samples, exists := metricSet.set[memoryMemTotalBytes]; exists { // Use a known metric type to gather names for _, sample := range samples { projectName := sample.labels["project"] instName := sample.labels["name"] if names[projectName] == nil { names[projectName] = []string{} } names[projectName] = append(names[projectName], instName) } } return metricSet, names, nil } func parseLabels(input string) map[string]string { labels := make(map[string]string) for _, pair := range strings.Split(input, ",") { kv := strings.Split(pair, "=") if len(kv) != 2 { continue } key := strings.TrimSpace(kv[0]) value := strings.Trim(kv[1], "\"") labels[key] = value } return labels } func findMetricTypeByName(name string) (metricType, bool) { for typ, typName := range metricNames { if typName == name { return typ, true } } return 0, false } incus-7.0.0/cmd/incus/usage/000077500000000000000000000000001517523235500156375ustar00rootroot00000000000000incus-7.0.0/cmd/incus/usage/legacysupport.go000066400000000000000000000005321517523235500210670ustar00rootroot00000000000000package usage import ( "github.com/lxc/incus/v7/internal/i18n" ) // LegacyKV is a backward-compatible key/value parsing atom. var LegacyKV = hide{alternative{[]Atom{compound{"=", []Atom{Key, Value}}, deprecated{compound{" ", []Atom{Key, Value}}, i18n.G("please switch to the “=” syntax")}}}, compound{"=", []Atom{Key, Value}}} incus-7.0.0/cmd/incus/usage/parse.go000066400000000000000000000206641517523235500173100ustar00rootroot00000000000000package usage import ( "errors" "fmt" "slices" "strconv" "strings" "github.com/fatih/color" "github.com/mattn/go-runewidth" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" cliColor "github.com/lxc/incus/v7/cmd/incus/color" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/cliconfig" ) // ExplainOnly is a global switch putting the parser into explain mode, i.e. showing the user how // their arguments are parsed. var ExplainOnly = false func formatAlternatives(alternatives []string) string { n := len(alternatives) if n == 1 { return quote(alternatives[0]) } quoted := make([]string, n) for i, exp := range alternatives { quoted[i] = quote(exp) } return fmt.Sprintf(i18n.G("one of %s or %s"), strings.Join(quoted[:n-1], i18n.G(", ")), quoted[n-1]) } type notEnoughArgumentsError struct { atom Atom } func (a *notEnoughArgumentsError) Error() string { // This special case handles compounds that end with a suffix (and therefore end with a blank // verbatim atom. v, ok := a.atom.(verbatim) if ok && v.element == "" { return i18n.G("unexpected end of argument; did you forget a suffix?") } return fmt.Sprintf(i18n.G("not enough arguments; expected a value for %s"), quote(renderRaw(a.atom))) } type tooManyArgumentsError struct { args []string } func (t *tooManyArgumentsError) Error() string { escapedArgs := make([]string, len(t.args)) for i, arg := range t.args { if strings.Contains(arg, " ") { escapedArgs[i] = strconv.Quote(arg) } else { escapedArgs[i] = arg } } return fmt.Sprintf(i18n.G("too many arguments; unexpected %s"), quote(strings.Join(escapedArgs, " "))) } type argumentNotFullyConsumedError struct { rest string parent string } func (a *argumentNotFullyConsumedError) Error() string { if a.rest == a.parent { return fmt.Sprintf(i18n.G("cannot parse this argument; unexpected %s"), quote(a.rest)) } return fmt.Sprintf(i18n.G("cannot parse this argument; unexpected %s in %s"), quote(a.rest), quote(a.parent)) } type argumentMismatchError struct { arg string expected []string } func (a *argumentMismatchError) Error() string { n := len(a.expected) if n == 0 { return fmt.Sprintf(i18n.G("unexpected %s"), quote(a.arg)) } if a.arg == "" { return fmt.Sprintf(i18n.G("expected %s"), formatAlternatives(a.expected)) } return fmt.Sprintf(i18n.G("unexpected %s; expected %s"), quote(a.arg), formatAlternatives(a.expected)) } func isParsingError(err error) bool { switch err.(type) { case *notEnoughArgumentsError, *tooManyArgumentsError, *argumentNotFullyConsumedError, *argumentMismatchError: return true default: return false } } // ErrExplainOnly is returned when --explain is used and the CLI invocation is valid. var ErrExplainOnly = errors.New(i18n.G("This command was called with --explain; its arguments are valid, but no further processing is done")) // Parsed is the type of parsed atoms. type Parsed struct { // The atom which has been parsed. source Atom // The remote name, if the parsed atom describes a remote. RemoteName string // The remote instance server, if the parsed atom describes a remote. RemoteServer incus.InstanceServer // The remote object parsed sub-atom, if the parsed atom describes a remote. RemoteObject *Parsed // The string argument(s) mapped to this parsed atom. String string // The parsed sub-atoms. List []*Parsed // The parsed sub-atoms as strings. StringList []string // The error that led to the atom parsing being skipped. err error // Whether the atom parsing has been skipped. Skipped bool // The branch number of an alternative. BranchID int } // Get gets a parsed atom’s string representation, or a default value if the atom was skipped. func (p Parsed) Get(def string) string { if p.Skipped { return def } return p.String } func underline(width int, cursor *int) (string, int) { var str string switch width { case 1: str = "┬" case 2: str = "├┘" default: dashCount := width - 3 str = "└" + strings.Repeat("─", dashCount/2) + "┬" + strings.Repeat("─", dashCount-(dashCount/2)) + "┘" } middle := *cursor + (width-1)/2 *cursor = *cursor + width + 1 return str, middle } func (u Usage) diagnose(cmd *cobra.Command, parsedValues []*Parsed, parseRTL bool) { nAtoms := len(u) renderedAtoms := make([]string, nAtoms) // To properly support international characters, we have to count printed columns and not bytes. wcWidths := make([]int, nAtoms) for i, atom := range u { renderedAtoms[i] = atom.Render() wcWidths[i] = runewidth.StringWidth(renderRaw(atom)) } commandPath := cmd.CommandPath() fmt.Println(cliColor.UsagePrefix + " " + commandPath + " " + strings.Join(renderedAtoms, " ")) parsedCount := len(parsedValues) usagePrefixLen := runewidth.StringWidth(cliColor.RawUsagePrefix) + runewidth.StringWidth(commandPath) + 1 cursor := 0 // We shrink renderedAtoms to the atoms we actually parsed. if parseRTL { slices.Reverse(parsedValues) renderedAtoms = renderedAtoms[nAtoms-parsedCount:] for i := range nAtoms - parsedCount { cursor = cursor + wcWidths[i] + 1 } } else { renderedAtoms = renderedAtoms[:parsedCount] } padding := usagePrefixLen + 1 if parsedCount > 0 { underlinedAtoms := make([]string, parsedCount) underlinedAtomMids := make([]int, parsedCount) offset := 0 if parseRTL { offset = nAtoms - parsedCount } for i := range parsedCount { str, middle := underline(wcWidths[offset+i], &cursor) underlinedAtoms[i] = str underlinedAtomMids[i] = middle } if parseRTL { if parsedCount < nAtoms { underlinedAtoms = append([]string{color.RedString(strings.Repeat("┅", wcWidths[offset-1]))}, underlinedAtoms...) } // In RTL mode, we need to properly pad the strings so that the diagnosis is right-aligned. for i := range offset - 1 { padding = padding + wcWidths[i] + 1 } } else { if parsedCount < nAtoms { underlinedAtoms = append(underlinedAtoms, color.RedString(strings.Repeat("┅", wcWidths[parsedCount]))) } } fmt.Println(strings.Repeat(" ", padding) + strings.Join(underlinedAtoms, " ")) for i := range parsedCount { fmt.Print(strings.Repeat(" ", parsedCount+1-i) + "┌" + strings.Repeat("│", i) + strings.Repeat("─", usagePrefixLen+underlinedAtomMids[i]-parsedCount-1) + "┘") j := i + 1 for j < parsedCount { fmt.Print(strings.Repeat(" ", underlinedAtomMids[j]-underlinedAtomMids[j-1]-1), "│") j++ } fmt.Print("\n") } for i, parsedValue := range parsedValues { fmt.Print(" ", strings.Repeat("│", parsedCount-i-1)+"└"+strings.Repeat("─", i+1)+" ") if parsedValue.err != nil { color.New(color.Faint).Printf(i18n.G("(skipped: %s)\n"), parsedValue.err) } else if parsedValue.Skipped { color.New(color.Faint).Println(i18n.G("(skipped: no value given)")) } else { fmt.Println(quote(parsedValue.String)) } } } else if nAtoms > 0 { i := 0 if parseRTL { // In RTL mode, we need to properly pad the strings so that the diagnosis is right-aligned. for i < nAtoms-1 { padding = padding + wcWidths[i] + 1 i++ } } fmt.Println(strings.Repeat(" ", padding) + color.RedString(strings.Repeat("┅", wcWidths[i]))) } // This makes the output error/status a bit easier to read. fmt.Println() } // Parse parses a usage. func (u Usage) Parse(conf *cliconfig.Config, cmd *cobra.Command, args []string, rtl ...bool) ([]*Parsed, error) { // Build a local server cache. servers := map[string]incus.InstanceServer{} nArgs := len(args) nAtoms := len(u) parseRTL := false if len(rtl) > 0 { parseRTL = rtl[0] } argsInUse := args atoms := u if parseRTL { slices.Reverse(argsInUse) // We don’t want to modify the original slice, as this may be reused. atoms = make([]Atom, nAtoms) for i, atom := range u { atoms[nAtoms-1-i] = atom } } var result []*Parsed for _, atom := range atoms { p, err := atom.Parse(conf, cmd, servers, &argsInUse, parseRTL) if err != nil { _, ok := err.(*notEnoughArgumentsError) if ok && nArgs == 0 { _ = cmd.Help() // This makes the output error a bit easier to read. fmt.Println() return nil, err } u.diagnose(cmd, result, parseRTL) return nil, err } result = append(result, p) } if len(argsInUse) != 0 { err := &tooManyArgumentsError{argsInUse} u.diagnose(cmd, result, parseRTL) return nil, err } if ExplainOnly { u.diagnose(cmd, result, parseRTL) return nil, ErrExplainOnly } if parseRTL { slices.Reverse(result) } return result, nil } incus-7.0.0/cmd/incus/usage/usage.go000066400000000000000000000652421517523235500173030ustar00rootroot00000000000000package usage import ( "fmt" "os" "slices" "strings" "github.com/fatih/color" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" cliColor "github.com/lxc/incus/v7/cmd/incus/color" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/cliconfig" ) // makeList is a helper function building list atoms. func makeList(atom Atom, minOccurrences int, separator ...string) Atom { if len(separator) == 0 { return list{atom, minOccurrences, " "} } return list{atom, minOccurrences, separator[0]} } // makeOptional is a helper function building optional atoms. func makeOptional(atom Atom, chain []Atom) Atom { if len(chain) == 0 { return optional{atom} } return optional{compound{" ", append([]Atom{atom}, chain...)}} } // Atom is the type of command-line atoms. type Atom interface { List(minOccurrences int, separator ...string) Atom Optional(chain ...Atom) Atom Parse(conf *cliconfig.Config, cmd *cobra.Command, servers map[string]incus.InstanceServer, args *[]string, parseRTL bool) (*Parsed, error) Render() string Remote() Atom } // alternative represents alternatives between several atoms. type alternative struct { atoms []Atom } // List makes the atom accept a list. func (a alternative) List(minOccurrences int, separator ...string) Atom { return makeList(a, minOccurrences, separator...) } // Optional makes the atom optional. func (a alternative) Optional(chain ...Atom) Atom { return makeOptional(a, chain) } // Parse parses the atom. func (a alternative) Parse(conf *cliconfig.Config, cmd *cobra.Command, servers map[string]incus.InstanceServer, args *[]string, parseRTL bool) (*Parsed, error) { // Parsing alternatives is not implemented cleverly at all: the first matching atom is the one // that will be used. This means that matching against something optional will always succeed, // which may not be intended. verbatimOnly := true verbatimElements := make([]string, len(a.atoms)) for i, atom := range a.atoms { // For diagnosis purposes, we record verbatim atoms. v, ok := atom.(verbatim) if ok { verbatimElements[i] = v.Render() } else { verbatimOnly = false } // We need to perform a deep copy of the arguments here, to easily rollback to a clean state if // something bad happened. argsCopy := make([]string, len(*args)) copy(argsCopy, *args) p, err := atom.Parse(conf, cmd, servers, &argsCopy, parseRTL) if err != nil { if isParsingError(err) { continue } return nil, err } *args = argsCopy p.BranchID = i return p, nil } // We defer this check to the very end, in case we have some exotic parsing rules. If we get out // of the loop, this probably means that the argument count is a problem. if len(*args) == 0 { return nil, ¬EnoughArgumentsError{a} } arg := (*args)[0] // If nothing is found, try to emit a nice diagnosis if possible. if verbatimOnly { return nil, &argumentMismatchError{arg, verbatimElements} } return nil, &argumentMismatchError{arg, []string{}} } // Render renders the atom's usage string. func (a alternative) Render() string { elements := make([]string, len(a.atoms)) for i, atom := range a.atoms { elements[i] = atom.Render() } faint := color.New(color.Faint) return faint.Sprint("(") + strings.Join(elements, faint.Sprint("|")) + faint.Sprint(")") } // Remote prefixes the atom with a remote. func (a alternative) Remote() Atom { return remote{Remote, a, true} } // compound represents a sequence of atoms separated with a separator. type compound struct { separator string atoms []Atom } // List makes the atom accept a list. func (c compound) List(minOccurrences int, separator ...string) Atom { return makeList(c, minOccurrences, separator...) } // Optional makes the atom optional. func (c compound) Optional(chain ...Atom) Atom { return makeOptional(c, chain) } // Parse parses the atom. func (c compound) Parse(conf *cliconfig.Config, cmd *cobra.Command, servers map[string]incus.InstanceServer, args *[]string, parseRTL bool) (*Parsed, error) { var consumed []string n := len(c.atoms) ps := make([]*Parsed, n) atoms := c.atoms // RTL parsing requires us to reverse our atoms when the sequence is separated by spaces. if c.separator == " " && parseRTL { // We don’t want to modify the original slice, as this may be reused. atoms = make([]Atom, n) for i, atom := range c.atoms { atoms[n-1-i] = atom } } // If the compound atom starts with optional atom(s), our dumb greedy parsing fails if those // optional atoms are not set. Instead of coming up with something more clever, we just shift them // as needed. We first have to count the optional atoms at the end of our compound. nOpt := 0 for i := range n { _, ok := atoms[n-i-1].(optional) if !ok { break } nOpt++ } // The parsing method differs a bit depending on the separator in use. if c.separator == " " { nSub := len(*args) i := 0 // Ignore optional atoms that cannot be set. for i < n-nOpt-nSub { o, ok := atoms[i].(optional) if !ok { break } p, err := o.Parse(conf, cmd, servers, &[]string{}, false) if err != nil { return nil, err } ps[i] = p i++ } for i < n { p, err := atoms[i].Parse(conf, cmd, servers, args, false) if err != nil { return nil, err } ps[i] = p // Prevent displaying useless spaces if !p.Skipped { consumed = append(consumed, p.String) } i++ } if parseRTL { slices.Reverse(consumed) slices.Reverse(ps) } } else { if len(*args) == 0 { return nil, ¬EnoughArgumentsError{c} } arg := (*args)[0] consumed = []string{arg} subArgs := strings.Split(arg, c.separator) nSub := len(subArgs) i := 0 // Ignore optional atoms that cannot be set. for i < n-nOpt-nSub { o, ok := atoms[i].(optional) if !ok { break } p, err := o.Parse(conf, cmd, servers, &[]string{}, false) if err != nil { return nil, err } ps[i] = p i++ } // Parsing up to the penultimate atom behaves normally. for i < n-1 { p, err := atoms[i].Parse(conf, cmd, servers, &subArgs, false) if err != nil { return nil, err } ps[i] = p i++ } // For the last atom, we join all the remaining sub-arguments. There are multiple reasons for // this: when parsing K/V pairs, we want to allow the user to enter `=` characters in the values // and have them processed normally, but in some very specific cases where the number of atoms // not being ignored cannot easily be known in advance, we may end up messing with the arguments // if we use `SplitN` naïvely. lastArgs := []string{} if len(subArgs) > 0 { lastArgs = append(lastArgs, strings.Join(subArgs, c.separator)) } p, err := atoms[n-1].Parse(conf, cmd, servers, &lastArgs, false) if err != nil { return nil, err } ps[n-1] = p *args = (*args)[1:] } stringList := make([]string, len(ps)) for i, p := range ps { stringList[i] = p.String } return &Parsed{source: c, String: strings.Join(consumed, " "), List: ps, StringList: stringList}, nil } // Render renders the atom's usage string. func (c compound) Render() string { if len(c.atoms) == 1 { return c.atoms[0].Render() } var sb strings.Builder firstNonOptionalAtom := 0 for i, atom := range c.atoms { // If our atom is optional, its separator should be included in the optional string, because it // wouldn't appear if the atom is not specified by the user. o, ok := atom.(optional) // Spaces convey no semantic value in this case, so it feels more natural not to include them in // optional blocks (` []` vs `[ ]`). if ok && c.separator != " " { if i == firstNonOptionalAtom { // If optional atoms appear at the beginning of the compound atom, they behave differently // with regard to the separator (`[/][/]` vs `[][/]/`). sb.WriteString(optional{verbatim{o.atom.Render() + c.separator}}.Render()) firstNonOptionalAtom++ } else { sb.WriteString(optional{verbatim{c.separator + o.atom.Render()}}.Render()) } } else { if i == firstNonOptionalAtom { // The separator must be omitted at the beginning of the compound, or just after optional // atoms at the beginning of the compound. sb.WriteString(atom.Render()) } else { sb.WriteString(c.separator + atom.Render()) } } } return sb.String() } // Remote prefixes the atom with a remote. func (c compound) Remote() Atom { return remote{Remote, c, true} } // deprecated represents an atom whose usage is deprecated. type deprecated struct { atom Atom warning string } // List makes the atom accept a list. func (d deprecated) List(minOccurrences int, separator ...string) Atom { return deprecated{d.atom.List(minOccurrences, separator...), d.warning} } // Optional makes the atom optional. func (d deprecated) Optional(chain ...Atom) Atom { return deprecated{d.atom.Optional(chain...), d.warning} } // Parse parses the atom. func (d deprecated) Parse(conf *cliconfig.Config, cmd *cobra.Command, servers map[string]incus.InstanceServer, args *[]string, parseRTL bool) (*Parsed, error) { parsed, err := d.atom.Parse(conf, cmd, servers, args, parseRTL) if err != nil { return nil, err } syntax := renderRaw(d.atom) fmt.Fprintf(os.Stderr, i18n.G("%s the %s syntax is deprecated; %s\n"), cliColor.WarningPrefix, quote(syntax), d.warning) return parsed, nil } // Render renders the atom's usage string. func (d deprecated) Render() string { return d.atom.Render() } // Remote prefixes the atom with a remote. func (d deprecated) Remote() Atom { return remote{Remote, d, true} } // flag represents a command-line flag. type flag struct { name string } // List makes the atom accept a list. This is probably a very bad idea. func (f flag) List(minOccurrences int, separator ...string) Atom { return makeList(f, minOccurrences, separator...) } // Optional makes the atom optional. func (f flag) Optional(chain ...Atom) Atom { return makeOptional(f, chain) } // Parse parses the atom. func (f flag) Parse(conf *cliconfig.Config, cmd *cobra.Command, servers map[string]incus.InstanceServer, args *[]string, parseRTL bool) (*Parsed, error) { cmdFlag := cmd.Flag(f.name) if cmdFlag == nil { return nil, fmt.Errorf(i18n.G("Unknown flag --%s"), f.name) } if cmdFlag.Changed { return &Parsed{source: f, String: "--" + f.name}, nil } return nil, &argumentMismatchError{"", []string{"--" + f.name}} } // Render renders the atom's usage string. func (f flag) Render() string { return verbatim{"--" + f.name}.Render() } // Remote is a no-op. func (f flag) Remote() Atom { return f } // hide represents an atom whose internal value is hidden. type hide struct { atom Atom replacement Atom } // List makes the atom accept a list. func (h hide) List(minOccurrences int, separator ...string) Atom { return makeList(h, minOccurrences, separator...) } // Optional makes the atom optional. func (h hide) Optional(chain ...Atom) Atom { return makeOptional(h, chain) } // Parse parses the atom. func (h hide) Parse(conf *cliconfig.Config, cmd *cobra.Command, servers map[string]incus.InstanceServer, args *[]string, parseRTL bool) (*Parsed, error) { return h.atom.Parse(conf, cmd, servers, args, parseRTL) } // Render renders the atom's usage string. func (h hide) Render() string { return h.replacement.Render() } // Remote prefixes the atom with a remote. func (h hide) Remote() Atom { return remote{Remote, h, true} } // list represents a list of atoms of arbitrary length. type list struct { atom Atom minOccurrences int separator string } // List makes the atom accept a list. func (l list) List(minOccurrences int, separator ...string) Atom { return makeList(l, minOccurrences, separator...) } // Optional makes the atom optional. func (l list) Optional(chain ...Atom) Atom { return makeOptional(list{l.atom, max(l.minOccurrences, 1), l.separator}, chain) } // Parse parses the atom. func (l list) Parse(conf *cliconfig.Config, cmd *cobra.Command, servers map[string]incus.InstanceServer, args *[]string, parseRTL bool) (*Parsed, error) { var ps []*Parsed var consumed []string // The parsing method differs a bit depending on the separator in use. if l.separator == " " { // The required occurrences of the list can have direct access to the args. for range l.minOccurrences { p, err := l.atom.Parse(conf, cmd, servers, args, parseRTL) if err != nil { return nil, err } ps = append(ps, p) consumed = append(consumed, p.String) } // For the rest, because we stop processing the list as soon as a problem is encountered, only a // copy of the args should be manipulated. We do a full copy every time which is a bit costly, // but it’s better to be safe here. for { argsCopy := make([]string, len(*args)) copy(argsCopy, *args) p, err := l.atom.Parse(conf, cmd, servers, &argsCopy, parseRTL) if err != nil { // If there is a parsing error, simply throw it away. if isParsingError(err) { break } return nil, err } *args = argsCopy ps = append(ps, p) consumed = append(consumed, p.String) } if parseRTL { slices.Reverse(consumed) slices.Reverse(ps) } } else { // In this case, the whole argument needs to be consumed, but parsing failures when the list is // optional are perfectly fine. var subArgs []string if len(*args) == 0 { subArgs = []string{} consumed = []string{} } else { arg := (*args)[0] subArgs = strings.Split(arg, l.separator) consumed = []string{arg} } i := 0 for i < l.minOccurrences || len(subArgs) > 0 { p, err := l.atom.Parse(conf, cmd, servers, &subArgs, false) if err != nil { // If there is a parsing error and the list is optional, simply throw it away and rollback // the whole thing. if l.minOccurrences == 0 && isParsingError(err) { return &Parsed{source: l, err: err, Skipped: true}, nil } return nil, err } ps = append(ps, p) i++ } // If everything went right, we can safely consume the argument. if len(*args) > 0 { *args = (*args)[1:] } } stringList := make([]string, len(ps)) for i, p := range ps { stringList[i] = p.String } skipped := len(ps) == 0 return &Parsed{source: l, String: strings.Join(consumed, " "), List: ps, StringList: stringList, Skipped: skipped}, nil } // Render renders the atom's usage string. func (l list) Render() string { faint := color.New(color.Faint) switch l.minOccurrences { case 0: // Lists with 0 minimum elements behave like optional lists with at least 1 element. return optional{list{l.atom, 1, l.separator}}.Render() case 1: element := l.atom.Render() if l.separator == " " { // If the separator is a space, `...` is widely understood as a valid repetition token // (`...`). return element + faint.Sprint("...") } // Else, we are a bit more explicit (e.g., with `,` as the separator, `[,...]`). return element + optional{verbatim{l.separator + element + faint.Sprint("...")}}.Render() default: // We recurse when the list has more that 1 minimum elements. return l.atom.Render() + l.separator + list{l.atom, l.minOccurrences - 1, l.separator}.Render() } } // Remote prefixes the atom with a remote. func (l list) Remote() Atom { // It doesn't really make sense to prefix a list with a remote, so we distribute the operation. return remote{Remote, l.atom, true}.List(l.minOccurrences, l.separator) } // optional represents an optional atom. type optional struct { atom Atom } // List makes the atom accept a list. func (o optional) List(minOccurrences int, separator ...string) Atom { // We define optional.List as list.Optional. return makeList(o.atom, minOccurrences, separator...).Optional() } // Optional makes the atom optional. func (o optional) Optional(chain ...Atom) Atom { return makeOptional(o.atom, chain) } // Parse parses the atom. func (o optional) Parse(conf *cliconfig.Config, cmd *cobra.Command, servers map[string]incus.InstanceServer, args *[]string, parseRTL bool) (*Parsed, error) { // We need to perform a deep copy of the arguments here, to easily rollback to a clean state if // something bad happened. argsCopy := make([]string, len(*args)) copy(argsCopy, *args) p, err := o.atom.Parse(conf, cmd, servers, &argsCopy, parseRTL) if err != nil { // If there is a parsing error, simply throw it away. if isParsingError(err) { return &Parsed{source: o, err: err, Skipped: true}, nil } return nil, err } *args = argsCopy return p, nil } // Render renders the atom's usage string. func (o optional) Render() string { faint := color.New(color.Faint) return faint.Sprint("[") + o.atom.Render() + faint.Sprint("]") } // Remote prefixes the atom with a remote. func (o optional) Remote() Atom { return remote{Remote, o, true} } // placeholder represents a placeholder atom. type placeholder struct { element string } // List makes the atom accept a list. func (p placeholder) List(minOccurrences int, separator ...string) Atom { return makeList(p, minOccurrences, separator...) } // Optional makes the atom optional. func (p placeholder) Optional(chain ...Atom) Atom { return makeOptional(p, chain) } // Parse parses the atom. func (p placeholder) Parse(conf *cliconfig.Config, cmd *cobra.Command, servers map[string]incus.InstanceServer, args *[]string, parseRTL bool) (*Parsed, error) { if len(*args) == 0 { return nil, ¬EnoughArgumentsError{p} } arg := (*args)[0] *args = (*args)[1:] return &Parsed{source: p, String: arg}, nil } // Render renders the atom's usage string. func (p placeholder) Render() string { return color.GreenString("<" + p.element + ">") } // Remote prefixes the atom with a remote. func (p placeholder) Remote() Atom { return remote{Remote, p, true} } // remote represents an atom prefixed with a remote. type remote struct { atom Atom suffix Atom optional bool } // List makes the atom accept a list. func (r remote) List(minOccurrences int, separator ...string) Atom { return makeList(r, minOccurrences, separator...) } // Optional makes the atom optional. func (r remote) Optional(chain ...Atom) Atom { return makeOptional(r, chain) } // Parse parses the atom. func (r remote) Parse(conf *cliconfig.Config, cmd *cobra.Command, servers map[string]incus.InstanceServer, args *[]string, parseRTL bool) (*Parsed, error) { skipped := false arg := "" if len(*args) == 0 { skipped = true } else { arg = (*args)[0] } if skipped && !r.optional { return nil, ¬EnoughArgumentsError{r} } remoteName, rest, err := conf.ParseRemote(arg) if err != nil { return nil, err } restArgs := []string{} if rest != "" { restArgs = append(restArgs, rest) } var p *Parsed // From here, we soft-fail if the remote is of the form `[:]` and hard-fail otherwise. if r.suffix == nil { if rest != "" { // Because this atom is skipped, we fallback to the default remote. remoteServer, serverErr := getInstanceServer(conf, servers, conf.DefaultRemote) if serverErr != nil { return nil, serverErr } if strings.Contains(arg, ":") { // If `:` is in the original argument, it means there is superfluous data after the `:`. err = &argumentNotFullyConsumedError{rest, arg} } else { // Otherwise, it is the same thing, but it may suggest that the user forgot to type `:`. The // error type is semantically not the most appropriate, but it nicely mirrors what is done // in compound atoms. err = ¬EnoughArgumentsError{verbatim{}} } if r.optional { return &Parsed{source: r, err: err, RemoteName: conf.DefaultRemote, RemoteServer: remoteServer, Skipped: true}, nil } return nil, err } } else { p, err = r.suffix.Parse(conf, cmd, servers, &restArgs, false) if err != nil { return nil, err } } if len(restArgs) != 0 { return nil, &argumentNotFullyConsumedError{restArgs[0], arg} } remoteServer, err := getInstanceServer(conf, servers, remoteName) if err != nil { return nil, err } // We defer the argument shifting so that code paths ignoring this atom can consume its arguments // again. if len(*args) > 0 { *args = (*args)[1:] } return &Parsed{source: r, String: arg, RemoteName: remoteName, RemoteServer: remoteServer, RemoteObject: p, Skipped: skipped}, nil } // Render renders the atom's usage string. func (r remote) Render() string { suffix := r.suffix if suffix == nil { suffix = verbatim{} } var prefix Atom prefix = compound{":", []Atom{r.atom, verbatim{""}}} if r.optional { prefix = prefix.Optional() } return prefix.Render() + suffix.Render() } // Remote prefixes the atom with a remote. func (r remote) Remote() Atom { // This is obviously a no-op. return r } // remote represents a verbatim atom. type verbatim struct { element string } // List makes the atom accept a list. func (v verbatim) List(minOccurrences int, separator ...string) Atom { return makeList(v, minOccurrences, separator...) } // Optional makes the atom optional. func (v verbatim) Optional(chain ...Atom) Atom { return makeOptional(v, chain) } // Parse parses the atom. func (v verbatim) Parse(conf *cliconfig.Config, cmd *cobra.Command, servers map[string]incus.InstanceServer, args *[]string, parseRTL bool) (*Parsed, error) { if len(*args) == 0 { return nil, ¬EnoughArgumentsError{v} } arg := (*args)[0] *args = (*args)[1:] if arg != v.element { expected := []string{} if v.element != "" { expected = append(expected, v.element) } return nil, &argumentMismatchError{arg, expected} } return &Parsed{source: v, String: arg}, nil } // Render renders the atom's usage string. func (v verbatim) Render() string { return v.element } // Remote prefixes the atom with a remote. func (v verbatim) Remote() Atom { return remote{Remote, v, true} } // A few strings used throughout the Incus client. var ( ACL = placeholder{i18n.G("ACL")} Address = placeholder{i18n.G("address")} AddressSet = placeholder{i18n.G("address set")} Alias = placeholder{i18n.G("alias")} Backend = placeholder{i18n.G("backend")} BackupFile = placeholder{i18n.G("backup file")} Bucket = placeholder{i18n.G("bucket")} Client = placeholder{i18n.G("client")} CommandLine = list{placeholder{i18n.G("command-line argument")}, 1, " "} Device = placeholder{i18n.G("device")} Direction = alternative{[]Atom{verbatim{"ingress"}, verbatim{"egress"}}} Directory = placeholder{i18n.G("directory")} Driver = placeholder{i18n.G("driver")} EndOfFlags = hide{optional{verbatim{"--"}}, verbatim{"[flags] [--]"}} Expiry = placeholder{i18n.G("expiry")} File = placeholder{i18n.G("file")} Filter = placeholder{i18n.G("filter")} Fingerprint = placeholder{i18n.G("fingerprint")} Group = placeholder{i18n.G("group")} Image = placeholder{i18n.G("image")} Instance = placeholder{i18n.G("instance")} Interface = placeholder{i18n.G("interface")} ListenAddress = placeholder{i18n.G("listen address")} ListenPort = placeholder{i18n.G("listen port")} Key = placeholder{i18n.G("key")} KV = compound{"=", []Atom{Key, Value}} Member = placeholder{i18n.G("member")} Network = placeholder{i18n.G("network")} NetworkIntegration = placeholder{i18n.G("network integration")} Operation = placeholder{i18n.G("operation")} Path = placeholder{i18n.G("path")} Peer = placeholder{i18n.G("peer")} Pool = placeholder{i18n.G("pool")} Port = placeholder{i18n.G("port")} Profile = placeholder{i18n.G("profile")} Project = placeholder{i18n.G("project")} Protocol = placeholder{i18n.G("protocol")} Query = placeholder{i18n.G("query")} Record = placeholder{i18n.G("record")} Remote = placeholder{i18n.G("remote")} RemoteColon = remote{Remote, nil, false} RemoteColonOpt = remote{Remote, nil, true} RemoteImage = compound{":", []Atom{optional{Remote}, Image}} Role = placeholder{i18n.G("role")} Snapshot = placeholder{i18n.G("snapshot")} StorageVolumeType = hide{alternative{[]Atom{verbatim{"custom"}, verbatim{"image"}, verbatim{"container"}, verbatim{"virtual-machine"}}}, placeholder{i18n.G("type")}} SymlinkTargetPath = placeholder{i18n.G("symlink target path")} Tarball = placeholder{i18n.G("tarball")} Template = placeholder{i18n.G("template")} Token = placeholder{i18n.G("token")} Type = placeholder{i18n.G("type")} URL = placeholder{i18n.G("URL")} Value = placeholder{i18n.G("value")} Volume = placeholder{i18n.G("volume")} WarningUUID = placeholder{i18n.G("warning UUID")} Zone = placeholder{i18n.G("zone")} ) // Either builds an alternative atom from several atoms. func Either(atoms ...Atom) Atom { return alternative{atoms} } // EitherVerbatim builds an alternative of verbatim atoms from several strings. func EitherVerbatim(elements ...string) Atom { atoms := make([]Atom, len(elements)) for i, element := range elements { atoms[i] = verbatim{element} } return alternative{atoms} } // EitherPlaceholder builds an alternative of placeholder atoms from several strings. func EitherPlaceholder(elements ...string) Atom { atoms := make([]Atom, len(elements)) for i, element := range elements { atoms[i] = placeholder{element} } return alternative{atoms} } // NewName transforms a placeholder (e.g. ``) into a placeholder suggesting that a new name is // requested (e.g. ``). func NewName(p placeholder) Atom { return placeholder{fmt.Sprintf(i18n.G("new %s name"), p.element)} } // Target transforms a placeholder (e.g. ``) into a placeholder suggesting that the requested // object is the target of an operation (e.g. ``). func Target(p placeholder) Atom { return placeholder{fmt.Sprintf(i18n.G("target %s"), p.element)} } // Colon suffixes an atom with `:`. func Colon(a Atom) Atom { return compound{":", []Atom{a, verbatim{""}}} } // MakePath builds an atom compound separated by `/`. func MakePath(atoms ...Atom) Atom { return compound{"/", atoms} } // Placeholder builds a placeholder atom from a string. func Placeholder(element string) placeholder { return placeholder{element} } // Verbatim builds a verbatim atom from a string. func Verbatim(element string) verbatim { return verbatim{element} } // Sequence builds a space-separated sequence of atoms. func Sequence(atoms ...Atom) Atom { return compound{" ", atoms} } // Flag builds a flag atom from a string. func Flag(name string) Atom { return flag{name} } // MakeRemote transforms an atom into a remote. func MakeRemote(atom Atom, optional bool) remote { return remote{atom, nil, optional} } // Usage is the type of CLI usages. type Usage []Atom incus-7.0.0/cmd/incus/usage/utils.go000066400000000000000000000025051517523235500173300ustar00rootroot00000000000000package usage import ( "fmt" "github.com/fatih/color" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/cliconfig" ) func getInstanceServer(conf *cliconfig.Config, servers map[string]incus.InstanceServer, remoteName string) (incus.InstanceServer, error) { // Look for a the remote in our cache. remoteServer, ok := servers[remoteName] if !ok { // New connection d, err := conf.GetInstanceServer(remoteName) if err != nil { return nil, err } servers[remoteName] = d remoteServer = d } return remoteServer, nil } // ParseString returns a parsed atom corresponding to a single string. func ParseString(s string) *Parsed { p, _ := placeholder{}.Parse(nil, nil, nil, &[]string{s}, false) return p } // ParseDefault returns a parsed atom corresponding to how the given atom is parsed without any // argument. func ParseDefault(atom Atom, conf *cliconfig.Config) (*Parsed, error) { return atom.Parse(conf, nil, map[string]incus.InstanceServer{}, &[]string{}, false) } // renderRaw returns the atom rendered after disabling terminal coloring. func renderRaw(atom Atom) string { noColor := color.NoColor color.NoColor = true s := atom.Render() color.NoColor = noColor return s } func quote(s string) string { return fmt.Sprintf(i18n.G("“%s”"), s) } incus-7.0.0/cmd/incus/utils.go000066400000000000000000000534611517523235500162330ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "io" "math/rand" "net" "os" "os/exec" "os/signal" "path/filepath" "reflect" "slices" "sort" "strings" "github.com/spf13/cobra" "golang.org/x/crypto/ssh" incus "github.com/lxc/incus/v7/client" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/shared/api" config "github.com/lxc/incus/v7/shared/cliconfig" "github.com/lxc/incus/v7/shared/termios" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) // Date layout to be used throughout the client. const dateLayout = "2006/01/02 15:04 MST" // Add a device to an instance. func instanceDeviceAdd(client incus.InstanceServer, name string, devName string, dev map[string]string) error { // Get the instance entry inst, etag, err := client.GetInstance(name) if err != nil { return err } // Check if the device already exists _, ok := inst.Devices[devName] if ok { return fmt.Errorf(i18n.G("Device already exists: %s"), devName) } inst.Devices[devName] = dev op, err := client.UpdateInstance(name, inst.Writable(), etag) if err != nil { return err } return op.Wait() } // Add a device to a profile. func profileDeviceAdd(client incus.InstanceServer, name string, devName string, dev map[string]string) error { // Get the profile entry profile, profileEtag, err := client.GetProfile(name) if err != nil { return err } // Check if the device already exists _, ok := profile.Devices[devName] if ok { return fmt.Errorf(i18n.G("Device already exists: %s"), devName) } // Add the device to the instance profile.Devices[devName] = dev err = client.UpdateProfile(name, profile.Writable(), profileEtag) if err != nil { return err } return nil } // parseDeviceOverrides parses device overrides of the form ",=" into a device map. // The resulting device map is unlikely to contain valid devices as these are simply values to be overridden. func parseDeviceOverrides(deviceOverrideArgs []string) (map[string]map[string]string, error) { deviceMap := map[string]map[string]string{} for _, entry := range deviceOverrideArgs { if !strings.Contains(entry, "=") || !strings.Contains(entry, ",") { return nil, fmt.Errorf(i18n.G("Bad device override syntax, expecting ,=: %s"), entry) } deviceFields := strings.SplitN(entry, ",", 2) keyFields := strings.SplitN(deviceFields[1], "=", 2) if deviceMap[deviceFields[0]] == nil { deviceMap[deviceFields[0]] = map[string]string{} } deviceMap[deviceFields[0]][keyFields[0]] = keyFields[1] } return deviceMap, nil } // isAliasesSubset returns true if the first array is completely contained in the second array. func isAliasesSubset(a1 []api.ImageAlias, a2 []api.ImageAlias) bool { set := make(map[string]any) for _, alias := range a2 { set[alias.Name] = nil } for _, alias := range a1 { _, found := set[alias.Name] if !found { return false } } return true } // getCommonAliases returns the common aliases between a list of aliases and all the existing ones. func getCommonAliases(client incus.InstanceServer, aliases ...api.ImageAlias) ([]api.ImageAliasesEntry, error) { if len(aliases) == 0 { return nil, nil } names := make([]string, len(aliases)) for i, alias := range aliases { names[i] = alias.Name } // 'getExistingAliases' which is using 'sort.SearchStrings' requires sorted slice sort.Strings(names) resp, err := client.GetImageAliases() if err != nil { return nil, err } return getExistingAliases(names, resp), nil } // Create the specified image aliases, updating those that already exist. func ensureImageAliases(client incus.InstanceServer, aliases []api.ImageAlias, fingerprint string) error { if len(aliases) == 0 { return nil } names := make([]string, len(aliases)) for i, alias := range aliases { names[i] = alias.Name } sort.Strings(names) resp, err := client.GetImageAliases() if err != nil { return err } // Delete existing aliases that match provided ones for _, alias := range getExistingAliases(names, resp) { err := client.DeleteImageAlias(alias.Name) if err != nil { return fmt.Errorf(i18n.G("Failed to remove alias %s: %w"), alias.Name, err) } } // Create new aliases. for _, alias := range aliases { aliasPost := api.ImageAliasesPost{} aliasPost.Name = alias.Name aliasPost.Target = fingerprint err := client.CreateImageAlias(aliasPost) if err != nil { return fmt.Errorf(i18n.G("Failed to create alias %s: %w"), alias.Name, err) } } return nil } // getExistingAliases returns the intersection between a list of aliases and all the existing ones. func getExistingAliases(aliases []string, allAliases []api.ImageAliasesEntry) []api.ImageAliasesEntry { existing := []api.ImageAliasesEntry{} for _, alias := range allAliases { name := alias.Name pos := sort.SearchStrings(aliases, name) if pos < len(aliases) && aliases[pos] == name { existing = append(existing, alias) } } return existing } // deleteImagesByAliases deletes images based on provided aliases. E.g. // aliases=[a1], image aliases=[a1] - image will be deleted // aliases=[a1, a2], image aliases=[a1] - image will be deleted // aliases=[a1], image aliases=[a1, a2] - image will be preserved. func deleteImagesByAliases(client incus.InstanceServer, aliases []api.ImageAlias) error { existingAliases, err := getCommonAliases(client, aliases...) if err != nil { return fmt.Errorf(i18n.G("Error retrieving aliases: %w"), err) } // Nothing to do. Just return. if len(existingAliases) == 0 { return nil } // Delete images if necessary visitedImages := make(map[string]any) for _, alias := range existingAliases { image, _, _ := client.GetImage(alias.Target) // If the image has already been visited then continue if image != nil { _, found := visitedImages[image.Fingerprint] if found { continue } visitedImages[image.Fingerprint] = nil } // An image can have multiple aliases. If an image being published // reuses all the aliases from an existing image then that existing image is removed. // In other case only specific aliases should be removed. E.g. // 1. If image with 'foo' and 'bar' aliases already exists and new image is published // with aliases 'foo' and 'bar'. Old image should be removed. // 2. If image with 'foo' and 'bar' aliases already exists and new image is published // with alias 'foo'. Old image should be kept with alias 'bar' // and new image will have 'foo' alias. if image != nil && isAliasesSubset(image.Aliases, aliases) { op, err := client.DeleteImage(alias.Target) if err != nil { return err } err = op.Wait() if err != nil { return err } } } return nil } func getConfig(args ...string) (map[string]string, error) { if len(args) == 2 && !strings.Contains(args[0], "=") { if args[1] == "-" && !termios.IsTerminal(getStdinFd()) { buf, err := io.ReadAll(os.Stdin) if err != nil { return nil, fmt.Errorf(i18n.G("Can't read from stdin: %w"), err) } args[1] = string(buf[:]) } return map[string]string{args[0]: args[1]}, nil } values := map[string]string{} for _, arg := range args { fields := strings.SplitN(arg, "=", 2) if len(fields) != 2 { return nil, fmt.Errorf(i18n.G("Invalid key=value configuration: %s"), arg) } if fields[1] == "-" && !termios.IsTerminal(getStdinFd()) { buf, err := io.ReadAll(os.Stdin) if err != nil { return nil, fmt.Errorf(i18n.G("Can't read from stdin: %w"), err) } fields[1] = string(buf[:]) } values[fields[0]] = fields[1] } return values, nil } // kvToMap converts a parsed KV list to a KV map. func kvToMap(p *u.Parsed) (map[string]string, error) { values := map[string]string{} stdinRead := false for _, kv := range p.List { key := kv.StringList[0] value := kv.StringList[1] if value == "-" && !termios.IsTerminal(getStdinFd()) { if stdinRead { return nil, errors.New(i18n.G("Cannot read the stdin twice")) } buf, err := io.ReadAll(os.Stdin) if err != nil { return nil, fmt.Errorf(i18n.G("Can't read from stdin: %w"), err) } value = string(buf[:]) } values[key] = value } return values, nil } // settable abstracts commands that set something. type settable interface { set(cmd *cobra.Command, parsed []*u.Parsed) error } // unsetKey reparses the last argument passed to an `unset` command to make it suitable for `set` // commands. func unsetKey(s settable, cmd *cobra.Command, parsed []*u.Parsed) error { i := len(parsed) - 1 parsed[i], _ = u.KV.List(0).Parse(nil, nil, nil, &[]string{parsed[i].String + "="}, false) return s.set(cmd, parsed) } func readEnvironmentFile(path string) (map[string]string, error) { content, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf(i18n.G("Can't read from environment file: %w"), err) } // Split the file into lines. lines := strings.Split(string(content), "\n") // Create a map to store the key value pairs. envMap := make(map[string]string) // Iterate over the lines. for _, line := range lines { if line == "" { continue } pieces := strings.SplitN(line, "=", 2) value := "" if len(pieces) > 1 { value = pieces[1] } envMap[pieces[0]] = value } return envMap, nil } // instancesExist iterates over a list of instances (or snapshots) and checks that they exist. func instancesExist(resources []remoteResource) error { for _, resource := range resources { // Handle snapshots. if instance.IsSnapshot(resource.name) { parent, snap, _ := api.GetParentAndSnapshotName(resource.name) _, _, err := resource.server.GetInstanceSnapshot(parent, snap) if err != nil { return fmt.Errorf(i18n.G("Failed checking instance snapshot exists \"%s:%s\": %w"), resource.remote, resource.name, err) } continue } _, _, err := resource.server.GetInstance(resource.name) if err != nil { return fmt.Errorf(i18n.G("Failed checking instance exists \"%s:%s\": %w"), resource.remote, resource.name, err) } } return nil } // structHasField checks if specified struct includes field with given name. func structHasField(typ reflect.Type, field string) bool { var parent reflect.Type for i := range typ.NumField() { fieldType := typ.Field(i) yaml := fieldType.Tag.Get("yaml") if yaml == ",inline" { parent = fieldType.Type } if yaml == field { return true } } if parent != nil { return structHasField(parent, field) } return false } // getServerSupportedFilters returns two lists: one with filters supported by server and second one with not supported. func getServerSupportedFilters(filters []string, clientFilters []string, singleValueServerSupport bool) ([]string, []string) { supportedFilters := []string{} unsupportedFilters := []string{} for _, filter := range filters { membs := strings.SplitN(filter, "=", 2) if len(membs) == 1 && singleValueServerSupport { supportedFilters = append(supportedFilters, filter) continue } else if len(membs) == 1 && !singleValueServerSupport { unsupportedFilters = append(unsupportedFilters, filter) continue } found := false if slices.Contains(clientFilters, membs[0]) { found = true unsupportedFilters = append(unsupportedFilters, filter) } if found { continue } supportedFilters = append(supportedFilters, filter) } return supportedFilters, unsupportedFilters } // guessImage checks that the image name (provided by the user) is correct given an instance remote and image remote. func guessImage(conf *config.Config, d incus.InstanceServer, instRemote string, imgRemote string, imageRef string) (string, string) { if instRemote != imgRemote { return imgRemote, imageRef } fields := strings.SplitN(imageRef, "/", 2) _, ok := conf.Remotes[fields[0]] if !ok { return imgRemote, imageRef } _, _, err := d.GetImageAlias(imageRef) if err == nil { return imgRemote, imageRef } _, _, err = d.GetImage(imageRef) if err == nil { return imgRemote, imageRef } if len(fields) == 1 { fmt.Fprintf(os.Stderr, i18n.G("The local image '%q' couldn't be found, trying '%q:' instead.")+"\n", imageRef, fields[0]) return fields[0], "default" } fmt.Fprintf(os.Stderr, i18n.G("The local image '%q' couldn't be found, trying '%q:%q' instead.")+"\n", imageRef, fields[0], fields[1]) return fields[0], fields[1] } // getImgInfo returns an image server and image info for the given image name (given by a user) // an image remote and an instance remote. func getImgInfo(d incus.InstanceServer, conf *config.Config, imgRemote string, instRemote string, imageRef string, source *api.InstanceSource) (incus.ImageServer, *api.Image, error) { var imgRemoteServer incus.ImageServer var imgInfo *api.Image var err error // Connect to the image server if imgRemote == instRemote { imgRemoteServer = d } else { imgRemoteServer, err = conf.GetImageServer(imgRemote) if err != nil { return nil, nil, err } } // Optimisation for public image servers. if conf.Remotes[imgRemote].Protocol != "incus" { imgInfo = &api.Image{} imgInfo.Fingerprint = imageRef imgInfo.Public = true source.Alias = imageRef } else { // Attempt to resolve an image alias alias, _, err := imgRemoteServer.GetImageAlias(imageRef) if err == nil { source.Alias = imageRef imageRef = alias.Target } // Get the image info imgInfo, _, err = imgRemoteServer.GetImage(imageRef) if err != nil { return nil, nil, err } } return imgRemoteServer, imgInfo, nil } // removeElementsFromSlice returns a slice equivalent to removing the given elements from the given list. // Elements not present in the list are ignored. func removeElementsFromSlice[T comparable](list []T, elements ...T) []T { for i := len(elements) - 1; i >= 0; i-- { element := elements[i] match := false for j := len(list) - 1; j >= 0; j-- { if element == list[j] { match = true list = slices.Delete(list, j, j+1) break } } if match { elements = slices.Delete(elements, i, i+1) } } return list } // sshfsMount mounts the instance's filesystem using sshfs by piping the instance's SFTP connection to sshfs. func sshfsMount(ctx context.Context, sftpConn net.Conn, entity string, relPath string, targetPath string) error { // Use the format "incus." as the source "host" (although not used for communication) // so that the mount can be seen to be associated with Incus and the instance in the local mount table. sourceURL := fmt.Sprintf("incus.%s:%s", entity, relPath) sshfsCmd := exec.Command("sshfs", "-o", "slave", sourceURL, targetPath) // Setup pipes. stdin, err := sshfsCmd.StdinPipe() if err != nil { return err } stdout, err := sshfsCmd.StdoutPipe() if err != nil { return err } sshfsCmd.Stderr = os.Stderr err = sshfsCmd.Start() if err != nil { return fmt.Errorf(i18n.G("Failed starting sshfs: %w"), err) } fmt.Printf(i18n.G("sshfs mounting %q on %q")+"\n", fmt.Sprintf("%s%s", entity, relPath), targetPath) fmt.Println(i18n.G("Press ctrl+c to finish")) ctx, cancel := context.WithCancel(ctx) chSignal := make(chan os.Signal, 1) signal.Notify(chSignal, os.Interrupt) go func() { select { case <-chSignal: case <-ctx.Done(): } cancel() // Prevents error output when the util.SafeCopy functions finish. _ = sshfsCmd.Process.Signal(os.Interrupt) // This will cause sshfs to unmount. _ = stdin.Close() }() go func() { _, err := util.SafeCopy(stdin, sftpConn) if ctx.Err() == nil { if err != nil { fmt.Fprintf(os.Stderr, i18n.G("I/O copy from instance to sshfs failed: %v")+"\n", err) } else { fmt.Println(i18n.G("Instance disconnected")) } } cancel() // Ask sshfs to end. }() _, err = util.SafeCopy(sftpConn, stdout) if err != nil && ctx.Err() == nil { fmt.Fprintf(os.Stderr, i18n.G("I/O copy from sshfs to instance failed: %v")+"\n", err) } cancel() // Ask sshfs to end. err = sshfsCmd.Wait() if err != nil { return err } fmt.Println(i18n.G("sshfs has stopped")) return sftpConn.Close() } // sshSFTPServer runs an SSH server listening on a random port of 127.0.0.1. // It provides an unauthenticated SFTP server connected to the instance's filesystem. func sshSFTPServer(ctx context.Context, sftpConn func() (net.Conn, error), authNone bool, authUser string, listenAddr string) error { randString := func(length int) string { chars := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321") randStr := make([]rune, length) for i := range randStr { randStr[i] = chars[rand.Intn(len(chars))] } return string(randStr) } // Setup an SSH SFTP server. sshConfig := &ssh.ServerConfig{} var authPass string if authNone { sshConfig.NoClientAuth = true } else { if authUser == "" { authUser = randString(8) } authPass = randString(8) sshConfig.PasswordCallback = func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) { if c.User() == authUser && string(pass) == authPass { return nil, nil } return nil, fmt.Errorf(i18n.G("Password rejected for %q"), c.User()) } } // Generate random host key. _, privKey, err := localtls.GenerateMemCert(false, false) if err != nil { return fmt.Errorf(i18n.G("Failed generating SSH host key: %w"), err) } private, err := ssh.ParsePrivateKey(privKey) if err != nil { return fmt.Errorf(i18n.G("Failed parsing SSH host key: %w"), err) } sshConfig.AddHostKey(private) if listenAddr == "" { listenAddr = "127.0.0.1:0" // Listen on a random local port if not specified. } listener, err := net.Listen("tcp", listenAddr) if err != nil { return fmt.Errorf(i18n.G("Failed to listen for connection: %w"), err) } fmt.Printf(i18n.G("SSH SFTP listening on %v")+"\n", listener.Addr()) if sshConfig.PasswordCallback != nil { fmt.Printf(i18n.G("Login with username %q and password %q")+"\n", authUser, authPass) } else { fmt.Println(i18n.G("Login without username and password")) } for { // Wait for new SSH connections. nConn, err := listener.Accept() if err != nil { return fmt.Errorf(i18n.G("Failed to accept incoming connection: %w"), err) } // Handle each SSH connection in its own go routine. go func() { fmt.Printf(i18n.G("SSH client connected %q")+"\n", nConn.RemoteAddr()) defer fmt.Printf(i18n.G("SSH client disconnected %q")+"\n", nConn.RemoteAddr()) defer func() { _ = nConn.Close() }() // Before use, a handshake must be performed on the incoming net.Conn. _, chans, reqs, err := ssh.NewServerConn(nConn, sshConfig) if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Failed SSH handshake with client %q: %v")+"\n", nConn.RemoteAddr(), err) return } // The incoming Request channel must be serviced. go ssh.DiscardRequests(reqs) // Service the incoming Channel requests. for newChannel := range chans { localChannel := newChannel // Channels have a type, depending on the application level protocol intended. // In the case of an SFTP session, this is "subsystem" with a payload string of // "sftp" if localChannel.ChannelType() != "session" { _ = localChannel.Reject(ssh.UnknownChannelType, "unknown channel type") fmt.Fprintf(os.Stderr, i18n.G("Unknown channel type for client %q: %s")+"\n", nConn.RemoteAddr(), localChannel.ChannelType()) continue } // Accept incoming channel request. channel, requests, err := localChannel.Accept() if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Failed accepting channel client %q: %v")+"\n", err) return } // Sessions have out-of-band requests such as "shell", "pty-req" and "env". // Here we handle only the "subsystem" request. go func(in <-chan *ssh.Request) { for req := range in { ok := false switch req.Type { case "subsystem": if string(req.Payload[4:]) == "sftp" { ok = true } } _ = req.Reply(ok, nil) } }(requests) // Handle each channel in its own go routine. go func() { defer func() { _ = channel.Close() }() // Connect to the instance's SFTP server. sftpConn, err := sftpConn() if err != nil { fmt.Fprintf(os.Stderr, i18n.G("Failed connecting to instance SFTP for client %q: %v")+"\n", nConn.RemoteAddr(), err) return } defer func() { _ = sftpConn.Close() }() // Copy SFTP data between client and remote instance. ctx, cancel := context.WithCancel(ctx) go func() { _, err := util.SafeCopy(channel, sftpConn) if ctx.Err() == nil { if err != nil { fmt.Fprintf(os.Stderr, i18n.G("I/O copy from instance to SSH failed: %v")+"\n", err) } else { fmt.Printf(i18n.G("Instance disconnected for client %q")+"\n", nConn.RemoteAddr()) } } cancel() // Prevents error output when other util.SafeCopy finishes. _ = channel.Close() }() _, err = util.SafeCopy(sftpConn, channel) if err != nil && ctx.Err() == nil { fmt.Fprintf(os.Stderr, i18n.G("I/O copy from SSH to instance failed: %v")+"\n", err) } cancel() // Prevents error output when other util.SafeCopy finishes. _ = sftpConn.Close() }() } }() } } // formatRemote formats a remote object. func formatRemote(conf *config.Config, p *u.Parsed) string { if p.RemoteName == conf.DefaultRemote { return p.RemoteObject.String } return p.RemoteName + ":" + p.RemoteObject.String } // normalizePath normalizes a path and return whether it looks like a directory. func normalizePath(path string) (string, bool) { // On Windows, the SFTP server expects the file path to start with `/`. path = "/" + path return filepath.Clean(path), strings.HasSuffix(path, "/") } // isStdin returns whether the provided path looks like stdin. func isStdin(path string) bool { return slices.Contains([]string{"-", "/dev/stdin", "/dev/fd/0"}, path) } // isStdout returns whether the provided path looks like stdout. func isStdout(path string) bool { return slices.Contains([]string{"-", "/dev/stdout", "/dev/fd/1"}, path) } incus-7.0.0/cmd/incus/utils_copy.go000066400000000000000000000107531517523235500172620ustar00rootroot00000000000000package main import ( "errors" "fmt" "os" "path/filepath" "strings" "github.com/pkg/sftp" "github.com/lxc/incus/v7/internal/i18n" ) // fileCopiable abstracts commands that pull files. type fileCopiable struct { flagRecursive bool flagNoDereference bool flagFollow bool flagDereference bool } // preCheck performs flag validation. func (f *fileCopiable) preCheck() error { // --no-dereference/-P, --follow/-H, and --dereference/-L are mutually exclusive. found := 0 if f.flagNoDereference { found++ } if f.flagFollow { found++ } if f.flagDereference { found++ } if found > 1 { return errors.New(i18n.G("--no-dereference/-P, --follow/-H, and --dereference/-L are mutually exclusive")) } return nil } // pullable abstracts commands that pull files. type pullable struct { fileCopiable } // preCheck performs flag validation. func (p *pullable) preCheck(target string) error { err := p.fileCopiable.preCheck() if err != nil { return err } // Using stdout as a target implicitly sets -H if -L is not set, but fails if -P is set. if isStdout(target) { if p.flagDereference { return nil } if p.flagNoDereference { return errors.New(i18n.G("--no-dereference/-P cannot be used together with stdout as a target")) } p.flagFollow = true } return nil } // statFile returns the proper stat struct for the given flags, along with the normalized file name. func (p *pullable) statFile(sftpConn *sftp.Client, path string) (os.FileInfo, string, error) { normalizedPath, _ := normalizePath(path) srcLstat, err := sftpConn.Lstat(normalizedPath) if err != nil { return nil, "", err } isSymlink := srcLstat.Mode()&os.ModeSymlink != 0 srcStat := srcLstat var errSymlink error if isSymlink { // We defer dereferencing error handling, as chances are we aren’t even interested in the // symlink target. srcStat, errSymlink = sftpConn.Stat(normalizedPath) } directoryRequested := strings.HasSuffix(path, "/") if errSymlink == nil { // Let’s be extra careful and check that explicit requests for directories actually point to // directories. if directoryRequested && !srcStat.IsDir() { return nil, "", fmt.Errorf(i18n.G("%s is not a directory"), normalizedPath) } // Here, we perform a special handling if -P is used on a directory symlink. if srcStat.IsDir() && !p.flagRecursive && (!isSymlink || !p.flagNoDereference) { return nil, "", errors.New(i18n.G("--recursive/-r is required when pulling directories")) } } // Under a few conditions, return the file the link points to and not the link itself. if p.flagDereference || !p.flagRecursive && !p.flagNoDereference || isSymlink && p.flagFollow || directoryRequested { if errSymlink != nil { return nil, "", err } return srcStat, normalizedPath, nil } return srcLstat, normalizedPath, nil } // pushable abstracts commands that push files. type pushable struct { fileCopiable } // statFile returns the proper stat struct for the given flags, along with the walkable file name. func (p *pushable) statFile(path string) (os.FileInfo, string, error) { absPath, err := filepath.Abs(path) if err != nil { return nil, "", err } srcLstat, err := os.Lstat(absPath) if err != nil { return nil, "", err } isSymlink := srcLstat.Mode()&os.ModeSymlink != 0 srcStat := srcLstat var errSymlink error if isSymlink { // We defer dereferencing error handling, as chances are we aren’t even interested in the // symlink target. srcStat, errSymlink = os.Stat(absPath) } if errSymlink == nil { // Here, we perform a special handling if -P is used on a directory symlink. if srcStat.IsDir() && !p.flagRecursive && (!isSymlink || !p.flagNoDereference) { return nil, "", errors.New(i18n.G("--recursive/-r is required when pulling directories")) } } // Under a few conditions, return the file the link points to and not the link itself. if p.flagDereference || !p.flagRecursive && !p.flagNoDereference || isSymlink && p.flagFollow || strings.HasSuffix(path, "/") { if errSymlink != nil { return nil, "", err } // This is a bit of a hack, but as we are using `filepath.Walk`, we need to point to an actual // directory to be able to walk it, hence the early dereferencing. if isSymlink && srcStat.IsDir() { target, err := os.Readlink(absPath) if err != nil { return nil, "", err } if !filepath.IsAbs(target) { target = filepath.Join(filepath.Dir(absPath), target) } absPath = target } return srcStat, absPath, nil } return srcLstat, absPath, nil } incus-7.0.0/cmd/incus/utils_notwindows.go000066400000000000000000000006651517523235500205240ustar00rootroot00000000000000//go:build (linux && !appengine) || darwin || freebsd || openbsd package main import ( "io" "os" "golang.org/x/sys/unix" ) func getStdout() io.WriteCloser { return os.Stdout } func getStdoutFd() int { return unix.Stdout } func getStdinFd() int { return unix.Stdin } func getEnviron() []string { return unix.Environ() } func doExec(argv0 string, argv []string, envv []string) error { return unix.Exec(argv0, argv, envv) } incus-7.0.0/cmd/incus/utils_properties.go000066400000000000000000000127111517523235500205000ustar00rootroot00000000000000package main import ( "errors" "fmt" "reflect" "strconv" "strings" "time" "github.com/mitchellh/mapstructure" "github.com/lxc/incus/v7/internal/i18n" ) // stringToTimeHookFunc is a custom decoding hook that converts string values to time.Time using the given layout. func stringToTimeHookFunc(layout string) mapstructure.DecodeHookFuncType { return func(from reflect.Type, to reflect.Type, data any) (any, error) { if from.Kind() == reflect.String && to == reflect.TypeOf(time.Time{}) { strValue, ok := data.(string) if !ok { return nil, errors.New("Unexpected data type") } t, err := time.Parse(layout, strValue) if err != nil { return nil, err } return t, nil } return data, nil } } // stringToBoolHookFunc is a custom decoding hook that converts string values to bool. func stringToBoolHookFunc() mapstructure.DecodeHookFunc { return func(f reflect.Kind, t reflect.Kind, data any) (any, error) { if f != reflect.String || t != reflect.Bool { return data, nil } str, ok := data.(string) if !ok { return false, errors.New("Unexpected data type") } str = strings.ToLower(str) switch str { case "1", "t", "true": return true, nil case "0", "f", "false": return false, nil default: return false, fmt.Errorf(i18n.G("Invalid boolean value: %s"), str) } } } // stringToIntHookFunc is a custom decoding hook that converts string values to int. func stringToIntHookFunc() mapstructure.DecodeHookFunc { return func(f reflect.Kind, t reflect.Kind, data any) (any, error) { if f != reflect.String || (t != reflect.Int && t != reflect.Int8 && t != reflect.Int16 && t != reflect.Int32 && t != reflect.Int64) { return data, nil } str, ok := data.(string) if !ok { return data, errors.New("Unexpected data type") } value, err := strconv.Atoi(str) if err != nil { return data, err } return value, nil } } // stringToFloatHookFunc is a custom decoding hook that converts string values to float. func stringToFloatHookFunc() mapstructure.DecodeHookFunc { return func(f reflect.Kind, t reflect.Kind, data any) (any, error) { if f != reflect.String || (t != reflect.Float32 && t != reflect.Float64) { return data, nil } str, ok := data.(string) if !ok { return data, errors.New("Unexpected data type") } value, err := strconv.ParseFloat(str, 64) if err != nil { return data, err } return value, nil } } // getFieldByJSONTag gets the value of a struct field by its JSON tag. func getFieldByJSONTag(obj any, tag string) (any, error) { var res any v := reflect.ValueOf(obj) if v.Kind() == reflect.Ptr { v = v.Elem() } if v.Kind() != reflect.Struct { return nil, fmt.Errorf(i18n.G("Expected a struct, got a %v"), v.Kind()) } ok, res := getFromStruct(v, tag) if !ok { return nil, fmt.Errorf(i18n.G("The property with tag %q does not exist"), tag) } return res, nil } // getFromStruct scans a struct for a field with the given JSON tag, including fields of inline structs. func getFromStruct(v reflect.Value, tag string) (bool, any) { for i := range v.NumField() { field := v.Field(i) jsonTag := v.Type().Field(i).Tag.Get("json") // Ignore any options that might be specified after a comma in the tag. commaIdx := strings.Index(jsonTag, ",") if commaIdx > 0 { jsonTag = jsonTag[:commaIdx] } if strings.EqualFold(jsonTag, tag) { return true, field.Interface() } if v.Type().Field(i).Anonymous { if field.Kind() == reflect.Ptr { field = field.Elem() } if field.Kind() == reflect.Struct { ok, res := getFromStruct(field, tag) if ok { return ok, res } } } } return false, nil } // setFieldByJSONTag sets the value of a struct field by its JSON tag. func setFieldByJSONTag(obj any, tag string, value any) { v := reflect.ValueOf(obj).Elem() var fieldName string for i := range v.NumField() { jsonTag := v.Type().Field(i).Tag.Get("json") commaIdx := strings.Index(jsonTag, ",") if commaIdx > 0 { jsonTag = jsonTag[:commaIdx] } if strings.EqualFold(jsonTag, tag) { fieldName = v.Type().Field(i).Name } } if fieldName != "" { if v.FieldByName(fieldName).CanSet() { v.FieldByName(fieldName).Set(reflect.ValueOf(value)) } } } // unsetFieldByJSONTag unsets (give a default value) the value of a struct field by its JSON tag. func unsetFieldByJSONTag(obj any, tag string) error { v, err := getFieldByJSONTag(obj, tag) if err != nil { return err } switch v.(type) { case string: setFieldByJSONTag(obj, tag, "") case int: setFieldByJSONTag(obj, tag, 0) case bool: setFieldByJSONTag(obj, tag, false) case float32, float64: setFieldByJSONTag(obj, tag, 0.0) case time.Time: setFieldByJSONTag(obj, tag, time.Time{}) case *time.Time: setFieldByJSONTag(obj, tag, &time.Time{}) } return nil } // unpackKVToWritable unpacks a map[string]string into a writable API struct. func unpackKVToWritable(writable any, keys map[string]string) error { data := make(map[string]any) for k, v := range keys { data[k] = v } decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ TagName: "json", Result: writable, DecodeHook: mapstructure.ComposeDecodeHookFunc( stringToBoolHookFunc(), stringToIntHookFunc(), stringToFloatHookFunc(), stringToTimeHookFunc(time.RFC3339), ), }) if err != nil { return fmt.Errorf(i18n.G("Error creating decoder: %v"), err) } err = decoder.Decode(data) if err != nil { return fmt.Errorf(i18n.G("Error decoding data: %v"), err) } return nil } incus-7.0.0/cmd/incus/utils_properties_test.go000066400000000000000000000107671517523235500215500ustar00rootroot00000000000000package main import ( "reflect" "testing" "time" "github.com/stretchr/testify/suite" ) type utilsPropertiesTestSuite struct { suite.Suite } func TestUtilsPropertiesTestSuite(t *testing.T) { suite.Run(t, &utilsPropertiesTestSuite{}) } func (s *utilsPropertiesTestSuite) TestStringToTimeHookFuncValidData() { layout := time.RFC3339 hook := stringToTimeHookFunc(layout) result, err := hook(reflect.TypeOf(""), reflect.TypeOf(time.Time{}), "2023-07-12T07:34:00Z") s.NoError(err) s.Equal(time.Date(2023, 7, 12, 7, 34, 0, 0, time.UTC), result) } func (s *utilsPropertiesTestSuite) TestStringToTimeHookFuncInvalidData() { layout := time.RFC3339 hook := stringToTimeHookFunc(layout) _, err := hook(reflect.TypeOf(""), reflect.TypeOf(time.Time{}), "not a time") s.Error(err, "Expected an error but got nil") } func (s *utilsPropertiesTestSuite) TestStringToBoolHookFuncValidData() { hookFunc := stringToBoolHookFunc() hook, ok := hookFunc.(func(reflect.Kind, reflect.Kind, any) (any, error)) s.Equal(true, ok) result, err := hook(reflect.String, reflect.Bool, "t") s.NoError(err) s.Equal(true, result) } func (s *utilsPropertiesTestSuite) TestStringToBoolHookFuncInvalidData() { hookFunc := stringToBoolHookFunc() hook, ok := hookFunc.(func(reflect.Kind, reflect.Kind, any) (any, error)) s.Equal(true, ok) _, err := hook(reflect.String, reflect.Bool, "not a boolean") s.Error(err, "Expected an error but got nil") } func (s *utilsPropertiesTestSuite) TestStringToIntHookFuncValidData() { hookFunc := stringToIntHookFunc() hook, ok := hookFunc.(func(reflect.Kind, reflect.Kind, any) (any, error)) s.Equal(true, ok) result, err := hook(reflect.String, reflect.Int, "123") s.NoError(err) s.Equal(123, result) } func (s *utilsPropertiesTestSuite) TestStringToIntHookFuncInvalidData() { hookFunc := stringToIntHookFunc() hook, ok := hookFunc.(func(reflect.Kind, reflect.Kind, any) (any, error)) s.Equal(true, ok) _, err := hook(reflect.String, reflect.Int, "not an int") s.Error(err, "Expected an error but got nil") } func (s *utilsPropertiesTestSuite) TestStringToFloatHookFuncValidData() { hookFunc := stringToFloatHookFunc() hook, ok := hookFunc.(func(reflect.Kind, reflect.Kind, any) (any, error)) s.Equal(true, ok) result, err := hook(reflect.String, reflect.Float64, "123.45") s.NoError(err) s.Equal(123.45, result) } func (s *utilsPropertiesTestSuite) TestStringToFloatHookFuncInvalidData() { hookFunc := stringToFloatHookFunc() hook, ok := hookFunc.(func(reflect.Kind, reflect.Kind, any) (any, error)) s.Equal(true, ok) _, err := hook(reflect.String, reflect.Float64, "not a float") s.Error(err, "Expected an error but got nil") } type testStruct struct { Name string `json:"name"` Age int `json:"age"` } func (s *utilsPropertiesTestSuite) TestSetFieldByJSONTagSettable() { ts := testStruct{ Name: "John Doe", Age: 30, } setFieldByJSONTag(&ts, "name", "Jane Doe") s.Equal("Jane Doe", ts.Name) } func (s *utilsPropertiesTestSuite) TestSetFieldByJSONTagNonSettable() { ts := testStruct{ Name: "John Doe", Age: 30, } setFieldByJSONTag(&ts, "invalid name", "Jane Doe") s.NotEqual(ts.Name, "Jane Doe") } func (s *utilsPropertiesTestSuite) TestUnsetFieldByJSONTagValid() { ts := testStruct{ Name: "John Doe", Age: 30, } err := unsetFieldByJSONTag(&ts, "name") s.NoError(err) s.Equal("", ts.Name) } func (s *utilsPropertiesTestSuite) TestUnsetFieldByJSONTagInvalid() { ts := testStruct{ Name: "John Doe", Age: 30, } err := unsetFieldByJSONTag(&ts, "invalid") s.Error(err, "Expected an error but got nil") } type writableStruct struct { Name string `json:"name"` Age int `json:"age"` Score float64 `json:"score"` Alive bool `json:"alive"` Birth time.Time `json:"birth"` } func (s *utilsPropertiesTestSuite) TestUnpackKVToWritable() { ws := &writableStruct{} keys := map[string]string{ "name": "John Doe", "age": "30", "score": "85.5", "alive": "true", "birth": "2000-01-01T00:00:00Z", } err := unpackKVToWritable(ws, keys) s.NoError(err) s.Equal("John Doe", ws.Name) s.Equal(30, ws.Age) s.Equal(85.5, ws.Score) s.Equal(true, ws.Alive) s.Equal("2000-01-01T00:00:00Z", ws.Birth.Format(time.RFC3339)) } func (s *utilsPropertiesTestSuite) TestUnpackKVToWritableInvalidData() { ws := &writableStruct{} keys := map[string]string{ "name": "John Doe", "age": "not an int", "score": "not a float", "alive": "not a bool", "birth": "not a time", } err := unpackKVToWritable(ws, keys) s.Error(err, "Expected an error but got nil") } incus-7.0.0/cmd/incus/utils_sftp.go000066400000000000000000000225361517523235500172660ustar00rootroot00000000000000package main import ( "errors" "fmt" "io" "io/fs" "os" "path/filepath" "strings" "github.com/pkg/sftp" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/i18n" internalIO "github.com/lxc/incus/v7/internal/io" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) func sftpSetOwnerMode(sftpConn *sftp.Client, targetPath string, args incus.InstanceFileArgs) error { // Skip if not on UNIX. _, err := sftpConn.StatVFS("/") if err != nil { return nil } // Get the current stat information. st, err := sftpConn.Stat(targetPath) if err != nil { return err } fileStat, ok := st.Sys().(*sftp.FileStat) if !ok { return fmt.Errorf("Invalid filestat data for %q", targetPath) } // Set owner. if args.UID >= 0 || args.GID >= 0 { if args.UID == -1 { args.UID = int64(fileStat.UID) } if args.GID == -1 { args.GID = int64(fileStat.GID) } err = sftpConn.Chown(targetPath, int(args.UID), int(args.GID)) if err != nil { return err } } // Set mode. if args.Mode >= 0 { err = sftpConn.Chmod(targetPath, fs.FileMode(args.Mode)) if err != nil { return err } } return nil } func sftpCreateFile(sftpConn *sftp.Client, targetPath string, args incus.InstanceFileArgs, push bool) error { switch args.Type { case "file": file, err := sftpConn.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC) if err != nil { return fmt.Errorf(i18n.G("Failed to open target file %q: %w"), targetPath, err) } defer func() { _ = file.Close() }() if push { _, err = util.SafeCopy(file, args.Content) if err != nil { return err } } err = sftpSetOwnerMode(sftpConn, targetPath, args) if err != nil { return err } case "directory": err := sftpConn.MkdirAll(targetPath) if err != nil { return err } err = sftpSetOwnerMode(sftpConn, targetPath, args) if err != nil { return err } case "symlink": // If already a symlink, re-create it. fInfo, err := sftpConn.Lstat(targetPath) if err == nil && fInfo.Mode()&os.ModeSymlink == os.ModeSymlink { err = sftpConn.Remove(targetPath) if err != nil { return err } } dest, err := io.ReadAll(args.Content) if err != nil { return err } err = sftpConn.Symlink(string(dest), targetPath) if err != nil { return err } } return nil } func sftpRecursivePullFile(sftpConn *sftp.Client, fInfo os.FileInfo, source string, normalizedSource string, targetDir string, quiet bool, dereference bool, createRoot bool) error { var fileType string if fInfo.IsDir() { fileType = "directory" } else if fInfo.Mode()&os.ModeSymlink == os.ModeSymlink { fileType = "symlink" } else { fileType = "file" } target := targetDir if createRoot { root := filepath.Base(source) // `cp` has a special behavior with the following paths. if root == "." || root == ".." { root = "" } target = filepath.Join(targetDir, root) } logger.Infof("Pulling %s from %s (%s)", target, normalizedSource, fileType) if fileType == "directory" { err := os.Mkdir(target, fInfo.Mode()) if err != nil { // If the error isn’t that the path already exists, there’s nothing we can do about it. if !errors.Is(err, os.ErrExist) { return err } // The error is pretty wide, so we must check whether the existing path it a directory (in // which case we can continue) or not (in which case we must fail). stat, statErr := os.Stat(target) if statErr != nil || !stat.IsDir() { // Even if the stat error can contain interesting data, the actual error that led us here in // the first place is `err`. return err } } entries, err := sftpConn.ReadDir(normalizedSource) if err != nil { return err } for _, ent := range entries { nextP := filepath.Join(normalizedSource, ent.Name()) stat := sftpConn.Lstat if dereference { stat = sftpConn.Stat } nextInfo, err := stat(nextP) if err != nil { return err } err = sftpRecursivePullFile(sftpConn, nextInfo, nextP, nextP, target, quiet, dereference, true) if err != nil { return err } } } else if fileType == "file" { src, err := sftpConn.Open(normalizedSource) if err != nil { return err } defer func() { _ = src.Close() }() dst, err := os.Create(target) if err != nil { return err } defer func() { _ = dst.Close() }() err = os.Chmod(target, fInfo.Mode()) if err != nil { return err } progress := cli.ProgressRenderer{ Format: fmt.Sprintf(i18n.G("Pulling %s from %s: %%s"), normalizedSource, target), Quiet: quiet, } writer := &ioprogress.ProgressWriter{ WriteCloser: dst, Tracker: &ioprogress.ProgressTracker{ Handler: func(bytesReceived int64, speed int64) { progress.UpdateProgress(ioprogress.ProgressData{ Text: fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(bytesReceived, 2), units.GetByteSizeString(speed, 2)), }) }, }, } _, err = util.SafeCopy(writer, src) if err != nil { progress.Done("") return err } err = src.Close() if err != nil { progress.Done("") return err } err = dst.Close() if err != nil { progress.Done("") return err } progress.Done("") } else if fileType == "symlink" { linkTarget, err := sftpConn.ReadLink(normalizedSource) if err != nil { return err } err = os.Symlink(linkTarget, target) if err != nil { return err } } else { return fmt.Errorf(i18n.G("Unknown file type '%s'"), fileType) } return nil } func sftpRecursivePushFile(sftpConn *sftp.Client, walkableSource string, source string, target string, quiet bool, dereference bool, createRoot bool) error { root := "" if createRoot { root = filepath.Base(source) // `cp` has a special behavior with the following paths. if root == "." || root == ".." { root = "" } } sendFile := func(p string, fInfo os.FileInfo, err error) error { if err != nil { return fmt.Errorf(i18n.G("Failed to walk path for %s: %s"), p, err) } // Detect unsupported files if !fInfo.Mode().IsRegular() && !fInfo.Mode().IsDir() && fInfo.Mode()&os.ModeSymlink != os.ModeSymlink { return fmt.Errorf(i18n.G("'%s' isn't a supported file type"), p) } // Prepare for file transfer targetPath := filepath.Join(target, root, p[len(walkableSource):]) mode, uid, gid := internalIO.GetOwnerMode(fInfo) args := incus.InstanceFileArgs{ UID: int64(uid), GID: int64(gid), Mode: int(mode.Perm()), } var readCloser io.ReadCloser if fInfo.IsDir() { // Directory handling args.Type = "directory" } else if fInfo.Mode()&os.ModeSymlink == os.ModeSymlink && !dereference { // Symlink handling symlinkTarget, err := os.Readlink(p) if err != nil { return err } args.Type = "symlink" args.Content = strings.NewReader(symlinkTarget) readCloser = io.NopCloser(args.Content) } else { // File handling f, err := os.Open(p) if err != nil { return fmt.Errorf(i18n.G("Failed to open source file %q: %v"), p, err) } defer func() { _ = f.Close() }() args.Type = "file" args.Content = f readCloser = f } progress := cli.ProgressRenderer{ Format: fmt.Sprintf(i18n.G("Pushing %s to %s: %%s"), p, targetPath), Quiet: quiet, } if args.Type != "directory" { contentLength, err := args.Content.Seek(0, io.SeekEnd) if err != nil { return err } _, err = args.Content.Seek(0, io.SeekStart) if err != nil { return err } args.Content = internalIO.NewReadSeeker(&ioprogress.ProgressReader{ ReadCloser: readCloser, Tracker: &ioprogress.ProgressTracker{ Length: contentLength, Handler: func(percent int64, speed int64) { progress.UpdateProgress(ioprogress.ProgressData{ Text: fmt.Sprintf("%d%% (%s/s)", percent, units.GetByteSizeString(speed, 2)), }) }, }, }, args.Content) } logger.Infof("Pushing %s to %s (%s)", p, targetPath, args.Type) err = sftpCreateFile(sftpConn, targetPath, args, true) if err != nil { if args.Type != "directory" { progress.Done("") } return err } if args.Type != "directory" { progress.Done("") } return nil } return filepath.Walk(walkableSource, sendFile) } func sftpRecursiveMkdir(sftpConn *sftp.Client, p string, mode *os.FileMode, uid int64, gid int64) error { /* special case, every instance has a /, we don't need to do anything */ if p == "/" { return nil } // Remove trailing "/" e.g. /A/B/C/. Otherwise we will end up with an // empty array entry "" which will confuse the Mkdir() loop below. pclean := filepath.Clean(p) parts := strings.Split(pclean, "/") i := len(parts) for ; i >= 1; i-- { cur := filepath.Join(parts[:i]...) fInfo, err := sftpConn.Lstat(cur) if err != nil { continue } if !fInfo.IsDir() { return fmt.Errorf(i18n.G("%s is not a directory"), cur) } i++ break } for ; i <= len(parts); i++ { cur := filepath.Join(parts[:i]...) if cur == "" { continue } cur = "/" + cur cur = strings.TrimLeft(cur, "/") modeArg := -1 if mode != nil { modeArg = int(mode.Perm()) } args := incus.InstanceFileArgs{ UID: uid, GID: gid, Mode: modeArg, Type: "directory", } logger.Infof("Creating %s (%s)", cur, args.Type) err := sftpCreateFile(sftpConn, cur, args, false) if err != nil { return err } } return nil } incus-7.0.0/cmd/incus/utils_test.go000066400000000000000000000045241517523235500172660ustar00rootroot00000000000000package main import ( "reflect" "testing" "github.com/stretchr/testify/suite" "github.com/lxc/incus/v7/shared/api" ) type utilsTestSuite struct { suite.Suite } func TestUtilsTestSuite(t *testing.T) { suite.Run(t, &utilsTestSuite{}) } func (s *utilsTestSuite) TestisAliasesSubsetTrue() { a1 := []api.ImageAlias{ {Name: "foo"}, } a2 := []api.ImageAlias{ {Name: "foo"}, {Name: "bar"}, {Name: "baz"}, } s.Exactly(isAliasesSubset(a1, a2), true) } func (s *utilsTestSuite) TestisAliasesSubsetFalse() { a1 := []api.ImageAlias{ {Name: "foo"}, {Name: "bar"}, } a2 := []api.ImageAlias{ {Name: "foo"}, {Name: "baz"}, } s.Exactly(isAliasesSubset(a1, a2), false) } func (s *utilsTestSuite) TestgetExistingAliases() { images := []api.ImageAliasesEntry{ {Name: "foo"}, {Name: "bar"}, {Name: "baz"}, } aliases := getExistingAliases([]string{"bar", "foo", "other"}, images) s.Exactly([]api.ImageAliasesEntry{images[0], images[1]}, aliases) } func (s *utilsTestSuite) TestgetExistingAliasesEmpty() { images := []api.ImageAliasesEntry{ {Name: "foo"}, {Name: "bar"}, {Name: "baz"}, } aliases := getExistingAliases([]string{"other1", "other2"}, images) s.Exactly([]api.ImageAliasesEntry{}, aliases) } func (s *utilsTestSuite) TestStructHasFields() { s.Equal(structHasField(reflect.TypeOf(api.Image{}), "type"), true) s.Equal(structHasField(reflect.TypeOf(api.Image{}), "public"), true) s.Equal(structHasField(reflect.TypeOf(api.Image{}), "foo"), false) } func (s *utilsTestSuite) TestGetServerSupportedFilters() { filters := []string{ "foo", "type=container", "user.blah=a", "status=running,stopped", } supportedFilters, unsupportedFilters := getServerSupportedFilters(filters, []string{}, false) s.Equal([]string{"type=container", "user.blah=a", "status=running,stopped"}, supportedFilters) s.Equal([]string{"foo"}, unsupportedFilters) supportedFilters, unsupportedFilters = getServerSupportedFilters(filters, []string{}, true) s.Equal([]string{"foo", "type=container", "user.blah=a", "status=running,stopped"}, supportedFilters) s.Equal([]string{}, unsupportedFilters) supportedFilters, unsupportedFilters = getServerSupportedFilters(filters, []string{"type", "status"}, true) s.Equal([]string{"foo", "user.blah=a"}, supportedFilters) s.Equal([]string{"type=container", "status=running,stopped"}, unsupportedFilters) } incus-7.0.0/cmd/incus/utils_windows.go000066400000000000000000000010071517523235500177720ustar00rootroot00000000000000//go:build windows package main import ( "errors" "io" "os" "github.com/mattn/go-colorable" "golang.org/x/sys/windows" ) func getStdout() io.WriteCloser { return &WrappedWriteCloser{os.Stdout, colorable.NewColorableStdout()} } func getStdoutFd() int { return int(windows.Stdout) } func getStdinFd() int { return int(windows.Stdin) } func getEnviron() []string { return windows.Environ() } func doExec(argv0 string, argv []string, envv []string) error { return errors.New("not supported by windows") } incus-7.0.0/cmd/incus/version.go000066400000000000000000000023131517523235500165460ustar00rootroot00000000000000package main import ( "fmt" "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/internal/version" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdVersion struct { global *cmdGlobal } var cmdVersionUsage = u.Usage{u.Colon(u.Remote).Optional()} func (c *cmdVersion) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("version", cmdVersionUsage...) cmd.Short = i18n.G("Show local and remote versions") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show local and remote versions`)) cmd.RunE = c.run return cmd } func (c *cmdVersion) run(cmd *cobra.Command, args []string) error { parsed, err := cmdVersionUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } fmt.Printf(i18n.G("Client version: %s\n"), version.Version) ver := i18n.G("unreachable") resources, err := c.global.parseServers(parsed[0].String) if err == nil { resource := resources[0] info, _, err := resource.server.GetServer() if err == nil { ver = info.Environment.ServerVersion } } fmt.Printf(i18n.G("Server version: %s\n"), ver) return nil } incus-7.0.0/cmd/incus/wait.go000066400000000000000000000076041517523235500160350ustar00rootroot00000000000000package main import ( "fmt" "strings" "time" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdWait struct { global *cmdGlobal flagInterval int flagTimeOut int } var cmdWaitUsage = u.Usage{u.Instance.Remote(), u.Placeholder(i18n.G("condition"))} func (c *cmdWait) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("wait", cmdWaitUsage...) cmd.Short = i18n.G("Wait for an instance to satisfy a condition") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `Wait for an instance to satisfy a condition Supported Conditions: agent Wait for the VM agent to be running ip Wait for any globally routable IP address ipv4 Wait for a globally routable IPv4 address ipv6 Wait for a globally routable IPv6 address status=STATUS Wait for the instance status to become STATUS`)) cmd.Example = cli.FormatSection("", i18n.G(`incus wait v1 agent Wait for VM instance v1 to have a functional agent.`)) cmd.RunE = c.run cli.AddIntFlag(cmd.Flags(), &c.flagInterval, "interval", 5, i18n.G("Polling interval (in seconds)")) cli.AddIntFlag(cmd.Flags(), &c.flagTimeOut, "timeout", -1, i18n.G("Maximum wait time")) cmd.ValidArgsFunction = func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return c.global.cmpInstances(toComplete) } return nil, cobra.ShellCompDirectiveNoFileComp } return cmd } func (c *cmdWait) run(cmd *cobra.Command, args []string) error { parsed, err := cmdWaitUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer instanceName := parsed[0].RemoteObject.String condition := parsed[1].String inst, _, err := d.GetInstance(instanceName) if err != nil { return err } start := time.Now() for { ok, err := c.checkCondition(d, inst, condition) if err != nil { return err } if ok { return nil } if c.flagTimeOut > 0 && time.Since(start) > time.Duration(c.flagTimeOut)*time.Second { return fmt.Errorf("Timeout for instance %s for condition: %s", formatRemote(c.global.conf, parsed[0]), condition) } time.Sleep(time.Duration(c.flagInterval) * time.Second) } } // check the conditions. func (c *cmdWait) checkCondition(d incus.InstanceServer, inst *api.Instance, condition string) (bool, error) { state, _, err := d.GetInstanceState(inst.Name) if err != nil { return false, nil } switch { case strings.ToLower(condition) == "agent": if inst.Type != "virtual-machine" { return false, fmt.Errorf("The agent condition is only valid for virtual-machines") } if state.Processes > 0 { return true, nil } case condition == "ip": if c.hasGlobalIP(state, "") { return true, nil } case condition == "ipv4": if c.hasGlobalIP(state, "ipv4") { return true, nil } case condition == "ipv6": if c.hasGlobalIP(state, "ipv6") { return true, nil } case strings.HasPrefix(condition, "status="): status := strings.TrimPrefix(condition, "status=") if strings.EqualFold(state.Status, status) { return true, nil } default: return false, fmt.Errorf("Unknown condition %q", condition) } return false, nil } // Check if the instance has a globally routable IP address of the specified family. func (c *cmdWait) hasGlobalIP(state *api.InstanceState, family string) bool { for _, net := range state.Network { for _, addr := range net.Addresses { if addr.Scope != "global" { continue } switch family { case "": return true case "ipv4": if addr.Family == "inet" { return true } case "ipv6": if addr.Family == "inet6" { return true } } } } return false } incus-7.0.0/cmd/incus/warning.go000066400000000000000000000232051517523235500165310ustar00rootroot00000000000000package main import ( "errors" "fmt" "os" "sort" "strings" "github.com/spf13/cobra" yaml "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" ) type warningColumn struct { Name string Data func(api.Warning) string } type cmdWarning struct { global *cmdGlobal } func (c *cmdWarning) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("warning") cmd.Short = i18n.G("Manage warnings") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Manage warnings`)) cmd.Hidden = true // List warningListCmd := cmdWarningList{global: c.global, warning: c} cmd.AddCommand(warningListCmd.command()) // Acknowledge warningAcknowledgeCmd := cmdWarningAcknowledge{global: c.global, warning: c} cmd.AddCommand(warningAcknowledgeCmd.command()) // Show warningShowCmd := cmdWarningShow{global: c.global, warning: c} cmd.AddCommand(warningShowCmd.command()) // Delete warningDeleteCmd := cmdWarningDelete{global: c.global, warning: c} cmd.AddCommand(warningDeleteCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, _ []string) { _ = cmd.Usage() } return cmd } // List. type cmdWarningList struct { global *cmdGlobal warning *cmdWarning flagColumns string flagFormat string flagAll bool } const defaultWarningColumns = "utSscpLl" var cmdWarningListUsage = u.Usage{u.RemoteColonOpt} func (c *cmdWarningList) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("list", cmdWarningListUsage...) cmd.Aliases = []string{"ls"} cmd.Short = i18n.G("List warnings") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G( `List warnings The -c option takes a (optionally comma-separated) list of arguments that control which warning attributes to output when displaying in table or csv format. Default column layout is: utSscpLl Column shorthand chars: c - Count l - Last seen L - Location f - First seen p - Project s - Severity S - Status u - UUID t - Type`)) cli.AddStringFlag(cmd.Flags(), &c.flagColumns, "columns|c", defaultWarningColumns, "", i18n.G("Columns")) cli.AddStringFlag(cmd.Flags(), &c.flagFormat, "format|f", c.global.defaultListFormat(), "", i18n.G(`Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`)) cli.AddBoolFlag(cmd.Flags(), &c.flagAll, "all|a", i18n.G("List all warnings")) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run return cmd } func (c *cmdWarningList) run(cmd *cobra.Command, args []string) error { parsed, err := cmdWarningListUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer allWarnings, err := d.GetWarnings() if err != nil { return err } // Per default, acknowledged and resolved warnings are not shown. Using the --all flag will show // those as well. var warnings []api.Warning if c.flagAll { warnings = allWarnings } else { for _, warning := range allWarnings { if warning.Status == "acknowledged" || warning.Status == "resolved" { continue } warnings = append(warnings, warning) } } // Process the columns columns, err := c.parseColumns(d.IsClustered()) if err != nil { return err } // Render the table data := [][]string{} for _, warning := range warnings { row := []string{} for _, column := range columns { row = append(row, column.Data(warning)) } data = append(data, row) } sort.Sort(cli.StringList(data)) rawData := make([]*api.Warning, len(warnings)) for i := range warnings { rawData[i] = &warnings[i] } headers := []string{} for _, column := range columns { headers = append(headers, column.Name) } return cli.RenderTable(os.Stdout, c.flagFormat, headers, data, rawData) } func (c *cmdWarningList) countColumnData(warning api.Warning) string { return fmt.Sprintf("%d", warning.Count) } func (c *cmdWarningList) firstSeenColumnData(warning api.Warning) string { return warning.FirstSeenAt.Local().Format(dateLayout) } func (c *cmdWarningList) lastSeenColumnData(warning api.Warning) string { return warning.LastSeenAt.Local().Format(dateLayout) } func (c *cmdWarningList) locationColumnData(warning api.Warning) string { return warning.Location } func (c *cmdWarningList) projectColumnData(warning api.Warning) string { return warning.Project } func (c *cmdWarningList) severityColumnData(warning api.Warning) string { return strings.ToUpper(warning.Severity) } func (c *cmdWarningList) stateColumnData(warning api.Warning) string { return strings.ToUpper(warning.Status) } func (c *cmdWarningList) typeColumnData(warning api.Warning) string { return warning.Type } func (c *cmdWarningList) uuidColumnData(warning api.Warning) string { return warning.UUID } func (c *cmdWarningList) parseColumns(clustered bool) ([]warningColumn, error) { columnsShorthandMap := map[rune]warningColumn{ 'c': {i18n.G("COUNT"), c.countColumnData}, 'f': {i18n.G("FIRST SEEN"), c.firstSeenColumnData}, 'l': {i18n.G("LAST SEEN"), c.lastSeenColumnData}, 'p': {i18n.G("PROJECT"), c.projectColumnData}, 's': {i18n.G("SEVERITY"), c.severityColumnData}, 'S': {i18n.G("STATE"), c.stateColumnData}, 't': {i18n.G("TYPE"), c.typeColumnData}, 'u': {i18n.G("UUID"), c.uuidColumnData}, } if clustered { columnsShorthandMap['L'] = warningColumn{i18n.G("LOCATION"), c.locationColumnData} } else { if c.flagColumns != defaultWarningColumns { if strings.ContainsAny(c.flagColumns, "L") { return nil, errors.New(i18n.G("Can't specify column L when not clustered")) } } c.flagColumns = strings.ReplaceAll(c.flagColumns, "L", "") } columnList := strings.Split(c.flagColumns, ",") columns := []warningColumn{} for _, columnEntry := range columnList { if columnEntry == "" { return nil, fmt.Errorf(i18n.G("Empty column entry (redundant, leading or trailing command) in '%s'"), c.flagColumns) } for _, columnRune := range columnEntry { column, ok := columnsShorthandMap[columnRune] if !ok { return nil, fmt.Errorf(i18n.G("Unknown column shorthand char '%c' in '%s'"), columnRune, columnEntry) } columns = append(columns, column) } } return columns, nil } // Acknowledge. type cmdWarningAcknowledge struct { global *cmdGlobal warning *cmdWarning } var cmdWarningAcknowledgeUsage = u.Usage{u.WarningUUID.Remote()} func (c *cmdWarningAcknowledge) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("acknowledge", cmdWarningAcknowledgeUsage...) cmd.Aliases = []string{"ack"} cmd.Short = i18n.G("Acknowledge warning") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Acknowledge warning`)) cmd.RunE = c.run return cmd } func (c *cmdWarningAcknowledge) run(cmd *cobra.Command, args []string) error { parsed, err := cmdWarningAcknowledgeUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer warningUUID := parsed[0].RemoteObject.String warning := api.WarningPut{Status: "acknowledged"} return d.UpdateWarning(warningUUID, warning, "") } // Show. type cmdWarningShow struct { global *cmdGlobal warning *cmdWarning } var cmdWarningShowUsage = u.Usage{u.WarningUUID.Remote()} func (c *cmdWarningShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("show", cmdWarningShowUsage...) cmd.Short = i18n.G("Show warning") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Show warning`)) cmd.RunE = c.run return cmd } func (c *cmdWarningShow) run(cmd *cobra.Command, args []string) error { parsed, err := cmdWarningShowUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer warningUUID := parsed[0].RemoteObject.String warning, _, err := d.GetWarning(warningUUID) if err != nil { return err } data, err := yaml.Dump(&warning, yaml.V2) if err != nil { return err } fmt.Printf("%s", data) return nil } // Delete. type cmdWarningDelete struct { global *cmdGlobal warning *cmdWarning flagAll bool } var cmdWarningDeleteUsage = u.Usage{u.Either(u.WarningUUID.Remote().List(1), u.Sequence(u.Flag("all"), u.RemoteColonOpt))} func (c *cmdWarningDelete) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("delete", cmdWarningDeleteUsage...) cmd.Aliases = []string{"rm", "remove"} cmd.Short = i18n.G("Delete warnings") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Delete warnings`)) cli.AddBoolFlag(cmd.Flags(), &c.flagAll, "all|a", i18n.G("Delete all warnings")) cmd.RunE = c.run return cmd } func (c *cmdWarningDelete) run(cmd *cobra.Command, args []string) error { parsed, err := cmdWarningDeleteUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } var errs []error if c.flagAll { if parsed[0].BranchID != 1 { return errors.New(i18n.G("--all cannot be used together with other arguments")) } d := parsed[0].List[1].RemoteServer allWarnings, err := d.GetWarnings() if err != nil { return err } for _, warning := range allWarnings { err = d.DeleteWarning(warning.UUID) if err != nil { errs = append(errs, err) } } } else { for _, p := range parsed[0].List { d := p.RemoteServer warningUUID := p.RemoteObject.String // Delete warnings err = d.DeleteWarning(warningUUID) if err != nil { errs = append(errs, err) } } } if len(errs) > 0 { return errors.Join(errs...) } return nil } incus-7.0.0/cmd/incus/webui.go000066400000000000000000000011271517523235500161760ustar00rootroot00000000000000package main import ( "github.com/spf13/cobra" "github.com/lxc/incus/v7/cmd/incus/color" u "github.com/lxc/incus/v7/cmd/incus/usage" "github.com/lxc/incus/v7/internal/i18n" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdWebui struct { global *cmdGlobal } var cmdWebuiUsage = u.Usage{u.RemoteColonOpt} func (c *cmdWebui) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = cli.U("webui", cmdWebuiUsage...) cmd.Short = i18n.G("Open the web interface") cmd.Long = cli.FormatSection(color.DescriptionPrefix, i18n.G(`Open the web interface`)) cmd.RunE = c.run return cmd } incus-7.0.0/cmd/incus/webui_unix.go000066400000000000000000000043221517523235500172410ustar00rootroot00000000000000//go:build !windows package main import ( "errors" "fmt" "net" "net/http" "net/url" "sync" "github.com/google/uuid" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/util" ) func (c *cmdWebui) run(cmd *cobra.Command, args []string) error { parsed, err := cmdWebuiUsage.Parse(c.global.conf, cmd, args) if err != nil { return err } d := parsed[0].RemoteServer // Create localhost socket. server, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { return fmt.Errorf("Unable to setup TCP socket: %w", err) } // Get the connection info. info, err := d.GetConnectionInfo() if err != nil { return err } uri, err := url.Parse(info.URL) if err != nil { return err } // Check that the target supports the UI. req, err := http.NewRequest("GET", fmt.Sprintf("%s/ui/", info.URL), nil) if err != nil { return err } resp, err := d.DoHTTP(req) if err != nil { return err } if resp.StatusCode == http.StatusNotFound { return errors.New(i18n.G("The server doesn't have a web UI installed")) } // Enable keep-alive for proxied connections. httpClient, err := d.GetHTTPClient() if err != nil { return err } httpTransport, ok := httpClient.Transport.(*http.Transport) if ok { httpTransport.DisableKeepAlives = false } // Get server info. api10, api10Etag, err := d.GetServer() if err != nil { return err } // Generate credentials. token := uuid.New().String() // Handle inbound connections. transport := remoteProxyTransport{ s: d, baseURL: uri, } connections := uint64(0) transactions := uint64(0) handler := remoteProxyHandler{ s: d, transport: transport, api10: api10, api10Etag: api10Etag, mu: &sync.RWMutex{}, connections: &connections, transactions: &transactions, token: token, url: "http://" + server.Addr().String(), } // Print address. uiURL := fmt.Sprintf("http://%s/ui?auth_token=%s", server.Addr().String(), token) fmt.Printf(i18n.G("Web server running at: %s")+"\n", uiURL) // Attempt to automatically open the web browser. _ = util.OpenBrowser(uiURL) // Start the server. err = http.Serve(server, handler) if err != nil { return err } return nil } incus-7.0.0/cmd/incus/webui_windows.go000066400000000000000000000004071517523235500177500ustar00rootroot00000000000000//go:build windows package main import ( "errors" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/i18n" ) func (c *cmdWebui) run(cmd *cobra.Command, args []string) error { return errors.New(i18n.G("This command isn't supported on Windows")) } incus-7.0.0/cmd/incusd/000077500000000000000000000000001517523235500146775ustar00rootroot00000000000000incus-7.0.0/cmd/incusd/.dir-locals.el000066400000000000000000000004001517523235500173220ustar00rootroot00000000000000;;; Directory Local Variables ;;; For more information see (info "(emacs) Directory Variables") ((go-mode . ((go-test-args . "-tags libsqlite3 -timeout 120s") (eval . (set (make-local-variable 'flycheck-go-build-tags) '("libsqlite3")))))) incus-7.0.0/cmd/incusd/api.go000066400000000000000000000341241517523235500160030ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "io/fs" "net/http" "net/url" "os" "strings" "time" "github.com/gorilla/mux" clusterConfig "github.com/lxc/incus/v7/internal/server/cluster/config" clusterRequest "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" storagePools "github.com/lxc/incus/v7/internal/server/storage" "github.com/lxc/incus/v7/internal/server/storage/s3" "github.com/lxc/incus/v7/internal/server/storage/s3/local" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) // swagger:operation GET / server api_get // // Get the supported API endpoints // // Returns a list of supported API versions (URLs). // // Internal API endpoints are not reported as those aren't versioned // and should only be used by the daemon itself. // // --- // produces: // - application/json // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: ["/1.0"] func restServer(d *Daemon) *http.Server { /* Setup the web server */ router := mux.NewRouter() router.StrictSlash(false) // Don't redirect to URL with trailing slash. router.SkipClean(true) router.UseEncodedPath() // Allow encoded values in path segments. // Serving the UI. uiPath := os.Getenv("INCUS_UI") uiEnabled := uiPath != "" && util.PathExists(fmt.Sprintf("%s/index.html", uiPath)) if uiEnabled { uiHttpDir := uiHttpDir{http.Dir(uiPath)} router.PathPrefix("/ui/").Handler(http.StripPrefix("/ui/", http.FileServer(uiHttpDir))) router.HandleFunc("/ui", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/ui/", http.StatusMovedPermanently) }) } // Serving the documentation. documentationPath := os.Getenv("INCUS_DOCUMENTATION") docEnabled := documentationPath != "" && util.PathExists(documentationPath) if docEnabled { documentationHttpDir := documentationHttpDir{http.Dir(documentationPath)} router.PathPrefix("/documentation/").Handler(http.StripPrefix("/documentation/", http.FileServer(documentationHttpDir))) router.HandleFunc("/documentation", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/documentation/", http.StatusMovedPermanently) }) } // Serving the OS API. d.createCmd(router, "os", apiOS) router.HandleFunc("/os", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/os/", http.StatusMovedPermanently) }) // OIDC browser login (code flow). router.HandleFunc("/oidc/login", func(w http.ResponseWriter, r *http.Request) { if d.oidcVerifier == nil { w.WriteHeader(http.StatusNotFound) return } d.oidcVerifier.Login(w, r) }) router.HandleFunc("/oidc/callback", func(w http.ResponseWriter, r *http.Request) { if d.oidcVerifier == nil { w.WriteHeader(http.StatusNotFound) return } d.oidcVerifier.Callback(w, r) }) router.HandleFunc("/oidc/logout", func(w http.ResponseWriter, r *http.Request) { if d.oidcVerifier == nil { w.WriteHeader(http.StatusNotFound) return } d.oidcVerifier.Logout(w, r) }) router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") ua := r.Header.Get("User-Agent") if uiEnabled && strings.Contains(ua, "Gecko") { // Web browser handling. http.Redirect(w, r, "/ui/", http.StatusMovedPermanently) } else { // Normal client handling. _ = response.SyncResponse(true, []string{"/1.0"}).Render(w) } }) for endpoint, f := range d.gateway.HandlerFuncs(d.heartbeatHandler, d.getTrustedCertificates) { router.HandleFunc(endpoint, f) } for _, c := range api10 { d.createCmd(router, "1.0", c) // Create any alias endpoints using the same handlers as the parent endpoint but // with a different path and name (so the handler can differentiate being called via // a different endpoint) if it wants to. for _, alias := range c.Aliases { ac := c ac.Name = alias.Name ac.Path = alias.Path d.createCmd(router, "1.0", ac) } } for _, c := range apiInternal { d.createCmd(router, "internal", c) } for _, c := range apiACME { d.createCmd(router, "", c) } router.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { logger.Info("Sending top level 404", logger.Ctx{"url": r.URL, "method": r.Method, "remote": r.RemoteAddr}) w.Header().Set("Content-Type", "application/json") _ = response.NotFound(nil).Render(w) }) return &http.Server{ Handler: &httpServer{r: router, d: d}, ConnContext: request.SaveConnectionInContext, IdleTimeout: 30 * time.Second, ReadHeaderTimeout: 3 * time.Second, } } func hoistReqVM(f func(*Daemon, instance.Instance, http.ResponseWriter, *http.Request) response.Response, d *Daemon) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { trusted, inst, err := authenticateAgentCert(d.State(), r) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } if !trusted { http.Error(w, "", http.StatusUnauthorized) return } resp := f(d, inst, w, r) _ = resp.Render(w) } } func vSockServer(d *Daemon) *http.Server { return &http.Server{ Handler: devIncusAPI(d, hoistReqVM), IdleTimeout: 30 * time.Second, ReadHeaderTimeout: 3 * time.Second, ReadTimeout: 3 * time.Second, } } func metricsServer(d *Daemon) *http.Server { /* Setup the web server */ router := mux.NewRouter() router.StrictSlash(false) router.SkipClean(true) router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _ = response.SyncResponse(true, []string{"/1.0"}).Render(w) }) for endpoint, f := range d.gateway.HandlerFuncs(d.heartbeatHandler, d.getTrustedCertificates) { router.HandleFunc(endpoint, f) } d.createCmd(router, "1.0", api10Cmd) d.createCmd(router, "1.0", metricsCmd) router.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { logger.Info("Sending top level 404", logger.Ctx{"url": r.URL, "method": r.Method, "remote": r.RemoteAddr}) w.Header().Set("Content-Type", "application/json") _ = response.NotFound(nil).Render(w) }) return &http.Server{ Handler: &httpServer{r: router, d: d}, IdleTimeout: 30 * time.Second, ReadHeaderTimeout: 3 * time.Second, ReadTimeout: 3 * time.Second, WriteTimeout: 30 * time.Second, } } func storageBucketsServer(d *Daemon) *http.Server { /* Setup the web server */ router := mux.NewRouter() router.StrictSlash(false) router.SkipClean(true) router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { // Wait until daemon is fully started. <-d.waitReady.Done() s := d.State() // Check if request contains an access key, and if so try and route it to the associated bucket. accessKey := s3.AuthorizationHeaderAccessKey(r.Header.Get("Authorization")) if accessKey != "" { // Lookup access key to ascertain if it maps to a bucket. var err error var bucket *db.StorageBucket err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { bucket, err = tx.GetStoragePoolLocalBucketByAccessKey(ctx, accessKey) return err }) if err != nil { if api.StatusErrorCheck(err, http.StatusNotFound) { errResult := s3.Error{Code: s3.ErrorCodeInvalidAccessKeyID} errResult.Response(w) return } errResult := s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()} errResult.Response(w) return } serveLocalBucket(d, w, r, bucket) return } // Otherwise treat request as anonymous. listResult := s3.ListAllMyBucketsResult{Owner: s3.Owner{ID: "anonymous"}} listResult.Response(w) }) // We use the NotFoundHandler to dispatch requests to local buckets by path. router.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Wait until daemon is fully started. <-d.waitReady.Done() s := d.State() reqURL, err := url.Parse(r.RequestURI) if err != nil { errResult := s3.Error{Code: s3.ErrorInvalidRequest, Message: err.Error()} errResult.Response(w) return } pathParts := strings.Split(reqURL.Path, "/") if len(pathParts) < 2 { errResult := s3.Error{Code: s3.ErrorInvalidRequest, Message: "Bucket name not specified"} errResult.Response(w) return } bucketName, err := url.PathUnescape(pathParts[1]) if err != nil { errResult := s3.Error{Code: s3.ErrorCodeNoSuchBucket, BucketName: pathParts[1]} errResult.Response(w) return } // Lookup bucket. var bucket *db.StorageBucket err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { bucket, err = tx.GetStoragePoolLocalBucket(ctx, bucketName) return err }) if err != nil { if api.StatusErrorCheck(err, http.StatusNotFound) { errResult := s3.Error{Code: s3.ErrorCodeNoSuchBucket, BucketName: bucketName} errResult.Response(w) return } errResult := s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error(), BucketName: bucketName} errResult.Response(w) return } serveLocalBucket(d, w, r, bucket) }) return &http.Server{ Handler: &httpServer{r: router, d: d}, IdleTimeout: 30 * time.Second, ReadHeaderTimeout: 3 * time.Second, } } // serveLocalBucket mounts the bucket volume, loads its keys, and dispatches the // request to the in-process S3 handler. Always called for buckets backed by // local storage drivers (dir, btrfs, zfs). func serveLocalBucket(d *Daemon, w http.ResponseWriter, r *http.Request, bucket *db.StorageBucket) { s := d.State() pool, err := storagePools.LoadByName(s, bucket.PoolName) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } // Load credentials for the bucket. var keys []*db.StorageBucketKey err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error keys, err = tx.GetStoragePoolBucketKeys(ctx, bucket.ID) return err }) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } creds := make([]local.Credential, 0, len(keys)) for _, k := range keys { creds = append(creds, local.Credential{ AccessKey: k.AccessKey, SecretKey: k.SecretKey, Role: local.Role(k.Role), }) } bucketDir, unmount, err := pool.MountLocalBucket(bucket.Project, bucket.Name, nil) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } defer func() { err := unmount() if err != nil { logger.Errorf("Failed unmounting bucket %q after S3 request: %v", bucket.Name, err) } }() srv := local.NewServer(bucketDir, creds) // Migrate any data left over from the legacy minio layout, but only // once the request has cleared authentication. This is a no-op once // the bucket has been migrated. srv.OnAuthenticated = func() error { return local.MigrateMinioBucket(bucketDir, bucket.Name) } srv.ServeHTTP(w, r) } type httpServer struct { r *mux.Router d *Daemon } func (s *httpServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if !strings.HasPrefix(req.URL.Path, "/internal") { // Wait for startup if not cluster internal queries. if !isClusterNotification(req) { <-s.d.setupChan } // Set CORS headers, unless this is an internal request. setCORSHeaders(rw, req, s.d.State().GlobalConfig) } // OPTIONS request don't need any further processing if req.Method == "OPTIONS" { return } // Call the original server s.r.ServeHTTP(rw, req) } func setCORSHeaders(rw http.ResponseWriter, req *http.Request, config *clusterConfig.Config) { // Check if we have a working config. if config == nil { return } allowedOrigin := config.HTTPSAllowedOrigin() origin := req.Header.Get("Origin") if allowedOrigin != "" && origin != "" { rw.Header().Set("Access-Control-Allow-Origin", allowedOrigin) } allowedMethods := config.HTTPSAllowedMethods() if allowedMethods != "" && origin != "" { rw.Header().Set("Access-Control-Allow-Methods", allowedMethods) } allowedHeaders := config.HTTPSAllowedHeaders() if allowedHeaders != "" && origin != "" { rw.Header().Set("Access-Control-Allow-Headers", allowedHeaders) } allowedCredentials := config.HTTPSAllowedCredentials() if allowedCredentials { rw.Header().Set("Access-Control-Allow-Credentials", "true") } } // Return true if this an API request coming from a cluster node that is // notifying us of some user-initiated API request that needs some action to be // taken on this node as well. func isClusterNotification(r *http.Request) bool { return r.Header.Get("User-Agent") == clusterRequest.UserAgentNotifier } func isClusterInternal(r *http.Request) bool { return r.Header.Get("User-Agent") == clusterRequest.UserAgentClient } type uiHttpDir struct { http.FileSystem } // Open is part of the http.FileSystem interface. func (httpFS uiHttpDir) Open(name string) (http.File, error) { fsFile, err := httpFS.FileSystem.Open(name) if err != nil && errors.Is(err, fs.ErrNotExist) { return httpFS.FileSystem.Open("index.html") } return fsFile, err } type documentationHttpDir struct { http.FileSystem } // Open is part of the http.FileSystem interface. func (httpFS documentationHttpDir) Open(name string) (http.File, error) { fsFile, err := httpFS.FileSystem.Open(name) if err != nil && errors.Is(err, fs.ErrNotExist) { return httpFS.FileSystem.Open("index.html") } return fsFile, err } incus-7.0.0/cmd/incusd/api_1.0.go000066400000000000000000000664331517523235500163710ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "net" "net/http" "os" "strings" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/auth/oidc" "github.com/lxc/incus/v7/internal/server/cluster" clusterConfig "github.com/lxc/incus/v7/internal/server/cluster/config" "github.com/lxc/incus/v7/internal/server/config" "github.com/lxc/incus/v7/internal/server/db" instanceDrivers "github.com/lxc/incus/v7/internal/server/instance/drivers" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" scriptletLoad "github.com/lxc/incus/v7/internal/server/scriptlet/load" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/revert" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) var api10Cmd = APIEndpoint{ Get: APIEndpointAction{Handler: api10Get, AllowUntrusted: true}, Patch: APIEndpointAction{Handler: api10Patch, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Put: APIEndpointAction{Handler: api10Put, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var api10 = []APIEndpoint{ api10Cmd, api10ResourcesCmd, certificateCmd, certificatesCmd, clusterCmd, clusterGroupCmd, clusterGroupsCmd, clusterNodeCmd, clusterNodeStateCmd, clusterNodesCmd, clusterCertificateCmd, instanceBackupCmd, instanceBackupExportCmd, instanceBackupsCmd, instanceBitmapsCmd, instanceCmd, instanceConsoleCmd, instanceExecCmd, instanceFileCmd, instanceExecOutputCmd, instanceExecOutputsCmd, instanceLogCmd, instanceLogsCmd, instanceMetadataCmd, instanceMetadataTemplatesCmd, instancesCmd, instanceRebuildCmd, instanceSFTPCmd, instanceSnapshotCmd, instanceSnapshotsCmd, instanceStateCmd, instanceAccessCmd, instanceDebugMemoryCmd, instanceDebugRepairCmd, eventsCmd, imageAliasCmd, imageAliasesCmd, imageCmd, imageExportCmd, imageRefreshCmd, imagesCmd, imageSecretCmd, metadataConfigurationCmd, networkCmd, networkLeasesCmd, networksCmd, networkStateCmd, networkACLCmd, networkACLsCmd, networkACLLogCmd, networkAddressSetCmd, networkAddressSetsCmd, networkAllocationsCmd, networkForwardCmd, networkForwardsCmd, networkIntegrationCmd, networkIntegrationsCmd, networkLoadBalancerCmd, networkLoadBalancerStateCmd, networkLoadBalancersCmd, networkPeerCmd, networkPeersCmd, networkZoneCmd, networkZonesCmd, networkZoneRecordCmd, networkZoneRecordsCmd, operationCmd, operationsCmd, operationWait, operationWebsocket, profileCmd, profilesCmd, projectCmd, projectsCmd, projectStateCmd, projectAccessCmd, storagePoolCmd, storagePoolResourcesCmd, storagePoolsCmd, storagePoolBucketsCmd, storagePoolBucketCmd, storagePoolBucketKeysCmd, storagePoolBucketKeyCmd, storagePoolBucketBackupsCmd, storagePoolBucketBackupCmd, storagePoolBucketBackupsExportCmd, storagePoolVolumesCmd, storagePoolVolumeSnapshotsTypeCmd, storagePoolVolumeSnapshotTypeCmd, storagePoolVolumesTypeCmd, storagePoolVolumeTypeCmd, storagePoolVolumeTypeBitmapCmd, storagePoolVolumeTypeBitmapsCmd, storagePoolVolumeTypeNBDCmd, storagePoolVolumeTypeSFTPCmd, storagePoolVolumeTypeFileCmd, storagePoolVolumeTypeCustomBackupsCmd, storagePoolVolumeTypeCustomBackupCmd, storagePoolVolumeTypeCustomBackupExportCmd, storagePoolVolumeTypeStateCmd, warningsCmd, warningCmd, metricsCmd, } // swagger:operation GET /1.0?public server server_get_untrusted // // Get the server environment // // Shows a small subset of the server environment and configuration // which is required by untrusted clients to reach a server. // // The `?public` part of the URL isn't required, it's simply used to // separate the two behaviors of this endpoint. // // --- // produces: // - application/json // responses: // "200": // description: Server environment and configuration // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/ServerUntrusted" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0 server server_get // // Get the server environment and configuration // // Shows the full server environment and configuration. // // --- // produces: // - application/json // parameters: // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Server environment and configuration // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Server" // "500": // $ref: "#/responses/InternalServerError" func api10Get(d *Daemon, r *http.Request) response.Response { s := d.State() // Pull the full server config. fullSrvConfig, err := daemonConfigRender(s) if err != nil { return response.InternalError(err) } // Get the authentication methods. authMethods := []string{api.AuthenticationMethodTLS} oidcIssuer, oidcClientID, _, _, _ := s.GlobalConfig.OIDCServer() if oidcIssuer != "" && oidcClientID != "" { authMethods = append(authMethods, api.AuthenticationMethodOIDC) } srv := api.ServerUntrusted{ APIStatus: "stable", APIVersion: version.APIVersion, Public: false, Auth: "untrusted", AuthMethods: authMethods, } // Populate the untrusted config (user.ui.XYZ). srv.Config = map[string]string{} for k, v := range fullSrvConfig { if strings.HasPrefix(k, "user.ui.") { srv.Config[k] = v } } // If untrusted, return now if d.checkTrustedClient(r) != nil { return response.SyncResponseETag(true, srv, nil) } // If not authorized, return now. err = s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectServer(), auth.EntitlementCanView) if err != nil { return response.SmartError(err) } // Add the API extensions to authenticated requests. srv.APIExtensions = version.APIExtensions[:d.apiExtensions] // If a target was specified, forward the request to the relevant node. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } srv.Auth = "trusted" localHTTPSAddress := s.LocalConfig.HTTPSAddress() addresses, err := localUtil.ListenAddresses(localHTTPSAddress) if err != nil { return response.InternalError(err) } // When clustered, use the node name, otherwise use the hostname. var serverName string if s.ServerClustered { serverName = s.ServerName } else { hostname, err := os.Hostname() if err != nil { return response.SmartError(err) } serverName = hostname } certificate := string(s.Endpoints.NetworkPublicKey()) var certificateFingerprint string if certificate != "" { certificateFingerprint, err = localtls.CertFingerprintStr(certificate) if err != nil { return response.InternalError(err) } } architectures := []string{} for _, architecture := range s.OS.Architectures { architectureName, err := osarch.ArchitectureName(architecture) if err != nil { return response.InternalError(err) } architectures = append(architectures, architectureName) } projectName := r.FormValue("project") if projectName == "" { projectName = api.ProjectDefaultName } env := api.ServerEnvironment{ Addresses: addresses, Architectures: architectures, Certificate: certificate, CertificateFingerprint: certificateFingerprint, Kernel: s.OS.Uname.Sysname, KernelArchitecture: s.OS.Uname.Machine, KernelVersion: s.OS.Uname.Release, OSName: s.OS.ReleaseInfo["NAME"], OSVersion: s.OS.ReleaseInfo["VERSION_ID"], Project: projectName, Server: "incus", ServerPid: os.Getpid(), ServerVersion: version.Version, ServerClustered: s.ServerClustered, ServerEventMode: string(cluster.ServerEventMode()), ServerName: serverName, Firewall: s.Firewall.String(), } env.KernelFeatures = map[string]string{} drivers := instanceDrivers.DriverStatuses() for _, driver := range drivers { // Only report the supported drivers. if !driver.Supported { continue } if env.Driver != "" { env.Driver = env.Driver + " | " + driver.Info.Name } else { env.Driver = driver.Info.Name } // Get the version of the instance drivers in use. if env.DriverVersion != "" { env.DriverVersion = env.DriverVersion + " | " + driver.Info.Version } else { env.DriverVersion = driver.Info.Version } } if s.OS.LXCFeatures != nil { env.LXCFeatures = map[string]string{} for k, v := range s.OS.LXCFeatures { env.LXCFeatures[k] = fmt.Sprintf("%v", v) } } supportedStorageDrivers, usedStorageDrivers := readStoragePoolDriversCache() for driver, version := range usedStorageDrivers { if env.Storage != "" { env.Storage = env.Storage + " | " + driver } else { env.Storage = driver } // Get the version of the storage drivers in use. if env.StorageVersion != "" { env.StorageVersion = env.StorageVersion + " | " + version } else { env.StorageVersion = version } } env.StorageSupportedDrivers = supportedStorageDrivers fullSrv := api.Server{ServerUntrusted: srv} fullSrv.Environment = env requestor := request.CreateRequestor(r) fullSrv.AuthUserName = requestor.Username fullSrv.AuthUserMethod = requestor.Protocol err = s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectServer(), auth.EntitlementCanViewSensitive) if err == nil { fullSrv.Config = fullSrvConfig } else if !api.StatusErrorCheck(err, http.StatusForbidden) { return response.SmartError(err) } return response.SyncResponseETag(true, fullSrv, fullSrv.Config) } // swagger:operation PUT /1.0 server server_put // // Update the server configuration // // Updates the entire server configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: server // description: Server configuration // required: true // schema: // $ref: "#/definitions/ServerPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func api10Put(d *Daemon, r *http.Request) response.Response { s := d.State() // If a target was specified, forward the request to the relevant node. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } // Don't apply changes to settings until daemon is fully started. <-d.waitReady.Done() req := api.ServerPut{} err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // If this is a notification from a cluster node, just run the triggers // for reacting to the values that changed. if isClusterNotification(r) { logger.Debug("Handling config changed notification") changed := util.CloneMap(req.Config) // Get the current (updated) config. var config *clusterConfig.Config err := s.DB.Cluster.Transaction(context.Background(), func(ctx context.Context, tx *db.ClusterTx) error { var err error config, err = clusterConfig.Load(ctx, tx) return err }) if err != nil { return response.SmartError(err) } // Update the daemon config. d.globalConfigMu.Lock() d.globalConfig = config d.globalConfigMu.Unlock() // Run any update triggers. err = doApi10UpdateTriggers(d, nil, changed, s.LocalConfig, config) if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } render, err := daemonConfigRender(s) if err != nil { return response.SmartError(err) } err = localUtil.EtagCheck(r, render) if err != nil { return response.PreconditionFailed(err) } return doApi10Update(d, r, req, false) } // swagger:operation PATCH /1.0 server server_patch // // Partially update the server configuration // // Updates a subset of the server configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: server // description: Server configuration // required: true // schema: // $ref: "#/definitions/ServerPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func api10Patch(d *Daemon, r *http.Request) response.Response { s := d.State() // If a target was specified, forward the request to the relevant node. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } // Don't apply changes to settings until daemon is fully started. <-d.waitReady.Done() render, err := daemonConfigRender(s) if err != nil { return response.InternalError(err) } err = localUtil.EtagCheck(r, render) if err != nil { return response.PreconditionFailed(err) } req := api.ServerPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } if req.Config == nil { return response.EmptySyncResponse } return doApi10Update(d, r, req, true) } func doApi10Update(d *Daemon, r *http.Request, req api.ServerPut, patch bool) response.Response { s := d.State() // First deal with config specific to the local daemon nodeValues := map[string]string{} for key := range node.ConfigSchema { value, ok := req.Config[key] if ok { nodeValues[key] = value delete(req.Config, key) } } nodeChanged := map[string]string{} var newNodeConfig *node.Config var oldNodeConfig map[string]string err := s.DB.Node.Transaction(r.Context(), func(ctx context.Context, tx *db.NodeTx) error { var err error newNodeConfig, err = node.ConfigLoad(ctx, tx) if err != nil { return fmt.Errorf("Failed to load node config: %w", err) } // Keep old config around in case something goes wrong. In that case the config will be reverted. oldNodeConfig = util.CloneMap(newNodeConfig.Dump()) // We currently don't allow changing the cluster.https_address once it's set. if s.ServerClustered { curConfig, err := tx.Config(ctx) if err != nil { return fmt.Errorf("Cannot fetch node config from database: %w", err) } newClusterHTTPSAddress, found := nodeValues["cluster.https_address"] if !found && patch { newClusterHTTPSAddress = curConfig["cluster.https_address"] } else if !found { newClusterHTTPSAddress = "" } if curConfig["cluster.https_address"] != newClusterHTTPSAddress { return errors.New("Changing cluster.https_address is currently not supported") } } // Validate the storage volumes if nodeValues["storage.backups_volume"] != "" && nodeValues["storage.backups_volume"] != newNodeConfig.StorageBackupsVolume() { err := daemonStorageValidate(s, "backups", nodeValues["storage.backups_volume"]) if err != nil { return fmt.Errorf("Failed validation of %q: %w", "storage.backups_volume", err) } } if nodeValues["storage.images_volume"] != "" && nodeValues["storage.images_volume"] != newNodeConfig.StorageImagesVolume() { err := daemonStorageValidate(s, "images", nodeValues["storage.images_volume"]) if err != nil { return fmt.Errorf("Failed validation of %q: %w", "storage.images_volume", err) } } if nodeValues["storage.logs_volume"] != "" && nodeValues["storage.logs_volume"] != newNodeConfig.StorageLogsVolume() { err := daemonStorageValidate(s, "logs", nodeValues["storage.logs_volume"]) if err != nil { return fmt.Errorf("Failed validation of %q: %w", "storage.logs_volume", err) } } if patch { nodeChanged, err = newNodeConfig.Patch(nodeValues) } else { nodeChanged, err = newNodeConfig.Replace(nodeValues) } return err }) if err != nil { var errorList *config.ErrorList switch { case errors.As(err, &errorList): return response.BadRequest(err) default: return response.SmartError(err) } } reverter := revert.New() defer reverter.Fail() reverter.Add(func() { for key := range nodeValues { val, ok := oldNodeConfig[key] if !ok { nodeValues[key] = "" } else { nodeValues[key] = val } } err = s.DB.Node.Transaction(r.Context(), func(ctx context.Context, tx *db.NodeTx) error { newNodeConfig, err := node.ConfigLoad(ctx, tx) if err != nil { return fmt.Errorf("Failed to load node config: %w", err) } if patch { _, err = newNodeConfig.Patch(nodeValues) } else { _, err = newNodeConfig.Replace(nodeValues) } if err != nil { return fmt.Errorf("Failed updating node config: %w", err) } return nil }) if err != nil { logger.Warn("Failed reverting node config", logger.Ctx{"err": err}) } }) // Then deal with cluster wide configuration var clusterChanged map[string]string var newClusterConfig *clusterConfig.Config var oldClusterConfig map[string]string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error newClusterConfig, err = clusterConfig.Load(ctx, tx) if err != nil { return fmt.Errorf("Failed to load cluster config: %w", err) } // Keep old config around in case something goes wrong. In that case the config will be reverted. oldClusterConfig = util.CloneMap(newClusterConfig.Dump()) if patch { clusterChanged, err = newClusterConfig.Patch(req.Config) } else { clusterChanged, err = newClusterConfig.Replace(req.Config) } return err }) if err != nil { var errorList *config.ErrorList switch { case errors.As(err, &errorList): return response.BadRequest(err) default: return response.SmartError(err) } } reverter.Add(func() { for key := range req.Config { val, ok := oldClusterConfig[key] if !ok { req.Config[key] = "" } else { req.Config[key] = val } } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { newClusterConfig, err = clusterConfig.Load(ctx, tx) if err != nil { return fmt.Errorf("Failed to load cluster config: %w", err) } if patch { _, err = newClusterConfig.Patch(req.Config) } else { _, err = newClusterConfig.Replace(req.Config) } if err != nil { return fmt.Errorf("Failed updating cluster config: %w", err) } return nil }) if err != nil { logger.Warn("Failed reverting cluster config", logger.Ctx{"err": err}) } }) // Notify the other nodes about changes notifier, err := cluster.NewNotifier(s, s.Endpoints.NetworkCert(), s.ServerCert(), cluster.NotifyAlive) if err != nil { return response.SmartError(err) } err = notifier(func(client incus.InstanceServer) error { server, etag, err := client.GetServer() if err != nil { return err } serverPut := server.Writable() // Only propagated cluster-wide changes serverPut.Config = util.CloneMap(clusterChanged) return client.UpdateServer(serverPut, etag) }) if err != nil { logger.Error("Failed to notify other members about config change", logger.Ctx{"err": err}) return response.SmartError(err) } // Update the daemon config. d.globalConfigMu.Lock() d.globalConfig = newClusterConfig d.localConfig = newNodeConfig d.globalConfigMu.Unlock() // Run any update triggers. err = doApi10UpdateTriggers(d, nodeChanged, clusterChanged, newNodeConfig, newClusterConfig) if err != nil { return response.SmartError(err) } reverter.Success() s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.ConfigUpdated.Event(request.CreateRequestor(r), nil)) return response.EmptySyncResponse } func doApi10UpdateTriggers(d *Daemon, nodeChanged, clusterChanged map[string]string, nodeConfig *node.Config, clusterConfig *clusterConfig.Config) error { s := d.State() acmeChanged := false bgpChanged := false dnsChanged := false oidcChanged := false openFGAChanged := false ovnChanged := false linstorChanged := false ovsChanged := false syslogChanged := false loggingChanges := map[string]struct{}{} for key := range clusterChanged { switch key { case "acme.agree_tos", "acme.ca_url", "acme.challenge", "acme.domain", "acme.email", "acme.provider", "acme.provider.environment", "acme.provider.resolvers", "acme.http.port": acmeChanged = true case "cluster.images_minimal_replica": err := autoSyncImages(s.ShutdownCtx, s) if err != nil { logger.Warn("Could not auto-sync images", logger.Ctx{"err": err}) } case "cluster.offline_threshold": d.gateway.HeartbeatOfflineThreshold = clusterConfig.OfflineThreshold() d.taskClusterHeartbeat.Reset() case "core.bgp_asn": bgpChanged = true case "core.https_trusted_proxy": s.Endpoints.NetworkUpdateTrustedProxy(clusterChanged[key]) case "core.proxy_http", "core.proxy_https", "core.proxy_ignore_hosts": daemonConfigSetProxy(d, clusterConfig) case "images.auto_update_interval", "images.remote_cache_expiry": if !s.OS.MockMode { d.taskPruneImages.Reset() } case "loki.api.url", "loki.auth.username", "loki.auth.password", "loki.api.ca_cert", "loki.instance", "loki.labels", "loki.loglevel", "loki.types": // Notify the logging mechanism about changes to the deprecated keys for backward compatibility. loggingChanges["loki"] = struct{}{} case "network.ovn.northbound_connection", "network.ovn.ca_cert", "network.ovn.client_cert", "network.ovn.client_key": ovnChanged = true case "oidc.issuer", "oidc.client.id", "oidc.audience", "oidc.claim", "oidc.scopes": oidcChanged = true case "openfga.api.url", "openfga.api.token", "openfga.store.id": openFGAChanged = true case "storage.linstor.controller_connection", "storage.linstor.ca_cert", "storage.linstor.client_cert", "storage.linstor.client_key": linstorChanged = true default: if strings.HasPrefix(key, "logging.") { fields := strings.Split(key, ".") if len(fields) > 2 { loggingChanges[fields[1]] = struct{}{} } } } } for key := range nodeChanged { switch key { case "core.bgp_address", "core.bgp_routerid": bgpChanged = true case "core.dns_address": dnsChanged = true case "core.syslog_socket": syslogChanged = true case "network.ovs.connection": ovsChanged = true } } // Process some additional keys. We do it sequentially because some keys are // correlated with others, and need to be processed first (for example // core.https_address need to be processed before // cluster.https_address). value, ok := nodeChanged["core.https_address"] if ok { if value == "" && s.OS.IncusOS != nil { return errors.New("Cannot unset HTTPS address on an IncusOS system") } err := s.Endpoints.NetworkUpdateAddress(value) if err != nil { return err } s.Endpoints.NetworkUpdateTrustedProxy(clusterConfig.HTTPSTrustedProxy()) } value, ok = nodeChanged["cluster.https_address"] if ok { err := s.Endpoints.ClusterUpdateAddress(value) if err != nil { return err } s.Endpoints.NetworkUpdateTrustedProxy(clusterConfig.HTTPSTrustedProxy()) } value, ok = nodeChanged["core.debug_address"] if ok { err := s.Endpoints.PprofUpdateAddress(value) if err != nil { return err } } value, ok = nodeChanged["core.metrics_address"] if ok { err := s.Endpoints.MetricsUpdateAddress(value, s.Endpoints.NetworkCert()) if err != nil { return err } } value, ok = nodeChanged["core.storage_buckets_address"] if ok { err := s.Endpoints.StorageBucketsUpdateAddress(value, s.Endpoints.NetworkCert()) if err != nil { return err } } value, ok = nodeChanged["storage.backups_volume"] if ok { err := daemonStorageMove(s, "backups", value) if err != nil { return err } } value, ok = nodeChanged["storage.images_volume"] if ok { err := daemonStorageMove(s, "images", value) if err != nil { return err } } value, ok = nodeChanged["storage.logs_volume"] if ok { err := daemonStorageMove(s, "logs", value) if err != nil { return err } } // Apply larger changes. if acmeChanged { err := autoRenewCertificate(s.ShutdownCtx, d, true) if err != nil { return err } } if bgpChanged { address := nodeConfig.BGPAddress() asn := clusterConfig.BGPASN() routerid := nodeConfig.BGPRouterID() err := s.BGP.Configure(address, uint32(asn), net.ParseIP(routerid)) if err != nil { return fmt.Errorf("Failed reconfiguring BGP: %w", err) } } if dnsChanged { address := nodeConfig.DNSAddress() err := s.DNS.Reconfigure(address) if err != nil { return fmt.Errorf("Failed reconfiguring DNS: %w", err) } } if len(loggingChanges) > 0 { err := d.loggingController.Reconfigure(d.State(), loggingChanges) if err != nil { return err } } if oidcChanged { oidcIssuer, oidcClientID, oidcScope, oidcAudience, oidcClaim := clusterConfig.OIDCServer() if oidcIssuer == "" || oidcClientID == "" { d.oidcVerifier = nil } else { var err error d.oidcVerifier, err = oidc.NewVerifier(oidcIssuer, oidcClientID, oidcScope, oidcAudience, oidcClaim) if err != nil { return fmt.Errorf("Failed creating verifier: %w", err) } } } if openFGAChanged { openfgaAPIURL, openfgaAPIToken, openfgaStoreID := d.globalConfig.OpenFGA() err := d.setupOpenFGA(openfgaAPIURL, openfgaAPIToken, openfgaStoreID) if err != nil { return err } } if ovnChanged { err := d.setupOVN() if err != nil { return err } } if ovsChanged { err := d.setupOVS() if err != nil { return err } } if syslogChanged { err := d.setupSyslogSocket(nodeConfig.SyslogSocket()) if err != nil { return err } } if linstorChanged { err := d.setupLinstor() if err != nil { return err } } // Compile and load the instance placement scriptlet. value, ok = clusterChanged["instances.placement.scriptlet"] if ok { err := scriptletLoad.InstancePlacementSet(value) if err != nil { return fmt.Errorf("Failed saving instance placement scriptlet: %w", err) } } // Setup the authorization scriptlet. value, ok = clusterChanged["authorization.scriptlet"] if ok { err := d.setupAuthorizationScriptlet(value) if err != nil { return err } } return nil } incus-7.0.0/cmd/incusd/api_acme.go000066400000000000000000000114171517523235500167700ustar00rootroot00000000000000package main import ( "context" "io" "net" "net/http" "strings" "github.com/lxc/incus/v7/internal/server/acme" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/task" "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" localtls "github.com/lxc/incus/v7/shared/tls" ) var apiACME = []APIEndpoint{ acmeChallengeCmd, } var acmeChallengeCmd = APIEndpoint{ Path: ".well-known/acme-challenge/{token}", Get: APIEndpointAction{Handler: acmeProvideChallenge, AllowUntrusted: true}, } func acmeProvideChallenge(d *Daemon, r *http.Request) response.Response { s := d.State() // Redirect to the leader when clustered. if s.ServerClustered { leader, err := s.Cluster.LeaderAddress() if err != nil { return response.SmartError(err) } // This gives me the correct value clusterAddress := s.LocalConfig.ClusterAddress() if clusterAddress != "" && clusterAddress != leader { // Forward the request to the leader client, err := cluster.Connect(leader, s.Endpoints.NetworkCert(), s.ServerCert(), r, true) if err != nil { return response.SmartError(err) } return response.ForwardedResponse(client, r) } } // Forward to the lego listener. addr := s.GlobalConfig.ACMEHTTP() if strings.HasPrefix(addr, ":") { addr = "127.0.0.1" + addr } client := http.Client{} client.Transport = &http.Transport{ DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { return net.Dial("tcp", addr) }, } // Get the forwarded host with fallback to the one from the request. host := r.Header.Get("X-Incus-forwarded-host") if host == "" { host = r.Host } if host == "" { host = r.URL.Host } // Prepare the request to lego. req, err := http.NewRequest("GET", "http://"+host+r.URL.String(), nil) if err != nil { return response.InternalError(err) } resp, err := client.Do(req) if err != nil { return response.InternalError(err) } defer resp.Body.Close() challenge, err := io.ReadAll(resp.Body) if err != nil { return response.InternalError(err) } return response.ManualResponse(func(w http.ResponseWriter) error { w.Header().Set("Content-Type", "text/plain") _, err = w.Write(challenge) if err != nil { return err } return nil }) } func autoRenewCertificate(ctx context.Context, d *Daemon, force bool) error { s := d.State() domain, email, caURL, agreeToS, challengeType := s.GlobalConfig.ACME() if domain == "" || email == "" || !agreeToS || challengeType == "" { return nil } // If we are clustered, let the leader handle the certificate renewal. if s.ServerClustered { leader, err := s.Cluster.LeaderAddress() if err != nil { return err } // Figure out our own cluster address. clusterAddress := s.LocalConfig.ClusterAddress() if clusterAddress != leader { return nil } } opRun := func(op *operations.Operation) error { newCert, err := acme.UpdateCertificate(s, challengeType, s.ServerClustered, domain, email, caURL, force) if err != nil { return err } // If cert is nil, there's no need to update it as it's still valid. if newCert == nil { return nil } if s.ServerClustered { req := api.ClusterCertificatePut{ ClusterCertificate: string(newCert.Certificate), ClusterCertificateKey: string(newCert.PrivateKey), } err = updateClusterCertificate(s.ShutdownCtx, s, d.gateway, nil, req) if err != nil { return err } return nil } cert, err := localtls.KeyPairFromRaw(newCert.Certificate, newCert.PrivateKey) if err != nil { return err } s.Endpoints.NetworkUpdateCert(cert) err = util.WriteCert(s.OS.VarDir, "server", newCert.Certificate, newCert.PrivateKey, nil) if err != nil { return err } return nil } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.RenewServerCertificate, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating renew server certificate operation", logger.Ctx{"err": err}) return err } logger.Info("Starting automatic server certificate renewal check") err = op.Start() if err != nil { logger.Error("Failed starting renew server certificate operation", logger.Ctx{"err": err}) return err } err = op.Wait(ctx) if err != nil { logger.Error("Failed server certificate renewal", logger.Ctx{"err": err}) return err } logger.Info("Done automatic server certificate renewal check") return nil } func autoRenewCertificateTask(d *Daemon) (task.Func, task.Schedule) { f := func(ctx context.Context) { _ = autoRenewCertificate(ctx, d, false) } return f, task.Daily() } incus-7.0.0/cmd/incusd/api_cluster.go000066400000000000000000002545741517523235500175610ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/url" "os" "path/filepath" "slices" "sort" "strconv" "strings" "sync" "time" "github.com/gorilla/mux" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/filter" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/certificate" "github.com/lxc/incus/v7/internal/server/cluster" clusterConfig "github.com/lxc/incus/v7/internal/server/cluster/config" clusterRequest "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/revert" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var clusterCmd = APIEndpoint{ Path: "cluster", Get: APIEndpointAction{Handler: clusterGet, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanView)}, Put: APIEndpointAction{Handler: clusterPut, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var clusterNodesCmd = APIEndpoint{ Path: "cluster/members", Get: APIEndpointAction{Handler: clusterNodesGet, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanView)}, Post: APIEndpointAction{Handler: clusterNodesPost, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var clusterNodeCmd = APIEndpoint{ Path: "cluster/members/{name}", Delete: APIEndpointAction{Handler: clusterNodeDelete, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Get: APIEndpointAction{Handler: clusterNodeGet, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanView)}, Patch: APIEndpointAction{Handler: clusterNodePatch, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Put: APIEndpointAction{Handler: clusterNodePut, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Post: APIEndpointAction{Handler: clusterNodePost, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var clusterNodeStateCmd = APIEndpoint{ Path: "cluster/members/{name}/state", Get: APIEndpointAction{Handler: clusterNodeStateGet, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanView)}, Post: APIEndpointAction{Handler: clusterNodeStatePost, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } // swagger:operation GET /1.0/cluster cluster cluster_get // // Get the cluster configuration // // Gets the current cluster configuration. // // --- // produces: // - application/json // responses: // "200": // description: Cluster configuration // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Cluster" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func clusterGet(d *Daemon, r *http.Request) response.Response { s := d.State() serverName := s.ServerName // If the name is set to the hard-coded default node name, then // clustering is not enabled. if serverName == "none" { serverName = "" } memberConfig, err := clusterGetMemberConfig(r.Context(), s.DB.Cluster) if err != nil { return response.SmartError(err) } // Sort the member config. sort.Slice(memberConfig, func(i, j int) bool { left := memberConfig[i] right := memberConfig[j] if left.Entity != right.Entity { return left.Entity < right.Entity } if left.Name != right.Name { return left.Name < right.Name } if left.Key != right.Key { return left.Key < right.Key } return left.Description < right.Description }) cluster := api.Cluster{ ServerName: serverName, Enabled: serverName != "", MemberConfig: memberConfig, } return response.SyncResponseETag(true, cluster, cluster) } // Fetch information about all node-specific configuration keys set on the // storage pools and networks of this cluster. func clusterGetMemberConfig(ctx context.Context, clusterDB *db.Cluster) ([]api.ClusterMemberConfigKey, error) { var pools map[string]map[string]string var networks map[string]map[string]string keys := []api.ClusterMemberConfigKey{} err := clusterDB.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error pools, err = tx.GetStoragePoolsLocalConfig(ctx) if err != nil { return fmt.Errorf("Failed to fetch storage pools configuration: %w", err) } networks, err = tx.GetNetworksLocalConfig(ctx) if err != nil { return fmt.Errorf("Failed to fetch networks configuration: %w", err) } return nil }) if err != nil { return nil, err } for pool, config := range pools { for key := range config { if strings.HasPrefix(key, internalInstance.ConfigVolatilePrefix) { continue } key := api.ClusterMemberConfigKey{ Entity: "storage-pool", Name: pool, Key: key, Description: fmt.Sprintf("\"%s\" property for storage pool \"%s\"", key, pool), } keys = append(keys, key) } } for network, config := range networks { for key := range config { if strings.HasPrefix(key, internalInstance.ConfigVolatilePrefix) { continue } key := api.ClusterMemberConfigKey{ Entity: "network", Name: network, Key: key, Description: fmt.Sprintf("\"%s\" property for network \"%s\"", key, network), } keys = append(keys, key) } } return keys, nil } // Depending on the parameters passed and on local state this endpoint will // either: // // - bootstrap a new cluster (if this node is not clustered yet) // - request to join an existing cluster // - disable clustering on a node // // The client is required to be trusted. // swagger:operation PUT /1.0/cluster cluster cluster_put // // Update the cluster configuration // // Updates the entire cluster configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: body // name: cluster // description: Cluster configuration // required: true // schema: // $ref: "#/definitions/ClusterPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func clusterPut(d *Daemon, r *http.Request) response.Response { req := api.ClusterPut{} // Parse the request err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. if req.ServerName == "" && req.Enabled { return response.BadRequest(errors.New("ServerName is required when enabling clustering")) } if req.ServerName != "" && !req.Enabled { return response.BadRequest(errors.New("ServerName must be empty when disabling clustering")) } if req.ServerName != "" && strings.HasPrefix(req.ServerName, targetGroupPrefix) { return response.BadRequest(fmt.Errorf("ServerName may not start with %q", targetGroupPrefix)) } // Disable clustering. if !req.Enabled { return clusterPutDisable(d, r, req) } // Depending on the provided parameters we either bootstrap a brand new // cluster with this node as first node, or perform a request to join a // given cluster. if req.ClusterAddress == "" { return clusterPutBootstrap(d, r, req) } return clusterPutJoin(d, r, req) } func clusterPutBootstrap(d *Daemon, r *http.Request, req api.ClusterPut) response.Response { s := d.State() logger.Info("Bootstrapping cluster", logger.Ctx{"serverName": req.ServerName}) run := func(op *operations.Operation) error { // Update server name. d.globalConfigMu.Lock() d.serverName = req.ServerName d.serverClustered = true d.globalConfigMu.Unlock() d.events.SetLocalLocation(d.serverName) // Refresh the state. s = d.State() // Start clustering tasks d.startClusterTasks() err := cluster.Bootstrap(s, d.gateway, req.ServerName) if err != nil { d.stopClusterTasks() return err } // Restart the networks. err = networkStartup(s) if err != nil { return err } // Return the new server certificate to the client. clusterCertPath := internalUtil.VarPath("cluster.crt") if util.PathExists(clusterCertPath) { cert, err := os.ReadFile(clusterCertPath) if err != nil { return err } err = op.UpdateMetadata(map[string]any{"certificate": string(cert)}) if err != nil { return err } } s.Events.SendLifecycle(request.ProjectParam(r), lifecycle.ClusterEnabled.Event(req.ServerName, op.Requestor(), nil)) return nil } resources := map[string][]api.URL{} resources["cluster"] = []api.URL{} // If there's no cluster.https_address set, but core.https_address is, // let's default to it. var err error var config *node.Config err = s.DB.Node.Transaction(r.Context(), func(ctx context.Context, tx *db.NodeTx) error { config, err = node.ConfigLoad(ctx, tx) if err != nil { return fmt.Errorf("Failed to fetch member configuration: %w", err) } localClusterAddress := config.ClusterAddress() if localClusterAddress != "" { return nil } localHTTPSAddress := config.HTTPSAddress() if internalUtil.IsWildCardAddress(localHTTPSAddress) { return fmt.Errorf("Cannot use wildcard core.https_address %q for cluster.https_address. Please specify a new cluster.https_address or core.https_address", localClusterAddress) } _, err = config.Patch(map[string]string{ "cluster.https_address": localHTTPSAddress, }) if err != nil { return fmt.Errorf("Copy core.https_address to cluster.https_address: %w", err) } return nil }) if err != nil { return response.SmartError(err) } // Update local config cache. d.globalConfigMu.Lock() d.localConfig = config d.globalConfigMu.Unlock() op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.ClusterBootstrap, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } // Add the cluster flag from the agent version.UserAgentFeatures([]string{"cluster"}) return operations.OperationResponse(op) } func clusterPutJoin(d *Daemon, r *http.Request, req api.ClusterPut) response.Response { s := d.State() logger.Info("Joining cluster", logger.Ctx{"serverName": req.ServerName}) // Make sure basic pre-conditions are met. if len(req.ClusterCertificate) == 0 { return response.BadRequest(errors.New("No target cluster member certificate provided")) } if s.ServerClustered { return response.BadRequest(errors.New("This server is already clustered")) } // Validate server address. if req.ServerAddress == "" { return response.BadRequest(errors.New("No server address provided for this member")) } // Check that the provided address is an IP address or DNS, not wildcard and isn't required to specify a port. err := validate.IsListenAddress(true, false, false)(req.ServerAddress) if err != nil { return response.BadRequest(fmt.Errorf("Invalid server address %q: %w", req.ServerAddress, err)) } // Verify provided address against cluster.https_address if set. localHTTPSAddress := s.LocalConfig.ClusterAddress() if localHTTPSAddress != "" { if !internalUtil.IsAddressCovered(req.ServerAddress, localHTTPSAddress) { return response.BadRequest(fmt.Errorf(`Server address %q is not covered by %q from "cluster.https_address"`, req.ServerAddress, localHTTPSAddress)) } } else { // If cluster.https_address is not set, check against core.https_address localHTTPSAddress = s.LocalConfig.HTTPSAddress() var config *node.Config if localHTTPSAddress == "" { // As the user always provides a server address, but no networking // was setup on this node, let's do the job and open the // port. We'll use the same address both for the REST API and // for clustering. // First try to listen to the provided address. If we fail, we // won't actually update the database config. err := s.Endpoints.NetworkUpdateAddress(req.ServerAddress) if err != nil { return response.SmartError(err) } err = s.DB.Node.Transaction(r.Context(), func(ctx context.Context, tx *db.NodeTx) error { config, err = node.ConfigLoad(ctx, tx) if err != nil { return fmt.Errorf("Failed to load cluster config: %w", err) } _, err = config.Patch(map[string]string{ "core.https_address": req.ServerAddress, "cluster.https_address": req.ServerAddress, }) return err }) if err != nil { return response.SmartError(err) } } else { // The user has previously set core.https_address and // is now providing a cluster address as well. If they // differ we need to listen to it. if !internalUtil.IsAddressCovered(req.ServerAddress, localHTTPSAddress) { err := s.Endpoints.ClusterUpdateAddress(req.ServerAddress) if err != nil { return response.SmartError(err) } } // Update the cluster.https_address config key. err := s.DB.Node.Transaction(r.Context(), func(ctx context.Context, tx *db.NodeTx) error { var err error config, err = node.ConfigLoad(ctx, tx) if err != nil { return fmt.Errorf("Failed to load cluster config: %w", err) } _, err = config.Patch(map[string]string{ "cluster.https_address": req.ServerAddress, }) return err }) if err != nil { return response.SmartError(err) } } // Update local config cache. d.globalConfigMu.Lock() d.localConfig = config d.globalConfigMu.Unlock() } // Client parameters to connect to the target cluster node. serverCert := s.ServerCert() args := &incus.ConnectionArgs{ TLSClientCert: string(serverCert.PublicKey()), TLSClientKey: string(serverCert.PrivateKey()), TLSServerCert: string(req.ClusterCertificate), UserAgent: version.UserAgent, } // Always set a proxy function to have cluster traffic bypass any configured HTTP proxy. proxy := func(req *http.Request) (*url.URL, error) { return nil, nil } args.Proxy = proxy // Asynchronously join the cluster. run := func(op *operations.Operation) error { logger.Debug("Running cluster join operation") // If the user has provided a join token, setup the trust // relationship by adding our own certificate to the cluster. if req.ClusterToken != "" { err := cluster.SetupTrust(serverCert, req.ServerName, req.ClusterAddress, req.ClusterCertificate, req.ClusterToken) if err != nil { return fmt.Errorf("Failed to setup cluster trust: %w", err) } } // Now we are in the remote trust store, ensure our name and type are correct to allow the cluster // to associate our member name to the server certificate. err := cluster.UpdateTrust(serverCert, req.ServerName, req.ClusterAddress, req.ClusterCertificate) if err != nil { return fmt.Errorf("Failed to update cluster trust: %w", err) } // Connect to the target cluster node. client, err := incus.ConnectIncus(fmt.Sprintf("https://%s", req.ClusterAddress), args) if err != nil { return err } // Get the cluster members members, err := client.GetClusterMembers() if err != nil { return err } // Verify if a node with the same name already exists in the cluster. for _, member := range members { if member.ServerName == req.ServerName { return fmt.Errorf("The cluster already has a member with name: %s", req.ServerName) } } // As ServerAddress field is required to be set it means that we're using the new join API // introduced with the 'clustering_join' extension. // Connect to ourselves to initialize storage pools and networks using the API. localClient, err := incus.ConnectIncusUnix(d.os.GetUnixSocket(), &incus.ConnectionArgs{UserAgent: clusterRequest.UserAgentJoiner}) if err != nil { return fmt.Errorf("Failed to connect to local server: %w", err) } reverter := revert.New() defer reverter.Fail() // Update server name. oldServerName := d.serverName d.globalConfigMu.Lock() d.serverName = req.ServerName d.serverClustered = true d.globalConfigMu.Unlock() reverter.Add(func() { d.globalConfigMu.Lock() d.serverName = oldServerName d.serverClustered = false d.globalConfigMu.Unlock() d.events.SetLocalLocation(d.serverName) }) d.events.SetLocalLocation(d.serverName) // Create all storage pools and networks. err = clusterInitMember(localClient, client, req.MemberConfig) if err != nil { return fmt.Errorf("Failed to initialize member: %w", err) } // Get all defined storage pools and networks, so they can be compared to the ones in the cluster. pools := []api.StoragePool{} networks := []api.InitNetworksProjectPost{} err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { poolNames, err := tx.GetStoragePoolNames(ctx) if err != nil && !response.IsNotFoundError(err) { return err } for _, name := range poolNames { _, pool, _, err := tx.GetStoragePoolInAnyState(ctx, name) if err != nil { return err } pools = append(pools, *pool) } // Get a list of projects for networks. var projects []dbCluster.Project projects, err = dbCluster.GetProjects(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed to load projects for networks: %w", err) } for _, p := range projects { networkNames, err := tx.GetNetworks(ctx, p.Name) if err != nil && !response.IsNotFoundError(err) { return err } for _, name := range networkNames { _, network, _, err := tx.GetNetworkInAnyState(ctx, p.Name, name) if err != nil { return err } internalNetwork := api.InitNetworksProjectPost{ NetworksPost: api.NetworksPost{ NetworkPut: network.NetworkPut, Name: network.Name, Type: network.Type, }, Project: p.Name, } networks = append(networks, internalNetwork) } } return nil }) if err != nil { return err } reverter.Add(func() { err = client.DeletePendingClusterMember(req.ServerName, true) if err != nil { logger.Errorf("Failed request to delete cluster member: %v", err) } }) // Now request for this node to be added to the list of cluster nodes. info, err := clusterAcceptMember(client, req.ServerName, req.ServerAddress, cluster.SchemaVersion, version.APIExtensionsCount(), pools, networks) if err != nil { return fmt.Errorf("Failed request to add member: %w", err) } // Update our TLS configuration using the returned cluster certificate. err = internalUtil.WriteCert(s.OS.VarDir, "cluster", info.PublicKey, info.PrivateKey, nil) if err != nil { return fmt.Errorf("Failed to save cluster certificate: %w", err) } networkCert, err := internalUtil.LoadClusterCert(s.OS.VarDir) if err != nil { return fmt.Errorf("Failed to parse cluster certificate: %w", err) } s.Endpoints.NetworkUpdateCert(networkCert) // Add trusted certificates of other members to local trust store. trustedCerts, err := client.GetCertificates() if err != nil { return fmt.Errorf("Failed to get trusted certificates: %w", err) } for _, trustedCert := range trustedCerts { if trustedCert.Type == api.CertificateTypeServer { dbType, err := certificate.FromAPIType(trustedCert.Type) if err != nil { return err } // Store the certificate in the local database. dbCert := dbCluster.Certificate{ Fingerprint: trustedCert.Fingerprint, Type: dbType, Name: trustedCert.Name, Certificate: trustedCert.Certificate, Restricted: trustedCert.Restricted, } logger.Debugf("Adding certificate %q (%s) to local trust store", trustedCert.Name, trustedCert.Fingerprint) err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { id, err := dbCluster.CreateCertificate(ctx, tx.Tx(), dbCert) if err != nil { return err } err = dbCluster.UpdateCertificateProjects(ctx, tx.Tx(), int(id), trustedCert.Projects) if err != nil { return err } return nil }) if err != nil && !api.StatusErrorCheck(err, http.StatusConflict) { return fmt.Errorf("Failed adding local trusted certificate %q (%s): %w", trustedCert.Name, trustedCert.Fingerprint, err) } } } // Update cached trusted certificates (this adds the server certificates we collected above) so that we are able to join. // Client and metric type certificates from the cluster we are joining will not be added until later. s.UpdateCertificateCache() // Update local setup and possibly join the raft dqlite cluster. nodes := make([]db.RaftNode, len(info.RaftNodes)) for i, node := range info.RaftNodes { nodes[i].ID = node.ID nodes[i].Address = node.Address nodes[i].Role = db.RaftRole(node.Role) } err = cluster.Join(s, d.gateway, networkCert, serverCert, req.ServerName, nodes) if err != nil { return err } // Add the new node to the default cluster group. err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { err := tx.AddNodeToClusterGroup(ctx, "default", req.ServerName) if err != nil { return fmt.Errorf("Failed to add new member to the default cluster group: %w", err) } return nil }) if err != nil { return err } // Start clustering tasks. d.startClusterTasks() reverter.Add(func() { d.stopClusterTasks() }) // Load the configuration. var nodeConfig *node.Config err = s.DB.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { var err error nodeConfig, err = node.ConfigLoad(ctx, tx) return err }) if err != nil { return err } // Get the current (updated) config. var currentClusterConfig *clusterConfig.Config err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { currentClusterConfig, err = clusterConfig.Load(ctx, tx) if err != nil { return err } return nil }) if err != nil { return err } d.globalConfigMu.Lock() d.localConfig = nodeConfig d.globalConfig = currentClusterConfig d.globalConfigMu.Unlock() changes := util.CloneMap(currentClusterConfig.Dump()) err = doApi10UpdateTriggers(d, nil, changes, nodeConfig, currentClusterConfig) if err != nil { return err } // Refresh the state. s = d.State() // Re-connect OVS if needed. _ = d.setupOVS() // Re-connect OVN if needed. _ = d.setupOVN() // Start up networks so any post-join changes can be applied now that we have a Node ID. logger.Debug("Starting networks after cluster join") err = networkStartup(s) if err != nil { logger.Errorf("Failed starting networks: %v", err) } client, err = cluster.Connect(req.ClusterAddress, s.Endpoints.NetworkCert(), serverCert, r, true) if err != nil { return err } // Add the cluster flag from the agent version.UserAgentFeatures([]string{"cluster"}) // Notify the leader of successful join, possibly triggering // role changes. _, _, err = client.RawQuery("POST", "/internal/cluster/rebalance", nil, "") if err != nil { logger.Warnf("Failed to trigger cluster rebalance: %v", err) } // Ensure all images are available after this node has joined. err = autoSyncImages(s.ShutdownCtx, s) if err != nil { logger.Warn("Failed to sync images") } // Update the cert cache again to add client and metric certs to the cache. s.UpdateCertificateCache() s.Events.SendLifecycle(request.ProjectParam(r), lifecycle.ClusterMemberAdded.Event(req.ServerName, op.Requestor(), nil)) reverter.Success() return nil } resources := map[string][]api.URL{} resources["cluster"] = []api.URL{} op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.ClusterJoin, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // clusterPutDisableMu is used to prevent the daemon from being replaced/stopped during removal from the // cluster until such time as the request that initiated the removal has finished. This allows for self removal // from the cluster when not the leader. var clusterPutDisableMu sync.Mutex // Disable clustering on a node. func clusterPutDisable(d *Daemon, r *http.Request, req api.ClusterPut) response.Response { s := d.State() logger.Info("Disabling clustering", logger.Ctx{"serverName": req.ServerName}) // Close the cluster database err := s.DB.Cluster.Close() if err != nil { return response.SmartError(err) } // Update our TLS configuration using our original certificate. for _, suffix := range []string{"crt", "key", "ca"} { path := filepath.Join(s.OS.VarDir, "cluster."+suffix) if !util.PathExists(path) { continue } err := os.Remove(path) if err != nil { return response.InternalError(err) } } networkCert, err := internalUtil.LoadCert(s.OS.VarDir) if err != nil { return response.InternalError(fmt.Errorf("Failed to parse member certificate: %w", err)) } // Reset the cluster database and make it local to this node. err = d.gateway.Reset(networkCert) if err != nil { return response.SmartError(err) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(request.ProjectParam(r), lifecycle.ClusterDisabled.Event(req.ServerName, requestor, nil)) // Stop database cluster connection. d.gateway.Kill() go func() { <-r.Context().Done() // Wait until request has finished. // Wait until we can acquire the lock. This way if another request is holding the lock we won't // replace/stop the daemon until that request has finished. clusterPutDisableMu.Lock() defer clusterPutDisableMu.Unlock() if d.systemdSocketActivated { logger.Info("Exiting daemon following removal from cluster") os.Exit(0) } else { logger.Info("Restarting daemon following removal from cluster") err = localUtil.ReplaceDaemon() if err != nil { logger.Error("Failed restarting daemon", logger.Ctx{"err": err}) } } }() return response.ManualResponse(func(w http.ResponseWriter) error { err := response.EmptySyncResponse.Render(w) if err != nil { return err } // Send the response before replacing the daemon process. f, ok := w.(http.Flusher) if ok { f.Flush() } else { return errors.New("http.ResponseWriter is not type http.Flusher") } return nil }) } // clusterInitMember initializes storage pools and networks on this member. We pass two client instances, one // connected to ourselves (the joining member) and one connected to the target cluster member to join. func clusterInitMember(d incus.InstanceServer, client incus.InstanceServer, memberConfig []api.ClusterMemberConfigKey) error { data := api.InitLocalPreseed{} // Fetch all pools currently defined in the cluster. pools, err := client.GetStoragePools() if err != nil { return fmt.Errorf("Failed to fetch information about cluster storage pools: %w", err) } // Merge the returned storage pools configs with the node-specific // configs provided by the user. for _, pool := range pools { // Skip pending pools. if pool.Status == "Pending" { continue } logger.Debugf("Populating init data for storage pool %q", pool.Name) post := api.StoragePoolsPost{ StoragePoolPut: pool.StoragePoolPut, Driver: pool.Driver, Name: pool.Name, } // Delete config keys that are automatically populated by the daemon. delete(post.Config, "volatile.initial_source") delete(post.Config, "zfs.pool_name") // Apply the node-specific config supplied by the user. nodeSpecificConfig := db.NodeSpecificStorageConfig(pool.Driver) for _, config := range memberConfig { if config.Entity != "storage-pool" { continue } if config.Name != pool.Name { continue } if !slices.Contains(nodeSpecificConfig, config.Key) { logger.Warnf("Ignoring config key %q for storage pool %q", config.Key, config.Name) continue } post.Config[config.Key] = config.Value } data.StoragePools = append(data.StoragePools, post) } projects, err := client.GetProjects() if err != nil { return fmt.Errorf("Failed to fetch project information about cluster networks: %w", err) } for _, p := range projects { if util.IsFalseOrEmpty(p.Config["features.networks"]) && p.Name != api.ProjectDefaultName { // Skip non-default projects that can't have their own networks so we don't try // and add the same default project networks twice. continue } // We only care about project features at this stage, leave the restrictions and limits for later. features := map[string]string{} for k, v := range p.Config { if strings.HasPrefix(k, "features.") { features[k] = v } } // Request that the project be created first before the project specific networks. data.Projects = append(data.Projects, api.ProjectsPost{ Name: p.Name, ProjectPut: api.ProjectPut{ Description: p.Description, Config: features, }, }) // Fetch all project specific networks currently defined in the cluster for the project. networks, err := client.UseProject(p.Name).GetNetworks() if err != nil { return fmt.Errorf("Failed to fetch network information about cluster networks in project %q: %w", p.Name, err) } // Merge the returned networks configs with the node-specific configs provided by the user. for _, network := range networks { // Skip unmanaged or pending networks. if !network.Managed || network.Status != api.NetworkStatusCreated { continue } // OVN networks don't need local creation. if network.Type == "ovn" { continue } post := api.InitNetworksProjectPost{ NetworksPost: api.NetworksPost{ NetworkPut: network.NetworkPut, Name: network.Name, Type: network.Type, }, Project: p.Name, } // Apply the node-specific config supplied by the user for networks in the default project. // At this time project specific networks don't have node specific config options. if p.Name == api.ProjectDefaultName { for _, config := range memberConfig { if config.Entity != "network" { continue } if config.Name != network.Name { continue } if !db.IsNodeSpecificNetworkConfig(config.Key) { logger.Warnf("Ignoring config key %q for network %q in project %q", config.Key, config.Name, p.Name) continue } post.Config[config.Key] = config.Value } } data.Networks = append(data.Networks, post) } } err = d.ApplyServerPreseed(api.InitPreseed{InitLocalPreseed: data}) if err != nil { return fmt.Errorf("Failed to initialize storage pools and networks: %w", err) } return nil } // Perform a request to the /internal/cluster/accept endpoint to check if a new // node can be accepted into the cluster and obtain joining information such as // the cluster private certificate. func clusterAcceptMember(client incus.InstanceServer, name string, address string, schema int, apiExt int, pools []api.StoragePool, networks []api.InitNetworksProjectPost) (*internalClusterPostAcceptResponse, error) { architecture, err := osarch.ArchitectureGetLocalID() if err != nil { return nil, err } req := internalClusterPostAcceptRequest{ Name: name, Address: address, Schema: schema, API: apiExt, StoragePools: pools, Networks: networks, Architecture: architecture, } info := &internalClusterPostAcceptResponse{} resp, _, err := client.RawQuery("POST", "/internal/cluster/accept", req, "") if err != nil { return nil, err } err = resp.MetadataAsStruct(&info) if err != nil { return nil, err } return info, nil } // swagger:operation GET /1.0/cluster/members cluster cluster_members_get // // Get the cluster members // // Returns a list of cluster members (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/cluster/members/server01", // "/1.0/cluster/members/server02" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/cluster/members?recursion=1 cluster cluster_members_get_recursion1 // // Get the cluster members // // Returns a list of cluster members (structs). // // --- // produces: // - application/json // parameters: // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of cluster members // items: // $ref: "#/definitions/ClusterMember" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func clusterNodesGet(d *Daemon, r *http.Request) response.Response { recursion := localUtil.IsRecursionRequest(r) s := d.State() // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } leaderAddress, err := s.Cluster.LeaderAddress() if err != nil { return response.InternalError(err) } var raftNodes []db.RaftNode err = s.DB.Node.Transaction(r.Context(), func(ctx context.Context, tx *db.NodeTx) error { raftNodes, err = tx.GetRaftNodes(ctx) if err != nil { return fmt.Errorf("Failed loading RAFT nodes: %w", err) } return nil }) if err != nil { return response.SmartError(err) } var members []api.ClusterMember err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { failureDomains, err := tx.GetFailureDomainsNames(ctx) if err != nil { return fmt.Errorf("Failed loading failure domains names: %w", err) } memberFailureDomains, err := tx.GetNodesFailureDomains(ctx) if err != nil { return fmt.Errorf("Failed loading member failure domains: %w", err) } nodes, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } maxVersion, err := tx.GetNodeMaxVersion(ctx) if err != nil { return fmt.Errorf("Failed getting max member version: %w", err) } args := db.NodeInfoArgs{ LeaderAddress: leaderAddress, FailureDomains: failureDomains, MemberFailureDomains: memberFailureDomains, OfflineThreshold: s.GlobalConfig.OfflineThreshold(), MaxMemberVersion: maxVersion, RaftNodes: raftNodes, } members = make([]api.ClusterMember, 0, len(nodes)) for i := range nodes { member, err := nodes[i].ToAPI(ctx, tx, args) if err != nil { return err } members = append(members, *member) } return nil }) if err != nil { return response.SmartError(err) } // Apply filters. filtered := make([]api.ClusterMember, 0) for _, member := range members { if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(member, *clauses) if err != nil { return response.SmartError(err) } if !match { continue } } filtered = append(filtered, member) } // Return full responses. if recursion { return response.SyncResponse(true, filtered) } // Return URLs only. urls := make([]string, 0, len(members)) for _, member := range members { u := api.NewURL().Path(version.APIVersion, "cluster", "members", member.ServerName) urls = append(urls, u.String()) } return response.SyncResponse(true, urls) } var clusterNodesPostMu sync.Mutex // Used to prevent races when creating cluster join tokens. // swagger:operation POST /1.0/cluster/members cluster cluster_members_post // // Request a join token // // Requests a join token to add a cluster member. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: body // name: cluster // description: Cluster member add request // required: true // schema: // $ref: "#/definitions/ClusterMembersPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func clusterNodesPost(d *Daemon, r *http.Request) response.Response { s := d.State() req := api.ClusterMembersPost{} // Parse the request. err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. err = validate.IsAPIName(req.ServerName, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid cluster member name: %w", err)) } if !s.ServerClustered { return response.BadRequest(errors.New("This server is not clustered")) } expiry, err := internalInstance.GetExpiry(time.Now(), s.GlobalConfig.ClusterJoinTokenExpiry()) if err != nil { return response.BadRequest(err) } // Get target addresses for existing online members, so that it can be encoded into the join token so that // the joining member will not have to specify a joining address during the join process. // Use anonymous interface type to align with how the API response will be returned for consistency when // retrieving remote operations. onlineNodeAddresses := make([]any, 0) // Get cluster database state. leaderAddress, err := s.Cluster.LeaderAddress() if err != nil { return response.InternalError(err) } var raftNodes []db.RaftNode err = s.DB.Node.Transaction(r.Context(), func(ctx context.Context, tx *db.NodeTx) error { raftNodes, err = tx.GetRaftNodes(ctx) if err != nil { return fmt.Errorf("Failed loading RAFT nodes: %w", err) } return nil }) if err != nil { return response.SmartError(err) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Get global cluster state. failureDomains, err := tx.GetFailureDomainsNames(ctx) if err != nil { return fmt.Errorf("Failed loading failure domains names: %w", err) } memberFailureDomains, err := tx.GetNodesFailureDomains(ctx) if err != nil { return fmt.Errorf("Failed loading member failure domains: %w", err) } maxVersion, err := tx.GetNodeMaxVersion(ctx) if err != nil { return fmt.Errorf("Failed getting max member version: %w", err) } // Get the nodes. members, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } args := db.NodeInfoArgs{ LeaderAddress: leaderAddress, FailureDomains: failureDomains, MemberFailureDomains: memberFailureDomains, OfflineThreshold: s.GlobalConfig.OfflineThreshold(), MaxMemberVersion: maxVersion, RaftNodes: raftNodes, } // Filter to online members. for _, member := range members { memberInfo, err := member.ToAPI(ctx, tx, args) if err != nil { return err } // Verify if a node with the same name already exists in the cluster. if member.Name == req.ServerName { return fmt.Errorf("The cluster already has a member with name: %s", req.ServerName) } // Skip servers that are offline. if slices.Contains([]int{db.ClusterMemberStateEvacuated, db.ClusterMemberStateEvacuating, db.ClusterMemberStateRestoring}, member.State) || member.IsOffline(s.GlobalConfig.OfflineThreshold()) { continue } // Only include servers that have a one of the database roles. if !slices.Contains(memberInfo.Roles, "database") && !slices.Contains(memberInfo.Roles, "database-standby") { continue } onlineNodeAddresses = append(onlineNodeAddresses, member.Address) } return nil }) if err != nil { return response.SmartError(err) } if len(onlineNodeAddresses) < 1 { return response.InternalError(errors.New("There are no online cluster members")) } // Lock to prevent concurrent requests racing the operationsGetByType function and creating duplicates. // We have to do this because collecting all of the operations from existing cluster members can take time. clusterNodesPostMu.Lock() defer clusterNodesPostMu.Unlock() // Remove any existing join tokens for the requested cluster member, this way we only ever have one active // join token for each potential new member, and it has the most recent active members list for joining. // This also ensures any historically unused (but potentially published) join tokens are removed. ops, err := operationsGetByType(s, r, api.ProjectDefaultName, operationtype.ClusterJoinToken) if err != nil { return response.InternalError(fmt.Errorf("Failed getting cluster join token operations: %w", err)) } for _, op := range ops { if op.StatusCode != api.Running { continue // Tokens are single use, so if cancelled but not deleted yet its not available. } opServerName, ok := op.Metadata["serverName"] if !ok { continue } if opServerName == req.ServerName { // Join token operation matches requested server name, so lets cancel it. logger.Warn("Cancelling duplicate join token operation", logger.Ctx{"operation": op.ID, "serverName": opServerName}) err = operationCancel(s, r, api.ProjectDefaultName, op) if err != nil { return response.InternalError(fmt.Errorf("Failed to cancel operation %q: %w", op.ID, err)) } } } // Generate join secret for new member. This will be stored inside the join token operation and will be // supplied by the joining member (encoded inside the join token) which will allow us to lookup the correct // operation in order to validate the requested joining server name is correct and authorised. joinSecret, err := internalUtil.RandomHexString(32) if err != nil { return response.InternalError(err) } // Generate fingerprint of network certificate so joining member can automatically trust the correct // certificate when it is presented during the join process. fingerprint, err := localtls.CertFingerprintStr(string(s.Endpoints.NetworkPublicKey())) if err != nil { return response.InternalError(err) } meta := map[string]any{ "serverName": req.ServerName, // Add server name to allow validation of name during join process. "secret": joinSecret, "fingerprint": fingerprint, "addresses": onlineNodeAddresses, "expiresAt": expiry, } resources := map[string][]api.URL{} resources["cluster"] = []api.URL{} op, err := operations.OperationCreate(s, api.ProjectDefaultName, operations.OperationClassToken, operationtype.ClusterJoinToken, resources, meta, nil, nil, nil, r) if err != nil { return response.InternalError(err) } s.Events.SendLifecycle(request.ProjectParam(r), lifecycle.ClusterTokenCreated.Event("members", op.Requestor(), nil)) return operations.OperationResponse(op) } // swagger:operation GET /1.0/cluster/members/{name} cluster cluster_member_get // // Get the cluster member // // Gets a specific cluster member. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Cluster member name // type: string // required: true // responses: // "200": // description: Cluster member // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/ClusterMember" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func clusterNodeGet(d *Daemon, r *http.Request) response.Response { s := d.State() name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } leaderAddress, err := s.Cluster.LeaderAddress() if err != nil { return response.InternalError(err) } var raftNodes []db.RaftNode err = s.DB.Node.Transaction(r.Context(), func(ctx context.Context, tx *db.NodeTx) error { raftNodes, err = tx.GetRaftNodes(ctx) if err != nil { return fmt.Errorf("Failed loading RAFT nodes: %w", err) } return nil }) if err != nil { return response.SmartError(err) } var memberInfo *api.ClusterMember err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { failureDomains, err := tx.GetFailureDomainsNames(ctx) if err != nil { return fmt.Errorf("Failed loading failure domains names: %w", err) } memberFailureDomains, err := tx.GetNodesFailureDomains(ctx) if err != nil { return fmt.Errorf("Failed loading member failure domains: %w", err) } member, err := tx.GetNodeByName(ctx, name) if err != nil { return err } maxVersion, err := tx.GetNodeMaxVersion(ctx) if err != nil { return fmt.Errorf("Failed getting max member version: %w", err) } args := db.NodeInfoArgs{ LeaderAddress: leaderAddress, FailureDomains: failureDomains, MemberFailureDomains: memberFailureDomains, OfflineThreshold: s.GlobalConfig.OfflineThreshold(), MaxMemberVersion: maxVersion, RaftNodes: raftNodes, } memberInfo, err = member.ToAPI(ctx, tx, args) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } return response.SyncResponseETag(true, memberInfo, memberInfo.ClusterMemberPut) } // swagger:operation PATCH /1.0/cluster/members/{name} cluster cluster_member_patch // // Partially update the cluster member // // Updates a subset of the cluster member configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Cluster member name // type: string // required: true // - in: body // name: cluster // description: Cluster member configuration // required: true // schema: // $ref: "#/definitions/ClusterMemberPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func clusterNodePatch(d *Daemon, r *http.Request) response.Response { return updateClusterNode(d.State(), d.gateway, r, true) } // swagger:operation PUT /1.0/cluster/members/{name} cluster cluster_member_put // // Update the cluster member // // Updates the entire cluster member configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Cluster member name // type: string // required: true // - in: body // name: cluster // description: Cluster member configuration // required: true // schema: // $ref: "#/definitions/ClusterMemberPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func clusterNodePut(d *Daemon, r *http.Request) response.Response { return updateClusterNode(d.State(), d.gateway, r, false) } // updateClusterNode is shared between clusterNodePut and clusterNodePatch. func updateClusterNode(s *state.State, gateway *cluster.Gateway, r *http.Request, isPatch bool) response.Response { name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } leaderAddress, err := s.Cluster.LeaderAddress() if err != nil { return response.InternalError(err) } var raftNodes []db.RaftNode err = s.DB.Node.Transaction(r.Context(), func(ctx context.Context, tx *db.NodeTx) error { raftNodes, err = tx.GetRaftNodes(ctx) if err != nil { return fmt.Errorf("Failed loading RAFT nodes: %w", err) } return nil }) if err != nil { return response.SmartError(err) } var member db.NodeInfo var memberInfo *api.ClusterMember err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { failureDomains, err := tx.GetFailureDomainsNames(ctx) if err != nil { return fmt.Errorf("Failed loading failure domains names: %w", err) } memberFailureDomains, err := tx.GetNodesFailureDomains(ctx) if err != nil { return fmt.Errorf("Failed loading member failure domains: %w", err) } member, err = tx.GetNodeByName(ctx, name) if err != nil { return err } maxVersion, err := tx.GetNodeMaxVersion(ctx) if err != nil { return fmt.Errorf("Failed getting max member version: %w", err) } args := db.NodeInfoArgs{ LeaderAddress: leaderAddress, FailureDomains: failureDomains, MemberFailureDomains: memberFailureDomains, OfflineThreshold: s.GlobalConfig.OfflineThreshold(), MaxMemberVersion: maxVersion, RaftNodes: raftNodes, } memberInfo, err = member.ToAPI(ctx, tx, args) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // Validate the request is fine err = localUtil.EtagCheck(r, memberInfo.ClusterMemberPut) if err != nil { return response.PreconditionFailed(err) } // Parse the request req := api.ClusterMemberPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Validate the request if slices.Contains(memberInfo.Roles, string(db.ClusterRoleDatabase)) && !slices.Contains(req.Roles, string(db.ClusterRoleDatabase)) { return response.BadRequest(fmt.Errorf("The %q role cannot be dropped at this time", db.ClusterRoleDatabase)) } if !slices.Contains(memberInfo.Roles, string(db.ClusterRoleDatabase)) && slices.Contains(req.Roles, string(db.ClusterRoleDatabase)) { return response.BadRequest(fmt.Errorf("The %q role cannot be added at this time", db.ClusterRoleDatabase)) } // Nodes must belong to at least one group. if len(req.Groups) == 0 { return response.BadRequest(errors.New("Cluster members need to belong to at least one group")) } // Prevent assigning all nodes the 'database-client' role. if slices.Contains(req.Roles, string(db.ClusterRoleDatabaseClient)) { clientNodes := 1 nodesCount := 0 err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { nodesCount, err = tx.GetNodesCount(ctx) if err != nil { return fmt.Errorf("Failed loading nodes count: %w", err) } nodes, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed loading nodes: %w", err) } for _, n := range nodes { // Ignore the node currently being updated. if n.Name == member.Name { continue } if slices.Contains(n.Roles, db.ClusterRoleDatabaseClient) { clientNodes++ } } return nil }) if err != nil { return response.SmartError(err) } if clientNodes >= nodesCount { return response.BadRequest(errors.New("Assigning the 'database-client' role to all nodes is not allowed")) } } // Convert the roles. newRoles := make([]db.ClusterRole, 0, len(req.Roles)) for _, role := range req.Roles { newRoles = append(newRoles, db.ClusterRole(role)) } // Update the database err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { nodeInfo, err := tx.GetNodeByName(ctx, name) if err != nil { return fmt.Errorf("Loading node information: %w", err) } err = clusterValidateConfig(req.Config) if err != nil { return err } if isPatch { // Populate request config with current values. if req.Config == nil { req.Config = nodeInfo.Config } else { for k, v := range nodeInfo.Config { _, ok := req.Config[k] if !ok { req.Config[k] = v } } } } // Update node config. err = tx.UpdateNodeConfig(ctx, nodeInfo.ID, req.Config) if err != nil { return fmt.Errorf("Failed to update cluster member config: %w", err) } // Update the description. if req.Description != memberInfo.Description { err = tx.SetDescription(nodeInfo.ID, req.Description) if err != nil { return fmt.Errorf("Update description: %w", err) } } // Update the roles. err = tx.UpdateNodeRoles(nodeInfo.ID, newRoles) if err != nil { return fmt.Errorf("Update roles: %w", err) } err = tx.UpdateNodeFailureDomain(ctx, nodeInfo.ID, req.FailureDomain) if err != nil { return fmt.Errorf("Update failure domain: %w", err) } // Update the cluster groups. err = tx.UpdateNodeClusterGroups(ctx, nodeInfo.ID, req.Groups) if err != nil { return fmt.Errorf("Update cluster groups: %w", err) } return nil }) if err != nil { return response.SmartError(err) } // If cluster roles changed, then distribute the info to all members. if s.Endpoints != nil && clusterRolesChanged(member.Roles, newRoles) { cluster.NotifyHeartbeat(s, gateway) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(request.ProjectParam(r), lifecycle.ClusterMemberUpdated.Event(name, requestor, nil)) return response.EmptySyncResponse } // clusterRolesChanged checks whether the non-internal roles have changed between oldRoles and newRoles. func clusterRolesChanged(oldRoles []db.ClusterRole, newRoles []db.ClusterRole) bool { // Build list of external-only roles from the newRoles list (excludes internal roles added by raft). newExternalRoles := make([]db.ClusterRole, 0, len(newRoles)) for _, r := range newRoles { // Check list of known external roles. for _, externalRole := range db.ClusterRoles { if r == externalRole { newExternalRoles = append(newExternalRoles, r) // Found external role. break } } } for _, r := range oldRoles { if !cluster.RoleInSlice(r, newExternalRoles) { return true } } for _, r := range newExternalRoles { if !cluster.RoleInSlice(r, oldRoles) { return true } } return false } // clusterValidateConfig validates the configuration keys/values for cluster members. func clusterValidateConfig(config map[string]string) error { clusterConfigKeys := map[string]func(value string) error{ // gendoc:generate(entity=cluster, group=cluster, key=scheduler.instance) // Possible values are `all`, `manual`, and `group`. See // {ref}`clustering-instance-placement` for more information. // --- // type: string // defaultdesc: `all` // shortdesc: Controls how instances are scheduled to run on this member "scheduler.instance": validate.Optional(validate.IsOneOf("all", "group", "manual")), } for k, v := range config { // User keys are free for all. // gendoc:generate(entity=cluster, group=cluster, key=user.*) // User keys can be used in search. // --- // type: string // shortdesc: Free form user key/value storage if strings.HasPrefix(k, "user.") { continue } validator, ok := clusterConfigKeys[k] if !ok { return fmt.Errorf("Invalid cluster configuration key %q", k) } err := validator(v) if err != nil { return fmt.Errorf("Invalid cluster configuration key %q value", k) } } return nil } // swagger:operation POST /1.0/cluster/members/{name} cluster cluster_member_post // // Rename the cluster member // // Renames an existing cluster member. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Cluster member name // type: string // required: true // - in: body // name: cluster // description: Cluster member rename request // required: true // schema: // $ref: "#/definitions/ClusterMemberPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func clusterNodePost(d *Daemon, r *http.Request) response.Response { s := d.State() memberName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Forward request. resp := forwardedResponseToNode(s, r, memberName) if resp != nil { return resp } req := api.ClusterMemberPost{} // Parse the request err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. err = validate.IsAPIName(req.ServerName, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid cluster member name: %w", err)) } // Perform the rename. err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RenameNode(ctx, memberName, req.ServerName) }) if err != nil { return response.SmartError(err) } // Update local server name. d.globalConfigMu.Lock() d.serverName = req.ServerName d.globalConfigMu.Unlock() d.events.SetLocalLocation(d.serverName) requestor := request.CreateRequestor(r) s.Events.SendLifecycle(request.ProjectParam(r), lifecycle.ClusterMemberRenamed.Event(req.ServerName, requestor, logger.Ctx{"old_name": memberName})) return response.EmptySyncResponse } // swagger:operation DELETE /1.0/cluster/members/{name} cluster cluster_member_delete // // Delete the cluster member // // Removes the member from the cluster. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Cluster member name // type: string // required: true // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func clusterNodeDelete(d *Daemon, r *http.Request) response.Response { s := d.State() force, err := strconv.Atoi(r.FormValue("force")) if err != nil { force = 0 } pending, err := strconv.Atoi(r.FormValue("pending")) if err != nil { pending = 0 } name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Redirect all requests to the leader, which is the one with // knowing what nodes are part of the raft cluster. localClusterAddress := s.LocalConfig.ClusterAddress() leader, err := s.Cluster.LeaderAddress() if err != nil { return response.InternalError(err) } var localInfo, leaderInfo db.NodeInfo err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { localInfo, err = tx.GetNodeByAddress(ctx, localClusterAddress) if err != nil { return fmt.Errorf("Failed loading local member info %q: %w", localClusterAddress, err) } leaderInfo, err = tx.GetNodeByAddress(ctx, leader) if err != nil { return fmt.Errorf("Failed loading leader member info %q: %w", leader, err) } return nil }) if err != nil { return response.SmartError(err) } // Get information about the cluster. var nodes []db.RaftNode err = s.DB.Node.Transaction(r.Context(), func(ctx context.Context, tx *db.NodeTx) error { var err error nodes, err = tx.GetRaftNodes(ctx) return err }) if err != nil { return response.SmartError(fmt.Errorf("Unable to get raft nodes: %w", err)) } if localClusterAddress != leader { if localInfo.Name == name { // If the member being removed is ourselves and we are not the leader, then lock the // clusterPutDisableMu before we forward the request to the leader, so that when the leader // goes on to request clusterPutDisable back to ourselves it won't be actioned until we // have returned this request back to the original client. clusterPutDisableMu.Lock() logger.Info("Acquired cluster self removal lock", logger.Ctx{"member": localInfo.Name}) go func() { <-r.Context().Done() // Wait until request is finished. logger.Info("Releasing cluster self removal lock", logger.Ctx{"member": localInfo.Name}) clusterPutDisableMu.Unlock() }() } logger.Debugf("Redirect member delete request to %s", leader) client, err := cluster.Connect(leader, s.Endpoints.NetworkCert(), s.ServerCert(), r, false) if err != nil { return response.SmartError(err) } if pending == 0 { err = client.DeleteClusterMember(name, force == 1) if err != nil { return response.SmartError(err) } } else { err = client.DeletePendingClusterMember(name, force == 1) if err != nil { return response.SmartError(err) } } // If we are the only remaining node, wait until promotion to leader, // then update cluster certs. if name == leaderInfo.Name && len(nodes) == 2 { err = d.gateway.WaitLeadership() if err != nil { return response.SmartError(err) } s.UpdateCertificateCache() } return response.ManualResponse(func(w http.ResponseWriter) error { err := response.EmptySyncResponse.Render(w) if err != nil { return err } // Send the response before replacing the daemon process. f, ok := w.(http.Flusher) if ok { f.Flush() } else { return errors.New("http.ResponseWriter is not type http.Flusher") } return nil }) } // Get lock now we are on leader. d.clusterMembershipMutex.Lock() defer d.clusterMembershipMutex.Unlock() // If we are removing the leader of a 2 node cluster, ensure the other node can be a leader. if name == leaderInfo.Name && len(nodes) == 2 { for i := range nodes { if nodes[i].Address != leader && nodes[i].Role != db.RaftVoter { // Promote the remaining node. nodes[i].Role = db.RaftVoter err := changeMemberRole(s, r, nodes[i].Address, nodes) if err != nil { return response.SmartError(fmt.Errorf("Unable to promote remaining cluster member to leader: %w", err)) } break } } } logger.Info("Deleting member from cluster", logger.Ctx{"name": name, "force": force}) err = autoSyncImages(s.ShutdownCtx, s) if err != nil { if force == 0 { return response.SmartError(fmt.Errorf("Failed to sync images: %w", err)) } // If force is set, only show a warning instead of returning an error. logger.Warn("Failed to sync images") } // First check that the node is clear from containers and images and // make it leave the database cluster, if it's part of it. address, err := cluster.Leave(s, d.gateway, name, force == 1, pending == 1) if err != nil { return response.SmartError(err) } if force != 1 { // Try to gracefully delete all networks and storage pools on it. // Delete all networks on this node client, err := cluster.Connect(address, s.Endpoints.NetworkCert(), s.ServerCert(), r, true) if err != nil { return response.SmartError(err) } // Get a list of projects for networks. var networkProjectNames []string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { networkProjectNames, err = dbCluster.GetProjectNames(ctx, tx.Tx()) return err }) if err != nil { return response.SmartError(fmt.Errorf("Failed to load projects for networks: %w", err)) } for _, networkProjectName := range networkProjectNames { var networks []string err := s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { networks, err = tx.GetNetworks(ctx, networkProjectName) return err }) if err != nil { return response.SmartError(err) } for _, name := range networks { err := client.UseProject(networkProjectName).DeleteNetwork(name) if err != nil { return response.SmartError(err) } } } var pools []string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Delete all the pools on this node pools, err = tx.GetStoragePoolNames(ctx) return err }) if err != nil && !response.IsNotFoundError(err) { return response.SmartError(err) } for _, name := range pools { err := client.DeleteStoragePool(name) if err != nil { return response.SmartError(err) } } } // Remove node from the database err = cluster.Purge(s.DB.Cluster, name, pending == 1) if err != nil { return response.SmartError(fmt.Errorf("Failed to remove member from database: %w", err)) } err = rebalanceMemberRoles(s, d.gateway, r, nil) if err != nil { logger.Warnf("Failed to rebalance dqlite nodes: %v", err) } // If this leader node removed itself, just disable clustering. if address == localClusterAddress { return clusterPutDisable(d, r, api.ClusterPut{}) } else if force != 1 { // Try to gracefully reset the database on the node. client, err := cluster.Connect(address, s.Endpoints.NetworkCert(), s.ServerCert(), r, true) if err != nil { return response.SmartError(err) } put := api.ClusterPut{} put.Enabled = false _, err = client.UpdateCluster(put, "") if err != nil { return response.SmartError(fmt.Errorf("Failed to cleanup the member: %w", err)) } } // Refresh the trusted certificate cache now that the member certificate has been removed. // We do not need to notify the other members here because the next heartbeat will trigger member change // detection and updateCertificateCache is called as part of that. s.UpdateCertificateCache() // Ensure all images are available after this node has been deleted. err = autoSyncImages(s.ShutdownCtx, s) if err != nil { logger.Warn("Failed to sync images") } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(request.ProjectParam(r), lifecycle.ClusterMemberRemoved.Event(name, requestor, nil)) return response.EmptySyncResponse } func internalClusterPostAccept(d *Daemon, r *http.Request) response.Response { s := d.State() req := internalClusterPostAcceptRequest{} // Parse the request err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. if req.Name == "" { return response.BadRequest(errors.New("No name provided")) } // Redirect all requests to the leader, which is the one // knowing what nodes are part of the raft cluster. localClusterAddress := s.LocalConfig.ClusterAddress() leader, err := s.Cluster.LeaderAddress() if err != nil { return response.InternalError(err) } if localClusterAddress != leader { logger.Debugf("Redirect member accept request to %s", leader) if leader == "" { return response.SmartError(errors.New("Unable to find leader address")) } url := &url.URL{ Scheme: "https", Path: "/internal/cluster/accept", Host: leader, } return response.SyncResponseRedirect(url.String()) } // Get lock now we are on leader. d.clusterMembershipMutex.Lock() defer d.clusterMembershipMutex.Unlock() // Make sure we have all the expected storage pools. err = clusterCheckStoragePoolsMatch(r.Context(), s.DB.Cluster, req.StoragePools) if err != nil { return response.SmartError(err) } // Make sure we have all the expected networks. err = clusterCheckNetworksMatch(r.Context(), s.DB.Cluster, req.Networks) if err != nil { return response.SmartError(err) } nodes, err := cluster.Accept(s, d.gateway, req.Name, req.Address, req.Schema, req.API, req.Architecture) if err != nil { return response.BadRequest(err) } accepted := internalClusterPostAcceptResponse{ RaftNodes: make([]internalRaftNode, len(nodes)), PublicKey: s.Endpoints.NetworkPublicKey(), PrivateKey: s.Endpoints.NetworkPrivateKey(), } for i, node := range nodes { accepted.RaftNodes[i].ID = node.ID accepted.RaftNodes[i].Address = node.Address accepted.RaftNodes[i].Role = int(node.Role) } return response.SyncResponse(true, accepted) } // A request for the /internal/cluster/accept endpoint. type internalClusterPostAcceptRequest struct { Name string `json:"name" yaml:"name"` Address string `json:"address" yaml:"address"` Schema int `json:"schema" yaml:"schema"` API int `json:"api" yaml:"api"` StoragePools []api.StoragePool `json:"storage_pools" yaml:"storage_pools"` Networks []api.InitNetworksProjectPost `json:"networks" yaml:"networks"` Architecture int `json:"architecture" yaml:"architecture"` } // A Response for the /internal/cluster/accept endpoint. type internalClusterPostAcceptResponse struct { RaftNodes []internalRaftNode `json:"raft_nodes" yaml:"raft_nodes"` PublicKey []byte `json:"public_key" yaml:"public_key"` PrivateKey []byte `json:"private_key" yaml:"private_key"` } // Represent a node that is part of the dqlite raft cluster. type internalRaftNode struct { ID uint64 `json:"id" yaml:"id"` Address string `json:"address" yaml:"address"` Role int `json:"role" yaml:"role"` Name string `json:"name" yaml:"name"` } // Used to update the cluster after a database node has been removed, and // possibly promote another one as database node. func internalClusterPostRebalance(d *Daemon, r *http.Request) response.Response { s := d.State() // Redirect all requests to the leader, which is the one with with // up-to-date knowledge of what nodes are part of the raft cluster. localClusterAddress := s.LocalConfig.ClusterAddress() leader, err := s.Cluster.LeaderAddress() if err != nil { return response.InternalError(err) } if localClusterAddress != leader { logger.Debugf("Redirect cluster rebalance request to %s", leader) url := &url.URL{ Scheme: "https", Path: "/internal/cluster/rebalance", Host: leader, } return response.SyncResponseRedirect(url.String()) } // Get lock now we are on leader. d.clusterMembershipMutex.Lock() defer d.clusterMembershipMutex.Unlock() err = rebalanceMemberRoles(s, d.gateway, r, nil) if err != nil { return response.SmartError(err) } return response.SyncResponse(true, nil) } // Check if there's a dqlite node whose role should be changed, and post a // change role request if so. func rebalanceMemberRoles(s *state.State, gateway *cluster.Gateway, r *http.Request, unavailableMembers []string) error { if s.ShutdownCtx.Err() != nil { return nil } again: address, nodes, err := cluster.Rebalance(s, gateway, unavailableMembers) if err != nil { return err } if address == "" { // Nothing to do. return nil } // Process demotions of offline nodes immediately. for _, node := range nodes { if node.Address != address { continue } reachable := cluster.HasConnectivity(s.Endpoints.NetworkCert(), s.ServerCert(), address, true) if node.Role != db.RaftSpare { if !reachable { // The server isn't ready to be promoted yet, try again next time. return nil } logger.Info("Promoting cluster member", logger.Ctx{"name": node.Name, "role": node.Role}) break } if reachable { // Don't demote reachable servers. break } logger.Info("Demoting cluster member", logger.Ctx{"name": node.Name, "role": node.Role}) err := gateway.DemoteOfflineNode(node.ID) if err != nil { return fmt.Errorf("Failed to demote cluster member %q: %w", node.Name, err) } goto again } // Then handle the promotions. err = changeMemberRole(s, r, address, nodes) if err != nil { return err } goto again } // Check if there are nodes not part of the raft configuration and add them in // case. func upgradeNodesWithoutRaftRole(s *state.State, gateway *cluster.Gateway) error { if s.ShutdownCtx.Err() != nil { return nil } var members []db.NodeInfo err := s.DB.Cluster.Transaction(context.Background(), func(ctx context.Context, tx *db.ClusterTx) error { var err error members, err = tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } return nil }) if err != nil { return err } return cluster.UpgradeMembersWithoutRole(gateway, members) } // Post a change role request to the member with the given address. The nodes // slice contains details about all members, including the one being changed. func changeMemberRole(s *state.State, r *http.Request, address string, nodes []db.RaftNode) error { post := &internalClusterPostAssignRequest{} for _, node := range nodes { post.RaftNodes = append(post.RaftNodes, internalRaftNode{ ID: node.ID, Address: node.Address, Role: int(node.Role), Name: node.Name, }) } client, err := cluster.Connect(address, s.Endpoints.NetworkCert(), s.ServerCert(), r, true) if err != nil { return err } _, _, err = client.RawQuery("POST", "/internal/cluster/assign", post, "") if err != nil { return err } return nil } // Try to handover the role of this member to another one. func handoverMemberRole(s *state.State, gateway *cluster.Gateway) error { // If we aren't clustered, there's nothing to do. if !s.ServerClustered { return nil } // Figure out our own cluster address. localClusterAddress := s.LocalConfig.ClusterAddress() post := &internalClusterPostHandoverRequest{ Address: localClusterAddress, } logCtx := logger.Ctx{"address": localClusterAddress} // Find the cluster leader. findLeader: leader, err := s.Cluster.LeaderAddress() if err != nil { return err } if leader == "" { return errors.New("No leader address found") } if leader == localClusterAddress { logger.Info("Transferring leadership", logCtx) err := gateway.TransferLeadership() if err != nil { return fmt.Errorf("Failed to transfer leadership: %w", err) } goto findLeader } logger.Info("Handing over cluster member role", logCtx) client, err := cluster.Connect(leader, s.Endpoints.NetworkCert(), s.ServerCert(), nil, true) if err != nil { return fmt.Errorf("Failed handing over cluster member role: %w", err) } _, _, err = client.RawQuery("POST", "/internal/cluster/handover", post, "") if err != nil { return err } return nil } // Used to assign a new role to a the local dqlite node. func internalClusterPostAssign(d *Daemon, r *http.Request) response.Response { s := d.State() req := internalClusterPostAssignRequest{} // Parse the request err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. if len(req.RaftNodes) == 0 { return response.BadRequest(errors.New("No raft members provided")) } nodes := make([]db.RaftNode, len(req.RaftNodes)) for i, node := range req.RaftNodes { nodes[i].ID = node.ID nodes[i].Address = node.Address nodes[i].Role = db.RaftRole(node.Role) nodes[i].Name = node.Name } err = cluster.Assign(s, d.gateway, nodes) if err != nil { return response.SmartError(err) } return response.SyncResponse(true, nil) } // A request for the /internal/cluster/assign endpoint. type internalClusterPostAssignRequest struct { RaftNodes []internalRaftNode `json:"raft_nodes" yaml:"raft_nodes"` } // Used to to transfer the responsibilities of a member to another one. func internalClusterPostHandover(d *Daemon, r *http.Request) response.Response { s := d.State() req := internalClusterPostHandoverRequest{} // Parse the request err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. if req.Address == "" { return response.BadRequest(errors.New("No id provided")) } // Redirect all requests to the leader, which is the one with // authoritative knowledge of the current raft configuration. localClusterAddress := s.LocalConfig.ClusterAddress() leader, err := s.Cluster.LeaderAddress() if err != nil { return response.InternalError(err) } if leader == "" { return response.SmartError(errors.New("No leader address found")) } if localClusterAddress != leader { logger.Debugf("Redirect handover request to %s", leader) url := &url.URL{ Scheme: "https", Path: "/internal/cluster/handover", Host: leader, } return response.SyncResponseRedirect(url.String()) } // Get lock now we are on leader. d.clusterMembershipMutex.Lock() defer d.clusterMembershipMutex.Unlock() target, nodes, err := cluster.Handover(s, d.gateway, req.Address) if err != nil { return response.SmartError(err) } // If there's no other member we can promote, there's nothing we can // do, just return. if target == "" { goto out } logger.Info("Promoting member during handover", logger.Ctx{"address": localClusterAddress, "losingAddress": req.Address, "candidateAddress": target}) err = changeMemberRole(s, r, target, nodes) if err != nil { return response.SmartError(err) } // Demote the member that is handing over. for i, node := range nodes { if node.Address == req.Address { nodes[i].Role = db.RaftSpare } } logger.Info("Demoting member during handover", logger.Ctx{"address": localClusterAddress, "losingAddress": req.Address}) err = changeMemberRole(s, r, req.Address, nodes) if err != nil { return response.SmartError(err) } out: return response.SyncResponse(true, nil) } // A request for the /internal/cluster/handover endpoint. type internalClusterPostHandoverRequest struct { // Address of the server whose role should be transferred. Address string `json:"address" yaml:"address"` } func clusterCheckStoragePoolsMatch(ctx context.Context, clusterDB *db.Cluster, reqPools []api.StoragePool) error { return clusterDB.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { poolNames, err := tx.GetCreatedStoragePoolNames(ctx) if err != nil && !response.IsNotFoundError(err) { return err } for _, name := range poolNames { found := false for _, reqPool := range reqPools { if reqPool.Name != name { continue } found = true var pool *api.StoragePool _, pool, _, err = tx.GetStoragePoolInAnyState(ctx, name) if err != nil { return err } if pool.Driver != reqPool.Driver { return fmt.Errorf("Mismatching driver for storage pool %s", name) } // Exclude the keys which are node-specific. exclude := db.NodeSpecificStorageConfig(pool.Driver) err = localUtil.CompareConfigs(pool.Config, reqPool.Config, exclude) if err != nil { return fmt.Errorf("Mismatching config for storage pool %s: %w", name, err) } break } if !found { return fmt.Errorf("Missing storage pool %s", name) } } return nil }) } func clusterCheckNetworksMatch(ctx context.Context, clusterDB *db.Cluster, reqNetworks []api.InitNetworksProjectPost) error { return clusterDB.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Get a list of projects for networks. networkProjectNames, err := dbCluster.GetProjectNames(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed to load projects for networks: %w", err) } for _, networkProjectName := range networkProjectNames { networkNames, err := tx.GetCreatedNetworkNamesByProject(ctx, networkProjectName) if err != nil && !response.IsNotFoundError(err) { return err } for _, networkName := range networkNames { _, network, _, err := tx.GetNetworkInAnyState(ctx, networkProjectName, networkName) if err != nil { return err } // OVN networks don't need local creation. if network.Type == "ovn" { continue } // Check that the network is present locally. found := false for _, reqNetwork := range reqNetworks { if reqNetwork.Name != networkName || reqNetwork.Project != networkProjectName { continue } found = true if reqNetwork.Type != network.Type { return fmt.Errorf("Mismatching type for network %q in project %q", networkName, networkProjectName) } // Exclude the keys which are node-specific. networkConfigWithoutNodeSpecific := db.StripNodeSpecificNetworkConfig(network.Config) reqNetworkConfigwithoutNodeSpecific := db.StripNodeSpecificNetworkConfig(reqNetwork.Config) err = localUtil.CompareConfigs(networkConfigWithoutNodeSpecific, reqNetworkConfigwithoutNodeSpecific, nil) if err != nil { return fmt.Errorf("Mismatching config for network %q in project %q: %w", network.Name, networkProjectName, err) } break } if !found { return fmt.Errorf("Missing network %q in project %q", networkName, networkProjectName) } } } return nil }) } // Used as low-level recovering helper. func internalClusterRaftNodeDelete(d *Daemon, r *http.Request) response.Response { s := d.State() address, err := url.PathUnescape(mux.Vars(r)["address"]) if err != nil { return response.SmartError(err) } err = cluster.RemoveRaftNode(d.gateway, address) if err != nil { return response.SmartError(err) } err = rebalanceMemberRoles(s, d.gateway, r, nil) if err != nil && !errors.Is(err, cluster.ErrNotLeader) { logger.Warn("Could not rebalance cluster member roles after raft member removal", logger.Ctx{"err": err}) } return response.SyncResponse(true, nil) } // swagger:operation GET /1.0/cluster/members/{name}/state cluster cluster_member_state_get // // Get state of the cluster member // // Gets state of a specific cluster member. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Cluster member name // type: string // required: true // responses: // "200": // description: Cluster member state // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/ClusterMemberState" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func clusterNodeStateGet(d *Daemon, r *http.Request) response.Response { memberName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } s := d.State() // Forward request. resp := forwardedResponseToNode(s, r, memberName) if resp != nil { return resp } memberState, err := cluster.MemberState(r.Context(), s, memberName) if err != nil { return response.SmartError(err) } return response.SyncResponse(true, memberState) } // swagger:operation POST /1.0/cluster/members/{name}/state cluster cluster_member_state_post // // Evacuate or restore a cluster member // // Evacuates or restores a cluster member. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Cluster member name // type: string // required: true // - in: body // name: cluster // description: Cluster member state // required: true // schema: // $ref: "#/definitions/ClusterMemberStatePost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func clusterNodeStatePost(d *Daemon, r *http.Request) response.Response { name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } s := d.State() // Forward request. resp := forwardedResponseToNode(s, r, name) if resp != nil { return resp } // Parse the request. req := api.ClusterMemberStatePost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Handling of evacuation mode. if req.Action == "evacuate" { // Validate the mode if provided. if req.Mode != "" { validator := internalInstance.InstanceConfigKeysAny["cluster.evacuate"] err = validator(req.Mode) if err != nil { return response.BadRequest(err) } } // Get a count of the cluster members. var serverCount int err := s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { nodes, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } serverCount = len(nodes) return nil }) if err != nil { return response.InternalError(err) } // Handle single node clusters. if serverCount == 1 { if req.Mode == "" || req.Mode == "auto" { req.Mode = "stop" } else if req.Mode != "stop" { return response.BadRequest(fmt.Errorf("Can't perform %q evacuation on a single node cluster", req.Mode)) } } } if req.Action == "evacuate" { run := func(op *operations.Operation) error { return evacuateClusterMember(context.Background(), s, op, name, req.Mode, evacuateStopInstance, evacuateMigrateInstance(r)) } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.ClusterMemberEvacuate, nil, nil, run, nil, nil, r) if err != nil { return response.SmartError(err) } return operations.OperationResponse(op) } else if req.Action == "restore" { if req.Mode != "" && req.Mode != "skip" { return response.BadRequest(fmt.Errorf("Invalid restore mode %q", req.Mode)) } return restoreClusterMember(d, r, req.Mode == "skip") } return response.BadRequest(fmt.Errorf("Unknown action %q", req.Action)) } incus-7.0.0/cmd/incusd/api_cluster_certificate.go000066400000000000000000000145641517523235500221140ustar00rootroot00000000000000package main import ( "context" "encoding/json" "encoding/pem" "fmt" "net/http" "os" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/server/acme" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/warningtype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/server/warnings" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) var clusterCertificateCmd = APIEndpoint{ Path: "cluster/certificate", Put: APIEndpointAction{Handler: clusterCertificatePut, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } // swagger:operation PUT /1.0/cluster/certificate cluster clustering_update_cert // // Update the certificate for the cluster // // Replaces existing cluster certificate and reloads each cluster member. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: body // name: cluster // description: Cluster certificate replace request // required: true // schema: // $ref: "#/definitions/ClusterCertificatePut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func clusterCertificatePut(d *Daemon, r *http.Request) response.Response { s := d.State() req := api.ClusterCertificatePut{} // Parse the request err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } certBytes := []byte(req.ClusterCertificate) keyBytes := []byte(req.ClusterCertificateKey) certBlock, _ := pem.Decode(certBytes) if certBlock == nil { return response.BadRequest(fmt.Errorf("Certificate must be base64 encoded PEM certificate: %w", err)) } keyBlock, _ := pem.Decode(keyBytes) if keyBlock == nil { return response.BadRequest(fmt.Errorf("Private key must be base64 encoded PEM key: %w", err)) } err = updateClusterCertificate(r.Context(), s, d.gateway, r, req) if err != nil { return response.SmartError(err) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(request.ProjectParam(r), lifecycle.ClusterCertificateUpdated.Event("certificate", requestor, nil)) return response.EmptySyncResponse } func updateClusterCertificate(ctx context.Context, s *state.State, gateway *cluster.Gateway, r *http.Request, req api.ClusterCertificatePut) error { reverter := revert.New() defer reverter.Fail() newClusterCertFilename := internalUtil.VarPath(acme.ClusterCertFilename) // First node forwards request to all other cluster nodes if r == nil || !isClusterNotification(r) { var err error reverter.Add(func() { _ = s.DB.Cluster.Transaction(context.Background(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpsertWarningLocalNode(ctx, "", -1, -1, warningtype.UnableToUpdateClusterCertificate, err.Error()) }) }) oldCertBytes, err := os.ReadFile(internalUtil.VarPath("cluster.crt")) if err != nil { return err } keyBytes, err := os.ReadFile(internalUtil.VarPath("cluster.key")) if err != nil { return err } oldReq := api.ClusterCertificatePut{ ClusterCertificate: string(oldCertBytes), ClusterCertificateKey: string(keyBytes), } // Get all members in cluster. var members []db.NodeInfo err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { members, err = tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } return nil }) if err != nil { return err } localClusterAddress := s.LocalConfig.ClusterAddress() reverter.Add(func() { // If distributing the new certificate fails, store the certificate. This new file will // be considered when running the auto renewal again. err := os.WriteFile(newClusterCertFilename, []byte(req.ClusterCertificate), 0o600) if err != nil { logger.Error("Failed storing new certificate", logger.Ctx{"err": err}) } }) newCertInfo, err := localtls.KeyPairFromRaw([]byte(req.ClusterCertificate), []byte(req.ClusterCertificateKey)) if err != nil { return err } var c incus.InstanceServer for i := range members { member := members[i] if member.Address == localClusterAddress { continue } c, err = cluster.Connect(member.Address, s.Endpoints.NetworkCert(), s.ServerCert(), r, true) if err != nil { return err } err = c.UpdateClusterCertificate(req, "") if err != nil { return err } // When reverting the certificate, we need to connect to the cluster members using the // new certificate otherwise we'll get a bad certificate error. reverter.Add(func() { c, err := cluster.Connect(member.Address, newCertInfo, s.ServerCert(), r, true) if err != nil { logger.Error("Failed to connect to cluster member", logger.Ctx{"address": member.Address, "err": err}) return } err = c.UpdateClusterCertificate(oldReq, "") if err != nil { logger.Error("Failed to update cluster certificate on cluster member", logger.Ctx{"address": member.Address, "err": err}) } }) } } err := internalUtil.WriteCert(s.OS.VarDir, "cluster", []byte(req.ClusterCertificate), []byte(req.ClusterCertificateKey), nil) if err != nil { return err } if util.PathExists(newClusterCertFilename) { err := os.Remove(newClusterCertFilename) if err != nil { return fmt.Errorf("Failed to remove cluster certificate: %w", err) } } // Get the new cluster certificate struct cert, err := internalUtil.LoadClusterCert(s.OS.VarDir) if err != nil { return err } // Update the certificate on the network endpoint and gateway s.Endpoints.NetworkUpdateCert(cert) gateway.NetworkUpdateCert(cert) // Resolve warning of this type _ = warnings.ResolveWarningsByLocalNodeAndType(s.DB.Cluster, warningtype.UnableToUpdateClusterCertificate) reverter.Success() return nil } incus-7.0.0/cmd/incusd/api_cluster_evacuation.go000066400000000000000000000660441517523235500217700ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "net" "net/http" "net/url" "runtime" "slices" "strconv" "strings" "time" "github.com/gorilla/mux" "golang.org/x/sync/errgroup" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/instance" instanceDrivers "github.com/lxc/incus/v7/internal/server/instance/drivers" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/scriptlet" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" "github.com/lxc/incus/v7/internal/server/task" "github.com/lxc/incus/v7/shared/api" apiScriptlet "github.com/lxc/incus/v7/shared/api/scriptlet" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" ) type ( evacuateStopFunc func(inst instance.Instance, action string) error evacuateMigrateFunc func(ctx context.Context, s *state.State, inst instance.Instance, sourceMemberInfo *db.NodeInfo, targetMemberInfo *db.NodeInfo, live bool, startInstance bool, op *operations.Operation) error ) type evacuateOpts struct { s *state.State instances []instance.Instance mode string srcMemberName string stopInstance evacuateStopFunc migrateInstance evacuateMigrateFunc op *operations.Operation } func evacuateClusterSetState(s *state.State, name string, newState int) error { return s.DB.Cluster.Transaction(context.Background(), func(ctx context.Context, tx *db.ClusterTx) error { // Get the node. node, err := tx.GetNodeByName(ctx, name) if err != nil { return fmt.Errorf("Failed to get cluster member by name: %w", err) } if node.State == db.ClusterMemberStatePending { return errors.New("Cannot evacuate or restore a pending cluster member") } // Do nothing if the node is already in expected state. if node.State == newState { if newState == db.ClusterMemberStateEvacuated { return errors.New("Cluster member is already evacuated") } else if newState == db.ClusterMemberStateCreated { return errors.New("Cluster member is already restored") } return errors.New("Cluster member is already in requested state") } // Set node status to requested value. err = tx.UpdateNodeStatus(node.ID, newState) if err != nil { return fmt.Errorf("Failed to update cluster member status: %w", err) } return nil }) } // evacuateHostShutdownDefaultTimeout default timeout (in seconds) for waiting for clean shutdown to complete. const evacuateHostShutdownDefaultTimeout = 30 func evacuateStopInstance(inst instance.Instance, action string) error { l := logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) switch action { case "force-stop": // Handle forced shutdown. err := inst.Stop(false) if err != nil && !errors.Is(err, instanceDrivers.ErrInstanceIsStopped) { return fmt.Errorf("Failed to force stop instance %q in project %q: %w", inst.Name(), inst.Project().Name, err) } case "stateful-stop": // Handle stateful stop. err := inst.Stop(true) if err != nil && !errors.Is(err, instanceDrivers.ErrInstanceIsStopped) { return fmt.Errorf("Failed to stateful stop instance %q in project %q: %w", inst.Name(), inst.Project().Name, err) } default: // Get the shutdown timeout for the instance. timeout := inst.ExpandedConfig()["boot.host_shutdown_timeout"] val, err := strconv.Atoi(timeout) if err != nil { val = evacuateHostShutdownDefaultTimeout } // Start with a clean shutdown. err = inst.Shutdown(time.Duration(val) * time.Second) if err != nil { l.Warn("Failed shutting down instance, forcing stop", logger.Ctx{"err": err}) // Fallback to forced stop. err = inst.Stop(false) if err != nil && !errors.Is(err, instanceDrivers.ErrInstanceIsStopped) { return fmt.Errorf("Failed to stop instance %q in project %q: %w", inst.Name(), inst.Project().Name, err) } } } // Mark the instance as RUNNING in volatile so its state can be properly restored. err := inst.VolatileSet(map[string]string{"volatile.last_state.power": instance.PowerStateRunning}) if err != nil { l.Warn("Failed to set instance state to RUNNING", logger.Ctx{"err": err}) } return nil } func evacuateMigrateInstance(r *http.Request) evacuateMigrateFunc { return func(ctx context.Context, s *state.State, inst instance.Instance, sourceMemberInfo *db.NodeInfo, targetMemberInfo *db.NodeInfo, live bool, startInstance bool, op *operations.Operation) error { // Migrate the instance. req := api.InstancePost{ Migration: true, Live: live, } err := migrateInstance(ctx, s, inst, req, sourceMemberInfo, targetMemberInfo, "", op) if err != nil { return fmt.Errorf("Failed to migrate instance %q in project %q: %w", inst.Name(), inst.Project().Name, err) } if !startInstance || live { return nil } // Start it back up on target. dest, err := cluster.Connect(targetMemberInfo.Address, s.Endpoints.NetworkCert(), s.ServerCert(), r, true) if err != nil { return fmt.Errorf("Failed to connect to destination %q for instance %q in project %q: %w", targetMemberInfo.Address, inst.Name(), inst.Project().Name, err) } dest = dest.UseProject(inst.Project().Name) if op != nil { _ = op.ExtendMetadata(map[string]any{"evacuation_progress": fmt.Sprintf("Starting %q in project %q", inst.Name(), inst.Project().Name)}) } startOp, err := dest.UpdateInstanceState(inst.Name(), api.InstanceStatePut{Action: "start"}, "") if err != nil { return err } err = startOp.Wait() if err != nil { return err } return nil } } func evacuateClusterMember(ctx context.Context, s *state.State, op *operations.Operation, name string, mode string, stopInstance evacuateStopFunc, migrateInstance evacuateMigrateFunc) error { // Get the instance list for the server being evacuated. var dbInstances []dbCluster.Instance err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error dbInstances, err = dbCluster.GetInstances(ctx, tx.Tx(), dbCluster.InstanceFilter{Node: &name}) if err != nil { return fmt.Errorf("Failed to get instances: %w", err) } return nil }) if err != nil { return err } // Load the instance structs. instances := make([]instance.Instance, len(dbInstances)) for i, dbInst := range dbInstances { inst, err := instance.LoadByProjectAndName(s, dbInst.Project, dbInst.Name) if err != nil { return fmt.Errorf("Failed to load instance: %w", err) } instances[i] = inst } // Setup a reverter. reverter := revert.New() defer reverter.Fail() // Set cluster member status to EVACUATING. err = evacuateClusterSetState(s, name, db.ClusterMemberStateEvacuating) if err != nil { return err } reverter.Add(func() { _ = evacuateClusterSetState(s, name, db.ClusterMemberStateCreated) }) // Perform the evacuation. opts := evacuateOpts{ s: s, instances: instances, mode: mode, srcMemberName: name, stopInstance: stopInstance, migrateInstance: migrateInstance, op: op, } err = evacuateInstances(ctx, opts) if err != nil { return err } // Stop networks after evacuation. if mode != "heal" { networkShutdown(s) } // Set cluster member status to EVACUATED. err = evacuateClusterSetState(s, name, db.ClusterMemberStateEvacuated) if err != nil { return err } reverter.Success() if mode != "heal" { s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.ClusterMemberEvacuated.Event(name, op.Requestor(), nil)) } return nil } func evacuateInstances(ctx context.Context, opts evacuateOpts) error { if opts.migrateInstance == nil { return errors.New("Missing migration callback function") } // Limit the number of concurrent evacuations to run at the same time numParallelEvacs := max(runtime.NumCPU()/16, 1) group, groupCtx := errgroup.WithContext(ctx) group.SetLimit(numParallelEvacs) for _, inst := range opts.instances { group.Go(func() error { return evacuateInstancesFunc(groupCtx, inst, opts) }) } err := group.Wait() if err != nil { return fmt.Errorf("Failed to evacuate instances: %w", err) } return nil } func evacuateInstancesFunc(ctx context.Context, inst instance.Instance, opts evacuateOpts) error { instProject := inst.Project() l := logger.AddContext(logger.Ctx{"project": instProject.Name, "instance": inst.Name()}) // Check if migratable. action := inst.CanMigrate() // Apply overrides. if opts.mode != "" { if opts.mode == "heal" { // Source server is dead, live-migration isn't an option. if action == "live-migrate" { action = "migrate" } if action != "migrate" { // We can only migrate instances or leave them as they are. return nil } } else if opts.mode != "auto" { action = opts.mode } } // Stop the instance if needed. isRunning := inst.IsRunning() if action != "live-migrate" { if opts.stopInstance != nil && isRunning { _ = opts.op.ExtendMetadata(map[string]any{"evacuation_progress": fmt.Sprintf("Stopping %q in project %q", inst.Name(), instProject.Name)}) err := opts.stopInstance(inst, action) if err != nil { return err } } if action != "migrate" { // Done with this instance. return nil } } else if !isRunning { // Can't live migrate if we're stopped. action = "migrate" } // Find a new location for the instance. sourceMemberInfo, targetMemberInfo, err := evacuateClusterSelectTarget(ctx, opts.s, inst) if err != nil { if api.StatusErrorCheck(err, http.StatusNotFound) { // Skip migration if no target is available. l.Warn("No migration target available for instance") return nil } return err } // Start migrating the instance. _ = opts.op.ExtendMetadata(map[string]any{"evacuation_progress": fmt.Sprintf("Migrating %q in project %q to %q", inst.Name(), instProject.Name, targetMemberInfo.Name)}) // Set origin server (but skip if already set as that suggests more than one server being evacuated). if inst.LocalConfig()["volatile.evacuate.origin"] == "" { _ = inst.VolatileSet(map[string]string{"volatile.evacuate.origin": opts.srcMemberName}) } start := isRunning || instanceShouldAutoStart(inst) err = opts.migrateInstance(ctx, opts.s, inst, sourceMemberInfo, targetMemberInfo, action == "live-migrate", start, opts.op) if err != nil { return err } return nil } // evacuateShutdown performs an evacuation of the local cluster member as part of the daemon shutdown sequence. func evacuateShutdown(ctx context.Context, s *state.State, name string) error { run := func(op *operations.Operation) error { return evacuateClusterMember(ctx, s, op, name, "", evacuateStopInstance, evacuateMigrateInstance(nil)) } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.ClusterMemberEvacuate, nil, nil, run, nil, nil, nil) if err != nil { return fmt.Errorf("Failed creating cluster member evacuate operation: %w", err) } err = op.Start() if err != nil { return fmt.Errorf("Failed starting cluster member evacuate operation: %w", err) } err = op.Wait(ctx) if err != nil { return fmt.Errorf("Failed to evacuate cluster member: %w", err) } return nil } func restoreClusterMember(d *Daemon, r *http.Request, skipInstances bool) response.Response { s := d.State() originName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Handle instances. instances := make([]instance.Instance, 0) localInstances := make([]instance.Instance, 0) if !skipInstances { var dbInstances []dbCluster.Instance err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbInstances, err = dbCluster.GetInstances(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed to get instances: %w", err) } return nil }) if err != nil { return response.SmartError(err) } for _, dbInst := range dbInstances { inst, err := instance.LoadByProjectAndName(s, dbInst.Project, dbInst.Name) if err != nil { return response.SmartError(fmt.Errorf("Failed to load instance: %w", err)) } if dbInst.Node == originName { localInstances = append(localInstances, inst) continue } // Only consider instances where volatile.evacuate.origin is set to the node which needs to be restored. val, ok := inst.LocalConfig()["volatile.evacuate.origin"] if !ok || val != originName { continue } instances = append(instances, inst) } } run := func(op *operations.Operation) error { // Setup a reverter. reverter := revert.New() defer reverter.Fail() // Set node status to RESTORING. err := evacuateClusterSetState(s, originName, db.ClusterMemberStateRestoring) if err != nil { return err } // Ensure node is put into its previous state if anything fails. reverter.Add(func() { _ = evacuateClusterSetState(s, originName, db.ClusterMemberStateEvacuated) }) // Restart the networks. err = networkStartup(d.State()) if err != nil { return err } // Restart the local instances. for _, inst := range localInstances { // Don't start instances which were stopped by the user. if inst.LocalConfig()["volatile.last_state.power"] != instance.PowerStateRunning { continue } // Don't attempt to start instances which are already running. if inst.IsRunning() { continue } // Start the instance. _ = op.ExtendMetadata(map[string]any{"evacuation_progress": fmt.Sprintf("Starting %q in project %q", inst.Name(), inst.Project().Name)}) // If configured for stateful stop, try restoring its state. action := inst.CanMigrate() if action == "stateful-stop" { err = inst.Start(true) } else { err = inst.Start(false) } if err != nil { return fmt.Errorf("Failed to start instance %q: %w", inst.Name(), err) } } // Limit the number of concurrent migrations to run at the same time numParallelMigrations := max(runtime.NumCPU()/16, 1) group := &errgroup.Group{} group.SetLimit(numParallelMigrations) // Migrate back the remote instances. for _, inst := range instances { group.Go(func() error { return restoreClusterMemberFunc(inst, op, originName, r, s) }) } err = group.Wait() if err != nil { return fmt.Errorf("Failed to restore instances: %w", err) } // Set node status to CREATED. err = evacuateClusterSetState(s, originName, db.ClusterMemberStateCreated) if err != nil { return err } reverter.Success() s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.ClusterMemberRestored.Event(originName, op.Requestor(), nil)) return nil } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.ClusterMemberRestore, nil, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } func restoreClusterMemberFunc(inst instance.Instance, op *operations.Operation, originName string, r *http.Request, s *state.State) error { var err error var source incus.InstanceServer var sourceNode db.NodeInfo l := logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) // Check the action. live := inst.CanMigrate() == "live-migrate" _ = op.ExtendMetadata(map[string]any{"evacuation_progress": fmt.Sprintf("Migrating %q in project %q from %q", inst.Name(), inst.Project().Name, inst.Location())}) err = s.DB.Cluster.Transaction(context.Background(), func(ctx context.Context, tx *db.ClusterTx) error { sourceNode, err = tx.GetNodeByName(ctx, inst.Location()) if err != nil { return fmt.Errorf("Failed to get node %q: %w", inst.Location(), err) } return nil }) if err != nil { return fmt.Errorf("Failed to get node: %w", err) } source, err = cluster.Connect(sourceNode.Address, s.Endpoints.NetworkCert(), s.ServerCert(), r, true) if err != nil { return fmt.Errorf("Failed to connect to source: %w", err) } source = source.UseProject(inst.Project().Name) apiInst, _, err := source.GetInstance(inst.Name()) if err != nil { return fmt.Errorf("Failed to get instance %q: %w", inst.Name(), err) } isRunning := apiInst.StatusCode == api.Running if isRunning && !live { _ = op.ExtendMetadata(map[string]any{"evacuation_progress": fmt.Sprintf("Stopping %q in project %q", inst.Name(), inst.Project().Name)}) timeout := inst.ExpandedConfig()["boot.host_shutdown_timeout"] val, err := strconv.Atoi(timeout) if err != nil { val = evacuateHostShutdownDefaultTimeout } // Attempt a clean stop. stopOp, err := source.UpdateInstanceState(inst.Name(), api.InstanceStatePut{Action: "stop", Force: false, Timeout: val}, "") if err != nil { return fmt.Errorf("Failed to stop instance %q: %w", inst.Name(), err) } // Wait for the stop operation to complete or timeout. err = stopOp.Wait() if err != nil { l.Warn("Failed shutting down instance, forcing stop", logger.Ctx{"err": err}) // On failure, attempt a forceful stop. stopOp, err = source.UpdateInstanceState(inst.Name(), api.InstanceStatePut{Action: "stop", Force: true}, "") if err != nil { // If this fails too, fail the whole operation. return fmt.Errorf("Failed to stop instance %q: %w", inst.Name(), err) } // Wait for the forceful stop to complete. err = stopOp.Wait() if err != nil && !strings.Contains(err.Error(), "The instance is already stopped") { return fmt.Errorf("Failed to stop instance %q: %w", inst.Name(), err) } } } req := api.InstancePost{ Name: inst.Name(), Migration: true, Live: live, } source = source.UseTarget(originName) migrationOp, err := source.MigrateInstance(inst.Name(), req) if err != nil { return fmt.Errorf("Migration API failure: %w", err) } err = migrationOp.Wait() if err != nil { return fmt.Errorf("Failed to wait for migration to finish: %w", err) } // Reload the instance after migration. inst, err = instance.LoadByProjectAndName(s, inst.Project().Name, inst.Name()) if err != nil { return fmt.Errorf("Failed to load instance: %w", err) } config := inst.LocalConfig() delete(config, "volatile.evacuate.origin") args := db.InstanceArgs{ Architecture: inst.Architecture(), Config: config, Description: inst.Description(), Devices: inst.LocalDevices(), Ephemeral: inst.IsEphemeral(), Profiles: inst.Profiles(), Project: inst.Project().Name, ExpiryDate: inst.ExpiryDate(), } err = inst.Update(args, false) if err != nil { return fmt.Errorf("Failed to update instance %q: %w", inst.Name(), err) } if !isRunning || live { return nil } _ = op.ExtendMetadata(map[string]any{"evacuation_progress": fmt.Sprintf("Starting %q in project %q", inst.Name(), inst.Project().Name)}) err = inst.Start(false) if err != nil { return fmt.Errorf("Failed to start instance %q: %w", inst.Name(), err) } return nil } func evacuateClusterSelectTarget(ctx context.Context, s *state.State, inst instance.Instance) (*db.NodeInfo, *db.NodeInfo, error) { var sourceMemberInfo *db.NodeInfo var targetMemberInfo *db.NodeInfo // Get candidate cluster members to move instances to. var candidateMembers []db.NodeInfo err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Get the source member info. srcMember, err := tx.GetNodeByName(ctx, inst.Location()) if err != nil { return fmt.Errorf("Failed loading location details %q for instance %q in project %q: %w", inst.Location(), inst.Name(), inst.Project().Name, err) } sourceMemberInfo = &srcMember allMembers, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } // Filter candidates by group if needed. group := inst.LocalConfig()["volatile.cluster.group"] if group != "" { newMembers := make([]db.NodeInfo, 0, len(allMembers)) for _, member := range allMembers { if !slices.Contains(member.Groups, group) { continue } newMembers = append(newMembers, member) } allMembers = newMembers } // Filter offline servers. candidateMembers, err = tx.GetCandidateMembers(ctx, allMembers, []int{inst.Architecture()}, "", nil, s.GlobalConfig.OfflineThreshold()) if err != nil { return err } return nil }) if err != nil { return nil, nil, err } // Run instance placement scriptlet if enabled. if s.GlobalConfig.InstancesPlacementScriptlet() != "" { leaderAddress, err := s.Cluster.LeaderAddress() if err != nil { return nil, nil, err } // Copy request so we don't modify it when expanding the config. reqExpanded := apiScriptlet.InstancePlacement{ InstancesPost: api.InstancesPost{ Name: inst.Name(), Type: api.InstanceType(inst.Type().String()), InstancePut: api.InstancePut{ Config: inst.ExpandedConfig(), Devices: inst.ExpandedDevices().CloneNative(), }, }, Project: inst.Project().Name, Reason: apiScriptlet.InstancePlacementReasonEvacuation, } reqExpanded.Architecture, err = osarch.ArchitectureName(inst.Architecture()) if err != nil { return nil, nil, fmt.Errorf("Failed getting architecture for instance %q in project %q: %w", inst.Name(), inst.Project().Name, err) } for _, p := range inst.Profiles() { reqExpanded.Profiles = append(reqExpanded.Profiles, p.Name) } ctx, cancel := context.WithTimeout(ctx, time.Second*5) targetMemberInfo, err = scriptlet.InstancePlacementRun(ctx, logger.Log, s, &reqExpanded, candidateMembers, leaderAddress) if err != nil { cancel() return nil, nil, fmt.Errorf("Failed instance placement scriptlet for instance %q in project %q: %w", inst.Name(), inst.Project().Name, err) } cancel() } // If target member not specified yet, then find the least loaded cluster member which // supports the instance's architecture. if targetMemberInfo == nil && len(candidateMembers) > 0 { targetMemberInfo = &candidateMembers[0] } if targetMemberInfo == nil { return nil, nil, fmt.Errorf("Couldn't find a cluster member for instance %q in project %q", inst.Name(), inst.Project().Name) } return sourceMemberInfo, targetMemberInfo, nil } func autoHealClusterTask(d *Daemon) (task.Func, task.Schedule) { f := func(ctx context.Context) { s := d.State() healingThreshold := s.GlobalConfig.ClusterHealingThreshold() if healingThreshold == 0 { return // Skip healing if it's disabled. } leader, err := s.Cluster.LeaderAddress() if err != nil { if errors.Is(err, cluster.ErrNodeIsNotClustered) { return // Skip healing if not clustered. } logger.Error("Failed to get leader cluster member address", logger.Ctx{"err": err}) return } if s.LocalConfig.ClusterAddress() != leader { return // Skip healing if not cluster leader. } var offlineMembers []db.NodeInfo { var members []db.NodeInfo err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { members, err = tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } return nil }) if err != nil { logger.Error("Failed healing cluster instances", logger.Ctx{"err": err}) return } for _, member := range members { // Ignore members which have been evacuated, and those which haven't exceeded the // healing offline trigger threshold. if member.State == db.ClusterMemberStateEvacuated || !member.IsOffline(healingThreshold) { continue } // As an extra safety net, make sure the dead system doesn't still respond on the network. hostAddress, _, err := net.SplitHostPort(member.Address) if err == nil { _, err := subprocess.RunCommand("ping", "-w1", "-c1", "-n", "-q", hostAddress) if err == nil { // Server isn't fully dead, not risking auto-healing. continue } } offlineMembers = append(offlineMembers, member) } } if len(offlineMembers) == 0 { return // Skip healing if there are no cluster members to evacuate. } opRun := func(op *operations.Operation) error { for _, member := range offlineMembers { err := healClusterMember(d, op, member.Name) if err != nil { logger.Error("Failed healing cluster instances", logger.Ctx{"server": member.Name, "err": err}) return err } } return nil } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.ClusterHeal, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating cluster instances heal operation", logger.Ctx{"err": err}) return } err = op.Start() if err != nil { logger.Error("Failed starting cluster instances heal operation", logger.Ctx{"err": err}) return } err = op.Wait(ctx) if err != nil { logger.Error("Failed healing cluster instances", logger.Ctx{"err": err}) return } } return f, task.Every(time.Minute) } func healClusterMember(d *Daemon, op *operations.Operation, name string) error { s := d.State() logger.Info("Starting cluster healing", logger.Ctx{"server": name}) defer logger.Info("Completed cluster healing", logger.Ctx{"server": name}) migrateFunc := func(ctx context.Context, s *state.State, inst instance.Instance, sourceMemberInfo *db.NodeInfo, targetMemberInfo *db.NodeInfo, live bool, startInstance bool, op *operations.Operation) error { // This returns an error if the instance's storage pool is local. // Since we only care about remote backed instances, this can be ignored and return nil instead. poolName, err := inst.StoragePool() if err != nil { if api.StatusErrorCheck(err, http.StatusNotFound) { return nil // We only care about remote backed instances. } return err } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return err } // Ignore anything using a local storage pool. if !pool.Driver().Info().Remote { return nil } // Migrate the instance. req := api.InstancePost{ Migration: true, } dest, err := cluster.Connect(targetMemberInfo.Address, s.Endpoints.NetworkCert(), s.ServerCert(), nil, true) if err != nil { return err } dest = dest.UseProject(inst.Project().Name) dest = dest.UseTarget(targetMemberInfo.Name) migrateOp, err := dest.MigrateInstance(inst.Name(), req) if err != nil { return err } err = migrateOp.Wait() if err != nil { return err } if !startInstance { return nil } // Start it back up on target. startOp, err := dest.UpdateInstanceState(inst.Name(), api.InstanceStatePut{Action: "start"}, "") if err != nil { return err } err = startOp.Wait() if err != nil { return err } return nil } // Attempt up to 5 evacuations. var err error for range 5 { err = evacuateClusterMember(context.Background(), s, op, name, "heal", nil, migrateFunc) if err == nil { s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.ClusterMemberHealed.Event(name, op.Requestor(), nil)) return nil } } logger.Error("Failed to heal cluster member", logger.Ctx{"server": name, "err": err}) return err } incus-7.0.0/cmd/incusd/api_cluster_group.go000066400000000000000000000701061517523235500207600ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/url" "slices" "strings" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" instanceDrivers "github.com/lxc/incus/v7/internal/server/instance/drivers" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/validate" ) var targetGroupPrefix = "@" var clusterGroupsCmd = APIEndpoint{ Path: "cluster/groups", Get: APIEndpointAction{Handler: clusterGroupsGet, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanView)}, Post: APIEndpointAction{Handler: clusterGroupsPost, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var clusterGroupCmd = APIEndpoint{ Path: "cluster/groups/{name}", Get: APIEndpointAction{Handler: clusterGroupGet, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanView)}, Post: APIEndpointAction{Handler: clusterGroupPost, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Put: APIEndpointAction{Handler: clusterGroupPut, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Patch: APIEndpointAction{Handler: clusterGroupPatch, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Delete: APIEndpointAction{Handler: clusterGroupDelete, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } // swagger:operation POST /1.0/cluster/groups cluster cluster_groups_post // // Create a cluster group. // // Creates a new cluster group. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: body // name: cluster // description: Cluster group to create // required: true // schema: // $ref: "#/definitions/ClusterGroupsPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func clusterGroupsPost(d *Daemon, r *http.Request) response.Response { s := d.State() if !s.ServerClustered { return response.BadRequest(errors.New("This server is not clustered")) } req := api.ClusterGroupsPost{} // Parse the request. err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid cluster group name: %w", err)) } err = clusterGroupValidate(req.Config) if err != nil { return response.BadRequest(err) } // Fill in the auto values. err = clusterGroupFill(r.Context(), s, nil, &req.ClusterGroupPut) if err != nil { return response.BadRequest(err) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { obj := dbCluster.ClusterGroup{ Name: req.Name, Description: req.Description, Nodes: req.Members, } groupID, err := dbCluster.CreateClusterGroup(ctx, tx.Tx(), obj) if err != nil { return err } for _, node := range obj.Nodes { _, err = dbCluster.CreateNodeClusterGroup(ctx, tx.Tx(), dbCluster.NodeClusterGroup{GroupID: int(groupID), Node: node}) if err != nil { return err } } err = dbCluster.CreateClusterGroupConfig(ctx, tx.Tx(), groupID, req.Config) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } requestor := request.CreateRequestor(r) lc := lifecycle.ClusterGroupCreated.Event(req.Name, requestor, nil) s.Events.SendLifecycle(api.ProjectDefaultName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation GET /1.0/cluster/groups cluster-groups cluster_groups_get // // Get the cluster groups // // Returns a list of cluster groups (URLs). // // --- // produces: // - application/json // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/cluster/groups/server01", // "/1.0/cluster/groups/server02" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/cluster/groups?recursion=1 cluster-groups cluster_groups_get_recursion1 // // Get the cluster groups // // Returns a list of cluster groups (structs). // // --- // produces: // - application/json // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of cluster groups // items: // $ref: "#/definitions/ClusterGroup" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func clusterGroupsGet(d *Daemon, r *http.Request) response.Response { s := d.State() if !s.ServerClustered { return response.BadRequest(errors.New("This server is not clustered")) } recursion := localUtil.IsRecursionRequest(r) var result any err := s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error if recursion { clusterGroups, err := dbCluster.GetClusterGroups(ctx, tx.Tx()) if err != nil { return err } for i := range clusterGroups { nodeClusterGroups, err := dbCluster.GetNodeClusterGroups(ctx, tx.Tx(), dbCluster.NodeClusterGroupFilter{GroupID: &clusterGroups[i].ID}) if err != nil { return err } clusterGroups[i].Nodes = make([]string, 0, len(nodeClusterGroups)) for _, node := range nodeClusterGroups { clusterGroups[i].Nodes = append(clusterGroups[i].Nodes, node.Node) } } apiClusterGroups := make([]*api.ClusterGroup, len(clusterGroups)) for i, clusterGroup := range clusterGroups { members, err := tx.GetClusterGroupNodes(ctx, clusterGroup.Name) if err != nil { return err } apiClusterGroups[i] = db.ClusterGroupToAPI(&clusterGroup, members) } result = apiClusterGroups } else { result, err = tx.GetClusterGroupURIs(ctx, dbCluster.ClusterGroupFilter{}) } return err }) if err != nil { return response.SmartError(err) } return response.SyncResponse(true, result) } // swagger:operation GET /1.0/cluster/groups/{name} cluster-groups cluster_group_get // // Get the cluster group // // Gets a specific cluster group. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Cluster group name // type: string // required: true // responses: // "200": // description: Cluster group // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/ClusterGroup" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func clusterGroupGet(d *Daemon, r *http.Request) response.Response { s := d.State() name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if !s.ServerClustered { return response.BadRequest(errors.New("This server is not clustered")) } var apiGroup *api.ClusterGroup err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Get the cluster group. group, err := dbCluster.GetClusterGroup(ctx, tx.Tx(), name) if err != nil { return err } nodeClusterGroups, err := dbCluster.GetNodeClusterGroups(ctx, tx.Tx(), dbCluster.NodeClusterGroupFilter{GroupID: &group.ID}) if err != nil { return err } group.Nodes = make([]string, 0, len(nodeClusterGroups)) for _, node := range nodeClusterGroups { group.Nodes = append(group.Nodes, node.Node) } apiGroup, err = group.ToAPI(ctx, tx.Tx()) if err != nil { return err } apiGroup.UsedBy, err = clusterGroupUsedBy(ctx, tx, group) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } return response.SyncResponseETag(true, apiGroup, apiGroup.ClusterGroupPut) } // swagger:operation POST /1.0/cluster/groups/{name} cluster-groups cluster_group_post // // Rename the cluster group // // Renames an existing cluster group. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Cluster group name // type: string // required: true // - in: body // name: name // description: Cluster group rename request // required: true // schema: // $ref: "#/definitions/ClusterGroupPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func clusterGroupPost(d *Daemon, r *http.Request) response.Response { s := d.State() name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if name == "default" { return response.Forbidden(errors.New(`The "default" group cannot be renamed`)) } if !s.ServerClustered { return response.BadRequest(errors.New("This server is not clustered")) } req := api.ClusterGroupPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid cluster group name: %w", err)) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Check that the name isn't already in use. _, err = dbCluster.GetClusterGroup(ctx, tx.Tx(), req.Name) if err == nil { return fmt.Errorf("Name %q already in use", req.Name) } currentGroup, err := dbCluster.GetClusterGroup(ctx, tx.Tx(), name) if err != nil { return err } usedBy, err := clusterGroupUsedBy(ctx, tx, currentGroup) if err != nil { return err } if len(usedBy) > 0 { return errors.New("Cluster group is currently in use") } // Rename the cluster group. err = dbCluster.RenameClusterGroup(ctx, tx.Tx(), name, req.Name) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } requestor := request.CreateRequestor(r) lc := lifecycle.ClusterGroupRenamed.Event(req.Name, requestor, logger.Ctx{"old_name": name}) s.Events.SendLifecycle(api.ProjectDefaultName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation PUT /1.0/cluster/groups/{name} cluster-groups cluster_group_put // // Update the cluster group // // Updates the entire cluster group configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Cluster group name // type: string // required: true // - in: body // name: cluster group // description: cluster group configuration // required: true // schema: // $ref: "#/definitions/ClusterGroupPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func clusterGroupPut(d *Daemon, r *http.Request) response.Response { s := d.State() name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if !s.ServerClustered { return response.BadRequest(errors.New("This server is not clustered")) } req := api.ClusterGroupPut{} // Parse the request. err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Validate the config. err = clusterGroupValidate(req.Config) if err != nil { return response.BadRequest(err) } // Get the current state. var dbClusterGroup *dbCluster.ClusterGroup err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbClusterGroup, err = dbCluster.GetClusterGroup(ctx, tx.Tx(), name) if err != nil { return err } nodeClusterGroups, err := dbCluster.GetNodeClusterGroups(ctx, tx.Tx(), dbCluster.NodeClusterGroupFilter{GroupID: &dbClusterGroup.ID}) if err != nil { return err } dbClusterGroup.Nodes = make([]string, 0, len(nodeClusterGroups)) for _, node := range nodeClusterGroups { dbClusterGroup.Nodes = append(dbClusterGroup.Nodes, node.Node) } return nil }) if err != nil { return response.SmartError(err) } // Fill in the auto values. err = clusterGroupFill(r.Context(), s, dbClusterGroup.Nodes, &req) if err != nil { return response.BadRequest(err) } // Update the database. err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { group, err := dbCluster.GetClusterGroup(ctx, tx.Tx(), name) if err != nil { return err } obj := dbCluster.ClusterGroup{ Name: group.Name, Description: req.Description, } err = dbCluster.UpdateClusterGroup(ctx, tx.Tx(), name, obj) if err != nil { return err } err = dbCluster.UpdateClusterGroupConfig(ctx, tx.Tx(), int64(group.ID), req.Config) if err != nil { return err } members, err := tx.GetClusterGroupNodes(ctx, name) if err != nil { return err } // skipMembers is a list of members which already belong to the group. skipMembers := []string{} for _, oldMember := range members { if !slices.Contains(req.Members, oldMember) { // Get all cluster groups this member belongs to. groups, err := tx.GetClusterGroupsWithNode(ctx, oldMember) if err != nil { return err } // Note that members who only belong to this group will not be removed from it. // That is because each member needs to belong to at least one group. if len(groups) > 1 { memberInstances, err := tx.GetClusterGroupMemberInstances(ctx, group, oldMember) if err != nil { return err } if len(memberInstances) > 0 { return errors.New("Cluster group member is currently in use") } // Remove member from this group as it belongs to at least one other group. err = tx.RemoveNodeFromClusterGroup(ctx, name, oldMember) if err != nil { return err } } } else { skipMembers = append(skipMembers, oldMember) } } for _, member := range req.Members { // Skip these members as they already belong to this group. if slices.Contains(skipMembers, member) { continue } // Add new members to the group. err = tx.AddNodeToClusterGroup(ctx, name, member) if err != nil { return err } } return nil }) if err != nil { return response.SmartError(err) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.ClusterGroupUpdated.Event(name, requestor, logger.Ctx{"description": req.Description, "members": req.Members})) return response.EmptySyncResponse } // swagger:operation PATCH /1.0/cluster/groups/{name} cluster-groups cluster_group_patch // // Update the cluster group // // Updates the cluster group configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Cluster group name // type: string // required: true // - in: body // name: cluster group // description: cluster group configuration // required: true // schema: // $ref: "#/definitions/ClusterGroupPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func clusterGroupPatch(d *Daemon, r *http.Request) response.Response { s := d.State() name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if !s.ServerClustered { return response.BadRequest(errors.New("This server is not clustered")) } var clusterGroup *api.ClusterGroup var dbClusterGroup *dbCluster.ClusterGroup err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbClusterGroup, err = dbCluster.GetClusterGroup(ctx, tx.Tx(), name) if err != nil { return err } nodeClusterGroups, err := dbCluster.GetNodeClusterGroups(ctx, tx.Tx(), dbCluster.NodeClusterGroupFilter{GroupID: &dbClusterGroup.ID}) if err != nil { return err } dbClusterGroup.Nodes = make([]string, 0, len(nodeClusterGroups)) for _, node := range nodeClusterGroups { dbClusterGroup.Nodes = append(dbClusterGroup.Nodes, node.Node) } clusterGroup, err = dbClusterGroup.ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } req := clusterGroup.Writable() // Validate the ETag. etag := []any{clusterGroup.Description, clusterGroup.Members} err = localUtil.EtagCheck(r, etag) if err != nil { return response.PreconditionFailed(err) } // Parse the request. err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } if req.Members == nil { req.Members = clusterGroup.Members } if req.Config == nil { req.Config = clusterGroup.Config } else { for k, v := range clusterGroup.Config { _, ok := req.Config[k] if !ok { req.Config[k] = v } } } // Validate the config. err = clusterGroupValidate(req.Config) if err != nil { return response.BadRequest(err) } // Fill in the auto values. err = clusterGroupFill(r.Context(), s, dbClusterGroup.Nodes, &req) if err != nil { return response.BadRequest(err) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { obj := dbCluster.ClusterGroup{ Name: dbClusterGroup.Name, Description: req.Description, } err = dbCluster.UpdateClusterGroup(ctx, tx.Tx(), name, obj) if err != nil { return err } groupID, err := dbCluster.GetClusterGroupID(ctx, tx.Tx(), obj.Name) if err != nil { return err } err = dbCluster.UpdateClusterGroupConfig(ctx, tx.Tx(), groupID, req.Config) if err != nil { return err } err = dbCluster.DeleteNodeClusterGroup(ctx, tx.Tx(), int(groupID)) if err != nil { return err } for _, node := range obj.Nodes { _, err = dbCluster.CreateNodeClusterGroup(ctx, tx.Tx(), dbCluster.NodeClusterGroup{GroupID: int(groupID), Node: node}) if err != nil { return err } } members, err := tx.GetClusterGroupNodes(ctx, name) if err != nil { return err } // skipMembers is a list of members which already belong to the group. skipMembers := []string{} for _, oldMember := range members { if !slices.Contains(req.Members, oldMember) { // Get all cluster groups this member belongs to. groups, err := tx.GetClusterGroupsWithNode(ctx, oldMember) if err != nil { return err } // Cluster member cannot be removed from the group as it doesn't belong to any other. if len(groups) == 1 { return fmt.Errorf("Cannot remove %s from group as member needs to belong to at least one group", oldMember) } memberInstances, err := tx.GetClusterGroupMemberInstances(ctx, dbClusterGroup, oldMember) if err != nil { return err } if len(memberInstances) > 0 { return errors.New("Cluster group member is currently in use") } // Remove member from this group as it belongs to at least one other group. err = tx.RemoveNodeFromClusterGroup(ctx, name, oldMember) if err != nil { return err } } else { skipMembers = append(skipMembers, oldMember) } } for _, member := range req.Members { // Skip these members as they already belong to this group. if slices.Contains(skipMembers, member) { continue } // Add new members to the group. err = tx.AddNodeToClusterGroup(ctx, name, member) if err != nil { return err } } return nil }) if err != nil { return response.SmartError(err) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.ClusterGroupUpdated.Event(name, requestor, logger.Ctx{"description": req.Description, "members": req.Members})) return response.EmptySyncResponse } // swagger:operation DELETE /1.0/cluster/groups/{name} cluster-groups cluster_group_delete // // Delete the cluster group. // // Removes the cluster group. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Cluster group name // type: string // required: true // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func clusterGroupDelete(d *Daemon, r *http.Request) response.Response { s := d.State() name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Quick checks. if name == "default" { return response.Forbidden(errors.New("The 'default' cluster group cannot be deleted")) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { members, err := tx.GetClusterGroupNodes(ctx, name) if err != nil { return err } if len(members) > 0 { return errors.New("Only empty cluster groups can be removed") } // Get the cluster group. group, err := dbCluster.GetClusterGroup(ctx, tx.Tx(), name) if err != nil { return err } usedBy, err := clusterGroupUsedBy(ctx, tx, group) if err != nil { return err } if len(usedBy) > 0 { return errors.New("Cluster group is currently in use") } return dbCluster.DeleteClusterGroup(ctx, tx.Tx(), name) }) if err != nil { return response.SmartError(err) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(name, lifecycle.ClusterGroupDeleted.Event(name, requestor, nil)) return response.EmptySyncResponse } // clusterGroupValidate validates the configuration keys/values for cluster groups. func clusterGroupValidate(config map[string]string) error { configKeys := map[string]func(value string) error{} // Add architecture keys. for _, arch := range osarch.SupportedArchitectures() { // gendoc:generate(entity=cluster_group, group=common, key=instances.vm.cpu.ARCHITECTURE.baseline) // The CPU base architecture name as can be found through `qemu -cpu ?`. // // This can be a generic definition like `qemu64` or `kvm64`, or it can be a specific hardware architecture like `EPYC-v2`. // It's important to ensure that all servers in the group match that baseline. // --- // type: string // shortdesc: CPU base architecture name configKeys[fmt.Sprintf("instances.vm.cpu.%s.baseline", arch)] = validate.Optional(validate.IsAny) // gendoc:generate(entity=cluster_group, group=common, key=instances.vm.cpu.ARCHITECTURE.flags) // A comma separated list of CPU flags to add on top of CPU baseline or a list of flags to remove from it. // // To remove a flag, use `-flag`. // --- // type: string // shortdesc: CPU flags to add/remove to/from the baseline configKeys[fmt.Sprintf("instances.vm.cpu.%s.flags", arch)] = validate.Optional(validate.IsListOf(validate.IsAny)) } for k, v := range config { // User keys are free for all. // gendoc:generate(entity=cluster_group, group=common, key=user.*) // User keys can be used in search. // --- // type: string // shortdesc: Free form user key/value storage if strings.HasPrefix(k, "user.") { continue } validator, ok := configKeys[k] if !ok { return fmt.Errorf("Invalid cluster group configuration key %q", k) } err := validator(v) if err != nil { return fmt.Errorf("Invalid cluster group configuration key %q value", k) } } return nil } // clusterGroupFill fills in automatic values. func clusterGroupFill(ctx context.Context, s *state.State, servers []string, req *api.ClusterGroupPut) error { // If no config, nothing to fill. if req.Config == nil { return nil } for _, arch := range osarch.SupportedArchitectures() { baseline := req.Config[fmt.Sprintf("instances.vm.cpu.%s.baseline", arch)] flags := req.Config[fmt.Sprintf("instances.vm.cpu.%s.flags", arch)] // Check whether we need to fill in values. if flags != "auto" { continue } if baseline != "kvm64" || arch != "x86_64" { return errors.New("Automatic CPU flags are currently only supported on \"x86_64\" with the \"kvm64\" baseline") } if len(servers) == 0 { return errors.New("Can't compute automatic CPU flags when no servers are in the cluster group") } // Fill in the flags. cpuFlags, err := instanceDrivers.GetClusterCPUFlags(ctx, s, servers, arch) if err != nil { return fmt.Errorf("Couldn't compute automatic CPU flags: %w", err) } req.Config[fmt.Sprintf("instances.vm.cpu.%s.flags", arch)] = strings.Join(cpuFlags, ",") } return nil } // clusterGroupUsedBy returns URLs of all instances and projects using the specified cluster group. func clusterGroupUsedBy(ctx context.Context, tx *db.ClusterTx, clusterGroup *dbCluster.ClusterGroup) ([]string, error) { usedBy := []string{} instances, err := dbCluster.GetInstances(ctx, tx.Tx()) if err != nil { return nil, err } // Check which instances are assigned to the specified cluster group. for _, instance := range instances { config, err := dbCluster.GetInstanceConfig(ctx, tx.Tx(), instance.ID) if err != nil { return nil, err } group := config["volatile.cluster.group"] if group != clusterGroup.Name { continue } apiInstance := api.Instance{Name: instance.Name} usedBy = append(usedBy, apiInstance.URL(version.APIVersion, instance.Project).String()) } // Check which projects are using the specified cluster group. projects, err := dbCluster.GetProjects(ctx, tx.Tx()) if err != nil { return nil, fmt.Errorf("Failed loading projects: %w", err) } for _, dbProject := range projects { apiProject, err := dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return nil, err } projectClusterGroups := project.GetRestrictedClusterGroups(apiProject) if slices.Contains(projectClusterGroups, clusterGroup.Name) { usedBy = append(usedBy, apiProject.URL(version.APIVersion).String()) } } return usedBy, nil } incus-7.0.0/cmd/incusd/api_cluster_rebalance.go000066400000000000000000000363351517523235500215460ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "math" "sort" "strconv" "time" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/scriptlet" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/server/task" "github.com/lxc/incus/v7/shared/api" apiScriptlet "github.com/lxc/incus/v7/shared/api/scriptlet" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" ) // ServerScore represents server score taken into account during load balancing. type ServerScore struct { NodeInfo db.NodeInfo Resources *api.Resources Score uint8 } // ServerUsage represents current server load. type ServerUsage struct { MemoryUsage uint64 MemoryTotal uint64 CPUUsage float64 CPUTotal uint64 } // sortAndGroupByArch sorts servers by its score and groups them by cpu architecture. func sortAndGroupByArch(servers []*ServerScore) map[string][]*ServerScore { sort.Slice(servers, func(i, j int) bool { return servers[i].Score > servers[j].Score }) result := make(map[string][]*ServerScore) for _, s := range servers { arch := s.Resources.CPU.Architecture _, ok := result[arch] if !ok { result[arch] = []*ServerScore{} } result[arch] = append(result[arch], s) } return result } // calculateScore calculates score for single server. func calculateScore(su *ServerUsage, au *ServerUsage) uint8 { memoryUsage := su.MemoryUsage memoryTotal := su.MemoryTotal cpuUsage := su.CPUUsage cpuTotal := su.CPUTotal if au != nil { memoryUsage += au.MemoryUsage memoryTotal += au.MemoryTotal cpuUsage += au.CPUUsage cpuTotal += au.CPUTotal } memoryScore := uint8(float64(memoryUsage) * 100 / float64(memoryTotal)) cpuScore := uint8((cpuUsage * 100) / float64(cpuTotal)) return (memoryScore + cpuScore) / 2 } // calculateServersScore calculates score based on memory and CPU usage for servers in cluster. func calculateServersScore(s *state.State, members []db.NodeInfo) (map[string][]*ServerScore, error) { scores := []*ServerScore{} for _, member := range members { clusterMember, err := cluster.Connect(member.Address, s.Endpoints.NetworkCert(), s.ServerCert(), nil, true) if err != nil { return nil, fmt.Errorf("Failed to connect to cluster member: %w", err) } res, err := clusterMember.GetServerResources() if err != nil { return nil, fmt.Errorf("Failed to get resources for cluster member: %w", err) } su := &ServerUsage{ MemoryUsage: res.Memory.Used, MemoryTotal: res.Memory.Total, CPUUsage: res.Load.Average1Min, CPUTotal: res.CPU.Total, } serverScore := calculateScore(su, nil) scores = append(scores, &ServerScore{NodeInfo: member, Resources: res, Score: serverScore}) } return sortAndGroupByArch(scores), nil } // clusterRebalanceServers is responsible for instances migration from the most busy server to less busy candidates. func clusterRebalanceServers(ctx context.Context, s *state.State, srcServer *ServerScore, candidates []*ServerScore, leaderAddress string, maxToMigrate int64) (int64, error) { numOfMigrated := int64(0) // Restrict candidates to servers less loaded than the source. lessLoadedCandidates := make([]*ServerScore, 0, len(candidates)) for _, c := range candidates { if c.Score >= srcServer.Score { continue } lessLoadedCandidates = append(lessLoadedCandidates, c) } if len(lessLoadedCandidates) == 0 { return numOfMigrated, nil } // The default target is the least-loaded candidate (last in the sorted list). dstServer := lessLoadedCandidates[len(lessLoadedCandidates)-1] // Get the list of instances on the source. var dbInstances []dbCluster.Instance err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error instType := instancetype.VM dbInstances, err = dbCluster.GetInstances(ctx, tx.Tx(), dbCluster.InstanceFilter{Node: &srcServer.NodeInfo.Name, Type: &instType}) if err != nil { return fmt.Errorf("Failed to get instances: %w", err) } return nil }) if err != nil { return -1, fmt.Errorf("Failed to get instances: %w", err) } // Filter for instances that can be live migrated to a new target. var instances []instance.Instance for _, dbInst := range dbInstances { inst, err := instance.LoadByProjectAndName(s, dbInst.Project, dbInst.Name) if err != nil { return -1, fmt.Errorf("Failed to load instance: %w", err) } // Do not allow to migrate instance which doesn't support live migration. if inst.CanMigrate() != "live-migrate" { continue } // Check if instance is ready for next migration. lastMove := inst.LocalConfig()["volatile.rebalance.last_move"] cooldown := s.GlobalConfig.ClusterRebalanceCooldown() if lastMove != "" { v, err := strconv.ParseInt(lastMove, 10, 64) if err != nil { return -1, fmt.Errorf("Failed to parse last_move value: %w", err) } expiry, err := internalInstance.GetExpiry(time.Unix(v, 0), cooldown) if err != nil { return -1, fmt.Errorf("Failed to calculate expiration for cooldown time: %w", err) } if time.Now().Before(expiry) { continue } } instances = append(instances, inst) } // Map candidate name to its score data for quick lookup. candidateByName := make(map[string]*ServerScore, len(lessLoadedCandidates)) for _, c := range lessLoadedCandidates { candidateByName[c.NodeInfo.Name] = c } // Track running usage and score per target so multiple instances heading to the same target accumulate correctly. runningUsage := make(map[string]*ServerUsage, len(lessLoadedCandidates)) runningScore := make(map[string]uint8, len(lessLoadedCandidates)) for _, c := range lessLoadedCandidates { runningUsage[c.NodeInfo.Name] = &ServerUsage{ MemoryUsage: c.Resources.Memory.Used, MemoryTotal: c.Resources.Memory.Total, CPUUsage: c.Resources.Load.Average1Min, CPUTotal: c.Resources.CPU.Total, } runningScore[c.NodeInfo.Name] = c.Score } placementScriptletEnabled := s.GlobalConfig.InstancesPlacementScriptlet() != "" // Prepare the source API client. srcClient, err := cluster.Connect(srcServer.NodeInfo.Address, s.Endpoints.NetworkCert(), s.ServerCert(), nil, true) if err != nil { return -1, fmt.Errorf("Failed to connect to cluster member: %w", err) } for _, inst := range instances { if numOfMigrated >= maxToMigrate { // We're done moving instances for now. return numOfMigrated, nil } // Filter the candidate list for this instance using project restrictions. var instanceCandidates []*ServerScore err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := dbCluster.GetProject(ctx, tx.Tx(), inst.Project().Name) if err != nil { return fmt.Errorf("Failed to get project: %w", err) } apiProject, err := dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed to load project: %w", err) } for _, c := range lessLoadedCandidates { _, _, err := project.CheckTarget(ctx, s.Authorizer, nil, tx, apiProject, c.NodeInfo.Name, []db.NodeInfo{c.NodeInfo}) if err != nil { continue } instanceCandidates = append(instanceCandidates, c) } return nil }) if err != nil { return -1, fmt.Errorf("Failed to filter candidates for instance %q in project %q: %w", inst.Name(), inst.Project().Name, err) } if len(instanceCandidates) == 0 { // No allowed targets for this instance. continue } // Default target is the least-loaded allowed candidate. instanceDstServer := instanceCandidates[len(instanceCandidates)-1] chosenTarget := &instanceDstServer.NodeInfo // Default fallback: if scriptlet not enabled and the global least-loaded candidate is allowed, prefer it. if !placementScriptletEnabled { for _, c := range instanceCandidates { if c.NodeInfo.Name == dstServer.NodeInfo.Name { chosenTarget = &dstServer.NodeInfo break } } } if placementScriptletEnabled { archName, err := osarch.ArchitectureName(inst.Architecture()) if err != nil { return -1, fmt.Errorf("Failed getting architecture for instance %q in project %q: %w", inst.Name(), inst.Project().Name, err) } profileNames := make([]string, 0, len(inst.Profiles())) for _, p := range inst.Profiles() { profileNames = append(profileNames, p.Name) } placementReq := apiScriptlet.InstancePlacement{ InstancesPost: api.InstancesPost{ Name: inst.Name(), Type: api.InstanceType(inst.Type().String()), InstancePut: api.InstancePut{ Architecture: archName, Config: inst.ExpandedConfig(), Devices: inst.ExpandedDevices().CloneNative(), Profiles: profileNames, }, }, Project: inst.Project().Name, Reason: apiScriptlet.InstancePlacementReasonRebalance, } // Build the scriptlet candidate list sorted from least to most loaded. sortedCandidates := make([]db.NodeInfo, 0, len(instanceCandidates)) for i := len(instanceCandidates) - 1; i >= 0; i-- { sortedCandidates = append(sortedCandidates, instanceCandidates[i].NodeInfo) } scriptCtx, cancel := context.WithTimeout(ctx, time.Second*5) scriptTarget, err := scriptlet.InstancePlacementRun(scriptCtx, logger.Log, s, &placementReq, sortedCandidates, leaderAddress) cancel() if err != nil { return -1, fmt.Errorf("Failed instance placement scriptlet for instance %q in project %q: %w", inst.Name(), inst.Project().Name, err) } if scriptTarget != nil { chosenTarget = scriptTarget } } // Skip if the chosen target is the source itself. if chosenTarget.Name == srcServer.NodeInfo.Name { continue } // Look up the score data for the chosen target. chosenScore, ok := candidateByName[chosenTarget.Name] if !ok { // Chosen target isn't in our candidate list (shouldn't happen). continue } // Per-target target score: midpoint between source and chosen target initial loads. targetScore := (srcServer.Score + chosenScore.Score) / 2 if runningScore[chosenTarget.Name] >= targetScore { // We've already balanced load against this target. continue } // Calculate resource consumption. cpuUsage, memUsage, _, err := instance.ResourceUsage(inst.ExpandedConfig(), inst.ExpandedDevices().CloneNative(), api.InstanceType(inst.Type().String())) if err != nil { return -1, fmt.Errorf("Failed to establish instance resource usage: %w", err) } // Calculate impact of migration. additionalUsage := &ServerUsage{ MemoryUsage: uint64(cpuUsage), CPUUsage: float64(memUsage), } expectedScore := calculateScore(runningUsage[chosenTarget.Name], additionalUsage) if expectedScore >= targetScore { // Skip the instance as it would have too big an impact. continue } // Prepare for live migration. req := api.InstancePost{ Migration: true, Live: true, } targetClient := srcClient.UseTarget(chosenTarget.Name) migrationOp, err := targetClient.MigrateInstance(inst.Name(), req) if err != nil { return -1, fmt.Errorf("Migration API failure: %w", err) } err = migrationOp.Wait() if err != nil { return -1, fmt.Errorf("Failed to wait for migration to finish: %w", err) } // Record the migration in the instance volatile storage. err = inst.VolatileSet(map[string]string{"volatile.rebalance.last_move": strconv.FormatInt(time.Now().Unix(), 10)}) if err != nil { return -1, err } // Update counters and per-target running state. numOfMigrated++ runningScore[chosenTarget.Name] = expectedScore runningUsage[chosenTarget.Name].MemoryUsage += additionalUsage.MemoryUsage runningUsage[chosenTarget.Name].CPUUsage += additionalUsage.CPUUsage } return numOfMigrated, nil } // clusterRebalance performs cluster re-balancing. func clusterRebalance(ctx context.Context, s *state.State, servers map[string][]*ServerScore, leaderAddress string) error { rebalanceThreshold := s.GlobalConfig.ClusterRebalanceThreshold() rebalanceBatch := s.GlobalConfig.ClusterRebalanceBatch() numOfMigrated := int64(0) for archName, v := range servers { if numOfMigrated >= rebalanceBatch { // Maximum number of instances already migrated in this run. continue } if len(v) < 2 { // Skip if there isn't at least 2 servers with specific arch. continue } if v[0].Score == 0 { // Don't migrate anything if most loaded isn't loaded. continue } leastBusyIndex := len(v) - 1 percentageChange := int64(float64(v[0].Score-v[leastBusyIndex].Score) / float64(v[0].Score) * 100) logger.Debug("Automatic re-balancing", logger.Ctx{"Architecture": archName, "LeastBusy": v[leastBusyIndex].NodeInfo.Name, "LeastBusyScore": v[leastBusyIndex].Score, "MostBusy": v[0].NodeInfo.Name, "MostBusyScore": v[0].Score, "Difference": fmt.Sprintf("%d%%", percentageChange), "Threshold": fmt.Sprintf("%d%%", rebalanceThreshold)}) if percentageChange < rebalanceThreshold { continue // Skip as threshold condition is not met. } n, err := clusterRebalanceServers(ctx, s, v[0], v[1:], leaderAddress, rebalanceBatch-numOfMigrated) if err != nil { return fmt.Errorf("Failed to rebalance cluster: %w", err) } numOfMigrated += n } return nil } func autoRebalanceCluster(ctx context.Context, d *Daemon) error { s := d.State() // Confirm we should run the rebalance. leader, err := s.Cluster.LeaderAddress() if err != nil { if errors.Is(err, cluster.ErrNodeIsNotClustered) { // Not clustered. return nil } return fmt.Errorf("Failed to get leader cluster member address: %w", err) } if s.LocalConfig.ClusterAddress() != leader { // Not the leader. return nil } // Get all online members var onlineMembers []db.NodeInfo err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { members, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } onlineMembers, err = tx.GetCandidateMembers(ctx, members, nil, "", nil, s.GlobalConfig.OfflineThreshold()) if err != nil { return fmt.Errorf("Failed getting online cluster members: %w", err) } return nil }) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } servers, err := calculateServersScore(s, onlineMembers) if err != nil { return fmt.Errorf("Failed calculating servers score: %w", err) } err = clusterRebalance(ctx, s, servers, leader) if err != nil { return fmt.Errorf("Failed rebalancing cluster: %w", err) } return nil } func autoRebalanceClusterTask(d *Daemon) (task.Func, task.Schedule) { f := func(ctx context.Context) { s := d.State() // Check that we should run now. interval := s.GlobalConfig.ClusterRebalanceInterval() if interval <= 0 { // Re-balance is disabled. return } now := time.Now() elapsed := int64(math.Round(now.Sub(s.StartTime).Minutes())) if elapsed%interval != 0 { // It's not time for a re-balance. return } // Run the rebalance. err := autoRebalanceCluster(ctx, d) if err != nil { logger.Error("Failed during cluster auto rebalancing", logger.Ctx{"err": err}) } } return f, task.Every(time.Minute, task.SkipFirst) } incus-7.0.0/cmd/incusd/api_cluster_test.go000066400000000000000000000077031517523235500206060ustar00rootroot00000000000000package main import ( "fmt" "net" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/api" ) // allocatePort asks the kernel for a free open port that is ready to use. func allocatePort() (int, error) { addr, err := net.ResolveTCPAddr("tcp", "localhost:0") if err != nil { return -1, err } l, err := net.ListenTCP("tcp", addr) if err != nil { return -1, err } return l.Addr().(*net.TCPAddr).Port, l.Close() } // A node which is already configured for networking can be converted to a // single-node cluster. func TestCluster_Bootstrap(t *testing.T) { daemon, cleanup := newTestDaemon(t) defer cleanup() // Simulate what happens when running "init", where a PUT /1.0 // request is issued to set both core.https_address and // cluster.https_address to the same value. f := clusterFixture{t: t} f.EnableNetworkingWithClusterAddress(daemon) client := f.ClientUnix(daemon) cluster := api.ClusterPut{} cluster.ServerName = "buzz" cluster.Enabled = true op, err := client.UpdateCluster(cluster, "") require.NoError(t, err) require.NoError(t, op.Wait()) server, _, err := client.GetServer() require.NoError(t, err) assert.True(t, client.IsClustered()) assert.Equal(t, "buzz", server.Environment.ServerName) } // Check the cluster API on a non-clustered server. func TestCluster_Get(t *testing.T) { daemon, cleanup := newTestDaemon(t) defer cleanup() c, err := incus.ConnectIncusUnix(daemon.os.GetUnixSocket(), nil) require.NoError(t, err) cluster, _, err := c.GetCluster() require.NoError(t, err) assert.Equal(t, "", cluster.ServerName) assert.False(t, cluster.Enabled) } // A node can be renamed. func TestCluster_RenameNode(t *testing.T) { daemon, cleanup := newTestDaemon(t) defer cleanup() f := clusterFixture{t: t} f.EnableNetworking(daemon) client := f.ClientUnix(daemon) cluster := api.ClusterPut{} cluster.ServerName = "buzz" cluster.Enabled = true op, err := client.UpdateCluster(cluster, "") require.NoError(t, err) require.NoError(t, op.Wait()) node := api.ClusterMemberPost{ServerName: "rusp"} err = client.RenameClusterMember("buzz", node) require.NoError(t, err) _, _, err = client.GetClusterMember("rusp") require.NoError(t, err) } // Test helper for cluster-related APIs. type clusterFixture struct { t *testing.T clients map[*Daemon]incus.InstanceServer } // Enable networking in the given daemon. The password is optional and can be // an empty string. func (f *clusterFixture) EnableNetworking(daemon *Daemon) { port, err := allocatePort() require.NoError(f.t, err) address := fmt.Sprintf("127.0.0.1:%d", port) client := f.ClientUnix(daemon) server, _, err := client.GetServer() require.NoError(f.t, err) serverPut := server.Writable() serverPut.Config["core.https_address"] = address require.NoError(f.t, client.UpdateServer(serverPut, "")) } // Enable networking in the given daemon, and set cluster.https_address to the // same value as core.https address. The password is optional and can be an // empty string. func (f *clusterFixture) EnableNetworkingWithClusterAddress(daemon *Daemon) { port, err := allocatePort() require.NoError(f.t, err) address := fmt.Sprintf("127.0.0.1:%d", port) client := f.ClientUnix(daemon) server, _, err := client.GetServer() require.NoError(f.t, err) serverPut := server.Writable() serverPut.Config["core.https_address"] = address serverPut.Config["cluster.https_address"] = address require.NoError(f.t, client.UpdateServer(serverPut, "")) } // Get a client for the given daemon connected via UNIX socket, creating one if // needed. func (f *clusterFixture) ClientUnix(daemon *Daemon) incus.InstanceServer { if f.clients == nil { f.clients = make(map[*Daemon]incus.InstanceServer) } client, ok := f.clients[daemon] if !ok { var err error client, err = incus.ConnectIncusUnix(daemon.os.GetUnixSocket(), nil) require.NoError(f.t, err) } return client } incus-7.0.0/cmd/incusd/api_internal.go000066400000000000000000001042521517523235500176770ustar00rootroot00000000000000package main import ( "bytes" "context" "database/sql" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "os" "path/filepath" "runtime" runtimeDebug "runtime/debug" "slices" "strconv" "strings" "github.com/gorilla/mux" "golang.org/x/sys/unix" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/internal/server/db/warningtype" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" internalSQL "github.com/lxc/incus/v7/internal/sql" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) var apiInternal = []APIEndpoint{ internalBGPStateCmd, internalClusterAcceptCmd, internalClusterAssignCmd, internalClusterHandoverCmd, internalClusterRaftNodeCmd, internalClusterRebalanceCmd, internalContainerOnStartCmd, internalContainerOnStopCmd, internalContainerOnStopNSCmd, internalVirtualMachineOnResizeCmd, internalGarbageCollectorCmd, internalImageOptimizeCmd, internalImageRefreshCmd, internalRAFTSnapshotCmd, internalRebalanceLoadCmd, internalReadyCmd, internalShutdownCmd, internalSQLCmd, internalWarningCreateCmd, } // Daemon management internal commands. var internalReadyCmd = APIEndpoint{ Path: "ready", Get: APIEndpointAction{Handler: internalWaitReady, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var internalShutdownCmd = APIEndpoint{ Path: "shutdown", Put: APIEndpointAction{Handler: internalShutdown, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } // Internal managemnt traffic. var internalImageOptimizeCmd = APIEndpoint{ Path: "image-optimize", Post: APIEndpointAction{Handler: internalOptimizeImage, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var internalRebalanceLoadCmd = APIEndpoint{ Path: "rebalance", Get: APIEndpointAction{Handler: internalRebalanceLoad, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var internalSQLCmd = APIEndpoint{ Path: "sql", Get: APIEndpointAction{Handler: internalSQLGet, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Post: APIEndpointAction{Handler: internalSQLPost, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } // Internal cluster traffic. var internalClusterAcceptCmd = APIEndpoint{ Path: "cluster/accept", Post: APIEndpointAction{Handler: internalClusterPostAccept, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var internalClusterAssignCmd = APIEndpoint{ Path: "cluster/assign", Post: APIEndpointAction{Handler: internalClusterPostAssign, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var internalClusterHandoverCmd = APIEndpoint{ Path: "cluster/handover", Post: APIEndpointAction{Handler: internalClusterPostHandover, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var internalClusterRaftNodeCmd = APIEndpoint{ Path: "cluster/raft-node/{address}", Delete: APIEndpointAction{Handler: internalClusterRaftNodeDelete, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var internalClusterRebalanceCmd = APIEndpoint{ Path: "cluster/rebalance", Post: APIEndpointAction{Handler: internalClusterPostRebalance, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } // Container hooks. var internalContainerOnStartCmd = APIEndpoint{ Path: "containers/{instanceRef}/onstart", Get: APIEndpointAction{Handler: internalContainerOnStart, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var internalContainerOnStopNSCmd = APIEndpoint{ Path: "containers/{instanceRef}/onstopns", Get: APIEndpointAction{Handler: internalContainerOnStopNS, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var internalContainerOnStopCmd = APIEndpoint{ Path: "containers/{instanceRef}/onstop", Get: APIEndpointAction{Handler: internalContainerOnStop, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } // Virtual machine hooks. var internalVirtualMachineOnResizeCmd = APIEndpoint{ Path: "virtual-machines/{instanceRef}/onresize", Get: APIEndpointAction{Handler: internalVirtualMachineOnResize, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } // Debugging. var internalBGPStateCmd = APIEndpoint{ Path: "debug/bgp", Get: APIEndpointAction{Handler: internalBGPState, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var internalGarbageCollectorCmd = APIEndpoint{ Path: "debug/gc", Get: APIEndpointAction{Handler: internalGC, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var internalImageRefreshCmd = APIEndpoint{ Path: "debug/image-refresh", Get: APIEndpointAction{Handler: internalRefreshImage, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var internalRAFTSnapshotCmd = APIEndpoint{ Path: "debug/raft-snapshot", Get: APIEndpointAction{Handler: internalRAFTSnapshot, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var internalWarningCreateCmd = APIEndpoint{ Path: "debug/warnings", Post: APIEndpointAction{Handler: internalCreateWarning, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } type internalImageOptimizePost struct { Image api.Image `json:"image" yaml:"image"` Pool string `json:"pool" yaml:"pool"` } type internalWarningCreatePost struct { Location string `json:"location" yaml:"location"` Project string `json:"project" yaml:"project"` EntityTypeCode int `json:"entity_type_code" yaml:"entity_type_code"` EntityID int `json:"entity_id" yaml:"entity_id"` TypeCode int `json:"type_code" yaml:"type_code"` Message string `json:"message" yaml:"message"` } // internalCreateWarning creates a warning, and is used for testing only. func internalCreateWarning(d *Daemon, r *http.Request) response.Response { body, err := io.ReadAll(r.Body) if err != nil { return response.InternalError(err) } rdr1 := io.NopCloser(bytes.NewBuffer(body)) rdr2 := io.NopCloser(bytes.NewBuffer(body)) reqRaw := jmap.Map{} err = json.NewDecoder(rdr1).Decode(&reqRaw) if err != nil { return response.BadRequest(err) } req := internalWarningCreatePost{} err = json.NewDecoder(rdr2).Decode(&req) if err != nil { return response.BadRequest(err) } req.EntityTypeCode, _ = reqRaw.GetInt("entity_type_code") req.EntityID, _ = reqRaw.GetInt("entity_id") // Check if the entity exists, and fail if it doesn't. _, ok := cluster.EntityNames[req.EntityTypeCode] if req.EntityTypeCode != -1 && !ok { return response.SmartError(errors.New("Invalid entity type")) } err = d.State().DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpsertWarning(ctx, req.Location, req.Project, req.EntityTypeCode, req.EntityID, warningtype.Type(req.TypeCode), req.Message) }) if err != nil { return response.SmartError(fmt.Errorf("Failed to create warning: %w", err)) } return response.EmptySyncResponse } func internalOptimizeImage(d *Daemon, r *http.Request) response.Response { s := d.State() req := &internalImageOptimizePost{} // Parse the request. err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } err = imageCreateInPool(s, &req.Image, req.Pool) if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } func internalRefreshImage(d *Daemon, _ *http.Request) response.Response { s := d.State() err := autoUpdateImages(s.ShutdownCtx, s) if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } func internalWaitReady(d *Daemon, _ *http.Request) response.Response { // Check that we're not shutting down. isClosing := d.State().ShutdownCtx.Err() != nil if isClosing { return response.Unavailable(errors.New("Daemon is shutting down")) } if d.waitReady.Err() == nil { return response.Unavailable(errors.New("Daemon not ready yet")) } return response.EmptySyncResponse } func internalShutdown(d *Daemon, r *http.Request) response.Response { force := request.QueryParam(r, "force") logger.Info("Asked to shutdown by API", logger.Ctx{"force": force}) if d.State().ShutdownCtx.Err() != nil { return response.SmartError(api.StatusErrorf(http.StatusTooManyRequests, "Shutdown already in progress")) } forceCtx, forceCtxCancel := context.WithCancel(context.Background()) if force == "true" { forceCtxCancel() // Don't wait for operations to finish. } return response.ManualResponse(func(w http.ResponseWriter) error { defer forceCtxCancel() <-d.setupChan // Wait for daemon to start. // Run shutdown sequence synchronously. stopErr := d.Stop(forceCtx, unix.SIGPWR) err := response.SmartError(stopErr).Render(w) if err != nil { return err } // Send the response before the daemon process ends. f, ok := w.(http.Flusher) if ok { f.Flush() } else { return errors.New("http.ResponseWriter is not type http.Flusher") } // Send result of d.Stop() to cmdDaemon so that process stops with correct exit code from Stop(). go func() { <-r.Context().Done() // Wait until request is finished. d.shutdownDoneCh <- stopErr }() return nil }) } // internalContainerHookLoadFromRequestReference loads the container from the instance reference in the request. // It detects whether the instance reference is an instance ID or instance name and loads instance accordingly. func internalContainerHookLoadFromReference(s *state.State, r *http.Request) (instance.Instance, error) { var inst instance.Instance instanceRef, err := url.PathUnescape(mux.Vars(r)["instanceRef"]) if err != nil { return nil, err } projectName := request.ProjectParam(r) instanceID, err := strconv.Atoi(instanceRef) if err == nil { inst, err = instance.LoadByID(s, instanceID) if err != nil { return nil, err } } else { inst, err = instance.LoadByProjectAndName(s, projectName, instanceRef) if err != nil { if api.StatusErrorCheck(err, http.StatusNotFound) { return nil, err } // If DB not available, try loading from backup file. logger.Warn("Failed loading instance from database, trying backup file", logger.Ctx{"project": projectName, "instance": instanceRef, "err": err}) instancePath := filepath.Join(internalUtil.VarPath("containers"), project.Instance(projectName, instanceRef)) inst, err = instance.LoadFromBackup(s, projectName, instancePath, false) if err != nil { return nil, err } } } if inst.Type() != instancetype.Container { return nil, errors.New("Instance is not container type") } return inst, nil } func internalContainerOnStart(d *Daemon, r *http.Request) response.Response { s := d.State() inst, err := internalContainerHookLoadFromReference(s, r) if err != nil { logger.Error("The start hook failed to load", logger.Ctx{"err": err}) return response.SmartError(err) } err = inst.OnHook(instance.HookStart, nil) if err != nil { logger.Error("The start hook failed", logger.Ctx{"instance": inst.Name(), "err": err}) return response.SmartError(err) } return response.EmptySyncResponse } func internalContainerOnStopNS(d *Daemon, r *http.Request) response.Response { // Wait until daemon is fully started. <-d.waitReady.Done() s := d.State() inst, err := internalContainerHookLoadFromReference(s, r) if err != nil { logger.Error("The stopns hook failed to load", logger.Ctx{"err": err}) return response.SmartError(err) } target := request.QueryParam(r, "target") if target == "" { target = "unknown" } netns := request.QueryParam(r, "netns") args := map[string]string{ "target": target, "netns": netns, } err = inst.OnHook(instance.HookStopNS, args) if err != nil { logger.Error("The stopns hook failed", logger.Ctx{"instance": inst.Name(), "err": err}) return response.SmartError(err) } return response.EmptySyncResponse } func internalContainerOnStop(d *Daemon, r *http.Request) response.Response { // Wait until daemon is fully started. <-d.waitReady.Done() s := d.State() inst, err := internalContainerHookLoadFromReference(s, r) if err != nil { logger.Error("The stop hook failed to load", logger.Ctx{"err": err}) return response.SmartError(err) } target := request.QueryParam(r, "target") if target == "" { target = "unknown" } args := map[string]string{ "target": target, } err = inst.OnHook(instance.HookStop, args) if err != nil { logger.Error("The stop hook failed", logger.Ctx{"instance": inst.Name(), "err": err}) return response.SmartError(err) } return response.EmptySyncResponse } func internalVirtualMachineOnResize(d *Daemon, r *http.Request) response.Response { // Wait until daemon is fully started. <-d.waitReady.Done() s := d.State() // Get the instance ID. instanceID, err := strconv.Atoi(mux.Vars(r)["instanceRef"]) if err != nil { return response.BadRequest(err) } // Get the devices list. devices := request.QueryParam(r, "devices") if devices == "" { return response.BadRequest(errors.New("Resize hook requires a list of devices")) } // Load by ID. inst, err := instance.LoadByID(s, instanceID) if err != nil { return response.SmartError(err) } // Update the local instance. for _, dev := range strings.Split(devices, ",") { fields := strings.SplitN(dev, ":", 2) if len(fields) != 2 { return response.BadRequest(fmt.Errorf("Invalid device/size tuple: %s", dev)) } size, err := strconv.ParseInt(fields[1], 16, 64) if err != nil { return response.BadRequest(err) } runConf := deviceConfig.RunConfig{} runConf.Mounts = []deviceConfig.MountEntryItem{ { DevName: fields[0], Size: size, }, } err = inst.DeviceEventHandler(&runConf) if err != nil { return response.InternalError(err) } } return response.EmptySyncResponse } // Perform a database dump. func internalSQLGet(d *Daemon, r *http.Request) response.Response { s := d.State() database := r.FormValue("database") if !slices.Contains([]string{"local", "global"}, database) { return response.BadRequest(errors.New("Invalid database")) } dumpFormValue := r.FormValue("dump") dumpInt, err := strconv.Atoi(dumpFormValue) if err != nil { dumpInt = 0 } dumpOption := query.DumpOptions(dumpInt) var db *sql.DB if database == "global" { db = s.DB.Cluster.DB() } else { db = s.DB.Node.DB() } tx, err := db.BeginTx(r.Context(), nil) if err != nil { return response.SmartError(fmt.Errorf("Failed to start transaction: %w", err)) } defer func() { _ = tx.Rollback() }() dump, err := query.Dump(r.Context(), tx, dumpOption) if err != nil { return response.SmartError(fmt.Errorf("Failed dump database %s: %w", database, err)) } return response.SyncResponse(true, internalSQL.SQLDump{Text: dump}) } // Execute queries. func internalSQLPost(d *Daemon, r *http.Request) response.Response { s := d.State() req := &internalSQL.SQLQuery{} // Parse the request. err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } if !slices.Contains([]string{"local", "global"}, req.Database) { return response.BadRequest(errors.New("Invalid database")) } if req.Query == "" { return response.BadRequest(errors.New("No query provided")) } var db *sql.DB if req.Database == "global" { db = s.DB.Cluster.DB() } else { db = s.DB.Node.DB() } batch := internalSQL.SQLBatch{} if req.Query == ".sync" { d.gateway.Sync() return response.SyncResponse(true, batch) } for _, query := range strings.Split(req.Query, ";") { query = strings.TrimLeft(query, " ") if query == "" { continue } result := internalSQL.SQLResult{} tx, err := db.Begin() if err != nil { return response.SmartError(err) } if strings.HasPrefix(strings.ToUpper(query), "SELECT") { err = internalSQLSelect(tx, query, &result) _ = tx.Rollback() } else { err = internalSQLExec(tx, query, &result) if err != nil { _ = tx.Rollback() } else { err = tx.Commit() } } if err != nil { return response.SmartError(err) } batch.Results = append(batch.Results, result) } return response.SyncResponse(true, batch) } func internalSQLSelect(tx *sql.Tx, query string, result *internalSQL.SQLResult) error { result.Type = "select" rows, err := tx.Query(query) if err != nil { return fmt.Errorf("Failed to execute query: %w", err) } defer func() { _ = rows.Close() }() result.Columns, err = rows.Columns() if err != nil { return fmt.Errorf("Failed to fetch column names: %w", err) } for rows.Next() { row := make([]any, len(result.Columns)) rowPointers := make([]any, len(result.Columns)) for i := range row { rowPointers[i] = &row[i] } err := rows.Scan(rowPointers...) if err != nil { return fmt.Errorf("Failed to scan row: %w", err) } for i, column := range row { // Convert bytes to string. This is safe as // long as we don't have any BLOB column type. data, ok := column.([]byte) if ok { row[i] = string(data) } } result.Rows = append(result.Rows, row) } err = rows.Err() if err != nil { return fmt.Errorf("Got a row error: %w", err) } return nil } func internalSQLExec(tx *sql.Tx, query string, result *internalSQL.SQLResult) error { result.Type = "exec" r, err := tx.Exec(query) if err != nil { return fmt.Errorf("Failed to exec query: %w", err) } result.RowsAffected, err = r.RowsAffected() if err != nil { return fmt.Errorf("Failed to fetch affected rows: %w", err) } return nil } // internalImportFromBackup creates instance, storage pool and volume DB records from an instance's backup file. // It expects the instance volume to be mounted so that the backup.yaml file is readable. func internalImportFromBackup(ctx context.Context, s *state.State, projectName string, instName string, allowNameOverride bool, deviceMap map[string]map[string]string, configMap map[string]string) error { if instName == "" { return errors.New("The name of the instance is required") } storagePoolsPath := internalUtil.VarPath("storage-pools") storagePoolsDir, err := os.Open(storagePoolsPath) if err != nil { return err } // Get a list of all storage pools. storagePoolNames, err := storagePoolsDir.Readdirnames(-1) if err != nil { _ = storagePoolsDir.Close() return err } _ = storagePoolsDir.Close() // Check whether the instance exists on any of the storage pools as either a container or a VM. instanceMountPoints := []string{} instancePoolName := "" instanceType := instancetype.Container instanceVolType := storageDrivers.VolumeTypeContainer instanceDBVolType := db.StoragePoolVolumeTypeContainer for _, volType := range []storageDrivers.VolumeType{storageDrivers.VolumeTypeVM, storageDrivers.VolumeTypeContainer} { for _, poolName := range storagePoolNames { volStorageName := project.Instance(projectName, instName) instanceMntPoint := storageDrivers.GetVolumeMountPath(poolName, volType, volStorageName) if util.PathExists(instanceMntPoint) { instanceMountPoints = append(instanceMountPoints, instanceMntPoint) instancePoolName = poolName instanceVolType = volType if volType == storageDrivers.VolumeTypeVM { instanceType = instancetype.VM instanceDBVolType = db.StoragePoolVolumeTypeVM } else { instanceType = instancetype.Container instanceDBVolType = db.StoragePoolVolumeTypeContainer } } } } // Quick checks. if len(instanceMountPoints) > 1 { return fmt.Errorf(`The instance %q seems to exist on multiple storage pools`, instName) } else if len(instanceMountPoints) != 1 { return fmt.Errorf(`The instance %q does not seem to exist on any storage pool`, instName) } // User needs to make sure that we can access the directory where backup.yaml lives. instanceMountPoint := instanceMountPoints[0] isEmpty, err := internalUtil.PathIsEmpty(instanceMountPoint) if err != nil { return err } if isEmpty { return fmt.Errorf(`The instance's directory %q appears to be empty. Please ensure that the instance's storage volume is mounted`, instanceMountPoint) } // Read in the backup.yaml file. backupYamlPath := filepath.Join(instanceMountPoint, "backup.yaml") backupConf, err := backup.ParseConfigYamlFile(backupYamlPath) if err != nil { return err } if backupConf.Container == nil { return errors.New("No instance configuration found in backup file.") } if allowNameOverride { backupConf.Container.Name = instName } if instName != backupConf.Container.Name { return fmt.Errorf("Instance name requested %q doesn't match instance name in backup config %q", instName, backupConf.Container.Name) } if backupConf.Pool == nil { // We don't know what kind of storage type the pool is. return errors.New("No storage pool struct in the backup file found. The storage pool needs to be recovered manually") } // Try to retrieve the storage pool the instance supposedly lives on. pool, err := storagePools.LoadByName(s, instancePoolName) if response.IsNotFoundError(err) { // Create the storage pool db entry if it doesn't exist. _, err = storagePoolDBCreate(ctx, s, instancePoolName, "", backupConf.Pool.Driver, backupConf.Pool.Config) if err != nil { return fmt.Errorf("Create storage pool database entry: %w", err) } pool, err = storagePools.LoadByName(s, instancePoolName) if err != nil { return fmt.Errorf("Load storage pool database entry: %w", err) } } else if err != nil { return fmt.Errorf("Find storage pool database entry: %w", err) } if backupConf.Pool.Name != instancePoolName { return fmt.Errorf(`The storage pool %q the instance was detected on does not match the storage pool %q specified in the backup file`, instancePoolName, backupConf.Pool.Name) } if backupConf.Pool.Driver != pool.Driver().Info().Name { return fmt.Errorf(`The storage pool's %q driver %q conflicts with the driver %q recorded in the instance's backup file`, instancePoolName, pool.Driver().Info().Name, backupConf.Pool.Driver) } // Check snapshots are consistent. existingSnapshots, err := pool.CheckInstanceBackupFileSnapshots(backupConf, projectName, false, nil) if err != nil { return fmt.Errorf("Failed checking snapshots: %w", err) } // Check if a storage volume entry for the instance already exists. var dbVolume *db.StorageVolume err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, pool.ID(), projectName, instanceDBVolType, backupConf.Container.Name, true) if err != nil && !response.IsNotFoundError(err) { return err } return nil }) if err != nil { return err } if dbVolume != nil { return fmt.Errorf(`Storage volume for instance %q already exists in the database`, backupConf.Container.Name) } err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Check if an entry for the instance already exists in the db. _, err := tx.GetInstanceID(ctx, projectName, backupConf.Container.Name) return err }) if err != nil && !response.IsNotFoundError(err) { return err } if err == nil { return fmt.Errorf(`Entry for instance %q already exists in the database`, backupConf.Container.Name) } if backupConf.Volume == nil { return errors.New(`No storage volume struct in the backup file found. The storage volume needs to be recovered manually`) } var profiles []api.Profile err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { profiles, err = tx.GetProfiles(ctx, projectName, backupConf.Container.Profiles) return err }) if err != nil { return fmt.Errorf("Failed loading profiles (%v) for instance: %w", strings.Join(backupConf.Container.Profiles, ", "), err) } // Initialize configuration if missing. if backupConf.Container.Config == nil { backupConf.Container.Config = map[string]string{} } if backupConf.Container.Devices == nil { backupConf.Container.Devices = make(map[string]map[string]string) } if backupConf.Container.ExpandedDevices == nil { backupConf.Container.ExpandedDevices = make(map[string]map[string]string) } // Add root device if needed. internalImportRootDevicePopulate(instancePoolName, backupConf.Container.Devices, backupConf.Container.ExpandedDevices, profiles) // Override device. for k, m := range deviceMap { for key, value := range m { if backupConf.Container.Devices[k] == nil { backupConf.Container.Devices[k] = map[string]string{} } if backupConf.Container.ExpandedDevices[k] == nil { backupConf.Container.ExpandedDevices[k] = map[string]string{} } backupConf.Container.Devices[k][key] = value backupConf.Container.ExpandedDevices[k][key] = value } } // Override config. for key, value := range configMap { backupConf.Container.Config[key] = value backupConf.Container.ExpandedConfig[key] = value } reverter := revert.New() defer reverter.Fail() if backupConf.Container == nil { return errors.New("No instance config in backup config") } instDBArgs, err := backup.ConfigToInstanceDBArgs(s, backupConf, projectName, true) if err != nil { return err } _, instOp, cleanup, err := instance.CreateInternal(s, *instDBArgs, nil, true, true, false) if err != nil { return fmt.Errorf("Failed creating instance record: %w", err) } reverter.Add(cleanup) defer instOp.Done(err) instancePath := storagePools.InstancePath(instanceType, projectName, backupConf.Container.Name, false) isPrivileged := false if backupConf.Container.Config["security.privileged"] == "" { isPrivileged = true } err = storagePools.CreateContainerMountpoint(instanceMountPoint, instancePath, isPrivileged) if err != nil { return err } for _, snap := range existingSnapshots { snapInstName := fmt.Sprintf("%s%s%s", backupConf.Container.Name, internalInstance.SnapshotDelimiter, snap.Name) snapErr := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Check if an entry for the snapshot already exists in the db. _, err := tx.GetInstanceSnapshotID(ctx, projectName, backupConf.Container.Name, snap.Name) return err }) if snapErr != nil && !response.IsNotFoundError(snapErr) { return snapErr } if snapErr == nil { return fmt.Errorf(`Entry for snapshot %q already exists in the database`, snapInstName) } // Check if a storage volume entry for the snapshot already exists. var dbVolume *db.StorageVolume err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, pool.ID(), projectName, instanceDBVolType, snapInstName, true) if err != nil && !response.IsNotFoundError(err) { return err } return nil }) if err != nil { return err } // If the storage volume entry does already exist we error here if dbVolume != nil { return fmt.Errorf(`Storage volume for snapshot %q already exists in the database`, snapInstName) } baseImage := snap.Config["volatile.base_image"] arch, err := osarch.ArchitectureID(snap.Architecture) if err != nil { return err } err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { profiles, err = tx.GetProfiles(ctx, projectName, snap.Profiles) return err }) if err != nil { return fmt.Errorf("Failed loading profiles (%v) for instance snapshot %q: %w", strings.Join(snap.Profiles, ", "), snapInstName, err) } // Add root device if needed. if snap.Devices == nil { snap.Devices = make(map[string]map[string]string) } if snap.ExpandedDevices == nil { snap.ExpandedDevices = make(map[string]map[string]string) } internalImportRootDevicePopulate(instancePoolName, snap.Devices, snap.ExpandedDevices, profiles) _, snapInstOp, cleanup, err := instance.CreateInternal(s, db.InstanceArgs{ Project: projectName, Architecture: arch, BaseImage: baseImage, Config: snap.Config, Description: snap.Description, CreationDate: snap.CreatedAt, Type: instanceType, Snapshot: true, Devices: deviceConfig.NewDevices(snap.Devices), Ephemeral: snap.Ephemeral, LastUsedDate: snap.LastUsedAt, Name: snapInstName, Profiles: profiles, Stateful: snap.Stateful, }, nil, true, true, false) if err != nil { return fmt.Errorf("Failed creating instance snapshot record %q: %w", snap.Name, err) } reverter.Add(cleanup) defer snapInstOp.Done(err) // Recreate missing mountpoints and symlinks. volStorageName := project.Instance(projectName, snapInstName) snapshotMountPoint := storageDrivers.GetVolumeMountPath(instancePoolName, instanceVolType, volStorageName) snapshotPath := storagePools.InstancePath(instanceType, projectName, backupConf.Container.Name, true) snapshotTargetPath := storageDrivers.GetVolumeSnapshotDir(instancePoolName, instanceVolType, volStorageName) err = storagePools.CreateSnapshotMountpoint(snapshotMountPoint, snapshotTargetPath, snapshotPath) if err != nil { return err } } reverter.Success() return nil } // internalImportRootDevicePopulate considers the local and expanded devices from backup.yaml as well as the // expanded devices in the current profiles and if needed will populate localDevices with a new root disk config // to attempt to maintain the same effective config as specified in backup.yaml. Where possible no new root disk // device will be added, if the root disk config in the current profiles matches the effective backup.yaml config. func internalImportRootDevicePopulate(instancePoolName string, localDevices map[string]map[string]string, expandedDevices map[string]map[string]string, profiles []api.Profile) { // First, check if localDevices from backup.yaml has a root disk. rootName, _, _ := internalInstance.GetRootDiskDevice(localDevices) if rootName != "" { localDevices[rootName]["pool"] = instancePoolName return // Local root disk device has been set to target pool. } // Next check if expandedDevices from backup.yaml has a root disk. expandedRootName, expandedRootConfig, _ := internalInstance.GetRootDiskDevice(expandedDevices) // Extract root disk from expanded profile devices. profileExpandedDevices := db.ExpandInstanceDevices(deviceConfig.NewDevices(localDevices), profiles) profileExpandedRootName, profileExpandedRootConfig, _ := internalInstance.GetRootDiskDevice(profileExpandedDevices.CloneNative()) // Record whether we need to add a new local disk device. addLocalDisk := false // We need to add a local root disk if the profiles don't have a root disk. if profileExpandedRootName == "" { addLocalDisk = true } else { // Check profile expanded root disk is in the correct pool if profileExpandedRootConfig["pool"] != instancePoolName { addLocalDisk = true } else { // Check profile expanded root disk config matches the old expanded disk in backup.yaml. // Excluding the "pool" property, which we ignore, as we have already checked the new // profile root disk matches the target pool name. if expandedRootName != "" { for k := range expandedRootConfig { if k == "pool" { continue // Ignore old pool name. } if expandedRootConfig[k] != profileExpandedRootConfig[k] { addLocalDisk = true break } } for k := range profileExpandedRootConfig { if k == "pool" { continue // Ignore old pool name. } if expandedRootConfig[k] != profileExpandedRootConfig[k] { addLocalDisk = true break } } } } } // Add local root disk entry if needed. if addLocalDisk { rootDev := map[string]string{ "type": "disk", "path": "/", "pool": instancePoolName, } // Inherit any extra root disk config from the expanded root disk from backup.yaml. if expandedRootName != "" { for k, v := range expandedRootConfig { _, found := rootDev[k] if !found { rootDev[k] = v } } } // If there is already a device called "root" in the instance's config, but it does not qualify as // a root disk, then try to find a free name for the new root disk device. rootDevName := "root" for i := range 100 { if localDevices[rootDevName] == nil { break } rootDevName = fmt.Sprintf("root%d", i) continue } localDevices[rootDevName] = rootDev } } func internalGC(_ *Daemon, _ *http.Request) response.Response { logger.Infof("Started forced garbage collection run") runtime.GC() runtimeDebug.FreeOSMemory() var m runtime.MemStats runtime.ReadMemStats(&m) logger.Infof("Heap allocated: %s", units.GetByteSizeStringIEC(int64(m.Alloc), 2)) logger.Infof("Stack in use: %s", units.GetByteSizeStringIEC(int64(m.StackInuse), 2)) logger.Infof("Requested from system: %s", units.GetByteSizeStringIEC(int64(m.Sys), 2)) logger.Infof("Releasable to OS: %s", units.GetByteSizeStringIEC(int64(m.HeapIdle-m.HeapReleased), 2)) logger.Infof("Completed forced garbage collection run") return response.EmptySyncResponse } func internalRAFTSnapshot(_ *Daemon, _ *http.Request) response.Response { logger.Warn("Forced RAFT snapshot not supported") return response.InternalError(errors.New("Not supported")) } func internalBGPState(d *Daemon, _ *http.Request) response.Response { s := d.State() return response.SyncResponse(true, s.BGP.Debug()) } func internalRebalanceLoad(d *Daemon, _ *http.Request) response.Response { err := autoRebalanceCluster(context.TODO(), d) if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } incus-7.0.0/cmd/incusd/api_internal_recover.go000066400000000000000000000537141517523235500214320ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "net/http" "slices" internalInstance "github.com/lxc/incus/v7/internal/instance" internalRecover "github.com/lxc/incus/v7/internal/recover" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/backup" backupConfig "github.com/lxc/incus/v7/internal/server/backup/config" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/revert" ) // Define API endpoints for recover actions. var internalRecoverValidateCmd = APIEndpoint{ Path: "recover/validate", Post: APIEndpointAction{Handler: internalRecoverValidate, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var internalRecoverImportCmd = APIEndpoint{ Path: "recover/import", Post: APIEndpointAction{Handler: internalRecoverImport, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } // init recover adds API endpoints to handler slice. func init() { apiInternal = append(apiInternal, internalRecoverValidateCmd, internalRecoverImportCmd) } // internalRecoverScan provides the discovery and import functionality for both recovery validate and import steps. func internalRecoverScan(ctx context.Context, s *state.State, userPools []api.StoragePoolsPost, validateOnly bool) response.Response { var err error var projects map[string]*api.Project var projectProfiles map[string][]*api.Profile var projectNetworks map[string]map[int64]api.Network // Retrieve all project, profile and network info in a single transaction so we can use it for all // imported instances and volumes, and avoid repeatedly querying the same information. err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Load list of projects for validation. ps, err := dbCluster.GetProjects(ctx, tx.Tx()) if err != nil { return err } // Convert to map for lookups by name later. projects = make(map[string]*api.Project, len(ps)) for i := range ps { project, err := ps[i].ToAPI(ctx, tx.Tx()) if err != nil { return err } projects[ps[i].Name] = project } // Load list of project/profile names for validation. profiles, err := dbCluster.GetProfiles(ctx, tx.Tx()) if err != nil { return err } profileConfigs, err := dbCluster.GetAllProfileConfigs(ctx, tx.Tx()) if err != nil { return err } profileDevices, err := dbCluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return err } // Convert to map for lookups by project name later. projectProfiles = make(map[string][]*api.Profile) for _, profile := range profiles { if projectProfiles[profile.Project] == nil { projectProfiles[profile.Project] = []*api.Profile{} } apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices) if err != nil { return err } projectProfiles[profile.Project] = append(projectProfiles[profile.Project], apiProfile) } // Load list of project/network names for validation. projectNetworks, err = tx.GetCreatedNetworks(ctx) if err != nil { return err } return nil }) if err != nil { return response.SmartError(fmt.Errorf("Failed getting validate dependency check info: %w", err)) } res := internalRecover.ValidateResult{} reverter := revert.New() defer reverter.Fail() // addDependencyError adds an error to the list of dependency errors if not already present in list. addDependencyError := func(err error) { errStr := err.Error() if !slices.Contains(res.DependencyErrors, errStr) { res.DependencyErrors = append(res.DependencyErrors, errStr) } } // Used to store the unknown volumes for each pool & project. poolsProjectVols := make(map[string]map[string][]*backupConfig.Config) // Used to store a handle to each pool containing user supplied config. pools := make(map[string]storagePools.Pool) // Iterate the pools finding unknown volumes and perform validation. for _, p := range userPools { pool, err := storagePools.LoadByName(s, p.Name) if err != nil { if response.IsNotFoundError(err) { // If the pool DB record doesn't exist, and we are clustered, then don't proceed // any further as we do not support pool DB record recovery when clustered. if s.ServerClustered { return response.BadRequest(errors.New("Storage pool recovery not supported when clustered")) } // If pool doesn't exist in DB, initialize a temporary pool with the supplied info. poolInfo := api.StoragePool{ Name: p.Name, Driver: p.Driver, StoragePoolPut: p.StoragePoolPut, Status: api.StoragePoolStatusCreated, } pool, err = storagePools.NewTemporary(s, &poolInfo) if err != nil { return response.SmartError(fmt.Errorf("Failed to initialize unknown pool %q: %w", p.Name, err)) } // Populate configuration with default values. err := pool.Driver().FillConfig() if err != nil { return response.SmartError(fmt.Errorf("Failed to evaluate the default configuration values for unknown pool %q: %w", p.Name, err)) } err = pool.Driver().Validate(poolInfo.Config) if err != nil { return response.SmartError(fmt.Errorf("Failed config validation for unknown pool %q: %w", p.Name, err)) } } else { return response.SmartError(fmt.Errorf("Failed loading existing pool %q: %w", p.Name, err)) } } // Record this pool to be used during import stage, assuming validation passes. pools[p.Name] = pool // Try to mount the pool. ourMount, err := pool.Mount() if err != nil { return response.SmartError(fmt.Errorf("Failed mounting pool %q: %w", pool.Name(), err)) } // Unmount pool when done if not existing in DB after function has finished. // This way if we are dealing with an existing pool or have successfully created the DB record then // we won't unmount it. As we should leave successfully imported pools mounted. if ourMount { defer func() { cleanupPool := pools[pool.Name()] if cleanupPool != nil && cleanupPool.ID() == storagePools.PoolIDTemporary { _, _ = cleanupPool.Unmount() } }() reverter.Add(func() { cleanupPool := pools[pool.Name()] _, _ = cleanupPool.Unmount() // Defer won't do it if record exists, so unmount on failure. }) } // Get list of unknown volumes on pool. poolProjectVols, err := pool.ListUnknownVolumes(nil) if err != nil { if errors.Is(err, storageDrivers.ErrNotSupported) { continue // Ignore unsupported storage drivers. } return response.SmartError(fmt.Errorf("Failed checking volumes on pool %q: %w", pool.Name(), err)) } // Store for consumption after validation scan to avoid needing to reprocess. poolsProjectVols[p.Name] = poolProjectVols // Check dependencies are met for each volume. for projectName, poolVols := range poolProjectVols { // Check project exists in database. projectInfo := projects[projectName] // Look up effective project names for profiles and networks. var profileProjectname string var networkProjectName string if projectInfo != nil { profileProjectname = project.ProfileProjectFromRecord(projectInfo) networkProjectName = project.NetworkProjectFromRecord(projectInfo) } else { addDependencyError(fmt.Errorf("Project %q", projectName)) continue // Skip further validation if project is missing. } for _, poolVol := range poolVols { if poolVol.Container == nil { continue // Skip dependency checks for non-instance volumes. } // Check that the instance's profile dependencies are met. for _, poolInstProfileName := range poolVol.Container.Profiles { foundProfile := false for _, profile := range projectProfiles[profileProjectname] { if profile.Name == poolInstProfileName { foundProfile = true } } if !foundProfile { addDependencyError(fmt.Errorf("Profile %q in project %q", poolInstProfileName, projectName)) } } // Check that the instance's NIC network dependencies are met. for _, devConfig := range poolVol.Container.ExpandedDevices { if devConfig["type"] != "nic" { continue } if devConfig["network"] == "" { continue } foundNetwork := false for _, n := range projectNetworks[networkProjectName] { if n.Name == devConfig["network"] { foundNetwork = true break } } if !foundNetwork { addDependencyError(fmt.Errorf("Network %q in project %q", devConfig["network"], projectName)) } } } } } // If in validation mode or if there are dependency errors, return discovered unknown volumes, along with // any dependency errors. if validateOnly || len(res.DependencyErrors) > 0 { for poolName, poolProjectVols := range poolsProjectVols { for projectName, poolVols := range poolProjectVols { for _, poolVol := range poolVols { var displayType, displayName string var displaySnapshotCount int // Build display fields for scan results. if poolVol.Container != nil { displayType = poolVol.Container.Type displayName = poolVol.Container.Name displaySnapshotCount = len(poolVol.Snapshots) } else if poolVol.Bucket != nil { displayType = "bucket" displayName = poolVol.Bucket.Name displaySnapshotCount = 0 } else { displayType = "volume" displayName = poolVol.Volume.Name displaySnapshotCount = len(poolVol.VolumeSnapshots) } res.UnknownVolumes = append(res.UnknownVolumes, internalRecover.ValidateVolume{ Pool: poolName, Project: projectName, Type: displayType, Name: displayName, SnapshotCount: displaySnapshotCount, }) } } } return response.SyncResponse(true, &res) } // If in import mode and no dependency errors, then re-create missing DB records. // Create the pools themselves. for _, pool := range pools { // Create missing storage pool DB record if needed. if pool.ID() == storagePools.PoolIDTemporary { var instPoolVol *backupConfig.Config // Instance volume used for new pool record. var poolID int64 // Pool ID of created pool record. var poolVols []*backupConfig.Config for _, value := range poolsProjectVols[pool.Name()] { poolVols = append(poolVols, value...) } // Search unknown volumes looking for an instance volume that can be used to // restore the pool DB config from. This is preferable over using the user // supplied config as it will include any additional settings not supplied. for _, poolVol := range poolVols { if poolVol.Pool != nil && poolVol.Pool.Config != nil { instPoolVol = poolVol break // Stop search once we've found an instance with pool config. } } if instPoolVol != nil { // Create storage pool DB record from config in the instance. logger.Info("Creating storage pool DB record from instance config", logger.Ctx{"name": instPoolVol.Pool.Name, "description": instPoolVol.Pool.Description, "driver": instPoolVol.Pool.Driver, "config": instPoolVol.Pool.Config}) poolID, err = dbStoragePoolCreateAndUpdateCache(ctx, s, instPoolVol.Pool.Name, instPoolVol.Pool.Description, instPoolVol.Pool.Driver, instPoolVol.Pool.Config) if err != nil { return response.SmartError(fmt.Errorf("Failed creating storage pool %q database entry: %w", pool.Name(), err)) } } else { // Create storage pool DB record from config supplied by user if not // instance volume pool config found. poolDriverName := pool.Driver().Info().Name poolDriverConfig := pool.Driver().Config() logger.Info("Creating storage pool DB record from user config", logger.Ctx{"name": pool.Name(), "driver": poolDriverName, "config": poolDriverConfig}) poolID, err = dbStoragePoolCreateAndUpdateCache(ctx, s, pool.Name(), "", poolDriverName, poolDriverConfig) if err != nil { return response.SmartError(fmt.Errorf("Failed creating storage pool %q database entry: %w", pool.Name(), err)) } } reverter.Add(func() { _ = dbStoragePoolDeleteAndUpdateCache(context.Background(), s, pool.Name()) }) // Set storage pool node to storagePoolCreated. // Must come before storage pool is loaded from the database. err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.StoragePoolNodeCreated(poolID) }) if err != nil { return response.SmartError(fmt.Errorf("Failed marking storage pool %q local status as created: %w", pool.Name(), err)) } logger.Debug("Marked storage pool local status as created", logger.Ctx{"pool": pool.Name()}) newPool, err := storagePools.LoadByName(s, pool.Name()) if err != nil { return response.SmartError(fmt.Errorf("Failed loading created storage pool %q: %w", pool.Name(), err)) } // Record this newly created pool so that defer doesn't unmount on return. pools[pool.Name()] = newPool } } // Recover the storage volumes and buckets. for _, pool := range pools { for projectName, poolVols := range poolsProjectVols[pool.Name()] { projectInfo := projects[projectName] if projectInfo == nil { // Shouldn't happen as we validated this above, but be sure for safety. return response.SmartError(fmt.Errorf("Project %q not found", projectName)) } customStorageProjectName := project.StorageVolumeProjectFromRecord(projectInfo, db.StoragePoolVolumeTypeCustom) // Recover unknown custom volumes (do this first before recovering instances so that any // instances that reference unknown custom volume disk devices can be created). for _, poolVol := range poolVols { if poolVol.Container != nil || poolVol.Bucket != nil { continue // Skip instance volumes and buckets. } else if poolVol.Container == nil && poolVol.Volume == nil { return response.SmartError(errors.New("Volume is neither instance nor custom volume")) } // Import custom volume and any snapshots. cleanup, err := pool.ImportCustomVolume(customStorageProjectName, poolVol, nil) if err != nil { return response.SmartError(fmt.Errorf("Failed importing custom volume %q in project %q: %w", poolVol.Volume.Name, projectName, err)) } reverter.Add(cleanup) } // Recover unknown buckets. for _, poolVol := range poolVols { // Skip non bucket volumes. if poolVol.Bucket == nil { continue } // Import bucket. cleanup, err := pool.ImportBucket(projectName, poolVol, nil) if err != nil { return response.SmartError(fmt.Errorf("Failed importing bucket %q in project %q: %w", poolVol.Bucket.Name, projectName, err)) } reverter.Add(cleanup) } } } // Finally restore the instances. for _, pool := range pools { for projectName, poolVols := range poolsProjectVols[pool.Name()] { projectInfo := projects[projectName] if projectInfo == nil { // Shouldn't happen as we validated this above, but be sure for safety. return response.SmartError(fmt.Errorf("Project %q not found", projectName)) } profileProjectName := project.ProfileProjectFromRecord(projectInfo) // Recover unknown instance volumes. for _, poolVol := range poolVols { if poolVol.Container == nil && (poolVol.Volume != nil || poolVol.Bucket != nil) { continue // Skip custom volumes, invalid volumes and buckets. } // Recover instance volumes and any snapshots. profiles := make([]api.Profile, 0, len(poolVol.Container.Profiles)) for _, profileName := range poolVol.Container.Profiles { for i := range projectProfiles[profileProjectName] { if projectProfiles[profileProjectName][i].Name == profileName { profiles = append(profiles, *projectProfiles[profileProjectName][i]) } } } inst, cleanup, err := internalRecoverImportInstance(s, pool, projectName, poolVol, profiles) if err != nil { return response.SmartError(fmt.Errorf("Failed creating instance %q record in project %q: %w", poolVol.Container.Name, projectName, err)) } reverter.Add(cleanup) // Recover instance volume snapshots. for _, poolInstSnap := range poolVol.Snapshots { profiles := make([]api.Profile, 0, len(poolInstSnap.Profiles)) for _, profileName := range poolInstSnap.Profiles { for i := range projectProfiles[profileProjectName] { if projectProfiles[profileProjectName][i].Name == profileName { profiles = append(profiles, *projectProfiles[profileProjectName][i]) } } } cleanup, err := internalRecoverImportInstanceSnapshot(s, pool, projectName, poolVol, poolInstSnap, profiles) if err != nil { return response.SmartError(fmt.Errorf("Failed creating instance %q snapshot %q record in project %q: %w", poolVol.Container.Name, poolInstSnap.Name, projectName, err)) } reverter.Add(cleanup) } // Recreate instance mount path and symlinks (must come after snapshot recovery). cleanup, err = pool.ImportInstance(inst, poolVol, nil) if err != nil { return response.SmartError(fmt.Errorf("Failed importing instance %q in project %q: %w", poolVol.Container.Name, projectName, err)) } reverter.Add(cleanup) // Reinitialize the instance's root disk quota even if no size specified (allows the storage driver the // opportunity to reinitialize the quota based on the new storage volume's DB ID). _, rootConfig, err := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if err == nil { err = pool.SetInstanceQuota(inst, rootConfig["size"], rootConfig["size.state"], nil) if err != nil { return response.SmartError(fmt.Errorf("Failed reinitializing root disk quota %q for instance %q in project %q: %w", rootConfig["size"], poolVol.Container.Name, projectName, err)) } } } } } reverter.Success() return response.EmptySyncResponse } // internalRecoverImportInstance recreates the database records for an instance and returns the new instance. // Returns a revert fail function that can be used to undo this function if a subsequent step fails. func internalRecoverImportInstance(s *state.State, pool storagePools.Pool, projectName string, poolVol *backupConfig.Config, profiles []api.Profile) (instance.Instance, revert.Hook, error) { if poolVol.Container == nil { return nil, nil, errors.New("Pool volume is not an instance volume") } // Add root device if needed. if poolVol.Container.Devices == nil { poolVol.Container.Devices = make(map[string]map[string]string) } if poolVol.Container.ExpandedDevices == nil { poolVol.Container.ExpandedDevices = make(map[string]map[string]string) } internalImportRootDevicePopulate(pool.Name(), poolVol.Container.Devices, poolVol.Container.ExpandedDevices, profiles) dbInst, err := backup.ConfigToInstanceDBArgs(s, poolVol, projectName, true) if err != nil { return nil, nil, err } if dbInst.Type < 0 { return nil, nil, errors.New("Invalid instance type") } inst, instOp, cleanup, err := instance.CreateInternal(s, *dbInst, nil, false, true, false) if err != nil { return nil, nil, fmt.Errorf("Failed creating instance record: %w", err) } defer instOp.Done(err) return inst, cleanup, err } // internalRecoverImportInstance recreates the database records for an instance snapshot. func internalRecoverImportInstanceSnapshot(s *state.State, pool storagePools.Pool, projectName string, poolVol *backupConfig.Config, snap *api.InstanceSnapshot, profiles []api.Profile) (revert.Hook, error) { if poolVol.Container == nil || snap == nil { return nil, errors.New("Pool volume is not an instance volume") } // Add root device if needed. if snap.Devices == nil { snap.Devices = make(map[string]map[string]string) } if snap.ExpandedDevices == nil { snap.ExpandedDevices = make(map[string]map[string]string) } internalImportRootDevicePopulate(pool.Name(), snap.Devices, snap.ExpandedDevices, profiles) arch, err := osarch.ArchitectureID(snap.Architecture) if err != nil { return nil, err } instanceType, err := instancetype.New(poolVol.Container.Type) if err != nil { return nil, err } _, snapInstOp, cleanup, err := instance.CreateInternal(s, db.InstanceArgs{ Project: projectName, Architecture: arch, BaseImage: snap.Config["volatile.base_image"], Config: snap.Config, Description: snap.Description, CreationDate: snap.CreatedAt, Type: instanceType, Snapshot: true, Devices: deviceConfig.NewDevices(snap.Devices), Ephemeral: snap.Ephemeral, LastUsedDate: snap.LastUsedAt, Name: poolVol.Container.Name + internalInstance.SnapshotDelimiter + snap.Name, Profiles: profiles, Stateful: snap.Stateful, }, nil, false, true, false) if err != nil { return nil, fmt.Errorf("Failed creating instance snapshot record %q: %w", snap.Name, err) } defer snapInstOp.Done(err) return cleanup, err } // internalRecoverValidate validates the requested pools to be recovered. func internalRecoverValidate(d *Daemon, r *http.Request) response.Response { // Parse the request. req := &internalRecover.ValidatePost{} err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } return internalRecoverScan(r.Context(), d.State(), req.Pools, true) } // internalRecoverImport performs the pool volume recovery. func internalRecoverImport(d *Daemon, r *http.Request) response.Response { // Parse the request. req := &internalRecover.ImportPost{} err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } return internalRecoverScan(r.Context(), d.State(), req.Pools, false) } incus-7.0.0/cmd/incusd/api_internal_test.go000066400000000000000000000133471517523235500207420ustar00rootroot00000000000000package main import ( "testing" "github.com/stretchr/testify/assert" "github.com/lxc/incus/v7/shared/api" ) // Test that an instance with a local root disk device just gets its pool property modified. func TestInternalImportRootDevicePopulate_LocalDevice(t *testing.T) { instancePoolName := "test" localDevices := make(map[string]map[string]string) localRootDev := map[string]string{ "type": "disk", "path": "/", "pool": "oldpool", "size": "15GiB", } localDevices["root"] = localRootDev internalImportRootDevicePopulate(instancePoolName, localDevices, nil, nil) assert.Equal(t, instancePoolName, localDevices["root"]["pool"]) assert.Equal(t, localRootDev["type"], localDevices["root"]["type"]) assert.Equal(t, localRootDev["path"], localDevices["root"]["path"]) assert.Equal(t, localRootDev["size"], localDevices["root"]["size"]) } // Test that an instance with no local root disk device but has a root disk from its old expanded profile devices, // that doesn't match the root disk in the new profiles, gets it added back as a local disk, with the pool property // modified. func TestInternalImportRootDevicePopulate_ExpandedDeviceProfileDeviceMismatch(t *testing.T) { instancePoolName := "test" localDevices := make(map[string]map[string]string) expandedDevices := make(map[string]map[string]string) expandedRootDev := map[string]string{ "type": "disk", "path": "/", "pool": "oldpool", "size": "15GiB", } expandedDevices["root"] = expandedRootDev profiles := []api.Profile{ { Name: "default", }, } internalImportRootDevicePopulate(instancePoolName, localDevices, expandedDevices, profiles) assert.Equal(t, instancePoolName, localDevices["root"]["pool"]) assert.Equal(t, expandedRootDev["type"], localDevices["root"]["type"]) assert.Equal(t, expandedRootDev["path"], localDevices["root"]["path"]) assert.Equal(t, expandedRootDev["size"], localDevices["root"]["size"]) } // Test that an instance with no local root disk device but has a root disk from its old expanded profile devices, // that matches the new profile root disk device (excluding pool name), and the new profile root disk matches the // target pool, then no local root disk device is added, and the instance will continue to use the profile root // disk device. func TestInternalImportRootDevicePopulate_ExpandedDeviceProfileDeviceMatch(t *testing.T) { instancePoolName := "test" localDevices := make(map[string]map[string]string) expandedDevices := make(map[string]map[string]string) expandedRootDev := map[string]string{ "type": "disk", "path": "/", "pool": "oldpool", "size": "15GiB", } expandedDevices["root"] = expandedRootDev profiles := []api.Profile{ { Name: "default", ProfilePut: api.ProfilePut{ Devices: make(map[string]map[string]string), }, }, } profiles[0].Devices["root"] = map[string]string{ "type": "disk", "path": "/", "pool": instancePoolName, "size": "15GiB", } internalImportRootDevicePopulate(instancePoolName, localDevices, expandedDevices, profiles) assert.Equal(t, len(localDevices), 0) } // Test that for an instance with no local root disk device, if the new profile root disk device doesn't match the // target pool that the old expanded root device is added as a local root disk device (with the pool modified). func TestInternalImportRootDevicePopulate_ExpandedDeviceProfileDevicePoolMismatch(t *testing.T) { instancePoolName := "test" localDevices := make(map[string]map[string]string) expandedDevices := make(map[string]map[string]string) expandedRootDev := map[string]string{ "type": "disk", "path": "/", "pool": "oldpool", "size": "15GiB", } expandedDevices["root"] = expandedRootDev profiles := []api.Profile{ { Name: "default", ProfilePut: api.ProfilePut{ Devices: make(map[string]map[string]string), }, }, } profiles[0].Devices["root"] = map[string]string{ "type": "disk", "path": "/", "pool": "wrongpool", "size": "15GiB", } internalImportRootDevicePopulate(instancePoolName, localDevices, expandedDevices, profiles) assert.Equal(t, instancePoolName, localDevices["root"]["pool"]) assert.Equal(t, expandedRootDev["type"], localDevices["root"]["type"]) assert.Equal(t, expandedRootDev["path"], localDevices["root"]["path"]) assert.Equal(t, expandedRootDev["size"], localDevices["root"]["size"]) } // Test that if old config has no root disk device, and neither does new profiles, then a basic local root disk // device is added using the target pool. func TestInternalImportRootDevicePopulate_NoExistingRootDiskDevice(t *testing.T) { instancePoolName := "test" localDevices := make(map[string]map[string]string) internalImportRootDevicePopulate(instancePoolName, localDevices, nil, nil) assert.Equal(t, instancePoolName, localDevices["root"]["pool"]) assert.Equal(t, "disk", localDevices["root"]["type"]) assert.Equal(t, "/", localDevices["root"]["path"]) } // Test that if old config has no root disk device, and neither does new profiles, then a basic local root disk // device is added using the target pool, and if there is already a local device called "root", then this new root // disk device is added under an automatically generated name. func TestInternalImportRootDevicePopulate_NoExistingRootDiskDeviceNameConflict(t *testing.T) { instancePoolName := "test" localDevices := make(map[string]map[string]string) localConflictingRootDev := map[string]string{ "type": "nic", "nictype": "bridged", "name": "eth0", } localDevices["root"] = localConflictingRootDev // Conflicting device called "root". internalImportRootDevicePopulate(instancePoolName, localDevices, nil, nil) assert.Equal(t, instancePoolName, localDevices["root0"]["pool"]) assert.Equal(t, "disk", localDevices["root0"]["type"]) assert.Equal(t, "/", localDevices["root0"]["path"]) } incus-7.0.0/cmd/incusd/api_metrics.go000066400000000000000000000373161517523235500175370ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "io" "net" "net/http" "runtime" "slices" "strings" "sync" "time" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/instance" instanceDrivers "github.com/lxc/incus/v7/internal/server/instance/drivers" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/locking" "github.com/lxc/incus/v7/internal/server/metrics" projecthelpers "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) type metricsCacheEntry struct { metrics *metrics.MetricSet expiry time.Time } var ( metricsCache map[string]metricsCacheEntry metricsCacheLock sync.Mutex ) var metricsCmd = APIEndpoint{ Path: "metrics", Get: APIEndpointAction{Handler: metricsGet, AccessHandler: allowMetrics, AllowUntrusted: true}, } func allowMetrics(d *Daemon, r *http.Request) response.Response { s := d.State() if !s.GlobalConfig.MetricsAuthentication() { return response.EmptySyncResponse } return allowPermission(auth.ObjectTypeServer, auth.EntitlementCanViewMetrics)(d, r) } // swagger:operation GET /1.0/metrics metrics metrics_get // // Get metrics // // Gets metrics of instances. // // --- // produces: // - text/plain // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Metrics // schema: // type: string // description: Instance metrics // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func metricsGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.QueryParam(r, "project") compress := strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") // Forward if requested. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } // Wait until daemon is fully started. <-d.waitReady.Done() // Prepare response. metricSet := metrics.NewMetricSet(nil) var projectNames []string var intMetrics *metrics.MetricSet err := s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Figure out the projects to retrieve. if projectName != "" { projectNames = []string{projectName} } else { // Get all project names if no specific project requested. projects, err := dbCluster.GetProjects(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading projects: %w", err) } projectNames = make([]string, 0, len(projects)) for _, project := range projects { projectNames = append(projectNames, project.Name) } } // Add internal metrics. intMetrics = internalMetrics(ctx, s, tx) return nil }) if err != nil { return response.SmartError(err) } // invalidProjectFilters returns project filters which are either not in cache or have expired. invalidProjectFilters := func(projectNames []string) []dbCluster.InstanceFilter { metricsCacheLock.Lock() defer metricsCacheLock.Unlock() var filters []dbCluster.InstanceFilter for _, p := range projectNames { projectName := p // Local var for filter pointer. cache, ok := metricsCache[projectName] if !ok || cache.expiry.Before(time.Now()) { // If missing or expired, record it. filters = append(filters, dbCluster.InstanceFilter{ Project: &projectName, Node: &s.ServerName, }) continue } // If present and valid, merge the existing data. metricSet.Merge(cache.metrics) } return filters } // Review the cache for invalid projects. projectsToFetch := invalidProjectFilters(projectNames) // If all valid, return immediately. if len(projectsToFetch) == 0 { // Merge in the internal metrics. metricSet.Merge(intMetrics) return getFilteredMetrics(s, r, compress, metricSet) } cacheDuration := time.Duration(8) * time.Second // Acquire update lock. lockCtx, lockCtxCancel := context.WithTimeout(r.Context(), cacheDuration) defer lockCtxCancel() unlock, err := locking.Lock(lockCtx, "metricsGet") if err != nil { return response.SmartError(api.StatusErrorf(http.StatusLocked, "Metrics are currently being built by another request: %s", err)) } defer unlock() // Setup a new response. metricSet = metrics.NewMetricSet(nil) // Merge in the internal metrics. metricSet.Merge(intMetrics) // Check if any of the missing data has been filled in since acquiring the lock. // As its possible another request was already populating the cache when we tried to take the lock. projectsToFetch = invalidProjectFilters(projectNames) // If all valid, return immediately. if len(projectsToFetch) == 0 { return getFilteredMetrics(s, r, compress, metricSet) } // Gather information about host interfaces once. hostInterfaces, _ := net.Interfaces() var instances []instance.Instance err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.InstanceList(ctx, func(dbInst db.InstanceArgs, p api.Project) error { inst, err := instance.Load(s, dbInst, p) if err != nil { return fmt.Errorf("Failed loading instance %q in project %q: %w", dbInst.Name, dbInst.Project, err) } instances = append(instances, inst) return nil }, projectsToFetch...) }) if err != nil { return response.SmartError(err) } // Prepare temporary metrics storage. newMetrics := make(map[string]*metrics.MetricSet, len(projectsToFetch)) newMetricsLock := sync.Mutex{} // Limit metrics build concurrency to number of instances or number of CPU cores (which ever is less). var wg sync.WaitGroup instMetricsCh := make(chan instance.Instance) maxConcurrent := runtime.NumCPU() instCount := len(instances) if instCount < maxConcurrent { maxConcurrent = instCount } // Start metrics builder routines. for range maxConcurrent { go func(instMetricsCh <-chan instance.Instance) { for inst := range instMetricsCh { projectName := inst.Project().Name instanceMetrics, err := inst.Metrics(hostInterfaces) if err != nil { // Ignore stopped instances. if !errors.Is(err, instanceDrivers.ErrInstanceIsStopped) { logger.Warn("Failed getting instance metrics", logger.Ctx{"instance": inst.Name(), "project": projectName, "err": err}) } } else { // Add the metrics. newMetricsLock.Lock() // Initialize metrics set for project if needed. if newMetrics[projectName] == nil { newMetrics[projectName] = metrics.NewMetricSet(nil) } newMetrics[projectName].Merge(instanceMetrics) newMetricsLock.Unlock() } wg.Done() } }(instMetricsCh) } // Fetch what's missing. for _, inst := range instances { wg.Add(1) instMetricsCh <- inst } wg.Wait() close(instMetricsCh) // Put the new data in the global cache and in response. metricsCacheLock.Lock() if metricsCache == nil { metricsCache = map[string]metricsCacheEntry{} } updatedProjects := []string{} for project, entries := range newMetrics { metricsCache[project] = metricsCacheEntry{ expiry: time.Now().Add(cacheDuration), metrics: entries, } updatedProjects = append(updatedProjects, project) metricSet.Merge(entries) } for _, project := range projectsToFetch { if slices.Contains(updatedProjects, *project.Project) { continue } metricsCache[*project.Project] = metricsCacheEntry{ expiry: time.Now().Add(cacheDuration), } } metricsCacheLock.Unlock() return getFilteredMetrics(s, r, compress, metricSet) } func getFilteredMetrics(s *state.State, r *http.Request, compress bool, metricSet *metrics.MetricSet) response.Response { if !s.GlobalConfig.MetricsAuthentication() { return response.SyncResponsePlain(true, compress, metricSet.String()) } // Get instances the user is allowed to view. userHasPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanView, auth.ObjectTypeInstance) if err != nil && !api.StatusErrorCheck(err, http.StatusForbidden) { return response.SmartError(err) } else if err != nil { userHasPermission, err = s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanViewMetrics, auth.ObjectTypeInstance) if err != nil { return response.SmartError(err) } } metricSet.FilterSamples(userHasPermission) return response.SyncResponsePlain(true, compress, metricSet.String()) } func internalMetrics(ctx context.Context, s *state.State, tx *db.ClusterTx) *metrics.MetricSet { out := metrics.NewMetricSet(nil) warnings, err := dbCluster.GetWarnings(ctx, tx.Tx()) if err != nil { logger.Warn("Failed to get warnings", logger.Ctx{"err": err}) } else { // Total number of warnings out.AddSamples(metrics.WarningsTotal, metrics.Sample{Value: float64(len(warnings))}) } operations, err := dbCluster.GetOperations(ctx, tx.Tx()) if err != nil { logger.Warn("Failed to get operations", logger.Ctx{"err": err}) } else { // Total number of operations out.AddSamples(metrics.OperationsTotal, metrics.Sample{Value: float64(len(operations))}) } // Project metrics. projects, err := dbCluster.GetProjects(ctx, tx.Tx()) if err != nil { logger.Warn("Failed to get projects", logger.Ctx{"err": err}) } else { for _, p := range projects { project, err := p.ToAPI(ctx, tx.Tx()) if err != nil { logger.Warn("Failed to convert project", logger.Ctx{"project": p.Name, "err": err}) continue } containerType := instancetype.Container containers, err := dbCluster.GetInstances(ctx, tx.Tx(), dbCluster.InstanceFilter{Project: &p.Name, Type: &containerType}) if err != nil { logger.Warn("Failed to get container count", logger.Ctx{"project": p.Name, "err": err}) } else { out.AddSamples(metrics.ProjectResourcesTotal, metrics.Sample{ Labels: map[string]string{"project": p.Name, "resource": "containers"}, Value: float64(len(containers)), }) } vmType := instancetype.VM vms, err := dbCluster.GetInstances(ctx, tx.Tx(), dbCluster.InstanceFilter{Project: &p.Name, Type: &vmType}) if err != nil { logger.Warn("Failed to get VM count", logger.Ctx{"project": p.Name, "err": err}) } else { out.AddSamples(metrics.ProjectResourcesTotal, metrics.Sample{ Labels: map[string]string{"project": p.Name, "resource": "virtual-machines"}, Value: float64(len(vms)), }) } images, err := dbCluster.GetImages(ctx, tx.Tx(), dbCluster.ImageFilter{Project: &p.Name}) if err != nil { logger.Warn("Failed to get image count", logger.Ctx{"project": p.Name, "err": err}) } else { out.AddSamples(metrics.ProjectResourcesTotal, metrics.Sample{ Labels: map[string]string{"project": p.Name, "resource": "images"}, Value: float64(len(images)), }) } profiles, err := dbCluster.GetProfiles(ctx, tx.Tx(), dbCluster.ProfileFilter{Project: &p.Name}) if err != nil { logger.Warn("Failed to get profile count", logger.Ctx{"project": p.Name, "err": err}) } else { out.AddSamples(metrics.ProjectResourcesTotal, metrics.Sample{ Labels: map[string]string{"project": p.Name, "resource": "profiles"}, Value: float64(len(profiles)), }) } networks, err := tx.GetNetworks(ctx, p.Name) if err != nil { logger.Warn("Failed to get network count", logger.Ctx{"project": p.Name, "err": err}) } else { out.AddSamples(metrics.ProjectResourcesTotal, metrics.Sample{ Labels: map[string]string{"project": p.Name, "resource": "networks"}, Value: float64(len(networks)), }) } volumes, err := tx.GetCustomVolumesInProject(ctx, p.Name) if err != nil { logger.Warn("Failed to get storage volume count", logger.Ctx{"project": p.Name, "err": err}) } else { out.AddSamples(metrics.ProjectResourcesTotal, metrics.Sample{ Labels: map[string]string{"project": p.Name, "resource": "storage-volumes"}, Value: float64(len(volumes)), }) } hasLimits := false for limitKey := range project.Config { if strings.HasPrefix(limitKey, "limits.") { hasLimits = true break } } if hasLimits { allocations, err := projecthelpers.GetCurrentAllocations(ctx, tx, p.Name) if err != nil { logger.Warn("Failed to get project allocations", logger.Ctx{"project": p.Name, "err": err}) continue } for resource, alloc := range allocations { out.AddSamples(metrics.ProjectLimit, metrics.Sample{ Labels: map[string]string{"project": p.Name, "resource": resource}, Value: float64(alloc.Limit), }) out.AddSamples(metrics.ProjectUsage, metrics.Sample{ Labels: map[string]string{"project": p.Name, "resource": resource}, Value: float64(alloc.Usage), }) } } } } // Daemon uptime out.AddSamples(metrics.UptimeSeconds, metrics.Sample{Value: time.Since(s.StartTime).Seconds()}) // Number of goroutines out.AddSamples(metrics.GoGoroutines, metrics.Sample{Value: float64(runtime.NumGoroutine())}) // Go memory stats var ms runtime.MemStats runtime.ReadMemStats(&ms) out.AddSamples(metrics.GoAllocBytes, metrics.Sample{Value: float64(ms.Alloc)}) out.AddSamples(metrics.GoAllocBytesTotal, metrics.Sample{Value: float64(ms.TotalAlloc)}) out.AddSamples(metrics.GoBuckHashSysBytes, metrics.Sample{Value: float64(ms.BuckHashSys)}) out.AddSamples(metrics.GoFreesTotal, metrics.Sample{Value: float64(ms.Frees)}) out.AddSamples(metrics.GoGCSysBytes, metrics.Sample{Value: float64(ms.GCSys)}) out.AddSamples(metrics.GoHeapAllocBytes, metrics.Sample{Value: float64(ms.HeapAlloc)}) out.AddSamples(metrics.GoHeapIdleBytes, metrics.Sample{Value: float64(ms.HeapIdle)}) out.AddSamples(metrics.GoHeapInuseBytes, metrics.Sample{Value: float64(ms.HeapInuse)}) out.AddSamples(metrics.GoHeapObjects, metrics.Sample{Value: float64(ms.HeapObjects)}) out.AddSamples(metrics.GoHeapReleasedBytes, metrics.Sample{Value: float64(ms.HeapReleased)}) out.AddSamples(metrics.GoHeapSysBytes, metrics.Sample{Value: float64(ms.HeapSys)}) out.AddSamples(metrics.GoLookupsTotal, metrics.Sample{Value: float64(ms.Lookups)}) out.AddSamples(metrics.GoMallocsTotal, metrics.Sample{Value: float64(ms.Mallocs)}) out.AddSamples(metrics.GoMCacheInuseBytes, metrics.Sample{Value: float64(ms.MCacheInuse)}) out.AddSamples(metrics.GoMCacheSysBytes, metrics.Sample{Value: float64(ms.MCacheSys)}) out.AddSamples(metrics.GoMSpanInuseBytes, metrics.Sample{Value: float64(ms.MSpanInuse)}) out.AddSamples(metrics.GoMSpanSysBytes, metrics.Sample{Value: float64(ms.MSpanSys)}) out.AddSamples(metrics.GoNextGCBytes, metrics.Sample{Value: float64(ms.NextGC)}) out.AddSamples(metrics.GoOtherSysBytes, metrics.Sample{Value: float64(ms.OtherSys)}) out.AddSamples(metrics.GoStackInuseBytes, metrics.Sample{Value: float64(ms.StackInuse)}) out.AddSamples(metrics.GoStackSysBytes, metrics.Sample{Value: float64(ms.StackSys)}) out.AddSamples(metrics.GoSysBytes, metrics.Sample{Value: float64(ms.Sys)}) // If on IncusOS, include OS metrics. if s.OS.IncusOS != nil { client := http.Client{} client.Transport = &http.Transport{ DialContext: func(_ context.Context, network, addr string) (net.Conn, error) { return net.DialTimeout("tcp", "127.0.0.1:9100", 50*time.Millisecond) }, DisableKeepAlives: true, ExpectContinueTimeout: time.Second * 5, ResponseHeaderTimeout: time.Second * 5, } resp, err := client.Get("http://incus-os/metrics") if err == nil { defer resp.Body.Close() osMetrics, err := io.ReadAll(resp.Body) if err == nil { out.AddRaw(osMetrics) } } } return out } incus-7.0.0/cmd/incusd/api_os.go000066400000000000000000000037211517523235500165030ustar00rootroot00000000000000package main import ( "context" "errors" "net" "net/http" "net/http/httputil" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/response" ) var apiOS = APIEndpoint{ Path: "{name:.*}", Patch: APIEndpointAction{Handler: apiOSProxy, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Put: APIEndpointAction{Handler: apiOSProxy, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Get: APIEndpointAction{Handler: apiOSProxy, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Post: APIEndpointAction{Handler: apiOSProxy, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Delete: APIEndpointAction{Handler: apiOSProxy, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Head: APIEndpointAction{Handler: apiOSProxy, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } func apiOSProxy(d *Daemon, r *http.Request) response.Response { s := d.State() // If a target was specified, forward the request to the relevant node. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } // Check if this is an IncusOS system. if s.OS.IncusOS == nil { return response.BadRequest(errors.New("System isn't running IncusOS")) } // Prepare the proxy. proxy := &httputil.ReverseProxy{ Transport: &http.Transport{ DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { return net.Dial("unix", "/run/incus-os/unix.socket") }, }, Director: func(r *http.Request) { r.URL.Scheme = "http" r.URL.Host = "incus-os" }, } // Allow IncusOS to adjust the returned paths to the prefix used by the proxy. r.Header.Add("X-IncusOS-Proxy", "/os") // Handle the request. return response.ManualResponse(func(w http.ResponseWriter) error { http.StripPrefix("/os", proxy).ServeHTTP(w, r) return nil }) } incus-7.0.0/cmd/incusd/api_project.go000066400000000000000000001732171517523235500175400ustar00rootroot00000000000000package main import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net" "net/http" "net/url" "slices" "strings" "github.com/gorilla/mux" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/operations" projecthelpers "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var projectsCmd = APIEndpoint{ Path: "projects", Get: APIEndpointAction{Handler: projectsGet, AccessHandler: allowAuthenticated}, Post: APIEndpointAction{Handler: projectsPost, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanCreateProjects)}, } var projectCmd = APIEndpoint{ Path: "projects/{name}", Delete: APIEndpointAction{Handler: projectDelete, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanEdit, "name")}, Get: APIEndpointAction{Handler: projectGet, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanView, "name")}, Patch: APIEndpointAction{Handler: projectPatch, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanEdit, "name")}, Post: APIEndpointAction{Handler: projectPost, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanEdit, "name")}, Put: APIEndpointAction{Handler: projectPut, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanEdit, "name")}, } var projectStateCmd = APIEndpoint{ Path: "projects/{name}/state", Get: APIEndpointAction{Handler: projectStateGet, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanView, "name")}, } var projectAccessCmd = APIEndpoint{ Path: "projects/{name}/access", Get: APIEndpointAction{Handler: projectAccess, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanEdit, "name")}, } // swagger:operation GET /1.0/projects projects projects_get // // Get the projects // // Returns a list of projects (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/projects/default", // "/1.0/projects/foo" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/projects?recursion=1 projects projects_get_recursion1 // // Get the projects // // Returns a list of projects (structs). // // --- // produces: // - application/json // parameters: // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of projects // items: // $ref: "#/definitions/Project" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func projectsGet(d *Daemon, r *http.Request) response.Response { s := d.State() recursion := localUtil.IsRecursionRequest(r) // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } userHasPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanView, auth.ObjectTypeProject) if err != nil { return response.InternalError(err) } filtered := make([]api.Project, 0) err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { projects, err := cluster.GetProjects(ctx, tx.Tx()) if err != nil { return err } for _, project := range projects { if !userHasPermission(auth.ObjectProject(project.Name)) { continue } apiProject, err := project.ToAPI(ctx, tx.Tx()) if err != nil { return err } apiProject.UsedBy, err = projectUsedBy(ctx, tx, &project) if err != nil { return err } if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(*apiProject, *clauses) if err != nil { return err } if !match { continue } } filtered = append(filtered, *apiProject) } return nil }) if err != nil { return response.SmartError(err) } if recursion { return response.SyncResponse(true, filtered) } urls := make([]string, len(filtered)) for i, p := range filtered { urls[i] = p.URL(version.APIVersion).String() } return response.SyncResponse(true, urls) } // projectUsedBy returns a list of URLs for all instances, images, profiles, // storage volumes, networks, and acls that use this project. func projectUsedBy(ctx context.Context, tx *db.ClusterTx, project *cluster.Project) ([]string, error) { usedBy := []string{} instances, err := cluster.GetInstances(ctx, tx.Tx(), cluster.InstanceFilter{Project: &project.Name}) if err != nil { return nil, err } for _, instance := range instances { apiInstance := api.Instance{Name: instance.Name} usedBy = append(usedBy, apiInstance.URL(version.APIVersion, project.Name).String()) } images, err := cluster.GetImages(ctx, tx.Tx(), cluster.ImageFilter{Project: &project.Name}) if err != nil { return nil, err } for _, image := range images { apiImage := api.Image{Fingerprint: image.Fingerprint} usedBy = append(usedBy, apiImage.URL(version.APIVersion, project.Name).String()) } networks, err := tx.GetNetworkURIs(ctx, project.ID, project.Name) if err != nil { return nil, err } usedBy = append(usedBy, networks...) acls, err := cluster.GetNetworkACLs(ctx, tx.Tx(), cluster.NetworkACLFilter{Project: &project.Name}) if err != nil { return nil, fmt.Errorf("Unable to get URIs for network acl: %w", err) } for _, acl := range acls { apiNetworkACL := api.NetworkACL{NetworkACLPost: api.NetworkACLPost{Name: acl.Name}} usedBy = append(usedBy, apiNetworkACL.URL(version.APIVersion, project.Name).String()) } var zones []cluster.NetworkZone zones, err = cluster.GetNetworkZones(ctx, tx.Tx(), cluster.NetworkZoneFilter{Project: &project.Name}) if err != nil { return nil, fmt.Errorf("Unable to get URIs for network zones: %w", err) } // Create URIs for each zone. networkZones := make([]string, len(zones)) for i, zone := range zones { networkZones[i] = api.NewURL().Path(version.APIVersion, "network-zones", zone.Name).Project(project.Name).String() } usedBy = append(usedBy, networkZones...) profiles, err := cluster.GetProfiles(ctx, tx.Tx(), cluster.ProfileFilter{Project: &project.Name}) if err != nil { return nil, err } for _, profile := range profiles { apiProfile := api.Profile{Name: profile.Name} usedBy = append(usedBy, apiProfile.URL(version.APIVersion, project.Name).String()) } storageBuckets, err := tx.GetStorageBucketURIs(ctx, project.Name) if err != nil { return nil, err } usedBy = append(usedBy, storageBuckets...) storageVolumes, err := tx.GetStorageVolumeURIs(ctx, project.Name) if err != nil { return nil, err } usedBy = append(usedBy, storageVolumes...) return usedBy, nil } // swagger:operation POST /1.0/projects projects projects_post // // Add a project // // Creates a new project. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: body // name: project // description: Project // required: true // schema: // $ref: "#/definitions/ProjectsPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func projectsPost(d *Daemon, r *http.Request) response.Response { s := d.State() // Parse the request. project := api.ProjectsPost{} // Set default features. if project.Config == nil { project.Config = map[string]string{} } for featureName, featureInfo := range cluster.ProjectFeatures { _, ok := project.Config[featureName] if !ok && featureInfo.DefaultEnabled { project.Config[featureName] = "true" } } err := json.NewDecoder(r.Body).Decode(&project) if err != nil { return response.BadRequest(err) } // Quick checks. err = validate.IsAPIName(project.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid project name: %w", err)) } err = projectValidateName(project.Name) if err != nil { return response.BadRequest(err) } // Validate the configuration. err = projectValidateConfig(s, project.Config) if err != nil { return response.BadRequest(err) } var id int64 err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { id, err = cluster.CreateProject(ctx, tx.Tx(), cluster.Project{Description: project.Description, Name: project.Name}) if err != nil { return fmt.Errorf("Failed adding database record: %w", err) } err = cluster.CreateProjectConfig(ctx, tx.Tx(), id, project.Config) if err != nil { return fmt.Errorf("Unable to create project config for project %q: %w", project.Name, err) } if util.IsTrue(project.Config["features.profiles"]) { err = projectCreateDefaultProfile(ctx, tx, project.Name) if err != nil { return err } if project.Config["features.images"] == "false" { err = cluster.InitProjectWithoutImages(ctx, tx.Tx(), project.Name) if err != nil { return err } } } return nil }) if err != nil { return response.SmartError(fmt.Errorf("Failed creating project %q: %w", project.Name, err)) } err = s.Authorizer.AddProject(r.Context(), id, project.Name) if err != nil { logger.Error("Failed to add project to authorizer", logger.Ctx{"name": project.Name, "error": err}) } requestor := request.CreateRequestor(r) lc := lifecycle.ProjectCreated.Event(project.Name, requestor, nil) s.Events.SendLifecycle(project.Name, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // Create the default profile of a project. func projectCreateDefaultProfile(ctx context.Context, tx *db.ClusterTx, project string) error { // Create a default profile profile := cluster.Profile{} profile.Project = project profile.Name = api.ProjectDefaultName profile.Description = fmt.Sprintf("Default Incus profile for project %s", project) _, err := cluster.CreateProfile(ctx, tx.Tx(), profile) if err != nil { return fmt.Errorf("Add default profile to database: %w", err) } return nil } // swagger:operation GET /1.0/projects/{name} projects project_get // // Get the project // // Gets a specific project. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Project name // type: string // required: true // responses: // "200": // description: Project // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Project" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func projectGet(d *Daemon, r *http.Request) response.Response { s := d.State() name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Get the database entry var project *api.Project err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := cluster.GetProject(ctx, tx.Tx(), name) if err != nil { return err } project, err = dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return err } project.UsedBy, err = projectUsedBy(ctx, tx, dbProject) return err }) if err != nil { return response.SmartError(err) } etag := []any{ project.Description, project.Config, } return response.SyncResponseETag(true, project, etag) } // swagger:operation PUT /1.0/projects/{name} projects project_put // // Update the project // // Updates the entire project configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Project name // type: string // required: true // - in: body // name: project // description: Project configuration // required: true // schema: // $ref: "#/definitions/ProjectPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func projectPut(d *Daemon, r *http.Request) response.Response { s := d.State() name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Get the current data var project *api.Project err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := cluster.GetProject(ctx, tx.Tx(), name) if err != nil { return err } project, err = dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return err } project.UsedBy, err = projectUsedBy(ctx, tx, dbProject) if err != nil { return err } return err }) if err != nil { return response.SmartError(err) } // Validate ETag etag := []any{ project.Description, project.Config, } err = localUtil.EtagCheck(r, etag) if err != nil { return response.PreconditionFailed(err) } // Parse the request req := api.ProjectPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(project.Name, lifecycle.ProjectUpdated.Event(project.Name, requestor, nil)) return projectChange(r.Context(), s, project, req) } // swagger:operation PATCH /1.0/projects/{name} projects project_patch // // Partially update the project // // Updates a subset of the project configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Project name // type: string // required: true // - in: body // name: project // description: Project configuration // required: true // schema: // $ref: "#/definitions/ProjectPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func projectPatch(d *Daemon, r *http.Request) response.Response { s := d.State() name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Get the current data var project *api.Project err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := cluster.GetProject(ctx, tx.Tx(), name) if err != nil { return err } project, err = dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return err } project.UsedBy, err = projectUsedBy(ctx, tx, dbProject) if err != nil { return err } return err }) if err != nil { return response.SmartError(err) } // Validate ETag etag := []any{ project.Description, project.Config, } err = localUtil.EtagCheck(r, etag) if err != nil { return response.PreconditionFailed(err) } body, err := io.ReadAll(r.Body) if err != nil { return response.InternalError(err) } rdr1 := io.NopCloser(bytes.NewBuffer(body)) rdr2 := io.NopCloser(bytes.NewBuffer(body)) reqRaw := jmap.Map{} err = json.NewDecoder(rdr1).Decode(&reqRaw) if err != nil { return response.BadRequest(err) } req := api.ProjectPut{} err = json.NewDecoder(rdr2).Decode(&req) if err != nil { return response.BadRequest(err) } // Check what was actually set in the query _, err = reqRaw.GetString("description") if err != nil { req.Description = project.Description } config, err := reqRaw.GetMap("config") if err != nil { req.Config = project.Config } else { for k, v := range project.Config { _, ok := config[k] if !ok { config[k] = v } } } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(project.Name, lifecycle.ProjectUpdated.Event(project.Name, requestor, nil)) return projectChange(r.Context(), s, project, req) } // Common logic between PUT and PATCH. func projectChange(ctx context.Context, s *state.State, project *api.Project, req api.ProjectPut) response.Response { // Make a list of config keys that have changed. configChanged := []string{} for key := range project.Config { if req.Config[key] != project.Config[key] { configChanged = append(configChanged, key) } } for key := range req.Config { _, ok := project.Config[key] if !ok { configChanged = append(configChanged, key) } } // Record which features have been changed. var featuresChanged []string for _, configKeyChanged := range configChanged { _, isFeature := cluster.ProjectFeatures[configKeyChanged] if isFeature { featuresChanged = append(featuresChanged, configKeyChanged) } } // Quick checks. if len(featuresChanged) > 0 { if project.Name == api.ProjectDefaultName { return response.BadRequest(errors.New("You can't change the features of the default project")) } // Consider the project empty if it is only used by the default profile. usedByLen := len(project.UsedBy) projectInUse := usedByLen > 1 || (usedByLen == 1 && !strings.Contains(project.UsedBy[0], "/profiles/default")) if projectInUse { // Check if feature is allowed to be changed. for _, featureChanged := range featuresChanged { // If feature is currently enabled, and it is being changed in the request, it // must be being disabled. So prevent it on non-empty projects. if util.IsTrue(project.Config[featureChanged]) { return response.BadRequest(fmt.Errorf("Project feature %q cannot be disabled on non-empty projects", featureChanged)) } // If feature is currently disabled, and it is being changed in the request, it // must be being enabled. So check if feature can be enabled on non-empty projects. if util.IsFalse(project.Config[featureChanged]) && !cluster.ProjectFeatures[featureChanged].CanEnableNonEmpty { return response.BadRequest(fmt.Errorf("Project feature %q cannot be enabled on non-empty projects", featureChanged)) } } } } // Validate the configuration. err := projectValidateConfig(s, req.Config) if err != nil { return response.BadRequest(err) } // Update the database entry. err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { err := projecthelpers.AllowProjectUpdate(tx, project.Name, req.Config, configChanged) if err != nil { return err } err = cluster.UpdateProject(ctx, tx.Tx(), project.Name, req) if err != nil { return fmt.Errorf("Persist profile changes: %w", err) } if slices.Contains(configChanged, "features.profiles") { if util.IsTrue(req.Config["features.profiles"]) { err = projectCreateDefaultProfile(ctx, tx, project.Name) if err != nil { return err } } else { // Delete the project-specific default profile. err = cluster.DeleteProfile(ctx, tx.Tx(), project.Name, api.ProjectDefaultName) if err != nil { return fmt.Errorf("Delete project default profile: %w", err) } } } if slices.Contains(configChanged, "features.images") && util.IsFalse(req.Config["features.images"]) && util.IsTrue(req.Config["features.profiles"]) { err = cluster.InitProjectWithoutImages(ctx, tx.Tx(), project.Name) if err != nil { return err } } return nil }) if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } // swagger:operation POST /1.0/projects/{name} projects project_post // // Rename the project // // Renames an existing project. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Project name // type: string // required: true // - in: body // name: project // description: Project rename request // required: true // schema: // $ref: "#/definitions/ProjectPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func projectPost(d *Daemon, r *http.Request) response.Response { s := d.State() name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Parse the request. req := api.ProjectPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. if name == api.ProjectDefaultName { return response.Forbidden(errors.New("The 'default' project cannot be renamed")) } // Perform the rename. run := func(op *operations.Operation) error { var id int64 err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { project, err := cluster.GetProject(ctx, tx.Tx(), req.Name) if err != nil && !response.IsNotFoundError(err) { return fmt.Errorf("Failed checking if project %q exists: %w", req.Name, err) } if project != nil { return fmt.Errorf("A project named %q already exists", req.Name) } project, err = cluster.GetProject(ctx, tx.Tx(), name) if err != nil { return fmt.Errorf("Failed loading project %q: %w", name, err) } empty, err := projectIsEmpty(ctx, project, tx) if err != nil { return err } if !empty { return errors.New("Only empty projects can be renamed") } id, err = cluster.GetProjectID(ctx, tx.Tx(), name) if err != nil { return fmt.Errorf("Failed getting project ID for project %q: %w", name, err) } err = validate.IsAPIName(name, false) if err != nil { return fmt.Errorf("Invalid project name: %w", err) } err = projectValidateName(req.Name) if err != nil { return fmt.Errorf("Invalid project name: %w", err) } return cluster.RenameProject(ctx, tx.Tx(), name, req.Name) }) if err != nil { return err } err = s.Authorizer.RenameProject(s.ShutdownCtx, id, name, req.Name) if err != nil { logger.Error("Failed to rename project in authorizer", logger.Ctx{"name": name, "new_name": req.Name, "err": err}) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(req.Name, lifecycle.ProjectRenamed.Event(req.Name, requestor, logger.Ctx{"old_name": name})) return nil } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.ProjectRename, nil, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // swagger:operation DELETE /1.0/projects/{name} projects project_delete // // Delete the project // // Removes the project. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Project name // type: string // required: true // - in: query // name: force // description: Delete project and related artifacts // type: boolean // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func projectDelete(d *Daemon, r *http.Request) response.Response { s := d.State() name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } force := util.IsTrue(r.FormValue("force")) // Quick checks. if name == api.ProjectDefaultName { return response.Forbidden(errors.New("The 'default' project cannot be deleted")) } var id int64 var projectConfig map[string]string var usedBy []string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { project, err := cluster.GetProject(ctx, tx.Tx(), name) if err != nil { return fmt.Errorf("Fetch project %q: %w", name, err) } if !force { empty, err := projectIsEmpty(ctx, project, tx) if err != nil { return err } if !empty { return errors.New("Only empty projects can be removed.") } } else { usedBy, err = projectUsedBy(ctx, tx, project) if err != nil { return err } } id, err = cluster.GetProjectID(ctx, tx.Tx(), name) if err != nil { return fmt.Errorf("Fetch project id %q: %w", name, err) } projectConfig, err = cluster.GetProjectConfig(ctx, tx.Tx(), int(id)) if err != nil { return fmt.Errorf("Fetch project config %q: %w", name, err) } return nil }) if err != nil { return response.SmartError(err) } // Handle requests to empty the project. if force { // Parse used by list. defaultProfile := api.NewURL().Path(version.APIVersion, "profiles", api.ProjectDefaultName).Project(name).String() entries := map[string][]string{} var count int for _, u := range usedBy { // Skip the default profile. if u == defaultProfile { continue } // Parse the URL. uri, err := url.Parse(u) if err != nil { return response.InternalError(err) } elements := strings.Split(uri.Path, "/") if len(elements) < 4 { return response.InternalError(fmt.Errorf("Bad usedBy entry: %s", u)) } if elements[2] == "storage-pools" { if elements[4] == "buckets" { if entries["storage-buckets"] == nil { entries["storage-buckets"] = []string{} } entry := fmt.Sprintf("%s/%s", elements[3], elements[5]) target := uri.Query().Get("target") if target != "" { entry = fmt.Sprintf("%s/%s", entry, target) } entries["storage-buckets"] = append(entries["storage-buckets"], entry) } else if elements[4] == "volumes" { if entries["storage-volumes"] == nil { entries["storage-volumes"] = []string{} } entry := fmt.Sprintf("%s/%s", elements[3], elements[6]) target := uri.Query().Get("target") if target != "" { entry = fmt.Sprintf("%s/%s", entry, target) } entries["storage-volumes"] = append(entries["storage-volumes"], entry) } } else { if entries[elements[2]] == nil { entries[elements[2]] = []string{} } entries[elements[2]] = append(entries[elements[2]], elements[3]) } count++ } // Connect to the local server. target, err := incus.ConnectIncusUnix(s.OS.GetUnixSocket(), nil) if err != nil { return response.InternalError(err) } target = target.UseProject(name) // Delete instances. for _, instName := range entries["instances"] { // Get current instance state. instState, _, err := target.GetInstance(instName) if err != nil { return response.InternalError(err) } // If running, force stop it. if instState.StatusCode != api.Stopped { req := api.InstanceStatePut{ Action: "stop", Timeout: -1, Force: true, } op, err := target.UpdateInstanceState(instName, req, "") if err != nil { return response.InternalError(err) } err = op.Wait() if err != nil { return response.InternalError(err) } } // Get the instance configuration. inst, _, err := target.GetInstance(instName) if err != nil { return response.InternalError(err) } // Clear security.protection.delete if set. if util.IsTrue(inst.ExpandedConfig["security.protection.delete"]) { inst.Config["security.protection.delete"] = "false" op, err := target.UpdateInstance(instName, inst.Writable(), "") if err != nil { return response.InternalError(err) } err = op.Wait() if err != nil { return response.InternalError(err) } } // Delete the instance. op, err := target.DeleteInstance(instName) if err != nil { return response.InternalError(err) } err = op.Wait() if err != nil { return response.InternalError(err) } // Done deleting the instance. count-- } // Delete profiles. for _, profileName := range entries["profiles"] { err := target.DeleteProfile(profileName) if err != nil { return response.InternalError(err) } // Done deleting the profile. count-- } // Empty the default profile, if the project owns one. if util.IsTrue(projectConfig["features.profiles"]) { err = target.UpdateProfile("default", api.ProfilePut{}, "") if err != nil { return response.InternalError(err) } } // Delete images. for _, imageFingerprint := range entries["images"] { op, err := target.DeleteImage(imageFingerprint) if err != nil { return response.InternalError(err) } err = op.Wait() if err != nil { return response.InternalError(err) } // Done deleting the image. count-- } // Delete networks. for _, networkName := range entries["networks"] { err := target.DeleteNetwork(networkName) if err != nil { return response.InternalError(err) } // Done deleting the network. count-- } // Delete network ACLs. for _, networkACLName := range entries["network-acls"] { err := target.DeleteNetworkACL(networkACLName) if err != nil { return response.InternalError(err) } // Done deleting the network ACL. count-- } // Delete network address sets. for _, networkAddressSetName := range entries["network-address-sets"] { err := target.DeleteNetworkAddressSet(networkAddressSetName) if err != nil { return response.InternalError(err) } // Done deleting the network address set. count-- } // Delete network zones. for _, networkZoneName := range entries["network-zones"] { err := target.DeleteNetworkZone(networkZoneName) if err != nil { return response.InternalError(err) } // Done deleting the network zone. count-- } // Delete storage volumes. for _, volume := range entries["storage-volumes"] { fields := strings.Split(volume, "/") if len(fields) == 3 { target.UseTarget(fields[2]) } err := target.DeleteStoragePoolVolume(fields[0], "custom", fields[1]) if err != nil { return response.InternalError(err) } // Done deleting the storage volume. count-- } // Delete storage buckets. for _, volume := range entries["storage-buckets"] { fields := strings.Split(volume, "/") if len(fields) == 3 { target.UseTarget(fields[2]) } err := target.DeleteStoragePoolBucket(fields[0], fields[1]) if err != nil { return response.InternalError(err) } // Done deleting the storage volume. count-- } // Check if anything is left. if count != 0 { return response.BadRequest(errors.New("Project couldn't be automatically emptied")) } } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { return cluster.DeleteProject(ctx, tx.Tx(), name) }) if err != nil { return response.SmartError(err) } err = s.Authorizer.DeleteProject(r.Context(), id, name) if err != nil { logger.Error("Failed to remove project from authorizer", logger.Ctx{"name": name, "err": err}) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(name, lifecycle.ProjectDeleted.Event(name, requestor, nil)) return response.EmptySyncResponse } // swagger:operation GET /1.0/projects/{name}/state projects project_state_get // // Get the project state // // Gets a specific project resource consumption information. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Project name // type: string // required: true // responses: // "200": // description: Project state // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/ProjectState" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func projectStateGet(d *Daemon, r *http.Request) response.Response { s := d.State() name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Setup the state struct. state := api.ProjectState{} // Get current limits and usage. err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { result, err := projecthelpers.GetCurrentAllocations(ctx, tx, name) if err != nil { return err } state.Resources = result return nil }) if err != nil { return response.SmartError(err) } return response.SyncResponse(true, &state) } // Check if a project is empty. func projectIsEmpty(ctx context.Context, project *cluster.Project, tx *db.ClusterTx) (bool, error) { usedBy, err := projectUsedBy(ctx, tx, project) if err != nil { return false, err } defaultProfile := api.NewURL().Path(version.APIVersion, "profiles", api.ProjectDefaultName).Project(project.Name).String() for _, entry := range usedBy { // Ignore the default profile. if entry == defaultProfile { continue } return false, nil } return true, nil } func isEitherAllowOrBlock(value string) error { return validate.Optional(validate.IsOneOf("block", "allow"))(value) } func isEitherAllowOrBlockOrManaged(value string) error { return validate.Optional(validate.IsOneOf("block", "allow", "managed"))(value) } func projectValidateConfig(s *state.State, config map[string]string) error { // Validate the project configuration. projectConfigKeys := map[string]func(value string) error{ // gendoc:generate(entity=project, group=specific, key=backups.compression_algorithm) // Specify which compression algorithm to use for backups in this project. // Possible values are `bzip2`, `gzip`, `lz4`, `lzma`, `xz`, `zstd` or `none`. // --- // type: string // shortdesc: Compression algorithm to use for backups "backups.compression_algorithm": validate.IsCompressionAlgorithm, // gendoc:generate(entity=project, group=features, key=features.profiles) // // --- // type: bool // defaultdesc: `false` // initialvaluedesc: `true` // shortdesc: Whether to use a separate set of profiles for the project "features.profiles": validate.Optional(validate.IsBool), // gendoc:generate(entity=project, group=features, key=features.images) // This setting applies to both images and image aliases. // --- // type: bool // defaultdesc: `false` // initialvaluedesc: `true` // shortdesc: Whether to use a separate set of images for the project "features.images": validate.Optional(validate.IsBool), // gendoc:generate(entity=project, group=features, key=features.storage.volumes) // // --- // type: bool // defaultdesc: `false` // initialvaluedesc: `true` // shortdesc: Whether to use a separate set of storage volumes for the project "features.storage.volumes": validate.Optional(validate.IsBool), // gendoc:generate(entity=project, group=features, key=features.storage.buckets) // // --- // type: bool // defaultdesc: `false` // initialvaluedesc: `true` // shortdesc: Whether to use a separate set of storage buckets for the project "features.storage.buckets": validate.Optional(validate.IsBool), // gendoc:generate(entity=project, group=features, key=features.networks) // // --- // type: bool // defaultdesc: `false` // initialvaluedesc: `false` // shortdesc: Whether to use a separate set of networks for the project "features.networks": validate.Optional(validate.IsBool), // gendoc:generate(entity=project, group=features, key=features.networks.zones) // // --- // type: bool // defaultdesc: `false` // initialvaluedesc: `false` // shortdesc: Whether to use a separate set of network zones for the project "features.networks.zones": validate.Optional(validate.IsBool), // gendoc:generate(entity=project, group=specific, key=images.auto_update_cached) // // --- // type: bool // shortdesc: Whether to automatically update cached images in the project "images.auto_update_cached": validate.Optional(validate.IsBool), // gendoc:generate(entity=project, group=specific, key=images.auto_update_interval) // Specify the interval in hours. // To disable looking for updates to cached images, set this option to `0`. // --- // type: integer // shortdesc: Interval at which to look for updates to cached images "images.auto_update_interval": validate.Optional(validate.IsInt64), // gendoc:generate(entity=project, group=specific, key=images.compression_algorithm) // Possible values are `bzip2`, `gzip`, `lz4`, `lzma`, `xz`, `zstd` or `none`. // --- // type: string // shortdesc: Compression algorithm to use for new images in the project "images.compression_algorithm": validate.IsCompressionAlgorithm, // gendoc:generate(entity=project, group=specific, key=images.default_architecture) // // --- // type: string // shortdesc: Default architecture to use in a mixed-architecture cluster "images.default_architecture": validate.Optional(validate.IsArchitecture), // gendoc:generate(entity=project, group=specific, key=images.remote_cache_expiry) // Specify the number of days after which the unused cached image expires. // --- // type: integer // shortdesc: When an unused cached remote image is flushed in the project "images.remote_cache_expiry": validate.Optional(validate.IsInt64), // gendoc:generate(entity=project, group=limits, key=limits.instances) // // --- // type: integer // shortdesc: Maximum number of instances that can be created in the project "limits.instances": validate.Optional(validate.IsUint32), // gendoc:generate(entity=project, group=limits, key=limits.containers) // // --- // type: integer // shortdesc: Maximum number of containers that can be created in the project "limits.containers": validate.Optional(validate.IsUint32), // gendoc:generate(entity=project, group=limits, key=limits.virtual-machines) // // --- // type: integer // shortdesc: Maximum number of VMs that can be created in the project "limits.virtual-machines": validate.Optional(validate.IsUint32), // gendoc:generate(entity=project, group=limits, key=limits.memory) // The value is the maximum value for the sum of the individual {config:option}`instance-resource-limits:limits.memory` configurations set on the instances of the project. // --- // type: string // shortdesc: Usage limit for the host's memory for the project "limits.memory": validate.Optional(validate.IsSize), // gendoc:generate(entity=project, group=limits, key=limits.processes) // This value is the maximum value for the sum of the individual {config:option}`instance-resource-limits:limits.processes` configurations set on the instances of the project. // --- // type: integer // shortdesc: Maximum number of processes within the project "limits.processes": validate.Optional(validate.IsUint32), // gendoc:generate(entity=project, group=limits, key=limits.cpu) // This value is the maximum value for the sum of the individual {config:option}`instance-resource-limits:limits.cpu` configurations set on the instances of the project. // --- // type: integer // shortdesc: Maximum number of CPUs to use in the project "limits.cpu": validate.Optional(validate.IsUint32), // gendoc:generate(entity=project, group=limits, key=limits.disk) // This value is the maximum value of the aggregate disk space used by all instance volumes, custom volumes, and images of the project. // --- // type: string // shortdesc: Maximum disk space used by the project "limits.disk": validate.Optional(validate.IsSize), // gendoc:generate(entity=project, group=limits, key=limits.networks) // // --- // type: integer // shortdesc: Maximum number of networks that the project can have "limits.networks": validate.Optional(validate.IsUint32), // gendoc:generate(entity=project, group=specific, key=network.hwaddr_pattern) // Specify a MAC address template, e.g. `10:66:6a:xx:xx:xx`, to use within the cluster. // Every `x` in the template will be replaced by a random character in `0`–`f`. // Beware of the birthday paradox! A single `xx` block leads to a 10% collision probability with only 8 addresses; for a double `xx:xx` block, 118 addresses; for a triple `xx:xx:xx` block, 1881; for a quadruple `xx:xx:xx:xx` block, 30084. We provide absolutely no guardrail against that. // --- // type: string // scope: global // shortdesc: MAC address template "network.hwaddr_pattern": validate.Optional(validate.IsMACPattern), // gendoc:generate(entity=project, group=restricted, key=restricted) // This option must be enabled to allow the `restricted.*` keys to take effect. // To temporarily remove the restrictions, you can disable this option instead of clearing the related keys. // --- // type: bool // defaultdesc: `false` // shortdesc: Whether to block access to security-sensitive features "restricted": validate.Optional(validate.IsBool), // gendoc:generate(entity=project, group=restricted, key=restricted.backups) // Possible values are `allow` or `block`. // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent creating instance or volume backups "restricted.backups": isEitherAllowOrBlock, // gendoc:generate(entity=project, group=restricted, key=restricted.cluster.groups) // If specified, this option prevents targeting cluster groups other than the provided ones. // --- // type: string // shortdesc: Cluster groups that can be targeted "restricted.cluster.groups": validate.Optional(func(value string) error { // Basic format validation. err := validate.IsListOf(validate.IsAny)(value) if err != nil { return err } // Get all valid groups. groupNames := []string{} err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { clusterGroups, err := cluster.GetClusterGroups(ctx, tx.Tx()) if err != nil { return err } for _, group := range clusterGroups { groupNames = append(groupNames, group.Name) } return nil }) if err != nil { return err } // Confirm that the group names exist. for _, name := range util.SplitNTrimSpace(value, ",", -1, true) { if !slices.Contains(groupNames, name) { return fmt.Errorf("Cluster group %q doesn't exist", name) } } return nil }), // gendoc:generate(entity=project, group=restricted, key=restricted.cluster.target) // Possible values are `allow` or `block`. // When set to `allow`, this option allows targeting of cluster members (either directly or via a group) when creating or moving instances. // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent targeting of cluster members "restricted.cluster.target": isEitherAllowOrBlock, // gendoc:generate(entity=project, group=restricted, key=restricted.containers.interception) // Possible values are `allow`, `block`, or `full`. // When set to `allow`, interception options that are usually safe are allowed. // File system mounting remains blocked. // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent using system call interception options "restricted.containers.interception": validate.Optional(validate.IsOneOf("allow", "block", "full")), // gendoc:generate(entity=project, group=restricted, key=restricted.containers.nesting) // Possible values are `allow` or `block`. // When set to `allow`, {config:option}`instance-security:security.nesting` can be set to `true` for an instance. // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent running nested Incus "restricted.containers.nesting": isEitherAllowOrBlock, // gendoc:generate(entity=project, group=restricted, key=restricted.containers.lowlevel) // Possible values are `allow` or `block`. // When set to `allow`, low-level container options like {config:option}`instance-raw:raw.lxc`, {config:option}`instance-raw:raw.idmap`, `volatile.*`, etc. can be used. // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent using low-level container options "restricted.containers.lowlevel": isEitherAllowOrBlock, // gendoc:generate(entity=project, group=restricted, key=restricted.containers.privilege) // Possible values are `unprivileged`, `isolated`, and `allow`. // // - When set to `unprivileged`, this option prevents setting {config:option}`instance-security:security.privileged` to `true`. // - When set to `isolated`, this option prevents setting {config:option}`instance-security:security.privileged` and {config:option}`instance-security:security.idmap.isolated` to `true`. // - When set to `allow`, there is no restriction. // --- // type: string // defaultdesc: `unprivileged` // shortdesc: Which settings for privileged containers to prevent "restricted.containers.privilege": validate.Optional(validate.IsOneOf("allow", "unprivileged", "isolated")), // gendoc:generate(entity=project, group=restricted, key=restricted.virtual-machines.lowlevel) // Possible values are `allow` or `block`. // When set to `allow`, low-level VM options like {config:option}`instance-raw:raw.qemu`, `volatile.*`, etc. can be used. // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent using low-level VM options "restricted.virtual-machines.lowlevel": isEitherAllowOrBlock, // gendoc:generate(entity=project, group=restricted, key=restricted.devices.unix-char) // Possible values are `allow` or `block`. // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent using devices of type `unix-char` "restricted.devices.unix-char": isEitherAllowOrBlock, // gendoc:generate(entity=project, group=restricted, key=restricted.devices.unix-block) // Possible values are `allow` or `block`. // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent using devices of type `unix-block` "restricted.devices.unix-block": isEitherAllowOrBlock, // gendoc:generate(entity=project, group=restricted, key=restricted.devices.unix-hotplug) // Possible values are `allow` or `block`. // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent using devices of type `unix-hotplug` "restricted.devices.unix-hotplug": isEitherAllowOrBlock, // gendoc:generate(entity=project, group=restricted, key=restricted.devices.infiniband) // Possible values are `allow` or `block`. // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent using devices of type `infiniband` "restricted.devices.infiniband": isEitherAllowOrBlock, // gendoc:generate(entity=project, group=restricted, key=restricted.devices.gpu) // Possible values are `allow` or `block`. // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent using devices of type `gpu` "restricted.devices.gpu": isEitherAllowOrBlock, // gendoc:generate(entity=project, group=restricted, key=restricted.devices.usb) // Possible values are `allow` or `block`. // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent using devices of type `usb` "restricted.devices.usb": isEitherAllowOrBlock, // gendoc:generate(entity=project, group=restricted, key=restricted.devices.pci) // Possible values are `allow` or `block`. // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent using devices of type `pci` "restricted.devices.pci": isEitherAllowOrBlock, // gendoc:generate(entity=project, group=restricted, key=restricted.devices.proxy) // Possible values are `allow` or `block`. // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent using devices of type `proxy` "restricted.devices.proxy": isEitherAllowOrBlock, // gendoc:generate(entity=project, group=restricted, key=restricted.devices.nic) // Possible values are `allow`, `block`, or `managed`. // // - When set to `block`, this option prevents using all network devices. // - When set to `managed`, this option allows using network devices only if `network=` is set. // - When set to `allow`, there is no restriction on which network devices can be used. // --- // type: string // defaultdesc: `managed` // shortdesc: Which network devices can be used "restricted.devices.nic": isEitherAllowOrBlockOrManaged, // gendoc:generate(entity=project, group=restricted, key=restricted.devices.disk) // Possible values are `allow`, `block`, or `managed`. // // - When set to `block`, this option prevents using all disk devices except the root one. // - When set to `managed`, this option allows using disk devices only if `pool=` is set. // - When set to `allow`, there is no restriction on which disk devices can be used. // --- // type: string // defaultdesc: `managed` // shortdesc: Which disk devices can be used "restricted.devices.disk": isEitherAllowOrBlockOrManaged, // gendoc:generate(entity=project, group=restricted, key=restricted.devices.disk.paths) // If {config:option}`project-restricted:restricted.devices.disk` is set to `allow`, this option controls which `source` can be used for `disk` devices. // Specify a comma-separated list of path prefixes that restrict the `source` setting. // If this option is left empty, all paths are allowed. // --- // type: string // shortdesc: Which `source` can be used for `disk` devices "restricted.devices.disk.paths": validate.Optional(validate.IsListOf(validate.IsAbsFilePath)), // gendoc:generate(entity=project, group=restricted, key=restricted.idmap.uid) // This option specifies the host UID ranges that are allowed in the instance's {config:option}`instance-raw:raw.idmap` setting. // --- // type: string // shortdesc: Which host UID ranges are allowed in `raw.idmap` "restricted.idmap.uid": validate.Optional(validate.IsListOf(validate.IsUint32Range)), // gendoc:generate(entity=project, group=restricted, key=restricted.idmap.gid) // This option specifies the host GID ranges that are allowed in the instance's {config:option}`instance-raw:raw.idmap` setting. // --- // type: string // shortdesc: Which host GID ranges are allowed in `raw.idmap` "restricted.idmap.gid": validate.Optional(validate.IsListOf(validate.IsUint32Range)), // gendoc:generate(entity=project, group=restricted, key=restricted.images.servers) // Specify a comma-delimited list of image servers domains that are allowed for use in this project. // If this option is not set, all image servers are accessible. // --- // type: string // shortdesc: Which image servers (HTTP host) are allowed for us in this project "restricted.images.servers": validate.Optional(validate.IsListOf(validate.IsAny)), // gendoc:generate(entity=project, group=restricted, key=restricted.networks.access) // Specify a comma-delimited list of network names that are allowed for use in this project. // If this option is not set, all networks are accessible. // // Note that this setting depends on the {config:option}`project-restricted:restricted.devices.nic` setting. // --- // type: string // shortdesc: Which network names are allowed for use in this project "restricted.networks.access": validate.Optional(validate.IsListOf(validate.IsAny)), // gendoc:generate(entity=project, group=restricted, key=restricted.networks.integrations) // Specify a comma-delimited list of network integrations that can be used by networks in this project. // --- // type: string // shortdesc: Which network integrations can be used in this project "restricted.networks.integrations": validate.IsListOf(validate.IsAny), // gendoc:generate(entity=project, group=restricted, key=restricted.networks.uplinks) // Specify a comma-delimited list of network names that can be used as uplink for networks in this project. // --- // type: string // shortdesc: Which network names can be used as uplink in this project "restricted.networks.uplinks": validate.Optional(validate.IsListOf(validate.IsAny)), // gendoc:generate(entity=project, group=restricted, key=restricted.networks.subnets) // Specify a comma-delimited list of network subnets from the uplink networks that are allocated for use in this project. // Use the form `:`. // --- // type: string // defaultdesc: `block` // shortdesc: Which network subnets are allocated for use in this project "restricted.networks.subnets": validate.Optional(func(value string) error { return projectValidateRestrictedSubnets(s, value) }), // gendoc:generate(entity=project, group=restricted, key=restricted.networks.zones) // Specify a comma-delimited list of network zones that can be used (or something under them) in this project. // --- // type: string // defaultdesc: `block` // shortdesc: Which network zones can be used in this project "restricted.networks.zones": validate.IsListOf(validate.IsAny), // gendoc:generate(entity=project, group=restricted, key=restricted.snapshots) // // --- // type: string // defaultdesc: `block` // shortdesc: Whether to prevent creating instance or volume snapshots "restricted.snapshots": isEitherAllowOrBlock, // gendoc:generate(entity=project, group=restricted, key=restricted.storage-pools.access) // Specify a comma-delimited list of storage pool names that are allowed for use in this project. // If this option is not set, all storage pools are accessible. // --- // type: string // shortdesc: Which storage pool names are allowed for use in this project "restricted.storage-pools.access": validate.Optional(validate.IsListOf(validate.IsAny)), } // Add the storage pool keys. err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Load all the pools. pools, err := tx.GetStoragePoolNames(ctx) if err != nil { return err } // Add the storage-pool specific config keys. for _, poolName := range pools { // gendoc:generate(entity=project, group=limits, key=limits.disk.pool.POOL_NAME) // This value is the maximum value of the aggregate disk // space used by all instance volumes, custom volumes, and images of the // project on this specific storage pool. // --- // type: string // shortdesc: Maximum disk space used by the project on this pool projectConfigKeys[fmt.Sprintf("limits.disk.pool.%s", poolName)] = validate.Optional(validate.IsSize) } return nil }) if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { return fmt.Errorf("Failed loading storage pool names: %w", err) } for k, v := range config { key := k // User keys are free for all. // gendoc:generate(entity=project, group=specific, key=user.*) // // --- // type: string // shortdesc: User-provided free-form key/value pairs if strings.HasPrefix(key, "user.") { continue } // Then validate. validator, ok := projectConfigKeys[key] if !ok { return fmt.Errorf("Invalid project configuration key %q", k) } err := validator(v) if err != nil { return fmt.Errorf("Invalid project configuration key %q value: %w", k, err) } } // Ensure that restricted projects have their own profiles. Otherwise restrictions in this project could // be bypassed by settings from the default project's profiles that are not checked against this project's // restrictions when they are configured. if util.IsTrue(config["restricted"]) && util.IsFalse(config["features.profiles"]) { return errors.New("Projects without their own profiles cannot be restricted") } return nil } func projectValidateName(name string) error { if strings.Contains(name, "_") { return errors.New("Project names may not contain underscores") } if strings.Contains(name, "'") || strings.Contains(name, `"`) { return errors.New("Project names may not contain quotes") } if name == "*" { return errors.New("Reserved project name") } if slices.Contains([]string{".", ".."}, name) { return fmt.Errorf("Invalid project name %q", name) } return nil } // projectValidateRestrictedSubnets checks that the project's restricted.networks.subnets are properly formatted // and are within the specified uplink network's routes. func projectValidateRestrictedSubnets(s *state.State, value string) error { for _, subnetRaw := range util.SplitNTrimSpace(value, ",", -1, false) { subnetParts := strings.SplitN(subnetRaw, ":", 2) if len(subnetParts) != 2 { return fmt.Errorf(`Subnet %q invalid, must be in the format of ":"`, subnetRaw) } uplinkName := subnetParts[0] subnetStr := subnetParts[1] restrictedSubnetIP, restrictedSubnet, err := net.ParseCIDR(subnetStr) if err != nil { return err } if restrictedSubnetIP.String() != restrictedSubnet.IP.String() { return fmt.Errorf("Not an IP network address %q", subnetStr) } var uplink *api.Network err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Check uplink exists and load config to compare subnets. _, uplink, _, err = tx.GetNetworkInAnyState(ctx, api.ProjectDefaultName, uplinkName) return err }) if err != nil { return fmt.Errorf("Invalid uplink network %q: %w", uplinkName, err) } // Parse uplink route subnets. var uplinkRoutes []*net.IPNet for _, k := range []string{"ipv4.routes", "ipv6.routes"} { if uplink.Config[k] == "" { continue } uplinkRoutes, err = network.SubnetParseAppend(uplinkRoutes, util.SplitNTrimSpace(uplink.Config[k], ",", -1, false)...) if err != nil { return err } } foundMatch := false // Check that the restricted subnet is within one of the uplink's routes. for _, uplinkRoute := range uplinkRoutes { if network.SubnetContains(uplinkRoute, restrictedSubnet) { foundMatch = true break } } if !foundMatch { return fmt.Errorf("Uplink network %q doesn't contain %q in its routes", uplinkName, restrictedSubnet.String()) } } return nil } // swagger:operation GET /1.0/projects/{name}/access projects project_access // // Get who has access to a project // // Gets the access information for the project. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Project name // type: string // required: true // responses: // "200": // description: Access // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Access" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func projectAccess(d *Daemon, r *http.Request) response.Response { s := d.State() name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Quick checks. err = validate.IsAPIName(name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid project name: %w", err)) } err = projectValidateName(name) if err != nil { return response.BadRequest(err) } // get the access struct access, err := s.Authorizer.GetProjectAccess(context.TODO(), name) if err != nil { return response.InternalError(err) } return response.SyncResponse(true, access) } incus-7.0.0/cmd/incusd/api_vsock.go000066400000000000000000000024201517523235500172020ustar00rootroot00000000000000package main import ( "context" "crypto/x509" "fmt" "net/http" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" ) func authenticateAgentCert(s *state.State, r *http.Request) (bool, instance.Instance, error) { var vsockID int trusted := false _, err := fmt.Sscanf(r.RemoteAddr, "vm(%d)", &vsockID) if err != nil { return false, nil, err } var clusterInst *cluster.Instance err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error clusterInst, err = tx.GetLocalInstanceWithVsockID(ctx, vsockID) if err != nil { return err } return nil }) if err != nil { return false, nil, err } inst, err := instance.LoadByProjectAndName(s, clusterInst.Project, clusterInst.Name) if err != nil { return false, nil, err } agentCert := inst.(instance.VM).AgentCertificate() for _, cert := range r.TLS.PeerCertificates { trusted, _ = localUtil.CheckTrustState(*cert, map[string]x509.Certificate{"0": *agentCert}, nil, false) if trusted { return true, inst, nil } } return false, nil, nil } incus-7.0.0/cmd/incusd/backup.go000066400000000000000000000657121517523235500165060ustar00rootroot00000000000000package main import ( "bytes" "context" "errors" "fmt" "io" "os" "time" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/internal/server/task" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // Create a new backup. func backupCreate(s *state.State, args db.InstanceBackup, sourceInst instance.Instance, op *operations.Operation, writer *io.PipeWriter) error { l := logger.AddContext(logger.Ctx{"project": sourceInst.Project().Name, "instance": sourceInst.Name(), "name": args.Name}) l.Debug("Instance backup started") defer l.Debug("Instance backup finished") reverter := revert.New() defer reverter.Fail() // Get storage pool. pool, err := storagePools.LoadByInstance(s, sourceInst) if err != nil { return fmt.Errorf("Failed loading instance storage pool: %w", err) } // Ignore requests for optimized backups when pool driver doesn't support it. if args.OptimizedStorage && !pool.Driver().Info().OptimizedBackups { args.OptimizedStorage = false } var b *backup.InstanceBackup if args.Name == "" { b = backup.NewInstanceBackup(s, sourceInst, 0, "", args.CreationDate, args.ExpiryDate, args.InstanceOnly, args.RootOnly, args.OptimizedStorage) } else { // Create the database entry. err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.CreateInstanceBackup(ctx, args) }) if err != nil { if errors.Is(err, db.ErrAlreadyDefined) { return fmt.Errorf("Backup %q already exists", args.Name) } return fmt.Errorf("Insert backup info into database: %w", err) } reverter.Add(func() { _ = s.DB.Cluster.Transaction(context.Background(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.DeleteInstanceBackup(ctx, args.Name) }) }) // Get the backup struct. b, err = instance.BackupLoadByName(s, sourceInst.Project().Name, args.Name) if err != nil { return fmt.Errorf("Load backup object: %w", err) } } // Detect compression method. var compress string b.SetCompressionAlgorithm(args.CompressionAlgorithm) if b.CompressionAlgorithm() != "" { compress = b.CompressionAlgorithm() } else { var p *api.Project err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { project, err := dbCluster.GetProject(ctx, tx.Tx(), sourceInst.Project().Name) if err != nil { return err } p, err = project.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return err } if p.Config["backups.compression_algorithm"] != "" { compress = p.Config["backups.compression_algorithm"] } else { compress = s.GlobalConfig.BackupsCompressionAlgorithm() } } // Setup the tarball writer. var tarFileWriter io.WriteCloser if writer == nil { // Create the target path if needed. backupsPath := internalUtil.VarPath("backups", "instances", project.Instance(sourceInst.Project().Name, sourceInst.Name())) if !util.PathExists(backupsPath) { err := os.MkdirAll(backupsPath, 0o700) if err != nil { return err } reverter.Add(func() { _ = os.Remove(backupsPath) }) } target := internalUtil.VarPath("backups", "instances", project.Instance(sourceInst.Project().Name, b.Name())) l.Debug("Opening backup tarball for writing", logger.Ctx{"path": target}) tarFileWriter, err = os.OpenFile(target, os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { return fmt.Errorf("Error opening backup tarball for writing %q: %w", target, err) } reverter.Add(func() { _ = os.Remove(target) }) } else { tarFileWriter = writer } defer func() { _ = tarFileWriter.Close() }() // Get IDMap to unshift container as the tarball is created. var idmapSet *idmap.Set if sourceInst.Type() == instancetype.Container { c := sourceInst.(instance.Container) idmapSet, err = c.DiskIdmap() if err != nil { return fmt.Errorf("Error getting container IDMAP: %w", err) } } // Create the tarball. tarPipeReader, tarPipeWriter := io.Pipe() defer func() { _ = tarPipeWriter.Close() }() // Ensure that go routine below always ends. tarWriter := instancewriter.NewInstanceTarWriter(tarPipeWriter, idmapSet) // Setup tar writer go routine, with optional compression. tarWriterRes := make(chan error) var compressErr error backupProgressWriter := &ioprogress.ProgressWriter{ Tracker: &ioprogress.ProgressTracker{ Handler: func(value, speed int64) { _ = op.ExtendMetadata(map[string]any{"create_backup_progress": fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(value, 2), units.GetByteSizeString(speed, 2))}) }, }, } go func(resCh chan<- error) { l.Debug("Started backup tarball writer") defer l.Debug("Finished backup tarball writer") if compress != "none" { backupProgressWriter.WriteCloser = tarFileWriter compressErr = compressFile(compress, tarPipeReader, backupProgressWriter) // If a compression error occurred, close the tarPipeWriter to end the export. if compressErr != nil { _ = tarPipeWriter.Close() } } else { backupProgressWriter.WriteCloser = tarFileWriter _, err = util.SafeCopy(backupProgressWriter, tarPipeReader) } resCh <- err }(tarWriterRes) // Write index file. l.Debug("Adding backup index file") err = backupWriteIndex(sourceInst, pool, b.OptimizedStorage(), !b.InstanceOnly(), !b.RootOnly(), tarWriter) // Check compression errors. if compressErr != nil { return compressErr } // Check backupWriteIndex for errors. if err != nil { return fmt.Errorf("Error writing backup index file: %w", err) } err = pool.BackupInstance(sourceInst, tarWriter, b.OptimizedStorage(), !b.InstanceOnly(), !b.RootOnly(), nil) if err != nil { return fmt.Errorf("Backup create: %w", err) } // Close off the tarball file. err = tarWriter.Close() if err != nil { return fmt.Errorf("Error closing tarball writer: %w", err) } // Close off the tarball pipe writer (this will end the go routine above). err = tarPipeWriter.Close() if err != nil { return fmt.Errorf("Error closing tarball pipe writer: %w", err) } err = <-tarWriterRes if err != nil { return fmt.Errorf("Error writing tarball: %w", err) } err = tarFileWriter.Close() if err != nil { return fmt.Errorf("Error closing tar file: %w", err) } reverter.Success() s.Events.SendLifecycle(sourceInst.Project().Name, lifecycle.InstanceBackupCreated.Event(args.Name, b.Instance(), nil)) return nil } // backupWriteIndex generates an index.yaml file and then writes it to the root of the backup tarball. func backupWriteIndex(sourceInst instance.Instance, pool storagePools.Pool, optimized bool, snapshots bool, dependentVolumes bool, tarWriter *instancewriter.InstanceTarWriter) error { // Indicate whether the driver will include a driver-specific optimized header. poolDriverOptimizedHeader := false if optimized { poolDriverOptimizedHeader = pool.Driver().Info().OptimizedBackupHeader } backupType := backup.InstanceTypeToBackupType(api.InstanceType(sourceInst.Type().String())) if backupType == backup.TypeUnknown { return errors.New("Unrecognised instance type for backup type conversion") } // We only write backup files out for actual instances. if sourceInst.IsSnapshot() { return errors.New("Cannot generate backup config for snapshots") } // Immediately return if the instance directory doesn't exist yet. if !util.PathExists(sourceInst.Path()) { return os.ErrNotExist } config, err := pool.GenerateInstanceBackupConfig(sourceInst, snapshots, dependentVolumes, nil) if err != nil { return fmt.Errorf("Failed generating instance backup config: %w", err) } indexInfo := backup.Info{ Name: sourceInst.Name(), Pool: pool.Name(), Backend: pool.Driver().Info().Name, Type: backupType, OptimizedStorage: &optimized, OptimizedHeader: &poolDriverOptimizedHeader, Config: config, } if snapshots { indexInfo.Snapshots = make([]string, 0, len(config.Snapshots)) for _, s := range config.Snapshots { indexInfo.Snapshots = append(indexInfo.Snapshots, s.Name) } } // Convert to YAML. indexData, err := yaml.Dump(&indexInfo, yaml.V2) if err != nil { return err } r := bytes.NewReader(indexData) indexFileInfo := instancewriter.FileInfo{ FileName: "backup/index.yaml", FileSize: int64(len(indexData)), FileMode: 0o644, FileModTime: time.Now(), } // Write to tarball. err = tarWriter.WriteFileFromReader(r, &indexFileInfo) if err != nil { return err } return nil } func pruneExpiredBackupsTask(d *Daemon) (task.Func, task.Schedule) { f := func(ctx context.Context) { s := d.State() opRun := func(op *operations.Operation) error { err := pruneExpiredInstanceBackups(ctx, s) if err != nil { return fmt.Errorf("Failed pruning expired instance backups: %w", err) } err = pruneExpiredStorageVolumeBackups(ctx, s) if err != nil { return fmt.Errorf("Failed pruning expired storage volume backups: %w", err) } err = pruneExpiredStorageBucketBackups(ctx, s) if err != nil { return fmt.Errorf("Failed pruning expired storage bucket backups: %w", err) } return nil } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.BackupsExpire, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating expired backups operation", logger.Ctx{"err": err}) return } logger.Info("Pruning expired backups") err = op.Start() if err != nil { logger.Error("Failed starting expired backups operation", logger.Ctx{"err": err}) return } err = op.Wait(ctx) if err != nil { logger.Error("Failed pruning expired backups", logger.Ctx{"err": err}) return } logger.Info("Done pruning expired backups") } f(context.Background()) first := true schedule := func() (time.Duration, error) { interval := time.Hour if first { first = false return interval, task.ErrSkip } return interval, nil } return f, schedule } func pruneExpiredInstanceBackups(ctx context.Context, s *state.State) error { var backups []db.InstanceBackup // Get the list of expired backups. err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error backups, err = tx.GetExpiredInstanceBackups(ctx) return err }) if err != nil { return fmt.Errorf("Unable to retrieve the list of expired instance backups: %w", err) } for _, b := range backups { inst, err := instance.LoadByID(s, b.InstanceID) if err != nil { return fmt.Errorf("Error loading instance for deleting backup %q: %w", b.Name, err) } instBackup := backup.NewInstanceBackup(s, inst, b.ID, b.Name, b.CreationDate, b.ExpiryDate, b.InstanceOnly, b.RootOnly, b.OptimizedStorage) err = instBackup.Delete() if err != nil { return fmt.Errorf("Error deleting instance backup %q: %w", b.Name, err) } } return nil } func volumeBackupCreate(s *state.State, args db.StoragePoolVolumeBackup, projectName string, poolName string, volumeName string, writer *io.PipeWriter) error { l := logger.AddContext(logger.Ctx{"project": projectName, "storage_volume": volumeName, "name": args.Name}) l.Debug("Volume backup started") defer l.Debug("Volume backup finished") reverter := revert.New() defer reverter.Fail() // Get storage pool. pool, err := storagePools.LoadByName(s, poolName) if err != nil { return fmt.Errorf("Failed loading storage pool %q: %w", poolName, err) } // Get the DB volume. volume, err := storagePools.VolumeDBGet(pool, projectName, volumeName, drivers.VolumeTypeCustom) if err != nil { return fmt.Errorf("Failed getting volume record: %w", err) } contentDBType, err := storagePools.VolumeContentTypeNameToContentType(volume.ContentType) if err != nil { return err } contentType, err := storagePools.VolumeDBContentTypeToContentType(contentDBType) if err != nil { return err } // Ignore requests for optimized backups when pool driver doesn't support it, or when backing up ISO volumes. if args.OptimizedStorage && (!pool.Driver().Info().OptimizedBackups || contentType == drivers.ContentTypeISO) { args.OptimizedStorage = false } var backupRow db.StoragePoolVolumeBackup if args.Name == "" { backupRow = args } else { // Create the database entry. err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.CreateStoragePoolVolumeBackup(ctx, args) }) if err != nil { if errors.Is(err, db.ErrAlreadyDefined) { return fmt.Errorf("Backup %q already exists", args.Name) } return fmt.Errorf("Failed creating backup record: %w", err) } reverter.Add(func() { _ = s.DB.Cluster.Transaction(context.Background(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.DeleteStoragePoolVolumeBackup(ctx, args.Name) }) }) err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { backupRow, err = tx.GetStoragePoolVolumeBackup(ctx, projectName, poolName, args.Name) return err }) if err != nil { return fmt.Errorf("Failed getting backup record: %w", err) } } // Detect compression method. var compress string backupRow.CompressionAlgorithm = args.CompressionAlgorithm if backupRow.CompressionAlgorithm != "" { compress = backupRow.CompressionAlgorithm } else { compress = s.GlobalConfig.BackupsCompressionAlgorithm() } // Setup the writer. var fileWriter io.WriteCloser if writer == nil { // Create the target path if needed. backupsPath := internalUtil.VarPath("backups", "custom", pool.Name(), project.StorageVolume(projectName, volumeName)) if !util.PathExists(backupsPath) { err := os.MkdirAll(backupsPath, 0o700) if err != nil { return err } reverter.Add(func() { _ = os.Remove(backupsPath) }) } target := internalUtil.VarPath("backups", "custom", pool.Name(), project.StorageVolume(projectName, backupRow.Name)) l.Debug("Opening backup file for writing", logger.Ctx{"path": target}) fileWriter, err = os.OpenFile(target, os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { return fmt.Errorf("Error opening backup file for writing %q: %w", target, err) } reverter.Add(func() { _ = os.Remove(target) }) } else { fileWriter = writer } defer func() { _ = fileWriter.Close() }() // If dealing with an ISO volume, we want to return it unaltered. if contentType == drivers.ContentTypeISO { err = pool.BackupCustomVolume(projectName, volumeName, instancewriter.NewInstanceRawWriter(fileWriter), backup.DefaultBackupPrefix, backupRow.OptimizedStorage, !backupRow.VolumeOnly, nil) if err != nil { return fmt.Errorf("Backup create: %w", err) } } else { // Create the tarball. tarPipeReader, tarPipeWriter := io.Pipe() defer func() { _ = tarPipeWriter.Close() }() // Ensure that go routine below always ends. tarWriter := instancewriter.NewInstanceTarWriter(tarPipeWriter, nil) // Setup tar writer go routine, with optional compression. tarWriterRes := make(chan error) var compressErr error go func(resCh chan<- error) { l.Debug("Started backup tarball writer") defer l.Debug("Finished backup tarball writer") if compress != "none" { compressErr = compressFile(compress, tarPipeReader, fileWriter) // If a compression error occurred, close the tarPipeWriter to end the export. if compressErr != nil { _ = tarPipeWriter.Close() } } else { _, err = util.SafeCopy(fileWriter, tarPipeReader) } resCh <- err }(tarWriterRes) // Write index file. l.Debug("Adding backup index file") err = volumeBackupWriteIndex(projectName, volumeName, pool, backupRow.OptimizedStorage, !backupRow.VolumeOnly, tarWriter) // Check compression errors. if compressErr != nil { return compressErr } // Check backupWriteIndex for errors. if err != nil { return fmt.Errorf("Error writing backup index file: %w", err) } err = pool.BackupCustomVolume(projectName, volumeName, tarWriter, backup.DefaultBackupPrefix, backupRow.OptimizedStorage, !backupRow.VolumeOnly, nil) if err != nil { return fmt.Errorf("Backup create: %w", err) } // Close off the tarball file. err = tarWriter.Close() if err != nil { return fmt.Errorf("Error closing tarball writer: %w", err) } // Close off the pipe writer (this will end the go routine above). err = tarPipeWriter.Close() if err != nil { return fmt.Errorf("Error closing tarball pipe writer: %w", err) } err = <-tarWriterRes if err != nil { return fmt.Errorf("Error writing tarball: %w", err) } } err = fileWriter.Close() if err != nil { return fmt.Errorf("Error closing backup file: %w", err) } reverter.Success() return nil } // volumeBackupWriteIndex generates an index.yaml file and then writes it to the root of the backup tarball. func volumeBackupWriteIndex(projectName string, volumeName string, pool storagePools.Pool, optimized bool, snapshots bool, tarWriter *instancewriter.InstanceTarWriter) error { // Indicate whether the driver will include a driver-specific optimized header. poolDriverOptimizedHeader := false if optimized { poolDriverOptimizedHeader = pool.Driver().Info().OptimizedBackupHeader } config, err := pool.GenerateCustomVolumeBackupConfig(projectName, volumeName, snapshots, nil) if err != nil { return fmt.Errorf("Failed generating volume backup config: %w", err) } indexInfo := backup.Info{ Name: config.Volume.Name, Pool: pool.Name(), Backend: pool.Driver().Info().Name, OptimizedStorage: &optimized, OptimizedHeader: &poolDriverOptimizedHeader, Type: backup.TypeCustom, Config: config, } if snapshots { indexInfo.Snapshots = make([]string, 0, len(config.VolumeSnapshots)) for _, s := range config.VolumeSnapshots { indexInfo.Snapshots = append(indexInfo.Snapshots, s.Name) } } // Convert to YAML. indexData, err := yaml.Dump(indexInfo, yaml.V2) if err != nil { return err } r := bytes.NewReader(indexData) indexFileInfo := instancewriter.FileInfo{ FileName: "backup/index.yaml", FileSize: int64(len(indexData)), FileMode: 0o644, FileModTime: time.Now(), } // Write to tarball. err = tarWriter.WriteFileFromReader(r, &indexFileInfo) if err != nil { return err } return nil } func pruneExpiredStorageVolumeBackups(ctx context.Context, s *state.State) error { var volumeBackups []*backup.VolumeBackup // Get the list of expired backups. err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { nodeID := tx.GetNodeID() backups, err := tx.GetExpiredStorageVolumeBackups(ctx) if err != nil { return fmt.Errorf("Unable to retrieve the list of expired storage volume backups: %w", err) } for _, b := range backups { vol, err := tx.GetStoragePoolVolumeWithID(ctx, int(b.VolumeID)) if err != nil { logger.Warn("Failed getting storage pool of backup", logger.Ctx{"backup": b.Name, "err": err}) continue } // Ignore volumes on other nodes, but include remote pools (NodeID == -1). if vol.NodeID != -1 && vol.NodeID != nodeID { continue } volBackup := backup.NewVolumeBackup(s, vol.ProjectName, vol.PoolName, vol.Name, b.ID, b.Name, b.CreationDate, b.ExpiryDate, b.VolumeOnly, b.OptimizedStorage) volumeBackups = append(volumeBackups, volBackup) } return nil }) if err != nil { return err } // The deletion is done outside of the transaction to avoid any unnecessary IO while inside of // the transaction. for _, b := range volumeBackups { err := b.Delete() if err != nil { return fmt.Errorf("Error deleting storage volume backup %q: %w", b.Name(), err) } } return nil } func bucketBackupCreate(s *state.State, args db.StoragePoolBucketBackup, projectName string, poolName string, bucketName string, writer *io.PipeWriter) error { l := logger.AddContext(logger.Ctx{"project": projectName, "storage_bucket": bucketName, "name": args.Name}) l.Debug("Bucket backup started") defer l.Debug("Bucket backup finished") reverter := revert.New() defer reverter.Fail() pool, err := storagePools.LoadByName(s, poolName) if err != nil { return fmt.Errorf("Failed loading storage pool %q: %w", poolName, err) } var backupRow db.StoragePoolBucketBackup if args.Name == "" { backupRow = args } else { // Create the database entry err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.CreateStoragePoolBucketBackup(ctx, args) }) if err != nil { if errors.Is(err, db.ErrAlreadyDefined) { return fmt.Errorf("Backup %q already exists", args.Name) } return fmt.Errorf("Failed creating backup record: %w", err) } reverter.Add(func() { _ = s.DB.Cluster.Transaction(context.Background(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.DeleteStoragePoolBucketBackup(ctx, args.Name) }) }) err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { backupRow, err = tx.GetStoragePoolBucketBackup(ctx, projectName, poolName, args.Name) return err }) if err != nil { return fmt.Errorf("Failed getting backup record: %w", err) } } // Detect compression method var compress string backupRow.CompressionAlgorithm = args.CompressionAlgorithm if backupRow.CompressionAlgorithm != "" { compress = backupRow.CompressionAlgorithm } else { compress = s.GlobalConfig.BackupsCompressionAlgorithm() } // Setup the tarball writer. var tarFileWriter io.WriteCloser if writer == nil { // Create the target path if needed. backupsPath := internalUtil.VarPath("backups", "buckets", pool.Name(), project.StorageBucket(projectName, bucketName)) if !util.PathExists(backupsPath) { err := os.MkdirAll(backupsPath, 0o700) if err != nil { return err } reverter.Add(func() { _ = os.Remove(backupsPath) }) } target := internalUtil.VarPath("backups", "buckets", pool.Name(), project.StorageBucket(projectName, backupRow.Name)) l.Debug("Opening backup tarball for writing", logger.Ctx{"path": target}) tarFileWriter, err = os.OpenFile(target, os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { return fmt.Errorf("Error opening backup tarball for writing %q: %w", target, err) } reverter.Add(func() { _ = os.Remove(target) }) } else { tarFileWriter = writer } defer func() { _ = tarFileWriter.Close() }() // Create the tarball. tarPipeReader, tarPipeWriter := io.Pipe() defer func() { _ = tarPipeWriter.Close() }() // Ensure that go routine below always ends. tarWriter := instancewriter.NewInstanceTarWriter(tarPipeWriter, nil) // Setup tar writer go routine, with optional compression. tarWriterRes := make(chan error) var compressErr error go func(resCh chan<- error) { l.Debug("Started backup tarball writer") defer l.Debug("Finished backup tarball writer") if compress != "none" { compressErr = compressFile(compress, tarPipeReader, tarFileWriter) // If a compression error occurred, close the tarPipeWriter to end the export. if compressErr != nil { _ = tarPipeWriter.Close() } } else { _, err = util.SafeCopy(tarFileWriter, tarPipeReader) } resCh <- err }(tarWriterRes) // Write index file. l.Debug("Adding backup index file") err = bucketBackupWriteIndex(projectName, bucketName, pool, tarWriter) // Check compression errors. if compressErr != nil { return compressErr } // Check backupWriteIndex for errors. if err != nil { return fmt.Errorf("Error writing backup index file: %w", err) } err = pool.BackupBucket(projectName, bucketName, tarWriter, nil) if err != nil { return fmt.Errorf("Backup create: %w", err) } // Close off the tarball file. err = tarWriter.Close() if err != nil { return fmt.Errorf("Error closing tarball writer: %w", err) } // Close off the tarball pipe writer (this will end the go routine above). err = tarPipeWriter.Close() if err != nil { return fmt.Errorf("Error closing tarball pipe writer: %w", err) } err = <-tarWriterRes if err != nil { return fmt.Errorf("Error writing tarball: %w", err) } err = tarFileWriter.Close() if err != nil { return fmt.Errorf("Error closing tar file: %w", err) } reverter.Success() return nil } // bucketBackupWriteIndex generates an index.yaml file and then writes it to the root of the backup tarball. func bucketBackupWriteIndex(projectName string, bucketName string, pool storagePools.Pool, tarWriter *instancewriter.InstanceTarWriter) error { config, err := pool.GenerateBucketBackupConfig(projectName, bucketName, nil) if err != nil { return fmt.Errorf("Failed generating storage backup config: %w", err) } indexInfo := backup.Info{ Name: config.Bucket.Name, Pool: pool.Name(), Backend: pool.Driver().Info().Name, Type: backup.TypeBucket, Config: config, } // Convert to YAML. indexData, err := yaml.Dump(indexInfo, yaml.V2) if err != nil { return err } r := bytes.NewReader(indexData) indexFileInfo := instancewriter.FileInfo{ FileName: "backup/index.yaml", FileSize: int64(len(indexData)), FileMode: 0o644, FileModTime: time.Now(), } // Write to tarball. err = tarWriter.WriteFileFromReader(r, &indexFileInfo) if err != nil { return err } return nil } func pruneExpiredStorageBucketBackups(ctx context.Context, s *state.State) error { var bucketBackups []*backup.BucketBackup // Get the list of expired backups. err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { backups, err := tx.GetExpiredStorageBucketBackups(ctx) if err != nil { return fmt.Errorf("Unable to retrieve the list of expired storage bucket backups: %w", err) } for _, b := range backups { bucket, err := tx.GetStoragePoolBucketWithID(ctx, int(b.BucketID)) if err != nil { logger.Warn("Failed getting storage pool of backup", logger.Ctx{"backup": b.Name, "err": err}) continue } bucketBackup := backup.NewBucketBackup(s, bucket.Project, bucket.PoolName, bucket.Name, b.ID, b.Name, b.CreationDate, b.ExpiryDate) bucketBackups = append(bucketBackups, bucketBackup) } return nil }) if err != nil { return err } // The deletion is done outside of the transaction to avoid any unnecessary IO while inside of // the transaction. for _, b := range bucketBackups { err := b.Delete() if err != nil { return fmt.Errorf("Error deleting storage volume backup %q: %w", b.Name(), err) } } return nil } incus-7.0.0/cmd/incusd/certificates.go000066400000000000000000001114261517523235500177000ustar00rootroot00000000000000package main import ( "context" "crypto/rsa" "crypto/x509" "encoding/base64" "encoding/json" "encoding/pem" "errors" "fmt" "net" "net/http" "net/url" "time" "github.com/gorilla/mux" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/filter" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/certificate" "github.com/lxc/incus/v7/internal/server/cluster" clusterRequest "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" localtls "github.com/lxc/incus/v7/shared/tls" ) var certificatesCmd = APIEndpoint{ Path: "certificates", Get: APIEndpointAction{Handler: certificatesGet, AccessHandler: allowAuthenticated}, Post: APIEndpointAction{Handler: certificatesPost, AllowUntrusted: true}, } var certificateCmd = APIEndpoint{ Path: "certificates/{fingerprint}", Delete: APIEndpointAction{Handler: certificateDelete, AccessHandler: allowAuthenticated}, Get: APIEndpointAction{Handler: certificateGet, AccessHandler: allowPermission(auth.ObjectTypeCertificate, auth.EntitlementCanView, "fingerprint")}, Patch: APIEndpointAction{Handler: certificatePatch, AccessHandler: allowAuthenticated}, Put: APIEndpointAction{Handler: certificatePut, AccessHandler: allowAuthenticated}, } // swagger:operation GET /1.0/certificates certificates certificates_get // // Get the trusted certificates // // Returns a list of trusted certificates (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/certificates/390fdd27ed5dc2408edc11fe602eafceb6c025ddbad9341dfdcb1056a8dd98b1", // "/1.0/certificates/22aee3f051f96abe6d7756892eecabf4b4b22e2ba877840a4ca981e9ea54030a" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/certificates?recursion=1 certificates certificates_get_recursion1 // // Get the trusted certificates // // Returns a list of trusted certificates (structs). // // --- // produces: // - application/json // parameters: // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of certificates // items: // $ref: "#/definitions/Certificate" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func certificatesGet(d *Daemon, r *http.Request) response.Response { s := d.State() userHasPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanView, auth.ObjectTypeCertificate) if err != nil { return response.SmartError(err) } recursion := localUtil.IsRecursionRequest(r) // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } mustLoadObjects := recursion || (clauses != nil && len(clauses.Clauses) > 0) linkResults := make([]string, 0) fullResults := make([]api.Certificate, 0) if mustLoadObjects { var baseCerts []dbCluster.Certificate var err error err = d.State().DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { baseCerts, err = dbCluster.GetCertificates(ctx, tx.Tx()) if err != nil { return err } for _, baseCert := range baseCerts { if !userHasPermission(auth.ObjectCertificate(baseCert.Fingerprint)) { continue } apiCert, err := baseCert.ToAPI(ctx, tx.Tx()) if err != nil { return err } if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(*apiCert, *clauses) if err != nil { return err } if !match { continue } } fullResults = append(fullResults, *apiCert) certificateURL := fmt.Sprintf("/%s/certificates/%s", version.APIVersion, apiCert.Fingerprint) linkResults = append(linkResults, certificateURL) } return nil }) if err != nil { return response.SmartError(err) } } else { trustedCertificates, err := d.getTrustedCertificates() if err != nil { return response.SmartError(err) } for _, certs := range trustedCertificates { for _, cert := range certs { fingerprint := localtls.CertFingerprint(&cert) if !userHasPermission(auth.ObjectCertificate(fingerprint)) { continue } certificateURL := fmt.Sprintf("/%s/certificates/%s", version.APIVersion, fingerprint) linkResults = append(linkResults, certificateURL) } } } if recursion { return response.SyncResponse(true, fullResults) } return response.SyncResponse(true, linkResults) } func updateCertificateCache(d *Daemon) { s := d.State() logger.Debug("Refreshing trusted certificate cache") var certs []*api.Certificate var dbCerts []dbCluster.Certificate var localCerts []dbCluster.Certificate var err error err = s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { dbCerts, err = dbCluster.GetCertificates(ctx, tx.Tx()) if err != nil { return err } certs = make([]*api.Certificate, len(dbCerts)) for i, c := range dbCerts { certs[i], err = c.ToAPI(ctx, tx.Tx()) if err != nil { return err } if c.Type == certificate.TypeServer { localCerts = append(localCerts, c) } } return nil }) if err != nil { logger.Warn("Failed reading certificates from global database", logger.Ctx{"err": err}) return } // Write out the server certs to the local database to allow the cluster to restart. err = s.DB.Node.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.NodeTx) error { return tx.ReplaceCertificates(localCerts) }) if err != nil { logger.Warn("Failed writing certificates to local database", logger.Ctx{"err": err}) // Don't return here, as we still should update the in-memory cache to allow the cluster to // continue functioning, and hopefully the write will succeed on next update. } d.clientCerts.SetCertificates(certs) } // updateCertificateCacheFromLocal loads trusted server certificates from local database into memory. func updateCertificateCacheFromLocal(d *Daemon) error { s := d.State() logger.Debug("Refreshing local trusted certificate cache") var certs []*api.Certificate var dbCerts []dbCluster.Certificate var err error err = s.DB.Node.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.NodeTx) error { dbCerts, err = tx.GetCertificates(ctx) if err != nil { return err } certs = make([]*api.Certificate, len(dbCerts)) for i, c := range dbCerts { certs[i], err = c.ToAPI(ctx, nil) if err != nil { return err } } return nil }) if err != nil { return fmt.Errorf("Failed reading certificates from local database: %w", err) } d.clientCerts.SetCertificates(certs) return nil } // clusterMemberJoinTokenValid searches for cluster join token that matches the join token provided. // Returns matching operation if found and cancels the operation, otherwise returns nil. func clusterMemberJoinTokenValid(s *state.State, r *http.Request, projectName string, joinToken *api.ClusterMemberJoinToken) (*api.Operation, error) { ops, err := operationsGetByType(s, r, projectName, operationtype.ClusterJoinToken) if err != nil { return nil, fmt.Errorf("Failed getting cluster join token operations: %w", err) } var foundOp *api.Operation for _, op := range ops { if op.StatusCode != api.Running { continue // Tokens are single use, so if cancelled but not deleted yet its not available. } if op.Resources == nil { continue } opSecret, ok := op.Metadata["secret"] if !ok { continue } opServerName, ok := op.Metadata["serverName"] if !ok { continue } if opServerName == joinToken.ServerName && opSecret == joinToken.Secret { foundOp = op break } } if foundOp != nil { // Token is single-use, so cancel it now. err = operationCancel(s, r, projectName, foundOp) if err != nil { return nil, fmt.Errorf("Failed to cancel operation %q: %w", foundOp.ID, err) } expiresAt, ok := foundOp.Metadata["expiresAt"] if ok { var expiry time.Time // Depending on whether it's a local operation or not, expiry will either be a time.Time or a string. if s.ServerName == foundOp.Location { expiry, _ = expiresAt.(time.Time) } else { expiry, _ = time.Parse(time.RFC3339Nano, expiresAt.(string)) } // Check if token has expired. if time.Now().After(expiry) { return nil, api.StatusErrorf(http.StatusForbidden, "Token has expired") } } return foundOp, nil } // No operation found. return nil, nil } // certificateTokenValid searches for certificate token that matches the add token provided. // Returns matching operation if found and cancels the operation, otherwise returns nil. func certificateTokenValid(s *state.State, r *http.Request, addToken *api.CertificateAddToken) (*api.Operation, error) { ops, err := operationsGetByType(s, r, api.ProjectDefaultName, operationtype.CertificateAddToken) if err != nil { return nil, fmt.Errorf("Failed getting certificate token operations: %w", err) } var foundOp *api.Operation for _, op := range ops { if op.StatusCode != api.Running { continue // Tokens are single use, so if cancelled but not deleted yet its not available. } opSecret, ok := op.Metadata["secret"] if !ok { continue } if opSecret == addToken.Secret { foundOp = op break } } if foundOp != nil { // Token is single-use, so cancel it now. err = operationCancel(s, r, api.ProjectDefaultName, foundOp) if err != nil { return nil, fmt.Errorf("Failed to cancel operation %q: %w", foundOp.ID, err) } expiresAt, ok := foundOp.Metadata["expiresAt"] if ok { expiry, _ := expiresAt.(time.Time) // Check if token has expired. if time.Now().After(expiry) { return nil, api.StatusErrorf(http.StatusForbidden, "Token has expired") } } return foundOp, nil } // No operation found. return nil, nil } // swagger:operation POST /1.0/certificates?public certificates certificates_post_untrusted // // Add a trusted certificate // // Adds a certificate to the trust store as an untrusted user. // In this mode, the `token` property must be set to the correct value. // // The `certificate` field can be omitted in which case the TLS client // certificate in use for the connection will be retrieved and added to the // trust store. // // The `?public` part of the URL isn't required, it's simply used to // separate the two behaviors of this endpoint. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: body // name: certificate // description: Certificate // required: true // schema: // $ref: "#/definitions/CertificatesPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation POST /1.0/certificates certificates certificates_post // // Add a trusted certificate // // Adds a certificate to the trust store. // In this mode, the `token` property is always ignored. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: body // name: certificate // description: Certificate // required: true // schema: // $ref: "#/definitions/CertificatesPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func certificatesPost(d *Daemon, r *http.Request) response.Response { s := d.State() // Parse the request. req := api.CertificatesPost{} err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } localHTTPSAddress := s.LocalConfig.HTTPSAddress() // Quick check. if req.Token && req.Certificate != "" { return response.BadRequest(errors.New("Can't use certificate if token is requested")) } if req.Token { if req.Type != "client" { return response.BadRequest(errors.New("Tokens can only be issued for client certificates")) } if localHTTPSAddress == "" { return response.BadRequest(errors.New("Can't issue token when server isn't listening on network")) } } // Access check. // Check if the user is already trusted. trusted, _, _, err := d.Authenticate(nil, r) if err != nil { return response.SmartError(err) } // User isn't an admin and is already trusted, can't add more certs. if trusted && req.Certificate == "" && !req.Token { return response.BadRequest(errors.New("Client is already trusted")) } // Handle requests by non-admin users. var userCanCreateCertificates bool err = s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectServer(), auth.EntitlementCanCreateCertificates) if err == nil { userCanCreateCertificates = true } else if !api.StatusErrorCheck(err, http.StatusForbidden) { return response.SmartError(err) } if !trusted || !userCanCreateCertificates { // Non-admin cannot issue tokens. if req.Token { return response.Forbidden(nil) } // A token is required for non-admin users. if req.TrustToken == "" { return response.Forbidden(nil) } // Check if cluster member join token supplied as token. joinToken, err := internalUtil.JoinTokenDecode(req.TrustToken) if err == nil { // If so then check there is a matching join operation. joinOp, err := clusterMemberJoinTokenValid(s, r, api.ProjectDefaultName, joinToken) if err != nil { return response.InternalError(fmt.Errorf("Failed during search for join token operation: %w", err)) } if joinOp == nil { return response.Forbidden(errors.New("No matching cluster join operation found")) } } else { // Check if certificate add token supplied as token. joinToken, err := localtls.CertificateTokenDecode(req.TrustToken) if err == nil { // If so then check there is a matching join operation. joinOp, err := certificateTokenValid(s, r, joinToken) if err != nil { return response.InternalError(fmt.Errorf("Failed during search for certificate add token operation: %w", err)) } if joinOp == nil { return response.Forbidden(errors.New("No matching certificate add operation found")) } // Create a new request from the token data as the user isn't allowed to override anything. req = api.CertificatesPost{} switch tokenReq := joinOp.Metadata["request"].(type) { case api.CertificatesPost: req.Name = tokenReq.Name req.Type = tokenReq.Type req.Restricted = tokenReq.Restricted req.Projects = tokenReq.Projects case map[string]any: req.Name = tokenReq["name"].(string) req.Type = tokenReq["type"].(string) req.Restricted = tokenReq["restricted"].(bool) for _, project := range tokenReq["projects"].([]any) { req.Projects = append(req.Projects, project.(string)) } default: return response.InternalError(errors.New("Bad certificate add operation data")) } } else { return response.Forbidden(nil) } } } dbReqType, err := certificate.FromAPIType(req.Type) if err != nil { return response.BadRequest(err) } // Extract the certificate. var cert *x509.Certificate if req.Certificate != "" { var der []byte // Try to parse as PEM. block, rest := pem.Decode([]byte(req.Certificate)) if block != nil { der = block.Bytes } else { data, err := base64.StdEncoding.DecodeString(string(rest)) if err != nil { return response.BadRequest(err) } der = data } cert, err = x509.ParseCertificate(der) if err != nil { return response.BadRequest(fmt.Errorf("Invalid certificate material: %w", err)) } } else if req.Token { // Get all addresses the server is listening on. This is encoded in the certificate token, // so that the client will not have to specify a server address. The client will iterate // through all these addresses until it can connect to one of them. addresses, err := localUtil.ListenAddresses(localHTTPSAddress) if err != nil { return response.InternalError(err) } // Generate join secret for new client. This will be stored inside the join token operation and will be // supplied by the joining client (encoded inside the join token) which will allow us to lookup the correct // operation in order to validate the requested joining client name is correct and authorised. joinSecret, err := internalUtil.RandomHexString(32) if err != nil { return response.InternalError(err) } // Generate fingerprint of network certificate so joining member can automatically trust the correct // certificate when it is presented during the join process. fingerprint, err := localtls.CertFingerprintStr(string(s.Endpoints.NetworkPublicKey())) if err != nil { return response.InternalError(err) } if req.Projects == nil { req.Projects = []string{} } meta := map[string]any{ "secret": joinSecret, "fingerprint": fingerprint, "addresses": addresses, "request": req, } // If tokens should expire, add the expiry date to the op's metadata. expiry := s.GlobalConfig.RemoteTokenExpiry() if expiry != "" { expiresAt, err := internalInstance.GetExpiry(time.Now(), expiry) if err != nil { return response.InternalError(err) } meta["expiresAt"] = expiresAt } op, err := operations.OperationCreate(s, api.ProjectDefaultName, operations.OperationClassToken, operationtype.CertificateAddToken, nil, meta, nil, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } else if r.TLS != nil { // Add client's certificate. if len(r.TLS.PeerCertificates) < 1 { // This can happen if the client doesn't send a client certificate or if the server is in // CA mode. We rely on this check to prevent non-CA trusted client certificates from being // added when in CA mode. return response.BadRequest(errors.New("No client certificate provided")) } cert = r.TLS.PeerCertificates[len(r.TLS.PeerCertificates)-1] } else { return response.BadRequest(errors.New("Can't use TLS data on non-TLS link")) } // Check validity. err = certificateValidate(cert) if err != nil { return response.BadRequest(err) } // Calculate the fingerprint. fingerprint := localtls.CertFingerprint(cert) // Figure out a name. name := req.Name if name == "" { // Try to pull the CN. name = cert.Subject.CommonName if name == "" { // Fallback to the client's IP address. remoteHost, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { return response.InternalError(err) } name = remoteHost } } if !isClusterNotification(r) { err := s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Check if we already have the certificate. existingCert, _ := dbCluster.GetCertificateByFingerprintPrefix(ctx, tx.Tx(), fingerprint) if existingCert != nil { return api.StatusErrorf(http.StatusConflict, "Certificate already in trust store") } // Store the certificate in the cluster database. dbCert := dbCluster.Certificate{ Fingerprint: localtls.CertFingerprint(cert), Type: dbReqType, Name: name, Certificate: string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})), Restricted: req.Restricted, Description: req.Description, } _, err := dbCluster.CreateCertificateWithProjects(ctx, tx.Tx(), dbCert, req.Projects) return err }) if err != nil { return response.SmartError(err) } // Notify other nodes about the new certificate. notifier, err := cluster.NewNotifier(s, s.Endpoints.NetworkCert(), s.ServerCert(), cluster.NotifyAlive) if err != nil { return response.SmartError(err) } req := api.CertificatesPost{ CertificatePut: api.CertificatePut{ Certificate: base64.StdEncoding.EncodeToString(cert.Raw), Name: name, Type: api.CertificateTypeClient, }, } err = notifier(func(client incus.InstanceServer) error { return client.CreateCertificate(req) }) if err != nil { return response.SmartError(err) } // Add the certificate resource to the authorizer. err = s.Authorizer.AddCertificate(r.Context(), fingerprint) if err != nil { logger.Error("Failed to add certificate to authorizer", logger.Ctx{"fingerprint": fingerprint, "error": err}) } } // Reload the cache. s.UpdateCertificateCache() lc := lifecycle.CertificateCreated.Event(fingerprint, request.CreateRequestor(r), nil) s.Events.SendLifecycle(api.ProjectDefaultName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation GET /1.0/certificates/{fingerprint} certificates certificate_get // // Get the trusted certificate // // Gets a specific certificate entry from the trust store. // // --- // produces: // - application/json // parameters: // - in: path // name: fingerprint // description: Fingerprint // type: string // required: true // responses: // "200": // description: Certificate // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Certificate" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func certificateGet(d *Daemon, r *http.Request) response.Response { fingerprint, err := url.PathUnescape(mux.Vars(r)["fingerprint"]) if err != nil { return response.SmartError(err) } var cert *api.Certificate err = d.State().DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbCertInfo, err := dbCluster.GetCertificateByFingerprintPrefix(ctx, tx.Tx(), fingerprint) if err != nil { return err } cert, err = dbCertInfo.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return response.SmartError(err) } return response.SyncResponseETag(true, cert, cert) } // swagger:operation PUT /1.0/certificates/{fingerprint} certificates certificate_put // // Update the trusted certificate // // Updates the entire certificate configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: fingerprint // description: Fingerprint // type: string // required: true // - in: body // name: certificate // description: Certificate configuration // required: true // schema: // $ref: "#/definitions/CertificatePut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func certificatePut(d *Daemon, r *http.Request) response.Response { fingerprint, err := url.PathUnescape(mux.Vars(r)["fingerprint"]) if err != nil { return response.SmartError(err) } // Get current database record. var apiEntry *api.Certificate err = d.State().DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { oldEntry, err := dbCluster.GetCertificateByFingerprintPrefix(ctx, tx.Tx(), fingerprint) if err != nil { return err } apiEntry, err = oldEntry.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return response.SmartError(err) } // Validate the ETag. err = localUtil.EtagCheck(r, apiEntry) if err != nil { return response.PreconditionFailed(err) } // Parse the request. req := api.CertificatePut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) // Apply the update. return doCertificateUpdate(d, *apiEntry, req, clientType, r) } // swagger:operation PATCH /1.0/certificates/{fingerprint} certificates certificate_patch // // Partially update the trusted certificate // // Updates a subset of the certificate configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: fingerprint // description: Fingerprint // type: string // required: true // - in: body // name: certificate // description: Certificate configuration // required: true // schema: // $ref: "#/definitions/CertificatePut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func certificatePatch(d *Daemon, r *http.Request) response.Response { fingerprint, err := url.PathUnescape(mux.Vars(r)["fingerprint"]) if err != nil { return response.SmartError(err) } // Get current database record. var apiEntry *api.Certificate err = d.State().DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { oldEntry, err := dbCluster.GetCertificateByFingerprintPrefix(ctx, tx.Tx(), fingerprint) if err != nil { return err } apiEntry, err = oldEntry.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return response.SmartError(err) } // Validate the ETag. err = localUtil.EtagCheck(r, apiEntry) if err != nil { return response.PreconditionFailed(err) } // Apply the changes. req := *apiEntry err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) return doCertificateUpdate(d, *apiEntry, req.Writable(), clientType, r) } func doCertificateUpdate(d *Daemon, dbInfo api.Certificate, req api.CertificatePut, clientType clusterRequest.ClientType, r *http.Request) response.Response { s := d.State() if clientType == clusterRequest.ClientTypeNormal { // Prevent any type change. if dbInfo.Type != req.Type { return response.BadRequest(errors.New("The certificate type cannot be changed")) } reqDBType, err := certificate.FromAPIType(req.Type) if err != nil { return response.BadRequest(err) } // Convert to the database type. dbCert := dbCluster.Certificate{ Certificate: dbInfo.Certificate, Fingerprint: dbInfo.Fingerprint, Restricted: req.Restricted, Name: req.Name, Type: reqDBType, Description: req.Description, } var userCanEditCertificate bool err = s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectCertificate(dbInfo.Fingerprint), auth.EntitlementCanEdit) if err == nil { userCanEditCertificate = true } else if !api.StatusErrorCheck(err, http.StatusForbidden) { return response.SmartError(err) } // Non-admins are able to change their own certificate but no other fields. // In order to prevent possible future security issues, the certificate information is // reset in case a non-admin user is performing the update. certProjects := req.Projects if !userCanEditCertificate { if r.TLS == nil { response.Forbidden(errors.New("Cannot update certificate information")) } // Ensure the user in not trying to change fields other than the certificate. if dbInfo.Restricted != req.Restricted || dbInfo.Name != req.Name || len(dbInfo.Projects) != len(req.Projects) { return response.Forbidden(errors.New("Only the certificate can be changed")) } for i := range dbInfo.Projects { if dbInfo.Projects[i] != req.Projects[i] { return response.Forbidden(errors.New("Only the certificate can be changed")) } } // Reset dbCert in order to prevent possible future security issues. dbCert = dbCluster.Certificate{ Certificate: dbInfo.Certificate, Fingerprint: dbInfo.Fingerprint, Restricted: dbInfo.Restricted, Name: dbInfo.Name, Type: reqDBType, Description: req.Description, } certProjects = dbInfo.Projects if req.Certificate != "" && dbInfo.Certificate != req.Certificate { certBlock, _ := pem.Decode([]byte(dbInfo.Certificate)) oldCert, err := x509.ParseCertificate(certBlock.Bytes) if err != nil { // This should not happen return response.InternalError(err) } trustedCerts := map[string]x509.Certificate{ dbInfo.Name: *oldCert, } trusted := false for _, i := range r.TLS.PeerCertificates { trusted, _ = localUtil.CheckTrustState(*i, trustedCerts, s.Endpoints.NetworkCert(), false) if trusted { break } } if !trusted { return response.Forbidden(errors.New("Certificate cannot be changed")) } } } if req.Certificate != "" && dbInfo.Certificate != req.Certificate { // Add supplied certificate. block, _ := pem.Decode([]byte(req.Certificate)) if block == nil { return response.BadRequest(errors.New("Invalid PEM encoded certificate")) } cert, err := x509.ParseCertificate(block.Bytes) if err != nil { return response.BadRequest(fmt.Errorf("Invalid certificate material: %w", err)) } dbCert.Certificate = string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})) dbCert.Fingerprint = localtls.CertFingerprint(cert) // Check validity. err = certificateValidate(cert) if err != nil { return response.BadRequest(err) } } // Update the database record. err = s.DB.UpdateCertificate(context.Background(), dbInfo.Fingerprint, dbCert, certProjects) if err != nil { return response.SmartError(err) } // Notify other nodes about the new certificate. notifier, err := cluster.NewNotifier(s, s.Endpoints.NetworkCert(), s.ServerCert(), cluster.NotifyAlive) if err != nil { return response.SmartError(err) } err = notifier(func(client incus.InstanceServer) error { return client.UpdateCertificate(dbCert.Fingerprint, req, "") }) if err != nil { return response.SmartError(err) } } // Reload the cache. s.UpdateCertificateCache() s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.CertificateUpdated.Event(dbInfo.Fingerprint, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation DELETE /1.0/certificates/{fingerprint} certificates certificate_delete // // Delete the trusted certificate // // Removes the certificate from the trust store. // // --- // produces: // - application/json // parameters: // - in: path // name: fingerprint // description: Fingerprint // type: string // required: true // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func certificateDelete(d *Daemon, r *http.Request) response.Response { s := d.State() fingerprint, err := url.PathUnescape(mux.Vars(r)["fingerprint"]) if err != nil { return response.SmartError(err) } if !isClusterNotification(r) { var certInfo *dbCluster.Certificate err := s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Get current database record. var err error certInfo, err = dbCluster.GetCertificateByFingerprintPrefix(ctx, tx.Tx(), fingerprint) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } var userCanEditCertificate bool err = s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectCertificate(certInfo.Fingerprint), auth.EntitlementCanEdit) if err == nil { userCanEditCertificate = true } else if api.StatusErrorCheck(err, http.StatusForbidden) { return response.SmartError(err) } // Non-admins are able to delete only their own certificate. if !userCanEditCertificate { if r.TLS == nil { response.Forbidden(errors.New("Cannot delete certificate")) } certBlock, _ := pem.Decode([]byte(certInfo.Certificate)) cert, err := x509.ParseCertificate(certBlock.Bytes) if err != nil { // This should not happen return response.InternalError(err) } trustedCerts := map[string]x509.Certificate{ certInfo.Name: *cert, } trusted := false for _, i := range r.TLS.PeerCertificates { trusted, _ = localUtil.CheckTrustState(*i, trustedCerts, s.Endpoints.NetworkCert(), false) if trusted { break } } if !trusted { return response.Forbidden(errors.New("Certificate cannot be deleted")) } } // On IncusOS, confirm at least one client certificate is left. if s.OS.IncusOS != nil && certInfo.Type == certificate.TypeClient { certs, err := d.getTrustedCertificates() if err != nil { return response.InternalError(err) } if len(certs[certificate.TypeClient]) < 2 { return response.BadRequest(errors.New("Cannot remove last client certificate while running on IncusOS")) } } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Perform the delete with the expanded fingerprint. return dbCluster.DeleteCertificate(ctx, tx.Tx(), certInfo.Fingerprint) }) if err != nil { return response.SmartError(err) } // Notify other nodes about the new certificate. notifier, err := cluster.NewNotifier(s, s.Endpoints.NetworkCert(), s.ServerCert(), cluster.NotifyAlive) if err != nil { return response.SmartError(err) } err = notifier(func(client incus.InstanceServer) error { return client.DeleteCertificate(certInfo.Fingerprint) }) if err != nil { return response.SmartError(err) } // Remove the certificate from the authorizer. err = s.Authorizer.DeleteCertificate(r.Context(), certInfo.Fingerprint) if err != nil { logger.Error("Failed to remove certificate from authorizer", logger.Ctx{"fingerprint": certInfo.Fingerprint, "error": err}) } } // Reload the cache. s.UpdateCertificateCache() s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.CertificateDeleted.Event(fingerprint, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } func certificateValidate(cert *x509.Certificate) error { if time.Now().Before(cert.NotBefore) { return errors.New("The provided certificate isn't valid yet") } if time.Now().After(cert.NotAfter) { return errors.New("The provided certificate is expired") } if cert.PublicKeyAlgorithm == x509.RSA { pubKey, ok := cert.PublicKey.(*rsa.PublicKey) if !ok { return errors.New("Unable to validate the RSA certificate") } // Check that we're dealing with at least 2048bit (Size returns a value in bytes). if pubKey.Size()*8 < 2048 { return errors.New("RSA key is too weak (minimum of 2048bit)") } } return nil } incus-7.0.0/cmd/incusd/cgo.go000066400000000000000000000010061517523235500157730ustar00rootroot00000000000000//go:build linux && cgo package main // #cgo CFLAGS: -std=gnu11 -Wvla -Werror -fvisibility=hidden -Winit-self // #cgo CFLAGS: -Wformat=2 -Wshadow -Wendif-labels -fasynchronous-unwind-tables // #cgo CFLAGS: -pipe --param=ssp-buffer-size=4 -g -Wunused // #cgo CFLAGS: -Werror=implicit-function-declaration // #cgo CFLAGS: -Werror=return-type -Wendif-labels -Werror=overflow // #cgo CFLAGS: -Wnested-externs -fexceptions // #cgo CFLAGS: -I ../../shared/cgo // #cgo pkg-config: lxc // #cgo pkg-config: libcap import "C" incus-7.0.0/cmd/incusd/daemon.go000066400000000000000000002337441517523235500165060ustar00rootroot00000000000000package main import ( "bytes" "context" "crypto/x509" "database/sql" "errors" "fmt" "io" "net" "net/http" "net/url" "os" "os/exec" "os/user" "path/filepath" "slices" "strings" "sync" "time" dqliteClient "github.com/cowsql/go-cowsql/client" "github.com/cowsql/go-cowsql/driver" "github.com/gorilla/mux" liblxc "github.com/lxc/go-lxc" "golang.org/x/sys/unix" internalIO "github.com/lxc/incus/v7/internal/io" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/rsync" "github.com/lxc/incus/v7/internal/server/apparmor" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/auth/oidc" "github.com/lxc/incus/v7/internal/server/bgp" "github.com/lxc/incus/v7/internal/server/certificate" "github.com/lxc/incus/v7/internal/server/cgroup" "github.com/lxc/incus/v7/internal/server/cluster" clusterConfig "github.com/lxc/incus/v7/internal/server/cluster/config" "github.com/lxc/incus/v7/internal/server/daemon" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/internal/server/db/warningtype" "github.com/lxc/incus/v7/internal/server/dns" "github.com/lxc/incus/v7/internal/server/endpoints" "github.com/lxc/incus/v7/internal/server/events" "github.com/lxc/incus/v7/internal/server/firewall" "github.com/lxc/incus/v7/internal/server/fsmonitor" "github.com/lxc/incus/v7/internal/server/instance" instanceDrivers "github.com/lxc/incus/v7/internal/server/instance/drivers" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/logging" "github.com/lxc/incus/v7/internal/server/network/ovn" "github.com/lxc/incus/v7/internal/server/network/ovs" networkZone "github.com/lxc/incus/v7/internal/server/network/zone" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" scriptletLoad "github.com/lxc/incus/v7/internal/server/scriptlet/load" "github.com/lxc/incus/v7/internal/server/seccomp" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/internal/server/storage/linstor" "github.com/lxc/incus/v7/internal/server/sys" "github.com/lxc/incus/v7/internal/server/syslog" "github.com/lxc/incus/v7/internal/server/task" "github.com/lxc/incus/v7/internal/server/ucred" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/server/warnings" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" "github.com/lxc/incus/v7/shared/cancel" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/proxy" "github.com/lxc/incus/v7/shared/revert" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) // A Daemon can respond to requests from a shared client. type Daemon struct { clientCerts *certificate.Cache os *sys.OS db *db.DB firewall firewall.Firewall bgp *bgp.Server dns *dns.Server // Event servers devIncusEvents *events.DevIncusServer events *events.Server internalListener *events.InternalListener // Tasks registry for long-running background tasks // Keep clustering tasks separate as they cause a lot of CPU wakeups tasks task.Group clusterTasks task.Group // Indexes of tasks that need to be reset when their execution interval changes taskPruneImages *task.Task taskClusterHeartbeat *task.Task // Stores startup time of daemon startTime time.Time // Whether daemon was started by systemd socket activation. systemdSocketActivated bool config *DaemonConfig endpoints *endpoints.Endpoints gateway *cluster.Gateway seccomp *seccomp.Server proxy func(req *http.Request) (*url.URL, error) oidcVerifier *oidc.Verifier // Stores last heartbeat node information to detect node changes. lastNodeList *cluster.APIHeartbeat // Serialize changes to cluster membership (joins, leaves, role // changes). clusterMembershipMutex sync.RWMutex serverCert func() *localtls.CertInfo serverCertInt *localtls.CertInfo // Do not use this directly, use servertCert func. // Status control. setupChan chan struct{} // Closed when basic Daemon setup is completed waitReady *cancel.Canceller // Cancelled when fully ready shutdownCtx context.Context // Cancelled when shutdown starts. shutdownCancel context.CancelFunc // Cancels the shutdownCtx to indicate shutdown starting. shutdownDoneCh chan error // Receives the result of the d.Stop() function and tells the daemon to end. // Device monitor for watching filesystem events devmonitor fsmonitor.FSMonitor // Keep track of skews. timeSkew bool // Configuration. globalConfig *clusterConfig.Config localConfig *node.Config globalConfigMu sync.Mutex // Cluster. serverName string serverClustered bool loggingController *logging.Controller // Authorization. authorizer auth.Authorizer // Syslog listener cancel function. syslogSocketCancel context.CancelFunc // OVN clients. ovnnb *ovn.NB ovnsb *ovn.SB ovnMu sync.Mutex // OVS client. ovs *ovs.VSwitch ovsMu sync.Mutex // API info. apiExtensions int // Linstor client. linstor *linstor.Client linstorMu sync.Mutex } // DaemonConfig holds configuration values for Daemon. type DaemonConfig struct { Group string // Group name the local unix socket should be chown'ed to Trace []string // List of sub-systems to trace RaftLatency float64 // Coarse grain measure of the cluster latency DqliteSetupTimeout time.Duration // How long to wait for the cluster database to be up } // newDaemon returns a new Daemon object with the given configuration. func newDaemon(config *DaemonConfig, os *sys.OS) *Daemon { incusEvents := events.NewServer(daemon.Debug, daemon.Verbose, cluster.EventHubPush) devIncusEvents := events.NewDevIncusServer(daemon.Debug, daemon.Verbose) shutdownCtx, shutdownCancel := context.WithCancel(context.Background()) d := &Daemon{ clientCerts: &certificate.Cache{}, config: config, devIncusEvents: devIncusEvents, events: incusEvents, db: &db.DB{}, os: os, setupChan: make(chan struct{}), waitReady: cancel.New(context.Background()), shutdownCtx: shutdownCtx, shutdownCancel: shutdownCancel, shutdownDoneCh: make(chan error), apiExtensions: len(version.APIExtensions), } d.serverCert = func() *localtls.CertInfo { return d.serverCertInt } return d } // defaultDaemonConfig returns a DaemonConfig object with default values. func defaultDaemonConfig() *DaemonConfig { return &DaemonConfig{ RaftLatency: 3.0, DqliteSetupTimeout: 36 * time.Hour, // Account for snap refresh lag } } // defaultDaemon returns a new, un-initialized Daemon object with default values. func defaultDaemon() *Daemon { config := defaultDaemonConfig() os := sys.DefaultOS() return newDaemon(config, os) } // APIEndpoint represents a URL in our API. type APIEndpoint struct { Name string // Name for this endpoint. Path string // Path pattern for this endpoint. Aliases []APIEndpointAlias // Any aliases for this endpoint. Get APIEndpointAction Head APIEndpointAction Put APIEndpointAction Post APIEndpointAction Delete APIEndpointAction Patch APIEndpointAction } // APIEndpointAlias represents an alias URL of and APIEndpoint in our API. type APIEndpointAlias struct { Name string // Name for this alias. Path string // Path pattern for this alias. } // APIEndpointAction represents an action on an API endpoint. type APIEndpointAction struct { Handler func(d *Daemon, r *http.Request) response.Response AccessHandler func(d *Daemon, r *http.Request) response.Response AllowUntrusted bool LargeRequest bool // Whether the endpoint may be getting requests larger than 1MiB. } // allowAuthenticated is an AccessHandler which allows only authenticated requests. This should be used in conjunction // with further access control within the handler (e.g. to filter resources the user is able to view/edit). func allowAuthenticated(d *Daemon, r *http.Request) response.Response { err := d.checkTrustedClient(r) if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } // allowPermission is a wrapper to check access against a given object, an object being an image, instance, network, etc. // Mux vars should be passed in so that the object we are checking can be created. For example, a certificate object requires // a fingerprint, the mux var for certificate fingerprints is "fingerprint", so that string should be passed in. // Mux vars should always be passed in with the same order they appear in the API route. func allowPermission(objectType auth.ObjectType, entitlement auth.Entitlement, muxVars ...string) func(d *Daemon, r *http.Request) response.Response { return func(d *Daemon, r *http.Request) response.Response { // Expansion function to deal with partial fingerprints. expandFingerprint := func(projectName string, fingerprint string) string { if objectType == auth.ObjectTypeImage { var imgInfo *api.Image err := d.db.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error _, imgInfo, err = tx.GetImage(ctx, fingerprint, dbCluster.ImageFilter{Project: &projectName}) return err }) if err != nil { return fingerprint } fingerprint = imgInfo.Fingerprint } else if objectType == auth.ObjectTypeCertificate { err := d.db.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbCertInfo, err := dbCluster.GetCertificateByFingerprintPrefix(ctx, tx.Tx(), fingerprint) if err != nil { return err } fingerprint = dbCertInfo.Fingerprint return nil }) if err != nil { return fingerprint } } // Fallback to no expansion. return fingerprint } // Expansion function to deal with project inheritance. expandProject := func(projectName string) string { // Object types that aren't part of projects. if slices.Contains([]auth.ObjectType{auth.ObjectTypeUser, auth.ObjectTypeServer, auth.ObjectTypeCertificate, auth.ObjectTypeStoragePool, auth.ObjectTypeNetworkIntegration}, objectType) { return projectName } // Load the project. var p *api.Project err := d.db.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := dbCluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return err } p, err = dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return projectName } if objectType == auth.ObjectTypeProfile { projectName = project.ProfileProjectFromRecord(p) } else if objectType == auth.ObjectTypeStorageBucket { projectName = project.StorageBucketProjectFromRecord(p) } else if objectType == auth.ObjectTypeStorageVolume { dbVolType, err := storagePools.VolumeTypeNameToDBType(muxVars[1]) if err != nil { return projectName } projectName = project.StorageVolumeProjectFromRecord(p, dbVolType) } else if objectType == auth.ObjectTypeNetworkZone { projectName = project.NetworkZoneProjectFromRecord(p) } else if slices.Contains([]auth.ObjectType{auth.ObjectTypeImage, auth.ObjectTypeImageAlias}, objectType) { projectName = project.ImageProjectFromRecord(p) } else if slices.Contains([]auth.ObjectType{auth.ObjectTypeNetwork, auth.ObjectTypeNetworkACL}, objectType) { projectName = project.NetworkProjectFromRecord(p) } return projectName } // Expansion function for volume location. expandVolumeLocation := func(projectName string, poolName string, volumeTypeName string, volumeName string) string { // The location field is only relevant in clusters. if !d.serverClustered { return "" } var err error var nodes []db.NodeInfo var poolID int64 // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return "" } // Get the server list for the volume. err = d.db.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { poolID, err = tx.GetStoragePoolID(ctx, poolName) if err != nil { return err } nodes, err = tx.GetStorageVolumeNodes(ctx, poolID, projectName, volumeName, volumeType) if err != nil { return err } return nil }) if err != nil { return "" } if len(nodes) != 1 { return "" } return nodes[0].Name } objectName, err := auth.ObjectFromRequest(r, objectType, expandProject, expandFingerprint, expandVolumeLocation, muxVars...) if err != nil { return response.InternalError(fmt.Errorf("Failed to create authentication object: %w", err)) } s := d.State() // Validate whether the user has the needed permission err = s.Authorizer.CheckPermission(r.Context(), r, objectName, entitlement) if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } } // Convenience function around Authenticate. func (d *Daemon) checkTrustedClient(r *http.Request) error { trusted, _, _, err := d.Authenticate(nil, r) if !trusted || err != nil { if err != nil { return err } return errors.New("Not authorized") } return nil } // getTrustedCertificates returns trusted certificates key on DB type and fingerprint. // // When in PKI mode, this also filters out any non-server certificate which isn't issued by the PKI. func (d *Daemon) getTrustedCertificates() (map[certificate.Type]map[string]x509.Certificate, error) { certs := d.clientCerts.GetCertificates() // If not in PKI mode, return all certificates. if !util.PathExists(internalUtil.VarPath("server.ca")) { return certs, nil } // If in PKI mode, filter certificates that aren't trusted by the CA. ca, err := localtls.ReadCert(internalUtil.VarPath("server.ca")) if err != nil { return nil, err } certPool := x509.NewCertPool() certPool.AddCert(ca) for certType, certEntries := range certs { if certType == certificate.TypeServer { continue } for name, entry := range certEntries { _, err := entry.Verify(x509.VerifyOptions{ Roots: certPool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, }) if err != nil { // Skip certificates that aren't signed by the PKI. delete(certs[certType], name) } } } return certs, nil } // Authenticate validates an incoming http Request // It will check over what protocol it came, what type of request it is and // will validate the TLS certificate. // // This does not perform authorization, only validates authentication. // Returns whether trusted or not, the username (or certificate fingerprint) of the trusted client, and the type of // client that has been authenticated (cluster, unix, or tls). func (d *Daemon) Authenticate(w http.ResponseWriter, r *http.Request) (bool, string, string, error) { trustedCerts, err := d.getTrustedCertificates() if err != nil { return false, "", "", err } // Allow internal cluster traffic by checking against the trusted certfificates. if r.TLS != nil { for _, i := range r.TLS.PeerCertificates { trusted, fingerprint := localUtil.CheckTrustState(*i, trustedCerts[certificate.TypeServer], d.endpoints.NetworkCert(), false) if trusted { return true, fingerprint, "cluster", nil } } } // Local unix socket queries. if r.RemoteAddr == "@" && r.TLS == nil { if w != nil { cred, err := ucred.GetCredFromContext(r.Context()) if err != nil { return false, "", "", err } u, err := user.LookupId(fmt.Sprintf("%d", cred.Uid)) if err != nil { return true, fmt.Sprintf("uid=%d", cred.Uid), "unix", nil } return true, u.Username, "unix", nil } return true, "", "unix", nil } // DevIncus unix socket credentials on main API. if r.RemoteAddr == "@dev_incus" { return false, "", "", errors.New("Main API query can't come from /dev/incus socket") } // Cluster notification with wrong certificate. if isClusterNotification(r) { return false, "", "", errors.New("Cluster notification isn't using trusted server certificate") } // Cluster internal client with wrong certificate. if isClusterInternal(r) { return false, "", "", errors.New("Cluster internal client isn't using trusted server certificate") } // Bad query, no TLS found. if r.TLS == nil { return false, "", "", errors.New("Bad/missing TLS on network query") } // Load the certificates. trustCACertificates := d.globalConfig.TrustCACertificates() // Check for JWT token signed by a TLS certificate. jwtOk, _, cert := localUtil.CheckJwtToken(r, trustedCerts[certificate.TypeClient]) if jwtOk { trusted, username := localUtil.CheckTrustState(*cert, trustedCerts[certificate.TypeClient], d.endpoints.NetworkCert(), trustCACertificates) if trusted { return true, username, api.AuthenticationMethodTLS, nil } } // Check for JWT token signed by an OpenID Connect provider. if d.oidcVerifier != nil && d.oidcVerifier.IsRequest(r) { userName, err := d.oidcVerifier.Auth(d.shutdownCtx, w, r) if err != nil { return false, "", "", err } return true, userName, api.AuthenticationMethodOIDC, nil } // Validate metrics TLS certificates. if r.URL.Path == "/1.0/metrics" { for _, i := range r.TLS.PeerCertificates { trusted, username := localUtil.CheckTrustState(*i, trustedCerts[certificate.TypeMetrics], d.endpoints.NetworkCert(), trustCACertificates) if trusted { return true, username, api.AuthenticationMethodTLS, nil } } } // Validate regular TLS certificates. for _, i := range r.TLS.PeerCertificates { trusted, username := localUtil.CheckTrustState(*i, trustedCerts[certificate.TypeClient], d.endpoints.NetworkCert(), trustCACertificates) if trusted { return true, username, api.AuthenticationMethodTLS, nil } } // Reject unauthorized. return false, "", "", nil } // State creates a new State instance linked to our internal db and os. func (d *Daemon) State() *state.State { // If the daemon is shutting down, the context will be cancelled. // This information will be available throughout the code, and can be used to prevent new // operations from starting during shutdown. // Build a list of instance types. drivers := instanceDrivers.DriverStatuses() instanceTypes := make(map[instancetype.Type]error, len(drivers)) for driverType, driver := range drivers { instanceTypes[driverType] = driver.Info.Error } d.globalConfigMu.Lock() globalConfig := d.globalConfig localConfig := d.localConfig d.globalConfigMu.Unlock() return &state.State{ Authorizer: d.authorizer, BGP: d.bgp, Cluster: d.gateway, DB: d.db, DevIncusEvents: d.devIncusEvents, DevMonitor: d.devmonitor, DNS: d.dns, Endpoints: d.endpoints, Events: d.events, Firewall: d.firewall, GlobalConfig: globalConfig, InstanceTypes: instanceTypes, LocalConfig: localConfig, OS: d.os, OVN: d.getOVN, OVS: d.getOVS, Linstor: d.getLinstor, Proxy: d.proxy, ServerCert: d.serverCert, ServerClustered: d.serverClustered, ServerName: d.serverName, ShutdownCtx: d.shutdownCtx, StartTime: d.startTime, UpdateCertificateCache: func() { updateCertificateCache(d) }, } } func (d *Daemon) createCmd(restAPI *mux.Router, version string, c APIEndpoint) { var uri string if c.Path == "" { uri = fmt.Sprintf("/%s", version) } else if version != "" { uri = fmt.Sprintf("/%s/%s", version, c.Path) } else { uri = fmt.Sprintf("/%s", c.Path) } route := restAPI.HandleFunc(uri, func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") // Block on daemon startup except for the "internal" and "os" APIs. if !slices.Contains([]string{"internal", "os"}, version) { select { case <-d.setupChan: default: response := response.Unavailable(errors.New("Daemon is starting up")) _ = response.Render(w) return } } // Authentication trusted, username, protocol, err := d.Authenticate(w, r) if err != nil { var authError *oidc.AuthError if errors.As(err, &authError) { // Ensure the OIDC headers are set if needed. if d.oidcVerifier != nil { _ = d.oidcVerifier.WriteHeaders(w) } _ = response.Unauthorized(err).Render(w) return } } // Restrict internal queries to remote, non-cluster, clients if version == "internal" && !slices.Contains([]string{"unix", "cluster"}, protocol) { internalAllowed := func() bool { // Reject any unauthenticated request. if !trusted { return false } // Allow select endpoints (unstable API but CLI supported). if slices.Contains([]string{"recover/import", "recover/validate", "sql"}, c.Path) { return true } if c.Path == "cluster/accept" && protocol == api.AuthenticationMethodTLS { return true } // Default to rejecting access. return false }() // Except for the initial cluster accept request (done over trusted TLS) if !internalAllowed { logger.Warn("Rejecting remote internal API request", logger.Ctx{"ip": r.RemoteAddr}) _ = response.Forbidden(nil).Render(w) return } } logCtx := logger.Ctx{"method": r.Method, "url": r.URL.RequestURI(), "ip": r.RemoteAddr, "protocol": protocol} if protocol == "cluster" { logCtx["fingerprint"] = username } else { logCtx["username"] = username } untrustedOk := (r.Method == "GET" && c.Get.AllowUntrusted) || (r.Method == "POST" && c.Post.AllowUntrusted) if trusted { logger.Debug("Handling API request", logCtx) // Add authentication/authorization context data. ctx := context.WithValue(r.Context(), request.CtxUsername, username) ctx = context.WithValue(ctx, request.CtxProtocol, protocol) // Add forwarded requestor data. if protocol == "cluster" { // Add authentication/authorization context data. ctx = context.WithValue(ctx, request.CtxForwardedAddress, r.Header.Get(request.HeaderForwardedAddress)) ctx = context.WithValue(ctx, request.CtxForwardedUsername, r.Header.Get(request.HeaderForwardedUsername)) ctx = context.WithValue(ctx, request.CtxForwardedProtocol, r.Header.Get(request.HeaderForwardedProtocol)) } r = r.WithContext(ctx) } else if untrustedOk && r.Header.Get("X-Incus-authenticated") == "" { logger.Debug(fmt.Sprintf("Allowing untrusted %s", r.Method), logger.Ctx{"url": r.URL.RequestURI(), "ip": r.RemoteAddr}) } else { if d.oidcVerifier != nil { _ = d.oidcVerifier.WriteHeaders(w) } logger.Warn("Rejecting request from untrusted client", logger.Ctx{"ip": r.RemoteAddr}) _ = response.Forbidden(nil).Render(w) return } // Dump full request JSON when in debug mode if daemon.Debug && r.Method != "GET" && localUtil.IsJSONRequest(r) { newBody := &bytes.Buffer{} captured := &bytes.Buffer{} multiW := io.MultiWriter(newBody, captured) _, err := util.SafeCopy(multiW, r.Body) if err != nil { _ = response.InternalError(err).Render(w) return } r.Body = internalIO.BytesReadCloser{Buf: newBody} localUtil.DebugJSON("API Request", captured, logger.AddContext(logCtx)) } // Actually process the request var resp response.Response // Return Unavailable Error (503) if daemon is shutting down. // There are some exceptions: // - internal calls, e.g. shutdown // - events endpoint as this is accessed when running `shutdown` // - /1.0 endpoint // - /1.0/operations endpoints // - GET queries allowedDuringShutdown := func() bool { if version == "internal" { return true } if c.Path == "" || c.Path == "events" || c.Path == "operations" || strings.HasPrefix(c.Path, "operations/") { return true } if r.Method == "GET" { return true } return false } if errors.Is(d.shutdownCtx.Err(), context.Canceled) && !allowedDuringShutdown() { _ = response.Unavailable(errors.New("Incus is shutting down")).Render(w) return } handleRequest := func(action APIEndpointAction) response.Response { if action.Handler == nil { return response.NotImplemented(nil) } // All APIEndpointActions should have an access handler or should allow untrusted requests. if action.AccessHandler == nil && !action.AllowUntrusted { return response.InternalError(fmt.Errorf("Access handler not defined for %s %s", r.Method, r.URL.RequestURI())) } // If the request is not trusted, only call the handler if the action allows it. if !trusted && !action.AllowUntrusted { return response.Forbidden(errors.New("You must be authenticated")) } // Protect against CSRF when using UI with browser that supports Fetch metadata. // Deny Sec-Fetch-Site when set to cross-origin or same-site. if slices.Contains([]string{"cross-site", "same-site"}, r.Header.Get("Sec-Fetch-Site")) { return response.ErrorResponse(http.StatusForbidden, "Forbidden Sec-Fetch-Site header value") } // Call the access handler if there is one. if action.AccessHandler != nil { resp := action.AccessHandler(d, r) if resp != response.EmptySyncResponse { return resp } } // Limit request body size unless the endpoint requires a large body. if !action.LargeRequest { r.Body = http.MaxBytesReader(w, r.Body, 1024*1024) } return action.Handler(d, r) } switch r.Method { case "GET": resp = handleRequest(c.Get) case "HEAD": resp = handleRequest(c.Head) case "PUT": resp = handleRequest(c.Put) case "POST": resp = handleRequest(c.Post) case "DELETE": resp = handleRequest(c.Delete) case "PATCH": resp = handleRequest(c.Patch) default: resp = response.NotFound(fmt.Errorf("Method %q not found", r.Method)) } // If sending out Forbidden, make sure we have OIDC headers. if resp.Code() == http.StatusForbidden && d.oidcVerifier != nil { _ = d.oidcVerifier.WriteHeaders(w) } // Handle errors err = resp.Render(w) if err != nil { writeErr := response.SmartError(err).Render(w) if writeErr != nil { logger.Error("Failed writing error for HTTP response", logger.Ctx{"url": uri, "err": err, "writeErr": writeErr}) } } }) // If the endpoint has a canonical name then record it so it can be used to build URLS // and accessed in the context of the request by the handler function. if c.Name != "" { route.Name(c.Name) } } // have we setup shared mounts? var sharedMountsLock sync.Mutex // setupSharedMounts will mount any shared mounts needed, and set daemon.SharedMountsSetup to true. func setupSharedMounts() error { // Check if we already went through this if daemon.SharedMountsSetup { return nil } // Get a lock to prevent races sharedMountsLock.Lock() defer sharedMountsLock.Unlock() // Check if already setup path := internalUtil.VarPath("shmounts") if linux.IsMountPoint(path) { daemon.SharedMountsSetup = true return nil } // Mount a new tmpfs err := unix.Mount("tmpfs", path, "tmpfs", 0, "size=100k,mode=0711") if err != nil { return err } // Mark as MS_SHARED and MS_REC var flags uintptr = unix.MS_SHARED | unix.MS_REC err = unix.Mount(path, path, "none", flags, "") if err != nil { return err } daemon.SharedMountsSetup = true return nil } // Init starts daemon process. func (d *Daemon) Init() error { d.startTime = time.Now() err := d.init() // If an error occurred synchronously while starting up, let's try to // cleanup any state we produced so far. Errors happening here will be // ignored. if err != nil { logger.Error("Failed to start the daemon", logger.Ctx{"err": err}) _ = d.Stop(context.Background(), unix.SIGINT) return err } return nil } func (d *Daemon) init() error { var err error var dbWarnings []dbCluster.Warning // Set default authorizer. d.authorizer, err = auth.LoadAuthorizer(d.shutdownCtx, auth.DriverTLS, logger.Log, d.clientCerts) if err != nil { return err } // Setup logger events.LoggingServer = d.events // Setup internal event listener d.internalListener = events.NewInternalListener(d.shutdownCtx, d.events) // Lets check if there's an existing daemon running err = endpoints.CheckAlreadyRunning(d.os.GetUnixSocket()) if err != nil { return err } /* Set the LVM environment */ err = os.Setenv("LVM_SUPPRESS_FD_WARNINGS", "1") if err != nil { return err } /* Print welcome message */ mode := "normal" if d.os.MockMode { mode = "mock" } logger.Info("Starting up", logger.Ctx{"version": version.Version, "mode": mode, "path": internalUtil.VarPath("")}) /* List of sub-systems to trace */ trace := d.config.Trace /* Initialize the operating system facade */ dbWarnings, err = d.os.Init() if err != nil { return err } // Initialize apparmor. if d.os.AppArmorAvailable { err := apparmor.Init() if err != nil { return fmt.Errorf("Failed to initialize apparmor: %v", err) } } // Setup AppArmor wrapper. archive.RunWrapper = func(cmd *exec.Cmd, output string, allowedCmds []string) (func(), error) { return apparmor.ArchiveWrapper(d.os, cmd, output, allowedCmds) } rsync.RunWrapper = func(cmd *exec.Cmd, source string, destination string) (func(), error) { return apparmor.RsyncWrapper(d.os, cmd, source, destination) } // Bump some kernel limits to avoid issues for _, limit := range []int{unix.RLIMIT_NOFILE} { rLimit := unix.Rlimit{} err := unix.Getrlimit(limit, &rLimit) if err != nil { return err } rLimit.Cur = rLimit.Max err = unix.Setrlimit(limit, &rLimit) if err != nil { return err } } // Detect LXC features d.os.LXCFeatures = map[string]bool{} lxcExtensions := []string{} for _, extension := range lxcExtensions { d.os.LXCFeatures[extension] = liblxc.HasAPIExtension(extension) } // Get cgroup warnings. dbWarnings = append(dbWarnings, cgroup.Warnings()...) // Detect and cached available instance types from operational drivers. drivers := instanceDrivers.DriverStatuses() for _, driver := range drivers { if driver.Warning != nil { dbWarnings = append(dbWarnings, *driver.Warning) } } // Detect and setup missing temporary mounts. if !d.os.MockMode { devicesPath := filepath.Join(d.os.VarDir, "devices") devIncusPath := filepath.Join(d.os.VarDir, "guestapi") // Attempt to mount the devices tmpfs. // NOTE: The check for devIncusPath is to handle initial rollout // of the tmpfs on systems that have running instances. It can go away // after a little while. if !linux.IsMountPoint(devIncusPath) && !linux.IsMountPoint(devicesPath) { err = unix.Mount("tmpfs", devicesPath, "tmpfs", 0, "size=250M,mode=0711") if err != nil { logger.Warn("Failed to set up devices tmpfs", logger.Ctx{"err": err}) } } // Attempt to mount the shmounts tmpfs. err := setupSharedMounts() if err != nil { logger.Warn("Failed to set up shmounts tmpfs", logger.Ctx{"err": err}) } // Attempt to mount the guestapi tmpfs if !linux.IsMountPoint(devIncusPath) { err = unix.Mount("tmpfs", devIncusPath, "tmpfs", 0, "size=100k,mode=0755") if err != nil { logger.Warn("Failed to set up guestapi tmpfs", logger.Ctx{"err": err}) } } } // Show all persistent warnings. for _, w := range dbWarnings { logger.Warnf(" - %s, %s", warningtype.TypeNames[warningtype.Type(w.TypeCode)], w.LastMessage) } /* Initialize the database */ err = initializeDbObject(d) if err != nil { return err } /* Setup network endpoint certificate */ networkCert, err := internalUtil.LoadCert(d.os.VarDir) if err != nil { return err } /* Setup server certificate */ serverCert, err := internalUtil.LoadServerCert(d.os.VarDir) if err != nil { return err } // Load cached local trusted certificates before starting listener and cluster database. err = updateCertificateCacheFromLocal(d) if err != nil { return err } d.serverClustered, err = cluster.Enabled(d.db.Node) if err != nil { return fmt.Errorf("Failed checking if clustered: %w", err) } // Detect if clustered, but not yet upgraded to per-server client certificates. certificates := d.clientCerts.GetCertificates() if d.serverClustered && len(certificates[certificate.TypeServer]) < 1 { // If the cluster has not yet upgraded to per-server client certificates (by running patch // patchClusteringServerCertTrust) then temporarily use the network (cluster) certificate as client // certificate, and cause us to trust it for use as client certificate from the other members. networkCertFingerPrint := networkCert.Fingerprint() logger.Warn("No local trusted server certificates found, falling back to trusting network certificate", logger.Ctx{"fingerprint": networkCertFingerPrint}) logger.Info("Set client certificate to network certificate", logger.Ctx{"fingerprint": networkCertFingerPrint}) d.serverCertInt = networkCert } else { // If standalone or the local trusted certificates table is populated with server certificates then // use our local server certificate as client certificate for intra-cluster communication. logger.Info("Set client certificate to server certificate", logger.Ctx{"fingerprint": serverCert.Fingerprint()}) d.serverCertInt = serverCert } /* Setup dqlite */ clusterLogLevel := "ERROR" if slices.Contains(trace, "dqlite") { clusterLogLevel = "TRACE" } d.gateway, err = cluster.NewGateway( d.shutdownCtx, d.db.Node, networkCert, d.State, cluster.Latency(d.config.RaftLatency), cluster.LogLevel(clusterLogLevel)) if err != nil { return err } d.gateway.HeartbeatNodeHook = d.nodeRefreshTask logger.Info("Loading daemon configuration") err = d.db.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { d.localConfig, err = node.ConfigLoad(ctx, tx) return err }) if err != nil { return err } localHTTPAddress := d.localConfig.HTTPSAddress() localClusterAddress := d.localConfig.ClusterAddress() debugAddress := d.localConfig.DebugAddress() if os.Getenv("LISTEN_PID") != "" { d.systemdSocketActivated = true } /* Setup the web server */ config := &endpoints.Config{ Dir: d.os.VarDir, UnixSocket: d.os.GetUnixSocket(), Cert: networkCert, RestServer: restServer(d), DevIncusServer: devIncusServer(d), LocalUnixSocketGroup: d.config.Group, LocalUnixSocketLabel: "system_u:object_r:container_runtime_t:s0", NetworkAddress: localHTTPAddress, ClusterAddress: localClusterAddress, DebugAddress: debugAddress, MetricsServer: metricsServer(d), StorageBucketsServer: storageBucketsServer(d), VsockServer: vSockServer(d), VsockSupport: false, } // Enable vsock server support if VM instances supported. err, found := d.State().InstanceTypes[instancetype.VM] if found && err == nil { config.VsockSupport = true } d.endpoints, err = endpoints.Up(config) if err != nil { return err } // Have the db package determine remote storage drivers db.StorageRemoteDriverNames = storageDrivers.RemoteDriverNames /* Open the cluster database */ for { logger.Info("Initializing global database") dir := filepath.Join(d.os.VarDir, "database") store := d.gateway.NodeStore() contextTimeout := 30 * time.Second if !d.serverClustered { // FIXME: this is a workaround for #5234. We set a very // high timeout when we're not clustered, since there's // actually no networking involved. contextTimeout = time.Minute } options := []driver.Option{ driver.WithDialFunc(d.gateway.DialFunc()), driver.WithContext(d.gateway.Context()), driver.WithConnectionTimeout(10 * time.Second), driver.WithContextTimeout(contextTimeout), driver.WithLogFunc(cluster.DqliteLog), } if slices.Contains(trace, "database") { options = append(options, driver.WithTracing(dqliteClient.LogDebug)) } d.db.Cluster, err = db.OpenCluster(context.Background(), "db.bin", store, localClusterAddress, dir, d.config.DqliteSetupTimeout, options...) if err == nil { logger.Info("Initialized global database") break } else if errors.Is(err, db.ErrSomeNodesAreBehind) { // If some other nodes have schema or API versions less recent // than this node, we block until we receive a notification // from the last node being upgraded that everything should be // now fine, and then retry logger.Warn("Wait for other cluster nodes to upgrade their versions, cluster not started yet") // The only thing we want to still do on this node is // to run the heartbeat task, in case we are the raft // leader. d.gateway.Cluster = d.db.Cluster taskFunc, taskSchedule := cluster.HeartbeatTask(d.gateway) hbGroup := task.Group{} d.taskClusterHeartbeat = hbGroup.Add(taskFunc, taskSchedule) hbGroup.Start(d.shutdownCtx) d.gateway.WaitUpgradeNotification() _ = hbGroup.Stop(time.Second) d.gateway.Cluster = nil _ = d.db.Cluster.Close() continue } return fmt.Errorf("Failed to initialize global database: %w", err) } d.firewall = firewall.New() logger.Info("Firewall loaded driver", logger.Ctx{"driver": d.firewall}) err = cluster.NotifyUpgradeCompleted(d.State(), networkCert, d.serverCert()) if err != nil { // Ignore the error, since it's not fatal for this particular // node. In most cases it just means that some nodes are // offline. logger.Warn("Could not notify all nodes of database upgrade", logger.Ctx{"err": err}) } d.gateway.Cluster = d.db.Cluster // Setup the user-agent. if d.serverClustered { version.UserAgentFeatures([]string{"cluster"}) } // Load server name and config before patches run (so they can access them from d.State()). err = d.db.Cluster.Transaction(d.shutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { config, err := clusterConfig.Load(ctx, tx) if err != nil { return err } // Get the local node (will be used if clustered). serverName, err := tx.GetLocalNodeName(ctx) if err != nil { return err } d.globalConfigMu.Lock() d.serverName = serverName d.globalConfig = config d.globalConfigMu.Unlock() return nil }) if err != nil { return err } d.events.SetLocalLocation(d.serverName) // Mount the storage pools. logger.Infof("Initializing storage pools") err = storageStartup(d.State()) if err != nil { return err } // Apply all patches that need to be run before daemon storage is initialized. err = patchesApply(d, patchPreDaemonStorage) if err != nil { return err } // Mount any daemon storage volumes. logger.Infof("Initializing daemon storage mounts") err = daemonStorageMount(d.State()) if err != nil { return err } // Create directories on daemon storage mounts. err = d.os.InitStorage() if err != nil { return err } // Apply all patches that need to be run after daemon storage is initialized. err = patchesApply(d, patchPostDaemonStorage) if err != nil { return err } // Load server name and config after patches run (in case its been changed). err = d.db.Cluster.Transaction(d.shutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { config, err := clusterConfig.Load(ctx, tx) if err != nil { return err } // Get the local node (will be used if clustered). serverName, err := tx.GetLocalNodeName(ctx) if err != nil { return err } d.globalConfigMu.Lock() d.serverName = serverName d.globalConfig = config d.globalConfigMu.Unlock() return nil }) if err != nil { return err } d.events.SetLocalLocation(d.serverName) // Get daemon configuration. bgpAddress := d.localConfig.BGPAddress() bgpRouterID := d.localConfig.BGPRouterID() bgpASN := int64(0) dnsAddress := d.localConfig.DNSAddress() // Get specific config keys. d.globalConfigMu.Lock() bgpASN = d.globalConfig.BGPASN() d.proxy = proxy.FromConfig(d.globalConfig.ProxyHTTPS(), d.globalConfig.ProxyHTTP(), d.globalConfig.ProxyIgnoreHosts()) d.gateway.HeartbeatOfflineThreshold = d.globalConfig.OfflineThreshold() oidcIssuer, oidcClientID, oidcScope, oidcAudience, oidcClaim := d.globalConfig.OIDCServer() syslogSocketEnabled := d.localConfig.SyslogSocket() openfgaAPIURL, openfgaAPIToken, openfgaStoreID := d.globalConfig.OpenFGA() instancePlacementScriptlet := d.globalConfig.InstancesPlacementScriptlet() authorizationScriptlet := d.globalConfig.AuthorizationScriptlet() d.endpoints.NetworkUpdateTrustedProxy(d.globalConfig.HTTPSTrustedProxy()) d.globalConfigMu.Unlock() d.loggingController = logging.NewLoggingController(d.internalListener) err = d.loggingController.Setup(d.State()) if err != nil { return err } // Setup syslog listener. if syslogSocketEnabled { err = d.setupSyslogSocket(true) if err != nil { return err } } // Setup OIDC authentication. if oidcIssuer != "" && oidcClientID != "" { d.oidcVerifier, err = oidc.NewVerifier(oidcIssuer, oidcClientID, oidcScope, oidcAudience, oidcClaim) if err != nil { return err } } // Setup OpenFGA authorization. if openfgaAPIURL != "" && openfgaStoreID != "" && openfgaAPIToken != "" { err = d.setupOpenFGA(openfgaAPIURL, openfgaAPIToken, openfgaStoreID) if err != nil { return fmt.Errorf("Failed to configure OpenFGA: %w", err) } } // Setup the authorization scriptlet. if authorizationScriptlet != "" { err = d.setupAuthorizationScriptlet(authorizationScriptlet) if err != nil { return err } } // Setup BGP listener. d.bgp = bgp.NewServer() if bgpAddress != "" && bgpASN != 0 && bgpRouterID != "" { err := d.bgp.Configure(bgpAddress, uint32(bgpASN), net.ParseIP(bgpRouterID)) if err != nil { return err } logger.Info("Started BGP server") } // Setup DNS listener. d.dns = dns.NewServer(d.db.Cluster, func(name string, full bool) (*dns.Zone, error) { // Fetch the zone. zone, err := networkZone.LoadByName(d.State(), name) if err != nil { return nil, err } zoneInfo := zone.Info() // Fill in the zone information. resp := &dns.Zone{} resp.Info = *zoneInfo if full { // Full content was requested. zoneBuilder, err := zone.Content() if err != nil { logger.Errorf("Failed to render DNS zone %q: %v", name, err) return nil, err } resp.Content = strings.TrimSpace(zoneBuilder.String()) } else { // SOA only. zoneBuilder, err := zone.SOA() if err != nil { logger.Errorf("Failed to render DNS zone %q: %v", name, err) return nil, err } resp.Content = strings.TrimSpace(zoneBuilder.String()) } return resp, nil }) if dnsAddress != "" { err := d.dns.Start(dnsAddress) if err != nil { return err } logger.Info("Started DNS server") } // Setup the networks. if !d.serverClustered || !d.db.Cluster.LocalNodeIsEvacuated() { logger.Infof("Initializing networks") err = networkStartup(d.State()) if err != nil { return err } } // Setup tertiary listeners that may use managed network addresses and must be started after networks. metricsAddress := d.localConfig.MetricsAddress() if metricsAddress != "" { err = d.endpoints.UpMetrics(metricsAddress) if err != nil { return err } } storageBucketsAddress := d.localConfig.StorageBucketsAddress() if storageBucketsAddress != "" { err = d.endpoints.UpStorageBuckets(storageBucketsAddress) if err != nil { return err } } // Load instance placement scriptlet. if instancePlacementScriptlet != "" { err = scriptletLoad.InstancePlacementSet(instancePlacementScriptlet) if err != nil { logger.Warn("Failed loading instance placement scriptlet", logger.Ctx{"err": err}) } } // Apply all patches that need to be run after networks are initialized. err = patchesApply(d, patchPostNetworks) if err != nil { return err } // Cleanup leftover images. pruneLeftoverImages(d.State()) var instances []instance.Instance if !d.os.MockMode { // Start the scheduler go deviceEventListener(d.State) prefixPath := os.Getenv("INCUS_DEVMONITOR_DIR") if prefixPath == "" { prefixPath = "/dev" } logger.Info("Starting device monitor") d.devmonitor, err = fsmonitor.New(d.State().ShutdownCtx, prefixPath) if err != nil { return err } // Must occur after d.devmonitor has been initialized. instances, err = instance.LoadNodeAll(d.State(), instancetype.Any) if err != nil { return fmt.Errorf("Failed loading local instances: %w", err) } // Register devices on running instances to receive events and reconnect to VM monitor sockets. // This should come after the event handler go routines have been started. devicesRegister(instances) // Setup seccomp handler seccompServer, err := seccomp.NewSeccompServer(d.State(), internalUtil.RunPath("seccomp.socket"), func(pid int32, state *state.State) (seccomp.Instance, error) { return findContainerForPid(pid, state) }) if err != nil { return err } d.seccomp = seccompServer logger.Info("Started seccomp handler", logger.Ctx{"path": internalUtil.RunPath("seccomp.socket")}) // Read the trusted certificates updateCertificateCache(d) } err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Remove volatile.last_state.ready key as we don't know if the instances are ready. return tx.DeleteReadyStateFromLocalInstances(ctx) }) if err != nil { return fmt.Errorf("Failed deleting volatile.last_state.ready: %w", err) } close(d.setupChan) _ = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Create warnings that have been collected for _, w := range dbWarnings { err := tx.UpsertWarningLocalNode(ctx, "", -1, -1, warningtype.Type(w.TypeCode), w.LastMessage) if err != nil { logger.Warn("Failed to create warning", logger.Ctx{"err": err}) } } return nil }) // Resolve warnings older than the daemon start time err = warnings.ResolveWarningsByLocalNodeOlderThan(d.db.Cluster, d.startTime) if err != nil { logger.Warn("Failed to resolve warnings", logger.Ctx{"err": err}) } // Start cluster tasks if needed. if d.serverClustered { d.startClusterTasks() } // FIXME: There's no hard reason for which we should not run these // tasks in mock mode. However it requires that we tweak them so // they exit gracefully without blocking (something we should do // anyways) and they don't hit the internet or similar. Support // for proper cancellation is something that has been started // but has not been fully completed. if !d.os.MockMode { // Log expiry (daily) d.tasks.Add(expireLogsTask(d.State())) // Remove expired images (daily) d.taskPruneImages = d.tasks.Add(pruneExpiredImagesTask(d)) // Auto-update images (every 6 hours, configurable) d.tasks.Add(autoUpdateImagesTask(d)) // Auto-update instance types (daily) d.tasks.Add(instanceRefreshTypesTask(d)) // Remove expired backups (hourly) d.tasks.Add(pruneExpiredBackupsTask(d)) // Prune expired instance snapshots and take snapshot of instances (minutely check of configurable cron expression) d.tasks.Add(pruneExpiredAndAutoCreateInstanceSnapshotsTask(d)) // Prune expired custom volume snapshots and take snapshots of custom volumes (minutely check of configurable cron expression) d.tasks.Add(pruneExpiredAndAutoCreateCustomVolumeSnapshotsTask(d)) // Remove resolved warnings (daily) d.tasks.Add(pruneResolvedWarningsTask(d)) // Auto-renew server certificate (daily) d.tasks.Add(autoRenewCertificateTask(d)) // Remove expired tokens (hourly) d.tasks.Add(autoRemoveExpiredTokensTask(d)) } // Start all background tasks d.tasks.Start(d.shutdownCtx) // Restore instances instancesStart(d.State(), instances) // Re-balance in case things changed while the daemon was down deviceTaskBalance(d.State()) // Unblock incoming requests d.waitReady.Cancel() logger.Info("Daemon started") return nil } func (d *Daemon) startClusterTasks() { // Add initial event listeners from global database members. // Run asynchronously so that connecting to remote members doesn't delay starting up other cluster tasks. go cluster.EventsUpdateListeners(d.State(), nil, d.events.Inject) // Heartbeats d.taskClusterHeartbeat = d.clusterTasks.Add(cluster.HeartbeatTask(d.gateway)) // Auto-sync images across the cluster (hourly) d.clusterTasks.Add(autoSyncImagesTask(d.State())) // Remove orphaned operations d.clusterTasks.Add(autoRemoveOrphanedOperationsTask(d.State())) // Perform automatic evacuation for offline cluster members d.clusterTasks.Add(autoHealClusterTask(d)) // Perform automatic live-migration to alance load on cluster d.clusterTasks.Add(autoRebalanceClusterTask(d)) // Start all background tasks d.clusterTasks.Start(d.shutdownCtx) } func (d *Daemon) stopClusterTasks() { _ = d.clusterTasks.Stop(3 * time.Second) d.clusterTasks = task.Group{} } // numRunningInstances returns the number of running instances. func (d *Daemon) numRunningInstances(instances []instance.Instance) int { count := 0 for _, instance := range instances { if instance.IsRunning() { count = count + 1 } } return count } // Stop stops the shared daemon. func (d *Daemon) Stop(ctx context.Context, sig os.Signal) error { logger.Info("Starting shutdown sequence", logger.Ctx{"signal": sig}) // Cancelling the context will make everyone aware that we're shutting down. d.shutdownCancel() if d.loggingController != nil { d.loggingController.Shutdown() } if d.gateway != nil { d.stopClusterTasks() err := handoverMemberRole(d.State(), d.gateway) if err != nil { logger.Warn("Could not handover member's responsibilities", logger.Ctx{"err": err}) d.gateway.Kill() } } s := d.State() var err error var instances []instance.Instance var instancesLoaded bool // If this is left as false this indicates an error loading instances. if d.db.Cluster != nil { instances, err = instance.LoadNodeAll(s, instancetype.Any) if err != nil { // List all instances on disk. logger.Warn("Loading local instances from disk as database is not available", logger.Ctx{"err": err}) instances, err = instancesOnDisk(s) if err != nil { logger.Warn("Failed loading instances from disk", logger.Ctx{"err": err}) } // Make all future queries fail fast as DB is not available. d.gateway.Kill() _ = d.db.Cluster.Close() } if err == nil { instancesLoaded = true } } // Handle shutdown (unix.SIGPWR) and reload (unix.SIGTERM) signals. if sig == unix.SIGPWR || sig == unix.SIGTERM { if d.db.Cluster != nil { // waitForOperations will block until all operations are done, or it's forced to shut down. // For the latter case, we reuse the shutdown channel which is filled when a shutdown is // initiated using `shutdown`. waitForOperations(ctx, d.db.Cluster, s.GlobalConfig.ShutdownTimeout()) } // Unmount daemon image and backup volumes if set. logger.Info("Stopping daemon storage volumes") done := make(chan struct{}) go func() { err := daemonStorageVolumesUnmount(s) if err != nil { logger.Error("Failed to unmount image and backup volumes", logger.Ctx{"err": err}) } done <- struct{}{} }() // Only wait 60 seconds in case the storage backend is unreachable. select { case <-time.After(time.Minute): logger.Error("Timed out waiting for image and backup volume") case <-done: } // Full shutdown requested. if sig == unix.SIGPWR { // Check if we should evacuate the cluster member instead of just shutting down. evacuated := false if d.serverClustered && s.GlobalConfig.ShutdownAction() == "evacuate" && !s.DB.Cluster.LocalNodeIsEvacuated() { logger.Info("Evacuating cluster member") err := evacuateShutdown(ctx, s, d.serverName) if err != nil { logger.Error("Failed to evacuate cluster member, falling back to regular shutdown", logger.Ctx{"err": err}) } else { evacuated = true } } if !evacuated { instancesShutdown(instances) logger.Info("Stopping networks") networkShutdown(s) } // Unmount storage pools after instances stopped. logger.Info("Stopping storage pools") var pools []string err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error pools, err = tx.GetStoragePoolNames(ctx) return err }) if err != nil && !response.IsNotFoundError(err) { logger.Error("Failed to get storage pools", logger.Ctx{"err": err}) } for _, poolName := range pools { pool, err := storagePools.LoadByName(s, poolName) if err != nil { logger.Error("Failed to get storage pool", logger.Ctx{"pool": poolName, "err": err}) continue } _, err = pool.Unmount() if err != nil { logger.Error("Unable to unmount storage pool", logger.Ctx{"pool": poolName, "err": err}) continue } } } } if d.gateway != nil { d.gateway.Kill() } errs := []error{} trackError := func(err error, desc string) { if err != nil { errs = append(errs, fmt.Errorf(desc+": %w", err)) } } trackError(d.tasks.Stop(3*time.Second), "Stop tasks") // Give tasks a bit of time to cleanup. trackError(d.clusterTasks.Stop(3*time.Second), "Stop cluster tasks") // Give tasks a bit of time to cleanup. n := d.numRunningInstances(instances) shouldUnmount := instancesLoaded && n <= 0 if d.db.Cluster != nil { logger.Info("Closing the database") err := d.db.Cluster.Close() if err != nil { logger.Debug("Could not close global database cleanly", logger.Ctx{"err": err}) } } if d.db != nil && d.db.Node != nil { trackError(d.db.Node.Close(), "Close local database") } if d.gateway != nil { trackError(d.gateway.Shutdown(), "Shutdown dqlite") } if d.endpoints != nil { trackError(d.endpoints.Down(), "Shutdown endpoints") } if shouldUnmount { logger.Info("Unmounting temporary filesystems") _ = unix.Unmount(internalUtil.VarPath("devices"), unix.MNT_DETACH) _ = unix.Unmount(internalUtil.VarPath("guestapi"), unix.MNT_DETACH) _ = unix.Unmount(internalUtil.VarPath("shmounts"), unix.MNT_DETACH) logger.Info("Done unmounting temporary filesystems") } else { logger.Info("Not unmounting temporary filesystems (instances are still running)") } if d.seccomp != nil { trackError(d.seccomp.Stop(), "Stop seccomp") } n = len(errs) if n > 0 { format := "%v" if n > 1 { format += fmt.Sprintf(" (and %d more errors)", n) } err = fmt.Errorf(format, errs[0]) } if err != nil { logger.Error("Failed to cleanly shutdown daemon", logger.Ctx{"err": err}) } return err } // Setup OpenFGA. func (d *Daemon) setupOpenFGA(apiURL string, apiToken string, storeID string) error { var err error if d.authorizer != nil { err := d.authorizer.StopService(d.shutdownCtx) if err != nil { logger.Error("Failed to stop authorizer service", logger.Ctx{"error": err}) } } if apiURL == "" || apiToken == "" || storeID == "" { // Reset to default authorizer. d.authorizer, err = auth.LoadAuthorizer(d.shutdownCtx, auth.DriverTLS, logger.Log, d.clientCerts) if err != nil { return err } return nil } config := map[string]any{ "openfga.api.url": apiURL, "openfga.api.token": apiToken, "openfga.store.id": storeID, } reverter := revert.New() defer reverter.Fail() reverter.Add(func() { // Reset to default authorizer. d.authorizer, _ = auth.LoadAuthorizer(d.shutdownCtx, auth.DriverTLS, logger.Log, d.clientCerts) }) // Build the list of resources to update the model. refreshResources := func() (*auth.Resources, error) { isLeader := false leaderAddress, err := d.gateway.LeaderAddress() if err != nil { if errors.Is(err, cluster.ErrNodeIsNotClustered) { isLeader = true } else { return nil, err } } else if leaderAddress == d.localConfig.ClusterAddress() { isLeader = true } // If clustered and not running on a leader, skip the resource update. if !isLeader { return nil, nil } var resources auth.Resources err = d.db.Cluster.Transaction(d.shutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { err := query.Scan(ctx, tx.Tx(), "SELECT certificates.fingerprint FROM certificates", func(scan func(dest ...any) error) error { var fingerprint string err := scan(&fingerprint) if err != nil { return err } resources.CertificateObjects = append(resources.CertificateObjects, auth.ObjectCertificate(fingerprint)) return nil }) if err != nil { return err } err = query.Scan(ctx, tx.Tx(), "SELECT name FROM storage_pools", func(scan func(dest ...any) error) error { var storagePoolName string err := scan(&storagePoolName) if err != nil { return err } resources.StoragePoolObjects = append(resources.StoragePoolObjects, auth.ObjectStoragePool(storagePoolName)) return nil }) if err != nil { return err } err = query.Scan(ctx, tx.Tx(), "SELECT name FROM projects", func(scan func(dest ...any) error) error { var projectName string err := scan(&projectName) if err != nil { return err } resources.ProjectObjects = append(resources.ProjectObjects, auth.ObjectProject(projectName)) return nil }) if err != nil { return err } err = query.Scan(ctx, tx.Tx(), "SELECT images.fingerprint, projects.name FROM images JOIN projects ON projects.id=images.project_id", func(scan func(dest ...any) error) error { var imageFingerprint string var projectName string err := scan(&imageFingerprint, &projectName) if err != nil { return err } resources.ImageObjects = append(resources.ImageObjects, auth.ObjectImage(projectName, imageFingerprint)) return nil }) if err != nil { return err } err = query.Scan(ctx, tx.Tx(), "SELECT images_aliases.name, projects.name FROM images_aliases JOIN projects ON projects.id=images_aliases.project_id", func(scan func(dest ...any) error) error { var imageAliasName string var projectName string err := scan(&imageAliasName, &projectName) if err != nil { return err } resources.ImageAliasObjects = append(resources.ImageAliasObjects, auth.ObjectImageAlias(projectName, imageAliasName)) return nil }) if err != nil { return err } err = query.Scan(ctx, tx.Tx(), "SELECT instances.name, projects.name FROM instances JOIN projects ON projects.id=instances.project_id", func(scan func(dest ...any) error) error { var instanceName string var projectName string err := scan(&instanceName, &projectName) if err != nil { return err } resources.InstanceObjects = append(resources.InstanceObjects, auth.ObjectInstance(projectName, instanceName)) return nil }) if err != nil { return err } err = query.Scan(ctx, tx.Tx(), "SELECT networks.name, projects.name FROM networks JOIN projects ON projects.id=networks.project_id", func(scan func(dest ...any) error) error { var networkName string var projectName string err := scan(&networkName, &projectName) if err != nil { return err } resources.NetworkObjects = append(resources.NetworkObjects, auth.ObjectNetwork(projectName, networkName)) return nil }) if err != nil { return err } err = query.Scan(ctx, tx.Tx(), "SELECT networks_acls.name, projects.name FROM networks_acls JOIN projects ON projects.id=networks_acls.project_id", func(scan func(dest ...any) error) error { var networkACLName string var projectName string err := scan(&networkACLName, &projectName) if err != nil { return err } resources.NetworkACLObjects = append(resources.NetworkACLObjects, auth.ObjectNetworkACL(projectName, networkACLName)) return nil }) if err != nil { return err } err = query.Scan(ctx, tx.Tx(), "SELECT networks_address_sets.name, projects.name FROM networks_address_sets JOIN projects ON projects.id=networks_address_sets.project_id", func(scan func(dest ...any) error) error { var networkAddressSetName string var projectName string err := scan(&networkAddressSetName, &projectName) if err != nil { return err } resources.NetworkAddressSetObjects = append(resources.NetworkAddressSetObjects, auth.ObjectNetworkAddressSet(projectName, networkAddressSetName)) return nil }) if err != nil { return err } err = query.Scan(ctx, tx.Tx(), "SELECT networks_zones.name, projects.name FROM networks_zones JOIN projects ON projects.id=networks_zones.project_id", func(scan func(dest ...any) error) error { var networkZoneName string var projectName string err := scan(&networkZoneName, &projectName) if err != nil { return err } resources.NetworkZoneObjects = append(resources.NetworkZoneObjects, auth.ObjectNetworkZone(projectName, networkZoneName)) return nil }) if err != nil { return err } err = query.Scan(ctx, tx.Tx(), "SELECT profiles.name, projects.name FROM profiles JOIN projects ON projects.id=profiles.project_id", func(scan func(dest ...any) error) error { var profileName string var projectName string err := scan(&profileName, &projectName) if err != nil { return err } resources.ProfileObjects = append(resources.ProfileObjects, auth.ObjectProfile(projectName, profileName)) return nil }) if err != nil { return err } err = query.Scan(ctx, tx.Tx(), "SELECT storage_volumes.name, storage_volumes.type, storage_pools.name, projects.name, nodes.name FROM storage_volumes JOIN projects ON projects.id=storage_volumes.project_id JOIN storage_pools ON storage_pools.id=storage_volumes.storage_pool_id LEFT JOIN nodes ON storage_volumes.node_id=nodes.id", func(scan func(dest ...any) error) error { var storageVolumeName string var storageVolumeType int var storageVolumeLocation sql.NullString var storagePoolName string var projectName string err := scan(&storageVolumeName, &storageVolumeType, &storagePoolName, &projectName, &storageVolumeLocation) if err != nil { return err } storageVolumeTypeName, err := db.StoragePoolVolumeTypeToName(storageVolumeType) if err != nil { return err } var location string if d.serverClustered && storageVolumeType != db.StoragePoolVolumeTypeContainer && storageVolumeType != db.StoragePoolVolumeTypeVM && storageVolumeLocation.Valid { location = storageVolumeLocation.String } resources.StoragePoolVolumeObjects = append(resources.StoragePoolVolumeObjects, auth.ObjectStorageVolume(projectName, storagePoolName, storageVolumeTypeName, storageVolumeName, location)) return nil }) if err != nil { return err } err = query.Scan(ctx, tx.Tx(), "SELECT storage_buckets.name, storage_pools.name, projects.name, nodes.name FROM storage_buckets JOIN projects ON projects.id=storage_buckets.project_id JOIN storage_pools ON storage_pools.id=storage_buckets.storage_pool_id LEFT JOIN nodes ON storage_buckets.node_id=nodes.id", func(scan func(dest ...any) error) error { var storageBucketName string var storageBucketLocation sql.NullString var storagePoolName string var projectName string err := scan(&storageBucketName, &storagePoolName, &projectName, &storageBucketLocation) if err != nil { return err } var location string if d.serverClustered && storageBucketLocation.Valid { location = storageBucketLocation.String } resources.StorageBucketObjects = append(resources.StorageBucketObjects, auth.ObjectStorageBucket(projectName, storagePoolName, storageBucketName, location)) return nil }) if err != nil { return err } return nil }) if err != nil { return nil, err } return &resources, nil } openfgaAuthorizer, err := auth.LoadAuthorizer(d.shutdownCtx, auth.DriverOpenFGA, logger.Log, d.clientCerts, auth.WithConfig(config), auth.WithResourcesFunc(refreshResources)) if err != nil { return err } d.authorizer = openfgaAuthorizer reverter.Success() return nil } // Setup authorization scriptlet. func (d *Daemon) setupAuthorizationScriptlet(scriptlet string) error { err := scriptletLoad.AuthorizationSet(scriptlet) if err != nil { return fmt.Errorf("Failed saving authorization scriptlet: %w", err) } if scriptlet == "" { // Reset to default authorizer. d.authorizer, err = auth.LoadAuthorizer(d.shutdownCtx, auth.DriverTLS, logger.Log, d.clientCerts) if err != nil { return err } return nil } // Fail if not using the default tls or scriptlet authorizer. switch d.authorizer.(type) { case *auth.TLS, *auth.Scriptlet: d.authorizer, err = auth.LoadAuthorizer(d.shutdownCtx, auth.DriverScriptlet, logger.Log, d.clientCerts) if err != nil { return err } default: return errors.New("Attempting to setup scriptlet authorization while another authorizer is already set") } return nil } // Syslog listener. func (d *Daemon) setupSyslogSocket(enable bool) error { // Always cancel the context to ensure that no goroutines leak. if d.syslogSocketCancel != nil { logger.Debug("Stopping syslog socket") d.syslogSocketCancel() } if !enable { return nil } var ctx context.Context ctx, d.syslogSocketCancel = context.WithCancel(d.shutdownCtx) logger.Debug("Starting syslog socket") err := syslog.Listen(ctx, d.events) if err != nil { return err } return nil } // Create a database connection and perform any updates needed. func initializeDbObject(d *Daemon) error { logger.Info("Initializing local database") // Hook to run when the local database is created from scratch. It will // create the default profile and mark all patches as applied. freshHook := func(db *db.Node) error { for _, patchName := range patchesGetNames() { err := db.MarkPatchAsApplied(patchName) if err != nil { return err } } return nil } var err error d.db.Node, err = db.OpenNode(filepath.Join(d.os.VarDir, "database"), freshHook) if err != nil { return fmt.Errorf("Error creating database: %s", err) } return nil } // hasMemberStateChanged returns true if the number of members, their addresses or state has changed. func (d *Daemon) hasMemberStateChanged(heartbeatData *cluster.APIHeartbeat) bool { // No previous heartbeat data. if d.lastNodeList == nil { return true } // Member count has changed. if len(d.lastNodeList.Members) != len(heartbeatData.Members) { return true } // Check for member address or state changes. for lastMemberID, lastMember := range d.lastNodeList.Members { if heartbeatData.Members[lastMemberID].Address != lastMember.Address { return true } if heartbeatData.Members[lastMemberID].Online != lastMember.Online { return true } } return false } // heartbeatHandler handles heartbeat requests from other cluster members. func (d *Daemon) heartbeatHandler(w http.ResponseWriter, _ *http.Request, isLeader bool, hbData *cluster.APIHeartbeat) { var err error // Look for time skews. now := time.Now().UTC() if hbData.Time.Add(5*time.Second).Before(now) || hbData.Time.Add(-5*time.Second).After(now) { if !d.timeSkew { logger.Warn("Time skew detected between leader and local", logger.Ctx{"leaderTime": hbData.Time, "localTime": now}) if d.db.Cluster != nil { err := d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpsertWarningLocalNode(ctx, "", -1, -1, warningtype.ClusterTimeSkew, fmt.Sprintf("leaderTime: %s, localTime: %s", hbData.Time, now)) }) if err != nil { logger.Warn("Failed to create cluster time skew warning", logger.Ctx{"err": err}) } } } d.timeSkew = true } else { if d.timeSkew { logger.Warn("Time skew resolved") if d.db.Cluster != nil { err := warnings.ResolveWarningsByLocalNodeAndType(d.db.Cluster, warningtype.ClusterTimeSkew) if err != nil { logger.Warn("Failed to resolve cluster time skew warning", logger.Ctx{"err": err}) } } d.timeSkew = false } } // Extract the raft nodes from the heartbeat info. raftNodes := make([]db.RaftNode, 0) for _, node := range hbData.Members { if node.RaftID > 0 { raftNodes = append(raftNodes, db.RaftNode{ NodeInfo: dqliteClient.NodeInfo{ ID: node.RaftID, Address: node.Address, Role: db.RaftRole(node.RaftRole), }, Name: node.Name, }) } } // Check we have been sent at least 1 raft node before wiping our set. if len(raftNodes) <= 0 { logger.Error("Empty raft member set received") http.Error(w, "400 Empty raft member set received", http.StatusBadRequest) return } // Accept raft node list from any heartbeat type so that we get freshest data quickly. logger.Debug("Replace current raft nodes", logger.Ctx{"raftMembers": raftNodes}) err = d.db.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { return tx.ReplaceRaftNodes(raftNodes) }) if err != nil { logger.Error("Error updating raft members", logger.Ctx{"err": err}) http.Error(w, "500 failed to update raft nodes", http.StatusInternalServerError) return } if hbData.FullStateList { // If there is an ongoing heartbeat round (and by implication this is the leader), then this could // be a problem because it could be broadcasting the stale member state information which in turn // could lead to incorrect decisions being made. So calling heartbeatRestart will request any // ongoing heartbeat round to cancel itself prematurely and restart another one. If there is no // ongoing heartbeat round or this member isn't the leader then this function call is a no-op and // will return false. If the heartbeat is restarted, then the heartbeat refresh task will be called // at the end of the heartbeat so no need to do it here. if !isLeader || !d.gateway.HeartbeatRestart() { // Run heartbeat refresh task async so heartbeat response is sent to leader straight away. go d.nodeRefreshTask(hbData, isLeader, nil) } } else { if isLeader { logger.Error("Partial heartbeat should not be sent to leader") http.Error(w, "400 Partial heartbeat should not be sent to leader", http.StatusBadRequest) return } logger.Debug("Partial heartbeat received") } } // nodeRefreshTask is run when a full state heartbeat is sent (on the leader) or received (by a non-leader member). // Is is used to check for member state changes and trigger refreshes of the certificate cache. // It also triggers member role promotion when run on the isLeader is true. // When run on the leader, it accepts a list of unavailableMembers that have not responded to the current heartbeat // round (but may not be considered actually offline at this stage). These unavailable members will not be used for // role rebalancing. func (d *Daemon) nodeRefreshTask(heartbeatData *cluster.APIHeartbeat, isLeader bool, unavailableMembers []string) { s := d.State() // Don't process the heartbeat until we're fully online. if d.db.Cluster == nil || d.db.Cluster.GetNodeID() == 0 { return } localClusterAddress := s.LocalConfig.ClusterAddress() if !heartbeatData.FullStateList || len(heartbeatData.Members) <= 0 { logger.Error("Heartbeat member refresh task called with partial state list", logger.Ctx{"local": localClusterAddress}) return } if heartbeatData.Version.MinAPIExtensions > 0 && heartbeatData.Version.MinAPIExtensions != d.apiExtensions { d.apiExtensions = heartbeatData.Version.MinAPIExtensions } // If the max version of the cluster has changed, check whether we need to upgrade. if d.lastNodeList == nil || d.lastNodeList.Version.APIExtensions != heartbeatData.Version.APIExtensions || d.lastNodeList.Version.Schema != heartbeatData.Version.Schema { err := cluster.MaybeUpdate(s) if err != nil { logger.Error("Error updating", logger.Ctx{"err": err}) return } } stateChangeTaskFailure := false // Records whether any of the state change tasks failed. // Handle potential OVN chassis changes. err := networkUpdateOVNChassis(s, heartbeatData, localClusterAddress) if err != nil { stateChangeTaskFailure = true logger.Error("Error restarting OVN networks", logger.Ctx{"err": err}) } if d.hasMemberStateChanged(heartbeatData) { logger.Info("Cluster status has changed, refreshing") // Refresh cluster certificates cached. updateCertificateCache(d) } // Refresh event listeners from heartbeat members (after certificates refreshed if needed). // Run asynchronously so that connecting to remote members doesn't delay other heartbeat tasks. wg := sync.WaitGroup{} wg.Add(1) go func() { cluster.EventsUpdateListeners(d.State(), heartbeatData.Members, d.events.Inject) wg.Done() }() // Only update the node list if there are no state change task failures. // If there are failures, then we leave the old state so that we can re-try the tasks again next heartbeat. if !stateChangeTaskFailure { d.lastNodeList = heartbeatData } // If we are leader and called from the leader heartbeat send function (unavailbleMembers != nil) and there // are other members in the cluster, then check if we need to update roles. We do not want to do this if // we are called on the leader as part of a notification heartbeat being received from another member. if isLeader && unavailableMembers != nil && len(heartbeatData.Members) > 1 { isDegraded := false hasNodesNotPartOfRaft := false hasDbClientToProcess := false onlineVoters := 0 onlineStandbys := 0 for _, node := range heartbeatData.Members { role := db.RaftRole(node.RaftRole) if node.Online { // Count online members that have voter or stand-by raft role. switch role { case db.RaftVoter: onlineVoters++ case db.RaftStandBy: onlineStandbys++ } if node.RaftID == 0 { hasNodesNotPartOfRaft = true } // Check if a 'database-client' node currently has a raft role other than 'spare'. if slices.Contains(node.Roles, db.ClusterRoleDatabaseClient) && node.RaftRole != int(db.RaftSpare) { hasDbClientToProcess = true } } else if role != db.RaftSpare { isDegraded = true // Offline member that has voter or stand-by raft role. } } maxVoters := s.GlobalConfig.MaxVoters() maxStandBy := s.GlobalConfig.MaxStandBy() // If there are offline members that have voter or stand-by database roles, let's see if we can // replace them with spare ones. Also, if we don't have enough voters or standbys, let's see if we // can upgrade some member. if isDegraded || onlineVoters != int(maxVoters) || onlineStandbys != int(maxStandBy) || hasDbClientToProcess { d.clusterMembershipMutex.Lock() logger.Debug("Rebalancing member roles in heartbeat", logger.Ctx{"local": localClusterAddress}) err := rebalanceMemberRoles(d.State(), d.gateway, nil, unavailableMembers) if err != nil && !errors.Is(err, cluster.ErrNotLeader) { logger.Warn("Could not rebalance cluster member roles", logger.Ctx{"err": err, "local": localClusterAddress}) } d.clusterMembershipMutex.Unlock() } if hasNodesNotPartOfRaft { d.clusterMembershipMutex.Lock() logger.Debug("Upgrading members without raft role in heartbeat", logger.Ctx{"local": localClusterAddress}) err := upgradeNodesWithoutRaftRole(d.State(), d.gateway) if err != nil && !errors.Is(err, cluster.ErrNotLeader) { logger.Warn("Failed upgrading raft roles:", logger.Ctx{"err": err, "local": localClusterAddress}) } d.clusterMembershipMutex.Unlock() } } wg.Wait() } func (d *Daemon) setupOVN() error { d.ovnMu.Lock() defer d.ovnMu.Unlock() // Clear any existing clients. d.ovnnb = nil d.ovnsb = nil // Connect to OpenVswitch. vswitch, err := d.getOVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } // Get the OVN southbound address. ovnSBAddr, err := vswitch.GetOVNSouthboundDBRemoteAddress(d.shutdownCtx) if err != nil { return fmt.Errorf("Failed to get OVN southbound connection string: %w", err) } // Get the OVN northbound address. ovnNBAddr := d.globalConfig.NetworkOVNNorthboundConnection() // Get the SSL certificates if needed. sslCACert, sslClientCert, sslClientKey := d.globalConfig.NetworkOVNSSL() // Fallback to filesystem keys. if sslCACert == "" { content, err := os.ReadFile("/etc/ovn/ovn-central.crt") if err == nil { sslCACert = string(content) } } if sslClientCert == "" { content, err := os.ReadFile("/etc/ovn/cert_host") if err == nil { sslClientCert = string(content) } } if sslClientKey == "" { content, err := os.ReadFile("/etc/ovn/key_host") if err == nil { sslClientKey = string(content) } } // Get OVN northbound client. ovnnb, err := ovn.NewNB(ovnNBAddr, sslCACert, sslClientCert, sslClientKey) if err != nil { return err } // Get OVN southbound client. ovnsb, err := ovn.NewSB(ovnSBAddr, sslCACert, sslClientCert, sslClientKey) if err != nil { return err } // Set the clients. d.ovnnb = ovnnb d.ovnsb = ovnsb return nil } func (d *Daemon) getOVN() (*ovn.NB, *ovn.SB, error) { if d.ovnnb == nil || d.ovnsb == nil { err := d.setupOVN() if err != nil { return nil, nil, fmt.Errorf("Failed to connect to OVN: %w", err) } } return d.ovnnb, d.ovnsb, nil } func (d *Daemon) setupOVS() error { d.ovsMu.Lock() defer d.ovsMu.Unlock() // Clear any existing client. d.ovs = nil // Connect to OpenVswitch. vswitch, err := ovs.NewVSwitch(d.localConfig.NetworkOVSConnection()) if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } // Set the client. d.ovs = vswitch return nil } func (d *Daemon) getOVS() (*ovs.VSwitch, error) { if d.ovs == nil { err := d.setupOVS() if err != nil { return nil, fmt.Errorf("Failed to connect to OVS: %w", err) } } return d.ovs, nil } func (d *Daemon) setupLinstor() error { d.linstorMu.Lock() defer d.linstorMu.Unlock() // Clear any existing client. d.linstor = nil // Get the Linstor controller connection string. controllerConnection := d.globalConfig.LinstorControllerConnection() // Get the SSL certificates if needed. sslCACert, sslClientCert, sslClientKey := d.globalConfig.LinstorSSL() // Get Linstor client. client, err := linstor.NewClient(controllerConnection, sslCACert, sslClientCert, sslClientKey) if err != nil { return fmt.Errorf("Failed to connect to Linstor: %w", err) } // Set the client. d.linstor = client return nil } func (d *Daemon) getLinstor() (*linstor.Client, error) { // Setup the client if it does not exist. if d.linstor == nil { err := d.setupLinstor() if err != nil { return nil, err } } return d.linstor, nil } incus-7.0.0/cmd/incusd/daemon_config.go000066400000000000000000000020321517523235500200130ustar00rootroot00000000000000package main import ( "context" "maps" clusterConfig "github.com/lxc/incus/v7/internal/server/cluster/config" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/proxy" ) func daemonConfigRender(state *state.State) (map[string]string, error) { config := map[string]string{} // Turn the config into a JSON-compatible map. maps.Copy(config, state.GlobalConfig.Dump()) // Apply the local config. err := state.DB.Node.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { nodeConfig, err := node.ConfigLoad(ctx, tx) if err != nil { return err } maps.Copy(config, nodeConfig.Dump()) return nil }) if err != nil { return nil, err } return config, nil } func daemonConfigSetProxy(d *Daemon, config *clusterConfig.Config) { // Update the cached proxy function d.proxy = proxy.FromConfig( config.ProxyHTTPS(), config.ProxyHTTP(), config.ProxyIgnoreHosts(), ) } incus-7.0.0/cmd/incusd/daemon_images.go000066400000000000000000000441361517523235500200260ustar00rootroot00000000000000package main import ( "context" "crypto/sha256" "fmt" "io" "net/http" "os" "path/filepath" "slices" "time" incus "github.com/lxc/incus/v7/client" internalIO "github.com/lxc/incus/v7/internal/io" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/locking" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/cancel" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // imageDownloadArgs used with imageDownload. type imageDownloadArgs struct { ProjectName string Server string Protocol string Certificate string Secret string Alias string Type string SetCached bool PreferCached bool AutoUpdate bool Public bool StoragePool string Budget int64 SourceProjectName string } // imageOperationLock acquires a lock for operating on an image and returns the unlock function. func imageOperationLock(ctx context.Context, fingerprint string) (locking.UnlockFunc, error) { l := logger.AddContext(logger.Ctx{"fingerprint": fingerprint}) l.Debug("Acquiring lock for image") defer l.Debug("Lock acquired for image") return locking.Lock(ctx, fmt.Sprintf("ImageOperation_%s", fingerprint)) } // imageDownload resolves the image fingerprint and if not in the database, downloads it. func imageDownload(ctx context.Context, r *http.Request, s *state.State, op *operations.Operation, args *imageDownloadArgs) (*api.Image, bool, error) { var err error var ctxMap logger.Ctx var remote incus.ImageServer var info *api.Image // Check if the project allows retrieving the image. err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { err := project.AllowImageDownload(tx, args.ProjectName, args.Server) if err != nil { return err } return nil }) if err != nil { return nil, false, err } // Default protocol is Incus. Copy so that local modifications aren't propagated to args. protocol := args.Protocol if protocol == "" { protocol = "incus" } // Copy so that local modifications aren't propagated to args. alias := args.Alias // Default the fingerprint to the alias string we received fp := alias // Attempt to resolve the alias if args.Server != "" && slices.Contains([]string{"incus", "lxd", "oci", "simplestreams"}, protocol) { clientArgs := &incus.ConnectionArgs{ TLSServerCert: args.Certificate, UserAgent: version.UserAgent, Proxy: s.Proxy, CachePath: s.OS.CacheDir, CacheExpiry: time.Hour, SkipGetEvents: true, SkipGetServer: true, TempPath: internalUtil.VarPath("images"), } if slices.Contains([]string{"incus", "lxd"}, protocol) { // Setup client remote, err = incus.ConnectPublicIncus(args.Server, clientArgs) if err != nil { return nil, false, fmt.Errorf("Failed to connect to the server %q: %w", args.Server, err) } server, ok := remote.(incus.InstanceServer) if ok { remote = server.UseProject(args.SourceProjectName) } } else if protocol == "oci" { // Setup OCI client remote, err = incus.ConnectOCI(args.Server, clientArgs) if err != nil { return nil, false, fmt.Errorf("Failed to connect to oci server %q: %w", args.Server, err) } } else if protocol == "simplestreams" { // Setup simplestreams client remote, err = incus.ConnectSimpleStreams(args.Server, clientArgs) if err != nil { return nil, false, fmt.Errorf("Failed to connect to simple streams server %q: %w", args.Server, err) } } // For public images, handle aliases and initial metadata if args.Secret == "" { // Look for a matching alias. Note, this err message is lost! entry, _, err := remote.GetImageAliasType(args.Type, fp) if err == nil { fp = entry.Target } // Expand partial fingerprints info, _, err = remote.GetImage(fp) if err != nil { return nil, false, fmt.Errorf("Failed getting remote image info: %w", err) } fp = info.Fingerprint } } // Ensure we are the only ones operating on this image. unlock, err := imageOperationLock(ctx, fp) if err != nil { return nil, false, err } defer unlock() // If auto-update is on and we're being given the image by // alias, try to use a locally cached image matching the given // server/protocol/alias, regardless of whether it's stale or // not (we can assume that it will be not *too* stale since // auto-update is on). interval := s.GlobalConfig.ImagesAutoUpdateIntervalHours() if args.PreferCached && interval > 0 && alias != fp { err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { for _, architecture := range s.OS.Architectures { cachedFingerprint, err := tx.GetCachedImageSourceFingerprint(ctx, args.Server, args.Protocol, alias, args.Type, architecture) if err == nil && cachedFingerprint != fp { fp = cachedFingerprint break } } return nil }) if err != nil { return nil, false, err } } var imgInfo *api.Image err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Check if the image already exists in this project (partial hash match). _, imgInfo, err = tx.GetImage(ctx, fp, cluster.ImageFilter{Project: &args.ProjectName}) return err }) if err == nil { var nodeAddress string err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Check if the image is available locally or it's on another node. nodeAddress, err = tx.LocateImage(ctx, imgInfo.Fingerprint) return err }) if err != nil { return nil, false, fmt.Errorf("Failed locating image %q in the cluster: %w", imgInfo.Fingerprint, err) } if nodeAddress != "" { // The image is available from another node, let's try to import it. err = instanceImageTransfer(s, r, args.ProjectName, imgInfo.Fingerprint, nodeAddress) if err != nil { return nil, false, fmt.Errorf("Failed transferring image %q from %q: %w", imgInfo.Fingerprint, nodeAddress, err) } err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // As the image record already exists in the project, just add the node ID to the image. return tx.AddImageToLocalNode(ctx, args.ProjectName, imgInfo.Fingerprint) }) if err != nil { return nil, false, fmt.Errorf("Failed adding transferred image %q to local cluster member: %w", imgInfo.Fingerprint, err) } } } else if response.IsNotFoundError(err) { err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Check if the image already exists in some other project. _, imgInfo, err = tx.GetImageFromAnyProject(ctx, fp) return err }) if err == nil { var nodeAddress string err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Check if the image is available locally or it's on another node. Do this before creating // the missing DB record so we don't include ourself in the search results. nodeAddress, err = tx.LocateImage(ctx, imgInfo.Fingerprint) if err != nil { return fmt.Errorf("Locate image %q in the cluster: %w", imgInfo.Fingerprint, err) } // We need to insert the database entry for this project, including the node ID entry. err = tx.CreateImage(ctx, args.ProjectName, imgInfo.Fingerprint, imgInfo.Filename, imgInfo.Size, args.Public, imgInfo.AutoUpdate, imgInfo.Architecture, imgInfo.CreatedAt, imgInfo.ExpiresAt, imgInfo.Properties, imgInfo.Type, nil) if err != nil { return fmt.Errorf("Failed creating image record for project: %w", err) } // Mark the image as "cached" if downloading for an instance. if args.SetCached { err = tx.SetImageCachedAndLastUseDate(ctx, args.ProjectName, imgInfo.Fingerprint, time.Now().UTC()) if err != nil { return fmt.Errorf("Failed setting cached flag and last use date: %w", err) } } var id int id, imgInfo, err = tx.GetImage(ctx, fp, cluster.ImageFilter{Project: &args.ProjectName}) if err != nil { return err } return tx.CreateImageSource(ctx, id, args.Server, args.Protocol, args.Certificate, alias) }) if err != nil { return nil, false, err } // Transfer image if needed (after database record has been created above). if nodeAddress != "" { // The image is available from another node, let's try to import it. err = instanceImageTransfer(s, r, args.ProjectName, info.Fingerprint, nodeAddress) if err != nil { return nil, false, fmt.Errorf("Failed transferring image: %w", err) } } } } if imgInfo != nil { info = imgInfo ctxMap = logger.Ctx{"fingerprint": info.Fingerprint} logger.Debug("Image already exists in the DB", ctxMap) // If not requested in a particular pool, we're done. if args.StoragePool == "" { return info, false, nil } ctxMap["pool"] = args.StoragePool var poolID int64 var poolIDs []int64 err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Get the ID of the target storage pool. poolID, err = tx.GetStoragePoolID(ctx, args.StoragePool) if err != nil { return err } // Check if the image is already in the pool. poolIDs, err = tx.GetPoolsWithImage(ctx, info.Fingerprint) return err }) if err != nil { return nil, false, err } if slices.Contains(poolIDs, poolID) { logger.Debug("Image already exists on storage pool", ctxMap) return info, false, nil } // Import the image in the pool. logger.Debug("Image does not exist on storage pool", ctxMap) err = imageCreateInPool(s, info, args.StoragePool) if err != nil { ctxMap["err"] = err logger.Debug("Failed to create image on storage pool", ctxMap) return nil, false, fmt.Errorf("Failed to create image %q on storage pool %q: %w", info.Fingerprint, args.StoragePool, err) } logger.Debug("Created image on storage pool", ctxMap) return info, false, nil } // Begin downloading if op == nil { ctxMap = logger.Ctx{"alias": alias, "server": args.Server} } else { ctxMap = logger.Ctx{"trigger": op.URL(), "fingerprint": fp, "operation": op.ID(), "alias": alias, "server": args.Server} } logger.Info("Downloading image", ctxMap) // Cleanup any leftover from a past attempt destDir := internalUtil.VarPath("images") destName := filepath.Join(destDir, fp) failure := true cleanup := func() { if failure { _ = os.Remove(destName) _ = os.Remove(destName + ".rootfs") } } defer cleanup() // Setup a progress handler progress := func(progress ioprogress.ProgressData) { if op == nil { return } meta := op.Metadata() if meta == nil { meta = make(map[string]any) } if meta["download_progress"] != progress.Text { _ = op.ExtendMetadata(map[string]any{"download_progress": progress.Text}) } } var canceler *cancel.HTTPRequestCanceller if op != nil { canceler = cancel.NewHTTPRequestCanceller() op.SetCanceler(canceler) } if slices.Contains([]string{"incus", "lxd", "oci", "simplestreams"}, protocol) { // Create the target files dest, err := os.Create(destName) if err != nil { return nil, false, err } defer func() { _ = dest.Close() }() destRootfs, err := os.Create(destName + ".rootfs") if err != nil { return nil, false, err } defer func() { _ = destRootfs.Close() }() // Get the image information if info == nil { if args.Secret != "" { info, _, err = remote.GetPrivateImage(fp, args.Secret) if err != nil { return nil, false, err } // Expand the fingerprint now and mark alias string to match fp = info.Fingerprint alias = info.Fingerprint } else { info, _, err = remote.GetImage(fp) if err != nil { return nil, false, err } } } // Compatibility with older servers if info.Type == "" { info.Type = "container" } if args.Budget > 0 && info.Size > args.Budget { return nil, false, fmt.Errorf("Remote image with size %d exceeds allowed bugdget of %d", info.Size, args.Budget) } // Download the image var resp *incus.ImageFileResponse request := incus.ImageFileRequest{ MetaFile: io.ReadWriteSeeker(dest), RootfsFile: io.ReadWriteSeeker(destRootfs), ProgressHandler: progress, Canceler: canceler, DeltaSourceRetriever: func(fingerprint string, file string) string { path := internalUtil.VarPath("images", fmt.Sprintf("%s.%s", fingerprint, file)) if util.PathExists(path) { return path } return "" }, } if args.Secret != "" { resp, err = remote.GetPrivateImageFile(fp, args.Secret, request) } else { resp, err = remote.GetImageFile(fp, request) } if err != nil { return nil, false, err } // Truncate down to size if resp.RootfsSize > 0 { err = destRootfs.Truncate(resp.RootfsSize) if err != nil { return nil, false, err } } err = dest.Truncate(resp.MetaSize) if err != nil { return nil, false, err } // Deal with unified images if resp.RootfsSize == 0 { err := os.Remove(destName + ".rootfs") if err != nil { return nil, false, err } } err = dest.Close() if err != nil { return nil, false, err } err = destRootfs.Close() if err != nil { return nil, false, err } } else if protocol == "direct" { // Setup HTTP client httpClient, err := localUtil.HTTPClient(args.Certificate, s.Proxy) if err != nil { return nil, false, err } // Use relatively short response header timeout so as not to hold the image lock open too long. httpTransport := httpClient.Transport.(*http.Transport) httpTransport.ResponseHeaderTimeout = 30 * time.Second req, err := http.NewRequest("GET", args.Server, nil) if err != nil { return nil, false, err } req.Header.Set("User-Agent", version.UserAgent) // Make the request raw, doneCh, err := cancel.CancelableDownload(canceler, httpClient.Do, req) if err != nil { return nil, false, err } defer close(doneCh) if raw.StatusCode != http.StatusOK { return nil, false, fmt.Errorf("Unable to fetch %q: %s", args.Server, raw.Status) } // Progress handler body := &ioprogress.ProgressReader{ ReadCloser: raw.Body, Tracker: &ioprogress.ProgressTracker{ Length: raw.ContentLength, Handler: func(percent int64, speed int64) { progress(ioprogress.ProgressData{Text: fmt.Sprintf("%d%% (%s/s)", percent, units.GetByteSizeString(speed, 2))}) }, }, } // Create the target files f, err := os.Create(destName) if err != nil { return nil, false, err } defer func() { _ = f.Close() }() // Hashing hash256 := sha256.New() // Download the image writer := internalIO.NewQuotaWriter(io.MultiWriter(f, hash256), args.Budget) size, err := util.SafeCopy(writer, body) if err != nil { return nil, false, err } // Validate hash result := fmt.Sprintf("%x", hash256.Sum(nil)) if result != fp { return nil, false, fmt.Errorf("Hash mismatch for %q: %s != %s", args.Server, result, fp) } // Parse the image imageMeta, imageType, err := getImageMetadata(destName) if err != nil { return nil, false, err } info = &api.Image{} info.Fingerprint = fp info.Size = size info.Architecture = imageMeta.Architecture info.Properties = imageMeta.Properties info.Type = imageType if imageMeta.CreationDate > 0 { info.CreatedAt = time.Unix(imageMeta.CreationDate, 0) } if imageMeta.ExpiryDate > 0 { info.ExpiresAt = time.Unix(imageMeta.ExpiryDate, 0) } err = f.Close() if err != nil { return nil, false, err } } else { return nil, false, fmt.Errorf("Unsupported protocol: %v", protocol) } // Override visibility info.Public = args.Public // We want to enable auto-update only if we were passed an // alias name, so we can figure when the associated // fingerprint changes in the remote. if alias != fp { info.AutoUpdate = args.AutoUpdate } err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Create the database entry return tx.CreateImage(ctx, args.ProjectName, info.Fingerprint, info.Filename, info.Size, info.Public, info.AutoUpdate, info.Architecture, info.CreatedAt, info.ExpiresAt, info.Properties, info.Type, nil) }) if err != nil { return nil, false, fmt.Errorf("Failed creating image record: %w", err) } // Image is in the DB now, don't wipe on-disk files on failure failure = false // Check if the image path changed (private images) newDestName := filepath.Join(destDir, fp) if newDestName != destName { err = internalUtil.FileMove(destName, newDestName) if err != nil { return nil, false, err } if util.PathExists(destName + ".rootfs") { err = internalUtil.FileMove(destName+".rootfs", newDestName+".rootfs") if err != nil { return nil, false, err } } } // Record the image source if alias != fp { err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { id, _, err := tx.GetImage(ctx, fp, cluster.ImageFilter{Project: &args.ProjectName}) if err != nil { return err } return tx.CreateImageSource(ctx, id, args.Server, protocol, args.Certificate, alias) }) if err != nil { return nil, false, err } } // Import into the requested storage pool if args.StoragePool != "" { err = imageCreateInPool(s, info, args.StoragePool) if err != nil { return nil, false, err } } // Mark the image as "cached" if downloading for an instance if args.SetCached { err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.SetImageCachedAndLastUseDate(ctx, args.ProjectName, fp, time.Now().UTC()) }) if err != nil { return nil, false, fmt.Errorf("Failed setting cached flag and last use date: %w", err) } } logger.Info("Image downloaded", ctxMap) return info, true, nil } incus-7.0.0/cmd/incusd/daemon_integration_test.go000066400000000000000000000025171517523235500221400ustar00rootroot00000000000000package main import ( "context" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/server/sys" ) // The daemon is started and a client can connect to it via unix socket. func TestIntegration_UnixSocket(t *testing.T) { daemon, cleanup := newTestDaemon(t) defer cleanup() c, err := incus.ConnectIncusUnix(daemon.os.GetUnixSocket(), nil) require.NoError(t, err) server, _, err := c.GetServer() require.NoError(t, err) assert.Equal(t, "trusted", server.Auth) assert.False(t, server.Environment.ServerClustered) assert.False(t, c.IsClustered()) } // Create a new daemon for testing. // // Return a function that can be used to cleanup every associated state. func newTestDaemon(t *testing.T) (*Daemon, func()) { // OS os, osCleanup := sys.NewTestOS(t) // Daemon daemon := newDaemon(newConfig(), os) require.NoError(t, daemon.Init()) cleanup := func() { assert.NoError(t, daemon.Stop(context.Background(), unix.SIGQUIT)) osCleanup() } return daemon, cleanup } // Create a new DaemonConfig object for testing purposes. func newConfig() *DaemonConfig { return &DaemonConfig{ RaftLatency: 0.8, Trace: []string{"dqlite"}, DqliteSetupTimeout: 10 * time.Second, } } incus-7.0.0/cmd/incusd/daemon_storage.go000066400000000000000000000331471517523235500202250ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "os" "path/filepath" "strings" "github.com/lxc/incus/v7/internal/rsync" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) func daemonStorageVolumesUnmount(s *state.State) error { var storageBackups string var storageImages string err := s.DB.Node.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { nodeConfig, err := node.ConfigLoad(ctx, tx) if err != nil { return err } storageBackups = nodeConfig.StorageBackupsVolume() storageImages = nodeConfig.StorageImagesVolume() return nil }) if err != nil { return err } unmount := func(storageType string, source string) error { // Parse the source. poolName, volumeName, err := daemonStorageSplitVolume(source) if err != nil { return err } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return err } // Mount volume. _, err = pool.UnmountCustomVolume(api.ProjectDefaultName, volumeName, nil) if err != nil { return fmt.Errorf("Failed to unmount storage volume %q: %w", source, err) } return nil } if storageBackups != "" { err := unmount("backups", storageBackups) if err != nil { return fmt.Errorf("Failed to unmount backups storage: %w", err) } } if storageImages != "" { err := unmount("images", storageImages) if err != nil { return fmt.Errorf("Failed to unmount images storage: %w", err) } } return nil } func daemonStorageMount(s *state.State) error { var storageBackups string var storageImages string var storageLogs string err := s.DB.Node.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { nodeConfig, err := node.ConfigLoad(ctx, tx) if err != nil { return err } storageBackups = nodeConfig.StorageBackupsVolume() storageImages = nodeConfig.StorageImagesVolume() storageLogs = nodeConfig.StorageLogsVolume() return nil }) if err != nil { return err } mount := func(storageType string, source string) error { // Parse the source. poolName, volumeName, err := daemonStorageSplitVolume(source) if err != nil { return err } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return err } // Mount volume. _, err = pool.MountCustomVolume(api.ProjectDefaultName, volumeName, nil) if err != nil { return fmt.Errorf("Failed to mount storage volume %q: %w", source, err) } // Ensure we have the correct symlink in place. volStorageName := project.StorageVolume(api.ProjectDefaultName, volumeName) _ = os.RemoveAll(internalUtil.VarPath(storageType)) err = os.Symlink(internalUtil.VarPath("storage-pools", poolName, "custom", volStorageName), internalUtil.VarPath(storageType)) if err != nil { return fmt.Errorf("Failed to set up symlink for %q: %w", storageType, err) } return nil } if storageBackups != "" { err := mount("backups", storageBackups) if err != nil { return fmt.Errorf("Failed to mount backups storage: %w", err) } } if storageImages != "" { err := mount("images", storageImages) if err != nil { return fmt.Errorf("Failed to mount images storage: %w", err) } } if storageLogs != "" { err := mount("logs", storageLogs) if err != nil { return fmt.Errorf("Failed to mount logs storage: %w", err) } } return nil } func daemonStorageSplitVolume(volume string) (string, string, error) { fields := strings.Split(volume, "/") if len(fields) != 2 { return "", "", errors.New("Invalid syntax for volume, must be /") } poolName := fields[0] volumeName := fields[1] return poolName, volumeName, nil } func daemonStorageValidate(s *state.State, storageType string, target string) error { // Check syntax. if target == "" { return nil } poolName, volumeName, err := daemonStorageSplitVolume(target) if err != nil { return err } var poolID int64 var snapshots []db.StorageVolumeArgs err = s.DB.Cluster.Transaction(context.Background(), func(ctx context.Context, tx *db.ClusterTx) error { // Validate pool exists. poolID, _, _, err = tx.GetStoragePool(ctx, poolName) if err != nil { return fmt.Errorf("Unable to load storage pool %q: %w", poolName, err) } // Confirm volume exists. dbVol, err := tx.GetStoragePoolVolume(ctx, poolID, api.ProjectDefaultName, db.StoragePoolVolumeTypeCustom, volumeName, true) if err != nil { return fmt.Errorf("Failed loading storage volume %q in %q project: %w", target, api.ProjectDefaultName, err) } if dbVol.ContentType != db.StoragePoolVolumeContentTypeNameFS { return fmt.Errorf("Storage volume %q in %q project is not filesystem content type", target, api.ProjectDefaultName) } snapshots, err = tx.GetLocalStoragePoolVolumeSnapshotsWithType(ctx, api.ProjectDefaultName, volumeName, db.StoragePoolVolumeTypeCustom, poolID) if err != nil { return fmt.Errorf("Unable to load storage volume snapshots %q in %q project: %w", target, api.ProjectDefaultName, err) } return nil }) if err != nil { return err } if len(snapshots) != 0 { return errors.New("Storage volumes for use by Incus itself cannot have snapshots") } // If logs, ensure no running instances. if storageType == "logs" { localInstances, err := instance.LoadNodeAll(s, instancetype.Any) if err != nil { return fmt.Errorf("Failed loading local instance list: %w", err) } for _, inst := range localInstances { if inst.IsRunning() { return fmt.Errorf("`storage.logs_volume` cannot be changed if there are running instances") } } } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return err } // Mount volume. _, err = pool.MountCustomVolume(api.ProjectDefaultName, volumeName, nil) if err != nil { return fmt.Errorf("Failed to mount storage volume %q: %w", target, err) } defer func() { _, _ = pool.UnmountCustomVolume(api.ProjectDefaultName, volumeName, nil) }() // Validate volume is empty (ignore lost+found). volStorageName := project.StorageVolume(api.ProjectDefaultName, volumeName) mountpoint := storageDrivers.GetVolumeMountPath(poolName, storageDrivers.VolumeTypeCustom, volStorageName) entries, err := os.ReadDir(mountpoint) if err != nil { return fmt.Errorf("Failed to list %q: %w", mountpoint, err) } for _, entry := range entries { entryName := entry.Name() // Don't fail on clean ext4 volumes. if entryName == "lost+found" { continue } // Don't fail on systems with snapdir=visible. if entryName == ".zfs" { continue } return fmt.Errorf("Storage volume %q isn't empty", target) } return nil } func daemonStorageMove(s *state.State, storageType string, target string) error { var destPath string isLogs := storageType == "logs" if isLogs && target == "" && os.Getenv("INCUS_DIR") == "" { // We keep the system-wide location when dealing with logs without a custom daemon path. destPath = "/var/log/incus" } else { destPath = internalUtil.VarPath(storageType) } // Track down the current storage. var sourcePool string var sourceVolume string sourcePath, err := os.Readlink(internalUtil.VarPath(storageType)) if err != nil { sourcePath = destPath } else { fields := strings.Split(sourcePath, "/") sourcePool = fields[len(fields)-3] sourceVolume = fields[len(fields)-1] } moveContent := func(source string, target string) error { // Copy the content. _, err := rsync.LocalCopy(source, target, "", false) if err != nil { return err } // Remove the source content. entries, err := os.ReadDir(source) if err != nil { return err } for _, entry := range entries { err := os.RemoveAll(filepath.Join(source, entry.Name())) if err != nil { return err } } return nil } // We should not move all data away from /var/log/incus and move it into the volume as that would interfere // with /var/log/incus/incusd.log which is created prior to us setting up volume mounts moveInstanceDirs := func(source string, target string) error { entries, err := os.ReadDir(source) if err != nil { return err } for _, entry := range entries { if !entry.IsDir() { continue } src := filepath.Join(source, entry.Name()) dst := filepath.Join(target, entry.Name()) _, err := rsync.LocalCopy(src, dst, "", false) if err != nil { return err } err = os.RemoveAll(src) if err != nil { return err } } return nil } // Deal with unsetting. if target == "" { // Things already look correct. if sourcePath == destPath { return nil } // Remove the symlink. err = os.Remove(internalUtil.VarPath(storageType)) if err != nil { return fmt.Errorf("Failed to delete storage symlink at %q: %w", destPath, err) } // Re-create as a directory. // In the context of Logs, we ensure system log dir exists and move instance dirs back there err = os.MkdirAll(destPath, 0o700) if err != nil { return fmt.Errorf("Failed to create directory %q: %w", destPath, err) } if isLogs { err = moveInstanceDirs(sourcePath, destPath) if err != nil { return fmt.Errorf("Failed to move instance logs back to %q: %w", destPath, err) } } else { // Move the data across. err = moveContent(sourcePath, destPath) if err != nil { return fmt.Errorf("Failed to move data over to directory %q: %w", destPath, err) } } pool, err := storagePools.LoadByName(s, sourcePool) if err != nil { return err } // Unmount old volume. projectName, sourceVolumeName := project.StorageVolumeParts(sourceVolume) _, err = pool.UnmountCustomVolume(projectName, sourceVolumeName, nil) if err != nil { if !isLogs { return fmt.Errorf(`Failed to umount storage volume "%s/%s": %w`, sourcePool, sourceVolumeName, err) } logger.Warn("Unable to unmount logs storage, daemon restart required") } return nil } // Parse the target. poolName, volumeName, err := daemonStorageSplitVolume(target) if err != nil { return err } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return err } // Mount volume. _, err = pool.MountCustomVolume(api.ProjectDefaultName, volumeName, nil) if err != nil { return fmt.Errorf("Failed to mount storage volume %q: %w", target, err) } // Set ownership & mode. volStorageName := project.StorageVolume(api.ProjectDefaultName, volumeName) mountpoint := storageDrivers.GetVolumeMountPath(poolName, storageDrivers.VolumeTypeCustom, volStorageName) destPath = mountpoint err = os.Chmod(mountpoint, 0o700) if err != nil { return fmt.Errorf("Failed to set permissions on %q: %w", mountpoint, err) } err = os.Chown(mountpoint, 0, 0) if err != nil { return fmt.Errorf("Failed to set ownership on %q: %w", mountpoint, err) } // Handle changes. if sourcePath != internalUtil.VarPath(storageType) { // Remove the symlink. err := os.Remove(internalUtil.VarPath(storageType)) if err != nil { return fmt.Errorf("Failed to remove the new symlink at %q: %w", internalUtil.VarPath(storageType), err) } // Create the new symlink. err = os.Symlink(destPath, internalUtil.VarPath(storageType)) if err != nil { return fmt.Errorf("Failed to create the new symlink at %q: %w", internalUtil.VarPath(storageType), err) } // Move the data across. if isLogs { err = moveInstanceDirs(sourcePath, destPath) if err != nil { return fmt.Errorf("Failed to move data over to directory %q: %w", destPath, err) } } else { err = moveContent(sourcePath, destPath) if err != nil { return fmt.Errorf("Failed to move data over to directory %q: %w", destPath, err) } } pool, err := storagePools.LoadByName(s, sourcePool) if err != nil { return err } // Unmount old volume. projectName, sourceVolumeName := project.StorageVolumeParts(sourceVolume) _, err = pool.UnmountCustomVolume(projectName, sourceVolumeName, nil) if err != nil { if !isLogs { return fmt.Errorf(`Failed to umount storage volume "%s/%s": %w`, sourcePool, sourceVolumeName, err) } logger.Warn("Unable to unmount logs storage, daemon restart required") } return nil } // Rename the existing storage. if isLogs && os.Getenv("INCUS_DIR") == "" { sourcePath = "/var/log/incus" } else if util.PathExists(internalUtil.VarPath(storageType)) { sourcePath = internalUtil.VarPath(storageType) + ".temp" err = os.Rename(internalUtil.VarPath(storageType), sourcePath) if err != nil { return fmt.Errorf("Failed to rename existing storage %q: %w", internalUtil.VarPath(storageType), err) } } // Create the new symlink. err = os.Symlink(destPath, internalUtil.VarPath(storageType)) if err != nil { return fmt.Errorf("Failed to create the new symlink at %q: %w", internalUtil.VarPath(storageType), err) } // Move the data across. if isLogs { err = moveInstanceDirs(sourcePath, destPath) if err != nil { return fmt.Errorf("Failed to move data over to directory %q: %w", destPath, err) } } else { err = moveContent(sourcePath, destPath) if err != nil { return fmt.Errorf("Failed to move data over to directory %q: %w", destPath, err) } } // Remove the old data. if sourcePath != "/var/log/incus" { err = os.RemoveAll(sourcePath) if err != nil { return fmt.Errorf("Failed to cleanup old directory %q: %w", sourcePath, err) } } return nil } incus-7.0.0/cmd/incusd/dev_incus.go000066400000000000000000000422611517523235500172120ustar00rootroot00000000000000package main import ( "encoding/json" "errors" "fmt" "net" "net/http" "net/url" "os" "regexp" "strconv" "strings" "sync" "time" "github.com/gorilla/mux" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/events" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/server/ucred" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" apiGuest "github.com/lxc/incus/v7/shared/api/guest" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/ws" ) type hoistFunc func(f func(*Daemon, instance.Instance, http.ResponseWriter, *http.Request) response.Response, d *Daemon) func(http.ResponseWriter, *http.Request) // DevIncusServer creates an http.Server capable of handling requests against the // /dev/incus Unix socket endpoint created inside containers. func devIncusServer(d *Daemon) *http.Server { return &http.Server{ Handler: devIncusAPI(d, hoistReq), ConnState: pidMapper.ConnStateHandler, ConnContext: request.SaveConnectionInContext, IdleTimeout: 30 * time.Second, ReadHeaderTimeout: 3 * time.Second, ReadTimeout: 3 * time.Second, } } type devIncusHandler struct { path string /* * This API will have to be changed slightly when we decide to support * websocket events upgrading, but since we don't have events on the * server side right now either, I went the simple route to avoid * needless noise. */ f func(d *Daemon, c instance.Instance, w http.ResponseWriter, r *http.Request) response.Response } var devIncusConfigGet = devIncusHandler{"/1.0/config", func(d *Daemon, c instance.Instance, w http.ResponseWriter, r *http.Request) response.Response { if util.IsFalse(c.ExpandedConfig()["security.guestapi"]) { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusForbidden, "not authorized"), c.Type() == instancetype.VM) } filtered := []string{} for k := range c.ExpandedConfig() { if strings.HasPrefix(k, "user.") || strings.HasPrefix(k, "cloud-init.") { filtered = append(filtered, fmt.Sprintf("/1.0/config/%s", k)) } } return response.DevIncusResponse(http.StatusOK, filtered, "json", c.Type() == instancetype.VM) }} var devIncusConfigKeyGet = devIncusHandler{"/1.0/config/{key}", func(d *Daemon, c instance.Instance, w http.ResponseWriter, r *http.Request) response.Response { if util.IsFalse(c.ExpandedConfig()["security.guestapi"]) { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusForbidden, "not authorized"), c.Type() == instancetype.VM) } key, err := url.PathUnescape(mux.Vars(r)["key"]) if err != nil { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusBadRequest, "bad request"), c.Type() == instancetype.VM) } if !strings.HasPrefix(key, "user.") && !strings.HasPrefix(key, "cloud-init.") { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusForbidden, "not authorized"), c.Type() == instancetype.VM) } value, ok := c.ExpandedConfig()[key] if !ok { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusNotFound, "not found"), c.Type() == instancetype.VM) } return response.DevIncusResponse(http.StatusOK, value, "raw", c.Type() == instancetype.VM) }} var devIncusImageExport = devIncusHandler{"/1.0/images/{fingerprint}/export", func(d *Daemon, c instance.Instance, w http.ResponseWriter, r *http.Request) response.Response { if util.IsFalse(c.ExpandedConfig()["security.guestapi"]) { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusForbidden, "not authorized"), c.Type() == instancetype.VM) } if util.IsFalseOrEmpty(c.ExpandedConfig()["security.guestapi.images"]) { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusForbidden, "not authorized"), c.Type() == instancetype.VM) } // Use by security checks to distinguish /dev/incus vs REST APs r.RemoteAddr = "@dev_incus" resp := imageExport(d, r) err := resp.Render(w) if err != nil { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusInternalServerError, "internal server error"), c.Type() == instancetype.VM) } return response.DevIncusResponse(http.StatusOK, "", "raw", c.Type() == instancetype.VM) }} var devIncusMetadataGet = devIncusHandler{"/1.0/meta-data", func(d *Daemon, inst instance.Instance, w http.ResponseWriter, r *http.Request) response.Response { if util.IsFalse(inst.ExpandedConfig()["security.guestapi"]) { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusForbidden, "not authorized"), inst.Type() == instancetype.VM) } value := inst.ExpandedConfig()["user.meta-data"] return response.DevIncusResponse(http.StatusOK, fmt.Sprintf("#cloud-config\ninstance-id: %s\nlocal-hostname: %s\n%s", inst.CloudInitID(), inst.Name(), value), "raw", inst.Type() == instancetype.VM) }} var devIncusEventsGet = devIncusHandler{"/1.0/events", func(d *Daemon, c instance.Instance, w http.ResponseWriter, r *http.Request) response.Response { if util.IsFalse(c.ExpandedConfig()["security.guestapi"]) { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusForbidden, "not authorized"), c.Type() == instancetype.VM) } typeStr := r.FormValue("type") if typeStr == "" { typeStr = "config,device" } var listenerConnection events.EventListenerConnection var resp response.Response // If the client has not requested a websocket connection then fallback to long polling event stream mode. if r.Header.Get("Upgrade") == "websocket" { conn, err := ws.Upgrader.Upgrade(w, r, nil) if err != nil { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusInternalServerError, "internal server error"), c.Type() == instancetype.VM) } defer func() { _ = conn.Close() }() // Ensure listener below ends when this function ends. listenerConnection = events.NewWebsocketListenerConnection(conn) resp = response.DevIncusResponse(http.StatusOK, "websocket", "websocket", c.Type() == instancetype.VM) } else { h, ok := w.(http.Hijacker) if !ok { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusInternalServerError, "internal server error"), c.Type() == instancetype.VM) } conn, _, err := h.Hijack() if err != nil { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusInternalServerError, "internal server error"), c.Type() == instancetype.VM) } defer func() { _ = conn.Close() }() // Ensure listener below ends when this function ends. listenerConnection, err = events.NewStreamListenerConnection(conn) if err != nil { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusInternalServerError, "internal server error"), c.Type() == instancetype.VM) } resp = response.DevIncusResponse(http.StatusOK, "", "raw", c.Type() == instancetype.VM) } listener, err := d.State().DevIncusEvents.AddListener(c.ID(), listenerConnection, strings.Split(typeStr, ",")) if err != nil { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusInternalServerError, "internal server error"), c.Type() == instancetype.VM) } logger.Debug("New container event listener", logger.Ctx{"instance": c.Name(), "project": c.Project().Name, "listener_id": listener.ID}) listener.Wait(r.Context()) return resp }} var devIncusAPIHandler = devIncusHandler{"/1.0", func(d *Daemon, c instance.Instance, w http.ResponseWriter, r *http.Request) response.Response { s := d.State() if r.Method == "GET" { var location string if d.serverClustered { location = c.Location() } else { var err error location, err = os.Hostname() if err != nil { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusInternalServerError, "internal server error"), c.Type() == instancetype.VM) } } var state api.StatusCode if util.IsTrue(c.LocalConfig()["volatile.last_state.ready"]) { state = api.Ready } else { state = api.Started } return response.DevIncusResponse(http.StatusOK, apiGuest.DevIncusGet{APIVersion: version.APIVersion, Location: location, InstanceType: c.Type().String(), DevIncusPut: apiGuest.DevIncusPut{State: state.String()}}, "json", c.Type() == instancetype.VM) } else if r.Method == "PATCH" { if util.IsFalse(c.ExpandedConfig()["security.guestapi"]) { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusForbidden, "not authorized"), c.Type() == instancetype.VM) } req := apiGuest.DevIncusPut{} err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusBadRequest, "%s", err.Error()), c.Type() == instancetype.VM) } state := api.StatusCodeFromString(req.State) if state != api.Started && state != api.Ready { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusBadRequest, "Invalid state %q", req.State), c.Type() == instancetype.VM) } err = c.VolatileSet(map[string]string{"volatile.last_state.ready": strconv.FormatBool(state == api.Ready)}) if err != nil { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusInternalServerError, "%s", err.Error()), c.Type() == instancetype.VM) } if state == api.Ready { s.Events.SendLifecycle(c.Project().Name, lifecycle.InstanceReady.Event(c, nil)) } return response.DevIncusResponse(http.StatusOK, "", "raw", c.Type() == instancetype.VM) } return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusMethodNotAllowed, "%s", fmt.Sprintf("method %q not allowed", r.Method)), c.Type() == instancetype.VM) }} var devIncusDevicesGet = devIncusHandler{"/1.0/devices", func(d *Daemon, c instance.Instance, w http.ResponseWriter, r *http.Request) response.Response { if util.IsFalse(c.ExpandedConfig()["security.guestapi"]) { return response.DevIncusErrorResponse(api.StatusErrorf(http.StatusForbidden, "not authorized"), c.Type() == instancetype.VM) } // Populate NIC hwaddr from volatile if not explicitly specified. // This is so cloud-init running inside the instance can identify the NIC when the interface name is // different than the device name (such as when run inside a VM). localConfig := c.LocalConfig() devices := c.ExpandedDevices() for devName, devConfig := range devices { if devConfig["type"] == "nic" && devConfig["hwaddr"] == "" && localConfig[fmt.Sprintf("volatile.%s.hwaddr", devName)] != "" { devices[devName]["hwaddr"] = localConfig[fmt.Sprintf("volatile.%s.hwaddr", devName)] } } return response.DevIncusResponse(http.StatusOK, c.ExpandedDevices(), "json", c.Type() == instancetype.VM) }} var handlers = []devIncusHandler{ {"/", func(d *Daemon, c instance.Instance, w http.ResponseWriter, r *http.Request) response.Response { return response.DevIncusResponse(http.StatusOK, []string{"/1.0"}, "json", c.Type() == instancetype.VM) }}, devIncusAPIHandler, devIncusConfigGet, devIncusConfigKeyGet, devIncusMetadataGet, devIncusEventsGet, devIncusImageExport, devIncusDevicesGet, } func hoistReq(f func(*Daemon, instance.Instance, http.ResponseWriter, *http.Request) response.Response, d *Daemon) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { conn := ucred.GetConnFromContext(r.Context()) cred, ok := pidMapper.m[conn.(*net.UnixConn)] if !ok { http.Error(w, errPIDNotInContainer.Error(), http.StatusInternalServerError) return } s := d.State() c, err := findContainerForPid(cred.Pid, s) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } // Access control rootUID := uint32(0) idmapset, err := c.CurrentIdmap() if err == nil && idmapset != nil { uid, _ := idmapset.ShiftIntoNS(0, 0) rootUID = uint32(uid) } if rootUID != cred.Uid { http.Error(w, "Access denied for non-root user", http.StatusUnauthorized) return } resp := f(d, c, w, r) _ = resp.Render(w) } } func devIncusAPI(d *Daemon, f hoistFunc) http.Handler { router := mux.NewRouter() router.UseEncodedPath() // Allow encoded values in path segments. for _, handler := range handlers { router.HandleFunc(handler.path, f(handler.f, d)) } return router } /* * Everything below here is the guts of the unix socket bits. Unfortunately, * golang's API does not make this easy. What happens is: * * 1. We install a ConnState listener on the http.Server, which does the * initial unix socket credential exchange. When we get a connection started * event, we use SO_PEERCRED to extract the creds for the socket. * * 2. We store a map from the connection pointer to the pid for that * connection, so that once the HTTP negotiation occurs and we get a * ResponseWriter, we know (because we negotiated on the first byte) which * pid the connection belongs to. * * 3. Regular HTTP negotiation and dispatch occurs via net/http. * * 4. When rendering the response via ResponseWriter, we match its underlying * connection against what we stored in step (2) to figure out which container * it came from. */ /* * We keep this in a global so that we can reference it from the server and * from our http handlers, since there appears to be no way to pass information * around here. */ var pidMapper = ConnPidMapper{m: map[*net.UnixConn]*unix.Ucred{}} type ConnPidMapper struct { m map[*net.UnixConn]*unix.Ucred mLock sync.Mutex } func (m *ConnPidMapper) ConnStateHandler(conn net.Conn, state http.ConnState) { unixConn := conn.(*net.UnixConn) switch state { case http.StateNew: cred, err := linux.GetUcred(unixConn) if err != nil { logger.Debugf("Error getting ucred for conn %s", err) } else { m.mLock.Lock() m.m[unixConn] = cred m.mLock.Unlock() } case http.StateActive: return case http.StateIdle: return case http.StateHijacked: /* * The "Hijacked" state indicates that the connection has been * taken over from net/http. This is useful for things like * developing websocket libraries, who want to upgrade the * connection to a websocket one, and not use net/http any * more. Whatever the case, we want to forget about it since we * won't see it either. */ m.mLock.Lock() delete(m.m, unixConn) m.mLock.Unlock() case http.StateClosed: m.mLock.Lock() delete(m.m, unixConn) m.mLock.Unlock() default: logger.Debugf("Unknown state for connection %s", state) } } var errPIDNotInContainer = errors.New("pid not in container?") func findContainerForPid(pid int32, s *state.State) (instance.Container, error) { /* * Try and figure out which container a pid is in. There is probably a * better way to do this. Based on rharper's initial performance * metrics, looping over every container and loading them is * expensive, so I wanted to avoid that if possible, so this happens in * a two step process: * * 1. Walk up the process tree until you see something that looks like * an lxc monitor process and extract its name from there. * * 2. If this fails, it may be that someone did an `incus exec foo -- bash`, * so the process isn't actually a descendant of the container's * init. In this case we just look through all the containers until * we find an init with a matching pid namespace. This is probably * uncommon, so hopefully the slowness won't hurt us. */ origpid := pid for pid > 1 { cmdline, err := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid)) if err != nil { return nil, err } status, err := os.ReadFile(fmt.Sprintf("/proc/%d/status", pid)) if err != nil { return nil, err } if strings.HasPrefix(string(cmdline), "[lxc monitor]") && strings.Contains(string(status), fmt.Sprintf("NSpid: %d\n", pid)) { // container names can't have spaces parts := strings.Split(string(cmdline), " ") name := strings.TrimSuffix(parts[len(parts)-1], "\x00") projectName := api.ProjectDefaultName if strings.Contains(name, "_") { fields := strings.SplitN(name, "_", 2) projectName = fields[0] name = fields[1] } inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return nil, err } if inst.Type() != instancetype.Container { return nil, errors.New("Instance is not container type") } return inst.(instance.Container), nil } re, err := regexp.Compile(`^PPid:\s+([0-9]+)$`) if err != nil { return nil, err } for _, line := range strings.Split(string(status), "\n") { m := re.FindStringSubmatch(line) if len(m) > 1 { result, err := strconv.Atoi(m[1]) if err != nil { return nil, err } pid = int32(result) break } } } origPidNs, err := os.Readlink(fmt.Sprintf("/proc/%d/ns/pid", origpid)) if err != nil { return nil, err } instances, err := instance.LoadNodeAll(s, instancetype.Container) if err != nil { return nil, err } for _, inst := range instances { if inst.Type() != instancetype.Container { continue } if !inst.IsRunning() { continue } initpid := inst.InitPID() pidNs, err := os.Readlink(fmt.Sprintf("/proc/%d/ns/pid", initpid)) if err != nil { return nil, err } if origPidNs == pidNs { return inst.(instance.Container), nil } } return nil, errPIDNotInContainer } incus-7.0.0/cmd/incusd/dev_incus_test.go000066400000000000000000000063301517523235500202460ustar00rootroot00000000000000package main import ( "context" "fmt" "io" "net" "net/http" "os" "path/filepath" "strings" "testing" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/sys" ) var testDir string type DevIncusDialer struct { Path string } func (d DevIncusDialer) DevIncusDial(context.Context, string, string) (net.Conn, error) { addr, err := net.ResolveUnixAddr("unix", d.Path) if err != nil { return nil, err } conn, err := net.DialUnix("unix", nil, addr) if err != nil { return nil, err } return conn, err } func setupDir() error { var err error testDir, err = os.MkdirTemp("", "incus_test_devIncus_") if err != nil { return err } err = sys.SetupTestCerts(testDir) if err != nil { return err } err = os.Chmod(testDir, 0o700) if err != nil { return err } _ = os.MkdirAll(fmt.Sprintf("%s/devIncus", testDir), 0o755) return os.Setenv("INCUS_DIR", testDir) } func setupSocket() (*net.UnixListener, error) { _ = setupDir() path := filepath.Join(testDir, "test-devIncus-sock") addr, err := net.ResolveUnixAddr("unix", path) if err != nil { return nil, err } listener, err := net.ListenUnix("unix", addr) if err != nil { return nil, err } return listener, nil } func connect(path string) (*net.UnixConn, error) { addr, err := net.ResolveUnixAddr("unix", path) if err != nil { return nil, err } conn, err := net.DialUnix("unix", nil, addr) if err != nil { return nil, err } return conn, nil } func TestCredsSendRecv(t *testing.T) { result := make(chan int32, 1) listener, err := setupSocket() if err != nil { t.Fatal(err) } defer func() { _ = listener.Close() }() defer func() { _ = os.RemoveAll(testDir) }() go func() { conn, err := listener.AcceptUnix() if err != nil { t.Log(err) result <- -1 return } defer func() { _ = conn.Close() }() cred, err := linux.GetUcred(conn) if err != nil { t.Log(err) result <- -1 return } result <- cred.Pid }() conn, err := connect(fmt.Sprintf("%s/test-devIncus-sock", testDir)) if err != nil { t.Fatal(err) } defer func() { _ = conn.Close() }() pid := <-result if pid != int32(os.Getpid()) { t.Fatal("pid mismatch: ", pid, os.Getpid()) } } /* * Here we're not really testing the API functionality (we can't, since it * expects us to be inside a container to work), but it is useful to test that * all the grotty connection extracting stuff works (that is, it gets to the * point where it realizes the pid isn't in a container without crashing). */ func TestHttpRequest(t *testing.T) { _ = setupDir() defer func() { _ = os.RemoveAll(testDir) }() d := defaultDaemon() d.os.MockMode = true err := d.Init() if err != nil { t.Fatal(err) } defer func() { _ = d.Stop(context.Background(), unix.SIGQUIT) }() c := http.Client{Transport: &http.Transport{DialContext: DevIncusDialer{Path: fmt.Sprintf("%s/guestapi/sock", testDir)}.DevIncusDial}} raw, err := c.Get("http://1.0") if err != nil { t.Fatal(err) } if raw.StatusCode != 500 { t.Fatal(err) } resp, err := io.ReadAll(raw.Body) if err != nil { t.Fatal(err) } if !strings.Contains(string(resp), errPIDNotInContainer.Error()) { t.Fatal("resp error not expected: ", string(resp)) } } incus-7.0.0/cmd/incusd/devices.go000066400000000000000000000455171517523235500166640ustar00rootroot00000000000000package main import ( "fmt" "os" "path" "path/filepath" "slices" "sort" "strconv" "strings" "unsafe" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/cgroup" "github.com/lxc/incus/v7/internal/server/device" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/util" ) type deviceTaskCPU struct { id int64 strId string count *int } type deviceTaskCPUs []deviceTaskCPU func (c deviceTaskCPUs) Len() int { return len(c) } func (c deviceTaskCPUs) Less(i, j int) bool { return *c[i].count < *c[j].count } func (c deviceTaskCPUs) Swap(i, j int) { c[i], c[j] = c[j], c[i] } func deviceNetlinkListener() (chan []string, chan device.USBEvent, chan device.UnixHotplugEvent, error) { NETLINK_KOBJECT_UEVENT := 15 UEVENT_BUFFER_SIZE := 2048 fd, err := unix.Socket( unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, NETLINK_KOBJECT_UEVENT, ) if err != nil { return nil, nil, nil, err } nl := unix.SockaddrNetlink{ Family: unix.AF_NETLINK, Pid: uint32(os.Getpid()), Groups: 3, } err = unix.Bind(fd, &nl) if err != nil { return nil, nil, nil, err } chCPU := make(chan []string, 1) chUSB := make(chan device.USBEvent) chUnix := make(chan device.UnixHotplugEvent) go func(chCPU chan []string, chUSB chan device.USBEvent, chUnix chan device.UnixHotplugEvent) { b := make([]byte, UEVENT_BUFFER_SIZE*2) for { r, err := unix.Read(fd, b) if err != nil { continue } ueventBuf := make([]byte, r) copy(ueventBuf, b) udevEvent := false if strings.HasPrefix(string(ueventBuf), "libudev") { udevEvent = true // Skip the header that libudev prepends ueventBuf = ueventBuf[40 : len(ueventBuf)-1] } ueventLen := 0 ueventParts := strings.Split(string(ueventBuf), "\x00") for i, part := range ueventParts { if strings.HasPrefix(part, "SEQNUM=") { ueventParts = slices.Delete(ueventParts, i, i+1) break } } props := map[string]string{} for _, part := range ueventParts { // libudev string prefix distinguishes udev events from kernel uevents if strings.HasPrefix(part, "libudev") { udevEvent = true continue } ueventLen += len(part) + 1 fields := strings.SplitN(part, "=", 2) if len(fields) != 2 { continue } props[fields[0]] = fields[1] } ueventLen-- if udevEvent { // The kernel always prepends this and udev expects it. kernelPrefix := fmt.Sprintf("%s@%s", props["ACTION"], props["DEVPATH"]) ueventParts = append([]string{kernelPrefix}, ueventParts...) ueventLen += len(kernelPrefix) } if props["SUBSYSTEM"] == "cpu" && !udevEvent { if props["DRIVER"] != "processor" { continue } if props["ACTION"] != "offline" && props["ACTION"] != "online" { continue } // As CPU re-balancing affects all containers, no need to queue them select { case chCPU <- []string{path.Base(props["DEVPATH"]), props["ACTION"]}: default: // Channel is full, drop the event } } if props["SUBSYSTEM"] == "net" && !udevEvent { if props["ACTION"] != "add" && props["ACTION"] != "removed" { continue } if !util.PathExists(fmt.Sprintf("/sys/class/net/%s", props["INTERFACE"])) { continue } } if props["SUBSYSTEM"] == "usb" && !udevEvent { parts := strings.Split(props["PRODUCT"], "/") if len(parts) < 2 { continue } major, ok := props["MAJOR"] if !ok { continue } minor, ok := props["MINOR"] if !ok { continue } devname, ok := props["DEVNAME"] if !ok { continue } busnum, ok := props["BUSNUM"] if !ok { continue } devnum, ok := props["DEVNUM"] if !ok { continue } zeroPad := func(s string, l int) string { return strings.Repeat("0", l-len(s)) + s } usb, err := device.USBNewEvent( props["ACTION"], /* udev doesn't zero pad these, while * everything else does, so let's zero pad them * for consistency */ zeroPad(parts[0], 4), zeroPad(parts[1], 4), props["SERIAL"], major, minor, busnum, devnum, devname, ueventParts[:len(ueventParts)-1], ueventLen, ) if err != nil { logger.Error("Error reading usb device", logger.Ctx{"err": err, "path": props["PHYSDEVPATH"]}) continue } chUSB <- usb } // unix hotplug device events rely on information added by udev if udevEvent { action := props["ACTION"] if action != "add" && action != "remove" { continue } subsystem, ok := props["SUBSYSTEM"] if !ok { continue } devname, ok := props["DEVNAME"] if !ok { continue } major, ok := props["MAJOR"] if !ok { continue } minor, ok := props["MINOR"] if !ok { continue } var pci string if strings.HasPrefix(props["DEVPATH"], "/devices/pci") { pci = filepath.Base(strings.Split(props["DEVPATH"], "/usb")[0]) } vendor := "" product := "" if action == "add" { vendor, product, ok = ueventParseVendorProduct(props, subsystem, devname) if !ok { continue } } zeroPad := func(s string, l int) string { return strings.Repeat("0", l-len(s)) + s } // zeropad if len(vendor) < 4 { vendor = zeroPad(vendor, 4) } if len(product) < 4 { product = zeroPad(product, 4) } unix, err := device.UnixHotplugNewEvent( action, /* udev doesn't zero pad these, while * everything else does, so let's zero pad them * for consistency */ vendor, product, pci, major, minor, subsystem, devname, ueventParts[:len(ueventParts)-1], ueventLen, ) if err != nil { logger.Error("Error reading unix device", logger.Ctx{"err": err, "path": props["PHYSDEVPATH"]}) continue } chUnix <- unix } } }(chCPU, chUSB, chUnix) return chCPU, chUSB, chUnix, nil } /* * fillFixedInstances fills the `fixedInstances` map with the instances that have been pinned to specific CPUs. * The `fixedInstances` map is a map of CPU IDs to a list of instances that have been pinned to that CPU. * The `targetCpuPool` is a list of CPU IDs that are available for pinning. * The `targetCpuNum` is the number of CPUs that are required for pinning. * The `loadBalancing` flag indicates whether the CPU pinning should be load balanced or not (e.g, NUMA placement when `limits.cpu` is a single number which means * a required number of vCPUs per instance that can be chosen within a CPU pool). */ func fillFixedInstances(fixedInstances map[int64][]instance.Instance, inst instance.Instance, effectiveCpus []int64, targetCpuPool []int64, targetCpuNum int, loadBalancing bool) { if len(targetCpuPool) < targetCpuNum { diffCount := len(targetCpuPool) - targetCpuNum logger.Warnf("%v CPUs have been required for pinning, but %v CPUs won't be allocated", len(targetCpuPool), -diffCount) targetCpuNum = len(targetCpuPool) } // If the `targetCpuPool` has been manually specified (explicit CPU IDs/ranges specified with `limits.cpu`) if len(targetCpuPool) == targetCpuNum && !loadBalancing { for _, nr := range targetCpuPool { if !slices.Contains(effectiveCpus, nr) { continue } _, ok := fixedInstances[nr] if ok { fixedInstances[nr] = append(fixedInstances[nr], inst) } else { fixedInstances[nr] = []instance.Instance{inst} } } return } // If we need to load-balance the instance across the CPUs of `targetCpuPool` (e.g, NUMA placement), // the heuristic is to sort the `targetCpuPool` by usage (number of instances already pinned to each CPU) // and then assign the instance to the first `desiredCpuNum` least used CPUs. usage := map[int64]deviceTaskCPU{} for _, id := range targetCpuPool { cpu := deviceTaskCPU{} cpu.id = id cpu.strId = fmt.Sprintf("%d", id) count := 0 _, ok := fixedInstances[id] if ok { count = len(fixedInstances[id]) } cpu.count = &count usage[id] = cpu } sortedUsage := make(deviceTaskCPUs, 0) for _, value := range usage { sortedUsage = append(sortedUsage, value) } sort.Sort(sortedUsage) count := 0 for _, cpu := range sortedUsage { if count == targetCpuNum { break } id := cpu.id _, ok := fixedInstances[id] if ok { fixedInstances[id] = append(fixedInstances[id], inst) } else { fixedInstances[id] = []instance.Instance{inst} } count++ } } // deviceTaskBalance is used to balance the CPU load across containers running on a host. // It first checks if CGroup support is available and returns if it isn't. // It then retrieves the effective CPU list (the CPUs that are guaranteed to be online) and isolates any isolated CPUs. // After that, it loads all instances of containers running on the node and iterates through them. // // For each container, it checks its CPU limits and determines whether it is pinned to specific CPUs or can use the load-balancing mechanism. // If it is pinned, the function adds it to the fixedInstances map with the CPU numbers it is pinned to. // If not, the container will be included in the load-balancing calculation, // and the number of CPUs it can use is determined by taking the minimum of its assigned CPUs and the available CPUs. Note that if // NUMA placement is enabled (`limits.cpu.nodes` is not empty), we apply a similar load-balancing logic to the `fixedInstances` map // with a constraint being the number of vCPUs and the CPU pool being the CPUs pinned to a set of NUMA nodes. // // Next, the function balance the CPU usage by iterating over all the CPUs and dividing the containers into those that // are pinned to a specific CPU and those that are load-balanced. For the pinned containers, // it adds them to the pinning map with the CPU number it's pinned to. // For the load-balanced containers, it sorts the available CPUs based on their usage count and assigns them to containers // in ascending order until the required number of CPUs have been assigned. // Finally, the pinning map is used to set the new CPU pinning for each container, updating it to the new balanced state. // // Overall, this function ensures that the CPU resources of the host are utilized effectively amongst all the containers running on it. func deviceTaskBalance(s *state.State) { minFunc := func(x, y int) int { if x < y { return x } return y } // Don't bother running when CGroup support isn't there if !cgroup.Supports(cgroup.CPUSet) { return } // Get effective cpus list - those are all guaranteed to be online cg, err := cgroup.NewFileReadWriter(1) if err != nil { logger.Errorf("Unable to load cgroup writer: %v", err) return } effectiveCpus, err := cg.GetEffectiveCpuset() if err != nil { // Older kernel - use cpuset.cpus effectiveCpus, err = cg.GetCpuset() if err != nil { logger.Errorf("Error reading host's cpuset.cpus") return } } effectiveCpusInt, err := resources.ParseCpuset(effectiveCpus) if err != nil { logger.Errorf("Error parsing effective CPU set") return } isolatedCpusInt := resources.GetCPUIsolated() effectiveCpusSlice := []string{} for _, id := range effectiveCpusInt { if slices.Contains(isolatedCpusInt, id) { continue } effectiveCpusSlice = append(effectiveCpusSlice, fmt.Sprintf("%d", id)) } effectiveCpus = strings.Join(effectiveCpusSlice, ",") cpus, err := resources.ParseCpuset(effectiveCpus) if err != nil { logger.Error("Error parsing host's cpu set", logger.Ctx{"cpuset": effectiveCpus, "err": err}) return } // Iterate through the instances instances, err := instance.LoadNodeAll(s, instancetype.Container) if err != nil { logger.Error("Problem loading instances list", logger.Ctx{"err": err}) return } // Get CPU topology. cpusTopology, err := resources.GetCPU() if err != nil { logger.Errorf("Unable to load system CPUs information: %v", err) return } // Build a map of NUMA node to CPU threads. numaNodeToCPU := make(map[int64][]int64) for _, cpu := range cpusTopology.Sockets { for _, core := range cpu.Cores { for _, thread := range core.Threads { // Skip any isolated CPU thread. if slices.Contains(isolatedCpusInt, thread.ID) { continue } numaNodeToCPU[int64(thread.NUMANode)] = append(numaNodeToCPU[int64(thread.NUMANode)], thread.ID) } } } fixedInstances := map[int64][]instance.Instance{} balancedInstances := map[instance.Instance]int{} for _, c := range instances { var numaCpus []int64 var numaCpusStr []string conf := c.ExpandedConfig() cpuNodes := conf["limits.cpu.nodes"] if cpuNodes != "" { if cpuNodes == "balanced" { cpuNodes = conf["volatile.cpu.nodes"] } numaNodeSet, err := resources.ParseNumaNodeSet(cpuNodes) if err != nil { logger.Error("Error parsing numa node set", logger.Ctx{"numaNodes": cpuNodes, "err": err}) continue } for _, numaNode := range numaNodeSet { numaCpus = append(numaCpus, numaNodeToCPU[numaNode]...) } for _, numaCPU := range numaCpus { numaCpusStr = append(numaCpusStr, fmt.Sprintf("%d", numaCPU)) } } cpulimit, ok := conf["limits.cpu"] if !ok || cpulimit == "" { // If restricted to specific NUMA node(s), only use their CPU threads. if cpuNodes != "" { cpulimit = strings.Join(numaCpusStr, ",") } else { cpulimit = effectiveCpus } } // Check that the container is running. // We use InitPID here rather than IsRunning because this task is triggered during the container's // onStart hook, which is during the time that the start lock is held, which causes IsRunning to // return false (because the container hasn't fully started yet) but it is sufficiently started to // have its cgroup CPU limits set. if c.InitPID() <= 0 { continue } count, err := strconv.Atoi(cpulimit) if err == nil { // Load-balance count = minFunc(count, len(cpus)) if len(numaCpus) > 0 { fillFixedInstances(fixedInstances, c, cpus, numaCpus, count, true) } else { balancedInstances[c] = count } } else { // Pinned containerCpus, err := resources.ParseCpuset(cpulimit) if err != nil { return } if conf["limits.cpu"] != "" && len(numaCpus) > 0 { logger.Warnf("The pinned CPUs: %v, override the NUMA configuration with the CPUs: %v", containerCpus, numaCpus) } fillFixedInstances(fixedInstances, c, cpus, containerCpus, len(containerCpus), false) } } // Balance things pinning := map[instance.Instance][]string{} usage := map[int64]deviceTaskCPU{} for _, id := range cpus { cpu := deviceTaskCPU{} cpu.id = id cpu.strId = fmt.Sprintf("%d", id) count := 0 cpu.count = &count usage[id] = cpu } for cpu, ctns := range fixedInstances { c, ok := usage[cpu] if !ok { logger.Errorf("Internal error: container using unavailable cpu") continue } id := c.strId for _, ctn := range ctns { _, ok := pinning[ctn] if ok { pinning[ctn] = append(pinning[ctn], id) } else { pinning[ctn] = []string{id} } *c.count += 1 } } sortedUsage := make(deviceTaskCPUs, 0) for _, value := range usage { sortedUsage = append(sortedUsage, value) } for ctn, count := range balancedInstances { sort.Sort(sortedUsage) for _, cpu := range sortedUsage { if count == 0 { break } count -= 1 id := cpu.strId _, ok := pinning[ctn] if ok { pinning[ctn] = append(pinning[ctn], id) } else { pinning[ctn] = []string{id} } *cpu.count += 1 } } // Set the new pinning for ctn, set := range pinning { // Confirm the container didn't just stop if ctn.InitPID() <= 0 { continue } sort.Strings(set) cg, err := ctn.CGroup() if err != nil { logger.Error("balance: Unable to get cgroup struct", logger.Ctx{"name": ctn.Name(), "err": err, "value": strings.Join(set, ",")}) continue } err = cg.SetCpuset(strings.Join(set, ",")) if err != nil { logger.Error("balance: Unable to set cpuset", logger.Ctx{"name": ctn.Name(), "err": err, "value": strings.Join(set, ",")}) } } } // deviceEventListener starts the event listener for resource scheduling. // Accepts stateFunc which will be called each time it needs a fresh state.State. func deviceEventListener(stateFunc func() *state.State) { chNetlinkCPU, chUSB, chUnix, err := deviceNetlinkListener() if err != nil { logger.Errorf("scheduler: Couldn't setup netlink listener: %v", err) return } for { select { case e := <-chNetlinkCPU: if len(e) != 2 { logger.Errorf("Scheduler: received an invalid cpu hotplug event") continue } s := stateFunc() if !cgroup.Supports(cgroup.CPUSet) { continue } logger.Debugf("Scheduler: cpu: %s is now %s: re-balancing", e[0], e[1]) deviceTaskBalance(s) case e := <-chUSB: device.USBRunHandlers(stateFunc(), &e) case e := <-chUnix: device.UnixHotplugRunHandlers(stateFunc(), &e) case e := <-cgroup.DeviceSchedRebalance: if len(e) != 3 { logger.Errorf("Scheduler: received an invalid rebalance event") continue } s := stateFunc() if !cgroup.Supports(cgroup.CPUSet) { continue } logger.Debugf("Scheduler: %s %s %s: re-balancing", e[0], e[1], e[2]) deviceTaskBalance(s) } } } // devicesRegister calls the Register() function on all supported devices so they receive events. // This also has the effect of actively reconnecting to any running VM monitor sockets. func devicesRegister(instances []instance.Instance) { logger.Debug("Registering running instances") for _, inst := range instances { if !inst.IsRunning() { // For VMs this will also trigger a connection to the QMP socket if running. continue } inst.RegisterDevices() } } func getHidrawDevInfo(fd int) (string, string, error) { type hidInfo struct { busType uint32 vendor int16 product int16 } var info hidInfo _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), linux.IoctlHIDIOCGrawInfo, uintptr(unsafe.Pointer(&info))) if errno != 0 { return "", "", fmt.Errorf("Failed setting received UUID: %w", unix.Errno(errno)) } return fmt.Sprintf("%04x", info.vendor), fmt.Sprintf("%04x", info.product), nil } func ueventParseVendorProduct(props map[string]string, subsystem string, devname string) (string, string, bool) { vendor, vendorOk := props["ID_VENDOR_ID"] product, productOk := props["ID_MODEL_ID"] if vendorOk && productOk { return vendor, product, true } if subsystem != "hidraw" { return "", "", false } if !filepath.IsAbs(devname) { return "", "", false } file, err := os.OpenFile(devname, os.O_RDWR, 0o000) if err != nil { return "", "", false } defer func() { _ = file.Close() }() vendor, product, err = getHidrawDevInfo(int(file.Fd())) if err != nil { logger.Debugf("Failed to retrieve device info from hidraw device \"%s\"", devname) return "", "", false } return vendor, product, true } incus-7.0.0/cmd/incusd/events.go000066400000000000000000000143571517523235500165440ustar00rootroot00000000000000package main import ( "context" "fmt" "net/http" "slices" "strings" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/events" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/ws" ) var ( eventTypes = []string{api.EventTypeLogging, api.EventTypeOperation, api.EventTypeLifecycle, api.EventTypeNetworkACL} privilegedEventTypes = []string{api.EventTypeLogging} ) var eventsCmd = APIEndpoint{ Path: "events", Get: APIEndpointAction{Handler: eventsGet, AccessHandler: allowAuthenticated}, } type eventsServe struct { req *http.Request s *state.State } func (r *eventsServe) Render(w http.ResponseWriter) error { return eventsSocket(r.s, r.req, w) } func (r *eventsServe) String() string { return "event handler" } // Code returns the HTTP code. func (r *eventsServe) Code() int { return http.StatusOK } func eventsSocket(s *state.State, r *http.Request, w http.ResponseWriter) error { // Detect project mode. projectName := request.QueryParam(r, "project") allProjects := util.IsTrue(request.QueryParam(r, "all-projects")) if allProjects && projectName != "" { return api.StatusErrorf(http.StatusBadRequest, "Cannot specify a project when requesting all projects") } else if !allProjects && projectName == "" { projectName = api.ProjectDefaultName } if !allProjects && projectName != api.ProjectDefaultName { _, err := s.DB.GetProject(context.Background(), projectName) if err != nil { return err } } var projectPermissionFunc auth.PermissionChecker if projectName != "" { err := s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectProject(projectName), auth.EntitlementCanViewEvents) if err != nil { return err } } else if allProjects { var err error projectPermissionFunc, err = s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanViewEvents, auth.ObjectTypeProject) if err != nil { return err } } canViewPrivilegedEvents := s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectServer(), auth.EntitlementCanViewPrivilegedEvents) == nil types := strings.Split(r.FormValue("type"), ",") if len(types) == 1 && types[0] == "" { types = []string{} for _, entry := range eventTypes { if !canViewPrivilegedEvents && slices.Contains(privilegedEventTypes, entry) { continue } types = append(types, entry) } } // Validate event types. for _, entry := range types { if !slices.Contains(eventTypes, entry) { return api.StatusErrorf(http.StatusBadRequest, "%q isn't a supported event type", entry) } } if slices.Contains(types, api.EventTypeLogging) && !canViewPrivilegedEvents { return api.StatusErrorf(http.StatusForbidden, "Forbidden") } l := logger.AddContext(logger.Ctx{"remote": r.RemoteAddr}) var excludeLocations []string if isClusterNotification(r) { ctx := r.Context() // Try and match cluster member certificate fingerprint to member name. fingerprint, found := ctx.Value(request.CtxUsername).(string) if found { err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { cert, err := cluster.GetCertificateByFingerprintPrefix(context.Background(), tx.Tx(), fingerprint) if err != nil { return fmt.Errorf("Failed matching client certificate to cluster member: %w", err) } // Add the cluster member client's name to the excluded locations so that we can avoid // looping the event back to them when they send us an event via recvFunc. excludeLocations = append(excludeLocations, cert.Name) return nil }) if err != nil { l.Warn("Failed setting up event connection", logger.Ctx{"err": err}) return nil } } } var recvFunc events.EventHandler var excludeSources []events.EventSource if isClusterNotification(r) { // If client is another cluster member, it will already be pulling events from other cluster // members so no need to also deliver forwarded events that this member receives. excludeSources = append(excludeSources, events.EventSourcePull) recvFunc = func(event api.Event) { // Inject event received via push from event listener client so its forwarded to // other event hub members (if operating in event hub mode). s.Events.Inject(event, events.EventSourcePush) } } // Upgrade the connection to websocket as late as possible. // This is because the client will assume it's getting events as soon as the upgrade is performed. conn, err := ws.Upgrader.Upgrade(w, r, nil) if err != nil { l.Warn("Failed upgrading event connection", logger.Ctx{"err": err}) return nil } defer func() { _ = conn.Close() }() // Ensure listener below ends when this function ends. listenerConnection := events.NewWebsocketListenerConnection(conn) listener, err := s.Events.AddListener(projectName, allProjects, projectPermissionFunc, listenerConnection, types, excludeSources, recvFunc, excludeLocations) if err != nil { l.Warn("Failed to add event listener", logger.Ctx{"err": err}) return nil } listener.Wait(r.Context()) return nil } // swagger:operation GET /1.0/events server events_get // // Get the event stream // // Connects to the event API using websocket. // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: type // description: Event type(s), comma separated (valid types are logging, operation or lifecycle) // type: string // example: logging,lifecycle // - in: query // name: all-projects // description: Retrieve instances from all projects // type: boolean // responses: // "200": // description: Websocket message (JSON) // schema: // $ref: "#/definitions/Event" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func eventsGet(d *Daemon, r *http.Request) response.Response { return &eventsServe{req: r, s: d.State()} } incus-7.0.0/cmd/incusd/images.go000066400000000000000000004176641517523235500165150ustar00rootroot00000000000000package main import ( "archive/tar" "bytes" "context" "crypto/sha256" "encoding/json" "errors" "fmt" "io" "io/fs" "maps" "math" "math/rand" "mime" "mime/multipart" "net/http" "net/url" "os" "os/exec" "path/filepath" "slices" "strconv" "strings" "sync" "time" "github.com/gorilla/mux" "github.com/kballard/go-shellquote" "go.yaml.in/yaml/v4" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/filter" internalInstance "github.com/lxc/incus/v7/internal/instance" internalIO "github.com/lxc/incus/v7/internal/io" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/internal/server/operations" projectutils "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" "github.com/lxc/incus/v7/internal/server/task" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var imagesCmd = APIEndpoint{ Path: "images", Get: APIEndpointAction{Handler: imagesGet, AllowUntrusted: true}, Post: APIEndpointAction{Handler: imagesPost, AllowUntrusted: true, LargeRequest: true}, } var imageCmd = APIEndpoint{ Path: "images/{fingerprint}", Delete: APIEndpointAction{Handler: imageDelete, AccessHandler: allowPermission(auth.ObjectTypeImage, auth.EntitlementCanEdit, "fingerprint")}, Get: APIEndpointAction{Handler: imageGet, AllowUntrusted: true}, Patch: APIEndpointAction{Handler: imagePatch, AccessHandler: allowPermission(auth.ObjectTypeImage, auth.EntitlementCanEdit, "fingerprint")}, Put: APIEndpointAction{Handler: imagePut, AccessHandler: allowPermission(auth.ObjectTypeImage, auth.EntitlementCanEdit, "fingerprint")}, } var imageExportCmd = APIEndpoint{ Path: "images/{fingerprint}/export", Get: APIEndpointAction{Handler: imageExport, AllowUntrusted: true}, Post: APIEndpointAction{Handler: imageExportPost, AccessHandler: allowPermission(auth.ObjectTypeImage, auth.EntitlementCanEdit, "fingerprint")}, } var imageSecretCmd = APIEndpoint{ Path: "images/{fingerprint}/secret", Post: APIEndpointAction{Handler: imageSecret, AccessHandler: allowPermission(auth.ObjectTypeImage, auth.EntitlementCanEdit, "fingerprint")}, } var imageRefreshCmd = APIEndpoint{ Path: "images/{fingerprint}/refresh", Post: APIEndpointAction{Handler: imageRefresh, AccessHandler: allowPermission(auth.ObjectTypeImage, auth.EntitlementCanEdit, "fingerprint")}, } var imageAliasesCmd = APIEndpoint{ Path: "images/aliases", Get: APIEndpointAction{Handler: imageAliasesGet, AccessHandler: allowAuthenticated}, Post: APIEndpointAction{Handler: imageAliasesPost, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanCreateImageAliases)}, } var imageAliasCmd = APIEndpoint{ Path: "images/aliases/{name:.*}", Delete: APIEndpointAction{Handler: imageAliasDelete, AccessHandler: allowPermission(auth.ObjectTypeImageAlias, auth.EntitlementCanEdit, "name")}, Get: APIEndpointAction{Handler: imageAliasGet, AllowUntrusted: true}, Patch: APIEndpointAction{Handler: imageAliasPatch, AccessHandler: allowPermission(auth.ObjectTypeImageAlias, auth.EntitlementCanEdit, "name")}, Post: APIEndpointAction{Handler: imageAliasPost, AccessHandler: allowPermission(auth.ObjectTypeImageAlias, auth.EntitlementCanEdit, "name")}, Put: APIEndpointAction{Handler: imageAliasPut, AccessHandler: allowPermission(auth.ObjectTypeImageAlias, auth.EntitlementCanEdit, "name")}, } /* We only want a single publish running at any one time. The CPU and I/O load of publish is such that running multiple ones in parallel takes longer than running them serially. Additionally, publishing the same container or container snapshot twice would lead to storage problem, not to mention a conflict at the end for whichever finishes last. */ var imagePublishLock sync.Mutex // imageTaskMu prevents image related tasks from being scheduled at the same time as each other to prevent them // stepping on each other's toes. var imageTaskMu sync.Mutex func compressFile(compress string, infile io.Reader, outfile io.Writer) error { reproducible := []string{"gzip"} var cmd *exec.Cmd // Parse the command. fields, err := shellquote.Split(compress) if err != nil { return err } if fields[0] == "squashfs" { // 'tar2sqfs' do not support writing to stdout. So write to a temporary // file first and then replay the compressed content to outfile. tempfile, err := os.CreateTemp("", "incus_compress_") if err != nil { return err } defer func() { _ = tempfile.Close() }() defer func() { _ = os.Remove(tempfile.Name()) }() // Prepare 'tar2sqfs' arguments args := []string{"tar2sqfs"} if len(fields) > 1 { args = append(args, fields[1:]...) } args = append(args, "--no-skip", "--force", "--compressor", "xz", tempfile.Name()) cmd = exec.Command(args[0], args[1:]...) cmd.Stdin = infile output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("tar2sqfs: %v (%v)", err, strings.TrimSpace(string(output))) } // Replay the result to outfile _, err = tempfile.Seek(0, io.SeekStart) if err != nil { return err } _, err = util.SafeCopy(outfile, tempfile) if err != nil { return err } } else { args := []string{"-c"} if len(fields) > 1 { args = append(args, fields[1:]...) } if slices.Contains(reproducible, fields[0]) { args = append(args, "-n") } cmd := exec.Command(fields[0], args...) cmd.Stdin = infile cmd.Stdout = outfile err := cmd.Run() if err != nil { return err } } return nil } /* * This function takes a container or snapshot from the local image server and * exports it as an image. */ func imgPostInstanceInfo(ctx context.Context, s *state.State, r *http.Request, req api.ImagesPost, op *operations.Operation, builddir string, budget int64) (*api.Image, error) { info := api.Image{} info.Properties = map[string]string{} projectName := request.ProjectParam(r) name := req.Source.Name ctype := req.Source.Type imageType := req.Format if ctype == "" || name == "" { return nil, errors.New("No source provided") } if imageType != "" && imageType != "unified" && imageType != "split" { return nil, errors.New("Invalid image format") } switch ctype { case "snapshot": if !internalInstance.IsSnapshot(name) { return nil, errors.New("Not a snapshot") } case "container", "virtual-machine", "instance": if internalInstance.IsSnapshot(name) { return nil, errors.New("This is a snapshot") } default: return nil, errors.New("Bad type") } info.Filename = req.Filename switch req.Public { case true: info.Public = true case false: info.Public = false } c, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return nil, err } info.Type = c.Type().String() // Build the actual image file metaFile, err := os.CreateTemp(builddir, "incus_build_image_") if err != nil { return nil, err } rootfsFile, err := os.CreateTemp(builddir, "incus_build_image_") if err != nil { return nil, err } defer func() { _ = os.Remove(metaFile.Name()) }() defer func() { _ = os.Remove(rootfsFile.Name()) }() // Calculate (close estimate of) total size of input to image totalSize := int64(0) sumSize := func(path string, fi os.FileInfo, err error) error { if err == nil { totalSize += fi.Size() } return nil } err = filepath.Walk(c.RootfsPath(), sumSize) if err != nil { return nil, err } // Track progress creating image. metaProgressWriter := &ioprogress.ProgressWriter{ Tracker: &ioprogress.ProgressTracker{ Handler: func(value, speed int64) { percent := int64(0) var processed int64 if totalSize > 0 { percent = value processed = totalSize * (percent / 100.0) } else { processed = value } metadata := make(map[string]any) operations.SetProgressMetadata(metadata, "create_image_from_container_pack", "Image pack", percent, processed, speed) _ = op.UpdateMetadata(metadata) }, Length: totalSize, }, } rootfsProgressWriter := &ioprogress.ProgressWriter{ Tracker: &ioprogress.ProgressTracker{ Handler: func(value, speed int64) { percent := int64(0) var processed int64 if totalSize > 0 { percent = value processed = totalSize * (percent / 100.0) } else { processed = value } metadata := make(map[string]any) operations.SetProgressMetadata(metadata, "create_image_from_container_pack", "Image pack", percent, processed, speed) _ = op.UpdateMetadata(metadata) }, Length: totalSize, }, } hash256 := sha256.New() var compress string var metaWriter io.Writer var rootfsWriter io.Writer if req.CompressionAlgorithm != "" { err := validate.IsCompressionAlgorithm(req.CompressionAlgorithm) if err != nil { return nil, err } compress = req.CompressionAlgorithm } else { var p *api.Project err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { project, err := dbCluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return err } p, err = project.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return nil, err } if p.Config["images.compression_algorithm"] != "" { compress = p.Config["images.compression_algorithm"] } else { compress = s.GlobalConfig.ImagesCompressionAlgorithm() } } // Setup tar, optional compress and sha256 to happen in one pass. wg := sync.WaitGroup{} var compressErr error if compress != "none" { wg.Add(1) tarReader, tarWriter := io.Pipe() metaProgressWriter.WriteCloser = tarWriter metaWriter = metaProgressWriter var compressWriter io.Writer if imageType != "split" { compressWriter = io.MultiWriter(metaFile, hash256) } else { compressWriter = io.MultiWriter(metaFile) } go func() { defer wg.Done() compressErr = compressFile(compress, tarReader, compressWriter) // If a compression error occurred, close the writer to end the instance export. if compressErr != nil { _ = metaProgressWriter.Close() } }() } else { metaProgressWriter.WriteCloser = metaFile if imageType != "split" { metaWriter = io.MultiWriter(metaProgressWriter, hash256) } else { metaWriter = io.MultiWriter(metaProgressWriter) } } if compress != "none" && c.Info().Type.String() != "virtual-machine" { wg.Add(1) tarReader, tarWriter := io.Pipe() rootfsProgressWriter.WriteCloser = tarWriter rootfsWriter = rootfsProgressWriter compressWriter := io.MultiWriter(rootfsFile) go func() { defer wg.Done() compressErr = compressFile(compress, tarReader, compressWriter) // If a compression error occurred, close the writer to end the instance export. if compressErr != nil { _ = rootfsProgressWriter.Close() } }() } else { rootfsProgressWriter.WriteCloser = rootfsFile rootfsWriter = io.MultiWriter(rootfsProgressWriter) } // Tracker instance for the export phase. tracker := &ioprogress.ProgressTracker{ Handler: func(value, speed int64) { metadata := make(map[string]any) operations.SetProgressMetadata(metadata, "create_image_from_container_pack", "Exporting", value, 0, 0) _ = op.UpdateMetadata(metadata) }, } // Export instance to writer. var meta *api.ImageMetadata metaWriter = internalIO.NewQuotaWriter(metaWriter, budget) rootfsWriter = internalIO.NewQuotaWriter(rootfsWriter, budget) if imageType != "split" { meta, err = c.Export(metaWriter, nil, req.Properties, req.ExpiresAt, tracker) } else { meta, err = c.Export(metaWriter, rootfsWriter, req.Properties, req.ExpiresAt, tracker) } // Clean up file handles. // When compression is used, Close on imageProgressWriter/tarWriter is required for compressFile/gzip to // know it is finished. Otherwise it is equivalent to imageFile.Close. _ = metaProgressWriter.Close() _ = rootfsProgressWriter.Close() wg.Wait() // Wait until compression helper has finished if used. _ = metaFile.Close() _ = rootfsFile.Close() // Check compression errors. if compressErr != nil { return nil, compressErr } // Check instance export errors. if err != nil { return nil, err } // Get ExpiresAt if meta.ExpiryDate != 0 { info.ExpiresAt = time.Unix(meta.ExpiryDate, 0) } fi, err := os.Stat(metaFile.Name()) if err != nil { return nil, err } info.Size = fi.Size() // Make sure both files are included for size and hash when using split format if imageType == "split" { rootfsFi, err := os.Stat(rootfsFile.Name()) if err != nil { return nil, err } info.Size += rootfsFi.Size() metaData, err := os.ReadFile(metaFile.Name()) if err != nil { return nil, err } hash256.Write(metaData) rootfsData, err := os.ReadFile(rootfsFile.Name()) if err != nil { return nil, err } hash256.Write(rootfsData) } info.Fingerprint = fmt.Sprintf("%x", hash256.Sum(nil)) info.CreatedAt = time.Now().UTC() err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { _, _, err = tx.GetImage(ctx, info.Fingerprint, dbCluster.ImageFilter{Project: &projectName}) return err }) if !response.IsNotFoundError(err) { if err != nil { return nil, err } return &info, fmt.Errorf("The image already exists: %s", info.Fingerprint) } /* rename the file to the expected name so our caller can use it */ metaFinalName := internalUtil.VarPath("images", info.Fingerprint) err = internalUtil.FileMove(metaFile.Name(), metaFinalName) if err != nil { return nil, err } if imageType == "split" { rootfsFinalName := internalUtil.VarPath("images", info.Fingerprint+".rootfs") err = internalUtil.FileMove(rootfsFile.Name(), rootfsFinalName) if err != nil { return nil, err } } info.Architecture, _ = osarch.ArchitectureName(c.Architecture()) info.Properties = meta.Properties err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Create the database entry return tx.CreateImage(ctx, c.Project().Name, info.Fingerprint, info.Filename, info.Size, info.Public, info.AutoUpdate, info.Architecture, info.CreatedAt, info.ExpiresAt, info.Properties, info.Type, nil) }) if err != nil { return nil, err } return &info, nil } func imgPostRemoteInfo(ctx context.Context, s *state.State, r *http.Request, req api.ImagesPost, op *operations.Operation, project string, budget int64) (*api.Image, error) { var err error var hash string if req.Source.Fingerprint != "" { hash = req.Source.Fingerprint } else if req.Source.Alias != "" { hash = req.Source.Alias } else { return nil, errors.New("must specify one of alias or fingerprint for init from image") } info, _, err := imageDownload(ctx, r, s, op, &imageDownloadArgs{ Server: req.Source.Server, Protocol: req.Source.Protocol, Certificate: req.Source.Certificate, Secret: req.Source.Secret, Alias: hash, Type: req.Source.ImageType, AutoUpdate: req.AutoUpdate, Public: req.Public, ProjectName: project, Budget: budget, SourceProjectName: req.Source.Project, }) if err != nil { return nil, err } // If just dealing with an internal copy, we're done here. if isClusterNotification(r) && req.Source.Server == "" { return info, nil } err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var id int id, info, err = tx.GetImage(ctx, info.Fingerprint, dbCluster.ImageFilter{Project: &project}) if err != nil { return err } // Allow overriding or adding properties maps.Copy(info.Properties, req.Properties) // Get profile IDs if req.Profiles == nil { req.Profiles = []string{api.ProjectDefaultName} } profileIds := make([]int64, len(req.Profiles)) for i, profile := range req.Profiles { profileID, _, err := tx.GetProfile(ctx, project, profile) if response.IsNotFoundError(err) { return fmt.Errorf("Profile '%s' doesn't exist", profile) } else if err != nil { return err } profileIds[i] = profileID } // Update the DB record if needed if req.Public || req.AutoUpdate || req.Filename != "" || len(req.Properties) > 0 || len(req.Profiles) > 0 { err := tx.UpdateImage(ctx, id, req.Filename, info.Size, req.Public, req.AutoUpdate, info.Architecture, info.CreatedAt, info.ExpiresAt, info.Properties, project, profileIds) if err != nil { return err } } return nil }) if err != nil { return nil, err } return info, nil } func imgPostURLInfo(ctx context.Context, s *state.State, r *http.Request, req api.ImagesPost, op *operations.Operation, project string, budget int64) (*api.Image, error) { var err error // Check the request. if req.Source.URL == "" { return nil, errors.New("Missing URL") } // Validate that the initial image target is allowed. // The imageDownload function will validate the ultimate file download location later. err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { err := projectutils.AllowImageDownload(tx, project, req.Source.URL) if err != nil { return err } return nil }) if err != nil { return nil, err } // Get the image download headers from the provided URL. // Provide some information about the current server to the target. myhttp, err := localUtil.HTTPClient("", s.Proxy) if err != nil { return nil, err } head, err := http.NewRequest("HEAD", req.Source.URL, nil) if err != nil { return nil, err } architectures := []string{} for _, architecture := range s.OS.Architectures { architectureName, err := osarch.ArchitectureName(architecture) if err != nil { return nil, err } architectures = append(architectures, architectureName) } head.Header.Set("User-Agent", version.UserAgent) head.Header.Set("Incus-Server-Architectures", strings.Join(architectures, ", ")) head.Header.Set("Incus-Server-Version", version.Version) raw, err := myhttp.Do(head) if err != nil { return nil, err } // Get the image fingerprint and download URL. hash := raw.Header.Get("Incus-Image-Hash") if hash == "" { return nil, errors.New("Missing Incus-Image-Hash header") } url := raw.Header.Get("Incus-Image-URL") if url == "" { return nil, errors.New("Missing Incus-Image-URL header") } // Download the image itself. info, _, err := imageDownload(ctx, r, s, op, &imageDownloadArgs{ Server: url, Protocol: "direct", Alias: hash, AutoUpdate: req.AutoUpdate, Public: req.Public, ProjectName: project, Budget: budget, }) if err != nil { return nil, err } // Apply user provided attributes and overrides. err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var id int id, info, err = tx.GetImage(ctx, info.Fingerprint, dbCluster.ImageFilter{Project: &project}) if err != nil { return err } // Allow overriding or adding properties maps.Copy(info.Properties, req.Properties) if req.Public || req.AutoUpdate || req.Filename != "" || len(req.Properties) > 0 { err := tx.UpdateImage(ctx, id, req.Filename, info.Size, req.Public, req.AutoUpdate, info.Architecture, info.CreatedAt, info.ExpiresAt, info.Properties, "", nil) if err != nil { return err } } return nil }) if err != nil { return nil, err } return info, nil } func getImgPostInfo(ctx context.Context, s *state.State, r *http.Request, builddir string, project string, post *os.File, metadata map[string]any) (*api.Image, error) { info := api.Image{} var imageMeta *api.ImageMetadata l := logger.AddContext(logger.Ctx{"function": "getImgPostInfo"}) info.Public = util.IsTrue(r.Header.Get("X-Incus-public")) propHeaders := r.Header[http.CanonicalHeaderKey("X-Incus-properties")] profilesHeaders := r.Header.Get("X-Incus-profiles") aliasesHeaders := r.Header.Get("X-Incus-aliases") ctype, ctypeParams, err := mime.ParseMediaType(r.Header.Get("Content-Type")) if err != nil { ctype = "application/octet-stream" } hash256 := sha256.New() var size int64 if ctype == "multipart/form-data" { // Create a temporary file for the image tarball imageTarf, err := os.CreateTemp(builddir, "incus_tar_") if err != nil { return nil, err } defer func() { _ = os.Remove(imageTarf.Name()) }() // Parse the POST data _, err = post.Seek(0, io.SeekStart) if err != nil { return nil, err } mr := multipart.NewReader(post, ctypeParams["boundary"]) // Get the metadata tarball part, err := mr.NextPart() if err != nil { return nil, err } if part.FormName() != "metadata" { return nil, errors.New("Invalid multipart image") } size, err = util.SafeCopy(io.MultiWriter(imageTarf, hash256), part) info.Size += size _ = imageTarf.Close() if err != nil { l.Error("Failed to copy the image tarfile", logger.Ctx{"err": err}) return nil, err } // Get the rootfs tarball part, err = mr.NextPart() if err != nil { l.Error("Failed to get the next part", logger.Ctx{"err": err}) return nil, err } if part.FormName() == "rootfs" { info.Type = instancetype.Container.String() } else if part.FormName() == "rootfs.img" { info.Type = instancetype.VM.String() } else { l.Error("Invalid multipart image") return nil, errors.New("Invalid multipart image") } // Create a temporary file for the rootfs tarball rootfsTarf, err := os.CreateTemp(builddir, "incus_tar_") if err != nil { return nil, err } defer func() { _ = os.Remove(rootfsTarf.Name()) }() size, err = util.SafeCopy(io.MultiWriter(rootfsTarf, hash256), part) info.Size += size _ = rootfsTarf.Close() if err != nil { l.Error("Failed to copy the rootfs tarfile", logger.Ctx{"err": err}) return nil, err } info.Filename = part.FileName() info.Fingerprint = fmt.Sprintf("%x", hash256.Sum(nil)) expectedFingerprint := r.Header.Get("X-Incus-fingerprint") if expectedFingerprint != "" && info.Fingerprint != expectedFingerprint { err = fmt.Errorf("fingerprints don't match, got %s expected %s", info.Fingerprint, expectedFingerprint) return nil, err } imageMeta, _, err = getImageMetadata(imageTarf.Name()) if err != nil { l.Error("Failed to get image metadata", logger.Ctx{"err": err}) return nil, err } imgfname := internalUtil.VarPath("images", info.Fingerprint) err = internalUtil.FileMove(imageTarf.Name(), imgfname) if err != nil { l.Error("Failed to move the image tarfile", logger.Ctx{ "err": err, "source": imageTarf.Name(), "dest": imgfname, }) return nil, err } rootfsfname := internalUtil.VarPath("images", info.Fingerprint+".rootfs") err = internalUtil.FileMove(rootfsTarf.Name(), rootfsfname) if err != nil { l.Error("Failed to move the rootfs tarfile", logger.Ctx{ "err": err, "source": rootfsTarf.Name(), "dest": imgfname, }) return nil, err } } else { _, err = post.Seek(0, io.SeekStart) if err != nil { return nil, err } size, err = util.SafeCopy(hash256, post) if err != nil { l.Error("Failed to copy the tarfile", logger.Ctx{"err": err}) return nil, err } info.Size = size info.Filename = r.Header.Get("X-Incus-filename") info.Fingerprint = fmt.Sprintf("%x", hash256.Sum(nil)) expectedFingerprint := r.Header.Get("X-Incus-fingerprint") if expectedFingerprint != "" && info.Fingerprint != expectedFingerprint { l.Error("Fingerprints don't match", logger.Ctx{ "got": info.Fingerprint, "expected": expectedFingerprint, }) err = fmt.Errorf("fingerprints don't match, got %s expected %s", info.Fingerprint, expectedFingerprint) return nil, err } var imageType string imageMeta, imageType, err = getImageMetadata(post.Name()) if err != nil { l.Error("Failed to get image metadata", logger.Ctx{"err": err}) return nil, err } info.Type = imageType imgfname := internalUtil.VarPath("images", info.Fingerprint) err = internalUtil.FileMove(post.Name(), imgfname) if err != nil { l.Error("Failed to move the tarfile", logger.Ctx{ "err": err, "source": post.Name(), "dest": imgfname, }) return nil, err } } info.Architecture = imageMeta.Architecture if imageMeta.CreationDate > 0 { info.CreatedAt = time.Unix(imageMeta.CreationDate, 0) } expiresAt, ok := metadata["expires_at"] if ok { info.ExpiresAt = expiresAt.(time.Time) } else if imageMeta.ExpiryDate > 0 { info.ExpiresAt = time.Unix(imageMeta.ExpiryDate, 0) } properties, ok := metadata["properties"] if ok { info.Properties = properties.(map[string]string) } else { info.Properties = imageMeta.Properties } if len(propHeaders) > 0 { for _, ph := range propHeaders { p, _ := url.ParseQuery(ph) for pkey, pval := range p { info.Properties[pkey] = pval[0] } } } if len(aliasesHeaders) > 0 { info.Aliases = []api.ImageAlias{} aliasNames, _ := url.ParseQuery(aliasesHeaders) // Check if we're using the URL encoded syntax (multiple entries) or just a simple header (single entry). aliases, ok := aliasNames["alias"] if !ok { aliases = []string{aliasesHeaders} } for _, aliasName := range aliases { alias := api.ImageAlias{ Name: aliasName, } info.Aliases = append(info.Aliases, alias) } } var profileIds []int64 if len(profilesHeaders) > 0 { profileNames, _ := url.ParseQuery(profilesHeaders) // Check if we're using the URL encoded syntax (multiple entries) or just a simple header (single entry). profiles, ok := profileNames["profile"] if !ok { profiles = []string{profilesHeaders} } profileIds = make([]int64, len(profiles)) err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { for i, val := range profiles { profileID, _, err := tx.GetProfile(ctx, project, val) if response.IsNotFoundError(err) { return fmt.Errorf("Profile '%s' doesn't exist", val) } else if err != nil { return err } profileIds[i] = profileID } return nil }) if err != nil { return nil, err } } var exists bool err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Check if the image already exists exists, err = tx.ImageExists(ctx, project, info.Fingerprint) return err }) if err != nil { return nil, err } if exists { // Do not create a database entry if the request is coming from the internal // cluster communications for image synchronization if isClusterNotification(r) { err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.AddImageToLocalNode(ctx, project, info.Fingerprint) }) if err != nil { return nil, err } } else { return &info, errors.New("Image with same fingerprint already exists") } } else { public, ok := metadata["public"] if ok { info.Public = public.(bool) } err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Create the database entry return tx.CreateImage(ctx, project, info.Fingerprint, info.Filename, info.Size, info.Public, info.AutoUpdate, info.Architecture, info.CreatedAt, info.ExpiresAt, info.Properties, info.Type, profileIds) }) if err != nil { return nil, err } } return &info, nil } // imageCreateInPool() creates a new storage volume in a given storage pool for // the image. No entry in the images database will be created. This implies that // imageCreateinPool() should only be called when an image already exists in the // database and hence has already a storage volume in at least one storage pool. func imageCreateInPool(s *state.State, info *api.Image, storagePool string) error { if storagePool == "" { return errors.New("No storage pool specified") } pool, err := storagePools.LoadByName(s, storagePool) if err != nil { return err } err = pool.EnsureImage(info.Fingerprint, nil) if err != nil { return err } return nil } // swagger:operation POST /1.0/images?public images images_post_untrusted // // Add an image // // Pushes the data to the target image server. // This is meant for server to server communication where a new image entry is // prepared on the target server and the source server is provided that URL // and a secret token to push the image content over. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: image // description: Image // required: true // schema: // $ref: "#/definitions/ImagesPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation POST /1.0/images images images_post // // Add an image // // Adds a new image to the image store. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: image // description: Image // required: false // schema: // $ref: "#/definitions/ImagesPost" // - in: body // name: raw_image // description: Raw image file // required: false // - in: header // name: X-Incus-secret // description: Push secret for server to server communication // schema: // type: string // example: RANDOM-STRING // - in: header // name: X-Incus-fingerprint // description: Expected fingerprint when pushing a raw image // schema: // type: string // - in: header // name: X-Incus-aliases // description: List of aliases to assign // schema: // type: array // items: // type: string // - in: header // name: X-Incus-properties // description: Descriptive properties // schema: // type: object // additionalProperties: // type: string // - in: header // name: X-Incus-public // description: Whether the image is available to unauthenticated users // schema: // type: boolean // - in: header // name: X-Incus-filename // description: Original filename of the image // schema: // type: string // - in: header // name: X-Incus-profiles // description: List of profiles to use // schema: // type: array // items: // type: string // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func imagesPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) var userCanCreateImages bool err := s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectProject(projectName), auth.EntitlementCanCreateImages) if err == nil { userCanCreateImages = true } else if !api.StatusErrorCheck(err, http.StatusForbidden) { return response.SmartError(err) } trusted := d.checkTrustedClient(r) == nil && userCanCreateImages secret := r.Header.Get("X-Incus-secret") fingerprint := r.Header.Get("X-Incus-fingerprint") var imageMetadata map[string]any if !trusted && (secret == "" || fingerprint == "") { return response.Forbidden(nil) } else { // We need to invalidate the secret whether the source is trusted or not. op, err := imageValidSecret(s, r, projectName, fingerprint, secret) if err != nil { return response.SmartError(err) } if op != nil { imageMetadata = op.Metadata } else if !trusted { return response.Forbidden(nil) } } // create a directory under which we keep everything while building builddir, err := os.MkdirTemp(internalUtil.VarPath("images"), "incus_build_") if err != nil { return response.InternalError(err) } cleanup := func(path string, fd *os.File) { if fd != nil { _ = fd.Close() } err := os.RemoveAll(path) if err != nil { logger.Debugf("Error deleting temporary directory \"%s\": %s", path, err) } } // Store the post data to disk post, err := os.CreateTemp(builddir, "incus_post_") if err != nil { cleanup(builddir, nil) return response.InternalError(err) } // Possibly set a quota on the amount of disk space this project is // allowed to use. var budget int64 err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { budget, err = projectutils.GetImageSpaceBudget(tx, projectName) return err }) if err != nil { return response.SmartError(err) } _, err = util.SafeCopy(internalIO.NewQuotaWriter(post, budget), r.Body) if err != nil { logger.Errorf("Store image POST data to disk: %v", err) cleanup(builddir, post) return response.InternalError(err) } // Is this a container request? _, err = post.Seek(0, io.SeekStart) if err != nil { return response.InternalError(err) } decoder := json.NewDecoder(post) imageUpload := false req := api.ImagesPost{} err = decoder.Decode(&req) if err != nil { if r.Header.Get("Content-Type") == "application/json" { cleanup(builddir, post) return response.BadRequest(err) } imageUpload = true } if !imageUpload && req.Source.Mode == "push" { cleanup(builddir, post) metadata := map[string]any{ "aliases": req.Aliases, "expires_at": req.ExpiresAt, "properties": req.Properties, "public": req.Public, } return createTokenResponse(s, r, projectName, req.Source.Fingerprint, metadata) } if !imageUpload && !slices.Contains([]string{"container", "instance", "virtual-machine", "snapshot", "image", "url"}, req.Source.Type) { cleanup(builddir, post) return response.InternalError(errors.New("Invalid images JSON")) } /* Forward requests for containers on other nodes */ if !imageUpload && slices.Contains([]string{"container", "instance", "virtual-machine", "snapshot"}, req.Source.Type) { name := req.Source.Name if name != "" { _, err = post.Seek(0, io.SeekStart) if err != nil { return response.InternalError(err) } r.Body = post resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { cleanup(builddir, post) return response.SmartError(err) } if resp != nil { cleanup(builddir, nil) return resp } } } // Begin background operation run := func(op *operations.Operation) error { var err error var info *api.Image // Setup the cleanup function defer cleanup(builddir, post) if imageUpload { /* Processing image upload */ info, err = getImgPostInfo(context.TODO(), s, r, builddir, projectName, post, imageMetadata) } else { if req.Source.Type == "image" { /* Processing image copy from remote */ info, err = imgPostRemoteInfo(context.TODO(), s, r, req, op, projectName, budget) } else if req.Source.Type == "url" { /* Processing image copy from URL */ info, err = imgPostURLInfo(context.TODO(), s, r, req, op, projectName, budget) } else { /* Processing image creation from container */ imagePublishLock.Lock() info, err = imgPostInstanceInfo(context.TODO(), s, r, req, op, builddir, budget) imagePublishLock.Unlock() } } // Set the metadata if possible, even if there is an error if info != nil { metadata := make(map[string]string) metadata["fingerprint"] = info.Fingerprint metadata["size"] = strconv.FormatInt(info.Size, 10) // Keep secret if available secret, ok := op.Metadata()["secret"] if ok { metadata["secret"] = secret.(string) } _ = op.UpdateMetadata(metadata) } if err != nil { return err } if isClusterNotification(r) { // If dealing with in-cluster image copy, don't touch the database. return nil } // Apply any provided alias if len(req.Aliases) == 0 { aliases, ok := imageMetadata["aliases"] if ok { // Used to get aliases from push mode image copy operation. aliases, ok := aliases.([]api.ImageAlias) if ok { req.Aliases = aliases } } else if len(info.Aliases) > 0 { // Used to get aliases from HTTP headers on raw image imports. req.Aliases = info.Aliases } } err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { imgID, _, err := tx.GetImageByFingerprintPrefix(ctx, info.Fingerprint, dbCluster.ImageFilter{Project: &projectName}) if err != nil { return fmt.Errorf("Fetch image %q: %w", info.Fingerprint, err) } for _, alias := range req.Aliases { _, _, err := tx.GetImageAlias(ctx, projectName, alias.Name, true) if !response.IsNotFoundError(err) { if err != nil { return fmt.Errorf("Fetch image alias %q: %w", alias.Name, err) } return fmt.Errorf("Alias already exists: %s", alias.Name) } err = tx.CreateImageAlias(ctx, projectName, alias.Name, imgID, alias.Description) if err != nil { return fmt.Errorf("Add new image alias to the database: %w", err) } // Add the image alias to the authorizer. err = s.Authorizer.AddImageAlias(ctx, projectName, alias.Name) if err != nil { logger.Error("Failed to add image alias to authorizer", logger.Ctx{"name": alias.Name, "project": projectName, "error": err}) } s.Events.SendLifecycle(projectName, lifecycle.ImageAliasCreated.Event(alias.Name, projectName, op.Requestor(), logger.Ctx{"target": info.Fingerprint})) } return nil }) if err != nil { return err } // Sync the images between each node in the cluster on demand err = imageSyncBetweenNodes(context.TODO(), s, r, projectName, info.Fingerprint) if err != nil { return fmt.Errorf("Failed syncing image between servers: %w", err) } // Add the image to the authorizer. err = s.Authorizer.AddImage(s.ShutdownCtx, projectName, info.Fingerprint) if err != nil { logger.Error("Failed to add image to authorizer", logger.Ctx{"fingerprint": info.Fingerprint, "project": projectName, "error": err}) } s.Events.SendLifecycle(projectName, lifecycle.ImageCreated.Event(info.Fingerprint, projectName, op.Requestor(), logger.Ctx{"type": info.Type})) return nil } var metadata any if imageUpload && imageMetadata != nil { secret, _ := internalUtil.RandomHexString(32) if secret != "" { metadata = map[string]string{ "secret": secret, } } } op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.ImageDownload, nil, metadata, run, nil, nil, r) if err != nil { cleanup(builddir, post) return response.InternalError(err) } return operations.OperationResponse(op) } func getImageMetadata(fname string) (*api.ImageMetadata, string, error) { var tr *tar.Reader var result api.ImageMetadata // Open the file r, err := os.Open(fname) if err != nil { return nil, "unknown", err } defer func() { _ = r.Close() }() // Decompress if needed _, algo, unpacker, err := archive.DetectCompressionFile(r) if err != nil { return nil, "unknown", err } _, err = r.Seek(0, io.SeekStart) if err != nil { return nil, "", err } if unpacker == nil { return nil, "unknown", errors.New("Unsupported backup compression") } // Open the tarball if len(unpacker) > 0 { if algo == ".squashfs" { // sqfs2tar can only read from a file unpacker = append(unpacker, fname) } cmd := exec.Command(unpacker[0], unpacker[1:]...) cmd.Stdin = r stdout, err := cmd.StdoutPipe() if err != nil { return nil, "unknown", err } defer func() { _ = stdout.Close() }() err = cmd.Start() if err != nil { return nil, "unknown", err } defer func() { _ = cmd.Wait() }() // Double close stdout, this is to avoid blocks in Wait() defer func() { _ = stdout.Close() }() tr = tar.NewReader(stdout) } else { tr = tar.NewReader(r) } // Parse the content hasMeta := false hasRoot := false imageType := "unknown" for { hdr, err := tr.Next() if err == io.EOF { break // End of archive } if err != nil { return nil, "unknown", err } if hdr.Name == "metadata.yaml" || hdr.Name == "./metadata.yaml" { loader, err := yaml.NewLoader(localUtil.MaxBytesReader(tr, 1024*1024)) if err != nil { return nil, "unknown", err } err = loader.Load(&result) if err != nil { return nil, "unknown", err } hasMeta = true } if strings.HasPrefix(hdr.Name, "rootfs/") || strings.HasPrefix(hdr.Name, "./rootfs/") { hasRoot = true imageType = instancetype.Container.String() } if hdr.Name == "rootfs.img" || hdr.Name == "./rootfs.img" { hasRoot = true imageType = instancetype.VM.String() } if hasMeta && hasRoot { // Done with the bits we want, no need to keep reading break } } if !hasMeta { return nil, "unknown", errors.New("Metadata tarball is missing metadata.yaml") } _, err = osarch.ArchitectureID(result.Architecture) if err != nil { return nil, "unknown", err } if result.CreationDate == 0 { return nil, "unknown", errors.New("Missing creation date") } return &result, imageType, nil } func doImagesGet(ctx context.Context, tx *db.ClusterTx, recursion bool, projectName string, public bool, clauses *filter.ClauseSet, hasPermission auth.PermissionChecker, allProjects bool) (any, error) { mustLoadObjects := recursion || (clauses != nil && len(clauses.Clauses) > 0) imagesProjectsMap := map[string][]string{} if allProjects { var err error imagesProjectsMap, err = tx.GetImages(ctx) if err != nil { return nil, err } } else { fingerprints, err := tx.GetImagesFingerprints(ctx, projectName, public) if err != nil { return nil, err } for _, fp := range fingerprints { imagesProjectsMap[fp] = []string{projectName} } } var resultString []string var resultMap []*api.Image if recursion { resultMap = make([]*api.Image, 0, len(imagesProjectsMap)) } else { resultString = make([]string, 0, len(imagesProjectsMap)) } for fingerprint, projects := range imagesProjectsMap { for _, curProjectName := range projects { image, err := doImageGet(ctx, tx, curProjectName, fingerprint, public) if err != nil { continue } if !image.Public && !hasPermission(auth.ObjectImage(curProjectName, fingerprint)) { continue } if !mustLoadObjects { resultString = append(resultString, api.NewURL().Path(version.APIVersion, "images", fingerprint).String()) } else { if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(*image, *clauses) if err != nil { return nil, err } if !match { continue } } if recursion { resultMap = append(resultMap, image) } else { resultString = append(resultString, api.NewURL().Path(version.APIVersion, "images", image.Fingerprint).String()) } } } } if recursion { return resultMap, nil } return resultString, nil } // swagger:operation GET /1.0/images?public images images_get_untrusted // // Get the public images // // Returns a list of publicly available images (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // - in: query // name: all-projects // description: Retrieve images from all projects // type: boolean // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/images/06b86454720d36b20f94e31c6812e05ec51c1b568cf3a8abd273769d213394bb", // "/1.0/images/084dd79dd1360fd25a2479eb46674c2a5ef3022a40fe03c91ab3603e3402b8e1" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/images?public&recursion=1 images images_get_recursion1_untrusted // // Get the public images // // Returns a list of publicly available images (structs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // - in: query // name: all-projects // description: Retrieve images from all projects // type: boolean // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of images // items: // $ref: "#/definitions/Image" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/images images images_get // // Get the images // // Returns a list of images (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // - in: query // name: all-projects // description: Retrieve images from all projects // type: boolean // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/images/06b86454720d36b20f94e31c6812e05ec51c1b568cf3a8abd273769d213394bb", // "/1.0/images/084dd79dd1360fd25a2479eb46674c2a5ef3022a40fe03c91ab3603e3402b8e1" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/images?recursion=1 images images_get_recursion1 // // Get the images // // Returns a list of images (structs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // - in: query // name: all-projects // description: Retrieve images from all projects // type: boolean // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of images // items: // $ref: "#/definitions/Image" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func imagesGet(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the parameters. projectName := request.ProjectParam(r) allProjects := util.IsTrue(r.FormValue("all-projects")) filterStr := r.FormValue("filter") // Make sure that we're not dealing with conflicting parameters. if allProjects && projectName != api.ProjectDefaultName { return response.BadRequest(errors.New("Cannot specify a project when requesting all projects")) } // Check if the user is authenticated and what kind of access they may have. hasPermission, authorizationErr := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanView, auth.ObjectTypeImage) if authorizationErr != nil && !api.StatusErrorCheck(authorizationErr, http.StatusForbidden) { return response.SmartError(authorizationErr) } public := d.checkTrustedClient(r) != nil || authorizationErr != nil // For unauthenticated/public requests, only the default profile may be queried. if public && (projectName != api.ProjectDefaultName || allProjects) { return response.BadRequest(errors.New("Unauthenticated image queries are only possible against the default project")) } // Process the filters. clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.SmartError(fmt.Errorf("Invalid filter: %w", err)) } // Get the image list. var result any err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { result, err = doImagesGet(ctx, tx, localUtil.IsRecursionRequest(r), projectName, public, clauses, hasPermission, allProjects) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } return response.SyncResponse(true, result) } func autoUpdateImagesTask(d *Daemon) (task.Func, task.Schedule) { f := func(ctx context.Context) { s := d.State() opRun := func(op *operations.Operation) error { return autoUpdateImages(ctx, s) } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.ImagesUpdate, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating image update operation", logger.Ctx{"err": err}) return } logger.Debug("Acquiring image task lock") imageTaskMu.Lock() defer imageTaskMu.Unlock() logger.Debug("Acquired image task lock") logger.Info("Updating images") err = op.Start() if err != nil { logger.Error("Failed starting image update operation", logger.Ctx{"err": err}) return } err = op.Wait(ctx) if err != nil { logger.Error("Failed updating images", logger.Ctx{"err": err}) return } logger.Info("Done updating images") } return f, task.Hourly() } func autoUpdateImages(ctx context.Context, s *state.State) error { imageMap := make(map[string][]dbCluster.Image) err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error autoUpdate := true images, err := dbCluster.GetImages(ctx, tx.Tx(), dbCluster.ImageFilter{AutoUpdate: &autoUpdate}) if err != nil { return err } for _, image := range images { imageMap[image.Fingerprint] = append(imageMap[image.Fingerprint], image) } return nil }) if err != nil { return fmt.Errorf("Unable to retrieve image fingerprints: %w", err) } for fingerprint, images := range imageMap { skipFingerprint := false var nodes []string err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { nodes, err = tx.GetNodesWithImageAndAutoUpdate(ctx, fingerprint, true) return err }) if err != nil { logger.Error("Error getting cluster members for image auto-update", logger.Ctx{"fingerprint": fingerprint, "err": err}) continue } if len(nodes) > 1 { var nodeIDs []int64 for _, node := range nodes { err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error nodeInfo, err := tx.GetNodeByAddress(ctx, node) if err != nil { return err } nodeIDs = append(nodeIDs, nodeInfo.ID) return nil }) if err != nil { logger.Error("Unable to retrieve cluster member information for image update", logger.Ctx{"err": err}) skipFingerprint = true break } } if skipFingerprint { continue } // If multiple nodes have the image, select one to deal with it. if len(nodeIDs) > 1 { selectedNode, err := localUtil.GetStableRandomInt64FromList(int64(len(images)), nodeIDs) if err != nil { logger.Error("Failed to select cluster member for image update", logger.Ctx{"err": err}) continue } // Skip image update if we're not the chosen cluster member. // That way, an image is only updated by a single cluster member. if s.DB.Cluster.GetNodeID() != selectedNode { continue } } } var deleteIDs []int var newImage *api.Image for _, image := range images { filter := dbCluster.ImageFilter{Project: &image.Project} if image.Public { filter.Public = &image.Public } var imageInfo *api.Image err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { _, imageInfo, err = tx.GetImage(ctx, image.Fingerprint, filter) return err }) if err != nil { logger.Error("Failed to get image", logger.Ctx{"err": err, "project": image.Project, "fingerprint": image.Fingerprint}) continue } newInfo, err := autoUpdateImage(ctx, s, nil, image.ID, imageInfo, image.Project, false) if err != nil { logger.Error("Failed to update image", logger.Ctx{"err": err, "project": image.Project, "fingerprint": image.Fingerprint}) if errors.Is(err, context.Canceled) { return nil } } else { deleteIDs = append(deleteIDs, image.ID) } // newInfo will have the same content for each image in the list. // Therefore, we just pick the first. if newImage == nil { newImage = newInfo } } if newImage != nil { if len(nodes) > 1 { err := distributeImage(ctx, s, nodes, fingerprint, newImage) if err != nil { logger.Error("Failed to distribute new image", logger.Ctx{"err": err, "fingerprint": newImage.Fingerprint}) if errors.Is(err, context.Canceled) { return nil } } } _ = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { for _, ID := range deleteIDs { // Remove the database entry for the image after distributing to cluster members. err := tx.DeleteImage(ctx, ID) if err != nil { logger.Error("Error deleting old image from database", logger.Ctx{"err": err, "fingerprint": fingerprint, "ID": ID}) } } return nil }) } } return nil } func distributeImage(ctx context.Context, s *state.State, nodes []string, oldFingerprint string, newImage *api.Image) error { // Get config of all nodes (incl. own) and check for storage.images_volume. // If the setting is missing, distribute the image to the node. // If the option is set, only distribute the image once to nodes with this // specific pool/volume. // imageVolumes is a list containing of all image volumes specified by // storage.images_volume. Since this option is node specific, the values // may be different for each cluster member. var imageVolumes []string err := s.DB.Node.Transaction(ctx, func(ctx context.Context, tx *db.NodeTx) error { config, err := node.ConfigLoad(ctx, tx) if err != nil { return err } vol := config.StorageImagesVolume() if vol != "" { fields := strings.Split(vol, "/") var pool *api.StoragePool err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { _, pool, _, err = tx.GetStoragePool(ctx, fields[0]) return err }) if err != nil { return fmt.Errorf("Failed to get storage pool info: %w", err) } // Add the volume to the list if the pool is backed by remote // storage as only then the volumes are shared. if slices.Contains(db.StorageRemoteDriverNames(), pool.Driver) { imageVolumes = append(imageVolumes, vol) } } return nil }) // No need to return with an error as this is only an optimization in the // distribution process. Instead, only log the error. if err != nil { logger.Error("Failed to load config", logger.Ctx{"err": err}) } // Skip own node localClusterAddress := s.LocalConfig.ClusterAddress() var poolIDs []int64 var poolNames []string err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Get the IDs of all storage pools on which a storage volume // for the requested image currently exists. poolIDs, err = tx.GetPoolsWithImage(ctx, newImage.Fingerprint) if err != nil { logger.Error("Error getting image storage pools", logger.Ctx{"err": err, "fingerprint": oldFingerprint}) return err } // Translate the IDs to poolNames. poolNames, err = tx.GetPoolNamesFromIDs(ctx, poolIDs) if err != nil { logger.Error("Error getting image storage pools", logger.Ctx{"err": err, "fingerprint": oldFingerprint}) return err } return nil }) if err != nil { return err } for _, nodeAddress := range nodes { if nodeAddress == localClusterAddress { continue } var nodeInfo db.NodeInfo err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error nodeInfo, err = tx.GetNodeByAddress(ctx, nodeAddress) return err }) if err != nil { return fmt.Errorf("Failed to retrieve information about cluster member with address %q: %w", nodeAddress, err) } client, err := cluster.Connect(nodeAddress, s.Endpoints.NetworkCert(), s.ServerCert(), nil, true) if err != nil { return fmt.Errorf("Failed to connect to %q for image synchronization: %w", nodeAddress, err) } client = client.UseTarget(nodeInfo.Name) resp, _, err := client.GetServer() if err != nil { logger.Error("Failed to retrieve information about cluster member", logger.Ctx{"err": err, "remote": nodeAddress}) } else { vol := resp.Config["storage.images_volume"] skipDistribution := false // If storage.images_volume is set on the cluster member, check if // the image has already been downloaded to this volume. If so, // skip distributing the image to this cluster member. // If the option is unset, distribute the image. if vol != "" { if slices.Contains(imageVolumes, vol) { skipDistribution = true } if skipDistribution { continue } fields := strings.Split(vol, "/") pool, _, err := client.GetStoragePool(fields[0]) if err != nil { logger.Error("Failed to get storage pool info", logger.Ctx{"err": err, "pool": fields[0]}) } else { if slices.Contains(db.StorageRemoteDriverNames(), pool.Driver) { imageVolumes = append(imageVolumes, vol) } } } } createArgs := &incus.ImageCreateArgs{} imageMetaPath := internalUtil.VarPath("images", newImage.Fingerprint) imageRootfsPath := internalUtil.VarPath("images", newImage.Fingerprint+".rootfs") metaFile, err := os.Open(imageMetaPath) if err != nil { return err } defer func() { _ = metaFile.Close() }() createArgs.MetaFile = metaFile createArgs.MetaName = filepath.Base(imageMetaPath) createArgs.Type = newImage.Type if util.PathExists(imageRootfsPath) { rootfsFile, err := os.Open(imageRootfsPath) if err != nil { return err } defer func() { _ = rootfsFile.Close() }() createArgs.RootfsFile = rootfsFile createArgs.RootfsName = filepath.Base(imageRootfsPath) } image := api.ImagesPost{} image.Filename = createArgs.MetaName op, err := client.CreateImage(image, createArgs) if err != nil { return err } select { case <-ctx.Done(): _ = op.Cancel() return ctx.Err() default: } err = op.Wait() if err != nil { return err } for _, poolName := range poolNames { if poolName == "" { continue } req := internalImageOptimizePost{ Image: *newImage, Pool: poolName, } _, _, err = client.RawQuery("POST", "/internal/image-optimize", req, "") if err != nil { logger.Error("Failed creating new image in storage pool", logger.Ctx{"err": err, "remote": nodeAddress, "pool": poolName, "fingerprint": newImage.Fingerprint}) } err = client.DeleteStoragePoolVolume(poolName, "image", oldFingerprint) if err != nil { logger.Error("Failed deleting old image from storage pool", logger.Ctx{"err": err, "remote": nodeAddress, "pool": poolName, "fingerprint": oldFingerprint}) } } } return nil } // Update a single image. The operation can be nil, if no progress tracking is needed. // Returns whether the image has been updated. func autoUpdateImage(ctx context.Context, s *state.State, op *operations.Operation, id int, info *api.Image, projectName string, manual bool) (*api.Image, error) { fingerprint := info.Fingerprint var source api.ImageSource if !manual { var interval int64 var project *api.Project err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { p, err := dbCluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return err } project, err = p.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return nil, err } if project.Config["images.auto_update_interval"] != "" { interval, err = strconv.ParseInt(project.Config["images.auto_update_interval"], 10, 64) if err != nil { return nil, fmt.Errorf("Unable to fetch project configuration: %w", err) } } else { interval = s.GlobalConfig.ImagesAutoUpdateIntervalHours() } // Check if we're supposed to auto update at all (0 disables it) if interval <= 0 { return nil, nil } now := time.Now() elapsedHours := int64(math.Round(now.Sub(s.StartTime).Hours())) if elapsedHours%interval != 0 { return nil, nil } } var poolNames []string err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error _, source, err = tx.GetImageSource(ctx, id) if err != nil { logger.Error("Error getting source image", logger.Ctx{"err": err, "fingerprint": fingerprint}) return err } // Get the IDs of all storage pools on which a storage volume // for the requested image currently exists. poolIDs, err := tx.GetPoolsWithImage(ctx, fingerprint) if err != nil { logger.Error("Error getting image pools", logger.Ctx{"err": err, "fingerprint": fingerprint}) return err } // Translate the IDs to poolNames. poolNames, err = tx.GetPoolNamesFromIDs(ctx, poolIDs) if err != nil { logger.Error("Error getting image pools", logger.Ctx{"err": err, "fingerprint": fingerprint}) return err } return nil }) if err != nil { return nil, err } // If no optimized pools at least update the base store if len(poolNames) == 0 { poolNames = append(poolNames, "") } logger.Debug("Processing image", logger.Ctx{"fingerprint": fingerprint, "server": source.Server, "protocol": source.Protocol, "alias": source.Alias}) // Set operation metadata to indicate whether a refresh happened setRefreshResult := func(result bool) { if op == nil { return } metadata := map[string]any{"refreshed": result} _ = op.UpdateMetadata(metadata) // Sent a lifecycle event if the refresh actually happened. if result { s.Events.SendLifecycle(projectName, lifecycle.ImageRefreshed.Event(fingerprint, projectName, op.Requestor(), nil)) } } // Update the image on each pool where it currently exists. hash := fingerprint var newInfo *api.Image for _, poolName := range poolNames { select { case <-ctx.Done(): return nil, ctx.Err() default: } newInfo, _, err = imageDownload(ctx, nil, s, op, &imageDownloadArgs{ Server: source.Server, Protocol: source.Protocol, Certificate: source.Certificate, Alias: source.Alias, Type: info.Type, AutoUpdate: true, Public: info.Public, StoragePool: poolName, ProjectName: projectName, Budget: -1, }) if err != nil { logger.Error("Failed to update the image", logger.Ctx{"err": err, "fingerprint": fingerprint}) continue } hash = newInfo.Fingerprint if hash == fingerprint { logger.Debug("Image already up to date", logger.Ctx{"fingerprint": fingerprint}) continue } var newID int err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { newID, _, err = tx.GetImage(ctx, hash, dbCluster.ImageFilter{Project: &projectName}) return err }) if err != nil { logger.Error("Error loading image", logger.Ctx{"err": err, "fingerprint": hash}) continue } if info.Cached { err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.SetImageCachedAndLastUseDate(ctx, projectName, hash, info.LastUsedAt) }) if err != nil { logger.Error("Error setting cached flag and last use date", logger.Ctx{"err": err, "fingerprint": hash}) continue } } else { err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { err = tx.UpdateImageLastUseDate(ctx, projectName, hash, info.LastUsedAt) if err != nil { logger.Error("Error setting last use date", logger.Ctx{"err": err, "fingerprint": hash}) return err } return nil }) if err != nil { continue } } err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.MoveImageAlias(ctx, id, newID) }) if err != nil { logger.Error("Error moving aliases", logger.Ctx{"err": err, "fingerprint": hash}) continue } err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.CopyDefaultImageProfiles(ctx, id, newID) }) if err != nil { logger.Error("Copying default profiles", logger.Ctx{"err": err, "fingerprint": hash}) } // If we do have optimized pools, make sure we remove the volumes associated with the image. if poolName != "" { pool, err := storagePools.LoadByName(s, poolName) if err != nil { logger.Error("Error loading storage pool to delete image", logger.Ctx{"err": err, "pool": poolName, "fingerprint": fingerprint}) continue } err = pool.DeleteImage(fingerprint, op) if err != nil { logger.Error("Error deleting image from storage pool", logger.Ctx{"err": err, "pool": pool.Name(), "fingerprint": fingerprint}) continue } } // Add the image to the authorizer. err = s.Authorizer.AddImage(s.ShutdownCtx, projectName, info.Fingerprint) if err != nil { logger.Error("Failed to add image to authorizer", logger.Ctx{"fingerprint": info.Fingerprint, "project": projectName, "error": err}) } var requestor *api.EventLifecycleRequestor if op != nil { requestor = op.Requestor() } s.Events.SendLifecycle(projectName, lifecycle.ImageCreated.Event(info.Fingerprint, projectName, requestor, logger.Ctx{"type": info.Type})) } // Image didn't change, nothing to do. if hash == fingerprint { setRefreshResult(false) return nil, nil } // Remove main image file. fname := filepath.Join(s.OS.VarDir, "images", fingerprint) if util.PathExists(fname) { err = os.Remove(fname) if err != nil { logger.Error("Error deleting image file", logger.Ctx{"fingerprint": fingerprint, "file": fname, "err": err}) } } // Remove the rootfs file for the image. fname = filepath.Join(s.OS.VarDir, "images", fingerprint) + ".rootfs" if util.PathExists(fname) { err = os.Remove(fname) if err != nil { logger.Error("Error deleting image rootfs file", logger.Ctx{"fingerprint": fingerprint, "file": fname, "err": err}) } } setRefreshResult(true) return newInfo, nil } func pruneExpiredImagesTask(d *Daemon) (task.Func, task.Schedule) { f := func(ctx context.Context) { s := d.State() opRun := func(op *operations.Operation) error { return pruneExpiredImages(ctx, s, op) } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.ImagesExpire, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating expired image prune operation", logger.Ctx{"err": err}) return } logger.Debug("Acquiring image task lock") imageTaskMu.Lock() defer imageTaskMu.Unlock() logger.Debug("Acquired image task lock") logger.Info("Pruning expired images") err = op.Start() if err != nil { logger.Error("Failed starting expired image prune operation", logger.Ctx{"err": err}) return } err = op.Wait(ctx) if err != nil { logger.Error("Failed expiring images", logger.Ctx{"err": err}) return } logger.Info("Done pruning expired images") } // Skip the first run, and instead run an initial pruning synchronously // before we start updating images later on in the start up process. f(context.Background()) first := true schedule := func() (time.Duration, error) { interval := 24 * time.Hour if first { first = false return interval, task.ErrSkip } return interval, nil } return f, schedule } func pruneLeftoverImages(s *state.State) { opRun := func(op *operations.Operation) error { // Check if dealing with shared image storage. var storageImages string err := s.DB.Node.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.NodeTx) error { nodeConfig, err := node.ConfigLoad(ctx, tx) if err != nil { return err } storageImages = nodeConfig.StorageImagesVolume() return nil }) if err != nil { return err } if storageImages != "" { // Parse the source. poolName, _, err := daemonStorageSplitVolume(storageImages) if err != nil { return err } // Load the pool. pool, err := storagePools.LoadByName(s, poolName) if err != nil { return err } // Skip cleanup if image volume may be multi-node. // When such a volume is used, we may have images that are // tied to other servers in the shared images folder and don't want to // delete those. if pool.Driver().Info().VolumeMultiNode { return nil } } // Get all images var images []string err = s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { var err error images, err = tx.GetLocalImagesFingerprints(ctx) return err }) if err != nil { return fmt.Errorf("Unable to retrieve the list of images: %w", err) } // Look at what's in the images directory entries, err := os.ReadDir(internalUtil.VarPath("images")) if err != nil { return fmt.Errorf("Unable to list the images directory: %w", err) } // Check and delete leftovers for _, entry := range entries { fp := strings.Split(entry.Name(), ".")[0] if !slices.Contains(images, fp) { err = os.RemoveAll(internalUtil.VarPath("images", entry.Name())) if err != nil { return fmt.Errorf("Unable to remove leftover image: %v: %w", entry.Name(), err) } logger.Debugf("Removed leftover image file: %s", entry.Name()) } } return nil } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.ImagesPruneLeftover, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating leftover image clean up operation", logger.Ctx{"err": err}) return } logger.Debug("Acquiring image task lock") imageTaskMu.Lock() defer imageTaskMu.Unlock() logger.Debug("Acquired image task lock") logger.Info("Cleaning up leftover image files") err = op.Start() if err != nil { logger.Error("Failed starting leftover image clean up operation", logger.Ctx{"err": err}) return } err = op.Wait(s.ShutdownCtx) if err != nil { logger.Error("Failed cleaning up leftover image files", logger.Ctx{"err": err}) return } logger.Infof("Done cleaning up leftover image files") } func pruneExpiredImages(ctx context.Context, s *state.State, op *operations.Operation) error { var err error var projectsImageRemoteCacheExpiryDays map[string]int64 var allImages map[string][]dbCluster.Image err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Get an image remote cache expiry days value for each project and store keyed on project name. globalImageRemoteCacheExpiryDays := s.GlobalConfig.ImagesRemoteCacheExpiryDays() dbProjects, err := dbCluster.GetProjects(ctx, tx.Tx()) if err != nil { return err } projectsImageRemoteCacheExpiryDays = make(map[string]int64, len(dbProjects)) for _, p := range dbProjects { p, err := p.ToAPI(ctx, tx.Tx()) if err != nil { return err } // If there is a project specific image expiry set use that. if p.Config["images.remote_cache_expiry"] != "" { expiry, err := strconv.ParseInt(p.Config["images.remote_cache_expiry"], 10, 64) if err != nil { return fmt.Errorf("Unable to fetch project configuration: %w", err) } projectsImageRemoteCacheExpiryDays[p.Name] = expiry } else { // Otherwise use the global default. projectsImageRemoteCacheExpiryDays[p.Name] = globalImageRemoteCacheExpiryDays } } // Get all cached images across all projects and store them keyed on fingerprint. cached := true images, err := dbCluster.GetImages(ctx, tx.Tx(), dbCluster.ImageFilter{Cached: &cached}) if err != nil { return fmt.Errorf("Failed getting images: %w", err) } allImages = make(map[string][]dbCluster.Image, len(images)) for _, image := range images { allImages[image.Fingerprint] = append(allImages[image.Fingerprint], image) } return nil }) if err != nil { return fmt.Errorf("Unable to retrieve project names: %w", err) } for fingerprint, dbImages := range allImages { // At each iteration we check if we got cancelled in the meantime. It is safe to abort here since // anything not expired now will be expired at the next run. select { case <-ctx.Done(): return nil default: } dbImagesDeleted := 0 for _, dbImage := range dbImages { // Get expiry days for image's project. expiryDays := projectsImageRemoteCacheExpiryDays[dbImage.Project] // Skip if no project expiry time set. if expiryDays <= 0 { continue } // Figure out the expiry of image. timestamp := dbImage.UploadDate if !dbImage.LastUseDate.Time.IsZero() { timestamp = dbImage.LastUseDate.Time } imageExpiry := timestamp.Add(time.Duration(expiryDays) * time.Hour * 24) // Skip if image is not expired. if imageExpiry.After(time.Now()) { continue } err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Remove the database entry for the image. return tx.DeleteImage(ctx, dbImage.ID) }) if err != nil { return fmt.Errorf("Error deleting image %q in project %q from database: %w", fingerprint, dbImage.Project, err) } dbImagesDeleted++ logger.Info("Deleted expired cached image record", logger.Ctx{"fingerprint": fingerprint, "project": dbImage.Project, "expiry": imageExpiry}) s.Events.SendLifecycle(dbImage.Project, lifecycle.ImageDeleted.Event(fingerprint, dbImage.Project, op.Requestor(), nil)) } // Skip deleting the image files and image storage volumes on disk if image is not expired in all // of its projects. if dbImagesDeleted < len(dbImages) { continue } var poolIDs []int64 var poolNames []string err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Get the IDs of all storage pools on which a storage volume for the image currently exists. poolIDs, err = tx.GetPoolsWithImage(ctx, fingerprint) if err != nil { return err } // Translate the IDs to poolNames. poolNames, err = tx.GetPoolNamesFromIDs(ctx, poolIDs) if err != nil { return err } return nil }) if err != nil { continue } for _, poolName := range poolNames { pool, err := storagePools.LoadByName(s, poolName) if err != nil { return fmt.Errorf("Error loading storage pool %q to delete image volume %q: %w", poolName, fingerprint, err) } err = pool.DeleteImage(fingerprint, op) if err != nil { return fmt.Errorf("Error deleting image volume %q from storage pool %q: %w", fingerprint, pool.Name(), err) } } // Remove main image file. fname := filepath.Join(s.OS.VarDir, "images", fingerprint) err = os.Remove(fname) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Error deleting image file %q: %w", fname, err) } // Remove the rootfs file for the image. fname = filepath.Join(s.OS.VarDir, "images", fingerprint) + ".rootfs" err = os.Remove(fname) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Error deleting image file %q: %w", fname, err) } logger.Info("Deleted expired cached image files and volumes", logger.Ctx{"fingerprint": fingerprint}) } return nil } // swagger:operation DELETE /1.0/images/{fingerprint} images image_delete // // Delete the image // // Removes the image from the image store. // // --- // produces: // - application/json // parameters: // - in: path // name: fingerprint // description: Fingerprint // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func imageDelete(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) fingerprint, err := url.PathUnescape(mux.Vars(r)["fingerprint"]) if err != nil { return response.SmartError(err) } var imgID int var imgInfo *api.Image err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Use the fingerprint we received in a LIKE query and use the full // fingerprint we receive from the database in all further queries. imgID, imgInfo, err = tx.GetImage(ctx, fingerprint, dbCluster.ImageFilter{Project: &projectName}) return err }) if err != nil { return response.SmartError(err) } do := func(op *operations.Operation) error { // Lock this operation to ensure that concurrent image operations don't conflict. // Other operations will wait for this one to finish. unlock, err := imageOperationLock(context.TODO(), imgInfo.Fingerprint) if err != nil { return err } defer unlock() var exist bool err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Check image still exists and another request hasn't removed it since we resolved the image // fingerprint above. exist, err = tx.ImageExists(ctx, projectName, imgInfo.Fingerprint) return err }) if err != nil { return err } if !exist { return api.StatusErrorf(http.StatusNotFound, "Image not found") } if !isClusterNotification(r) { var referenced bool err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Check if the image being deleted is actually still // referenced by other projects. In that case we don't want to // physically delete it just yet, but just to remove the // relevant database entry. referenced, err = tx.ImageIsReferencedByOtherProjects(ctx, projectName, imgInfo.Fingerprint) if err != nil { return err } if referenced { err = tx.DeleteImage(ctx, imgID) if err != nil { return fmt.Errorf("Error deleting image info from the database: %w", err) } } return nil }) if err != nil { return err } if referenced { return nil } // Notify the other nodes about the removed image so they can remove it from disk too. notifier, err := cluster.NewNotifier(s, s.Endpoints.NetworkCert(), s.ServerCert(), cluster.NotifyAll) if err != nil { return err } err = notifier(func(client incus.InstanceServer) error { op, err := client.UseProject(projectName).DeleteImage(imgInfo.Fingerprint) if err != nil { return fmt.Errorf("Failed to request to delete image from peer node: %w", err) } err = op.Wait() if err != nil { return fmt.Errorf("Failed to delete image from peer node: %w", err) } return nil }) if err != nil { return err } // Delete the aliases. for _, alias := range imgInfo.Aliases { err = s.Authorizer.DeleteImageAlias(s.ShutdownCtx, projectName, alias.Name) if err != nil { logger.Error("Failed to remove image alias from authorizer", logger.Ctx{"name": alias.Name, "project": projectName, "error": err}) } s.Events.SendLifecycle(projectName, lifecycle.ImageAliasDeleted.Event(alias.Name, projectName, op.Requestor(), nil)) } // Remove image from authorizer. err = s.Authorizer.DeleteImage(s.ShutdownCtx, projectName, imgInfo.Fingerprint) if err != nil { logger.Error("Failed to remove image from authorizer", logger.Ctx{"fingerprint": imgInfo.Fingerprint, "project": projectName, "error": err}) } s.Events.SendLifecycle(projectName, lifecycle.ImageDeleted.Event(imgInfo.Fingerprint, projectName, op.Requestor(), nil)) } var poolIDs []int64 var poolNames []string err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Delete the pool volumes. poolIDs, err = tx.GetPoolsWithImage(ctx, imgInfo.Fingerprint) if err != nil { return err } poolNames, err = tx.GetPoolNamesFromIDs(ctx, poolIDs) if err != nil { return err } return nil }) if err != nil { return err } for _, poolName := range poolNames { pool, err := storagePools.LoadByName(s, poolName) if err != nil { return fmt.Errorf("Error loading storage pool %q to delete image %q: %w", poolName, imgInfo.Fingerprint, err) } // Only perform the deletion of remote volumes on the server handling the request. if !isClusterNotification(r) || !pool.Driver().Info().Remote { err = pool.DeleteImage(imgInfo.Fingerprint, op) if err != nil { return fmt.Errorf("Error deleting image %q from storage pool %q: %w", imgInfo.Fingerprint, pool.Name(), err) } } } // Remove the database entry. if !isClusterNotification(r) { err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.DeleteImage(ctx, imgID) }) if err != nil { return fmt.Errorf("Error deleting image info from the database: %w", err) } } // Remove main image file from disk. imageDeleteFromDisk(imgInfo.Fingerprint) return nil } resources := map[string][]api.URL{} resources["images"] = []api.URL{*api.NewURL().Path(version.APIVersion, "images", imgInfo.Fingerprint)} op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.ImageDelete, resources, nil, do, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // Helper to delete an image file from the local images directory. func imageDeleteFromDisk(fingerprint string) { // Remove main image file. fname := internalUtil.VarPath("images", fingerprint) if util.PathExists(fname) { err := os.Remove(fname) if err != nil && !errors.Is(err, fs.ErrNotExist) { logger.Errorf("Error deleting image file %s: %s", fname, err) } } // Remove the rootfs file for the image. fname = internalUtil.VarPath("images", fingerprint) + ".rootfs" if util.PathExists(fname) { err := os.Remove(fname) if err != nil && !errors.Is(err, fs.ErrNotExist) { logger.Errorf("Error deleting image file %s: %s", fname, err) } } } func doImageGet(ctx context.Context, tx *db.ClusterTx, project, fingerprint string, public bool) (*api.Image, error) { filter := dbCluster.ImageFilter{Project: &project} if public { filter.Public = &public } _, imgInfo, err := tx.GetImageByFingerprintPrefix(ctx, fingerprint, filter) if err != nil { return nil, err } return imgInfo, nil } // imageValidSecret searches for an ImageToken operation running on any member in the default project that has an // images resource matching the specified fingerprint and the metadata secret field matches the specified secret. // If an operation is found it is returned and the operation is cancelled. Otherwise nil is returned if not found. func imageValidSecret(s *state.State, r *http.Request, projectName string, fingerprint string, secret string) (*api.Operation, error) { ops, err := operationsGetByType(s, r, projectName, operationtype.ImageToken) if err != nil { return nil, fmt.Errorf("Failed getting image token operations: %w", err) } for _, op := range ops { if op.Resources == nil { continue } opImages, ok := op.Resources["images"] if !ok { continue } if !util.StringPrefixInSlice(api.NewURL().Path(version.APIVersion, "images", fingerprint).String(), opImages) { continue } opSecret, ok := op.Metadata["secret"] if !ok { continue } if opSecret == secret { // Check if the operation is currently running (we allow access while expired). if op.Status == api.Running.String() { // Token is single-use, so cancel it now. err = operationCancel(s, r, projectName, op) if err != nil { return nil, fmt.Errorf("Failed to cancel operation %q: %w", op.ID, err) } } return op, nil } } return nil, nil } // swagger:operation GET /1.0/images/{fingerprint}?public images image_get_untrusted // // Get the public image // // Gets a specific public image. // // --- // produces: // - application/json // parameters: // - in: path // name: fingerprint // description: Fingerprint // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: secret // description: Secret token to retrieve a private image // type: string // example: RANDOM-STRING // responses: // "200": // description: Image // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Image" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/images/{fingerprint} images image_get // // Get the image // // Gets a specific image. // // --- // produces: // - application/json // parameters: // - in: path // name: fingerprint // description: Fingerprint // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Image // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Image" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func imageGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) fingerprint, err := url.PathUnescape(mux.Vars(r)["fingerprint"]) if err != nil { return response.SmartError(err) } // Get the image (expand partial fingerprints). var info *api.Image err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { info, err = doImageGet(ctx, tx, projectName, fingerprint, false) if err != nil { return err } return nil }) if err != nil { // As this is a publicly available function, override any 404 to a standard reply. // This avoids leaking information about the image or project existence. if response.IsNotFoundError(err) { return response.NotFound(fmt.Errorf("Image %q not found", fingerprint)) } return response.SmartError(err) } var userCanViewImage bool err = s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectImage(projectName, info.Fingerprint), auth.EntitlementCanView) if err == nil { userCanViewImage = true } else if !api.StatusErrorCheck(err, http.StatusForbidden) { return response.SmartError(err) } public := d.checkTrustedClient(r) != nil || !userCanViewImage secret := r.FormValue("secret") op, err := imageValidSecret(s, r, projectName, info.Fingerprint, secret) if err != nil { return response.SmartError(err) } if !info.Public && public && op == nil { return response.NotFound(fmt.Errorf("Image %q not found", fingerprint)) } etag := []any{info.Public, info.AutoUpdate, info.Properties} return response.SyncResponseETag(true, info, etag) } // swagger:operation PUT /1.0/images/{fingerprint} images image_put // // Update the image // // Updates the entire image definition. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: fingerprint // description: Fingerprint // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: image // description: Image configuration // required: true // schema: // $ref: "#/definitions/ImagePut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func imagePut(d *Daemon, r *http.Request) response.Response { s := d.State() // Get current value projectName := request.ProjectParam(r) fingerprint, err := url.PathUnescape(mux.Vars(r)["fingerprint"]) if err != nil { return response.SmartError(err) } var id int var info *api.Image err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { id, info, err = tx.GetImage(ctx, fingerprint, dbCluster.ImageFilter{Project: &projectName}) return err }) if err != nil { return response.SmartError(err) } // Validate ETag etag := []any{info.Public, info.AutoUpdate, info.Properties} err = localUtil.EtagCheck(r, etag) if err != nil { return response.PreconditionFailed(err) } req := api.ImagePut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Get ExpiresAt if !req.ExpiresAt.IsZero() { info.ExpiresAt = req.ExpiresAt } // Get profile IDs if req.Profiles == nil { req.Profiles = []string{"default"} } profileIds := make([]int64, len(req.Profiles)) err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { for i, profile := range req.Profiles { profileID, _, err := tx.GetProfile(ctx, projectName, profile) if response.IsNotFoundError(err) { return fmt.Errorf("Profile '%s' doesn't exist", profile) } else if err != nil { return err } profileIds[i] = profileID } return tx.UpdateImage(ctx, id, info.Filename, info.Size, req.Public, req.AutoUpdate, info.Architecture, info.CreatedAt, info.ExpiresAt, req.Properties, projectName, profileIds) }) if err != nil { if response.IsNotFoundError(err) { return response.BadRequest(err) } return response.SmartError(err) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(projectName, lifecycle.ImageUpdated.Event(info.Fingerprint, projectName, requestor, nil)) return response.EmptySyncResponse } // swagger:operation PATCH /1.0/images/{fingerprint} images image_patch // // Partially update the image // // Updates a subset of the image definition. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: fingerprint // description: Fingerprint // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: image // description: Image configuration // required: true // schema: // $ref: "#/definitions/ImagePut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func imagePatch(d *Daemon, r *http.Request) response.Response { s := d.State() // Get current value projectName := request.ProjectParam(r) fingerprint, err := url.PathUnescape(mux.Vars(r)["fingerprint"]) if err != nil { return response.SmartError(err) } var id int var info *api.Image err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { id, info, err = tx.GetImage(ctx, fingerprint, dbCluster.ImageFilter{Project: &projectName}) return err }) if err != nil { return response.SmartError(err) } // Validate ETag etag := []any{info.Public, info.AutoUpdate, info.Properties} err = localUtil.EtagCheck(r, etag) if err != nil { return response.PreconditionFailed(err) } body, err := io.ReadAll(r.Body) if err != nil { return response.InternalError(err) } rdr1 := io.NopCloser(bytes.NewBuffer(body)) rdr2 := io.NopCloser(bytes.NewBuffer(body)) reqRaw := jmap.Map{} err = json.NewDecoder(rdr1).Decode(&reqRaw) if err != nil { return response.BadRequest(err) } req := api.ImagePut{} err = json.NewDecoder(rdr2).Decode(&req) if err != nil { return response.BadRequest(err) } // Get AutoUpdate autoUpdate, err := reqRaw.GetBool("auto_update") if err == nil { info.AutoUpdate = autoUpdate } // Get Public public, err := reqRaw.GetBool("public") if err == nil { info.Public = public } // Get Properties _, ok := reqRaw["properties"] if ok { properties := req.Properties for k, v := range info.Properties { _, ok := req.Properties[k] if !ok { properties[k] = v } } info.Properties = properties } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateImage(ctx, id, info.Filename, info.Size, info.Public, info.AutoUpdate, info.Architecture, info.CreatedAt, info.ExpiresAt, info.Properties, "", nil) }) if err != nil { return response.SmartError(err) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(projectName, lifecycle.ImageUpdated.Event(info.Fingerprint, projectName, requestor, nil)) return response.EmptySyncResponse } // swagger:operation POST /1.0/images/aliases images images_aliases_post // // Add an image alias // // Creates a new image alias. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: image alias // description: Image alias // required: true // schema: // $ref: "#/definitions/ImageAliasesPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func imageAliasesPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) req := api.ImageAliasesPost{} err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Validation. if req.Name == "" || req.Target == "" { return response.BadRequest(errors.New("Alias name and target are required")) } err = validate.IsAPIName(req.Name, true) if err != nil { return response.BadRequest(fmt.Errorf("Invalid image alias name: %w", err)) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // This is just to see if the alias name already exists. _, _, err = tx.GetImageAlias(ctx, projectName, req.Name, true) if !response.IsNotFoundError(err) { if err != nil { return err } return api.StatusErrorf(http.StatusConflict, "Alias %q already exists", req.Name) } imgID, _, err := tx.GetImageByFingerprintPrefix(ctx, req.Target, dbCluster.ImageFilter{Project: &projectName}) if err != nil { return err } err = tx.CreateImageAlias(ctx, projectName, req.Name, imgID, req.Description) if err != nil { return err } return err }) if err != nil { return response.SmartError(err) } // Add the image alias to the authorizer. err = s.Authorizer.AddImageAlias(r.Context(), projectName, req.Name) if err != nil { logger.Error("Failed to add image alias to authorizer", logger.Ctx{"name": req.Name, "project": projectName, "error": err}) } requestor := request.CreateRequestor(r) lc := lifecycle.ImageAliasCreated.Event(req.Name, projectName, requestor, logger.Ctx{"target": req.Target}) s.Events.SendLifecycle(projectName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation GET /1.0/images/aliases images images_aliases_get // // Get the image aliases // // Returns a list of image aliases (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/images/aliases/foo", // "/1.0/images/aliases/bar1" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/images/aliases?recursion=1 images images_aliases_get_recursion1 // // Get the image aliases // // Returns a list of image aliases (structs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of image aliases // items: // $ref: "#/definitions/ImageAliasesEntry" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func imageAliasesGet(d *Daemon, r *http.Request) response.Response { projectName := request.ProjectParam(r) recursion := localUtil.IsRecursionRequest(r) s := d.State() userHasPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanView, auth.ObjectTypeImageAlias) if err != nil { return response.InternalError(fmt.Errorf("Failed to get a permission checker: %w", err)) } var responseStr []string var responseMap []api.ImageAliasesEntry err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { names, err := tx.GetImageAliases(ctx, projectName) if err != nil { return err } if recursion { responseMap = make([]api.ImageAliasesEntry, 0, len(names)) } else { responseStr = make([]string, 0, len(names)) } for _, name := range names { if !userHasPermission(auth.ObjectImageAlias(projectName, name)) { continue } if !recursion { responseStr = append(responseStr, api.NewURL().Path(version.APIVersion, "images", "aliases", name).String()) } else { _, alias, err := tx.GetImageAlias(ctx, projectName, name, true) if err != nil { continue } responseMap = append(responseMap, alias) } } return nil }) if err != nil { return response.SmartError(err) } if !recursion { return response.SyncResponse(true, responseStr) } return response.SyncResponse(true, responseMap) } // swagger:operation GET /1.0/images/aliases/{name}?public images image_alias_get_untrusted // // Get the public image alias // // Gets a specific public image alias. // This untrusted endpoint only works for aliases pointing to public images. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Alias name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Image alias // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/ImageAliasesEntry" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/images/aliases/{name} images image_alias_get // // Get the image alias // // Gets a specific image alias. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Alias name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Image alias // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/ImageAliasesEntry" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func imageAliasGet(d *Daemon, r *http.Request) response.Response { projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } s := d.State() var userCanViewImageAlias bool err = s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectImageAlias(projectName, name), auth.EntitlementCanView) if err == nil { userCanViewImageAlias = true } else if !api.StatusErrorCheck(err, http.StatusForbidden) { return response.SmartError(err) } public := d.checkTrustedClient(r) != nil || !userCanViewImageAlias var alias api.ImageAliasesEntry err = d.State().DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { _, alias, err = tx.GetImageAlias(ctx, projectName, name, !public) return err }) if err != nil { return response.SmartError(err) } return response.SyncResponseETag(true, alias, alias) } // swagger:operation DELETE /1.0/images/aliases/{name} images image_alias_delete // // Delete the image alias // // Deletes a specific image alias. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Alias name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func imageAliasDelete(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { _, _, err = tx.GetImageAlias(ctx, projectName, name, true) if err != nil { return err } err = tx.DeleteImageAlias(ctx, projectName, name) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // Remove image alias from authorizer. err = s.Authorizer.DeleteImageAlias(r.Context(), projectName, name) if err != nil { logger.Error("Failed to remove image alias from authorizer", logger.Ctx{"name": name, "project": projectName, "error": err}) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(projectName, lifecycle.ImageAliasDeleted.Event(name, projectName, requestor, nil)) return response.EmptySyncResponse } // swagger:operation PUT /1.0/images/aliases/{name} images images_aliases_put // // Update the image alias // // Updates the entire image alias configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Alias name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: image alias // description: Image alias configuration // required: true // schema: // $ref: "#/definitions/ImageAliasesEntryPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func imageAliasPut(d *Daemon, r *http.Request) response.Response { s := d.State() // Get current value projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } req := api.ImageAliasesEntryPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } if req.Target == "" { return response.BadRequest(errors.New("The target field is required")) } var imgAlias api.ImageAliasesEntry err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var imgAliasID int imgAliasID, imgAlias, err = tx.GetImageAlias(ctx, projectName, name, true) if err != nil { return err } // Validate ETag err = localUtil.EtagCheck(r, imgAlias) if err != nil { return err } imageID, _, err := tx.GetImageByFingerprintPrefix(ctx, req.Target, dbCluster.ImageFilter{Project: &projectName}) if err != nil { return err } err = tx.UpdateImageAlias(ctx, imgAliasID, imageID, req.Description) if err != nil { return err } return err }) if err != nil { return response.SmartError(err) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(projectName, lifecycle.ImageAliasUpdated.Event(imgAlias.Name, projectName, requestor, logger.Ctx{"target": req.Target})) return response.EmptySyncResponse } // swagger:operation PATCH /1.0/images/aliases/{name} images images_alias_patch // // Partially update the image alias // // Updates a subset of the image alias configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Alias name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: image alias // description: Image alias configuration // required: true // schema: // $ref: "#/definitions/ImageAliasesEntryPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func imageAliasPatch(d *Daemon, r *http.Request) response.Response { s := d.State() // Get current value projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } req := jmap.Map{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } var imgAlias api.ImageAliasesEntry err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var imgAliasID int imgAliasID, imgAlias, err = tx.GetImageAlias(ctx, projectName, name, true) if err != nil { return err } // Validate ETag err = localUtil.EtagCheck(r, imgAlias) if err != nil { return err } _, ok := req["target"] if ok { target, err := req.GetString("target") if err != nil { return api.StatusErrorf(http.StatusBadRequest, "%v", err) } imgAlias.Target = target } _, ok = req["description"] if ok { description, err := req.GetString("description") if err != nil { return api.StatusErrorf(http.StatusBadRequest, "%v", err) } imgAlias.Description = description } imageID, _, err := tx.GetImage(ctx, imgAlias.Target, dbCluster.ImageFilter{Project: &projectName}) if err != nil { return err } err = tx.UpdateImageAlias(ctx, imgAliasID, imageID, imgAlias.Description) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(projectName, lifecycle.ImageAliasUpdated.Event(imgAlias.Name, projectName, requestor, logger.Ctx{"target": imgAlias.Target})) return response.EmptySyncResponse } // swagger:operation POST /1.0/images/aliases/{name} images images_alias_post // // Rename the image alias // // Renames an existing image alias. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Alias name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: image alias // description: Image alias rename request // required: true // schema: // $ref: "#/definitions/ImageAliasesEntryPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func imageAliasPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } req := api.ImageAliasesEntryPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Validation. err = validate.IsAPIName(req.Name, true) if err != nil { return response.BadRequest(fmt.Errorf("Invalid image alias name: %w", err)) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // This is just to see if the alias name already exists. _, _, err := tx.GetImageAlias(ctx, projectName, req.Name, true) if !response.IsNotFoundError(err) { if err != nil { return err } return api.StatusErrorf(http.StatusConflict, "Alias %q already exists", req.Name) } imgAliasID, _, err := tx.GetImageAlias(ctx, projectName, name, true) if err != nil { return err } return tx.RenameImageAlias(ctx, imgAliasID, req.Name) }) if err != nil { return response.SmartError(err) } // Rename image alias in authorizer. err = s.Authorizer.RenameImageAlias(r.Context(), projectName, name, req.Name) if err != nil { logger.Error("Failed to rename image alias in authorizer", logger.Ctx{"old_name": name, "new_name": req.Name, "project": projectName}) } requestor := request.CreateRequestor(r) lc := lifecycle.ImageAliasRenamed.Event(req.Name, projectName, requestor, logger.Ctx{"old_name": name}) s.Events.SendLifecycle(projectName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation GET /1.0/images/{fingerprint}/export?public images image_export_get_untrusted // // Get the raw image file(s) // // Download the raw image file(s) of a public image from the server. // If the image is in split format, a multipart http transfer occurs. // // --- // produces: // - application/octet-stream // - multipart/form-data // parameters: // - in: path // name: fingerprint // description: Fingerprint // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: secret // description: Secret token to retrieve a private image // type: string // example: RANDOM-STRING // responses: // "200": // description: Raw image data // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/images/{fingerprint}/export images image_export_get // // Get the raw image file(s) // // Download the raw image file(s) from the server. // If the image is in split format, a multipart http transfer occurs. // // --- // produces: // - application/octet-stream // - multipart/form-data // parameters: // - in: path // name: fingerprint // description: Fingerprint // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Raw image data // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func imageExport(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) fingerprint, err := url.PathUnescape(mux.Vars(r)["fingerprint"]) if err != nil { return response.SmartError(err) } var imgInfo *api.Image err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Get the image (expand the fingerprint). _, imgInfo, err = tx.GetImage(ctx, fingerprint, dbCluster.ImageFilter{Project: &projectName}) return err }) if err != nil { // As this is a publicly available function, override any 404 to a standard reply. // This avoids leaking information about the image or project existence. if response.IsNotFoundError(err) { return response.NotFound(fmt.Errorf("Image %q not found", fingerprint)) } return response.SmartError(err) } // Access control. var userCanViewImage bool err = s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectImage(projectName, imgInfo.Fingerprint), auth.EntitlementCanView) if err == nil { userCanViewImage = true } else if !api.StatusErrorCheck(err, http.StatusForbidden) { return response.SmartError(err) } public := d.checkTrustedClient(r) != nil || !userCanViewImage secret := r.FormValue("secret") if r.RemoteAddr == "@dev_incus" { // /dev/incus API requires exact match if imgInfo.Fingerprint != fingerprint { return response.NotFound(fmt.Errorf("Image %q not found", fingerprint)) } if !imgInfo.Public && !imgInfo.Cached { return response.NotFound(fmt.Errorf("Image %q not found", fingerprint)) } } else { op, err := imageValidSecret(s, r, projectName, imgInfo.Fingerprint, secret) if err != nil { return response.SmartError(err) } if !imgInfo.Public && public && op == nil { return response.NotFound(fmt.Errorf("Image %q not found", fingerprint)) } } var address string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Check if the image is only available on another node. address, err = tx.LocateImage(ctx, imgInfo.Fingerprint) return err }) if err != nil { return response.SmartError(err) } if address != "" { // Forward the request to the other node client, err := cluster.Connect(address, s.Endpoints.NetworkCert(), s.ServerCert(), r, false) if err != nil { return response.SmartError(err) } return response.ForwardedResponse(client, r) } // Set image type header. headers := map[string]string{} headers["X-Incus-Type"] = "incus" if imgInfo.Properties != nil && imgInfo.Properties["type"] == "oci" { headers["X-Incus-Type"] = "oci" } imagePath := internalUtil.VarPath("images", imgInfo.Fingerprint) rootfsPath := imagePath + ".rootfs" _, ext, _, err := archive.DetectCompression(imagePath) if err != nil { ext = "" } filename := fmt.Sprintf("%s%s", imgInfo.Fingerprint, ext) if util.PathExists(rootfsPath) { files := make([]response.FileResponseEntry, 2) files[0].Identifier = "metadata" files[0].Path = imagePath files[0].Filename = "meta-" + filename // Recompute the extension for the root filesystem, it may use a different // compression algorithm than the metadata. _, ext, _, err = archive.DetectCompression(rootfsPath) if err != nil { ext = "" } filename = fmt.Sprintf("%s%s", imgInfo.Fingerprint, ext) if imgInfo.Type == "virtual-machine" { files[1].Identifier = "rootfs.img" } else { files[1].Identifier = "rootfs" } files[1].Path = rootfsPath files[1].Filename = filename return response.FileResponse(r, files, headers) } files := make([]response.FileResponseEntry, 1) files[0].Identifier = filename files[0].Path = imagePath files[0].Filename = filename requestor := request.CreateRequestor(r) s.Events.SendLifecycle(projectName, lifecycle.ImageRetrieved.Event(imgInfo.Fingerprint, projectName, requestor, nil)) return response.FileResponse(r, files, headers) } // swagger:operation POST /1.0/images/{fingerprint}/export images images_export_post // // Make the server push the image to a remote server // // Gets the server to connect to a remote server and push the image to it. // // --- // produces: // - application/json // parameters: // - in: path // name: fingerprint // description: Fingerprint // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: image // description: Image push request // required: true // schema: // $ref: "#/definitions/ImageExportPost" // responses: // "202": // $ref: "#/responses/Operation" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func imageExportPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) fingerprint, err := url.PathUnescape(mux.Vars(r)["fingerprint"]) if err != nil { return response.SmartError(err) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Check if the image exists _, _, err = tx.GetImage(ctx, fingerprint, dbCluster.ImageFilter{Project: &projectName}) return err }) if err != nil { return response.SmartError(err) } req := api.ImageExportPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { response.SmartError(err) } // Connect to the target and push the image args := &incus.ConnectionArgs{ TLSServerCert: req.Certificate, UserAgent: version.UserAgent, Proxy: s.Proxy, CachePath: s.OS.CacheDir, CacheExpiry: time.Hour, SkipGetEvents: true, SkipGetServer: true, } // Setup client remote, err := incus.ConnectIncus(req.Target, args) if err != nil { return response.SmartError(err) } var imageCreateOp incus.Operation run := func(op *operations.Operation) error { createArgs := &incus.ImageCreateArgs{} imageMetaPath := internalUtil.VarPath("images", fingerprint) imageRootfsPath := internalUtil.VarPath("images", fingerprint+".rootfs") metaFile, err := os.Open(imageMetaPath) if err != nil { return err } defer func() { _ = metaFile.Close() }() createArgs.MetaFile = metaFile createArgs.MetaName = filepath.Base(imageMetaPath) if util.PathExists(imageRootfsPath) { rootfsFile, err := os.Open(imageRootfsPath) if err != nil { return err } defer func() { _ = rootfsFile.Close() }() createArgs.RootfsFile = rootfsFile createArgs.RootfsName = filepath.Base(imageRootfsPath) } image := api.ImagesPost{ Filename: createArgs.MetaName, Source: &api.ImagesPostSource{ Fingerprint: fingerprint, Secret: req.Secret, Mode: "push", }, ImagePut: api.ImagePut{ Profiles: req.Profiles, }, } if req.Project != "" { remote = remote.UseProject(req.Project) } imageCreateOp, err = remote.CreateImage(image, createArgs) if err != nil { return err } opAPI := imageCreateOp.Get() var secret string val, ok := opAPI.Metadata["secret"] if ok { secret = val.(string) } opWaitAPI, _, err := remote.GetOperationWaitSecret(opAPI.ID, secret, -1) if err != nil { return err } if opWaitAPI.StatusCode != api.Success { return fmt.Errorf("Failed operation %q: %q", opWaitAPI.Status, opWaitAPI.Err) } s.Events.SendLifecycle(projectName, lifecycle.ImageRetrieved.Event(fingerprint, projectName, op.Requestor(), logger.Ctx{"target": req.Target})) return nil } op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.ImageDownload, nil, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // swagger:operation POST /1.0/images/{fingerprint}/secret images images_secret_post // // Generate secret for retrieval of the image by an untrusted client // // This generates a background operation including a secret one time key // in its metadata which can be used to fetch this image from an untrusted // client. // // --- // produces: // - application/json // parameters: // - in: path // name: fingerprint // description: Fingerprint // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "202": // $ref: "#/responses/Operation" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func imageSecret(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) fingerprint, err := url.PathUnescape(mux.Vars(r)["fingerprint"]) if err != nil { return response.SmartError(err) } var imgInfo *api.Image err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { _, imgInfo, err = tx.GetImage(ctx, fingerprint, dbCluster.ImageFilter{Project: &projectName}) return err }) if err != nil { return response.SmartError(err) } return createTokenResponse(s, r, projectName, imgInfo.Fingerprint, nil) } func imageImportFromNode(imagesDir string, client incus.InstanceServer, fingerprint string) error { // Prepare the temp files buildDir, err := os.MkdirTemp(imagesDir, "incus_build_") if err != nil { return fmt.Errorf("failed to create temporary directory for download: %w", err) } defer func() { _ = os.RemoveAll(buildDir) }() metaFile, err := os.CreateTemp(buildDir, "incus_tar_") if err != nil { return err } defer func() { _ = metaFile.Close() }() rootfsFile, err := os.CreateTemp(buildDir, "incus_tar_") if err != nil { return err } defer func() { _ = rootfsFile.Close() }() getReq := incus.ImageFileRequest{ MetaFile: io.ReadWriteSeeker(metaFile), RootfsFile: io.ReadWriteSeeker(rootfsFile), } getResp, err := client.GetImageFile(fingerprint, getReq) if err != nil { return err } // Truncate down to size if getResp.RootfsSize > 0 { err = rootfsFile.Truncate(getResp.RootfsSize) if err != nil { return err } } err = metaFile.Truncate(getResp.MetaSize) if err != nil { return err } if getResp.RootfsSize == 0 { // This is a unified image. rootfsPath := filepath.Join(imagesDir, fingerprint) err := internalUtil.FileMove(metaFile.Name(), rootfsPath) if err != nil { return err } } else { // This is a split image. metaPath := filepath.Join(imagesDir, fingerprint) rootfsPath := filepath.Join(imagesDir, fingerprint+".rootfs") err := internalUtil.FileMove(metaFile.Name(), metaPath) if err != nil { return nil } err = internalUtil.FileMove(rootfsFile.Name(), rootfsPath) if err != nil { return nil } } return nil } // swagger:operation POST /1.0/images/{fingerprint}/refresh images images_refresh_post // // Refresh an image // // This causes the server to check the image source server for an updated // version of the image and if available to refresh the local copy with the // new version. // // --- // produces: // - application/json // parameters: // - in: path // name: fingerprint // description: Fingerprint // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "202": // $ref: "#/responses/Operation" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func imageRefresh(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) fingerprint, err := url.PathUnescape(mux.Vars(r)["fingerprint"]) if err != nil { return response.SmartError(err) } var imageID int var imageInfo *api.Image err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { imageID, imageInfo, err = tx.GetImage(ctx, fingerprint, dbCluster.ImageFilter{Project: &projectName}) return err }) if err != nil { return response.SmartError(err) } // Begin background operation run := func(op *operations.Operation) error { var nodes []string err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { nodes, err = tx.GetNodesWithImageAndAutoUpdate(ctx, fingerprint, true) return err }) if err != nil { return fmt.Errorf("Error getting cluster members for refreshing image %q in project %q: %w", fingerprint, projectName, err) } newImage, err := autoUpdateImage(context.TODO(), s, op, imageID, imageInfo, projectName, true) if err != nil { return fmt.Errorf("Failed to update image %q in project %q: %w", fingerprint, projectName, err) } if newImage != nil { if len(nodes) > 1 { err := distributeImage(context.TODO(), s, nodes, fingerprint, newImage) if err != nil { return fmt.Errorf("Failed to distribute new image %q: %w", newImage.Fingerprint, err) } } err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Remove the database entry for the image after distributing to cluster members. return tx.DeleteImage(ctx, imageID) }) if err != nil { logger.Error("Error deleting old image from database", logger.Ctx{"err": err, "fingerprint": fingerprint, "ID": imageID}) } } return err } op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.ImageRefresh, nil, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } func autoSyncImagesTask(s *state.State) (task.Func, task.Schedule) { f := func(ctx context.Context) { // In order to only have one task operation executed per image when syncing the images // across the cluster, only leader node can launch the task, no others. localClusterAddress := s.LocalConfig.ClusterAddress() leader, err := s.Cluster.LeaderAddress() if err != nil { if errors.Is(err, cluster.ErrNodeIsNotClustered) { return // No error if not clustered. } logger.Error("Failed to get leader cluster member address", logger.Ctx{"err": err}) return } if localClusterAddress != leader { logger.Debug("Skipping image synchronization task since we're not leader") return } opRun := func(op *operations.Operation) error { return autoSyncImages(ctx, s) } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.ImagesSynchronize, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating image synchronization operation", logger.Ctx{"err": err}) return } logger.Debug("Acquiring image task lock") imageTaskMu.Lock() defer imageTaskMu.Unlock() logger.Debug("Acquired image task lock") logger.Info("Synchronizing images across the cluster") err = op.Start() if err != nil { logger.Error("Failed starting image synchronization operation", logger.Ctx{"err": err}) return } err = op.Wait(ctx) if err != nil { logger.Error("Failed synchronizing images", logger.Ctx{"err": err}) return } logger.Info("Done synchronizing images across the cluster") } return f, task.Hourly() } func autoSyncImages(ctx context.Context, s *state.State) error { var imageProjectInfo map[string][]string err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get all images. imageProjectInfo, err = tx.GetImages(ctx) return err }) if err != nil { return fmt.Errorf("Failed to query image fingerprints: %w", err) } for fingerprint, projects := range imageProjectInfo { ch := make(chan error) go func(projectName string, fingerprint string) { err := imageSyncBetweenNodes(ctx, s, nil, projectName, fingerprint) if err != nil { logger.Error("Failed to synchronize images", logger.Ctx{"err": err, "project": projectName, "fingerprint": fingerprint}) } ch <- nil }(projects[0], fingerprint) select { case <-ctx.Done(): return nil case <-ch: } } return nil } func imageSyncBetweenNodes(ctx context.Context, s *state.State, r *http.Request, project string, fingerprint string) error { logger.Info("Syncing image to members started", logger.Ctx{"fingerprint": fingerprint, "project": project}) defer logger.Info("Syncing image to members finished", logger.Ctx{"fingerprint": fingerprint, "project": project}) var desiredSyncNodeCount int64 var syncNodeAddresses []string err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { desiredSyncNodeCount = s.GlobalConfig.ImagesMinimalReplica() // -1 means that we want to replicate the image on all nodes if desiredSyncNodeCount == -1 { nodesCount, err := tx.GetNodesCount(ctx) if err != nil { return fmt.Errorf("Failed to get the number of nodes: %w", err) } desiredSyncNodeCount = int64(nodesCount) } var err error // Check how many nodes already have this image syncNodeAddresses, err = tx.GetNodesWithImage(ctx, fingerprint) if err != nil { return fmt.Errorf("Failed to get nodes for the image synchronization: %w", err) } return nil }) if err != nil { return err } // If none of the nodes have the image, there's nothing to sync. if len(syncNodeAddresses) == 0 { logger.Info("No members have image, nothing to do", logger.Ctx{"fingerprint": fingerprint, "project": project}) return nil } nodeCount := desiredSyncNodeCount - int64(len(syncNodeAddresses)) if nodeCount <= 0 { logger.Info("Sufficient members have image", logger.Ctx{"fingerprint": fingerprint, "project": project, "desiredSyncCount": desiredSyncNodeCount, "syncedCount": len(syncNodeAddresses)}) return nil } // Pick a random node from that slice as the source. syncNodeAddress := syncNodeAddresses[rand.Intn(len(syncNodeAddresses))] source, err := cluster.Connect(syncNodeAddress, s.Endpoints.NetworkCert(), s.ServerCert(), r, true) if err != nil { return fmt.Errorf("Failed to connect to source node for image synchronization: %w", err) } source = source.UseProject(project) var image *api.Image err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Get the image. _, image, err = tx.GetImage(ctx, fingerprint, dbCluster.ImageFilter{Project: &project}) return err }) if err != nil { return fmt.Errorf("Failed to get image: %w", err) } // Set up the image download request. req := api.ImagesPost{ Source: &api.ImagesPostSource{ Fingerprint: image.Fingerprint, Mode: "pull", Type: "image", Project: project, }, } // Replicate on as many nodes as needed. for range int(nodeCount) { var addresses []string err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Get a list of nodes that do not have the image. addresses, err = tx.GetNodesWithoutImage(ctx, fingerprint) return err }) if err != nil { return fmt.Errorf("Failed to get nodes for the image synchronization: %w", err) } if len(addresses) <= 0 { logger.Info("All members have image", logger.Ctx{"fingerprint": fingerprint, "project": project}) return nil } // Pick a random node from that slice as the target. targetNodeAddress := addresses[rand.Intn(len(addresses))] client, err := cluster.Connect(targetNodeAddress, s.Endpoints.NetworkCert(), s.ServerCert(), r, true) if err != nil { return fmt.Errorf("Failed to connect node for image synchronization: %w", err) } // Select the right project. client = client.UseProject(project) // Copy the image to the target server. logger.Info("Copying image to member", logger.Ctx{"fingerprint": fingerprint, "address": targetNodeAddress, "project": project}) op, err := client.CreateImage(req, nil) if err != nil { return fmt.Errorf("Failed to copy image to %q: %w", targetNodeAddress, err) } err = op.Wait() if err != nil { return err } } return nil } func createTokenResponse(s *state.State, r *http.Request, projectName string, fingerprint string, metadata jmap.Map) response.Response { secret, err := internalUtil.RandomHexString(32) if err != nil { return response.InternalError(err) } meta := jmap.Map{} maps.Copy(meta, metadata) meta["secret"] = secret resources := map[string][]api.URL{} resources["images"] = []api.URL{*api.NewURL().Path(version.APIVersion, "images", fingerprint)} op, err := operations.OperationCreate(s, projectName, operations.OperationClassToken, operationtype.ImageToken, resources, meta, nil, nil, nil, r) if err != nil { return response.InternalError(err) } s.Events.SendLifecycle(projectName, lifecycle.ImageSecretCreated.Event(fingerprint, projectName, op.Requestor(), nil)) return operations.OperationResponse(op) } incus-7.0.0/cmd/incusd/instance.go000066400000000000000000000657621517523235500170520ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "net/http" "os" "path/filepath" "strings" "sync" "time" "github.com/kballard/go-shellquote" ociSpecs "github.com/opencontainers/runtime-spec/specs-go" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/instance/operationlock" "github.com/lxc/incus/v7/internal/server/locking" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" "github.com/lxc/incus/v7/internal/server/task" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" ) // Helper functions // instanceCreateAsEmpty creates an empty instance. func instanceCreateAsEmpty(s *state.State, args db.InstanceArgs, op *operations.Operation) (instance.Instance, error) { reverter := revert.New() defer reverter.Fail() // Create the instance record. inst, instOp, cleanup, err := instance.CreateInternal(s, args, op, true, true, false) if err != nil { return nil, fmt.Errorf("Failed creating instance record: %w", err) } reverter.Add(cleanup) defer instOp.Done(err) pool, err := storagePools.LoadByInstance(s, inst) if err != nil { return nil, fmt.Errorf("Failed loading instance storage pool: %w", err) } err = pool.CreateInstance(inst, nil) if err != nil { return nil, fmt.Errorf("Failed creating instance: %w", err) } reverter.Add(func() { _ = inst.Delete(true, true) }) err = inst.UpdateBackupFile() if err != nil { return nil, err } reverter.Success() return inst, nil } // instanceImageTransfer transfers an image from another cluster node. func instanceImageTransfer(s *state.State, r *http.Request, projectName string, hash string, nodeAddress string) error { logger.Debugf("Transferring image %q from node %q", hash, nodeAddress) client, err := cluster.Connect(nodeAddress, s.Endpoints.NetworkCert(), s.ServerCert(), r, false) if err != nil { return err } client = client.UseProject(projectName) err = imageImportFromNode(filepath.Join(s.OS.VarDir, "images"), client, hash) if err != nil { return err } return nil } func ensureImageIsLocallyAvailable(ctx context.Context, s *state.State, r *http.Request, img *api.Image, projectName string) error { // Check if the image is available locally or it's on another member. // Ensure we are the only ones operating on this image. Otherwise another instance created at the same // time may also arrive at the conclusion that the image doesn't exist on this cluster member and then // think it needs to download the image and store the record in the database as well, which will lead to // duplicate record errors. unlock, err := imageOperationLock(ctx, img.Fingerprint) if err != nil { return err } defer unlock() var memberAddress string err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { memberAddress, err = tx.LocateImage(ctx, img.Fingerprint) return err }) if err != nil { return fmt.Errorf("Failed locating image %q: %w", img.Fingerprint, err) } if memberAddress != "" { // The image is available from another node, let's try to import it. err = instanceImageTransfer(s, r, projectName, img.Fingerprint, memberAddress) if err != nil { return fmt.Errorf("Failed transferring image %q from %q: %w", img.Fingerprint, memberAddress, err) } err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // As the image record already exists in the project, just add the node ID to the image. return tx.AddImageToLocalNode(ctx, projectName, img.Fingerprint) }) if err != nil { return fmt.Errorf("Failed adding transferred image %q record to local cluster member: %w", img.Fingerprint, err) } } return nil } // instanceCreateFromImage creates an instance from a rootfs image. func instanceCreateFromImage(ctx context.Context, s *state.State, img *api.Image, args db.InstanceArgs, op *operations.Operation) error { reverter := revert.New() defer reverter.Fail() // Validate the type of the image matches the type of the instance. imgType, err := instancetype.New(img.Type) if err != nil { return err } if imgType != args.Type { return fmt.Errorf("Requested image's type %q doesn't match instance type %q", imgType, args.Type) } // Set the "image.*" keys. if img.Properties != nil { for k, v := range img.Properties { args.Config[fmt.Sprintf("image.%s", k)] = v } } // Set the BaseImage field (regardless of previous value). args.BaseImage = img.Fingerprint // Create the instance. inst, instOp, cleanup, err := instance.CreateInternal(s, args, op, true, true, false) if err != nil { return fmt.Errorf("Failed creating instance record: %w", err) } reverter.Add(cleanup) defer instOp.Done(nil) err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { err = tx.UpdateImageLastUseDate(ctx, args.Project, img.Fingerprint, time.Now().UTC()) if err != nil { return fmt.Errorf("Error updating image last use date: %w", err) } return nil }) if err != nil { return err } pool, err := storagePools.LoadByInstance(s, inst) if err != nil { return fmt.Errorf("Failed loading instance storage pool: %w", err) } // Lock this operation to ensure that concurrent image operations don't conflict. // Other operations will wait for this one to finish. unlock, err := imageOperationLock(ctx, img.Fingerprint) if err != nil { return err } defer unlock() err = pool.CreateInstanceFromImage(inst, img.Fingerprint, op) if err != nil { return fmt.Errorf("Failed creating instance from image: %w", err) } reverter.Add(func() { _ = inst.Delete(true, true) }) // If dealing with an OCI image, parse the configuration. if args.Type == instancetype.Container && inst.LocalConfig()["image.type"] == "oci" { // Reset the config to the post-generation one. args.Config = inst.LocalConfig() expandedConfig := inst.ExpandedConfig() // Mount the instance. _, err = pool.MountInstance(inst, nil) if err != nil { return err } // Parse the OCI config. data, err := os.ReadFile(filepath.Join(inst.Path(), "config.json")) if err != nil { return err } var config ociSpecs.Spec err = json.Unmarshal([]byte(data), &config) if err != nil { return err } // Unmount the instance. err = pool.UnmountInstance(inst, nil) if err != nil { return err } // Update the config for the environment variables. args.Config["volatile.container.oci"] = "true" for _, env := range config.Process.Env { fields := strings.SplitN(env, "=", 2) if len(fields) != 2 { return fmt.Errorf("Bad OCI environment variable: %s", env) } key := fmt.Sprintf("environment.%s", fields[0]) value := fields[1] _, ok := expandedConfig[key] if !ok { args.Config[key] = value } } // Set the entrypoint configuration options. if len(config.Process.Args) > 0 && expandedConfig["oci.entrypoint"] == "" { args.Config["oci.entrypoint"] = shellquote.Join(config.Process.Args...) } if config.Process.Cwd != "" && expandedConfig["oci.cwd"] == "" { args.Config["oci.cwd"] = config.Process.Cwd } if expandedConfig["oci.uid"] == "" { args.Config["oci.uid"] = fmt.Sprintf("%d", config.Process.User.UID) } if expandedConfig["oci.gid"] == "" { args.Config["oci.gid"] = fmt.Sprintf("%d", config.Process.User.GID) } err = inst.Update(args, false) if err != nil { return err } } err = inst.UpdateBackupFile() if err != nil { return err } reverter.Success() return nil } func instanceRebuildFromImage(ctx context.Context, s *state.State, r *http.Request, inst instance.Instance, img *api.Image, op *operations.Operation) error { // Validate the type of the image matches the type of the instance. imgType, err := instancetype.New(img.Type) if err != nil { return err } if imgType != inst.Type() { return fmt.Errorf("Requested image's type %q doesn't match instance type %q", imgType, inst.Type()) } err = ensureImageIsLocallyAvailable(ctx, s, r, img, inst.Project().Name) if err != nil { return err } err = inst.Rebuild(img, op) if err != nil { return fmt.Errorf("Failed rebuilding instance from image: %w", err) } return nil } func instanceRebuildFromEmpty(inst instance.Instance, op *operations.Operation) error { err := inst.Rebuild(nil, op) // Rebuild as empty. if err != nil { return fmt.Errorf("Failed rebuilding as an empty instance: %w", err) } return nil } // instanceCreateAsCopyOpts options for copying an instance. type instanceCreateAsCopyOpts struct { sourceInstance instance.Instance // Source instance. targetInstance db.InstanceArgs // Configuration for new instance. instanceOnly bool // Only copy the instance and not it's snapshots. refresh bool // Refresh an existing target instance. refreshExcludeOlder bool // During refresh, exclude source snapshots earlier than latest target snapshot applyTemplateTrigger bool // Apply deferred TemplateTriggerCopy. allowInconsistent bool // Ignore some copy errors } // instanceCreateAsCopy create a new instance by copying from an existing instance. func instanceCreateAsCopy(s *state.State, opts instanceCreateAsCopyOpts, op *operations.Operation) (instance.Instance, error) { var inst instance.Instance var instOp *operationlock.InstanceOperation var err error var cleanup revert.Hook reverter := revert.New() defer reverter.Fail() if opts.refresh { // Load the target instance. inst, err = instance.LoadByProjectAndName(s, opts.targetInstance.Project, opts.targetInstance.Name) if err != nil { opts.refresh = false // Instance doesn't exist, so switch to copy mode. } } // If we are not in refresh mode, then create a new instance as we are in copy mode. if !opts.refresh { // Create the instance. inst, instOp, cleanup, err = instance.CreateInternal(s, opts.targetInstance, op, true, false, true) if err != nil { return nil, fmt.Errorf("Failed creating instance record: %w", err) } reverter.Add(cleanup) } else { instOp, err = inst.LockExclusive() if err != nil { return nil, fmt.Errorf("Failed getting exclusive access to target instance: %w", err) } } defer instOp.Done(err) // At this point we have already figured out the instance's root disk device so we can simply retrieve it // from the expanded devices. instRootDiskDeviceKey, instRootDiskDevice, err := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if err != nil { return nil, err } var snapshots []instance.Instance if !opts.instanceOnly { if opts.refresh { // Compare snapshots. sourceSnaps, err := opts.sourceInstance.Snapshots() if err != nil { return nil, err } sourceSnapshotComparable := make([]storagePools.ComparableSnapshot, 0, len(sourceSnaps)) for _, sourceSnap := range sourceSnaps { _, sourceSnapName, _ := api.GetParentAndSnapshotName(sourceSnap.Name()) sourceSnapshotComparable = append(sourceSnapshotComparable, storagePools.ComparableSnapshot{ Name: sourceSnapName, CreationDate: sourceSnap.CreationDate(), }) } targetSnaps, err := inst.Snapshots() if err != nil { return nil, err } targetSnapshotsComparable := make([]storagePools.ComparableSnapshot, 0, len(targetSnaps)) for _, targetSnap := range targetSnaps { _, targetSnapName, _ := api.GetParentAndSnapshotName(targetSnap.Name()) targetSnapshotsComparable = append(targetSnapshotsComparable, storagePools.ComparableSnapshot{ Name: targetSnapName, CreationDate: targetSnap.CreationDate(), }) } syncSourceSnapshotIndexes, deleteTargetSnapshotIndexes := storagePools.CompareSnapshots(sourceSnapshotComparable, targetSnapshotsComparable, opts.refreshExcludeOlder) // Delete extra snapshots first. for _, deleteTargetSnapIndex := range deleteTargetSnapshotIndexes { err := targetSnaps[deleteTargetSnapIndex].Delete(true, true) if err != nil { return nil, err } } // Only send the snapshots that need updating. snapshots = make([]instance.Instance, 0, len(syncSourceSnapshotIndexes)) for _, syncSourceSnapIndex := range syncSourceSnapshotIndexes { snapshots = append(snapshots, sourceSnaps[syncSourceSnapIndex]) } } else { // Get snapshots of source instance. snapshots, err = opts.sourceInstance.Snapshots() if err != nil { return nil, err } } for _, srcSnap := range snapshots { snapLocalDevices := srcSnap.LocalDevices().Clone() // Load snap root disk from expanded devices (in case it doesn't have its own root disk). snapExpandedRootDiskDevKey, snapExpandedRootDiskDev, err := internalInstance.GetRootDiskDevice(srcSnap.ExpandedDevices().CloneNative()) if err == nil { // If the expanded devices has a root disk, but its pool doesn't match our new // parent instance's pool, then either modify the device if it is local or add a // new one to local devices if its coming from the profiles. if snapExpandedRootDiskDev["pool"] != instRootDiskDevice["pool"] { localRootDiskDev, found := snapLocalDevices[snapExpandedRootDiskDevKey] if found { // Modify exist local device's pool. localRootDiskDev["pool"] = instRootDiskDevice["pool"] snapLocalDevices[snapExpandedRootDiskDevKey] = localRootDiskDev } else { // Add a new local device using parent instance's pool. snapLocalDevices[instRootDiskDeviceKey] = map[string]string{ "type": "disk", "path": "/", "pool": instRootDiskDevice["pool"], } } } } else if errors.Is(err, internalInstance.ErrNoRootDisk) { // If no root disk defined in either local devices or profiles, then add one to the // snapshot local devices using the same device name from the parent instance. snapLocalDevices[instRootDiskDeviceKey] = map[string]string{ "type": "disk", "path": "/", "pool": instRootDiskDevice["pool"], } } else { //nolint:staticcheck // (keep the empty branch for the comment) // Snapshot has multiple root disk devices, we can't automatically fix this so // leave alone so we don't prevent copy. } fields := strings.SplitN(srcSnap.Name(), internalInstance.SnapshotDelimiter, 2) newSnapName := fmt.Sprintf("%s/%s", inst.Name(), fields[1]) snapInstArgs := db.InstanceArgs{ Architecture: srcSnap.Architecture(), Config: srcSnap.LocalConfig(), Type: opts.sourceInstance.Type(), Snapshot: true, Devices: snapLocalDevices, Description: srcSnap.Description(), Ephemeral: srcSnap.IsEphemeral(), Name: newSnapName, Profiles: srcSnap.Profiles(), Project: opts.targetInstance.Project, ExpiryDate: srcSnap.ExpiryDate(), CreationDate: srcSnap.CreationDate(), } // Create the snapshots. _, snapInstOp, cleanup, err := instance.CreateInternal(s, snapInstArgs, op, true, false, false) if err != nil { return nil, fmt.Errorf("Failed creating instance snapshot record %q: %w", newSnapName, err) } reverter.Add(cleanup) defer snapInstOp.Done(err) } } // Copy the storage volume. pool, err := storagePools.LoadByInstance(s, inst) if err != nil { return nil, fmt.Errorf("Failed loading instance storage pool: %w", err) } if opts.refresh { err = pool.RefreshInstance(inst, opts.sourceInstance, snapshots, opts.allowInconsistent, op) if err != nil { return nil, fmt.Errorf("Refresh instance: %w", err) } } else { err = pool.CreateInstanceFromCopy(inst, opts.sourceInstance, !opts.instanceOnly, opts.allowInconsistent, op) if err != nil { return nil, fmt.Errorf("Create instance from copy: %w", err) } reverter.Add(func() { _ = inst.Delete(true, true) }) if opts.applyTemplateTrigger { // Trigger the templates on next start. err = inst.DeferTemplateApply(instance.TemplateTriggerCopy) if err != nil { return nil, err } } } err = inst.UpdateBackupFile() if err != nil { return nil, err } reverter.Success() return inst, nil } // Load all instances of this nodes under the given project. func instanceLoadNodeProjectAll(ctx context.Context, s *state.State, projectName string) ([]instance.Instance, error) { var err error var instances []instance.Instance filter := dbCluster.InstanceFilter{Project: &projectName} if s.ServerName != "" { filter.Node = &s.ServerName } err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.InstanceList(ctx, func(dbInst db.InstanceArgs, p api.Project) error { inst, err := instance.Load(s, dbInst, p) if err != nil { return fmt.Errorf("Failed loading instance %q in project %q: %w", dbInst.Name, dbInst.Project, err) } instances = append(instances, inst) return nil }, filter) }) if err != nil { return nil, err } return instances, nil } func autoCreateInstanceSnapshots(ctx context.Context, s *state.State, instances []instance.Instance) error { // Make the snapshots. for _, inst := range instances { err := ctx.Err() if err != nil { return err } l := logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) snapshotName, err := instance.NextSnapshotName(s, inst, "snap%d") if err != nil { l.Error("Error retrieving next snapshot name", logger.Ctx{"err": err}) return err } expiry, err := internalInstance.GetExpiry(time.Now(), inst.ExpandedConfig()["snapshots.expiry"]) if err != nil { l.Error("Error getting snapshots.expiry date") return err } err = inst.Snapshot(snapshotName, expiry, false) if err != nil { l.Error("Error creating snapshot", logger.Ctx{"snapshot": snapshotName, "err": err}) return err } } return nil } var instSnapshotsPruneRunning = sync.Map{} func pruneExpiredInstanceSnapshots(ctx context.Context, snapshots []instance.Instance) error { // Find snapshots to delete for _, snapshot := range snapshots { err := ctx.Err() if err != nil { return err } _, loaded := instSnapshotsPruneRunning.LoadOrStore(snapshot.ID(), struct{}{}) if loaded { continue // Deletion of this snapshot is already running, skip. } err = snapshot.Delete(true, true) instSnapshotsPruneRunning.Delete(snapshot.ID()) if err != nil { return fmt.Errorf("Failed to delete expired instance snapshot %q in project %q: %w", snapshot.Name(), snapshot.Project().Name, err) } logger.Debug("Deleted instance snapshot", logger.Ctx{"project": snapshot.Project().Name, "snapshot": snapshot.Name()}) } return nil } func pruneExpiredAndAutoCreateInstanceSnapshotsTask(d *Daemon) (task.Func, task.Schedule) { // `f` creates new scheduled instance snapshots and then, prune the expired ones f := func(ctx context.Context) { s := d.State() var instances, expiredSnapshotInstances []instance.Instance // Get list of expired instance snapshots for this local member. err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { expiredSnaps, err := tx.GetLocalExpiredInstanceSnapshots(ctx) if err != nil { return fmt.Errorf("Failed loading expired instance snapshots: %w", err) } if len(expiredSnaps) > 0 { expiredSnapshots := make([]dbCluster.Instance, 0, len(expiredSnaps)) parents := make(map[string]*dbCluster.Instance) // Enrich expired snapshot list with info from parent (opportunistically loading // the parent info from the DB if not already loaded). for _, snapshot := range expiredSnaps { parentInstanceKey := snapshot.Project + "/" + snapshot.Instance parent, ok := parents[parentInstanceKey] if !ok { parent, err = dbCluster.GetInstance(ctx, tx.Tx(), snapshot.Project, snapshot.Instance) if err != nil { return fmt.Errorf("Failed loading instance %q (project %q): %w", snapshot.Instance, snapshot.Project, err) } parents[parentInstanceKey] = parent } expiredSnapshots = append(expiredSnapshots, snapshot.ToInstance(parent.Name, parent.Node, parent.Type, parent.Architecture)) } // Load expired snapshot configs. snapshotArgs, err := tx.InstancesToInstanceArgs(ctx, true, expiredSnapshots...) if err != nil { return fmt.Errorf("Failed loading expired instance snapshots info: %w", err) } projects := make(map[string]*api.Project) expiredSnapshotInstances = make([]instance.Instance, 0) for _, snapshotArg := range snapshotArgs { // Load project if not already loaded. p, found := projects[snapshotArg.Project] if !found { dbProject, err := dbCluster.GetProject(ctx, tx.Tx(), snapshotArg.Project) if err != nil { return fmt.Errorf("Failed loading project %q: %w", snapshotArg.Project, err) } p, err = dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading project %q config: %w", snapshotArg.Project, err) } projects[snapshotArg.Project] = p } inst, err := instance.Load(s, snapshotArg, *p) if err != nil { return fmt.Errorf("Failed loading instance snapshot %q (project %q) for prune task: %w", snapshotArg.Name, snapshotArg.Project, err) } logger.Debug("Scheduling instance snapshot expiry", logger.Ctx{"instance": inst.Name(), "project": inst.Project().Name}) expiredSnapshotInstances = append(expiredSnapshotInstances, inst) } } return nil }) if err != nil { logger.Error("Failed getting instance snapshot expiry info", logger.Ctx{"err": err}) return } // Get list of instances on the local member that are due to have snapshots creating. filter := dbCluster.InstanceFilter{Node: &s.ServerName} err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.InstanceList(ctx, func(dbInst db.InstanceArgs, p api.Project) error { err = project.AllowSnapshotCreation(&p) if err != nil { return nil } inst, err := instance.Load(s, dbInst, p) if err != nil { return fmt.Errorf("Failed loading instance %q (project %q) for snapshot task: %w", dbInst.Name, dbInst.Project, err) } // Check if instance has snapshot schedule enabled. schedule, ok := inst.ExpandedConfig()["snapshots.schedule"] if !ok || schedule == "" { return nil } // Check if snapshot is scheduled. if !snapshotIsScheduledNow(schedule, int64(inst.ID())) { return nil } // If snapshot should only be taken if instance is running, check if running. if util.IsFalseOrEmpty(inst.ExpandedConfig()["snapshots.schedule.stopped"]) && !inst.IsRunning() { return nil } logger.Debug("Scheduling auto instance snapshot", logger.Ctx{"instance": inst.Name(), "project": inst.Project().Name}) instances = append(instances, inst) return nil }, filter) }) if err != nil { logger.Error("Failed getting instance snapshot schedule info", logger.Ctx{"err": err}) return } // Handle snapshot expiry first before creating new ones to reduce the chances of running out of // disk space. if len(expiredSnapshotInstances) > 0 { opRun := func(op *operations.Operation) error { return pruneExpiredInstanceSnapshots(ctx, expiredSnapshotInstances) } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.SnapshotsExpire, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating instance snapshots expiry operation", logger.Ctx{"err": err}) } else { logger.Info("Pruning expired instance snapshots") err = op.Start() if err != nil { logger.Error("Failed starting instance snapshots expiry operation", logger.Ctx{"err": err}) } else { err = op.Wait(ctx) if err != nil { logger.Error("Failed pruning instance snapshots", logger.Ctx{"err": err}) } else { logger.Info("Done pruning expired instance snapshots") } } } } // Handle snapshot auto creation. if len(instances) > 0 { opRun := func(op *operations.Operation) error { return autoCreateInstanceSnapshots(ctx, s, instances) } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.SnapshotCreate, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating scheduled instance snapshot operation", logger.Ctx{"err": err}) } else { logger.Info("Creating scheduled instance snapshots") err = op.Start() if err != nil { logger.Error("Failed starting scheduled instance snapshot operation", logger.Ctx{"err": err}) } else { err = op.Wait(ctx) if err != nil { logger.Error("Failed scheduled instance snapshots", logger.Ctx{"err": err}) } else { logger.Info("Done creating scheduled instance snapshots") } } } } } first := true schedule := func() (time.Duration, error) { interval := time.Minute if first { first = false return interval, task.ErrSkip } return interval, nil } return f, schedule } // getSourceImageFromInstanceSource returns the image to use for an instance source. func getSourceImageFromInstanceSource(ctx context.Context, s *state.State, tx *db.ClusterTx, project string, source api.InstanceSource, imageRef *string, instType string) (*api.Image, error) { // Resolve the image. sourceImageRefUpdate, err := instance.ResolveImage(ctx, tx, project, source) if err != nil { return nil, err } *imageRef = sourceImageRefUpdate sourceImageHash := *imageRef // If a remote server is being used, check whether we have a cached image for the alias. // If so then use the cached image fingerprint for loading the cache image profiles. // As its possible for a remote cached image to have its profiles modified after download. if source.Server != "" { for _, architecture := range s.OS.Architectures { cachedFingerprint, err := tx.GetCachedImageSourceFingerprint(ctx, source.Server, source.Protocol, *imageRef, instType, architecture) if err == nil && cachedFingerprint != sourceImageHash { sourceImageHash = cachedFingerprint break } } } // Check if image has an entry in the database. _, sourceImage, err := tx.GetImageByFingerprintPrefix(ctx, sourceImageHash, dbCluster.ImageFilter{Project: &project}) if err != nil { return nil, err } return sourceImage, nil } // instanceOperationLock acquires a lock for operating on an instance and returns the unlock function. func instanceOperationLock(ctx context.Context, projectName string, instanceName string) (locking.UnlockFunc, error) { l := logger.AddContext(logger.Ctx{"project": projectName, "instance": instanceName}) l.Debug("Acquiring lock for instance") defer l.Debug("Lock acquired for instance") return locking.Lock(ctx, fmt.Sprintf("InstanceOperation_%s", project.Instance(projectName, instanceName))) } incus-7.0.0/cmd/incusd/instance_access.go000066400000000000000000000042101517523235500203500ustar00rootroot00000000000000package main import ( "context" "errors" "net/http" "net/url" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" ) // swagger:operation GET /1.0/instances/{name}/access instances instance_access // // Get who has access to an instance // // Gets the access information for the instance. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // responses: // "200": // description: Access // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Access" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceAccess(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } access, err := s.Authorizer.GetInstanceAccess(context.TODO(), projectName, mux.Vars(r)["name"]) if err != nil { return response.InternalError(err) } return response.SyncResponse(true, access) } incus-7.0.0/cmd/incusd/instance_backup.go000066400000000000000000000474561517523235500203770ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "strings" "time" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/jmap" internalBackup "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/validate" ) // swagger:operation GET /1.0/instances/{name}/backups instances instance_backups_get // // Get the backups // // Returns a list of instance backups (URLs). // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/instances/foo/backups/backup0", // "/1.0/instances/foo/backups/backup1" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/instances/{name}/backups?recursion=1 instances instance_backups_get_recursion1 // // Get the backups // // Returns a list of instance backups (structs). // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of instance backups // items: // $ref: "#/definitions/InstanceBackup" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceBackupsGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) cname, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(cname) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, cname) if err != nil { return response.SmartError(err) } if resp != nil { return resp } recursion := localUtil.IsRecursionRequest(r) c, err := instance.LoadByProjectAndName(s, projectName, cname) if err != nil { return response.SmartError(err) } backups, err := c.Backups() if err != nil { return response.SmartError(err) } resultString := []string{} resultMap := []*api.InstanceBackup{} for _, backup := range backups { if !recursion { url := fmt.Sprintf("/%s/instances/%s/backups/%s", version.APIVersion, cname, strings.Split(backup.Name(), "/")[1]) resultString = append(resultString, url) } else { render := backup.Render() resultMap = append(resultMap, render) } } if !recursion { return response.SyncResponse(true, resultString) } return response.SyncResponse(true, resultMap) } // swagger:operation POST /1.0/instances/{name}/backups instances instance_backups_post // // Create a backup // // Creates a new backup. // // If the `Accept` header is set to `application/octet-stream`, this directly streams the backup // tarball to the client without any intermediate operation. // // --- // consumes: // - application/json // produces: // - application/json // - application/octet-stream // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: backup // description: Backup request // required: false // schema: // $ref: "#/definitions/InstanceBackupsPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceBackupsPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { err := project.AllowBackupCreation(tx, projectName) return err }) if err != nil { return response.SmartError(err) } // Handle requests targeted to a container on a different node. resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } rj := jmap.Map{} err = json.NewDecoder(r.Body).Decode(&rj) if err != nil { return response.InternalError(err) } expiry, _ := rj.GetString("expires_at") if expiry == "" { // Disable expiration by setting it to zero time. rj["expires_at"] = time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC) } // Create body with correct expiry. body, err := json.Marshal(rj) if err != nil { return response.InternalError(err) } req := api.InstanceBackupsPost{} err = json.Unmarshal(body, &req) if err != nil { return response.BadRequest(err) } direct := r.Header.Get("Accept") == "application/octet-stream" if direct && req.Target != nil { return response.BadRequest(errors.New("application/octet-stream is not a valid content type when a target is defined")) } if req.CompressionAlgorithm != "" { err := validate.IsCompressionAlgorithm(req.CompressionAlgorithm) if err != nil { return response.BadRequest(err) } } var reader *io.PipeReader var writer *io.PipeWriter var fullName string if direct || req.Target != nil { if req.Name != "" { if direct { return response.BadRequest(errors.New("No backup name can be set when requesting a direct backup with Accept: application/octet-stream")) } return response.BadRequest(errors.New("No backup name can be set when setting a backup target")) } reader, writer = io.Pipe() } else { if req.Name == "" { // come up with a name. backups, err := inst.Backups() if err != nil { return response.BadRequest(err) } base := name + internalInstance.SnapshotDelimiter + "backup" length := len(base) backupID := 0 for _, backup := range backups { // Ignore backups not containing base. if !strings.HasPrefix(backup.Name(), base) { continue } substr := backup.Name()[length:] var num int count, err := fmt.Sscanf(substr, "%d", &num) if err != nil || count != 1 { continue } if num >= backupID { backupID = num + 1 } } req.Name = fmt.Sprintf("backup%d", backupID) } // Validate the name. if strings.Contains(req.Name, "/") { return response.BadRequest(errors.New("Backup names may not contain slashes")) } fullName = name + internalInstance.SnapshotDelimiter + req.Name } backup := func(op *operations.Operation) error { args := db.InstanceBackup{ Name: fullName, InstanceID: inst.ID(), CreationDate: time.Now(), InstanceOnly: req.InstanceOnly, RootOnly: req.RootOnly, OptimizedStorage: req.OptimizedStorage, CompressionAlgorithm: req.CompressionAlgorithm, } if !direct && req.Target == nil { args.ExpiryDate = req.ExpiresAt } uploadRes := make(chan error) // Start the upload in the background if requested. if req.Target != nil { go func(resCh chan<- error) { resCh <- internalBackup.Upload(reader, req.Target) }(uploadRes) } // Create the backup. err := backupCreate(s, args, inst, op, writer) if err != nil { // If we receive a pipe closed error, we first check for an explicit error returned by the // reader. if errors.Is(err, io.ErrClosedPipe) { select { case readerErr := <-uploadRes: err = readerErr default: } } // In order to actually fail piped exports, we use a dirty trick where we close the reader. // This doesn't provide a clean error message in the case of direct backups, but it is a // convenient tradeoff ensuring that the client reports an error. _ = reader.Close() return err } return nil } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", name)} if !direct && req.Target == nil { resources["backups"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", name, "backups", req.Name)} } op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.BackupCreate, resources, nil, backup, nil, nil, r) if err != nil { return response.InternalError(err) } if direct { err = op.Start() if err != nil { return response.InternalError(err) } return response.PipeResponse(r, reader) } return operations.OperationResponse(op) } // swagger:operation GET /1.0/instances/{name}/backups/{backup} instances instance_backup_get // // Get the backup // // Gets a specific instance backup. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: path // name: backup // description: Backup name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Instance backup // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/InstanceBackup" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceBackupGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } backupName, err := url.PathUnescape(mux.Vars(r)["backupName"]) if err != nil { return response.SmartError(err) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } fullName := name + internalInstance.SnapshotDelimiter + backupName backup, err := instance.BackupLoadByName(s, projectName, fullName) if err != nil { return response.SmartError(err) } return response.SyncResponse(true, backup.Render()) } // swagger:operation POST /1.0/instances/{name}/backups/{backup} instances instance_backup_post // // Rename a backup // // Renames an instance backup. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: path // name: backup // description: Backup name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: backup // description: Backup rename // required: false // schema: // $ref: "#/definitions/InstanceBackupPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceBackupPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } backupName, err := url.PathUnescape(mux.Vars(r)["backupName"]) if err != nil { return response.SmartError(err) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } req := api.InstanceBackupPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Validate the name if strings.Contains(req.Name, "/") { return response.BadRequest(errors.New("Backup names may not contain slashes")) } oldName := name + internalInstance.SnapshotDelimiter + backupName backup, err := instance.BackupLoadByName(s, projectName, oldName) if err != nil { return response.SmartError(err) } newName := name + internalInstance.SnapshotDelimiter + req.Name rename := func(op *operations.Operation) error { err := backup.Rename(newName) if err != nil { return err } return nil } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", name)} op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.BackupRename, resources, nil, rename, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // swagger:operation DELETE /1.0/instances/{name}/backups/{backup} instances instance_backup_delete // // Delete a backup // // Deletes the instance backup. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: path // name: backup // description: Backup name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceBackupDelete(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } backupName, err := url.PathUnescape(mux.Vars(r)["backupName"]) if err != nil { return response.SmartError(err) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } fullName := name + internalInstance.SnapshotDelimiter + backupName backup, err := instance.BackupLoadByName(s, projectName, fullName) if err != nil { return response.SmartError(err) } remove := func(op *operations.Operation) error { err := backup.Delete() if err != nil { return err } return nil } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", name)} op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.BackupRemove, resources, nil, remove, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // swagger:operation GET /1.0/instances/{name}/backups/{backup}/export instances instance_backup_export // // Get the raw backup file(s) // // Download the raw backup file(s) from the server. // // --- // produces: // - application/octet-stream // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: path // name: backup // description: Backup name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Raw image data // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceBackupExportGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } backupName, err := url.PathUnescape(mux.Vars(r)["backupName"]) if err != nil { return response.SmartError(err) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } fullName := name + internalInstance.SnapshotDelimiter + backupName backup, err := instance.BackupLoadByName(s, projectName, fullName) if err != nil { return response.SmartError(err) } ent := response.FileResponseEntry{ Path: internalUtil.VarPath("backups", "instances", project.Instance(projectName, backup.Name())), } s.Events.SendLifecycle(projectName, lifecycle.InstanceBackupRetrieved.Event(fullName, backup.Instance(), nil)) return response.FileResponse(r, []response.FileResponseEntry{ent}, nil) } incus-7.0.0/cmd/incusd/instance_bitmap.go000066400000000000000000000056561517523235500204020ustar00rootroot00000000000000package main import ( "encoding/json" "errors" "fmt" "net/http" "net/url" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/shared/api" ) // swagger:operation POST /1.0/instances/{name}/bitmaps instances instance_bitmaps_post // // Create a bitmap // // Creates a new bitmap. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: bitmap // description: Bitmap request // required: false // schema: // $ref: "#/definitions/StorageVolumeBitmapsPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceBitmapsPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) cname, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(cname) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to an instance on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, cname) if err != nil { return response.SmartError(err) } if resp != nil { return resp } inst, err := instance.LoadByProjectAndName(s, projectName, cname) if err != nil { return response.SmartError(err) } if !inst.IsRunning() { return response.BadRequest(fmt.Errorf("Creating bitmaps requires the instance to be running")) } if inst.Type() != instancetype.VM { return response.BadRequest(fmt.Errorf("Only VMs are supported.")) } req := api.StorageVolumeBitmapsPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } rootDiskName, _, err := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if err != nil { return response.BadRequest(fmt.Errorf("Failed getting instance root disk: %w", err)) } devNames := []string{rootDiskName} err = inst.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error { devNames = append(devNames, dev.Name) return nil }) if err != nil { return response.SmartError(err) } err = inst.CreateBitmap(devNames, req) if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } incus-7.0.0/cmd/incusd/instance_console.go000066400000000000000000000546161517523235500205700ustar00rootroot00000000000000package main import ( "bytes" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "os" "path" "slices" "strconv" "sync" "time" "github.com/gorilla/mux" "github.com/gorilla/websocket" liblxc "github.com/lxc/go-lxc" "golang.org/x/sys/unix" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/ws" ) type consoleWs struct { // instance currently worked on instance instance.Instance // daemon state (used to emit lifecycle events) state *state.State // websocket connections to bridge pty fds to conns map[int]*websocket.Conn // map dynamic websocket connections to their associated console file dynamic map[*websocket.Conn]*os.File // locks needed to access the "conns" member connsLock sync.Mutex // channel to wait until all websockets are properly connected allConnected chan bool // channel to wait until the control socket is connected controlConnected chan bool // map file descriptors to secret fds map[int]string // terminal width width int // terminal height height int // channel type (either console or vga) protocol string } func (s *consoleWs) metadata() any { fds := jmap.Map{} for fd, secret := range s.fds { if fd == -1 { fds[api.SecretNameControl] = secret } else { fds[strconv.Itoa(fd)] = secret } } return jmap.Map{"fds": fds} } func (s *consoleWs) connect(op *operations.Operation, r *http.Request, w http.ResponseWriter) error { // Check that the user connecting is the same who started the session. if !op.IsSameRequestor(r) { return api.StatusErrorf(http.StatusForbidden, "Requestor mismatch") } switch s.protocol { case instance.ConsoleTypeConsole: return s.connectConsole(r, w) case instance.ConsoleTypeVGA: return s.connectVGA(r, w) default: return fmt.Errorf("Unknown protocol %q", s.protocol) } } func (s *consoleWs) connectConsole(r *http.Request, w http.ResponseWriter) error { secret := r.FormValue("secret") if secret == "" { return errors.New("missing secret") } for fd, fdSecret := range s.fds { if secret == fdSecret { conn, err := ws.Upgrader.Upgrade(w, r, nil) if err != nil { return err } s.connsLock.Lock() s.conns[fd] = conn s.connsLock.Unlock() if fd == -1 { s.controlConnected <- true return nil } s.connsLock.Lock() for i, c := range s.conns { if i != -1 && c == nil { s.connsLock.Unlock() return nil } } s.connsLock.Unlock() s.allConnected <- true return nil } } /* If we didn't find the right secret, the user provided a bad one, * which 403, not 404, since this operation actually exists */ return os.ErrPermission } func (s *consoleWs) connectVGA(r *http.Request, w http.ResponseWriter) error { secret := r.FormValue("secret") if secret == "" { return errors.New("missing secret") } for fd, fdSecret := range s.fds { if secret != fdSecret { continue } conn, err := ws.Upgrader.Upgrade(w, r, nil) if err != nil { return err } if fd == -1 { logger.Debug("VGA control websocket connected") s.connsLock.Lock() s.conns[fd] = conn s.connsLock.Unlock() s.controlConnected <- true // Emit a single instance-console event per session here. SPICE clients open one // dynamic websocket per channel (display, cursor, inputs, ...) and emitting from the // per-channel path would produce many duplicate events for one user-visible session. s.state.Events.SendLifecycle(s.instance.Project().Name, lifecycle.InstanceConsole.Event(s.instance, logger.Ctx{"type": s.protocol})) return nil } logger.Debug("VGA dynamic websocket connected") console, _, err := s.instance.Console("vga") if err != nil { _ = conn.Close() return err } // Mirror the console and websocket. go func() { l := logger.AddContext(logger.Ctx{"address": conn.RemoteAddr().String()}) defer l.Debug("Finished mirroring websocket to console") l.Debug("Started mirroring websocket") readDone, writeDone := ws.Mirror(conn, console) <-readDone l.Debug("Finished mirroring console to websocket") _ = conn.Close() <-writeDone }() s.connsLock.Lock() s.dynamic[conn] = console s.connsLock.Unlock() return nil } // If we didn't find the right secret, the user provided a bad one, // which 403, not 404, since this operation actually exists. return os.ErrPermission } func (s *consoleWs) do(op *operations.Operation) error { s.instance.SetOperation(op) switch s.protocol { case instance.ConsoleTypeConsole: return s.doConsole() case instance.ConsoleTypeVGA: return s.doVGA() default: return fmt.Errorf("Unknown protocol %q", s.protocol) } } func (s *consoleWs) doConsole() error { defer logger.Debug("Console websocket finished") <-s.allConnected // Get console from instance. console, consoleDisconnectCh, err := s.instance.Console(s.protocol) if err != nil { return err } // Cleanup the console when we're done. defer func() { _ = console.Close() }() // Detect size of window and set it into console. if s.width > 0 && s.height > 0 { _ = linux.SetPtySize(int(console.Fd()), s.width, s.height) } consoleDoneCh := make(chan struct{}) // Wait for control socket to connect and then read messages from the remote side in a loop. go func() { defer logger.Debugf("Console control websocket finished") res := <-s.controlConnected if !res { return } for { s.connsLock.Lock() conn := s.conns[-1] s.connsLock.Unlock() _, r, err := conn.NextReader() if err != nil { logger.Debugf("Got error getting next reader: %v", err) close(consoleDoneCh) return } buf, err := io.ReadAll(r) if err != nil { logger.Debugf("Failed to read message: %v", err) break } command := api.InstanceConsoleControl{} err = json.Unmarshal(buf, &command) if err != nil { logger.Debugf("Failed to unmarshal control socket command: %s", err) continue } if command.Command == "window-resize" { winchWidth, err := strconv.Atoi(command.Args["width"]) if err != nil { logger.Debugf("Unable to extract window width: %s", err) continue } winchHeight, err := strconv.Atoi(command.Args["height"]) if err != nil { logger.Debugf("Unable to extract window height: %s", err) continue } err = linux.SetPtySize(int(console.Fd()), winchWidth, winchHeight) if err != nil { logger.Debugf("Failed to set window size to: %dx%d", winchWidth, winchHeight) continue } logger.Debugf("Set window size to: %dx%d", winchWidth, winchHeight) } } }() // Mirror the console and websocket. mirrorDoneCh := make(chan struct{}) go func() { s.connsLock.Lock() conn := s.conns[0] s.connsLock.Unlock() l := logger.AddContext(logger.Ctx{"address": conn.RemoteAddr().String()}) defer l.Debug("Finished mirroring websocket to console") l.Debug("Started mirroring websocket") readDone, writeDone := ws.Mirror(conn, console) <-readDone l.Debug("Finished mirroring console to websocket") _ = conn.Close() <-writeDone close(mirrorDoneCh) }() // Wait until either the console or the websocket is done. select { case <-mirrorDoneCh: close(consoleDisconnectCh) case <-consoleDoneCh: close(consoleDisconnectCh) } // Once this function ends ensure that any connected websockets are closed. defer func() { s.connsLock.Lock() consoleConn := s.conns[0] ctrlConn := s.conns[-1] if consoleConn != nil { _ = consoleConn.Close() } if ctrlConn != nil { _ = ctrlConn.Close() } s.connsLock.Unlock() }() // Write a reset escape sequence to the console to cancel any ongoing reads to the handle // and then close it. This ordering is important, close the console before closing the // websocket to ensure console doesn't get stuck reading. _, _ = console.Write([]byte("\x1bc")) err = console.Close() if err != nil && !errors.Is(err, os.ErrClosed) { return err } // Indicate to the control socket go routine to end if not already. close(s.controlConnected) return nil } func (s *consoleWs) doVGA() error { defer logger.Debug("VGA websocket finished") consoleDoneCh := make(chan struct{}) // The control socket is used to terminate the operation. go func() { defer logger.Debugf("VGA control websocket finished") res := <-s.controlConnected if !res { return } for { s.connsLock.Lock() conn := s.conns[-1] s.connsLock.Unlock() _, _, err := conn.NextReader() if err != nil { logger.Debugf("Got error getting next reader: %v", err) close(consoleDoneCh) return } } }() // Wait until the control channel is done. <-consoleDoneCh s.connsLock.Lock() control := s.conns[-1] s.connsLock.Unlock() err := control.Close() // Close all dynamic connections. for conn, console := range s.dynamic { _ = conn.Close() _ = console.Close() } // Indicate to the control socket go routine to end if not already. close(s.controlConnected) return err } // Cancel is responsible for closing websocket connections. func (s *consoleWs) cancel(*operations.Operation) error { s.connsLock.Lock() conn := s.conns[-1] s.connsLock.Unlock() if conn == nil { return nil } _ = conn.Close() // Close all dynamic connections. for conn, console := range s.dynamic { _ = conn.Close() _ = console.Close() } return nil } // swagger:operation POST /1.0/instances/{name}/console instances instance_console_post // // Connect to console // // Connects to the console of an instance. // // The returned operation metadata will contain two websockets, one for data and one for control. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: console // description: Console request // schema: // $ref: "#/definitions/InstanceConsolePost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceConsolePost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } post := api.InstanceConsolePost{} buf, err := io.ReadAll(r.Body) if err != nil { return response.BadRequest(err) } err = json.Unmarshal(buf, &post) if err != nil { return response.BadRequest(err) } // Forward the request if the container is remote. client, err := cluster.ConnectIfInstanceIsRemote(s, projectName, name, r) if err != nil { return response.SmartError(err) } if client != nil { url := api.NewURL().Path(version.APIVersion, "instances", name, "console").Project(projectName) resp, _, err := client.RawQuery("POST", url.String(), post, "") if err != nil { return response.SmartError(err) } opAPI, err := resp.MetadataAsOperation() if err != nil { return response.SmartError(err) } return operations.ForwardedOperationResponse(opAPI) } if post.Type == "" { post.Type = instance.ConsoleTypeConsole } // Basic parameter validation. if !slices.Contains([]string{instance.ConsoleTypeConsole, instance.ConsoleTypeVGA}, post.Type) { return response.BadRequest(fmt.Errorf("Unknown console type %q", post.Type)) } inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } if post.Type == instance.ConsoleTypeVGA && inst.Type() != instancetype.VM { return response.BadRequest(errors.New("VGA console is only supported by virtual machines")) } if !inst.IsRunning() { return response.BadRequest(errors.New("Instance is not running")) } if inst.IsFrozen() { return response.BadRequest(errors.New("Instance is frozen")) } // Find any running 'ConsoleShow' operation for the instance. // If the '--force' flag was used, cancel the running operation. Otherwise, notify the user about the operation. for _, op := range operations.Clone() { // Consider only console show operations with Running status. if op.Type() != operationtype.ConsoleShow || op.Project() != projectName || op.Status() != api.Running { continue } // Fetch instance name from operation. r := op.Resources() apiUrls := r["instances"] if len(apiUrls) < 1 { return response.SmartError(errors.New("Operation does not have an instance URL defined")) } urlPrefix, instanceName := path.Split(apiUrls[0].URL.Path) if urlPrefix == "" || instanceName == "" { return response.SmartError(errors.New("Instance URL has incorrect format")) } if instanceName != inst.Name() { continue } if !post.Force { return response.SmartError(errors.New("This console is already connected. Force is required to take it over.")) } _, err = op.Cancel() if err != nil { return response.SmartError(err) } } ws := &consoleWs{} ws.fds = map[int]string{} ws.conns = map[int]*websocket.Conn{} ws.conns[-1] = nil ws.conns[0] = nil ws.dynamic = map[*websocket.Conn]*os.File{} for i := -1; i < len(ws.conns)-1; i++ { ws.fds[i], err = internalUtil.RandomHexString(32) if err != nil { return response.InternalError(err) } } ws.allConnected = make(chan bool, 1) ws.controlConnected = make(chan bool, 1) ws.instance = inst ws.state = s ws.width = post.Width ws.height = post.Height ws.protocol = post.Type resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", ws.instance.Name())} op, err := operations.OperationCreate(s, projectName, operations.OperationClassWebsocket, operationtype.ConsoleShow, resources, ws.metadata(), ws.do, ws.cancel, ws.connect, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // swagger:operation GET /1.0/instances/{name}/console instances instance_console_get // // Get console output // // Gets the console output for the instance either as text log or as vga // screendump. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: type // description: Console type // type: string // enum: [log, vga] // default: log // example: vga // responses: // "200": // description: | // Console output either as raw console log or as vga screendump in PNG // format depending on the `type` parameter provided with the request. // content: // application/octet-stream: // schema: // type: string // example: some-text // image/png: // schema: // type: string // format: binary // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceConsoleLogGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } consoleLogType := request.QueryParam(r, "type") if consoleLogType != "" && consoleLogType != "log" && consoleLogType != "vga" { return response.SmartError(fmt.Errorf("Invalid value for type parameter: %s", consoleLogType)) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Forward the request if the container is remote. resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } ent := response.FileResponseEntry{} if !inst.IsRunning() { // Check if we have data we can return. consoleBufferLogPath := inst.ConsoleBufferLogPath() if !util.PathExists(consoleBufferLogPath) { return response.FileResponse(r, nil, nil) } ent.Path = consoleBufferLogPath ent.Filename = consoleBufferLogPath return response.FileResponse(r, []response.FileResponseEntry{ent}, nil) } if inst.Type() == instancetype.Container { c, ok := inst.(instance.Container) if !ok { return response.SmartError(errors.New("Failed to cast inst to Container")) } // Query the container's console ringbuffer. console := liblxc.ConsoleLogOptions{ ClearLog: false, ReadLog: true, ReadMax: 0, WriteToLogFile: true, } // Send a ringbuffer request to the container. logContents, err := c.ConsoleLog(console) if err != nil { errno, isErrno := linux.GetErrno(err) if !isErrno { return response.SmartError(err) } if errors.Is(errno, unix.ENODATA) { return response.FileResponse(r, nil, nil) } return response.SmartError(err) } ent.File = bytes.NewReader([]byte(logContents)) ent.FileModified = time.Now() ent.FileSize = int64(len(logContents)) return response.FileResponse(r, []response.FileResponseEntry{ent}, nil) } else if inst.Type() == instancetype.VM { v, ok := inst.(instance.VM) if !ok { return response.SmartError(errors.New("Failed to cast inst to VM")) } var headers map[string]string if consoleLogType == "vga" { screenShotPath := fmt.Sprintf("/tmp/incus_screenshot_%d", inst.ID()) // Delete then create the path with O_EXCL to ensure that we are the creator of the path. // Any attempt at racing with us will cause in a (already exists) failure. _ = os.Remove(screenShotPath) screenshotFile, err := os.OpenFile(screenShotPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) if err != nil { return response.SmartError(fmt.Errorf("Couldn't create screenshot file: %w", err)) } ent.Cleanup = func() { _ = screenshotFile.Close() _ = os.Remove(screenshotFile.Name()) } err = v.ConsoleScreenshot(screenshotFile) if err != nil { return response.SmartError(err) } fileInfo, err := screenshotFile.Stat() if err != nil { return response.SmartError(fmt.Errorf("Couldn't stat screenshot file for filesize: %w", err)) } headers = map[string]string{ "Content-Type": "image/png", } ent.File = screenshotFile ent.FileSize = fileInfo.Size() ent.Filename = screenshotFile.Name() } else { logContents, err := v.ConsoleLog() if err != nil { return response.SmartError(err) } ent.File = bytes.NewReader([]byte(logContents)) ent.FileSize = int64(len(logContents)) } ent.FileModified = time.Now() return response.FileResponse(r, []response.FileResponseEntry{ent}, headers) } return response.SmartError(fmt.Errorf("Unsupported instance type %q", inst.Type())) } // swagger:operation DELETE /1.0/instances/{name}/console instances instance_console_delete // // Clear the console log // // Clears the console log buffer. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceConsoleLogDelete(d *Daemon, r *http.Request) response.Response { if !liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 3, 0, 0) { return response.BadRequest(errors.New("Clearing the console buffer requires liblxc >= 3.0")) } name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } projectName := request.ProjectParam(r) inst, err := instance.LoadByProjectAndName(d.State(), projectName, name) if err != nil { return response.SmartError(err) } if inst.Type() != instancetype.Container { return response.SmartError(errors.New("Instance is not container type")) } c, ok := inst.(instance.Container) if !ok { return response.SmartError(errors.New("Instance is not container type")) } truncateConsoleLogFile := func(path string) error { // Check that this is a regular file. We don't want to try and unlink // /dev/stderr or /dev/null or something. st, err := os.Stat(path) if err != nil { return err } if !st.Mode().IsRegular() { return errors.New("The console log is not a regular file") } if path == "" { return errors.New("Container does not keep a console logfile") } return os.Truncate(path, 0) } if !inst.IsRunning() { consoleLogpath := c.ConsoleBufferLogPath() return response.SmartError(truncateConsoleLogFile(consoleLogpath)) } // Send a ringbuffer request to the container. console := liblxc.ConsoleLogOptions{ ClearLog: true, ReadLog: false, ReadMax: 0, WriteToLogFile: false, } _, err = c.ConsoleLog(console) if err != nil { errno, isErrno := linux.GetErrno(err) if !isErrno { return response.SmartError(err) } if errors.Is(errno, unix.ENODATA) { return response.SmartError(nil) } return response.SmartError(err) } return response.SmartError(nil) } incus-7.0.0/cmd/incusd/instance_debug.go000066400000000000000000000165671517523235500202170ustar00rootroot00000000000000package main import ( "encoding/json" "errors" "fmt" "net/http" "net/url" "os" "slices" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) // swagger:operation GET /1.0/instances/{name}/debug/memory instances instance_debug_memory_get // // Get memory debug information of an instance // // Returns memory debug information of a running instance. // Only supported for VMs. // // --- // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: format // description: Memory dump format // type: string // example: elf // responses: // "200": // description: Success // content: // application/octet-stream: // schema: // type: string // example: raw memory dump // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceDebugMemoryGet(d *Daemon, r *http.Request) response.Response { s := d.State() format := request.QueryParam(r, "format") projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } // Ensure instance exists. inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } if inst.Type() != instancetype.VM { return response.BadRequest(errors.New("Memory dumps are only supported for virtual machines")) } if !inst.IsRunning() { return response.BadRequest(errors.New("Instance must be running to dump memory")) } v, ok := inst.(instance.VM) if !ok { return response.InternalError(errors.New("Failed to cast inst to VM")) } // Wrap up the request. return response.ManualResponse(func(w http.ResponseWriter) error { // Start streaming back to the client. w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "application/octet-stream") // Setup a PIPE for the data. reader, writer, err := os.Pipe() if err != nil { return err } defer reader.Close() defer writer.Close() chCopy := make(chan error) go func() { _, err := util.SafeCopy(w, reader) chCopy <- err }() // Start dumping into the PIPE. err = v.DumpGuestMemory(writer, format) if err != nil { return err } err = <-chCopy if err != nil { return err } return nil }) } // swagger:operation GET /1.0/instances/{name}/debug/repair instances instance_debug_repair_post // // Trigger a repair action on the instance. // // Runs an internal repair action on the instance. // // --- // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: state // description: State // required: false // schema: // $ref: "#/definitions/InstanceDebugRepairPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceDebugRepairPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to an instance on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } // Parse the request. req := api.InstanceDebugRepairPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Validate the repair action. if !slices.Contains([]string{"rebuild-config-volume"}, req.Action) { return response.BadRequest(fmt.Errorf("Invalid repair action %q", req.Action)) } // Load the instance. inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Running the action. switch req.Action { case "rebuild-config-volume": err := instanceDebugRepairRebuildConfigVolume(s, inst) if err != nil { return response.SmartError(err) } } return response.EmptySyncResponse } func instanceDebugRepairRebuildConfigVolume(s *state.State, inst instance.Instance) error { // Initial validation. if inst.Type() != instancetype.VM { return errors.New("Config volume rebuild is only possible on VMs") } if inst.IsRunning() { return errors.New("Config volume rebuild is only possible on stopped VMs") } // Load the storage pool. pool, err := storagePools.LoadByInstance(s, inst) if err != nil { return err } // Load the volume. dbVol, err := storagePools.VolumeDBGet(pool, inst.Project().Name, inst.Name(), storageDrivers.VolumeTypeVM) if err != nil { return err } if dbVol.Config["block.type"] != "qcow2" { return errors.New("Config volume rebuild is only possible on QCOW2 backed VMs") } volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := pool.GetVolume(storageDrivers.VolumeTypeVM, storageDrivers.ContentTypeFS, volStorageName, dbVol.Config) // Re-create the filesystem. err = pool.Driver().ActivateTask(vol, func(devPath string, op *operations.Operation) error { _, err = subprocess.RunCommand("mkfs.btrfs", "-f", devPath) if err != nil { return err } return nil }, nil) if err != nil { return err } // Re-configure the sub-volumes. err = storageDrivers.Qcow2CreateConfig(vol, nil) if err != nil { return err } snaps, err := inst.Snapshots() if err != nil { return err } for _, snap := range snaps { snapVolStorageName := project.Instance(snap.Project().Name, snap.Name()) snapVol := pool.GetVolume(storageDrivers.VolumeTypeVM, storageDrivers.ContentTypeFS, snapVolStorageName, dbVol.Config) err = storageDrivers.Qcow2CreateConfigSnapshot(vol, snapVol, nil) if err != nil { return err } } return nil } incus-7.0.0/cmd/incusd/instance_delete.go000066400000000000000000000050241517523235500203550ustar00rootroot00000000000000package main import ( "errors" "net/http" "net/url" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // swagger:operation DELETE /1.0/instances/{name} instances instance_delete // // Delete an instance // // Deletes a specific instance. // // This also deletes anything owned by the instance such as snapshots and backups. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceDelete(d *Daemon, r *http.Request) response.Response { // Don't mess with instance while in setup mode. <-d.waitReady.Done() s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } if inst.IsRunning() { return response.BadRequest(errors.New("Instance is running")) } run := func(op *operations.Operation) error { inst.SetOperation(op) return inst.Delete(false, true) } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", name)} op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.InstanceDelete, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } incus-7.0.0/cmd/incusd/instance_exec.go000066400000000000000000000540201517523235500200370ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "io" "io/fs" "net/http" "net/url" "os" "path/filepath" "strconv" "strings" "sync" "time" "github.com/gorilla/mux" "github.com/gorilla/websocket" "golang.org/x/sys/unix" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/drivers" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/cancel" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/tcp" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/ws" ) const ( execWSControl = -1 execWSStdin = 0 execWSStdout = 1 execWSStderr = 2 ) type execWs struct { req api.InstanceExecPost instance instance.Instance conns map[int]*websocket.Conn connsLock sync.Mutex waitRequiredConnected *cancel.Canceller waitControlConnected *cancel.Canceller fds map[int]string s *state.State } func (s *execWs) metadata() any { fds := jmap.Map{} for fd, secret := range s.fds { if fd == execWSControl { fds[api.SecretNameControl] = secret } else { fds[strconv.Itoa(fd)] = secret } } return jmap.Map{ "fds": fds, "command": s.req.Command, "environment": s.req.Environment, "interactive": s.req.Interactive, } } func (s *execWs) cancel(op *operations.Operation) error { s.connsLock.Lock() conn := s.conns[-1] s.connsLock.Unlock() if conn == nil { return nil } _ = conn.Close() return nil } func (s *execWs) connect(op *operations.Operation, r *http.Request, w http.ResponseWriter) error { // Check that the user connecting is the same who started the session. if !op.IsSameRequestor(r) { return api.StatusErrorf(http.StatusForbidden, "Requestor mismatch") } secret := r.FormValue("secret") if secret == "" { return errors.New("missing secret") } for fd, fdSecret := range s.fds { if secret == fdSecret { conn, err := ws.Upgrader.Upgrade(w, r, nil) if err != nil { return err } s.connsLock.Lock() defer s.connsLock.Unlock() val, found := s.conns[fd] if found && val == nil { s.conns[fd] = conn // Set TCP timeout options. remoteTCP, _ := tcp.ExtractConn(conn.UnderlyingConn()) if remoteTCP != nil { err = tcp.SetTimeouts(remoteTCP, 0) if err != nil { logger.Warn("Failed setting TCP timeouts on remote connection", logger.Ctx{"err": err}) } } // Start channel keep alive to run until channel is closed. go func() { pingInterval := time.Second * 10 t := time.NewTicker(pingInterval) defer t.Stop() for { err := conn.WriteControl(websocket.PingMessage, []byte("keepalive"), time.Now().Add(5*time.Second)) if err != nil { return } <-t.C } }() if fd == execWSControl { s.waitControlConnected.Cancel() // Control connection connected. } for i, c := range s.conns { if i == execWSControl && s.req.WaitForWS && !s.req.Interactive { // Due to a historical bug in the LXC CLI command, we cannot force // the client to connect a control socket when in non-interactive // mode. This is because the older CLI tools did not connect this // channel and so we would prevent the older CLIs connecting to // newer servers. So skip the control connection from being // considered as a required connection in this case. continue } if c == nil { return nil // Not all required connections connected yet. } } s.waitRequiredConnected.Cancel() // All required connections now connected. return nil } else if !found { return errors.New("Unknown websocket number") } else { return errors.New("Websocket number already connected") } } } /* If we didn't find the right secret, the user provided a bad one, * which 403, not 404, since this operation actually exists */ return os.ErrPermission } func (s *execWs) do(op *operations.Operation) error { s.instance.SetOperation(op) // Once this function ends ensure that any connected websockets are closed. defer func() { s.connsLock.Lock() for i := range s.conns { if s.conns[i] != nil { _ = s.conns[i].Close() } } s.connsLock.Unlock() }() // As this function only gets called when the exec request has WaitForWS enabled, we expect the client to // connect to all of the required websockets within a short period of time and we won't proceed until then. logger.Debug("Waiting for exec websockets to connect") select { case <-s.waitRequiredConnected.Done(): break case <-time.After(time.Second * 10): return errors.New("Timed out waiting for websockets to connect") } var err error var ttys []*os.File var ptys []*os.File var stdin *os.File var stdout *os.File var stderr *os.File if s.req.Interactive { if s.instance.Type() == instancetype.Container { // For containers, we setup a PTY on the server. ttys = make([]*os.File, 1) ptys = make([]*os.File, 1) var rootUID, rootGID int64 var devptsFd *os.File c := s.instance.(instance.Container) idmapset, err := c.CurrentIdmap() if err != nil { return err } if idmapset != nil { rootUID, rootGID = idmapset.ShiftIntoNS(0, 0) } devptsFd, _ = c.DevptsFd() if devptsFd != nil { ptys[0], ttys[0], err = linux.OpenPtyInDevpts(int(devptsFd.Fd()), rootUID, rootGID) _ = devptsFd.Close() devptsFd = nil } else { ptys[0], ttys[0], err = linux.OpenPty(rootUID, rootGID) } if err != nil { return fmt.Errorf("Unable to open the PTY device: %w", err) } stdin = ttys[0] stdout = ttys[0] stderr = ttys[0] if s.req.Width > 0 && s.req.Height > 0 { _ = linux.SetPtySize(int(ptys[0].Fd()), s.req.Width, s.req.Height) } } else { // For VMs we rely on the agent PTY running inside the VM guest. ttys = make([]*os.File, 2) ptys = make([]*os.File, 2) for i := range ttys { ptys[i], ttys[i], err = os.Pipe() if err != nil { return err } } stdin = ptys[execWSStdin] stdout = ttys[execWSStdout] } } else { ttys = make([]*os.File, 3) ptys = make([]*os.File, 3) for i := range ttys { ptys[i], ttys[i], err = os.Pipe() if err != nil { return err } } stdin = ptys[execWSStdin] stdout = ttys[execWSStdout] stderr = ttys[execWSStderr] } waitAttachedChildIsDead, markAttachedChildIsDead := context.WithCancel(context.Background()) var wgEOF sync.WaitGroup // Define a function to clean up TTYs and sockets when done. finisher := func(cmdResult int, cmdErr error) error { // Cancel this before closing the control connection so control handler can detect command ending. markAttachedChildIsDead() for _, tty := range ttys { _ = tty.Close() } s.connsLock.Lock() conn := s.conns[execWSControl] s.connsLock.Unlock() if conn == nil { s.waitControlConnected.Cancel() // Request control go routine to end if no control connection. } else { err = conn.Close() // Close control connection (will cause control go routine to end). if err != nil && cmdErr == nil { cmdErr = err } } wgEOF.Wait() for _, pty := range ptys { _ = pty.Close() } // Make VM disconnections (shutdown/reboot) match containers. if errors.Is(cmdErr, drivers.ErrExecDisconnected) { cmdResult = 129 cmdErr = nil } metadata := jmap.Map{"return": cmdResult} err = op.ExtendMetadata(metadata) if err != nil { return err } return cmdErr } cmd, err := s.instance.Exec(s.req, stdin, stdout, stderr) if err != nil { return finisher(-1, err) } l := logger.AddContext(logger.Ctx{"project": s.instance.Project().Name, "instance": s.instance.Name(), "PID": cmd.PID(), "interactive": s.req.Interactive}) l.Debug("Instance process started") var cmdKillOnce sync.Once cmdKill := func() { err := cmd.Signal(unix.SIGKILL) if err != nil { l.Debug("Failed to send SIGKILL signal", logger.Ctx{"err": err}) } else { l.Debug("Sent SIGKILL signal") } } // Now that process has started, we can start the control handler. wgEOF.Add(1) go func() { defer wgEOF.Done() <-s.waitControlConnected.Done() // Indicates control connection has started or command has ended. s.connsLock.Lock() conn := s.conns[execWSControl] s.connsLock.Unlock() if conn == nil { return // No connection, command has ended, being asked to end. } l.Debug("Exec control handler started") defer l.Debug("Exec control handler finished") for { mt, r, err := conn.NextReader() if err != nil || mt == websocket.CloseMessage { // Check if command process has finished normally, if so, no need to kill it. if waitAttachedChildIsDead.Err() != nil { return } if mt == websocket.CloseMessage { l.Warn("Got exec control websocket close message, killing command") } else { l.Warn("Failed getting exec control websocket reader, killing command", logger.Ctx{"err": err}) } cmdKillOnce.Do(cmdKill) return } buf, err := io.ReadAll(r) if err != nil { // Check if command process has finished normally, if so, no need to kill it. if waitAttachedChildIsDead.Err() != nil { return } l.Warn("Failed reading control websocket message, killing command", logger.Ctx{"err": err}) cmdKillOnce.Do(cmdKill) return } command := api.InstanceExecControl{} err = json.Unmarshal(buf, &command) if err != nil { l.Debug("Failed to unmarshal control socket command", logger.Ctx{"err": err}) continue } // Only handle window-resize requests for interactive sessions. if command.Command == "window-resize" && s.req.Interactive { winchWidth, err := strconv.Atoi(command.Args["width"]) if err != nil { l.Debug("Unable to extract window width", logger.Ctx{"err": err}) continue } winchHeight, err := strconv.Atoi(command.Args["height"]) if err != nil { l.Debug("Unable to extract window height", logger.Ctx{"err": err}) continue } err = cmd.WindowResize(int(ptys[0].Fd()), winchWidth, winchHeight) if err != nil { l.Debug("Failed to set window size", logger.Ctx{"err": err, "width": winchWidth, "height": winchHeight}) continue } } else if command.Command == "signal" { err := cmd.Signal(unix.Signal(command.Signal)) if err != nil { l.Debug("Failed forwarding signal", logger.Ctx{"err": err, "signal": command.Signal}) continue } } } }() // Now that process has started, we can start the mirroring of the process channels and websockets. if s.req.Interactive { wgEOF.Add(1) go func() { defer wgEOF.Done() var readErr, writeErr error l.Debug("Exec mirror websocket started", logger.Ctx{"number": 0}) defer func() { l.Debug("Exec mirror websocket finished", logger.Ctx{"number": 0, "readErr": readErr, "writeErr": writeErr}) }() s.connsLock.Lock() conn := s.conns[0] s.connsLock.Unlock() var readDone, writeDone chan error if s.instance.Type() == instancetype.Container { // For containers, we are running the command via the locally managed PTY and so // need to use the same PTY handle for both read and write. readDone, writeDone = ws.Mirror(conn, linux.NewExecWrapper(waitAttachedChildIsDead, ptys[0])) } else { readDone = ws.MirrorRead(conn, ptys[execWSStdout]) writeDone = ws.MirrorWrite(conn, ttys[execWSStdin]) } readErr = <-readDone writeErr = <-writeDone _ = conn.Close() }() } else { wgEOF.Add(len(ttys) - 1) for i := range ttys { go func(i int) { var err error l.Debug("Exec mirror websocket started", logger.Ctx{"number": i}) defer func() { l.Debug("Exec mirror websocket finished", logger.Ctx{"number": i, "err": err}) }() s.connsLock.Lock() conn := s.conns[i] s.connsLock.Unlock() if i == execWSStdout { // Launch a go routine that reads from stdout. This will be used to detect // when the client disconnects, as normally there should be no data // received on the stdout channel from the client. This is needed in cases // where the control connection isn't used by the client and so we need to // detect when the client disconnects to avoid leaving the command running // in the background. go func() { _, _, err := conn.ReadMessage() // If there is a control connection, then leave it to that handler // to clean the command up. If there's no control connection, the // control context gets cancelled when the command exits, so this // can also be used indicate that the command has already finished. // In either case there is no need to kill the command, but if not // then it is our responsibility to kill the command now. if s.waitControlConnected.Err() == nil { l.Warn("Unexpected read on stdout websocket, killing command", logger.Ctx{"number": i, "err": err}) cmdKillOnce.Do(cmdKill) } }() } if i == execWSStderr { // Consume data (e.g. websocket pings) from stderr too to // avoid a situation where we hit an inactivity timeout on // stderr during long exec sessions go func() { _, _, _ = conn.ReadMessage() }() } if i == execWSStdin { err = <-ws.MirrorWrite(conn, ttys[i]) _ = ttys[i].Close() } else { err = <-ws.MirrorRead(conn, linux.NewExecWrapper(waitAttachedChildIsDead, ptys[i])) _ = ptys[i].Close() wgEOF.Done() } }(i) } } exitStatus, err := cmd.Wait() l.Debug("Instance process stopped", logger.Ctx{"err": err, "exitStatus": exitStatus}) return finisher(exitStatus, err) } // swagger:operation POST /1.0/instances/{name}/exec instances instance_exec_post // // Run a command // // Executes a command inside an instance. // // The returned operation metadata will contain either 2 or 4 websockets. // In non-interactive mode, you'll get one websocket for each of stdin, stdout and stderr. // In interactive mode, a single bi-directional websocket is used for stdin and stdout/stderr. // // An additional "control" socket is always added on top which can be used for out of band communications. // This allows sending signals and window sizing information through. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: exec // description: Exec request // schema: // $ref: "#/definitions/InstanceExecPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceExecPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } post := api.InstanceExecPost{} buf, err := io.ReadAll(r.Body) if err != nil { return response.BadRequest(err) } err = json.Unmarshal(buf, &post) if err != nil { return response.BadRequest(err) } // Constraint validations. if post.RecordOutput && post.WaitForWS { return response.BadRequest(fmt.Errorf("Cannot use %q in combination with %q", "record-output", "wait-for-websocket")) } if post.Interactive && post.RecordOutput { return response.BadRequest(fmt.Errorf("Cannot use %q in combination with %q", "interactive", "record-output")) } // Forward the request if the container is remote. client, err := cluster.ConnectIfInstanceIsRemote(s, projectName, name, r) if err != nil { return response.SmartError(err) } if client != nil { url := api.NewURL().Path(version.APIVersion, "instances", name, "exec").Project(projectName) resp, _, err := client.RawQuery("POST", url.String(), post, "") if err != nil { return response.SmartError(err) } opAPI, err := resp.MetadataAsOperation() if err != nil { return response.SmartError(err) } return operations.ForwardedOperationResponse(opAPI) } inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } if !inst.IsRunning() { return response.BadRequest(errors.New("Instance is not running")) } if inst.IsFrozen() { return response.BadRequest(errors.New("Instance is frozen")) } // Process environment. if post.Environment == nil { post.Environment = map[string]string{} } // Override any environment variable settings from the instance if not manually specified in post. for k, v := range inst.ExpandedConfig() { after, ok := strings.CutPrefix(k, "environment.") if ok { envKey := after _, found := post.Environment[envKey] if !found { post.Environment[envKey] = v } } } // Set default value for PATH. _, ok := post.Environment["PATH"] if !ok { post.Environment["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" if inst.Type() == instancetype.Container { // Add some additional paths. This directly looks through /proc // rather than use FileExists as none of those paths are expected to be // symlinks and this is much faster than forking a sub-process and // attaching to the instance. extraPaths := map[string]string{ "/snap": "/snap/bin", "/etc/NIXOS": "/run/current-system/sw/bin", } instPID := inst.InitPID() for k, v := range extraPaths { if util.PathExists(fmt.Sprintf("/proc/%d/root%s", instPID, k)) { post.Environment["PATH"] = fmt.Sprintf("%s:%s", post.Environment["PATH"], v) } } } } // If running as root, set some env variables. if post.User == 0 { // Set default value for HOME. _, ok = post.Environment["HOME"] if !ok { post.Environment["HOME"] = "/root" } // Set default value for USER. _, ok = post.Environment["USER"] if !ok { post.Environment["USER"] = "root" } } // Set default value for LANG. _, ok = post.Environment["LANG"] if !ok { post.Environment["LANG"] = "C.UTF-8" } if post.WaitForWS { ws := &execWs{} ws.s = d.State() ws.fds = map[int]string{} ws.conns = map[int]*websocket.Conn{} ws.conns[execWSControl] = nil ws.conns[0] = nil // This is used for either TTY or Stdin. if !post.Interactive { ws.conns[execWSStdout] = nil ws.conns[execWSStderr] = nil } ws.waitRequiredConnected = cancel.New(context.Background()) ws.waitControlConnected = cancel.New(context.Background()) for i := range ws.conns { ws.fds[i], err = internalUtil.RandomHexString(32) if err != nil { return response.InternalError(err) } } ws.instance = inst ws.req = post resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", ws.instance.Name())} op, err := operations.OperationCreate(s, projectName, operations.OperationClassWebsocket, operationtype.CommandExec, resources, ws.metadata(), ws.do, ws.cancel, ws.connect, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } run := func(op *operations.Operation) error { inst.SetOperation(op) metadata := jmap.Map{} var err error var stdout, stderr *os.File if post.RecordOutput { // Ensure exec-output directory exists execOutputDir := inst.ExecOutputPath() err = os.Mkdir(execOutputDir, 0o600) if err != nil && !errors.Is(err, fs.ErrExist) { return err } // Prepare stdout and stderr recording. stdout, err = os.OpenFile(filepath.Join(execOutputDir, fmt.Sprintf("exec_%s.stdout", op.ID())), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o666) if err != nil { return err } defer func() { _ = stdout.Close() }() stderr, err = os.OpenFile(filepath.Join(execOutputDir, fmt.Sprintf("exec_%s.stderr", op.ID())), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o666) if err != nil { return err } defer func() { _ = stderr.Close() }() // Update metadata with the right URLs. metadata["output"] = jmap.Map{ "1": fmt.Sprintf("/%s/instances/%s/logs/exec-output/%s", version.APIVersion, inst.Name(), filepath.Base(stdout.Name())), "2": fmt.Sprintf("/%s/instances/%s/logs/exec-output/%s", version.APIVersion, inst.Name(), filepath.Base(stderr.Name())), } } // Run the command. cmd, err := inst.Exec(post, nil, stdout, stderr) if err != nil { return err } l := logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "PID": cmd.PID(), "recordOutput": post.RecordOutput}) l.Debug("Instance process started") exitStatus, cmdErr := cmd.Wait() l.Debug("Instance process stopped", logger.Ctx{"err": cmdErr, "exitStatus": exitStatus}) metadata["return"] = exitStatus err = op.ExtendMetadata(metadata) if err != nil { l.Error("Error updating metadata for cmd", logger.Ctx{"err": err, "cmd": post.Command}) } if cmdErr != nil { return cmdErr } return nil } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", name)} if inst.Type() == instancetype.Container { resources["containers"] = resources["instances"] } op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.CommandExec, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } incus-7.0.0/cmd/incusd/instance_file.go000066400000000000000000000407061517523235500200400ustar00rootroot00000000000000package main import ( "bytes" "errors" "fmt" "io" "io/fs" "net/http" "net/url" "os" "path/filepath" "slices" "strings" "time" "github.com/gorilla/mux" "github.com/pkg/sftp" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" ) func instanceFileHandler(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Redirect to correct server if needed. resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } // Load the instance. inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Parse and cleanup the path. path := r.FormValue("path") if path == "" { return response.BadRequest(errors.New("Missing path argument")) } if !strings.HasPrefix(path, "/") { path = "/" + path } switch r.Method { case "GET": return instanceFileGet(s, inst, path, r) case "HEAD": return instanceFileHead(s, inst, path, r) case "POST": return instanceFilePost(s, inst, path, r) case "DELETE": return instanceFileDelete(s, inst, path, r) default: return response.NotFound(fmt.Errorf("Method %q not found", r.Method)) } } // swagger:operation GET /1.0/instances/{name}/files instances instance_files_get // // Get a file // // Gets the file content. If it's a directory, a json list of files will be returned instead. // // --- // produces: // - application/json // - application/octet-stream // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: path // description: Path to the file // type: string // example: default // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Raw file or directory listing // headers: // X-Incus-uid: // description: File owner UID // schema: // type: integer // X-Incus-gid: // description: File owner GID // schema: // type: integer // X-Incus-mode: // description: Mode mask // schema: // type: integer // X-Incus-modified: // description: Last modified date // schema: // type: string // X-Incus-type: // description: Type of file (file, symlink or directory) // schema: // type: string // content: // application/octet-stream: // schema: // type: string // example: some-text // application/json: // schema: // type: array // items: // type: string // example: |- // [ // "/etc", // "/home" // ] // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceFileGet(s *state.State, inst instance.Instance, path string, r *http.Request) response.Response { reverter := revert.New() defer reverter.Fail() // Get a SFTP client. client, err := inst.FileSFTP() if err != nil { return response.InternalError(err) } reverter.Add(func() { _ = client.Close() }) return fileSFTPGet(client, path, r, reverter, func() { s.Events.SendLifecycle(inst.Project().Name, lifecycle.InstanceFileRetrieved.Event(inst, logger.Ctx{"path": path})) }) } // swagger:operation HEAD /1.0/instances/{name}/files instances instance_files_head // // Get metadata for a file // // Gets the file or directory metadata. // // --- // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: path // description: Path to the file // type: string // example: default // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Raw file or directory listing // headers: // X-Incus-uid: // description: File owner UID // schema: // type: integer // X-Incus-gid: // description: File owner GID // schema: // type: integer // X-Incus-mode: // description: Mode mask // schema: // type: integer // X-Incus-modified: // description: Last modified date // schema: // type: string // X-Incus-type: // description: Type of file (file, symlink or directory) // schema: // type: string // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceFileHead(_ *state.State, inst instance.Instance, path string, _ *http.Request) response.Response { reverter := revert.New() defer reverter.Fail() // Get a SFTP client. client, err := inst.FileSFTP() if err != nil { return response.InternalError(err) } reverter.Add(func() { _ = client.Close() }) return fileSFTPHead(client, path) } // swagger:operation POST /1.0/instances/{name}/files instances instance_files_post // // Create or replace a file // // Creates a new file in the instance. // // --- // consumes: // - application/octet-stream // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: path // description: Path to the file // type: string // example: default // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: raw_file // description: Raw file content // - in: header // name: X-Incus-uid // description: File owner UID // schema: // type: integer // example: 1000 // - in: header // name: X-Incus-gid // description: File owner GID // schema: // type: integer // example: 1000 // - in: header // name: X-Incus-mode // description: File mode // schema: // type: integer // example: 0644 // - in: header // name: X-Incus-type // description: Type of file (file, symlink or directory) // schema: // type: string // example: file // - in: header // name: X-Incus-write // description: Write mode (overwrite or append) // schema: // type: string // example: overwrite // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceFilePost(s *state.State, inst instance.Instance, path string, r *http.Request) response.Response { // Get a SFTP client. client, err := inst.FileSFTP() if err != nil { return response.InternalError(err) } defer func() { _ = client.Close() }() return fileSFTPPost(client, path, r, func() { s.Events.SendLifecycle(inst.Project().Name, lifecycle.InstanceFilePushed.Event(inst, logger.Ctx{"path": path})) }) } // swagger:operation DELETE /1.0/instances/{name}/files instances instance_files_delete // // Delete a file // // Removes the file. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: path // description: Path to the file // type: string // example: default // - in: query // name: project // description: Project name // type: string // example: default // - in: header // name: X-Incus-force // description: Perform recursive deletion // schema: // type: boolean // example: true // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceFileDelete(s *state.State, inst instance.Instance, path string, r *http.Request) response.Response { // Get a SFTP client. client, err := inst.FileSFTP() if err != nil { return response.InternalError(err) } defer func() { _ = client.Close() }() return fileSFTPDelete(client, path, r, func() { s.Events.SendLifecycle(inst.Project().Name, lifecycle.InstanceFileDeleted.Event(inst, logger.Ctx{"path": path})) }) } func fileSFTPGet(client *sftp.Client, path string, r *http.Request, reverter *revert.Reverter, onSuccess func()) response.Response { // Get the file stats. stat, err := client.Lstat(path) if err != nil { return response.SmartError(err) } fileType := "file" if stat.Mode().IsDir() { fileType = "directory" } else if stat.Mode()&os.ModeSymlink == os.ModeSymlink { fileType = "symlink" } fileStat, ok := stat.Sys().(*sftp.FileStat) if !ok { return response.InternalError(fmt.Errorf("Unexpected stat type %T", stat.Sys())) } // Prepare the response. headers := map[string]string{ "X-Incus-uid": fmt.Sprintf("%d", fileStat.UID), "X-Incus-gid": fmt.Sprintf("%d", fileStat.GID), "X-Incus-mode": fmt.Sprintf("%04o", stat.Mode().Perm()), "X-Incus-modified": stat.ModTime().UTC().String(), "X-Incus-type": fileType, } switch fileType { case "file": // Open the file. file, err := client.Open(path) if err != nil { return response.SmartError(err) } reverter.Add(func() { _ = file.Close() }) // Setup cleanup logic. cleanup := reverter.Clone() reverter.Success() // Make a file response struct. files := make([]response.FileResponseEntry, 1) files[0].Identifier = filepath.Base(path) files[0].Filename = filepath.Base(path) files[0].File = file files[0].FileSize = stat.Size() files[0].FileModified = stat.ModTime() files[0].Cleanup = func() { cleanup.Fail() } onSuccess() return response.FileResponse(r, files, headers) case "symlink": // Find symlink target. target, err := client.ReadLink(path) if err != nil { return response.SmartError(err) } // If not an absolute symlink, need to mangle to something // relative to the source path. This is required because there // is no sftp function to get the final target path and RealPath doesn't // allow specifying the path to resolve from. if !strings.HasPrefix(target, "/") { target = filepath.Join(filepath.Dir(path), target) } // Convert to absolute path. target, err = client.RealPath(target) if err != nil { return response.SmartError(err) } // Make a file response struct. files := make([]response.FileResponseEntry, 1) files[0].Identifier = filepath.Base(path) files[0].Filename = filepath.Base(path) files[0].File = bytes.NewReader([]byte(target)) files[0].FileModified = time.Now() files[0].FileSize = int64(len(target)) onSuccess() return response.FileResponse(r, files, headers) case "directory": dirEnts := []string{} // List the directory. entries, err := client.ReadDir(path) if err != nil { return response.SmartError(err) } for _, entry := range entries { dirEnts = append(dirEnts, entry.Name()) } onSuccess() return response.SyncResponseHeaders(true, dirEnts, headers) } return response.InternalError(fmt.Errorf("Bad file type: %s", fileType)) } func fileSFTPHead(client *sftp.Client, path string) response.Response { // Get the file stats. stat, err := client.Lstat(path) if err != nil { return response.SmartError(err) } fileType := "file" if stat.Mode().IsDir() { fileType = "directory" } else if stat.Mode()&os.ModeSymlink == os.ModeSymlink { fileType = "symlink" } fileStat, ok := stat.Sys().(*sftp.FileStat) if !ok { return response.InternalError(fmt.Errorf("Unexpected stat type %T", stat.Sys())) } // Prepare the response. headers := map[string]string{ "X-Incus-uid": fmt.Sprintf("%d", fileStat.UID), "X-Incus-gid": fmt.Sprintf("%d", fileStat.GID), "X-Incus-mode": fmt.Sprintf("%04o", stat.Mode().Perm()), "X-Incus-modified": stat.ModTime().UTC().String(), "X-Incus-type": fileType, } if fileType == "file" { headers["Content-Type"] = "application/octet-stream" headers["Content-Length"] = fmt.Sprintf("%d", stat.Size()) } // Return an empty body (per RFC for HEAD). return response.ManualResponse(func(w http.ResponseWriter) error { // Set the headers. for k, v := range headers { w.Header().Set(k, v) } // Flush the connection. w.WriteHeader(http.StatusOK) return nil }) } func fileSFTPPost(client *sftp.Client, path string, r *http.Request, onSuccess func()) response.Response { // Extract file ownership and mode from headers uid, gid, mode, type_, write := api.ParseFileHeaders(r.Header) if !slices.Contains([]string{"overwrite", "append"}, write) { return response.BadRequest(fmt.Errorf("Bad file write mode: %s", write)) } // Check if the file already exists. _, err := client.Stat(path) exists := err == nil if type_ == "file" { fileMode := os.O_RDWR if write == "overwrite" { fileMode |= os.O_CREATE | os.O_TRUNC } // Open/create the file. file, err := client.OpenFile(path, fileMode) if err != nil { return response.SmartError(err) } defer func() { _ = file.Close() }() // Go to the end of the file. _, err = file.Seek(0, io.SeekEnd) if err != nil { return response.InternalError(err) } // Transfer the file into the instance. _, err = util.SafeCopy(file, r.Body) if err != nil { return response.InternalError(err) } if !exists { // Set file permissions. if mode >= 0 { err = file.Chmod(fs.FileMode(mode)) if err != nil { return response.SmartError(err) } } // Set file ownership. if uid >= 0 || gid >= 0 { err = file.Chown(int(uid), int(gid)) if err != nil { return response.SmartError(err) } } } onSuccess() return response.EmptySyncResponse } else if type_ == "symlink" { // Figure out target. target, err := io.ReadAll(r.Body) if err != nil { return response.InternalError(err) } // Check if already setup. currentTarget, err := client.ReadLink(path) if err == nil && currentTarget == string(target) { return response.EmptySyncResponse } // Create the symlink. err = client.Symlink(string(target), path) if err != nil { return response.SmartError(err) } onSuccess() return response.EmptySyncResponse } else if type_ == "directory" { // Check if it already exists. if exists { return response.EmptySyncResponse } // Create the directory. err = client.Mkdir(path) if err != nil { return response.SmartError(err) } // Set file permissions. if mode < 0 { // Default mode for directories (sftp doesn't know about umask). mode = 0o755 } err = client.Chmod(path, fs.FileMode(mode)) if err != nil { return response.SmartError(err) } // Set file ownership. if uid >= 0 || gid >= 0 { err = client.Chown(path, int(uid), int(gid)) if err != nil { return response.SmartError(err) } } onSuccess() return response.EmptySyncResponse } else { return response.BadRequest(fmt.Errorf("Bad file type: %s", type_)) } } func fileSFTPDelete(client *sftp.Client, path string, r *http.Request, onSuccess func()) response.Response { if util.IsFalseOrEmpty(r.Header.Get("X-Incus-force")) { // Delete the file. err := client.Remove(path) if err != nil { return response.SmartError(err) } } else { // Delete the tree. err := client.RemoveAll(path) if err != nil { return response.SmartError(err) } } onSuccess() return response.EmptySyncResponse } incus-7.0.0/cmd/incusd/instance_get.go000066400000000000000000000073251517523235500177000ustar00rootroot00000000000000package main import ( "errors" "net" "net/http" "net/url" "strconv" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" ) // swagger:operation GET /1.0/instances/{name} instances instance_get // // Get the instance // // Gets a specific instance (basic struct). // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Instance // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Instance" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/instances/{name}?recursion=1 instances instance_get_recursion1 // // Get the instance // // Gets a specific instance (full struct). // // recursion=1 also includes information about state, snapshots and backups. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Instance // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/InstanceFull" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Parse the recursion field recursionStr := r.FormValue("recursion") recursion, err := strconv.Atoi(recursionStr) if err != nil { recursion = 0 } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } c, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } var state any var etag any if recursion == 0 { state, etag, err = c.Render() } else { hostInterfaces, _ := net.Interfaces() state, etag, err = c.RenderFull(hostInterfaces) } if err != nil { return response.SmartError(err) } return response.SyncResponseETag(true, state, etag) } incus-7.0.0/cmd/incusd/instance_instance_types.go000066400000000000000000000145531517523235500221520ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "io" "net/http" "os" "strconv" "strings" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/server/task" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) type instanceType struct { // Amount of CPUs (can be a fraction) CPU float32 `yaml:"cpu"` // Amount of memory in GiB Memory float32 `yaml:"mem"` } var instanceTypes map[string]map[string]*instanceType func instanceSaveCache() error { if instanceTypes == nil { return nil } data, err := yaml.Dump(&instanceTypes, yaml.V2) if err != nil { return err } err = os.WriteFile(internalUtil.CachePath("instance_types.yaml"), data, 0o600) if err != nil { return err } return nil } func instanceLoadCache() error { if !util.PathExists(internalUtil.CachePath("instance_types.yaml")) { return nil } content, err := os.ReadFile(internalUtil.CachePath("instance_types.yaml")) if err != nil { return err } err = yaml.Load(content, &instanceTypes) if err != nil { return err } return nil } func instanceRefreshTypesTask(d *Daemon) (task.Func, task.Schedule) { f := func(ctx context.Context) { s := d.State() opRun := func(op *operations.Operation) error { return instanceRefreshTypes(ctx, s) } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.InstanceTypesUpdate, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating instance types update operation", logger.Ctx{"err": err}) return } logger.Info("Updating instance types") err = op.Start() if err != nil { logger.Error("Failed starting instance types update operation", logger.Ctx{"err": err}) return } err = op.Wait(ctx) if err != nil { logger.Error("Failed updating instance types", logger.Ctx{"err": err}) return } logger.Info("Done updating instance types") } return f, task.Daily() } func instanceRefreshTypes(ctx context.Context, s *state.State) error { // Attempt to download the new definitions downloadParse := func(filename string, target any) error { url := fmt.Sprintf("https://images.linuxcontainers.org/meta/instance-types/%s", filename) httpClient, err := localUtil.HTTPClient("", s.Proxy) if err != nil { return err } httpReq, err := http.NewRequest("GET", url, nil) if err != nil { return err } httpReq.Header.Set("User-Agent", version.UserAgent) cancelableRequest, ok := any(httpReq).(localUtil.ContextAwareRequest) if ok { httpReq = cancelableRequest.WithContext(ctx) } resp, err := httpClient.Do(httpReq) if err != nil { return err } if ctx.Err() != nil { return ctx.Err() } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return fmt.Errorf("Failed to get %s", url) } content, err := io.ReadAll(resp.Body) if err != nil { return err } err = yaml.Load(content, target) if err != nil { return err } return nil } // Set an initial value from the cache if instanceTypes == nil { instanceTypes = map[string]map[string]*instanceType{} } if len(instanceTypes) == 0 { err := instanceLoadCache() if err != nil { return err } } // Allow disabling instance type download. if util.IsTrue(os.Getenv("INCUS_SKIP_INSTANCE_TYPES")) { return nil } // Get the list of instance type sources sources := map[string]string{} err := downloadParse(".yaml", &sources) if err != nil { if !errors.Is(err, ctx.Err()) { logger.Warnf("Failed to update instance types: %v", err) } return err } // Parse the individual files newInstanceTypes := map[string]map[string]*instanceType{} for name, filename := range sources { types := map[string]*instanceType{} err = downloadParse(filename, &types) if err != nil { logger.Warnf("Failed to update instance types: %v", err) return err } newInstanceTypes[name] = types } // Update the global map instanceTypes = newInstanceTypes // And save in the cache err = instanceSaveCache() if err != nil { logger.Warnf("Failed to update instance types cache: %v", err) return err } return nil } func instanceParseType(value string) (map[string]string, error) { sourceName := "" sourceType := "" fields := strings.SplitN(value, ":", 2) // Check if the name of the source was provided if len(fields) != 2 { sourceType = value } else { sourceName = fields[0] sourceType = fields[1] } // If not, lets go look for a match if instanceTypes != nil && sourceName == "" { for name, types := range instanceTypes { _, ok := types[sourceType] if ok { if sourceName != "" { return nil, fmt.Errorf("Ambiguous instance type provided: %s", value) } sourceName = name } } } // Check if we have a limit for the provided value limits, ok := instanceTypes[sourceName][sourceType] if !ok { // Check if it's maybe just a resource limit if sourceName == "" && value != "" { newLimits := instanceType{} fields := strings.Split(value, "-") for _, field := range fields { if len(field) < 2 || (field[0] != 'c' && field[0] != 'm') { return nil, fmt.Errorf("Provided instance type doesn't exist: %s", value) } floatValue, err := strconv.ParseFloat(field[1:], 32) if err != nil { return nil, fmt.Errorf("Bad custom instance type: %s", value) } if field[0] == 'c' { newLimits.CPU = float32(floatValue) } else if field[0] == 'm' { newLimits.Memory = float32(floatValue) } } limits = &newLimits } if limits == nil { return nil, fmt.Errorf("Provided instance type doesn't exist: %s", value) } } out := map[string]string{} // Handle CPU if limits.CPU > 0 { cpuCores := int(limits.CPU) if float32(cpuCores) < limits.CPU { cpuCores++ } cpuTime := int(limits.CPU / float32(cpuCores) * 100.0) out["limits.cpu"] = fmt.Sprintf("%d", cpuCores) if cpuTime < 100 { out["limits.cpu.allowance"] = fmt.Sprintf("%d%%", cpuTime) } } // Handle memory if limits.Memory > 0 { rawLimit := int64(limits.Memory * 1024) out["limits.memory"] = fmt.Sprintf("%dMiB", rawLimit) } return out, nil } incus-7.0.0/cmd/incusd/instance_logs.go000066400000000000000000000423601517523235500200630ustar00rootroot00000000000000package main import ( "errors" "fmt" "net/http" "net/url" "os" "path/filepath" "slices" "strings" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/storage" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/revert" ) var instanceLogCmd = APIEndpoint{ Name: "instanceLog", Path: "instances/{name}/logs/{file}", Delete: APIEndpointAction{Handler: instanceLogDelete, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanEdit, "name")}, Get: APIEndpointAction{Handler: instanceLogGet, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanView, "name")}, } var instanceLogsCmd = APIEndpoint{ Name: "instanceLogs", Path: "instances/{name}/logs", Get: APIEndpointAction{Handler: instanceLogsGet, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanView, "name")}, } var instanceExecOutputCmd = APIEndpoint{ Name: "instanceExecOutput", Path: "instances/{name}/logs/exec-output/{file}", Delete: APIEndpointAction{Handler: instanceExecOutputDelete, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanExec, "name")}, Get: APIEndpointAction{Handler: instanceExecOutputGet, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanExec, "name")}, } var instanceExecOutputsCmd = APIEndpoint{ Name: "instanceExecOutputs", Path: "instances/{name}/logs/exec-output", Get: APIEndpointAction{Handler: instanceExecOutputsGet, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanExec, "name")}, } // swagger:operation GET /1.0/instances/{name}/logs instances instance_logs_get // // Get the log files // // Returns a list of log files (URLs). // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/instances/foo/logs/lxc.log" // ] // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceLogsGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Ensure instance exists. inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(d.State(), r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } result := []string{} dents, err := os.ReadDir(inst.LogPath()) if err != nil { return response.SmartError(err) } for _, f := range dents { if !validLogFileName(f.Name()) { continue } result = append(result, fmt.Sprintf("/%s/instances/%s/logs/%s", version.APIVersion, name, f.Name())) } return response.SyncResponse(true, result) } // swagger:operation GET /1.0/instances/{name}/logs/{filename} instances instance_log_get // // Get the log file // // Gets the log file. // // --- // produces: // - application/json // - application/octet-stream // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: path // name: filename // description: Log file name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Raw file // content: // application/octet-stream: // schema: // type: string // example: some-text // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceLogGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Ensure instance exists. inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } file, err := url.PathUnescape(mux.Vars(r)["file"]) if err != nil { return response.SmartError(err) } if !validLogFileName(file) { return response.BadRequest(fmt.Errorf("Log file name %q not valid", file)) } ent := response.FileResponseEntry{ Path: filepath.Join(inst.LogPath(), file), Filename: file, } s.Events.SendLifecycle(projectName, lifecycle.InstanceLogRetrieved.Event(file, inst, request.CreateRequestor(r), nil)) return response.FileResponse(r, []response.FileResponseEntry{ent}, nil) } // swagger:operation DELETE /1.0/instances/{name}/logs/{filename} instances instance_log_delete // // Delete the log file // // Removes the log file. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: path // name: filename // description: Log file name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceLogDelete(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Ensure instance exists. inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } file, err := url.PathUnescape(mux.Vars(r)["file"]) if err != nil { return response.SmartError(err) } if !validLogFileName(file) { return response.BadRequest(fmt.Errorf("Log file name %q not valid", file)) } if !strings.HasSuffix(file, ".log") || file == "lxc.log" || file == "qemu.log" { return response.BadRequest(errors.New("Only log files excluding qemu.log and lxc.log may be deleted")) } err = os.Remove(filepath.Join(inst.LogPath(), file)) if err != nil { return response.SmartError(err) } s.Events.SendLifecycle(projectName, lifecycle.InstanceLogDeleted.Event(file, inst, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation GET /1.0/instances/{name}/logs/exec-output instances instance_exec-outputs_get // // Get the exec record-output files // // Returns a list of exec record-output files (URLs). // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/instances/foo/logs/exec-output/exec_d0a89537-0617-4ed6-a79b-c2e88a970965.stdout", // "/1.0/instances/foo/logs/exec-output/exec_d0a89537-0617-4ed6-a79b-c2e88a970965.stderr", // ] // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceExecOutputsGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Ensure instance exists. inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(d.State(), r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } // Mount the instance's root volume pool, err := storage.LoadByInstance(s, inst) if err != nil { return response.SmartError(err) } _, err = pool.MountInstance(inst, nil) if err != nil { return response.SmartError(err) } defer func() { _ = pool.UnmountInstance(inst, nil) }() // Read exec record-output files dents, err := os.ReadDir(inst.ExecOutputPath()) if err != nil { return response.SmartError(err) } result := []string{} for _, f := range dents { if !validExecOutputFileName(f.Name()) { continue } result = append(result, fmt.Sprintf("/%s/instances/%s/logs/exec-output/%s", version.APIVersion, name, f.Name())) } return response.SyncResponse(true, result) } // swagger:operation GET /1.0/instances/{name}/logs/exec-output/{filename} instances instance_exec-output_get // // Get the exec-output log file // // Gets the exec-output file. // // --- // produces: // - application/json // - application/octet-stream // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: path // name: filename // description: Log file name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Raw file // content: // application/octet-stream: // schema: // type: string // example: some-text // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceExecOutputGet(d *Daemon, r *http.Request) response.Response { reverter := revert.New() defer reverter.Fail() s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Ensure instance exists. inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } file, err := url.PathUnescape(mux.Vars(r)["file"]) if err != nil { return response.SmartError(err) } if !validExecOutputFileName(file) { return response.BadRequest(fmt.Errorf("Exec record-output file name %q not valid", file)) } // Mount the instance's root volume pool, err := storage.LoadByInstance(s, inst) if err != nil { return response.SmartError(err) } _, err = pool.MountInstance(inst, nil) if err != nil { return response.SmartError(err) } reverter.Add(func() { _ = pool.UnmountInstance(inst, nil) }) cleanup := reverter.Clone() reverter.Success() ent := response.FileResponseEntry{ Path: filepath.Join(inst.ExecOutputPath(), file), Filename: file, Cleanup: cleanup.Fail, } s.Events.SendLifecycle(projectName, lifecycle.InstanceLogRetrieved.Event(file, inst, request.CreateRequestor(r), nil)) return response.FileResponse(r, []response.FileResponseEntry{ent}, nil) } // swagger:operation DELETE /1.0/instances/{name}/logs/exec-output/{filename} instances instance_exec-output_delete // // Delete the exec record-output file // // Removes the exec record-output file. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: path // name: filename // description: Log file name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceExecOutputDelete(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Ensure instance exists. inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } file, err := url.PathUnescape(mux.Vars(r)["file"]) if err != nil { return response.SmartError(err) } if !validExecOutputFileName(file) { return response.BadRequest(fmt.Errorf("Exec record-output file name %q not valid", file)) } // Mount the instance's root volume pool, err := storage.LoadByInstance(s, inst) if err != nil { return response.SmartError(err) } _, err = pool.MountInstance(inst, nil) if err != nil { return response.SmartError(err) } defer func() { _ = pool.UnmountInstance(inst, nil) }() err = os.Remove(filepath.Join(inst.ExecOutputPath(), file)) if err != nil { return response.SmartError(err) } s.Events.SendLifecycle(projectName, lifecycle.InstanceLogDeleted.Event(file, inst, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } func validLogFileName(fname string) bool { // Make sure that there's nothing fishy about the provided file name. if filepath.Base(fname) != fname { return false } /* Let's just require that the paths be relative, so that we don't have * to deal with any escaping or whatever. */ return slices.Contains([]string{"lxc.log", "qemu.log", "qemu.early.log", "qemu.qmp.log"}, fname) || strings.HasPrefix(fname, "migration_") || strings.HasPrefix(fname, "snapshot_") } func validExecOutputFileName(fName string) bool { // Make sure that there's nothing fishy about the provided file name. if filepath.Base(fName) != fName { return false } return (strings.HasSuffix(fName, ".stdout") || strings.HasSuffix(fName, ".stderr")) && strings.HasPrefix(fName, "exec_") } incus-7.0.0/cmd/incusd/instance_metadata.go000066400000000000000000000462211517523235500206770ustar00rootroot00000000000000package main import ( "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "os" "path/filepath" "strings" "github.com/gorilla/mux" "go.yaml.in/yaml/v4" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) // swagger:operation GET /1.0/instances/{name}/metadata instances instance_metadata_get // // Get the instance image metadata // // Gets the image metadata for the instance. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Image metadata // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/ImageMetadata" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceMetadataGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } // Load the container c, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Start the storage if needed pool, err := storagePools.LoadByInstance(s, c) if err != nil { return response.SmartError(err) } _, err = storagePools.InstanceMount(pool, c, nil) if err != nil { return response.SmartError(err) } defer func() { _ = storagePools.InstanceUnmount(pool, c, nil) }() // If missing, just return empty result metadataPath := filepath.Join(c.Path(), "metadata.yaml") if !util.PathExists(metadataPath) { return response.SyncResponse(true, api.ImageMetadata{}) } // Read the metadata metadataFile, err := os.Open(metadataPath) if err != nil { return response.InternalError(err) } defer func() { _ = metadataFile.Close() }() data, err := io.ReadAll(metadataFile) if err != nil { return response.InternalError(err) } // Parse into the API struct metadata := api.ImageMetadata{} err = yaml.Load(data, &metadata) if err != nil { return response.SmartError(err) } s.Events.SendLifecycle(projectName, lifecycle.InstanceMetadataRetrieved.Event(c, request.CreateRequestor(r), nil)) return response.SyncResponseETag(true, metadata, metadata) } // swagger:operation PATCH /1.0/instances/{name}/metadata instances instance_metadata_patch // // Partially update the image metadata // // Updates a subset of the instance image metadata. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: metadata // description: Image metadata // required: true // schema: // $ref: "#/definitions/ImageMetadata" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func instanceMetadataPatch(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to an instance on a different node. resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } // Load the instance. inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Start the storage if needed. pool, err := storagePools.LoadByInstance(s, inst) if err != nil { return response.SmartError(err) } _, err = storagePools.InstanceMount(pool, inst, nil) if err != nil { return response.SmartError(err) } defer func() { _ = storagePools.InstanceUnmount(pool, inst, nil) }() // Read the existing data. metadataPath := filepath.Join(inst.Path(), "metadata.yaml") metadata := api.ImageMetadata{} if util.PathExists(metadataPath) { metadataFile, err := os.Open(metadataPath) if err != nil { return response.InternalError(err) } defer func() { _ = metadataFile.Close() }() data, err := io.ReadAll(metadataFile) if err != nil { return response.InternalError(err) } // Parse into the API struct err = yaml.Load(data, &metadata) if err != nil { return response.SmartError(err) } } // Validate ETag err = localUtil.EtagCheck(r, metadata) if err != nil { return response.PreconditionFailed(err) } // Apply the new metadata on top. err = json.NewDecoder(r.Body).Decode(&metadata) if err != nil { return response.BadRequest(err) } // Update the file. return doInstanceMetadataUpdate(s, inst, metadata, r) } // swagger:operation PUT /1.0/instances/{name}/metadata instances instance_metadata_put // // Update the image metadata // // Updates the instance image metadata. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: metadata // description: Image metadata // required: true // schema: // $ref: "#/definitions/ImageMetadata" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func instanceMetadataPut(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to an instance on a different node. resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } // Read the new metadata. metadata := api.ImageMetadata{} err = json.NewDecoder(r.Body).Decode(&metadata) if err != nil { return response.BadRequest(err) } // Load the instance. inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Start the storage if needed. pool, err := storagePools.LoadByInstance(s, inst) if err != nil { return response.SmartError(err) } _, err = storagePools.InstanceMount(pool, inst, nil) if err != nil { return response.SmartError(err) } defer func() { _ = storagePools.InstanceUnmount(pool, inst, nil) }() return doInstanceMetadataUpdate(s, inst, metadata, r) } func doInstanceMetadataUpdate(s *state.State, inst instance.Instance, metadata api.ImageMetadata, r *http.Request) response.Response { // Convert YAML. data, err := yaml.Dump(metadata, yaml.V2) if err != nil { return response.BadRequest(err) } // Update the metadata. metadataPath := filepath.Join(inst.Path(), "metadata.yaml") err = os.WriteFile(metadataPath, data, 0o644) if err != nil { return response.InternalError(err) } s.Events.SendLifecycle(inst.Project().Name, lifecycle.InstanceMetadataUpdated.Event(inst, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation GET /1.0/instances/{name}/metadata/templates instances instance_metadata_templates_get // // Get the template file names or a specific // // If no path specified, returns a list of template file names. // If a path is specified, returns the file content. // // --- // produces: // - application/json // - application/octet-stream // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: path // description: Template name // type: string // example: hostname.tpl // responses: // "200": // description: Raw template file or file listing // content: // application/octet-stream: // schema: // type: string // example: some-text // application/json: // schema: // type: array // items: // type: string // example: |- // [ // "hostname.tpl", // "hosts.tpl" // ] // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceMetadataTemplatesGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } // Load the container c, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Start the storage if needed pool, err := storagePools.LoadByInstance(s, c) if err != nil { return response.SmartError(err) } _, err = storagePools.InstanceMount(pool, c, nil) if err != nil { return response.SmartError(err) } defer func() { _ = storagePools.InstanceUnmount(pool, c, nil) }() // Look at the request templateName := r.FormValue("path") if templateName == "" { templates := []string{} if !util.PathExists(filepath.Join(c.Path(), "templates")) { return response.SyncResponse(true, templates) } // List templates templatesPath := filepath.Join(c.Path(), "templates") entries, err := os.ReadDir(templatesPath) if err != nil { return response.InternalError(err) } for _, entry := range entries { if !entry.IsDir() { templates = append(templates, entry.Name()) } } return response.SyncResponse(true, templates) } // Check if the template exists templatePath, err := getContainerTemplatePath(c, templateName) if err != nil { return response.SmartError(err) } if !util.PathExists(templatePath) { return response.NotFound(fmt.Errorf("Template %q not found", templateName)) } // Create a temporary file with the template content (since the container // storage might not be available when the file is read from FileResponse) template, err := os.Open(templatePath) if err != nil { return response.SmartError(err) } defer func() { _ = template.Close() }() tempfile, err := os.CreateTemp("", "incus_template") if err != nil { return response.SmartError(err) } _, err = util.SafeCopy(tempfile, template) if err != nil { return response.InternalError(err) } err = tempfile.Close() if err != nil { return response.InternalError(err) } files := make([]response.FileResponseEntry, 1) files[0].Identifier = templateName files[0].Path = tempfile.Name() files[0].Filename = templateName files[0].Cleanup = func() { _ = os.Remove(tempfile.Name()) } s.Events.SendLifecycle(projectName, lifecycle.InstanceMetadataTemplateRetrieved.Event(c, request.CreateRequestor(r), logger.Ctx{"path": templateName})) return response.FileResponse(r, files, nil) } // swagger:operation POST /1.0/instances/{name}/metadata/templates instances instance_metadata_templates_post // // Create or replace a template file // // Creates a new image template file for the instance. // // --- // consumes: // - application/octet-stream // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: path // description: Template name // type: string // example: default // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: raw_file // description: Raw file content // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceMetadataTemplatesPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } // Load the container c, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Start the storage if needed pool, err := storagePools.LoadByInstance(s, c) if err != nil { return response.SmartError(err) } _, err = storagePools.InstanceMount(pool, c, nil) if err != nil { return response.SmartError(err) } defer func() { _ = storagePools.InstanceUnmount(pool, c, nil) }() // Look at the request templateName := r.FormValue("path") if templateName == "" { return response.BadRequest(errors.New("missing path argument")) } if !util.PathExists(filepath.Join(c.Path(), "templates")) { err := os.MkdirAll(filepath.Join(c.Path(), "templates"), 0o711) if err != nil { return response.SmartError(err) } } // Check if the template already exists templatePath, err := getContainerTemplatePath(c, templateName) if err != nil { return response.SmartError(err) } // Write the new template template, err := os.OpenFile(templatePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) if err != nil { return response.SmartError(err) } _, err = util.SafeCopy(template, r.Body) if err != nil { return response.InternalError(err) } err = template.Close() if err != nil { return response.InternalError(err) } s.Events.SendLifecycle(projectName, lifecycle.InstanceMetadataTemplateCreated.Event(c, request.CreateRequestor(r), logger.Ctx{"path": templateName})) return response.EmptySyncResponse } // swagger:operation DELETE /1.0/instances/{name}/metadata/templates instances instance_metadata_templates_delete // // Delete a template file // // Removes the template file. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: path // description: Template name // type: string // example: default // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceMetadataTemplatesDelete(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } // Load the container c, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Start the storage if needed pool, err := storagePools.LoadByInstance(s, c) if err != nil { return response.SmartError(err) } _, err = storagePools.InstanceMount(pool, c, nil) if err != nil { return response.SmartError(err) } defer func() { _ = storagePools.InstanceUnmount(pool, c, nil) }() // Look at the request templateName := r.FormValue("path") if templateName == "" { return response.BadRequest(errors.New("missing path argument")) } templatePath, err := getContainerTemplatePath(c, templateName) if err != nil { return response.SmartError(err) } if !util.PathExists(templatePath) { return response.NotFound(fmt.Errorf("Template %q not found", templateName)) } // Delete the template err = os.Remove(templatePath) if err != nil { return response.InternalError(err) } s.Events.SendLifecycle(projectName, lifecycle.InstanceMetadataTemplateDeleted.Event(c, request.CreateRequestor(r), logger.Ctx{"path": templateName})) return response.EmptySyncResponse } // Return the full path of a container template. func getContainerTemplatePath(c instance.Instance, filename string) (string, error) { if strings.Contains(filename, "/") { return "", errors.New("Invalid template filename") } return filepath.Join(c.Path(), "templates", filename), nil } incus-7.0.0/cmd/incusd/instance_patch.go000066400000000000000000000131071517523235500202130ustar00rootroot00000000000000package main import ( "bytes" "context" "encoding/json" "errors" "io" "net/http" "net/url" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" projecthelpers "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/osarch" ) // swagger:operation PATCH /1.0/instances/{name} instances instance_patch // // Partially update the instance // // Updates a subset of the instance configuration // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: instance // description: Update request // schema: // $ref: "#/definitions/InstancePut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instancePatch(d *Daemon, r *http.Request) response.Response { // Don't mess with instance while in setup mode. <-d.waitReady.Done() s := d.State() projectName := request.ProjectParam(r) // Get the container name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } unlock, err := instanceOperationLock(s.ShutdownCtx, projectName, name) if err != nil { return response.SmartError(err) } defer unlock() c, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Validate the ETag err = localUtil.EtagCheck(r, c.ETag()) if err != nil { return response.PreconditionFailed(err) } body, err := io.ReadAll(r.Body) if err != nil { return response.InternalError(err) } rdr1 := io.NopCloser(bytes.NewBuffer(body)) rdr2 := io.NopCloser(bytes.NewBuffer(body)) reqRaw := jmap.Map{} err = json.NewDecoder(rdr1).Decode(&reqRaw) if err != nil { return response.BadRequest(err) } req := api.InstancePut{} err = json.NewDecoder(rdr2).Decode(&req) if err != nil { return response.BadRequest(err) } if req.Restore != "" { return response.BadRequest(errors.New("Can't call PATCH in restore mode")) } // Check if architecture was passed var architecture int _, err = reqRaw.GetString("architecture") if err != nil { architecture = c.Architecture() } else { architecture, err = osarch.ArchitectureID(req.Architecture) if err != nil { architecture = 0 } } // Check if description was passed _, err = reqRaw.GetString("description") if err != nil { req.Description = c.Description() } // Check if ephemeral was passed _, err = reqRaw.GetBool("ephemeral") if err != nil { req.Ephemeral = c.IsEphemeral() } profileNames := make([]string, 0, len(c.Profiles())) for _, profile := range c.Profiles() { profileNames = append(profileNames, profile.Name) } // Check if profiles was passed if req.Profiles == nil { req.Profiles = profileNames } // Check if config was passed if req.Config == nil { req.Config = c.LocalConfig() } else { for k, v := range c.LocalConfig() { _, ok := req.Config[k] if !ok { req.Config[k] = v } } } // Check if devices was passed if req.Devices == nil { req.Devices = c.LocalDevices().CloneNative() } else { for k, v := range c.LocalDevices() { _, ok := req.Devices[k] if !ok { req.Devices[k] = v } } } // Check project limits. apiProfiles := make([]api.Profile, 0, len(req.Profiles)) err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { profiles, err := cluster.GetProfilesIfEnabled(ctx, tx.Tx(), projectName, req.Profiles) if err != nil { return err } profileConfigs, err := cluster.GetAllProfileConfigs(ctx, tx.Tx()) if err != nil { return err } profileDevices, err := cluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return err } for _, profile := range profiles { apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices) if err != nil { return err } apiProfiles = append(apiProfiles, *apiProfile) } return projecthelpers.AllowInstanceUpdate(tx, projectName, name, req, c.LocalConfig()) }) if err != nil { return response.SmartError(err) } // Update container configuration args := db.InstanceArgs{ Architecture: architecture, Config: req.Config, Description: req.Description, Devices: deviceConfig.NewDevices(req.Devices), Ephemeral: req.Ephemeral, Profiles: apiProfiles, Project: projectName, } err = c.Update(args, true) if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } incus-7.0.0/cmd/incusd/instance_post.go000066400000000000000000001042101517523235500200750ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "maps" "net/http" "net/url" "slices" "strings" "github.com/gorilla/mux" incus "github.com/lxc/incus/v7/client" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/cluster" clusterRequest "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/scriptlet" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" apiScriptlet "github.com/lxc/incus/v7/shared/api/scriptlet" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) // swagger:operation POST /1.0/instances/{name} instances instance_post // // Rename or move/migrate an instance // // Renames, moves an instance between pools or migrates an instance to another server. // // The returned operation metadata will vary based on what's requested. // For rename or move within the same server, this is a simple background operation with progress data. // For migration, in the push case, this will similarly be a background // operation with progress data, for the pull case, it will be a websocket // operation with a number of secrets to be passed to the target server. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: migration // description: Migration request // schema: // $ref: "#/definitions/InstancePost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instancePost(d *Daemon, r *http.Request) response.Response { s := d.State() // Don't mess with instance while in setup mode. <-d.waitReady.Done() // Parse the request URL. projectName := request.ProjectParam(r) target := request.QueryParam(r, "target") name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Quick checks. if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } if target != "" && !s.ServerClustered { return response.BadRequest(errors.New("Target only allowed when clustered")) } // Check if the server the instance is running on is currently online. var sourceMemberInfo *db.NodeInfo err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Load source node. sourceAddress, err := tx.GetNodeAddressOfInstance(ctx, projectName, name) if err != nil { return fmt.Errorf("Failed to get address of instance's member: %w", err) } if sourceAddress == "" { // Local node. return nil } info, err := tx.GetNodeByAddress(ctx, sourceAddress) if err != nil { return fmt.Errorf("Failed to get source member for %q: %w", sourceAddress, err) } sourceMemberInfo = &info return nil }) if err != nil { return response.SmartError(err) } // More checks. if target == "" && sourceMemberInfo != nil && sourceMemberInfo.IsOffline(s.GlobalConfig.OfflineThreshold()) { return response.BadRequest(errors.New("Can't perform action as server is currently offline")) } // Handle request forwarding. if sourceMemberInfo != nil && sourceMemberInfo.IsOffline(s.GlobalConfig.OfflineThreshold()) { // Current location of the instance isn't available and we've been asked to relocate it, forward to target. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } } else if target == "" || sourceMemberInfo == nil || !sourceMemberInfo.IsOffline(s.GlobalConfig.OfflineThreshold()) { // Forward the request to the instance's current location (if not local). resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } } // Parse the request. req := api.InstancePost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Target instance properties. instProject := projectName instLocation := target // Clear instance name if it's the same. if req.Name != "" && req.Name == name { req.Name = "" } // Validate the new target project (if provided). if req.Project != "" { // Confirm access to target project. err := s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectProject(req.Project), auth.EntitlementCanCreateInstances) if err != nil { return response.SmartError(err) } instProject = req.Project } // Validate the new instance name (if provided). if req.Name != "" { // Check the new instance name is valid. err = instance.ValidName(req.Name, false) if err != nil { return response.BadRequest(err) } // Check that the new isn't already in use. var id int err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Check that the name isn't already in use. id, _ = tx.GetInstanceID(ctx, instProject, req.Name) return nil }) if id > 0 { return response.Conflict(fmt.Errorf("Name %q already in use", req.Name)) } } // Load the local instance. inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Handle simple instance renaming. if !req.Migration { run := func(op *operations.Operation) error { inst.SetOperation(op) return inst.Rename(req.Name, true) } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", name)} op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.InstanceRename, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // Start handling migrations. if inst.IsSnapshot() { return response.BadRequest(errors.New("Instance snapshots cannot be moved on their own")) } // Checks for running instances. if inst.IsRunning() { if req.Pool != "" || req.Project != "" || target != "" { // Stateless migrations need the instance stopped. if !req.Live { return response.BadRequest(errors.New("Instance must be stopped to be moved statelessly")) } // Storage pool changes require a target flag. if req.Pool != "" { if inst.Type() != instancetype.VM { return response.BadRequest(errors.New("Live storage pool changes aren't supported for containers")) } if !s.ServerClustered { return response.BadRequest(errors.New("Live storage pool changes aren't supported on standalone systems")) } if target == "" { return response.BadRequest(errors.New("Live storage pool changes require the VM be moved to another cluster member")) } } // Project changes require a stopped instance. if req.Project != "" { return response.BadRequest(errors.New("Instance must be stopped to be moved across projects")) } // Name changes require a stopped instance. if req.Name != "" { return response.BadRequest(errors.New("Instance must be stopped to change their names")) } } } else { // Clear Live flag if instance isn't running. req.Live = false } // Check for offline sources. if sourceMemberInfo != nil && sourceMemberInfo.IsOffline(s.GlobalConfig.OfflineThreshold()) && (req.Pool != "" || req.Project != "" || req.Name != "") { return response.BadRequest(errors.New("Instance server is currently offline")) } // When in a cluster, default to keeping current location. if instLocation == "" && inst.Location() != "" { instLocation = inst.Location() } // If clustered, consider a new location for the instance. var targetMemberInfo *db.NodeInfo var targetCandidates []db.NodeInfo if s.ServerClustered && (target != "" || req.Project != "") { err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var targetGroupName string // Load the target project. p, err := dbCluster.GetProject(ctx, tx.Tx(), instProject) if err != nil { return err } targetProject, err := p.ToAPI(ctx, tx.Tx()) if err != nil { return err } // Load the cluster members. allMembers, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } // Check if the current location is fine. targetMemberInfo, _, err = project.CheckTarget(ctx, s.Authorizer, r, tx, targetProject, instLocation, allMembers) if err == nil && targetMemberInfo != nil { return nil } // If we must change location, validate access to requested member/group (if provided). targetMemberInfo, targetGroupName, err = project.CheckTarget(ctx, s.Authorizer, r, tx, targetProject, target, allMembers) if err != nil { return err } // If no specific server, get a list of allowed candidates. if targetMemberInfo == nil { clusterGroupsAllowed := project.GetRestrictedClusterGroups(targetProject) targetCandidates, err = tx.GetCandidateMembers(ctx, allMembers, []int{inst.Architecture()}, targetGroupName, clusterGroupsAllowed, s.GlobalConfig.OfflineThreshold()) if err != nil { return err } } return nil }) if err != nil { return response.SmartError(err) } // Run instance placement scriptlet if enabled. if s.GlobalConfig.InstancesPlacementScriptlet() != "" { // If a target was specified, limit the list of candidates to that target. if targetMemberInfo != nil { targetCandidates = []db.NodeInfo{*targetMemberInfo} } leaderAddress, err := s.Cluster.LeaderAddress() if err != nil { return response.InternalError(err) } // Load profiles. profileNames := make([]string, 0, len(inst.Profiles())) for _, profile := range inst.Profiles() { profileNames = append(profileNames, profile.Name) } profiles := make([]api.Profile, 0, len(profileNames)) if len(profileNames) > 0 { err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { profileFilters := make([]dbCluster.ProfileFilter, 0, len(profileNames)) for _, profileName := range profileNames { profileFilters = append(profileFilters, dbCluster.ProfileFilter{ Project: &instProject, Name: &profileName, }) } dbProfiles, err := dbCluster.GetProfiles(ctx, tx.Tx(), profileFilters...) if err != nil { return err } dbProfileConfigs, err := dbCluster.GetAllProfileConfigs(ctx, tx.Tx()) if err != nil { return err } dbProfileDevices, err := dbCluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return err } profilesByName := make(map[string]dbCluster.Profile, len(dbProfiles)) for _, dbProfile := range dbProfiles { profilesByName[dbProfile.Name] = dbProfile } for _, profileName := range profileNames { profile, found := profilesByName[profileName] if !found { return fmt.Errorf("Requested profile %q doesn't exist", profileName) } apiProfile, err := profile.ToAPI(ctx, tx.Tx(), dbProfileConfigs, dbProfileDevices) if err != nil { return err } profiles = append(profiles, *apiProfile) } return nil }) if err != nil { return response.SmartError(err) } } // Prepare the placement scriptlet context. req := apiScriptlet.InstancePlacement{ InstancesPost: api.InstancesPost{ Name: name, Type: api.InstanceTypeAny, InstancePut: api.InstancePut{ Config: db.ExpandInstanceConfig(inst.LocalConfig(), profiles), Devices: db.ExpandInstanceDevices(deviceConfig.NewDevices(inst.LocalDevices().CloneNative()), profiles).CloneNative(), Profiles: profileNames, }, }, Project: instProject, Reason: apiScriptlet.InstancePlacementReasonRelocation, } if targetMemberInfo == nil { // Get a new target. targetMemberInfo, err = scriptlet.InstancePlacementRun(r.Context(), logger.Log, s, &req, targetCandidates, leaderAddress) if err != nil { return response.BadRequest(fmt.Errorf("Failed instance placement scriptlet: %w", err)) } } else { // Validate the current target. _, err = scriptlet.InstancePlacementRun(r.Context(), logger.Log, s, &req, targetCandidates, leaderAddress) if err != nil { return response.BadRequest(fmt.Errorf("Failed instance placement scriptlet: %w", err)) } } } // If no member was selected yet, pick the member with the least number of instances. if targetMemberInfo == nil { var filteredCandidateMembers []db.NodeInfo // The instance might already be placed on the node with least number of instances. // Therefore remove it from the list of possible candidates if existent. for _, candidateMember := range targetCandidates { if candidateMember.Name != inst.Location() { filteredCandidateMembers = append(filteredCandidateMembers, candidateMember) } } if len(filteredCandidateMembers) == 0 { return response.InternalError(errors.New("Couldn't find a cluster member for the instance")) } targetMemberInfo = &filteredCandidateMembers[0] } if targetMemberInfo.IsOffline(s.GlobalConfig.OfflineThreshold()) { return response.BadRequest(errors.New("Target cluster member is offline")) } } // If the user requested a specific server group, make sure we can have it recorded. var targetGroupName string after, ok := strings.CutPrefix(target, targetGroupPrefix) if ok { targetGroupName = after } // Check that we're not requested to move to the same location we're currently on. if target != "" && targetMemberInfo.Name == inst.Location() { return response.BadRequest(errors.New("Requested target server is the same as current server")) } // If the instance needs to move, make sure it doesn't have backups. if targetMemberInfo != nil && targetMemberInfo.Name != inst.Location() { // Check if instance has backups. var backups []string err := s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { backups, err = tx.GetInstanceBackups(ctx, projectName, name) return err }) if err != nil { err = fmt.Errorf("Failed to fetch instance's backups: %w", err) return response.SmartError(err) } if len(backups) > 0 { return response.BadRequest(errors.New("Instances with backups cannot be moved")) } } // Server-side instance migration. if req.Pool != "" || req.Project != "" || target != "" { // Clear targetMemberInfo if no target change required. if targetMemberInfo != nil && inst.Location() == targetMemberInfo.Name { targetMemberInfo = nil } // Setup the instance move operation. run := func(op *operations.Operation) error { inst.SetOperation(op) return migrateInstance(context.TODO(), s, inst, req, sourceMemberInfo, targetMemberInfo, targetGroupName, op) } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", name)} op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.InstanceMigrate, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // Cross-server instance migration. ws, err := newMigrationSource(inst, req.Live, req.InstanceOnly, req.AllowInconsistent, "", "", req.Devices, req.Target) if err != nil { return response.InternalError(err) } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", name)} run := func(op *operations.Operation) error { return ws.do(op) } cancel := func(op *operations.Operation) error { ws.disconnect() return nil } if req.Target != nil { // Push mode. op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.InstanceMigrate, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // Pull mode. op, err := operations.OperationCreate(s, projectName, operations.OperationClassWebsocket, operationtype.InstanceMigrate, resources, ws.Metadata(), run, cancel, ws.Connect, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // Perform the server-side migration. func migrateInstance(ctx context.Context, s *state.State, inst instance.Instance, req api.InstancePost, sourceMemberInfo *db.NodeInfo, targetMemberInfo *db.NodeInfo, targetGroupName string, op *operations.Operation) error { // Load the instance storage pool. sourcePool, err := storagePools.LoadByInstance(s, inst) if err != nil { return fmt.Errorf("Failed loading instance storage pool: %w", err) } // Check that we're not requested to move to the same storage pool we're currently use. if req.Pool != "" && req.Pool == sourcePool.Name() { return errors.New("Requested storage pool is the same as current pool") } // Get the DB volume type for the instance. volType, err := storagePools.InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } volDBType, err := storagePools.VolumeTypeToDBType(volType) if err != nil { return err } // Handle migration of an instance away from an offline server (on shared storage). if targetMemberInfo != nil && sourceMemberInfo != nil && sourceMemberInfo.IsOffline(s.GlobalConfig.OfflineThreshold()) && sourcePool.Driver().Info().Remote { // Update the database records. err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { err := tx.UpdateInstanceNode(ctx, inst.Project().Name, inst.Name(), inst.Name(), targetMemberInfo.Name, sourcePool.ID(), volDBType) if err != nil { return fmt.Errorf("Failed updating cluster member to %q for instance %q: %w", targetMemberInfo.Name, inst.Name(), err) } return nil }) if err != nil { return fmt.Errorf("Failed to relink instance database data: %w", err) } // Import the instance into the storage. _, err = sourcePool.ImportInstance(inst, nil, nil) if err != nil { return fmt.Errorf("Failed creating mount point of instance on target node: %w", err) } // Perform any remaining instance rename. if req.Name != "" { err = inst.Rename(req.Name, true) if err != nil { return err } } // Record the new group name if needed. if targetGroupName != "" { err = inst.VolatileSet(map[string]string{"volatile.cluster.group": targetGroupName}) if err != nil { return err } } return nil } // Save the original value of the "volatile.apply_template" config key, // since we'll want to preserve it in the copied container. instVolatileApplyTemplate := inst.LocalConfig()["volatile.apply_template"] // Get the current instance info. instInfoRaw, _, err := inst.Render() if err != nil { return fmt.Errorf("Failed getting source instance info: %w", err) } targetInstInfo, ok := instInfoRaw.(*api.Instance) if !ok { return fmt.Errorf("Unexpected result from source instance render: %w", err) } // Apply the config overrides. maps.Copy(targetInstInfo.Config, req.Config) // If "security.secureboot" has changed, force a NVRAM reset. if util.IsTrueOrEmpty(inst.ExpandedConfig()["security.secureboot"]) != util.IsTrueOrEmpty(req.Config["security.secureboot"]) { targetInstInfo.Config["volatile.apply_nvram"] = "true" } // Apply the device overrides. maps.Copy(targetInstInfo.Devices, req.Devices) // Apply the profile overrides. if req.Profiles != nil { targetInstInfo.Profiles = req.Profiles } // Handle storage pool override. if req.Pool != "" { err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { _, err := tx.GetStoragePoolID(ctx, req.Pool) return err }) if response.IsNotFoundError(err) { return fmt.Errorf("Can't find a storage pool '%s' for the instance to use", req.Pool) } rootDevKey, rootDev, err := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if err != nil { return err } // Apply the override. rootDev["pool"] = req.Pool // Add the device to local config. targetInstInfo.Devices[rootDevKey] = rootDev } // Handle local changes (name, project, storage). // Handle the renames first. if req.Name != "" { err := inst.Rename(req.Name, true) if err != nil { return err } inst, err = instance.LoadByProjectAndName(s, inst.Project().Name, req.Name) if err != nil { return err } // Clear the rename part of the request. req.Name = "" } // Handle pool and project moves for stopped instances. if (req.Project != "" || req.Pool != "") && !req.Live && (targetMemberInfo == nil || inst.Location() == targetMemberInfo.Name) { // Get a local client. args := &incus.ConnectionArgs{ SkipGetServer: true, UserAgent: clusterRequest.UserAgentClient, } target, err := incus.ConnectIncusUnix(s.OS.GetUnixSocket(), args) if err != nil { return err } if targetMemberInfo != nil { target = target.UseTarget(targetMemberInfo.Name) } else if s.ServerClustered { target = target.UseTarget(inst.Location()) } targetProject := inst.Project().Name if req.Project != "" { targetProject = req.Project } target = target.UseProject(targetProject) // Check if we have a root disk in local config. _, _, err = internalInstance.GetRootDiskDevice(targetInstInfo.Devices) if err != nil && req.Project != "" { // If not and we're dealing with project copy, let's get one. var newRootDev map[string]string // Get current root disk. currentRootDevKey, currentRootDev, err := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if err != nil { return err } // Load the profiles. profiles := []api.Profile{} err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { rawProfiles, err := dbCluster.GetProfilesIfEnabled(ctx, tx.Tx(), targetProject, targetInstInfo.Profiles) if err != nil { return err } profileConfigs, err := dbCluster.GetAllProfileConfigs(ctx, tx.Tx()) if err != nil { return err } profileDevices, err := dbCluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return err } for _, profile := range rawProfiles { apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices) if err != nil { return err } profiles = append(profiles, *apiProfile) } return nil }) if err != nil { return err } // Go through expected profiles and look for a root disk. for _, profile := range profiles { _, dev, err := internalInstance.GetRootDiskDevice(profile.Devices) if err != nil { continue } newRootDev = dev break } // Check if root disk coming from profiles is suitable, if not, copy the current one. if newRootDev == nil || newRootDev["pool"] != currentRootDev["pool"] || newRootDev["size"] != currentRootDev["size"] || newRootDev["size.state"] != currentRootDev["size.state"] { targetInstInfo.Devices[currentRootDevKey] = currentRootDev } } // Use a temporary instance name if needed. targetInstName := inst.Name() if req.Project == "" { targetInstName, err = instance.MoveTemporaryName(inst) if err != nil { return err } } // Connect the event handler. _, err = target.GetEvents() if err != nil { return err } defer target.Disconnect() // Create the target instance. destOp, err := target.CreateInstance(api.InstancesPost{ Name: targetInstName, InstancePut: targetInstInfo.Writable(), Type: api.InstanceType(targetInstInfo.Type), Source: api.InstanceSource{ Type: "copy", Source: inst.Name(), Project: inst.Project().Name, InstanceOnly: req.InstanceOnly, }, }) if err != nil { return fmt.Errorf("Failed requesting instance create on destination: %w", err) } // Setup a progress handler. handler := func(newOp api.Operation) { _ = op.UpdateMetadata(newOp.Metadata) } _, err = destOp.AddHandler(handler) if err != nil { return err } // Wait for the migration to complete. err = destOp.Wait() if err != nil { return fmt.Errorf("Instance move to destination failed: %w", err) } // Delete the source instance. err = inst.Delete(true, false) if err != nil { return err } // If using a temporary name, rename it. if targetInstName != inst.Name() { op, err := target.RenameInstance(targetInstName, api.InstancePost{Name: inst.Name()}) if err != nil { return err } err = op.Wait() if err != nil { return err } } err = cleanupDependentDisks(s, inst, req.Devices, op) if err != nil { return fmt.Errorf("Failed deleting instance dependent volumes on source member: %w", err) } // Reload the instance. inst, err = instance.LoadByProjectAndName(s, targetProject, inst.Name()) if err != nil { return err } // Clear the pool and project part of the request. req.Pool = "" req.Project = "" } // Handle remote migrations (location and storage pool changes). if targetMemberInfo != nil && inst.Location() != targetMemberInfo.Name { // Get the client. networkCert := s.Endpoints.NetworkCert() target, err := cluster.Connect(targetMemberInfo.Address, networkCert, s.ServerCert(), nil, true) if err != nil { return fmt.Errorf("Failed to connect to destination server %q: %w", targetMemberInfo.Address, err) } target = target.UseProject(inst.Project().Name) target = target.UseTarget(targetMemberInfo.Name) // Get the source member info if missing. if sourceMemberInfo == nil { err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Get the source member info. srcMember, err := tx.GetNodeByName(ctx, inst.Location()) if err != nil { return fmt.Errorf("Failed getting current cluster member of instance %q", inst.Name()) } sourceMemberInfo = &srcMember return nil }) if err != nil { return err } } // Get the current instance snapshot list. snapshots, err := inst.Snapshots() if err != nil { return fmt.Errorf("Failed getting source instance snapshots: %w", err) } // Setup a new migration source. sourceMigration, err := newMigrationSource(inst, req.Live, false, req.AllowInconsistent, inst.Name(), req.Pool, req.Devices, nil) if err != nil { return fmt.Errorf("Failed setting up instance migration on source: %w", err) } run := func(_ *operations.Operation) error { return sourceMigration.do(op) } cancel := func(op *operations.Operation) error { sourceMigration.disconnect() return nil } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", inst.Name())} sourceOp, err := operations.OperationCreate(s, inst.Project().Name, operations.OperationClassWebsocket, operationtype.InstanceMigrate, resources, sourceMigration.Metadata(), run, cancel, sourceMigration.Connect, nil) if err != nil { return err } sourceOp.CopyRequestor(op) // Start the migration source. err = sourceOp.Start() if err != nil { return fmt.Errorf("Failed starting migration source operation: %w", err) } // Extract the migration secrets. sourceSecrets := make(map[string]string, len(sourceMigration.conns)) for connName, conn := range sourceMigration.conns { sourceSecrets[connName] = conn.Secret() } // Connect the event handler. _, err = target.GetEvents() if err != nil { return err } defer target.Disconnect() // Create the target instance. destOp, err := target.CreateInstance(api.InstancesPost{ Name: inst.Name(), InstancePut: targetInstInfo.Writable(), Type: api.InstanceType(targetInstInfo.Type), Source: api.InstanceSource{ Type: "migration", Mode: "pull", Operation: fmt.Sprintf("https://%s%s", sourceMemberInfo.Address, sourceOp.URL()), Websockets: sourceSecrets, Certificate: string(networkCert.PublicKey()), Live: req.Live, Source: inst.Name(), }, }) if err != nil { return fmt.Errorf("Failed requesting instance create on destination: %w", err) } // Setup a progress handler. handler := func(newOp api.Operation) { _ = op.UpdateMetadata(newOp.Metadata) } _, err = destOp.AddHandler(handler) if err != nil { return err } // Wait for the migration to complete. err = sourceOp.Wait(context.Background()) if err != nil { return fmt.Errorf("Instance move to destination failed on source: %w", err) } err = destOp.Wait() if err != nil { return fmt.Errorf("Instance move to destination failed: %w", err) } // Update the database post-migration. err = s.DB.Cluster.Transaction(context.Background(), func(ctx context.Context, tx *db.ClusterTx) error { // Update instance DB record to indicate its location on the new cluster member. err = tx.UpdateInstanceNode(ctx, inst.Project().Name, inst.Name(), inst.Name(), targetMemberInfo.Name, sourcePool.ID(), volDBType) if err != nil { return fmt.Errorf("Failed updating cluster member to %q for instance %q: %w", targetMemberInfo.Name, inst.Name(), err) } id, err := dbCluster.GetInstanceID(ctx, tx.Tx(), inst.Project().Name, inst.Name()) if err != nil { return fmt.Errorf("Failed to get ID of moved instance: %w", err) } // Set the cluster group record if needed. if targetGroupName != "" { err = tx.DeleteInstanceConfigKey(ctx, id, "volatile.cluster.group") if err != nil { return fmt.Errorf("Failed to remove volatile.cluster.group config key: %w", err) } err = tx.CreateInstanceConfig(ctx, int(id), map[string]string{"volatile.cluster.group": targetGroupName}) if err != nil { return fmt.Errorf("Failed to set volatile.apply_template config key: %w", err) } } else if targetMemberInfo != nil { config, err := dbCluster.GetInstanceConfig(ctx, tx.Tx(), inst.ID()) if err != nil { return err } // Remove 'volatile.cluster.group' if the instance is on a node outside the specified cluster group. group := config["volatile.cluster.group"] if group != "" { groupMembers, err := tx.GetClusterGroupNodes(ctx, group) if err != nil { return err } if !slices.Contains(groupMembers, targetMemberInfo.Name) { err = tx.DeleteInstanceConfigKey(ctx, id, "volatile.cluster.group") if err != nil { return fmt.Errorf("Failed to remove volatile.cluster.group config key: %w", err) } } } } // Restore the original value of "volatile.apply_template". err = tx.DeleteInstanceConfigKey(ctx, id, "volatile.apply_template") if err != nil { return fmt.Errorf("Failed to remove volatile.apply_template config key: %w", err) } if instVolatileApplyTemplate != "" { config := map[string]string{ "volatile.apply_template": instVolatileApplyTemplate, } err = tx.CreateInstanceConfig(ctx, int(id), config) if err != nil { return fmt.Errorf("Failed to set volatile.apply_template config key: %w", err) } } return nil }) if err != nil { return err } // Cleanup instance paths on source member if using remote shared storage // and there was no storage pool change. if sourcePool.Driver().Info().Remote && req.Pool == "" { err = sourcePool.CleanupInstancePaths(inst, nil) if err != nil { return fmt.Errorf("Failed cleaning up instance paths on source member: %w", err) } } else { // Delete the instance on source member if pool isn't remote shared storage. // We cannot use the normal delete process, as that would remove the instance DB record. // So instead we need to delete just the local storage volume(s) for the instance. snapshotCount := len(snapshots) for k := range snapshots { // Delete the snapshots in reverse order. k = snapshotCount - 1 - k err = sourcePool.DeleteInstanceSnapshot(snapshots[k], nil) if err != nil { return fmt.Errorf("Failed delete instance snapshot %q on source member: %w", snapshots[k].Name(), err) } } err = sourcePool.DeleteInstance(inst, nil) if err != nil { return fmt.Errorf("Failed deleting instance on source member: %w", err) } } err = cleanupDependentDisks(s, inst, req.Devices, op) if err != nil { return fmt.Errorf("Failed deleting instance dependent volumes on source member: %w", err) } } return nil } // cleanupDependentDisks removes dependent volumes from the source after migration if needed. func cleanupDependentDisks(s *state.State, inst instance.Instance, deviceOverrides api.DevicesMap, op *operations.Operation) error { err := inst.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error { diskPool, err := storagePools.LoadByName(s, dev.Config["pool"]) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } // If new disk was created than delete source volume. override, ok := deviceOverrides[dev.Name] if ok { if (override["source"] != "" && override["source"] != dev.Config["source"]) || (override["pool"] != "" && override["pool"] != dev.Config["pool"]) { _ = diskPool.DeleteCustomVolume(inst.Project().Name, dev.Config["source"], op) } } // Volumes on remote pools cannot be removed. if diskPool.Driver().Info().Remote { return nil } _ = diskPool.DeleteCustomVolume(inst.Project().Name, dev.Config["source"], op) return nil }) if err != nil { return err } return nil } incus-7.0.0/cmd/incusd/instance_put.go000066400000000000000000000146461517523235500177350ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/url" "github.com/google/uuid" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/operations" projecthelpers "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/revert" ) // swagger:operation PUT /1.0/instances/{name} instances instance_put // // Update the instance // // Updates the instance configuration or trigger a snapshot restore. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: instance // description: Update request // schema: // $ref: "#/definitions/InstancePut" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instancePut(d *Daemon, r *http.Request) response.Response { // Don't mess with instance while in setup mode. <-d.waitReady.Done() s := d.State() projectName := request.ProjectParam(r) // Get the container name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } reverter := revert.New() defer reverter.Fail() unlock, err := instanceOperationLock(s.ShutdownCtx, projectName, name) if err != nil { return response.SmartError(err) } reverter.Add(func() { unlock() }) inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Validate the ETag err = localUtil.EtagCheck(r, inst.ETag()) if err != nil { return response.PreconditionFailed(err) } configRaw := api.InstancePut{} err = json.NewDecoder(r.Body).Decode(&configRaw) if err != nil { return response.BadRequest(err) } architecture, err := osarch.ArchitectureID(configRaw.Architecture) if err != nil { architecture = 0 } var do func(*operations.Operation) error var opType operationtype.Type if configRaw.Restore == "" { // Check project limits. apiProfiles := make([]api.Profile, 0, len(configRaw.Profiles)) err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { profiles, err := cluster.GetProfilesIfEnabled(ctx, tx.Tx(), projectName, configRaw.Profiles) if err != nil { return err } profileConfigs, err := cluster.GetAllProfileConfigs(ctx, tx.Tx()) if err != nil { return err } profileDevices, err := cluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return err } for _, profile := range profiles { apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices) if err != nil { return err } apiProfiles = append(apiProfiles, *apiProfile) } return projecthelpers.AllowInstanceUpdate(tx, projectName, name, configRaw, inst.LocalConfig()) }) if err != nil { return response.SmartError(err) } // Update container configuration do = func(op *operations.Operation) error { inst.SetOperation(op) defer unlock() args := db.InstanceArgs{ Architecture: architecture, Config: configRaw.Config, Description: configRaw.Description, Devices: deviceConfig.NewDevices(configRaw.Devices), Ephemeral: configRaw.Ephemeral, Profiles: apiProfiles, Project: projectName, } err = inst.Update(args, true) if err != nil { return err } return nil } opType = operationtype.InstanceUpdate } else { // Snapshot Restore if configRaw.DiskOnly && configRaw.Stateful { return response.BadRequest(errors.New("Cannot use option disk_only and stateful together on snapshot restore")) } do = func(op *operations.Operation) error { defer unlock() return instanceSnapRestore(s, projectName, name, configRaw.Restore, configRaw.Stateful, configRaw.DiskOnly, op) } opType = operationtype.SnapshotRestore } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", name)} op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, opType, resources, nil, do, nil, nil, r) if err != nil { return response.InternalError(err) } reverter.Success() return operations.OperationResponse(op) } func instanceSnapRestore(s *state.State, projectName string, name string, snap string, stateful bool, diskOnly bool, op *operations.Operation) error { // normalize snapshot name if !internalInstance.IsSnapshot(snap) { snap = name + internalInstance.SnapshotDelimiter + snap } inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return err } inst.SetOperation(op) source, err := instance.LoadByProjectAndName(s, projectName, snap) if err != nil { switch { case response.IsNotFoundError(err): return fmt.Errorf("Snapshot %s does not exist", snap) default: return err } } source.SetOperation(op) // Generate a new `volatile.uuid.generation` to differentiate this instance restored from a snapshot from the original instance. source.LocalConfig()["volatile.uuid.generation"] = uuid.New().String() err = inst.Restore(source, stateful, diskOnly) if err != nil { return err } return nil } incus-7.0.0/cmd/incusd/instance_rebuild.go000066400000000000000000000106341517523235500205440ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/url" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // swagger:operation POST /1.0/instances/{name}/rebuild instances instance_rebuild_post // // Rebuild an instance // // Rebuild an instance using an alternate image or as empty. // --- // consumes: // - application/octet-stream // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: instance // description: InstanceRebuild request // required: true // schema: // $ref: "#/definitions/InstanceRebuildPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func instanceRebuildPost(d *Daemon, r *http.Request) response.Response { s := d.State() targetProjectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, targetProjectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } // Parse the request req := api.InstanceRebuildPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } var targetProject *api.Project var sourceImage *api.Image var inst instance.Instance var sourceImageRef string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := dbCluster.GetProject(ctx, tx.Tx(), targetProjectName) if err != nil { return fmt.Errorf("Failed loading project: %w", err) } targetProject, err = dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return err } dbInst, err := dbCluster.GetInstance(ctx, tx.Tx(), targetProject.Name, name) if err != nil { return fmt.Errorf("Failed loading instance: %w", err) } if req.Source.Type != "none" { sourceImage, err = getSourceImageFromInstanceSource(ctx, s, tx, targetProject.Name, req.Source, &sourceImageRef, dbInst.Type.String()) if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { return err } } return nil }) if err != nil { return response.SmartError(err) } inst, err = instance.LoadByProjectAndName(s, targetProject.Name, name) if err != nil { return response.SmartError(err) } if inst.IsRunning() { return response.BadRequest(errors.New("Instance must be stopped to be rebuilt")) } run := func(op *operations.Operation) error { if req.Source.Type == "none" { return instanceRebuildFromEmpty(inst, op) } if req.Source.Server != "" { sourceImage, err = ensureDownloadedImageFitWithinBudget(context.TODO(), s, r, op, *targetProject, sourceImageRef, req.Source, inst.Type().String()) if err != nil { return err } } if sourceImage == nil { return errors.New("Image not provided for instance rebuild") } return instanceRebuildFromImage(context.TODO(), s, r, inst, sourceImage, op) } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", name)} op, err := operations.OperationCreate(s, targetProject.Name, operations.OperationClassTask, operationtype.InstanceRebuild, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } incus-7.0.0/cmd/incusd/instance_sftp.go000066400000000000000000000044561517523235500200770ustar00rootroot00000000000000package main import ( "errors" "net" "net/http" "net/url" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/shared/api" ) // swagger:operation GET /1.0/instances/{name}/sftp instances instance_sftp // // Get the instance SFTP connection // // Upgrades the request to an SFTP connection of the instance's filesystem. // // --- // produces: // - application/json // - application/octet-stream // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // responses: // "101": // description: Switching protocols to SFTP // "400": // $ref: "#/responses/BadRequest" // "404": // $ref: "#/responses/NotFound" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceSFTPHandler(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) instName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(instName) { return response.BadRequest(errors.New("Invalid instance name")) } if r.Header.Get("Upgrade") != "sftp" { return response.SmartError(api.StatusErrorf(http.StatusBadRequest, "Missing or invalid upgrade header")) } // Forward the request if the instance is remote. client, err := cluster.ConnectIfInstanceIsRemote(s, projectName, instName, r) if err != nil { return response.SmartError(err) } // Redirect to correct server if needed. var conn net.Conn if client != nil { conn, err = client.GetInstanceFileSFTPConn(instName) if err != nil { return response.SmartError(err) } } else { inst, err := instance.LoadByProjectAndName(s, projectName, instName) if err != nil { return response.SmartError(err) } conn, err = inst.FileSFTPConn() if err != nil { return response.SmartError(api.StatusErrorf(http.StatusInternalServerError, "Failed getting instance SFTP connection: %v", err)) } } return response.UpgradeResponse(r, conn, "sftp", nil) } incus-7.0.0/cmd/incusd/instance_snapshot.go000066400000000000000000000547731517523235500207710ustar00rootroot00000000000000package main import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "time" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/validate" ) // swagger:operation GET /1.0/instances/{name}/snapshots instances instance_snapshots_get // // Get the snapshots // // Returns a list of instance snapshots (URLs). // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/instances/foo/snapshots/snap0", // "/1.0/instances/foo/snapshots/snap1" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/instances/{name}/snapshots?recursion=1 instances instance_snapshots_get_recursion1 // // Get the snapshots // // Returns a list of instance snapshots (structs). // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of instance snapshots // items: // $ref: "#/definitions/InstanceSnapshot" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceSnapshotsGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) cname, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(cname) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, cname) if err != nil { return response.SmartError(err) } if resp != nil { return resp } recursion := localUtil.IsRecursionRequest(r) resultString := []string{} resultMap := []*api.InstanceSnapshot{} if !recursion { var snaps []string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error snaps, err = tx.GetInstanceSnapshotsNames(ctx, projectName, cname) return err }) if err != nil { return response.SmartError(err) } for _, snap := range snaps { _, snapName, _ := api.GetParentAndSnapshotName(snap) if projectName == api.ProjectDefaultName { url := fmt.Sprintf("/%s/instances/%s/snapshots/%s", version.APIVersion, cname, snapName) resultString = append(resultString, url) } else { url := fmt.Sprintf("/%s/instances/%s/snapshots/%s?project=%s", version.APIVersion, cname, snapName, projectName) resultString = append(resultString, url) } } } else { c, err := instance.LoadByProjectAndName(s, projectName, cname) if err != nil { return response.SmartError(err) } snaps, err := c.Snapshots() if err != nil { return response.SmartError(err) } for _, snap := range snaps { render, _, err := snap.RenderWithUsage() if err != nil { continue } resultMap = append(resultMap, render.(*api.InstanceSnapshot)) } } if !recursion { return response.SyncResponse(true, resultString) } return response.SyncResponse(true, resultMap) } // swagger:operation POST /1.0/instances/{name}/snapshots instances instance_snapshots_post // // Create a snapshot // // Creates a new snapshot. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: snapshot // description: Snapshot request // required: false // schema: // $ref: "#/definitions/InstanceSnapshotsPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceSnapshotsPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := cluster.GetProject(context.Background(), tx.Tx(), projectName) if err != nil { return err } p, err := dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return err } err = project.AllowSnapshotCreation(p) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } /* * snapshot is a three step operation: * 1. choose a new name * 2. copy the database info over * 3. copy over the rootfs */ inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } req := api.InstanceSnapshotsPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } if req.Name == "" { req.Name, err = instance.NextSnapshotName(s, inst, "snap%d") if err != nil { return response.SmartError(err) } } // Validate the name err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid snapshot name: %w", err)) } var expiry time.Time if req.ExpiresAt != nil { expiry = *req.ExpiresAt } else { duration := inst.ExpandedConfig()["snapshots.expiry.manual"] if duration == "" { duration = inst.ExpandedConfig()["snapshots.expiry"] } expiry, err = internalInstance.GetExpiry(time.Now(), duration) if err != nil { return response.BadRequest(err) } } snapshot := func(op *operations.Operation) error { inst.SetOperation(op) return inst.Snapshot(req.Name, expiry, req.Stateful) } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", name)} resources["instances_snapshots"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", name, "snapshots", req.Name)} op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.SnapshotCreate, resources, nil, snapshot, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } func instanceSnapshotHandler(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) instName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } snapshotName, err := url.PathUnescape(mux.Vars(r)["snapshotName"]) if err != nil { return response.SmartError(err) } resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, instName) if err != nil { return response.SmartError(err) } if resp != nil { return resp } snapshotName, err = url.QueryUnescape(snapshotName) if err != nil { return response.SmartError(err) } snapInst, err := instance.LoadByProjectAndName(s, projectName, instName+internalInstance.SnapshotDelimiter+snapshotName) if err != nil { return response.SmartError(err) } switch r.Method { case "GET": return snapshotGet(snapInst) case "POST": return snapshotPost(s, r, snapInst) case "DELETE": return snapshotDelete(s, r, snapInst) case "PUT": return snapshotPut(s, r, snapInst) case "PATCH": return snapshotPatch(s, r, snapInst) default: return response.NotFound(fmt.Errorf("Method %q not found", r.Method)) } } // swagger:operation PATCH /1.0/instances/{name}/snapshots/{snapshot} instances instance_snapshot_patch // // Partially update snapshot // // Updates a subset of the snapshot config. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: path // name: snapshot // description: Snapshot name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: snapshot // description: Snapshot update // required: false // schema: // $ref: "#/definitions/InstanceSnapshotPut" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func snapshotPatch(s *state.State, r *http.Request, snapInst instance.Instance) response.Response { // Only expires_at is currently editable, so PATCH is equivalent to PUT. return snapshotPut(s, r, snapInst) } // swagger:operation PUT /1.0/instances/{name}/snapshots/{snapshot} instances instance_snapshot_put // // Update snapshot // // Updates the snapshot config. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: path // name: snapshot // description: Snapshot name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: snapshot // description: Snapshot update // required: false // schema: // $ref: "#/definitions/InstanceSnapshotPut" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func snapshotPut(s *state.State, r *http.Request, snapInst instance.Instance) response.Response { // Validate the ETag etag := []any{snapInst.ExpiryDate()} err := localUtil.EtagCheck(r, etag) if err != nil { return response.PreconditionFailed(err) } rj := jmap.Map{} err = json.NewDecoder(r.Body).Decode(&rj) if err != nil { return response.InternalError(err) } var do func(op *operations.Operation) error _, err = rj.GetString("expires_at") if err != nil { // Skip updating the snapshot since the requested key wasn't provided do = func(op *operations.Operation) error { return nil } } else { body, err := json.Marshal(rj) if err != nil { return response.InternalError(err) } configRaw := api.InstanceSnapshotPut{} err = json.Unmarshal(body, &configRaw) if err != nil { return response.BadRequest(err) } // Update instance configuration do = func(op *operations.Operation) error { snapInst.SetOperation(op) args := db.InstanceArgs{ Architecture: snapInst.Architecture(), Config: snapInst.LocalConfig(), Description: snapInst.Description(), Devices: snapInst.LocalDevices(), Ephemeral: snapInst.IsEphemeral(), Profiles: snapInst.Profiles(), Project: snapInst.Project().Name, ExpiryDate: configRaw.ExpiresAt, Type: snapInst.Type(), Snapshot: snapInst.IsSnapshot(), } err = snapInst.Update(args, false) if err != nil { return err } return nil } } opType := operationtype.SnapshotUpdate parentName, snapName, _ := api.GetParentAndSnapshotName(snapInst.Name()) resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", parentName)} resources["instances_snapshots"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", parentName, "snapshots", snapName)} op, err := operations.OperationCreate(s, snapInst.Project().Name, operations.OperationClassTask, opType, resources, nil, do, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // swagger:operation GET /1.0/instances/{name}/snapshots/{snapshot} instances instance_snapshot_get // // Get the snapshot // // Gets a specific instance snapshot. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: path // name: snapshot // description: Snapshot name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Instance snapshot // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/InstanceSnapshot" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func snapshotGet(snapInst instance.Instance) response.Response { render, _, err := snapInst.RenderWithUsage() if err != nil { return response.SmartError(err) } etag := []any{snapInst.ExpiryDate()} return response.SyncResponseETag(true, render.(*api.InstanceSnapshot), etag) } // swagger:operation POST /1.0/instances/{name}/snapshots/{snapshot} instances instance_snapshot_post // // Rename or move/migrate a snapshot // // Renames or migrates an instance snapshot to another server. // // The returned operation metadata will vary based on what's requested. // For rename or move within the same server, this is a simple background operation with progress data. // For migration, in the push case, this will similarly be a background // operation with progress data, for the pull case, it will be a websocket // operation with a number of secrets to be passed to the target server. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: path // name: snapshot // description: Snapshot name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: snapshot // description: Snapshot migration // required: false // schema: // $ref: "#/definitions/InstanceSnapshotPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func snapshotPost(s *state.State, r *http.Request, snapInst instance.Instance) response.Response { body, err := io.ReadAll(r.Body) if err != nil { return response.InternalError(err) } rdr1 := io.NopCloser(bytes.NewBuffer(body)) raw := jmap.Map{} err = json.NewDecoder(rdr1).Decode(&raw) if err != nil { return response.BadRequest(err) } parentName, snapName, _ := api.GetParentAndSnapshotName(snapInst.Name()) rdr2 := io.NopCloser(bytes.NewBuffer(body)) reqNew := api.InstanceSnapshotPost{} err = json.NewDecoder(rdr2).Decode(&reqNew) if err != nil { return response.BadRequest(err) } migration, err := raw.GetBool("migration") if err == nil && migration { rdr3 := io.NopCloser(bytes.NewBuffer(body)) req := api.InstancePost{} err = json.NewDecoder(rdr3).Decode(&req) if err != nil { return response.BadRequest(err) } if reqNew.Live { if parentName != reqNew.Name { return response.BadRequest(fmt.Errorf("Instance name cannot be changed during stateful copy (%q to %q)", parentName, reqNew.Name)) } } ws, err := newMigrationSource(snapInst, reqNew.Live, true, false, "", "", nil, req.Target) if err != nil { return response.SmartError(err) } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", parentName)} resources["instances_snapshots"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", parentName, "snapshots", snapName)} run := func(op *operations.Operation) error { ws.instance.SetOperation(op) return ws.do(op) } if req.Target != nil { // Push mode. op, err := operations.OperationCreate(s, snapInst.Project().Name, operations.OperationClassTask, operationtype.SnapshotTransfer, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // Pull mode. op, err := operations.OperationCreate(s, snapInst.Project().Name, operations.OperationClassWebsocket, operationtype.SnapshotTransfer, resources, ws.Metadata(), run, ws.Cancel, ws.Connect, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } else if !migration { if reqNew.Name == "" { return response.BadRequest(errors.New("A new name for the instance must be provided")) } } newName, err := raw.GetString("name") if err != nil { return response.BadRequest(err) } // Validate the name err = validate.IsAPIName(newName, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid snapshot name: %w", err)) } fullName := parentName + internalInstance.SnapshotDelimiter + newName err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Check that the name isn't already in use id, _ := tx.GetInstanceSnapshotID(ctx, snapInst.Project().Name, parentName, newName) if id > 0 { return fmt.Errorf("Name '%s' already in use", fullName) } return nil }) if err != nil { return response.Conflict(err) } rename := func(op *operations.Operation) error { snapInst.SetOperation(op) return snapInst.Rename(fullName, false) } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", parentName)} resources["instances_snapshots"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", parentName, "snapshots", snapName)} op, err := operations.OperationCreate(s, snapInst.Project().Name, operations.OperationClassTask, operationtype.SnapshotRename, resources, nil, rename, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // swagger:operation DELETE /1.0/instances/{name}/snapshots/{snapshot} instances instance_snapshot_delete // // Delete a snapshot // // Deletes the instance snapshot. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: path // name: snapshot // description: Snapshot name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func snapshotDelete(s *state.State, r *http.Request, snapInst instance.Instance) response.Response { remove := func(op *operations.Operation) error { snapInst.SetOperation(op) return snapInst.Delete(false, true) } parentName, snapName, _ := api.GetParentAndSnapshotName(snapInst.Name()) resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", parentName)} resources["instances_snapshots"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", parentName, "snapshots", snapName)} op, err := operations.OperationCreate(s, snapInst.Project().Name, operations.OperationClassTask, operationtype.SnapshotDelete, resources, nil, remove, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } incus-7.0.0/cmd/incusd/instance_state.go000066400000000000000000000154531517523235500202420ustar00rootroot00000000000000package main import ( "encoding/json" "errors" "fmt" "net" "net/http" "net/url" "time" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // swagger:operation GET /1.0/instances/{name}/state instances instance_state_get // // Get the runtime state // // Gets the runtime state of the instance. // // This is a reasonably expensive call as it causes code to be run // inside of the instance to retrieve the resource usage and network // information. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // responses: // "200": // description: State // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/InstanceState" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceState(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } c, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } hostInterfaces, _ := net.Interfaces() state, err := c.RenderState(hostInterfaces) if err != nil { return response.InternalError(err) } return response.SyncResponse(true, state) } // swagger:operation PUT /1.0/instances/{name}/state instances instance_state_put // // Change the state // // Changes the running state of the instance. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Instance name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: state // description: State // required: false // schema: // $ref: "#/definitions/InstanceStatePut" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instanceStatePut(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(name) { return response.BadRequest(errors.New("Invalid instance name")) } // Handle requests targeted to a container on a different node resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, name) if err != nil { return response.SmartError(err) } if resp != nil { return resp } req := api.InstanceStatePut{} // We default to -1 (i.e. no timeout) here instead of 0 (instant timeout). req.Timeout = -1 err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Check if the cluster member is evacuated. if s.ServerClustered && req.Action != "stop" && s.DB.Cluster.LocalNodeIsEvacuated() { return response.Forbidden(errors.New("Cluster member is evacuated")) } // Don't mess with instances while in setup mode. <-d.waitReady.Done() inst, err := instance.LoadByProjectAndName(s, projectName, name) if err != nil { return response.SmartError(err) } // Actually perform the change. opType, err := instanceActionToOpType(req.Action) if err != nil { return response.BadRequest(err) } do := func(op *operations.Operation) error { inst.SetOperation(op) return doInstanceStatePut(inst, req) } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", name)} op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, opType, resources, nil, do, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } func instanceActionToOpType(action string) (operationtype.Type, error) { switch internalInstance.InstanceAction(action) { case internalInstance.Start: return operationtype.InstanceStart, nil case internalInstance.Stop: return operationtype.InstanceStop, nil case internalInstance.Restart: return operationtype.InstanceRestart, nil case internalInstance.Freeze: return operationtype.InstanceFreeze, nil case internalInstance.Unfreeze: return operationtype.InstanceUnfreeze, nil default: return operationtype.Unknown, fmt.Errorf("Unknown action: '%s'", action) } } func doInstanceStatePut(inst instance.Instance, req api.InstanceStatePut) error { if req.Force { // A zero timeout indicates to do a forced stop/restart. req.Timeout = 0 } else if req.Timeout < 0 { // If no timeout requested set a high default shutdown timeout. This way if the instance does not // respond to shutdown request the operation lock won't linger forever. req.Timeout = 600 } timeout := time.Duration(req.Timeout) * time.Second switch internalInstance.InstanceAction(req.Action) { case internalInstance.Start: return inst.Start(req.Stateful) case internalInstance.Stop: if req.Stateful { return inst.Stop(req.Stateful) } else if req.Timeout == 0 { return inst.Stop(false) } else { return inst.Shutdown(timeout) } case internalInstance.Restart: return inst.Restart(timeout) case internalInstance.Freeze: return inst.Freeze() case internalInstance.Unfreeze: return inst.Unfreeze() } return fmt.Errorf("Unknown action: '%s'", req.Action) } incus-7.0.0/cmd/incusd/instance_test.go000066400000000000000000000364541517523235500201050ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "net" "testing" "time" "github.com/stretchr/testify/suite" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" storagePools "github.com/lxc/incus/v7/internal/server/storage" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/idmap" ) type containerTestSuite struct { daemonTestSuite } func (suite *containerTestSuite) TestContainer_ProfilesDefault() { args := db.InstanceArgs{ Type: instancetype.Container, Ephemeral: false, Name: "testFoo", } c, op, _, err := instance.CreateInternal(suite.d.State(), args, nil, true, true, false) suite.Req.Nil(err) op.Done(nil) defer func() { _ = c.Delete(true, true) }() profiles := c.Profiles() suite.Len( profiles, 1, "No default profile created on instanceCreateInternal.") suite.Equal( "default", profiles[0].Name, "First profile should be the default profile.") } func (suite *containerTestSuite) TestContainer_ProfilesMulti() { // Create an unprivileged profile err := suite.d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { profile := cluster.Profile{ Name: "unprivileged", Description: "unprivileged", Project: "default", } id, err := cluster.CreateProfile(ctx, tx.Tx(), profile) if err != nil { return err } err = cluster.CreateProfileConfig(ctx, tx.Tx(), id, map[string]string{"security.privileged": "true"}) if err != nil { return err } return err }) suite.Req.Nil(err, "Failed to create the unprivileged profile.") defer func() { _ = suite.d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return cluster.DeleteProfile(ctx, tx.Tx(), "default", "unprivileged") }) }() var testProfiles []api.Profile err = suite.d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { testProfiles, err = tx.GetProfiles(ctx, "default", []string{"default", "unprivileged"}) return err }) suite.Req.Nil(err) args := db.InstanceArgs{ Type: instancetype.Container, Ephemeral: false, Profiles: testProfiles, Name: "testFoo", } c, op, _, err := instance.CreateInternal(suite.d.State(), args, nil, true, true, false) suite.Req.Nil(err) op.Done(nil) defer func() { _ = c.Delete(true, true) }() profiles := c.Profiles() suite.Len( profiles, 2, "Didn't get both profiles in instanceCreateInternal.") suite.True( c.IsPrivileged(), "The container is not privileged (didn't apply the unprivileged profile?).") } func (suite *containerTestSuite) TestContainer_ProfilesOverwriteDefaultNic() { args := db.InstanceArgs{ Type: instancetype.Container, Ephemeral: false, Config: map[string]string{"security.privileged": "true"}, Devices: deviceConfig.Devices{ "eth0": deviceConfig.Device{ "type": "nic", "nictype": "bridged", "parent": "unknownbr0", }, }, Name: "testFoo", } err := suite.d.State().DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, err := tx.CreateNetwork(ctx, api.ProjectDefaultName, "unknownbr0", "", db.NetworkTypeBridge, nil) return err }) suite.Req.Nil(err) c, op, _, err := instance.CreateInternal(suite.d.State(), args, nil, true, true, false) suite.Req.Nil(err) op.Done(nil) suite.True(c.IsPrivileged(), "This container should be privileged.") out, _, err := c.Render() suite.Req.Nil(err) state := out.(*api.Instance) defer func() { _ = c.Delete(true, true) }() suite.Equal( "unknownbr0", state.Devices["eth0"]["parent"], "Container config doesn't overwrite profile config.") } func (suite *containerTestSuite) TestContainer_LoadFromDB() { args := db.InstanceArgs{ Type: instancetype.Container, Ephemeral: false, Config: map[string]string{"security.privileged": "true"}, Devices: deviceConfig.Devices{ "eth0": deviceConfig.Device{ "type": "nic", "nictype": "bridged", "parent": "unknownbr0", }, }, Name: "testFoo", } state := suite.d.State() err := state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, err := tx.CreateNetwork(ctx, api.ProjectDefaultName, "unknownbr0", "", db.NetworkTypeBridge, nil) return err }) suite.Req.Nil(err) // Create the container c, op, _, err := instance.CreateInternal(suite.d.State(), args, nil, true, true, false) suite.Req.Nil(err) op.Done(nil) defer func() { _ = c.Delete(true, true) }() poolName, err := c.StoragePool() suite.Req.Nil(err) pool, err := storagePools.LoadByName(state, poolName) suite.Req.Nil(err) err = state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, err = tx.CreateStoragePoolVolume(ctx, c.Project().Name, c.Name(), "", db.StoragePoolVolumeContentTypeFS, pool.ID(), nil, db.StoragePoolVolumeContentTypeFS, time.Now()) return err }) suite.Req.Nil(err) // Load the container and trigger initLXC() c2, err := instance.LoadByProjectAndName(state, "default", "testFoo") c2.IsRunning() suite.Req.Nil(err) hostInterfaces, _ := net.Interfaces() apiC1, etagC1, err := c.RenderFull(hostInterfaces) suite.Req.Nil(err) apiC2, etagC2, err := c2.RenderFull(hostInterfaces) suite.Req.Nil(err) suite.Equal(etagC1, etagC2) suite.Exactly( apiC1, apiC2, "The loaded container isn't exactly the same as the created one.", ) } func (suite *containerTestSuite) TestContainer_Path_Regular() { // Regular args := db.InstanceArgs{ Type: instancetype.Container, Ephemeral: false, Name: "testFoo", } c, op, _, err := instance.CreateInternal(suite.d.State(), args, nil, true, true, false) suite.Req.Nil(err) op.Done(nil) defer func() { _ = c.Delete(true, true) }() suite.Req.False(c.IsSnapshot(), "Shouldn't be a snapshot.") suite.Req.Equal(internalUtil.VarPath("containers", "testFoo"), c.Path()) suite.Req.Equal(internalUtil.VarPath("containers", "testFoo2"), storagePools.InstancePath(instancetype.Container, "default", "testFoo2", false)) } func (suite *containerTestSuite) TestContainer_LogPath() { args := db.InstanceArgs{ Type: instancetype.Container, Ephemeral: false, Name: "testFoo", } c, op, _, err := instance.CreateInternal(suite.d.State(), args, nil, true, true, false) suite.Req.Nil(err) op.Done(nil) defer func() { _ = c.Delete(true, true) }() suite.Req.Equal(internalUtil.VarPath("logs", "testFoo"), c.LogPath()) } func (suite *containerTestSuite) TestContainer_IsPrivileged_Privileged() { args := db.InstanceArgs{ Type: instancetype.Container, Ephemeral: false, Config: map[string]string{"security.privileged": "true"}, Name: "testFoo", } c, op, _, err := instance.CreateInternal(suite.d.State(), args, nil, true, true, false) suite.Req.Nil(err) op.Done(nil) suite.Req.True(c.IsPrivileged(), "This container should be privileged.") suite.Req.Nil(c.Delete(true, true), "Failed to delete the container.") } func (suite *containerTestSuite) TestContainer_AddRoutedNicValidation() { eth0 := deviceConfig.Device{ "name": "eth0", "type": "nic", "ipv4.gateway": "none", "ipv6.gateway": "none", "nictype": "routed", "parent": "unknownbr0", } eth1 := deviceConfig.Device{ "name": "eth1", "type": "nic", "ipv4.gateway": "none", "ipv6.gateway": "none", "nictype": "routed", "parent": "unknownbr0", } eth2 := deviceConfig.Device{"name": "eth2", "type": "nic", "nictype": "bridged", "parent": "unknownbr0"} var testProfiles []api.Profile err := suite.d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error testProfiles, err = tx.GetProfiles(ctx, "default", []string{"default"}) return err }) suite.Req.Nil(err) args := db.InstanceArgs{ Type: instancetype.Container, Profiles: testProfiles, Devices: deviceConfig.Devices{ "eth0": eth0, }, Name: "testFoo", } c, op, _, err := instance.CreateInternal(suite.d.State(), args, nil, true, true, false) suite.Req.NoError(err) op.Done(nil) err = c.Update(db.InstanceArgs{ Type: instancetype.Container, Profiles: testProfiles, Config: c.LocalConfig(), Devices: deviceConfig.Devices{ "eth0": eth0, "eth1": eth1, }, Name: "testFoo", }, true) suite.Req.NoError(err, errors.New("Adding multiple routed with gateway mode ['none'] should succeed. ")) eth0["ipv6.gateway"] = "auto" eth1["ipv6.gateway"] = "" err = c.Update(db.InstanceArgs{ Type: instancetype.Container, Profiles: testProfiles, Config: c.LocalConfig(), Devices: deviceConfig.Devices{ "eth0": eth0, "eth1": eth1, }, Name: "testFoo", }, true) suite.Req.Error(err, errors.New("Adding multiple routed nic devices with any gateway mmode ['auto',''] should throw error. ")) err = c.Update(db.InstanceArgs{ Type: instancetype.Container, Profiles: testProfiles, Config: c.LocalConfig(), Devices: deviceConfig.Devices{ "eth0": eth0, "eth2": eth2, }, Name: "testFoo", }, true) suite.Req.NoError(err, errors.New("Adding multiple nic devices with unicque nictype ['routed'] should throw error. ")) } func (suite *containerTestSuite) TestContainer_IsPrivileged_Unprivileged() { args := db.InstanceArgs{ Type: instancetype.Container, Ephemeral: false, Config: map[string]string{"security.privileged": "false"}, Name: "testFoo", } c, op, _, err := instance.CreateInternal(suite.d.State(), args, nil, true, true, false) suite.Req.Nil(err) op.Done(nil) suite.Req.False(c.IsPrivileged(), "This container should be unprivileged.") suite.Req.Nil(c.Delete(true, true), "Failed to delete the container.") } func (suite *containerTestSuite) TestContainer_Rename() { args := db.InstanceArgs{ Type: instancetype.Container, Ephemeral: false, Name: "testFoo", } c, op, _, err := instance.CreateInternal(suite.d.State(), args, nil, true, true, false) suite.Req.Nil(err) op.Done(nil) defer func() { _ = c.Delete(true, true) }() suite.Req.Nil(c.Rename("testFoo2", true), "Failed to rename the container.") suite.Req.Equal(internalUtil.VarPath("containers", "testFoo2"), c.Path()) } func (suite *containerTestSuite) TestContainer_findIdmap_isolated() { c1, op, _, err := instance.CreateInternal(suite.d.State(), db.InstanceArgs{ Type: instancetype.Container, Name: "isol-1", Config: map[string]string{ "security.idmap.isolated": "true", }, }, nil, true, true, false) suite.Req.Nil(err) op.Done(nil) defer func() { _ = c1.Delete(true, true) }() c2, op, _, err := instance.CreateInternal(suite.d.State(), db.InstanceArgs{ Type: instancetype.Container, Name: "isol-2", Config: map[string]string{ "security.idmap.isolated": "true", }, }, nil, true, true, false) suite.Req.Nil(err) op.Done(nil) defer func() { _ = c2.Delete(true, true) }() map1, err := c1.(instance.Container).NextIdmap() suite.Req.Nil(err) map2, err := c2.(instance.Container).NextIdmap() suite.Req.Nil(err) host := suite.d.os.IdmapSet.Entries[0] for i := range 2 { suite.Req.Equal(host.HostID+65536, map1.Entries[i].HostID, "hostids don't match %d", i) suite.Req.Equal(int64(0), map1.Entries[i].NSID, "nsid nonzero") suite.Req.Equal(int64(65536), map1.Entries[i].MapRange, "incorrect maprange") } for i := range 2 { suite.Req.Equal(host.HostID+65536*2, map2.Entries[i].HostID, "hostids don't match") suite.Req.Equal(int64(0), map2.Entries[i].NSID, "nsid nonzero") suite.Req.Equal(int64(65536), map2.Entries[i].MapRange, "incorrect maprange") } } func (suite *containerTestSuite) TestContainer_findIdmap_mixed() { c1, op, _, err := instance.CreateInternal(suite.d.State(), db.InstanceArgs{ Type: instancetype.Container, Name: "isol-1", Config: map[string]string{ "security.idmap.isolated": "false", }, }, nil, true, true, false) suite.Req.Nil(err) op.Done(nil) defer func() { _ = c1.Delete(true, true) }() c2, op, _, err := instance.CreateInternal(suite.d.State(), db.InstanceArgs{ Type: instancetype.Container, Name: "isol-2", Config: map[string]string{ "security.idmap.isolated": "true", }, }, nil, true, true, false) suite.Req.Nil(err) op.Done(nil) defer func() { _ = c2.Delete(true, true) }() map1, err := c1.(instance.Container).NextIdmap() suite.Req.Nil(err) map2, err := c2.(instance.Container).NextIdmap() suite.Req.Nil(err) host := suite.d.os.IdmapSet.Entries[0] for i := range 2 { suite.Req.Equal(host.HostID, map1.Entries[i].HostID, "hostids don't match %d", i) suite.Req.Equal(int64(0), map1.Entries[i].NSID, "nsid nonzero") suite.Req.Equal(host.MapRange, map1.Entries[i].MapRange, "incorrect maprange") } for i := range 2 { suite.Req.Equal(host.HostID+65536, map2.Entries[i].HostID, "hostids don't match") suite.Req.Equal(int64(0), map2.Entries[i].NSID, "nsid nonzero") suite.Req.Equal(int64(65536), map2.Entries[i].MapRange, "incorrect maprange") } } func (suite *containerTestSuite) TestContainer_findIdmap_raw() { c1, op, _, err := instance.CreateInternal(suite.d.State(), db.InstanceArgs{ Type: instancetype.Container, Name: "isol-1", Config: map[string]string{ "security.idmap.isolated": "false", "raw.idmap": "both 1000 1000", }, }, nil, true, true, false) suite.Req.Nil(err) op.Done(nil) defer func() { _ = c1.Delete(true, true) }() map1, err := c1.(instance.Container).NextIdmap() suite.Req.Nil(err) host := suite.d.os.IdmapSet.Entries[0] for _, i := range []int{0, 3} { suite.Req.Equal(host.HostID, map1.Entries[i].HostID, "hostids don't match") suite.Req.Equal(int64(0), map1.Entries[i].NSID, "nsid nonzero") suite.Req.Equal(int64(1000), map1.Entries[i].MapRange, "incorrect maprange") } suite.Req.Equal(int64(1000), map1.Entries[1].HostID, "hostids don't match") suite.Req.Equal(int64(1000), map1.Entries[1].NSID, "invalid nsid") suite.Req.Equal(int64(1), map1.Entries[1].MapRange, "incorrect maprange") for _, i := range []int{2, 4} { suite.Req.Equal(host.HostID+1001, map1.Entries[i].HostID, "hostids don't match") suite.Req.Equal(int64(1001), map1.Entries[i].NSID, "invalid nsid") suite.Req.Equal(host.MapRange-1000-1, map1.Entries[i].MapRange, "incorrect maprange") } } func (suite *containerTestSuite) TestContainer_findIdmap_maxed() { maps := []*idmap.Set{} instances := []instance.Instance{} for i := range 7 { c, op, _, err := instance.CreateInternal(suite.d.State(), db.InstanceArgs{ Type: instancetype.Container, Name: fmt.Sprintf("isol-%d", i), Config: map[string]string{ "security.idmap.isolated": "true", }, }, nil, true, true, false) instances = append(instances, c) /* we should fail if there are no ids left */ if i != 6 { suite.Req.Nil(err) } else { suite.Req.NotNil(err) return } op.Done(nil) m, err := c.(instance.Container).NextIdmap() suite.Req.Nil(err) maps = append(maps, m) } defer func() { for _, c := range instances { _ = c.Delete(true, true) } }() for i, m1 := range maps { for j, m2 := range maps { if m1 == m2 { continue } for _, e := range m2.Entries { suite.Req.False(m1.HostIDsIntersect(e), "%d and %d's idmaps intersect %v %v", i, j, m1, m2) } } } } func TestContainerTestSuite(t *testing.T) { suite.Run(t, &containerTestSuite{}) } incus-7.0.0/cmd/incusd/instances.go000066400000000000000000000511221517523235500172160ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "io/fs" "net/http" "os" "path/filepath" "runtime" "sort" "strconv" "sync" "time" "golang.org/x/sync/errgroup" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/warningtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/server/warnings" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) var instancesCmd = APIEndpoint{ Name: "instances", Path: "instances", Get: APIEndpointAction{Handler: instancesGet, AccessHandler: allowAuthenticated}, Post: APIEndpointAction{Handler: instancesPost, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanCreateInstances), LargeRequest: true}, Put: APIEndpointAction{Handler: instancesPut, AccessHandler: allowAuthenticated}, } var instanceCmd = APIEndpoint{ Name: "instance", Path: "instances/{name}", Get: APIEndpointAction{Handler: instanceGet, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanView, "name")}, Put: APIEndpointAction{Handler: instancePut, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanEdit, "name")}, Delete: APIEndpointAction{Handler: instanceDelete, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanEdit, "name")}, Post: APIEndpointAction{Handler: instancePost, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanEdit, "name")}, Patch: APIEndpointAction{Handler: instancePatch, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanEdit, "name")}, } var instanceRebuildCmd = APIEndpoint{ Name: "instanceRebuild", Path: "instances/{name}/rebuild", Post: APIEndpointAction{Handler: instanceRebuildPost, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanEdit, "name")}, } var instanceStateCmd = APIEndpoint{ Name: "instanceState", Path: "instances/{name}/state", Get: APIEndpointAction{Handler: instanceState, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanView, "name")}, Put: APIEndpointAction{Handler: instanceStatePut, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanUpdateState, "name")}, } var instanceSFTPCmd = APIEndpoint{ Name: "instanceFile", Path: "instances/{name}/sftp", Get: APIEndpointAction{Handler: instanceSFTPHandler, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanConnectSFTP, "name")}, } var instanceFileCmd = APIEndpoint{ Name: "instanceFile", Path: "instances/{name}/files", Get: APIEndpointAction{Handler: instanceFileHandler, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanAccessFiles, "name")}, Head: APIEndpointAction{Handler: instanceFileHandler, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanAccessFiles, "name")}, Post: APIEndpointAction{Handler: instanceFileHandler, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanAccessFiles, "name"), LargeRequest: true}, Delete: APIEndpointAction{Handler: instanceFileHandler, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanAccessFiles, "name")}, } var instanceSnapshotsCmd = APIEndpoint{ Name: "instanceSnapshots", Path: "instances/{name}/snapshots", Get: APIEndpointAction{Handler: instanceSnapshotsGet, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanView, "name")}, Post: APIEndpointAction{Handler: instanceSnapshotsPost, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanManageSnapshots, "name")}, } var instanceSnapshotCmd = APIEndpoint{ Name: "instanceSnapshot", Path: "instances/{name}/snapshots/{snapshotName}", Get: APIEndpointAction{Handler: instanceSnapshotHandler, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanView, "name")}, Post: APIEndpointAction{Handler: instanceSnapshotHandler, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanManageSnapshots, "name")}, Delete: APIEndpointAction{Handler: instanceSnapshotHandler, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanManageSnapshots, "name")}, Patch: APIEndpointAction{Handler: instanceSnapshotHandler, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanManageSnapshots, "name")}, Put: APIEndpointAction{Handler: instanceSnapshotHandler, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanManageSnapshots, "name")}, } var instanceConsoleCmd = APIEndpoint{ Name: "instanceConsole", Path: "instances/{name}/console", Get: APIEndpointAction{Handler: instanceConsoleLogGet, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanView, "name")}, Post: APIEndpointAction{Handler: instanceConsolePost, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanAccessConsole, "name")}, Delete: APIEndpointAction{Handler: instanceConsoleLogDelete, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanEdit, "name")}, } var instanceExecCmd = APIEndpoint{ Name: "instanceExec", Path: "instances/{name}/exec", Post: APIEndpointAction{Handler: instanceExecPost, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanExec, "name")}, } var instanceMetadataCmd = APIEndpoint{ Name: "instanceMetadata", Path: "instances/{name}/metadata", Get: APIEndpointAction{Handler: instanceMetadataGet, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanView, "name")}, Patch: APIEndpointAction{Handler: instanceMetadataPatch, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanEdit, "name")}, Put: APIEndpointAction{Handler: instanceMetadataPut, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanEdit, "name")}, } var instanceMetadataTemplatesCmd = APIEndpoint{ Name: "instanceMetadataTemplates", Path: "instances/{name}/metadata/templates", Get: APIEndpointAction{Handler: instanceMetadataTemplatesGet, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanView, "name")}, Post: APIEndpointAction{Handler: instanceMetadataTemplatesPost, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanEdit, "name")}, Delete: APIEndpointAction{Handler: instanceMetadataTemplatesDelete, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanEdit, "name")}, } var instanceBackupsCmd = APIEndpoint{ Name: "instanceBackups", Path: "instances/{name}/backups", Get: APIEndpointAction{Handler: instanceBackupsGet, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanView, "name")}, Post: APIEndpointAction{Handler: instanceBackupsPost, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanManageBackups, "name")}, } var instanceBackupCmd = APIEndpoint{ Name: "instanceBackup", Path: "instances/{name}/backups/{backupName}", Get: APIEndpointAction{Handler: instanceBackupGet, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanView, "name")}, Post: APIEndpointAction{Handler: instanceBackupPost, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanManageBackups, "name")}, Delete: APIEndpointAction{Handler: instanceBackupDelete, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanManageBackups, "name")}, } var instanceBackupExportCmd = APIEndpoint{ Name: "instanceBackupExport", Path: "instances/{name}/backups/{backupName}/export", Get: APIEndpointAction{Handler: instanceBackupExportGet, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanManageBackups, "name")}, } var instanceBitmapsCmd = APIEndpoint{ Name: "instanceBitmaps", Path: "instances/{name}/bitmaps", Post: APIEndpointAction{Handler: instanceBitmapsPost, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanEdit, "name")}, } var instanceAccessCmd = APIEndpoint{ Name: "access", Path: "instances/{name}/access", Get: APIEndpointAction{Handler: instanceAccess, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanView, "name")}, } var instanceDebugMemoryCmd = APIEndpoint{ Name: "instanceDebugMemory", Path: "instances/{name}/debug/memory", Get: APIEndpointAction{Handler: instanceDebugMemoryGet, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanEdit, "name")}, } var instanceDebugRepairCmd = APIEndpoint{ Name: "instanceDebugRepair", Path: "instances/{name}/debug/repair", Post: APIEndpointAction{Handler: instanceDebugRepairPost, AccessHandler: allowPermission(auth.ObjectTypeInstance, auth.EntitlementCanEdit, "name")}, } type instanceAutostartList []instance.Instance func (slice instanceAutostartList) Len() int { return len(slice) } func (slice instanceAutostartList) Less(i, j int) bool { iOrder := slice[i].ExpandedConfig()["boot.autostart.priority"] jOrder := slice[j].ExpandedConfig()["boot.autostart.priority"] if iOrder != jOrder { iOrderInt, _ := strconv.Atoi(iOrder) jOrderInt, _ := strconv.Atoi(jOrder) return iOrderInt > jOrderInt } return slice[i].Name() < slice[j].Name() } func (slice instanceAutostartList) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] } var instancesStartMu sync.Mutex // bulkStartInstances splits instances into those that can be started in parallel // and those that must be started sequentially. func bulkStartInstances(instances []instance.Instance) ([]instance.Instance, []instance.Instance) { var bulkStart, rest []instance.Instance for _, inst := range instances { config := inst.ExpandedConfig() autoStartDelay := config["boot.autostart.delay"] autoStartPriority := config["boot.autostart.priority"] if autoStartDelay == "" && autoStartPriority == "" { bulkStart = append(bulkStart, inst) } else { rest = append(rest, inst) } } return bulkStart, rest } // instanceShouldAutoStart returns whether the instance should be auto-started. // Returns true if boot.autostart is enabled or boot.autostart is not set and instance was previously running. func instanceShouldAutoStart(inst instance.Instance) bool { config := inst.ExpandedConfig() autoStart := config["boot.autostart"] lastState := config["volatile.last_state.power"] return util.IsTrue(autoStart) || ((autoStart == "" || autoStart == "last-state") && lastState == instance.PowerStateRunning) } // instanceStart starts a single instance. func instanceStart(s *state.State, inst instance.Instance) error { // Let's make up to 3 attempts to start instances. maxAttempts := 3 if !instanceShouldAutoStart(inst) { return nil } // If already running, we're done. if inst.IsRunning() { return nil } // Get the instance config. config := inst.ExpandedConfig() autoStartDelay := config["boot.autostart.delay"] shutdownAction := config["boot.host_shutdown_action"] instLogger := logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) // Try to start the instance. attempt := 0 for { attempt++ var err error if shutdownAction == "stateful-stop" { // Attempt to restore state. err = inst.Start(true) } else { // Normal startup. err = inst.Start(false) } if err != nil { if api.StatusErrorCheck(err, http.StatusServiceUnavailable) { break // Don't log or retry instances that are not ready to start yet. } instLogger.Warn("Failed auto start instance attempt", logger.Ctx{"attempt": attempt, "maxAttempts": maxAttempts, "err": err}) if attempt >= maxAttempts { warnErr := s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { // If unable to start after 3 tries, record a warning. return tx.UpsertWarningLocalNode(ctx, inst.Project().Name, cluster.TypeInstance, inst.ID(), warningtype.InstanceAutostartFailure, fmt.Sprintf("%v", err)) }) if warnErr != nil { instLogger.Warn("Failed to create instance autostart failure warning", logger.Ctx{"err": warnErr}) } instLogger.Error("Failed to auto start instance", logger.Ctx{"err": err}) break } time.Sleep(5 * time.Second) continue } // Resolve any previous warning. warnErr := warnings.ResolveWarningsByLocalNodeAndProjectAndTypeAndEntity(s.DB.Cluster, inst.Project().Name, warningtype.InstanceAutostartFailure, cluster.TypeInstance, inst.ID()) if warnErr != nil { instLogger.Warn("Failed to resolve instance autostart failure warning", logger.Ctx{"err": warnErr}) } // Wait the auto-start delay if set. autoStartDelayInt, err := strconv.Atoi(autoStartDelay) if err == nil { time.Sleep(time.Duration(autoStartDelayInt) * time.Second) } break } return nil } func instancesStart(s *state.State, instances []instance.Instance) { // Check if the cluster is currently evacuated. if s.ServerClustered && s.DB.Cluster.LocalNodeIsEvacuated() { return } // Acquire startup lock. instancesStartMu.Lock() defer instancesStartMu.Unlock() bulkInstances, sequentialInstances := bulkStartInstances(instances) // Limit the number of concurrent tasks. numParallel := max(runtime.NumCPU()/4, 1) group := new(errgroup.Group) group.SetLimit(numParallel) // Start instances that support bulk startup. for _, inst := range bulkInstances { i := inst group.Go(func() error { _ = instanceStart(s, i) return nil }) } _ = group.Wait() // Sort based on instance boot priority. sort.Sort(instanceAutostartList(sequentialInstances)) for _, inst := range sequentialInstances { _ = instanceStart(s, inst) } } type instanceStopList []instance.Instance func (slice instanceStopList) Len() int { return len(slice) } func (slice instanceStopList) Less(i, j int) bool { iOrder := slice[i].ExpandedConfig()["boot.stop.priority"] jOrder := slice[j].ExpandedConfig()["boot.stop.priority"] if iOrder != jOrder { iOrderInt, _ := strconv.Atoi(iOrder) jOrderInt, _ := strconv.Atoi(jOrder) return iOrderInt > jOrderInt // check this line (prob <) } return slice[i].Name() < slice[j].Name() } func (slice instanceStopList) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] } // Return all local instances on disk (if instance is running, it will attempt to populate the instance's local // and expanded config using the backup.yaml file). It will clear the instance's profiles property to avoid needing // to enrich them from the database. func instancesOnDisk(s *state.State) ([]instance.Instance, error) { var err error instancePaths := map[instancetype.Type]string{ instancetype.Container: internalUtil.VarPath("containers"), instancetype.VM: internalUtil.VarPath("virtual-machines"), } instanceTypeNames := make(map[instancetype.Type][]os.DirEntry, 2) instanceTypeNames[instancetype.Container], err = os.ReadDir(instancePaths[instancetype.Container]) if err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, err } instanceTypeNames[instancetype.VM], err = os.ReadDir(instancePaths[instancetype.VM]) if err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, err } instances := make([]instance.Instance, 0, len(instanceTypeNames[instancetype.Container])+len(instanceTypeNames[instancetype.VM])) for instanceType, instanceNames := range instanceTypeNames { for _, file := range instanceNames { // Convert file name to project name and instance name. projectName, instanceName := project.InstanceParts(file.Name()) var inst instance.Instance // Try and parse the backup file (if instance is running). // This allows us to stop VMs which require access to the vsock ID and volatile UUID. // Also generally it ensures that all devices are stopped cleanly too. backupYamlPath := filepath.Join(instancePaths[instanceType], file.Name(), "backup.yaml") if util.PathExists(backupYamlPath) { inst, err = instance.LoadFromBackup(s, projectName, filepath.Join(instancePaths[instanceType], file.Name()), false) if err != nil { logger.Warn("Failed loading instance", logger.Ctx{"project": projectName, "instance": instanceName, "backup_file": backupYamlPath, "err": err}) } } if inst == nil { // Initialize dbArgs with a very basic config. // This will not be sufficient to stop an instance cleanly. instDBArgs := &db.InstanceArgs{ Type: instanceType, Project: projectName, Name: instanceName, Config: make(map[string]string), } emptyProject := api.Project{ Name: projectName, } inst, err = instance.Load(s, *instDBArgs, emptyProject) if err != nil { logger.Warn("Failed loading instance", logger.Ctx{"project": projectName, "instance": instanceName, "err": err}) continue } } instances = append(instances, inst) } } return instances, nil } func instancesShutdown(instances []instance.Instance) { sort.Sort(instanceStopList(instances)) // Limit shutdown concurrency to number of instances or number of CPU cores (which ever is less). var wg sync.WaitGroup instShutdownCh := make(chan instance.Instance) maxConcurrent := runtime.NumCPU() instCount := len(instances) if instCount < maxConcurrent { maxConcurrent = instCount } for range maxConcurrent { go func(instShutdownCh <-chan instance.Instance) { for inst := range instShutdownCh { // Determine how long to wait for the instance to shutdown cleanly. timeoutSeconds := 30 value, ok := inst.ExpandedConfig()["boot.host_shutdown_timeout"] if ok { timeoutSeconds, _ = strconv.Atoi(value) } // Shutdown the instance. func() { // Save and restore the ephemeral bit (if DB is available). if inst.ID() > 0 { ephemeral := inst.IsEphemeral() if ephemeral { // Unset ephemeral flag if present. args := db.InstanceArgs{ Architecture: inst.Architecture(), Config: inst.LocalConfig(), Description: inst.Description(), Devices: inst.LocalDevices(), Ephemeral: false, Profiles: inst.Profiles(), Project: inst.Project().Name, Type: inst.Type(), Snapshot: inst.IsSnapshot(), } err := inst.Update(args, false) if err == nil { // On function return, set the flag back on. defer func() { args.Ephemeral = ephemeral _ = inst.Update(args, false) }() } } } // Perform the shutdown action. action := inst.ExpandedConfig()["boot.host_shutdown_action"] switch action { case "stateful-stop": err := inst.Stop(true) if err != nil { logger.Warn("Failed statefully stopping instance", logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "err": err}) } case "force-stop": err := inst.Stop(false) if err != nil { logger.Warn("Failed forcefully stopping instance", logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "err": err}) } default: err := inst.Shutdown(time.Second * time.Duration(timeoutSeconds)) if err != nil { logger.Warn("Failed shutting down instance, forcefully stopping", logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "err": err}) err = inst.Stop(false) if err != nil { logger.Warn("Failed forcefully stopping instance", logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "err": err}) } } } }() if inst.ID() > 0 { // If DB was available then the instance shutdown process will have set // the last power state to STOPPED, so set that back to RUNNING so that // when the daemon restarts the instance will be started again. _ = inst.VolatileSet(map[string]string{"volatile.last_state.power": instance.PowerStateRunning}) } wg.Done() } }(instShutdownCh) } var currentBatchPriority int for i, inst := range instances { // Skip stopped instances. if !inst.IsRunning() { continue } priority, _ := strconv.Atoi(inst.ExpandedConfig()["boot.stop.priority"]) // Shutdown instances in priority batches, logging at the start of each batch. if i == 0 || priority != currentBatchPriority { currentBatchPriority = priority // Wait for instances with higher priority to finish before starting next batch. wg.Wait() logger.Info("Stopping instances", logger.Ctx{"stopPriority": currentBatchPriority}) } wg.Add(1) instShutdownCh <- inst } wg.Wait() close(instShutdownCh) } incus-7.0.0/cmd/incusd/instances_get.go000066400000000000000000000360171517523235500200630ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "net" "net/http" "runtime" "sort" "strconv" "sync" "time" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) // swagger:operation GET /1.0/instances instances instances_get // // Get the instances // // Returns a list of instances (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // - in: query // name: all-projects // description: Retrieve instances from all projects // type: boolean // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/instances/foo", // "/1.0/instances/bar" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/instances?recursion=1 instances instances_get_recursion1 // // Get the instances // // Returns a list of instances (basic structs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // - in: query // name: all-projects // description: Retrieve instances from all projects // type: boolean // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of instances // items: // $ref: "#/definitions/Instance" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/instances?recursion=2 instances instances_get_recursion2 // // Get the instances // // Returns a list of instances (full structs). // // The main difference between recursion=1 and recursion=2 is that the // latter also includes state and snapshot information allowing for a // single API call to return everything needed by most clients. // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // - in: query // name: all-projects // description: Retrieve instances from all projects // type: boolean // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of instances // items: // $ref: "#/definitions/InstanceFull" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instancesGet(d *Daemon, r *http.Request) response.Response { s := d.State() resultFullList := []*api.InstanceFull{} resultMu := sync.Mutex{} // Parse the recursion field. recursion, err := strconv.Atoi(r.FormValue("recursion")) if err != nil { recursion = 0 } // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } mustLoadObjects := recursion > 0 || (recursion == 0 && clauses != nil && len(clauses.Clauses) > 0) // Detect project mode. projectName := request.QueryParam(r, "project") allProjects := util.IsTrue(r.FormValue("all-projects")) if allProjects && projectName != "" { return response.BadRequest(errors.New("Cannot specify a project when requesting all projects")) } else if !allProjects && projectName == "" { projectName = api.ProjectDefaultName } // Get the list and location of all instances. var filteredProjects []string var memberAddressInstances map[string][]db.Instance err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { if allProjects { projects, err := dbCluster.GetProjects(context.Background(), tx.Tx()) if err != nil { return err } for _, project := range projects { filteredProjects = append(filteredProjects, project.Name) } } else { filteredProjects = []string{projectName} } offlineThreshold := s.GlobalConfig.OfflineThreshold() memberAddressInstances, err = tx.GetInstancesByMemberAddress(ctx, offlineThreshold, filteredProjects) if err != nil { return fmt.Errorf("Failed getting instances by member address: %w", err) } return nil }) if err != nil { return response.SmartError(err) } userHasPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanView, auth.ObjectTypeInstance) if err != nil { return response.InternalError(err) } // Removes instances the user doesn't have access to. for address, instances := range memberAddressInstances { var filteredInstances []db.Instance for _, inst := range instances { if !userHasPermission(auth.ObjectInstance(inst.Project, inst.Name)) { continue } filteredInstances = append(filteredInstances, inst) } memberAddressInstances[address] = filteredInstances } resultErrListAppend := func(inst db.Instance, err error) { instFull := &api.InstanceFull{ Instance: api.Instance{ Name: inst.Name, Status: api.Error.String(), StatusCode: api.Error, Location: inst.Location, Project: inst.Project, Type: inst.Type.String(), }, } resultMu.Lock() resultFullList = append(resultFullList, instFull) resultMu.Unlock() } resultFullListAppend := func(instFull *api.InstanceFull) { if instFull != nil { resultMu.Lock() resultFullList = append(resultFullList, instFull) resultMu.Unlock() } } // Get the data wg := sync.WaitGroup{} networkCert := s.Endpoints.NetworkCert() for memberAddress, instances := range memberAddressInstances { // If this is an internal request from another cluster node, ignore instances from other // projectInstanceToNodeName, and return only the ones on this member. if isClusterNotification(r) && memberAddress != "" { continue } // Mark instances on unavailable projectInstanceToNodeName as down. if mustLoadObjects && memberAddress == "0.0.0.0" { for _, inst := range instances { resultErrListAppend(inst, errors.New("unavailable")) } continue } // For recursion requests we need to fetch the state of remote instances from their respective // projectInstanceToNodeName. if mustLoadObjects && memberAddress != "" && !isClusterNotification(r) { wg.Add(1) go func(memberAddress string, instances []db.Instance) { defer wg.Done() if recursion == 1 { apiInsts, err := doInstancesGetFromNode(filteredProjects, memberAddress, allProjects, networkCert, s.ServerCert(), r) if err != nil { for _, inst := range instances { resultErrListAppend(inst, err) } return } for _, apiInst := range apiInsts { resultFullListAppend(&api.InstanceFull{Instance: apiInst}) } return } cs, err := doInstancesFullGetFromNode(filteredProjects, memberAddress, allProjects, networkCert, s.ServerCert(), r) if err != nil { for _, inst := range instances { resultErrListAppend(inst, err) } return } for _, c := range cs { resultFullListAppend(&c) } }(memberAddress, instances) continue } if !mustLoadObjects { for _, inst := range instances { resultFullListAppend(&api.InstanceFull{Instance: api.Instance{ Project: inst.Project, Name: inst.Name, Location: inst.Location, }}) } } else { threads := min(len(instances), max(runtime.NumCPU()/2, 1)) hostInterfaces, _ := net.Interfaces() // Get the local instances. localInstancesByID := make(map[int64]instance.Instance) for _, projectName := range filteredProjects { insts, err := instanceLoadNodeProjectAll(r.Context(), s, projectName) if err != nil { return response.InternalError(fmt.Errorf("Failed loading instances for project %q: %w", projectName, err)) } for _, inst := range insts { localInstancesByID[int64(inst.ID())] = inst } } queue := make(chan db.Instance, threads) for range threads { wg.Add(1) go func() { for { dbInst, more := <-queue if !more { break } inst, found := localInstancesByID[dbInst.ID] if !found { continue } if recursion < 2 { c, _, err := inst.Render() if err != nil { resultErrListAppend(dbInst, err) } else { resultFullListAppend(&api.InstanceFull{Instance: *c.(*api.Instance)}) } continue } c, _, err := inst.RenderFull(hostInterfaces) if err != nil { resultErrListAppend(dbInst, err) } else { resultFullListAppend(c) } } wg.Done() }() } for _, inst := range instances { queue <- inst } close(queue) } } wg.Wait() // Sort the result list by project and then instance name. sort.SliceStable(resultFullList, func(i, j int) bool { if resultFullList[i].Project == resultFullList[j].Project { return resultFullList[i].Name < resultFullList[j].Name } return resultFullList[i].Project < resultFullList[j].Project }) // Filter result list if needed. if clauses != nil && len(clauses.Clauses) > 0 { resultFullList, err = instance.FilterFull(resultFullList, *clauses) if err != nil { return response.SmartError(err) } } if recursion == 0 { resultList := make([]string, 0, len(resultFullList)) for i := range resultFullList { url := api.NewURL().Path(version.APIVersion, "instances", resultFullList[i].Name).Project(resultFullList[i].Project) resultList = append(resultList, url.String()) } return response.SyncResponse(true, resultList) } if recursion == 1 { resultList := make([]*api.Instance, 0, len(resultFullList)) for i := range resultFullList { resultList = append(resultList, &resultFullList[i].Instance) } return response.SyncResponse(true, resultList) } return response.SyncResponse(true, resultFullList) } // Fetch information about the containers on the given remote node, using the // rest API and with a timeout of 30 seconds. func doInstancesGetFromNode(projects []string, node string, allProjects bool, networkCert *localtls.CertInfo, serverCert *localtls.CertInfo, r *http.Request) ([]api.Instance, error) { f := func() ([]api.Instance, error) { client, err := cluster.Connect(node, networkCert, serverCert, r, true) if err != nil { return nil, fmt.Errorf("Failed to connect to member %s: %w", node, err) } var containers []api.Instance if allProjects { containers, err = client.GetInstancesAllProjects(api.InstanceTypeAny) if err != nil { return nil, fmt.Errorf("Failed to get instances from member %s: %w", node, err) } } else { for _, project := range projects { client = client.UseProject(project) tmpContainers, err := client.GetInstances(api.InstanceTypeAny) if err != nil { return nil, fmt.Errorf("Failed to get instances from member %s: %w", node, err) } containers = append(containers, tmpContainers...) } } return containers, nil } timeout := time.After(30 * time.Second) done := make(chan struct{}) var containers []api.Instance var err error go func() { containers, err = f() done <- struct{}{} }() select { case <-timeout: err = fmt.Errorf("Timeout getting instances from member %s", node) case <-done: } return containers, err } func doInstancesFullGetFromNode(projects []string, node string, allProjects bool, networkCert *localtls.CertInfo, serverCert *localtls.CertInfo, r *http.Request) ([]api.InstanceFull, error) { f := func() ([]api.InstanceFull, error) { client, err := cluster.Connect(node, networkCert, serverCert, r, true) if err != nil { return nil, fmt.Errorf("Failed to connect to member %s: %w", node, err) } var instances []api.InstanceFull if allProjects { instances, err = client.GetInstancesFullAllProjects(api.InstanceTypeAny) if err != nil { return nil, fmt.Errorf("Failed to get instances from member %s: %w", node, err) } } else { for _, project := range projects { client = client.UseProject(project) tmpInstances, err := client.GetInstancesFull(api.InstanceTypeAny) if err != nil { return nil, fmt.Errorf("Failed to get instances from member %s: %w", node, err) } instances = append(instances, tmpInstances...) } } return instances, nil } timeout := time.After(30 * time.Second) done := make(chan struct{}) var instances []api.InstanceFull var err error go func() { instances, err = f() done <- struct{}{} }() select { case <-timeout: err = fmt.Errorf("Timeout getting instances from member %s", node) case <-done: } return instances, err } incus-7.0.0/cmd/incusd/instances_post.go000066400000000000000000001451271517523235500202740ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "os" "slices" "strings" petname "github.com/dustinkirkland/golang-petname" "github.com/gorilla/websocket" internalInstance "github.com/lxc/incus/v7/internal/instance" internalIO "github.com/lxc/incus/v7/internal/io" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/instance/operationlock" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/scriptlet" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" apiScriptlet "github.com/lxc/incus/v7/shared/api/scriptlet" "github.com/lxc/incus/v7/shared/archive" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" ) func ensureDownloadedImageFitWithinBudget(ctx context.Context, s *state.State, r *http.Request, op *operations.Operation, p api.Project, imgAlias string, source api.InstanceSource, imgType string) (*api.Image, error) { var autoUpdate bool var err error if p.Config["images.auto_update_cached"] != "" { autoUpdate = util.IsTrue(p.Config["images.auto_update_cached"]) } else { autoUpdate = s.GlobalConfig.ImagesAutoUpdateCached() } var budget int64 err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { budget, err = project.GetImageSpaceBudget(tx, p.Name) return err }) if err != nil { return nil, err } imgDownloaded, created, err := imageDownload(ctx, r, s, op, &imageDownloadArgs{ Server: source.Server, Protocol: source.Protocol, Certificate: source.Certificate, Secret: source.Secret, Alias: imgAlias, SetCached: true, Type: imgType, AutoUpdate: autoUpdate, Public: false, PreferCached: true, ProjectName: p.Name, Budget: budget, }) if err != nil { return nil, err } if created { // Add the image to the authorizer. err = s.Authorizer.AddImage(s.ShutdownCtx, p.Name, imgDownloaded.Fingerprint) if err != nil { logger.Error("Failed to add image to authorizer", logger.Ctx{"fingerprint": imgDownloaded.Fingerprint, "project": p.Name, "error": err}) } s.Events.SendLifecycle(p.Name, lifecycle.ImageCreated.Event(imgDownloaded.Fingerprint, p.Name, op.Requestor(), logger.Ctx{"type": imgDownloaded.Type})) } return imgDownloaded, nil } func createFromImage(s *state.State, r *http.Request, p api.Project, profiles []api.Profile, img *api.Image, imgAlias string, req *api.InstancesPost) response.Response { if s.ServerClustered && s.DB.Cluster.LocalNodeIsEvacuated() { return response.Forbidden(errors.New("Cluster member is evacuated")) } dbType, err := instancetype.New(string(req.Type)) if err != nil { return response.BadRequest(err) } run := func(op *operations.Operation) error { devices := deviceConfig.NewDevices(req.Devices) args := db.InstanceArgs{ Project: p.Name, Config: req.Config, Type: dbType, Description: req.Description, Devices: deviceConfig.ApplyDeviceInitialValues(devices, profiles), Ephemeral: req.Ephemeral, Name: req.Name, Profiles: profiles, } if req.Source.Server != "" { img, err = ensureDownloadedImageFitWithinBudget(context.TODO(), s, r, op, p, imgAlias, req.Source, string(req.Type)) if err != nil { return err } } else if img != nil { err := ensureImageIsLocallyAvailable(context.TODO(), s, r, img, args.Project) if err != nil { return err } } else { return errors.New("Image not provided for instance creation") } args.Architecture, err = osarch.ArchitectureID(img.Architecture) if err != nil { return err } // Actually create the instance. err = instanceCreateFromImage(context.TODO(), s, img, args, op) if err != nil { return err } return instanceCreateFinish(s, req, args, op) } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", req.Name)} op, err := operations.OperationCreate(s, p.Name, operations.OperationClassTask, operationtype.InstanceCreate, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } func createFromNone(s *state.State, r *http.Request, projectName string, profiles []api.Profile, req *api.InstancesPost) response.Response { if s.ServerClustered && s.DB.Cluster.LocalNodeIsEvacuated() { return response.Forbidden(errors.New("Cluster member is evacuated")) } dbType, err := instancetype.New(string(req.Type)) if err != nil { return response.BadRequest(err) } devices := deviceConfig.NewDevices(req.Devices) args := db.InstanceArgs{ Project: projectName, Config: req.Config, Type: dbType, Description: req.Description, Devices: deviceConfig.ApplyDeviceInitialValues(devices, profiles), Ephemeral: req.Ephemeral, Name: req.Name, Profiles: profiles, } if req.Architecture != "" { architecture, err := osarch.ArchitectureID(req.Architecture) if err != nil { return response.InternalError(err) } args.Architecture = architecture } run := func(op *operations.Operation) error { // Actually create the instance. _, err := instanceCreateAsEmpty(s, args, op) if err != nil { return err } return instanceCreateFinish(s, req, args, op) } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", req.Name)} op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.InstanceCreate, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } func createFromMigration(ctx context.Context, s *state.State, r *http.Request, projectName string, profiles []api.Profile, req *api.InstancesPost) response.Response { if s.ServerClustered && r != nil && r.Context().Value(request.CtxProtocol) != "cluster" && s.DB.Cluster.LocalNodeIsEvacuated() { return response.Forbidden(errors.New("Cluster member is evacuated")) } // Validate migration mode. if req.Source.Mode != "pull" && req.Source.Mode != "push" { return response.NotImplemented(fmt.Errorf("Mode %q not implemented", req.Source.Mode)) } // Parse the architecture name architecture, err := osarch.ArchitectureID(req.Architecture) if err != nil { return response.BadRequest(err) } dbType, err := instancetype.New(string(req.Type)) if err != nil { return response.BadRequest(err) } if dbType != instancetype.Container && dbType != instancetype.VM { return response.BadRequest(fmt.Errorf("Instance type not supported %q", req.Type)) } // Prepare the instance creation request. args := db.InstanceArgs{ Project: projectName, Architecture: architecture, BaseImage: req.Source.BaseImage, Config: req.Config, Type: dbType, Devices: deviceConfig.NewDevices(req.Devices), Description: req.Description, Ephemeral: req.Ephemeral, Name: req.Name, Profiles: profiles, Stateful: req.Stateful, } storagePool, storagePoolProfile, localRootDiskDeviceKey, localRootDiskDevice, resp := instanceFindStoragePool(ctx, s, projectName, req) if resp != nil { return resp } if storagePool == "" { return response.BadRequest(errors.New("Can't find a storage pool for the instance to use")) } if localRootDiskDeviceKey == "" && storagePoolProfile == "" { // Give the container it's own local root disk device with a pool property. rootDev := map[string]string{} rootDev["type"] = "disk" rootDev["path"] = "/" rootDev["pool"] = storagePool if args.Devices == nil { args.Devices = deviceConfig.Devices{} } // Make sure that we do not overwrite a device the user is currently using under the // name "root". rootDevName := "root" for i := range 100 { if args.Devices[rootDevName] == nil { break } rootDevName = fmt.Sprintf("root%d", i) continue } args.Devices[rootDevName] = rootDev } else if localRootDiskDeviceKey != "" && localRootDiskDevice["pool"] == "" { args.Devices[localRootDiskDeviceKey]["pool"] = storagePool } var inst instance.Instance var instOp *operationlock.InstanceOperation var cleanup revert.Hook // Decide if this is an internal cluster move request. var clusterMoveSourceName string if r != nil && isClusterNotification(r) && req.Source.Source != "" { clusterMoveSourceName = req.Source.Source } // Early check for refresh and cluster same name move to check instance exists. if req.Source.Refresh || (clusterMoveSourceName != "" && clusterMoveSourceName == req.Name) { inst, err = instance.LoadByProjectAndName(s, projectName, req.Name) if err != nil { if response.IsNotFoundError(err) { if clusterMoveSourceName != "" { // Cluster move doesn't allow renaming as part of migration so fail here. return response.SmartError(errors.New("Cluster move doesn't allow renaming")) } req.Source.Refresh = false } else { return response.SmartError(err) } } } reverter := revert.New() defer reverter.Fail() instanceOnly := req.Source.InstanceOnly if inst == nil { _, err := storagePools.LoadByName(s, storagePool) if err != nil { return response.InternalError(err) } // Create the instance DB record for main instance. // Note: At this stage we do not yet know if snapshots are going to be received and so we cannot // create their DB records. This will be done if needed in the migrationSink.do() function called // as part of the operation below. inst, instOp, cleanup, err = instance.CreateInternal(s, args, nil, true, false, true) if err != nil { return response.InternalError(fmt.Errorf("Failed creating instance record: %w", err)) } reverter.Add(cleanup) } else { instOp, err = inst.LockExclusive() if err != nil { return response.SmartError(fmt.Errorf("Failed getting exclusive access to instance: %w", err)) } } reverter.Add(func() { instOp.Done(err) }) push := false var dialer *websocket.Dialer if req.Source.Mode == "push" { push = true } else { dialer, err = setupWebsocketDialer(req.Source.Certificate) if err != nil { return response.SmartError(fmt.Errorf("Failed setting up websocket dialer for migration sink connections: %w", err)) } } // Override existing devices with values provided in the request. devs := inst.LocalDevices().CloneNative() if req.Devices != nil { err = inst.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error { reqDevice, ok := req.Devices[dev.Name] if !ok { return nil } for k, v := range reqDevice { devs[dev.Name][k] = v } return nil }) if err != nil { return response.InternalError(err) } } // Update devices for the target instance without saving changes to the database, // as the same instance is still being used by the source. err = inst.UpdateDevices(deviceConfig.NewDevices(devs)) if err != nil { return response.InternalError(fmt.Errorf("Failed to update instance %q: %w", inst.Name(), err)) } migrationArgs := migrationSinkArgs{ URL: req.Source.Operation, Dialer: dialer, Instance: inst, Secrets: req.Source.Websockets, Push: push, Live: req.Source.Live, InstanceOnly: instanceOnly, ClusterMoveSourceName: clusterMoveSourceName, Refresh: req.Source.Refresh, RefreshExcludeOlder: req.Source.RefreshExcludeOlder, StoragePool: storagePool, } // Check if the pool is changing at all. if r != nil && isClusterNotification(r) && inst != nil { _, currentPool, _ := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if currentPool["pool"] == storagePool { migrationArgs.StoragePool = "" } } sink, err := newMigrationSink(&migrationArgs) if err != nil { return response.InternalError(err) } // Copy reverter so far so we can use it inside run after this function has finished. runReverter := reverter.Clone() run := func(op *operations.Operation) error { defer runReverter.Fail() sink.instance.SetOperation(op) // And finally run the migration. err = sink.do(instOp) if err != nil { err = fmt.Errorf("Error transferring instance data: %w", err) instOp.Done(err) // Complete operation that was created earlier, to release lock. return err } instOp.Done(nil) // Complete operation that was created earlier, to release lock. if migrationArgs.StoragePool != "" || req.Devices != nil { // Update root device for the instance if needed. updateNeeded := false devs := inst.LocalDevices().CloneNative() rootDevKey, _, err := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if err != nil { if !errors.Is(err, internalInstance.ErrNoRootDisk) { return err } rootDev := map[string]string{} rootDev["type"] = "disk" rootDev["path"] = "/" rootDev["pool"] = storagePool devs["root"] = rootDev updateNeeded = true } else { // Copy the device if not a local device. _, ok := devs[rootDevKey] if !ok { devs[rootDevKey] = inst.ExpandedDevices().CloneNative()[rootDevKey] } // Apply the override. if devs[rootDevKey]["pool"] != storagePool { devs[rootDevKey]["pool"] = storagePool updateNeeded = true } } if req.Devices != nil { err = inst.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error { updateNeeded = true reqDevice, ok := req.Devices[dev.Name] if !ok { return nil } for k, v := range reqDevice { devs[dev.Name][k] = v } return nil }) if err != nil { return err } } if updateNeeded { devices, err := dbCluster.APIToDevices(devs) if err != nil { return err } err = s.DB.Cluster.Transaction(context.Background(), func(ctx context.Context, tx *db.ClusterTx) error { id, err := dbCluster.GetInstanceID(ctx, tx.Tx(), inst.Project().Name, inst.Name()) if err != nil { return fmt.Errorf("Failed to get ID of moved instance: %w", err) } err = dbCluster.UpdateInstanceDevices(ctx, tx.Tx(), int64(id), devices) if err != nil { return err } return nil }) if err != nil { return err } } } runReverter.Success() return instanceCreateFinish(s, req, args, op) } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", req.Name)} var op *operations.Operation if push { op, err = operations.OperationCreate(s, projectName, operations.OperationClassWebsocket, operationtype.InstanceCreate, resources, sink.Metadata(), run, sink.Cancel, sink.Connect, r) if err != nil { return response.InternalError(err) } } else { op, err = operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.InstanceCreate, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } } reverter.Success() return operations.OperationResponse(op) } // validateDependentVolumes validates dependent volumes during copy. func validateDependentVolumes(source instance.Instance, req *api.InstancesPost) error { // Fetch all dependent devices belonging to the instance. dependentVolumes := []string{} err := source.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error { dependentVolumes = append(dependentVolumes, dev.Name) return nil }) if err != nil { return err } sourceDevices := source.LocalDevices() for _, key := range dependentVolumes { newDevice, exists := req.Devices[key] if !exists { return fmt.Errorf("Missing device %s in request", key) } oldDevice, exists := sourceDevices[key] if !exists { return fmt.Errorf("Missing device %s on source", key) } if newDevice["pool"] != "" && oldDevice["pool"] != newDevice["pool"] { continue } // Check if the source was overridden. if oldDevice["source"] == newDevice["source"] { return fmt.Errorf("Device source name should be different during copy for dependent disk: %s", key) } } return nil } // ErrPoolNotRemote indicates the pool is not remote. var ErrPoolNotRemote error = errors.New("Pool is not remote") // checkVolumesOnRemoteStorage checks whether root and dependent disks are located on remote storage. func checkVolumesOnRemoteStorage(s *state.State, pool *api.StoragePool, inst instance.Instance) error { if !slices.Contains(db.StorageRemoteDriverNames(), pool.Driver) { return ErrPoolNotRemote } err := inst.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error { diskPool, err := storagePools.LoadByName(s, dev.Config["pool"]) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } if !slices.Contains(db.StorageRemoteDriverNames(), diskPool.Driver().Name()) { return ErrPoolNotRemote } return nil }) if err != nil { return err } return nil } func createFromCopy(ctx context.Context, s *state.State, r *http.Request, projectName string, profiles []api.Profile, req *api.InstancesPost) response.Response { if s.ServerClustered && s.DB.Cluster.LocalNodeIsEvacuated() { return response.Forbidden(errors.New("Cluster member is evacuated")) } if req.Source.Source == "" { return response.BadRequest(errors.New("Must specify a source instance")) } sourceProject := req.Source.Project if sourceProject == "" { sourceProject = projectName } targetProject := projectName source, err := instance.LoadByProjectAndName(s, sourceProject, req.Source.Source) if err != nil { return response.SmartError(err) } // If "security.secureboot" has changed, force a NVRAM reset. if util.IsTrueOrEmpty(source.ExpandedConfig()["security.secureboot"]) != util.IsTrueOrEmpty(req.Config["security.secureboot"]) { req.Config["volatile.apply_nvram"] = "true" } err = validateDependentVolumes(source, req) if err != nil { return response.SmartError(err) } // When clustered, use the node name, otherwise use the hostname. if s.ServerClustered { serverName := s.ServerName if serverName != source.Location() { // Check if we are copying from a remote storage instance. _, rootDevice, _ := internalInstance.GetRootDiskDevice(source.ExpandedDevices().CloneNative()) sourcePoolName := rootDevice["pool"] destPoolName, _, _, _, resp := instanceFindStoragePool(r.Context(), s, targetProject, req) if resp != nil { return resp } if sourcePoolName != destPoolName { // Redirect to migration return clusterCopyContainerInternal(ctx, s, r, source, projectName, profiles, req) } var pool *api.StoragePool err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { _, pool, _, err = tx.GetStoragePoolInAnyState(ctx, sourcePoolName) return err }) if err != nil { err = fmt.Errorf("Failed to fetch instance's pool info: %w", err) return response.SmartError(err) } err = checkVolumesOnRemoteStorage(s, pool, source) if err != nil { if errors.Is(err, ErrPoolNotRemote) { // Redirect to migration return clusterCopyContainerInternal(ctx, s, r, source, projectName, profiles, req) } return response.SmartError(err) } } } // Config override sourceConfig := source.LocalConfig() if req.Config == nil { req.Config = make(map[string]string) } for key, value := range sourceConfig { if !internalInstance.InstanceIncludeWhenCopying(key, false) { logger.Debug("Skipping key from copy source", logger.Ctx{"key": key, "sourceProject": source.Project().Name, "sourceInstance": source.Name(), "project": targetProject, "instance": req.Name}) continue } _, exists := req.Config[key] if exists { continue } req.Config[key] = value } // Devices override sourceDevices := source.LocalDevices() if req.Devices == nil { req.Devices = make(map[string]map[string]string) } for key, value := range sourceDevices { _, exists := req.Devices[key] if exists { continue } req.Devices[key] = value } if req.Stateful { sourceName, _, _ := api.GetParentAndSnapshotName(source.Name()) if sourceName != req.Name { return response.BadRequest(fmt.Errorf("Instance name cannot be changed during stateful copy (%q to %q)", sourceName, req.Name)) } } dbType, err := instancetype.New(string(req.Type)) if err != nil { return response.BadRequest(err) } // If type isn't specified, match the source type. if req.Type == "" { dbType = source.Type() } if dbType != instancetype.Any && dbType != source.Type() { return response.BadRequest(errors.New("Instance type should not be specified or should match source type")) } args := db.InstanceArgs{ Project: targetProject, Architecture: source.Architecture(), BaseImage: req.Source.BaseImage, Config: req.Config, Type: source.Type(), Description: req.Description, Devices: deviceConfig.NewDevices(req.Devices), Ephemeral: req.Ephemeral, Name: req.Name, Profiles: profiles, Stateful: req.Stateful, } run := func(op *operations.Operation) error { // Actually create the instance. _, err := instanceCreateAsCopy(s, instanceCreateAsCopyOpts{ sourceInstance: source, targetInstance: args, instanceOnly: req.Source.InstanceOnly, refresh: req.Source.Refresh, refreshExcludeOlder: req.Source.RefreshExcludeOlder, applyTemplateTrigger: true, allowInconsistent: req.Source.AllowInconsistent, }, op) if err != nil { return err } return instanceCreateFinish(s, req, args, op) } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", req.Name), *api.NewURL().Path(version.APIVersion, "instances", req.Source.Source)} op, err := operations.OperationCreate(s, targetProject, operations.OperationClassTask, operationtype.InstanceCreate, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } func createFromBackup(s *state.State, r *http.Request, projectName string, data io.Reader, pool string, instanceName string, config string, device string) response.Response { reverter := revert.New() defer reverter.Fail() // Create temporary file to store uploaded backup data. backupFile, err := os.CreateTemp(internalUtil.VarPath("backups"), fmt.Sprintf("%s_", backup.WorkingDirPrefix)) if err != nil { return response.InternalError(err) } defer func() { _ = os.Remove(backupFile.Name()) }() reverter.Add(func() { _ = backupFile.Close() }) // Get disk budget for the project if any. var budget int64 err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { budget, err = project.GetSpaceBudget(tx, projectName) if err != nil { return err } return nil }) if err != nil { return response.InternalError(err) } // Stream uploaded backup data into temporary file. _, err = util.SafeCopy(internalIO.NewQuotaWriter(backupFile, budget), data) if err != nil { return response.InternalError(err) } // Detect squashfs compression and convert to tarball. _, err = backupFile.Seek(0, io.SeekStart) if err != nil { return response.InternalError(err) } _, algo, decomArgs, err := archive.DetectCompressionFile(backupFile) if err != nil { return response.InternalError(err) } if algo == ".squashfs" { // Pass the temporary file as program argument to the decompression command. decomArgs := append(decomArgs, backupFile.Name()) // Create temporary file to store the decompressed tarball in. tarFile, err := os.CreateTemp(internalUtil.VarPath("backups"), fmt.Sprintf("%s_decompress_", backup.WorkingDirPrefix)) if err != nil { return response.InternalError(err) } defer func() { _ = os.Remove(tarFile.Name()) }() // Decompress to tarFile temporary file. err = archive.ExtractWithFds(decomArgs[0], decomArgs[1:], nil, nil, tarFile) if err != nil { return response.InternalError(err) } // We don't need the original squashfs file anymore. _ = backupFile.Close() _ = os.Remove(backupFile.Name()) // Replace the backup file handle with the handle to the tar file. backupFile = tarFile } // Parse the backup information. _, err = backupFile.Seek(0, io.SeekStart) if err != nil { return response.InternalError(err) } bInfo, err := backup.GetInfo(backupFile, s.OS, backupFile.Name()) if err != nil { return response.BadRequest(err) } // Detect broken legacy backups. if bInfo.Config == nil || bInfo.Config.Container == nil { return response.BadRequest(errors.New("Backup file is missing required information")) } // Early project permissions check (pre-override and pre-backup.yaml). var req api.InstancesPost err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { req = api.InstancesPost{ InstancePut: bInfo.Config.Container.InstancePut, Name: bInfo.Name, Source: api.InstanceSource{}, // Only relevant for "copy" or "migration", but may not be nil. Type: api.InstanceType(bInfo.Config.Container.Type), } return project.AllowInstanceCreation(tx, projectName, req) }) if err != nil { return response.SmartError(err) } bInfo.Project = projectName // Override pool. if pool != "" { bInfo.Pool = pool } // Override instance name. if instanceName != "" { bInfo.Name = instanceName } // Override config. configMap := map[string]string{} if config != "" { configOverride := strings.Split(config, " ") for _, entry := range configOverride { key, value, found := strings.Cut(entry, "=") if !found { return response.BadRequest(fmt.Errorf("Failed to parse config =: %q", entry)) } configMap[key] = value } } // Override device. deviceMap := map[string]map[string]string{} if device != "" { deviceOverride := strings.Split(device, " ") for _, entry := range deviceOverride { if !strings.Contains(entry, "=") || !strings.Contains(entry, ",") { return response.BadRequest(fmt.Errorf("Failed to parse device ,=: %q", entry)) } deviceFields := strings.SplitN(entry, ",", 2) keyFields := strings.SplitN(deviceFields[1], "=", 2) if deviceMap[deviceFields[0]] == nil { deviceMap[deviceFields[0]] = map[string]string{} } deviceMap[deviceFields[0]][keyFields[0]] = keyFields[1] } } logger.Debug("Backup file info loaded", logger.Ctx{ "type": bInfo.Type, "name": bInfo.Name, "project": bInfo.Project, "backend": bInfo.Backend, "pool": bInfo.Pool, "optimized": *bInfo.OptimizedStorage, "snapshots": bInfo.Snapshots, }) err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Check storage pool exists. _, _, _, err = tx.GetStoragePoolInAnyState(ctx, bInfo.Pool) return err }) if response.IsNotFoundError(err) { // The storage pool doesn't exist. If backup is in binary format (so we cannot alter // the backup.yaml) or the pool has been specified directly from the user restoring // the backup then we cannot proceed so return an error. if *bInfo.OptimizedStorage || pool != "" { return response.InternalError(fmt.Errorf("Storage pool not found: %w", err)) } var profile *api.Profile err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Otherwise try and restore to the project's default profile pool. _, profile, err = tx.GetProfile(ctx, bInfo.Project, "default") return err }) if err != nil { return response.InternalError(fmt.Errorf("Failed to get default profile: %w", err)) } _, v, err := internalInstance.GetRootDiskDevice(profile.Devices) if err != nil { return response.InternalError(fmt.Errorf("Failed to get root disk device: %w", err)) } // Use the default-profile's root pool. bInfo.Pool = v["pool"] } else if err != nil { return response.InternalError(err) } // Copy reverter so far so we can use it inside run after this function has finished. runReverter := reverter.Clone() run := func(op *operations.Operation) error { defer func() { _ = backupFile.Close() }() defer runReverter.Fail() pool, err := storagePools.LoadByName(s, bInfo.Pool) if err != nil { return err } // Check if the backup is optimized that the source pool driver matches the target pool driver. if *bInfo.OptimizedStorage && pool.Driver().Info().Name != bInfo.Backend { return fmt.Errorf("Optimized backup storage driver %q differs from the target storage pool driver %q", bInfo.Backend, pool.Driver().Info().Name) } // Dump tarball to storage. Because the backup file is unpacked and restored onto the storage // device before the instance is created in the database it is necessary to return two functions; // a post hook that can be run once the instance has been created in the database to run any // storage layer finalisations, and a revert hook that can be run if the instance database load // process fails that will remove anything created thus far. postHook, revertHook, err := pool.CreateInstanceFromBackup(*bInfo, backupFile, nil) if err != nil { return fmt.Errorf("Create instance from backup: %w", err) } runReverter.Add(revertHook) err = internalImportFromBackup(context.TODO(), s, bInfo.Project, bInfo.Name, instanceName != "", deviceMap, configMap) if err != nil { return fmt.Errorf("Failed importing backup: %w", err) } // Load the newly imported instance. inst, err := instance.LoadByProjectAndName(s, bInfo.Project, bInfo.Name) if err != nil { return fmt.Errorf("Failed loading instance: %w", err) } // Clean up created instance if the post hook fails below. runReverter.Add(func() { _ = inst.Delete(true, true) }) // Run a late project restriction check on the instance. instState, _, err := inst.Render() if err != nil { return fmt.Errorf("Failed loading instance state: %w", err) } instStateAPI, ok := instState.(*api.Instance) if !ok { return fmt.Errorf("Unexpected instance state type %T", instStateAPI) } err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { req = api.InstancesPost{ InstancePut: instStateAPI.Writable(), Name: inst.Name(), Source: api.InstanceSource{}, Type: inst.Type().ToAPI(), } return project.AllowInstanceCreation(tx, projectName, req) }) if err != nil { return err } // Run any post hook for the instance. if postHook != nil { err = postHook(inst) if err != nil { return fmt.Errorf("Post hook failed: %w", err) } } // And wrap up validation by running a check on all snapshots too. snaps, err := inst.Snapshots() if err != nil { return fmt.Errorf("Failed loading instance snapshots: %w", err) } for _, snap := range snaps { snapState, _, err := snap.Render() if err != nil { return fmt.Errorf("Failed loading instance snapshot state: %w", err) } snapStateAPI, ok := snapState.(*api.InstanceSnapshot) if !ok { return fmt.Errorf("Unexpected snapshot type %T", snapStateAPI) } err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { req = api.InstancesPost{ InstancePut: api.InstancePut{ Architecture: snapStateAPI.Architecture, Config: snapStateAPI.Config, Description: snapStateAPI.Description, Devices: snapStateAPI.Devices, Ephemeral: snapStateAPI.Ephemeral, Profiles: snapStateAPI.Profiles, }, Name: inst.Name(), Source: api.InstanceSource{}, Type: inst.Type().ToAPI(), } return project.AllowInstanceCreation(tx, projectName, req) }) if err != nil { return err } } runReverter.Success() return instanceCreateFinish(s, &req, db.InstanceArgs{Name: bInfo.Name, Project: bInfo.Project}, op) } resources := map[string][]api.URL{} resources["instances"] = []api.URL{*api.NewURL().Path(version.APIVersion, "instances", bInfo.Name)} op, err := operations.OperationCreate(s, bInfo.Project, operations.OperationClassTask, operationtype.BackupRestore, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } reverter.Success() return operations.OperationResponse(op) } // swagger:operation POST /1.0/instances instances instances_post // // Create a new instance // // Creates a new instance. // Depending on the source, this can create an instance from an existing // local image, remote image, existing local instance or snapshot, remote // migration stream or backup file. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member // type: string // example: default // - in: body // name: instance // description: Instance request // required: false // schema: // $ref: "#/definitions/InstancesPost" // - in: body // name: raw_backup // description: Raw backup file // required: false // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instancesPost(d *Daemon, r *http.Request) response.Response { s := d.State() targetProjectName := request.ProjectParam(r) clusterNotification := isClusterNotification(r) clusterInternal := isClusterInternal(r) logger.Debug("Responding to instance create") // If we're getting binary content, process separately if r.Header.Get("Content-Type") == "application/octet-stream" { return createFromBackup(s, r, targetProjectName, r.Body, r.Header.Get("X-Incus-pool"), r.Header.Get("X-Incus-name"), r.Header.Get("X-Incus-config"), r.Header.Get("X-Incus-devices")) } // Parse the request req := api.InstancesPost{} err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Set type from URL if missing if req.Type == "" { req.Type = api.InstanceTypeContainer // Default to container if not specified. } if req.Devices == nil { req.Devices = map[string]map[string]string{} } if req.Config == nil { req.Config = map[string]string{} } if req.InstanceType != "" { conf, err := instanceParseType(req.InstanceType) if err != nil { return response.BadRequest(err) } for k, v := range conf { if req.Config[k] == "" { req.Config[k] = v } } } // Special handling for instance refresh. // For all other situations, we're headed towards the scheduler, but for this case, we can short circuit it. if s.ServerClustered && !clusterNotification && req.Source.Type == "migration" && req.Source.Refresh { client, err := cluster.ConnectIfInstanceIsRemote(s, targetProjectName, req.Name, r) if err != nil && !response.IsNotFoundError(err) { return response.SmartError(err) } if client != nil { // The request needs to be forwarded to the correct server. op, err := client.CreateInstance(req) if err != nil { return response.SmartError(err) } opAPI := op.Get() return operations.ForwardedOperationResponse(&opAPI) } if err == nil { // The instance is valid and the request wasn't forwarded, so the instance is local. return createFromMigration(r.Context(), s, r, targetProjectName, nil, &req) } } var targetProject *api.Project var profiles []api.Profile var sourceInst *dbCluster.Instance var sourceImage *api.Image var sourceImageRef string var candidateMembers []db.NodeInfo var targetMemberInfo *db.NodeInfo var targetGroupName string target := request.QueryParam(r, "target") if !s.ServerClustered && target != "" { return response.BadRequest(errors.New("Target only allowed when clustered")) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := dbCluster.GetProject(ctx, tx.Tx(), targetProjectName) if err != nil { return fmt.Errorf("Failed loading project: %w", err) } targetProject, err = dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return err } var allMembers []db.NodeInfo if s.ServerClustered && !clusterNotification { allMembers, err = tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } // Check if the given target is allowed and try to resolve the right member or group targetMemberInfo, targetGroupName, err = project.CheckTarget(ctx, s.Authorizer, r, tx, targetProject, target, allMembers) if err != nil { return err } } profileProject := project.ProfileProjectFromRecord(targetProject) switch req.Source.Type { case "copy": if req.Source.Source == "" { return api.StatusErrorf(http.StatusBadRequest, "Must specify a source instance") } if req.Source.Project == "" { req.Source.Project = targetProjectName } sourceInst, err = instance.LoadInstanceDatabaseObject(ctx, tx, req.Source.Project, req.Source.Source) if err != nil { return err } req.Type = api.InstanceType(sourceInst.Type.String()) // Use source instance's profiles if no profile override. if req.Profiles == nil { sourceInstArgs, err := tx.InstancesToInstanceArgs(ctx, true, *sourceInst) if err != nil { return err } req.Profiles = make([]string, 0, len(sourceInstArgs[sourceInst.ID].Profiles)) for _, profile := range sourceInstArgs[sourceInst.ID].Profiles { req.Profiles = append(req.Profiles, profile.Name) } } case "image": // Check if the image has an entry in the database but fail only if the error // is different than the image not being found. sourceImage, err = getSourceImageFromInstanceSource(ctx, s, tx, targetProject.Name, req.Source, &sourceImageRef, string(req.Type)) if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { return err } // If image has an entry in the database then use its profiles if no override provided. if sourceImage != nil && req.Profiles == nil { req.Architecture = sourceImage.Architecture req.Profiles = sourceImage.Profiles } } // Use default profile if no profile list specified (not even an empty list). // This mirrors the logic in instance.CreateInternal() that would occur anyway. if req.Profiles == nil { req.Profiles = []string{"default"} } // Initialize the profile info list (even if an empty list is provided so this isn't left as nil). // This way instances can still be created without any profiles by providing a non-nil empty list. profiles = make([]api.Profile, 0, len(req.Profiles)) // Load profiles. if len(req.Profiles) > 0 { profileFilters := make([]dbCluster.ProfileFilter, 0, len(req.Profiles)) for _, profileName := range req.Profiles { profileFilters = append(profileFilters, dbCluster.ProfileFilter{ Project: &profileProject, Name: &profileName, }) } dbProfiles, err := dbCluster.GetProfiles(ctx, tx.Tx(), profileFilters...) if err != nil { return err } dbProfileConfigs, err := dbCluster.GetAllProfileConfigs(ctx, tx.Tx()) if err != nil { return err } dbProfileDevices, err := dbCluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return err } profilesByName := make(map[string]dbCluster.Profile, len(dbProfiles)) for _, dbProfile := range dbProfiles { profilesByName[dbProfile.Name] = dbProfile } for _, profileName := range req.Profiles { profile, found := profilesByName[profileName] if !found { return fmt.Errorf("Requested profile %q doesn't exist", profileName) } apiProfile, err := profile.ToAPI(ctx, tx.Tx(), dbProfileConfigs, dbProfileDevices) if err != nil { return err } profiles = append(profiles, *apiProfile) } } // Generate automatic instance name if not specified. if req.Name == "" { names, err := tx.GetInstanceNames(ctx, targetProjectName) if err != nil { return err } i := 0 for { i++ req.Name = strings.ToLower(petname.Generate(2, "-")) if !slices.Contains(names, req.Name) { break } if i > 100 { return errors.New("Couldn't generate a new unique name after 100 tries") } } logger.Debug("No name provided for new instance, using auto-generated name", logger.Ctx{"project": targetProjectName, "instance": req.Name}) } else if req.Source.Type != "migration" && !req.Source.Refresh { // Check if the instance name is already in use. id, err := tx.GetInstanceID(ctx, targetProjectName, req.Name) if err == nil && id > 0 { return fmt.Errorf("Instance %q already exists", req.Name) } } if s.ServerClustered && !clusterNotification && targetMemberInfo == nil { architectures, err := instance.SuitableArchitectures(ctx, s, tx, targetProjectName, sourceInst, sourceImageRef, req) if err != nil { return err } // If no architectures have been ascertained from the source then use the default // architecture from project or global config if available. if len(architectures) < 1 { defaultArch := targetProject.Config["images.default_architecture"] if defaultArch == "" { defaultArch = s.GlobalConfig.ImagesDefaultArchitecture() } if defaultArch != "" { defaultArchID, err := osarch.ArchitectureID(defaultArch) if err != nil { return err } architectures = append(architectures, defaultArchID) } else { architectures = nil // Don't exclude candidate members based on architecture. } } clusterGroupsAllowed := project.GetRestrictedClusterGroups(targetProject) candidateMembers, err = tx.GetCandidateMembers(ctx, allMembers, architectures, targetGroupName, clusterGroupsAllowed, s.GlobalConfig.OfflineThreshold()) if err != nil { return err } } if !clusterNotification { // Check that the project's limits are not violated. Note this check is performed after // automatically generated config values (such as ones from an InstanceType) have been set. err = project.AllowInstanceCreation(tx, targetProjectName, req) if err != nil { return err } } return nil }) if err != nil { return response.SmartError(err) } err = instance.ValidName(req.Name, false) if err != nil { return response.BadRequest(err) } if s.ServerClustered && !clusterNotification && !clusterInternal { // If a target was specified, limit the list of candidates to that target. if targetMemberInfo != nil { candidateMembers = []db.NodeInfo{*targetMemberInfo} } // Run instance placement scriptlet if enabled. if s.GlobalConfig.InstancesPlacementScriptlet() != "" { leaderAddress, err := s.Cluster.LeaderAddress() if err != nil { return response.InternalError(err) } // Copy request so we don't modify it when expanding the config. reqExpanded := apiScriptlet.InstancePlacement{ InstancesPost: req, Project: targetProjectName, Reason: apiScriptlet.InstancePlacementReasonNew, } reqExpanded.Config = db.ExpandInstanceConfig(reqExpanded.Config, profiles) reqExpanded.Devices = db.ExpandInstanceDevices(deviceConfig.NewDevices(reqExpanded.Devices), profiles).CloneNative() targetMemberInfo, err = scriptlet.InstancePlacementRun(r.Context(), logger.Log, s, &reqExpanded, candidateMembers, leaderAddress) if err != nil { return response.SmartError(fmt.Errorf("Failed instance placement scriptlet: %w", err)) } } // If no target member was selected yet, pick the member with the least number of instances. if targetMemberInfo == nil && len(candidateMembers) > 0 { targetMemberInfo = &candidateMembers[0] } if targetMemberInfo == nil { return response.InternalError(errors.New("Couldn't find a cluster member for the instance")) } } // Record the cluster group as a volatile config key if present. if !clusterNotification && !clusterInternal && targetGroupName != "" { req.Config["volatile.cluster.group"] = targetGroupName } if targetMemberInfo != nil && targetMemberInfo.Address != "" && targetMemberInfo.Name != s.ServerName { client, err := cluster.Connect(targetMemberInfo.Address, s.Endpoints.NetworkCert(), s.ServerCert(), r, true) if err != nil { return response.SmartError(err) } client = client.UseProject(targetProjectName) client = client.UseTarget(targetMemberInfo.Name) logger.Debug("Forward instance post request", logger.Ctx{"local": s.ServerName, "target": targetMemberInfo.Name, "targetAddress": targetMemberInfo.Address}) op, err := client.CreateInstance(req) if err != nil { return response.SmartError(err) } opAPI := op.Get() return operations.ForwardedOperationResponse(&opAPI) } switch req.Source.Type { case "image": return createFromImage(s, r, *targetProject, profiles, sourceImage, sourceImageRef, &req) case "none": return createFromNone(s, r, targetProjectName, profiles, &req) case "migration": return createFromMigration(r.Context(), s, r, targetProjectName, profiles, &req) case "copy": return createFromCopy(r.Context(), s, r, targetProjectName, profiles, &req) default: return response.BadRequest(fmt.Errorf("Unknown source type %s", req.Source.Type)) } } func instanceFindStoragePool(ctx context.Context, s *state.State, projectName string, req *api.InstancesPost) (string, string, string, map[string]string, response.Response) { // Grab the container's root device if one is specified storagePool := "" storagePoolProfile := "" localRootDiskDeviceKey, localRootDiskDevice, _ := internalInstance.GetRootDiskDevice(req.Devices) if localRootDiskDeviceKey != "" { storagePool = localRootDiskDevice["pool"] } // Handle copying/moving between two storage-api instances. if storagePool != "" { err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { _, err := tx.GetStoragePoolID(ctx, storagePool) return err }) if response.IsNotFoundError(err) { storagePool = "" // Unset the local root disk device storage pool if not // found. localRootDiskDevice["pool"] = "" } } // If we don't have a valid pool yet, look through profiles if storagePool == "" { err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { for _, pName := range req.Profiles { _, p, err := tx.GetProfile(ctx, projectName, pName) if err != nil { return err } k, v, _ := internalInstance.GetRootDiskDevice(p.Devices) if k != "" && v["pool"] != "" { // Keep going as we want the last one in the profile chain storagePool = v["pool"] storagePoolProfile = pName } } return nil }) if err != nil { return "", "", "", nil, response.SmartError(err) } } // If there is just a single pool in the database, use that if storagePool == "" { logger.Debug("No valid storage pool in the container's local root disk device and profiles found") var pools []string err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error pools, err = tx.GetStoragePoolNames(ctx) return err }) if err != nil { if response.IsNotFoundError(err) { return "", "", "", nil, response.BadRequest(errors.New("This instance does not have any storage pools configured")) } return "", "", "", nil, response.SmartError(err) } if len(pools) == 1 { storagePool = pools[0] } } return storagePool, storagePoolProfile, localRootDiskDeviceKey, localRootDiskDevice, nil } func clusterCopyContainerInternal(ctx context.Context, s *state.State, r *http.Request, source instance.Instance, projectName string, profiles []api.Profile, req *api.InstancesPost) response.Response { // Locate the source of the container var nodeAddress string err := s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Load source node. nodeAddress, err = tx.GetNodeAddressOfInstance(ctx, source.Project().Name, source.Name()) if err != nil { return fmt.Errorf("Failed to get address of instance's member: %w", err) } return nil }) if err != nil { return response.SmartError(err) } if nodeAddress == "" { return response.BadRequest(errors.New("The source instance is currently offline")) } // Connect to the container source client, err := cluster.Connect(nodeAddress, s.Endpoints.NetworkCert(), s.ServerCert(), r, false) if err != nil { return response.SmartError(err) } client = client.UseProject(source.Project().Name) // Setup websockets var opAPI api.Operation if internalInstance.IsSnapshot(req.Source.Source) { cName, sName, _ := api.GetParentAndSnapshotName(req.Source.Source) pullReq := api.InstanceSnapshotPost{ Migration: true, Live: req.Source.Live, } op, err := client.MigrateInstanceSnapshot(cName, sName, pullReq) if err != nil { return response.SmartError(err) } opAPI = op.Get() } else { instanceOnly := req.Source.InstanceOnly pullReq := api.InstancePost{ Migration: true, Live: req.Source.Live, InstanceOnly: instanceOnly, Devices: req.Devices, } op, err := client.MigrateInstance(req.Source.Source, pullReq) if err != nil { return response.SmartError(err) } opAPI = op.Get() } websockets := map[string]string{} for k, v := range opAPI.Metadata { websockets[k] = v.(string) } // Reset the source for a migration req.Source.Type = "migration" req.Source.Certificate = string(s.Endpoints.NetworkCert().PublicKey()) req.Source.Mode = "pull" req.Source.Operation = fmt.Sprintf("https://%s/%s/operations/%s", nodeAddress, version.APIVersion, opAPI.ID) req.Source.Websockets = websockets req.Source.Source = "" req.Source.Project = "" // Run the migration return createFromMigration(ctx, s, nil, projectName, profiles, req) } func instanceCreateFinish(s *state.State, req *api.InstancesPost, args db.InstanceArgs, op *operations.Operation) error { if req == nil || !req.Start { return nil } // Start the instance. inst, err := instance.LoadByProjectAndName(s, args.Project, args.Name) if err != nil { return fmt.Errorf("Failed to load the instance: %w", err) } inst.SetOperation(op) return inst.Start(false) } incus-7.0.0/cmd/incusd/instances_put.go000066400000000000000000000145121517523235500201100ustar00rootroot00000000000000package main import ( "context" "encoding/json" "fmt" "net/http" "strings" "sync" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) func coalesceErrors(local bool, errors map[string]error) error { if len(errors) == 0 { return nil } var errorMsg string if local { errorMsg += "The following instances failed to update state:\n" } for instName, err := range errors { if local { errorMsg += fmt.Sprintf(" - Instance: %s: %v\n", instName, err) } else { errorMsg += strings.TrimSpace(fmt.Sprintf("%v\n", err)) } } return fmt.Errorf("%s", errorMsg) } // swagger:operation PUT /1.0/instances instances instances_put // // Bulk instance state update // // Changes the running state of all instances. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: state // description: State // required: false // schema: // $ref: "#/definitions/InstancesPut" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func instancesPut(d *Daemon, r *http.Request) response.Response { projectName := request.ProjectParam(r) // Don't mess with instances while in setup mode. <-d.waitReady.Done() s := d.State() c, err := instance.LoadNodeAll(s, instancetype.Any) if err != nil { return response.BadRequest(err) } req := api.InstancesPut{} req.State = &api.InstanceStatePut{} req.State.Timeout = -1 err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } action := internalInstance.InstanceAction(req.State.Action) userHasPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanUpdateState, auth.ObjectTypeInstance) if err != nil { return response.SmartError(err) } var names []string var instances []instance.Instance for _, inst := range c { if inst.Project().Name != projectName { continue } // Only allow changing the state of instances the user has permission for. if !userHasPermission(auth.ObjectInstance(inst.Project().Name, inst.Name())) { continue } switch action { case internalInstance.Freeze: if !inst.IsRunning() { continue } case internalInstance.Restart: if !inst.IsRunning() { continue } case internalInstance.Start: if inst.IsRunning() { continue } case internalInstance.Stop: if !inst.IsRunning() { continue } case internalInstance.Unfreeze: if !inst.IsFrozen() { continue } } instances = append(instances, inst) names = append(names, inst.Name()) } // Determine operation type. opType, err := instanceActionToOpType(req.State.Action) if err != nil { return response.BadRequest(err) } // Batch the changes. do := func(op *operations.Operation) error { localAction := func(local bool) error { failures := map[string]error{} failuresLock := sync.Mutex{} wgAction := sync.WaitGroup{} for _, inst := range instances { wgAction.Add(1) go func(inst instance.Instance) { defer wgAction.Done() inst.SetOperation(op) err := doInstanceStatePut(inst, *req.State) if err != nil { failuresLock.Lock() failures[inst.Name()] = err failuresLock.Unlock() } }(inst) } wgAction.Wait() return coalesceErrors(local, failures) } // Only return the local data if asked by cluster member. if isClusterNotification(r) { return localAction(false) } // If not clustered, return the local data. if !s.ServerClustered { return localAction(true) } // Get all members in cluster. var members []db.NodeInfo err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error members, err = tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } return nil }) if err != nil { return err } // Get local cluster address. localClusterAddress := s.LocalConfig.ClusterAddress() // Record the results. failures := map[string]error{} failuresLock := sync.Mutex{} wgAction := sync.WaitGroup{} networkCert := s.Endpoints.NetworkCert() for _, member := range members { wgAction.Add(1) go func(member db.NodeInfo) { defer wgAction.Done() // Special handling for the local member. if member.Address == localClusterAddress { err := localAction(false) if err != nil { failuresLock.Lock() failures[member.Name] = err failuresLock.Unlock() } return } // Connect to the remote server. client, err := cluster.Connect(member.Address, networkCert, s.ServerCert(), r, true) if err != nil { failuresLock.Lock() failures[member.Name] = err failuresLock.Unlock() return } client = client.UseProject(projectName) // Perform the action. op, err := client.UpdateInstances(req, "") if err != nil { failuresLock.Lock() failures[member.Name] = err failuresLock.Unlock() return } err = op.Wait() if err != nil { failuresLock.Lock() failures[member.Name] = err failuresLock.Unlock() return } }(member) } wgAction.Wait() return coalesceErrors(true, failures) } resources := map[string][]api.URL{} for _, instName := range names { resources["instances"] = append(resources["instances"], *api.NewURL().Path(version.APIVersion, "instances", instName)) } op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, opType, resources, nil, do, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } incus-7.0.0/cmd/incusd/logging.go000066400000000000000000000077011517523235500166610ustar00rootroot00000000000000package main import ( "context" "os" "slices" "strings" "time" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/server/task" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" ) // This task function expires logs when executed. It's started by the Daemon // and will run once every 24h. func expireLogsTask(state *state.State) (task.Func, task.Schedule) { f := func(ctx context.Context) { opRun := func(op *operations.Operation) error { return expireLogs(ctx, state) } op, err := operations.OperationCreate(state, "", operations.OperationClassTask, operationtype.LogsExpire, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating log files expiry operation", logger.Ctx{"err": err}) return } logger.Info("Expiring log files") err = op.Start() if err != nil { logger.Error("Failed starting log files expiry operation", logger.Ctx{"err": err}) return } err = op.Wait(ctx) if err != nil { logger.Error("Failed expiring log files", logger.Ctx{"err": err}) return } logger.Info("Done expiring log files") } return f, task.Daily() } func expireLogs(ctx context.Context, state *state.State) error { // List the instances. instances, err := instance.LoadNodeAll(state, instancetype.Any) if err != nil { return err } // List the directory. entries, err := os.ReadDir(state.OS.LogDir) if err != nil { return err } // Build the expected names. names := []string{} for _, inst := range instances { names = append(names, project.Instance(inst.Project().Name, inst.Name())) } newestFile := func(path string, dir os.FileInfo) time.Time { newest := dir.ModTime() entries, err := os.ReadDir(path) if err != nil { return newest } for _, entry := range entries { info, err := entry.Info() if err != nil { continue } if info.ModTime().After(newest) { newest = info.ModTime() } } return newest } for _, entry := range entries { // At each iteration we check if we got cancelled in the meantime. select { case <-ctx.Done(): return nil default: } // We only care about instance directories. if !entry.IsDir() { continue } // Skip if we are unable to read the file info, e.g. the file might // be deleted. fi, err := entry.Info() if err != nil { continue } // Check if the instance still exists. if slices.Contains(names, fi.Name()) { instDirEntries, err := os.ReadDir(internalUtil.LogPath(fi.Name())) if err != nil { return err } for _, instDirEntry := range instDirEntries { path := internalUtil.LogPath(fi.Name(), instDirEntry.Name()) instInfo, err := instDirEntry.Info() if err != nil { continue } // Deal with directories (snapshots). if instInfo.IsDir() { newest := newestFile(path, instInfo) if time.Since(newest).Hours() >= 48 { err := os.RemoveAll(path) if err != nil { return err } } continue } // Only remove old log files (keep other files, such as conf, pid, monitor etc). if strings.HasSuffix(instInfo.Name(), ".log") || strings.HasSuffix(instInfo.Name(), ".log.old") { // Remove any log file which wasn't modified in the past 48 hours. if time.Since(instInfo.ModTime()).Hours() >= 48 { err := os.Remove(path) if err != nil { return err } } } } } else { // Empty directory if unchanged in the past 24 hours. path := internalUtil.LogPath(fi.Name()) newest := newestFile(path, fi) if time.Since(newest).Hours() >= 24 { err := os.RemoveAll(path) if err != nil { return err } } } } return nil } incus-7.0.0/cmd/incusd/main.go000066400000000000000000000140451517523235500161560ustar00rootroot00000000000000package main import ( "os" dqlite "github.com/cowsql/go-cowsql" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/rsync" "github.com/lxc/incus/v7/internal/server/daemon" "github.com/lxc/incus/v7/internal/server/events" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/logger" ) type cmdGlobal struct { cmd *cobra.Command flagHelp bool flagVersion bool flagLogFile string flagLogDebug bool flagLogSyslog bool flagLogTrace []string flagLogVerbose bool } func (c *cmdGlobal) run(_ *cobra.Command, _ []string) error { // Configure dqlite to *not* disable internal SQLite locking, since we // use SQLite both through dqlite and through go-dqlite, potentially // from different threads at the same time. We need to call this // function as early as possible since this is a global setting in // SQLite, which can't be changed afterwise. err := dqlite.ConfigMultiThread() if err != nil { return err } // Set logging global variables daemon.Debug = c.flagLogDebug rsync.Debug = c.flagLogDebug daemon.Verbose = c.flagLogVerbose // Set debug for the operations package operations.Init(daemon.Debug) // Set debug for the response package response.Init(daemon.Debug) // Setup logger syslog := "" if c.flagLogSyslog { syslog = "incus" } err = logger.InitLogger(c.flagLogFile, syslog, c.flagLogVerbose, c.flagLogDebug, events.NewEventHandler()) if err != nil { return err } return nil } // rawArgs returns the raw unprocessed arguments from os.Args after the command name arg is found. func (c *cmdGlobal) rawArgs(cmd *cobra.Command) []string { for i, arg := range os.Args { if arg == cmd.Name() && len(os.Args)-1 > i { return os.Args[i+1:] } } return []string{} } func main() { // daemon command (main) daemonCmd := cmdDaemon{} app := daemonCmd.command() app.SilenceUsage = true app.CompletionOptions = cobra.CompletionOptions{DisableDefaultCmd: true} // Workaround for main command app.Args = cobra.ArbitraryArgs // Workaround for being called through "incus admin cluster". if len(os.Args) >= 3 && os.Args[0] == "incusd" && os.Args[1] == "admin" && os.Args[2] == "cluster" { app.Use = "incus" } // Global flags globalCmd := cmdGlobal{cmd: app} daemonCmd.global = &globalCmd app.PersistentPreRunE = globalCmd.run app.PersistentFlags().BoolVar(&globalCmd.flagVersion, "version", false, "Print version number") app.PersistentFlags().BoolVarP(&globalCmd.flagHelp, "help", "h", false, "Print help") app.PersistentFlags().StringVar(&globalCmd.flagLogFile, "logfile", "", "Path to the log file"+"``") app.PersistentFlags().BoolVar(&globalCmd.flagLogSyslog, "syslog", false, "Log to syslog") app.PersistentFlags().StringArrayVar(&globalCmd.flagLogTrace, "trace", []string{}, "Log tracing targets"+"``") app.PersistentFlags().BoolVarP(&globalCmd.flagLogDebug, "debug", "d", false, "Show all debug messages") app.PersistentFlags().BoolVarP(&globalCmd.flagLogVerbose, "verbose", "v", false, "Show all information messages") // Version handling app.SetVersionTemplate("{{.Version}}\n") app.Version = version.Version // activateifneeded sub-command activateifneededCmd := cmdActivateifneeded{global: &globalCmd} app.AddCommand(activateifneededCmd.command()) // callhook sub-command callhookCmd := cmdCallhook{global: &globalCmd} app.AddCommand(callhookCmd.command()) // forkconsole sub-command forkconsoleCmd := cmdForkconsole{global: &globalCmd} app.AddCommand(forkconsoleCmd.command()) // forkexec sub-command forkexecCmd := cmdForkexec{global: &globalCmd} app.AddCommand(forkexecCmd.command()) // forkfile sub-command forkfileCmd := cmdForkfile{global: &globalCmd} app.AddCommand(forkfileCmd.command()) // forklimits sub-command forklimitsCmd := cmdForklimits{global: &globalCmd} app.AddCommand(forklimitsCmd.command()) // forkmigrate sub-command forkmigrateCmd := cmdForkmigrate{global: &globalCmd} app.AddCommand(forkmigrateCmd.command()) // forksyscall sub-command forksyscallCmd := cmdForksyscall{global: &globalCmd} app.AddCommand(forksyscallCmd.command()) // forkcoresched sub-command forkbpfCmd := cmdForkbpf{global: &globalCmd} app.AddCommand(forkbpfCmd.command()) // forkcoresched sub-command forkcoreschedCmd := cmdForkcoresched{global: &globalCmd} app.AddCommand(forkcoreschedCmd.command()) // forkmount sub-command forkmountCmd := cmdForkmount{global: &globalCmd} app.AddCommand(forkmountCmd.command()) // forknet sub-command forknetCmd := cmdForknet{global: &globalCmd} app.AddCommand(forknetCmd.command()) // forkproxy sub-command forkproxyCmd := cmdForkproxy{global: &globalCmd} app.AddCommand(forkproxyCmd.command()) // forkstart sub-command forkstartCmd := cmdForkstart{global: &globalCmd} app.AddCommand(forkstartCmd.command()) // forkuevent sub-command forkueventCmd := cmdForkuevent{global: &globalCmd} app.AddCommand(forkueventCmd.command()) // forkzfs sub-command forkzfsCmd := cmdForkZFS{global: &globalCmd} app.AddCommand(forkzfsCmd.command()) // manpage sub-command manpageCmd := cmdManpage{global: &globalCmd} app.AddCommand(manpageCmd.command()) // migratedumpsuccess sub-command migratedumpsuccessCmd := cmdMigratedumpsuccess{global: &globalCmd} app.AddCommand(migratedumpsuccessCmd.command()) // netcat sub-command netcatCmd := cmdNetcat{global: &globalCmd} app.AddCommand(netcatCmd.command()) // shutdown sub-command shutdownCmd := cmdShutdown{global: &globalCmd} app.AddCommand(shutdownCmd.command()) // version sub-command versionCmd := cmdVersion{global: &globalCmd} app.AddCommand(versionCmd.command()) // waitready sub-command waitreadyCmd := cmdWaitready{global: &globalCmd} app.AddCommand(waitreadyCmd.command()) // cluster sub-command (also admin cluster) adminCmd := cmdAdmin{global: &globalCmd} app.AddCommand(adminCmd.command()) clusterCmd := cmdCluster{global: &globalCmd} app.AddCommand(clusterCmd.command()) // Run the main command and handle errors err := app.Execute() if err != nil { os.Exit(1) } } incus-7.0.0/cmd/incusd/main_activateifneeded.go000066400000000000000000000122131517523235500215150ustar00rootroot00000000000000package main import ( "context" "database/sql" "errors" "os" sqlite3 "github.com/mattn/go-sqlite3" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) func init() { sql.Register("dqlite_direct_access", &sqlite3.SQLiteDriver{ConnectHook: sqliteDirectAccess}) } type cmdActivateifneeded struct { global *cmdGlobal } func (c *cmdActivateifneeded) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "activateifneeded" cmd.Short = "Check if the daemon should be started" cmd.Long = `Description: Check if the daemon should be started This command will check if the daemon has any auto-started instances, instances which were running prior to the last shutdown or if it's configured to listen on the network address. If at least one of those is true, then a connection will be attempted to the socket which will cause a socket-activated daemon to be spawned. ` cmd.RunE = c.run cmd.Hidden = true return cmd } func (c *cmdActivateifneeded) run(_ *cobra.Command, _ []string) error { // Only root should run this if os.Geteuid() != 0 { return errors.New("This must be run as root") } // Don't start a full daemon, we just need database access d := defaultDaemon() // Check if either the local database files exists. path := d.os.LocalDatabasePath() if !util.PathExists(d.os.LocalDatabasePath()) { logger.Debugf("No local database, so no need to start the daemon now") return nil } // Open the database directly to avoid triggering any initialization // code, in particular the data migration from node to cluster db. sqldb, err := sql.Open("sqlite3", path) if err != nil { return err } d.db.Node = db.DirectAccess(sqldb) // Load the configured address from the database var localConfig *node.Config err = d.db.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { localConfig, err = node.ConfigLoad(ctx, tx) return err }) if err != nil { return err } localHTTPAddress := localConfig.HTTPSAddress() // Look for network socket if localHTTPAddress != "" { logger.Debugf("Daemon has core.https_address set, activating...") _, err := incus.ConnectIncusUnix("", nil) return err } // Set a non-nil IdmapSet to be able to load unprivileged instances d.os.IdmapSet = &idmap.Set{} // Look for auto-started or previously started instances path = d.os.GlobalDatabasePath() if !util.PathExists(path) { logger.Debugf("No global database, so no need to start the daemon now") return nil } sqldb, err = sql.Open("dqlite_direct_access", path+"?mode=ro") if err != nil { return err } defer func() { _ = sqldb.Close() }() d.db.Cluster, err = db.ForLocalInspectionWithPreparedStmts(sqldb) if err != nil { return err } instances, err := instance.LoadNodeAll(d.State(), instancetype.Any) if err != nil { return err } for _, inst := range instances { if instanceShouldAutoStart(inst) { logger.Debugf("Daemon has auto-started instances, activating...") _, err := incus.ConnectIncusUnix("", nil) return err } if inst.IsRunning() { logger.Debugf("Daemon has running instances, activating...") _, err := incus.ConnectIncusUnix("", nil) return err } // Check for scheduled instance snapshots config := inst.ExpandedConfig() if config["snapshots.schedule"] != "" { logger.Debugf("Daemon has scheduled instance snapshots, activating...") _, err := incus.ConnectIncusUnix("", nil) return err } } // Check for scheduled volume snapshots var volumes []db.StorageVolumeArgs err = d.State().DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { volumes, err = tx.GetStoragePoolVolumesWithType(ctx, db.StoragePoolVolumeTypeCustom, false) if err != nil { return err } return nil }) if err != nil { return err } for _, vol := range volumes { if vol.Config["snapshots.schedule"] != "" { logger.Debugf("Daemon has scheduled volume snapshots, activating...") _, err := incus.ConnectIncusUnix("", nil) return err } } logger.Debugf("No need to start the daemon now") return nil } // Configure the sqlite connection so that it's safe to access the // dqlite-managed sqlite file, also without setting up raft. func sqliteDirectAccess(conn *sqlite3.SQLiteConn) error { // Ensure journal mode is set to WAL, as this is a requirement for // replication. _, err := conn.Exec("PRAGMA journal_mode=wal", nil) if err != nil { return err } // Ensure we don't truncate or checkpoint the WAL on exit, as this // would bork replication which must be in full control of the WAL // file. _, err = conn.Exec("PRAGMA journal_size_limit=-1", nil) if err != nil { return err } // Ensure WAL autocheckpoint is disabled, since checkpoints are // triggered explicitly by dqlite. _, err = conn.Exec("PRAGMA wal_autocheckpoint=0", nil) if err != nil { return err } return nil } incus-7.0.0/cmd/incusd/main_callhook.go000066400000000000000000000052271517523235500200340ustar00rootroot00000000000000package main import ( "errors" "fmt" "net/url" "os" "path/filepath" "time" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" ) type cmdCallhook struct { global *cmdGlobal } func (c *cmdCallhook) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "callhook [| ] " cmd.Short = "Call container lifecycle hook" cmd.Long = `Description: Call container lifecycle hook This internal command notifies the daemon about a container lifecycle event (start, stopns, stop, restart) and blocks until it has been processed. ` cmd.RunE = c.run cmd.Hidden = true return cmd } func (c *cmdCallhook) run(cmd *cobra.Command, args []string) error { // Quick checks. if len(args) < 2 { _ = cmd.Help() if len(args) == 0 { return nil } return errors.New("Missing required arguments") } path := args[0] var projectName string var instanceRef string var hook string if len(args) == 3 { instanceRef = args[1] hook = args[2] } else if len(args) == 4 { projectName = args[1] instanceRef = args[2] hook = args[3] } target := "" // Only root should run this. if os.Geteuid() != 0 { return errors.New("This must be run as root") } // Connect to daemon. socket := os.Getenv("INCUS_SOCKET") if socket == "" { socket = filepath.Join(path, "unix.socket") } clientArgs := incus.ConnectionArgs{ SkipGetServer: true, } d, err := incus.ConnectIncusUnix(socket, &clientArgs) if err != nil { return err } // Prepare the request URL query parameters. v := url.Values{} if projectName != "" { v.Set("project", projectName) } if hook == "stop" || hook == "stopns" { target = os.Getenv("LXC_TARGET") if target == "" { target = "unknown" } v.Set("target", target) } if hook == "stopns" { v.Set("netns", os.Getenv("LXC_NET_NS")) } // Setup the request. response := make(chan error, 1) go func() { url := fmt.Sprintf("/internal/containers/%s/%s?%s", url.PathEscape(instanceRef), url.PathEscape(fmt.Sprintf("on%s", hook)), v.Encode()) _, _, err := d.RawQuery("GET", url, nil, "") response <- err }() // Handle the timeout. select { case err := <-response: if err != nil { return err } break case <-time.After(30 * time.Second): return errors.New("Hook didn't finish within 30s") } // If the container is rebooting, we purposefully tell LXC that this hook failed so that // it won't reboot the container, which lets us start it again in the OnStop function. // Other hook types can return without error safely. if hook == "stop" && target == "reboot" { return errors.New("Reboot must be handled by Incus") } return nil } incus-7.0.0/cmd/incusd/main_cluster.go000066400000000000000000000327201517523235500177170ustar00rootroot00000000000000package main import ( "bufio" "context" "errors" "fmt" "io" "os" "path/filepath" "slices" "strings" "github.com/cowsql/go-cowsql/client" "github.com/spf13/cobra" "go.yaml.in/yaml/v4" "golang.org/x/sys/unix" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/ports" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/internal/server/sys" internalUtil "github.com/lxc/incus/v7/internal/util" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/termios" ) type cmdAdmin struct { global *cmdGlobal } func (c *cmdAdmin) command() *cobra.Command { cmd := &cobra.Command{} cmd.Hidden = true cmd.Use = "admin" // Cluster clusterCmd := cmdCluster{global: c.global} cmd.AddCommand(clusterCmd.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } return cmd } type cmdCluster struct { global *cmdGlobal } // Command returns a cobra command for inclusion. func (c *cmdCluster) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "cluster" cmd.Short = "Low-level cluster administration commands" cmd.Long = `Description: Low level administration tools for inspecting and recovering clusters. ` // List database nodes listDatabase := cmdClusterListDatabase{global: c.global} cmd.AddCommand(listDatabase.command()) // Recover recoverFromQuorumLoss := cmdClusterRecoverFromQuorumLoss{global: c.global} cmd.AddCommand(recoverFromQuorumLoss.command()) // Remove a raft node. removeRaftNode := cmdClusterRemoveRaftNode{global: c.global} cmd.AddCommand(removeRaftNode.command()) // Edit cluster configuration. clusterEdit := cmdClusterEdit{global: c.global} cmd.AddCommand(clusterEdit.command()) // Show cluster configuration. clusterShow := cmdClusterShow{global: c.global} cmd.AddCommand(clusterShow.command()) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } return cmd } const SegmentComment = "# Latest cowsql segment ID: %s" // ClusterMember is a more human-readable representation of the db.RaftNode struct. type ClusterMember struct { ID uint64 `yaml:"id"` Name string `yaml:"name,omitempty"` Address string `yaml:"address"` Role string `yaml:"role"` } // ClusterConfig is a representation of the current cluster configuration. type ClusterConfig struct { Members []ClusterMember `yaml:"members"` } // ToRaftNode converts a ClusterConfig struct to a RaftNode struct. func (c ClusterMember) ToRaftNode() (*db.RaftNode, error) { node := &db.RaftNode{ NodeInfo: client.NodeInfo{ ID: c.ID, Address: c.Address, }, Name: c.Name, } var role db.RaftRole switch c.Role { case "voter": role = db.RaftVoter case "stand-by": role = db.RaftStandBy case "spare": role = db.RaftSpare default: return nil, fmt.Errorf("unknown raft role: %q", c.Role) } node.Role = role return node, nil } type cmdClusterEdit struct { global *cmdGlobal } func (c *cmdClusterEdit) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "edit" cmd.Short = "Edit cluster configuration as YAML" cmd.Long = `Description: Edit cluster configuration as YAML.` cmd.RunE = c.run return cmd } func (c *cmdClusterEdit) run(_ *cobra.Command, _ []string) error { // Make sure that the daemon is not running. _, err := incus.ConnectIncusUnix("", nil) if err == nil { return errors.New("The daemon is running, please stop it first.") } database, err := db.OpenNode(filepath.Join(sys.DefaultOS().VarDir, "database"), nil) if err != nil { return err } var nodes []db.RaftNode err = database.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { config, err := node.ConfigLoad(ctx, tx) if err != nil { return err } clusterAddress := config.ClusterAddress() if clusterAddress == "" { return errors.New(`Can't edit cluster configuration as server isn't clustered (missing "cluster.https_address" config)`) } nodes, err = tx.GetRaftNodes(ctx) return err }) if err != nil { return err } segmentID, err := db.DqliteLatestSegment() if err != nil { return err } config := ClusterConfig{Members: []ClusterMember{}} for _, node := range nodes { member := ClusterMember{ID: node.ID, Name: node.Name, Address: node.Address, Role: node.Role.String()} config.Members = append(config.Members, member) } data, err := yaml.Dump(config, yaml.V2) if err != nil { return err } var content []byte if !termios.IsTerminal(unix.Stdin) { content, err = io.ReadAll(os.Stdin) if err != nil { return err } } else { if len(config.Members) > 0 { data = []byte(fmt.Sprintf(SegmentComment, segmentID) + "\n\n" + string(data)) } content, err = cli.TextEditor("", data) if err != nil { return err } } for { newConfig := ClusterConfig{} err = yaml.Load(content, &newConfig) if err == nil { // Convert ClusterConfig back to RaftNodes. newNodes := []db.RaftNode{} var newNode *db.RaftNode for _, node := range newConfig.Members { newNode, err = node.ToRaftNode() if err != nil { break } newNodes = append(newNodes, *newNode) } // Ensure new configuration is valid. if err == nil { err = validateNewConfig(nodes, newNodes) if err == nil { err = cluster.Reconfigure(database, newNodes) } } } if err != nil { fmt.Fprintf(os.Stderr, "Config validation error: %s\n", err) fmt.Println("Press enter to open the editor again or ctrl+c to abort change") _, err := os.Stdin.Read(make([]byte, 1)) if err != nil { return err } content, err = cli.TextEditor("", content) if err != nil { return err } continue } break } return nil } func validateNewConfig(oldNodes []db.RaftNode, newNodes []db.RaftNode) error { if len(oldNodes) > len(newNodes) { return errors.New("Removing cluster members is not supported") } if len(oldNodes) < len(newNodes) { return errors.New("Adding cluster members is not supported") } numNewVoters := 0 for i, newNode := range newNodes { oldNode := oldNodes[i] // IDs should not be reordered among cluster members. if oldNode.ID != newNode.ID { return errors.New("Changing cluster member ID is not supported") } // If the name field could not be populated, just ignore the new value. if oldNode.Name != "" && newNode.Name != "" && oldNode.Name != newNode.Name { return errors.New("Changing cluster member name is not supported") } if oldNode.Role == db.RaftSpare && newNode.Role == db.RaftVoter { return fmt.Errorf("A %q cluster member cannot become a %q", db.RaftSpare.String(), db.RaftVoter.String()) } if newNode.Role == db.RaftVoter { numNewVoters++ } } if numNewVoters < 2 && len(newNodes) > 2 { return fmt.Errorf("Number of %q must be 2 or more", db.RaftVoter.String()) } else if numNewVoters < 1 { return fmt.Errorf("At least one member must be a %q", db.RaftVoter.String()) } return nil } type cmdClusterShow struct { global *cmdGlobal } func (c *cmdClusterShow) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "show" cmd.Short = "Show cluster configuration as YAML" cmd.Long = `Description: Show cluster configuration as YAML.` cmd.RunE = c.run return cmd } func (c *cmdClusterShow) run(_ *cobra.Command, _ []string) error { database, err := db.OpenNode(filepath.Join(sys.DefaultOS().VarDir, "database"), nil) if err != nil { return err } var nodes []db.RaftNode err = database.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { var err error nodes, err = tx.GetRaftNodes(ctx) return err }) if err != nil { return err } segmentID, err := db.DqliteLatestSegment() if err != nil { return err } config := ClusterConfig{Members: []ClusterMember{}} for _, node := range nodes { member := ClusterMember{ID: node.ID, Name: node.Name, Address: node.Address, Role: node.Role.String()} config.Members = append(config.Members, member) } data, err := yaml.Dump(config, yaml.V2) if err != nil { return err } if len(config.Members) > 0 { fmt.Printf(SegmentComment+"\n\n%s", segmentID, data) } else { fmt.Print(data) } return nil } type cmdClusterListDatabase struct { global *cmdGlobal flagFormat string } func (c *cmdClusterListDatabase) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "list-database" cmd.Aliases = []string{"ls"} cmd.Short = "Print the addresses of the cluster members serving the database" cmd.Flags().StringVarP(&c.flagFormat, "format", "f", "table", `Format (csv|json|table|yaml|compact|markdown), use suffix ",noheader" to disable headers and ",header" to enable it if missing, e.g. csv,header`) cmd.PreRunE = func(cmd *cobra.Command, _ []string) error { return cli.ValidateFlagFormatForListOutput(cmd.Flag("format").Value.String()) } cmd.RunE = c.run return cmd } func (c *cmdClusterListDatabase) run(_ *cobra.Command, _ []string) error { defaultOS := sys.DefaultOS() dbconn, err := db.OpenNode(filepath.Join(defaultOS.VarDir, "database"), nil) if err != nil { return fmt.Errorf("Failed to open local database: %w", err) } addresses, err := cluster.ListDatabaseNodes(dbconn) if err != nil { return fmt.Errorf("Failed to get database nodes: %w", err) } columns := []string{"Address"} data := make([][]string, len(addresses)) for i, address := range addresses { data[i] = []string{address} } _ = cli.RenderTable(os.Stdout, c.flagFormat, columns, data, nil) return nil } type cmdClusterRecoverFromQuorumLoss struct { global *cmdGlobal flagNonInteractive bool } func (c *cmdClusterRecoverFromQuorumLoss) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "recover-from-quorum-loss" cmd.Short = "Recover an instance whose cluster has lost quorum" cmd.RunE = c.run cmd.Flags().BoolVarP(&c.flagNonInteractive, "quiet", "q", false, "Don't require user confirmation") return cmd } func (c *cmdClusterRecoverFromQuorumLoss) run(_ *cobra.Command, _ []string) error { // Make sure that the daemon is not running. _, err := incus.ConnectIncusUnix("", nil) if err == nil { return errors.New("The daemon is running, please stop it first.") } // Prompt for confirmation unless --quiet was passed. if !c.flagNonInteractive { err := c.promptConfirmation() if err != nil { return err } } os := sys.DefaultOS() db, err := db.OpenNode(filepath.Join(os.VarDir, "database"), nil) if err != nil { return fmt.Errorf("Failed to open local database: %w", err) } return cluster.Recover(db) } func (c *cmdClusterRecoverFromQuorumLoss) promptConfirmation() error { reader := bufio.NewReader(os.Stdin) fmt.Print(`You should run this command only if you are *absolutely* certain that this is the only database node left in your cluster AND that other database nodes will never come back (i.e. their daemon won't ever be started again). This will make this server the only member of the cluster, and it won't be possible to perform operations on former cluster members anymore. However all information about former cluster members will be preserved in the database, so you can possibly inspect it for further recovery. You'll be able to permanently delete from the database all information about former cluster members by running "incus cluster remove --force". See https://linuxcontainers.org/incus/docs/main/howto/cluster_recover/#recover-from-quorum-loss for more info. Do you want to proceed? (yes/no): `) input, _ := reader.ReadString('\n') input = strings.TrimSuffix(input, "\n") if !slices.Contains([]string{"yes"}, strings.ToLower(input)) { return errors.New("Recover operation aborted") } return nil } type cmdClusterRemoveRaftNode struct { global *cmdGlobal flagNonInteractive bool } func (c *cmdClusterRemoveRaftNode) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "remove-raft-node
" cmd.Short = "Remove a raft node from the raft configuration" cmd.RunE = c.run cmd.Flags().BoolVarP(&c.flagNonInteractive, "quiet", "q", false, "Don't require user confirmation") return cmd } func (c *cmdClusterRemoveRaftNode) run(cmd *cobra.Command, args []string) error { if len(args) != 1 { _ = cmd.Help() return errors.New("Missing required arguments") } address := internalUtil.CanonicalNetworkAddress(args[0], ports.HTTPSDefaultPort) // Prompt for confirmation unless --quiet was passed. if !c.flagNonInteractive { err := c.promptConfirmation() if err != nil { return err } } client, err := incus.ConnectIncusUnix("", nil) if err != nil { return fmt.Errorf("Failed to connect to daemon: %w", err) } endpoint := fmt.Sprintf("/internal/cluster/raft-node/%s", address) _, _, err = client.RawQuery("DELETE", endpoint, nil, "") if err != nil { return err } return nil } func (c *cmdClusterRemoveRaftNode) promptConfirmation() error { reader := bufio.NewReader(os.Stdin) fmt.Print(`You should run this command only if you ended up in an inconsistent state where a node has been uncleanly removed (i.e. it doesn't show up in "incus cluster list" but it's still in the raft configuration). Do you want to proceed? (yes/no): `) input, _ := reader.ReadString('\n') input = strings.TrimSuffix(input, "\n") if !slices.Contains([]string{"yes"}, strings.ToLower(input)) { return errors.New("Remove raft node operation aborted") } return nil } incus-7.0.0/cmd/incusd/main_daemon.go000066400000000000000000000041761517523235500175050ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "os" "os/exec" "os/signal" "github.com/spf13/cobra" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/server/sys" "github.com/lxc/incus/v7/shared/logger" ) type cmdDaemon struct { global *cmdGlobal // Common options flagGroup string } func (c *cmdDaemon) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "incusd" cmd.Short = "The Incus daemon" cmd.Long = `Description: The Incus daemon This is the incus daemon command line. It's typically started directly by your init system and interacted with through a tool like ` + "`incus`" + `. ` cmd.RunE = c.run cmd.Flags().StringVar(&c.flagGroup, "group", "", "The group of users that will be allowed to talk to Incus"+"``") return cmd } func (c *cmdDaemon) run(cmd *cobra.Command, args []string) error { if len(args) > 1 || (len(args) == 1 && args[0] != "daemon" && args[0] != "") { return fmt.Errorf("unknown command \"%s\" for \"%s\"", args[0], cmd.CommandPath()) } // Only root should run this if os.Geteuid() != 0 { return errors.New("This must be run as root") } neededPrograms := []string{"ip", "rsync", "setfattr", "tar", "unsquashfs", "xz"} for _, p := range neededPrograms { _, err := exec.LookPath(p) if err != nil { return err } } defer logger.Info("Daemon stopped") conf := defaultDaemonConfig() conf.Group = c.flagGroup conf.Trace = c.global.flagLogTrace d := newDaemon(conf, sys.DefaultOS()) sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, unix.SIGPWR) signal.Notify(sigCh, unix.SIGINT) signal.Notify(sigCh, unix.SIGQUIT) signal.Notify(sigCh, unix.SIGTERM) chIgnore := make(chan os.Signal, 1) signal.Notify(chIgnore, unix.SIGHUP) err := d.Init() if err != nil { return err } for { select { case sig := <-sigCh: logger.Info("Received signal", logger.Ctx{"signal": sig}) if d.shutdownCtx.Err() != nil { logger.Warn("Ignoring signal, shutdown already in progress", logger.Ctx{"signal": sig}) } else { go func() { d.shutdownDoneCh <- d.Stop(context.Background(), sig) }() } case err = <-d.shutdownDoneCh: return err } } } incus-7.0.0/cmd/incusd/main_forkbpf.go000066400000000000000000000150721517523235500176700ustar00rootroot00000000000000package main /* #include "config.h" #include #include #include #include #include #include #include #include #include #include "incus.h" #include "memory_utils.h" pid_t incus_forkbpf_pid; int incus_forkbpf_socket_parent; int incus_forkbpf_socket_child; static int dosetns_file(char *file) { __do_close int ns_fd = -EBADF; ns_fd = open(file, O_RDONLY); if (ns_fd < 0) { fprintf(stderr, "%m - Failed to open \"%s\": %s", file, strerror(errno)); return -1; } if (setns(ns_fd, 0) < 0) { fprintf(stderr, "%m - Failed to attach to namespace \"%s\": %s", file, strerror(errno)); return -1; } return 0; } void forkbpf(void) { char *pidstr; char path[PATH_MAX]; int fds[2]; pidstr = getenv("LXC_PID"); if (!pidstr) { fprintf(stderr, "No LXC_PID in environment\n"); _exit(EXIT_FAILURE); } if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) < 0) { fprintf(stderr, "Failed to create socket pair: %s", strerror(errno)); _exit(EXIT_FAILURE); } incus_forkbpf_socket_parent = fds[0]; incus_forkbpf_socket_child = fds[1]; incus_forkbpf_pid = fork(); if (incus_forkbpf_pid < 0) { fprintf(stderr, "%s - Failed to create new process\n", strerror(errno)); _exit(EXIT_FAILURE); } if (incus_forkbpf_pid == 0) { // Attach to the user namespace. snprintf(path, sizeof(path), "/proc/%s/ns/user", pidstr); if (dosetns_file(path) < 0) { _exit(EXIT_FAILURE); } // Attach to the mount namespace. snprintf(path, sizeof(path), "/proc/%s/ns/mnt", pidstr); if (dosetns_file(path) < 0) { _exit(EXIT_FAILURE); } } } */ import "C" import ( "fmt" "os" "strings" "github.com/spf13/cobra" "golang.org/x/sys/unix" // Used by cgo. _ "github.com/lxc/incus/v7/shared/cgo" ) type cmdForkbpf struct { global *cmdGlobal } func (c *cmdForkbpf) command() *cobra.Command { // Main subcommand cmd := &cobra.Command{} cmd.Use = "forkbpf " cmd.Args = cobra.ExactArgs(5) cmd.Short = "Mount bpffs" cmd.Long = `Description: Mount bpffs This internal command is used to mount a bpf filesystem into the container for bpf token delegation. ` cmd.Hidden = true cmd.Run = c.run return cmd } func (c *cmdForkbpf) run(_ *cobra.Command, args []string) { mountPath := args[0] cmdTypes := strings.Split(args[1], ",") mapTypes := strings.Split(args[2], ",") progTypes := strings.Split(args[3], ",") attachTypes := strings.Split(args[4], ",") if C.incus_forkbpf_pid == 0 { err := c.runChild(int(C.incus_forkbpf_socket_child), mountPath) if err != nil { _, _ = fmt.Fprintf(os.Stdout, "[child]: %v", err) os.Exit(1) // nolint:revive } } else { childProc, err := os.FindProcess(int(C.incus_forkbpf_pid)) if err != nil { _, _ = fmt.Fprint(os.Stdout, "[parent]: Couldn't find child, assuming it already failed, exiting") os.Exit(1) // nolint:revive } err = c.runParent(int(C.incus_forkbpf_socket_parent), cmdTypes, mapTypes, progTypes, attachTypes) if err != nil { _, _ = fmt.Fprintf(os.Stdout, "[parent]: Encountered error, killing child: %v", err) err2 := childProc.Kill() if err2 != nil { _, _ = fmt.Fprintf(os.Stdout, "[parent]: Failed to kill child: %v", err2) } else { _, err2 = childProc.Wait() if err2 != nil { _, _ = fmt.Fprintf(os.Stdout, "[parent]: Failed to wait for child: %v", err2) } } os.Exit(1) // nolint:revive } procState, err := childProc.Wait() if err != nil { _, _ = fmt.Fprintf(os.Stdout, "[parent]: Failed to wait for child: %v", err) } else { if !procState.Success() { _, _ = fmt.Fprint(os.Stdout, "[parent]: Child process failed") os.Exit(1) // nolint:revive } } } } func (c *cmdForkbpf) runChild(socket int, mountPath string) error { fsfd, err := unix.Fsopen("bpf", 0) if err != nil { return fmt.Errorf("Failed to open bpf fs: %v", err) } rights := unix.UnixRights(fsfd) err = unix.Sendmsg(socket, nil, rights, nil, 0) if err != nil { return fmt.Errorf("Failed to send bpf fs fd to parent: %v", err) } data := make([]byte, unix.CmsgSpace(4)) _, _, _, _, err = unix.Recvmsg(socket, nil, data, 0) if err != nil { return fmt.Errorf("Failed to receive from parent: %v", err) } cmsgs, err := unix.ParseSocketControlMessage(data) if err != nil { return fmt.Errorf("Failed to parse message from parent: %v", err) } fds, err := unix.ParseUnixRights(&cmsgs[0]) if err != nil { return fmt.Errorf("Failed to parse fd from parent: %v", err) } err = unix.MoveMount(fds[0], "", unix.AT_FDCWD, mountPath, unix.MOVE_MOUNT_F_EMPTY_PATH) if err != nil { return fmt.Errorf("Failed to attach bpf fs mount: %v", err) } return nil } func (c *cmdForkbpf) setBpfDelegate(bpfFd int, key string, values []string) error { valuesTrimmed := make([]string, 0, len(values)) for _, value := range values { trimmed := strings.TrimSpace(value) if trimmed != "" { valuesTrimmed = append(valuesTrimmed, trimmed) } } if len(valuesTrimmed) == 0 { return nil } valuesJoined := strings.Join(valuesTrimmed, ":") err := unix.FsconfigSetString(bpfFd, key, valuesJoined) if err != nil { return fmt.Errorf("Failed to set %s=%v on bpf fs: %v", key, valuesJoined, err) } return nil } func (c *cmdForkbpf) runParent(socket int, cmdTypes []string, mapTypes []string, progTypes []string, attachTypes []string) error { data := make([]byte, unix.CmsgSpace(4)) _, _, _, _, err := unix.Recvmsg(socket, nil, data, 0) if err != nil { return fmt.Errorf("Failed to receive from child: %v", err) } cmsgs, err := unix.ParseSocketControlMessage(data) if err != nil { return fmt.Errorf("Failed to parse message from child: %v", err) } fds, err := unix.ParseUnixRights(&cmsgs[0]) if err != nil { return fmt.Errorf("Failed to parse fd from child: %v", err) } bpfFd := fds[0] err = c.setBpfDelegate(bpfFd, "delegate_cmds", cmdTypes) if err != nil { return err } err = c.setBpfDelegate(bpfFd, "delegate_maps", mapTypes) if err != nil { return err } err = c.setBpfDelegate(bpfFd, "delegate_progs", progTypes) if err != nil { return err } err = c.setBpfDelegate(bpfFd, "delegate_attachs", attachTypes) if err != nil { return err } err = unix.FsconfigCreate(bpfFd) if err != nil { return fmt.Errorf("Failed to create bpf fs: %v", err) } mountFd, err := unix.Fsmount(bpfFd, 0, 0) if err != nil { return fmt.Errorf("Failed to mount bpf fs: %v", err) } rights := unix.UnixRights(mountFd) err = unix.Sendmsg(socket, nil, rights, nil, 0) if err != nil { return fmt.Errorf("Failed to send mount fd to child: %v", err) } return nil } incus-7.0.0/cmd/incusd/main_forkconsole.go000066400000000000000000000036011517523235500205560ustar00rootroot00000000000000package main import ( "errors" "fmt" "os" "strconv" "strings" liblxc "github.com/lxc/go-lxc" "github.com/spf13/cobra" ) type cmdForkconsole struct { global *cmdGlobal } func (c *cmdForkconsole) command() *cobra.Command { // Main subcommand cmd := &cobra.Command{} cmd.Use = "forkconsole " cmd.Short = "Attach to the console of a container" cmd.Long = `Description: Attach to the console of a container This internal command is used to attach to one of the container's tty devices. ` cmd.RunE = c.run cmd.Hidden = true return cmd } func (c *cmdForkconsole) run(cmd *cobra.Command, args []string) error { // Quick checks. if len(args) != 5 { _ = cmd.Help() if len(args) == 0 { return nil } return errors.New("Missing required arguments") } // Only root should run this if os.Geteuid() != 0 { return errors.New("This must be run as root") } name := args[0] lxcpath := args[1] configPath := args[2] ttyNum := strings.TrimPrefix(args[3], "tty=") tty, err := strconv.Atoi(ttyNum) if err != nil { return fmt.Errorf("Failed to retrieve tty number: %q", err) } escapeNum := strings.TrimPrefix(args[4], "escape=") escape, err := strconv.Atoi(escapeNum) if err != nil { return fmt.Errorf("Failed to retrieve escape character: %q", err) } d, err := liblxc.NewContainer(name, lxcpath) if err != nil { return fmt.Errorf("Error initializing container: %q", err) } err = d.LoadConfigFile(configPath) if err != nil { return fmt.Errorf("Error opening config file: %q", err) } opts := liblxc.ConsoleOptions{} opts.Tty = tty opts.StdinFd = uintptr(os.Stdin.Fd()) opts.StdoutFd = uintptr(os.Stdout.Fd()) opts.StderrFd = uintptr(os.Stderr.Fd()) opts.EscapeCharacter = rune(escape) err = d.Console(opts) if err != nil { return fmt.Errorf("Failed running forkconsole: %q", err) } return nil } incus-7.0.0/cmd/incusd/main_forkcoresched.go000066400000000000000000000044201517523235500210530ustar00rootroot00000000000000package main /* #include "config.h" #include #include #include #include #include #include #include #include #include #include #include "incus.h" #include "memory_utils.h" #include "mount_utils.h" #include "syscall_numbers.h" #include "syscall_wrappers.h" void forkcoresched(void) { char *cur = NULL; char *pidstr; int hook; int ret; __u64 cookie; // Check that we're root if (geteuid() != 0) _exit(EXIT_FAILURE); // Get the subcommand cur = advance_arg(false); if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) _exit(EXIT_SUCCESS); ret = core_scheduling_cookie_create_thread(0); if (ret) _exit(EXIT_FAILURE); cookie = core_scheduling_cookie_get(0); if (!core_scheduling_cookie_valid(cookie)) _exit(EXIT_FAILURE); hook = atoi(cur); switch (hook) { case 0: for (pidstr = cur; pidstr; pidstr = advance_arg(false)) { ret = core_scheduling_cookie_share_to(atoi(pidstr)); if (ret) _exit(EXIT_FAILURE); cookie = core_scheduling_cookie_get(0); if (!core_scheduling_cookie_valid(cookie)) _exit(EXIT_FAILURE); } break; case 1: pidstr = getenv("LXC_PID"); if (!pidstr) _exit(EXIT_FAILURE); ret = core_scheduling_cookie_share_to(atoi(pidstr)); if (ret) _exit(EXIT_FAILURE); cookie = core_scheduling_cookie_get(0); if (!core_scheduling_cookie_valid(cookie)) _exit(EXIT_FAILURE); break; default: _exit(EXIT_FAILURE); } _exit(EXIT_SUCCESS); } */ import "C" import ( "errors" "github.com/spf13/cobra" // Used by cgo _ "github.com/lxc/incus/v7/shared/cgo" ) type cmdForkcoresched struct { global *cmdGlobal } func (c *cmdForkcoresched) command() *cobra.Command { // Main subcommand cmd := &cobra.Command{} cmd.Use = "forkcoresched [...]" cmd.Short = "Create new core scheduling domain" cmd.Long = `Description: Create new core scheduling domain This command is used to move a set of processes into a new core scheduling domain. ` cmd.RunE = c.run cmd.Hidden = true return cmd } func (c *cmdForkcoresched) run(_ *cobra.Command, _ []string) error { return errors.New("This command should have been intercepted in cgo") } incus-7.0.0/cmd/incusd/main_forkexec.go000066400000000000000000000211131517523235500200360ustar00rootroot00000000000000package main /* #include "config.h" #include #include #include #include #include #include #include #include #include #include #include #include "incus.h" #include "file_utils.h" #include "macro.h" #include "memory_utils.h" #include "process_utils.h" #include "syscall_wrappers.h" #include #include define_cleanup_function(struct lxc_container *, lxc_container_put); static int fd_cloexec(int fd, bool cloexec) { int oflags, nflags; oflags = fcntl(fd, F_GETFD, 0); if (oflags < 0) return -errno; if (cloexec) nflags = oflags | FD_CLOEXEC; else nflags = oflags & ~FD_CLOEXEC; if (nflags == oflags) return 0; if (fcntl(fd, F_SETFD, nflags) < 0) return -errno; return 0; } static int safe_int(const char *numstr, int *converted) { char *err = NULL; signed long int sli; errno = 0; sli = strtol(numstr, &err, 0); if (errno == ERANGE && (sli == LONG_MAX || sli == LONG_MIN)) return -ERANGE; if (errno != 0 && sli == 0) return -EINVAL; if (err == numstr || *err != '\0') return -EINVAL; if (sli > INT_MAX || sli < INT_MIN) return -ERANGE; *converted = (int)sli; return 0; } static inline bool match_stdfds(int fd) { return (fd == STDIN_FILENO || fd == STDOUT_FILENO || fd == STDERR_FILENO); } int close_inherited(int *fds_to_ignore, size_t len_fds) { int fddir; DIR *dir; struct dirent *direntp; restart: dir = opendir("/proc/self/fd"); if (!dir) return -errno; fddir = dirfd(dir); while ((direntp = readdir(dir))) { int fd, ret; size_t i; if (strcmp(direntp->d_name, ".") == 0) continue; if (strcmp(direntp->d_name, "..") == 0) continue; ret = safe_int(direntp->d_name, &fd); if (ret < 0) continue; for (i = 0; i < len_fds; i++) if (fds_to_ignore[i] == fd) break; if (fd == fddir || (i < len_fds && fd == fds_to_ignore[i])) continue; if (match_stdfds(fd)) continue; if (close(fd)) { return log_error(-errno, "%s - Failed to close file descriptor %d", strerror(errno), fd); } else { char fdpath[PATH_MAX], realpath[PATH_MAX]; snprintf(fdpath, sizeof(fdpath), "/proc/self/fd/%d", fd); ret = readlink(fdpath, realpath, PATH_MAX); if (ret < 0) snprintf(realpath, sizeof(realpath), "unknown"); else if (ret >= sizeof(realpath)) realpath[sizeof(realpath) - 1] = '\0'; log_error(-errno, "Closing unexpected file descriptor %d -> %s", fd, realpath); } closedir(dir); goto restart; } closedir(dir); return 0; } #define EXEC_STDIN_FD 3 #define EXEC_STDOUT_FD 4 #define EXEC_STDERR_FD 5 #define EXEC_PIPE_FD 6 #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) // We use a separate function because cleanup macros are called during stack // unwinding if I'm not mistaken and if the compiler knows it exits it won't // call them. That's not a problem since we're exiting but I just like to be on // the safe side in case we ever call this from a different context. We also // tell the compiler to not inline us. __attribute__ ((noinline)) static int __forkexec(void) { __do_close int status_pipe = EXEC_PIPE_FD; __do_free_string_list char **argvp = NULL, **envvp = NULL; call_cleaner(lxc_container_put) struct lxc_container *c = NULL; const char *config_path = NULL, *lxcpath = NULL, *name = NULL; char *cwd = NULL; pid_t init_pid; lxc_attach_options_t attach_options = LXC_ATTACH_OPTIONS_DEFAULT; lxc_attach_command_t command = { .program = NULL, }; int fds_to_ignore[] = {EXEC_STDIN_FD, EXEC_STDOUT_FD, EXEC_STDERR_FD, EXEC_PIPE_FD}; ssize_t ret; pid_t attached_pid; uid_t uid; gid_t gid; int coresched; if (geteuid() != 0) return log_error(EXIT_FAILURE, "Error: forkexec requires root privileges"); name = advance_arg(false); if (name == NULL || (strcmp(name, "--help") == 0 || strcmp(name, "--version") == 0 || strcmp(name, "-h") == 0)) return 0; lxcpath = advance_arg(true); config_path = advance_arg(true); cwd = advance_arg(true); uid = atoi(advance_arg(true)); if (uid < 0) uid = (uid_t) - 1; gid = atoi(advance_arg(true)); if (gid < 0) gid = (gid_t) - 1; coresched = atoi(advance_arg(true)); if (coresched != 0 && coresched != 1) _exit(EXIT_FAILURE); for (char *arg = NULL, *section = NULL; (arg = advance_arg(false)); ) { if (!strcmp(arg, "--") && (!section || strcmp(section, "cmd"))) { section = NULL; continue; } if (!section) { section = arg; continue; } if (!strcmp(section, "env")) { if (!strncmp(arg, "HOME=", STRLITERALLEN("HOME="))) attach_options.initial_cwd = arg + STRLITERALLEN("HOME="); ret = push_vargs(&envvp, arg); if (ret < 0) return log_error(ret, "Failed to add %s to env array", arg); } else if (!strcmp(section, "cmd")) { ret = push_vargs(&argvp, arg); if (ret < 0) return log_error(ret, "Failed to add %s to arg array", arg); } else { return log_error(EXIT_FAILURE, "Invalid exec section %s", section); } } if (!argvp || !*argvp) return log_error(EXIT_FAILURE, "No command specified"); ret = incus_close_range(EXEC_PIPE_FD + 1, UINT_MAX, CLOSE_RANGE_UNSHARE); if (ret) { // Fallback to close_inherited() when the syscall is not // available or when CLOSE_RANGE_UNSHARE isn't supported. // On a regular kernel CLOSE_RANGE_UNSHARE should always be // available but openSUSE Leap 15.3 seems to have a partial // backport without CLOSE_RANGE_UNSHARE support. if (errno == ENOSYS || errno == EINVAL) ret = close_inherited(fds_to_ignore, ARRAY_SIZE(fds_to_ignore)); } if (ret) return log_error(EXIT_FAILURE, "Aborting attach to prevent leaking file descriptors into container"); ret = fd_cloexec(status_pipe, true); if (ret) return log_errno(EXIT_FAILURE, "Failed to make pipe close-on-exec"); c = lxc_container_new(name, lxcpath); if (!c) return log_error(EXIT_FAILURE, "Failed to load new container %s/%s", lxcpath, name); c->clear_config(c); if (!c->load_config(c, config_path)) return log_error(EXIT_FAILURE, "Failed to load config file %s for %s/%s", config_path, lxcpath, name); if (strcmp(cwd, "")) attach_options.initial_cwd = cwd; attach_options.env_policy = LXC_ATTACH_CLEAR_ENV; attach_options.extra_env_vars = envvp; attach_options.stdin_fd = 3; attach_options.stdout_fd = 4; attach_options.stderr_fd = 5; attach_options.uid = uid; attach_options.gid = gid; command.program = argvp[0]; command.argv = argvp; ret = c->attach(c, lxc_attach_run_command, &command, &attach_options, &attached_pid); if (ret < 0) return EXIT_FAILURE; ret = write_nointr(status_pipe, &attached_pid, sizeof(attached_pid)); if (ret < 0) { // Kill the child just to be safe. fprintf(stderr, "Failed to send pid %d of executing child to daemon. Killing child\n", attached_pid); kill(attached_pid, SIGKILL); goto out_reap; } if (coresched == 1) { pid_t pid; init_pid = c->init_pid(c); if (init_pid < 0) { kill(attached_pid, SIGKILL); goto out_reap; } pid = vfork(); if (pid < 0) { kill(attached_pid, SIGKILL); goto out_reap; } if (pid == 0) { __u64 cookie; ret = core_scheduling_cookie_share_with(init_pid); if (ret) _exit(EXIT_FAILURE); ret = core_scheduling_cookie_share_to(attached_pid); if (ret) _exit(EXIT_FAILURE); cookie = core_scheduling_cookie_get(attached_pid); if (!core_scheduling_cookie_valid(cookie)) _exit(EXIT_FAILURE); _exit(EXIT_SUCCESS); } ret = wait_for_pid(pid); if (ret) kill(attached_pid, SIGKILL); } out_reap: ret = wait_for_pid_status_nointr(attached_pid); if (ret < 0) return log_error(EXIT_FAILURE, "Failed to wait for child process %d", attached_pid); if (WIFEXITED(ret)) return WEXITSTATUS(ret); if (WIFSIGNALED(ret)) return 128 + WTERMSIG(ret); return EXIT_FAILURE; } void forkexec(void) { _exit(__forkexec()); } */ import "C" import ( "errors" "github.com/spf13/cobra" // Used by cgo _ "github.com/lxc/incus/v7/shared/cgo" ) type cmdForkexec struct { global *cmdGlobal } func (c *cmdForkexec) command() *cobra.Command { // Main subcommand cmd := &cobra.Command{} cmd.Use = "forkexec -- env [key=value...] -- cmd " cmd.Short = "Execute a task inside the container" cmd.Long = `Description: Execute a task inside the container This internal command is used to spawn a task inside the container and allow the daemon to interact with it. ` cmd.RunE = c.run cmd.Hidden = true return cmd } func (c *cmdForkexec) run(_ *cobra.Command, _ []string) error { return errors.New("This command should have been intercepted in cgo") } incus-7.0.0/cmd/incusd/main_forkfile.go000066400000000000000000000107521517523235500200400ustar00rootroot00000000000000package main /* #include "config.h" #include #include #include #include #include #include #include #include #include #include #include #include #include "incus.h" #include "memory_utils.h" void forkfile(void) { int ns_fd = -EBADF, pidfd = -EBADF, rootfs_fd = -EBADF; char *listenfd = NULL; pid_t pid = 0; // Check that we're root. if (geteuid() != 0) { fprintf(stderr, "Error: forkfile requires root privileges\n"); _exit(1); } // Check the first argument. listenfd = advance_arg(false); if (listenfd == NULL) return; if (strcmp(listenfd, "--") == 0) listenfd = advance_arg(false); if (listenfd == NULL || (strcmp(listenfd, "--help") == 0 || strcmp(listenfd, "--version") == 0 || strcmp(listenfd, "-h") == 0)) return; // Get the container rootfs. rootfs_fd = atoi(advance_arg(true)); // Get the container PID. pidfd = atoi(advance_arg(true)); pid = atoi(advance_arg(true)); if (pid > 0 || pidfd >= 0) { ns_fd = pidfd_nsfd(pidfd, pid); if (ns_fd < 0) { _exit(1); } } // Attach to the container. if (ns_fd >= 0) { int setns_flags = CLONE_NEWNS; if (in_same_namespace(getpid(), ns_fd, "user") > 0) setns_flags |= CLONE_NEWUSER; if (!change_namespaces(pidfd, ns_fd, setns_flags)) { error("error: setns"); _exit(1); } if (setns_flags & CLONE_NEWUSER) finalize_userns(); } else { if (fchdir(rootfs_fd) < 0) { error("error: fchdir"); _exit(1); } if (chroot(".") < 0) { error("error: chroot"); _exit(1); } if (chdir("/") < 0) { error("error: chdir"); _exit(1); } } } */ import "C" import ( "net" "os" "os/signal" "strconv" "sync" "time" "github.com/pkg/sftp" "github.com/spf13/cobra" "golang.org/x/sys/unix" ) type cmdForkfile struct { global *cmdGlobal } func (c *cmdForkfile) command() *cobra.Command { // Main subcommand cmd := &cobra.Command{} cmd.Use = "forkfile " cmd.Short = "Perform container file operations" cmd.Long = `Description: Perform container file operations This spawns a daemon inside of the instance's filesystem which can then receive command over a simple SFTP API operating on the provided listen fd. The command can be called with PID and PIDFd set to 0 to just operate on the rootfs fd. In such cases, it's the responsibility of the caller to handle any kind of userns shifting. ` cmd.Hidden = true cmd.Args = cobra.ExactArgs(4) cmd.RunE = c.run return cmd } func (c *cmdForkfile) run(_ *cobra.Command, args []string) error { var mu sync.RWMutex var connections uint64 var transactions uint64 // Convert the listener FD number. listenFD, err := strconv.Atoi(args[0]) if err != nil { return err } // Setup listener. listenerFile := os.NewFile(uintptr(listenFD), "forkfile.sock") listener, err := net.FileListener(listenerFile) if err != nil { return err } defer func() { _ = listener.Close() }() // Convert the rootfs FD number. rootfsFD, err := strconv.Atoi(args[1]) if err != nil { return err } // Automatically shutdown after inactivity. go func() { for { time.Sleep(10 * time.Second) // Check for active connections. mu.RLock() if connections > 0 { mu.RUnlock() continue } // Look for recent activity oldCount := transactions mu.RUnlock() time.Sleep(5 * time.Second) mu.RLock() if oldCount == transactions { mu.RUnlock() // Daemon has been inactive for 10s, exit. os.Exit(0) } mu.RUnlock() } }() // Signal handler. go func() { // Wait for SIGINT. sigs := make(chan os.Signal, 1) signal.Notify(sigs, unix.SIGINT) <-sigs // Prevent new connections. _ = listener.Close() // Wait for connections to be gone and exit. for { mu.RLock() if connections == 0 { mu.RUnlock() break } mu.RUnlock() time.Sleep(time.Second) } os.Exit(0) }() // Connection handler. for { // Accept new connection. conn, err := listener.Accept() if err != nil { continue } go func(conn net.Conn) { defer func() { _ = conn.Close() mu.Lock() connections -= 1 mu.Unlock() }() // Increase counters. mu.Lock() transactions += 1 connections += 1 mu.Unlock() // Spawn the server. server, err := sftp.NewServer(conn, sftp.WithAllocator()) if err != nil { return } _ = server.Serve() // Sync the filesystem. _ = unix.Syncfs(int(rootfsFD)) }(conn) } } incus-7.0.0/cmd/incusd/main_forklimits.go000066400000000000000000000064141517523235500204220ustar00rootroot00000000000000package main import ( "errors" "fmt" "os" "regexp" "strconv" "strings" "github.com/spf13/cobra" "golang.org/x/sys/unix" ) var reLimitsArg = regexp.MustCompile(`^limit=(\w+):(\w+):(\w+)$`) type cmdForklimits struct { global *cmdGlobal } func (c *cmdForklimits) command() *cobra.Command { // Main subcommand cmd := &cobra.Command{} cmd.Use = "forklimits [fd=...] [limit=::...] -- [...]" cmd.Short = "Execute a task inside the container" cmd.Long = `Description: Execute a command with specific limits set. This internal command is used to spawn a command with limits set. It can also pass through one or more filed escriptors specified by fd=n arguments. These are passed through in the order they are specified. ` cmd.RunE = c.run cmd.Hidden = true return cmd } func (c *cmdForklimits) run(cmd *cobra.Command, _ []string) error { // Use raw args instead of cobra passed args, as we need to access the "--" argument. args := c.global.rawArgs(cmd) if len(args) == 0 { _ = cmd.Help() return nil } // Only root should run this if os.Geteuid() != 0 { return errors.New("This must be run as root") } type limit struct { name string soft string hard string } var limits []limit var fds []uintptr var cmdParts []string for i, arg := range args { matches := reLimitsArg.FindStringSubmatch(arg) if len(matches) == 4 { limits = append(limits, limit{ name: matches[1], soft: matches[2], hard: matches[3], }) } else if strings.HasPrefix(arg, "fd=") { fdParts := strings.SplitN(arg, "=", 2) fdNum, err := strconv.Atoi(fdParts[1]) if err != nil { _ = cmd.Help() return errors.New("Invalid file descriptor number") } fds = append(fds, uintptr(fdNum)) } else if arg == "--" { if len(args)-1 > i { cmdParts = args[i+1:] } break // No more passing of arguments needed. } else { _ = cmd.Help() return errors.New("Unrecognised argument") } } // Setup rlimits. for _, limit := range limits { var resource int var rLimit unix.Rlimit if limit.name == "memlock" { resource = unix.RLIMIT_MEMLOCK } else { return fmt.Errorf("Unsupported limit type: %q", limit.name) } if limit.soft == "unlimited" { rLimit.Cur = unix.RLIM_INFINITY } else { softLimit, err := strconv.ParseUint(limit.soft, 10, 64) if err != nil { return fmt.Errorf("Invalid soft limit for %q", limit.name) } rLimit.Cur = softLimit } if limit.hard == "unlimited" { rLimit.Max = unix.RLIM_INFINITY } else { hardLimit, err := strconv.ParseUint(limit.hard, 10, 64) if err != nil { return fmt.Errorf("Invalid hard limit for %q", limit.name) } rLimit.Max = hardLimit } err := unix.Setrlimit(resource, &rLimit) if err != nil { return err } } if len(cmdParts) == 0 { _ = cmd.Help() return errors.New("Missing required command argument") } // Clear the cloexec flag on the file descriptors we are passing through. for _, fd := range fds { _, _, syscallErr := unix.Syscall(unix.SYS_FCNTL, fd, unix.F_SETFD, uintptr(0)) if syscallErr != 0 { err := os.NewSyscallError(fmt.Sprintf("fcntl failed on FD %d", fd), syscallErr) if err != nil { return err } } } return unix.Exec(cmdParts[0], cmdParts, os.Environ()) } incus-7.0.0/cmd/incusd/main_forkmigrate.go000066400000000000000000000032071517523235500205460ustar00rootroot00000000000000package main import ( "errors" "fmt" "os" "strconv" liblxc "github.com/lxc/go-lxc" "github.com/spf13/cobra" ) type cmdForkmigrate struct { global *cmdGlobal } func (c *cmdForkmigrate) command() *cobra.Command { // Main subcommand cmd := &cobra.Command{} cmd.Use = "forkmigrate " cmd.Short = "Restore the container from saved state" cmd.Long = `Description: Restore the container from saved state This internal command is used to start the container as a separate process, restoring its recorded state. ` cmd.RunE = c.run cmd.Hidden = true return cmd } func (c *cmdForkmigrate) run(cmd *cobra.Command, args []string) error { // Quick checks. if len(args) != 5 { _ = cmd.Help() if len(args) == 0 { return nil } return errors.New("Missing required arguments") } // Only root should run this if os.Geteuid() != 0 { return errors.New("This must be run as root") } name := args[0] lxcpath := args[1] configPath := args[2] imagesDir := args[3] preservesInodes, err := strconv.ParseBool(args[4]) if err != nil { return err } d, err := liblxc.NewContainer(name, lxcpath) if err != nil { return err } err = d.LoadConfigFile(configPath) if err != nil { return fmt.Errorf("Failed loading config file %q: %w", configPath, err) } /* see https://github.com/golang/go/issues/13155, startContainer, and dc3a229 */ _ = os.Stdin.Close() _ = os.Stdout.Close() _ = os.Stderr.Close() return d.Migrate(liblxc.MIGRATE_RESTORE, liblxc.MigrateOptions{ Directory: imagesDir, Verbose: true, PreservesInodes: preservesInodes, }) } incus-7.0.0/cmd/incusd/main_forkmount.go000066400000000000000000000362551517523235500202710ustar00rootroot00000000000000package main /* #include "config.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "incus.h" #include "memory_utils.h" #include "mount_utils.h" #include "syscall_numbers.h" #include "syscall_wrappers.h" #define VERSION_AT_LEAST(major, minor, micro) \ ((LXC_DEVEL == 1) || (!(major > LXC_VERSION_MAJOR || \ major == LXC_VERSION_MAJOR && minor > LXC_VERSION_MINOR || \ major == LXC_VERSION_MAJOR && minor == LXC_VERSION_MINOR && micro > LXC_VERSION_MICRO))) static int mkdir_p(const char *dir, mode_t mode) { const char *tmp = dir; const char *orig = dir; do { __do_free char *makeme = NULL; dir = tmp + strspn(tmp, "/"); tmp = dir + strcspn(dir, "/"); makeme = strndup(orig, dir - orig); if (*makeme) { if (mkdir(makeme, mode) && errno != EEXIST) { fprintf(stderr, "failed to create directory '%s': %s\n", makeme, strerror(errno)); return -1; } } } while(tmp != dir); return 0; } static void ensure_dir(char *dest) { struct stat sb; if (stat(dest, &sb) == 0) { if ((sb.st_mode & S_IFMT) == S_IFDIR) return; if (unlink(dest) < 0) { fprintf(stderr, "Failed to remove old %s: %s\n", dest, strerror(errno)); _exit(1); } } if (mkdir(dest, 0755) < 0) { fprintf(stderr, "Failed to mkdir %s: %s\n", dest, strerror(errno)); _exit(1); } } static void ensure_file(char *dest) { __do_close int fd = -EBADF; struct stat sb; if (stat(dest, &sb) == 0) { if ((sb.st_mode & S_IFMT) != S_IFDIR) return; if (rmdir(dest) < 0) { fprintf(stderr, "Failed to remove old %s: %s\n", dest, strerror(errno)); _exit(1); } } fd = creat(dest, 0755); if (fd < 0) { fprintf(stderr, "Failed to mkdir %s: %s\n", dest, strerror(errno)); _exit(1); } } static void create(int fd_src, char *src, char *dest) { __do_free char *dirdup = NULL; char *destdirname; struct stat sb; if (src) { if (stat(src, &sb) < 0) die("source %s does not exist", src); } else { if (fstat(fd_src, &sb) < 0) die("source %s does not exist", src); } dirdup = strdup(dest); if (!dirdup) _exit(1); destdirname = dirname(dirdup); if (mkdir_p(destdirname, 0755) < 0) { fprintf(stderr, "failed to create path: %s\n", destdirname); _exit(1); } switch (sb.st_mode & S_IFMT) { case S_IFDIR: ensure_dir(dest); return; default: ensure_file(dest); return; } } static int lxc_safe_ulong(const char *numstr, unsigned long *converted) { char *err = NULL; unsigned long int uli; while (isspace(*numstr)) numstr++; if (*numstr == '-') return -EINVAL; errno = 0; uli = strtoul(numstr, &err, 0); if (errno == ERANGE && uli == ULONG_MAX) return -ERANGE; if (err == numstr || *err != '\0') return -EINVAL; *converted = uli; return 0; } static void do_incus_forkmount(int pidfd, int ns_fd) { unsigned long mntflags = 0; int fd_tree = -EBADF; int ret; char *src, *dest, *idmapType, *flags; src = advance_arg(true); dest = advance_arg(true); idmapType = advance_arg(true); flags = advance_arg(true); if (strcmp(idmapType, "idmapped") == 0) { int fd_mntns, fd_userns; fd_userns = preserve_ns(-ESRCH, ns_fd, "user"); if (fd_userns < 0) { fprintf(stderr, "Failed to open user namespace of container: %s\n", strerror(errno)); _exit(1); } fd_mntns = preserve_ns(getpid(), -EBADF, "mnt"); if (fd_mntns < 0) { fprintf(stderr, "Failed to open mount namespace of container: %s\n", strerror(errno)); _exit(1); } if (!change_namespaces(pidfd, ns_fd, CLONE_NEWNS)) { fprintf(stderr, "Failed setns to container mount namespace: %s\n", strerror(errno)); _exit(1); } fd_tree = mount_detach_idmap(src, fd_userns); if (fd_tree < 0) { fprintf(stderr, "Failed to create detached idmapped mount \"%s\": %s\n", src, strerror(errno)); _exit(1); } ret = setns(fd_mntns, CLONE_NEWNS); if (ret) { fprintf(stderr, "Failed to switch to original mount namespace: %s\n", strerror(errno)); _exit(1); } close_prot_errno_disarm(fd_userns); close_prot_errno_disarm(fd_mntns); } attach_userns_fd(ns_fd); if (!change_namespaces(pidfd, ns_fd, CLONE_NEWNS)) { fprintf(stderr, "Failed setns to container mount namespace: %s\n", strerror(errno)); _exit(1); } create(-EBADF, src, dest); if (access(src, F_OK) < 0) { fprintf(stderr, "Mount source doesn't exist: %s\n", strerror(errno)); _exit(1); } if (access(dest, F_OK) < 0) { fprintf(stderr, "Mount destination doesn't exist: %s\n", strerror(errno)); _exit(1); } if (fd_tree >= 0) { ret = incus_move_mount(fd_tree, "", -EBADF, dest, MOVE_MOUNT_F_EMPTY_PATH); if (ret) { fprintf(stderr, "Failed to move detached mount to target from %d to %s: %s\n", fd_tree, dest, strerror(errno)); _exit(1); } close_prot_errno_disarm(fd_tree); _exit(0); } ret = lxc_safe_ulong(flags, &mntflags); if (ret < 0) _exit(1); // Here, we always move recursively, because we sometimes allow // recursive mounts. If the mount has no kids then it doesn't matter, // but if it does, we want to move those too. if (mount(src, dest, "none", MS_MOVE | MS_REC, NULL) < 0) { fprintf(stderr, "Failed mounting %s onto %s: %s\n", src, dest, strerror(errno)); _exit(1); } _exit(0); } static int lxc_safe_uint(const char *numstr, unsigned int *converted) { char *err = NULL; unsigned long int uli; while (isspace(*numstr)) numstr++; if (*numstr == '-') return -EINVAL; errno = 0; uli = strtoul(numstr, &err, 0); if (errno == ERANGE && uli == ULONG_MAX) return -ERANGE; if (err == numstr || *err != '\0') return -EINVAL; if (uli > UINT_MAX) return -ERANGE; *converted = (unsigned int)uli; return 0; } static int mnt_attributes_new(unsigned int old_flags, unsigned int *new_flags) { unsigned int flags = 0; if (old_flags & MS_RDONLY) { flags |= MOUNT_ATTR_RDONLY; old_flags &= ~MS_RDONLY; } if (old_flags & MS_NOSUID) { flags |= MOUNT_ATTR_NOSUID; old_flags &= ~MS_NOSUID; } if (old_flags & MS_NODEV) { flags |= MOUNT_ATTR_NODEV; old_flags &= ~MS_NODEV; } if (old_flags & MS_NOEXEC) { flags |= MOUNT_ATTR_NOEXEC; old_flags &= ~MS_NOEXEC; } if (old_flags & MS_RELATIME) { flags |= MOUNT_ATTR_RELATIME; old_flags &= ~MS_RELATIME; } if (old_flags & MS_NOATIME) { flags |= MOUNT_ATTR_NOATIME; old_flags &= ~MS_NOATIME; } if (old_flags & MS_STRICTATIME) { flags |= MOUNT_ATTR_STRICTATIME; old_flags &= ~MS_STRICTATIME; } if (old_flags & MS_NODIRATIME) { flags |= MOUNT_ATTR_NODIRATIME; old_flags &= ~MS_NODIRATIME; } *new_flags |= flags; return old_flags; } static int make_final_open(struct stat *st_src, const char *dest) { int ret; int flags = O_CLOEXEC | O_NOFOLLOW | O_NOCTTY | O_CLOEXEC; struct stat st_dest; ret = stat(dest, &st_dest); if (ret == 0) { if ((st_dest.st_mode & S_IFMT) == (st_src->st_mode & S_IFMT)) goto out_open; ret = remove(dest); if (ret) return -1; } if ((st_src->st_mode & S_IFMT) == S_IFDIR) ret = mkdir(dest, 0000); else ret = mknod(dest, S_IFREG | 0000, 0); if (ret) return -1; out_open: return open(dest, flags | O_PATH); } static int make_dest_open(int fd_src, const char *dest) { __do_free char *dirdup = NULL; int ret; char *destdirname; struct stat st_src; ret = fstat(fd_src, &st_src); if (ret) return -1; dirdup = strdup(dest); if (!dirdup) return -1; destdirname = dirname(dirdup); ret = mkdir_p(destdirname, 0755); if (ret) return -1; return make_final_open(&st_src, dest); } static void do_move_forkmount(int pidfd, int ns_fd) { __do_close int fs_fd = -EBADF, mnt_fd = -EBADF, fd_userns = -EBADF, dest_fd = -EBADF; int ret; char *fstype, *src, *dest, *idmapType, *flags; unsigned int old_mntflags = 0, new_mntflags = 0; fstype = advance_arg(true); src = advance_arg(true); dest = advance_arg(true); idmapType = advance_arg(true); flags = advance_arg(true); ret = lxc_safe_uint(flags, &old_mntflags); if (ret < 0) die("parse mount flags"); mnt_attributes_new(old_mntflags, &new_mntflags); if (strcmp(fstype, "") && strcmp(fstype, "none")) { fs_fd = incus_fsopen(fstype, FSOPEN_CLOEXEC); if (fs_fd < 0) die("fsopen: %s", fstype); ret = incus_fsconfig(fs_fd, FSCONFIG_SET_STRING, "source", src, 0); if (ret < 0) die("fsconfig: source"); ret = incus_fsconfig(fs_fd, FSCONFIG_CMD_CREATE, NULL, NULL, 0); if (ret < 0) die("fsconfig: create"); mnt_fd = incus_fsmount(fs_fd, FSMOUNT_CLOEXEC, new_mntflags); if (mnt_fd < 0) die("fsmount"); } else { mnt_fd = incus_open_tree(-EBADF, src, OPEN_TREE_CLOEXEC | OPEN_TREE_CLONE); if (mnt_fd < 0) die("open_tree"); } fd_userns = preserve_ns(-ESRCH, ns_fd, "user"); if (fd_userns < 0) die("preserve userns"); if (strcmp(idmapType, "idmapped") == 0) { struct lxc_mount_attr attr = { .attr_set = MOUNT_ATTR_IDMAP, .userns_fd = fd_userns, }; ret = incus_mount_setattr(mnt_fd, "", AT_EMPTY_PATH, &attr, sizeof(attr)); if (ret) die("idmap mount"); } attach_userns_fd(ns_fd); if (!change_namespaces(pidfd, ns_fd, CLONE_NEWNS)) die("Failed setns to container mount namespace"); dest_fd = make_dest_open(mnt_fd, dest); if (dest_fd < 0) die("Failed to create destination mount point"); ret = incus_move_mount(mnt_fd, "", dest_fd, "", MOVE_MOUNT_F_EMPTY_PATH | MOVE_MOUNT_T_EMPTY_PATH); if (ret) die("Failed to move detached mount to target from %d to %s", mnt_fd, dest); _exit(EXIT_SUCCESS); } static void do_incus_forkumount(int pidfd, int ns_fd) { int ret; char *path = NULL; if (!change_namespaces(pidfd, ns_fd, CLONE_NEWNS)) { fprintf(stderr, "Failed to setns to container mount namespace: %s\n", strerror(errno)); _exit(1); } path = advance_arg(true); ret = umount2(path, MNT_DETACH); if (ret < 0) { // - ENOENT: The user must have unmounted and removed the path. // - EINVAL: The user must have unmounted. Other explanations // for EINVAL do not apply. if (errno == ENOENT || errno == EINVAL) _exit(0); fprintf(stderr, "Error unmounting %s: %s\n", path, strerror(errno)); _exit(1); } _exit(0); } static void do_lxc_forkmount(void) { #if VERSION_AT_LEAST(3, 1, 0) int ret; char *config, *flags, *fstype, *lxcpath, *name, *source, *target; struct lxc_container *c; struct lxc_mount mnt = {0}; unsigned long mntflags = 0; name = advance_arg(true); lxcpath = advance_arg(true); config = advance_arg(true); source = advance_arg(true); target = advance_arg(true); fstype = advance_arg(true); flags = advance_arg(true); c = lxc_container_new(name, lxcpath); if (!c) _exit(1); c->clear_config(c); if (!c->load_config(c, config)) { lxc_container_put(c); _exit(1); } ret = lxc_safe_ulong(flags, &mntflags); if (ret < 0) { lxc_container_put(c); _exit(1); } ret = c->mount(c, source, target, fstype, mntflags, NULL, &mnt); lxc_container_put(c); if (ret < 0) _exit(1); _exit(0); #else fprintf(stderr, "error: Called lxc_forkmount when missing LXC support\n"); _exit(1); #endif } static void do_lxc_forkumount(void) { #if VERSION_AT_LEAST(3, 1, 0) int ret; char *config, *lxcpath, *name, *target; struct lxc_container *c; struct lxc_mount mnt = {0}; name = advance_arg(true); lxcpath = advance_arg(true); config = advance_arg(true); target = advance_arg(true); c = lxc_container_new(name, lxcpath); if (!c) _exit(1); c->clear_config(c); if (!c->load_config(c, config)) { lxc_container_put(c); _exit(1); } ret = c->umount(c, target, MNT_DETACH, &mnt); lxc_container_put(c); if (ret < 0) _exit(1); _exit(0); #else fprintf(stderr, "error: Called lxc_forkumount when missing LXC support\n"); _exit(1); #endif } void forkmount(void) { char *command = NULL, *cur = NULL; int ns_fd = -EBADF, pidfd = -EBADF; pid_t pid = 0; // Get the subcommand command = advance_arg(false); if (command == NULL || (strcmp(command, "--help") == 0 || strcmp(command, "--version") == 0 || strcmp(command, "-h") == 0)) return; // Check that we're root if (geteuid() != 0) { fprintf(stderr, "Error: forkmount requires root privileges\n"); _exit(1); } // skip "--" advance_arg(true); // Call the subcommands if (strcmp(command, "go-mount") == 0) { // Get the pid cur = advance_arg(false); if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) return; pid = atoi(cur); if (pid <= 0) _exit(EXIT_FAILURE); pidfd = atoi(advance_arg(true)); ns_fd = pidfd_nsfd(pidfd, pid); if (ns_fd < 0) _exit(EXIT_FAILURE); do_incus_forkmount(pidfd, ns_fd); } else if (strcmp(command, "lxc-mount") == 0) { do_lxc_forkmount(); } else if (strcmp(command, "move-mount") == 0) { // Get the pid cur = advance_arg(false); if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) return; pid = atoi(cur); if (pid <= 0) _exit(EXIT_FAILURE); pidfd = atoi(advance_arg(true)); ns_fd = pidfd_nsfd(pidfd, pid); if (ns_fd < 0) _exit(EXIT_FAILURE); do_move_forkmount(pidfd, ns_fd); } else if (strcmp(command, "go-umount") == 0) { // Get the pid cur = advance_arg(false); if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) return; pid = atoi(cur); if (pid <= 0) _exit(EXIT_FAILURE); pidfd = atoi(advance_arg(true)); ns_fd = pidfd_nsfd(pidfd, pid); if (ns_fd < 0) _exit(EXIT_FAILURE); do_incus_forkumount(pidfd, ns_fd); } else if (strcmp(command, "lxc-umount") == 0) { do_lxc_forkumount(); } } */ import "C" import ( "errors" "github.com/spf13/cobra" // Used by cgo _ "github.com/lxc/incus/v7/shared/cgo" ) type cmdForkmount struct { global *cmdGlobal } func (c *cmdForkmount) command() *cobra.Command { // Main subcommand cmd := &cobra.Command{} cmd.Use = "forkmount" cmd.Short = "Perform mount operations" cmd.Long = `Description: Perform mount operations This set of internal commands are used for all container mount operations. ` cmd.Hidden = true // mount cmdLXCMount := &cobra.Command{} cmdLXCMount.Use = "lxc-mount " cmdLXCMount.Args = cobra.ExactArgs(7) cmdLXCMount.RunE = c.run cmd.AddCommand(cmdLXCMount) cmdGoMount := &cobra.Command{} cmdGoMount.Use = "go-mount " cmdGoMount.Args = cobra.ExactArgs(6) cmdGoMount.RunE = c.run cmd.AddCommand(cmdGoMount) // umount cmdLXCUmount := &cobra.Command{} cmdLXCUmount.Use = "lxc-umount " cmdLXCUmount.Args = cobra.ExactArgs(4) cmdLXCUmount.RunE = c.run cmd.AddCommand(cmdLXCUmount) cmdGoUmount := &cobra.Command{} cmdGoUmount.Use = "go-umount " cmdGoUmount.Args = cobra.ExactArgs(3) cmdGoUmount.RunE = c.run cmd.AddCommand(cmdGoUmount) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } return cmd } func (c *cmdForkmount) run(_ *cobra.Command, _ []string) error { return errors.New("This command should have been intercepted in cgo") } incus-7.0.0/cmd/incusd/main_forknet.go000066400000000000000000000532121517523235500177050ustar00rootroot00000000000000package main /* #include "config.h" #include #include #include #include #include #include #include #include #include #include #include "incus.h" #include "macro.h" #include "memory_utils.h" #include "process_utils.h" static int dosetns_file(char *file, char *nstype) { __do_close int ns_fd = -EBADF; ns_fd = open(file, O_RDONLY); if (ns_fd < 0) { fprintf(stderr, "%m - Failed to open \"%s\"", file); return -1; } if (setns(ns_fd, 0) < 0) { fprintf(stderr, "%m - Failed to attach to namespace \"%s\"", file); return -1; } return 0; } static void forkdonetdetach(char *file) { // Attach to the network namespace. if (dosetns_file(file, "net") < 0) { fprintf(stderr, "Failed setns to container network namespace: %s\n", strerror(errno)); _exit(1); } if (unshare(CLONE_NEWNS) < 0) { fprintf(stderr, "Failed to create new mount namespace: %s\n", strerror(errno)); _exit(1); } if (mount(NULL, "/", NULL, MS_REC | MS_PRIVATE, NULL) < 0) { fprintf(stderr, "Failed to mark / private: %s\n", strerror(errno)); _exit(1); } if (mount("sysfs", "/sys", "sysfs", 0, NULL) < 0) { fprintf(stderr, "Failed mounting new sysfs: %s\n", strerror(errno)); _exit(1); } // Jump back to Go for the rest } int forknet_dhcp_logfile = -1; static void forkdonetdhcp(char *logfilestr) { char *pidstr; char path[PATH_MAX]; pid_t pid; pidstr = getenv("LXC_PID"); if (!pidstr) { fprintf(stderr, "No LXC_PID in environment\n"); _exit(1); } // Attach to the network namespace. snprintf(path, sizeof(path), "/proc/%s/ns/net", pidstr); if (dosetns_file(path, "net") < 0) { fprintf(stderr, "Failed setns to container network namespace: %s\n", strerror(errno)); _exit(1); } forknet_dhcp_logfile = open(logfilestr, O_WRONLY | O_APPEND); if (forknet_dhcp_logfile < 0) { fprintf(stderr, "Failed to open logfile %s: %s\n", logfilestr, strerror(errno)); fprintf(stderr, "Execution will continue but log output will be lost after daemonize\n"); } // Run in the background. pid = fork(); if (pid < 0) { fprintf(stderr, "%s - Failed to create new process\n", strerror(errno)); _exit(EXIT_FAILURE); } if (pid > 0) { _exit(EXIT_SUCCESS); } if (!freopen("/dev/null", "r", stdin)) { fprintf(stderr, "Failed to reconfigure stdin: %s\n", strerror(errno)); _exit(1); } if (!freopen("/dev/null", "w", stdout)) { fprintf(stderr, "Failed to reconfigure stdout: %s\n", strerror(errno)); _exit(1); } if (!freopen("/dev/null", "w", stderr)) { fprintf(stderr, "Failed to reconfigure stderr: %s\n", strerror(errno)); _exit(1); } if (setsid() < 0) { fprintf(stderr, "%s - Failed to setup new session\n", strerror(errno)); _exit(EXIT_FAILURE); } pid = fork(); if (pid < 0) { fprintf(stderr, "%s - Failed to create new process\n", strerror(errno)); _exit(EXIT_FAILURE); } if (pid > 0) { _exit(EXIT_SUCCESS); } // Set the process title. char *workdir = advance_arg(false); if (workdir != NULL) { char *title = malloc(sizeof(char)*strlen(workdir)+19); if (title != NULL) { sprintf(title, "[incus dhcp] %s eth0", workdir); (void)setproctitle(title); } } // Jump back to Go for the rest } void forknet(void) { char *command = NULL; char *cur = NULL; // Get the subcommand command = advance_arg(false); if (command == NULL || (strcmp(command, "--help") == 0 || strcmp(command, "--version") == 0 || strcmp(command, "-h") == 0)) { return; } if (strcmp(command, "dhcp") == 0) { advance_arg(false); // skip instance directory cur = advance_arg(false); // get the logfile path forkdonetdhcp(cur); return; } // skip "--" advance_arg(true); // Get the netns file path. cur = advance_arg(false); if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) { return; } // Check that we're root if (geteuid() != 0) { fprintf(stderr, "Error: forknet requires root privileges\n"); _exit(1); } if (strcmp(command, "detach") == 0) forkdonetdetach(cur); } */ import "C" import ( "context" "errors" "fmt" "io" "math/rand" "net" "os" "path/filepath" "strings" "sync" "time" "github.com/insomniacslk/dhcp/dhcpv4" "github.com/insomniacslk/dhcp/dhcpv4/nclient4" "github.com/insomniacslk/dhcp/dhcpv6" "github.com/insomniacslk/dhcp/dhcpv6/nclient6" "github.com/insomniacslk/dhcp/iana" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/server/ip" _ "github.com/lxc/incus/v7/shared/cgo" // Used by cgo "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) type cmdForknet struct { global *cmdGlobal applyDNSMu sync.Mutex dhcpv4Leases map[string]*nclient4.Lease dhcpv6Leases map[string]*dhcpv6.Message instNetworkPath string } func (c *cmdForknet) command() *cobra.Command { // Main subcommand cmd := &cobra.Command{} cmd.Use = "forknet" cmd.Short = "Perform container network operations" cmd.Long = `Description: Perform container network operations This set of internal commands are used for some container network operations which require attaching to the container's network namespace. ` cmd.Hidden = true // detach cmdDetach := &cobra.Command{} cmdDetach.Use = "detach " cmdDetach.Args = cobra.ExactArgs(4) cmdDetach.RunE = c.runDetach cmd.AddCommand(cmdDetach) // dhclient cmdDHCP := &cobra.Command{} cmdDHCP.Use = "dhcp " cmdDHCP.Args = cobra.ExactArgs(2) cmdDHCP.RunE = c.runDHCP cmd.AddCommand(cmdDHCP) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } return cmd } // RunDHCP spawns the DHCP client(s) and applies address, route and DNS configuration. func (c *cmdForknet) runDHCP(_ *cobra.Command, args []string) error { logger := logrus.New() logger.Level = logrus.DebugLevel c.instNetworkPath = args[0] if C.forknet_dhcp_logfile >= 0 { logger.SetOutput(os.NewFile(uintptr(C.forknet_dhcp_logfile), "incus-dhcp-logfile")) } else { logger.SetOutput(io.Discard) } // Read the hostname. bb, err := os.ReadFile(filepath.Join(c.instNetworkPath, "hostname")) if err != nil { logger.WithError(err).Error("Unable to read hostname file") } hostname := strings.TrimSpace(string(bb)) // Create PID file. err = os.WriteFile(filepath.Join(c.instNetworkPath, "dhcp.pid"), []byte(fmt.Sprintf("%d", os.Getpid())), 0o644) if err != nil { logger.WithError(err).Error("Giving up on DHCP, couldn't write PID file") return err } // Enumerate network interfaces and skip loopback. ifaces, err := net.Interfaces() if err != nil { logger.WithError(err).Error("Giving up on DHCP, couldn't list interfaces") return err } var names []string for _, ifi := range ifaces { if ifi.Flags&net.FlagLoopback != 0 { continue } names = append(names, ifi.Name) } if len(names) == 0 { logger.Info("No non-loopback interfaces found; nothing to do for DHCP") return nil } // Initialize per-interface lease maps. c.applyDNSMu.Lock() c.dhcpv4Leases = map[string]*nclient4.Lease{} c.dhcpv6Leases = map[string]*dhcpv6.Message{} c.applyDNSMu.Unlock() // Buffer size is 2 goroutines per iface. errorChannel := make(chan error, len(names)*2) // Launch DHCP clients for each iface. for _, iface := range names { logger := logger.WithField("interface", iface).Logger logger.Info("running dhcp on interface") link := &ip.Link{ Name: iface, } err := link.SetUp() if err != nil { logger.WithField("interface", iface).WithError(err).Error("Giving up on DHCP for this interface, couldn't bring up interface") // continue to try other interfaces continue } go c.dhcpRunV4(errorChannel, iface, hostname, logger) go c.dhcpRunV6(errorChannel, iface, hostname, logger) } // Wait for all goroutines to return (2 per interface). var finalErr error for i := 0; i < len(names)*2; i++ { err := <-errorChannel if err != nil { logger.WithError(err).Error("DHCP client failed") finalErr = fmt.Errorf("some DHCP clients failed (one or more)") } } return finalErr } func (c *cmdForknet) dhcpRunV4(errorChannel chan error, iface string, hostname string, logger *logrus.Logger) { // Try to get a lease. client, err := nclient4.New(iface) if err != nil { logger.WithError(err).Error("Giving up on DHCPv4, couldn't set up client") errorChannel <- err return } defer func() { _ = client.Close() }() lease, err := client.Request(context.Background(), dhcpv4.WithoutOption(dhcpv4.OptionIPAddressLeaseTime), dhcpv4.WithRequestedOptions( dhcpv4.OptionSubnetMask, // 1 dhcpv4.OptionRouter, // 3 dhcpv4.OptionDomainNameServer, // 6 dhcpv4.OptionDomainName, // 15 dhcpv4.OptionClasslessStaticRoute, // 121 (if present) dhcpv4.OptionIPAddressLeaseTime, // 51 dhcpv4.OptionRenewTimeValue, // 58 (T1) dhcpv4.OptionRebindingTimeValue, // 59 (T2) ), dhcpv4.WithOption(dhcpv4.OptHostName(hostname))) if err != nil { logger.WithError(err).WithField("hostname", hostname). Error("Giving up on DHCPv4, couldn't get a lease") errorChannel <- err return } // Parse the response. if lease.Offer == nil { logger.WithField("hostname", hostname). Error("Giving up on DHCPv4, couldn't get a lease after 5s") errorChannel <- err return } if lease.Offer.YourIPAddr == nil || lease.Offer.YourIPAddr.Equal(net.IPv4zero) || lease.Offer.SubnetMask() == nil || len(lease.Offer.Router()) != 1 { logger.Error("Giving up on DHCPv4, lease didn't contain required fields") errorChannel <- errors.New("Giving up on DHCPv4, lease didn't contain required fields") return } c.applyDNSMu.Lock() c.dhcpv4Leases[iface] = lease c.applyDNSMu.Unlock() err = c.dhcpApplyDNS(logger) if err != nil { logger.WithError(err).Error("Giving up on DHCPv4, error applying DNS") errorChannel <- err return } // Network configuration. addr := &ip.Addr{ DevName: iface, Address: &net.IPNet{ IP: lease.Offer.YourIPAddr, Mask: lease.Offer.SubnetMask(), }, Family: ip.FamilyV4, } err = addr.Add() if err != nil { logger.WithError(err).Error("Giving up on DHCPv4, couldn't add IP") errorChannel <- err return } if lease.Offer.Options.Has(dhcpv4.OptionClasslessStaticRoute) { for _, staticRoute := range lease.Offer.ClasslessStaticRoute() { route := &ip.Route{ DevName: iface, Route: staticRoute.Dest, Family: ip.FamilyV4, } if !staticRoute.Router.IsUnspecified() { route.Via = staticRoute.Router } err = route.Add() if err != nil { logger.WithError(err).Error("Giving up on DHCPv4, couldn't add classless static route") errorChannel <- err return } } } else { gws := lease.Offer.Router() if len(gws) == 0 || gws[0] == nil || gws[0].IsUnspecified() { logger.WithField("interface", iface).Info("No default gateway provided by DHCPv4; skipping default route") } else { err := c.installDefaultRouteV4(iface, gws[0]) if err != nil { logger.WithError(err).Error("Giving up on DHCPv4, couldn't add default route") errorChannel <- err return } } } // Handle DHCP renewal. for { // Calculate the renewal time. var t1 time.Duration if lease.ACK != nil { t1 = lease.ACK.IPAddressRenewalTime(0) } if t1 == 0 && lease.Offer != nil { t1 = lease.Offer.IPAddressRenewalTime(0) } if t1 == 0 && lease.Offer != nil { lt := lease.Offer.IPAddressLeaseTime(0) if lt > 0 { t1 = lt / 2 } } if t1 == 0 { t1 = time.Minute } j := time.Duration(int64(t1) / 20) // 5% if j > 0 { t1 += time.Duration(rand.Int63n(int64(2*j))) - j } // Wait until it's renewal time. time.Sleep(t1) // Renew the lease. newLease, err := client.Renew(context.Background(), lease, dhcpv4.WithRequestedOptions( dhcpv4.OptionIPAddressLeaseTime, // 51 dhcpv4.OptionRenewTimeValue, // 58 dhcpv4.OptionRebindingTimeValue, // 59 ), dhcpv4.WithOption(dhcpv4.OptHostName(hostname))) if err != nil { logger.WithError(err).Error("Giving up on DHCPv4, couldn't renew the lease") errorChannel <- err return } lease = newLease } } func (c *cmdForknet) dhcpRunV6(errorChannel chan error, iface string, hostname string, logger *logrus.Logger) { // Wait a couple of seconds for IPv6 link-local. time.Sleep(2 * time.Second) // Get a new DHCPv6 client. client, err := nclient6.New(iface) if err != nil { logger.WithError(err).Error("Giving up on DHCPv6, couldn't set up client") errorChannel <- err return } defer func() { _ = client.Close() }() // Try to get a lease. advertisement, err := client.Solicit(context.Background(), dhcpv6.WithFQDN(0, hostname)) if err != nil { logger.WithError(err).Error("Giving up on DHCPv6, error during DHCPv6 Solicit") errorChannel <- err return } // Check if we're dealing with stateless DHCPv6. if advertisement.Options.Status() == nil || advertisement.Options.Status().StatusCode == iana.StatusNoAddrsAvail { // Get interface details. i, err := net.InterfaceByName(iface) if err != nil { logger.WithError(err).Error("Giving up on DHCPv6, couldn't get interface details") errorChannel <- err return } // Try to get some information. infoRequest, err := dhcpv6.NewSolicit(i.HardwareAddr) if err != nil { logger.WithError(err).Error("Giving up on DHCPv6, error preparing DHCPv6 Info Request") errorChannel <- err return } infoRequest.MessageType = dhcpv6.MessageTypeInformationRequest infoRequest.Options.Del(dhcpv6.OptionIANA) reply, err := client.SendAndRead(context.Background(), nclient6.AllDHCPRelayAgentsAndServers, infoRequest, nclient6.IsMessageType(dhcpv6.MessageTypeReply)) if err != nil { logger.WithError(err).Error("Giving up on DHCPv6, error during DHCPv6 Info Request") errorChannel <- err return } // Update DNS. c.applyDNSMu.Lock() c.dhcpv6Leases[iface] = reply c.applyDNSMu.Unlock() err = c.dhcpApplyDNS(logger) if err != nil { logger.WithError(err).Error("Giving up on DHCPv6, error applying DNS") errorChannel <- err return } // We're dealing with stateless DHCPv6, no need to keep running. errorChannel <- nil return } reply, err := client.Request(context.Background(), advertisement, dhcpv6.WithFQDN(0, hostname)) if err != nil { logger.WithError(err).Error("Giving up on DHCPv6, error during DHCPv6 Request") errorChannel <- err return } c.applyDNSMu.Lock() c.dhcpv6Leases[iface] = reply c.applyDNSMu.Unlock() err = c.dhcpApplyDNS(logger) if err != nil { logger.WithError(err).Error("Giving up on DHCPv6, error applying DNS") errorChannel <- err return } // Network configuration. ia := reply.Options.OneIANA() if ia == nil { logger.Error("Giving up on DHCPv6 renewal, reply missing IANA") errorChannel <- errors.New("Giving up on DHCPv6 renewal, reply missing IANA") return } for _, iaaddr := range ia.Options.Addresses() { addr := &ip.Addr{ DevName: iface, Address: &net.IPNet{ IP: iaaddr.IPv6Addr, Mask: net.CIDRMask(64, 128), }, Family: ip.FamilyV6, } err = addr.Add() if err != nil { logger.WithError(err).Error("Giving up on DHCPv6, couldn't add IP") errorChannel <- err return } } // Handle DHCP Renewal. for { // Wait until it's renewal time. t1 := ia.T1 time.Sleep(t1) // Renew the lease. var optIAAddrs []dhcpv6.OptIAAddress for _, optIAAddr := range ia.Options.Addresses() { optIAAddrs = append(optIAAddrs, *optIAAddr) } modifiers := []dhcpv6.Modifier{ dhcpv6.WithClientID(reply.Options.ClientID()), dhcpv6.WithServerID(reply.Options.ServerID()), dhcpv6.WithIAID(ia.IaId), dhcpv6.WithIANA(optIAAddrs...), } renew, err := dhcpv6.NewMessage(modifiers...) if err != nil { logger.WithError(err).Error("Giving up on DHCv6, couldn't create renew message") errorChannel <- err return } renew.MessageType = dhcpv6.MessageTypeRenew newReply, err := dhcpv6.NewReplyFromMessage(renew, dhcpv6.WithFQDN(0, hostname)) if err != nil { logger.WithError(err).Error("Giving up on DHCPv6, couldn't renew the lease") errorChannel <- err return } reply = newReply } } func (c *cmdForknet) dhcpApplyDNS(logger *logrus.Logger) error { nameservers := map[string]struct{}{} searchLabels := []string{} domainNames := []string{} c.applyDNSMu.Lock() // IPv4 leases. for _, lease := range c.dhcpv4Leases { if lease == nil || lease.Offer == nil { continue } // Nameservers from DHCPv4. for _, ns := range lease.Offer.DNS() { nameservers[ns.String()] = struct{}{} } // Domain name (option 15). dn := lease.Offer.DomainName() if dn != "" { domainNames = append(domainNames, dn) } // Domain search list (option 119). ds := lease.Offer.DomainSearch() if ds != nil && len(ds.Labels) > 0 { searchLabels = append(searchLabels, ds.Labels...) } } // IPv6 leases. for _, reply := range c.dhcpv6Leases { if reply == nil { continue } // Nameservers from DHCPv6. for _, ns := range reply.Options.DNS() { nameservers[ns.String()] = struct{}{} } // Domain search list. dsl := reply.Options.DomainSearchList() if dsl != nil && len(dsl.Labels) > 0 { searchLabels = append(searchLabels, dsl.Labels...) } } c.applyDNSMu.Unlock() // Create resolv.conf. f, err := os.Create(filepath.Join(c.instNetworkPath, "resolv.conf")) if err != nil { logger.WithError(err).Error("Giving up on DHCP, couldn't create resolv.conf") return err } defer f.Close() // Write unique nameservers. for ns := range nameservers { _, err = fmt.Fprintf(f, "nameserver %s\n", ns) if err != nil { logger.WithError(err).Error("Giving up on DHCP, couldn't write resolv.conf") return err } } // Prefer a search list if present; otherwise write a single domain if available. if len(searchLabels) > 0 { seen := map[string]struct{}{} out := []string{} for _, s := range searchLabels { if s == "" { continue } _, ok := seen[s] if ok { continue } seen[s] = struct{}{} out = append(out, s) } if len(out) > 0 { _, err = fmt.Fprintf(f, "search %s\n", strings.Join(out, ", ")) if err != nil { logger.WithError(err).Error("Giving up on DHCP, couldn't write resolv.conf") return err } } } else if len(domainNames) > 0 { _, err = fmt.Fprintf(f, "domain %s\n", domainNames[0]) if err != nil { logger.WithError(err).Error("Giving up on DHCP, couldn't write resolv.conf") return err } } return nil } func (c *cmdForknet) installDefaultRouteV4(iface string, gw net.IP) error { // List all IPv4 routes in the main table; we'll filter default routes (Dst == nil) locally. routes, err := (&ip.Route{ Family: ip.FamilyV4, Table: "main", }).List() if err != nil { return err } var currentOwnerIf string var currentOwnerGw net.IP for _, r := range routes { // Only consider default routes (no destination) if r.Route != nil { continue } // r.DevName may be empty if not resolvable; skip such entries if r.DevName == "" { continue } if currentOwnerIf == "" || r.DevName < currentOwnerIf { currentOwnerIf = r.DevName currentOwnerGw = r.Via } } // Decide based on lexical order. switch { case currentOwnerIf == "": // No default route yet; we can install ours. case currentOwnerIf == iface: // We already own the default; if gateway unchanged, nothing to do. if currentOwnerGw != nil && gw != nil && currentOwnerGw.Equal(gw) { return nil } case iface < currentOwnerIf: // We win; replace the current default with ours. default: // We lose; keep existing default route. return nil } defRoute := &ip.Route{ DevName: iface, Route: nil, Via: gw, Family: ip.FamilyV4, Proto: "dhcp", } err = defRoute.Replace() if err != nil { return err } return nil } func (c *cmdForknet) runDetach(_ *cobra.Command, args []string) error { daemonPID := args[1] ifName := args[2] hostName := args[3] if daemonPID == "" { return errors.New("Daemon PID argument is required") } if ifName == "" { return errors.New("ifname argument is required") } if hostName == "" { return errors.New("hostname argument is required") } // Check if the interface exists. if !util.PathExists(fmt.Sprintf("/sys/class/net/%s", ifName)) { return fmt.Errorf("Couldn't restore host interface %q as container interface %q couldn't be found", hostName, ifName) } // Remove all IP addresses from interface before moving to parent netns. // This is to avoid any container address config leaking into host. addr := &ip.Addr{ DevName: ifName, } err := addr.Flush() if err != nil { return err } // Set interface down. link := &ip.Link{Name: ifName} err = link.SetDown() if err != nil { return err } // Rename it back to the host name. err = link.SetName(hostName) if err != nil { // If the interface has an altname that matches the target name, this can prevent rename of the // interface, so try removing it and trying the rename again if succeeds. _, altErr := subprocess.RunCommand("ip", "link", "property", "del", "dev", ifName, "altname", hostName) if altErr == nil { err = link.SetName(hostName) } return err } // Move it back to the host. phyPath := fmt.Sprintf("/sys/class/net/%s/phy80211/name", hostName) if util.PathExists(phyPath) { // Get the phy name. phyName, err := os.ReadFile(phyPath) if err != nil { return err } // Wifi cards (move the phy instead). _, err = subprocess.RunCommand("iw", "phy", strings.TrimSpace(string(phyName)), "set", "netns", daemonPID) if err != nil { return err } } else { // Regular NICs. link = &ip.Link{Name: hostName} err = link.SetNetns(daemonPID) if err != nil { return err } } return nil } incus-7.0.0/cmd/incusd/main_forkproxy.go000066400000000000000000000571151517523235500203060ustar00rootroot00000000000000package main /* #include "config.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "incus.h" #include "macro.h" #include "memory_utils.h" #include "process_utils.h" int whoami = -ESRCH; #define FORKPROXY_CHILD 1 #define FORKPROXY_PARENT 0 #define FORKPROXY_UDS_SOCK_FD_NUM 200 static int switch_uid_gid(uint32_t uid, uint32_t gid) { if (setgid((gid_t)gid) < 0) return -1; if (setuid((uid_t)uid) < 0) return -1; return 0; } static int lxc_epoll_wait_nointr(int epfd, struct epoll_event* events, int maxevents, int timeout) { int ret; again: ret = epoll_wait(epfd, events, maxevents, timeout); if (ret < 0 && errno == EINTR) goto again; return ret; } static void *async_wait_kludge(void *args) { pid_t pid = PTR_TO_INT(args); wait_for_pid(pid); return NULL; } #define LISTEN_NEEDS_MNTNS 1U #define CONNECT_NEEDS_MNTNS 2U void forkproxy(void) { unsigned int needs_mntns = 0; int connect_pid, connect_pidfd, listen_pid, listen_pidfd; size_t unix_prefix_len = sizeof("unix:") - 1; ssize_t ret; pid_t pid; char *connect_addr, *cur, *listen_addr; int sk_fds[2] = {-EBADF, -EBADF}; // Get the pid cur = advance_arg(false); if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) _exit(EXIT_FAILURE); listen_pid = atoi(advance_arg(true)); listen_pidfd = atoi(advance_arg(true)); listen_addr = advance_arg(true); connect_pid = atoi(advance_arg(true)); connect_pidfd = atoi(advance_arg(true)); connect_addr = advance_arg(true); if (strncmp(listen_addr, "udp:", sizeof("udp:") - 1) == 0 && strncmp(connect_addr, "udp:", sizeof("udp:") - 1) != 0) { fprintf(stderr, "Error: Proxying from udp to non-udp protocol is not supported\n"); _exit(EXIT_FAILURE); } // We only need to attach to the mount namespace for // non-abstract unix sockets. if ((strncmp(listen_addr, "unix:", unix_prefix_len) == 0) && (listen_addr[unix_prefix_len] != '@')) needs_mntns |= LISTEN_NEEDS_MNTNS; if ((strncmp(connect_addr, "unix:", unix_prefix_len) == 0) && (connect_addr[unix_prefix_len] != '@')) needs_mntns |= CONNECT_NEEDS_MNTNS; ret = socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, sk_fds); if (ret < 0) { fprintf(stderr, "%s - Failed to create anonymous unix socket pair\n", strerror(errno)); _exit(EXIT_FAILURE); } pid = fork(); if (pid < 0) { fprintf(stderr, "%s - Failed to create new process\n", strerror(errno)); _exit(EXIT_FAILURE); } if (pid == 0) { int listen_nsfd; int setns_flags; whoami = FORKPROXY_CHILD; ret = close(sk_fds[0]); if (ret < 0) fprintf(stderr, "%s - Failed to close fd %d\n", strerror(errno), sk_fds[0]); listen_nsfd = pidfd_nsfd(listen_pidfd, listen_pid); if (listen_nsfd < 0) { fprintf(stderr, "Error: %m - Failed to safely open namespace file descriptor based on pidfd %d\n", listen_pidfd); _exit(EXIT_FAILURE); } // Attach to the namespaces of the listener setns_flags = CLONE_NEWNET; if (in_same_namespace(getpid(), listen_nsfd, "user") > 0) setns_flags |= CLONE_NEWUSER; if (needs_mntns & LISTEN_NEEDS_MNTNS) setns_flags |= CLONE_NEWNS; if (!change_namespaces(listen_pidfd, listen_nsfd, setns_flags)) { fprintf(stderr, "Error: %m - Failed setns to listener namespaces\n"); _exit(EXIT_FAILURE); } // Complete switch to the user namespace of the connector if (setns_flags & CLONE_NEWUSER) finalize_userns(); close_prot_errno_disarm(listen_nsfd); close_prot_errno_disarm(listen_pidfd); ret = dup3(sk_fds[1], FORKPROXY_UDS_SOCK_FD_NUM, O_CLOEXEC); if (ret < 0) { fprintf(stderr, "%s - Failed to duplicate fd %d to fd 200\n", strerror(errno), sk_fds[1]); _exit(EXIT_FAILURE); } ret = close(sk_fds[1]); if (ret < 0) fprintf(stderr, "%s - Failed to close fd %d\n", strerror(errno), sk_fds[1]); } else { pthread_t thread; int connect_nsfd; int setns_flags; whoami = FORKPROXY_PARENT; ret = close(sk_fds[1]); if (ret < 0) fprintf(stderr, "%s - Failed to close fd %d\n", strerror(errno), sk_fds[1]); connect_nsfd = pidfd_nsfd(connect_pidfd, connect_pid); if (connect_nsfd < 0) { fprintf(stderr, "Error: %m - Failed to safely open namespace file descriptor based on pidfd %d\n", connect_pidfd); _exit(EXIT_FAILURE); } // Attach to the namespaces of the connector setns_flags = CLONE_NEWNET; if (in_same_namespace(getpid(), connect_nsfd, "user") > 0) setns_flags |= CLONE_NEWUSER; if (needs_mntns & CONNECT_NEEDS_MNTNS) setns_flags |= CLONE_NEWNS; if (!change_namespaces(connect_pidfd, connect_nsfd, setns_flags)) { fprintf(stderr, "Error: %m - Failed setns to connector namespaces\n"); _exit(EXIT_FAILURE); } // Complete switch to the user namespace of the connector if (setns_flags & CLONE_NEWUSER) finalize_userns(); close_prot_errno_disarm(connect_nsfd); close_prot_errno_disarm(connect_pidfd); ret = dup3(sk_fds[0], FORKPROXY_UDS_SOCK_FD_NUM, O_CLOEXEC); if (ret < 0) { fprintf(stderr, "%s - Failed to duplicate fd %d to fd 200\n", strerror(errno), sk_fds[1]); _exit(EXIT_FAILURE); } ret = close(sk_fds[0]); if (ret < 0) fprintf(stderr, "%s - Failed to close fd %d\n", strerror(errno), sk_fds[0]); // Usually we should wait for the child process somewhere here. // But we cannot really do this. The listener file descriptors // are retrieved in the go runtime but at that point we have // already double-fork()ed to daemonize ourselves and so we // can't wait on the child anymore after we received the // listener fds. On the other hand, if we wait on the child // here we wait on the child before the receive. However, if we // do this then we can end up in a situation where the socket // send buffer is full and we need to retrieve some file // descriptors first before we can go on sending more. But this // won't be possible because we're waiting before the call to // receive the file descriptor in the go runtime. Luckily, we // can just rely on init doing it's job and reaping the zombie // process. So, technically unsatisfying but pragmatically // correct. // Create detached waiting thread after all namespace // interactions have concluded since some of them require // single-threadedness. if (pthread_create(&thread, NULL, async_wait_kludge, INT_TO_PTR(pid)) || pthread_detach(thread)) { fprintf(stderr, "%m - Failed to create detached thread\n"); _exit(EXIT_FAILURE); } } } */ import "C" import ( "errors" "fmt" "io" "io/fs" "net" "os" "os/signal" "strconv" "strings" "sync" "time" "unsafe" "github.com/spf13/cobra" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/netutils" "github.com/lxc/incus/v7/internal/server/daemon" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/network" _ "github.com/lxc/incus/v7/shared/cgo" // Used by cgo ) const forkproxyUDSSockFDNum int = C.FORKPROXY_UDS_SOCK_FD_NUM type cmdForkproxy struct { global *cmdGlobal } // UDP session tracking (map "client tuple" to udp session) var ( udpSessions = map[string]*udpSession{} udpSessionsLock sync.Mutex ) type udpSession struct { client net.Addr target net.Conn timer *time.Timer timerLock sync.Mutex } func (c *cmdForkproxy) command() *cobra.Command { // Main subcommand cmd := &cobra.Command{} cmd.Use = "forkproxy " cmd.Short = "Setup network connection proxying" cmd.Long = `Description: Setup network connection proxying This internal command will spawn a new proxy process for a particular container, connecting one side to the host and the other to the container. ` cmd.Args = cobra.ExactArgs(12) cmd.RunE = c.run cmd.Hidden = true return cmd } func rearmUDPFd(epFd C.int, connFd C.int) { var ev C.struct_epoll_event ev.events = C.EPOLLIN | C.EPOLLONESHOT *(*C.int)(unsafe.Pointer(uintptr(unsafe.Pointer(&ev)) + unsafe.Sizeof(ev.events))) = connFd ret := C.epoll_ctl(epFd, C.EPOLL_CTL_MOD, connFd, &ev) if ret < 0 { fmt.Println("Error: Failed to add listener fd to epoll instance") } } func listenerInstance(epFd C.int, lAddr *deviceConfig.ProxyAddress, cAddr *deviceConfig.ProxyAddress, connFd C.int, lStruct *lStruct, proxy bool) error { // Single or multiple port -> single port connectAddr := cAddr.Address if cAddr.ConnType != "unix" { connectPort := cAddr.Ports[0] if lAddr.ConnType != "unix" && cAddr.ConnType != "unix" && len(cAddr.Ports) > 1 { // multiple port -> multiple port connectPort = cAddr.Ports[(*lStruct).lAddrIndex] } connectAddr = net.JoinHostPort(cAddr.Address, fmt.Sprintf("%d", connectPort)) } if lAddr.ConnType == "udp" { // This only handles udp <-> udp. The C constructor will have verified this before go func() { srcConn, err := net.FileConn((*lStruct).f) if err != nil { fmt.Printf("Warning: Failed to re-assemble listener: %v\n", err) rearmUDPFd(epFd, connFd) return } dstConn, err := net.Dial(cAddr.ConnType, connectAddr) if err != nil { fmt.Printf("Warning: Failed to connect to target: %v\n", err) rearmUDPFd(epFd, connFd) return } genericRelay(srcConn, dstConn) rearmUDPFd(epFd, connFd) }() return nil } // Accept a new client listener := (*lStruct).lConn srcConn, err := (*listener).Accept() if err != nil { fmt.Printf("Warning: Failed to accept new connection: %v\n", err) return err } dstConn, err := net.Dial(cAddr.ConnType, connectAddr) if err != nil { _ = srcConn.Close() fmt.Printf("Warning: Failed to connect to target: %v\n", err) return err } if proxy && cAddr.ConnType == "tcp" { if lAddr.ConnType == "unix" { _, _ = dstConn.Write([]byte("PROXY UNKNOWN\r\n")) } else { cHost, cPort, err := net.SplitHostPort(srcConn.RemoteAddr().String()) if err != nil { return err } dHost, dPort, err := net.SplitHostPort(srcConn.LocalAddr().String()) if err != nil { return err } proto := srcConn.LocalAddr().Network() proto = strings.ToUpper(proto) if strings.Contains(cHost, ":") { proto = fmt.Sprintf("%s6", proto) } else { proto = fmt.Sprintf("%s4", proto) } _, _ = dstConn.Write([]byte(fmt.Sprintf("PROXY %s %s %s %s %s\r\n", proto, cHost, dHost, cPort, dPort))) } } if cAddr.ConnType == "unix" && lAddr.ConnType == "unix" { // Handle OOB if both src and dst are using unix sockets go unixRelay(srcConn, dstConn) } else { go genericRelay(srcConn, dstConn) } return nil } type lStruct struct { f *os.File lConn *net.Listener lAddrIndex int } func (c *cmdForkproxy) run(cmd *cobra.Command, args []string) error { // Only root should run this if os.Geteuid() != 0 { return errors.New("This must be run as root") } // Quick checks. if len(args) != 12 { _ = cmd.Help() if len(args) == 0 { return nil } return errors.New("Missing required arguments") } // Check where we are in initialization if C.whoami != C.FORKPROXY_PARENT && C.whoami != C.FORKPROXY_CHILD { return errors.New("Failed to call forkproxy constructor") } listenAddr := args[2] lAddr, err := network.ProxyParseAddr(listenAddr) if err != nil { return err } connectAddr := args[5] cAddr, err := network.ProxyParseAddr(connectAddr) if err != nil { return err } if (lAddr.ConnType == "udp" || lAddr.ConnType == "tcp") && cAddr.ConnType == "udp" || cAddr.ConnType == "tcp" { err := errors.New("Invalid port range") if len(lAddr.Ports) > 1 && len(cAddr.Ports) > 1 && (len(cAddr.Ports) != len(lAddr.Ports)) { fmt.Println(err) return err } else if len(lAddr.Ports) == 1 && len(cAddr.Ports) > 1 { fmt.Println(err) return err } } if C.whoami == C.FORKPROXY_CHILD { defer func() { _ = unix.Close(forkproxyUDSSockFDNum) }() if lAddr.ConnType == "unix" && !lAddr.Abstract { err := os.Remove(lAddr.Address) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } } var listenAddresses []string if lAddr.ConnType == "unix" { listenAddresses = []string{lAddr.Address} } else { listenPortCount := len(lAddr.Ports) listenAddresses = make([]string, 0, listenPortCount) for i := 0; i < listenPortCount; i++ { listenAddresses = append(listenAddresses, net.JoinHostPort(lAddr.Address, fmt.Sprintf("%d", lAddr.Ports[i]))) } } for _, listenAddress := range listenAddresses { file, err := getListenerFile(lAddr.ConnType, listenAddress) if err != nil { return err } sAgain: err = netutils.AbstractUnixSendFd(forkproxyUDSSockFDNum, int(file.Fd())) if err != nil { errno, ok := linux.GetErrno(err) if ok && (errors.Is(errno, unix.EAGAIN)) { goto sAgain } break } _ = file.Close() } if lAddr.ConnType == "unix" && !lAddr.Abstract { var err error listenAddrGID := -1 if args[6] != "" { listenAddrGID, err = strconv.Atoi(args[6]) if err != nil { return err } } listenAddrUID := -1 if args[7] != "" { listenAddrUID, err = strconv.Atoi(args[7]) if err != nil { return err } } if listenAddrGID != -1 || listenAddrUID != -1 { err = os.Chown(lAddr.Address, listenAddrUID, listenAddrGID) if err != nil { return err } } var listenAddrMode os.FileMode if args[8] != "" { tmp, err := strconv.ParseUint(args[8], 8, 0) if err != nil { return err } listenAddrMode = os.FileMode(tmp) err = os.Chmod(lAddr.Address, listenAddrMode) if err != nil { return err } } } return err } addrRecvCount := 1 if lAddr.ConnType != "unix" { addrRecvCount = len(lAddr.Ports) } files := []*os.File{} for i := 0; i < addrRecvCount; i++ { rAgain: f, err := netutils.AbstractUnixReceiveFd(forkproxyUDSSockFDNum, netutils.UnixFdsAcceptExact) if err != nil { errno, ok := linux.GetErrno(err) if ok && (errors.Is(errno, unix.EAGAIN)) { goto rAgain } fmt.Printf("Error: Failed to receive fd from listener process: %v\n", err) _ = unix.Close(forkproxyUDSSockFDNum) return err } if f == nil { fmt.Println("Error: Failed to receive fd from listener process") _ = unix.Close(forkproxyUDSSockFDNum) return err } files = append(files, f) } _ = unix.Close(forkproxyUDSSockFDNum) var listenerMap map[int]*lStruct isUDPListener := lAddr.ConnType == "udp" listenerMap = make(map[int]*lStruct, addrRecvCount) if isUDPListener { for i, f := range files { listenerMap[int(f.Fd())] = &lStruct{ f: f, lAddrIndex: i, } } } else { for i, f := range files { listener, err := net.FileListener(f) if err != nil { fmt.Printf("Error: Failed to re-assemble listener: %v\n", err) return err } listenerMap[int(f.Fd())] = &lStruct{ lConn: &listener, lAddrIndex: i, } } } // Drop privilege if requested gid := uint64(0) if args[9] != "" { gid, err = strconv.ParseUint(args[9], 10, 32) if err != nil { return err } } uid := uint64(0) if args[10] != "" { uid, err = strconv.ParseUint(args[10], 10, 32) if err != nil { return err } } if uid != 0 || gid != 0 { ret := C.switch_uid_gid(C.uint32_t(uid), C.uint32_t(gid)) if ret < 0 { return fmt.Errorf("Failed to switch to uid %d and gid %d", uid, gid) } } // Handle SIGTERM which is sent when the proxy is to be removed sigs := make(chan os.Signal, 1) signal.Notify(sigs, unix.SIGTERM) if lAddr.ConnType == "unix" && !lAddr.Abstract { defer func() { _ = os.Remove(lAddr.Address) }() } epFd := C.epoll_create1(C.EPOLL_CLOEXEC) if epFd < 0 { return errors.New("Failed to create new epoll instance") } // Wait for SIGTERM and close the listener in order to exit the loop below self := unix.Getpid() go func() { <-sigs for _, f := range files { C.epoll_ctl(epFd, C.EPOLL_CTL_DEL, C.int(f.Fd()), nil) _ = f.Close() } _ = unix.Close(int(epFd)) if !isUDPListener { for _, l := range listenerMap { conn := (*l).lConn _ = (*conn).Close() } } _ = unix.Kill(self, unix.SIGKILL) }() defer func() { _ = unix.Kill(self, unix.SIGTERM) }() for _, f := range files { var ev C.struct_epoll_event ev.events = C.EPOLLIN if isUDPListener { ev.events |= C.EPOLLONESHOT } *(*C.int)(unsafe.Pointer(&ev.data)) = C.int(f.Fd()) ret := C.epoll_ctl(epFd, C.EPOLL_CTL_ADD, C.int(f.Fd()), &ev) if ret < 0 { return errors.New("Error: Failed to add listener fd to epoll instance") } } // This line is used by the daemon to check forkproxy has started OK. fmt.Println("Status: Started") for { var events [10]C.struct_epoll_event nfds := C.lxc_epoll_wait_nointr(epFd, &events[0], 10, -1) if nfds < 0 { fmt.Println("Error: Failed to wait on epoll instance") break } for i := C.int(0); i < nfds; i++ { curFd := *(*C.int)(unsafe.Pointer(&events[i].data)) srcConn, ok := listenerMap[int(curFd)] if !ok { continue } err := listenerInstance(epFd, lAddr, cAddr, curFd, srcConn, args[11] == "true") if err != nil { fmt.Printf("Warning: Failed to prepare new listener instance: %v\n", err) } } } fmt.Println("Status: Stopping proxy") return nil } func proxyCopy(dst net.Conn, src net.Conn) error { var err error // Attempt casting to UDP connections srcUdp, srcIsUdp := src.(*net.UDPConn) dstUdp, dstIsUdp := dst.(*net.UDPConn) buf := make([]byte, 32*1024) for { rAgain: var nr int var er error if srcIsUdp && srcUdp.RemoteAddr() == nil { var addr net.Addr nr, addr, er = srcUdp.ReadFrom(buf) if er == nil { // Look for existing UDP session udpSessionsLock.Lock() us, ok := udpSessions[addr.String()] udpSessionsLock.Unlock() if !ok { dc, err := net.Dial(dst.RemoteAddr().Network(), dst.RemoteAddr().String()) if err != nil { return err } us = &udpSession{ client: addr, target: dc, } udpSessionsLock.Lock() udpSessions[addr.String()] = us udpSessionsLock.Unlock() go func() { _ = proxyCopy(src, dc) }() us.timer = time.AfterFunc(30*time.Minute, func() { _ = us.target.Close() udpSessionsLock.Lock() delete(udpSessions, addr.String()) udpSessionsLock.Unlock() }) } us.timerLock.Lock() us.timer.Reset(30 * time.Minute) us.timerLock.Unlock() dst = us.target dstUdp, dstIsUdp = dst.(*net.UDPConn) } } else { nr, er = src.Read(buf) } // keep retrying on EAGAIN errno, ok := linux.GetErrno(er) if ok && (errors.Is(errno, unix.EAGAIN)) { goto rAgain } if nr > 0 { wAgain: var nw int var ew error if dstIsUdp && dstUdp.RemoteAddr() == nil { var us *udpSession udpSessionsLock.Lock() for _, v := range udpSessions { if v.target.LocalAddr() == src.LocalAddr() { us = v break } } udpSessionsLock.Unlock() if us == nil { return errors.New("Connection expired") } us.timerLock.Lock() us.timer.Reset(30 * time.Minute) us.timerLock.Unlock() nw, ew = dstUdp.WriteTo(buf[0:nr], us.client) } else { nw, ew = dst.Write(buf[0:nr]) } // keep retrying on EAGAIN errno, ok := linux.GetErrno(ew) if ok && (errors.Is(errno, unix.EAGAIN)) { goto wAgain } if ew != nil { err = ew break } if nr != nw { err = io.ErrShortWrite break } } if er != nil { if er != io.EOF { err = er } break } } return err } func genericRelay(dst net.Conn, src net.Conn) { relayer := func(src net.Conn, dst net.Conn, ch chan error) { ch <- proxyCopy(src, dst) close(ch) } chSend := make(chan error) chRecv := make(chan error) go relayer(src, dst, chRecv) _, isUDP := dst.(*net.UDPConn) if !isUDP { go relayer(dst, src, chSend) } select { case errSnd := <-chSend: if daemon.Debug && errSnd != nil { fmt.Printf("Warning: Error while sending data: %v\n", errSnd) } case errRcv := <-chRecv: if daemon.Debug && errRcv != nil { fmt.Printf("Warning: Error while reading data: %v\n", errRcv) } } _ = src.Close() _ = dst.Close() // Empty the channels if !isUDP { <-chSend } <-chRecv } func unixRelayer(src *net.UnixConn, dst *net.UnixConn, ch chan error) { dataBuf := make([]byte, 4096) oobBuf := make([]byte, 4096) for { // Read from the source readAgain: sData, sOob, _, _, err := src.ReadMsgUnix(dataBuf, oobBuf) if err != nil { errno, ok := linux.GetErrno(err) if ok && errors.Is(errno, unix.EAGAIN) { goto readAgain } ch <- err return } var fds []int if sOob > 0 { entries, err := unix.ParseSocketControlMessage(oobBuf[:sOob]) if err != nil { ch <- err return } for _, msg := range entries { fds, err = unix.ParseUnixRights(&msg) if err != nil { ch <- err return } } } // Send to the destination writeAgain: tData, tOob, err := dst.WriteMsgUnix(dataBuf[:sData], oobBuf[:sOob], nil) if err != nil { errno, ok := linux.GetErrno(err) if ok && errors.Is(errno, unix.EAGAIN) { goto writeAgain } ch <- err return } if sData != tData || sOob != tOob { ch <- errors.New("Lost oob data during transfer") return } // Close those fds we received for _, fd := range fds { err := unix.Close(fd) if err != nil { ch <- err return } } } } func unixRelay(dst io.ReadWriteCloser, src io.ReadWriteCloser) { chSend := make(chan error) go unixRelayer(dst.(*net.UnixConn), src.(*net.UnixConn), chSend) chRecv := make(chan error) go unixRelayer(src.(*net.UnixConn), dst.(*net.UnixConn), chRecv) select { case errSnd := <-chSend: if daemon.Debug && errSnd != nil { fmt.Printf("Warning: Error while sending data: %v\n", errSnd) } case errRcv := <-chRecv: if daemon.Debug && errRcv != nil { fmt.Printf("Warning: Error while reading data: %v\n", errRcv) } } _ = src.Close() _ = dst.Close() // Empty the channels <-chSend <-chRecv } func tryListen(protocol string, addr string) (net.Listener, error) { var listener net.Listener var err error for i := 0; i < 10; i++ { listener, err = net.Listen(protocol, addr) if err == nil { break } time.Sleep(500 * time.Millisecond) } if err != nil { return nil, err } return listener, nil } func tryListenUDP(protocol string, addr string) (*os.File, error) { var UDPConn *net.UDPConn var err error udpAddr, err := net.ResolveUDPAddr(protocol, addr) if err != nil { return nil, err } for i := 0; i < 10; i++ { UDPConn, err = net.ListenUDP(protocol, udpAddr) if err == nil { file, err := UDPConn.File() _ = UDPConn.Close() return file, err } time.Sleep(500 * time.Millisecond) } if err != nil { return nil, err } if UDPConn == nil { return nil, errors.New("Failed to setup UDP listener") } file, err := UDPConn.File() _ = UDPConn.Close() return file, err } func getListenerFile(protocol string, addr string) (*os.File, error) { if protocol == "udp" { return tryListenUDP("udp", addr) } listener, err := tryListen(protocol, addr) if err != nil { return nil, fmt.Errorf("Failed to listen on %s: %w", addr, err) } var file *os.File switch l := listener.(type) { case *net.TCPListener: file, err = l.File() case *net.UnixListener: file, err = l.File() default: return nil, errors.New("Could not get listener file: invalid listener type") } if err != nil { return nil, fmt.Errorf("Failed to get file from listener: %w", err) } return file, nil } incus-7.0.0/cmd/incusd/main_forkproxy_test.go000066400000000000000000000064701517523235500213430ustar00rootroot00000000000000package main import ( "log" "testing" "github.com/stretchr/testify/require" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/network" ) func TestParseAddr(t *testing.T) { tests := []struct { name string address string expected *deviceConfig.ProxyAddress shouldFail bool }{ // Port testing { "Single port", "tcp:127.0.0.1:2000", &deviceConfig.ProxyAddress{ ConnType: "tcp", Address: "127.0.0.1", Ports: []uint64{2000}, Abstract: false, }, false, }, { "Multiple ports", "tcp:127.0.0.1:2000,2002", &deviceConfig.ProxyAddress{ ConnType: "tcp", Address: "127.0.0.1", Ports: []uint64{ 2000, 2002, }, Abstract: false, }, false, }, { "Port range", "tcp:127.0.0.1:2000-2002", &deviceConfig.ProxyAddress{ ConnType: "tcp", Address: "127.0.0.1", Ports: []uint64{ 2000, 2001, 2002, }, Abstract: false, }, false, }, { "Mixed ports and port ranges", "tcp:127.0.0.1:2000,2002,3000-3003,4000-4003", &deviceConfig.ProxyAddress{ ConnType: "tcp", Address: "127.0.0.1", Ports: []uint64{ 2000, 2002, 3000, 3001, 3002, 3003, 4000, 4001, 4002, 4003, }, Abstract: false, }, false, }, // connType testing { "UDP", "udp:127.0.0.1:2000", &deviceConfig.ProxyAddress{ ConnType: "udp", Address: "127.0.0.1", Ports: []uint64{2000}, Abstract: false, }, false, }, { "Unix socket", "unix:/foobar", &deviceConfig.ProxyAddress{ ConnType: "unix", Address: "/foobar", Abstract: false, }, false, }, { "Abstract unix socket", "unix:@/foobar", &deviceConfig.ProxyAddress{ ConnType: "unix", Address: "@/foobar", Abstract: true, }, false, }, { "Unknown connection type", "bla:blub", nil, true, }, // Address testing { "Valid IPv6 address (1)", "tcp:[fd39:2561:7238:91b5:0:0:0:0]:2000", &deviceConfig.ProxyAddress{ ConnType: "tcp", Address: "fd39:2561:7238:91b5:0:0:0:0", Ports: []uint64{2000}, Abstract: false, }, false, }, { "Valid IPv6 address (2)", "tcp:[fd39:2561:7238:91b5::0]:2000", &deviceConfig.ProxyAddress{ ConnType: "tcp", Address: "fd39:2561:7238:91b5::0", Ports: []uint64{2000}, Abstract: false, }, false, }, { "Valid IPv6 address (3)", "tcp:[::1]:2000", &deviceConfig.ProxyAddress{ ConnType: "tcp", Address: "::1", Ports: []uint64{2000}, Abstract: false, }, false, }, { "Valid IPv6 address (4)", "tcp:[::]:2000", &deviceConfig.ProxyAddress{ ConnType: "tcp", Address: "::", Ports: []uint64{2000}, Abstract: false, }, false, }, { "Invalid IPv6 address (1)", "tcp:fd39:2561:7238:91b5:0:0:0:0:2000", nil, true, }, { "Invalid IPv6 address (2)", "tcp:fd39:2561:7238:91b5::0:2000", nil, true, }, } for i, tt := range tests { log.Printf("Running test #%d: %s", i, tt.name) addr, err := network.ProxyParseAddr(tt.address) if tt.shouldFail { require.Error(t, err) require.Nil(t, addr) continue } require.NoError(t, err) require.Equal(t, tt.expected, addr) } } incus-7.0.0/cmd/incusd/main_forkstart.go000066400000000000000000000044751517523235500202630ustar00rootroot00000000000000package main import ( "errors" "fmt" "os" "path/filepath" liblxc "github.com/lxc/go-lxc" "github.com/spf13/cobra" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/shared/util" ) type cmdForkstart struct { global *cmdGlobal } func (c *cmdForkstart) command() *cobra.Command { // Main subcommand cmd := &cobra.Command{} cmd.Use = "forkstart " cmd.Short = "Start the container" cmd.Long = `Description: Start the container This internal command is used to start the container as a separate process. ` cmd.RunE = c.run cmd.Hidden = true return cmd } func (c *cmdForkstart) run(cmd *cobra.Command, args []string) error { // Quick checks. if len(args) != 4 { _ = cmd.Help() if len(args) == 0 { return nil } return errors.New("Missing required arguments") } // Only root should run this if os.Geteuid() != 0 { return errors.New("This must be run as root") } name := args[0] lxcpath := args[1] configPath := args[2] logDir := args[3] err := linux.CloseRange(uint32(os.Stderr.Fd())+1, ^uint32(0), linux.CLOSE_RANGE_CLOEXEC) if err != nil { return errors.New("Aborting attach to prevent leaking file descriptors into container") } d, err := liblxc.NewContainer(name, lxcpath) if err != nil { return fmt.Errorf("Error initializing container for start: %q", err) } err = d.LoadConfigFile(configPath) if err != nil { return fmt.Errorf("Error opening startup config file: %q", err) } /* due to https://github.com/golang/go/issues/13155 and the * CollectOutput call we make for the forkstart process, we need to * close our stdin/stdout/stderr here. Collecting some of the logs is * better than collecting no logs, though. */ _ = os.Stdin.Close() _ = os.Stderr.Close() _ = os.Stdout.Close() // Redirect stdout and stderr to a log file logPath := filepath.Join(logDir, "forkstart.log") if util.PathExists(logPath) { _ = os.Remove(logPath) } logFile, err := os.OpenFile(logPath, os.O_WRONLY|os.O_CREATE|os.O_SYNC, 0o644) if err == nil { _ = unix.Dup3(int(logFile.Fd()), 1, 0) _ = unix.Dup3(int(logFile.Fd()), 2, 0) } // Handle application containers. execCmd := d.ConfigItem("lxc.execute.cmd") if len(execCmd) > 0 && execCmd[0] != "" { return d.StartExecute(nil) } return d.Start() } incus-7.0.0/cmd/incusd/main_forksyscall.go000066400000000000000000000340761517523235500206000ustar00rootroot00000000000000package main /* #ifndef _GNU_SOURCE #define _GNU_SOURCE 1 #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../../shared/cgo/macro.h" #include "../../shared/cgo/memory_utils.h" #include "../../shared/cgo/mount_utils.h" #include "../../shared/cgo/syscall_numbers.h" #include "../../shared/cgo/syscall_wrappers.h" extern char* advance_arg(bool required); extern void attach_userns_fd(int ns_fd); extern int pidfd_nsfd(int pidfd, pid_t pid); extern int preserve_ns(pid_t pid, int ns_fd, const char *ns); extern bool change_namespaces(int pidfd, int nsfd, unsigned int flags); extern int mount_detach_idmap(const char *path, int fd_userns); static bool chdirchroot_in_mntns(int cwd_fd, int root_fd) { ssize_t len; char buf[PATH_MAX]; if (fchdir(cwd_fd)) return false; len = readlinkat(root_fd, "", buf, sizeof(buf)); if (len < 0 || len >= sizeof(buf)) return false; buf[len] = '\0'; if (chroot(buf)) return false; return true; } static bool acquire_basic_creds(pid_t pid, int pidfd, int ns_fd, int *rootfd, int *cwdfd) { __do_close int cwd_fd = -EBADF, root_fd = -EBADF; char buf[256]; snprintf(buf, sizeof(buf), "/proc/%d/root", pid); root_fd = open(buf, O_PATH | O_RDONLY | O_CLOEXEC | O_NOFOLLOW); if (root_fd < 0) return false; snprintf(buf, sizeof(buf), "/proc/%d/cwd", pid); cwd_fd = open(buf, O_PATH | O_RDONLY | O_CLOEXEC); if (cwd_fd < 0) return false; if (!change_namespaces(pidfd, ns_fd, CLONE_NEWNS)) return false; if (!chdirchroot_in_mntns(cwd_fd, root_fd)) return false; if (rootfd) *rootfd = move_fd(root_fd); if (cwdfd) *cwdfd = move_fd(cwd_fd); return true; } static bool reacquire_basic_creds(int pidfd, int ns_fd, int root_fd, int cwd_fd) { if (!change_namespaces(pidfd, ns_fd, CLONE_NEWNS)) return false; if (!chdirchroot_in_mntns(cwd_fd, root_fd)) return false; return true; } static bool acquire_final_creds(pid_t pid, uid_t uid, gid_t gid, uid_t fsuid, gid_t fsgid) { int ret; cap_t caps; caps = cap_get_pid(pid); if (!caps) { fprintf(stderr, "%d", ENOANO); return false; } ret = prctl(PR_SET_KEEPCAPS, 1); if (ret) { fprintf(stderr, "%d", ENOANO); return false; } ret = setegid(gid); if (ret) { fprintf(stderr, "%d", ENOANO); return false; } setfsgid(fsgid); if (setfsgid(-1) != fsgid) { fprintf(stderr, "%d", ENOANO); return false; } ret = seteuid(uid); if (ret) { fprintf(stderr, "%d", ENOANO); return false; } setfsuid(fsuid); if (setfsuid(-1) != fsuid) { fprintf(stderr, "%d", ENOANO); return false; } ret = cap_set_proc(caps); if (ret) { fprintf(stderr, "%d", ENOANO); return false; } return true; } // Expects command line to be in the form: // static void mknod_emulate(void) { __do_close int target_dir_fd = -EBADF, pidfd = -EBADF, ns_fd = -EBADF; char *target = NULL, *target_dir = NULL; int ret; char path[PATH_MAX]; mode_t mode; dev_t dev; pid_t pid; uid_t fsuid, uid; gid_t fsgid, gid; struct statfs sfs; pid = atoi(advance_arg(true)); pidfd = atoi(advance_arg(true)); ns_fd = pidfd_nsfd(pidfd, pid); if (ns_fd < 0) _exit(EXIT_FAILURE); target = advance_arg(true); mode = atoi(advance_arg(true)); dev = atoi(advance_arg(true)); uid = atoi(advance_arg(true)); gid = atoi(advance_arg(true)); fsuid = atoi(advance_arg(true)); fsgid = atoi(advance_arg(true)); if (!acquire_basic_creds(pid, pidfd, ns_fd, NULL, NULL)) { fprintf(stderr, "%d", ENOANO); _exit(EXIT_FAILURE); } snprintf(path, sizeof(path), "%s", target); target_dir = dirname(path); target_dir_fd = open(target_dir, O_PATH | O_RDONLY | O_CLOEXEC | O_DIRECTORY); if (target_dir_fd < 0) { fprintf(stderr, "%d", ENOANO); _exit(EXIT_FAILURE); } if (!acquire_final_creds(pid, uid, gid, fsuid, fsgid)) { fprintf(stderr, "%d", ENOANO); _exit(EXIT_FAILURE); } ret = fstatfs(target_dir_fd, &sfs); if (ret) { fprintf(stderr, "%d", ENOANO); _exit(EXIT_FAILURE); } if (sfs.f_flags & MS_NODEV) { fprintf(stderr, "%d", EPERM); _exit(EXIT_FAILURE); } // basename() can modify its argument so accessing target_host is // invalid from now on. ret = mknodat(target_dir_fd, target, mode, dev); if (ret) { if (errno == EPERM) fprintf(stderr, "%d", ENOMEDIUM); else fprintf(stderr, "%d", errno); _exit(EXIT_FAILURE); } } const static int ns_flags[] = { CLONE_NEWUSER, CLONE_NEWPID, CLONE_NEWUTS, CLONE_NEWIPC, CLONE_NEWNET, CLONE_NEWCGROUP }; static bool change_creds(int pidfd, int ns_fd, cap_t caps, uid_t nsuid, gid_t nsgid, uid_t nsfsuid, gid_t nsfsgid) { if (prctl(PR_SET_KEEPCAPS, 1)) return false; for (int i = 0; ARRAY_SIZE(ns_flags); i++) { if (!change_namespaces(pidfd, ns_fd, ns_flags[i])) return false; } if (setegid(nsgid)) return false; setfsgid(nsfsgid); if (seteuid(nsuid)) return false; setfsuid(nsfsuid); if (cap_set_proc(caps)) return false; return true; } static void setxattr_emulate(void) { __do_close int ns_fd = -EBADF, pidfd = -EBADF, target_fd = -EBADF; int flags = 0; char *name, *target; uid_t nsfsuid, nsuid; gid_t nsfsgid, nsgid; pid_t pid = 0; cap_t caps; cap_flag_value_t flag; int whiteout; void *data; size_t size; pid = atoi(advance_arg(true)); pidfd = atoi(advance_arg(true)); ns_fd = pidfd_nsfd(pidfd, pid); if (ns_fd < 0) _exit(EXIT_FAILURE); nsuid = atoi(advance_arg(true)); nsgid = atoi(advance_arg(true)); nsfsuid = atoi(advance_arg(true)); nsfsgid = atoi(advance_arg(true)); name = advance_arg(true); target = advance_arg(true); flags = atoi(advance_arg(true)); whiteout = atoi(advance_arg(true)); size = atoi(advance_arg(true)); data = advance_arg(true); if (!acquire_basic_creds(pid, pidfd, ns_fd, NULL, NULL)) { fprintf(stderr, "%d", ENOANO); _exit(EXIT_FAILURE); } target_fd = open(target, O_RDONLY | O_CLOEXEC); if (target_fd < 0) { fprintf(stderr, "%d", errno); _exit(EXIT_FAILURE); } caps = cap_get_pid(pid); if (!caps) { fprintf(stderr, "%d", ENOANO); _exit(EXIT_FAILURE); } if (whiteout == 1) { if (cap_get_flag(caps, CAP_SYS_ADMIN, CAP_EFFECTIVE, &flag) != 0) { fprintf(stderr, "%d", EPERM); _exit(EXIT_FAILURE); } if (flag == CAP_CLEAR) { fprintf(stderr, "%d", EPERM); _exit(EXIT_FAILURE); } } if (whiteout == 1) { if (fsetxattr(target_fd, "trusted.overlay.opaque", "y", 1, flags)) { fprintf(stderr, "%d", errno); _exit(EXIT_FAILURE); } } else { if (!change_creds(pidfd, ns_fd, caps, nsuid, nsgid, nsfsuid, nsfsgid)) { fprintf(stderr, "%d", EFAULT); _exit(EXIT_FAILURE); } if (fsetxattr(target_fd, name, data, size, flags)) { fprintf(stderr, "%d", errno); _exit(EXIT_FAILURE); } } } static void mount_emulate(void) { __do_close int fd_mntns = -EBADF, fd_userns = -EBADF, pidfd = -EBADF, ns_fd = -EBADF, root_fd = -EBADF, cwd_fd = -EBADF; char *source = NULL, *shift = NULL, *target = NULL, *fstype = NULL; bool use_fuse; uid_t nsuid = -1, uid = -1, nsfsuid = -1, fsuid = -1; gid_t nsgid = -1, gid = -1, nsfsgid = -1, fsgid = -1; int ret; pid_t pid = -1; unsigned long flags = 0; const void *data; pid = atoi(advance_arg(true)); pidfd = atoi(advance_arg(true)); ns_fd = pidfd_nsfd(pidfd, pid); if (ns_fd < 0) _exit(EXIT_FAILURE); use_fuse = (atoi(advance_arg(true)) == 1); if (!use_fuse) { source = advance_arg(true); target = advance_arg(true); fstype = advance_arg(true); flags = atoi(advance_arg(true)); shift = advance_arg(true); } uid = atoi(advance_arg(true)); gid = atoi(advance_arg(true)); fsuid = atoi(advance_arg(true)); fsgid = atoi(advance_arg(true)); if (!use_fuse) { nsuid = atoi(advance_arg(true)); nsgid = atoi(advance_arg(true)); nsfsuid = atoi(advance_arg(true)); nsfsgid = atoi(advance_arg(true)); data = advance_arg(false); } fd_userns = preserve_ns(-ESRCH, ns_fd, "user"); if (fd_userns < 0) _exit(EXIT_FAILURE); fd_mntns = preserve_ns(getpid(), ns_fd, "mnt"); if (fd_mntns < 0) _exit(EXIT_FAILURE); if (use_fuse) { attach_userns_fd(ns_fd); // Attach to pid namespace so that if we spawn a fuse daemon // it'll belong to the correct pid namespace and dies with the // container. change_namespaces(pidfd, ns_fd, CLONE_NEWPID); } if (!acquire_basic_creds(pid, pidfd, ns_fd, &root_fd, &cwd_fd)) _exit(EXIT_FAILURE); if (!acquire_final_creds(pid, uid, gid, fsuid, fsgid)) _exit(EXIT_FAILURE); if (use_fuse) { int status; pid_t pid_fuse; pid_fuse = fork(); if (pid_fuse < 0) _exit(EXIT_FAILURE); if (pid_fuse == 0) { const char *fuse_source, *fuse_target, *fuse_opts; fuse_source = advance_arg(true); fuse_target = advance_arg(true); fuse_opts = advance_arg(true); if (strcmp(fuse_opts, "") == 0) execlp("mount.fuse", "mount.fuse", fuse_source, fuse_target, (char *) NULL); else execlp("mount.fuse", "mount.fuse", fuse_source, fuse_target, "-o", fuse_opts, (char *) NULL); _exit(EXIT_FAILURE); } ret = waitpid(pid_fuse, &status, 0); if ((ret != pid_fuse) || !WIFEXITED(status) || WEXITSTATUS(status)) _exit(EXIT_FAILURE); } else if (strcmp(shift, "idmapped") == 0) { int fd_tree; int fs_fd = -EBADF; struct lxc_mount_attr attr = { .attr_set = MOUNT_ATTR_IDMAP, }; fs_fd = incus_fsopen(fstype, FSOPEN_CLOEXEC); if (fs_fd < 0) die("error: failed to create detached idmapped mount: fsopen"); ret = incus_fsconfig(fs_fd, FSCONFIG_SET_STRING, "source", source, 0); if (ret < 0) die("error: failed to create detached idmapped mount: fsconfig"); if (data && strcmp((const char *)data, "") != 0) { char *buf, *cur, *tok; buf = strdup((const char *)data); if (!buf) die("error: failed to allocate memory for mount data"); for (cur = buf; (tok = strsep(&cur, ",")); ) { char *val; if (*tok == '\0') continue; val = strchr(tok, '='); if (val) { *val = '\0'; val++; } if (val && *val != '\0') { ret = incus_fsconfig(fs_fd, FSCONFIG_SET_STRING, tok, val, 0); } else { ret = incus_fsconfig(fs_fd, FSCONFIG_SET_FLAG, tok, NULL, 0); } if (ret < 0) die("error: failed to create detached idmapped mount: fsconfig"); } free(buf); } ret = incus_fsconfig(fs_fd, FSCONFIG_CMD_CREATE, NULL, NULL, 0); if (ret < 0) die("error: failed to create detached idmapped mount: fsconfig"); fd_tree = incus_fsmount(fs_fd, FSMOUNT_CLOEXEC, flags); if (fd_tree < 0) die("error: failed to create detached idmapped mount: fsmount"); attr.userns_fd = fd_userns; ret = incus_mount_setattr(fd_tree, "", AT_EMPTY_PATH, &attr, sizeof(attr)); if (ret < 0) die("error: failed to create detached idmapped mount"); ret = setns(fd_mntns, CLONE_NEWNS); if (ret) die("error: failed to attach to old mount namespace"); attach_userns_fd(ns_fd); if (!change_namespaces(pidfd, ns_fd, CLONE_NEWUSER)) die("error: failed to change to target user namespace"); if (!reacquire_basic_creds(pidfd, ns_fd, root_fd, cwd_fd)) die("error: failed to acquire basic creds"); if (!acquire_final_creds(pid, nsuid, nsgid, nsfsuid, nsfsgid)) die("error: failed to acquire final creds"); ret = incus_move_mount(fd_tree, "", -EBADF, target, MOVE_MOUNT_F_EMPTY_PATH); if (ret) die("error: failed to attach detached mount"); } else { if (mount(source, target, fstype, flags, data) < 0) _exit(EXIT_FAILURE); } } static bool incus_cap_is_set(cap_t caps, cap_value_t cap, cap_flag_t flag) { int ret; cap_flag_value_t flagval; ret = cap_get_flag(caps, cap, flag, &flagval); if (ret < 0) return false; return flagval == CAP_SET; } static void sched_setscheduler_emulate(void) { __do_close int pidfd = -EBADF, ns_fd = -EBADF; pid_t pid_caller = -ESRCH, pid_target = -ESRCH; int policy = -1, sched_priority = -1; int switch_pidns = 0; struct sched_param param = {}; cap_t caps; pid_caller = atoi(advance_arg(true)); pidfd = atoi(advance_arg(true)); ns_fd = pidfd_nsfd(pidfd, pid_caller); if (ns_fd < 0) _exit(EXIT_FAILURE); switch_pidns = atoi(advance_arg(true)); pid_target = atoi(advance_arg(true)); if (pid_target < 0) _exit(EXIT_FAILURE); policy = atoi(advance_arg(true)); if (policy < 0) _exit(EXIT_FAILURE); sched_priority = atoi(advance_arg(true)); if (sched_priority < 0) _exit(EXIT_FAILURE); param.sched_priority = sched_priority; caps = cap_get_pid(pid_caller); if (!caps) _exit(EXIT_FAILURE); if (!incus_cap_is_set(caps, CAP_SYS_NICE, CAP_EFFECTIVE)) _exit(EXIT_FAILURE); if (switch_pidns && !change_namespaces(pidfd, ns_fd, CLONE_NEWPID)) _exit(EXIT_FAILURE); if (sched_setscheduler(pid_target, policy, ¶m)) _exit(EXIT_FAILURE); } void forksyscall(void) { char *syscall = NULL; // Check that we're root if (geteuid() != 0) _exit(EXIT_FAILURE); // Get the subcommand syscall = advance_arg(false); if (syscall == NULL || (strcmp(syscall, "--help") == 0 || strcmp(syscall, "--version") == 0 || strcmp(syscall, "-h") == 0)) _exit(EXIT_SUCCESS); if (strcmp(syscall, "mknod") == 0) mknod_emulate(); else if (strcmp(syscall, "sched_setscheduler") == 0) sched_setscheduler_emulate(); else if (strcmp(syscall, "setxattr") == 0) setxattr_emulate(); else if (strcmp(syscall, "mount") == 0) mount_emulate(); else _exit(EXIT_FAILURE); _exit(EXIT_SUCCESS); } */ import "C" import ( "errors" "github.com/spf13/cobra" // Used by cgo _ "github.com/lxc/incus/v7/shared/cgo" ) type cmdForksyscall struct { global *cmdGlobal } func (c *cmdForksyscall) command() *cobra.Command { // Main subcommand cmd := &cobra.Command{} cmd.Use = "forksyscall [...]" cmd.Short = "Perform syscall operations" cmd.Long = `Description: Perform syscall operations This set of internal commands is used for all seccomp-based container syscall operations. ` cmd.RunE = c.run cmd.Hidden = true return cmd } func (c *cmdForksyscall) run(_ *cobra.Command, _ []string) error { return errors.New("This command should have been intercepted in cgo") } incus-7.0.0/cmd/incusd/main_forkuevent.go000066400000000000000000000126601517523235500204270ustar00rootroot00000000000000package main /* #ifndef _GNU_SOURCE #define _GNU_SOURCE 1 #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../../internal/netutils/network.c" #include "../../shared/cgo/memory_utils.h" #ifndef UEVENT_SEND #define UEVENT_SEND 16 #endif extern char *advance_arg(bool required); extern void attach_userns_fd(int ns_fd); extern int pidfd_nsfd(int pidfd, pid_t pid); extern bool change_namespaces(int pidfd, int nsfd, unsigned int flags); struct nlmsg { struct nlmsghdr *nlmsghdr; ssize_t cap; }; static struct nlmsg *nlmsg_alloc(size_t size) { __do_free struct nlmsg *nlmsg = NULL; size_t len = NLMSG_HDRLEN + NLMSG_ALIGN(size); nlmsg = (struct nlmsg *)malloc(sizeof(struct nlmsg)); if (!nlmsg) return NULL; nlmsg->nlmsghdr = (struct nlmsghdr *)malloc(len); if (!nlmsg->nlmsghdr) return NULL; memset(nlmsg->nlmsghdr, 0, len); nlmsg->cap = len; nlmsg->nlmsghdr->nlmsg_len = NLMSG_HDRLEN; return move_ptr(nlmsg); } static void *nlmsg_reserve_unaligned(struct nlmsg *nlmsg, size_t len) { char *buf; size_t nlmsg_len = nlmsg->nlmsghdr->nlmsg_len; size_t tlen = len; if ((ssize_t)(nlmsg_len + tlen) > nlmsg->cap) return NULL; buf = ((char *)(nlmsg->nlmsghdr)) + nlmsg_len; nlmsg->nlmsghdr->nlmsg_len += tlen; if (tlen > len) memset(buf + len, 0, tlen - len); return buf; } int can_inject_uevent(const char *uevent, size_t len) { __do_close int sock_fd = -EBADF; __do_free struct nlmsg *nlmsg = NULL; int ret; char *umsg = NULL; sock_fd = netlink_open(NETLINK_KOBJECT_UEVENT); if (sock_fd < 0) { return -1; } nlmsg = nlmsg_alloc(len); if (!nlmsg) return -1; nlmsg->nlmsghdr->nlmsg_flags = NLM_F_REQUEST; nlmsg->nlmsghdr->nlmsg_type = UEVENT_SEND; nlmsg->nlmsghdr->nlmsg_pid = 0; umsg = nlmsg_reserve_unaligned(nlmsg, len); if (!umsg) return -1; memcpy(umsg, uevent, len); ret = __netlink_send(sock_fd, nlmsg->nlmsghdr); if (ret < 0) return -1; return 0; } static int inject_uevent(const char *uevent, size_t len) { __do_close int sock_fd = -EBADF; __do_free struct nlmsg *nlmsg = NULL; int ret; char *umsg = NULL; sock_fd = netlink_open(NETLINK_KOBJECT_UEVENT); if (sock_fd < 0) return -1; nlmsg = nlmsg_alloc(len); if (!nlmsg) return -1; nlmsg->nlmsghdr->nlmsg_flags = NLM_F_ACK | NLM_F_REQUEST; nlmsg->nlmsghdr->nlmsg_type = UEVENT_SEND; nlmsg->nlmsghdr->nlmsg_pid = 0; umsg = nlmsg_reserve_unaligned(nlmsg, len); if (!umsg) return -1; memcpy(umsg, uevent, len); ret = netlink_transaction(sock_fd, nlmsg->nlmsghdr, nlmsg->nlmsghdr); if (ret < 0) return -1; return 0; } void forkuevent(void) { char *uevent = NULL; char *cur = NULL; pid_t pid = 0; size_t len = 0; int ns_fd = -EBADF, pidfd = -EBADF; cur = advance_arg(false); if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) { fprintf(stderr, "Error: Missing PID\n"); _exit(1); } // skip "--" advance_arg(false); // Get the pid cur = advance_arg(false); if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) { fprintf(stderr, "Error: Missing PID\n"); _exit(1); } pid = atoi(cur); pidfd = atoi(advance_arg(true)); ns_fd = pidfd_nsfd(pidfd, pid); if (ns_fd < 0) _exit(1); // Get the size cur = advance_arg(false); if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) { fprintf(stderr, "Error: Missing uevent length\n"); _exit(1); } len = atoi(cur); // Get the uevent cur = advance_arg(false); if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) { fprintf(stderr, "Error: Missing uevent\n"); _exit(1); } uevent = cur; // Check that we're root if (geteuid() != 0) { fprintf(stderr, "Error: forkuevent requires root privileges\n"); _exit(1); } attach_userns_fd(ns_fd); if (!change_namespaces(pidfd, ns_fd, CLONE_NEWNET)) { fprintf(stderr, "Failed to setns to container network namespace: %s\n", strerror(errno)); _exit(1); } if (inject_uevent(uevent, len) < 0) { fprintf(stderr, "Failed to inject uevent\n"); _exit(1); } _exit(0); } */ import "C" import ( "errors" "github.com/spf13/cobra" ) type cmdForkuevent struct { global *cmdGlobal } func (c *cmdForkuevent) command() *cobra.Command { // Main subcommand cmd := &cobra.Command{} cmd.Use = "forkuevent" cmd.Short = "Inject uevents into container's network namespace" cmd.Long = `Description: Inject uevent into a container's network namespace This internal command is used to inject uevents into unprivileged container's network namespaces. ` cmd.Hidden = true cmdInject := &cobra.Command{} cmdInject.Use = "inject ..." cmdInject.Args = cobra.MinimumNArgs(4) cmdInject.RunE = c.run cmd.AddCommand(cmdInject) // Workaround for subcommand usage errors. See: https://github.com/spf13/cobra/issues/706 cmd.Args = cobra.NoArgs cmd.Run = func(cmd *cobra.Command, args []string) { _ = cmd.Usage() } return cmd } func (c *cmdForkuevent) run(_ *cobra.Command, _ []string) error { return errors.New("This command should have been intercepted in cgo") } incus-7.0.0/cmd/incusd/main_forkzfs.go000066400000000000000000000036201517523235500177170ustar00rootroot00000000000000package main import ( "bufio" "errors" "os" "os/exec" "path/filepath" "strings" "github.com/spf13/cobra" "golang.org/x/sys/unix" internalUtil "github.com/lxc/incus/v7/internal/util" ) type cmdForkZFS struct { global *cmdGlobal } func (c *cmdForkZFS) command() *cobra.Command { // Main subcommand cmd := &cobra.Command{} cmd.Use = "forkzfs [...]" cmd.Short = "Run ZFS inside a cleaned up mount namespace" cmd.Long = `Description: Run ZFS inside a cleaned up mount namespace This internal command is used to run ZFS in some specific cases. ` cmd.RunE = c.run cmd.Hidden = true return cmd } func (c *cmdForkZFS) run(cmd *cobra.Command, args []string) error { // Quick checks. if len(args) < 1 { _ = cmd.Help() if len(args) == 0 { return nil } return errors.New("Missing required arguments") } // Only root should run this if os.Geteuid() != 0 { return errors.New("This must be run as root") } // Mark mount tree as private err := unix.Mount("none", "/", "", unix.MS_REC|unix.MS_PRIVATE, "") if err != nil { return err } // Expand the mount path absPath, err := filepath.Abs(internalUtil.VarPath()) if err != nil { return err } expPath, err := filepath.EvalSymlinks(absPath) if err != nil { expPath = absPath } // Find the source mount of the path file, err := os.Open("/proc/self/mountinfo") if err != nil { return err } defer func() { _ = file.Close() }() // Unmount all mounts under the main directory scanner := bufio.NewScanner(file) for scanner.Scan() { line := scanner.Text() rows := strings.Fields(line) if !strings.HasPrefix(rows[4], expPath) { continue } _ = unix.Unmount(rows[4], unix.MNT_DETACH) } // Run the ZFS command command := exec.Command("zfs", args...) command.Stdin = os.Stdin command.Stdout = os.Stdout command.Stderr = os.Stderr err = command.Run() if err != nil { return err } return nil } incus-7.0.0/cmd/incusd/main_manpage.go000066400000000000000000000017261517523235500176500ustar00rootroot00000000000000package main import ( "errors" "github.com/spf13/cobra" "github.com/spf13/cobra/doc" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdManpage struct { global *cmdGlobal } func (c *cmdManpage) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "manpage " cmd.Short = "Generate manpages for all commands" cmd.Long = cli.FormatSection("Description:", `Generate manpages for all commands`) cmd.Hidden = true cmd.RunE = c.run return cmd } func (c *cmdManpage) run(cmd *cobra.Command, args []string) error { // Quick checks. if len(args) != 1 { _ = cmd.Help() if len(args) == 0 { return nil } return errors.New("Missing required arguments") } // Generate the manpages header := &doc.GenManHeader{ Title: "Incus - Daemon", Section: "1", } opts := doc.GenManTreeOptions{ Header: header, Path: args[0], CommandSeparator: ".", } _ = doc.GenManTreeFromOpts(c.global.cmd, opts) return nil } incus-7.0.0/cmd/incusd/main_migratedumpsuccess.go000066400000000000000000000031121517523235500221360ustar00rootroot00000000000000package main import ( "errors" "fmt" "os" "strings" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/api" ) type cmdMigratedumpsuccess struct { global *cmdGlobal } func (c *cmdMigratedumpsuccess) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "migratedumpsuccess " cmd.Short = "Tell the daemon that a particular CRIU dump succeeded" cmd.Long = `Description: Tell the daemon that a particular CRIU dump succeeded This internal command is used from the CRIU dump script and is called as soon as the script is done running. ` cmd.RunE = c.run cmd.Hidden = true return cmd } func (c *cmdMigratedumpsuccess) run(cmd *cobra.Command, args []string) error { // Quick checks. if len(args) < 2 { _ = cmd.Help() if len(args) == 0 { return nil } return errors.New("Missing required arguments") } // Only root should run this if os.Geteuid() != 0 { return errors.New("This must be run as root") } clientArgs := incus.ConnectionArgs{ SkipGetServer: true, } d, err := incus.ConnectIncusUnix("", &clientArgs) if err != nil { return err } url := fmt.Sprintf("%s/websocket?secret=%s", strings.TrimPrefix(args[0], "/1.0"), args[1]) conn, err := d.RawWebsocket(url) if err != nil { return err } _ = conn.Close() resp, _, err := d.RawQuery("GET", fmt.Sprintf("%s/wait", args[0]), nil, "") if err != nil { return err } op, err := resp.MetadataAsOperation() if err != nil { return err } if op.StatusCode == api.Success { return nil } return errors.New(op.Err) } incus-7.0.0/cmd/incusd/main_netcat.go000066400000000000000000000046531517523235500175200ustar00rootroot00000000000000package main import ( "errors" "fmt" "net" "os" "sync" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/eagain" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/util" ) type cmdNetcat struct { global *cmdGlobal } func (c *cmdNetcat) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "netcat
" cmd.Short = "Send stdin data to a unix socket" cmd.Long = `Description: Send stdin data to a unix socket This internal command is used to forward the output of a program over a websocket by first forwarding it to a unix socket controlled by the daemon. Its main use is when running rsync or btrfs/zfs send/receive between two machines over the websocket API. ` cmd.RunE = c.run cmd.Hidden = true return cmd } func (c *cmdNetcat) run(cmd *cobra.Command, args []string) error { // Quick checks. if len(args) < 2 { _ = cmd.Help() if len(args) == 0 { return nil } return errors.New("Missing required arguments") } // Only root should run this if os.Geteuid() != 0 { return errors.New("This must be run as root") } logPath := internalUtil.LogPath(args[1], "netcat.log") if util.PathExists(logPath) { _ = os.Remove(logPath) } logFile, logErr := os.OpenFile(logPath, os.O_WRONLY|os.O_CREATE|os.O_SYNC, 0o644) if logErr == nil { defer func() { _ = logFile.Close() }() } uAddr, err := net.ResolveUnixAddr("unix", args[0]) if err != nil { if logErr == nil { _, _ = logFile.WriteString(fmt.Sprintf("Could not resolve unix domain socket \"%s\": %s\n", args[0], err)) } return err } conn, err := net.DialUnix("unix", nil, uAddr) if err != nil { if logErr == nil { _, _ = logFile.WriteString(fmt.Sprintf("Could not dial unix domain socket \"%s\": %s\n", args[0], err)) } return err } wg := sync.WaitGroup{} wg.Add(1) go func() { _, err := util.SafeCopy(eagain.Writer{Writer: os.Stdout}, eagain.Reader{Reader: conn}) if err != nil && logErr == nil { _, _ = logFile.WriteString(fmt.Sprintf("Error while copying from stdout to unix domain socket \"%s\": %s\n", args[0], err)) } _ = conn.Close() wg.Done() }() go func() { _, err := util.SafeCopy(eagain.Writer{Writer: conn}, eagain.Reader{Reader: os.Stdin}) if err != nil && logErr == nil { _, _ = logFile.WriteString(fmt.Sprintf("Error while copying from unix domain socket \"%s\" to stdin: %s\n", args[0], err)) } }() wg.Wait() return nil } incus-7.0.0/cmd/incusd/main_nsexec.go000066400000000000000000000210751517523235500175240ustar00rootroot00000000000000/** * This file is a bit funny. The goal here is to use setns() to manipulate * files inside the container, so we don't have to reason about the paths to * make sure they don't escape (we can simply rely on the kernel for * correctness). Unfortunately, you can't setns() to a mount namespace with a * multi-threaded program, which every golang binary is. However, by declaring * our init as an initializer, we can capture process control before it is * transferred to the golang runtime, so we can then setns() as we'd like * before golang has a chance to set up any threads. So, we implement two new * fork* commands which are captured here, and take a file on the host fs * and copy it into the container ns. * * An alternative to this would be to move this code into a separate binary, * which of course has problems of its own when it comes to packaging (how do * we find the binary, what do we do if someone does file push and it is * missing, etc.). After some discussion, even though the embedded method is * somewhat convoluted, it was preferred. */ package main /* #include "config.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../../shared/cgo/incus.h" #include "../../shared/cgo/file_utils.h" #include "../../shared/cgo/memory_utils.h" #include "../../shared/cgo/mount_utils.h" #include "../../shared/cgo/process_utils.h" #include "../../shared/cgo/syscall_numbers.h" #include "../../shared/cgo/syscall_wrappers.h" // Command line parsing and tracking char *cmdline_buf = NULL; char *cmdline_cur = NULL; ssize_t cmdline_size = -1; char* advance_arg(bool required) { while (*cmdline_cur != 0) cmdline_cur++; cmdline_cur++; if (cmdline_size <= cmdline_cur - cmdline_buf) { if (!required) return NULL; fprintf(stderr, "not enough arguments\n"); _exit(1); } return cmdline_cur; } void error(char *msg) { int old_errno = errno; if (old_errno == 0) { fprintf(stderr, "%s\n", msg); fprintf(stderr, "errno: 0\n"); return; } perror(msg); fprintf(stderr, "errno: %d\n", old_errno); } int preserve_ns(pid_t pid, int ns_fd, const char *ns) { int ret; if (ns_fd >= 0) return openat(ns_fd, ns, O_RDONLY | O_CLOEXEC); // 5 /proc + 21 /int_as_str + 3 /ns + 20 /NS_NAME + 1 \0 #define __NS_PATH_LEN 50 char path[__NS_PATH_LEN]; // This way we can use this function to also check whether namespaces // are supported by the kernel by passing in the NULL or the empty // string. ret = snprintf(path, __NS_PATH_LEN, "/proc/%d/ns%s%s", pid, !ns || strcmp(ns, "") == 0 ? "" : "/", !ns || strcmp(ns, "") == 0 ? "" : ns); errno = EFBIG; if (ret < 0 || (size_t)ret >= __NS_PATH_LEN) return -EFBIG; return open(path, O_RDONLY | O_CLOEXEC); } // in_same_namespace - Check whether two processes are in the same namespace. // @pid1 - PID of the first process. // @pid2 - PID of the second process. // @ns_fd2 - ns_fd @pid2. // @ns - Name of the namespace to check. Must correspond to one of the names // for the namespaces as shown in /proc/= sizeof(path)) return -E2BIG; ns_fd = open(path, O_DIRECTORY | O_RDONLY | O_CLOEXEC); if (ns_fd < 0) return -errno; if (pidfd >= 0) { // Verify that the pid has not been recycled and our /proc/ handle // is still valid. ret = incus_pidfd_send_signal(pidfd, 0, NULL, 0); if (ret && errno != EPERM) return -errno; } return move_fd(ns_fd); } static const struct ns_info { const char *proc_name; int clone_flag; } ns_info[] = { { "user", CLONE_NEWUSER }, { "mnt", CLONE_NEWNS }, { "pid", CLONE_NEWPID }, { "uts", CLONE_NEWUTS }, { "ipc", CLONE_NEWIPC }, { "net", CLONE_NEWNET }, { "cgroup", CLONE_NEWCGROUP }, { "time", CLONE_NEWTIME }, }; bool change_namespaces(int pidfd, int nsfd, unsigned int flags) { __do_close int fd = -EBADF; if (pidfd >= 0 && setns(pidfd, flags) == 0) return true; if (nsfd < 0) return false; for (int i = 0; i < ARRAY_SIZE(ns_info); i++) { if (flags & ns_info[i].clone_flag) { fd = openat(nsfd, ns_info[i].proc_name, O_RDONLY | O_CLOEXEC); if (fd < 0) return false; if (setns(fd, 0) < 0) return false; } } return true; } static char *file_to_buf(char *path, ssize_t *length) { __do_close int fd = -EBADF; __do_free char *copy = NULL; char buf[PATH_MAX]; if (!length) return NULL; fd = open(path, O_RDONLY | O_CLOEXEC); if (fd < 0) return NULL; *length = 0; for (;;) { ssize_t n; char *old = copy; n = read_nointr(fd, buf, sizeof(buf)); if (n < 0) return NULL; if (!n) break; copy = realloc(old, (*length + n) * sizeof(*old)); if (!copy) return NULL; memcpy(copy + *length, buf, n); *length += n; } return move_ptr(copy); } int mount_detach_idmap(const char *path, int fd_userns) { __do_close int fd_tree = -EBADF; struct lxc_mount_attr attr = { .attr_set = MOUNT_ATTR_IDMAP, }; int ret; fd_tree = incus_open_tree(-EBADF, path, OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); if (fd_tree < 0) return -errno; attr.userns_fd = fd_userns; ret = incus_mount_setattr(fd_tree, "", AT_EMPTY_PATH, &attr, sizeof(attr)); if (ret < 0) return -errno; return move_fd(fd_tree); } __attribute__((constructor)) void init(void) { __do_free char *cmdline = NULL; int ret; cmdline_buf = file_to_buf("/proc/self/cmdline", &cmdline_size); if (!cmdline_buf) _exit(232); // Skip the first argument (but don't fail on missing second argument) cmdline = cmdline_cur = cmdline_buf; while (*cmdline_cur != 0) cmdline_cur++; cmdline_cur++; if (cmdline_size <= cmdline_cur - cmdline_buf) { return; } // Intercepts some subcommands if (strcmp(cmdline_cur, "forkexec") == 0) forkexec(); else if (strcmp(cmdline_cur, "forkfile") == 0) forkfile(); else if (strcmp(cmdline_cur, "forksyscall") == 0) forksyscall(); else if (strcmp(cmdline_cur, "forkmount") == 0) forkmount(); else if (strcmp(cmdline_cur, "forkbpf") == 0) forkbpf(); else if (strcmp(cmdline_cur, "forknet") == 0) forknet(); else if (strcmp(cmdline_cur, "forkproxy") == 0) forkproxy(); else if (strcmp(cmdline_cur, "forkuevent") == 0) forkuevent(); else if (strcmp(cmdline_cur, "forkcoresched") == 0) forkcoresched(); else if (strcmp(cmdline_cur, "forkzfs") == 0) { ret = unshare(CLONE_NEWNS); if (ret < 0) { fprintf(stderr, "Failed unshare of mount namespace: %s\n", strerror(errno)); return; } } } */ import "C" import ( // Used by cgo _ "github.com/lxc/incus/v7/shared/cgo" ) incus-7.0.0/cmd/incusd/main_shutdown.go000066400000000000000000000040361517523235500201100ustar00rootroot00000000000000package main import ( "fmt" "net/http" "net/url" "strconv" "time" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" ) type cmdShutdown struct { global *cmdGlobal flagForce bool flagTimeout int } func (c *cmdShutdown) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "shutdown" cmd.Short = "Tell the daemon to shutdown all instances and exit" cmd.Long = `Description: Tell the daemon to shutdown all instances and exit This will tell the daemon to start a clean shutdown of all instances, followed by having itself shutdown and exit. This can take quite a while as instances can take a long time to shutdown, especially if a non-standard timeout was configured for them. ` cmd.RunE = c.run cmd.Flags().IntVarP(&c.flagTimeout, "timeout", "t", 0, "Number of seconds to wait before giving up"+"``") cmd.Flags().BoolVarP(&c.flagForce, "force", "f", false, "Force shutdown instead of waiting for running operations to finish"+"``") cmd.Hidden = true return cmd } func (c *cmdShutdown) run(_ *cobra.Command, _ []string) error { connArgs := &incus.ConnectionArgs{ SkipGetServer: true, } d, err := incus.ConnectIncusUnix("", connArgs) if err != nil { return err } v := url.Values{} v.Set("force", strconv.FormatBool(c.flagForce)) chResult := make(chan error, 1) go func() { defer close(chResult) httpClient, err := d.GetHTTPClient() if err != nil { chResult <- err return } // Request shutdown, this shouldn't return until daemon has stopped so use a large request timeout. httpTransport := httpClient.Transport.(*http.Transport) httpTransport.ResponseHeaderTimeout = 3600 * time.Second _, _, err = d.RawQuery("PUT", fmt.Sprintf("/internal/shutdown?%s", v.Encode()), nil, "") if err != nil { chResult <- err return } }() if c.flagTimeout > 0 { select { case err = <-chResult: return err case <-time.After(time.Second * time.Duration(c.flagTimeout)): return fmt.Errorf("Daemon still running after %ds timeout", c.flagTimeout) } } return <-chResult } incus-7.0.0/cmd/incusd/main_test.go000066400000000000000000000057261517523235500172230ustar00rootroot00000000000000package main import ( "context" "fmt" "os" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/sys" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/idmap" ) func mockStartDaemon() (*Daemon, error) { d := defaultDaemon() d.os.MockMode = true // Setup test certificates. We reuse the ones already on disk under // the test/ directory, to avoid generating new ones, which is // expensive. err := sys.SetupTestCerts(internalUtil.VarPath()) if err != nil { return nil, err } err = d.Init() if err != nil { return nil, err } d.os.IdmapSet = &idmap.Set{Entries: []idmap.Entry{ {IsUID: true, HostID: 100000, NSID: 0, MapRange: 500000}, {IsGID: true, HostID: 100000, NSID: 0, MapRange: 500000}, }} return d, nil } type daemonTestSuite struct { suite.Suite d *Daemon Req *require.Assertions tmpdir string } const daemonTestSuiteDefaultStoragePool string = "testrunPool" func (suite *daemonTestSuite) SetupTest() { tmpdir, err := os.MkdirTemp("", "incus_testrun_") if err != nil { suite.T().Errorf("failed to create temp dir: %v", err) } suite.tmpdir = tmpdir err = os.Setenv("INCUS_DIR", suite.tmpdir) if err != nil { suite.T().Errorf("failed to set INCUS_DIR: %v", err) } suite.d, err = mockStartDaemon() if err != nil { suite.T().Errorf("failed to start daemon: %v", err) } // Create default storage pool. Make sure that we don't pass a nil to // the next function. poolConfig := map[string]string{} // Create the database entry for the storage pool. poolDescription := fmt.Sprintf("%s storage pool", daemonTestSuiteDefaultStoragePool) _, err = dbStoragePoolCreateAndUpdateCache(context.Background(), suite.d.State(), daemonTestSuiteDefaultStoragePool, poolDescription, "mock", poolConfig) if err != nil { suite.T().Errorf("failed to create default storage pool: %v", err) } rootDev := map[string]string{} rootDev["path"] = "/" rootDev["pool"] = daemonTestSuiteDefaultStoragePool device := cluster.Device{ Name: "root", Type: cluster.TypeDisk, Config: rootDev, } err = suite.d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { profile, err := cluster.GetProfile(ctx, tx.Tx(), "default", "default") if err != nil { return err } return cluster.UpdateProfileDevices(ctx, tx.Tx(), int64(profile.ID), map[string]cluster.Device{"root": device}) }) if err != nil { suite.T().Errorf("failed to update default profile: %v", err) } suite.Req = require.New(suite.T()) } func (suite *daemonTestSuite) TearDownTest() { err := suite.d.Stop(context.Background(), unix.SIGQUIT) if err != nil { suite.T().Errorf("failed to stop daemon: %v", err) } err = os.RemoveAll(suite.tmpdir) if err != nil { suite.T().Errorf("failed to remove temp dir: %v", err) } } incus-7.0.0/cmd/incusd/main_version.go000066400000000000000000000010411517523235500177130ustar00rootroot00000000000000package main import ( "fmt" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/version" cli "github.com/lxc/incus/v7/shared/cmd" ) type cmdVersion struct { global *cmdGlobal } func (c *cmdVersion) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "version" cmd.Short = "Show the server version" cmd.Long = cli.FormatSection("Description:", `Show the server version`) cmd.RunE = c.run return cmd } func (c *cmdVersion) run(_ *cobra.Command, _ []string) error { fmt.Println(version.Version) return nil } incus-7.0.0/cmd/incusd/main_waitready.go000066400000000000000000000042571517523235500202330ustar00rootroot00000000000000package main import ( "fmt" "time" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/logger" ) type cmdWaitready struct { global *cmdGlobal flagTimeout int } func (c *cmdWaitready) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "waitready" cmd.Short = "Wait for the daemon to be ready to process requests" cmd.Long = `Description: Wait for the daemon to be ready to process requests This command will block until the daemon is reachable over its REST API and is done with early start tasks like re-starting previously started containers. ` cmd.RunE = c.run cmd.Hidden = true cmd.Flags().IntVarP(&c.flagTimeout, "timeout", "t", 0, "Number of seconds to wait before giving up"+"``") return cmd } func (c *cmdWaitready) run(_ *cobra.Command, _ []string) error { finger := make(chan error, 1) var errLast error go func() { for i := 0; ; i++ { // Start logging only after the 10'th attempt (about 5 // seconds). Then after the 30'th attempt (about 15 // seconds), log only only one attempt every 10 // attempts (about 5 seconds), to avoid being too // verbose. doLog := false if i > 10 { doLog = i < 30 || ((i % 10) == 0) } if doLog { logger.Debugf("Connecting to the daemon (attempt %d)", i) } d, err := incus.ConnectIncusUnix("", nil) if err != nil { errLast = err if doLog { logger.Debugf("Failed connecting to the daemon (attempt %d): %v", i, err) } time.Sleep(500 * time.Millisecond) continue } if doLog { logger.Debugf("Checking if the daemon is ready (attempt %d)", i) } _, _, err = d.RawQuery("GET", "/internal/ready", nil, "") if err != nil { errLast = err if doLog { logger.Debugf("Failed to check if the daemon is ready (attempt %d): %v", i, err) } time.Sleep(500 * time.Millisecond) continue } finger <- nil return } }() if c.flagTimeout > 0 { select { case <-finger: break case <-time.After(time.Second * time.Duration(c.flagTimeout)): return fmt.Errorf("Daemon still not running after %ds timeout (%v)", c.flagTimeout, errLast) } } else { <-finger } return nil } incus-7.0.0/cmd/incusd/metadata.go000066400000000000000000000025431517523235500170120ustar00rootroot00000000000000package main import ( "net/http" "github.com/lxc/incus/v7/internal/server/metadata" "github.com/lxc/incus/v7/internal/server/response" ) var metadataConfigurationCmd = APIEndpoint{ Path: "metadata/configuration", Get: APIEndpointAction{Handler: metadataConfigurationGet, AllowUntrusted: true}, } // swagger:operation GET /1.0/metadata/configuration metadata_configuration_get // // Get the metadata configuration // // Returns the generated metadata configuration in YAML format. // // --- // produces: // - text/plain // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: string // description: The generated metadata configuration // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func metadataConfigurationGet(_ *Daemon, _ *http.Request) response.Response { return response.SyncResponse(true, metadata.Data) } incus-7.0.0/cmd/incusd/migrate.go000066400000000000000000000163201517523235500166600ustar00rootroot00000000000000// Package migration provides the primitives for server to server migration. // // See https://github.com/lxc/incus/blob/main/doc/migration.md for a complete // description. package main import ( "context" "fmt" "net/http" "sync" "time" "github.com/gorilla/websocket" "google.golang.org/protobuf/proto" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/server/instance" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/idmap" ) type migrationFields struct { controlLock sync.Mutex conns map[string]*migrationConn // container specific fields live bool instanceOnly bool instance instance.Instance // storage specific fields volumeOnly bool allowInconsistent bool storagePool string } func (c *migrationFields) send(m proto.Message) error { /* gorilla websocket doesn't allow concurrent writes, and * panic()s if it sees them (which is reasonable). If e.g. we * happen to fail, get scheduled, start our write, then get * unscheduled before the write is bit to a new thread which is * receiving an error from the other side (due to our previous * close), we can engage in these concurrent writes, which * casuses the whole daemon to panic. * * Instead, let's lock sends to the controlConn so that we only ever * write one message at the time. */ c.controlLock.Lock() defer c.controlLock.Unlock() conn, err := c.conns[api.SecretNameControl].WebSocket(context.TODO()) if err != nil { return fmt.Errorf("Control connection not initialized: %w", err) } _ = conn.SetWriteDeadline(time.Now().Add(30 * time.Second)) err = migration.ProtoSend(conn, m) if err != nil { return err } return nil } func (c *migrationFields) recv(m proto.Message, handshake bool) error { conn, err := c.conns[api.SecretNameControl].WebSocket(context.TODO()) if err != nil { return fmt.Errorf("Control connection not initialized: %w", err) } // If dealing with the initial handshake, use a short timeout to // prevent lingering create operations on communication failure. // // Later calls are done during migration as migration barrier and // can potentially take multiple hours. if handshake { _ = conn.SetReadDeadline(time.Now().Add(30 * time.Second)) // Remove the deadline after the request. defer func() { _ = conn.SetReadDeadline(time.Time{}) }() } return migration.ProtoRecv(conn, m) } func (c *migrationFields) Cancel(op *operations.Operation) error { c.controlLock.Lock() defer c.controlLock.Unlock() for _, conn := range c.conns { conn.Close() } return nil } func (c *migrationFields) disconnect() { c.controlLock.Lock() ctx, cancel := context.WithTimeout(context.TODO(), time.Second) defer cancel() conn, _ := c.conns[api.SecretNameControl].WebSocket(ctx) if conn != nil { closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") _ = conn.SetWriteDeadline(time.Now().Add(time.Second * 30)) _ = conn.WriteMessage(websocket.CloseMessage, closeMsg) } c.controlLock.Unlock() /* Below we just Close(), which doesn't actually write to the * websocket, it just closes the underlying connection. If e.g. there * is still a filesystem transfer going on, but the other side has run * out of disk space, writing an actual CloseMessage here will cause * gorilla websocket to panic. Instead, we just force close this * connection, since we report the error over the control channel * anyway. */ for _, conn := range c.conns { conn.Close() } } func (c *migrationFields) sendControl(err error) { c.controlLock.Lock() conn, _ := c.conns[api.SecretNameControl].WebSocket(context.TODO()) if conn != nil { _ = conn.SetWriteDeadline(time.Now().Add(time.Second * 10)) migration.ProtoSendControl(conn, err) } c.controlLock.Unlock() if err != nil { c.disconnect() } } func (c *migrationFields) controlChannel() <-chan *localMigration.ControlResponse { ch := make(chan *localMigration.ControlResponse) go func() { resp := localMigration.ControlResponse{} err := c.recv(&resp.MigrationControl, false) if err != nil { resp.Err = err ch <- &resp return } ch <- &resp }() return ch } type migrationSourceWs struct { migrationFields clusterMoveSourceName string devices api.DevicesMap pushCertificate string pushOperationURL string pushSecrets map[string]string } func (s *migrationSourceWs) Metadata() any { secrets := make(jmap.Map, len(s.conns)) for connName, conn := range s.conns { secrets[connName] = conn.Secret() } return secrets } func (s *migrationSourceWs) Connect(op *operations.Operation, r *http.Request, w http.ResponseWriter) error { incomingSecret := r.FormValue("secret") if incomingSecret == "" { return api.StatusErrorf(http.StatusBadRequest, "Missing migration source secret") } for connName, conn := range s.conns { if incomingSecret != conn.Secret() { continue } err := conn.AcceptIncoming(r, w) if err != nil { return fmt.Errorf("Failed accepting incoming migration source %q connection: %w", connName, err) } return nil } // If we didn't find the right secret, the user provided a bad one, so return 403, not 404, since this // operation actually exists. return api.StatusErrorf(http.StatusForbidden, "Invalid migration source secret") } type migrationSink struct { migrationFields url string push bool clusterMoveSourceName string refresh bool refreshExcludeOlder bool } // MigrationSinkArgs arguments to configure migration sink. type migrationSinkArgs struct { // General migration fields Dialer *websocket.Dialer Push bool Secrets map[string]string URL string // Instance specific fields Instance instance.Instance InstanceOnly bool Idmap *idmap.Set Live bool Refresh bool RefreshExcludeOlder bool ClusterMoveSourceName string Snapshots []*migration.Snapshot // Storage specific fields StoragePool string VolumeOnly bool VolumeSize int64 // Transport specific fields RsyncFeatures []string } // Metadata returns metadata for the migration sink. func (s *migrationSink) Metadata() any { secrets := make(jmap.Map, len(s.conns)) for connName, conn := range s.conns { secrets[connName] = conn.Secret() } return secrets } // Connect connects to the migration source. func (s *migrationSink) Connect(op *operations.Operation, r *http.Request, w http.ResponseWriter) error { incomingSecret := r.FormValue("secret") if incomingSecret == "" { return api.StatusErrorf(http.StatusBadRequest, "Missing migration sink secret") } for connName, conn := range s.conns { if incomingSecret != conn.Secret() { continue } err := conn.AcceptIncoming(r, w) if err != nil { return fmt.Errorf("Failed accepting incoming migration sink %q connection: %w", connName, err) } return nil } // If we didn't find the right secret, the user provided a bad one, so return 403, not 404, since this // operation actually exists. return api.StatusErrorf(http.StatusForbidden, "Invalid migration sink secret") } incus-7.0.0/cmd/incusd/migrate_instance.go000066400000000000000000000217201517523235500205440ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "io" "net/url" "os/exec" "strings" "time" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/instance/operationlock" "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) func newMigrationSource(inst instance.Instance, stateful bool, instanceOnly bool, allowInconsistent bool, clusterMoveSourceName string, storagePool string, devices api.DevicesMap, pushTarget *api.InstancePostTarget) (*migrationSourceWs, error) { ret := migrationSourceWs{ migrationFields: migrationFields{ instance: inst, allowInconsistent: allowInconsistent, storagePool: storagePool, }, clusterMoveSourceName: clusterMoveSourceName, devices: devices, } if pushTarget != nil { ret.pushCertificate = pushTarget.Certificate ret.pushOperationURL = pushTarget.Operation ret.pushSecrets = pushTarget.Websockets } ret.instanceOnly = instanceOnly secretNames := []string{api.SecretNameControl, api.SecretNameFilesystem} if stateful && inst.IsRunning() { if inst.Type() == instancetype.Container { _, err := exec.LookPath("criu") if err != nil { return nil, migration.ErrNoLiveMigrationSource } } ret.live = true secretNames = append(secretNames, api.SecretNameState) } ret.conns = make(map[string]*migrationConn, len(secretNames)) for _, connName := range secretNames { if ret.pushOperationURL != "" { if ret.pushSecrets[connName] == "" { return nil, fmt.Errorf("Expected %q connection secret missing from migration source target request", connName) } dialer, err := setupWebsocketDialer(ret.pushCertificate) if err != nil { return nil, fmt.Errorf("Failed setting up websocket dialer for migration source %q connection: %w", connName, err) } u, err := url.Parse(fmt.Sprintf("wss://%s/websocket", strings.TrimPrefix(ret.pushOperationURL, "https://"))) if err != nil { return nil, fmt.Errorf("Failed parsing websocket URL for migration source %q connection: %w", connName, err) } ret.conns[connName] = newMigrationConn(ret.pushSecrets[connName], dialer, u) } else { secret, err := internalUtil.RandomHexString(32) if err != nil { return nil, fmt.Errorf("Failed creating migration source secret for %q connection: %w", connName, err) } ret.conns[connName] = newMigrationConn(secret, nil, nil) } } return &ret, nil } func (s *migrationSourceWs) do(migrateOp *operations.Operation) error { l := logger.AddContext(logger.Ctx{"project": s.instance.Project().Name, "instance": s.instance.Name(), "live": s.live, "clusterMoveSourceName": s.clusterMoveSourceName, "push": s.pushOperationURL != ""}) ctx, cancel := context.WithTimeout(context.TODO(), time.Second*30) defer cancel() l.Debug("Waiting for migration control connection on source") _, err := s.conns[api.SecretNameControl].WebSocket(ctx) if err != nil { return fmt.Errorf("Failed waiting for migration control connection on source: %w", err) } l.Debug("Migration control connection established on source") defer l.Debug("Migration channels disconnected on source") defer s.disconnect() stateConnFunc := func(ctx context.Context) (io.ReadWriteCloser, error) { conn := s.conns[api.SecretNameState] if conn == nil { return nil, errors.New("Migration source control connection not initialized") } wsConn, err := conn.WebsocketIO(ctx) if err != nil { return nil, fmt.Errorf("Failed getting migration source control connection: %w", err) } return wsConn, nil } filesystemConnFunc := func(ctx context.Context) (io.ReadWriteCloser, error) { conn := s.conns[api.SecretNameFilesystem] if conn == nil { return nil, errors.New("Migration source filesystem connection not initialized") } wsConn, err := conn.WebsocketIO(ctx) if err != nil { return nil, fmt.Errorf("Failed getting migration source filesystem connection: %w", err) } return wsConn, nil } s.instance.SetOperation(migrateOp) err = s.instance.MigrateSend(instance.MigrateSendArgs{ MigrateArgs: instance.MigrateArgs{ ControlSend: s.send, ControlReceive: s.recv, StateConn: stateConnFunc, FilesystemConn: filesystemConnFunc, Snapshots: !s.instanceOnly, Live: s.live, Disconnect: func() { for connName, conn := range s.conns { if connName != api.SecretNameControl { conn.Close() } } }, ClusterMoveSourceName: s.clusterMoveSourceName, StoragePool: s.storagePool, }, AllowInconsistent: s.allowInconsistent, Devices: s.devices, }) if err != nil { l.Error("Failed migration on source", logger.Ctx{"err": err}) errMsg := fmt.Errorf("Failed migration on source: %w", err) s.sendControl(errMsg) return errMsg } return nil } func newMigrationSink(args *migrationSinkArgs) (*migrationSink, error) { sink := migrationSink{ migrationFields: migrationFields{ instance: args.Instance, instanceOnly: args.InstanceOnly, live: args.Live, storagePool: args.StoragePool, }, url: args.URL, clusterMoveSourceName: args.ClusterMoveSourceName, push: args.Push, refresh: args.Refresh, refreshExcludeOlder: args.RefreshExcludeOlder, } secretNames := []string{api.SecretNameControl, api.SecretNameFilesystem} if sink.live { if sink.instance.Type() == instancetype.Container { _, err := exec.LookPath("criu") if err != nil { return nil, migration.ErrNoLiveMigrationTarget } } secretNames = append(secretNames, api.SecretNameState) } sink.conns = make(map[string]*migrationConn, len(secretNames)) for _, connName := range secretNames { if !sink.push { if args.Secrets[connName] == "" { return nil, fmt.Errorf("Expected %q connection secret missing from migration sink target request", connName) } u, err := url.Parse(fmt.Sprintf("wss://%s/websocket", strings.TrimPrefix(args.URL, "https://"))) if err != nil { return nil, fmt.Errorf("Failed parsing websocket URL for migration sink %q connection: %w", connName, err) } sink.conns[connName] = newMigrationConn(args.Secrets[connName], args.Dialer, u) } else { secret, err := internalUtil.RandomHexString(32) if err != nil { return nil, fmt.Errorf("Failed creating migration sink secret for %q connection: %w", connName, err) } sink.conns[connName] = newMigrationConn(secret, nil, nil) } } return &sink, nil } func (c *migrationSink) do(instOp *operationlock.InstanceOperation) error { l := logger.AddContext(logger.Ctx{"project": c.instance.Project().Name, "instance": c.instance.Name(), "live": c.live, "clusterMoveSourceName": c.clusterMoveSourceName, "push": c.push}) ctx, cancel := context.WithTimeout(context.TODO(), time.Second*30) defer cancel() l.Debug("Waiting for migration control connection on target") _, err := c.conns[api.SecretNameControl].WebSocket(ctx) if err != nil { return fmt.Errorf("Failed waiting for migration control connection on target: %w", err) } l.Debug("Migration control connection established on target") defer l.Debug("Migration channels disconnected on target") if c.push { defer c.disconnect() } stateConnFunc := func(ctx context.Context) (io.ReadWriteCloser, error) { conn := c.conns[api.SecretNameState] if conn == nil { return nil, errors.New("Migration target control connection not initialized") } wsConn, err := conn.WebsocketIO(ctx) if err != nil { return nil, fmt.Errorf("Failed getting migration target control connection: %w", err) } return wsConn, nil } filesystemConnFunc := func(ctx context.Context) (io.ReadWriteCloser, error) { conn := c.conns[api.SecretNameFilesystem] if conn == nil { return nil, errors.New("Migration target filesystem connection not initialized") } wsConn, err := conn.WebsocketIO(ctx) if err != nil { return nil, fmt.Errorf("Failed getting migration target filesystem connection: %w", err) } return wsConn, nil } err = c.instance.MigrateReceive(instance.MigrateReceiveArgs{ MigrateArgs: instance.MigrateArgs{ ControlSend: c.send, ControlReceive: c.recv, StateConn: stateConnFunc, FilesystemConn: filesystemConnFunc, Snapshots: !c.instanceOnly, Live: c.live, Disconnect: func() { for connName, conn := range c.conns { if connName != api.SecretNameControl { conn.Close() } } }, ClusterMoveSourceName: c.clusterMoveSourceName, StoragePool: c.storagePool, }, InstanceOperation: instOp, Refresh: c.refresh, RefreshExcludeOlder: c.refreshExcludeOlder, }) if err != nil { l.Error("Failed migration on target", logger.Ctx{"err": err}) errMsg := fmt.Errorf("Failed migration on target: %w", err) c.sendControl(errMsg) return errMsg } return nil } incus-7.0.0/cmd/incusd/migrate_storage_volumes.go000066400000000000000000000435031517523235500221610ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "io" "net/url" "slices" "strings" "time" "github.com/lxc/incus/v7/internal/migration" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) func newStorageMigrationSource(volumeOnly bool, pushTarget *api.StorageVolumePostTarget) (*migrationSourceWs, error) { ret := migrationSourceWs{ migrationFields: migrationFields{}, } if pushTarget != nil { ret.pushCertificate = pushTarget.Certificate ret.pushOperationURL = pushTarget.Operation ret.pushSecrets = pushTarget.Websockets } ret.volumeOnly = volumeOnly secretNames := []string{api.SecretNameControl, api.SecretNameFilesystem} ret.conns = make(map[string]*migrationConn, len(secretNames)) for _, connName := range secretNames { if ret.pushOperationURL != "" { if ret.pushSecrets[connName] == "" { return nil, fmt.Errorf("Expected %q connection secret missing from migration source target request", connName) } dialer, err := setupWebsocketDialer(ret.pushCertificate) if err != nil { return nil, fmt.Errorf("Failed setting up websocket dialer for migration source %q connection: %w", connName, err) } u, err := url.Parse(fmt.Sprintf("wss://%s/websocket", strings.TrimPrefix(ret.pushOperationURL, "https://"))) if err != nil { return nil, fmt.Errorf("Failed parsing websocket URL for migration source %q connection: %w", connName, err) } ret.conns[connName] = newMigrationConn(ret.pushSecrets[connName], dialer, u) } else { secret, err := internalUtil.RandomHexString(32) if err != nil { return nil, fmt.Errorf("Failed creating migration source secret for %q connection: %w", connName, err) } ret.conns[connName] = newMigrationConn(secret, nil, nil) } } return &ret, nil } func (s *migrationSourceWs) DoStorage(state *state.State, projectName string, poolName string, volName string, migrateOp *operations.Operation) error { l := logger.AddContext(logger.Ctx{"project": projectName, "pool": poolName, "volume": volName, "push": s.pushOperationURL != ""}) ctx, cancel := context.WithTimeout(state.ShutdownCtx, time.Second*30) defer cancel() l.Info("Waiting for migration connections on source") for _, connName := range []string{api.SecretNameControl, api.SecretNameFilesystem} { _, err := s.conns[connName].WebSocket(ctx) if err != nil { return fmt.Errorf("Failed waiting for migration %q connection on source: %w", connName, err) } } l.Info("Migration channels connected on source") defer l.Info("Migration channels disconnected on source") defer s.disconnect() var poolMigrationTypes []localMigration.Type pool, err := storagePools.LoadByName(state, poolName) if err != nil { return err } srcConfig, err := pool.GenerateCustomVolumeBackupConfig(projectName, volName, !s.volumeOnly, migrateOp) if err != nil { return fmt.Errorf("Failed generating volume migration config: %w", err) } dbContentType, err := storagePools.VolumeContentTypeNameToContentType(srcConfig.Volume.ContentType) if err != nil { return err } contentType, err := storagePools.VolumeDBContentTypeToContentType(dbContentType) if err != nil { return err } volStorageName := project.StorageVolume(projectName, volName) vol := pool.GetVolume(storageDrivers.VolumeTypeCustom, contentType, volStorageName, srcConfig.Volume.Config) var volSize int64 if contentType == storageDrivers.ContentTypeBlock { err = vol.MountTask(func(mountPath string, op *operations.Operation) error { volDiskPath, err := pool.Driver().GetVolumeDiskPath(vol) if err != nil { return err } volSize, err = storageDrivers.BlockDiskSizeBytes(volDiskPath) if err != nil { return err } return nil }, nil) if err != nil { return err } } // The refresh argument passed to MigrationTypes() is always set // to false here. The migration source/sender doesn't need to care whether // or not it's doing a refresh as the migration sink/receiver will know // this, and adjust the migration types accordingly. // The same applies for clusterMove and storageMove, which are set to the most optimized defaults. poolMigrationTypes = pool.MigrationTypes(storageDrivers.ContentType(srcConfig.Volume.ContentType), false, !s.volumeOnly, true, false) if len(poolMigrationTypes) == 0 { return errors.New("No source migration types available") } // Convert the pool's migration type options to an offer header to target. offerHeader := localMigration.TypesToHeader(poolMigrationTypes...) // Offer to send index header. indexHeaderVersion := localMigration.IndexHeaderVersion offerHeader.IndexHeaderVersion = &indexHeaderVersion offerHeader.VolumeSize = &volSize // Only send snapshots when requested. if !s.volumeOnly { offerHeader.Snapshots = make([]*migration.Snapshot, 0, len(srcConfig.VolumeSnapshots)) offerHeader.SnapshotNames = make([]string, 0, len(srcConfig.VolumeSnapshots)) for i := range srcConfig.VolumeSnapshots { offerHeader.SnapshotNames = append(offerHeader.SnapshotNames, srcConfig.VolumeSnapshots[i].Name) // Set size for snapshot volume snapSize, err := storagePools.CalculateVolumeSnapshotSize(projectName, pool, contentType, storageDrivers.VolumeTypeCustom, volName, srcConfig.VolumeSnapshots[i].Name) if err != nil { return err } srcConfig.VolumeSnapshots[i].Config["size"] = fmt.Sprintf("%d", snapSize) offerHeader.Snapshots = append(offerHeader.Snapshots, localMigration.VolumeSnapshotToProtobuf(srcConfig.VolumeSnapshots[i])) } } // Send offer to target. err = s.send(offerHeader) if err != nil { logger.Errorf("Failed to send storage volume migration header") s.sendControl(err) return err } // Receive response from target. respHeader := &migration.MigrationHeader{} err = s.recv(respHeader, true) if err != nil { logger.Errorf("Failed to receive storage volume migration header") s.sendControl(err) return err } migrationTypes, err := localMigration.MatchTypes(respHeader, storagePools.FallbackMigrationType(storageDrivers.ContentType(srcConfig.Volume.ContentType)), poolMigrationTypes) if err != nil { logger.Errorf("Failed to negotiate migration type: %v", err) s.sendControl(err) return err } volSourceArgs := &localMigration.VolumeSourceArgs{ IndexHeaderVersion: respHeader.GetIndexHeaderVersion(), // Enable index header frame if supported. Name: srcConfig.Volume.Name, MigrationType: migrationTypes[0], Snapshots: offerHeader.SnapshotNames, TrackProgress: true, ContentType: srcConfig.Volume.ContentType, Info: &localMigration.Info{Config: srcConfig}, VolumeOnly: s.volumeOnly, } // Only send the snapshots that the target requests when refreshing. if respHeader.GetRefresh() { volSourceArgs.Refresh = true volSourceArgs.Snapshots = respHeader.GetSnapshotNames() allSnapshots := volSourceArgs.Info.Config.VolumeSnapshots // Ensure that only the requested snapshots are included in the migration index header. volSourceArgs.Info.Config.VolumeSnapshots = make([]*api.StorageVolumeSnapshot, 0, len(volSourceArgs.Snapshots)) for i := range allSnapshots { if slices.Contains(volSourceArgs.Snapshots, allSnapshots[i].Name) { volSourceArgs.Info.Config.VolumeSnapshots = append(volSourceArgs.Info.Config.VolumeSnapshots, allSnapshots[i]) } } } fsConn, err := s.conns[api.SecretNameFilesystem].WebsocketIO(state.ShutdownCtx) if err != nil { return err } err = pool.MigrateCustomVolume(projectName, fsConn, volSourceArgs, migrateOp) if err != nil { s.sendControl(err) return err } msg := migration.MigrationControl{} err = s.recv(&msg, false) if err != nil { logger.Errorf("Failed to receive storage volume migration control message") return err } if !msg.GetSuccess() { logger.Errorf("Failed to send storage volume") return errors.New(msg.GetMessage()) } logger.Debugf("Migration source finished transferring storage volume") return nil } func newStorageMigrationSink(args *migrationSinkArgs) (*migrationSink, error) { sink := migrationSink{ migrationFields: migrationFields{ volumeOnly: args.VolumeOnly, }, url: args.URL, push: args.Push, refresh: args.Refresh, refreshExcludeOlder: args.RefreshExcludeOlder, } secretNames := []string{api.SecretNameControl, api.SecretNameFilesystem} sink.conns = make(map[string]*migrationConn, len(secretNames)) for _, connName := range secretNames { if !sink.push { if args.Secrets[connName] == "" { return nil, fmt.Errorf("Expected %q connection secret missing from migration sink target request", connName) } u, err := url.Parse(fmt.Sprintf("wss://%s/websocket", strings.TrimPrefix(args.URL, "https://"))) if err != nil { return nil, fmt.Errorf("Failed parsing websocket URL for migration sink %q connection: %w", connName, err) } sink.conns[connName] = newMigrationConn(args.Secrets[connName], args.Dialer, u) } else { secret, err := internalUtil.RandomHexString(32) if err != nil { return nil, fmt.Errorf("Failed creating migration sink secret for %q connection: %w", connName, err) } sink.conns[connName] = newMigrationConn(secret, nil, nil) } } return &sink, nil } func (c *migrationSink) DoStorage(state *state.State, projectName string, poolName string, req *api.StorageVolumesPost, op *operations.Operation) error { l := logger.AddContext(logger.Ctx{"project": projectName, "pool": poolName, "volume": req.Name, "push": c.push}) ctx, cancel := context.WithTimeout(state.ShutdownCtx, time.Second*30) defer cancel() l.Info("Waiting for migration connections on target") for _, connName := range []string{api.SecretNameControl, api.SecretNameFilesystem} { _, err := c.conns[connName].WebSocket(ctx) if err != nil { return fmt.Errorf("Failed waiting for migration %q connection on target: %w", connName, err) } } l.Info("Migration channels connected on target") defer l.Info("Migration channels disconnected on target") if c.push { defer c.disconnect() } offerHeader := &migration.MigrationHeader{} err := c.recv(offerHeader, true) if err != nil { logger.Errorf("Failed to receive storage volume migration header") c.sendControl(err) return err } // The function that will be executed to receive the sender's migration data. var myTarget func(conn io.ReadWriteCloser, op *operations.Operation, args migrationSinkArgs) error pool, err := storagePools.LoadByName(state, poolName) if err != nil { return err } dbContentType, err := storagePools.VolumeContentTypeNameToContentType(req.ContentType) if err != nil { return err } contentType, err := storagePools.VolumeDBContentTypeToContentType(dbContentType) if err != nil { return err } // The source/sender will never set Refresh. However, to determine the correct migration type // Refresh needs to be set. offerHeader.Refresh = &c.refresh clusterMove := req.Source.Location != "" // Extract the source's migration type and then match it against our pool's // supported types and features. If a match is found the combined features list // will be sent back to requester. respTypes, err := localMigration.MatchTypes(offerHeader, storagePools.FallbackMigrationType(contentType), pool.MigrationTypes(contentType, c.refresh, !c.volumeOnly, clusterMove, poolName != "" && req.Source.Pool != poolName || !clusterMove)) if err != nil { return err } // The migration header to be sent back to source with our target options. // Convert response type to response header and copy snapshot info into it. respHeader := localMigration.TypesToHeader(respTypes...) // Respond with our maximum supported header version if the requested version is higher than ours. // Otherwise just return the requested header version to the source. indexHeaderVersion := min(offerHeader.GetIndexHeaderVersion(), localMigration.IndexHeaderVersion) respHeader.IndexHeaderVersion = &indexHeaderVersion respHeader.SnapshotNames = offerHeader.SnapshotNames respHeader.Snapshots = offerHeader.Snapshots respHeader.Refresh = &c.refresh respHeader.VolumeSize = offerHeader.VolumeSize // Translate the legacy MigrationSinkArgs to a VolumeTargetArgs suitable for use // with the new storage layer. myTarget = func(conn io.ReadWriteCloser, op *operations.Operation, args migrationSinkArgs) error { volTargetArgs := localMigration.VolumeTargetArgs{ IndexHeaderVersion: respHeader.GetIndexHeaderVersion(), Name: req.Name, Config: req.Config, Description: req.Description, MigrationType: respTypes[0], TrackProgress: true, ContentType: req.ContentType, Refresh: args.Refresh, RefreshExcludeOlder: args.RefreshExcludeOlder, VolumeSize: args.VolumeSize, VolumeOnly: args.VolumeOnly, } // A zero length Snapshots slice indicates volume only migration in // VolumeTargetArgs. So if VoluneOnly was requested, do not populate them. if !args.VolumeOnly { volTargetArgs.Snapshots = make([]*migration.Snapshot, 0, len(args.Snapshots)) for _, snap := range args.Snapshots { volTargetArgs.Snapshots = append(volTargetArgs.Snapshots, &migration.Snapshot{Name: snap.Name, LocalConfig: snap.LocalConfig}) } } return pool.CreateCustomVolumeFromMigration(projectName, conn, volTargetArgs, op) } if c.refresh { // Get the remote snapshots on the source. sourceSnapshots := offerHeader.GetSnapshots() sourceSnapshotComparable := make([]storagePools.ComparableSnapshot, 0, len(sourceSnapshots)) for _, sourceSnap := range sourceSnapshots { sourceSnapshotComparable = append(sourceSnapshotComparable, storagePools.ComparableSnapshot{ Name: sourceSnap.GetName(), CreationDate: time.Unix(0, sourceSnap.GetCreationDate()), }) } // Get existing snapshots on the local target. targetSnapshots, err := storagePools.VolumeDBSnapshotsGet(pool, projectName, req.Name, storageDrivers.VolumeTypeCustom) if err != nil { c.sendControl(err) return err } targetSnapshotsComparable := make([]storagePools.ComparableSnapshot, 0, len(targetSnapshots)) for _, targetSnap := range targetSnapshots { _, targetSnapName, _ := api.GetParentAndSnapshotName(targetSnap.Name) targetSnapshotsComparable = append(targetSnapshotsComparable, storagePools.ComparableSnapshot{ Name: targetSnapName, CreationDate: targetSnap.CreationDate, }) } // Compare the two sets. syncSourceSnapshotIndexes, deleteTargetSnapshotIndexes := storagePools.CompareSnapshots(sourceSnapshotComparable, targetSnapshotsComparable, c.refreshExcludeOlder) // Delete the extra local snapshots first. for _, deleteTargetSnapshotIndex := range deleteTargetSnapshotIndexes { err := pool.DeleteCustomVolumeSnapshot(projectName, targetSnapshots[deleteTargetSnapshotIndex].Name, op) if err != nil { c.sendControl(err) return err } } // Only request to send the snapshots that need updating. syncSnapshotNames := make([]string, 0, len(syncSourceSnapshotIndexes)) syncSnapshots := make([]*migration.Snapshot, 0, len(syncSourceSnapshotIndexes)) for _, syncSourceSnapshotIndex := range syncSourceSnapshotIndexes { syncSnapshotNames = append(syncSnapshotNames, sourceSnapshots[syncSourceSnapshotIndex].GetName()) syncSnapshots = append(syncSnapshots, sourceSnapshots[syncSourceSnapshotIndex]) } respHeader.Snapshots = syncSnapshots respHeader.SnapshotNames = syncSnapshotNames offerHeader.Snapshots = syncSnapshots offerHeader.SnapshotNames = syncSnapshotNames } err = c.send(respHeader) if err != nil { logger.Errorf("Failed to send storage volume migration header") c.sendControl(err) return err } restore := make(chan error) go func(c *migrationSink) { // We do the fs receive in parallel so we don't have to reason about when to receive // what. The sending side is smart enough to send the filesystem bits that it can // before it seizes the container to start checkpointing, so the total transfer time // will be minimized even if we're dumb here. fsTransfer := make(chan error) go func() { // Get rsync options from sender, these are passed into mySink function // as part of MigrationSinkArgs below. rsyncFeatures := respHeader.GetRsyncFeaturesSlice() args := migrationSinkArgs{ RsyncFeatures: rsyncFeatures, Snapshots: respHeader.Snapshots, VolumeOnly: c.volumeOnly, VolumeSize: respHeader.GetVolumeSize(), Refresh: c.refresh, RefreshExcludeOlder: c.refreshExcludeOlder, } fsConn, err := c.conns[api.SecretNameFilesystem].WebsocketIO(state.ShutdownCtx) if err != nil { fsTransfer <- err return } err = myTarget(fsConn, op, args) if err != nil { fsTransfer <- err return } fsTransfer <- nil }() err := <-fsTransfer if err != nil { restore <- err return } restore <- nil }(c) for { select { case err = <-restore: if err != nil { c.disconnect() return err } c.sendControl(nil) logger.Debug("Migration sink finished receiving storage volume") return nil case msg := <-c.controlChannel(): if msg.Err != nil { c.disconnect() return fmt.Errorf("Got error reading migration source: %w", msg.Err) } if !msg.GetSuccess() { c.disconnect() return errors.New(msg.GetMessage()) } // The source can only tell us it failed (e.g. if // checkpointing failed). We have to tell the source // whether or not the restore was successful. logger.Warn("Unknown message from migration source", logger.Ctx{"message": msg.GetMessage()}) } } } incus-7.0.0/cmd/incusd/migration_connection.go000066400000000000000000000104521517523235500214400ustar00rootroot00000000000000package main import ( "context" "crypto/x509" "encoding/pem" "errors" "fmt" "io" "net/http" "net/url" "sync" "time" "github.com/gorilla/websocket" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/tcp" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/ws" ) // setupWebsocketDialer uses a certificate to parse and configure a websocket.Dialer. func setupWebsocketDialer(certificate string) (*websocket.Dialer, error) { var err error var cert *x509.Certificate if certificate != "" { certBlock, _ := pem.Decode([]byte(certificate)) if certBlock == nil { return nil, errors.New("Failed PEM decoding certificate") } cert, err = x509.ParseCertificate(certBlock.Bytes) if err != nil { return nil, fmt.Errorf("Failed parsing certificate: %w", err) } } config, err := localtls.GetTLSConfig(cert) if err != nil { return nil, fmt.Errorf("Failed configuring TLS: %w", err) } dialer := &websocket.Dialer{ TLSClientConfig: config, NetDialContext: localtls.RFC3493Dialer, HandshakeTimeout: time.Second * 5, } return dialer, nil } // newMigrationConn configures a new migration connection handler. func newMigrationConn(secret string, outgoingDialer *websocket.Dialer, outgoingURL *url.URL) *migrationConn { return &migrationConn{ secret: secret, outgoingDialer: outgoingDialer, outgoingURL: outgoingURL, connected: make(chan struct{}), } } // migrationConn represents a handler for both accepting and making new migration connections. type migrationConn struct { mu sync.Mutex secret string outgoingDialer *websocket.Dialer outgoingURL *url.URL conn *websocket.Conn connected chan struct{} disconnected bool } // Secret returns the secret for this connection. func (c *migrationConn) Secret() string { return c.secret } // AcceptIncoming takes an incoming HTTP request and upgrades it to a websocket. func (c *migrationConn) AcceptIncoming(r *http.Request, w http.ResponseWriter) error { c.mu.Lock() defer c.mu.Unlock() if c.disconnected { return errors.New("Connection already disconnected") } if c.conn != nil { return api.StatusErrorf(http.StatusConflict, "Connection already established") } var err error c.conn, err = ws.Upgrader.Upgrade(w, r, nil) if err != nil { return fmt.Errorf("Failed upgrading incoming request to websocket: %w", err) } // Set TCP timeout options. remoteTCP, _ := tcp.ExtractConn(c.conn.UnderlyingConn()) if remoteTCP != nil { err = tcp.SetTimeouts(remoteTCP, 0) if err != nil { logger.Warn("Failed setting TCP timeouts on incoming websocket connection", logger.Ctx{"err": err}) } } close(c.connected) return nil } // WebSocket returns the underlying websocket connection. // If the connection isn't yet active it will either wait for an incoming connection or if configured, will attempt // to initiate a new outbound connection. If the context is cancelled before the connection is established it // will return with an error. func (c *migrationConn) WebSocket(ctx context.Context) (*websocket.Conn, error) { c.mu.Lock() if c.disconnected { c.mu.Unlock() return nil, errors.New("Connection already disconnected") } if c.conn != nil { c.mu.Unlock() return c.conn, nil } if c.outgoingURL != nil && c.outgoingDialer != nil { var err error q := c.outgoingURL.Query() q.Set("secret", c.secret) c.outgoingURL.RawQuery = q.Encode() c.conn, _, err = c.outgoingDialer.DialContext(ctx, c.outgoingURL.String(), http.Header{}) if err != nil { c.mu.Unlock() return nil, err } c.mu.Unlock() return c.conn, nil } c.mu.Unlock() select { case <-c.connected: return c.conn, nil case <-ctx.Done(): return nil, ctx.Err() } } // WebsocketIO calls WebSocket and returns it wrapped for io.ReadWriteCloser compatibility. func (c *migrationConn) WebsocketIO(ctx context.Context) (io.ReadWriteCloser, error) { wsConn, err := c.WebSocket(ctx) if err != nil { return nil, err } return ws.NewWrapper(wsConn), nil } // Close closes the connection (if established) and marks it as disconnected so that it cannot be used again. func (c *migrationConn) Close() { c.mu.Lock() defer c.mu.Unlock() c.disconnected = true if c.conn != nil { c.conn.Close() c.conn = nil } } incus-7.0.0/cmd/incusd/network_acls.go000066400000000000000000000464531517523235500177350ustar00rootroot00000000000000package main import ( "bytes" "context" "encoding/json" "errors" "fmt" "net/http" "net/url" "time" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/server/auth" clusterRequest "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/network/acl" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) var networkACLsCmd = APIEndpoint{ Path: "network-acls", Get: APIEndpointAction{Handler: networkACLsGet, AccessHandler: allowAuthenticated}, Post: APIEndpointAction{Handler: networkACLsPost, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanCreateNetworkACLs)}, } var networkACLCmd = APIEndpoint{ Path: "network-acls/{name}", Delete: APIEndpointAction{Handler: networkACLDelete, AccessHandler: allowPermission(auth.ObjectTypeNetworkACL, auth.EntitlementCanEdit, "name")}, Get: APIEndpointAction{Handler: networkACLGet, AccessHandler: allowPermission(auth.ObjectTypeNetworkACL, auth.EntitlementCanView, "name")}, Put: APIEndpointAction{Handler: networkACLPut, AccessHandler: allowPermission(auth.ObjectTypeNetworkACL, auth.EntitlementCanEdit, "name")}, Patch: APIEndpointAction{Handler: networkACLPut, AccessHandler: allowPermission(auth.ObjectTypeNetworkACL, auth.EntitlementCanEdit, "name")}, Post: APIEndpointAction{Handler: networkACLPost, AccessHandler: allowPermission(auth.ObjectTypeNetworkACL, auth.EntitlementCanEdit, "name")}, } var networkACLLogCmd = APIEndpoint{ Path: "network-acls/{name}/log", Get: APIEndpointAction{Handler: networkACLLogGet, AccessHandler: allowPermission(auth.ObjectTypeNetworkACL, auth.EntitlementCanView, "name")}, } // API endpoints. // swagger:operation GET /1.0/network-acls network-acls network_acls_get // // Get the network ACLs // // Returns a list of network ACLs (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve network ACLs from all projects // type: boolean // example: true // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/network-acls/foo", // "/1.0/network-acls/bar" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/network-acls?recursion=1 network-acls network_acls_get_recursion1 // // Get the network ACLs // // Returns a list of network ACLs (structs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve network ACLs from all projects // type: boolean // example: true // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of network ACLs // items: // $ref: "#/definitions/NetworkACL" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkACLsGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } recursion := localUtil.IsRecursionRequest(r) allProjects := util.IsTrue(r.FormValue("all-projects")) // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } mustLoadObjects := recursion || (clauses != nil && len(clauses.Clauses) > 0) aclNames := map[string][]string{} err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var acls []dbCluster.NetworkACL // Get list of Network ACLs. if allProjects { acls, err = dbCluster.GetNetworkACLs(ctx, tx.Tx()) if err != nil { return err } } else { acls, err = dbCluster.GetNetworkACLs(ctx, tx.Tx(), dbCluster.NetworkACLFilter{Project: &projectName}) if err != nil { return err } } for _, acl := range acls { if aclNames[acl.Project] == nil { aclNames[acl.Project] = []string{} } aclNames[acl.Project] = append(aclNames[acl.Project], acl.Name) } return nil }) if err != nil { return response.InternalError(err) } userHasPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanView, auth.ObjectTypeNetworkACL) if err != nil { return response.SmartError(err) } linkResults := make([]string, 0) fullResults := make([]api.NetworkACL, 0) for projectName, acls := range aclNames { for _, aclName := range acls { if !userHasPermission(auth.ObjectNetworkACL(projectName, aclName)) { continue } if mustLoadObjects { netACL, err := acl.LoadByName(s, projectName, aclName) if err != nil { continue } netACLInfo := netACL.Info() netACLInfo.UsedBy, _ = netACL.UsedBy() // Ignore errors in UsedBy, will return nil. if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(*netACLInfo, *clauses) if err != nil { return response.SmartError(err) } if !match { continue } } fullResults = append(fullResults, *netACLInfo) } linkResults = append(linkResults, fmt.Sprintf("/%s/network-acls/%s", version.APIVersion, aclName)) } } if !recursion { return response.SyncResponse(true, linkResults) } return response.SyncResponse(true, fullResults) } // swagger:operation POST /1.0/network-acls network-acls network_acls_post // // Add a network ACL // // Creates a new network ACL. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: acl // description: ACL // required: true // schema: // $ref: "#/definitions/NetworkACLsPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkACLsPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } req := api.NetworkACLsPost{} // Parse the request into a record. err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } _, err = acl.LoadByName(s, projectName, req.Name) if err == nil { return response.BadRequest(errors.New("The network ACL already exists")) } err = acl.Create(s, projectName, &req) if err != nil { return response.SmartError(err) } netACL, err := acl.LoadByName(s, projectName, req.Name) if err != nil { return response.BadRequest(err) } err = s.Authorizer.AddNetworkACL(r.Context(), projectName, req.Name) if err != nil { logger.Error("Failed to add network ACL to authorizer", logger.Ctx{"name": req.Name, "project": projectName, "error": err}) } lc := lifecycle.NetworkACLCreated.Event(netACL, request.CreateRequestor(r), nil) s.Events.SendLifecycle(projectName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation DELETE /1.0/network-acls/{name} network-acls network_acl_delete // // Delete the network ACL // // Removes the network ACL. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: ACL name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkACLDelete(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } aclName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } netACL, err := acl.LoadByName(s, projectName, aclName) if err != nil { return response.SmartError(err) } err = netACL.Delete() if err != nil { return response.SmartError(err) } err = s.Authorizer.DeleteNetworkACL(r.Context(), projectName, aclName) if err != nil { logger.Error("Failed to remove network ACL from authorizer", logger.Ctx{"name": aclName, "project": projectName, "error": err}) } s.Events.SendLifecycle(projectName, lifecycle.NetworkACLDeleted.Event(netACL, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation GET /1.0/network-acls/{name} network-acls network_acl_get // // Get the network ACL // // Gets a specific network ACL. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: ACL name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: ACL // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/NetworkACL" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkACLGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } aclName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } netACL, err := acl.LoadByName(s, projectName, aclName) if err != nil { return response.SmartError(err) } info := netACL.Info() info.UsedBy, err = netACL.UsedBy() if err != nil { return response.SmartError(err) } return response.SyncResponseETag(true, info, netACL.Etag()) } // swagger:operation PATCH /1.0/network-acls/{name} network-acls network_acl_patch // // Partially update the network ACL // // Updates a subset of the network ACL configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: ACL name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: acl // description: ACL configuration // required: true // schema: // $ref: "#/definitions/NetworkACLPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation PUT /1.0/network-acls/{name} network-acls network_acl_put // // Update the network ACL // // Updates the entire network ACL configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: ACL name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: acl // description: ACL configuration // required: true // schema: // $ref: "#/definitions/NetworkACLPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func networkACLPut(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } aclName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Get the existing Network ACL. netACL, err := acl.LoadByName(s, projectName, aclName) if err != nil { return response.SmartError(err) } // Validate the ETag. err = localUtil.EtagCheck(r, netACL.Etag()) if err != nil { return response.PreconditionFailed(err) } req := api.NetworkACLPut{} // Decode the request. err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } if r.Method == http.MethodPatch { // If config being updated via "patch" method, then merge all existing config with the keys that // are present in the request config. for k, v := range netACL.Info().Config { _, ok := req.Config[k] if !ok { req.Config[k] = v } } } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) err = netACL.Update(&req, clientType) if err != nil { return response.SmartError(err) } s.Events.SendLifecycle(projectName, lifecycle.NetworkACLUpdated.Event(netACL, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation POST /1.0/network-acls/{name} network-acls network_acl_post // // Rename the network ACL // // Renames an existing network ACL. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: ACL name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: acl // description: ACL rename request // required: true // schema: // $ref: "#/definitions/NetworkACLPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkACLPost(d *Daemon, r *http.Request) response.Response { s := d.State() aclName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } projectName, _, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } req := api.NetworkACLPost{} // Parse the request. err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Get the existing Network ACL. netACL, err := acl.LoadByName(s, projectName, aclName) if err != nil { return response.SmartError(err) } err = netACL.Rename(req.Name) if err != nil { return response.SmartError(err) } err = s.Authorizer.RenameNetworkACL(r.Context(), projectName, aclName, req.Name) if err != nil { logger.Error("Failed to rename network ACL in authorizer", logger.Ctx{"old_name": aclName, "new_name": req.Name, "project": projectName, "error": err}) } lc := lifecycle.NetworkACLRenamed.Event(netACL, request.CreateRequestor(r), logger.Ctx{"old_name": aclName}) s.Events.SendLifecycle(projectName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation GET /1.0/network-acls/{name}/log network-acls network_acl_log_get // // Get the network ACL log // // Gets a specific network ACL log entries. // // --- // produces: // - application/octet-stream // parameters: // - in: path // name: name // description: ACL name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Raw log file // content: // application/octet-stream: // schema: // type: string // example: LOG-ENTRY // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkACLLogGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } aclName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } netACL, err := acl.LoadByName(s, projectName, aclName) if err != nil { return response.SmartError(err) } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) log, err := netACL.GetLog(clientType) if err != nil { return response.SmartError(err) } ent := response.FileResponseEntry{} ent.File = bytes.NewReader([]byte(log)) ent.FileModified = time.Now() ent.FileSize = int64(len(log)) return response.FileResponse(r, []response.FileResponseEntry{ent}, nil) } incus-7.0.0/cmd/incusd/network_address_sets.go000066400000000000000000000443301517523235500214660ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/url" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/server/auth" clusterRequest "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/lifecycle" addressset "github.com/lxc/incus/v7/internal/server/network/address-set" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) var networkAddressSetsCmd = APIEndpoint{ Path: "network-address-sets", Get: APIEndpointAction{Handler: networkAddressSetsGet, AccessHandler: allowAuthenticated}, Post: APIEndpointAction{Handler: networkAddressSetsPost, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanCreateNetworkAddressSets)}, } var networkAddressSetCmd = APIEndpoint{ Path: "network-address-sets/{name}", Delete: APIEndpointAction{Handler: networkAddressSetDelete, AccessHandler: allowPermission(auth.ObjectTypeNetworkAddressSet, auth.EntitlementCanEdit, "name")}, Get: APIEndpointAction{Handler: networkAddressSetGet, AccessHandler: allowPermission(auth.ObjectTypeNetworkAddressSet, auth.EntitlementCanView, "name")}, Put: APIEndpointAction{Handler: networkAddressSetPut, AccessHandler: allowPermission(auth.ObjectTypeNetworkAddressSet, auth.EntitlementCanEdit, "name")}, Patch: APIEndpointAction{Handler: networkAddressSetPut, AccessHandler: allowPermission(auth.ObjectTypeNetworkAddressSet, auth.EntitlementCanEdit, "name")}, Post: APIEndpointAction{Handler: networkAddressSetPost, AccessHandler: allowPermission(auth.ObjectTypeNetworkAddressSet, auth.EntitlementCanEdit, "name")}, } // API endpoints. // swagger:operation GET /1.0/network-address-sets network-address-sets network_address_sets_get // // Get the network address sets // // Returns a list of network address sets (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve network address sets from all projects // type: boolean // example: true // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/network-address-sets/foo", // "/1.0/network-address-sets/bar" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/network-address-sets?recursion=1 network-address-sets network_address_sets_get_recursion1 // // # Get the network address sets // // Returns a list of network address sets (structs). // // --- // produces: // - application/json // // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve network address sets from all projects // type: boolean // example: true // - in: query // name: filter // description: Collection filter // type: string // example: default // // responses: // // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of network address sets // items: // $ref: "#/definitions/NetworkAddressSet" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkAddressSetsGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } recursion := localUtil.IsRecursionRequest(r) allProjects := util.IsTrue(r.FormValue("all-projects")) filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } mustLoadObjects := recursion || (clauses != nil && len(clauses.Clauses) > 0) var addrSets []dbCluster.NetworkAddressSet err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error if allProjects { addrSets, err = dbCluster.GetNetworkAddressSets(ctx, tx.Tx()) if err != nil { return err } } else { addrSets, err = dbCluster.GetNetworkAddressSets(ctx, tx.Tx(), dbCluster.NetworkAddressSetFilter{Project: &projectName}) if err != nil { return err } } return nil }) if err != nil { return response.InternalError(err) } userHasPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanView, auth.ObjectTypeNetworkAddressSet) if err != nil { return response.SmartError(err) } linkResults := make([]string, 0) fullResults := make([]api.NetworkAddressSet, 0) for _, addrSet := range addrSets { if !userHasPermission(auth.ObjectNetworkAddressSet(addrSet.Project, addrSet.Name)) { continue } if mustLoadObjects { netAddressSet, err := addressset.LoadByName(s, addrSet.Project, addrSet.Name) if err != nil { continue } netAddressSetInfo := netAddressSet.Info() netAddressSetInfo.UsedBy, _ = netAddressSet.UsedBy() // Ignore errors in UsedBy, will return nil. if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(*netAddressSetInfo, *clauses) if err != nil { return response.SmartError(err) } if !match { continue } } fullResults = append(fullResults, *netAddressSetInfo) } linkResults = append(linkResults, fmt.Sprintf("/%s/network-address-sets/%s", version.APIVersion, addrSet.Name)) } if !recursion { return response.SyncResponse(true, linkResults) } return response.SyncResponse(true, fullResults) } // swagger:operation POST /1.0/network-address-sets network-address-sets network_address_sets_post // // Add a network address set // // Creates a new network address set. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: address set // description: address set // required: true // schema: // $ref: "#/definitions/NetworkAddressSetsPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkAddressSetsPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } req := api.NetworkAddressSetsPost{} // Parse the request into a record. err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } _, err = addressset.LoadByName(s, projectName, req.Name) if err == nil { return response.BadRequest(errors.New("The network address set already exists")) } err = addressset.Create(s, projectName, &req) if err != nil { return response.SmartError(err) } netAddrSet, err := addressset.LoadByName(s, projectName, req.Name) if err != nil { return response.BadRequest(err) } err = s.Authorizer.AddNetworkAddressSet(r.Context(), projectName, req.Name) if err != nil { logger.Error("Failed to add network address set to authorizer", logger.Ctx{"name": req.Name, "project": projectName, "error": err}) } lc := lifecycle.NetworkAddressSetCreated.Event(netAddrSet, request.CreateRequestor(r), nil) s.Events.SendLifecycle(projectName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation DELETE /1.0/network-address-sets/{name} network-address-sets network_address_set_delete // // Delete the network address set // // Removes the network address set. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Address set name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkAddressSetDelete(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } addrSetName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } netAddrSet, err := addressset.LoadByName(s, projectName, addrSetName) if err != nil { return response.SmartError(err) } err = netAddrSet.Delete() if err != nil { return response.SmartError(err) } err = s.Authorizer.DeleteNetworkAddressSet(r.Context(), projectName, addrSetName) if err != nil { logger.Error("Failed to remove network address set from authorizer", logger.Ctx{"name": addrSetName, "project": projectName, "error": err}) } s.Events.SendLifecycle(projectName, lifecycle.NetworkAddressSetDeleted.Event(netAddrSet, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation GET /1.0/network-address-sets/{name} network-address-sets network_address_set_get // // Get the network address set // // Gets a specific network address set. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Address set name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: address set // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/NetworkAddressSet" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkAddressSetGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } addrSetName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } netAddrSet, err := addressset.LoadByName(s, projectName, addrSetName) if err != nil { return response.SmartError(err) } info := netAddrSet.Info() info.UsedBy, err = netAddrSet.UsedBy() if err != nil { return response.SmartError(err) } return response.SyncResponseETag(true, info, netAddrSet.Etag()) } // swagger:operation PATCH /1.0/network-address-sets/{name} network-address-sets network_address_set_patch // // Partially update the network address set // // Updates a subset of the network address set configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Address set name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: address set // description: Address set configuration // required: true // schema: // $ref: "#/definitions/NetworkAddressSetPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation PUT /1.0/network-address-sets/{name} network-address-sets network_address_set_put // // Update the network address set // // Updates the entire network address set configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Address set name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: address set // description: Address set configuration // required: true // schema: // $ref: "#/definitions/NetworkAddressSetPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func networkAddressSetPut(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } addrSetName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Get the existing network address set. netAddrSet, err := addressset.LoadByName(s, projectName, addrSetName) if err != nil { return response.SmartError(err) } // Validate ETag. err = localUtil.EtagCheck(r, netAddrSet.Etag()) if err != nil { return response.PreconditionFailed(err) } req := api.NetworkAddressSetPut{} // Decode the request. err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } if r.Method == http.MethodPatch { for k, v := range netAddrSet.Info().Config { _, ok := req.Config[k] if !ok { req.Config[k] = v } } } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) err = netAddrSet.Update(&req, clientType) if err != nil { return response.SmartError(err) } s.Events.SendLifecycle(projectName, lifecycle.NetworkAddressSetUpdated.Event(netAddrSet, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation POST /1.0/network-address-sets/{name} network-address-sets network_address_set_post // // Rename the network address set // // Renames an existing network address set. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Address set name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: address set // description: Address set rename request // required: true // schema: // $ref: "#/definitions/NetworkAddressSetPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkAddressSetPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } addrSetName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Parse the request. req := api.NetworkAddressSetPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Get the existing network address set. netAddrSet, err := addressset.LoadByName(s, projectName, addrSetName) if err != nil { return response.SmartError(err) } oldName := addrSetName err = netAddrSet.Rename(req.Name) if err != nil { return response.SmartError(err) } err = s.Authorizer.RenameNetworkAddressSet(r.Context(), projectName, oldName, req.Name) if err != nil { logger.Error("Failed to rename network address set in authorizer", logger.Ctx{"old_name": oldName, "new_name": req.Name, "project": projectName, "error": err}) } lc := lifecycle.NetworkAddressSetRenamed.Event(netAddrSet, request.CreateRequestor(r), logger.Ctx{"old_name": oldName}) s.Events.SendLifecycle(projectName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } incus-7.0.0/cmd/incusd/network_allocations.go000066400000000000000000000200471517523235500213120ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "net" "net/http" "slices" clusterRequest "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) var networkAllocationsCmd = APIEndpoint{ Path: "network-allocations", Get: APIEndpointAction{Handler: networkAllocationsGet, AccessHandler: allowAuthenticated}, } // swagger:operation GET /1.0/network-allocations network-allocations network_allocations_get // // Get the network allocations in use (`network`, `network-forward` and `load-balancer` and `instance`) // // Returns a list of network allocations. // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve entities from all projects // type: boolean // responses: // "200": // description: API endpoints // schema: // type: object // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of network allocations used by a consuming entity // items: // $ref: "#/definitions/NetworkAllocations" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkAllocationsGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, reqProject, err := project.NetworkProject(d.State().DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } allProjects := util.IsTrue(request.QueryParam(r, "all-projects")) var projectNames []string if allProjects { err = d.db.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Get all project names if no specific project requested. projectNames, err = dbCluster.GetProjectNames(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading projects: %w", err) } return nil }) if err != nil { return response.SmartError(err) } } else { projectNames = []string{projectName} } // Returns IP address in its canonical CIDR form and whether the network is using NAT for that IP family. ipToCIDR := func(addr string, netConf map[string]string) (string, bool, error) { ip := net.ParseIP(addr) if ip == nil { return "", false, fmt.Errorf("Invalid IP address %q", addr) } if ip.To4() != nil { return fmt.Sprintf("%s/32", ip.String()), util.IsTrue(netConf["ipv4.nat"]), nil } return fmt.Sprintf("%s/128", ip.String()), util.IsTrue(netConf["ipv6.nat"]), nil } result := make([]api.NetworkAllocations, 0) // Then, get all the networks, their network forwards and their network load balancers. for _, projectName := range projectNames { var networkNames []string err := d.db.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error networkNames, err = tx.GetNetworks(ctx, projectName) return err }) if err != nil { return response.SmartError(fmt.Errorf("Failed loading networks: %w", err)) } // Get all the networks, their attached instances, their network forwards and their network load balancers. for _, networkName := range networkNames { ok, err := canAccessNetwork(s, r, projectName, reqProject.Config, networkName, true) if err != nil { return response.SmartError(err) } if !ok { continue } n, err := network.LoadByName(d.State(), projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network %q in project %q: %w", networkName, projectName, err)) } netConf := n.Config() for _, keyPrefix := range []string{"ipv4", "ipv6"} { ipNet, _ := network.ParseIPCIDRToNet(netConf[fmt.Sprintf("%s.address", keyPrefix)]) if ipNet == nil { continue } result = append(result, api.NetworkAllocations{ Address: ipNet.String(), UsedBy: api.NewURL().Path(version.APIVersion, "networks", networkName).Project(projectName).String(), Type: "network", NAT: util.IsTrue(netConf[fmt.Sprintf("%s.nat", keyPrefix)]), }) } leases, err := n.Leases(projectName, clusterRequest.ClientTypeNormal) if err != nil && !errors.Is(network.ErrNotImplemented, err) { return response.SmartError(fmt.Errorf("Failed getting leases for network %q in project %q: %w", networkName, projectName, err)) } for _, lease := range leases { if slices.Contains([]string{"static", "dynamic"}, lease.Type) { cidrAddr, nat, err := ipToCIDR(lease.Address, netConf) if err != nil { return response.SmartError(err) } result = append(result, api.NetworkAllocations{ Address: cidrAddr, UsedBy: api.NewURL().Path(version.APIVersion, "instances", lease.Hostname).Project(projectName).String(), Type: "instance", Hwaddr: lease.Hwaddr, NAT: nat, }) } } var forwards map[int64]*api.NetworkForward err = d.db.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() dbRecords, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, }) if err != nil { return err } forwards = make(map[int64]*api.NetworkForward) for _, dbRecord := range dbRecords { forward, err := dbRecord.ToAPI(ctx, tx.Tx()) if err != nil { return err } forwards[dbRecord.ID] = forward } return nil }) if err != nil { return response.SmartError(fmt.Errorf("Failed getting forwards for network %q in project %q: %w", networkName, projectName, err)) } for _, forward := range forwards { cidrAddr, _, err := ipToCIDR(forward.ListenAddress, netConf) if err != nil { return response.SmartError(err) } result = append( result, api.NetworkAllocations{ Address: cidrAddr, UsedBy: api.NewURL().Path(version.APIVersion, "networks", networkName, "forwards", forward.ListenAddress).Project(projectName).String(), Type: "network-forward", NAT: false, // Network forwards are ingress and so aren't affected by SNAT. }, ) } var dbLoadBalancers []dbCluster.NetworkLoadBalancer err = d.db.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() // Get the load balancers. dbLoadBalancers, err = dbCluster.GetNetworkLoadBalancers(ctx, tx.Tx(), dbCluster.NetworkLoadBalancerFilter{NetworkID: &networkID}) if err != nil { return err } return nil }) if err != nil { return response.SmartError(fmt.Errorf("Failed getting load-balancers for network %q in project %q: %w", networkName, projectName, err)) } for _, loadBalancer := range dbLoadBalancers { cidrAddr, _, err := ipToCIDR(loadBalancer.ListenAddress, netConf) if err != nil { return response.SmartError(err) } result = append( result, api.NetworkAllocations{ Address: cidrAddr, UsedBy: api.NewURL().Path(version.APIVersion, "networks", networkName, "load-balancers", loadBalancer.ListenAddress).Project(projectName).String(), Type: "network-load-balancer", NAT: false, // Network load-balancers are ingress and so aren't affected by SNAT. }, ) } } } return response.SyncResponse(true, result) } incus-7.0.0/cmd/incusd/network_forwards.go000066400000000000000000000544501517523235500206360ustar00rootroot00000000000000package main import ( "context" "encoding/json" "fmt" "net/http" "net/url" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/server/auth" clusterRequest "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) var networkForwardsCmd = APIEndpoint{ Path: "networks/{networkName}/forwards", Get: APIEndpointAction{Handler: networkForwardsGet, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanView, "networkName")}, Post: APIEndpointAction{Handler: networkForwardsPost, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, } var networkForwardCmd = APIEndpoint{ Path: "networks/{networkName}/forwards/{listenAddress}", Delete: APIEndpointAction{Handler: networkForwardDelete, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, Get: APIEndpointAction{Handler: networkForwardGet, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanView, "networkName")}, Put: APIEndpointAction{Handler: networkForwardPut, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, Patch: APIEndpointAction{Handler: networkForwardPut, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, } // API endpoints // swagger:operation GET /1.0/networks/{networkName}/forwards network-forwards network_forwards_get // // Get the network address forwards // // Returns a list of network address forwards (URLs). // // --- // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/networks/mybr0/forwards/192.0.2.1", // "/1.0/networks/mybr0/forwards/192.0.2.2" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/networks/{networkName}/forwards?recursion=1 network-forwards network_forward_get_recursion1 // // Get the network address forwards // // Returns a list of network address forwards (structs). // // --- // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of network address forwards // items: // $ref: "#/definitions/NetworkForward" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkForwardsGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().AddressForwards { return response.BadRequest(fmt.Errorf("Network driver %q does not support forwards", n.Type())) } recursion := localUtil.IsRecursionRequest(r) // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } mustLoadObjects := recursion || (clauses != nil && len(clauses.Clauses) > 0) linkResults := make([]string, 0) fullResults := make([]api.NetworkForward, 0) if mustLoadObjects { var records map[int64]*api.NetworkForward err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() dbRecords, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, }) if err != nil { return err } records = make(map[int64]*api.NetworkForward) for _, dbRecord := range dbRecords { forward, err := dbRecord.ToAPI(ctx, tx.Tx()) if err != nil { return err } records[dbRecord.ID] = forward } return err }) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network forwards: %w", err)) } for _, record := range records { if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(*record, *clauses) if err != nil { return response.SmartError(err) } if !match { continue } } fullResults = append(fullResults, *record) linkResults = append(linkResults, fmt.Sprintf("/%s/networks/%s/forwards/%s", version.APIVersion, url.PathEscape(n.Name()), url.PathEscape(record.ListenAddress))) } } else { var listenAddresses map[int64]string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() dbRecords, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, }) if err != nil { return err } listenAddresses = make(map[int64]string) for _, dbRecord := range dbRecords { listenAddresses[dbRecord.ID] = dbRecord.ListenAddress } return err }) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network forwards: %w", err)) } for _, listenAddress := range listenAddresses { linkResults = append(linkResults, fmt.Sprintf("/%s/networks/%s/forwards/%s", version.APIVersion, url.PathEscape(n.Name()), url.PathEscape(listenAddress))) } } if recursion { return response.SyncResponse(true, fullResults) } return response.SyncResponse(true, linkResults) } // swagger:operation POST /1.0/networks/{networkName}/forwards network-forwards network_forwards_post // // Add a network address forward // // Creates a new network address forward. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: forward // description: Forward // required: true // schema: // $ref: "#/definitions/NetworkForwardsPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkForwardsPost(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } // Parse the request into a record. req := api.NetworkForwardsPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } req.Normalise() // So we handle the request in normalised/canonical form. networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().AddressForwards { return response.BadRequest(fmt.Errorf("Network driver %q does not support forwards", n.Type())) } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) err = n.ForwardCreate(req, clientType) if err != nil { return response.SmartError(fmt.Errorf("Failed creating forward: %w", err)) } lc := lifecycle.NetworkForwardCreated.Event(n, req.ListenAddress, request.CreateRequestor(r), nil) s.Events.SendLifecycle(projectName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation DELETE /1.0/networks/{networkName}/forwards/{listenAddress} network-forwards network_forward_delete // // Delete the network address forward // // Removes the network address forward. // // --- // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: path // name: listenAddress // description: Listen address // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkForwardDelete(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().AddressForwards { return response.BadRequest(fmt.Errorf("Network driver %q does not support forwards", n.Type())) } listenAddress, err := url.PathUnescape(mux.Vars(r)["listenAddress"]) if err != nil { return response.SmartError(err) } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) err = n.ForwardDelete(listenAddress, clientType) if err != nil { return response.SmartError(fmt.Errorf("Failed deleting forward: %w", err)) } s.Events.SendLifecycle(projectName, lifecycle.NetworkForwardDeleted.Event(n, listenAddress, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation GET /1.0/networks/{networkName}/forwards/{listenAddress} network-forwards network_forward_get // // Get the network address forward // // Gets a specific network address forward. // // --- // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: path // name: listenAddress // description: Listen address // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Address forward // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/NetworkForward" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkForwardGet(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().AddressForwards { return response.BadRequest(fmt.Errorf("Network driver %q does not support forwards", n.Type())) } listenAddress, err := url.PathUnescape(mux.Vars(r)["listenAddress"]) if err != nil { return response.SmartError(err) } targetMember := request.QueryParam(r, "target") memberSpecific := targetMember != "" var forward *api.NetworkForward err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() dbRecords, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, ListenAddress: &listenAddress, }) if err != nil { return err } filteredRecords := make([]dbCluster.NetworkForward, 0, len(dbRecords)) for _, dbRecord := range dbRecords { // Include all records if memberSpecific is turned off // Otherwise, filter based offed of dbRecords with same node id if !memberSpecific || (!dbRecord.NodeID.Valid || (dbRecord.NodeID.Int64 == tx.GetNodeID())) { filteredRecords = append(filteredRecords, dbRecord) } } if len(filteredRecords) == 0 { return api.StatusErrorf(http.StatusNotFound, "Network forward not found") } if len(filteredRecords) > 1 { return api.StatusErrorf(http.StatusConflict, "Network forward found on more than one cluster member. Please target a specific member") } dbNetworkForward := filteredRecords[0] forward, err = dbNetworkForward.ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } return response.SyncResponseETag(true, forward, forward.Etag()) } // swagger:operation PATCH /1.0/networks/{networkName}/forwards/{listenAddress} network-forwards network_forward_patch // // Partially update the network address forward // // Updates a subset of the network address forward configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: path // name: listenAddress // description: Listen address // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: forward // description: Address forward configuration // required: true // schema: // $ref: "#/definitions/NetworkForwardPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation PUT /1.0/networks/{networkName}/forwards/{listenAddress} network-forwards network_forward_put // // Update the network address forward // // Updates the entire network address forward configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: path // name: listenAddress // description: Listen address // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: forward // description: Address forward configuration // required: true // schema: // $ref: "#/definitions/NetworkForwardPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func networkForwardPut(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().AddressForwards { return response.BadRequest(fmt.Errorf("Network driver %q does not support forwards", n.Type())) } listenAddress, err := url.PathUnescape(mux.Vars(r)["listenAddress"]) if err != nil { return response.SmartError(err) } // Decode the request. req := api.NetworkForwardPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } targetMember := request.QueryParam(r, "target") memberSpecific := targetMember != "" if r.Method == http.MethodPatch { var forward *api.NetworkForward err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() dbRecords, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, ListenAddress: &listenAddress, }) if err != nil { return err } filteredRecords := make([]dbCluster.NetworkForward, 0, len(dbRecords)) for _, dbRecord := range dbRecords { // Include all records if memberSpecific is turned off // Otherwise, filter based offed of dbRecords with same node id if !memberSpecific || (!dbRecord.NodeID.Valid || (dbRecord.NodeID.Int64 == tx.GetNodeID())) { filteredRecords = append(filteredRecords, dbRecord) } } if len(filteredRecords) == 0 { return api.StatusErrorf(http.StatusNotFound, "Network forward not found") } if len(filteredRecords) > 1 { return api.StatusErrorf(http.StatusConflict, "Network forward found on more than one cluster member. Please target a specific member") } dbNetworkForward := filteredRecords[0] forward, err = dbNetworkForward.ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // If config being updated via "patch" method, then merge all existing config with the keys that // are present in the request config. for k, v := range forward.Config { _, ok := req.Config[k] if !ok { req.Config[k] = v } } // If forward being updated via "patch" method and ports not specified, then merge existing ports // into forward. if req.Ports == nil { req.Ports = forward.Ports } } req.Normalise() // So we handle the request in normalised/canonical form. clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) err = n.ForwardUpdate(listenAddress, req, clientType) if err != nil { return response.SmartError(fmt.Errorf("Failed updating forward: %w", err)) } s.Events.SendLifecycle(projectName, lifecycle.NetworkForwardUpdated.Event(n, listenAddress, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } incus-7.0.0/cmd/incusd/network_integrations.go000066400000000000000000000674251517523235500215230ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/url" "strings" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/validate" ) var networkIntegrationsCmd = APIEndpoint{ Path: "network-integrations", Get: APIEndpointAction{Handler: networkIntegrationsGet, AccessHandler: allowAuthenticated}, Post: APIEndpointAction{Handler: networkIntegrationsPost, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanCreateNetworkIntegrations)}, } var networkIntegrationCmd = APIEndpoint{ Path: "network-integrations/{integration}", Delete: APIEndpointAction{Handler: networkIntegrationDelete, AccessHandler: allowPermission(auth.ObjectTypeNetworkIntegration, auth.EntitlementCanEdit, "integration")}, Get: APIEndpointAction{Handler: networkIntegrationGet, AccessHandler: allowPermission(auth.ObjectTypeNetworkIntegration, auth.EntitlementCanView, "integration")}, Put: APIEndpointAction{Handler: networkIntegrationPut, AccessHandler: allowPermission(auth.ObjectTypeNetworkIntegration, auth.EntitlementCanEdit, "integration")}, Patch: APIEndpointAction{Handler: networkIntegrationPut, AccessHandler: allowPermission(auth.ObjectTypeNetworkIntegration, auth.EntitlementCanEdit, "integration")}, Post: APIEndpointAction{Handler: networkIntegrationPost, AccessHandler: allowPermission(auth.ObjectTypeNetworkIntegration, auth.EntitlementCanEdit, "integration")}, } // API endpoints. // swagger:operation GET /1.0/network-integrations network-integrations network_integrations_get // // Get the network integrations // // Returns a list of network integrations (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/network-integrations/region2", // "/1.0/network-integrations/region3" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/network-integrations?recursion=1 network-integrations network_integrations_get_recursion1 // // Get the network integrations // // Returns a list of network integrations (structs). // // --- // produces: // - application/json // parameters: // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of network integrations // items: // $ref: "#/definitions/NetworkIntegration" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkIntegrationsGet(d *Daemon, r *http.Request) response.Response { s := d.State() recursion := localUtil.IsRecursionRequest(r) // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } mustLoadObjects := recursion || (clauses != nil && len(clauses.Clauses) > 0) // Network integrations aren't project aware, we only load the per-project data to apply name restrictions. projectName := request.ProjectParam(r) // Get list of Network integrations. linkResults := make([]string, 0) fullResults := make([]api.NetworkIntegration, 0) err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Load the project. dbProject, err := dbCluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return fmt.Errorf("Failed to load network restrictions from project %q: %w", projectName, err) } p, err := dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed to load network restrictions from project %q: %w", projectName, err) } // Load the integrations. integrations, err := dbCluster.GetNetworkIntegrations(ctx, tx.Tx()) if err != nil { return err } for _, integration := range integrations { // Filter for project restrictions. if !project.NetworkIntegrationAllowed(p.Config, integration.Name) { continue } if mustLoadObjects { // Get the integration. result, err := integration.ToAPI(r.Context(), tx.Tx()) if err != nil { return err } // Check if the user should see the configuration. err = s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectNetworkIntegration(result.Name), auth.EntitlementCanEdit) if err != nil { result.Config = map[string]string{} } // Add UsedBy field. integrationID := integration.ID allPeers, err := dbCluster.GetNetworkPeers(ctx, tx.Tx()) // Fetch all peers if err != nil { return fmt.Errorf("Failed to load network peers: %w", err) } usedBy := []string{} for _, peer := range allPeers { if peer.TargetNetworkIntegrationID.Valid && peer.TargetNetworkIntegrationID.Int64 == int64(integrationID) { // Fetch the network associated with the peer networkName, networkProjectName, err := tx.GetNetworkNameAndProjectWithID(ctx, int(peer.NetworkID)) if err != nil { continue } _, network, _, err := tx.GetNetworkInAnyState(ctx, networkProjectName, networkName) if err != nil { continue } // Fetch the project associated with the network project, err := dbCluster.GetProject(ctx, tx.Tx(), networkProjectName) if err != nil { continue } // Construct the URL url := api.NewURL().Path(version.APIVersion, "networks", network.Name, "peers", peer.Name).Project(project.Name).String() usedBy = append(usedBy, url) } } // Assign the collected URLs (original 'err' check is implicitly handled by checks above) result.UsedBy = usedBy if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(*result, *clauses) if err != nil { return err } if !match { continue } } fullResults = append(fullResults, *result) } linkResults = append(linkResults, api.NewURL().Path(version.APIVersion, "network-integrations", integration.Name).String()) } return nil }) if err != nil { return response.InternalError(err) } if !recursion { return response.SyncResponse(true, linkResults) } return response.SyncResponse(true, fullResults) } // swagger:operation POST /1.0/network-integrations network-integrations network_integrations_post // // Add a network integration // // Creates a new network integration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: body // name: integration // description: integration // required: true // schema: // $ref: "#/definitions/NetworkIntegrationsPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkIntegrationsPost(d *Daemon, r *http.Request) response.Response { s := d.State() req := api.NetworkIntegrationsPost{} // Parse the request into a record. err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid network integration name: %w", err)) } // Validate the config. err = networkIntegrationValidate(req.Type, false, nil, req.Config) if err != nil { return response.BadRequest(err) } // Convert the API type to DB type. dbType := -1 for k, v := range dbCluster.NetworkIntegrationTypeNames { if v == req.Type { dbType = k break } } if dbType == -1 { return response.BadRequest(fmt.Errorf("Unsupported integration type %q", req.Type)) } // Create the DB record. err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbRecord := dbCluster.NetworkIntegration{ Name: req.Name, Description: req.Description, Type: dbType, } id, err := dbCluster.CreateNetworkIntegration(ctx, tx.Tx(), dbRecord) if err != nil { return err } err = dbCluster.CreateNetworkIntegrationConfig(ctx, tx.Tx(), id, req.Config) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // Add the integration to the auth backend. err = s.Authorizer.AddNetworkIntegration(r.Context(), req.Name) if err != nil { logger.Error("Failed to add network integration to authorizer", logger.Ctx{"name": req.Name, "error": err}) } // Emit the lifecycle event. lc := lifecycle.NetworkIntegrationCreated.Event(req.Name, request.CreateRequestor(r), nil) s.Events.SendLifecycle(api.ProjectDefaultName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation DELETE /1.0/network-integrations/{integration} network-integrations network_integration_delete // // Delete the network integration // // Removes the network integration. // // --- // produces: // - application/json // parameters: // - in: path // name: integration // description: Integration name // type: string // required: true // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkIntegrationDelete(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the integration name. integrationName, err := url.PathUnescape(mux.Vars(r)["integration"]) if err != nil { return response.SmartError(err) } // Delete the DB record. err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Get UsedBy for the integration. integrationID, err := dbCluster.GetNetworkIntegrationID(ctx, tx.Tx(), integrationName) if err != nil { return fmt.Errorf("Failed to get network integration ID: %w", err) } allPeers, err := dbCluster.GetNetworkPeers(ctx, tx.Tx()) // Fetch all peers if err != nil { return fmt.Errorf("Failed to load network peers: %w", err) } usedBy := []string{} for _, peer := range allPeers { if peer.TargetNetworkIntegrationID.Valid && peer.TargetNetworkIntegrationID.Int64 == int64(integrationID) { // Fetch the network associated with the peer networkName, networkProjectName, err := tx.GetNetworkNameAndProjectWithID(ctx, int(peer.NetworkID)) if err != nil { continue } _, network, _, err := tx.GetNetworkInAnyState(ctx, networkProjectName, networkName) if err != nil { continue } // Fetch the project associated with the network project, err := dbCluster.GetProject(ctx, tx.Tx(), networkProjectName) if err != nil { continue } // Construct the URL url := api.NewURL().Path(version.APIVersion, "networks", network.Name, "peers", peer.Name).Project(project.Name).String() usedBy = append(usedBy, url) } } if len(usedBy) > 0 { return errors.New("Network integration is currently in use") } err = dbCluster.DeleteNetworkIntegration(ctx, tx.Tx(), integrationName) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // Delete the integration from the auth backend. err = s.Authorizer.DeleteNetworkIntegration(r.Context(), integrationName) if err != nil { logger.Error("Failed to remove network integration from authorizer", logger.Ctx{"name": integrationName, "error": err}) } // Emit the lifecycle event. s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.NetworkIntegrationDeleted.Event(integrationName, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation GET /1.0/network-integrations/{integration} network-integrations network_integration_get // // Get the network integration // // Gets a specific network integration. // // --- // produces: // - application/json // parameters: // - in: path // name: integration // description: Integration name // type: string // required: true // responses: // "200": // description: integration // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/NetworkIntegration" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkIntegrationGet(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the integration name. integrationName, err := url.PathUnescape(mux.Vars(r)["integration"]) if err != nil { return response.SmartError(err) } // Network integrations aren't project aware, we only load the per-project data to apply name restrictions. projectName := request.ProjectParam(r) // Get the integration. var info *api.NetworkIntegration err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Get the project. dbProject, err := dbCluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return fmt.Errorf("Failed to load network restrictions from project %q: %w", projectName, err) } p, err := dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed to load network restrictions from project %q: %w", projectName, err) } // Filter for project restrictions. if !project.NetworkIntegrationAllowed(p.Config, integrationName) { return nil } // Get the integration. dbRecord, err := dbCluster.GetNetworkIntegration(ctx, tx.Tx(), integrationName) if err != nil { return err } info, err = dbRecord.ToAPI(ctx, tx.Tx()) if err != nil { return err } // Check if the user should see the configuration. err = s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectNetworkIntegration(info.Name), auth.EntitlementCanEdit) if err != nil { info.Config = map[string]string{} } // Add UsedBy field. integrationID := dbRecord.ID allPeers, err := dbCluster.GetNetworkPeers(ctx, tx.Tx()) // Fetch all peers if err != nil { return fmt.Errorf("Failed to load network peers: %w", err) } usedBy := []string{} for _, peer := range allPeers { if peer.TargetNetworkIntegrationID.Valid && peer.TargetNetworkIntegrationID.Int64 == int64(integrationID) { // Fetch the network associated with the peer networkName, networkProjectName, err := tx.GetNetworkNameAndProjectWithID(ctx, int(peer.NetworkID)) if err != nil { continue } _, network, _, err := tx.GetNetworkInAnyState(ctx, networkProjectName, networkName) if err != nil { continue } // Fetch the project associated with the network project, err := dbCluster.GetProject(ctx, tx.Tx(), networkProjectName) if err != nil { continue } // Construct the URL url := api.NewURL().Path(version.APIVersion, "networks", network.Name, "peers", peer.Name).Project(project.Name).String() usedBy = append(usedBy, url) } } // Assign the collected URLs (original 'err' check is implicitly handled by checks above) info.UsedBy = usedBy return nil }) if err != nil { return response.SmartError(err) } if info == nil { return response.NotFound(nil) } return response.SyncResponseETag(true, info, info.Writable()) } // swagger:operation PATCH /1.0/network-integrations/{integration} network-integrations network_integration_patch // // Partially update the network integration // // Updates a subset of the network integration configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: integration // description: Integration name // type: string // required: true // - in: body // name: integration // description: integration configuration // required: true // schema: // $ref: "#/definitions/NetworkIntegrationPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation PUT /1.0/network-integrations/{integration} network-integrations network_integration_put // // Update the network integration // // Updates the entire network integration configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: integration // description: Integration name // type: string // required: true // - in: body // name: integration // description: integration configuration // required: true // schema: // $ref: "#/definitions/NetworkIntegrationPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func networkIntegrationPut(d *Daemon, r *http.Request) response.Response { s := d.State() integrationName, err := url.PathUnescape(mux.Vars(r)["integration"]) if err != nil { return response.SmartError(err) } // Get the existing network integration. var dbRecord *dbCluster.NetworkIntegration var info *api.NetworkIntegration var usedBy []string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbRecord, err = dbCluster.GetNetworkIntegration(ctx, tx.Tx(), integrationName) if err != nil { return err } info, err = dbRecord.ToAPI(ctx, tx.Tx()) if err != nil { return err } integrationID := dbRecord.ID allPeers, err := dbCluster.GetNetworkPeers(ctx, tx.Tx()) // Fetch all peers if err != nil { return fmt.Errorf("Failed to load network peers: %w", err) } // Build usedBy slice in two passes to avoid SA4010. count := 0 for _, peer := range allPeers { if peer.TargetNetworkIntegrationID.Valid && peer.TargetNetworkIntegrationID.Int64 == int64(integrationID) { count++ } } usedBy = make([]string, count) idx := 0 for _, peer := range allPeers { if peer.TargetNetworkIntegrationID.Valid && peer.TargetNetworkIntegrationID.Int64 == int64(integrationID) { // Fetch the network associated with the peer networkName, networkProjectName, err := tx.GetNetworkNameAndProjectWithID(ctx, int(peer.NetworkID)) if err != nil { continue } _, network, _, err := tx.GetNetworkInAnyState(ctx, networkProjectName, networkName) if err != nil { continue } // Fetch the project associated with the network project, err := dbCluster.GetProject(ctx, tx.Tx(), networkProjectName) if err != nil { continue } // Construct the URL usedBy[idx] = api.NewURL().Path(version.APIVersion, "networks", network.Name, "peers", peer.Name).Project(project.Name).String() idx++ } } return nil }) if err != nil { return response.SmartError(err) } // Validate the ETag. err = localUtil.EtagCheck(r, info.Writable()) if err != nil { return response.PreconditionFailed(err) } // Decode the request. req := api.NetworkIntegrationPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } if r.Method == http.MethodPatch { // If config being updated via "patch" method, then merge all existing config with the keys that // are present in the request config. for k, v := range info.Config { _, ok := req.Config[k] if !ok { req.Config[k] = v } } if req.Description == "" { req.Description = info.Description } } // Validate the resulting config. err = networkIntegrationValidate(info.Type, len(usedBy) > 0, info.Config, req.Config) if err != nil { return response.BadRequest(err) } // Update the database record. err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Update the description if needed. if dbRecord.Description != req.Description { dbRecord.Description = req.Description err := dbCluster.UpdateNetworkIntegration(ctx, tx.Tx(), integrationName, *dbRecord) if err != nil { return err } } // Update the configuration. err := dbCluster.UpdateNetworkIntegrationConfig(ctx, tx.Tx(), int64(dbRecord.ID), req.Config) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // Emit the lifecycle event. s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.NetworkIntegrationUpdated.Event(integrationName, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation POST /1.0/network-integrations/{integration} network-integrations network_integration_post // // Rename the network integration // // Renames the network integration. // // --- // produces: // - application/json // parameters: // - in: path // name: integration // description: Integration name // type: string // required: true // - in: body // name: integration // description: integration configuration // required: true // schema: // $ref: "#/definitions/NetworkIntegrationPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkIntegrationPost(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the integration name. integrationName, err := url.PathUnescape(mux.Vars(r)["integration"]) if err != nil { return response.SmartError(err) } // Decode the request. req := api.NetworkIntegrationPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid network integration name: %w", err)) } // Rename the DB record. err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { err := dbCluster.RenameNetworkIntegration(ctx, tx.Tx(), integrationName, req.Name) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // Rename the integration in the auth backend. err = s.Authorizer.RenameNetworkIntegration(r.Context(), integrationName, req.Name) if err != nil { logger.Error("Failed to remove network integration from authorizer", logger.Ctx{"name": integrationName, "error": err}) } // Emit the lifecycle event. s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.NetworkIntegrationDeleted.Event(req.Name, request.CreateRequestor(r), logger.Ctx{"old_name": integrationName})) return response.EmptySyncResponse } // networkIntegrationValidate validates the configuration keys/values for network integration. func networkIntegrationValidate(integrationType string, inUse bool, oldConfig map[string]string, config map[string]string) error { if integrationType != "ovn" { return fmt.Errorf("Invalid integration type %q", integrationType) } configKeys := map[string]func(value string) error{ // gendoc:generate(entity=network_integration, group=ovn, key=ovn.northbound_connection) // // --- // type: string // scope: global // shortdesc: OVN northbound inter-connection connection string "ovn.northbound_connection": validate.IsAny, // gendoc:generate(entity=network_integration, group=ovn, key=ovn.southbound_connection) // // --- // type: string // scope: global // shortdesc: OVN southbound inter-connection connection string "ovn.southbound_connection": validate.IsAny, // gendoc:generate(entity=network_integration, group=ovn, key=ovn.ca_cert) // // --- // type: string // scope: global // shortdesc: OVN SSL certificate authority for the inter-connection database "ovn.ca_cert": validate.Optional(validate.IsAny), // gendoc:generate(entity=network_integration, group=ovn, key=ovn.client_cert) // // --- // type: string // scope: global // shortdesc: OVN SSL client certificate "ovn.client_cert": validate.Optional(validate.IsAny), // gendoc:generate(entity=network_integration, group=ovn, key=ovn.client_key) // // --- // type: string // scope: global // shortdesc: OVN SSL client key "ovn.client_key": validate.Optional(validate.IsAny), // gendoc:generate(entity=network_integration, group=ovn, key=ovn.transit.pattern) // Specify a Pongo2 template string that represents the transit switch name. // This template gets access to the project name (`projectName`), // integration name (`integrationName`), network name (`networkName`) // and peer name (`peerName`). // // --- // type: string // defaultdesc: `ts-incus-{{ integrationName }}-{{ projectName }}-{{ networkName }}` // shortdesc: Template for the transit switch name "ovn.transit.pattern": validate.IsAny, } for k, v := range config { // User keys are free for all. // gendoc:generate(entity=network_integration, group=common, key=user.*) // User keys can be used in search. // --- // type: string // shortdesc: Free form user key/value storage if strings.HasPrefix(k, "user.") { continue } validator, ok := configKeys[k] if !ok { return fmt.Errorf("Invalid network integration configuration key %q", k) } err := validator(v) if err != nil { return fmt.Errorf("Invalid network integration configuration key %q value", k) } } if oldConfig != nil && oldConfig["ovn.transit.pattern"] != config["ovn.transit.pattern"] && inUse { return errors.New("The OVN transit switch pattern cannot be changed while the integration is in use") } return nil } incus-7.0.0/cmd/incusd/network_load_balancers.go000066400000000000000000000627401517523235500217410ustar00rootroot00000000000000package main import ( "context" "encoding/json" "fmt" "net/http" "net/url" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/server/auth" clusterRequest "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) var networkLoadBalancersCmd = APIEndpoint{ Path: "networks/{networkName}/load-balancers", Get: APIEndpointAction{Handler: networkLoadBalancersGet, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanView, "networkName")}, Post: APIEndpointAction{Handler: networkLoadBalancersPost, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, } var networkLoadBalancerCmd = APIEndpoint{ Path: "networks/{networkName}/load-balancers/{listenAddress}", Delete: APIEndpointAction{Handler: networkLoadBalancerDelete, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, Get: APIEndpointAction{Handler: networkLoadBalancerGet, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanView, "networkName")}, Put: APIEndpointAction{Handler: networkLoadBalancerPut, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, Patch: APIEndpointAction{Handler: networkLoadBalancerPut, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, } var networkLoadBalancerStateCmd = APIEndpoint{ Path: "networks/{networkName}/load-balancers/{listenAddress}/state", Get: APIEndpointAction{Handler: networkLoadBalancerStateGet, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanView, "networkName")}, } // API endpoints // swagger:operation GET /1.0/networks/{networkName}/load-balancers network-load-balancers network_load_balancers_get // // Get the network address of load balancers // // Returns a list of network address load balancers (URLs). // // --- // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/networks/mybr0/load-balancers/192.0.2.1", // "/1.0/networks/mybr0/load-balancers/192.0.2.2" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/networks/{networkName}/load-balancers?recursion=1 network-load-balancers network_load_balancer_get_recursion1 // // Get the network address load balancers // // Returns a list of network address load balancers (structs). // // --- // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of network address load balancers // items: // $ref: "#/definitions/NetworkLoadBalancer" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkLoadBalancersGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().LoadBalancers { return response.BadRequest(fmt.Errorf("Network driver %q does not support load balancers", n.Type())) } recursion := localUtil.IsRecursionRequest(r) // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } mustLoadObjects := recursion || (clauses != nil && len(clauses.Clauses) > 0) fullResults := make([]api.NetworkLoadBalancer, 0) linkResults := make([]string, 0) if mustLoadObjects { var records map[int64]*api.NetworkLoadBalancer err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() // Get the load balancers. dbLoadBalancers, err := dbCluster.GetNetworkLoadBalancers(ctx, tx.Tx(), dbCluster.NetworkLoadBalancerFilter{NetworkID: &networkID}) if err != nil { return err } records = make(map[int64]*api.NetworkLoadBalancer) for _, lb := range dbLoadBalancers { // Get the full API record. apiLoadBalancer, err := lb.ToAPI(ctx, tx.Tx()) if err != nil { return err } records[lb.ID] = apiLoadBalancer } return nil }) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network load balancers: %w", err)) } for _, record := range records { if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(*record, *clauses) if err != nil { return response.SmartError(err) } if !match { continue } } fullResults = append(fullResults, *record) u := api.NewURL().Path(version.APIVersion, "networks", n.Name(), "load-balancers", record.ListenAddress) linkResults = append(linkResults, u.String()) } } else { listenAddresses := []string{} err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() // Get the load balancers. dbLoadBalancers, err := dbCluster.GetNetworkLoadBalancers(ctx, tx.Tx(), dbCluster.NetworkLoadBalancerFilter{ NetworkID: &networkID, }) if err != nil { return fmt.Errorf("Failed loading network load balancers: %w", err) } for _, lb := range dbLoadBalancers { listenAddresses = append(listenAddresses, lb.ListenAddress) } return nil }) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network load balancers: %w", err)) } for _, listenAddress := range listenAddresses { u := api.NewURL().Path(version.APIVersion, "networks", n.Name(), "load-balancers", listenAddress) linkResults = append(linkResults, u.String()) } } if recursion { return response.SyncResponse(true, fullResults) } return response.SyncResponse(true, linkResults) } // swagger:operation POST /1.0/networks/{networkName}/load-balancers network-load-balancers network_load_balancers_post // // Add a network load balancer // // Creates a new network load balancer. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: load-balancer // description: Load Balancer // required: true // schema: // $ref: "#/definitions/NetworkLoadBalancersPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkLoadBalancersPost(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } // Parse the request into a record. req := api.NetworkLoadBalancersPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } req.Normalise() // So we handle the request in normalised/canonical form. networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().LoadBalancers { return response.BadRequest(fmt.Errorf("Network driver %q does not support load balancers", n.Type())) } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) err = n.LoadBalancerCreate(req, clientType) if err != nil { return response.SmartError(fmt.Errorf("Failed creating load balancer: %w", err)) } lc := lifecycle.NetworkLoadBalancerCreated.Event(n, req.ListenAddress, request.CreateRequestor(r), nil) s.Events.SendLifecycle(projectName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation DELETE /1.0/networks/{networkName}/load-balancers/{listenAddress} network-load-balancers network_load_balancer_delete // // Delete the network address load balancer // // Removes the network address load balancer. // // --- // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: path // name: listenAddress // description: Listen address // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkLoadBalancerDelete(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().LoadBalancers { return response.BadRequest(fmt.Errorf("Network driver %q does not support load balancers", n.Type())) } listenAddress, err := url.PathUnescape(mux.Vars(r)["listenAddress"]) if err != nil { return response.SmartError(err) } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) err = n.LoadBalancerDelete(listenAddress, clientType) if err != nil { return response.SmartError(fmt.Errorf("Failed deleting load balancer: %w", err)) } s.Events.SendLifecycle(projectName, lifecycle.NetworkLoadBalancerDeleted.Event(n, listenAddress, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation GET /1.0/networks/{networkName}/load-balancers/{listenAddress} network-load-balancers network_load_balancer_get // // Get the network address load balancer // // Gets a specific network address load balancer. // // --- // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: path // name: listenAddress // description: Listen address // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Load Balancer // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/NetworkLoadBalancer" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkLoadBalancerGet(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().LoadBalancers { return response.BadRequest(fmt.Errorf("Network driver %q does not support load balancers", n.Type())) } listenAddress, err := url.PathUnescape(mux.Vars(r)["listenAddress"]) if err != nil { return response.SmartError(err) } var loadBalancer *api.NetworkLoadBalancer err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() // Get the load balancer. dbLoadBalancers, err := dbCluster.GetNetworkLoadBalancers(ctx, tx.Tx(), dbCluster.NetworkLoadBalancerFilter{ NetworkID: &networkID, ListenAddress: &listenAddress, }) if err != nil { return err } if len(dbLoadBalancers) != 1 { return api.StatusErrorf(http.StatusNotFound, "Network load balancer not found") } // Get API struct. loadBalancer, err = dbLoadBalancers[0].ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } return response.SyncResponseETag(true, loadBalancer, loadBalancer.Etag()) } // swagger:operation PATCH /1.0/networks/{networkName}/load-balancers/{listenAddress} network-load-balancers network_load_balancer_patch // // Partially update the network address load balancer // // Updates a subset of the network address load balancer configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: path // name: listenAddress // description: Listen address // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: load-balancer // description: Address load balancer configuration // required: true // schema: // $ref: "#/definitions/NetworkLoadBalancerPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation PUT /1.0/networks/{networkName}/load-balancers/{listenAddress} network-load-balancers network_load_balancer_put // // Update the network address load balancer // // Updates the entire network address load balancer configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: path // name: listenAddress // description: Listen address // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: load-balancer // description: Address load balancer configuration // required: true // schema: // $ref: "#/definitions/NetworkLoadBalancerPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func networkLoadBalancerPut(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().LoadBalancers { return response.BadRequest(fmt.Errorf("Network driver %q does not support load balancers", n.Type())) } listenAddress, err := url.PathUnescape(mux.Vars(r)["listenAddress"]) if err != nil { return response.SmartError(err) } // Decode the request. req := api.NetworkLoadBalancerPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } if r.Method == http.MethodPatch { var loadBalancer *api.NetworkLoadBalancer err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() // Get the load balancer. dbLoadBalancers, err := dbCluster.GetNetworkLoadBalancers(ctx, tx.Tx(), dbCluster.NetworkLoadBalancerFilter{ NetworkID: &networkID, ListenAddress: &listenAddress, }) if err != nil { return err } if len(dbLoadBalancers) != 1 { return api.StatusErrorf(http.StatusNotFound, "Network load balancer not found") } // Get the API struct. loadBalancer, err = dbLoadBalancers[0].ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // If config being updated via "patch" method, then merge all existing config with the keys that // are present in the request config. for k, v := range loadBalancer.Config { _, ok := req.Config[k] if !ok { req.Config[k] = v } } // If load balancer being updated via "patch" method and backends not specified, then merge // existing backends into load balancer. if req.Backends == nil { req.Backends = loadBalancer.Backends } // If load balancer being updated via "patch" method and ports not specified, then merge existing // ports into load balancer. if req.Ports == nil { req.Ports = loadBalancer.Ports } } req.Normalise() // So we handle the request in normalised/canonical form. clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) err = n.LoadBalancerUpdate(listenAddress, req, clientType) if err != nil { return response.SmartError(fmt.Errorf("Failed updating load balancer: %w", err)) } s.Events.SendLifecycle(projectName, lifecycle.NetworkLoadBalancerUpdated.Event(n, listenAddress, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation GET /1.0/networks/{networkName}/load-balancers/{listenAddress}/state network-load-balancers network_load_balancer_state_get // // Get the network address load balancer state // // Get the current state of a specific network address load balancer. // // --- // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: path // name: listenAddress // description: Listen address // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Load Balancer state // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/NetworkLoadBalancerState" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkLoadBalancerStateGet(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().LoadBalancers { return response.BadRequest(fmt.Errorf("Network driver %q does not support load balancers", n.Type())) } listenAddress, err := url.PathUnescape(mux.Vars(r)["listenAddress"]) if err != nil { return response.SmartError(err) } var loadBalancer *api.NetworkLoadBalancer err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() // Get the load balancers. dbLoadBalancers, err := dbCluster.GetNetworkLoadBalancers(ctx, tx.Tx(), dbCluster.NetworkLoadBalancerFilter{ NetworkID: &networkID, ListenAddress: &listenAddress, }) if err != nil { return err } if len(dbLoadBalancers) != 1 { return api.StatusErrorf(http.StatusNotFound, "Network load balancer not found") } // Get the API struct. loadBalancer, err = dbLoadBalancers[0].ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } lbState, err := n.LoadBalancerState(*loadBalancer) if err != nil { return response.SmartError(fmt.Errorf("Failed fetching load balancer state: %w", err)) } return response.SyncResponse(true, lbState) } incus-7.0.0/cmd/incusd/network_peers.go000066400000000000000000000465621517523235500201320ustar00rootroot00000000000000package main import ( "context" "encoding/json" "fmt" "net/http" "net/url" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/validate" ) var networkPeersCmd = APIEndpoint{ Path: "networks/{networkName}/peers", Get: APIEndpointAction{Handler: networkPeersGet, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanView, "networkName")}, Post: APIEndpointAction{Handler: networkPeersPost, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, } var networkPeerCmd = APIEndpoint{ Path: "networks/{networkName}/peers/{peerName}", Delete: APIEndpointAction{Handler: networkPeerDelete, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, Get: APIEndpointAction{Handler: networkPeerGet, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanView, "networkName")}, Put: APIEndpointAction{Handler: networkPeerPut, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, Patch: APIEndpointAction{Handler: networkPeerPut, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, } // API endpoints // swagger:operation GET /1.0/networks/{networkName}/peers network-peers network_peers_get // // Get the network peers // // Returns a list of network peers (URLs). // // --- // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/networks/mybr0/peers/my-peer-1", // "/1.0/networks/mybr0/peers/my-peer-2" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/networks/{networkName}/peers?recursion=1 network-peers network_peer_get_recursion1 // // Get the network peers // // Returns a list of network peers (structs). // // --- // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of network peers // items: // $ref: "#/definitions/NetworkPeer" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkPeersGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().Peering { return response.BadRequest(fmt.Errorf("Network driver %q does not support peering", n.Type())) } recursion := localUtil.IsRecursionRequest(r) // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } mustLoadObjects := recursion || (clauses != nil && len(clauses.Clauses) > 0) fullResults := make([]api.NetworkPeer, 0) linkResults := make([]string, 0) if mustLoadObjects { var peers map[int64]*api.NetworkPeer err := s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Use generated function to get peers. netID := n.ID() filter := dbCluster.NetworkPeerFilter{NetworkID: &netID} dbPeers, err := dbCluster.GetNetworkPeers(ctx, tx.Tx(), filter) if err != nil { return fmt.Errorf("Failed loading network peer DB objects: %w", err) } // Convert DB objects to API objects and build the map. peers = make(map[int64]*api.NetworkPeer, len(dbPeers)) for _, dbPeer := range dbPeers { peer, err := dbPeer.ToAPI(ctx, tx.Tx()) if err != nil { // Use fmt.Errorf as requested, though logging might be preferable in some contexts. return fmt.Errorf("Failed converting network peer DB object to API object for peer ID %d: %w", dbPeer.ID, err) } peers[dbPeer.ID] = peer } return nil }) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network peers: %w", err)) } for _, peer := range peers { peer.UsedBy, _ = n.PeerUsedBy(peer.Name) if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(*peer, *clauses) if err != nil { return response.SmartError(err) } if !match { continue } } fullResults = append(fullResults, *peer) linkResults = append(linkResults, fmt.Sprintf("/%s/networks/%s/peers/%s", version.APIVersion, url.PathEscape(n.Name()), url.PathEscape(peer.Name))) } } else { var peerNames map[int64]string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Use the generated GetNetworkPeers function with a filter. netID := n.ID() filter := dbCluster.NetworkPeerFilter{NetworkID: &netID} peers, err := dbCluster.GetNetworkPeers(ctx, tx.Tx(), filter) if err != nil { return err } peerNames = make(map[int64]string, len(peers)) for _, peer := range peers { peerNames[peer.ID] = peer.Name } return nil }) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network peers: %w", err)) } for _, peerName := range peerNames { linkResults = append(linkResults, fmt.Sprintf("/%s/networks/%s/peers/%s", version.APIVersion, url.PathEscape(n.Name()), url.PathEscape(peerName))) } } if recursion { return response.SyncResponse(true, fullResults) } return response.SyncResponse(true, linkResults) } // swagger:operation POST /1.0/networks/{networkName}/peers network-peers network_peers_post // // Add a network peer // // Initiates/creates a new network peering. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: peer // description: Peer // required: true // schema: // $ref: "#/definitions/NetworkPeersPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "202": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkPeersPost(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } // Parse the request into a record. req := api.NetworkPeersPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid network peer name: %w", err)) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().Peering { return response.BadRequest(fmt.Errorf("Network driver %q does not support peering", n.Type())) } err = n.PeerCreate(req) if err != nil { return response.SmartError(fmt.Errorf("Failed creating peer: %w", err)) } lc := lifecycle.NetworkPeerCreated.Event(n, req.Name, request.CreateRequestor(r), nil) s.Events.SendLifecycle(projectName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation DELETE /1.0/networks/{networkName}/peers/{peerName} network-peers network_peer_delete // // Delete the network peer // // Removes the network peering. // // --- // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: path // name: peerName // description: Peer name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkPeerDelete(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().Peering { return response.BadRequest(fmt.Errorf("Network driver %q does not support peering", n.Type())) } peerName, err := url.PathUnescape(mux.Vars(r)["peerName"]) if err != nil { return response.SmartError(err) } err = n.PeerDelete(peerName) if err != nil { return response.SmartError(fmt.Errorf("Failed deleting peer: %w", err)) } s.Events.SendLifecycle(projectName, lifecycle.NetworkPeerDeleted.Event(n, peerName, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation GET /1.0/networks/{networkName}/peers/{peerName} network-peers network_peer_get // // Get the network peer // // Gets a specific network peering. // // --- // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: path // name: peerName // description: Peer name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Peer // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/NetworkPeer" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkPeerGet(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().Peering { return response.BadRequest(fmt.Errorf("Network driver %q does not support peering", n.Type())) } peerName, err := url.PathUnescape(mux.Vars(r)["peerName"]) if err != nil { return response.SmartError(err) } var peer *api.NetworkPeer err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { netID := n.ID() dbPeer, err := dbCluster.GetNetworkPeer(ctx, tx.Tx(), netID, peerName) if err != nil { return fmt.Errorf("Failed getting network peer DB object: %w", err) } peer, err = dbPeer.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed converting network peer DB object to API object: %w", err) } return nil }) if err != nil { return response.SmartError(err) } peer.UsedBy, _ = n.PeerUsedBy(peer.Name) return response.SyncResponseETag(true, peer, peer.Etag()) } // swagger:operation PATCH /1.0/networks/{networkName}/peers/{peerName} network-peers network_peer_patch // // Partially update the network peer // // Updates a subset of the network peering configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: path // name: peerName // description: Peer name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: Peer // description: Peer configuration // required: true // schema: // $ref: "#/definitions/NetworkPeerPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation PUT /1.0/networks/{networkName}/peers/{peerName} network-peers network_peer_put // // Update the network peer // // Updates the entire network peering configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: networkName // description: Network name // type: string // required: true // - in: path // name: peerName // description: Peer name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: peer // description: Peer configuration // required: true // schema: // $ref: "#/definitions/NetworkPeerPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func networkPeerPut(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if !n.Info().Peering { return response.BadRequest(fmt.Errorf("Network driver %q does not support peering", n.Type())) } peerName, err := url.PathUnescape(mux.Vars(r)["peerName"]) if err != nil { return response.SmartError(err) } // Decode the request. req := api.NetworkPeerPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } err = n.PeerUpdate(peerName, req) if err != nil { return response.SmartError(fmt.Errorf("Failed updating peer: %w", err)) } s.Events.SendLifecycle(projectName, lifecycle.NetworkPeerUpdated.Event(n, peerName, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } incus-7.0.0/cmd/incusd/network_zones.go000066400000000000000000000372541517523235500201500ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/url" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/server/auth" clusterRequest "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/network/zone" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) var networkZonesCmd = APIEndpoint{ Path: "network-zones", Get: APIEndpointAction{Handler: networkZonesGet, AccessHandler: allowAuthenticated}, Post: APIEndpointAction{Handler: networkZonesPost, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanCreateNetworkZones)}, } var networkZoneCmd = APIEndpoint{ Path: "network-zones/{zone}", Delete: APIEndpointAction{Handler: networkZoneDelete, AccessHandler: allowPermission(auth.ObjectTypeNetworkZone, auth.EntitlementCanEdit, "zone")}, Get: APIEndpointAction{Handler: networkZoneGet, AccessHandler: allowPermission(auth.ObjectTypeNetworkZone, auth.EntitlementCanView, "zone")}, Put: APIEndpointAction{Handler: networkZonePut, AccessHandler: allowPermission(auth.ObjectTypeNetworkZone, auth.EntitlementCanEdit, "zone")}, Patch: APIEndpointAction{Handler: networkZonePut, AccessHandler: allowPermission(auth.ObjectTypeNetworkZone, auth.EntitlementCanEdit, "zone")}, } // API endpoints. // swagger:operation GET /1.0/network-zones network-zones network_zones_get // // Get the network zones // // Returns a list of network zones (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve network zones from all projects // type: boolean // example: true // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/network-zones/example.net", // "/1.0/network-zones/example.com" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/network-zones?recursion=1 network-zones network_zones_get_recursion1 // // Get the network zones // // Returns a list of network zones (structs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve network zones from all projects // type: boolean // example: true // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of network zones // items: // $ref: "#/definitions/NetworkZone" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkZonesGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkZoneProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } recursion := localUtil.IsRecursionRequest(r) // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } mustLoadObjects := recursion || (clauses != nil && len(clauses.Clauses) > 0) var zones []dbCluster.NetworkZone var zoneNamesMap map[string]string allProjects := util.IsTrue(request.QueryParam(r, "all-projects")) err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { if allProjects { zones, err = dbCluster.GetNetworkZones(ctx, tx.Tx()) if err != nil { return err } zoneNamesMap = map[string]string{} for _, zone := range zones { zoneNamesMap[zone.Name] = zone.Project } } else { filter := dbCluster.NetworkZoneFilter{Project: &projectName} zones, err = dbCluster.GetNetworkZones(ctx, tx.Tx(), filter) if err != nil { return err } zoneNamesMap = map[string]string{} for _, zone := range zones { zoneNamesMap[zone.Name] = projectName } } return err }) if err != nil { return response.InternalError(err) } userHasPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanView, auth.ObjectTypeNetworkZone) if err != nil { return response.InternalError(err) } linkResults := make([]string, 0) fullResults := make([]api.NetworkZone, 0) for zoneName, projectName := range zoneNamesMap { if !userHasPermission(auth.ObjectNetworkZone(projectName, zoneName)) { continue } if mustLoadObjects { netzone, err := zone.LoadByNameAndProject(s, projectName, zoneName) if err != nil { continue } netzoneInfo := netzone.Info() netzoneInfo.UsedBy, _ = netzone.UsedBy() // Ignore errors in UsedBy, will return nil. netzoneInfo.Project = projectName if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(*netzoneInfo, *clauses) if err != nil { return response.SmartError(err) } if !match { continue } } fullResults = append(fullResults, *netzoneInfo) } linkResults = append(linkResults, api.NewURL().Path(version.APIVersion, "network-zones", zoneName).String()) } if !recursion { return response.SyncResponse(true, linkResults) } return response.SyncResponse(true, fullResults) } // swagger:operation POST /1.0/network-zones network-zones network_zones_post // // Add a network zone // // Creates a new network zone. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: zone // description: zone // required: true // schema: // $ref: "#/definitions/NetworkZonesPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkZonesPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkZoneProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } req := api.NetworkZonesPost{} // Parse the request into a record. err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Create the zone. err = zone.Exists(s, req.Name) if err == nil { return response.BadRequest(errors.New("The network zone already exists")) } err = zone.Create(s, projectName, &req) if err != nil { return response.SmartError(err) } netzone, err := zone.LoadByNameAndProject(s, projectName, req.Name) if err != nil { return response.BadRequest(err) } err = s.Authorizer.AddNetworkZone(r.Context(), projectName, req.Name) if err != nil { logger.Error("Failed to add network zone to authorizer", logger.Ctx{"name": req.Name, "project": projectName, "error": err}) } lc := lifecycle.NetworkZoneCreated.Event(netzone, request.CreateRequestor(r), nil) s.Events.SendLifecycle(projectName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation DELETE /1.0/network-zones/{zone} network-zones network_zone_delete // // Delete the network zone // // Removes the network zone. // // --- // produces: // - application/json // parameters: // - in: path // name: zone // description: Network zone name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkZoneDelete(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkZoneProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } zoneName, err := url.PathUnescape(mux.Vars(r)["zone"]) if err != nil { return response.SmartError(err) } netzone, err := zone.LoadByNameAndProject(s, projectName, zoneName) if err != nil { return response.SmartError(err) } err = netzone.Delete() if err != nil { return response.SmartError(err) } err = s.Authorizer.DeleteNetworkZone(r.Context(), projectName, zoneName) if err != nil { logger.Error("Failed to remove network zone from authorizer", logger.Ctx{"name": zoneName, "project": projectName, "error": err}) } s.Events.SendLifecycle(projectName, lifecycle.NetworkZoneDeleted.Event(netzone, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation GET /1.0/network-zones/{zone} network-zones network_zone_get // // Get the network zone // // Gets a specific network zone. // // --- // produces: // - application/json // parameters: // - in: path // name: zone // description: Network zone name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: zone // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/NetworkZone" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkZoneGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkZoneProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } zoneName, err := url.PathUnescape(mux.Vars(r)["zone"]) if err != nil { return response.SmartError(err) } netzone, err := zone.LoadByNameAndProject(s, projectName, zoneName) if err != nil { return response.SmartError(err) } info := netzone.Info() info.UsedBy, err = netzone.UsedBy() if err != nil { return response.SmartError(err) } return response.SyncResponseETag(true, info, netzone.Etag()) } // swagger:operation PATCH /1.0/network-zones/{zone} network-zones network_zone_patch // // Partially update the network zone // // Updates a subset of the network zone configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: zone // description: Network zone name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: zone // description: zone configuration // required: true // schema: // $ref: "#/definitions/NetworkZonePut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation PUT /1.0/network-zones/{zone} network-zones network_zone_put // // Update the network zone // // Updates the entire network zone configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: zone // description: Network zone name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: zone // description: zone configuration // required: true // schema: // $ref: "#/definitions/NetworkZonePut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func networkZonePut(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkZoneProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } zoneName, err := url.PathUnescape(mux.Vars(r)["zone"]) if err != nil { return response.SmartError(err) } // Get the existing Network zone. netzone, err := zone.LoadByNameAndProject(s, projectName, zoneName) if err != nil { return response.SmartError(err) } // Validate the ETag. err = localUtil.EtagCheck(r, netzone.Etag()) if err != nil { return response.PreconditionFailed(err) } req := api.NetworkZonePut{} // Decode the request. err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } if r.Method == http.MethodPatch { // If config being updated via "patch" method, then merge all existing config with the keys that // are present in the request config. for k, v := range netzone.Info().Config { _, ok := req.Config[k] if !ok { req.Config[k] = v } } } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) err = netzone.Update(&req, clientType) if err != nil { return response.SmartError(err) } s.Events.SendLifecycle(projectName, lifecycle.NetworkZoneUpdated.Event(netzone, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } incus-7.0.0/cmd/incusd/network_zones_records.go000066400000000000000000000371551517523235500216710ustar00rootroot00000000000000package main import ( "encoding/json" "fmt" "net/http" "net/url" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/server/auth" clusterRequest "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/network/zone" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) var networkZoneRecordsCmd = APIEndpoint{ Path: "network-zones/{zone}/records", Get: APIEndpointAction{Handler: networkZoneRecordsGet, AccessHandler: allowPermission(auth.ObjectTypeNetworkZone, auth.EntitlementCanView, "zone")}, Post: APIEndpointAction{Handler: networkZoneRecordsPost, AccessHandler: allowPermission(auth.ObjectTypeNetworkZone, auth.EntitlementCanEdit, "zone")}, } var networkZoneRecordCmd = APIEndpoint{ Path: "network-zones/{zone}/records/{name}", Delete: APIEndpointAction{Handler: networkZoneRecordDelete, AccessHandler: allowPermission(auth.ObjectTypeNetworkZone, auth.EntitlementCanEdit, "zone")}, Get: APIEndpointAction{Handler: networkZoneRecordGet, AccessHandler: allowPermission(auth.ObjectTypeNetworkZone, auth.EntitlementCanView, "zone")}, Put: APIEndpointAction{Handler: networkZoneRecordPut, AccessHandler: allowPermission(auth.ObjectTypeNetworkZone, auth.EntitlementCanEdit, "zone")}, Patch: APIEndpointAction{Handler: networkZoneRecordPut, AccessHandler: allowPermission(auth.ObjectTypeNetworkZone, auth.EntitlementCanEdit, "zone")}, } // API endpoints. // swagger:operation GET /1.0/network-zones/{zone}/records network-zones network_zone_records_get // // Get the network zone records // // Returns a list of network zone records (URLs). // // --- // produces: // - application/json // parameters: // - in: path // name: zone // description: Network zone name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/network-zones/example.net/records/foo", // "/1.0/network-zones/example.net/records/bar" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/network-zones/{zone}/records?recursion=1 network-zones network_zone_records_get_recursion1 // // Get the network zone records // // Returns a list of network zone records (structs). // // --- // produces: // - application/json // parameters: // - in: path // name: zone // description: Network zone name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of network zone records // items: // $ref: "#/definitions/NetworkZoneRecord" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkZoneRecordsGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkZoneProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } recursion := localUtil.IsRecursionRequest(r) // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } mustLoadObjects := recursion || (clauses != nil && len(clauses.Clauses) > 0) zoneName, err := url.PathUnescape(mux.Vars(r)["zone"]) if err != nil { return response.SmartError(err) } // Get the network zone. netzone, err := zone.LoadByNameAndProject(s, projectName, zoneName) if err != nil { return response.SmartError(err) } // Get the records. records, err := netzone.GetRecords() if err != nil { return response.SmartError(err) } linkResults := make([]string, 0) fullResults := make([]api.NetworkZoneRecord, 0) for _, record := range records { if mustLoadObjects { if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(record, *clauses) if err != nil { return response.SmartError(err) } if !match { continue } } fullResults = append(fullResults, record) } linkResults = append(linkResults, api.NewURL().Path(version.APIVersion, "network-zones", zoneName, "records", record.Name).String()) } if !recursion { return response.SyncResponse(true, linkResults) } return response.SyncResponse(true, fullResults) } // swagger:operation POST /1.0/network-zones/{zone}/records network-zones network_zone_records_post // // Add a network zone record // // Creates a new network zone record. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: zone // description: Network zone name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: zone // description: zone // required: true // schema: // $ref: "#/definitions/NetworkZoneRecordsPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkZoneRecordsPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkZoneProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } zoneName, err := url.PathUnescape(mux.Vars(r)["zone"]) if err != nil { return response.SmartError(err) } // Get the network zone. netzone, err := zone.LoadByNameAndProject(s, projectName, zoneName) if err != nil { return response.SmartError(err) } // Parse the request into a record. req := api.NetworkZoneRecordsPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Create the record. err = netzone.AddRecord(req) if err != nil { return response.SmartError(err) } lc := lifecycle.NetworkZoneRecordCreated.Event(netzone, req.Name, request.CreateRequestor(r), nil) s.Events.SendLifecycle(projectName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation DELETE /1.0/network-zones/{zone}/records/{name} network-zones network_zone_record_delete // // Delete the network zone record // // Removes the network zone record. // // --- // produces: // - application/json // parameters: // - in: path // name: zone // description: Network zone name // type: string // required: true // - in: path // name: name // description: Record name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkZoneRecordDelete(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkZoneProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } zoneName, err := url.PathUnescape(mux.Vars(r)["zone"]) if err != nil { return response.SmartError(err) } recordName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Get the network zone. netzone, err := zone.LoadByNameAndProject(s, projectName, zoneName) if err != nil { return response.SmartError(err) } // Delete the record. err = netzone.DeleteRecord(recordName) if err != nil { return response.SmartError(err) } s.Events.SendLifecycle(projectName, lifecycle.NetworkZoneRecordDeleted.Event(netzone, recordName, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation GET /1.0/network-zones/{zone}/records/{name} network-zones network_zone_record_get // // Get the network zone record // // Gets a specific network zone record. // // --- // produces: // - application/json // parameters: // - in: path // name: zone // description: Network zone name // type: string // required: true // - in: path // name: name // description: Record name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: zone // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/NetworkZoneRecord" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkZoneRecordGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkZoneProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } zoneName, err := url.PathUnescape(mux.Vars(r)["zone"]) if err != nil { return response.SmartError(err) } recordName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Get the network zone. netzone, err := zone.LoadByNameAndProject(s, projectName, zoneName) if err != nil { return response.SmartError(err) } // Get the record. record, err := netzone.GetRecord(recordName) if err != nil { return response.SmartError(err) } return response.SyncResponseETag(true, record, record.Writable()) } // swagger:operation PATCH /1.0/network-zones/{zone}/records/{name} network-zones network_zone_record_patch // // Partially update the network zone record // // Updates a subset of the network zone record configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: zone // description: Network zone name // type: string // required: true // - in: path // name: name // description: Record name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: zone // description: zone record configuration // required: true // schema: // $ref: "#/definitions/NetworkZoneRecordPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation PUT /1.0/network-zones/{zone}/records/{name} network-zones network_zone_record_put // // Update the network zone record // // Updates the entire network zone record configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: zone // description: Network zone name // type: string // required: true // - in: path // name: name // description: Record name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: zone // description: zone record configuration // required: true // schema: // $ref: "#/definitions/NetworkZoneRecordPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func networkZoneRecordPut(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, _, err := project.NetworkZoneProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } zoneName, err := url.PathUnescape(mux.Vars(r)["zone"]) if err != nil { return response.SmartError(err) } recordName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } // Get the network zone. netzone, err := zone.LoadByNameAndProject(s, projectName, zoneName) if err != nil { return response.SmartError(err) } // Get the record. record, err := netzone.GetRecord(recordName) if err != nil { return response.SmartError(err) } // Validate the ETag. err = localUtil.EtagCheck(r, record.Writable()) if err != nil { return response.PreconditionFailed(err) } // Decode the request. req := api.NetworkZoneRecordPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } if r.Method == http.MethodPatch { // If config being updated via "patch" method, then merge all existing config with the keys that // are present in the request config. for k, v := range netzone.Info().Config { _, ok := req.Config[k] if !ok { req.Config[k] = v } } } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) err = netzone.UpdateRecord(recordName, req, clientType) if err != nil { return response.SmartError(err) } s.Events.SendLifecycle(projectName, lifecycle.NetworkZoneRecordUpdated.Event(netzone, recordName, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } incus-7.0.0/cmd/incusd/networks.go000066400000000000000000001740601517523235500171120ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "maps" "net" "net/http" "net/url" "slices" "strconv" "strings" "sync" "time" "github.com/gorilla/mux" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/cluster" clusterRequest "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/warningtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/server/warnings" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // Lock to prevent concurrent networks creation. var networkCreateLock sync.Mutex var networksCmd = APIEndpoint{ Path: "networks", Get: APIEndpointAction{Handler: networksGet, AccessHandler: allowAuthenticated}, Post: APIEndpointAction{Handler: networksPost, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanCreateNetworks)}, } var networkCmd = APIEndpoint{ Path: "networks/{networkName}", Delete: APIEndpointAction{Handler: networkDelete, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, Get: APIEndpointAction{Handler: networkGet, AccessHandler: allowAuthenticated}, Patch: APIEndpointAction{Handler: networkPatch, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, Post: APIEndpointAction{Handler: networkPost, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, Put: APIEndpointAction{Handler: networkPut, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanEdit, "networkName")}, } var networkLeasesCmd = APIEndpoint{ Path: "networks/{networkName}/leases", Get: APIEndpointAction{Handler: networkLeasesGet, AccessHandler: allowPermission(auth.ObjectTypeNetwork, auth.EntitlementCanView, "networkName")}, } var networkStateCmd = APIEndpoint{ Path: "networks/{networkName}/state", Get: APIEndpointAction{Handler: networkStateGet, AccessHandler: allowAuthenticated}, } // API endpoints // swagger:operation GET /1.0/networks networks networks_get // // Get the networks // // Returns a list of networks (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve networks from all projects // type: boolean // example: true // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/networks/mybr0", // "/1.0/networks/mybr1" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/networks?recursion=1 networks networks_get_recursion1 // // Get the networks // // Returns a list of networks (structs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve networks from all projects // type: boolean // example: true // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of networks // items: // $ref: "#/definitions/Network" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networksGet(d *Daemon, r *http.Request) response.Response { s := d.State() // If a target was specified, forward the request to the relevant node. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } // Track down the project holding networks based on current project configuration. projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } recursion := localUtil.IsRecursionRequest(r) // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } mustLoadObjects := recursion || (clauses != nil && len(clauses.Clauses) > 0) allProjects := util.IsTrue(r.FormValue("all-projects")) unmanagedNetworkNames := []string{} var networkNames map[string][]string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { if allProjects { // Get list of managed networks from all projects. networkNames, err = tx.GetNetworksAllProjects(ctx) if err != nil { return err } } else { // Get list of managed networks (that may or may not have network interfaces on the host). networks, err := tx.GetNetworks(ctx, projectName) if err != nil { return err } networkNames = map[string][]string{} networkNames[projectName] = networks } return nil }) if err != nil { return response.SmartError(err) } // Get list of actual network interfaces on the host as well if the effective project is Default. if projectName == api.ProjectDefaultName { if s.OS.IncusOS != nil { ns, err := s.OS.IncusOS.GetSystemNetwork() if err != nil { return response.InternalError(err) } // Get the interfaces. for _, iface := range ns.State.GetInterfaceNamesByRole("instances") { // Append to the list of networks if a managed network of same name doesn't exist. if !slices.Contains(networkNames[projectName], iface) { networkNames[projectName] = append(networkNames[projectName], iface) unmanagedNetworkNames = append(unmanagedNetworkNames, iface) } } } else { ifaces, err := net.Interfaces() if err != nil { return response.InternalError(err) } for _, iface := range ifaces { // Ignore veth pairs (for performance reasons). if strings.HasPrefix(iface.Name, "veth") { continue } // Append to the list of networks if a managed network of same name doesn't exist. if !slices.Contains(networkNames[projectName], iface.Name) { networkNames[projectName] = append(networkNames[projectName], iface.Name) unmanagedNetworkNames = append(unmanagedNetworkNames, iface.Name) } } } } linkResults := make([]string, 0) fullResults := make([]api.Network, 0) for projectName, networks := range networkNames { for _, networkName := range networks { if mustLoadObjects { netInfo, err := doNetworkGet(s, r, s.ServerClustered, projectName, reqProject.Config, networkName) if err != nil { continue } if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(netInfo, *clauses) if err != nil { return response.SmartError(err) } if !match { continue } } fullResults = append(fullResults, netInfo) } else { ok, err := canAccessNetwork(s, r, projectName, reqProject.Config, networkName, !slices.Contains(unmanagedNetworkNames, networkName)) if err != nil { return response.SmartError(err) } if !ok { continue } } linkResults = append(linkResults, fmt.Sprintf("/%s/networks/%s", version.APIVersion, networkName)) } } if !recursion { return response.SyncResponse(true, linkResults) } return response.SyncResponse(true, fullResults) } // swagger:operation POST /1.0/networks networks networks_post // // Add a network // // Creates a new network. // When clustered, most network types require individual POST for each cluster member prior to a global POST. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: network // description: Network // required: true // schema: // $ref: "#/definitions/NetworksPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networksPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkCreateLock.Lock() defer networkCreateLock.Unlock() req := api.NetworksPost{} // Parse the request. err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. if req.Name == "" { return response.BadRequest(errors.New("No name provided")) } if req.Name == "none" { return response.BadRequest(errors.New("Invalid network name: 'none' is a reserved name")) } err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid network name: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, req.Name, true) { return response.SmartError(api.StatusErrorf(http.StatusForbidden, "Network not allowed in project")) } if req.Type == "" { if projectName != api.ProjectDefaultName { req.Type = "ovn" // Only OVN networks are allowed inside network enabled projects. } else { req.Type = "bridge" // Default to bridge for non-network enabled projects. } } if req.Config == nil { req.Config = map[string]string{} } netType, err := network.LoadByType(req.Type) if err != nil { return response.BadRequest(err) } // Driver specific name validation. err = netType.ValidateName(req.Name) if err != nil { return response.BadRequest(fmt.Errorf("Invalid network name: %w", err)) } netTypeInfo := netType.Info() if projectName != api.ProjectDefaultName && !netTypeInfo.Projects { return response.BadRequest(errors.New("Network type does not support non-default projects")) } // Check if project has limits.network and if so check we are allowed to create another network. if projectName != api.ProjectDefaultName && reqProject.Config != nil && reqProject.Config["limits.networks"] != "" { networksLimit, err := strconv.Atoi(reqProject.Config["limits.networks"]) if err != nil { return response.InternalError(fmt.Errorf("Invalid project limits.network value: %w", err)) } var networks []string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { networks, err = tx.GetNetworks(ctx, projectName) return err }) if err != nil { return response.InternalError(fmt.Errorf("Failed loading project's networks for limits check: %w", err)) } // Only check network limits if the new network name doesn't exist already in networks list. // If it does then this create request will either be for adding a target node to an existing // pending network or it will fail anyway as it is a duplicate. if !slices.Contains(networks, req.Name) && len(networks) >= networksLimit { return response.BadRequest(errors.New("Networks limit has been reached for project")) } } u := api.NewURL().Path(version.APIVersion, "networks", req.Name).Project(projectName) resp := response.SyncResponseLocation(true, nil, u.String()) clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) if isClusterNotification(r) { n, err := network.LoadByName(s, projectName, req.Name) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // This is an internal request which triggers the actual creation of the network across all nodes // after they have been previously defined. err = doNetworksCreate(r.Context(), s, n, clientType) if err != nil { return response.SmartError(err) } return resp } targetNode := request.QueryParam(r, "target") if targetNode != "" { if !netTypeInfo.NodeSpecificConfig { return response.BadRequest(fmt.Errorf("Network type %q does not support member specific config", netType.Type())) } // A targetNode was specified, let's just define the node's network without actually creating it. // Check that only NodeSpecificNetworkConfig keys are specified. for key := range req.Config { if !db.IsNodeSpecificNetworkConfig(key) { return response.BadRequest(fmt.Errorf("Config key %q may not be used as member-specific key", key)) } } // Make sure that no description is set through the member-specific path. if req.Description != "" { return response.BadRequest(errors.New("The network description cannot be set for a specific member")) } exists := false err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { _, err := tx.GetNetworkID(ctx, projectName, req.Name) if err == nil { exists = true } return tx.CreatePendingNetwork(ctx, targetNode, projectName, req.Name, req.Description, netType.DBType(), req.Config) }) if err != nil { if errors.Is(err, db.ErrAlreadyDefined) { return response.Conflict(fmt.Errorf("Network %q is already defined on member %q", req.Name, targetNode)) } return response.SmartError(err) } if !exists { err = s.Authorizer.AddNetwork(r.Context(), projectName, req.Name) if err != nil { logger.Error("Failed to add network to authorizer", logger.Ctx{"name": req.Name, "project": projectName, "error": err}) } } return resp } var netInfo *api.Network err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Load existing network if exists, if not don't fail. _, netInfo, _, err = tx.GetNetworkInAnyState(ctx, projectName, req.Name) return err }) if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { return response.InternalError(err) } // Check if we're clustered. count, err := cluster.Count(s) if err != nil { return response.SmartError(err) } // No targetNode was specified and we're clustered or there is an existing partially created single node // network, either way finalize the config in the db and actually create the network on all cluster nodes. if count > 1 || (netInfo != nil && netInfo.Status != api.NetworkStatusCreated) { // Simulate adding pending node network config when the driver doesn't support per-node config. if !netTypeInfo.NodeSpecificConfig && clientType != clusterRequest.ClientTypeJoiner { // Create pending entry for each node. err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { members, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } for _, member := range members { // Don't pass in any config, as these nodes don't have any node-specific // config and we don't want to create duplicate global config. err = tx.CreatePendingNetwork(ctx, member.Name, projectName, req.Name, "", netType.DBType(), nil) if err != nil && !errors.Is(err, db.ErrAlreadyDefined) { return fmt.Errorf("Failed creating pending network for member %q: %w", member.Name, err) } } return nil }) if err != nil { return response.SmartError(err) } // Create the authorization entry and advertise the network as existing. err = s.Authorizer.AddNetwork(r.Context(), projectName, req.Name) if err != nil { logger.Error("Failed to add network to authorizer", logger.Ctx{"name": req.Name, "project": projectName, "error": err}) } } err = networksPostCluster(r.Context(), s, projectName, netInfo, req, clientType, netType) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, req.Name) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(projectName, lifecycle.NetworkCreated.Event(n, requestor, nil)) return resp } // Non-clustered network creation. if netInfo != nil { return response.Conflict(fmt.Errorf("Network %q already exists", req.Name)) } reverter := revert.New() defer reverter.Fail() // Populate default config. if clientType != clusterRequest.ClientTypeJoiner { err = netType.FillConfig(req.Config) if err != nil { return response.SmartError(err) } } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Create the database entry. _, err = tx.CreateNetwork(ctx, projectName, req.Name, req.Description, netType.DBType(), req.Config) return err }) if err != nil { return response.SmartError(fmt.Errorf("Error inserting %q into database: %w", req.Name, err)) } reverter.Add(func() { _ = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.DeleteNetwork(ctx, projectName, req.Name) }) }) n, err := network.LoadByName(s, projectName, req.Name) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } err = doNetworksCreate(r.Context(), s, n, clientType) if err != nil { return response.SmartError(err) } err = s.Authorizer.AddNetwork(r.Context(), projectName, req.Name) if err != nil { logger.Error("Failed to add network to authorizer", logger.Ctx{"name": req.Name, "project": projectName, "error": err}) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(projectName, lifecycle.NetworkCreated.Event(n, requestor, nil)) reverter.Success() return resp } // networkPartiallyCreated returns true of supplied network has properties that indicate it has had previous // create attempts run on it but failed on one or more nodes. func networkPartiallyCreated(netInfo *api.Network) bool { // If the network status is NetworkStatusErrored, this means create has been run in the past and has // failed on one or more nodes. Hence it is partially created. if netInfo.Status == api.NetworkStatusErrored { return true } // If the network has global config keys, then it has previously been created by having its global config // inserted, and this means it is partialled created. for key := range netInfo.Config { if !db.IsNodeSpecificNetworkConfig(key) { return true } } return false } // networksPostCluster checks that there is a pending network in the database and then attempts to setup the // network on each node. If all nodes are successfully setup then the network's state is set to created. // Accepts an optional existing network record, which will exist when performing subsequent re-create attempts. func networksPostCluster(ctx context.Context, s *state.State, projectName string, netInfo *api.Network, req api.NetworksPost, clientType clusterRequest.ClientType, netType network.Type) error { // Check that no node-specific config key has been supplied in request. for key := range req.Config { if db.IsNodeSpecificNetworkConfig(key) { return fmt.Errorf("Config key %q is cluster member specific", key) } } // If network already exists, perform quick checks. if netInfo != nil { // Check network isn't already created. if netInfo.Status == api.NetworkStatusCreated { return errors.New("The network is already created") } // Check the requested network type matches the type created when adding the local member config. if req.Type != netInfo.Type { return fmt.Errorf("Requested network type %q doesn't match type in existing database record %q", req.Type, netInfo.Type) } } // Check that the network is properly defined, get the node-specific configs and merge with global config. var nodeConfigs map[string]map[string]string err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Check if any global config exists already, if so we should not create global config again. if netInfo != nil && networkPartiallyCreated(netInfo) { if len(req.Config) > 0 { return errors.New("Network already partially created. Please do not specify any global config when re-running create") } logger.Debug("Skipping global network create as global config already partially created", logger.Ctx{"project": projectName, "network": req.Name}) return nil } // Fetch the network ID. networkID, err := tx.GetNetworkID(ctx, projectName, req.Name) if err != nil { return err } // Fetch the node-specific configs and check the network is defined for all nodes. nodeConfigs, err = tx.NetworkNodeConfigs(ctx, networkID) if err != nil { return err } // Add default values if we are inserting global config for first time. err = netType.FillConfig(req.Config) if err != nil { return err } // Insert the global config keys. err = tx.CreateNetworkConfig(networkID, 0, req.Config) if err != nil { return err } // Set the network description if provided err = tx.UpdateNetworkDescription(networkID, req.Description) if err != nil { return err } // Assume failure unless we succeed later on. return tx.NetworkErrored(projectName, req.Name) }) if err != nil { if response.IsNotFoundError(err) { return errors.New("Network not pending on any node (use --target first)") } return err } // Create notifier for other nodes to create the network. notifier, err := cluster.NewNotifier(s, s.Endpoints.NetworkCert(), s.ServerCert(), cluster.NotifyAll) if err != nil { return err } // Load the network from the database for the local member. n, err := network.LoadByName(s, projectName, req.Name) if err != nil { return fmt.Errorf("Failed loading network: %w", err) } netConfig := n.Config() err = doNetworksCreate(ctx, s, n, clientType) if err != nil { return err } logger.Debug("Created network on local cluster member", logger.Ctx{"project": projectName, "network": req.Name, "config": netConfig}) // Remove this node's node specific config keys. netConfig = db.StripNodeSpecificNetworkConfig(netConfig) // Notify other nodes to create the network. err = notifier(func(client incus.InstanceServer) error { server, _, err := client.GetServer() if err != nil { return err } // Clone the network config for this node so we don't modify it and potentially end up sending // this node's config to another node. nodeConfig := util.CloneMap(netConfig) // Merge node specific config items into global config. maps.Copy(nodeConfig, nodeConfigs[server.Environment.ServerName]) // Create fresh request based on existing network to send to node. nodeReq := api.NetworksPost{ NetworkPut: api.NetworkPut{ Config: nodeConfig, Description: n.Description(), }, Name: n.Name(), Type: n.Type(), } err = client.UseProject(n.Project()).CreateNetwork(nodeReq) if err != nil { return err } logger.Debug("Created network on cluster member", logger.Ctx{"project": n.Project(), "network": n.Name(), "member": server.Environment.ServerName, "config": nodeReq.Config}) return nil }) if err != nil { return err } // Mark network global status as networkCreated now that all nodes have succeeded. err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.NetworkCreated(projectName, req.Name) }) if err != nil { return err } logger.Debug("Marked network global status as created", logger.Ctx{"project": projectName, "network": req.Name}) return nil } // Create the network on the system. The clusterNotification flag is used to indicate whether creation request // is coming from a cluster notification (and if so we should not delete the database record on error). func doNetworksCreate(ctx context.Context, s *state.State, n network.Network, clientType clusterRequest.ClientType) error { reverter := revert.New() defer reverter.Fail() validateConfig := n.Config() // Skip the ACLs during validation on cluster join as those aren't yet available in the database. if clientType == clusterRequest.ClientTypeJoiner { validateConfig = map[string]string{} for k, v := range n.Config() { if k == "security.acls" || strings.HasPrefix(k, "security.acls.") { continue } validateConfig[k] = v } } // Validate so that when run on a cluster node the full config (including node specific config) is checked. err := n.Validate(validateConfig, clientType) if err != nil { return err } if n.LocalStatus() == api.NetworkStatusCreated { logger.Debug("Skipping local network create as already created", logger.Ctx{"project": n.Project(), "network": n.Name()}) return nil } // Run initial creation setup for the network driver. err = n.Create(clientType) if err != nil { return err } reverter.Add(func() { _ = n.Delete(clientType) }) // Only start networks when not doing a cluster pre-join phase (this ensures that networks are only started // once the node has fully joined the clustered database and has consistent config with rest of the nodes). if clientType != clusterRequest.ClientTypeJoiner { err = n.Start() if err != nil { return err } } // Mark local as status as networkCreated. err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.NetworkNodeCreated(n.ID()) }) if err != nil { return err } logger.Debug("Marked network local status as created", logger.Ctx{"project": n.Project(), "network": n.Name()}) reverter.Success() return nil } // swagger:operation GET /1.0/networks/{name} networks network_get // // Get the network // // Gets a specific network. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Network // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Network" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkGet(d *Daemon, r *http.Request) response.Response { s := d.State() // If a target was specified, forward the request to the relevant node. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } allNodes := false if s.ServerClustered && request.QueryParam(r, "target") == "" { allNodes = true } n, err := doNetworkGet(s, r, allNodes, projectName, reqProject.Config, networkName) if err != nil { return response.SmartError(err) } etag := []any{n.Name, n.Managed, n.Type, n.Description, n.Config} return response.SyncResponseETag(true, &n, etag) } // canAccessNetwork checks if the network can be viewed/accessed by the user. func canAccessNetwork(s *state.State, r *http.Request, projectName string, projectConfig map[string]string, networkName string, managed bool) (bool, error) { // Don't allow retrieving info about the local server interfaces when not using default project. if projectName != api.ProjectDefaultName && !managed { return false, nil } // Check for basic access. if managed { userHasPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanView, auth.ObjectTypeNetwork) if err != nil { return false, err } if !userHasPermission(auth.ObjectNetwork(projectName, networkName)) { return false, nil } } else { userHasResourcesPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanViewResources, auth.ObjectTypeServer) if err != nil { return false, err } if !userHasResourcesPermission(auth.ObjectServer()) { return false, nil } } // Check if project allows access to network. if !project.NetworkAllowed(projectConfig, networkName, managed) { return false, nil } return true, nil } // doNetworkGet returns information about the specified network. // If the network being requested is a managed network and allNodes is true then node specific config is removed. // Otherwise if allNodes is false then the network's local status is returned. func doNetworkGet(s *state.State, r *http.Request, allNodes bool, projectName string, reqProjectConfig map[string]string, networkName string) (api.Network, error) { // Ignore veth pairs (for performance reasons). if strings.HasPrefix(networkName, "veth") { return api.Network{}, api.StatusErrorf(http.StatusNotFound, "Network not found") } // Get some information. n, err := network.LoadByName(s, projectName, networkName) if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { return api.Network{}, fmt.Errorf("Failed loading network: %w", err) } // Validate access. ok, err := canAccessNetwork(s, r, projectName, reqProjectConfig, networkName, n != nil) if err != nil { return api.Network{}, err } if !ok { return api.Network{}, api.StatusErrorf(http.StatusNotFound, "Network not found") } // Get OS network details. osInfo, _ := net.InterfaceByName(networkName) // Quick check. if osInfo == nil && n == nil { return api.Network{}, api.StatusErrorf(http.StatusNotFound, "Network not found") } // Prepare the response. apiNet := api.Network{} apiNet.Name = networkName apiNet.UsedBy = []string{} apiNet.Config = map[string]string{} apiNet.Project = projectName // Set the device type as needed. if n != nil { apiNet.Managed = true apiNet.Description = n.Description() apiNet.Type = n.Type() err = s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectNetwork(projectName, networkName), auth.EntitlementCanEdit) if err == nil { // Only allow admins to see network config as sensitive info can be stored there. apiNet.Config = n.Config() } else if !api.StatusErrorCheck(err, http.StatusForbidden) { return api.Network{}, err } // If no member is specified, we omit the node-specific fields. if allNodes { apiNet.Config = db.StripNodeSpecificNetworkConfig(apiNet.Config) } } else if osInfo != nil && int(osInfo.Flags&net.FlagLoopback) > 0 { apiNet.Type = "loopback" } else if util.PathExists(fmt.Sprintf("/sys/class/net/%s/bridge", apiNet.Name)) { apiNet.Type = "bridge" } else if util.PathExists(fmt.Sprintf("/proc/net/vlan/%s", apiNet.Name)) { apiNet.Type = "vlan" } else if util.PathExists(fmt.Sprintf("/sys/class/net/%s/device", apiNet.Name)) { apiNet.Type = "physical" } else if util.PathExists(fmt.Sprintf("/sys/class/net/%s/bonding", apiNet.Name)) { apiNet.Type = "bond" } else { vswitch, err := s.OVS() if err != nil { return api.Network{}, fmt.Errorf("Failed to connect to OVS: %w", err) } _, err = vswitch.GetBridge(context.TODO(), apiNet.Name) if err == nil { apiNet.Type = "bridge" } else { apiNet.Type = "unknown" } } // Look for instances using the interface. if apiNet.Type != "loopback" { var networkID int64 if n != nil { networkID = n.ID() } usedBy, err := network.UsedBy(s, projectName, networkID, apiNet.Name, apiNet.Type, false) if err != nil { return api.Network{}, err } apiNet.UsedBy = project.FilterUsedBy(s.Authorizer, r, usedBy) } if n != nil { if allNodes { apiNet.Status = n.Status() } else { apiNet.Status = n.LocalStatus() } apiNet.Locations = n.Locations() } return apiNet, nil } // swagger:operation DELETE /1.0/networks/{name} networks network_delete // // Delete the network // // Removes the network. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkDelete(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } // Get the existing network. n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) clusterNotification := isClusterNotification(r) if !clusterNotification { // Quick checks. inUse, err := n.IsUsed(false) if err != nil { return response.SmartError(err) } if inUse { return response.BadRequest(errors.New("The network is currently in use")) } } if n.LocalStatus() != api.NetworkStatusPending { err = n.Delete(clientType) if err != nil { return response.InternalError(err) } } // If this is a cluster notification, we're done, any database work will be done by the node that is // originally serving the request. if clusterNotification { return response.EmptySyncResponse } // If we are clustered, also notify all other nodes, if any. if s.ServerClustered { notifier, err := cluster.NewNotifier(s, s.Endpoints.NetworkCert(), s.ServerCert(), cluster.NotifyAll) if err != nil { return response.SmartError(err) } err = notifier(func(client incus.InstanceServer) error { return client.UseProject(n.Project()).DeleteNetwork(n.Name()) }) if err != nil { return response.SmartError(err) } } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Remove the network from the database. err = tx.DeleteNetwork(ctx, n.Project(), n.Name()) return err }) if err != nil { return response.SmartError(err) } err = s.Authorizer.DeleteNetwork(r.Context(), projectName, networkName) if err != nil { logger.Error("Failed to remove network from authorizer", logger.Ctx{"name": networkName, "project": projectName, "error": err}) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(projectName, lifecycle.NetworkDeleted.Event(n, requestor, nil)) return response.EmptySyncResponse } // swagger:operation POST /1.0/networks/{name} networks network_post // // Rename the network // // Renames an existing network. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: network // description: Network rename request // required: true // schema: // $ref: "#/definitions/NetworkPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkPost(d *Daemon, r *http.Request) response.Response { s := d.State() // FIXME: renaming a network is currently not supported in clustering // mode. The difficulty is that network.Start() depends on the // network having already been renamed in the database, which is // a chicken-and-egg problem for cluster notifications (the // serving node should typically do the database job, so the // network is not yet renamed in the db when the notified node // runs network.Start). if s.ServerClustered { return response.BadRequest(errors.New("Renaming clustered network not supported")) } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } req := api.NetworkPost{} // Parse the request. err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Get the existing network. n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } if n.Status() != api.NetworkStatusCreated { return response.BadRequest(errors.New("Cannot rename network when not in created state")) } // Ensure new name is supplied. if req.Name == "" { return response.BadRequest(errors.New("New network name not provided")) } // Perform generic name validation. err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid network name: %w", err)) } // Perform driver-specific name validation. err = n.ValidateName(req.Name) if err != nil { return response.BadRequest(fmt.Errorf("Invalid network name: %w", err)) } // Check network isn't in use. inUse, err := n.IsUsed(false) if err != nil { return response.InternalError(fmt.Errorf("Failed checking network in use: %w", err)) } if inUse { return response.BadRequest(errors.New("Network is currently in use")) } var networks []string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Check that the name isn't already in used by an existing managed network. networks, err = tx.GetNetworks(ctx, projectName) return err }) if err != nil { return response.InternalError(err) } if slices.Contains(networks, req.Name) { return response.Conflict(fmt.Errorf("Network %q already exists", req.Name)) } // Rename it. err = n.Rename(req.Name) if err != nil { return response.SmartError(err) } err = s.Authorizer.RenameNetwork(r.Context(), projectName, networkName, req.Name) if err != nil { logger.Error("Failed to rename network in authorizer", logger.Ctx{"old_name": networkName, "new_name": req.Name, "project": projectName, "error": err}) } requestor := request.CreateRequestor(r) lc := lifecycle.NetworkRenamed.Event(n, requestor, map[string]any{"old_name": networkName}) s.Events.SendLifecycle(projectName, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation PUT /1.0/networks/{name} networks network_put // // Update the network // // Updates the entire network configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: network // description: Network configuration // required: true // schema: // $ref: "#/definitions/NetworkPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func networkPut(d *Daemon, r *http.Request) response.Response { s := d.State() // If a target was specified, forward the request to the relevant node. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } // Get the existing network. n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } targetNode := request.QueryParam(r, "target") if targetNode == "" && n.Status() != api.NetworkStatusCreated { return response.BadRequest(errors.New("Cannot update network global config when not in created state")) } // Duplicate config for etag modification and generation. etagConfig := localUtil.CopyConfig(n.Config()) // If no target node is specified and the daemon is clustered, we omit the node-specific fields so that // the e-tag can be generated correctly. This is because the GET request used to populate the request // will also remove node-specific keys when no target is specified. if targetNode == "" && s.ServerClustered { etagConfig = db.StripNodeSpecificNetworkConfig(etagConfig) } // Validate the ETag. etag := []any{n.Name(), n.IsManaged(), n.Type(), n.Description(), etagConfig} err = localUtil.EtagCheck(r, etag) if err != nil { return response.PreconditionFailed(err) } // Decode the request. req := api.NetworkPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) // In clustered mode, we differentiate between node specific and non-node specific config keys based on // whether the user has specified a target to apply the config to. if s.ServerClustered && clientType == clusterRequest.ClientTypeNormal { curConfig := n.Config() changedConfig := make(map[string]string, len(req.Config)) for key, value := range req.Config { if curConfig[key] == value { continue } changedConfig[key] = value } if targetNode == "" { // If no target is specified, then ensure only non-node-specific config keys are changed. for k := range changedConfig { if db.IsNodeSpecificNetworkConfig(k) { return response.BadRequest(fmt.Errorf("Config key %q is cluster member specific", k)) } } } else { // If a target is specified, then ensure only node-specific config keys are changed. for k := range changedConfig { if !db.IsNodeSpecificNetworkConfig(k) { return response.BadRequest(fmt.Errorf("Config key %q may not be used as member-specific key", k)) } } } } resp = doNetworkUpdate(n, req, targetNode, clientType, r.Method, s.ServerClustered) // Send a single update event when the server is clustered. if !s.ServerClustered || (s.ServerClustered && clientType == clusterRequest.ClientTypeNormal) { requestor := request.CreateRequestor(r) s.Events.SendLifecycle(projectName, lifecycle.NetworkUpdated.Event(n, requestor, nil)) } return resp } // swagger:operation PATCH /1.0/networks/{name} networks network_patch // // Partially update the network // // Updates a subset of the network configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: network // description: Network configuration // required: true // schema: // $ref: "#/definitions/NetworkPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func networkPatch(d *Daemon, r *http.Request) response.Response { return networkPut(d, r) } // doNetworkUpdate loads the current local network config, merges with the requested network config, validates // and applies the changes. Will also notify other cluster nodes of non-node specific config if needed. func doNetworkUpdate(n network.Network, req api.NetworkPut, targetNode string, clientType clusterRequest.ClientType, httpMethod string, clustered bool) response.Response { if req.Config == nil { req.Config = map[string]string{} } // Normally a "put" request will replace all existing config, however when clustered, we need to account // for the node specific config keys and not replace them when the request doesn't specify a specific node. if targetNode == "" && httpMethod != http.MethodPatch && clustered { // If non-node specific config being updated via "put" method in cluster, then merge the current // node-specific network config with the submitted config to allow validation. // This allows removal of non-node specific keys when they are absent from request config. for k, v := range n.Config() { if db.IsNodeSpecificNetworkConfig(k) { req.Config[k] = v } } } else if httpMethod == http.MethodPatch { // If config being updated via "patch" method, then merge all existing config with the keys that // are present in the request config. for k, v := range n.Config() { _, ok := req.Config[k] if !ok { req.Config[k] = v } } } // Validate the merged configuration. err := n.Validate(req.Config, clientType) if err != nil { return response.BadRequest(err) } // Apply the new configuration (will also notify other cluster nodes if needed). err = n.Update(req, targetNode, clientType) if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } // swagger:operation GET /1.0/networks/{name}/leases networks networks_leases_get // // Get the DHCP leases // // Returns a list of DHCP leases for the network. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of DHCP leases // items: // $ref: "#/definitions/NetworkLease" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkLeasesGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } // Attempt to load the network. n, err := network.LoadByName(s, projectName, networkName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. if !project.NetworkAllowed(reqProject.Config, networkName, n.IsManaged()) { return response.SmartError(api.StatusErrorf(http.StatusNotFound, "Network not found")) } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) leases, err := n.Leases(reqProject.Name, clientType) if err != nil { return response.SmartError(err) } return response.SyncResponse(true, leases) } func networkStartup(s *state.State) error { var err error // Get a list of projects. var projectNames []string err = s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { projectNames, err = dbCluster.GetProjectNames(ctx, tx.Tx()) return err }) if err != nil { return fmt.Errorf("Failed to load projects: %w", err) } // Build a list of networks to initialize, keyed by project and network name. const networkPriorityStandalone = 0 // Start networks not dependent on any other network first. const networkPriorityPhysical = 1 // Start networks dependent on physical interfaces second. const networkPriorityLogical = 2 // Start networks dependent logical networks third. initNetworks := []map[network.ProjectNetwork]struct{}{ networkPriorityStandalone: make(map[network.ProjectNetwork]struct{}), networkPriorityPhysical: make(map[network.ProjectNetwork]struct{}), networkPriorityLogical: make(map[network.ProjectNetwork]struct{}), } err = s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { for _, projectName := range projectNames { networkNames, err := tx.GetCreatedNetworkNamesByProject(ctx, projectName) if err != nil { return fmt.Errorf("Failed to load networks for project %q: %w", projectName, err) } for _, networkName := range networkNames { pn := network.ProjectNetwork{ ProjectName: projectName, NetworkName: networkName, } // Assume all networks are networkPriorityStandalone initially. initNetworks[networkPriorityStandalone][pn] = struct{}{} } } return nil }) if err != nil { return err } loadedNetworks := make(map[network.ProjectNetwork]network.Network) initNetwork := func(n network.Network, priority int) error { err = n.Start() if err != nil { err = fmt.Errorf("Failed starting: %w", err) _ = s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpsertWarningLocalNode(ctx, n.Project(), dbCluster.TypeNetwork, int(n.ID()), warningtype.NetworkUnvailable, err.Error()) }) return err } logger.Info("Initialized network", logger.Ctx{"project": n.Project(), "name": n.Name()}) // Network initialized successfully so remove it from the list so its not retried. pn := network.ProjectNetwork{ ProjectName: n.Project(), NetworkName: n.Name(), } delete(initNetworks[priority], pn) _ = warnings.ResolveWarningsByLocalNodeAndProjectAndTypeAndEntity(s.DB.Cluster, n.Project(), warningtype.NetworkUnvailable, dbCluster.TypeNetwork, int(n.ID())) return nil } loadAndInitNetwork := func(pn network.ProjectNetwork, priority int, firstPass bool) error { var err error var n network.Network if firstPass && loadedNetworks[pn] != nil { // Check if network already loaded from during first pass phase. n = loadedNetworks[pn] } else { n, err = network.LoadByName(s, pn.ProjectName, pn.NetworkName) if err != nil { if api.StatusErrorCheck(err, http.StatusNotFound) { // Network has been deleted since we began trying to start it so delete // entry. delete(initNetworks[priority], pn) return nil } return fmt.Errorf("Failed loading: %w", err) } } netConfig := n.Config() err = n.Validate(netConfig, clusterRequest.ClientTypeNormal) if err != nil { return fmt.Errorf("Failed validating: %w", err) } // Update network start priority based on dependencies. if netConfig["parent"] != "" && priority != networkPriorityPhysical { // Start networks that depend on physical interfaces existing after // non-dependent networks. delete(initNetworks[priority], pn) initNetworks[networkPriorityPhysical][pn] = struct{}{} return nil } else if netConfig["network"] != "" && priority != networkPriorityLogical { // Start networks that depend on other logical networks after networks after // non-dependent networks and networks that depend on physical interfaces. delete(initNetworks[priority], pn) initNetworks[networkPriorityLogical][pn] = struct{}{} return nil } err = initNetwork(n, priority) if err != nil { return err } return nil } // Try initializing networks in priority order. for priority := range initNetworks { for pn := range initNetworks[priority] { err := loadAndInitNetwork(pn, priority, true) if err != nil { logger.Error("Failed initializing network", logger.Ctx{"project": pn.ProjectName, "network": pn.NetworkName, "err": err}) continue } } } loadedNetworks = nil // Don't store loaded networks after first pass. remainingNetworks := 0 for _, networks := range initNetworks { remainingNetworks += len(networks) } // For any remaining networks that were not successfully initialized, we now start a go routine to // periodically try to initialize them again in the background. if remainingNetworks > 0 { go func() { for { t := time.NewTimer(time.Duration(time.Minute)) select { case <-s.ShutdownCtx.Done(): t.Stop() return case <-t.C: t.Stop() tryInstancesStart := false // Try initializing networks in priority order. for priority := range initNetworks { for pn := range initNetworks[priority] { err := loadAndInitNetwork(pn, priority, false) if err != nil { logger.Error("Failed initializing network", logger.Ctx{"project": pn.ProjectName, "network": pn.NetworkName, "err": err}) continue } tryInstancesStart = true // We initialized at least one network. } } remainingNetworks := 0 for _, networks := range initNetworks { remainingNetworks += len(networks) } if remainingNetworks <= 0 { logger.Info("All networks initialized") } // At least one remaining network was initialized, check if any instances // can now start. if tryInstancesStart { instances, err := instance.LoadNodeAll(s, instancetype.Any) if err != nil { logger.Warn("Failed loading instances to start", logger.Ctx{"err": err}) } else { instancesStart(s, instances) } } if remainingNetworks <= 0 { return // Our job here is done. } } } }() } else { logger.Info("All networks initialized") } return nil } func networkShutdown(s *state.State) { var err error // Get a list of projects. var projectNames []string err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { projectNames, err = dbCluster.GetProjectNames(ctx, tx.Tx()) return err }) if err != nil { logger.Error("Failed shutting down networks, couldn't load projects", logger.Ctx{"err": err}) return } for _, projectName := range projectNames { var networks []string err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get a list of managed networks. networks, err = tx.GetNetworks(ctx, projectName) return err }) if err != nil { logger.Error("Failed shutting down networks, couldn't load networks for project", logger.Ctx{"project": projectName, "err": err}) continue } // Bring them all down. for _, name := range networks { n, err := network.LoadByName(s, projectName, name) if err != nil { logger.Error("Failed shutting down network, couldn't load network", logger.Ctx{"network": name, "project": projectName, "err": err}) continue } err = n.Stop() if err != nil { logger.Error("Failed to bring down network", logger.Ctx{"err": err, "project": projectName, "name": name}) } } } } // networkRestartOVN is used to trigger a restart of all OVN networks. func networkRestartOVN(s *state.State) error { logger.Infof("Restarting OVN networks") // Get a list of projects. var projectNames []string var err error err = s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { projectNames, err = dbCluster.GetProjectNames(ctx, tx.Tx()) return err }) if err != nil { return fmt.Errorf("Failed to load projects: %w", err) } // Go over all the networks in every project. for _, projectName := range projectNames { var networkNames []string err := s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { networkNames, err = tx.GetCreatedNetworkNamesByProject(ctx, projectName) return err }) if err != nil { return fmt.Errorf("Failed to load networks for project %q: %w", projectName, err) } for _, networkName := range networkNames { // Load the network struct. n, err := network.LoadByName(s, projectName, networkName) if err != nil { return fmt.Errorf("Failed to load network %q in project %q: %w", networkName, projectName, err) } // Skip non-OVN networks. if n.DBType() != db.NetworkTypeOVN { continue } // Restart the network. err = n.Start() if err != nil { return fmt.Errorf("Failed to restart network %q in project %q: %w", networkName, projectName, err) } } } return nil } // swagger:operation GET /1.0/networks/{name}/state networks networks_state_get // // Get the network state // // Returns the current network state information. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Network name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/NetworkState" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func networkStateGet(d *Daemon, r *http.Request) response.Response { s := d.State() // If a target was specified, forward the request to the relevant node. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, reqProject, err := project.NetworkProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } networkName, err := url.PathUnescape(mux.Vars(r)["networkName"]) if err != nil { return response.SmartError(err) } n, err := network.LoadByName(s, projectName, networkName) if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { return response.SmartError(fmt.Errorf("Failed loading network: %w", err)) } // Check if project allows access to network. ok, err := canAccessNetwork(s, r, projectName, reqProject.Config, networkName, n != nil) if err != nil { return response.SmartError(err) } if !ok { return response.NotFound(errors.New("Network not found")) } var state *api.NetworkState if n != nil { state, err = n.State() if err != nil { return response.SmartError(err) } } else { state, err = resources.GetNetworkState(networkName) if err != nil { return response.SmartError(err) } } return response.SyncResponse(true, state) } incus-7.0.0/cmd/incusd/networks_utils.go000066400000000000000000000021471517523235500203260ustar00rootroot00000000000000package main import ( "slices" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/logger" ) var networkOVNChassis *bool // networkUpdateOVNChassis gets called on heartbeats to check if OVN needs reconfiguring. func networkUpdateOVNChassis(s *state.State, heartbeatData *cluster.APIHeartbeat, localAddress string) error { // Check if we have at least one active OVN chassis. hasOVNChassis := false localOVNChassis := false for _, n := range heartbeatData.Members { if slices.Contains(n.Roles, db.ClusterRoleOVNChassis) { if n.Address == localAddress { localOVNChassis = true } hasOVNChassis = true } } runChassis := !hasOVNChassis || localOVNChassis if networkOVNChassis != nil && *networkOVNChassis != runChassis { // Detected that the local OVN chassis setup may be incorrect, restarting. err := networkRestartOVN(s) if err != nil { logger.Error("Error restarting OVN networks", logger.Ctx{"err": err}) } } networkOVNChassis = &runChassis return nil } incus-7.0.0/cmd/incusd/operations.go000066400000000000000000001017751517523235500174240ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "net/http" "net/url" "strconv" "strings" "time" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/server/task" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) var operationCmd = APIEndpoint{ Path: "operations/{id}", Delete: APIEndpointAction{Handler: operationDelete, AccessHandler: allowAuthenticated}, Get: APIEndpointAction{Handler: operationGet, AccessHandler: allowAuthenticated}, } var operationsCmd = APIEndpoint{ Path: "operations", Get: APIEndpointAction{Handler: operationsGet, AccessHandler: allowAuthenticated}, } var operationWait = APIEndpoint{ Path: "operations/{id}/wait", Get: APIEndpointAction{Handler: operationWaitGet, AllowUntrusted: true}, } var operationWebsocket = APIEndpoint{ Path: "operations/{id}/websocket", Get: APIEndpointAction{Handler: operationWebsocketGet, AllowUntrusted: true}, } // waitForOperations waits for operations to finish. // There's a timeout for console/exec operations that when reached will shut down the instances forcefully. func waitForOperations(ctx context.Context, cluster *db.Cluster, consoleShutdownTimeout time.Duration) { timeout := time.After(consoleShutdownTimeout) defer func() { _ = cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { err := dbCluster.DeleteOperations(ctx, tx.Tx(), cluster.GetNodeID()) if err != nil { logger.Error("Failed cleaning up operations") } return nil }) }() // Check operation status every second. tick := time.NewTicker(time.Second) defer tick.Stop() var i int for { // Get all the operations ops := operations.Clone() var runningOps, execConsoleOps int for _, op := range ops { if op.Status() != api.Running || op.Class() == operations.OperationClassToken { continue } runningOps++ opType := op.Type() if opType == operationtype.CommandExec || opType == operationtype.ConsoleShow { execConsoleOps++ } _, opAPI, err := op.Render() if err != nil { logger.Warn("Failed to render operation", logger.Ctx{"operation": op, "err": err}) } else if opAPI.MayCancel { _, _ = op.Cancel() } } // No more running operations left. Exit function. if runningOps == 0 { logger.Info("All running operations finished, shutting down") return } // Print log message every minute. if i%60 == 0 { logger.Infof("Waiting for %d operation(s) to finish", runningOps) } i++ select { case <-timeout: // We wait up to core.shutdown_timeout minutes for exec/console operations to finish. // If there are still running operations, we continue shutdown which will stop any running // instances and terminate the operations. if execConsoleOps > 0 { logger.Info("Shutdown timeout reached, continuing with shutdown") } return case <-ctx.Done(): // Return here, and ignore any running operations. logger.Info("Forcing shutdown, ignoring running operations") return case <-tick.C: } } } // API functions // swagger:operation GET /1.0/operations/{id} operations operation_get // // Get the operation state // // Gets the operation state. // // --- // produces: // - application/json // parameters: // - in: path // name: id // description: Operation ID // type: string // required: true // responses: // "200": // description: Operation // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Operation" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func operationGet(d *Daemon, r *http.Request) response.Response { s := d.State() id, err := url.PathUnescape(mux.Vars(r)["id"]) if err != nil { return response.SmartError(err) } var body *api.Operation // First check if the query is for a local operation from this node op, err := operations.OperationGetInternal(id) if err == nil { _, body, err = op.Render() if err != nil { return response.SmartError(err) } return response.SyncResponse(true, body) } // Then check if the query is from an operation on another node, and, if so, forward it var address string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { filter := dbCluster.OperationFilter{UUID: &id} ops, err := dbCluster.GetOperations(ctx, tx.Tx(), filter) if err != nil { return err } if len(ops) < 1 { return api.StatusErrorf(http.StatusNotFound, "Operation not found") } if len(ops) > 1 { return errors.New("More than one operation matches") } operation := ops[0] address = operation.NodeAddress return nil }) if err != nil { return response.SmartError(err) } client, err := cluster.Connect(address, s.Endpoints.NetworkCert(), s.ServerCert(), r, false) if err != nil { return response.SmartError(err) } return response.ForwardedResponse(client, r) } // swagger:operation DELETE /1.0/operations/{id} operations operation_delete // // Cancel the operation // // Cancels the operation if supported. // // --- // produces: // - application/json // parameters: // - in: path // name: id // description: Operation ID // type: string // required: true // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func operationDelete(d *Daemon, r *http.Request) response.Response { s := d.State() id, err := url.PathUnescape(mux.Vars(r)["id"]) if err != nil { return response.SmartError(err) } // First check if the query is for a local operation from this node op, err := operations.OperationGetInternal(id) if err == nil { projectName := op.Project() if projectName == "" { projectName = api.ProjectDefaultName } objectType, entitlement := op.Permission() if objectType != "" { for _, v := range op.Resources() { for _, u := range v { // When dealing with specific objects, get the arguments from the URL. var pathArgs []string if objectType != auth.ObjectTypeProject { var err error _, _, _, pathArgs, err = dbCluster.URLToEntityType(u.String()) if err != nil { return response.InternalError(fmt.Errorf("Unable to parse operation resource URL: %w", err)) } } // Check that the access is allowed. object, err := auth.NewObject(objectType, projectName, pathArgs...) if err != nil { return response.InternalError(fmt.Errorf("Unable to create authorization object for operation: %w", err)) } err = s.Authorizer.CheckPermission(r.Context(), r, object, entitlement) if err != nil { return response.SmartError(err) } } } } _, err = op.Cancel() if err != nil { return response.BadRequest(err) } s.Events.SendLifecycle(projectName, lifecycle.OperationCancelled.Event(op, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // Then check if the query is from an operation on another node, and, if so, forward it var address string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { filter := dbCluster.OperationFilter{UUID: &id} ops, err := dbCluster.GetOperations(ctx, tx.Tx(), filter) if err != nil { return err } if len(ops) < 1 { return api.StatusErrorf(http.StatusNotFound, "Operation not found") } if len(ops) > 1 { return errors.New("More than one operation matches") } operation := ops[0] address = operation.NodeAddress return nil }) if err != nil { return response.SmartError(err) } client, err := cluster.Connect(address, s.Endpoints.NetworkCert(), s.ServerCert(), r, false) if err != nil { return response.SmartError(err) } return response.ForwardedResponse(client, r) } // operationCancel cancels an operation that exists on any member. func operationCancel(s *state.State, r *http.Request, projectName string, op *api.Operation) error { // Check if operation is local and if so, cancel it. localOp, _ := operations.OperationGetInternal(op.ID) if localOp != nil { if localOp.Status() == api.Running { _, err := localOp.Cancel() if err != nil { return fmt.Errorf("Failed to cancel local operation %q: %w", op.ID, err) } } s.Events.SendLifecycle(projectName, lifecycle.OperationCancelled.Event(localOp, request.CreateRequestor(r), nil)) return nil } // If not found locally, try connecting to remote member to delete it. var memberAddress string var err error err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { filter := dbCluster.OperationFilter{UUID: &op.ID} ops, err := dbCluster.GetOperations(ctx, tx.Tx(), filter) if err != nil { return fmt.Errorf("Failed loading operation %q: %w", op.ID, err) } if len(ops) < 1 { return api.StatusErrorf(http.StatusNotFound, "Operation not found") } if len(ops) > 1 { return errors.New("More than one operation matches") } operation := ops[0] memberAddress = operation.NodeAddress return nil }) if err != nil { return err } client, err := cluster.Connect(memberAddress, s.Endpoints.NetworkCert(), s.ServerCert(), r, true) if err != nil { return fmt.Errorf("Failed to connect to %q: %w", memberAddress, err) } err = client.UseProject(projectName).DeleteOperation(op.ID) if err != nil { return fmt.Errorf("Failed to delete remote operation %q on %q: %w", op.ID, memberAddress, err) } return nil } // swagger:operation GET /1.0/operations operations operations_get // // Get the operations // // Returns a JSON object of operation type to operation list (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve operations from all projects // type: boolean // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: object // additionalProperties: // type: array // items: // type: string // description: JSON object of operation types to operation URLs // example: |- // { // "running": [ // "/1.0/operations/6916c8a6-9b7d-4abd-90b3-aedfec7ec7da" // ] // } // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/operations?recursion=1 operations operations_get_recursion1 // // Get the operations // // Returns a list of operations (structs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve operations from all projects // type: boolean // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of operations // items: // $ref: "#/definitions/Operation" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func operationsGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.QueryParam(r, "project") allProjects := util.IsTrue(request.QueryParam(r, "all-projects")) recursion := localUtil.IsRecursionRequest(r) if allProjects && projectName != "" { return response.SmartError( api.StatusErrorf(http.StatusBadRequest, "Cannot specify a project when requesting all projects"), ) } else if !allProjects && projectName == "" { projectName = api.ProjectDefaultName } userHasPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanViewOperations, auth.ObjectTypeProject) if err != nil { return response.InternalError(fmt.Errorf("Failed to get operation permission checker: %w", err)) } localOperationURLs := func() (jmap.Map, error) { // Get all the operations. localOps := operations.Clone() // Build a list of URLs. body := jmap.Map{} for _, v := range localOps { if !allProjects && v.Project() != "" && v.Project() != projectName { continue } if !userHasPermission(auth.ObjectProject(v.Project())) { continue } status := strings.ToLower(v.Status().String()) _, ok := body[status] if !ok { body[status] = make([]string, 0) } body[status] = append(body[status].([]string), v.URL()) } return body, nil } localOperations := func() (jmap.Map, error) { // Get all the operations. localOps := operations.Clone() // Build a list of operations. body := jmap.Map{} for _, v := range localOps { if !allProjects && v.Project() != "" && v.Project() != projectName { continue } if !userHasPermission(auth.ObjectProject(v.Project())) { continue } status := strings.ToLower(v.Status().String()) _, ok := body[status] if !ok { body[status] = make([]*api.Operation, 0) } _, op, err := v.Render() if err != nil { return nil, err } body[status] = append(body[status].([]*api.Operation), op) } return body, nil } // Check if called from a cluster node. if isClusterNotification(r) { // Only return the local data. if recursion { // Recursive queries. body, err := localOperations() if err != nil { return response.InternalError(err) } return response.SyncResponse(true, body) } // Normal queries body, err := localOperationURLs() if err != nil { return response.InternalError(err) } return response.SyncResponse(true, body) } // Start with local operations. var md jmap.Map if recursion { md, err = localOperations() if err != nil { return response.InternalError(err) } } else { md, err = localOperationURLs() if err != nil { return response.InternalError(err) } } // If not clustered, then just return local operations. if !s.ServerClustered { return response.SyncResponse(true, md) } // Get all nodes with running operations in this project. var membersWithOps []string var members []db.NodeInfo err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error if allProjects { membersWithOps, err = tx.GetAllNodesWithOperations(ctx) } else { membersWithOps, err = tx.GetNodesWithOperations(ctx, projectName) } if err != nil { return fmt.Errorf("Failed getting members with operations: %w", err) } members, err = tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } return nil }) if err != nil { return response.SmartError(err) } // Get local address. localClusterAddress := s.LocalConfig.ClusterAddress() offlineThreshold := s.GlobalConfig.OfflineThreshold() memberOnline := func(memberAddress string) bool { for _, member := range members { if member.Address == memberAddress { if member.IsOffline(offlineThreshold) { logger.Warn("Excluding offline member from operations list", logger.Ctx{"member": member.Name, "address": member.Address, "ID": member.ID, "lastHeartbeat": member.Heartbeat}) return false } return true } } return false } networkCert := s.Endpoints.NetworkCert() for _, memberAddress := range membersWithOps { if memberAddress == localClusterAddress { continue } if !memberOnline(memberAddress) { continue } // Connect to the remote server. Use notify=true to only get local operations on remote member. client, err := cluster.Connect(memberAddress, networkCert, s.ServerCert(), r, true) if err != nil { return response.SmartError(fmt.Errorf("Failed connecting to member %q: %w", memberAddress, err)) } // Get operation data. var ops []api.Operation if allProjects { ops, err = client.GetOperationsAllProjects() } else { ops, err = client.UseProject(projectName).GetOperations() } if err != nil { logger.Warn("Failed getting operations from member", logger.Ctx{"address": memberAddress, "err": err}) continue } // Merge with existing data. for _, o := range ops { op := o // Local var for pointer. status := strings.ToLower(op.Status) _, ok := md[status] if !ok { if recursion { md[status] = make([]*api.Operation, 0) } else { md[status] = make([]string, 0) } } if recursion { md[status] = append(md[status].([]*api.Operation), &op) } else { md[status] = append(md[status].([]string), fmt.Sprintf("/1.0/operations/%s", op.ID)) } } } return response.SyncResponse(true, md) } // operationsGetByType gets all operations for a project and type. func operationsGetByType(s *state.State, r *http.Request, projectName string, opType operationtype.Type) ([]*api.Operation, error) { ops := make([]*api.Operation, 0) // Get local operations for project. for _, op := range operations.Clone() { if op.Project() != projectName || op.Type() != opType { continue } _, apiOp, err := op.Render() if err != nil { return nil, fmt.Errorf("Failed converting local operation %q to API representation: %w", op.ID(), err) } ops = append(ops, apiOp) } // Return just local operations if not clustered. if !s.ServerClustered { return ops, nil } // Get all operations of the specified type in project. var members []db.NodeInfo memberOps := make(map[string]map[string]dbCluster.Operation) err := s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error members, err = tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } ops, err := tx.GetOperationsOfType(ctx, projectName, opType) if err != nil { return fmt.Errorf("Failed getting operations for project %q and type %d: %w", projectName, opType, err) } // Group operations by member address and UUID. for _, op := range ops { if memberOps[op.NodeAddress] == nil { memberOps[op.NodeAddress] = make(map[string]dbCluster.Operation) } memberOps[op.NodeAddress][op.UUID] = op } return nil }) if err != nil { return nil, err } // Get local address. localClusterAddress := s.LocalConfig.ClusterAddress() offlineThreshold := s.GlobalConfig.OfflineThreshold() memberOnline := func(memberAddress string) bool { for _, member := range members { if member.Address == memberAddress { if member.IsOffline(offlineThreshold) { logger.Warn("Excluding offline member from operations by type list", logger.Ctx{"member": member.Name, "address": member.Address, "ID": member.ID, "lastHeartbeat": member.Heartbeat, "opType": opType}) return false } return true } } return false } networkCert := s.Endpoints.NetworkCert() serverCert := s.ServerCert() for memberAddress := range memberOps { if memberAddress == localClusterAddress { continue } if !memberOnline(memberAddress) { continue } // Connect to the remote server. Use notify=true to only get local operations on remote member. client, err := cluster.Connect(memberAddress, networkCert, serverCert, r, true) if err != nil { return nil, fmt.Errorf("Failed connecting to member %q: %w", memberAddress, err) } // Get all remote operations in project. remoteOps, err := client.UseProject(projectName).GetOperations() if err != nil { logger.Warn("Failed getting operations from member", logger.Ctx{"address": memberAddress, "err": err}) continue } for _, o := range remoteOps { op := o // Local var for pointer. // Exclude remote operations that don't have the desired type. if memberOps[memberAddress][op.ID].Type != opType { continue } ops = append(ops, &op) } } return ops, nil } // swagger:operation GET /1.0/operations/{id}/wait?public operations operation_wait_get_untrusted // // Wait for the operation // // Waits for the operation to reach a final state (or timeout) and retrieve its final state. // // When accessed by an untrusted user, the secret token must be provided. // // --- // produces: // - application/json // parameters: // - in: path // name: id // description: Operation ID // type: string // required: true // - in: query // name: secret // description: Authentication token // type: string // example: random-string // - in: query // name: timeout // description: Timeout in seconds (-1 means never) // type: integer // example: -1 // responses: // "200": // description: Operation // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Operation" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/operations/{id}/wait operations operation_wait_get // // Wait for the operation // // Waits for the operation to reach a final state (or timeout) and retrieve its final state. // // --- // produces: // - application/json // parameters: // - in: path // name: id // description: Operation ID // type: string // required: true // - in: query // name: timeout // description: Timeout in seconds (-1 means never) // type: integer // example: -1 // responses: // "200": // description: Operation // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Operation" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func operationWaitGet(d *Daemon, r *http.Request) response.Response { s := d.State() id, err := url.PathUnescape(mux.Vars(r)["id"]) if err != nil { return response.SmartError(err) } secret := r.FormValue("secret") trusted, _, _, _ := d.Authenticate(nil, r) if !trusted && secret == "" { return response.Forbidden(nil) } timeoutSecs := -1 if r.FormValue("timeout") != "" { timeoutSecs, err = strconv.Atoi(r.FormValue("timeout")) if err != nil { return response.InternalError(err) } } // First check if the query is for a local operation from this node op, err := operations.OperationGetInternal(id) if err == nil { if secret != "" && op.Metadata()["secret"] != secret { return response.Forbidden(nil) } var ctx context.Context var cancel context.CancelFunc // If timeout is -1, it will wait indefinitely otherwise it will timeout after timeoutSecs. if timeoutSecs > -1 { ctx, cancel = context.WithDeadline(r.Context(), time.Now().Add(time.Second*time.Duration(timeoutSecs))) } else { ctx, cancel = context.WithCancel(r.Context()) } waitResponse := func(w http.ResponseWriter) error { defer cancel() // Write header to avoid client side timeouts. w.Header().Set("Connection", "keep-alive") w.Header().Set("Content-Type", "application/json") w.Header().Set("X-Content-Type-Options", "nosniff") w.WriteHeader(http.StatusOK) f, ok := w.(http.Flusher) if ok { f.Flush() } // Wait for the operation. _ = op.Wait(ctx) // Render the current state. _, body, err := op.Render() if err != nil { _ = response.SmartError(err).Render(w) return nil } _ = response.SyncResponse(true, body).Render(w) return nil } return response.ManualResponse(waitResponse) } // Then check if the query is from an operation on another node, and, if so, forward it var address string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { filter := dbCluster.OperationFilter{UUID: &id} ops, err := dbCluster.GetOperations(ctx, tx.Tx(), filter) if err != nil { return err } if len(ops) < 1 { return api.StatusErrorf(http.StatusNotFound, "Operation not found") } if len(ops) > 1 { return errors.New("More than one operation matches") } operation := ops[0] address = operation.NodeAddress return nil }) if err != nil { return response.SmartError(err) } client, err := cluster.Connect(address, s.Endpoints.NetworkCert(), s.ServerCert(), r, false) if err != nil { return response.SmartError(err) } return response.ForwardedResponse(client, r) } type operationWebSocket struct { req *http.Request op *operations.Operation } func (r *operationWebSocket) Render(w http.ResponseWriter) error { chanErr, err := r.op.Connect(r.req, w) if err != nil { return err } err = <-chanErr return err } func (r *operationWebSocket) String() string { _, md, err := r.op.Render() if err != nil { return fmt.Sprintf("error: %s", err) } return md.ID } // Code returns the HTTP code. func (r *operationWebSocket) Code() int { return http.StatusOK } // swagger:operation GET /1.0/operations/{id}/websocket?public operations operation_websocket_get_untrusted // // Get the websocket stream // // Connects to an associated websocket stream for the operation. // This should almost never be done directly by a client, instead it's // meant for server to server communication with the client only relaying the // connection information to the servers. // // The untrusted endpoint is used by the target server to connect to the source server. // Authentication is performed through the secret token. // // --- // produces: // - application/json // parameters: // - in: path // name: id // description: Operation ID // type: string // required: true // - in: query // name: secret // description: Authentication token // type: string // example: random-string // responses: // "200": // description: Websocket operation messages (dependent on operation) // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/operations/{id}/websocket operations operation_websocket_get // // Get the websocket stream // // Connects to an associated websocket stream for the operation. // This should almost never be done directly by a client, instead it's // meant for server to server communication with the client only relaying the // connection information to the servers. // // --- // produces: // - application/json // parameters: // - in: path // name: id // description: Operation ID // type: string // required: true // - in: query // name: secret // description: Authentication token // type: string // example: random-string // responses: // "200": // description: Websocket operation messages (dependent on operation) // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func operationWebsocketGet(d *Daemon, r *http.Request) response.Response { s := d.State() id, err := url.PathUnescape(mux.Vars(r)["id"]) if err != nil { return response.SmartError(err) } // First check if the query is for a local operation from this node op, err := operations.OperationGetInternal(id) if err == nil { return &operationWebSocket{r, op} } // Then check if the query is from an operation on another node, and, if so, forward it secret := r.FormValue("secret") if secret == "" { return response.BadRequest(errors.New("Missing websocket secret")) } var address string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { filter := dbCluster.OperationFilter{UUID: &id} ops, err := dbCluster.GetOperations(ctx, tx.Tx(), filter) if err != nil { return err } if len(ops) < 1 { return api.StatusErrorf(http.StatusNotFound, "Operation not found") } if len(ops) > 1 { return errors.New("More than one operation matches") } operation := ops[0] address = operation.NodeAddress return nil }) if err != nil { return response.SmartError(err) } client, err := cluster.Connect(address, s.Endpoints.NetworkCert(), s.ServerCert(), r, false) if err != nil { return response.SmartError(err) } source, err := client.GetOperationWebsocket(id, secret) if err != nil { return response.SmartError(err) } return operations.ForwardedOperationWebSocket(r, id, source) } func autoRemoveOrphanedOperationsTask(s *state.State) (task.Func, task.Schedule) { f := func(ctx context.Context) { localClusterAddress := s.LocalConfig.ClusterAddress() leader, err := s.Cluster.LeaderAddress() if err != nil { if errors.Is(err, cluster.ErrNodeIsNotClustered) { return // No error if not clustered. } logger.Error("Failed to get leader cluster member address", logger.Ctx{"err": err}) return } if localClusterAddress != leader { logger.Debug("Skipping remove orphaned operations task since we're not leader") return } opRun := func(op *operations.Operation) error { return autoRemoveOrphanedOperations(ctx, s) } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.RemoveOrphanedOperations, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating remove orphaned operations operation", logger.Ctx{"err": err}) return } err = op.Start() if err != nil { logger.Error("Failed starting remove orphaned operations operation", logger.Ctx{"err": err}) return } err = op.Wait(ctx) if err != nil { logger.Error("Failed removing orphaned operations", logger.Ctx{"err": err}) return } } return f, task.Hourly() } // autoRemoveOrphanedOperations removes old operations from offline members. Operations can be left // behind if a cluster member abruptly becomes unreachable. If the affected cluster members comes // back online, these operations won't be cleaned up. We therefore need to periodically clean up // such operations. func autoRemoveOrphanedOperations(ctx context.Context, s *state.State) error { logger.Debug("Removing orphaned operations across the cluster") offlineThreshold := s.GlobalConfig.OfflineThreshold() err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { members, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } for _, member := range members { // Skip online nodes if !member.IsOffline(offlineThreshold) { continue } err = dbCluster.DeleteOperations(ctx, tx.Tx(), member.ID) if err != nil { return fmt.Errorf("Failed to delete operations: %w", err) } } return nil }) if err != nil { return fmt.Errorf("Failed to remove orphaned operations: %w", err) } logger.Debug("Done removing orphaned operations across the cluster") return nil } incus-7.0.0/cmd/incusd/patches.go000066400000000000000000001556431517523235500166730ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" "io/fs" "net/http" "os" "path/filepath" "slices" "strings" "time" linstorClient "github.com/LINBIT/golinstor/client" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/certificate" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/query" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/network/acl" "github.com/lxc/incus/v7/internal/server/network/ovn" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/internal/server/project" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) type patchStage int // Define the stages that patches can run at. const ( patchNoStageSet patchStage = iota patchPreDaemonStorage patchPostDaemonStorage patchPostNetworks ) /* Patches are one-time actions that are sometimes needed to update existing container configuration or move things around on the filesystem. Those patches are applied at startup time after the database schema has been fully updated. Patches can therefore assume a working database. At the time the patches are applied, the containers aren't started yet and the daemon isn't listening to requests. DO NOT use this mechanism for database update. Schema updates must be done through the separate schema update mechanism. Only append to the patches list, never remove entries and never re-order them. */ var patches = []patch{ {name: "storage_lvm_skipactivation", stage: patchPostDaemonStorage, run: patchGenericStorage}, {name: "clustering_drop_database_role", stage: patchPostDaemonStorage, run: patchClusteringDropDatabaseRole}, {name: "network_clear_bridge_volatile_hwaddr", stage: patchPostDaemonStorage, run: patchGenericNetwork(patchNetworkClearBridgeVolatileHwaddr)}, {name: "move_backups_instances", stage: patchPostDaemonStorage, run: patchMoveBackupsInstances}, {name: "network_ovn_enable_nat", stage: patchPostDaemonStorage, run: patchGenericNetwork(patchNetworkOVNEnableNAT)}, {name: "network_ovn_remove_routes", stage: patchPostDaemonStorage, run: patchGenericNetwork(patchNetworkOVNRemoveRoutes)}, {name: "thinpool_typo_fix", stage: patchPostDaemonStorage, run: patchThinpoolTypoFix}, {name: "vm_rename_uuid_key", stage: patchPostDaemonStorage, run: patchVMRenameUUIDKey}, {name: "db_nodes_autoinc", stage: patchPreDaemonStorage, run: patchDBNodesAutoInc}, {name: "network_acl_remove_defaults", stage: patchPostDaemonStorage, run: patchGenericNetwork(patchNetworkACLRemoveDefaults)}, {name: "clustering_server_cert_trust", stage: patchPreDaemonStorage, run: patchClusteringServerCertTrust}, {name: "warnings_remove_empty_node", stage: patchPostDaemonStorage, run: patchRemoveWarningsWithEmptyNode}, {name: "dnsmasq_entries_include_device_name", stage: patchPostDaemonStorage, run: patchDnsmasqEntriesIncludeDeviceName}, {name: "storage_missing_snapshot_records", stage: patchPostDaemonStorage, run: patchGenericStorage}, {name: "storage_delete_old_snapshot_records", stage: patchPostDaemonStorage, run: patchGenericStorage}, {name: "storage_zfs_drop_block_volume_filesystem_extension", stage: patchPostDaemonStorage, run: patchGenericStorage}, {name: "storage_prefix_bucket_names_with_project", stage: patchPostDaemonStorage, run: patchGenericStorage}, {name: "storage_move_custom_iso_block_volumes", stage: patchPostDaemonStorage, run: patchStorageRenameCustomISOBlockVolumes}, {name: "zfs_set_content_type_user_property", stage: patchPostDaemonStorage, run: patchZfsSetContentTypeUserProperty}, {name: "snapshots_rename", stage: patchPreDaemonStorage, run: patchSnapshotsRename}, {name: "storage_zfs_unset_invalid_block_settings", stage: patchPostDaemonStorage, run: patchStorageZfsUnsetInvalidBlockSettings}, {name: "storage_zfs_unset_invalid_block_settings_v2", stage: patchPostDaemonStorage, run: patchStorageZfsUnsetInvalidBlockSettingsV2}, {name: "runtime_directory", stage: patchPostDaemonStorage, run: patchRuntimeDirectory}, {name: "lvm_node_force_reuse", stage: patchPostDaemonStorage, run: patchLvmForceReuseKey}, {name: "auth_openfga_viewer", stage: patchPostNetworks, run: patchGenericAuthorization}, {name: "auth_openfga_network_address_set", stage: patchPostNetworks, run: patchGenericAuthorization}, {name: "db_json_columns", stage: patchPreDaemonStorage, run: patchConvertJSONColumn}, {name: "network_ovn_directional_port_groups", stage: patchPostDaemonStorage, run: patchGenericNetwork(patchNetworkOVNPortGroups)}, {name: "pool_fix_default_permissions", stage: patchPostDaemonStorage, run: patchDefaultStoragePermissions}, {name: "auth_openfga_volume_files", stage: patchPostNetworks, run: patchGenericAuthorization}, {name: "btrfs_config_volume_subvolume_names", stage: patchPostNetworks, run: patchBtrfsSubvolumeNames}, {name: "linstor_tune_rs_discard_granularity", stage: patchPostDaemonStorage, run: patchLinstorDiscardGranularity}, } type patchRun func(name string, d *Daemon) error var errRetryNextTime = errors.New("skipped") type patch struct { name string stage patchStage run patchRun } func (p *patch) apply(d *Daemon) error { logger.Info("Applying patch", logger.Ctx{"name": p.name}) err := p.run(p.name, d) if err != nil { if errors.Is(err, errRetryNextTime) { return nil } return fmt.Errorf("Failed applying patch %q: %w", p.name, err) } err = d.db.Node.MarkPatchAsApplied(p.name) if err != nil { return fmt.Errorf("Failed marking patch applied %q: %w", p.name, err) } return nil } // Return the names of all available patches. func patchesGetNames() []string { names := make([]string, len(patches)) for i, patch := range patches { if patch.stage == patchNoStageSet { continue // Ignore any patch without explicitly set stage (it is defined incorrectly). } names[i] = patch.name } return names } // patchesApplyPostDaemonStorage applies the patches that need to run after the daemon storage is initialized. func patchesApply(d *Daemon, stage patchStage) error { appliedPatches, err := d.db.Node.GetAppliedPatches() if err != nil { return err } for _, patch := range patches { if patch.stage == patchNoStageSet { return fmt.Errorf("Patch %q has no stage set: %d", patch.name, patch.stage) } if patch.stage != stage { continue } if slices.Contains(appliedPatches, patch.name) { continue } err := patch.apply(d) if err != nil { return err } } return nil } // Patches begin here func patchDnsmasqEntriesIncludeDeviceName(_ string, d *Daemon) error { err := network.UpdateDNSMasqStatic(d.State(), "") if err != nil { return err } return nil } func patchRemoveWarningsWithEmptyNode(_ string, d *Daemon) error { err := d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { warnings, err := dbCluster.GetWarnings(ctx, tx.Tx()) if err != nil { return err } for _, w := range warnings { if w.Node == "" { err = dbCluster.DeleteWarning(ctx, tx.Tx(), w.UUID) if err != nil { return err } } } return nil }) if err != nil { return err } return nil } func patchClusteringServerCertTrust(name string, d *Daemon) error { if !d.serverClustered { return nil } var serverName string err := d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error serverName, err = tx.GetLocalNodeName(ctx) return err }) if err != nil { return err } // Add our server cert to DB trust store. serverCert, err := internalUtil.LoadServerCert(d.os.VarDir) if err != nil { return err } // Update our own entry in the nodes table. logger.Infof("Adding local server certificate to global trust store for %q patch", name) err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return cluster.EnsureServerCertificateTrusted(serverName, serverCert, tx) }) if err != nil { return err } logger.Infof("Added local server certificate to global trust store for %q patch", name) // Check all other members have done the same. for { var err error var dbCerts []dbCluster.Certificate err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbCerts, err = dbCluster.GetCertificates(ctx, tx.Tx()) return err }) if err != nil { return err } trustedServerCerts := make(map[string]*dbCluster.Certificate) for _, c := range dbCerts { if c.Type == certificate.TypeServer { trustedServerCerts[c.Name] = &c } } var members []db.NodeInfo err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { members, err = tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } return nil }) if err != nil { return err } missingCerts := false for _, member := range members { _, found := trustedServerCerts[member.Name] if !found { logger.Warnf("Missing trusted server certificate for cluster member %q", member.Name) missingCerts = true break } } if missingCerts { logger.Warnf("Waiting for %q patch to be applied on all cluster members", name) time.Sleep(time.Second) continue } logger.Infof("Trusted server certificates found in trust store for all cluster members") break } // Now switch to using our server certificate for intra-cluster communication and load the trusted server // certificates for the other members into the in-memory trusted cache. logger.Infof("Set client certificate to server certificate %v", serverCert.Fingerprint()) d.serverCertInt = serverCert updateCertificateCache(d) return nil } // patchNetworkACLRemoveDefaults removes the "default.action" and "default.logged" settings from network ACLs. // It was decided that the user experience of having the default actions at the ACL level was confusing when using // multiple ACLs, and that the interplay between conflicting default actions on multiple ACLs was difficult to // understand. Instead it will be replace with a network and NIC level defaults settings. func patchNetworkACLRemoveDefaults(_ string, d *Daemon) error { var err error var projectNames []string // Get projects. err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { projectNames, err = dbCluster.GetProjectNames(ctx, tx.Tx()) return err }) if err != nil { return err } err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get ACLs in projects. for _, projectName := range projectNames { acls, err := dbCluster.GetNetworkACLs(ctx, tx.Tx(), dbCluster.NetworkACLFilter{Project: &projectName}) if err != nil { return err } for _, acl := range acls { aclAPI, err := acl.ToAPI(ctx, tx.Tx()) if err != nil { return err } modified := false // Remove the offending keys if found. _, found := aclAPI.Config["default.action"] if found { delete(aclAPI.Config, "default.action") modified = true } _, found = aclAPI.Config["default.logged"] if found { delete(aclAPI.Config, "default.logged") modified = true } // Write back modified config if needed. if modified { err = dbCluster.UpdateNetworkACLAPI(ctx, tx.Tx(), int64(acl.ID), &aclAPI.NetworkACLPut) if err != nil { return fmt.Errorf("Failed updating network ACL %d: %w", acl.ID, err) } } } } return nil }) if err != nil { return err } return nil } // patchDBNodesAutoInc re-creates the nodes table id column as AUTOINCREMENT. // Its done as a patch rather than a schema update so we can use PRAGMA foreign_keys = OFF without a transaction. func patchDBNodesAutoInc(name string, d *Daemon) error { s := d.State() for { // Only apply patch if schema needs it. var schemaSQL string row := s.DB.Cluster.DB().QueryRow("SELECT sql FROM sqlite_master WHERE name = 'nodes'") err := row.Scan(&schemaSQL) if err != nil { return err } if strings.Contains(schemaSQL, "id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL") { logger.Debugf(`Skipping %q patch as "nodes" table id column already AUTOINCREMENT`, name) return nil // Nothing to do. } // Only apply patch on leader, otherwise wait for it to be applied. var localConfig *node.Config err = d.db.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { localConfig, err = node.ConfigLoad(ctx, tx) return err }) if err != nil { return err } leaderAddress, err := s.Cluster.LeaderAddress() if err != nil { if errors.Is(err, cluster.ErrNodeIsNotClustered) { break // Apply change on standalone node. } return err } if localConfig.ClusterAddress() == leaderAddress { break // Apply change on leader node. } logger.Warnf("Waiting for %q patch to be applied on leader cluster member", name) time.Sleep(time.Second) } // Apply patch. _, err := s.DB.Cluster.DB().Exec(` PRAGMA foreign_keys=OFF; -- So that integrity doesn't get in the way for now. PRAGMA legacy_alter_table = ON; -- So that views referencing this table don't block change. CREATE TABLE nodes_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT DEFAULT '', address TEXT NOT NULL, schema INTEGER NOT NULL, api_extensions INTEGER NOT NULL, heartbeat DATETIME DEFAULT CURRENT_TIMESTAMP, state INTEGER NOT NULL DEFAULT 0, arch INTEGER NOT NULL DEFAULT 0 CHECK (arch > 0), failure_domain_id INTEGER DEFAULT NULL REFERENCES nodes_failure_domains (id) ON DELETE SET NULL, UNIQUE (name), UNIQUE (address) ); INSERT INTO nodes_new (id, name, description, address, schema, api_extensions, heartbeat, state, arch, failure_domain_id) SELECT id, name, description, address, schema, api_extensions, heartbeat, state, arch, failure_domain_id FROM nodes; DROP TABLE nodes; ALTER TABLE nodes_new RENAME TO nodes; PRAGMA foreign_keys=ON; -- Make sure we turn integrity checks back on. PRAGMA legacy_alter_table = OFF; -- So views check integrity again. `) return err } // patchVMRenameUUIDKey renames the volatile.vm.uuid key to volatile.uuid in instance and snapshot configs. func patchVMRenameUUIDKey(_ string, d *Daemon) error { oldUUIDKey := "volatile.vm.uuid" newUUIDKey := "volatile.uuid" s := d.State() return s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.InstanceList(ctx, func(inst db.InstanceArgs, p api.Project) error { if inst.Type != instancetype.VM { return nil } uuid := inst.Config[oldUUIDKey] if uuid != "" { changes := map[string]string{ oldUUIDKey: "", newUUIDKey: uuid, } logger.Debugf("Renaming config key %q to %q for VM %q (Project %q)", oldUUIDKey, newUUIDKey, inst.Name, inst.Project) err := tx.UpdateInstanceConfig(inst.ID, changes) if err != nil { return fmt.Errorf("Failed renaming config key %q to %q for VM %q (Project %q): %w", oldUUIDKey, newUUIDKey, inst.Name, inst.Project, err) } } snaps, err := tx.GetInstanceSnapshotsWithName(ctx, inst.Project, inst.Name) if err != nil { return err } for _, snap := range snaps { config, err := dbCluster.GetInstanceConfig(ctx, tx.Tx(), snap.ID) if err != nil { return err } uuid := config[oldUUIDKey] if uuid != "" { changes := map[string]string{ oldUUIDKey: "", newUUIDKey: uuid, } logger.Debugf("Renaming config key %q to %q for VM %q (Project %q)", oldUUIDKey, newUUIDKey, snap.Name, snap.Project) err = tx.UpdateInstanceSnapshotConfig(snap.ID, changes) if err != nil { return fmt.Errorf("Failed renaming config key %q to %q for VM %q (Project %q): %w", oldUUIDKey, newUUIDKey, snap.Name, snap.Project, err) } } } return nil }) }) } // patchThinpoolTypoFix renames any config incorrectly set config file entries due to the lvm.thinpool_name typo. func patchThinpoolTypoFix(_ string, d *Daemon) error { reverter := revert.New() defer reverter.Fail() // Setup a transaction. tx, err := d.db.Cluster.Begin() if err != nil { return fmt.Errorf("Failed to begin transaction: %w", err) } reverter.Add(func() { _ = tx.Rollback() }) // Fetch the IDs of all existing nodes. nodeIDs, err := query.SelectIntegers(context.TODO(), tx, "SELECT id FROM nodes") if err != nil { return fmt.Errorf("Failed to get IDs of current nodes: %w", err) } // Fetch the IDs of all existing lvm pools. poolIDs, err := query.SelectIntegers(context.TODO(), tx, "SELECT id FROM storage_pools WHERE driver='lvm'") if err != nil { return fmt.Errorf("Failed to get IDs of current lvm pools: %w", err) } for _, poolID := range poolIDs { // Fetch the config for this lvm pool and check if it has the lvm.thinpool_name. config, err := query.SelectConfig(context.TODO(), tx, "storage_pools_config", "storage_pool_id=? AND node_id IS NULL", poolID) if err != nil { return fmt.Errorf("Failed to fetch of lvm pool config: %w", err) } value, ok := config["lvm.thinpool_name"] if !ok { continue } // Delete the current key _, err = tx.Exec(` DELETE FROM storage_pools_config WHERE key='lvm.thinpool_name' AND storage_pool_id=? AND node_id IS NULL `, poolID) if err != nil { return fmt.Errorf("Failed to delete lvm.thinpool_name config: %w", err) } // Add the config entry for each node for _, nodeID := range nodeIDs { _, err := tx.Exec(` INSERT INTO storage_pools_config(storage_pool_id, node_id, key, value) VALUES(?, ?, 'lvm.thinpool_name', ?) `, poolID, nodeID, value) if err != nil { return fmt.Errorf("Failed to create lvm.thinpool_name node config: %w", err) } } } err = tx.Commit() if err != nil { return fmt.Errorf("Failed to commit transaction: %w", err) } reverter.Success() return nil } // patchNetworkOVNRemoveRoutes removes the "ipv4.routes.external" and "ipv6.routes.external" settings from OVN // networks. It was decided that the OVN NIC level equivalent settings were sufficient. func patchNetworkOVNRemoveRoutes(_ string, d *Daemon) error { err := d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { projectNetworks, err := tx.GetCreatedNetworks(ctx) if err != nil { return err } for projectName, networks := range projectNetworks { for networkID, network := range networks { if network.Type != "ovn" { continue } modified := false // Ensure existing behaviour of having NAT enabled if IP address was set. _, found := network.Config["ipv4.routes.external"] if found { modified = true delete(network.Config, "ipv4.routes.external") } _, found = network.Config["ipv6.routes.external"] if found { modified = true delete(network.Config, "ipv6.routes.external") } if modified { err = tx.UpdateNetwork(ctx, projectName, network.Name, network.Description, network.Config) if err != nil { return fmt.Errorf("Failed removing OVN external route settings for %q (%d): %w", network.Name, networkID, err) } logger.Debugf("Removing external route settings for OVN network %q (%d)", network.Name, networkID) } } } return nil }) if err != nil { return err } return nil } // patchNetworkOVNEnableNAT adds "ipv4.nat" and "ipv6.nat" keys set to "true" to OVN networks if not present. // This is to ensure existing networks retain the old behaviour of always having NAT enabled as we introduce // the new NAT settings which default to disabled if not specified. // patchNetworkCearBridgeVolatileHwaddr removes the unsupported `volatile.bridge.hwaddr` config key from networks. func patchNetworkOVNEnableNAT(_ string, d *Daemon) error { err := d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { projectNetworks, err := tx.GetCreatedNetworks(ctx) if err != nil { return err } for projectName, networks := range projectNetworks { for networkID, network := range networks { if network.Type != "ovn" { continue } modified := false // Ensure existing behaviour of having NAT enabled if IP address was set. if network.Config["ipv4.address"] != "" && network.Config["ipv4.nat"] == "" { modified = true network.Config["ipv4.nat"] = "true" } if network.Config["ipv6.address"] != "" && network.Config["ipv6.nat"] == "" { modified = true network.Config["ipv6.nat"] = "true" } if modified { err = tx.UpdateNetwork(ctx, projectName, network.Name, network.Description, network.Config) if err != nil { return fmt.Errorf("Failed saving OVN NAT settings for %q (%d): %w", network.Name, networkID, err) } logger.Debugf("Enabling NAT for OVN network %q (%d)", network.Name, networkID) } } } return nil }) if err != nil { return err } return nil } // Moves backups from internalUtil.VarPath("backups") to internalUtil.VarPath("backups", "instances"). func patchMoveBackupsInstances(_ string, _ *Daemon) error { if !util.PathExists(internalUtil.VarPath("backups")) { return nil // Nothing to do, no backups directory. } backupsPath := internalUtil.VarPath("backups", "instances") err := os.MkdirAll(backupsPath, 0o700) if err != nil { return fmt.Errorf("Failed creating instances backup directory %q: %w", backupsPath, err) } backups, err := os.ReadDir(internalUtil.VarPath("backups")) if err != nil { return fmt.Errorf("Failed listing existing backup directory %q: %w", internalUtil.VarPath("backups"), err) } for _, backupDir := range backups { if backupDir.Name() == "instances" || strings.HasPrefix(backupDir.Name(), backup.WorkingDirPrefix) { continue // Don't try and move our new instances directory or temporary directories. } oldPath := internalUtil.VarPath("backups", backupDir.Name()) newPath := filepath.Join(backupsPath, backupDir.Name()) logger.Debugf("Moving backup from %q to %q", oldPath, newPath) err = os.Rename(oldPath, newPath) if err != nil { return fmt.Errorf("Failed moving backup from %q to %q: %w", oldPath, newPath, err) } } return nil } func patchGenericAuthorization(name string, d *Daemon) error { return d.authorizer.ApplyPatch(d.shutdownCtx, name) } func patchGenericStorage(name string, d *Daemon) error { return storagePools.Patch(d.State(), name) } func patchGenericNetwork(f func(name string, d *Daemon) error) func(name string, d *Daemon) error { return func(name string, d *Daemon) error { err := network.PatchPreCheck() if err != nil { return err } return f(name, d) } } func patchClusteringDropDatabaseRole(_ string, d *Daemon) error { return d.State().DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { members, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } for _, member := range members { err := tx.UpdateNodeRoles(member.ID, nil) if err != nil { return err } } return nil }) } // patchNetworkClearBridgeVolatileHwaddr removes the unsupported `volatile.bridge.hwaddr` config key from networks. func patchNetworkClearBridgeVolatileHwaddr(_ string, d *Daemon) error { // Use api.ProjectDefaultName, as bridge networks don't support projects. projectName := api.ProjectDefaultName err := d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get the list of networks. networks, err := tx.GetNetworks(ctx, projectName) if err != nil { return fmt.Errorf("Failed loading networks for network_clear_bridge_volatile_hwaddr patch: %w", err) } for _, networkName := range networks { _, net, _, err := tx.GetNetworkInAnyState(ctx, projectName, networkName) if err != nil { return fmt.Errorf("Failed loading network %q for network_clear_bridge_volatile_hwaddr patch: %w", networkName, err) } if net.Config["volatile.bridge.hwaddr"] != "" { delete(net.Config, "volatile.bridge.hwaddr") err = tx.UpdateNetwork(ctx, projectName, net.Name, net.Description, net.Config) if err != nil { return fmt.Errorf("Failed updating network %q for network_clear_bridge_volatile_hwaddr patch: %w", networkName, err) } } } return nil }) if err != nil { return err } return nil } // patchStorageRenameCustomISOBlockVolumes renames existing custom ISO volumes by adding the ".iso" suffix so they can be distinguished from regular custom block volumes. // This patch doesn't use the patchGenericStorage function because the storage drivers themselves aren't aware of custom ISO volumes. func patchStorageRenameCustomISOBlockVolumes(_ string, d *Daemon) error { s := d.State() var pools []string err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get all storage pool names. pools, err = tx.GetStoragePoolNames(ctx) return err }) if err != nil { // Skip the rest of the patch if no storage pools were found. if api.StatusErrorCheck(err, http.StatusNotFound) { return nil } return fmt.Errorf("Failed getting storage pool names: %w", err) } // Only apply patch on leader. var localConfig *node.Config isLeader := false err = d.db.Node.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.NodeTx) error { localConfig, err = node.ConfigLoad(ctx, tx) return err }) if err != nil { return err } leaderAddress, err := s.Cluster.LeaderAddress() if err != nil { // If we're not clustered, we're the leader. if errors.Is(err, cluster.ErrNodeIsNotClustered) { isLeader = true } else { return err } } else if localConfig.ClusterAddress() == leaderAddress { isLeader = true } volTypeCustom := db.StoragePoolVolumeTypeCustom customPoolVolumes := make(map[string][]*db.StorageVolume) err = s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { for _, pool := range pools { // Get storage pool ID. poolID, err := tx.GetStoragePoolID(ctx, pool) if err != nil { return fmt.Errorf("Failed getting storage pool ID of pool %q: %w", pool, err) } // Get the pool's custom storage volumes. customVolumes, err := tx.GetStoragePoolVolumes(ctx, poolID, false, db.StorageVolumeFilter{Type: &volTypeCustom}) if err != nil { return fmt.Errorf("Failed getting custom storage volumes of pool %q: %w", pool, err) } if customPoolVolumes[pool] == nil { customPoolVolumes[pool] = []*db.StorageVolume{} } customPoolVolumes[pool] = append(customPoolVolumes[pool], customVolumes...) } return nil }) if err != nil { return err } for poolName, volumes := range customPoolVolumes { // Load storage pool. p, err := storagePools.LoadByName(s, poolName) if err != nil { return fmt.Errorf("Failed loading pool %q: %w", poolName, err) } // Ensure the renaming is done only on the cluster leader for remote storage pools. if p.Driver().Info().Remote && !isLeader { continue } for _, vol := range volumes { // In a non-clusted environment ServerName will be empty. if s.ServerName != "" && vol.Location != s.ServerName { continue } // Exclude non-ISO custom volumes. if vol.ContentType != db.StoragePoolVolumeContentTypeNameISO { continue } // We need to use ContentTypeBlock here in order for the driver to figure out the correct (old) location. oldVol := storageDrivers.NewVolume(p.Driver(), p.Name(), storageDrivers.VolumeTypeCustom, storageDrivers.ContentTypeBlock, project.StorageVolume(vol.Project, vol.Name), nil, nil) err = p.Driver().RenameVolume(oldVol, fmt.Sprintf("%s.iso", oldVol.Name()), nil) if err != nil { return fmt.Errorf("Failed renaming volume: %w", err) } } } return nil } // patchZfsSetContentTypeUserProperty adds the `incus:content_type` user property to custom storage volumes. In case of recovery, this allows for proper detection of block-mode enabled volumes. func patchZfsSetContentTypeUserProperty(_ string, d *Daemon) error { s := d.State() var pools []string err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get all storage pool names. pools, err = tx.GetStoragePoolNames(ctx) return err }) if err != nil { // Skip the rest of the patch if no storage pools were found. if api.StatusErrorCheck(err, http.StatusNotFound) { return nil } return fmt.Errorf("Failed getting storage pool names: %w", err) } volTypeCustom := db.StoragePoolVolumeTypeCustom customPoolVolumes := make(map[string][]*db.StorageVolume) err = s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { for _, pool := range pools { // Get storage pool ID. poolID, err := tx.GetStoragePoolID(ctx, pool) if err != nil { return fmt.Errorf("Failed getting storage pool ID of pool %q: %w", pool, err) } // Get the pool's custom storage volumes. customVolumes, err := tx.GetStoragePoolVolumes(ctx, poolID, false, db.StorageVolumeFilter{Type: &volTypeCustom}) if err != nil { return fmt.Errorf("Failed getting custom storage volumes of pool %q: %w", pool, err) } if customPoolVolumes[pool] == nil { customPoolVolumes[pool] = []*db.StorageVolume{} } customPoolVolumes[pool] = append(customPoolVolumes[pool], customVolumes...) } return nil }) if err != nil { return err } for poolName, volumes := range customPoolVolumes { // Load storage pool. p, err := storagePools.LoadByName(s, poolName) if err != nil { return fmt.Errorf("Failed loading pool %q: %w", poolName, err) } if p.Driver().Info().Name != "zfs" { continue } for _, vol := range volumes { // In a non-clusted environment ServerName will be empty. if s.ServerName != "" && vol.Location != s.ServerName { continue } zfsPoolName := p.Driver().Config()["zfs.pool_name"] if zfsPoolName != "" { poolName = zfsPoolName } zfsVolName := fmt.Sprintf("%s/%s/%s", poolName, storageDrivers.VolumeTypeCustom, project.StorageVolume(vol.Project, vol.Name)) _, err = subprocess.RunCommand("zfs", "set", fmt.Sprintf("incus:content_type=%s", vol.ContentType), zfsVolName) if err != nil { logger.Debug("Failed setting incus:content_type property", logger.Ctx{"name": zfsVolName, "err": err}) } } } return nil } // patchSnapshotsRename renames the "snapshots" directory to "container-snapshots". func patchSnapshotsRename(_ string, _ *Daemon) error { // Remove what should be an empty directory. os.Remove(internalUtil.VarPath("containers-snapshots")) return os.Rename(internalUtil.VarPath("snapshots"), internalUtil.VarPath("containers-snapshots")) } // patchStorageZfsUnsetInvalidBlockSettings removes invalid block settings from volumes. func patchStorageZfsUnsetInvalidBlockSettings(_ string, d *Daemon) error { s := d.State() var pools []string err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get all storage pool names. pools, err = tx.GetStoragePoolNames(ctx) return err }) if err != nil { // Skip the rest of the patch if no storage pools were found. if api.StatusErrorCheck(err, http.StatusNotFound) { return nil } return fmt.Errorf("Failed getting storage pool names: %w", err) } volTypeCustom := db.StoragePoolVolumeTypeCustom volTypeVM := db.StoragePoolVolumeTypeVM poolIDNameMap := make(map[int64]string) poolVolumes := make(map[int64][]*db.StorageVolume) err = s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { for _, pool := range pools { // Get storage pool ID. poolID, err := tx.GetStoragePoolID(ctx, pool) if err != nil { return fmt.Errorf("Failed getting storage pool ID of pool %q: %w", pool, err) } driverName, err := tx.GetStoragePoolDriver(ctx, poolID) if err != nil { return fmt.Errorf("Failed getting storage pool driver of pool %q: %w", pool, err) } if driverName != "zfs" { continue } // Get the pool's custom storage volumes. volumes, err := tx.GetStoragePoolVolumes(ctx, poolID, false, db.StorageVolumeFilter{Type: &volTypeCustom}, db.StorageVolumeFilter{Type: &volTypeVM}) if err != nil { return fmt.Errorf("Failed getting custom storage volumes of pool %q: %w", pool, err) } if poolVolumes[poolID] == nil { poolVolumes[poolID] = []*db.StorageVolume{} } poolIDNameMap[poolID] = pool poolVolumes[poolID] = append(poolVolumes[poolID], volumes...) } return nil }) if err != nil { return err } var volType int for pool, volumes := range poolVolumes { for _, vol := range volumes { // In a non-clusted environment ServerName will be empty. if s.ServerName != "" && vol.Location != s.ServerName { continue } config := vol.Config if util.IsTrue(config["zfs.block_mode"]) { continue } update := false for _, k := range []string{"block.filesystem", "block.mount_options"} { _, found := config[k] if found { delete(config, k) update = true } } if !update { continue } if vol.Type == db.StoragePoolVolumeTypeNameVM { volType = volTypeVM } else if vol.Type == db.StoragePoolVolumeTypeNameCustom { volType = volTypeCustom } else { // Should not happen. continue } err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateStoragePoolVolume(ctx, vol.Project, vol.Name, volType, pool, vol.Description, config) }) if err != nil { return fmt.Errorf("Failed updating volume %q in project %q on pool %q: %w", vol.Name, vol.Project, poolIDNameMap[pool], err) } } } return nil } // patchStorageZfsUnsetInvalidBlockSettingsV2 removes invalid block settings from volumes. // This patch fixes the previous one. // - Handle non-clusted environments correctly. // - Always remove block.* settings from VMs. func patchStorageZfsUnsetInvalidBlockSettingsV2(_ string, d *Daemon) error { s := d.State() var pools []string err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get all storage pool names. pools, err = tx.GetStoragePoolNames(ctx) return err }) if err != nil { // Skip the rest of the patch if no storage pools were found. if api.StatusErrorCheck(err, http.StatusNotFound) { return nil } return fmt.Errorf("Failed getting storage pool names: %w", err) } volTypeCustom := db.StoragePoolVolumeTypeCustom volTypeVM := db.StoragePoolVolumeTypeVM poolIDNameMap := make(map[int64]string) poolVolumes := make(map[int64][]*db.StorageVolume) err = s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { for _, pool := range pools { // Get storage pool ID. poolID, err := tx.GetStoragePoolID(ctx, pool) if err != nil { return fmt.Errorf("Failed getting storage pool ID of pool %q: %w", pool, err) } driverName, err := tx.GetStoragePoolDriver(ctx, poolID) if err != nil { return fmt.Errorf("Failed getting storage pool driver of pool %q: %w", pool, err) } if driverName != "zfs" { continue } // Get the pool's custom storage volumes. volumes, err := tx.GetStoragePoolVolumes(ctx, poolID, false, db.StorageVolumeFilter{Type: &volTypeCustom}, db.StorageVolumeFilter{Type: &volTypeVM}) if err != nil { return fmt.Errorf("Failed getting custom storage volumes of pool %q: %w", pool, err) } if poolVolumes[poolID] == nil { poolVolumes[poolID] = []*db.StorageVolume{} } poolIDNameMap[poolID] = pool poolVolumes[poolID] = append(poolVolumes[poolID], volumes...) } return nil }) if err != nil { return err } var volType int for pool, volumes := range poolVolumes { for _, vol := range volumes { // In a non-clusted environment ServerName will be empty. if s.ServerName != "" && vol.Location != s.ServerName { continue } config := vol.Config // Only check zfs.block_mode for custom volumes. VMs should never have any block.* settings // regardless of the zfs.block_mode setting. if util.IsTrue(config["zfs.block_mode"]) && vol.Type == db.StoragePoolVolumeTypeNameCustom { continue } update := false for _, k := range []string{"block.filesystem", "block.mount_options"} { _, found := config[k] if found { delete(config, k) update = true } } if !update { continue } if vol.Type == db.StoragePoolVolumeTypeNameVM { volType = volTypeVM } else if vol.Type == db.StoragePoolVolumeTypeNameCustom { volType = volTypeCustom } else { // Should not happen. continue } err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateStoragePoolVolume(ctx, vol.Project, vol.Name, volType, pool, vol.Description, config) }) if err != nil { return fmt.Errorf("Failed updating volume %q in project %q on pool %q: %w", vol.Name, vol.Project, poolIDNameMap[pool], err) } } } return nil } func patchRuntimeDirectory(_ string, d *Daemon) error { s := d.State() // Get the list of local instances. instances, err := instance.LoadNodeAll(s, instancetype.Any) if err != nil { return fmt.Errorf("Failed loading local instances: %w", err) } for _, inst := range instances { if !util.PathExists(inst.LogPath()) { continue } err = os.MkdirAll(inst.RunPath(), 0o700) if err != nil && !os.IsExist(err) { return fmt.Errorf("Failed to create runtime directory for %q in project %q: %w", inst.Name(), inst.Project(), err) } files, err := os.ReadDir(inst.LogPath()) if err != nil { return fmt.Errorf("Failed to list log files for %q in project %q: %w", inst.Name(), inst.Project(), err) } for _, fi := range files { name := fi.Name() // Keep actual log files where they are. if strings.HasSuffix(name, ".log") || strings.HasSuffix(name, ".log.old") || strings.HasSuffix(name, ".gz") { continue } info, err := fi.Info() if err != nil { return fmt.Errorf("Failed getting file info on %q for instance %q in project %q: %w", name, inst.Name(), inst.Project(), err) } if info.Mode().IsRegular() { // Relocate the file. _, err := subprocess.RunCommand("mv", filepath.Join(inst.LogPath(), name), inst.RunPath()) if err != nil { return fmt.Errorf("Failed to relocate runtime file %q for instance %q in project %q: %w", name, inst.Name(), inst.Project(), err) } } else { // For pipes and sockets, we need to use a symlink to avoid breaking the listener. err := os.Symlink(filepath.Join(inst.LogPath(), name), filepath.Join(inst.RunPath(), name)) if err != nil { return fmt.Errorf("Failed to symlink runtime file %q for instance %q in project %q: %w", name, inst.Name(), inst.Project(), err) } } } } return nil } // The lvm.vg.force_reuse config key is node-specific and need to be linked to nodes. func patchLvmForceReuseKey(_ string, d *Daemon) error { reverter := revert.New() defer reverter.Fail() // Setup a transaction. tx, err := d.db.Cluster.Begin() if err != nil { return fmt.Errorf("Failed to begin transaction: %w", err) } reverter.Add(func() { _ = tx.Rollback() }) // Fetch the IDs of all existing nodes. nodeIDs, err := query.SelectIntegers(context.TODO(), tx, "SELECT id FROM nodes") if err != nil { return fmt.Errorf("Failed to get IDs of current nodes: %w", err) } // Fetch the IDs of all existing lvm pools. poolIDs, err := query.SelectIntegers(context.TODO(), tx, "SELECT id FROM storage_pools WHERE driver='lvm'") if err != nil { return fmt.Errorf("Failed to get IDs of current LVM pools: %w", err) } for _, poolID := range poolIDs { // Fetch the config for this LVM pool and check if it has the lvm.vg.force_reuse key. config, err := query.SelectConfig(context.TODO(), tx, "storage_pools_config", "storage_pool_id=? AND node_id IS NULL", poolID) if err != nil { return fmt.Errorf("Failed to fetch of lvm pool config: %w", err) } value, ok := config["lvm.vg.force_reuse"] if !ok { continue } // Delete the current key. _, err = tx.Exec("DELETE FROM storage_pools_config WHERE key='lvm.vg.force_reuse' AND storage_pool_id=? AND node_id IS NULL", poolID) if err != nil { return fmt.Errorf("Failed to delete old config: %w", err) } // Add the config entry for each node. for _, nodeID := range nodeIDs { _, err := tx.Exec(` INSERT INTO storage_pools_config(storage_pool_id, node_id, key, value) VALUES(?, ?, 'lvm.vg.force_reuse', ?) `, poolID, nodeID, value) if err != nil { return fmt.Errorf("Failed to create new config: %w", err) } } } err = tx.Commit() if err != nil { return fmt.Errorf("Failed to commit transaction: %w", err) } reverter.Success() return nil } // The database generator expects valid JSON. An empty string isn't valid JSON. func patchConvertJSONColumn(_ string, d *Daemon) error { s := d.State() _, err := s.DB.Cluster.DB().Exec(` UPDATE networks_acls SET egress="null" WHERE egress=""; UPDATE networks_acls SET ingress="null" WHERE ingress=""; UPDATE networks_forwards SET ports="null" WHERE ports=""; UPDATE networks_load_balancers SET backends="null" WHERE backends=""; UPDATE networks_load_balancers SET ports="null" WHERE ports=""; `) if err != nil { return fmt.Errorf("Failed to fix JSON columns: %w", err) } return nil } func patchNetworkOVNPortGroups(_ string, d *Daemon) error { reverter := revert.New() defer reverter.Fail() s := d.State() // Only apply patch on leader. var err error var localConfig *node.Config isLeader := false err = d.db.Node.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.NodeTx) error { localConfig, err = node.ConfigLoad(ctx, tx) return err }) if err != nil { return err } leaderAddress, err := s.Cluster.LeaderAddress() if err != nil { // If we're not clustered, we're the leader. if !errors.Is(err, cluster.ErrNodeIsNotClustered) { return err } isLeader = true } else if localConfig.ClusterAddress() == leaderAddress { isLeader = true } if !isLeader { return nil } projectACLs := make(map[string][]dbCluster.NetworkACL) err = d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get all project names. projects, err := dbCluster.GetProjects(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading projects: %w", err) } for _, project := range projects { networkACLs, err := dbCluster.GetNetworkACLs(ctx, tx.Tx(), dbCluster.NetworkACLFilter{Project: &project.Name}) if err != nil { return err } projectACLs[project.Name] = networkACLs } return nil }) if err != nil { return err } instanceDeviceACLDefaults := func(networkConfig map[string]string, deviceCfg deviceConfig.Device, direction string) string { defaults := map[string]string{ fmt.Sprintf("security.acls.default.%s.action", direction): "reject", } for k := range defaults { if deviceCfg[k] != "" { defaults[k] = deviceCfg[k] } else if networkConfig[k] != "" { defaults[k] = networkConfig[k] } } return defaults[fmt.Sprintf("security.acls.default.%s.action", direction)] } for projectName, networkACLs := range projectACLs { var projectID int64 err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get project ID. projectID, err = dbCluster.GetProjectID(ctx, tx.Tx(), projectName) if err != nil { return fmt.Errorf("Failed getting project ID for project %q: %w", projectName, err) } return nil }) if err != nil { return err } aclNameIDs := make(map[string]int64, len(networkACLs)) for _, networkACL := range networkACLs { aclNameIDs[networkACL.Name] = int64(networkACL.ID) } allNetworks := []string{} for _, networkACL := range networkACLs { // Get a list of networks that are using this ACL (either directly or indirectly via a NIC). aclNets := map[string]acl.NetworkACLUsage{} err = acl.NetworkUsage(s, projectName, []string{networkACL.Name}, aclNets) if err != nil { return fmt.Errorf("Failed getting ACL network usage: %w", err) } aclOVNNets := map[string]acl.NetworkACLUsage{} for k, v := range aclNets { if v.Type != "ovn" { continue } aclOVNNets[k] = v if slices.Contains(allNetworks, k) { continue } allNetworks = append(allNetworks, k) } if len(aclOVNNets) < 1 { continue } // Check that OVN is available. ovnnb, _, err := s.OVN() if err != nil { return err } // Get list of OVN port groups associated to this project. portGroups, err := ovnnb.GetPortGroupsByProject(context.TODO(), projectID) if err != nil { return fmt.Errorf("Failed getting port groups for project %q: %w", projectName, err) } // Prepare a slice `removePortGroup` containing port groups to be removed with the prefix "incus_acl". removePortGroups := []ovn.OVNPortGroup{} for _, portGroup := range portGroups { if !strings.HasPrefix(string(portGroup), acl.OVNACLPortGroupNamePrefix(int64(networkACL.ID))) { continue } removePortGroups = append(removePortGroups, portGroup) } // Delete 'old' port groups err = ovnnb.DeletePortGroup(context.TODO(), removePortGroups...) if err != nil { return fmt.Errorf("Failed to delete unused OVN port groups: %w", err) } cleanup, err := acl.OVNEnsureACLs(s, logger.AddContext(logger.Ctx{}), ovnnb, projectName, aclNameIDs, aclOVNNets, []string{networkACL.Name}, true) if err != nil { return fmt.Errorf("Failed ensuring ACL is configured in OVN: %w", err) } reverter.Add(cleanup) } if len(allNetworks) < 1 { continue } // Check that OVN is available. ovnnb, _, err := s.OVN() if err != nil { return err } for _, networkName := range allNetworks { n, err := network.LoadByName(s, projectName, networkName) if err != nil { return err } // Get list of active switch ports. switchName := acl.OVNIntSwitchName(n.ID()) activePorts, err := ovnnb.GetLogicalSwitchPorts(context.TODO(), switchName) if err != nil { return fmt.Errorf("Failed getting active ports: %w", err) } addedACLs := util.SplitNTrimSpace(n.Config()["security.acls"], ",", -1, true) addChangeSet := map[ovn.OVNPortGroup][]ovn.OVNSwitchPortUUID{} err = network.UsedByInstanceDevices(s, n.Project(), n.Name(), n.Type(), func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error { // Combine NIC ACLs (including ACLs from the profile) with network ACLs. nicACLs := util.SplitNTrimSpace(nicConfig["security.acls"], ",", -1, true) for _, addedACL := range addedACLs { if slices.Contains(nicACLs, addedACL) { continue } nicACLs = append(nicACLs, addedACL) } // Get logical port UUID and name. networkPrefix := acl.OVNNetworkPrefix(n.ID()) intSwitchInstancePortPrefix := fmt.Sprintf("%s-instance", networkPrefix) instancePortName := ovn.OVNSwitchPort(fmt.Sprintf("%s-%s-%s", intSwitchInstancePortPrefix, inst.Config["volatile.uuid"], nicName)) portUUID, found := activePorts[instancePortName] if !found { return nil // No need to update a port that isn't started yet. } ingressAction := instanceDeviceACLDefaults(n.Config(), nicConfig, "ingress") egressAction := instanceDeviceACLDefaults(n.Config(), nicConfig, "egress") // Add NIC port to ACL port group. for _, nicACL := range nicACLs { aclID, found := aclNameIDs[nicACL] if !found { return fmt.Errorf("Cannot find security ACL ID for %q", nicACL) } directionalPortGroups := acl.OVNACLDirectionalPortGroups(aclID) var ingressPortGroupName ovn.OVNPortGroup if ingressAction == "allow" { ingressPortGroupName = directionalPortGroups.IngressReversed } else { ingressPortGroupName = directionalPortGroups.Ingress } acl.OVNPortGroupInstanceNICSchedule(portUUID, addChangeSet, ingressPortGroupName) logger.Debug("Scheduled logical port for ACL port group addition", logger.Ctx{"networkACL": nicACL, "portGroup": ingressPortGroupName, "port": instancePortName}) var egressPortGroupName ovn.OVNPortGroup if egressAction == "allow" { egressPortGroupName = directionalPortGroups.EgressReversed } else { egressPortGroupName = directionalPortGroups.Egress } acl.OVNPortGroupInstanceNICSchedule(portUUID, addChangeSet, egressPortGroupName) logger.Debug("Scheduled logical port for ACL port group addition", logger.Ctx{"networkACL": nicACL, "portGroup": egressPortGroupName, "port": instancePortName}) } return nil }) if err != nil { return err } // Apply add changesets. if len(addChangeSet) > 0 { logger.Debug("Applying ACL port group member change sets") removeChangeSet := map[ovn.OVNPortGroup][]ovn.OVNSwitchPortUUID{} err = ovnnb.UpdatePortGroupMembers(context.TODO(), addChangeSet, removeChangeSet) if err != nil { return fmt.Errorf("Failed applying OVN port group member change sets for instance NIC: %w", err) } } } } reverter.Success() return nil } // patchDefaultStoragePermissions re-applies the default modes to all storage pools. func patchDefaultStoragePermissions(_ string, d *Daemon) error { s := d.State() var pools []string err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get all storage pool names. pools, err = tx.GetStoragePoolNames(ctx) return err }) if err != nil { // Skip the rest of the patch if no storage pools were found. if api.StatusErrorCheck(err, http.StatusNotFound) { return nil } return fmt.Errorf("Failed getting storage pool names: %w", err) } for _, pool := range pools { for _, volEntry := range storageDrivers.BaseDirectories { for _, volDir := range volEntry.Paths { path := filepath.Join(storagePools.GetStoragePoolMountPoint(pool), volDir) err := os.Chmod(path, volEntry.Mode) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to set directory mode %q: %w", path, err) } } } } return nil } // patchBtrfsSubvolumeNames updates Btrfs subvolume names for instance config volumes, // replacing the pattern '-' with 'instance-'. func patchBtrfsSubvolumeNames(_ string, d *Daemon) error { s := d.State() // Get the list of local instances. instances, err := instance.LoadNodeAll(s, instancetype.Any) if err != nil { return fmt.Errorf("Failed loading local instances: %w", err) } for _, inst := range instances { volType, err := storagePools.InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } // Get the volume name on storage. volStorageName := project.Instance(inst.Project().Name, inst.Name()) contentType := storagePools.InstanceContentType(inst) _, currentPool, _ := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) // Load storage pool. p, err := storagePools.LoadByName(s, currentPool["pool"]) if err != nil { logger.Error("Failed loading pool", logger.Ctx{"pool": currentPool["pool"], "err": err}) continue } // Check if driver is 'lvmcluster'. if p.Driver().Info().Name != "lvmcluster" { continue } // Load storage volume from database. dbVol, err := storagePools.VolumeDBGet(p, inst.Project().Name, inst.Name(), volType) if err != nil { logger.Error("Failed loading volume", logger.Ctx{"vol": inst.Name(), "err": err}) continue } // Process only qcow2 instances. if dbVol.Config["block.type"] != "qcow2" { continue } vol := p.GetVolume(volType, contentType, volStorageName, dbVol.Config) newName := storageDrivers.Qcow2ConfigVolumeBase fsVol := vol.NewVMBlockFilesystemVolume() // Increment the ref count for running instances, // since it is equal zero at this stage and may cause an unintended volume deactivation. if inst.IsRunning() { fsVol.MountRefCountIncrement() } err = storageDrivers.Qcow2MountConfigTask(fsVol, nil, func(mountPath string) error { _, volName := project.StorageVolumeParts(fsVol.Name()) entries, err := os.ReadDir(mountPath) if err != nil { return err } // Iterate through all entries (directories) and rename them. for _, entry := range entries { if !entry.IsDir() { continue } oldName := entry.Name() if oldName == volName || strings.HasPrefix(oldName, volName) { newName := newName + strings.TrimPrefix(oldName, volName) oldPath := filepath.Join(mountPath, oldName) newPath := filepath.Join(mountPath, newName) err := os.Rename(oldPath, newPath) if err != nil { return fmt.Errorf("Failed to rename %q to %q: %w", oldPath, newPath, err) } } } return nil }) if err != nil { logger.Error("Failed to rename btrfs subvolumes", logger.Ctx{"err": err}) continue } if inst.IsRunning() { fsVol.MountRefCountDecrement() } } return nil } // patchLinstorDiscardGranularity sets `DrbdOptions/Disk/rs-discard-granularity` on pools that don’t // already set it. func patchLinstorDiscardGranularity(_ string, d *Daemon) error { s := d.State() var pools map[int64]api.StoragePool err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get all storage pools. pools, _, err = tx.GetStoragePools(ctx, nil) return err }) if err != nil { // Skip the rest of the patch if no storage pools were found. if api.StatusErrorCheck(err, http.StatusNotFound) { return nil } return fmt.Errorf("Failed getting storage pools: %w", err) } var resourceGroups []string for _, pool := range pools { if pool.Driver == "linstor" { resourceGroups = append(resourceGroups, pool.Config[storageDrivers.LinstorResourceGroupNameConfigKey]) } } if len(resourceGroups) == 0 { return nil } // Retrieve the Linstor client. linstor, err := s.Linstor() if err != nil { // Let’s not fail the daemon here. logger.Errorf("Failed to load LINSTOR client: %v", err) return errRetryNextTime } retryNextTime := false for _, resourceGroup := range resourceGroups { rg, err := linstor.Client.ResourceGroups.Get(context.TODO(), resourceGroup) if err != nil { logger.Errorf("Could not get LINSTOR resource group %s: %v", resourceGroup, err) retryNextTime = true continue } if rg.Props["DrbdOptions/Disk/rs-discard-granularity"] == "" { err = linstor.Client.ResourceGroups.Modify(context.TODO(), resourceGroup, linstorClient.ResourceGroupModify{ OverrideProps: map[string]string{"DrbdOptions/Disk/rs-discard-granularity": "1048576"}, }) if err != nil { logger.Errorf("Could not set LINSTOR resource group %s property: %v", resourceGroup, err) retryNextTime = true } } } if retryNextTime { return errRetryNextTime } return nil } // Patches end here incus-7.0.0/cmd/incusd/profiles.go000066400000000000000000000577161517523235500170710ustar00rootroot00000000000000package main import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "github.com/gorilla/mux" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var profilesCmd = APIEndpoint{ Path: "profiles", Get: APIEndpointAction{Handler: profilesGet, AccessHandler: allowAuthenticated}, Post: APIEndpointAction{Handler: profilesPost, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanCreateProfiles)}, } var profileCmd = APIEndpoint{ Path: "profiles/{name}", Delete: APIEndpointAction{Handler: profileDelete, AccessHandler: allowPermission(auth.ObjectTypeProfile, auth.EntitlementCanEdit, "name")}, Get: APIEndpointAction{Handler: profileGet, AccessHandler: allowPermission(auth.ObjectTypeProfile, auth.EntitlementCanView, "name")}, Patch: APIEndpointAction{Handler: profilePatch, AccessHandler: allowPermission(auth.ObjectTypeProfile, auth.EntitlementCanEdit, "name")}, Post: APIEndpointAction{Handler: profilePost, AccessHandler: allowPermission(auth.ObjectTypeProfile, auth.EntitlementCanEdit, "name")}, Put: APIEndpointAction{Handler: profilePut, AccessHandler: allowPermission(auth.ObjectTypeProfile, auth.EntitlementCanEdit, "name")}, } // swagger:operation GET /1.0/profiles profiles profiles_get // // Get the profiles // // Returns a list of profiles (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve profiles from all projects // type: boolean // example: true // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/profiles/default", // "/1.0/profiles/foo" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/profiles?recursion=1 profiles profiles_get_recursion1 // // Get the profiles // // Returns a list of profiles (structs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve profiles from all projects // type: boolean // example: true // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of profiles // items: // $ref: "#/definitions/Profile" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func profilesGet(d *Daemon, r *http.Request) response.Response { s := d.State() p, err := project.ProfileProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } recursion := localUtil.IsRecursionRequest(r) // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } mustLoadObjects := recursion || (clauses != nil && len(clauses.Clauses) > 0) allProjects := util.IsTrue(request.QueryParam(r, "all-projects")) userHasPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanView, auth.ObjectTypeProfile) if err != nil { return response.InternalError(err) } fullResults := make([]api.Profile, 0) linkResults := make([]string, 0) err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var profiles []dbCluster.Profile if !allProjects { filter := dbCluster.ProfileFilter{ Project: &p.Name, } profiles, err = dbCluster.GetProfiles(ctx, tx.Tx(), filter) if err != nil { return err } } else { profiles, err = dbCluster.GetProfiles(ctx, tx.Tx()) if err != nil { return err } } if mustLoadObjects { profileConfigs, err := dbCluster.GetAllProfileConfigs(ctx, tx.Tx()) if err != nil { return err } profileDevices, err := dbCluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return err } for _, profile := range profiles { if !userHasPermission(auth.ObjectProfile(p.Name, profile.Name)) { continue } apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices) if err != nil { return err } apiProfile.UsedBy, err = profileUsedBy(ctx, tx, profile) if err != nil { return err } apiProfile.UsedBy = project.FilterUsedBy(s.Authorizer, r, apiProfile.UsedBy) if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(*apiProfile, *clauses) if err != nil { return err } if !match { continue } } fullResults = append(fullResults, *apiProfile) linkResults = append(linkResults, apiProfile.URL(version.APIVersion, profile.Project).String()) } } else { for _, profile := range profiles { if !userHasPermission(auth.ObjectProfile(p.Name, profile.Name)) { continue } apiProfile := api.Profile{Name: profile.Name} linkResults = append(linkResults, apiProfile.URL(version.APIVersion, profile.Project).String()) } } return err }) if err != nil { return response.SmartError(err) } if recursion { return response.SyncResponse(true, fullResults) } return response.SyncResponse(true, linkResults) } // profileUsedBy returns all the instance URLs that are using the given profile. func profileUsedBy(ctx context.Context, tx *db.ClusterTx, profile dbCluster.Profile) ([]string, error) { instances, err := dbCluster.GetProfileInstances(ctx, tx.Tx(), profile.ID) if err != nil { return nil, err } usedBy := make([]string, len(instances)) for i, inst := range instances { apiInst := &api.Instance{Name: inst.Name} usedBy[i] = apiInst.URL(version.APIVersion, inst.Project).String() } return usedBy, nil } // swagger:operation POST /1.0/profiles profiles profiles_post // // Add a profile // // Creates a new profile. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: profile // description: Profile // required: true // schema: // $ref: "#/definitions/ProfilesPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func profilesPost(d *Daemon, r *http.Request) response.Response { s := d.State() p, err := project.ProfileProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } req := api.ProfilesPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. if req.Name == "" { return response.BadRequest(errors.New("No name provided")) } err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid profile name: %w", err)) } err = instance.ValidConfig(d.os, req.Config, false, instancetype.Any) if err != nil { return response.BadRequest(err) } // At this point we don't know the instance type, so just use instancetype.Any type for validation. err = instance.ValidDevices(s, *p, instancetype.Any, deviceConfig.NewDevices(req.Devices), nil) if err != nil { return response.BadRequest(err) } // Update DB entry. err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { devices, err := dbCluster.APIToDevices(req.Devices) if err != nil { return err } current, _ := dbCluster.GetProfile(ctx, tx.Tx(), p.Name, req.Name) if current != nil { return errors.New("The profile already exists") } profile := dbCluster.Profile{ Project: p.Name, Name: req.Name, Description: req.Description, } id, err := dbCluster.CreateProfile(ctx, tx.Tx(), profile) if err != nil { return err } err = dbCluster.CreateProfileConfig(ctx, tx.Tx(), id, req.Config) if err != nil { return err } err = dbCluster.CreateProfileDevices(ctx, tx.Tx(), id, devices) if err != nil { return err } return err }) if err != nil { return response.SmartError(fmt.Errorf("Error inserting %q into database: %w", req.Name, err)) } err = s.Authorizer.AddProfile(r.Context(), p.Name, req.Name) if err != nil { logger.Error("Failed to add profile to authorizer", logger.Ctx{"name": req.Name, "project": p.Name, "error": err}) } requestor := request.CreateRequestor(r) lc := lifecycle.ProfileCreated.Event(req.Name, p.Name, requestor, nil) s.Events.SendLifecycle(p.Name, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation GET /1.0/profiles/{name} profiles profile_get // // Get the profile // // Gets a specific profile. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Profile name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Profile // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Profile" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func profileGet(d *Daemon, r *http.Request) response.Response { s := d.State() p, err := project.ProfileProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } var resp *api.Profile err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { profile, err := dbCluster.GetProfile(ctx, tx.Tx(), p.Name, name) if err != nil { return fmt.Errorf("Fetch profile: %w", err) } profileConfigs, err := dbCluster.GetAllProfileConfigs(ctx, tx.Tx()) if err != nil { return err } profileDevices, err := dbCluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return err } resp, err = profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices) if err != nil { return err } resp.UsedBy, err = profileUsedBy(ctx, tx, *profile) if err != nil { return err } resp.UsedBy = project.FilterUsedBy(s.Authorizer, r, resp.UsedBy) return nil }) if err != nil { return response.SmartError(err) } etag := []any{resp.Config, resp.Description, resp.Devices} return response.SyncResponseETag(true, resp, etag) } // swagger:operation PUT /1.0/profiles/{name} profiles profile_put // // Update the profile // // Updates the entire profile configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Profile name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: profile // description: Profile configuration // required: true // schema: // $ref: "#/definitions/ProfilePut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func profilePut(d *Daemon, r *http.Request) response.Response { s := d.State() p, err := project.ProfileProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if isClusterNotification(r) { // In this case the ProfilePut request payload contains information about the old profile, since // the new one has already been saved in the database. old := api.ProfilePut{} err := json.NewDecoder(r.Body).Decode(&old) if err != nil { return response.BadRequest(err) } err = doProfileUpdateCluster(r.Context(), s, p.Name, name, old) return response.SmartError(err) } var profile *api.Profile err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { current, err := dbCluster.GetProfile(ctx, tx.Tx(), p.Name, name) if err != nil { return fmt.Errorf("Failed to retrieve profile %q: %w", name, err) } profile, err = current.ToAPI(ctx, tx.Tx(), nil, nil) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // Validate the ETag. etag := []any{profile.Config, profile.Description, profile.Devices} err = localUtil.EtagCheck(r, etag) if err != nil { return response.PreconditionFailed(err) } req := api.ProfilePut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } err = doProfileUpdate(r.Context(), s, *p, name, profile, req) if err == nil && !isClusterNotification(r) { // Notify all other nodes. If a node is down, it will be ignored. notifier, err := cluster.NewNotifier(s, s.Endpoints.NetworkCert(), s.ServerCert(), cluster.NotifyAlive) if err != nil { return response.SmartError(err) } err = notifier(func(client incus.InstanceServer) error { return client.UseProject(p.Name).UpdateProfile(name, profile.ProfilePut, "") }) if err != nil { return response.SmartError(err) } } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(p.Name, lifecycle.ProfileUpdated.Event(name, p.Name, requestor, nil)) return response.SmartError(err) } // swagger:operation PATCH /1.0/profiles/{name} profiles profile_patch // // Partially update the profile // // Updates a subset of the profile configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Profile name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: profile // description: Profile configuration // required: true // schema: // $ref: "#/definitions/ProfilePut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func profilePatch(d *Daemon, r *http.Request) response.Response { s := d.State() p, err := project.ProfileProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } var profile *api.Profile err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { current, err := dbCluster.GetProfile(ctx, tx.Tx(), p.Name, name) if err != nil { return fmt.Errorf("Failed to retrieve profile=%q: %w", name, err) } profile, err = current.ToAPI(ctx, tx.Tx(), nil, nil) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // Validate the ETag. etag := []any{profile.Config, profile.Description, profile.Devices} err = localUtil.EtagCheck(r, etag) if err != nil { return response.PreconditionFailed(err) } body, err := io.ReadAll(r.Body) if err != nil { return response.InternalError(err) } rdr1 := io.NopCloser(bytes.NewBuffer(body)) rdr2 := io.NopCloser(bytes.NewBuffer(body)) reqRaw := jmap.Map{} err = json.NewDecoder(rdr1).Decode(&reqRaw) if err != nil { return response.BadRequest(err) } req := api.ProfilePut{} err = json.NewDecoder(rdr2).Decode(&req) if err != nil { return response.BadRequest(err) } // Get Description. _, err = reqRaw.GetString("description") if err != nil { req.Description = profile.Description } // Get Config. if req.Config == nil { req.Config = profile.Config } else { for k, v := range profile.Config { _, ok := req.Config[k] if !ok { req.Config[k] = v } } } // Get Devices. if req.Devices == nil { req.Devices = profile.Devices } else { for k, v := range profile.Devices { _, ok := req.Devices[k] if !ok { req.Devices[k] = v } } } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(p.Name, lifecycle.ProfileUpdated.Event(name, p.Name, requestor, nil)) return response.SmartError(doProfileUpdate(r.Context(), s, *p, name, profile, req)) } // swagger:operation POST /1.0/profiles/{name} profiles profile_post // // Rename the profile // // Renames an existing profile. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Profile name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: profile // description: Profile rename request // required: true // schema: // $ref: "#/definitions/ProfilePost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func profilePost(d *Daemon, r *http.Request) response.Response { s := d.State() p, err := project.ProfileProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if name == "default" { return response.Forbidden(errors.New(`The "default" profile cannot be renamed`)) } req := api.ProfilePost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. if req.Name == "" { return response.BadRequest(errors.New("No name provided")) } err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid profile name: %w", err)) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Check that the profile exists. _, err = dbCluster.GetProfile(ctx, tx.Tx(), p.Name, name) if err != nil { return fmt.Errorf("Profile %q doesn't exist", name) } // Check that the name isn't already in use. _, err = dbCluster.GetProfile(ctx, tx.Tx(), p.Name, req.Name) if err == nil { return fmt.Errorf("Profile %q already exists", req.Name) } return dbCluster.RenameProfile(ctx, tx.Tx(), p.Name, name, req.Name) }) if err != nil { return response.SmartError(err) } err = s.Authorizer.RenameProfile(r.Context(), p.Name, name, req.Name) if err != nil { logger.Error("Failed to rename profile in authorizer", logger.Ctx{"old_name": name, "new_name": req.Name, "project": p.Name, "error": err}) } requestor := request.CreateRequestor(r) lc := lifecycle.ProfileRenamed.Event(req.Name, p.Name, requestor, logger.Ctx{"old_name": name}) s.Events.SendLifecycle(p.Name, lc) return response.SyncResponseLocation(true, nil, lc.Source) } // swagger:operation DELETE /1.0/profiles/{name} profiles profile_delete // // Delete the profile // // Removes the profile. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Profile name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func profileDelete(d *Daemon, r *http.Request) response.Response { s := d.State() p, err := project.ProfileProject(s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } name, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } if name == "default" { return response.Forbidden(errors.New(`The "default" profile cannot be deleted`)) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { profile, err := dbCluster.GetProfile(ctx, tx.Tx(), p.Name, name) if err != nil { return err } usedBy, err := profileUsedBy(ctx, tx, *profile) if err != nil { return err } if len(usedBy) > 0 { return errors.New("Profile is currently in use") } return dbCluster.DeleteProfile(ctx, tx.Tx(), p.Name, name) }) if err != nil { return response.SmartError(err) } err = s.Authorizer.DeleteProfile(r.Context(), p.Name, name) if err != nil { logger.Error("Failed to remove profile from authorizer", logger.Ctx{"name": name, "project": p.Name, "error": err}) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(p.Name, lifecycle.ProfileDeleted.Event(name, p.Name, requestor, nil)) return response.EmptySyncResponse } incus-7.0.0/cmd/incusd/profiles_utils.go000066400000000000000000000224161517523235500202760ustar00rootroot00000000000000package main import ( "context" "errors" "fmt" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" ) func doProfileUpdate(ctx context.Context, s *state.State, p api.Project, profileName string, profile *api.Profile, req api.ProfilePut) error { // Check project limits. err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { return project.AllowProfileUpdate(tx, p.Name, profileName, req) }) if err != nil { return err } // Quick checks. err = instance.ValidConfig(s.OS, req.Config, false, instancetype.Any) if err != nil { return err } // Profiles can be applied to any instance type, so just use instancetype.Any type for validation so that // instance type specific validation checks are not performed. err = instance.ValidDevices(s, p, instancetype.Any, deviceConfig.NewDevices(req.Devices), nil) if err != nil { return err } insts, projects, err := getProfileInstancesInfo(ctx, s.DB.Cluster, p.Name, profileName) if err != nil { return fmt.Errorf("Failed to query instances associated with profile %q: %w", profileName, err) } // Check if the root disk device's pool would be changed or removed and prevent that if there are instances // using that root disk device. oldProfileRootDiskDeviceKey, oldProfileRootDiskDevice, _ := internalInstance.GetRootDiskDevice(profile.Devices) _, newProfileRootDiskDevice, _ := internalInstance.GetRootDiskDevice(req.Devices) if len(insts) > 0 && oldProfileRootDiskDevice["pool"] != "" && newProfileRootDiskDevice["pool"] == "" || (oldProfileRootDiskDevice["pool"] != newProfileRootDiskDevice["pool"]) { // Check for instances using the device. for _, inst := range insts { // Check if the device is locally overridden. k, v, _ := internalInstance.GetRootDiskDevice(inst.Devices.CloneNative()) if k != "" && v["pool"] != "" { continue } err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Check what profile the device comes from by working backwards along the profiles list. for i := len(inst.Profiles) - 1; i >= 0; i-- { _, profile, err := tx.GetProfile(ctx, p.Name, inst.Profiles[i].Name) if err != nil { return err } // Check if we find a match for the device. _, ok := profile.Devices[oldProfileRootDiskDeviceKey] if ok { // Found the profile. if inst.Profiles[i].Name == profileName { // If it's the current profile, then we can't modify that root device. return errors.New("At least one instance relies on this profile's root disk device") } // If it's not, then move on to the next instance. break } } return nil }) if err != nil { return err } } } // Update the database. err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { devices, err := cluster.APIToDevices(req.Devices) if err != nil { return err } err = cluster.UpdateProfile(ctx, tx.Tx(), p.Name, profileName, cluster.Profile{ Project: p.Name, Name: profileName, Description: req.Description, }) if err != nil { return err } id, err := cluster.GetProfileID(ctx, tx.Tx(), p.Name, profileName) if err != nil { return err } err = cluster.UpdateProfileConfig(ctx, tx.Tx(), id, req.Config) if err != nil { return err } err = cluster.UpdateProfileDevices(ctx, tx.Tx(), id, devices) if err != nil { return err } newProfiles, err := cluster.GetProfilesIfEnabled(ctx, tx.Tx(), p.Name, []string{profileName}) if err != nil { return err } if len(newProfiles) != 1 { return fmt.Errorf("Failed to find profile %q in project %q", profileName, p.Name) } return nil }) if err != nil { return err } // Update all the instances on this node using the profile. Must be done after db.TxCommit due to DB lock. failures := map[*db.InstanceArgs]error{} for _, it := range insts { inst := it // Local var for instance pointer. if inst.Node != "" && inst.Node != s.ServerName { continue // This instance does not belong to this member, skip. } err := doProfileUpdateInstance(ctx, s, inst, *projects[inst.Project]) if err != nil { failures[&inst] = err } } if len(failures) != 0 { msg := "The following instances failed to update (profile change still saved):\n" for inst, err := range failures { msg += fmt.Sprintf(" - Project: %s, Instance: %s: %v\n", inst.Project, inst.Name, err) } return fmt.Errorf("%s", msg) } return nil } // Like doProfileUpdate but does not update the database, since it was already // updated by doProfileUpdate itself, called on the notifying node. func doProfileUpdateCluster(ctx context.Context, s *state.State, projectName string, profileName string, old api.ProfilePut) error { insts, projects, err := getProfileInstancesInfo(ctx, s.DB.Cluster, projectName, profileName) if err != nil { return fmt.Errorf("Failed to query instances associated with profile %q: %w", profileName, err) } failures := map[*db.InstanceArgs]error{} for _, it := range insts { inst := it // Local var for instance pointer. if inst.Node != "" && inst.Node != s.ServerName { continue // This instance does not belong to this member, skip. } for i, profile := range inst.Profiles { if profile.Name == profileName { // As profile has already been updated in the database by this point, overwrite the // new config from the database with the old config and devices, so that // doProfileUpdateInstance will detect the changes and apply them. inst.Profiles[i].Config = old.Config inst.Profiles[i].Devices = old.Devices break } } err := doProfileUpdateInstance(ctx, s, inst, *projects[inst.Project]) if err != nil { failures[&inst] = err } } if len(failures) != 0 { msg := "The following instances failed to update (profile change still saved):\n" for inst, err := range failures { msg += fmt.Sprintf(" - Project: %s, Instance: %s: %v\n", inst.Project, inst.Name, err) } return fmt.Errorf("%s", msg) } return nil } // Profile update of a single instance. func doProfileUpdateInstance(ctx context.Context, s *state.State, args db.InstanceArgs, p api.Project) error { profileNames := make([]string, 0, len(args.Profiles)) for _, profile := range args.Profiles { profileNames = append(profileNames, profile.Name) } var profiles []api.Profile err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error profiles, err = tx.GetProfiles(ctx, args.Project, profileNames) return err }) if err != nil { return err } // Load the instance using the old profile config. inst, err := instance.Load(s, args, p) if err != nil { return err } // Update will internally load the new profile configs and detect the changes to apply. return inst.Update(db.InstanceArgs{ Architecture: inst.Architecture(), Config: inst.LocalConfig(), Description: inst.Description(), Devices: inst.LocalDevices(), Ephemeral: inst.IsEphemeral(), Profiles: profiles, // Supply with new profile config. Project: inst.Project().Name, Type: inst.Type(), Snapshot: inst.IsSnapshot(), }, true) } // Query the db for information about instances associated with the given profile. func getProfileInstancesInfo(ctx context.Context, dbCluster *db.Cluster, projectName string, profileName string) (map[int]db.InstanceArgs, map[string]*api.Project, error) { var projectInstNames map[string][]string // Query the db for information about instances associated with the given profile. err := dbCluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error projectInstNames, err = tx.GetInstancesWithProfile(ctx, projectName, profileName) return err }) if err != nil { return nil, nil, fmt.Errorf("Failed to query instances with profile %q: %w", profileName, err) } var instances map[int]db.InstanceArgs projects := make(map[string]*api.Project) err = dbCluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var dbInstances []cluster.Instance for instProject, instNames := range projectInstNames { // Load project if not already loaded. _, found := projects[instProject] if !found { dbProject, err := cluster.GetProject(context.Background(), tx.Tx(), instProject) if err != nil { return err } projects[instProject], err = dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return err } } for _, instName := range instNames { dbInst, err := cluster.GetInstance(ctx, tx.Tx(), instProject, instName) if err != nil { return err } dbInstances = append(dbInstances, *dbInst) } } instances, err = tx.InstancesToInstanceArgs(ctx, true, dbInstances...) if err != nil { return err } return nil }) if err != nil { return nil, nil, fmt.Errorf("Failed to fetch instances: %w", err) } return instances, projects, nil } incus-7.0.0/cmd/incusd/resources.go000066400000000000000000000077211517523235500172470ustar00rootroot00000000000000package main import ( "net/http" "net/url" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/response" storagePools "github.com/lxc/incus/v7/internal/server/storage" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/resources" ) var api10ResourcesCmd = APIEndpoint{ Path: "resources", Get: APIEndpointAction{Handler: api10ResourcesGet, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanViewResources)}, } var storagePoolResourcesCmd = APIEndpoint{ Path: "storage-pools/{name}/resources", Get: APIEndpointAction{Handler: storagePoolResourcesGet, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanViewResources)}, } // swagger:operation GET /1.0/resources server resources_get // // Get system resources information // // Gets the hardware information profile of the server. // // --- // produces: // - application/json // parameters: // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Hardware resources // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Resources" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func api10ResourcesGet(d *Daemon, r *http.Request) response.Response { s := d.State() // If a target was specified, forward the request to the relevant node. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } // Get the local resource usage res, err := resources.GetResources() if err != nil { return response.SmartError(err) } return response.SyncResponse(true, res) } // swagger:operation GET /1.0/storage-pools/{name}/resources storage storage_pool_resources // // Get storage pool resources information // // Gets the usage information for the storage pool. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Resource name // type: string // required: true // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Hardware resources // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/ResourcesStoragePool" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolResourcesGet(d *Daemon, r *http.Request) response.Response { s := d.State() // If a target was specified, forward the request to the relevant node. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } // Get the existing storage pool poolName, err := url.PathUnescape(mux.Vars(r)["name"]) if err != nil { return response.SmartError(err) } var res *api.ResourcesStoragePool pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.InternalError(err) } res, err = pool.GetResources() if err != nil { return response.InternalError(err) } return response.SyncResponse(true, res) } incus-7.0.0/cmd/incusd/response.go000066400000000000000000000051151517523235500170660ustar00rootroot00000000000000package main import ( "net/http" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" ) func forwardedResponseToNode(s *state.State, r *http.Request, memberName string) response.Response { // Figure out the address of the target member (which is possibly this very same member). address, err := cluster.ResolveTarget(r.Context(), s, memberName) if err != nil { return response.SmartError(err) } // Forward the response if not local. if address != "" { client, err := cluster.Connect(address, s.Endpoints.NetworkCert(), s.ServerCert(), r, false) if err != nil { return response.SmartError(err) } return response.ForwardedResponse(client, r) } return nil } // forwardedResponseIfTargetIsRemote forwards a request to the request has a target parameter pointing to a member // which is not the local one. func forwardedResponseIfTargetIsRemote(s *state.State, r *http.Request) response.Response { targetNode := request.QueryParam(r, "target") if targetNode == "" { return nil } return forwardedResponseToNode(s, r, targetNode) } // forwardedResponseIfInstanceIsRemote redirects a request to the node running // the container with the given name. If the container is local, nothing gets // done and nil is returned. func forwardedResponseIfInstanceIsRemote(s *state.State, r *http.Request, project, name string) (response.Response, error) { client, err := cluster.ConnectIfInstanceIsRemote(s, project, name, r) if err != nil { return nil, err } if client == nil { return nil, nil } return response.ForwardedResponse(client, r), nil } // forwardedResponseIfVolumeIsRemote redirects a request to the node hosting // the volume with the given pool ID, name and type. If the container is local, // nothing gets done and nil is returned. If more than one node has a matching // volume, an error is returned. // // This is used when no targetNode is specified, and saves users some typing // when the volume name/type is unique to a node. func forwardedResponseIfVolumeIsRemote(s *state.State, r *http.Request, poolName string, projectName string, volumeName string, volumeType int) response.Response { if request.QueryParam(r, "target") != "" { return nil } client, err := cluster.ConnectIfVolumeIsRemote(s, poolName, projectName, volumeName, volumeType, s.Endpoints.NetworkCert(), s.ServerCert(), r) if err != nil { return response.SmartError(err) } if client == nil { return nil } return response.ForwardedResponse(client, r) } incus-7.0.0/cmd/incusd/snapshot_common.go000066400000000000000000000060701517523235500204400ustar00rootroot00000000000000package main import ( "fmt" "strconv" "strings" "time" "github.com/adhocore/gronx" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/util" ) // SnapshotScheduleAliases contains the mapping of scheduling aliases to cron syntax // including placeholders for scheduled time obfuscation. var SnapshotScheduleAliases = map[string]string{ "@hourly": "%s * * * *", "@daily": "%s %s * * *", "@midnight": "%s 0 * * *", "@weekly": "%s %s * * 0", "@monthly": "%s %s 1 * *", "@annually": "%s %s 1 1 *", "@yearly": "%s %s 1 1 *", "@never": "", } func snapshotIsScheduledNow(spec string, subjectID int64) bool { result := false specs := buildCronSpecs(spec, subjectID) for _, curSpec := range specs { isNow, err := cronSpecIsNow(curSpec) if err == nil && isNow { result = true } } return result } func buildCronSpecs(spec string, subjectID int64) []string { var result []string if strings.Contains(spec, ", ") { for _, curSpec := range util.SplitNTrimSpace(spec, ",", -1, true) { entry := getCronSyntax(curSpec, subjectID) if entry != "" { result = append(result, entry) } } } else { entry := getCronSyntax(spec, subjectID) if entry != "" { result = append(result, entry) } } return result } func getCronSyntax(spec string, subjectID int64) string { alias, isAlias := SnapshotScheduleAliases[strings.ToLower(spec)] if isAlias { if alias == "@never" { return "" } obfuscatedMinute, obfuscatedHour := getObfuscatedTimeValuesForSubject(subjectID) if strings.Count(alias, "%s") > 1 { return fmt.Sprintf(alias, obfuscatedMinute, obfuscatedHour) } else { return fmt.Sprintf(alias, obfuscatedMinute) } } return spec } func getObfuscatedTimeValuesForSubject(subjectID int64) (string, string) { minuteResult := "0" hourResult := "0" minSequence, minSequenceErr := localUtil.GenerateSequenceInt64(0, 60, 1) min, minErr := localUtil.GetStableRandomInt64FromList(subjectID, minSequence) if minErr == nil && minSequenceErr == nil { minuteResult = strconv.FormatInt(min, 10) } hourSequence, hourSequenceErr := localUtil.GenerateSequenceInt64(0, 24, 1) hour, hourErr := localUtil.GetStableRandomInt64FromList(subjectID, hourSequence) if hourErr == nil && hourSequenceErr == nil { hourResult = strconv.FormatInt(hour, 10) } return minuteResult, hourResult } func cronSpecIsNow(spec string) (bool, error) { // Check if it's time to snapshot now := time.Now() // Truncate the time now back to the start of the minute. // This is needed because the cron scheduler will add a minute to the scheduled time // and we don't want the next scheduled time to roll over to the next minute and break // the time comparison below. now = now.Truncate(time.Minute) // Calculate the next scheduled time based on the snapshots.schedule // pattern and the time now. next, err := gronx.NextTickAfter(spec, now, false) if err != nil { return false, fmt.Errorf("Could not parse cron '%s': %w", spec, err) } if !now.Add(time.Minute).Equal(next) { return false, nil } return true, nil } incus-7.0.0/cmd/incusd/snapshot_common_test.go000066400000000000000000000020241517523235500214720ustar00rootroot00000000000000package main import ( "testing" "github.com/stretchr/testify/suite" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" ) type snapshotCommonTestSuite struct { daemonTestSuite } func (s *snapshotCommonTestSuite) TestSnapshotScheduling() { args := db.InstanceArgs{ Type: instancetype.Container, Ephemeral: false, Name: "hal9000", } c, op, _, err := instance.CreateInternal(s.d.State(), args, nil, true, true, false) s.Req.Nil(err) s.Equal(true, snapshotIsScheduledNow("* * * * *", int64(c.ID())), "snapshot.schedule config '* * * * *' should have matched now") s.Equal(true, snapshotIsScheduledNow("@daily,"+ "@hourly,"+ "@midnight,"+ "@weekly,"+ "@monthly,"+ "@annually,"+ "@yearly,"+ " * * * * *", int64(c.ID())), "snapshot.schedule config '* * * * *' should have matched now") op.Done(nil) } func TestSnapshotCommon(t *testing.T) { suite.Run(t, &snapshotCommonTestSuite{}) } incus-7.0.0/cmd/incusd/storage.go000066400000000000000000000151671517523235500167040ustar00rootroot00000000000000package main import ( "context" "fmt" "slices" "sync" "sync/atomic" "time" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/warningtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/internal/server/warnings" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) // Simple cache used to store the activated drivers on this server. // This allows us to avoid querying the database every time an API call is made. var ( storagePoolUsedDriversCacheVal atomic.Value storagePoolSupportedDriversCacheVal atomic.Value storagePoolDriversCacheLock sync.Mutex ) // readStoragePoolDriversCache returns supported and used storage driver info. func readStoragePoolDriversCache() ([]api.ServerStorageDriverInfo, map[string]string) { usedDrivers := storagePoolUsedDriversCacheVal.Load() if usedDrivers == nil { usedDrivers = map[string]string{} } supportedDrivers := storagePoolSupportedDriversCacheVal.Load() if supportedDrivers == nil { supportedDrivers = []api.ServerStorageDriverInfo{} } return supportedDrivers.([]api.ServerStorageDriverInfo), usedDrivers.(map[string]string) } func storageStartup(s *state.State) error { // Update the storage drivers supported and used cache in api_1.0.go. storagePoolDriversCacheUpdate(s.ShutdownCtx, s) var poolNames []string err := s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { var err error poolNames, err = tx.GetCreatedStoragePoolNames(ctx) return err }) if err != nil { if response.IsNotFoundError(err) { logger.Debug("No existing storage pools detected") return nil } return fmt.Errorf("Failed loading existing storage pools: %w", err) } initPools := make(map[string]struct{}, len(poolNames)) for _, poolName := range poolNames { initPools[poolName] = struct{}{} } initPool := func(poolName string) bool { logger.Debug("Initializing storage pool", logger.Ctx{"pool": poolName}) pool, err := storagePools.LoadByName(s, poolName) if err != nil { if response.IsNotFoundError(err) { return true // Nothing to activate as pool has been deleted. } logger.Error("Failed loading storage pool", logger.Ctx{"pool": poolName, "err": err}) return false } _, err = pool.Mount() if err != nil { logger.Error("Failed mounting storage pool", logger.Ctx{"pool": poolName, "err": err}) _ = s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpsertWarningLocalNode(ctx, "", cluster.TypeStoragePool, int(pool.ID()), warningtype.StoragePoolUnvailable, err.Error()) }) return false } logger.Info("Initialized storage pool", logger.Ctx{"pool": poolName}) _ = warnings.ResolveWarningsByLocalNodeAndProjectAndTypeAndEntity(s.DB.Cluster, "", warningtype.StoragePoolUnvailable, cluster.TypeStoragePool, int(pool.ID())) return true } // Try initializing storage pools in random order. for poolName := range initPools { if initPool(poolName) { // Storage pool initialized successfully so remove it from the list so its not retried. delete(initPools, poolName) } } // For any remaining storage pools that were not successfully initialized, we now start a go routine to // periodically try to initialize them again in the background. if len(initPools) > 0 { go func() { for { t := time.NewTimer(time.Duration(time.Minute)) select { case <-s.ShutdownCtx.Done(): t.Stop() return case <-t.C: t.Stop() // Try initializing remaining storage pools in random order. tryInstancesStart := false for poolName := range initPools { if initPool(poolName) { // Storage pool initialized successfully or deleted so // remove it from the list so its not retried. delete(initPools, poolName) tryInstancesStart = true } } if len(initPools) <= 0 { logger.Info("All storage pools initialized") } // At least one remaining storage pool was initialized, check if any // instances can now start. if tryInstancesStart { instances, err := instance.LoadNodeAll(s, instancetype.Any) if err != nil { logger.Error("Failed loading instances to start", logger.Ctx{"err": err}) } else { instancesStart(s, instances) } } if len(initPools) <= 0 { return // Our job here is done. } } } }() } else { logger.Info("All storage pools initialized") } return nil } func storagePoolDriversCacheUpdate(ctx context.Context, s *state.State) { // Get a list of all storage drivers currently in use // on this server. Only do this when we do not already have done // this once to avoid unnecessarily querying the db. All subsequent // updates of the cache will be done when we create or delete storage // pools in the db. Since this is a rare event, this cache // implementation is a classic frequent-read, rare-update case so // copy-on-write semantics without locking in the read case seems // appropriate. (Should be cheaper then querying the db all the time, // especially if we keep adding more storage drivers.) var drivers []string err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error drivers, err = tx.GetStoragePoolDrivers(ctx) return err }) if err != nil && !response.IsNotFoundError(err) { return } usedDrivers := map[string]string{} // Get the driver info. info := storageDrivers.SupportedDrivers(s) supportedDrivers := make([]api.ServerStorageDriverInfo, 0, len(info)) for _, entry := range info { supportedDrivers = append(supportedDrivers, api.ServerStorageDriverInfo{ Name: entry.Name, Version: entry.Version, Remote: entry.Remote, }) if slices.Contains(drivers, entry.Name) { usedDrivers[entry.Name] = entry.Version } } // Prepare the cache entries. backends := []string{} for k, v := range usedDrivers { backends = append(backends, fmt.Sprintf("%s %s", k, v)) } // Update the user agent. version.UserAgentStorageBackends(backends) storagePoolDriversCacheLock.Lock() storagePoolUsedDriversCacheVal.Store(usedDrivers) storagePoolSupportedDriversCacheVal.Store(supportedDrivers) storagePoolDriversCacheLock.Unlock() } incus-7.0.0/cmd/incusd/storage_buckets.go000066400000000000000000001323131517523235500204150ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "os" "sort" "strconv" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/filter" internalIO "github.com/lxc/incus/v7/internal/io" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var storagePoolBucketsCmd = APIEndpoint{ Path: "storage-pools/{poolName}/buckets", Get: APIEndpointAction{Handler: storagePoolBucketsGet, AccessHandler: allowAuthenticated}, Post: APIEndpointAction{Handler: storagePoolBucketsPost, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanCreateStorageBuckets), LargeRequest: true}, } var storagePoolBucketCmd = APIEndpoint{ Path: "storage-pools/{poolName}/buckets/{bucketName}", Delete: APIEndpointAction{Handler: storagePoolBucketDelete, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanEdit, "poolName", "bucketName", "location")}, Get: APIEndpointAction{Handler: storagePoolBucketGet, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanView, "poolName", "bucketName", "location")}, Patch: APIEndpointAction{Handler: storagePoolBucketPut, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanEdit, "poolName", "bucketName", "location")}, Put: APIEndpointAction{Handler: storagePoolBucketPut, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanEdit, "poolName", "bucketName", "location")}, } var storagePoolBucketKeysCmd = APIEndpoint{ Path: "storage-pools/{poolName}/buckets/{bucketName}/keys", Get: APIEndpointAction{Handler: storagePoolBucketKeysGet, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanView, "poolName", "bucketName", "location")}, Post: APIEndpointAction{Handler: storagePoolBucketKeysPost, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanEdit, "poolName", "bucketName", "location")}, } var storagePoolBucketKeyCmd = APIEndpoint{ Path: "storage-pools/{poolName}/buckets/{bucketName}/keys/{keyName}", Delete: APIEndpointAction{Handler: storagePoolBucketKeyDelete, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanEdit, "poolName", "bucketName", "location")}, Get: APIEndpointAction{Handler: storagePoolBucketKeyGet, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanView, "poolName", "bucketName", "location")}, Put: APIEndpointAction{Handler: storagePoolBucketKeyPut, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanEdit, "poolName", "bucketName", "location")}, } // API endpoints // swagger:operation GET /1.0/storage-pools/{poolName}/buckets storage storage_pool_buckets_get // // Get the storage pool buckets // // Returns a list of storage pool buckets (URLs). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve storage pool buckets from all projects // type: boolean // example: true // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/storage-pools/default/buckets/foo", // "/1.0/storage-pools/default/buckets/bar", // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/storage-pools/{poolName}/buckets?recursion=1 storage storage_pool_buckets_get_recursion1 // // Get the storage pool buckets // // Returns a list of storage pool buckets (structs). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve storage pool buckets from all projects // type: boolean // example: true // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of storage pool buckets // items: // $ref: "#/definitions/StorageBucket" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/storage-pools/{poolName}/buckets?recursion=2 storage storage_pool_buckets_get_recursion2 // // Get the storage pool bucket details // // Returns a list of storage pool buckets with all details (structs). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: all-projects // description: Retrieve storage pool buckets from all projects // type: boolean // example: true // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of storage pool buckets // items: // $ref: "#/definitions/StorageBucketFull" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketsGet(d *Daemon, r *http.Request) response.Response { s := d.State() requestProjectName := request.ProjectParam(r) allProjects := util.IsTrue(r.FormValue("all-projects")) bucketProjectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, requestProjectName) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } driverInfo := pool.Driver().Info() if !driverInfo.Buckets { return response.BadRequest(fmt.Errorf("Storage pool driver %q does not support buckets", driverInfo.Name)) } // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } memberSpecific := false // Get buckets for all cluster members. var dbBuckets []*db.StorageBucket err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { poolID := pool.ID() filter := db.StorageBucketFilter{ PoolID: &poolID, } if !allProjects { filter.Project = &bucketProjectName } dbBuckets, err = tx.GetStoragePoolBuckets(ctx, memberSpecific, filter) if err != nil { return fmt.Errorf("Failed loading storage buckets: %w", err) } return nil }) if err != nil { return response.SmartError(err) } userHasPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanView, auth.ObjectTypeStorageBucket) if err != nil { return response.SmartError(err) } var filteredDBBuckets []*db.StorageBucket for _, bucket := range dbBuckets { var location string if s.ServerClustered && !pool.Driver().Info().Remote { location = bucket.Location } if !userHasPermission(auth.ObjectStorageBucket(requestProjectName, poolName, bucket.Name, location)) { continue } if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(bucket.StorageBucket, *clauses) if err != nil { return response.SmartError(err) } if !match { continue } } filteredDBBuckets = append(filteredDBBuckets, bucket) } // Sort by bucket name. sort.SliceStable(filteredDBBuckets, func(i, j int) bool { bucketA := filteredDBBuckets[i] bucketB := filteredDBBuckets[j] return bucketA.Name < bucketB.Name }) recursionStr := r.FormValue("recursion") recursion, err := strconv.Atoi(recursionStr) if err != nil { recursion = 0 } if recursion > 0 { buckets := make([]*api.StorageBucket, 0, len(filteredDBBuckets)) for _, dbBucket := range filteredDBBuckets { u := pool.GetBucketURL(dbBucket.Name) if u != nil { dbBucket.S3URL = u.String() } buckets = append(buckets, &dbBucket.StorageBucket) } if recursion == 2 { bucketsFull := make([]*api.StorageBucketFull, 0, len(buckets)) for i, bucket := range buckets { fullBucket, err := getBucketFull(r.Context(), s, pool, filteredDBBuckets[i].ID, *bucket) if err != nil { return response.InternalError(err) } bucketsFull = append(bucketsFull, fullBucket) } return response.SyncResponse(true, bucketsFull) } return response.SyncResponse(true, buckets) } urls := make([]string, 0, len(filteredDBBuckets)) for _, dbBucket := range filteredDBBuckets { urls = append(urls, dbBucket.StorageBucket.URL(version.APIVersion, poolName, requestProjectName).String()) } return response.SyncResponse(true, urls) } // swagger:operation GET /1.0/storage-pools/{poolName}/buckets/{bucketName} storage storage_pool_bucket_get // // Get the storage pool bucket // // Gets a specific storage pool bucket. // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Storage pool bucket // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/StorageBucket" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/storage-pools/{poolName}/buckets/{bucketName}?recursion=1 storage storage_pool_bucket_get_recursion1 // // Get the full storage pool bucket details // // Gets a specific storage pool bucket with all details (backups and keys). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Storage pool bucket // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/StorageBucketFull" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketGet(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } bucketProjectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } if !pool.Driver().Info().Buckets { return response.BadRequest(errors.New("Storage pool does not support buckets")) } bucketName, err := url.PathUnescape(mux.Vars(r)["bucketName"]) if err != nil { return response.SmartError(err) } targetMember := request.QueryParam(r, "target") memberSpecific := targetMember != "" var bucket *db.StorageBucket err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { bucket, err = tx.GetStoragePoolBucket(ctx, pool.ID(), bucketProjectName, memberSpecific, bucketName) return err }) if err != nil { return response.SmartError(err) } u := pool.GetBucketURL(bucket.Name) if u != nil { bucket.S3URL = u.String() } // Prepare the response. if localUtil.IsRecursionRequest(r) { bucketFull, err := getBucketFull(r.Context(), s, pool, bucket.ID, bucket.StorageBucket) if err != nil { return response.InternalError(err) } return response.SyncResponseETag(true, bucketFull, bucket.Etag()) } return response.SyncResponseETag(true, bucket.StorageBucket, bucket.Etag()) } func getBucketFull(ctx context.Context, s *state.State, pool storagePools.Pool, id int64, bucket api.StorageBucket) (*api.StorageBucketFull, error) { // Set the base object. resp := api.StorageBucketFull{ StorageBucket: bucket, Backups: []api.StorageBucketBackup{}, Keys: []api.StorageBucketKey{}, } // Add all backups. err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { bucketBackups, err := tx.GetStoragePoolBucketBackups(ctx, bucket.Project, bucket.Name, pool.ID()) if err != nil { return err } for _, entry := range bucketBackups { _, backupName, _ := api.GetParentAndSnapshotName(entry.Name) resp.Backups = append(resp.Backups, *backup.NewBucketBackup(s, bucket.Project, pool.Name(), bucket.Name, entry.ID, backupName, entry.CreationDate, entry.ExpiryDate).Render()) } return nil }) if err != nil { return nil, err } // Add all keys. err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { dbBucketKeys, err := tx.GetStoragePoolBucketKeys(ctx, id) if err != nil { return fmt.Errorf("Failed loading storage bucket keys: %w", err) } for _, dbBucketKey := range dbBucketKeys { resp.Keys = append(resp.Keys, dbBucketKey.StorageBucketKey) } return nil }) if err != nil { return nil, err } return &resp, nil } // swagger:operation POST /1.0/storage-pools/{poolName}/buckets storage storage_pool_bucket_post // // Add a storage pool bucket. // // Creates a new storage pool bucket. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: bucket // description: Bucket // required: true // schema: // $ref: "#/definitions/StorageBucketsPost" // responses: // "200": // $ref: '#/definitions/StorageBucketKey' // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketsPost(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } bucketProjectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } if r.Header.Get("Content-Type") == "application/octet-stream" { return createStoragePoolBucketFromBackup(s, r, request.ProjectParam(r), bucketProjectName, r.Body, poolName, r.Header.Get("X-Incus-name")) } // Parse the request into a record. req := api.StorageBucketsPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } // Quick checks. err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid storage bucket name: %w", err)) } reverter := revert.New() defer reverter.Fail() err = pool.CreateBucket(bucketProjectName, req, nil) if err != nil { return response.SmartError(fmt.Errorf("Failed creating storage bucket: %w", err)) } reverter.Add(func() { _ = pool.DeleteBucket(bucketProjectName, req.Name, nil) }) // Create admin key for new bucket. adminKeyReq := api.StorageBucketKeysPost{ StorageBucketKeyPut: api.StorageBucketKeyPut{ Role: "admin", Description: "Admin user", }, Name: "admin", } adminKey, err := pool.CreateBucketKey(bucketProjectName, req.Name, adminKeyReq, nil) if err != nil { return response.SmartError(fmt.Errorf("Failed creating storage bucket admin key: %w", err)) } var location string if s.ServerClustered && !pool.Driver().Info().Remote { location = s.ServerName } err = s.Authorizer.AddStorageBucket(r.Context(), bucketProjectName, poolName, req.Name, location) if err != nil { logger.Error("Failed to add storage bucket to authorizer", logger.Ctx{"name": req.Name, "pool": poolName, "project": bucketProjectName, "error": err}) } s.Events.SendLifecycle(bucketProjectName, lifecycle.StorageBucketCreated.Event(pool, bucketProjectName, req.Name, request.CreateRequestor(r), nil)) u := api.NewURL().Path(version.APIVersion, "storage-pools", pool.Name(), "buckets", req.Name) reverter.Success() return response.SyncResponseLocation(true, adminKey, u.String()) } // swagger:operation PATCH /1.0/storage-pools/{name}/buckets/{bucketName} storage storage_pool_bucket_patch // // Partially update the storage bucket. // // Updates a subset of the storage bucket configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Resource name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: storage bucket // description: Storage bucket configuration // required: true // schema: // $ref: "#/definitions/StorageBucketPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation PUT /1.0/storage-pools/{name}/buckets/{bucketName} storage storage_pool_bucket_put // // Update the storage bucket // // Updates the entire storage bucket configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Resource name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: storage bucket // description: Storage bucket configuration // required: true // schema: // $ref: "#/definitions/StorageBucketPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketPut(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } bucketProjectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } bucketName, err := url.PathUnescape(mux.Vars(r)["bucketName"]) if err != nil { return response.SmartError(err) } // Decode the request. req := api.StorageBucketPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } if r.Method == http.MethodPatch { targetMember := request.QueryParam(r, "target") memberSpecific := targetMember != "" var bucket *db.StorageBucket err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { bucket, err = tx.GetStoragePoolBucket(ctx, pool.ID(), bucketProjectName, memberSpecific, bucketName) return err }) if err != nil { return response.SmartError(err) } // If config being updated via "patch" method, then merge all existing config with the keys that // are present in the request config. for k, v := range bucket.Config { _, ok := req.Config[k] if !ok { req.Config[k] = v } } } err = pool.UpdateBucket(bucketProjectName, bucketName, req, nil) if err != nil { return response.SmartError(fmt.Errorf("Failed updating storage bucket: %w", err)) } s.Events.SendLifecycle(bucketProjectName, lifecycle.StorageBucketUpdated.Event(pool, bucketProjectName, bucketName, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation DELETE /1.0/storage-pools/{name}/buckets/{bucketName} storage storage_pool_bucket_delete // // Delete the storage bucket // // Removes the storage bucket. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Resource name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketDelete(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } bucketProjectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } bucketName, err := url.PathUnescape(mux.Vars(r)["bucketName"]) if err != nil { return response.SmartError(err) } err = pool.DeleteBucket(bucketProjectName, bucketName, nil) if err != nil { return response.SmartError(fmt.Errorf("Failed deleting storage bucket: %w", err)) } s.Events.SendLifecycle(bucketProjectName, lifecycle.StorageBucketDeleted.Event(pool, bucketProjectName, bucketName, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // API endpoints // swagger:operation GET /1.0/storage-pools/{poolName}/buckets/{bucketName}/keys storage storage_pool_bucket_keys_get // // Get the storage pool bucket keys // // Returns a list of storage pool bucket keys (URLs). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/storage-pools/default/buckets/foo/keys/my-read-only-key", // "/1.0/storage-pools/default/buckets/bar/keys/admin", // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/storage-pools/{poolName}/buckets/{bucketName}/keys?recursion=1 storage storage_pool_bucket_keys_get_recursion1 // // Get the storage pool bucket keys // // Returns a list of storage pool bucket keys (structs). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of storage pool bucket keys // items: // $ref: "#/definitions/StorageBucketKey" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketKeysGet(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } bucketProjectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } driverInfo := pool.Driver().Info() if !driverInfo.Buckets { return response.BadRequest(fmt.Errorf("Storage pool driver %q does not support buckets", driverInfo.Name)) } bucketName, err := url.PathUnescape(mux.Vars(r)["bucketName"]) if err != nil { return response.SmartError(err) } // If target is set, get buckets only for this cluster members. targetMember := request.QueryParam(r, "target") memberSpecific := targetMember != "" var dbBucket *db.StorageBucket var dbBucketKeys []*db.StorageBucketKey err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbBucket, err = tx.GetStoragePoolBucket(ctx, pool.ID(), bucketProjectName, memberSpecific, bucketName) if err != nil { return fmt.Errorf("Failed loading storage bucket: %w", err) } dbBucketKeys, err = tx.GetStoragePoolBucketKeys(ctx, dbBucket.ID) if err != nil { return fmt.Errorf("Failed loading storage bucket keys: %w", err) } return nil }) if err != nil { return response.SmartError(err) } if localUtil.IsRecursionRequest(r) { bucketKeys := make([]*api.StorageBucketKey, 0, len(dbBucketKeys)) for _, dbBucketKey := range dbBucketKeys { bucketKeys = append(bucketKeys, &dbBucketKey.StorageBucketKey) } return response.SyncResponse(true, bucketKeys) } bucketKeyURLs := make([]string, 0, len(dbBucketKeys)) for _, dbBucketKey := range dbBucketKeys { bucketKeyURLs = append(bucketKeyURLs, dbBucketKey.URL(version.APIVersion, poolName, bucketProjectName, bucketName).String()) } return response.SyncResponse(true, bucketKeyURLs) } // swagger:operation POST /1.0/storage-pools/{poolName}/buckets/{bucketName}/keys storage storage_pool_bucket_key_post // // Add a storage pool bucket key. // // Creates a new storage pool bucket key. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: bucket // description: Bucket // required: true // schema: // $ref: "#/definitions/StorageBucketKeysPost" // responses: // "200": // $ref: '#/definitions/StorageBucketKey' // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketKeysPost(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } bucketProjectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } bucketName, err := url.PathUnescape(mux.Vars(r)["bucketName"]) if err != nil { return response.SmartError(err) } // Parse the request into a record. req := api.StorageBucketKeysPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } key, err := pool.CreateBucketKey(bucketProjectName, bucketName, req, nil) if err != nil { return response.SmartError(fmt.Errorf("Failed creating storage bucket key: %w", err)) } lc := lifecycle.StorageBucketKeyCreated.Event(pool, bucketProjectName, pool.Name(), req.Name, request.CreateRequestor(r), nil) s.Events.SendLifecycle(bucketProjectName, lc) return response.SyncResponseLocation(true, key, lc.Source) } // swagger:operation DELETE /1.0/storage-pools/{name}/buckets/{bucketName}/keys/{keyName} storage storage_pool_bucket_key_delete // // Delete the storage bucket key // // Removes the storage bucket key. // // --- // produces: // - application/json // parameters: // - in: path // name: name // description: Resource name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: path // name: keyName // description: Storage bucket key name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketKeyDelete(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } bucketProjectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } bucketName, err := url.PathUnescape(mux.Vars(r)["bucketName"]) if err != nil { return response.SmartError(err) } keyName, err := url.PathUnescape(mux.Vars(r)["keyName"]) if err != nil { return response.SmartError(err) } err = pool.DeleteBucketKey(bucketProjectName, bucketName, keyName, nil) if err != nil { return response.SmartError(fmt.Errorf("Failed deleting storage bucket key: %w", err)) } s.Events.SendLifecycle(bucketProjectName, lifecycle.StorageBucketKeyDeleted.Event(pool, bucketProjectName, pool.Name(), bucketName, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } // swagger:operation GET /1.0/storage-pools/{poolName}/buckets/{bucketName}/keys/{keyName} storage storage_pool_bucket_key_get // // Get the storage pool bucket key // // Gets a specific storage pool bucket key. // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: path // name: keyName // description: Storage bucket key name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Storage pool bucket key // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/StorageBucketKey" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketKeyGet(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } bucketProjectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } if !pool.Driver().Info().Buckets { return response.BadRequest(errors.New("Storage pool does not support buckets")) } bucketName, err := url.PathUnescape(mux.Vars(r)["bucketName"]) if err != nil { return response.SmartError(err) } keyName, err := url.PathUnescape(mux.Vars(r)["keyName"]) if err != nil { return response.SmartError(err) } targetMember := request.QueryParam(r, "target") memberSpecific := targetMember != "" var bucketKey *db.StorageBucketKey err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { bucket, err := tx.GetStoragePoolBucket(ctx, pool.ID(), bucketProjectName, memberSpecific, bucketName) if err != nil { return err } bucketKey, err = tx.GetStoragePoolBucketKey(ctx, bucket.ID, keyName) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } return response.SyncResponseETag(true, bucketKey.StorageBucketKey, bucketKey.Etag()) } // swagger:operation PUT /1.0/storage-pools/{name}/buckets/{bucketName}/keys/{keyName} storage storage_pool_bucket_key_put // // Update the storage bucket key // // Updates the entire storage bucket key configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: name // description: Resource name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: path // name: keyName // description: Storage bucket key name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: storage bucket // description: Storage bucket key configuration // required: true // schema: // $ref: "#/definitions/StorageBucketKeyPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketKeyPut(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } bucketProjectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } bucketName, err := url.PathUnescape(mux.Vars(r)["bucketName"]) if err != nil { return response.SmartError(err) } keyName, err := url.PathUnescape(mux.Vars(r)["keyName"]) if err != nil { return response.SmartError(err) } // Decode the request. req := api.StorageBucketKeyPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } err = pool.UpdateBucketKey(bucketProjectName, bucketName, keyName, req, nil) if err != nil { return response.SmartError(fmt.Errorf("Failed updating storage bucket key: %w", err)) } s.Events.SendLifecycle(bucketProjectName, lifecycle.StorageBucketKeyUpdated.Event(pool, bucketProjectName, pool.Name(), bucketName, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } func createStoragePoolBucketFromBackup(s *state.State, r *http.Request, requestProjectName string, projectName string, data io.Reader, pool string, bucketName string) response.Response { reverter := revert.New() defer reverter.Fail() // Create temporary file to store uploaded backup data. backupFile, err := os.CreateTemp(internalUtil.VarPath("backups"), fmt.Sprintf("%s_", backup.WorkingDirPrefix)) if err != nil { return response.InternalError(err) } defer func() { _ = os.Remove(backupFile.Name()) }() reverter.Add(func() { _ = backupFile.Close() }) // Get disk budget for the project if any. var budget int64 err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { budget, err = project.GetSpaceBudget(tx, projectName) if err != nil { return err } return nil }) if err != nil { return response.InternalError(err) } // Stream uploaded backup data into temporary file. _, err = util.SafeCopy(internalIO.NewQuotaWriter(backupFile, budget), data) if err != nil { return response.InternalError(err) } // Parse the backup information. _, err = backupFile.Seek(0, io.SeekStart) if err != nil { return response.InternalError(err) } logger.Debug("Reading backup file info") bInfo, err := backup.GetInfo(backupFile, s.OS, backupFile.Name()) if err != nil { return response.BadRequest(err) } bInfo.Project = projectName // Override pool. if pool != "" { bInfo.Pool = pool } // Override bucket name. if bucketName != "" { bInfo.Name = bucketName } logger.Debug("Backup file info loaded", logger.Ctx{ "type": bInfo.Type, "name": bInfo.Name, "project": bInfo.Project, "backend": bInfo.Backend, "pool": bInfo.Pool, }) runRevert := reverter.Clone() run := func(op *operations.Operation) error { defer func() { _ = backupFile.Close() }() defer runRevert.Fail() pool, err := storagePools.LoadByName(s, bInfo.Pool) if err != nil { return err } err = pool.CreateBucketFromBackup(*bInfo, backupFile, nil) if err != nil { return fmt.Errorf("Create storage bucket from backup: %w", err) } runRevert.Success() return nil } resources := map[string][]api.URL{} resources["storage_buckets"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", bInfo.Pool, "buckets", string(bInfo.Type), bInfo.Name)} op, err := operations.OperationCreate(s, requestProjectName, operations.OperationClassTask, operationtype.BucketBackupRestore, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } reverter.Success() return operations.OperationResponse(op) } incus-7.0.0/cmd/incusd/storage_buckets_backup.go000066400000000000000000000663461517523235500217560ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "strings" "time" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/validate" ) var storagePoolBucketBackupsCmd = APIEndpoint{ Path: "storage-pools/{poolName}/buckets/{bucketName}/backups", Get: APIEndpointAction{Handler: storagePoolBucketBackupsGet, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanView, "poolName", "bucketName", "location")}, Post: APIEndpointAction{Handler: storagePoolBucketBackupsPost, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanManageBackups, "poolName", "bucketName", "location")}, } var storagePoolBucketBackupCmd = APIEndpoint{ Path: "storage-pools/{poolName}/buckets/{bucketName}/backups/{backupName}", Get: APIEndpointAction{Handler: storagePoolBucketBackupGet, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanView, "poolName", "bucketName", "location")}, Post: APIEndpointAction{Handler: storagePoolBucketBackupPost, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanManageBackups, "poolName", "bucketName", "location")}, Delete: APIEndpointAction{Handler: storagePoolBucketBackupDelete, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanManageBackups, "poolName", "bucketName", "location")}, } var storagePoolBucketBackupsExportCmd = APIEndpoint{ Path: "storage-pools/{poolName}/buckets/{bucketName}/backups/{backupName}/export", Get: APIEndpointAction{Handler: storagePoolBucketBackupExportGet, AccessHandler: allowPermission(auth.ObjectTypeStorageBucket, auth.EntitlementCanView, "poolName", "bucketName", "location")}, } // swagger:operation GET /1.0/storage-pools/{poolName}/buckets/{bucketName}/backups storage storage_pool_buckets_backups_get // // Get the storage bucket backups // // Returns a list of storage bucket backups (URLs). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/storage-pools/local/buckets/foo/backups/backup0", // "/1.0/storage-pools/local/buckets/foo/backups/backup1" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/storage-pools/{poolName}/buckets/{bucketName}/backups?recursion=1 storage storage_pool_buckets_backups_get_recursion1 // // Get the storage bucket backups // // Returns a list of storage bucket backups (structs). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of storage bucket backups // items: // $ref: "#/definitions/StorageBucketBackup" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketBackupsGet(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } // Get the name of the storage bucket. bucketName, err := url.PathUnescape(mux.Vars(r)["bucketName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage pool the bucket is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the storage pool itself. var poolID int64 err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error poolID, _, _, err = tx.GetStoragePool(ctx, poolName) return err }) if err != nil { return response.SmartError(err) } // Handle the request. recursion := localUtil.IsRecursionRequest(r) var bucketBackups []db.StoragePoolBucketBackup err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { bucketBackups, err = tx.GetStoragePoolBucketBackups(ctx, projectName, bucketName, poolID) return err }) if err != nil { return response.SmartError(err) } backups := make([]*backup.BucketBackup, len(bucketBackups)) for i, b := range bucketBackups { _, backupName, _ := api.GetParentAndSnapshotName(b.Name) backups[i] = backup.NewBucketBackup(s, projectName, poolName, bucketName, b.ID, backupName, b.CreationDate, b.ExpiryDate) } resultString := []string{} resultMap := []*api.StorageBucketBackup{} for _, entry := range backups { if !recursion { resultString = append(resultString, api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "buckets", bucketName, "backups", strings.Split(entry.Name(), "/")[1]).String()) } else { render := entry.Render() resultMap = append(resultMap, render) } } if !recursion { return response.SyncResponse(true, resultString) } return response.SyncResponse(true, resultMap) } // swagger:operation POST /1.0/storage-pools/{poolName}/buckets/{bucketName}/backups storage storage_pool_buckets_backups_post // // Create a storage bucket backup // // Creates a new storage bucket backup. // // If the `Accept` header is set to `application/octet-stream`, this directly streams the backup // tarball to the client without any intermediate operation. // // --- // consumes: // - application/json // produces: // - application/json // - application/octet-stream // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: bucket // description: Storage bucket backup // required: true // schema: // $ref: "#/definitions/StorageBucketBackupsPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketBackupsPost(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } if !pool.Driver().Info().Buckets { return response.BadRequest(errors.New("Storage pool does not support buckets")) } bucketName, err := url.PathUnescape(mux.Vars(r)["bucketName"]) if err != nil { return response.SmartError(err) } targetMember := request.QueryParam(r, "target") memberSpecific := targetMember != "" // Quick checks. err = validate.IsAPIName(bucketName, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid storage bucket backup name: %w", err)) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { err := project.AllowBackupCreation(tx, projectName) return err }) if err != nil { return response.SmartError(err) } var bucket *db.StorageBucket err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { bucket, err = tx.GetStoragePoolBucket(ctx, pool.ID(), projectName, memberSpecific, bucketName) return err }) if err != nil { return response.SmartError(err) } rj := jmap.Map{} err = json.NewDecoder(r.Body).Decode(&rj) if err != nil { return response.InternalError(err) } expiry, _ := rj.GetString("expires_at") if expiry == "" { // Disable expiration by setting it to zero time. rj["expires_at"] = time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC) } body, err := json.Marshal(rj) if err != nil { return response.InternalError(err) } req := api.StorageBucketBackupsPost{} err = json.Unmarshal(body, &req) if err != nil { return response.BadRequest(err) } direct := r.Header.Get("Accept") == "application/octet-stream" var reader *io.PipeReader var writer *io.PipeWriter var fullName string if direct { if req.Name != "" { return response.BadRequest(errors.New("No backup name can be set when requesting a direct backup with Accept: application/octet-stream")) } reader, writer = io.Pipe() } else { if req.Name == "" { var backups []string // come up with a name. err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { backups, err = tx.GetStoragePoolBucketBackupsName(ctx, projectName, bucketName) return err }) if err != nil { return response.BadRequest(err) } base := bucketName + internalInstance.SnapshotDelimiter + "backup" length := len(base) backupID := 0 for _, entry := range backups { // Ignore backups not containing base. if !strings.HasPrefix(entry, base) { continue } substr := entry[length:] var num int count, err := fmt.Sscanf(substr, "%d", &num) if err != nil || count != 1 { continue } if num >= backupID { backupID = num + 1 } } req.Name = fmt.Sprintf("backup%d", backupID) } // Validate the name. if strings.Contains(req.Name, "/") { return response.BadRequest(errors.New("Backup names may not contain slashes")) } fullName = bucketName + internalInstance.SnapshotDelimiter + req.Name } do := func(op *operations.Operation) error { args := db.StoragePoolBucketBackup{ Name: fullName, BucketID: bucket.ID, CreationDate: time.Now(), } if !direct { args.ExpiryDate = req.ExpiresAt } // Create the backup. err := bucketBackupCreate(s, args, projectName, poolName, bucketName, writer) if err != nil { // In order to actually fail piped exports, we use a dirty trick where we close the reader. // This doesn't provide a clean error message in the case of direct backups, but it is a // convenient tradeoff ensuring that the client reports an error. _ = reader.Close() return err } s.Events.SendLifecycle(projectName, lifecycle.StorageBucketBackupCreated.Event(poolName, args.Name, projectName, op.Requestor(), nil)) return nil } resources := map[string][]api.URL{} resources["storage_buckets"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "buckets", bucketName)} if !direct { resources["backups"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "buckets", bucketName, "backups", req.Name)} } op, err := operations.OperationCreate(s, request.ProjectParam(r), operations.OperationClassTask, operationtype.BucketBackupCreate, resources, nil, do, nil, nil, r) if err != nil { return response.InternalError(err) } if direct { err = op.Start() if err != nil { return response.InternalError(err) } return response.PipeResponse(r, reader) } return operations.OperationResponse(op) } // swagger:operation GET /1.0/storage-pools/{poolName}/buckets/{bucketName}/backups/{backupName} storage storage_pool_buckets_backup_get // // Get the storage bucket backup // // Gets a specific storage bucket backup. // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: path // name: backupName // description: Backup name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Storage bucket backup // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/StorageBucketBackup" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketBackupGet(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } if !pool.Driver().Info().Buckets { return response.BadRequest(errors.New("Storage pool does not support buckets")) } bucketName, err := url.PathUnescape(mux.Vars(r)["bucketName"]) if err != nil { return response.SmartError(err) } backupName, err := url.PathUnescape(mux.Vars(r)["backupName"]) if err != nil { return response.SmartError(err) } fullName := bucketName + internalInstance.SnapshotDelimiter + backupName entry, err := storagePoolBucketBackupLoadByName(r.Context(), s, projectName, poolName, fullName) if err != nil { return response.SmartError(err) } return response.SyncResponse(true, entry.Render()) } // swagger:operation POST /1.0/storage-pools/{poolName}/buckets/{bucketName}/backups/{backupName} storage storage_pool_buckets_backup_post // // Rename a storage bucket backup // // Renames a storage bucket backup. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: path // name: backupName // description: Backup name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: bucket rename // description: Storage bucket backup // required: true // schema: // $ref: "#/definitions/StorageBucketBackupPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketBackupPost(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } if !pool.Driver().Info().Buckets { return response.BadRequest(errors.New("Storage pool does not support buckets")) } bucketName, err := url.PathUnescape(mux.Vars(r)["bucketName"]) if err != nil { return response.SmartError(err) } backupName, err := url.PathUnescape(mux.Vars(r)["backupName"]) if err != nil { return response.SmartError(err) } req := api.StorageBucketBackupPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid storage bucket backup name: %w", err)) } oldName := bucketName + internalInstance.SnapshotDelimiter + backupName entry, err := storagePoolBucketBackupLoadByName(r.Context(), s, projectName, poolName, oldName) if err != nil { return response.SmartError(err) } newName := bucketName + internalInstance.SnapshotDelimiter + req.Name rename := func(op *operations.Operation) error { err := entry.Rename(newName) if err != nil { return err } s.Events.SendLifecycle(projectName, lifecycle.StorageBucketBackupRenamed.Event(poolName, newName, projectName, op.Requestor(), nil)) return nil } resources := map[string][]api.URL{} resources["storage_buckets"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "buckets", bucketName)} resources["backups"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "buckets", bucketName, "backups", backupName)} op, err := operations.OperationCreate(s, request.ProjectParam(r), operations.OperationClassTask, operationtype.BucketBackupRemove, resources, nil, rename, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // swagger:operation DELETE /1.0/storage-pools/{poolName}/buckets/{bucketName}/backups/{backupName} storage storage_pool_buckets_backup_delete // // Delete a storage bucket backup // // Deletes a new storage bucket backup. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: path // name: backupName // description: Backup name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketBackupDelete(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } if !pool.Driver().Info().Buckets { return response.BadRequest(errors.New("Storage pool does not support buckets")) } bucketName, err := url.PathUnescape(mux.Vars(r)["bucketName"]) if err != nil { return response.SmartError(err) } backupName, err := url.PathUnescape(mux.Vars(r)["backupName"]) if err != nil { return response.SmartError(err) } fullName := bucketName + internalInstance.SnapshotDelimiter + backupName entry, err := storagePoolBucketBackupLoadByName(r.Context(), s, projectName, poolName, fullName) if err != nil { return response.SmartError(err) } remove := func(op *operations.Operation) error { err := entry.Delete() if err != nil { return err } s.Events.SendLifecycle(projectName, lifecycle.StorageBucketBackupDeleted.Event(poolName, fullName, projectName, op.Requestor(), nil)) return nil } resources := map[string][]api.URL{} resources["storage_buckets"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "buckets", bucketName)} resources["backups"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "buckets", bucketName, "backups", backupName)} op, err := operations.OperationCreate(s, request.ProjectParam(r), operations.OperationClassTask, operationtype.BucketBackupRemove, resources, nil, remove, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // swagger:operation GET /1.0/storage-pools/{poolName}/buckets/{bucketName}/backups/{backupName}/export storage storage_pool_buckets_backup_export_get // // Get the raw backup file // // Download the raw backup file from the server. // // --- // produces: // - application/octet-stream // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: bucketName // description: Storage bucket name // type: string // required: true // - in: path // name: backupName // description: Backup name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Raw backup data // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolBucketBackupExportGet(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } projectName, err := project.StorageBucketProject(r.Context(), s.DB.Cluster, request.ProjectParam(r)) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(fmt.Errorf("Failed loading storage pool: %w", err)) } if !pool.Driver().Info().Buckets { return response.BadRequest(errors.New("Storage pool does not support buckets")) } bucketName, err := url.PathUnescape(mux.Vars(r)["bucketName"]) if err != nil { return response.SmartError(err) } // Get backup name. backupName, err := url.PathUnescape(mux.Vars(r)["backupName"]) if err != nil { return response.SmartError(err) } targetMember := request.QueryParam(r, "target") memberSpecific := targetMember != "" fullName := bucketName + internalInstance.SnapshotDelimiter + backupName // Ensure the bucket exists err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { _, err = tx.GetStoragePoolBucket(ctx, pool.ID(), projectName, memberSpecific, bucketName) return err }) if err != nil { return response.SmartError(err) } ent := response.FileResponseEntry{ Path: internalUtil.VarPath("backups", "buckets", poolName, project.StorageBucket(projectName, fullName)), } s.Events.SendLifecycle(projectName, lifecycle.StorageBucketBackupRetrieved.Event(poolName, fullName, projectName, request.CreateRequestor(r), nil)) return response.FileResponse(r, []response.FileResponseEntry{ent}, nil) } func storagePoolBucketBackupLoadByName(ctx context.Context, s *state.State, projectName, poolName, backupName string) (*backup.BucketBackup, error) { var b db.StoragePoolBucketBackup err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error b, err = tx.GetStoragePoolBucketBackup(ctx, projectName, poolName, backupName) return err }) if err != nil { return nil, err } bucketName, snapName, _ := api.GetParentAndSnapshotName(b.Name) entry := backup.NewBucketBackup(s, projectName, poolName, bucketName, b.ID, snapName, b.CreationDate, b.ExpiryDate) return entry, nil } incus-7.0.0/cmd/incusd/storage_pools.go000066400000000000000000001067161517523235500201210ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "maps" "net/http" "net/url" "slices" "sync" "github.com/gorilla/mux" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/cluster" clusterRequest "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // Lock to prevent concurrent storage pools creation. var storagePoolCreateLock sync.Mutex var storagePoolsCmd = APIEndpoint{ Path: "storage-pools", Get: APIEndpointAction{Handler: storagePoolsGet, AccessHandler: allowAuthenticated}, Post: APIEndpointAction{Handler: storagePoolsPost, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanCreateStoragePools)}, } var storagePoolCmd = APIEndpoint{ Path: "storage-pools/{poolName}", Delete: APIEndpointAction{Handler: storagePoolDelete, AccessHandler: allowPermission(auth.ObjectTypeStoragePool, auth.EntitlementCanEdit, "poolName")}, Get: APIEndpointAction{Handler: storagePoolGet, AccessHandler: allowPermission(auth.ObjectTypeStoragePool, auth.EntitlementCanView, "poolName")}, Patch: APIEndpointAction{Handler: storagePoolPatch, AccessHandler: allowPermission(auth.ObjectTypeStoragePool, auth.EntitlementCanEdit, "poolName")}, Put: APIEndpointAction{Handler: storagePoolPut, AccessHandler: allowPermission(auth.ObjectTypeStoragePool, auth.EntitlementCanEdit, "poolName")}, } // swagger:operation GET /1.0/storage-pools storage storage_pools_get // // Get the storage pools // // Returns a list of storage pools (URLs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/storage-pools/local", // "/1.0/storage-pools/remote" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/storage-pools?recursion=1 storage storage_pools_get_recursion1 // // Get the storage pools // // Returns a list of storage pools (structs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of storage pools // items: // $ref: "#/definitions/StoragePool" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolsGet(d *Daemon, r *http.Request) response.Response { s := d.State() recursion := localUtil.IsRecursionRequest(r) // Parse filter value. filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.BadRequest(fmt.Errorf("Invalid filter: %w", err)) } mustLoadObjects := recursion || (clauses != nil && len(clauses.Clauses) > 0) var poolNames []string var hiddenPoolNames []string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Load the pool names. poolNames, err = tx.GetStoragePoolNames(ctx) if err != nil { return err } // Load the project limits. hiddenPoolNames, err = project.HiddenStoragePools(ctx, tx, request.ProjectParam(r), poolNames) if err != nil { return err } return nil }) if err != nil && !response.IsNotFoundError(err) { return response.SmartError(err) } hasEditPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanEdit, auth.ObjectTypeStoragePool) if err != nil { return response.InternalError(err) } linkResults := make([]string, 0) fullResults := make([]api.StoragePool, 0) for _, poolName := range poolNames { // Hide storage pools with a 0 project limit. if slices.Contains(hiddenPoolNames, poolName) { continue } if mustLoadObjects { pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } // Get all users of the storage pool. poolUsedBy, err := storagePools.UsedBy(r.Context(), s, pool, false, false) if err != nil { return response.SmartError(err) } poolAPI := pool.ToAPI() poolAPI.UsedBy = project.FilterUsedBy(s.Authorizer, r, poolUsedBy) if !hasEditPermission(auth.ObjectStoragePool(poolName)) { // Don't allow non-admins to see pool config as sensitive info can be stored there. poolAPI.Config = nil } // If no member is specified and the daemon is clustered, we omit the node-specific fields. if s.ServerClustered { nodeSpecificConfig := db.NodeSpecificStorageConfig(pool.Driver().Info().Name) for _, key := range nodeSpecificConfig { delete(poolAPI.Config, key) } } else { // Use local status if not clustered. To allow seeing unavailable pools. poolAPI.Status = pool.LocalStatus() } if clauses != nil && len(clauses.Clauses) > 0 { match, err := filter.Match(poolAPI, *clauses) if err != nil { return response.SmartError(err) } if !match { continue } } fullResults = append(fullResults, poolAPI) } linkResults = append(linkResults, fmt.Sprintf("/%s/storage-pools/%s", version.APIVersion, poolName)) } if !recursion { return response.SyncResponse(true, linkResults) } return response.SyncResponse(true, fullResults) } // swagger:operation POST /1.0/storage-pools storage storage_pools_post // // Add a storage pool // // Creates a new storage pool. // When clustered, storage pools require individual POST for each cluster member prior to a global POST. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: storage // description: Storage pool // required: true // schema: // $ref: "#/definitions/StoragePoolsPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolsPost(d *Daemon, r *http.Request) response.Response { s := d.State() storagePoolCreateLock.Lock() defer storagePoolCreateLock.Unlock() req := api.StoragePoolsPost{} // Parse the request. err := json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. if req.Name == "" { return response.BadRequest(errors.New("No name provided")) } err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid storage pool name: %w", err)) } if req.Driver == "" { return response.BadRequest(errors.New("No driver provided")) } if req.Config == nil { req.Config = map[string]string{} } ctx := logger.Ctx{} targetNode := request.QueryParam(r, "target") if targetNode != "" { ctx["target"] = targetNode } lc := lifecycle.StoragePoolCreated.Event(req.Name, request.CreateRequestor(r), ctx) resp := response.SyncResponseLocation(true, nil, lc.Source) clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) if isClusterNotification(r) { // This is an internal request which triggers the actual // creation of the pool across all nodes, after they have been // previously defined. err = storagePoolValidate(s, req.Name, req.Driver, req.Config) if err != nil { return response.BadRequest(err) } var poolID int64 err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error poolID, err = tx.GetStoragePoolID(ctx, req.Name) return err }) if err != nil { return response.SmartError(err) } _, err = storagePoolCreateLocal(r.Context(), s, poolID, req, clientType) if err != nil { return response.SmartError(err) } return resp } if targetNode != "" { // A targetNode was specified, let's just define the node's storage without actually creating it. // The only legal key values for the storage config are the ones in NodeSpecificStorageConfig. nodeSpecificConfig := db.NodeSpecificStorageConfig(req.Driver) for key := range req.Config { if !slices.Contains(nodeSpecificConfig, key) { return response.SmartError(fmt.Errorf("Config key %q may not be used as member-specific key", key)) } } // Make sure that no description is set through the member-specific path. if req.Description != "" { return response.BadRequest(errors.New("The storage pool description cannot be set for a specific member")) } err = storagePoolValidate(s, req.Name, req.Driver, req.Config) if err != nil { return response.BadRequest(err) } exists := false err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { _, err = tx.GetStoragePoolID(ctx, req.Name) if err == nil { exists = true } return tx.CreatePendingStoragePool(ctx, targetNode, req.Name, req.Driver, req.Config) }) if err != nil { if errors.Is(err, db.ErrAlreadyDefined) { return response.BadRequest(fmt.Errorf("The storage pool already defined on member %q", targetNode)) } return response.SmartError(err) } if !exists { // Add the storage pool to the authorizer. err = s.Authorizer.AddStoragePool(r.Context(), req.Name) if err != nil { logger.Error("Failed to add storage pool to authorizer", logger.Ctx{"name": req.Name, "error": err}) } } return resp } var pool *api.StoragePool err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Load existing pool if exists, if not don't fail. _, pool, _, err = tx.GetStoragePoolInAnyState(ctx, req.Name) return err }) if err != nil && !response.IsNotFoundError(err) { return response.InternalError(err) } // Check if we're clustered. count, err := cluster.Count(s) if err != nil { return response.SmartError(err) } // No targetNode was specified and we're clustered or there is an existing partially created single node // pool, either way finalize the config in the db and actually create the pool on all nodes in the cluster. if count > 1 || (pool != nil && pool.Status != api.StoragePoolStatusCreated) { err = storagePoolsPostCluster(r.Context(), s, pool, req, clientType) if err != nil { return response.InternalError(err) } // Send out the lifecycle event. s.Events.SendLifecycle(api.ProjectDefaultName, lc) } else { // Create new single node storage pool. err = storagePoolCreateGlobal(r.Context(), s, req, clientType) if err != nil { return response.SmartError(err) } // Add the storage pool to the authorizer. err = s.Authorizer.AddStoragePool(r.Context(), req.Name) if err != nil { logger.Error("Failed to add storage pool to authorizer", logger.Ctx{"name": req.Name, "error": err}) } // Send out the lifecycle event. s.Events.SendLifecycle(api.ProjectDefaultName, lc) } return resp } // storagePoolPartiallyCreated returns true of supplied storage pool has properties that indicate it has had // previous create attempts run on it but failed on one or more nodes. func storagePoolPartiallyCreated(pool *api.StoragePool) bool { // If the pool status is StoragePoolStatusErrored, this means create has been run in the past and has // failed on one or more nodes. Hence it is partially created. if pool.Status == api.StoragePoolStatusErrored { return true } // If the pool has global config keys, then it has previously been created by having its global config // inserted, and this means it is partialled created. nodeSpecificConfig := db.NodeSpecificStorageConfig(pool.Driver) for key := range pool.Config { if !slices.Contains(nodeSpecificConfig, key) { return true } } return false } // storagePoolsPostCluster handles creating storage pools after the per-node config records have been created. // Accepts an optional existing pool record, which will exist when performing subsequent re-create attempts. func storagePoolsPostCluster(ctx context.Context, s *state.State, pool *api.StoragePool, req api.StoragePoolsPost, clientType clusterRequest.ClientType) error { // Check that no node-specific config key has been defined. nodeSpecificConfig := db.NodeSpecificStorageConfig(req.Driver) creationDisallowedConfig := storagePools.DisallowedStorageConfigForCreation(req.Driver) for key := range req.Config { if slices.Contains(nodeSpecificConfig, key) { return fmt.Errorf("Config key %q is cluster member specific", key) } if slices.Contains(creationDisallowedConfig, key) { return fmt.Errorf("Config key %q is not allowed during creation", key) } } // If pool already exists, perform quick checks. if pool != nil { // Check pool isn't already created. if pool.Status == api.StoragePoolStatusCreated { return errors.New("The storage pool is already created") } // Check the requested pool type matches the type created when adding the local member config. if req.Driver != pool.Driver { return fmt.Errorf("Requested storage pool driver %q doesn't match driver in existing database record %q", req.Driver, pool.Driver) } } // Check that the pool is properly defined, fetch the node-specific configs and insert the global config. var configs map[string]map[string]string var poolID int64 err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error // Check that the pool was defined at all. Must come before partially created checks. poolID, err = tx.GetStoragePoolID(ctx, req.Name) if err != nil { return err } // Check if any global config exists already, if so we should not create global config again. if pool != nil && storagePoolPartiallyCreated(pool) { if len(req.Config) > 0 { return errors.New("Storage pool already partially created. Please do not specify any global config when re-running create") } logger.Debug("Skipping global storage pool create as global config already partially created", logger.Ctx{"pool": req.Name}) return nil } // Fetch the node-specific configs and check the pool is defined for all nodes. configs, err = tx.GetStoragePoolNodeConfigs(ctx, poolID) if err != nil { return err } // Insert the global config keys. err = tx.CreateStoragePoolConfig(poolID, 0, req.Config) if err != nil { return err } // Assume failure unless we succeed later on. return tx.StoragePoolErrored(req.Name) }) if err != nil { if response.IsNotFoundError(err) { return errors.New("Pool not pending on any node (use --target first)") } return err } // Create notifier for other nodes to create the storage pool. notifier, err := cluster.NewNotifier(s, s.Endpoints.NetworkCert(), s.ServerCert(), cluster.NotifyAll) if err != nil { return err } // Create the pool on this node. nodeReq := req // Merge node specific config items into global config. maps.Copy(nodeReq.Config, configs[s.ServerName]) updatedConfig, err := storagePoolCreateLocal(ctx, s, poolID, req, clientType) if err != nil { return err } // Clone the config so that updatedConfig retains the node-specific key for later use. req.Config = util.CloneMap(updatedConfig) logger.Debug("Created storage pool on local cluster member", logger.Ctx{"pool": req.Name}) // Strip node specific config keys from config. Very important so we don't forward node-specific config. nodeSpecificConfig = db.NodeSpecificStorageConfig(req.Driver) for _, k := range nodeSpecificConfig { delete(req.Config, k) } // Notify all other nodes to create the pool. err = notifier(func(client incus.InstanceServer) error { server, _, err := client.GetServer() if err != nil { return err } nodeReq := req // Clone fresh node config so we don't modify req.Config with this node's specific config which // could result in it being sent to other nodes later. nodeReq.Config = util.CloneMap(req.Config) // Merge node specific config items into global config. maps.Copy(nodeReq.Config, configs[server.Environment.ServerName]) err = client.CreateStoragePool(nodeReq) if err != nil { return err } logger.Debug("Created storage pool on cluster member", logger.Ctx{"pool": req.Name, "member": server.Environment.ServerName}) return nil }) if err != nil { return err } // Get the existing storage pool. p, err := storagePools.LoadByName(s, pool.Name) if err != nil { return err } if p.Driver().Info().SameSource { err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { nodes, err := tx.GetNodes(ctx) if err != nil { return err } for _, node := range nodes { if node.Name == s.ServerName { continue } err := tx.UpdateStoragePoolConfig(poolID, node.ID, updatedConfig) if err != nil { return err } } return nil }) if err != nil { return err } } // Finally update the storage pool state. err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.StoragePoolCreated(req.Name) }) if err != nil { return err } logger.Debug("Marked storage pool global status as created", logger.Ctx{"pool": req.Name}) return nil } // swagger:operation GET /1.0/storage-pools/{poolName} storage storage_pool_get // // Get the storage pool // // Gets a specific storage pool. // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Storage pool // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/StoragePool" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolGet(d *Daemon, r *http.Request) response.Response { s := d.State() // If a target was specified, forward the request to the relevant node. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } memberSpecific := false if request.QueryParam(r, "target") != "" { memberSpecific = true } var hiddenPoolNames []string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Load the project limits. hiddenPoolNames, err = project.HiddenStoragePools(ctx, tx, request.ProjectParam(r), []string{poolName}) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // Hide storage pools with a 0 project limit. if slices.Contains(hiddenPoolNames, poolName) { return response.NotFound(nil) } // Get the existing storage pool. pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } // Get all users of the storage pool. poolUsedBy, err := storagePools.UsedBy(r.Context(), s, pool, false, memberSpecific) if err != nil { return response.SmartError(err) } poolAPI := pool.ToAPI() poolAPI.UsedBy = project.FilterUsedBy(s.Authorizer, r, poolUsedBy) err = s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectStoragePool(poolName), auth.EntitlementCanEdit) if err != nil && api.StatusErrorCheck(err, http.StatusForbidden) { // Don't allow non-admins to see pool config as sensitive info can be stored there. poolAPI.Config = nil } else if err != nil { return response.SmartError(err) } // If no member is specified and the daemon is clustered, we omit the node-specific fields. if s.ServerClustered && !memberSpecific { nodeSpecificConfig := db.NodeSpecificStorageConfig(pool.Driver().Info().Name) for _, key := range nodeSpecificConfig { delete(poolAPI.Config, key) } } else { // Use local status if not clustered or memberSpecific. To allow seeing unavailable pools. poolAPI.Status = pool.LocalStatus() } etag := []any{pool.Name(), pool.Driver().Info().Name, pool.Description(), poolAPI.Config} return response.SyncResponseETag(true, &poolAPI, etag) } // swagger:operation PUT /1.0/storage-pools/{poolName} storage storage_pool_put // // Update the storage pool // // Updates the entire storage pool configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: storage pool // description: Storage pool configuration // required: true // schema: // $ref: "#/definitions/StoragePoolPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func storagePoolPut(d *Daemon, r *http.Request) response.Response { s := d.State() // If a target was specified, forward the request to the relevant node. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the existing storage pool. pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } targetNode := request.QueryParam(r, "target") if targetNode == "" && pool.Status() != api.StoragePoolStatusCreated { return response.BadRequest(errors.New("Cannot update storage pool global config when not in created state")) } // Duplicate config for etag modification and generation. etagConfig := localUtil.CopyConfig(pool.Driver().Config()) nodeSpecificConfig := db.NodeSpecificStorageConfig(pool.Driver().Info().Name) // If no target node is specified and the daemon is clustered, we omit the node-specific fields so that // the e-tag can be generated correctly. This is because the GET request used to populate the request // will also remove node-specific keys when no target is specified. if targetNode == "" && s.ServerClustered { for _, key := range nodeSpecificConfig { delete(etagConfig, key) } } // Validate the ETag. etag := []any{pool.Name(), pool.Driver().Info().Name, pool.Description(), etagConfig} err = localUtil.EtagCheck(r, etag) if err != nil { return response.PreconditionFailed(err) } // Decode the request. req := api.StoragePoolPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // In clustered mode, we differentiate between node specific and non-node specific config keys based on // whether the user has specified a target to apply the config to. if s.ServerClustered { if targetNode == "" { // If no target is specified, then ensure only non-node-specific config keys are changed. for k := range req.Config { if slices.Contains(nodeSpecificConfig, k) { return response.BadRequest(fmt.Errorf("Config key %q is cluster member specific", k)) } } } else { curConfig := pool.Driver().Config() // If a target is specified, then ensure only node-specific config keys are changed. for k, v := range req.Config { if !slices.Contains(nodeSpecificConfig, k) && curConfig[k] != v { return response.BadRequest(fmt.Errorf("Config key %q may not be used as cluster member specific key", k)) } } } } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) response := doStoragePoolUpdate(s, pool, req, targetNode, clientType, r.Method, s.ServerClustered) requestor := request.CreateRequestor(r) ctx := logger.Ctx{} if targetNode != "" { ctx["target"] = targetNode } // Send a single update event when the server is clustered. if !s.ServerClustered || (s.ServerClustered && clientType == clusterRequest.ClientTypeNormal) { s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.StoragePoolUpdated.Event(pool.Name(), requestor, ctx)) } return response } // swagger:operation PATCH /1.0/storage-pools/{poolName} storage storage_pool_patch // // Partially update the storage pool // // Updates a subset of the storage pool configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: storage pool // description: Storage pool configuration // required: true // schema: // $ref: "#/definitions/StoragePoolPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func storagePoolPatch(d *Daemon, r *http.Request) response.Response { return storagePoolPut(d, r) } // doStoragePoolUpdate takes the current local storage pool config, merges with the requested storage pool config, // validates and applies the changes. Will also notify other cluster nodes of non-node specific config if needed. func doStoragePoolUpdate(s *state.State, pool storagePools.Pool, req api.StoragePoolPut, targetNode string, clientType clusterRequest.ClientType, httpMethod string, clustered bool) response.Response { if req.Config == nil { req.Config = map[string]string{} } // Normally a "put" request will replace all existing config, however when clustered, we need to account // for the node specific config keys and not replace them when the request doesn't specify a specific node. if targetNode == "" && httpMethod != http.MethodPatch && clustered { // If non-node specific config being updated via "put" method in cluster, then merge the current // node-specific network config with the submitted config to allow validation. // This allows removal of non-node specific keys when they are absent from request config. nodeSpecificConfig := db.NodeSpecificStorageConfig(pool.Driver().Info().Name) for k, v := range pool.Driver().Config() { if slices.Contains(nodeSpecificConfig, k) { req.Config[k] = v } } } else if httpMethod == http.MethodPatch { // If config being updated via "patch" method, then merge all existing config with the keys that // are present in the request config. for k, v := range pool.Driver().Config() { _, ok := req.Config[k] if !ok { req.Config[k] = v } } } // Validate the configuration. err := pool.Validate(req.Config) if err != nil { return response.BadRequest(err) } // Notify the other nodes, unless this is itself a notification. if clustered && clientType != clusterRequest.ClientTypeNotifier && targetNode == "" { notifier, err := cluster.NewNotifier(s, s.Endpoints.NetworkCert(), s.ServerCert(), cluster.NotifyAll) if err != nil { return response.SmartError(err) } sendPool := req sendPool.Config = make(map[string]string) driverName := pool.Driver().Info().Name nodeSpecificConfig := db.NodeSpecificStorageConfig(driverName) clusterWideConfig := storagePools.ClusterWideStorageConfig(driverName) for k, v := range req.Config { // Don't forward node specific keys (these will be merged in on recipient node). // Don't forward cluster wide keys as performing operation on one node is enough. if slices.Contains(nodeSpecificConfig, k) || slices.Contains(clusterWideConfig, k) { continue } sendPool.Config[k] = v } err = notifier(func(client incus.InstanceServer) error { return client.UpdateStoragePool(pool.Name(), sendPool, "") }) if err != nil { return response.SmartError(err) } } err = pool.Update(clientType, req.Description, req.Config, nil) if err != nil { return response.InternalError(err) } return response.EmptySyncResponse } // swagger:operation DELETE /1.0/storage-pools/{poolName} storage storage_pools_delete // // Delete the storage pool // // Removes the storage pool. // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolDelete(d *Daemon, r *http.Request) response.Response { s := d.State() poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } clientType := clusterRequest.UserAgentClientType(r.Header.Get("User-Agent")) clusterNotification := isClusterNotification(r) var notifier cluster.Notifier if !clusterNotification { // Quick checks. inUse, err := pool.IsUsed() if err != nil { return response.SmartError(err) } if inUse { return response.BadRequest(errors.New("The storage pool is currently in use")) } // Get the cluster notifier notifier, err = cluster.NewNotifier(s, s.Endpoints.NetworkCert(), s.ServerCert(), cluster.NotifyAll) if err != nil { return response.SmartError(err) } } // Only perform the deletion of remote image volumes on the server handling the request. // Otherwise delete local image volumes on each server. if !clusterNotification || !pool.Driver().Info().Remote { var removeImgFingerprints []string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Get all the volumes using the storage pool on this server. // Only image volumes should remain now. volumes, err := tx.GetStoragePoolVolumes(ctx, pool.ID(), true) if err != nil { return fmt.Errorf("Failed loading storage volumes: %w", err) } for _, vol := range volumes { if vol.Type != db.StoragePoolVolumeTypeNameImage { return fmt.Errorf("Volume %q of type %q in project %q still exists in storage pool %q", vol.Name, vol.Type, vol.Project, pool.Name()) } removeImgFingerprints = append(removeImgFingerprints, vol.Name) } return nil }) if err != nil { return response.SmartError(err) } for _, removeImgFingerprint := range removeImgFingerprints { err = pool.DeleteImage(removeImgFingerprint, nil) if err != nil { return response.InternalError(fmt.Errorf("Error deleting image %q from storage pool %q: %w", removeImgFingerprint, pool.Name(), err)) } } } // If the pool requires deactivation, go through it first. if !clusterNotification && pool.Driver().Info().Remote && pool.Driver().Info().Deactivate { err = notifier(func(client incus.InstanceServer) error { _, _, err := client.GetServer() if err != nil { return err } return client.DeleteStoragePool(pool.Name()) }) if err != nil { return response.SmartError(err) } } if pool.LocalStatus() != api.StoragePoolStatusPending { err = pool.Delete(clientType, nil) if err != nil { return response.InternalError(err) } } // If this is a cluster notification, we're done, any database work will be done by the node that is // originally serving the request. if clusterNotification { return response.EmptySyncResponse } // If clustered and dealing with a normal pool, notify all other nodes. if !pool.Driver().Info().Remote || !pool.Driver().Info().Deactivate { err = notifier(func(client incus.InstanceServer) error { _, _, err := client.GetServer() if err != nil { return err } return client.DeleteStoragePool(pool.Name()) }) } if err != nil { return response.SmartError(err) } err = dbStoragePoolDeleteAndUpdateCache(r.Context(), s, pool.Name()) if err != nil { return response.SmartError(err) } // Remove the storage pool from the authorizer. err = s.Authorizer.DeleteStoragePool(r.Context(), pool.Name()) if err != nil { logger.Error("Failed to remove storage pool from authorizer", logger.Ctx{"name": pool.Name(), "error": err}) } requestor := request.CreateRequestor(r) s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.StoragePoolDeleted.Event(pool.Name(), requestor, nil)) return response.EmptySyncResponse } incus-7.0.0/cmd/incusd/storage_pools_utils.go000066400000000000000000000135321517523235500213320ustar00rootroot00000000000000package main import ( "context" "fmt" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/validate" ) // storagePoolDBCreate creates a storage pool DB entry and returns the created Pool ID. func storagePoolDBCreate(ctx context.Context, s *state.State, poolName string, poolDescription string, driver string, config map[string]string) (int64, error) { err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Check that the storage pool does not already exist. _, err := tx.GetStoragePoolID(ctx, poolName) return err }) if err == nil { return -1, fmt.Errorf("The storage pool already exists: %w", db.ErrAlreadyDefined) } // Make sure that we don't pass a nil to the next function. if config == nil { config = map[string]string{} } err = storagePoolValidate(s, poolName, driver, config) if err != nil { return -1, err } // Create the database entry for the storage pool. id, err := dbStoragePoolCreateAndUpdateCache(ctx, s, poolName, poolDescription, driver, config) if err != nil { return -1, fmt.Errorf("Error inserting %s into database: %w", poolName, err) } return id, nil } func storagePoolValidate(s *state.State, poolName string, driverName string, config map[string]string) error { poolType, err := storagePools.LoadByType(s, driverName) if err != nil { return err } // Check if the storage pool name is valid. err = validate.IsAPIName(poolName, false) if err != nil { return err } // Validate the requested storage pool configuration. err = poolType.Validate(config) if err != nil { return err } return nil } func storagePoolCreateGlobal(ctx context.Context, s *state.State, req api.StoragePoolsPost, clientType request.ClientType) error { // Create the database entry. id, err := storagePoolDBCreate(ctx, s, req.Name, req.Description, req.Driver, req.Config) if err != nil { return err } // Define a function which reverts everything. Defer this function // so that it doesn't need to be explicitly called in every failing // return path. Track whether or not we want to undo the changes // using a closure. reverter := revert.New() defer reverter.Fail() reverter.Add(func() { _ = dbStoragePoolDeleteAndUpdateCache(context.Background(), s, req.Name) }) _, err = storagePoolCreateLocal(ctx, s, id, req, clientType) if err != nil { return err } reverter.Success() return nil } // This performs local pool setup and updates DB record if config was changed during pool setup. // Returns resulting config. func storagePoolCreateLocal(ctx context.Context, s *state.State, poolID int64, req api.StoragePoolsPost, clientType request.ClientType) (map[string]string, error) { // Setup revert. reverter := revert.New() defer reverter.Fail() // Load pool record. pool, err := storagePools.LoadByName(s, req.Name) if err != nil { return nil, err } if pool.LocalStatus() == api.NetworkStatusCreated { logger.Debug("Skipping local storage pool create as already created", logger.Ctx{"pool": pool.Name()}) return pool.Driver().Config(), nil } // Create the pool. err = pool.Create(clientType, nil) if err != nil { return nil, err } reverter.Add(func() { _ = pool.Delete(clientType, nil) }) // Mount the pool. _, err = pool.Mount() if err != nil { return nil, err } // In case the storage pool config was changed during the pool creation, we need to update the database to // reflect this change. This can e.g. happen, when we create a loop file image. This means we append ".img" // to the path the user gave us and update the config in the storage callback. So diff the config here to // see if something like this has happened. configDiff, _ := storagePools.ConfigDiff(req.Config, pool.Driver().Config()) if len(configDiff) > 0 { err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Update the database entry for the storage pool. return tx.UpdateStoragePool(ctx, req.Name, req.Description, pool.Driver().Config()) }) if err != nil { return nil, fmt.Errorf("Error updating storage pool config after local create for %q: %w", req.Name, err) } } // Set storage pool node to storagePoolCreated. err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.StoragePoolNodeCreated(poolID) }) if err != nil { return nil, err } logger.Debug("Marked storage pool local status as created", logger.Ctx{"pool": req.Name}) reverter.Success() return pool.Driver().Config(), nil } // Helper around the low-level DB API, which also updates the driver names cache. func dbStoragePoolCreateAndUpdateCache(ctx context.Context, s *state.State, poolName string, poolDescription string, poolDriver string, poolConfig map[string]string) (int64, error) { var id int64 err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error id, err = tx.CreateStoragePool(ctx, poolName, poolDescription, poolDriver, poolConfig) return err }) if err != nil { return id, err } // Update the storage drivers cache in api_1.0.go. storagePoolDriversCacheUpdate(ctx, s) return id, nil } // Helper around the low-level DB API, which also updates the driver names // cache. func dbStoragePoolDeleteAndUpdateCache(ctx context.Context, s *state.State, poolName string) error { err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { _, err := tx.RemoveStoragePool(ctx, poolName) return err }) if err != nil { return err } // Update the storage drivers cache in api_1.0.go. storagePoolDriversCacheUpdate(ctx, s) return err } incus-7.0.0/cmd/incusd/storage_volumes.go000066400000000000000000002554101517523235500204530ustar00rootroot00000000000000package main import ( "bytes" "context" "crypto/x509" "encoding/json" "encoding/pem" "errors" "fmt" "io" "net/http" "net/url" "os" "slices" "sort" "strconv" "strings" "time" "github.com/gorilla/mux" "github.com/gorilla/websocket" "github.com/lxc/incus/v7/internal/filter" internalInstance "github.com/lxc/incus/v7/internal/instance" internalIO "github.com/lxc/incus/v7/internal/io" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var storagePoolVolumesCmd = APIEndpoint{ Path: "storage-pools/{poolName}/volumes", Get: APIEndpointAction{Handler: storagePoolVolumesGet, AccessHandler: allowAuthenticated}, Post: APIEndpointAction{Handler: storagePoolVolumesPost, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanCreateStorageVolumes), LargeRequest: true}, } var storagePoolVolumesTypeCmd = APIEndpoint{ Path: "storage-pools/{poolName}/volumes/{type}", Get: APIEndpointAction{Handler: storagePoolVolumesGet, AccessHandler: allowAuthenticated}, Post: APIEndpointAction{Handler: storagePoolVolumesPost, AccessHandler: allowPermission(auth.ObjectTypeProject, auth.EntitlementCanCreateStorageVolumes), LargeRequest: true}, } var storagePoolVolumeTypeCmd = APIEndpoint{ Path: "storage-pools/{poolName}/volumes/{type}/{volumeName}", Delete: APIEndpointAction{Handler: storagePoolVolumeDelete, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanEdit, "poolName", "type", "volumeName", "location")}, Get: APIEndpointAction{Handler: storagePoolVolumeGet, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanView, "poolName", "type", "volumeName", "location")}, Patch: APIEndpointAction{Handler: storagePoolVolumePatch, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanEdit, "poolName", "type", "volumeName", "location")}, Post: APIEndpointAction{Handler: storagePoolVolumePost, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanEdit, "poolName", "type", "volumeName", "location")}, Put: APIEndpointAction{Handler: storagePoolVolumePut, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanEdit, "poolName", "type", "volumeName", "location")}, } var storagePoolVolumeTypeNBDCmd = APIEndpoint{ Path: "storage-pools/{poolName}/volumes/{type}/{volumeName}/nbd", Get: APIEndpointAction{Handler: storagePoolVolumeTypeNBDHandler, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanConnectNBD, "poolName", "type", "volumeName", "location")}, } var storagePoolVolumeTypeSFTPCmd = APIEndpoint{ Path: "storage-pools/{poolName}/volumes/{type}/{volumeName}/sftp", Get: APIEndpointAction{Handler: storagePoolVolumeTypeSFTPHandler, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanConnectSFTP, "poolName", "type", "volumeName", "location")}, } var storagePoolVolumeTypeFileCmd = APIEndpoint{ Path: "storage-pools/{poolName}/volumes/{type}/{volumeName}/files", Delete: APIEndpointAction{Handler: storagePoolVolumeTypeFileHandler, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanAccessFiles, "poolName", "type", "volumeName", "location")}, Get: APIEndpointAction{Handler: storagePoolVolumeTypeFileHandler, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanAccessFiles, "poolName", "type", "volumeName", "location")}, Head: APIEndpointAction{Handler: storagePoolVolumeTypeFileHandler, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanAccessFiles, "poolName", "type", "volumeName", "location")}, Post: APIEndpointAction{Handler: storagePoolVolumeTypeFileHandler, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanAccessFiles, "poolName", "type", "volumeName", "location"), LargeRequest: true}, } // swagger:operation GET /1.0/storage-pools/{poolName}/volumes storage storage_pool_volumes_get // // Get the storage volumes // // Returns a list of storage volumes (URLs). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/storage-pools/local/volumes/container/a1", // "/1.0/storage-pools/local/volumes/container/a2", // "/1.0/storage-pools/local/volumes/custom/backups", // "/1.0/storage-pools/local/volumes/custom/images" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/storage-pools/{poolName}/volumes?recursion=1 storage storage_pool_volumes_get_recursion1 // // Get the storage volumes // // Returns a list of storage volumes (structs). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: query // name: filter // description: Collection filter // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of storage volumes // items: // $ref: "#/definitions/StorageVolume" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type} storage storage_pool_volumes_type_get // // Get the storage volumes // // Returns a list of storage volumes (URLs) (type specific endpoint). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/storage-pools/local/volumes/custom/backups", // "/1.0/storage-pools/local/volumes/custom/images" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}?recursion=1 storage storage_pool_volumes_type_get_recursion1 // // Get the storage volumes // // Returns a list of storage volumes (structs) (type specific endpoint). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of storage volumes // items: // $ref: "#/definitions/StorageVolume" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}?recursion=2 storage storage_pool_volumes_type_get_recursion2 // // Get the storage volumes with all details // // Returns a list of storage volumes (structs) including all details (type specific endpoint). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of storage volumes // items: // $ref: "#/definitions/StorageVolumeFull" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumesGet(d *Daemon, r *http.Request) response.Response { s := d.State() resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } targetMember := request.QueryParam(r, "target") memberSpecific := targetMember != "" poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the name of the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Convert volume type name to internal integer representation if requested. var volumeType int if volumeTypeName != "" { volumeType, err = storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } } filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.SmartError(fmt.Errorf("Invalid filter: %w", err)) } // Retrieve the storage pool (and check if the storage pool exists). pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } // Detect project mode. requestProjectName := request.QueryParam(r, "project") allProjects := util.IsTrue(request.QueryParam(r, "all-projects")) if allProjects && requestProjectName != "" { return response.SmartError(api.StatusErrorf(http.StatusBadRequest, "Cannot specify a project when requesting all projects")) } else if !allProjects && requestProjectName == "" { requestProjectName = api.ProjectDefaultName } var dbVolumes []*db.StorageVolume var projectImages []string err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var customVolProjectName string if !allProjects { dbProject, err := dbCluster.GetProject(ctx, tx.Tx(), requestProjectName) if err != nil { return err } p, err := dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return err } // The project name used for custom volumes varies based on whether the // project has the features.storage.volumes feature enabled. customVolProjectName = project.StorageVolumeProjectFromRecord(p, db.StoragePoolVolumeTypeCustom) projectImages, err = tx.GetImagesFingerprints(ctx, requestProjectName, false) if err != nil { return err } } filters := make([]db.StorageVolumeFilter, 0) for i := range supportedVolumeTypes { supportedVolType := supportedVolumeTypes[i] // Local variable for use as pointer below. if volumeTypeName != "" && supportedVolType != volumeType { continue // Only include the requested type if specified. } switch supportedVolType { case db.StoragePoolVolumeTypeCustom: volTypeCustom := db.StoragePoolVolumeTypeCustom filter := db.StorageVolumeFilter{ Type: &volTypeCustom, } if !allProjects { filter.Project = &customVolProjectName } filters = append(filters, filter) case db.StoragePoolVolumeTypeImage: // Image volumes are effectively a cache and are always linked to default project. // We filter the ones relevant to requested project below after the query has run. volTypeImage := db.StoragePoolVolumeTypeImage filters = append(filters, db.StorageVolumeFilter{ Type: &volTypeImage, }) default: // Include instance volume types using the specified project. filter := db.StorageVolumeFilter{ Type: &supportedVolType, } if !allProjects { filter.Project = &requestProjectName } filters = append(filters, filter) } } dbVolumes, err = tx.GetStoragePoolVolumes(ctx, pool.ID(), memberSpecific, filters...) if err != nil { return fmt.Errorf("Failed loading storage volumes: %w", err) } return err }) if err != nil { return response.SmartError(err) } // Pre-fill UsedBy if using filtering. if clauses != nil && len(clauses.Clauses) > 0 { for i, vol := range dbVolumes { volumeUsedBy, err := storagePoolVolumeUsedByGet(s, requestProjectName, poolName, vol) if err != nil { return response.InternalError(err) } dbVolumes[i].UsedBy = project.FilterUsedBy(s.Authorizer, r, volumeUsedBy) } } // Filter the results. dbVolumes, err = filterVolumes(dbVolumes, clauses, allProjects, projectImages) if err != nil { return response.SmartError(err) } // Sort by type then volume name. sort.SliceStable(dbVolumes, func(i, j int) bool { volA := dbVolumes[i] volB := dbVolumes[j] if volA.Type != volB.Type { return dbVolumes[i].Type < dbVolumes[j].Type } return volA.Name < volB.Name }) userHasPermission, err := s.Authorizer.GetPermissionChecker(r.Context(), r, auth.EntitlementCanView, auth.ObjectTypeStorageVolume) if err != nil { return response.SmartError(err) } recursionStr := r.FormValue("recursion") recursion, err := strconv.Atoi(recursionStr) if err != nil { recursion = 0 } if recursion > 0 { volumes := make([]*api.StorageVolume, 0, len(dbVolumes)) for _, dbVol := range dbVolumes { vol := &dbVol.StorageVolume var location string if s.ServerClustered && !pool.Driver().Info().Remote { location = vol.Location } volumeName, snapName, _ := api.GetParentAndSnapshotName(vol.Name) if snapName != "" { continue } if !userHasPermission(auth.ObjectStorageVolume(vol.Project, poolName, dbVol.Type, volumeName, location)) { continue } // Fill in UsedBy if we haven't previously done so. if clauses == nil || len(clauses.Clauses) == 0 { volumeUsedBy, err := storagePoolVolumeUsedByGet(s, requestProjectName, poolName, dbVol) if err != nil { return response.InternalError(err) } vol.UsedBy = project.FilterUsedBy(s.Authorizer, r, volumeUsedBy) } volumes = append(volumes, vol) } if recursion == 2 { volumesFull := make([]*api.StorageVolumeFull, 0, len(volumes)) for _, vol := range volumes { if s.ServerClustered && !pool.Driver().Info().Remote && vol.Location != "" && vol.Location != s.ServerName { // Get the remote address. var volNode db.NodeInfo err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { volNode, err = tx.GetNodeByName(ctx, vol.Location) return err }) if err != nil { return response.InternalError(fmt.Errorf("Failed getting cluster member info for %q: %w", vol.Location, err)) } client, err := cluster.Connect(volNode.Address, s.Endpoints.NetworkCert(), s.ServerCert(), r, false) if err != nil { return response.InternalError(err) } fullVol, _, err := client.UseTarget(vol.Location).UseProject(vol.Project).GetStoragePoolVolumeFull(poolName, vol.Type, vol.Name) if err != nil { return response.InternalError(err) } volumesFull = append(volumesFull, fullVol) continue } // Handle local volumes. fullVol, err := getVolumeFull(r.Context(), s, poolName, *vol) if err != nil { return response.InternalError(err) } volumesFull = append(volumesFull, fullVol) } return response.SyncResponse(true, volumesFull) } return response.SyncResponse(true, volumes) } urls := make([]string, 0, len(dbVolumes)) for _, dbVol := range dbVolumes { volumeName, snapName, _ := api.GetParentAndSnapshotName(dbVol.Name) if snapName != "" { continue } var location string if s.ServerClustered && !pool.Driver().Info().Remote { location = dbVol.Location } if !userHasPermission(auth.ObjectStorageVolume(dbVol.Project, poolName, dbVol.Type, volumeName, location)) { continue } urls = append(urls, dbVol.StorageVolume.URL(version.APIVersion, poolName).String()) } return response.SyncResponse(true, urls) } // filterVolumes returns a filtered list of volumes that match the given clauses. func filterVolumes(volumes []*db.StorageVolume, clauses *filter.ClauseSet, allProjects bool, filterProjectImages []string) ([]*db.StorageVolume, error) { // FilterStorageVolume is for filtering purpose only. // It allows to filter snapshots by using default filter mechanism. type FilterStorageVolume struct { api.StorageVolume `yaml:",inline"` Snapshot string `yaml:"snapshot"` } filtered := []*db.StorageVolume{} for _, volume := range volumes { // Filter out image volumes that are not used by this project. if volume.Type == db.StoragePoolVolumeTypeNameImage && !allProjects && !slices.Contains(filterProjectImages, volume.Name) { continue } tmpVolume := FilterStorageVolume{ StorageVolume: volume.StorageVolume, Snapshot: strconv.FormatBool(strings.Contains(volume.Name, internalInstance.SnapshotDelimiter)), } match, err := filter.Match(tmpVolume, *clauses) if err != nil { return nil, err } if !match { continue } filtered = append(filtered, volume) } return filtered, nil } // swagger:operation POST /1.0/storage-pools/{poolName}/volumes storage storage_pool_volumes_post // // Add a storage volume // // Creates a new storage volume. // Will return an empty sync response on simple volume creation but an operation on copy or migration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: volume // description: Storage volume // required: true // schema: // $ref: "#/definitions/StorageVolumesPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation POST /1.0/storage-pools/{poolName}/volumes/{type} storage storage_pool_volumes_type_post // // Add a storage volume // // Creates a new storage volume (type specific endpoint). // Will return an empty sync response on simple volume creation but an operation on copy or migration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: volume // description: Storage volume // required: true // schema: // $ref: "#/definitions/StorageVolumesPost" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumesPost(d *Daemon, r *http.Request) response.Response { s := d.State() poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), db.StoragePoolVolumeTypeCustom) if err != nil { return response.SmartError(err) } resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } // If we're getting binary content, process separately. if r.Header.Get("Content-Type") == "application/octet-stream" { if r.Header.Get("X-Incus-type") == "iso" { return createStoragePoolVolumeFromISO(s, r, request.ProjectParam(r), projectName, r.Body, poolName, r.Header.Get("X-Incus-name")) } return createStoragePoolVolumeFromBackup(s, r, request.ProjectParam(r), projectName, r.Body, poolName, r.Header.Get("X-Incus-name")) } req := api.StorageVolumesPost{} // Parse the request. err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid storage volume name: %w", err)) } // Backward compatibility. if req.ContentType == "" { req.ContentType = db.StoragePoolVolumeContentTypeNameFS } _, err = storagePools.VolumeContentTypeNameToContentType(req.ContentType) if err != nil { return response.BadRequest(err) } // Handle being called through the typed URL. _, ok := mux.Vars(r)["type"] if ok { req.Type, err = url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } } // We currently only allow to create storage volumes of type storagePoolVolumeTypeCustom. // So check, that nothing else was requested. if req.Type != db.StoragePoolVolumeTypeNameCustom { return response.BadRequest(fmt.Errorf("Currently not allowed to create storage volumes of type %q", req.Type)) } var poolID int64 var dbVolume *db.StorageVolume err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { poolID, err = tx.GetStoragePoolID(ctx, poolName) if err != nil { return err } // Check if destination volume exists. dbVolume, err = tx.GetStoragePoolVolume(ctx, poolID, projectName, db.StoragePoolVolumeTypeCustom, req.Name, true) if err != nil && !response.IsNotFoundError(err) { return err } err = project.AllowVolumeCreation(tx, projectName, poolName, req) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } else if dbVolume != nil && !req.Source.Refresh { return response.Conflict(errors.New("Volume by that name already exists")) } // Check if we need to switch to migration serverName := s.ServerName var nodeAddress string if s.ServerClustered && (req.Source.Location != "" && serverName != req.Source.Location) { err := s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { nodeInfo, err := tx.GetNodeByName(ctx, req.Source.Location) if err != nil { return err } nodeAddress = nodeInfo.Address return nil }) if err != nil { return response.SmartError(err) } if nodeAddress == "" { return response.BadRequest(errors.New("The source is currently offline")) } return clusterCopyCustomVolumeInternal(s, r, nodeAddress, projectName, poolName, &req) } switch req.Source.Type { case "": err = validateCreateConfig(req.Config) if err != nil { return response.SmartError(err) } return doVolumeCreateOrCopy(s, r, request.ProjectParam(r), projectName, poolName, &req) case "copy": if dbVolume != nil { return doCustomVolumeRefresh(s, r, request.ProjectParam(r), projectName, poolName, &req) } return doVolumeCreateOrCopy(s, r, request.ProjectParam(r), projectName, poolName, &req) case "migration": return doVolumeMigration(s, r, request.ProjectParam(r), projectName, poolName, &req) default: return response.BadRequest(fmt.Errorf("Unknown source type %q", req.Source.Type)) } } // validateCreateConfig validates the configuration at creation time // and rejects keys that are not allowed when creating the storage volume. func validateCreateConfig(config map[string]string) error { if util.IsTrue(config["dependent"]) { return errors.New("Config key 'dependent' cannot be set on creation") } return nil } func clusterCopyCustomVolumeInternal(s *state.State, r *http.Request, sourceAddress string, projectName string, poolName string, req *api.StorageVolumesPost) response.Response { websockets := map[string]string{} client, err := cluster.Connect(sourceAddress, s.Endpoints.NetworkCert(), s.ServerCert(), r, false) if err != nil { return response.SmartError(err) } sourceProject := projectName if req.Source.Project != "" { sourceProject = req.Source.Project } client = client.UseProject(sourceProject) pullReq := api.StorageVolumePost{ Name: req.Source.Name, Pool: req.Source.Pool, Migration: true, VolumeOnly: req.Source.VolumeOnly, Source: api.StorageVolumeSource{ Location: req.Source.Location, }, } if sourceProject != projectName { pullReq.Project = projectName } op, err := client.MigrateStoragePoolVolume(req.Source.Pool, pullReq) if err != nil { return response.SmartError(err) } opAPI := op.Get() for k, v := range opAPI.Metadata { websockets[k] = v.(string) } // Reset the source for a migration req.Source.Type = "migration" req.Source.Certificate = string(s.Endpoints.NetworkCert().PublicKey()) req.Source.Mode = "pull" req.Source.Operation = fmt.Sprintf("https://%s/%s/operations/%s", sourceAddress, version.APIVersion, opAPI.ID) req.Source.Websockets = websockets req.Source.Project = "" return doVolumeMigration(s, r, req.Source.Project, projectName, poolName, req) } func doCustomVolumeRefresh(s *state.State, r *http.Request, requestProjectName string, projectName string, poolName string, req *api.StorageVolumesPost) response.Response { pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } var srcProjectName string if req.Source.Project != "" { srcProjectName, err = project.StorageVolumeProject(s.DB.Cluster, req.Source.Project, db.StoragePoolVolumeTypeCustom) if err != nil { return response.SmartError(err) } } run := func(op *operations.Operation) error { reverter := revert.New() defer reverter.Fail() if req.Source.Name == "" { return errors.New("No source volume name supplied") } err = pool.RefreshCustomVolume(projectName, srcProjectName, req.Name, req.Description, req.Config, req.Source.Pool, req.Source.Name, !req.Source.VolumeOnly, req.Source.RefreshExcludeOlder, op) if err != nil { return err } reverter.Success() return nil } op, err := operations.OperationCreate(s, requestProjectName, operations.OperationClassTask, operationtype.VolumeCopy, nil, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } func doVolumeCreateOrCopy(s *state.State, r *http.Request, requestProjectName string, projectName string, poolName string, req *api.StorageVolumesPost) response.Response { pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } var srcProjectName string if req.Source.Project != "" { srcProjectName, err = project.StorageVolumeProject(s.DB.Cluster, req.Source.Project, db.StoragePoolVolumeTypeCustom) if err != nil { return response.SmartError(err) } } volumeDBContentType, err := storagePools.VolumeContentTypeNameToContentType(req.ContentType) if err != nil { return response.SmartError(err) } contentType, err := storagePools.VolumeDBContentTypeToContentType(volumeDBContentType) if err != nil { return response.SmartError(err) } run := func(op *operations.Operation) error { if req.Source.Name == "" { // Use an empty operation for this sync response to pass the requestor op := &operations.Operation{} op.SetRequestor(r) return pool.CreateCustomVolume(projectName, req.Name, req.Description, req.Config, contentType, op) } return pool.CreateCustomVolumeFromCopy(projectName, srcProjectName, req.Name, req.Description, req.Config, req.Source.Pool, req.Source.Name, !req.Source.VolumeOnly, op) } // If no source name supplied then this a volume create operation. if req.Source.Name == "" { err := run(nil) if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } // Volume copy operations potentially take a long time, so run as an async operation. op, err := operations.OperationCreate(s, requestProjectName, operations.OperationClassTask, operationtype.VolumeCopy, nil, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } func doVolumeMigration(s *state.State, r *http.Request, requestProjectName string, projectName string, poolName string, req *api.StorageVolumesPost) response.Response { // Validate migration mode if req.Source.Mode != "pull" && req.Source.Mode != "push" { return response.NotImplemented(fmt.Errorf("Mode '%s' not implemented", req.Source.Mode)) } // create new certificate var err error var cert *x509.Certificate if req.Source.Certificate != "" { certBlock, _ := pem.Decode([]byte(req.Source.Certificate)) if certBlock == nil { return response.InternalError(errors.New("Invalid certificate")) } cert, err = x509.ParseCertificate(certBlock.Bytes) if err != nil { return response.InternalError(err) } } config, err := localtls.GetTLSConfig(cert) if err != nil { return response.InternalError(err) } push := false if req.Source.Mode == "push" { push = true } // Initialize migrationArgs, don't set the Storage property yet, this is done in DoStorage, // to avoid this function relying on the legacy storage layer. migrationArgs := migrationSinkArgs{ URL: req.Source.Operation, Dialer: &websocket.Dialer{ TLSClientConfig: config, NetDialContext: localtls.RFC3493Dialer, HandshakeTimeout: time.Second * 5, }, Secrets: req.Source.Websockets, Push: push, VolumeOnly: req.Source.VolumeOnly, Refresh: req.Source.Refresh, RefreshExcludeOlder: req.Source.RefreshExcludeOlder, } sink, err := newStorageMigrationSink(&migrationArgs) if err != nil { return response.InternalError(err) } resources := map[string][]api.URL{} resources["storage_volumes"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", "custom", req.Name)} run := func(op *operations.Operation) error { // And finally run the migration. err = sink.DoStorage(s, projectName, poolName, req, op) if err != nil { logger.Error("Error during migration sink", logger.Ctx{"err": err}) return fmt.Errorf("Error transferring storage volume: %s", err) } return nil } var op *operations.Operation if push { op, err = operations.OperationCreate(s, requestProjectName, operations.OperationClassWebsocket, operationtype.VolumeCreate, resources, sink.Metadata(), run, sink.Cancel, sink.Connect, r) if err != nil { return response.InternalError(err) } } else { op, err = operations.OperationCreate(s, requestProjectName, operations.OperationClassTask, operationtype.VolumeCopy, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } } return operations.OperationResponse(op) } // swagger:operation POST /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName} storage storage_pool_volume_type_post // // Rename or move/migrate a storage volume // // Renames, moves a storage volume between pools or migrates an instance to another server. // // The returned operation metadata will vary based on what's requested. // For rename or move within the same server, this is a simple background operation with progress data. // For migration, in the push case, this will similarly be a background // operation with progress data, for the pull case, it will be a websocket // operation with a number of secrets to be passed to the target server. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: migration // description: Migration request // schema: // $ref: "#/definitions/StorageVolumePost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumePost(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(volumeName) { return response.BadRequest(errors.New("Invalid volume name")) } // Get the name of the storage pool the volume is supposed to be attached to. srcPoolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } req := api.StorageVolumePost{} // Parse the request. err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid storage volume name: %w", err)) } // Check requested new volume name is not a snapshot volume. if internalInstance.IsSnapshot(req.Name) { return response.BadRequest(errors.New("Storage volume names may not contain slashes")) } // We currently only allow to create storage volumes of type storagePoolVolumeTypeCustom. // So check, that nothing else was requested. if volumeTypeName != db.StoragePoolVolumeTypeNameCustom { return response.BadRequest(fmt.Errorf("Renaming storage volumes of type %q is not allowed", volumeTypeName)) } projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), db.StoragePoolVolumeTypeCustom) if err != nil { return response.SmartError(err) } targetProjectName := projectName if req.Project != "" { targetProjectName, err = project.StorageVolumeProject(s.DB.Cluster, req.Project, db.StoragePoolVolumeTypeCustom) if err != nil { return response.SmartError(err) } // Check whether the effective storage project differs from the requested target project. // If they do it means that the requested target project doesn't have features.storage.volumes // and this means that the volume would effectively be moved into the default project, and so we // require the user explicitly indicates this by targeting it directly. if targetProjectName != req.Project { return response.BadRequest(errors.New("Target project does not have features.storage.volumes enabled")) } if projectName == targetProjectName { return response.BadRequest(errors.New("Project and target project are the same")) } // Check if user has access to effective storage target project err := s.Authorizer.CheckPermission(r.Context(), r, auth.ObjectProject(targetProjectName), auth.EntitlementCanCreateStorageVolumes) if err != nil { return response.SmartError(err) } } // We need to restore the body of the request since it has already been read, and if we // forwarded it now no body would be written out. buf := bytes.Buffer{} err = json.NewEncoder(&buf).Encode(req) if err != nil { return response.SmartError(err) } r.Body = internalIO.BytesReadCloser{Buf: &buf} target := request.QueryParam(r, "target") // Check if clustered. if s.ServerClustered && target != "" && req.Source.Location != "" && req.Migration { var sourceNodeOffline bool err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Load source node. nodeInfo, err := tx.GetNodeByName(ctx, req.Source.Location) if err != nil { return err } sourceAddress := nodeInfo.Address if sourceAddress == "" { // Local node. sourceNodeOffline = false return nil } sourceMemberInfo, err := tx.GetNodeByAddress(ctx, sourceAddress) if err != nil { return fmt.Errorf("Failed to get source member for %q: %w", sourceAddress, err) } sourceNodeOffline = sourceMemberInfo.IsOffline(s.GlobalConfig.OfflineThreshold()) return nil }) if err != nil { return response.SmartError(err) } var targetProject *api.Project var targetMemberInfo *db.NodeInfo if sourceNodeOffline { resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } srcPool, err := storagePools.LoadByName(s, srcPoolName) if err != nil { return response.SmartError(err) } if srcPool.Driver().Info().Remote { var dbVolume *db.StorageVolume var volumeNotFound bool var targetIsSet bool err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Load source volume. srcPoolID, err := tx.GetStoragePoolID(ctx, srcPoolName) if err != nil { return err } dbVolume, err = tx.GetStoragePoolVolume(ctx, srcPoolID, projectName, db.StoragePoolVolumeTypeCustom, volumeName, true) if err != nil { // Check if the user provided an incorrect target query parameter and return a helpful error message. _, volumeNotFound = api.StatusErrorMatch(err, http.StatusNotFound) targetIsSet = r.URL.Query().Get("target") != "" return err } return nil }) if err != nil { if s.ServerClustered && targetIsSet && volumeNotFound { return response.NotFound(errors.New("Storage volume not found on this cluster member")) } return response.SmartError(err) } req := api.StorageVolumePost{ Name: req.Name, } return storagePoolVolumeTypePostRename(s, r, srcPool.Name(), projectName, &dbVolume.StorageVolume, req) } } else { resp := forwardedResponseToNode(s, r, req.Source.Location) if resp != nil { return resp } } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { p, err := dbCluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return err } targetProject, err = p.ToAPI(ctx, tx.Tx()) if err != nil { return err } allMembers, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } targetMemberInfo, _, err = project.CheckTarget(ctx, s.Authorizer, r, tx, targetProject, target, allMembers) if err != nil { return err } if targetMemberInfo == nil { return fmt.Errorf("Failed checking cluster member %q", target) } return nil }) if err != nil { return response.SmartError(err) } if targetMemberInfo.IsOffline(s.GlobalConfig.OfflineThreshold()) { return response.BadRequest(errors.New("Target cluster member is offline")) } run := func(op *operations.Operation) error { return migrateStorageVolume(s, r, volumeName, srcPoolName, targetMemberInfo.Name, targetProjectName, req, op) } resources := map[string][]api.URL{} resources["storage_volumes"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", srcPoolName, "volumes", "custom", volumeName)} op, err := operations.OperationCreate(s, projectName, operations.OperationClassTask, operationtype.VolumeMigrate, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // If source is set, we know the source and the target, and therefore don't need this function to figure out where to forward the request to. if req.Source.Location == "" { resp = forwardedResponseIfVolumeIsRemote(s, r, srcPoolName, projectName, volumeName, volumeType) if resp != nil { return resp } } // This is a migration request so send back requested secrets. if req.Migration { return storagePoolVolumeTypePostMigration(s, r, request.ProjectParam(r), projectName, srcPoolName, volumeName, req) } // Retrieve ID of the storage pool (and check if the storage pool exists). var targetPoolID int64 var targetPoolName string if req.Pool != "" { targetPoolName = req.Pool } else { targetPoolName = srcPoolName } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { targetPoolID, err = tx.GetStoragePoolID(ctx, targetPoolName) return err }) if err != nil { return response.SmartError(err) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Check that the name isn't already in use. _, err = tx.GetStoragePoolNodeVolumeID(ctx, targetProjectName, req.Name, volumeType, targetPoolID) return err }) if !response.IsNotFoundError(err) { if err != nil { return response.InternalError(err) } return response.Conflict(errors.New("Volume by that name already exists")) } // Check if the daemon itself is using it. used, err := storagePools.VolumeUsedByDaemon(s, srcPoolName, volumeName) if err != nil { return response.SmartError(err) } if used { return response.SmartError(errors.New("Volume is used by Incus itself and cannot be renamed")) } var dbVolume *db.StorageVolume var volumeNotFound bool var targetIsSet bool err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Load source volume. srcPoolID, err := tx.GetStoragePoolID(ctx, srcPoolName) if err != nil { return err } dbVolume, err = tx.GetStoragePoolVolume(ctx, srcPoolID, projectName, volumeType, volumeName, true) if err != nil { // Check if the user provided an incorrect target query parameter and return a helpful error message. _, volumeNotFound = api.StatusErrorMatch(err, http.StatusNotFound) targetIsSet = r.URL.Query().Get("target") != "" return err } return nil }) if err != nil { if s.ServerClustered && targetIsSet && volumeNotFound { return response.NotFound(errors.New("Storage volume not found on this cluster member")) } return response.SmartError(err) } // Check if a running instance is using it. err = storagePools.VolumeUsedByInstanceDevices(s, srcPoolName, projectName, &dbVolume.StorageVolume, true, func(dbInst db.InstanceArgs, project api.Project, usedByDevices []string) error { inst, err := instance.Load(s, dbInst, project) if err != nil { return err } if inst.IsRunning() { return errors.New("Volume is still in use by running instances") } return nil }) if err != nil { return response.SmartError(err) } // Detect a rename request. if (req.Pool == "" || req.Pool == srcPoolName) && (projectName == targetProjectName) { return storagePoolVolumeTypePostRename(s, r, srcPoolName, projectName, &dbVolume.StorageVolume, req) } // Otherwise this is a move request. return storagePoolVolumeTypePostMove(s, r, srcPoolName, projectName, targetProjectName, &dbVolume.StorageVolume, req) } func migrateStorageVolume(s *state.State, r *http.Request, sourceVolumeName string, sourcePoolName string, targetNode string, projectName string, req api.StorageVolumePost, op *operations.Operation) error { if targetNode == req.Source.Location { return errors.New("Target must be different than storage volumes' current location") } var err error var srcMember, newMember db.NodeInfo // If the source member is online then get its address so we can connect to it and see if the // instance is running later. err = s.DB.Cluster.Transaction(s.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { srcMember, err = tx.GetNodeByName(ctx, req.Source.Location) if err != nil { return fmt.Errorf("Failed getting current cluster member of storage volume %q", req.Source.Name) } newMember, err = tx.GetNodeByName(ctx, targetNode) if err != nil { return fmt.Errorf("Failed loading new cluster member for storage volume: %w", err) } return nil }) if err != nil { return err } srcPool, err := storagePools.LoadByName(s, sourcePoolName) if err != nil { return fmt.Errorf("Failed loading storage volume storage pool: %w", err) } f, err := storageVolumePostClusteringMigrate(s, r, srcPool, projectName, sourceVolumeName, req.Pool, req.Project, req.Name, srcMember, newMember, req.VolumeOnly) if err != nil { return err } return f(op) } func storageVolumePostClusteringMigrate(s *state.State, r *http.Request, srcPool storagePools.Pool, srcProjectName string, srcVolumeName string, newPoolName string, newProjectName string, newVolumeName string, srcMember db.NodeInfo, newMember db.NodeInfo, volumeOnly bool) (func(op *operations.Operation) error, error) { srcMemberOffline := srcMember.IsOffline(s.GlobalConfig.OfflineThreshold()) // Make sure that the source member is online if we end up being called from another member after a // redirection due to the source member being offline. if srcMemberOffline { return nil, errors.New("The cluster member hosting the storage volume is offline") } run := func(op *operations.Operation) error { if newVolumeName == "" { newVolumeName = srcVolumeName } networkCert := s.Endpoints.NetworkCert() // Connect to the destination member, i.e. the member to migrate the custom volume to. // Use the notify argument to indicate to the destination that we are moving a custom volume between // cluster members. dest, err := cluster.Connect(newMember.Address, networkCert, s.ServerCert(), r, true) if err != nil { return fmt.Errorf("Failed to connect to destination server %q: %w", newMember.Address, err) } dest = dest.UseTarget(newMember.Name).UseProject(srcProjectName) resources := map[string][]api.URL{} resources["storage_volumes"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", srcPool.Name(), "volumes", "custom", srcVolumeName)} srcMigration, err := newStorageMigrationSource(volumeOnly, nil) if err != nil { return fmt.Errorf("Failed setting up storage volume migration on source: %w", err) } run := func(op *operations.Operation) error { err := srcMigration.DoStorage(s, srcProjectName, srcPool.Name(), srcVolumeName, op) if err != nil { return err } err = srcPool.DeleteCustomVolume(srcProjectName, srcVolumeName, op) if err != nil { return err } return nil } cancel := func(op *operations.Operation) error { srcMigration.disconnect() return nil } srcOp, err := operations.OperationCreate(s, srcProjectName, operations.OperationClassWebsocket, operationtype.VolumeMigrate, resources, srcMigration.Metadata(), run, cancel, srcMigration.Connect, r) if err != nil { return err } err = srcOp.Start() if err != nil { return fmt.Errorf("Failed starting migration source operation: %w", err) } sourceSecrets := make(map[string]string, len(srcMigration.conns)) for connName, conn := range srcMigration.conns { sourceSecrets[connName] = conn.Secret() } // Request pull mode migration on destination. err = dest.CreateStoragePoolVolume(newPoolName, api.StorageVolumesPost{ Name: newVolumeName, Type: "custom", Source: api.StorageVolumeSource{ Type: "migration", Mode: "pull", Operation: fmt.Sprintf("https://%s%s", srcMember.Address, srcOp.URL()), Websockets: sourceSecrets, Certificate: string(networkCert.PublicKey()), Name: newVolumeName, Pool: newPoolName, Project: newProjectName, }, }) if err != nil { return fmt.Errorf("Failed requesting volume create on destination: %w", err) } return nil } return run, nil } // storagePoolVolumeTypePostMigration handles volume migration type POST requests. func storagePoolVolumeTypePostMigration(state *state.State, r *http.Request, requestProjectName string, projectName string, poolName string, volumeName string, req api.StorageVolumePost) response.Response { ws, err := newStorageMigrationSource(req.VolumeOnly, req.Target) if err != nil { return response.InternalError(err) } resources := map[string][]api.URL{} srcVolParentName, srcVolSnapName, srcIsSnapshot := api.GetParentAndSnapshotName(volumeName) if srcIsSnapshot { resources["storage_volume_snapshots"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", "custom", srcVolParentName, "snapshots", srcVolSnapName)} } else { resources["storage_volumes"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", "custom", volumeName)} } run := func(op *operations.Operation) error { return ws.DoStorage(state, projectName, poolName, volumeName, op) } if req.Target != nil { // Push mode. op, err := operations.OperationCreate(state, requestProjectName, operations.OperationClassTask, operationtype.VolumeMigrate, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // Pull mode. op, err := operations.OperationCreate(state, requestProjectName, operations.OperationClassWebsocket, operationtype.VolumeMigrate, resources, ws.Metadata(), run, ws.Cancel, ws.Connect, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // storagePoolVolumeTypePostRename handles volume rename type POST requests. func storagePoolVolumeTypePostRename(s *state.State, r *http.Request, poolName string, projectName string, vol *api.StorageVolume, req api.StorageVolumePost) response.Response { newVol := *vol newVol.Name = req.Name pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } reverter := revert.New() defer reverter.Fail() // Update devices using the volume in instances and profiles. err = storagePoolVolumeUpdateUsers(r.Context(), s, projectName, pool.Name(), vol, pool.Name(), &newVol) if err != nil { return response.SmartError(err) } // Use an empty operation for this sync response to pass the requestor op := &operations.Operation{} op.SetRequestor(r) err = pool.RenameCustomVolume(projectName, vol.Name, req.Name, op) if err != nil { return response.SmartError(err) } reverter.Success() u := api.NewURL().Path(version.APIVersion, "storage-pools", pool.Name(), "volumes", db.StoragePoolVolumeTypeNameCustom, req.Name).Project(projectName) return response.SyncResponseLocation(true, nil, u.String()) } // storagePoolVolumeTypePostMove handles volume move type POST requests. func storagePoolVolumeTypePostMove(s *state.State, r *http.Request, poolName string, requestProjectName string, projectName string, vol *api.StorageVolume, req api.StorageVolumePost) response.Response { newVol := *vol newVol.Name = req.Name pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } newPool, err := storagePools.LoadByName(s, req.Pool) if err != nil { return response.SmartError(err) } run := func(op *operations.Operation) error { reverter := revert.New() defer reverter.Fail() // Update devices using the volume in instances and profiles. err = storagePoolVolumeUpdateUsers(context.TODO(), s, requestProjectName, pool.Name(), vol, newPool.Name(), &newVol) if err != nil { return err } reverter.Add(func() { _ = storagePoolVolumeUpdateUsers(context.TODO(), s, projectName, newPool.Name(), &newVol, pool.Name(), vol) }) // Provide empty description and nil config to instruct CreateCustomVolumeFromCopy to copy it // from source volume. err = newPool.CreateCustomVolumeFromCopy(projectName, requestProjectName, newVol.Name, "", nil, pool.Name(), vol.Name, true, op) if err != nil { return err } err = pool.DeleteCustomVolume(requestProjectName, vol.Name, op) if err != nil { return err } reverter.Success() return nil } op, err := operations.OperationCreate(s, requestProjectName, operations.OperationClassTask, operationtype.VolumeMove, nil, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName} storage storage_pool_volume_type_get // // Get the storage volume // // Gets a specific storage volume. // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Storage volume // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/StorageVolume" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}?recursion=1 storage storage_pool_volume_type_get_recursion1 // // Get the full storage volume details // // Gets a specific storage volume with all details (backups, snapshots and state0.. // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Storage volume // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/StorageVolumeFull" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeGet(d *Daemon, r *http.Request) response.Response { s := d.State() volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage pool the volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if !slices.Contains(supportedVolumeTypes, volumeType) { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } requestProjectName := request.ProjectParam(r) volumeProjectName, err := project.StorageVolumeProject(s.DB.Cluster, requestProjectName, volumeType) if err != nil { return response.SmartError(err) } resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } resp = forwardedResponseIfVolumeIsRemote(s, r, poolName, volumeProjectName, volumeName, volumeType) if resp != nil { return resp } var dbVolume *db.StorageVolume var poolID int64 err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get the ID of the storage pool the storage volume is supposed to be attached to. poolID, err = tx.GetStoragePoolID(ctx, poolName) if err != nil { return err } // Get the storage volume. dbVolume, err = tx.GetStoragePoolVolume(ctx, poolID, volumeProjectName, volumeType, volumeName, true) return err }) if err != nil { return response.SmartError(err) } volumeUsedBy, err := storagePoolVolumeUsedByGet(s, requestProjectName, poolName, dbVolume) if err != nil { return response.SmartError(err) } dbVolume.UsedBy = project.FilterUsedBy(s.Authorizer, r, volumeUsedBy) etag := []any{volumeName, dbVolume.Type, dbVolume.Config} // Prepare the response. if localUtil.IsRecursionRequest(r) { volFull, err := getVolumeFull(r.Context(), s, poolName, dbVolume.StorageVolume) if err != nil { return response.SmartError(err) } return response.SyncResponseETag(true, volFull, etag) } return response.SyncResponseETag(true, dbVolume.StorageVolume, etag) } func getVolumeFull(ctx context.Context, s *state.State, poolName string, vol api.StorageVolume) (*api.StorageVolumeFull, error) { // Convert the volume type name to our internal integer representation. volType, err := storagePools.VolumeTypeNameToDBType(vol.Type) if err != nil { return nil, err } // Set the base object. resp := api.StorageVolumeFull{ StorageVolume: vol, Backups: []api.StorageVolumeBackup{}, } // Get the pool. pool, err := storagePools.LoadByName(s, poolName) if err != nil { return nil, err } // Add all backups. err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { volumeBackups, err := tx.GetStoragePoolVolumeBackups(ctx, vol.Project, vol.Name, pool.ID()) if err != nil { return err } for _, entry := range volumeBackups { resp.Backups = append(resp.Backups, *backup.NewVolumeBackup(s, vol.Project, poolName, vol.Name, entry.ID, entry.Name, entry.CreationDate, entry.ExpiryDate, entry.VolumeOnly, entry.OptimizedStorage).Render()) } return nil }) if err != nil { return nil, err } // Add all snapshots. err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { volumeSnapshots, err := tx.GetLocalStoragePoolVolumeSnapshotsWithType(ctx, vol.Project, vol.Name, volType, pool.ID()) if err != nil { return err } for _, entry := range volumeSnapshots { _, snapName, _ := api.GetParentAndSnapshotName(entry.Name) snap := api.StorageVolumeSnapshot{} snap.Config = entry.Config snap.Description = entry.Description snap.Name = snapName snap.CreatedAt = entry.CreationDate if entry.ExpiryDate.Unix() > 0 { snap.ExpiresAt = &entry.ExpiryDate } resp.Snapshots = append(resp.Snapshots, snap) } return nil }) if err != nil { return nil, err } // Add the state. var usage *storagePools.VolumeUsage switch volType { case db.StoragePoolVolumeTypeCustom: usage, err = pool.GetCustomVolumeUsage(vol.Project, vol.Name) if err != nil && !errors.Is(err, storageDrivers.ErrNotSupported) { return nil, err } case db.StoragePoolVolumeTypeContainer, db.StoragePoolVolumeTypeVM: inst, err := instance.LoadByProjectAndName(s, vol.Project, vol.Name) if err != nil { return nil, err } usage, err = pool.GetInstanceUsage(inst) if err != nil && !errors.Is(err, storageDrivers.ErrNotSupported) { return nil, err } default: } volState := api.StorageVolumeState{} if usage != nil { volState.Usage = &api.StorageVolumeStateUsage{} // Only fill 'used' field if receiving a valid value. if usage.Used >= 0 { volState.Usage.Used = uint64(usage.Used) } // Only fill 'total' field if receiving a valid value. if usage.Total >= 0 { volState.Usage.Total = usage.Total } } resp.State = &volState return &resp, nil } // swagger:operation PUT /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName} storage storage_pool_volume_type_put // // Update the storage volume // // Updates the entire storage volume configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: storage volume // description: Storage volume configuration // required: true // schema: // $ref: "#/definitions/StorageVolumePut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumePut(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage pool the volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } projectName, err = project.StorageVolumeProject(s.DB.Cluster, projectName, volumeType) if err != nil { return response.SmartError(err) } // Check that the storage volume type is valid. if !slices.Contains(supportedVolumeTypes, volumeType) { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } resp = forwardedResponseIfVolumeIsRemote(s, r, pool.Name(), projectName, volumeName, volumeType) if resp != nil { return resp } // Get the existing storage volume. var dbVolume *db.StorageVolume err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, pool.ID(), projectName, volumeType, volumeName, true) return err }) if err != nil { return response.SmartError(err) } // Validate the ETag etag := []any{volumeName, dbVolume.Type, dbVolume.Config} err = localUtil.EtagCheck(r, etag) if err != nil { return response.PreconditionFailed(err) } req := api.StorageVolumePut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Use an empty operation for this sync response to pass the requestor op := &operations.Operation{} op.SetRequestor(r) if volumeType == db.StoragePoolVolumeTypeCustom { // Restore custom volume from snapshot if requested. This should occur first // before applying config changes so that changes are applied to the // restored volume. if req.Restore != "" { err = pool.RestoreCustomVolume(projectName, dbVolume.Name, req.Restore, op) if err != nil { return response.SmartError(err) } } // Handle custom volume update requests. // Only apply changes during a snapshot restore if a non-nil config is supplied to avoid clearing // the volume's config if only restoring snapshot. if req.Config != nil || req.Restore == "" { // Possibly check if project limits are honored. err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { return project.AllowVolumeUpdate(tx, projectName, volumeName, req, dbVolume.Config) }) if err != nil { return response.SmartError(err) } err = pool.UpdateCustomVolume(projectName, dbVolume.Name, req.Description, req.Config, op) if err != nil { return response.SmartError(err) } } } else if volumeType == db.StoragePoolVolumeTypeContainer || volumeType == db.StoragePoolVolumeTypeVM { inst, err := instance.LoadByProjectAndName(s, projectName, dbVolume.Name) if err != nil { return response.SmartError(err) } // Handle instance volume update requests. err = pool.UpdateInstance(inst, req.Description, req.Config, op) if err != nil { return response.SmartError(err) } } else if volumeType == db.StoragePoolVolumeTypeImage { // Handle image update requests. err = pool.UpdateImage(dbVolume.Name, req.Description, req.Config, op) if err != nil { return response.SmartError(err) } } else { return response.SmartError(errors.New("Invalid volume type")) } return response.EmptySyncResponse } // swagger:operation PATCH /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName} storage storage_pool_volume_type_patch // // Partially update the storage volume // // Updates a subset of the storage volume configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: storage volume // description: Storage volume configuration // required: true // schema: // $ref: "#/definitions/StorageVolumePut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumePatch(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(volumeName) { return response.BadRequest(errors.New("Invalid volume name")) } // Get the name of the storage pool the volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is custom. if volumeType != db.StoragePoolVolumeTypeCustom { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), volumeType) if err != nil { return response.SmartError(err) } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } resp = forwardedResponseIfVolumeIsRemote(s, r, pool.Name(), projectName, volumeName, volumeType) if resp != nil { return resp } // Get the existing storage volume. var dbVolume *db.StorageVolume err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, pool.ID(), projectName, volumeType, volumeName, true) return err }) if err != nil { return response.SmartError(err) } // Validate the ETag. etag := []any{volumeName, dbVolume.Type, dbVolume.Config} err = localUtil.EtagCheck(r, etag) if err != nil { return response.PreconditionFailed(err) } req := api.StorageVolumePut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } if req.Config == nil { req.Config = map[string]string{} } // Merge current config with requested changes. for k, v := range dbVolume.Config { _, ok := req.Config[k] if !ok { req.Config[k] = v } } // Use an empty operation for this sync response to pass the requestor op := &operations.Operation{} op.SetRequestor(r) err = pool.UpdateCustomVolume(projectName, dbVolume.Name, req.Description, req.Config, op) if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } // swagger:operation DELETE /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName} storage storage_pool_volume_type_delete // // Delete the storage volume // // Removes the storage volume. // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeDelete(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } if internalInstance.IsSnapshot(volumeName) { return response.BadRequest(fmt.Errorf("Invalid storage volume %q", volumeName)) } // Get the name of the storage pool the volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } requestProjectName := request.ProjectParam(r) volumeProjectName, err := project.StorageVolumeProject(s.DB.Cluster, requestProjectName, volumeType) if err != nil { return response.SmartError(err) } // Check that the storage volume type is valid. if !slices.Contains(supportedVolumeTypes, volumeType) { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } resp = forwardedResponseIfVolumeIsRemote(s, r, poolName, volumeProjectName, volumeName, volumeType) if resp != nil { return resp } if volumeType != db.StoragePoolVolumeTypeCustom && volumeType != db.StoragePoolVolumeTypeImage { return response.BadRequest(fmt.Errorf("Storage volumes of type %q cannot be deleted with the storage API", volumeTypeName)) } // Get the storage pool the storage volume is supposed to be attached to. pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } // Get the storage volume. var dbVolume *db.StorageVolume err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, pool.ID(), volumeProjectName, volumeType, volumeName, true) return err }) if err != nil { return response.SmartError(err) } volumeUsedBy, err := storagePoolVolumeUsedByGet(s, requestProjectName, poolName, dbVolume) if err != nil { return response.SmartError(err) } // isImageURL checks whether the provided usedByURL represents an image resource for the fingerprint. isImageURL := func(usedByURL string, fingerprint string) bool { usedBy, _ := url.Parse(usedByURL) if usedBy == nil { return false } img := api.NewURL().Path(version.APIVersion, "images", fingerprint) return usedBy.Path == img.URL.Path } if len(volumeUsedBy) > 0 { if len(volumeUsedBy) != 1 || volumeType != db.StoragePoolVolumeTypeImage || !isImageURL(volumeUsedBy[0], dbVolume.Name) { return response.BadRequest(errors.New("The storage volume is still in use")) } } // Use an empty operation for this sync response to pass the requestor op := &operations.Operation{} op.SetRequestor(r) switch volumeType { case db.StoragePoolVolumeTypeCustom: err = pool.DeleteCustomVolume(volumeProjectName, volumeName, op) case db.StoragePoolVolumeTypeImage: err = pool.DeleteImage(volumeName, op) default: return response.BadRequest(fmt.Errorf(`Storage volumes of type %q cannot be deleted with the storage API`, volumeTypeName)) } if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } func createStoragePoolVolumeFromISO(s *state.State, r *http.Request, requestProjectName string, projectName string, data io.Reader, pool string, volName string) response.Response { reverter := revert.New() defer reverter.Fail() if volName == "" { return response.BadRequest(errors.New("Missing volume name")) } // Create isos directory if needed. if !util.PathExists(internalUtil.VarPath("isos")) { err := os.MkdirAll(internalUtil.VarPath("isos"), 0o644) if err != nil { return response.InternalError(err) } } // Create temporary file to store uploaded ISO data. isoFile, err := os.CreateTemp(internalUtil.VarPath("isos"), fmt.Sprintf("%s_", "incus_iso")) if err != nil { return response.InternalError(err) } defer func() { _ = os.Remove(isoFile.Name()) }() reverter.Add(func() { _ = isoFile.Close() }) // Get disk budget for the project if any. var budget int64 err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { budget, err = project.GetSpaceBudget(tx, projectName) if err != nil { return err } return nil }) if err != nil { return response.InternalError(err) } // Stream uploaded ISO data into temporary file. size, err := util.SafeCopy(internalIO.NewQuotaWriter(isoFile, budget), data) if err != nil { return response.InternalError(err) } // Copy reverter so far so we can use it inside run after this function has finished. runReverter := reverter.Clone() run := func(op *operations.Operation) error { defer func() { _ = isoFile.Close() }() defer runReverter.Fail() pool, err := storagePools.LoadByName(s, pool) if err != nil { return err } // Dump ISO to storage. err = pool.CreateCustomVolumeFromISO(projectName, volName, isoFile, size, op) if err != nil { return fmt.Errorf("Failed creating custom volume from ISO: %w", err) } runReverter.Success() return nil } resources := map[string][]api.URL{} resources["storage_volumes"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", pool, "volumes", "custom", volName)} op, err := operations.OperationCreate(s, requestProjectName, operations.OperationClassTask, operationtype.VolumeCreate, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } reverter.Success() return operations.OperationResponse(op) } func createStoragePoolVolumeFromBackup(s *state.State, r *http.Request, requestProjectName string, projectName string, data io.Reader, pool string, volName string) response.Response { reverter := revert.New() defer reverter.Fail() // Create temporary file to store uploaded backup data. backupFile, err := os.CreateTemp(internalUtil.VarPath("backups"), fmt.Sprintf("%s_", backup.WorkingDirPrefix)) if err != nil { return response.InternalError(err) } defer func() { _ = os.Remove(backupFile.Name()) }() reverter.Add(func() { _ = backupFile.Close() }) // Get disk budget for the project if any. var budget int64 err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { budget, err = project.GetSpaceBudget(tx, projectName) if err != nil { return err } return nil }) if err != nil { return response.InternalError(err) } // Stream uploaded backup data into temporary file. _, err = util.SafeCopy(internalIO.NewQuotaWriter(backupFile, budget), data) if err != nil { return response.InternalError(err) } // Detect squashfs compression and convert to tarball. _, err = backupFile.Seek(0, io.SeekStart) if err != nil { return response.InternalError(err) } _, algo, decomArgs, err := archive.DetectCompressionFile(backupFile) if err != nil { return response.InternalError(err) } if algo == ".squashfs" { // Pass the temporary file as program argument to the decompression command. decomArgs := append(decomArgs, backupFile.Name()) // Create temporary file to store the decompressed tarball in. tarFile, err := os.CreateTemp(internalUtil.VarPath("backups"), fmt.Sprintf("%s_decompress_", backup.WorkingDirPrefix)) if err != nil { return response.InternalError(err) } defer func() { _ = os.Remove(tarFile.Name()) }() // Decompress to tarFile temporary file. err = archive.ExtractWithFds(decomArgs[0], decomArgs[1:], nil, nil, tarFile) if err != nil { return response.InternalError(err) } // We don't need the original squashfs file anymore. _ = backupFile.Close() _ = os.Remove(backupFile.Name()) // Replace the backup file handle with the handle to the tar file. backupFile = tarFile } // Parse the backup information. _, err = backupFile.Seek(0, io.SeekStart) if err != nil { return response.InternalError(err) } logger.Debug("Reading backup file info") bInfo, err := backup.GetInfo(backupFile, s.OS, backupFile.Name()) if err != nil { return response.BadRequest(err) } bInfo.Project = projectName // Override pool. if pool != "" { bInfo.Pool = pool } // Override volume name. if volName != "" { bInfo.Name = volName } logger.Debug("Backup file info loaded", logger.Ctx{ "type": bInfo.Type, "name": bInfo.Name, "project": bInfo.Project, "backend": bInfo.Backend, "pool": bInfo.Pool, "optimized": *bInfo.OptimizedStorage, "snapshots": bInfo.Snapshots, }) err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Check storage pool exists. _, _, _, err = tx.GetStoragePoolInAnyState(ctx, bInfo.Pool) return err }) if response.IsNotFoundError(err) { // The storage pool doesn't exist. If backup is in binary format (so we cannot alter // the backup.yaml) or the pool has been specified directly from the user restoring // the backup then we cannot proceed so return an error. if *bInfo.OptimizedStorage || pool != "" { return response.InternalError(fmt.Errorf("Storage pool not found: %w", err)) } var profile *api.Profile err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Otherwise try and restore to the project's default profile pool. _, profile, err = tx.GetProfile(ctx, bInfo.Project, "default") return err }) if err != nil { return response.InternalError(fmt.Errorf("Failed to get default profile: %w", err)) } _, v, err := internalInstance.GetRootDiskDevice(profile.Devices) if err != nil { return response.InternalError(fmt.Errorf("Failed to get root disk device: %w", err)) } // Use the default-profile's root pool. bInfo.Pool = v["pool"] } else if err != nil { return response.InternalError(err) } // Copy reverter so far so we can use it inside run after this function has finished. runReverter := reverter.Clone() run := func(op *operations.Operation) error { defer func() { _ = backupFile.Close() }() defer runReverter.Fail() pool, err := storagePools.LoadByName(s, bInfo.Pool) if err != nil { return err } // Check if the backup is optimized that the source pool driver matches the target pool driver. if *bInfo.OptimizedStorage && pool.Driver().Info().Name != bInfo.Backend { return fmt.Errorf("Optimized backup storage driver %q differs from the target storage pool driver %q", bInfo.Backend, pool.Driver().Info().Name) } // Dump tarball to storage. err = pool.CreateCustomVolumeFromBackup(*bInfo, backupFile, backup.DefaultBackupPrefix, nil) if err != nil { return fmt.Errorf("Create custom volume from backup: %w", err) } runReverter.Success() return nil } resources := map[string][]api.URL{} resources["storage_volumes"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", bInfo.Pool, "volumes", string(bInfo.Type), bInfo.Name)} op, err := operations.OperationCreate(s, requestProjectName, operations.OperationClassTask, operationtype.CustomVolumeBackupRestore, resources, nil, run, nil, nil, r) if err != nil { return response.InternalError(err) } reverter.Success() return operations.OperationResponse(op) } incus-7.0.0/cmd/incusd/storage_volumes_backup.go000066400000000000000000001005171517523235500217750ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "strings" "time" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/server/auth" internalBackup "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" storagePools "github.com/lxc/incus/v7/internal/server/storage" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/validate" ) var storagePoolVolumeTypeCustomBackupsCmd = APIEndpoint{ Path: "storage-pools/{poolName}/volumes/{type}/{volumeName}/backups", Get: APIEndpointAction{Handler: storagePoolVolumeTypeCustomBackupsGet, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanView, "poolName", "type", "volumeName", "location")}, Post: APIEndpointAction{Handler: storagePoolVolumeTypeCustomBackupsPost, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanManageBackups, "poolName", "type", "volumeName", "location")}, } var storagePoolVolumeTypeCustomBackupCmd = APIEndpoint{ Path: "storage-pools/{poolName}/volumes/{type}/{volumeName}/backups/{backupName}", Get: APIEndpointAction{Handler: storagePoolVolumeTypeCustomBackupGet, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanView, "poolName", "type", "volumeName", "location")}, Post: APIEndpointAction{Handler: storagePoolVolumeTypeCustomBackupPost, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanManageBackups, "poolName", "type", "volumeName", "location")}, Delete: APIEndpointAction{Handler: storagePoolVolumeTypeCustomBackupDelete, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanManageBackups, "poolName", "type", "volumeName", "location")}, } var storagePoolVolumeTypeCustomBackupExportCmd = APIEndpoint{ Path: "storage-pools/{poolName}/volumes/{type}/{volumeName}/backups/{backupName}/export", Get: APIEndpointAction{Handler: storagePoolVolumeTypeCustomBackupExportGet, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanView, "poolName", "type", "volumeName", "location")}, } // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/backups storage storage_pool_volumes_type_backups_get // // Get the storage volume backups // // Returns a list of storage volume backups (URLs). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/storage-pools/local/volumes/custom/foo/backups/backup0", // "/1.0/storage-pools/local/volumes/custom/foo/backups/backup1" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/backups?recursion=1 storage storage_pool_volumes_type_backups_get_recursion1 // // Get the storage volume backups // // Returns a list of storage volume backups (structs). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of storage volume backups // items: // $ref: "#/definitions/StorageVolumeBackup" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeTypeCustomBackupsGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), db.StoragePoolVolumeTypeCustom) if err != nil { return response.SmartError(err) } // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage pool the volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if volumeType != db.StoragePoolVolumeTypeCustom { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } var poolID int64 err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error poolID, _, _, err = tx.GetStoragePool(ctx, poolName) return err }) if err != nil { return response.SmartError(err) } // Handle requests targeted to a volume on a different node resp := forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, volumeName, db.StoragePoolVolumeTypeCustom) if resp != nil { return resp } recursion := localUtil.IsRecursionRequest(r) var volumeBackups []db.StoragePoolVolumeBackup err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { volumeBackups, err = tx.GetStoragePoolVolumeBackups(ctx, projectName, volumeName, poolID) return err }) if err != nil { return response.SmartError(err) } backups := make([]*internalBackup.VolumeBackup, len(volumeBackups)) for i, b := range volumeBackups { backups[i] = internalBackup.NewVolumeBackup(s, projectName, poolName, volumeName, b.ID, b.Name, b.CreationDate, b.ExpiryDate, b.VolumeOnly, b.OptimizedStorage) } resultString := []string{} resultMap := []*api.StorageVolumeBackup{} for _, backup := range backups { if !recursion { url := api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", "custom", volumeName, "backups", strings.Split(backup.Name(), "/")[1]).String() resultString = append(resultString, url) } else { render := backup.Render() resultMap = append(resultMap, render) } } if !recursion { return response.SyncResponse(true, resultString) } return response.SyncResponse(true, resultMap) } // swagger:operation POST /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/backups storage storage_pool_volumes_type_backups_post // // Create a storage volume backup // // Creates a new storage volume backup. // // If the `Accept` header is set to `application/octet-stream`, this directly streams the backup // tarball to the client without any intermediate operation. // // --- // consumes: // - application/json // produces: // - application/json // - application/octet-stream // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: volume // description: Storage volume backup // required: true // schema: // $ref: "#/definitions/StorageVolumeBackupsPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeTypeCustomBackupsPost(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage pool the volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if volumeType != db.StoragePoolVolumeTypeCustom { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), db.StoragePoolVolumeTypeCustom) if err != nil { return response.SmartError(err) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { err := project.AllowBackupCreation(tx, projectName) return err }) if err != nil { return response.SmartError(err) } resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } var poolID int64 err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error poolID, _, _, err = tx.GetStoragePool(ctx, poolName) return err }) if err != nil { return response.SmartError(err) } resp = forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, volumeName, db.StoragePoolVolumeTypeCustom) if resp != nil { return resp } var dbVolume *db.StorageVolume err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, volumeName, true) return err }) if err != nil { return response.SmartError(err) } rj := jmap.Map{} err = json.NewDecoder(r.Body).Decode(&rj) if err != nil { return response.InternalError(err) } expiry, _ := rj.GetString("expires_at") if expiry == "" { // Disable expiration by setting it to zero time. rj["expires_at"] = time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC) } // Create body with correct expiry. body, err := json.Marshal(rj) if err != nil { return response.InternalError(err) } req := api.StorageVolumeBackupsPost{} err = json.Unmarshal(body, &req) if err != nil { return response.BadRequest(err) } direct := r.Header.Get("Accept") == "application/octet-stream" if direct && req.Target != nil { return response.BadRequest(errors.New("application/octet-stream is not a valid content type when a target is defined")) } if req.CompressionAlgorithm != "" { err := validate.IsCompressionAlgorithm(req.CompressionAlgorithm) if err != nil { return response.BadRequest(err) } } var reader *io.PipeReader var writer *io.PipeWriter var fullName string if direct || req.Target != nil { if req.Name != "" { if direct { return response.BadRequest(errors.New("No backup name can be set when requesting a direct backup with Accept: application/octet-stream")) } return response.BadRequest(errors.New("No backup name can be set when setting a backup target")) } reader, writer = io.Pipe() } else { if req.Name == "" { var backups []string // come up with a name. err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { backups, err = tx.GetStoragePoolVolumeBackupsNames(ctx, projectName, volumeName, poolID) return err }) if err != nil { return response.BadRequest(err) } base := volumeName + internalInstance.SnapshotDelimiter + "backup" length := len(base) backupID := 0 for _, backup := range backups { // Ignore backups not containing base. if !strings.HasPrefix(backup, base) { continue } substr := backup[length:] var num int count, err := fmt.Sscanf(substr, "%d", &num) if err != nil || count != 1 { continue } if num >= backupID { backupID = num + 1 } } req.Name = fmt.Sprintf("backup%d", backupID) } // Quick checks. err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid storage volume backup name: %w", err)) } fullName = volumeName + internalInstance.SnapshotDelimiter + req.Name } backup := func(op *operations.Operation) error { args := db.StoragePoolVolumeBackup{ Name: fullName, VolumeID: dbVolume.ID, CreationDate: time.Now(), VolumeOnly: req.VolumeOnly, OptimizedStorage: req.OptimizedStorage, CompressionAlgorithm: req.CompressionAlgorithm, } if !direct && req.Target == nil { args.ExpiryDate = req.ExpiresAt } uploadRes := make(chan error) // Start the upload in the background if requested. if req.Target != nil { go func(resCh chan<- error) { resCh <- internalBackup.Upload(reader, req.Target) }(uploadRes) } // Create the backup. err := volumeBackupCreate(s, args, projectName, poolName, volumeName, writer) if err != nil { // If we receive a pipe closed error, we first check for an explicit error returned by the // reader. if errors.Is(err, io.ErrClosedPipe) { select { case readerErr := <-uploadRes: err = readerErr default: } } // In order to actually fail piped exports, we use a dirty trick where we close the reader. // This doesn't provide a clean error message in the case of direct backups, but it is a // convenient tradeoff ensuring that the client reports an error. _ = reader.Close() return err } s.Events.SendLifecycle(projectName, lifecycle.StorageVolumeBackupCreated.Event(poolName, volumeTypeName, args.Name, projectName, op.Requestor(), logger.Ctx{"type": volumeTypeName})) return nil } resources := map[string][]api.URL{} resources["storage_volumes"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", volumeTypeName, volumeName)} if !direct && req.Target == nil { resources["backups"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", volumeTypeName, volumeName, "backups", req.Name)} } op, err := operations.OperationCreate(s, request.ProjectParam(r), operations.OperationClassTask, operationtype.CustomVolumeBackupCreate, resources, nil, backup, nil, nil, r) if err != nil { return response.InternalError(err) } if direct { err = op.Start() if err != nil { return response.InternalError(err) } return response.PipeResponse(r, reader) } return operations.OperationResponse(op) } // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/backups/{backupName} storage storage_pool_volumes_type_backup_get // // Get the storage volume backup // // Gets a specific storage volume backup. // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: path // name: backupName // description: Backup name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Storage volume backup // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/StorageVolumeBackup" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeTypeCustomBackupGet(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage pool the volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get backup name. backupName, err := url.PathUnescape(mux.Vars(r)["backupName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if volumeType != db.StoragePoolVolumeTypeCustom { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), db.StoragePoolVolumeTypeCustom) if err != nil { return response.SmartError(err) } resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } resp = forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, volumeName, db.StoragePoolVolumeTypeCustom) if resp != nil { return resp } fullName := volumeName + internalInstance.SnapshotDelimiter + backupName entry, err := storagePoolVolumeBackupLoadByName(r.Context(), s, projectName, poolName, fullName) if err != nil { return response.SmartError(err) } return response.SyncResponse(true, entry.Render()) } // swagger:operation POST /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/backups/{backupName} storage storage_pool_volumes_type_backup_post // // Rename a storage volume backup // // Renames a storage volume backup. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: path // name: backupName // description: Backup name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: volume rename // description: Storage volume backup // required: true // schema: // $ref: "#/definitions/StorageVolumeSnapshotPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeTypeCustomBackupPost(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage pool the volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get backup name. backupName, err := url.PathUnescape(mux.Vars(r)["backupName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if volumeType != db.StoragePoolVolumeTypeCustom { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), db.StoragePoolVolumeTypeCustom) if err != nil { return response.SmartError(err) } resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } resp = forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, volumeName, db.StoragePoolVolumeTypeCustom) if resp != nil { return resp } req := api.StorageVolumeBackupPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid storage volume backup name: %w", err)) } oldName := volumeName + internalInstance.SnapshotDelimiter + backupName entry, err := storagePoolVolumeBackupLoadByName(r.Context(), s, projectName, poolName, oldName) if err != nil { return response.SmartError(err) } newName := volumeName + internalInstance.SnapshotDelimiter + req.Name rename := func(op *operations.Operation) error { err := entry.Rename(newName) if err != nil { return err } s.Events.SendLifecycle(projectName, lifecycle.StorageVolumeBackupRenamed.Event(poolName, volumeTypeName, newName, projectName, op.Requestor(), logger.Ctx{"old_name": oldName})) return nil } resources := map[string][]api.URL{} resources["storage_volumes"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", volumeTypeName, volumeName)} resources["backups"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", volumeTypeName, volumeName, "backups", oldName)} op, err := operations.OperationCreate(s, request.ProjectParam(r), operations.OperationClassTask, operationtype.CustomVolumeBackupRename, resources, nil, rename, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // swagger:operation DELETE /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/backups/{backupName} storage storage_pool_volumes_type_backup_delete // // Delete a storage volume backup // // Deletes a new storage volume backup. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: path // name: backupName // description: Backup name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeTypeCustomBackupDelete(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage pool the volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get backup name. backupName, err := url.PathUnescape(mux.Vars(r)["backupName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if volumeType != db.StoragePoolVolumeTypeCustom { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), db.StoragePoolVolumeTypeCustom) if err != nil { return response.SmartError(err) } resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } resp = forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, volumeName, db.StoragePoolVolumeTypeCustom) if resp != nil { return resp } fullName := volumeName + internalInstance.SnapshotDelimiter + backupName entry, err := storagePoolVolumeBackupLoadByName(r.Context(), s, projectName, poolName, fullName) if err != nil { return response.SmartError(err) } remove := func(op *operations.Operation) error { err := entry.Delete() if err != nil { return err } s.Events.SendLifecycle(projectName, lifecycle.StorageVolumeBackupDeleted.Event(poolName, volumeTypeName, fullName, projectName, op.Requestor(), nil)) return nil } resources := map[string][]api.URL{} resources["storage_volumes"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", volumeTypeName, volumeName)} resources["backups"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", volumeTypeName, volumeName, "backups", backupName)} op, err := operations.OperationCreate(s, request.ProjectParam(r), operations.OperationClassTask, operationtype.CustomVolumeBackupRemove, resources, nil, remove, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/backups/{backupName}/export storage storage_pool_volumes_type_backup_export_get // // Get the raw backup file // // Download the raw backup file from the server. // // --- // produces: // - application/octet-stream // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: path // name: backupName // description: Backup name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Raw backup data // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeTypeCustomBackupExportGet(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage pool the volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get backup name. backupName, err := url.PathUnescape(mux.Vars(r)["backupName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if volumeType != db.StoragePoolVolumeTypeCustom { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), db.StoragePoolVolumeTypeCustom) if err != nil { return response.SmartError(err) } resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } resp = forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, volumeName, db.StoragePoolVolumeTypeCustom) if resp != nil { return resp } fullName := volumeName + internalInstance.SnapshotDelimiter + backupName // Ensure the volume exists _, err = storagePoolVolumeBackupLoadByName(r.Context(), s, projectName, poolName, fullName) if err != nil { return response.SmartError(err) } ent := response.FileResponseEntry{ Path: internalUtil.VarPath("backups", "custom", poolName, project.StorageVolume(projectName, fullName)), } s.Events.SendLifecycle(projectName, lifecycle.StorageVolumeBackupRetrieved.Event(poolName, volumeTypeName, fullName, projectName, request.CreateRequestor(r), nil)) return response.FileResponse(r, []response.FileResponseEntry{ent}, nil) } incus-7.0.0/cmd/incusd/storage_volumes_bitmap.go000066400000000000000000000453531517523235500220120ustar00rootroot00000000000000package main import ( "encoding/json" "fmt" "net/http" "net/url" "slices" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" storagePools "github.com/lxc/incus/v7/internal/server/storage" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) var storagePoolVolumeTypeBitmapsCmd = APIEndpoint{ Path: "storage-pools/{poolName}/volumes/{type}/{volumeName}/bitmaps", Get: APIEndpointAction{Handler: storagePoolVolumeTypeBitmapsGet, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanView, "poolName", "type", "volumeName", "location")}, Post: APIEndpointAction{Handler: storagePoolVolumeTypeBitmapsPost, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanEdit, "poolName", "type", "volumeName", "location")}, } var storagePoolVolumeTypeBitmapCmd = APIEndpoint{ Path: "storage-pools/{poolName}/volumes/{type}/{volumeName}/bitmaps/{bitmapName}", Get: APIEndpointAction{Handler: storagePoolVolumeTypeBitmapGet, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanView, "poolName", "type", "volumeName", "location")}, Delete: APIEndpointAction{Handler: storagePoolVolumeTypeBitmapDelete, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanEdit, "poolName", "type", "volumeName", "location")}, } // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/bitmaps storage storage_pool_volume_type_bitmaps_get // // Get the storage volume dirty bitmaps // // Gets a specific storage volume bitmaps // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Storage volume bitmaps // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/storage-pools/shared/volumes/custom/foo/bitmaps/bitmap0", // "/1.0/storage-pools/shared/volumes/custom/foo/bitmaps/bitmap1" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/bitmaps?recursion=1 storage storage_pool_volume_type_bitmaps_get_recursion1 // // Get the storage volume dirty bitmaps // // Gets a specific storage volume bitmaps // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Storage volume bitmaps // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of storage volume bitmaps // items: // $ref: "#/definitions/StorageVolumeBitmap" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeTypeBitmapsGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) // Get the volume details. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeDBType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if !slices.Contains([]int{db.StoragePoolVolumeTypeVM, db.StoragePoolVolumeTypeCustom}, volumeDBType) { return response.BadRequest(fmt.Errorf("Unsupported storage volume type %q", volumeTypeName)) } resp := forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, volumeName, volumeDBType) if resp != nil { return resp } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } volumeType, err := storagePools.VolumeDBTypeToType(volumeDBType) if err != nil { return response.SmartError(err) } // Get the volume. dbVol, err := storagePools.VolumeDBGet(pool, projectName, volumeName, volumeType) if err != nil { return response.SmartError(err) } contentType, err := storagePools.VolumeContentTypeNameToContentType(dbVol.ContentType) if err != nil { return response.SmartError(err) } if contentType != db.StoragePoolVolumeContentTypeBlock { return response.BadRequest(fmt.Errorf("Only block volumes are supported")) } inst, deviceName, err := storagePools.InstanceByVolumeName(s, poolName, projectName, volumeName, volumeDBType) if err != nil { return response.SmartError(err) } if !inst.IsRunning() { return response.BadRequest(fmt.Errorf("Listing bitmaps requires the instance to be running")) } bitmaps, err := inst.GetBitmaps(deviceName) if err != nil { return response.SmartError(err) } recursion := localUtil.IsRecursionRequest(r) if recursion { return response.SyncResponse(true, bitmaps) } resultString := []string{} for _, bitmap := range bitmaps { bitmapURL := api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", volumeTypeName, volumeName, "bitmaps", bitmap.Name).String() resultString = append(resultString, bitmapURL) } return response.SyncResponse(true, resultString) } // swagger:operation POST /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/bitmaps storage storage_pool_volumes_type_bitmaps_post // // Create a storage volume bitmap // // Creates a new storage volume bitmap. // // --- // consumes: // - application/json // produces: // - application/json // - application/octet-stream // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: volume // description: Storage volume bitmap // required: true // schema: // $ref: "#/definitions/StorageVolumeBitmapsPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeTypeBitmapsPost(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) // Get the volume details. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeDBType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if !slices.Contains([]int{db.StoragePoolVolumeTypeVM, db.StoragePoolVolumeTypeCustom}, volumeDBType) { return response.BadRequest(fmt.Errorf("Unsupported storage volume type %q", volumeTypeName)) } resp := forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, volumeName, volumeDBType) if resp != nil { return resp } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } volumeType, err := storagePools.VolumeDBTypeToType(volumeDBType) if err != nil { return response.SmartError(err) } // Get the volume. dbVol, err := storagePools.VolumeDBGet(pool, projectName, volumeName, volumeType) if err != nil { return response.SmartError(err) } contentType, err := storagePools.VolumeContentTypeNameToContentType(dbVol.ContentType) if err != nil { return response.SmartError(err) } if contentType != db.StoragePoolVolumeContentTypeBlock { return response.BadRequest(fmt.Errorf("Only block volumes are supported")) } inst, deviceName, err := storagePools.InstanceByVolumeName(s, poolName, projectName, volumeName, volumeDBType) if err != nil { return response.SmartError(err) } if !inst.IsRunning() { return response.BadRequest(fmt.Errorf("Creating bitmaps requires the instance to be running")) } req := api.StorageVolumeBitmapsPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } err = inst.CreateBitmap([]string{deviceName}, req) if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/bitmaps/{bitmapName} storage storage_pool_volume_type_bitmap_get // // Get the storage volume dirty bitmap // // Gets a specific storage volume bitmap // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: path // name: bitmapName // description: Bitmap name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Storage volume bitmap // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/StorageVolumeBitmap" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeTypeBitmapGet(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) // Get the volume details. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } bitmapName, err := url.PathUnescape(mux.Vars(r)["bitmapName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeDBType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if !slices.Contains([]int{db.StoragePoolVolumeTypeVM, db.StoragePoolVolumeTypeCustom}, volumeDBType) { return response.BadRequest(fmt.Errorf("Unsupported storage volume type %q", volumeTypeName)) } resp := forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, volumeName, volumeDBType) if resp != nil { return resp } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } volumeType, err := storagePools.VolumeDBTypeToType(volumeDBType) if err != nil { return response.SmartError(err) } // Get the volume. dbVol, err := storagePools.VolumeDBGet(pool, projectName, volumeName, volumeType) if err != nil { return response.SmartError(err) } contentType, err := storagePools.VolumeContentTypeNameToContentType(dbVol.ContentType) if err != nil { return response.SmartError(err) } if contentType != db.StoragePoolVolumeContentTypeBlock { return response.BadRequest(fmt.Errorf("Only block volumes are supported")) } inst, deviceName, err := storagePools.InstanceByVolumeName(s, poolName, projectName, volumeName, volumeDBType) if err != nil { return response.SmartError(err) } if !inst.IsRunning() { return response.BadRequest(fmt.Errorf("Listing bitmap requires the instance to be running")) } bitmaps, err := inst.GetBitmaps(deviceName) if err != nil { return response.SmartError(err) } for _, b := range bitmaps { if b.Name == bitmapName { return response.SyncResponse(true, b) } } return response.BadRequest(fmt.Errorf("Bitmap %q not found", bitmapName)) } // swagger:operation DELETE /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/bitmaps/{bitmapName} storage storage_pool_volumes_type_bitmap_delete // // Delete a storage volume bitmap // // Deletes a storage volume bitmap. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: path // name: bitmapName // description: Bitmap name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeTypeBitmapDelete(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) // Get the volume details. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } bitmapName, err := url.PathUnescape(mux.Vars(r)["bitmapName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeDBType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if !slices.Contains([]int{db.StoragePoolVolumeTypeVM, db.StoragePoolVolumeTypeCustom}, volumeDBType) { return response.BadRequest(fmt.Errorf("Unsupported storage volume type %q", volumeTypeName)) } resp := forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, volumeName, volumeDBType) if resp != nil { return resp } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } volumeType, err := storagePools.VolumeDBTypeToType(volumeDBType) if err != nil { return response.SmartError(err) } // Get the volume. dbVol, err := storagePools.VolumeDBGet(pool, projectName, volumeName, volumeType) if err != nil { return response.SmartError(err) } contentType, err := storagePools.VolumeContentTypeNameToContentType(dbVol.ContentType) if err != nil { return response.SmartError(err) } if contentType != db.StoragePoolVolumeContentTypeBlock { return response.BadRequest(fmt.Errorf("Only block volumes are supported")) } inst, deviceName, err := storagePools.InstanceByVolumeName(s, poolName, projectName, volumeName, volumeDBType) if err != nil { return response.SmartError(err) } if !inst.IsRunning() { return response.BadRequest(fmt.Errorf("Deleting bitmaps requires the instance to be running")) } err = inst.DeleteBitmap(deviceName, bitmapName) if err != nil { return response.SmartError(err) } return response.EmptySyncResponse } incus-7.0.0/cmd/incusd/storage_volumes_file.go000066400000000000000000000274461517523235500214600ustar00rootroot00000000000000package main import ( "errors" "fmt" "net/http" "net/url" "slices" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" ) func storagePoolVolumeTypeFileHandler(d *Daemon, r *http.Request) response.Response { s := d.State() volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage pool the volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if !slices.Contains(supportedVolumeTypes, volumeType) { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } requestProjectName := request.ProjectParam(r) volumeProjectName, err := project.StorageVolumeProject(s.DB.Cluster, requestProjectName, volumeType) if err != nil { return response.SmartError(err) } // Redirect to correct server if needed. resp := forwardedResponseIfVolumeIsRemote(s, r, poolName, volumeProjectName, volumeName, volumeType) if resp != nil { return resp } if resp != nil { return resp } // Load the storage volume. pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } volumeDB, err := storagePools.VolumeDBGet(pool, volumeProjectName, volumeName, storageDrivers.VolumeTypeCustom) if err != nil { return response.SmartError(err) } diskVolName := project.StorageVolume(volumeProjectName, volumeName) vol := pool.GetVolume(storageDrivers.VolumeTypeCustom, storageDrivers.ContentTypeFS, diskVolName, volumeDB.Config) // Parse the path. path := r.FormValue("path") if path == "" { return response.BadRequest(errors.New("Missing path argument")) } switch r.Method { case "GET": return storageVolumeFileGet(s, vol, volumeProjectName, path, r) case "HEAD": return storageVolumeFileHead(s, vol, volumeProjectName, path, r) case "DELETE": return storageVolumeFileDelete(s, vol, volumeProjectName, path, r) case "POST": return storageVolumeFilePost(s, vol, volumeProjectName, path, r) default: return response.NotFound(fmt.Errorf("Method %q not found", r.Method)) } } // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/files storage storage_pool_volume_type_files_get // // Get a file // // Gets the file content. If it's a directory, a json list of files will be returned instead. // // --- // produces: // - application/json // - application/octet-stream // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: path // description: Path to the file // type: string // example: default // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Raw file or directory listing // headers: // X-Incus-uid: // description: File owner UID // schema: // type: integer // X-Incus-gid: // description: File owner GID // schema: // type: integer // X-Incus-mode: // description: Mode mask // schema: // type: integer // X-Incus-modified: // description: Last modified date // schema: // type: string // X-Incus-type: // description: Type of file (file, symlink or directory) // schema: // type: string // content: // application/octet-stream: // schema: // type: string // example: some-text // application/json: // schema: // type: array // items: // type: string // example: |- // [ // "/etc", // "/home" // ] // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func storageVolumeFileGet(s *state.State, vol storageDrivers.Volume, volumeProjectName string, path string, r *http.Request) response.Response { reverter := revert.New() defer reverter.Fail() client, err := vol.FileSFTP(s) if err != nil { return response.SmartError(err) } reverter.Add(func() { _ = client.Close() }) return fileSFTPGet(client, path, r, reverter, func() { s.Events.SendLifecycle(volumeProjectName, lifecycle.StorageVolumeFileRetrieved.Event(vol, string(vol.Type()), volumeProjectName, nil, logger.Ctx{"path": path})) }) } // swagger:operation HEAD /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/files storage storage_pool_volume_type_files_head // // Get metadata for a file // // Gets the file or directory metadata. // // --- // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: path // description: Path to the file // type: string // example: default // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Raw file or directory listing // headers: // X-Incus-uid: // description: File owner UID // schema: // type: integer // X-Incus-gid: // description: File owner GID // schema: // type: integer // X-Incus-mode: // description: Mode mask // schema: // type: integer // X-Incus-modified: // description: Last modified date // schema: // type: string // X-Incus-type: // description: Type of file (file, symlink or directory) // schema: // type: string // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func storageVolumeFileHead(s *state.State, vol storageDrivers.Volume, volumeProjectName string, path string, _ *http.Request) response.Response { reverter := revert.New() defer reverter.Fail() client, err := vol.FileSFTP(s) if err != nil { return response.SmartError(err) } reverter.Add(func() { _ = client.Close() }) return fileSFTPHead(client, path) } // swagger:operation POST /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/files storage storage_pool_volume_type_files_post // // Create or replace a file // // Creates a new file in the storage volume. // // --- // consumes: // - application/octet-stream // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: path // description: Path to the file // type: string // example: default // - in: query // name: project // description: Project name // type: string // example: default // - in: body // name: raw_file // description: Raw file content // - in: header // name: X-Incus-uid // description: File owner UID // schema: // type: integer // example: 1000 // - in: header // name: X-Incus-gid // description: File owner GID // schema: // type: integer // example: 1000 // - in: header // name: X-Incus-mode // description: File mode // schema: // type: integer // example: 0644 // - in: header // name: X-Incus-type // description: Type of file (file, symlink or directory) // schema: // type: string // example: file // - in: header // name: X-Incus-write // description: Write mode (overwrite or append) // schema: // type: string // example: overwrite // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func storageVolumeFilePost(s *state.State, vol storageDrivers.Volume, volumeProjectName string, path string, r *http.Request) response.Response { client, err := vol.FileSFTP(s) if err != nil { return response.SmartError(err) } defer func() { _ = client.Close() }() return fileSFTPPost(client, path, r, func() { s.Events.SendLifecycle(volumeProjectName, lifecycle.StorageVolumeFilePushed.Event(vol, string(vol.Type()), volumeProjectName, nil, logger.Ctx{"path": path})) }) } // swagger:operation DELETE /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/files storage storage_pool_volume_type_files_delete // // Delete a file // // Removes the file. // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: path // description: Path to the file // type: string // example: default // - in: query // name: project // description: Project name // type: string // example: default // - in: header // name: X-Incus-force // description: Perform recursive deletion // schema: // type: boolean // example: true // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func storageVolumeFileDelete(s *state.State, vol storageDrivers.Volume, volumeProjectName string, path string, r *http.Request) response.Response { client, err := vol.FileSFTP(s) if err != nil { return response.SmartError(err) } defer func() { _ = client.Close() }() return fileSFTPDelete(client, path, r, func() { s.Events.SendLifecycle(volumeProjectName, lifecycle.StorageVolumeFileDeleted.Event(vol, string(vol.Type()), volumeProjectName, nil, logger.Ctx{"path": path})) }) } incus-7.0.0/cmd/incusd/storage_volumes_nbd.go000066400000000000000000000101751517523235500212730ustar00rootroot00000000000000package main import ( "fmt" "net/http" "net/url" "slices" "github.com/gorilla/mux" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" storagePools "github.com/lxc/incus/v7/internal/server/storage" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/nbd storage storage_pool_volume_type_nbd_get // // Get the storage volume NBD connection // // Upgrades the request to an NBD connection of the storage volume's block device. // // --- // produces: // - application/json // - application/octet-stream // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // responses: // "101": // description: Switching protocols to NBD // "400": // $ref: "#/responses/BadRequest" // "404": // $ref: "#/responses/NotFound" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeTypeNBDHandler(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) writable := util.IsTrue(request.QueryParam(r, "writable")) if r.Header.Get("Upgrade") != "nbd" { return response.SmartError(api.StatusErrorf(http.StatusBadRequest, "Missing or invalid upgrade header")) } // Get the volume details. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if !slices.Contains([]int{db.StoragePoolVolumeTypeVM, db.StoragePoolVolumeTypeCustom}, volumeType) { return response.BadRequest(fmt.Errorf("Unsupported storage volume type %q", volumeTypeName)) } // Determine the relevant project. volumeProjectName, err := project.StorageVolumeProject(s.DB.Cluster, projectName, volumeType) if err != nil { return response.SmartError(err) } // Forward the request if the instance is remote. c, err := cluster.ConnectIfVolumeIsRemote(s, poolName, volumeProjectName, volumeName, volumeType, s.Endpoints.NetworkCert(), s.ServerCert(), r) if err != nil { return response.SmartError(err) } if c != nil { conn, err := c.GetStoragePoolVolumeBlockNBDConn(poolName, volumeTypeName, volumeName, incus.StorageVolumeNBDPost{Writable: writable}) if err != nil { return response.SmartError(err) } return response.UpgradeResponse(r, conn, "nbd", nil) } // Local requests. pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } // Special handling for instances. if volumeType == db.StoragePoolVolumeTypeVM { inst, err := instance.LoadByProjectAndName(s, volumeProjectName, volumeName) if err != nil { return response.SmartError(err) } conn, disconnect, err := pool.GetInstanceNBD(inst, writable) if err != nil { return response.SmartError(err) } return response.UpgradeResponse(r, conn, "nbd", disconnect) } // Handle custom volumes. conn, disconnect, err := pool.GetCustomVolumeNBD(volumeProjectName, volumeName, writable) if err != nil { return response.SmartError(err) } return response.UpgradeResponse(r, conn, "nbd", disconnect) } incus-7.0.0/cmd/incusd/storage_volumes_sftp.go000066400000000000000000000075271517523235500215130ustar00rootroot00000000000000package main import ( "fmt" "net" "net/http" "net/url" "slices" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/shared/api" ) // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/sftp storage storage_pool_volume_type_sftp_get // // Get the storage volume SFTP connection // // Upgrades the request to an SFTP connection of the storage volume's filesystem. // // --- // produces: // - application/json // - application/octet-stream // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // responses: // "101": // description: Switching protocols to SFTP // "400": // $ref: "#/responses/BadRequest" // "404": // $ref: "#/responses/NotFound" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeTypeSFTPHandler(d *Daemon, r *http.Request) response.Response { s := d.State() projectName := request.ProjectParam(r) if r.Header.Get("Upgrade") != "sftp" { return response.SmartError(api.StatusErrorf(http.StatusBadRequest, "Missing or invalid upgrade header")) } // Get the volume details. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if !slices.Contains(supportedVolumeTypes, volumeType) { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } volumeProjectName, err := project.StorageVolumeProject(s.DB.Cluster, projectName, volumeType) if err != nil { return response.SmartError(err) } // Redirect to correct server if needed. var conn net.Conn // Forward the request if the instance is remote. client, err := cluster.ConnectIfVolumeIsRemote(s, poolName, volumeProjectName, volumeName, volumeType, s.Endpoints.NetworkCert(), s.ServerCert(), r) if err != nil { return response.SmartError(err) } if client != nil { conn, err = client.GetStoragePoolVolumeFileSFTPConn(poolName, volumeTypeName, volumeName) if err != nil { return response.SmartError(err) } } else { pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } volumeDB, err := storagePools.VolumeDBGet(pool, volumeProjectName, volumeName, storageDrivers.VolumeTypeCustom) if err != nil { return response.SmartError(err) } diskVolName := project.StorageVolume(volumeProjectName, volumeName) vol := pool.GetVolume(storageDrivers.VolumeTypeCustom, storageDrivers.ContentTypeFS, diskVolName, volumeDB.Config) conn, err = vol.FileSFTPConn(d.State()) if err != nil { return response.SmartError(api.StatusErrorf(http.StatusInternalServerError, "Failed getting storage volume SFTP connection: %v", err)) } } return response.UpgradeResponse(r, conn, "sftp", nil) } incus-7.0.0/cmd/incusd/storage_volumes_snapshot.go000066400000000000000000001456361517523235500224020ustar00rootroot00000000000000package main import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/url" "slices" "strconv" "strings" "sync" "time" "github.com/flosch/pongo2/v6" "github.com/gorilla/mux" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" "github.com/lxc/incus/v7/internal/server/task" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var storagePoolVolumeSnapshotsTypeCmd = APIEndpoint{ Path: "storage-pools/{poolName}/volumes/{type}/{volumeName}/snapshots", Get: APIEndpointAction{Handler: storagePoolVolumeSnapshotsTypeGet, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanView, "poolName", "type", "volumeName", "location")}, Post: APIEndpointAction{Handler: storagePoolVolumeSnapshotsTypePost, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanManageSnapshots, "poolName", "type", "volumeName", "location")}, } var storagePoolVolumeSnapshotTypeCmd = APIEndpoint{ Path: "storage-pools/{poolName}/volumes/{type}/{volumeName}/snapshots/{snapshotName}", Delete: APIEndpointAction{Handler: storagePoolVolumeSnapshotTypeDelete, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanManageSnapshots, "poolName", "type", "volumeName", "location")}, Get: APIEndpointAction{Handler: storagePoolVolumeSnapshotTypeGet, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanView, "poolName", "type", "volumeName", "location")}, Post: APIEndpointAction{Handler: storagePoolVolumeSnapshotTypePost, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanManageSnapshots, "poolName", "type", "volumeName", "location")}, Patch: APIEndpointAction{Handler: storagePoolVolumeSnapshotTypePatch, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanManageSnapshots, "poolName", "type", "volumeName", "location")}, Put: APIEndpointAction{Handler: storagePoolVolumeSnapshotTypePut, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanManageSnapshots, "poolName", "type", "volumeName", "location")}, } // swagger:operation POST /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/snapshots storage storage_pool_volumes_type_snapshots_post // // Create a storage volume snapshot // // Creates a new storage volume snapshot. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: volume // description: Storage volume snapshot // required: true // schema: // $ref: "#/definitions/StorageVolumeSnapshotsPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeSnapshotsTypePost(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the pool. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the name of the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get the name of the volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if volumeType != db.StoragePoolVolumeTypeCustom { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } // Get the project name. projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), volumeType) if err != nil { return response.SmartError(err) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := dbCluster.GetProject(context.Background(), tx.Tx(), projectName) if err != nil { return err } p, err := dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return err } err = project.AllowSnapshotCreation(p) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // Forward if needed. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } resp = forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, volumeName, volumeType) if resp != nil { return resp } // Parse the request. req := api.StorageVolumeSnapshotsPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Check that this isn't a restricted volume used, err := storagePools.VolumeUsedByDaemon(s, poolName, volumeName) if err != nil { return response.InternalError(err) } if used { return response.BadRequest(errors.New("Volumes used by Incus itself cannot have snapshots")) } // Retrieve the storage pool (and check if the storage pool exists). pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } // Get the parent volume. var parentDBVolume *db.StorageVolume err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Get the parent volume so we can get the config. parentDBVolume, err = tx.GetStoragePoolVolume(ctx, pool.ID(), projectName, volumeType, volumeName, true) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } if util.IsTrue(parentDBVolume.Config["dependent"]) { return response.BadRequest(fmt.Errorf("Direct snapshots are not allowed for dependent volumes")) } // Get the snapshot pattern. pattern := parentDBVolume.Config["snapshots.pattern"] if pattern == "" { pattern = "snap%d" } renderedPattern, err := internalUtil.RenderTemplate(pattern, pongo2.Context{ "creation_date": time.Now(), }) if err != nil { return response.InternalError(err) } // Get a snapshot name. if req.Name == "" && strings.Count(renderedPattern, "%d") == 1 { var i int _ = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { i = tx.GetNextStorageVolumeSnapshotIndex(ctx, poolName, volumeName, volumeType, renderedPattern) return nil }) req.Name = fmt.Sprintf(renderedPattern, i) } else if req.Name == "" && renderedPattern != pattern { req.Name = renderedPattern } else if req.Name != "" { // Make sure the snapshot doesn't already exist. err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { snapDBVolume, err := tx.GetStoragePoolVolume(ctx, pool.ID(), projectName, volumeType, fmt.Sprintf("%s/%s", volumeName, req.Name), true) if err != nil && !response.IsNotFoundError(err) { return err } else if snapDBVolume != nil { return api.StatusErrorf(http.StatusConflict, "Snapshot %q already in use", req.Name) } return nil }) if err != nil { return response.SmartError(err) } } else { return response.BadRequest(errors.New("Couldn't determine snapshot name")) } // Quick checks. err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid storage volume snapshot name: %w", err)) } // Fill in the expiry. var expiry time.Time if req.ExpiresAt != nil { expiry = *req.ExpiresAt } else { duration := parentDBVolume.Config["snapshots.expiry.manual"] if duration == "" { duration = parentDBVolume.Config["snapshots.expiry"] } expiry, err = internalInstance.GetExpiry(time.Now(), duration) if err != nil { return response.BadRequest(err) } } // Create the snapshot. snapshot := func(op *operations.Operation) error { return pool.CreateCustomVolumeSnapshot(projectName, volumeName, req.Name, expiry, false, op) } resources := map[string][]api.URL{} resources["storage_volumes"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", volumeTypeName, volumeName)} resources["storage_volume_snapshots"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", volumeTypeName, volumeName, "snapshots", req.Name)} op, err := operations.OperationCreate(s, request.ProjectParam(r), operations.OperationClassTask, operationtype.VolumeSnapshotCreate, resources, nil, snapshot, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/snapshots storage storage_pool_volumes_type_snapshots_get // // Get the storage volume snapshots // // Returns a list of storage volume snapshots (URLs). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/storage-pools/local/volumes/custom/foo/snapshots/snap0", // "/1.0/storage-pools/local/volumes/custom/foo/snapshots/snap1" // ] // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/snapshots?recursion=1 storage storage_pool_volumes_type_snapshots_get_recursion1 // // Get the storage volume snapshots // // Returns a list of storage volume snapshots (structs). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of storage volume snapshots // items: // $ref: "#/definitions/StorageVolumeSnapshot" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeSnapshotsTypeGet(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the pool the storage volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } recursion := localUtil.IsRecursionRequest(r) // Get the name of the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get the name of the volume type. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if !slices.Contains(supportedVolumeTypes, volumeType) { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), volumeType) if err != nil { return response.SmartError(err) } var poolID int64 var volumes []db.StorageVolumeArgs // Forward if needed. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } resp = forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, volumeName, volumeType) if resp != nil { return resp } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Retrieve ID of the storage pool (and check if the storage pool exists). poolID, err = tx.GetStoragePoolID(ctx, poolName) if err != nil { return err } // Get the names of all storage volume snapshots of a given volume. volumes, err = tx.GetLocalStoragePoolVolumeSnapshotsWithType(ctx, projectName, volumeName, volumeType, poolID) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // Prepare the response. resultString := []string{} resultMap := []*api.StorageVolumeSnapshot{} for _, volume := range volumes { _, snapshotName, _ := api.GetParentAndSnapshotName(volume.Name) if !recursion { resultString = append(resultString, fmt.Sprintf("/%s/storage-pools/%s/volumes/%s/%s/snapshots/%s", version.APIVersion, poolName, volumeTypeName, volumeName, snapshotName)) } else { var vol *db.StorageVolume err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { vol, err = tx.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, volume.Name, true) return err }) if err != nil { return response.SmartError(err) } volumeUsedBy, err := storagePoolVolumeUsedByGet(s, projectName, poolName, vol) if err != nil { return response.SmartError(err) } vol.UsedBy = project.FilterUsedBy(s.Authorizer, r, volumeUsedBy) tmp := &api.StorageVolumeSnapshot{} tmp.Config = vol.Config tmp.Description = vol.Description tmp.Name = vol.Name tmp.CreatedAt = vol.CreatedAt expiryDate := volume.ExpiryDate if expiryDate.Unix() > 0 { tmp.ExpiresAt = &expiryDate } resultMap = append(resultMap, tmp) } } if !recursion { return response.SyncResponse(true, resultString) } return response.SyncResponse(true, resultMap) } // swagger:operation POST /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/snapshots/{snapshotName} storage storage_pool_volumes_type_snapshot_post // // Rename a storage volume snapshot // // Renames a storage volume snapshot. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: path // name: snapshotName // description: Snapshot name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: volume rename // description: Storage volume snapshot // required: true // schema: // $ref: "#/definitions/StorageVolumeSnapshotPost" // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeSnapshotTypePost(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the storage pool the volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the name of the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage volume. snapshotName, err := url.PathUnescape(mux.Vars(r)["snapshotName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if volumeType != db.StoragePoolVolumeTypeCustom { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } // Get the project name. projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), volumeType) if err != nil { return response.SmartError(err) } // Forward if needed. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } fullSnapshotName := fmt.Sprintf("%s/%s", volumeName, snapshotName) resp = forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, volumeName, volumeType) if resp != nil { return resp } // Parse the request. req := api.StorageVolumeSnapshotPost{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Quick checks. err = validate.IsAPIName(req.Name, false) if err != nil { return response.BadRequest(fmt.Errorf("Invalid storage volume snapshot name: %w", err)) } // This is a migration request so send back requested secrets. if req.Migration { req := api.StorageVolumePost{ Name: req.Name, Target: req.Target, } return storagePoolVolumeTypePostMigration(s, r, request.ProjectParam(r), projectName, poolName, fullSnapshotName, req) } // Rename the snapshot. snapshotRename := func(op *operations.Operation) error { pool, err := storagePools.LoadByName(s, poolName) if err != nil { return err } return pool.RenameCustomVolumeSnapshot(projectName, fullSnapshotName, req.Name, op) } resources := map[string][]api.URL{} resources["storage_volume_snapshots"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", volumeTypeName, volumeName, "snapshots", snapshotName)} op, err := operations.OperationCreate(s, request.ProjectParam(r), operations.OperationClassTask, operationtype.VolumeSnapshotRename, resources, nil, snapshotRename, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/snapshots/{snapshotName} storage storage_pool_volumes_type_snapshot_get // // Get the storage volume snapshot // // Gets a specific storage volume snapshot. // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: path // name: snapshotName // description: Snapshot name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Storage volume snapshot // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/StorageVolumeSnapshot" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeSnapshotTypeGet(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the storage pool the volume is supposed to be // attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the name of the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage volume. snapshotName, err := url.PathUnescape(mux.Vars(r)["snapshotName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Get the project name. projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), volumeType) if err != nil { return response.SmartError(err) } // Forward if needed. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } fullSnapshotName := fmt.Sprintf("%s/%s", volumeName, snapshotName) resp = forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, fullSnapshotName, volumeType) if resp != nil { return resp } var poolID int64 var dbVolume *db.StorageVolume var expiry time.Time err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Get the snapshot. poolID, _, _, err = tx.GetStoragePool(ctx, poolName) if err != nil { return err } dbVolume, err = tx.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, fullSnapshotName, true) if err != nil { return err } expiry, err = tx.GetStorageVolumeSnapshotExpiry(ctx, dbVolume.ID) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } snapshot := api.StorageVolumeSnapshot{} snapshot.Config = dbVolume.Config snapshot.Description = dbVolume.Description snapshot.Name = snapshotName snapshot.ExpiresAt = &expiry snapshot.ContentType = dbVolume.ContentType snapshot.CreatedAt = dbVolume.CreatedAt etag := []any{snapshot.Description, expiry} return response.SyncResponseETag(true, &snapshot, etag) } // swagger:operation PUT /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/snapshots/{snapshotName} storage storage_pool_volumes_type_snapshot_put // // Update the storage volume snapshot // // Updates the entire storage volume snapshot configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: path // name: snapshotName // description: Snapshot name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: storage volume snapshot // description: Storage volume snapshot configuration // required: true // schema: // $ref: "#/definitions/StorageVolumeSnapshotPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeSnapshotTypePut(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the storage pool the volume is supposed to be // attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the name of the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage volume. snapshotName, err := url.PathUnescape(mux.Vars(r)["snapshotName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Get the project name. projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), volumeType) if err != nil { return response.SmartError(err) } // Forward if needed. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } fullSnapshotName := fmt.Sprintf("%s/%s", volumeName, snapshotName) resp = forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, fullSnapshotName, volumeType) if resp != nil { return resp } var poolID int64 var dbVolume *db.StorageVolume var expiry time.Time err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Get the snapshot. poolID, _, _, err = tx.GetStoragePool(ctx, poolName) if err != nil { return err } dbVolume, err = tx.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, fullSnapshotName, true) if err != nil { return err } expiry, err = tx.GetStorageVolumeSnapshotExpiry(ctx, dbVolume.ID) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // Validate the ETag etag := []any{dbVolume.Description, expiry} err = localUtil.EtagCheck(r, etag) if err != nil { return response.PreconditionFailed(err) } req := api.StorageVolumeSnapshotPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } return doStoragePoolVolumeSnapshotUpdate(s, r, poolName, projectName, dbVolume.Name, volumeType, req) } // swagger:operation PATCH /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/snapshots/{snapshotName} storage storage_pool_volumes_type_snapshot_patch // // Partially update the storage volume snapshot // // Updates a subset of the storage volume snapshot configuration. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: path // name: snapshotName // description: Snapshot name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // - in: body // name: storage volume snapshot // description: Storage volume snapshot configuration // required: true // schema: // $ref: "#/definitions/StorageVolumeSnapshotPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "412": // $ref: "#/responses/PreconditionFailed" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeSnapshotTypePatch(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the storage pool the volume is supposed to be // attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the name of the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage volume. snapshotName, err := url.PathUnescape(mux.Vars(r)["snapshotName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Get the project name. projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), volumeType) if err != nil { return response.SmartError(err) } // Forward if needed. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } fullSnapshotName := fmt.Sprintf("%s/%s", volumeName, snapshotName) resp = forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, fullSnapshotName, volumeType) if resp != nil { return resp } var poolID int64 var dbVolume *db.StorageVolume var expiry time.Time err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { // Get the snapshot. poolID, _, _, err = tx.GetStoragePool(ctx, poolName) if err != nil { return err } dbVolume, err = tx.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, fullSnapshotName, true) if err != nil { return err } expiry, err = tx.GetStorageVolumeSnapshotExpiry(ctx, dbVolume.ID) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } // Validate the ETag etag := []any{dbVolume.Description, expiry} err = localUtil.EtagCheck(r, etag) if err != nil { return response.PreconditionFailed(err) } req := api.StorageVolumeSnapshotPut{ Description: dbVolume.Description, ExpiresAt: &expiry, } err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } return doStoragePoolVolumeSnapshotUpdate(s, r, poolName, projectName, dbVolume.Name, volumeType, req) } func doStoragePoolVolumeSnapshotUpdate(s *state.State, r *http.Request, poolName string, projectName string, volName string, volumeType int, req api.StorageVolumeSnapshotPut) response.Response { expiry := time.Time{} if req.ExpiresAt != nil { expiry = *req.ExpiresAt } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } // Use an empty operation for this sync response to pass the requestor op := &operations.Operation{} op.SetRequestor(r) // Update the database. if volumeType == db.StoragePoolVolumeTypeCustom { err = pool.UpdateCustomVolumeSnapshot(projectName, volName, req.Description, nil, expiry, op) if err != nil { return response.SmartError(err) } } else { inst, err := instance.LoadByProjectAndName(s, projectName, volName) if err != nil { return response.SmartError(err) } err = pool.UpdateInstanceSnapshot(inst, req.Description, nil, op) if err != nil { return response.SmartError(err) } } return response.EmptySyncResponse } // swagger:operation DELETE /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/snapshots/{snapshotName} storage storage_pool_volumes_type_snapshot_delete // // Delete a storage volume snapshot // // Deletes a new storage volume snapshot. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: path // name: snapshotName // description: Snapshot name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "202": // $ref: "#/responses/Operation" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeSnapshotTypeDelete(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the storage pool the volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the name of the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get the name of the storage volume. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Get the name of the storage volume. snapshotName, err := url.PathUnescape(mux.Vars(r)["snapshotName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if volumeType != db.StoragePoolVolumeTypeCustom { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } // Get the project name. projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), volumeType) if err != nil { return response.SmartError(err) } // Forward if needed. resp := forwardedResponseIfTargetIsRemote(s, r) if resp != nil { return resp } fullSnapshotName := fmt.Sprintf("%s/%s", volumeName, snapshotName) resp = forwardedResponseIfVolumeIsRemote(s, r, poolName, projectName, fullSnapshotName, volumeType) if resp != nil { return resp } pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } // Get the parent volume. var parentDBVolume *db.StorageVolume err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { parentDBVolume, err = tx.GetStoragePoolVolume(ctx, pool.ID(), projectName, volumeType, volumeName, true) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } if util.IsTrue(parentDBVolume.Config["dependent"]) { return response.BadRequest(fmt.Errorf("Direct snapshot removal is not allowed for dependent volumes")) } snapshotDelete := func(op *operations.Operation) error { return pool.DeleteCustomVolumeSnapshot(projectName, fullSnapshotName, op) } resources := map[string][]api.URL{} resources["storage_volume_snapshots"] = []api.URL{*api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", volumeTypeName, volumeName, "snapshots", snapshotName)} op, err := operations.OperationCreate(s, request.ProjectParam(r), operations.OperationClassTask, operationtype.VolumeSnapshotDelete, resources, nil, snapshotDelete, nil, nil, r) if err != nil { return response.InternalError(err) } return operations.OperationResponse(op) } func pruneExpiredAndAutoCreateCustomVolumeSnapshotsTask(d *Daemon) (task.Func, task.Schedule) { f := func(ctx context.Context) { s := d.State() var volumes, remoteVolumes, expiredSnapshots, expiredRemoteSnapshots []db.StorageVolumeArgs var memberCount int var onlineMemberIDs []int64 err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Get the list of expired custom volume snapshots for this member (or remote). allExpiredSnapshots, err := tx.GetExpiredStorageVolumeSnapshots(ctx, true) if err != nil { return fmt.Errorf("Failed getting expired custom volume snapshots: %w", err) } for _, v := range allExpiredSnapshots { if v.NodeID < 0 { // Keep a separate list of remote volumes in order to select a member to // perform the snapshot expiry on later. expiredRemoteSnapshots = append(expiredRemoteSnapshots, v) } else { logger.Debug("Scheduling local custom volume snapshot expiry", logger.Ctx{"volName": v.Name, "project": v.ProjectName, "pool": v.PoolName}) expiredSnapshots = append(expiredSnapshots, v) // Always include local volumes. } } projs, err := dbCluster.GetProjects(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading projects: %w", err) } // Key by project name for lookup later. projects := make(map[string]*api.Project, len(projs)) for _, p := range projs { projects[p.Name], err = p.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading project %q: %w", p.Name, err) } } allVolumes, err := tx.GetStoragePoolVolumesWithType(ctx, db.StoragePoolVolumeTypeCustom, true) if err != nil { return fmt.Errorf("Failed getting volumes for auto custom volume snapshot task: %w", err) } for _, v := range allVolumes { err = project.AllowSnapshotCreation(projects[v.ProjectName]) if err != nil { continue } schedule, ok := v.Config["snapshots.schedule"] if !ok || schedule == "" { continue } // Check if snapshot is scheduled. if !snapshotIsScheduledNow(schedule, v.ID) { continue } if v.NodeID < 0 { // Keep a separate list of remote volumes in order to select a member to // perform the snapshot later. remoteVolumes = append(remoteVolumes, v) } else { logger.Debug("Scheduling local auto custom volume snapshot", logger.Ctx{"volName": v.Name, "project": v.ProjectName, "pool": v.PoolName}) volumes = append(volumes, v) // Always include local volumes. } } if len(remoteVolumes) > 0 || len(expiredRemoteSnapshots) > 0 { // Get list of cluster members. members, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } memberCount = len(members) // Filter to online members. for _, member := range members { if member.IsOffline(s.GlobalConfig.OfflineThreshold()) { continue } onlineMemberIDs = append(onlineMemberIDs, member.ID) } return nil } return nil }) if err != nil { logger.Error("Failed getting custom volume info", logger.Ctx{"err": err}) return } localMemberID := s.DB.Cluster.GetNodeID() if len(expiredRemoteSnapshots) > 0 { // Skip expiring remote custom volume snapshots if there are no online members, as we can't // be sure that the cluster isn't partitioned and we may end up attempting to expire // snapshot on multiple members. if memberCount > 1 && len(onlineMemberIDs) <= 0 { logger.Error("Skipping remote volumes for expire custom volume snapshot task due to no online members") } else { for _, v := range expiredRemoteSnapshots { // If there are multiple cluster members, a stable random member is chosen // to perform the snapshot expiry. This avoids expiring the snapshot on // every member and spreads the load across the online cluster members. if memberCount > 1 { selectedMemberID, err := localUtil.GetStableRandomInt64FromList(int64(v.ID), onlineMemberIDs) if err != nil { logger.Error("Failed scheduling remote expire custom volume snapshot task", logger.Ctx{"volName": v.Name, "project": v.ProjectName, "pool": v.PoolName, "err": err}) continue } // Don't snapshot, if we're not the chosen one. if localMemberID != selectedMemberID { continue } } logger.Debug("Scheduling remote custom volume snapshot expiry", logger.Ctx{"volName": v.Name, "project": v.ProjectName, "pool": v.PoolName}) expiredSnapshots = append(expiredSnapshots, v) } } } if len(remoteVolumes) > 0 { // Skip snapshotting remote custom volumes if there are no online members, as we can't be // sure that the cluster isn't partitioned and we may end up attempting the snapshot on // multiple members. if memberCount > 1 && len(onlineMemberIDs) <= 0 { logger.Error("Skipping remote volumes for auto custom volume snapshot task due to no online members") } else { for _, v := range remoteVolumes { // If there are multiple cluster members, a stable random member is chosen // to perform the snapshot from. This avoids taking the snapshot on every // member and spreads the load taking the snapshots across the online // cluster members. if memberCount > 1 { selectedNodeID, err := localUtil.GetStableRandomInt64FromList(int64(v.ID), onlineMemberIDs) if err != nil { logger.Error("Failed scheduling remote auto custom volume snapshot task", logger.Ctx{"volName": v.Name, "project": v.ProjectName, "pool": v.PoolName, "err": err}) continue } // Don't snapshot, if we're not the chosen one. if localMemberID != selectedNodeID { continue } } logger.Debug("Scheduling remote auto custom volume snapshot", logger.Ctx{"volName": v.Name, "project": v.ProjectName, "pool": v.PoolName}) volumes = append(volumes, v) } } } // Handle snapshot expiry first before creating new ones to reduce the chances of running out of // disk space. if len(expiredSnapshots) > 0 { opRun := func(op *operations.Operation) error { return pruneExpiredCustomVolumeSnapshots(ctx, s, expiredSnapshots) } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.CustomVolumeSnapshotsExpire, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating expired custom volume snapshots prune operation", logger.Ctx{"err": err}) } else { logger.Info("Pruning expired custom volume snapshots") err = op.Start() if err != nil { logger.Error("Failed starting expired custom volume snapshots prune operation", logger.Ctx{"err": err}) } else { err = op.Wait(ctx) if err != nil { logger.Error("Failed pruning expired custom volume snapshots", logger.Ctx{"err": err}) } else { logger.Info("Done pruning expired custom volume snapshots") } } } } // Handle snapshot auto creation. if len(volumes) > 0 { opRun := func(op *operations.Operation) error { return autoCreateCustomVolumeSnapshots(ctx, s, volumes) } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.VolumeSnapshotCreate, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating scheduled volume snapshot operation", logger.Ctx{"err": err}) } else { logger.Info("Creating scheduled volume snapshots") err = op.Start() if err != nil { logger.Error("Failed starting scheduled volume snapshot operation", logger.Ctx{"err": err}) } else { err = op.Wait(ctx) if err != nil { logger.Error("Failed scheduled custom volume snapshots", logger.Ctx{"err": err}) } else { logger.Info("Done creating scheduled volume snapshots") } } } } } first := true schedule := func() (time.Duration, error) { interval := time.Minute if first { first = false return interval, task.ErrSkip } return interval, nil } return f, schedule } var customVolSnapshotsPruneRunning = sync.Map{} func pruneExpiredCustomVolumeSnapshots(ctx context.Context, s *state.State, expiredSnapshots []db.StorageVolumeArgs) error { for _, v := range expiredSnapshots { err := ctx.Err() if err != nil { return err // Stop if context is cancelled. } _, loaded := customVolSnapshotsPruneRunning.LoadOrStore(v.ID, struct{}{}) if loaded { continue // Deletion of this snapshot is already running, skip. } pool, err := storagePools.LoadByName(s, v.PoolName) if err != nil { customVolSnapshotsPruneRunning.Delete(v.ID) return fmt.Errorf("Error loading pool for volume snapshot %q (project %q, pool %q): %w", v.Name, v.ProjectName, v.PoolName, err) } err = pool.DeleteCustomVolumeSnapshot(v.ProjectName, v.Name, nil) customVolSnapshotsPruneRunning.Delete(v.ID) if err != nil { return fmt.Errorf("Error deleting custom volume snapshot %q (project %q, pool %q): %w", v.Name, v.ProjectName, v.PoolName, err) } } return nil } func autoCreateCustomVolumeSnapshots(ctx context.Context, s *state.State, volumes []db.StorageVolumeArgs) error { // Make the snapshots sequentially. for _, v := range volumes { err := ctx.Err() if err != nil { return err // Stop if context is cancelled. } snapshotName, err := volumeDetermineNextSnapshotName(ctx, s, v, "snap%d") if err != nil { return fmt.Errorf("Error retrieving next snapshot name for volume %q (project %q, pool %q): %w", v.Name, v.ProjectName, v.PoolName, err) } expiry, err := internalInstance.GetExpiry(time.Now(), v.Config["snapshots.expiry"]) if err != nil { return fmt.Errorf("Error getting snapshot expiry for volume %q (project %q, pool %q): %w", v.Name, v.ProjectName, v.PoolName, err) } pool, err := storagePools.LoadByName(s, v.PoolName) if err != nil { return fmt.Errorf("Error loading pool for volume %q (project %q, pool %q): %w", v.Name, v.ProjectName, v.PoolName, err) } err = pool.CreateCustomVolumeSnapshot(v.ProjectName, v.Name, snapshotName, expiry, false, nil) if err != nil { return fmt.Errorf("Error creating snapshot for volume %q (project %q, pool %q): %w", v.Name, v.ProjectName, v.PoolName, err) } } return nil } func volumeDetermineNextSnapshotName(ctx context.Context, s *state.State, volume db.StorageVolumeArgs, defaultPattern string) (string, error) { var err error pattern, ok := volume.Config["snapshots.pattern"] if !ok { pattern = defaultPattern } pattern, err = internalUtil.RenderTemplate(pattern, pongo2.Context{ "creation_date": time.Now(), }) if err != nil { return "", err } count := strings.Count(pattern, "%d") if count > 1 { return "", fmt.Errorf("Snapshot pattern may contain '%%d' only once") } else if count == 1 { var i int _ = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { i = tx.GetNextStorageVolumeSnapshotIndex(ctx, volume.PoolName, volume.Name, db.StoragePoolVolumeTypeCustom, pattern) return nil }) return strings.Replace(pattern, "%d", strconv.Itoa(i), 1), nil } snapshotExists := false var snapshots []db.StorageVolumeArgs var projects []string var pools []string err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { projects, err = dbCluster.GetProjectNames(ctx, tx.Tx()) if err != nil { return err } pools, err = tx.GetStoragePoolNames(ctx) if err != nil { return err } return nil }) if err != nil { return "", err } for _, pool := range pools { var poolID int64 err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { poolID, err = tx.GetStoragePoolID(ctx, pool) if err != nil { return err } for _, project := range projects { snaps, err := tx.GetLocalStoragePoolVolumeSnapshotsWithType(ctx, project, volume.Name, db.StoragePoolVolumeTypeCustom, poolID) if err != nil { return err } snapshots = append(snapshots, snaps...) } return nil }) if err != nil { return "", err } } for _, snap := range snapshots { _, snapOnlyName, _ := api.GetParentAndSnapshotName(snap.Name) if snapOnlyName == pattern { snapshotExists = true break } } if snapshotExists && count == 1 { var i int _ = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { i = tx.GetNextStorageVolumeSnapshotIndex(ctx, volume.PoolName, volume.Name, db.StoragePoolVolumeTypeCustom, pattern) return nil }) return strings.Replace(pattern, "%d", strconv.Itoa(i), 1), nil } else if snapshotExists { return "", errors.New("Snapshot with that name already exists") } return pattern, nil } incus-7.0.0/cmd/incusd/storage_volumes_state.go000066400000000000000000000117451517523235500216540ustar00rootroot00000000000000package main import ( "errors" "fmt" "net/http" "net/url" "slices" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/shared/api" ) var storagePoolVolumeTypeStateCmd = APIEndpoint{ Path: "storage-pools/{poolName}/volumes/{type}/{volumeName}/state", Get: APIEndpointAction{Handler: storagePoolVolumeTypeStateGet, AccessHandler: allowPermission(auth.ObjectTypeStorageVolume, auth.EntitlementCanView, "poolName", "type", "volumeName")}, } // swagger:operation GET /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/state storage storage_pool_volume_type_state_get // // Get the storage volume state // // Gets a specific storage volume state (usage data). // // --- // produces: // - application/json // parameters: // - in: path // name: poolName // description: Storage pool name // type: string // required: true // - in: path // name: type // description: Storage volume type // type: string // required: true // - in: path // name: volumeName // description: Storage volume name // type: string // required: true // - in: query // name: project // description: Project name // type: string // example: default // - in: query // name: target // description: Cluster member name // type: string // example: server01 // responses: // "200": // description: Storage pool // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/StorageVolumeState" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func storagePoolVolumeTypeStateGet(d *Daemon, r *http.Request) response.Response { s := d.State() // Get the name of the pool the storage volume is supposed to be attached to. poolName, err := url.PathUnescape(mux.Vars(r)["poolName"]) if err != nil { return response.SmartError(err) } // Get the name of the volume type. volumeTypeName, err := url.PathUnescape(mux.Vars(r)["type"]) if err != nil { return response.SmartError(err) } // Get the name of the volume type. volumeName, err := url.PathUnescape(mux.Vars(r)["volumeName"]) if err != nil { return response.SmartError(err) } // Convert the volume type name to our internal integer representation. volumeType, err := storagePools.VolumeTypeNameToDBType(volumeTypeName) if err != nil { return response.BadRequest(err) } // Check that the storage volume type is valid. if !slices.Contains([]int{db.StoragePoolVolumeTypeCustom, db.StoragePoolVolumeTypeContainer, db.StoragePoolVolumeTypeVM}, volumeType) { return response.BadRequest(fmt.Errorf("Invalid storage volume type %q", volumeTypeName)) } // Get the storage project name. projectName, err := project.StorageVolumeProject(s.DB.Cluster, request.ProjectParam(r), volumeType) if err != nil { return response.SmartError(err) } // Load the storage pool. pool, err := storagePools.LoadByName(s, poolName) if err != nil { return response.SmartError(err) } // Fetch the current usage. var usage *storagePools.VolumeUsage if volumeType == db.StoragePoolVolumeTypeCustom { // Custom volumes. usage, err = pool.GetCustomVolumeUsage(projectName, volumeName) if err != nil && !errors.Is(err, storageDrivers.ErrNotSupported) { return response.SmartError(err) } } else { resp, err := forwardedResponseIfInstanceIsRemote(s, r, projectName, volumeName) if err != nil { return response.SmartError(err) } if resp != nil { return resp } // Instance volumes. inst, err := instance.LoadByProjectAndName(s, projectName, volumeName) if err != nil { return response.SmartError(err) } usage, err = pool.GetInstanceUsage(inst) if err != nil && !errors.Is(err, storageDrivers.ErrNotSupported) { return response.SmartError(err) } } // Prepare the state struct. state := api.StorageVolumeState{} if usage != nil { state.Usage = &api.StorageVolumeStateUsage{} // Only fill 'used' field if receiving a valid value. if usage.Used >= 0 { state.Usage.Used = uint64(usage.Used) } // Only fill 'total' field if receiving a valid value. if usage.Total >= 0 { state.Usage.Total = usage.Total } } return response.SyncResponse(true, state) } incus-7.0.0/cmd/incusd/storage_volumes_utils.go000066400000000000000000000126631517523235500216740ustar00rootroot00000000000000package main import ( "context" "slices" "strings" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) var supportedVolumeTypes = []int{db.StoragePoolVolumeTypeContainer, db.StoragePoolVolumeTypeVM, db.StoragePoolVolumeTypeCustom, db.StoragePoolVolumeTypeImage} func storagePoolVolumeUpdateUsers(ctx context.Context, s *state.State, projectName string, oldPoolName string, oldVol *api.StorageVolume, newPoolName string, newVol *api.StorageVolume) error { // Update all instances that are using the volume with a local (non-expanded) device. err := storagePools.VolumeUsedByInstanceDevices(s, oldPoolName, projectName, oldVol, false, func(dbInst db.InstanceArgs, project api.Project, usedByDevices []string) error { inst, err := instance.Load(s, dbInst, project) if err != nil { return err } localDevices := inst.LocalDevices() for _, devName := range usedByDevices { _, exists := localDevices[devName] if exists { localDevices[devName]["pool"] = newPoolName volFields := strings.SplitN(localDevices[devName]["source"], "/", 2) volFields[0] = newVol.Name localDevices[devName]["source"] = strings.Join(volFields, "/") } } args := db.InstanceArgs{ Architecture: inst.Architecture(), Description: inst.Description(), Config: inst.LocalConfig(), Devices: localDevices, Ephemeral: inst.IsEphemeral(), Profiles: inst.Profiles(), Project: inst.Project().Name, Type: inst.Type(), Snapshot: inst.IsSnapshot(), } err = inst.Update(args, false) if err != nil { return err } return nil }) if err != nil { return err } // Update all profiles that are using the volume with a device. err = storagePools.VolumeUsedByProfileDevices(s, oldPoolName, projectName, oldVol, func(profileID int64, profile api.Profile, p api.Project, usedByDevices []string) error { for name, dev := range profile.Devices { if slices.Contains(usedByDevices, name) { dev["pool"] = newPoolName volFields := strings.SplitN(dev["source"], "/", 2) volFields[0] = newVol.Name dev["source"] = strings.Join(volFields, "/") } } pUpdate := api.ProfilePut{} pUpdate.Config = profile.Config pUpdate.Description = profile.Description pUpdate.Devices = profile.Devices err = doProfileUpdate(ctx, s, p, profile.Name, &profile, pUpdate) if err != nil { return err } return nil }) if err != nil { return err } return nil } // storagePoolVolumeUsedByGet returns a list of URL resources that use the volume. func storagePoolVolumeUsedByGet(s *state.State, requestProjectName string, poolName string, vol *db.StorageVolume) ([]string, error) { // Handle instance volumes. if vol.Type == db.StoragePoolVolumeTypeNameContainer || vol.Type == db.StoragePoolVolumeTypeNameVM { volName, snapName, isSnap := api.GetParentAndSnapshotName(vol.Name) if isSnap { return []string{api.NewURL().Path(version.APIVersion, "instances", volName, "snapshots", snapName).Project(vol.Project).String()}, nil } return []string{api.NewURL().Path(version.APIVersion, "instances", volName).Project(vol.Project).String()}, nil } // Handle image volumes. if vol.Type == db.StoragePoolVolumeTypeNameImage { return []string{api.NewURL().Path(version.APIVersion, "images", vol.Name).Project(requestProjectName).Target(vol.Location).String()}, nil } // Check if the daemon itself is using it. used, err := storagePools.VolumeUsedByDaemon(s, poolName, vol.Name) if err != nil { return []string{}, err } if used { return []string{api.NewURL().Path(version.APIVersion).String()}, nil } // Look for instances using this volume. volumeUsedBy := []string{} // Pass false to expandDevices, as we only want to see instances directly using a volume, rather than their // profiles using a volume. err = storagePools.VolumeUsedByInstanceDevices(s, poolName, vol.Project, &vol.StorageVolume, false, func(inst db.InstanceArgs, p api.Project, usedByDevices []string) error { volumeUsedBy = append(volumeUsedBy, api.NewURL().Path(version.APIVersion, "instances", inst.Name).Project(inst.Project).String()) return nil }) if err != nil { return []string{}, err } err = storagePools.VolumeUsedByProfileDevices(s, poolName, requestProjectName, &vol.StorageVolume, func(profileID int64, profile api.Profile, p api.Project, usedByDevices []string) error { volumeUsedBy = append(volumeUsedBy, api.NewURL().Path(version.APIVersion, "profiles", profile.Name).Project(p.Name).String()) return nil }) if err != nil { return []string{}, err } return volumeUsedBy, nil } func storagePoolVolumeBackupLoadByName(ctx context.Context, s *state.State, projectName, poolName, backupName string) (*backup.VolumeBackup, error) { var b db.StoragePoolVolumeBackup err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error b, err = tx.GetStoragePoolVolumeBackup(ctx, projectName, poolName, backupName) return err }) if err != nil { return nil, err } volumeName := strings.Split(backupName, "/")[0] backup := backup.NewVolumeBackup(s, projectName, poolName, volumeName, b.ID, b.Name, b.CreationDate, b.ExpiryDate, b.VolumeOnly, b.OptimizedStorage) return backup, nil } incus-7.0.0/cmd/incusd/swagger.go000066400000000000000000000012051517523235500166630ustar00rootroot00000000000000// Incus external REST API // // This is the REST API used by all Incus clients. // Internal endpoints aren't included in this documentation. // // The Incus API is available over both a local unix+http and remote https API. // Authentication for local users relies on group membership and access to the unix socket. // For remote users, the default authentication method is TLS client // certificates. // // Version: 1.0 // License: Apache-2.0 https://www.apache.org/licenses/LICENSE-2.0 // Contact: Incus upstream https://github.com/lxc/incus // // swagger:meta package main // Common error definitions. incus-7.0.0/cmd/incusd/tokens.go000066400000000000000000000037621517523235500165410ustar00rootroot00000000000000package main import ( "context" "time" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/server/task" "github.com/lxc/incus/v7/shared/logger" ) func autoRemoveExpiredTokens(ctx context.Context, s *state.State) { expiredTokenOps := make([]*operations.Operation, 0) for _, op := range operations.Clone() { // Only consider token operations if op.Type() != operationtype.ClusterJoinToken && op.Type() != operationtype.CertificateAddToken { continue } // Instead of cancelling the operation here, we add it to a list of expired token operations. // This allows us to only show log messages if there are expired tokens. expiry, ok := op.Metadata()["expiresAt"].(time.Time) if ok && time.Now().After(expiry) { expiredTokenOps = append(expiredTokenOps, op) } } if len(expiredTokenOps) == 0 { return } opRun := func(op *operations.Operation) error { for _, op := range expiredTokenOps { _, err := op.Cancel() if err != nil { logger.Debug("Failed removing expired token", logger.Ctx{"err": err, "id": op.ID()}) } } return nil } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.RemoveExpiredTokens, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating remove expired tokens operation", logger.Ctx{"err": err}) return } logger.Info("Removing expired tokens") err = op.Start() if err != nil { logger.Error("Failed starting remove expired tokens operation", logger.Ctx{"err": err}) return } err = op.Wait(ctx) if err != nil { logger.Error("Failed removing expired tokens", logger.Ctx{"err": err}) return } logger.Debug("Done removing expired tokens") } func autoRemoveExpiredTokensTask(d *Daemon) (task.Func, task.Schedule) { f := func(ctx context.Context) { autoRemoveExpiredTokens(ctx, d.State()) } return f, task.Every(time.Minute) } incus-7.0.0/cmd/incusd/warnings.go000066400000000000000000000364241517523235500170670ustar00rootroot00000000000000package main import ( "context" "database/sql" "encoding/json" "errors" "fmt" "net/http" "net/url" "strconv" "time" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/db/warningtype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/server/task" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) var warningsCmd = APIEndpoint{ Path: "warnings", Get: APIEndpointAction{Handler: warningsGet, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } var warningCmd = APIEndpoint{ Path: "warnings/{id}", Get: APIEndpointAction{Handler: warningGet, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Patch: APIEndpointAction{Handler: warningPatch, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Put: APIEndpointAction{Handler: warningPut, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, Delete: APIEndpointAction{Handler: warningDelete, AccessHandler: allowPermission(auth.ObjectTypeServer, auth.EntitlementCanEdit)}, } func filterWarnings(warnings []api.Warning, clauses *filter.ClauseSet) ([]api.Warning, error) { filtered := []api.Warning{} for _, warning := range warnings { match, err := filter.Match(warning, *clauses) if err != nil { return nil, err } if !match { continue } filtered = append(filtered, warning) } return filtered, nil } // swagger:operation GET /1.0/warnings warnings warnings_get // // List the warnings // // Returns a list of warnings. // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: Sync response // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of endpoints // items: // type: string // example: |- // [ // "/1.0/warnings/39c61a48-cc17-40ae-8248-4f7b4cadedf4", // "/1.0/warnings/951779a5-2820-4d96-b01e-88fe820e5310" // ] // "500": // $ref: "#/responses/InternalServerError" // swagger:operation GET /1.0/warnings?recursion=1 warnings warnings_get_recursion1 // // Get the warnings // // Returns a list of warnings (structs). // // --- // produces: // - application/json // parameters: // - in: query // name: project // description: Project name // type: string // example: default // responses: // "200": // description: API endpoints // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // type: array // description: List of warnings // items: // $ref: "#/definitions/Warning" // "500": // $ref: "#/responses/InternalServerError" func warningsGet(d *Daemon, r *http.Request) response.Response { // Parse the recursion field recursionStr := r.FormValue("recursion") recursion, err := strconv.Atoi(recursionStr) if err != nil { recursion = 0 } // Parse filter value filterStr := r.FormValue("filter") clauses, err := filter.Parse(filterStr, filter.QueryOperatorSet()) if err != nil { return response.SmartError(fmt.Errorf("Failed to filter warnings: %w", err)) } // Parse the project field projectName := request.QueryParam(r, "project") var warnings []api.Warning err = d.State().DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { filters := []cluster.WarningFilter{} if projectName != "" { filter := cluster.WarningFilter{Project: &projectName} filters = append(filters, filter) } dbWarnings, err := cluster.GetWarnings(ctx, tx.Tx(), filters...) if err != nil { return fmt.Errorf("Failed to get warnings: %w", err) } warnings = make([]api.Warning, len(dbWarnings)) for i, w := range dbWarnings { warning := w.ToAPI() warning.EntityURL, err = getWarningEntityURL(ctx, tx.Tx(), &w) if err != nil { return err } warnings[i] = warning } return nil }) if err != nil { return response.SmartError(err) } var filters []api.Warning if recursion == 0 { var resultList []string filters, err = filterWarnings(warnings, clauses) if err != nil { return response.SmartError(err) } for _, w := range filters { url := fmt.Sprintf("/%s/warnings/%s", version.APIVersion, w.UUID) resultList = append(resultList, url) } return response.SyncResponse(true, resultList) } if filters == nil { filters, err = filterWarnings(warnings, clauses) if err != nil { return response.SmartError(err) } } // Return detailed list of warning return response.SyncResponse(true, filters) } // swagger:operation GET /1.0/warnings/{uuid} warnings warning_get // // Get the warning // // Gets a specific warning. // // --- // produces: // - application/json // parameters: // - in: path // name: uuid // description: UUID // type: string // required: true // responses: // "200": // description: Warning // schema: // type: object // description: Sync response // properties: // type: // type: string // description: Response type // example: sync // status: // type: string // description: Status description // example: Success // status_code: // type: integer // description: Status code // example: 200 // metadata: // $ref: "#/definitions/Warning" // "404": // $ref: "#/responses/NotFound" // "500": // $ref: "#/responses/InternalServerError" func warningGet(d *Daemon, r *http.Request) response.Response { id, err := url.PathUnescape(mux.Vars(r)["id"]) if err != nil { return response.SmartError(err) } var resp api.Warning err = d.State().DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { dbWarning, err := cluster.GetWarning(ctx, tx.Tx(), id) if err != nil { return err } resp = dbWarning.ToAPI() resp.EntityURL, err = getWarningEntityURL(ctx, tx.Tx(), dbWarning) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } return response.SyncResponse(true, resp) } // swagger:operation PATCH /1.0/warnings/{uuid} warnings warning_patch // // Partially update the warning // // Updates a subset of the warning status. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: uuid // description: UUID // type: string // required: true // - in: body // name: warning // description: Warning status // required: true // schema: // $ref: "#/definitions/WarningPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func warningPatch(d *Daemon, r *http.Request) response.Response { return warningPut(d, r) } // swagger:operation PUT /1.0/warnings/{uuid} warnings warning_put // // Update the warning // // Updates the warning status. // // --- // consumes: // - application/json // produces: // - application/json // parameters: // - in: path // name: uuid // description: UUID // type: string // required: true // - in: body // name: warning // description: Warning status // required: true // schema: // $ref: "#/definitions/WarningPut" // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "400": // $ref: "#/responses/BadRequest" // "403": // $ref: "#/responses/Forbidden" // "500": // $ref: "#/responses/InternalServerError" func warningPut(d *Daemon, r *http.Request) response.Response { s := d.State() id, err := url.PathUnescape(mux.Vars(r)["id"]) if err != nil { return response.SmartError(err) } req := api.WarningPut{} err = json.NewDecoder(r.Body).Decode(&req) if err != nil { return response.BadRequest(err) } // Currently, we only allow changing the status to acknowledged or new. status, ok := warningtype.StatusTypes[req.Status] if !ok { // Invalid status return response.BadRequest(fmt.Errorf("Invalid warning type %q", req.Status)) } if status != warningtype.StatusAcknowledged && status != warningtype.StatusNew { return response.Forbidden(errors.New(`Status may only be set to "acknowledge" or "new"`)) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { err := tx.UpdateWarningStatus(id, status) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } if status == warningtype.StatusAcknowledged { s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.WarningAcknowledged.Event(id, request.CreateRequestor(r), nil)) } else { s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.WarningReset.Event(id, request.CreateRequestor(r), nil)) } return response.EmptySyncResponse } // swagger:operation DELETE /1.0/warnings/{uuid} warnings warning_delete // // Delete the warning // // Removes the warning. // // --- // produces: // - application/json // parameters: // - in: path // name: uuid // description: UUID // type: string // required: true // responses: // "200": // $ref: "#/responses/EmptySyncResponse" // "500": // $ref: "#/responses/InternalServerError" func warningDelete(d *Daemon, r *http.Request) response.Response { s := d.State() id, err := url.PathUnescape(mux.Vars(r)["id"]) if err != nil { return response.SmartError(err) } err = s.DB.Cluster.Transaction(r.Context(), func(ctx context.Context, tx *db.ClusterTx) error { err := cluster.DeleteWarning(ctx, tx.Tx(), id) if err != nil { return err } return nil }) if err != nil { return response.SmartError(err) } s.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.WarningDeleted.Event(id, request.CreateRequestor(r), nil)) return response.EmptySyncResponse } func pruneResolvedWarningsTask(d *Daemon) (task.Func, task.Schedule) { f := func(ctx context.Context) { s := d.State() opRun := func(op *operations.Operation) error { return pruneResolvedWarnings(ctx, s) } op, err := operations.OperationCreate(s, "", operations.OperationClassTask, operationtype.WarningsPruneResolved, nil, nil, opRun, nil, nil, nil) if err != nil { logger.Error("Failed creating prune resolved warnings operation", logger.Ctx{"err": err}) return } logger.Info("Pruning resolved warnings") err = op.Start() if err != nil { logger.Error("Failed starting prune resolved warnings operation", logger.Ctx{"err": err}) return } err = op.Wait(ctx) if err != nil { logger.Error("Failed pruning resolved warnings", logger.Ctx{"err": err}) return } logger.Info("Done pruning resolved warnings") } return f, task.Daily() } func pruneResolvedWarnings(ctx context.Context, s *state.State) error { err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Retrieve warnings by resolved status. statusResolved := warningtype.StatusResolved filter := cluster.WarningFilter{ Status: &statusResolved, } warnings, err := cluster.GetWarnings(ctx, tx.Tx(), filter) if err != nil { return fmt.Errorf("Failed to get resolved warnings: %w", err) } for _, w := range warnings { // Delete the warning if it has been resolved for at least 24 hours if time.Since(w.UpdatedDate) >= 24*time.Hour { err = cluster.DeleteWarning(ctx, tx.Tx(), w.UUID) if err != nil { return err } } } return nil }) if err != nil { return fmt.Errorf("Failed to delete warnings: %w", err) } return nil } // getWarningEntityURL fetches the entity corresponding to the warning from the database, and generates a URL. func getWarningEntityURL(ctx context.Context, tx *sql.Tx, warning *cluster.Warning) (string, error) { if warning.EntityID == -1 || warning.EntityTypeCode == -1 { return "", nil } _, ok := cluster.EntityNames[warning.EntityTypeCode] if !ok { return "", errors.New("Unknown entity type") } var url string switch warning.EntityTypeCode { case cluster.TypeImage: entities, err := cluster.GetImages(ctx, tx, cluster.ImageFilter{ID: &warning.EntityID}) if err != nil { return "", err } if len(entities) == 0 { return "", db.ErrUnknownEntityID } apiImage := api.Image{Fingerprint: entities[0].Fingerprint} url = apiImage.URL(version.APIVersion, entities[0].Project).String() case cluster.TypeProfile: entities, err := cluster.GetProfiles(ctx, tx, cluster.ProfileFilter{ID: &warning.EntityID}) if err != nil { return "", err } if len(entities) == 0 { return "", db.ErrUnknownEntityID } apiProfile := api.Profile{Name: entities[0].Name} url = apiProfile.URL(version.APIVersion, entities[0].Project).String() case cluster.TypeProject: entities, err := cluster.GetProjects(ctx, tx, cluster.ProjectFilter{ID: &warning.EntityID}) if err != nil { return "", err } if len(entities) == 0 { return "", db.ErrUnknownEntityID } apiProject := api.Project{Name: entities[0].Name} url = apiProject.URL(version.APIVersion).String() case cluster.TypeCertificate: entities, err := cluster.GetCertificates(ctx, tx, cluster.CertificateFilter{ID: &warning.EntityID}) if err != nil { return "", err } if len(entities) == 0 { return "", db.ErrUnknownEntityID } apiCertificate := api.Certificate{Fingerprint: entities[0].Fingerprint} url = apiCertificate.URL(version.APIVersion).String() case cluster.TypeContainer: fallthrough case cluster.TypeInstance: entities, err := cluster.GetInstances(ctx, tx, cluster.InstanceFilter{ID: &warning.EntityID}) if err != nil { return "", err } if len(entities) == 0 { return "", db.ErrUnknownEntityID } apiInstance := api.Instance{Name: entities[0].Name} url = apiInstance.URL(version.APIVersion, entities[0].Project).String() } return url, nil } incus-7.0.0/cmd/lxc-to-incus/000077500000000000000000000000001517523235500157375ustar00rootroot00000000000000incus-7.0.0/cmd/lxc-to-incus/config.go000066400000000000000000000076531517523235500175460ustar00rootroot00000000000000package main import ( "bufio" "fmt" "os" "path/filepath" "slices" "strings" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/util" ) var checkedKeys = []string{ "lxc.aa_allow_incomplete", "lxc.aa_profile", "lxc.apparmor.allow_incomplete", "lxc.apparmor.allow_nesting", "lxc.apparmor.profile", "lxc.arch", "lxc.autodev", "lxc.cap.drop", "lxc.environment", "lxc.haltsignal", "lxc.id_map", "lxc.idmap", "lxc.include", "lxc.loglevel", "lxc.mount", "lxc.mount.auto", "lxc.mount.entry", "lxc.pts", "lxc.pty.max", "lxc.rebootsignal", "lxc.rootfs", "lxc.rootfs.backend", "lxc.rootfs.mount", "lxc.rootfs.path", "lxc.seccomp", "lxc.signal.halt", "lxc.signal.reboot", "lxc.signal.stop", "lxc.start.auto", "lxc.start.delay", "lxc.start.order", "lxc.stopsignal", "lxc.tty", "lxc.tty.max", "lxc.uts.name", "lxc.utsname", "incus.migrated", } func getUnsupportedKeys(config []string) []string { var out []string for _, a := range config { supported := slices.Contains(checkedKeys, a) if !supported { out = append(out, a) } } return out } func getConfig(config []string, key string) []string { // Return an array since keys can be specified more than once var out []string for _, c := range config { text := strings.TrimSpace(c) // Ignore empty lines and comments if len(text) == 0 || strings.HasPrefix(text, "#") { continue } line := strings.Split(text, "=") if len(line) != 2 { continue } k := strings.TrimSpace(line[0]) v := strings.Trim(strings.TrimSpace(line[1]), "\"") if k == key && len(v) > 0 { out = append(out, v) } } if len(out) == 0 { return nil } return out } func getConfigKeys(config []string) []string { // Make sure we don't have duplicate keys m := make(map[string]bool) for _, c := range config { text := strings.TrimSpace(c) // Ignore empty lines and comments if len(text) == 0 || strings.HasPrefix(text, "#") { continue } line := strings.Split(text, "=") key := strings.TrimSpace(line[0]) if strings.HasPrefix(key, "lxc.") { m[key] = true } } var out []string for k := range m { out = append(out, k) } return out } func parseConfig(path string) ([]string, error) { file, err := os.Open(path) if err != nil { return nil, err } defer func() { _ = file.Close() }() var config []string // Parse config sc := bufio.NewScanner(file) for sc.Scan() { text := strings.TrimSpace(sc.Text()) // Ignore empty lines and comments if len(text) == 0 || strings.HasPrefix(text, "#") { continue } line := strings.Split(text, "=") if len(line) != 2 { continue } key := strings.TrimSpace(line[0]) value := strings.TrimSpace(line[1]) switch key { // Parse user-added includes case "lxc.include": // Ignore our own default configs if strings.HasPrefix(value, "/usr/share/lxc/config/") { continue } if util.PathExists(value) { if internalUtil.IsDir(value) { files, err := os.ReadDir(value) if err != nil { return nil, err } for _, file := range files { path := filepath.Join(value, file.Name()) if !strings.HasSuffix(path, ".conf") { continue } config = append(config, path) } } else { c, err := parseConfig(value) if err != nil { return nil, err } config = append(config, c...) } continue } // Expand any fstab case "lxc.mount": if !util.PathExists(value) { fmt.Println("Container fstab file doesn't exist, skipping...") continue } file, err := os.Open(value) if err != nil { return nil, err } sc := bufio.NewScanner(file) for sc.Scan() { text := strings.TrimSpace(sc.Text()) if len(text) > 0 && !strings.HasPrefix(text, "#") { config = append(config, fmt.Sprintf("lxc.mount.entry = %s", text)) } } _ = file.Close() continue default: config = append(config, text) } } return config, nil } incus-7.0.0/cmd/lxc-to-incus/main.go000066400000000000000000000034421517523235500172150ustar00rootroot00000000000000package main /* #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include __attribute__((constructor)) void init(void) { int ret; int mntns_fd; if (getenv("SNAP") == NULL) return; mntns_fd = open("/proc/1/ns/mnt", O_RDONLY | O_CLOEXEC); if (ret < 0) { fprintf(stderr, "Failed open mntns: %s\n", strerror(errno)); _exit(EXIT_FAILURE); } ret = setns(mntns_fd, CLONE_NEWNS); close(mntns_fd); if (ret < 0) { fprintf(stderr, "Failed setns to outside mount namespace: %s\n", strerror(errno)); _exit(EXIT_FAILURE); } ret = chdir("/"); if (ret < 0) { fprintf(stderr, "Failed chdir /: %s\n", strerror(errno)); _exit(EXIT_FAILURE); } // We're done, jump back to Go } */ import "C" import ( "os" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/version" ) type cmdGlobal struct { flagVersion bool flagHelp bool } func main() { // migrate command (main) migrateCmd := cmdMigrate{} app := migrateCmd.command() app.SilenceUsage = true app.CompletionOptions = cobra.CompletionOptions{DisableDefaultCmd: true} // Workaround for main command app.Args = cobra.ArbitraryArgs // Global flags globalCmd := cmdGlobal{} migrateCmd.global = &globalCmd app.PersistentFlags().BoolVar(&globalCmd.flagVersion, "version", false, "Print version number") app.PersistentFlags().BoolVarP(&globalCmd.flagHelp, "help", "h", false, "Print help") // Version handling app.SetVersionTemplate("{{.Version}}\n") app.Version = version.Version // netcat sub-command netcatCmd := cmdNetcat{global: &globalCmd} app.AddCommand(netcatCmd.command()) // Run the main command and handle errors err := app.Execute() if err != nil { os.Exit(1) } } incus-7.0.0/cmd/lxc-to-incus/main_migrate.go000066400000000000000000000362641517523235500207350ustar00rootroot00000000000000package main import ( "encoding/json" "errors" "fmt" "runtime" "slices" "strconv" "strings" liblxc "github.com/lxc/go-lxc" "github.com/spf13/cobra" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/i18n" "github.com/lxc/incus/v7/shared/api" cli "github.com/lxc/incus/v7/shared/cmd" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/util" ) type cmdMigrate struct { global *cmdGlobal // Flags flagDryRun bool flagDebug bool flagAll bool flagDelete bool flagStorage string flagLXCPath string flagRsyncArgs string flagContainers []string } func (c *cmdMigrate) command() *cobra.Command { cmd := &cobra.Command{ Use: "lxc-to-incus", Short: i18n.G("Command line client for container migration"), } // Wrappers cmd.RunE = c.run // Flags cmd.Flags().BoolVar(&c.flagDryRun, "dry-run", false, i18n.G("Dry run mode")) cmd.Flags().BoolVar(&c.flagDebug, "debug", false, i18n.G("Print debugging output")) cmd.Flags().BoolVar(&c.flagAll, "all", false, i18n.G("Import all containers")) cmd.Flags().BoolVar(&c.flagDelete, "delete", false, i18n.G("Delete the source container")) cmd.Flags().StringVar(&c.flagStorage, "storage", "", i18n.G("Storage pool to use for the container")+"``") cmd.Flags().StringVar(&c.flagLXCPath, "lxcpath", liblxc.DefaultConfigPath(), i18n.G("Alternate LXC path")+"``") cmd.Flags().StringVar(&c.flagRsyncArgs, "rsync-args", "", "Extra arguments to pass to rsync"+"``") cmd.Flags().StringSliceVar(&c.flagContainers, "containers", nil, i18n.G("Container(s) to import")+"``") return cmd } func (c *cmdMigrate) run(cmd *cobra.Command, args []string) error { if (len(c.flagContainers) == 0 && !c.flagAll) || (len(c.flagContainers) > 0 && c.flagAll) { return errors.New("You must either pass container names or --all") } // Connect to the daemon d, err := incus.ConnectIncusUnix("", nil) if err != nil { return err } // Retrieve LXC containers for _, container := range liblxc.Containers(c.flagLXCPath) { if !c.flagAll && !slices.Contains(c.flagContainers, container.Name()) { continue } err := convertContainer(d, container, c.flagStorage, c.flagDryRun, c.flagRsyncArgs, c.flagDebug) if err != nil { fmt.Printf("Skipping container '%s': %v\n", container.Name(), err) continue } // Delete container if c.flagDelete { if c.flagDryRun { fmt.Println("Would destroy container now") continue } err := container.Destroy() if err != nil { fmt.Printf("Failed to destroy container '%s': %v\n", container.Name(), err) } } } return nil } func validateConfig(conf []string, container *liblxc.Container) error { // Checking whether container has already been migrated fmt.Println("Checking whether container has already been migrated") if len(getConfig(conf, "incus.migrated")) > 0 { return errors.New("Container has already been migrated") } // Validate lxc.utsname / lxc.uts.name value := getConfig(conf, "lxc.uts.name") if value == nil { value = getConfig(conf, "lxc.utsname") } if value == nil || value[0] != container.Name() { return errors.New("Container name doesn't match lxc.uts.name / lxc.utsname") } // Validate lxc.aa_allow_incomplete: must be set to 0 or unset. fmt.Println("Validating whether incomplete AppArmor support is enabled") value = getConfig(conf, "lxc.apparmor.allow_incomplete") if value == nil { value = getConfig(conf, "lxc.aa_allow_incomplete") } if value != nil { v, err := strconv.Atoi(value[0]) if err != nil { return err } if v != 0 { return errors.New("Container allows incomplete AppArmor support") } } // Validate lxc.autodev: must be set to 1 or unset. fmt.Println("Validating whether mounting a minimal /dev is enabled") value = getConfig(conf, "lxc.autodev") if value != nil { v, err := strconv.Atoi(value[0]) if err != nil { return err } if v != 1 { return errors.New("Container doesn't mount a minimal /dev filesystem") } } // Extract and valid rootfs key fmt.Println("Validating container rootfs") rootfs, err := getRootfs(conf) if err != nil { return err } if !util.PathExists(rootfs) { return fmt.Errorf("Couldn't find the container rootfs '%s'", rootfs) } return nil } func convertContainer(d incus.InstanceServer, container *liblxc.Container, storage string, dryRun bool, rsyncArgs string, debug bool) error { // Don't migrate running containers if container.Running() { return errors.New("Only stopped containers can be migrated") } fmt.Println("Parsing LXC configuration") conf, err := parseConfig(container.ConfigFileName()) if err != nil { return err } if debug { fmt.Printf("Container configuration:\n %v\n", strings.Join(conf, "\n ")) } // Check whether there are unsupported keys in the config fmt.Println("Checking for unsupported LXC configuration keys") keys := getUnsupportedKeys(getConfigKeys(conf)) for _, k := range keys { if !strings.HasPrefix(k, "lxc.net.") && !strings.HasPrefix(k, "lxc.network.") && !strings.HasPrefix(k, "lxc.cgroup.") && !strings.HasPrefix(k, "lxc.cgroup2.") { return fmt.Errorf("Found unsupported config key '%s'", k) } } // Make sure we don't have a conflict fmt.Println("Checking for existing containers") containers, err := d.GetInstanceNames(api.InstanceTypeContainer) if err != nil { return err } found := false for _, name := range containers { if name == container.Name() { found = true } } if found { return errors.New("Container already exists") } // Validate config err = validateConfig(conf, container) if err != nil { return err } newConfig := make(map[string]string) value := getConfig(conf, "lxc.idmap") if value == nil { value = getConfig(conf, "lxc.id_map") } if value == nil { // Privileged container newConfig["security.privileged"] = "true" } else { // Unprivileged container newConfig["security.privileged"] = "false" } newDevices := make(map[string]map[string]string) // Convert network configuration err = convertNetworkConfig(container, newDevices) if err != nil { return err } // Convert storage configuration err = convertStorageConfig(conf, newDevices) if err != nil { return err } // Convert environment fmt.Println("Processing environment configuration") value = getConfig(conf, "lxc.environment") for _, env := range value { entry := strings.Split(env, "=") key := strings.TrimSpace(entry[0]) val := strings.TrimSpace(entry[len(entry)-1]) newConfig[fmt.Sprintf("environment.%s", key)] = val } // Convert auto-start fmt.Println("Processing container boot configuration") value = getConfig(conf, "lxc.start.auto") if value != nil { val, err := strconv.Atoi(value[0]) if err != nil { return err } if val > 0 { newConfig["boot.autostart"] = "true" } } value = getConfig(conf, "lxc.start.delay") if value != nil { val, err := strconv.Atoi(value[0]) if err != nil { return err } if val > 0 { newConfig["boot.autostart.delay"] = value[0] } } value = getConfig(conf, "lxc.start.order") if value != nil { val, err := strconv.Atoi(value[0]) if err != nil { return err } if val > 0 { newConfig["boot.autostart.priority"] = value[0] } } // Convert apparmor fmt.Println("Processing container apparmor configuration") value = getConfig(conf, "lxc.apparmor.profile") if value == nil { value = getConfig(conf, "lxc.aa_profile") } if value != nil { if value[0] == "lxc-container-default-with-nesting" { newConfig["security.nesting"] = "true" } else if value[0] != "lxc-container-default" { newConfig["raw.lxc"] = fmt.Sprintf("lxc.apparmor.profile=%s\n", value[0]) } } // Convert seccomp fmt.Println("Processing container seccomp configuration") value = getConfig(conf, "lxc.seccomp.profile") if value == nil { value = getConfig(conf, "lxc.seccomp") } if value != nil && value[0] != "/usr/share/lxc/config/common.seccomp" { return errors.New("Custom seccomp profiles aren't supported") } // Convert SELinux fmt.Println("Processing container SELinux configuration") value = getConfig(conf, "lxc.selinux.context") if value == nil { value = getConfig(conf, "lxc.se_context") } if value != nil { return errors.New("Custom SELinux policies aren't supported") } // Convert capabilities fmt.Println("Processing container capabilities configuration") value = getConfig(conf, "lxc.cap.drop") if value != nil { for _, cap := range strings.Split(value[0], " ") { // Ignore capabilities that are dropped in containers by default. if slices.Contains([]string{"mac_admin", "mac_override", "sys_module", "sys_time"}, cap) { continue } return errors.New("Custom capabilities aren't supported") } } value = getConfig(conf, "lxc.cap.keep") if value != nil { return errors.New("Custom capabilities aren't supported") } // Add rest of the keys to lxc.raw for _, c := range conf { parts := strings.SplitN(c, "=", 2) if len(parts) != 2 { continue } key := strings.TrimSpace(parts[0]) val := strings.TrimSpace(parts[1]) switch key { case "lxc.signal.halt", "lxc.haltsignal": newConfig["raw.lxc"] += fmt.Sprintf("lxc.signal.halt=%s\n", val) case "lxc.signal.reboot", "lxc.rebootsignal": newConfig["raw.lxc"] += fmt.Sprintf("lxc.signal.reboot=%s\n", val) case "lxc.signal.stop", "lxc.stopsignal": newConfig["raw.lxc"] += fmt.Sprintf("lxc.signal.stop=%s\n", val) case "lxc.apparmor.allow_incomplete", "lxc.aa_allow_incomplete": newConfig["raw.lxc"] += fmt.Sprintf("lxc.apparmor.allow_incomplete=%s\n", val) case "lxc.pty.max", "lxc.pts": newConfig["raw.lxc"] += fmt.Sprintf("lxc.pty.max=%s\n", val) case "lxc.tty.max", "lxc.tty": newConfig["raw.lxc"] += fmt.Sprintf("lxc.tty.max=%s\n", val) } } // Setup the container creation request req := api.InstancesPost{ Name: container.Name(), Source: api.InstanceSource{ Type: "migration", Mode: "push", }, } req.Config = newConfig req.Devices = newDevices req.Profiles = []string{"default"} // Set the container architecture if set in LXC fmt.Println("Processing container architecture configuration") var arch string value = getConfig(conf, "lxc.arch") if value == nil { fmt.Println("Couldn't find container architecture, assuming native") arch = runtime.GOARCH } else { arch = value[0] } archID, err := osarch.ArchitectureID(arch) if err != nil { // If arch is linux32 or linux64, the architecture ID cannot be determined as multiple // architectures have the linux32 or linux64 personality. In this case, assume the native // architecture. arch = runtime.GOARCH archID, err = osarch.ArchitectureID(arch) if err != nil { return err } // If the instance architecture is 32bit but the local architecture is 64bit, iterate // through the local architecture's personalities until the supported architecture // personality matches the instance's architecture. if len(value) > 0 && value[0] == "linux32" { personalities, err := osarch.ArchitecturePersonalities(archID) if err != nil { return err } for id, personality := range personalities { arch, err = osarch.ArchitecturePersonality(personality) if err != nil { return err } if arch == value[0] { archID = id break } } } } req.Architecture, err = osarch.ArchitectureName(archID) if err != nil { return err } if storage != "" { req.Devices["root"] = map[string]string{ "type": "disk", "pool": storage, "path": "/", } } if debug { out, _ := json.MarshalIndent(req, "", " ") fmt.Printf("Container config:\n%v\n", string(out)) } // Create container fmt.Println("Creating container") if dryRun { fmt.Println("Would create container now") } else { op, err := d.CreateInstance(req) if err != nil { return err } progress := cli.ProgressRenderer{Format: "Transferring container: %s"} _, err = op.AddHandler(progress.UpdateOp) if err != nil { progress.Done("") return err } rootfs, _ := getRootfs(conf) err = transferRootfs(d, op, rootfs, rsyncArgs) if err != nil { return err } progress.Done(fmt.Sprintf("Container '%s' successfully created", container.Name())) } return nil } func convertNetworkConfig(container *liblxc.Container, devices map[string]map[string]string) error { networkDevice := func(network map[string]string) map[string]string { if network == nil { return nil } device := make(map[string]string) device["type"] = "nic" // Get the device type device["nictype"] = network["type"] // Convert the configuration for k, v := range network { switch k { case "hwaddr", "mtu", "name": device[k] = v case "link": device["parent"] = v case "veth_pair": device["host_name"] = v case "": // empty key return nil } } switch device["nictype"] { case "veth": _, ok := device["parent"] if ok { device["nictype"] = "bridged" } else { device["nictype"] = "p2p" } case "phys": device["nictype"] = "physical" case "empty": return nil } return device } fmt.Println("Processing network configuration") devices["eth0"] = make(map[string]string) devices["eth0"]["type"] = "none" // New config key for i := range container.ConfigItem("lxc.net") { network := networkGet(container, i, "lxc.net") dev := networkDevice(network) if dev == nil { continue } devices[fmt.Sprintf("net%d", i)] = dev } // Old config key for i := range container.ConfigItem("lxc.network") { network := networkGet(container, i, "lxc.network") dev := networkDevice(network) if dev == nil { continue } devices[fmt.Sprintf("net%d", len(devices))] = dev } return nil } func convertStorageConfig(conf []string, devices map[string]map[string]string) error { fmt.Println("Processing storage configuration") i := 0 for _, mount := range getConfig(conf, "lxc.mount.entry") { parts := strings.Split(mount, " ") if len(parts) < 4 { return fmt.Errorf("Invalid mount configuration: %s", mount) } // Ignore mounts that are present in containers by default. if slices.Contains([]string{"proc", "sysfs"}, parts[0]) { continue } device := make(map[string]string) device["type"] = "disk" // Deal with read-only mounts if slices.Contains(strings.Split(parts[3], ","), "ro") { device["readonly"] = "true" } // Deal with optional mounts if slices.Contains(strings.Split(parts[3], ","), "optional") { device["optional"] = "true" } else { if !strings.HasPrefix(parts[0], "/") { continue } if !util.PathExists(parts[0]) { return fmt.Errorf("Invalid path: %s", parts[0]) } } // Set the source device["source"] = parts[0] // Figure out the target if !strings.HasPrefix(parts[1], "/") { device["path"] = fmt.Sprintf("/%s", parts[1]) } else { rootfs, err := getRootfs(conf) if err != nil { return err } device["path"] = strings.TrimPrefix(parts[1], rootfs) } devices[fmt.Sprintf("mount%d", i)] = device i++ } return nil } func getRootfs(conf []string) (string, error) { value := getConfig(conf, "lxc.rootfs.path") if value == nil { value = getConfig(conf, "lxc.rootfs") if value == nil { return "", errors.New("Invalid container, missing lxc.rootfs key") } } // Get the rootfs path parts := strings.SplitN(value[0], ":", 2) return parts[len(parts)-1], nil } incus-7.0.0/cmd/lxc-to-incus/main_migrate_test.go000066400000000000000000000201131517523235500217560ustar00rootroot00000000000000package main import ( "log" "os" "strings" "testing" liblxc "github.com/lxc/go-lxc" "github.com/stretchr/testify/require" ) func TestValidateConfig(t *testing.T) { tests := []struct { name string config []string err string shouldFail bool }{ { "container migrated", []string{ "incus.migrated = 1", }, "Container has already been migrated", true, }, { "container name mismatch (1)", []string{ "lxc.uts.name = c2", }, "Container name doesn't match lxc.uts.name / lxc.utsname", true, }, { "container name mismatch (2)", []string{ "lxc.utsname = c2", }, "Container name doesn't match lxc.uts.name / lxc.utsname", true, }, { "incomplete AppArmor support (1)", []string{ "lxc.uts.name = c1", "lxc.apparmor.allow_incomplete = 1", }, "Container allows incomplete AppArmor support", true, }, { "incomplete AppArmor support (2)", []string{ "lxc.uts.name = c1", "lxc.aa_allow_incomplete = 1", }, "Container allows incomplete AppArmor support", true, }, { "missing minimal /dev filesystem", []string{ "lxc.uts.name = c1", "lxc.apparmor.allow_incomplete = 0", "lxc.autodev = 0", }, "Container doesn't mount a minimal /dev filesystem", true, }, { "missing lxc.rootfs key", []string{ "lxc.uts.name = c1", "lxc.apparmor.allow_incomplete = 0", "lxc.autodev = 1", }, "Invalid container, missing lxc.rootfs key", true, }, { "non-existent rootfs path", []string{ "lxc.uts.name = c1", "lxc.apparmor.allow_incomplete = 0", "lxc.autodev = 1", "lxc.rootfs = dir:/invalid/path", }, "Couldn't find the container rootfs '/invalid/path'", true, }, } lxcPath, err := os.MkdirTemp("", "lxc-to-incus-test-") require.NoError(t, err) defer require.NoError(t, os.RemoveAll(lxcPath)) c, err := liblxc.NewContainer("c1", lxcPath) require.NoError(t, err) for i, tt := range tests { log.Printf("Running test #%d: %s", i, tt.name) err := validateConfig(tt.config, c) if tt.shouldFail { require.EqualError(t, err, tt.err) } else { require.NoError(t, err) } } } func TestConvertNetworkConfig(t *testing.T) { tests := []struct { name string config []string expectedDevices map[string]map[string]string expectedError string shouldFail bool }{ { "loopback only", []string{}, map[string]map[string]string{ "eth0": { "type": "none", }, }, "", false, }, { "multiple network devices (sorted)", []string{ "lxc.net.0.type = macvlan", "lxc.net.0.macvlan.mode = bridge", "lxc.net.0.link = mvlan0", "lxc.net.0.hwaddr = 10:66:6a:8d:4f:51", "lxc.net.0.name = eth1", "lxc.net.1.type = veth", "lxc.net.1.link = lxcbr0", "lxc.net.1.hwaddr = 10:66:6a:a2:7d:54", "lxc.net.1.name = eth2", }, map[string]map[string]string{ "net1": { "type": "nic", "nictype": "bridged", "parent": "lxcbr0", "name": "eth2", "hwaddr": "10:66:6a:a2:7d:54", }, "eth0": { "type": "none", }, "net0": { "name": "eth1", "hwaddr": "10:66:6a:8d:4f:51", "type": "nic", "nictype": "macvlan", "parent": "mvlan0", }, }, "", false, }, { "multiple network devices (unsorted)", []string{ "lxc.net.0.type = macvlan", "lxc.net.0.macvlan.mode = bridge", "lxc.net.0.link = mvlan0", "lxc.net.1.type = veth", "lxc.net.0.hwaddr = 10:66:6a:8d:4f:51", "lxc.net.0.name = eth1", "lxc.net.1.name = eth2", "lxc.net.1.link = lxcbr0", "lxc.net.1.hwaddr = 10:66:6a:a2:7d:54", }, map[string]map[string]string{ "net1": { "type": "nic", "nictype": "bridged", "parent": "lxcbr0", "name": "eth2", "hwaddr": "10:66:6a:a2:7d:54", }, "eth0": { "type": "none", }, "net0": { "name": "eth1", "hwaddr": "10:66:6a:8d:4f:51", "type": "nic", "nictype": "macvlan", "parent": "mvlan0", }, }, "", false, }, } lxcPath, err := os.MkdirTemp("", "lxc-to-incus-test-") require.NoError(t, err) defer func() { _ = os.RemoveAll(lxcPath) }() for i, tt := range tests { log.Printf("Running test #%d: %s", i, tt.name) c, err := liblxc.NewContainer("c1", lxcPath) require.NoError(t, err) err = c.Create(liblxc.TemplateOptions{Template: "busybox"}) require.NoError(t, err) // In case the system uses a lxc.conf file err = c.ClearConfigItem("lxc.net.0") require.NoError(t, err) for _, conf := range tt.config { parts := strings.SplitN(conf, "=", 2) require.Equal(t, 2, len(parts)) err := c.SetConfigItem(strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1])) require.NoError(t, err) } devices := make(map[string]map[string]string) err = convertNetworkConfig(c, devices) if tt.shouldFail { require.EqualError(t, err, tt.expectedError) } else { require.NoError(t, err) require.Equal(t, tt.expectedDevices, devices) } err = c.Destroy() if err != nil && strings.Contains(err.Error(), string(liblxc.ErrNotDefined)) { continue } require.NoError(t, err) } } func TestConvertStorageConfig(t *testing.T) { tests := []struct { name string config []string expectedDevices map[string]map[string]string expectedError string shouldFail bool }{ { "invalid path", []string{ "lxc.mount.entry = /foo lib none ro,bind 0 0", }, map[string]map[string]string{}, "Invalid path: /foo", true, }, { "ignored default mounts", []string{ "lxc.mount.entry = proc /proc proc defaults 0 0", }, map[string]map[string]string{}, "", false, }, { "ignored mounts", []string{ "lxc.mount.entry = shm /dev/shm tmpfs defaults 0 0", }, map[string]map[string]string{}, "", false, }, { "valid mount configuration", []string{ "lxc.rootfs.path = dir:/tmp", "lxc.mount.entry = /lib lib none ro,bind 0 0", "lxc.mount.entry = /usr/lib usr/lib none ro,bind 1 0", "lxc.mount.entry = /home home none ro,bind 0 0", "lxc.mount.entry = /sys/kernel/security /sys/kernel/security none ro,bind,optional 1 0", "lxc.mount.entry = /mnt /tmp/mnt none ro,bind 0 0", }, map[string]map[string]string{ "mount0": { "type": "disk", "readonly": "true", "source": "/lib", "path": "/lib", }, "mount1": { "type": "disk", "readonly": "true", "source": "/usr/lib", "path": "/usr/lib", }, "mount2": { "type": "disk", "readonly": "true", "source": "/home", "path": "/home", }, "mount3": { "type": "disk", "readonly": "true", "optional": "true", "source": "/sys/kernel/security", "path": "/sys/kernel/security", }, "mount4": { "type": "disk", "readonly": "true", "source": "/mnt", "path": "/mnt", }, }, "", false, }, } for i, tt := range tests { log.Printf("Running test #%d: %s", i, tt.name) devices := make(map[string]map[string]string) err := convertStorageConfig(tt.config, devices) if tt.shouldFail { require.EqualError(t, err, tt.expectedError) } else { require.NoError(t, err) require.Equal(t, tt.expectedDevices, devices) } } } func TestGetRootfs(t *testing.T) { tests := []struct { name string config []string expectedOutput string expectedError string shouldFail bool }{ { "missing lxc.rootfs key", []string{}, "", "Invalid container, missing lxc.rootfs key", true, }, { "valid lxc.rootfs key (1)", []string{ "lxc.rootfs = foobar", }, "foobar", "", false, }, { "valid lxc.rootfs key (2)", []string{ "lxc.rootfs = dir:foobar", }, "foobar", "", false, }, } for i, tt := range tests { log.Printf("Running test #%d: %s", i, tt.name) rootfs, err := getRootfs(tt.config) require.Equal(t, tt.expectedOutput, rootfs) if tt.shouldFail { require.EqualError(t, err, tt.expectedError) } else { require.NoError(t, err) } } } incus-7.0.0/cmd/lxc-to-incus/main_netcat.go000066400000000000000000000023751517523235500205570ustar00rootroot00000000000000package main import ( "errors" "net" "os" "sync" "github.com/spf13/cobra" "github.com/lxc/incus/v7/internal/eagain" "github.com/lxc/incus/v7/shared/util" ) type cmdNetcat struct { global *cmdGlobal } func (c *cmdNetcat) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "netcat
" cmd.Short = "Sends stdin data to a unix socket" cmd.RunE = c.run cmd.Hidden = true return cmd } func (c *cmdNetcat) run(cmd *cobra.Command, args []string) error { // Help and usage if len(args) == 0 { _ = cmd.Help() return nil } // Handle mandatory arguments if len(args) != 1 { _ = cmd.Help() return errors.New("Missing required argument") } // Connect to the provided address uAddr, err := net.ResolveUnixAddr("unix", args[0]) if err != nil { return err } conn, err := net.DialUnix("unix", nil, uAddr) if err != nil { return err } // We'll wait until we're done reading from the socket wg := sync.WaitGroup{} wg.Add(1) go func() { defer func() { _ = conn.Close() }() defer wg.Done() _, _ = util.SafeCopy(eagain.Writer{Writer: os.Stdout}, eagain.Reader{Reader: conn}) }() go func() { _, _ = util.SafeCopy(eagain.Writer{Writer: conn}, eagain.Reader{Reader: os.Stdin}) }() // Wait wg.Wait() return nil } incus-7.0.0/cmd/lxc-to-incus/network.go000066400000000000000000000010671517523235500177630ustar00rootroot00000000000000package main import ( "fmt" "strings" liblxc "github.com/lxc/go-lxc" ) func networkGet(container *liblxc.Container, index int, configKey string) map[string]string { keys := container.ConfigKeys(fmt.Sprintf("%s.%d", configKey, index)) if len(keys) == 0 { return nil } dev := make(map[string]string) for _, k := range keys { value := container.ConfigItem(fmt.Sprintf("%s.%d.%s", configKey, index, k)) if len(value) == 0 || strings.TrimSpace(value[0]) == "" { continue } dev[k] = value[0] } if len(dev) == 0 { return nil } return dev } incus-7.0.0/cmd/lxc-to-incus/transfer.go000066400000000000000000000050221517523235500201110ustar00rootroot00000000000000package main import ( "fmt" "io" "net" "os" "os/exec" "strings" "github.com/google/uuid" "github.com/gorilla/websocket" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/shared/ws" ) // Send an rsync stream of a path over a websocket. func rsyncSend(conn *websocket.Conn, path string, rsyncArgs string) error { cmd, dataSocket, stderr, err := rsyncSendSetup(path, rsyncArgs) if err != nil { return err } if dataSocket != nil { defer func() { _ = dataSocket.Close() }() } readDone, writeDone := ws.Mirror(conn, dataSocket) <-writeDone _ = dataSocket.Close() output, err := io.ReadAll(stderr) if err != nil { _ = cmd.Process.Kill() _ = cmd.Wait() return fmt.Errorf("Failed to rsync: %v\n%s", err, output) } err = cmd.Wait() <-readDone if err != nil { return fmt.Errorf("Failed to rsync: %v\n%s", err, output) } return nil } // Spawn the rsync process. func rsyncSendSetup(path string, rsyncArgs string) (*exec.Cmd, net.Conn, io.ReadCloser, error) { auds := fmt.Sprintf("@lxc-to-incus/%s", uuid.New().String()) if len(auds) > linux.ABSTRACT_UNIX_SOCK_LEN-1 { auds = auds[:linux.ABSTRACT_UNIX_SOCK_LEN-1] } l, err := net.Listen("unix", auds) if err != nil { return nil, nil, nil, err } execPath, err := os.Readlink("/proc/self/exe") if err != nil { return nil, nil, nil, err } rsyncCmd := fmt.Sprintf("sh -c \"%s netcat %s\"", execPath, auds) args := []string{ "-ar", "--devices", "--numeric-ids", "--partial", "--sparse", "--xattrs", "--delete", "--compress", "--compress-level=2", } args = append(args, "--filter=-x security.selinux", "--ignore-missing-args") if rsyncArgs != "" { args = append(args, strings.Split(rsyncArgs, " ")...) } args = append(args, []string{path, "localhost:/tmp/foo"}...) args = append(args, []string{"-e", rsyncCmd}...) cmd := exec.Command("rsync", args...) cmd.Stdout = os.Stderr stderr, err := cmd.StderrPipe() if err != nil { return nil, nil, nil, err } err = cmd.Start() if err != nil { return nil, nil, nil, err } conn, err := l.Accept() if err != nil { _ = cmd.Process.Kill() _ = cmd.Wait() return nil, nil, nil, err } _ = l.Close() return cmd, conn, stderr, nil } func protoSendError(webSocket *websocket.Conn, err error) { migration.ProtoSendControl(webSocket, err) if err != nil { closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") _ = webSocket.WriteMessage(websocket.CloseMessage, closeMsg) _ = webSocket.Close() } } incus-7.0.0/cmd/lxc-to-incus/utils.go000066400000000000000000000035161517523235500174330ustar00rootroot00000000000000package main import ( "errors" "fmt" "reflect" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/shared/api" ) func transferRootfs(dst incus.InstanceServer, op incus.Operation, rootfs string, rsyncArgs string) error { opAPI := op.Get() // Connect to the websockets wsControl, err := op.GetWebsocket(opAPI.Metadata[api.SecretNameControl].(string)) if err != nil { return err } abort := func(err error) error { protoSendError(wsControl, err) return err } wsFs, err := op.GetWebsocket(opAPI.Metadata[api.SecretNameFilesystem].(string)) if err != nil { return abort(err) } // Setup control struct fs := migration.MigrationFSType_RSYNC rsyncHasFeature := true offerHeader := migration.MigrationHeader{ Fs: &fs, RsyncFeatures: &migration.RsyncFeatures{ Xattrs: &rsyncHasFeature, Delete: &rsyncHasFeature, Compress: &rsyncHasFeature, }, } err = migration.ProtoSend(wsControl, &offerHeader) if err != nil { return abort(err) } var respHeader migration.MigrationHeader err = migration.ProtoRecv(wsControl, &respHeader) if err != nil { return abort(err) } rsyncFeaturesOffered := offerHeader.GetRsyncFeaturesSlice() rsyncFeaturesResponse := respHeader.GetRsyncFeaturesSlice() if !reflect.DeepEqual(rsyncFeaturesOffered, rsyncFeaturesResponse) { return abort(fmt.Errorf("Offered rsync features (%v) differ from those in the migration response (%v)", rsyncFeaturesOffered, rsyncFeaturesResponse)) } // Send the filesystem err = rsyncSend(wsFs, rootfs, rsyncArgs) if err != nil { return abort(err) } // Check the result msg := migration.MigrationControl{} err = migration.ProtoRecv(wsControl, &msg) if err != nil { _ = wsControl.Close() return err } if !msg.GetSuccess() { return errors.New(msg.GetMessage()) } return nil } incus-7.0.0/cmd/lxd-to-incus/000077500000000000000000000000001517523235500157405ustar00rootroot00000000000000incus-7.0.0/cmd/lxd-to-incus/db.go000066400000000000000000000061211517523235500166540ustar00rootroot00000000000000package main import ( "encoding/binary" "fmt" "io" "os" "path/filepath" "github.com/pierrec/lz4/v4" internalIO "github.com/lxc/incus/v7/internal/io" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/util" ) // Uncompress the raft snapshot files in the given database directory. // // A backup will be created and kept around in case of errors. func migrateDatabase(dir string) error { global := filepath.Join(dir, "global") err := internalUtil.DirCopy(global, global+".bak") if err != nil { return fmt.Errorf("Failed to backup database directory %q: %w", global, err) } files, err := os.ReadDir(global) if err != nil { return fmt.Errorf("Failed to list database directory %q: %w", global, err) } for _, file := range files { var timestamp uint64 var first uint64 var last uint64 if !file.Type().IsRegular() { continue } n, err := fmt.Sscanf(file.Name(), "snapshot-%d-%d-%d\n", ×tamp, &first, &last) if err != nil || n != 3 { continue } filename := filepath.Join(global, file.Name()) err = lz4Uncompress(filename) if err != nil { return fmt.Errorf("Failed to uncompress snapshot %q: %w", filename, err) } } return nil } // Uncompress the given file, preserving its mode and ownership. // // If the file is not lz4-compressed, this is a no-op. func lz4Uncompress(zfilename string) error { zr := lz4.NewReader(nil) zfile, err := os.Open(zfilename) if err != nil { return fmt.Errorf("Failed to open file %q: %w", zfilename, err) } buf := make([]byte, 4) n, err := zfile.Read(buf) if err != nil { return fmt.Errorf("Failed to read header file %q: %w", zfilename, err) } if n != 4 { return fmt.Errorf("Read only %d bytes from %q", n, zfilename) } // Check the file magic, and return now if it's not an lz4 file. magic := binary.LittleEndian.Uint32(buf) if magic != 0x184D2204 { zfile.Close() return nil } off, err := zfile.Seek(0, 0) if err != nil { return fmt.Errorf("Failed to seek %q: %w", zfilename, err) } if off != 0 { return fmt.Errorf("Seek %q to offset: %d", zfilename, off) } zinfo, err := zfile.Stat() if err != nil { return fmt.Errorf("Failed to get file info for %q: %w", zfilename, err) } // use the same mode for the output file mode := zinfo.Mode() _, uid, gid := internalIO.GetOwnerMode(zinfo) filename := zfilename + ".uncompressed" file, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) if err != nil { return fmt.Errorf("Failed to open file %q: %w", filename, err) } zr.Reset(zfile) _, err = util.SafeCopy(file, zr) if err != nil { return fmt.Errorf("Failed to uncompress %q into %q: %w", zfilename, filename, err) } for _, c := range []io.Closer{zfile, file} { err := c.Close() if err != nil { return fmt.Errorf("Failed to close file: %w", err) } } err = os.Chown(filename, uid, gid) if err != nil { return fmt.Errorf("Failed to set ownership of %q: %w", filename, err) } err = os.Rename(filename, zfilename) if err != nil { return fmt.Errorf("Failed to rename %q to %q: %w", filename, zfilename, err) } return nil } incus-7.0.0/cmd/lxd-to-incus/main.go000066400000000000000000000750261517523235500172250ustar00rootroot00000000000000package main import ( "bufio" "encoding/json" "errors" "fmt" "io/fs" "os" "path/filepath" "slices" "strings" "time" "github.com/spf13/cobra" "golang.org/x/sys/unix" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/ask" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) type cmdGlobal struct { asker ask.Asker flagHelp bool flagVersion bool } func main() { // Setup command line parser. migrateCmd := cmdMigrate{} app := migrateCmd.command() app.Use = "lxd-to-incus" app.Short = "LXD to Incus migration tool" app.Long = `Description: LXD to Incus migration tool This tool allows an existing LXD user to move all their data over to Incus. ` app.SilenceUsage = true app.CompletionOptions = cobra.CompletionOptions{DisableDefaultCmd: true} // Global flags. globalCmd := cmdGlobal{asker: ask.NewAsker(bufio.NewReader(os.Stdin))} migrateCmd.global = globalCmd app.PersistentFlags().BoolVar(&globalCmd.flagVersion, "version", false, "Print version number") app.PersistentFlags().BoolVarP(&globalCmd.flagHelp, "help", "h", false, "Print help") // Version handling. app.SetVersionTemplate("{{.Version}}\n") app.Version = version.Version // Run the main command and handle errors. err := app.Execute() if err != nil { os.Exit(1) } } type cmdMigrate struct { global cmdGlobal flagYes bool flagClusterMember bool flagIgnoreVersionCheck bool } func (c *cmdMigrate) command() *cobra.Command { cmd := &cobra.Command{} cmd.Use = "lxd-to-incus" cmd.RunE = c.run cmd.PersistentFlags().BoolVar(&c.flagYes, "yes", false, "Migrate without prompting") cmd.PersistentFlags().BoolVar(&c.flagClusterMember, "cluster-member", false, "Used internally for cluster migrations") cmd.PersistentFlags().BoolVar(&c.flagIgnoreVersionCheck, "ignore-version-check", false, "Bypass source version check") return cmd } func (c *cmdMigrate) run(app *cobra.Command, args []string) error { var err error var srcClient incus.InstanceServer var targetClient incus.InstanceServer // Confirm that we're root. if os.Geteuid() != 0 { return errors.New("This tool must be run as root") } // Create log file. logFile, err := os.Create(fmt.Sprintf("/var/log/lxd-to-incus.%d.log", os.Getpid())) if err != nil { return fmt.Errorf("Failed to create log file: %w", err) } defer logFile.Close() err = logFile.Chmod(0o600) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to set permissions on log file: %w", err) } if c.flagClusterMember { _, _ = logFile.WriteString("Running in cluster member mode\n") } // Iterate through potential sources. fmt.Println("=> Looking for source server") var source source for _, candidate := range sources { if !candidate.present() { continue } source = candidate break } if source == nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return errors.New("No source server could be found") } fmt.Printf("==> Detected: %s\n", source.name()) _, _ = fmt.Fprintf(logFile, "Source server: %s\n", source.name()) // Iterate through potential targets. fmt.Println("=> Looking for target server") var target target for _, candidate := range targets { if !candidate.present() { continue } target = candidate break } if target == nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return errors.New("No target server could be found") } fmt.Printf("==> Detected: %s\n", target.name()) _, _ = fmt.Fprintf(logFile, "Target server: %s\n", target.name()) // Connect to the servers. clustered := c.flagClusterMember if !c.flagClusterMember { fmt.Println("=> Connecting to source server") srcClient, err = source.connect() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to connect to the source: %w", err) } // Look for API incompatibility (bool in /1.0 config). resp, _, err := srcClient.RawQuery("GET", "/1.0", nil, "") if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to get source server info: %w", err) } type lxdServer struct { Config map[string]any `json:"config"` } s := lxdServer{} err = json.Unmarshal(resp.Metadata, &s) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to parse source server config: %w", err) } badEntries := []string{} for k, v := range s.Config { _, ok := v.(string) if !ok { badEntries = append(badEntries, k) } } if len(badEntries) > 0 { fmt.Println("") fmt.Println("The source server (LXD) has the following configuration keys that are incompatible with Incus:") for _, k := range badEntries { fmt.Printf(" - %s\n", k) } fmt.Println("") fmt.Println("The present migration tool cannot properly connect to the LXD server with those configuration keys present.") fmt.Println("Please unset those configuration keys through the `lxc config unset` command and retry `lxd-to-incus`.") fmt.Println("") _, _ = fmt.Fprintf(logFile, "ERROR: Bad config keys: %v\n", badEntries) return errors.New("Unable to interact with the source server") } // Get the source server info. srcServerInfo, _, err := srcClient.GetServer() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to get source server info: %w", err) } clustered = srcServerInfo.Environment.ServerClustered } if clustered { _, _ = logFile.WriteString("Source server is a cluster\n") } fmt.Println("=> Connecting to the target server") targetClient, err = target.connect() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to connect to the target: %w", err) } // Configuration validation. if !c.flagClusterMember { err = c.validate(source, target) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return err } } // Grab the path information. sourcePaths, err := source.paths() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to get source paths: %w", err) } _, _ = fmt.Fprintf(logFile, "Source server paths: %+v\n", sourcePaths) targetPaths, err := target.paths() if err != nil { return fmt.Errorf("Failed to get target paths: %w", err) } _, _ = fmt.Fprintf(logFile, "Target server paths: %+v\n", targetPaths) // Mangle storage pool sources. rewriteStatements := []string{} rewriteCommands := [][]string{} if !c.flagClusterMember { var storagePools []api.StoragePool if !clustered { storagePools, err = srcClient.GetStoragePools() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Couldn't list storage pools: %w", err) } } else { clusterMembers, err := srcClient.GetClusterMembers() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return errors.New("Failed to retrieve the list of cluster members") } for _, member := range clusterMembers { poolNames, err := srcClient.UseTarget(member.ServerName).GetStoragePoolNames() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Couldn't list storage pools: %w", err) } for _, poolName := range poolNames { pool, _, err := srcClient.UseTarget(member.ServerName).GetStoragePool(poolName) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Couldn't get storage pool: %w", err) } storagePools = append(storagePools, *pool) } } } rbdRenamed := []string{} for _, pool := range storagePools { if pool.Driver == "ceph" { cephCluster, ok := pool.Config["ceph.cluster_name"] if !ok { cephCluster = "ceph" } cephUser, ok := pool.Config["ceph.user.name"] if !ok { cephUser = "admin" } cephPool, ok := pool.Config["ceph.osd.pool_name"] if !ok { cephPool = pool.Name } renameCmd := []string{"rbd", "rename", "--cluster", cephCluster, "--name", fmt.Sprintf("client.%s", cephUser), fmt.Sprintf("%s/lxd_%s", cephPool, cephPool), fmt.Sprintf("%s/incus_%s", cephPool, cephPool)} if !slices.Contains(rbdRenamed, pool.Name) { rewriteCommands = append(rewriteCommands, renameCmd) rbdRenamed = append(rbdRenamed, pool.Name) } } source := pool.Config["source"] if source == "" || source[0] != byte('/') { continue } if !strings.HasPrefix(source, sourcePaths.daemon) { continue } newSource := strings.Replace(source, sourcePaths.daemon, targetPaths.daemon, 1) rewriteStatements = append(rewriteStatements, fmt.Sprintf("UPDATE storage_pools_config SET value='%s' WHERE value='%s';", newSource, source)) } } // Mangle OVN. if !c.flagClusterMember { srcServerInfo, _, err := srcClient.GetServer() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to get source server info: %w", err) } ovnNB, ok := srcServerInfo.Config["network.ovn.northbound_connection"] if !ok && util.PathExists("/run/ovn/ovnnb_db.sock") { ovnNB = "unix:/run/ovn/ovnnb_db.sock" } if ovnNB != "" { if !c.flagClusterMember { out, err := subprocess.RunCommand("ovs-vsctl", "get", "open_vswitch", ".", "external_ids:ovn-remote") if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to get OVN southbound database address: %w", err) } ovnSB := strings.TrimSpace(strings.ReplaceAll(out, "\"", "")) commands, err := ovnConvert(ovnNB, ovnSB) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to prepare OVN conversion: %v", err) } rewriteCommands = append(rewriteCommands, commands...) err = ovnBackup(ovnNB, ovnSB, "/var/backups/") if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to backup the OVN database: %v", err) } } } } // General database updates. if !c.flagClusterMember { // Mangle profile and project descriptions. rewriteStatements = append(rewriteStatements, "UPDATE profiles SET description='Default Incus profile' WHERE description='Default LXD profile';") rewriteStatements = append(rewriteStatements, "UPDATE projects SET description='Default Incus project' WHERE description='Default LXD project';") // Remove volatile.uuid key from storage volumes (not used by Incus). rewriteStatements = append(rewriteStatements, "DELETE FROM storage_volumes_config WHERE key='volatile.uuid';") rewriteStatements = append(rewriteStatements, "DELETE FROM storage_volumes_snapshots_config WHERE key='volatile.uuid';") // Remove volatile.uuid key from instances (not used by Incus). rewriteStatements = append(rewriteStatements, "DELETE FROM instances_config WHERE key='volatile.uuid';") rewriteStatements = append(rewriteStatements, "DELETE FROM instances_snapshots_config WHERE key='volatile.uuid';") } // Mangle database schema to be compatible. if !c.flagClusterMember { srcServerInfo, _, err := srcClient.GetServer() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to get source server info: %w", err) } srcVersion, err := version.Parse(srcServerInfo.Environment.ServerVersion) if err != nil { return fmt.Errorf("Couldn't parse source server version: %w", err) } lxdVersionAuth := &version.DottedVersion{Major: 5, Minor: 21, Patch: 0} if srcVersion.Compare(lxdVersionAuth) >= 0 { // Re-create the certificate tables. rewriteStatements = append(rewriteStatements, `CREATE TABLE certificates ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, fingerprint TEXT NOT NULL, type INTEGER NOT NULL, name TEXT NOT NULL, certificate TEXT NOT NULL, restricted INTEGER NOT NULL DEFAULT 0, UNIQUE (fingerprint) ); CREATE TABLE "certificates_projects" ( certificate_id INTEGER NOT NULL, project_id INTEGER NOT NULL, FOREIGN KEY (certificate_id) REFERENCES certificates (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE, UNIQUE (certificate_id, project_id) );`) // Revert the schema version. rewriteStatements = append(rewriteStatements, `DELETE FROM schema WHERE version < 73; UPDATE schema SET version=69 WHERE version=73;`) // Convert back the entries. rewriteStatements = append(rewriteStatements, `INSERT INTO certificates (id, fingerprint, type, name, certificate, restricted) SELECT id, identifier, 1, name, json_extract(metadata, "$.cert"), 1 FROM identities WHERE type=1; INSERT INTO certificates (id, fingerprint, type, name, certificate, restricted) SELECT id, identifier, 1, name, json_extract(metadata, "$.cert"), 0 FROM identities WHERE type=2; INSERT INTO certificates (id, fingerprint, type, name, certificate, restricted) SELECT id, identifier, 2, name, json_extract(metadata, "$.cert"), 0 FROM identities WHERE type=3; INSERT INTO certificates (id, fingerprint, type, name, certificate, restricted) SELECT id, identifier, 3, name, json_extract(metadata, "$.cert"), 1 FROM identities WHERE type=4; INSERT INTO certificates (id, fingerprint, type, name, certificate, restricted) SELECT id, identifier, 3, name, json_extract(metadata, "$.cert"), 0 FROM identities WHERE type=6; INSERT INTO certificates_projects (certificate_id, project_id) SELECT identity_id, project_id FROM identities_projects;`) // Drop the other tables. rewriteStatements = append(rewriteStatements, `DROP TRIGGER IF EXISTS on_auth_group_delete; DROP TRIGGER IF EXISTS on_cluster_group_delete; DROP TRIGGER IF EXISTS on_identity_delete; DROP TRIGGER IF EXISTS on_identity_provider_group_delete; DROP TRIGGER IF EXISTS on_image_alias_delete; DROP TRIGGER IF EXISTS on_image_delete; DROP TRIGGER IF EXISTS on_instance_backup_delete; DROP TRIGGER IF EXISTS on_instance_delete; DROP TRIGGER IF EXISTS on_instance_snapshot_delete; DROP TRIGGER IF EXISTS on_instance_snaphot_delete; DROP TRIGGER IF EXISTS on_network_acl_delete; DROP TRIGGER IF EXISTS on_network_delete; DROP TRIGGER IF EXISTS on_network_zone_delete; DROP TRIGGER IF EXISTS on_node_delete; DROP TRIGGER IF EXISTS on_operation_delete; DROP TRIGGER IF EXISTS on_profile_delete; DROP TRIGGER IF EXISTS on_project_delete; DROP TRIGGER IF EXISTS on_storage_bucket_delete; DROP TRIGGER IF EXISTS on_storage_pool_delete; DROP TRIGGER IF EXISTS on_storage_volume_backup_delete; DROP TRIGGER IF EXISTS on_storage_volume_delete; DROP TRIGGER IF EXISTS on_storage_volume_snapshot_delete; DROP TRIGGER IF EXISTS on_warning_delete; DROP TABLE IF EXISTS identities_projects; DROP TABLE IF EXISTS auth_groups_permissions; DROP TABLE IF EXISTS auth_groups_identity_provider_groups; DROP TABLE IF EXISTS identities_auth_groups; DROP TABLE IF EXISTS identity_provider_groups; DROP TABLE IF EXISTS identities; DROP TABLE IF EXISTS auth_groups;`) } } // Log rewrite actions. _, _ = logFile.WriteString("Rewrite SQL statements:\n") for _, entry := range rewriteStatements { _, _ = fmt.Fprintf(logFile, " - %s\n", entry) } _, _ = logFile.WriteString("Rewrite commands:\n") for _, entry := range rewriteCommands { _, _ = fmt.Fprintf(logFile, " - %s\n", strings.Join(entry, " ")) } // Confirm migration. if !c.flagClusterMember && !c.flagYes { if !clustered { fmt.Println(` The migration is now ready to proceed. At this point, the source server and all its instances will be stopped. Instances will come back online once the migration is complete.`) ok, err := c.global.asker.AskBool("Proceed with the migration? [default=no]: ", "no") if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return err } if !ok { _, _ = logFile.WriteString("User aborted migration\n") return errors.New("User aborted migration") } } else { fmt.Println(` The migration is now ready to proceed. A cluster environment was detected. Manual action will be needed on each of the server prior to Incus being functional.`) if os.Getenv("CLUSTER_NO_STOP") != "1" { fmt.Println("The migration will begin by shutting down instances on all servers.") } fmt.Println(` It will then convert the current server over to Incus and then wait for the other servers to be converted. Do not attempt to manually run this tool on any of the other servers in the cluster. Instead this tool will be providing specific commands for each of the servers.`) ok, err := c.global.asker.AskBool("Proceed with the migration? [default=no]: ", "no") if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return err } if !ok { return errors.New("User aborted migration") } } } _, _ = logFile.WriteString("Migration started\n") // Cluster evacuation. if os.Getenv("CLUSTER_NO_STOP") == "1" { _, _ = logFile.WriteString("WARN: User requested no instance stop during migration\n") } if !c.flagClusterMember && clustered && os.Getenv("CLUSTER_NO_STOP") != "1" { fmt.Println("=> Stopping all workloads on the cluster") clusterMembers, err := srcClient.GetClusterMembers() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return errors.New("Failed to retrieve the list of cluster members") } for _, member := range clusterMembers { fmt.Printf("==> Stopping all workloads on server %q\n", member.ServerName) _, _ = fmt.Fprintf(logFile, "Stopping instances on server %qn\n", member.ServerName) op, err := srcClient.UpdateClusterMemberState(member.ServerName, api.ClusterMemberStatePost{Action: "evacuate", Mode: "stop"}) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to stop workloads %q: %w", member.ServerName, err) } err = op.Wait() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to stop workloads %q: %w", member.ServerName, err) } } } // Stop source. fmt.Println("=> Stopping the source server") _, _ = logFile.WriteString("Stopping the source server\n") err = source.stop() if err != nil { return fmt.Errorf("Failed to stop the source server: %w", err) } // Stop target. fmt.Println("=> Stopping the target server") _, _ = logFile.WriteString("Stopping the target server\n") err = target.stop() if err != nil { return fmt.Errorf("Failed to stop the target server: %w", err) } // Unmount potential mount points. for _, mount := range []string{"devlxd", "shmounts"} { _, _ = fmt.Fprintf(logFile, "Unmounting %q\n", filepath.Join(targetPaths.daemon, mount)) _ = unix.Unmount(filepath.Join(targetPaths.daemon, mount), unix.MNT_DETACH) } // Wipe the target. fmt.Println("=> Wiping the target server") _, _ = logFile.WriteString("Wiping the target server\n") err = os.RemoveAll(targetPaths.logs) if err != nil && !errors.Is(err, fs.ErrNotExist) { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to remove %q: %w", targetPaths.logs, err) } err = os.RemoveAll(targetPaths.cache) if err != nil && !errors.Is(err, fs.ErrNotExist) { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to remove %q: %w", targetPaths.cache, err) } err = os.RemoveAll(targetPaths.daemon) if err != nil && !errors.Is(err, fs.ErrNotExist) { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to remove %q: %w", targetPaths.daemon, err) } // Migrate data. fmt.Println("=> Migrating the data") _, _ = logFile.WriteString("Migrating the data\n") _, err = subprocess.RunCommand("mv", sourcePaths.logs, targetPaths.logs) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to move %q to %q: %w", sourcePaths.logs, targetPaths.logs, err) } _, err = subprocess.RunCommand("mv", sourcePaths.cache, targetPaths.cache) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to move %q to %q: %w", sourcePaths.cache, targetPaths.cache, err) } if linux.IsMountPoint(sourcePaths.daemon) { _, _ = logFile.WriteString("Source daemon path is a mountpoint\n") err = os.MkdirAll(targetPaths.daemon, 0o711) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to create target directory: %w", err) } _, _ = logFile.WriteString("Creating bind-mount of daemon path\n") err = unix.Mount(sourcePaths.daemon, targetPaths.daemon, "none", unix.MS_BIND|unix.MS_REC, "") if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to bind mount %q to %q: %w", sourcePaths.daemon, targetPaths.daemon, err) } _, _ = logFile.WriteString("Unmounting former mountpoint\n") err = unix.Unmount(sourcePaths.daemon, unix.MNT_DETACH) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to unmount source mount %q: %w", sourcePaths.daemon, err) } fmt.Println("") fmt.Printf("WARNING: %s was detected to be a mountpoint.\n", sourcePaths.daemon) fmt.Printf("The migration logic has moved this mount to the new target path at %s.\n", targetPaths.daemon) fmt.Print("However it is your responsibility to modify your system settings to ensure this mount will be properly restored on reboot.\n") fmt.Println("") } else { _, _ = logFile.WriteString("Moving data over\n") _, err = subprocess.RunCommand("mv", sourcePaths.daemon, targetPaths.daemon) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to move %q to %q: %w", sourcePaths.daemon, targetPaths.daemon, err) } } // Migrate database format. fmt.Println("=> Migrating database") _, _ = logFile.WriteString("Migrating database files\n") _, err = subprocess.RunCommand("cp", "-R", filepath.Join(targetPaths.daemon, "database"), filepath.Join(targetPaths.daemon, "database.pre-migrate")) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to backup the database: %w", err) } err = migrateDatabase(filepath.Join(targetPaths.daemon, "database")) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to migrate database in %q: %w", filepath.Join(targetPaths.daemon, "database"), err) } // Apply custom migration statements. if len(rewriteStatements) > 0 { fmt.Println("=> Writing database patch") _, _ = logFile.WriteString("Writing the database patch\n") err = os.WriteFile(filepath.Join(targetPaths.daemon, "database", "patch.global.sql"), []byte(strings.Join(rewriteStatements, "\n")+"\n"), 0o600) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to write database path: %w", err) } } if len(rewriteCommands) > 0 { fmt.Println("=> Running data migration commands") _, _ = logFile.WriteString("Running data migration commands:\n") failures := 0 for _, cmd := range rewriteCommands { _, _ = fmt.Fprintf(logFile, " - %+v\n", cmd) _, err := subprocess.RunCommand(cmd[0], cmd[1:]...) if err != nil { _, _ = fmt.Fprintf(logFile, "Failed to run command: %v\n", err) failures++ } } if failures > 0 { fmt.Printf("==> WARNING: %d commands out of %d succeeded (%d failures)\n", len(rewriteCommands)-failures, len(rewriteCommands), failures) fmt.Println(" Please review the log file for details.") fmt.Println(" Note that in OVN environments, it's normal to see some failures") fmt.Println(" related to Flow Rules and Switch Ports as those often change during the migration.") } } // Cleanup paths. fmt.Println("=> Cleaning up target paths") _, _ = logFile.WriteString("Cleaning up target paths\n") for _, dir := range []string{"backups", "images"} { _, _ = fmt.Fprintf(logFile, "Cleaning up path %q\n", filepath.Join(targetPaths.daemon, dir)) // Remove any potential symlink (ignore errors for real directories). _ = os.Remove(filepath.Join(targetPaths.daemon, dir)) } for _, dir := range []string{"devices", "devlxd", "security", "shmounts"} { _, _ = fmt.Fprintf(logFile, "Cleaning up path %q\n", filepath.Join(targetPaths.daemon, dir)) _ = unix.Unmount(filepath.Join(targetPaths.daemon, dir), unix.MNT_DETACH) err = os.RemoveAll(filepath.Join(targetPaths.daemon, dir)) if err != nil && !errors.Is(err, fs.ErrNotExist) { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to delete %q: %w", dir, err) } } for _, dir := range []string{"containers", "containers-snapshots", "snapshots", "virtual-machines", "virtual-machines-snapshots"} { entries, err := os.ReadDir(filepath.Join(targetPaths.daemon, dir)) if err != nil { if errors.Is(err, fs.ErrNotExist) { continue } _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to read entries in %q: %w", filepath.Join(targetPaths.daemon, dir), err) } _, _ = logFile.WriteString("Rewrite symlinks:\n") for _, entry := range entries { srcPath := filepath.Join(targetPaths.daemon, dir, entry.Name()) if entry.Type()&os.ModeSymlink != os.ModeSymlink { continue } oldTarget, err := os.Readlink(srcPath) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to resolve symlink %q: %w", srcPath, err) } newTarget := strings.Replace(oldTarget, sourcePaths.daemon, targetPaths.daemon, 1) err = os.Remove(srcPath) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to delete symlink %q: %w", srcPath, err) } _, _ = fmt.Fprintf(logFile, " - %q to %q\n", newTarget, srcPath) err = os.Symlink(newTarget, srcPath) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to create symlink %q: %w", srcPath, err) } } } // Cleanup the cache. cacheEntries, err := os.ReadDir(targetPaths.cache) if err == nil { for _, entry := range cacheEntries { if !entry.IsDir() { continue } err := os.RemoveAll(filepath.Join(targetPaths.cache, entry.Name())) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to clear cache file %q: %w", filepath.Join(targetPaths.cache, entry.Name()), err) } } } // Start target. fmt.Println("=> Starting the target server") _, _ = logFile.WriteString("Starting the target server\n") err = target.start() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to start the target server: %w", err) } // Cluster handling. if clustered { if !c.flagClusterMember { _, _ = logFile.WriteString("Waiting for user to run command on other cluster members\n") fmt.Println("=> Waiting for other cluster servers") fmt.Println("") fmt.Print("Please run `lxd-to-incus --cluster-member` on all other servers in the cluster\n\n") for { ok, err := c.global.asker.AskBool("The command has been started on all other servers? [default=no]: ", "no") if !ok || err != nil { continue } break } fmt.Println("") _, _ = logFile.WriteString("User confirmed command was run on other members\n") } // Wait long enough that we get accurate heartbeat information. fmt.Println("=> Waiting for cluster to be fully migrated") _, _ = logFile.WriteString("Waiting for cluster to come back online\n") time.Sleep(30 * time.Second) for { clusterMembers, err := targetClient.GetClusterMembers() if err != nil { time.Sleep(30 * time.Second) continue } ready := true for _, member := range clusterMembers { info, _, err := targetClient.UseTarget(member.ServerName).GetServer() if err != nil || info.Environment.Server != "incus" { ready = false break } if member.Status == "Evacuated" && member.Message == "Unavailable due to maintenance" { continue } if member.Status == "Online" && member.Message == "Fully operational" { continue } ready = false break } if !ready { time.Sleep(30 * time.Second) continue } break } } // Validate target. fmt.Println("=> Checking the target server") _, _ = logFile.WriteString("Checking target server\n") targetServerInfo, _, err := targetClient.GetServer() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to get target server info: %w", err) } // Fix OVS. ovnNB, ok := targetServerInfo.Config["network.ovn.northbound_connection"] if ok && ovnNB != "" { commands, err := ovsConvert() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to prepare OVS conversion: %v", err) } _, _ = logFile.WriteString("Running OVS conversion commands:\n") for _, cmd := range commands { _, _ = fmt.Fprintf(logFile, " - %+v\n", cmd) _, err := subprocess.RunCommand(cmd[0], cmd[1:]...) if err != nil { _, _ = fmt.Fprintf(logFile, "Failed to run command: %v\n", err) fmt.Fprintf(os.Stderr, "Failed to run command: %v\n", err) } } } // Cluster restore. if !c.flagClusterMember && clustered { fmt.Println("=> Restoring the cluster") _, _ = logFile.WriteString("Restoring cluster state\n") clusterMembers, err := targetClient.GetClusterMembers() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return errors.New("Failed to retrieve the list of cluster members") } for _, member := range clusterMembers { fmt.Printf("==> Restoring workloads on server %q\n", member.ServerName) _, _ = fmt.Fprintf(logFile, "Restoring workloads on %q\n", member.ServerName) op, err := targetClient.UpdateClusterMemberState(member.ServerName, api.ClusterMemberStatePost{Action: "restore"}) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to restore %q: %w", member.ServerName, err) } err = op.Wait() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to restore %q: %w", member.ServerName, err) } } } // Writing completion stamp file. completeFile, err := os.Create(filepath.Join(targetPaths.daemon, ".migrated-from-lxd")) if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) } defer completeFile.Close() // Confirm uninstall. if !c.flagYes { ok, err := c.global.asker.AskBool("Uninstall the LXD package? [default=no]: ", "no") if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return err } if !ok { _, _ = logFile.WriteString("User decided not ro remove the package\n") return nil } } // Purge source. fmt.Println("=> Uninstalling the source server") _, _ = logFile.WriteString("Uninstalling the source package\n") err = source.purge() if err != nil { _, _ = fmt.Fprintf(logFile, "ERROR: %v\n", err) return fmt.Errorf("Failed to uninstall the source server: %w", err) } return nil } incus-7.0.0/cmd/lxd-to-incus/ovn.go000066400000000000000000000162041517523235500170740ustar00rootroot00000000000000package main import ( "context" "encoding/csv" "fmt" "os" "path/filepath" "strings" "github.com/lxc/incus/v7/shared/subprocess" ) func ovsConvert() ([][]string, error) { commands := [][]string{} output, err := subprocess.RunCommand("ovs-vsctl", "get", "open_vswitch", ".", "external-ids:ovn-bridge-mappings") if err != nil { // Assume that being unable to read the key means it's not set. return nil, nil } oldValue := strings.TrimSpace(strings.ReplaceAll(output, "\"", "")) oldBridges := []string{} values := strings.Split(oldValue, ",") for i, value := range values { fields := strings.Split(value, ":") oldBridges = append(oldBridges, fields[1]) fields[1] = strings.ReplaceAll(fields[1], "lxdovn", "incusovn") values[i] = strings.Join(fields, ":") } newValue := strings.Join(values, ",") if oldValue != newValue { commands = append(commands, []string{"ovs-vsctl", "set", "open_vswitch", ".", fmt.Sprintf("external-ids:ovn-bridge-mappings=%s", newValue)}) } for _, bridge := range oldBridges { commands = append(commands, []string{"ovs-vsctl", "del-br", bridge}) } return commands, nil } func ovnBackup(nbDB string, sbDB string, target string) error { // Backup the Northbound database. nbStdout, err := os.Create(filepath.Join(target, fmt.Sprintf("lxd-to-incus.ovn-nb.%d.backup", os.Getpid()))) if err != nil { return err } defer nbStdout.Close() err = nbStdout.Chmod(0o600) if err != nil { return err } args := []string{"dump", "-f", "csv", nbDB, "OVN_Northbound"} if strings.Contains(nbDB, "ssl:") { args = append(args, "-c", "/etc/ovn/cert_host") args = append(args, "-p", "/etc/ovn/key_host") args = append(args, "-C", "/etc/ovn/ovn-central.crt") } err = subprocess.RunCommandWithFds(context.Background(), nil, nbStdout, "ovsdb-client", args...) if err != nil { return err } // Backup the Southbound database. sbStdout, err := os.Create(filepath.Join(target, fmt.Sprintf("lxd-to-incus.ovn-sb.%d.backup", os.Getpid()))) if err != nil { return err } defer sbStdout.Close() err = sbStdout.Chmod(0o600) if err != nil { return err } args = []string{"dump", "-f", "csv", sbDB, "OVN_Southbound"} if strings.Contains(sbDB, "ssl:") { args = append(args, "-c", "/etc/ovn/cert_host") args = append(args, "-p", "/etc/ovn/key_host") args = append(args, "-C", "/etc/ovn/ovn-central.crt") } err = subprocess.RunCommandWithFds(context.Background(), nil, sbStdout, "ovsdb-client", args...) if err != nil { return err } return nil } func ovnConvert(nbDB string, sbDB string) ([][]string, error) { commands := [][]string{} // Patch the Northbound records. args := []string{"dump", "-f", "csv", nbDB, "OVN_Northbound"} if strings.Contains(sbDB, "ssl:") { args = append(args, "-c", "/etc/ovn/cert_host") args = append(args, "-p", "/etc/ovn/key_host") args = append(args, "-C", "/etc/ovn/ovn-central.crt") } output, err := subprocess.RunCommand("ovsdb-client", args...) if err != nil { return nil, err } data, err := ovnParseDump(output) if err != nil { return nil, err } for table, records := range data { for _, record := range records { for k, v := range record { needsFixing, newValue, err := ovnCheckValue(table, k, v) if err != nil { return nil, err } if needsFixing { cmd := []string{"ovn-nbctl", "--db", nbDB} if strings.Contains(nbDB, "ssl:") { cmd = append(cmd, "-c", "/etc/ovn/cert_host") cmd = append(cmd, "-p", "/etc/ovn/key_host") cmd = append(cmd, "-C", "/etc/ovn/ovn-central.crt") } cmd = append(cmd, []string{"set", table, record["_uuid"], fmt.Sprintf("%s=%s", k, newValue)}...) commands = append(commands, cmd) } } } } // Patch the Southbound records. args = []string{"dump", "-f", "csv", sbDB, "OVN_Southbound"} if strings.Contains(sbDB, "ssl:") { args = append(args, "-c", "/etc/ovn/cert_host") args = append(args, "-p", "/etc/ovn/key_host") args = append(args, "-C", "/etc/ovn/ovn-central.crt") } output, err = subprocess.RunCommand("ovsdb-client", args...) if err != nil { return nil, err } data, err = ovnParseDump(output) if err != nil { return nil, err } for table, records := range data { for _, record := range records { for k, v := range record { needsFixing, newValue, err := ovnCheckValue(table, k, v) if err != nil { return nil, err } if needsFixing { cmd := []string{"ovn-sbctl", "--db", sbDB} if strings.Contains(sbDB, "ssl:") { cmd = append(cmd, "-c", "/etc/ovn/cert_host") cmd = append(cmd, "-p", "/etc/ovn/key_host") cmd = append(cmd, "-C", "/etc/ovn/ovn-central.crt") } cmd = append(cmd, []string{"set", table, record["_uuid"], fmt.Sprintf("%s=%s", k, newValue)}...) commands = append(commands, cmd) } } } } return commands, nil } func ovnCheckValue(table string, k string, v string) (bool, string, error) { if !strings.Contains(v, "lxd") { return false, "", nil } if table == "DNS" && k == "records" { return false, "", nil } if table == "Chassis" && k == "other_config" { return false, "", nil } if table == "Chassis" && k == "external_ids" { return false, "", nil } if table == "Logical_Flow" && k == "actions" { return false, "", nil } if table == "DHCP_Options" && k == "options" { return false, "", nil } if table == "Logical_Router_Port" && k == "ipv6_ra_configs" { return false, "", nil } if (table == "Logical_Switch_Port" || table == "Port_Binding") && k == "options" && (v == "{network_name=lxdbr0}" || v == "{network_name=lxdbr1}") { return false, "", nil } newValue := strings.ReplaceAll(v, "lxd-net", "incus-net") newValue = strings.ReplaceAll(newValue, "lxd_acl", "incus_acl") newValue = strings.ReplaceAll(newValue, "lxd_location", "incus_location") newValue = strings.ReplaceAll(newValue, "lxd_net", "incus_net") newValue = strings.ReplaceAll(newValue, "lxd_port_group", "incus_port_group") newValue = strings.ReplaceAll(newValue, "lxd_project_id", "incus_project_id") newValue = strings.ReplaceAll(newValue, "lxd_switch", "incus_switch") newValue = strings.ReplaceAll(newValue, "lxd_switch_port", "incus_switch_port") if v == newValue { return true, "", fmt.Errorf("Couldn't convert value %q for key %q in table %q", v, k, table) } return true, newValue, nil } func ovnParseDump(data string) (map[string][]map[string]string, error) { output := map[string][]map[string]string{} tableName := "" fields := []string{} newTable := false for _, line := range strings.Split(data, "\n") { if line == "" { continue } if !strings.Contains(line, ",") && strings.HasSuffix(line, " table") { newTable = true tableName = strings.Split(line, " ")[0] output[tableName] = []map[string]string{} continue } if newTable { newTable = false var err error fields, err = csv.NewReader(strings.NewReader(line)).Read() if err != nil { return nil, err } continue } record := map[string]string{} entry, err := csv.NewReader(strings.NewReader(line)).Read() if err != nil { return nil, err } for k, v := range entry { record[fields[k]] = v } output[tableName] = append(output[tableName], record) } return output, nil } incus-7.0.0/cmd/lxd-to-incus/paths.go000066400000000000000000000001271517523235500174060ustar00rootroot00000000000000package main type daemonPaths struct { daemon string logs string cache string } incus-7.0.0/cmd/lxd-to-incus/sources.go000066400000000000000000000005231517523235500177520ustar00rootroot00000000000000package main import incus "github.com/lxc/incus/v7/client" type source interface { present() bool stop() error start() error purge() error connect() (incus.InstanceServer, error) paths() (*daemonPaths, error) name() string } var sources = []source{ &srcSnap{}, &srcDeb{}, &srcCOPR{}, &srcXbps{}, &srcAPK{}, &srcManual{}, } incus-7.0.0/cmd/lxd-to-incus/sources_apk.go000066400000000000000000000020661517523235500206110ustar00rootroot00000000000000package main import ( incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) type srcAPK struct{} func (s *srcAPK) present() bool { if !util.PathExists("/var/lib/incus/") { return false } _, err := subprocess.RunCommand("rc-service", "--exists", "incusd") return err == nil } func (s *srcAPK) name() string { return "apk package" } func (s *srcAPK) stop() error { _, err := subprocess.RunCommand("rc-service", "lxd", "stop") return err } func (s *srcAPK) start() error { _, err := subprocess.RunCommand("rc-service", "lxd", "start") return err } func (s *srcAPK) purge() error { _, err := subprocess.RunCommand("apk", "del", "lxd", "lxd-client") return err } func (s *srcAPK) connect() (incus.InstanceServer, error) { return incus.ConnectIncusUnix("/var/lib/lxd/unix.socket", &incus.ConnectionArgs{SkipGetServer: true}) } func (s *srcAPK) paths() (*daemonPaths, error) { return &daemonPaths{ daemon: "/var/lib/lxd", logs: "/var/log/lxd", cache: "/var/cache/lxd", }, nil } incus-7.0.0/cmd/lxd-to-incus/sources_copr.go000066400000000000000000000022761517523235500210040ustar00rootroot00000000000000package main import ( incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) type srcCOPR struct{} func (s *srcCOPR) present() bool { // Validate that the RPM package is installed. _, err := subprocess.RunCommand("rpm", "-q", "lxd") if err != nil { return false } if !util.PathExists("/run/lxd.socket") { return false } return true } func (s *srcCOPR) name() string { return "COPR package" } func (s *srcCOPR) stop() error { _, err := subprocess.RunCommand("systemctl", "stop", "lxd-containers.service", "lxd.service", "lxd.socket") return err } func (s *srcCOPR) start() error { _, err := subprocess.RunCommand("systemctl", "start", "lxd.socket", "lxd-containers.service") return err } func (s *srcCOPR) purge() error { _, err := subprocess.RunCommand("dnf", "remove", "-y", "lxd") return err } func (s *srcCOPR) connect() (incus.InstanceServer, error) { return incus.ConnectIncusUnix("/run/lxd.socket", &incus.ConnectionArgs{SkipGetServer: true}) } func (s *srcCOPR) paths() (*daemonPaths, error) { return &daemonPaths{ daemon: "/var/lib/lxd", logs: "/var/log/lxd", cache: "/var/cache/lxd", }, nil } incus-7.0.0/cmd/lxd-to-incus/sources_deb.go000066400000000000000000000023171517523235500205670ustar00rootroot00000000000000package main import ( incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) type srcDeb struct{} func (s *srcDeb) present() bool { // Validate that the Debian package is installed. if !util.PathExists("/var/lib/dpkg/info/lxd.list") { return false } if !util.PathExists("/var/lib/lxd") { return false } return true } func (s *srcDeb) name() string { return ".deb package" } func (s *srcDeb) stop() error { _, err := subprocess.RunCommand("systemctl", "stop", "lxd-containers.service", "lxd.service", "lxd.socket") return err } func (s *srcDeb) start() error { _, err := subprocess.RunCommand("systemctl", "start", "lxd.socket", "lxd-containers.service") return err } func (s *srcDeb) purge() error { _, err := subprocess.RunCommand("apt-get", "remove", "--yes", "--purge", "lxd", "lxd-client") return err } func (s *srcDeb) connect() (incus.InstanceServer, error) { return incus.ConnectIncusUnix("/var/lib/lxd/unix.socket", &incus.ConnectionArgs{SkipGetServer: true}) } func (s *srcDeb) paths() (*daemonPaths, error) { return &daemonPaths{ daemon: "/var/lib/lxd", logs: "/var/log/lxd", cache: "/var/cache/lxd", }, nil } incus-7.0.0/cmd/lxd-to-incus/sources_manual.go000066400000000000000000000024171517523235500213130ustar00rootroot00000000000000package main import ( "errors" "net/http" "time" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/util" ) type srcManual struct{} func (s *srcManual) present() bool { return util.PathExists("/var/lib/lxd") } func (s *srcManual) name() string { return "manual installation" } func (s *srcManual) stop() error { d, err := s.connect() if err != nil { return err } httpClient, err := d.GetHTTPClient() if err != nil { return err } // Request shutdown, this shouldn't return until daemon has stopped so use a large request timeout. httpTransport, ok := httpClient.Transport.(*http.Transport) if !ok { return errors.New("Bad transport type") } httpTransport.ResponseHeaderTimeout = 3600 * time.Second _, _, err = d.RawQuery("PUT", "/internal/shutdown", nil, "") if err != nil { return err } return nil } func (s *srcManual) start() error { return nil } func (s *srcManual) purge() error { return nil } func (s *srcManual) connect() (incus.InstanceServer, error) { return incus.ConnectIncusUnix("/var/lib/lxd/unix.socket", &incus.ConnectionArgs{SkipGetServer: true}) } func (s *srcManual) paths() (*daemonPaths, error) { return &daemonPaths{ daemon: "/var/lib/lxd", logs: "/var/log/lxd", cache: "/var/cache/lxd", }, nil } incus-7.0.0/cmd/lxd-to-incus/sources_snap.go000066400000000000000000000022571517523235500210010ustar00rootroot00000000000000package main import ( incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) type srcSnap struct{} func (s *srcSnap) present() bool { // Validate that the snap is installed. if !util.PathExists("/snap/lxd") && !util.PathExists("/var/lib/snapd/snap/lxd") { return false } if !util.PathExists("/var/snap/lxd") { return false } return true } func (s *srcSnap) name() string { return "snap package" } func (s *srcSnap) stop() error { _, err := subprocess.RunCommand("snap", "stop", "lxd") return err } func (s *srcSnap) start() error { _, err := subprocess.RunCommand("snap", "start", "lxd") return err } func (s *srcSnap) purge() error { _, err := subprocess.RunCommand("snap", "remove", "lxd", "--purge") return err } func (s *srcSnap) connect() (incus.InstanceServer, error) { return incus.ConnectIncusUnix("/var/snap/lxd/common/lxd/unix.socket", &incus.ConnectionArgs{SkipGetServer: true}) } func (s *srcSnap) paths() (*daemonPaths, error) { return &daemonPaths{ daemon: "/var/snap/lxd/common/lxd", logs: "/var/snap/lxd/common/lxd/logs", cache: "/var/snap/lxd/common/lxd/cache", }, nil } incus-7.0.0/cmd/lxd-to-incus/sources_xbps.go000066400000000000000000000021551517523235500210110ustar00rootroot00000000000000package main import ( incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) type srcXbps struct{} func (s *srcXbps) present() bool { if !util.PathExists("/var/db/xbps/.lxd-files.plist") { return false } if !util.PathExists("/var/service/lxd") { return false } if !util.PathExists("/var/lib/lxd/unix.socket") { return false } return true } func (s *srcXbps) name() string { return "xbps" } func (s *srcXbps) stop() error { _, err := subprocess.RunCommand("sv", "stop", "lxd") return err } func (s *srcXbps) start() error { _, err := subprocess.RunCommand("sv", "start", "lxd") return err } func (s *srcXbps) purge() error { _, err := subprocess.RunCommand("xbps-remove", "-R", "-y", "lxd") return err } func (s *srcXbps) connect() (incus.InstanceServer, error) { return incus.ConnectIncusUnix("/var/lib/lxd/unix.socket", &incus.ConnectionArgs{SkipGetServer: true}) } func (s *srcXbps) paths() (*daemonPaths, error) { return &daemonPaths{ daemon: "/var/lib/lxd", logs: "/var/log/lxd", cache: "/var/cache/lxd", }, nil } incus-7.0.0/cmd/lxd-to-incus/targets.go000066400000000000000000000004531517523235500177420ustar00rootroot00000000000000package main import incus "github.com/lxc/incus/v7/client" type target interface { present() bool stop() error start() error connect() (incus.InstanceServer, error) paths() (*daemonPaths, error) name() string } var targets = []target{ &targetSystemd{}, &targetOpenRC{}, &targetXbps{}, } incus-7.0.0/cmd/lxd-to-incus/targets_openrc.go000066400000000000000000000026301517523235500213070ustar00rootroot00000000000000package main import ( "time" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) type targetOpenRC struct { service string } func (s *targetOpenRC) present() bool { if !util.PathExists("/var/lib/incus/") { return false } _, err := subprocess.RunCommand("rc-service", "--exists", "incus") if err == nil { s.service = "incus" return true } _, err = subprocess.RunCommand("rc-service", "--exists", "incusd") if err == nil { s.service = "incusd" return true } return false } func (s *targetOpenRC) stop() error { _, err := subprocess.RunCommand("rc-service", s.service, "stop") if err != nil { return err } // Wait for the service to fully stop. time.Sleep(5 * time.Second) return nil } func (s *targetOpenRC) start() error { _, err := subprocess.RunCommand("rc-service", s.service, "start") if err != nil { return err } // Wait for the socket to become available. time.Sleep(5 * time.Second) return nil } func (s *targetOpenRC) connect() (incus.InstanceServer, error) { return incus.ConnectIncusUnix("/var/lib/incus/unix.socket", &incus.ConnectionArgs{SkipGetServer: true}) } func (s *targetOpenRC) paths() (*daemonPaths, error) { return &daemonPaths{ daemon: "/var/lib/incus", logs: "/var/log/incus", cache: "/var/cache/incus", }, nil } func (s *targetOpenRC) name() string { return "openrc" } incus-7.0.0/cmd/lxd-to-incus/targets_systemd.go000066400000000000000000000025061517523235500215130ustar00rootroot00000000000000package main import ( "time" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) type targetSystemd struct{} func (s *targetSystemd) present() bool { if !util.PathExists("/var/lib/incus/") { return false } _, err := subprocess.RunCommand("systemctl", "list-unit-files", "incus.service") return err == nil } func (s *targetSystemd) stop() error { _, err := subprocess.RunCommand("systemctl", "stop", "incus.service", "incus.socket") return err } func (s *targetSystemd) start() error { _, err := subprocess.RunCommand("systemctl", "start", "incus.service", "incus.socket") if err != nil { return err } // Wait for the socket to become available. time.Sleep(5 * time.Second) return nil } func (s *targetSystemd) connect() (incus.InstanceServer, error) { if util.PathExists("/run/incus/unix.socket") { return incus.ConnectIncusUnix("/run/incus/unix.socket", &incus.ConnectionArgs{SkipGetServer: true}) } return incus.ConnectIncusUnix("/var/lib/incus/unix.socket", &incus.ConnectionArgs{SkipGetServer: true}) } func (s *targetSystemd) paths() (*daemonPaths, error) { return &daemonPaths{ daemon: "/var/lib/incus", logs: "/var/log/incus", cache: "/var/cache/incus", }, nil } func (s *targetSystemd) name() string { return "systemd" } incus-7.0.0/cmd/lxd-to-incus/targets_xbps.go000066400000000000000000000022261517523235500207760ustar00rootroot00000000000000package main import ( "time" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) type targetXbps struct{} func (s *targetXbps) present() bool { if !util.PathExists("/var/db/xbps/.incus-files.plist") { return false } if !util.PathExists("/var/service/incus") { return false } if !util.PathExists("/var/lib/incus/unix.socket") { return false } return true } func (s *targetXbps) stop() error { _, err := subprocess.RunCommand("sv", "stop", "incus") return err } func (s *targetXbps) start() error { _, err := subprocess.RunCommand("sv", "start", "incus") if err != nil { return err } // Wait for the socket to become available. time.Sleep(5 * time.Second) return nil } func (s *targetXbps) connect() (incus.InstanceServer, error) { return incus.ConnectIncusUnix("/var/lib/incus/unix.socket", &incus.ConnectionArgs{SkipGetServer: true}) } func (s *targetXbps) paths() (*daemonPaths, error) { return &daemonPaths{ daemon: "/var/lib/incus", logs: "/var/log/incus", cache: "/var/cache/incus", }, nil } func (s *targetXbps) name() string { return "xbps" } incus-7.0.0/cmd/lxd-to-incus/validate.go000066400000000000000000000273071517523235500200710ustar00rootroot00000000000000package main import ( "errors" "fmt" "os" "os/exec" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) var ( minLXDVersion = &version.DottedVersion{Major: 4, Minor: 0, Patch: 0} maxLXDVersion = &version.DottedVersion{Major: 5, Minor: 21, Patch: 99} ) func (c *cmdMigrate) validate(source source, target target) error { srcClient, err := source.connect() if err != nil { return fmt.Errorf("Failed to connect to source: %v", err) } targetClient, err := target.connect() if err != nil { return fmt.Errorf("Failed to connect to target: %v", err) } // Get versions. fmt.Println("=> Checking server versions") srcServerInfo, _, err := srcClient.GetServer() if err != nil { return fmt.Errorf("Failed getting source server info: %w", err) } targetServerInfo, _, err := targetClient.GetServer() if err != nil { return fmt.Errorf("Failed getting target server info: %w", err) } fmt.Printf("==> Source version: %s\n", srcServerInfo.Environment.ServerVersion) fmt.Printf("==> Target version: %s\n", targetServerInfo.Environment.ServerVersion) // Compare versions. fmt.Println("=> Validating version compatibility") srcVersion, err := version.Parse(srcServerInfo.Environment.ServerVersion) if err != nil { return fmt.Errorf("Couldn't parse source server version: %w", err) } if srcVersion.Compare(minLXDVersion) < 0 { return fmt.Errorf("LXD version is lower than minimal version %q", minLXDVersion) } if !c.flagIgnoreVersionCheck { if srcVersion.Compare(maxLXDVersion) > 0 { return fmt.Errorf("LXD version is newer than maximum version %q", maxLXDVersion) } } else { fmt.Println("==> WARNING: User asked to bypass version check") } // Validate source non-empty. srcCheckEmpty := func() (bool, error) { // Check if more than one project. names, err := srcClient.GetProjectNames() if err != nil { return false, err } if len(names) > 1 { return false, nil } // Check if more than one profile. names, err = srcClient.GetProfileNames() if err != nil { return false, err } if len(names) > 1 { return false, nil } // Check if any instance is present. names, err = srcClient.GetInstanceNames(api.InstanceTypeAny) if err != nil { return false, err } if len(names) > 0 { return false, nil } // Check if any storage pool is present. names, err = srcClient.GetStoragePoolNames() if err != nil { return false, err } if len(names) > 0 { return false, nil } // Check if any network is present. networks, err := srcClient.GetNetworks() if err != nil { return false, err } for _, network := range networks { if network.Managed { return false, nil } } return true, nil } fmt.Println("=> Checking that the source server isn't empty") isEmpty, err := srcCheckEmpty() if err != nil { return fmt.Errorf("Failed to check source server: %w", err) } if isEmpty { return errors.New("Source server is empty, migration not needed") } // Validate target empty. targetCheckEmpty := func() (bool, string, error) { // Check if more than one project. names, err := targetClient.GetProjectNames() if err != nil { return false, "", err } if len(names) > 1 { return false, "projects", nil } // Check if more than one profile. names, err = targetClient.GetProfileNames() if err != nil { return false, "", err } if len(names) > 1 { return false, "profiles", nil } // Check if any instance is present. names, err = targetClient.GetInstanceNames(api.InstanceTypeAny) if err != nil { return false, "", err } if len(names) > 0 { return false, "instances", nil } // Check if any storage pool is present. names, err = targetClient.GetStoragePoolNames() if err != nil { return false, "", err } if len(names) > 0 { return false, "storage pools", nil } // Check if any network is present. networks, err := targetClient.GetNetworks() if err != nil { return false, "", err } for _, network := range networks { if network.Managed { return false, "networks", nil } } return true, "", nil } fmt.Println("=> Checking that the target server is empty") isEmpty, found, err := targetCheckEmpty() if err != nil { return fmt.Errorf("Failed to check target server: %w", err) } if !isEmpty { return fmt.Errorf("Target server isn't empty (%s found), can't proceed with migration.", found) } // Validate configuration. validationErrors := []error{} fmt.Println("=> Validating source server configuration") deprecatedConfigs := []string{ "candid.api.key", "candid.api.url", "candid.domains", "candid.expiry", "core.trust_password", "maas.api.key", "maas.api.url", "rbac.agent.url", "rbac.agent.username", "rbac.agent.private_key", "rbac.agent.public_key", "rbac.api.expiry", "rbac.api.key", "rbac.api.url", "rbac.expiry", } for _, key := range deprecatedConfigs { _, ok := srcServerInfo.Config[key] if ok { validationErrors = append(validationErrors, fmt.Errorf("Source server is using deprecated server configuration key %q", key)) } } networks, err := srcClient.GetNetworks() if err != nil { return fmt.Errorf("Couldn't list source networks: %w", err) } deprecatedNetworkConfigs := []string{ "bridge.mode", "fan.overlay_subnet", "fan.underlay_subnet", "fan.type", } for _, network := range networks { if !network.Managed { continue } for _, key := range deprecatedNetworkConfigs { _, ok := network.Config[key] if ok { validationErrors = append(validationErrors, fmt.Errorf("Source server has network %q using deprecated configuration key %q", network.Name, key)) } } } storagePools, err := srcClient.GetStoragePools() if err != nil { return fmt.Errorf("Couldn't list storage pools: %w", err) } for _, pool := range storagePools { switch pool.Driver { case "zfs": _, err = exec.LookPath("zfs") if err != nil { validationErrors = append(validationErrors, fmt.Errorf("Required command %q is missing for storage pool %q", "zfs", pool.Name)) } case "btrfs": _, err = exec.LookPath("btrfs") if err != nil { validationErrors = append(validationErrors, fmt.Errorf("Required command %q is missing for storage pool %q", "btrfs", pool.Name)) } case "ceph", "cephfs", "cephobject": _, err = exec.LookPath("ceph") if err != nil { validationErrors = append(validationErrors, fmt.Errorf("Required command %q is missing for storage pool %q", "ceph", pool.Name)) } case "lvm", "lvmcluster": _, err = exec.LookPath("lvm") if err != nil { validationErrors = append(validationErrors, fmt.Errorf("Required command %q is missing for storage pool %q", "lvm", pool.Name)) } if pool.Driver == "lvmcluster" { _, err = exec.LookPath("lvmlockctl") if err != nil { validationErrors = append(validationErrors, fmt.Errorf("Required command %q is missing for storage pool %q", "lvmlockctl", pool.Name)) } } } } deprecatedInstanceConfigs := []string{ "boot.debug_edk2", "limits.network.priority", "security.devlxd", "security.devlxd.images", } deprecatedInstanceDeviceConfigs := []string{ "maas.subnet.ipv4", "maas.subnet.ipv6", } projects, err := srcClient.GetProjects() if err != nil { return fmt.Errorf("Couldn't list source projects: %w", err) } for _, project := range projects { c := srcClient.UseProject(project.Name) instances, err := c.GetInstances(api.InstanceTypeAny) if err != nil { return fmt.Errorf("Couldn't list instances in project %q: %w", project.Name, err) } for _, inst := range instances { for _, key := range deprecatedInstanceConfigs { _, ok := inst.Config[key] if ok { validationErrors = append(validationErrors, fmt.Errorf("Source server has instance %q in project %q using deprecated configuration key %q", inst.Name, project.Name, key)) } } for deviceName, device := range inst.Devices { for _, key := range deprecatedInstanceDeviceConfigs { _, ok := device[key] if ok { validationErrors = append(validationErrors, fmt.Errorf("Source server has device %q for instance %q in project %q using deprecated configuration key %q", deviceName, inst.Name, project.Name, key)) } } } } profiles, err := c.GetProfiles() if err != nil { return fmt.Errorf("Couldn't list profiles in project %q: %w", project.Name, err) } for _, profile := range profiles { for _, key := range deprecatedInstanceConfigs { _, ok := profile.Config[key] if ok { validationErrors = append(validationErrors, fmt.Errorf("Source server has profile %q in project %q using deprecated configuration key %q", profile.Name, project.Name, key)) } } for deviceName, device := range profile.Devices { for _, key := range deprecatedInstanceDeviceConfigs { _, ok := device[key] if ok { validationErrors = append(validationErrors, fmt.Errorf("Source server has device %q for profile %q in project %q using deprecated configuration key %q", deviceName, profile.Name, project.Name, key)) } } } } } // Cluster validation. if srcServerInfo.Environment.ServerClustered { clusterMembers, err := srcClient.GetClusterMembers() if err != nil { return errors.New("Failed to retrieve the list of cluster members") } for _, member := range clusterMembers { if member.Status != "Online" { if os.Getenv("CLUSTER_NO_STOP") == "1" && member.Status == "Evacuated" { continue } validationErrors = append(validationErrors, fmt.Errorf("Cluster member %q isn't in the online state", member.ServerName)) } } } if len(validationErrors) > 0 { fmt.Println("") fmt.Println("Source server uses obsolete features:") for _, err := range validationErrors { fmt.Printf(" - %s\n", err.Error()) } return errors.New("Source server is using incompatible configuration") } // Storage validation. targetPaths, err := target.paths() if err != nil { return fmt.Errorf("Failed to get target paths: %w", err) } sourcePaths, err := source.paths() if err != nil { return fmt.Errorf("Failed to get source paths: %w", err) } fi, err := os.Lstat(sourcePaths.daemon) if err != nil { return err } if fi.Mode()&os.ModeSymlink != 0 { return fmt.Errorf("The source path %q is a symlink. Incus does not support its daemon directory being a symlink, please switch to a bind-mount.", sourcePaths.daemon) } if linux.IsMountPoint(targetPaths.daemon) { return fmt.Errorf("The target path %q is a mountpoint. This isn't currently supported as the target path needs to be deleted during the migration.", targetPaths.daemon) } srcFilesystem, _ := linux.DetectFilesystem(sourcePaths.daemon) targetFilesystem, _ := linux.DetectFilesystem(targetPaths.daemon) if srcFilesystem == "btrfs" && targetFilesystem != "btrfs" && !linux.IsMountPoint(sourcePaths.daemon) { return errors.New("Source daemon running on btrfs but being moved to non-btrfs target") } // Shiftfs check. if util.PathExists("/sys/module/shiftfs/") { fmt.Println("") fmt.Println("WARNING: The shiftfs kernel module was detected on your system.") fmt.Println(" This may indicate that your LXD installation is using shiftfs") fmt.Println(" to allow shifted passthrough of some disks to your instance.") fmt.Println("") fmt.Println(" Incus does not support shiftfs but instead relies on a recent") fmt.Println(" feature of the Linux kernel instead, VFS idmap.") fmt.Println("") fmt.Println(" If your instances actively rely on shiftfs today, you may need") fmt.Println(" to update to a more recent Linux kernel or ZFS version to keep") fmt.Println(" using this shifted passthrough features.") fmt.Println("") } return nil } incus-7.0.0/doc/000077500000000000000000000000001517523235500134145ustar00rootroot00000000000000incus-7.0.0/doc/.sphinx/000077500000000000000000000000001517523235500150035ustar00rootroot00000000000000incus-7.0.0/doc/.sphinx/.markdownlint/000077500000000000000000000000001517523235500175725ustar00rootroot00000000000000incus-7.0.0/doc/.sphinx/.markdownlint/doc-lint.sh000077500000000000000000000014421517523235500216430ustar00rootroot00000000000000#!/bin/sh -eu if ! command -v mdl >/dev/null; then echo "Install mdl with 'snap install mdl' first." exit 1 fi trap "rm -rf .tmp/" EXIT ## Preprocessing for fn in $(find doc/ -name '*.md'); do mkdir -p "$(dirname ".tmp/$fn")"; sed -n -E '/^\([^)]+\)=$/{ N /\n$/ { P; D } s/\n/\n\n/ P D } p' "$fn" > ".tmp/$fn" done rm -rf .tmp/doc/reference/manpages/ mdl .tmp/doc -sdoc/.sphinx/.markdownlint/style.rb -udoc/.sphinx/.markdownlint/rules.rb --ignore-front-matter > .tmp/errors.txt || true ## Postprocessing sed -i '/^$/,$d' .tmp/errors.txt filtered_errors="$(grep -vxFf doc/.sphinx/.markdownlint/exceptions.txt .tmp/errors.txt)" || true if [ -z "$filtered_errors" ]; then echo "Passed!" exit 0 else echo "Failed!" echo "$filtered_errors" exit 1 fi incus-7.0.0/doc/.sphinx/.markdownlint/exceptions.txt000066400000000000000000000013341517523235500225150ustar00rootroot00000000000000.tmp/doc/howto/import_machines_to_instances.md:106: MD034 Bare URL used .tmp/doc/howto/import_machines_to_instances.md:210: MD034 Bare URL used .tmp/doc/howto/network_forwards.md:52: MD004 Unordered list style .tmp/doc/howto/network_forwards.md:56: MD004 Unordered list style .tmp/doc/howto/network_forwards.md:53: MD005 Inconsistent indentation for list items at the same level .tmp/doc/howto/network_forwards.md:57: MD005 Inconsistent indentation for list items at the same level .tmp/doc/howto/network_forwards.md:53: MD032 Lists should be surrounded by blank lines .tmp/doc/howto/network_forwards.md:57: MD032 Lists should be surrounded by blank lines .tmp/doc/contributing.md:9: MD002 First header should be a top level header incus-7.0.0/doc/.sphinx/.markdownlint/rules.rb000066400000000000000000000024401517523235500212510ustar00rootroot00000000000000rule 'Myst-MD031', 'Fenced code blocks should be surrounded by blank lines' do tags :code, :blank_lines aliases 'blanks-around-fences' check do |doc| errors = [] # Some parsers (including kramdown) have trouble detecting fenced code # blocks without surrounding whitespace, so examine the lines directly. in_code = false fence = nil lines = [''] + doc.lines + [''] lines.each_with_index do |line, linenum| line.strip.match(/^(`{3,}|~{3,})/) unless Regexp.last_match(1) && ( !in_code || (Regexp.last_match(1).slice(0, fence.length) == fence) ) next end fence = in_code ? nil : Regexp.last_match(1) in_code = !in_code if (in_code && !(lines[linenum - 1].empty? || lines[linenum - 1].match(/^[:\-\*]*\s*\% /))) || (!in_code && !(lines[linenum + 1].empty? || lines[linenum + 1].match(/^\s*:/))) errors << linenum end end errors end end rule 'Myst-IDs', 'MyST IDs should be preceded by a blank line' do check do |doc| errors = [] ids = doc.matching_text_element_lines(/^\(.+\)=\s*$/) ids.each do |linenum| if (linenum > 1) && !doc.lines[linenum - 2].empty? errors << linenum end end errors.sort end end incus-7.0.0/doc/.sphinx/.markdownlint/style.rb000066400000000000000000000004121517523235500212540ustar00rootroot00000000000000all exclude_rule 'MD013' exclude_rule 'MD046' exclude_rule 'MD041' exclude_rule 'MD040' exclude_rule 'MD024' exclude_rule 'MD033' exclude_rule 'MD022' exclude_rule 'MD031' rule 'MD026', :punctuation => '.,;:!' rule 'MD003', :style => :atx rule 'MD007', :indent => 3 incus-7.0.0/doc/.sphinx/_extra/000077500000000000000000000000001517523235500162655ustar00rootroot00000000000000incus-7.0.0/doc/.sphinx/_extra/rest-api.yaml000077700000000000000000000000001517523235500237262../../rest-api.yamlustar00rootroot00000000000000incus-7.0.0/doc/.sphinx/_static/000077500000000000000000000000001517523235500164315ustar00rootroot00000000000000incus-7.0.0/doc/.sphinx/_static/custom.css000066400000000000000000000101201517523235500204470ustar00rootroot00000000000000/** Fix the font weight (300 for normal, 400 for slightly bold) **/ div.page, h1, h2, h3, h4, h5, h6, .sidebar-tree .current-page>.reference, button, input, optgroup, select, textarea, th.head { font-weight: 300 } .toc-tree li.scroll-current>.reference, dl.glossary dt, dl.simple dt, dl:not([class]) dt { font-weight: 400; } /** Table styling **/ th.head { text-transform: uppercase; font-size: var(--font-size--small); } table.docutils { border: 0; box-shadow: none; width:100%; } table.docutils td, table.docutils th, table.docutils td:last-child, table.docutils th:last-child, table.docutils td:first-child, table.docutils th:first-child { border-right: none; border-left: none; } /* Allow to centre text horizontally in table data cells */ table.align-center { text-align: center !important; } /** No rounded corners **/ .admonition, code.literal, .sphinx-tabs-tab, .sphinx-tabs-panel, .highlight { border-radius: 0; } /** Admonition styling **/ .admonition { border-top: 1px solid #d9d9d9; border-right: 1px solid #d9d9d9; border-bottom: 1px solid #d9d9d9; } /** Color for the "copy link" symbol next to headings **/ a.headerlink { color: var(--color-brand-primary); } /** Line to the left of the current navigation entry **/ .sidebar-tree li.current-page { border-left: 2px solid var(--color-brand-primary); } /** Some tweaks for issue #16 **/ [role="tablist"] { border-bottom: 1px solid var(--color-sidebar-item-background--hover); } .sphinx-tabs-tab[aria-selected="true"] { border: 0; border-bottom: 2px solid var(--color-brand-primary); background-color: var(--color-sidebar-item-background--current); font-weight:300; } .sphinx-tabs-tab{ color: var(--color-brand-primary); font-weight:300; } .sphinx-tabs-panel { border: 0; border-bottom: 1px solid var(--color-sidebar-item-background--hover); background: var(--color-background-primary); } button.sphinx-tabs-tab:hover { background-color: var(--color-sidebar-item-background--hover); } /** Custom classes to fix scrolling in tables by decreasing the font size or breaking certain columns. Specify the classes in the Markdown file with, for example: ```{rst-class} break-col-4 min-width-4-8 ``` **/ table.dec-font-size { font-size: smaller; } table.break-col-1 td.text-left:first-child { word-break: break-word; } table.break-col-4 td.text-left:nth-child(4) { word-break: break-word; } table.min-width-1-15 td.text-left:first-child { min-width: 15em; } table.min-width-4-8 td.text-left:nth-child(4) { min-width: 8em; } /** Underline for abbreviations **/ abbr[title] { text-decoration: underline solid #cdcdcd; } /** Use the same style for right-details as for left-details **/ .bottom-of-page .right-details { font-size: var(--font-size--small); display: block; } /** Version switcher */ button.version_select { color: var(--color-foreground-primary); background-color: var(--color-toc-background); padding: 5px 10px; border: none; } .version_select:hover, .version_select:focus { background-color: var(--color-sidebar-item-background--hover); } .version_dropdown { position: relative; display: inline-block; text-align: right; font-size: var(--sidebar-item-font-size); } .available_versions { display: none; position: absolute; right: 0px; background-color: var(--color-toc-background); box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2); z-index: 11; } .available_versions a { color: var(--color-foreground-primary); padding: 12px 16px; text-decoration: none; display: block; } .available_versions a:hover {background-color: var(--color-sidebar-item-background--current)} .show {display:block;} /** Fix for nested numbered list - the nested list is lettered **/ ol.arabic ol.arabic { list-style: lower-alpha; } /** Make expandable sections look like links **/ details summary { color: var(--color-link); } .rst-versions .rst-current-version { color: var(--color-version-popup); font-weight: bolder; } /** Hide Expand all options **/ #expand-options { display: none; } incus-7.0.0/doc/.sphinx/_static/favicon.ico000066400000000000000000000102761517523235500205600ustar00rootroot00000000000000  ( @   @zp~Li3a4a'W4a^]>h RE2`*ZFO;gݧko"SJHF3`*ZGHJLe#ZGpFHHHF3`*ZGHHHH:eh"R!R*oOEHHHHF>>oooDDD111222???wwwEEE444333...vvv...111:::]]]bbb---222333333222333hhh\\\<<<...vvv999LLLaaa@@@///333333333333333333,,,KKKjjjIII|||VVVSSS555111333333333333333333333333111111XXX;۱tttBBB///333333333333333333333333333333///;;;lllIcHHH333222333333333333333333333333333222---```nWWW555333333333333333333333333333333333222???T777222333333333333333333333333333333333555SSSd^^^)))222333333333333333333333333333222666???󁁁WXXX@@@222333333333333333333333333000@@@\\\ɶY|||ZZZ222111333333333333000666XXXyyy7wwwKKK---333333///GGGuuu:Tccc444///eee욚Xsss?incus-7.0.0/doc/.sphinx/_static/furo_colors.css000066400000000000000000000103671517523235500215060ustar00rootroot00000000000000body { --color-code-background: #f8f8f8; --color-code-foreground: black; --color-foreground-primary: #111; --color-foreground-secondary: var(--color-foreground-primary); --color-foreground-muted: #333; --color-background-secondary: #FFF; --color-background-hover: #f2f2f2; --color-brand-primary: #111; --color-brand-content: #06C; --color-api-background: #cdcdcd; --color-inline-code-background: rgba(0,0,0,.03); --color-sidebar-link-text: #111; --color-sidebar-item-background--current: #ebebeb; --color-sidebar-item-background--hover: #f2f2f2; --toc-font-size: var(--font-size--small); --color-admonition-title-background--note: var(--color-background-primary); --color-admonition-title-background--tip: var(--color-background-primary); --color-admonition-title-background--important: var(--color-background-primary); --color-admonition-title-background--caution: var(--color-background-primary); --color-admonition-title--note: #24598F; --color-admonition-title--tip: #24598F; --color-admonition-title--important: #C7162B; --color-admonition-title--caution: #F99B11; --color-highlighted-background: #EbEbEb; --color-link-underline: var(--color-background-primary); --color-link-underline--hover: var(--color-background-primary); --color-version-popup: #772953; } @media not print { body[data-theme="dark"] { --color-code-background: #202020; --color-code-foreground: #d0d0d0; --color-foreground-secondary: var(--color-foreground-primary); --color-foreground-muted: #CDCDCD; --color-background-secondary: var(--color-background-primary); --color-background-hover: #666; --color-brand-primary: #fff; --color-brand-content: #06C; --color-sidebar-link-text: #f7f7f7; --color-sidebar-item-background--current: #666; --color-sidebar-item-background--hover: #333; --color-admonition-background: transparent; --color-admonition-title-background--note: var(--color-background-primary); --color-admonition-title-background--tip: var(--color-background-primary); --color-admonition-title-background--important: var(--color-background-primary); --color-admonition-title-background--caution: var(--color-background-primary); --color-admonition-title--note: #24598F; --color-admonition-title--tip: #24598F; --color-admonition-title--important: #C7162B; --color-admonition-title--caution: #F99B11; --color-highlighted-background: #666; --color-link-underline: var(--color-background-primary); --color-link-underline--hover: var(--color-background-primary); --color-version-popup: #F29879; } @media (prefers-color-scheme: dark) { body:not([data-theme="light"]) { --color-code-background: #202020; --color-code-foreground: #d0d0d0; --color-foreground-secondary: var(--color-foreground-primary); --color-foreground-muted: #CDCDCD; --color-background-secondary: var(--color-background-primary); --color-background-hover: #666; --color-brand-primary: #fff; --color-brand-content: #06C; --color-sidebar-link-text: #f7f7f7; --color-sidebar-item-background--current: #666; --color-sidebar-item-background--hover: #333; --color-admonition-background: transparent; --color-admonition-title-background--note: var(--color-background-primary); --color-admonition-title-background--tip: var(--color-background-primary); --color-admonition-title-background--important: var(--color-background-primary); --color-admonition-title-background--caution: var(--color-background-primary); --color-admonition-title--note: #24598F; --color-admonition-title--tip: #24598F; --color-admonition-title--important: #C7162B; --color-admonition-title--caution: #F99B11; --color-highlighted-background: #666; --color-link-underline: var(--color-background-primary); --color-link-underline--hover: var(--color-background-primary); --color-version-popup: #F29879; } } } incus-7.0.0/doc/.sphinx/_static/swagger-override.css000066400000000000000000000045241517523235500224240ustar00rootroot00000000000000.swagger-ui { background-color: white; } .swagger-ui .markdown p, .swagger-ui .markdown pre, .swagger-ui .renderedMarkdown p, .swagger-ui .renderedMarkdown pre { margin-left: 0em; } .swagger-ui, .swagger-ui textarea, .swagger-ui .info li, .swagger-ui .info a, .swagger-ui .info p, .swagger-ui .info table, .swagger-ui .info .title .swagger-ui .opblock-tag, .swagger-ui .opblock .opblock-summary-description, .swagger-ui .opblock-description-wrapper p, .swagger-ui .opblock-external-docs-wrapper p, .swagger-ui .opblock-title_normal p, .swagger-ui .opblock .opblock-section-header h4, .swagger-ui .opblock-tag:hover, .swagger-ui .opblock-tag small, .swagger-ui .opblock .opblock-section-header>label, .swagger-ui .opblock .opblock-summary-method, .swagger-ui .tab li, .swagger-ui .opblock-description-wrapper,.swagger-ui .opblock-external-docs-wrapper,.swagger-ui .opblock-title_normal, .swagger-ui .opblock-description-wrapper h4,.swagger-ui .opblock-external-docs-wrapper h4,.swagger-ui .opblock-title_normal h4, .swagger-ui .responses-inner h4,.swagger-ui .responses-inner h5, .swagger-ui .response-col_status, .swagger-ui .response-col_links, .swagger-ui .download-contents, .swagger-ui .scheme-container .schemes>label, .swagger-ui .loading-container .loading:after, .swagger-ui section h3, .swagger-ui .btn, .swagger-ui .btn.cancel, .swagger-ui select, .swagger-ui label, .swagger-ui .dialog-ux .modal-ux-content p, .swagger-ui .dialog-ux .modal-ux-content p, .swagger-ui .dialog-ux .modal-ux-content h4, .swagger-ui .dialog-ux .modal-ux-header h3, .swagger-ui section.models h4, .swagger-ui section.models h5, .swagger-ui .model-title, .swagger-ui .servers>label, .swagger-ui .model-deprecated-warning, .swagger-ui table thead tr td,.swagger-ui table thead tr th, .swagger-ui .parameter__name, .swagger-ui .topbar a, .swagger-ui .topbar .download-url-wrapper .download-url-button, .swagger-ui .info h1,.swagger-ui .info h2,.swagger-ui .info h3,.swagger-ui .info h4,.swagger-ui .info h5, .swagger-ui .info .title small pre, .swagger-ui .errors-wrapper hgroup h4 { font-family: "Ubuntu", -apple-system, "Segoe UI", "Roboto", "Oxygen", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", sans-serif; } .swagger-ui .response-col_description, .swagger-ui .parameters-col_description { width: 85% } .swagger-ui .information-container .url { display: none; } incus-7.0.0/doc/.sphinx/_static/tag.png000066400000000000000000000234711517523235500177210ustar00rootroot00000000000000PNG  IHDR>%})htEXtSoftwareAdobe ImageReadyqe<&IDATxmeUyAC2&&@ &bT#v_0i F`ӹi?`4MmN? %m\2U xx V*Avκ뜳>{<{\9zJ1+AQ7.c EQ}Z8!{wW<(;P<6 Bo|x[n=} ?;Q5K@ޡ}ţ,ey/}?u~Ɔ+[gǑhW"(-?/~axS}co7^z|T _Uࣲ-k/GGY;wΜw㡟ׅq @]p>EQСwHZPֆVoEQRVRWVV[}BAo^Y֢lk+t=w"Ǟ>60R[n)3|(ZқSP"yGh>)6/6=f em]]y殻*w+z8w"e9SB }?lXiR<B|xkR€l,WUnWݦ>o=Ex)7|[77/>^*{JhwQG x xBi(o[JZ(o80=xm5oͰGQޞd^>eԞR7ybGL 2{J ͉Ge NS*_@_|TC9;8{Jh1ޚ/ ,B ל,m{5ȵp -=j_#}5|l^4gBihWkOA`o|"?JU=%'|lGҼ7ӣ_/糧uU֦wQ!+Ҟ3 Th͉R=|!).E\ 㯨#_u iO>kaZ`C{*hoThiRXWThi|)':p@YWThi|)}l ߨRS>)l ߨ r |v)K.~WTh<|9S >ka㦛cURfO+Czh"}?B)$՞RG♓U&ГM峿` d 9oQ=eْ=E/w/BK_gm3V/ >6.hO2e:S=iWoTDdO%`6 ikaZs⯂_n^O?tk?EPLj|UeZ)_}3B5%DSg#Si.S Wޡ]UV{J i5쳿0 Po+O=>'{ X1 ZoTv{hZv};ߨ O=N*cOdaZ]f{ >;ij)_QaZ̜2|-id]SbHMwZ:G+F!_|&m'rحC _[=[4䮲_:2s﷛:nyb[=;_r!T=]_Kx_s)wer3FSDnu)Yo^IdKC P_dL޷,L]?_:GwA'?g5';^B=?O4oxV >{*jG}",ȊWӇTdǽ +e!_U)YvxpF_h<0(DSɜN$Ng}U.79bO>tSb#vU8|sWֱ~k666첊WX .vsX |oKAN9|!N$GJImRO&{rBS/WpUWO 㯺x,|; uX5Y>cꪓRj n3ͪ=G[9b_s хC:0/ТM'bU}Ҝ^=&d (,ks4 {w'm rvA)LkUJYy}dOi zx'ڼL+ᮀc_C&(ZU rN;-CS 94<]X-;Cҭq-uw7Օ)wmk\gw 69_~_Prtcǎ|]zRC}~?('z{cMo&(S-:+P)ف{)y5טnSIO=mΞ=4MP)j^ rvO(Rsn-.:#QuMWB4SO}{;"JHmWC,.z՟(7]Mb  .D\JH WmNrr㵕r4A{<{JgOEm%dnW-9.Qx]:ڨrAO܉5Pr4lOqa>vk )wM.c.W9 ܉(Ҕ&uk=%M7M2[޾SO CNiaIJXra$)t4A3*!݃]r'; 9 9Z߅53WמZ _UW5wN|M ]_iSuaRpri-CW]) T>)E\'UpZZx1xmnijT`Tt|sb;59vգSʅu({Jh6)yN_iS֫*wӭWԦ |R)Ji3'/=%eY}'ezjNAۧ͞STO5JI}j`^ r1\9 PaM9 >|8P4媺)CVXɦ^xॣ߭0'"Y@PhԚZMS(\!/VRXQ$ ʯ3N| ˡ_%y.Z.h`3}>1|HO8Q~`D宖|Y5Vx𺡬 7]\o^/@϶P\wb(d !nxm/q/8!G%$b QM}kH*T+vK9?Ƀ믿~fFqB)?gi^7] dd>LlF̀Rr࣯ʧأOxՓ릋M7ȁOTBJ;3fM2r-9!+e FW:6{?e|fۮ6GUCG8ѯe`sNAgP2 * 'Y|\trm7S2慯m P;['>ځPkO,<=CYKqaۅ`1SK`z/wJhϮ.WP@쎿1oz,sN|=yhM7; oj_}10=v_՟K~m..(% S%(V_R(t+,ЏQnf!o*yZ-L fic=6g?wol,rz ?3=ʅX4Sg*O|\`MdW%g- memB` XE(o+W޹.g 2tM759@|emOAc=% d@-Û|ȵ*)Ԣ. AWǞ |"UO{'V_1jzmշ=+||=%9De͟?t{ <4d O.lD'B))TouljO|SbW'o`:4d3na7x*=(Cˬ>"R=`?y5_a6յ|WXX j F%mW:B ڌ=さ_=xJ|W峧`>"%9e͎$ʾ*7#=yT.-}>,cT~] =(aC+*vl^ףBYdWd`@Ҟ|"WN85mPo߾r#wɹL)'++-o_یH!|kL% Ox!L =E-J} #^+-7T>eMeOQ>HFTAH$E[l0>Z$ٹVSؗ@P|sԅ9}ҴyC-keBH*;E2ffS |K$W2&W;}N|MgPL,5Y|)ȿ**5 5](`GHTmW.JY)kEb|6)_-J.a>0䳧hKA{_is4mPe:SHA&VׯoTWZ6oz{*vijrH7SK(d̬OS $fɗퟦoPVQk:z]rmOb`_n/d +?P/J͐%V%{L{; $,_weU{4]2IJֶȘUzWt|WAԚ.c җn/CW2&4 ;K=+tM;9 |%oiW酖^KYɞ]ZBM)P_⯤ kJۉqH7DW&{J)_bI]l徃pS4dO!V+{W*kL N4!Jl.A=EMO;;d/CI`~4;2&?M7) kRfO!:V" tOc)LPh^t-^;wn܅=B_er7Y%hA楥Յ4 ]s 2TE/)UweC_!$WRjZAhR#_ {e=)BK+U/l)M;3zg| ?Յ]Bq@%}ܩ%HBHжh*ݶwj >+imj|HܹSKB+(-tsqGSKN4a9~5r^;P '_IOS=b)sg@9 ĿiҼ[1䶙 [G~ŗO˕vk+~d:eLMp=%¢V82vF/Ǻ,ùo).WҒrGUs Ϟjs,VBey;m7FX caqyˈ'͇tsC:6udTe-Vx[- bwxq/Ꮏi:@h+͇t#rli613ZVqf0lHWiOɣB&~ߨ_PpZsg+y>&Qӿ؜Cxm^k+}W ]|ǚ9O'/L%Ҝk?u`,{J?oʓG{4eƻi-7NN|;uĪV1xW@H |:—N;{JRY1j'%*wRyxյWTUN{FUS]4%Bc^ϵ09<3C} /8[vU.7oux&}Ҟ,!nrr |LANw]cU bΚSUs3iOIS)ك7To)4cON~./DeXwM|<;]YIArTi |,fectǞe&)(W"* @8r-4 Ӟ&Z7plEy"*$hOɣmb"Sz> WI.W!4jL-7%| i')_}/c^qfX`_!i4'9=86JVn@fd=@⯚/+~9 ?x_5NW:(J6WMCӬM }P_kO{FS |aa843|<;]bC |U6?ľPSYH)_x 㯲($ :|<;QjO!+?%rb+ALAt|yO⯎WR9ֈ؆R 4\3J"7OIVݝM0* TnxsTCU~Q`x܈ wAB|_(7k\Bzu#fIJC֝nظq^ݼ%>zS} [%ofY{{?1S"@_ Vz)7ҞB ~u⯜>#I׊~Gߗo7'2~H)֎>Z |4}y+;ٲ=p̉|13')~;۶=e ; |TMn3J~,HA=Ze_}{T²Vv rc 2GuPne"3~UKK-8)9t![+uY|H_!J?˕GiaZL{ GeIeaՊxH7Ge @_Pn @џ7j!LA&(2J5[.8}3e9[RyH7EQA8$t3"(*9_!GikmbՍwH7S)^Fxaҋrr;[v?W~vv-(k|5) `fhգ(J;ЦuZE vk>|{wIENDB`incus-7.0.0/doc/.sphinx/requirements.txt000066400000000000000000000005621517523235500202720ustar00rootroot00000000000000docutils<0.22 furo gitpython linkify-it-py canonical-sphinx-extensions @ git+https://github.com/canonical/canonical-sphinx-extensions@bc648473619f93fdfefb9ae8becbe7d00dad675a myst-parser pyspelling sphinx sphinx-autobuild sphinx-copybutton sphinx-design sphinx-notfound-page sphinx-remove-toctrees sphinx-reredirects sphinx-tabs sphinxcontrib-jquery sphinxext-opengraph incus-7.0.0/doc/.sphinx/spellingcheck.yaml000066400000000000000000000011141517523235500204770ustar00rootroot00000000000000jobs: 2 matrix: - name: Markdown files aspell: lang: en d: en_US dictionary: wordlists: - doc/.wordlist.txt output: doc/.sphinx/.wordlist.dic sources: - doc/html/**/*.html|!doc/html/config-options/index.html|!doc/html/reference/manpages/**/*.html pipeline: - pyspelling.filters.html: comments: false attributes: - title - alt ignores: - code - pre - spellexception - link - title - div.relatedlinks - span.guilabel - div.visually-hidden - img - a.p-navigation__link incus-7.0.0/doc/.wordlist.txt000066400000000000000000000047541517523235500161140ustar00rootroot00000000000000AAAA AAVMF ABI ACL ACLs AI AIO allocator AMD Ansible Ansible's API APIs AppArmor ARMv ARP ASN AXFR backend backends backported Backports balancer balancers benchmarking BFD BGP bibi BitLocker BMC bool bootable BPF Btrfs bugfix bugfixes Centos Ceph CephFS Ceph's CFS cgroup cgroupfs cgroups checksum checksums Chocolatey CIDR CLI Colima COPR Cowsql CPUs CRIU CRL cron CSV CUDA customizable dataset DCO dereferenced devtmpfs DHCP DHCPv Diátaxis Diffie diskful diskless Diskless Distrobuilder DNAT DNS dnsmasq DNSSEC DoS DRBD DRM EB Ebit eBPF ECDHE ECDSA ECMP EDK EiB Eibit endian EPEL ES ESA ETag failover formatters FQDNs FreeBSD frontend Furo gapped GARM GARP GbE Gbit Geneve GiB Gibit GID GIDs Github Golang goroutines GPUs Grafana HAProxy hardcoded HDDs Hellman Homebrew hostname hotplug hotplugged hotplugging HTTPS hwdata ICMP idmap idmapped idmaps incrementing Incus Incus' InfiniBand InfluxDB init initramfs integrations IOMMU IOPS IOV IPAM IPs IPv IPVLAN iSCSI JIT jq JSON kB kbit KiB kibi Kibit Kubernetes KVM LINBIT LINSTOR LINSTOR's LLM LLMs lookups Loongarch LRU LTS LV LVM LXC LXCFS LXC's LXD LXD's macOS macvlan Makefile manpages Mbit mDNS MiB Mibit MicroCeph MicroCloud MII MITM MTU Mullvad multicast MyST namespace namespaced namespaces NATed natively NDP netmask NFS NIC NICs NixOS NUMA NVMe NVRAM OCI OData OIDC OpenFGA OpenID OpenMetrics OpenSSL openSUSE OpenSUSE OpenTofu OSD overcommit overcommitting overlayfs OVMF OVN OVS PackageHub passthrough Pbit PCI PCIe PDU peerings Permalink PFs PiB Pibit PID PKI PNG Podman Pongo POSIX PPA pre preselects preseed proxied proxying PTS qdisc QEMU qgroup qgroups QMP RADOS RBAC RBD RDNSS README reconfiguring requestor resolvers RESTful RHEL rootfs RSA rST RTC runtime SATA scalable scriptlet SDN Seccomp SELinux SEV SFTP SHA shiftfs SIGHUP SIGTERM simplestreams SLAAC SMTP Snapcraft snapshotted SNAT Solaris SPAs SPL SquashFS SSDs SSL Starlark stateful stderr stdin stdout STP struct structs subcommand subcommands subitem subnet subnets subpage substep subtree subtrees subvolume subvolumes superset SVG symlink symlinks syscall syscalls sysfs syslog systemd Tbit TCP Telegraf Terraform TiB Tibit TLS tmpfs toolchain topologies TPM TrueNAS TSIG TTL UDP UEFI UFW UI UID UIDs uncomment unconfigured unevictable unixgram unmanaged unmount unmounting uplink uptime URI URIs userspace UUID vCPU vCPUs VDPA VFs VFS VirtIO virtualize virtualized VLAN VLANs VM VMs VPD VPN VPS VRF vSwitch VXLAN webhook WebSocket WebSockets Winget XFS XHR YAML YAML's Zabbly Zettabyte ZFS zpool zpools incus-7.0.0/doc/README.md000066400000000000000000000027721517523235500147030ustar00rootroot00000000000000# Incus documentation The Incus documentation is available at: GitHub provides a basic rendering of the documentation as well, but important features like includes and clickable links are missing. Therefore, we recommend reading the [published documentation](https://linuxcontainers.org/incus/docs/main/). ## Documentation framework Incus' documentation is built with [Sphinx](https://www.sphinx-doc.org/en/master/index.html). It is written in [Markdown](https://commonmark.org/) with [MyST](https://myst-parser.readthedocs.io/) extensions. For syntax help and guidelines, see the [documentation cheat sheet](https://linuxcontainers.org/incus/docs/main/doc-cheat-sheet/) ([source](https://raw.githubusercontent.com/lxc/incus/main/doc/doc-cheat-sheet.md)). For structuring, the documentation uses the [Diátaxis](https://diataxis.fr/) approach. ## Build the documentation To build the documentation, run `make doc` from the root directory of the repository. This command installs the required tools and renders the output to the `doc/html/` directory. To update the documentation for changed files only (without re-installing the tools), run `make doc-incremental`. Before opening a pull request, make sure that the documentation builds without any warnings (warnings are treated as errors). To preview the documentation locally, run `make doc-serve` and go to [`http://localhost:8001`](http://localhost:8001) to view the rendered documentation. incus-7.0.0/doc/api-extensions.md000066400000000000000000003316471517523235500167220ustar00rootroot00000000000000# API extensions The changes below were introduced to the Incus API after the 1.0 API was finalized. They are all backward compatible and can be detected by client tools by looking at the `api_extensions` field in `GET /1.0`. ## `storage_zfs_remove_snapshots` A `storage.zfs_remove_snapshots` daemon configuration key was introduced. It's a Boolean that defaults to `false` and that when set to `true` instructs Incus to remove any needed snapshot when attempting to restore another. This is needed as ZFS will only let you restore the latest snapshot. ## `container_host_shutdown_timeout` A `boot.host_shutdown_timeout` container configuration key was introduced. It's an integer which indicates how long Incus should wait for the container to stop before killing it. Its value is only used on clean Incus daemon shutdown. It defaults to 30s. ## `container_stop_priority` A `boot.stop.priority` container configuration key was introduced. It's an integer which indicates the priority of a container during shutdown. Containers will shutdown starting with the highest priority level. Containers with the same priority will shutdown in parallel. It defaults to 0. ## `container_syscall_filtering` A number of new syscalls related container configuration keys were introduced. * `security.syscalls.blacklist_default` * `security.syscalls.blacklist_compat` * `security.syscalls.blacklist` * `security.syscalls.whitelist` See [Instance configuration](instance-config) for how to use them. ## `auth_pki` This indicates support for PKI authentication mode. In this mode, the client and server both must use certificates issued by the same PKI. See [Security](security.md) for details. ## `container_last_used_at` A `last_used_at` field was added to the `GET /1.0/containers/` endpoint. It is a timestamp of the last time the container was started. If a container has been created but not started yet, `last_used_at` field will be `1970-01-01T00:00:00Z` ## `etag` Add support for the ETag header on all relevant endpoints. This adds the following HTTP header on answers to GET: * ETag (SHA-256 of user modifiable content) And adds support for the following HTTP header on PUT requests: * If-Match (ETag value retrieved through previous GET) This makes it possible to GET an Incus object, modify it and PUT it without risking to hit a race condition where Incus or another client modified the object in the meantime. ## `patch` Add support for the HTTP PATCH method. PATCH allows for partial update of an object in place of PUT. ## `usb_devices` Add support for USB hotplug. ## `https_allowed_credentials` To use Incus API with all Web Browsers (via SPAs) you must send credentials (certificate) with each XHR (in order for this to happen, you should set [`withCredentials=true`](https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/withCredentials) flag to each XHR Request). Some browsers like Firefox and Safari can't accept server response without `Access-Control-Allow-Credentials: true` header. To ensure that the server will return a response with that header, set `core.https_allowed_credentials=true`. ## `image_compression_algorithm` This adds support for a `compression_algorithm` property when creating an image (`POST /1.0/images`). Setting this property overrides the server default value (`images.compression_algorithm`). ## `directory_manipulation` This allows for creating and listing directories via the Incus API, and exports the file type via the X-Incus-type header, which can be either `file` or `directory` right now. ## `container_cpu_time` This adds support for retrieving CPU time for a running container. ## `storage_zfs_use_refquota` Introduces a new server property `storage.zfs_use_refquota` which instructs Incus to set the `refquota` property instead of `quota` when setting a size limit on a container. Incus will also then use `usedbydataset` in place of `used` when being queried about disk utilization. This effectively controls whether disk usage by snapshots should be considered as part of the container's disk space usage. ## `storage_lvm_mount_options` Adds a new `storage.lvm_mount_options` daemon configuration option which defaults to `discard` and allows the user to set addition mount options for the file system used by the LVM LV. ## `network` Network management API for Incus. This includes: * Addition of the `managed` property on `/1.0/networks` entries * All the network configuration options (see [Network configuration](networks.md) for details) * `POST /1.0/networks` (see [RESTful API](rest-api.md) for details) * `PUT /1.0/networks/` (see [RESTful API](rest-api.md) for details) * `PATCH /1.0/networks/` (see [RESTful API](rest-api.md) for details) * `DELETE /1.0/networks/` (see [RESTful API](rest-api.md) for details) * `ipv4.address` property on `nic` type devices (when `nictype` is `bridged`) * `ipv6.address` property on `nic` type devices (when `nictype` is `bridged`) * `security.mac_filtering` property on `nic` type devices (when `nictype` is `bridged`) ## `profile_usedby` Adds a new `used_by` field to profile entries listing the containers that are using it. ## `container_push` When a container is created in push mode, the client serves as a proxy between the source and target server. This is useful in cases where the target server is behind a NAT or firewall and cannot directly communicate with the source server and operate in pull mode. ## `container_exec_recording` Introduces a new Boolean `record-output`, parameter to `/1.0/containers//exec` which when set to `true` and combined with with `wait-for-websocket` set to `false`, will record stdout and stderr to disk and make them available through the logs interface. The URL to the recorded output is included in the operation metadata once the command is done running. That output will expire similarly to other log files, typically after 48 hours. ## `certificate_update` Adds the following to the REST API: * ETag header on GET of a certificate * PUT of certificate entries * PATCH of certificate entries ## `container_exec_signal_handling` Adds support `/1.0/containers//exec` for forwarding signals sent to the client to the processes executing in the container. Currently SIGTERM and SIGHUP are forwarded. Further signals that can be forwarded might be added later. ## `gpu_devices` Enables adding GPUs to a container. ## `container_image_properties` Introduces a new `image` configuration key space. Read-only, includes the properties of the parent image. ## `migration_progress` Transfer progress is now exported as part of the operation, on both sending and receiving ends. This shows up as a `fs_progress` attribute in the operation metadata. ## `id_map` Enables setting the `security.idmap.isolated` and `security.idmap.isolated`, `security.idmap.size`, and `raw.id_map` fields. ## `network_firewall_filtering` Add two new keys, `ipv4.firewall` and `ipv6.firewall` which if set to `false` will turn off the generation of `iptables` FORWARDING rules. NAT rules will still be added so long as the matching `ipv4.nat` or `ipv6.nat` key is set to `true`. Rules necessary for `dnsmasq` to work (DHCP/DNS) will always be applied if `dnsmasq` is enabled on the bridge. ## `network_routes` Introduces `ipv4.routes` and `ipv6.routes` which allow routing additional subnets to an Incus bridge. ## `storage` Storage management API for Incus. This includes: * `GET /1.0/storage-pools` * `POST /1.0/storage-pools` (see [RESTful API](rest-api.md) for details) * `GET /1.0/storage-pools/` (see [RESTful API](rest-api.md) for details) * `POST /1.0/storage-pools/` (see [RESTful API](rest-api.md) for details) * `PUT /1.0/storage-pools/` (see [RESTful API](rest-api.md) for details) * `PATCH /1.0/storage-pools/` (see [RESTful API](rest-api.md) for details) * `DELETE /1.0/storage-pools/` (see [RESTful API](rest-api.md) for details) * `GET /1.0/storage-pools//volumes` (see [RESTful API](rest-api.md) for details) * `GET /1.0/storage-pools//volumes/` (see [RESTful API](rest-api.md) for details) * `POST /1.0/storage-pools//volumes/` (see [RESTful API](rest-api.md) for details) * `GET /1.0/storage-pools//volumes//` (see [RESTful API](rest-api.md) for details) * `POST /1.0/storage-pools//volumes//` (see [RESTful API](rest-api.md) for details) * `PUT /1.0/storage-pools//volumes//` (see [RESTful API](rest-api.md) for details) * `PATCH /1.0/storage-pools//volumes//` (see [RESTful API](rest-api.md) for details) * `DELETE /1.0/storage-pools//volumes//` (see [RESTful API](rest-api.md) for details) * All storage configuration options (see [Storage configuration](storage.md) for details) ## `file_delete` Implements `DELETE` in `/1.0/containers//files` ## `file_append` Implements the `X-Incus-write` header which can be one of `overwrite` or `append`. ## `network_dhcp_expiry` Introduces `ipv4.dhcp.expiry` and `ipv6.dhcp.expiry` allowing to set the DHCP lease expiry time. ## `storage_lvm_vg_rename` Introduces the ability to rename a volume group by setting `storage.lvm.vg_name`. ## `storage_lvm_thinpool_rename` Introduces the ability to rename a thin pool name by setting `storage.thinpool_name`. ## `network_vlan` This adds a new `vlan` property to `macvlan` network devices. When set, this will instruct Incus to attach to the specified VLAN. Incus will look for an existing interface for that VLAN on the host. If one can't be found it will create one itself and then use that as the macvlan parent. ## `image_create_aliases` Adds a new `aliases` field to `POST /1.0/images` allowing for aliases to be set at image creation/import time. ## `container_stateless_copy` This introduces a new `live` attribute in `POST /1.0/containers/`. Setting it to `false` tells Incus not to attempt running state transfer. ## `container_only_migration` Introduces a new Boolean `container_only` attribute. When set to `true` only the container will be copied or moved. ## `storage_zfs_clone_copy` Introduces a new Boolean `storage_zfs_clone_copy` property for ZFS storage pools. When set to `false` copying a container will be done through `zfs send` and receive. This will make the target container independent of its source container thus avoiding the need to keep dependent snapshots in the ZFS pool around. However, this also entails less efficient storage usage for the affected pool. The default value for this property is `true`, i.e. space-efficient snapshots will be used unless explicitly set to `false`. ## `unix_device_rename` Introduces the ability to rename the `unix-block`/`unix-char` device inside container by setting `path`, and the `source` attribute is added to specify the device on host. If `source` is set without a `path`, we should assume that `path` will be the same as `source`. If `path` is set without `source` and `major`/`minor` isn't set, we should assume that `source` will be the same as `path`. So at least one of them must be set. ## `storage_rsync_bwlimit` When `rsync` has to be invoked to transfer storage entities setting `rsync.bwlimit` places an upper limit on the amount of socket I/O allowed. ## `network_vxlan_interface` This introduces a new `tunnel.NAME.interface` option for networks. This key control what host network interface is used for a VXLAN tunnel. ## `storage_btrfs_mount_options` This introduces the `btrfs.mount_options` property for Btrfs storage pools. This key controls what mount options will be used for the Btrfs storage pool. ## `entity_description` This adds descriptions to entities like containers, snapshots, networks, storage pools and volumes. ## `image_force_refresh` This allows forcing a refresh for an existing image. ## `storage_lvm_lv_resizing` This introduces the ability to resize logical volumes by setting the `size` property in the containers root disk device. ## `id_map_base` This introduces a new `security.idmap.base` allowing the user to skip the map auto-selection process for isolated containers and specify what host UID/GID to use as the base. ## `file_symlinks` This adds support for transferring symlinks through the file API. X-Incus-type can now be `symlink` with the request content being the target path. ## `container_push_target` This adds the `target` field to `POST /1.0/containers/` which can be used to have the source Incus host connect to the target during migration. ## `network_vlan_physical` Allows use of `vlan` property with `physical` network devices. When set, this will instruct Incus to attach to the specified VLAN on the `parent` interface. Incus will look for an existing interface for that `parent` and VLAN on the host. If one can't be found it will create one itself. Then, Incus will directly attach this interface to the container. ## `storage_images_delete` This enabled the storage API to delete storage volumes for images from a specific storage pool. ## `container_edit_metadata` This adds support for editing a container `metadata.yaml` and related templates via API, by accessing URLs under `/1.0/containers//metadata`. It can be used to edit a container before publishing an image from it. ## `container_snapshot_stateful_migration` This enables migrating stateful container snapshots to new containers. ## `storage_driver_ceph` This adds a Ceph storage driver. ## `storage_ceph_user_name` This adds the ability to specify the Ceph user. ## `instance_types` This adds the `instance_type` field to the container creation request. Its value is expanded to Incus resource limits. ## `storage_volatile_initial_source` This records the actual source passed to Incus during storage pool creation. ## `storage_ceph_force_osd_reuse` This introduces the `ceph.osd.force_reuse` property for the Ceph storage driver. When set to `true` Incus will reuse an OSD storage pool that is already in use by another Incus instance. ## `storage_block_filesystem_btrfs` This adds support for Btrfs as a storage volume file system, in addition to `ext4` and `xfs`. ## `resources` This adds support for querying an Incus daemon for the system resources it has available. ## `kernel_limits` This adds support for setting process limits such as maximum number of open files for the container via `nofile`. The format is `limits.kernel.[limit name]`. ## `storage_api_volume_rename` This adds support for renaming custom storage volumes. ## `network_sriov` This adds support for SR-IOV enabled network devices. ## `console` This adds support to interact with the container console device and console log. ## `restrict_dev_incus` A new `security.guestapi` container configuration key was introduced. The key controls whether the `/dev/incus` interface is made available to the container. If set to `false`, this effectively prevents the container from interacting with the Incus daemon. ## `migration_pre_copy` This adds support for optimized memory transfer during live migration. ## `infiniband` This adds support to use InfiniBand network devices. ## `dev_incus_events` This adds a WebSocket API to the `/dev/incus` socket. When connecting to `/1.0/events` over the `/dev/incus` socket, you will now be getting a stream of events over WebSocket. ## `proxy` This adds a new `proxy` device type to containers, allowing forwarding of connections between the host and container. ## `network_dhcp_gateway` Introduces a new `ipv4.dhcp.gateway` network configuration key to set an alternate gateway. ## `file_get_symlink` This makes it possible to retrieve symlinks using the file API. ## `network_leases` Adds a new `/1.0/networks/NAME/leases` API endpoint to query the lease database on bridges which run an Incus-managed DHCP server. ## `unix_device_hotplug` This adds support for the `required` property for Unix devices. ## `storage_api_local_volume_handling` This add the ability to copy and move custom storage volumes locally in the same and between storage pools. ## `operation_description` Adds a `description` field to all operations. ## `clustering` Clustering API for Incus. This includes the following new endpoints (see [RESTful API](rest-api.md) for details): * `GET /1.0/cluster` * `UPDATE /1.0/cluster` * `GET /1.0/cluster/members` * `GET /1.0/cluster/members/` * `POST /1.0/cluster/members/` * `DELETE /1.0/cluster/members/` The following existing endpoints have been modified: * `POST /1.0/containers` accepts a new `target` query parameter * `POST /1.0/storage-pools` accepts a new `target` query parameter * `GET /1.0/storage-pool/` accepts a new `target` query parameter * `POST /1.0/storage-pool//volumes/` accepts a new `target` query parameter * `GET /1.0/storage-pool//volumes//` accepts a new `target` query parameter * `POST /1.0/storage-pool//volumes//` accepts a new `target` query parameter * `PUT /1.0/storage-pool//volumes//` accepts a new `target` query parameter * `PATCH /1.0/storage-pool//volumes//` accepts a new `target` query parameter * `DELETE /1.0/storage-pool//volumes//` accepts a new `target` query parameter * `POST /1.0/networks` accepts a new `target` query parameter * `GET /1.0/networks/` accepts a new `target` query parameter ## `event_lifecycle` This adds a new `lifecycle` message type to the events API. ## `storage_api_remote_volume_handling` This adds the ability to copy and move custom storage volumes between remote. ## `nvidia_runtime` Adds a `nvidia_runtime` configuration option for containers, setting this to `true` will have the NVIDIA runtime and CUDA libraries passed to the container. ## `container_mount_propagation` This adds a new `propagation` option to the disk device type, allowing the configuration of kernel mount propagation. ## `container_backup` Add container backup support. This includes the following new endpoints (see [RESTful API](rest-api.md) for details): * `GET /1.0/containers//backups` * `POST /1.0/containers//backups` * `GET /1.0/containers//backups/` * `POST /1.0/containers//backups/` * `DELETE /1.0/containers//backups/` * `GET /1.0/containers//backups//export` The following existing endpoint has been modified: * `POST /1.0/containers` accepts the new source type `backup` ## `dev_incus_images` Adds a `security.guestapi.images` configuration option for containers which controls the availability of a `/1.0/images/FINGERPRINT/export` API over `/dev/incus`. This can be used by a container running nested Incus to access raw images from the host. ## `container_local_cross_pool_handling` This enables copying or moving containers between storage pools on the same Incus instance. ## `proxy_unix` Add support for both Unix sockets and abstract Unix sockets in proxy devices. They can be used by specifying the address as `unix:/path/to/unix.sock` (normal socket) or `unix:@/tmp/unix.sock` (abstract socket). Supported connections are now: * `TCP <-> TCP` * `UNIX <-> UNIX` * `TCP <-> UNIX` * `UNIX <-> TCP` ## `proxy_udp` Add support for UDP in proxy devices. Supported connections are now: * `TCP <-> TCP` * `UNIX <-> UNIX` * `TCP <-> UNIX` * `UNIX <-> TCP` * `UDP <-> UDP` * `TCP <-> UDP` * `UNIX <-> UDP` ## `clustering_join` This makes `GET /1.0/cluster` return information about which storage pools and networks are required to be created by joining nodes and which node-specific configuration keys they are required to use when creating them. Likewise the `PUT /1.0/cluster` endpoint now accepts the same format to pass information about storage pools and networks to be automatically created before attempting to join a cluster. ## `proxy_tcp_udp_multi_port_handling` Adds support for forwarding traffic for multiple ports. Forwarding is allowed between a range of ports if the port range is equal for source and target (for example `1.2.3.4 0-1000 -> 5.6.7.8 1000-2000`) and between a range of source ports and a single target port (for example `1.2.3.4 0-1000 -> 5.6.7.8 1000`). ## `network_state` Adds support for retrieving a network's state. This adds the following new endpoint (see [RESTful API](rest-api.md) for details): * `GET /1.0/networks//state` ## `proxy_unix_dac_properties` This adds support for GID, UID, and mode properties for non-abstract Unix sockets. ## `container_protection_delete` Enables setting the `security.protection.delete` field which prevents containers from being deleted if set to `true`. Snapshots are not affected by this setting. ## `proxy_priv_drop` Adds `security.uid` and `security.gid` for the proxy devices, allowing privilege dropping and effectively changing the UID/GID used for connections to Unix sockets too. ## `pprof_http` This adds a new `core.debug_address` configuration option to start a debugging HTTP server. That server currently includes a `pprof` API and replaces the old `cpu-profile`, `memory-profile` and `print-goroutines` debug options. ## `proxy_haproxy_protocol` Adds a `proxy_protocol` key to the proxy device which controls the use of the HAProxy PROXY protocol header. ## `network_hwaddr` Adds a `bridge.hwaddr` key to control the MAC address of the bridge. ## `proxy_nat` This adds optimized UDP/TCP proxying. If the configuration allows, proxying will be done via `iptables` instead of proxy devices. ## `network_nat_order` This introduces the `ipv4.nat.order` and `ipv6.nat.order` configuration keys for Incus bridges. Those keys control whether to put the Incus rules before or after any pre-existing rules in the chain. ## `container_full` This introduces a new `recursion=2` mode for `GET /1.0/containers` which allows for the retrieval of all container structs, including the state, snapshots and backup structs. This effectively allows for [`incus list`](incus_list.md) to get all it needs in one query. ## `backup_compression` This introduces a new `backups.compression_algorithm` configuration key which allows configuration of backup compression. ## `nvidia_runtime_config` This introduces a few extra configuration keys when using `nvidia.runtime` and the `libnvidia-container` library. Those keys translate pretty much directly to the matching NVIDIA container environment variables: * `nvidia.driver.capabilities` => `NVIDIA_DRIVER_CAPABILITIES` * `nvidia.require.cuda` => `NVIDIA_REQUIRE_CUDA` * `nvidia.require.driver` => `NVIDIA_REQUIRE_DRIVER` ## `storage_api_volume_snapshots` Add support for storage volume snapshots. They work like container snapshots, only for volumes. This adds the following new endpoint (see [RESTful API](rest-api.md) for details): * `GET /1.0/storage-pools//volumes///snapshots` * `POST /1.0/storage-pools//volumes///snapshots` * `GET /1.0/storage-pools//volumes///snapshots/` * `PUT /1.0/storage-pools//volumes///snapshots/` * `POST /1.0/storage-pools//volumes///snapshots/` * `DELETE /1.0/storage-pools//volumes///snapshots/` ## `storage_unmapped` Introduces a new `security.unmapped` Boolean on storage volumes. Setting it to `true` will flush the current map on the volume and prevent any further idmap tracking and remapping on the volume. This can be used to share data between isolated containers after attaching it to the container which requires write access. ## `projects` Add a new project API, supporting creation, update and deletion of projects. Projects can hold containers, profiles or images at this point and let you get a separate view of your Incus resources by switching to it. ## `network_vxlan_ttl` This adds a new `tunnel.NAME.ttl` network configuration option which makes it possible to raise the TTL on VXLAN tunnels. ## `container_incremental_copy` This adds support for incremental container copy. When copying a container using the `--refresh` flag, only the missing or outdated files will be copied over. Should the target container not exist yet, a normal copy operation is performed. ## `usb_optional_vendorid` As the name implies, the `vendorid` field on USB devices attached to containers has now been made optional, allowing for all USB devices to be passed to a container (similar to what's done for GPUs). ## `snapshot_scheduling` This adds support for snapshot scheduling. It introduces three new configuration keys: `snapshots.schedule`, `snapshots.schedule.stopped`, and `snapshots.pattern`. Snapshots can be created automatically up to every minute. ## `snapshots_schedule_aliases` Snapshot schedule can be configured by a comma-separated list of schedule aliases. Available aliases are `<@hourly> <@daily> <@midnight> <@weekly> <@monthly> <@annually> <@yearly> <@startup>` for instances, and `<@hourly> <@daily> <@midnight> <@weekly> <@monthly> <@annually> <@yearly>` for storage volumes. ## `container_copy_project` Introduces a `project` field to the container source JSON object, allowing for copy/move of containers between projects. ## `clustering_server_address` This adds support for configuring a server network address which differs from the REST API client network address. When bootstrapping a new cluster, clients can set the new `cluster.https_address` configuration key to specify the address of the initial server. When joining a new server, clients can set the `core.https_address` configuration key of the joining server to the REST API address the joining server should listen at, and set the `server_address` key in the `PUT /1.0/cluster` API to the address the joining server should use for clustering traffic (the value of `server_address` will be automatically copied to the `cluster.https_address` configuration key of the joining server). ## `clustering_image_replication` Enable image replication across the nodes in the cluster. A new `cluster.images_minimal_replica` configuration key was introduced can be used to specify to the minimal numbers of nodes for image replication. ## `container_protection_shift` Enables setting the `security.protection.shift` option which prevents containers from having their file system shifted. ## `snapshot_expiry` This adds support for snapshot expiration. The task is run minutely. The configuration option `snapshots.expiry` takes an expression in the form of `1M 2H 3d 4w 5m 6y` (1 minute, 2 hours, 3 days, 4 weeks, 5 months, 6 years), however not all parts have to be used. Snapshots which are then created will be given an expiry date based on the expression. This expiry date, defined by `expires_at`, can be manually edited using the API or [`incus config edit`](incus_config_edit.md). Snapshots with a valid expiry date will be removed when the task in run. Expiry can be disabled by setting `expires_at` to an empty string or `0001-01-01T00:00:00Z` (zero time). This is the default if `snapshots.expiry` is not set. This adds the following new endpoint (see [RESTful API](rest-api.md) for details): * `PUT /1.0/containers//snapshots/` ## `snapshot_expiry_creation` Adds `expires_at` to container creation, allowing for override of a snapshot's expiry at creation time. ## `network_leases_location` Introduces a `Location` field in the leases list. This is used when querying a cluster to show what node a particular lease was found on. ## `resources_cpu_socket` Add Socket field to CPU resources in case we get out of order socket information. ## `resources_gpu` Add a new GPU struct to the server resources, listing all usable GPUs on the system. ## `resources_numa` Shows the NUMA node for all CPUs and GPUs. ## `kernel_features` Exposes the state of optional kernel features through the server environment. ## `id_map_current` This introduces a new internal `volatile.idmap.current` key which is used to track the current mapping for the container. This effectively gives us: * `volatile.last_state.idmap` => On-disk idmap * `volatile.idmap.current` => Current kernel map * `volatile.idmap.next` => Next on-disk idmap This is required to implement environments where the on-disk map isn't changed but the kernel map is (e.g. `idmapped mounts`). ## `event_location` Expose the location of the generation of API events. ## `storage_api_remote_volume_snapshots` This allows migrating storage volumes including their snapshots. ## `network_nat_address` This introduces the `ipv4.nat.address` and `ipv6.nat.address` configuration keys for Incus bridges. Those keys control the source address used for outbound traffic from the bridge. ## `container_nic_routes` This introduces the `ipv4.routes` and `ipv6.routes` properties on `nic` type devices. This allows adding static routes on host to container's NIC. ## `cluster_internal_copy` This makes it possible to do a normal `POST /1.0/containers` to copy a container between cluster nodes with Incus internally detecting whether a migration is required. ## `seccomp_notify` If the kernel supports `seccomp`-based syscall interception Incus can be notified by a container that a registered syscall has been performed. Incus can then decide to trigger various actions. ## `lxc_features` This introduces the `lxc_features` section output from the [`incus info`](incus_info.md) command via the `GET /1.0` route. It outputs the result of checks for key features being present in the underlying LXC library. ## `container_nic_ipvlan` This introduces the `ipvlan` `nic` device type. ## `network_vlan_sriov` This introduces VLAN (`vlan`) and MAC filtering (`security.mac_filtering`) support for SR-IOV devices. ## `storage_cephfs` Add support for CephFS as a storage pool driver. This can only be used for custom volumes, images and containers should be on Ceph (RBD) instead. ## `container_nic_ipfilter` This introduces container IP filtering (`security.ipv4_filtering` and `security.ipv6_filtering`) support for `bridged` NIC devices. ## `resources_v2` Rework the resources API at `/1.0/resources`, especially: * CPU * Fix reporting to track sockets, cores and threads * Track NUMA node per core * Track base and turbo frequency per socket * Track current frequency per core * Add CPU cache information * Export the CPU architecture * Show online/offline status of threads * Memory * Add huge-pages tracking * Track memory consumption per NUMA node too * GPU * Split DRM information to separate struct * Export device names and nodes in DRM struct * Export device name and node in NVIDIA struct * Add SR-IOV VF tracking ## `container_exec_user_group_cwd` Adds support for specifying `User`, `Group` and `Cwd` during `POST /1.0/containers/NAME/exec`. ## `container_syscall_intercept` Adds the `security.syscalls.intercept.*` configuration keys to control what system calls will be intercepted by Incus and processed with elevated permissions. ## `container_disk_shift` Adds the `shift` property on `disk` devices which controls the use of the `idmapped mounts` overlay. ## `storage_shifted` Introduces a new `security.shifted` Boolean on storage volumes. Setting it to `true` will allow multiple isolated containers to attach the same storage volume while keeping the file system writable from all of them. This makes use of `idmapped mounts` as an overlay file system. ## `resources_infiniband` Export InfiniBand character device information (`issm`, `umad`, `uverb`) as part of the resources API. ## `daemon_storage` This introduces two new configuration keys `storage.images_volume` and `storage.backups_volume` to allow for a storage volume on an existing pool be used for storing the daemon-wide images and backups artifacts. ## `instances` This introduces the concept of instances, of which currently the only type is `container`. ## `image_types` This introduces support for a new Type field on images, indicating what type of images they are. ## `resources_disk_sata` Extends the disk resource API struct to include: * Proper detection of SATA devices (type) * Device path * Drive RPM * Block size * Firmware version * Serial number ## `clustering_roles` This adds a new `roles` attribute to cluster entries, exposing a list of roles that the member serves in the cluster. ## `images_expiry` This allows for editing of the expiry date on images. ## `resources_network_firmware` Adds a `FirmwareVersion` field to network card entries. ## `backup_compression_algorithm` This adds support for a `compression_algorithm` property when creating a backup (`POST /1.0/containers//backups`). Setting this property overrides the server default value (`backups.compression_algorithm`). ## `ceph_data_pool_name` This adds support for an optional argument (`ceph.osd.data_pool_name`) when creating storage pools using Ceph RBD, when this argument is used the pool will store it's actual data in the pool specified with `data_pool_name` while keeping the metadata in the pool specified by `pool_name`. ## `container_syscall_intercept_mount` Adds the `security.syscalls.intercept.mount`, `security.syscalls.intercept.mount.allowed`, and `security.syscalls.intercept.mount.shift` configuration keys to control whether and how the `mount` system call will be intercepted by Incus and processed with elevated permissions. ## `compression_squashfs` Adds support for importing/exporting of images/backups using SquashFS file system format. ## `container_raw_mount` This adds support for passing in raw mount options for disk devices. ## `container_nic_routed` This introduces the `routed` `nic` device type. ## `container_syscall_intercept_mount_fuse` Adds the `security.syscalls.intercept.mount.fuse` key. It can be used to redirect file-system mounts to their fuse implementation. To this end, set e.g. `security.syscalls.intercept.mount.fuse=ext4=fuse2fs`. ## `container_disk_ceph` This allows for existing a Ceph RBD or CephFS to be directly connected to an Incus container. ## `virtual-machines` Add virtual machine support. ## `image_profiles` Allows a list of profiles to be applied to an image when launching a new container. ## `clustering_architecture` This adds a new `architecture` attribute to cluster members which indicates a cluster member's architecture. ## `resources_disk_id` Add a new `device_id` field in the disk entries on the resources API. ## `storage_lvm_stripes` This adds the ability to use LVM stripes on normal volumes and thin pool volumes. ## `vm_boot_priority` Adds a `boot.priority` property on NIC and disk devices to control the boot order. ## `unix_hotplug_devices` Adds support for Unix char and block device hotplugging. ## `api_filtering` Adds support for filtering the result of a GET request for instances and images. ## `instance_nic_network` Adds support for the `network` property on a NIC device to allow a NIC to be linked to a managed network. This allows it to inherit some of the network's settings and allows better validation of IP settings. ## `clustering_sizing` Support specifying a custom values for database voters and standbys. The new `cluster.max_voters` and `cluster.max_standby` configuration keys were introduced to specify to the ideal number of database voter and standbys. ## `firewall_driver` Adds the `Firewall` property to the `ServerEnvironment` struct indicating the firewall driver being used. ## `storage_lvm_vg_force_reuse` Introduces the ability to create a storage pool from an existing non-empty volume group. This option should be used with care, as Incus can then not guarantee that volume name conflicts won't occur with non-Incus created volumes in the same volume group. This could also potentially lead to Incus deleting a non-Incus volume should name conflicts occur. ## `container_syscall_intercept_hugetlbfs` When mount syscall interception is enabled and `hugetlbfs` is specified as an allowed file system type Incus will mount a separate `hugetlbfs` instance for the container with the UID and GID mount options set to the container's root UID and GID. This ensures that processes in the container can use huge pages. ## `limits_hugepages` This allows to limit the number of huge pages a container can use through the `hugetlb` cgroup. This means the `hugetlb` cgroup needs to be available. Note, that limiting huge pages is recommended when intercepting the mount syscall for the `hugetlbfs` file system to avoid allowing the container to exhaust the host's huge pages resources. ## `container_nic_routed_gateway` This introduces the `ipv4.gateway` and `ipv6.gateway` NIC configuration keys that can take a value of either `auto` or `none`. The default value for the key if unspecified is `auto`. This will cause the current behavior of a default gateway being added inside the container and the same gateway address being added to the host-side interface. If the value is set to `none` then no default gateway nor will the address be added to the host-side interface. This allows multiple routed NIC devices to be added to a container. ## `projects_restrictions` This introduces support for the `restricted` configuration key on project, which can prevent the use of security-sensitive features in a project. ## `custom_volume_snapshot_expiry` This allows custom volume snapshots to expiry. Expiry dates can be set individually, or by setting the `snapshots.expiry` configuration key on the parent custom volume which then automatically applies to all created snapshots. ## `volume_snapshot_scheduling` This adds support for custom volume snapshot scheduling. It introduces two new configuration keys: `snapshots.schedule` and `snapshots.pattern`. Snapshots can be created automatically up to every minute. ## `trust_ca_certificates` This allows for checking client certificates trusted by the provided CA (`server.ca`). It can be enabled by setting `core.trust_ca_certificates` to `true`. If enabled, it will perform the check, and bypass the trusted password if `true`. An exception will be made if the connecting client certificate is in the provided CRL (`ca.crl`). In this case, it will ask for the password. ## `snapshot_disk_usage` This adds a new `size` field to the output of `/1.0/instances//snapshots/` which represents the disk usage of the snapshot. ## `clustering_edit_roles` This adds a writable endpoint for cluster members, allowing the editing of their roles. ## `container_nic_routed_host_address` This introduces the `ipv4.host_address` and `ipv6.host_address` NIC configuration keys that can be used to control the host-side `veth` interface's IP addresses. This can be useful when using multiple routed NICs at the same time and needing a predictable next-hop address to use. This also alters the behavior of `ipv4.gateway` and `ipv6.gateway` NIC configuration keys. When they are set to `auto` the container will have its default gateway set to the value of `ipv4.host_address` or `ipv6.host_address` respectively. The default values are: `ipv4.host_address`: `169.254.0.1` `ipv6.host_address`: `fe80::1` This is backward compatible with the previous default behavior. ## `container_nic_ipvlan_gateway` This introduces the `ipv4.gateway` and `ipv6.gateway` NIC configuration keys that can take a value of either `auto` or `none`. The default value for the key if unspecified is `auto`. This will cause the current behavior of a default gateway being added inside the container and the same gateway address being added to the host-side interface. If the value is set to `none` then no default gateway nor will the address be added to the host-side interface. This allows multiple IPVLAN NIC devices to be added to a container. ## `resources_usb_pci` This adds USB and PCI devices to the output of `/1.0/resources`. ## `resources_cpu_threads_numa` This indicates that the `numa_node` field is now recorded per-thread rather than per core as some hardware apparently puts threads in different NUMA domains. ## `resources_cpu_core_die` Exposes the `die_id` information on each core. ## `api_os` This introduces two new fields in `/1.0`, `os` and `os_version`. Those are taken from the OS-release data on the system. ## `container_nic_routed_host_table` This introduces the `ipv4.host_table` and `ipv6.host_table` NIC configuration keys that can be used to add static routes for the instance's IPs to a custom policy routing table by ID. ## `container_nic_ipvlan_host_table` This introduces the `ipv4.host_table` and `ipv6.host_table` NIC configuration keys that can be used to add static routes for the instance's IPs to a custom policy routing table by ID. ## `container_nic_ipvlan_mode` This introduces the `mode` NIC configuration key that can be used to switch the `ipvlan` mode into either `l2` or `l3s`. If not specified, the default value is `l3s` (which is the old behavior). In `l2` mode the `ipv4.address` and `ipv6.address` keys will accept addresses in either CIDR or singular formats. If singular format is used, the default subnet size is taken to be /24 and /64 for IPv4 and IPv6 respectively. In `l2` mode the `ipv4.gateway` and `ipv6.gateway` keys accept only a singular IP address. ## `resources_system` This adds system information to the output of `/1.0/resources`. ## `images_push_relay` This adds the push and relay modes to image copy. It also introduces the following new endpoint: * `POST 1.0/images//export` ## `network_dns_search` This introduces the `dns.search` configuration option on networks. ## `container_nic_routed_limits` This introduces `limits.ingress`, `limits.egress` and `limits.max` for routed NICs. ## `instance_nic_bridged_vlan` This introduces the `vlan` and `vlan.tagged` settings for `bridged` NICs. `vlan` specifies the non-tagged VLAN to join, and `vlan.tagged` is a comma-delimited list of tagged VLANs to join. ## `network_state_bond_bridge` This adds a `bridge` and `bond` section to the `/1.0/networks/NAME/state` API. Those contain additional state information relevant to those particular types. Bond: * Mode * Transmit hash * Up delay * Down delay * MII frequency * MII state * Lower devices Bridge: * ID * Forward delay * STP mode * Default VLAN * VLAN filtering * Upper devices ## `resources_cpu_isolated` Add an `Isolated` property on CPU threads to indicate if the thread is physically `Online` but is configured not to accept tasks. ## `usedby_consistency` This extension indicates that `UsedBy` should now be consistent with suitable `?project=` and `?target=` when appropriate. The 5 entities that have `UsedBy` are: * Profiles * Projects * Networks * Storage pools * Storage volumes ## `custom_block_volumes` This adds support for creating and attaching custom block volumes to instances. It introduces the new `--type` flag when creating custom storage volumes, and accepts the values `fs` and `block`. ## `clustering_failure_domains` This extension adds a new `failure_domain` field to the `PUT /1.0/cluster/` API, which can be used to set the failure domain of a node. ## `container_syscall_filtering_allow_deny_syntax` A number of new syscalls related container configuration keys were updated. * `security.syscalls.deny_default` * `security.syscalls.deny_compat` * `security.syscalls.deny` * `security.syscalls.allow` ## `resources_gpu_mdev` Expose available mediated device profiles and devices in `/1.0/resources`. ## `console_vga_type` This extends the `/1.0/console` endpoint to take a `?type=` argument, which can be set to `console` (default) or `vga` (the new type added by this extension). When doing a `POST` to `/1.0//console?type=vga` the data WebSocket returned by the operation in the metadata field will be a bidirectional proxy attached to a SPICE Unix socket of the target virtual machine. ## `projects_limits_disk` Add `limits.disk` to the available project configuration keys. If set, it limits the total amount of disk space that instances volumes, custom volumes and images volumes can use in the project. ## `network_type_macvlan` Adds support for additional network type `macvlan` and adds `parent` configuration key for this network type to specify which parent interface should be used for creating NIC device interfaces on top of. Also adds `network` configuration key support for `macvlan` NICs to allow them to specify the associated network of the same type that they should use as the basis for the NIC device. ## `network_type_sriov` Adds support for additional network type `sriov` and adds `parent` configuration key for this network type to specify which parent interface should be used for creating NIC device interfaces on top of. Also adds `network` configuration key support for `sriov` NICs to allow them to specify the associated network of the same type that they should use as the basis for the NIC device. ## `container_syscall_intercept_bpf_devices` This adds support to intercept the `bpf` syscall in containers. Specifically, it allows to manage device cgroup `bpf` programs. ## `network_type_ovn` Adds support for additional network type `ovn` with the ability to specify a `bridge` type network as the `parent`. Introduces a new NIC device type of `ovn` which allows the `network` configuration key to specify which `ovn` type network they should connect to. Also introduces two new global configuration keys that apply to all `ovn` networks and NIC devices: * `network.ovn.integration_bridge` - the OVS integration bridge to use. * `network.ovn.northbound_connection` - the OVN northbound database connection string. ## `projects_networks` Adds the `features.networks` configuration key to projects and the ability for a project to hold networks. ## `projects_networks_restricted_uplinks` Adds the `restricted.networks.uplinks` project configuration key to indicate (as a comma-delimited list) which networks the networks created inside the project can use as their uplink network. ## `custom_volume_backup` Add custom volume backup support. This includes the following new endpoints (see [RESTful API](rest-api.md) for details): * `GET /1.0/storage-pools////backups` * `POST /1.0/storage-pools////backups` * `GET /1.0/storage-pools////backups/` * `POST /1.0/storage-pools////backups/` * `DELETE /1.0/storage-pools////backups/` * `GET /1.0/storage-pools////backups//export` The following existing endpoint has been modified: * `POST /1.0/storage-pools///` accepts the new source type `backup` ## `backup_override_name` Adds `Name` field to `InstanceBackupArgs` to allow specifying a different instance name when restoring a backup. Adds `Name` and `PoolName` fields to `StoragePoolVolumeBackupArgs` to allow specifying a different volume name when restoring a custom volume backup. ## `storage_rsync_compression` Adds `rsync.compression` configuration key to storage pools. This key can be used to disable compression in `rsync` while migrating storage pools. ## `network_type_physical` Adds support for additional network type `physical` that can be used as an uplink for `ovn` networks. The interface specified by `parent` on the `physical` network will be connected to the `ovn` network's gateway. ## `network_ovn_external_subnets` Adds support for `ovn` networks to use external subnets from uplink networks. Introduces the `ipv4.routes` and `ipv6.routes` setting on `physical` networks that defines the external routes allowed to be used in child OVN networks in their `ipv4.routes.external` and `ipv6.routes.external` settings. Introduces the `restricted.networks.subnets` project setting that specifies which external subnets are allowed to be used by OVN networks inside the project (if not set then all routes defined on the uplink network are allowed). ## `network_ovn_nat` Adds support for `ipv4.nat` and `ipv6.nat` settings on `ovn` networks. When creating the network if these settings are unspecified, and an equivalent IP address is being generated for the subnet, then the appropriate NAT setting will added set to `true`. If the setting is missing then the value is taken as `false`. ## `network_ovn_external_routes_remove` Removes the settings `ipv4.routes.external` and `ipv6.routes.external` from `ovn` networks. The equivalent settings on the `ovn` NIC type can be used instead for this, rather than having to specify them both at the network and NIC level. ## `tpm_device_type` This introduces the `tpm` device type. ## `storage_zfs_clone_copy_rebase` This introduces `rebase` as a value for `zfs.clone_copy` causing Incus to track down any `image` dataset in the ancestry line and then perform send/receive on top of that. ## `gpu_mdev` This adds support for virtual GPUs. It introduces the `mdev` configuration key for GPU devices which takes a supported `mdev` type, e.g. `i915-GVTg_V5_4`. ## `resources_pci_iommu` This adds the `IOMMUGroup` field for PCI entries in the resources API. ## `resources_network_usb` Adds the `usb_address` field to the network card entries in the resources API. ## `resources_disk_address` Adds the `usb_address` and `pci_address` fields to the disk entries in the resources API. ## `network_physical_ovn_ingress_mode` Adds `ovn.ingress_mode` setting for `physical` networks. Sets the method that OVN NIC external IPs will be advertised on uplink network. Either `l2proxy` (proxy ARP/NDP) or `routed`. ## `network_ovn_dhcp` Adds `ipv4.dhcp` and `ipv6.dhcp` settings for `ovn` networks. Allows DHCP (and RA for IPv6) to be disabled. Defaults to on. ## `network_physical_routes_anycast` Adds `ipv4.routes.anycast` and `ipv6.routes.anycast` Boolean settings for `physical` networks. Defaults to `false`. Allows OVN networks using physical network as uplink to relax external subnet/route overlap detection when used with `ovn.ingress_mode=routed`. ## `projects_limits_instances` Adds `limits.instances` to the available project configuration keys. If set, it limits the total number of instances (VMs and containers) that can be used in the project. ## `network_state_vlan` This adds a `vlan` section to the `/1.0/networks/NAME/state` API. Those contain additional state information relevant to VLAN interfaces: * `lower_device` * `vid` ## `instance_nic_bridged_port_isolation` This adds the `security.port_isolation` field for bridged NIC instances. ## `instance_bulk_state_change` Adds the following endpoint for bulk state change (see [RESTful API](rest-api.md) for details): * `PUT /1.0/instances` ## `network_gvrp` This adds an optional `gvrp` property to `macvlan` and `physical` networks, and to `ipvlan`, `macvlan`, `routed` and `physical` NIC devices. When set, this specifies whether the VLAN should be registered using GARP VLAN Registration Protocol. Defaults to `false`. ## `instance_pool_move` This adds a `pool` field to the `POST /1.0/instances/NAME` API, allowing for easy move of an instance root disk between pools. ## `gpu_sriov` This adds support for SR-IOV enabled GPUs. It introduces the `sriov` GPU type property. ## `pci_device_type` This introduces the `pci` device type. ## `storage_volume_state` Add new `/1.0/storage-pools/POOL/volumes/VOLUME/state` API endpoint to get usage data on a volume. ## `network_acl` This adds the concept of network ACLs to API under the API endpoint prefix `/1.0/network-acls`. ## `migration_stateful` Add a new `migration.stateful` configuration key. ## `disk_state_quota` This introduces the `size.state` device configuration key on `disk` devices. ## `storage_ceph_features` Adds a new `ceph.rbd.features` configuration key on storage pools to control the RBD features used for new volumes. ## `projects_compression` Adds new `backups.compression_algorithm` and `images.compression_algorithm` configuration keys which allows configuration of backup and image compression per-project. ## `projects_images_remote_cache_expiry` Add new `images.remote_cache_expiry` configuration key to projects, allowing for set number of days after which an unused cached remote image will be flushed. ## `certificate_project` Adds a new `restricted` property to certificates in the API as well as `projects` holding a list of project names that the certificate has access to. ## `network_ovn_acl` Adds a new `security.acls` property to OVN networks and OVN NICs, allowing Network ACLs to be applied. ## `projects_images_auto_update` Adds new `images.auto_update_cached` and `images.auto_update_interval` configuration keys which allows configuration of images auto update in projects ## `projects_restricted_cluster_target` Adds new `restricted.cluster.target` configuration key to project which prevent the user from using --target to specify what cluster member to place a workload on or the ability to move a workload between members. ## `images_default_architecture` Adds new `images.default_architecture` global configuration key and matching per-project key which lets user tell Incus what architecture to go with when no specific one is specified as part of the image request. ## `network_ovn_acl_defaults` Adds new `security.acls.default.{in,e}gress.action` and `security.acls.default.{in,e}gress.logged` configuration keys for OVN networks and NICs. This replaces the removed ACL `default.action` and `default.logged` keys. ## `gpu_mig` This adds support for NVIDIA MIG. It introduces the `mig` GPU type and associated configuration keys. ## `project_usage` Adds an API endpoint to get current resource allocations in a project. Accessible at API `GET /1.0/projects//state`. ## `network_bridge_acl` Adds a new `security.acls` configuration key to `bridge` networks, allowing Network ACLs to be applied. Also adds `security.acls.default.{in,e}gress.action` and `security.acls.default.{in,e}gress.logged` configuration keys for specifying the default behavior for unmatched traffic. ## `warnings` Warning API for Incus. This includes the following endpoints (see [Restful API](rest-api.md) for details): * `GET /1.0/warnings` * `GET /1.0/warnings/` * `PUT /1.0/warnings/` * `DELETE /1.0/warnings/` ## `projects_restricted_backups_and_snapshots` Adds new `restricted.backups` and `restricted.snapshots` configuration keys to project which prevents the user from creation of backups and snapshots. ## `clustering_join_token` Adds `POST /1.0/cluster/members` API endpoint for requesting a join token used when adding new cluster members without using the trust password. ## `clustering_description` Adds an editable description to the cluster members. ## `server_trusted_proxy` This introduces support for `core.https_trusted_proxy` which has Incus parse a HAProxy style connection header on such connections and if present, will rewrite the request's source address to that provided by the proxy server. ## `clustering_update_cert` Adds `PUT /1.0/cluster/certificate` endpoint for updating the cluster certificate across the whole cluster ## `storage_api_project` This adds support for copy/move custom storage volumes between projects. ## `server_instance_driver_operational` This modifies the `driver` output for the `/1.0` endpoint to only include drivers which are actually supported and operational on the server (as opposed to being included in Incus but not operational on the server). ## `server_supported_storage_drivers` This adds supported storage driver info to server environment info. ## `event_lifecycle_requestor_address` Adds a new address field to `lifecycle` requestor. ## `resources_gpu_usb` Add a new `USBAddress` (`usb_address`) field to `ResourcesGPUCard` (GPU entries) in the resources API. ## `clustering_evacuation` Adds `POST /1.0/cluster/members//state` endpoint for evacuating and restoring cluster members. It also adds the configuration keys `cluster.evacuate` and `volatile.evacuate.origin` for setting the evacuation method (`auto`, `stop` or `migrate`) and the origin of any migrated instance respectively. ## `network_ovn_nat_address` This introduces the `ipv4.nat.address` and `ipv6.nat.address` configuration keys for Incus `ovn` networks. Those keys control the source address used for outbound traffic from the OVN virtual network. These keys can only be specified when the OVN network's uplink network has `ovn.ingress_mode=routed`. ## `network_bgp` This introduces support for Incus acting as a BGP router to advertise routes to `bridge` and `ovn` networks. This comes with the addition to global configuration of: * `core.bgp_address` * `core.bgp_asn` * `core.bgp_routerid` The following network configurations keys (`bridge` and `physical`): * `bgp.peers..address` * `bgp.peers..asn` * `bgp.peers..password` The `nexthop` configuration keys (`bridge`): * `bgp.ipv4.nexthop` * `bgp.ipv6.nexthop` And the following NIC-specific configuration keys (`bridged` NIC type): * `ipv4.routes.external` * `ipv6.routes.external` ## `network_forward` This introduces the networking address forward functionality. Allowing for `bridge` and `ovn` networks to define external IP addresses that can be forwarded to internal IP(s) inside their respective networks. ## `custom_volume_refresh` Adds support for refresh during volume migration. ## `network_counters_errors_dropped` This adds the received and sent errors as well as inbound and outbound dropped packets to the network counters. ## `metrics` This adds metrics to Incus. It returns metrics of running instances using the OpenMetrics format. This includes the following endpoints: * `GET /1.0/metrics` ## `image_source_project` Adds a new `project` field to `POST /1.0/images` allowing for the source project to be set at image copy time. ## `clustering_config` Adds new `config` property to cluster members with configurable key/value pairs. ## `network_peer` This adds network peering to allow traffic to flow between OVN networks without leaving the OVN subsystem. ## `linux_sysctl` Adds new `linux.sysctl.*` configuration keys allowing users to modify certain kernel parameters within containers. ## `network_dns` Introduces a built-in DNS server and zones API to provide DNS records for Incus instances. This introduces the following server configuration key: * `core.dns_address` The following network configuration key: * `dns.zone.forward` * `dns.zone.reverse.ipv4` * `dns.zone.reverse.ipv6` And the following project configuration key: * `restricted.networks.zones` A new REST API is also introduced to manage DNS zones: * `/1.0/network-zones` (GET, POST) * `/1.0/network-zones/` (GET, PUT, PATCH, DELETE) ## `ovn_nic_acceleration` Adds new `acceleration` configuration key to OVN NICs which can be used for enabling hardware offloading. It takes the values `none` or `sriov`. ## `certificate_self_renewal` This adds support for renewing a client's own trust certificate. ## `instance_project_move` This adds a `project` field to the `POST /1.0/instances/NAME` API, allowing for easy move of an instance between projects. ## `storage_volume_project_move` This adds support for moving storage volume between projects. ## `cloud_init` This adds a new `cloud-init` configuration key namespace which contains the following keys: * `cloud-init.vendor-data` * `cloud-init.user-data` * `cloud-init.network-config` It also adds a new endpoint `/1.0/devices` to `/dev/incus` which shows an instance's devices. ## `network_dns_nat` This introduces `network.nat` as a configuration option on network zones (DNS). It defaults to the current behavior of generating records for all instances NICs but if set to `false`, it will instruct Incus to only generate records for externally reachable addresses. ## `database_leader` Adds new `database-leader` role which is assigned to cluster leader. ## `instance_all_projects` This adds support for displaying instances from all projects. ## `clustering_groups` Add support for grouping cluster members. This introduces the following new endpoints: * `/1.0/cluster/groups` (GET, POST) * `/1.0/cluster/groups/` (GET, POST, PUT, PATCH, DELETE) The following project restriction is added: * `restricted.cluster.groups` ## `ceph_rbd_du` Adds a new `ceph.rbd.du` Boolean on Ceph storage pools which allows disabling the use of the potentially slow `rbd du` calls. ## `instance_get_full` This introduces a new `recursion=1` mode for `GET /1.0/instances/{name}` which allows for the retrieval of all instance structs, including the state, snapshots and backup structs. ## `qemu_metrics` This adds a new `security.agent.metrics` Boolean which defaults to `true`. When set to `false`, it doesn't connect to the `incus-agent` for metrics and other state information, but relies on stats from QEMU. ## `gpu_mig_uuid` Adds support for the new MIG UUID format used by NVIDIA `470+` drivers (for example, `MIG-74c6a31a-fde5-5c61-973b-70e12346c202`), the `MIG-` prefix can be omitted This extension supersedes old `mig.gi` and `mig.ci` parameters which are kept for compatibility with old drivers and cannot be set together. ## `event_project` Expose the project an API event belongs to. ## `clustering_evacuation_live` This adds `live-migrate` as a configuration option to `cluster.evacuate`, which forces live-migration of instances during cluster evacuation. ## `instance_allow_inconsistent_copy` Adds `allow_inconsistent` field to instance source on `POST /1.0/instances`. If `true`, `rsync` will ignore the `Partial transfer due to vanished source files` (code 24) error when creating an instance from a copy. ## `network_state_ovn` This adds an `ovn` section to the `/1.0/networks/NAME/state` API which contains additional state information relevant to OVN networks: * chassis ## `storage_volume_api_filtering` Adds support for filtering the result of a GET request for storage volumes. ## `image_restrictions` This extension adds on to the image properties to include image restrictions/host requirements. These requirements help determine the compatibility between an instance and the host system. ## `storage_zfs_export` Introduces the ability to disable zpool export when unmounting pool by setting `zfs.export`. ## `network_dns_records` This extends the network zones (DNS) API to add the ability to create and manage custom records. This adds: * `GET /1.0/network-zones/ZONE/records` * `POST /1.0/network-zones/ZONE/records` * `GET /1.0/network-zones/ZONE/records/RECORD` * `PUT /1.0/network-zones/ZONE/records/RECORD` * `PATCH /1.0/network-zones/ZONE/records/RECORD` * `DELETE /1.0/network-zones/ZONE/records/RECORD` ## `network_zones_all_projects` This adds support for listing network zones across all projects through the `all-projects` parameter on the `GET /1.0/network-zones`API. ## `storage_zfs_reserve_space` Adds ability to set the `reservation`/`refreservation` ZFS property along with `quota`/`refquota`. ## `network_acl_log` Adds a new `GET /1.0/networks-acls/NAME/log` API to retrieve ACL firewall logs. ## `storage_zfs_blocksize` Introduces a new `zfs.blocksize` property for ZFS storage volumes which allows to set volume block size. ## `metrics_cpu_seconds` This is used to detect whether Incus was fixed to output used CPU time in seconds rather than as milliseconds. ## `instance_snapshot_never` Adds a `@never` option to `snapshots.schedule` which allows disabling inheritance. ## `certificate_token` This adds token-based certificate addition to the trust store as a safer alternative to a trust password. It adds the `token` field to `POST /1.0/certificates`. ## `instance_nic_routed_neighbor_probe` This adds the ability to disable the `routed` NIC IP neighbor probing for availability on the parent network. Adds the `ipv4.neighbor_probe` and `ipv6.neighbor_probe` NIC settings. Defaulting to `true` if not specified. ## `event_hub` This adds support for `event-hub` cluster member role and the `ServerEventMode` environment field. ## `agent_nic_config` If set to `true`, on VM start-up the `incus-agent` will apply NIC configuration to change the names and MTU of the instance NIC devices. ## `projects_restricted_intercept` Adds new `restricted.container.intercept` configuration key to allow usually safe system call interception options. ## `metrics_authentication` Introduces a new `core.metrics_authentication` server configuration option to allow for the `/1.0/metrics` endpoint to be generally available without client authentication. ## `images_target_project` Adds ability to copy image to a project different from the source. ## `images_all_projects` This adds support for listing images across all projects through the `all-projects` parameter on the `GET /1.0/images`API. ## `cluster_migration_inconsistent_copy` Adds `allow_inconsistent` field to `POST /1.0/instances/`. Set to `true` to allow inconsistent copying between cluster members. ## `cluster_ovn_chassis` Introduces a new `ovn-chassis` cluster role which allows for specifying what cluster member should act as an OVN chassis. ## `container_syscall_intercept_sched_setscheduler` Adds the `security.syscalls.intercept.sched_setscheduler` to allow advanced process priority management in containers. ## `storage_lvm_thinpool_metadata_size` Introduces the ability to specify the thin pool metadata volume size via `storage.thinpool_metadata_size`. If this is not specified then the default is to let LVM pick an appropriate thin pool metadata volume size. ## `storage_volume_state_total` This adds `total` field to the `GET /1.0/storage-pools/{name}/volumes/{type}/{volume}/state` API. ## `instance_file_head` Implements HEAD on `/1.0/instances/NAME/file`. ## `instances_nic_host_name` This introduces the `instances.nic.host_name` server configuration key that can take a value of either `random` or `mac`. The default value for the key if unspecified is `random`. If it is set to random then use the random host interface names. If it's set to `mac`, then generate a name in the form `inc1122334455`. ## `image_copy_profile` Adds ability to modify the set of profiles when image is copied. ## `container_syscall_intercept_sysinfo` Adds the `security.syscalls.intercept.sysinfo` to allow the `sysinfo` syscall to be populated with cgroup-based resource usage information. ## `clustering_evacuation_mode` This introduces a `mode` field to the evacuation request which allows for overriding the evacuation mode traditionally set through `cluster.evacuate`. ## `resources_pci_vpd` Adds a new VPD struct to the PCI resource entries. This struct extracts vendor provided data including the full product name and additional key/value configuration pairs. ## `qemu_raw_conf` Introduces a `raw.qemu.conf` configuration key to override select sections of the generated `qemu.conf`. ## `storage_cephfs_fscache` Add support for `fscache`/`cachefilesd` on CephFS pools through a new `cephfs.fscache` configuration option. ## `network_load_balancer` This introduces the networking load balancer functionality. Allowing `ovn` networks to define port(s) on external IP addresses that can be forwarded to one or more internal IP(s) inside their respective networks. ## `vsock_api` This introduces a bidirectional `vsock` interface which allows the `incus-agent` and the Incus server to communicate better. ## `instance_ready_state` This introduces a new `Ready` state for instances which can be set using `/dev/incus`. ## `network_bgp_holdtime` This introduces a new `bgp.peers..holdtime` configuration key to control the BGP hold time for a particular peer. ## `storage_volumes_all_projects` This introduces the ability to list storage volumes from all projects. ## `metrics_memory_oom_total` This introduces a new `incus_memory_OOM_kills_total` metric to the `/1.0/metrics` API. It reports the number of times the out of memory killer (`OOM`) has been triggered. ## `storage_buckets` This introduces the storage bucket API. It allows the management of S3 object storage buckets for storage pools. ## `storage_buckets_create_credentials` This updates the storage bucket API to return initial admin credentials at bucket creation time. ## `metrics_cpu_effective_total` This introduces a new `incus_cpu_effective_total` metric to the `/1.0/metrics` API. It reports the total number of effective CPUs. ## `projects_networks_restricted_access` Adds the `restricted.networks.access` project configuration key to indicate (as a comma-delimited list) which networks can be accessed inside the project. If not specified, all networks are accessible (assuming it is also allowed by the `restricted.devices.nic` setting, described below). This also introduces a change whereby network access is controlled by the project's `restricted.devices.nic` setting: * If `restricted.devices.nic` is set to `managed` (the default if not specified), only managed networks are accessible. * If `restricted.devices.nic` is set to `allow`, all networks are accessible (dependent on the `restricted.networks.access` setting). * If `restricted.devices.nic` is set to `block`, no networks are accessible. ## `storage_buckets_local` This introduces the ability to use storage buckets on local storage pools by setting the new `core.storage_buckets_address` global configuration setting. ## `loki` This adds support for sending life cycle and logging events to a Loki server. It adds the following global configuration keys: * `loki.api.ca_cert`: CA certificate which can be used when sending events to the Loki server * `loki.api.url`: URL to the Loki server (protocol, name or IP and port) * `loki.auth.username` and `loki.auth.password`: Used if Loki is behind a reverse proxy with basic authentication enabled * `loki.labels`: Comma-separated list of values which are to be used as labels for Loki events. * `loki.loglevel`: Minimum log level for events sent to the Loki server. * `loki.types`: Types of events which are to be sent to the Loki server (`lifecycle` and/or `logging`). ## `acme` This adds ACME support, which allows [Let's Encrypt](https://letsencrypt.org/) or other ACME services to issue certificates. It adds the following global configuration keys: * `acme.domain`: The domain for which the certificate should be issued. * `acme.email`: The email address used for the account of the ACME service. * `acme.ca_url`: The directory URL of the ACME service, defaults to `https://acme-v02.api.letsencrypt.org/directory`. It also adds the following endpoint, which is required for the HTTP-01 challenge: * `/.well-known/acme-challenge/` ## `internal_metrics` This adds internal metrics to the list of metrics. These include: * Total running operations * Total active warnings * Daemon uptime in seconds * Go memory stats * Number of goroutines ## `cluster_join_token_expiry` This adds an expiry to cluster join tokens which defaults to 3 hours, but can be changed by setting the `cluster.join_token_expiry` configuration key. ## `remote_token_expiry` This adds an expiry to remote add join tokens. It can be set in the `core.remote_token_expiry` configuration key, and default to no expiry. ## `storage_volumes_created_at` This change adds support for storing the creation date and time of storage volumes and their snapshots. This adds the `CreatedAt` field to the `StorageVolume` and `StorageVolumeSnapshot` API types. ## `cpu_hotplug` This adds CPU hotplugging for VMs. Hotplugging is disabled when using CPU pinning, because this would require hotplugging NUMA devices as well, which is not possible. ## `projects_networks_zones` This adds support for the `features.networks.zones` project feature, which changes which project network zones are associated with when they are created. Previously network zones were tied to the value of `features.networks`, meaning they were created in the same project as networks were. Now this has been decoupled from `features.networks` to allow projects that share a network in the default project (i.e those with `features.networks=false`) to have their own project level DNS zones that give a project oriented "view" of the addresses on that shared network (which only includes addresses from instances in their project). This also introduces a change to the network `dns.zone.forward` setting, which now accepts a comma-separated of DNS zone names (a maximum of one per project) in order to associate a shared network with multiple zones. No change to the `dns.zone.reverse.*` settings have been made, they still only allow a single DNS zone to be set. However the resulting zone content that is generated now includes `PTR` records covering addresses from all projects that are referencing that network via one of their forward zones. Existing projects that have `features.networks=true` will have `features.networks.zones=true` set automatically, but new projects will need to specify this explicitly. ## `instance_nic_txqueuelength` Adds a `txqueuelen` key to control the `txqueuelen` parameter of the NIC device. ## `cluster_member_state` Adds `GET /1.0/cluster/members//state` API endpoint and associated `ClusterMemberState` API response type. ## `instances_placement_scriptlet` Adds support for a Starlark scriptlet to be provided to Incus to allow customized logic that controls placement of new instances in a cluster. The Starlark scriptlet is provided to Incus via the new global configuration option `instances.placement.scriptlet`. ## `storage_pool_source_wipe` Adds support for a `source.wipe` Boolean on the storage pool, indicating that Incus should wipe partition headers off the requested disk rather than potentially fail due to pre-existing file systems. ## `zfs_block_mode` This adds support for using ZFS block {spellexception}`filesystem` volumes allowing the use of different file systems on top of ZFS. This adds the following new configuration options for ZFS storage pools: * `volume.zfs.block_mode` * `volume.block.mount_options` * `volume.block.filesystem` ## `instance_generation_id` Adds support for instance generation ID. The VM or container generation ID will change whenever the instance's place in time moves backwards. As of now, the generation ID is only exposed through to VM type instances. This allows for the VM guest OS to reinitialize any state it needs to avoid duplicating potential state that has already occurred: * `volatile.uuid.generation` ## `disk_io_cache` This introduces a new `io.cache` property to disk devices which can be used to override the VM caching behavior. ## `amd_sev` Adds support for AMD SEV (Secure Encrypted Virtualization) that can be used to encrypt the memory of a guest VM. This adds the following new configuration options for SEV encryption: * `security.sev` : (bool) is SEV enabled for this VM * `security.sev.policy.es` : (bool) is SEV-ES enabled for this VM * `security.sev.session.dh` : (string) guest owner's `base64`-encoded Diffie-Hellman key * `security.sev.session.data` : (string) guest owner's `base64`-encoded session blob ## `storage_pool_loop_resize` This allows growing loop file backed storage pools by changing the `size` setting of the pool. ## `migration_vm_live` This adds support for performing VM QEMU to QEMU live migration for both shared storage (clustered Ceph) and non-shared storage pools. This also adds the `CRIUType_VM_QEMU` value of `3` for the migration `CRIUType` `protobuf` field. ## `ovn_nic_nesting` This adds support for nesting an `ovn` NIC inside another `ovn` NIC on the same instance. This allows for an OVN logical switch port to be tunneled inside another OVN NIC using VLAN tagging. This feature is configured by specifying the parent NIC name using the `nested` property and the VLAN ID to use for tunneling with the `vlan` property. ## `oidc` This adds support for OpenID Connect (OIDC) authentication. This adds the following new configuration keys: * `oidc.issuer` * `oidc.client.id` * `oidc.audience` ## `network_ovn_l3only` This adds the ability to set an `ovn` network into "layer 3 only" mode. This mode can be enabled at IPv4 or IPv6 level using `ipv4.l3only` and `ipv6.l3only` configuration options respectively. With this mode enabled the following changes are made to the network: * The virtual router's internal port address will be configured with a single host netmask (e.g. /32 for IPv4 or /128 for IPv6). * Static routes for active instance NIC addresses will be added to the virtual router. * A discard route for the entire internal subnet will be added to the virtual router to prevent packets destined for inactive addresses from escaping to the uplink network. * The DHCPv4 server will be configured to indicate that a netmask of 255.255.255.255 be used for instance configuration. ## `ovn_nic_acceleration_vdpa` This updates the `ovn_nic_acceleration` API extension. The `acceleration` configuration key for OVN NICs can now takes the value `vdpa` to support Virtual Data Path Acceleration (VDPA). ## `cluster_healing` This adds cluster healing which automatically evacuates offline cluster members. This adds the following new configuration key: * `cluster.healing_threshold` The configuration key takes an integer, and can be disabled by setting it to 0 (default). If set, the value represents the threshold after which an offline cluster member is to be evacuated. In case the value is lower than `cluster.offline_threshold`, that value will be used instead. When the offline cluster member is evacuated, only remote-backed instances will be migrated. Local instances will be ignored as there is no way of migrating them once the cluster member is offline. ## `instances_state_total` This extension adds a new `total` field to `InstanceStateDisk` and `InstanceStateMemory`, both part of the instance's state API. ## `auth_user` Add current user details to the main API endpoint. This introduces: * `auth_user_name` * `auth_user_method` ## `security_csm` Introduce a new `security.csm` configuration key to control the use of `CSM` (Compatibility Support Module) to allow legacy operating systems to be run in Incus VMs. ## `instances_rebuild` This extension adds the ability to rebuild an instance with the same origin image, alternate image or as empty. A new `POST /1.0/instances//rebuild?project=` API endpoint has been added as well as a new CLI command [`incus rebuild`](incus_rebuild.md). ## `numa_cpu_placement` This adds the possibility to place a set of CPUs in a desired set of NUMA nodes. This adds the following new configuration key: * `limits.cpu.nodes` : (string) comma-separated list of NUMA node IDs or NUMA node ID ranges to place the CPUs (chosen with a dynamic value of `limits.cpu`) in. ## `custom_volume_iso` This adds the possibility to import ISO images as custom storage volumes. This adds the `--type` flag to [`incus storage volume import`](incus_storage_volume_import.md). ## `network_allocations` This adds the possibility to list an Incus deployment's network allocations. Through the [`incus network list-allocations`](incus_network_list-allocations.md) command and the `--project | --all-projects` flags, you can list all the used IP addresses, hardware addresses (for instances), resource URIs and whether it uses NAT for each `instance`, `network`, `network forward` and `network load-balancer`. ## `zfs_delegate` This implements a new `zfs.delegate` volume Boolean for volumes on a ZFS storage driver. When enabled and a suitable system is in use (requires ZFS 2.2 or higher), the ZFS dataset will be delegated to the container, allowing for its use through the `zfs` command line tool. ## `storage_api_remote_volume_snapshot_copy` This allows copying storage volume snapshots to and from remotes. ## `operations_get_query_all_projects` This introduces support for the `all-projects` query parameter for the GET API calls to both `/1.0/operations` and `/1.0/operations?recursion=1`. This parameter allows bypassing the project name filter. ## `metadata_configuration` Adds the `GET /1.0/metadata/configuration` API endpoint to retrieve the generated metadata configuration in a JSON format. The JSON structure adopts the structure ```"configs" > `ENTITY` > `ENTITY_SECTION` > "keys" > [, , ...]```. ## `syslog_socket` This introduces a syslog socket that can receive syslog formatted log messages. These can be viewed in the events API and `incus monitor`, and can be forwarded to Loki. To enable this feature, set `core.syslog_socket` to `true`. ## `event_lifecycle_name_and_project` This adds the fields `Name` and `Project` to `lifecycle` events. ## `instances_nic_limits_priority` This introduces a new per-NIC `limits.priority` option that works with both cgroup1 and cgroup2 unlike the deprecated `limits.network.priority` instance setting, which only worked with cgroup1. ## `disk_initial_volume_configuration` This API extension provides the capability to set initial volume configurations for instance root devices. Initial volume configurations are prefixed with `initial.` and can be specified either through profiles or directly during instance initialization using the `--device` flag. Note that these configuration are applied only at the time of instance creation and subsequent modifications have no effect on existing devices. ## `operation_wait` This API extension indicates that the `/1.0/operations/{id}/wait` endpoint exists on the server. This indicates to the client that the endpoint can be used to wait for an operation to complete rather than waiting for an operation event via the `/1.0/events` endpoint. ## `image_restriction_privileged` This extension adds a new image restriction, `requirements.privileged` which when `false` indicates that an image cannot be run in a privileged container. ## `cluster_internal_custom_volume_copy` This extension adds support for copying and moving custom storage volumes within a cluster with a single API call. Calling `POST /1.0/storage-pools//custom?target=` will copy the custom volume specified in the `source` part of the request. Calling `POST /1.0/storage-pools//custom/?target=` will move the custom volume from the source, specified in the `source` part of the request, to the target. ## `disk_io_bus` This introduces a new `io.bus` property to disk devices which can be used to override the bus the disk is attached to. ## `storage_cephfs_create_missing` This introduces the configuration keys `cephfs.create_missing`, `cephfs.osd_pg_num`, `cephfs.meta_pool` and `cephfs.osd_pool` to be used when adding a `cephfs` storage pool to instruct Incus to create the necessary entities for the storage pool, if they do not exist. ## `instance_move_config` This API extension provides the ability to use flags `--profile`, `--no-profile`, `--device`, and `--config` when moving an instance between projects and/or storage pools. ## `ovn_ssl_config` This introduces new server configuration keys to provide the SSL CA and client key pair to access the OVN databases. The new configuration keys are `network.ovn.ca_cert`, `network.ovn.client_cert` and `network.ovn.client_key`. ## `certificate_description` Adds a `description` field to certificate. ## `disk_io_bus_virtio_blk` Adds a new `virtio-blk` value for `io.bus` on `disk` devices which allows for the attached disk to be connected to the `virtio-blk` bus. ## `loki_config_instance` Adds a new `loki.instance` server configuration key to customize the `instance` field in Loki events. This can be used to expose the name of the cluster rather than the individual system name sending the event as that's usually already covered by the `location` field. ## `instance_create_start` Adds a new `start` field to the `POST /1.0/instances` API which when set to `true` will have the instance automatically start upon creation. In this scenario, the creation and startup is part of a single background operation. ## `clustering_evacuation_stop_options` This introduces new options for the `cluster.evacuate` option: * `stateful-stop` has the instance store its state to disk to be resume on restore. * `force-stop` has the instance immediately stopped without waiting for it to shut down. ## `boot_host_shutdown_action` This introduces a new `boot.host_shutdown_action` instance configuration key which can be used to override the default `stop` behavior on system shutdown. It supports the value `stop`, `stateful-stop` and `force-stop`. ## `agent_config_drive` This introduces a new `agent:config` disk `source` which can be used to expose an ISO to the VM guest containing the agent and its configuration. ## `network_state_ovn_lr` Adds a new `LogicalRouter` field to the `NetworkStateOVN` struct which is part of the `GET /1.0/networks/NAME/state` API. This is used to get the OVN logical router name. ## `image_template_permissions` This adds `uid`, `gid` and `mode` fields to the image metadata template entries. ## `storage_bucket_backup` Add storage bucket backup support. This includes the following new endpoints (see [RESTful API](rest-api.md) for details): * `GET /1.0/storage-pools//buckets//backups` * `POST /1.0/storage-pools//buckets//backups` * `GET /1.0/storage-pools//buckets//backups/` * `POST /1.0/storage-pools//buckets//backups/` * `DELETE /1.0/storage-pools//buckets//backups/` * `GET /1.0/storage-pools//buckets//backups//export` ## `storage_lvm_cluster` This adds a new `lvmcluster` storage driver which makes use of LVM shared VG through `lvmlockd`. With this, it's possible to have a single shared LVM pool across multiple servers so long as they all see the same backing device(s). ## `shared_custom_block_volumes` This adds a new configuration key `security.shared` to custom block volumes. If unset or `false`, the custom block volume cannot be attached to multiple instances. This feature was added to prevent data loss which can happen when custom block volumes are attached to multiple instances at once. ## `auth_tls_jwt` This adds the ability to use a signed `JSON Web Token` (`JWT`) instead of using the TLS client certificate directly. In this scenario, the client derives a `JWT` from their own TLS client certificate providing it as a `bearer` token through the `Authorization` HTTP header. The `JWT` must have the certificate's fingerprint as its `Subject` and must be signed by the client's private key. ## `oidc_claim` This introduces a new `oidc.claim` server configuration key which can be used to specify what OpenID Connect claim to use as the username. ## `device_usb_serial` This adds a new configuration key `serial` for device type `usb`. Feature has been added, to make it possible to distinguish between devices with identical `vendorid` and `productid`. ## `numa_cpu_balanced` This adds `balanced` as a new value for `limits.cpu.nodes`. When set to `balanced`, Incus will attempt to select the least busy NUMA node at startup time for the instance, trying to keep the load spread across NUMA nodes on the system. ## `image_restriction_nesting` This extension adds a new image restriction, `requirements.nesting` which when `true` indicates that an image cannot be run without nesting. ## `network_integrations` Adds the concept of network integrations and initial support for OVN Interconnection. New API: * `/1.0/network-integrations` (GET, POST) * `/1.0/network-integrations/NAME` (GET, PUT, PATCH, DELETE, POST) Each integration is made of: * name * description * type (only `ovn` for now) * configuration * `ovn.northbound_connection` (database connection string for the OVN Interconnection database) * `ovn.ca_cert` (optional, SSL CA certificate for the OVN Interconnection database) * `ovn.client_cert` (optional, SSL client certificate to connect to the OVN Interconnection database) * `ovn.client_key` (optional, SSL client key to connect to the OVN Interconnection database) * `ovn.transit.pattern` (Pongo2 template to generate the transit switch name) Those integrations attach to network peers through some new fields: * `type` (`local` for current behavior, `remote` for integrations) * `target_integration` (reference to the integration) ## `instance_memory_swap_bytes` This extends `limits.memory.swap` to allow for a total limit in bytes. ## `network_bridge_external_create` This adds the ability for `bridge.external_interfaces` to create a parent interface using a `interface/parent/vlan` syntax. ## `storage_zfs_vdev` This adds support for `mirror`, `raidz1` and `raidz2` ZFS `vdev` types by extending storage `source` configuration. ## `container_migration_stateful` A `migration.stateful` configuration key was introduced. It's a Boolean flag set to true whenever the container is in a stateful mode during the start, stop, and snapshot functions. This makes it less likely for users to run into CRIU errors when copying containers to another system. ## `profiles_all_projects` This adds support for listing profiles across all projects through the `all-projects` parameter on the `GET /1.0/profiles`API. ## `instances_scriptlet_get_instances` This allows the instance scriptlet to fetch a list of instances given an optional Project or Location filter. ## `instances_scriptlet_get_cluster_members` This allows the instance scriptlet to fetch a list of cluster members given an optional cluster group. ## `instances_scriptlet_get_project` This allows the instance scriptlet to fetch a project given name of a project. ## `network_acl_stateless` This adds support for stateless rules in network ACLs. ## `instance_state_started_at` This adds a `started_at` timestamp to the instance state API. ## `networks_all_projects` This adds support for listing networks across all projects through the `all-projects` parameter on the `GET /1.0/networks`API. ## `network_acls_all_projects` This adds support for listing network ACLs across all projects through the `all-projects` parameter on the `GET /1.0/network-acls`API. ## `storage_buckets_all_projects` This adds support for listing storage buckets across all projects through the `all-projects` parameter on the `GET /1.0/storage-pools/POOL/buckets`API. ## `resources_load` Add a new Load section to the resources API. ## `instance_access` This introduces a new API endpoint at `GET /1.0/instances/NAME/access` which exposes who can interact with the instance and what role they have. ## `project_access` This introduces a new API endpoint at `GET /1.0/projects/NAME/access` which exposes who can interact with the project and what role they have. ## `projects_force_delete` This extends `DELETE /1.0/projects` to allow `?force=true` which will delete everything inside of the project along with the project itself. ## `resources_cpu_flags` This exposes the CPU flags/extensions in our resources API to check the CPU features. ## `disk_io_bus_cache_filesystem` This adds support for both `io.bus` and `io.cache` to disks that are backed by a file system. ## `instance_oci` Adds initial support for running OCI containers. ## `clustering_groups_config` This introduces a standard key/value `config` option to clustering groups which will allow placing some restrictions or configuration on those groups. ## `instances_lxcfs_per_instance` This introduces a new `instances.lxcfs.per_instance` server configuration key to control whether to run LXCFS per instance instead of globally on the system. ## `clustering_groups_vm_cpu_definition` This introduces a few new configuration options to control the virtual machine CPU definitions through cluster group configuration. The new configuration keys are: * `instances.vm.cpu.ARCHITECTURE.baseline` * `instances.vm.cpu.ARCHITECTURE.flag` ## `disk_volume_subpath` This introduces the ability to access the sub-path of a file system custom volume by using the `source=volume/path` syntax. ## `projects_limits_disk_pool` This introduces per-pool project disk limits, introducing a `limits.disk.pool.NAME` configuration option to the project limits. ## `network_ovn_isolated` This allows using `none` as the uplink network for an OVN network, making the network isolated. ## `qemu_raw_qmp` This adds new configuration options to virtual machines to directly issue QMP commands at various stages of startup: * `raw.qemu.qmp.early` * `raw.qemu.qmp.pre-start` * `raw.qemu.qmp.post-start` ## `network_load_balancer_health_check` This adds the ability to perform health checks for load balancer backends. The following new configuration options are introduced: * `healthcheck` * `healthcheck.interval` * `healthcheck.timeout` * `healthcheck.failure_count` * `healthcheck.success_count` ## `oidc_scopes` This introduces a new `oidc.scopes` server configuration key which can take a comma separate list of OIDC scopes to request from the identity provider. ## `network_integrations_peer_name` This extends `ovn.transit.pattern` to allow `peerName` as a template variable. ## `qemu_scriptlet` This adds the ability to run a scriptlet at various stages of startup: using the `raw.qemu.scriptlet` configuration key. ## `instance_auto_restart` This introduces a new `boot.autorestart` configuration key which when set to `true` will have the instance automatically be restarted upon unexpected exit for up to 10 times over a 1 minute period. ## `storage_lvm_metadatasize` This introduces a new `lvm.metadata_size` option for LVM storage pools which allows overriding the default metadata size when creating a new LVM physical volume. ## `ovn_nic_promiscuous` This implements a new `security.promiscuous` configuration option on OVN NICs. ## `ovn_nic_ip_address_none` This adds `none` as a value for `ipv4.address` and `ipv6.address` for OVN NICs. ## `instances_state_os_info` This extension adds a pointer to an `InstanceStateOSInfo` struct to the instance's state API. ## `network_load_balancer_state` This adds a new `/1.0/networks/NAME/load-balancers/IP/state` API endpoint which returns load-balancer health check information (when configured). ## `instance_nic_macvlan_mode` This adds a `mode` configuration key on `macvlan` network interfaces which allows for configuring the Macvlan mode. ## `storage_lvm_cluster_create` Allow for creating new LVM cluster pools by setting the `source` to the shared block device. ## `network_ovn_external_interfaces` This adds support for `bridge.external_interfaces` on OVN networks. ## `instances_scriptlet_get_instances_count` This allows the instance scriptlet to fetch the count instances given an optional Project or Location filter as well as including pending instances. ## `cluster_rebalance` This adds automatic live-migration to balance load on cluster again. As part of this, the following configuration options have been added: * `cluster.rebalance.batch` * `cluster.rebalance.cooldown` * `cluster.rebalance.interval` * `cluster.rebalance.threshold` ## `custom_volume_refresh_exclude_older_snapshots` This adds support for excluding source snapshots earlier than latest target snapshot. ## `storage_initial_owner` This adds ability to set the initial owner of a custom volume. The following configuration options have been added: * `initial.gid` * `initial.mode` * `initial.uid` ## `storage_live_migration` This adds support for virtual-machines live-migration between storage pools. ## `instance_console_screenshot` This adds support to take screenshots of the current VGA console of a VM. ## `image_import_alias` Adds a new `X-Incus-aliases` HTTP header to set aliases while uploading an image. ## `authorization_scriptlet` This adds the ability to define a scriptlet in a new configuration key, `authorization.scriptlet`, managing authorization on the Incus cluster. ## `console_force` This adds support for forcing a connection to the console, even if there is already an active session. It introduces the new `--force` flag for connecting to the instance console. ## `network_ovn_state_addresses` This adds extra fields to the OVN network state struct for the IPv4 and IPv6 addresses used on the uplink. ## `qemu_scriptlet_config` This extends the QEMU scriptlet feature by allowing to modify QEMU configuration before a VM starts, and passing information about the instance to the scriptlet. ## `network_bridge_acl_devices` This adds support for device ACLs when attached to a bridged network. ## `instance_debug_memory` Add new memory dump API at `/1.0/instances/NAME/debug/memory`. ## `init_preseed_storage_volumes` This API extension provides the ability to configure storage volumes in preseed init. ## `init_preseed_profile_project` This API extension provides the ability to specify the project as part of profile definitions in preseed init. ## `instance_nic_routed_host_address` Adds support for specifying the VRF to add the routes to. ## `instance_smbios11` A new category of configuration options, `smbios11.XYZ` has been added which allows passing key/value pairs through `SMBIOS Type 11` on systems that support it. ## `api_filtering_extended` This extends the API filtering mechanism to all API collections. ## `acme_dns01` Adds support for `DNS-01` challenge to the Incus ACME support for certificate generation. ## `security_iommu` Introduce a new `security.iommu` configuration key to control whether to enable IOMMU emulation. This is done through `virtio_iommu` on Linux and the emulated Intel IOMMU on Windows. ## `network_ipv4_dhcp_routes` Introduces a new `ipv4.dhcp.routes` configuration option on bridged and OVN networks. This allows specifying pairs of CIDR networks and gateway address to be announced by the DHCP server. ## `network_state_ovn_ls` Adds a new `LogicalSwitch` field to the `NetworkStateOVN` struct which is part of the `GET /1.0/networks/NAME/state` API. This is used to get the OVN logical switch name. ## `network_dns_nameservers` Introduces the `dns.nameservers` configuration option on bridged and OVN networks. This allows specifying IPv4 and IPv6 DNS server addresses to be announced by the DHCP server and via Router Advertisements. ## `acme_http01_port` Adds `acme.http.port` to control an alternative HTTP port for `HTTP-01` validation. ## `network_ovn_ipv4_dhcp_expiry` Introduces `ipv4.dhcp.expiry` for OVN networks. ## `instance_state_cpu_time` This adds an `allocated_time` field below `CPU` in the instance state API. ## `network_io_bus` This introduces a new `io.bus` property for compatible network devices allowing to choose between `virtio` (default) and `usb`. ## `disk_io_bus_usb` Adds a new `usb` value for `io.bus` on `disk` devices. ## `storage_driver_linstor` This adds a LINSTOR storage driver. ## `instance_oci_entrypoint` This introduces a set of new configuration options on the container to configure the OCI entry point: * `oci.entrypoint` * `oci.cwd` * `oci.uid` * `oci.gid` Those are initialized at creation time using the values from the OCI image. ## `network_address_set` This adds the concept of network address sets to API under the API endpoint prefix `/1.0/network-address-sets`. ## `server_logging` This implements a new set of `logging` configuration keys on the server, allowing for multiple logging targets. The former `loki` configuration keys are being transitioned over as part of this. ## `network_forward_snat` Adds a `snat` configuration option for network forwards which will cause any DNAT to get a matching SNAT applied. So new connections from the target will appear as coming from the network forward address. This is limited to bridged networks as OVN doesn't support flexible enough SNAT for this. ## `memory_hotplug` This adds memory hotplugging for VMs, allowing them to add memory at runtime without rebooting. ## `instance_nic_routed_host_tables` This adds support for specifying host-routing tables on `nic` devices that use the routed mode. ## `instance_publish_split` This adds support for creating a split format image out of an existing instance. ## `init_preseed_certificates` This API extension provides the ability to configure certificates in preseed init. ## `custom_volume_sftp` This adds the SFTP API to custom storage volumes. ## `network_ovn_external_nic_address` This adds support for configuring a custom external IPv4 or IPv6 address for a given instance so long as that address is available through a network forward. ## `network_physical_gateway_hwaddr` Allows setting the MAC address of the IPv4 and IPv6 gateways when used with OVN. ## `backup_s3_upload` Adds support for immediately uploading instance or volume backups to an S3 compatible endpoint. ## `snapshot_manual_expiry` Introduces a `snapshots.expiry.manual` configuration key to both instances and storage volumes which allows overriding the default expiry value for snapshots created directly by the user as opposed to created on schedule. ## `resources_cpu_address_sizes` This adds tracking of CPU address sizes in the resources API. The main use of this is within clusters to calculate a cluster-wide maximum memory amount for hotplugging into virtual machines. ## `disk_attached` This introduces a new `attached` property to disk devices describing whether disks are attached or ejected. ## `limits_memory_hotplug` The `limits.memory.hotplug` option controls how memory hotplug is handled for the virtual machine. It can be set to `false` to completely disable memory hotplugging. Alternatively, it can be set to a value that defines the maximum amount of memory the VM can reach through hotplug. This value must be greater than or equal to `limits.memory`. ## `disk_wwn` Add support for setting the disk World Wide Name property through the new `wwn` disk configuration option. ## `server_logging_webhook` This adds support for basic webhook as a logging target. It can be selected through `logging.NAME.target.type` with the `webhook` value. The following target keys are supported: * `logging.NAME.target.address` (URL of the target) * `logging.NAME.target.ca_cert` (Certificate when using an HTTPS target with a self-signed certificate) * `logging.NAME.target.username` (Username for HTTP authentication) * `logging.NAME.target.password` (Password for HTTP authentication) * `logging.NAME.target.retry` (How many times to retry the transmission) The webhook data matches what's sent over the existing events API. ## `storage_driver_truenas` This adds a TrueNAS storage driver. ## `container_disk_tmpfs` This adds tmpfs support for disk devices. * `source=tmpfs:` mounts a tmpfs file system, respecting `size`, `uid`, `gid` and `mode` options * `source=tmpfs-overlay:` same as tmpfs but with additional overlayfs behavior ## `instance_limits_oom` This adds a new `limits.memory.oom_priority` configuration option to configure the Out Of Memory score for the container or virtual-machine. ## `backup_override_config` This adds support for overriding both configuration and devices during backup import by using the `X-Incus-config` and `X-Incus-devices` HTTP headers. ## `network_ovn_tunnels` This adds support for network tunnels to OVN networks. ## `init_preseed_cluster_groups` This API extension provides the ability to configure cluster groups in preseed init. ## `usb_attached` This introduces a new `attached` property to USB devices describing whether they are plugged in or not. ## `backup_iso` This allows to backup ISO custom volumes by simply copying them. It enables exporting ISO volumes with `incus storage volume export` and get an ISO back. ## `instance_systemd_credentials` This adds two categories of configuration options, `systemd.credential.*` and `systemd.credential-binary.*`, which allow passing systemd credentials through a bind-mounted directory for containers and `SMBIOS Type 11` for virtual machines. ## `cluster_group_usedby` A `used_by` field was added to the `GET /1.0/cluster/groups/{name}` endpoint. `used_by` holds the URLs of all instances and projects using the cluster group. ## `bpf_token_delegation` This adds support for [eBPF token delegation](https://docs.ebpf.io/linux/concepts/token/). ## `file_storage_volume` This adds file transfer API on the custom volumes. Implements `DELETE`, `GET`, `HEAD`, `POST` operations on the `/1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/files` endpoint. ## `network_hwaddr_pattern` This adds `network.hwaddr_pattern` global and per-project configuration keys to customize MAC address allocation. ## `storage_volume_full` This adds `?recursion=1` support to `/1.0/storage-pools/POOL/volumes/VOLUME` and `?recursion=2` support to `/1.0/storage-pools/POOL/volumes`. As well as a matching `StorageVolumeFull` struct. ## `storage_bucket_full` This adds `?recursion=1` support to `/1.0/storage-pools/POOL/buckets/VOLUME` and `?recursion=2` support to `/1.0/storage-pools/POOL/buckets`. As well as a matching `StorageBucketFull` struct. ## `device_pci_firmware` This extension introduces a new boolean `firmware` option for PCI devices. When set to `false`, Incus tells QEMU to disable the ROM-BAR (`rombar=0`) for the device. ## `resources_serial` This adds serial device tracking to the resources API. ## `ovn_nic_limits` This adds support for `limits.egress`, `limits.ingress`, `limits.max` and `limits.priority` on `ovn` type network devices. ## `storage_lvmcluster_qcow2` This adds support for running virtual machines from `QCOW2` images. Support is currently limited to `lvmcluster` storage driver. ## `oidc_allowed_subnets` This indicates support by Incus for the custom `incus.allowed_subnets` OIDC claim. ## `file_delete_force` This adds a new `X-Incus-force` HTTP header that when set to `true` allows for recursive deletion of instance or custom volume files. ## `nic_sriov_select_ext` Adds support for selecting an SR-IOV network interface by vendor ID, product ID, or PCI address. ## `network_zones_dns_contact` Adds a `dns.contact` configuration key to network zones. ## `nic_attached_connected` This introduces two new properties for NICs: * `attached`, behaving like the `attached` key for disk and USB devices; * `connected`, setting the up/down link state for the NIC (when supported). ## `nic_sriov_security_trusted` This introduces a new property for SR-IOV NICs: * `security.trusted` allows setting the `trusted` flag for the virtual function if the parent NIC supports it. ## `direct_backup` This allows to perform backups of instances, custom storage volumes and storage buckets without disk buffering when the `Accept` header is set to `application/octet-stream` in the respective POST API endpoints. Doing so, said POST endpoints return the data stream instead of creating a backup entry in the database. ## `instance_snapshot_disk_only_restore` This adds support for only restoring the disk of a snapshotted instance. ## `unix_hotplug_pci` Adds a `pci` configuration key on the `unix-hotplug` device to allow filtering based on the USB controller PCI address. ## `cluster_evacuating_restoring` Introduces new `Evacuating` and `Restoring` cluster member states. ## `projects_restricted_image_servers` Introduces a new `restricted.images.servers` project configuration option. This allows specifying a comma separate list of image server domains from which the user may download images. ## `storage_lvmcluster_size` Adds support for specifying the `size` parameter when updating `lvmcluster` storage pool. ## `authorization_scriptlet_cert` This adds two fields, `Chain` and `Certificate`, to the `details` argument of the authorization scriptlet, to give more information about the client to authorize. ## `lvmcluster_remove_snapshots` It is a Boolean that defaults to `false`. When set to `true`, it instructs Incus to remove any required snapshots when attempting to restore another snapshot. This is necessary because `lvmcluster` only allows restoring the latest snapshot. ## `daemon_storage_logs` This adds `storage.logs_volume` alongside the existing `storage.backups_volume` and `storage.images_volume`. ## `instances_debug_repair` This adds a new API at `POST /1.0/instances/NAME/debug/repair` to trigger low-level repair actions. ## `network_io_bus_ovn` This ports the `io.bus` property available on most NIC devices to non-accelerated OVN NICs. ## `dependent` This introduces a new `dependent` configuration key for disk devices and custom volumes. Volumes marked as `dependent` are tied to the `lifecycle` of the instance they are attached to. Snapshot operations on the instance are automatically propagated to the dependent volume. Creating an instance snapshot creates a corresponding snapshot on the dependent volume, and deleting the instance snapshot removes it as well. Direct snapshot creation or deletion on a dependent volume is not allowed. Exporting and importing an instance also includes dependent volumes. ## `metrics_project_resources` This adds project-level metrics to the `/1.0/metrics` endpoint including resource counts, configured limits, and current usage per project. ## `storage_volume_nbd` This adds a few new APIs for storage volumes and instances: * `POST /1.0/storage-pools/POOL/volumes/TYPE/VOLUME/nbd` * `GET /1.0/storage-pools/POOL/volumes/TYPE/VOLUME/bitmaps` * `POST /1.0/storage-pools/POOL/volumes/TYPE/VOLUME/bitmaps` * `GET /1.0/storage-pools/POOL/volumes/TYPE/VOLUME/bitmaps/NAME` * `DELETE /1.0/storage-pools/POOL/volumes/TYPE/VOLUME/bitmaps/NAME` * `POST /1.0/instances/NAME/bitmaps` This introduces the ability to get a raw `NBD` connection to an Incus block storage volume. It also allows interacting with dirty bitmaps on those volumes which can then be used as the basis for incremental backups. The instance level endpoint allows for consistent bitmap creation across the VM and its dependent volumes. ## `projects_restricted_storage_pool_access` Adds the `restricted.storage-pools.access` project configuration key to indicate (as a comma-delimited list) which storage pools can be accessed inside the project. If not specified, all storage pools are accessible. Storage pools that are not in the list are treated as equivalent to having a pool size limit of 0 (`limits.disk.pool.POOLNAME=0`), making them inaccessible and hidden from the project. ## `server_shutdown_action` This adds a new `core.shutdown_action` server configuration option. Its default value of `shutdown` keeps the current behavior but `evacuate` can be set to have a clustered server attempt an evacuation on shutdown with any instance left after it getting shutdown. ## `instances_placement_scriptlet_rebalance` Add a new placement scriptlet trigger for cluster re-balancing. incus-7.0.0/doc/api.md000066400000000000000000000004001517523235500145010ustar00rootroot00000000000000# REST API ```{toctree} :maxdepth: 1 Main API documentation rest-api-spec Main API extensions Instance API documentation Events API documentation Metrics API documentation ``` incus-7.0.0/doc/architectures.md000066400000000000000000000046231517523235500166100ustar00rootroot00000000000000(architectures)= # Architectures Incus can run on just about any architecture that is supported by the Linux kernel and by Go. Some entities in Incus are tied to an architecture, for example, the instances, instance snapshots and images. The following table lists all supported architectures including their unique identifier and the name used to refer to them. The architecture names are typically aligned with the Linux kernel architecture names. | ID | Kernel name | Description | Personalities | | :--- | :--- | :--- | :--- | | 1 | `i686` | 32bit Intel x86 | | | 2 | `x86_64` | 64bit Intel x86 | `x86` | | 3 | `armv7l` | 32bit ARMv7 little-endian | | | 4 | `aarch64` | 64bit ARMv8 little-endian | `armv7l` (optional) | | 5 | `ppc` | 32bit PowerPC big-endian | | | 6 | `ppc64` | 64bit PowerPC big-endian | `powerpc` | | 7 | `ppc64le` | 64bit PowerPC little-endian | | | 8 | `s390x` | 64bit ESA/390 big-endian | | | 9 | `mips` | 32bit MIPS | | | 10 | `mips64` | 64bit MIPS | `mips` | | 11 | `riscv32` | 32bit RISC-V little-endian | | | 12 | `riscv64` | 64bit RISC-V little-endian | | | 13 | `armv6l` | 32bit ARMv6 little-endian | | | 14 | `armv8l` | 32bit ARMv8 little-endian | | | 15 | `loongarch64` | 64bit Loongarch | | ```{note} Incus cares only about the kernel architecture, not the particular userspace flavor as determined by the toolchain. That means that Incus considers ARMv7 hard-float to be the same as ARMv7 soft-float and refers to both as `armv7l`. If useful to the user, the exact userspace ABI may be set as an image and container property, allowing easy query. ``` ## Virtual-machine support Incus only supports running virtual-machines on the following host architectures: - `x86_64` - `aarch64` - `ppc64le` - `s390x` The virtual machine guest architecture can usually be the 32bit personality of the host architecture, so long as the virtual machine firmware is capable of booting it. incus-7.0.0/doc/authentication.md000066400000000000000000000252751517523235500167700ustar00rootroot00000000000000(authentication)= # Remote API authentication Remote communications with the Incus daemon happen using JSON over HTTPS. To be able to access the remote API, clients must authenticate with the Incus server. The following authentication methods are supported: - {ref}`authentication-tls-certs` - {ref}`authentication-openid` (authentication-tls-certs)= ## TLS client certificates When using {abbr}`TLS (Transport Layer Security)` client certificates for authentication, both the client and the server will generate a key pair the first time they're launched. The server will use that key pair for all HTTPS connections to the Incus socket. The client will use its certificate as a client certificate for any client-server communication. To cause certificates to be regenerated, simply remove the old ones. On the next connection, a new certificate is generated. ### Communication protocol The supported protocol must be TLS 1.3 or better. It's possible to force Incus to accept TLS 1.2 by setting the `INCUS_INSECURE_TLS` environment variable on both client and server. However this isn't a supported setup and should only ever be used when forced to use an outdated corporate proxy. All communications must use perfect forward secrecy, and ciphers must be limited to strong elliptic curve ones (such as ECDHE-RSA or ECDHE-ECDSA). Any generated key should be at least 4096 bit RSA, preferably 384 bit ECDSA. When using signatures, only SHA-2 signatures should be trusted. Since we control both client and server, there is no reason to support any backward compatibility to broken protocol or ciphers. (authentication-trusted-clients)= ### Trusted TLS clients You can obtain the list of TLS certificates trusted by an Incus server with [`incus config trust list`](incus_config_trust_list.md). Trusted clients can be added in either of the following ways: - {ref}`authentication-add-certs` - {ref}`authentication-token` The workflow to authenticate with the server is similar to that of SSH, where an initial connection to an unknown server triggers a prompt: 1. When the user adds a server with [`incus remote add`](incus_remote_add.md), the server is contacted over HTTPS, its certificate is downloaded and the fingerprint is shown to the user. 1. The user is asked to confirm that this is indeed the server's fingerprint, which they can manually check by connecting to the server or by asking someone with access to the server to run the info command and compare the fingerprints. 1. The server attempts to authenticate the client: - If the client certificate is in the server's trust store, the connection is granted. - If the client certificate is not in the server's trust store, the server prompts the user for a token. If the provided token matches, the client certificate is added to the server's trust store and the connection is granted. Otherwise, the connection is rejected. It is possible to restrict a TLS client's access to Incus via {ref}`authorization-tls`. To revoke trust to a client, remove its certificate from the server with [`incus config trust remove `](incus_config_trust_remove.md). (authentication-tls-jwt)= #### Using `JSON Web Token` (`JWT`) to perform TLS authentication As an alternative to directly using the client's TLS certificate for authentication, Incus also supports the user derive a `bearer` token and use it through the HTTP `Authorization` header. To do this, the user must generate a signed `JWT` which has its `Subject` field set to the full fingerprint of their client certificate, it must have valid `NotBefore` and `NotAfter` fields and be signed by the client certificate's private key. (authentication-add-certs)= #### Adding trusted certificates to the server The preferred way to add trusted clients is to directly add their certificates to the trust store on the server. To do so, copy the client certificate to the server and register it using [`incus config trust add-certificate `](incus_config_trust_add-certificate.md). (authentication-token)= #### Adding client certificates using tokens You can also add new clients by using tokens. Tokens expire after a configurable time ({config:option}`server-core:core.remote_token_expiry`) or once they've been used. To use this method, generate a token for each client by calling [`incus config trust add`](incus_config_trust_add.md), which will prompt for the client name. The clients can then add their certificates to the server's trust store by providing the generated token when prompted. ```{note} If your Incus server is behind NAT, you must specify its external public address when adding it as a remote for a client: incus remote add When generating the token on the server, Incus includes a list of IP addresses that the client can use to access the server. However, if the server is behind NAT, these addresses might be local addresses that the client cannot connect to. In this case, you must specify the external address manually. ``` Alternatively, the clients can provide the token directly when adding the remote: [`incus remote add `](incus_remote_add.md). ### Using a PKI system In a {abbr}`PKI (Public key infrastructure)` setup, a system administrator manages a central PKI that issues client certificates for all the Incus clients and server certificates for all the Incus daemons. To enable PKI mode, complete the following steps: 1. Add the {abbr}`CA (Certificate authority)` certificate to all machines: - Place the `client.ca` file in the clients' configuration directories (`~/.config/incus`). - Place the `server.ca` file in the server's configuration directory (`/var/lib/incus`). 1. Place the certificates issued by the CA on the clients and the server, replacing the automatically generated ones. 1. Restart the server. In that mode, any connection to an Incus daemon will be done using the pre-seeded CA certificate. If the server certificate isn't signed by the CA, the connection will simply go through the normal authentication mechanism. If the server certificate is valid and signed by the CA, then the connection continues without prompting the user for the certificate. Note that the generated certificates are not automatically trusted. You must still add them to the server in one of the ways described in {ref}`authentication-trusted-clients`. ### Encrypting local keys The `incus` client also supports encrypted client keys. Keys generated via the methods above can be encrypted with a password, using: ``` ssh-keygen -p -o -f .config/incus/client.key ``` ```{note} Unless you enable [`keepalive` mode](remote-keepalive), then every single call to Incus will cause the prompt which may get a bit annoying: $ incus list remote-host: Password for client.key: +------+-------+------+------+------+-----------+ | NAME | STATE | IPV4 | IPV6 | TYPE | SNAPSHOTS | +------+-------+------+------+------+-----------+ ``` ```{note} While the `incus` command line supports encrypted keys, tools such as [Ansible's connection plugin](https://docs.ansible.com/ansible/latest/collections/community/general/incus_connection.html) do not. ``` (authentication-openid)= ## OpenID Connect authentication Incus supports using [OpenID Connect](https://openid.net/connect/) to authenticate users through an {abbr}`OIDC (OpenID Connect)` Identity Provider. ```{note} Authentication through OpenID Connect is supported, but there is no user role handling in place so far. Any user that authenticates through the configured OIDC Identity Provider gets full access to Incus. ``` To configure Incus to use OIDC authentication, set the [`oidc.*`](server-options-oidc) server configuration options. Your OIDC provider must be configured to enable the [Device Authorization Grant](https://oauth.net/2/device-flow/) type. To add a remote pointing to an Incus server configured with OIDC authentication, run [`incus remote add `](incus_remote_add.md). You are then prompted to authenticate through your web browser, where you must confirm the device code that Incus uses. The Incus client then retrieves and stores the access and refresh tokens and provides those to Incus for all interactions. Incus supports a custom OIDC claim of `incus.allowed_subnets` (list of strings), if the claim is set, the user will only be allowed if connecting from an IP address that's part of one of the CIDR subnets listed in the claim. ```{important} Any user that authenticates through the configured OIDC Identity Provider gets full access to Incus. To restrict user access, you must also configure {ref}`authorization`. Currently, the only authorization method that is compatible with OIDC is {ref}`authorization-openfga`. ``` (authentication-server-certificate)= ## TLS server certificate Incus supports issuing server certificates using {abbr}`ACME (Automatic Certificate Management Environment)` services, for example, [Let's Encrypt](https://letsencrypt.org/). To enable this feature, set the [relevant server configuration options](server-options-acme). Incus supports both `HTTP-01` and `DNS-01` challenges. The set of configuration option varies between the two. For `DNS-01`, the relevant {config:option}`server-acme:acme.provider` and {config:option}`server-acme:acme.provider.environment` values can be found directly in the [documentation of `lego`](https://go-acme.github.io/lego/dns/index.html), the ACME client that Incus uses behind the scenes. For `HTTP-01`, Incus will cause `lego` to temporarily listen on port `80` so the the HTTP challenge can go through. If your Incus server sits behind a reverse proxy, you'll need that reverse proxy to redirect HTTP traffic to HTTPS. ## Failure scenarios In the following scenarios, authentication is expected to fail. ### Server certificate changed The server certificate might change in the following cases: - The server was fully reinstalled and therefore got a new certificate. - The connection is being intercepted ({abbr}`MITM (Machine in the middle)`). In such cases, the client will refuse to connect to the server because the certificate fingerprint does not match the fingerprint in the configuration for this remote. It is then up to the user to contact the server administrator to check if the certificate did in fact change. If it did, the certificate can be replaced by the new one, or the remote can be removed altogether and re-added. ### Server trust relationship revoked The server trust relationship is revoked for a client if another trusted client or the local server administrator removes the trust entry for the client on the server. In this case, the server still uses the same certificate, but all API calls return a 403 code with an error indicating that the client isn't trusted. incus-7.0.0/doc/authorization.md000066400000000000000000000113311517523235500166350ustar00rootroot00000000000000(authorization)= # Authorization When interacting with Incus over the Unix socket, members of the `incus-admin` group will have full access to the Incus API. Those who are only members of the `incus` group will instead be restricted to a single project tied to their user. When interacting with Incus over the network (see {ref}`server-expose` for instructions), it is possible to further authenticate and restrict user access. There are three supported authorization methods: - {ref}`authorization-tls` - {ref}`authorization-openfga` - {ref}`authorization-scriptlet` (authorization-tls)= ## TLS authorization Incus natively supports restricting {ref}`authentication-trusted-clients` to one or more projects. When a client certificate is restricted, the client will also be prevented from performing global configuration changes or altering the configuration (limits, restrictions) of the projects it's allowed access to. To restrict access, use [`incus config trust edit `](incus_config_trust_edit.md). Set the `restricted` key to `true` and specify a list of projects to restrict the client to. If the list of projects is empty, the client will not be allowed access to any of them. This authorization method is used if a client authenticates with TLS even if {ref}`OpenFGA authorization ` is configured. (authorization-openfga)= ## Open Fine-Grained Authorization (OpenFGA) Incus supports integrating with [{abbr}`OpenFGA (Open Fine-Grained Authorization)`](https://openfga.dev). This authorization method is highly granular. For example, it can be used to restrict user access to a single instance. To use OpenFGA for authorization, you must configure and run an OpenFGA server yourself. Incus will connect to the OpenFGA server, write the {ref}`openfga-model`, and query this server for authorization for all subsequent requests. To enable this authorization method in Incus, set the [`openfga.*`](server-options-openfga) server configuration options. All options must be set in order to enable OpenFGA. Though, you do not have to create the authorization-model yourself, Incus will generate it including the initial tuple to allow only authenticated users: `server:incus#authenticated@user:*`. (openfga-model)= ### OpenFGA model With OpenFGA, access to a particular API resource is determined by the user's relationship to it. These relationships are determined by an [OpenFGA authorization model](https://openfga.dev/docs/concepts#what-is-an-authorization-model). The Incus OpenFGA authorization model describes API resources in terms of their relationship to other resources, and a relationship a user or group might have with that resource. The full Incus OpenFGA authorization model is defined in `internal/server/auth/driver_openfga_model.openfga`: ```{literalinclude} ../internal/server/auth/driver_openfga_model.openfga --- language: none --- ``` ```{important} Users that you do not trust with root access to the host should not be granted the following relations: - `server -> admin` - `server -> operator` - `server -> can_edit` - `server -> can_create_storage_pools` - `server -> can_create_projects` - `server -> can_create_certificates` - `certificate -> can_edit` - `storage_pool -> can_edit` - `project -> manager` The remaining relations may be granted. However, you must apply appropriate {ref}`project-restrictions`. ``` (authorization-scriptlet)= ## Scriptlet authorization Incus supports defining a scriptlet to manage fine-grained authorization, allowing to write precise authorization rules with no dependency on external tools. To use scriptlet authorization, you can write a scriptlet in the `authorization.scriptlet` server configuration option implementing a function `authorize`, which takes three arguments: - `details`, an object with the following attributes: - `Username`: the user name or certificate fingerprint - `Protocol`: the authentication protocol - `IsAllProjectsRequest`: whether the request is made on all projects - `ProjectName`: the project name - `Chain`: the certificate chain as a list of dissected x509 certificates - `Certificate`: the certificate data stored in the database - `object`, the object on which the user requests authorization - `entitlement`, the authorization level asked by the user This function must return a Boolean indicating whether the user has access or not to the given object with the given entitlement. Additionally, two optional functions can be defined so that users can be listed through the access API: - `get_instance_access`, with two arguments (`project_name` and `instance_name`), returning a list of users able to access a given instance - `get_project_access`, with one argument (`project_name`), returning a list of users able to access a given project incus-7.0.0/doc/backup.md000066400000000000000000000125011517523235500152020ustar00rootroot00000000000000(backups)= # How to back up an Incus server In a production setup, you should always back up the contents of your Incus server. The Incus server contains a variety of different entities, and when choosing your backup strategy, you must decide which of these entities you want to back up and how frequently you want to save them. ## What to back up The various contents of your Incus server are located on your file system and, in addition, recorded in the {ref}`Incus database `. Therefore, only backing up the database or only backing up the files on disk does not give you a full functional backup. Your Incus server contains the following entities: - Instances (database records and file systems) - Images (database records, image files, and file systems) - Networks (database records and state files) - Profiles (database records) - Storage volumes (database records and file systems) Consider which of these you need to back up. For example, if you don't use custom images, you don't need to back up your images since they are available on the image server. If you use only the `default` profile, or only the standard `incusbr0` network bridge, you might not need to worry about backing them up, because they can easily be re-created. ## Full backup To create a full backup of all contents of your Incus server, back up the `/var/lib/incus` directory. This directory contains your local storage, the Incus database, and your configuration. It does not contain separate storage devices, however. That means that whether the directory also contains the data of your instances depends on the storage drivers that you use. ```{important} If your Incus server uses any external storage (for example, LVM volume groups, ZFS zpools, or any other resource that isn't directly self-contained to Incus), you must back this up separately. ``` To back up your data, create a tarball of `/var/lib/incus`. If your system uses `/etc/subuid` and `/etc/subgid` file, you should also back up these files. Restoring them avoids needless shifting of instance file systems. To restore your data, complete the following steps: 1. Stop Incus on your server (for example, with `sudo systemctl stop incus.service incus.socket`). 1. Delete the directory (`/var/lib/incus/`). 1. Restore the directory from the backup. 1. Delete and restore any external storage devices. 1. Restore the `/etc/subuid` and `/etc/subgid` files if present. 1. Restart Incus (for example, with `sudo systemctl start incus.socket incus.service` or by restarting your machine). ## Partial backup If you decide to only back up specific entities, you have different options for how to do this. You should consider doing some of these partial backups even if you are doing full backups in addition. It can be easier and safer to, for example, restore a single instance or reconfigure a profile than to restore the full Incus server. ### Back up instances and volumes Instances and storage volumes are backed up in a very similar way (because when backing up an instance, you basically back up its instance volume, see {ref}`storage-volume-types`). See {ref}`instances-backup` and {ref}`howto-storage-backup-volume` for detailed information. The following sections give a brief summary of the options you have for backing up instances and volumes. #### Secondary backup Incus server Incus supports copying and moving instances and storage volumes between two hosts. See {ref}`move-instances` and {ref}`howto-storage-move-volume` for instructions. So if you have a spare server, you can regularly copy your instances and storage volumes to that secondary server to back them up. If needed, you can either switch over to the secondary server or copy your instances or storage volumes back from it. If you use the secondary server as a pure storage server, it doesn't need to be as powerful as your main Incus server. #### Export tarballs You can use the `export` command to export instances and volumes to a backup tarball. By default, those tarballs include all snapshots. You can use an optimized export option, which is usually quicker and results in a smaller size of the tarball. However, you must then use the same storage driver when restoring the backup tarball. See {ref}`instances-backup-export` and {ref}`storage-backup-export` for instructions. #### Snapshots Snapshots save the state of an instance or volume at a specific point in time. However, they are stored in the same storage pool and are therefore likely to be lost if the original data is deleted or lost. This means that while snapshots are very quick and easy to create and restore, they don't constitute a secure backup. See {ref}`instances-snapshots` and {ref}`storage-backup-snapshots` for more information. (backup-database)= ### Back up the database While there is no trivial method to restore the contents of the {ref}`Incus database `, it can still be very convenient to keep a backup of its content. Such a backup can make it much easier to re-create, for example, networks or profiles if the need arises. Use the following command to dump the content of the local database to a file: ```bash incus admin sql local .dump > ``` Use the following command to dump the content of the global database to a file: ```bash incus admin sql global .dump > ``` You should include these two commands in your regular Incus backup. incus-7.0.0/doc/client.md000066400000000000000000000002301517523235500152070ustar00rootroot00000000000000(incus-client)= # Incus client ```{toctree} :maxdepth: 1 Add remote servers Add command aliases /reference/manpages ``` incus-7.0.0/doc/cloud-init.md000066400000000000000000000212171517523235500160100ustar00rootroot00000000000000--- relatedlinks: https://cloudinit.readthedocs.org/ --- (cloud-init)= # How to use `cloud-init` [`cloud-init`](https://cloud-init.io/) is a tool for automatically initializing and customizing an instance of a Linux distribution. By adding `cloud-init` configuration to your instance, you can instruct `cloud-init` to execute specific actions at the first start of an instance. Possible actions include, for example: * Updating and installing packages * Applying certain configurations * Adding users * Enabling services * Running commands or scripts * Automatically growing the file system of a VM to the size of the disk See the {ref}`cloud-init:index` for detailed information. ```{note} The `cloud-init` actions are run only once on the first start of the instance. Rebooting the instance does not re-trigger the actions. ``` ## `cloud-init` support in images To use `cloud-init`, you must base your instance on an image that has `cloud-init` installed. Images from the [`images` remote](https://images.linuxcontainers.org/) have `cloud-init`-enabled variants, which are usually bigger in size than the default variant. The cloud variants use the `/cloud` suffix, for example, `images:debian/12/cloud`. ## `cloud-init` and virtual machines For `cloud-init` to work inside of a virtual machine, you need to either have a functional `incus-agent` in the VM or need to provide the `cloud-init` data through a special extra disk. All images coming from the `images:` remote will have the agent already setup and so are good to go from the start. For instances that do not have the `incus-agent`, you can pass in the extra `cloud-init` disk with: incus config device add INSTANCE cloud-init disk source=cloud-init:config ## Configuration options Incus supports two different sets of configuration options for configuring `cloud-init`: `cloud-init.*` and `user.*`. Which of these sets you must use depends on the `cloud-init` support in the image that you use. As a rule of thumb, newer images support the `cloud-init.*` configuration options, while older images support `user.*`. However, there might be exceptions to that rule. The following configuration options are supported: * `cloud-init.vendor-data` or `user.vendor-data` (see {ref}`cloud-init:vendor-data`) * `cloud-init.user-data` or `user.user-data` (see {ref}`cloud-init:user_data_formats`) * `cloud-init.network-config` or `user.network-config` (see {ref}`cloud-init:network_config`) For more information about the configuration options, see the [`cloud-init` instance options](instance-options-cloud-init), and the documentation for the {ref}`LXD data source ` in the `cloud-init` documentation. ### Vendor data and user data Both `vendor-data` and `user-data` are used to provide cloud configuration data to `cloud-init`. The main idea is that `vendor-data` is used for the general default configuration, while `user-data` is used for instance-specific configuration. This means that you should specify `vendor-data` in a profile and `user-data` in the instance configuration. Incus does not enforce this method, but allows using both `vendor-data` and `user-data` in profiles and in the instance configuration. If both `vendor-data` and `user-data` are supplied for an instance, `cloud-init` merges the two configurations. However, if you use the same keys in both configurations, merging might not be possible. In this case, configure how `cloud-init` should merge the provided data. See the {ref}`cloud-init documentation ` for instructions. ## How to configure `cloud-init` To configure `cloud-init` for an instance, add the corresponding configuration options to a {ref}`profile ` that the instance uses or directly to the {ref}`instance configuration `. When configuring `cloud-init` directly for an instance, keep in mind that `cloud-init` runs only on the first start of the instance. That means that you must configure `cloud-init` before you start the instance. To do so, create the instance with [`incus init`](incus_create.md) instead of [`incus launch`](incus_launch.md), and then start it after completing the configuration. ### YAML format for `cloud-init` configuration The `cloud-init` options require YAML's [literal style format](https://yaml.org/spec/1.2.2/#812-literal-style). You use a pipe symbol (`|`) to indicate that all indented text after the pipe should be passed to `cloud-init` as a single string, with new lines and indentation preserved. The `vendor-data` and `user-data` options usually start with `#cloud-config`. For example: ```yaml config: cloud-init.user-data: | #cloud-config package_upgrade: true packages: - package1 - package2 ``` ```{tip} See {ref}`How to validate user data ` for information on how to check whether the syntax is correct. ``` ## How to check the `cloud-init` status `cloud-init` runs automatically on the first start of an instance. Depending on the configured actions, it might take a while until it finishes. To check the `cloud-init` status, log on to the instance and enter the following command: cloud-init status If the result is `status: running`, `cloud-init` is still working. If the result is `status: done`, it has finished. Alternatively, use the `--wait` flag to be notified only when `cloud-init` is finished: ```{terminal} :input: cloud-init status --wait :user: root :host: instance ..................................... status: done ``` ## How to specify user or vendor data The `user-data` and `vendor-data` configuration can be used to, for example, upgrade or install packages, add users, or run commands. The provided values must have a first line that indicates what type of {ref}`user data format ` is being passed to `cloud-init`. For activities like upgrading packages or setting up a user, `#cloud-config` is the data format to use. The configuration data is stored in the following files in the instance's root file system: * `/var/lib/cloud/instance/cloud-config.txt` * `/var/lib/cloud/instance/user-data.txt` ### Examples See the following sections for the user data (or vendor data) configuration for different example use cases. You can find more advanced {ref}`examples ` in the `cloud-init` documentation. #### Upgrade packages To trigger a package upgrade from the repositories for the instance right after the instance is created, use the `package_upgrade` key: ```yaml config: cloud-init.user-data: | #cloud-config package_upgrade: true ``` #### Install packages To install specific packages when the instance is set up, use the `packages` key and specify the package names as a list: ```yaml config: cloud-init.user-data: | #cloud-config packages: - git - openssh-server ``` #### Set the time zone To set the time zone for the instance on instance creation, use the `timezone` key: ```yaml config: cloud-init.user-data: | #cloud-config timezone: Europe/Rome ``` #### Run commands To run a command (such as writing a marker file), use the `runcmd` key and specify the commands as a list: ```yaml config: cloud-init.user-data: | #cloud-config runcmd: - [touch, /run/cloud.init.ran] ``` #### Add a user account To add a user account, use the `users` key. See the {ref}`cloud-init:reference/examples:including users and groups` example in the `cloud-init` documentation for details about default users and which keys are supported. ```yaml config: cloud-init.user-data: | #cloud-config users: - name: documentation_example ``` ## How to specify network configuration data By default, `cloud-init` configures a DHCP client on an instance's `eth0` interface. You can define your own network configuration using the `network-config` option to override the default configuration (this is due to how the template is structured). `cloud-init` then renders the relevant network configuration on the system using either `ifupdown` or `netplan`, depending on the distribution. The configuration data is stored in the following files in the instance's root file system: * `/var/lib/cloud/seed/nocloud-net/network-config` * `/etc/network/interfaces.d/50-cloud-init.cfg` (if using `ifupdown`) * `/etc/netplan/50-cloud-init.yaml` (if using `netplan`) ### Example To configure a specific network interface with a static IPv4 address and also use a custom name server, use the following configuration: ```yaml config: cloud-init.network-config: | version: 2 ethernets: eth1: addresses: - 10.10.101.20/24 gateway4: 10.10.101.1 nameservers: addresses: - 10.10.10.254 ``` incus-7.0.0/doc/clustering.md000066400000000000000000000007251517523235500161210ustar00rootroot00000000000000(clustering)= # Clustering ```{toctree} :titlesonly: explanation/clustering.md Form a cluster Access a cluster Manage a cluster Recover a cluster Manage cluster groups Manage instances Configure storage Configure networks reference/cluster_member_config ``` incus-7.0.0/doc/conf.py000066400000000000000000000170151517523235500147170ustar00rootroot00000000000000import contextlib import datetime import os import stat import subprocess import tempfile import yaml from git import Repo import filecmp # Download and link swagger-ui files if not os.path.isdir('.sphinx/deps/swagger-ui'): Repo.clone_from('https://github.com/swagger-api/swagger-ui', '.sphinx/deps/swagger-ui', depth=1) os.makedirs('.sphinx/_static/swagger-ui/', exist_ok=True) if not os.path.islink('.sphinx/_static/swagger-ui/swagger-ui-bundle.js'): os.symlink('../../deps/swagger-ui/dist/swagger-ui-bundle.js', '.sphinx/_static/swagger-ui/swagger-ui-bundle.js') if not os.path.islink('.sphinx/_static/swagger-ui/swagger-ui-standalone-preset.js'): os.symlink('../../deps/swagger-ui/dist/swagger-ui-standalone-preset.js', '.sphinx/_static/swagger-ui/swagger-ui-standalone-preset.js') if not os.path.islink('.sphinx/_static/swagger-ui/swagger-ui.css'): os.symlink('../../deps/swagger-ui/dist/swagger-ui.css', '.sphinx/_static/swagger-ui/swagger-ui.css') ### MAN PAGES ### # Find the path to the incus binary path = str(subprocess.check_output(['go', 'env', 'GOPATH'], encoding="utf-8").strip()) incus = os.path.join(path, 'bin', 'incus') if os.path.isfile(incus): print("Using " + incus + " to generate man pages.") else: print("Cannot find incus in " + incus) exit(2) # Generate man pages content os.makedirs('.sphinx/deps/manpages', exist_ok=True) subprocess.run([incus, 'manpage', '.sphinx/deps/manpages/', '--format=md', '--all'], check=True) # Preprocess man pages content for page in [x for x in os.listdir('.sphinx/deps/manpages') if os.path.isfile(os.path.join('.sphinx/deps/manpages/', x))]: # replace underscores with slashes to create a directory structure pagepath = page.replace('_', '/') # for each generated page, add an anchor, fix the title, and adjust the # heading levels with open(os.path.join('.sphinx/deps/manpages/', page), 'r') as mdfile: content = mdfile.readlines() os.makedirs(os.path.dirname(os.path.join('.sphinx/deps/manpages/', pagepath)), exist_ok=True) with open(os.path.join('.sphinx/deps/manpages/', pagepath), 'w') as mdfile: mdfile.write('(' + page + ')=\n') line_block = False for line in content: if line.startswith('###### Auto generated'): continue elif line.startswith('### Synopsis'): mdfile.write('## Synopsis\n') mdfile.write('```{line-block}\n') line_block = True elif line.startswith('## '): mdfile.write('# `' + line[3:].rstrip() + '`\n') elif line.startswith('```'): if line_block: mdfile.write('```\n') line_block = False mdfile.write(line) elif line.startswith('##'): if line_block: mdfile.write('```\n') line_block = False mdfile.write(line[1:]) else: mdfile.write(line) # remove the input page (unless the file path doesn't change) if '_' in page: os.remove(os.path.join('.sphinx/deps/manpages/', page)) # Complete and copy man pages content for folder, subfolders, files in os.walk('.sphinx/deps/manpages'): # for each subfolder, add toctrees to the parent page that # include the subpages for subfolder in subfolders: with open(os.path.join(folder, subfolder + '.md'), 'a') as parent: parent.write('```{toctree}\n:titlesonly:\n:glob:\n:hidden:\n\n' + subfolder + '/*\n```\n') # for each file, if the content is different to what has been generated # before, copy the file to the reference/manpages folder # (copying all would mess up the incremental build) for f in files: sourcefile = os.path.join(folder, f) targetfile = os.path.join('reference/manpages/', os.path.relpath(folder, '.sphinx/deps/manpages'), f) if (not os.path.isfile(targetfile) or not filecmp.cmp(sourcefile, targetfile, shallow=False)): os.makedirs(os.path.dirname(targetfile), exist_ok=True) os.system('cp ' + sourcefile + ' ' + targetfile) ### End MAN PAGES ### # Project config. project = "Incus" author = "Incus contributors" copyright = "2014-%s %s" % (datetime.date.today().year, author) with open("../internal/version/flex.go") as fd: version = fd.read().split("\n")[-2].split()[-1].strip("\"") # Extensions. extensions = [ "config-options", "custom-rst-roles", "myst_parser", "notfound.extension", "related-links", "sphinxcontrib.jquery", "sphinx_copybutton", "sphinx_design", "sphinx.ext.intersphinx", "sphinxext.opengraph", "sphinx_remove_toctrees", "sphinx_reredirects", "sphinx_tabs.tabs", "terminal-output", "youtube-links" ] myst_enable_extensions = [ "deflist", "linkify", "substitution" ] myst_linkify_fuzzy_links = False myst_heading_anchors = 7 if os.path.exists("./substitutions.yaml"): with open("./substitutions.yaml", "r") as fd: myst_substitutions = yaml.safe_load(fd.read()) if os.path.exists("./related_topics.yaml"): with open("./related_topics.yaml", "r") as fd: myst_substitutions.update(yaml.safe_load(fd.read())) intersphinx_mapping = { 'cloud-init': ('https://cloudinit.readthedocs.io/en/latest/', None) } if ("LOCAL_SPHINX_BUILD" in os.environ) and (os.environ["LOCAL_SPHINX_BUILD"] == "True"): swagger_url_scheme = "/api/#{{path}}" else: swagger_url_scheme = "/incus/docs/main/api/#{{path}}" myst_url_schemes = { "http": None, "https": None, "swagger": swagger_url_scheme, } remove_from_toctrees = ["reference/manpages/incus/*.md"] # Setup theme. html_theme = "furo" html_show_sphinx = False html_last_updated_fmt = "" html_favicon = ".sphinx/_static/favicon.ico" html_static_path = ['.sphinx/_static'] html_css_files = ['custom.css', 'furo_colors.css'] html_extra_path = ['.sphinx/_extra'] html_theme_options = { "sidebar_hide_name": True, } html_context = { "github_url": "https://github.com/lxc/incus", "github_version": "main", "github_folder": "/doc/", "github_filetype": "md", "discourse_prefix": { "lxc": "https://discuss.linuxcontainers.org/t/"} } source_suffix = ".md" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ['html', 'README.md', '.sphinx', 'config_options_cheat_sheet.md'] # Open Graph configuration ogp_site_url = "https://linuxcontainers.org/incus/docs/main/" ogp_site_name = "Incus documentation" ogp_image = "https://linuxcontainers.org/static/img/containers.png" # Links to ignore when checking links linkcheck_ignore = [ 'https://127.0.0.1:8443/1.0', 'https://web.libera.chat/#lxc', 'http://localhost:8001', 'https://www.schlachter.tech/solutions/pongo2-template-engine/', r'https://gitlab.alpinelinux.org/.*', r'https://developer.hashicorp.com/.*', r'https://docutils.sourceforge.io/.*', r'/incus/docs/main/api/.*', r'/api/.*' ] linkcheck_exclude_documents = [r'.*/manpages/.*'] linkcheck_anchors_ignore_for_url = [ r'https://github\.com/.*' ] # Setup redirects (https://documatt.gitlab.io/sphinx-reredirects/usage.html) redirects = { "howto/instances_snapshots/index": "../instances_backup/", } incus-7.0.0/doc/config_options.txt000066400000000000000000006036071517523235500172110ustar00rootroot00000000000000// Code generated by generate-config from the incus project; DO NOT EDIT. ```{config:option} scheduler.instance cluster-cluster :defaultdesc: "`all`" :shortdesc: "Controls how instances are scheduled to run on this member" :type: "string" Possible values are `all`, `manual`, and `group`. See {ref}`clustering-instance-placement` for more information. ``` ```{config:option} user.* cluster-cluster :shortdesc: "Free form user key/value storage" :type: "string" User keys can be used in search. ``` ```{config:option} instances.vm.cpu.ARCHITECTURE.baseline cluster_group-common :shortdesc: "CPU base architecture name" :type: "string" The CPU base architecture name as can be found through `qemu -cpu ?`. This can be a generic definition like `qemu64` or `kvm64`, or it can be a specific hardware architecture like `EPYC-v2`. It's important to ensure that all servers in the group match that baseline. ``` ```{config:option} instances.vm.cpu.ARCHITECTURE.flags cluster_group-common :shortdesc: "CPU flags to add/remove to/from the baseline" :type: "string" A comma separated list of CPU flags to add on top of CPU baseline or a list of flags to remove from it. To remove a flag, use `-flag`. ``` ```{config:option} user.* cluster_group-common :shortdesc: "Free form user key/value storage" :type: "string" User keys can be used in search. ``` ```{config:option} attached devices-disk :default: "`true`" :required: "no" :shortdesc: "Only for VMs: Whether the disk is attached or ejected" :type: "bool" ``` ```{config:option} boot.priority devices-disk :required: "no" :shortdesc: "Boot priority for VMs (higher value boots first)" :type: "integer" ``` ```{config:option} ceph.cluster_name devices-disk :default: "`ceph`" :required: "no" :shortdesc: "The cluster name of the Ceph cluster (required for Ceph or CephFS sources)" :type: "string" ``` ```{config:option} ceph.user_name devices-disk :default: "`admin`" :required: "no" :shortdesc: "The user name of the Ceph cluster (required for Ceph or CephFS sources)" :type: "string" ``` ```{config:option} dependent devices-disk :default: "`false`" :required: "no" :shortdesc: "Specifies if the disk is instance dependent" :type: "bool" ``` ```{config:option} initial.* devices-disk :required: "no" :shortdesc: "Initial volume configuration for instance root disk devices" :type: "string" For root disk devices, this is used to override the storage pool's default volume configuration when creating the instance's root volume. For custom volumes, only `initial.uid`, `initial.gid` and `initial.mode` are accepted and they are used when auto-creating sub-directories inside the custom volume (when the `source` includes a sub-path that doesn't exist). `initial.uid`, `initial.gid` and `initial.mode` are also used to set the ownership and mode of the file system when the `source` is `tmpfs:` or `tmpfs-overlay:`. ``` ```{config:option} io.bus devices-disk :default: "`virtio-scsi` for block, `auto` for file system" :required: "no" :shortdesc: "Only for VMs: Override the bus for the device" :type: "string" This controls what bus a disk device should be attached to. For block devices (disks), this is one of: - `nvme` - `virtio-blk` - `virtio-scsi` (default) - `usb` For file systems (shared directories or custom volumes), this is one of: - `9p` - `auto` (default) (`virtiofs` if possible, else `9p`) - `virtiofs` `9p` doesn't support hotplugging and `virtiofs` doesn't support live migration. `auto` tries to use `virtiofs` if possible (`migration.stateful` not set to `true` and host support for `virtiofsd`) and falls back to `9p` otherwise. ``` ```{config:option} io.cache devices-disk :default: "`none`" :required: "no" :shortdesc: "Only for VMs: Override the caching mode for the device" :type: "string" This controls what bus a disk device should be attached to. For block devices (disks), this is one of: - `none` (default) - `writeback` - `unsafe` For file systems (shared directories or custom volumes), this is one of: - `none` (default) - `metadata` - `unsafe` ``` ```{config:option} limits.max devices-disk :required: "no" :shortdesc: "I/O limit in byte/s or IOPS for both read and write (same as setting both `limits.read` and `limits.write`)" :type: "string" ``` ```{config:option} limits.read devices-disk :required: "no" :shortdesc: "I/O limit in byte/s (various suffixes supported, see {ref}`instances-limit-units`) or in IOPS (must be suffixed with `iops`) - see also {ref}`storage-configure-IO`" :type: "string" ``` ```{config:option} limits.write devices-disk :required: "no" :shortdesc: "I/O limit in byte/s (various suffixes supported, see {ref}`instances-limit-units`) or in IOPS (must be suffixed with `iops`) - see also {ref}`storage-configure-IO`" :type: "string" ``` ```{config:option} path devices-disk :required: "yes" :shortdesc: "Path inside the instance where the disk will be mounted (only for file system disk devices)" :type: "string" This controls which path inside the instance the disk should be mounted on. With containers, this option supports mounting file system disk devices, and paths and single files within them. With VMs, this option supports mounting file system disk devices and paths within them. Mounting single files is not supported. ``` ```{config:option} pool devices-disk :required: "no" :shortdesc: "The storage pool to which the disk device belongs (only applicable for storage volumes managed by Incus)" :type: "string" ``` ```{config:option} propagation devices-disk :required: "no" :shortdesc: "Controls how a bind-mount is shared between the instance and the host (can be one of `private`, the default, or `shared`, `slave`, `unbindable`, `rshared`, `rslave`, `runbindable`, `rprivate`; see the Linux Kernel [shared subtree](https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt) documentation for a full explanation)" :type: "string" ``` ```{config:option} raw.mount.options devices-disk :required: "no" :shortdesc: "File system specific mount options" :type: "string" ``` ```{config:option} readonly devices-disk :default: "`false`" :required: "no" :shortdesc: "Controls whether to make the mount read-only" :type: "bool" ``` ```{config:option} recursive devices-disk :default: "`false`" :required: "no" :shortdesc: "Controls whether to recursively mount the source path" :type: "bool" ``` ```{config:option} required devices-disk :default: "`true`" :required: "no" :shortdesc: "Controls whether to fail if the source doesn't exist" :type: "bool" ``` ```{config:option} shift devices-disk :default: "`false`" :required: "no" :shortdesc: "Sets up a shifting overlay to translate the source UID/GID to match the instance (only for containers)" :type: "bool" ``` ```{config:option} size devices-disk :required: "no" :shortdesc: "Disk size in bytes (various suffixes supported, see {ref}`instances-limit-units`) - only supported for the `rootfs` (`/`)" :type: "string" ``` ```{config:option} size.state devices-disk :required: "no" :shortdesc: "Same as `size`, but applies to the file-system volume used for saving runtime state in VMs" :type: "string" ``` ```{config:option} source devices-disk :required: "yes" :shortdesc: "Source of a file system or block device (see {ref}`devices-disk-types` for details)" :type: "string" ``` ```{config:option} wwn devices-disk :default: "``" :required: "no" :shortdesc: "Only for VMs: Set the disk World Wide Name (only supported on `virtio-scsi` bus)" :type: "bool" ``` ```{config:option} id devices-gpu_mdev :required: "no" :shortdesc: "The DRM card ID of the GPU device" :type: "string" ``` ```{config:option} mdev devices-gpu_mdev :required: "yes" :shortdesc: "The mediated device profile to use (required - for example, `i915-GVTg_V5_4`)" :type: "string" ``` ```{config:option} productid devices-gpu_mdev :required: "no" :shortdesc: "The product ID of the GPU device" :type: "string" ``` ```{config:option} vendorid devices-gpu_mdev :required: "no" :shortdesc: "The vendor ID of the GPU device" :type: "string" ``` ```{config:option} id devices-gpu_mig :required: "no" :shortdesc: "The DRM card ID of the GPU device" :type: "string" ``` ```{config:option} mig.ci devices-gpu_mig :required: "no" :shortdesc: "Existing MIG compute instance ID" :type: "int" ``` ```{config:option} mig.gi devices-gpu_mig :required: "no" :shortdesc: "Existing MIG GPU instance ID" :type: "int" ``` ```{config:option} mig.uuid devices-gpu_mig :required: "no" :shortdesc: "Existing MIG device UUID (MIG- prefix can be omitted)" :type: "string" ``` ```{config:option} pci devices-gpu_mig :required: "no" :shortdesc: "The PCI address of the GPU device" :type: "string" ``` ```{config:option} productid devices-gpu_mig :required: "no" :shortdesc: "The product ID of the GPU device" :type: "string" ``` ```{config:option} vendorid devices-gpu_mig :required: "no" :shortdesc: "The vendor ID of the GPU device" :type: "string" ``` ```{config:option} gid devices-gpu_physical :default: "0" :required: "no" :shortdesc: "GID of the device owner in the instance (container only)" :type: "int" ``` ```{config:option} id devices-gpu_physical :required: "no" :shortdesc: "The DRM card ID of the GPU device" :type: "string" ``` ```{config:option} mode devices-gpu_physical :default: "0660" :required: "no" :shortdesc: "Mode of the device in the instance (container only)" :type: "int" ``` ```{config:option} pci devices-gpu_physical :required: "no" :shortdesc: "The PCI address of the GPU device" :type: "string" ``` ```{config:option} productid devices-gpu_physical :required: "no" :shortdesc: "The product ID of the GPU device" :type: "string" ``` ```{config:option} uid devices-gpu_physical :default: "0" :required: "no" :shortdesc: "UID of the device owner in the instance (container only)" :type: "int" ``` ```{config:option} vendorid devices-gpu_physical :required: "no" :shortdesc: "The vendor ID of the GPU device" :type: "string" ``` ```{config:option} id devices-gpu_sriov :required: "no" :shortdesc: "The DRM card ID of the parent GPU device" :type: "string" ``` ```{config:option} pci devices-gpu_sriov :required: "no" :shortdesc: "The PCI address of the parent GPU device" :type: "string" ``` ```{config:option} productid devices-gpu_sriov :required: "no" :shortdesc: "The product ID of the parent GPU device" :type: "string" ``` ```{config:option} vendorid devices-gpu_sriov :required: "no" :shortdesc: "The vendor ID of the parent GPU device" :type: "string" ``` ```{config:option} hwaddr devices-infiniband :defaultdesc: "randomly assigned" :required: "no" :shortdesc: "The MAC address of the new interface (can be either the full 20-byte variant or the short 8-byte variant, which will only modify the last 8 bytes of the parent device)" :type: "string" ``` ```{config:option} mtu devices-infiniband :defaultdesc: "parent MTU" :required: "no" :shortdesc: "The MTU of the new interface" :type: "integer" ``` ```{config:option} name devices-infiniband :defaultdesc: "kernel assigned" :required: "no" :shortdesc: "The name of the interface inside the instance" :type: "string" ``` ```{config:option} nictype devices-infiniband :required: "yes" :shortdesc: "The device type (one of `physical` or `sriov`)" :type: "string" ``` ```{config:option} parent devices-infiniband :defaultdesc: "kernel assigned" :required: "no" :shortdesc: "The name of the interface inside the instance" :type: "string" ``` ```{config:option} attached devices-nic_bridged :default: "`true`" :required: "no" :shortdesc: "Whether the NIC is plugged in or not" :type: "bool" ``` ```{config:option} boot.priority devices-nic_bridged :managed: "no" :shortdesc: "Boot priority for VMs (higher value boots first)" :type: "integer" ``` ```{config:option} connected devices-nic_bridged :default: "`true`" :required: "no" :shortdesc: "Whether the NIC is connected to the host network" :type: "bool" ``` ```{config:option} host_name devices-nic_bridged :default: "randomly assigned" :managed: "no" :shortdesc: "The name of the interface on the host" :type: "string" ``` ```{config:option} hwaddr devices-nic_bridged :default: "randomly assigned" :managed: "no" :shortdesc: "The MAC address of the new interface" :type: "string" ``` ```{config:option} io.bus devices-nic_bridged :default: "`virtio`" :managed: "no" :shortdesc: "Override the bus for the device (can be `virtio` or `usb`) (VM only)" :type: "string" ``` ```{config:option} ipv4.address devices-nic_bridged :managed: "no" :shortdesc: "An IPv4 address to assign to the instance through DHCP (can be `none` to restrict all IPv4 traffic when `security.ipv4_filtering` is set)" :type: "string" ``` ```{config:option} ipv4.routes devices-nic_bridged :managed: "no" :shortdesc: "Comma-delimited list of IPv4 static routes to add on host to NIC" :type: "string" ``` ```{config:option} ipv4.routes.external devices-nic_bridged :managed: "no" :shortdesc: "Comma-delimited list of IPv4 static routes to route to the NIC and publish on uplink network (BGP)" :type: "string" ``` ```{config:option} ipv6.address devices-nic_bridged :managed: "no" :shortdesc: "An IPv6 address to assign to the instance through DHCP (can be `none` to restrict all IPv6 traffic when `security.ipv6_filtering` is set)" :type: "string" ``` ```{config:option} ipv6.routes devices-nic_bridged :managed: "no" :shortdesc: "Comma-delimited list of IPv6 static routes to add on host to NIC" :type: "string" ``` ```{config:option} ipv6.routes.external devices-nic_bridged :managed: "no" :shortdesc: "Comma-delimited list of IPv6 static routes to route to the NIC and publish on uplink network (BGP)" :type: "string" ``` ```{config:option} limits.egress devices-nic_bridged :managed: "no" :shortdesc: "I/O limit in bit/s for outgoing traffic (various suffixes supported, see {ref}`instances-limit-units`)" :type: "string" ``` ```{config:option} limits.ingress devices-nic_bridged :managed: "no" :shortdesc: "I/O limit in bit/s for incoming traffic (various suffixes supported, see {ref}`instances-limit-units`)" :type: "string" ``` ```{config:option} limits.max devices-nic_bridged :managed: "no" :shortdesc: "I/O limit in bit/s for both incoming and outgoing traffic (same as setting both limits.ingress and limits.egress)" :type: "string" ``` ```{config:option} limits.priority devices-nic_bridged :managed: "no" :shortdesc: "The priority for outgoing traffic, to be used by the kernel queuing discipline to prioritize network packets" :type: "integer" ``` ```{config:option} mtu devices-nic_bridged :default: "MTU of the parent device" :managed: "yes" :shortdesc: "The Maximum Transmit Unit (MTU) of the new interface" :type: "integer" ``` ```{config:option} name devices-nic_bridged :default: "kernel assigned" :managed: "no" :shortdesc: "The name of the interface inside the instance" :type: "string" ``` ```{config:option} network devices-nic_bridged :managed: "no" :shortdesc: "The managed network to link the device to (instead of specifying the `nictype` directly)" :type: "string" ``` ```{config:option} parent devices-nic_bridged :managed: "yes" :shortdesc: "The name of the parent host device (required if specifying the `nictype` directly)" :type: "string" ``` ```{config:option} queue.tx.length devices-nic_bridged :managed: "no" :shortdesc: "The transmit queue length for the NIC" :type: "integer" ``` ```{config:option} security.acls devices-nic_bridged :managed: "no" :shortdesc: "Comma-separated list of network ACLs to apply" :type: "string" ``` ```{config:option} security.acls.default.egress.action devices-nic_bridged :default: "drop" :managed: "no" :shortdesc: "Action to use for egress traffic that doesn't match any ACL rule" :type: "string" ``` ```{config:option} security.acls.default.egress.logged devices-nic_bridged :default: "false" :managed: "no" :shortdesc: "Whether to log egress traffic that doesn't match any ACL rule" :type: "bool" ``` ```{config:option} security.acls.default.ingress.action devices-nic_bridged :default: "drop" :managed: "no" :shortdesc: "Action to use for ingress traffic that doesn't match any ACL rule" :type: "string" ``` ```{config:option} security.acls.default.ingress.logged devices-nic_bridged :default: "false" :managed: "no" :shortdesc: "Whether to log ingress traffic that doesn't match any ACL rule" :type: "bool" ``` ```{config:option} security.ipv4_filtering devices-nic_bridged :default: "false" :managed: "no" :shortdesc: "Prevent the instance from spoofing another instance's IPv4 address (enables `security.mac_filtering`)" :type: "bool" ``` ```{config:option} security.ipv6_filtering devices-nic_bridged :default: "false" :managed: "no" :shortdesc: "Prevent the instance from spoofing another instance's IPv6 address (enables `security.mac_filtering`)" :type: "bool" ``` ```{config:option} security.mac_filtering devices-nic_bridged :default: "false" :managed: "no" :shortdesc: "Prevent the instance from spoofing another instance's MAC address" :type: "bool" ``` ```{config:option} security.port_isolation devices-nic_bridged :default: "false" :managed: "no" :shortdesc: "Prevent the NIC from communicating with other NICs in the network that have port isolation enabled" :type: "bool" ``` ```{config:option} vlan devices-nic_bridged :managed: "no" :shortdesc: "The VLAN ID to use for non-tagged traffic (can be none to remove port from default VLAN)" :type: "integer" ``` ```{config:option} vlan.tagged devices-nic_bridged :managed: "no" :shortdesc: "Comma-delimited list of VLAN IDs or VLAN ranges to join for tagged traffic" :type: "integer" ``` ```{config:option} attached devices-nic_ipvlan :default: "`true`" :required: "no" :shortdesc: "Whether the NIC is plugged in or not" :type: "bool" ``` ```{config:option} gvrp devices-nic_ipvlan :default: "false" :shortdesc: "Register VLAN using GARP VLAN Registration Protocol" :type: "bool" ``` ```{config:option} hwaddr devices-nic_ipvlan :default: "randomly assigned" :shortdesc: "The MAC address of the new interface" :type: "string" ``` ```{config:option} ipv4.address devices-nic_ipvlan :shortdesc: "Comma-delimited list of IPv4 static addresses to add to the instance (in l2 mode, these can be specified as CIDR values or singular addresses using a subnet of /24)" :type: "string" ``` ```{config:option} ipv4.gateway devices-nic_ipvlan :default: "`auto` (in `l3s` mode), `-` (in `l2` mode)" :shortdesc: "In `l3s` mode, whether to add an automatic default IPv4 gateway (can be `auto` or `none`). In `l2` mode, the IPv4 address of the gateway" :type: "string" ``` ```{config:option} ipv4.host_table devices-nic_ipvlan :shortdesc: "The custom policy routing table ID to add IPv4 static routes to (in addition to the main routing table)" :type: "integer" ``` ```{config:option} ipv6.address devices-nic_ipvlan :shortdesc: "Comma-delimited list of IPv6 static addresses to add to the instance (in `l2` mode, these can be specified as CIDR values or singular addresses using a subnet of /64)" :type: "string" ``` ```{config:option} ipv6.gateway devices-nic_ipvlan :default: "`auto` (in `l3s` mode), `-` (in `l2` mode)" :shortdesc: "In `l3s` mode, whether to add an automatic default IPv6 gateway (can be `auto` or `none`). In `l2` mode, the IPv6 address of the gateway" :type: "string" ``` ```{config:option} ipv6.host_table devices-nic_ipvlan :shortdesc: "The custom policy routing table ID to add IPv6 static routes to (in addition to the main routing table)" :type: "integer" ``` ```{config:option} mode devices-nic_ipvlan :default: "`l3s`" :shortdesc: "The IPVLAN mode (either `l2` or `l3s`)" :type: "string" ``` ```{config:option} mtu devices-nic_ipvlan :default: "MTU of the parent device" :shortdesc: "The Maximum Transmit Unit (MTU) of the new interface" :type: "integer" ``` ```{config:option} name devices-nic_ipvlan :default: "kernel assigned" :shortdesc: "The name of the interface inside the instance" :type: "string" ``` ```{config:option} parent devices-nic_ipvlan :shortdesc: "The name of the host device (required)" :type: "string" ``` ```{config:option} vlan devices-nic_ipvlan :shortdesc: "The VLAN ID to attach to" :type: "integer" ``` ```{config:option} attached devices-nic_macvlan :default: "`true`" :required: "no" :shortdesc: "Whether the NIC is plugged in or not" :type: "bool" ``` ```{config:option} boot.priority devices-nic_macvlan :managed: "no" :shortdesc: "Boot priority for VMs (higher value boots first)" :type: "integer" ``` ```{config:option} connected devices-nic_macvlan :default: "`true`" :required: "no" :shortdesc: "Whether the NIC is connected to the host network (VM only)" :type: "bool" ``` ```{config:option} gvrp devices-nic_macvlan :default: "false" :managed: "no" :shortdesc: "Register VLAN using GARP VLAN Registration Protocol" :type: "bool" ``` ```{config:option} hwaddr devices-nic_macvlan :default: "randomly assigned" :managed: "no" :shortdesc: "The MAC address of the new interface" :type: "string" ``` ```{config:option} io.bus devices-nic_macvlan :default: "`virtio`" :managed: "no" :shortdesc: "Override the bus for the device (can be `virtio` or `usb`) (VM only)" :type: "string" ``` ```{config:option} mode devices-nic_macvlan :default: "bridge" :managed: "no" :shortdesc: "Macvlan mode (one of `bridge`, `vepa`, `passthru` or `private`)" :type: "string" ``` ```{config:option} mtu devices-nic_macvlan :default: "MTU of the parent device" :managed: "yes" :shortdesc: "The Maximum Transmit Unit (MTU) of the new interface" :type: "integer" ``` ```{config:option} name devices-nic_macvlan :default: "kernel assigned" :managed: "no" :shortdesc: "The name of the interface inside the instance" :type: "string" ``` ```{config:option} network devices-nic_macvlan :managed: "no" :shortdesc: "The managed network to link the device to (instead of specifying the `nictype` directly)" :type: "string" ``` ```{config:option} parent devices-nic_macvlan :managed: "yes" :shortdesc: "The name of the parent host device (required if specifying the `nictype` directly)" :type: "string" ``` ```{config:option} vlan devices-nic_macvlan :managed: "no" :shortdesc: "The VLAN ID to attach to" :type: "integer" ``` ```{config:option} acceleration devices-nic_ovn :default: "none" :managed: "no" :shortdesc: "Enable hardware offloading (either `none`, `sriov` or `vdpa`)" :type: "string" ``` ```{config:option} attached devices-nic_ovn :default: "`true`" :required: "no" :shortdesc: "Whether the NIC is plugged in or not" :type: "bool" ``` ```{config:option} boot.priority devices-nic_ovn :managed: "no" :shortdesc: "Boot priority for VMs (higher value boots first)" :type: "integer" ``` ```{config:option} connected devices-nic_ovn :default: "`true`" :required: "no" :shortdesc: "Whether the NIC is connected to the host network (requires `acceleration` set to `none`)" :type: "bool" ``` ```{config:option} host_name devices-nic_ovn :default: "randomly assigned" :managed: "no" :shortdesc: "The name of the interface inside the host" :type: "string" ``` ```{config:option} hwaddr devices-nic_ovn :default: "randomly assigned" :managed: "no" :shortdesc: "The MAC address of the new interface" :type: "string" ``` ```{config:option} io.bus devices-nic_ovn :default: "`virtio`" :managed: "no" :shortdesc: "Override the bus for the device (can be `virtio` or `usb`, requires `acceleration` set to `none`) (VM only)" :type: "string" ``` ```{config:option} ipv4.address devices-nic_ovn :managed: "no" :shortdesc: "An IPv4 address to assign to the instance through DHCP, `none` can be used to disable IP allocation" :type: "string" ``` ```{config:option} ipv4.address.external devices-nic_ovn :managed: "no" :shortdesc: "Select a specific external address (typically from a network forward)" :type: "string" ``` ```{config:option} ipv4.routes devices-nic_ovn :managed: "no" :shortdesc: "Comma-delimited list of IPv4 static routes to route to the NIC" :type: "string" ``` ```{config:option} ipv4.routes.external devices-nic_ovn :managed: "no" :shortdesc: "Comma-delimited list of IPv4 static routes to route to the NIC and publish on uplink network" :type: "string" ``` ```{config:option} ipv6.address devices-nic_ovn :managed: "no" :shortdesc: "An IPv6 address to assign to the instance through DHCP, `none` can be used to disable IP allocation" :type: "string" ``` ```{config:option} ipv6.address.external devices-nic_ovn :managed: "no" :shortdesc: "Select a specific external address (typically from a network forward)" :type: "string" ``` ```{config:option} ipv6.routes devices-nic_ovn :managed: "no" :shortdesc: "Comma-delimited list of IPv6 static routes to route to the NIC" :type: "string" ``` ```{config:option} ipv6.routes.external devices-nic_ovn :managed: "no" :shortdesc: "Comma-delimited list of IPv6 static routes to route to the NIC and publish on uplink network" :type: "string" ``` ```{config:option} limits.egress devices-nic_ovn :managed: "no" :shortdesc: "I/O limit in bit/s for outgoing traffic (various suffixes supported, see {ref}`instances-limit-units`)" :type: "string" ``` ```{config:option} limits.ingress devices-nic_ovn :managed: "no" :shortdesc: "I/O limit in bit/s for incoming traffic (various suffixes supported, see {ref}`instances-limit-units`)" :type: "string" ``` ```{config:option} limits.max devices-nic_ovn :managed: "no" :shortdesc: "I/O limit in bit/s for both incoming and outgoing traffic. (same as setting both limits.ingress and limits.egress / mutually exclusive with limits.ingress and limits.egress)" :type: "string" ``` ```{config:option} limits.priority devices-nic_ovn :default: "100" :managed: "no" :shortdesc: "The priority for outgoing traffic, to be used by the kernel queuing discipline to prioritize network packets" :type: "integer" ``` ```{config:option} mtu devices-nic_ovn :default: "MTU of the parent network" :managed: "yes" :shortdesc: "The Maximum Transmit Unit (MTU) of the new interface" :type: "integer" ``` ```{config:option} name devices-nic_ovn :default: "kernel assigned" :managed: "no" :shortdesc: "The name of the interface inside the instance" :type: "string" ``` ```{config:option} nested devices-nic_ovn :managed: "no" :shortdesc: "The parent NIC name to nest this NIC under (see also `vlan`)" :type: "string" ``` ```{config:option} network devices-nic_ovn :managed: "yes" :shortdesc: "The managed network to link the device to (required)" :type: "string" ``` ```{config:option} security.acls devices-nic_ovn :managed: "no" :shortdesc: "Comma-separated list of network ACLs to apply" :type: "string" ``` ```{config:option} security.acls.default.egress.action devices-nic_ovn :default: "reject" :managed: "no" :shortdesc: "Action to use for egress traffic that doesn't match any ACL rule" :type: "string" ``` ```{config:option} security.acls.default.egress.logged devices-nic_ovn :default: "false" :managed: "no" :shortdesc: "Whether to log egress traffic that doesn't match any ACL rule" :type: "bool" ``` ```{config:option} security.acls.default.ingress.action devices-nic_ovn :default: "reject" :managed: "no" :shortdesc: "Action to use for ingress traffic that doesn't match any ACL rule" :type: "string" ``` ```{config:option} security.acls.default.ingress.logged devices-nic_ovn :default: "false" :managed: "no" :shortdesc: "Whether to log ingress traffic that doesn't match any ACL rule" :type: "bool" ``` ```{config:option} security.promiscuous devices-nic_ovn :default: "false" :managed: "no" :shortdesc: "Have OVN send unknown network traffic to this network interface (required for some nesting cases)" :type: "bool" ``` ```{config:option} vlan devices-nic_ovn :managed: "no" :shortdesc: "The VLAN ID to use when nesting (see also `nested`)" :type: "integer" ``` ```{config:option} attached devices-nic_p2p :default: "`true`" :required: "no" :shortdesc: "Whether the NIC is plugged in or not" :type: "bool" ``` ```{config:option} boot.priority devices-nic_p2p :shortdesc: "Boot priority for VMs (higher value boots first)" :type: "integer" ``` ```{config:option} connected devices-nic_p2p :default: "`true`" :required: "no" :shortdesc: "Whether the NIC is connected to the host network" :type: "bool" ``` ```{config:option} host_name devices-nic_p2p :default: "randomly assigned" :shortdesc: "The name of the interface on the host" :type: "string" ``` ```{config:option} hwaddr devices-nic_p2p :default: "randomly assigned" :shortdesc: "The MAC address of the new interface" :type: "string" ``` ```{config:option} io.bus devices-nic_p2p :default: "`virtio`" :shortdesc: "Override the bus for the device (can be `virtio` or `usb`) (VM only)" :type: "string" ``` ```{config:option} ipv4.routes devices-nic_p2p :shortdesc: "Comma-delimited list of IPv4 static routes to add on host to NIC" :type: "string" ``` ```{config:option} ipv6.routes devices-nic_p2p :shortdesc: "Comma-delimited list of IPv6 static routes to add on host to NIC" :type: "string" ``` ```{config:option} limits.egress devices-nic_p2p :shortdesc: "I/O limit in bit/s for outgoing traffic (various suffixes supported, see {ref}`instances-limit-units`)" :type: "string" ``` ```{config:option} limits.ingress devices-nic_p2p :shortdesc: "I/O limit in bit/s for incoming traffic (various suffixes supported, see {ref}`instances-limit-units`)" :type: "string" ``` ```{config:option} limits.max devices-nic_p2p :shortdesc: "I/O limit in bit/s for both incoming and outgoing traffic (same as setting both limits.ingress and limits.egress)" :type: "string" ``` ```{config:option} limits.priority devices-nic_p2p :shortdesc: "The priority for outgoing traffic, to be used by the kernel queuing discipline to prioritize network packets" :type: "integer" ``` ```{config:option} mtu devices-nic_p2p :default: "kernel assigned" :shortdesc: "The Maximum Transmit Unit (MTU) of the new interface" :type: "integer" ``` ```{config:option} name devices-nic_p2p :default: "kernel assigned" :shortdesc: "The name of the interface inside the instance" :type: "string" ``` ```{config:option} queue.tx.length devices-nic_p2p :shortdesc: "The transmit queue length for the NIC" :type: "integer" ``` ```{config:option} attached devices-nic_physical :default: "`true`" :required: "no" :shortdesc: "Whether the NIC is plugged in or not" :type: "bool" ``` ```{config:option} boot.priority devices-nic_physical :managed: "no" :shortdesc: "Boot priority for VMs (higher value boots first)" :type: "integer" ``` ```{config:option} gvrp devices-nic_physical :condition: "container" :default: "false" :managed: "no" :shortdesc: "Register VLAN using GARP VLAN Registration Protocol" :type: "bool" ``` ```{config:option} hwaddr devices-nic_physical :condition: "container" :default: "randomly assigned" :managed: "no" :shortdesc: "The MAC address of the new interface" :type: "string" ``` ```{config:option} mtu devices-nic_physical :condition: "container" :default: "MTU of the parent device" :managed: "no" :shortdesc: "The Maximum Transmit Unit (MTU) of the new interface" :type: "integer" ``` ```{config:option} name devices-nic_physical :default: "kernel assigned" :managed: "no" :shortdesc: "The name of the interface inside the instance" :type: "string" ``` ```{config:option} network devices-nic_physical :managed: "no" :shortdesc: "The managed network to link the device to (instead of specifying the `nictype` directly)" :type: "string" ``` ```{config:option} parent devices-nic_physical :managed: "yes" :shortdesc: "The name of the parent host device (required if specifying the `nictype` directly)" :type: "string" ``` ```{config:option} vlan devices-nic_physical :condition: "container" :managed: "no" :shortdesc: "The VLAN ID to attach to" :type: "integer" ``` ```{config:option} vlan.tagged devices-nic_physical :condition: "container" :managed: "no" :shortdesc: "Comma-delimited list of VLAN IDs or VLAN ranges to join for tagged traffic" :type: "integer" ``` ```{config:option} attached devices-nic_routed :default: "`true`" :required: "no" :shortdesc: "Whether the NIC is plugged in or not" :type: "bool" ``` ```{config:option} connected devices-nic_routed :default: "`true`" :required: "no" :shortdesc: "Whether the NIC is connected to the host network" :type: "bool" ``` ```{config:option} gvrp devices-nic_routed :default: "false" :shortdesc: "Register VLAN using GARP VLAN Registration Protocol" :type: "bool" ``` ```{config:option} host_name devices-nic_routed :default: "randomly assigned" :shortdesc: "The name of the interface on the host" :type: "string" ``` ```{config:option} hwaddr devices-nic_routed :default: "randomly assigned" :shortdesc: "The MAC address of the new interface" :type: "string" ``` ```{config:option} io.bus devices-nic_routed :default: "`virtio`" :shortdesc: "Override the bus for the device (can be `virtio` or `usb`) (VM only)" :type: "string" ``` ```{config:option} ipv4.address devices-nic_routed :shortdesc: "Comma-delimited list of IPv4 static addresses to add to the instance" :type: "string" ``` ```{config:option} ipv4.gateway devices-nic_routed :default: "auto" :shortdesc: "Whether to add an automatic default IPv4 gateway (can be `auto` or `none`)" :type: "string" ``` ```{config:option} ipv4.host_address devices-nic_routed :default: "`169.254.0.1`" :shortdesc: "The IPv4 address to add to the host-side `veth` interface" :type: "string" ``` ```{config:option} ipv4.host_table devices-nic_routed :shortdesc: "Deprecated: Use `ipv4.host_tables` instead" :type: "integer" The custom policy routing table ID to add IPv4 static routes to (in addition to the main routing table) ``` ```{config:option} ipv4.host_tables devices-nic_routed :default: "254" :shortdesc: "Comma-delimited list of routing tables IDs to add IPv4 static routes to" :type: "string" ``` ```{config:option} ipv4.neighbor_probe devices-nic_routed :default: "true" :shortdesc: "Whether to probe the parent network for IP address availability" :type: "bool" ``` ```{config:option} ipv4.routes devices-nic_routed :shortdesc: "Comma-delimited list of IPv4 static routes to add on host to NIC (without L2 ARP/NDP proxy)" :type: "string" ``` ```{config:option} ipv6.address devices-nic_routed :shortdesc: "Comma-delimited list of IPv6 static addresses to add to the instance" :type: "string" ``` ```{config:option} ipv6.gateway devices-nic_routed :default: "auto" :shortdesc: "Whether to add an automatic default IPv6 gateway (can be `auto` or `none`)" :type: "string" ``` ```{config:option} ipv6.host_address devices-nic_routed :default: "`fe80::1`" :shortdesc: "The IPv6 address to add to the host-side `veth` interface" :type: "string" ``` ```{config:option} ipv6.host_table devices-nic_routed :shortdesc: "Deprecated: Use `ipv6.host_tables` instead" :type: "integer" The custom policy routing table ID to add IPv6 static routes to (in addition to the main routing table) ``` ```{config:option} ipv6.host_tables devices-nic_routed :default: "254" :shortdesc: "Comma-delimited list of routing tables IDs to add IPv6 static routes to" :type: "string" ``` ```{config:option} ipv6.neighbor_probe devices-nic_routed :default: "true" :shortdesc: "Whether to probe the parent network for IP address availability" :type: "bool" ``` ```{config:option} ipv6.routes devices-nic_routed :shortdesc: "Comma-delimited list of IPv6 static routes to add on host to NIC (without L2 ARP/NDP proxy)" :type: "string" ``` ```{config:option} limits.egress devices-nic_routed :shortdesc: "I/O limit in bit/s for outgoing traffic (various suffixes supported, see {ref}`instances-limit-units`)" :type: "string" ``` ```{config:option} limits.ingress devices-nic_routed :shortdesc: "I/O limit in bit/s for incoming traffic (various suffixes supported, see {ref}`instances-limit-units`)" :type: "string" ``` ```{config:option} limits.max devices-nic_routed :shortdesc: "I/O limit in bit/s for both incoming and outgoing traffic (same as setting both limits.ingress and limits.egress)" :type: "string" ``` ```{config:option} limits.priority devices-nic_routed :shortdesc: "The priority for outgoing traffic, to be used by the kernel queuing discipline to prioritize network packets" :type: "integer" ``` ```{config:option} mtu devices-nic_routed :default: "parent MTU" :shortdesc: "The Maximum Transmit Unit (MTU) of the new interface" :type: "integer" ``` ```{config:option} name devices-nic_routed :default: "kernel assigned" :shortdesc: "The name of the interface inside the instance" :type: "string" ``` ```{config:option} parent devices-nic_routed :shortdesc: "The name of the parent host device to join the instance to" :type: "string" ``` ```{config:option} queue.tx.length devices-nic_routed :shortdesc: "The transmit queue length for the NIC" :type: "integer" ``` ```{config:option} vlan devices-nic_routed :shortdesc: "The VLAN ID to attach to" :type: "integer" ``` ```{config:option} vrf devices-nic_routed :shortdesc: "The VRF on the host in which the host-side interface and routes are created" :type: "string" ``` ```{config:option} attached devices-nic_sriov :default: "`true`" :required: "no" :shortdesc: "Whether the NIC is plugged in or not" :type: "bool" ``` ```{config:option} boot.priority devices-nic_sriov :managed: "no" :shortdesc: "Boot priority for VMs (higher value boots first)" :type: "integer" ``` ```{config:option} hwaddr devices-nic_sriov :default: "randomly assigned" :managed: "no" :shortdesc: "The MAC address of the new interface" :type: "string" ``` ```{config:option} mtu devices-nic_sriov :default: "kernel assigned" :managed: "yes" :shortdesc: "The Maximum Transmit Unit (MTU) of the new interface" :type: "integer" ``` ```{config:option} name devices-nic_sriov :default: "kernel assigned" :managed: "no" :shortdesc: "The name of the interface inside the instance" :type: "string" ``` ```{config:option} network devices-nic_sriov :managed: "no" :shortdesc: "The managed network to link the device to (instead of specifying the `nictype` directly)" :type: "string" ``` ```{config:option} parent devices-nic_sriov :managed: "yes" :shortdesc: "The name of the parent host device (required if specifying the `nictype` directly)" :type: "string" ``` ```{config:option} pci devices-nic_sriov :required: "no" :shortdesc: "The PCI address of the parent host device" :type: "string" ``` ```{config:option} productid devices-nic_sriov :required: "no" :shortdesc: "The product ID of the parent host device" :type: "string" ``` ```{config:option} security.mac_filtering devices-nic_sriov :default: "false" :managed: "no" :shortdesc: "Prevent the instance from spoofing another instance's MAC address" :type: "bool" ``` ```{config:option} security.trusted devices-nic_sriov :default: "false, if supported by parent device" :managed: "no" :shortdesc: "Allows the instance to configure the NIC in ways that may negatively impact security." :type: "bool" ``` ```{config:option} vendorid devices-nic_sriov :required: "no" :shortdesc: "The vendor ID of the parent host device" :type: "string" ``` ```{config:option} vlan devices-nic_sriov :managed: "no" :shortdesc: "The VLAN ID to attach to" :type: "integer" ``` ```{config:option} address devices-pci :required: "yes" :shortdesc: "PCI address of the device" :type: "string" ``` ```{config:option} firmware devices-pci :default: "true" :required: "no" :shortdesc: "Whether to expose the device's option ROM to the VM" :type: "bool" ``` ```{config:option} bind devices-proxy :default: "`host`" :required: "no" :shortdesc: "Which side to bind on (`host`/`instance`)" :type: "string" ``` ```{config:option} connect devices-proxy :required: "yes" :shortdesc: "The address and port to connect to (`::[-][,]`)" :type: "string" ``` ```{config:option} gid devices-proxy :default: "`0`" :required: "no" :shortdesc: "GID of the owner of the listening Unix socket" :type: "int" ``` ```{config:option} listen devices-proxy :required: "yes" :shortdesc: "The address and port to bind and listen (`::[-][,]`)" :type: "string" ``` ```{config:option} mode devices-proxy :default: "`0644`" :required: "no" :shortdesc: "Mode for the listening Unix socket" :type: "int" ``` ```{config:option} nat devices-proxy :default: "`false`" :required: "no" :shortdesc: "Whether to optimize proxying via NAT (requires that the instance NIC has a static IP address)" :type: "bool" ``` ```{config:option} proxy_protocol devices-proxy :default: "`false`" :required: "no" :shortdesc: "Whether to use the HAProxy PROXY protocol to transmit sender information" :type: "bool" ``` ```{config:option} security.gid devices-proxy :default: "`0`" :required: "no" :shortdesc: "What GID to drop privilege to" :type: "int" ``` ```{config:option} security.uid devices-proxy :default: "`0`" :required: "no" :shortdesc: "What UID to drop privilege to" :type: "int" ``` ```{config:option} uid devices-proxy :default: "`0`" :required: "no" :shortdesc: "UID of the owner of the listening Unix socket" :type: "int" ``` ```{config:option} path devices-tpm :default: "-" :required: "for containers" :shortdesc: "Only for containers: path inside the instance (for example, `/dev/tpm0`)" :type: "string" ``` ```{config:option} pathrm devices-tpm :default: "-" :required: "for containers" :shortdesc: "Only for containers: resource manager path inside the instance (for example, `/dev/tpmrm0`)" :type: "string" ``` ```{config:option} gid devices-unix-char-block :default: "0" :shortdesc: "GID of the device owner in the instance" :type: "int" ``` ```{config:option} major devices-unix-char-block :default: "device on host" :shortdesc: "Device major number" :type: "int" ``` ```{config:option} minor devices-unix-char-block :default: "device on host" :shortdesc: "Device minor number" :type: "int" ``` ```{config:option} mode devices-unix-char-block :default: "0660" :shortdesc: "Mode of the device in the instance" :type: "int" ``` ```{config:option} path devices-unix-char-block :shortdesc: "Path inside the instance (one of `source` and `path` must be set)" :type: "string" ``` ```{config:option} required devices-unix-char-block :default: "true" :shortdesc: "Whether this device is required to start the instance" :type: "bool" ``` ```{config:option} source devices-unix-char-block :shortdesc: "Path on the host (one of `source` and `path` must be set)" :type: "string" ``` ```{config:option} uid devices-unix-char-block :default: "0" :shortdesc: "UID of the device owner in the instance" :type: "int" ``` ```{config:option} gid devices-unix-hotplug :default: "0" :shortdesc: "GID of the device owner in the instance" :type: "int" ``` ```{config:option} mode devices-unix-hotplug :default: "0660" :shortdesc: "Mode of the device in the instance" :type: "int" ``` ```{config:option} pci devices-unix-hotplug :shortdesc: "The PCI address of a USB controller to monitor" :type: "string" ``` ```{config:option} productid devices-unix-hotplug :shortdesc: "The product ID of the USB device" :type: "string" ``` ```{config:option} required devices-unix-hotplug :default: "true" :shortdesc: "Whether this device is required to start the instance" :type: "bool" ``` ```{config:option} uid devices-unix-hotplug :default: "0" :shortdesc: "UID of the device owner in the instance" :type: "int" ``` ```{config:option} vendorid devices-unix-hotplug :shortdesc: "The vendor ID of the USB device" :type: "string" ``` ```{config:option} attached devices-usb :default: "`true`" :required: "no" :shortdesc: "Whether the USB device is plugged in or not" :type: "bool" ``` ```{config:option} busnum devices-usb :shortdesc: "The bus number of which the USB device is attached" :type: "int" ``` ```{config:option} devnum devices-usb :shortdesc: "The device number of the USB device" :type: "int" ``` ```{config:option} gid devices-usb :defaultdesc: "`0`" :shortdesc: "Only for containers: GID of the device owner in the instance" :type: "int" ``` ```{config:option} mode devices-usb :defaultdesc: "`0660`" :shortdesc: "Only for containers: Mode of the device in the instance" :type: "int" ``` ```{config:option} productid devices-usb :shortdesc: "The product ID of the USB device" :type: "string" ``` ```{config:option} required devices-usb :defaultdesc: "`false`" :shortdesc: "Whether this device is required to start the instance (the default is `false`, and all devices can be hotplugged)" :type: "bool" ``` ```{config:option} serial devices-usb :shortdesc: "The serial number of the USB device" :type: "string" ``` ```{config:option} uid devices-usb :defaultdesc: "`0`" :shortdesc: "Only for containers: UID of the device owner in the instance" :type: "int" ``` ```{config:option} vendorid devices-usb :shortdesc: "The vendor ID of the USB device" :type: "string" ``` ```{config:option} requirements.cdrom_agent image-requirements :shortdesc: "If set to `true`, indicates that the VM requires an `agent:config` disk be added." :type: "bool" ``` ```{config:option} requirements.cdrom_cloud_init image-requirements :shortdesc: "If set to `true`, indicates that the VM requires a `cloud-init:config` disk be present every time `cloud-init` should be run." :type: "bool" ``` ```{config:option} requirements.cgroup image-requirements :shortdesc: "If set to `v1`, indicates that the image requires the host to run cgroup v1." :type: "string" ``` ```{config:option} requirements.nesting image-requirements :shortdesc: "If set to `true`, indicates that the image cannot work without nesting enabled." :type: "bool" ``` ```{config:option} requirements.privileged image-requirements :shortdesc: "If set to `false`, indicates that the image cannot work as a privileged container." :type: "bool" ``` ```{config:option} requirements.secureboot image-requirements :shortdesc: "If set to `false`, indicates that the image cannot boot under secure boot." :type: "bool" ``` ```{config:option} boot.autorestart instance-boot :liveupdate: "no" :shortdesc: "Whether to automatically restart an instance on unexpected exit" :type: "bool" If set to `true` will attempt up to 10 restarts over a 1 minute period upon unexpected instance exit. ``` ```{config:option} boot.autostart instance-boot :liveupdate: "no" :shortdesc: "Whether to always start the instance when the daemon starts" :type: "bool" If unset or set to `last-state`, restores the last state. ``` ```{config:option} boot.autostart.delay instance-boot :defaultdesc: "0" :liveupdate: "no" :shortdesc: "Delay after starting the instance" :type: "integer" The number of seconds to wait after the instance started before starting the next one. ``` ```{config:option} boot.autostart.priority instance-boot :liveupdate: "no" :shortdesc: "What order to start the instances in" :type: "integer" The instance with the highest value is started first. Instances without a priority set will be started (with some parallelism) ahead of instances with a priority set. ``` ```{config:option} boot.host_shutdown_action instance-boot :defaultdesc: "stop" :liveupdate: "yes" :shortdesc: "What action to take on the instance when the host is shut down" :type: "string" Action to take on host shut down Valid values are: `stop`, `force-stop` or `stateful-stop` ``` ```{config:option} boot.host_shutdown_timeout instance-boot :defaultdesc: "30" :liveupdate: "yes" :shortdesc: "How long to wait for the instance to shut down" :type: "integer" Number of seconds to wait for the instance to shut down before it is force-stopped. ``` ```{config:option} boot.stop.priority instance-boot :defaultdesc: "0" :liveupdate: "no" :shortdesc: "What order to shut down the instances in" :type: "integer" The instance with the highest value is shut down first. ``` ```{config:option} cloud-init.network-config instance-cloud-init :condition: "If supported by image" :defaultdesc: "`DHCP on eth0`" :liveupdate: "no" :shortdesc: "Network configuration for `cloud-init`" :type: "string" The content is used as seed value for `cloud-init`. ``` ```{config:option} cloud-init.user-data instance-cloud-init :condition: "If supported by image" :defaultdesc: "`#cloud-config`" :liveupdate: "no" :shortdesc: "User data for `cloud-init`" :type: "string" The content is used as seed value for `cloud-init`. ``` ```{config:option} cloud-init.vendor-data instance-cloud-init :condition: "If supported by image" :defaultdesc: "`#cloud-config`" :liveupdate: "no" :shortdesc: "Vendor data for `cloud-init`" :type: "string" The content is used as seed value for `cloud-init`. ``` ```{config:option} user.network-config instance-cloud-init :condition: "If supported by image" :defaultdesc: "`DHCP on eth0`" :liveupdate: "no" :shortdesc: "Legacy version of `cloud-init.network-config`" :type: "string" ``` ```{config:option} user.user-data instance-cloud-init :condition: "If supported by image" :defaultdesc: "`#cloud-config`" :liveupdate: "no" :shortdesc: "Legacy version of `cloud-init.user-data`" :type: "string" ``` ```{config:option} user.vendor-data instance-cloud-init :condition: "If supported by image" :defaultdesc: "`#cloud-config`" :liveupdate: "no" :shortdesc: "Legacy version of `cloud-init.vendor-data`" :type: "string" ``` ```{config:option} migration.incremental.memory instance-migration :condition: "container" :defaultdesc: "`false`" :liveupdate: "yes" :shortdesc: "Whether to use incremental memory transfer" :type: "bool" Using incremental memory transfer of the instance's memory can reduce downtime. ``` ```{config:option} migration.incremental.memory.goal instance-migration :condition: "container" :defaultdesc: "`70`" :liveupdate: "yes" :shortdesc: "Percentage of memory to have in sync before stopping the instance" :type: "integer" ``` ```{config:option} migration.incremental.memory.iterations instance-migration :condition: "container" :defaultdesc: "`10`" :liveupdate: "yes" :shortdesc: "Maximum number of transfer operations to go through before stopping the instance" :type: "integer" ``` ```{config:option} migration.stateful instance-migration :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to allow for stateful stop/start and snapshots" :type: "bool" Enabling this option prevents the use of some features that are incompatible with it. ``` ```{config:option} agent.nic_config instance-miscellaneous :condition: "virtual machine" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to use the name and MTU of the default network interfaces" :type: "bool" For containers, the name and MTU of the default network interfaces is used for the instance devices. For virtual machines, set this option to `true` to set the name and MTU of the default network interfaces to be the same as the instance devices. ``` ```{config:option} cluster.evacuate instance-miscellaneous :defaultdesc: "`auto`" :liveupdate: "no" :shortdesc: "What to do when evacuating the instance" :type: "string" The `cluster.evacuate` provides control over how instances are handled when a cluster member is being evacuated. Available Modes: - `auto` *(default)*: The system will automatically decide the best evacuation method based on the instance's type and configured devices: + If any device is not suitable for migration, the instance will not be migrated (only stopped). + Live migration will be used only for virtual machines with the `migration.stateful` setting enabled and for which all its devices can be migrated as well. - `live-migrate`: Instances are live-migrated to another server. This means the instance remains running and operational during the migration process, ensuring minimal disruption. - `migrate`: In this mode, instances are migrated to another server in the cluster. The migration process will not be live, meaning there will be a brief downtime for the instance during the migration. - `stop`: Instances are not migrated. Instead, they are stopped on the current server. - `stateful-stop`: Instances are not migrated. Instead, they are stopped on the current server but with their runtime state (memory) stored on disk for resuming on restore. - `force-stop`: Instances are not migrated. Instead, they are forcefully stopped. See {ref}`cluster-evacuate` for more information. ``` ```{config:option} environment.* instance-miscellaneous :liveupdate: "yes" :shortdesc: "Free-form environment key/value" :type: "string" Extra environment variables to set on boot and during exec. ``` ```{config:option} linux.kernel_modules instance-miscellaneous :condition: "container" :liveupdate: "yes" :shortdesc: "Kernel modules to load before starting the instance" :type: "string" Specify the kernel modules as a comma-separated list. ``` ```{config:option} linux.sysctl.* instance-miscellaneous :condition: "container" :liveupdate: "no" :shortdesc: "Override for the corresponding `sysctl` setting in the container" :type: "string" ``` ```{config:option} smbios11.* instance-miscellaneous :liveupdate: "yes" :shortdesc: "Free-form `SMBIOS Type 11` key/value" :type: "string" `SMBIOS Type 11` configuration keys. ``` ```{config:option} systemd.credential-binary.* instance-miscellaneous :liveupdate: "yes" :shortdesc: "Systemd credential key/value, where value is Base64 encoded" :type: "string" Systemd credential key/value pair passed as a read-only bind mount in containers and as `SMBIOS Type 11` data in virtual machines. The value is Base64 encoded. ``` ```{config:option} systemd.credential.* instance-miscellaneous :liveupdate: "yes" :shortdesc: "Systemd credential key/value" :type: "string" Systemd credential key/value pair passed as a read-only bind mount in containers and as `SMBIOS Type 11` data in virtual machines. ``` ```{config:option} user.* instance-miscellaneous :liveupdate: "yes" :shortdesc: "Free-form user key/value storage" :type: "string" User keys can be used in search. ``` ```{config:option} nvidia.driver.capabilities instance-nvidia :condition: "container" :defaultdesc: "`compute,utility`" :liveupdate: "no" :shortdesc: "What driver capabilities the instance needs" :type: "string" The specified driver capabilities are used to set `libnvidia-container NVIDIA_DRIVER_CAPABILITIES`. ``` ```{config:option} nvidia.require.cuda instance-nvidia :condition: "container" :liveupdate: "no" :shortdesc: "Required CUDA version" :type: "string" The specified version expression is used to set `libnvidia-container NVIDIA_REQUIRE_CUDA`. ``` ```{config:option} nvidia.require.driver instance-nvidia :condition: "container" :liveupdate: "no" :shortdesc: "Required driver version" :type: "string" The specified version expression is used to set `libnvidia-container NVIDIA_REQUIRE_DRIVER`. ``` ```{config:option} nvidia.runtime instance-nvidia :condition: "container" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to pass the host NVIDIA and CUDA runtime libraries into the instance" :type: "bool" ``` ```{config:option} oci.cwd instance-oci :condition: "OCI container" :liveupdate: "no" :shortdesc: "OCI container working directory" :type: "string" Override the working directory of an OCI container. ``` ```{config:option} oci.entrypoint instance-oci :condition: "OCI container" :liveupdate: "no" :shortdesc: "OCI container entry point" :type: "string" Override the entry point of an OCI container. ``` ```{config:option} oci.gid instance-oci :condition: "OCI container" :liveupdate: "no" :shortdesc: "OCI container GID" :type: "string" Override the GID of the process run in an OCI container. ``` ```{config:option} oci.uid instance-oci :condition: "OCI container" :liveupdate: "no" :shortdesc: "OCI container UID" :type: "string" Override the UID of the process run in an OCI container. ``` ```{config:option} raw.apparmor instance-raw :liveupdate: "yes" :shortdesc: "AppArmor profile entries" :type: "blob" The specified entries are appended to the generated profile. ``` ```{config:option} raw.idmap instance-raw :condition: "unprivileged container" :liveupdate: "no" :shortdesc: "Raw idmap configuration" :type: "blob" For example: `both 1000 1000` ``` ```{config:option} raw.lxc instance-raw :condition: "container" :liveupdate: "no" :shortdesc: "Raw LXC configuration to be appended to the generated one" :type: "blob" ``` ```{config:option} raw.qemu instance-raw :condition: "virtual machine" :liveupdate: "no" :shortdesc: "Raw QEMU configuration to be appended to the generated command line" :type: "blob" ``` ```{config:option} raw.qemu.conf instance-raw :condition: "virtual machine" :liveupdate: "no" :shortdesc: "Addition/override to the generated `qemu.conf` file" :type: "blob" See {ref}`instance-options-qemu` for more information. ``` ```{config:option} raw.qemu.qmp.early instance-raw :condition: "virtual machine" :liveupdate: "no" :shortdesc: "QMP commands to run before Incus QEMU initialization" :type: "blob" ``` ```{config:option} raw.qemu.qmp.post-start instance-raw :condition: "virtual machine" :liveupdate: "no" :shortdesc: "QMP commands to run after the VM has started" :type: "blob" ``` ```{config:option} raw.qemu.qmp.pre-start instance-raw :condition: "virtual machine" :liveupdate: "no" :shortdesc: "QMP commands to run after Incus QEMU initialization and before the VM has started" :type: "blob" ``` ```{config:option} raw.qemu.scriptlet instance-raw :condition: "virtual machine" :liveupdate: "no" :shortdesc: "QEMU scriptlet to run at early, pre-start and post-start stages" :type: "string" ``` ```{config:option} raw.seccomp instance-raw :condition: "container" :liveupdate: "no" :shortdesc: "Raw Seccomp configuration" :type: "blob" ``` ```{config:option} limits.cpu instance-resource-limits :defaultdesc: "1 (VMs)" :liveupdate: "yes" :shortdesc: "Which CPUs to expose to the instance" :type: "string" A number or a specific range of CPUs to expose to the instance. See {ref}`instance-options-limits-cpu` for more information. ``` ```{config:option} limits.cpu.allowance instance-resource-limits :condition: "container" :defaultdesc: "100%" :liveupdate: "yes" :shortdesc: "How much of the CPU can be used" :type: "string" To control how much of the CPU can be used, specify either a percentage (`50%`) for a soft limit or a chunk of time (`25ms/100ms`) for a hard limit. See {ref}`instance-options-limits-cpu-container` for more information. ``` ```{config:option} limits.cpu.nodes instance-resource-limits :liveupdate: "yes" :shortdesc: "Which NUMA nodes to place the instance CPUs on" :type: "string" A comma-separated list of NUMA node IDs or ranges to place the instance CPUs on. Alternatively, the value `balanced` may be used to have Incus pick the least busy NUMA node on startup. See {ref}`instance-options-limits-cpu-container` for more information. ``` ```{config:option} limits.cpu.priority instance-resource-limits :condition: "container" :defaultdesc: "`10` (maximum)" :liveupdate: "yes" :shortdesc: "CPU scheduling priority compared to other instances" :type: "integer" When overcommitting resources, specify the CPU scheduling priority compared to other instances that share the same CPUs. Specify an integer between 0 and 10. See {ref}`instance-options-limits-cpu-container` for more information. ``` ```{config:option} limits.disk.priority instance-resource-limits :defaultdesc: "`5` (medium)" :liveupdate: "yes" :shortdesc: "Priority of the instance's I/O requests" :type: "integer" Controls how much priority to give to the instance's I/O requests when under load. Specify an integer between 0 and 10. ``` ```{config:option} limits.hugepages.1GB instance-resource-limits :condition: "container" :liveupdate: "yes" :shortdesc: "Limit for the number of 1 GB huge pages" :type: "string" Fixed value (in bytes) to limit the number of 1 GB huge pages. Various suffixes are supported (see {ref}`instances-limit-units`). See {ref}`instance-options-limits-hugepages` for more information. ``` ```{config:option} limits.hugepages.1MB instance-resource-limits :condition: "container" :liveupdate: "yes" :shortdesc: "Limit for the number of 1 MB huge pages" :type: "string" Fixed value (in bytes) to limit the number of 1 MB huge pages. Various suffixes are supported (see {ref}`instances-limit-units`). See {ref}`instance-options-limits-hugepages` for more information. ``` ```{config:option} limits.hugepages.2MB instance-resource-limits :condition: "container" :liveupdate: "yes" :shortdesc: "Limit for the number of 2 MB huge pages" :type: "string" Fixed value (in bytes) to limit the number of 2 MB huge pages. Various suffixes are supported (see {ref}`instances-limit-units`). See {ref}`instance-options-limits-hugepages` for more information. ``` ```{config:option} limits.hugepages.64KB instance-resource-limits :condition: "container" :liveupdate: "yes" :shortdesc: "Limit for the number of 64 KB huge pages" :type: "string" Fixed value (in bytes) to limit the number of 64 KB huge pages. Various suffixes are supported (see {ref}`instances-limit-units`). See {ref}`instance-options-limits-hugepages` for more information. ``` ```{config:option} limits.memory instance-resource-limits :defaultdesc: "`1GiB` (VMs)" :liveupdate: "yes" :shortdesc: "Usage limit for the host's memory" :type: "string" Percentage of the host's memory or a fixed value in bytes. Various suffixes are supported. See {ref}`instances-limit-units` for details. ``` ```{config:option} limits.memory.enforce instance-resource-limits :condition: "container" :defaultdesc: "`hard`" :liveupdate: "yes" :shortdesc: "Whether the memory limit is `hard` or `soft`" :type: "string" If the instance's memory limit is `hard`, the instance cannot exceed its limit. If it is `soft`, the instance can exceed its memory limit when extra host memory is available. ``` ```{config:option} limits.memory.hotplug instance-resource-limits :condition: "virtual machine" :defaultdesc: "`true`" :liveupdate: "yes" :shortdesc: "Control upper limit for hotplugged memory or disable memory hotplug." :type: "string" If this option is set to `false`, disable memory hotplug entirely. Alternatively, it can be set to a bytes value which will define an upper limit for hotplugged memory. The value must be greater than or equal to limits.memory. ``` ```{config:option} limits.memory.hugepages instance-resource-limits :condition: "virtual machine" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to back the instance using huge pages" :type: "bool" If this option is set to `false`, regular system memory is used. ``` ```{config:option} limits.memory.oom_priority instance-resource-limits :defaultdesc: "`0`" :liveupdate: "yes" :shortdesc: "Out Of Memory killer priority adjustment for the instance" :type: "integer" Specify an integer between -1000 and 1000. A negative value makes the instance less likely to be killed by the Out Of Memory killer, while a positive value makes it more likely to be killed. The default value of 0 means no adjustment to the Out Of Memory score. ``` ```{config:option} limits.memory.swap instance-resource-limits :condition: "container" :defaultdesc: "`true`" :liveupdate: "yes" :shortdesc: "Control swap usage by the instance" :type: "string" When set to `true` or `false`, it controls whether the container is likely to get some of its memory swapped by the kernel. Alternatively, it can be set to a bytes value which will then allow the container to make use of additional memory through swap. ``` ```{config:option} limits.memory.swap.priority instance-resource-limits :condition: "container" :defaultdesc: "`10` (maximum)" :liveupdate: "yes" :shortdesc: "Prevents the instance from being swapped to disk" :type: "integer" Specify an integer between 0 and 10. The higher the value, the less likely the instance is to be swapped to disk. ``` ```{config:option} limits.processes instance-resource-limits :condition: "container" :defaultdesc: "empty" :liveupdate: "yes" :shortdesc: "Maximum number of processes that can run in the instance" :type: "integer" If left empty, no limit is set. ``` ```{config:option} security.agent.metrics instance-security :condition: "virtual machine" :defaultdesc: "`true`" :liveupdate: "no" :shortdesc: "Whether the `incus-agent` is queried for state information and metrics" :type: "bool" ``` ```{config:option} security.bpffs.delegate_attachs instance-security :condition: "unprivileged container" :liveupdate: "no" :shortdesc: "What BPF attach types to delegate" :type: "string" See {ref}`bpf-tokens` for more information. ``` ```{config:option} security.bpffs.delegate_cmds instance-security :condition: "unprivileged container" :liveupdate: "no" :shortdesc: "What BPF command types to delegate" :type: "string" See {ref}`bpf-tokens` for more information. ``` ```{config:option} security.bpffs.delegate_maps instance-security :condition: "unprivileged container" :liveupdate: "no" :shortdesc: "What BPF map types to delegate" :type: "string" See {ref}`bpf-tokens` for more information. ``` ```{config:option} security.bpffs.delegate_progs instance-security :condition: "unprivileged container" :liveupdate: "no" :shortdesc: "What BPF program types to delegate" :type: "string" See {ref}`bpf-tokens` for more information. ``` ```{config:option} security.bpffs.path instance-security :condition: "unprivileged container" :defaultdesc: "`/sys/fs/bpf`" :liveupdate: "no" :shortdesc: "The path to mount the BPF file system at" :type: "string" The specified path must exist in the container. The BPF file system is only mounted if any of the `security.bpffs.delegate_*` options are set. See {ref}`bpf-tokens` for more information. ``` ```{config:option} security.csm instance-security :condition: "virtual machine" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to use a firmware that supports UEFI-incompatible operating systems" :type: "bool" When enabling this option, set {config:option}`instance-security:security.secureboot` to `false`. ``` ```{config:option} security.guestapi instance-security :defaultdesc: "`true`" :liveupdate: "no" :shortdesc: "Whether `/dev/incus` is present in the instance" :type: "bool" See {ref}`dev-incus` for more information. ``` ```{config:option} security.guestapi.images instance-security :condition: "container" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Controls the availability of the `/1.0/images` API over `guestapi`" :type: "bool" ``` ```{config:option} security.idmap.base instance-security :condition: "unprivileged container" :liveupdate: "no" :shortdesc: "The base host ID to use for the allocation" :type: "integer" Setting this option overrides auto-detection. ``` ```{config:option} security.idmap.isolated instance-security :condition: "unprivileged container" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to use a unique idmap for this instance" :type: "bool" If specified, the idmap used for this instance is unique among instances that have this option set. ``` ```{config:option} security.idmap.size instance-security :condition: "unprivileged container" :liveupdate: "no" :shortdesc: "The size of the idmap to use" :type: "integer" ``` ```{config:option} security.iommu instance-security :condition: "virtual machine" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to enable virtual IOMMU, useful for device passthrough and nesting" :type: "bool" ``` ```{config:option} security.nesting instance-security :condition: "container" :defaultdesc: "`false`" :liveupdate: "yes" :shortdesc: "Whether to support running Incus (nested) inside the instance" :type: "bool" ``` ```{config:option} security.privileged instance-security :condition: "container" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to run the instance in privileged mode" :type: "bool" ``` ```{config:option} security.protection.delete instance-security :defaultdesc: "`false`" :liveupdate: "yes" :shortdesc: "Prevents the instance from being deleted" :type: "bool" ``` ```{config:option} security.protection.shift instance-security :condition: "container" :defaultdesc: "`false`" :liveupdate: "yes" :shortdesc: "Whether to protect the file system from being UID/GID shifted" :type: "bool" Set this option to `true` to prevent the instance's file system from being UID/GID shifted on startup. ``` ```{config:option} security.secureboot instance-security :condition: "virtual machine" :defaultdesc: "`true`" :liveupdate: "no" :shortdesc: "Whether UEFI secure boot is enforced with the default Microsoft keys" :type: "bool" When disabling this option, consider enabling {config:option}`instance-security:security.csm`. ``` ```{config:option} security.sev instance-security :condition: "virtual machine" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether AMD SEV (Secure Encrypted Virtualization) is enabled for this VM" :type: "bool" ``` ```{config:option} security.sev.policy.es instance-security :condition: "virtual machine" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether AMD SEV-ES (SEV Encrypted State) is enabled for this VM" :type: "bool" ``` ```{config:option} security.sev.session.data instance-security :condition: "virtual machine" :defaultdesc: "`true`" :liveupdate: "no" :shortdesc: "The guest owner's `base64`-encoded session blob" :type: "string" ``` ```{config:option} security.sev.session.dh instance-security :condition: "virtual machine" :defaultdesc: "`true`" :liveupdate: "no" :shortdesc: "The guest owner's `base64`-encoded Diffie-Hellman key" :type: "string" ``` ```{config:option} security.syscalls.allow instance-security :condition: "container" :liveupdate: "no" :shortdesc: "List of syscalls to allow" :type: "string" A `\n`-separated list of syscalls to allow. This list must be mutually exclusive with `security.syscalls.deny*`. ``` ```{config:option} security.syscalls.deny instance-security :condition: "container" :liveupdate: "no" :shortdesc: "List of syscalls to deny" :type: "string" A `\n`-separated list of syscalls to deny. This list must be mutually exclusive with `security.syscalls.allow`. ``` ```{config:option} security.syscalls.deny_compat instance-security :condition: "container" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to block `compat_*` syscalls (`x86_64` only)" :type: "bool" On `x86_64`, this option controls whether to block `compat_*` syscalls. On other architectures, the option is ignored. ``` ```{config:option} security.syscalls.deny_default instance-security :condition: "container" :defaultdesc: "`true`" :liveupdate: "no" :shortdesc: "Whether to enable the default syscall deny" :type: "bool" ``` ```{config:option} security.syscalls.intercept.bpf instance-security :condition: "container" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to handle the `bpf()` system call" :type: "bool" ``` ```{config:option} security.syscalls.intercept.bpf.devices instance-security :condition: "container" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to allow BPF programs" :type: "bool" This option controls whether to allow BPF programs for the devices cgroup in the unified hierarchy to be loaded. ``` ```{config:option} security.syscalls.intercept.mknod instance-security :condition: "container" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to handle the `mknod` and `mknodat` system calls" :type: "bool" These system calls allow creation of a limited subset of char/block devices. ``` ```{config:option} security.syscalls.intercept.mount instance-security :condition: "container" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to handle the `mount` system call" :type: "bool" ``` ```{config:option} security.syscalls.intercept.mount.allowed instance-security :condition: "container" :liveupdate: "yes" :shortdesc: "File systems that can be mounted" :type: "string" Specify a comma-separated list of file systems that are safe to mount for processes inside the instance. ``` ```{config:option} security.syscalls.intercept.mount.fuse instance-security :condition: "container" :liveupdate: "yes" :shortdesc: "File system that should be redirected to FUSE implementation" :type: "string" Specify the mounts of a given file system that should be redirected to their FUSE implementation (for example, `ext4=fuse2fs`). ``` ```{config:option} security.syscalls.intercept.mount.shift instance-security :condition: "container" :defaultdesc: "`false`" :liveupdate: "yes" :shortdesc: "Whether to use idmapped mounts for syscall interception" :type: "bool" ``` ```{config:option} security.syscalls.intercept.sched_setscheduler instance-security :condition: "container" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to handle the `sched_setscheduler` system call" :type: "bool" This system call allows increasing process priority. ``` ```{config:option} security.syscalls.intercept.setxattr instance-security :condition: "container" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to handle the `setxattr` system call" :type: "bool" This system call allows setting a limited subset of restricted extended attributes. ``` ```{config:option} security.syscalls.intercept.sysinfo instance-security :condition: "container" :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to handle the `sysinfo` system call" :type: "bool" This system call can be used to get cgroup-based resource usage information. ``` ```{config:option} snapshots.expiry instance-snapshots :liveupdate: "no" :shortdesc: "When snapshots are to be deleted" :type: "string" Specify an expression like `1M 2H 3d 4w 5m 6y`. ``` ```{config:option} snapshots.expiry.manual instance-snapshots :liveupdate: "no" :shortdesc: "When snapshots are to be deleted (for those not created through scheduling)" :type: "string" Specify an expression like `1M 2H 3d 4w 5m 6y`. ``` ```{config:option} snapshots.pattern instance-snapshots :defaultdesc: "`snap%d`" :liveupdate: "no" :shortdesc: "Template for the snapshot name" :type: "string" Specify a Pongo2 template string that represents the snapshot name. This template is used for scheduled snapshots and for unnamed snapshots. See {ref}`instance-options-snapshots-names` for more information. ``` ```{config:option} snapshots.schedule instance-snapshots :defaultdesc: "empty" :liveupdate: "no" :shortdesc: "Schedule for automatic instance snapshots" :type: "string" Specify either a cron expression (` `), a comma-and-space-separated list of schedule aliases (`@startup`, `@hourly`, `@daily`, `@midnight`, `@weekly`, `@monthly`, `@annually`, `@yearly`), or leave empty to disable automatic snapshots. Note that unlike most other configuration keys, this one must be comma-and-space-separated and not just comma-separated as cron expression can themselves contain commas. ``` ```{config:option} snapshots.schedule.stopped instance-snapshots :defaultdesc: "`false`" :liveupdate: "no" :shortdesc: "Whether to automatically snapshot stopped instances" :type: "bool" ``` ```{config:option} volatile..apply_quota instance-volatile :shortdesc: "Disk quota" :type: "string" The disk quota is applied the next time the instance starts. ``` ```{config:option} volatile..ceph_rbd instance-volatile :shortdesc: "RBD device path for Ceph disk devices" :type: "string" ``` ```{config:option} volatile..host_name instance-volatile :shortdesc: "Network device name on the host" :type: "string" ``` ```{config:option} volatile..hwaddr instance-volatile :shortdesc: "Network device MAC address" :type: "string" The network device MAC address is used when no `hwaddr` property is set on the device itself. ``` ```{config:option} volatile..io.bus instance-volatile :shortdesc: "IO bus in use" :type: "string" The IO bus stores the actual IO bus being used, checked in case `io.bus=auto`. ``` ```{config:option} volatile..last_state.created instance-volatile :shortdesc: "Whether the network device physical device was created" :type: "string" Possible values are `true` or `false`. ``` ```{config:option} volatile..last_state.hwaddr instance-volatile :shortdesc: "Network device original MAC" :type: "string" The original MAC that was used when moving a physical device into an instance. ``` ```{config:option} volatile..last_state.ip_addresses instance-volatile :shortdesc: "Last used IP addresses" :type: "string" Comma-separated list of the last used IP addresses of the network device. ``` ```{config:option} volatile..last_state.mtu instance-volatile :shortdesc: "Network device original MTU" :type: "string" The original MTU that was used when moving a physical device into an instance. ``` ```{config:option} volatile..last_state.pci.driver instance-volatile :shortdesc: "PCI original host driver" :type: "string" The original host driver for the PCI device. ``` ```{config:option} volatile..last_state.pci.parent instance-volatile :shortdesc: "PCI parent host device" :type: "string" The parent host device used when allocating a PCI device to an instance. ``` ```{config:option} volatile..last_state.pci.slot.name instance-volatile :shortdesc: "PCI parent slot name" :type: "string" The parent host device PCI slot name. ``` ```{config:option} volatile..last_state.usb.bus instance-volatile :shortdesc: "USB bus address" :type: "string" The original USB bus address. ``` ```{config:option} volatile..last_state.usb.device instance-volatile :shortdesc: "USB device identifier" :type: "string" The original USB device identifier. ``` ```{config:option} volatile..last_state.vdpa.name instance-volatile :shortdesc: "VDPA device name" :type: "string" The VDPA device name used when moving a VDPA device file descriptor into an instance. ``` ```{config:option} volatile..last_state.vf.hwaddr instance-volatile :shortdesc: "SR-IOV virtual function original MAC" :type: "string" The original MAC used when moving a VF into an instance. ``` ```{config:option} volatile..last_state.vf.id instance-volatile :shortdesc: "SR-IOV virtual function ID" :type: "string" The ID used when moving a VF into an instance. ``` ```{config:option} volatile..last_state.vf.parent instance-volatile :shortdesc: "SR-IOV parent host device" :type: "string" The parent host device used when allocating a VF into an instance. ``` ```{config:option} volatile..last_state.vf.spoofcheck instance-volatile :shortdesc: "SR-IOV virtual function original spoof check setting" :type: "string" The original spoof check setting used when moving a VF into an instance. ``` ```{config:option} volatile..last_state.vf.trusted instance-volatile :shortdesc: "SR-IOV virtual function original trusted setting" :type: "string" The original trusted setting used when moving a VF into an instance. ``` ```{config:option} volatile..last_state.vf.vlan instance-volatile :shortdesc: "SR-IOV virtual function original VLAN" :type: "string" The original VLAN used when moving a VF into an instance. ``` ```{config:option} volatile..mig.uuid instance-volatile :shortdesc: "MIG instance UUID" :type: "string" The NVIDIA MIG instance UUID. ``` ```{config:option} volatile..name instance-volatile :shortdesc: "Network interface name inside of the instance" :type: "string" The network interface name inside of the instance when no `name` property is set on the device itself. ``` ```{config:option} volatile..vgpu.uuid instance-volatile :shortdesc: "virtual GPU instance UUID" :type: "string" The NVIDIA virtual GPU instance UUID. ``` ```{config:option} volatile.apply_nvram instance-volatile :shortdesc: "Whether to regenerate VM NVRAM the next time the instance starts" :type: "bool" ``` ```{config:option} volatile.apply_template instance-volatile :shortdesc: "Template hook" :type: "string" The template with the given name is triggered upon next startup. ``` ```{config:option} volatile.base_image instance-volatile :shortdesc: "Hash of the base image" :type: "string" The hash of the image that the instance was created from (empty if the instance was not created from an image). ``` ```{config:option} volatile.cloud_init.instance-id instance-volatile :shortdesc: "`instance-id` (UUID) exposed to `cloud-init`" :type: "string" ``` ```{config:option} volatile.cluster.group instance-volatile :shortdesc: "The original cluster group for the instance" :type: "string" The cluster group(s) that the instance was restricted to at creation time. This is used during re-scheduling events like an evacuation to keep the instance within the requested set. ``` ```{config:option} volatile.container.oci instance-volatile :defaultdesc: "`false`" :shortdesc: "Whether the container is an OCI application container" :type: "bool" ``` ```{config:option} volatile.cpu.nodes instance-volatile :shortdesc: "Instance NUMA node" :type: "string" The NUMA node that was selected for the instance. ``` ```{config:option} volatile.evacuate.origin instance-volatile :shortdesc: "The origin of the evacuated instance" :type: "string" The cluster member that the instance lived on before evacuation. ``` ```{config:option} volatile.idmap.base instance-volatile :shortdesc: "The first ID in the instance's primary idmap range" :type: "integer" ``` ```{config:option} volatile.idmap.current instance-volatile :shortdesc: "The idmap currently in use by the instance" :type: "string" ``` ```{config:option} volatile.idmap.next instance-volatile :shortdesc: "The idmap to use the next time the instance starts" :type: "string" ``` ```{config:option} volatile.last_state.idmap instance-volatile :shortdesc: "Serialized instance UID/GID map" :type: "string" ``` ```{config:option} volatile.last_state.power instance-volatile :shortdesc: "Instance state as of last host shutdown" :type: "string" ``` ```{config:option} volatile.last_state.ready instance-volatile :shortdesc: "Instance marked itself as ready" :type: "string" ``` ```{config:option} volatile.rebalance.last_move instance-volatile :shortdesc: "Timestamp of last move by automatic live-migration" :type: "integer" ``` ```{config:option} volatile.uuid instance-volatile :shortdesc: "Instance UUID" :type: "string" The instance UUID is globally unique across all servers and projects. ``` ```{config:option} volatile.uuid.generation instance-volatile :shortdesc: "Instance generation UUID" :type: "string" The instance generation UUID changes whenever the instance's place in time moves backwards. It is globally unique across all servers and projects. ``` ```{config:option} volatile.vm.boot_state instance-volatile :shortdesc: "JSON encoded VM properties used during live migration and other state restoration." :type: "string" ``` ```{config:option} volatile.vm.needs_reset instance-volatile :shortdesc: "Indicates that the VM needs a full reset on next reboot" :type: "bool" ``` ```{config:option} volatile.vm.rtc_adjustment instance-volatile :shortdesc: "Real Time Clock change adjustment" :type: "int64" Real Time Clock adjustment time to allow virtual machines to run on a different base than the host. ``` ```{config:option} volatile.vm.rtc_offset instance-volatile :shortdesc: "Real Time Clock change offset" :type: "int64" Real Time Clock offset to allow virtual machines to run on a different base than the host. ``` ```{config:option} volatile.vsock_id instance-volatile :shortdesc: "Instance `vsock ID` used as of last start" :type: "string" ``` ```{config:option} limits.kernel.as kernel-limits :resource: "`RLIMIT_AS`" :shortdesc: "Maximum size of the process's virtual memory" :type: "string" ``` ```{config:option} limits.kernel.core kernel-limits :resource: "`RLIMIT_CORE`" :shortdesc: "Maximum size of the process's core dump file" :type: "string" ``` ```{config:option} limits.kernel.cpu kernel-limits :resource: "`RLIMIT_CPU`" :shortdesc: "Limit in seconds on the amount of CPU time the process can consume" :type: "string" ``` ```{config:option} limits.kernel.data kernel-limits :resource: "`RLIMIT_DATA`" :shortdesc: "Maximum size of the process's data segment" :type: "string" ``` ```{config:option} limits.kernel.fsize kernel-limits :resource: "`RLIMIT_FSIZE`" :shortdesc: "Maximum size of files the process may create" :type: "string" ``` ```{config:option} limits.kernel.locks kernel-limits :resource: "`RLIMIT_LOCKS`" :shortdesc: "Limit on the number of file locks that this process may establish" :type: "string" ``` ```{config:option} limits.kernel.memlock kernel-limits :resource: "`RLIMIT_MEMLOCK`" :shortdesc: "Limit on the number of bytes of memory that the process may lock in RAM" :type: "string" ``` ```{config:option} limits.kernel.nice kernel-limits :resource: "`RLIMIT_NICE`" :shortdesc: "Maximum value to which the process's nice value can be raised" :type: "string" ``` ```{config:option} limits.kernel.nofile kernel-limits :resource: "`RLIMIT_NOFILE`" :shortdesc: "Maximum number of open files for the process" :type: "string" ``` ```{config:option} limits.kernel.nproc kernel-limits :resource: "`RLIMIT_NPROC`" :shortdesc: "Maximum number of processes that can be created for the user of the calling process" :type: "string" ``` ```{config:option} limits.kernel.rtprio kernel-limits :resource: "`RLIMIT_RTPRIO`" :shortdesc: "Maximum value on the real-time-priority that may be set for this process" :type: "string" ``` ```{config:option} limits.kernel.sigpending kernel-limits :resource: "`RLIMIT_SIGPENDING`" :shortdesc: "Limit on the number of bytes of memory that the process may lock in RAM" :type: "string" ``` ```{config:option} user.* network_address_set-common :shortdesc: "Free form user key/value storage" :type: "string" User keys can be used in search. ``` ```{config:option} bgp.peers.NAME.address network_bridge-bgp :condition: "BGP server" :defaultdesc: "-" :shortdesc: "Peer address (IPv4 or IPv6) for use by `ovn` downstream networks" :type: "string" ``` ```{config:option} bgp.peers.NAME.asn network_bridge-bgp :condition: "BGP server" :defaultdesc: "-" :shortdesc: "Peer AS number for use by `ovn` downstream networks" :type: "integer" ``` ```{config:option} bgp.peers.NAME.holdtime network_bridge-bgp :condition: "BGP server" :defaultdesc: "`180`" :shortdesc: "Peer session hold time (in seconds; optional)" :type: "integer" ``` ```{config:option} bgp.peers.NAME.password network_bridge-bgp :condition: "BGP server" :defaultdesc: "- (no password)" :shortdesc: "Peer session password (optional) for use by `ovn` downstream networks" :type: "string" ``` ```{config:option} bgp.ipv4.nexthop network_bridge-common :condition: "BGP server" :default: "local address" :shortdesc: "Override the next-hop for advertised prefixes" :type: "string" ``` ```{config:option} bgp.ipv6.nexthop network_bridge-common :condition: "BGP server" :default: "local address" :shortdesc: "Override the next-hop for advertised prefixes" :type: "string" ``` ```{config:option} bridge.driver network_bridge-common :condition: "-" :default: "`native`" :shortdesc: "Bridge driver: `native` or `openvswitch`" :type: "string" ``` ```{config:option} bridge.external_interfaces network_bridge-common :condition: "-" :default: "-" :shortdesc: "Comma-separated list of unconfigured network interfaces to include in the bridge" :type: "string" ``` ```{config:option} bridge.hwaddr network_bridge-common :condition: "-" :default: "-" :shortdesc: "MAC address for the bridge" :type: "string" ``` ```{config:option} bridge.mtu network_bridge-common :condition: "-" :default: "`1500`" :shortdesc: "Bridge MTU (default varies if tunnel in use)" :type: "integer" ``` ```{config:option} dns.domain network_bridge-common :condition: "-" :default: "`incus`" :shortdesc: "Domain to advertise to DHCP clients and use for DNS resolution" :type: "string" ``` ```{config:option} dns.mode network_bridge-common :condition: "-" :default: "`managed`" :shortdesc: "DNS registration mode: none for no DNS record, managed for Incus-generated static records or dynamic for client-generated records" :type: "string" ``` ```{config:option} dns.nameservers network_bridge-common :condition: "-" :default: "IPv4 and IPv6 address" :shortdesc: "DNS server IPs to advertise to DHCP clients and via Router Advertisements. Both IPv4 and IPv6 addresses get pushed via DHCP, and IPv6 addresses are also advertised as RDNSS via RA." :type: "string" ``` ```{config:option} dns.search network_bridge-common :condition: "-" :default: "-" :shortdesc: "Full comma-separated domain search list, defaulting to `dns.domain` value" :type: "string" ``` ```{config:option} dns.zone.forward network_bridge-common :condition: "-" :default: "`managed`" :shortdesc: "Comma-separated list of DNS zone names for forward DNS records" :type: "string" ``` ```{config:option} dns.zone.reverse.ipv4 network_bridge-common :condition: "-" :default: "`managed`" :shortdesc: "DNS zone name for IPv4 reverse DNS records" :type: "string" ``` ```{config:option} dns.zone.reverse.ipv6 network_bridge-common :condition: "-" :default: "`managed`" :shortdesc: "DNS zone name for IPv6 reverse DNS records" :type: "string" ``` ```{config:option} ipv4.address network_bridge-common :condition: "standard mode" :default: "- (initial value on creation: `auto`)" :shortdesc: "IPv4 address for the bridge (use `none` to turn off IPv4 or `auto` to generate a new random unused subnet) (CIDR)" :type: "string" ``` ```{config:option} ipv4.dhcp network_bridge-common :condition: "IPv4 address" :default: "`true`" :shortdesc: "Whether to allocate addresses using DHCP" :type: "bool" ``` ```{config:option} ipv4.dhcp.expiry network_bridge-common :condition: "IPv4 DHCP" :default: "`1h`" :shortdesc: "When to expire DHCP leases" :type: "string" ``` ```{config:option} ipv4.dhcp.gateway network_bridge-common :condition: "IPv4 DHCP" :default: "IPv4 address" :shortdesc: "Address of the gateway for the subnet (use `none` to turn off gateway announcement)" :type: "string" ``` ```{config:option} ipv4.dhcp.ranges network_bridge-common :condition: "IPv4 DHCP" :default: "all addresses" :shortdesc: "Comma-separated list of IP ranges to use for DHCP (FIRST-LAST format)" :type: "string" ``` ```{config:option} ipv4.dhcp.routes network_bridge-common :condition: "IPv4 DHCP" :default: "-" :shortdesc: "Static routes to provide via DHCP option 121, as a comma-separated list of alternating subnets (CIDR) and gateway addresses (same syntax as dnsmasq)" :type: "string" ``` ```{config:option} ipv4.firewall network_bridge-common :condition: "IPv4 address" :default: "`true`" :shortdesc: "Whether to generate filtering firewall rules for this network" :type: "bool" ``` ```{config:option} ipv4.nat network_bridge-common :condition: "IPv4 address" :default: "`false`(initial value on creation if `ipv4.address` is set to `auto`: `true`)" :shortdesc: "Whether to NAT" :type: "bool" ``` ```{config:option} ipv4.nat.address network_bridge-common :condition: "IPv4 address" :default: "-" :shortdesc: "The source address used for outbound traffic from the bridge" :type: "string" ``` ```{config:option} ipv4.nat.order network_bridge-common :condition: "IPv4 address" :default: "`before`" :shortdesc: "Whether to add the required NAT rules before or after any pre-existing rules" :type: "string" ``` ```{config:option} ipv4.ovn.ranges network_bridge-common :condition: "-" :default: "-" :shortdesc: "Comma-separated list of IPv4 ranges to use for child OVN network routers (FIRST-LAST format)" :type: "string" ``` ```{config:option} ipv4.routes network_bridge-common :condition: "IPv4 address" :default: "-" :shortdesc: "Comma-separated list of additional IPv4 CIDR subnets to route to the bridge" :type: "string" ``` ```{config:option} ipv4.routing network_bridge-common :condition: "IPv4 DHCP" :default: "`true`" :shortdesc: "Whether to route traffic in and out of the bridge" :type: "bool" ``` ```{config:option} ipv6.address network_bridge-common :condition: "standard mode" :default: "- (initial value on creation: `auto`)" :shortdesc: "IPv6 address for the bridge (use `none` to turn off IPv6 or `auto` to generate a new random unused subnet) (CIDR)" :type: "string" ``` ```{config:option} ipv6.dhcp network_bridge-common :condition: "IPv6 DHCP" :default: "`true`" :shortdesc: "Whether to provide additional network configuration over DHCP" :type: "bool" ``` ```{config:option} ipv6.dhcp.expiry network_bridge-common :condition: "IPv6 DHCP" :default: "`1h`" :shortdesc: "When to expire DHCP leases" :type: "string" ``` ```{config:option} ipv6.dhcp.ranges network_bridge-common :condition: "IPv6 stateful DHCP" :default: "all addresses" :shortdesc: "Comma-separated list of IPv6 ranges to use for DHCP (FIRST-LAST format)" :type: "string" ``` ```{config:option} ipv6.dhcp.stateful network_bridge-common :condition: "IPv6 DHCP" :default: "`false`" :shortdesc: "Whether to allocate addresses using DHCP" :type: "bool" ``` ```{config:option} ipv6.firewall network_bridge-common :condition: "IPv6 address" :default: "`true`" :shortdesc: "Whether to generate filtering firewall rules for this network" :type: "bool" ``` ```{config:option} ipv6.nat network_bridge-common :condition: "IPv6 address" :default: "`false` (initial value on creation if `ipv6.address` is set to `auto`: `true`)" :shortdesc: "Whether to NAT" :type: "bool" ``` ```{config:option} ipv6.nat.address network_bridge-common :condition: "IPv6 address" :default: "-" :shortdesc: "The source address used for outbound traffic from the bridge" :type: "string" ``` ```{config:option} ipv6.nat.order network_bridge-common :condition: "IPv6 address" :default: "`before`" :shortdesc: "Whether to add the required NAT rules before or after any pre-existing rules" :type: "string" ``` ```{config:option} ipv6.ovn.ranges network_bridge-common :condition: "-" :default: "-" :shortdesc: "Comma-separated list of IPv6 ranges to use for child OVN network routers (FIRST-LAST format)" :type: "string" ``` ```{config:option} ipv6.routes network_bridge-common :condition: "IPv6 address" :default: "-" :shortdesc: "Comma-separated list of additional IPv6 CIDR subnets to route to the bridge" :type: "string" ``` ```{config:option} ipv6.routing network_bridge-common :condition: "IPv6 address" :default: "`true`" :shortdesc: "Whether to route traffic in and out of the bridge" :type: "bool" ``` ```{config:option} raw.dnsmasq network_bridge-common :condition: "-" :default: "-" :shortdesc: "Additional dnsmasq configuration to append to the configuration file" :type: "string" ``` ```{config:option} security.acls network_bridge-common :condition: "-" :default: "-" :shortdesc: "Comma-separated list of Network ACLs to apply to NICs connected to this network (see {ref}`network-acls-bridge-limitations`)" :type: "string" ``` ```{config:option} security.acls.default.egress.action network_bridge-common :condition: "`security.acls`" :default: "`reject`" :shortdesc: "Action to use for egress traffic that doesn't match any ACL rule" :type: "string" ``` ```{config:option} security.acls.default.egress.logged network_bridge-common :condition: "`security.acls`" :default: "`false`" :shortdesc: "Whether to log egress traffic that doesn't match any ACL rule" :type: "bool" ``` ```{config:option} security.acls.default.ingress.action network_bridge-common :condition: "`security.acls`" :default: "`reject`" :shortdesc: "Action to use for ingress traffic that doesn't match any ACL rule" :type: "string" ``` ```{config:option} security.acls.default.ingress.logged network_bridge-common :condition: "`security.acls`" :default: "`false`" :shortdesc: "Whether to log ingress traffic that doesn't match any ACL rule" :type: "bool" ``` ```{config:option} tunnel.NAME.group network_bridge-common :condition: "`vxlan`" :default: "`239.0.0.1`" :shortdesc: "Multicast address for `vxlan` (used if local and remote aren't set)" :type: "string" ``` ```{config:option} tunnel.NAME.id network_bridge-common :condition: "`vxlan`" :default: "`0`" :shortdesc: "Specific tunnel ID to use for the `vxlan` tunnel" :type: "integer" ``` ```{config:option} tunnel.NAME.interface network_bridge-common :condition: "`vxlan`" :default: "-" :shortdesc: "Specific host interface to use for the tunnel" :type: "string" ``` ```{config:option} tunnel.NAME.local network_bridge-common :condition: "`gre` or `vxlan`" :default: "-" :shortdesc: "Local address for the tunnel (not necessary for multicast `vxlan`)" :type: "string" ``` ```{config:option} tunnel.NAME.port network_bridge-common :condition: "`vxlan`" :default: "`0`" :shortdesc: "Specific port to use for the `vxlan` tunnel" :type: "integer" ``` ```{config:option} tunnel.NAME.protocol network_bridge-common :condition: "standard mode" :default: "-" :shortdesc: "Tunneling protocol: `vxlan` or `gre`" :type: "string" ``` ```{config:option} tunnel.NAME.remote network_bridge-common :condition: "`gre` or `vxlan`" :default: "-" :shortdesc: "Remote address for the tunnel (not necessary for multicast `vxlan`)" :type: "string" ``` ```{config:option} tunnel.NAME.ttl network_bridge-common :condition: "`vxlan`" :default: "`1`" :shortdesc: "Specific TTL to use for multicast routing topologies" :type: "integer" ``` ```{config:option} user.* network_bridge-common :condition: "-" :default: "-" :shortdesc: "User-provided free-form key/value pairs" :type: "string" ``` ```{config:option} target_address network_forward-common :shortdesc: "Default target address for anything not covered through a port definition" :type: "string" ``` ```{config:option} user.* network_forward-common :shortdesc: "User defined key/value configuration" :type: "string" ``` ```{config:option} user.* network_integration-common :shortdesc: "Free form user key/value storage" :type: "string" User keys can be used in search. ``` ```{config:option} ovn.ca_cert network_integration-ovn :scope: "global" :shortdesc: "OVN SSL certificate authority for the inter-connection database" :type: "string" ``` ```{config:option} ovn.client_cert network_integration-ovn :scope: "global" :shortdesc: "OVN SSL client certificate" :type: "string" ``` ```{config:option} ovn.client_key network_integration-ovn :scope: "global" :shortdesc: "OVN SSL client key" :type: "string" ``` ```{config:option} ovn.northbound_connection network_integration-ovn :scope: "global" :shortdesc: "OVN northbound inter-connection connection string" :type: "string" ``` ```{config:option} ovn.southbound_connection network_integration-ovn :scope: "global" :shortdesc: "OVN southbound inter-connection connection string" :type: "string" ``` ```{config:option} ovn.transit.pattern network_integration-ovn :defaultdesc: "`ts-incus-{{ integrationName }}-{{ projectName }}-{{ networkName }}`" :shortdesc: "Template for the transit switch name" :type: "string" Specify a Pongo2 template string that represents the transit switch name. This template gets access to the project name (`projectName`), integration name (`integrationName`), network name (`networkName`) and peer name (`peerName`). ``` ```{config:option} healthcheck network_load_balancer-common :defaultdesc: "`false`" :shortdesc: "Whether to perform checks on the backends" :type: "bool" ``` ```{config:option} healthcheck.failure_count network_load_balancer-common :defaultdesc: "`3`" :shortdesc: "Number of failed tests to consider the backend offline" :type: "integer" ``` ```{config:option} healthcheck.interval network_load_balancer-common :defaultdesc: "`10`" :shortdesc: "Interval in seconds between health checks" :type: "integer" ``` ```{config:option} healthcheck.success_count network_load_balancer-common :defaultdesc: "`3`" :shortdesc: "Number of successful tests to consider the backend online" :type: "integer" ``` ```{config:option} healthcheck.timeout network_load_balancer-common :defaultdesc: "`30`" :shortdesc: "Test timeout" :type: "integer" ``` ```{config:option} user.* network_load_balancer-common :shortdesc: "Free form user key/value storage" :type: "string" User keys can be used in search. ``` ```{config:option} gvrp network_macvlan-common :condition: "-" :default: "`false`" :shortdesc: "Register VLAN using GARP VLAN Registration Protocol" :type: "bool" ``` ```{config:option} mtu network_macvlan-common :condition: "-" :shortdesc: "The MTU of the new interface" :type: "int" ``` ```{config:option} parent network_macvlan-common :condition: "-" :shortdesc: "Parent interface to create macvlan NICs on" :type: "string" ``` ```{config:option} user.* network_macvlan-common :shortdesc: "User-provided free-form key/value pairs" :type: "string" ``` ```{config:option} vlan network_macvlan-common :condition: "-" :shortdesc: "The VLAN ID to attach to" :type: "int" ``` ```{config:option} bridge.external_interfaces network_ovn-common :shortdesc: "Comma-separated list of unconfigured network interfaces to include in the bridge" :type: "string" ``` ```{config:option} bridge.hwaddr network_ovn-common :shortdesc: "MAC address for the virtual bridge interface" :type: "string" ``` ```{config:option} bridge.mtu network_ovn-common :default: "`1442`" :shortdesc: "Bridge MTU (default allows host to host Geneve tunnels)" :type: "integer" ``` ```{config:option} dns.domain network_ovn-common :default: "`incus`" :shortdesc: "Domain to advertise to DHCP clients and use for DNS resolution" :type: "string" ``` ```{config:option} dns.mode network_ovn-common :condition: "-" :default: "`managed`" :shortdesc: "DNS registration mode: none for no DNS record, managed for OVN managed records" :type: "string" ``` ```{config:option} dns.nameservers network_ovn-common :default: "Uplink DNS servers (IPv4 and IPv6 address if no uplink is configured)" :shortdesc: "DNS server IPs to advertise to DHCP clients and via Router Advertisements. Both IPv4 and IPv6 addresses get pushed via DHCP, and the first IPv6 address is also advertised as RDNSS via RA." :type: "string" ``` ```{config:option} dns.search network_ovn-common :shortdesc: "Full comma-separated domain search list, defaulting to `dns.domain` value" :type: "string" ``` ```{config:option} dns.zone.forward network_ovn-common :shortdesc: "Comma-separated list of DNS zone names for forward DNS records" :type: "string" ``` ```{config:option} dns.zone.reverse.ipv4 network_ovn-common :shortdesc: "DNS zone name for IPv4 reverse DNS records" :type: "string" ``` ```{config:option} dns.zone.reverse.ipv6 network_ovn-common :shortdesc: "DNS zone name for IPv6 reverse DNS records" :type: "string" ``` ```{config:option} ipv4.address network_ovn-common :condition: "standard mode" :default: "(initial value on creation: `auto`)" :shortdesc: "IPv4 address for the bridge (use `none` to turn off IPv4 or `auto` to generate a new random unused subnet) (CIDR)" :type: "string" ``` ```{config:option} ipv4.dhcp network_ovn-common :condition: "IPv4 address" :default: "`true`" :shortdesc: "Whether to allocate addresses using DHCP" :type: "bool" ``` ```{config:option} ipv4.dhcp.expiry network_ovn-common :condition: "IPv4 DHCP" :default: "`1h`" :shortdesc: "When to expire DHCP leases" :type: "string" ``` ```{config:option} ipv4.dhcp.gateway network_ovn-common :condition: "IPv4 DHCP" :default: "IPv4 address" :shortdesc: "Address of the gateway for the subnet (use `none` to turn off gateway announcement)" :type: "string" ``` ```{config:option} ipv4.dhcp.ranges network_ovn-common :condition: "IPv4 DHCP" :default: "all addresses" :shortdesc: "Comma-separated list of IP ranges to use for DHCP (FIRST-LAST format)" :type: "string" ``` ```{config:option} ipv4.dhcp.routes network_ovn-common :condition: "IPv4 DHCP" :shortdesc: "Static routes to provide via DHCP option 121, as a comma-separated list of alternating subnets (CIDR) and gateway addresses (same syntax as dnsmasq and OVN)" :type: "string" ``` ```{config:option} ipv4.l3only network_ovn-common :condition: "IPv4 address" :default: "`false`" :shortdesc: "Whether to enable layer 3 only mode." :type: "bool" ``` ```{config:option} ipv4.nat network_ovn-common :condition: "IPv4 address" :default: "`false` initial value on creation if `ipv4.address` is set to `auto: true`)" :shortdesc: "Whether to NAT" :type: "bool" ``` ```{config:option} ipv4.nat.address network_ovn-common :condition: "IPv4 address" :shortdesc: "The source address used for outbound traffic from the network (requires uplink `ovn.ingress_mode=routed`)" :type: "string" ``` ```{config:option} ipv6.address network_ovn-common :condition: "standard mode" :default: "(initial value on creation: `auto`)" :shortdesc: "IPv6 address for the bridge (use `none` to turn off IPv6 or `auto` to generate a new random unused subnet) (CIDR)" :type: "string" ``` ```{config:option} ipv6.dhcp network_ovn-common :condition: "IPv6 address" :default: "`true`" :shortdesc: "Whether to provide additional network configuration over DHCP" :type: "bool" ``` ```{config:option} ipv6.dhcp.stateful network_ovn-common :condition: "IPv6 DHCP" :default: "`false`" :shortdesc: "Whether to allocate addresses using DHCP" :type: "bool" ``` ```{config:option} ipv6.l3only network_ovn-common :condition: "IPv6 DHCP stateful" :default: "`false`" :shortdesc: "Whether to enable layer 3 only mode." :type: "bool" ``` ```{config:option} ipv6.nat network_ovn-common :condition: "IPv6 address" :default: "`false` (initial value on creation if `ipv6.address` is set to `auto: true`)" :shortdesc: "Whether to NAT" :type: "bool" ``` ```{config:option} ipv6.nat.address network_ovn-common :condition: "IPv6 address" :shortdesc: "The source address used for outbound traffic from the network (requires uplink `ovn.ingress_mode=routed`)" :type: "string" ``` ```{config:option} network network_ovn-common :shortdesc: "Uplink network to use for external network access or `none` to keep isolated" :type: "string" ``` ```{config:option} security.acls network_ovn-common :shortdesc: "Comma-separated list of Network ACLs to apply to NICs connected to this network" :type: "string" ``` ```{config:option} security.acls.default.egress.action network_ovn-common :condition: "`security.acls`" :default: "`reject`" :shortdesc: "Action to use for egress traffic that doesn't match any ACL rule" :type: "string" ``` ```{config:option} security.acls.default.egress.logged network_ovn-common :condition: "`security.acls`" :default: "`false`" :shortdesc: "Whether to log egress traffic that doesn't match any ACL rule" :type: "bool" ``` ```{config:option} security.acls.default.ingress.action network_ovn-common :condition: "`security.acls`" :default: "`reject`" :shortdesc: "Action to use for ingress traffic that doesn't match any ACL rule" :type: "string" ``` ```{config:option} security.acls.default.ingress.logged network_ovn-common :condition: "`security.acls`" :default: "`false`" :shortdesc: "Whether to log ingress traffic that doesn't match any ACL rule" :type: "bool" ``` ```{config:option} tunnel.NAME.group network_ovn-common :condition: "`vxlan`" :default: "`239.0.0.1`" :shortdesc: "Multicast address for `vxlan`" :type: "string" ``` ```{config:option} tunnel.NAME.id network_ovn-common :condition: "`vxlan`" :default: "`0`" :shortdesc: "Specific tunnel ID to use for the `vxlan` tunnel" :type: "integer" ``` ```{config:option} tunnel.NAME.interface network_ovn-common :condition: "`vxlan`" :default: "-" :shortdesc: "Specific host interface to use for the tunnel" :type: "string" ``` ```{config:option} tunnel.NAME.local network_ovn-common :condition: "`gre`" :default: "-" :shortdesc: "Local address for the tunnel" :type: "string" ``` ```{config:option} tunnel.NAME.port network_ovn-common :condition: "`vxlan`" :default: "`0`" :shortdesc: "Specific port to use for the `vxlan` tunnel" :type: "integer" ``` ```{config:option} tunnel.NAME.protocol network_ovn-common :condition: "standard mode" :default: "-" :shortdesc: "Tunneling protocol: `vxlan` or `gre`" :type: "string" ``` ```{config:option} tunnel.NAME.remote network_ovn-common :condition: "`gre`" :default: "-" :shortdesc: "Remote address for the tunnel" :type: "string" ``` ```{config:option} tunnel.NAME.ttl network_ovn-common :condition: "`vxlan`" :default: "`1`" :shortdesc: "Specific TTL to use for multicast routing topologies" :type: "integer" ``` ```{config:option} user.* network_ovn-common :shortdesc: "User-provided free-form key/value pairs" :type: "string" ``` ```{config:option} bgp.peers.NAME.address network_physical-bgp :condition: "BGP server" :defaultdesc: "-" :shortdesc: "Peer address (IPv4 or IPv6) for use by `ovn` downstream networks" :type: "string" ``` ```{config:option} bgp.peers.NAME.asn network_physical-bgp :condition: "BGP server" :defaultdesc: "-" :shortdesc: "Peer AS number for use by `ovn` downstream networks" :type: "integer" ``` ```{config:option} bgp.peers.NAME.holdtime network_physical-bgp :condition: "BGP server" :defaultdesc: "`180`" :shortdesc: "Peer session hold time (in seconds; optional)" :type: "integer" ``` ```{config:option} bgp.peers.NAME.password network_physical-bgp :condition: "BGP server" :defaultdesc: "- (no password)" :shortdesc: "Peer session password (optional) for use by `ovn` downstream networks" :type: "string" ``` ```{config:option} gvrp network_physical-common :condition: "-" :defaultdesc: "'false'" :shortdesc: "Register VLAN using GARP VLAN Registration Protocol" :type: "bool" ``` ```{config:option} mtu network_physical-common :condition: "-" :shortdesc: "The MTU of the new interface" :type: "integer" ``` ```{config:option} parent network_physical-common :condition: "-" :shortdesc: "Existing interface to use for network" :type: "string" ``` ```{config:option} vlan network_physical-common :condition: "-" :shortdesc: "The VLAN ID to attach to" :type: "integer" ``` ```{config:option} vlan.tagged network_physical-common :condition: "Parent must be an existing bridge" :shortdesc: "Comma-delimited list of VLAN IDs or VLAN ranges to join for tagged traffic" :type: "integer" ``` ```{config:option} dns.nameservers network_physical-dns :condition: "standard mode" :shortdesc: "List of DNS server IPs on `physical` network" :type: "string" ``` ```{config:option} ipv4.gateway network_physical-ipv4 :condition: "standard mode" :shortdesc: "IPv4 address for the gateway and network (CIDR)" :type: "string" ``` ```{config:option} ipv4.gateway.hwaddr network_physical-ipv4 :shortdesc: "MAC address of the gateway (to avoid discovery)" :type: "string" ``` ```{config:option} ipv4.ovn.ranges network_physical-ipv4 :condition: "-" :shortdesc: "Comma-separated list of IPv4 ranges to use for child OVN network routers (FIRST-LAST format)" :type: "string" ``` ```{config:option} ipv4.routes network_physical-ipv4 :condition: "IPv4 address" :shortdesc: "Comma-separated list of additional IPv4 CIDR subnets that can be used with child OVN networks `ipv4.routes.external` setting" :type: "string" ``` ```{config:option} ipv4.routes.anycast network_physical-ipv4 :condition: "IPv4 address" :defaultdesc: "'false'" :shortdesc: "Allow the overlapping routes to be used on multiple networks/NIC at the same time" :type: "bool" ``` ```{config:option} ipv6.gateway network_physical-ipv6 :condition: "standard mode" :shortdesc: "IPv6 address for the gateway and network (CIDR)" :type: "string" ``` ```{config:option} ipv6.gateway.hwaddr network_physical-ipv6 :shortdesc: "MAC address of the gateway (to avoid discovery)" :type: "string" ``` ```{config:option} ipv6.ovn.ranges network_physical-ipv6 :condition: "-" :shortdesc: "Comma-separated list of IPv6 ranges to use for child OVN network routers (FIRST-LAST format)" :type: "string" ``` ```{config:option} ipv6.routes network_physical-ipv6 :condition: "IPv6 address" :shortdesc: "Comma-separated list of additional IPv6 CIDR subnets that can be used with child OVN networks `ipv6.routes.external` setting" :type: "string" ``` ```{config:option} ipv6.routes.anycast network_physical-ipv6 :condition: "IPv6 address" :defaultdesc: "'false'" :shortdesc: "Allow the overlapping routes to be used on multiple networks/NIC at the same time" :type: "bool" ``` ```{config:option} ovn.ingress_mode network_physical-ovn :condition: "standard mode" :defaultdesc: "`l2proxy`" :shortdesc: "Sets the method how OVN NIC external IPs will be advertised on uplink network: `l2proxy` (proxy ARP/NDP) or `routed`" :type: "string" ``` ```{config:option} mtu network_sriov-common :condition: "-" :shortdesc: "The MTU of the new interface" :type: "integer" ``` ```{config:option} parent network_sriov-common :condition: "-" :shortdesc: "Parent interface to create `sriov` NICs on" :type: "string" ``` ```{config:option} user.* network_sriov-common :condition: "-" :shortdesc: "User-provided free-form key/value pairs" :type: "string" ``` ```{config:option} vlan network_sriov-common :condition: "-" :shortdesc: "The VLAN ID to attach to" :type: "integer" ``` ```{config:option} dns.contact network_zone-common :required: "no" :shortdesc: "Admin contact email for DNS server" :type: "string" ``` ```{config:option} dns.nameservers network_zone-common :required: "no" :shortdesc: "Comma-separated list of DNS server FQDNs (for NS records)" :type: "string set" ``` ```{config:option} network.nat network_zone-common :defaultdesc: "`true`" :required: "no" :shortdesc: "Whether to generate records for NAT-ed subnets" :type: "bool" ``` ```{config:option} peers.NAME.address network_zone-common :required: "no" :shortdesc: "IP address of a DNS server" :type: "string" ``` ```{config:option} peers.NAME.key network_zone-common :required: "no" :shortdesc: "TSIG key for the server" :type: "string" ``` ```{config:option} user.* network_zone-common :required: "no" :shortdesc: "User-provided free-form key/value pairs" :type: "string" ``` ```{config:option} features.images project-features :defaultdesc: "`false`" :initialvaluedesc: "`true`" :shortdesc: "Whether to use a separate set of images for the project" :type: "bool" This setting applies to both images and image aliases. ``` ```{config:option} features.networks project-features :defaultdesc: "`false`" :initialvaluedesc: "`false`" :shortdesc: "Whether to use a separate set of networks for the project" :type: "bool" ``` ```{config:option} features.networks.zones project-features :defaultdesc: "`false`" :initialvaluedesc: "`false`" :shortdesc: "Whether to use a separate set of network zones for the project" :type: "bool" ``` ```{config:option} features.profiles project-features :defaultdesc: "`false`" :initialvaluedesc: "`true`" :shortdesc: "Whether to use a separate set of profiles for the project" :type: "bool" ``` ```{config:option} features.storage.buckets project-features :defaultdesc: "`false`" :initialvaluedesc: "`true`" :shortdesc: "Whether to use a separate set of storage buckets for the project" :type: "bool" ``` ```{config:option} features.storage.volumes project-features :defaultdesc: "`false`" :initialvaluedesc: "`true`" :shortdesc: "Whether to use a separate set of storage volumes for the project" :type: "bool" ``` ```{config:option} limits.containers project-limits :shortdesc: "Maximum number of containers that can be created in the project" :type: "integer" ``` ```{config:option} limits.cpu project-limits :shortdesc: "Maximum number of CPUs to use in the project" :type: "integer" This value is the maximum value for the sum of the individual {config:option}`instance-resource-limits:limits.cpu` configurations set on the instances of the project. ``` ```{config:option} limits.disk project-limits :shortdesc: "Maximum disk space used by the project" :type: "string" This value is the maximum value of the aggregate disk space used by all instance volumes, custom volumes, and images of the project. ``` ```{config:option} limits.disk.pool.POOL_NAME project-limits :shortdesc: "Maximum disk space used by the project on this pool" :type: "string" This value is the maximum value of the aggregate disk space used by all instance volumes, custom volumes, and images of the project on this specific storage pool. ``` ```{config:option} limits.instances project-limits :shortdesc: "Maximum number of instances that can be created in the project" :type: "integer" ``` ```{config:option} limits.memory project-limits :shortdesc: "Usage limit for the host's memory for the project" :type: "string" The value is the maximum value for the sum of the individual {config:option}`instance-resource-limits:limits.memory` configurations set on the instances of the project. ``` ```{config:option} limits.networks project-limits :shortdesc: "Maximum number of networks that the project can have" :type: "integer" ``` ```{config:option} limits.processes project-limits :shortdesc: "Maximum number of processes within the project" :type: "integer" This value is the maximum value for the sum of the individual {config:option}`instance-resource-limits:limits.processes` configurations set on the instances of the project. ``` ```{config:option} limits.virtual-machines project-limits :shortdesc: "Maximum number of VMs that can be created in the project" :type: "integer" ``` ```{config:option} restricted project-restricted :defaultdesc: "`false`" :shortdesc: "Whether to block access to security-sensitive features" :type: "bool" This option must be enabled to allow the `restricted.*` keys to take effect. To temporarily remove the restrictions, you can disable this option instead of clearing the related keys. ``` ```{config:option} restricted.backups project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent creating instance or volume backups" :type: "string" Possible values are `allow` or `block`. ``` ```{config:option} restricted.cluster.groups project-restricted :shortdesc: "Cluster groups that can be targeted" :type: "string" If specified, this option prevents targeting cluster groups other than the provided ones. ``` ```{config:option} restricted.cluster.target project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent targeting of cluster members" :type: "string" Possible values are `allow` or `block`. When set to `allow`, this option allows targeting of cluster members (either directly or via a group) when creating or moving instances. ``` ```{config:option} restricted.containers.interception project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent using system call interception options" :type: "string" Possible values are `allow`, `block`, or `full`. When set to `allow`, interception options that are usually safe are allowed. File system mounting remains blocked. ``` ```{config:option} restricted.containers.lowlevel project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent using low-level container options" :type: "string" Possible values are `allow` or `block`. When set to `allow`, low-level container options like {config:option}`instance-raw:raw.lxc`, {config:option}`instance-raw:raw.idmap`, `volatile.*`, etc. can be used. ``` ```{config:option} restricted.containers.nesting project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent running nested Incus" :type: "string" Possible values are `allow` or `block`. When set to `allow`, {config:option}`instance-security:security.nesting` can be set to `true` for an instance. ``` ```{config:option} restricted.containers.privilege project-restricted :defaultdesc: "`unprivileged`" :shortdesc: "Which settings for privileged containers to prevent" :type: "string" Possible values are `unprivileged`, `isolated`, and `allow`. - When set to `unprivileged`, this option prevents setting {config:option}`instance-security:security.privileged` to `true`. - When set to `isolated`, this option prevents setting {config:option}`instance-security:security.privileged` and {config:option}`instance-security:security.idmap.isolated` to `true`. - When set to `allow`, there is no restriction. ``` ```{config:option} restricted.devices.disk project-restricted :defaultdesc: "`managed`" :shortdesc: "Which disk devices can be used" :type: "string" Possible values are `allow`, `block`, or `managed`. - When set to `block`, this option prevents using all disk devices except the root one. - When set to `managed`, this option allows using disk devices only if `pool=` is set. - When set to `allow`, there is no restriction on which disk devices can be used. ``` ```{config:option} restricted.devices.disk.paths project-restricted :shortdesc: "Which `source` can be used for `disk` devices" :type: "string" If {config:option}`project-restricted:restricted.devices.disk` is set to `allow`, this option controls which `source` can be used for `disk` devices. Specify a comma-separated list of path prefixes that restrict the `source` setting. If this option is left empty, all paths are allowed. ``` ```{config:option} restricted.devices.gpu project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent using devices of type `gpu`" :type: "string" Possible values are `allow` or `block`. ``` ```{config:option} restricted.devices.infiniband project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent using devices of type `infiniband`" :type: "string" Possible values are `allow` or `block`. ``` ```{config:option} restricted.devices.nic project-restricted :defaultdesc: "`managed`" :shortdesc: "Which network devices can be used" :type: "string" Possible values are `allow`, `block`, or `managed`. - When set to `block`, this option prevents using all network devices. - When set to `managed`, this option allows using network devices only if `network=` is set. - When set to `allow`, there is no restriction on which network devices can be used. ``` ```{config:option} restricted.devices.pci project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent using devices of type `pci`" :type: "string" Possible values are `allow` or `block`. ``` ```{config:option} restricted.devices.proxy project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent using devices of type `proxy`" :type: "string" Possible values are `allow` or `block`. ``` ```{config:option} restricted.devices.unix-block project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent using devices of type `unix-block`" :type: "string" Possible values are `allow` or `block`. ``` ```{config:option} restricted.devices.unix-char project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent using devices of type `unix-char`" :type: "string" Possible values are `allow` or `block`. ``` ```{config:option} restricted.devices.unix-hotplug project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent using devices of type `unix-hotplug`" :type: "string" Possible values are `allow` or `block`. ``` ```{config:option} restricted.devices.usb project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent using devices of type `usb`" :type: "string" Possible values are `allow` or `block`. ``` ```{config:option} restricted.idmap.gid project-restricted :shortdesc: "Which host GID ranges are allowed in `raw.idmap`" :type: "string" This option specifies the host GID ranges that are allowed in the instance's {config:option}`instance-raw:raw.idmap` setting. ``` ```{config:option} restricted.idmap.uid project-restricted :shortdesc: "Which host UID ranges are allowed in `raw.idmap`" :type: "string" This option specifies the host UID ranges that are allowed in the instance's {config:option}`instance-raw:raw.idmap` setting. ``` ```{config:option} restricted.images.servers project-restricted :shortdesc: "Which image servers (HTTP host) are allowed for us in this project" :type: "string" Specify a comma-delimited list of image servers domains that are allowed for use in this project. If this option is not set, all image servers are accessible. ``` ```{config:option} restricted.networks.access project-restricted :shortdesc: "Which network names are allowed for use in this project" :type: "string" Specify a comma-delimited list of network names that are allowed for use in this project. If this option is not set, all networks are accessible. Note that this setting depends on the {config:option}`project-restricted:restricted.devices.nic` setting. ``` ```{config:option} restricted.networks.integrations project-restricted :shortdesc: "Which network integrations can be used in this project" :type: "string" Specify a comma-delimited list of network integrations that can be used by networks in this project. ``` ```{config:option} restricted.networks.subnets project-restricted :defaultdesc: "`block`" :shortdesc: "Which network subnets are allocated for use in this project" :type: "string" Specify a comma-delimited list of network subnets from the uplink networks that are allocated for use in this project. Use the form `:`. ``` ```{config:option} restricted.networks.uplinks project-restricted :shortdesc: "Which network names can be used as uplink in this project" :type: "string" Specify a comma-delimited list of network names that can be used as uplink for networks in this project. ``` ```{config:option} restricted.networks.zones project-restricted :defaultdesc: "`block`" :shortdesc: "Which network zones can be used in this project" :type: "string" Specify a comma-delimited list of network zones that can be used (or something under them) in this project. ``` ```{config:option} restricted.snapshots project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent creating instance or volume snapshots" :type: "string" ``` ```{config:option} restricted.storage-pools.access project-restricted :shortdesc: "Which storage pool names are allowed for use in this project" :type: "string" Specify a comma-delimited list of storage pool names that are allowed for use in this project. If this option is not set, all storage pools are accessible. ``` ```{config:option} restricted.virtual-machines.lowlevel project-restricted :defaultdesc: "`block`" :shortdesc: "Whether to prevent using low-level VM options" :type: "string" Possible values are `allow` or `block`. When set to `allow`, low-level VM options like {config:option}`instance-raw:raw.qemu`, `volatile.*`, etc. can be used. ``` ```{config:option} backups.compression_algorithm project-specific :shortdesc: "Compression algorithm to use for backups" :type: "string" Specify which compression algorithm to use for backups in this project. Possible values are `bzip2`, `gzip`, `lz4`, `lzma`, `xz`, `zstd` or `none`. ``` ```{config:option} images.auto_update_cached project-specific :shortdesc: "Whether to automatically update cached images in the project" :type: "bool" ``` ```{config:option} images.auto_update_interval project-specific :shortdesc: "Interval at which to look for updates to cached images" :type: "integer" Specify the interval in hours. To disable looking for updates to cached images, set this option to `0`. ``` ```{config:option} images.compression_algorithm project-specific :shortdesc: "Compression algorithm to use for new images in the project" :type: "string" Possible values are `bzip2`, `gzip`, `lz4`, `lzma`, `xz`, `zstd` or `none`. ``` ```{config:option} images.default_architecture project-specific :shortdesc: "Default architecture to use in a mixed-architecture cluster" :type: "string" ``` ```{config:option} images.remote_cache_expiry project-specific :shortdesc: "When an unused cached remote image is flushed in the project" :type: "integer" Specify the number of days after which the unused cached image expires. ``` ```{config:option} network.hwaddr_pattern project-specific :scope: "global" :shortdesc: "MAC address template" :type: "string" Specify a MAC address template, e.g. `10:66:6a:xx:xx:xx`, to use within the cluster. Every `x` in the template will be replaced by a random character in `0`–`f`. Beware of the birthday paradox! A single `xx` block leads to a 10% collision probability with only 8 addresses; for a double `xx:xx` block, 118 addresses; for a triple `xx:xx:xx` block, 1881; for a quadruple `xx:xx:xx:xx` block, 30084. We provide absolutely no guardrail against that. ``` ```{config:option} user.* project-specific :shortdesc: "User-provided free-form key/value pairs" :type: "string" ``` ```{config:option} acme.agree_tos server-acme :defaultdesc: "`false`" :scope: "global" :shortdesc: "Agree to ACME terms of service" :type: "bool" ``` ```{config:option} acme.ca_url server-acme :defaultdesc: "`https://acme-v02.api.letsencrypt.org/directory`" :scope: "global" :shortdesc: "URL to the directory resource of the ACME service" :type: "string" ``` ```{config:option} acme.challenge server-acme :defaultdesc: "`HTTP-01`" :scope: "global" :shortdesc: "ACME challenge type to use" :type: "string" Possible values are `DNS-01` and `HTTP-01`. ``` ```{config:option} acme.domain server-acme :scope: "global" :shortdesc: "Domain for which the certificate is issued" :type: "string" ``` ```{config:option} acme.email server-acme :scope: "global" :shortdesc: "Email address used for the account registration" :type: "string" ``` ```{config:option} acme.http.port server-acme :defaultdesc: "`:80`" :scope: "global" :shortdesc: "Port and interface for HTTP server (used by HTTP-01)" :type: "string" Set the port and interface to use for HTTP-01 based challenges to listen on ``` ```{config:option} acme.provider server-acme :defaultdesc: "``" :scope: "global" :shortdesc: "Backend provider for the challenge (used by DNS-01)" :type: "string" ``` ```{config:option} acme.provider.environment server-acme :defaultdesc: "``" :scope: "global" :shortdesc: "Environment variables to set during the challenge (used by DNS-01)" :type: "string" ``` ```{config:option} acme.provider.resolvers server-acme :defaultdesc: "``" :scope: "global" :shortdesc: "Comma-separated list of DNS resolvers (used by DNS-01)" :type: "string" DNS resolvers to use for performing (recursive) `CNAME` resolving and apex domain determination during DNS-01 challenge. ``` ```{config:option} cluster.healing_threshold server-cluster :defaultdesc: "`0`" :scope: "global" :shortdesc: "Threshold when to evacuate an offline cluster member" :type: "integer" Specify the number of seconds after which an offline cluster member is to be evacuated. To disable evacuating offline members, set this option to `0`. ``` ```{config:option} cluster.https_address server-cluster :scope: "local" :shortdesc: "Address to use for clustering traffic" :type: "string" See {ref}`cluster-https-address`. ``` ```{config:option} cluster.images_minimal_replica server-cluster :defaultdesc: "`3`" :scope: "global" :shortdesc: "Number of cluster members that replicate an image" :type: "integer" Specify the minimal number of cluster members that keep a copy of a particular image. Set this option to `1` for no replication, or to `-1` to replicate images on all members. ``` ```{config:option} cluster.join_token_expiry server-cluster :defaultdesc: "`3H`" :scope: "global" :shortdesc: "Time after which a cluster join token expires" :type: "string" ``` ```{config:option} cluster.max_standby server-cluster :defaultdesc: "`2`" :scope: "global" :shortdesc: "Number of database stand-by members" :type: "integer" Specify the maximum number of cluster members that are assigned the database stand-by role. This must be a number between `0` and `5`. ``` ```{config:option} cluster.max_voters server-cluster :defaultdesc: "`3`" :scope: "global" :shortdesc: "Number of database voter members" :type: "integer" Specify the maximum number of cluster members that are assigned the database voter role. This must be an odd number >= `3`. ``` ```{config:option} cluster.offline_threshold server-cluster :defaultdesc: "`20`" :scope: "global" :shortdesc: "Threshold when an unresponsive member is considered offline" :type: "integer" Specify the number of seconds after which an unresponsive member is considered offline. ``` ```{config:option} cluster.rebalance.batch server-cluster :defaultdesc: "`1`" :scope: "global" :shortdesc: "Maximum number of instances to move during one re-balancing run" :type: "integer" ``` ```{config:option} cluster.rebalance.cooldown server-cluster :defaultdesc: "`6H`" :scope: "global" :shortdesc: "Amount of time during which an instance will not be moved again" :type: "string" ``` ```{config:option} cluster.rebalance.interval server-cluster :defaultdesc: "`0`" :scope: "global" :shortdesc: "How often (in minutes) to consider re-balancing things. 0 to disable (default)" :type: "integer" ``` ```{config:option} cluster.rebalance.threshold server-cluster :defaultdesc: "`20`" :scope: "global" :shortdesc: "Percentage load difference between most and least busy server needed to trigger a migration" :type: "integer" ``` ```{config:option} core.bgp_address server-core :scope: "local" :shortdesc: "Address to bind the BGP server to" :type: "string" See {ref}`network-bgp`. ``` ```{config:option} core.bgp_asn server-core :scope: "global" :shortdesc: "BGP Autonomous System Number for the local server" :type: "string" ``` ```{config:option} core.bgp_routerid server-core :scope: "local" :shortdesc: "A unique identifier for the BGP server" :type: "string" The identifier must be formatted as an IPv4 address. ``` ```{config:option} core.debug_address server-core :scope: "local" :shortdesc: "Address to bind the `pprof` debug server to (HTTP)" :type: "string" ``` ```{config:option} core.dns_address server-core :scope: "local" :shortdesc: "Address to bind the authoritative DNS server to" :type: "string" See {ref}`network-dns-server`. ``` ```{config:option} core.https_address server-core :scope: "local" :shortdesc: "Address to bind for the remote API (HTTPS)" :type: "string" See {ref}`server-expose`. ``` ```{config:option} core.https_allowed_credentials server-core :defaultdesc: "`false`" :scope: "global" :shortdesc: "Whether to set `Access-Control-Allow-Credentials`" :type: "bool" If enabled, the `Access-Control-Allow-Credentials` HTTP header value is set to `true`. ``` ```{config:option} core.https_allowed_headers server-core :scope: "global" :shortdesc: "`Access-Control-Allow-Headers` HTTP header value" :type: "string" ``` ```{config:option} core.https_allowed_methods server-core :scope: "global" :shortdesc: "`Access-Control-Allow-Methods` HTTP header value" :type: "string" ``` ```{config:option} core.https_allowed_origin server-core :scope: "global" :shortdesc: "`Access-Control-Allow-Origin` HTTP header value" :type: "string" ``` ```{config:option} core.https_trusted_proxy server-core :scope: "global" :shortdesc: "Trusted servers to provide the client's address" :type: "string" Specify a comma-separated list of IP addresses of trusted servers that provide the client's address through the proxy connection header. ``` ```{config:option} core.metrics_address server-core :scope: "local" :shortdesc: "Address to bind the metrics server to (HTTPS)" :type: "string" See {ref}`metrics`. ``` ```{config:option} core.metrics_authentication server-core :defaultdesc: "`true`" :scope: "global" :shortdesc: "Whether to enforce authentication on the metrics endpoint" :type: "bool" ``` ```{config:option} core.proxy_http server-core :scope: "global" :shortdesc: "HTTP proxy to use" :type: "string" If this option is not specified, the daemon falls back to the `HTTP_PROXY` environment variable (if set). ``` ```{config:option} core.proxy_https server-core :scope: "global" :shortdesc: "HTTPS proxy to use" :type: "string" If this option is not specified, the daemon falls back to the `HTTPS_PROXY` environment variable (if set). ``` ```{config:option} core.proxy_ignore_hosts server-core :scope: "global" :shortdesc: "Hosts that don't need the proxy" :type: "string" Specify this option in a similar format to `NO_PROXY` (for example, `1.2.3.4,1.2.3.5`) If this option is not specified, the daemon falls back to the `NO_PROXY` environment variable (if set). ``` ```{config:option} core.remote_token_expiry server-core :defaultdesc: "no expiry" :scope: "global" :shortdesc: "Time after which a remote add token expires" :type: "string" ``` ```{config:option} core.shutdown_action server-core :defaultdesc: "`shutdown`" :scope: "global" :shortdesc: "Action to perform on server shutdown" :type: "string" Specify the action to take when the daemon is being shut down. Supported values are `shutdown` (stop all instances) and `evacuate` (attempt to evacuate the clustered server). ``` ```{config:option} core.shutdown_timeout server-core :defaultdesc: "`5`" :scope: "global" :shortdesc: "How long to wait before shutdown" :type: "integer" Specify the number of minutes to wait for running operations to complete before the daemon shuts down. ``` ```{config:option} core.storage_buckets_address server-core :scope: "local" :shortdesc: "Address to bind the storage object server to (HTTPS)" :type: "string" See {ref}`howto-storage-buckets`. ``` ```{config:option} core.syslog_socket server-core :defaultdesc: "`false`" :scope: "local" :shortdesc: "Whether to enable the syslog unixgram socket listener" :type: "bool" Set this option to `true` to enable the syslog unixgram socket to receive log messages from external processes. ``` ```{config:option} core.trust_ca_certificates server-core :defaultdesc: "`false`" :scope: "global" :shortdesc: "Whether to automatically trust clients signed by the CA" :type: "bool" ``` ```{config:option} images.auto_update_cached server-images :defaultdesc: "`true`" :scope: "global" :shortdesc: "Whether to automatically update cached images" :type: "bool" ``` ```{config:option} images.auto_update_interval server-images :defaultdesc: "`6`" :scope: "global" :shortdesc: "Interval at which to look for updates to cached images" :type: "integer" Specify the interval in hours. To disable looking for updates to cached images, set this option to `0`. ``` ```{config:option} images.compression_algorithm server-images :defaultdesc: "`gzip`" :scope: "global" :shortdesc: "Compression algorithm to use for new images" :type: "string" Possible values are `bzip2`, `gzip`, `lz4`, `lzma`, `xz`, `zstd` or `none`. ``` ```{config:option} images.default_architecture server-images :shortdesc: "Default architecture to use in a mixed-architecture cluster" :type: "string" ``` ```{config:option} images.remote_cache_expiry server-images :defaultdesc: "`10`" :scope: "global" :shortdesc: "When an unused cached remote image is flushed" :type: "integer" Specify the number of days after which the unused cached image expires. ``` ```{config:option} logging.NAME.lifecycle.projects server-logging :scope: "global" :shortdesc: "Comma separate list of projects, empty means all" :type: "string" ``` ```{config:option} logging.NAME.lifecycle.types server-logging :scope: "global" :shortdesc: "E.g., `instance`, comma separate, empty means all" :type: "string" ``` ```{config:option} logging.NAME.logging.level server-logging :defaultdesc: "`info`" :scope: "global" :shortdesc: "Minimum log level to send to the logger" :type: "string" ``` ```{config:option} logging.NAME.target.address server-logging :scope: "global" :shortdesc: "Address of the logger" :type: "string" Specify the protocol, name or IP and port. For example `tcp://syslog01.int.example.net:514`. ``` ```{config:option} logging.NAME.target.ca_cert server-logging :scope: "global" :shortdesc: "CA certificate for the server" :type: "string" ``` ```{config:option} logging.NAME.target.facility server-logging :scope: "global" :shortdesc: "The syslog facility defines the category of the log message" :type: "string" ``` ```{config:option} logging.NAME.target.instance server-logging :defaultdesc: "Local server host name or cluster member name" :scope: "global" :shortdesc: "Name to use as the instance field in Loki events." :type: "string" This allows replacing the default instance value (server host name) by a more relevant value like a cluster identifier. ``` ```{config:option} logging.NAME.target.labels server-logging :scope: "global" :shortdesc: "Labels for a Loki log entry" :type: "string" Specify a comma-separated list of values that should be used as labels for a Loki log entry. ``` ```{config:option} logging.NAME.target.password server-logging :scope: "global" :shortdesc: "Password used for authentication" :type: "string" ``` ```{config:option} logging.NAME.target.retry server-logging :scope: "global" :shortdesc: "number of delivery retries, default 3" :type: "integer" ``` ```{config:option} logging.NAME.target.type server-logging :scope: "global" :shortdesc: "The type of the logger. One of `loki`, `syslog` or `webhook`." :type: "string" ``` ```{config:option} logging.NAME.target.username server-logging :scope: "global" :shortdesc: "User name used for authentication" :type: "string" ``` ```{config:option} logging.NAME.types server-logging :defaultdesc: "`lifecycle,logging`" :scope: "global" :shortdesc: "Events to send to the logger" :type: "string" Specify a comma-separated list of events to send to the logger. The events can be any combination of `lifecycle`, `logging`, and `network-acl`. ``` ```{config:option} loki.api.ca_cert server-loki :scope: "global" :shortdesc: "CA certificate for the Loki server" :type: "string" ``` ```{config:option} loki.api.url server-loki :scope: "global" :shortdesc: "URL to the Loki server" :type: "string" Specify the protocol, name or IP and port. For example `https://loki.example.com:3100`. Incus will automatically add the `/loki/api/v1/push` suffix so there's no need to add it here. ``` ```{config:option} loki.auth.password server-loki :scope: "global" :shortdesc: "Password used for Loki authentication" :type: "string" ``` ```{config:option} loki.auth.username server-loki :scope: "global" :shortdesc: "User name used for Loki authentication" :type: "string" ``` ```{config:option} loki.instance server-loki :defaultdesc: "Local server host name or cluster member name" :scope: "global" :shortdesc: "Name to use as the instance field in Loki events." :type: "string" This allows replacing the default instance value (server host name) by a more relevant value like a cluster identifier. ``` ```{config:option} loki.labels server-loki :scope: "global" :shortdesc: "Labels for a Loki log entry" :type: "string" Specify a comma-separated list of values that should be used as labels for a Loki log entry. ``` ```{config:option} loki.loglevel server-loki :defaultdesc: "`info`" :scope: "global" :shortdesc: "Minimum log level to send to the Loki server" :type: "string" ``` ```{config:option} loki.types server-loki :defaultdesc: "`lifecycle,logging`" :scope: "global" :shortdesc: "Events to send to the Loki server" :type: "string" Specify a comma-separated list of events to send to the Loki server. The events can be any combination of `lifecycle`, `logging`, and `network-acl`. ``` ```{config:option} authorization.scriptlet server-miscellaneous :scope: "global" :shortdesc: "Authorization scriptlet" :type: "string" When using scriptlet-based authorization, this option stores the scriptlet. ``` ```{config:option} backups.compression_algorithm server-miscellaneous :defaultdesc: "`gzip`" :scope: "global" :shortdesc: "Compression algorithm to use for backups" :type: "string" Possible values are `bzip2`, `gzip`, `lz4`, `lzma`, `xz`, `zstd` or `none`. ``` ```{config:option} instances.lxcfs.per_instance server-miscellaneous :defaultdesc: "`false`" :scope: "global" :shortdesc: "Whether to run LXCFS on a per-instance basis" :type: "bool" LXCFS is used to provide overlays for common `/proc` and `/sys` files which reflect the resource limits applied to the container. It normally operates through a single file system mount on the host which is then shared by all containers. This is very efficient but comes with the downside that a crash of LXCFS will break all containers. With this option, it's now possible to run a LXCFS instance per container instead, using more system resources but reducing the impact of a crash. ``` ```{config:option} instances.nic.host_name server-miscellaneous :defaultdesc: "`random`" :scope: "global" :shortdesc: "How to set the host name for a NIC" :type: "string" Possible values are `random` and `mac`. If set to `random`, use the random host interface name as the host name. If set to `mac`, generate a host name in the form `inc` (MAC without leading two digits). ``` ```{config:option} instances.placement.scriptlet server-miscellaneous :scope: "global" :shortdesc: "Instance placement scriptlet for automatic instance placement" :type: "string" When using custom automatic instance placement logic, this option stores the scriptlet. See {ref}`clustering-instance-placement-scriptlet` for more information. ``` ```{config:option} network.ovn.ca_cert server-miscellaneous :defaultdesc: "Content of `/etc/ovn/ovn-central.crt` if present" :scope: "global" :shortdesc: "OVN SSL certificate authority" :type: "string" ``` ```{config:option} network.ovn.client_cert server-miscellaneous :defaultdesc: "Content of `/etc/ovn/cert_host` if present" :scope: "global" :shortdesc: "OVN SSL client certificate" :type: "string" ``` ```{config:option} network.ovn.client_key server-miscellaneous :defaultdesc: "Content of `/etc/ovn/key_host` if present" :scope: "global" :shortdesc: "OVN SSL client key" :type: "string" ``` ```{config:option} network.ovn.integration_bridge server-miscellaneous :defaultdesc: "`br-int`" :scope: "global" :shortdesc: "OVS integration bridge to use for OVN networks" :type: "string" ``` ```{config:option} network.ovn.northbound_connection server-miscellaneous :defaultdesc: "`unix:/run/ovn/ovnnb_db.sock`" :scope: "global" :shortdesc: "OVN northbound database connection string" :type: "string" ``` ```{config:option} network.ovs.connection server-miscellaneous :defaultdesc: "`unix:/run/openvswitch/db.sock`" :scope: "global" :shortdesc: "OVS socket path" :type: "string" ``` ```{config:option} storage.backups_volume server-miscellaneous :scope: "local" :shortdesc: "Volume to use to store backup tarballs" :type: "string" Specify the volume using the syntax `POOL/VOLUME`. ``` ```{config:option} storage.images_volume server-miscellaneous :scope: "local" :shortdesc: "Volume to use to store the image tarballs" :type: "string" Specify the volume using the syntax `POOL/VOLUME`. ``` ```{config:option} storage.linstor.ca_cert server-miscellaneous :scope: "global" :shortdesc: "LINSTOR SSL certificate authority" :type: "string" ``` ```{config:option} storage.linstor.client_cert server-miscellaneous :scope: "global" :shortdesc: "LINSTOR SSL client certificate" :type: "string" ``` ```{config:option} storage.linstor.client_key server-miscellaneous :scope: "global" :shortdesc: "LINSTOR SSL client key" :type: "string" ``` ```{config:option} storage.linstor.controller_connection server-miscellaneous :scope: "global" :shortdesc: "LINSTOR controller connection string" :type: "string" ``` ```{config:option} storage.linstor.satellite.name server-miscellaneous :scope: "global" :shortdesc: "LINSTOR satellite node name override" :type: "string" Set this option to the name of the local LINSTOR satellite node, should it be different from the Incus server name. ``` ```{config:option} storage.logs_volume server-miscellaneous :scope: "local" :shortdesc: "Volume to use to store instance log directories" :type: "string" Specify the volume using the syntax `POOL/VOLUME`. ``` ```{config:option} network.hwaddr_pattern server-network :defaultdesc: "`10:66:6a:xx:xx:xx`" :scope: "global" :shortdesc: "MAC address template" :type: "string" Specify a MAC address template, e.g. `10:66:6a:xx:xx:xx`, to use within the cluster. Every `x` in the template will be replaced by a random character in `0`–`f`. Beware of the birthday paradox! A single `xx` block leads to a 10% collision probability with only 8 addresses; for a double `xx:xx` block, 118 addresses; for a triple `xx:xx:xx` block, 1881; for a quadruple `xx:xx:xx:xx` block, 30084. We provide absolutely no guardrail against that. ``` ```{config:option} oidc.audience server-oidc :scope: "global" :shortdesc: "Expected audience value for the application" :type: "string" This value is required by some providers. ``` ```{config:option} oidc.claim server-oidc :scope: "global" :shortdesc: "OpenID Connect claim to use as the username" :type: "string" Note that the claim must be contained in the access token. ``` ```{config:option} oidc.client.id server-oidc :scope: "global" :shortdesc: "OpenID Connect client ID" :type: "string" ``` ```{config:option} oidc.issuer server-oidc :scope: "global" :shortdesc: "OpenID Connect Discovery URL for the provider" :type: "string" ``` ```{config:option} oidc.scopes server-oidc :scope: "global" :shortdesc: "Comma separated list of OpenID Connect scopes" :type: "string" ``` ```{config:option} openfga.api.token server-openfga :scope: "global" :shortdesc: "API token of the OpenFGA server" :type: "string" ``` ```{config:option} openfga.api.url server-openfga :scope: "global" :shortdesc: "URL of the OpenFGA server" :type: "string" ``` ```{config:option} openfga.store.id server-openfga :scope: "global" :shortdesc: "ID of the OpenFGA permission store" :type: "string" ``` ```{config:option} btrfs.mount_options storage_btrfs-common :default: "`user_subvol_rm_allowed`" :scope: "global" :shortdesc: "Mount options for block devices" :type: "string" ``` ```{config:option} size storage_btrfs-common :default: "auto (20% of free disk space, >= 5 GiB and <= 30 GiB)" :scope: "local" :shortdesc: "Size of the storage pool when creating loop-based pools (in bytes, suffixes supported, can be increased to grow storage pool)" :type: "string" ``` ```{config:option} source storage_btrfs-common :default: "-" :scope: "local" :shortdesc: "Path to an existing block device, loop file or Btrfs subvolume" :type: "string" ``` ```{config:option} source.wipe storage_btrfs-common :default: "`false`" :scope: "local" :shortdesc: "Wipe the block device specified in `source` prior to creating the storage pool" :type: "bool" ``` ```{config:option} size storage_bucket_btrfs-common :condition: "appropriate driver" :default: "same as `volume.size`" :shortdesc: "Size/quota of the storage bucket" :type: "string" ``` ```{config:option} size storage_bucket_cephobject-common :default: "-" :shortdesc: "Quota of the storage bucket" :type: "string" ``` ```{config:option} size storage_bucket_lvm-common :condition: "appropriate driver" :default: "same as `volume.size`" :shortdesc: "Size/quota of the storage bucket" :type: "string" ``` ```{config:option} size storage_bucket_zfs-common :condition: "appropriate driver" :default: "same as `volume.size`" :shortdesc: "Size/quota of the storage bucket" :type: "string" ``` ```{config:option} ceph.cluster_name storage_ceph-common :default: "`ceph`" :scope: "global" :shortdesc: "Name of the Ceph cluster in which to create new storage pools" :type: "string" ``` ```{config:option} ceph.osd.data_pool_name storage_ceph-common :default: "-" :scope: "global" :shortdesc: "Name of the OSD data pool" :type: "string" ``` ```{config:option} ceph.osd.force_reuse storage_ceph-common :default: "-" :scope: "global" :shortdesc: "Deprecated, should not be used." :type: "bool" ``` ```{config:option} ceph.osd.pg_name storage_ceph-common :default: "`32`" :scope: "global" :shortdesc: "Number of placement groups for the OSD storage pool" :type: "string" ``` ```{config:option} ceph.osd.pool_name storage_ceph-common :default: "name of the pool" :scope: "global" :shortdesc: "Name of the OSD storage pool" :type: "string" ``` ```{config:option} ceph.rbd.clone_copy storage_ceph-common :default: "`true`" :scope: "global" :shortdesc: "Whether to use RBD lightweight clones rather than full dataset copies" :type: "bool" ``` ```{config:option} ceph.rbd.du storage_ceph-common :default: "`true`" :scope: "global" :shortdesc: "Whether to use RBD `du` to obtain disk usage data for stopped instances" :type: "bool" ``` ```{config:option} ceph.rbd.features storage_ceph-common :default: "`layering`" :scope: "global" :shortdesc: "Comma-separated list of RBD features to enable on the volumes" :type: "string" ``` ```{config:option} ceph.user.name storage_ceph-common :default: "`admin`" :scope: "global" :shortdesc: "The Ceph user to use when creating storage pools and volumes" :type: "string" ``` ```{config:option} source storage_ceph-common :default: "-" :scope: "local" :shortdesc: "Existing OSD storage pool to use" :type: "string" ``` ```{config:option} volatile.pool.pristine storage_ceph-common :default: "`true`" :scope: "global" :shortdesc: "Whether the pool was empty on creation time" :type: "string" ``` ```{config:option} cephfs.cluster_name storage_cephfs-common :default: "`ceph`" :scope: "global" :shortdesc: "Name of the Ceph cluster that contains the CephFS file system" :type: "string" ``` ```{config:option} cephfs.create_missing storage_cephfs-common :default: "`false`" :scope: "global" :shortdesc: "Create the file system and the missing data and metadata OSD pools" :type: "bool" ``` ```{config:option} cephfs.data_pool storage_cephfs-common :default: "-" :scope: "global" :shortdesc: "Data OSD pool name to create for the file system" :type: "string" ``` ```{config:option} cephfs.fscache storage_cephfs-common :default: "`false`" :scope: "global" :shortdesc: "Enable use of kernel `fscache` and `cachefilesd`" :type: "bool" ``` ```{config:option} cephfs.meta_pool storage_cephfs-common :default: "-" :scope: "global" :shortdesc: "Metadata OSD pool name to create for the file system" :type: "string" ``` ```{config:option} cephfs.osd_pg_num storage_cephfs-common :default: "-" :scope: "global" :shortdesc: "OSD pool `pg_num` to use when creating missing OSD pools" :type: "string" ``` ```{config:option} cephfs.path storage_cephfs-common :default: "`/`" :scope: "global" :shortdesc: "The base path for the CephFS mount" :type: "string" ``` ```{config:option} cephfs.user.name storage_cephfs-common :default: "`admin`" :scope: "global" :shortdesc: "The Ceph user to use" :type: "string" ``` ```{config:option} source storage_cephfs-common :default: "-" :scope: "local" :shortdesc: "Existing CephFS file system or file system path to use" :type: "string" ``` ```{config:option} volatile.pool.pristine storage_cephfs-common :default: "`true`" :scope: "global" :shortdesc: "Whether the CephFS file system was empty on creation time" :type: "string" ``` ```{config:option} cephobject.bucket_name_prefix storage_cephobject-common :default: "-" :scope: "global" :shortdesc: "Prefix to add to bucket names in Ceph" :type: "string" ``` ```{config:option} cephobject.cluster_name storage_cephobject-common :default: "`ceph`" :scope: "global" :shortdesc: "The Ceph cluster to use" :type: "string" ``` ```{config:option} cephobject.radosgw.endpoint storage_cephobject-common :default: "-" :scope: "global" :shortdesc: "URL of the `radosgw` gateway process" :type: "string" ``` ```{config:option} cephobject.radosgw.endpoint_cert_file storage_cephobject-common :default: "-" :scope: "global" :shortdesc: "Path to the file containing the TLS client certificate to use for endpoint communication" :type: "string" ``` ```{config:option} cephobject.user.name storage_cephobject-common :default: "`admin`" :scope: "global" :shortdesc: "The Ceph user to use" :type: "string" ``` ```{config:option} volatile.pool.pristine storage_cephobject-common :default: "`true`" :scope: "global" :shortdesc: "Whether the `radosgw` `incus-admin` user existed at creation time" :type: "string" ``` ```{config:option} rsync.bwlimit storage_dir-common :default: "`0` (no limit)" :scope: "global" :shortdesc: "The upper limit to be placed on the socket I/O when `rsync` must be used to transfer storage entities" :type: "string" ``` ```{config:option} rsync.compression storage_dir-common :default: "`true`" :scope: "global" :shortdesc: "Whether to use compression while migrating storage pools" :type: "bool" ``` ```{config:option} source storage_dir-common :default: "-" :scope: "local" :shortdesc: "Path to an existing directory" :type: "string" ``` ```{config:option} drbd.auto_add_quorum_tiebreaker storage_linstor-common :default: "`true`" :scope: "global" :shortdesc: "Whether to allow LINSTOR to automatically create diskless resources to act as quorum tiebreakers if needed (applied to the resource group)" :type: "bool" ``` ```{config:option} drbd.auto_diskful storage_linstor-common :default: "-" :scope: "global" :shortdesc: "A duration string describing the time after which a primary diskless resource can be converted to diskful if storage is available on the node (applied to the resource group)" :type: "string" ``` ```{config:option} drbd.on_no_quorum storage_linstor-common :default: "-" :scope: "global" :shortdesc: "The DRBD policy to use on resources when quorum is lost (applied to the resource group)" :type: "string" ``` ```{config:option} linstor.resource_group.name storage_linstor-common :default: "`incus`" :scope: "global" :shortdesc: "Name of the LINSTOR resource group that will be used for the storage pool" :type: "string" ``` ```{config:option} linstor.resource_group.place_count storage_linstor-common :default: "`2`" :scope: "global" :shortdesc: "Number of diskful replicas that should be created for resources in the resource group. Increasing the value of this option on a pool that already has volumes will result in LINSTOR creating new diskful replicas for all existing resources to match the new value" :type: "int" ``` ```{config:option} linstor.resource_group.storage_pool storage_linstor-common :default: "-" :scope: "global" :shortdesc: "The storage pool name in which resources should be placed on satellite nodes" :type: "string" ``` ```{config:option} linstor.volume.prefix storage_linstor-common :default: "`incus-volume-`" :scope: "global" :shortdesc: "The prefix to use for the internal names of LINSTOR-managed volumes. Cannot be updated after the storage pool is created" :type: "string" ``` ```{config:option} source storage_linstor-common :default: "`incus`" :scope: "global" :shortdesc: "LINSTOR storage pool name. Alias for `linstor.resource_group.name`. Use either either one or the other or make sure they have the same value." :type: "string" ``` ```{config:option} volatile.pool.pristine storage_linstor-common :default: "`true`" :scope: "global" :shortdesc: "Whether the pool was empty on creation time" :type: "string" ``` ```{config:option} block.type storage_lvm-common :condition: "block-based volume" :default: "same as `volume.block.type`" :shortdesc: "Type of the block volume" ``` ```{config:option} lvm.metadata_size storage_lvm-common :default: "`0` (auto)" :scope: "global" :shortdesc: "The size of the metadata space for the physical volume." :type: "string" ``` ```{config:option} lvm.thinpool_metadata_size storage_lvm-common :default: "`0` (auto)" :scope: "global" :shortdesc: "The size of the thin pool metadata volume (the default is to let LVM calculate an appropriate size). Not usable with `lvmcluster`." :type: "string" ``` ```{config:option} lvm.thinpool_name storage_lvm-common :default: "`IncusThinPool`" :scope: "local" :shortdesc: "Thin pool where volumes are created. Not usable with `lvmcluster`." :type: "string" ``` ```{config:option} lvm.use_thinpool storage_lvm-common :default: "`true`" :scope: "global" :shortdesc: "Whether the storage pool uses a thin pool for logical volumes. Not usable with `lvmcluster`." :type: "bool" ``` ```{config:option} lvm.vg.force_reuse storage_lvm-common :default: "`false`" :scope: "local" :shortdesc: "Force using an existing non-empty volume group. Not usable with `lvmcluster`." :type: "bool" ``` ```{config:option} lvm.vg_name storage_lvm-common :default: "name of the pool" :scope: "local" :shortdesc: "Name of the volume group to create." :type: "string" ``` ```{config:option} size storage_lvm-common :default: "auto (20% of free disk space, >= 5 GiB and <= 30 GiB) for `lvm`." :scope: "local" :shortdesc: "Storage pool size (in bytes, suffixes supported, can be increased to grow storage pool). `lvmcluster` pools: cannot set at creation, but can be updated." :type: "string" ``` ```{config:option} source storage_lvm-common :default: "-" :scope: "local" :shortdesc: "Path to an existing block device, loop file or LVM volume group." :type: "string" ``` ```{config:option} source.wipe storage_lvm-common :default: "`false`" :scope: "local" :shortdesc: "Wipe the block device specified in `source` prior to creating the storage pool." :type: "bool" ``` ```{config:option} source storage_truenas-common :default: "-" :scope: "local" :shortdesc: "ZFS dataset to use on the remote TrueNAS host. Format: `[:][/][/]`. If `host` is omitted here, it must be set via `truenas.host`." :type: "string" ``` ```{config:option} truenas.allow_insecure storage_truenas-common :default: "`false`" :scope: "global" :shortdesc: "If set to `true`, allows insecure (non-TLS) connections to the TrueNAS API." :type: "bool" ``` ```{config:option} truenas.api_key storage_truenas-common :default: "-" :scope: "global" :shortdesc: "API key used to authenticate with the TrueNAS host." :type: "string" ``` ```{config:option} truenas.clone_copy storage_truenas-common :default: "`true`" :scope: "global" :shortdesc: "Whether to use lightweight clones rather than full {spellexception}`dataset` copies." :type: "bool" ``` ```{config:option} truenas.config storage_truenas-common :default: "-" :scope: "global" :shortdesc: "Path to a configuration file for the TrueNAS client tool." :type: "string" ``` ```{config:option} truenas.dataset storage_truenas-common :default: "-" :scope: "global" :shortdesc: "Remote dataset name. Typically inferred from `source`, but can be overridden." :type: "string" ``` ```{config:option} truenas.force_reuse storage_truenas-common :default: "`false`" :scope: "global" :shortdesc: "Allow to use an existing non-empty pool." :type: "bool" ``` ```{config:option} truenas.host storage_truenas-common :default: "-" :scope: "global" :shortdesc: "Hostname or IP address of the remote TrueNAS system. Optional if included in the `source`, or a configuration is used." :type: "string" ``` ```{config:option} truenas.initiator storage_truenas-common :default: "-" :scope: "global" :shortdesc: "iSCSI initiator name used during block volume attachment." :type: "string" ``` ```{config:option} truenas.portal storage_truenas-common :default: "-" :scope: "global" :shortdesc: "iSCSI portal address to use for block volume connections." :type: "string" ``` ```{config:option} initial.gid storage_volume_btrfs-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.gid` or `0`" :shortdesc: "GID of the volume owner in the instance" :type: "int" ``` ```{config:option} initial.mode storage_volume_btrfs-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.mode` or `711`" :shortdesc: "Mode of the volume in the instance" :type: "int" ``` ```{config:option} initial.uid storage_volume_btrfs-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.uid` or `0`" :shortdesc: "UID of the volume owner in the instance" :type: "int" ``` ```{config:option} security.shared storage_volume_btrfs-common :condition: "custom block volume" :default: "same as `volume.security.shared` or `false`" :shortdesc: "Enable sharing the volume across multiple instances" :type: "bool" ``` ```{config:option} security.shifted storage_volume_btrfs-common :condition: "custom volume" :default: "same as `volume.security.shifted` or `false`" :shortdesc: "{{enable_ID_shifting}}" :type: "bool" ``` ```{config:option} security.unmapped storage_volume_btrfs-common :condition: "custom volume" :default: "same as `volume.security.unmapped` or `false`" :shortdesc: "Disable ID mapping for the volume" :type: "bool" ``` ```{config:option} size storage_volume_btrfs-common :condition: "appropriate driver" :default: "same as `volume.size`" :shortdesc: "Size/quota of the storage volume" :type: "string" ``` ```{config:option} snapshots.expiry storage_volume_btrfs-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.expiry.manual storage_volume_btrfs-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry.manual`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.pattern storage_volume_btrfs-common :condition: "custom volume" :default: "same as `volume.snapshot.pattern` or `snap%d`" :shortdesc: "{{snapshot_pattern_format}} [^*]" :type: "string" ``` ```{config:option} snapshots.schedule storage_volume_btrfs-common :condition: "custom volume" :default: "same as `volume.snapshot.schedule`" :shortdesc: "{{snapshot_schedule_format}}" :type: "string" ``` ```{config:option} block.filesystem storage_volume_ceph-common :condition: "block-based volume with content type `filesystem`" :default: "same as `volume.block.filesystem`" :shortdesc: "{{block_filesystem}}" :type: "string" ``` ```{config:option} block.mount_options storage_volume_ceph-common :condition: "block-based volume with content type `filesystem`" :default: "same as `volume.block.mount_options`" :shortdesc: "Mount options for block-backed file system volumes" :type: "string" ``` ```{config:option} initial.gid storage_volume_ceph-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.gid` or `0`" :shortdesc: "GID of the volume owner in the instance" :type: "int" ``` ```{config:option} initial.mode storage_volume_ceph-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.mode` or `711`" :shortdesc: "Mode of the volume in the instance" :type: "int" ``` ```{config:option} initial.uid storage_volume_ceph-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.uid` or `0`" :shortdesc: "UID of the volume owner in the instance" :type: "int" ``` ```{config:option} security.shared storage_volume_ceph-common :condition: "custom block volume" :default: "same as `volume.security.shared` or `false`" :shortdesc: "Enable sharing the volume across multiple instances" :type: "bool" ``` ```{config:option} security.shifted storage_volume_ceph-common :condition: "custom volume" :default: "same as `volume.security.shifted` or `false`" :shortdesc: "{{enable_ID_shifting}}" :type: "bool" ``` ```{config:option} security.unmapped storage_volume_ceph-common :condition: "custom volume" :default: "same as `volume.security.unmapped` or `false`" :shortdesc: "Disable ID mapping for the volume" :type: "bool" ``` ```{config:option} size storage_volume_ceph-common :condition: "-" :default: "same as `volume.size`" :shortdesc: "Size/quota of the storage volume" :type: "string" ``` ```{config:option} snapshots.expiry storage_volume_ceph-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.expiry.manual storage_volume_ceph-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry.manual`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.pattern storage_volume_ceph-common :condition: "custom volume" :default: "same as `volume.snapshot.pattern` or `snap%d`" :shortdesc: "{{snapshot_pattern_format}} [^*]" :type: "string" ``` ```{config:option} snapshots.schedule storage_volume_ceph-common :condition: "custom volume" :default: "same as `volume.snapshot.schedule`" :shortdesc: "{{snapshot_schedule_format}}" :type: "string" ``` ```{config:option} initial.gid storage_volume_cephfs-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.gid` or `0`" :shortdesc: "GID of the volume owner in the instance" :type: "int" ``` ```{config:option} initial.mode storage_volume_cephfs-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.mode` or `711`" :shortdesc: "Mode of the volume in the instance" :type: "int" ``` ```{config:option} initial.uid storage_volume_cephfs-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.uid` or `0`" :shortdesc: "UID of the volume owner in the instance" :type: "int" ``` ```{config:option} security.shared storage_volume_cephfs-common :condition: "custom block volume" :default: "same as `volume.security.shared` or `false`" :shortdesc: "Enable sharing the volume across multiple instances" :type: "bool" ``` ```{config:option} security.shifted storage_volume_cephfs-common :condition: "custom volume" :default: "same as `volume.security.shifted` or `false`" :shortdesc: "{{enable_ID_shifting}}" :type: "bool" ``` ```{config:option} security.unmapped storage_volume_cephfs-common :condition: "custom volume" :default: "same as `volume.security.unmapped` or `false`" :shortdesc: "Disable ID mapping for the volume" :type: "bool" ``` ```{config:option} size storage_volume_cephfs-common :condition: "appropriate driver" :default: "same as `volume.size`" :shortdesc: "Size/quota of the storage volume" :type: "string" ``` ```{config:option} snapshots.expiry storage_volume_cephfs-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.expiry.manual storage_volume_cephfs-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry.manual`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.pattern storage_volume_cephfs-common :condition: "custom volume" :default: "same as `volume.snapshot.pattern` or `snap%d`" :shortdesc: "{{snapshot_pattern_format}} [^*]" :type: "string" ``` ```{config:option} snapshots.schedule storage_volume_cephfs-common :condition: "custom volume" :default: "same as `volume.snapshot.schedule`" :shortdesc: "{{snapshot_schedule_format}}" :type: "string" ``` ```{config:option} initial.gid storage_volume_dir-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.gid` or `0`" :shortdesc: "GID of the volume owner in the instance" :type: "int" ``` ```{config:option} initial.mode storage_volume_dir-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.mode` or `711`" :shortdesc: "Mode of the volume in the instance" :type: "int" ``` ```{config:option} initial.uid storage_volume_dir-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.uid` or `0`" :shortdesc: "UID of the volume owner in the instance" :type: "int" ``` ```{config:option} security.shared storage_volume_dir-common :condition: "custom block volume" :default: "same as `volume.security.shared` or `false`" :shortdesc: "Enable sharing the volume across multiple instances" :type: "bool" ``` ```{config:option} security.shifted storage_volume_dir-common :condition: "custom volume" :default: "same as `volume.security.shifted` or `false`" :shortdesc: "{{enable_ID_shifting}}" :type: "bool" ``` ```{config:option} security.size storage_volume_dir-common :condition: "appropriate driver" :default: "same as `volume.size`" :shortdesc: "Size/quota of the storage volume" :type: "string" ``` ```{config:option} security.unmapped storage_volume_dir-common :condition: "custom volume" :default: "same as `volume.security.unmapped` or `false`" :shortdesc: "Disable ID mapping for the volume" :type: "bool" ``` ```{config:option} snapshots.expiry storage_volume_dir-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.expiry.manual storage_volume_dir-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry.manual`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.pattern storage_volume_dir-common :condition: "custom volume" :default: "same as `volume.snapshot.pattern` or `snap%d`" :shortdesc: "{{snapshot_pattern_format}} [^*]" :type: "string" ``` ```{config:option} snapshots.schedule storage_volume_dir-common :condition: "custom volume" :default: "same as `volume.snapshot.schedule`" :shortdesc: "{{snapshot_schedule_format}}" :type: "string" ``` ```{config:option} block.filesystem storage_volume_linstor-common :condition: "block-based volume with content type `filesystem`" :default: "same as `volume.block.filesystem`" :shortdesc: "{{block_filesystem}}" :type: "string" ``` ```{config:option} block.mount_options storage_volume_linstor-common :condition: "block-based volume with content type `filesystem`" :default: "same as `volume.block.mount_options`" :shortdesc: "Mount options for block-backed file system volumes" :type: "string" ``` ```{config:option} drbd.auto_add_quorum_tiebreaker storage_volume_linstor-common :condition: "-" :default: "`true`" :shortdesc: "Whether to allow LINSTOR to automatically create diskless resources to act as quorum tiebreakers if needed (applied to the resource definition)" :type: "bool" ``` ```{config:option} drbd.auto_diskful storage_volume_linstor-common :condition: "-" :default: "-" :shortdesc: "A duration string describing the time after which a primary diskless resource can be converted to diskful if storage is available on the node (applied to the resource definition)" :type: "string" ``` ```{config:option} drbd.on_no_quorum storage_volume_linstor-common :condition: "-" :default: "-" :shortdesc: "The DRBD policy to use on resources when quorum is lost (applied to the resource definition)" :type: "string" ``` ```{config:option} initial.gid storage_volume_linstor-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.gid` or `0`" :shortdesc: "GID of the volume owner in the instance" :type: "int" ``` ```{config:option} initial.mode storage_volume_linstor-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.mode` or `711`" :shortdesc: "Mode of the volume in the instance" :type: "int" ``` ```{config:option} initial.uid storage_volume_linstor-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.uid` or `0`" :shortdesc: "UID of the volume owner in the instance" :type: "int" ``` ```{config:option} linstor.remove_snapshots storage_volume_linstor-common :condition: "-" :default: "same as `volume.linstor.remove_snapshots` or `false`" :shortdesc: "Remove snapshots as needed" :type: "bool" ``` ```{config:option} security.shared storage_volume_linstor-common :condition: "custom block volume" :default: "same as `volume.security.shared` or `false`" :shortdesc: "Enable sharing the volume across multiple instances" :type: "bool" ``` ```{config:option} security.shifted storage_volume_linstor-common :condition: "custom volume" :default: "same as `volume.security.shifted` or `false`" :shortdesc: "{{enable_ID_shifting}}" :type: "bool" ``` ```{config:option} security.unmapped storage_volume_linstor-common :condition: "custom volume" :default: "same as `volume.security.unmapped` or `false`" :shortdesc: "Disable ID mapping for the volume" :type: "bool" ``` ```{config:option} size storage_volume_linstor-common :condition: "-" :default: "same as `volume.size`" :shortdesc: "Size/quota of the storage volume" :type: "string" ``` ```{config:option} snapshots.expiry storage_volume_linstor-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.expiry.manual storage_volume_linstor-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry.manual`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.pattern storage_volume_linstor-common :condition: "custom volume" :default: "same as `volume.snapshot.pattern` or `snap%d`" :shortdesc: "{{snapshot_pattern_format}} [^*]" :type: "string" ``` ```{config:option} snapshots.schedule storage_volume_linstor-common :condition: "custom volume" :default: "same as `volume.snapshot.schedule`" :shortdesc: "{{snapshot_schedule_format}}" :type: "string" ``` ```{config:option} block.filesystem storage_volume_lvm-common :condition: "block-based volume with content type `filesystem`" :default: "same as `volume.block.filesystem`" :shortdesc: "{{block_filesystem}}" :type: "string" ``` ```{config:option} block.mount_options storage_volume_lvm-common :condition: "block-based volume with content type `filesystem`" :default: "same as `volume.block.mount_options`" :shortdesc: "Mount options for block-backed file system volumes" :type: "string" ``` ```{config:option} initial.gid storage_volume_lvm-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.gid` or `0`" :shortdesc: "GID of the volume owner in the instance" :type: "int" ``` ```{config:option} initial.mode storage_volume_lvm-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.mode` or `711`" :shortdesc: "Mode of the volume in the instance" :type: "int" ``` ```{config:option} initial.uid storage_volume_lvm-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.uid` or `0`" :shortdesc: "UID of the volume owner in the instance" :type: "int" ``` ```{config:option} lvm.stripes storage_volume_lvm-common :condition: "-" :default: "same as `volume.lvm.stripes`" :shortdesc: "Number of stripes to use for new volumes (or thin pool volume)" :type: "string" ``` ```{config:option} lvm.stripes.size storage_volume_lvm-common :condition: "-" :default: "same as `volume.lvm.stripes.size`" :shortdesc: "Size of stripes to use (at least 4096 bytes and multiple of 512 bytes)" :type: "string" ``` ```{config:option} lvmcluster.remove_snapshots storage_volume_lvm-common :condition: "-" :default: "same as `volume.lvmcluster.remove_snapshots` or `false`" :shortdesc: "Remove snapshots as needed" :type: "bool" ``` ```{config:option} security.shared storage_volume_lvm-common :condition: "custom block volume" :default: "same as `volume.security.shared` or `false`" :shortdesc: "Enable sharing the volume across multiple instances" :type: "bool" ``` ```{config:option} security.shifted storage_volume_lvm-common :condition: "custom volume" :default: "same as `volume.security.shifted` or `false`" :shortdesc: "{{enable_ID_shifting}}" :type: "bool" ``` ```{config:option} security.unmapped storage_volume_lvm-common :condition: "custom volume" :default: "same as `volume.security.unmapped` or `false`" :shortdesc: "Disable ID mapping for the volume" :type: "bool" ``` ```{config:option} size storage_volume_lvm-common :condition: "default: same as `volume.size`" :shortdesc: "Size/quota of the storage volume" :type: "string" ``` ```{config:option} snapshots.expiry storage_volume_lvm-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.expiry.manual storage_volume_lvm-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry.manual`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.pattern storage_volume_lvm-common :condition: "custom volume" :default: "same as `volume.snapshot.pattern` or `snap%d`" :shortdesc: "{{snapshot_pattern_format}} [^*]" :type: "string" ``` ```{config:option} snapshots.schedule storage_volume_lvm-common :condition: "custom volume" :default: "same as `volume.snapshot.schedule`" :shortdesc: "{{snapshot_schedule_format}}" :type: "string" ``` ```{config:option} block.filesystem storage_volume_truenas-common :condition: "-" :default: "same as `volume.block.filesystem`" :shortdesc: "{{block_filesystem}}" :type: "string" ``` ```{config:option} block.mount_options storage_volume_truenas-common :condition: "-" :default: "same as `volume.block.mount_options`" :shortdesc: "Mount options for block-backed file system volumes" :type: "string" ``` ```{config:option} initial.gid storage_volume_truenas-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.gid` or `0`" :shortdesc: "GID of the volume owner in the instance" :type: "int" ``` ```{config:option} initial.mode storage_volume_truenas-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.mode` or `711`" :shortdesc: "Mode of the volume in the instance" :type: "int" ``` ```{config:option} initial.uid storage_volume_truenas-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.uid` or `0`" :shortdesc: "UID of the volume owner in the instance" :type: "int" ``` ```{config:option} security.shared storage_volume_truenas-common :condition: "custom block volume" :default: "same as `volume.security.shared` or `false`" :shortdesc: "Enable sharing the volume across multiple instances" :type: "bool" ``` ```{config:option} security.shifted storage_volume_truenas-common :condition: "custom volume" :default: "same as `volume.security.shifted` or `false`" :shortdesc: "{{enable_ID_shifting}}" :type: "bool" ``` ```{config:option} security.unmapped storage_volume_truenas-common :condition: "custom volume" :default: "same as `volume.security.unmapped` or `false`" :shortdesc: "Disable ID mapping for the volume" :type: "bool" ``` ```{config:option} size storage_volume_truenas-common :condition: "appropriate driver" :default: "same as `volume.size`" :shortdesc: "Size/quota of the storage volume" :type: "string" ``` ```{config:option} snapshots.expiry storage_volume_truenas-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.expiry.manual storage_volume_truenas-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry.manual`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.pattern storage_volume_truenas-common :condition: "custom volume" :default: "same as `volume.snapshot.pattern` or `snap%d`" :shortdesc: "{{snapshot_pattern_format}}" :type: "string" ``` ```{config:option} snapshots.schedule storage_volume_truenas-common :condition: "custom volume" :default: "same as `volume.snapshot.schedule`" :shortdesc: "{{snapshot_schedule_format}}" :type: "string" ``` ```{config:option} truenas.blocksize storage_volume_truenas-common :condition: "-" :default: "same as `volume.truenas.blocksize`" :shortdesc: "Size of the ZFS block in range from 512 bytes to 16 MiB (must be power of 2) - for block volume, a maximum value of 128 KiB will be used even if a higher value is set" :type: "string" ``` ```{config:option} truenas.remove_snapshots storage_volume_truenas-common :condition: "-" :default: "same as `volume.truenas.remove_snapshots` or `false`" :shortdesc: "Remove snapshots as needed" :type: "bool" ``` ```{config:option} truenas.use_refquota storage_volume_truenas-common :condition: "-" :default: "same as `volume.truenas.use_refquota` or `false`" :shortdesc: "Use `refquota` instead of `quota` for space" :type: "bool" ``` ```{config:option} block.filesystem storage_volume_zfs-common :condition: "block-based volume with content type `filesystem` (`zfs.block_mode` enabled)" :default: "same as `volume.block.filesystem`" :shortdesc: "{{block_filesystem}}" :type: "string" ``` ```{config:option} block.mount_options storage_volume_zfs-common :condition: "block-based volume with content type `filesystem` (`zfs.block_mode` enabled)" :default: "same as `volume.block.mount_options`" :shortdesc: "Mount options for block-backed file system volumes" :type: "string" ``` ```{config:option} initial.gid storage_volume_zfs-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.gid` or `0`" :shortdesc: "GID of the volume owner in the instance" :type: "int" ``` ```{config:option} initial.mode storage_volume_zfs-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.mode` or `711`" :shortdesc: "Mode of the volume in the instance" :type: "int" ``` ```{config:option} initial.uid storage_volume_zfs-common :condition: "custom volume with content type `filesystem`" :default: "same as `volume.initial.uid` or `0`" :shortdesc: "UID of the volume owner in the instance" :type: "int" ``` ```{config:option} security.shared storage_volume_zfs-common :condition: "custom block volume" :default: "same as `volume.security.shared` or `false`" :shortdesc: "Enable sharing the volume across multiple instances" :type: "bool" ``` ```{config:option} security.shifted storage_volume_zfs-common :condition: "custom volume" :default: "same as `volume.security.shifted` or `false`" :shortdesc: "{{enable_ID_shifting}}" :type: "bool" ``` ```{config:option} security.unmapped storage_volume_zfs-common :condition: "custom volume" :default: "same as `volume.security.unmapped` or `false`" :shortdesc: "Disable ID mapping for the volume" :type: "bool" ``` ```{config:option} size storage_volume_zfs-common :condition: "-" :default: "same as `volume.size`" :shortdesc: "Size/quota of the storage volume" :type: "string" ``` ```{config:option} snapshots.expiry storage_volume_zfs-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.expiry.manual storage_volume_zfs-common :condition: "custom volume" :default: "same as `volume.snapshot.expiry.manual`" :shortdesc: "{{snapshot_expiry_format}}" :type: "string" ``` ```{config:option} snapshots.pattern storage_volume_zfs-common :condition: "custom volume" :default: "same as `volume.snapshot.pattern` or `snap%d`" :shortdesc: "{{snapshot_pattern_format}} [^*]" :type: "string" ``` ```{config:option} snapshots.schedule storage_volume_zfs-common :condition: "custom volume" :default: "same as `volume.snapshot.schedule`" :shortdesc: "{{snapshot_schedule_format}}" :type: "string" ``` ```{config:option} zfs.block_mode storage_volume_zfs-common :condition: "-" :default: "same as `volume.zfs.block_mode`" :shortdesc: "Whether to use a formatted `zvol` rather than a {spellexception}`dataset` (`zfs.block_mode` can be set only for custom storage volumes; use `volume.zfs.block_mode` to enable ZFS block mode for all storage volumes in the pool, including instance volumes)" :type: "bool" ``` ```{config:option} zfs.blocksize storage_volume_zfs-common :condition: "-" :default: "same as `volume.zfs.blocksize`" :shortdesc: "Size of the ZFS block in range from 512 bytes to 16 MiB (must be power of 2) - for block volume, a maximum value of 128 KiB will be used even if a higher value is set" :type: "string" ``` ```{config:option} zfs.delegate storage_volume_zfs-common :condition: "ZFS 2.2 or higher" :default: "same as `volume.zfs.delegate`" :shortdesc: "Controls whether to delegate the ZFS dataset and anything underneath it to the container(s) using it. Allows the use of the `zfs` command in the container" :type: "bool" ``` ```{config:option} zfs.remove_snapshots storage_volume_zfs-common :condition: "-" :default: "same as `volume.zfs.remove_snapshots` or `false`" :shortdesc: "Remove snapshots as needed" :type: "bool" ``` ```{config:option} zfs.reserve_space storage_volume_zfs-common :condition: "-" :default: "same as `volume.zfs.reserve_space` or `false`" :shortdesc: "Use `reservation`/`refreservation` along with `quota`/`refquota`" :type: "bool" ``` ```{config:option} zfs.use_refquota storage_volume_zfs-common :condition: "-" :default: "same as `volume.zfsuse_refquota` or `false`" :shortdesc: "Use `refquota` instead of `quota` for space" :type: "bool" ``` ```{config:option} size storage_zfs-common :default: "auto (20% of free disk space, >= 5 GiB and <= 30 GiB)" :scope: "local" :shortdesc: "Size of the storage pool when creating loop-based pools (in bytes, suffixes supported, can be increased to grow storage pool)" :type: "string" ``` ```{config:option} source storage_zfs-common :default: "-" :scope: "local" :shortdesc: "Path to existing block device(s), loop file or ZFS dataset/pool. Multiple block devices should be separated by `,`. When listing block devices, you can also prefix them with `vdev` type. To specify a `vdev` type, use an `=` sign between the `vdev` type and the block devices (e.g., `mirror=/dev/sda,/dev/sdb`). Only `stripe`, `mirror`, `raidz1` and `raidz2` `vdev` types are supported." :type: "string" ``` ```{config:option} source.wipe storage_zfs-common :default: "`false`" :scope: "local" :shortdesc: "Wipe the block device specified in `source` prior to creating the storage pool" :type: "bool" ``` ```{config:option} zfs.clone_copy storage_zfs-common :default: "`true`" :scope: "global" :shortdesc: "Whether to use ZFS lightweight clones rather than full {spellexception}`dataset` copies (Boolean), or `rebase` to copy based on the initial image" :type: "string" ``` ```{config:option} zfs.export storage_zfs-common :default: "`true`" :scope: "global" :shortdesc: "Disable zpool export while unmount performed" :type: "bool" ``` ```{config:option} zfs.pool_name storage_zfs-common :default: "name of the pool" :scope: "local" :shortdesc: "Name of the zpool" :type: "string" ``` incus-7.0.0/doc/config_options_cheat_sheet.md000066400000000000000000000100631517523235500213120ustar00rootroot00000000000000--- orphan: true nosearch: true --- # Configuration options ```{important} This page shows how to output configuration option documentation. The content in this page is for demonstration purposes only. ``` Some instance options: ```{config:option} agent.nic_config instance :shortdesc: Set the name and MTU to be the same as the instance devices :default: "`false`" :type: bool :liveupdate: "`no`" :condition: Virtual machine Controls whether to set the name and MTU of the default network interfaces to be the same as the instance devices (this happens automatically for containers) ``` ```{config:option} migration.incremental.memory.iterations instance :shortdesc: Maximum number of transfer operations :condition: container :default: 10 :type: integer :liveupdate: "yes" Maximum number of transfer operations to go through before stopping the instance ``` ```{config:option} cluster.evacuate instance :shortdesc: What to do when evacuating the instance :default: "`auto`" :type: string :liveupdate: "no" Controls what to do when evacuating the instance (`auto`, `migrate`, `live-migrate`, or `stop`) ``` These need the `instance` scope to be specified as second argument. The default scope is `server`, so this argument isn't required. Some server options: ```{config:option} backups.compression_algorithm server :shortdesc: Compression algorithm for images :type: string :scope: global :default: "`gzip`" Compression algorithm to use for new images (`bzip2`, `gzip`, `lzma`, `xz` or `none`) ``` ```{config:option} instances.nic.host_name :shortdesc: How to generate a host name :type: string :scope: global :default: "`random`" If set to `random`, use the random host interface name as the host name; if set to `mac`, generate a host name in the form `inc` (MAC without leading two digits) ``` ```{config:option} instances.placement.scriptlet :shortdesc: Custom automatic instance placement logic :type: string :scope: global Stores the {ref}`clustering-instance-placement-scriptlet` for custom automatic instance placement logic ``` Any other scope is also possible. This scope shows that you can use formatting, mainly in the short description and the description, and the available options. ```{config:option} test1 something :shortdesc: testing Testing. ``` ```{config:option} test2 something :shortdesc: Hello! **bold** and `code` This is the real text. With two paragraphs. And a list: - Item - Item - Item And a table: Key | Type | Scope | Default | Description :-- | :--- | :---- | :------ | :---------- `acme.agree_tos` | bool | global | `false` | Agree to ACME terms of service `acme.ca_url` | string | global | `https://acme-v02.api.letsencrypt.org/directory` | URL to the directory resource of the ACME service `acme.domain` | string | global | - | Domain for which the certificate is issued `acme.email` | string | global | - | Email address used for the account registration ``` ```{config:option} test3 something :shortdesc: testing :default: "`false`" :type: Type :liveupdate: Python parses the options, so "no" is converted to "False" - to prevent this, put quotes around the text ("no" or "`no`") :condition: "yes" :readonly: "`maybe` - also add quotes if the option starts with code" :resource: Resource, :managed: Managed :required: Required :scope: (this is something like "global" or "local", **not** the scope of the option (`server`, `instance`, ...) Content ``` To reference an option, use `{config:option}`. It is not possible to override the link text. Except for server options (default), you must specify the scope. {config:option}`instance:migration.incremental.memory.iterations` {config:option}`something:test1` The index is here: {ref}`config-options` incus-7.0.0/doc/container-environment.md000066400000000000000000000060211517523235500202610ustar00rootroot00000000000000(container-runtime-environment)= # Container runtime environment Incus attempts to present a consistent environment to all containers it runs. The exact environment will differ slightly based on kernel features and user configuration, but otherwise, it is identical for all containers. ## File system Incus assumes that any image it uses to create a new container comes with at least the following root-level directories: - `/dev` (empty) - `/proc` (empty) - `/sbin/init` (executable) - `/sys` (empty) ## Devices Incus containers have a minimal and ephemeral `/dev` based on a `tmpfs` file system. Since this is a `tmpfs` and not a `devtmpfs` file system, device nodes appear only if manually created. The following standard set of device nodes is set up automatically: - `/dev/console` - `/dev/fd` - `/dev/full` - `/dev/log` - `/dev/null` - `/dev/ptmx` - `/dev/random` - `/dev/stdin` - `/dev/stderr` - `/dev/stdout` - `/dev/tty` - `/dev/urandom` - `/dev/zero` In addition to the standard set of devices, the following devices are also set up for convenience: - `/dev/fuse` - `/dev/net/tun` - `/dev/mqueue` ### Network Incus containers may have any number of network devices attached to them. The naming for those (unless overridden by the user) is `ethX`, where `X` is an incrementing number. ### Container-to-host communication Incus sets up a socket at `/dev/incus/sock` that the root user in the container can use to communicate with Incus on the host. See {doc}`dev-incus` for the API documentation. ## Mounts The following mounts are set up by default: - `/proc` ({spellexception}`proc`) - `/sys` (`sysfs`) - `/sys/fs/cgroup/*` (`cgroupfs`) (only on kernels that lack cgroup namespace support) If they are present on the host, the following paths will also automatically be mounted: - `/proc/sys/fs/binfmt_misc` - `/sys/firmware/efi/efivars` - `/sys/fs/fuse/connections` - `/sys/fs/pstore` - `/sys/kernel/debug` - `/sys/kernel/security` The reason for passing all of those paths is that legacy init systems require them to be mounted, or be mountable, inside the container. The majority of those paths will not be writable (or even readable) from inside an unprivileged container. In privileged containers, they will be blocked by the AppArmor policy. ### LXCFS If LXCFS is present on the host, it is automatically set up for the container. This normally results in a number of `/proc` files being overridden through bind-mounts. ## PID1 Incus spawns whatever is located at `/sbin/init` as the initial process of the container (PID 1). This binary should act as a proper init system, including handling re-parented processes. Incus' communication with PID1 in the container is limited to two signals: - `SIGINT` to trigger a reboot of the container - `SIGPWR` (or alternatively `SIGRTMIN`+3) to trigger a clean shutdown of the container The initial environment of PID1 is blank except for `container=lxc`, which can be used by the init system to detect the runtime. All file descriptors above the default three are closed prior to PID1 being spawned. incus-7.0.0/doc/contributing.md000066400000000000000000000003031517523235500164410ustar00rootroot00000000000000# Contributing to Incus ```{toctree} :maxdepth: 1 Introduction Contribute to the code Contribute to the documentation ``` incus-7.0.0/doc/contributing/000077500000000000000000000000001517523235500161235ustar00rootroot00000000000000incus-7.0.0/doc/contributing/code.md000066400000000000000000000031541517523235500173620ustar00rootroot00000000000000# Contribute to the code Follow the steps below to set up your development environment to get started working on new features for Incus. ## Install Incus from source To build the dependencies, follow the instructions in {ref}`installing_from_source`. ## Add your fork as a remote After setting up your build environment, add your GitHub fork as a remote: git remote add myfork git@github.com:/incus.git git remote update Then switch to it: git checkout myfork/main ## Build Incus Finally, you should be able to run `make` inside the repository and build your fork of the project. At this point, you most likely want to create a new branch for your changes on your fork: ```bash git checkout -b [name_of_your_new_branch] git push myfork [name_of_your_new_branch] ``` ## Important notes for new Incus contributors - Persistent data is stored in the `INCUS_DIR` directory, which is generated by `incus admin init`. The `INCUS_DIR` defaults to `/var/lib/incus`. - As you develop, you may want to change the `INCUS_DIR` for your fork of Incus so as to avoid version conflicts. - Binaries compiled from your source will be generated in the `$(go env GOPATH)/bin` directory by default. - You will need to explicitly invoke these binaries (not the global `incusd` you may have installed) when testing your changes. - You may choose to create an alias in your `~/.bashrc` to call these binaries with the appropriate flags more conveniently. - If you have a `systemd` service configured to run the Incus daemon from a previous installation of Incus, you may want to disable it to avoid version conflicts. incus-7.0.0/doc/contributing/docs.md000066400000000000000000000066041517523235500174030ustar00rootroot00000000000000# Contribute to the documentation We want Incus to be as easy and straight-forward to use as possible. Therefore, we aim to provide documentation that contains the information that users need to work with Incus, that covers all common use cases, and that answers typical questions. You can contribute to the documentation in various different ways. We appreciate your contributions! Typical ways to contribute are: - Add or update documentation for new features or feature improvements that you contribute to the code. We'll review the documentation update and merge it together with your code. - Add or update documentation that clarifies any doubts you had when working with the product. Such contributions can be done through a pull request or through a post in the [Tutorials](https://discuss.linuxcontainers.org/c/tutorials/16) section on the forum. New tutorials will be considered for inclusion in the docs (through a link or by including the actual content). - To request a fix to the documentation, open a documentation issue on [GitHub](https://github.com/lxc/incus/issues). We'll evaluate the issue and update the documentation accordingly. - Post a question or a suggestion on the [forum](https://discuss.linuxcontainers.org). We'll monitor the posts and, if needed, update the documentation accordingly. % Include content from [../README.md](../README.md) ```{include} ../README.md :start-after: ``` When you open a pull request, a preview of the documentation output is built automatically. ## Automatic documentation checks GitHub runs automatic checks on the documentation to verify the spelling, the validity of links, correct formatting of the Markdown files, and the use of inclusive language. You can (and should!) run these tests locally as well with the following commands: - Check the spelling: `make doc-spellcheck` - Check the validity of links: `make doc-linkcheck` - Check the Markdown formatting: `make doc-lint` - Check for inclusive language: `make doc-woke` To run the above, you will need the following: - Python 3.8 or higher - The `venv` python package - The `aspell` tool for spellchecking - The `mdl` markdown lint tool ## Document configuration options ```{note} We are currently in the process of moving the documentation of configuration options to code comments. At the moment, not all configuration options follow this approach. ``` The documentation of configuration options is extracted from comments in the Go code. Look for comments that start with `gendoc:generate` in the code. When you add or change a configuration option, make sure to include the required documentation comment for it. Then run `make generate-config` to re-generate the `doc/config_options.txt` file. The updated file should be checked in. The documentation includes sections from the `doc/config_options.txt` to display a group of configuration options. For example, to include the core server options: ```` % Include content from [config_options.txt](config_options.txt) ```{include} config_options.txt :start-after: :end-before: ``` ```` If you add a configuration option to an existing group, you don't need to do any updates to the documentation files. The new option will automatically be picked up. You only need to add an include to a documentation file if you are defining a new group. incus-7.0.0/doc/contributing/introduction.md000066400000000000000000000003641517523235500211710ustar00rootroot00000000000000# How to contribute to Incus % Include content from [../../CONTRIBUTING.md](../../CONTRIBUTING.md) ```{include} ../../CONTRIBUTING.md :start-after: :end-before: ``` incus-7.0.0/doc/daemon-behavior.md000066400000000000000000000023251517523235500170000ustar00rootroot00000000000000(daemon-behavior)= # Daemon behavior This specification covers some of the Incus daemon's behavior. ## Startup On every start, Incus checks that its directory structure exists. If it doesn't, it creates the required directories, generates a key pair and initializes the database. Once the daemon is ready for work, Incus scans the instances table for any instance for which the stored power state differs from the current one. If an instance's power state was recorded as running and the instance isn't running, Incus starts it. ## Signal handling ### `SIGINT`, `SIGQUIT`, `SIGTERM` For those signals, Incus assumes that it's being temporarily stopped and will be restarted at a later time to continue handling the instances. The instances will keep running and Incus will close all connections and exit cleanly. ### `SIGPWR` Indicates to Incus that the host is going down. Incus will attempt a clean shutdown of all the instances. After 30 seconds, it kills any remaining instance. The instance `power_state` in the instances table is kept as it was so that Incus can restore the instances as they were after the host is done rebooting. ### `SIGUSR1` Write a memory profile dump to the file specified with `--memprofile`. incus-7.0.0/doc/database.md000066400000000000000000000030721517523235500155040ustar00rootroot00000000000000(database)= # About the Incus database Incus uses a distributed database to store the server configuration and state, which allows for quicker queries than if the configuration was stored inside each instance's directory (as it is done by LXC, for example). To understand the advantages, consider a query against the configuration of all instances, like "what instances are using `br0`?". To answer that question without a database, you would have to iterate through every single instance, load and parse its configuration, and then check which network devices are defined in there. With a database, you can run a simple query on the database to retrieve this information. ## Cowsql In an Incus cluster, all members of the cluster must share the same database state. Therefore, Incus uses [Cowsql](https://github.com/cowsql/cowsql), a distributed version of SQLite. Cowsql provides replication, fault-tolerance, and automatic failover without the need of external database processes. When using Incus as a single machine and not as a cluster, the Cowsql database effectively behaves like a regular SQLite database. ## File location The database files are stored in the `database` sub-directory of your Incus data directory (`/var/lib/incus/database/`). Upgrading Incus to a newer version might require updating the database schema. In this case, Incus automatically stores a backup of the database and then runs the update. See {ref}`installing-upgrade` for more information. ## Backup See {ref}`backup-database` for instructions on how to back up the contents of the Incus database. incus-7.0.0/doc/debugging.md000066400000000000000000000106571517523235500157020ustar00rootroot00000000000000# How to debug Incus For information on debugging instance issues, see {ref}`instances-troubleshoot`. ## Debugging `incus` and `incusd` Here are different ways to help troubleshooting `incus` and `incusd` code. ### `incus --debug` Adding `--debug` flag to any client command will give extra information about internals. If there is no useful info, it can be added with the logging call: logger.Debugf("Hello: %s", "Debug") ### `incus monitor` This command will monitor messages as they appear on remote server. ## REST API through local socket On server side the most easy way is to communicate with Incus through local socket. This command accesses `GET /1.0` and formats JSON into human readable form using [jq](https://stedolan.github.io/jq/tutorial/) utility: ```bash curl --unix-socket /var/lib/incus/unix.socket incus/1.0 | jq . ``` See the [RESTful API](rest-api.md) for available API. ## REST API through HTTPS {ref}`HTTPS connection to Incus ` requires valid client certificate that is generated on first [`incus remote add`](incus_remote_add.md). This certificate should be passed to connection tools for authentication and encryption. If desired, `openssl` can be used to examine the certificate (`~/.config/incus/client.crt`): ```bash openssl x509 -text -noout -in client.crt ``` Among the lines you should see: Certificate purposes: SSL client : Yes ### With command line tools ```bash wget --no-check-certificate --certificate=$HOME/.config/incus/client.crt --private-key=$HOME/.config/incus/client.key -qO - https://127.0.0.1:8443/1.0 ``` ### With browser Some browser plugins provide convenient interface to create, modify and replay web requests. To authenticate against Incus server, convert `incus` client certificate into importable format and import it into browser. For example this produces `client.pfx` in Windows-compatible format: ```bash openssl pkcs12 -clcerts -inkey client.key -in client.crt -export -out client.pfx ``` After that, opening [`https://127.0.0.1:8443/1.0`](https://127.0.0.1:8443/1.0) should work as expected. ## Debug the Incus database The files of the global {ref}`database ` are stored under the `./database/global` sub-directory of your Incus data directory (`/var/lib/incus/database/global`). Since each member of the cluster also needs to keep some data which is specific to that member, Incus also uses a plain SQLite database (the "local" database), which you can find in `./database/local.db`. Backups of the global database directory and of the local database file are made before upgrades, and are tagged with the `.bak` suffix. You can use those if you need to revert the state as it was before the upgrade. ### Dumping the database content or schema If you want to get a SQL text dump of the content or the schema of the databases, use the `incus admin sql [.dump|.schema]` command, which produces the equivalent output of the `.dump` or `.schema` directives of the `sqlite3` command line tool. ### Running custom queries from the console If you need to perform SQL queries (e.g. `SELECT`, `INSERT`, `UPDATE`) against the local or global database, you can use the `incus admin sql` command (run `incus admin sql --help` for details). You should only need to do that in order to recover from broken updates or bugs. Please consult the Incus team first (creating a [GitHub issue](https://github.com/lxc/incus/issues/new) or [forum](https://discuss.linuxcontainers.org) post). ### Running custom queries at Incus daemon startup In case the Incus daemon fails to start after an upgrade because of SQL data migration bugs or similar problems, it's possible to recover the situation by creating `.sql` files containing queries that repair the broken update. To perform repairs against the local database, write a `./database/patch.local.sql` file containing the relevant queries, and similarly a `./database/patch.global.sql` for global database repairs. Those files will be loaded very early in the daemon startup sequence and deleted if the queries were successful (if they fail, no state will change as they are run in a SQL transaction). As above, please consult the Incus team first. ### Syncing the cluster database to disk If you want to flush the content of the cluster database to disk, use the `incus admin sql global .sync` command, that will write a plain SQLite database file into `./database/global/db.bin`, which you can then inspect with the `sqlite3` command line tool. incus-7.0.0/doc/dev-incus.md000066400000000000000000000111171517523235500156340ustar00rootroot00000000000000(dev-incus)= # Communication between instance and host Communication between the hosted workload (instance) and its host while not strictly needed is a pretty useful feature. In Incus, this feature is implemented through a `/dev/incus/sock` node which is created and set up for all Incus instances. This file is a Unix socket which processes inside the instance can connect to. It's multi-threaded so multiple clients can be connected at the same time. ```{note} {config:option}`instance-security:security.guestapi` must be set to `true` (which is the default) for an instance to allow access to the socket. ``` ## Implementation details Incus on the host binds `/var/lib/incus/guestapi/sock` and starts listening for new connections on it. This socket is then exposed into every single instance started by Incus at `/dev/incus/sock`. The single socket is required so we can exceed 4096 instances, otherwise, Incus would have to bind a different socket for every instance, quickly reaching the FD limit. ## Authentication Queries on `/dev/incus/sock` will only return information related to the requesting instance. To figure out where a request comes from, Incus will extract the initial socket's user credentials and compare that to the list of instances it manages. ## Protocol The protocol on `/dev/incus/sock` is plain-text HTTP with JSON messaging, so very similar to the local version of the Incus protocol. Unlike the main Incus API, there is no background operation and no authentication support in the `/dev/incus/sock` API. ## REST-API ### API structure * `/` * `/1.0` * `/1.0/config` * `/1.0/config/{key}` * `/1.0/devices` * `/1.0/events` * `/1.0/images/{fingerprint}/export` * `/1.0/meta-data` ### API details #### `/` ##### GET * Description: List of supported APIs * Return: list of supported API endpoint URLs (by default `['/1.0']`) Return value: ```json [ "/1.0" ] ``` #### `/1.0` ##### GET * Description: Information about the 1.0 API * Return: JSON object Return value: ```json { "api_version": "1.0", "location": "foo.example.com", "instance_type": "container", "state": "Started", } ``` #### PATCH * Description: Update instance state (valid states are `Ready` and `Started`) * Return: none Input: ```json { "state": "Ready" } ``` #### `/1.0/config` ##### GET * Description: List of configuration keys * Return: list of configuration keys URL Note that the configuration key names match those in the instance configuration, however not all configuration namespaces will be exported to `/dev/incus/sock`. Currently only the `cloud-init.*` and `user.*` keys are accessible to the instance. At this time, there also aren't any instance-writable namespace. Return value: ```json [ "/1.0/config/user.a" ] ``` #### `/1.0/config/` ##### GET * Description: Value of that key * Return: Plain-text value Return value: blah #### `/1.0/devices` ##### GET * Description: Map of instance devices * Return: JSON object Return value: ```json { "eth0": { "name": "eth0", "network": "incusbr0", "type": "nic" }, "root": { "path": "/", "pool": "default", "type": "disk" } } ``` #### `/1.0/events` ##### GET * Description: WebSocket upgrade * Return: none (never ending flow of events) Supported arguments are: * type: comma-separated list of notifications to subscribe to (defaults to all) The notification types are: * `config` (changes to any of the `user.*` configuration keys) * `device` (any device addition, change or removal) This never returns. Each notification is sent as a separate JSON object: ```json { "timestamp": "2017-12-21T18:28:26.846603815-05:00", "type": "device", "metadata": { "name": "kvm", "action": "added", "config": { "type": "unix-char", "path": "/dev/kvm" } } } ``` ```json { "timestamp": "2017-12-21T18:28:26.846603815-05:00", "type": "config", "metadata": { "key": "user.foo", "old_value": "", "value": "bar" } } ``` #### `/1.0/images//export` ##### GET * Description: Download a public/cached image from the host * Return: raw image or error * Access: Requires `security.guestapi.images` set to `true` Return value: See /1.0/images//export in the daemon API. #### `/1.0/meta-data` ##### GET * Description: Container meta-data compatible with cloud-init * Return: cloud-init meta-data Return value: #cloud-config instance-id: af6a01c7-f847-4688-a2a4-37fddd744625 local-hostname: abc incus-7.0.0/doc/doc-cheat-sheet.md000066400000000000000000000414441517523235500167020ustar00rootroot00000000000000--- orphan: true nosearch: true myst: substitutions: reuse_key: "This is **included** text." advanced_reuse_key: "This is a substitution that includes a code block: ``` code block ```" --- # Documentation cheat sheet The documentation files use a mixture of [Markdown](https://commonmark.org/) and [MyST](https://myst-parser.readthedocs.io/) syntax. See the following sections for syntax help and conventions. ## Headings ```{list-table} :header-rows: 1 * - Input - Description * - `# Title` - Page title and H1 heading * - `## Heading` - H2 heading * - `### Heading` - H3 heading * - `#### Heading` - H4 heading * - ... - Further headings ``` Adhere to the following conventions: - Do not use consecutive headings without intervening text. - Use sentence style for headings (capitalize only the first word). - Do not skip levels (for example, always follow an H2 with an H3, not an H4). ## Inline formatting ```{list-table} :header-rows: 1 * - Input - Output * - `` {guilabel}`UI element` `` - {guilabel}`UI element` * - `` `code` `` - `code` * - `` {command}`command` `` - {command}`command` * - `*Italic*` - *Italic* * - `**Bold**` - **Bold** ``` Adhere to the following conventions: - Use italics sparingly. Common uses for italics are titles and names (for example, when referring to a section title that you cannot link to, or when introducing the name for a concept). - Use bold sparingly. A common use for bold is UI elements ("Click **OK**"). Avoid using bold for emphasis and rather rewrite the sentence to get your point across. ## Code blocks Start and end a code block with three back ticks: ``` You can specify the code language after the back ticks to enforce a specific lexer, but in many cases, the default lexer works just fine. ```{list-table} :header-rows: 1 * - Input - Output * - ```` ``` # Demonstrate a code block code: - example: true ``` ```` - ``` # Demonstrate a code block code: - example: true ``` * - ```` ```yaml # Demonstrate a code block code: - example: true ``` ```` - ```yaml # Demonstrate a code block code: - example: true ``` ``` To include back ticks in a code block, increase the number of surrounding back ticks: ```{list-table} :header-rows: 1 * - Input - Output * - ````` ```` ``` ```` ````` - ```` ``` ```` ``` ## Links How to link depends on if you are linking to an external URL or to another page in the documentation. ### External links For external links, use only the URL, or Markdown syntax if you want to override the link text. ```{list-table} :header-rows: 1 * - Input - Output * - `https://linuxcontainers.org/incus` - [{spellexception}`https://linuxcontainers.org/incus`](https://linuxcontainers.org/incus) * - `[Incus](https://linuxcontainers.org/incus)` - [Incus](https://linuxcontainers.org/incus) ``` To display a URL as text and prevent it from being linked, add a ``: ```{list-table} :header-rows: 1 * - Input - Output * - `https://linuxcontainers.org/incus` - {spellexception}`https://linuxcontainers.org/incus` ``` ### Internal references For internal references, both Markdown and MyST syntax are supported. In most cases, you should use MyST syntax though, because it resolves the link text automatically and gives an indication of the link in GitHub rendering. #### Referencing a page To reference a documentation page, use MyST syntax to automatically extract the link text. When overriding the link text, use Markdown syntax. ```{list-table} :header-rows: 1 * - Input - Output - Output on GitHub - Status * - `` {doc}`index` `` - {doc}`index` - {doc}`index` - Preferred. * - `[](index)` - [](index) - - Do not use. * - `[Incus documentation](index)` - [Incus documentation](index) - [Incus documentation](index) - Preferred when overriding the link text. * - `` {doc}`Incus documentation ` `` - {doc}`Incus documentation ` - {doc}`Incus documentation ` - Alternative when overriding the link text. ``` Adhere to the following conventions: - Override the link text only when it is necessary. If you can use the document title as link text, do so, because the text will then update automatically if the title changes. - Never "override" the link text with the same text that would be generated automatically. (a_section_target)= #### Referencing a section To reference a section within the documentation (on the same page or on another page), you can either add a target to it and reference that target, or you can use an automatically generated anchor in combination with the file name. Adhere to the following conventions: - Add targets for sections that are central and a "typical" place to link to, so you expect they will be linked frequently. For "one-off" links, you can use the automatically generated anchors. - Override the link text only when it is necessary. If you can use the section title as link text, do so, because the text will then update automatically if the title changes. - Never "override" the link text with the same text that would be generated automatically. ##### Using a target You can add targets at any place in the documentation. However, if there is no heading or title for the targeted element, you must specify a link text. (a_random_target)= ```{list-table} :header-rows: 1 * - Input - Output - Output on GitHub - Description * - `(target_ID)=` - - \(target_ID\)= - Adds the target ``target_ID``. * - `` {ref}`a_section_target` `` - {ref}`a_section_target` - \{ref\}`a_section_target` - References a target that has a title. * - `` {ref}`link text ` `` - {ref}`link text ` - \{ref\}`link text ` - References a target and specifies a title. * - ``[`option name\](a_random_target)`` - [`option name`](a_random_target) - [`option name`](a_random_target) (link is broken) - Use Markdown syntax if you need markup on the link text. ``` ##### Using an automatically generated anchor You must use Markdown syntax to use automatically generated anchors. You can leave out the file name when linking within the same file. ```{list-table} :header-rows: 1 * - Input - Output - Output on GitHub - Description * - `[](#referencing-a-section)` - [](#referencing-a-section) - - Do not use. * - `[link text](#referencing-a-section)` - [link text](#referencing-a-section) - [link text](#referencing-a-section) - Preferred when overriding the link text. ``` ## Navigation Every documentation page must be included as a subpage to another page in the navigation. This is achieved with the [`toctree`](https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-toctree) directive in the parent page: ```` ```{toctree} :hidden: subpage1 subpage2 ``` ```` If a page should not be included in the navigation, you can suppress the resulting build warning by putting the following instruction at the top of the file: ``` --- orphan: true --- ``` Use orphan pages sparingly and only if there is a clear reason for it. ## Lists ```{list-table} :header-rows: 1 * - Input - Output * - ``` - Item 1 - Item 2 - Item 3 ``` - - Item 1 - Item 2 - Item 3 * - ``` 1. Step 1 1. Step 2 1. Step 3 ``` - 1. Step 1 1. Step 2 1. Step 3 * - ``` 1. Step 1 - Item 1 * Subitem - Item 2 1. Step 2 1. Substep 1 1. Substep 2 ``` - 1. Step 1 - Item 1 * Subitem - Item 2 1. Step 2 1. Substep 1 1. Substep 2 ``` Adhere to the following conventions: - In numbered lists, use ``1.`` for all items to generate the step numbers automatically. - Use `-` for unordered lists. When using nested lists, you can use `*` for the nested level. ### Definition lists ```{list-table} :header-rows: 1 * - Input - Output * - ``` Term 1 : Definition Term 2 : Definition ``` - Term 1 : Definition Term 2 : Definition ``` ## Tables You can use standard Markdown tables. However, using the rST [list table](https://docutils.sourceforge.io/docs/ref/rst/directives.html#list-table) syntax is usually much easier. Both markups result in the following output: ```{list-table} :header-rows: 1 * - Header 1 - Header 2 * - Cell 1 Second paragraph cell 1 - Cell 2 * - Cell 3 - Cell 4 ``` ### Markdown tables ``` | Header 1 | Header 2 | |------------------------------------|----------| | Cell 1

2nd paragraph cell 1 | Cell 2 | | Cell 3 | Cell 4 | ``` ### List tables ```` ```{list-table} :header-rows: 1 * - Header 1 - Header 2 * - Cell 1 2nd paragraph cell 1 - Cell 2 * - Cell 3 - Cell 4 ``` ```` ## Notes ```{list-table} :header-rows: 1 * - Input - Output * - ```` ```{note} A note. ``` ```` - ```{note} A note. ``` * - ```` ```{tip} A tip. ``` ```` - ```{tip} A tip. ``` * - ```` ```{important} Important information ``` ```` - ```{important} Important information. ``` * - ```` ```{caution} This might damage your hardware! ``` ```` - ```{caution} This might damage your hardware! ``` ``` Adhere to the following conventions: - Use notes sparingly. - Only use the following note types: `note`, `tip`, `important`, `caution` - Only use a caution if there is a clear hazard of hardware damage or data loss. ## Images ```{list-table} :header-rows: 1 * - Input - Output * - ``` ![Alt text](https://linuxcontainers.org/incus/docs/main/_static/tag.png) ``` - ![Alt text](https://linuxcontainers.org/incus/docs/main/_static/tag.png) * - ```` ```{figure} https://linuxcontainers.org/incus/docs/main/_static/tag.png :width: 100px :alt: Alt text Figure caption ``` ```` - ```{figure} https://linuxcontainers.org/incus/docs/main/_static/tag.png :width: 100px :alt: Alt text Figure caption ``` ``` Adhere to the following conventions: - For pictures in the `doc` directory, start the path with `/` (for example, `/images/image.png`). - Use PNG format for screenshots and SVG format for graphics. ## Reuse A big advantage of MyST in comparison to plain Markdown is that it allows to reuse content. ### Substitution To reuse sentences or paragraphs without too much markup and special formatting, use substitutions. Substitutions can be defined in the following locations: - In the `substitutions.yaml` file. Substitutions defined in this file are available in all documentation pages. - At the top of a single file in the following format: ```` --- myst: substitutions: reuse_key: "This is **included** text." advanced_reuse_key: "This is a substitution that includes a code block: ``` code block ```" --- ```` You can combine both options by defining a default substitution in `reuse/substitutions.py` and overriding it at the top of a file. ```{list-table} :header-rows: 1 * - Input - Output * - `{{reuse_key}}` - {{reuse_key}} * - `{{advanced_reuse_key}}` - {{advanced_reuse_key}} ``` Adhere to the following convention: - Substitutions do not work on GitHub. Therefore, use key names that indicate the included text (for example, `note_not_supported` instead of `reuse_note`). ### File inclusion To reuse longer sections or text with more advanced markup, you can put the content in a separate file and include the file or parts of the file in several locations. You cannot put any targets into the content that is being reused (because references to this target would be ambiguous then). You can, however, put a target right before including the file. By combining file inclusion and substitutions, you can even replace parts of the included text. `````{list-table} :header-rows: 1 * - Input - Output * - ```` % Include parts of the content from file [../README.md](../README.md) ```{include} ../README.md :start-after: :end-before: ``` ```` - % Include parts of the content from file [../README.md](../README.md) ```{include} ../README.md :start-after: :end-before: ``` ````` Adhere to the following convention: - File inclusion does not work on GitHub. Therefore, always add a comment linking to the included file. - To select parts of the text, add HTML comments for the start and end points and use `:start-after:` and `:end-before:`, if possible. You can combine `:start-after:` and `:end-before:` with `:start-line:` and `:end-line:` if required. Using only `:start-line:` and `:end-line:` is error-prone though. ## Tabs ``````{list-table} :header-rows: 1 * - Input - Output * - ````` ````{tabs} ```{group-tab} Tab 1 Content Tab 1 ``` ```{group-tab} Tab 2 Content Tab 2 ``` ```` ````` - ````{tabs} ```{group-tab} Tab 1 Content Tab 1 ``` ```{group-tab} Tab 2 Content Tab 2 ``` ```` `````` ## Collapsible sections There is no support for details sections in rST, but you can insert HTML to create them. ```{list-table} :header-rows: 1 * - Input - Output * - ```
Details Content
``` -
Details Content
``` ## Glossary You can define glossary terms in any file. Ideally, all terms should be collected in one glossary file though, and they can then be referenced from any file. `````{list-table} :header-rows: 1 * - Input - Output * - ```` ```{glossary} example term Definition of the example term. ``` ```` - ```{glossary} example term Definition of the example term. ``` * - ``{term}`example term` `` - {term}`example term` ````` ## More useful markup `````{list-table} :header-rows: 1 * - Input - Output * - ```` ```{versionadded} X.Y ``` ```` - ```{versionadded} X.Y ``` * - `` {abbr}`API (Application Programming Interface)` `` - {abbr}`API (Application Programming Interface)` ````` ## Custom extensions The documentation uses some custom extensions. ### Related links You can add links to related websites to the sidebar by adding the following field at the top of the page: relatedlinks: https://github.com/canonical/lxd-sphinx-extensions, [RTFM](https://www.google.com) To override the title, use Markdown syntax. Note that spaces are ignored; if you need spaces in the title, replace them with ` `, and include the value in quotes if Sphinx complains about the metadata value because it starts with `[`. To add a link to a Discourse topic, add the following field at the top of the page (where `12345` is the ID of the Discourse topic): discourse: 12345 ### YouTube links To add a link to a YouTube video, use the following directive: `````{list-table} :header-rows: 1 * - Input - Output * - ```` ```{youtube} https://www.youtube.com/watch?v=iMLiK1fX4I0 :title: Demo ``` ```` - ```{youtube} https://www.youtube.com/watch?v=iMLiK1fX4I0 :title: Demo ``` ````` The video title is extracted automatically and displayed when hovering over the link. To override the title, add the `:title:` option. ### Spelling exceptions If you need to use a word that does not comply to the spelling conventions, but is correct in a certain context, you can exempt it from the spelling checker by surrounding it with `{spellexception}`. ```{list-table} :header-rows: 1 * - Input - Output * - `` {spellexception}`PurposelyWrong` `` - {spellexception}`PurposelyWrong` ``` ### Terminal output To show a terminal view with commands and output, use the following directive: `````{list-table} :header-rows: 1 * - Input - Output * - ```` ```{terminal} :input: command number one :user: root :host: vm output line one output line two :input: another command more output ``` ```` - ```{terminal} :input: command number one :user: root :host: vm output line one output line two :input: another command more output ``` ````` Input is specified as the `:input:` option (or prefixed with `:input:` as part of the main content of the directive). Output is the main content of the directive. To override the prompt (`user@host:~$` by default), specify the `:user:` and/or `:host:` options. To make the terminal scroll horizontally instead of wrapping long lines, add `:scroll:`. incus-7.0.0/doc/environment.md000066400000000000000000000071701517523235500163070ustar00rootroot00000000000000# Environment variables The Incus client and daemon respect some environment variables to adapt to the user's environment and to turn some advanced features on and off. ## Common | Name | Description | | :--- | :--- | | `INCUS_DIR` | The Incus data directory | | `INCUS_INSECURE_TLS` | If set to true, allows all default Go ciphers both for client <-> server communication and server <-> image servers (server <-> server and clustering are not affected) | | `PATH` | List of paths to look into when resolving binaries | | `http_proxy` | Proxy server URL for HTTP | | `https_proxy` | Proxy server URL for HTTPS | | `no_proxy` | List of domains, IP addresses or CIDR ranges that don't require the use of a proxy | ## Client environment variable | Name | Description | | :--- | :--- | | `EDITOR` | What text editor to use | | `INCUS_CONF` | Path to the client configuration directory | | `INCUS_GLOBAL_CONF` | Path to the global client configuration directory | | `INCUS_PROJECT` | Name of the project to use (overrides configured default project) | | `INCUS_REMOTE` | Name of the remote to use (overrides configured default remote) | | `VISUAL` | What text editor to use (if `EDITOR` isn't set) | ## Server environment variable | Name | Description | | :--- | :--- | | `INCUS_AGENT_PATH` | Path to the directory including the `incus-agent` builds | | `INCUS_CLUSTER_UPDATE` | Script to call on a cluster update | | `INCUS_DEVMONITOR_DIR` | Path to be monitored by the device monitor. This is primarily for testing | | `INCUS_DOCUMENTATION` | Path to the documentation to serve through the web server | | `INCUS_EDK2_PATH` | Path to EDK2 firmware build including `*_CODE.fd` and `*_VARS.fd` | | `INCUS_EXEC_PATH` | Full path to the Incus binary (used when forking subcommands) | | `INCUS_IDMAPPED_MOUNTS_DISABLE` | Disable idmapped mounts support (useful when testing traditional UID shifting) | | `INCUS_LXC_TEMPLATE_CONFIG` | Path to the LXC template configuration directory | | `INCUS_SECURITY_APPARMOR` | If set to `false`, forces AppArmor off | | `INCUS_SECURITY_SELINUX` | If set to `true`, turns on SELinux integration | | `INCUS_SKIP_INSTANCE_TYPES` | If set to `true`, skip downloading instance type definitions | | `INCUS_UI` | Path to the web UI to serve through the web server | | `INCUS_USBIDS_PATH` | Path to the hwdata `usb.ids` file | incus-7.0.0/doc/events.md000066400000000000000000000637601517523235500152560ustar00rootroot00000000000000# Events ## Introduction Events are messages about actions that have occurred over Incus. Using the API endpoint `/1.0/events` directly or via [`incus monitor`](incus_monitor.md) will connect to a WebSocket through which logs and life-cycle messages will be streamed. ## Event types Incus Currently supports three event types. - `logging`: Shows all logging messages regardless of the server logging level. - `operation`: Shows all ongoing operations from creation to completion (including updates to their state and progress metadata). - `lifecycle`: Shows an audit trail for specific actions occurring over Incus. ## Event structure ### Example ```yaml location: cluster_name metadata: action: network-updated requestor: protocol: unix username: root source: /1.0/networks/incusbr0 timestamp: "2021-03-14T00:00:00Z" type: lifecycle ``` - `location`: The cluster member name (if clustered). - `timestamp`: Time that the event occurred in RFC3339 format. - `type`: The type of event this is (one of `logging`, `operation`, or `lifecycle`). - `metadata`: Information about the specific event type. ### Logging event structure - `message`: The log message. - `level`: The log-level of the log. - `context`: Additional information included in the event. ### Operation event structure - `id`: The UUID of the operation. - `class`: The type of operation (`task`, `token`, or `websocket`). - `description`: A description of the operation. - `created_at`: The operation's creation date. - `updated_at`: The operation's date of last change. - `status`: The current state of the operation. - `status_code`: The operation status code. - `resources`: Resources affected by this operation. - `metadata`: Operation specific metadata. - `may_cancel`: Whether the operation may be canceled. - `err`: Error message of the operation. - `location`: The cluster member name (if clustered). ### Life-cycle event structure - `action`: The life-cycle action that occurred. - `requestor`: Information about who is making the request (if applicable). - `source`: Path to what is being acted upon. - `context`: Additional information included in the event. ## Supported life-cycle events | Name | Description | Additional Information | | :------------------------------------- | :-------------------------------------------------------------------- | :--------------------------------------------------------------------------------------------------- | | `certificate-created` | A new certificate has been added to the server trust store. | | | `certificate-deleted` | The certificate has been deleted from the trust store. | | | `certificate-updated` | The certificate's configuration has been updated. | | | `cluster-certificate-updated` | The certificate for the whole cluster has changed. | | | `cluster-disabled` | Clustering has been disabled for this machine. | | | `cluster-enabled` | Clustering has been enabled for this machine. | | | `cluster-group-created` | A new cluster group has been created. | | | `cluster-group-deleted` | A cluster group has been deleted. | | | `cluster-group-renamed` | A cluster group has been renamed. | | | `cluster-group-updated` | A cluster group has been updated. | | | `cluster-member-added` | A new machine has joined the cluster. | | | `cluster-member-removed` | The cluster member has been removed from the cluster. | | | `cluster-member-renamed` | The cluster member has been renamed. | `old_name`: the previous name. | | `cluster-member-updated` | The cluster member's configuration been edited. | | | `cluster-token-created` | A join token for adding a cluster member has been created. | | | `config-updated` | The server configuration has changed. | | | `image-alias-created` | An alias has been created for an existing image. | `target`: the original instance. | | `image-alias-deleted` | An alias has been deleted for an existing image. | `target`: the original instance. | | `image-alias-renamed` | The alias for an existing image has been renamed. | `old_name`: the previous name. | | `image-alias-updated` | The configuration for an image alias has changed. | `target`: the original instance. | | `instance-agent-started` | The instance agent has connected to the host. | | | `instance-agent-stopped` | The instance agent has disconnected from the host. | | | `image-created` | A new image has been added to the image store. | `type`: `container` or `vm`. | | `image-deleted` | The image has been deleted from the image store. | | | `image-refreshed` | The local image copy has updated to the current source image version. | | | `image-retrieved` | The raw image file has been downloaded from the server. | `target`: destination server. | | `image-secret-created` | A one-time key to fetch this image has been created. | | | `image-updated` | The image's configuration has changed. | | | `instance-backup-created` | A backup of the instance has been created. | | | `instance-backup-deleted` | The instance backup has been deleted. | | | `instance-backup-renamed` | The instance backup has been renamed. | `old_name`: the previous name. | | `instance-backup-retrieved` | The raw instance backup file has been downloaded. | | | `instance-console` | Connected to the console of the instance. | `type`: `console` or `vga`. | | `instance-console-reset` | The console buffer has been reset. | | | `instance-console-retrieved` | The console log has been downloaded. | | | `instance-created` | A new instance has been created. | | | `instance-deleted` | The instance has been deleted. | | | `instance-exec` | A command has been executed on the instance. | `command`: the command to be executed. | | `instance-file-deleted` | A file on the instance has been deleted. | `file`: path to the file. | | `instance-file-pushed` | The file has been pushed to the instance. | `file-source`: local file path. `file-destination`: destination file path. `info`: file information. | | `instance-file-retrieved` | The file has been downloaded from the instance. | `file-source`: instance file path. `file-destination`: destination file path. | | `instance-log-deleted` | The instance's specified log file has been deleted. | | | `instance-log-retrieved` | The instance's specified log file has been downloaded. | | | `instance-metadata-retrieved` | The instance's image metadata has been downloaded. | | | `instance-metadata-template-created` | A new image template file for the instance has been created. | `path`: relative file path. | | `instance-metadata-template-deleted` | The image template file for the instance has been deleted. | `path`: relative file path. | | `instance-metadata-template-retrieved` | The image template file for the instance has been downloaded. | `path`: relative file path. | | `instance-metadata-updated` | The instance's image metadata has changed. | | | `instance-paused` | The instance has been put in a paused state. | | | `instance-ready` | The instance is ready. | | | `instance-renamed` | The instance has been renamed. | `old_name`: the previous name. | | `instance-restarted` | The instance has restarted. | | | `instance-restored` | The instance has been restored from a snapshot. | `snapshot`: name of the snapshot being restored. | | `instance-resumed` | The instance has resumed after being paused. | | | `instance-shutdown` | The instance has shut down. | | | `instance-snapshot-created` | A snapshot of the instance has been created. | | | `instance-snapshot-deleted` | The instance snapshot has been deleted. | | | `instance-snapshot-renamed` | The instance snapshot has been renamed. | `old_name`: the previous name. | | `instance-snapshot-updated` | The instance snapshot's configuration has changed. | | | `instance-started` | The instance has started. | | | `instance-stopped` | The instance has stopped. | | | `instance-updated` | The instance's configuration has changed. | | | `network-acl-created` | A new network ACL has been created. | | | `network-acl-deleted` | The network ACL has been deleted. | | | `network-acl-renamed` | The network ACL has been renamed. | `old_name`: the previous name. | | `network-acl-updated` | The network ACL configuration has changed. | | | `network-created` | A network device has been created. | | | `network-deleted` | The network device has been deleted. | | | `network-forward-created` | A new network forward has been created. | | | `network-forward-deleted` | The network forward has been deleted. | | | `network-forward-updated` | The network forward has been updated. | | | `network-peer-created` | A new network peer has been created. | | | `network-peer-deleted` | The network peer has been deleted. | | | `network-peer-updated` | The network peer has been updated. | | | `network-renamed` | The network device has been renamed. | `old_name`: the previous name. | | `network-updated` | The network device's configuration has changed. | | | `network-zone-created` | A new network zone has been created. | | | `network-zone-deleted` | The network zone has been deleted. | | | `network-zone-record-created` | A new network zone record has been created. | | | `network-zone-record-deleted` | The network zone record has been deleted. | | | `network-zone-record-updated` | The network zone record has been updated. | | | `network-zone-updated` | The network zone has been updated. | | | `operation-cancelled` | The operation has been canceled. | | | `profile-created` | A new profile has been created. | | | `profile-deleted` | The profile has been deleted. | | | `profile-renamed` | The profile has been renamed . | `old_name`: the previous name. | | `profile-updated` | The profile's configuration has changed. | | | `project-created` | A new project has been created. | | | `project-deleted` | The project has been deleted. | | | `project-renamed` | The project has been renamed. | `old_name`: the previous name. | | `project-updated` | The project's configuration has changed. | | | `storage-pool-created` | A new storage pool has been created. | `target`: cluster member name. | | `storage-pool-deleted` | The storage pool has been deleted. | | | `storage-pool-updated` | The storage pool's configuration has changed. | `target`: cluster member name. | | `storage-volume-backup-created` | A new backup for the storage volume has been created. | `type`: `container`, `virtual-machine`, `image`, or `custom`. | | `storage-volume-backup-deleted` | The storage volume's backup has been deleted. | | | `storage-volume-backup-renamed` | The storage volume's backup has been renamed. | `old_name`: the previous name. | | `storage-volume-backup-retrieved` | The storage volume's backup has been downloaded. | | | `storage-volume-created` | A new storage volume has been created. | `type`: `container`, `virtual-machine`, `image`, or `custom`. | | `storage-volume-deleted` | The storage volume has been deleted. | | | `storage-volume-renamed` | The storage volume has been renamed. | `old_name`: the previous name. | | `storage-volume-restored` | The storage volume has been restored from a snapshot. | `snapshot`: name of the snapshot being restored. | | `storage-volume-snapshot-created` | A new storage volume snapshot has been created. | `type`: `container`, `virtual-machine`, `image`, or `custom`. | | `storage-volume-snapshot-deleted` | The storage volume's snapshot has been deleted. | | | `storage-volume-snapshot-renamed` | The storage volume's snapshot has been renamed. | `old_name`: the previous name. | | `storage-volume-snapshot-updated` | The configuration for the storage volume's snapshot has changed. | | | `storage-volume-updated` | The storage volume's configuration has changed. | | | `warning-acknowledged` | The warning's status has been set to "acknowledged". | | | `warning-deleted` | The warning has been deleted. | | | `warning-reset` | The warning's status has been set to "new". | | incus-7.0.0/doc/explanation/000077500000000000000000000000001517523235500157365ustar00rootroot00000000000000incus-7.0.0/doc/explanation/bpf-tokens.md000066400000000000000000000041061517523235500203310ustar00rootroot00000000000000(bpf-tokens)= # BPF token delegation Incus supports delegating BPF capabilities via [BPF tokens](https://docs.ebpf.io/linux/concepts/token/), introduced in Linux kernel 6.9. If any of the instance options {config:option}`instance-security:security.bpffs.delegate_cmds`, {config:option}`instance-security:security.bpffs.delegate_maps`, {config:option}`instance-security:security.bpffs.delegate_progs` or {config:option}`instance-security:security.bpffs.delegate_attachs` is set, Incus mounts a BPF file system into the container at the path specified by the {config:option}`instance-security:security.bpffs.path` option and delegates the configured capabilities to it. The permissible values for these options depend on the kernel version and can be found in `enums` in the BPF header file (`include/uapi/linux/bpf.h` in the kernel tree, `/usr/include/linux/bpf.h` on most distributions if you have the kernel sources installed): | Key | Kernel `enum` | Remove prefix | | :--- | :--- | :--- | | `security.bpffs.delegate_cmds` | `bpf_cmd` | `BPF_` | | `security.bpffs.delegate_maps` | `bpf_map_type` | `BPF_MAP_TYPE_` | | `security.bpffs.delegate_progs` | `bpf_prog_type` | `BPF_PROG_TYPE_` | | `security.bpffs.delegate_attachs` | `bpf_attach_type` | `BPF_` | Each of these options takes a comma-separated list of values, additionally the value `any` is supported to delegate all possible values of the type. ## Example | Key | Value | | :--- | :--- | | `security.bpffs.delegate_cmds` | `map_create,obj_get,link_create` | | `security.bpffs.delegate_maps` | `hash,array,devmap,queue,stack` | | `security.bpffs.delegate_progs` | `socket_filter,kprobe,cgroup_sysctl` | | `security.bpffs.delegate_attachs` | `any` | ```bash $ mount -t bpf none on /sys/fs/bpf type bpf (rw,relatime,delegate_cmds=map_create:obj_get:link_create,delegate_maps=hash:array:devmap:queue:stack,delegate_progs=socket_filter:kprobe:cgroup_sysctl,delegate_attachs=any) ``` incus-7.0.0/doc/explanation/clustering.md000066400000000000000000000354311517523235500204450ustar00rootroot00000000000000(exp-clustering)= # About clustering To spread the total workload over several servers, Incus can be run in clustering mode. In this scenario, any number of Incus servers share the same distributed database that holds the configuration for the cluster members and their instances. The Incus cluster can be managed uniformly using the [`incus`](incus.md) client or the REST API. (clustering-members)= ## Cluster members An Incus cluster consists of one bootstrap server and at least two further cluster members. It stores its state in a [distributed database](../database.md), which is a [Cowsql](https://github.com/cowsql/cowsql/) database replicated using the Raft algorithm. While you could create a cluster with only two members, it is strongly recommended that the number of cluster members be at least three. With this setup, the cluster can survive the loss of at least one member and still be able to establish quorum for its distributed state. When you create the cluster, the Cowsql database runs on only the bootstrap server until a third member joins the cluster. Then both the second and the third server receive a replica of the database. See {ref}`cluster-form` for more information. (clustering-member-roles)= ### Member roles In a cluster with three members, all members replicate the distributed database that stores the state of the cluster. If the cluster has more members, only some of them replicate the database. The remaining members have access to the database, but don't replicate it. At each time, there is an elected cluster leader that monitors the health of the other members. Each member that replicates the database has either the role of a *voter* or of a *stand-by*. If the cluster leader goes offline, one of the voters is elected as the new leader. If a voter member goes offline, a stand-by member is automatically promoted to voter. The database (and hence the cluster) remains available as long as a majority of voters is online. The following roles can be assigned to Incus cluster members. Automatic roles are assigned by Incus itself and cannot be modified by the user. | Role | Automatic | Description | | :--- | :-------- | :---------- | | `database` | yes | Voting member of the distributed database | | `database-client` | no | Prevents the affected cluster member from being elected as a voter or stand-by| | `database-leader` | yes | Current leader of the distributed database | | `database-standby` | yes | Stand-by (non-voting) member of the distributed database | | `event-hub` | no | Exchange point (hub) for the internal Incus events (requires at least two) | | `ovn-chassis` | no | Uplink gateway candidate for OVN networks | The default number of voter members ({config:option}`server-cluster:cluster.max_voters`) is three. The default number of stand-by members ({config:option}`server-cluster:cluster.max_standby`) is two. With this configuration, your cluster will remain operational as long as you switch off at most one voting member at a time. See {ref}`cluster-manage` for more information. (clustering-offline-members)= #### Offline members and fault tolerance If a cluster member is down for more than the configured offline threshold, its status is marked as offline. In this case, no operations are possible on this member, and neither are operations that require a state change across all members. As soon as the offline member comes back online, operations are available again. If the member that goes offline is the leader itself, the other members will elect a new leader. If you can't or don't want to bring the server back online, you can [delete it from the cluster](cluster-manage-delete-members). You can tweak the amount of seconds after which a non-responding member is considered offline by setting the {config:option}`server-cluster:cluster.offline_threshold` configuration. The default value is 20 seconds. The minimum value is 10 seconds. To automatically {ref}`evacuate ` instances from an offline member, set the {config:option}`server-cluster:cluster.healing_threshold` configuration to a non-zero value. See {ref}`cluster-recover` for more information. #### Failure domains You can use failure domains to indicate which cluster members should be given preference when assigning roles to a cluster member that has gone offline. For example, if a cluster member that currently has the database role gets shut down, Incus tries to assign its database role to another cluster member in the same failure domain, if one is available. To update the failure domain of a cluster member, use the [`incus cluster edit `](incus_cluster_edit.md) command and change the `failure_domain` property from `default` to another string. (clustering-member-config)= ### Member configuration Incus cluster members are generally assumed to be identical systems. This means that all Incus servers joining a cluster must have an identical configuration to the bootstrap server, in terms of storage pools and networks. To accommodate things like slightly different disk ordering or network interface naming, there is an exception for some configuration options related to storage and networks, which are member-specific. When such settings are present in a cluster, any server that is being added must provide a value for them. Most often, this is done through the interactive `incus admin init` command, which asks the user for the value for a number of configuration keys related to storage or networks. Those settings typically include: - The source device and size for a storage pool - The name for a ZFS zpool, LVM thin pool or LVM volume group - External interfaces and BGP next-hop for a bridged network - The name of the parent network device for managed `physical` or `macvlan` networks See {ref}`cluster-config-storage` and {ref}`cluster-config-networks` for more information. If you want to look up the questions ahead of time (which can be useful for scripting), query the `/1.0/cluster` API endpoint. This can be done through `incus query /1.0/cluster` or through other API clients. ## Images By default, Incus replicates images on as many cluster members as there are database members. This typically means up to three copies within the cluster. You can increase that number to improve fault tolerance and the likelihood of the image being locally available. To do so, set the {config:option}`server-cluster:cluster.images_minimal_replica` configuration. The special value of `-1` can be used to have the image copied to all cluster members. (cluster-groups)= ## Cluster groups In an Incus cluster, you can add members to cluster groups. You can use these cluster groups to launch instances on a cluster member that belongs to a subset of all available members. For example, you could create a cluster group for all members that have a GPU and then launch all instances that require a GPU on this cluster group. By default, all cluster members belong to the `default` group. See {ref}`howto-cluster-groups` and {ref}`cluster-target-instance` for more information. (cluster-cpu)= ## Cluster CPU baseline For live-migration of virtual machines to be possible, the CPU of the target server must have at least the same capabilities as the source CPU. To achieve that, Incus automatically scans CPU features of all servers within a cluster and tries to calculate a baseline set of CPU capabilities. The guest then gets exposed a virtual QEMU CPU with those capabilities exposed. Homogeneous clusters can significantly benefit from turning this behavior off and instead directly exposing the host CPU to the guest with: incus cluster group set default instances.vm.cpu.x86_64.baseline=host ```{note} The automated baseline may not work in all environments, especially when mixing CPU generations or CPU vendors. For mixed environment, creating a cluster group per platform is recommended. ``` (clustering-instance-placement)= ## Automatic placement of instances In a cluster setup, each instance lives on one of the cluster members. When you launch an instance, you can target it to a specific cluster member, to a cluster group or have Incus automatically assign it to a cluster member. By default, the automatic assignment picks the cluster member that has the lowest number of instances. If several members have the same amount of instances, one of the members is chosen at random. However, you can control this behavior with the {config:option}`cluster-cluster:scheduler.instance` configuration option: - If `scheduler.instance` is set to `all` for a cluster member, this cluster member is selected for an instance if: - The instance is created without `--target` and the cluster member has the lowest number of instances. - The instance is targeted to live on this cluster member. - The instance is targeted to live on a member of a cluster group that the cluster member is a part of, and the cluster member has the lowest number of instances compared to the other members of the cluster group. - If `scheduler.instance` is set to `manual` for a cluster member, this cluster member is selected for an instance if: - The instance is targeted to live on this cluster member. - If `scheduler.instance` is set to `group` for a cluster member, this cluster member is selected for an instance if: - The instance is targeted to live on this cluster member. - The instance is targeted to live on a member of a cluster group that the cluster member is a part of, and the cluster member has the lowest number of instances compared to the other members of the cluster group. (clustering-instance-placement-scriptlet)= ### Instance placement scriptlet Incus supports using custom logic to control automatic instance placement by using an embedded script (scriptlet). This method provides more flexibility than the built-in instance placement functionality. The instance placement scriptlet must be written in the [Starlark language](https://github.com/bazelbuild/starlark) (which is a subset of Python). The scriptlet is invoked each time Incus needs to know where to place an instance. The scriptlet receives information about the instance that is being placed and the candidate cluster members that could host the instance. It is also possible for the scriptlet to request information about each candidate cluster member's state and the hardware resources available. An instance placement scriptlet must implement the `instance_placement` function with the following signature: `instance_placement(request, candidate_members)`: - `request` is an object that contains an expanded representation of [`scriptlet.InstancePlacement`](https://pkg.go.dev/github.com/lxc/incus/shared/api/scriptlet/#InstancePlacement). This request includes `project` and `reason` fields. The `reason` can be `new`, `evacuation`, `relocation` or `rebalance`. - `candidate_members` is a `list` of cluster member objects representing [`api.ClusterMember`](https://pkg.go.dev/github.com/lxc/incus/shared/api#ClusterMember) entries. When the reason is `rebalance`, the list contains all compatible cluster members other than the source, sorted from least to most loaded. For example: ```python def instance_placement(request, candidate_members): # Example of logging info, this will appear in Incus' log. log_info("instance placement started: ", request) # Example of applying logic based on the instance request. if request.name == "foo": # Example of logging an error, this will appear in Incus' log. log_error("Invalid name supplied: ", request.name) fail("Invalid name") # Exit with an error to reject instance placement. # Place the instance on the first candidate server provided. set_target(candidate_members[0].server_name) return # Return empty to allow instance placement to proceed. ``` The scriptlet must be applied to Incus by storing it in the `instances.placement.scriptlet` global configuration setting. For example, if the scriptlet is saved inside a file called `instance_placement.star`, then it can be applied to Incus with the following command: cat instance_placement.star | incus config set instances.placement.scriptlet=- To see the current scriptlet applied to Incus, use the `incus config get instances.placement.scriptlet` command. The following functions are available to the scriptlet (in addition to those provided by Starlark): - `log_info(*messages)`: Add a log entry to Incus' log at `info` level. `messages` is one or more message arguments. - `log_warn(*messages)`: Add a log entry to Incus' log at `warn` level. `messages` is one or more message arguments. - `log_error(*messages)`: Add a log entry to Incus' log at `error` level. `messages` is one or more message arguments. - `set_target(member_name)`: Set the cluster member where the instance should be created. `member_name` is the name of the cluster member the instance should be created on. If this function is not called, then Incus will use its built-in instance placement logic. - `get_cluster_member_resources(member_name)`: Get information about resources on the cluster member. Returns an object with the resource information in the form of [`api.Resources`](https://pkg.go.dev/github.com/lxc/incus/shared/api#Resources). `member_name` is the name of the cluster member to get the resource information for. - `get_cluster_member_state(member_name)`: Get the cluster member's state. Returns an object with the cluster member's state in the form of [`api.ClusterMemberState`](https://pkg.go.dev/github.com/lxc/incus/shared/api#ClusterMemberState). `member_name` is the name of the cluster member to get the state for. - `get_instance_resources()`: Get information about the resources the instance will require. Returns an object with the resource information in the form of [`scriptlet.InstanceResources`](https://pkg.go.dev/github.com/lxc/incus/shared/api/scriptlet/#InstanceResources). - `get_instances(location, project)`: Get a list of instances based on project and/or location filters. Returns the list of instances in the form of [`[]api.Instance`](https://pkg.go.dev/github.com/lxc/incus/shared/api#Instance). - `get_instances_count(location, project, pending)`: Get a count of the instances based on project and/or location filters. The count may include instances currently being created for which no database record exists yet.. - `get_cluster_members(group)`: Get a list of cluster members based on the cluster group. Returns the list of cluster members in the form of [`[]api.ClusterMember`](https://pkg.go.dev/github.com/lxc/incus/shared/api#ClusterMember). - `get_project(name)`: Get a project object based on the project name. Returns a project object in the form of [`api.Project`](https://pkg.go.dev/github.com/lxc/incus/shared/api#Project). ```{note} Field names in the object types are equivalent to the JSON field names in the associated Go types. ``` incus-7.0.0/doc/explanation/containers_and_vms.md000066400000000000000000000066171517523235500221460ustar00rootroot00000000000000(containers-and-vms)= # About containers and VMs Incus provides support for two different types of {ref}`instances `: *system containers* and *virtual machines*. Incus uses features of the Linux kernel (such as `namespaces` and `cgroups`) in the implementation of system containers. These features provide a software-only way to isolate and restrict a running system container. A system container can only be based on the Linux kernel. When running a virtual machine, Incus uses hardware features of the the host system as a way to isolate and restrict a running virtual machine. Therefore, virtual machines can be used to run, for example, different operating systems than the host system. | Virtual Machines | Application Containers | System Containers | | :--- | :--- | :--- | | Uses a dedicated kernel | Uses the kernel of the host | Uses the kernel of the host | | Can host different types of OS | Can only host Linux | Can only host Linux | | Uses more resources | Uses less resources | Uses less resources | | Requires hardware virtualization | Software-only | Software-only | | Can host multiple applications | Can host a single app | Can host multiple applications | | Supported by Incus | Supported by Docker | Supported by Incus | ## Application containers vs. system containers Application containers (as provided by, for example, Docker) package a single process or application. System containers, on the other hand, simulate a full operating system similar to what you would be running on a host or in a virtual machine. You can run Docker in an Incus system container, but you would not run Incus in a Docker application container. Therefore, application containers are suitable to provide separate components, while system containers provide a full solution of libraries, applications, databases and so on. In addition, you can use system containers to create different user spaces and isolate all processes belonging to each user space, which is not what application containers are intended for. ![Application and system containers](/images/application-vs-system-containers.svg "Application and system containers") ## Virtual machines vs. system containers Virtual machines create a virtual version of a physical machine, using hardware features of the host system. The boundaries between the host system and virtual machines is enforced by those hardware features. System containers, on the other hand, use the already running OS kernel of the host system instead of launching their own kernel. If you run several system containers, they all share the same kernel, which makes them faster and more lightweight than virtual machines. With Incus, you can create both system containers and virtual machines. You should use a system container to leverage the smaller size and increased performance if all functionality you require is compatible with the kernel of your host operating system. If you need functionality that is not supported by the OS kernel of your host system or you want to run a completely different OS, use a virtual machine. ![Virtual machines and system containers](/images/virtual-machines-vs-system-containers.svg "Virtual machines and system containers") incus-7.0.0/doc/explanation/instance_config.md000066400000000000000000000044441517523235500214170ustar00rootroot00000000000000(instance-config)= # Instance configuration The instance configuration consists of different categories: Instance properties : Instance properties are specified when the instance is created. They include, for example, the instance name and architecture. Some of the properties are read-only and cannot be changed after creation, while others can be updated by {ref}`setting their property value ` or {ref}`editing the full instance configuration `. In the YAML configuration, properties are on the top level. See {ref}`instance-properties` for a reference of available instance properties. Instance options : Instance options are configuration options that are related directly to the instance. They include, for example, startup options, security settings, hardware limits, kernel modules, snapshots and user keys. These options can be specified as key/value pairs during instance creation (through the `--config key=value` flag). After creation, they can be configured with the [`incus config set`](incus_config_set.md) and [`incus config unset`](incus_config_unset.md) commands. In the YAML configuration, options are located under the `config` entry. See {ref}`instance-options` for a reference of available instance options, and {ref}`instances-configure-options` for instructions on how to configure the options. Instance devices : Instance devices are attached to an instance. They include, for example, network interfaces, mount points, USB and GPU devices. Devices are usually added after an instance is created with the [`incus config device add`](incus_config_device_add.md) command, but they can also be added to a profile or a YAML configuration file that is used to create an instance. Each type of device has its own specific set of options, referred to as *instance device options*. In the YAML configuration, devices are located under the `devices` entry. See {ref}`devices` for a reference of available devices and the corresponding instance device options, and {ref}`instances-configure-devices` for instructions on how to add and configure instance devices. ```{toctree} :maxdepth: 1 :hidden: ../reference/instance_properties.md ../reference/instance_options.md ../reference/devices.md ../reference/instance_units.md ``` incus-7.0.0/doc/explanation/instances.md000066400000000000000000000043021517523235500202460ustar00rootroot00000000000000(expl-instances)= # About instances Incus supports the following types of instances: Systems Containers : System containers run full Linux distributions using a shared kernel. Those containers run a full Linux distribution, very similar to a virtual machine but sharing kernel with the host system. They have an extremely low overhead, can be packed very densely and generally provide a near identical experience to virtual machines without the required hardware support and overhead. System containers are implemented through the use of `liblxc` (LXC). Application containers : Application containers run a single application through a pre-built image. Those kind of containers got popularized by the likes of Docker and Kubernetes. Rather than provide a pristine Linux environment on top of which software needs to be installed, they instead come with a pre-installed and mostly pre-configured piece of software. Incus can consume application container images from any OCI-compatible image registry (e.g. the Docker Hub). Application containers are implemented through the use of `liblxc` (LXC) with help from `umoci` and `skopeo`. Virtual machines : {abbr}`VMs (Virtual machines)` are a full virtualized system. Virtual machines are also natively supported by Incus and provide an alternative to system containers. Not everything can run properly in containers. Anything that requires a different kernel or its own kernel modules should be run in a virtual machine instead. Similarly, some kind of device pass-through, such as full PCI devices will only work properly with virtual machines. To keep the user experience consistent, a built-in agent is provided by Incus to allow for interactive command execution and file transfers. Virtual machines are implemented through the use of QEMU. ```{note} Currently, virtual machines support fewer features than containers, but the plan is to support the same set of features for both instance types in the future. To see which features are available for virtual machines, check the condition column in the {ref}`instance-options` documentation. ``` See {ref}`containers-and-vms` for more information about the different instance types. incus-7.0.0/doc/explanation/networks.md000066400000000000000000000142711517523235500201410ustar00rootroot00000000000000(networks)= # About networking There are different ways to connect your instances to the Internet. The easiest method is to have Incus create a network bridge during initialization and use this bridge for all instances, but Incus supports many different and advanced setups for networking. ## Network devices To grant direct network access to an instance, you must assign it at least one network device, also called {abbr}`NIC (Network Interface Controller)`. You can configure the network device in one of the following ways: - Use the default network bridge that you set up during the Incus initialization. Check the default profile to see the default configuration: incus profile show default This method is used if you do not specify a network device for your instance. - Use an existing network interface by adding it as a network device to your instance. This network interface is outside of Incus control. Therefore, you must specify all information that Incus needs to use the network interface. Use a command similar to the following: incus config device add nic nictype= ... See [Type: `nic`](devices-nic) for a list of available NIC types and their configuration properties. For example, you could add a pre-existing Linux bridge (`br0`) with the following command: incus config device add eth0 nic nictype=bridged parent=br0 - {doc}`Create a managed network ` and add it as a network device to your instance. With this method, Incus has all required information about the configured network, and you can directly attach it to your instance as a device: incus network attach See {ref}`network-attach` for more information. (managed-networks)= ## Managed networks Managed networks in Incus are created and configured with the `incus network [create|edit|set]` command. Depending on the network type, Incus either fully controls the network or just manages an external network interface. Note that not all {ref}`NIC types ` are supported as network types. Incus can only set up some of the types as managed networks. ### Fully controlled networks Fully controlled networks create network interfaces and provide most functionality, including, for example, the ability to do IP management. Incus supports the following network types: {ref}`network-bridge` : % Include content from [../reference/network_bridge.md](../reference/network_bridge.md) ```{include} ../reference/network_bridge.md :start-after: :end-before: ``` In Incus context, the `bridge` network type creates an L2 bridge that connects the instances that use it together into a single network L2 segment. This makes it possible to pass traffic between the instances. The bridge can also provide local DHCP and DNS. This is the default network type. {ref}`network-ovn` : % Include content from [../reference/network_ovn.md](../reference/network_ovn.md) ```{include} ../reference/network_ovn.md :start-after: :end-before: ``` In Incus context, the `ovn` network type creates a logical network. To set it up, you must install and configure the OVN tools. In addition, you must create an uplink network that provides the network connection for OVN. As the uplink network, you should use one of the external network types or a managed Incus bridge. ```{tip} Unlike the other network types, you can create and manage an OVN network inside a {ref}`project `. This means that you can create your own OVN network as a non-admin user, even in a restricted project. ``` ### External networks % Include content from [../reference/network_external.md](../reference/network_external.md) ```{include} ../reference/network_external.md :start-after: :end-before: ``` {ref}`network-macvlan` : % Include content from [../reference/network_macvlan.md](../reference/network_macvlan.md) ```{include} ../reference/network_macvlan.md :start-after: :end-before: ``` In Incus context, the `macvlan` network type provides a preset configuration to use when connecting instances to a parent macvlan interface. {ref}`network-sriov` : % Include content from [../reference/network_sriov.md](../reference/network_sriov.md) ```{include} ../reference/network_sriov.md :start-after: :end-before: ``` In Incus context, the `sriov` network type provides a preset configuration to use when connecting instances to a parent SR-IOV interface. {ref}`network-physical` : % Include content from [../reference/network_physical.md](../reference/network_physical.md) ```{include} ../reference/network_physical.md :start-after: :end-before: ``` It provides a preset configuration to use when connecting OVN networks to a parent interface. ## Recommendations In general, if you can use a managed network, you should do so because networks are easy to configure and you can reuse the same network for several instances without repeating the configuration. Which network type to choose depends on your specific use case. If you choose a fully controlled network, it provides more functionality than using a network device. As a general recommendation: - If you are running Incus on a single system or in a public cloud, use a {ref}`network-bridge`. - If you are running Incus in your own private cloud, use an {ref}`network-ovn`. ```{note} OVN requires a shared L2 uplink network for proper operation. Therefore, using OVN is usually not possible if you run Incus in a public cloud. ``` - To connect an instance NIC to a managed network, use the `network` property rather than the `parent` property, if possible. This way, the NIC can inherit the settings from the network and you don't need to specify the `nictype`. incus-7.0.0/doc/explanation/performance_tuning.md000066400000000000000000000045451517523235500221550ustar00rootroot00000000000000(performance-tuning)= # About performance tuning When you are ready to move your Incus setup to production, you should take some time to optimize the performance of your system. There are different aspects that impact performance. The following steps help you to determine the choices and settings that you should tune to improve your Incus setup. ## Run benchmarks Incus provides a benchmarking tool to evaluate the performance of your system. You can use the tool to initialize or launch a number of containers and measure the time it takes for the system to create the containers. By running the tool repeatedly with different Incus configurations, system settings or even hardware setups, you can compare the performance and evaluate which is the ideal configuration. See {ref}`benchmark-performance` for instructions on running the tool. ## Monitor instance metrics % Include content from [../metrics.md](../metrics.md) ```{include} ../metrics.md :start-after: :end-before: ``` You should regularly monitor the metrics to evaluate the resources that your instances use. The numbers help you to determine if there are any spikes or bottlenecks, or if usage patterns change and require updates to your configuration. See {ref}`metrics` for more information about metrics collection. ## Tune server settings The default kernel settings for most Linux distributions are not optimized for running a large number of containers or virtual machines. Therefore, you should check and modify the relevant server settings to avoid hitting limits caused by the default settings. Typical errors that you might see when you encounter those limits are: * `Failed to allocate directory watch: Too many open files` * ` : Too many open files` * `failed to open stream: Too many open files in...` * `neighbour: ndisc_cache: neighbor table overflow!` See {ref}`server-settings` for a list of relevant server settings and suggested values. ## Tune the network bandwidth If you have a lot of local activity between instances or between the Incus host and the instances, or if you have a fast internet connection, you should consider increasing the network bandwidth of your Incus setup. You can do this by increasing the transmit and receive queue lengths. See {ref}`network-increase-bandwidth` for instructions. incus-7.0.0/doc/explanation/projects.md000066400000000000000000000121161517523235500201120ustar00rootroot00000000000000(exp-projects)= # About projects You can use projects to keep your Incus server clean by grouping related instances together. In addition to isolated instances, each project can also have specific images, profiles, networks, and storage. For example, projects can be useful in the following scenarios: - You run a huge number of instances for different purposes, for example, for different customer projects. You want to keep these instances separate to make it easier to locate and maintain them, and you might want to reuse the same instance names in each customer project for consistency reasons. Each instance in a customer project should use the same base configuration (for example, networks and storage), but the configuration might differ between customer projects. In this case, you can create an Incus project for each customer project (thus each group of instances) and use different profiles, networks, and storage for each Incus project. - Your Incus server is shared between multiple users. Each user runs their own instances, and might want to configure their own profiles. You want to keep the user instances confined, so that each user can interact only with their own instances and cannot see the instances created by other users. In addition, you want to be able to limit resources for each user and make sure that the instances of different users cannot interfere with one another. In this case, you can set up a multi-user environment with confined projects. Incus comes with a `default` project. See {ref}`projects-create` for instructions on how to add projects. (projects-isolation)= ## Isolation of projects Projects always encapsulate the instances they contain, which means that instances cannot be shared between projects and instance names can be duplicated in several projects. When you are in a specific project, you can see only the instances that belong to this project. Other entities (images, profiles, networks, and storage) can be either isolated in the project or inherited from the `default` project. To configure which entities are isolated, you enable or disable the respective *feature* in the project. If a feature is enabled, the corresponding entity is isolated in the project; if the feature is disabled, it is inherited from the `default` project. For example, if you enable {config:option}`project-features:features.networks` for a project, the project uses a separate set of networks and not the networks defined in the `default` project. If you disable {config:option}`project-features:features.images`, the project has access to the images defined in the `default` project, and any images you add while you're using the project are also added to the `default` project. See the list of available {ref}`project-features` for information about which features are enabled or disabled when you create a project. ```{note} You must select the features that you want to enable before starting to use a new project. When a project contains instances, the features are locked. To edit them, you must remove all instances first. New features that are added in an upgrade are disabled for existing projects. ``` (projects-confined)= ## Confined projects in a multi-user environment If your Incus server is used by multiple users (for example, in a lab environment), you can use projects to confine the activities of each user. This method isolates the instances and other entities (depending on the feature configuration), as described in {ref}`projects-isolation`. It also confines users to their own user space and prevents them from gaining access to other users' instances or data. Any changes that affect the Incus server and its configuration, for example, adding or removing storage, are not permitted. In addition, this method allows users to work with Incus without being a member of the `incus-admin` group (see {ref}`security-daemon-access`). Members of the `incus-admin` group have full access to Incus, including permission to attach file system paths and tweak the security features of an instance, which makes it possible to gain root access to the host system. Using confined projects limits what users can do in Incus, but it also prevents users from gaining root access. ### Authentication methods for projects There are different ways of authentication that you can use to confine projects to specific users: Client certificates : You can restrict the {ref}`authentication-tls-certs` to allow access to specific projects only. The projects must exist before you can restrict access to them. A client that connects using a restricted certificate can see only the project or projects that the client has been granted access to. Multi-user Incus daemon : A multi-user Incus daemon allows dynamic project creation on a per-user basis. This is usually used for users that are a member of the `incus` group but aren't in the more privileged `incus-admin` group. When a user that is a member of this group starts using Incus, Incus automatically creates a confined project for this user. See {ref}`projects-confine` for instructions on how to enable and configure the different authentication methods. incus-7.0.0/doc/explanation/security.md000066400000000000000000000216441517523235500201360ustar00rootroot00000000000000(exp-security)= # About security % Include content from [../../README.md](../../README.md) ```{include} ../../README.md :start-after: :end-before: ``` See the following sections for detailed information. If you discover a security issue, see the [Incus security policy](https://github.com/lxc/incus/blob/main/SECURITY.md) for information on how to report the issue. ## Supported versions Never use unsupported Incus versions in a production environment. % Include content from [../../SECURITY.md](../../SECURITY.md) ```{include} ../../SECURITY.md :start-after: :end-before: ``` (security-daemon-access)= ## Access to the Incus daemon Incus is a daemon that can be accessed locally over a Unix socket or, if configured, remotely over a {abbr}`TLS (Transport Layer Security)` socket. Anyone with access to the socket can fully control Incus, which includes the ability to attach host devices and file systems or to tweak the security features for all instances. Therefore, make sure to restrict the access to the daemon to trusted users. ### Local access to the Incus daemon The Incus daemon runs as root and provides a Unix socket for local communication. Access control for Incus is based on group membership. The root user and all members of the `incus-admin` group can interact with the local daemon. ````{important} % Include content from [../../README.md](../../README.md) ```{include} ../../README.md :start-after: :end-before: ``` ```` (security_remote_access)= ### Access to the remote API By default, access to the daemon is only possible locally. By setting the `core.https_address` configuration option, you can expose the same API over the network on a {abbr}`TLS (Transport Layer Security)` socket. See {ref}`server-expose` for instructions. Remote clients can then connect to Incus and access any image that is marked for public use. There are several ways to authenticate remote clients as trusted clients to allow them to access the API. See {ref}`authentication` for details. In a production setup, you should set `core.https_address` to the single address where the server should be available (rather than any address on the host). In addition, you should set firewall rules to allow access to the Incus port only from authorized hosts/subnets. (container-security)= ## Container security Incus containers can use a wide range of features for security. By default, containers are *unprivileged*, meaning that they operate inside a user namespace, restricting the abilities of users in the container to that of regular users on the host with limited privileges on the devices that the container owns. If data sharing between containers isn't needed, you can enable {config:option}`instance-security:security.idmap.isolated`, which will use non-overlapping UID/GID maps for each container, preventing potential {abbr}`DoS (Denial of Service)` attacks on other containers. Incus can also run *privileged* containers. Note, however, that those aren't root safe, and a user with root access in such a container will be able to DoS the host as well as find ways to escape confinement. More details on container security and the kernel features we use can be found on the [LXC security page](https://linuxcontainers.org/lxc/security/). ### Container name leakage The default server configuration makes it easy to list all cgroups on a system and, by extension, all running containers. You can prevent this name leakage by blocking access to `/sys/kernel/slab` and `/proc/sched_debug` before you start any containers. To do so, run the following commands: chmod 400 /proc/sched_debug chmod 700 /sys/kernel/slab/ ## Network security Make sure to configure your network interfaces to be secure. Which aspects you should consider depends on the networking mode you decide to use. ### Bridged NIC security The default networking mode in Incus is to provide a "managed" private network bridge that each instance connects to. In this mode, there is an interface on the host called `incusbr0` that acts as the bridge for the instances. The host runs an instance of `dnsmasq` for each managed bridge, which is responsible for allocating IP addresses and providing both authoritative and recursive DNS services. Instances using DHCPv4 will be allocated an IPv4 address, and a DNS record will be created for their instance name. This prevents instances from being able to spoof DNS records by providing false host name information in the DHCP request. The `dnsmasq` service also provides IPv6 router advertisement capabilities. This means that instances will auto-configure their own IPv6 address using SLAAC, so no allocation is made by `dnsmasq`. However, instances that are also using DHCPv4 will also get an AAAA DNS record created for the equivalent SLAAC IPv6 address. This assumes that the instances are not using any IPv6 privacy extensions when generating IPv6 addresses. In this default configuration, whilst DNS names cannot not be spoofed, the instance is connected to an Ethernet bridge and can transmit any layer 2 traffic that it wishes, which means an instance that is not trusted can effectively do MAC or IP spoofing on the bridge. In the default configuration, it is also possible for instances connected to the bridge to modify the Incus host's IPv6 routing table by sending (potentially malicious) IPv6 router advertisements to the bridge. This is because the `incusbr0` interface is created with `/proc/sys/net/ipv6/conf/incusbr0/accept_ra` set to `2`, meaning that the Incus host will accept router advertisements even though `forwarding` is enabled (see [`/proc/sys/net/ipv4/*` Variables](https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt) for more information). However, Incus offers several bridged {abbr}`NIC (Network interface controller)` security features that can be used to control the type of traffic that an instance is allowed to send onto the network. These NIC settings should be added to the profile that the instance is using, or they can be added to individual instances, as shown below. The following security features are available for bridged NICs: | Key | Type | Default | Required | Description | | :--- | :--- | :--- | :--- | :--- | | `security.mac_filtering` | bool | `false` | no | Prevent the instance from spoofing another instance's MAC address | | `security.ipv4_filtering` | bool | `false` | no | Prevent the instance from spoofing another instance's IPv4 address (enables `mac_filtering`) | | `security.ipv6_filtering` | bool | `false` | no | Prevent the instance from spoofing another instance's IPv6 address (enables `mac_filtering`) | One can override the default bridged NIC settings from the profile on a per-instance basis using: ``` incus config device override security.mac_filtering=true ``` Used together, these features can prevent an instance connected to a bridge from spoofing MAC and IP addresses. These options are implemented using `nftables`. It's worth noting that those options effectively prevent nested containers from using the parent network with a different MAC address (i.e using bridged or `macvlan` NICs). The IP filtering features block ARP and NDP advertisements that contain a spoofed IP, as well as blocking any packets that contain a spoofed source address. If `security.ipv4_filtering` or `security.ipv6_filtering` is enabled and the instance cannot be allocated an IP address (because `ipvX.address=none` or there is no DHCP service enabled on the bridge), then all IP traffic for that protocol is blocked from the instance. When `security.ipv6_filtering` is enabled, IPv6 router advertisements are blocked from the instance. When `security.ipv4_filtering` or `security.ipv6_filtering` is enabled, any Ethernet frames that are not ARP, IPv4 or IPv6 are dropped. This prevents stacked VLAN Q-in-Q (802.1ad) frames from bypassing the IP filtering. ### Routed NIC security An alternative networking mode is available called "routed". It provides a virtual Ethernet device pair between container and host. In this networking mode, the Incus host functions as a router, and static routes are added to the host directing traffic for the container's IPs towards the container's `veth` interface. By default, the `veth` interface created on the host has its `accept_ra` setting disabled to prevent router advertisements from the container modifying the IPv6 routing table on the Incus host. In addition to that, the `rp_filter` on the host is set to `1` to prevent source address spoofing for IPs that the host does not know the container has. incus-7.0.0/doc/explanation/storage.md000066400000000000000000000214501517523235500177260ustar00rootroot00000000000000(exp-storage)= # About storage pools, volumes and buckets Incus stores its data in storage pools, divided into storage volumes of different content types (like images or instances). You could think of a storage pool as the disk that is used to store data, while storage volumes are different partitions on this disk that are used for specific purposes. In addition to storage volumes, there are storage buckets, which use the [Amazon {abbr}`S3 (Simple Storage Service)`](https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html) protocol. Like storage volumes, storage buckets are part of a storage pool. (storage-pools)= ## Storage pools During initialization, Incus prompts you to create a first storage pool. If required, you can create additional storage pools later (see {ref}`storage-create-pool`). Each storage pool uses a storage driver. The following storage drivers are supported: - [Directory - `dir`](storage-dir) - [Btrfs - `btrfs`](storage-btrfs) - [LVM - `lvm`](storage-lvm) - [LVM Cluster - `lvmcluster`](storage-lvmcluster) - [ZFS - `zfs`](storage-zfs) - [Ceph RBD - `ceph`](storage-ceph) - [CephFS - `cephfs`](storage-cephfs) - [Ceph Object - `cephobject`](storage-cephobject) - [LINSTOR - `linstor`](storage-linstor) - [TrueNAS - `truenas`](storage-truenas) See the following how-to guides for additional information: - {ref}`howto-storage-pools` - {ref}`howto-storage-create-instance` (storage-location)= ### Data storage location Where the Incus data is stored depends on the configuration and the selected storage driver. Depending on the storage driver that is used, Incus can either share the file system with its host or keep its data separate. | Storage location | Directory | Btrfs | LVM (all) | ZFS | Ceph (all) | LINSTOR | | :--- | :--- | :--- | :--- | :--- | :--- | :--- | | Shared with the host | ✓ | ✓ | - | ✓ | - | - | | Dedicated disk/partition | - | ✓ | ✓ | ✓ | - | ✓ | | Loop disk | - | ✓ | ✓ | ✓ | - | ✓ | | Remote storage | - | - | ✓ | - | ✓ | ✓ | #### Shared with the host Sharing the file system with the host is usually the most space-efficient way to run Incus. In most cases, it is also the easiest to manage. This option is supported for the `dir` driver, the `btrfs` driver (if the host is Btrfs and you point Incus to a dedicated sub-volume) and the `zfs` driver (if the host is ZFS and you point Incus to a dedicated dataset on your zpool). #### Dedicated disk or partition Having Incus use an empty partition on your main disk or a full dedicated disk keeps its storage completely independent from the host. This option is supported for the `btrfs` driver, the `lvm` driver, the `zfs` driver and the `linstor` driver. #### Loop disk Incus can create a loop file on your main drive and have the selected storage driver use that. This method is functionally similar to using a disk or partition, but it uses a large file on your main drive instead. This means that every write must go through the storage driver and your main drive's file system, which leads to decreased performance. The loop files reside in `/var/lib/incus/disks/`. Loop files usually cannot be shrunk. They will grow up to the configured limit, but deleting instances or images will not cause the file to shrink. You can increase their size though; see {ref}`storage-resize-pool`. #### Remote storage The `ceph`, `cephfs` and `cephobject` drivers store the data in a completely independent Ceph storage cluster that must be set up separately. The `lvmcluster` driver relies on a shared block device being available to all cluster members and on a pre-existing `lvmlockd` setup. The `linstor` driver stores the data in a LINSTOR storage cluster that must be setup separately. The `truenas` driver stores the data on a TrueNAS storage server that must be setup separately. (storage-default-pool)= ### Default storage pool There is no concept of a default storage pool in Incus. When you create a storage volume, you must specify the storage pool to use. When Incus automatically creates a storage volume during instance creation, it uses the storage pool that is configured for the instance. This configuration can be set in either of the following ways: - Directly on an instance: [`incus launch --storage `](incus_launch.md) - Through a profile: [`incus profile device add root disk path=/ pool=`](incus_profile_device_add.md) and [`incus launch --profile `](incus_launch.md) - Through the default profile In a profile, the storage pool to use is defined by the pool for the root disk device: ```yaml root: type: disk path: / pool: default ``` In the default profile, this pool is set to the storage pool that was created during initialization. (storage-volumes)= ## Storage volumes When you create an instance, Incus automatically creates the required storage volumes for it. You can create additional storage volumes. See the following how-to guides for additional information: - {ref}`howto-storage-volumes` - {ref}`howto-storage-move-volume` - {ref}`howto-storage-backup-volume` (storage-volume-types)= ### Storage volume types Storage volumes can be of the following types: `container`/`virtual-machine` : Incus automatically creates one of these storage volumes when you launch an instance. It is used as the root disk for the instance, and it is destroyed when the instance is deleted. This storage volume is created in the storage pool that is specified in the profile used when launching the instance (or the default profile, if no profile is specified). The storage pool can be explicitly specified by providing the `--storage` flag to the launch command. `image` : Incus automatically creates one of these storage volumes when it unpacks an image to launch one or more instances from it. You can delete it after the instance has been created. If you do not delete it manually, it is deleted automatically ten days after it was last used to launch an instance. The image storage volume is created in the same storage pool as the instance storage volume, and only for storage pools that use a {ref}`storage driver ` that supports optimized image storage. `custom` : You can add one or more custom storage volumes to hold data that you want to store separately from your instances. Custom storage volumes can be shared between instances, and they are retained until you delete them. You can also use custom storage volumes to hold your backups or images. You must specify the storage pool for the custom volume when you create it. (storage-content-types)= ### Content types Each storage volume uses one of the following content types: `filesystem` : This content type is used for containers and container images. It is the default content type for custom storage volumes. Custom storage volumes of content type `filesystem` can be attached to both containers and virtual machines, and they can be shared between instances. `block` : This content type is used for virtual machines and virtual machine images. You can create a custom storage volume of type `block` by using the `--type=block` flag. Custom storage volumes of content type `block` can only be attached to virtual machines. They should not be shared between instances, because simultaneous access can lead to data corruption. `iso` : This content type is used for custom ISO volumes. A custom storage volume of type `iso` can only be created by importing an ISO file using [`incus storage volume import`](incus_storage_volume_import.md). Custom storage volumes of content type `iso` can only be attached to virtual machines. They can be attached to multiple machines simultaneously as they are always read-only. (storage-buckets)= ## Storage buckets Storage buckets provide object storage functionality via the S3 protocol. They can be used in a way that is similar to custom storage volumes. However, unlike storage volumes, storage buckets are not attached to an instance. Instead, applications can access a storage bucket directly using its URL. Each storage bucket is assigned one or more access keys, which the applications must use to access it. Storage buckets can be located on local storage (with `dir`, `btrfs`, `lvm` or `zfs` pools) or on remote storage (with `cephobject` pools). To enable storage buckets for local storage pool drivers and allow applications to access the buckets via the S3 protocol, you must configure the {config:option}`server-core:core.storage_buckets_address` server setting. See the following how-to guide for additional information: - {ref}`howto-storage-buckets` incus-7.0.0/doc/external_resources.md000066400000000000000000000002721517523235500176530ustar00rootroot00000000000000# External resources ```{toctree} :maxdepth: 1 Project repository Image server Third party tools ``` incus-7.0.0/doc/faq.md000066400000000000000000000154441517523235500145150ustar00rootroot00000000000000# Frequently asked questions The following sections give answers to frequently asked questions. They explain how to resolve common issues and point you to more detailed information. ## Why do my instances not have network access? Most likely, your firewall blocks network access for your instances. See {ref}`network-bridge-firewall` for more information about the problem and how to fix it. Another frequent reason for connectivity issues is running Incus and Docker on the same host. See {ref}`network-incus-docker` for instructions on how to fix such issues. ## How to enable the Incus server for remote access? By default, the Incus server is not accessible from the network, because it only listens on a local Unix socket. You can enable it for remote access by following the instructions in {ref}`server-expose`. ## When I do a `incus remote add`, it asks for a token? To be able to access the remote API, clients must authenticate with the Incus server. See {ref}`server-authenticate` for instructions on how to authenticate using a trust token. ## Why should I not run privileged containers? A privileged container can do things that affect the entire host - for example, it can use things in `/sys` to reset the network card, which will reset it for the entire host, causing network blips. See {ref}`container-security` for more information. Almost everything can be run in an unprivileged container, or - in cases of things that require unusual privileges, like wanting to mount NFS file systems inside the container - you might need to use bind mounts. ## Can I bind-mount my home directory in a container? Yes, you can do this by using a {ref}`disk device `: incus config device add container-name home disk source=/home/${USER} path=/home/ubuntu For unprivileged containers, you need to make sure that the user in the container has working read/write permissions. Otherwise, all files will show up as the overflow UID/GID (`65536:65536`) and access to anything that's not world-readable will fail. Use either of the following methods to grant the required permissions: - Pass `shift=true` to the [`incus config device add`](incus_config_device_add.md) call. This depends on the kernel and file system supporting either idmapped mounts (see [`incus info`](incus_info.md)). - Add a `raw.idmap` entry (see [Idmaps for user namespace](userns-idmap.md)). - Place recursive POSIX ACLs on your home directory. Privileged containers do not have this issue because all UID/GID in the container are the same as outside. But that's also the cause of most of the security issues with such privileged containers. ## How can I run Docker inside an Incus container? To run Docker inside an Incus container, set the {config:option}`instance-security:security.nesting` property of the container to `true`: incus config set security.nesting true Note that Incus containers cannot load kernel modules, so depending on your Docker configuration, you might need to have extra kernel modules loaded by the host. You can do so by setting a comma-separated list of kernel modules that your container needs: incus config set linux.kernel_modules In addition, creating a `/.dockerenv` file in your container can help Docker ignore some errors it's getting due to running in a nested environment. ## Where does the Incus client (`incus`) store its configuration? The [`incus`](incus.md) command stores its configuration under `~/.config/incus`. Various configuration files are stored in that directory, for example: - `client.crt`: client certificate (generated on demand) - `client.key`: client key (generated on demand) - `config.yml`: configuration file (info about `remotes`, `aliases`, etc.) - `clientcerts/`: directory with per-remote client certificates - `servercerts/`: directory with server certificates belonging to `remotes` ## Why can I not ping my Incus instance from another host? Many switches do not allow MAC address changes, and will either drop traffic with an incorrect MAC or disable the port totally. If you can ping an Incus instance from the host, but are not able to ping it from a different host, this could be the cause. The way to diagnose this problem is to run a `tcpdump` on the uplink and you will see either ``ARP Who has `xx.xx.xx.xx` tell `yy.yy.yy.yy` ``, with you sending responses but them not getting acknowledged, or ICMP packets going in and out successfully, but never being received by the other host. (faq-monitor)= ## How can I monitor what Incus is doing? To see detailed information about what Incus is doing and what processes it is running, use the [`incus monitor`](incus_monitor.md) command. For example, to show a human-readable output of all types of messages, enter the following command: incus monitor --pretty See [`incus monitor --help`](incus_monitor.md) for all options, and {doc}`debugging` for more information. ## Why does Incus stall when creating an instance? Check if your storage pool is out of space (by running [`incus storage info `](incus_storage_info.md)). In that case, Incus cannot finish unpacking the image, and the instance that you're trying to create shows up as stopped. To get more insight into what is happening, run [`incus monitor`](incus_monitor.md) (see {ref}`faq-monitor`), and check `sudo dmesg` for any I/O errors. ## Why does starting containers suddenly fail? If starting containers suddenly fails with a cgroup-related error message (`Failed to mount "/sys/fs/cgroup"`), this might be due to running a VPN client on the host. This is a known issue for both [Mullvad VPN](https://github.com/mullvad/mullvadvpn-app/issues/3651) and [Private Internet Access VPN](https://github.com/pia-foss/desktop/issues/50), but might occur for other VPN clients as well. The problem is that the VPN client mounts the `net_cls` cgroup1 over cgroup2 (which Incus uses). The easiest fix for this problem is to stop the VPN client and unmount the `net_cls` cgroup1 with the following command: umount /sys/fs/cgroup/net_cls If you need to keep the VPN client running, mount the `net_cls` cgroup1 in another location and reconfigure your VPN client accordingly. See [this Discourse post](https://discuss.linuxcontainers.org/t/help-help-help-cgroup2-related-issue-on-ubuntu-jammy-with-mullvad-and-privateinternetaccess-vpn/14705/18) for instructions for Mullvad VPN. ## What is this `incusbr0-mtu` device? When setting the `bridge.mtu` option on an Incus managed bridge network, Incus will create a dummy network interface named `BRIDGE-mtu`. That interface will never be used to carry traffic but it has the requested MTU set to it and is bridged into the network bridge. This has the effect of forcing the bridge to adopt that MTU and avoids issues where the bridge's configured MTU would change as interfaces get added to it. incus-7.0.0/doc/general.md000066400000000000000000000011711517523235500153530ustar00rootroot00000000000000# General See the following sections for information on how to get started with Incus: ```{toctree} :maxdepth: 1 Containers and VMs Install Incus Initialize Incus Get support Frequently asked ``` You can find a series of demos and tutorials on YouTube: incus-7.0.0/doc/howto/000077500000000000000000000000001517523235500145545ustar00rootroot00000000000000incus-7.0.0/doc/howto/benchmark_performance.md000066400000000000000000000104711517523235500214140ustar00rootroot00000000000000(benchmark-performance)= # How to benchmark performance The performance of your Incus server or cluster depends on a lot of different factors, ranging from the hardware, the server configuration, the selected storage driver and the network bandwidth to the overall usage patterns. To find the optimal configuration, you should run benchmark tests to evaluate different setups. Incus provides a benchmarking tool for this purpose. This tool allows you to initialize or launch a number of containers and measure the time it takes for the system to create the containers. If you run this tool repeatedly with different configurations, you can compare the performance and evaluate which is the ideal configuration. ## Get the tool If the `incus-benchmark` tool isn't provided with your installation, you can build it from source. Make sure that you have `go` (see {ref}`requirements-go`) installed and install the tool with the following command: go install github.com/lxc/incus/cmd/incus-benchmark@latest After installation, `incus-benchmark` will be available at your [`GOPATH`](https://go.dev/wiki/SettingGOPATH). If no `GOPATH` is set, it is assumed to be `$HOME/go` on Unix systems and `%USERPROFILE%\go` on Windows. ## Run the tool Run `incus-benchmark [action]` to measure the performance of your Incus setup. The benchmarking tool uses the current Incus configuration. If you want to use a different project, specify it with `--project`. For all actions, you can specify the number of parallel threads to use (default is to use a dynamic batch size). You can also choose to append the results to a CSV report file and label them in a certain way. See `incus-benchmark help` for all available actions and flags. ### Select an image Before you run the benchmark, select what kind of image you want to use. Local image : If you want to measure the time it takes to create a container and ignore the time it takes to download the image, you should copy the image to your local image store before you run the benchmarking tool. To do so, run a command similar to the following and specify the fingerprint (for example, `2d21da400963`) of the image when you run `incus-benchmark`: incus image copy images:debian/12 local: You can also assign an alias to the image and specify that alias (for example, `debian`) when you run `incus-benchmark`: incus image copy images:debian/12 local: --alias debian Remote image : If you want to include the download time in the overall result, specify a remote image (for example, `images:debian/12`). The default image that `incus-benchmark` uses is the latest Debian image (`images:debian/12`), so if you want to use this image, you can leave out the image name when running the tool. ### Create and launch containers Run the following command to create a number of containers: incus-benchmark init --count Add `--privileged` to the command to create privileged containers. For example: ```{list-table} :header-rows: 1 * - Command - Description * - `incus-benchmark init --count 10 --privileged` - Create ten privileged containers that use the latest Debian image. * - `incus-benchmark init --count 20 --parallel 4 images:alpine/edge` - Create 20 containers that use the Alpine Edge image, using four parallel threads. * - `incus-benchmark init 2d21da400963` - Create one container that uses the local image with the fingerprint `2d21da400963`. * - `incus-benchmark init --count 10 debian` - Create ten containers that use the image with the alias `debian`. ``` If you use the `init` action, the benchmarking containers are created but not started. To start the containers that you created, run the following command: incus-benchmark start Alternatively, use the `launch` action to both create and start the containers: incus-benchmark launch --count 10 For this action, you can add the `--freeze` flag to freeze each container right after it starts. Freezing a container pauses its processes, so this flag allows you to measure the pure launch times without interference of the processes that run in each container after startup. ### Delete containers To delete the benchmarking containers that you created, run the following command: incus-benchmark delete ```{note} You must delete all existing benchmarking containers before you can run a new benchmark. ``` incus-7.0.0/doc/howto/cluster_access.md000066400000000000000000000114701517523235500201030ustar00rootroot00000000000000(cluster-access)= # Accessing a cluster An Incus cluster generally behaves much like a standalone Incus server. A client can talk to any of the servers within a cluster and will get an identical experience. Requests can be directed at a specific server through the API and doesn't need a direct connection to that server. ```{note} Targeting a specific server is done through `?target=SERVER` at the API level or `--target` at the CLI level). ``` The cluster uses a single client facing TLS certificate for all servers, this makes it easier to expose a valid HTTPS endpoint to clients, avoiding having to manually check fingerprints. You can use `incus cluster update-certificate` to load your own cluster-wide TLS certificate, or you can use ACME to automatically issue and deploy a certificate across the cluster (see {ref}`authentication-server-certificate`). ## Authentication ### HTTPS with TLS The default authentication method when dealing with a remote Incus cluster. This works fine in a cluster, but may cause some issues with some proxies and load-balancers that want to establish their own TLS connection to the cluster. See {ref}`authentication-tls-certs` for details. ### HTTPS with OpenID Connect (OIDC) OpenID Connect authentication on Incus requires an external OpenID Identity Provider but then has the advantage of offering fine grained authentication to a cluster, making managing and auditing access easy. For OpenID Connect to work properly, the cluster will need to have a DNS record, a valid certificate and be able to reach the OpenID Identity Provider. See {ref}`authentication-openid` for details. ### Local access You can also interact with a cluster by connecting to any of the clustered servers and talking to the local Incus daemon running on that server. ## High availability To provide a highly available Incus API on a cluster, you need to have client requests always make it to at least one responsive server. Here are a few common ways to handle it. ### DNS round-robin DNS is a very easy way to balance API traffic over multiple servers. Simply create a DNS record with an `A` or `AAAA` for each server in the cluster. While this is trivial to put in place, it will only properly handle falling back to another server if the server quickly rejects the connection. Any stuck server may cause significant delays for some clients as they'll need to wait for a full connection timeout before another server is contacted. ### External load-balancer A reasonably easy solution is to run a load-balancer, either a self-hosted one like `haproxy` or one provide by your existing network or cloud infrastructure. Those load-balancers can often monitor service health and only send requests to servers that are currently responsive. Incus supports the `haproxy` proxy protocol headers so the original client IP address is reported in log and audit messages. ```{note} TLS client certificate authentication only works with load-balancers that act at the TCP level. Load-balancers which terminate the TLS session and then establish their own to Incus can only be used with OIDC authentication. ``` ### Floating IP address It's possible to use Incus with an additional floating IP, effectively a virtual IP address which is only live on one of the servers. This centralizes all client API traffic to that single server but may be easier to manage in some environments. For that you'll need to make sure that all servers are configured to listen on all interfaces (e.g. `core.https_address=:8443`) and then make use of a local firewall to only allow external clients to connect to the virtual IP address. Common solutions to handle a virtual IP address are `VRRP` (through something like `frr`) and `corosync/pacemaker`. ### ECMP For those running a full L3 network infrastructure with BGP to each individual host, it's possible to advertise an IP address for use for Incus client traffic. This IP address would be added to all servers in their network configuration (as a `/32` for IPv4 or `/128` for IPv6) and then advertised to their router. This will result in the router having an equal cost route for the IP address to all servers in the cluster (ECMP). Traffic will then get balanced between all servers and as soon as a server goes down, its route will go away and traffic will head to the remaining servers. ```{note} To minimize fallback delay, one can make use of BFD alongside BGP to get sub-1s fallback time. ``` ### mDNS in L2 Network If you are running in an L2 network (for instance, in a typical home network) you can use mDNS and publish the same `.local` domain (something like `incus.local`) from multiple hosts. More than one host may send a reply in response to the multicast request, and the client will receive multiple mDNS response packets - this way, at the cost of packet flood, you get a simple way to ensure you reach *some* cluster node. incus-7.0.0/doc/howto/cluster_config_networks.md000066400000000000000000000052751517523235500220510ustar00rootroot00000000000000(cluster-config-networks)= # How to configure networks for a cluster All members of a cluster must have identical networks defined. The only configuration keys that may differ between networks on different members are [`bridge.external_interfaces`](network-bridge-options), [`parent`](network-external), [`bgp.ipv4.nexthop`](network-bridge-options) and [`bgp.ipv6.nexthop`](network-bridge-options). See {ref}`clustering-member-config` for more information. Creating additional networks is a two-step process: 1. Define and configure the new network across all cluster members. For example, for a cluster that has three members: incus network create --target server1 my-network incus network create --target server2 my-network incus network create --target server3 my-network ```{note} You can pass only the member-specific configuration keys `bridge.external_interfaces`, `parent`, `bgp.ipv4.nexthop` and `bgp.ipv6.nexthop`. Passing other configuration keys results in an error. ``` These commands define the network, but they don't create it. If you run [`incus network list`](incus_network_list.md), you can see that the network is marked as "pending". 1. Run the following command to instantiate the network on all cluster members: incus network create my-network ```{note} You can add configuration keys that are not member-specific to this command. ``` If you missed a cluster member when defining the network, or if a cluster member is down, you get an error. Also see {ref}`network-create-cluster`. (cluster-https-address)= ## Separate REST API and clustering networks You can configure different networks for the REST API endpoint of your clients and for internal traffic between the members of your cluster. This separation can be useful, for example, to use a virtual address for your REST API, with DNS round robin. To do so, you must specify different addresses for {config:option}`server-cluster:cluster.https_address` (the address for internal cluster traffic) and {config:option}`server-core:core.https_address` (the address for the REST API): 1. Create your cluster as usual, and make sure to use the address that you want to use for internal cluster traffic as the cluster address. This address is set as the `cluster.https_address` configuration. 1. After joining your members, set the `core.https_address` configuration to the address for the REST API. For example: incus config set core.https_address 0.0.0.0:8443 ```{note} `core.https_address` is specific to the cluster member, so you can use different addresses on different members. You can also use a wildcard address to make the member listen on multiple interfaces. ``` incus-7.0.0/doc/howto/cluster_config_storage.md000066400000000000000000000064521517523235500216370ustar00rootroot00000000000000(cluster-config-storage)= # How to configure storage for a cluster All members of a cluster must have identical storage pools. The only configuration keys that may differ between pools on different members are [`source`](storage-drivers), [`size`](storage-drivers), [`zfs.pool_name`](storage-zfs-pool-config), [`lvm.thinpool_name`](storage-lvm-pool-config) and [`lvm.vg_name`](storage-lvm-pool-config). See {ref}`clustering-member-config` for more information. Incus creates a default `local` storage pool for each cluster member during initialization. Creating additional storage pools is a two-step process: 1. Define and configure the new storage pool across all cluster members. For example, for a cluster that has three members: incus storage create --target server1 data zfs source=/dev/vdb1 incus storage create --target server2 data zfs source=/dev/vdc1 incus storage create --target server3 data zfs source=/dev/vdb1 size=10GiB ```{note} You can pass only the member-specific configuration keys `source`, `size`, `zfs.pool_name`, `lvm.thinpool_name` and `lvm.vg_name`. Passing other configuration keys results in an error. ``` These commands define the storage pool, but they don't create it. If you run [`incus storage list`](incus_storage_list.md), you can see that the pool is marked as "pending". 1. Run the following command to instantiate the storage pool on all cluster members: incus storage create data zfs ```{note} You can add configuration keys that are not member-specific to this command. ``` If you missed a cluster member when defining the storage pool, or if a cluster member is down, you get an error. Also see {ref}`storage-pools-cluster`. ## View member-specific pool configuration Running [`incus storage show `](incus_storage_show.md) shows the cluster-wide configuration of the storage pool. To view the member-specific configuration, use the `--target` flag. For example: incus storage show data --target server2 ## Create storage volumes For most storage drivers (all except for Ceph-based storage drivers), storage volumes are not replicated across the cluster and exist only on the member for which they were created. Run [`incus storage volume list `](incus_storage_volume_list.md) to see on which member a certain volume is located. When creating a storage volume, use the `--target` flag to create a storage volume on a specific cluster member. Without the flag, the volume is created on the cluster member on which you run the command. For example, to create a volume on the current cluster member `server1`: incus storage volume create local vol1 To create a volume with the same name on another cluster member: incus storage volume create local vol1 --target server2 Different volumes can have the same name as long as they live on different cluster members. Typical examples for this are image volumes. You can manage storage volumes in a cluster in the same way as you do in non-clustered deployments, except that you must pass the `--target` flag to your commands if more than one cluster member has a volume with the given name. For example, to show information about the storage volumes: incus storage volume show local vol1 --target server1 incus storage volume show local vol1 --target server2 incus-7.0.0/doc/howto/cluster_form.md000066400000000000000000000223161517523235500176060ustar00rootroot00000000000000(cluster-form)= # How to form a cluster When forming an Incus cluster, you start with a bootstrap server. This bootstrap server can be an existing Incus server or a newly installed one. After initializing the bootstrap server, you can join additional servers to the cluster. See {ref}`clustering-members` for more information. You can form the Incus cluster interactively by providing configuration information during the initialization process or by using preseed files that contain the full configuration. ## Configure the cluster interactively To form your cluster, you must first run `incus admin init` on the bootstrap server. After that, run it on the other servers that you want to join to the cluster. When forming a cluster interactively, you answer the questions that `incus admin init` prompts you with to configure the cluster. ### Initialize the bootstrap server To initialize the bootstrap server, run `incus admin init` and answer the questions according to your desired configuration. You can accept the default values for most questions, but make sure to answer the following questions accordingly: - `Would you like to use Incus clustering?` Select **yes**. - `What IP address or DNS name should be used to reach this server?` Make sure to use an IP or DNS address that other servers can reach. - `Are you joining an existing cluster?` Select **no**.
Expand to see a full example for incus admin init on the bootstrap server ```{terminal} :input: incus admin init Would you like to use Incus clustering? (yes/no) [default=no]: yes What IP address or DNS name should be used to reach this server? [default=192.0.2.101]: Are you joining an existing cluster? (yes/no) [default=no]: no What member name should be used to identify this server in the cluster? [default=server1]: Do you want to configure a new local storage pool? (yes/no) [default=yes]: Name of the storage backend to use (btrfs, dir, lvm, zfs) [default=zfs]: Create a new ZFS pool? (yes/no) [default=yes]: Would you like to use an existing empty block device (e.g. a disk or partition)? (yes/no) [default=no]: Size in GiB of the new loop device (1GiB minimum) [default=9GiB]: Do you want to configure a new remote storage pool? (yes/no) [default=no]: Would you like to configure Incus to use an existing bridge or host interface? (yes/no) [default=no]: Would you like stale cached images to be updated automatically? (yes/no) [default=yes]: Would you like a YAML "incus admin init" preseed to be printed? (yes/no) [default=no]: ```
After the initialization process finishes, your first cluster member should be up and available on your network. You can check this with [`incus cluster list`](incus_cluster_list.md). ### Convert an existing server into bootstrap server If you are intending to convert an existing Incus server with instances into the bootstrap server for a new cluster, there is a slightly different procedure. Firstly, ensure the `core.https_address` (or `cluster.https_address`) is configured to a specific IP or DNS address using `incus config set core.https_address [IP_OR_DNS]:8443`. The default wildcard value cannot be used in clustered mode. Then you can run `incus cluster enable memberName` and continue with the steps below to join in additional cluster members. ### Join additional servers You can now join further servers to the cluster. ```{note} The servers that you add should be newly installed Incus servers. If you are using existing servers, make sure to clear their contents before joining them, because any existing data on them will be lost. ``` To join a server to the cluster, run `incus admin init` on the cluster. Joining an existing cluster requires root privileges, so make sure to run the command as root or with `sudo`. Basically, the initialization process consists of the following steps: 1. Request to join an existing cluster. Answer the first questions that `incus admin init` asks accordingly: - `Would you like to use Incus clustering?` Select **yes**. - `What IP address or DNS name should be used to reach this server?` Make sure to use an IP or DNS address that other servers can reach. - `Are you joining an existing cluster?` Select **yes**. 1. Authenticate with the cluster. There are two alternative methods, depending on which authentication method you choose when configuring the bootstrap server. `````{tabs} ````{group-tab} Authentication tokens If you configured your cluster to use {ref}`authentication tokens `, you must generate a join token for each new member. To do so, run the following command on an existing cluster member (for example, the bootstrap server): incus cluster add This command returns a single-use join token that is valid for a configurable time (see {config:option}`server-cluster:cluster.join_token_expiry`). Enter this token when `incus admin init` prompts you for the join token. The join token contains the addresses of the existing online members, as well as a single-use secret and the fingerprint of the cluster certificate. This reduces the amount of questions that you must answer during `incus admin init`, because the join token can be used to answer these questions automatically. ```` ````` 1. Confirm that all local data for the server is lost when joining a cluster. 1. Configure server-specific settings (see {ref}`clustering-member-config` for more information). You can accept the default values or specify custom values for each server.
Expand to see full examples for incus admin init on additional servers `````{tabs} ````{group-tab} Authentication tokens ```{terminal} :input: sudo incus admin init Would you like to use Incus clustering? (yes/no) [default=no]: yes What IP address or DNS name should be used to reach this server? [default=192.0.2.102]: Are you joining an existing cluster? (yes/no) [default=no]: yes Do you have a join token? (yes/no/[token]) [default=no]: yes Please provide join token: eyJzZXJ2ZXJfbmFtZSI6InJwaTAxIiwiZmluZ2VycHJpbnQiOiIyNjZjZmExZDk0ZDZiMjk2Nzk0YjU0YzJlYzdjOTMwNDA5ZjIzNjdmNmM1YjRhZWVjOGM0YjAxYTc2NjU0MjgxIiwiYWRkcmVzc2VzIjpbIjE3Mi4xNy4zMC4xODM6ODQ0MyJdLCJzZWNyZXQiOiJmZGI1OTgyNjgxNTQ2ZGQyNGE2ZGE0Mzg5MTUyOGM1ZGUxNWNmYmQ5M2M3OTU3ODNkNGI5OGU4MTQ4MWMzNmUwIn0= All existing data is lost when joining a cluster, continue? (yes/no) [default=no] yes Choose "size" property for storage pool "local": Choose "source" property for storage pool "local": Choose "zfs.pool_name" property for storage pool "local": Would you like a YAML "incus admin init" preseed to be printed? (yes/no) [default=no]: ``` ```` `````
After the initialization process finishes, your server is added as a new cluster member. You can check this with [`incus cluster list`](incus_cluster_list.md). ## Configure the cluster through preseed files To form your cluster, you must first run `incus admin init` on the bootstrap server. After that, run it on the other servers that you want to join to the cluster. Instead of answering the `incus admin init` questions interactively, you can provide the required information through preseed files. You can feed a file to `incus admin init` with the following command: cat | incus admin init --preseed You need a different preseed file for every server. ### Initialize the bootstrap server `````{tabs} ````{group-tab} Authentication tokens To enable clustering, the preseed file for the bootstrap server must contain the following fields: ```yaml config: core.https_address: cluster: server_name: enabled: true ``` Here is an example preseed file for the bootstrap server: ```yaml config: core.https_address: 192.0.2.101:8443 images.auto_update_interval: 15 storage_pools: - name: default driver: dir - name: my-pool driver: zfs networks: - name: incusbr0 type: bridge profiles: - name: default devices: root: path: / pool: my-pool type: disk eth0: name: eth0 nictype: bridged parent: incusbr0 type: nic cluster: server_name: server1 enabled: true ``` ```` ````` ### Join additional servers The preseed files for new cluster members require only a `cluster` section with data and configuration values that are specific to the joining server. `````{tabs} ````{group-tab} Authentication tokens The preseed file for additional servers must include the following fields: ```yaml cluster: enabled: true server_address: cluster_token: ``` Here is an example preseed file for a new cluster member: ```yaml cluster: enabled: true server_address: 192.0.2.102:8443 cluster_token: eyJzZXJ2ZXJfbmFtZSI6Im5vZGUyIiwiZmluZ2VycHJpbnQiOiJjZjlmNmVhMWIzYjhiNjgxNzQ1YTY1NTY2YjM3ZGUwOTUzNjRmM2MxMDAwMGNjZWQyOTk5NDU5YzY2MGIxNWQ4IiwiYWRkcmVzc2VzIjpbIjE3Mi4xNy4zMC4xODM6ODQ0MyJdLCJzZWNyZXQiOiIxNGJmY2EzMDhkOTNhY2E3MGJmYThkMzE0NWM4NWY3YmE0ZmU1YmYyNmJiNDhmMmUwNzhhOGZhMDczZDc0YTFiIn0= member_config: - entity: storage-pool name: default key: source value: "" - entity: storage-pool name: my-pool key: source value: "" - entity: storage-pool name: my-pool key: driver value: "zfs" ``` ```` ````` incus-7.0.0/doc/howto/cluster_groups.md000066400000000000000000000051031517523235500201550ustar00rootroot00000000000000(howto-cluster-groups)= # How to set up cluster groups Cluster members can be assigned to {ref}`cluster-groups`. By default, all cluster members belong to the `default` group. To create a cluster group, use the [`incus cluster group create`](incus_cluster_group_create.md) command. For example: incus cluster group create gpu To assign a cluster member to one or more groups, use the [`incus cluster group assign`](incus_cluster_group_assign.md) command. This command removes the specified cluster member from all the cluster groups it currently is a member of and then adds it to the specified group or groups. For example, to assign `server1` to only the `gpu` group, use the following command: incus cluster group assign server1 gpu To assign `server1` to the `gpu` group and also keep it in the `default` group, use the following command: incus cluster group assign server1 default,gpu To add a cluster member to a specific group without removing it from other groups, use the [`incus cluster group add`](incus_cluster_group_add.md) command. For example, to add `server1` to the `gpu` group and also keep it in the `default` group, use the following command: incus cluster group add server1 gpu ## Configuration options The following configuration options are available for cluster groups: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ## Launch an instance on a cluster group member With cluster groups, you can target an instance to run on one of the members of the cluster group, instead of targeting it to run on a specific member. ```{note} {config:option}`cluster-cluster:scheduler.instance` must be set to either `all` (the default) or `group` to allow instances to be targeted to a cluster group. See {ref}`clustering-instance-placement` for more information. ``` To launch an instance on a member of a cluster group, follow the instructions in {ref}`cluster-target-instance`, but use the group name prefixed with `@` for the `--target` flag. For example: incus launch images:debian/12 c1 --target=@gpu ## Use with restricted projects A project can be configured to only have access to servers that are part of specific cluster groups. This is done by setting both `restricted=true` and `restricted.cluster.groups` to a comma separated list of group names. ```{note} If the cluster group is renamed, the project restrictions will need to be updated for the new group name. ``` incus-7.0.0/doc/howto/cluster_manage.md000066400000000000000000000220721517523235500200720ustar00rootroot00000000000000(cluster-manage)= # How to manage a cluster After your cluster is formed, use [`incus cluster list`](incus_cluster_list.md) to see a list of its members and their status: ```{terminal} :input: incus cluster list :scroll: +---------+----------------------------+------------------+--------------+----------------+-------------+--------+-------------------+ | NAME | URL | ROLES | ARCHITECTURE | FAILURE DOMAIN | DESCRIPTION | STATE | MESSAGE | +---------+----------------------------+------------------+--------------+----------------+-------------+--------+-------------------+ | server1 | https://192.0.2.101:8443 | database-leader | x86_64 | default | | ONLINE | Fully operational | | | | database | | | | | | +---------+----------------------------+------------------+--------------+----------------+-------------+--------+-------------------+ | server2 | https://192.0.2.102:8443 | database-standby | aarch64 | default | | ONLINE | Fully operational | +---------+----------------------------+------------------+--------------+----------------+-------------+--------+-------------------+ | server3 | https://192.0.2.103:8443 | database-standby | aarch64 | default | | ONLINE | Fully operational | +---------+----------------------------+------------------+--------------+----------------+-------------+--------+-------------------+ ``` To see more detailed information about an individual cluster member, run the following command: incus cluster show To see state and usage information for a cluster member, run the following command: incus cluster info ## Configure your cluster To configure your cluster, use [`incus config`](incus_config.md). For example: incus config set cluster.max_voters 5 Keep in mind that some {ref}`server configuration options ` are global and others are local. You can configure the global options on any cluster member, and the changes are propagated to the other cluster members through the distributed database. The local options are set only on the server where you configure them (or alternatively on the server that you target with `--target`). In addition to the server configuration, there are a few cluster configurations that are specific to each cluster member. See {ref}`cluster-member-config` for all available configurations. To set these configuration options, use [`incus cluster set`](incus_cluster_set.md) or [`incus cluster edit`](incus_cluster_edit.md). For example: incus cluster set server1 scheduler.instance manual ### Assign member roles To add or remove a {ref}`member role ` for a cluster member, use the [`incus cluster role`](incus_cluster_role.md) command. For example: incus cluster role add server1 event-hub ```{note} You can add or remove only those roles that are not assigned automatically by Incus. ``` ### Edit the cluster member configuration To edit all properties of a cluster member, including the member-specific configuration, the member roles, the failure domain and the cluster groups, use the [`incus cluster edit`](incus_cluster_edit.md) command. (cluster-evacuate)= ## Evacuate and restore cluster members There are scenarios where you might need to empty a given cluster member of all its instances (for example, for routine maintenance like applying system updates that require a reboot, or to perform hardware changes). To do so, use the [`incus cluster evacuate`](incus_cluster_evacuate.md) command. This command migrates all instances on the given server, moving them to other cluster members. The evacuated cluster member is then transitioned to an "evacuated" state, which prevents the creation of any instances on it. You can control how each instance is moved through the {config:option}`instance-miscellaneous:cluster.evacuate` instance configuration key. Instances are shut down cleanly, respecting the `boot.host_shutdown_timeout` configuration key. When the evacuated server is available again, use the [`incus cluster restore`](incus_cluster_restore.md) command to move the server back into a normal running state. This command also moves the evacuated instances back from the servers that were temporarily holding them. (cluster-automatic-evacuation)= ### Cluster healing Incus can automatically detect and recover from a broken server. This is done by setting the {config:option}`server-cluster:cluster.healing_threshold` configuration to a non-zero value. Instances are automatically evacuated to other servers after the leader has marked a cluster member has offline. When the broken server is available again, you must manually restore it as if it had been manually evacuated. ```{note} This automatic cluster healing only applies to instances on shared storage and which don't use any local devices. ``` ```{warning} Enabling this feature can come at the risk of data corruption should a server be deemed offline as a result of partial connectivity issues. Incus considers a server to be offline when it fails to respond to heartbeat packets and when it also fails to respond to ICMP packets. It's critical to ensure that a server which is considered offline is in fact offline and isn't still running its instances. One way to automatically achieve this is to have a piece of software monitor Incus for a `cluster-member-healed` event and promptly cut the power to the server in question by interacting with its BMC or PDU. ``` (cluster-automatic-balancing)= ### Cluster re-balancing Incus can automatically balance the load across cluster members. This is done through a few configuration options: - {config:option}`server-cluster:cluster.rebalance.batch` - {config:option}`server-cluster:cluster.rebalance.cooldown` - {config:option}`server-cluster:cluster.rebalance.interval` - {config:option}`server-cluster:cluster.rebalance.threshold` Incus will compare the load across all servers and if the difference in percent exceeds the threshold, it will start identifying virtual-machines that can be safely live-migrated to the least loaded server. (cluster-manage-delete-members)= ## Delete cluster members To cleanly delete a member from the cluster, use the following command: incus cluster remove You can only cleanly delete members that are online and that don't have any instances located on them. ### Deal with offline cluster members If a cluster member goes permanently offline, you can force-remove it from the cluster. Make sure to do so as soon as you discover that you cannot recover the member. If you keep an offline member in your cluster, you might encounter issues when upgrading your cluster to a newer version. To force-remove a cluster member, enter the following command on one of the cluster members that is still online: incus cluster remove --force ```{caution} Force-removing a cluster member will leave the member's database in an inconsistent state (for example, the storage pool on the member will not be removed). As a result, it will not be possible to re-initialize Incus later, and the server must be fully reinstalled. ``` ## Upgrade cluster members To upgrade a cluster, you must upgrade all of its members. All members must be upgraded to the same version of Incus. ```{caution} Do not attempt to upgrade your cluster if any of its members are offline. Offline members cannot be upgraded, and your cluster will end up in a blocked state. ``` To upgrade a single member, simply upgrade the Incus package on the host and restart the Incus daemon. If the new version of the daemon has database schema or API changes, the upgraded member might transition into a "blocked" state. In this case, the member does not serve any Incus API requests (which means that `incus` commands don't work on that member anymore), but any running instances will continue to run. This happens if there are other cluster members that have not been upgraded and are therefore running an older version. Run [`incus cluster list`](incus_cluster_list.md) on a cluster member that is not blocked to see if any members are blocked. As you proceed upgrading the rest of the cluster members, they will all transition to the "blocked" state. When you upgrade the last member, the blocked members will notice that all servers are now up-to-date, and the blocked members become operational again. ## Update the cluster certificate In an Incus cluster, the API on all servers responds with the same shared certificate, which is usually a standard self-signed certificate with an expiry set to ten years. The certificate is stored at `/var/lib/incus/cluster.crt` and is the same on all cluster members. You can replace the standard certificate with another one, for example, a valid certificate obtained through ACME services (see {ref}`authentication-server-certificate` for more information). To do so, use the [`incus cluster update-certificate`](incus_cluster_update-certificate.md) command. This command replaces the certificate on all servers in your cluster. incus-7.0.0/doc/howto/cluster_manage_instance.md000066400000000000000000000031021517523235500217470ustar00rootroot00000000000000(cluster-manage-instance)= # How to manage instances in a cluster In a cluster setup, each instance lives on one of the cluster members. You can operate each instance from any cluster member, so you do not need to log on to the cluster member on which the instance is located. (cluster-target-instance)= ## Launch an instance on a specific cluster member When you launch an instance, you can target it to run on a specific cluster member. You can do this from any cluster member. For example, to launch an instance named `c1` on the cluster member `server2`, use the following command: incus launch images:debian/12 c1 --target server2 You can launch instances on specific cluster members or on specific {ref}`cluster groups `. If you do not specify a target, the instance is assigned to a cluster member automatically. See {ref}`clustering-instance-placement` for more information. ## Check where an instance is located To check on which member an instance is located, list all instances in the cluster: incus list The location column indicates the member on which each instance is running. ## Move an instance You can move an existing instance to another cluster member. For example, to move the instance `c1` to the cluster member `server1`, use the following commands: incus stop c1 incus move c1 --target server1 incus start c1 See {ref}`move-instances` for more information. To move an instance to a member of a cluster group, use the group name prefixed with `@` for the `--target` flag. For example: incus move c1 --target @group1 incus-7.0.0/doc/howto/cluster_recover.md000066400000000000000000000131741517523235500203120ustar00rootroot00000000000000(cluster-recover)= # How to recover a cluster It might happen that one or several members of your cluster go offline or become unreachable. In that case, no operations are possible on this member, and neither are operations that require a state change across all members. See {ref}`clustering-offline-members` and {ref}`cluster-automatic-evacuation` for more information. If you can bring the offline cluster members back or delete them from the cluster, operation resumes as normal. If this is not possible, there are a few ways to recover the cluster, depending on the scenario that caused the failure. See the following sections for details. ```{note} Run `incus admin cluster --help` for an overview of all available commands. ``` ## Recover from quorum loss Every Incus cluster has a specific number of members (configured through {config:option}`server-cluster:cluster.max_voters`) that serve as voting members of the distributed database. If you permanently lose a majority of these cluster members (for example, you have a three-member cluster and you lose two members), the cluster loses quorum and becomes unavailable. However, if at least one database member survives, it is possible to recover the cluster. To do so, complete the following steps: 1. Log on to any surviving member of your cluster and run the following command: sudo incus admin cluster list-database This command shows which cluster members have one of the database roles. 1. Pick one of the listed database members that is still online as the new leader. Log on to the machine (if it differs from the one you are already logged on to). 1. Make sure that the Incus daemon is not running on the machine. sudo systemctl stop incus.service incus.socket 1. Log on to all other cluster members that are still online and stop the Incus daemon. 1. On the server that you picked as the new leader, run the following command: sudo incus admin cluster recover-from-quorum-loss 1. Start the Incus daemon again on all machines, starting with the new leader. sudo systemctl start incus.socket incus.service The database should now be back online. No information has been deleted from the database. All information about the cluster members that you have lost is still there, including the metadata about their instances. This can help you with further recovery steps if you need to re-create the lost instances. To permanently delete the cluster members that you have lost, force-remove them. See {ref}`cluster-manage-delete-members`. ## Recover cluster members with changed addresses If some members of your cluster are no longer reachable, or if the cluster itself is unreachable due to a change in IP address or listening port number, you can reconfigure the cluster. To do so, edit the cluster configuration on each member of the cluster and change the IP addresses or listening port numbers as required. You cannot remove any members during this process. The cluster configuration must contain the description of the full cluster, so you must do the changes for all cluster members on all cluster members. You can edit the {ref}`clustering-member-roles` of the different members, but with the following limitations: - A cluster member that does not have a `database*` role cannot become a voter, because it might lack a global database. - At least two members must remain voters (except in the case of a two-member cluster, where one voter suffices), or there will be no quorum. Log on to each cluster member and complete the following steps: 1. Stop the Incus daemon. sudo systemctl stop incus.service incus.socket 1. Run the following command: sudo incus admin cluster edit 1. Edit the YAML representation of the information that this cluster member has about the rest of the cluster: ```yaml # Latest cowsql segment ID: 1234 members: - id: 1 # Internal ID of the member (Read-only) name: server1 # Name of the cluster member (Read-only) address: 192.0.2.10:8443 # Last known address of the member (Writeable) role: voter # Last known role of the member (Writeable) - id: 2 # Internal ID of the member (Read-only) name: server2 # Name of the cluster member (Read-only) address: 192.0.2.11:8443 # Last known address of the member (Writeable) role: stand-by # Last known role of the member (Writeable) - id: 3 # Internal ID of the member (Read-only) name: server3 # Name of the cluster member (Read-only) address: 192.0.2.12:8443 # Last known address of the member (Writeable) role: spare # Last known role of the member (Writeable) ``` You can edit the addresses and the roles. After doing the changes on all cluster members, start the Incus daemon on all members again. sudo systemctl start incus.socket incus.service The cluster should now be fully available again with all members reporting in. No information has been deleted from the database. All information about the cluster members and their instances is still there. ## Manually alter Raft membership In some situations, you might need to manually alter the Raft membership configuration of the cluster because of some unexpected behavior. For example, if you have a cluster member that was removed uncleanly, it might not show up in [`incus cluster list`](incus_cluster_list.md) but still be part of the Raft configuration. To see the Raft configuration, run the following command: incus admin sql local "SELECT * FROM raft_nodes" In that case, run the following command to remove the leftover node: incus admin cluster remove-raft-node
incus-7.0.0/doc/howto/disaster_recovery.md000066400000000000000000000131451517523235500206360ustar00rootroot00000000000000(disaster-recovery)= # How to recover instances in case of disaster Incus provides a tool for disaster recovery in case the {ref}`Incus database ` is corrupted or otherwise lost. The tool scans the storage pools for instances and imports the instances that it finds back into the database. You need to re-create the required entities that are missing (usually profiles, projects, and networks). ```{important} This tool should be used for disaster recovery only. Do not rely on this tool as an alternative to proper backups; you will lose data like profiles, network definitions, or server configuration. The tool must be run interactively and cannot be used in automated scripts. ``` ## Recovery process When you run the tool, it scans all storage pools that still exist in the database, looking for missing volumes that can be recovered. You can also specify the details of any unknown storage pools (those that exist on disk but do not exist in the database), and the tool attempts to scan those too. After mounting the specified storage pools (if not already mounted), the tool scans them for unknown volumes that look like they are associated with Incus. Incus maintains a `backup.yaml` file in each instance's storage volume, which contains all necessary information to recover a given instance (including instance configuration, attached devices, storage volume, and pool configuration). This data can be used to rebuild the instance, storage volume, and storage pool database records. Before recovering an instance, the tool performs some consistency checks to compare what is in the `backup.yaml` file with what is actually on disk (such as matching snapshots). If all checks out, the database records are re-created. If the storage pool database record also needs to be created, the tool uses the information from an instance's `backup.yaml` file as the basis of its configuration, rather than what the user provided during the discovery phase. However, if this information is not available, the tool falls back to restoring the pool's database record with what was provided by the user. The tool asks you to re-create missing entities like networks. However, the tool does not know how the instance was configured. That means that if some configuration was specified through the `default` profile, you must also re-add the required configuration to the profile. For example, if the `incusbr0` bridge is used in an instance and you are prompted to re-create it, you must add it back to the `default` profile so that the recovered instance uses it. ## Example This is how a recovery process could look: ```{terminal} :input: incus admin recover This Incus server currently has the following storage pools: Would you like to recover another storage pool? (yes/no) [default=no]: yes Name of the storage pool: default Name of the storage backend (btrfs, ceph, cephfs, cephobject, dir, lvm, lvmcluster, zfs): zfs Source of the storage pool (block device, volume group, dataset, path, ... as applicable): /var/lib/incus/storage-pools/default/containers Additional storage pool configuration property (KEY=VALUE, empty when done): zfs.pool_name=default Additional storage pool configuration property (KEY=VALUE, empty when done): Would you like to recover another storage pool? (yes/no) [default=no]: The recovery process will be scanning the following storage pools: - NEW: "default" (backend="zfs", source="/var/lib/incus/storage-pools/default/containers") Would you like to continue with scanning for lost volumes? (yes/no) [default=yes]: yes Scanning for unknown volumes... The following unknown volumes have been found: - Container "u1" on pool "default" in project "default" (includes 0 snapshots) - Container "u2" on pool "default" in project "default" (includes 0 snapshots) You are currently missing the following: - Network "incusbr0" in project "default" Please create those missing entries and then hit ENTER: ^Z [1]+ Stopped incus admin recover :input: incus network create incusbr0 Network incusbr0 created :input: fg incus admin recover The following unknown volumes have been found: - Container "u1" on pool "default" in project "default" (includes 0 snapshots) - Container "u2" on pool "default" in project "default" (includes 0 snapshots) Would you like those to be recovered? (yes/no) [default=no]: yes Starting recovery... :input: incus list +------+---------+------+------+-----------+-----------+ | NAME | STATE | IPV4 | IPV6 | TYPE | SNAPSHOTS | +------+---------+------+------+-----------+-----------+ | u1 | STOPPED | | | CONTAINER | 0 | +------+---------+------+------+-----------+-----------+ | u2 | STOPPED | | | CONTAINER | 0 | +------+---------+------+------+-----------+-----------+ :input: incus profile device add default eth0 nic network=incusbr0 name=eth0 Device eth0 added to default :input: incus start u1 :input: incus list +------+---------+-------------------+----------------------------------------------+-----------+-----------+ | NAME | STATE | IPV4 | IPV6 | TYPE | SNAPSHOTS | +------+---------+-------------------+----------------------------------------------+-----------+-----------+ | u1 | RUNNING | 192.0.2.49 (eth0) | 2001:db8:8b6:abfe:1266:6aff:fe82:918e (eth0) | CONTAINER | 0 | +------+---------+-------------------+----------------------------------------------+-----------+-----------+ | u2 | STOPPED | | | CONTAINER | 0 | +------+---------+-------------------+----------------------------------------------+-----------+-----------+ ``` incus-7.0.0/doc/howto/images_copy.md000066400000000000000000000063051517523235500174010ustar00rootroot00000000000000(images-copy)= # How to copy and import images To add images to an image store, you can either copy them from another server or import them from files (either local files or files on a web server). ## Copy an image from a remote To copy an image from one server to another, enter the following command: incus image copy [:] : ```{note} To copy the image to your local image store, specify `local:` as the target remote. ``` See [`incus image copy --help`](incus_image_copy.md) for a list of all available flags. The most relevant ones are: `--alias` : Assign an alias to the copy of the image. `--copy-aliases` : Copy the aliases that the source image has. `--auto-update` : Keep the copy up-to-date with the original image. `--vm` : When copying from an alias, copy the image that can be used to create virtual machines. ## Import an image from files If you have image files that use the required {ref}`image-format`, you can import them into your image store. There are several ways of obtaining such image files: - Exporting an existing image (see {ref}`images-manage-export`) - Building your own image using `distrobuilder` (see {ref}`images-create-build`) - Downloading image files from a {ref}`remote image server ` (note that it is usually easier to {ref}`use the remote image ` directly instead of downloading it to a file and importing it) ### Import from the local file system To import an image from the local file system, use the [`incus image import`](incus_image_import.md) command. This command supports both {ref}`unified images ` (compressed file or directory) and {ref}`split images ` (two files). To import a unified image from one file or directory, enter the following command: incus image import [:] To import a split image, enter the following command: incus image import [:] In both cases, you can assign an alias with the `--alias` flag. See [`incus image import --help`](incus_image_import.md) for all available flags. ### Import from a file on a remote web server You can import image files from a remote web server by URL. This method is an alternative to running an Incus server for the sole purpose of distributing an image to users. It only requires a basic web server with support for custom headers (see {ref}`images-copy-http-headers`). The image files must be provided as unified images (see {ref}`image-format-unified`). To import an image file from a remote web server, enter the following command: incus image import You can assign an alias to the local image with the `--alias` flag. (images-copy-http-headers)= #### Custom HTTP headers Incus requires the following custom HTTP headers to be set by the web server: `Incus-Image-Hash` : The SHA256 of the image that is being downloaded. `Incus-Image-URL` : The URL from which to download the image. Incus sets the following headers when querying the server: `Incus-Server-Architectures` : A comma-separated list of architectures that the client supports. `Incus-Server-Version` : The version of Incus in use. incus-7.0.0/doc/howto/images_create.md000066400000000000000000000043111517523235500176650ustar00rootroot00000000000000(images-create)= # How to create images If you want to create and share your own images, you can do this either based on an existing instance or snapshot or by building your own image from scratch. (images-create-publish)= ## Publish an image from an instance or snapshot If you want to be able to use an instance or an instance snapshot as the base for new instances, you should create and publish an image from it. To publish an image from an instance, make sure that the instance is stopped. Then enter the following command: incus publish [:] To publish an image from a snapshot, enter the following command: incus publish / [:] In both cases, you can specify an alias for the new image with the `--alias` flag, set an expiration date with `--expire` and make the image publicly available with `--public`. If an image with the same name already exists, add the `--reuse` flag to overwrite it. See [`incus publish --help`](incus_publish.md) for a full list of available flags. The publishing process can take quite a while because it generates a tarball from the instance or snapshot and then compresses it. As this can be particularly I/O and CPU intensive, publish operations are serialized by Incus. ```{note} For OCI containers, it's typically best to update the source image definition and build a new OCI image to publish on a registry than to publish an altered version of the OCI container. ``` ### Prepare the instance for publishing Before you publish an image from an instance, clean up all data that should not be included in the image. Usually, this includes the following data: - Instance metadata (use [`incus config metadata`](incus_config_metadata.md) to edit) - File templates (use [`incus config template`](incus_config_template.md) to edit) - Instance-specific data inside the instance itself (for example, host SSH keys and `dbus/systemd machine-id`) (images-create-build)= ## Build an image For building your own images, you can use [`distrobuilder`](https://github.com/lxc/distrobuilder). See the [`distrobuilder` documentation](https://linuxcontainers.org/distrobuilder/docs/latest/) for instructions for installing and using the tool. incus-7.0.0/doc/howto/images_manage.md000066400000000000000000000107251517523235500176600ustar00rootroot00000000000000(images-manage)= # How to manage images When working with images, you can inspect various information about the available images, view and edit their properties and configure aliases to refer to specific images. You can also export an image to a file, which can be useful to {ref}`copy or import it ` on another machine. ## List available images To list all images on a server, enter the following command: incus image list [:] If you do not specify a remote, the {ref}`default remote ` is used. (images-manage-filter)= ### Filter available images To filter the results that are displayed, specify a part of the alias or fingerprint after the command. For example, to show all Debian images, enter the following command: incus image list images: debian You can specify several filters as well. For example, to show all Arm 64-bit Debian images, enter the following command: incus image list images: debian arm64 To filter for properties other than alias or fingerprint, specify the filter in `=` format. For example: incus image list images: debian architecture=x86_64 ## View image information To view information about an image, enter the following command: incus image info As the image ID, you can specify either the image's alias or its fingerprint. For a remote image, remember to include the remote server (for example, `images:debian/12`). To display only the image properties, enter the following command: incus image show You can also display a specific image property (located under the `properties` key) with the following command: incus image get-property For example, to show the release name of the latest Debian 12 image, enter the following command: incus image get-property images:debian/12 release (images-manage-edit)= ## Edit image properties To set a specific image property that is located under the `properties` key, enter the following command: incus image set-property ```{note} These properties can be used to convey information about the image. They do not configure Incus' behavior in any way. ``` To edit the full image properties, including the top-level properties, enter the following command: incus image edit ## Delete an image To delete a local copy of an image, enter the following command: incus image delete Deleting an image won't affect running instances that are already using it, but it will remove the image locally. After deletion, if the image was downloaded from a remote server, it will be removed from local cache and downloaded again on next use. However, if the image was manually created (not cached), the image will be deleted. ## Configure image aliases Configuring an alias for an image can be useful to make it easier to refer to an image, since remembering an alias is usually easier than remembering a fingerprint. Most importantly, however, you can change an alias to point to a different image, which allows creating an alias that always provides a current image (for example, the latest version of a release). You can see some of the existing aliases in the image list. To see the full list, enter the following command: incus image alias list You can directly assign an alias to an image when you {ref}`copy or import ` or {ref}`publish ` it. Alternatively, enter the following command: incus image alias create You can also delete an alias: incus image alias delete To rename an alias, enter the following command: incus image alias rename If you want to keep the alias name, but point the alias to a different image (for example, a newer version), you must delete the existing alias and then create a new one. (images-manage-export)= ## Export an image to a file Images are located in the image store of your local server or a remote Incus server. You can export them to a file though. This method can be useful to back up image files or to transfer them to an air-gapped environment. To export a container image to a file, enter the following command: incus image export [:] [] To export a virtual machine image to a file, add the `--vm` flag: incus image export [:] [] --vm See {ref}`image-format` for a description of the file structure used for the image. incus-7.0.0/doc/howto/images_profiles.md000066400000000000000000000017001517523235500202440ustar00rootroot00000000000000(images-profiles)= # How to associate profiles with an image You can associate one or more profiles with a specific image. Instances that are created from the image will then automatically use the associated profiles in the order they were specified. To associate a list of profiles with an image, use the [`incus image edit`](incus_image_edit.md) command and edit the `profiles:` section: ```yaml profiles: - default ``` Most provided images come with a profile list that includes only the `default` profile. To prevent any profile (including the `default` profile) from being associated with an image, pass an empty list. ```{note} Passing an empty list is different than passing `nil`. If you pass `nil` as the profile list, only the `default` profile is associated with the image. ``` You can override the associated profiles for an image when creating an instance by adding the `--profile` or the `--no-profiles` flag to the launch or init command. incus-7.0.0/doc/howto/images_remote.md000066400000000000000000000047551517523235500177310ustar00rootroot00000000000000(images-remote)= # How to use remote images The [`incus`](incus.md) CLI command can support several image servers and comes pre-configured with our own. See {ref}`image-servers` for an overview. ## List configured remotes To see all configured remote servers, enter the following command: incus remote list Remote servers that use the [simple streams format](https://git.launchpad.net/simplestreams/tree/) are pure image servers. Servers that use the `incus` format are Incus servers, which either serve solely as image servers or might provide some images in addition to serving as regular Incus servers. See {ref}`image-server-types` for more information. ## List available images on a remote To list all remote images on a server, enter the following command: incus image list : You can filter the results. See {ref}`images-manage-filter` for instructions. ## Add a remote server How to add a remote depends on the protocol that the server uses. ### Add a simple streams server To add a simple streams server as a remote, enter the following command: incus remote add --protocol=simplestreams The URL must use HTTPS. ### Add a remote Incus server To add an Incus server as a remote, enter the following command: incus remote add [flags] Some authentication methods require specific flags (for example, use [`incus remote add --auth-type=oidc`](incus_remote_add.md) for OIDC authentication). See {ref}`server-authenticate` and {ref}`authentication` for more information. For example, enter the following command to add a remote through an IP address: incus remote add my-remote 192.0.2.10 You are prompted to confirm the remote server fingerprint and then asked for the token. ## Reference an image To reference an image, specify its remote and its alias or fingerprint, separated with a colon. For example: images:debian/12 images:debian/12 local:ed7509d7e83f (images-remote-default)= ## Select a default remote If you specify an image name without the name of the remote, the default image server is used. To see which server is configured as the default image server, enter the following command: incus remote get-default To select a different remote as the default image server, enter the following command: incus remote switch incus-7.0.0/doc/howto/import_machines_to_instances.md000066400000000000000000000302031517523235500230260ustar00rootroot00000000000000(import-machines-to-instances)= # How to import physical or virtual machines to Incus instances Incus provides a tool (`incus-migrate`) to create an Incus instance or volume based on an existing disk or image. You can run the tool on any Linux machine. It connects to a local or remote Incus server and creates a blank instance or volume, which you can configure during or after the migration. The tool then copies the data from the disk or image that you provide. `incus-migrate` can import images in `raw`, `qcow2`, `ova` and `vmdk` file formats. It can also directly download an image from a provided URL. ```{note} If you want to configure a new instance during the migration process, set up the entities that you want your instance to use before starting the migration process. By default, the new instance will use the entities specified in the `default` profile. You can specify a different profile (or a profile list) to customize the configuration. See {ref}`profiles` for more information. You can also override {ref}`instance-options`, the {ref}`storage pool ` to be used and the size for the {ref}`storage volume `, and the {ref}`network ` to be used. Alternatively, you can update the instance configuration after the migration is complete. ``` When working with instances, the tool can create both containers and virtual machines: * When creating a container, you must provide a disk or partition that contains the root file system for the container. For example, this could be the `/` root disk of the machine or container where you are running the tool. * When creating a virtual machine, you must provide a bootable disk, partition or image. This means that just providing a file system is not sufficient, and you cannot create a virtual machine from a container that you are running. It is also not possible to create a virtual machine from the physical machine that you are using to do the migration, because the migration tool would be using the disk that it is copying. Instead, you could provide a bootable image, or a bootable partition or disk that is currently not in use. ````{tip} If you want to convert a Windows VM from a foreign hypervisor (not from QEMU/KVM with Q35/`virtio-scsi`), you must install the `virtio-win` drivers to your Windows. Otherwise, your VM won't boot.
Expand to see how to integrate the required drivers to your Windows VM Install the required tools on the host: 1. Install `virt-v2v` version >= 2.3.4 (this is the minimal version that supports the `--block-driver` option). 1. Install the `virtio-win` package, or download the [`virtio-win.iso`](https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/stable-virtio/virtio-win.iso) image and put it into the `/usr/share/virtio-win` folder. 1. You might also need to install [`rhsrvany`](https://github.com/rwmjones/rhsrvany). Now you can use `virt-v2v` to convert images from a foreign hypervisor to `raw` images for Incus and include the required drivers: ``` # Example 1. Convert a vmdk disk image to a raw image suitable for incus-migrate sudo virt-v2v --block-driver virtio-scsi -o local -of raw -os ./os -i vmx ./test-vm.vmx # Example 2. Convert a QEMU/KVM qcow2 image and integrate virtio-scsi driver sudo virt-v2v --block-driver virtio-scsi -o local -of raw -os ./os -if qcow2 -i disk test-vm-disk.qcow2 ``` You can find the resulting image in the `os` directory and use it with `incus-migrate` on the next steps.
```` Complete the following steps to migrate an existing machine to an Incus instance: 1. Download the `bin.linux.incus-migrate` tool ([`bin.linux.incus-migrate.aarch64`](https://github.com/lxc/incus/releases/latest/download/bin.linux.incus-migrate.aarch64) or [`bin.linux.incus-migrate.x86_64`](https://github.com/lxc/incus/releases/latest/download/bin.linux.incus-migrate.x86_64)) from the **Assets** section of the latest [Incus release](https://github.com/lxc/incus/releases). 1. Place the tool on the machine that you want to use to create the instance. Make it executable (usually by running `chmod u+x bin.linux.incus-migrate`). 1. Make sure that the machine has `rsync` installed. If it is missing, install it (for example, with `sudo apt install rsync`). 1. Run the tool: sudo ./bin.linux.incus-migrate The tool then asks you to provide the information required for the migration. ```{tip} As an alternative to running the tool interactively, you can provide the configuration as parameters to the command. See `./bin.linux.incus-migrate --help` for more information. ``` 1. Specify the Incus server URL, either as an IP address or as a DNS name. ```{note} The Incus server must be {ref}`exposed to the network `. If you want to import to a local Incus server, you must still expose it to the network. You can then specify `127.0.0.1` as the IP address to access the local server. ``` 1. Check and confirm the certificate fingerprint. 1. Choose a method for authentication (see {ref}`authentication`). For example, if you choose using a certificate token, log on to the Incus server and create a token for the machine on which you are running the migration tool with [`incus config trust add`](incus_config_trust_add.md). Then use the generated token to authenticate the tool. 1. Choose whether to create a container or a virtual machine. See {ref}`containers-and-vms`. 1. Specify a name for the instance that you are creating. 1. Provide the path to a root file system (for containers) or a bootable disk, partition or image file (for virtual machines). 1. For containers, optionally add additional file system mounts. 1. For virtual machines, specify whether secure boot is supported. 1. Optionally, configure the new instance. You can do so by specifying {ref}`profiles `, directly setting {ref}`configuration options ` or changing {ref}`storage ` or {ref}`network ` settings. Alternatively, you can configure the new instance after the migration. 1. When you are done with the configuration, start the migration process.
Expand to see an example output for importing to a container ```{terminal} :input: sudo ./bin.linux.incus-migrate The local Incus server is the target [default=yes]: What would you like to create? 1) Container 2) Virtual Machine 3) Virtual Machine (from .ova) 4) Custom Volume Please enter the number of your choice: 1 Name of the new instance: foo Please provide the path to a root filesystem: / Do you want to add additional filesystem mounts? [default=no]: Instance to be created: Name: foo Project: default Type: container Source: / Additional overrides can be applied at this stage: 1) Begin the migration with the above configuration 2) Override profile list 3) Set additional configuration options 4) Change instance storage pool or volume size 5) Change instance network Please pick one of the options above [default=1]: 3 Please specify config keys and values (key=value ...): limits.cpu=2 Instance to be created: Name: foo Project: default Type: container Source: / Config: limits.cpu: "2" Additional overrides can be applied at this stage: 1) Begin the migration with the above configuration 2) Override profile list 3) Set additional configuration options 4) Change instance storage pool or volume size 5) Change instance network Please pick one of the options above [default=1]: 4 Please provide the storage pool to use: default Do you want to change the storage size? [default=no]: yes Please specify the storage size: 20GiB Instance to be created: Name: foo Project: default Type: container Source: / Storage pool: default Storage pool size: 20GiB Config: limits.cpu: "2" Additional overrides can be applied at this stage: 1) Begin the migration with the above configuration 2) Override profile list 3) Set additional configuration options 4) Change instance storage pool or volume size 5) Change instance network Please pick one of the options above [default=1]: 5 Please specify the network to use for the instance: incusbr0 Instance to be created: Name: foo Project: default Type: container Source: / Storage pool: default Storage pool size: 20GiB Network name: incusbr0 Config: limits.cpu: "2" Additional overrides can be applied at this stage: 1) Begin the migration with the above configuration 2) Override profile list 3) Set additional configuration options 4) Change instance storage pool or volume size 5) Change instance network Please pick one of the options above [default=1]: 1 Instance foo successfully created ```
Expand to see an example output for importing to a VM ```{terminal} :input: sudo ./bin.linux.incus-migrate The local Incus server is the target [default=yes]: What would you like to create? 1) Container 2) Virtual Machine 3) Virtual Machine (from .ova) 4) Custom Volume Please enter the number of your choice: 2 Name of the new instance: foo Please provide the path to a root filesystem: ./virtual-machine.img Does the VM support UEFI Secure Boot? [default=no]: no Instance to be created: Name: foo Project: default Type: virtual-machine Source: ./virtual-machine.img Config: security.secureboot: "false" Additional overrides can be applied at this stage: 1) Begin the migration with the above configuration 2) Override profile list 3) Set additional configuration options 4) Change instance storage pool or volume size 5) Change instance network Please pick one of the options above [default=1]: 3 Please specify config keys and values (key=value ...): limits.cpu=2 Instance to be created: Name: foo Project: default Type: virtual-machine Source: ./virtual-machine.img Config: limits.cpu: "2" security.secureboot: "false" Additional overrides can be applied at this stage: 1) Begin the migration with the above configuration 2) Override profile list 3) Set additional configuration options 4) Change instance storage pool or volume size 5) Change instance network Please pick one of the options above [default=1]: 4 Please provide the storage pool to use: default Do you want to change the storage size? [default=no]: yes Please specify the storage size: 20GiB Instance to be created: Name: foo Project: default Type: virtual-machine Source: ./virtual-machine.img Storage pool: default Storage pool size: 20GiB Config: limits.cpu: "2" security.secureboot: "false" Additional overrides can be applied at this stage: 1) Begin the migration with the above configuration 2) Override profile list 3) Set additional configuration options 4) Change instance storage pool or volume size 5) Change instance network Please pick one of the options above [default=1]: 5 Please specify the network to use for the instance: incusbr0 Instance to be created: Name: foo Project: default Type: virtual-machine Source: ./virtual-machine.img Storage pool: default Storage pool size: 20GiB Network name: incusbr0 Config: limits.cpu: "2" security.secureboot: "false" Additional overrides can be applied at this stage: 1) Begin the migration with the above configuration 2) Override profile list 3) Set additional configuration options 4) Change instance storage pool or volume size 5) Change instance network Please pick one of the options above [default=1]: 1 Instance foo successfully created ```
1. When the migration is complete, check the new instance and update its configuration to the new environment. Typically, you must update at least the storage configuration (`/etc/fstab`) and the network configuration. incus-7.0.0/doc/howto/incus_alias.md000066400000000000000000000147311517523235500173760ustar00rootroot00000000000000(incus-alias)= # How to manage command aliases The Incus command-line client `incus` has support for adding aliases for commands that you use frequently. You can use aliases as shortcuts for longer commands, or to automatically add flags to existing commands. Managing aliases is done through the [`incus alias`](incus_alias.md) command. Within the [`incus alias`](incus_alias.md) command, you can use the following subcommands: - `incus alias add` to add a new command alias - `incus alias list` to list all command aliases - `incus alias remove` to remove a command alias - `incus alias rename` to rename a command alias Run [`incus alias --help`](incus_alias.md) to see all available subcommands and parameters. ```{note} _Command aliases_ are different from {ref}`_image aliases_ `. An image alias is an alternative name for an image, usually a shorter name or another common mnemonic for that image. Image aliases are a server-side concept part of the Incus API whereas command aliases are purely part of the command line tool configuration. ``` ## How to add a command alias To always ask for confirmation when deleting an instance, create an alias for [`incus delete`](incus_delete.md) that always runs `incus delete --interactive`. The following command for `incus alias`, will _add_ the command alias with name `delete`, and will invoke the same Incus command but with the added `--interactive` flag. incus alias add delete "delete --interactive" Note that when you now run `incus delete mycontainer` to delete an instance called `myinstance`, the Incus command-line client will replace `incus delete` with `incus delete --interactive` and will instead execute `incus delete --interactive myinstance`. When a command alias has the same name as an Incus command, the command alias will mask the Incus command. You would need to remove first the command alias if you want to run verbatim the Incus command of the same name. In addition, when you use a command alias with parameters (in this case, the name of the container), the Incus command-line client will place those parameters at the end of the aliased command unless they are manually placed elsewhere through the `@ARGS@` string. Finally, the command in the command alias should be enclosed in quotes. ## How to list all command aliases To see all configured aliases, run [`incus alias list`](incus_alias_list.md). ## How to remove a command alias To remove an existing command alias, type [`incus alias remove`](incus_alias_remove.md) and add the name of that command alias. ## How to rename a command alias To rename an existing command alias, type [`incus alias rename`](incus_alias_rename.md), then add the name of that existing command alias, and finally the name of the new command alias. ## Built-in `shell` alias Incus comes with the `shell` built-in command alias. That alias is based on the [`incus exec`](incus_exec.md) command, executing `exec @ARGS@ -- su -l`. ``` $ incus alias list +-----------+----------------------+ | ALIAS | TARGET | +-----------+----------------------+ | shell | exec @ARGS@ -- su -l | +-----------+----------------------+ ``` If you run `incus shell myinstance`, this command alias will expand into `incus exec myinstance -- su -l`. The `--` construct is a command-line artifact that instructs the Incus command-line client to stop processing further parameters, like the `-l` that follows. Without `--`, the expanded command `incus exec mycontainer su -l` would fail, because the Incus command-line client would try to parse the `-l` flag. In this particular case, it would fail with an error because there is no `-l` parameter for `incus shell`. The `su -l` command is synonymous to `su -` or `su --login`. It launches a login shell in the instance as the `root` user. The command reads the necessary configuration files to launch a login shell for user `root`. The `shell` alias is built-in into the Incus server. Therefore, the Incus client is not able to remove it. If you try to remove it, there will be an error that the alias does not exist. ``` $ incus alias remove shell Error: Alias shell doesn't exist $ ``` If you add a new command alias with the name `shell`, the new command alias will be masking the built-in command alias. That is, the Incus command-line client will be using your newly added alias instead and the built-in command alias will be hidden. When you remove the newly added alias `shell`, the built-in alias will appear again. ## How to use a command alias to get a non-root shell in an instance Several Incus images have been configured to create a non-root username as shown in the table below. | Distribution | Username | Image | | :----------- | :--------------: | :----------- | | Alpine | `alpine` | `images:alpine/edge/cloud` | | Debian | `debian` | `images:debian/12/cloud` | | Fedora | `fedora` | `images:fedora/42/cloud` | | Ubuntu | `ubuntu` | `images:ubuntu/24.04/cloud` | You can get a shell into the instance for this non-root username with the following command. ``` $ incus launch images:debian/12/cloud mycontainer Launching mycontainer $ incus exec mycontainer -- su -l debian debian@mycontainer:~$ ``` By using the Incus command aliases, you can also create a command alias to get a shell into that instance. In this command alias, you specify to `su -l` into the username `debian`. ``` $ incus alias add debian 'exec @ARGS@ -- su -l debian' $ ``` Finally, you can now get a shell into the instance with the following convenient command: ``` $ incus debian mycontainer debian@mycontainer:~$ ``` ```{note} As an alternative to `su`, you may use instead `sudo`. In that case, the command would be as follows. incus alias add debian `exec @ARGS@ -- sudo --login --user debian` ``` ```{note} When launching a system container or a virtual machine, Incus allows to specify environment variables. incus launch -c environment.MYVARIABLE=myvalue images:debian/12 myinstance A login shell in such an instance does not have access to those environment variables. This is due to the semantics of login shells with either `su -l` or `sudo --login` which do not preserve any environment variables. If you want to preserve any environment variables, you would instead use either `su --preserve-environment` or `sudo --preserve-env`. Another alternative is to add the environment variables into the instance to the file system file `/etc/environment`. By doing so, any new login shell to the instance will be able to parse this file and enable any environment variables. ``` incus-7.0.0/doc/howto/initialize.md000066400000000000000000000163741517523235500172520ustar00rootroot00000000000000(initialize)= # How to initialize Incus Before you can create an Incus instance, you must configure and initialize Incus. ## Interactive configuration Run the following command to start the interactive configuration process: incus admin init ```{note} For simple configurations, you can run this command as a normal user. However, some more advanced operations during the initialization process (for example, joining an existing cluster) require root privileges. In this case, run the command with `sudo` or as root. ``` The tool asks a series of questions to determine the required configuration. The questions are dynamically adapted to the answers that you give. They cover the following areas: Clustering (see {ref}`exp-clustering` and {ref}`cluster-form`) : A cluster combines several Incus servers. The cluster members share the same distributed database and can be managed uniformly using the Incus client ([`incus`](incus.md)) or the REST API. The default answer is `no`, which means clustering is not enabled. If you answer `yes`, you can either connect to an existing cluster or create one. Networking (see {ref}`networks` and {ref}`Network devices `) : Provides network access for the instances. You can let Incus create a new bridge (recommended) or use an existing network bridge or interface. You can create additional bridges and assign them to instances later. Storage pools (see {ref}`exp-storage` and {ref}`storage-drivers`) : Instances (and other data) are stored in storage pools. For testing purposes, you can create a loop-backed storage pool. For production use, however, you should use an empty partition (or full disk) instead of loop-backed storage (because loop-backed pools are slower and their size can't be reduced). The recommended backends are `zfs` and `btrfs`. You can create additional storage pools later. Remote access (see {ref}`security_remote_access` and {ref}`authentication`) : Allows remote access to the server over the network. The default answer is `no`, which means remote access is not allowed. If you answer `yes`, you can connect to the server over the network. You can choose to add client certificates to the server (manually or through tokens). Automatic image update (see {ref}`about-images`) : You can download images from image servers. In this case, images can be updated automatically. The default answer is `yes`, which means that Incus will update the downloaded images regularly. YAML `incus admin init` preseed (see {ref}`initialize-preseed`) : If you answer `yes`, the command displays a summary of your chosen configuration options in the terminal. ### Minimal setup To create a minimal setup with default options, you can skip the configuration steps by adding the `--minimal` flag to the `incus admin init` command: incus admin init --minimal ```{note} The minimal setup provides a basic configuration, but the configuration is not optimized for speed or functionality. Especially the [`dir` storage driver](storage-dir), which is used by default, is slower than other drivers and doesn't provide fast snapshots, fast copy/launch, quotas and optimized backups. If you want to use an optimized setup, go through the interactive configuration process instead. ``` (initialize-preseed)= ## Non-interactive configuration The `incus admin init` command supports a `--preseed` command line flag that makes it possible to fully configure the Incus daemon settings, storage pools, network devices and profiles, in a non-interactive way through a preseed YAML file. For example, starting from a brand new Incus installation, you could configure Incus with the following command: ```bash cat </ For example, to edit the `/etc/hosts` file in the instance, enter the following command: incus file edit my-container/etc/hosts ```{note} The file must already exist on the instance. You cannot use the `edit` command to create a file on the instance. ``` ## Delete files from the instance To delete a file from your instance, enter the following command: incus file delete / ## Pull files from the instance to the local machine To pull a file from your instance to your local machine, enter the following command: incus file pull / For example, to pull the `/etc/hosts` file to the current directory, enter the following command: incus file pull my-instance/etc/hosts . Instead of pulling the instance file into a file on the local system, you can also pull it to stdout and pipe it to stdin of another command. This can be useful, for example, to check a log file: incus file pull my-instance/var/log/syslog - | less To pull a directory with all contents, enter the following command: incus file pull -r / ## Push files from the local machine to the instance To push a file from your local machine to your instance, enter the following command: incus file push / To push a directory with all contents, enter the following command: incus file push -r / ## Mount a file system from the instance You can mount an instance file system into a local path on your client. To do so, make sure that you have `sshfs` installed. Then run the following command: incus file mount / You can then access the files from your local machine. ### Set up an SSH SFTP listener Alternatively, you can set up an SSH SFTP listener. This method allows you to connect with any SFTP client and with a dedicated user name. To do so, first set up the listener by entering the following command: incus file mount [--listen
:] For example, to set up the listener on a random port on the local machine (for example, `127.0.0.1:45467`): incus file mount my-instance If you want to access your instance files from outside your local network, you can pass a specific address and port: incus file mount my-instance --listen 192.0.2.50:2222 ```{caution} Be careful when doing this, because it exposes your instance remotely. ``` To set up the listener on a specific address and a random port: incus file mount my-instance --listen 192.0.2.50:0 The command prints out the assigned port and a user name and password for the connection. ```{tip} You can specify a user name by passing the `--auth-user` flag. ``` Use this information to access the file system. For example, if you want to use `sshfs` to connect, enter the following command: sshfs @
: -p For example: sshfs xFn8ai8c@127.0.0.1:/home my-instance-files -p 35147 You can then access the file system of your instance at the specified location on the local machine. incus-7.0.0/doc/howto/instances_backup.md000066400000000000000000000133251517523235500204160ustar00rootroot00000000000000--- myst: substitutions: type: "instance" --- (instances-backup)= # How to back up instances There are different ways of backing up your instances: - {ref}`instances-snapshots` - {ref}`instances-backup-export` - {ref}`instances-backup-copy` % Include content from [storage_backup_volume.md](storage_backup_volume.md) ```{include} storage_backup_volume.md :start-after: :end-before: ``` ```{note} Custom storage volumes might be attached to an instance, but they are not part of the instance. Therefore, the content of a custom storage volume is not stored when you back up your instance. You must back up the data of your storage volume separately. See {ref}`howto-storage-backup-volume` for instructions. ``` (instances-snapshots)= ## Use snapshots for instance backup You can save your instance at a point in time by creating an instance snapshot, which makes it easy to restore the instance to a previous state. Instance snapshots are stored in the same storage pool as the instance volume itself. % Include content from [storage_backup_volume.md](storage_backup_volume.md) ```{include} storage_backup_volume.md :start-after: :end-before: ``` ### Create a snapshot Use the following command to create a snapshot of an instance: incus snapshot create [] % Include content from [storage_backup_volume.md](storage_backup_volume.md) ```{include} storage_backup_volume.md :start-after: :end-before: ``` For virtual machines, you can add the `--stateful` flag to capture not only the data included in the instance volume but also the running state of the instance. Note that this feature is not fully supported for containers because of CRIU limitations. ### View, edit or delete snapshots Use the following command to display the snapshots for an instance: incus info You can view or modify snapshots in a similar way to instances, by referring to the snapshot with `/`. To show configuration information about a snapshot, use the following command: incus config show / To change the expiry date of a snapshot, use the following command: incus config edit / ```{note} In general, snapshots cannot be edited, because they preserve the state of the instance. The only exception is the expiry date. Other changes to the configuration are silently ignored. ``` To delete a snapshot, use the following command: incus snapshot delete ### Schedule instance snapshots You can configure an instance to automatically create snapshots at specific times (at most once every minute). To do so, set the {config:option}`instance-snapshots:snapshots.schedule` instance option. For example, to configure daily snapshots, use the following command: incus config set snapshots.schedule @daily To configure taking a snapshot every day at 6 am, use the following command: incus config set snapshots.schedule "0 6 * * *" When scheduling regular snapshots, consider setting an automatic expiry ({config:option}`instance-snapshots:snapshots.expiry`) and a naming pattern for snapshots ({config:option}`instance-snapshots:snapshots.pattern`). You should also configure whether you want to take snapshots of instances that are not running ({config:option}`instance-snapshots:snapshots.schedule.stopped`). ### Restore an instance snapshot You can restore an instance to any of its snapshots. To do so, use the following command: incus snapshot restore If the snapshot is stateful (which means that it contains information about the running state of the instance), you can add the `--stateful` flag to restore the state. (instances-backup-export)= ## Use export files for instance backup You can export the full content of your instance to a standalone file that can be stored at any location. For highest reliability, store the backup file on a different file system to ensure that it does not get lost or corrupted. ### Export an instance Use the following command to export an instance to a compressed file (for example, `/path/to/my-instance.tgz`): incus export [] If you do not specify a file path, the export file is saved as `.` in the working directory (for example, `my-container.tar.gz`). % Include content from [storage_backup_volume.md](storage_backup_volume.md) ```{include} storage_backup_volume.md :start-after: :end-before: ``` `--instance-only` : By default, the export file contains all snapshots of the instance. Add this flag to export the instance without its snapshots. ### Restore an instance from an export file You can import an export file (for example, `/path/to/my-backup.tgz`) as a new instance. To do so, use the following command: incus import [] If you do not specify an instance name, the original name of the exported instance is used for the new instance. If an instance with that name already (or still) exists in the specified storage pool, the command returns an error. In that case, either delete the existing instance before importing the backup or specify a different instance name for the import. (instances-backup-copy)= ## Copy an instance to a backup server You can copy an instance to a secondary backup server to back it up. See {ref}`move-instances` for instructions. incus-7.0.0/doc/howto/instances_configure.md000066400000000000000000000204751517523235500211360ustar00rootroot00000000000000(instances-configure)= # How to configure instances You can configure instances by setting {ref}`instance-properties`, {ref}`instance-options`, or by adding and configuring {ref}`devices`. See the following sections for instructions. ```{note} To store and reuse different instance configurations, use {ref}`profiles `. ``` (instances-configure-options)= ## Configure instance options You can specify instance options when you {ref}`create an instance `. Alternatively, you can update the instance options after the instance is created. ````{tabs} ```{group-tab} CLI Use the [`incus config set`](incus_config_set.md) command to update instance options. Specify the instance name and the key and value of the instance option: incus config set = = ... ``` ```{group-tab} API Send a PATCH request to the instance to update instance options. Specify the instance name and the key and value of the instance option: incus query --request PATCH /1.0/instances/ --data '{"config": {"":"","":""}}' See [`PATCH /1.0/instances/{name}`](swagger:/instances/instance_patch) for more information. ``` ```` See {ref}`instance-options` for a list of available options and information about which options are available for which instance type. For example, change the memory limit for your container: ````{tabs} ```{group-tab} CLI To set the memory limit to 8 GiB, enter the following command: incus config set my-container limits.memory=8GiB ``` ```{group-tab} API To set the memory limit to 8 GiB, send the following request: incus query --request PATCH /1.0/instances/my-container --data '{"config": {"limits.memory":"8GiB"}}' ``` ```` ```{note} Some of the instance options are updated immediately while the instance is running. Others are updated only when the instance is restarted. See the "Live update" information in the {ref}`instance-options` reference for information about which options are applied immediately while the instance is running. ``` (instances-configure-properties)= ## Configure instance properties ````{tabs} ```{group-tab} CLI To update instance properties after the instance is created, use the [`incus config set`](incus_config_set.md) command with the `--property` flag. Specify the instance name and the key and value of the instance property: incus config set = = ... --property Using the same flag, you can also unset a property just like you would unset a configuration option: incus config unset --property You can also retrieve a specific property value with: incus config get --property ``` ```{group-tab} API To update instance properties through the API, use the same mechanism as for configuring instance options. The only difference is that properties are on the root level of the configuration, while options are under the `config` field. Therefore, to set an instance property, send a PATCH request to the instance: incus query --request PATCH /1.0/instances/ --data '{"":"","":"property_value>"}}' To unset an instance property, send a PUT request that contains the full instance configuration that you want except for the property that you want to unset. See [`PATCH /1.0/instances/{name}`](swagger:/instances/instance_patch) and [`PUT /1.0/instances/{name}`](swagger:/instances/instance_put) for more information. ``` ```` (instances-configure-devices)= ## Configure devices Generally, devices can be added or removed for a container while it is running. VMs support hotplugging for some device types, but not all. See {ref}`devices` for a list of available device types and their options. ```{note} Every device entry is identified by a name unique to the instance. Devices from profiles are applied to the instance in the order in which the profiles are assigned to the instance. Devices defined directly in the instance configuration are applied last. At each stage, if a device with the same name already exists from an earlier stage, the whole device entry is overridden by the latest definition. Device names are limited to a maximum of 64 characters. ``` `````{tabs} ````{group-tab} CLI To add and configure an instance device for your instance, use the [`incus config device add`](incus_config_device_add.md) command. Specify the instance name, a device name, the device type and maybe device options (depending on the {ref}`device type `): incus config device add = = ... For example, to add the storage at `/share/c1` on the host system to your instance at path `/opt`, enter the following command: incus config device add my-container disk-storage-device disk source=/share/c1 path=/opt To configure instance device options for a device that you have added earlier, use the [`incus config device set`](incus_config_device_set.md) command: incus config device set = = ... ```{note} You can also specify device options by using the `--device` flag when {ref}`creating an instance `. This is useful if you want to override device options for a device that is provided through a {ref}`profile `. ``` To remove a device, use the [`incus config device remove`](incus_config_device_remove.md) command. See [`incus config device --help`](incus_config_device.md) for a full list of available commands. ```` ````{group-tab} API To add and configure an instance device for your instance, use the same mechanism of patching the instance configuration. The device configuration is located under the `devices` field of the configuration. Specify the instance name, a device name, the device type and maybe device options (depending on the {ref}`device type `): incus query --request PATCH /1.0/instances/ --data '{"devices": {"": {"type":"","":"","":"device_option_value>"}}}' For example, to add the storage at `/share/c1` on the host system to your instance at path `/opt`, enter the following command: incus query --request PATCH /1.0/instances/my-container --data '{"devices": {"disk-storage-device": {"type":"disk","source":"/share/c1","path":"/opt"}}}' See [`PATCH /1.0/instances/{name}`](swagger:/instances/instance_patch) for more information. ```` ````` ## Display instance configuration ````{tabs} ```{group-tab} CLI To display the current configuration of your instance, including writable instance properties, instance options, devices and device options, enter the following command: incus config show --expanded ``` ```{group-tab} API To retrieve the current configuration of your instance, including writable instance properties, instance options, devices and device options, send a GET request to the instance: incus query /1.0/instances/ See [`GET /1.0/instances/{name}`](swagger:/instances/instance_get) for more information. ``` ```` (instances-configure-edit)= ## Edit the full instance configuration `````{tabs} ````{group-tab} CLI To edit the full instance configuration, including writable instance properties, instance options, devices and device options, enter the following command: incus config edit ```{note} For convenience, the [`incus config edit`](incus_config_edit.md) command displays the full configuration including read-only instance properties. However, you cannot edit those properties. Any changes are ignored. ``` ```` ````{group-tab} API To update the full instance configuration, including writable instance properties, instance options, devices and device options, send a PUT request to the instance: incus query --request PUT /1.0/instances/ --data '' See [`PUT /1.0/instances/{name}`](swagger:/instances/instance_put) for more information. ```{note} If you include changes to any read-only instance properties in the configuration you provide, they are ignored. ``` ```` ````` incus-7.0.0/doc/howto/instances_console.md000066400000000000000000000027151517523235500206140ustar00rootroot00000000000000(instances-console)= # How to access the console Use the [`incus console`](incus_console.md) command to attach to instance consoles. The console is available at boot time already, so you can use it to see boot messages and, if necessary, debug startup issues of a container or VM. To get an interactive console, enter the following command: incus console To show log output, pass the `--show-log` flag: incus console --show-log You can also immediately attach to the console when you start your instance: incus start --console incus start --console=vga ## Access the graphical console (for virtual machines) On virtual machines, log on to the console to get graphical output. Using the console you can, for example, install an operating system using a graphical interface or run a desktop environment. An additional advantage is that the console is available even if the `incus-agent` process is not running. This means that you can access the VM through the console before the `incus-agent` starts up, and also if the `incus-agent` is not available at all. To start the VGA console with graphical output for your VM, you must install a SPICE client. Incus supports two common clients: - `remote-viewer` (often part of the `virt-viewer` package) - `spicy` (part of the `spice-client-gtk` or `spice-gtk-tools` package) Then enter the following command: incus console --type vga incus-7.0.0/doc/howto/instances_create.md000066400000000000000000000265151517523235500204210ustar00rootroot00000000000000(instances-create)= # How to create instances To create an instance, you can use either the [`incus init`](incus_create.md) or the [`incus launch`](incus_launch.md) command. The [`incus init`](incus_create.md) command only creates the instance, while the [`incus launch`](incus_launch.md) command creates and starts it. ## Usage Enter the following command to create a container: incus launch|init : [flags] Image : Images contain a basic operating system (for example, a Linux distribution) and some Incus-related information. Images for various operating systems are available on the built-in remote image servers. See {ref}`images` for more information. Unless the image is available locally, you must specify the name of the image server and the name of the image (for example, `images:debian/12` for a Debian 12 image). Instance name : Instance names must be unique within an Incus deployment (also within a cluster). See {ref}`instance-properties` for additional requirements. Flags : See [`incus launch --help`](incus_launch.md) or [`incus init --help`](incus_create.md) for a full list of flags. The most common flags are: - `--config` to specify a configuration option for the new instance - `--device` to override {ref}`device options ` for a device provided through a profile, or to specify an {ref}`initial configuration for the root disk device ` - `--profile` to specify a {ref}`profile ` to use for the new instance - `--network` or `--storage` to make the new instance use a specific network or storage pool - `--target` to create the instance on a specific cluster member - `--vm` to create a virtual machine instead of a container ## Pass a configuration file Instead of specifying the instance configuration as flags, you can pass it to the command as a YAML file. For example, to launch a container with the configuration from `config.yaml`, enter the following command: incus launch images:debian/12 debian-config < config.yaml ```{tip} Check the contents of an existing instance configuration ([`incus config show --expanded`](incus_config_show.md)) to see the required syntax of the YAML file. ``` ## Examples The following examples use [`incus launch`](incus_launch.md), but you can use [`incus init`](incus_create.md) in the same way. ### Launch a system container To launch a system container with a Debian 12 image from the `images` server using the instance name `debian-container`, enter the following command: incus launch images:debian/12 debian-container ### Launch an application container To launch an application (OCI) container, you first need to add an image registry: incus remote add oci-docker https://docker.io --protocol=oci And then can launch a container from one of its images: incus launch oci-docker:hello-world --ephemeral --console ### Launch a virtual machine To launch a virtual machine with a Debian 12 image from the `images` server using the instance name `debian-vm`, enter the following command: incus launch images:debian/12 debian-vm --vm Or with a bigger disk: incus launch images:debian/12 debian-vm-big --vm --device root,size=30GiB ### Launch a container with specific configuration options To launch a container and limit its resources to one vCPU and 192 MiB of RAM, enter the following command: incus launch images:debian/12 debian-limited --config limits.cpu=1 --config limits.memory=192MiB ### Launch a VM on a specific cluster member To launch a virtual machine on the cluster member `server2`, enter the following command: incus launch images:debian/12 debian-container --vm --target server2 ### Launch a container with a specific instance type Incus supports simple instance types for clouds. Those are represented as a string that can be passed at instance creation time. The syntax allows the three following forms: - `` - `:` - `c-m` For example, the following three instance types are equivalent: - `t2.micro` - `aws:t2.micro` - `c1-m1` To launch a container with this instance type, enter the following command: incus launch images:debian/12 my-instance --type t2.micro The list of supported clouds and instance types can be found at [`https://github.com/dustinkirkland/instance-type`](https://github.com/dustinkirkland/instance-type). ### Launch a VM that boots from an ISO ```{note} When creating a Windows, macOS, or FreeBSD virtual machine, make sure to set the `image.os` property to something starting with `Windows`, `macOS`, or `FreeBSD` respectively. Doing so will tell Incus to expect the correct OS to be running inside of the virtual machine and to tweak behavior accordingly. This notably will cause: - On Windows: - Some unsupported virtual devices to be disabled - The {abbr}`RTC (Real Time Clock)` clock to be based on system local time rather than UTC - IOMMU handling to switch to an Intel IOMMU controller - On FreeBSD: - Memory hotplug to be disabled ``` To launch a VM that boots from an ISO, you must first create a VM. Let's assume that we want to create a VM and install it from the ISO image. In this scenario, use the following command to create an empty VM: incus init iso-vm --empty --vm ```{note} Depending on the needs of the operating system being installed, you may want to allocate more CPU, memory or storage to the virtual machine. For example, for 2 CPUs, 4 GiB of memory and 50 GiB of storage, you can do: incus init iso-vm --empty --vm -c limits.cpu=2 -c limits.memory=4GiB -d root,size=50GiB ``` The second step is to import an ISO image that can later be attached to the VM as a storage volume: incus storage volume import iso-volume --type=iso Lastly, you need to attach the custom ISO volume to the VM using the following command: incus config device add iso-vm iso-volume disk pool= source=iso-volume boot.priority=10 The `boot.priority` configuration key ensures that the VM will boot from the ISO first. Start the VM and connect to the console as there might be a menu you need to interact with: incus start iso-vm --console Once you're done in the serial console, you need to disconnect from the console using `ctrl+a-q`, and connect to the VGA console using the following command: incus console iso-vm --type=vga You should now see the installer. After the installation is done, you need to detach the custom ISO volume: incus storage volume detach iso-volume iso-vm Now the VM can be rebooted, and it will boot from disk. ### Install the Incus Agent into virtual machine instances ```{warning} The Incus agent relies on TLS certificates for communication between the host and guest. For this to work correctly, the guest clock needs to be reasonably in sync with the host. ``` In order for features like direct command execution (`incus exec`), file transfers (`incus file`) and detailed usage metrics (`incus info`) to work properly with virtual machines, an agent software is provided by Incus. The virtual machine images from the [images](https://images.linuxcontainers.org) remote are pre-configured to load that agent on startup. For other virtual machines, you may want to manually install the agent. ```{note} The Incus Agent is currently available only on Linux, Windows, macOS, and FreeBSD virtual machines. ``` Incus provides the agent primarily through a remote `9p` file system with mount name `config`. Alternatively, it is possible to get the agent files through a virtual CD-ROM drive by adding a `disk` device to the instance and using `agent:config` as the `source` property. incus config device add INSTANCE-NAME agent disk source=agent:config ```{note} The agent CD-ROM drive must remain attached to the VM as it is accessed on every boot to refresh the agent and to get the needed credentials to interact with Incus. ``` #### On Linux To install the agent on a Linux system with `9p`, you'll need to get access to the virtual machine and run the following commands: mount -t 9p config /mnt cd /mnt ./install.sh When using the virtual CD-ROM drive, you can use the following instead: mount /dev/disk/by-label/incus-agent /mnt cd /mnt ./install.sh ```{note} All installation commands showed above should be run from a `root` shell. They require a Linux system using `systemd` as its init system. The first line will mount the remote file system on the mount point `/mnt`. The subsequent commands will run the installation script `install.sh` to install and run the Incus Agent. ``` #### On Windows For Windows systems, the virtual CD-ROM drive must be used. The agent can be installed as a service by running the `install.ps1` file from the CD-ROM drive (by either executing the file from the explorer or terminal). To automatically let the agent update itself, leave the CD-ROM available to the virtual machine ```{note} To install: The agent as a service, local administrator privileges are required. To update: When rebooting the virtual machine, the CD-ROM will be mounted again with the newest version and the service will automatically update itself using the files in the CD-ROM drive. ``` Otherwise, the agent can manually be started by opening a terminal and running (assuming `d:\` is the CD-ROM): d:\ .\incus-agent.exe #### On macOS For macOS systems, the agent can manually be installed using a `9p` mount by opening a terminal **as root** and running the following commands: mount_9p config cd /Volumes/config ./install.sh ```{warning} Apple's Transparency, Consent, Control daemon requires you to allow full disk access to `sh` for the agent to be automatically started. This reduces the overall security of the system, by relaxing some of Apple's additional security restrictions. This does not in any way bypass UNIX permissions, however, if you are not comfortable with that, you will need to manually run `incus-agent` each time. ``` #### On FreeBSD For FreeBSD systems, the agent can manually be installed using a `9p` mount by running the following commands **as root** (running `kldload virtio_p9fs` beforehand may be necessary if this module is not loaded): mount -t p9fs config /mnt cd /mnt ./install.sh ### Configure the Incus Agent By default the Incus Agent will have all features enabled. In some environments, VM owners may want to turn off specific features. This can be done through a `incus-agent.yml` file which is located at: - `/etc/incus-agent.yml` on Linux - `/usr/local/etc/incus-agent.yml` on supported BSD-like OSes (FreeBSD and macOS) - `C:\Program Files\Incus Agent\incus-agent.yml` on Windows If the file is missing or is empty, all features will be enabled. If the file contains a `features` map, then all features will be disabled unless specifically enabled. The supported features are: - `guestapi` controls whether the agent exposes the `/dev/incus` API within the guest - `exec` controls whether commands can be executed through the agent - `files` controls whether the files transfer API is available - `mounts` controls whether to setup the file system mounts for shared disk devices - `metrics` controls access to detailed OpenMetrics data - `state` controls access to basic OS state information (OS version, network interface details, ...) An example YAML file would be: ``` features: guestapi: true metrics: true state: true ``` incus-7.0.0/doc/howto/instances_manage.md000066400000000000000000000141571517523235500204050ustar00rootroot00000000000000(instances-manage)= # How to manage instances When listing the existing instances, you can see their type, status, and location (if applicable). You can filter the instances and display only the ones that you are interested in. ````{tabs} ```{group-tab} CLI Enter the following command to list all instances: incus list You can filter the instances that are displayed, for example, by type, status or the cluster member where the instance is located: incus list type=container incus list status=running incus list location=server1 You can also filter by name. To list several instances, use a regular expression for the name. For example: incus list debian.* Enter [`incus list --help`](incus_list.md) to see all filter options. ``` ```{group-tab} API Query the `/1.0/instances` endpoint to list all instances. You can use {ref}`rest-api-recursion` to display more information about the instances: incus query /1.0/instances?recursion=2 You can {ref}`filter ` the instances that are displayed, by name, type, status or the cluster member where the instance is located: incus query /1.0/instances?filter=name+eq+debian incus query /1.0/instances?filter=type+eq+container incus query /1.0/instances?filter=status+eq+running incus query /1.0/instances?filter=location+eq+server1 To list several instances, use a regular expression for the name. For example: incus query /1.0/instances?filter=name+eq+debian.* See [`GET /1.0/instances`](swagger:/instances/instances_get) for more information. ``` ```` ## Show information about an instance ````{tabs} ```{group-tab} CLI Enter the following command to show detailed information about an instance: incus info Add `--show-log` to the command to show the latest log lines for the instance: incus info --show-log ``` ```{group-tab} API Query the following endpoint to show detailed information about an instance: incus query /1.0/instances/ See [`GET /1.0/instances/{name}`](swagger:/instances/instance_get) for more information. ``` ```` ## Start an instance ````{tabs} ```{group-tab} CLI Enter the following command to start an instance: incus start You will get an error if the instance does not exist or if it is running already. To immediately attach to the console when starting, pass the `--console` flag. For example: incus start --console See {ref}`instances-console` for more information. ``` ```{group-tab} API To start an instance, send a PUT request to change the instance state: incus query --request PUT /1.0/instances//state --data '{"action":"start"}' The return value of this query contains an operation ID, which you can use to query the status of the operation: incus query /1.0/operations/ Use the following query to monitor the state of the instance: incus query /1.0/instances//state See [`GET /1.0/instances/{name}/state`](swagger:/instances/instance_state_get) and [`PUT /1.0/instances/{name}/state`](swagger:/instances/instance_state_put)for more information. ``` ```` (instances-manage-stop)= ## Stop an instance `````{tabs} ````{group-tab} CLI Enter the following command to stop an instance: incus stop You will get an error if the instance does not exist or if it is not running. ```` ````{group-tab} API To stop an instance, send a PUT request to change the instance state: incus query --request PUT /1.0/instances//state --data '{"action":"stop"}' % Include content from above ```{include} ./instances_manage.md :start-after: :end-before: ``` ```` ````` ## Delete an instance If you don't need an instance anymore, you can remove it. The instance must be stopped before you can delete it. `````{tabs} ```{group-tab} CLI Enter the following command to delete an instance: incus delete ``` ```{group-tab} API To delete an instance, send a DELETE request to the instance: incus query --request DELETE /1.0/instances/ See [`DELETE /1.0/instances/{name}`](swagger:/instances/instance_delete) for more information. ``` ````` ```{caution} This command permanently deletes the instance and all its snapshots. ``` ### Prevent accidental deletion of instances There are different ways to prevent accidental deletion of instances: - To protect a specific instance from being deleted, set {config:option}`instance-security:security.protection.delete` to `true` for the instance. See {ref}`instances-configure` for instructions. - In the CLI client, you can create an alias to be prompted for approval every time you use the [`incus delete`](incus_delete.md) command: incus alias add delete "delete -i" ## Rebuild an instance If you want to wipe and re-initialize the root disk of your instance but keep the instance configuration, you can rebuild the instance. Rebuilding is only possible for instances that do not have any snapshots. Stop your instance before rebuilding it. ````{tabs} ```{group-tab} CLI Enter the following command to rebuild the instance with a different image: incus rebuild Enter the following command to rebuild the instance with an empty root disk: incus rebuild --empty For more information about the `rebuild` command, see [`incus rebuild --help`](incus_rebuild.md). ``` ```{group-tab} API To rebuild the instance with a different image, send a POST request to the instance's `rebuild` endpoint. For example: incus query --request POST /1.0/instances//rebuild --data '{"source": {"alias":"","server":"", protocol:"simplestreams"}}' To rebuild the instance with an empty root disk, specify the source type as `none`: incus query --request POST /1.0/instances//rebuild --data '{"source": {"type":"none"}}' See [`POST /1.0/instances/{name}/rebuild`](swagger:/instances/instance_rebuild_post) for more information. ``` ```` incus-7.0.0/doc/howto/instances_routed_nic_vm.md000066400000000000000000000042601517523235500220040ustar00rootroot00000000000000(instances-routed-nic-vm)= # How to add a routed NIC device to a virtual machine When adding a {ref}`routed NIC device ` to an instance, you must configure the instance to use the link-local gateway IPs as default routes. For containers, this is configured for you automatically. For virtual machines, the gateways must be configured manually or via a mechanism like `cloud-init`. To configure the gateways with `cloud-init`, firstly initialize an instance: incus init images:debian/12 bookworm --vm Then add the routed NIC device: incus config device add bookworm eth0 nic nictype=routed parent=my-parent-network ipv4.address=192.0.2.2 ipv6.address=2001:db8::2 In this command, `my-parent-network` is your parent network, and the IPv4 and IPv6 addresses are within the subnet of the parent. Next we will add some `netplan` configuration to the instance using the `cloud-init.network-config` configuration key: cat <` (`169.254.0.1` and `fe80::1`) that are required. For each of these routes we set `on-link` to `true`, which specifies that the route is directly connected to the interface. We also add the addresses that we configured in our routed NIC device. For more information on `netplan`, see [their documentation](https://netplan.readthedocs.io/en/latest/). ```{note} This `netplan` configuration does not include a name server. To enable DNS within the instance, you must set a valid DNS IP address. If there is a `incusbr0` network on the host, the name server can be set to that IP instead. ``` You can then start your instance with: incus start bookworm ```{note} Before you start your instance, make sure that you have {ref}`configured the parent network ` to enable proxy ARP/NDP. ``` incus-7.0.0/doc/howto/instances_troubleshoot.md000066400000000000000000000060621517523235500217020ustar00rootroot00000000000000(instances-troubleshoot)= # How to troubleshoot failing instances If your instance fails to start and ends up in an error state, this usually indicates a bigger issue related to either the image that you used to create the instance or the server configuration. To troubleshoot the problem, complete the following steps: 1. Save the relevant log files and debug information: Instance log : Enter the following command to display the instance log: incus info --show-log Console log : Enter the following command to display the console log: incus console --show-log 1. Reboot the machine that runs your Incus server. 1. Try starting your instance again. If the error occurs again, compare the logs to check if it is the same error. If it is, and if you cannot figure out the source of the error from the log information, open a question in the [forum](https://discuss.linuxcontainers.org). Make sure to include the log files you collected. ## Troubleshooting example In this example, let's investigate a RHEL 7 system in which `systemd` cannot start. ```{terminal} :input: incus console --show-log systemd Console log: Failed to insert module 'autofs4' Failed to insert module 'unix' Failed to mount sysfs at /sys: Operation not permitted Failed to mount proc at /proc: Operation not permitted [!!!!!!] Failed to mount API filesystems, freezing. ``` The errors here say that `/sys` and `/proc` cannot be mounted - which is correct in an unprivileged container. However, Incus mounts these file systems automatically if it can. The {doc}`container requirements <../container-environment>` specify that every container must come with an empty `/dev`, `/proc` and `/sys` directory, and that `/sbin/init` must exist. If those directories don't exist, Incus cannot mount them, and `systemd` will then try to do so. As this is an unprivileged container, `systemd` does not have the ability to do this, and it then freezes. So you can see the environment before anything is changed, and you can explicitly change the init system in a container using the `raw.lxc` configuration parameter. This is equivalent to setting `init=/bin/bash` on the Linux kernel command line. incus config set systemd raw.lxc 'lxc.init.cmd = /bin/bash' Here is what it looks like: ```{terminal} :input: incus config set systemd raw.lxc 'lxc.init.cmd = /bin/bash' :input: incus start systemd :input: incus console --show-log systemd Console log: [root@systemd /]# ``` Now that the container has started, you can check it and see that things are not running as well as expected: ```{terminal} :input: incus exec systemd -- bash [root@systemd ~]# ls [root@systemd ~]# mount mount: failed to read mtab: No such file or directory [root@systemd ~]# cd / [root@systemd /]# ls /proc/ sys [root@systemd /]# exit ``` Because Incus tries to auto-heal, it created some of the directories when it was starting up. Shutting down and restarting the container fixes the problem, but the original cause is still there - the template does not contain the required files. incus-7.0.0/doc/howto/migrate_from_lxc.md000066400000000000000000000066701517523235500204300ustar00rootroot00000000000000(migrate-from-lxc)= # How to migrate containers from LXC to Incus Incus provides a tool (`lxc-to-incus`) that you can use to import LXC containers into your Incus server. The LXC containers must exist on the same machine as the Incus server. The tool analyzes the LXC containers and migrates both their data and their configuration into new Incus containers. ```{note} Alternatively, you can use the `incus-migrate` tool within a LXC container to migrate it to Incus (see {ref}`import-machines-to-instances`). However, this tool does not migrate any of the LXC container configuration. ``` ## Get the tool If the tool isn't provided alongside your Incus installation, you can build it yourself. Make sure that you have `go` ({ref}`requirements-go`) installed and get the tool with the following command: go install github.com/lxc/incus/cmd/lxc-to-incus@latest ## Prepare your LXC containers You can migrate one container at a time or all of your LXC containers at the same time. ```{note} Migrated containers use the same name as the original containers. You cannot migrate containers with a name that already exists as an instance name in Incus. Therefore, rename any LXC containers that might cause name conflicts before you start the migration process. ``` Before you start the migration process, stop the LXC containers that you want to migrate. ## Start the migration process Run `sudo lxc-to-incus [flags]` to migrate the containers. For example, to migrate all containers: sudo lxc-to-incus --all To migrate only the `lxc1` container: sudo lxc-to-incus --containers lxc1 To migrate two containers (`lxc1` and `lxc2`) and use the `my-storage` storage pool in Incus: sudo lxc-to-incus --containers lxc1,lxc2 --storage my-storage To test the migration of all containers without actually running it: sudo lxc-to-incus --all --dry-run To migrate all containers but limit the `rsync` bandwidth to 5000 KB/s: sudo lxc-to-incus --all --rsync-args --bwlimit=5000 Run `sudo lxc-to-incus --help` to check all available flags. ```{note} If you get an error that the `linux64` architecture isn't supported, either update the tool to the latest version or change the architecture in the LXC container configuration from `linux64` to either `amd64` or `x86_64`. ``` ## Check the configuration The tool analyzes the LXC configuration and the configuration of the container (or containers) and migrates as much of the configuration as possible. You will see output similar to the following: ```{terminal} :input: sudo lxc-to-incus --containers lxc1 Parsing LXC configuration Checking for unsupported LXC configuration keys Checking for existing containers Checking whether container has already been migrated Validating whether incomplete AppArmor support is enabled Validating whether mounting a minimal /dev is enabled Validating container rootfs Processing network configuration Processing storage configuration Processing environment configuration Processing container boot configuration Processing container apparmor configuration Processing container seccomp configuration Processing container SELinux configuration Processing container capabilities configuration Processing container architecture configuration Creating container Transferring container: lxc1: ... Container 'lxc1' successfully created ``` After the migration process is complete, you can check and, if necessary, update the configuration in Incus before you start the migrated Incus container. incus-7.0.0/doc/howto/move_instances.md000066400000000000000000000073671517523235500201300ustar00rootroot00000000000000(move-instances)= # How to move existing Incus instances between servers To move an instance from one Incus server to another, use the [`incus move`](incus_move.md) command: incus move [:] :[] ```{note} When moving a container, you must stop it first. See {ref}`live-migration-containers` for more information. When moving a virtual machine, you must either enable {ref}`live-migration-vms` or stop it first. ``` Alternatively, you can use the [`incus copy`](incus_copy.md) command if you want to duplicate the instance: incus copy [:] :[] In both cases, you don't need to specify the source remote if it is your default remote, and you can leave out the target instance name if you want to use the same instance name. If you want to move the instance to a specific cluster member, specify it with the `--target` flag. In this case, do not specify the source and target remote. You can add the `--mode` flag to choose a transfer mode, depending on your network setup: `pull` (default) : Instruct the target server to connect to the source server and pull the respective instance. `push` : Instruct the source server to connect to the target server and push the instance. `relay` : Instruct the client to connect to both the source and the target server and transfer the data through the client. If you need to adapt the configuration for the instance to run on the target server, you can either specify the new configuration directly (using `--config`, `--device`, `--storage` or `--target-project`) or through profiles (using `--no-profiles` or `--profile`). See [`incus move --help`](incus_move.md) for all available flags. (live-migration)= ## Live migration Live migration means migrating an instance while it is running. This method is supported for virtual machines. For containers, there is limited support. (live-migration-vms)= ### Live migration for virtual machines Virtual machines can be moved to another server while they are running, thus without any downtime. To allow for live migration, you must enable support for stateful migration. To do so, ensure the following configuration: * Set {config:option}`instance-migration:migration.stateful` to `true` on the instance. (live-migration-containers)= ### Live migration for containers For containers, there is limited support for live migration using [{abbr}`CRIU (Checkpoint/Restore in Userspace)`](https://criu.org/). However, because of extensive kernel dependencies, only very basic containers (non-`systemd` containers without a network device) can be migrated reliably. In most real-world scenarios, you should stop the container, move it over and then start it again. If you want to use live migration for containers, you must first make sure that CRIU is installed on both systems. To optimize the memory transfer for a container, set the {config:option}`instance-migration:migration.incremental.memory` property to `true` to make use of the pre-copy features in CRIU. With this configuration, Incus instructs CRIU to perform a series of memory dumps for the container. After each dump, Incus sends the memory dump to the specified remote. In an ideal scenario, each memory dump will decrease the delta to the previous memory dump, thereby increasing the percentage of memory that is already synced. When the percentage of synced memory is equal to or greater than the threshold specified via {config:option}`instance-migration:migration.incremental.memory.goal`, or the maximum number of allowed iterations specified via {config:option}`instance-migration:migration.incremental.memory.iterations` is reached, Incus instructs CRIU to perform a final memory dump and transfers it. incus-7.0.0/doc/howto/network_acls.md000066400000000000000000000257771517523235500176130ustar00rootroot00000000000000(network-acls)= # How to configure network ACLs ```{note} Network ACLs are available for the {ref}`OVN NIC type `, the {ref}`network-ovn` and the {ref}`network-bridge` (with some exceptions, see {ref}`network-acls-bridge-limitations`). ``` Network {abbr}`ACLs (Access Control Lists)` define traffic rules that allow controlling network access between different instances connected to the same network, and access to and from other networks. Network ACLs can be assigned directly to the {abbr}`NIC (Network Interface Controller)` of an instance or to a network. When assigned to a network, the ACL applies to all NICs connected to the network. The instance NICs that have a particular ACL applied (either explicitly or implicitly through a network) make up a logical group, which can be referenced from other rules as a source or destination. See {ref}`network-acls-groups` for more information. ## Create an ACL Use the following command to create an ACL: ```bash incus network acl create [configuration_options...] ``` This command creates an ACL without rules. As a next step, {ref}`add rules ` to the ACL. Valid network ACL names must adhere to the following rules: - Names must be between 1 and 63 characters long. - Names must be made up exclusively of letters, numbers and dashes from the ASCII table. - Names must not start with a digit or a dash. - Names must not end with a dash. ### ACL properties ACLs have the following properties: | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | `name` | string | yes | Unique name of the network ACL in the project | | `description` | string | no | Description of the network ACL | | `ingress` | rule list | no | Ingress traffic rules | | `egress` | rule list | no | Egress traffic rules | | `config` | string set | no | Configuration options as key/value pairs (only `user.*` custom keys supported) | (network-acls-rules)= ## Add or remove rules Each ACL contains two lists of rules: - *Ingress* rules apply to inbound traffic going towards the NIC. - *Egress* rules apply to outbound traffic leaving the NIC. To add a rule to an ACL, use the following command, where `` can be either `ingress` or `egress`: ```bash incus network acl rule add [properties...] ``` This command adds a rule to the list for the specified direction. You cannot edit a rule (except if you {ref}`edit the full ACL `), but you can delete rules with the following command: ```bash incus network acl rule remove [properties...] ``` You must either specify all properties needed to uniquely identify a rule or add `--force` to the command to delete all matching rules. ### Rule ordering and priorities Rules are provided as lists. However, the order of the rules in the list is not important and does not affect filtering. Incus automatically orders the rules based on the `action` property as follows: - `drop` - `reject` - `allow` - Automatic default action for any unmatched traffic (defaults to `reject`, see {ref}`network-acls-defaults`). This means that when you apply multiple ACLs to a NIC, there is no need to specify a combined rule ordering. If one of the rules in the ACLs matches, the action for that rule is taken and no other rules are considered. (network-acls-rules-properties)= ### Rule properties ACL rules have the following properties: | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | `action` | string | yes | Action to take for matching traffic (`allow`, `allow-stateless`, `reject`, or `drop`) | | `state` | string | yes | State of the rule (`enabled`, `disabled` or `logged`), defaulting to `enabled` if not specified | | `description` | string | no | Description of the rule | | `source` | string | no | Comma-separated list of CIDR or IP ranges, source subject name selectors (for ingress rules), or empty for any | | `destination` | string | no | Comma-separated list of CIDR or IP ranges, destination subject name selectors (for egress rules), or empty for any | | `protocol` | string | no | Protocol to match (`icmp4`, `icmp6`, `tcp`, `udp`) or empty for any | | `source_port` | string | no | If protocol is `udp` or `tcp`, then a comma-separated list of ports or port ranges (start-end inclusive), or empty for any | | `destination_port` | string | no | If protocol is `udp` or `tcp`, then a comma-separated list of ports or port ranges (start-end inclusive), or empty for any | | `icmp_type` | string | no | If protocol is `icmp4` or `icmp6`, then ICMP type number, or empty for any | | `icmp_code` | string | no | If protocol is `icmp4` or `icmp6`, then ICMP code number, or empty for any | (network-acls-selectors)= ### Use selectors in rules ```{note} This feature is supported only for the {ref}`OVN NIC type ` and the {ref}`network-ovn`. ``` The `source` field (for ingress rules) and the `destination` field (for egress rules) support using selectors instead of CIDR or IP ranges. With this method, you can use ACL groups or network selectors to define rules for groups of instances without needing to maintain IP lists or create additional subnets. (network-acls-address-sets)= ### Use address sets in rules ```{note} This feature is supported only for the {ref}`bridge network using ` `nftables` and the {ref}`network-ovn`. ``` The `source` field (for ingress rules) and the `destination` field (for egress rules) support using address sets. With this feature you can create groups of addresses and / or networks to match rules against. You can eventually mix them with literals addresses and CIDR. To use one in a rule: ``` source=\$ ``` (network-acls-groups)= #### ACL groups Instance NICs that are assigned a particular ACL (either explicitly or implicitly through a network) make up a logical port group. Such ACL groups are called *subject name selectors*, and they can be referenced with the name of the ACL in other ACL groups. For example, if you have an ACL with the name `foo`, you can specify the group of instance NICs that are assigned this ACL as source with `source=foo`. #### Network selectors You can use *network subject selectors* to define rules based on the network that the traffic is coming from or going to. There are two special network subject selectors called `@internal` and `@external`. They represent network local and external traffic, respectively. For example: ```bash source=@internal ``` If your network supports [network peers](network_ovn_peers.md), you can reference traffic to or from the peer connection by using a network subject selector in the format `@/`. For example: ```bash source=@ovn1/mypeer ``` When using a network subject selector, the network that has the ACL applied to it must have the specified peer connection. Otherwise, the ACL cannot be applied to it. ### Log traffic Generally, ACL rules are meant to control the network traffic between instances and networks. However, you can also use them to log specific network traffic, which can be useful for monitoring, or to test rules before actually enabling them. To add a rule for logging, create it with the `state=logged` property. You can then display the log output for all logging rules in the ACL with the following command: ```bash incus network acl show-log ``` (network-acls-edit)= ## Edit an ACL Use the following command to edit an ACL: ```bash incus network acl edit ``` This command opens the ACL in YAML format for editing. You can edit both the ACL configuration and the rules. ## Assign an ACL After configuring an ACL, you must assign it to a network or an instance NIC. To do so, add it to the `security.acls` list of the network or NIC configuration. For networks, use the following command: ```bash incus network set security.acls="" ``` For instance NICs, use the following command: ```bash incus config device set security.acls="" ``` (network-acls-defaults)= ## Configure default actions When one or more ACLs are applied to a NIC (either explicitly or implicitly through a network), a default reject rule is added to the NIC. This rule rejects all traffic that doesn't match any of the rules in the applied ACLs. You can change this behavior with the network and NIC level `security.acls.default.ingress.action` and `security.acls.default.egress.action` settings. The NIC level settings override the network level settings. For example, to set the default action for inbound traffic to `allow` for all instances connected to a network, use the following command: ```bash incus network set security.acls.default.ingress.action=allow ``` To configure the same default action for an instance NIC, use the following command: ```bash incus config device set security.acls.default.ingress.action=allow ``` (network-acls-bridge-limitations)= ## Bridge limitations When using network ACLs with a bridge network, be aware of the following limitations: - Unlike OVN ACLs, bridge ACLs are applied only on the boundary between the bridge and the Incus host. This means they can only be used to apply network policies for traffic going to or from external networks. They cannot be used for to create {spellexception}`intra-bridge` firewalls, thus firewalls that control traffic between instances connected to the same bridge, except when ACLs are applied directly to the NIC device. In that case the `reject` ACL rules applied to the ingress traffic are converted to `drop` to address `nftables` limitation. - {ref}`ACL groups and network selectors ` are not supported. - Baseline network service rules are added before ACL rules (in their respective INPUT/OUTPUT chains), because we cannot differentiate between INPUT/OUTPUT and FORWARD traffic once we have jumped into the ACL chain. Because of this, ACL rules cannot be used to block baseline service rules. incus-7.0.0/doc/howto/network_address_sets.md000066400000000000000000000044441517523235500213400ustar00rootroot00000000000000(network-address-sets)= # How to use network address sets ```{note} Network address sets are working with {ref}`ACLs ` and work only with {ref}`network-ovn` or with {ref}`bridged networks ` using `nftables` only. ``` Network address sets are a list of either IPv4, IPv6 addresses with or without CIDR suffix. They can be used in source or destination fields of {ref}`ACLs `. ## Address set properties Address sets have the following properties: | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | `name` | string | yes | Name of the network address set | | `description` | string | no | Description of the network address set | | `addresses` | string list | no | Ingress traffic rules | ## Address set configuration options The following configuration options are available for all network address sets: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ## Creating an address set Use the following command to create an address set. ```bash incus network address-set create [configuration_options...] ``` This will create an address set without any addresses, after this you can {ref}`add addresses `. (manage-addresses-in-set)= ## Add or remove addresses Adding addresses is pretty straightforward: ```bash incus network address-set add ``` There is no restriction about the kind of address you are appending in your set, a mix of IPv4, IPv6 and CIDR can be used without disruption. To remove addresses, the same `remove` command can be used instead. ```bash incus network address-set remove ``` ## Use of address sets in ACL rules In order to use an address set in an {ref}`ACL `, we need to prepend `name` with `$` (you need to escape the dollar in command line). Then we can refer the address set in `source` or `destination` fields of an ACL rule. incus-7.0.0/doc/howto/network_bgp.md000066400000000000000000000075241517523235500174270ustar00rootroot00000000000000(network-bgp)= # How to configure Incus as a BGP server ```{note} The BGP server feature is available for the {ref}`network-bridge` and the {ref}`network-physical`. ``` {abbr}`BGP (Border Gateway Protocol)` is a protocol that allows exchanging routing information between autonomous systems. If you want to directly route external addresses to specific Incus servers or instances, you can configure Incus as a BGP server. Incus will then act as a BGP peer and advertise relevant routes and next hops to external routers, for example, your network router. It automatically establishes sessions with upstream BGP routers and announces the addresses and subnets that it's using. The BGP server feature can be used to allow an Incus server or cluster to directly use internal/external address space by getting the specific subnets or addresses routed to the correct host. This way, traffic can be forwarded to the target instance. For bridge networks, the following addresses and networks are being advertised: - Network `ipv4.address` or `ipv6.address` subnets (if the matching `nat` property isn't set to `true`) - Network `ipv4.nat.address` or `ipv6.nat.address` subnets (if the matching `nat` property is set to `true`) - Network forward addresses - Addresses or subnets specified in `ipv4.routes.external` or `ipv6.routes.external` on an instance NIC that is connected to the bridge network Make sure to add your subnets to the respective configuration options. Otherwise, they won't be advertised. For physical networks, no addresses are advertised directly at the level of the physical network. Instead, the networks, forwards and routes of all downstream networks (the networks that specify the physical network as their uplink network through the `network` option) are advertised in the same way as for bridge networks. ```{note} At this time, it is not possible to announce only some specific routes/addresses to particular peers. If you need this, filter prefixes on the upstream routers. ``` ## Configure the BGP server To configure Incus as a BGP server, set the following server configuration options on all cluster members: - {config:option}`server-core:core.bgp_address` - the IP address for the BGP server - {config:option}`server-core:core.bgp_asn` - the {abbr}`ASN (Autonomous System Number)` for the local server - {config:option}`server-core:core.bgp_routerid` - the unique identifier for the BGP server For example, set the following values: ```bash incus config set core.bgp_address=192.0.2.50:179 incus config set core.bgp_asn=65536 incus config set core.bgp_routerid=192.0.2.50 ``` Once these configuration options are set, Incus starts listening for BGP sessions. ### Configure next-hop (`bridge` only) For bridge networks, you can override the next-hop configuration. By default, the next-hop is set to the address used for the BGP session. To configure a different address, set `bgp.ipv4.nexthop` or `bgp.ipv6.nexthop`. ### Configure BGP peers for OVN networks If you run an OVN network with an uplink network (`physical` or `bridge`), the uplink network is the one that holds the list of allowed subnets and the BGP configuration. Therefore, you must configure BGP peers on the uplink network that contain the information that is required to connect to the BGP server. Set the following configuration options on the uplink network: - `bgp.peers..address` - the peer address to be used by the downstream networks - `bgp.peers..asn` - the {abbr}`ASN (Autonomous System Number)` for the local server - `bgp.peers..password` - an optional password for the peer session - `bgp.peers..holdtime` - an optional hold time for the peer session (in seconds) Once the uplink network is configured, downstream OVN networks will get their external subnets and addresses announced over BGP. The next-hop is set to the address of the OVN router on the uplink network. incus-7.0.0/doc/howto/network_bridge_firewalld.md000066400000000000000000000170211517523235500221350ustar00rootroot00000000000000(network-bridge-firewall)= # How to configure your firewall Linux firewalls are based on `netfilter`. Incus uses the same subsystem, which can lead to connectivity issues. If you run a firewall on your system, you might need to configure it to allow network traffic between the managed Incus bridge and the host. Otherwise, some network functionality (DHCP, DNS and external network access) might not work as expected. You might also see conflicts between the rules defined by your firewall (or another application) and the firewall rules that Incus adds. For example, your firewall might erase Incus rules if it is started after the Incus daemon, which might interrupt network connectivity to the instance. ## `nftables` Incus uses `nftables` to manage its `netfilter` rules. It places its rules in their own `nftables` namespace to keep them separated from rules added by other applications. However, if a packet is blocked in one namespace, it is not possible for another namespace to allow it. Therefore, rules in one namespace can still affect rules in another namespace, and firewall applications can still impact Incus network functionality. ## Use Incus' firewall By default, managed Incus bridges add firewall rules to ensure full functionality. If you do not run another firewall on your system, you can let Incus manage its firewall rules. To enable or disable this behavior, use the `ipv4.firewall` or `ipv6.firewall` {ref}`configuration options `. ## Use another firewall Firewall rules added by other applications might interfere with the firewall rules that Incus adds. Therefore, if you use another firewall, you should disable Incus' firewall rules. You must also configure your firewall to allow network traffic between the instances and the Incus bridge, so that the Incus instances can access the DHCP and DNS server that Incus runs on the host. See the following sections for instructions on how to disable Incus' firewall rules and how to properly configure `firewalld` and UFW, respectively. ### Disable Incus' firewall rules Run the following commands to prevent Incus from setting firewall rules for a specific network bridge (for example, `incusbr0`): incus network set ipv6.firewall false incus network set ipv4.firewall false ### `firewalld`: Add the bridge to the trusted zone To allow traffic to and from the Incus bridge in `firewalld`, add the bridge interface to the `trusted` zone. To do this permanently (so that it persists after a reboot), run the following commands: sudo firewall-cmd --zone=trusted --change-interface= --permanent sudo firewall-cmd --reload For example: sudo firewall-cmd --zone=trusted --change-interface=incusbr0 --permanent sudo firewall-cmd --reload ```{warning} The commands given above show a simple example configuration. Depending on your use case, you might need more advanced rules and the example configuration might inadvertently introduce a security risk. ``` ### UFW: Add rules for the bridge If UFW has a rule to drop all unrecognized traffic, it blocks the traffic to and from the Incus bridge. In this case, you must add rules to allow traffic to and from the bridge, as well as allowing traffic forwarded to it. To do so, run the following commands: sudo ufw allow in on sudo ufw route allow in on sudo ufw route allow out on For example: sudo ufw allow in on incusbr0 sudo ufw route allow in on incusbr0 sudo ufw route allow out on incusbr0 ````{warning} % Repeat warning from above ```{include} network_bridge_firewalld.md :start-after: :end-before: ``` Here's an example for more restrictive firewall rules that limit access from the guests to the host to only DHCP and DNS and allow all outbound connections: ``` # allow the guest to get an IP from the Incus host sudo ufw allow in on incusbr0 to any port 67 proto udp sudo ufw allow in on incusbr0 to any port 547 proto udp # allow the guest to resolve host names from the Incus host sudo ufw allow in on incusbr0 to any port 53 # allow the guest to have access to outbound connections CIDR4="$(incus network get incusbr0 ipv4.address | sed 's|\.[0-9]\+/|.0/|')" CIDR6="$(incus network get incusbr0 ipv6.address | sed 's|:[0-9]\+/|:/|')" sudo ufw route allow in on incusbr0 from "${CIDR4}" sudo ufw route allow in on incusbr0 from "${CIDR6}" ``` ```` (network-incus-docker)= ## Prevent connectivity issues with Incus and Docker Running Incus and Docker on the same host can cause connectivity issues. A common reason for these issues is that Docker sets the global FORWARD policy to `drop`, which prevents Incus from forwarding traffic and thus causes the instances to lose network connectivity. See [Docker on a router](https://docs.docker.com/engine/network/packet-filtering-firewalls/#docker-on-a-router) for detailed information. There are different ways of working around this problem: Uninstall Docker : The easiest way to prevent such issues is to uninstall Docker from the system that runs Incus and restart the system. You can run Docker inside an Incus container or virtual machine instead. Preventing Docker from dropping traffic : Most issues with Docker networking come from it dropping all traffic that it doesn't directly manage. This behavior can be changed through the `ip-forward-no-drop` Docker configuration option in `daemon.json`. Setting the option to `true` may be sufficient to allow both Incus and Docker to operate on the same system. Enable IPv4 forwarding : If uninstalling or reconfiguring Docker is not an option, enabling IPv4 forwarding before the Docker service starts will prevent Docker from modifying the global FORWARD policy. Incus bridge networks enable this setting normally. However, if Incus starts after Docker, then Docker will already have modified the global FORWARD policy. To enable IPv4 forwarding before Docker starts, ensure that the following `sysctl` setting is enabled: net.ipv4.conf.all.forwarding=1 ```{important} You must make this setting persistent across host reboots. One way of doing this is to add a file to the `/etc/sysctl.d/` directory using the following commands: echo "net.ipv4.conf.all.forwarding=1" > /etc/sysctl.d/99-forwarding.conf systemctl restart systemd-sysctl ``` Allow egress network traffic flows : If you do not want the Docker container ports to be potentially reachable from any machine on your local network, you can apply a more complex solution provided by Docker. Use the following commands to explicitly allow egress network traffic flows from your Incus managed bridge interface: iptables -I DOCKER-USER -i -j ACCEPT iptables -I DOCKER-USER -o -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT For example, if your Incus managed bridge is called `incusbr0`, you can allow egress traffic to flow using the following commands: iptables -I DOCKER-USER -i incusbr0 -j ACCEPT iptables -I DOCKER-USER -o incusbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT Or the equivalent `nft` commands: nft insert rule ip filter DOCKER-USER iifname incusbr0 counter accept nft insert rule ip filter DOCKER-USER oifname incusbr0 ct state related,established counter accept ```{important} You must make these firewall rules persistent across host reboots. How to do this depends on your Linux distribution. ``` incus-7.0.0/doc/howto/network_bridge_resolved.md000066400000000000000000000124541517523235500220140ustar00rootroot00000000000000(network-bridge-resolved)= # How to integrate with `systemd-resolved` If the system that runs Incus uses `systemd-resolved` to perform DNS lookups, you should notify `resolved` of the domains that Incus can resolve. To do so, add the DNS servers and domains provided by an Incus network bridge to the `resolved` configuration. ```{note} The `dns.mode` option (see {ref}`network-bridge-options`) must be set to `managed` or `dynamic` if you want to use this feature. ``` (network-bridge-resolved-configure)= ## Configure resolved To add a network bridge to the `resolved` configuration, specify the DNS addresses and domains for the respective bridge. DNS address : You can use the IPv4 address, the IPv6 address or both. The address must be specified without the subnet netmask. To retrieve the IPv4 address for the bridge, use the following command: incus network get ipv4.address To retrieve the IPv6 address for the bridge, use the following command: incus network get ipv6.address DNS domain : To retrieve the DNS domain name for the bridge, use the following command: incus network get dns.domain If this option is not set, the default domain name is `incus`. Use the following commands to configure `resolved`: resolvectl dns resolvectl domain ~ ```{note} When configuring `resolved` with the DNS domain name, you should prefix the name with `~`. The `~` tells `resolved` to use the respective name server to look up only this domain. Depending on which shell you use, you might need to include the DNS domain in quotes to prevent the `~` from being expanded. ``` DNSSEC and DNS over TLS : The `incus` DNS server does not support DNSSEC or DNS over TLS. Depending on your resolved configuration the configuration will fail as the server does not support DNSSEC or DNS over TLS. To disable both only for the bridge, use the following commands: resolvectl dnssec off resolvectl dnsovertls off For example: resolvectl dns incusbr0 192.0.2.10 resolvectl domain incusbr0 '~incus' resolvectl dnssec incusbr0 off resolvectl dnsovertls incusbr0 off ```{note} Alternatively, you can use the `systemd-resolve` command. This command has been deprecated in newer releases of `systemd`, but it is still provided for backwards compatibility. systemd-resolve --interface --set-domain ~ --set-dns --set-dnsovertls=off --set-dnssec=off ``` The `resolved` configuration persists as long as the bridge exists. You must repeat the commands after each reboot and after Incus is restarted, or make it persistent as described below. ## Make the `resolved` configuration persistent You can automate the `systemd-resolved` DNS configuration, so that it is applied on system start and takes effect when Incus creates the network interface. To do so, create a `systemd` unit file named `/etc/systemd/system/incus-dns-.service` with the following content: ``` [Unit] Description=Incus per-link DNS configuration for BindsTo=sys-subsystem-net-devices-.device After=sys-subsystem-net-devices-.device [Service] Type=oneshot ExecStart=/usr/bin/resolvectl dns ExecStart=/usr/bin/resolvectl domain ~ ExecStart=/usr/bin/resolvectl dnssec off ExecStart=/usr/bin/resolvectl dnsovertls off ExecStopPost=/usr/bin/resolvectl revert RemainAfterExit=yes [Install] WantedBy=sys-subsystem-net-devices-.device ``` Replace `` in the file name and content with the name of your bridge (for example, `incusbr0`). Also replace `` and `` as described in {ref}`network-bridge-resolved-configure`. Then enable and start the service with the following commands: sudo systemctl daemon-reload sudo systemctl enable --now incus-dns- If the respective bridge already exists (because Incus is already running), you can use the following command to check that the new service has started: sudo systemctl status incus-dns-.service You should see output similar to the following: ```{terminal} :input: sudo systemctl status incus-dns-incusbr0.service ● incus-dns-incusbr0.service - Incus per-link DNS configuration for incusbr0 Loaded: loaded (/etc/systemd/system/incus-dns-incusbr0.service; enabled; vendor preset: enabled) Active: inactive (dead) since Mon 2021-06-14 17:03:12 BST; 1min 2s ago Process: 9433 ExecStart=/usr/bin/resolvectl dns incusbr0 n.n.n.n (code=exited, status=0/SUCCESS) Process: 9434 ExecStart=/usr/bin/resolvectl domain incusbr0 ~incus (code=exited, status=0/SUCCESS) Main PID: 9434 (code=exited, status=0/SUCCESS) ``` To check that `resolved` has applied the settings, use `resolvectl status `: ```{terminal} :input: resolvectl status incusbr0 Link 6 (incusbr0) Current Scopes: DNS DefaultRoute setting: no LLMNR setting: yes MulticastDNS setting: no DNSOverTLS setting: no DNSSEC setting: no DNSSEC supported: no Current DNS Server: n.n.n.n DNS Servers: n.n.n.n DNS Domain: ~incus ``` incus-7.0.0/doc/howto/network_configure.md000066400000000000000000000017621517523235500206360ustar00rootroot00000000000000(network-configure)= # How to configure a network To configure an existing network, use either the [`incus network set`](incus_network_set.md) and [`incus network unset`](incus_network_unset.md) commands (to configure single settings) or the `incus network edit` command (to edit the full configuration). To configure settings for specific cluster members, add the `--target` flag. For example, the following command configures a DNS server for a physical network: ```bash incus network set UPLINK dns.nameservers=8.8.8.8 ``` The available configuration options differ depending on the network type. See {ref}`network-types` for links to the configuration options for each network type. There are separate commands to configure advanced networking features. See the following documentation: - {doc}`/howto/network_acls` - {doc}`/howto/network_forwards` - {doc}`/howto/network_integrations` - {doc}`/howto/network_load_balancers` - {doc}`/howto/network_zones` - {doc}`/howto/network_ovn_peers` (OVN only) incus-7.0.0/doc/howto/network_create.md000066400000000000000000000072171517523235500201210ustar00rootroot00000000000000# How to create a network To create a managed network, use the [`incus network`](incus_network.md) command and its subcommands. Append `--help` to any command to see more information about its usage and available flags. (network-types)= ## Network types The following network types are available: ```{list-table} :header-rows: 1 * - Network type - Documentation - Configuration options * - `bridge` - {ref}`network-bridge` - {ref}`network-bridge-options` * - `ovn` - {ref}`network-ovn` - {ref}`network-ovn-options` * - `macvlan` - {ref}`network-macvlan` - {ref}`network-macvlan-options` * - `sriov` - {ref}`network-sriov` - {ref}`network-sriov-options` * - `physical` - {ref}`network-physical` - {ref}`network-physical-options` ``` ## Create a network Use the following command to create a network: ```bash incus network create --type= [configuration_options...] ``` See {ref}`network-types` for a list of available network types and links to their configuration options. If you do not specify a `--type` argument, the default type of `bridge` is used. (network-create-cluster)= ### Create a network in a cluster If you are running an Incus cluster and want to create a network, you must create the network for each cluster member separately. The reason for this is that the network configuration, for example, the name of the parent network interface, might be different between cluster members. Therefore, you must first create a pending network on each member with the `--target=` flag and the appropriate configuration for the member. Make sure to use the same network name for all members. Then create the network without specifying the `--target` flag to actually set it up. For example, the following series of commands sets up a physical network with the name `UPLINK` on three cluster members: ```{terminal} :input: incus network create UPLINK --type=physical parent=br0 --target=vm01 Network UPLINK pending on member vm01 :input: incus network create UPLINK --type=physical parent=br0 --target=vm02 Network UPLINK pending on member vm02 :input: incus network create UPLINK --type=physical parent=br0 --target=vm03 Network UPLINK pending on member vm03 :input: incus network create UPLINK --type=physical Network UPLINK created ``` Also see {ref}`cluster-config-networks`. (network-attach)= ## Attach a network to an instance After creating a managed network, you can attach it to an instance as a {ref}`NIC device `. To do so, use the following command: incus network attach [] [] The device name and the interface name are optional, but we recommend specifying at least the device name. If not specified, Incus uses the network name as the device name, which might be confusing and cause problems. For example, Incus images perform IP auto-configuration on the `eth0` interface, which does not work if the interface is called differently. For example, to attach the network `my-network` to the instance `my-instance` as `eth0` device, enter the following command: incus network attach my-network my-instance eth0 ### Attach the network as a device The [`incus network attach`](incus_network_attach.md) command is a shortcut for adding a NIC device to an instance. Alternatively, you can add a NIC device based on the network configuration in the usual way: incus config device add nic network= When using this way, you can add further configuration to the command to override the default settings for the network if needed. See {ref}`NIC device ` for all available device options. incus-7.0.0/doc/howto/network_forwards.md000066400000000000000000000144371517523235500205070ustar00rootroot00000000000000(network-forwards)= # How to configure network forwards ```{note} Network forwards are available for the {ref}`network-ovn` and the {ref}`network-bridge`. ``` Network forwards allow an external IP address (or specific ports on it) to be forwarded to an internal IP address (or specific ports on it) in the network that the forward belongs to. This feature can be useful if you have limited external IP addresses and want to share a single external address between multiple instances. There are two different ways how you can use network forwards in this case: - Forward all traffic from the external address to the internal address of one instance. This method makes it easy to move the traffic destined for the external address to another instance by simply reconfiguring the network forward. - Forward traffic from different port numbers of the external address to different instances (and optionally different ports on those instances). This method allows to "share" your external IP address and expose more than one instance at a time. ## Create a network forward Use the following command to create a network forward: ```bash incus network forward create [configuration_options...] ``` Each forward is assigned to a network. It requires a single external listen address (see {ref}`network-forwards-listen-addresses` for more information about which addresses can be forwarded, depending on the network that you are using). You can specify an optional default target address by adding the `target_address=` configuration option. If you do, any traffic that does not match a port specification is forwarded to this address. Note that this target address must be within the same subnet as the network that the forward is associated to. ### Forward properties Network forwards have the following properties: | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | `listen_address` | string | yes | IP address to listen on | | `description` | string | no | Description of the network forward | | `config` | string set | no | See table below | | `ports` | port list | no | List of {ref}`port specifications ` | ### Forward configuration Network forwards have the following configuration options: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (network-forwards-listen-addresses)= ### Requirements for listen addresses The requirements for valid listen addresses vary depending on which network type the forward is associated to. #### Bridge network - Any non-conflicting listen address is allowed. - The listen address must not overlap with a subnet that is in use with another network. #### OVN network - Allowed listen addresses must be defined in the uplink network's `ipv{n}.routes` settings or the project's {config:option}`project-restricted:restricted.networks.subnets` setting (if set). - The listen address must not overlap with a subnet that is in use with another network. (network-forwards-port-specifications)= ## Configure ports You can add port specifications to the network forward to forward traffic from specific ports on the listen address to specific ports on the target address. This target address must be different from the default target address. It must be within the same subnet as the network that the forward is associated to. Use the following command to add a port specification: ```bash incus network forward port add [] ``` You can specify a single listen port or a set of ports. If you want to forward the traffic to different ports, you have two options: - Specify a single target port to forward traffic from all listen ports to this target port. - Specify a set of target ports with the same number of ports as the listen ports to forward traffic from the first listen port to the first target port, the second listen port to the second target port, and so on. ### Port properties Network forward ports have the following properties: | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | `protocol` | string | yes | Protocol for the port(s) (`tcp` or `udp`) | | `listen_port` | string | yes | Listen port(s) (e.g. `80,90-100`) | | `target_address` | string | yes | IP address to forward to | | `target_port` | string | no | Target port(s) (e.g. `70,80-90` or `90`), same as `listen_port` if empty | | `description` | string | no | Description of port(s) | | `snat` | bool | no | Whether to place a matching SNAT rule to rewrite any new traffic coming from the target | ```{note} The `snat` property is currently only supported on managed `bridge` networks and with the `nftables` firewall driver. You also need to ensure that the target instance's port(s) aren't covered by multiple forwards to guarantee a consistent external address. ``` ## Edit a network forward Use the following command to edit a network forward: ```bash incus network forward edit ``` This command opens the network forward in YAML format for editing. You can edit both the general configuration and the port specifications. ## Delete a network forward Use the following command to delete a network forward: ```bash incus network forward delete ``` incus-7.0.0/doc/howto/network_increase_bandwidth.md000066400000000000000000000043421517523235500224670ustar00rootroot00000000000000(network-increase-bandwidth)= # How to increase the network bandwidth You can increase the network bandwidth of your Incus setup by configuring the transmit queue length (`txqueuelen`). This change makes sense in the following scenarios: - You have a NIC with 1 GbE or higher on an Incus host with a lot of local activity (instance-instance connections or host-instance connections). - You have an internet connection with 1 GbE or higher on your Incus host. The more instances you use, the more you can benefit from this tweak. ```{note} The following instructions use a `txqueuelen` value of 10000, which is commonly used with 10GbE NICs, and a `net.core.netdev_max_backlog` value of 182757. Depending on your network, you might need to use different values. In general, you should use small `txqueuelen` values with slow devices with a high latency, and high `txqueuelen` values with devices with a low latency. For the `net.core.netdev_max_backlog` value, a good guideline is to use the minimum value of the `net.ipv4.tcp_mem` configuration. ``` ## Increase the network bandwidth on the Incus host Complete the following steps to increase the network bandwidth on the Incus host: 1. Increase the transmit queue length (`txqueuelen`) of both the real NIC and the Incus NIC (for example, `incusbr0`). You can do this temporarily for testing with the following command: ifconfig txqueuelen 10000 To make the change permanent, add the following command to your interface configuration in `/etc/network/interfaces`: up ip link set eth0 txqueuelen 10000 1. Increase the receive queue length (`net.core.netdev_max_backlog`). You can do this temporarily for testing with the following command: echo 182757 > /proc/sys/net/core/netdev_max_backlog To make the change permanent, add the following configuration to `/etc/sysctl.conf`: net.core.netdev_max_backlog = 182757 ## Increase the transmit queue length on the instances You must also change the `txqueuelen` value for all Ethernet interfaces in your instances. To do this, use one of the following methods: - Apply the same changes as described above for the Incus host. - Set the `queue.tx.length` device option on the instance profile or configuration. incus-7.0.0/doc/howto/network_integrations.md000066400000000000000000000060451517523235500213620ustar00rootroot00000000000000(network-integrations)= # How to configure network integrations ```{note} Network integrations are currently only available for the {ref}`network-ovn`. ``` Network integrations can be used to connect networks on the local Incus deployment to remote networks hosted on Incus or other platforms. ## OVN interconnection At this time the only type of network integrations supported is OVN which makes use of OVN interconnection gateways to peer OVN networks together across multiple deployments. For this to work one needs a working OVN interconnection setup with: - OVN interconnection `NorthBound` and `SouthBound` databases - Two or more OVN clusters with their availability-zone names set properly (`name` property) - All OVN clusters need to have the `ovn-ic` daemon running - OVN clusters configured to advertise and learn routes from interconnection - At least one server marked as an OVN interconnection gateway More details can be found in the [upstream documentation](https://docs.ovn.org/en/latest/tutorials/ovn-interconnection.html). ## Creating a network integration A network integration can be created with `incus network integration create`. Integrations are global to the Incus deployment, they are not tied to a network or project. An example for an OVN integration would be: ``` incus network integration create ovn-region ovn incus network integration set ovn-region ovn.northbound_connection tcp:[192.0.2.12]:6645,tcp:[192.0.3.13]:6645,tcp:[192.0.3.14]:6645 incus network integration set ovn-region ovn.southbound_connection tcp:[192.0.2.12]:6646,tcp:[192.0.3.13]:6646,tcp:[192.0.3.14]:6646 ``` ## Using a network integration To make use of a network integration, one needs to peer with it. This is done through `incus network peer create`, for example: ``` incus network peer create default region ovn-region --type=remote ``` ## Integration properties Address sets have the following properties: | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | `name` | string | yes | Name of the network integration | | `description` | string | no | Description of the network integration | | `type` | string | yes | Type of network integration (currently only `ovn`) | ## Integration configuration options The following configuration options are available for all network integrations: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ### OVN configuration options Those options are specific to the OVN network integrations: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/howto/network_ipam.md000066400000000000000000000043131517523235500175760ustar00rootroot00000000000000(network-ipam)= # How to display IPAM information of an Incus deployment {abbr}`IPAM (IP Address Management)` is a method used to plan, track, and manage the information associated with a computer network's IP address space. In essence, it's a way of organizing, monitoring, and manipulating the IP space in a network. Checking the IPAM information for your Incus setup can help you debug networking issues. You can see which IP addresses are used for instances, network interfaces, forwards, and load balancers and use this information to track down where traffic is lost. To display IPAM information, enter the following command: ```bash incus network list-allocations ``` By default, this command shows the IPAM information for the `default` project. You can select a different project with the `--project` flag, or specify `--all-projects` to display the information for all projects. The resulting output will look something like this: ``` +------------------------+-----------------+----------+------+-------------------+ | USED BY | ADDRESS | TYPE | NAT | HARDWARE ADDRESS | +------------------------+-----------------+----------+------+-------------------+ | /1.0/networks/incusbr0 | 192.0.2.0/24 | network | true | | +------------------------+-----------------+----------+------+-------------------+ | /1.0/networks/incusbr0 | 2001:db8::/32 | network | true | | +------------------------+-----------------+----------+------+-------------------+ | /1.0/instances/u1 | 2001:db8::1/128 | instance | true | 10:66:6a:04:f0:95 | +------------------------+-----------------+----------+------+-------------------+ | /1.0/instances/u1 | 192.0.2.2/32 | instance | true | 10:66:6a:04:f0:95 | +------------------------+-----------------+----------+------+-------------------+ ... ``` Each listed entry lists the IP address (in CIDR notation) of one of the following Incus entities: `network`, `network-forward`, `network-load-balancer`, and `instance`. An entry contains an IP address using the CIDR notation. It also contains an Incus resource URI, the type of the entity, whether it is in NAT mode, and the hardware address (only for the `instance` entity). incus-7.0.0/doc/howto/network_load_balancers.md000066400000000000000000000156731517523235500216140ustar00rootroot00000000000000(network-load-balancers)= # How to configure network load balancers ```{note} Network load balancers are currently available for the {ref}`network-ovn`. ``` Network load balancers are similar to forwards in that they allow specific ports on an external IP address to be forwarded to specific ports on internal IP addresses in the network that the load balancer belongs to. The difference between load balancers and forwards is that load balancers can be used to share ingress traffic between multiple internal backend addresses. This feature can be useful if you have limited external IP addresses or want to share a single external address and ports over multiple instances. A load balancer is made up of: - A single external listen IP address. - One or more named backends consisting of an internal IP and optional port ranges. - One or more listen port ranges that are configured to forward to one or more named backends. ## Create a network load balancer Use the following command to create a network load balancer: ```bash incus network load-balancer create [configuration_options...] ``` Each load balancer is assigned to a network. It requires a single external listen address (see {ref}`network-load-balancers-listen-addresses` for more information about which addresses can be load-balanced). ### Load balancer properties Network load balancers have the following properties: | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | `listen_address` | string | yes | IP address to listen on | | `description` | string | no | Description of the network load balancer | | `config` | string set | no | Configuration options as key/value pairs (see below) | | `backends` | backend list | no | List of {ref}`backend specifications ` | | `ports` | port list | no | List of {ref}`port specifications ` | ### Configuration options The following configuration options are available for load balancers: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (network-load-balancers-listen-addresses)= ### Requirements for listen addresses The following requirements must be met for valid listen addresses: - Allowed listen addresses must be defined in the uplink network's `ipv{n}.routes` settings or the project's {config:option}`project-restricted:restricted.networks.subnets` setting (if set). - The listen address must not overlap with a subnet that is in use with another network or entity in that network. (network-load-balancers-backend-specifications)= ## Configure backends You can add backend specifications to the network load balancer to define target addresses (and optionally ports). The backend target address must be within the same subnet as the network that the load balancer is associated to. Use the following command to add a backend specification: ```bash incus network load-balancer backend add [] ``` The target ports are optional. If not specified, the load balancer defaults to using the frontend port as the backend target port. If you want to forward the traffic to different ports, you have two options: - Specify a single target port to forward traffic from all listen ports to this target port. - Specify a set of target ports with the same number of ports as the listen ports to forward traffic from the first listen port to the first target port, the second listen port to the second target port, and so on. ### Backend properties Network load balancer backends have the following properties: | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | `name` | string | yes | Name of the backend | | `target_address` | string | yes | IP address to forward to | | `target_port` | string | no | Target port(s) (e.g. `70,80-90` or `90`), same as the {ref}`port `'s `listen_port` if empty | | `description` | string | no | Description of backend | (network-load-balancers-port-specifications)= ## Configure ports You can add port specifications to the network load balancer to forward traffic from specific ports on the listen address to specific ports on one or more target backends. Use the following command to add a port specification: ```bash incus network load-balancer port add [,...] ``` You can specify a single listen port or a set of ports. The backend(s) specified must have target port(s) settings compatible with the port's listen port(s) setting. ### Port properties Network load balancer ports have the following properties: | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | `protocol` | string | yes | Protocol for the port(s) (`tcp` or `udp`) | | `listen_port` | string | yes | Listen port(s) (e.g. `80,90-100`) | | `target_backend` | backend list | yes | Backend name(s) to forward to | | `description` | string | no | Description of port(s) | ## Edit a network load balancer Use the following command to edit a network load balancer: ```bash incus network load-balancer edit ``` This command opens the network load balancer in YAML format for editing. You can edit both the general configuration, backend and the port specifications. ## Delete a network load balancer Use the following command to delete a network load balancer: ```bash incus network load-balancer delete ``` incus-7.0.0/doc/howto/network_ovn_peers.md000066400000000000000000000103121517523235500206440ustar00rootroot00000000000000(network-ovn-peers)= # How to create peer routing relationships By default, traffic between two OVN networks goes through the uplink network. This path is inefficient, however, because packets must leave the OVN subsystem and transit through the host's networking stack (and, potentially, an external network) and back into the OVN subsystem of the target network. Depending on how the host's networking is configured, this might limit the available bandwidth (if the OVN overlay network is on a higher bandwidth network than the host's external network). Therefore, Incus allows creating peer routing relationships between two OVN networks. Using this method, traffic between the two networks can go directly from one OVN network to the other and thus stays within the OVN subsystem, rather than transiting through the uplink network. Additionally, with network integrations, it's possible to peer two OVN networks even when they're running on different clusters. ## Create a routing relationship between networks To add a peer routing relationship between two networks, you must create a network peering for both networks. The relationship must be mutual. If you set it up on only one network, the routing relationship will be in pending state, but not active. When creating the peer routing relationship, specify a peering name that identifies the relationship for the respective network. The name can be chosen freely, and you can use it later to edit or delete the relationship. Use the following commands to create a peer routing relationship between networks in the same project: incus network peer create [configuration_options] incus network peer create [configuration_options] You can also create peer routing relationships between OVN networks in different projects: incus network peer create [configuration_options] --project= incus network peer create [configuration_options] --project= For remote peering through a network integration: incus network peer create [configuration_options] --type=remote ```{important} If the project or the network name is incorrect, the command will not return any error indicating that the respective project/network does not exist, and the routing relationship will remain in pending state. This behavior prevents users in a different project from discovering whether a project and network exists. ``` ### Peering properties Peer routing relationships have the following properties: | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | `name` | string | yes | Name of the network peering on the local network | | `description` | string | no | Description of the network peering | | `config` | string set | no | Configuration options as key/value pairs (only `user.*` custom keys supported) | | `target_integration` | string | no | Name of the integration (required at create time for remote peers) | | `target_project` | string | yes | Which project the target network exists in (required at create time for local peers) | | `target_network` | string | yes | Which network to create a peering with (required at create time for local peers) | | `status` | string | -- | Status indicating if pending or created (mutual peering exists with the target network) | ## List routing relationships To list all network peerings for a network, use the following command: incus network peer list ## Edit a routing relationship Use the following command to edit a network peering: incus network peer edit This command opens the network peering in YAML format for editing. incus-7.0.0/doc/howto/network_ovn_setup.md000066400000000000000000000214101517523235500206670ustar00rootroot00000000000000(network-ovn-setup)= # How to set up OVN with Incus See the following sections for how to set up a basic OVN network, either as a standalone network or to host a small Incus cluster. ## Set up a standalone OVN network Complete the following steps to create a standalone OVN network that is connected to a managed Incus parent bridge network (for example, `incusbr0`) for outbound connectivity. 1. Install the OVN tools on the local server: sudo apt install ovn-host ovn-central 1. Configure the OVN integration bridge: sudo ovs-vsctl set open_vswitch . \ external_ids:ovn-remote=unix:/run/ovn/ovnsb_db.sock \ external_ids:ovn-encap-type=geneve \ external_ids:ovn-encap-ip=127.0.0.1 1. Create an OVN network: incus network set ipv4.dhcp.ranges= ipv4.ovn.ranges= incus network create ovntest --type=ovn network= 1. Create an instance that uses the `ovntest` network: incus init images:debian/12 c1 incus config device override c1 eth0 network=ovntest incus start c1 1. Run [`incus list`](incus_list.md) to show the instance information: ```{terminal} :input: incus list :scroll: +------+---------+---------------------+-----------------------------------------------+-----------+-----------+ | NAME | STATE | IPV4 | IPV6 | TYPE | SNAPSHOTS | +------+---------+---------------------+-----------------------------------------------+-----------+-----------+ | c1 | RUNNING | 192.0.2.2 (eth0) | 2001:db8:cff3:5089:1266:6aff:fef0:549f (eth0) | CONTAINER | 0 | +------+---------+---------------------+-----------------------------------------------+-----------+-----------+ ``` ## Set up an Incus cluster on OVN Complete the following steps to set up an Incus cluster that uses an OVN network. Just like Incus, the distributed database for OVN must be run on a cluster that consists of an odd number of members. The following instructions use the minimum of three servers, which run both the distributed database for OVN and the OVN controller. In addition, you can add any number of servers to the Incus cluster that run only the OVN controller. 1. Complete the following steps on the three machines that you want to run the distributed database for OVN: 1. Install the OVN tools: sudo apt install ovn-central ovn-host 1. Mark the OVN services as enabled to ensure that they are started when the machine boots: systemctl enable ovn-central systemctl enable ovn-host 1. Stop OVN for now: systemctl stop ovn-central 1. Note down the IP address of the machine: ip -4 a 1. Open `/etc/default/ovn-central` for editing. 1. Paste in one of the following configurations (replace ``, `` and `` with the IP addresses of the respective machines, and `` with the IP address of the machine that you are on). - For the first machine: ``` OVN_CTL_OPTS=" \ --db-nb-addr= \ --db-nb-create-insecure-remote=yes \ --db-sb-addr= \ --db-sb-create-insecure-remote=yes \ --db-nb-cluster-local-addr= \ --db-sb-cluster-local-addr= \ --ovn-northd-nb-db=tcp::6641,tcp::6641,tcp::6641 \ --ovn-northd-sb-db=tcp::6642,tcp::6642,tcp::6642" ``` - For the second and third machine: ``` OVN_CTL_OPTS=" \ --db-nb-addr= \ --db-nb-cluster-remote-addr= \ --db-nb-create-insecure-remote=yes \ --db-sb-addr= \ --db-sb-cluster-remote-addr= \ --db-sb-create-insecure-remote=yes \ --db-nb-cluster-local-addr= \ --db-sb-cluster-local-addr= \ --ovn-northd-nb-db=tcp::6641,tcp::6641,tcp::6641 \ --ovn-northd-sb-db=tcp::6642,tcp::6642,tcp::6642" ``` 1. Start OVN: systemctl start ovn-central 1. On the remaining machines, install only `ovn-host` and make sure it is enabled: sudo apt install ovn-host systemctl enable ovn-host 1. On all machines, configure Open vSwitch (replace the variables as described above): sudo ovs-vsctl set open_vswitch . \ external_ids:ovn-remote=tcp::6642,tcp::6642,tcp::6642 \ external_ids:ovn-encap-type=geneve \ external_ids:ovn-encap-ip= 1. Create an Incus cluster by running `incus admin init` on all machines. On the first machine, create the cluster. Then join the other machines with tokens by running [`incus cluster add `](incus_cluster_add.md) on the first machine and specifying the token when initializing Incus on the other machine. 1. On the first machine, create and configure the uplink network: incus network create UPLINK --type=physical parent= --target= incus network create UPLINK --type=physical parent= --target= incus network create UPLINK --type=physical parent= --target= incus network create UPLINK --type=physical parent= --target= incus network create UPLINK --type=physical \ ipv4.ovn.ranges= \ ipv6.ovn.ranges= \ ipv4.gateway= \ ipv6.gateway= \ dns.nameservers= To determine the required values: Uplink interface : A high availability OVN cluster requires a shared layer 2 network, so that the active OVN chassis can move between cluster members (which effectively allows the OVN router's external IP to be reachable from a different host). Therefore, you must specify either an unmanaged bridge interface or an unused physical interface as the parent for the physical network that is used for OVN uplink. The instructions assume that you are using a manually created unmanaged bridge. See [How to configure network bridges](https://netplan.readthedocs.io/en/stable/examples/#how-to-configure-network-bridges) for instructions on how to set up this bridge. Gateway : Run `ip -4 route show default` and `ip -6 route show default`. Name server : Run `resolvectl`. IP ranges : Use suitable IP ranges based on the assigned IPs. 1. Still on the first machine, configure Incus to be able to communicate with the OVN DB cluster. To do so, find the value for `ovn-northd-nb-db` in `/etc/default/ovn-central` and provide it to Incus with the following command: incus config set network.ovn.northbound_connection 1. Finally, create the actual OVN network (on the first machine): incus network create my-ovn --type=ovn 1. To test the OVN network, create some instances and check the network connectivity: incus launch images:debian/12 c1 --network my-ovn incus launch images:debian/12 c2 --network my-ovn incus launch images:debian/12 c3 --network my-ovn incus launch images:debian/12 c4 --network my-ovn incus list incus exec c4 -- bash ping ping ping6 -n www.example.com ## Send OVN logs to Incus Complete the following steps to have the OVN controller send its logs to Incus. 1. Enable the syslog socket: incus config set core.syslog_socket=true 1. Open `/etc/default/ovn-host` for editing. 1. Paste the following configuration: OVN_CTL_OPTS=" \ --ovn-controller-log='-vsyslog:info --syslog-method=unix:/var/lib/incus/syslog.socket'" 1. Restart the OVN controller: systemctl restart ovn-controller.service You can now use [`incus monitor`](incus_monitor.md) to see logged network ACL traffic from the OVN controller: incus monitor --type=network-acls You can also send the logs to Loki. To do so, add the `network-acl` value to the {config:option}`server-logging:logging.NAME.types` configuration key, for example: incus config set logging.NAME.types=network-acl ```{tip} You can include logs for OVN `northd`, OVN north-bound `ovsdb-server`, and OVN south-bound `ovsdb-server` as well. To do so, edit `/etc/default/ovn-central`: OVN_CTL_OPTS=" \ --ovn-northd-log='-vsyslog:info --syslog-method=unix:/var/lib/incus/syslog.socket' \ --ovn-nb-log='-vsyslog:info --syslog-method=unix:/var/lib/incus/syslog.socket' \ --ovn-sb-log='-vsyslog:info --syslog-method=unix:/var/lib/incus/syslog.socket'" sudo systemctl restart ovn-central.service ``` incus-7.0.0/doc/howto/network_zones.md000066400000000000000000000254711517523235500200160ustar00rootroot00000000000000(network-zones)= # How to configure network zones ```{note} Network zones are available for the {ref}`network-ovn` and the {ref}`network-bridge`. ``` Network zones can be used to serve DNS records for Incus networks. You can use network zones to automatically maintain valid forward and reverse records for all your instances. This can be useful if you are operating an Incus cluster with multiple instances across many networks. Having DNS records for each instance makes it easier to access network services running on an instance. It is also important when hosting, for example, an outbound SMTP service. Without correct forward and reverse DNS entries for the instance, sent mail might be flagged as potential spam. Each network can be associated to different zones: - Forward DNS records - multiple comma-separated zones (no more than one per project) - IPv4 reverse DNS records - single zone - IPv6 reverse DNS records - single zone Incus will then automatically manage forward and reverse records for all instances, network gateways and downstream network ports and serve those zones for zone transfer to the operator’s production DNS servers. ## Project views Projects have a {config:option}`project-features:features.networks.zones` feature, which is disabled by default. This controls which project new networks zones are created in. When this feature is enabled new zones are created in the project, otherwise they are created in the default project. This allows projects that share a network in the default project (i.e those with `features.networks=false`) to have their own project level DNS zones that give a project oriented "view" of the addresses on that shared network (which only includes addresses from instances in their project). ## Generated records ### Forward records If you configure a zone with forward DNS records for `incus.example.net` for your network, it generates records that resolve the following DNS names: - For all instances in the network: `.incus.example.net` - For the network gateway: `.gw.incus.example.net` - For downstream network ports (for network zones set on an uplink network with a downstream OVN network): `-.uplink.incus.example.net` - Manual records added to the zone. You can check the records that are generated with your zone setup with the `dig` command. This assumes that {config:option}`server-core:core.dns_address` was set to `:`. (Setting that configuration option causes the backend to immediately start serving on that address.) In order for the `dig` request to be allowed for a given zone, you must set the `peers.NAME.address` configuration option for that zone. `NAME` can be anything random. The value must match the IP address where your `dig` is calling from. You must leave `peers.NAME.key` for that same random `NAME` unset. For example: `incus network zone set incus.example.net peers.whatever.address=192.0.2.1`. ```{note} It is not enough for the address to be of the same machine that `dig` is calling from; it needs to match as a string with what the DNS server in `incus` thinks is the exact remote address. `dig` binds to `0.0.0.0`, therefore the address you need is most likely the same that you provided to `core.dns_address`. ``` For example, running `dig @ -p axfr incus.example.net` might give the following output: ```{terminal} :input: dig @192.0.2.200 -p 1053 axfr incus.example.net incus.example.net. 3600 IN SOA incus.example.net. ns1.incus.example.net. 1669736788 120 60 86400 30 incus.example.net. 300 IN NS ns1.incus.example.net. inctest.gw.incus.example.net. 300 IN A 192.0.2.1 inctest.gw.incus.example.net. 300 IN AAAA fd42:4131:a53c:7211::1 default-ovntest.uplink.incus.example.net. 300 IN A 192.0.2.20 default-ovntest.uplink.incus.example.net. 300 IN AAAA fd42:4131:a53c:7211:1266:6aff:fe4e:b794 c1.incus.example.net. 300 IN AAAA fd42:4131:a53c:7211:1266:6aff:fe19:6ede c1.incus.example.net. 300 IN A 192.0.2.125 manualtest.incus.example.net. 300 IN A 8.8.8.8 incus.example.net. 3600 IN SOA incus.example.net. ns1.incus.example.net. 1669736788 120 60 86400 30 ``` ### Reverse records If you configure a zone for IPv4 reverse DNS records for `2.0.192.in-addr.arpa` for a network using `192.0.2.0/24`, it generates reverse `PTR` DNS records for addresses from all projects that are referencing that network via one of their forward zones. For example, running `dig @ -p axfr 2.0.192.in-addr.arpa` might give the following output: ```{terminal} :input: dig @192.0.2.200 -p 1053 axfr 2.0.192.in-addr.arpa 2.0.192.in-addr.arpa. 3600 IN SOA 2.0.192.in-addr.arpa. ns1.2.0.192.in-addr.arpa. 1669736828 120 60 86400 30 2.0.192.in-addr.arpa. 300 IN NS ns1.2.0.192.in-addr.arpa. 1.2.0.192.in-addr.arpa. 300 IN PTR inctest.gw.incus.example.net. 20.2.0.192.in-addr.arpa. 300 IN PTR default-ovntest.uplink.incus.example.net. 125.2.0.192.in-addr.arpa. 300 IN PTR c1.incus.example.net. 2.0.192.in-addr.arpa. 3600 IN SOA 2.0.192.in-addr.arpa. ns1.2.0.192.in-addr.arpa. 1669736828 120 60 86400 30 ``` (network-dns-server)= ## Enable the built-in DNS server To make use of network zones, you must enable the built-in DNS server. To do so, set the {config:option}`server-core:core.dns_address` configuration option to a local address on the Incus server. To avoid conflicts with an existing DNS we suggest not using the port 53. This is the address on which the DNS server will listen. Note that in an Incus cluster, the address may be different on each cluster member. ```{note} The built-in DNS server supports only zone transfers through AXFR. It cannot be directly queried for DNS records. Therefore, the built-in DNS server must be used in combination with an external DNS server (`bind9`, `nsd`, ...), which will transfer the entire zone from Incus, refresh it upon expiry and provide authoritative answers to DNS requests. Authentication for zone transfers is configured on a per-zone basis, with peers defined in the zone configuration and a combination of IP address matching and TSIG-key based authentication. ``` ## Create and configure a network zone Use the following command to create a network zone: ```bash incus network zone create [configuration_options...] ``` The following examples show how to configure a zone for forward DNS records, one for IPv4 reverse DNS records and one for IPv6 reverse DNS records, respectively: ```bash incus network zone create incus.example.net incus network zone create 2.0.192.in-addr.arpa incus network zone create 1.0.0.0.1.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa ``` ```{note} Zones must be globally unique, even across projects. If you get a creation error, it might be due to the zone already existing in another project. ``` You can either specify the configuration options when you create the network or configure them afterwards with the following command: ```bash incus network zone set = ``` Use the following command to edit a network zone in YAML format: ```bash incus network zone edit ``` ### Configuration options The following configuration options are available for network zones: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ```{note} When generating the TSIG key using `tsig-keygen`, the key name must follow the format `_.`. For example, if your zone name is `incus.example.net` and the peer name is `bind9`, then the key name must be `incus.example.net_bind9.`. If this format is not followed, zone transfer might fail. ``` ## Add a network zone to a network To add a zone to a network, set the corresponding configuration option in the network configuration: - For forward DNS records: `dns.zone.forward` - For IPv4 reverse DNS records: `dns.zone.reverse.ipv4` - For IPv6 reverse DNS records: `dns.zone.reverse.ipv6` For example: ```bash incus network set dns.zone.forward="incus.example.net" ``` Zones belong to projects and are tied to the `networks` features of projects. You can restrict projects to specific domains and sub-domains through the {config:option}`project-restricted:restricted.networks.zones` project configuration key. ## Add custom records A network zone automatically generates forward and reverse records for all instances, network gateways and downstream network ports. If required, you can manually add custom records to a zone. To do so, use the [`incus network zone record`](incus_network_zone_record.md) command. ### Create a record Use the following command to create a record: ```bash incus network zone record create ``` This command creates an empty record without entries and adds it to a network zone. #### Record properties Records have the following properties: | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | `name` | string | yes | Unique name of the record | | `description` | string | no | Description of the record | | `entries` | entry list | no | A list of DNS entries | | `config` | string set | no | Configuration options as key/value pairs (only `user.*` custom keys supported) | ### Add or remove entries To add an entry to the record, use the following command: ```bash incus network zone record entry add [--ttl ] ``` This command adds a DNS entry with the specified type and value to the record. For example, to create a dual-stack web server, add a record with two entries similar to the following: ```bash incus network zone record entry add A 1.2.3.4 incus network zone record entry add AAAA 1234::1234 ``` You can use the `--ttl` flag to set a custom time-to-live (in seconds) for the entry. Otherwise, the default of 300 seconds is used. You cannot edit an entry (except if you edit the full record with [`incus network zone record edit`](incus_network_zone_record_edit.md)), but you can delete entries with the following command: ```bash incus network zone record entry remove ``` incus-7.0.0/doc/howto/projects_confine.md000066400000000000000000000052051517523235500204320ustar00rootroot00000000000000(projects-confine)= # How to confine projects to specific users You can use projects to confine the activities of different users or clients. See {ref}`projects-confined` for more information. How to confine a project to a specific user depends on the authentication method you choose. ## Confine projects to specific TLS clients You can confine access to specific projects by restricting the TLS client certificate that is used to connect to the Incus server. See {ref}`authentication-tls-certs` for detailed information. To confine the access from the time the client certificate is added, you must either use token authentication or add the client certificate to the server directly. Use the following command to add a restricted client certificate: ````{tabs} ```{group-tab} Token authentication incus config trust add --projects --restricted ``` ```{group-tab} Add client certificate incus config trust add-certificate --projects --restricted ``` ```` The client can then add the server as a remote in the usual way ([`incus remote add `](incus_remote_add.md) or [`incus remote add `](incus_remote_add.md)) and can only access the project or projects that have been specified. To confine access for an existing certificate, use the following command: incus config trust edit Make sure that `restricted` is set to `true` and specify the projects that the certificate should give access to under `projects`. ```{note} You can specify the `--project` flag when adding a remote. This configuration preselects the specified project. However, it does not confine the client to this project. ``` ## Confine projects to specific Incus users Incus can be configured to dynamically create projects for all users in a specific user group. This is usually achieved by having some users be a member of the `incus` group but not the `incus-admin` group. Make sure that all user accounts that you want to be able to use Incus are a member of this group. Once a member of the group issues an Incus command, Incus creates a confined project for this user and switches to this project. If Incus has not been {ref}`initialized ` at this point, it is automatically initialized (with the default settings). If you want to customize the project settings, for example, to impose limits or restrictions, you can do so after the project has been created. To modify the project configuration, you must have full access to Incus, which means you must be part of the `incus-admin` group and not only the group that you configured as the Incus user group. incus-7.0.0/doc/howto/projects_create.md000066400000000000000000000052371517523235500202610ustar00rootroot00000000000000(projects-create)= # How to create and configure projects You can configure projects at creation time or later. However, note that it is not possible to modify the features that are enabled for a project when the project contains instances. ## Create a project To create a project, use the [`incus project create`](incus_project_create.md) command. For example, to create a project called `my-project`, enter the following command: incus project create my-project You can specify configuration options by using the `--config` flag. See {ref}`ref-projects` for the available configuration options. For example, to create a project called `my-project-shared-images` that isolates instances but allows access to the default project's images, enter the following command: incus project create my-project-shared-images --config features.images=false To create a project called `my-restricted-project` that blocks access to security-sensitive features (for example, container nesting) but allows backups, enter the following command: incus project create my-restricted-project --config restricted=true --config restricted.backups=allow ```{tip} When you create a project without specifying configuration options, {config:option}`project-features:features.profiles` is set to `true`, which means that profiles are isolated in the project. Consequently, the new project does not have access to the `default` profile of the `default` project and therefore misses required configuration for creating instances (like the root disk). To fix this, use the [`incus profile device add`](incus_profile_device_add.md) command to add a root disk device to the project's `default` profile. ``` (projects-configure)= ## Configure a project To configure a project, you can either set a specific configuration option or edit the full project. Some configuration options can only be set for projects that do not contain any instances. ### Set specific configuration options To set a specific configuration option, use the [`incus project set`](incus_project_set.md) command. For example, to limit the number of containers that can be created in `my-project` to five, enter the following command: incus project set my-project limits.containers=5 To unset a specific configuration option, use the [`incus project unset`](incus_project_unset.md) command. ```{note} If you unset a configuration option, it is set to its default value. This default value might differ from the initial value that is set when the project is created. ``` ### Edit the project To edit the full project configuration, use the [`incus project edit`](incus_project_edit.md) command. For example: incus project edit my-project incus-7.0.0/doc/howto/projects_work.md000066400000000000000000000101321517523235500177660ustar00rootroot00000000000000(projects-work)= # How to work with different projects If you have more projects than just the `default` project, you must make sure to use or address the correct project when working with Incus. ```{note} If you have projects that are {ref}`confined to specific users `, only users with full access to Incus can see all projects. Users without full access can only see information for the projects to which they have access. ``` ## List projects To list all projects (that you have permission to see), enter the following command: incus project list By default, the output is presented as a list: ```{terminal} :input: incus project list :scroll: +----------------------+--------+----------+-----------------+-----------------+----------+---------------+---------------------+---------+ | NAME | IMAGES | PROFILES | STORAGE VOLUMES | STORAGE BUCKETS | NETWORKS | NETWORK ZONES | DESCRIPTION | USED BY | +----------------------+--------+----------+-----------------+-----------------+----------+---------------+---------------------+---------+ | default | YES | YES | YES | YES | YES | YES | Default Incus project | 19 | +----------------------+--------+----------+-----------------+-----------------+----------+---------------+---------------------+---------+ | my-project (current) | YES | NO | NO | NO | YES | YES | | 0 | +----------------------+--------+----------+-----------------+-----------------+----------+---------------+---------------------+---------+ ``` You can request a different output format by adding the `--format` flag. See [`incus project list --help`](incus_project_list.md) for more information. ## Switch projects By default, all commands that you issue in Incus affect the project that you are currently using. To see which project you are in, use the [`incus project list`](incus_project_list.md) command. To switch to a different project, enter the following command: incus project switch ## Target a project Instead of switching to a different project, you can target a specific project when running a command. Many Incus commands support the `--project` flag to run an action in a different project. ```{note} You can target only projects that you have permission for. ``` The following sections give some typical examples where you would typically target a project instead of switching to it. ### List instances in a project To list the instances in a specific project, add the `--project` flag to the [`incus list`](incus_list.md) command. For example: incus list --project my-project ### Move an instance to another project To move an instance from one project to another, enter the following command: incus move --project --target-project You can keep the same instance name if no instance with that name exists in the target project. For example, to move the instance `my-instance` from the `default` project to `my-project` and keep the instance name, enter the following command: incus move my-instance my-instance --project default --target-project my-project ### Copy a profile to another project If you create a project with the default settings, profiles are isolated in the project ([`features.profiles`](project-features) is set to `true`). Therefore, the project does not have access to the default profile (which is part of the `default` project), and you will see an error similar to the following when trying to create an instance: ```{terminal} :input: incus launch images:debian/12 my-instance Creating my-instance Error: Failed instance creation: Failed creating instance record: Failed initializing instance: Failed getting root disk: No root device could be found ``` To fix this, you can copy the contents of the `default` project's default profile into the current project's default profile. To do so, enter the following command: incus profile show default --project default | incus profile edit default incus-7.0.0/doc/howto/server_configure.md000066400000000000000000000030371517523235500204500ustar00rootroot00000000000000(server-configure)= # How to configure the Incus server See {ref}`server` for all configuration options that are available for the Incus server. If the Incus server is part of a cluster, some of the options apply to the cluster, while others apply only to the local server, thus the cluster member. In the {ref}`server` option tables, options that apply to the cluster are marked with a `global` scope, while options that apply to the local server are marked with a `local` scope. ## Configure server options You can configure a server option with the following command: incus config set For example, to allow remote access to the Incus server on port 8443, enter the following command: incus config set core.https_address :8443 In a cluster setup, to configure a server option for a cluster member only, add the `--target` flag. For example, to configure where to store image tarballs on a specific cluster member, enter a command similar to the following: incus config set storage.images_volume my-pool/my-volume --target member02 ## Display the server configuration To display the current server configuration, enter the following command: incus config show In a cluster setup, to show the local configuration for a specific cluster member, add the `--target` flag. ## Edit the full server configuration To edit the full server configuration as a YAML file, enter the following command: incus config edit In a cluster setup, to edit the local configuration for a specific cluster member, add the `--target` flag. incus-7.0.0/doc/howto/server_expose.md000066400000000000000000000057411517523235500177760ustar00rootroot00000000000000(server-expose)= # How to expose Incus to the network By default, Incus can be used only by local users through a Unix socket and is not accessible over the network. To expose Incus to the network, you must configure it to listen to addresses other than the local Unix socket. To do so, set the {config:option}`server-core:core.https_address` server configuration option. For example, to allow access to the Incus server on port `8443`, enter the following command: incus config set core.https_address :8443 To allow access through a specific IP address, use `ip addr` to find an available address and then set it. For example: ```{terminal} :input: ip addr 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: enp5s0: mtu 1500 qdisc mq state UP group default qlen 1000 link/ether 10:66:6a:e3:f3:3f brd ff:ff:ff:ff:ff:ff inet 10.68.216.12/24 metric 100 brd 10.68.216.255 scope global dynamic enp5s0 valid_lft 3028sec preferred_lft 3028sec inet6 fd42:e819:7a51:5a7b:1266:6aff:fee3:f33f/64 scope global mngtmpaddr noprefixroute valid_lft forever preferred_lft forever inet6 fe80::1266:6aff:fee3:f33f/64 scope link valid_lft forever preferred_lft forever 3: incusbr0: mtu 1500 qdisc noqueue state DOWN group default qlen 1000 link/ether 10:66:6a:8d:f3:72 brd ff:ff:ff:ff:ff:ff inet 10.64.82.1/24 scope global incusbr0 valid_lft forever preferred_lft forever inet6 fd42:f4ab:4399:e6eb::1/64 scope global valid_lft forever preferred_lft forever :input: incus config set core.https_address 10.68.216.12 ``` All remote clients can then connect to Incus and access any image that is marked for public use. (server-authenticate)= ## Authenticate with the Incus server To be able to access the remote API, clients must authenticate with the Incus server. There are several authentication methods; see {ref}`authentication` for detailed information. The recommended method is to add the client's TLS certificate to the server's trust store through a trust token. To authenticate a client using a trust token, complete the following steps: 1. On the server, enter the following command: incus config trust add The command generates and prints a token that can be used to add the client certificate. 1. On the client, add the server with the following command: incus remote add % Include content from [../authentication.md](../authentication.md) ```{include} ../authentication.md :start-after: :end-before: ``` See {ref}`authentication` for detailed information and other authentication methods. incus-7.0.0/doc/howto/server_migrate_lxd.md000066400000000000000000000061361517523235500207710ustar00rootroot00000000000000(server-migrate-lxd)= # Migrating from LXD Incus includes a tool named `lxd-to-incus` which can be used to convert an existing LXD installation into an Incus one. For this to work properly, you should make sure to {doc}`install ` the latest stable release of Incus but not initialize it. Instead, make sure that both `incus info` and `lxc info` both work properly, then run `lxd-to-incus` to migrate your data. This process transfers the entire database and all storage from LXD to Incus, resulting in an identical setup after the migration. ```{note} Following the migration, you will need to add any user that was in the `lxd` group into the equivalent `incus-admin` group. As group membership only updates on login, users may need to close their session and re-open it for it to take effect. ``` ```{note} Additionally, this process doesn't migrate the command line tool configuration. For this you may want to transfer the content of `~/.config/lxc/` or `~/snap/lxd/common/config/` over to `~/.config/incus/`. This is mostly useful to those who interact with other remote servers or have configured custom aliases. ``` ```{terminal} :input: lxd-to-incus :user: root => Looking for source server ==> Detected: snap package => Looking for target server => Connecting to source server => Connecting to the target server => Checking server versions ==> Source version: 5.19 ==> Target version: 0.1 => Validating version compatibility => Checking that the source server isn't empty => Checking that the target server is empty => Validating source server configuration The migration is now ready to proceed. At this point, the source server and all its instances will be stopped. Instances will come back online once the migration is complete. Proceed with the migration? [default=no]: yes => Stopping the source server => Stopping the target server => Wiping the target server => Migrating the data => Migrating database => Cleaning up target paths => Starting the target server => Checking the target server Uninstall the LXD package? [default=no]: yes => Uninstalling the source server ``` ```{terminal} :input: incus list :user: root To start your first container, try: incus launch images:debian/12 Or for a virtual machine: incus launch images:debian/12 --vm +------+---------+-----------------------+------------------------------------------------+-----------+-----------+ | NAME | STATE | IPV4 | IPV6 | TYPE | SNAPSHOTS | +------+---------+-----------------------+------------------------------------------------+-----------+-----------+ | u1 | RUNNING | 10.204.220.101 (eth0) | fd42:1eb6:f1d8:4e2a:1266:6aff:fe65:940d (eth0) | CONTAINER | 0 | +------+---------+-----------------------+------------------------------------------------+-----------+-----------+ ``` The tool will also look for any configuration that is incompatible with Incus and fail before any data is migrated. ```{warning} All instances will be stopped during the migration. Once the migration process is started, it cannot easily be reversed so make sure to plan adequate downtime. ``` incus-7.0.0/doc/howto/storage_backup_volume.md000066400000000000000000000161351517523235500214640ustar00rootroot00000000000000--- myst: substitutions: type: "volume" --- (howto-storage-backup-volume)= # How to back up custom storage volumes There are different ways of backing up your custom storage volumes: - {ref}`storage-backup-snapshots` - {ref}`storage-backup-export` - {ref}`storage-copy-volume` Which method to choose depends both on your use case and on the storage driver you use. In general, snapshots are quick and space efficient (depending on the storage driver), but they are stored in the same storage pool as the {{type}} and therefore not too reliable. Export files can be stored on different disks and are therefore more reliable. They can also be used to restore the {{type}} into a different storage pool. If you have a separate, network-connected Incus server available, regularly copying {{type}}s to this other server gives high reliability as well, and this method can also be used to back up snapshots of the {{type}}. ```{note} Custom storage volumes might be attached to an instance, but they are not part of the instance. Therefore, the content of a custom storage volume is not stored when you {ref}`back up your instance `. You must back up the data of your storage volume separately. ``` (storage-backup-snapshots)= ## Use snapshots for volume backup A snapshot saves the state of the storage volume at a specific time, which makes it easy to restore the volume to a previous state. It is stored in the same storage pool as the volume itself. Most storage drivers support optimized snapshot creation (see {ref}`storage-drivers-features`). For these drivers, creating snapshots is both quick and space-efficient. For the `dir` driver, snapshot functionality is available but not very efficient. For the `lvm` driver, snapshot creation is quick, but restoring snapshots is efficient only when using thin-pool mode. ### Create a snapshot of a custom storage volume Use the following command to create a snapshot for a custom storage volume: incus storage volume snapshot create [] Add the `--reuse` flag in combination with a snapshot name to replace an existing snapshot. By default, snapshots are kept forever, unless the `snapshots.expiry` configuration option is set. To retain a specific snapshot even if a general expiry time is set, use the `--no-expiry` flag. (storage-edit-snapshots)= ### View, edit or delete snapshots Use the following command to display the snapshots for a storage volume: incus storage volume info You can view or modify snapshots in a similar way to custom storage volumes, by referring to the snapshot with `/`. To show information about a snapshot, use the following command: incus storage volume show / To edit a snapshot (for example, to add a description or change the expiry date), use the following command: incus storage volume edit / To delete a snapshot, use the following command: incus storage volume snapshot delete ### Schedule snapshots of a custom storage volume You can configure a custom storage volume to automatically create snapshots at specific times. To do so, set the `snapshots.schedule` configuration option for the storage volume (see {ref}`storage-configure-volume`). For example, to configure daily snapshots, use the following command: incus storage volume set snapshots.schedule @daily To configure taking a snapshot every day at 6 am, use the following command: incus storage volume set snapshots.schedule "0 6 * * *" When scheduling regular snapshots, consider setting an automatic expiry (`snapshots.expiry`) and a naming pattern for snapshots (`snapshots.pattern`). See the {ref}`storage-drivers` documentation for more information about those configuration options. ### Restore a snapshot of a custom storage volume You can restore a custom storage volume to the state of any of its snapshots. To do so, you must first stop all instances that use the storage volume. Then use the following command: incus storage volume snapshot restore You can also restore a snapshot into a new custom storage volume, either in the same storage pool or in a different one (even a remote storage pool). To do so, use the following command: incus storage volume copy // / (storage-backup-export)= ## Use export files for volume backup You can export the full content of your custom storage volume to a standalone file that can be stored at any location. For highest reliability, store the backup file on a different file system to ensure that it does not get lost or corrupted. ### Export a custom storage volume Use the following command to export a custom storage volume to a compressed file (for example, `/path/to/my-backup.tgz`): incus storage volume export [] If you do not specify a file path, the export file is saved as `backup.tar.gz` in the working directory. ```{warning} If the output file already exists, the command overwrites the existing file without warning. ``` You can add any of the following flags to the command: `--compression` : By default, the output file uses `gzip` compression. You can specify a different compression algorithm (for example, `bzip2`) or turn off compression with `--compression=none`. `--optimized-storage` : If your storage pool uses the `btrfs` or the `zfs` driver, add the `--optimized-storage` flag to store the data as a driver-specific binary blob instead of an archive of individual files. In this case, the export file can only be used with pools that use the same storage driver. Exporting a volume in optimized mode is usually quicker than exporting the individual files. Snapshots are exported as differences from the main volume, which decreases their size and makes them easily accessible. `--volume-only` : By default, the export file contains all snapshots of the storage volume. Add this flag to export the volume without its snapshots. ### Restore a custom storage volume from an export file You can import an export file (for example, `/path/to/my-backup.tgz`) as a new custom storage volume. To do so, use the following command: incus storage volume import [] If you do not specify a volume name, the original name of the exported storage volume is used for the new volume. If a volume with that name already (or still) exists in the specified storage pool, the command returns an error. In that case, either delete the existing volume before importing the backup or specify a different volume name for the import. incus-7.0.0/doc/howto/storage_buckets.md000066400000000000000000000120221517523235500202570ustar00rootroot00000000000000(howto-storage-buckets)= # How to manage storage buckets and keys See the following sections for instructions on how to create, configure, view and resize {ref}`storage-buckets` and how to manage storage bucket keys. ## Configure the S3 address If you want to use storage buckets on local storage (thus in a `dir`, `btrfs`, `lvm`, or `zfs` pool), you must configure the S3 address for your Incus server. This is the address that you can then use to access the buckets through the S3 protocol. To configure the S3 address, set the {config:option}`server-core:core.storage_buckets_address` server configuration option. For example: incus config set core.storage_buckets_address :8555 ## Manage storage buckets Storage buckets provide access to object storage exposed using the S3 protocol. Unlike custom storage volumes, storage buckets are not added to an instance, but applications can instead access them directly via their URL. See {ref}`storage-buckets` for detailed information. ### Create a storage bucket Use the following command to create a storage bucket in a storage pool: incus storage bucket create [configuration_options...] See the {ref}`storage-drivers` documentation for a list of available storage bucket configuration options for each driver that supports object storage. To add a storage bucket on a cluster member, add the `--target` flag: incus storage bucket create --target= [configuration_options...] ```{note} For most storage drivers, storage buckets are not replicated across the cluster and exist only on the member for which they were created. This behavior is different for `cephobject` storage pools, where buckets are available from any cluster member. ``` ### Configure storage bucket settings See the {ref}`storage-drivers` documentation for the available configuration options for each storage driver that supports object storage. Use the following command to set configuration options for a storage bucket: incus storage bucket set For example, to set the quota size of a bucket, use the following command: incus storage bucket set my-pool my-bucket size 1MiB You can also edit the storage bucket configuration by using the following command: incus storage bucket edit Use the following command to delete a storage bucket and its keys: incus storage bucket delete ### View storage buckets You can display a list of all available storage buckets in a storage pool and check their configuration. To list all available storage buckets in a storage pool, use the following command: incus storage bucket list To show detailed information about a specific bucket, use the following command: incus storage bucket show ### Resize a storage bucket By default, storage buckets do not have a quota applied. To set or change a quota for a storage bucket, set its size configuration: incus storage bucket set size ```{important} - Growing a storage bucket usually works (if the storage pool has sufficient storage). - You cannot shrink a storage bucket below its current used size. ``` ## Manage storage bucket keys To access a storage bucket, applications must use a set of S3 credentials made up of an *access key* and a *secret key*. You can create multiple sets of credentials for a specific bucket. Each set of credentials is given a key name. The key name is used only for reference and does not need to be provided to the application that uses the credentials. Each set of credentials has a *role* that specifies what operations they can perform on the bucket. The roles available are: - `admin` - Full access to the bucket - `read-only` - Read-only access to the bucket (list and get files only) If the role is not specified when creating a bucket key, the role used is `read-only`. ### Create storage bucket keys Use the following command to create a set of credentials for a storage bucket: incus storage bucket key create [configuration_options...] Use the following command to create a set of credentials for a storage bucket with a specific role: incus storage bucket key create --role=admin [configuration_options...] These commands will generate and display a random set of credential keys. ### Edit or delete storage bucket keys Use the following command to edit an existing bucket key: incus storage bucket key edit Use the following command to delete an existing bucket key: incus storage bucket key delete ### View storage bucket keys Use the following command to see the keys defined for an existing bucket: incus storage bucket key list Use the following command to see a specific bucket key: incus storage bucket key show incus-7.0.0/doc/howto/storage_create_instance.md000066400000000000000000000013501517523235500217500ustar00rootroot00000000000000(howto-storage-create-instance)= # How to create an instance in a specific storage pool Instance storage volumes are created in the storage pool that is specified by the instance's root disk device. This configuration is normally provided by the profile or profiles applied to the instance. See {ref}`storage-default-pool` for detailed information. To use a different storage pool when creating or launching an instance, add the `--storage` flag. This flag overrides the root disk device from the profile. For example: incus launch --storage % Include content from [storage_move_volume.md](storage_move_volume.md) ```{include} storage_move_volume.md :start-after: (storage-move-instance)= ``` incus-7.0.0/doc/howto/storage_linstor_setup.md000066400000000000000000000415201517523235500215360ustar00rootroot00000000000000(storage-linstor-setup)= # How to set up LINSTOR with Incus Follow this guide to setup a LINSTOR cluster and configure Incus to use it as a storage provider. In this guide, we'll demonstrate the setup across three nodes (`server01`, `server02`, `server03`) running Ubuntu 24.04, all of which will run Incus instances and contribute storage to the LINSTOR cluster. Other configurations are also supported, such as having dedicated storage nodes and only consuming that storage via the network. Regardless of the underlying storage setup, all Incus nodes should run the LINSTOR satellite service to be able to mount volumes on the node. It's also worth noting that we'll be using LVM Thin as the LINSTOR storage backend, but regular LVM and ZFS are also supported. 1. Complete the following steps on the three machines to install the required LINSTOR components: 1. Add the LINBIT PPA: sudo add-apt-repository ppa:linbit/linbit-drbd9-stack sudo apt update 1. Install the required packages: sudo apt install lvm2 drbd-dkms drbd-utils linstor-satellite 1. Enable the LINSTOR satellite service to ensure it is always started with the machine: sudo systemctl enable --now linstor-satellite 1. Complete the following steps on the first machine (`server01`) to setup the LINSTOR controller and bootstrap the LINSTOR cluster: 1. Install the LINSTOR controller and client packages: sudo apt install linstor-controller linstor-client python3-setuptools 1. Enable the LINSTOR controller service to ensure it is always started with the machine: sudo systemctl enable --now linstor-controller 1. Add the nodes to the LINSTOR cluster (replace ``, `` and `` with the IP addresses of the respective machines). In this case `server01` is a combined node (controller + satellite), while the other two nodes are just satellites: linstor node create server01 --node-type combined linstor node create server02 --node-type satellite linstor node create server03 --node-type satellite 1. Verify that all nodes are online and that their node names match the node names in the Incus cluster: ```{terminal} :input: linstor node list :scroll: ╭─────────────────────────────────────────────────────────────╮ ┊ Node ┊ NodeType ┊ Addresses ┊ State ┊ ╞═════════════════════════════════════════════════════════════╡ ┊ server01 ┊ COMBINED ┊ 10.172.117.211:3366 (PLAIN) ┊ Online ┊ ┊ server02 ┊ SATELLITE ┊ 10.172.117.35:3366 (PLAIN) ┊ Online ┊ ┊ server03 ┊ SATELLITE ┊ 10.172.117.232:3366 (PLAIN) ┊ Online ┊ ╰─────────────────────────────────────────────────────────────╯ ``` 1. Verify that all nodes have the desired features available. In this case we're interested in `LVMThin` and `DRBD`, which are available: ```{terminal} :input: linstor node info :scroll: ╭───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ ┊ Node ┊ Diskless ┊ LVM ┊ LVMThin ┊ ZFS/Thin ┊ File/Thin ┊ SPDK ┊ EXOS ┊ Remote SPDK ┊ Storage Spaces ┊ Storage Spaces/Thin ┊ ╞═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╡ ┊ server01 ┊ + ┊ + ┊ + ┊ + ┊ + ┊ - ┊ - ┊ + ┊ - ┊ - ┊ ┊ server02 ┊ + ┊ + ┊ + ┊ + ┊ + ┊ - ┊ - ┊ + ┊ - ┊ - ┊ ┊ server03 ┊ + ┊ + ┊ + ┊ + ┊ + ┊ - ┊ - ┊ + ┊ - ┊ - ┊ ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭───────────────────────────────────────────────────────────────────────╮ ┊ Node ┊ DRBD ┊ LUKS ┊ NVMe ┊ Cache ┊ BCache ┊ WriteCache ┊ Storage ┊ ╞═══════════════════════════════════════════════════════════════════════╡ ┊ server01 ┊ + ┊ - ┊ - ┊ + ┊ - ┊ + ┊ + ┊ ┊ server02 ┊ + ┊ - ┊ - ┊ + ┊ - ┊ + ┊ + ┊ ┊ server03 ┊ + ┊ - ┊ - ┊ + ┊ - ┊ + ┊ + ┊ ╰───────────────────────────────────────────────────────────────────────╯ ``` 1. Create the storage pools in each satellite node that will contribute storage to the cluster. In this case all satellite nodes will contribute storage, but in a setup with dedicated storage nodes we'd only create the storage pools on those nodes. We could setup the LVM volume group manually using `vgcreate` and `pvcreate` and tell LINSTOR to use this volume group to setup its storage pool, but the `linstor physical-storage create-device-pool` automates this setup in a convenient way. We could also specify multiple devices to compose the pool, but in this case we have a single `/dev/nvme1n1` device available on each node: linstor physical-storage create-device-pool --storage-pool nvme_pool --pool-name nvme_pool lvmthin server01 /dev/nvme1n1 linstor physical-storage create-device-pool --storage-pool nvme_pool --pool-name nvme_pool lvmthin server02 /dev/nvme1n1 linstor physical-storage create-device-pool --storage-pool nvme_pool --pool-name nvme_pool lvmthin server03 /dev/nvme1n1 1. Verify that all storage pools are created and report the expected size: ```{terminal} :input: linstor storage-pool list :scroll: ╭────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ ┊ StoragePool ┊ Node ┊ Driver ┊ PoolName ┊ FreeCapacity ┊ TotalCapacity ┊ CanSnapshots ┊ State ┊ SharedName ┊ ╞════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╡ ┊ DfltDisklessStorPool ┊ server01 ┊ DISKLESS ┊ ┊ ┊ ┊ False ┊ Ok ┊ server01;DfltDisklessStorPool ┊ ┊ DfltDisklessStorPool ┊ server02 ┊ DISKLESS ┊ ┊ ┊ ┊ False ┊ Ok ┊ server02;DfltDisklessStorPool ┊ ┊ DfltDisklessStorPool ┊ server03 ┊ DISKLESS ┊ ┊ ┊ ┊ False ┊ Ok ┊ server03;DfltDisklessStorPool ┊ ┊ nvme_pool ┊ server01 ┊ LVM_THIN ┊ linstor_nvme_pool/nvme_pool ┊ 49.89 GiB ┊ 49.89 GiB ┊ True ┊ Ok ┊ server01;nvme_pool ┊ ┊ nvme_pool ┊ server02 ┊ LVM_THIN ┊ linstor_nvme_pool/nvme_pool ┊ 49.89 GiB ┊ 49.89 GiB ┊ True ┊ Ok ┊ server02;nvme_pool ┊ ┊ nvme_pool ┊ server03 ┊ LVM_THIN ┊ linstor_nvme_pool/nvme_pool ┊ 49.89 GiB ┊ 49.89 GiB ┊ True ┊ Ok ┊ server03;nvme_pool ┊ ╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` 1. Configure Incus to be able to communicate with the LINSTOR controller (replace `` with the IP address of the controller machine): incus config set storage.linstor.controller_connection=http://:3370 1. Create the storage pool on Incus. We'll specify the `linstor.resource_group.storage_pool` option to ensure that LINSTOR uses the `nvme_pool` storage pool for the volumes on this Incus storage pool. This is specially useful if you have multiple LINSTOR storage pools (e.g. one for NVMe drives and another for SATA HDDs): incus storage create remote linstor --target server01 incus storage create remote linstor --target server02 incus storage create remote linstor --target server03 incus storage create remote linstor linstor.resource_group.storage_pool=nvme_pool 1. Verify the LINSTOR resource group created by Incus: ```{terminal} :input: linstor resource-group list :scroll: ╭──────────────────────────────────────────────────────────────────────────────────────╮ ┊ ResourceGroup ┊ SelectFilter ┊ VlmNrs ┊ Description ┊ ╞══════════════════════════════════════════════════════════════════════════════════════╡ ┊ DfltRscGrp ┊ PlaceCount: 2 ┊ ┊ ┊ ╞┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄╡ ┊ remote ┊ PlaceCount: 2 ┊ ┊ Resource group managed by Incus ┊ ┊ ┊ StoragePool(s): nvme_pool ┊ ┊ ┊ ╰──────────────────────────────────────────────────────────────────────────────────────╯ ``` 1. To test the storage, create some volumes and instances: incus launch images:debian/12 c1 --storage remote incus storage volume create remote fsvol incus storage volume attach remote fsvol c1 /mnt incus launch images:debian/12 v1 --storage remote --vm -c migration.stateful=true incus storage volume create remote vol --type block size=42GiB incus storage volume attach remote vol v1 1. Verify the LINSTOR view of the resources created by Incus: ```{terminal} :input: linstor resource-definition list --show-props Aux/Incus/name Aux/Incus/type Aux/Incus/content-type :scroll: ╭─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ ┊ ResourceName ┊ Port ┊ ResourceGroup ┊ Layers ┊ State ┊ Aux/Incus/name ┊ Aux/Incus/type ┊ Aux/Incus/content-type ┊ ╞═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╡ ┊ incus-volume-1cb987892f6748299a7f894a483e4e7e ┊ 7004 ┊ remote ┊ DRBD,STORAGE ┊ ok ┊ incus-volume-v1 ┊ virtual-machines ┊ block ┊ ┊ incus-volume-5b680bf0dd6f4b39b784c1c151dd510c ┊ 7002 ┊ remote ┊ DRBD,STORAGE ┊ ok ┊ incus-volume-default_fsvol ┊ custom ┊ filesystem ┊ ┊ incus-volume-5d7ee1b9c5224f73b3dd3c3a4ff46fed ┊ 7000 ┊ remote ┊ DRBD,STORAGE ┊ ok ┊ incus-volume-198e0b3f6b3685418d9c21b58445686f939596b1fccd8e295191fe515d1ab32c ┊ images ┊ filesystem ┊ ┊ incus-volume-9f7ed7091da346e2b7c764348ffada54 ┊ 7001 ┊ remote ┊ DRBD,STORAGE ┊ ok ┊ incus-volume-c1 ┊ containers ┊ filesystem ┊ ┊ incus-volume-10991980d449418b9b8714b769f030d7 ┊ 7005 ┊ remote ┊ DRBD,STORAGE ┊ ok ┊ incus-volume-default_vol ┊ custom ┊ block ┊ ┊ incus-volume-af0e3529ad514b7b89c7a3a9b8b718ff ┊ 7003 ┊ remote ┊ DRBD,STORAGE ┊ ok ┊ incus-volume-dfc28af5f731668509b897ce7eb30d07c5bfe50502da4b2f19421a8a0b05137a ┊ images ┊ block ┊ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` incus-7.0.0/doc/howto/storage_move_volume.md000066400000000000000000000067741517523235500211750ustar00rootroot00000000000000(howto-storage-move-volume)= # How to move or copy storage volumes You can {ref}`copy ` or {ref}`move ` custom storage volumes from one storage pool to another, or copy or rename them within the same storage pool. To move instance storage volumes from one storage pool to another, {ref}`move the corresponding instance ` to another pool. When copying or moving a volume between storage pools that use different drivers, the volume is automatically converted. (storage-copy-volume)= ## Copy custom storage volumes Use the following command to copy a custom storage volume: incus storage volume copy / / Add the `--volume-only` flag to copy only the volume and skip any snapshots that the volume might have. If the volume already exists in the target location, use the `--refresh` flag to update the copy. Specify the same pool as the source and target pool to copy the volume within the same storage pool. You must specify different volume names for source and target in this case. When copying from one storage pool to another, you can either use the same name for both volumes or rename the new volume. (storage-move-volume)= ## Move or rename custom storage volumes Before you can move or rename a custom storage volume, all instances that use it must be {ref}`stopped `. Use the following command to move or rename a storage volume: incus storage volume move / / Specify the same pool as the source and target pool to rename the volume while keeping it in the same storage pool. You must specify different volume names for source and target in this case. When moving from one storage pool to another, you can either use the same name for both volumes or rename the new volume. ## Copy or move between cluster members For most storage drivers (except for `ceph` and `ceph-fs`), storage volumes exist only on the cluster member for which they were created. To copy or move a custom storage volume from one cluster member to another, add the `--target` and `--destination-target` flags to specify the source cluster member and the target cluster member, respectively. ## Copy or move between projects Add the `--target-project` to copy or move a custom storage volume to a different project. ## Copy or move between Incus servers You can copy or move custom storage volumes between different Incus servers by specifying the remote for each pool: incus storage volume copy :/ :/ incus storage volume move :/ :/ You can add the `--mode` flag to choose a transfer mode, depending on your network setup: `pull` (default) : Instruct the target server to pull the respective storage volume. `push` : Push the storage volume from the source server to the target server. `relay` : Pull the storage volume from the source server to the local client, and then push it to the target server. (storage-move-instance)= ## Move instance storage volumes to another pool To move an instance storage volume to another storage pool, make sure the instance is stopped. Then use the following command to move the instance to a different pool: incus move --storage incus-7.0.0/doc/howto/storage_pools.md000066400000000000000000000204361517523235500177630ustar00rootroot00000000000000(howto-storage-pools)= # How to manage storage pools See the following sections for instructions on how to create, configure, view and resize {ref}`storage-pools`. (storage-create-pool)= ## Create a storage pool Incus creates a storage pool during initialization. You can add more storage pools later, using the same driver or different drivers. To create a storage pool, use the following command: incus storage create [configuration_options...] Unless specified otherwise, Incus sets up loop-based storage with a sensible default size (20% of the free disk space, but at least 5 GiB and at most 30 GiB). See the {ref}`storage-drivers` documentation for a list of available configuration options for each driver. ### Examples See the following examples for how to create a storage pool using different storage drivers. `````{tabs} ````{group-tab} Directory Create a directory pool named `pool1`: incus storage create pool1 dir Use the existing directory `/data/incus` for `pool2`: incus storage create pool2 dir source=/data/incus ```` ````{group-tab} Btrfs Create a loop-backed pool named `pool1`: incus storage create pool1 btrfs Use the existing Btrfs file system at `/some/path` for `pool2`: incus storage create pool2 btrfs source=/some/path Create a pool named `pool3` on `/dev/sdX`: incus storage create pool3 btrfs source=/dev/sdX ```` ````{group-tab} LVM Create a loop-backed pool named `pool1` (the LVM volume group will also be called `pool1`): incus storage create pool1 lvm Use the existing LVM volume group called `my-pool` for `pool2`: incus storage create pool2 lvm source=my-pool Use the existing LVM thin pool called `my-pool` in volume group `my-vg` for `pool3`: incus storage create pool3 lvm source=my-vg lvm.thinpool_name=my-pool Create a pool named `pool4` on `/dev/sdX` (the LVM volume group will also be called `pool4`): incus storage create pool4 lvm source=/dev/sdX Create a pool named `pool5` on `/dev/sdX` with the LVM volume group name `my-pool`: incus storage create pool5 lvm source=/dev/sdX lvm.vg_name=my-pool ```` ````{group-tab} ZFS Create a loop-backed pool named `pool1` (the ZFS zpool will also be called `pool1`): incus storage create pool1 zfs Create a loop-backed pool named `pool2` with the ZFS zpool name `my-tank`: incus storage create pool2 zfs zfs.pool_name=my-tank Use the existing ZFS zpool `my-tank` for `pool3`: incus storage create pool3 zfs source=my-tank Use the existing ZFS dataset `my-tank/slice` for `pool4`: incus storage create pool4 zfs source=my-tank/slice Use the existing ZFS dataset `my-tank/zvol` for `pool5` and configure it to use ZFS block mode: incus storage create pool5 zfs source=my-tank/zvol volume.zfs.block_mode=yes Create a pool named `pool6` on `/dev/sdX` (the ZFS zpool will also be called `pool6`): incus storage create pool6 zfs source=/dev/sdX Create a pool named `pool7` on `/dev/sdX` with the ZFS zpool name `my-tank`: incus storage create pool7 zfs source=/dev/sdX zfs.pool_name=my-tank ```` ````{group-tab} Ceph RBD Create an OSD storage pool named `pool1` in the default Ceph cluster (named `ceph`): incus storage create pool1 ceph Create an OSD storage pool named `pool2` in the Ceph cluster `my-cluster`: incus storage create pool2 ceph ceph.cluster_name=my-cluster Create an OSD storage pool named `pool3` with the on-disk name `my-osd` in the default Ceph cluster: incus storage create pool3 ceph ceph.osd.pool_name=my-osd Use the existing OSD storage pool `my-already-existing-osd` for `pool4`: incus storage create pool4 ceph source=my-already-existing-osd Use the existing OSD erasure-coded pool `ecpool` and the OSD replicated pool `rpl-pool` for `pool5`: incus storage create pool5 ceph source=rpl-pool ceph.osd.data_pool_name=ecpool ```` ````{group-tab} CephFS ```{note} Each CephFS file system consists of two OSD storage pools, one for the actual data and one for the file metadata. ``` Use the existing CephFS file system `my-filesystem` for `pool1`: incus storage create pool1 cephfs source=my-filesystem Use the sub-directory `my-directory` from the `my-filesystem` file system for `pool2`: incus storage create pool2 cephfs source=my-filesystem/my-directory Create a CephFS file system `my-filesystem` with a data pool called `my-data` and a metadata pool called `my-metadata` for `pool3`: incus storage create pool3 cephfs source=my-filesystem cephfs.create_missing=true cephfs.data_pool=my-data cephfs.meta_pool=my-metadata ```` ````{group-tab} Ceph Object ```{note} When using the Ceph Object driver, you must have a running Ceph Object Gateway [`radosgw`](https://docs.ceph.com/en/latest/radosgw/) URL available beforehand. ``` Use the existing Ceph Object Gateway `https://www.example.com/radosgw` to create `pool1`: incus storage create pool1 cephobject cephobject.radosgw.endpoint=https://www.example.com/radosgw ```` ````` (storage-pools-cluster)= ### Create a storage pool in a cluster If you are running an Incus cluster and want to add a storage pool, you must create the storage pool for each cluster member separately. The reason for this is that the configuration, for example, the storage location or the size of the pool, might be different between cluster members. Therefore, you must first create a pending storage pool on each member with the `--target=` flag and the appropriate configuration for the member. Make sure to use the same storage pool name for all members. Then create the storage pool without specifying the `--target` flag to actually set it up. For example, the following series of commands sets up a storage pool with the name `my-pool` at different locations and with different sizes on three cluster members: ```{terminal} :input: incus storage create my-pool zfs source=/dev/sdX size=10GiB --target=vm01 Storage pool my-pool pending on member vm01 :input: incus storage create my-pool zfs source=/dev/sdX size=15GiB --target=vm02 Storage pool my-pool pending on member vm02 :input: incus storage create my-pool zfs source=/dev/sdY size=10GiB --target=vm03 Storage pool my-pool pending on member vm03 :input: incus storage create my-pool zfs Storage pool my-pool created ``` Also see {ref}`cluster-config-storage`. ```{note} For most storage drivers, the storage pools exist locally on each cluster member. That means that if you create a storage volume in a storage pool on one member, it will not be available on other cluster members. This behavior is different for Ceph-based storage pools (`ceph`, `cephfs` and `cephobject`) where each storage pool exists in one central location and therefore, all cluster members access the same storage pool with the same storage volumes. ``` ## Configure storage pool settings See the {ref}`storage-drivers` documentation for the available configuration options for each storage driver. General keys for a storage pool (like `source`) are top-level. Driver-specific keys are namespaced by the driver name. Use the following command to set configuration options for a storage pool: incus storage set For example, to turn off compression during storage pool migration for a `dir` storage pool, use the following command: incus storage set my-dir-pool rsync.compression false You can also edit the storage pool configuration by using the following command: incus storage edit ## View storage pools You can display a list of all available storage pools and check their configuration. Use the following command to list all available storage pools: incus storage list The resulting table contains the storage pool that you created during initialization (usually called `default` or `local`) and any storage pools that you added. To show detailed information about a specific pool, use the following command: incus storage show To see usage information for a specific pool, run the following command: incus storage info (storage-resize-pool)= ## Resize a storage pool If you need more storage, you can increase the size of your storage pool by changing the `size` configuration key: incus storage set size= This will only work for loop-backed storage pools that are managed by Incus. You can only grow the pool (increase its size), not shrink it. incus-7.0.0/doc/howto/storage_volumes.md000066400000000000000000000236051517523235500203220ustar00rootroot00000000000000(howto-storage-volumes)= # How to manage storage volumes See the following sections for instructions on how to create, configure, view and resize {ref}`storage-volumes`. ## Create a custom storage volume When you create an instance, Incus automatically creates a storage volume that is used as the root disk for the instance. You can add custom storage volumes to your instances. Such custom storage volumes are independent of the instance, which means that they can be backed up separately and are retained until you delete them. Custom storage volumes with content type `filesystem` can also be shared between different instances. See {ref}`storage-volumes` for detailed information. ### Create the volume Use the following command to create a custom storage volume of type `block` or `filesystem` in a storage pool: incus storage volume create [configuration_options...] See the {ref}`storage-drivers` documentation for a list of available storage volume configuration options for each driver. By default, custom storage volumes use the `filesystem` {ref}`content type `. To create a custom storage volume with the content type `block`, add the `--type` flag: incus storage volume create --type=block [configuration_options...] To add a custom storage volume on a cluster member, add the `--target` flag: incus storage volume create --target= [configuration_options...] ```{note} For most storage drivers, custom storage volumes are not replicated across the cluster and exist only on the member for which they were created. This behavior is different for Ceph-based storage pools (`ceph` and `cephfs`), clustered LVM (`lvmcluster`), LINSTOR (`linstor`) and TrueNAS (`truenas`), where volumes are available from any cluster member. ``` To create a custom storage volume of type `iso`, use the `import` command instead of the `create` command: incus storage volume import --type=iso (storage-attach-volume)= ### Attach the volume to an instance After creating a custom storage volume, you can add it to one or more instances as a {ref}`disk device `. The following restrictions apply: - Custom storage volumes of {ref}`content type ` `block` or `iso` cannot be attached to containers, but only to virtual machines. - To avoid data corruption, storage volumes of {ref}`content type ` `block` should never be attached to more than one virtual machine at a time. - Storage volumes of {ref}`content type ` `iso` are always read-only, and can therefore be attached to more than one virtual machine at a time without corrupting data. - File system storage volumes can't be attached to virtual machines while they're running. For custom storage volumes with the content type `filesystem`, use the following command, where `` is the path for accessing the storage volume inside the instance (for example, `/data`): incus storage volume attach Custom storage volumes with the content type `block` do not take a location: incus storage volume attach By default, the custom storage volume is added to the instance with the volume name as the {ref}`device ` name. If you want to use a different device name, you can add it to the command: incus storage volume attach incus storage volume attach #### Attach the volume as a device The [`incus storage volume attach`](incus_storage_volume_attach.md) command is a shortcut for adding a disk device to an instance. Alternatively, you can add a disk device for the storage volume in the usual way: incus config device add disk pool= source= [path=] When using this way, you can add further configuration to the command if needed. See {ref}`disk device ` for all available device options. (storage-configure-IO)= #### Configure I/O limits When you attach a storage volume to an instance as a {ref}`disk device `, you can configure I/O limits for it. To do so, set the `limits.read`, `limits.write` or `limits.max` properties to the corresponding limits. See the {ref}`devices-disk` reference for more information. The limits are applied through the Linux `blkio` cgroup controller, which makes it possible to restrict I/O at the disk level (but nothing finer grained than that). ```{note} Because the limits apply to a whole physical disk rather than a partition or path, the following restrictions apply: - Limits will not apply to file systems that are backed by virtual devices (for example, device mapper). - If a file system is backed by multiple block devices, each device will get the same limit. - If two disk devices that are backed by the same disk are attached to the same instance, the limits of the two devices will be averaged. ``` All I/O limits only apply to actual block device access. Therefore, consider the file system's own overhead when setting limits. Access to cached data is not affected by the limit. (storage-volume-special)= ### Use the volume for backups or images Instead of attaching a custom volume to an instance as a disk device, you can also use it as a special kind of volume to store {ref}`backups ` or {ref}`images `. To do so, you must set the corresponding {ref}`server configuration `: - To use a custom volume to store the backup tarballs: incus config set storage.backups_volume / - To use a custom volume to store the image tarballs: incus config set storage.images_volume / (storage-configure-volume)= ## Configure storage volume settings See the {ref}`storage-drivers` documentation for the available configuration options for each storage driver. Use the following command to set configuration options for a storage volume: incus storage volume set [/] The default {ref}`storage volume type ` is `custom`, so you can leave out the `/` when configuring a custom storage volume. For example, to set the size of your custom storage volume `my-volume` to 1 GiB, use the following command: incus storage volume set my-pool my-volume size=1GiB To set the snapshot expiry time for your virtual machine `my-vm` to one month, use the following command: incus storage volume set my-pool virtual-machine/my-vm snapshots.expiry 1M You can also edit the storage volume configuration by using the following command: incus storage volume edit [/] (storage-configure-vol-default)= ### Configure default values for storage volumes You can define default volume configurations for a storage pool. To do so, set a storage pool configuration with a `volume` prefix, thus `volume.=`. This value is then used for all new storage volumes in the pool, unless it is set explicitly for a volume or an instance. In general, the defaults set on a storage pool level (before the volume was created) can be overridden through the volume configuration, and the volume configuration can be overridden through the instance configuration (for storage volumes of {ref}`type ` `container` or `virtual-machine`). For example, to set a default volume size for a storage pool, use the following command: incus storage set [:] volume.size ## View storage volumes You can display a list of all available storage volumes in a storage pool and check their configuration. To list all available storage volumes in a storage pool, use the following command: incus storage volume list To display the storage volumes for all projects (not only the default project), add the `--all-projects` flag. The resulting table contains the {ref}`storage volume type ` and the {ref}`content type ` for each storage volume in the pool. ```{note} Custom storage volumes might use the same name as instance volumes (for example, you might have a container named `c1` with a container storage volume named `c1` and a custom storage volume named `c1`). Therefore, to distinguish between instance storage volumes and custom storage volumes, all instance storage volumes must be referred to as `/` (for example, `container/c1` or `virtual-machine/vm`) in commands. ``` To show detailed configuration information about a specific volume, use the following command: incus storage volume show [/] To show state information about a specific volume, use the following command: incus storage volume info [/] In both commands, the default {ref}`storage volume type ` is `custom`, so you can leave out the `/` when displaying information about a custom storage volume. ## Resize a storage volume If you need more storage in a volume, you can increase the size of your storage volume. In some cases, it is also possible to reduce the size of a storage volume. To resize a storage volume, set its size configuration: incus storage volume set size ```{important} - Growing a storage volume usually works (if the storage pool has sufficient storage). - Shrinking a storage volume is only possible for storage volumes with content type `filesystem`. It is not guaranteed to work though, because you cannot shrink storage below its current used size. - Shrinking a storage volume with content type `block` is not possible. ``` incus-7.0.0/doc/image-handling.md000066400000000000000000000067071517523235500166140ustar00rootroot00000000000000(about-images)= # About images Incus uses an image-based workflow. Each instance is based on an image, which contains a basic operating system (for example, a Linux distribution) and some Incus-related information. Images are available from remote image stores (see {ref}`image-servers` for an overview), but you can also create your own images, either based on an existing instances or a rootfs image. You can copy images from remote servers to your local image store, or copy local images to remote servers. You can also use a local image to create a remote instance. Each image is identified by a fingerprint (SHA256). To make it easier to manage images, Incus allows defining one or more aliases for each image. ## Caching When you create an instance using a remote image, Incus downloads the image and caches it locally. It is stored in the local image store with the cached flag set. The image is kept locally as a private image until either: - The image has not been used to create a new instance for the number of days set in {config:option}`server-images:images.remote_cache_expiry`. - The image's expiry date (one of the image properties; see {ref}`images-manage-edit` for information on how to change it) is reached. Incus keeps track of the image usage by updating the `last_used_at` image property every time a new instance is spawned from the image. ## Auto-update Incus can automatically keep images that come from a remote server up to date. ```{note} Only images that are requested through an alias can be updated. If you request an image through a fingerprint, you request an exact image version. ``` Whether auto-update is enabled for an image depends on how the image was downloaded: - If the image was downloaded and cached when creating an instance, it is automatically updated if {config:option}`server-images:images.auto_update_cached` was set to `true` (the default) at download time. - If the image was copied from a remote server using the [`incus image copy`](incus_image_copy.md) command, it is automatically updated only if the `--auto-update` flag was specified. You can change this behavior for an image by [editing the `auto_update` property](images-manage-edit). On startup and after every {config:option}`server-images:images.auto_update_interval` (by default, every six hours), the Incus daemon checks for more recent versions of all the images in the store that are marked to be auto-updated and have a recorded source server. When a new version of an image is found, it is downloaded into the image store. Then any aliases pointing to the old image are moved to the new one, and the old image is removed from the store. To not delay instance creation, Incus does not check if a new version is available when creating an instance from a cached image. This means that the instance might use an older version of an image for the new instance until the image is updated at the next update interval. ## Special image properties Image properties that begin with the prefix `requirements` (for example, `requirements.XYZ`) are used by Incus to determine the compatibility of the host system and the instance that is created based on the image. If these are incompatible, Incus does not start the instance. The following requirements are supported: % Include content from [config_options.txt](config_options.txt) ```{include} config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/images.md000066400000000000000000000004671517523235500152120ustar00rootroot00000000000000(images)= # Images ```{toctree} :maxdepth: 1 image-handling Use remote images Manage images Copy and import images Create images Associate profiles reference/image_format reference/image_servers ``` incus-7.0.0/doc/images/000077500000000000000000000000001517523235500146615ustar00rootroot00000000000000incus-7.0.0/doc/images/UI/000077500000000000000000000000001517523235500151765ustar00rootroot00000000000000incus-7.0.0/doc/images/UI/limits_memory_example.png000066400000000000000000002447731517523235500223310ustar00rootroot00000000000000PNG  IHDR{~-sBIT|dtEXtSoftwaregnome-screenshot>*tEXtCreation TimeMo 11 Sep 2023 15:05:00 CEST3dK IDATxpU'Քv(4EvStoH_: dtG8*d5l`ઉ*waGg4wifՑV!΅ɈթC}tB~ŀUȧst;$0 DDDDDDDDDD.?LtDDDDDDDDDƠ`DDI"""""""""EDDDDDDDD$% &HJL((""""""""")Q0QDDDDDDDDDR`DDT R]nj*F=յȄQ0q | 1@T""""" &#+0o6 3/rW"HN\Dd%"/Od~g پ[EaXf{)[쀮Fo {]:(FNƶQ_)./S̘d0Bli$}?sc|- KL4isI߫On1LP.#|_$""""7kfW?2;Ƞ R`E3yI\_`DckSv^$Ô z*D2<רB DVҲӇc q9witm$}tS^[ 6y)|)gmXI$DJpY'߮]ľR|]&5[VMR6𾟒9%=s-39t $Xk9ٞӳ)xOڨwf2ɶrw+=^dcntMymvDwkYA1 MU}ٸWUјjߖuJ-?S&1ɜ Vx M*m!""""2k80A`2Jb]h^\=؟ :y%|?DmûIJJ4pY`[ d Bo|G+58aNbBtzq pkbܶ1 e%4X|T׫\%b>8yo v rS}uojڮ6o Qە\a^j Ni b r,}%Շ U0|Kr<@sO3a5Qb5i&_iY͚e x`9zOm+46i "lZHQZ;iNj(c'7J*Vaz?DE߽}p;0Eﯧ~75߇w}Z/Di0}إoSLvrE^/XÕ>ag93M *hzdՄu#9{E1uS|TVMvt u[abD6oh3b]$_ʨyDt-_7XRF2 r-fuдz %eq`OKyM^|:hzp_o&O?]Oj˪;D>:mP ?pIW/VH|_~ SR[LcA++~2%AT3WoJ-DDDDDd\+_ -ƅq^0V Ӣja&d=$U3Nô0 8am2lD6VQncRa[{ddÔgԜSn|}CO9 2z¨^d2lIsT dLyFMǠWbXm>eJ)aRO\Vdl ӊ(ơGWb؆8U~dUQb1%p5  20Y@x(\l"t(0hX\KkG`cF!޻o|z 6$ r(se4'%Nb6"ō'BGCŎiˇoAցim=O&Umi:Ё}u! p/Óphf|9tqSu{4X#98"DWW[pǕ-1-(4&Z)9ٸ<{^L'ƫS6,rӁMm(oZဏi|JD,Wp yC[ D4\a8ǴSjyl.D'>+ZvJםvOMMGͭ& ]C9́b4|j^k]A[$7)8zm q\` t %S m!=TO\/ޛ\KlAݱ>-:^P / ا7Btsaq?& (\&$HchbK&Žz}H SJmH_݀ә35kY9y2%H~8]c#jYsrv&Hx#[\w_EEާvsd{|8ڪ)+ɠؠ5 4SP9Yv2g0>B*嘖k@:Sﯤk]\S6'٣Lt3-^j^~Gv5Hxo()W)-A֌A"K09{FvG6{"ULpn›8*6$A'ΑxA\:NWkt}M+h'$l.yǵ &9q:M$> 10k]RMGl#/(;w~iN*ѧf$@ /oSpd|z^\_ |fY">m}AON.NJse"Ft0H`ؗS'r j81=ABG' h4mj(D,uŮF|KJi2hz2^JLCW59N}?BTߟMc`r}CK9r-D̡`w^4rK#컊r"ԯP~0gn,[rt'8v)6qJikUwMH Qu[w5Yq3ֱi>eû=VDRb46w^Uĥow`#J4n99ǁÖZ0AvAW ͭJK!͎}tE/,9)xA$ ċw!iA "ZUwVJŝVle9p̂Q:8 k4;DN^~jk h"(Z9Eþ8R47ye"v:rE#GhynƷ,|'.KtchZ*TzedsQ:tL&gɁgm -V$"pDz7tgW{[ĹЃÿ7GnG;Pfilccw8qQH XY\A[mdbg:f[Ei>=ĩF7Н_Iղ}Mx,&㰟~N)Y9/4QIO`RޅwȁzF9l ԋ;8r=l(~m%E%Gߨqw »y(Zr 's,ұOrABvG( v$H0 L6MO{*{-= dZPg.D>ĺ)HmC]Խ)ݞڴ4ޥ.L_FިN]Z%=_W}4vENKU?7&bg( 5+h b ׇ/wMo q\I?i.6@dY)wSp)޻dvy?fshS1tuGMc!LKl{MExP _v:etJq?_p(8*5,|I!K̵A( xXY{K%54^UUfJLyTQGr ]<\ #-A--3P`1D6pDSаXJBd59LVk1W%4SX1j25yfcש ӻuNQTqbnFŊ{ip-G1f]|hyjtm`5B/?f >#b5J~Lj[fa2EGSt㮙%Wgnr=7DDF8UD!6Ͻ|r諵|w P4} 3qO;el ȍ'FN8e`m4XCdI >EDDD`|g$> @]q0MK*V(QDD뺉?NǙ.XOpc2""""r4gIȍADI"""""""""EDDDDDDDD$% &HJL((""""""""")Q0QDDDDDDDDDR`DDI"""""""""EDDDDDDDD$% &HJ&͛4iDAC Ø"uB#EDDDDDDDD$% &HJFLs?7Ђh"""""""""Ms((""""""""")Q0QDDDDDDDDDR`DDI"""""""""EDDDDDDDD$% &HJL((""""""""")Q0QDDDDDDDDDR`DDI"""""""""EDDDDDDDD$% &HJL((""""""""")Q0QDDDDDDDDDR`DDI"""""""""EDDDDDDDD$% &HJL((""""""""")Q0QDDDDDDDDDR`DDIߨ',Wh-e%/[ƲtsgK=Llœ[_LOpn""""""""raٖ?R]7Luq<.;;–U?sb~G}'?oT""""""""i]oWs!Ǜ;GI~!O瀙Wpt\7ǿNG&Wu^__͂;2֮=cqc&gß~ey/;dOs7sϝ7-N83g $]ٺ(\'n`ı}EK8/om+X;7y@`/x?h>Dž܁dgC9qQ>@t~qxO:[K}9~*=otڲ_nK9$tӱ;\AAN?EQ]!֙]orYexcZ['3cKVfœmPx5~9Yq?R@YV?|C,a[[O_=N!=3?w'tj s˃|64rY't1~۳45|^Nvnb Ys| IDATnNf{9쟒9s=,ok:npn`b,q,L]Uo4姪{&^z8nc^x܇tztwM=L})r-c!"""""""qs9vrb˛>gU_)8fg3r8C|G> }BPUw<`m_] ]?C\˼QnQtQ*fY{)x\.O|+m|\S#[)%}K/r,oTO{8Lg |keO;vu\e=x]#g.\Ke?ɲV9c^@[mdɺl>=e|(߼ 'rrȝNwZK}&wM4yzҧ>]qN|8o$Yy`XO#ǒ>3]WXt-&N:c$:{ɽ@ q'y+.UW1IE'*|$d_?w̑={uϗ>;uxY,\޿]]8ԩx4SPv3f GKe Ÿ20سo'8K8]OLΆ !ox:ĤIC̫I􆩺s懚f=T@WH=mq7H퓘mI}PE-9wÕd0Ob-f2aMbD0-噼4IzDOvs3 O!%fc!x=Ļ5Q}siQ>LwML<闂_{<́CaS{AjtCO现r'f|d-?Es5@7%Hgw;F{zU <, :;î~#aן\z>YKxV. ^-ԟqSz^_0o [)#sMTRX2x;,Qowtp=<>I;?yE CђD^j z[- i?[n!gRP\Bl^RB ? V\JLr ؜x{sFEK@8!~J7.sE *܋Q;~亂lsSO6X39jdzXG ɑ@Ohu#1;sɚz d1BUmm?26G:v88~]3gB痟n ,gwwl~|9,f:؆2I =)h3il(=frWs| G 8ćo9-u8@R@τ'3ɿ񃁧?|_CL!bl৾ՊRJ)q x0@þp\=t:Y6YE^RyXE]_^H$~FIԛB7^?כfF`+7K#.0HSnz]NJ$&GwTS:vuU9yw/bj$akK{M̗t6V@>J 6(Mgd@/ixw4P5#@cDF]+A1;nM{c6|gNs ہu̷Pbn* +Z#b|K 52rbtiv죌LΰcZ8V׳Qh>ss!+Ύl?rk}Kr=74M*\W;W|%3MT(ٱ6XfǞ EBs=#7H;kp#Mg>2"?FqrϾ-!M-NYh`O:wxqTذL=0v_1EDFrSMspӇ{zEOZI<=/{p/vcJ:=$3Yf9lEQ9d D3 X嶓r-v+ !U' {07- =G?E$?o~u{NB^y Wp#z'ŕT",乗-Ф嗳gp Spos޽曖5kDOGTb=E ,k\l~P^k^*5Hf>wuVj4YXw;ʷ4'eށa~ҥe3#8V68)YzT/#QYɘfpH/8 DmDeųOUٔrLTgޯah%_\ I`F Tm"aiNwiiV9\L2"!(^}R|K)olcmMT{kZZCp@4cd|ӲB[=txS;% /.] fe3(EDFr}Bmfm2dd"ӽTc]{}[b=9roŵÎ!L$(?\̆^fo<ލucd`5HC.\ ]7RNՊuK2ftk P' k̺,Eٖxqi|׊c_ӆ%6-jUo8}# o(%guZua 7Pk^=??x{y0""t巩&!,wV:~[:hÝki 0{-sX\.19?9|rsy-x-ޑ)a<|2wwiKvm}h0Β;xr?d-}gY}Sn'F4'Sjem -->Kj.RX!i&_va÷ÇgK T-Y3̅S!_'h)b֢Q'p<\OAk\4;h^ .*a%^PEc ʗ:tȯh6-fu3;+>Bٜ {$Ѵ7hd7 To$݃ ZWP͚|TFIs=54~\_Aݖ A.3SQD#Uf̷;ɛ3t>YKYM5G/Sқ ~ࠥSi|ZCWƺa;752_H4ǛVRJ5c`uZZCˎq$[nٌjwxvVP`4(Ǔ5?P+5/jj|=JbǕcU,uTˡ:E5f=C69<§;Gy:)E4r_E~)? i3fkF0Yq^ZÑ eEReT 9jf}=18\^ka\PXצ2<.q!k(i@òQ[F~ ^~} 2Oi&2V8ZEA,6x6wؒ ڞ+E;҂OEDs&aLt!կHs ֩d<;o8 &HJD@DDDDDDDDDn &HJL((""""""""")Q0QDDDDDDDDDR`DDI"""""""""EDDDDDDDD$% &HJL((""""""""")Q0QDDDDDDDDDR`DDI""""""""".ȷDADDDDDDdqs9_s7OtAEDDDDDDD(xmIDQDDDDDDDDLR]e?=w""""""""")i#8F&|K#""""""""r9OI=\sN.u/ZgFپ};[n%##fŊ׬L=ő?]sٻ;k6[@t~qxO:[K}C΂e#;uxpS{Ewr6μ\/> +;G.1O</¸ş̜9e˖v-\ rjZZג{eHʞ.6l~m8&s0/&}nAO~2Y͙e*ɜCG~ [LjȜL,{~t_y-_3/W[IKǥ3ɜC4r[ߡzNMz૚G慓˽<,;2/=so!rJbQ__?j@_q?5 $_Mv w`vq_Ooo9K ;yk[D]O/[u06D6X3\bto8.}o, Vf]:<6r ]ZO p|ZMzu|8RAe^UL~_]Y8.݃lϒ9-裏RRRB4!$ XkV葷9{oIu_3M%:{-Rmt<83|o9;=qμ٤Igcn 8D1T)ZX4S8y(""""""""-L裏mɑߝd^S{zq#'Yph.f3cpv=۽wrw<~h`a$7y :8L|s<ӭSB p.bXhsQ=g\9|kDH'MDCCׯ>կۯ} ۼ}?yyosTbzb ǵr]k;.irEy]>y7ٻw/W_}5?|'@iz 7/()K#z39|8 FLZn/>0'޽6WVH;X 'vC"Mi$ޝDTđb176c?ٌ  ѷF`JI5.J'n$wfa[YHc!V,"r+\0S2O/E`|v|oA: $,Č䋗/&=N IAلL訩;uB7 ?yXީ?sk#+ %M gse|^G38FsS.d/ƝH$}w !g*pp<"1I^#ÚO sPK\d?70&|}"s4۬X"Bbύ^U3;xVvU;Hvbx-6ͩ(=˽kJ!C$n%gYDDDD~3QD'wHZD҄H6ZDJ3H_3]۪lF ־|q|G_Z{S22׊Ia-kz'1fb,.;N-{u !D w] 9r}cm]6&EM%s/{Y2PZI}OdsJܬrU/jy62\~T:[̘5q?#X\B{1BԞO?˅>3f±0` 2v0""""rP2QD T0x6OG$CH\LFx2Y)LM,é&Kɟ_;$,[cQ*ƒ IJ@g1ӊpO"T2N"}l>G4^ԙث$fe3oDMIDLIg޳Ŀ1#n&@d]w[,$?UH  \tf@xunHR֯%..dΈ0??g5 \Xd* ^zD?9\D;.&n8\ XlW]uӦMϯCː9{E&p EiL1Me:D\!n3WEdluC8RHIeYd.xn߱WHYdluc1ؠr;8v:Fn'Wչds$ODHyb Ec44D`K#Grt""r9RDm'Ù=?$;f*[K(opȧ.B>)H)d/#]5ǶXjr!Sc5C,3sػO(f$cj4| .yp-P$7'kތr pLZO$mn"|qWHǎmڴ 0Ӯ%ED2a #`g ADRF".&aU=IXB5ޡLZk'IC/H$EC"L-iث;=)({"ƓȚV'bp̽È5ᡘal:'׀s}aC51a$>{f04 cx[40/`-f|; D&z[6&j6)($>ș֗ gňG2{-M7؟&4|$#a&\\dM'><0IkeiqDbII ?<^^^߿˔L9^;y{|bE)d$$O*CÑdb,&-n?dmo!|Bs>iTrB7w10H)$=>ٍw$Ƶ䯷6`JC $?171jCHyF;y9`0I]Yu{8 'b& ;׌Q !enM. !!$&^v瑶M\Jiʟfp2s0?O)5Z0cFrsSP~/c8/TF^DN?6zv0`@k)""1/i DD.wU0ݞyϏlpDDD/m6fϞM=سg ,vG8 sx< g&>>^D 4JDDDDDgmƶmС +4#%ED.TzVv"""r ۹*H:t[nh""rhDK|#G ꫯ˫C!%EDDDDDDDDI4YDDDDDs-)("""""r 9ӵkFDD~4YDDDDDDDDDMk """""""""%EDDDDDDDDI4gUUeMgv넟Y(x8vmMDDD'*acF>~kX#p>ITO` čӛkV d?/T9<[ xp?f?ʰ>MݮJ&ի?ك{ 2o***;xoEDDD.}Jٱ5}u;]ų78HU3e*+pRA)!NsSU݌-a'$eeD_LU@E)_ѧ"-frJƍǝwމaϞ=wjz$:t#QXsAO>8u_-=q@sMANUULө5{pOFP)9΁\Ll!k֬[o%!!^ɓ[)"iG̝ܷ~22Z]Q ځ\HUTUJAMEDDQ2=z~IwЁu7m믿W_}E:t(qqqtرNaa!+V˅`஻b|go>X,q7֋˗i׮]ݰ)Sԫή]07aʔ)x{$""rk >I&|h w*|<(?(ݴ5vpSObo"eoL9농 Wlx_}W v#/=Fc:zZʨl@r(Wkt&A*U_3oqs]W|=~CjR%o0m=+w2s 7ߧ'ITU* `[mgweTVO@#3zϫ5krRv |{2nL⍬YC~jz θ ?/bֿj#:s]Ӊj/=Wʘ8 SDDD._mһwo֭[GTTg?}tPo!w}DGG3aYr%[n套^`0駟}vҥ n憻cǎ7 Xv-=߹ꪫ8r> L0k[o{W/oh"&Nȴiػw/6C5uIe_px >@نFLL`uT;]ac~yOP_ 򃊒-\ƚ1͏,^N5sw|mO6gx fGLeoYò'*UZ'0sn_b_e/C6:u(_0cm]:HѺ%|==tB3$}|3*1s91=q{QO=TTPe,m<>~{`C>ԧ/=~J)"""/%[#C&LU())W_KB||u>̋/H|||ݼ7p]t'{kW}%-""r:m>~t]/h#ew}G1-vBߠU?d@\;,;A̩UyxjLD-| %LÏNom|:gMt&CH!ɔ-#Q@[?:f{9Y7.~ J~wDq_]]O(BYl?y7x׉⮻" o޼Jnz&QF{K&67AAA۷'22^"16mѣDGG+ˋ?aÆsL"""rhu<=)3Ks ^d׮Cutkl9:wLžJ6胯ɏ?=cuA=ϸXo(_][X;UflbTT`:{x-}OÛ{d3gϞJz&uݷ ;t=EDD䲠db j߾=zlܸW^y{ｗ_|񬉻{ҹsgL&SmdffrQڶm{vJJJXl۷oØL&9F5w=KYlz qY1?`Aw[-BLzB M6 6 ɓIOO'11ŏ{!ի> ݺu,XرcMZ<رc ^~Fkϐ)(yn'%!.@nRNmң/P1:O^}U;/}%*'{LD"_:CA)eX9N-G9UP#:}bbxＯ\64Eu]޽{;zWW]ur5p8++O?СCG: "3#ؾe9EUTQU^Ǿu9ΗG2J9"""rPxW?R\\_ o.7x`ڷoϪU{].׿j|cǎ<;5J@(t%(""""""""?_,\.֯_OII CDQDi6\L&Q2QDDDDDDDDDDDi%EDDDDDDDDIL&Q2QDDDDDDDDDDDi%EDDDDDDDDIL&Q2QDDDDDDDDDDDi%EDDDDDDDDIL&Q2QDDDDDDDDDDDi%EDDDDDDDDIL&Q2QDDDDDDDDDDDi%EfΈ IDATDDDDDDDDILfw^|A[;iJ&^=!4P^^ζmۨnPDDDDDDDD(B֮]Ktt4_5#F`֭vRRO QDDDDDDDD(؂*++={6jv].Y9%[P6mh׮<""""""""rn.w)))L6UVqΝ;yWرc>>> 4?Lzx x㍺篼 dff[oի3p!Ν[=0awf ߿?SNkgҥl۶///_b3fڈ4oCiW]uUCֽ{w|A{9իi~L>!CORQQ믿̙3Yt)'^QF1~AAA 0^{رzٷo p0}tzŃ>,\={5s=[{ｷM;nٲ3{l%Kꫯ~8K*\_"""""""+%/QFǜ9sxi׮]^x̟pwaFQWn4$&CBBauxػw/ݻw`rupBZRSSi۶-9s&/ .w(Ӟ_~O}(((81ڵkGϯ+۲e !!!իAjm۶|l۶n.5=Ǐώ;طo_믿;`رuޝsjODDDDDDDD.J&^ ;vd֬YZ? ;NyFUg5yEn_>cnF ħ~ZWo˖-tM@M`X ޽{|O>$fɓ'7ؖHN;IMM튈H0 (""H{9xzێ;O<5\`:4/"enV=ʚ5k8z({t2x&wy'K.e̜9޶.]xb6oʕ+뮻6mwqG֡dɓywPYYI=؆GmtW_M޽ͥ6mۛ[RTTDhh(ݺuN,Hp8$2G:M1|p, =;vdԩ{yy1x`̚5kx8p`݂.""""""""rq0 sdff+֭={7ޠmt҅o###fÆ :///ڴiÐ!CߵkWKfff]IÛoIHHW^y97ȣ>Jzz:9997l0sj_DDDDDDDD.<%[ѣ iP>c Ϛ5kؼy3ׯfի7h 6o믿Nnn.VnmowީY~=#G68_~#<† ذa<<9r$vͫf{}vHMM}缎!""""""""9///O_,\_ Po[KFFѽ{]ve7,,:Я_?:w\޽{xbl6>,}h"z}G^^/< ]`0/~yҥyCDDDDDDDD. /eEDDDDDDDDD@ÜEDDDDDDDDIL&ќ""""""uCGڱ-3 n,-\Xl۶b\.F` @߾}[9Bi 3QDDϮ,ҖgS+ W[vW|7ˎન/~\kVյƼq` Gt w+Z= i8.`x+wa6;]87d:{r哝ĵ3ît"NsBDNe9>KX ίlz-Javb 3ekxRc{q EX]@ʓyhf=@CIY-J>%m n]i }8lk@dd$^^zyy1rHy())^D`LrQ2QD1n'.ځ$ Iܥ8/7/V̟MʲyL l&1f_AXCMa8\p9YFyegNڛl,(U͔O!6~YңgJY>@e%YYXb(رsU1qw@D7bsZf _uNÖ{QXfM FvyrĥoXDD,69ސFjNf„a؋ɵ;>űklo#7@A^w)`#%.;ҴeGx{-"LAAGo>}G)((Т,"?-iLY)j0 2q֫`Ţ8 !\ ]5=Oj#`nI mSJL%fL^v49; 30SG_|͡najd(f/&s(#!蔡ٻ3#֚|MH 3egd|vʹ4Fz;p I8~Ǐ틹H^þzSG\#$O^c{TL^^xyyJҦ)"gVAAr>5v}۟);CqLuM"k0p5O}>&ō sn}TVCsj[9̛xN_co#;sJy1$_\\D^>a$o=[6OݓJ޳]m۶1t3m>///ʶmۚ7]$OIX}k!aQn '1/?:id&׾̈́Js;b#6MVp$GIz2rQ(WIS.8E^ϙ^ME0fZyOi&?>4""+wQZ DHZ7[DfuSi9n,&3$g9.P;Q2`$$&\R}ᆳ4=aj AHorcn* hwsSD>mnAUp:gtO#CE.~Jy߽%}}[WmBˌגI='$9\q?ߍcYd;7ZG ;1̂oe)[ݘ>dN`j#1R]o c wYO8dymܫMs=$R C +H{| Y]6}ԴM42+XyXv0׾ 9#I=BGA9s;FjYUq,^%Nuc/%x '$50v]GLD"`v(ǣ ($cvbCV `ąc]IK2"ps]<""@<֧==x,TldL`zRvTz223zo'>ky= ўe;RG<{Myx׿xLyPc \ܷô-}Z(h?c,v<z ~7c1X}Jc@'09~:zybCxk"7 eu6o +'Ñ,O=FH"^Û|'NJ#:>q$zf<֮GXi?f6ごR_DWL.Lʞq|aox aX`Ǘg;[c,O#kU1~MvXG_;y1r\n6xP~X+5\hK:(f&Fۃ/>LLo2;u=eL@C&k_QsI^ßEi@Yǃ =.ԕi];廃'ushQc#-xڱy?EM#+s.7 27}ξ/5Dg:@ 6N"i㈟O\Gta i$Ei!$Ākg!D%88Eb;u[C`ù~F#_5}4 ޽{1&[F! fxKw:{$'D *^-y`4c.pWNEr_y89H]-ל6wcXAQxş Li&g%L'fn6#m7՝ FIkn0 M6+7ޖ3' P=>o>iY D[%?dzM."ͮ ,8uuX}|t˟I -t<}L-LN g'; Ji9z޼r7g xjh6t)Y5Ɋq:5j? !ĽOL#4R摑ʟ~?B>uQSN%%%7sTgr]O|pƯ1 {wiGTIKI 1c.%{Tv3 D ~z6;΃U6Ñc~ϵmS?x;_?>I7u٘w"LiT0[LM @qXk;$] o XBi-! 8|4Lȝ}C4Y8 o+qSI}8k8VBxb.ƙ  ހ,T@P?+q*?1w= ؼᣛOMgUG޵=ԝHg|pUȈ_de+Q#rk `!W4[< 2]SXy̌LӒec a1$ΘØI$3-\km/AaCN߄e>NX1B 6B 0իW3|pڶm{NVUUi&bbbZ(S9ptaԼ\kBe@-2 P=,27ٻ7lچ}cQȯV(#o?koQ~N(b&&[lְ8;l _ݯK]vz;{珍Lr=A~2 w7,'OrÅ ~.rjH29<|4ۼbfvO'MQoUUCp:_Uq@]z'+Ge^1K7kI-6uy@'HZ3/ONYSI%oΐqt,:=X܃~;Y掻XbWRrf-&0aB^ (EЋ pObJ0|9䂋`:At\nm비ܵad%%:"^=jȃ멦FjT[Y'GSnA'GH$jT੦Rǒ[}iaܥ]g($N&rnݣ+a^Se<¾wW$W!;w3FU!"> qTJϏU}>wTV ??2ZƁ's_G|;&va+º ^SNK]b@@c7]bٔ/ԡwlJXrf} 6EһYUV F&8@w,{Iܐ |琗TKK֟qDzUQ y{V0ؘ1gPW_7g=X3\?fWJ晌 H7cL:u%ϮFE}akrv 71M9*X tPqTWVuQ~]U9% kqROͺ{]" ͛wt-lNE~޽x||.3ѣ5a8z>DY27 O1+թD0l?>ךJ/)j$dfXN<޾5:A#NXK IDATMFoȈY#sN2nH ݜ (_w'=nXHH .I&xyySN_2z ʲ:{8s80>_G|{g0Lq7 'Rd+T@hCJ_g2c` cƤPZBi]*X.phRLaң+I—s2Yz?Hyr}އu/dh~/ZR,oِsظ7ov; a)8]QQDPƽ1gn~Sԙhz#X(oP%wTF $9L!l{[ִvҤg;ƅ6C'k+;=D\Rf 8_Op̀PR>W?'#|[II3vъ#^'.K# lzn9O,RNn"qӋui^Px)qo1y$]'w W+Jߟy)Xhآ+Lfl˟,eײ5u|[h3z\ƳiLK7,8x4Sp/%4f3ƇBQ`LT,NnH>Xƥ ̘J׋H7rױv 7vaʬ0TphH._dȐ!,]Ν;Ӿ}Bî]رcC ӳlmH+kfOY®7B mwGӯ}0\*ן0C62OyT^=~^ߖEDDDDDDD42.p$aD^c0Ӂ}a/YY 2`y&-zN`;)O㕆M{ګaA4f!\38EKCpM�>I2BFs[wzL:L~#3⏚n]&-KGذx>eӧiq~ 료4u!-k7ų o cn<Be3'w|=7.i75raY~P23qXӟyfj05+l@K =#}&8{y.VZ>T’sku܍j]l6t]A[Wm|+^jM_Iu1¦Twٽ[t5rMeY䫬x6r΍yg7罊6q8`)MXvyuPʁYWR 5ƫ`ٙ}j&sqvuĠyK"""rPֲLȸd">z/„Hz$X{?ۉE/7gOyX /g{m >dmǨDv % c'0m}qѣ>jnׅDG Z,<ıㅧ 7~ SħB7wXط6 7wZJT$gȑ9q;n|NI#Y. s4WҢ3gx,_w7 :Jf6bmJDDDB42,] z9fTcU dA LkGC^y-=(!h\-j)A&؄^5.h\*v)@jשp*YeeZ9'zR#.׋z*ݠEDDޒȑ1mRsaQ8}ݱ3a5N8Kr)PHt{X:5~SWE*"S 9݋TDDD1J&%L<)ojY>9y)^f4}lZĚ~Nn `oDVL ?K ]:ʞȼddJR {_}c[d]8nܻb־Ƃk؏YW2ڒ-1TvĮ.vfkYݜ#t!uiܥP yfbtKq=OIJdkyә9 E]1H-w#9 uFψC0pn! 6;(|N1~/+9aP$Y #B u}1eABbع!= 0ӒBwRI/ВYDf$omgn5a ʂ!W]0pF?|3c}?%dLa>Iئ(O! vnxk6xW|_DFl%$R.Zb]Bƌ!&u7ȴui;Br.@v;Wi5xA:/_wSsy ç'WyC&49z=#&~$B[كi/}etr0Z w/S{h9eەYlӫK_vKe/P$7R7FcN_zď C\,cհMOGxFkKW#?Ɓ50[] U bRށ^`.d&E5<B 3)hZ2`< `NCآUD{jK\ku:=fɢDkcV q#Yuv'/ZM?%+X\&0r:l2 YӈWkbpy؆I6qڵd[ۃ99}WxQ Ɔm\ϥMU;0:5ae BbHOU,^dbk!x_vjGIW% kq2 O?p<۝l,@ECT)ӑEr} i郓CgGB=6mÚXH=+?|!+Sv/+]n""""""dI +xe?<O/'$ƁJ/\'3cYԥ=S:<]&/tl/?1uCoڼs,sgȐ!TRlBժU (5Lwh]$pkȕD"@ _|!܅`opc(@[LL_vv9ɓ'Yx1Gulݻٷo}?R2QDDDDD䮨=I1-X|W~=WVkߥD {F}ʴ@& s㖹[j v/iЮ];:to}݄ƍٽ{79[[;%EDDDDD BϦstL$"yI-v|"Bf,17jm~eQg*+QX gF7o;?@ح[7:tsL(nܸ{ʨQ\]TnR""""""wKggt8~@t'b\Yp :9pqcV {w0Qn+IqkߛCp$">OgR֭_|o6 UTQ"r_;}4?o&Ƚ5s ;wش-^g?Sw;:*T "[`~v*VM5iyzl6nW2}ElYWt֍;r9/^̆ ػw/իWW"d>|8SL)T}){bZ˖-[3pэ=HɴEDDD>Йa/5˿,胓W\19'2ai2 rqhń*=3]r61wF9u38`b:w>u_f3_;q'w%@VV.]⡇*Q/~Wܹ3 ӓG}֭[IJJRJ4lؐYf0h ^P=Jxx8111ӧcǎ͍^{P?iӆ={2vXV^G=z`E"<m3 nr &b#|l!eŔ週F-|=M?XGZ:٢!|#GRZ5, ŞdTիW`0pBĄRSSyGl'::_~QFQZ5.]Į]?~{=^x-[;IIIdffҭ[k?̙3ŎXp!_~e~2СC 6rrrZj>}ӧOC:u pSӉorrr8{,]бlU-gfԩۗ=zн{wK|-w 3p@""";v,888|"svo>'';;^#++c{{{:t~Jpp0'N 55VZR{9997UfF?:駟o5 nӨQ#"""ؾ};֭?7ߤYf7u=wPʕ~ łC܈a.\(x/'஗֭C%..~u!KҥK\p!#5k\=ɓDIrJjj*^^^7_BBBB֭gfl޼{fb)j]FFqqqT^-Z`ggGxxxmmۖ8֩W*щ9sקy[{ҡC[Ά rGdqF~z6lH5T̼n%鿫nݚ]vTsʗ/O֭IKK#==版/O6mAs)΅ ?8s dѢE$''3n8X"vvvDDDǟ9q 4ӓcǎAFF8q;vЬYtk׮ѣّđ#G믩_~ÇsqzuM;vrʴi]p۶mDGGc08y$w?WWW/_NZZ'Nraq lllHNN&22 D֭֯_p>LŊgҥOL6  7n:W{,IZmRn]|||عs'W˹ٝ/D HLLdРA,X??b^O?ӺEDDDDwfn 2 ҎާzidܴǏArr2+W'a"QDDDDDDDDJ&M۱c7oRJ2lذ,"""""R4}Rh_9;L)eʕ!^öSw;Ƚ'%%k2x`Trd$}1`VgΖ8ܺNaSsؙ\DuڍI?J!j~Kcm=՛z7x6̉D}=cO6a-_uy]∕$ j-YG4`go솧O+uno+Lk$&&ayRM&feeQ|RkODDDDAv><<5.8/Jf&%\.?6[hHCJ–9zƯƧHQ2fBBB0`999dzcw~ûʺa=?[Π܇afl+fG31՟~}£ NdTmɨEI'jW= ?ם쌋%)ϫ_q{Ã^..W%l2i 5n.@VKq."rL,#у#G*8p]HDDDDDFW?#F`C>Bps#׬lMvTlʿHC+lҎ>]h`=ܹ46\[Ү)"LV;[ܷ7m NNaHD'`ˆOK 99v !)n 8‰ӡ~y`Ҭ}e%ƖLn o~_3i3QcEڇ Ϸ*IwS;Y gFҦj_s֟w/{gJvvT%s2\vpڄ : {l;|MɶbNf۞X}hF%HR2 deeq%z]p!?3O&##H֭y1O?1o<̤N:裏׉"<szf!ܹs9~8.]Օnݺѷoko>ĉTTmzzǎ}GA޽Z|Hvxo"""""4>Xd -)?dm|¬`cc51ٞ T \-L[ګd9^`#vYiDo^ĻOf`&z☝B% \1_D~KV"Q[Wj9L 2b% =-ZEd |7qX4͵f1Ϯ}a%hL>Ø9V0`n6bqs'gL\<4:^mCgm\ 3u)~7Bq/' $ wSqۖ O^U05|K0|RTω83\ûS>{1fګySsV;o|}J#>~x;aG]E&"ݽbI `iu^x9A)pĪȍ黉2dooO^Xf 'O^ 11T /^/KHH`̙<3tڕ_VOG1fƏϡCJt?GV^~h4Rjkz{{TƎ# `޼y$$$( 1\F_/s«2v媵O`C6$]" fjb;‰/wKJŰc97sR4GL>4Wg=wpiћЮ-N)3vnu."(}G5GD!8pkGǮ&ߎ60Gofbv G9v./)Ex?$""c{N ?`ȑԫWѣGŋY`Ac֖qѣG>C^{5ڶmɓP;lms_.*-ժUcɒ%߿5k0x`Zf/"""""m꿏U+ҿ~ζd'g7EχI:|ǂkH')O\qo~ X3 l+Vxm-|]Zr׻]NߧZR)tgkEJlמvsHo•iD]~c^5*&wVNNaGyuߧ$\z^\3,;省62ώm9Hb\ iOa!^ohw&~a7>>d.Lbl:PYcw~;vX1$}8wqQ _(`5SdnRZ2tWYoinYa[mem.劚9BV+y`HVFDYxx?u\3s`17 :lkeϞ=tާ~yg,ҩS`r?ak'|ѣG3qD֬YCJJJꪫ(++bX+**+m[ !!!'8p Wfp \sMDDDDD.8pB[ſ! :Zя⼡mX^Forhz'71׵ 2B#^GXud.X͔'NK}Y/}]_ЙʼnoX z+Z+dW_TA WBQm"I;taJ"F `Эl'k" x9_Q!CxW.눭s{ XBBhJ7C),""&0 j|W:vav+VЦM={b}J8.˗_v;kEDDcǎѦ͉IHH`V(""""‹6fN1UQfN 'm_lvUf!fC5Խ6]q>b=Cx|GӋiHn?-5j Gzz:bz1Omt""gP\\?NRRфߓ?Prbcc1L#ϟjw[""""""""raPxرc}Y]ժU|eذal۶~HOOoرc\֮ìZ Mqq1m۶sߟ1cЩS֮IUUUQYYѣGO\Çk?Yr%aaag""""""""&{2m4JJJ2dw}7ڵcϞ=n|AfРA򹯿z***""""""""rPB `ƌ:t^{nݺ5؟zZ\k9ݺ^H(""""""""'0c̙s\ldffΧx3~xzQ{geƍ yꩧxi۶m{˗{n"""p8:t/2e o6;w`l߾oÇӱcG8q"ڵ=7-- c=֠LYn.-oVEEE_G̛7g᭷#s8=`r6m5j_|)saFyZu ďcvb֣0tЁ:o0 vqسg)wѾ}{,i)۷M6}""kz`<T'|-V/ rUWh\ڴG=5Ƕmrر]9Ɠ<- w2/NUE=Y=KΓɌ5rv#>,{?*EO%WYDDDD-[naϞ=ߺB #F?LbCtR,G;p#EDDD¡0zDEE1gΜoضm[>9vԉ}ݠa%&&+vUO>8p ڵ78[:+"""kC*"U,z.OJJ\2B}zl0bRI==>,V`/7c-6C zJ9+2`,2RtszH/f񦧵kӊ|W#oM Zlb;k9 cH{g ۿ7'aײ""""ZByyq\'?!::P{>S7UW]ſorrrHJJ.cʔ)L2Çf y饗jΠAXp!?=z7Ovxy)++nl6SZZ{sEEEK/=܃a|"PRr1o$nٻn ?}Sϟo[V^MxxxP[o+PVV@tt4 .$$$ٳg /ЦM^yy2d˗/'d|y I2dsΥ9s㏳pBuF߾}Imۖ~Gꫯ{nͳ(}%x+;EdݰgEdCLdY&SG9Q̢T2㉍DXpz'g/Nf Q#֛ (8.Qf{c #֛{gQ߶s5%$plTꡌzb}NlHa/>ZUr~JR?8F,ؗɰAu^M0^UrĢF|M‰K!}yQ{E+Iuo㦾^p\ϗxGDSDƎk0ڷucm 㙷Q?MI!)Y ?djw)!5s̅?捎wpQJgʨmad4)^?ֳqSM"6FXXއ/?'P[n]{]a5aS2)8QѪBĒY]W "Bɤ_~9#?CӘ~W 瑳hz6gqaAa$D_g}*ٗugr\ CKCL&EiFL0}b8cnaÌj v1%sIEqa0' ㌘Hq+zӰcl1bq,#uשx{n8nN41}'k+ "eX1L#]rad&0ag8nj5ƎL4bz$Ka|ex|m/ۻp1al9ߴvb30pLk#}hɈ4V=Pѝf؃1flh2q}㌸[?k D&ag$I4=L׸$cl0& nčkNI-v1i$t״Xc^~ƒSqm7$HaČYb'{.ά-z5QYF-GVCu1dͯ<Y,g+n놇X<b - I52*W'.FL+HK7iLu?L5?-F<0w[S[?H M/""""gD&lpbf5-+Y c(S0#?HKs}.Y6ְ7PeL욣* Ñ1YFi͇sD3Ɗgi=skJ,5f0L5IMc^xVZe 2Ȫn]2Z+c5u5OdhuXwzmpX񎫶}ˉ:q|F's Mz hsÄH|6~2J^M"V'lV9KSu[քga&,FF -c!%F$QSFaɯ&m?9l9޽Hb`J4Քy|XӐuccR Lj vS ۽t1ւemqļ_;+ è0acM[f;FFaGH s!""""g94t0n`k*^P"_ghI0Y,@)ޚEշ3)t-hWEp$g'ylivN-nƞ8WHM5p{:&kZ{,"n֜HP+sv}pV_\/pد±U\|1$kya<>=+!nK`'1&)izuOcqXB&%{kdݠ Wݑ͊|ECu PX>rXM1DŽ‚“]}<{ݜ獯ݬ~Ex&&'YϊsIiL7&(/b ^/>[pk"(/ٱP5E.[neڴi,XCvuDD"0QD r&s8;pMHĴ/RL>;S0RZ^]NQ>L'>:,+=~p,@y)kBs,‡b_Xיܭq)؃,S7}`$Ee,i~6 =hs2,n/iTUXD ˠJ?wNc*>=.[0x.j}0S]˅z'_M%AXq`G&;jB̓hB;نs8 9i$~z8gܖ&,ဏR ^8\cPGJyUYpޕ=d2 \D<#3Vbۡ-s"bg%(YV;v x)='cE?gѢEQXXHFFljEHaH|b,DGk.&^ W&ïfe`tsY+Xkfj5=#a͜s60% X၂r(Žsd|MXV=J#Il=n[}s.>xYX!-N%e2 LLy Yo`d짺Rv4ֳ <ΤQ LB%A'1bY,8J6q3}nk=sq*lŌshlUE+$eqr>jKI]O9Ջ?ϺuZj""rR("҄Ro)`":L5c"q GuaiJѕݎ=|^I.ֳPG{>aB>=:‚u!NNwY$?E}_g>?>_"kY?sV!g".g*^qEC.ƎJ5&10ѳ*0h~d:yo$2_l`\KAA!>N'd|UիZ=ƒ|>S7  ƙk21u!CwSvH~@6 X%73X Ksd} d{^ޗ'hGx'&N [#4vH+Z }9/ J=л^!c_{UzcuIڔ8LU^r #uq>UWHΛnϬ^ɤ7EL} ϺtBj۱w5zm[`ϟY塤;Km E-ҖCثJ(+bG3pWYqJQD (lwKl9$5laKA'Yz?} M; j x}@p)#c/M%kizb0!sN$3ZDr7>S 1ZyDGw+d̙tM|W˴iӆn(""!""ǩDQ#o"[3gbӈ1yq?-:ۓHKGg]\̝gvn'>Nx#ɘsE&3L^/^72o u KN!excQTe!~H\]8cq: aIMSpZx M%%rbòc=Ɠ:%?|L-&㉊Ƣ>5w4HIhĒH` fŧI~m [3pnSDτׯ'\fA,r-`G)ǒG/OoXn#Yd=a)`-,MǒqYIl{q N/?? 4᜙,ExQ {~]`d /[a"hԫT}n8ou,eߊ q Ilg',҆`5Utۥz%,yĉ5gk?'itT\m}L9 [1lj"? q,X x֮\ 0Z""r^]Ȓ.>)6^26~oxə0fo4*-qI,8YRWN;^!MpeKg\1o$7=Xf0<4&,`gm|n6<놱h'6 \2{0%3]A}\.۷o 'N QDDZDKZ %{"WXL_[FAp:ߨ9b_~%.۶mc޼y|>&M""rNh\Ҋ>* Z*U(QDOJzW_MPPPkWIDD.A EDDDDDDDD$ ,"""""r7o׿NGPDDR0QDDDDDǏ~S:SN-TTiMkW@DDDDDDDDD. EDDDDDDDD$ EDDDDDDDD$ EDDDDDDDD$ EDDDDDDDD$ ]9?c㶇Čṇѡt"G#Do&?P A=kvL<7=#ռC<:!:bED^vnIDDD伧0QDDDD!!ח/б,ck6Su5 /e˕#x6GX,^&8r W[~Ye{SފyCahK"wG b :TJh֮騤ׇEb#""BH3eܯ2B];R2-9O1s\L߾;`נ%1 mx6N| 2{B,LVm=?Nz\CN~Ct/j2Wqc;pAXww~DDDD.&HP?̀ /$pwY.o[L$t}Wi&"tks] =S~KNBji%p={vTn^˦LmCtymJʽЭ}8Z>t$""r>䇏3ylk_\S nבvmBh8}E༾zV72f>2e!!v]ob¿]]_yoW0zm]1_ 9NCq 995"ͼCнݙEDDDw@g'W1ݥlH ۙ0'ݽ ɌS|h7.׻m#$w̪f":s~t鴮{Pqaiπ֨es⩔z^`N2+=-ru/zu\7+ Bء#TBXx;*s7б| KitD zޛҿl#!su ya>BD:ػaeTvwdo91/r]$m[LnYY]9,}c?J'k4 g21:+INV&rQ\ඟo/ .e?wQm ** ؝>Fzp}Gbŕ/7 IٖI滛*ۚv'm{p}€xo}2~wjYO6sPI{,]_/ЈhF<uPzox1A W7Z K7m^&3ap]ӻ V/e黛$p%;YwԽ]%/+=*""ҎͺmcGl %,O=rZ`XV;Xg{?guZX#B跋ǟ/SrF?{G8m1S`UFNQI+yl}p~]̣ߐ.uv=79]Za"1t"?]7l g_PFMɆ ;ad.X̋x,f:uLw.eO[rr`4WlrlV~)Ž\,|^ 4޹|1 Wۀ'3L6$$^Jh98:oYL_T^3* Һ$/6cgG0>8l9)M\ד4ǝ\?] poXMOn%&O_LfP n9^ Y7+M\^ =9aPt$Uvz3(|Gr#<Gp޲!m2sh]gPtp0g_0OKGtE]9G#9T竭lve2fVrqɅI#xWIYf2(}aø%tks0. ,?a #;y?OaN8_f4aAʁ&r g3m e@h;ٔ[LA_(#p,I|e>hJcm沿&]n?[۷ٿbmnb}$h 0;;Cήb)__DDBf; )]b}9UY S7Fw/-xQv=}gl߉t̑X}[ϡ#.Qraz|_"/;\za"{ry"7@Ȩ 4WRY;iOnIM QJ%儆|#e2c$8CٳˠgRb<؈LuCuk9h+~[^VRY~:aJOhԏ;[@;ضtܞ8z:rBwobݻRm1petW1EDDΕ^?Ŝf"Ǐ n}om6,5:t|~b]'ff˵w͗#"""".0X lָ\ePw)w4g~3*9XkPUlnOɆJnv[m/DsR~8GA PN{!i~%~=c;ầ,P^ϼ}Ph{'"""""""rDU~*0q87C y78HyUd^@e Q+lD>_梅V nkf ؖCW7ߧ̙JeI %p{@X{̔g&J)C#̓Aq^"n5S̬QBqq%]j+"""""""raT ٽCp:Чw$:Q&vVRy:$mZVI; +, gc Y94x7ņwTsJ6upK\$>?veNúa? N?` ( lON :}nHʂ-79tDDDDDDDD.\X_f ϣXO:]D.smPGO.B 兆BA+핞mlq9Aqktgl'JX;Bg'%I=j"μ~|%QI?Vtu1}/l01h?}Q2za fۭ_>%G6}x0c䏻cls$E8󺷘󪙟&DZIqhEn^]6Bp%Lfo[z)ߵyyVhKz䭧ਃ'DAPYaC2BBh9;߬!}a1/(?J'kwz''8_͝O伞P:8-d\R;Bsxc)/ go%n}'O(nɖ< |7r*ۚv'z.'9W<Jfv&PN(ݯOB;cڷsXD \܂ 0Zri(_71Y6D ЈHS3Qξʝ侻07zϮM\^@\ EDDDDDDD.P +?CbJ!,gs0ZDDDDDDDDZ9H@ڴvDDDDDDDDD 0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD9֮HꪫZAa-~iDaDaDaDaDaDaDaDaDaDaDaDaH+ k"""""""rr ED 'uھn.͖-X$%:(ݢ=+ӻinghiE EDY9$Ga-:ѓoU3Xg_!QS+tœfYNrP1mFutv%"ga{Z~kWNDDDD]QŮ-}VvbYʋwO!/"4i9#I,w|=ؚUihYtI%zOEܴ%,4crN"%_a# VD>JL`1l7ajijyCa b|md֣d~8mö[.؎ -Q5I3/PUD+{0œnv}DDDD|a""3a<%2|%ͺSl gLKT0ªJm>OQ=- FgRd/>ߎgXBolᄅ9Xdml4_gǓ_0L]=H "Y?ʃ sD> [t<)ӖQh(g"ޛDl_p/ |SIUEK66<ou=Nݠa,NPH76`"r6Uy)-ApuD&gȩ&>!'qqo,M~޷ICdzlN#[T@I):( 0ݜA]؛)]UBۙ|fAuaTNEL^C75^]7]e/p(lJaٽY;~| ȞJ-]?S`R. (lvY1uog8>q("gU)}D>>~[z롈i-8VA\S?Mx2.-o$7 ʰ)x{MbęOR S"/|Ű X6-aCY~SY 9H4Җa;ht",bTtԯ/o!kdiF ̽fp擓S@ed"'n<9[=xe/cBTfd"续[opUW1aڵkU9ވ>/c OV .:̕ -(1cP xYtg^ʂsZF]L1L9G+(hTbF0A /覰DĦ{~C 2J5&u?$NsLDZ΢E(++{<&fd|DI8u@% p*ʥ.z Bz6ʥK_(SJXǴ!! h3*)II0S,;\J &\~^ke3{?y~&'υWZ""50QD|4DrO Ԕ0`g&EmIbȿ|+egT,h̎;tDDkHay޵vL 78` ?,fl7C[1usPl?}P;E\8[NnfbL}[gνzwan,Ki%D˩a79݌"_K|dܹ\u]Ӈ{1v=*""I("r g0ḍ4Rl{6%,EC;f^R$5Ba\r3;#(z$1Y?sկӧӧO[o)LF=T_YN{^-N-~\% n9[^B_qO^KɍL>WHnR<ΪrL}\*?!"""""""""("""""rXl_zuηmΝ{j$""&\F^ӯ_KTgi"""""""""+]:(L(L(L(L(L^ WCͫXKY@S1٘qu#6o1K~Fμ;a0neI4&bwata{ٚ$"""0QDDDD:\KY,U:%=?ËkKɘi}m=߼'q` ;1D2,`[W|#t&ϜFV+ """"r5`X̦]4~ފAbV ݅I;[Z Ƀp cw]$t-eݶZ`$I5!1ux;Qyx7}ߝюh rnK }M<8/c\k1 xvy IDATp:5Krιy8PUDDDbR(""""Q0 |TEkX4?+&@`jo3neb7D +, [AD2[ ֗b޸q1\c ~ם 'QlLtsyW|4''u_;vߗǸϿ""""_ EDDDq3?cpO9kM``'j({skǪ7f1yjg} J5{3X'#/ZI/c{\N` oV?M)89'ٜhj  UNat{?)%lZV;Cn:c{2H,Sw$&$(&thl\hC:DX88; """"E+)_Pk]0v-/UL|oc{h4s&m/ϗAGGP("p+J{}f_]GTpyoVc%Ze]1mY=o߉: ' %;IISjZSHS/v+kw]ة}mdO'5 o&y*"""r5P("rŵQ'/9; OR4j\3pL8-UQ&guigdሻ4}T[)3A޽i4n[KiUc&ml\_F<_@Gjغz5><341;}q` JxTr:|$܂T|޵&p<a[Oәvr9Ik&(oO)wy4@4&I<'e{.cMI\eg1/MYw6qc.w4Otf&u,*g59Vn[+$:q,`w#;9ci=@_ƒu}͚eH*<9['2ѵv3dF6 6'yeM067̂.l׀D{6 義L!o;5R[oXPlY'2}Nk6+cl!+8.muͬg{VXLXC{Zt-(ŸS&t#P^Ə1mtOnхN7ZGL D WQ@[ kAd7C;9ٙT`Y#v6$ -ੇX1e(F3_]z!e~RWZ1c oȡ`8Һ=(^]ȼm!9ۼ%>$Ȥ[KX095ƨm7w51b&gT ?MG>Ln:c17'qm~V.BƝ\g錛1Q`&^px)a9şlMk='41ix~$pIhBMo`t0nB /ˬ*fu?al[ +p[N}<:gqyc臜Pc+wWkb^c%) 8Qú'gvC+ +H/aX<'ҏ >ikc<&N8nH;cx?N(ˊ6ZG 3|<. Q.""g:czlv ,aCSz:v+Z{]*`ŌNaXsF۩[I}hm<~H%k?z0e/d>$pݰ㜶SeVq"}d+_8dPFskyD?=lf=;팩Q|-DjH40~}?*'_GDf8#D E&:vjwTPrb1:F.Yq5;n Ú b VJcہl=gl(pq&Xlʲ e-XNP(n| @}Gj9hÙXǁ̎{]KM WO ׽8Ñ$scŶm=CZ?+a0A©V;FY.<>H |Nքxj | d`XeTVWeĆV;ښ%XONzz/V;~Yt9O#Ǝτxu[WU79 VqP?nYA|k \xsÇ5oNh}ՃPFQ 4GDDDDD5H Dިv=os+=n(p643$soH'=F g%?gY!ֳL!\ImaL_ M˙yt&Dɣ׻t 56efK!r!KJaș  5&5ꉳť)'(tySϰuG?5hV\C>|LJ\pXSI4x \S{X1``bv%v \K?I"r\v8|ays)e{|xwRWUO^}|r_~Vʾ?ȻZԒI1 38Z<9/"""")&aCw=ˬV3[[ 5gL !$Qݸ0c<>9x?gTZjI9p6\'2p2vT3Evp쫭eAÆ9 w}5{xyաG\w0#=bOc멫`ωrA}m-]fm!Hҫ*8rwvR[ .wQbOnL'!1vRX]EԳ-מb 繇aMq`k@7\|)LsL~U??Gvb>_w&psk+e/od XMM>&RHj@lk?K4bd (fM}u/ac7b[!Hc _c&m:A8yP}]dl#!@|U*fv~U}~uv7fIkfy/cp0r>@x&ڎ>vcYћVcrߪ!4 bz0vhF%:v}$P#@5n0:ˎy, 졐grȹOMx? F~Ͼf ھTg^m^LLӤ|=HIo4Mz8sOeERMHFEDDDDMsN_', eQLAo~sa$Г7}|ֵB=a`v~V9ߟHc Xm g:{<6z&snח(Fsz]my佳g71 SID; ȿ|Fv,]~Ça۱k'ъq2G n(eO~x7Pe;/<6 dX&u  Had^bq̙Ȧ7ֳh[Ӱ5";9֐A3:6Mv3E'LcSFr7cƖlZLgל!䍲u]7Md +m3ooKM-YLVٯXȞ: yGx}/.dc c)"""""]p8ҕ~ XL_"urpWSǞ #Ƒ j^xqꙟ_DzH#_GSOπ`_[ɺD1/떯)78 6d_YV^ΟNFoFVgFj# bZ]E,m-`AXֱ\ʜHZ_ q?bcFV-Ɩ$ymI+8D%#>ct]㸳1cILDTFGy qRWuX <Ku~Hs;Hk`pAoV>A&F?>jq&Xl16FMjWq54M)=obu2IӞw5c`VQZLF 1 7qv"UbQ970bÊяgoҹekĎe Ie7 9P±Z4p&q࣎hRSg%8}AJHls1.ګPwz/[֓:z AUT#/y oA +[<(ĸH6"Q@ײFYS0&/uOoc9MV?u """"rDb $0rbyf+&=[yK V0:%͆i+G#Nii]]g\D:A ϋ3꒟,rF}#ZCJH`#m_yoOab# n_kȠYPN]9 vЋs3 [|Ĭ6?+ >9@`Yha h.}lMx }Ak+0FF) DS?. ě]Ni$!zA2J| )7{-m֑FDe_toY]~Af3}dCL:Gdzz&UkS|엽68arNJkh:܇;kk*?Uk׵10Npz/N:k߻'p21t?/oY0OSY 5w*@XAW }46kcϨܶO* sO ~7gc6v *(+u`MBOpDPǘY̱lb˙r+6qaKؼ˞`uH9R GڅE]B?;l۩`j&J2161RКaK59쩏0{s #7vW`}jc0l;r;qpfOh87g~a`'bEJ|/3mе[t'Gx}/.dc c)ыM =Cښb53+MogѶayk3gul11ƕV1HLvuUDDDDDN}pJWBDDfmymuyտ['#ne'"""""r4YDD.Ls% $ȅS(""$}9k"""""""9HT42QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD{+ "r9}gW """""""Ā.5%\4YDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDDDDDDDDD0QDD.LrW.""""""rI)L La#<4_DOpw8⾑UNJKDDDDD*0QD;t̲cק0X=9g3bqG2;d+]1@앮WROS @/W17o.g^ͪ[MM!UU)~i/$+ΌWB""""""WL鎿oCVlG'X;^7~''@i$ߺҕ(LNvddJr" lx~~K&PKd$csi8#s$S~Y9Ɍ&H)[cYqUkxG!7;H>>qq8nGV쥥2C}]^߱ÐQHq}>l3{GݏB@9EG󧰬-Y(}A82f/9=TʢOnwZGqP @ v$`𽏰lC;M>}' 0v`!i d3f/a¨q$ߝQ!~GrH#1-bKM7mShLtj{<{47Ut}(-}C"ycڐzn;#ufd*Ch!_+dˆ>r}2i."""""_+a҂pSg /ͳE:wn83pc'^Z}uT׈pƇǏNM" {q|k턱o𞟺Öؤ=Va l;'l9|DxDFR[nxvx{8ppwuׂ;0{8s'ikpѻemC8qÞa=ʼHnOsSO*g֌pj,aS#OGdd DDDDD䪦0QDG+[X©#SI0{{.嘆(o]j(|vOx,|4|ȽGWoMؓ :y{NeS^HQ);yH[-O$RI"z[$ O9m5a0s{zk>=2` {^n l87o^xŊW:""5i""hHd@Ul 6u?IcfP`IJiiP ~\F5t9,K¿q#(bol.F.Ic| 8TMksWr{H#ޮxn@Cgt 2BU)=Rɝ4_h#XګٻiR YS2'R@JټbM2IO.Γ7("gQfI;/}W'07pԝˈ$hyw7pB˻RTl]_>"geFBTo|?@é/?~ZXjTWWSTT+]-R("ҍ#̈́K|2ĺ)x4KT}P*SHd~WȄ;] 9y$˺]1 ɵi8=g)s3Σ±6K?I$G9& h>WhIfhhh!~J &S_w"_s94, !KD; !>j`gԨm1K7,Cpq>hXRN0_ e>BI4G8E?tc)a`LVu -,'˪i8xYw&qFq(n-HjT??4N}9+]-9>իWYx1=fǎWj""5˱/""B\R|$I:)y!/W#YⓀ&B--_Ą/kqBrd/ag9KfDyTt3,,W׳#pJ3K0o%ݒڛ%arCpQⵎXK_El*#cN*fՊUX(grIRSX|6na[ذm pǟ.*)Ռ]S㔿VL嬙d8$Z, k֬__8~|)LFdԕ0$S)YX/4HrT= ?sGjOry7aǫ9k+G(\*G\u&?M`T@bz3@*N L()0R T?\Ro!ehCGMLB߭$C{tZ*ikgY3X:zﭢly~O.svaK>]΢ \J3<*ΈP!g2b^Ɍ:=ZdtGzk!"zK|dܹ޽{+]E4g/  6˩BL>=*.>.r;[h[9F5P<#x~)33v~q޹ؾf ]yv? W<%,A]rmτ-PWxOsBl&tZ)u?;w1&cE;O>..d!8}rܟg )kB;'YE{i<2rϡ:Τ'i&!Z:w {.;SMHji-d~/xwGDDWeee 箻O>cak\|/ "%'#2aW.LFtʐ,cKǚy}aCj<tB w n )`˽X6{Sv-8c ^Rk/<>GQgBU+0롩zC!ty'%M.hf(a-6Gin4/6+}"ʟe7~o;/%v! KŮ!Z8^5P#.[Q>9,72Tq&Q~3wblfܷ &f>1,s2~c5!wBh RK&Y5ܔKn53 Zس 7cGGjr_Kb;$K&o=b7$ws% eXgXG,)E(jM{5pomV(҄:Ӕ{ެJ":SdeRdx`B3< Npsni;w?/u!5)w"ͭJ-(Sܯܿ$9CE~yf(~TᦨZM|KոQO%.PTkj(~DJ3vi̬Vx[*o(;SS3KTQ Er&KS}Z!c[ oPjUb,PݶFULb~,o^D28kmzN jU󶨢TptNP݈B[B̖gQު/KRo O m )nBٳTzXOߕ)J 4z5V(NgKN=?T0"]k׮iZt.pK,"8ƴ;3TsFS pذavڥJ]q:p֭['4l2V$F+`,K|$7_4M-_ iF08p8IvҮ]t?Ǹ:08SjsUWb*`zꪫN%\2JWls`c]0-l!LJ)BPc]άbjU`"],٬ JZ]pv#LHRw{΍7ڛr}Bu1c7Wj/#v~4-7Ή8e0YՏ_K6+h^3EP753W՜f]SAER"춝A* >Zfy1*F=RjMj;V+0US=٫Y3F"S5媨4)UIyzTvhx\%^Tj\늀3nۡ/O/X5? cP_P+^уOܪ)CXP?!^BnAnkI)=s\ySݿhvQr.>Y0WݦRG+xֶ^ KV+rʳF-wW;Z~U93Tp{ǺqːRy~YȦ:ŗT=%gԅPt,.Tt/"-~bX.I۲I-==\IROWڏPߦ|C9OZWkQ1(gq!a5… RhM"{qP eA$ȿ$LcWUIf+#;Wn[ojw\4e\:E3U b/mo\Ҕwe=ϱ7+Ҵ>h ?ޘV{Ҕq?٬ܬy}ۻ rSLIXXmiޤU_?9W2*s9@)JK_O*HOSM땔€JfsX'!_]GOP+r?B٣IR;^5}xboѡ#mƟ*+U\+w󭗴?W\Tkccm|dmKTT|A^|M=+֝~rL]hK+=+Gym?zX.@oR0CgXC0ߍKr{*8v+Qك'vT7oS\yjm>UÒkRQ~RDN =*PGQ^]S{F=*(˓s4ȼ}|dg{RƕEGJ[}Vwʎ)08;º{B$E]ӭjEGNտt^n%>͖3G=§b$| rIb ug Sߨey>~Tt[DKj5)od^^Z>l^ I=h>>~4_aSJԱ}6JsJu9MX2=TRLIG7˔_k߾Mݦ2uxXZﾵFwK]V%~yy>oyGC7 Oߜ~tVvo~ V&m2e]0'jMI)7z[U{\*;h~8 =R+nT[U--cHē?ޠǾ0Qu~DjXLE.`Huo|㛊RrZ~|f0:u+Gg*0adH2mط;-wdTm_s?/*uTgjSuzf4<9~Uܠ Ϩp'{}wySS*=4f$_J5թ!)~V\n jov2u8e銉I;!k>BW\~ [GA~]h-zq{ȍtho+wnZD+%CR29`{',SpѕkqO![:(J);<+õ!0$uK z 7%%g@5U,)QO!-wN N*T}yxNяqG$a y0ᕑ(B^m&D2!ɐ+ufƳ U|xE$t=:ԾS75wfn<t鵍ʟOGQπKN^>ܦWkӁ?Vg'ybI?y\}AŎh[UKZ4k Oަ ;Ը#ב?W}˵uzK-R#wKʇ8X/묳NYʯ=_>܆,Iܓ]%oC͕wϙﵼ׻-C.+.۟|,jk^/Z^sU%rYX +i5$Y$X\fSbf,Iɣ=Xe[%Nc:dٝn3gxay~2$˽b5^za A rdIN˷a53?Vw1eH2vB1=Q"_VݧÖ $V&P>- Ӡ:U.3ɐ+4U`Y[f̬Vx[*o(;SS3KTܖ;J ./P⽈m) re:_ۨUy o *)ly>L*QݶF,T4ٯ66TJnO]REdoasrҐɠ*x+ufp|ɘS'4Ѡ>9/_༒fY5E|nAiQZ_ 7%c\MJHeV=wo>ZY* -g쉔0.2a̫Y[UސO9#4w͚tsRyժeplW-ITI image/svg+xml Container runtime App 1 App 2 App 3 Host OS kernel FullOS Host OS kernel Application containers System containers FullOS FullOS incus-7.0.0/doc/images/grafana_add_datasource.png000066400000000000000000001226561517523235500220240ustar00rootroot00000000000000PNG  IHDR cm}zTXtRaw profile type exifxڥir$9sn p CdJZZjefGFpVO5*}4m-Sj|3by_zy[^^4$_ <~(ފ?@=|m+6Ͽ_:_O߉)9[I%Z{J(T*7r*cKWqQVt}7}O?dbcj~R~r|/u|9׊~{}.fm7-G`i-v!P‰;.~wҔM54MYbrC;(=kW7܋SIOkɟkIϴwxf|`G7z)$ߜ=,3FF}Uy-dP<^"J`k%'хW ~"  kHj){Jr35@ɇEZJ#7#Guh1tLZ MѦ]ɜ'OOnsUV]V[}knvc'rN?虖 (Y5f݆ n ^;wZw)f NsFrwOΞ8R39 ,RvI缫Kl@B4HcR&/#I.dcHC6GRLgOEb˘e:r_3jd0 b(=fe:Ұ>vm`-dp횓¬ҸXQn֦^nyhw- 6ZYzoskRk˒vGZ=q tyu¶`BhsQ^rv`Gi. eͅ0GvXFAqՙ*Я90ʄ:KRH֡^9`xCfKָ78}AD:ZWRp5Rk#(暲%Zvv2 B;+*77iJm I[ D32zwdC Za HHPYحr>{ksYW&6@&4O ix$b ڂo\Q."YLv:sW>IKdSX9d]gˎsQpGa(ݰ>`H1S)լp2~ha Y:xLS5uc3Z] ,S:qiqjr Ik2, 1:`[-7;iJ$].X`M,n0'F,} c%{f׺ ۨʳe0&]u -#4,fǭvȼׁ׵w^ TI̳WՂfA ~gAx:]=99uzv}Ţh2?P0ʞ r¹":<>˲U|3CJϑ"8Xb PҾgi:s^6E TҲVV7Z'{G1|p#h+f"1NpYFS)[ZB'li+ H$JrP7è I;#"g̃ЌT$Am:rI⁀i}cpAZ[EgP܊UC86r'pVt}>M2iQ{7-$}+A_/O'?mgNQq>e_S$Zص Q! ֥a3^ :R8AO8XH@;*fa/I{ 3Xݘ##56bMą1x]k[@<갰- "OK ~,~pUYV ulF<ۈ.dO@(iYphnxbT_j$ b[q&}馕['TXvPcLWi)&Lvvo3:d% rYl`AL[p R~ls9oKADߥ c?ӄ@eON~qFoZ@{+*>$P[Gg]pbRN-EQ@ұz)IVH׈ gʀOu,ά#ʣwd ϋFkyNG߬>@ tUrHYym$eΰ; 'q U Gܻ_0X -FK;_VLLj+F 2qrk5LW!BQ`1h!!vC@i\hwhšޱEc%} ?r[|~[#4Ҵj=4^{ezʋILfFBy0,h|jc3h(y#q0^Z;mi<$&lԆIJk$hHqrܷ[(ѵ H < c~ t!4z UPS_YϰX⺇Nl$V`= RwbBCMd}\c`BXPx uIT&1NRn~*,n_fLuh4.?Lo31bx~Eez K`5;;?ӷh ,?!UE98懋Ww! ZOisAśDʑHYƀ[t@+5O˨TO${ẞC99A,FL@$XDGo~6%,(T!rZ {ő~ﶉV{<*: ܹ0ܸ^kg^vv KayqyJ(mX$6.M9Pedp/+@t|/ z*b;O xtz.9 ވafE˅@Z~LDXW667 M=!-J{AyP %P?l.4 FF$yl5Z;{ ձK4:2cTd& K&|wtV,*RsH]+k>8`?54UߠRhZђT*sȱA"n8J w/&KvR ~(ffJV(DdH?[sc^(ѫ4$n/v.%z7K ʙS5[c:-xhOt TD]?}zbsC[$D븝e?ER]]Q;o/3?%rQI&S?_рUzr~icJا9p򧝅o_. I@ ~x (ϼ9 4Q?uvo+w n{k9Fsp{s@g>_xvo=)WRO;ZCinQq[oբxg|,xL/Xm?o>ͼC҆1ln_3p %:aqjF[S.^NPiX/n:{݀k/H鲂 "iCCPICC profilex}=H@_?EZ ␡:Y(*U(BP+`r41$).kŪ "%/)=ff0 ed)!_XBaDϊb>%x?GT) 3L7,uMKOcI!>'3ď\]~\v3cF.;G#],w1*$q\Q5]V8oqVku־'a-/q0X"DQG5XHЪb"K)% Fyl@wfibMB@aǶ:o4O-~oM.w']2$G J}SzW8}rU88F˔pwoiJrk1f iTXtXML:com.adobe.xmp .dbKGDi*Y pHYs  tIME5ף IDATxy|TO@& l@ a"\֥<iEl*qyERV KH $L$$$ _/^drsq lZblV""""""rrU(((((((((ǟ7eV&udQ6Mo|1Rbʔ&JK^O 21ם֦wp#۠!"""""5h2 x0- c ..۽(夽<0jO.uWDDDDDDLsf7p/7DAyR:2ʏCiGO7&6ѳ;1 wrQ6(Ep{rYSDaA-AG=,Ty(ֿՃ1MՖ#<[3VJ;HcР 4 0]2.qyN+!Jt;g;KzS;#"""""H4Xif:Pޅ=\yd֝t˗'yɟ{Z4}/яj-i,t\;Ky0RrJG;o*X#^FvdΈ(O^ς+.&\eW 'λ.W .&zt]Q07-o)LRs'mv-Ikqe݁EDDDDDD'Z;[<]eO4l6Zv pypb6\q?5U4`Vl?T6uCE}L'c}𓎡ss*K7( C7r[DOygzU3''EEEPZzqRN'NÁѝ&z d6C?_2|w`؉ڇwl6'ӫ5 А0/vӮ֟"í\)?jGJ1?>EL cqgw?3H Q:Ӻal8تl*""""-z4z~p8p:4Օ[.V.7w}nU몛x݆a5Z{s2>kiXLEDDDn'fIaaU -Jsrkv-[7^|֍'7/pj6oȵZ`ƨPil8>Ұ:b٩ lX<秾3`$3o6gI?Sb+2Vp`=P*""""%Um5W8?? /ȯWyFgn;-Lgѣ]t۫lҔ$VܜZaVpî];߳0oSYV9>ʎɎ9tʺf9e甞z6-eI8+fS'^8ZF'cV]y L& ޤRIJ>q2hC J&<@Va kl) \mnnlۇj/pƖ}mXY<-mƻ t;K.K촷9*#8FJ{&ҩb?-sm Oǚ=;ciՙNm{3D= +<nlIXخ#e!r-lF|dŞFϴhP&= šqcpZCO.{ךMn fxhv (ФOMG0AIDDDDLoɄy.YY1WN+Fmڤ)yE7v 䔤Mrj˱ll.9jrJuNN#qq$~qy,X9||^7 Fdyr.O%Mf-X|z`۳i7W y:dgK esjS 1aFǿW!lGcLl=^EAb`3ոf7nV.üZG-3O)YkPϒ9Wy7>zDqɥ^,h^!,sPq1eUv\dFWox{Tm|dJnM3y\Foyj*rYZt[!q9U/aqIgzv3^F=,_JgKsؚo`߱`[-S$-a+ﳘA.z>*ЩkrDDDDDj /L 8~صl]:w>S>EOɓ7veGYvܮ7dT^,^خ~cc4VxG6eX>ؖ5}HfZ:ͱVo|b{doZwY䆯dz}e0e,2/6;y6O""""re*Ek$ފpKs 3sO]2ݔkڵkG}u=z1xаF^lٲVЊNkFpBO`X `|s&C*t "NWm;tp֜0BhXsC|2`"VB4p=/H`h: '4$ #$,Zpo$"""" 7P)!;j-;mO˗V[W[Õ@Uu56-NּK%Z,Mn]ģln)a=<^*05iS3c!t kPO""""`z JJJpqqli.W/e7d /BJ-uPZ1=z]M8awdtFݰmZu RMHxobWr ;ngٳf=Ǭ(( zK6Zx3ޡ)kԳ٤p5Ϙwl^#=+fG|))IsSXT#Ml|lٺրZ۹5D-ZO6Fڍ~uor03/ scF򀶃ףN!%zG(`(ޟSi=}-+9 @/(zgdh,kM8[[ܥ=/eS=](^5ݎ`ό^i(uqq`0PPXpSKyckZ 2),ڲe%[lw{îӺ XKz˂eg?o&8yş& ῝=|) FfQnrRn{Ա|u-3sSĴ߳y Vcd~S1JڊZ^esx`!fUNISuHr60v&vhX&IDDDDn-3Nv;FMA~>*ݝEE7.Lɵʒ {rEy;LZFGI/%o:c~>zf`ԣx iX)?Y2j7kr<aŚ>Oٗ8mqDDDDzp~www\]k?zRWWW|Vr>,>Zc r`=m_K?Y 9h}o>s>`d ڵ ŗܶ'1DnSgl3Ӯ'U4ױck^yyK7q,Ê1=!ffd^Yϼ9v|.DvӼPW@tpӘzp`kY}[' -N)OO 5iBӦM9M ;|ϗ/dÓRKjS*_?{Z;ڲW罬Ouf5O0ˡxJDDDD-1-BS \4i҄Jk3nDlva֍=gT-ѫ`wEd6@.Xw8HU00" >}."""""ċ(-)zSF#&sX@h0hժ5ww &`_vNQ e!4oނ@VvN0j6|ZY>OiFF:)I۷}sss˧ڱl٨ """"" l(۷ddΖ8޾}{;Z0ݵkZMEDDDD DEv!$8VRRSH8F_=z6Bon^umܵk1=zUv6j5;/z{y hQ>DEv!555߬$7/V~rm6fE-l:z}jfU{F54$Z_EDDDDDn`OGd2cwIHcdff@vDEu%88K?[TINI"J8 d),^1=~ҋ.]SU_Pf3cN 9۷-[7b6p]1[%g#q_#ٔh#xtp7 8~ 3(" &xaQC>JN 6pUOADF]QQ q -~2.|1~ ~lQi455ԓeQ,lQ>|&cbzBcGЕ|h5/Qԛx4צ]C'n`uM4;bN_eD6stkaH Ks| q桧_w|}hDDDDL֚5+y' eC曕uia*Gf݉;ֆȎMHڸ2Vjw' hȌ#;_1GpODs?98SmY#ߵ=F IDATغpBYwc!]hINjkWnx>`hgGS>$`x_ 9G60}ĉ5(;׭ >͉9e&.gXz=SxwD$>v_ެ _gӿk0'lZ,cx豁vǒsuRaiߝH_ط s!Zѧ53} .}3<{mkVT->im5η[vsDyfy]ǿHtNkyv~̴51iarf6eudt4M$"""`Jn^.K?!88 q233}"`2]9|(~^m3@9vom:]>( lml?36:'nW$|7 XOǩm7 *نGF}S"5t_ C;Sʣl4<3>=Q6AF!8Kggm!̳{5-]A|욾 _e[%عLБ#Yd>u+qӮ?"}m0G3#'E4rY{ՙV'[so,Y (b%vڐ~$w~7;aȱqFm<ĊwM:2t\v}߅ws1ɱjinzqڑG{I6c6}rbJRٻ'>c/o5{3@[pu"o '4JHGYB H c۹Tm^Ȇ^&@X9r1#P s$gi(G]N!|,Wٕ@ia0hWeͱ_}lZC2'nW gd<q G,{GpZ>]OU{YFO5^N9uM@Avgy߫7#{b(:ϙݬX#gv|ʧ =MNm{|Y񌩫K3?ZZpӧNDDD*N8œYY˷;Wjtm=S~E;Ω~{}3wq슈HSuofփqz:j ͳ{B1 "vOp܄[O*b*w)P9n篻 """""҈5t1&rdxJSSٶ}K刽^!q b҂e<~a#/cq sF5$""""" V*j>lQ]^;218/KN-88HY19Jf7;Na[YlȨg1O|> ."""""0SLTd;q'>(Oﾕ˛T 4RNI]~?qlª*3̕?8mo7{#/%ؙ_1pތcof~2˰1ts[Ǭ)vu,~kN;.>9^;IӦ2G{|ݲIٱ7\A%zLV_5VoVDEuG.8Xڷ Fb OGijٙIg3Oq 8Y4}*z{VL~oelf?Ԃ~y{{TfN{Z3fNM{s 9c!o1#"^9{L}u&9 b_X-uftJP/2ҟZb 1^gZx_!9w?88OFH쳳ﻊy#ѿ͌xD%j {XQ:M -[( mUxk'qoN&VilXɃ"dc(veEX#Xv"}D3Z ·8v$ ߂|= 9wNU,F3/YFķCY6ogM lWiKckFGsCsv=?M:kz7'yea8sxb1}u(Oaxa\u8X|y+9ƙJP5BI4| ,xlk,U)YH1I_>9[3g%k ] ;vbɮU,Gof~2=lO"(*ƑDڹ=h0z?ӟ[BrϐM1ãi`Β 0f,A͸OT^*N`@F YRRpf6xෳy{a<0c64IJuez-1ф '3^\f+(((靠ɤFh2SRRz3 M!"BII19Q0USP򊈈((((ueP\Zs??UU8`P+`x{ ' 6,=.|[ͰhQĉ HԖWs8eGI[d}瑑 >c_d9poFԀv sx2= C{t GNe=4#v뀿9Sb#Щ -wן,_ҌX'k `.ֆrEw5SʭzƂT6,c*inMϡ#Hܴ;2nIf<;ZpoٛQ#0V/ti4^3X2{3q]x1?/lz܅#VV9e> kW"ӻ/z๩3ݗvU""]Dx_~Ɔ]{.):BbR|%F#a&~(QĻ*=+;8{{Ήe-xw?׳3+NgT`Y Tr}l~H:]x xfd=Vڌy'{}gfdYφ=h>ۛ#~e< Q_aƈښǟ3E_"zg߇xt@3rog h=#K<ӷ-G0cK;G®M)߾Xy4#/İ9Ч2ʯtk4/͛I_TvΉ#shSҧTɴ)8{NL7O(Hv~ؽ6pT zxQaVL9ss>M~CӧGW_1ZQeD)G:$L- F IDAT_&7v"{><0|=""RdV}&>衡į+7t8]v2j{*5RK01#H8&|l65=x4RLD $[" g5~Noey>i&^aDo]#_VٰGfb 8HfZwC]̖y;l: $ uFΰEX,7rC)zޅȘQuE Kn6l~C:??/H?mӦGџ,Z *;*HϘ)K%(~c0^ذWoW(uPjCw'uKjëƜ$2`wjfy0 x}VX,X,8v ~*8p?#"" 'lٙAлJH3H°Dz`||dlw ᠵ^%뱍}tI?Ev^rW8L3"*KeCnmP4z@ r03IIod;6I虌Ρ?xEP~2ԋyur<+j^} ?kzyk&k*F:޿}֝XX9}"yp,S]҈{q~Mog'JG6KQ=)γySݻ|s3$:&}ˇzCA:ua"<,Ka,v5]ȏ;>5tTT}^c_L6;묓WDz_PɜIk:clOc G0{܉Mϊ],>lhCG*X~uHk=xQey9휄(1/N$`Dy¤d>2G <M5aiY0*i{6^^P^n|e>ҁuz}թڰvnp/ %`!%ˈ'd~'X^Of,OT/6d9텗H!vB#?/c1^GBwMiq0ƒmvGen]&?9"+PXuuu+_8u~X xc|)VoDx[G0yM$q;y02 f8z\lZCt*$=v199cq> G`<}N3n]=bբwL} i|lQ)x$g OLn! [G1qTO UR*K^>tLjr<+}G=&` vw>h2bM o 5eq苆<.rʆA.&BFe ǜK.@ԝ}"52r>+%ʣ:cTJJnmɎ}un' xt[v!""ߛ1%}((ȯw»G=fVZYn}FAA>>tz2DD䧠Z6R 3zV?ơbӳXH`CHH܎Hf? VlŇߟm2}}ˬӊ͖ϡ/˲ܒ[DʡTƌ}U3g^,iv}V9iXdnx=ӜALzƔ,>_ҭgmzccy|2OwfSxX>ED9Cg#bySܿ0r< Y0,6e$q>gg)'XI m}bIyz /n''z4LX+FY{J9z6>4X:= sNevv5.DD{׿ӹ+-zlwn`Gxyݪ7Vi'Aȟs$I䳤-q*(2;vh]-MD¬ Syb[al*Œgm6Mf(ϯwk0aZ] [A6[sS<6,yE ^R ^ ֡ > X,׈>KO#m,ͨot>eM8uwYbՌv!""ee,Hi`|uW"^3Kb:}&}ouY"""ٰs)7\K);^dL{KZsk^R֡_O"Ui8/ Rg״D;)hEDj1fc~q.2{vV^Ӕ׌wbdz(6_w<]>)Gy+HkЉ}^z9{K=R3P><0X,y+""""""ב10v];O*l$I`.KDDDDDD-c%=&tvZ\O8n[(6[C!~鱯Қ }G %ϒv7J]]ň_+CDDDDDDL/ŒOҞD\]ݘ#n#aL.L\]WDDDDD:Ԧ% ,i7`_c)'%K&01C`̈́晥޹ DDDDDD䧯s-]156 [ZO )--2ڶuWKk\ XYΛt! iVt6D>s9(uLCJEDOաtQ\7vv8R^-tn ;7 h=""" ""r˓s`wJ Ǿ9&7{x(PUζ ńL_H;pmgq+:432XQ|^m}#onmNe}ʍ]ºw) tVle˿8[AsۥS6|SU` qɏwcrU9kq1Fs; fX7nhZA"^B֥+ÆvdBΞ.Pan-:d-=ڕH=KzӮFuaE/x_oVmwYӿ` 3ltv;C`apo~qw(qz e4xfzڟ#d ,pt_/5VmܸoJ'fuqVRArt#"ܛ d]flm`,@NY*4܏U=koItd"ftgB=sz]9RqzYō?5ouǞi{@ҞD V@W?7Bݸˑ)Sαr~;c V)K8QUɷKݭz{!jf; uǛ\y*6FN3_Prf~ [){-ga } Y= pʫ}ͺs@+:Gb+pv玭&tg3Go$$ҋmg8~X\͹̻2Pzx ƍ~sUms3χ]\ΫzK:m)Gd.[8QmLL6gIpyI7KW_obvY8֭45G֞"FB_َC۱=-" H.}ZccDD#/O ੷nh6p NvҔ=ޡuAѝH +jeկn7~a7`m| _gٹzҫW- ֐{[[Bqbގ~u_}_xU 7i ˰9z'olK?Vr֭d]zܣmRɁ߲wÂ1)]Vd%II=ż&T Yw;!*g60gm1bg~+6/W:Wx)&uq'KYDD~@4{q{y q CTD:10܃?o盢n```G-ϽL`4;ЙRU%֖;nue]goT^yyHͳaN0mc&FZVہVpzuؚϫ# -Kӓam qm6Μse؎tB ־7`jtɬޮ53v`,ׁl]{7!uk ʇiSkE{LCde_[=z,>9fDD~ !uw4+ڕa=0RIҗ%38ǁ/K9ۓ ΜOJ7ڝcVNT!S؂G=7Kt'9ǎ[qRh ^SW߲41}vW([WW"|8Uw`-%x3כ>JRN%mu>uepY5 ]&mRFCA: a&z{]J9\ށm-TG@Mubzo>i"9q1~m09ʏ kz5KS9̝aXx[[Kl|k*JYq1`S kk:݀9w"N|Ǻ739ć7lOrR7Wz^e'"""W{;VߝtMdω9&; Vnʙ+*Ʉve F4A3f w^|She*QzƑPn``oW}|,7y3g Rwo``eqo$$~ _f_ >7놟{%4ӼRA%Y{Rʩt7ѕP_#.E%|0/n5g|Qkx6e~l|Q hM֑oٖ T`6FnoKR+YșvHO( tq%г5ߤB%qa}YIKڍŒ+݃SWTwǎ\QmۺU~;#ݭ%Uon+Α,GT "VVVzX1K>ܨoED֛ xs[b+vKTr,Ϗ[UI"""ב6|~ܪ*"""k""""""" """"""`*""""""`*"""""" """"""" """"""`*""""""`*"""""" """"""`*""""""`*"""""" """"""" """"""`*""ד*iU9S}W"0}W!"RGUw"DDD{SQ0SSQ0Q0SSQ0Q0SSQ0Q0SSQ0Q0SSQ0Q0SSQ0Q0SSQ0)A?B&6yg <5(+XxQDz[K[l"wk}q,9V煘bߛc^&v(l^ \J yW^[OksjbY1/kYYݧb\ :g4O`:ds+y0*: akkëkn['`i,dRy%Hf&no?sLUMnښn Xˬ ~l=m0y5Af??6Xşߋu(㟙yS,\˱=R[oeH{iE5(|n+[H~0`0,X~82C{0fa!K$3Vn~ϑ/0XT-|D%ddC' fF?g6񣒌n.7WyiXN|DDLE_;gc2qh 'Nfbjkձtđ) ^%/bl2s2_w|EJcɫR?`%lf!0f,{WRRk IDAT\./dljOL&M+HQ͈ &b_Fgl}ksubݼ4H߿ SK4_n;w=BNA~ [g^ҋ*";Aގ{ewIWD\L w%ӈ5/X?ً1Ǽ+`C$Oö~=a 0aO&!LSP /%Xefǒ8i& S?W`/ ˱]įYCBur^Fz/3ZL795|(7'#*V2o'mg˧a۸rlwA6*w&GMghm*~wm4ɣ샱"oZAgL&%Y|VoH܂>y:1񢐴-k. #?y |0 b3Dz x_Ī=z#3bڢD,19O1g]Oᕧ}H+%Axذnfb9T\{4tSc ּT7b۠mxxPLXws$ VL#x`T:ʝz@}0g3i{z [a[n +E|EʳblhSlٍ鎑9H[_rVyQŋcgr=^_QsD{xo)kpkr ;#"1.&s,c9v=vb"1eq;MW%i$n4i,?OguMe:ܚk'?8 օd&Ʋﻱ 2,f Y峰_8uWݮq)1܉{d%$̞bc#k`/^Ƙ@ fü5n;\/p,vטtKd8wTwg HIes`S㕻84xLe [! њx-ρN^ TAel5f=,uv.WЃ~dNc֢&WZ"/܂ཌྷ~ =s Zf{15ѹSХ/FqI[P~m!sf\0Ge޸`2/ɧD;QW2F;[m|wui+';BŲvA6$ck"ѽqϭlX1 <ܣ-g^f˩>=(˾5eyeX#y N.,zڷ|4=*)̞Yg,e>isX}"35:4`.@3s&?Yhh\K(?:kX8{ sls?wa:O51 ϼ1noņesxr*%sVP&=1Lc,iq̛1Ox|&,D0c67|6&5}>6;M`ެ'mmfN{O| LxJ㳆bb>:9مaL`jx=G1H?ZȜSX1Ns:|݇_#ĒV騯,w/Dzէg\f36{0iƨ t)oMc4̙2N(CP=|tvN>ovi<^-q",zZ7|L|}&]]ʱ6tz}di7ۧ3u *y L4Ԟ2bٮ=vK92gGmhݚ>\/+'^b_Swbܻ0}Od=98XSoc!P ;1sXDF>괵X=D=2ݚOO`sd2C}jjY]EMoOGnE \'ʷ59q2=JΎ`mOmK )IRlN0#"|Չifp(~@Lq.WE>r+ +#a~$ϼ܋S6yߑbd`˪l=MNV2[Wʖ ,=(z2_浝Yi<&Rr(=H L]8o{O˲t;ھEZ_fcY8ͤҹ/YX%mzt;mO}q{H&s,O.K!37UN̯7Ru[a`c:saa9T#]oix}&4,Ed&,c#H(t|2ǖlM˧Ċ%##|2w:ݰkU^^O/5li=_ =$6}d%no!Aa*kKE[Coġkqw]ʱM(>!(نvlU@ev{*G܅y gfs.yzff[s/r7t ^DX{'szA_\xlM.9 'j Xl؜-NMdذf yi/{),ZތFt+50׿>yx6hŒwı8LmMg G8kLIBKO"Dp9Am]naD (09N6#ْKVp2)d,f[ͪ;+EuxuzII3sfRM好HIN&]|yOVU_o³+cg=Ʈ FǸCX2np4sOR\Y< xor1JK{9ruCϋ}2U`-r.v<^W6;&8>b/hdnk!Vv*׶=GkqWnȏ_ntXxy.Q5&~L&w>GE˵gg]UEX ̨ryJW ׆ڏS]fkї/嶂]$<@BR0U5&Rv"eArsj$Z3Gv%GNcƱe\ۮz=}c7ﺶa=Ւ$!Z^h5h: ’a|ž-xX+ Oc_D <)s WSY< г{#fc+y;ꬻ-y+W|ؚK?_7Knח|ߥ׏͋X_*K)\x,˿z!wͩokC\eGǥٹ;zp h(5wṅ4\Ͽ>MNV69 l2 :ͧRx[d]7.&Bꎉ5q[l9-rk: [(.3&rNd`6psRfocey[0G$-6=~iqw4llEcxK׮gkRrdjNgaU,}n.S}dfY1nGDo irloHAH`mdecT><m-uab&3fO{2}@#{fmUػ,|mPZE.g$lXO/c/ѷwlmpǫ8/W 46>  ("%qk__Ĝ?nc(Qy%[K^>FxݡaZ`9W<kʎnԶζ}hIܮ/p痖WX_O"f7gx wU" "?ctO 'NfJTf%O1u@(ԨR>|~/٥kAfDGL1sߣ[O`O32/A^WF̘#>}.#q[x*ߡo0Qcg2b,L}̛41>dFenãg2Y kvmv,a=ֱ!1Wm>V#!(*( 5Ą5MSXӌcу1ckޱw̒`OBc;&s6U]]s:8zÿ֌4~^xl}0p|K^oѷb}ǭuf+q}Sӏ+c9}llAq=*s`v]ypݓ#:&ښn6xuDT<R/-Fʶt%χ[G1z G (f>/?5`<}}0il/q\Vmz{E+r mۖN͸۽[8xi۶mN(u$,ca ]GZmk\ɍkVm,yЄ-?Wb!L嗿gymt S߯~i /36`=WXIO|0iՓSז:fmbg><_m'nW*/BkeMŒFa,󏷓xĆ{@6|3?Œ&(&%q91k=3L?F$Ϸ/blByp:SU!)O]{2_Zcyx<]d"e,n=G?c-fF9W 3ya] fW2u <04{V&f`̽E#ĕ,~ʧo-yRǖ=762)~ n&n[*7瘝c8Ć=4ŊAYC~l.qs_?g S؋Ӝ')#|uc9fJ1v./5bڍ{0O0֬kW:A~q(܃"[܌v-g=lY4'+/W,3?~2F`٫,^_x4b|S#x7"de^Ŕ?l/q\fmb{EVT7݃SWTwǎ^٧Q۶jiAnlvJ~;޽8#,y_lgx֞HKqҶCd/ڌֶzeՃ6^qz&EDVVVzzLE3nl=s {pUH5~q2OVA)ѣ /I&>ǽiSX2=}&!$1DbO]ABU5E*a"Pic*W kX ,tή γMWW1 Lg(<1ً9ćki|c"f_D?1rwXV i9$wF㡩㓭uՏGg?;58""""""ס&zL:JeVI6RR>IVw2+CΣVreY4Qi{LρMs{M(bZuL&wVUj-eG0L,29II=mQMӪtڤ̌ښZ[jVk)eI}15+㧦2 `UVnQUUuMۼ1Mwo㓄m=Qw }Ψ1݂/Y;NQw;&?$a\-]DDDDDRf>}ngvL&?cKcNwFĎoӣdDDDDDD#\?w>;PI¶1w_ԇ&w_~sZ=!RCAVDDDDDD`z 鱦7Wxodd'ߒǺ@PP0P;q2""""""r}iVczZR틈HS=8W Uο<:q))<4an}krO""""""ס&clDw ϐǐ;ӪU+^6Hxx}^3Z9$};j\G.>s|i/aA$FcLdeVыZ=#/OhF^L=˝цzbasG;f wFc{e*""""""L5c&jARɄٗc~_E5ܹo$>'&woQ0Q0L?s޽I6LԌ9PI6+"""""rj9cZ핿,ᓄm5z01kBiCiupKcLqpӼ뿴#ޝ_'I\cg};weVkISO=c'ƹqwv&&;bU<<a 0a 0a׿)s$ٷ]scs_Ld]w}  L:(IDAT}$9qӝ;wС# Ljbm~~Ō(/0o4/r8VkyW7r#5;* F.h4O>֭_t$/䵟]fg7]],^vOkY^^Nؤ+~@{3^DZ {1?x7I>Sx_ki46Bn=mǎyWlRmZמ̟<}WBXy_{ￛK#翝$?"v O&x~jpK0@0@0@0@0@0@0@0@0@0@0@0@zʛpVZSj'I˅,0Ug4 v@nl>{)$v{xNwmW3xc|:T}`>yBK]y'Tռxcӵ}yk{>^wKQ;ٰ䱼Ա Iؕ'jBcJu_DmŮ ɫOO# c-6gn:W֛Lr0oώazwvL㧺T;7$#Iu;:: ^bX;݅- ׼tU|iOk9<7fϣ9Y{s񡶻t}Zws97u>]9Y"nNôi/a:59]bjM{a6{%YjBv{T_RR._?!s5W2l/Zȧk)w SazJBͳݏͭ6張x1ӭś9x1'r2oΖ?/>j[;h.Rԗ]{?m|ؘysܜ˶RwVfwo؅SHsy,R?D׮|Pؿ\; 0]T)۶;gҟG:ίL1LJv@R9LkJUv$=I_:+y@UvVvm L-j9'Ko:Y|hjJǃn+ࣼ. z 3PNҝJʹҜ8^IGnglREaCwYOdw[B~p.-]VwO{wka<^ۓԏg{ML9KݒI p+o"c幜:~ʹ2њttm#fW%ʹ$#͙>7Ke LƇy̷]I|2?tdte&p>j9(J@Qϫ{=ͬb )sյc3W*eiv䣼 EPrC jZj]\|! S|6^]a9 a.a a a a a w~ރvIENDB`incus-7.0.0/doc/images/grafana_configure_datasource.png000066400000000000000000001426251517523235500232530ustar00rootroot00000000000000PNG  IHDRc0sBIT|dtEXtSoftwaregnome-screenshot> IDATxyxTw2$!$@KDR#*Ȣт]@-ȯ"Ej)R mօ* iYd $dA$d&1I ˄r3<-(ŏ\{[W@DDDDDDo """"""fLEDDDDDmj16`*""""""SiS """"""ҦLEDDDDDM)HR06`*""""""mJTDDDDDDڔ)SiS """"""ҦLEDDDDDM)HR06uU3=;yJy\L@^H/ZkUi1]v@Y=<}_W-#5ĥiÍāɍ>Lbps>~@5*j 9RpeDDDDDD`H7s| ܠ^ԭn;~^z_uu-nnW[ g >38RTEn51yqs' g UIDDDDDD}. EOܠ.u#6NUT]5 +3=}dHOWZ""""""rsGWf7w7LUܛ\|Q(P\G"|mi<ſS(^MtG&~lCi?ӻ{qs'O;yPl&pNQ^%Ã=2ARaǚ^q=.[{n^΂9W`?|m}y4FQe5p tutӛC< 0ûyK.%U^ps'o?n*jWVKDDDDDDa.k1AtS):;ߧ ni+z{8_7j-^(/޺̀^ѝ=6W% ^8RJI{v-ѝ=X>\%v8/)[ó;n@u5 ɿSiC&z 4h6d6i A pS hR""""""wp֨YƔ.&v|ͬ3Dyp8~Ql2=LkMŋ}g=g8h`q\ZLM`pTY nnpP&px K7.> =qvjuDRgOuǡڿtxf,EU*ȵ>`﷕ 'nT;nn7ngLŘfN;8UDy:wu:\q߬ ;ύ~X<<\xZ;`%jU[HSc}Yקw?mLћgr0!y ŗ<,?rrZ\0ϋ9ʁc_M>߾2V*@f@l`si0uۭnۇ Cύ/q㗷uࡡ+;8CZ%N Nx]yϖgdÀ SU]ۥˬeK .ysJY. }Y 1WXnrwcA|<&ADDDDDZe]L5֮K3oj>]oB2ʝFU5]Ζͫq޻G4.""""""-qsȨ˭Wq ֨5NߔU^Tn{g` #JiQEŎ*ygG9(ǿ}#g7yC5RsM p-~ɇIfV {ΖqĠsٞMgl|]b`n˜Usi;LAVjq9Rɩ*nb A& o/o<=xxxL;FnTVSZZ~qt{/OйIzx.~^`p>/ *Q3j>ȽHh;%z9k6w uUDDDD:}x{wbnꋻWWWc۱ٓP (] &[yW3h)1\{dY%V9^jmۚ[ڬɕ:{lM>"g_'[`N[ؙNIzSQ1J2I?f)|Gg&~f5+my\eg˝ ,ݮofణ+$"""׺6k1m)QV^a UWWcetZ 5uǓ{,LFy%6gM 6ƞn-!Ƹ'؏|Ze ?;?fB"c:~:gKҭ;hRF]XeXoļF~A>w(QG1}᫼D7obfi S?bYvMHsZb%`>WVch`U'nk-kd+TUUQZZǎ8J=!~=@mk)8g=l{o+*ɬ84afS;6>&w [c}Sc/?mKX#{=q);-e.WX: m1Hx/kFMVnó;p3rښ"1/Ԕ)輵?1- glG79؏0݅-#M/=Cr Bn1֫v~Ws}Ϭ=Jx%~tXMdDv#d@Mp<)_d}eO:\]v;UUU ԅҒJJlw!7 ׌QO։Dvn^͚5ƚGtbkpN&9^Pғ/ْfŋϝ&ܧs{&14f ^]>.։/1>]Ĕe/ǩ,dmݼf"62KQ ~ںb>(` 63o[\bJ|`#C`[X^ &:3V3 295ג{zIz#3zeRJtRm4~p״ifGٷmmcWk'C_p~w2c#yqv Qwffł{غE9{{2IOIffj;q.3\wY'Jl=y~cl;WҎyC,""r`j2򢴴1T䷿&5ˬo< =[m)[L6qɖ[Gؘ5&k κ_` mOYа쵒Gj^02b ̿-3`q<" 0]p=ӕXGa1#\YS˽,< d^"aI 2f)' 哞QJx( gĠe,x#\8ubu04S1c"G| 0NfLHL5EH%#3c,-7t kNay7p5Ero_`MVpJM!̋ȞĎCl\?r۲pa>rٗTB0c b/{?{Mes:OacqE+Ѓc /0!2{IlLO'/?r$O%A>7Տ>4d3,lTݳuB8y ^|~؇_PIbzayn+p kO?LɏFܻ>+ytgvLW/ۅ7' ,5*ۡCji&fĢg߽r;Uߛ\Aawӻ,ч&wJ PD5zSQp =dXv-aeCy,D&ᮏ~ʆӀ#$:'kIxxA/x|k˽mYDYyyuO7lN)KX6Mi,,bo]4}5Wb!~$~j174ڵg@Rqj#ʈ~ # Uli9 ^÷ 'pg+H_MV8~m5/#19 oY%Gied lyvi̜!A֎,{1u83LdT~ Oc#3WTse/J԰'Xhqݬܘs_dg4o}^oE;"#nboooWU<:5o8nCVv3ǙBˮldv~ Z5?蹿;kUls/nmZ./7u5K~3aϳ3;KdjnOhMPcj>tq±]ϲd! 0]8E[[J-,ZXG_{]}s+doa٢@a2]Ě#fs4Wo@υR"`49VUU:k|߷yT:\JyR6UӫT_͆ q4(6]+-Ml|% %K`J8FLż߾ʺYDb/p3lpal=b%Z἞4B1436vv]SV #svT8zpdCfC)ґ,DG]ϩ2q^SY ;&KS$7 _~yHbco&nOWA]/u? wYa`%~X E] lu:(W0~(- ^-.W'7p9tBhй+ls,\ ʍzadil]uusskLYJDD$ /̾UUUNFlbMm-l5ZiT6Хl%*={&[(!AVBBz jrRٹ~;Pbo7 f8bG쭓\V a78\pB3I: 1 $7:P3"+8ʡFi!"3!#`y%v p~ad  o}mw.fD,pH$))CI:NN|x+úϹċ{W/ Pp?S9t҉'p 6"ʞ ]o E`cQ^XFtc?OY37P23yll{g b&>f)o#)1Cɉߟ|""""-֮iSUSbݚ>鏫zWѣᲹ2BCR}6G2b/9:=mgpj͈Un}L>]Kҧky'3`&_ȉ ؚ f~mEu=d-Cxf,fa`ktˑ>{e#0 |=8 b4A0rϿKt|!K\Q&H?ⱚbܤE8 bM[ش35ת]欅y =k&+_i~X-иDXQ`cےX-Olx& &3!=q@}b_c1k9Q]?'6"3JLiZj%g6:_YbqK8&e5Z 禙lGXD+Vb∍G߁?$seliXcjWb2]>$]J+Nee| lA7~~^WީZL[r >n u;hc+8xə,[ɩu̘Ӭr3' /'y-x3R&vK%+0H?zns%j:[ߚH#4`oLcWY){غ~%+C|̄ 5#-ޯVP|Ԛgm1EZܸ79GlxuKLg40Y1i4!M4C ǟlR(宪% 4[c}55 %n8F^q\3^=B(2idqL`WEstƘ,{c7#-z MoMeǜOFxCo3x>aDݬw:FT87=y1 :u]h6ah'tNd÷1hL@0'pYWJe瞚 πL-g=.I11s5{9lcf5Y6oζ\0҅S 6yAL~Gu86o}nQ,FD;B,|p#&L:oAS{T~lsb{Ɉ'mN'tɉ )p 'n2-}*f9f6#ޭq>o ofok&ga&,wu8?5 q3{?@sf=ďȈs?5{3{p,@ĆgOj?h8hۂ'ҒFIYI)vkr^(py}?>?N;gC3-Q rl_f˙5eYA[e8Q\F]K+Y Kmq䓴v^sA7T6N$jr֍'=;|:Պ9u"` IDAT{;wcvD>#$oOlؒW ䷤K8IwD;y1O(ÌB]#u >^0GV/,pg?emK( ,Y}r2[xq>CԄ1,Kמ3;c'Hv}oXE9KTLm` `%:P/ms2W>@zil%֟Vwf%>6ihʧWbo{CLl΄wb6ֶz 2W93t /< s߭q#c3_ynh޳oNcÉ}%wYӕ0g_#5g_kVkӮ)@II1NT50aQNΙFy99yr&[X#]Ģry܁Ξ&^;]xקѩG{ g|~WÖOVI'ma? :gLi9:VL^M6\b ïeQ|f9` 栮L0r/fO1yp{ɊFQ&'Sؿ#&iگ=B,عy5?Mmyh63|mobSuFS.;?MiSߏX$m[k {+LGoL1 4JΦZ"Mdd\?{`{~Ʒ؟݈`^fN$K\FZNi;(𰞄8lg$o[o%l~NMH߼{147!a=16N$rxּewS4㹁=Y.gNd=w/D=l۞ \nkkC󣴴Q6͍:POf6},7j_=gWSPBF73x.}'EAYQ.ϪKDDDZ]ڮƘ*//F.)СEg϶j(ꬹ:{FRh5\ oß7fYik+oUY,TTVV^ OOO<=-G3'7e:.+rr졼FkcLb2ӌg͒2nTW94(//f߽!~<$LgmV,vhfn1p,snůUgm]gx 'Z]﮳RDDDDDZw&^2.fr__ui7/G7..&*.gm*,sTUj-aaOxX8vΒNHkS_NUNv13uUbN]"""""rqy0 G?L^HLߎOFF[?LaQpZmc\뇶Jmm] NtVް{xyySaT|cS8s&.]Ջx(g;oֽjG m]zD.[rJH kw|0u}xyyq* 룇)**p)**d7|!cќL!ًR)TTUss@5^L/Pk\e>]b?!((Si]f:>ºGwwLS\-N6oLE4z?zm/m1uɬaaSaTuFuf* " k|.hjc[XzB|I|Q\Qv{hԾw6uYrI0 w.rxJ=Kgw0gX4h6d6X3&5A pS V2ʪ\Lmm{v+z"3qgdWTbkPZN֑\và`2Vm>By[W>1g[cJ""""~$fdq*}x7ߖ q?FZ{N6>t=0 ⊶m[R̾_f{7~0фzP^pC;-t4tN;!TN֗yhBޔg͛8Wsb zCT}>9wkl#hOw_8{ >>Gގ1 |Ovֵ㠇Vl'<c/l= x3C={q%~"""""MuL Y * "qOq(˛pF:sÌr*ׯ8ы+ ~p(v1WX?f9=Mgxc<9y}_ޛ0/@U9r m8R9 ޔclH>UYͮ<;)BޱW_}?1 >{U7DŽ?WbjhFw7=LgLX2. B9zx%yu*)?3/m"=vg򏍜ȝR?)Âv^jN;3[# dV#kvw0su\˂)8'0Z 2NMxc3Gg=?&n`|]ol3u} ׳cJ 74ѻgG[9CXsI=IjhI\i&c oA% 099٤g|諺1o{iS k׽IE]9z$cnC7cwLm9KU9y9m1At@z2сQ@?4!C~k~tcKƭ_wxy,Hp@1iꊝx ;|[ss?UM%@s(5CWApn6Eo/+w;{91߭/4?/I̻1~Aߓ'I;yC_+""""t.22tLEEy'ɓWpa!A^.G؞[RqJݚgWyw^Zْ?㭇;PqOxܘe-P@{o;󂂍bu vGow D ظ]^3+H{ҮUNsr/r0|2w'NV8wo[do"p(. [erN%Ⱦauc]˓cu#o!>P^bͼTtnJŏ+(0̾x7^[GO&|=vىW-r7(}G|R43BT a 2 ~Q+$"""""W򊈈H{+""""""'SiS """"""ҦLEDDDDDM)HR06`*""""""mJTDDDDDDڔ)SiS """"""ҦLEDDDDDM)HR06`*""""""mJTDDDDDDڔ)SiS """"""ҦLEDDDDDM)HR06uՂiϨ}N'""""""U hh\SH;qU[5oVۿ}TDDDDDDx j[H p! 6 :wwS[WED䪨rPXxҒL뷖ڽpӾ}k [ׄbTTuUDD  """h. ^^t҅A@)u!С۷?SǙ3dɦ¢9srWUnR(QQ^""""$. /zmkiz߿#` 39٬]Q-i\ޕ7T9MyE9i^XT /%.] 7WWHDDDDDDi.uo @`?ʱG/ edqQYzsry{ݛ. u˛G.[˛SKPp](1s:M """""". pq8yF ..A?P*"""""rsi0g8- g/_z """"""/Suku* ]FDDDDDDpr1^^ufd[GխWƮ=_bJԹ̼s0Qf4>~Wo'3<+d?eCki*ſS,knOEE9Vk coHQjLLt7q*zb6{QVZr?+]_dXw'^~#=i&S9i0z.B9@VEKNDaODDDwnnn@+tvT'q?ELuꊴ'^~kIokۜRS8\hf~DVӌx9'"""""ҕ֡C_kƚڽ e}ki@V:ddĬ_r񴑕 /` Yf&plH1WVDff:]q%عz Ͼz(r6#b.Jf߶?=@@aM=I>6DDDDDD.49@߅[XTlf ;Hf릃<>?H!.gnT"_vaț'`r _Yg0 Hp%Z^oG rb J)N8p0c £"1z=q6mv&W$D}ikY$,g8u/Զ|G)^2IpI'&`Vg,6yKpKx` Co!vxo>‚'Յӧ'a{{6,"""ƽ+ @ 9x*Yt6euOnOtA mkoO 1pfՏX#{a-OfVrL}3QjyEm'ֲ`d(H+S0dNrK~`?f6-lr~wj1w $$,~҅Z7,!L\жHh?Pc΃?R) OJHL6,W.s#"""""I ׄL6,yZ9k)?y ]|l|^<`e¯^U/$͈""""""-`*wKjjw򊈈HR06`*""""""mJTDDDDDDڔi+r`jj\5f/oWQDDDfmegl U*mjH;`JKK(--ij\ԕWDDDDDDڔ)SiS """"""ҦLEDDDDDM)HR06`*""""""mJTDDDDDDڔG[WZ9( """"".4j?Fيۺ """"""}]LN|=##}EDDDDD%cL/4νޔ}EDDDDDϥ]y?m /j""""""ݥYyEDDDDDM)HR06`*""""""mʥ5e"Mt$""""""ӌSi~)g,?Xc"0A8qܥe\e㱲L,ͥKـ IDAT̴G̕HӴrkHTPQ@pX?#((p@˗r}܊7s-We4k7exQjHDDDDDt;nd3ط&QlMEjNGUl ID8aT .T{3]"QnaDzwguy7pľFٟŔ˽Į5{w].o]DDDDDnB?+[o*rC(urv۽9Ɓ!֬K0uf p`xˣ9ryЅW7 1y֡k]h1uɀ=^ęLϖ,!͍eѡ/Q+wAͩg|bX-oY;zvjXͯXv]LZ߮<=֢2czNopg.CԴ]ׯ!:]NsɸvBhV] '\]^$LamXYv CdѦծeæ$[MHl]['D-Z[&H_j7pL?kϐ{Q/P+rjVN ٷt/;d{;riC5Ԫj4;zkoIDDDDQi19L\BB3זxyu,el 'lr.VWYfgG>ksdR^v`_65o~bzAp9Ｕ)k$b1$0%\;k© v M\:彞KI>v/Xĥج_D)\1JDDDD(ZѩxUIqZ y/_BWϛ㵫@FBMež ی!#wX^+q⎲/7!j"WWǛy۳+P{@O?uis!%l3urlr\I8F d\Wr$HN1a7ncR \e9LW߽ܵEDDDD$ 嵢mNg=zPU 7 ޵]8CΝ>ʱSu!qL^L}恑]T1a$>D[ڷ(=`lؚFUorryaB;HN^bx;w͛\YU{\ѽ)l*Yh,﨎 LWNs;<M:4փpSZ?%=5_AUm O 冻րyui/hP=1.["""""*?yL=lH1|kV၆Ձԅ~ޥŚCeڽ݊WkHN _DԺ """""rP0-&N~vC JMy1\ɯ9c*"""""(J馡""""r7O:""""""bU E -l4hZr`cCDDDD4S0- gWU8ll- gnU)*)[6vni)Z\=oXiz(^DDDDDD$+""""""V`*""""""V`*""""""V`*""""""V`*""""""V`*""""""V`*""""""VUVZwwTDDs>""""""RJ0S.0/>޾@zH]~511х )Whޚ.vHvmRn,?bc/b,WmlKs511H 4|4>Un%( W\f-sLCBҥY9_peН=V-HT ;EFf6?2,ۗrȎ[3x ud^YcKX顳Z㑧# 9] .) $i/Ž6ouT uWЩc8::[ NtX:DDDDDD(11عZ1t[&[giٶV%nu񊈈#e<p wr;]7-ğGVoLh4TDJ]m-4o֊ǻ=IhBBy:>xW!0e=NEDDDx-~/SAZiaT{oѯ%S8_DJ_K|- ;B]py˟cǰn*pޕ}\ooN̙Ӝ>Y$OV{_6GrrRvǜ i׾"""bckt5JӶ+Uu l<̢_ZL׌FHV(\D`E<<Ɯ&22h-tlgr ˨%GȌRMb |9|Hͨ_/Çqq+ӴIsfߊ-˾[?`cU(Sƙ>Z$[&޽Z)""R3~9@8sb'cqTCSwd{#Yz>4IM޴ Z̶7J*Pz-UfَԬbf#my[ɹD$+ݗ+yqrE՟FV>48g-h&(1|zJSBiV5be<4T+`*D#aߘy$\@准0f fXq(| CvAx#ټ`> W#O fg9Ҝ^O p,3mQkwem-_U6n\OgrxX t qu-1|,X8ǻ=Sz遽=]N)S!9sΝw&NjPn}"#px:tLN]HNNf,@ 2UV XpًO>eI)"R|oBط;KJǛ-Y.ͣo n~ۓC>bsz @|p!-fxcIgXF!]æIKgXbl>buѵ@攦һu9j!ÇYJ^HIIu҃iNf xWiԨ)0/ $&:O|sg5;vtnDEc7>}zS:t* ?=;|_f~)}H׮=0L:r1|˖cÆDӦyoh^xy\̔xK6y^7m ʼna۾""wluC:ܼ#Yz vMc>co,8Q-үK>^?Ef_ʋ*P \aԒI_hW<^u ER0-NY8BZbaCo.q/I̿hGX8Iy΄dX =pj~|_ ՋrڧM<ʕ}8ѣ)pCpʕ6{пl߾k].]DH_\r,:uO&rAΜ9͌ϦhNZٽg{dO?ё%.e}z['ؾm ~sn35鿒7 /3 >ı,Z5FGƸFs!Ξ=ʹO'LÃgϞCpTt/6`vsbc/ТEk|||o֯a ýz3G6mS^>Uxs;dٳ7Oӳ:>FlE""w?{ϻcg`YL9?Wo$N̠" Ϝgs͋!Ќo_>qeRKDQFGVUbcEPXeǹ󝽮h6^}o>3rOEwwLwT/l06 !gtw f-؛=Ln9uKvҥ?ˁ,e 0:Q3v1YaiР!3ǻw1g",^jdȐT^˗/˦ |<-GKY?3{!_Zf-><.]Zٯ ڷ넳 ǎ˯fXzvx7hp[s΢_,8::WhAߡ|9qZr*\;\-~h/ӴRWX;B͙[9qѻ>˳ޜEOS0̓i޼+cY}̙!)A8M#-%/O>RN}8z NozjU0)YǴܱCxEDD _bFi+logl+S{G~o6m"5)} 36$ROLEDDD~D#p 8$֦ػxj&9r%g=w_DJSѿ1jfH)UyEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪL@JJPHA+ϖEdR0-IIfk7hֈHIDjj B6 IDATs5-D?WHMKv3Dh"JBB<t˳JII&)Lj恭Z&""%Ejj^͡=s}ƛpm"->RR7Y"RHJ@jj* nF@=sUD5LEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪ,z/~^NDDDDDDJz!*ER""""""Rj0utt"VmUa);vCILL(̗Ђ'zGGl>޾xҢyk~ 11х) %:::YBihI>UԳ̚zNEDDDDDJBYE֖PDF[gdd8K-"tN4 j\MP'9wlǷ0 """"""%Dx##ozNR+"""""R>ӼR+SP0++bZ4omk7kŎ[ )""""""Rhiiެm]5ЊHQ=uX7ț/zZj[ң@{L)[}e[ׂl>n}nHFʵ""""""Rrhe}ݩck!t8֯n=p$_2) e(o{V>޾ y~<11{ EDDDDD(`Ѣyk<<<-ewl!&&0_ZDDDDDDJB DW_FDDDDDDJ]HDDDDDD$LEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪLEDDDDDĪ݀³ 0 v |/͢\ϰG}>L18?x*=_ЄN{6E)PDDDDD(W)\>hЇXf)I7b ht^fgX[w只Q~, vHS0-b9a9,_{ahևNar9S;=.W^کun1|VDcu5=dPzjɚ +#"""""31^dž͸TOռ!SɁS)d[ TU@ԭfsLjS)d Eʋ}n\(dp2ecj>=9b1E)}LT,;}onrP4/Շ0w:ɣvxaνnd׌3i.b;F05L}ccމD&~-xVlk1IL4yu #3N03u&cTm~06i 'n >>c3ۋ*ycPÿy3DDDV($fNr5ܩ 9S'h԰ 6TI>r:;uV߻AJ9ȾyRU:i&-`ᣄ:13~ǭ34`J4]z({Vg/q04[ mfLᝬ:ˣѳ_z%Xϒ]G8Y'Wte%oTI~߲209VnșXn3+Oa`n4Y;U 6֎k12^e,Z1tTfU,ێCvd')]c$񷘾h=klf) mQ)u3z*V%3̾CTӇ[ϢnҚ=¢*U _̲7vrփc=v~5;WqLY3N却y3E=;0DzXWJ%Cp n4Df_fo T+tCc4^Jfu)}Hz~~EDD%Ӓօ#Kc:w&/%KvD ͩS'f~}sSNR+v!H͈u yyGx!461te)`>ن6bʡ,3?Cύؑ'}Geє\:s,ƍƵ+v`exu~58uԍfZz>8 NexcN0^On/|ЀXA}y'3!TPctw&ғx$u† ~86l`lUı=2h8zd+]yz$ g!voK&6Oȣ>b #Y.^/[ǯ:^vf;!*T9E]E*q&L)5Q{o *fFJ/#"M7eJ׆ W\)* Ņ-p sE.N糐 i"wЃmadQ"CXŷԦ_1s0d6cNy),b aW̘dތomԛcݤa1'K\ԯz]ɩ,]u:=DwKGٳ|A׎/-  ׮H0c@27mGeܨk4jpr> 0c7!KD |DDDp) KtGB>8]RfQj1'9zl' o!jg*R| =ms(`!ĹYo{ـm|˘%,KU ?Ie5QJ0(dE8Uٿ*~TsdQeVW^ |i>#سDe ߱P6}rU? aϺPNB+Qe1<6ӧ0^3oLc,[3Pal6]0qՔlp-v0`Llm *Ƶ<ǒ1D`pĜ%V 3˰a)J AR0wE-^ٖ&M[e jѭs&|p7tkkf{z鿪݇SQ)BCCxu|fٻן[VVh 0d*Ujc rr*uyDq?9Ldi_3hf4LU-tە=z2f %"H9Iy*~x\ BצnOI^m>b 1 '0ETRe$칌0F y>b̾2c}4mQBe9_rcLjz MN"Ph?[ _C!~0e~ZRf599O N?Z.?SD fKSYp¢+PnŎ ԫ[s)"Rv҄Wf-ԃM+A Z6Hd'o0s m &4MF)*A el@9"L7%?W9Evi ͙[̝؋*dpv'{.IJ?^2O@Af1pb̯~g26\}o_[ןlOV3\+rXq m?HX92!#.LeUyU6>q/2kQOˏa̧8w۶#J& 7SukLW)  ߉2uͤ6'g2iNb5ÃqdMAةX;}:9}ilp0aZUcs53c<G2ճ|(l&}HsA8l4Q) [ό;,_7)&<=xdNQ5z/.p3DDDH(Hp.@7j ?Xfy4S^oOSe,>)bg"lL|1=ٕ|yDCBNNe׋i]È-6b54UDJx񊈈ɵ6I4hڴcGV6a8v~{]%9i6js[ԭk<==hۦ 3<==֥{>+""""""wܷIyeiA/˦ 4m{?%?дIs^}ymi5;xy[{2Ci\|%xnڼ8qq&6mސX\|miϫ/w!""""""%V=)XbvMҷI|]bd}a9/Ϧ8V>FGڳkriZZ669IWKKĸX*b>+HDUOb\!"Rrp8y2 gMDGG3'K g׆zz_MR©H>tޒ!'8YU""E/1. B4.\Of <Ҧ=]D^8y8..FzHT$lllpp0L˔%66D+LDh9::`*"w{LS|K0imti}=66a֪CӦ-,sP3-^>?i@\_6|{S(=9}(HQ""r{i\ڵ kYȔ1t׮L4C@ȡ,+DD&M]+"""""R9.56qW]s VO.=xM{,]pe*""""""9+9URgg#'pq1ZBLWgg#8F#ULEDDDDDJ9/x OO/ZnCÆMirmY#Z'`: ;Tz7}f._Iݚ {7u d/j33Xn5k׭/g0PEN}|PV"ySOScggGӦ-s4oު곆wj?hذ M)W̉ǩV#"R)^M'S@VA] ĝsgs}g1{ad#2!HEj6&sEG1xg!'_ab( ]Oܳ`+Lb (3ǟ?W8`{~>SW?RJJ&fSKf$< ̙9}={vjw„oq]Ϗ^dדz//oƍ@Hh$''QWW4~?;ػwA9sCxv睱>^c:Cc@X8 CލRuf}fF ߣ'N!,,NfF;wmfMtwС0,Zv?#hhh`͚444\1nɓK?z6|&7 1q])**`sF*++ڍȨQcݽϜ"9ycC ##M[^^^LtAA!?_W_ms\PbfeeK}C=ѭaÆ3ftǎ% ȈhPiSg0 ,]wP]]=""re+'d#Cس;])`ث%K9_,ZBNVtL=-t@]1!(o7{*p'&sYNA G(;k( 00>-#1~#=Hf7_VZ ֒`w>>}ilbbժey ŇA"ILŽ[Y8q|ssPwY3%2inK9Eʁ},\r 3gn._32eʴV r77w34%..]Z),*h5l'Y|'ǸYf%>^̹s<+7n嚕,=˘6\5d"ʸkh|s3ϟn5`KI15 77F~~^w FQQIב~/׬p4{yy10;}8q"klk>v4fz%c!s1@d ۄBM5X\c#p%<&;8/g@Tl8qSgX2>xwQ\hނtO*b]9BN񼬬F?_LQxܖ{>\mϬ WAJoO7~ƬXU~*&N zUUUl߱)):t'2ٷoUm ,t)ڔfLw8w8p;hllz35TTS^^Ɖ ͝;ꫵ>}Rj Fci2ybV7?|($;wÇv;cձ{j9u V^o)vp\n%e奔v*۷S 9|1O8>|s)(rsmN\JZ˩SETTiW8"l7έ[7ⳏ9s4U:B_?ƦaܹJMK2+IIk)//Sܹ '֯aãvZNNNrх^zQTTfsT...xyybTBcc#gNj׷/TVVPSODD:7灌u߻2t~x]8vmWZqo`0cm H=ͼә sH9ϔˎbFye5\^ 6j͘P6򊦺QЇgPx5Mef?# eX4%@3ƪ4?է͝œ4Ύd$-6QL}<ݙk+Cw;2`@8ǎ톈9rٸq}2BCxǠm'CsOH]]Ϝ"4tٸ"%e/,..M22DG/onzGCcg^m)+/Rsm1e4Ʒx(BvNV8_{^z]f|||x䄧gϥ#c$?0L'1q2}R^^FĠ(vz]חJJZ/w)HҧO!..<"K})piȧRD<ŋR^A?g~UYdD28~dh z "rmN`d@1]KϷy+dfɱ+y _<ļ Y~NDU>Y?g3n7oeڰz'Qom'/Mft>?^y<6E,-gSR:Ө$X6b4\dzrR6g|̛NidjH!AY=ʼG&պK وnêS?W~Úl逰p&į^{jll$i:oS?e/GlE9ل \jkSZzA`Օ8UUdeC F$--8v[m[?,p\Slذ£gO8q2Zvw]]-yy9DDDsGdjI0 Q[{9yv;NX.6WWWytrV~[p;EXY fzS˹  _^Z_ZRSp*i0|(I8nDGRSSƷ3}~塞 1iNtss)ٱs+=z\d3ӿ&]3ڔ<ĉl{>F#C #|@RAA2?xz&>~}7omO}}=wmgQhMxDF=N>|)*)ۖ昊`mĸ5;Srª/IeeE9z#WQU;aضms{of{:TDs9֮ø!mbG"""rsP^QJLEDDDDDG)1TDDDDDDznpEڴЈ[D#"3ihg iUn`0]qrrPDnmXs __3~DHCDK)1ajj0DDDDi((%""""""ңHnbG"_ŏ.aADsX{:.Ĵ()0]$^^>Em@ۃ;-[Ӹ!Ν[I=v;{PR*ʹc0mVDDשiĠ(掷7!!ax{4י6ƌOAAuKJ,dfwf8""""""rt惏\U%;vl`x{Ӧd(]ۀ'yX,eǎ`6;cF'tE""""""rtέ[,ŭU%""""""/ͤTDDDDDDzSQSsg11q?;ÑȍF h4k`;%V/x*f\Cgv{xm93nؽ=hL&ydߞ6-[7 g#aͺ|dQr#mFt=#طy9?7ľ{nXJeŅ\msxSq;;L(ط \]&b6cט5+BZ忈s|~:/gάx6yDߋpsϗ FO$2^_'yWgiGÅSX=[ʕJ8lԪtp)1aM|SL4I#)x6S78*ٙ 띓g^T{h=vs-x41 jJ)8ͻiɘ {X%k[tǿ3JP6<<$PWy!.=ʖd{1sy|@>I%ɩKv'k`0>z 1 v~ 80QAx;QY|[7ZҶ蠬8{7'VFs 9BMHpRWL֮/IJl1Cu8;_vf ^J$h3Bʛz!pfs"[+נk6Q'aSR:Eyv >fL@ G(;k( 0Of>-o/&!!Vc۱VSQrIl`rs[M޲-Zm!6fpwvM|#"qw-c ?>PULaŝ,XJ|0vImbIL0ø&u؛]IbAcQyjo};7u2TTb9u{1GkAÈu`]䔔Rv87,hC8.rۯ}i}u&{c;p6ڭkKuY*V>EYð:R֯'|R7'VI p8i=K)+p*;7,_a{$1RsG>vˑU~ ,e>'*/-j䃷{eXTǗƺq+""rw{ o/|XGնΜ^>+:; Fye+G IDAT5(Wͧz`v>Xs+zΗRTS \ձGvdp? }=筗j- y\p.~죠Csٜ>7_78E'qa8mH8@AL"@ )=NZa}s w3N.X6~9WUɿVhJHݶ&3kx:d)dl.irx*_ k֮ngzy™0~zAɁXZzBC-z&vkPv IxpC g|.&{٨ J\+y.q7tGE"AB 8GF5_"""7.dÿ;)q ÿcb޸K7yv8Y5@.d{&/L=̦9EkyS^ŗ9?ʆaKwx*k׭Ͻm9{VG͊l^>X+*eu3YYR?C B ;bw>x`e>&}ƒ/R:A^ఔRp*+xT"u4}w n--h $Ѝ2K5JYY^Aa9( Q=rŇl>M읡zIMvղ)\is)vաobŬ~u6/\?7- m1wۥKanw66-M =={-obmakϰŦKq?p3%xy? ™8ak׭wvGpD`b"11sg,ٔO]|ӆS+J o6sSHt>,nynKc|j8|BDZӧ<ߙBGI6tPJ);؏rrS]VH`;}sGHS)KEĀSgC݅)Sy) 5fx%W N`Zp7Lza0[J\|ce*^WL?TN9 h^{J[(50~DfK}׳rK~SRT֕_☘ MTPG֍NKd QMJqʾuёg9Z')xrߚP˹l~D)qÏKb/)&eM]* |q݈k#UNX˜(6PgUn,"""""r;@ 1Dhll.75 ?)9DqTT&3"+#ؗ4YkUo{Ѯ}9i?ޠ#,;ՙ:GCcxit NN`22=Ҫn B{%ix:y%""""""Nےz|9hN8^\VqƅәZ;+"""""3}(oKV'rf2Ѐɷ 6UNX\("""""".;[g5Pml +Sw(%""""""ңבx#_~vB{<ēLl{\F~{obloh6c]-nĴy2:q&v:j㎱Ӹùc9JL[IOYDf|T͘Ӹ ;k~{#8³{rAo?L]vTkĴEON |`G} r=,IV_ņpFLEDDDDi(owqBG!N [Y.4(77v|n;[u+ѰдZ8WDDDDDD:֩3v AHHaX0̈́7o/Z^!#""""""N_1`έdf3m0\º_TD&qqqߗ܈QO]]-55VlwOiH꒥,b>\0 kNP-%ak#r>>xxxPcRQQnoï`0`4A߾PQQIHw}BDuYJ>E1 FKtt,febҎknCCuuuQU]/~~fJK-]q" B -'DDn z{t^h%גe,|iF_O}ż;<>Ub3gK ˟,`Vn @_ ()՛'|)ggg8'|>}|;()`04k2e,僘/n2wW7m'| " Ѵ~ij}>M5Ϝ-`M_wf/`s x}uU;\777'LYMvIOóe|nֺOܔ0%^~%^^SxבTZ9ĶX;1ʎDMf"׫ieeM(X?_ |YVVnNlK!Dz\4p01Hc&c&J}KDԻ`=ʬN8N>:ȍb1??E`˓,? G?d>#țS'n%#/*mDJ}QЯȨQcؽ{7:f}S,rb=z燈hRE8 QJj0Ƴ-y3o5?\HODc-`oζх{.w֫o/Gߎg;G9ICi My22Ą6-_M f޳O0)kns][ 3Wp>4ԕDNM+`f;a+:s\{l:qΦa*ğ+8(,lӰ;dv%g-$%?1Na}BDfQ(& ƚyzwE`ٻmDERvƞw~t; ?ro<Üxwo<ۗIedP{Z>Qf3y.< x+qs?y܏/^bW|`3Qomre1j.?^AA&Hr;rqqnDWo qvLtBR1#"r* <;SUU ,~5)sVR퐺r 2 L`SQVޜ^J%LnZ ;#"G&6b@+{l:fey=<07 -]øߏfd >h(ddV':\ w1Qww=T`K5QD.vL'un6[Gl)C5s.]MZѱ44ECM LB[NrGy?q =%yҡSL-e ].&f55_;啊;s68q'u4L8!$ Dz9ZBh7j<ϡ#I? px|;{ivSʾMH= ћ6PZrS0ߗܡ<H0!*3(B}fsNǪUj+usUsʁuѮv%Ѵxh6ɩj0f˻#AhFWT'XFtI9`=CmtPk:h|kLYMU5&bx$$PT,:h~]lq~ӡ62885Y]COJJsqWVL$Q[ROqUT)5xt*Ng=с-t9r(t:mmԴoЀr7iJVWYƋ=sL %c~^Zw;r(+k^`:k1ʫjr}}o7CkFj68)C/deFj/nlC9bv{DZ&%IDAT15N!x%i"H=Y3K4M`ل5niIsdrПX`&e2`̫,'¦U53Uve𽿺hbY SX ~z|D&WTVa&ɕrI# 5SQg$1){e*jbUka1 'SX839胩=Ki^"cٳ,|El& nݽƲ3摶2`-ICc_),\ԤxS7*wAg_U aRL8=Y,d/o}_ٱ:)Jדt6[ sDZzPKo6N!8%CCGس{{ve1Ep r9Zi"ߺړxj%>!y[Fjh!|\~pssgi}ERׅcВֲ}Z luONM㧿$=n>'!z0( AA!}΍oZƷ).|ƂC0( ^hΟvQRY5Tw%zQOe=ʩ;*e)3W8̴oJjƯX/'^yCs!^av#RzNxoaT >Ip#=RmٻM 88C~[hn!>6_! Ľu]*a=ɩ^׳%ʲ)T2j+DB1>M$sdEX,5b02f+v-g4 ̋+wč|/]^iEkk|gL44Hx,y||y`X0K%=IzTB7Cǂçw~2#pePTu܅N$!]iZ!Lir>- ne.iaa tvvpsB'?-2 !2"Np*3eTHkn#44448)..Ґ !oGB MFhZ<JJEtvvI Bj@Q^o@= B+' bBsz'f`BBzwP褭V NT~߶8!cOS!Ą(B%qB!Ɩ+B!bLIb*B!bLa_(IENDB`incus-7.0.0/doc/images/grafana_dashboard_id.png000066400000000000000000001227101517523235500214540ustar00rootroot00000000000000PNG  IHDRC&sBIT|dtEXtSoftwaregnome-screenshot>-tEXtCreation TimeMon 09 Oct 2023 11:38:02 AM EDT/ IDATxw|UϹ7B$$ {m]mj_mֺ몊 BBdwr g]2 ]#`5.ajgH Ϭ/!0 nm?22Zϲ%YdK-7_Cpl83e13֦ >Եo/ЦXCgȣo~}Ϙ7ȋ Ҷ9Fr0ܷv\1!ͽqnua..׫}p=?(3z/O ]F0sK6/~.|YvS,g`o∈Mjt=1k($3x??v\|_Rvr73FI,=J 8瑷2; K3-xwMo?.e_>o:Ǩn8p`0EgҬs'>n_p  O${fA;־ ފ\yezIGM^}P(Kդ΄X!F~~ _2`?~]Kg/&L#q`07rSnۧq Fr yYyE7ۦ# A [':  #ӭd,EFn!(aN^ ˳xGe@Wu |yxxonWϾ5viKNj]s/1m``W2ݧJX;d. L ?no69v;Y'7p6&Y#q7* fM|^׏?^| sϜ;X(5Kp3k)&̢"*#ޥ$:Dr~9/]CMU%MRys[E]V{g[IZW؛Mm ψ51$^,*s˔Yf0)J[K'nه/]Ymx7?9SM[d p%&9JHo &LjJTo )eBB $~ŀ$Tҡ`R7ޮ`Ŗ@3H=8Qu l6Fn=*Z@muśQa(`iq|K;8VR^Y{GB3dna[\V * %{l EDDDg~E8 . œ;@eee6&5׼6IY,ޓy~0< yϹ`R| 28PNQv{e%gֱ`NJ 8t";~!y~\󽬩2 <ˬnV 51];1ruC`G:wimmTXٹUa'?;34K(*z9~"\1Iq2.]Y&&DNNvcn8܄&%EؑЀLb{N,Dl.4ؚEۨPUӎݬ b`@qٳȮ7v՞EztB1nħP<Y.=kc\ PEypeû2ytҲm)JȾo@{ũlj9t[nzx#zFEEoGPH.%0.2Úz,>Er8}!2~Axy۞nG!iK)0/*EDDD.iέᣯQI8/O.=3&EH4jܷ>a_(feՍ֔+pL^D f;V:vHWK9xhs?OsY3|SW'.(աL5R+^4hQ""""N^{HG MEECӐ2gB)p^g\gN2w/-Ϭ3hُ̿Q}tO"*ړsvت_޸e\Vp;STTxհIh'#}=ں`խ`mK`>6,}ʃzciztϪ{χ=ɎIo1[xo BvM4W'MXh4{XuKeޑcu[؁@|f%9e @!g5(,v^Ll*`sH̺/AY!Vntq:%I_WkJEvr8oe|L;u./:thYTm 0-dcJ1pHG>9Йn8bdd[Sٲ&B:,8A3k%D;/k N^zpa(=+ _9Q W}MJh)̠,wK vc?źXPkOӮM<$!^[P:~6p;;F1slxo tc;Y{;Mx?̣ F3%4{YAԧ+=֟?an1>}$6KֿN F:OeXK4O{dϱ -$ s?7toc*9y4 ̍!i?¢{ə;C+z<ӻv"G6]rCP|/xY0e2VՎ.Çd-c/Χȱ?#(H))cmm$IoW+Yw(ȱl^ʶյCqx\&Nbͮ9g(|׳"F!%FFPNU鍻>sڟ3]#Y&=ߧ4>~.O.ĝ%;; g)HII6^p+Q?WF{H[i-u1<nY{겂9&9į-sߜW1 591lE0 }+TL̚rӏ}>vhR ^ y o-g${%oO5o k_+1qnbؓY&ovtNʬK-_;Ma t!7v;1+yiB팙s| f³;; gg/++--#0(ubÂ|sWvDDDDD~`n6vhS#d-R3۷G~ HEDDDDD*++߽iȮ&ߤ!""""""r(HP V@*""""""BTDDDDDDZ Ri """"""*HEDDDDDU(Hph2!ju0[""""""NEDDDDDZ2^UbQ aX087h*""""""8j} """"""]p0 8D/ U? p8cü{kDDDDDDD .w{a"PthjnSaUDDDDDDjhȮ Ri """"""*.C6mbQ>SZZr?@EA~56[kEDDDDDD`}@nX, """"""?56[F`\_HEDDDDDU(HP V@*""""""BTDDDDDDZkH===kH'NĉӮ.DDDDDDi@@ .}₫K#k y^JEDDDDDDbO?STDDDDDDr1ѣ5}yE̟w/ϫ-\.+Ƶ(@'~w̻^233]FDDDDDDԨO>ʊڬ\.{nc>C FEDDDDDYZ,fff\Yy©4iC֯_u-7/""""""?`4\͋XMj$""""""r9HEDDDDDU(HP V@*""""""BTDDDDDDZ ҋXC,Yҏ5}S\.kh=˼N.Ǿf|MW Iz}g9x ,""""ׂ\5c//\ϲo%xiRi """"""*HEDDDDDU聿Ǿ7= sw11 ^BV~6,E{ө2& Io[,9$Gocd1fSz;odHD0^&i%~/,fsءDՠ0lZ5뢲n`5ױ/#qz7r-͝qF 'qn&z5szhj.^t;7MNϰx8)ˎ/x62 GL6Q{ҩ/m祓p|[V,b}LnernπI4b(pPUɩ]_kc 7wvng0G>n҉?=6zk}̝:n=q)!'5[xNW6]7p K{_ܬUg'qfV+\J͟zH#ϟ_U fLЍޮd|S8cv>+F v6dtȼ)Ͼ8C|`=.Co~FЧygh`vL v3ki(X}G/gfԟ#=GvU~-d@VOx$>ֺXd&f0a/~͜=0w}#ϛڂ#Ac?F' ߙ3j*cK^Y˅?G_z]Ϋ3GDHa5 3mgtn ^d#ވ F<<7=g6A`a2Woƶ"A^wͧ18!a":q&[^_ysuwv+am0mYDDDD We r#n+NGNe#1&ȋ?'̶%SKwlzzqÏb$wWZlxK{_DS;y t[a ,fd k\<ɳnaXH0?ƫrUUzc= hWŊwbáxrpe,_u;D Y|))س=MQ] |~"\8ϽW u&M"7|F2D8X# O U^NN&>m9/G3;T?8uէXgyk_޹ 9k_~53i3gü}q0pMSPR0*1j,-^f~?w ?oM8׃x};rG?\ ^]-nj -p^9R>@͟!L:ώ<kD&EblӼl._OR07̟:UBEaܛ-go2q.eW[D{#.H3߷{vlIblaۑgϦо<|Z*\5Xp[fؘq'wvx D IDATdˡ?̝eś?5/{,""""Mjt̒|ME?kRw7&f/,ͱr,~X~liyKbM,>ް\!̜a^0zv5ĤP!YdaaE6I屏ysiA,TMÝff1߼_xP\͖=aN@Ԝ":aԕ|ݷ‹(Yâ gTqjl/0pKX `ﾷ^=NxgS.H)3q]"frCw 3ΦKhSebxᦱΛsjHZk/ L;uE Ҏ@'f)euWdgPЯj[ e&`}lllqvb!akg- k}ΏX֬ņA>25' :-h׏]0 [Ofc]MqMt)M;뭺6-Zڴcqqi`2˚{~f62QIRlOφoB`'w6Rz뷐e@t<kt4 س7j_c%:V44 rWq'lkg^hG?c7=+h-$IM!>>{dmB2inYpo}Hڻ`aש {Z}(AS8~:yٮA! jߞ@C"4Vnݍl`ͱ;Q6  [vr qiT7,ʧ*7+VVk#5$ĝlۚc@B! 7:b".MYDlL!=pL\N+!ՍZp)~O%E O4tB`2mylY]"ȵC{K;<-p\+Fv5[>yM'?{Q>v *΁r ܏@wOiod{_{^ʜScT"_ƣ߈0vr ˛FU>%&Xiۮ-eԶIaN%&{yi^q5CK\H,F3e`i0hFori3(H=sCHG)$!1 2OCV#aa] 1~{g}LJwܭܲ$_{h*b]}]J˴\OH,~x51,`lŅL TM틷0(htw7⃟E*˙ɇ.,p}F:@UV>{]VJ _,f)|W3zl&G@E@؟rvf5 kWSop8Qix32ڿ8ҕ[[@/g`K7g=oYDDDD/9NYۿW?@AM`-7Yl*bz͡ᅷ^+pĤM@wOtRW}[iT :=rK<븂(to}O'G}9,ްļ*\|1xM͸K `Os _P<9tc[>ؗ<z_x^?g¢W/>r ؙw0 Y̡k 9-,`ߦㄏ}W}z-Ĥ{0naxYbYx{:),{~zOW$SXoAL5öagWW>ng(~_݌yufM{?x7w]4Yv_N/<njf]I^wQ |uf̝1;I.:ozWѽk' ~:jۓoo2'?o ;ʛ'gVW " d,˼ 1E_[.*_ueEի|,ѨHY{c^9 xv.=z_O^tvk%ckmg(~7/<9wftNTt+k ō#[6\,&fء ٠(=lY5k7NbR~#^Ew̹ѽ;\Wf^ 'άÉ 8n"{vW.O%0qLڗ0O܌rrSصs\]EpTN;ΖJsS?Mx^2'y!>A8KK;΁߲hN*P#Ow0ƛ0/A޸Y)9ʼnC[Xt)S4v_gˢ(lӓ1)9+S (."""r]R rnnTv Ԧ ּĦeShde}ݷF+|NacإƞO|;.'Yw=͆K *6V kE 1,͋j1"?RRdm7#&3Gٸb vO1([ybdW_Y_ˆlqg1 ^Hz.6o;Df%g1h~CT?dMD}Xw<'#'wֲTmnÙ=WpimBۿ7~Y|{R]9~"CڗoB9Xs(.|#FЧ?|r=SL v pbӣ|/kVaOlm"'3Sc3‚oHs)+6 7QcEl~D[|sm^ۉ-c>8#!fFiX]Qz aW)3MMdgϲ~,L4*B7v_8w9C m;oNxcK(4y&*.t!Yl_-(dGǑ_frX7.aƝĕ3rROjGQI]!(V`lj,slmguKt4Ɋ3fɇ3߄WI,cZ< 8E[ SXHXe6pà^!Nf-ٻ2fh:Ū%߲=Cе(1v!<>Nï؟Z$|@%9|r8}* PۖV/gˁ{gp_2Sx ,C}0bVb6N[2j=89vØ-~:s.܉(bҊkip{8 w\2y[{ w!s7,]PFL{ӕ`w rtKl`)1 -:ĉ܋=#n^Xj c3>M,kog+)[a(Jڏb^Z77E q:yuyϐjK138; rqXUlI>|"Js?MWkڮNRVKOx{s"yݶ cٶ+CGMlpԼRva1pMuDRʱ _0_d%d?H|E |Lg}RVYSq<ʺH- ?(QnQcu( [3 `/<ƞiTrQҌJN:LvE 9LjJǻ^[^Cӎ e^< ӈ޼# ql` /u+v$ ؒπA[9ㄳS ]K :oLv+ $8ځ f~.M$Mާ?N-uS@~16.=q4-WB*ʳ>@59FYu% 8Y腯\4dRr\1&5Cm&a |f2L]>6_?;I7,>Er8}!T_:.L7Ϥ =RөނC/(//|`9Z|}$ʂ{Drb,'Rzr.8KYT2啚d2?>թ߻f&9X(X=??oo< lHB絍__,ɨW42k^nMU+Re.\ۛvf gf3X,-4;_{>::AP|}ȉO9,O=EKO$^|wg83`g"AucK_χ_~XCkl/IYiɹ f%UVp[0}}? xpj/PA~ͼ+_v7wM|wIX.o1X;0 ?8gÝ3eO:GK>JAu(o =Ypo>H"(,А0ȴ|, ] pX%DhNsJč7Rk6t-P.w0 egI T\DF Md#;Nd8$Q%GYixY{xq^:Mc{[_O&69han{:,x^ooxe[m0'[@(AFT ';0kOa |CBp=up;C0:Uqxb6=Dlb 6smq.!-;6,E܏.m#Bz>@3B;Lr ?G rrv@ޞŬض L]ڿœj.[`W.Y)/\'m,؆dsAWd]>JcM{Z r+Ӕ5z3Y<;GEkp :SX\j՚0 .xvO ܌EDDDP Ll^2f"U8\)g@wFOon4k̊J@{֩%lKnx Ënأ!BK ?w@!w0 tkG[tٙvU4ȉ ;?'vl@I ;TzyL 72)Nj0r *܇N qtunzU{9ZՓSWP_&C^Kpef+OW:&n5N ,;2cDL}*f9yXYΙzGLbHy!V/Lf=@d)Lc dNmmMyl}49wMV}61ғ2PO憷Y3|"fd-JgNJ8+`ܘ,58w)Q^OS7p4CMeX+dF[]=M[ǡ3wr*3eDtuݏPl~-qch<>f'Y-W,""""!¿STL EEVBs,"KqiuMK(%t 5QXBTDDDDDD,@*""""""P K(%HEDDDDD """"""b RXBTDDDDDD,@*""""""P K(%HEDDDDD """"""b RXn4q]! 4ٹk;6IC޳www  @SS{Ȍ5 IDATӓPPOJ>}ںZL׏}&AիI; @PP= iG>]Jz1BC5b233FllW N@NtNйSQQ瑗 @dd#8ɓDGЩcgs(,<@.qu"Ӿ}2nk ciн[CHK;LEe={&0 ÇOpP $퇏/)Q__@~`=4660p@޳ !عk;i0t09sJ:tN霂o9U\\LfqDDZc 7.JDPTTx]ەe_툈u U( '7'e4eIa """"@yy9:Y]HvKi3[+ N^#""""_z+qETDDDDDD,@DDDDSo%"(:IDDDD V"XBIy""""rWH=zDDDD V"XBIX0Ki3[+ NbcZ]HJD\Q K(:˵6C@$ DDDD V"XBI.f[+ N:"ryE.GDUDR ??~g'..RZS uvDD.9P*"7:V" K0u3+ M@*"ro|.O?RR'.ADΆњs?>J1 4V"⊻H|}.AD თ}EDJD\XBɮ;.ADDDPo%"(:q8V """fWHl:"b%o|D}K84D..+qq7zDD V"XB}qR__gu "".ef+JD\Q ur`%玊ׅz+qESvEDDDDD N.ADDDPo%"(:٣%7\Mm WC^DD,V"⊮!\}M!nנ"GIqI%7[ R(**dOϭKD*D#]iuW"*&FDDDDDD&iu]C*""""""P u9DDDD V"hKi3[+ """"""b R'YٙV """fWH.""""׎z+qETDDDDDD,@{V """fWHX]HJD\Q K(:9x0DDDD V"ںZKi3[+ """"""b R'6C@DDDD V"4.AkVS~=Y#J1`g_"ظ*D䪩WHܵDDXO|g|('6?_YfM# ~g?=OX}8_ ܣoZ3;IlJfϲPb )_] W4BDD LfcsOue#/pD}CË޷ y@er|wv'y/|~瀉qenY|ekol)P\FEEDD,@ˋi[|Mk|ߧ 9@G6.%ks31!]CU^Y9gdt2(Lf/p@B3XbN׃u%!Kޣe<:bѽ3g4<8כ.]gonx5sljr}ww A4o/iU>#wMw%;ϗWp? 7{'u'/lѮmܷh*}c:PMo6o1fOnxxci6V6]3c©+:p>tb֒eL|ʖhܳu7lRNtzv݀_情GYx?{WoK+ыֱt(=hȕQo%"hʮZ]Hv?̛ZziO{{ec?V^ҚbвOI_~g4{|nD ~v{`54aF{?K1&upѳg̉(`?Ϯ$~)ɣ}˿1~%QN f-S1/<{^NB's /3/Owwj><|S?YaLȖym7OgF<^:3?}Go~337>^A fЕ O+l<ʁ::b {~n3v 9;c+adfb'1j̟ il:Š⢐gj8U}4ud-5^{t]}޻ \ի"hDR'Z]׆Dii ᆟ/FS#MNnn8?e^k̟cO-l';؃/\ԸxjekS-Mhn)k]8m׳=|hI'.21+yM |?^b30A^ʓ "WHkHEK1ˎQb#n~,Ґu삛͸OpC~=lDE9 Ω-a]`AD̳cdDoo,/h6k=8{yY{hwv $lgO}#g[x"MzŐLyݖBu?#}5^#z^>I#v݈ןprɷcVgUlȹKmIAN] #3,Y[c,ts]Si'++11 Ͽ(Y/hj4`gE}WVŎ ۩d7)O~Wvٓ| OH;rNdEI(mzLȵR')-- aV?\! ۏQlhy6a01SuR(j؏#L~=_>HȹoƬ8Ngծ_KQ̠p> ًOQPFΞASx%Lg4x>S'i4tθX=\[c63GOa|̿%dc4>Y)?Oa ۻUE8B3c(6>@cۺCvu؁ym{R6ey!W~]q\ˊ7=;>UA츻XT֎JD\Q u5A_"W̤dx.;Mf̜;!V_x̥wb|{_wViwc&y彽Ԙ@Ny {?Ku`=Q >;yx=}3#+I[>cI۝6>3?ʐL6o]kƛ~t~;f>}U ,luW̚^q&qC>b_;-u<3nϨY3jٹ,^ٜkV楟O0o7~WxyϞ};`1)*Nf>Dz,\)V".kq 6Ύu"8gCzY*ʿ8""r-f '7'et V """fWH涞EDDDʩWt \_f |*Y]p4B$&&DDDD V".ADDDPo%"(%HK?ju """"mz+qEIYiKi3[+ """"""b R'}z6C@DDDD V"XB}.ADDDPo%"(:ijj6C@*""""""P u2tpKi3[+ """"""b R'V """fWHۿDDDD V"XBԉ%Xs=xăwgd|Vv5ls= mg\e ,x`nw3)n 잸cx+ܜDCx=.G̓scV|9"mz+qko^o{edOG͇ИAt;S|^muqr1 kJ#.SukRz+qETD.d4s+ Qp3}r`؟;@&ʲYn7[ ]Gd|b$~ELΦ O-^KD1ddD5ٞC-6&dG6p]ILٛE9pɝjeY#w"0b_M`8JIUDFeNdOp*rv}S\^tNZA]trt V>+0$n%)xfGE#d-/t LӏN>v*Sؼn Y՗o1C3(Jš)my<ܷF<}2hr#4qGӣ,lyu.>)=r3ul\̋ӪGG̛G弿?udm?>]wû;0"4q XBլ_pB0k(g%vN /^wS 3GolaӘ-j?LNɏ0 ]gL8wl6;fn9B2|):9}Dn GB/F@$AoRN$ec7wT8)z]$MDKlŠ]a%KGe©$e̖B7:Ez/=sd\4[-|$Gp`?xԝad[qs>[YHfkϑgYA9̙9w)$ig₹>/6}aۆO =f" %m%az׮SЛ)0_l.07;Xs4>ϲ7"I}ys.̤}g3I>~}9yM!Fds&¨_*68F]^<\gI2}ů}De[aF/ؠ̞Ї%+RnLKB*9Q)Y=^:M s۰^HַcB+|r 5'=`U nc2N K”ˬ;8'<`PXLTMD$SVFtD-1 0Wy>SЭiyԨ<>~}90θx{WHJD\5N3Y]ȍQKm'^g.sAV{#96|}7"G(۷[?Gd'nrz{7> :N6ͧI?^BPlg7`:j۵/AeI?A=zqd 1:N>plɧGqv!G$1h0Ԝ8HFM0 wT2l6i*K%P#qag#q^7>l'7ўm[d8sJIݺD)ОKQ-/jwp uf.;r܏lfoad}0juҭ _ϊmἀws[:\y%`ԁIl ҷ,Qm/ +/H h"mؑDTgOh"lOiz.ʴOYKDB`?Zf)L'+ez+qE#"ri6_|)7a9]>~ =(QjᇿOvTUPe^r3>C?z>TchܛϮa:0l6χJWSu{%:;/KUrʪ|5jϾhRSUgnŹʹ7tvSÆf`ۓǝ͍ޜ{ 3~g6[ hpڗKJhϭgN{UӨ/}ˠU}j̥n[8apzGǷJy 4s.|Ȯl`wD^vc#m&d Y rwR(aϧ5ےi.Ng/#?[ Ht؉SNZ] ;'5'U ]^%ЀA;Y0k#-G' IDATߥF?ck3Oz>\VgR[]P 6j[{ _iٖkjj,_Ӳ-:τX+@/ (ښ/V|uǾpœWLjk(>ol/n}ڧYCu]nPڒ `kin-j.<2w兟A}u`#41aM '/ ZWMW$pv3(.y Ǡ(h0F-/qau7ȤtxMaeP_BeR<}BO?wJSٴ2M6og3]U >y O;ؼ;4.^jB£GXycOֹI|5fGJ2l$Q6 PzJA@{!%1g(Xh龗(0^ca%bB'u5Ԕz}]L%:LS?ԜpH 1*)L53t9^*Zw]ٵ=ID~f]f?" ;p=2;fq8r2up"}͟ uXeFE6V"FHEp#~<1Ĵ7PUo32;8g3q|i.;ܓ8LJd,f7 bL%q[9>?ɤ2g̈́\Am|m }NjYW^Q7cG)_3yԬ;ʑDn}h.R2KШu5]Y<|F֜hR+t}2yÌ p8o``RqC>o=rݣ[X-&.7IS[>I$_l|M87'SX:WO"bKaG!vpӤ[cW0q*?<Ϧ2vK0I?tsǴ@(Gki>׶nJFf5l$nS> 1h8u{(5M8!}2l"w/\,DDD!v_+Zbb(**Ա'5D3Bs\_6.^O.^ȂEaxSN(aJ7[XX89O Mu/Lk\nڛ=.{X|1i[++"טIuNLa1pɖ"""" )N㻑e\D4e* 6C@*""""""P uz%DR'UV """fWHEDDDDD Nhu """"mz+qEԉ%YF0-Jh0c!,d".i#,zta1)0Fy#;\׮H~Dxr|>r5W;DuDֽ;FQ%=𴸖v v->)Zc^* IݶE_~F ]q.KDriVrUu<DDD%7_Z]ĕ n#??ﺮ_֙I=qQj?Ai\&NH*r(k ?baެ:ZOSo^ z sLdhb7B8QXQajg2ipR{F1`V@º$1chfMĈx[NHgFy/8Vz 8FκY079݀ x&2q-?p;}ʦ3ȷ;1v`xq'?jg?$ ROցwKxA*!|m+DhuR"1s>[wXzp%-#ezg2VɊM{( >v-'q>wX+ am Ȳ?fGf3 '^*ʰn&Vͧo(3<8Oa>NXY&gn\x0*'dnG˺}h4^%wQ 5+>Od4Bs i~䥞]?Ys>i_*|z0bS(n#Heeq̞rCy\v?,=6?I`~JISI6E5D&"G/bP*X]9t߁jݣ`2>{e9x:@a[ڰWf{Z-q.?'j./dR}gl6RRdk|_2pt*Ӝx٫=EDD䲮$jʮwwM܉@M1*ӱ{~#[X%;,YND^Jʉfc{?|Uq#y[&Me$j$6.̗IyN*@#EӮ'j3?'ցp};J-dnO&kӸlt9.oL;q:YwuP4qdaܺ[+i;5}*R+M-ʩnetL3=!4U(v[D"ّRM5H wB ㇳ1QuchvGeOʸ"$PPo%"[ɠٱ32Dz%]gv?d_/;t4PS B$V&0BɌOq|Ի%`R[Ss>@٫ǀʋ3Gu~eتmϻj+6!1|\Shi;sاͣTמ*j}:kke? 8U>&yVT~ B%`1LsuZgGJpǽ2s%6SQ^@3c:f)LG Y`"z+qETD+Ϩ^Ĝ^ˋ/:sͣ' 3b`|WSSSP 6jZB ?_ pҖbx@U] 5uG }abPJ݃T G> ~y _3B ĩ[@W"5[5GQ^HpSou97(jI`ꪩuTsb*N8FϝøX×RֿCz7|;9Sjo@ FPpu%5\ϻqo!Y4tLUW(:5"ז]zR~zFrЮGO Eipo /Crפ ҏ;|$Q6 06{bgr8/# tw'fسpyc:#?SDsS3qYNƑ2N5b!,'X]'džݙAIv.wLS?8($=ӝnU|t]<4HWGESzVwpf<=!]8zk ۰Qt Wf?~Nsc3vGy9;Q=C4lEbx"gTl2pЙ)cȸ>$ 3<ݞ/^VDV"nj䤨_ǻ;e7'U†73|NT4iF - vR`PM&Nؤ9:(-#ai"wlH)bOG.ˮgI|*US^ f,àx}ʮ: >ǐة>foQElvFXNJEbNfROH|.W7y3V1j:G/:ȵJJnjd ї\Ǿ ?u͜{{=Rώac6WV~0ua,3w=Ku*"""2$$8DD\r ctOhlq:Fd)|WHtDDZgcGg}ƍGHv/C0;k_cS׉z+qESv 2\&F[|iU*,Duu}zR o?|||H9]`]$% ]1MC)S:9sJ|sΩb2#"#]iuW"*&B+NnNNhʮXBTDDDDDD,@*""""""P K(%HEDDDDD """"""b RXBTDDDDDD,@*""""""P K(%HEDDDDD """"""b RXBTDDDDDD,@*""""""P K(%HEDDDDD """"""b RXBTDDDDDD,@*""""""P K(%HEDDDDD """"""b RXBTDDDDDD,@/[[] =pj_{[+q[WSq'rS<,az[}8EDDDD䆤@z 1[%*׭Pv%0q7EQ!Rn)LN#;wLW{yX["_YBIpx~Xx90=z~>E&آYnĹ#غq%*Mcg c|,.]Ό },ՉS6vQ V_nwOJ_)KW=/mC}UL F I"Ћ @"HcA9 "R"C&-QB 陴I/H0@@PI/^xZ{^k>NϴL{{)OWV__kf\N#>Z~9MҮ\8y)P8̯9*udGsn-;L%Ǐ^}_;2#ξ=Oat~w9[_:1.}ٵWcNu;)k{7ܨk|;c'{G~: j5{jBjΣSM[ؤɩκ/c8:kiIҖ w^LK[wN' ?Ɲv1W盷lo^,hN1sf/͵<jn[nߞͷ˚FgZ˝rn;3-5нoϞe`drZF~L1i͘\|vr<^?fıI]뚆`:Wܹܒ 9[1;IZ2k3Rی|osĵImԐY :dp^ɂ׬Xvt|yfA c)._>.H2nK󉏮._<lS$wW/,z>>Mo|sN_\9['9~eR.˺kJڤ\|KSwJUN IDAT:G9Gr9?ɳϽ ok&GJ~Į5 JbKsWme=6:)'~FYXx4`ӷoL?-ϱBJ {Ź'kKfϽM o>e$m|/r:#o_׽'ز Ζ]VZ")EPt%zԧRXŵ#7s3oVbt9zgqijj.= ho/ R-ٲJ"F'")ER!H(BP vX5u;i'A}5},륝~,G{HvpT?qI~vihKv8+7PyӵN5_88Кʇ?)Gmvmʔn\4'5k .ak3ߧ=m\+;?3=OO/[!G}qp<%{_x/ wvr3ea)/=7+#rUMc?ɈcrA~#r12if[ s?fvS4]w }jnmܙop]j>zHVl7rےd_%/nN^8=OUᛧYդfh>y?M54^|t͙;y\.7O+[|.7}oW_'HߡlMk.Hmc~q˕̋Cj kS;d|z{eurnqcƼZ[m3]^.K`Us=v]$>ܐ l̔?vkOvmy酹<8畟6*:f}3wH%yCީִVWx sfIkSs6չ}Ҙ. 2cѲIdϵԦxbF]ܾgSY|uezc˹ϭ*]1gA~+_cJ^߬靓Ϻ+vYg@MZ',}1/,;g~:͆g3WFTi5*-oYvcQ۬Ey*H }WfCC.Mi=+ 3'&JdQޑi+ɺ.s~/fvYkt9) XM9ճ ss2r>9}',tm/!}TRrfPuQfϘ-IfʟiM}lxysnxt g>;9םf\ NM>G}-ygmJ[jX-v\J]S,ɘiԭ%u^\r~t̩y58[g4eb]>n]j~WlR/IwM5Df0"1'|15k]7yu77Y{vZ[̪_3C_yٓ2ynsپ|e7lTڷ[tK~ =+ޒыԷ[ۡ.S{͙^?={9~i9/7<3_e@쌚Smv+8{A3n\-7"itFCscMt`>zG51nX9`t]&|9-{ao^J[OV,#WZr_fܝ24ܐ.s{qL%y)p,&ؐoT/'gmQͤgrv˞%tTsldIuFnޡ/^|l>w@NY8atnǹT}&?8wISd3 ˹7<$52lsLJeuȄdǽK]7I>l6s<~\󗹩uMF9fE[m$e-y=CVHRMzl~]ZN9,Ĝ1}n9~ŋ߱٭K.v$_xByy9ҵԭsHN?y,/g M:.{z|OE-̜{ӯ|9ڕ˃֛N%IVjԌشfC˿7N<"_?;[{Bv齢eךv x[Sد 'C۾GǭÎ7C Zg3\>>&IM񵜶k5wĜrs ްMM~0}y9#s}q]+udGsn-;;\t#-UԭS2驻L}k49c/cӻ~%l˄;ceքQaavNe[K~qekvݤ+mW]ǎˌTe΂T[3g,n}X-假ޖ[{ޛo5^wb˹%Ξ2lywrԋ9slo|uڥ:ߏIc7!~i?􇌝69vKh5&5nC껤29Iuq/kɘȜ3Ͱnf õ-sr{kI/7[˫fI_6-ݻUcǜچ{*5X]E>%-miW>gC~+/=jX(=BZɵ?e|㧧&3}$=9=.}ŋlIssRSw0/5TӽiN.lIct2 =*jRYm֔Ƽ ]^2zz~i}K5Q}HN}rn7/9M}G%QM*yya^Gaf6S׶iӜ:#߾}ګڶ$ ;fsy]^_|g{p]wMM|do~dwY斌wd&7<9,Coݎ:$ޝ~OYhQ*7 I1ma^=3OO߾}J)ct̙X*@O[QN:ݖ]mSVSN͜9 ߿U {;gN6|̚5*]X}ꩿx[.]gtI\shIU ]N5IuXi?`@fΜfAygҒeɩXٲYgeȐR)=Jwo]t{kCYgsA `Ac1#ز @")ER!H(BP A @")ER!H(BPDL{zortj;t/=JKfSd޸;.]>TާV}TАG ]r&Im0Hp´UާV}5 RX-Oe5"FߡjexA @")ER!H(BPD[}e|jҔyEcߧ#?Z4]bPkk*8pv5R߱% Ο~unylJJL>|ziN]5..RDWvr>~M=xKnLhqqVNi@jK笐Y#ol <2tkv9bPHw&Hog;eLИn'}ms2]:57GrgB+;?]]ץmVc.Twv~<+Mid]k%i]߭6Ly۫nS n=hs倝6H< g ZkX~@vpt'n5ܚ1;>(ge3wkn 1l}?Uj忷KMYWʟ瓽^9_|^hדsKëk;OMJ9~٫974-<ܚ>3kB ILc~։wח]?MiK;'{Ӱq؎\`\57g=>lHA{U1T2_-Y'=漸0/:AxѢ-W?%'tv<0 Ȑ5WOMۋУjj[ט!Ԥ[L2nV3˧6 䀳^}J*S_RxҕLEGvK͸|c{5?.=,^stigF>[~ͶA;fd]>,|"j$/U2lW/|Kl>01yIT+Yo.Y}ܜs\r|s=C56Cl>ߺkwu<0&~lskf}ǶY=in2rn}+zZg6vG~dɋϙ=#lw؜O-O{%ݐ_+yXNھ)Ӟ%wo>;xsgGkfWܓjEO7( 7rJ.FgW]d J~ľ5 X?Ͽ{noy|࡙8}G.]3~KE߾2q)ZT&#~by>_V7o?uZ>/NFchy}F>#߶},}ZVl:%~_ w'zV'r3!O6 )N65fgqQOC{lyru2>g~}*figC#0;*ϯ;M)Xq&!7difENΆȖJϚ-,sIϰwxFlS%q%~{RXWixH'?x^3eo%uU7$O3'V_=o@2YKb)A&SzfTs#7C|h.9U;0#5ɍIJO/5RkmQΖ[i֛칗koѵϑGu:f@jӮCUdɝ'WONsʫV[}kng]w}{X6Úu6O:)vGϼ[ow~fֿF;kdOYcoH'sF`DƻYy΢ 3K12XNzG2^˛OofyYih?{u5fJbԬ @ZұXh=Ny!e2g Mp|f>s]ӚyW)G Y Tsz"ɚ\[`^ccd3e"٭tfXN>ZEw`-X_Zou)s azҺO[SGzvu _J^ 6%qfz} JouiBJ&2֧r5vWxn 4˂0Iz)C^!̗.gwU jvPjٚ)>+CVhQV} 97IFۙ!NrӵZq.R|O P~Kɗ<3$eFc. srUƭV펯׭x*$O Xe:Rw녰D?X 0Ltv hT=XvS=Diyv͝%m$pTJۅ mIJXy;/m$&yl% Ah F|ZQ݋(zU~J݀}Ŋ 5[(<6dl^ / 2p8mWa \3IQQSn1}?{dʮQSYb$E92ˆ."&=T<j[DJY-k:E+P'FDLU'dTX6i{ :#isKQ$N&o; (Rg96:,Nf!h}־̀AoH@џT,B6Gv,!4?~ٞfB '=4{>G:h u/=wlmCJ0tU ?b dk8u7_[Xcu(z(DK^m@{0Bi~'2yI'ZseQ* @ŪW&q`ndQAy7c=jtB A[Rΰua#sdzPaVo ݓQ [`y3M"v ^Ϫ΋l3`̏nv$ʎUFx\\ 3;MU=gR=A:y-άHaN66TEhJ^ݓBC8Pm qi\HpNR2V \u?u@;7@Ky$̡0)jOX]{|{51CnWgvߟ>+YK%\2HGT4,j3\,}2,TK 99HA+2(Yb%А`RL@'N8\;ԣ+f),2SC{@ziRxGkR(?pus3$&]1|_[GW_12_j ]3FAHvpqg1#^`ŵ۫o%YJSv v֚l5dsc' {p ya zs E0cE K`:[ r8q_`Q c4+xfAQyyDQP,93sZ)R\H3lnAc1@dTa?쏦vcE(NԸDqi!-%tƛ ޯ?، ]qs60.`;/_5"{koF\'E9SZny/1F0z#H/_@ O[z(&sVIMeq5xf2Jhi=PDfLb ĨvǞL`#A݌ST߀> (>&J B;RO]CޢDy98:y|Ql/t#F'fe %XB9m 9FE0^yd&e"&X"K2؟Dc9xUN Pe"u5*5Z?W&K]YD;h5NTI-'ݘX`7&#ڴ4 N5_($Y_!,t@:Cz̟KᬸުB8`'ipn2 s״_GDrF4Gy?rڏ{*#ƝPhlޟyFgSM(c,z.Q9p`@Qx[4_J!u.oSlx| )Ș1_5=6FQwš>٦bf"vH bt |Vc_7Ÿ\u_ >2E-oiQn1w}uu!G`E]Ɋ;I d]k|vS縿w7 ɟ'iCCPICC profilex}=H@_?EZ ␡:Y(*U(BP+`r41$).kŪ "%/)=ff0 ed)!_XBaDϊb>%x?GT) 3L7,uMKOcI!>'3ď\]~\v3cF.;G#],w1*$q\Q5]V8oqVku־'a-/q0X"DQG5XHЪb"K)% Fyl@wfibMB@aǶ:o4O-~oM.w']2$G J}SzW8}rU88F˔pwoiJrk1f iTXtXML:com.adobe.xmp T,bKGDi*Y pHYs  tIME;Q{ IDATxy|ա$IB&dAH DT,XqEzkb{+^"JeMAAd $Db6 Y2&IB!}^ 3vy099!OD3W)ٓDDDDDD*UUQXQXUUQXQXQXUUQXQXUUQXQXQXճ}t/sWHjG׋AT7T""""""ҩKҳՉzPWnsc/Ćl #$·Ogp_|Tۚ):&^.:ʋwdrV,"""""rـ\UM>i˃=׵k1v5 UP  ,"""""r1+` ;;3s%jyZ +X/\ SMZFDDDDDc=qF₍NQujo_'wgQ+YGy >iĻ<ߪWUDDDDDZ#=wE-`L͎?o9cm4kd`׉Fꅋ7$3>^N\ o݀gTЈ]*"""""" jQ\ ev6:^ |QKS""""""z>lp& Uv+9 :JG*"""""r:Įc/g87ۖˮhL5]Jǰ_LE # 2r脍2=gUDDDDDZecω 54FD8&nmažs nsoٕ}>ctF+"""""rՄPn`kٙxΌFsK+ ^Ugg#nb4qrrU撼%yb "g2U3"""""׬+gLccYQ[SCM'o466CYYΆ[V^J_MN>,̦PnG鏟v:^=E~O6D2g,&&Dǻ?ysO}}s1:`G=g|oُ!?Č #k9Ba_\DDDD°lՕڳ.KfWs腱E3_*u&O M`P>\Q|3`)'?; ?!L!̂䷛|W3 w<3r{3~>!O[igHX漴cR\HqOBC1 <4f[; w&`_;H ^lcdSapכkAč˜7~Ƣo$O"3+ySHqf7b$=k{ YdLggRD!egbovˏai/6})w 3OMr3_JHd4C>ɐā,? R\DDDDx "" rEUf?Fف?¡XT/dB&7obƖp'fnв,XBkl3Nj $瘳ax#w-}ֶ<|E&0)lX L,^YBy#{1󙟝 |wT-i|"ffgcIzEᵽğ.dFo_au[/G˳ز56;߈XwO~#l$w#Z2-a}$r2L'xF?8)b6P${cIͧvd9+UW]?_/ //^,mjjMgWbL~k}1]Uq:' R1Du)z/C@.ao^bE ),yնں"0X5X)*  f̺!뺏g] (g+?=¡ XNVPͽS%`?c[oU{Y@DwrkyeAL7+ک P»v TP X,VUªhZS[jqq1lLXTpߵw?N4X똽=۽XFy>B ,ڻ-{strJY3e'WZ3SOWg|d~uJ@HĴá7 eٮc\woSMXRf<;yN՝z܈Dc{g RXYV_2rm 9}ul_CV""""r~JNN5pk`}^$""矛O2:;;lL=Z i"d@"qQ1DOHH$gQ;Y*1s J!9u/SHNM.>,!Y)Su#ϲѦʱ2#؟p3Y Jpm$ƾHJD?xd&H$O(E:ݗ>#3Ϻn\ʉ>׿˖ 39C)ߟBzj QL6vW3-`l_{)'LOS5OP$ =#XY8ó0"#81GOힵ,{cI!0W 19`Rճ~˨8ٽ#y'cDDXOF۝`b~/Oc@܈ۈqc4 ^_r}"""""WzX s[[[m;99sx\JRbGoB^~j$fk9|])ɥԂ0Mxw+G>{< 1gX}݌8aZ(˳av:tה"wEn5w(#Xs Owj5:fqvd<3pKS)|l""7 Nd^&*y~9]Qls==ɉK񹲲⢷Q|mR3%όXr[댹fLHČCo=Yʖ=iU9VJyǼaDfLxlKd@?PF^(1_l"g; ćM崔8f c"MP,5[ٟT;۰-+LD% vCE9] $z!Kz{$y=L1Hj =<'XݼxiY}|$f,=βeX~"'e{Z'#2us,| SS!~D~~3ɯ[ySC湟>{OmW}VMaj(ͣ́x7ڞvY?swXEV ̣{;0(Q&Uj+>3,gȲyC<|dGiRe VOYd _,kw~I<}"ɤāOKiɟod{+I.J""""r~ !}"vwӓzj7ի'˩TL\Y@/3uuuv www**.yPcn"88֯6qw]ӕz55Xqwwd2a0ι`d2r*/q|-(|};K]J>ZV<< FSSv '''1RVZMs,~WQXMM6*pv6ꊋ+&WNNN-7hbN(^ZSkN=l_8~9b""""""@DDDDDDVEDDDDDD;AB rLSUUA~A>i((S틈ȥ ^y񞗗 >$ P=<>FDDDDDDf܅ R:ɉCB`"xwѰT ~y䑒̴i7<*""""""K?w[YUߡhnLiZBDDDDDDz6ELלzׯ@XXaaj 鹰:,1 nMTYUIZZ*Q1j zUo/odv{ݬ#$M"((l_ڮȥ#=-IZ a""""""҃a'4X m(dbjIɩvW:'g.*VVV_\튈|z' &:*n:ҵonq#qDcCq7b;Y759koeşXQYlG]B0@H?˶Lz9~[xuAl:/}1w0@RweodK!}&>eM_Kf[_!""ׄY=~Am-o/oֲ G3sV/>bM=}a~}t?vkCD=?n΢rWq4oPaWS'"" 硠 y1{inՍ#ٙL)b̭ f/YYdm9o{/!SH T-\Qy6.k2kwD5$zk;1"}߰\f-Վmj GwrB^}Ƿl9;e=nFi8_XcARp ItҶa[VۇU2z K=OdPGgt rX _W@'0|(due)ISH݆(ݛVRXgģ̻.O?8 -`[w M$/ƺRdt]wSw/mĎ@s=EVp 㧍غf5e-ȤIPOѷl*4؃ˠQa+}q܈9 6b, }:6 ?F}I[~;[_kݺmi[y{W8?n#,ld%iƒy:wDy\MMRںrQq?n稻)̻/;lĎƧ`~tu^G~0ӟ{GFaSG9o[̓۠9jVm5٭# xj~e ׅbvPzdz:6d.O+`[[(=ψG;0fu׮9xd9=ŀOp_ܚ):͛e=mȲ˜:o.m/![˵Hݯ-_s&PTsMu{k_wZDDly'6@Nn6OB <<ؚl444`4ݻ72y~~m݇o9DSS-f2RW[V>y'06L)?zN͡b(ܾ՟"ʆo7?a͆9XDQStԣ;f'ᔾk7'ϙ؛&e *dA?p޻/}̖ aҘPN8Iۙe{`e:725ћ#iC1g+>^ǞwbOkҋm`Mdv}kcmn:;$&"vk>^E13'3w=?^Y8EOblX%2N`s e[5VWK+ IDATcxbRʱ;sCfϚX #<ԃ}T7[½̽ɇӝd6D0>LJqa?%cRٻ[@D~xP?pU(7I غs׿ 1'Ξf>J|:ԝyԚ8z|ZH P!6 į&k7m=}Mat`!Yp#vN+ >C7e%L!)7>+OkÇ]LICoa[vQٟA:HHGΚVc&Lco)9Ur-2ὫHݗCm˵o>Rr,Jb@))۫8aGj>Rsm[׏N6|8^a6ȡֹ~x/~v.+~*dcMTLy(\+"ql"[;~Ĵ>_1כc{Wp׾s]#?\f='y8a2a}#HDФN=W;q-{PV1ROy6%k[@İ%28սЁ$o^wSԂq i9`ak6lzbjRKJظysr:ldi)G F'[z$l|3;$,bFo<5;ȶ:̛u IݶjzKzi mPobWZ!qʢeGn"'}>(pӖ#-؟L{ldIÒ)0ֱѯHaCZ۸l˜7eX}'O$Ɇӷ$[p $w?/}ݺ5$gJJX1L"vKơ$^HH)%~e`_:yκkmϔM;ȫnH8yqβ~m [S2N ֏111OiGl f$ bk^)or3~Hlt ?h;+/7,dz+`#l.T=TC} ߬b]?g;=rn'1-OVmkk} э$&luqj` btMF<\7aaJS!8jKaex`[ETRd{tr]|H9Gz܌,Afank IݯG%߳@TD?bus^GERwq^PٺvySf2dI|j}Ē0F1cܡ|.Eʌ!vƟi=@t[} O^de^zWr뉺.h %%`P\Ul9g9b"=1c 8pjNkr$-,FM»6{A 1v}lET|0;fF<̝S?b#Id%MnzN l!h׃iӏ7(S?X.ɹ;GKP*vGA^jrv({@Hh[ vRMii=>ahjpXةwo %ؽldKV$Syݨi>r(¥/Dq9ϋn/#lWz!EDb++l_^زHˆr4Yf5_oaG% Hᨸav?[ρLeJغu5Gr4@ۗEg}bGJpsmw&|oh}O|!׾F\a5իzU#z`&~,5M.bݽTDLeMKpQ̘5CC1   >Ď0cmX,#''z/3I}#~;c"{tv%Tx0>D 0w㷧>sL}} 1{1[E $ Go }0%1'u#G3|DoX"rh[1-6KEDf@ }nejmDYꋏoPAi7f890~("} 7a;ẖr}Br#qLMN=yuwւP#|FԄ0|C?bS#=]ʄ؉$q:2lƣ?5> 1*RL's( %qP:wR9o|I>-Jy\?[dR0i(g SG@R +-v" fqʽt!q '8ѷOŅ\.!""ril#|r`eQ !m۳UrZnedM<2 1714Ə |)qS;z,H -Y}%xǎd܈XsSm,|>z"=Wz7Wq츍C&0~8۟[7%_㘕358lvS\?n"yvR.f0;C(þ8 bdx-S9FAm '3a0<ٓe: {1sc6).}J -/&rc\(Mqܯc&ؤID_ L ݬ[2K݈q7N"ƻ_p+y5@P`p[PY)^H`UP`%;+`OGy<:WEDDDDDaUWW7`m %%SJFm ^-Z ?^x.|¡&Տ\;zUWW`7j6ihXk8V5Si<5'_]'kH;~C{>ymI9'۶2|EƸ>M-z嫶ռĝZo׿6 tKDDDDDxjZZj'XjuՎm$$ n[GEޓ7k E/H5&3M=Ny%9 ga] ~5| ?2/c~ iV10 2$fLB[g5?[s l6/.e+,LGPm ))?5H޻CqC$M$VI%Ĺ٘eVfhS,|deޜBQN~wZRz94 Uy$~2O-Swhg۽GRR(ۓ}ikd^鹼泘qOlRI^ZȡH2hZ g6{RVRC>;3MD)0`QXQXUUQXQXUYv{&WWU\3LnM(FUwU"rMۛPE嬶UH7h((ªª((ªªi] Ͱg?/*?0Pg(+)n1f @3Ƙɰ `T,2:5͎c8r/* T/xˊȵ˰UȪ]O /hYQX$#7U]""""""zJA5$""""""YXm;""""""" 6\.aތ|fXTavLɔDDDDDDlz >]YVDDDDDDM>g+=u^Q3=)+)mbTLy\j]l7/* T/xˊp>w>& eEDDDDDDaY4nqm,+"""""" 5[DDDDDD{  Ս^8T""""rU7۩PPYd|<<55Trr[Xr>j3-iߝGͽy|8ۢ#/ȯougbUҝ}{mֵFP:pW,VvTFuZ:'3Gg0PׁK_(]9C*uT8r0t5Px-CI.6r7v!L@d?kTDy?/$oWl a̛5 F~ GHGTG7HftEmnbޔ׮bk^1ة|{|q3s\1-^Aٛ1 _oM2> kSՔo&IfPلt#H^h5iC>G~=_LLQs/aM50[׭q&wQB!` Ʉj0_m|j=n,`ũ)lZ ǮpB?+n ДRlMpO[e RbǮ\/Ա(d3(7tq+Ͷ#gM0F !,|,S+rHpZb#wPP><0z'=%[k(E[ pؤ ꎳe *vW: cx_8ng mGAZM/ѤGZő~s40Tͫ0ʠ R^d_ԯ}h K3,G8z_x1$;r4ZN}3汛xh*|uԥ@NNC%PHssy);TG) Gc~1{lT&+'!}߶׫2 %1ЅX5ڤYi< gP TΖ] }ފk)r\4Y yQ !Bf~pOŃuJݜSw7Zn`Ft&=9|cdE/Gܜ`\{[S^s3XL Hs3ДWvڮP_9ž+g_F7%pnv^Fc*lurԀ=~rWw/Ks-NRˊ[uZ}ɱ9PJ(5hmnf7z7lN[,LVX\=q W' Yu++[MeknUq+|IK$(3UER.e-EX֕Fiٲ!+ mns>Ǣ,Qei j M xgXQ^%ϻ2c W3P.s)"YyE !B:o,`j 9Zpe'zYtǥ&h7.xvJ1GwuNR18(s j),(-K˖^+/|`^(9=; A獕W/7 99r?;|š. /F !BMҡ,~S^ķFF>i]6•?/cL7WQKl'?| K9p!ˆ~~6k Ja". 6zKa?23d #6͞!/Ǽa:A]@/ IL qu&K :üO7L˴`/зSӦႶg >4},Wp>;1p5G3)WW/`PWCMKfS_{ Gh/\ RCaQ N`_7\Η" <&BYVQ94a*g.eDlxDRwbƴa[\gQ~dr&/b@h0ӷj[?m>vwIŌvm5nx]Y8FTJMªB!iiY5gF/ZUXDUªB![-4u x62=W&fWEsOU!B!- w!T(nqښ Xp*B!!<US#1բouH5&ҚkU)LeKXB!Bm#t6['`i>OzUQ7*uέe࿛U]`4Q*hZ) !Bѡj5;=Fݷ\X9'۲^?^E@k(PJmIdkU]PV+B!hjj 6nskA4*HJ9V k2jѢZpjR.S(S`2ŨAQ׍?],a.VPB!m`Ӽi缶40**nεe`mӰ70O 77LUy9^!B!oT[+,]rۙdq[\2IXbxzx[_9'rss !BqjK-l %sוqptt"//Sr4C  'G'<=j櫮fSR^VvGNׅB!A9J84%!u4j CRSjfn9o`3gf\xt:-cc!BъAZPJm{ u-fd0naUxwhVeDJ{?gvJVtyyw74RB!D !3٦Z\[Da~mqKe֌DEdP"tlJ=ZmQQ?0jB!B&S/1z4~.[Snz^:u9Zy_/rtA! R3B\JM!//O/rB!BHPjj7kDZmshݖPyYn&L[X[#JX-M…z˔J{F EYpvB&NKªB!f-èBJ J4h4Z+ myUhn~~d2-9b4Bܕu@KK5Uo[/!B!ĭhPxP$:utê7K#YRgو8txkO\ZWj[aE,ac4贈{ !B!Aj<-w4XMmZZBP)))Qܪمu]+V!BѢ@ќ*iUvUYƯG@[όjUUYs5׬T7GnMnn !B!jfSIAU.-~*54*-*U&Uy%.~~8;wErhNGB!B4divPnW-կ;vԲjnM4(& W3HHk/.ÖB!BBZUoT%@V-u[S[uUjCI̥Ԕ&hs*B! [ nVT54PQos\V`?0sF8J=9;~ɠT3sFx}!B![&݀m0S9u ۭzVsss8k4J=˟zN7`0ߞ bSϡTs>,B!M~jii]Y oC M]o[ʦ5ʱGƼCByTW3a@  <枪kp%!B!DF*AUI}Rpsw]z]Zzzz1kрϻjU'(˻}yxܐ:*B!nDhMu RRSSk2fרZtl;::Q_dO*77/}-~xzzW-F$7WSB!%Aj-{}NêD0)@Q8ZHjٺ6#۴SX!Bڡ電$ێ&U݀n&{LZغj%/B!D uCE0Urn;3_[ǧ/M$a’wHۻRO?yG/,O-㖯y 㹏a#+ϓqhx]j|֨o6B!=Җt:ng6bHo[X86 zqodfEA{# T;`jsy}U!B!:d?ݶ񛹄 nu))'?/$NE/O01fJiz֜U!B!:fhdu[3HGri<5є4Cp~?o |,Rbyg%sث=1߉DeN/o.7x7* 0wi]Wۆ[Gøؑ s|ͼ wlYƆc,y?ݺ/Nk|mrf<ذ?rǵӏg-9qq#QY4g A>]-"X$j~,Ͱ^8[J<ȝhadOjFR] & '~kNPjE@K Pj{bK'Ґ-:.HY mYtq++!FJJԔ6 ,-nIco&f^ʹɄhd2blʹjxO {m>FE:^SS(;gY4+Y$ơ?n~!Ym8GeQ G:P-=0I@h> r^u\zowV;>}6ɊyKS˃&NFV5@d6.#%g>A]+O D?lA9EAljiš8?~+J̡xf3W$AW\9F~ p z ^~bÑ$@H\IP/H ֐U?;Lׇ0m57gys_O>x}5z8`@ܬ6)6r$VLH7zqw~sXԊ_Xӛo_Sv6V;+AOʷBځViAH kj`iM D  [\uǹ=زBb[YMgBcp2R6E%zrvIz_8؂2Йb^:᜗XHI٤:%y5P PV+\N"|Ł|w .Nw$#?t A>ymIe5elwW5,W+[qdѦoI{U!B9{{:M`шF\Nh'Qn$3ٿy^`A6`GucPg2UKDKoTvdfR ԍ\6BAumV~'xsׯ25|3F?^l ?i1sV?%wMEY4iK^{K#u};y֯b7A[{4!u Vc'''9f׻X̟gejTppA]5Zo~^>j[ܔԶRǑFOKF)zP%%p u!->ŷq2P,bNW-}}$qyےMԶ/B`"kwzqQg_|sY8<U1A],?_ώќMLCoD_GUBP_˽3ktն5MªB!J$o^2T蹼kG>x;K>@AAo/wN牧’0O"> Fl8 샷WF_LTDrٽl|2sP}= V2x{{d|n:?۴,o/yq6tN !BY3z };dj]?s"xn;|.^ϪmzSq$!j$RTzGUh{U*#?+s#&1JgبQ㉁LZ^O f:AYA*"?bGzK c;bq^[_yUdZX4g NY$D̺̪{X o.hvH&5z_8<1AOv6vDD֬ACy^[ `/N'(˻}yxܸ}zrZ[{w sh{ uo_7)cnn=sso2oui#74fyjRRRQ(:ua1vE7%P $hnr!VZm^{IJJ܅XŴJ-nQmVۑmpY|9I!voa Bqτ).kպ6w` )˛zp 9u|ЍqdHvؕA^6qжϷYzl _!m‰ә0/`(##(&QlxϦ%_+'Kҥ ~=S]͠L G4kb"Wpp${r* x!uQK2 W7O V vj1ku@j_TM IDAT 2~A4Iu ϼ2oa?#!i `K-X0*|!vW7q$C'%D?h+$v!!C:u6}INN6^^PGFAvD)σ<{R/]Uk>L^a7qB4{!`Ca%[OKy.;0g4ary<Ɋy[#y`̔) 'Eٗc8(Ƈ!f1.'npϤyG0˓ Ol}g9Mp _DG6]}0"^XN@)u|*6c.aLt<bxp;nG;b.㱏N5i9/>:ݠ(_#U/nCŧ96qZNc\TL5t sg/u֗׎T?,^8k׮F.!&k0XuMŸA~t5P|-r>aOtoGԽ3ЃCM@,zZ7naӿv+M WjG7 'F,{ԇn`n<`4wi{6x3,ЭUζd׆]V4zM_(tGϔ:eY<:_DFE sCcυ 'Cshݠ `ȗKWW/0!vҧ ۿ#׮K'hb{s`2*4~2czؠ&C:GRB&=29[sL*EDVuXgyl\ aG vJ.;^ɔV2:_W3PUva篰.#>gƠ\_W &TΓyů᭏.<;tg)+/33gc6uӦΨ{MrJR~^f̦B[CxWP)&L t8>Ct1 22*ΟQ1d“ HȢ{f.!tl_CƉg1=b.kRM\7Uh'7\1R+0VZ.,noV; ..nƛ cX a?s7U0TjހFk.8d(;$*<=NFS]Ȯscw44A/q>#T]P~cܡ`` Cm1' * 8ň?Z]"q8#f+i puey lbMw*//o^SNsD{β7ȉXħ?Ig iQD~Y:J tؕRTj +~ )p¯7`/.;VGE`踼g5?2鐅S_+;R nggal⇾SiwիsH_π*oeBWvz1.tBǎҾmJѢ?ߨ O;ť+Ӧ@ip:uU11XW0QØ2<Uz GU}ؼXY`q)ct#DQg8M"΀4`"h+a=Iǟ#y m4.!C5fUgL>ճ9ޞp] ljpV@S W봒Td3wWZ:. / w78O.]Ⴭ& kOwW{y:W "ş#8d(?^ӍjQķB*:jp cyM'V }g9Y`[w^BS5e+vCg݆Yv8jHs9ՍAd˵B e奸TksssLG^mEsG2Pvӎ]7cL\{8ns ՟,>U\V/olCAo55mڱE ߿';wج}k/m@M8[\{*af7F3# >~~h.$%(0q>W(ҐǶ#)w***شK/~ŋ7(S.I[HAiF4;x 77FXwO+rI1i/G{l ? q̚1g禿.=sK)lb~sPWDFA2%[i;w#FW 0V@MYm8 ;qݜ U`ыNI6=nw&^xOJ'khp=H9ղH]{AJ!wڮD<4`zAZv~xXNT>? ^uC:2qtNR9\tv/ŐE;G7\= hIgܡhޜID"o5~9Oޟ}֭ ⹅nKGҶ*hT3>Uh+aV}.nceMY6vF0Sl;^V=ۓt5[]w[TLyCj Az6UAC{"Gm L ŋ3#iJ{[ͤ KN0%Cbd`U$`Pxl6TLP fQZvCzQA3 #< '2Y;EFA M'S]xemx'pJ',,{ ng$Ò5Žg]`M.83Y[ (ouB* j=] Ať+KlmH/> I eCT?Õ 2_g8܇P(i'?vIl?_SZaà]$'O\*{2 p G~CYOSs` "^w[mH?ȱ<q2}a;{ IOEr|8yәh/bYŁkٱ+X@s GTW ?mi,&<2'SWN$6k.AR ;hIΕ\SXVmvJW7 DAuڞ^w&ߎii-u_"|7o9^eFo3s2˿[v",  hdiGG' Hfv..u?i=]:$e/Ɍp|նPV+U3i88j75_꧟} 'Z!vX]LR-tbr딼ByP CuWVą{p2ƀ&/?}{@V3!ZױGxfݿ75{o#B #@J[LAx`? s@J\!ڵ$S53E*2)8!ZRiA f#KB…: !B4$)!z<=XJ: `\x6U&B!B!aIJ}MPMMrssj+j~=X{ !hʪ='!mMk ?*&Tl܈jJ=#G BtrS;шF##{7gg}"1ϬowWn "g?ߺ;\߿˄[ߣ\&ڤe^jcc]h=Т!B1UTSQQ.!D'a7p1̌捝ٿK )ut*umҲZ=6V^%h2T*Mh.oB3lmZ5\>uF:yc&@NNR`[=tjYW7VRDi2RV&]ބWRͽ]z/}PRAm̄z|S&Sմ Su~FL&SM'ְ7uLEQ#9/2* 29h=s"Yʫ/!p;ki2^p? >oò:?|9FnvfOY{"Mlκ 7;&,}Ee!I~X{w ld~7vJEha^RBBq)++T B>砖W M[6Ovt:Zž՛ tw5z,+>Sxy^!.?9!cat̤WF:w۹׃+xQW{\5j Ͼ.o?m^ob~.½Rm̻Y־K'>!OadgX~R wBB!B.k d3r LVx~DMډM'L1TpW  1`h {s9`!dA?ٹgV>ց<8{(eDTWS9û|~‰Is0:Pm{Go 5mnS8 >deO9 -&UA!=iqk_}fm:6FB!Q>叏qzek+:Fn e *]&e>t(v{!(teH;~Fҳ~87>"Dk֟>[]qR-c돟{!I] INFXg@D v\vs--D#<:.t(:KvkPSbHy&.tDcIߜ}22RktHp Ѩ*X?|,)խބycWu˙X* ]?› mUYuvvhvP\ٹ !B!:^\%Uc?o';V<FW^fTwSNLH9s0% g297LeO𞿒g& '[4~҆y+xf@z<裫 Ql;DࡡxȄWO*QMhͽVU!Bj"Y;/}+߃'[[setVl-q\"8}y8YHNr)gﻼi?>.Tߺm$DŻl;PŬgCWVW]MU,ZNLFsf5d`!aU!B,gUCeb*nPv2MXzy|Z8r*$ !B!D5BBa7͛ٸq LDFFb4hVܠTAuKB!B>s$aU!Bq1A`Fo7%KYrϾmY=w\VV F#V,S ņbW0wKxI?_߰\#z1^O鑰*B!WQ/;jq%l_?0ܧJaee`2g˝$h;~+0 [3$әFR=:y AUªh;J{tqJѲdLCB Q!B)BTJJ]]1LKX[h_ָTpݵ$k-qp6~] D4H$*~]8QXVmvJW7V{2q遛J x$ch@=؝nJ)Bitoh`4bee-{fժU -ڏTʭl:fݓzORX8hfΚ1~(s~.Γpd# hL&V#R߷ȓHB!踱d2RPpj/>czͧ~ڢ}Y[[S7Pq- WFszsYUIL$E}obР6T̜J`s7oDb*T#<ۘ) s B~B k2˯N޽N5#Z>Ogpw{Ϣt#OyI1zIjJGҲznMMW3/77ABx7ꬫVp!5z0=ǎySa=++1mHzCi>G,R),ߕ_*7v0WO?KkאLOG]%=X ˨,u_=5#<L{Wud&fJ!ӅSWP`P`gk^ş_~?^z5k//^`4P(@U__^ouM6Vm 11x9FЮ7x刄UQ'~?5A&N#66uI3U7bҮ6pU;ܙx`\bө7e ~ٽq=8o3_㋨T ,gO^KqL7r8w0v tübŦapfg5QB!DGϥ  VͨD?3#U=~EA%*o6otoTVZVg0cU3Xs GRYαeq˕ZJa\R~*YPig_urr Gn^h%4[#qIRcCO4tnG(0x,DBܬR{PpwMׇ_&i)m"mAP-U8 IDAT(0qX.͙s9}{vw_øeE pe e~)-Җ$M#mIФ|Ú&|qϾ?gv&Ij,٪ݓEqo#i 5vOizsR{*U 'O8Nc{%Wt긾'd6B.%ө;1dgf%6;/_- ̒wJR'¯SndJy{KwN5~~w'~*}o2w)%AXUZzvwtm6*I JgӔo&t޻;djJ0A))>Ѭ/ 4tQy5Q.54OSn6uoRqmf--ᵯ U61]5'?\&+kV?ukixmujY6; 8Ck|P3]0S%oϥ́wu= Ijoj>Opq؜}<鋓e:|Q9i!991kfL&<.n$iF۽U}Tq^ـմtUW^mJT;yUZ١)sok}f59Fg&v3TwC)cjknVKvsR٥m܍?{-͒}la6~zRN}M>\Um2[ LRˆ_DDad2{GO32L}X 0ft5*d4qb.5e{kU'K^Vyٚuf="־>D-YwhMi7iI wVgǕ~JݑwƧNД9#Zڤ 9aN[kn& FrρKՂF/lW־JKK$ D&p+PT-h+5Y3:şމ"i;t_AL:uzgFV%|Uo'|EoWXjFU{CO~/Zq-4*=9`r'eLlw1jlh:OAcL:Us&KSmɋz):T?`)/7_iizw[%ԋ/5JɅ v}999|ȈXEE wr_ym=Iga^T^^%K5{RP=*@-,K`D~<0j# ZtEE">l&&< ި hE{$O:L?ߦq'*n.OΟoR4kʹ}lMX:ptA_F;R][>q2P_C]_bXFPF=3pL0"ߌ>bb9ve Cr: V d`i0aC T;o@_k}oa{ۇFeu({kErp&Va"᮵uS{g;Kp=a{{ 0tk/` }o$x; xBūvdჵ7L{oNbh5{EGk}dX.x TC%m7>öoJ/႔C*d: GV ;m߾+*02sE ukۑ= 8T)^(aNDWT7wtuH%?JW8ߡE$<Dq{o{|q}oibFNJݩ.wTP` p MXb AH ?%QϞzf%vwRҞK с~4k`0HNx@ Fġub ?<\>?8R6iDۥA#B Ubb6|j970Z{BR6A3_ SFUa.U3:\#oWbQAz*OwW*ZCu~hQ,= 6au!*pm`f(kO`j#-f!g5Aj,!V^j`x?R{6B< ہf=|xBX.`{jh\#%k"{ImBm/7h%V>j'Q7pPnmGIc}LbakGkp!#vjۅ>guhBXFhz"xiM>^y+#/Y|nx* \a+ vɚ&T3aȶk75T?GX!"sB_51 ]#DŽVOmN,cxo#cvbCן!.%V$\BcR1 rX0l) .Ab8* \qB] 1kͺ-cʳyl#Z_[,?3;kkh?׊ɗ N\c@x\ԑ"7XEt&j=9\r~k @oi].F2={/T+-]cլ;^׎8aNVY?~KK~JlSYfm]Aܦg_~J;_k֒>&f~}c|MKOWi zuwITƢojͪە3.VWR+]+~Awu%T|{$gt~ˇ4$iů6>#h}:u寴~wTv3[i~zYq|M(g!//]Ur{8u ?ŸůI=KZw*_X5b+z}j͚qRCީbaNG6Sg>5t5٧RX|7zuO})xMaVy{2l^جRZهKm~AEUvwveCzK5fsXROi܋8wq^[:얚k_^ZBK~=rx6A;jjɣɅQK:kD^̥'fag1fcZ2^6mЦiu2LYE׊oW=G5gyuC~*IP7Uusv4_+Z>﨩)Gk͖kϵ&oEex25Ԯ={ϜgT;I.dY+.^{P{G%-QJeO9T XlxQk;[|HzݳPůFFZW7la\ʜrgI{JP7ӎ^;蔴S[|k6;,`fdh۲mo^g?{U=dԴ,tT2TU}iM"V;JxteR<YDRMRP%O#TvTԧؽS`_}.$ktN,Fy6թu~^Xe%K!RոU>7ӑ#j9ע~HeRS=S%>fK4j =5ejw:K;^WQYޮi6K*;R%.wmӎ[x2$BfW9kga_#n}!=vUUv}ajݒԱBtSZmۧIdntWm.ަђ$WkW3s|PA<٬6oPIGlZWeaeiw:ʩ5/=*Fe;~W7(gk~z\>.K-U:q[7v׺kG;,rT{h6N~J<5+睭c&,yc}vX=Vퟗh׈}lq}c*5Q3_iÛƗpyxjn;=QjܺOŋ7В=.g]^ۢ9?x}#1)c'+JINc>!Izj9E")i'uP<9SN\ =zt{;p8-E b6rwoA<A9s.զڻoOH IP\19˝Ij5YYS:k6mO͗jUYA+\Lf^>_K-(+I;ҒK2{o}iIj]V'~HK[|eO|m/RIʜ9=$޵BgJZVVV+\LFVTV-o&J>gHel}[5'p:$yG[{ffIމ3bwՔKJJ> pZR SnnȵVմIZm6H*('զ]{7Cj]#Ym6s6M"V#*//UnnV?u$K|JW/̙ RHl۶=jjB籞PY̱^Eoӑ#j9"ͦ3.MSym/5Ue ~R`鹯\ 8RY_i@**XXE$<*IA\+⽚WN:;]|FFV**XX+Y"'g!:|:UÇQ0` V V**XXXD</[,$-ޣR>yM&CVM))5EɩeZB,X'[^G*+"ꪕܹ-Pt˂۔~E9 xIѡ'T__{Tii隒5UVM`B 3J}}]@ V V**XDJrJ\Q9!Iʚszy.1s--3;V{ @Lc`y$)/7?`MKKWv̀ÀO愬V^yV[e9j!V%p:~=hZ6~JNNл[fK|LIɩ:;~TLq権IMM9ZוsZ;ssjbUΟ:Fr:)0=fLʘft@G9$yg}wr8:$%QcCà1~DXbǍxҀ}}RѰpU4=fIG:Y}BVHm/R}}[pbUʫɧ HXXbb@@@UU V V***XXbbxj(5-xlI=N6C&5יh7 ƪѤl&L(A$uz<-VF ХΜ2tB<#Sǯ9SٮtrN,w-K'>R):tu H [sVpݵzz2 ɞ);)_7X3-{5Sw'km 2 ]fdictulXrUX]'Z2@^<#z}~kM%I9@^l0]yRKׅRFia axyDѳ={;2 z=KY'|ثܪ0 jR.egv L]gxuUa!%%aO`nijcC{ A]ƖuA ]=2dhy&$$Ce/U9Zwʜ_C,`{쌻];˜HzLɺmt}]}N}s/!gYRt j[QuyU V/BӐf*S}N\AG}vUM42NLɾǎ8 CNfb"t/;Fgܷ|ʜu@FѤ.iI2̀F[5י4% LFK }p2k-e3,! Vh7x*AJLUDu*=WLvmڭN͛KJƆ20XIܪ;VMU&Lq;-tEXtCreation TimeMon 09 Oct 2023 12:00:42 PM EDTuA IDATxw|V}l@@{ⶭZGZmΟZuUBYd<L/H=ssF`P5r+ """""" """"""&(H+""""""m """"""&(H+""""""mծ4{Ҭ+++8+\+ 5v%DDDDDD<<x͇fokV9hBI}GREqqv"++Ypם-<\^YWXhŲ<7䢓1򿟾ǏL׳,zK/^批pl83e:+֦ BWhwXCgso"7s f yo0Hnx>Fߚ{xƾ8_Vskm]\.oZGJCZbⳓ6phQ̞دvuDDDDDE2̪ 3=)/s>=,"""""m,t{１]}0 OgQ Ma ϗ1k>:ZG q?ՆBÕq܉Rz}ͤ2j6x0Uڰ<ӂwyw RUg{ɫK.ӹSy ʫ_Luǟadz8c<= q8i߼:_+ Зh1p ws"vżbbJLj2oq-U?)1aq4]=If? GpoDuetNUerƴ}lWC^f9HN XƱ-ׄ(L; ͷ-5h({($6OaLoƾ#Aw06Ǔ{kQ=n:#{v"'ټVFM _[8V\0i =;y`!jVNlӸ4F톧ME^9c<>gq`ZAcqgq ^Hz.6o;DfϘI IqZ47hv5@'Zd *g;ϙ2iHn5maWA .~P8Yo旿ʯʯ^I4zgxS ?ƌ??~__r ykDx~.7~ Cq|#QcӧV 6Rbcҕ+{_ϞC/?S0? OvcZ{r??ES;~Nǫ퀅 skwpXū,.oG̼i2! ,C_r1z1nm|/vjAHO֯z5TڪHe {Q m %zW'$8/YAz;]ObDSuX6{=6~ ~3׍ tb'|'ӨtɤsoV'U` oV|j>L99)|vfF^o$p>@ A&"68;a&̴󟯏Pl:2&%$8i؍\7cm%`/2u_Ίb*C9u<ˎwx 8ݢK@DDD{ÃF+Ϝq=}zŧruF`[s'-o9f¢'_2_>a?dwz{2uKgSa‘I=w`&0G;kGdOowcg2ﶅ̻}aX g\ 0o,* k.⾈;]JC$CR5TUBR5?QTibń7ቑxfeӽ6x9Yaۿ:Ϟ[ȲBJj X87M뉟0ivڶtH>>ܕUW3 5_k{QbBɑa+Pq LƠ.RF-$؀JⷯH JrL%/uo:I@Vl qC>djHܳcU*bf3ll N57#ܣ%:qpv:w3^gO(]A[6RFfhNȷt]%YI),{8 Tqo!d~ھ6d\Ů"L`Ruph[]%r+t0F=Mc%啵g$!:Cv9IYegȠPwYI0[H_]*UQQIee~}Wvvva 9{᥵-BcϞPÃP &>0/yoٱWVRyf[ vXC;3G>Z9++޺6kei_ӵ '{Xg:vkv-U\}+yoϞ;;v򳳩>Wpq='΅ 3t'#ebAę Bp`':wc1-lRRT| $x|NĦaF3ϭyZM50L, Ԟ=zsiYgAw_/,FN| ǓSOvɥU_^/?+؝WUNZ\0\ٟ= 舵8D.?)DȨw,)޵QQQ+X{>Ks iLL<|f$^ŧH.#dWC1 (//|;`9]y;neyc8QHZzRkK6qກa%0/=ǠuݽS ]\-X.7ܥsk]rѠ^es q1.kL&FqIQ3lPIwy3&aat6ƌv&q6a/ԫ-`kLIhtj1GA$:lGX_;~x/9'2sx[`zb.x=5)--='aΧ%uӦniU6rAII6|3"21v?֊N0G2Tm-qX"0_wϚxŨ63Q~l[]@U>iYu"@MozV($ĝ*ҩj q}q78fQ<~6cr$ĻxTcο""""?xnOnĝ%;; RROLEEmxmbV YĩylK x7dySd1g~c{ɉaۻξW;~WL?LĬ)'?8WO-Gc&[^吟q3opCq[]}̙d՗ɝ{ݱ$qp ee:&M{2D.2k[5gG㏼| 4uP Gx7wsέZG-eOb|M&iGXd#)uW0atnJ3?\" I߿#3r;Ⱥpr7|ʆ3}ݫiu[Y΋iqTԃl?|bJ]\0˲I:1;ڞˁ-xݾNTdDZ>PuMۗ,fxf͟'6Ŗ֚ZS0;DP5)> _:Mcą v!7v;1'yim?P;3_ pvva-y2?rE6B#}",q,ԍ7Bn .%wyVDDDDD{feNģxmHUYYqv:HJDDDDDDD """"""&(H+""""""m """"""&(H+""""""m """"""&8\:\a:W"""""""R\ F~`upBUDDDDDD-FrX\ aA#""""""rm1΅׺?8F\SU0.x|.Vm` 0Km'""""""8:4 k 6nzVDDDDDDC e 944}9;|4XDDDDDDXi`EDDDDDMpvŢ-"""""}`)˧ylW*"""""" Vxl`oV3;n0wgzLQ1sK[ȴ5P@m)Nzx4>ֺR ,ff0c/üޞ=0#^O*ϠtJ3q/.g?.絙wg"Gw׈ 3>3 r:WW(}'EYxȅ7.èG'vLvnÙ8ylj(ts'BxZ띃")gᕵ ]7-koﻟgʰa+,""""?t m q#nNGNgC0.ȋ?t$̶&%SKs\z{qݏ`";ӏ+}-~6Mopf#nȨʂ֏~I >1ao?]g,'w9>~?a ԅ. tw~oH={S"Gwq(Otw&3m[uA017/ܳsO{d ێ<ş~>o̬珞ز7ƌs5?y۶sH?w?[]g\p& IDATe7JhqYDDDD-t̒|UE_kRw7v&f/^-ͱr,~\ˡliy3bM,94aa̞avz}2_ĔGP%Yk27c8 c҄Y^S5 ʙ~N|weBtE4[Ɔ[h8z/ PsuW~w_\^^x=S>e{ ztHSxvrwΛr1 GNEs1z`l}6e\\C,_*k7?wݴΩ!i۬(:,""""e'A4R~^AaCm%k囥غg%g9BJN#["D5"UY vF۹mjNZa9`^N& Sv[uUmkO3|j#i^`ACK t0ݷ35YDDDD`=<f} _rniKƴ4q+[ w.xC;hueVSa7p&$_Lf&[V4p>GnJpee3CF s[cΉ޷}[ІS(al)d\wcI\t)aVv10W_ʊ7)c.ݹEq60 tseL"""""g`95vri(~w¾(iG@Q̺;=~ȑ^ZE篰t1 =:>gN ipݠN8SCWXomD'T3n~Wpt؛ܜ]oѤ;K6'1  ~xy3.c)|r>]"<_y6s'B x3RG@ O+4~]]X gR>a?6tw09| q5tOO\bp&rM̝/ TZ›KO=eo|K/I!g֐ovr^}(qɏxP~BƳ,`^%o2[F+^@EFE*^+xO]S{z3oۖϡl-g`IsHesۧ >iʚdsBqAʖ>˶s5~8p6, )f[V}ɚMmnbl.8`qՠ,:=)2gH"h8+>%թ Ls#'7ܔ(v:gV#{WDNf`dBRq<"Jq" yh#Q1zeqi̜>)3w8NdfSw"`lns={y$֨\kڵkOQaep"â;/9^}k}a!hT;FG-ʿ۱&wbpJOGCnUU wzL_uٵ홥8`Ȩ[9/% ٮ7SVP]DDDDQ(77WSs*;=ԮּĦeShde}oX9crb1{rjjx:e~v;o?mÍ bȹC""""rQmonzϱ+Yͨ)LEQ6XClp b cckyײa*g&%[>~qY2m o8{Gn0gXqE6hyS&Ụ̈̌l>|A,sϘI IqZ47hv0ў߿+>&%Ql[輚s2[' !IQA6H|q!7ÍQ<3EoehM)cQܼسi$ 'Õ$vaۉ|lgƇ6zGWx` Zʡэ>K0]'bcfg7c z]G:X]Yו};VyqaF!] yqY g29Vzz9]xg9|ߝ)\:RV/zʨ>x:TPp87m"&!KנL;n,Cش?25|ډ1K8pr`{`% [cY±WL\?8 Cy1ۖ>:sb>tF#,lb]Ѿ 'LlzmgyHZG28[^ [Wn &L Hܳ{SEDDDizpS!hZ9FP_~V/_ŞTzO̰e[ke<9 [ Z)&gi;NLf81[U5+װ'6vSإ㱙Ta7b4]عVRMM `o"6[_->#`1dt?||ӑ w2+4[W|]Qz c W)3] bgϲ/XY߁iTҿ$-)ZGq19r" wD](ٗP:MTb6\2yBؾj[Q3#Ɏ#\{owW)9EF%u}ӏ꣬['rtbmsV+ױ/#$+jE$C|>ߟo&Ȏi$|`zyoWOa#aM9C`;egθIcJ%t6GɼpX$?~Qsǻ}R'" n/ E&΁ÁTLھܶz9[c Gy,‹n`[GuB13m^ȹΟVmsAl$GV\OYn,̊fk-ˠm ;bU2jR$أ`nm+XfOJh!N^tGuǚUkghbf~w+Vl˩||Q9^ 4HXWQe,6&5ԃ]wz &h׋!=9z5Qirl2WE2g 7l9EIy1Y_H'z1ٿyUe(nRr O?ʆm145nf3llאg2(+֌: p%>r4S]QCE1*npTشPc7c5xUSU〻a(5R{=KcRVZr&YIe(;@__<}t~ uWd)klf\y,xw+u#l@t,2J86y\oų+zf,""""-jLl^6e"U8\iCg@Oon4̊J@{ީ-lKnDf Ëܫ)BK N}8o@aI@{tݕU4R ;x S'vl@I ;0ptyLI3)Nj0r *ڏ.t e twnU{9Z՛WP&C _KչGuf+CgNO>n3. 2;<û_߅Hmj*7!ѓ1gB:m0nHl=pPkK O<2t&#BYv g- baxЋnbD5f>""""2BjL n&hcE{?cڐS%K0Y'b/M?{Uqu" qT{(5ԲLҬa{=ҴlYVf2P9}\wkstvKgɗeXHUñZGCEDDDDDJP*A^5^2DDDDDD+""""""UT """"""R%(H+""""""UT """"""R%(H+""""""UT """"""R%(H+""""""UT """"""R%(H+""""""UT """"""R%(H+""""""UT """"""R%(  IDATH+""""""U.@Ŭuhݯ=bqODajR|ԩD4&=~9l)}(̻)]s8`]n5{੻lCf8\:imٱtxO7;[i+a{EDDD ^@}8^<ܙ^MF>԰qdR>[d|-8'-ݱO̟sO_|ζיa:4Q=Z iQ# ù]D:j׏DfQ߼ŋ&,^uO嵨= }p7'JkiXX2E;:~ސw$%uOWY4Vz㖑}iCA1 >Vÿ0bb'2Uuf}8df.:F,;iMۉ&uݱpd|ɏ_ͻ#CFGFŞIB.E=Y/ս;ߋqҢ~|YI^EoWOh낙D䮕|7lJ(?RRg LQf$;?{q=e2Jt3,Jsqǥ9tWyݚŮ?e7$;5 \ӫ-<9zm~KvTޜ]pq(哱9fc%o,*Miz|c=|/KY5ŽFA,Q+\>Q ٥Bq;¹]i@${,;գz'~s<|qJlq ,O8u]h5`Dy^&ŽVweg&mpsqv?n.1I9gܔhܪXɍ[G̦9~3>~]m7{hu^ I;bתwhNPiں+Ng#aY|=!""XWw}rq>jqE\IF&߼vRO^bT{kTG't!\ж]wzԠAul v ڶAylZFY50zLf9ٰM{] z_ɕ|8)ceF}#hc%nIm:f@Zӗ%u7?ٱg?QG`26'$֤;f\(?ػ;Ăo(B#ԁ\6&FO;[~`v+4Å_)iRdԝ[oUH {~`Ex֌zZP$T;m3hXȁߗ3d.-nṗgPY߻&>ͻҭ4Ⱥ٧c? jlj#ޱlܙn}ЪhkK. wZ^96RHqp"&Qـ-:Ы֮'Lv9yĦ0zW/G Y?]wTFSX0mՙjD\б hm2aͨՅ)/ƤͩC2Lѥ@zc]hN^k븪IԞ싈D8\ͣ/?Vuq<}I($5H%˷ e;EG"Rf81- ۷Js9}z{Gܶl=SP[uG'"dLdvnb_L5m~s{R.<=M/:}ʞH/]Hrqͨ3歛b>{ F"cTýs+5)JNr>JKI) (<Ռ٤E Ǡ{=ea=;go8=YcqAM5}-kȩ{~\E͌M,;C}~NϤi臑0Mj7HvqӍ6?#eK \ԭF\DT9,h29yz\NBL;"F=ZdP% ;!}#ӿ7)^}wÓV^j-'-qK]ǻϿʊ ](OMoy:sϊk?V5[Y8)cb>$N`Cv?6ǝ7+C~dG~}g'sZI}sW1X[7]'qp6*3-≯`6~"`\s,bg4㍟#N: Wa썴^8n IF< Nuub3x(:*e{:pUq %u\LЇC&#q }~>p ~ynr3KON;P#݇{ܫ_sd ΁ Ҟ4GAHXϼ;q'ϗjM,k~ZF47ȴ_Ĉq__HYDWKcax"lRoS2&y~JKxDN|wOmf('hY&%NU\+h{x9ر7@fOq4q67Co!nw%:-naY16봷 0|Dvi2u;wӟ?4=[)&ۿ]/^lqlpbL6.aMىO ٸ6A_KQ-es.^N6׎p[cԢ5C7Rc+y#^壍Y8Nn^$|\x$9~+;6ܚ"ʩ}1l ޤNWd6}&gsхb^NW37}i=%e}^' M*uf؈۷oM_93Ϊ'˟Yoނgk3zg͜,|SpЫi~jqus[_N$;>^'_Oe〈^L 'ȩ;[yS%|ǬK9FX/ge+-yË0jԣ^J45p2$ڣ&Eb ,f}mTN}\o5+ۏrn,=Q[Y^ K(SIᑹDcꀗ{".Y\M!Iֆe+@1_do*٭dfkb2&Aݞ7~5e=gǎ惙G/OW3a85DDD`+NF|Y:vr VPp'ggʳlekT}f[6G`4$$ؽ߱%[VLrrffefquyId'=.g4*.ͼdsM,.niQ3=ջ+^x* rwb[;gl`sl͖CѲuhލ0|pح_{7to韝JőGhO8 wv Vz_<;yE9Q'd4t÷ n8Z 0%QEpHWf|fn0K<|.E݆.VEQ[bתA9i/bƑ [ '|7%DDD{9X-Xv'g~*a2=;{5M9gyh~ǹ~\6k˺]Olt$C^1sAn[Ӓ]&%-g@ v<=s_Ӌf9X8+00nS~7@gx.xCl.vLJr"_dfi-,%?$Y96ǫAC# 'nŽ iBs܌H^!M}S+HHM٨Kਯ&ˉҎn%&{2I DZBu's1l]EDD(^V@:BY݇Vb#=% ;`f%z$n ~VH`Ȫ_V)gNJ/a椑Ylb㇏PۻًSȲ[ȳ2[3'qr>~_ȳ,8yй.PMN$cX^Ù6s ]MC,xřgQ`$%$2VphVŤ&KTƘOx敹f7pؓNZj:6w؛vv懟}/_?Ŷ4ؓ3QؓNN X!:vV5k{.2YnMHWvNum&r<^ iԬkd'?3h[mO8|"F]ܼ\1G ;)b?ޓ_%S1jogcԤc.]+iY&%=?RӝMzXtRVPPtun/z6>{k}ew_ZѸ~r},{7c=cs}5ج>s}3Z{ۗ0S?4F`o@ -KQ8K"QőkYW.<9Ѱq߃v!tE՚xtC<z'yd;EV?r?@ geV戅0>3[k<.ɷls` y6hmG ڤv 63H>ڴ|AǨelqӉmƵx^fFR`EDD.7ē͠_ z m L3 K~&άÀ;c@@5?d&8Gl͹DZ̼a EX,\=.oq՘ȡ!0o Vڝ[ﺆ3sٿ|g|ImgVN},@Oxm~ЗoeD~Ԫג@w-ysEakhm V%| wϫ{E fh`y .[v.w~黓1Y.4i ?;fq_> `#b4f_hriԲ%uko;?ڔb]ݦ졄GasſYkz8>?)][Yh/]&tf۟3pǭ?ףE͒w> 4M>FXd6 Z,iv#^CT) X{ҵo7 _@ád&RhSOED/*;|FΎw! |ʐ;HMHgP9.q~:6A};neDlT.A}.Aݎ([b4 sO~n.n7N+볍 P|&if,G h{CZ"5>S COؾl"""r9(^f*NƾVʉ(_-\f*G'r8_s%wGu()LXfܬ@q ≓ұ ;K0ܿ:F KH˞4q( Z9u3 ?[UY;fFDNAX N>׸5-Œ]CLǛ+6zNqiKq:qKP*""`xx-t˸$ <3~B">`#ߑ>$oǨ0G$]dջ!-קC.a_O4KTLVqy6+e`\Ef&e1 W,]yDDDDD,X`EDDDDDJP*AVDDDDDDX`EDDDDDJP*AVDDDDDDX`EDDDDDJP*AVDDDDDDX`EDDDDDJP*AVDDDDDDX`EDDDDDJP*AVDDDDDD]HYŗVDDDDDDX`EDDDDDJP*AVDDDDDDX`EDDDDDJP*AVDDDDDDX`EDDDDDJP*A2 o~nKʒEw?sxrB_*+n×8c+Vy5U~z^Fe #GR+O""#<{_g""" .wU #n~f<ým'ϼԯg'Z&~-"7p{x]|J3):JJrv 9)w`} 2v}ͼ]z""""""U_p".Dk_kY(|g8#y᭎8edEȘj __! {]q3?fBzW2a hS1v.{!dBkk:cIяn0_Ny$ZˢO?e]]i:x*FtA"6Ʒ8''Bn1͵W;OD^!.mW:W{SsOξvM2v.~am|> ;c׌HG1y6E.42u+Ec|@پ]ǀAL:>²(i \ g#ܠ̣Z ,ƓgKL'=4X<4̩ac幖Y5.ϜB mny;Z'kw3>%?2M ]͞OdyL' Ir}*$|C({5iߥ5p@W"Uy#3(fK#s\7s,w/T7 J> wMܺ#sǸ{ʭ<4f0v-~\9 eƽ2w<(C|K6n_wyީ<9w=΃uOo5pE{}]UDD_JO1pݎ 5:^k_ O 6l9Y/YkqwEe56>Ufr:b-I-_ɠq&8{OFbYs0p_zӯ3Xhk{/7GtKf "ۧkٔ׎n!%@n Uȿy#[a6ӎb&44z{2-D%sx\>ZMIV|(iY9LrB_OŖertr4YD-oE慯UDSom1k6޹wG>[ُ%fw֐Jsf k1)͏Vq8GQڨ0b/xX <+_sx4ص]5t۲'vj';9XT)~Sk88;rc |9p ScfqPa`֖6bmۓ|*ReU=:NP=wm ;ɼZMgPIuUCo݆4vf{^(vs@ddP?1kvs^^H0sȾ4"""ry(V_^_b6 sH>qօ F3,g-S!8`6oۖN cXC|ytb@^(={ϘԤ('sbrF%:ϿODl@H)1ʯ"Uyۣf3XnR4߆b!`x]uv\͞ [Y@Ƴ3!,퐁a8Xv|4IH!ĕf~ظcǓR>'((tƥ{Rvhؒ&Χ4&1.]9VW|s3p?z7][QY|𰤒T':6Cp Ys $'ѵf{to7Tai2">.fͨqN&:<Ó.͈Ef}?6m@kVRͼʴ]^sB3Lzt˴zdGUDDjQTl10xMtƯ& ;vdmCſPN@l=?ч;;ϳ-нJ=Kjf n>-mBi_{4,ۅץ1> 8j:C\ddt:t ֲ1AU*`{dfB.iE_sh;ҠL| KbR>ȇ:! }]jKf~3s#||eLcGQ́Ko?{Fv/-0c@!T1}gh~Tyu8'6hD7 >cmJ~NI~ÅFLfy3e5 {VV[$`Vfˎy\;Lu3:_%W_w0vcSu >/Ovb6#߰d=f&R=챬Z~uXËwǞL\3?p9 عe~2^K(6M&SbcZ:/ϊ=dg)<}q_&/~B) Ou "99o} ;_{)'"Ղvzc{=|=+""S\XtE!2i q#ۻЋYD3NwetvI?Ə鉹wG4Toe /H|_E'^̜n03'sx\Wi8 !FCEDDDDDb?IVDDDDDDX`EDDDDDJP*AVDDDDDDX`EDDDDDJP*AVDDDDDD]Aqq.ADDDDDD.@=""""""R%(H+""""""UT """"""R%(H+""""""UT """"""R%(H+""""""UT *f wTf.:<Cu]|k?&BK _+ɡT>xԍٷƶEjx2rvk!aO٘XtjfK:z~Gj;j?y}[`:¦0wY9%>Icυ X;|5J5Y>t! &<[&'7] +im!jΛGNmQ$n; fGsXUv3:a#5L̙SKP`OM8Fmه^smBxk\]<;Y;NEýna]edp$yٰGM+;Sf1)V՜=Qg\pؓ;> RZDD4VI^qk=V?ţWZYtivG'ܩtCўۘTzwW>c5Ʊtõ =q'MԺYW<{gҖk뾧_{rKsoB`D}S=vޕ˘ĽuQ6p2&57O2&=~\pu7Ɯ'nc3` OLU:Wrӓ ^zv|o=5~2 ~tU6GgV>}nӟMxDsnؙ^>v|8=¤L{l2\ƂgmrּcRɏ,« ""rq,ٷ! GIʹs uK&%ҡF6 Y1v,yzl9Jz~a/`i!}PӺ{;r~9{? WX[K}{3{+S߹7E`ś>bbX=et+K~Z1D,ѣY9M~@5+ g7h9_%2ë?o9Eݮ ĩdQJEs"ujU|uZ_w =@o F|l8ElZ>ŝr*8|uɁ5m9}iz#^5Ɍ$19?i9!""r42=u>谁W2idG4mт (ÕfCkzнO&Gߞ8Fz!͋xbsO}ОOwKթoFym+ٱY|}jL0.EnP3Zn4Ȋ`˲^LfwTt3=껒l߼6?909V~7e6`?--eK%Lrwc44r`Ӿbg- mv0ҀYxƋ۱M .&k^I ڸ*W99Ml}{“faXF 2jw~ڔv,̹+^n!3L۱aq&8s=FS̞ހMd^ 1|^2۰E}Ƀ+n+ydtӖxn֭8~s3Ecf]y Oj{dl$7zUxHnuՙw*Q0ys,7 8ݛ`mȨG/8&;of !8QLmN>da<23yf|<֊<,ԻGúy)<4cS<άc=1g69t^Û>:k78c4 =""UY@ZlCgGV}p|xÍtՇJU'=-x7|7 4?0j Pߡh;}| g7f/ۣ㽎)0,o&`S2&mmʇղwmؿl>bs\6~ 랻Ld[^ɿ99;Q].u8Npf4h{Cˏ H~76L&Ç!4 M5?hlf| cN8Sȉr (r`qqgW0+z4:x O^,zw7çǡؕ/䪤Sdz8~5?:sjctwbZe{1]J~B^zs " JbSҶ]n&#zz3wۯȮք&ԃv{z) vC7h鿛)8uY $Vffr`2Lio5 ׏z☺ܓ84Q=7O 8TDz6#znl+`'`8s6tpie9wrԭ?= v䒞YfO59u*"1k{&>|;[^⁕>' qr.茳8QXep ʇO&_= ʭ_ef|e7xraIv9ݣgfr&ʹ`}f[x8S=Œav;8A徒jGSwBLIۑ%SUhH#5Q,]UYHwF@ <6v[/54>_ =)*, ("@`/0~PPRog|g|I,C;!@puRc__#G"P4}]G:ns _O1j;N 202eRLJ \бCЦ8kVԕ/CaĴ@v\uϵEF2e)S,LA0MpM IDATiMѽxpg>7g?"@DMO:r F咭#S߾&':3IOgP2#)g5!ou '_sȠxɎۺ=m8SIY5ZsŃ96u4?Gf CI(gSyrnj-L{*p> Vː X~#)s*[ys$"Y>uʉN+ߢR/H/SJ+!5Pn=1 PXz_zm6Űsv_+I[cƍLlBzRkצuTnǙqu(_, ꃗxkm[mNx8_DWyf؝*.ێZrY틌L?c&j'\ƕР\us>5͋/bB=fߛ㪖${8H`ǽ:(t.ܗk'@^n-?a ^~*3^Cjŵ/=@{TQW/RzM|f_B< j$5 @T%Zy14VB i a܋ 2%.z z˱m￾)Fp9]iC֒ox빧({(rdZ}%:K҅o"_$ok.橇ϡڞSA<7(Nܷ3GW/KF:E;Oup9<1m=9＾mPP+gM/}QQ7[֧b|6$gSwL=/S[&>E߿>`?pJ^ŊHN>e m\vN$I7[J$I‚V$I*ĺ17r֘C$Iz$I$$I$),`%I$Ia+I$I XI$IRX0J$I‚V$I $I`$I$$I$),`%I$Ia+I$I XI$IRX0J$I‚V$I $I`$I$$I$),`%I$Ia+I$I XI$IRX0J$I‚V$I $I`$I$$I$),`%I$Ia+I$I XI$IRX0J$I‚V$I $I`$I$$I$),`%I$Ia+I$I XI$IRX0J$I‚V$I $I`$I$nP>k$IUӸS_Ntj1Lsظ'~Zmx$I.mC$WL^A@ j-NrB7&Oj03787s7$Itsv@)Z q#f{xer<:vF/ɧs e͒$I76 ۡlÿc>ꆲfڰѴ|Z9_fY[N`+1trmνiܽ7įbYRM._Q2s>XzǶcTNY;Blx=NӖJ[(}'-ɓI:-hq|Z]ȗ?bg&-I%]ӧ0sq*Ѫmw:\."˓s%I$86mxH8 @$k$-,d} 2o:SF (ju=>Oc`ktQnpמ }`(=njT8 S#X^J|{ s&)?^noVӥq\k2G?mwGGQu蹜tK\<)DGU(Y=C<9Sz3c*XW(Nβxi#I$Iv)_}94]nJe̩L=jW e ,$ q\{#c4}:o_ WnfKߥ9IubɭYȟgd3D/ù9 :5kLZ1ft(ƾ&dE3ﺆ姝)miFg )Ƈ7,e%I$~ GtڵE]An=oWsV+I$IPb"+iρ$I$$I$),`%I$Ias`%I$I:ρ$I$e`%I$Ia+I$I XI$IRX0J$I‚V$I $I`$I$$I$),`%I$Ia+I$I XI$IRX0J$I‚V$I $I`$I$$I$),`%I$Ia+I$I XI$IRX0J$I‚V$I $I`$I$,=o?'fǣ[ʨ]t&I$I]uN+J$I$I{U1BNϕzph9Kyd{|8/@syV};}S)>{Um.gPD}/VTn{W AÄRNaÙVZ$I‡#;0w,/W݈=aeqy-<^j_z/;[0>VnU[oټ'H'.:%ŀzukֿ{'tctO{B$IشyuNs/y!<8vYشnjT x"W/`⤹o/Ϛ?մ5K(Ol 'vmW bۻu99MM"7`Evdt& DBӈ%I$i_ _xq䪳 @| T{|CyC5-$#='P!9WI$I͖YGW:yyF$><嫷_o#{S'~/jQrӟ1^Z$I 3g$O~Cd}?W#x̙ࡷ7o<5r{;$I$&$ڵE]$I$Tb"+iG`%I$Ia+I$I XI$IRX0J$I‚V$I $IU bc(W.D$)̅*R$`reˑ.E$)DT+p 1ë$I //Ym$I$),`%I$Ia+I$I XI$IRX0J$I‚V$I $I`$I$ $_n4I$I*f1I$IRX0J$I‚7}ϼy;_ǣ/'} EWyh[)z}pwdAԈJ+xDF8_&I$o<;bE g\psˠGU:]q'e{S/mGT;[nl7n`ȹgr+uX6p%I$/ %Zưhcʨ죩#;qf|>Q.IdvO"sP|Bm n#\]a_/I$I:`֧w7/;Ge.Lp1h`'|qn|v8&䙬= !I$I)Ŀ>D8w-[qكӆsCMbI$IOäp"?cri۳uJn&8+'!%*Rj g|#]̈́'H$I{6|_ 7rr$VmB+^u \;wH7%V^[mSl~9=z[ԤBj4ԍcSX*L$I.L]̀;^\Rf{G`= _:-^gv \z (f0ᱡhK$IHlB}YII]zXHĂk$IGիO.CbDV$gS%I$Ia+I$I XI$IRX0J$I‚V$I $I`$I$,bb IMU ֧')6.D$)̅J,EvvVQ!I$IK$I‚V$I $I`$I$$I$),`%I$Ia+I$I XI$IRX0J$I‚V$I $I`$I$$I$),`%I$Ia+I$I XI$IRX0J$I‚V$I @ 3?8\~v.?NA3[Y$I~.dmN&TEI$I3$kb,-h*+QNdԳ0ayZ,ךW_ɍˑCYؽp4L( u^6k v.<֖I|$I$'@t݁ rˍ7;iqЋiZ|*3INܵ[;#7x:նh^c/} O\CI$I t~ݩ`~,巙$I$&$'^VOJbԢ.C$IX1ɇS%I$Ia+I$I XI$IRX0J$I‚V$I $I}`8ʕ-G $I s!`}zIAX\r$'/!//K$I k11$%6J:,B *IwV+I$I XI$IRX0J$I‚V$I $I`$I$$I$),`%I$Ia! 7Fx!7Z K[7-$9׼|;E]$I$!5璇G>6欋q}9ghX*.R$I -nW=ni~`Μ%A*zLO҂E\$I$5`VTc:w*˔0{[x*/FvK\eJ5:K/<(ܰ>ygޙNF"k_SVa̳8a4([@Oo_fmmJPe\rNWU`|ym B'x<%ҧOq$4ȥNZ(y ?|,O6$I$/t uSzWY`os/L ~}7]r<z=vw?ޚʊ q30øm$IRx9g6n`@!$&V"1$)B|B[6%Glldm %;onO xJڵ+Q{y}"Qv U{ f2M<|@0[ 9!i]3V tY:&vYBjGsT`)ipQduemjڶIPrͺ}H߁3d^?hHLQ!I9-9o<θuB 5h=.垇Μ}E IDATSϜaUk_;VL柏sɵ`2GDws X䬆lHYCމ8&Juiy-{QH$Ik0Ц))g[A$IFlB}II]zXHĂk$IGիO.CbDV$gS%I$Ia+I$I XI$IRX0J$I‚V$I $I`$I$,bb IMU ֧')6.D$)̅J,EvvVQ!I$IK$I‚V$I $I`$I$$I$),`%I$Ia+I$I XI$IRX0J$I‚V$I $I`$I$$I$),`%I$Ia+I$I XI$IRX0J$I‚VGJԖT<,Gj{緤Tp/I$dq=\,jG)ʣ~rEQ{reRv҈<}bqUܶ^{E.0ze4.C^c'Gl@Y: 2IF`ˌ~L*vq\٭ vM 3?onuⶭO+Fa]m)I$I1iFBR| ?:pCs\EgjB_o ?e?)K6DDloZo^P? n3iӒІov~UvmL/Y$IaAv.<ڒ_^[I]GMvD"N>*FoNGi:~‰Esx䱶LZF!#v[:'kLKI\O*]0I+ P9\u4b#qן&"qYrQ)b&ưdz<7@|~j o\Sz4t>v?#e7U1<61ҧXO2N(^Ua}S]w&-?p}hA-[Wkw]囷Nt^ZRƕs7?dӿ30oe0wJ5F n$I$vF=gn;^Q\U܄jT 5hݶ&)ͩY:gߟ~PS`eХYc^zd/wjd[5HbSx}X~YyS;й>{f.cyoӞJm?EJ|& թV"@~vYy)ح|7_ dd>}`7G}ͼ5lڽqđP6=mwBl [mGlr#JPWr$Id9m"@`[* maKζ#OҔ)"-5ඖsRq"ʔlt-μMNٖx#J (Y:荋HP0ulbh0p|TIGlLB[v("<Ѕ2oy˦ɿƥy`.PV~CټJnbpM$֢z Hټer5ڍ9a#Ҿ]}6y*&>߲$I,N!Ͽ\C~3ҪyN|t)/bU .rnH&=q;җ*&'F}@lR>/frTVlNOpM=O/!xι'M4u9¶]Lfҗֵ?'$S<:m/x$3f)tݖ[a3~rIYbO% W2NnۚV' |־&G_! 7֧2T}.o=$u$wmV$IґX0sĭ]ǐdP\R|k~KF.LUf^ӏ[qO| P_fk#xZ1;מFtg桑1c> *`:'|+tmAW$I GՓX6]\ư{ky,ŚqœC)1|0~}:i`rH$IߥbDV$8XG2ٝʇH 2V.WI$) 8XG)ĠO_y9x$I ypE]$I$7K$I‚V$I $I`T1*5:j%O[ѫ[CbO$IRdqu_ ه;fWE6h۶tm=6fZtj(Q-ӜX'tĉΡn l4Ϳ%F]9]c*zT2`~)VxztiERQnZķc2k(DPɜ׺1 >O_Jvq $>r k}fH6B6 I޺|;~<7Y/@]_'#dzދkYVdʋX^D䋚b}p lQI+q:{-b?G)T|m Yj)AɌ;5yς{24eLI@|ߦ4>Rf>ɘ%-RnH藘>Hh5|m"~csDP}_^P6ͱe(ޜʶϾElw:;ףI?*_:rt=#sV҅剬sg/I‰ EVIA[ݟȌA^m~ϠX.~r61i]jU(xU^XArF,pSbjq©- } mxΜvJ[R~{ѣJƼ4<ן]̖R޷m  UKLfJT^?P(ي%!- W2:FW}2g]#}NIހK0 U P5L1]z"ѫ])f$oj뾜`Y[f7%-L\t>2& ~hTm_dž$:֋F,J݋j0Ѭ/K.WZIA$=4=)9LuhȒ~!ѕ+K K%D, s\$Fxمd53z:NvD~gAbݞư{R(׈Z Wؚ^lLo|+c~ڏ1s¾,^ۿH*W+Ƥwela9H&xsӈv )ωջl,\Vvu Ԯ]eSYzo{g0y[g_̙ob>7/XI熵1l"Xvύi$‹8`L&3.\=0PEjV"d5۱%-v\$$'=v?'q7qܛzPAB{ 3L~` r"'{g<^Ϫ6ڮI9Dy7ՍzH1D1CSHMǾ@ f`'t켫E$0 3Z{YI#xM LW7J"Xc`bv@1y 3o%#[.$A`N |[c1[ @`41-i^^x#Q Y-H7[eBUP{v$cp?|e $WL/k!1eKL6Ζ tgIXE 50TD37;+[ʺU;pHc64̺-U]\8I_L08܉֚B cSXl!Dhd buA@6ĶtFYKOc,h&t.3٬6Q׸MyϮWB6bY4Sv LU#K{l]K[OM?:BM.`;c]T5nfQR4p.^>W.V&aRuUJM":4Mˇ} @p/#}O%T,fuV$:IB9ͻeMS:K#xkT2#I)5n4J_O?鳉хg #r>zIY` h$eq\lyK` öNB{E^ü $IKhj.;XDr:фM!<;^d McѶ'YC { '4`JdĜi&tO_ƦqV?qT̔5lI]#L3n."x!Y$CB6gtpApvAFy Ƣ:*aj$d13H' ܧgYL (.m3>CZ<): !K^QMV,#D2+{Zܠ\(,A@$ fM1|gKde:&D@,9"%PܸUV)zwX(,4m)+ҭTNzUfbB1`o{zS;"e$2p+0ŬIQi0/@ a @B\#T{`v]xI",\*a m9j)wܧx9O\{;9`-gg<1VźvbG~+!Fr=:M8~Bg+{ӿ6RPRKBJ9-/:D EG]fm8gTLTq].?AXivÍ<5BS:bc4\kt >e-ϱBe'\Qא*{sG-%GFQݍsH su=z.K*N"y * nuQwicӃeM R;::yۯYыt(b3 Jk~^>xy߆O!(#{kNs,e;^Xnzz^a[/G3Hel䱗 MMζ)@'&q߱JR\J5oxBBv-O<6J(:^ /}ԞFcDm}?|Ǘ?8u+3;YT!oN!G.u*_[\(=i~r#C{DF&db#`88SdbV>>;:@ DG0wbH,gi&}<_U Q a_Ĭ*&2z/!Kg~JU@ K3q6r?\!L[4;g)>?2 g&U%ǽ,ৄXB,@ 8b @ @ ~V @ ? 4zg а0V?8W IyaCUu<\"mY;eTS4'?>+c7]Ofr~"w<& z6 cCjOeд5swTVNg2hI_jeɃ[Y3GP52 ǂ<$EwQs #ed~[t['u.~=W_D2-fף1ZĥY7/WHAn@&A^%9a pj>Jeֿr yě뻃fՂjT~Ty4>HФ41Wo!qlNk^Jo.cYH^f6 ˖N¸`d{+}sY@}u\:v-㟓"Cvdw${VFyib)9{0 :Oۈfb}$3q L$YNz 9rm:t]w8[ m@ޘBH&Re#ŊǷ\JdV*+wA 0v)s=P(b;OYFJl)-ˉj)ǖR'ihM]Cw3rG @inz%)S9vMo>I%lѓ ?_Y kNSYoRa"cVeD7{|vQ˒#՜9KNE6çW^39$9,iE/]FKD6e"Zg廳t'ӝ4;rZzAFGyO2g#yR P6s\wHD,<}wjY>T/~ہwn/GW_H޷8 @B%fb ȧ&2SF~XFOQ,Y#2K7ڽw^߼e) "GjU'<^O-eժ,7 '}.g:TmgCl3O!g%i.ϟ'>򭫈~O_\Dhc5 U|;X˿7RXqhiD`ZB 2?]9ؘW6/Doò*`L?w9/-eݜ0->+_oe2E{>0=k>312HSX~wx/͇O#@iO©GER f!~ʛODܺlf?y"JWӫgh$FlzM{_OюD6跒>njw~|ǒyI* ubg0EH09'&HN]:G[W$6؈eG1E`ђZJʺqHafQ{:wʱq5\Eu}bg12%3+ T^uAT, ucg0ENJ%.7=FR(iYaT9C|ES0>~B.'0DGU-(L5ZvbiJkm C&&O9J.[PPR 2GG9Dy7ՍzH6V@샃7g svxq#G^qs}hdѷu|~AQ6Ҿ \\=Tt=_O͵:*(J*B*1"c@A*lntL_܉Kg ;҆+2DB".Tg'e-D##9;쨊giIg{g۠]P>d}]K<D@l"3\O\u7*HAaDqpv^J<;#z15[Qm $L-PC$yiI:=,2YhSXTDlj:іR UG_] %}ϝiPAuvSz"ďة7ߥ#B ;֌ݹ~ۻ] @p"4;;[5xz $ ͅ_Qr5yO%J|NuT4UE[lπLNB5cnR5@NFvR#{ D#줪T)C˭ejR*vVG^`'*XB4b o/kpUyiSkitofzxR}HĴ^ K1y 3o%lHl@u9ѷTUE roݩۥuVIat]O_7I#xM v #b-.ُ-Qv;ɘuY}YL* @AU ^ϣL:TvO#={k&cR a$qDɤ?9[юۭ"2iOu(!I0A @?=`!}!2IJuMŇp}J ]Ժ07)\goZoy JOqtCF*%\%)J'Q7*x(g]0cnUNG2NZ9L~t #_\R‘#&V/s$z4a}wFI﷽ٵ@ *"V!a%FNc2: Muν˛UTtd-7^r%#k]! `aqXCMX;Gr(!6,*k'? rZ% u- CB '7i$])Afj蝯ȑJ'uMH|sFNdYX)>:cI$x #2+|+C C gDBCu &V%|2Ԅslo$Bz.işśȰ= !aZ?[;|7R]q9x(o -s샆٬6Q׸MyϮ A_N?c&n6'aaȴ $c8Z#HK q3!#Iph]ԟQXBㅽ4^mH2οM| &wȻ}Ў!$Fx =I ` ՊBm~W2arYf=5H2孼41oߝ1ķ@ ~/nFq!+3 R74n7dd Kp#10sۋy2S\R\pgsTJ"Ձ*N0SSu:);~&5eγ9-"+=;|Ii` K SV\O{yGCr/lʙ+\yQ(+ap݀3 hu "+rJcn ;ꊸ0k=ѷ>\qxHi;F=o$stG[qu7Qt^7=FGi\HGx!4}|'JhIxr‘\ؿ @B\#T{:V5S~o(ֱtsܯ쪥Gk2%SwǢv(oRu}YT6o5KC=՜;znP۹z^ɫ2NMcl&6HQ3ڕf-=J9-/:D<:*,/7 ٺi.?gg*v˲'~UUp vRU_^C䚔ߗO7P-0HCcC\UzZf6Wrvr7HySʩ|Z 0?hY^ꤐ=dždQb*Ŋ {8LS6| qE_^J4lJ3(;oM(||wk>a&\FQYyOD޿p-@ oxKHL-@0a4irf)'M YMQ=GItt M zG,!"󳝹$h瓢i,W@ b@ 6߱Mb¾4@ 3%@ @ # @ '`@ @@d6 CG_v'n๧-@fq<˝:v?^B_w/Y͊'eP?ID-#kʭc4imY}s%2bߣktYI+X> f<ҨEX䘵>9taV;aNK1 u{3V@ ;DuTr91D`02laUF4Ai/9 zf:kzpL9jF9Ȝ_ǥc9dEYVL]L]>],#%BD3q,cIR:WUg 0&ϋ\ptp㷌@}oLe(Xq9;@2$"/SݔcU(,\͹&,<3J=1h);%V ˫YHnJR"4Xpeh!3ZQ8\E~ WݷM1H&WnbxLZ'u9z6_rHGذp&8tfGT9+I Q+H~v ik˞KQZLT(ȸGv&BF/vsH-Aɹflh|<'H\ufQ^%t{|H'eL v~{WbV}Wf[l%2+HJD6r n:NwG=t=A|>26Ms<Dzds5gRӏmܦ~ yU-Nݐ^M5䭾iz7-W8q:[{*}ͨʇFҼ,W*-Ox6ӿ1X%ūN0x9<^aɴ]P~9,i{CFL` sXEu!/gf)O}ZZ>/Lto0%>Ȍ/B;ΉՂ@ "L"^?麔uӷFDz$(ө4r|HO`b7TH`l}eįfǦ-X? O!NO{LR f!~ʛŝGg3~?u`r`zV,6l;JAwI,ciu G=C x!lYly4oq]%lV[4+1,0s5둛cor@/mx/J## A 1KgREZQ3t+e"_|c۝O{9jDPU>gl0}]Vў|I6a;ZYlx~/*wnoC d{Fw˟J/xܫFc}'Ж MK$ft3oJ b@ {9o؞ɱ#oy ٽ{G^%ve8Tki b6WJ顪z| )?>;qdCp‚$JfVb;E+wrÏЭC uPXOfs(ơ1מRO4(PAN~Ʒ#LQkqu_tzP9)Ǎ`s9Mj8z M,/rb{ifj^1מzoc4 zv0dr:RƐ`m,F|3T O"zfΜE@YߵQUm (&s컍쵔uw0R3¨=[HMAj XGWR oIϜB4 *nJO]Ğ1 }+8U\\k/O͵:*(J*BӀJjf$gGd_҅ ]#o(׋يhVFgh$ao۸{DgD-9;bYbҙddAdŷRt[[ʹΛ3>Lw?coQ4aY&:a*cH@0y=13A̚A*\f%8U{%/`STmEV[d$eNtC`e]97UAQd$hFՃ`KmM;g%c ׭ffL^dRSu;pHsD6Yաf $Tc0斁[3>zz%m ؄!€hj=g7`ԛ=^1hI|ϰ8qޜ^Qq$IF"); * ױ4^6܏Fp7qsHU($ ۪Wӝ a04//t,w$mmF/v29̷lHxܩ~6Ơ(X *ѧ'>iԟfBxa;eC} KL(`VXn9'LH 1[dI^&$zΙG@ w`T;!VǮ2 t?9"/hCYvo*}C;-C0Jj eE7n͚ln$aqlltȾ1ބefCAF=ʠ{1vv ,k SS c._ʛ (Vĕ| ؾc#s? vd\v!H4qӍ:_ɤ2q7a{B´ ~1˩%$Ԉ0L 5uc p+b2lqXm!h)U %D!{Jscy[1 ˊuH/53ې Ýg`0x`2z4YAej|pjZ'ܞc'gc` #.HL7N7ik(ٛyWm J'uMH|sFNdyG{uZMзJPL7(b nDw/ݓ_[_cw]YݲowXj@ a6X9L M+h-x:P-Lwxx/3 р9ƭaK^HvV$:IƐ\q2q2 R1-&U?򛽖Ʃ,YI+6~΢H wU* XN`LZϮ!^3,OebVgE$tsټkYF yz,z Vpm/}pV&VZʨ [Ph0&`,-ר6.deF8Sj68.$YV)ES"fv6aZ$m+^w&BGy)̏ BL],g)e6Gue/)KbY(Xh>SUŌe+I"}e -c'}O&SQe梁B%L ch&#Ӆ)r z76<$[Adǟ^Q0/:I16ɴx1k)oƒ)5^23f=;͏ϼ6&,@h6҆Hä)E1ԕ/gX(,4m)+ҭTx=ڬXp(dLvI 3(t=z<G EPp[8DZsurrO|!Zf6Wrv su=z.Kɀ0VɳIܿwdq/ c/=@=mS sC0@=o{ Yrv* PmQQ9ٸgQvtCPE$=KQmT5+^{F4ެRv<=먥`rs<s).WXI@Qij [(˩کza βpp',{%o˯6 {ҥkq{yGCr/lʙ+\Wڽzxy߆O(MV_z9.{ְfjQ)Q_:=MBz< ys}nJc.,^"v_8443*gr-b"u9_v2+{ӿ6RPRKzeo|&w'ٶq>_]{@ #b) ttShIk|IYHy`U=@ gCSCäK>8V|d$cn;>SB˜mEx51cLL'7#H G M]ˆ9ą Sy ߖv;X%0X^rtcGO^/=@ X @ XB,@ ,"@ CБlM~IXkv3U!EgY9{<տx. xML׀{ gs_Rj>|S<)ǖR'isg9F. ,ep/إlYBB@5{ ^Ms<Dzds5gRFaJ̓f1aêĥY7o>QB˗n&A^cXJeNh-;L.yY8=U@ =`wou'afת,j D惷LȣIO}e3볒1\txRGRf|?5w g2WC"fvlYJDžtiޞM๏xشGRbdHgö ;'`}=>UOk7,! 3iԹ,`3)ͩ棋2_| _og㣸fWDb`zj؎s؉%sr\rIqc0`.!:jwJZ vrbh_D CEʼ2LH {#7wCݙܡ}H< wfѓV$p kFWPόKH(u+ {kɪ gdC,V›{, DQUg$2cx !#b1ܓ-[خ5|c+ G-=Ie81J$ 'ߠ6-! ,<Rs_q{ p|SJ Bԝ#񾸗yf 'r.mo4jbvEUG3rHsJ/wl ;R*kli\L)Dž|WRg9Ws+ 8rH䏳NHXX×E~IjZnwI,YM +uyvk1]Ud=LfP|,D >N}Jj׮ϢKX24߸~ӅFlo2+c. s܁YWGwaw}iR|j5HGZ[a4 $ۤ~rFg%v? ';Dž#{;Un؞az3ߧu(Nw`GMnOM41, :3:ݗ&3gX)/)o`7 0 9e'~ɜػSy:;͙߉bhGPAEesQ% ʛife ^qzv:Mag+-WVTBdv`A7x:D扅]Y<;^Na/C_!F5)vjBCPU1o %C G$U]K0o țұa2q`sƹdXםeR]UAypApgOeM6VoW!PN]f{ !O{sîYIƱdAX/ѥ{Uw?ϰϛTw/ތ3k9/C<ןgŏKX0Q![ J@M5**k BC_Z#(GU v5#0 GÁ˰e1;;Ǎ W5iIȬ>\i0A]i O@E~yg26'f,םb gY3}x$>LυMٜܛ ~xeLO_ *|q,..];"t|K0b&czbX47i&P=z)3cq; ΆA9.VK9썿W@e9Zl_:p:mł;a,x1DˁV!K('5p S'ŒBYFBLFl;',kn^!1A'r4FD47Ĥ{F[DDDDn[u2}m2><.IwBr" ?ƞ_%\w,{{C40%׏[p gp߳Lʫٹ0. 3a>gY'll -ld>\_x["./>$9Mg>G7CKd=Xp3 ^ѝ)6l8+,j|eI.\H;q.$,.UAι-lbo+sGY:|o}fgu%k}{ O>oS-B}ipfq'|o7)O?R9zh0|q"ΟLLj/~"lP E{;'eݪ"q7Cpg+ lh gpΜEDDDg2#o+x=,l}W5HDEE~M˨ x6L:j>,""""""KM|A~!;j>,""""""M:]Kݥ=  """"""#(H>7~~| ]EkwWnb-Hnc7#qEDDDzUυ`(`EDDDDDGPAVDDDDDDzX`EDDDDDGPAVDDDDDDzX=̨#lX>;n߰Q̙cGH*Cjl"t{F.z]ޙL{ %)TlzCy.:q3_$F-9aXE# 07]uTsN^BiOddFu'+.Aeb=<>%ὬXj?Wk[J9[Y?(z ƋifQ %v~\vUxWǃWsXv%u_xED:{y&}OwHЉܳ/!<;)85!ƏϕM'؉*䇗=׮𿃙!<7fSE0C'1EX}ąq] {>O 9ḍjF\\#3'[|.""ËVr*Xwځ5ҟ9 o~}oO`R]VGF2GwEDS0FN@`VIm yW(5f1;JGq碩u>%.fԄ!/|ȅϞ}g@968G RLb0ʫg4먮, PNʼnlQ]B{EwwA%1R&ߠđ{C1qI.a ]~u:jk?/Z9I֐Ը .Dw3șTϷ̾o&A|aeʄ Bb j^?.$[A4gחŽH?b87'&w,婵vI¢ 2J,7sQ<2ڗ0)чUywy|i$ !rRӕ+Y v\%&?77/ogqyk{`7aTcJ(a|YąI0>߀ IaOr97a\|`㱫ڍDDgUN<V\~{؁ uJFG™<7e.'kdOm?~y>(dsp0ȟjƒYyYt 'I:zYa_n@Gˬ!̋)ի7 w+GX%ח!fuYD{M\庅H\l##5 ||1oӟaf-I^goVc53'ɺk&ӧ)j:> %] ^DџA'0~r?~ù ؿ~?WŇ>:s%3];ݭqΝPc̆Mhp')kך.2 M7m߻ou34K|/ol8v2rʚWZiBѾ6'ۛ 9]b6X~yy?>a)unet+"75kdiDTkYs8.*9l]vNsa=]ᤦ8r{{inV$Nuye o[V]t@_hw #²8]kK(5 Δ1l8""rqٹgϝ^q?YV&̎GihIdiqÊJ֟ _VX0qXMZni_,T;&%]cltlzE5}[VD d.0zOc2JKKDi_KӟCX{`xx2ΓQ,=uIT[b-X2,a~H/jgw:raʪ( gEB> W^9]W ʬL`̥gJ<O<I%i.of͋dO9߽PӶ4zO%%w`j@zr^cJWjeʌPt:؆HR&.N$LN è t{#S6m(c1L;@/'UY_˅FF0TtJ >G_{*MEZ8NZb%<"[UJ ΝLI'-]x/G ^^|s ,9oȭB8)).ïO?41a1q:Tm ·1\|ﱺx,_yfDDDDDDD8{9Cec8jfP1e2x\|~ Kk<>p/Wm 6;GDDDDDPkdO~Ηo>C1 %wאVǏ/(:[PxGSo#+}m'}_HH7u'??!""""""7PTT4״H+""""""=m6|*f:rbRMI.bnԗ>So.-ðx>!&Ϩ[m`kMQ_Q\N[wKDDDGX} ;c?r}b->lUSxM^EDDDnFy~R7?GMQC%"۲68n6_;va~. n' L Jk-d^9qVDn2Q\`ugDDDDz<okk/u[6!nmٰ*"""ruMښAn+""""""= """"""#(H+""""""=lmv """""r[Ә""""C#:1gNFFZwMDlE\ Gff:Ægddwj"" ì~_LRE[XzOUz^?gɟm^ +EZɌ Yām6w'('Fda PS;9Z+`4+8 ;,?Κ?!3Y9nTda\hHpO˔ Qs{T\PX6.`a_"lNqo_Ξ˸AQ5f箓[51$^_{JK9}<1I9 {N4|pUqv/{OdR*SxDYa^Q#g1}PX-JܡO8V3[6#(G֝xs1:s>=Y &gƨx"M*s9Kn6""QtT 6 NJJˈAD[ݦګL4XNF`0jk͆53tX{βZl>L^%*1}8ywPaZ:VYqlZOL[81ioi]Ȳ{v? q}6GϲOz|1^'.`bZb54_g'ьs/ Dj%n̍Ne]\^2wvam4/cIBW/FνsxP.m*MX& '[_agU aDr9@>}xS7zs6\{_eg2'q5lZ,@YjTԿhdlf+0{%{W>T6EK&am#1w%j^KOZ C2{9ֽɇgJ[|̒#ɷ79>uqԗ] ʇS~?6umWMϥ->;hC^#J̟;ܵ)]jEDn!QQ]!"7f^qLg$ɨtvs1Z;ָq',*48v#"w+G9Y iqRSz+eaDYαqġ˹uv.sh,uL':'N[t49e\ɸbi0vX5Ƕ~BbN1WIܿΡϻ&-OB >P{Ԕgpl˼)aN:N{ )NҏhDH j=7'&oͷr0ʚ NZۛ毼YȱT99'1}{aB̨ Def맗/+AmCFbF'v?(kyeTz]^]*"Sg|~EmZK۪ >7"(w%R8ښa~ {Gb?JNۚ*2,ATښ X҂@}`ly&r+'jkY}6Cw }zIMsXbP6PED&k!} 1tia,R*bۡ\LI#|hz V/FU}G20F@lMA9Ֆ vzmtRPB~AnWQ>vqg.LuGHw " "'4G @Y>h|[uh|}QQ̝_4M|:ojܨ+V""Y; /`O?5]^+3mN?۠F"Οd=b˅\}f!o`VSU˰AC.Ϥ uxDN^@‚T?Av7d: . r4q;I f3 `GZfPӛ#Mjc3=`obb&c?1Rʰ apA2\2aGy"W(OcAFu|IAZ$ yc{ 8=wg`#Ʌ8?{1ikx@pm4p*Q31r7a6~ 6e gLlyGbLr{Y`%zrIaǮ3|@""=V{gq-EΞ=nkv jqW!쭦9s8aәj&~Jr/S|ҽ6 4M4JL瞉y$gE.>@jCsg\V'͝d7M%gkYˮ3QT_]\h(hՅO^ tIsضY~ʦ}t]iWefMY£}qq#mSs0o$|E\9~~zMUz7{rBFu!ױ'6L2e+#+q3u^[JGLlEۺcu47?k<]f >ɼ=U\=}8>30}Y^bߦdvٴ܌Zv~l%662&/9q+ӭ31#o6u^o`$_DGIawl@8DDDcp]c8wZX F> ˫}E&**@=4uj\EDDe7n#D=lnLCܹ< IDATe@˱Ct %"7`fǵ,Y:!""" Mm}gŠu=l;$uIp{ZY[]gA3MƵ)ȊuaV2uTe[Yu)Cw1Rk+bAV!VDѭ`[պB􎞽ZDDDDz.\m~hoa=fXvf0X浄UZI sh[N0[^¨I!VD צٜv ӎ4mkmcTRc66z{{2 bӅrkVmt`,^O6Td <5M@VDDD3ZIk2m1c1M{YкZ"ҩ['z Td#2t #2$Izh{$mZ«"""rkv-sg9a0M*a؍"Kw4뱕\/l q~NpIM\o2h1i׶6|""""7ki1MZ\OX};y?/l ˔&4`ܑUxvk^=k'|ЁMIƣvMByg5tV5F_FCsaa5Zl`VBޱ^Z_j&VME#=?vtsxGU.Y{~@wP惺{@u\Blp>nelYXb 5uiJV1pn6\UfQ}+5?"A16n2}VDۑaa1:קRbEz>64Ēٔ_IDDDDnU#xN;?].DY t=ljtEDDDnX+C~ D`;},n;8hgdn[tb;X"ҠX'jt`ZXρ{EAK_nuK9]HU_ySxձPDDD3 trvDrpYë""""7&ĺOSSb,zVwZ=MPWDDDDkY`Ң{Sbs>뙇;{6k[DDDDn05Wӟa=6VDnk='v4[d?n\55NuSNt&r[9p/U4fc gr۲)5ODz`=k|ؽսp+"""kx-ka[,f"WwFc5zdNӝ9AVaL! ƧAEduT:u[a d&FSCnfF}؆.""mn;_Oò7wMk`U_j'C03WsjXW +"""Ɓ[4tz@N)ж]wJEcwa=ޢ-j_5\V&V:M}2_{Boc`kl9j|6KV.ipzgQ?qjN\_KSml`WFil.؂IkpLJor8jd/rewEDD>/ Mm '=<ܿknFױA^k-~RTq=?XiV ^~Vb稩ս+^:Wf`maO}_k@:Y cדةtԿ:rm.VaYuQ*ѭ=""ҊaȤ-+υm1ЦH'zvmj lr-ǾC{ڭ%yw8\`_i96j_zn*l*MDDD T6of?nu:zpvGذW01o@ Wx\$bKS͉7{#=կ 谖=C;A&Ηx=q轌-4;#ΦV淟bJec>Cˇ_=T%rx6NwET*L h+[pv_5wl _'VD#A+ FҙaNk^rt+iѺoYq52uGZ `ߏ :ayO2ܷ+""7˛/s~Xя)Ϥ_7 XF=6ۙmBikNtWDZ5* PZ7CPv]| =^q0#:hD⛅ћ.Oz~\-'/eZ3cV0^zͯ eo tvޯ|kc Wu~ۛ +VMG31qL,Ha՗|.9D{0Ȯ˪/œ̰!έ}O^Ĝj!?q/!+xfd)' y N{3~bf6O%0i`BRn3yԅg3~m7=*"i:^(oκX,SdrL &|;ymO6f IX,uE$YǮ򮵆26[F6&FSڽizD٢aPa=^g`uwYC^k~XKmT5y~u'?|oGΪY`'?y?_ [+02?ۅxo0 -ҧ}yǙ6 +o^d_n}i bX/+Ws`xk_K`ȟYQ~a[|q? OU ܞ~ń|_AJ9Ma _W,<_.ń'.Ԡ"_-.Ÿ;ٴH/@+Ș_o<%uY~ϟ<5ڝܠii]FVDnm y#?y՝Ea2z 7֒UȄX:7dabݧ7Uʟ*˜S,tw{B,~t6/qȟ!s1zS 5#IXe}חG덝\uA`ӷ}xcբY I$[@ZBɋ\1SrNOy>am|n7k,"݄3izVlpY h>@o绸oQm-yk?^;g~η7L,1]6W類0>|wQ /g|gOE1n2 1'c~(sOhhd_1o~&3Ży^L}?#G*ͦmJ˯?}՗k#_ed .1?~Nگ#wg^x`W?Y{ 5Kn.rqYce> {Ϭ7-D?z=~ט3ΈGO|aY̳O_^k=_H|~#/ jh.݋_ k2)s&Z9۰!,D0H/i8G^"PMڑdV088Fӿ̹8έ42u`Vgq4 bdl\aRWrC =gr.2r* 㳭i"< Gq6oBxm^j,"7@ml9*]g7VSZ'˵p *ʏ Lkz^-^0.ǯN J?}s:E88_v>X9=wo> :_CÊJbC/H g=e7Z cECwxv*c0cynҒ(q:υĴpvuyym?Ud/5,5}J| :vOI&$0DNwsyӕ&g8j'Ĉ[Rl=wKf[e ;o()UUDUMMr)ړkE*wC2K٢<ҊUPuɭwA,Йκ~C,yNRYSoGg)V?7m::ȩ̑6~6uymѮ4qݯ 4uٻvkȴyZ89ZNuPe[?+Ov в:{Zji '%F^,g;vZ(R4kp>V28<zwZOOkqpvk'C+R)p){>`-X'{kR8Sfɕ?%KTTbqstao٧vW\}?P'|zss{߮TžY>MS[ʨ^wOaYXUs1JvrOrt3;Ϫkt&~=OjT+G?䞮k:r` |DTQ)zdTz7;>v*R>ճF]R$ t8>[T3MsrjTwꆂ=:ZZu+/fNZջ.v;-ˮ*OU(}ݲo9xulkZ~Ap W/n]g {q\Rҫ.AMuhWBfO^Шk5ZuWKek 4uv m;W)(4GV>;JZ8㿗J~&\v?R;׷5A>@K< מ+φMVuF6fz9Rާ!EW-֗_ /yܭkfߴ\ OkFL@O~u/wNj =_Νy <Ԇ-˻,МZZx#Z"e1Z|mk_{TU7MzcM{iwg:l]ym߼R7?E?Ҏ5 Ҧmi9z{{[õDC6C.իw+z`Saa7S;R=Ĝe }2xF' ZSjq3If֌ _{mGsvNpSMt{77e$+4OU[!qj׆ĀɒԉzU8ۨ3/ڎʩG4^:o(JloS臭N}]8a\+`L|WKƇ-?#-Ց= ?\o _\+`~7$4&{"Ztذxxݦs _nN]wm79J20abc411RUY=n n׵Z_ihUVD̥Lue tfʑd9ӍlKu1-;I3J &/v_Erlm/+pEyo(-SyZgl^r}^Qj_-ܳjqYMzX W}@@@@@@@@@p%HJ՛ЭxM  zbXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX&o"IDATX`Wo]J??:8LIENDB`incus-7.0.0/doc/images/grafana_instances.png000066400000000000000000002331001517523235500210340ustar00rootroot00000000000000PNG  IHDR#,0SsBIT|dtEXtSoftwaregnome-screenshot>-tEXtCreation TimeMon 09 Oct 2023 12:07:23 PM EDTv-) IDATxyuϜ98z,m꫔%-vI/BEHY*mJ*K}+ -Pc=!g_朙083#13}ύgM|r9@L~_;W@B0fHzib$ ((F b$ ((F b$ ((F b$i1Wk񢕊 e8ʕ'?6w2B}'߄WS/fL_O?WrOh~}hzsؿu-׿vd䥗6E+uS[Ct*&bcUTaUN}}Zf5^ÇV%}.ƍk9CӮ]; 5SX%I?Qԝwܭƿ#IQ>|HOxLߛnsI}IϿkC1 4T/$m޲QWxN&7\RO>7jj";OKXqRn-ܩo전/\Qnn=Kv$Wիz?6nUNSwI͚]sΩ-]DLlIRԪՍ^=ҫjպPG?sg|W]Tw߯{｣?|m*W:W]xX^*3B?Aƽ={%INS#Gòr4?Xu]M_t>}T-\8GTl9}8uΝ#qU>*]l٤ֆ ?xo Abt q㯾B$-Z4_]HM^+FNVXkCFy˖-#ǪnK5w1]rq=1Fe˖$EFFZj\\9؟xXƎR~l'SؓZ+͝7Kի!Tj|)J;|9[Xǭ_7M/CRt}jZx1kԉ:uԴɵt}jlk]h.+IڸW67%{JOOWHIRDӹջL?JtbzufIon?;Meff衇-[%_vim|k$iYj6{S|z[6IVXa+pcq#G8JWZZj.W\P 4l`UZM)JNN$sN/_vZw Eϧp@Qy-ws}5k_߁Zd_0k~{vvIT |۶lMW]HO?_-Z8OnXEK.1F'R*\ʕKPllqY}GoKKT̙+DJ} jǎ-ڻw+&Iڱcw:G*ݷ'hK֭u%uԽ#ڴ7}Gưuf_!R֯_={v~=9$5*mۖ$مwZ5iLժVӦͿ)!nw/kio]EEEOLT$I,8i7xB%`EE:d;]zQJJ:EXy)HnY|-[Ne˖ӬϗE%/Ï}[u'˥˗wG*%%%8=x_T5K>M|RG${өn)JH]b⊆?]%x>X>&)omĝJHT.+FojQE?@]Frj{5~»^<]vY{OWPfH*W}ڲey %&mzUCS2e lw*UJ}8pz]:uumԾ-,^y|+SIb+ E[vuu[.DUZZ`Uݣ7~gVJJ:ޕ$u]u% ;#Ա}[~PݵKwM2Q2J}ͱdRҁV;[t"ǭm۶OjݺTL96=b1|Ǖ/_A˿Ԯ];d۶/5?ݭqk~-wj挅roVJ::m.b;[JUtU^ c/Y5A99uR*U Fi:22X=L}4Iu.nz+iza@_7+##][S)33CO=Ow߯v:h˯pbWz;qkW_'S^w ծM9zjْ$+GseeeZ'ӊ_Uv%$TлcԤ+]6|5Fmܤ;'{kǎmSHҮ];շ_O[]D^}R3dOі:u$)>.7E;1qwjwt@qSN m3)U:!t(M4S4GZjJNI=&!~}heZ|iI>Y_~kF>o?@84mhժT ԯ߶!IJJ:U z|Æ>ᄑ@EEE+תcM:QuתU)'';(4hPM7 ZbV~qd(FCݺUN}I܄j v No(DB1@POp ((F b$ ((F b$ ((F b$ ((F b$ ((F b$ (PAUTUBBEIRR~%&RRҁG Rp2T;AVTN}%$+5%YT2.^qZZ-^@99!fK@A^,^:u_M7S\\ݣsgjYxܒ-ڨ}Ǹ\.ع]?VWצK׻{w57Y3>~&OL7_yo{&ܳ/Wڴ@~쐊VS||̙yg]kAɚx>GG{?F^ԻcF)2"B/wίY+!7ox5zI;6FP||7olmQQjD@_ b=shSjЗ4i=t}0iڴiڵ/k+䋅={GnGmvdHz TN}͙3v:]֜93UN}LJ"Toh)ᷭikO/EXoRׯ7mG*W=ȑڷo*S)P!ʲ,sOWjV%Ji3f/ zW2EFF55r{ԡm*V,FfM3"BM\:z;\Nwo75oRQQڰI%UTSrJ C=ܓ!š#Tp c.b=7w]M}*[E6=6@7tL7F Qtto*(T 6߱{w6nU-[mP֭E!T1r&N j up *T#(Ti U\ETJU ymzu7o:}=_*UZ{E7oW_}!ǣnkr\Z('K"`##K()IJKK-1qqbbԩS_ uNsg+++S7RJ+++Sm뙧˕K.ڴwM7ݦu~7᯿R>&c$o [4{ IҪUjgմɵutp5x>v kᢹճ㕜-(==M+W~׮qktUJL%I:t^~uUZMv$}ך4I%Q5j\:p`j<_{$[˕A/TjjVXҥK][)##] /&6CIRFMG_ +F+B+J~+*B}bi^'}~Nk׭_/m{0k5.P .LJ*kžnW]x%5kޏW}|,ˡ  gHJ:*UXtI'$TW=-B=~T]R3f|hٲ%kwN:*+Fؿ߷=''G99rΒ$EEFIΫq6o-IV}vnVժյaϒ{ t=[D-^*WTH_ +F>|HGQj+VL]>[~޷+qwrs]ڽ{ߚ7Y'\+6233%;;K'l/^$)豛7oԽݩMQ:{޺sWu^y<}}rEZع]7Idwuwyyߏ}ߗ}LJ'&TZv :sII'N]EEEk:} 22ҵrjղvئ Zp u}'űsl_W>ƊWff|/_ lHǣ/XkNuwmw*--oV맟V_֝ ڲeln6hPe˖ӦeoTޥQ|6áV-*55UII$IWS+V糦륗?e˩nKu%uT\y >4A_T11|֡Cu%u|ی1:۷olܹ];wntLꓒhkնmEEyZd}TTڶ?UrkۂTz uvک'm۶Um+77ןSj0jo۬ksW(77Wv(ؔjtuS5V~rs]_J HI`5hP2RK-ցTjذfϞM6\׬3tS[u֪|j޼ݭg{G~&!hgkjj'm|2uE>4^n[t(\o]wgZ^ϧʹUծ]| =~ٲ%z't:5i/_ `v%RڵդI3}ԐŔto/54i=gf_ չS5nLʕÇŗ 5iO3e9чӤ5wn#3HAa:gb$ ((F b$ ((F b$ ((F b$ ((F b$ ((F g OvS[իW?Pgܹ]ժUW>Tz m߾MxUv$թSO=S#hǎm2Fjıڰ@ :2rܙjZ?k߾r8`-Y@ZӋ^ᭇv]zaj׶$hM"@LLLC{z܃}=3|<4{ 9h1vr^v>\կ5k \;s^˼)2*ZQsbJjѲ&?.!o u!pB{=ƃy>r=5Y.u IDATmʕե63%I5jƍ#uj֬)IOjr\+DA5#쮎jO)IUjj_ x۷oI=l#+VmMId[ݷ}V=ۿz8>;33CO?? ((F b$ ((F b$ ((F b$ (;J.~}~նݵkٲn%$TT`lM<$i{S4b: 4X l$۶vcvnF1gsϭW^sf~mwtO4rmڴQ+?ҥwGI&L;:%-blF|N<ةHq=vN P?p$E8M0*<4{ sV ӑo̼<غ}vr=nݼauB(FRvv灓q8ڵ^xa6mM_Ν}=_yNiF/^=1Fmɿ%J(=-r=yh=_9#UDI ڵ)v_;9#Ux}էGoA4bM9P:吆OQ4#pٿ$)..>6իl_!Çj!^#22JwqVHHk_Nb,ݞ,exVЮ'[W;X [~Yr? QCW?xCD:+Wјwߗ$EDDnso á=ܓADDi2Q<ي0F##JW1NcHpcEYb9GtP*]~ϞD߃n}:OYYY'=nذW}kF6ϕzqЫ:xQ111oHsU(Iz ~L|]RTNTl^+krM?1 d[{?1ŝGe =%pc9 HhvP]Bmڴѷv e:T𔤼\} t.-[rBL)_ye!bKWzZz8 GCe9XC.\o[+*=H<)}^z5ڗxcvB$YAX#ˊ,1vcܳ+V\\.|9},K5kRjARdQu$mmۊryu?t89}7ƒ"CouԵKwUt:3>ϧ{?m{ta}$mR<&0l&AFC9 ^p(c SC8\3RN_[F%d y HcN ]¤$ցϒ/j C|?PTmi ):=l"-[l[*v|h[1rXF=\pc,XɘPGSe+ e GC2r:2)0Omd9;-?/",,RXb$@1%YzW#VR(n%5Jdl[4%-)O d#Xñ)IY"ԑ lEDX*)q m9Jelt*>WKedd+=ƒN짊b$@1<\y0=^F2yh#A 1F.,Od]ittY91R˖sn|_ =ZӔ34g8yEG_)Z5rfZ#ƨjj}uӛj1^$!CF颋. ta#pD8[Q% m#9%/uD[Hc$#׻LwQO?ON.Wz/Ir8`-Y@ZӋ^ymw7i]7ޢ[6iÆ_&@x-2&"ԑ Eے,::]Ƕjd@ioƍ{[;wn$(URMTJKO׬3$Is~֭nTzi͚Ur)2*ZQsbJjѲžmlGLe:C0qlrt<4{ )ciEtT1;F2pJer-dYTDБk_hfX=&kԨG8zլYS4yDhZӧ>}r"veH8[8=mdKHrȒ/l9dt8ˏ7ݦARZzx^-=~K>uVXb}{8~J!۶mU]_t%0a17=~ճ>qc'[̤s$ u!pB{Ubr=yhȲ-2):(Q˖XOkF%sȒeԢ<% ͚=]?JzOOX{v'jHN~a@i^Mnh_\IRŊe8I?Rժ5|ۚ>@ <A`-'o[ے2RًʹeY'ju_Hǣƿƿs۷o#W3FF-ە^xc_+ W#Aރy/;Ғ4ʋ,TgKc}qGF:1p(F#[2QAHNCNORKXJ1J)acJ)cy׋Z<<5R1 8ؒa6 2r=yhyda? S\GG}HYoo[䐭9##Œm$[+-ԡ (̇ly׊t8 Ɩ#$ï E1 4m#Y!-ω1adl[, l}1RP462yh#A feۺ=׶W;n2FazP f,ɖYBiW_9?Qf7EEE_9cN{:w;T2zC5wR}⊫Nz5ki񢕺'+SZ7\R4)S$iРZ`Zs,[oW͚N*9-)Xb2F:Zpm˲dl5#Ž1cBi1p ڴ7rkk ~m_Ω,Iz*]v=X75nLZSum+ǟUJ{6mn:a[VpO0F;vl:%jT7uhgicdI% nW"VlQf-?8,KrؖLxҦ ~Mcw>MzZh\moߦ*UI:RSSu1j٢$Z?LmڴA樓^gD]z劋;eYjٲ6mͷRٲN8>//W/9T*m^xŋ lkD˕#%$j?)]?X(F#vY*i^Q:\~WI1lx k_m6_l|7nʕ+#GNz,-_T-Zm+t`vQhQ󎻵z~1m$md;m#џ2,cN.(F:?ה)u -5{x]}釾KЃ=Ud=:w_jںuA&S\^kYjF6mk޼Yׯ@͝T>_v7ެ~PgۡZoy\E"n[݅xlؒ׼%9<k$o맵?+iΜ'֭z.Ubb.^}]zY|c6U-b5ϕzqЫ:x mcbb{%K ŋ:9 |<4{y"!JDЗH,d)"*ZpyiE闔*effQ.ȒPzrZIRbN-Z!.xu>Wk-]X.HVulْdffuܣJEŇ+@iiCC9 Ըxd*5&Bi))xRzˑijG)'+KْRKTZr9?Ǒ"%i!]0/x`IJڧ2eʪqkJTVf7hv:CkK?JeʔSv7P&תnە^s%jBheYYj׾P. ^xŋ3Q1)xjut[RӒXtZ~mْ1rXF]vѮd3XmKa5hSzz ϧIygyRM8zIs?ޏSno^C^3;;K+V,Sj)1qg1>3iٶGGM-gȗ122R`hK(Bsg,##[Y*xHP3$5?=t_FFҳuf~[غuz<~^?tK|P d[H?g2yh#A d[? ʓ9:ј6k%siKagLv"|331,<<#c۲ddy+ab$@1m{Y*h[2>FI6F}P}Iic$yd{ܡȖdgC.y>r`Ť,)-gcӴ-YH˫cKƖGRӄe nl['OHRRlvClymeF4wE4mFFcdC|o pel['N琑ll)c_.㝪]B1 slMx~c#"AFC9 ^0[,h0-ږ;͖rH/Wa#/ԁ l[#f9V҂sԼy85uzkH%# okR[GO-m˶ui[n?ЪUʶz31 bbb>33C11۷jS$I7ut;ԑ bsmEɒ-R*el6iir2?b=ڳ'wFw!C^RffJ*׾D8KGX_\tbI}}k\<{)s?a22jp]%׭u]zkP{Aa2[Ig]zdjbYeym(է׆;s\28)H7gx3f80~9O:O>K0jhO?F> 6K}0 c°E""""C>""""RqlwqC#IXZa#aވ}叨MF>"?:;n{DQfόGr!qeEyƏö>͞{Nko )"""R߂t]tW_gs_y;sY (kmɕ߾G%w~!cYد8`=΍3~x'7YO2yd7#&O{ H.p ya=X3ᘉ  jן-+[rqrs ;m‹O-: IDAT= d䷯EˋgL?<#FpхWpOOUW}9es]w}'%Sr9|ߋIu??ʕ/wN=yi-0֑cA0m,[| '|c{nEy,τ İԇmi3ؖ#(b#m H[9㚆r/e>/WUW}=X}Jdi%>͌Gƻ߽7{?\noogCgԩnkW[&<1Yϵ1L2ż6SkCq]S=9kw<-Kvߦ}F|?c^ˢ""""uɆ욑]0yG֑|3r?s[e^Yx睷s!xxB⾟ͦ76;̝G_Ϙqs_3gϜG1q䧹MSDDD/ iC4ؕ _!jfH]r ݴ?Ox?EQyKrf=kʷu ?ēo]ڵxT+mkcʔXv-W^y֨k,y?峮Obݺu=677R|=o5$Ͱaj݅]b^{)浡. BazKSAhFa*YQ[sd%}S2RDDDОe㯼ŒsAR~4id8fXv5-G?&#}ӏb͚U,Z\.ׯ>FQ_SL_v$}/^أM6-Az[j݋]b^{)浡*ޑ-o'7( Kon֨a3VMiԉG,Ou:;~lK ӏݏL&IuګZ^{ƌ?|d}|cj^kѢxAqħ,.&Oޛ}y;EDDD9p~݋c䱛朣ɻ4맙""""uCѱ؀ 4`kݭA-܊˹8 7v=fͺ7xY.ओO .c֭<\z^Zĉb}s./̳a',aY!WXVb^{)浡;N.gViۙm̕Ԩqxǀ3\ZlP_/E XhAsK~[N9>cŊzvpM~KEDDDZ6d3֕5xW1Δ-3x[M]Ho]N 킡flט7Fczme#qXp.saL@Sɮ:͌1c.OpZ%oXVb^{)浡+KZБ4V)HfP/ս.)"""Rk. }!fF +$H 䯆"h(ݿ#Y/Y1Ywˌx;9JFBJ]+4mOa2k@e""""yQ~̌j'yKP,L1 ŽP{W85NFA޵}cdBb1eQ2RDDD֜aD^)9ըC""""RM\?ʴ0d62h\ߟU#JFԚs԰8Em&Ըcg,XVb^{)浡==^{{+MyT}5#EDDDj̹4ljĥ6]DDDDjch{mx&eRi}%"AHڳ!Yr;:j3g#8|kDc ÀlX pjEe""""5f΁y| ;-׭+w2XVb^{)浡 hG$cS MQLS\W_(Z1_1S3#EDDDj9,4M^ZaqY;&""""U(L죝%p&e[JFԘs+ɽ}j'%7d&e$A5L[DDDJ]Jkdt)U׆^}ym(2dDh{#C kÀ+K6q>>F5]S6)znHӃaNJ3)=0\J3#EDDDk-J7X>[b@P,O1 ŽP{Myh{S[KH6mIٽ}yB>c5#EDDDj!AkHU^HX kiH9:fR)G:Pu7JFsF] 2a=j89|_XM:v4E{6q83>U-"""R{׸O; agXVb^{)浡9'$͒c,@hvfus͌1+ujC]Jv;n茎(d&ejNgE %#s<̟D'q\[sωs7o &U~}MŗѦe5뒈T9p$qB1sR#& F޻ 4^3rPʴg8\.[0Ŭˮ?>'?9g} q1Ǎ7^˨c8OqMrԑa ӟ8rYW3RY3ס*SkCq>ż6mvi3ɧٶq#{)|8̒쮶Frc{k :S9.#G|[7Tn$3gClݺ)S "2 d2 Dqsù tEDDDA`؄6Qpſ_͜G;~}6oҦɓfc87f8{Cq?fͺG?yO0Etӏ;ᎋ}f8DZA׷v V,+8lG9S2ol%q9))浡Wb^{%}0, ȈD# ^d?~U h2r)瞓x{knna˖%Dz6[Xr_0/9~*'Dz l,;]xVƂ?x`pM.ޞeٲ?q9ֶVڔKk9, igN8ᫌ;(ς0awAT\);+a> Jۚ%cz ؚtN;6q8Ͷ1zcÇn_Kug{uy7o6?^{ini)vf)e ;]tпXz8yqQ#FSI=I{Ѩɴߓ1#ٶ6@l}3Ї6m[od3J͒%OvY=t?]Lz0ֽƫ}Ͼf2 /3DDDd(1*wG; %#*xǞ|khmm/_:=>nWιbyS$s[dd6P~ݺZwAukPt<3,HqF,m?ۍ|cFl%mmFfKh2lܸewk0rH>izxxԃY|_̺u+mLW{d\pb2rƌ;w6x3syya-oJR\_F1 ŽP{ YdOm|5ē(Fyt׫mlK9FE~|wG/\t _gkoΞy!_?+8ӏ䗿\ag,]g'A%k촜allcqP\ \]m`EQIJe$|GJVXN)S`ڵ~\ye&#J>(T.u%~*$˫tFQ)/#պ ż6SkCqy#( !MiZ36>,X$A>ilj.mӌrTR_1j,JhX(s٬9Sdժ<O`ao| iz8]<}MqA ݂s7A8_9*pA n ? t: CTT2gC̘~k֬bѢ 1b#y{xmʫt\cL z沙2PܫO1 ŽWҙq[DgglǵiF`pl[{m~0#ldlw7nTժKF[b9_ %V\g\)l\fm\xك=p n\%֬K57=gfrfGa۶-sM%-Wo;~5{Oo{k?XDDDI5Whkv;Oʰ9ǰ惖2AЫ4h]wM~ qNr\r79|_OW^/dS͎ttS8]]>sE}y6l=/,{?ZDDD/ݴ #t-mFCtsFl8RAa dH#eeڅfuMׁd|i₝{brW7SiSSO^us ה+O/>?WV)U׆^}ym(c'0aٌG@`xsA )KvjBG*,-4;mu#Y|p; ]Mjzn#"""";5sK9s5fav3()"""R5{V`kXU-Yۉ˴EDDD'o{|$O޸JҾ>gEʴEDDD\@{q# #FݴTsr.P2 ~ʛL6ż6SkCq$Un탠G| 0'ػiR0WdEDDD|\a7`QG7oCC8%6Û'U՘; )vA2tetm|c|NH*qA*أzfTcZ5#Q%Q;ŲPܫO1 Ž/8.0::Chkr6G opֵԬ5V8Tm;׌;{LkF )'];Ԕ@KN!P[ck][e3]Rj֌taIIwYܢ؈qD0KüᢀƼem0]=RH. HH1#hX3V\OvӮϱm8ŲPܫO1 Žw>#NI1s ]ff# ~ļʗE3#EDDD$#N'wn,r=GubQȐbVځsx0cÓ$c JFȅ$*볚FHj)lwXc҄kFʃGn! L:,""""<.瓵]>q{`Dޱ%>o`L[DDDZY6j=$̬nrKBg`$+f)դ׆^}ym(^Ks돎5:0ic(vX`fdࠤ632L}\H*q.0sF3+-1YX,&$"""";\ [_.Q x]Lvcu~l|!lۈ=g Y7ȍsf8БbT. ̓++S2RDDDZ\toY< EM8 IMK8@~./G1 ŽP{{+!>[ %#EDDD̃yz|L2X#ޙ+f ĥ˚9<-]61l6k}jRkCq>ż6R4#9R@c#l*m bAdDuX JFTsi 03ʳ)sIRSSF=7emmjݍ]b^{)浡j*{ 7}7C&7'z 0Hi] fGv#|n3A׬Ɋe4$qAy9eJwqf\@a `KhΜfF Af8`++0,IjȐ 9#랳sxV:]S#Yj1Y4ѬP\ܹM~3'EDDDdd8cǎ [23ʚ g.RH R z 2q۫\a ?L3#EDDD=`x{H;e1x͌ٵ9&?0R-o6xW?wo䑇m?S?P'Oޛܘ1? iw~S5*$=GqM?bg3F}t6`;ddah7m]sXWN4Z/ssTH9Es{Z6~ɷ}Is6wߝْ],^ /<۟ϻl۶g̘qtϘ~aX|}mj뫯?J*fڴK8/""""2X"3:C(g |i 54[2|>imu ž3KOQ;q*uxpnk=";,6lP<)֬YG.ɳWv}rSimY<ӦβpW;v\GQ 3a7 nAfAqdw# `dZQ2RDDDJ,0 W>᷽p7.+iԚoLr0l_҉Ť.>#+$Q;zXƎ_緷䓋8cS̺ufͪ>4p3EDDdhH g\zɧ y#񋒑""""Uv0 \e|2kKާ%Ux{ߝ?9i^8M:/B6۶vsf#g8sg3gϜG1q䧹EQl=QidcޒdTp֒jFkFTKW,hp8DKիWqw>:98 yvŊdژ2֮]˾ǕW^\{5#z|֕vI[Gf[ͻaI](浡Wb^{Rw^eRH! )io|;6*Hg<[ż6Viq{FF>8\I{c4aQܿoDG0#sϽ,?Sy8pɂ. c؂\>;a=k;8Opԑa ˔!IgʓnYnȰ)d|ɝԩsa{r_ɧ>%n}o!LdgJ-ZȁG| `gDDDDެp[:kmpd7mǐ,qKi1̀Ό\h> llK~[&MzK'"""YO_`2Ƹ˖cѝ:1'X${xA<d|L<%Kw'}(9n~ш=% g=FN, O5rG#q5k67tmoC?ܯ^b9zb7\Srꫯ(>?_,>""""̒Z>vnya4Ib٬AfERiNVX@ss [l.iͶܜ,r ο`&GYgȐcaq,,]Wr kFexK""""2dD0>uZr>c+N,yϾnUs3&7|O9\>n;Yn] :]5o^zD(&e+Q[:I5myƍ-6>ϖn'wflo5"<؁Ya|وLFXOǜq|s_v{Gq4˖ K0jhO?F> 6K?a*$A}("""3p8,l !HBjߕsf*bp{, cKcFo3|_y.>A~~\楗_;n \6|fL??C9K/;(J2O`as?෿}=uƝ F\ςŘ%#L6j:#!s7m A;:S +F@Үi$+"""2\:;ae 4 )\jXX,ʮmgV-Y3_;8Acz7DuZQdȠ3\1I&bXff#)""""CY>fFf3|dI6)=d9f'6""""&.~-o_7Oa;m7 汾V6J_^Ӝ3/$4/gkf.|Ϸ-O`aa)@pȥ[""""R}y3bz so p(($&됒"""", Oq.cYddqVd{)"""2D7f :|n_X!,ZQ2RDDDd9|ƊN3}xjWfPa)"""2 ̣sDK֗tor[v1b֚ƛǙ#q>JkEHAt A ݄3Q;bWJ7 IJe#EDDD(vDaߥչ#2OLHhŽjb1eY#JF 2|d,d5H֌Ģd&d\@0lqtR5#EDDDN|؛LdDZZiwr9/;_'8r3J>#p)͘qtc;0 oVZY|}Wpre޼ٜKyS|gZ qTxbs4"MMtƽ?Y+JF 2 3Z(̀!ѥR@WٶQ>#qUho[߾\.ǣaҤwOlٲ_I&M|Wg <drSimY<ӦβpW;gL(ς0a3EDDD,ݪCFfF'/2u^ !.(‹;[2mmMv.&-;eD^z/,];[gg.v|2bD+ƍ>1]L;ޱ'']<䢒S̺ufͪ>ߟ4p3dJDDDo{^^F>ȟ=sNdrY 2yO,ZQ2RDDDdܖI 0_^2̄t&\9zi;f~_3 ǟ/|P(sz7_}#|ǰmVN.`?r䑟a='rʇ#}ݹ3!fL?5kVhrM<ߧH),^Gl6[2pό-=[ֶ](浡Wb^m8GE;Ӛd:i"E*EV0@GCִё G̷mJHPȠ훈J7p!5=Yq'g]Ş{N$i#8o{ۋk8YO}hiƈ~ħY]8w%/'O _?+,a7t<^sѢxAqħ,.&Oޛ}yEDDD1{ uMۍ AVhf K @8b:#糘!8(ݳ@0#;#Yf:Mol+.S➻AB<6l{~,"""ZVe[#cH#E|d *vu6Xk'\v*zp?m'U<ꫯpEg.[b9zb7\Srꫯ(>?_g 3Û82Ei m:Q2RDDDd5 #$V:H꼵ҎPU\ӧ~b{G=╶_HdEDDD (β.FZL2GeC$d#""""o">y !8WeJF 2jے${Y'X!΁sd}@j18%#EDDDO9>l6.\"U-"""2‘"Aرš9p)af/b?wEDDDdKs)4EfF?Ͻ9)f]v Ls|wdr1Ǎ7^]w +^XƟǁ즈HUh+x,Zܹ:pAfxm*=B>,u+ kF644pʫ֭[7a=y*[mc3!nʔ)GF28b9t~""""U6MD9 Kʴ-8.% lj-k|ݹEDDDd(J0? }3+,93! ؀?˒?e^{M.ɓYi~z|׉n3/曯# dEDDD.޼@ g]o| # Xi$Zx9ΌbqG"" ecFz7h|z"-lٲM6Fss +W fpQUy Aur9,Gْ&ɼHs c H'"όt,nys,pX{1A; =:fPwrEgͶIq%oeݺ*>38x3ol/~w577R|̴ 3vL :ԃ7s ߣAMH5gǍ+6҄Fkl%ȵir0fX tG #N5ȶfn(""""5bz>-mtNdb{.\ye^xW/Xv}m9Ǚg~oH.tl6u_]tꁮA]qDy,\5ցɯa8(ac͛W}c""""CDcĠiiFr@k|F˷u z#˗L&C&,tFGG}ÆtBrqLM @6 >unad%nM։I;ͺzc<+{^Vp6 <3̄ {pEWmܴc=(s٬9Sdժdp>~v8ofڡsݵ7o^Cۨ$hG8s m3"-\s8i !Ya&cInټJ؅xH%#EDDD R֌l>7s[8$|/DΣ!px\k[qF+%L[#YAdA ,&,0$Hq]w1=0L&Sf׹umy""""UiqmܐDDDDvhl[Cr ١'q` |_ 8lrsr8)h\n׋⣏?iqC| gСz'^z˿AOm8`0dɦ#Fצ \}#gs6r-j&1`MKSY6_ɦQH:d&.oOGP rSЦ S1] vMII~B-&]wK>lٲ%~\Y:uڕ{iJJb,XW^uEU|={q6q,2ϥy^y f̘J۶u!q5mk\,>TM䴒 }g"""" |Մ8LrH },N"Mz_0#^Wzq.Hы ЪUkyg0~&];FG7/^_/MGE]㔔曯sуRz>UVt3r '%׋TΧ ֩x0?LU1RDDDp.&\M;(^ *jh04&ƸX?U.w[v?oFg~.?\yhW6R?kIyzXuL Aj?~9#EDDD1n?y+` o&2<VyE+.G,[nݺs7ӳ[<#H~r;YjU؜rrsSMVAVTSߧQ{ڽzA" Nh4Jnx}' H"Pܐޔ{x0=\#5%H?A,R *F%'vX k"bK  * 1l=#=Æ_@Iɦw=%$\uroM7ߚܹӬY >P(a>wT{_I^3xq5ޜ8t;{'oanmb55|un gH2&~/ps[c|:5D 6 Xzi7 pyrԅ^N-yѩmfmٯӲeka(oy7xoK.?wr=?㕖[iW-6`͚5<,j/f$/45_XC85OS'뉊""""u(8"ͻ1O`\/-Ƅr\ N7+9n=#=jx|nѢ\j<ƢE 3SδG59瞒q""""۞!*9i.{JV)fH SN4dnɢI2F ݏoև8?6޸ 8JDDDDv$V#_KX+1i"AHe\DcWUm7&M[ܼd1|N$[ َ{FHUjrAbiHe""""u m*@ZkJ8ZRB0.&Ʌ+Ndmdx#""""[m<ᶦ %YγU惛*F!c\I_|x qܨClm[VXB Ydxej. $ 5jUn)""""u8ajqgD M2htuj nFH{EBP&E=)mh')"""R_bdiA%xƄ $ִBє'AEDDDd;f,60r`0U̦BdH)"""Rh.|`ʢ]|HŘPeE>qJ3r1_ lVHqi6,*F%' Urf4N 1C8˥$)eP>f^ h"e6M[V 7$HHdB#(~16H5ZZrxyG=q"""";=( 1SiĆBH:d_TL p#!ldm6Lga% ,&JDDDDv,6EL+!BXݸM-ĩZQDDDe (,8`?J}-/ BX YgwNj' """"!tMZp 0q6Q+"""Rk~)cq*# &&ߖa&' <]scLrrbȎÀMHScTU4\N, sY0m^:u枻f}t)\=p]ӹs Эuq""""u C6j b_9t6~ nqEoO+:.xqo2t""""Raa/=X5Vn(y8(p+Omwo;3gN d2nr~w$>$-C cW mcB A߀c7: Lb}zPP Bb5ao16( WBC]QDDDD1$ꩡg5.5`+|ZuCC"Hmw1~X&=z{hHhӦG?QO>R("""Mm/Ɣ-]lix)ۦ-v"nN8"""""u|q.fGá=K6]$cm2$}٩ۼ}\i{.]??>]Q8 \vEDDD2zx|/|My]F\M&r 1-pvc1&h;g oఽ֓Syv䶎-,'* eHˆ CbbrrrE\{;x 6 9@r DkAAq8k-^ʼdr9#EDDDv |ܐC8M Wa 8;A4+/ц #cb6IטU~H۶qx_r _v Zaʔ̚Z!'77XUOmYf} CC5@ 4 EhR9G1DiޢŦHH8DPbiڤ!! 06DpljĊbuqy"""" gc<Ky‘dOH8I544+F.^CN=3m[n{sR1E<GXLu^OA}htAàP;y++f4xǺگ QZTJ~  e%$J=֯߀Wor)-)Z6! ?' .!k<,v-X‰Xc'a̛M4e!DY;x1oGWvC.R|u+N$"""00p1>E!,c0`-l*k~":d3ecp@',рP=#C!(KB!p(, UI4Rd2hP^~i2 ছ+niӖ6ӱcg5^;&S)"""MA,`$' /@v {'AOTKM6[dӶ߁zPj[jJ.]R9F"QN=m DDDd  B4b +c ðf7i ~(#EDDDB!k=kN ,TX&`}:mAhٲ]x9?|Ojѣׯ??Ęc[Ǥ8pHAC@$!1isш9/ G6 bض*)"""RW\ % ܐ5I~DŽ#$ńéDЊ 0 z_*o`IڴΝ>3_ e0}d?-#&MǠCYt >x<^ az0kJ1X,-q8g*..S=Ce%2CY<`XR{9c}DĦ2- ZyZJ3T ;&DPHK4'uLr'= ܖKh,UG; Ɨ `D"D"1\=͚5C~C$]ӏ%T?ϤW<hoqvݝ=ؓ^eECk*ń] "#JXhbơ 7WTHFp!8~'*jz4jb\0Rpo1@g~C9v gGv)Ç_\uLaa!ӦMfܸgii o5Nveٲoj<+k֬Y?¯NvZK%q1! 7˒c1Ƨ 7ub4LQH:bWa""DXL)/ zeN܂ #X~ >"fqчl1? άq- 7}ϝiϏqks=%M7]SٖrCr*C!!0v>QVbS8,-"""RW,ܬ* 1C@#1XbmbBPk-lC)""")-5SA9[Bn`lz 'kT#qe e~{8Lr2'dpjv_^A~#x%ЮEY e!|0%d|0BɆIYH]qX7qpB.w[ArܖYZK8+9{sAiȎ/0 a@4VM*ZhгF*# /(P x%~"`Mlu)6C>:s""""2}Cvr9(q,XPzkqi%I#EDDD@F-T9S>;Lr„67v$H7 eg*DDDD~b-x (WٲXl`Ǔ\'=ZC&چ9P[H:bBċ+b'b>Aƒ kOc>$ H2VKRDDDD_ppOD–PvʱI~5%lB5a"y78g' ^UZC)&{O#.5p.Drb8hƮCDDDDOXM&VKQ CF~4.oXW"a|kh)w}웍ڕ"+-)"""RW|'iuK)+<驡osO.`SZ Y6X`6 Z,XFߙC6nE};6jS{M{-1K= lkI$ -&0! { oWg Y<񇯤W5UH=o> 4&]kp=Cr汝v @4dԢ/֜L ~ЭɆDDDDWaE /`T`Wn@0bB4NDDDdG X\S}u!|sµ1He""""ud7#&q kTXg<恁Z0যm%8""""0X'ql?pB쀲%+vv4q-Ʊ[KR1RDDDx%Rʊ+b<"-,dVbmrl`1& 8_CqCDDDDմM c$!7Nꗊ""""uq)ZU_,a[zFz1[ ,RZ 2x%""""R Sc).5֐F yc >AìE)"""RWEvظQl2Knk+ }*JBm`p6a7""""uH.PSC$'\K¯PXB4m0W/W1RDDDزuN)eq6Pc ^'XEp2Yb Brs$ 6"""";ܨ $X| 9 ôS#oe""""uĺQrBY9Xߒm¬/GS1[V~I*8؆\}CQI(@M5<`K4m.04) kR H1N[ZD8`Y"#ؠBh[Xeׇ"h!]L(Zק/""""dE Y eh\JɺMOƲ8' """"ۚ"ؐ]!N>[RVa|rAގ>!Vm+$WOaLrRrӂp/Vc3t--u__q IDATD9j9lJDDD籁%'QKiPT1H[JCEj amX199yz^w=^óEDDdny/|?'aۤmPXK.\pdi-Y imK4pӂ?X1hW_qŵ\v5 ƫ3y"""C,rA8YC"AL,Y9gdƊX1M4MۖטU~H۶qx_r _v Zaʔ̚Z!'wўsV5$uO?݃AjQ2>~qPh9YF]y }mLLV3Z pKӬb?gSVVFvvX1yyVY9f6P(-ࢋfժUb+梱gc[gH$yPgڼ~kW9릘^(q/}l޻l~U 5O5+sɉe,_X1rERancRóыx"_z-#c 5Z-X]E.PtAà 򗴇նxW̝A/^TI 8h4cO^<(n%^뺙:ML63<[Һu8\LPǡkc=o3p"""";7. /%Ksy ڴiKpߧ1:ML2֭Ѓ4ySNbUWW^k֬fW /""""g4mRvD?݃AȎNS{=j)"""""""""0"""""""""*FHF)"""""""""bN$''z+^pЩӮ̘>ױC>6:><9j ^ 윌\6jܸ1w/&M/O="}Y>N;,^zq2/8N=3;Gy g2رӱcg?L?zRluE"a9M}=+}k޻jmcKb=rku~}it)mmڴeĈ{4<س}_Ґ)ڵ^͝#oPZV_^/PF]7 sܰ%WҮm{ >Դήrchٲ_7 `[F.~\`)L1[3[qvvp=BX}s5w7{qGq_nw߮5{\|w427m 7qrwq57ӨQ}QY5S>Y?fގ/CCC,V̗_~i8AXkI$⩯#_/bW mo^~i2yyу6m"}6Rz;YYY| GMmԹsOL<^׋Сퟜqy54p?0VȚ5yb4p>y7e g^@=w2~X&p=z8?ozSY?_Gcf;weŊ),ܐҵniqr҉'Xk|njG׮]?sEE,_:mWjj~CDt֝^oV~j{wV]_lѥmq֦ c@=eTtڵ O?ʖŗ_~AYYi'>}[}MӼY3v)Py5S>Y?fގ/t`RXX-+&''}N~}RoԒgL>rrrٰa}nkٞFG?wGᡇeZbjuwFD)*>X1h{s;:y>b!t+ V}rrrk!}9?2p`Om*|z(++%|~aw}8w/-ZĞ{z.//6mڲlْJum]}M%lRMCsS{3[諴ߺuj\gۏ#ne7_`կN{nHjz>UVC KD}ϝi@?3OfP~p(lxuk{UI۳9Kf}c1s2\i'UT=zS>f̜1˯~ՍF8?oPZZyވUumr4kւC pvmӧO?|Lwm:uR~jO;&ryn݆3N?)S';g ;;SN>kװ|2 ̛M6e!DY;xW>GuǑm߮={P݃I=q睷.pI4Py5ˆCdã6;sh4mekAnn#:}֭G`8#8~I~>6眳/ };＝:Mumԫ~}t+EE:u2|n;|o8Uxsڇλ&M2r*u׮\~t܅%K3_gɒŵ:$=3y¶潫߶* àC˫|sEWzJ7nK/VNN&N[i~HC3kGdQY;RbdiHF)"""""""""bd"""""""""*FHF)"""""""""bd"""""""""*FHF)"""""""""bdN_gذ4DDDDdzu4Mʹ0~&6֭ۤ8 Ϗ㜳/gW}+.;9mLVQV{[6nܘ_RL>ۿIgѤIq%[1k~6{k7x87tqL8F*~dgTzM>}1}͛خrig1W0~&wzGS]~2Kݘ1}Ӧfټd.UÑ:2c*_sޕݬYsM]{6~p5={?ΤxvXN:Zlūc8J}/1&)'z\ӽx/Ǽ0SO=3m_{uJ?'{O N>x8~0|5W\q]ۛU~7[qܴmݻCUo%p8ŌSSg̬x礙nݴi?pӇ_ˉ'>H۷?/Tz}VV6}(r tlǽҫٷG/r=v{k{u;g9Sh߾#'xJZL߾?sytRg?w! P˖ ӏk/_۟\ɺueT|=iԨ;v޴m|݋/GyGUI"l#8}Lo#++ qyq$ڷ;SRvƌ}=.tEZf̘ t%ڱc'{u:gj> <9j /k_n݆<؃=rq3hhڶm۱KYcu؉8o nO 8O? K/Nf 9^oᔓ`̘ }e*saZm=^b|ݘ18 qteݺ1c2]w7pD~Vz:,n={GƍSǡcX˟un YO;vf%8˖-e-ױ䛯اG/ k9s = @^^[a/VW^Ͽs܎pSxWk}|,TW;nwoM$v(1%6IA.rt^ν}ͻaΜ78yg8y5漝ާO?~qN?D9d2-#= Ϙ¡[N=Jh߾gyk_ ~=FE|ӓ9uiѢ%VȥXOB˖my,v) &6mqi'㋈{@.<6/UOqm3͐Xdqj[$eMvN}ϧ~^cwAc>pc>x.G}Lz>UV[umw16g~ߢ-* Oߒ58w|/ڸ?EV1}V^ŷ.K=? ^z9wGŸ/O|<|jEus2zs'y>\}:HCWg=# q|x<CǎÃ˻`8hڴ9mڴOP~dee1ik-o5[^{퓊>c A0oGtǴnզn񺷢 DyVD"Nnn#ڶmG,V̍7]ͪU?D9C,_)S'r!^;n*D65H jۈ}0~L<(!|13bbzsۜ9}=R;wޙ]SH;LYVgGqQ/--墋biRЄRڵe"X=+^؆֛Gq'eѢgD"Q~sx_&O_>=dڼt:V",C R>ݛ<(ƾ:P̞_ 9opGUǤ80;2 ѯ_Lc6.(৵k= 2rciҤ 7l]\G=uk)))ᖿ]ͦ/7EX>]v٥n:bbF~C76 ny۶1}3a Y{}<7Hv$yתO`a=xZ܋ZnqNޮumnKu>∣56l@,Vرc=zF֕ ~Ӹ/7oCa͛K67ѤI3>0yg6p9X-^s?Ui͚`}e̘MápZ=$,A$m#{mڵۅ믻Lѣ((( ;;ѣ_NźN7ޘz~""wxSI ] *CPq8Á(![T@2d "lAe!H)RJ4'MI};<Sll} 9sZ~/cǏz+IҒx{ʳZ"-YHjƩVz׿JLY,p*&&ѣUZeuTyJΝsgdլf,Fʱ+?Aը^[up=8'O ;~.{+<,\&MOU0)SN/^Bߍ5ӦIdFʕ޽{￧xWdljެrΣcǎ*$8ӊ??2… pqTTߍ$sOE7TEs4 ?_KǏ[M>Nwlu[]zȔ}(6&V*ܧ#GU^tUZtj ՌK]{[z1L•ۗ"##5~H?R^F騀""f+::ZOWw}ԧW֩'TTiUpz'3bnQļ咤HE̟.TZaJ~JutΗ/NL{rώY#8y]m~ZO'okĈr:Zf} =eTk;/"ͮ_~Euч͚\gϝծ];ղkui_nf̜i}٣缳gbz!88X9sӽ_mפakLjz]mjzi3͟?G.S+?;')wUOVyF礎w٪]y:t@D 0 =զ}5wusjMj֬^1 IDATz'1[yz=Xڴyt0i}ɌDOr[-nӪ[72߹ϝ;Eo-zeYA#Oףt;ԶmguZGtyXLoDEhhO^?Mƅ ܹsڱczZx/(:::{C ح< yի,[^Qll.^R\EYR>XÆVppڵLu}n\̙Z|6|] *UѦRZj>xTBI:z숎8h/?^=?YV2 WҰ%]>&3l6rΣW_MT~^p?^_z҅@9rX֭QաZt`{V\={wǟsP׮boo~mܯHkCy5fM8FOzNvi9nVU>S]Ƨ.8_0[}*uW4r{*Vo_neB$i-%KI׮JHH_ЪUkܙ^w鯋onF[|Te3=ʃ׼Xw5}C=Գzjd˗W_m|Y1rٲ%jP~uJޔ^ Q?>~G\վ}UKV\뿦eSyܼyvZ^ynu˾$]S.2bѢmzB]wݭfZjA\˖-UӮ];rz[ WS?ۣ?׬13+0s| *^֭[iNLL=UZt͜ƎA*UV||>WY3[ޚ1L)DfÇi1bvg1p= Ҵs5|xmڴ^k׹ ꣠`M:GF~#GjǎW Maڴzʟ}VHKiTTvܦFDR|W #鯜9rjy2x~YeǤ6֭5q4 6VKΝj5M:O} -5dӐ!ԩSW͚H ^nII۷GÇVf:q F%8q\WTTpj!IZW=LUFVm۶*>>AGzuР>ںmuɓ~_׽k,A~WYnu,ϟka :t0C"Λ7Kn4WE3syz7GyL+V.OuMkUx +vK/^UYqޒ^ `ߏS64k"}e_[Fӧt@عUvmޔgm׮l?ה6͝7S]>?GG5fό=gLmݺI'LС5o^zfe+|}CAߌ)nh*?6IO``ʖ- 5 T1MUj SN絭 se6m͉b$ '(F |b$ '(F |b$ '(F |b$ '(F |b$ '(F |b$ '(F |b$ '(Fo…x*\$ԩ:|N:i[= V UZCO#u>R'oi-Zd.Z|^^Z<.\822RFV.8uteuKݺu-3իRv]}KK;{6nToDkT}RHQM~yf:֢E[UVSu=ioݪ~1'c>iڵSs"ðT˯)_kGo_bbu!DgFsg*8$Du뼨_Sjݙ~j ӷ9g:8lZ|ZUмy,Kp=g"V@@n>WCj9sZtQ\L~/꯿k츑 Qz/jzu"/_a@bS'E6]_zE  .3=Zk֬ґ#%IVДɿv4hp_\?+*\*{~T:u'B.j޼Y_UZ\/6'Nׂs=gϞaߎUfWI iׯ]yEFSvrJ6o٨=*k͟2H)7+*RΜ9)љ[yΙ6mVXoWߪg/Ԡ*Yv<_{%ItMDDqI=|\;vlTx jAݥǏifp wrz﬎?nn+=f-_o 裏ZJri>b6=Xr[y>RwQN:~̝ )44bcc.37_!^CcG k^͚5'-Ȟl} J5L͚詂 s?P!,XH|kVW'w~PPz=HEհჴ`\}v (=N5j0-^9r!*Wn FʕQ^fKn"˥~{jyzհaTXM[ozGdճG?ȑC_tFzZjڛ:WҴ_tԹg՗*)i=̶j5ڷuZ*=ߨ…x.ԩOԿu>2ZcVxRe˖g]…i"%i հa#(Q2r̩_*SN3fkbCu\.z I7>-?nP>}-ުN9r_!*u{i O&փ>z{^_zv n+^B> ץ=y}fd|_amoر:t5ϴhB w)o޼:Tp~z4MmܸNO<*V|@+W.O;ǞTԶGڶm$).>N[u} o4Uhh.XfWϞ_#g[o-Zz6q8b/~ZUǩg&CX9T*YjתM]gkH~LIRy.;vM[ݪ_Ӗ-4xH+^?PǎZNfzݏo 13#k׮$ӥ_ԽTm5nMg# uZ%K?1ݍ(VV:|@+W.i]2f[>zF:|$̙٣J(Czm [in}ec>6#+TOӦI\.4mÇz~>N)F>rNS)88J.˩]vx߹s{=?'&&U=Ϲ\NO!RJ^F z@7y;ʔkU掲ڳwg*I3fL\掲*T;1d-_7~G+?[7+t1Io-Q4y/ڰOmٺI+V,i ҭ׭O|m60zo8uꤊ/yAAi_p{%uٱm$w_ S~_]yڼeɣ{R5P?zS\t\ڷ/eS¿v%K][mVLl*}IESr%I9r7:t/[q?iӦuڸq$2wJFt)'NHZxg|+=T{_Ȟ^Q^_(O0u֮]}MR׺WzMNKNg9 ?oG+!!^GK|-#fxαW7 CT_v,>>^Z`*W~X+?W_m1cGhڽ{j<~}AK}-̙f{IRψÇs5TrN]ޮSr Ҟ= LU>%$$СiOol,4d+^@r8r8b ju2yTyZ&6Mv{IR@@sujyZlX"Ug7tJB |i\TΝ۷GB]mܸVvi'# >lr:u3n6lvwWxb@5#ׯ__izGU/^P=)oq|ʑ#ʕ]Mkڱck qQΜ9t%*?={$߷[KI5ӳaoߣ:~ܯ+<,\ RHH*W~HQY&ڶm>ny2*Zfϙ=?W׮U@AUp.\'+W\>xpnCzo8uꤶmߢڵ_$IK.Ԓ =T ڶ} *pSE^RGeǯ6I5iXխ~7b:u"sKg1J fswEIҡC??Jߍ%K\΍uqUZKr3g ,M?ڵ~Z[>ڰO͞=C R2w+$Ij=jUkjA:}_խN/I:mZQiz{}jذ }\ްdB.TD5oY䔽j~AGjɒWx-5*G@(PLLF|786mNSLTŊO:ݻt9fwԫgmظ^yNe˖x3MS\EU9ոqS8O۶oaO?8BxntLczhZ|sg+g@}?iLR,\jq{Mcǎa:y򸢣crr5nTztM3gc$i_].ڰa{r://|d?ZLHԡg\ -Z<_3fLSTF=У9[}ԼufUEGGOۨEj3:u\#^=zZqj߾7o-riݺ5>bgmCcVzj3F맩4}SVzi3u]/i؞9sZAM~__}[.\Ԋi!Q͐6@BsСڭ;,I-NtY 7.j ZU#u>aa m۷hɒ-{ʖKNSgΜ֒ 5yEFKm.r9un5{uUڷlɓ+66Vk=w6S\رMcǍٳg<癦KG|V-;H"ںu~vi?SwnM8FvFtiYb>V7zK 4Ν5~(diϿZceٴc6 o9ծ} 5kJDڴi Yhƌ Qz/O cGT剧k˜W;䂃i7G~p"*^ ."ҳgߴLӸqSzUQ[6=dțEb$|w*.eϗ.}vJŸsv)sGұ2IҎTLI)z7UjM͘1Umt԰ay%@fhsGd^p*$$T^u$zic#"amd{ A.Y)NSqqq +Iʝ;.tU|^;w^>}2s:'m:pZ`/5놄($40MO`mdkZ}9ֵ볱%Hxf_jnf A.Y-]wݣJʗGM:Wߣq<6 Ӟ}_JOH矷QXt8Y*@:sշkkkkY3[h6@ R"Ev`\I֭O5kU``W_s֭jFSn I*Zר֭?$mذNի5c4K>f+lZ}:}7!\r?ȅ zZZzZڴH>A1OP#H>A1OP#H>A5Z IDAT1OP#H>A1OP#H>A1OP#H>A1OP#H>A1OP#POX} QCE) V]#a2 Z(F!SշxH?ď@j|zTN2.oh"&yn8kIR>UcN[M^mLc7([+6JHL65r`-o-o=r Ў9|grt솞6{r~{fj*^/FW@11z啺i/T{9Kd!&Fy<[J۴ * t@2 ֌@ Qy4fd.\DkC)66Fdѣ*W^7z޾ )CL1SIثLZX[[X+q>3r=վC+5~e꣏ZK4l` :@_M׫{>WyWldHd=^ya=zxĈ1b!#V<&NkL;/{z)&VD h{z{z{pC ,8W\8bcp8rz1lr:w{L+e]`!*UZ֭ej:p` P>5a(׎݄i73ZX[[X+q `1Ӕlvw2y:a(e3loDzY=#V$nPntB//^_СÏU zu_=d  `1Cr$?nN}ڥJSTTΜW׬TKEhܙOPDlEGGbJ$gbr)g@%:U-V&LmG@h)S 0#3쵛M3մzJ=cIJ.]Rpܱsʔ)Mi zfJLtaն6l> Zɼ$#""#Sa g7mo7?4˖-s#dV\͛7JBBBu>GBBB%IUOJWl?F1r=kFjȶtցԭ{gqjֻ3vX ϗܹ铩KnС>iIՂs|i^3$$D!y(4o?\6'wno!#ֺ)3o=r`v1@r8|Gn#a7d(5TMGP7_VLL$iڴ)3v۫7Mu~hܸ?< P65l@W5z+#GAwb`n2e_LE65r`-o-o=r`?l X4 Mk^i:|P/@'O^ծ: Iںufͺ +wںum+>NvU i`f$LriH0$W$=z|}~%өozKԭ{giQ5okNJLtb,Zר֭?$mذNի5c ߃iSkkke0HbdϒfIf'NW]{qSڸt ֡Mb!m6ϒ_QgZ$ fFXΔ-nNM;deZX[[XOHl5 S ďiJ&6HTZ3/LV*@JZZzZsV3.M;MۤIYHm%Obd5iXΔ- iF.H2Ҷ1>bj=r`-o-o=r`-??3#,fHjӖdK^3Ґ釃GFP! ܍)3$ 6m˙HazZiLc v2Sj5r`-o-o=r`&23b ndxꏆHd=̌)6Cr2*M̤=zZZzZsH?`+i֌@E1b$[@6Hpϖڴ,fʔ$ӕ<;29!E$#""#h ; `10d7RZ  d#,fJ2II{61ٿYH )24j(FX4%۴%L@63$ RZ IS)&1&bj=r`-o-o=r`{ 13b$ٓB&if$3 f3diФ ,6mr!/ i uRV.7r`-o-o=r`-??3#,fȐfK63E ˡ `5CQYʚ3#Ц lv$S4$|:nnȁȁ4̌)ffL#p `9S6{}qn&]ھgZX[[X+m `9C͐̔MkLL~f3ddHR$ڴf]M;:ɼӅr`-o-o=r`-?3#a7d)֘ti bf$LS6-i6)CLfF駳nfzZZzZ13rl`rnr>3{*jar\ ZK.$=Szf k5r8b}7!ty։d7mdE^/F*TXzcZ\ZuP/;kmբE۷pSS~4 fFZ}JY)bj5r`-o-o=r`-?۴ *=jUkim߾UqqwCO)((۷p1RoZc2XPYׇ QŊkԹs7ni7*]v97&&ZGQ%}7Y˔lm-6pަ|ڴy6n\B W￧hQBBBu>C!޾ iH!󒩑axЅP. V#""#{uf?N?E"p* sQ6 QHhidPio-@ A.<6!ݦ/)SyU4Ͽ\8bcp8uuz1{BKwVt쪽o^u=Z*IRf?y&199VFyM0 .H)ifd6F֨QGS&Լs +Y sj*QؽV!4xH*UZ;}Qc #f2 =3# Sf6EW6h. (>>N-[$ޭ.]/%e>M>/~:_$իGòyM4m4K׌ #46իFKKJEhܙ٪Y*VMəAʙ3PD-zU6m>Oȁȁ2爙0$ilMٲw+0(HF~Y3y֞dekW;vnS2e$ILT55cTiQÆ T||?~IRZMp(0SEDsz6lBBBu>GBBB%IUOJW۳]L99Ɵb$_0%WJk쵛vbb&NcǏJF޽iذr8b/<_sΫӧOz.=CM'.RT 忦y͐z TpX?Y'oo!#ֺfo=r`?"Wp82(FXݞr.i6ǎU@nw/)i߾zqT/ƍ0 O{}+)>!A^GMb1pt_Trك</2?o!#""#Hc^Y!R `=ÐI;5;y~dvx|5}C8p@ lRI֭޲j֬~[k+wںu]ӊ(.j^3;maILFEEGg%/㺤5^̙;C7Ӱoh℩:zƍ)ݭ{gժYOL瞫:)11ATh1UQ[&$mذN%J5c4>.H?`\fnچif6m˥1cGhi߿>n^N8SڸXu]'n H`.@٨Eo{,ZX[[X+qh{ޯ%ifz `9CLfFi`dSkkke4N3#f$ O10l ~d)!MC2)F MjfR{%! eݴddZ}99Ɵv*ݣJ;p^izfCX3Y3#2a3Tl~]WP@]<yۦ /ܑfd?!:6.99V13 6C.K JKgC d#2aqI3#cʼdHv@C6@&2l t1:^!ǎ4ݻx 8|{kkki M. HӼMi  l6tWp:mf7F*X3YmȰ2MS1]84MI6H֌1~t3ELFEEG:ĚL!T\tB%kFp(Fd"&6<i0dؘGEEG rSP:kFJVaf$]\41|n2D5Y d"͐:c;7q]{)iؿ,UflXX[[X_Hd3\{4M3MӔ4:3e01YHLtLHeʰixfC ,6mLd2Md7qOwL i*!f*Skkke4>3#2ѥ3# Ӕl/i6fFd"&.+q5# lK1{ûc=r`-o-o=r`-??3#2a۴MLZ32i2Mg6$3#QD͐tlL}^32o1ڴ2ae&玲[KfN,JdS !#""#hMr33 6w [`4%Cb$@&{mɳj$)欃6md9id"fȕT4ͫ̌tɽlPh`n鎓73ZX[[X+q l6NS2WZ34=3#mvI.Ѧ ,f̤Mӽr+m6HX4ȁ;sdǪ5;5st>eoM|BP/ 1r]+kݴf6||@ےҙ999?~nvbX#J͝~@ÆPie  $ke!6ݺw $"8Ep7ھ~8!w+VhG^)cGIztQ8Q{HsIwHe"ȷ~-mIl܉!#""#j]MF9pwƩ$I]t$=pؾ]=jjj5wЀ$IhTyS4UϞLQ IDATt~4b$@xE7Ɨ`7z˔H+P }s.SOQyI+ҥ3?Jyt~qY:2=Mpf͚L]:ZebLK[vS{BzK3uǴə R>%6&Kz,.жm[E" $IV\.I:i_g1R5viD7mSHi]Z,Hx˳6nR1GGR4m#r42Al9,199W0H41!GFόfgF5vGf6&xM&WzH41$kv.6陑^Hn,\o,Wu$fFURyJbTt &\+tM#m&)&̀t\o)7 l(FqIJ=#G7Y,׵G8d'iKh-Ӷj~52 y{C:͎4URaS*___:@8!Z(kemr%cM~(Ro> ;I Mcd$٘WT:a7=nOI1^ҩȵVZi kL M1ʍӶVMόd6!k`S|KoZtQ99WPH4q͸l|cS#I^5qXv( Ht Iٺmf]DzWln7$SGEEGSHx ݘwﱹbWfT&k#kľ@6 &c ]WMBSGll]WffWAF41&M;k{F:̀4kj/_R199Wkomf?[23 MP|vCu%̧/J2FqV?:!##r,HtqmgdoideqKpt=w阣S~~t\d3x陑nHцcY$}' Ky1277GMnsd~Jtlj۷Wh/ԧϾ ķ/ec8FNȉόeޓ,@J2N;K [,_X8\m۶&D"*`H9xܹgk>U:!EGo,/XN;-{V##""#jm3#{GC~zSmڴIi[sM$7 m۲UhĹյ}Tur9 hMY'}zs/5s$/;ﾥk?E]z]UU;Ry\~z n#I321MXy9@XkȁȁZLg(ܞ_\N(-[Hbyz6T^@`um& VuuKuޥp&8@M}O?=7|MoZ: ݴXَ +k$8Ece]ڿg?;]>U[ki$o&N~ZjfqsbC_]kMW1FzfϾOK|Ժ aFJb;r//?rV?9je8F6Vo-nu%cWܵzg( ;QM3B~ْPLE翤ѧW)*W?19s:qO$IF~B$2b$@xKfbdSD+ɍO ٨Tu%j{ȼ<=}ZKUTT~H#""# j)FYw'nMIim˵mCGkܯϓ$ڶmk9HIҪU+t5KN5F1 M5dcMFxC]7i˴+uuTJ EIvQycu˳M0I{S/^>W8V q?'ϑJ+jߊ*?km jgxwk]mݴMH$WE1 }BFuwܺ 68I3#rf{}GMՉ+WQgtnN7h„4k 0H5iD>tw CT۴qߗ&꘢ձ \6ix7D5ļBן|#wjݴfjUnn$iE***VYHQUةP/J|١!TW))$̌Hx:6|n^]Edžc`}m̘޽uuSǾN=uZMr&\v~=nV^MhVԳ:l."I{-Fh3kEċH!b?r//?r?9 &qqm{Fz?`$Ǩ*ݴl9k ͚5WZ/֭J\TH5ۣ &1n Rk~VV[ysek H41Ii]9MSk8Fh|3#+9m߼%K%k-199Wk :@1ZkcjblI>dfFqTM[eڊ)(Zb^]r͌3# 3#gFZk3YܺZ Htq\[i+Qo6eScy=n(F ;L ]vn*#x1FZ7dñ4I/#e5g___;03 M2mkZYIѪvm|X7ulGn ( &NI*Fʵr?~Yo˺ui`2m41f2ֵMj4VY[׿&uŞie`J*cȁȁZ戙i8.6jrHIu c1oz r]+(F ;13 M$ۦgF&U*&II lc4AJ1y#?r//?r௠Ɵb$@8!#׵kbhmhMâ1ʍce6˴8F91=z1&πZ+eŘHcLFkXbdZwR2e33WѾ4#ql4جܦmhUTf!rc7KB,H'yf^QIVUѺl0uD1Yibɍ\'l JڪW: l*'YPx'___A?3##-g5Zke$Ūbu4˺-̮)Rvñ23/+@Yת3??Aڰl51u3I{C~l+'оsFLGEEGg8G#R7A.yܫSh9w%Iuʖ{CZ֪@L S1\m5 N2Ms)F k13 v#wSն֝,Z`v%R(g1#S5̌p|UmiGK7md/fFdHcyQѰ"):ƬNN~-99W@03 S=ӱK6>38 ?Z۰ d )di6FkǶFW d idJ#=36gd#MU 7yjS6N3#2Z+K4z~cje:  J|l݆3#i ~κjؿZ٘Cd)b]iòo4cǶ}UH+mXKʓ8T[Ss[R_&V+ Ao~[;wԽ{wmܸ1bV?n#R(S@|X <ި#""# b$H2ú+j=3zvx t92|u.f /0ǷKȁȁZϔ)/F^yuڲ6'*ZnK~\-$ǞWg/֤Xʋ."JE"Z{*)(F辇zC@曯I1گt :z'%IEErSoAʕ+tngNenY5o___ʋu}M:hΜGbrIRuuf}>?ڰaN; M|bt] H[1rڻg/M0I9K}ϊD*5w<6tJO/k0F8V qtH/9[.!Ghߗ A.\4oӧ[ս{IRyyN^]vzC˖-ѵ]{E27mD|<r"8EԲ"~#""# hSg+uo79瞚@%8aJJiժqZ3IҡЯ.\wަիW[oSgߧ%K>PLE翤ѧW)*W?19s:qO$IF~B! `M;8M v<r"8ZPzg$Is>a#5` -Z@hTyS4UϞt0z:iCEEG33֯_.MTȣ Ǘh„4k phNV͌@h۶I"JUVk.$4j<;xөGEEGgW1Ҋd- "8Ephm.@"J'+,Ig_y%8aܫ^|9?7+\Po5&/֭哐V_~N#D"i vg:|}cLbyTS[&H$c۳~_BGEEGg$mE***VYHQUةP/JS{vCH5Ub B>^9vE1 "8EphOFk5y^6JO=9OC &OR4Z+Iso 6B_{ ԧOfL[O?7?/0E{fGY'P>M=zrn{C\rͅu%7f0K}FEEGSK-IDATA!""# jm#d`!5Cp A.u@HA1@Fd2mo*.+_ߑ A.\GksaY af$&S_ó~t$i}Wnp;iVQ瘣óg1ז-O.]tM34?_Rڳ|b|b:ǎh)FhJcNne&L޽5kVkۨƍZ#I8-!Iz.JӦߨ:RUU≾ j)>\3E,_\oK/B-)9-\ڳCOQ?:Vcǝ /:K?1:t$i=ԵLуޫ/P(N=S<$iUӟTe(--ҥ'۾Bk׮>ekx(Cջwo-[?gΣz8Nk%KvZ>CG&c]W}#)9험5\磏?PiiiӞ>O-g7,اBSyXl'ŋbbޭbϪ6?<"8Ephu.Hp@I"J{:P[e⏝;"z[5F8\m۶6xSAAJ_Kj)>>Mt9{[6K~Ʊ2s{8!iD*/ _m}ŋT6lJJX4pqX_iiʗ_՗_D@3+'f{/ڱ#") QAҩ"Htژh|VQGi};wT\|rrruđ?/?]eNw5yn\7Fƴ~:|]xxyy+_o,o0Vee{]T|u:_Zh.>}*j?^Rmm>|^~e=f$l~d&Պ&uD\v-ֆ uc4!گt}*I_Sx!IctP4^{q VK[үqZQ{/.R=^=u9ҋ/=/I1_ӧ;_͛7i%%E*..VYHQչP{56 :+ёG|O[lܭ7|Mt챃5tecԍsԵk[@Ӧݤi}]٤ :H8ۧoߦ^i)Gٹ^GDܿ롇M޴%k$cQ O/ƍ㏨K.z;^x "8Ep lUݴ餝^6yr"8Ep d3@V #j6+Ҍ@\rYH0ln!8Ep A.(F"ҭ<Cp A.\ Q#dH$D*+< r"8)*ޓ3#&t}b$ڤ?ߗz(FMxZE~_ 6iAz? @juז-u}Ｉ}KzZ5Fc>N?N]ϸY;vDڲIs1ҥ&];v@mڴI>t^}nR9_jђ{R˃ǎ<‹{iOt۵fgӧO.xJJjժqI6<.)e.ҥKڽ2yHq5~z]}F<֐$칷&NZ:X6׬ 4o<wkU[#i?״xn.]M6~@ 13?~vTUi?ͷLք Իwfj 18quڸ\K~$Ir1$W}teWiSGJ_UK1)dR2|d]zwoIyhiС'х :B{kOO7~ Bs tÔ[5}J^yunr[q4A}XK\i*N;DRkȐ޻Z]2ͺzduT(m[;HI#8UvZ%AC1 1G.E"Zl^~y?aIUVhCԓTXع1$i /Js98ꃣ+SMM.XV>+It8NKygxH=н*/Z6mC𲑒(-\o[FzTB6HRr.m0;ͺs]FkZz/3DR|[Ǝ;OK_T![iiӦıP(5kV稦Z . w+hPmH>}nWؖ8~IB!vzч$IWҟt*+8Fiiib)$m^kj}JҲJK1Z޿5lrsԿݻ-Ο3Q?ZnJKӒ%;*cd#kmug>sѯ~Yu><𸮫>hѶ؟%Tj*˖-QuuU!}+b1vTb1IRqqwuM_|-MnKVGZiƀG03 3 TQQt,T8N:6Cn?^vW_lpmN)}-٬=C6m~8w6o,IZʴ*ㄔw)T^^'u?8J/Rٰ*)+c\4pqX_ivkwΗ_՗_DDA^3E=>mߊ\ӪwWqqwbQmٲ%cafd ;+\qB:mYz4>+rwǨΝ;k{ee*^B\rrruđ?/?]eNw5n\7ju3Nƴ~:|]xxyy+_o,o0VeeÂ{]T۷T'\~ZzϸEWl ?3nN[;ezC98 /ϐ?rmܸA?t'hX0izg d2e2b$ #(F2b$ #(F2b$ #(F2b$ #(F2b$ #(F2VfR/%IENDB`incus-7.0.0/doc/images/grafana_resources.png000066400000000000000000006631721517523235500210770ustar00rootroot00000000000000PNG  IHDR#uzBsBIT|dtEXtSoftwaregnome-screenshot>-tEXtCreation TimeMon 09 Oct 2023 12:06:41 PM EDTO IDATxw|M͐a%AEګT)JKu֮UQRj&H$12rDZMs{9ǐ5ȃ {?`0{ n""""""""""  ?&Fjͩ綠}E>niulDDDDcI_}HT'#zظ'j(X[= 3+/#::E&!- ~kMKw^":6""""by>V/Ӷ2<+6.. _@ttLy^hZR7^>ձ{^Ϗ%}%#tH%#EDDDDDDDD"P2RDDDDDDDDD,BH%#EDDDDDDDD"lu""""""""*Wٲֳag'%#EDDDDDDDPpQ2:fP2%J_];/%G)Y4>U+7RXtܚ[(^۟ءEC7ӛ^{=Ѻǖ m$ Vn_͟uX""""h٢-#~_m9rԗCqz2:f;:\IG832nYh*jU]֐d 6l912s%(͛7pfx+;ˆCx|:FgY͚~FYG3LG 85c|cWc9bSUvOֵOU=V|f=G=qsQ6ب4Rv]bccS{zXT|4u'M{^dɊzJXXi͛aJDDDD$uZhĹsg-S^# FSnG7a4)@TT/eٽPV=nݾɁR,-[vjԌK``,ZM=-{/Ӿv*ӧOymٱc +i"[Kd{o}?]O /qOO=y,(S/v|iTZ9sg^N%&&Ye1aWN9 E^kaO"o_%%:ȯEDDDD[58w OrÉ קشً5kW`ookΐC='ؒ[^u LU22{6>x#W,!::шZΜ=eHTK*oKV+"Vr|\Hƚx#*U{ѨAZh᣾4/2ױtڇ"q~FFllYVl`Ęὕ֭[:e㫁Rr5 W?zڥUy<0?ps_prLu+i٢-7os2fD\*^h4>2d{FZMMӓ}T+rf_†,cf!Ⱦ=U[#R&h޼ 6w1|59\rtΝ|헴lM vTZ_fq|iTʰaCq;/Rl֖Q#'~3?fء3֦2֌> |ޅ_Gezk{tڑ{ӿ FʇGN$Ni7Y~u_W3zEh=WR .6+[$n(^Su:C2}/D=)RD8.Mw'6?ʹvr"<6#.67:` 8bN}͕˕Q&^>Ҟ)SӣԪUܹ0jDg]3iXkޔE֌?=>[>TVG΄1f$쒍e;Y\ cc8D#Ș-?UڌIԀځ7.cn½NDžCK(K"ƒX7'NbjPwػ?;RB+Voteү2\ALVȮg'j@ٺ,ԩSm7ԩ]{RS2eˏ}gͣ;Ȕ`Ɣ%'F#Ƴ|'>kvlDDDDIJl٢G>V]n WNW\]scggGrUM { (@iۺ~ːoQ:~vs?}w'cF w?|EEԬQF-=G:jUkL͛xz~Nлg¬Sq)B>РAc/ȗ2ǯiҤ5"00a?%6[Ϗٸ)e} $|= ~ o5lbBH{%oڽཛྷ[3L(T^#&q!Wq_b 4#wf>cg=x{on]VZ(U F1fZYY1ex.^;DГ's5-B_{:p.qQf+CUɞ݅#rN;aӜ93gΞ{6/z5)R 8&>>3cƌv0f$ƌ39GU$V}p_بExIl2d$Oˇ ȹ)T] z-ۿ[zqtKA\>mӸy{ H0Bl]"o]#&Vsm9䳟"((ݻ:lo'ۢE[||on~c27 ѣ 'SL4mڒĉ `Q9X^s_lL$ǷNN%NSl28&4˘xэ0ypo\L$gq?b"} 9oB|f;.#K΄݂@H/[+͉mSzn!s|dK8:fJjl߾m6SZG&S228dM՘'"o]֩ ­[k9z5|ڵN e:6""""bYI͈VժxBȜ9 qrrf_,ԭ0^FX/{|7+ ;wnrR._N;~?Kpx:zX<\ bŪɑ#'ܹ{BCCOۿ;׮]%66;wn'&whh˔)3ժ`Ƭ9}ƏvnjobBibٙЋD:M?ړ5K{  4=&>w}F+iW^6͛ا@O>h4RL*T [>NHdȐF@r{dЯ)T0gΞ&6O2K-RcǏ%D O6X""\M Oxؘ1?ѽL61MLom3|f;'L6y >sN +nݸ`Zy<>,)Y/Ϝ ,~軡y-hڵlE !;L.899Veꔙf9883#yb"M}% V6 )Q o !ISƧٕ¯Wσ"u%Y߼}윻 ιPnXY>H f'NWtt4U_7}Ԍ)o]N'495{w=hٲѺz/˱I*pf8ŋ$4A/% 4.]6[Iufxd"RpQn666?/ݾ} ϓ1 ktӞ8;9slllJy^6l .fo>ni% kέ 8uXl">*!V(69cg#`FFH"o28fˇ JFKf p~)ƍA9%f VMɋtשּ$N-yFc<N%2e/Dĭė!F޺JؕllytD(ɞiYGg fS2 G X቞fKn-uf۾h}'.˹ŕ*GQH6KT!+a۟rzvrTq+,.Mgv)c6_:kg>[~HWv~H^ hkkKVsl2u5] tzc=zė6mKUٗ؈ee"n_!8 7n\9;DF&\;w^||&㄄ o|mq#K}ő!鞂$-oZf|=eǎ-k&Fgܻ/ܹrJyrg`4hYnGwT) 8XB|r;_7v'iIi׮=~lYE?H۰qS'"eNL}hfѻW۾Gؼ k-[̝;3opnݺu+0LOHHM7iӖt]ϛvrb*6 YYx4+k[L~9yb$M]DVjx&Sfљ*Gaekrbtg5jruWڷjj_e))xElt{5Cz8w CIhXGRrUZzA3P}?a?}þ仡ٽ/k= |2eLg?VYA|8y͛(]cǷ|dBر#xA]o~1NN2ahzOZὕ'\gŪslllYvY]&gy,ZuW=%s\J+PW=Yə3/-+^N%E 34hps 1g SSflb<=hJF>7||w.T_0oiBxx#ʕ+7~Iٲ 1?#uN: @6՟J^#>> k&ӄض} Hx<|N]s\ճ?53r$_㕒gG3h;bm>iT2tڛbEKܹ3Y@TdiuC" tY[Ka؏C#gccCǎiؠ 9}$~ę3roSٓ#)S'Z׿ fu-ɍE++G:Y5_mH;ꚇKs:;wn8DZԞlіgYP",{W,I%$#^dͩ綠}E>niulDDDDcI_}H-sv OXz7h+(P+ZI>ͩ綠}E>niulDDDDcI_}H-s\ """""""""NP2RDDDDDDDDD,BH%#EDDDDDDDD"P2RDDDDDDDDD,BH%#EDDDDDDDD"P2RDDDDDDDDD,BH%#EDDDDDDDD"P2RDDDDDDDDD,BH%#EDDDDDDDD"P2RDDDDDDDDD,BHg1|^ަnc3lmmҬ>`ѢN:_vuc2/jzY"""""""""/=O||<2P~Љ 0l7iRƌpw6M{5nciX7 IDAT{{{u(""""""""x.fO'&&++kF7jI"8{S?bPlll}^ׯbuzDDDDDDDD$ףH[ Ν;  :: _߃xxTSܽ{'bkwڵ%K/RDDDDDDD D8xpu3-zZDFFpo?8C9zԗG};FReptp ?WQ899mӦ˖RK_r`bFڵꚞ> PvӪmZsQQ4~9زeic`?W©S'M߸qKb#6r~F#sfU&~ڵ+q5jSDiƍA~W'~4 pɞ p {e 쏻{%?VsDm^ҥ i=F ߃-Rڵs?1""]MIևpm4_t6o"o5jF,YM ڔ<^f%j6Yd rV\~/yt" ^GPRU.8EDDDDDDD呦3#--66_~ΊKRMFƿ`I ]8v0͚COeԨYv|@BBBxY6[n˯`&ȑ8sƏuRѣrdMH\th)%Jb_3oݴ969yٺ۷w_q>mГ Υ{TI:u+sG]]۶o=2 O`ex})"""""""*28geL؋? h-nJEDDDDDDDD$yi~ʊ]zRF-._R"RDDDDDDDDz-L8Y#""""""""J{%.g~8P2RDDDDDDDDD,BH%#EDDDDDDDD"P2RDDDDDDDDD,BH%#EDDDDDDDD"P2RDDDDDDDDD,BH%#EDDDDDDDD"P2RDDDDDDDDD,BH%#EDDDDDDDD"P2RDDDDDDDDD,BH%#EDDDDDDDD"P2RDDDDDDDDD,Y """""""rr%ʕ .]Hpg^ipΖIvv4h˹0du)嵎gj^վfƍӿߠdϙ;sgESL:bEK- GMvL2ӾǼQYfV^ƪ+Q& 0ĴMtt4/?`߾=fL4Uѫ4jؘ-ծ EEDқ9;9R\֭_uH"b]B\quv_&,,TnԨ`ARCOha'U+~Rf$#G XvvNNNZ S'֗(^fZ_ftIzYb?Ytڑ7oZ'*4LaƎyX|1gNpj*.l߱L޽lyG&Fnlٺ+A*U޽PHqOeV~r4o֚~C^r̩t'yH㢈%h\L~67,/oOĉc$.y6NzrlllpsOoz|޷+ׯ_ (0g鋩Aprrb I&sN” ٗ bժe o\?ʖqS0=̽}3͒ @ReN:m)ltɚ Cٰq}/{siyбCg֬]q]v ;1%4mڊ GΈKEED,Ab\s8+W.MTYv3e zta?~cUx/\\)_Ν%K6K4k ܉dժem;vl!,, }qڕfzşݨQ3f͜US%{MZo2q^)SfRldsvvʚ7n`c֭ڵ+ɚ5kۇ 02M>7~{|7b3^ի63eLʗZ7C~EY̓9QߑE Wr>ݻ΂鶶6|iO-\e> ?H=qƣ?aq^iSg斟O;w%kYpmڴ3ۦtrL?U+72h׮=VVOUg287VĂرs:?uȒ% Cd<;pu͝h_￦͚4f͚g,_O?%|J㯃#_~-˗yh j׮of3Legwob,Z%Kȫy;ޤc&Shq\\r鍕5;tӛL2lz*T\9w)}̟GM$66#f 1]0Mw8}$?6o@9)^jTʼnGr%(G!>QqG2j01R5~7ԮUh׮ѓ(T&aTR?|xE^&Ӹ&576me='3:ȑPOO={.O||1߃])P_'2dlݺBy'z]SA b0ᇡ9C(Q4a/s:wfFjD$eGbccC`kkwŬٿӨa>xc%J!Xn5͛l{*U9cΩS'R{NAf,\8Pe{0>~<\\MyX+_Ma&JKӧOF̙ܹDBEpPwrQv}~{1sgOg̽[ŤԿ=8)zLR:r䈏杻S`a;wm7[_֭[ٳXJ5j c=HxXFKtҋ?WOϜ9[[R#ӬYy3Gڷ3mۦ''o>""vJ""rq7C͵kW~{ߤl67>m;g[]hP-~钠{wzMV2!t<ܵePL97nIʕ ^E# mۼGXX(? Ltt ;wm&[2kOTm'::sMvBlh֬-sΝx{o5{"^N- %JfʔOzXȓǍγ}F#'N`pG..9Y6s`s8~g-0_fm+@O?ҥ\~Kx.{7)⒃b,Z4'3kBZhfLQ_8zԗG9qG!6.ִt2 p|a-/<7#۾c3/nUtq6mEE6DFFͷ˦Mȑ7l`j# "=?>l\Xz9!fqs>p~:t/ȿ·rrZ5o9{cGzwo%{[/:KE{p{%_FTT8w =@lliW?,,,G IճMUJUKFۓ7^^kM8th͛5%L?HBI_:|im,:*PxDS^'!CJi_0;z'iM௹Kؿ7>ٶm3FcoݾɛuPxIrd2$Lv͐yؼT_\\'QdY n@qu͓'{^=EETT$ 7~-];+AΜXbC5ޞǮpb:gԻ{wӬYk ( 'wEFFkW%gOz70ٺmsk.w/5oˁص۴Qrȃ[\|fw?&j5zŇǛVH1 3z5KSXT ||pٟ'Çضm/22{;;>'nQ, DD{֮]AU\ׁ͘…o>cүtf?ƥKhPD_<%uUxI9}IK0{^3fFdDBX[[oާw||SUgddd$XʃƭpĹsMʔ.o9g\ȑ 'YoMX0C ʖMr`)lܸڵ= ߧeܺulC8thG&J҅sJ 4!ȘٲPjuNLDEE2[_pQr_⒃=([9rdЯ3w&۶mׯQl92@nxz2e̅ pAA*Ukk?H󼌋Oܹe 04~8;9#Goγ(VJ^w޷o.ܹMo^M7$[6̜VVV/_X͟W1olTyY8^HxRp2K{$={HLL`O q15=J2..9ȟ gN2-3S~N9P-[60~(>=HZ,5NNـ|򧺾:us+Fcժ\7? E ѴdRfeAU/_~5kΝ۹{N\5NSޣРCtܹ3h޼ 5k~R{N|[Ŋ,X8nRO#8 дi+8ڰ L;@{Kbɲel{`Ƅ@ N y% IDAT`S.8`cnK,*VJmǴ{mj43v>y9_W_|V<Lg]wphjK/-/ rЁ  3;{LlNkdwm?⥗^ .f|*w:ʭ/d/w s?C³V:߾8{~#Ylw?|ng/x׿sWZޫn檫X7nWu׽8vo ˲x뭷ot+O>80YGy;]>яyK\tѥTǁ >?Gkns#czA83.N!_v5ws'4;=[^xṱ(~.j>{x1na>#*M$<ۧx{>cʱgnН1.v8m(eo~]x)GfhhHyeKw2::BOOr[nD|N{eY|wy_OӅlǺcǺ~}h4JW}H<>/#G:O.yWɿ,޷ٲ%֬]/?$?#\x%45͢g\;Ū5\|ehyͷC\~|CO{B;ϽЉn3\{x=/h+axxKB|Sŗ塇g֬Y,^l²,v)ox#X3g>#/ٳ;77fY'BOOZ׸i-x>c|'/k솳eɒe|?3"wow錄׶¹ϦMѱ_7O?ex'Xx .y7p{ӉS.  M\r=z{ϕٵ{v. ~GFG뱵/3exhko|$./<@? loXr57.]}}Ͽ^kmѣ>| ο{JO~6σq}7ÂmogÆ7S{7}}=rN1::۸ [YzO?k/J%Yvk֜}}+) xM^Gs@CaY{?34p57| ?…7O?11UXVm۶R[[˥\5gtr֬> /6퍭<ܵmggt\s-[ƅ^ݓ}k Si\}\w;>3"F~?M7Yt˽ɾ5A)}<__HLݴ+~Cw{H7^|QLW^wlz-lt X_x+'AA8ŘQӴAAAAA8}1N     pf b     ӂ     L "F    0-)    ´ b     ӂ     L "F    0-)    ´ b     ӂ     L "F    0-)    ´ b     ӂ     L "F    0-)    ´ b     ӂ     L "F    0-)    ´ b     ӂ     L "F    0-)    ´ b     ӂ     L "F    0-)    ´97 (@)hh[  :Jj ©FaC*meh  bA8]PPw1vǰmN?M} P#2hFaIT%Ɉ"vUn&9L P"27Ԉ6 v*4`l&jm>AN"baT)fmWPA ,f] TGWFI&$- +eaeX,ƲRvSnk-VQ2 e0* ETj"",]LLm,@zHY$ 4:gAADa*2&Chmҁۤ6hJ)0Tx^+^Dƚw`4$$$ѮVfho ӧXqd4mRt pbPaD6 m ʠEXA6.z Ρb 5DX"A,`4g4%HD2ZL&m| A0 e(" Eؠ Va6cΦb ˲IY)8XX֧N],/ hij5opXiK$  TDH0 lep} gEYI\H{i ﮧWjv`%wrb "utI2*vz5K]ZŢg1?Jϖ!4~Ω 'մU%Uv((3>tJiJEFXb:`:Hl,(KiݑyAiq=cć2[NL/ e >:>oek85vQ=Q;vvĝyٵֲWTSf pX7C#X)x<>DOob&j,$`gArXmv*TŲ*CW_e4'LJ&- g("Fa(àg*ny Q;F qI#Ż-oAI8cybQ]Q)@;"!s L dCG0ݯq>l&6D)׃0p[G-ȂN+ hbM?,n+ytkkj!LpcP@4 D)Eob&׶ كDiqӜAf jMNK贋Muu̪#Hy~  baD&FGX(F%lR9ǯsYwd ;*'Xj-PEdʤ:Zϯg0!1">)9p `^/MKSLNV,#@}"GsFWq#gGLP6\69tqhdlK t à:`s鬑GW[yBXfb)o}>msf6g6=9pd*I,AE4 VVǹen‰Qc stxCY,׆+7]1YiXa*)\#(z@aVê0 rW=vʼn01L:vV |]VWEl\ot˅q:+ wF1Qdxtx<(3 4Uh e^/P7#Za@bvZ{0X\kA3#OS Ӡ:`&m}4q%WҼlX,K9# Os9B.v BqRrnSB,tFKFYy\b}I>|Q#))3>? ڤ^ll|'{+fsLL\sW$脻ҋ;) O(bEDG{ã#qRIFN60LVP{/jkٮR,SJ=],Ցk/cdt;%() 1y:aEM.8` M~jDJiڰ ӑ7NRL-ɸ)X uOk@D9rCnsLv\+ߎSzYٌv%P#1# %$Q zMV"XQcYҝW*\9 3-SCqP!R-JCt,YH4ÝFc ai0(CQ0EGiJN_㭇I_`aJ||+q !E쩶+Qټljq$* /g ҎP0ȊE Ƣwh,~jAꠇƀ͇v"3{2bߢk)/b'o}>K142νĢqԂ p*bOpuRK% b:Fyc37h:&VXke‚|UUYXN)w3Y9|n-wA#SqbxLjVh M/7~*cw'kIxLu\/:s(Y<(-}Z<߇w{^Z GI*%SD ̍;}4h(Ͷ +lEK:Ptbv1{V#`(JX p"0P(ewSjiTNDx:FlV/&;ҭɇLW\r#3=%#wvEK:ps֩YTņ?_G9t?h'S 4M.5pC^ؗ"dY1R?gtԢR +rOqt ukG|O\PaTPvb>"̹sb}M-r(;` L!PTk |76fpjbw^:: 63{VXO/#c Pڐ!?_q*]شUyj؏Nzy"8esضk7#DGAA8e10 >?+Q:vqh'/XW;.r"H7]ayYPa'Օ SjӚB8\fN3Y-ȭ-9.Q2S t|h.F#$b DI D K>ZdήROw:ƵfDRy)\t g'a̗) W>̵ `ἹDضkV"J{'A(ix}^> G\t^2vaqUzpZ"Jѱd1}}gD ˒qAF֌<rӥpǺyXwR1( {z,^TXF jQer GٴJ wj!fRVǜIXqq0a|hL64&#Sv{8ɳǪα\X"xȒl]h.>iPWNiUT\Y5g{TQ` 8I')c54M64&k;J'RJ\QIɚ] hU~e5P緸x/63lW){(GQ9J9M]̿/#}˛,-N6le/sW?ogK1\6̙C0ppF5Ԅ2{]Tv̇_\]%4x.N]Xsf   fnO4_9;]$Do\z]_~<mcǖLٰߎI j/xyI! rj&u-49Qy&uXrN4MfjdWfۀ5tʽ]Z*mu}=!6~I**+̅-jWR F깦֨}GQ<Z+G;G5o-=錪NV:]WkhXKdA h-p34XRKkC:;8 cG#|Λk o|mVδ;s6prvQa*G iG{zbQAeUӃi٩u*rTzgٺ0I2d9 %;-/;(_bvq2eu l3N/*!vfjlپT<%{0} ezHUTƬj_:Fl횳>Hpι17;уa'G.=T_~K?M}-Kr4mk䈅ʜtɲ1m݉/_G] Ek1&jڪ0k??+!vv* lf^Hus׬8!wg1:Zx\s=~3S;V*Mǻ"Db*_¹ 3__bBU<L8Y\!LTSťǪL ʪ=*x{I=;X1;%O8 :,>(a]c=NGi^V̱CDqȐv* 1i-.TE|2nTU"0tx.46lP*_qzB0% g$!ZW>L:nXQ|/=6/wWQ5li}>_.m9ѩv|`\Ȓk 9'SQ_a]Yz]Ms_Fbh+QQ 3XK2fu[#h+5r"8l<Ý$zv7M8!>b@^ ++Zriў87eųr۾8K"+YuV .C %>$1d(_z~ZLI~4D0+gtS|t )-R:ϝC\{j3qLQcMծ Xx Fځ41B:Re $?]8s%d:zv2m끻Nz%ʺJK|ZϫO%&!+Zci0ݪv Iｹ%(^elTr-c.<ESrβ.kކ:mlg0\q=rv˘7g6`p")A8 nJeRj`bGk:3 ' 4PWm7[u7pul0J|v+G^*9*GҼI]TNk'1̥5e tD!CAaIq2O9uP\]$kIe>TuJK<8j(bE`E"eNy ìKq8%qqj򎩫 uskiśZT:™Am>&lzܱyؓ_;29uvKt eHXJvseKi:۔=/v=AMXWۙ׶9,Z0_:6Პ)EIPt*'.ϝ6UP0g]*:kZV Siergv +}ųFڢy $JfڷS,qw]t9mGn.Waoɾ:\W;0s:V-]D@6S8qa04Ox AjjkXt1JL?5󑆣-+ "#ST&zwY>ѳu긮/L/ڲyua9zg_ɋ=ƞwxru*_By*!Ld  ra͉NRt8 S,Bf}>/ ldnU2>yDZ0k1&QmvonIn!L\%R㎼\}Gh/Ftۙ2i9I;/~/h =oA$J)C^@*Y9/.^mk^KvBE(atZ ;ݩ ,*3ƙ_YV3FT9!!W o'3MhMtf7 Jg wm3Rիx׉bHA8Q |: oP'"aǘzX78*i<55Q,.]>ذj%/:xm^݈ϝoe=129r 6[>`ØuEdɵ7Mf25CTˋݯ*9E e(Ul\% HTvu aVz>=V5iǵԣe1z SqolCW(0j{yM2T)[L5O$SV|}á)SG/!eeW(E:Nl`g v,WNgc|Rt޹k:Ay:bq bvqՄ<"H\G;V4@ZRQf@ڒn؁Y̫peEwR ֮}w>IOe@mw;dC/?7uu>|̾kD@~^p`xoo˝[ Ok{ ]D؅OQDf^/PWud} r/% Aѐ55]4/SZ*:Qvv1ؘǮ0q8JeDl:3cF9o h\YzizJ%={]y:4B*>)0èjD >氫CR%f#> [xPJauk3g[)Q3mapNK4E1kt:_':&V,mo048eG'j*o{ ?`1^ug݉WSg ))~m{ԭ68ټMRE(2<Ԯy([ۋŊശH ɝ{B VoZt\,,:ÓZ_c4&GXo781G]¢f-͓%w ^C*&9l1Og۩)b4xvҜ`X(W,!Bda~Fjν:n[7"dE1b%[e?CFINB>ŢZGǛ:WR7Dg+"v gYzJ德vv./8vV+Z3-k 3V2ciEoV~劌 9e@pG=wgn&El'Ƕ=L66vq2"f[un*:΀Oxei80e01_'v_ffށS=+4|hf_ԬOWsC )o\w|WX؉(EԮP%X~`-@IѲI XPDdA漣yv?ݑ^q o+ofbE{&t̤\ 1&$@::hgf ͚Ŝ x:]{ÇFP+ㆩxi쨦?ǁ''x1r0 Huu !2%/D" yͱ@t~!?9)!D|6ASk ̀pܯD1+J YHlMC,%Hwb䪏g0xhTT:0spwG׸D j|m cA&g\چ93Ṉr6ipl'[ww3iFe~oJ``pP6oN <0٫BC GU_|i |fM')Kb]?[Ͻ&։$8qrEw 7JԮy_<ѽU׊ E=τ朋f>%~_ |5#5uɧK:UTͿ;1:Vez,{}/}(?4w#H ڶH n!V4֕|~W܄v6"F9%;7wMf-΅(S1gI[h iYt)&/ 9cJ/۝g,v|zLb Kgayl0)!LiO$Omp\w}Np Jw:Ew  #k9 3e~c G7gc.DfNUgaB + Gg U /[_bh#s_m;($Щu[1^=~ӛ_/*_~SD]_>""r(ebHyzN+n_;O x_v`Y2=o ޤ7«zO?׋o2jӪ.q]Ί[L 1j|脣lpu2%K}ug=ּ划D׵66p$gkopBNxY^%4ێs/V NhQ$J~JT-d2$W5`m-LR4GLpI;T;fy]bo_ JcL!!mRXV&rqwWyۥnG<h{UU*#Rw~UY `N],g-sVw . (ڵȝ&zw1z8I}Tһ_OЮNj(7c=;U6ڼj h~'BW'LhFR#*)96 O 52{=dӼѩ_hWlm.n mۇ]fɒo㕯H/zxC@5ɑt̘p9ϱkv1Ȝptޙ{yot8ZT_DZ>xp\L}ޖJ,c ER(8 eY ,1[QM^ ̠2ՙxB&g)q)Z|Z:ŝ+7אYH+Sܧv!0])8O+ñΝ+TSܫܜ/PtcJ1_) EdpdP a<1xL(힦6\9E1^NZ鉮mcRk6mЮ׮闷]f 4#ϗ ٷ=ϼ'2ukdCtSCi(P3C *Y/ |^czxL^/~ dk[M4"m-f(XEaĎJȫe=`^Ќ mK,Ȟsee0è jhIw{_ %,Vײ,Dikh%l^Ny'u ![Z6'w,T,b{Im3݈y zC8̿WtM."ǂ4 3[w??PvV ~IR6 ÆW)ZmP$`kѶF'_dހ @'Mb)7 zh⭞xH\(<KsHfE@$?GrD ܚS"DU o_Ϳ~{L.Cc8hTY6CJaf`i4v]gb42 P i`fC4 O($'~$m})fZَ7Y{IMn05VbQ _5f]/ooǦ.NӀE([cV4ej615[oc65d`c1JHab(#m6 Txa:9L~ Әڵ'1 Ne IDATVR"46}/~JW uєeomf1ZᅗobNlCh+l]fɁ(Cdhbph\16 zܹ߸o\Z +kXEivRkkxդhOf/iucqmԴvVJKESl=$g2"F BU!,[q#Djg{N1%KWC~;XE=ny@ }#۶9b<3Q@?- ;'8+F&# Le(Vj >qyE~,L+ܦ8-8N\MV5 ѡ aԆ|jU'~3;,d+>fW%YTW=!ޕH0WL=) X6NT? 3;E TJ1:epxLO?7A*D8FO:#Ǝ*ġD;x;Kz=,_ko":*vQ8ki~P8!2@pçyOЁ$7xEqSB)F!<>/JiT9=LӤcb^߹KӐм /Νz/E05kޛ;$ +ks/84mo9I w]<6{v0Ed&z0{k{->"g=oG^9ŀK3yVؙpw۶`жM2jIc 0ߠ S&K>pLqEIeJ chB[?9J|d 8IR2*&Dtr^ ՀNnS=biϤC2lYxrz m:b(h4ƀCJzPŮ+FUQx;"pEk>x 'u-ݍaHU!NJ -}C/SOq~IV6/9&`ib[J1 q2be4=PĢR[:")v&f=x}!ԭ#w<_n6t*hEװ$w>Ee?* cMU.fH='o'û{T/Ko\3+Ej(|/E<]ᬏ. l8gf`cwȎぼXx?$G?+k߯E@-vU CG9b$~z1r|476RW]}Bd>11c4ր͞A?)Up5 T"^嵐lϧ E$/c ]]jjGosYg<>e&.Hf)1jiZ_(]/Ϭ&3٣\:kdLqBV ˝#y{XfW*K^('rP"J !d[ ,18`^Z& ^0I P|Jwt']ݽ͓{:tvVUťfbdsqaynhg+0 `Y\3"|^/"aD-n'8bIZ## _߶u|-BM%䳷#gb'ebL5olbg^W۩3/"+Sg#s^ֿ*|vaľ@= 06ty5]vO|,t#Doz-zv +CKsېn7̾‰&UCbd'pJ'v !D?lòn9Ћ;wLtDpY7N^ nr] ʬ'Rsn RkDG$Xɮv1GkAЛ-Or]W~j}#ݟ7NӿE׳ R6¦R"z|:tx@>в~;o+;+`͇ZwtM;?~i$tuđ 2'swÓB>elb5`$l Ʌ $F߇m [+DVn&zC+0:>G#xTz'B4#|^T*rS9}/"vv?0DeHv5ArD$?ȥ)Oq`@XSg!Q-Vڕ;fqx^(WCQ:r84UdEjݏ{O1Ɗ{"_Z oT[m?mMZǰ?7H;2vxKb=>Ɲǻ`9?_]h TUEF7E_wES.8}+ݻT |VFV@`ub+¡ w5z/0"[YW|| zzjCcף?H'nvB= _zpCC_6ܗ `U? F#[HxH> HRXQu<<#8y݆CEp bå+{zLm# .VΈb~@H*Hl3l/DT"-enwaL]bcy<0%r4M(i`}&(zsfQV" ZI8sDbdWx(\R?{z9:І8tI^ә' SYOti<0M?Wq1~qWW7ԗoi4M,zb]i⻸/E[O܌=ϿhCh/ֿyH'"d-ɮ7=ٟl§[O;ڴ]oODqD0mO0 @U'g`oO~F;37mSByu,]ۯ)~7i4.q]CGj:H+w`fzCE}pndp򎿀4t/ x=3n$ᆟ?G#s&aL'h+<ˮ-m"燬GVqdd ._GcDwTFK'KY}/BϮo&ssL̅Ȁ/ԯv8QϊZ0|$#wL@PrbH"NU"]??$"TQO /Co55¼=h`g6}X>ڟ#PsKh_]X񲏡{5C/I7Bdឞ<]dž,>xDKtwDq`ȩ&:.z^==h(v(Q^] Z%_9}zOX {p֡wkA޽^ _xż\g#Ocjy1{?3@AU1L씂}q8\#1I&`Alݰ"b^$OBCGdY ȩ:qRYOBz0[8To@P FXz]}>yDsdΌ[]f^<5e+&Eq EÅwrQ/p{.U]Xt)H:,XOLJp/ DZx^x} +<>x"Iv h9$?I~8Nr! BXGH a`&PuY " CёՐyAFpMbyIb io'X|ܺ]E9xnc'`tZ"zbAyȚ" >dw[fvތ@TM!u `k?ol]uA.D:"!oCuI?`{1k3Wг1<|2G%z{ b&yD]uabz:  Hx> DW$2/ .)?y'ט2qLƑ܅%t:H$YhUµ-K's̘ C%0٣`dvV 8'"Hei/,,* PҊa1 Kɑ0M's||BZczus~Vf11君tf&iTEj舆OsG&R 8~IA4;`͊6 OGy"yY})##cMy (HO/ě*JpnX {jxyvi[WvEMpL''*s`}xp4$ 9LN6G1wrَ]wiɻP=_k(9ݕʅX^z =7 0ťf+ኬ,c&ᑱX ~h>IA @K~l^AIK~Vgl& NZq`́#GJSS0& NkbXm*Fs!E0fC87lxd҇AIG\v-KS+p(.RE0 COW۸9IXb-/ 0@OetmR^CᰪA%6$yvTP=hw'ct]C2+7uͩp$E aHtc P֮Lk]ͭe#JnD|d2Ymq/fBMȡ{{~b87ot%:[*bJ}><)Q&K0vq<1Ny}1:Vcp2 h: iuZx̓.,p?$IBUUʡGԄ aO ]hWfkإkL4j*z] MKA4mX/(;4^l,s,$ك!genrJÂ^hS5G$='-&c|Hϒw$Q@JGV(#0pzT8~I=BJqf,W{|"?J'I\lLCu$Y5)HsUz{uE˜iZK, a5=׹_ u9(D:9v10 H 1(gdA@$B4//FiVYZQ9FBda&丌6C8 y{~,i(m\(P4&8غ*,{(_ញ\<(*GLmi'pxdN˧Ju7k IDAT#á EA~|]u4d7e2UPg0MdY=PSi BAA{3)z+$!;aζI^tɱH$hW1u<ǡL7HUX0}i$JzE{T[T{{NGtǟA8:rwư)T$]s`7@ӌE4M p0}y4][=}mȯV ($INw$QJ ڃ+ .{>ZN)wKAi7&nui" oPƩpIA@!)hxE]pKa ,'ggORRQx!ٽ)mKέLJo,YpqV;{GV xLyjzE*Y^x}z!<=[*ʷBo$&)$D.TElTvλ=H$I`BD!/?!hmSa"eY[huN"b6Lb0 +J# WgdM+΋JvKtg˹NOBQΟЦL ?(qX9ivNjN&R VR颹bfQ o!lG}xvZSr<͇SNHeI\j躎" #z\˓fwli59#I$;XO۵aX#Fr}j41k:rHPӾvb:  L`="bN`ڢ|ױ//jxr|Hbi"e[ۜ i+Yچ[wR~vtoP xO}E Dz~|~ Ş" RYZ -UrG˟lnS d c"]k4oM딕%{ePR7F8:6XSd0|^/yA!H?6 APʇob_ڎ"6$S8r klVJ Ц D6'#҉wd{Ho؃ċ2 <:W}[+{jz*jOylQNcaD|VUxӾALxcB6lNAOWu[{/g뼀<T Nvx 2hSƦ|߾jjm Y( H$0 AD}Hau5w)ˮ ;aZE fI_pT9`2WW L+DG PYcZC-}ڗPb)Q0'1k]U@gcmx^akMƋS2=PXEߙ{p,GOJIZp BRu{fY`ò.Yp[Ж%wFb=w8؜ IF'(T0GA%v\lDLT;a7w"^ۦ o# O@u-Au!1 #`ZDByO^?yE.-9EPX. ŀ t? 5Jޑz@x|N?b з^e¹p.- u3{wڏ#)?vYECA^C7:bD]Fx o@ rcIJ <~@KB}-qfE0/@yɦ~G}u# 2䩳:2,E .$F!IV \Sh,qstY2 /;sݎhotE2{{uRRwNxפt:?5o:8V3/ZwO]Bt+|[? ]W0> 8f2l}_xj"a"ˡ;溭uZ/yDz{\|\fi餀#/ ϐ9+R]4oݒ ɱfMP5uSaCR35NAԅ2L  I\ "ctj Ty-K *Z.,ѕwdq>$פ' bv=X!v۪zT)x߃wǟ{"~I`p~c8A988Awyu*.֫x>=$rByJ; ApW&S,뎘8.8Ck8|x{u|gi/ANBrood/x 2i" H 9sF?Z)+ &W툥0MĀ˰DޑxGL6߮0Q|HN\HBWH$| \x=|=J>6a o{ޑ0MIx(:{b8cI~xKT.YϷ^&Ru$.iF#>mT,2HߍS7B=1Կ`^fr949ՂߔhG Àf XhD*E9 Y/x"'0:1(֐Kyhh^ 8|b٬L^iS7Nrծw5$;Di)q8 8<CE|zM!t-4o'1"3z#bYT]w}{/8WR-_7q I0</"&~V{`Հv8$#&B!XbtK˽"2|$0M] IA4AD H,Y~ ӑ9TM$!|-/uwIx*d;@!,u֑h,|,kQB;#+rN;/­&>fygioCC>}>Wmmle7^@U@6C&KvSD矮bA濲!b۷*8,w#= !v& _B==j_Xp>t&lRXA!1r)/]otҊ~HCC!3,,N< Y͋EV'H&"8\³;S197]MHkZ^+^M x>l:+^FT*UUWgn"Su9UCl콀BQf-^@6,!Ql2EB˃:iUڢry=a OCXѰWd'%i !_.I&; L#>+v`p`^e^ =H`Fo,V-+Brbp"-;01d;0}Wfaj-,:4$?1.*Dݳ.+<#s1-: 9tMt<ޘDyf8sKyK9o?9 wX6qހr u&I,mz&6*’u8{kgmx<÷#=]fWϥ>LU&CuL&tSa{(AAX!HCnzN›^Wm]8{c[o{'~)LNO!L1~vh&M 5KC54{\@|̗TzGRYvv\AJb2G \׼]tk ʣBWԚNC7 ȪWyY` ʼ 2L޺Qqgm ѮM^.G$lM9v+@?y?rВN?59ۡerĒDut =ݮڝ dQ #?-׽"13șo+`&䉧s(zbw^HR0 ߏX(@Z_y>몡nDā,tBo3p }__t#XPi3R}KP'ii0A IwmYtED?UB5W^AoBg@)zuUlW]9LLs&w`<!Bps}G2]yDL)T ´@"-9}glD_9Gof'C*3X=]QKL*yt,jJ#Itt;2!c /emsĶ`*z6Ū^reW+LU {HJC9?, I,8T;#"[2C7l[Xvwd8iema|*C<߶b]#u[)thMʘf].FyEdAD%/F2A@`l@{f.)\\ Х0?|[Jbj6׋XG/x- % K#vܑ@Q7gI6W %cQ&' >vޔ%.P$01L;/kv9 Fz^4h,Lr "6Q׶ޏ֛(ۄRj`76M{:HBPBVp53"tӿ1 x}1#KYeAD w޵Vwݢ ;'_ PgAZV11=9cm$`jޑ- ɶS!9]hrw 2X!b"@~Y eiPN-k"$1k`T AˑpP/Ϩܤ`+RyA2mE`J /ݙA_ 9FjrQI0 نv4H.A! ku@e;W ɬcxaNMAQw`dr 7X'q./6j[ǵ<3K A`Ŋ>P.&.bu?~7-%; yv1 d R: F?mNAvm[?_;x{N]C0xmM5%ZG;NHK;9= Bo@u//);vgǝ7 :MUHe\[|[/[*{PepuޞU+c}?GBhHi⩴v i d R:Zd'v[çwݢܶm|.4Y \±H(k e_z9A&'QiOTʐ`:)9+<kF\{"Ec^ DmdƹPScxC|+5Dcӵ ^<8E1Mළ`A: 5 [e*L6_T9卬N޳w!m`^vqqJ3j9m|s0F HsE~=úQS^8Dmt]G&< cCR&'o^ b#'/.Ū-].! 1\1٬f#(y"@^K.cw.D=t@:^7u5 dyݸ $-uc ur:m1:m.4fݷRqѰ]z<`tA1GG` p傻7܅)uހ\x~j)w!Z1Zuh*4 zr|.FDg9tÄѰ@ck= +|^0K&0FǕ[ƚ8Y ʣKԅs/j.FC!H b#AgS Uul8%u&adedr,Y'e V^Tř'/E8CιIaa? y JA⒇1+Mz35y{#W$ 4U .|^ʧKA7 [nl73Jyg芎܄s1qBMɒ #MɲC`FtyDks1a2^B0GK3F TۨkS c^d$Fu\\h(h #7x]rhman'GpÄ*]13Gu# w = CzM1 rI {.N$H4MWau΀uIe_@{-eC$>ɴ^:A0 hs1ra`㖟A 6` G#&d.prF0 =fэe@+b@hrda`@!};Ee&e!%\_t7]F#*z6i2ג[턙A$9yG  tz~AxBj1k F#a -, .i81Mi-NJHK Ŷ9)4"pL;H{Ast {Uh0 bFDZKOx]biiI2 `NNY4}Ht <'F [1ҎMN# 47i; n| Fx @ea6rpzCӢ{ xֽؚ퉓I4>vQ4m "g 1ҎF^D4Fc#'Hnm#C3˥ g _hünK,Y8ވYыDNL]0멑D e8`Nj̦ߧ@tgppbpp%`|| ǎE~2XcHotb?p>Am%bY\p UOHOB4C*>|6*3}^/6BQ\f9|ڿߌ{O茻|7n+V{I4 _ފ _o;~[u_N|Ԓgw]X,[lW +po^"P5:q,_} y HCI/czZ@јɻ$-S MCٟk[j5:R2# LC7af.ꑩ < C0ihSˋ{pg FS- rza`0)nm^w~~kp5׾EO~/!zxK^ku?GݏG_!;]CT|[_ks\rɥbM_wxGn2>q%e?贱_O> ^.znPp1NX\:NS.LǍLE uC1fs ȤG= Od5L5SDfByNR>" meyL0a@;=,09|^v}{o@wO.U Nԣz:O2Ӫ٦(3`nzUJ1TF2OA5=2! =(pxDM(b3]yZˬׅy3+3P/N7{ i05x`R? -88X4;47ӛ0MRn`s'X7ЮXa#{kc \몞~g}~6m ~?LqWȇ᷿7}?1$)sy>>7cAz{^];χixч2$K_ LӸK/<%yeX ajj4s+X,9[ownu0NÏF72<D"k Oıy6=C;AcHpCc d2O\0`ArF6:!4\D^rfh|߬a)Gr^~-/Nk hHx~"Ӱ, 9aE/s ݅T$!hAlh&p#p[?IE5ky짱a&ѻ`7ނ{SS?x4^/ᦛn{s>𑏼w]w_,x9c=֯߈?ğ޾>ӥC|;sD?|_\BO؟XL18>?աZ:P^C2,/nTm=ARQ҆aU? *&0[D,{|^b tk"&g~#92Y̐ds䜀, PUΐSBp߷H=h]ɞO;n{{]s[g|G$ވ|=ދsϽO?';ؾ SSP۷ @w=Cu5Z|O=f3v:w? H$g>a5/ys}0ntwBwKpK^;UIY_]dB\ O[TıHnevXl,{!0r70:#Ea׹_!șyW8bIXfܖ;!$q0&7A[4i2uAK9F& E 2ya1- HyQ1LI ) & `@ށP4Oɐ7{ `Y(2(p 0E4-a.qo܄ϿwnFɽMb``ZJA@ @.gݏUX l߉ǟ؍\.į lٲ 7} i}=l޴c[z5"(|>Qwg݇oWKE+f׏.z%x޺};e|$UKmaˑNNZ:N`Empmj00PbY""(&syXj#'jpmb9⪫މT{^'k?ᖅ;v\|)nV"쌏Wnݲ ~/t tXtsm7͹a6㙕]WiE& ,RXd< GxYQtߍWAr6{%o5Na5rj ]هFyQ !Q_V؞Qx4_$!|sYhY7S{{G{oބ{kD7_;{nĊ+qYY2>~{=/y~s;k?/{jɞ^Nz:RAP*{n9LI4f$$ I +em~l0668#c`Aa&:W7uoU'Hz0шAps x!l m5 #ckpg6OwbAElca R\'xLthA:X>; oKF6? ho}?7݊7>wZxGp[j<v7NU+S{݋>pa<3-?w>G29[+_"(>zǡ%|kx'3O|n?R$>zDZu˶[T?G +n^P_\xRdQLUnZuݷoMGjS 0J&`;^0Y5h ֕'bEZ`3^CiB6"2RXĺ wkBCڌA{+I! ޿s2?x{{}3^7ҫK_8=48 U-S=>w3_Dvx7x[߉ݻolΉozOcxͷaͿ:Z/~yZQ*o~G}qއ}p}G7uJwc_tC}qq`}]7 %;, 0NU^mV^ikit=T}}Vg.L(c*UZ屡:zk *eTS%O4*;>*/6+k[׆ʵ :G1TJeD[0DpuXհU*װټNj>˜ ,xad3@͞/c@lC%Ͳhڵ~yZJE.p ,]w9X `ll=G=zhY/%'+/. .%>{ Ɯ{л4Eir.5cAjfd^/>Դ?RhXݢ~3WS8'6jڀcaDglz!`jGvh Wl6hx޷8g6S, ܉iX(h,lk6 qĕhg],4:E LWzzz)-})x=k[L2W~Co78)amC"ȆS}'ʙ` jvKLa|Rlʓf|\A0EP!2ET Xt4-FQll|gA\hLY whc'qNgh黤ۿbaBdbibKEBlkEOt=Q!b5m/C-K$F,DqFt V6l1ʃ$rItRob:#HAJ12-~d4Q] ;7>^eL2!@8(J?Zwh' #ȟjT6K'Ʈ DD;! 69MK @b{V>ŴlAADXXKi'U]7 ɯz=)SEt$">>3̫7q%$FL^dJFnV]XWB@/Zp@^Pi+4X?QyTwrN `C´o$ `fQb! lt < q6m]٧n;m`R<9a!|ѣ2[#eK,LtK .4bO_$g$DNҮ6UCzR1bx8/kopLP-ND#1I 0^\dXR5: h ţ4s;.V67@NY@bgK3Ɛ9Se8qÉaΙk;ݖV¥{.@;+XDl> \Q@M:'Fa/TO&VRmKM8L(DVI@ik‰ ϣt3A+# 2vE?vL]ºwI4 ķ*b?m^XR t4 ENI.QblfG4P-BW?GO"pV[=wC6LL¤}V#ygXTn08-&HXM.$uA @ۊ~ dPbDB5K^zQ'}iXqcl:'C"ak5ZHxp|gya>Kfa/J2-$jW\K{ @Wz+Ę)*xqdI0Kc3tx A@w`,tfEEGfKgt mNl EL 3ڕߠ7(KXÂfYj? m<"QZH$ =*5=WDk"phA7]/F2G|TT;>=  wʀ1DZ?=BMcHGd}K#W0mX[d ucEC}NmXiꀻ`f. "XYL{7?W 1Ew;,b(`T䥰$A']YvwT;謵Zz$8pM?^{*'r8q';SgܣJNoLϜ.I+0ql~VSAnTRxݎ_ wvpM;]l'%=*sy:ŕaHbs}jyЮ)݌] tA  Erˍ@hJH=kiAkkK˜ ˤx7"JÅ) Nw7"F )mX]9gW洏G ӘN'r &gadW* @GDl,).۳ 鍪`mc*Mv),cxT&$4"AHGF@);'[F9.=j:+M䝟DxEbETREAt$Ft s/}>mbC:k<2[| owd(%ݲ+ě^P,N| U[[%sEZr}'x L8ecYִlnۓnCe tTnlQXlWM.vcLRm=a9A} Y1QŶsI:E #Q^fh13 mʆGCcMEr?xN&0\s~^lP6{,/8^GpqVIg_ L'(59<7ޞA_DC9)Ea=}v_"v ǗU1QŊq ,'m]^4_b~|G_mD;  ̾lyit`;Q}$V ʒ3$ /AMHRlBtv F?.Td$aWjF%O,oH"Y"v=2`k ذZC&a괉ٗ2={5*zmKy9كY*d`UF5 ^ZmG|@~#PKQB [ 6ZgOV^'JajvUH1U zDZ!p&bWG6U=R@QB 6Y _zvq&tADHa9X9|{ٙkW;o`iI}0 :s%"<ݡb|+W)3q(N;'x<ObBn@Kn rE7_엞 arU5B`8?)Щ_D$Y@bHz;X-yhtK:\.Ov0U F5=L[FG'\)f}<B<uJiDJOdoQmOG*  :Lۆi(N :QCq8Y%x~;?^['XZ0Г`Lz ]_XosĽ"*ytI5>j gU&96݆e83! s׀Kv..Am@b SנN=Қ w{&2%{arA8,Ua`Eeu;صR {4d0b udwDfUIC5arFKTF5"22DjsM;۾{U VsPKy4L& kJ\l1**YP5dW,axh,pKe?RINAtd7psr .!'҃yC! umť3 J F'~jNl݂bu!:6e'+ #!DY#"=o \1hkxuY_eMIwoR-kߕ/On̼|X1,$1c^z..w$E汵xj){O=w;NkvbhvbQJ|6]&Dǚ,i32 >Q[$4p׬ \ (Q8A&$Fz`YRSpBAdCG,Bfg$.?ј7h)Nzy9vyPc#FgXqXLYgv`/~D ∌MEu161xSwA \ lYn1y8)I?|Q82/ {ytd|̬C1j1J".4PB!_5mDv1BTX&E =hHRF,zz3l6B#' K/olD^w{id%rWrq y}<9t)t`.+ᦫdH7(3ޛz1|gd/B(`hFXjȊ5#C­ :z_uW%V{F8l&Kb$T Zqe{)_7I+] EcxCC%kFD׽vsHAiY(͝BaeXc#%n\<|D!ȫ_ ^H催sM;)hdШȋ37 <(C, b3w-lZ}iM81 ՚ԥ&5פq,:Bh8 [&. #`pG/]!Kv #[Pr}[rMJ{}.;!r׏ WKa[𓳁fN{d[myp Kl2Y^e?Oƿg#GlD:ۊbG^ܰ F#شfuGcnpzop?)9tR,M!]S)QZ~ot$Z@]ljELݐgs䝝1>Ӟh/a4l{y%gOf2>/7byIH5`y/7ctc?7QQvm.JƑRT$ќL~':EKObs8aGAFeڎZ~X~5%U^S9:yE)f8ڽXC6׮yl⺷ #4GK$mú۸vJc$ ȶ0MZ!ԡo;omoU.?w 8ė ^1‡1y#Wivs$L2 941aBr3W's=qdzClcVC 昭Σ`{F]/i I[!x{.Cl^Q?Uazn|bRlѱ)N{Wv>0Pˈ(IŢذsKo4 "MAt &z!g04fZ>c7{>3^>Xӏ,NA$QqzΖf"xFLc i^d)*hjabVm׺##3 ɗB{/Xa,?P{x70_!ruѻ(0, f1pD7p+% žJkG׾G.GQD0 )I$:=d[Zĸe яկGa;I-QCר/&pkߓ #i gGpo5OZ<"zQ0;^޿&,+Y!pҴDBbaDP_o\*: N& 1ԉѳ-[8Z#w<4HD::w\¡xh MJA ЃsE{EN(*.?ỏGX؆%B| x+bBUa&,jAaC$5 ;op8GvrκOB:!Ht(\0pM @ݦհ %"$Y¦D:nrǎ#͓]$ LܻrJb 4Ꟶ$ AqQZPX&>CO*%Q^J"5e31_G.;݇W`;F+˩^;t=1j0KL'."#by`PӃ+\bN=Z5ye I&q4Mg'S7AkHkw wC S,$@Ȋh$m/K85;d  V!?q /'rJه^R, C".xŮi9"퉎1f-5{Q7эD2>Bƒ?@9l~bBuV9l+lBdg<~"(K!1Ȋ>,H{NnwfwǞZC9^aڳ}.N=BrTb0P(^ ._Q Ȏa ujԘH{C.TdGQ8wi[7oߜv̴O~k!> nO87iTbA t7dzH#ƒ(uAGJ_H U^+8&SpDAaZ]#Ve{;z lĂ H:wD4 CNbdbϔ(J2$"Rq*$yw'diHA,[#r }!Įx]9|n5emi , Pu̲ʘLd69Ȋ[/hSDs!^}}쬀gy:A, 3< &4YĪ;0;^afc@-cfYt+GTzW QLT#2Ɩw :仼j,˞g/x/=\ĥ` J*JmkᆕOV$kF!4Y7h6fd, }gB;˵],j9) T d]0  A%BuZ6yex⅗Q*l baeb FzͯmR(Xu@,x4f IDAT% *Dęa0n{yϲX>)Nic! Be^L%0DGXsgOWcdK9@Qh`DPB F12ad6X؉]Olf?t:M (0Ȝ.ax~E!ėb`j1JH,+X= 0 )"E77$߳0m|DD4"Q\xZ]K1rCysS﬍{8TBOD12uR8S>ό1@QȊACjWs9hᮕjU.NMCh bq0ud`Cl,`eE&0) z{wEF(P_?֭"?JvN˳ E\IL=yV2,@@d Go쥙?pŚdCWn.㑧wXF&P*+~C1(2IbuO/\ەo"좣kIu {;1~}nOP, sܹ""}*T}/4Y &k/RC+ A(zwE17'UZI[tc%0ۋh\Wi~6:܎l>.Tm+%7SGFxD^v8 5PGdAQ$X> .)gsc  'Öo/m806_/>ĥD{>rbnsNCͨGB``0 RiqdYsG_"s EV,[bތ> &(e,CܳB/_@|πqr''>8۲]yh'nMf=r>J0bZn~x|,ۊ`kxM1DW1œU`|˹]/(2rmE=%4:ђ#7kf3$8t('$̉(Lh_$E=vcEذ>baᘏY1<U]Y[@P7lĂE$2 eZ{!Y9-#c$R.Dc], Xs"dO|n?L'qWwB $Oea& W`apB]d<\Cq틑7c`GvoOc %J\`J)==mv[ h3ͅ" 2qӵ9tms<SpขTiXP5s,Yý1w}s "u墙X@Y"ev^NB/z!:Ѐf`# \QXeیc.b(iHCTj`P q٣c`I<8C_"XxgA_7Q5psgR%!lf~CdD^v8BO1mx@o K%92˲M \0cD1xhHK. ˘=][ 1& ; Py W/:{QJ;cս_D1ZQJ\n9W|mE#H\N/Ox+fB?{ՈC< 糎1pd?®_+]xH w#B ~c vqVED/a/n-c'j\=we6V/T\xj 1r(̝ş>]ckjfg^3!W<ോ{1k=9ܺ'\q )KsHE C7aVB6kɪbAV} }=&lcjĻEۗc/&D~v{LrG3ܝ K)[‰L"V/9n`+9:޵"\W0Vio$^C2,Ζ>Gl i]-GMZhѓb8 `D!D!DBʂsx\E|w&ÑSP($ 8i$DV).N=ekYVe(>t.Bd&, ݀YY41p\y$J" |v1]Le38};U7 Gt4}b.E'H\4r_[?48u6%;SQ),C$lb"vn)f29&E?#cjLR@Q :4APq-\0Lp 3Y21:a_Q4D#nSh&"s)[|ב Pbͯ#bA1]K|Fta;=6JTr$͑ ƴE4gW]_^uJ:J(3 Z^pF0U]X1pP8C ID4FHQsgy͎t ߉ xKUc(I,SyqFp꜆ 8<ϡ7Co'MƟ Vw$F29ƛגU?3ZsTX2kb$$'A8/ i PkoVd.8|e^]߆T18-nOva'N8[ m Yzzo}>6ȢdXDlC [6 a3I LlJZfS4A4˯aDIcx 1A xN$DqQ\R{BXl{K5u=N7) kRl̠&aZmɷ[մ=z!\h.0k3R-z:FI>B0j;۲,';#ጊ<}-X Up`G_t j㋖s~NmOz׹Ģu /Î1C%G=!D2/ ЈI1[w}ǻgU1ʧ{mOk(Wy˵u#@]'2hcx==CuٶĖNݡ#p iSV.ԋ/C- 0uT{dgez@T H"3 KRY@ .Yv^D!GBJ@>uP奾M{-ݙg*r~覻+F?7D@'Fs{lyʼn;z_;qv&9zo7uȨь͘;+޴!yśVcyFf׆1zC}2m$|gg^2uݗ*=AQtkKנ28* V<~qW3{ C_5sغD{x<k$KdchS/-D"jNA,5,Bn:,l}!RuU'5w:`:5_ͫ\=VķE,I*;n4lU{X;ƿis8ޝ:?|> BPv]~"f"lzeE\roNK>qk|Faaz6Q0^"5X?س`EL?;(N"4z_'y<̱u=[Uq_Bh 6 ^1" 5qk4ŒCt]wH}b"ۃ"#/z)Rz 'y/62$mryvG qOT}^o}w5w廯|(G& r%eZ=K_]»LHc7OBMkeZ (<N 1`Né|9$H6w!UT$JvD;"dw~s1>"+}0AB$|05^\u ;8CDǩ`(zKQ4V>aW; J4msޫBd$D5#7;+?fI駲сKZ,hإȃWHEȃnWˌcDߞ rØyfL?G( ,e+mvS_i ^ ?Px|dvw㰛c:Ri;矙װc\xf'{c!vEE^)E[״]k{r׮BxPƋ_;ɽs%=Dvgdzrcsغ^j5-jnn{xr]ht;z m& ,eH?9kXxW#}^hgR~8C}O51QbDZ=Z,`]о]kpר7GZ6nGPƳ#4)BX^2\,v|#B2Ź2BsnfŽV&h.v#bl_h"& X>$vk./[/!wǘz`?TcU=7'3UR '>0^Q moU'ܓnWp6gce1r}N"}\࿱{sO, YYC 0 j)ŕcIh$rqk`RKo)<D^o;!0DIY+ӘV캳V"n+K,_2.1s~Ke&I$/n"u6x~rCs-j剕4c( U%a0Rt9Sζrtd3C,_Iaϟ5b.<qX54X,وn[~vq?2L?-C~W*ZMhSnoG]{8Kܙ`1FEb;kD6NI`O͘=>0NA!Ոl|5 ^Bx}#sgm.Wː#א>"ߴkn:>Ǿ>9v= >2ԩG=XQgp(xF9 1yT$Kuwc}kP} P{ #/z!  ~L{ߔ D4 >`ofK_)CM`͑vDyiBt]vR_@&uX<'q/5$j2vAva;pur% XpFǺs:{ԯ;qwHbEPh0 I\?]zwQTk3[N!^QTr(*vⵁ# kX@F/B mǶ ~_?$;gNs9sF)_l ٿǖHen=lֆ@#Μ6^! PVuX6xIy}q̘z/$[!$ێm `ٹyMmn3}.A?_^$yHjjg̅! `+YVΜawX@؊5oa#Us rVV#z_Aܵp? ۫J\XHصp?]w@rwy([Wʭ]ћuprTGE6ZyH5eqla571($9aNUDSbrwvo|{{vG*,x)ׇrAH(H=ra_Z+ \&"E=X+{vxܾ6@MBxAP. vւ BLt:o@N@LS\1(  .+m`\_ = y)ε_ra٪5e\e਷!KM+AP:Wl:t:군xkQJLJE0dv U5?igK (]2ET%7J> .hhVU , zRӠ}CgHs?yvyHydt=7wT- ϔeDޠV嚝DƉ ?/ IDATPu&:*r~I=A;W:T=l h{R.AsLF6!Émoguo"RBjqy]e(:f=:E,٦|Q>Og;'qnX[^RÃfuM-,˰۰:=uZAj9SjvUuФ}`CINJvH]J١VyVO*XG[\/?wمtNJajv .fdqo5ii tcRSeysRb+ ͎s? ѐ6#g|poPF{sc[춣~ρe}JY]|TIyXrzBvbcP@}Й1I:.hXvԹ-U4Ƚ>咿cV{A4wTd^`GE=1?-/7``9`GRFw&dd=jPUϿBtU C֕mB!Ʋ?M @I{;hk&4p>'X'w!wUΐlp*-]m3`:$jZZ6ej5:t՝q.ޟS@m؉I 4D z,AŚMPhX pT -WWԸxn:jzZ 9e=DC DCJ`m-k;w]jY:BIjRpt.'eYBҧq{%'*z $w:ÑeōUl."'eT˶`=A' )=H0c>xMw*/+ $_mlh O:7!LXJca-^6g<-5B>wF)n SŦǣh9 ዁:*EZ|-D_VE""BzuԐתDݻsҰrv[ßxGt$ٱ/yʹKe.4v&Qc|XTOև-+ L6) j.\'\2R@ȁGY7<&P}4ZlWUg;16<) @4meQvu"S}?u$QT"x(wյh,pܰ zHIRXmp`)-{\-BvT$} 6oĤ2mXc'[r\\q-l/ygL`HVei_n#rB %1#H=YSrwjב1.T-;*E?F 9j#{Cbٖɹuy6a!a̭b~Aط"6,#26 Wv8#z tT$_Z!ECNs}Sm͌6cN_v[_OCtWU[t,xZ/^CZ"?|FrtB1uڷeޗQ|7;tr\^O }=1 hԷ(*놕+4z{ G2>x2DA  ?v jY_?7AQڿCAںz,[uuLDQQDzu;./G:+K7X&ݢ(DzUkP[SDQLu( ǁ&na>}GoOÐ.le"6u82:]@UV=8+It h.R@$lޜ 8HD*8!o_߆]F N6|C $G]׿ڟqz\.?++ԛ8yCC!8eiel$ ݳoBα"go@[PGܵ =;9ފ-!k1!l܊ְۅ~H!BA kKj eVG^V؏/yE2lƙԗH_{P 9<du?{lUX*Kt0fw!~5ʿɚ#phwVAWC4 )'z_ܾ\\jرȒ +^GZ )òcU;!{$ 4 Vm_+w e}jkU2U`G~ Uq9sUGUP,TבTo{vۢ3 )~-x%medw+:"LUb+$Tz<$etֹqD yd$:YӠ7!2ꊛLF6c99sa-^Fnr%UmF' #,JYHĤHTe5#h:d(3 b.I-ݿ߻"rO{??^A.9gE$8c}v kwr.ڝyҚ QA=4DY=¶۽IFt:+?P'}2IX50,dH֡C3 3p[јW| w>s !qt>F aPǂ~E^+z]>HS͟VN375QP~=:*9' (Q/"b2 9'5&ׅdŋ̟%)&־=b++Q27&Øf.IG [p`ujwǞ;*2/ll>"|c-wX$sbc--i)j<ҹ7AQ'b-`߼S/+$CHfyȂ/lhk#Vr1XOҿlXؔt#q4$0$|dvK._$T6YTqL䐠&3as\1Ny2O%eXdk htg۷BsO19sޠhD rz7D.u3]*.hj +UGcpC+5d?8"LF&$ם[~]x@mQ[v[#iF{\; Uт48sQ0>:]]lRRMvmCڏ4bRj}=JWKHOM^9͘!vctζz@yv qpIJKިC Qڂ~=!9BoɎ71+G(:\=# .UVa=pp9|_DDDBd=sӬ_RМRLQ92:1֍vmFQ^= T榡R*𵰉OS IFt(La HII 4_ {ʸ(WctZ.Fb6QCD1;Bu`YεvgjEp8p8 D tLt(iͪqX8GE M$F  [ wlds(,F""jlaQ@ZݸzgKq'%FKډIx5  .zL~LCq6o&J8Q`LMBJ[+\aIIȣ%CWk>Š-Bۭ I>hc|J#u+uU;۪=dłmE{`r8'v(nBj;:IpZ%l}ztV%!ӈh.m'bfg\$"JEѹ1g!71)}drղ>v%5H'5GA uh~v$$Q ")Iꗂg}b0TZ%(C疣j{^~&BԸکV  EaٽOe\$jr9+ A<s{D#_-i&e5V.ُ:6q|v xPWĆm"ٟ\|ڹ<z;Ӊ(rcd@~nZdvΑ#'.rHn v IDDdLohԡ Ǎ, 1ZW#7 -xT_;} vh~_oOQZ9zHnm@3?Z:Yhb2j1F0\NPz $ĸHԒ ,#?'YHO] | 4:R*"]nvᎋ_V*v ZDDD$:f ~vx~ N#loneIJ&lؑI\DG#Qި? 9I@c^H_ªiWL݁̌49cO5F(  @ ܐ SR<92*3!(X#$5hmFqQeT֢uupKv;"Q LF6OMv Ʌc8Ы9V䤪,@ <Ԉ+EJ>U밵ȀۓPYQoN":"xzތ܁icHNоnB^ltm߮\c"#vjjQVY $ 00&p*#YHh} iu:ĸhsQUSjY-SŋH^CZ`48m%t*puah`:2擌'tW$lڕ <\Dt4!:0!:'#in IDATL05] Yerbë{>y :˜1Lc]Áz ʫQ[_Ytyh32Qdz&S z=2Rd}S](7 Im*_ZVZa F""":xLFA "=UDnI@|7vv"/ˍ6R0A D}AubE2=D] ЋAP^#CxW":d:DQd )moDr~*A,_WMvklv;6[P[_Ȁ$!=i)7pKnzi00' F(ƚڢ,D՛x( n-CDDHF!" z&A`wN6WBNddJhAf&!޻!isQka Sf@qu!I/C'zpʰe^&KE42; D`jGR+ um}ˬ01l@ȝDx`w:p:a;a`t:<,#{IWd谒! "LIЉ:x Cx`aN2!5 sR̦$t({-.J.v6wrDH .߃ ^$"":8#yK#"J8AN- "=t:oRKAh4)0?#\Ȁ(9x[<t:=H2$E""a2)@eww @Vg-T_5eۨxdvEE}0aQcQIDt$?@FRm'5鏋QJ"""jO2ONd"Z.;k"ʲLD5/bS$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!$""""""""`2BX;5{σ׿wT^}tSUVRR}Xޯ-\ 7 }^̺>0#F~qHNI)w^ç~XסCGXx}vZczTF]܈TkPZz={}>mO>${!^QC0.&6/`O 2N8$EÀq1qZj{Qg-w%۴)YgO?N+OG6imE-M 1w7 o:4Z2r5> @ͷ_b[텍VarsoK2zz\-84+"N9tt8=QUU+Ï?Jm ˉ~]s 3^vg矿ISv;+c5=Ek%bXf%tz ;32EÀqy:#ACb#)aD-cbv8ڊG/ e;0[{zkVIJߗزaDQLY?<בhX~-UVN9t,ZSXu/c֛o />CDZǞw}8[· =3Ȟ6w17UUU`ܝ!j ۖ-g+o ._qp`\lbs4x@"`1&6oͽ zsxᙦ>:$|ȑ#ǻ|sYb3 SOK/s`wЯ߀kժDQ ^;p2\5Z<蓘;g>+ >BUOc a?W\q DQquwGX`N:>/s!`p2?ɓo`?&#===gbgq`8ah4`3GGR\gs9ݻmމ/?լ我xaƫ_{@gxw?śo۩~x< ddd. M=6om eǎ̞λ_}9 ȊÇ?]w~x°[c%x|8Q=fEC\rq|G_o Cg-ywb ޚ!N=y~Avo?7 ԓ3ЩSc(['`Ř93 ׷?V^6J ,W]Xn߾.XSN9oW'r'j>>&6F[|Ms0if||!1w0X@ØH$49ryCXv ~z v؆INQջwbޏcnf />ssg= ̞s'#WeA͘è lӾ}<˰Z-xǰ`ڛqu7>CX{ٟp31``@^^kiSLt0p`XVlܸӟ{z̙/`[baxT!CU,LIlݺIA?Ν⡉,C$~.+3/cxrj]/kޛ|]sxsLt:<ѣf:3^x}zń~g,＆G{{pϽwxk_;PZVvsӟoW9E &?} <\\5ZYgŠ)bU0az0Lx3_}}{y@#~хq[Tef3?B9k&̟{DŎ;cCa 4>"} ?@ح[<ؓ3ݻpMi3").j9׷1۷?M{~.8bzw.XF}.763N];1iTsL{]KqcOb/Ix /G^|& 3̈.Uc_|G&?ʊr<>9$'ޫOc`0v9yq ?njApzH(ڹ1Ć2& F<ܥ+^|9,{0Lyi۳ķb{ز4msX~ ^|YN`̘_5~q`1ry% k֮BMu5yqÔǞƸ;ox,+V?ګ+/B>Kp%W MyN +d;B(.ً aa>W 0p`Yn#FDvq ~gZ0iT 2˗;<6!x}a0eTYKf`JJ:؏0ܹbʿk֬DUw_[{eFMqyزe#N:4ukwu꘣~хqbVЪU6t:; ' ; XqQuxɁ[#'.-hApXn VY?Cq^.v?26-@nnN?,13.bi/*M6.V@AA[:|̚5=j<+njŪ og~5 ++['tn[+sp֙=￟ ?Wp)gkaY1o٬شi=ZʂSoؾ}jjj`D۟,1ۅ^kWc t"|gjz-T1OCqWa"@yy6;vBQѮC *ŴM’& m۶…e$aKe˖jWΝn^ T9? ={–5\ I6'u[E& v]s˗O? 0p0VZݎc޼|Y]tZ_ʿ=t3zݚAK@ZZ:~[K PY\Ղ\rpCڗ@]}-N~6ի/ֽ' m t8K=,cބCaUXRv;LII]bg4;u#aӦ }c>/jy6.^;&\;68nxs֫:=If*-ރ|k֬ut"nq]}{?UHEc\lŢB6 u)ڴ)];CɄvm;xᰋZn݄뮿r9fnvMPSS |N&?Gq޼Jh깡ǘyïCLö9p`Ŋo Xvu˜ز%lHݎb |lE ; Uoyy]P<\|7o(ϻ6v(DR%%{kڵ2[!777p:]pn*l۾wrsrrY f_vxF/Y mZ|Gp)qM:Mvpţ`ش+oїGcD]aʔxwtb$'t~;oE׮`67zո4;vY}):8l-܂7gw\u3c>L,No˗1cF|˜\|e8"Wr r&TCWh4a1||p%?&#LCnn^`}rr2{L0 s9ݺuBRvN=t?8=xɧc𠡸pig`)5Ճ',V+<~z*\.7z1t: ;]O8&N]Tڱs/Os9C.$aˠCiխ n}+p.7ݮZm񜷹sŐ!aN„ |l<熚cb󈉍V6&W'#Fb31uӸxG JL8P=_d<ۄbL<$49e/U{1``uxH \ U^^ƏCuM5\z̟=fxܹ>tdL4^{3~?ӟ*Pgq\9@}rA(@t:0apƍ{V^ǣ:ʕ/>eС=S1tP]](qwhn>Kի7n6: °a'b#OcΘ=C:PEE;wAcq٣g/?ѬyFzk&u;݋ݻw///#GvV4n,]2 `m#[nîĉ?+b;vl .Aq_W_F`ڴqHzU̟?@츸nZJVK/?. ]tv^|sfS!Yh-Zԇ_| S _<y(ǧ}>x'DDqzg˜ؼ4۴ɉ(!$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!$""""""""`2H"""""""""J&#(!M}DjDY!˞`(BȀ,ː!r%Q":@! Dw<$I!PrHDԒt: "#y`2D\AD-z ^ee55p0gь|#,{pֺz < {Iv9V[s5A0Jm9cKN؀maefawa6 !b.fc l#ɖ%YCśPU]yJ]So|%~G)E*d5z#gɨF=dz{ E\%KiSrDP0SЍENTFYX,!J) <d@ 3.x.^ Q b)Џݐ ڋf-#@ +nDCQ)JQS|R6.^>CKEP E:֬*gPӴ!Ґt[ WEſ ,$%a|ǎs3#O"-'>CR_ (^t)PE a|' tú"F)R Rrq'H'eݤ)~%wh^/Fa{325i_ @fM eQ۲ȤS$dE#LLL "~*sSx(>wa{o/^WpnxTnxx87< BۋfmLj( &Ll e!T*sĽZ  ZBR_ )]p)^*aBt[Q9tfmH!$&q{zܙE ]x.'47U?-`v:M2K4 `$$Fٝ kk.qF} 醦FY%dlAAoml/vJu(Ŷ72qSy  @O[h4B޴['8SO_ Ytw"( )#E^@Tl-zNЧ;x|e(5GwFjX-\$Quڍ׶pTղgYѥj@Q>fD?5/Iv_oe<7$ P~k4!$&޽3 'bX܋  ΋iRH0zg \bЍp&ݩiF8)Wy!g5U+7Q!h_1 H)Q4)+ /nK- /q'-xFYBJ$Smk$̪/l-ki<߫ 9 0Pgڊ{d&nn<Dn[ԥ4;#5" !Iσ?5S-0F⯒ kt#As5L>R9S̎@WGA>j) 9Q8[8>QPk4y$"ֵW$3^WǽHSyqቒUѮbZ)("QN{QwOEx[ҥ-4C v<޻}*׊p~GȗJdөI♾aF)كjA;^*pc8#^'u4M4HےN7L>-C:r:M;89W83]PwJj̋Txmc SfTRus㩱rn}-rVfc7!񃐘0>>'zgTUlȳV'VkŀK{ru'Bh4U!%zn5?E9^}/r¤ͽE'%+0E LY-/ۤ >?Q8\B_{QT$e~[ A]OooIH' dR&Av}^ i Tc;[qnz\'xEWgi4:xMd]޻})Q'N_?_,cc :q)%c"UbI\NI"H۰}=RM\?ηCM'<9VӴg#N/*ִƚfNz\(.nQOh63B ctczE zˋzq(c!8+7"3 z/gϷ\}l lb-].}^+l>(G*:dt;ڋ&EHA6yML-9@|H_wWsL`LՅkWۉfd Q*77Vb+F| x1ȿK(:͆Fx㛦{uKaB]f$5ŮCpS8W<4_#BѬkR&Tg͉ 8]sW{(W$Dّ1ƋkˉٽInN5 *8:TѝiH$w$a#R)w$w爵vUnU_V6NGsU`u2a[5&RmH5#4{}WYaU5 {;o9ȋ!'V14u ӷ1obd05-o%+w=laM~ud]"p/%~h)$&!ϊ(WyΫ۪[=:HT2 f# nnp(Չ ؑbB²<c0faך~/87Pgoz~[i~33M_TRo~nru@e):ѯsۚs.蠦=5?u1ɂjOF@c?s$rV;>FsI2;^LGJh'2=!.PKRQCcmz1:ݳ?ݥlc>?Z뚹PWS &%3My#ȋ}I ׍۩hW]Tyv:qzpcPXԉ!Z;EN:Xiؽ?m͎܄X>ARj>…E)sZih4mjFְi-uz_ls`H.Ff2}OwthqerRaTo{qF^Q@6{C#s溋 q5 X! (ٍAЮ)ϫ=$5J.i"赎9qzE[ i-zXE'0ޤc͎D!SMrwP} hKc`>On۾LJaNݭTC 0G#AS~<ߗew c%-pFB`% =1h۽=($''cHԝmN{4k+a#njh4 )H%Wyh<`|>WP*/i,ϱCX )uQYOtbȡQ޾%q'9RP,n;)%1ӠԶw[N7˾ a%*͆6 ʎz{˃3򜬟fA ?w6HT|ZfkQ/IP£8sSgoʵz^r]f7$2VzӤϮ~{U:#2<~}a/dc!H+0;ĠűMz07:5MHw֫.2X@vqhξ2R@Y|Qiӹ?4DAѴi]i'%$! RR)'c;I&7v_Xz6K, E]4Uc蛌T^) ytU*d;HOMyQǦGC;=#f뵡4iIm(avԋs7نAxQ)#;3GցFI DA?2;3* ,.cPLVM|)*FlHyid ŽhHBP[ M?=LCo*tgǴRd>n7G1o4Dr 6qWkܘV P+j]ē:HYK&%w}cf,Ӭr7CE%ݦ!4hw}w尻JZIjly ˧9]ɛUs".  ׍4 PH;>1@wFnPL{#ͳQM\h/Nb308wk%\NIjf:巪2#ù.mm`軫KwHj4k;7L2M^ +Cv# 5@{zñƮ>͛i<8("w&}(pbDP|!(D"r:*G[ӷ}IJ\BH4ݤhVQ{̋BH_al:nl~1$nM"Mg=hn!Bǖp7 ZiG$@0~f[#{!X1B3M"!R! $MÐȘ~,'1 $ EUBLbڕkW IDAT˿sbfR;[Eҧ~?~g4vs30""@ I{Q IbUOY&-SW~ CRYeS&?v;:"j_O_b'm{zBrQJ{TPJYެ4M SNy:@#b$ZCq[5^6Rڊv';pRP}RtMb1A;&{pmʺНi=QrJt:iugM"rE\*BJ!1&fB=L[3YYLQgY܊ȳh4e`Pq B{2/VQ' e4AHLA)nO#8uq%^Og5Ue{wyۖlgcCG|6ǧt(;)2% ۠ كjPXVF3+a{5o.V,nwFzv#dxdH )1 71lIn:^#mEF|㿝-k'n$t 2Q2G{=%ʳӳW-P)TA%|e"=ǰ5Vo9fϪ9*݇RLX!F3~^ݯ/{l3y+_"}g/:՝3TcDe'x^3ayE& K)V6:||{WqZRD1P0T <ZJ'ɟp7:3rEuq%N7} n.Q"c8*\[ {Keb݈ rO$nhuR)UP|l'=O>_2Jư-EzWƹ1Fn~݌J'X;>ogd;;"^e/" VPi&13L^}q+_^hڍ wͶ\N/?;wOՒg.ts:oQ3kÔqRmڶ"ΤMЙ)%ww-s!߃Lm{[?Ļ[n6"C ‹ܱn{ũ3;Ӓiw@j?5AѬg!mWķX^;6ɳ/<{?2bTobx>ŗIĊZ,׋ *<*#ѴiHͿPW ~Oxi⁁RG;"L8˙q煨5"&}'*87|2{ޙpE]VWtºyZڃvN{M|d/܄u5YQE pF} '*xc].N )MډnQX hc]"r_@Ef󦜯'(]x7"#xTT;%G}&.!İe|n Ls+5Ѭ_&?9^~ .}0^4Ew}xQuoʋp|E\/ܞW^EViV*㜀;R ~#ύlvD>q=/R^Pv5983wgB [oQ{ #Ot MN]fi۵C*Ɠ/"J_}aK"Jh(bܲ47c荀\tb|vuY?ӷwmW'N]o#UQSJq}a\(qH 4.|@U}Ѵl6Ƈޑ;5BzsB掮Ћ7(?XSٗ@EXO9hVg 1 3\-E+B??ȓT`Y ^t)rH/q g5b{|/cĨqbh~ɓ<5ids*϶b4C_)vܖD޲%p33r%Griӷ{rpN!XQ\k*_~/5Џ Ue gMnͷo:~Uw' t%f07SYЋJ[=r/?u+bfY\k*7YBҾ7uoҧp6_3m9^z/;87gC]Y+H 材 =$(_SSű!"`o0/My1^.88cڋr$5tKrbi k_Jak#)~!n#\CDADP`f ՏK\`M, P@2i#1>#ąo3x ^zc_z+_s;[.f,j=?ޓXͦ7ֹ7}4ϗqki41~;Emn_>I8 bm鹝>ʯFV>{o" wu{4s%ܢ,x"Ͷo۹<w 6O>\tTY~gDMy?^m/ޙƗ'u fmmGKvbmV䒞c( _KE?;S7j+*ۊXHfjӌ>]+zEwF3R)p0`,Vu^P\_#b;[zƿEJ@]WH 4OŐ|#uSitCo/0t2y<ſ$;Ǿ7kOt߱^4yEf9XA+IJSOt߅{g^Kc//y/(;{!D@{1cCѴD4KYb)Sh4-" iǾEu*t|_r‹RVאַƴ5Rƅ9ȳ)uOoYԓ81*q܍tO8kee?r}(M"GHi4KEHAb^R߳l'L #d0 doiġPR_Yv)8 qC[/:"6#y~h]*GO\Kʟ{d,~Q cT+K0ؔdw+U4F:eWZãAS'WA'F;&*(Tx.22ڋM+S]柞eb;w)O` ."J#UwO.>FٰR=t+vR ǞF joD¨Ce^/Ӊݏm\')6-N;8N@T+7c} tމ A_)xR;q;#0B,K{?'Y>sݢDKe3q0lx%ύye}~P0Rd%1ⲦVFRJ*vm>w]JڋF8AߪU/y!.zЭ ~P݋]F$(B^Mb2h6&4^Q'Fkh-x\qՈc^НkdJ/ʳ>'Zūx\ٙ$_MLh4u$q/_:zإ\.2Vlы0-m[VŒ_0^7A:\2:)Zm/>E,ۋ4u܉| \Zz51V\5LL A8rm_ɗtVs5Gv^ԝXߓňx͆!L9vԋ_?d^l(p'|&:žFS "{wϝv _ab Da;3qE]g\E E:3KX8QxL\(ZjiL+; ?ȎFU/>x^엲Lb7)a/>S5X"[͊nf?h1./s -ݧ}9hoݧ>J|ٛ Y'ޣtgDᇒ{̟/tUY%p|T-dP՚v<#=]R 5J& ww΋ \G/Z\r@Toݕ$rj50odnSY"+ ^ k/I#  o˗v +eb VcIN\54L쩎~jAxkdz׽\+-9|ZQ~u#=g AzGBlzaeuʋRJYO^)~Ɨ-k^T@f ޳yOyy+xiiw|FY/wzZfS#:} =={c[lCqɦzi\- bjAE9orǦ`;{2$lyqt\rAzKnKYzhXROlnL;Mzx1?*_¿B^\1Pt8. Sv͋H}sbX8E8񒎡@臔.t6?S8k &fd*b+<\ wZ[b2S3A^L鉐ܶt ]mͯm $zf;NGۆx?zj3t缘bcټH"1''~p/yStΉS2Yc!΄XqU= V0g^LZm#/wHj.HC t- &E.Vx6QiE8 KaU5!3QDLGJg 0|yyw^-1fS"ZD,'*/}RHLELi'atgCûھnbIPv ~N&kW2A/0mELta1'ꢥK>uΉٽIZOfa[{W'iԿ߿VI#Q2u͂( ~Y- m6^!QI -~Gxs\y"nToo^SWKVǦ٤`jf[xR KJ״;"ɀCv]BIQ!ωQaF5 "(مkɬ! cn'Q)HQ<”ڋj=O˚C^APv x$,/hoq@VtO鬻[2qoͦC䐖~(b(L{HĻL\l2҈c;2_f1:CKCYtgBplgacn/'=״_ɿBf b&)j4ŗXx?9/f&JM{?"+CcK>3-}[4u&fS! {cN*m IDAT1)Gm@ )u ѿTk!{تp㦩tE7֦7;uf[:H fkˋu"o ^{cHfCHIrCǞ"ErYV-x19EͦBHAr#q<)_δs%3rMkdR9]Rp%S;k40??M/u#5c'|Ki;iq冉Wzܥ8n\v0pX7r^czͦC1Gg2|j-%ad!ѯlK̃CKXpC/zDR8S vT=\HwniE^S,R0["7zq[htDͦCfjv]sɔVڋPͼzgڋzZ<֙v:1{НhKU'nD mRgk ^$KIGRt5+uPsliO";(w*9StInw=G Rs)^rrlHrmGDTD6u(+'NVD/ye[/ ~DCϽ%ߊ.SOl:Tckv2;yP:h4*P:~/&b\#~WӖ㭕, +:C2][5츢6>^IUH0Hu0-tnL4l7#5AݚM! duH$mܑ4*RS# o2q}7F}!BH#@A"xLI¿͂iƐ%|YҁwToP ^19UMAYHwĉQN8Q:Cgt Vѝk!H'k 6vH.r{!FMz:b7*%:U9?0j/_?5169_ o">e*W[եY BR=*4xQ)cnͦA Xq*%U6j ,bkFB`${;DߣU@Hб$8CY#`gkI_ɂiXED B#; m`h6R, YwG7FfozVNNla@/ Kjk6 B"geˍEe[*FJP/vfs H .xr^ bgڊۊk!&򴜌i %"W a@Ph߻4DTG3{1zra8H5 @+k6 fjξfJT2uZNDXisvmŨ(=>'^ۊ5=DwFR vHIsA+1E[-zՒ?_a<\b#0l/j/+ P  C -,>'¶tF% U~0u56!D(6o`ȑҔ(!Qد:Yoɏ"}!@ 77$ 14D.ցM뀥r\6hx镨M !T&ϥib.rJB`0𛯟+ [d_5ALOE~Zq݅lڄ@|8Zxdw@PC4zaEYG52,0Ds>h!Xs\;5! -8'2u)%vm/zϲ{\j F*|B iCllB|+ "p2"!ա!4&U! :{( B^HCRf7h{RZHIojh ~ZNB*MIPvxv ᧃVyNFbW*B U{/ i)!e\4lɆR]e0׼zڍ7彅 aD޴!dtÀi #hFStXdQyuhgkHtZ5P [ebᏪqɹ.ZHHagCVch8U1#q"_děA}I4T,Zt'* T>qEZkdC,6-SKMBvд!څk`ۡ|V[BS,]R#dh \}ow\tT #j7Z͖lkDC$&GIKYk'Jox7ЬxS'eܯF"\2Zgb6d&!]mDz)O4VѾ]Ccd mj)u u~4rV F͑f&%5猬7ٌ}ޥ:_MRӜ\_m"uXϽl!4'_miK,j*+*֡[ڄ64%K[@Hhjn)0Ρ4)- 0MEvP^*_2¶*f[A qe8 0N@k +j5B0Vk dP*<b;bcd1Z2X$`Ҳ^xS@#j!5i h՜\M_M'{Ssr16u CA!dn[@J{I誅D@@*487- DIe BpֺZXQBȬP(;bC< MJvϿl#JoQ8>J7uk8>mQ̦R"٪5d m$*@0wdS#"|GFHhNyKC,cS+LyPgdIHоB0Ϭ6ʯ~*Ը3ÈKDjՙW=sQgZù5%cMX:W7R}sANёDD> K Q!R>4Jk/dbsce"7UI:G4'#bɟ){Hy>5[$jvͼj2euN[3GUmQ XyJQ>j;rV{.ƷЬ5ؖZ*ݵ I%&Zhx,nt4@ eu FՒqA~FFM-m\z%;yphO#9="BfFtPh\2-C %7{^ 3]hFNFsNAsQH7ս?tVuTY4е/^ P؝aR2E#<M[`E $ax8\I:_ Y+=^T '~W<|߃*,FaD t_ռ3CiNΡbFq k8jM#:3]1Uo]m&v0~ Ws%8,5& ِPp(GAڍŮ=1L`J3@rWb :!4#h; a\/R[)Bxƾ aFa8pqꬅaAJ51"&{xג+:PJ#C58Bp~ߕ.IMbD$;SV|43-1K#;"yEp̧|2`$ `0, wWx &<:9'FꬃA$a0NZb}~gԡDأn5R__Jĭ4n>k9(cB:+iMH [Lpr.N\:X1_@@/lhr!qteFR Ӝbnˊ틍mO=7FԽ]it n~ɌfWm/ZaHyCx)NFNυH#Icd%lt\L;fM>[)xW*p7(;ĢmC 8\̘W^;ec$uLN6_48 񱖆yϺ.ofx~5CՐ1~튍m&C` TEٶj8nR1k?lHJ:s( u&;[\6KߙAVI=p)b8{,kcSC7BXP4rg/NN7a6~uņ@+/B,/RJ) C+R`iqrݗ?I iADSlbcdZO2!"¤/1_Nܵ/[ " W{T(x xYu,5BWAB]ݨ\=q" bQҜ/:[n}v؛C?sRFecϗ qY)s~0`d#k->\ 2!,s:V#"EZ/ն-`3367a>LLmcllG;Cuɲ?>Qs2gs+?VdL]ۋunb<;c u<+ɢ@)8pK䦘`ǣ{FT3G3Igl\D_B֗JQKCi@!V= >E&݌cfdz\'Z؈#RT1aêaݪM=ȩùiSO}&_e;5'K!p9ZHJ7OW9u)T|[X{v4n״HC8}(sQKg2KNJLܱcY @|K_k.+>AϤ6΄Waq$PGsusӈY^\-+jaE-/45K`w1Wb][\ @3Jh@kC'%*VQt4z,:СQlz~"5^^k."siGؙgݷ͠;ic腽M|&26F9woN ,F]{bޟm1#ڟ@׾GVnt{o3thY?]\Yؽ?]q6')_л/}^XG.` !h6+ڋՏ\2Y^4ѵ'-(/~g^ErTJ6=bv}9fDU to~YwggYV\/T9™/+NSVƱU$WͿR,~/ŁYGGܸ!ZV G-wB,d3bC6X\t=dG@rT09+1=+K+E0*Lڻf18HDr[ #u,rXs.^䱼HTV 3/Ewbh@3+Dl7N\uac:f\`2];X%ϩ7 #ʾq bzX=:6 /^nq3.|{:ɹx/ #6%ExP#~m^Ht93_x{iE~qLe7N"4|$@4ŶWm㵵#3py Ѳ!DK2z{y%M1!QԡOq.GE+l\T^atdIE|!]+ Ͱ`'[||#HЬУsaъB?Dw5WF`9d7rѝ>pT$ |OԅΏG*VKW?8z9j|ELz~* "ѲTpv,lI&\cYqb&s.=hż6FGGe+ AZg޷oI o!Ѣe*JGE~}sEU8}(v+rq۶B[m[{yx+9**){GCF,k}TUȞ1tS7OJKB!oBia:#D/oZWRe>1Df΅C7&`9ݘ{znmg!$#sǖxml06(8z%ߣ\8Q89jhQ9OHdJϠ%)lhL̥MƼd.Eρ.l{'!4~<<jy bl&gD0y3(Kb+ZZk^1rp}]\JV[N6 6TV,-/W>zq*n+ EvܱC*ΉVbd0K° @y À0> *f߼¶8Qd.eQ$DT"pqk8%.HSrJXDL /kyŃ<7Knu>_]=[ZdFg^C"LZp]r1b0- Ca>ȵſ?|s8]dim|&pÁLKh-#0m #akW>嘉#-? {3?4DW?gZls)hGC_Ҁe{o;4H@yUW??]@l!idι~rq4tPj\G2o\hH`uxH_̵z"ƙ(n>6`ۢlC>yMt:B CM ].EdƘDQ<ĭ>^pss\?kiD{ oBth2gF)+66FcZk(WC@"# r#}DQ~]{bCЮn+n#3/@UgIHb1 /G;)^H`Zi(#Vnu. )е7}q,+4EbXq ]=W'VGfqYsȹLS`´0*ߢb=E)\\U3e ۳nF $w$5>X+ŋߝ)>sQp<mlue"=;=ds&2 DBYqW[^>k[V.=:a4l\B/7"yEVO~? i t_G Ͽ f )aLD-K`݃>HX{Z-}ٯMMqh9Q3^9={c0kQ}qt_x k8( $`q.f98}%"QxKQ棷Ab!|ۖu҅D05i kK%ˍŰ0l^e. s<_h ЕXLG59aQDm`v‚{ -/@ ѵ|u0D_p{5S]X+ 7j"Z܎E|C9bsJ/׳蟞svGZy i8fFȍBl{ްV7=OH!u0 w" x!V5ˇ`F#|x&4Q=0t&@r8k߱rћ9`p~: 4 *r19fD2bm$wFm\| u1Sg& ߜXDV/S.zN3Sǣxp ٜVBH) a@FWp`0*J)?ac/ 곬6|̮(!@7/f ?m(Ώ[Ri |i= !1$D׾8a&WLr5N#;czacFг#}o޲8|ƛ${E w܇3!H+" 0W0,ȯ%E~@ÈtKķH\C|ݼ.ޔ%B? =AЈZư"t}7|p >1N~4y*+ jH! "ȯv-%@0`$$b-؛,6Ve5 y̜H7}t(-Rb.?7]:|+m .xĩsF/:C0_B 0$1 }[]/9i{F`vyӈ\E̞LqN4E%vmxg6y-:gab:@q42ŬBB|Ǵ0!! huhk0?z 6C8&a&n@l`]UVx9Eo.? t5| B$ - i+ܑƏYNi(bfp?=ci Dfuul  i  9#iKHKˆKDbVٲckH.~n ` CW(}wo17[].u~d*m p}ϓP)$b Bw"Q !X|{mj7"O|n SSZɎx.^}kfdbGNYILN*D Hl SBFDD ݑ:YԈL GCQ/]s2lhIكmw0Wxxb^HVIx+ YX_q{4]oVkg:KβHFQ7_ wLx'1 >ӢSsިz!AZ{OhŋmO$+8^Q>r˯M`9VXx3z%uhT.|pa/7ZL:kwLΌư~VG2̗/a9Y6Dndlܠ7 MݗUlc@e,Բ}b>,+8}8pXv!\ г+bG9VځiL˚ / ۢ["pi*sos8њ |&Xx#'l rgR~0;g6<6Fn`A.7";bMeC3eeU &٭tG1ÆHBs!42}zSO9Qq\ 6VX /]Rmu뽯'SF0KXkh.~fSOrHvyĉ6n3g.xvwkh&~"K3;#7;ægvWxyڋJ½ Vr.+}?t3\M.b.V\rdz<"Q䀙#gcx5흉ND16ceFr7I:@ ^:ci<eU^?ED!-*Vc)S\&@2*W4^֟.#Bxs!ҧrE4%5YgDm.2Y?s.B]])HQ28zDCtaJ7Q Y 5=ǜg4F6Cw,BeN 0t]WDavEFW*ۋ+k1+Tx,+jI/OyQ>|Rk1 *Tp=/}qbs1sS{ά2?p\ ۥ*AK+!x .е;֘\<ɿ=DJmlj609wrHte⏞Ñ&zO͖Ѭ3ɿ9 'f&v6Fv7a( #F|4||>lTU,) , I, Ͳ@~)$ٵDT00w؁a $F ű̿NI;kQFZ8QmAQbE\,l(gDvD <(DPē-чq+Eu( B34O!ŒE E~n={Śsq8hm H ܰ߅.dv P?a( +EQO~Gb귶לOqqx9Zs @J 6믙ޚgzn-5FeS8K\Su<`ZiA)1⅑'>3чgfUQ{QBȜv$sXB'>3GHaxb'۸v>+(ɼK22[a{O>yΏ|(}ˆH @_O'EcVG?~ˋDqh!!3pNo2Q)%}Uaˊ?_Q<pd$E ;1Bb7BNeE[~ +o<;(}(#ſDQ~5 CI@|S[^؇뻘rq.|gKb.mt%! i\۫8Ҩ ?Ҩ%PڊKYaIl  alYضl.>&{"ࢶ sLLg$A ݵ ]{.s8MÝ,+"6FR +aJ!ķFX.t/prN&@+Q\ݝތsߘ܉,ܴ DJn^w[ ۆeF+r%(Ad_}抲{B,WYXxG`Ŷ;KrљpKȍ\dy# )Е.^ U3y~l}fDꏵu@nԁXe1% ;a!֗ XĢI,=+KBEYg,3<{oM"{уqĆ!awŖ[;6x.:<\R8iiJJ/X+e+`QD,DQ+E>r_1|KzJ Ȇ9M;!"b<^ lZQӨL|/>؍mfXXV̍8$2\x9NSA+PRBn΁!:hb+ hʵh!v['re ޛҝ 9$nʅbeʨP!7;"uAl(3\4c.YK"ZB!fgC<20:ыߜ]9%tQj]/_6e. ,S% XfdYd&'}>W0gbΑp=HB.)-hrYqH=w:󡸶U0 Mp?#%=1_5 Y^+ؖ+Ks s!`rc šD*fspR.r`]\׹<0Y 3˂%L)]Rgn7dJ8P:h>J*#˗X-QbdپE06i@+nFpq"<*)x_7~8ٍĜ#FDtl-+Bk\FT3^M5RNZZ"]:Y6xr,C2]8Źrc.P!p|CDuB*b(6=;c*N0N;Jg.ѪX֛2JRV wLo;WqW7xhNYH( H h17p^@K'WTu4NcGsȜ_ExˊT76FҪO\`HlC$Q4gEJxѮe\2o( NA&Di851`Xq]@X~$6E*`):|> IDATDpBJ$b2cM\Eo~H]{?M?KFEuʤ2O(N7a[!fB!LHb{{JԖ}'Ɓn_/Y\I 39h83`c$5a&tDb{_]WDa&-t\?_|j,='̆Hq0w1;=\mefEoZHu0}89Wܐ lB@JH̀f#f >d"-耉hHXՈ!iΔwGṽ2=CK{Pq"Z{"?O$fBh$z'`vM&}&\U.;ÙdG]dϻ]|Wd.B$z~(MvlmǖM>1Dli^U;Ů\Zb6m 8u ΎE!9 Q+u5|:L@i02`x |8|R‹vU p|d.8\tNPTBivPs1ZFRh@0%)`X-aD%"qVOWrU=a8<) "J)kCD0cHSЁe@+ 7K fW:\WA~ "Ȅr aN!T!#"pX&AJ lS#D@<+$b D,D Hop BB(̦ e 9W2 z㲒MD嶆A *8N>b}1G,t!(P#tBG!xS!"?yF/eEj6F #L⢰MZ;[9ґ62 !$ ĒU 2z~f#mdLҬȯD[FH) \\((.U^FvY)D4 F"O㓟x*Bկ+^jl޼ 矐J##~v5J}_>O%{zz+>|F~5gQқނ^8|Pw333'?""ubǝw _/~~_Ry{?xWˎYX.eh>gO=#_ _w==ٳg159٠o_x]o>ɏ'+_-l;Zܯ~x{ӧOmom?߰c$wRxߒ>rۋ;x!kλ^fv<6X.%""j966o|kHk^z nĥ3I㝿[<̪rq/G>A/]}W,>O,l?|g+x|k/|Ǿ n>ɒ~ ?oC=59^"""F*/۷w?r7N/ cwmo;_K>Q?AV*–[j[o=/?}3x+^϶6|c×}ľ}J^[~=?ǝw |alݲm9w |_&}{ b'ffg<=rG˶>r33ػz./ٳ11q /-yם/G&cK§?Ews箅cm__zG?}>"qxu|_w/pW5gw`x/x;Z,Q%Moɯ47;;u9=z'z~睿)'Nlab+f{Gq6p7ulc q/ ŀw$JBB @BS ;{euҩI;$7Y~>ݖgvvy'4Gnp-e. ƍ?^|?}|3-}7\حkwVXm[5>#uuԅ?>$׬Oz3q=ѯ~. szOضų#2ݼ[xŗcr˭1uֹ<??6o'gmڴ<ڞ d|RaaA~SB!8m3eF<;2@3!5u ee1{#\x%|Ǽ x<^z 5kYfp}q xǹKxq0jhN?ѽo2&sϽ3z9oL8M6c6NrZd%0}>yB!jSӴHNNF)E.rՕ׳e&vYfΜkW]y=+V,W_`Yt| W]usͪK/<|Q˃̝7 ǹ\GK˖8"ӫ׮]W]Omغu3+W1xI\ֺCjGrR2ɘSNGu@)?3yOڵ?xcöm[p:P ~SG]{x<_Y| ϥ[U6:?efР,\T5 g}vD3=u WCii) 9} _w ̜93o2sִ3 !6>kN^?Phغus$B)Ŷm[k*N@!lܴ>}cf⢘+g͞mؓAFg'R 5OH9Ѫׇaض^x4T[̭o~3qEs?GZ2|Շ ~`)̙;{Cd{JzPx~|71˕N?:۷o3Φq=bN9q=p=l '}B!đtmhTN;u8냸3 }ƒ~Z;wngӦ r*ٙdڴɵ./ar^*Bzg Xt!K.b˖M<,--%=}PU(d@Al޼!7ov[7uf7kNVmM?Nyҥ+@di~I~eV\9暘7w.YKYYL:W\Cf)KdpM2=u27)?qMG}YzmڴaQlܴqP={M3w:駏rޯu3p%s]ңG/v=9eiNeCE{8,ϧ,_rٰa-ee0bImbOrxgۯN>93dBW\+M(u'Oe+B!hbfϞȑg1\rj,'svٳ'>O<863gr͵7C'c:LnVoX ~jTn6.2:TKBC$#'N`̘k8ؽ{'>sΌ-x{ӟ oGyKƐ~_"ӡNwwޅܮZO.gٸa={! $;;o䀧̜9?j,bժ|uܖy3a$<=|ѻ"͚ʛoE]]ENN6&'z?<O>W6n\ǟσT-{뮽?VX'=w?Yfoe^y/"77W^}SVVg̙gͥ^I~A>eNn֌fH[ɓ'PVVZB!DӔM##GIk]xZٺm3>tWa.o;۶Y`o~1#c7k׮G^LO)r^*B!ġRS zd|[|g^A?Ew1 W\qyyR6mұcgڴi lذV!"Bsm[nDyB$KRRc;vciq(sϽرa%Hi+'yn.Brx(z'0xɼK]9/B!~`dǎ]8o;oVcmEϿsхqEQZZƂx7hB!ŃwW3 ƏkDyBuDLB!B!B.B!B!A#B!B! B#B!B! B#B!B! B#B!B! B#B!B! B#B!B! B#B!B! B#B!B! B#B!B! B#B!B! B#B!B! B#B!B! B#B!B! B#B!B! B#B!B! B#B!B! Qt=q} ;mee|տ;?(4 Ì,?iOt܅{.]y&^zym @>5[lB)*.W!B!B!QFxoIKi܌B!B!0Oadja7uЦM[x)ѧHHеk7֬q4ug5W_Yg_{7xB!B!0indKظqϧ,X0-z1^^x| bؼy#.R-B!B! Fw |"/@IcWK߸ꊿ?7(.&4p80>OHľHľ4ȡCs%WSRRUݏDzBlܸ˶#mڴS9f;ߛ?yLo CgF&_ZՒ7.7.#/!AľHľHOS}3_~=<ԣq\8Nڴi#?Nviެ9m̚ KhM[!B!B!H!B!B!DH!B!B!DH!B!B!DH!B!B!DH!B!B!DH!B!B!DH!B!B!DH!B!B!DH!B!B!DH!B!B!DH!B!B!DH!B!B!DH!B!B!DH!B!B!DH!B!B!DH!B(>ɓpʐ"møo&2[QJպQFfpWcU9}੧ L0_nݎ;|B!58B!rqxҩc'r\.vĉk,{gsuukhnRRZa˖͑ug IDAT2u"Q呇緿UT!BĐ+#B!DYgΛ~.]I1B3;o]:Sc}tp-O?ʂ((gժ#G'_K0rY$%%ӪU+2KXKc()-e1dʫ/`YVep܌.*$BQE8ldB!hP@qϦOدƣb֬,]rDrs#;t;ow[n<}=ic̈́B!9#m.Yr Al m\zDgt Nũp:Hs*Cs4dfpBkDkMӱ+65ΠD"B!u95;5([nɧ0o,WPX@;3GJJJn3'Ss]%++Ʋ>_BB-S-[4v~$KsxI7}d]cw6O:边aAs/“SO/?A(ϣEN^~ov.'K]'lC1[x`:m1q)Cb1`U M=3f4He0R!B4ϿN:3HNNfoN/?gȐS(* _~Tm۶OH{[&~=&;bP(ȼgsκիSj,Zeq !o<#o<0O=h͸g6:ak[ۊ0\c4cqJ f|=?qp tSR]} 3zt`ΐ$zM.kz7؞B!AK8.Bx?pooo`uK8mE]NBdFw1[՜nmY}9N:iݻ?8}30 u;{kaB!D]4 Z=b86 6&uZS8m:]P&J)O+ln^/2ѡq.R74reB!DJt%: 7viI^~^L"|Ǹ?, ~`y9|+Mvv|%[z1|Ohm__`^e{E%(Ց΀}~ij/B_2!4z8eAN 2Θ6\>G(^(A&ɖ.ϷBb 3nsZrՏ6g%V'ſÚY[^"B!M`kr='y{w:z^ԉL:kkׁ]gi5:h[ Z:2Y=h96}Og(=Mcԗ[JӨ|eqL?abk Mmq9k+rmgI*+0l,r!<P6P@Y>x8nHn}".'sI7$2UnQ?߀ڵKก[:5GtL=Q=k* a(ih<2_#jWB _6566Oe]'Bl~KRY>h88u`hxL5}OgY,8IEq+QfҙiM_/{sl_ׂqΊ-Κ#00"1̃8R30A+MӘӹ/vG;RnaDatoT 9Vn _>~Cp=M`S`"u}V{PU 1*eWreŠųX<Ըx(ELND!h*\YQʼh.Ct0ʌٯIllǤEok@%UyUn0MS۪̭"9]q<Bḣ5=+r )g1P/AxeTOWcm9ii3w(.ÈzzVǢ1w]\a@H e1^8Gـf, 8aF~?fS9a,VTrbY썺`fSqU{uSt~1)`Y,tjE ,IeUzUgEmF{q8TźJ8Kf|ptI0c a;,Fl.M Ke_Wtـ_:僆kq*iqّvSU}a Z>r[r>z>ep-Ie ^:ERf]8T]Oz ¥[Qx]=h/ʁqTJ ٴT,3qPf6X1 Q4{20p[ dՌF~l.qxM' mnT FchόeO\ZJJN Q랸RBwRDW1ΚVÂ}ۓѳgH?gM+brR-RH$N{{dP7ARUgb[ /<2mܥ6ML.j 3~=6qT< %+fahW>IiElڝ.3Rܡ9FN> g&E;{r1H=6rI(Jo;6f0#&u5/bڮIoLU |wah۵ N'UʪΝ]CawގVvcH7:38kj;N9g~."{`WL EJH)&3>|9E fKtTw<ˈذݚ`d (y#m w:,JhY`**U% V]q# g$q"rumLvfxwahu6wͤmiE6y-qed&{`We3d_~-gUS&ԈIHAIZ4#aWV WF63OIɅ9zTߖ]xǨx^SziZL{e'0lN1Ƕtr,l }@'ڬΔ2]&4nB=kJg۽=k羥C ӷG-O'Xhdc$+]q-'jʺGR=+znJPК쾝9jQv󜍬=7p5~a6V]<-fH?',)jՒV߱1rsY0inan\Cu㝸#GMNNYÏˌ~cZk/D0yx~rPEm Ik)u]'C[(sH{&S4ZC]Pԋ77@IJ"ەǞ#[ ?)&ض%ͷdh\ПDŽ35m>tC8#]CU\X:Č! ~2`4?uP;߮| L*cN$*7l,2ɧv$hw|v$PTj)m͸Lzlkw0g*4g\N&\>&M7VfOs|u{g;MVmH{mH1';N^\)v^~.EM<<uG';b3%9`uxXN]Gۀ.YVHIľSMFܺfqu|HˬVg?{n`kI =bש=m(iѕDl Ųm'ݺmrQ(3Xu R\ۢ[q}+{x֧5Ɔ9(ڰd[Ԏ,:Op%)0o@Sgz&(A^:.7Xs~X=..:G[R3ۨ}#Tqcٖ"Nf ~s;v*6 -ߋ[N9tPhulEi!!};l+ulH5yImtspQXpvsnbͼutL:q9}径sYA“n7nKoֽEb8 ֟f_$S3b+Jb\N&V{?l9Sgz QyYn#`bvpFrFV(S5OСpٚokOaCKñ G6h[hȎ#B!hL(K"tՏhM2:ʟ/؆&WռM鹼9&1fM鹼sͻÃZ n~ˠہ3^cvQ)AT/U}ˎ&2jٕ'7 1dP`h3~Z5_m1,E[6] O߭KGPJ)V8?7)[t-C+6*ȭb~?Q(ZxݑY"WZS}Pkuw_J hK A&^έږ h/kr)T\AׯIF=al&diw.o\c6iY\/=Yte~Vdլ_Gy_'zy8vB? 6y_A"di{eNJo%UƎEӇ*svO3ry;mQW.E$ #pd$3Dhc]E]s7$UEnTȭbڴݑTYҰviFi 6Npc "^cVQ+[ٞofi,Gdۼ #*E,^C\g M F !B4q@)}7:'lkbޯ'm^箞ő/s|( Am~hR#Έ%x?b=IF;|?-q)fؚ{,֊{FS⮹2 _6#Pҵ~dzC( {<:Vw\;<Y}tR^*J _PSZ.6rt˘xV=ޤ^wvMfњL!o #njveE5U[.ٍU95I{y25me6l-۲㮷dBQw#Nu}m}BW`9)ʫ>ZMl0hnl63cʽT蜽b,΋l;ޚQ4.۠ԭj|Dc_gM IDATc=uvRCտԡI )296ڲZ_Ap\RwZ=}ҰT(E^} 6܀7$B!hMlJ浀C wJ]Z18 |&.ã^ZC{ÅV7+h(?uV|OoI$7+Β}+7+|RleD\+{u{^ (w(淨{S29cayjfbjwn<6\/FJ]ݛPW xd޻aq1*:D[i~#7+{+nJ笙=-Sq:@ KS}JU\>Ƨ(6tQj*)؟srt5z J븲.C)OnV/FylO$Or(b`I0 T^Rs >| ƹ/m2#Dzk|ova؊ʝq*ȕx1ٗ2w86կ'rWG^&W~3L YB!h4'-nMJnv.]n f/<-rV)xhJ~Akˁ҇G}`ӽ-_Me!-ʶq[1)q+~nUsx]'';Prn0(4/jB#>w'8B _}W8,(z2jV3EsoerMvukP:!ԡ>iˎ[\F+L6)v_GMlMr#& ӧQ_Tnv7H\>ƣ&˷iǣʴ[k;昤˺l<WiۡG[6w0{GN$KdE=>hezrwجyW1{hӤg5ɵJKT޺nlYaeRK(Ŋy=7QBqۏ !B4q?0ݍ]tGiMڪOP [+?Fi6fHQU49 RX㰔c rG`:{D ruKkVo}5dV϶(+q;ưh-t:OUvlR*}1+CaZPǢ:+c868CՖ!~ ?uZi5`qWiUvsPaI{&ϩWMvxEf5y 輯TnOۚb/8F~XR$h1^0lBJQybޯ>gP~s3d, aXvYzR @phTX8nL0?yUdչmQ\w F !B4u*z)wLR`k8lv{.;ST%m"׹ YR1cSxe+3dcV< /K7&S_?\r}2Eeᩜe%(q_>D?Fy^k9e.JkYsJP$έ@j+]$ֹ̾+sۂh%(">]iӂ9?r"gB!M r(qXYmIAKa؊PW{flylE1(˂{{S; ɲEpME)MC1 p;͒:ѳ X,$Ӵ갯ח{+U /[!;by 6O2w.~6/؍i8O3/pzٟV Vh_j#sֲEfQBWE>[]YqHw^ @WdڠHOKCȕB!MmPocE4R^>CV^Mg K—Jd\??5biW#܃@klbt[8+Vz~^8ڞTOrhА+(eSEx_i>v)oL`uvivp *gOB!h '_ 3Pr]Q_pl}E/cZ(b|sZ s6ٕdw(2wv/[+h0V=. XJhr@N /܂~S*05Vu=SK7 {ۋnt%uVS\)B)PhsQ7  +奛M m֚E3z:h=^ħlߑ}AD 5yƾ6|9R,B!hTŕ'%Hu0_N4`z}GdBƿJMb 6X"-BԙgJi7> Q֐W:<*?+wC;2b~r2 `}/Gbc4`B!DSчBpf(8r CYeDᐬ] D]#ы xU/o/ǧ0Y]Gҥ+7o⥗g۶ӏl[ʫ/e&RozB!DPqE4:nK-m$ {ېGnޛ6i#7Tb_|챿`<.xU}qDF:Smz#'?}7s짨GS,ٻwogz}bJCv(J;＝sxϱcY;DImtJXKiYLFݛկ~HӔFA\^`hhFc4MG9hd c׮Lnby_ygsȡٹ|_s,JJ:96NݶaㆼYP/*%6͹B񯏎R׻5m4]e3EOxV\O;Yw۵+!i|Yeϗſ; HC ]lE,9uӞA}) <jW}~}/Wox?N{ ϖ]{W8qzf^3ytӍt;vBX9 Ն;={mO5?N:#|ዟ[ɇa1-)N*AwWiT_zIB ^Q:a,׀~Ug{ΊNF < nsp;＝Ws׿/^e(oz?Xc1Ƙ>IyHdx V|%zdY:`1ӧ\y!4$H`ƘF *h zs|8wqݵʵ;k >Ou[fjPtW ?0_,(c1Ƙ>Jq>vfdHyi/~o{9|{aϞ[xyP*><ƍf>IZ\vؖ,KS(LccU:jVG~\yg?t.z-lYs綟xw^@ P(羀րjT!rW|C1@!i1Kd˴1c+ ZiWK/z}t{"{\WRlg:N` w.~\rq/;Y>s8̳޳={4`Y]X|ƕ(;=߃N8~دKm2c1OVQ㚓:o]4 JgrhhFc|V㞇pH/sN~p؜٩v;W]vڶRPNfVוKK~fqJrCXz{~bǥuA}3'?,j^dc1}nڧ'u\n:~IOҏ}3qO9G~4wqU8u?P wũw[7oM^^w.7!cšի+%$}A :O:}~X}cMAA5~MO~=w駿ﾋ(gIӔ˯s>'Gq}>ߚ9{q|ӗ-r;PVX|sث"{`_͒#f apcXp{d1cLr_.h JZlrݷ/~s㗮wyބj`ypƘR d\q# @(=c̠Hc1Ƙ~T!!peƒY]~%\~% _~ݼྖ*X"G楗vD}{R3_,ػ`1cfWi 'Ծc:4+c1B mTm=,c9 #ffY k VƘd˴1c7">') <*c\#X|tVkp`_A? nW}~%i1Ӈ+Bhd-1fvA;hJV3i1Kc1c+t2#|1 0L'DJhS Ƙec1}H &E[yk %Z|sث"}ZK!~5_bo11ch8P)T1PKh7,eFc1!E-ė@"Ao^z HӞ}ݫkg?0_,[f1cL9jۯ\p md -Ԧ#1N9*@9 cHc1ƘqwWWcQ(=s  b1c˴1czLèeyQjߠzK=Z>WרoNz(Q:OVҠUPT? nW}~%i1kQ Ahd { 탽<c?@jD hDcczŕ@<*οMq)6ڙ ٳ1cLW8;p.QiBmc2mc1ƘSTq.ޖ"@⊨/#X/kԮ ߜ2O3{/( N {03_,(c1ƘQiO4C[heB*Y& c\p:L*OcY"ˌ4c5Wh0.UQ˲šMqQVk%-I얀Z|sQ!=0" a>,Y0=|J<1cL>^xä1oU!qC3Ƙ >lu-c>91czǕ毙&"dk7{PRl23:i4GbDLc1ǤL覭iMdh7|H{4J3'k5椗f}H t0_ ,[f1cLv'ùSJԗMTDABP(O۶S]1Ƭ U.=aU6ƘHc1Ƙ sOF/gK. + N6.*ېMP楗O}V"5n`_>6gc X/c1=$n'F=ӾIH(XIc`PTڽ͠I cHc1ƘKC heKD%%jӶUƘ1 iY-f?ŖcΖic1/ ss7)"'&+U$1\) a/؁w楇Fz=߃ox@د I-3c1yP?W(jgNBHd M @4}2R ]1ƬZAiUccz- E%p. !E@-v1  2=gcb\Yj;>ɾ= c1s) i^;ڹ(=F\v9FӏA˽ lZT椗O]S>٠m~5geFUF"kf`1JQ*Pʡ3o-F<҆ji1@'bf"Z@A?1f,3ҘUG;.5Ƙ>q>uO^J0pN HJ{Ec*+uAi]QoNz%Efp_ 9xj`BMF +LM1ß/ M/!WEL]CZ8D[iw%_g1*!;Ւ=s@BHQc-6f_Hcz:١}NVYϜLs:q!y4(2^zZcĸD!5ȚeƘeHcVlyYˬ1W6̽E*| {綸v=K T? k`s#x#q,> ?ϟ>yJe_IT IDAT>^m4>x%\pQGw~[uǸp1|t1)8OukԶbk`dL?&dVMDUqȌtm'Gр"2"L |_$>7O6dvl69ahh?/漯g<};.C<q.ᮻ\~{O$v2g>`c4]c08:|vmlpcHW(.tvFO#?]põ>W?8Ox ,8[2fIj Wyڅ*9qE$Ҕ AΚjFb(*ы?׮I?.= @X?Ư+ΝWκ#^񶷝y[owb ﳘgqYO_-eƘnIy߉5hVOcY|sM׽/N{I>]3mN<ٻo_\s͗رN8qxHBDP$I6oIN3_ލ`` q&*z_g\ָƉU}lg?f֤i m/NF( ??M'6c8Ӯ߽{P(羀sNX|s+iB@5 apcX/d+֭[?C9a>~9Cwͅ|x÷Mp[of֭xg/K$z#]ydȘHΆ0!ATN4A}pw19 A4]:f1k''x~7T\>#+3^v*?]nwYpggC{872O")yHV={(Ƙo)O}Ք׿ͼ NR塇L۾^RyƳ?x-6f!>V]8.c:W994mYf׌TT@@C"R@_- 3n?ny)Ocǃ{[nH׫7룳n=42ҹ_\2666~wrׯ#b'k׮YV**jyُ,T+7|.z8KıOE"i.I .z˹S2rȴ퇆ֱko]7<ռ̳9rf~B0ݲaㆼmظ$6n]#tB^} $Lsaϟ|Y I1ų>[jPBju)$ A[ԆH\8b|oX,(z}F6o̭Bַ֛>÷羃w9.j!=8nG=8T7{rq$i;n;G=fmSקŭ֪Ry7G{sdfUGB=0Qدh+t2c'Mn'Sq=d/7#N}E {w>PUoDv$uGmeDhRhV\jH}њMiSBDľ{)fPL! i$o@ĿRwoۛ9Oz @6Y^.؇O~DPfsR4MK8-z?rGמ?|3K@sV/K[ছnW;Nzo;7tcSg?O1NnƘv6/eK? *DE2|$R]B$ڪCTh7RX#А-V Z'IXIEA࢙2m2!x i2+.2n 3RJ[ב^ƾUO1D{^ybcÃˌ4fRl6C"4YmRH<ӨKV2#2Li ]Is2m3 M4h{2+LT8ՙ9&(.n׃2`1&G6i*-{wo2O{(kL̚q:W.Bl]2HN&־_l/# AP?=ugrmӶW;c1l1$3F#h&Zyf͘/0xTL"qi&' _F!&Y9ChkEȚft@ۓޗ''CMX|%X|scϏ>?Xp{NNZʶO=T#(R)=f7^,fbMY^ c^>Cq)!4d[Ty)5h85٨HgMc1Ƙ>d MT\0LN|TF="j;駿XfٙQ./8립8ʴf8ϼ_'kJ!B\9P7]T& }9z%4p8MFϹX#@noN,b}~}ˌ\8WN,!䧏b$!M/H\QH &l*&u ? )ɚQڙ꡹K&ND/ (C!Ρ"R!$aJ)`_1cLoUTvߑ#qkzYpM8UŗFɚpElEq[6 r ׍2.o6BhIlF}J\Em~}y#NY)US ھ1jFc1Ƙ>fi+k:3RZCȗdݓCB4qi3:G7]"RMr(o8c}/E:W%{$]>dv)ͽ@@T;4:'ۓANeڸ"C$~٘Lƚ}~,b~eFR5*a ONJHLA83U{k^qB:ގeHO~ V{o B+ \#PaED ى)f> qT4md[_\X (YӛV}>d|c13\$ }qkx:6z; Wk9]=nl%n~W]lEs,c^apG).BQ5BSڙ@hճھhc@vj u.2}lc1Yj k{s0: ?ԝ&4!.c"\{vmt qBqe4[N"ϼ^c2Mu%4g;@7r4HM[ol-pQ4-v?|TM u^X-4S4*g|?K:dȼ8R`!l<ǽmWKW4ɺA-_3@$jצxBe?3efB1ETyi~ϙЅT\Wڔ5~- KO0~:kHB夏* hH\apQT /a"3rd4m7Ic1ƘdP>+4? 7u.vMTrةD\ \y=rWY 7u=) 5~8"jFխߺ&+/ẑ p~.*q!6}舧 ]P&wDeݳ!; 0 2#i׌tl1c[I`^ "w"#Љe["+V_h ' D|KA0)p. l6gK65|IƬy]\~>h]{2:P=c~->]qUG“˄D=hauM~ZcQIBV& 7'XcϏ>?g,^nҒ-\ Ws=i+n>*/DWX-umxQ==tTNʯf_HT -K;>AFݠaXGR5>?x 9{)ݹdB "'[BlBR.hͶWAjDd k.Mc1}L{!"hqC'}7<;>zkQ{$8/#=<@rF\' PCC#[puG皙U8B~q-KRԹ V=2KLqqh9I;\)@'\Th_~ɤ{ q1m3z݋ZYYg' c!I{2MC<2hf'J鍨Aq\_wO}~,b~}3}LCJa8( /z(⻿L[dD+my_Hiqc&.DԞd×6!q5!ǷJVC,OYça $CJ#N mɵLAJ{լriJL<| *pT_Ɂ;&Xs ,TrΌL8W| c:D%8 /s{ zv&ً%hOI MFӴF^s&UM1c9q1X\q]v/4,>KC>"M [Do,nA|g2 ?54`ȌՐեkӤi: ^PMo[|\ů;tϝ˺-]:d\m'ٷx'iBHw &+/~m֭2m|-g };ښ][B~$qiejUF:CSeTjɜ5+E" [}Qc[^h}~,b~eF.$HTjF.;~u'3>FTZQ]653Q 멜1cnMɜJQ$.-nš0t;L8/El)zN[RiٝWD*uvga_D])Qe23@V@!\.?T>mx2N*G9\mKm#!vD<W|MyDC?CC@ّ8̮ލ =?FEQB@5*YGhk 5"b](Egd61cYa6! ! ~ ;\v7w ֶ, Gm\4&&*jR=YV%YNf}hǧf]'~+#rQOvD'`\# "z([d$j!!n@܌1U[SUdfCϳ,CvIbnYȷ/L,BtS;KeT>!6crP*.*eR_yW4Ls|Tl/ڒb {BqŠi,[4ޣc"Y־l(fu 5m,c!4;5#u c1c}c]-QE C+$+o_9d֢D)dͅd`W#+YBmWQ6P8r[a3pƒl)YsIKK>{ًx ?93roX \{i+g33IPu^l2f=,W]˴eAP۵I9I)/2mkJqzMaI0Y3Tq ~3W_&4hk OS `]=t ;O"/S!m!?3Rl`7l@|QqQG~ߎACyń\_{k@/V6Z!7'XcϏ>? ž4YvE(Y?oS& 9~һP`L*OLP!*ʄE\yԾP)dVNdͲB$[+I-Dr_`ETy~mO贕+Wyb5dӉم>TCI,<1A/qWnh \ݤ92KBy IDATz] rݤ+{ a/i,n\JLBwBN}˄>\evDlyFf)ײ/T۸.,ӞX%!ᬡB'0Q^e_l~m7i_X')"C9)Znv4F2  @?JhSN/Qm>|pflD*mS^_ؓ .WGh/^pSCZ ҤK|I+{i<]MFfagdߧ6}s+c1ƘiG#$$_g߻l:K)>E(:1\&꣹x~֙U;J4Q;J6vg˾&[R!dKZvWNPA[x_ʪ_̞)C 09ŔU.*guzHϵfD|ktTV_\/ZP\~sQWȲ|@hEmH%ͭ?$-uk/yN|BT^B2!mMdi260l[fZETYvHMzO]&4ˮYLNֹ!iM/אHB~̂%0a''B Y!EJxe׶_DLkS}Qm@\BH&>|ޏ}iyΉV!87oᵯ=(7\ ;m/@lyk۶ys_!l{õ^UW}'C n&?]< OFUIӔ~v',~?XcO~͝::+o=A0@: ,ۛ7BC'pt>f<0#mJ]L>aH/[+o$m]hl#[7 jC/4\\,T_&4 먜M(x$2#\n I. 0v*H]5*t4|)|V/]mr"Gm;BJ\ay32_nq%75+Km!_2f}o aN4p!RɺQ PANH^NWpBmV&\ ait1+M'KW(CϝHq/i9;#rRM=Y@d3N} _!?}{O}8㌿lsep]wv.=ogt;w~p?@c135)i(U0jd)fF٤EMԌ\_Kw9ڒ${jPF@W@nJ3ߋkOR"![fM{f.Ǎ NF%DtZ C" G<ϮYc<0}3U#Ç@Lli~:M e4R1/g+ +Mxy%[Iޯ/S}G_bʿY".a!I!i'BuyBݔui#ܜ#[|9ɡ*~˓HTS' qhQ$ߺ}xWeww=hR)nҮ99&-mԿ0LHƧ݇De4%>dcCөn}׽l2Q|N /B(KHq $!дmzܢDHa=|pu if䔸UmvK@V熶'gk\E|mwVɷbbxqw/|4Mo [me֯_/x{8%ϋ_2;nɏسAz9/{Og8}7J_-|kEP57'XcϏ>?5ڂŮT:FڪCh!A9 >;= kN|ټԶ_ޙ W:~<Èz6!RS1'hoS_Ea8[秿\idͪ%vzHԤ:dDz."Mt9׬tS^tIj-|cAH_a⣞b̽' wch'q҉/=+!4LOdY8W7)Qt; " _^sCQ$BqES&j6*&Y.E1s!/tz]B9S74igo?H?!*#GHTpNɏ-w\RDz mr7I''c]y'C&ᣩ>ó҆9F - ѻ\i4ms~ŗ :<ɴiwzdd#7lyg*]?Iu1cl|39a]| HdP(羀v1cYkd,<5[ &'\+{/QQEAMɎOH-hPܺs~BhSQ-YpQ[49YCY/QpAlZ5av̩ YH[oF8n1 y|js'd2i i?GQ)ˌ9鰲,t۲ [ )n>ίq(V>'!Γ>SÞw *S òZ6!gn5LN8˜\UϜ>DS_#$+W3ԗɡ?]pL2]4'?Uܦu Jמ= W\Bf|鉥:/=Ӯƾdy0$)Ͽwbl'Yښ: anĤ\T#m&w!m aM|s-1a]o(=E+.qLgs7}]K}{UDH}} PGN]hܳs"cxy|^9x8cҗZC!?k 7u裏vgu||KSc1ƘYs˴BtTPB6|q8Cښ3s2$͟Y=IHTM:=P͊M kY? I=[;Q/LXMtw(ChZũ+n]ߚ偬Q}\Eens|͟lYR&{#״DM+W7җC܉{ie 'V}/TN"x hl7lbifYaϙVjB}3.a.jS?'5FHZ8h .=HyD(K'`}m iS> hlȶ[l rDWk/-3$g6 ;# 2?Q /W"8Ml22:aY-AHÞ_*CkBTF"⋤SK:Y6` 7r"ėнw#COߦ<-0-.ajVe~(Y&ߧ!meM[Oh:{[rgoUo;A3#( $kiDw Y?l﮽PۘldGHdP 7s:UQ}Oܙ;#y<3OwuUuWo\M'"/#?N-?lcr7~!^$ΐtzk~Ov y^oo_8p}kzGӬ.QJ<-0T>0_?k῟l<ゑBTVgE[`U*81@l! Q%qyUVZx3Now΁rlBG @ct8[tp7XB~,&9BX2P$-Χ2.kSAɲl;]eO)#@g\vBS?NPK͗ojCE]fz~0]و&)qIX4'iծGPٴd@uc3عQGc8n lXҽsƏR'NH$_/l1Ӫ-]U~Rwz1۞ TeQzac;*Lqa lრ}XPc] 848;ڀdJ0N7h1.amO=骁|~&g}NWk9+(d7^y]g?m^Tٸ' ˴=Gx0è_?~Z9xƅ]^`h\yˠs{VL.Jz[C Йw|\mw-jJ y!C`ETXGIH兽1pKջa*uX-([㊎7i ɥ[#gHe[-CD~{Y:'PJGd =h;︬YpiҏJ5uƞ6 zU]h b[4]<4l<2z =(WgU-s.iAuۥfX Eg@cx-nh6Js+;pL~_!aL1w?? c֣4r;)E]Bjt">k^t|;ٷU|Ψ+nuc@+Ds-<ӱXE3~@d*5ݓ:,|g}.o{o&"u~u͏G7??YQ1n~+e ?B~~Y3gã|d5uFaFaex1#MTP0}ܜ81 sX$}P;v1tcf+V/CWQ͠TҘ%j Π1+) J(+Q& ]77Wq| 4C&-FoTNpnsCw<>D2_;MDێu;:׹BOY:ʶlc?_ ^V zRY0t$8KOb=X4hťD w=Q`y+ Ԃs%3VɇduIXwк[ g<?ǩ8NFe WuvQ&g~_9ˡC?{?߼k#ܨ _?~0èOzZ= .>P䮪[z;3:1ǔB^Wcب)bBq"8 3QSTٔ6D2[(:wQtx!ҷsVpΔ1헃D`O}49܎:j%otu QV,gF G?Ge%pyM$[ջ&alY*U!.[(Y;kDdUǔßrc]f5{iaSILc*8dLzDT/ ?`F=UPB_WecHLvy|p/!NE8ɋnpdm൩^&TRڜ 2 `*bRWm?}l./`6'ʋi5 ?19+M<ԮPQ\R$`(פ!KH+k,ztV6T}{ÝAPh -~IC*pe U4Wu6d Y!C2h*zK i}EUq탫7[B!AKꆡI 4?}5帼@3Y>UX}BZ"FB?R\/w0d E $ss/^ӾM=>|3釺_5wFaFa/'*O+@)ϵ4 D0h3āpJiw(D$ J+X87Afwb- 8RZs" `gp:inBqֳgFgϚw͎&óIe5q#:FIIc;}'懻nۮnwE5q:Xb=uScSHt$mLcWv+?Â_&!Ao}iO%T4Ku _q[MSD|j>RYgy32~ zu<+Rs:~HXCO/9Q+Wc*M@ WT&גCm#XTz!XY e78w7mZ"&qy9a~ U[iE\rTcӊ`܎t@:UhAArd>CcD~w9MlFv;PstYNסwżgYz)@c|+Uv̏>lm/LkeHt}&9~4#GaFaFxf4yT8AenWH*1E'nu*bg~8Q8(6HH1Gٴ`]{9lZD TOX *c i7]>6 adA|XpP2Ud"A $Oix,u`EҘcy} ^ET^ðeQ}uu]F?l>vbd\6G:.u_neN7mԗ#J!h^B>;A{ (^آ- C #tq*ִ0 ZTD4X<.'m1W2,GSzEmcPCt%PeZէ,:Li_y3A0ZƕqY)Z}7$R>2ݽJ'U{)(^},ZC5U' $y` XSxBJ~mEb_Wp s:M zq(2*#8 [pYL.& >eR7Q!䙟o(Y.'Pajn~(H #&ED={}S$'^+sTh5pj撕ߣ>P4]Y1$Ȑ~uc *_/ ۞[//1Wg/LWYxǃ9DFTUo$5_n8 4gi2blmpIJvEuJ503jEGaFaFxiF-/B6c~oBI019XMrhi;]~; OYU'&riLX84؜`<\$|VE⃙oĥ!Y~!X$v39/ؼLalcOA̻Z]|6] 1nc橨N)(:H mHn|)}\Bte%-ڜק;A*vQmRѧoZo~WR>ЧۧNq-]gS[yls8nEco Tu-:=j: i=${nx'vI{uDM1^]KqE8;vC~"EUw-jlDZz@kIQW6Ա6(W* 3/m`#c{p; Ѽ!NϦjt >ߞHxw{'Piz ZIT4sg3;Ng=o1,_8 Ks[ELDa cz#=#U wP|#${o(z-tn[̀FDAVcL2 :GP_&{UAUpnteI3vmy>X}0\T Z~.u.LzC2$߽r+f'ss 1#lM Q`?S1x/6=}캩 'Npxʰ}ED$9-XΚLFxX]8;¨_?~0ӤOj}~?FC/> ~uonǎpG?׽N9Ϲo#]D8\/q9.=H2+8gHވk Hj=b x5O *fdDIiY*Ӹ"G*$_ 4/'J$.&p^ Bo0c5K :`4gmV6 ;B5߾FoՍ2MQl*Ė'qS"g1kwK mlL"A5 KiɍSq&AzwFn*x[MoES>g S%xc"ٝ'h"XDUTkzϿ`[ѓzvb ko"Uf~|Ԧ$ DG1$:3{6/9֓//A aA |P5LⲖ.T7 >p&~~7]&HXGVLeUE6O'GQd ?y{Úš/S兄{E 8W ׯ2@1 SE k(cwGtLn\:Q_JL:O!J#\6>祥y8LA24OD@;j2l a,آE޿2ߦ뇖<\lc9B\o$R9x k]f#0#0#<qR/}+xfǎW\j^X;{7`XL{?QP&"ɞkt 3WLf}zd]fs@R1.#>*hcAUt-"JrPLcKpW[&Ptxh)}(| uUAlO{?y*^Of8Ӹ=6C-7p9TZ4ٜ'i|5$_׾\"* l':GYg+!cW]̫괿E3w-JvPP+Ω]p5*lmv> @P݂dUڵ;quD޵X Τ]{BµS;9^jgRv5 c\,޾TWf[C0VlxbcF=YV+KUi=JpʋyMU(}J%{=M;s؉O V` 'Js b./NGxrLQF}~aQ߯&}R=(?7կ}e`U^nzС|˻ukKXX\YO5 \x +DQLa l+x7㪛^T4q/`)$9Vq֔&+k`JskS.fdF2L,T(c$Ayө_NԸ!c6:5}Y[§U;#@W&K`:Tqa +}璃yaUqj{mDp^M":fs=Gh`<.9Ğ}tL[)Aֆ42YD[_Q=ud;Sgx5©\< -pɑDŽ"m8-ђva{֌ BijB ̠>c.ZᶗzW٣V3,l s7;i{6^<*vMɳV Cz[b&Q߄2 a7KmQ ӭn@ aQAk-+o!:dЊcl©=t"12QĐwCHPŕZJ4aoǮe*ּuc-848{ђ(ExfqJgP[.Ņ5>H)LEY2LlP? Kxz0PMP RCJ3 5*.y+sR!C#A s#)}5)&rBE5j ktHחx߆ۂj3>`,+{@(ٯ]VwXdwOlrU`[("q9]8s4^n>~pgyݗ4A+ͮ6x#0#0#ͫ\0/x%։_(qa48J}gEyvvW.]vѱ5@T_+( P].70y{^,)ك&_8Hf4Ɗ *š׬4H}ve#HpCr|:[{ -):_V]ҥiM3Rۈ>V.>Bzndx9A كSu;7zF9Jy!?*Ė9~KԄ9Ztm"^7U&Ϳ@q^ʡ{$GF BWނcj iBW@5cf5p .l?앇|՟z@{f6kp}Rm#? Kg 6_X+A6 ʷx7mn0%-:*2v*WN%y' O'<{q,&C}cF N[)%,Q% wm +RZ3g$90#0#0“FkM,,v8V3??Ȋk[j>~;_;= 3(ATL/'nd[Ǥ Xp6Cᩳ.pH೬tulF"C .tէy)O`9WPje*#ͭ8gp eHPA .#v"%1 vԛtf~FamvC %7'{/T䝡mΖt>e9pyDc>Yn$ [܂S/s߂mC vX$ΠbL!<|bqY]Qץndδp&mv$jbC_lҟ^Y\s U4%UیOH뙱*^tB6RQÄg:R;T*'}afUsk6Gf73PMյ z8H;?*1BWpY8K7XWO$VJ"ts\?Īz. jbZ;g$I~w<:!c,-{*^LEpWљ`c;p"BL#*$s7KpE,` bqq׭37k,KՍm2'7L/EF9;qx; .ᩯ(Y`}90|S8ϊUoc;qE۝-ˌP^RAlS-*#rT4FtkГ"hȿ"ۙ \FuhNC L o90O{bm2x2lQ>wtUY@4! ;^*6rاLw0è_?~t,i!MSOCj6Hc vAY9΁-gϾɟg-|S峟j5j5`"6@;VǗ3R:EJy7v&ffXLa f|| x y)Ѽݨ|Q]q, 2@e GD#<[P N#vID-=>A"C$r>Sɝl=V7Q(1aƩq)cvyXŢ6fdeEא,[sm, ijđ>>ArV NlŅ!-$ VzZ۞KlL^s 33́mm ?Etׯ_kD{~ˌqPݿx6)oWKfj+I"3aӖnkA1y 9DYjc dkTǦp6A…03CCɰ͆PLn퍆=(翍N6;>)lT(DIX0&Hxf3o2[,}k5A^9 JCC_X@?smcl؀+#IUʽ7G6WDz?,֎&&7cZQV 4 ]p@bO#X[NG$/*~ #[f! jcڌO`+1hiSs$kTQ, |Yߚo AP/&枛O6ȷ_ O%w\ CVFué-pͭHz?3ϜOP!@ tmBpDW (fNgRjbN cA>?Xgѵ f~2|Q ElA%R'<#?%F'T"(4ڌ-wiShnEgz'ej5ٯ#0#0#T) F{ݜsqǿpsËk{}[D]x1Y+3>j0nJ*ۅ,mў=v} !mܸVmT U(ys"JWY .2wuk{]I"M2f-*R?O[x 3-/Lҟ¤ P߂kϓ~럨=t}X-!MD[`ny]flMY|Kē 1!Q? >$=XY[ b0r칞ԗclzy;7K%_$?Ňq#$sй9ߏ8|Yh9Tutafu+KǦk\V]?zOrbۏPM( UlGIk;Xl`o}whcZlaQe~v]T^!NE"yF{n 5(ŵ'G4$$9/BJUO"id+ۏ ~w~:\VvXkX|AهIn/\Kx&/# ]gSzQ"_GoNEGRE(Emu:.aHrKǫƭF !,.=?/T.yK/ #*٭;o˷b/M}_@e &^_KQil?vb]7e7ͱ(_E|8׍Mk~/iҏ%7oS?ec+ j_ I}:iS4JATRx]" IDAT8kE TΏ-n2[Hv_8`H@FqNelwbHx! E&=7{O;"L__{W3 .`| paVl?{l{ayoZ> nVjGl=Wz;6Al ͞oęUC aH-_j \ċ>'#nL~ϨG}L4O}ݻ?yǍ_ퟹP9U^_'+_yRi˖Sʞ{ݟvr5χ?kK ^g3(?WAr؆ݚqq9fX&s$ql~.CBPɗq;GphL:mhے=(bP&Ygv,' jp,wm't׳쁽}t/d vLMF~ck>Pk7]297Hq+Q5@ _N)g}/Nb3#b l2$ILB T䎷{g-tVP&8lwM]1x7*g7)|0dlW%%Gzj9΢Xp帢M0s.vk_  IR}[87fW11#Js^WQld2bs:wccrCHͧ?-OhQ9|}Ɵw.4N8ѩ#*$r79ރ'p};;d@D{՟t]=Y,R^luP$ҘtnrHPב<bmm.9Ƒ~l`-N.(;8EkWеͽU,>踉 9(КCCqb#Q A8E W%] Xg+&V]9]Fg:tGaFaFa=pRӴ?sm7.n]C{oo=أͽ4v/Jc!]p.JS9?@r510"8y1] 'i4_@&._9(n{Cwؿ_ƣ6H8TTEK~-sH85dUHm|ƙ5_S8ؿc蠛[{Em +!ʮ^&NǦ]oJ8snfG;)`NA'VBԤIgqEЕMN0 EWCux*`Rܷ ~o3zl'fr.<nj *Mс'}o})@+z8>a?ҩin.(PqMۯ;)[)@ǼFs۰"w1zÅ` K`9bU>~X C0;szp-<"MV[&ÔS3]*a!wZ _F(YBE.bT J G{vK΋5?Wh-(M~_lKCTe_<;+ 8GS,B)A-1/X[_>R^->vJctaINtqIGs'yn} %z/dὋK[O(/hp8p8M(1GkDd6w`H)Gi9GȥNg5"Zfmw>6B wRc$w4趫Yq](Гr Lէ"o Jy[-3|^D$>_/ >07oɋ5x|8#l?Tl^L5$!fA Q@p۱.Jg~?4/3x j#Q)iw&Zݤ=/iZAM[!EPAsn73ʃ\z>m{MgRAK7mĢzWNނɾ*|/vaM;^F4+96ʒ|ދ: OʨbcTLDp*l<Ҹ%0HzNzΕmP.?Jq!|1ﶼ,4ΰVbQ!mݘl-k݀E9 t?}-wۄ$IQoV (LfVccf#WjicJKDrmV]QY6&m|u˶NwmGr (?ߦ-fh:8C| =PJ/)^@᷷b#Pk JT)H7@l6i<lc`MUA ],Y%E`c#wWB ] &ž꘳(c_A靿5(bó=on-†SD_Xe_cs].hloyp8cwۍ6i:^(w*b#/[J]}[/[ _iϫ53 }`-xoDu<,+$}tqh . VW sJ1CnۯiZP&%h.\WM0t)CO{_^DiD,v1]סLF<:T1' VM;̦CWH]MZy j1_|ˎpUp*}aww8p8D>&m`/܅\SuU/$s(h8 =w;rH{_a*\wEBԗ7ԏMCc>&sxQ2cNBqϦx|a_|˹0fNZ6Lce뎓p8p8X|#|㜑;-,h4>'HK~6maGcO;P($m"/.rqq7CCrx l:VɾW_ (I{W N߱QՃN8!JKFݐ!LMF1j Ḛ4gީg1&kCgd1fP oI6MfIߢ,4+bu8f{me]c7I8OOG\GIvlc+QuՙlYXOG )ph$FaҨC,Z %D| 9FBiwj._[֯H lᬇw_iرy=6vn vfU 2ٞ]_,8ed˨߾'gQ(J&&ʼn&HSۼB{\W/=CL(5BZ%>J'#KqY'.+=V7 .-sGV(VX+af8u=9c'gy0:Ý}p|xs0ǼӹHW? XIQҤ;r<89Ƅ: h(Vf }MkЁt+LS<MB<8$qі8^>i;A䩙}AxDOG x(:@+Hxpn P(<&)3f,(in[K'-k(Эߒpa?_Lbz:"8#=AV;l+./0(8^tc#WG8*gx8 jLJg8i%meǢSL%ݶi"Nl;m#[ãmq^E_PtFpZIu=G"GX2ǖwMZqL*9)USaR1u>l]vʸX/v|(\sٌ9~wNx;>i(# r-cͻY(5ȐN&Vaj:(aJ1Tۯ^1@#W1WLmk*יR>ԒX<歶;հc V߫ |l/Ԏ,7[*0WQy\&V%#!.S|1n_kV1}6B ᣣVYc-sҹ֍G+q_NEsl`XS r{T{I=ϯJcd(l{4;1`bd3G8p8̜ ).Y[ QZFe~Вur1x|G7zepaLǙ;0l=оl`B ŲvZrG6Sj( !1I00RV%1X< -V{@`ꑌCD fBLSZ16ݹxWplIf(;6PA+wkp8;ÏS?St↏|J'fʧqݻo{?|ϡ:w'<~/x-+mkؠUMSæi`O)3gR‡JkF~%F|W!0(@۷iJuhMӞIUs=Ć0).O1iTmC'JvHOD5 Ҷ,mbw6ZiۉMS"[$E,%! +q0FCf ]諔F4eǢ9ƍZp*:Sl9j*7L[$@74v)DѨe]6F^> 5G+ot%9ܼ[2i>zA&ci-kB`"m^N=Z6B; L E\t1mO{I$2scd~.Kdkʄ]h>J@yٺd o\6M"s5ګHX=ly(#ƲWQ_x9v_]?3?x1gϞ;\}H<瞻K_mo{]vx9\|7W^T^?y%^ Q#ÊsIL](PR! %jVĔew,:o})QQP Q$EqD 쐱4#s bH7sfu?4ie!,d`%3fg|JȲ^M)(CaV]- -b q9t2t` Y@gM$2$P*zLOe 4/gU j $ʷI1DPP (TY,:@ sfK>R$Qm{V ZY2q-үI,81>TDmH` TdXl+|'iIIL]ȷbI2I$dC m! 49m5ǖ+$]'\Yp(瘱VB#.ӈ,E%.Em)]6:u k -!KϾGM'r\rnx !xX8;?BMi#؀$^%1qܽheI;G+D5Q) `kYXqjt]sX,εr5X{EL]l(,s1`yTzF 1*h(0l MXDU4B[/܋gfKfv&1T4.Kli9X 1?Qh~D$KIiϷV$Ki%Q[xqhmHl2=/[)S| ,m3uRXT#^svTEMIJhtz[kFKz=Z;Q["[hi9uELeDP#I8VheQs3m`<<)Rl_ñ |ԊcokUu8OK_LMMu}c?S΃O[ć?wK<>yի^ƭ~ju~˼/T,y<(Y"aҬVK:eZ:HS*;ZSXxxlu1G "8qO%l޶` At?]X VP33i,I--[Jijuҋ- Ϡ3ˆ gG)iSYvdrc7OuiǼeZ,Z܁xaFNUluԺ>Ja$qb$k[}]V$#:;AL!XBE׉s):+ fviPO`X*<3UiGruX,Ŷqbt껇ǫ8_LU#!1{M\ˬ]^ u*c4Hg%Aٶ^Zr|Y-n_, _͖:X.m{|n}#N[O˭6iVyٺtEg5(ɠ/}ZSt)ݖo۠MTNlw*$Σk{!J-z)d| criSr$.H(##-.6_Ġ~l&P^#noa/鿿,9&_=?Ͽ|S#|3̛|N(➯W~PY|69D6!,NΕ֋NZIˀnnmeEG56h`Eg( jy{=z/,#HldLFZZJMb0Ja,GAlJDy2􌺊"K4a~>c`#ld<V""$Rd9heI\ZYI>JvJag䔇xjtmN;\ PXG,Dxئ{hnDB^-tL^{Mwjv{\\ڕ}-M\Һ$Y?E3CWtlAr~};QdJ־)0砟E@7XY@\ҙTF7[$| vo E E!_7[HKA 0uvUS^I+ /P+,DXI2Š Z5(ֆbP"&UxP,AEЊbad+@)o/5^H( i)rrThWw"Bhzc =Xe@fh%ƍ;{+˽rOT:5-C080ģX~c\rE,Kyԧ>FA+U0c5PbP0IEƔRKF()>@/GZSbx4ƪv)m,zL BIbQRfg4WEE))QTB!>HuP>alb)q4*fDz'~!M,E*=&TchűX+DɮՖ%+e m,.&>-Gɿ9]tb)P"I<$ˈ;r8qd&NL,).9g=,gb Y8Kv[IldY[$I 6E4(D fn[i\G[ Y<#bJX' 26ˈ [Bɲmd+%b6mñqEFYV-D*nbñ|ǯpaFFF_ ne_?{##|?}fhoя~,O}smi]/(%4s5xE ,Qd۴Ky?ZpܴCzi蔙@V耩HLjz8P)jM/)U mN=/OPXDwo1IOzhRXV ΧVD%j9wad)PZoIC뜵*/Of+roEyY$&YhZ݉f-I0ZsZz'v\(Hñ5+q;&ñIs!wQ,Op_7$ṿ+;v).C[\#b+{;[N^$w>L%)SR X eZt)F@J0OJ Y">4zmE3`90UT9uczCQ"J fd|Ãt5(-D, E/c[v&vmxI)w>LnU@t&߽l6 ef*bUYɞ3i^j_4}+,t% Pk!뫌ٙAe`Ti*}n?Neik0*M({{YNeoo~]ZwuM7O޼ϷtcB"RK(j!L)jE/6c `Z̪ň^Ań*eZюuX(b0FiaրVBP&vܖX+{m*_z!\I)䋉Ui9ٷOۍRt#/2RkiSbIuqήz_EWp865y}=d9Iv&ip86ư7bSޮ ?-WYyW2]ϻ v\J׋ؼ]㳺EscSl/#7=*(vzQrs_}ҘEʣƷ@ܪȽ3.~y/w޶{NeWag-ʆp/NJoXyn|1WK܃}<Hbh4#XzRf̦:~h^ޱNѰvۢ~oRQU\dִu!r yv*5j;0bSdݏMi-p8Ȁbfִ3]O#kZ{cl9VFDlo0۫FouZn(Σ p86EXdfp8ñpz"n^GBiw; 8|m%y-F)Ys5틅Mۂڦ|-&V_E). wpoYFg ( U5jpNEFn06 [ ٶhP_Ѭłt8GG. v,p8!Jp`{}+M1hjpsl O/SpHñadX!"|q8pr7r/$W4N R|[cmFpۍ鸟핰AJ;q/Ne91q8u[8r8VEF: #6`]dp8c5D@}UᢤU]qt89#[+^_vR@G۱vw3q/NcO"#::9#džAf)-p8cE!J]X[c8g88.p85`imHbCMEq8{}X%v+WvR&FFHε@> IDATq_qw8{6ұo7"#Ʊ !lq8pvD4.נp[3pl+)C(,ix8p8v= r89#[]_vP&aʣps;nq/Np F=p8'p86N:Fѱp8X+zp8-` gM\{p ~E/ĉGqqz'<ቼx5ǿwo|E[_p/ M(EñQgAyֳ;y+^7?T^K}6d7^:~?Ž߸g͎Hñ4>~XcO(VzoqpS(ߎ:cEvtÇ9=IyȇHo0sss< 2 BdLLW>|;w ñ') ̠ .p8&q8ʎFF9raοȑ|t [%ޯe7Z^-o$I)WftG.y<_؍mTCc~t} ,7&cl؟3{߼zg1W1ZFN2~۞p8 3cV?p: e"z_:A:p8';Hw/ݙ3_[h?mq|:yO簑{$^DǮEodᦛ线x;"vOY|6GLȹ|Q~x_W||?"y75{wp8@6o/Q}"Y#%Mp\Iё;c:.h.c5J0%1cx"D.Yy.r~,ỿ{y9p8pu%)lX=M۱/Pb:TD AN'sg( *[ ( #G+;ڮñh{6bS8RqS_+?COx޲'0_|jno<_Kbfz%\ƍ77n Ǐyex-\g$n[_p8KwC TG#mb{TZR.;;v]+6%\B#&N5}ұ W=vǡG q{S'yq@ev⣋ }_~hZC-l?p3~/rgaT];9u>H177ˆF~، I+7T^ﺴB'`ZH9zpu!mQ/(-,#'R]Z%:1W#l'n^\+<< O0)?sYGRcKˑčYJJ5J4P,u|Qg/}??PF<|={+/u߳gh.]T."M~r9pi64k?-?፰8:W#_?h0È> ݜt .HgsEbPrOJrMi1Xpq%F,7@dIt0u}\O9<9 S޵$״FN'Z7 ?yqT8V=&j,%+ZiFBiv\ ۻ2p y Je|38@>_>B\C_(vVJɝwŞ=8!#0#0W7v@'p% \ !K ㆠ`Ӫ7gޕ0` 4}^SGzh%)"0?Wqqa6I u?o]vox?( V.%})mɞ_KRLa#,O@o>HLNk!h 5ƀjjlObkVV{W_G)ŏ#ſ߇?|$?177ϥK׾׿ ";W_{j gFaFaui\uAi-ЩjT$,ֶ g!x/@lTHيF?΀\YRٕc1<0ǫ:Q˷xVW@G4E}7a9^3h57 qks%_e>WTˆÕGyNj$a;+؞!gQڢ0|2&g|皨epqcHQ gϞuOɷ6L~}G}aFaF=hp.o.FaoqMq-`{~)sӶ~x'ኆa zkk !ïJKSkW8:n7ȶ4R͢952*Մw ^q47:;<ۺMJ֔QqK*j !R.Q! i%*$$at¸n+(*R+!"+=#0#07Ii4]͟Mn~[)Ri ' Zu=1¤_q=y{e7A823.xEf<|]ڏ¤_+C|sh>ۇ^omGn>%W䕴 is讵tO`ý74 Oaʡ0i4E`q{f܌H#ﳠ( Wcr6I|j6 lVHqF+އfu!"@58I*ʼnulFFaFaFqp)#MbpwMrvHG$:IX f;mW-EӐNGHW`@)7搫LVܶ9MoK&xc6 ڱQKe=╲9u<;' AemҴVe%&1*A7F(fl! Ƽck͆ŒN6ƥχpE&8miN"i("#I<lL䟝IbfaMxXe o*0#0#0#\#t3f(ݒP~[FlJoXT1ʼ#[R sݲ0%0x$ZM7[v I gn*njψKZUiGOΏW+&5wF%.`WInB\+eJO"@_S$Pymסt2^g^ā#p}*rsؾ=e>edUyKč7ާgX~JTO)L{HWoaT@L`m1-EZM%A0 +?r0#0#^»1n_:XF3\aT\Sb#}2ݼq]#4"o<" [9IzޠEy@k\_2P6uN<4߽@ash.n8˕X~6F+ />q'.}*`{M?}'},{ FZP>|V8˧ 9fum% C6N"Y}|b>O Ks%F:KJtjrcJW)&gʜKi&˂ņ~ab;EmpMĊog?v-<_~z˜auꟹH`\_zfgb9I="5Z-Z-c' .^7O"l[P+x#N"ȸ5c-8 A&F-Y9W#%eN4R #vBZ[k-54:u7~t[[e_?1_y@߹7.xfv<̒K=tK ~9~t]!v|'n'.EX[v9󯟘 GdO]k؝1*2eo̧/y|;|cKC?3K|(=^rߘW?uզ3~fw=_~/}|ܲۗąQ[pnE)Ko_m>Z?B;'.|>~F%qm_8r_Nρ_%4iFμwӟa]wq4Jk*\7΍OѮf(wi?GWuW(*-oKY厝 ]|I?&>Cy>l-_yVmq*{&|r`GyPvtzQ]߭K_$N.oN?*/o9őg–ǿ3ΑWK[A<&w59E>ٗ%7׏WVn&R+k=_]Xu?GVm|>t~yKw߷ܷzeЗUP> 4YY}2#ɂa~"gr'T߻_";v&Ԑӗn0{}s,_>>V߉ >R?O\b%~w~t8ad4Pvn w7 k/س=Gt_2̷GcEoԚOwm/]:׎f|O>lJmͧLRtS!ja3Œ $X2؎V[L^w*> 'M/=qvú\_w -^ە}}H8i )Lf,rR Rx~ l|g B`m 2ut88 *~Ji`7=,xwO_Yy50ɹz?n]5KGNg=gygMHJW?1 1¶'([%n@!f'V%P)o|otA{mjr.ouu5aY4R3^$)+żNa{&*zҷnDc[Gvsfi{! #Uޓ*|4e¯/~EitK+vxnmݫ['ϟ ^~3?w07Omg ｻSJv_Ww}ifobOK[*M+|]vj%Yg|xrW/;?gRꡤVM땕[~{lwuܓv䃋NūsR7x벳^vy}eC> W޺+6h_Xŕn9s5 ɨݓg42CN{}&-^.s/=k|Ns<ǽ/ym+iS-ۑ)&jv7%۽̫}msE^=aִxm /񡻖i?ִr`bT8|l1Le-ɾmxYEw˒4%7j%Ť-PJ24V[+@f˹QEZz`jla#DQ)[s)? +oa mK*ǷIZ)Ϋt;J8Y U*!{: 5qep7jr3czoWClBVQ5yqktYj,Q&5QZm7U Z_?_? ^[f|.+[hոҶ1T#EEpXvxI/`#`k!/m긞l}# 0~[”0l{W[gFe !NI5 'IcERlh9Y=7"ܒCFg>l T}cE$Pr1^0^3=k&3S\~GWk{O˼Xt͘WOX{fT|ߛ>ӯy<%JyLeuJs[]ϛĒ;HN{Rn{DDDlI)o9I!^[]1{f[}_n;<y/.đΙP܈|h?{m+auvpibf"fg^^q\tg~L׻t҅5eV-c&[_ n_441OhijCa#HuK2%Xc/˓E6v,3]Y{^M{';:2k#z7>&^82?<u3> 2W.l$ zn90;:z%zbҖvЛrƹkpp=9<(CMP ̿;fW/5iʏ*id)nqAc,¥2 $IvQp}J4/ih׳'}N6t!ۈ^N|\iS>JmwWkϝܹ1#xo ϑեzmZ|^N_玹{فRCgg)81Fix%}ő.v o 13CZݵ%mzm;ڴK [^Om=/o4rK,}#: u^:+˫5waF&.l]tEuuNxw#;Kí锆oh]V S>~/iprvԽYiJ÷~v{_6/;uEA)P|뇹 >}7[OݠOdo?!LW"vBtY^*\;+ME&wWЉΔO 7gF#i[; +TSidѻخ&S: бq)+•`r]y$Z*.q=@9iKHtr:xy Rtd7on§0 ȮߤE{I`YLi[u4q*!_f~,εlk=RD.윕A58~OY0)NخEPGҟ_,K]xc*iKc֭wFݳȾR\ȑDwKב+=u[d٫B;ܻz-qLڼNh#mATL˽|9Cy·e¥2z3sG涕.Y~6ggc;kA4kF3,- YЛaÕ *ѨP=-ç ?»OY1yENHkxJ36\D?q;c]K& 6JOu|͡[V|0Miޙ:e;SHofڐ4bfu#Lcõϼ>vra;ެҮ z'5%{w556EW"SzZߏ>p/ܜ}in_Fih_Suĵ)u׹ISvZԦUI[1*J@%s*߶.hM#_μV/4ꞹ+ "$1q+BzS0}׵ ^F)nF ۂ¸I+ &(9% LdwǵRMPu_,+SF*^zUq/`z[ „G} eՀ_rIb4g=8xtwm^^b{vv|~[w7Ow#nټk+mMpO{n`4AOO/sUxk=M\O0Zl}㑶4nYTbxk'Щ"(ڌ:v6nL_f367’<8h:R1>2q[c2OqKqǬϳP’i;V(M͟yݞrw@' 6d:dym-LHW`Yۛ$ u*s^Z#Ge}ǘgCxOr I=ݥ,O 66gt0O/|T秿zg*$:(FřlCa*Z./б"i]?4*=8CJo8Jd׌jlm8ؖ!~9P{i7}^yo=_$i)l IZYTl=l S>I=b8,ݏ~ǁ'/S]Pޙ'cos* ֩Y6EJb1z=B{yfHE~G)m8={>w˶Ng.!5ix喑B ?8ۭ+:v x`vR*~r-ZhyWxWiU[XVvX>]gtʮiu;:#Qzk@nY\,ώ,Cc|5%AymESwnh{ʻ*>z kڡB)IkzʢU[$(ckZ":t%I=t0m.Bchk6,[)2!xH"QZY4q3!'u,Ic)D:W7c,\FD|[Љr$g9KJl" kaIIjngPQL(Ld41># ;&ɺ]غ+W[7ۀ-՘}܂YD3xY N Jv[$lAq"ʳ9fLHB_rq}~ehjӄڸI3|qCsC-!{ f 'gFBdr=.,G+W>َz_] h]<8 uGqAnh{/ܜEflGo0zg/p[{|w-y؞ґ?8(rc.v:=|E&{-)2kVc$ ?7,j̾qʩT#/f'?iwpxP00V<?aVRQ?ȪN<{K6͢W+gB9g5GKOJ=W:%:U8~aX:VYL? ͕0BcPmJFV+#08KfGxNFp3OٶRL#MR4Vb\=6ˡkMlfJNvGf~Iȵ@nĪ:Co̢SR6`MbFJB,5+S=" I-E,jvn#X2"6F !!85Yid۵p ˵H#Maʧ0呴4Q-ssH[O/-7"o؞_Vs*5}VI<lO‚ҎhL.Z`J3~ubniv-IF{>=7\ W#lХ@Bx n&ps>V[Lܶ|bwt7mҮaN }tsyG!+wiUcam+9Ԡ±dzH[p賷eWviK8B H!Ks)&h(<:CbSq=xUVv(W4.Ff^H<I5o1+:4/Eh1yG{gr~FL}CEi3Ep9&(9L^$ZmqY~dMIš[H\+oWD9!ZNCHS!REas S{[br~_?[^jk5 K$"(g=c^PX<^c:zxyˑ[x\|mic m7Sn b`hw#BܤOR˔NN2sELUz_v^l .}$JFC,^d~Ul.2]?U?^eIx˕_r RBT[;Tj.GH'; *4Qbp/6hEc)ʮZg?&j%A`kq@XyA%K }uV:ӥ_2*>4(V,w#0#p)#{!%Ck5FXV-B% 'k&3rHT<8_ͿH" u*A7r8NTK' Մb8Ji,X3sh;$m´9h ;<Ku뙢xob-CM0搄 ])tO^]2#$ q+EE&R2m;̱̽??׿ 5>+=Q{fe&Aax\Q2o=H16cYd*Z٦~Ҹ3r$ fҹUv0~f\qbHvídSKxb'u1qg9Xv,šT~# SV;Kw?2JjJ-pM*>^ޡmܲE\n4q򒋯WBjB~F| \V4hxEf]Is1DMȘ94W(Ng֒:"V$*Lݕ͟\Kl Hi8(2E|'_94ν`9Ő/|x6ޢKaҧ;^jDPv}tiAk9WZI)LYxwaDT!q#˻G0a)^5A'°h,E]s5}uVOU@Xۻ$@-[A, )(Oq "8!e"ZhR4mH NVSxq]朓33pm;IZ`7q|VX?&Vsf@&UƉ&20)H-q~%@0 Zdh18p澨<  KU|1=o닾踇T$ku(Vevk31UI^mT*jo꺲xc,~V T;D%.c xDƻ\nt::YQp )JM%%YG1.7$m~-P պrSLU)=tɱ]ނ֌YgZ(Uh!0ҋȮTL`dUMpG7 >^@ ,K댗w ckM}KHZqm^% TڤWQޓPDbcq^eo_h10`|v e&t+6Nʪ#tͮp6[7 b2(F8s˞a՚ !;UU[1.WQ!]':7';O/TfDSi' S +U#//2ɫ91hL(Ssl۱[[|5&J:+Z ECʊ#4k6W:;Zl.OߖC>i}_jPbDac,5I@35fUe(5/2a1*+1WX Sc Rՙ1 Ppp1 oD~9Xau4EATI\΁%i*̔a`pF56r;QP<;* rLoT戧ls$f/2c,Qp Fa4xj)a1C|Ԙh6/D`,*O_׃bSs9E`ʲP)( t4xut)wM3f,@@7>(O;go1* QKR_ Sc-eTR`3C˃)ϩpįŒjj6=JDxQYlqm暰vMo J2+hXWm[23fsy%)\/Z3^j%jO %c[tb]/2aGj~*JffjB1Mʬ.,$ʇL--M6zǀ*6:6ulчid,3XnȈ:<#?ЛG|CK`/J;"cPv)qWGT Us-:ٺ<(ɬYgSIo6/1.3T*>j )!)! Վ$)1jxh!q{@q"j_OZ<Ό *Ve熗 쐼<Z3g@YN9+31!4f8҂^c`"G-}3?`4KE*6~aޘJtT,5VWB쯯&)eX*L9ʾfrl6@YnM: VUFEyyP]Qu|i]`n+48RWJj:fǨ5^J+ z+b}PXc'fG\jiSX̪ZUa_#k kM[b TRPw1݂h}mlXiAmVUQD "ye0qYdTBDҊ4zL#~?rl KԠ+59ktzQS=VTrlO1 yu$yТcQj`Xnf4B=h(<ُLt:2K2nGZiVi%S Gv8jp X_ی]y|ď%*> mafWAa3Gayv ,* t$&u^ #~*G?&j=&.;܈Ԛn9FZ ʇJE -~KW̹Rc wO Ye芍oYVRYfB_fDAkNy~F-(Xa"yy=neʓi9&j]LP'rj{E⣎bBZ R5Ֆ3VVbef ED ;dT =*G7'Jt[o@Ol&+>}[h^ $Қ8!/]6״bQvL̑O)#/lq=+6¦R-ĄďoʊǛrB[w!{̕):\BEUOmf8q 9<$-?|G 43hN> ;KNg5(b'?ED C9" NHoฆ oAm}}_DTP{qxKNju4H߅.g*?񌬳{+)5NQ**'gC孤v^p9rqmط8tІ}2Y۱)Xfտ`=#)*,AwQعbL,i?id+-Ns%qIVsn9|X47-9X_.+n8Rd7cc?@"M,]mBILmS~P4)k<:6g;zk*c|i?<ԕG(lX4nڰgaz CUMܔeCIZv G2lYD !uMNcH]{ޑSqdUOv<%it6BtN]x~J4%)t} r5)꺏/Vnᢛ~4-P*_Pw{ػ}$SWd%'qQ%)λeT8$aSԘmOdYDM8(mꌿvEӽr&‘p?2kq5y65 Guɭ3u1=$yI2’Sػ}$#w!dB"bԚUߝEU^w6HYM / +νvvc9*-g}̂q3z,i!%;w]!Bɰaӧw?~n|.Z U~6G~*^|F[!Bdiw֓ V^ڵ+1|4=س/ ^jX-DFF3t(~3ڎ]/]p{HK^ץ#F30;ٸqm߳Xl޼Gy B~N!LȶmۑZ@rڵ`~dexO0Ln[!Bmޥ v=//o&SB!ԌC-s ##)/= cn>Rּ}߽$%wK0* ooo˝zjT*Vw}v O?"*2mBcB!%$#z!.(,svy<4oغ닯nǮPNaa/$.{y\xr߽ꊿ^C74Պh^@@@ FD$OcO^Wi<}vf IDATV[r]ic^y_(h4M%Eb>{ػ}ꊽިdi9mN:3ggBl-&W_}Wk2R+ M]Q zx$%w/{IRza:vczzZ߳X7F3+g-Ֆy<취.M%M&PFL{˜:zC*ʍ{.B!e5sσ7EHoX]J%u:r<'T!B@iyc~}iI7^b1Űx)ٻvŰ,[]B!n~jZbsXn%6u{Fqq1?H{+B!!/Ņ+MH^߽$b'װHGb>{ػOS$#B!B!)B!B!wdB!B!QH2R!B!B($)B!B!h5Wf|77>ЪU6oQ cnn:N{mymc k-gϥl{YxKc{\u5_5իGתŵf/Yj ?]mٺs)atxs\sS9N|;gJAY|#'Ň~5?3yUߗgp')CGʎ#F2\EIF=3T zۍL{ ~Eb[y8aEE@TֻXz>Ohǟq˱65楗ϡTn;iOz/6[eӦuLFRSH'6ow`\F͜ogSXX@qq#Fk{v, +W-]{֭'Zڵ+)/߳\>_f͚bvN?vL@b>trssx{HJ ?oR~t);Kʌo(/z׵__?]:|}]֭۠P(__zcY/^|{?FcľI/4 Rȳ_4Rt)?&)76Ms1f ? ^TxdWELqAs@*tqb;|_у1z3ZOCfb2]: h4`Ye{SyXjڶ/[ںt/FAz#;kVb> KPspX+W-qy w2H=4I铸_8KyQ [\WtLzz)).&))9-44{WA]HKKcyDFFuBEشi–6`6du;#Fӣ{zӐs|JK;2\N]q %??ﾕ17 ߦ}=?=0_e;zԾQ]}Wtu*o\COLgh* >R~t);6=Rnl$b)/J2RTc0m<еk7 -(LN`[:6oYǨQ7pᇟd_1-q+6 >EtT QGChպm\6l\=>>qݔsX=2bh0fHߣfcc976}u$.}.* OO/JLjM!Gʏ#eǦGʍM»ʋ??{%z ɶmЯn;'zqb=!}.:]=E]ѽPQeÆu׵ϥQ7f vs^k޽w}.Mv< nݖ#G8r$AWi>䚕؟}3rLߑ؟=,EEbqqܸa;6>;ŵ>9kw\׹8\>C 8CC\ X^{ZRbqZbbbMx/m۶g/IHX -[ьsѼy VӥG4. V>::Wq|77E?G^)oqb_k{V*ߣ,[g+zB4ej-:Ӻ_ |Ca[9-: Ǟᡇxnzsv>}h<w/ܹ_wGUs t( zzz֛2lڼk\oϤ1 r=0m'L23g~rm÷kxī$ށ ZfbJS zK׮ݹIT*^|5""QXXSO>^c4k}FVv&9 L<xB!axABÃ|IfgDLJ׏ȑt4//ou _||}4pII\f F?v8^l͛C۶hxY '\OPPwyʤj''رxGx,g0 S5loӾb3{]h*>wV<3dds߽ܳSxi۶=>4=8Gd0iDD4;Z|a îl L<* h;J֭ң{/~ڲ9}l $CxxEE'㪫zsY˝ Q#o`5v)н[/~7*+.7oYۿ ,V3v#j?w~#p&c0xwq,]#z^C߅?w;vJR>F5&a2y;v79l=.@-_5DRm6BB4x8))OnSJK")i/{79ۻCG9gϫ(,'7/SFu/4Ը(lOikSl61-0 ׹\jj2͹lڼBwŰ#Yd!]ēɀCXj)]tgDV+P8Glԕ{tѓb^; ]gUѺ_zX( Z ;w`p"#cطo7Fj~StFaa7n+:ٴy=6=:#k\o}Hw ^'6Ng6'**^k@aa^^\s͵|>s:&cDzXa }s~wʥB!yq&H`x-+ؼiVna!|ѻORRmVOOێ2g` ?UK47}//!#k\`0cLHpF訖g$ lgʽ-b} šIK;f۹_ѽ{O ֭[Mzal6+ ~/]{Hc@Û>b.W$u۶f2pjXn%#F8jGzxx0x0֯_]6))-sV^OCBVKt!`wjJͷ^f)\>Φaa?~&\ee%Z{h2QՌsx>3J5%veeuGc:~<;r:sHr3yxx0z{Sќl~]]Wtwߠ{=MtkKSVW j[wch4|lv6mڴ㙧_Cڵ+)//xqڵmO|[TTKmڝ 9QQѬ]r k׭bOIMBɵ-j:߹CCC;y}X! <-﬚i7wdzL/cZݗ>t"wrrx͗Q*Urezj..)&55'|ArL?U3z8)eeXl-ZD2ݏHOc0L4NB!f3i%KtstzۍdȑcݻK `UtݍbСԳJFj4|}G]ո|^WѣǕ$ :tdʋvOY5>SkMi)QѮ#\7kBAEo̪ޒb"[D9?{yyvfs{gU33U)eӦuLp|*j_0sr[`N(/ײ{ 0 `eunk]vdf?C;>Ǟagwȹ\qqQ1}ˋ:}ˮ]Ad\8iEq=n*l6+z]́ d].~m+>KXN6ssxYtYddǎ<̘AA!v|>z5JLxn,q?ƌ-xo'dgg_߫lٲR;֭[xl q1xp^Em>~ѣúZ\xφ?2:Sh2o_q]4vKjIDAT9mxݹgt.bƵs$xz8r$3>_g򍌻6U'~3[prr9Ǩ\8w]SYw@?1p צIILf6nZVO'1io~X-7Od{S9pQpXd#vrݺUlm+_Ξǂ<ڠ~׬YΝ;5k.~\Mdd6kHhfeLp\sMm=GTTtoB^qgu//bFz,9@r"*9y@T:?3: kKSYx-=_w]y޹W^Nbg}Ś5u'노X)BBPMاySU~!B!B!u `Soo5]v9]FB!B!B߿*9y= 2?zuB!B!B\XfB!B!BF!B!BqqdB!B!QH2R!B!B($)B!B!hB!B!B4 IF !B!B!$#B!B!BB!B!BF!H!B!B!DdB!B!QH2R!B!B($)B!B!hB!B!B4 IF !B!B!$#B!B!BB!B!BF!H!B!B!DdB!B!QH2R!B!B(tN|PIENDB`incus-7.0.0/doc/images/grafana_select_prometheus.png000066400000000000000000001536111517523235500226070ustar00rootroot00000000000000PNG  IHDRcsBIT|dtEXtSoftwaregnome-screenshot> IDATxy\T03 ȦT#Q%6Z5&i%mo&ioަms%bTјh41{4(*nʀ&3?\39|A~++ڍ.CDDDDDD.G """"""p*""""""NTDDDDDD:©t8Sp """"""NEDDDDD)HS8p*""""""NTDDDDDD:©t8Spfkzi5%Up]Kя愛@7WXZiS""""""rykS8-uuusr]{u|Kя?$2;ľ /osiBOK3~ߏcZg<+[btUDDDDDDu ֠[{p`#}m` '*Ӂ Hn䤷DIkJ鎶?{j2ls;""""""ilG}kꄷ2kJy Iyu[F|yӫ#`L$i!]r/̨,-p ݱAqԵL4#W~zXn@ԅG<ܿdKn:W[d1'oRȷU{Nƛyn`LuF%3gURgcNA&?U̵*&y~ ~Y|[ ⇿ŏ#g]W?#{x2*z|S ~'ْ_{-G5 ȷUL׳%7K>GkN^ѾѤ!tdL@IQ """"""fZtoQ [V3oA~PGP?~~^ݰnj&p 7p#{,:""""""F~b۴KR22.7ioץb3cƬ{FʞÊ(Hb0 b6V{:ߥa00 F**qu:bHN9׊>`׮o\"""""" ],]vFuu5 %~~~R*cScĈ-nCaaҢ\kZ,"}ϤFDDG9z`mšOW%\yy!osmkoc, B2_{KmkgN ]h :ǃ{Hsk6kw: q\ssz|51qCTGIfcY&RO̩ϧ@ (J.O7Sf͏,Q^DDDDS.TKWZ`Xmm-hv *pcNhb񒅼Λڕ~A%.]y7Yd!nwӶ<74cƉ ~St>E85Xuxϥ*` h /NxDd/^Hfv?Of~/^xsS ;x `eEDDDDD:bXo`{L©mR]]-(.X̖9oǝ)!lD;73HD&r'3N|S}_՗ٻ77Yy"ӈxz!A~gd/ af0&?ݭ?`8?6㵿ALR~v[`boVHka-_/n5~˝\7qR.e4wRn"). +J 3ٻe￷$ȷBLeeeZꗋ9p"bcxuAK.]).9kW:YY6!RmY,$$ uH":h"@ W/3LX7i'3 } y_añpc8)ߓLG|g^^An gr􋽊ceҏbf?ckxw/7J=y|.nK~HKx3'DjEbHfOfwi u}<$(󿼛7^7 g:HC) ṅir ㇤L| f{]/I_Fe8#P^RҏjO?¤wvs7¤V ksܖ'$G:w/I4 LM~7Ob5~_9o555-791q0cǘ?.Ǝ@bbb4Sb!&&S;f#1n1S L8 u4Ɔ}i_ g_<8?y`0g{w:0dEn Ү^8)N#(ذ! ja5O<ճvOn֖Ln`]on`1%soen gΗjv$;/0$r[N/I?}'?};tIsQ$I Xˏk!Z n8q XuwFΘ=W'kϵ4k|#:}8z ^oL͓睱ߟ˨b׮.B;qV тf:gVܣ0,yZ$%kHH2C/ZPO4ę z&L鎻T8rH_Cj">8'Lˤ=df`)йsN\UR;c~ HcP;gr{_c{Ɯ!""""N}gC'.* ܝ{ض.b 5AM䴸g'0y鰲wK@Lj!N(7gFkrXX3k6|Z]O63)qMd>{8i{xxdv=g` Mi83$7;Ք|+ӓMPƆ-:Kq>?@+Gnrذ-w[iGF;ҵ|c foL흤ښkDҍO| KӇSۍhϐ^l0h4RYUٞ)1:6r-I&)&o=~<˜AL4;67Zxt9:p?1HowOkB'5Oiotj(''ۆveS~?i4 z'wOd]j),%1S7&7 zg} -F=4:3Gr8C)w>ˣ[dKo8ɳ#eIDiɹ<;Żۜc*A&5:JxL9fëN?Ƌ& O =LeEf@@Ux۳Զkˍ!ɓ{?_r.]ܙw3lzaqs$LJS͠+QXx)ڸ(8VEdO+\ y䋸VSߛw&3o3v J=`%KYͤfNyp+w*V9]ot[mg盼pns]uDzu)$_`+XPϾxoQ<|r `"& x(Oi2nފkozb5@Q:7کiKͪ"wj u#gob±=5oFYkYUA=&g86u @il-JxX!= 6S~0cPlX|3uL ly5YŚ;Yv'S0#'G^:ed9 ڝO'i&8(ބtTuײӹOE="@KWݺQYYyA6Ϗ]r 神=eOObf͚xر#ڧ"""""ɴ:FGruh7<;ɃJޑ\!:e rzQHRR21qo)۪- &sÌY$XGYkZUSyQ9dddT3ssضm #FlzlL\0\{n p\1Bxx$6[0`bbضm[ׯ*"""""ҙ:ڂldeg^Yٙ ~tͧKg\?퍃j{XZ;UDDDDD:VӰecZ3c{ǶW]ұs::\sdfg͚U+"""""ZsZnυ=z%s8Xx!6[0#Fx6<۶(eᴸ#.Sy Z{+QƚOWUlDDD6|XbZ⛛CTDDDDD.7EEpښIڐ9e8edfRڦK4Mbx(c:z 1.yBR IDATWwp0@<4 ᓼ.FDDDDk&09҅H4s*DN&G8=f>]v WxrtG ^\ZNc6[~ |7`6[>+1e(Y"ܕp;-oǘ2a]t&mi˼nc[\dcu74лqӺZ:3c Bk7]|ޅ[ߟ5aܺpm MbhLw,u')8>ц,jS 4:q=> G4Bo<]'Oo]Dd Ng|ְLbG] Dڌxgmtԫ;ƪ*|ôhhF7_bĎډoI{|{U|~<$S:{/KRn=&?̂av~iTcX˪uY-Oi?kPӭ=vs8 "$$!ZxqFzUS6u:!!=ٳ復_[d2SUYٮmE%36/"uF'Y 9h qgbgtbءX8|0a73еbˁb]2Sn ^ק[S&:j!l ±=Z\3e =)@[}/w'kϺOֳ#H¸I1f# 1ڬ-d/]bΰnI~B,t ˽Ԇ[Fboww㯥_e:{ =w뚿^D_}5A[j1:z՛SʀHeg yud{2QXsw`?Y :݅}-C~\jm^JbĩDeTKmv샙lܴ^wa̻6.ze1bąV}}}/p|jv&Ol/+ubILee-[A@1,NF0m~O28^Offo8iYyUKx%^?,;~>z|vY2F\ ,KJagBKhwM =o^…jx8w*\ۓؙ7 Eɡ, Oz&ckC{W#o-/QZe(0~_-8^ثFHvW]FVf+=<ƃYx B, ox|ySZ/cxtAxe>_u,ڇ ֵvDDDDiuN}mc' :|}2H"B-M~_NlԄ١I\suozv=nFrZd X5xqܛdi))}#փ#ތfksf==s3ٿgGOX Py֊Ç9~`"穥շݴݲ[ 8Ur`a2^,?4=}_Ժ(oĚhC/ J ]=N1(%),݉5rv6~w__@3?э`'>lEEWue^]α6k0Cdß.MЄejnh>ߛ- &i?f[/I^w& \G!!>}Χ$P5ҥэ1'׷Ф> Oފ,2C|0Α}|㍮WtT#>6 oI1e݊QNc06ӅފH[tX8_@}E5xo cms&9]3; Û +s+;]1`[)Ce>IIwD͵CУ{)㦛f2&w2/#՝~@&0~a''b68.+@p9!| =fru5׳ 㻷F;R Ǐz _oxI̜00"'1{BҶfH!9Aʴ1XSVĜIڝIR݊8}o&5NA =#oP9,e!NQ@<#LbH uq|]I-"'2 i9+w6cVKɑBL}R?n#9c!}07kʰ*=W9Ɛg,c>hUT.0'e"nj$%BX1jߝkQ3Qfk|&6}?OOЉ38#["E럇=!;Wyl#7Fʀظ!70}\*ݜd1^T,02c&0,1U,N=o*"""rE"ړڍ +47?$ͭ*""""FH`XMc)25FTD.ьNH[ Ml-( 1ѱl3=4hnB )/w(/Ֆ2DDDDDD2pz? ,,zMO}П`?cyBZ[\eXo;yyr˳7(wX,f G/(BDDDDDD.[_ަ}`X^|{ ע[[\ZNnWC@5-̻6[{laͷL̩ȷ[9؀`*""""""iS8j5vT">`*"""""" NP]A(kqFϣ*)NlA k R^\""""""rhdfKyyvlA6&Mڰi^7TdEDDDDDDQn y7ѱ̋%c.  ::ΈȷGӈpj6[H4MZVN &B=jlChke݆g.6nZӍ֑kFcP}@4WDDDDDD|%fdlPژǫ>du$'m8FDDDDDD/Wl]Gўnа0ΓTȷC[X,f ]ZkzDDDDDRYvut){"#0G: o|N(ptjѥl6vY(rrSUUѥH.]קCé.?Sdne;Jh0*v2UUޓ}y~7ETDDDDDD:©t8Sp """"""NEDDDDDiNdȐbbقɵ琙‚.KDDDDDS8bc3w¢.C0v9,Y[\YDDDDr?yf2!)&'esttq "ޚ9崁ica!46&7bwλo^@@z8{K?~s\$p|䟋hj"j} ߗ{ #""""^LIwϓ=l,Ev{'vtW6w<2x³ssx70uu| hKiH/^uYLH42{1l!; 1;n`ˋɨbDDDD:@+=;y =͗E>E~4}ۥlpځ Il(c/3v[vPZ\_mȻREDDDD:S4""?mkt&xꋧ)@M.n@9`{g"LTطY^ @^/y-7M&:CeoT0kCᡭ,}E>:Lk;^cѮP3-%>Ԍ,=װ?V\trO<¡^1Dҳ7f/9??Ju/3g>DO_0{J`|~~^}+<0?P̿ a>? O)|Uԍ!sSOP'X]ҦKTN;-8Lttg-dӿB ?!l1\ o `ɸ t3kaE)̚wՇ`XzkNp7L+7rѧTe)̝dC?`hDliϾ%!fLO ?y#70ku =ie`x3 dE:kLxDb% Ō̊w_43&2qH&v8 dn;} KVBh'`5øY3/o ;Y߅ws7{Dnז[k#p,#XWXN#c9JZ HHڷQ!rݴx?X|_eϾ@>(,*?qmekLny;&|b' j=SGG;0mQә=2?oezo)Aә=W+}ؔ!G6.KN;PaaCwؘ8\ HӯԔog??%۩o-P[4Ƨ2ȸl#56oH|.v䥭a籖bd=A6kUkCg06"#3D⃶VƘDk6铼tЁ(ڵ)2VŽ^5NވYD[Oغ#G-8wS4b,延QTTfϑL 8N5,\6K M"dzJϸa 2gffW'6oЈ;~a[jz} getr" )ľm! #)2V)~ek~&Qy-""""-NXy~`1{g&Itx v-axgz D X}z#+4.h ⁦./:+c&wnbe؃t Vl4QVPA}u&lVঢÛ)9Vph^2ץo0=,S=ˋMld\=o/c޳L#̐q_ھ(5|ݚrx$jH&>֖/eGb?A}l$y'`÷;I)! d.9sa&)gg*d+N4v$cØFD ^ u/k!#xW3L5۫s8v>Q2vf[Js=elǙ=-^{1kq FYx(3{ {eNm=H#f7}2.Ճm[Yfɟ}ƞ>䲛bظ6.ֽ ޫ$1 j[A4$:F@rrqSeqrSmܨ{o`n^Եgd++U06YFپ-eB6TJ"""rhժ ?e>=8q2Ð4LSrRL~OVڬ`Z O01}Q_)""""wSq9s LօHì2|w26b&;=Q8׳[;wiQaىn1QU*ADDDD䶧GɈջr:kB[K:+""""""O+ՅMe_i|Qa+/""""""xӛa]d|aIoWChõ*j7ɵk94M_DDDD^Aa׮zHv1Gnv$>P$ϚG5ddamFe=q9+uYHHe]9OT벯4>n!z ǧ)iN<0 4"<{NEDDDDD}鬙smݶ/뼯4>V6BC^}EDDDDDqz޷FmU[}EDDDDD=""""""r """"""r """"""r """"""rN/TTޕK~.HҪmѐ||Tɹ@#icTDDDDDD\NTDDDDDD\NTDDDDDD\NTDDDDDD\NTDDDDDD\NTDDDDDD\ܵzL&_._/s6-eƠ>m} #3aiʞM'=8^쉼 ?ʛ;lW~CjK16F,f龤ji ΐ!xqh;lx ,@!xؒpp.'2k+,"y[zzS!OQ;.֝)ߋ+SfiGnO!*N{1鹌ֆ`_O,m"~+zJ} 9$Gdwxz=׳(0̎tOoaٓXfZ#"""" fz DkNm6c5!3:8Sa 2&D~<4&vL;| &0wevE|W슅dV\l;Mv!y?ı鋏rv{eZxb_` Xl)ki2O$jぢiA_~Mm^"/g:C}/Vj׎c޶bXx9{e||h%ibPӋ0V"`6imcxN(jо F,dܓ{9[bOQ\:M_KwڬQU-_~%zәiKKrrW( f6t@E,@hk%ki "X"/"LrbKjessO"~>^؞Cƥ2rVHUL~Ɋع#[wv='a9 kv}63}lHbۊwCŇJۧ KO@t JgGﰮh";B=s~Q睆L=_'բѣdnWi$Iy6YfѤx0<ҳ'[ؖP$b+Kiɓ 2O`t=1Ey/zH˅[ۉ]DfХS@/VWu8q!t8u[D;J|e FN$RLdK<xکMaZx{I,qڲOO!4'|lޝx-IK tDO-o(&M"y9E!b#|D: r?l#ɯofr%X48sBL?cꔅ{[i_ү | M7CF΁{ol; f3D|n-~.xi'$m\ҳohёNz64@ Y0g}iI$]I*ߖ(Mg0y[{Ȉ=ؗT|ޞ;YxO7 jjGDDDDąN"9Ws vbxsTMJ>.1O>+m#͑áOqb#=İq"6.s;f.<{^md'U2,l򡙿Oa/0 2]x{5E?aXvÞ>ER|!'jQhfg>}6%90!3LK,VIp,3?2輓ɛV^XJs$T iG g"=|(#/3ЂXŞ` tF3zMeIpՇJÚ;uMiӞt8[Μ("s16~_Lޠ2'&rgK>w!'8b@;ˤm X Z>SE6^NgYG+;@!S&Q/hbr7E36p& {?´a@bS6XClD6{:/ʹ1Ƃ̓$`MzDDDDnpHnjdJ3#] NݍMlZQ:L;[b.?9̴]!fη̢JkyKI!Me[Rr^Obrxy )\6i{W{+~|,{ְtESfRWl̛Ēmd^3ݟhQMH}?o˨]jf /KSՀzW$""""RF&<̃!>@/l/]B 1Uf>'s -'cr+kunQz^ΡWf-2?oi wUA!s,ZO3`?G"l[Ep) xx0'0s$ٓ-|7237sj?dgG\#] m;ȯU<бwmEtbObI!""""i#htq5qfO|}wcdՙ|O6:חM))3z=cP.pZ@fTL?} ڇN,HB\}red <ЍB29ϒX:N! *{t ,x4 d6De:{ϲ:<̹ "M""""R; ȦF~܃=6Z]8i<l6IZ>?ގS}tz.v+\NU- >{CfZ|S2ۇf%>f=17k_˗}_ [7p,xΚ'@2myfgVm>6itT9Kmhv+ۮGݒmqA4?k^`fF[jf/9缼V=n*㛝 : }W5p`zOYu؂dRaL{Dž}MQEƗC' ub}f3aH(7k1jt dXg4 wr:yVF-W+ v,TTV^r^;Βr ?3E9]Ƨlw}o l7#?%헬HKjT{mOUY2͛c Ė~]+>IeBICgYS3x?#̄lm6[DDDDnKn!ަ˨֏OSR{vE@rrj[~SלC%ˌ|\?nmV>樂2E{)cJ)4WbJ׮8u3yLE&L{YZ>$Z"H;^=*A,+ IDATG<ܓ~*ދ]cKAz{9h zй#N^A#׊9kunL 2wI[[(%qbUs֢yZi%4^ȡ}_Ȏgun"""""rwP8 hjw%A󪣐\i+SdK-UT+""""Ҙ)qSaD]ՄhkBrLoqp6=iNp|tkD i1bmn_TꦆY30kƜE7ɊKp[XH*񶊯o̶_q칈ȝ릆m m{3hx٫UӨ<QyZ,nX9+tz@OEDDDDO&}z+W15c}zdj>N,pZ Z]մDr%@!^^pDDDDDԠ(5cNZTA2x+w -[Ĩ|É{`{yuf;Ƿk ѐO>Eslܼ?nbOϏ{~Epn hU<=M^Tf36ͩ6<<<:6~dVc lT8eT2D{Oww"WF~]G0axF B•L(7znmiJZ.=;w3b]=΅LT?Qӥ<Zjҵm2/]"D݃i|8K݇p~DDD@ gXM2xhi0z*/a+?'>&}onM x/\{~MO-ͩk]xx`5/ECI<`4ryb d(7՚ gTRnvE;Sf]HZ.con壸c?w"""R' 67((CT=;5-mFG)@l--4p#2SC{#OٛPX.L1}=q$'c:w5ɾN0{p& B`;ىgػe;19O_f<֒ &‚1g{-*ŧ#CS,b^pOOƎH\#rF^Uf=?eg Szj bڢ-ҷ3K 3ڲΣ SGx6hyzbZdLX 1:Oe~lNJ@)vH]6}Khf ЮmBl^r]2cczwIV߆Tw_#@Ȍ"1h҆~#GѣFk:c)8jx?nϠ1 Y5w@Q{fg\OvfL{ xv\{l6 5-0dpj;\+06ɤ3CvAmO|nu=N#3Cs#׿ݒ#&Z+٫3җ)D]϶s\/\ ^t" )8l\c3j r M`ud=J"""rS5X4uѪܧ^Ӓ}Zz8UnChΞ[w $7SWne *9-{a9>t5QM0q(hGa~jÈ=.i˻%Abٻ|5CЮbYio` L;цb8%w>ۙN1ݷvh$r,YG& vG2dĈtŬgٮ8dj!8ˮxٗ2 dRoعc>^ KpzSoχ:m_'oTlз'#8z K>\Ʈ̖t 5ת C`y{#Xύ\,t1ycz<ʷuxrŇiq_wS{zw7G,Yf ;~X+;cdK{˷c MgB{Y>uTm?`o8#DZ$b#~]˳{ w>$l_^]0]Ͳ%gWh1ta>5G9]Vɪ.%ҏ]<3t|O8K_(?:w0U1~q[܎i=0b6Ʊc;Lx+nԆKj2c6a5{g x*캬ז|)vJbtЦ J >S)uhWDDDn]Wng "oVrȵb71õrÀXb)S]ͱ`5av3ُn,vh{)p)=1T!95T Fo3ƫ>Nzoo3P󽓎2b6{aRU`Zvk՜G]W;j12w?~td!f~x[-X `~d-{+)a,W8eG0zMy͂f #\K&`,^\ ڎŒJ~Lw7l+f(ũmRM)NvvƘAWxKVmT5 Vk4#rRq$6LA( eJ{ddqpT1 zG"8ʜMQ_.4JlnMXxp=##'g}q|s/Ժ=-~:ƺtWsiẍ1u;)7V6}z2̑ ;37bK\+}2i""")Q <ϯjJ 4!n Mm&y )XyTm& '& tn7A@h!zMZү_GgH,syCL-*4SPK`g3v6{TpĝW7v xBiDEgAA>=F|KUuB 4'FݗWpDəN:-?P#̴2*Y%pg7oZBGڸQn4(z_'3^ս7qJ0a'w;FF#:[smͷcdcw`,zCҽe  ~t3EUj %׮QgMK!t1Aa]W⨰@|X D^}iW&uD}?wg~0{@zԩ N'CW2xnnn޳!4JIjNqg$˝fq[6GmYBm$FWt2~l-v<Ё3g2SA3_Y˜ovg 1Ȋשm!1ڍA&y\ܻS9cG`ūn-o+g"l$Nz!.&S)Y{8cϧ7XSΰs󱢀Q&q̘Ķ~MT]NcklLjYÁ vmZC#漄AlWc,q=#3cC ,bE+`(Ʃ?0v4O!ȊfOT_8.coILz%L?}/G"{#< Z{N:*r(V.FnF4GaS4VjI?xbx 69ث9$C3q>M`Ɉ1ޗHZ<Ԇ VRΐײb^sML΂EdEm[-U[׏_~^Jee֌9dggG9}>>MIKMu恁\u o묻ߵ01ԧv-<{T{!Tm\C{B=y窧1=_a_UUS0tֳ7ނ)'[zTTDDv`7Tn}rdnx9_JݚRɔߪDD#2#cFfk:gO&"""q:'\"uz+S=XxفEɩ1m*)SWBD}n mq0DDD+V~^Mx]  N'(0Sx8d^ݦvmb|SI`Z3Z[&"""""w#؉ж|ۭ6+h8&5~[brܚ/|fN:լ^FiJJ2+VF0d0&{RRFh){/j:>~f[MӂƂgS2P`n"Iw҂*Hpz.N;3kvVf %_m+lji6YTDDDD-ӻ?v6Mr~?ٝ+]ctGݦ?ٜ귾 =n`ZP`ubWR8,N}^ÝwR8|Q>x?V=`qk~tZϫ1=j}EDDDD]jl6gm`э:Nmntc@E[/֩[LЀ>| -Ƒ˅b#X\=4*wr~v 7|gx>г֏~~S{M"q|⫝{'&w79"""""7~޾˽d墭<1dek671ȃ>3YubAY bꤖlL`ڜ4 Ɩ5q 1CYR9|%мW7/b6'>#f-`bۚi#yz><̃5L dٛ V@IN5A{OwAѭƾ]s63-tKve?/07߬?z3K$`f>G;q,p+VYv&β7+Lxd"~g'""""r[S8m$Z8#~F5:$<\/J<0HN:Xeg[->xc)zct¿bQbWس5ǶDW\mM͈ '8=Xn9!NmW_#R#g nn] 8`}PZ’C^TP f eGS&_(9uwt)!k &w,""""(6RgȖntkbfMkDr gqpmJ x=E_SfjocL̡J{֘=X3s rȱdrb/uo+""""b ܕB٣np/[} H,{dp/imDc+}/N^b 9-Ga}/NtX"SO& ,CfՆ܁eFTʉuy{?O1| cT&G=ʂW]Df9F3<Ɋ """""7r iYOSRSjy` 99Wjǩ(=!""""صk9NL#""""""rk)))J^arp/fDDDDR8FS$7~[T&iz"""""2z4JS64f)ppX izS6t"""""p*VM """"" MS8S8S8S8S8S8S8S8S8S8S8S8S8I ]=ZsuQ8I.sEDDDDDj|K`tiw1ݎnw0DDDDDD))))))97hÄ""""""RBvAӛDTDDDDDnnnxx\:ӛDTDDDDD$0 """"""r """"""r """"""r """"""r """"""r """"""r """"""r """"""r """"""r """"""r """"""r """"""r """"""r """"""r ""ػswIadʐ% qEG~[[kֶv׺JAAK $@d$w7 |<499sox<%"""NEDDDDD)HS86p*""""""m i[:-^]SX&s\ 9nMG_%/tBgOny읂s>E=r s{/yOz:TS:;J8Em\#LV2RNfd|ΏG!;{x_ŝSN]C>L\)0[Ĝ_Ħ`_ش%%>cSxG IDAT] 4_UDDDD rir&q{~Gvk,ȫ/`_ICoyQw-eU]}=oؗ-7],˯y03bpo>[C~+5ԓ|&'+۴Եr]"""""mY8Iܖ yCQ"B^wNN=|t)}d՗8Kp W:w&-9)-x@C8-ȧG\'kc?vԇQJ*9X.UAz $=?nzM޺~(;çO:Y8W4pN+NNiᄖ)ˌ|8i0ynF{I{LIrJWw^oy2tԇ@>g牉w_Ğ~"""""gKpj33r~/{ɟv3Q`Q84PB:@io4>u~U6{ ?垞EcRФ֞?ު|@(w~G'吟ֿșj?f)>RS0i”fggX%wfI{֭m\蘬$)ظ` {c?l@>N =M)ɿhHTF݇KSx"""""mHa )C0q5w7Xl1#Go߯g﷿Ǥ=o[rva4]CA 6+ajj %bZ T۝Xs ;N?daltݱX|nv2a*_JN) ٩U>zSH~H~SC:Ce%xc2ux6mdն"(\ݿ) X3pM/#v8xCWAC0x=s%xՍOpM9o{i$ ኊnwbi 10cqzb䳵`4::WUIE&7,^QLBbr[?^=/YoGdֵW1eUxR^M~[6R|Xg"ws3eEy˷g7 ijo;s/fs2_b{ƀ=*y!/Ys~“e0k|{/~1gU}EDDDDFN]#^S0ϼ^0[>ex<<3=[͂oxxW\ldRr2-vG૫>a8b!e>C,N7ՁAGŞ\X:=nw&! Su-;?]Q%pϙ޳g>Ǐz}UEDDDD^MMuoI;NϩRT\/>Ӭ[ƟSSq{!m+saEoJ8l`b`v |o``qyi`qL.! ¡0Z:vl$q5X\nl)i+ۗK-u^RR:tEK!"""""msN#'TT\8dwt1L:=xٚ [>`ȂH˖/˹cXKH㌉ 1(t JXh?J{`KT »o7 O9tt?pְ9}2ja7[JHDDDDD.r©ysNGäa1@dn񣷑O>4l q{Xm3ӆLӤCA98<EԔ_8WSJ Gq/ATjgX{T! rJ.!j"l1|TW%]j"oǜsG[7BDDDD<ּɈw@-_kz[X O^/}AF_Ʋ'Dȑcq?Gy!:b:1: PPުX(| ()Ӛ¡B~ {\;.7ye-gpO8PaZlX:\܎szxTvM ?^|qk#F8<߯oǐ"CF|[t|8a#wm Z  }l|⛤ OeñǷ#//WYF%x<,fO=0qE """""95NaiSy"S %|!W- 0Qhc;~¡ uZ)i M0DwiԔ[pORb2qXaOL"aX%= }&)6?[MV}~xX}~qS9KKƻv ~3u,N!OGڏ--粙wLjGtnxz~GL7 :ա;طty0,M`F/14HOLgL0#F3i  y6%=ҫ<#G"sS._"bDcX)ݲC]e_ۥa,nd$)3ؽd!;?m,NW q]>|C bi^XDDDDD.iSB82Yv5,[qڵyɟo-+E]| Z 'ډv)HHlYZF]I Q \'O F_Euӹ;V._QvjŅTU\ZijJ{&N jژG){}Q @Hc ϯ5OĶe-n枮yaQ6MCL-6M{nnq眞kWl&N'x'\T`ci4e˗hHY櫫`0@tL<1q-[]UAmPngG\t2SW^[0,VIz\iX1lN ,W~`;Mao-/W[G{O0.E vcӎRB7ok**8)@h?9|x6_NS""""csz:hCl'\/>ss Zi}iRWp^L0 BPd\ ߇i\n6Iج6(B]A>>=Aꐑa4#_(<ۿ&a8?sŰ8̚/[galNa)8w'_/bmA*dtVJC6b{Q}I Ry`;kdg5=[oHn'={TsBfW4oKt/NGd@;.gm7G- U{% κNN۞ݝAM|+؎:E4 Z'y#8N+#ݛˇ,bgL깓yX3n>l3} g6CF%m2g18XZ-i07I7TgŲLkOt X3~ 'ᢎҜe_=H9ƜS"S?yXv5EŅx<ь16Oß~k€XL+p8azB0Hn͢P)`8RڧMt:]⭮ tŕܑ~é|=a`ZOz9ڝd$M;K"=;מ$n+RFpèXrbG`_9I)^~ NCE9*t1ne^q wdRb úy./l-u']w<%9ƻo5@n^`na}[fԜ9/L츙%ZS/^,Od/sÐ ̝"% n9AXsb_Ϥ,~m͎ vSҦaNYT8y5NI&PpBJ= u+B.\{4|M>ozV/ꘀdk3!=s B$cp+{RF>IAڵlJLDEV# j}&dUY8N˛|KFMOϸ4:v sBy_.K ՝3Db-r#J7c.I|69_jr*@9k6oV_Ys9}*!pƑࡴx/k_DDD36kSvjn>E SS^zCE 㶿323? ˦N#2YߑҾ1'aXlTst3G|1ˆCjVE~B&j}<ZȊ:zMs1l#elp+j9D WĆ,b1Lm$_;?]ɇۊ=تvGAJ˽=\?bhjGMeù<)GEφVoc#ߏ,3gY=u'{cqz1oqucX9ѤRyDVz!Z3v"WAZX*1T%yF+Hey)8[} jtjOv/{/bzpu,$/"""FVHN +vbd?VAbT N1kwשI \@I~*+JOXWWKeeYlY{A>os3 Cgۥ.Ot)a>'ByK3Cb\{H;VYcy: ދPv\ÐN" ]íӇxZk@ywט>8+AB`d(kB{5cv WPZMq[lF@v=`yԭ~u_sAM!yjj[)rtbm`mLnեy8ٳ& |4CWjsٶ#WEzl&pr:r2dTk\;Aذ{".E@OB*N+@`lV[}<.%G1|$2dkPkEDDDNK”9M')$! 7Mm*qXV, pp8 IuU^_-q=sꭄC!z#&;A?`{~pK XmW慼#oeHPu){>77L;`m1tj][MjǤ1*Jb/?Zy}z^nnQNBuY.۪ LV~R#'s={R=E{gqLrWW/xt۹z_ío}:z Y꼆I=wVV-;=7fCx,!*ʊ{OZlj;~w=ΰX'2G:-_L~P'{\AhZ֤ w=3PʮSA6P9e_밞ɖOyl >eOǞbcrߧ,  Sm5;'O` {9;@/"""vStt JJZTGRr2-vθװULYi >_>LG;<[RJI j"/Clv0MBwe.O16iJoDgk9W`62v-"""""SZ-9o E|sP H4ؾd96+!'NP[UM-~#6IP F^?~~-t:L/{nfEEDDDD.@rPKzN/PDc00M@Ͼ{:-V+CӾla7.)!clp+YV E\|ycvuEäaKb [IuDQ[VX{ V$X'SbR eA;(;2~-` FRmpzZm$$PVZL8&*]ߩ*˩6  msRk3FSdYںrlv,o+M|B6jQ [E0m` |xH[R%&>12,Ot1Xm5Tr X'${JEDDDDp*҄w1nH:scw`޶nELTqW~!k,YHp`[7PDDDD"p*6SX3O^ȒR;32 L%z@Y>êbʇ$vGInOu2nd(x, \&ZHd[A9qw~ 'x;yia=:zֲ;)=X;oNn/3mb~2©HO4jHe$ ٯjg}E ]g4^]{ʆpIܹx 4VSq5փőT,!bR|?x̎bN p,""""r9k=՟|Lֶ-gV"#+]wѷ(yޔ[T6Wz Ƨ*.߁2Ӵp5r dc\ZN{Crr >_NxtBll|c뮝ʘcٛƲ%%iiDZG0̏˳G; [6Oo`2r}qײwqe..Uaٟɷk&zyi_#84K<^YYO>`諈f~-mH跼qS~vn#/A}mb0G<0~yϰ ǜ]Y ~8GXoSڰO| W/(..j`u}fNEDDDDDL[S[|6co\TEgV""""""r8+_ylU/""""""VYWDDDDDD%NEDDDDD)HS8=KLA9k:(%\Sa&MpVA`[7CDDDDD䂠Sis """"""NEDDDDD)HS86p*""""""mNTDDDDDDڜ9=Kl6v0ں)""""""'d&`0fmPY`*""""" 0۝mӳDTDDDDD.$maNEDDDDD)HS86p*""""""mNTDDDDDDڜ©9Sis """"""NEDDDDDZ2$>5')+0 Yyּ\Z%:=&{hĂAfy$e.LZ+ tZ4t% Wzۑ?/LYq趻 s ZhC.Qa8ܙ֬4npW<]֒&E{N0sPfb-N;ZAX]φ3m\(:=&|x1Ϗ訮'YTKԞZ3@Eɱ8KQ #Rgv8uzL:NXfjΝ g^!DNS""""""'լiZ`GO;mVwVzUDDDDDD.Z'LͺEDDDDDuݔhl-Ҕ|G-Ji}.MHO9ɏ_<9sP(4E(?~4-p7?,ۊi-Nm67`8 ב?Pe**\G=vVw㹃!/lS$^ǽ_DƗ-;,Y˜VRR q<_iqWyvyA~1~{EDDDDNM]*G㱪O}XYf|RbLv"}O?".¦i[ptOY8%쯡n|߈iIhr돹=Zp'Ky5YCY|{H?X"""""ӞsX0`CD*>jvݾ`[}T}\_+k$ϸGcD%`gtz)?zۺg}üO:`:˼)݉pp2XӘ|8Kģ㻣n{s;H4>6WI'kC #zOx0Տ̔^'{ pWADDDDmvi]$4w2ݳɱ5;x8a]=`:W?t8{mg>0M_F7H8Y}{Sgo>>;.g7z|?+RN;zYIj>a4{zz79(9i8𼴞!~LLje.0qMF#@xk}dQ}=G3vxlB"Qo7/;p vcQ<:O&^Y(.xq?qG ys0?VDDDD v8-/ҥOp2\fO7Z쬬|xwW.'2|y;b F}z0 8R30NoĿwʶ{ Cqcl 5%>AwoaϠwՆ^Ey,RMbM?_[4WDDDD;N}^C-xU\ave}]/W~~\e؛,~ >mqc$z"=CxMwQ2N| T~acOh`YiDjv D 1n$T ~p6?7;+0;奧q=x(//nr8PGֶ#Dں,Sa\)i3ZwN+>MWֵb!DaƁ`uvlhM;kz%L.~7pxni6JU1_ !Cю&}f݁RcOVɶ*q3$,W=v*J 3|YGZO6g90lN]: g1XOJ^Ŗ[JO\n÷<}.^¦-^a#N7qeX}&NN g/YURħd};oG.JćH 0?kfGݮUZ0]: k4Y^bIFӚ|OvL} %Ϸq7?GGVn~7۳`o'|>|UPd}W ||@y9^.`_yyfFURՎbOwÁ:[`N‘1ڧ5[DDDDD.pJW1$NliŪgt.tH$\\="X{`i{FC-9s ll+{2>6C"ΘL5?K-ެwZ|`^;p &xj/ꣿ(%P=a0ĵǚ܃PɮODDDDD.\ πiď_li'Y*q3mPdȮ =F]Ӫ#X}H. """""8ɋ,f,N֢S`V 7xO05Ke v="-bmy""""": X3}?1mrQ{u'eS)_;0Mv]E;Fz`3Z"""""rҰ FWc Xŏ/>ޥ%;y7͔.yPu X*pt8p yލ[pc֔JEDDDD£  qu qcN̈ H0`?φ[)^15p W/1CgQc a@ 팆l|S? ;)^UZ<$\hNQ)MCSi~Ӿ 5CDDD.~rYF}ϩqՓ1qsFu6'I7<'?K?a`Q)D6:8zZ"q4nȁuo1/2wE.[rJRL`hǛ5wSnf\h?9Vl6{&z[ADA!KШHwM4&Ldg'6f93gf[gkl33&fId&FM~C5*"ȥ^vAb|w鶬z~O̴+ʄB!Rޫ|-­(J(D|w:E'jOPp߂[^lYі:cLy4D G%s!'1-ׄK?՜ijZ2R`>]Gj|Twex{"fiOqB̈gFƲ$͹&_˸Id28M6QxڿÛxsY\@T읟~8;y9{g”6ʷo[9d)jbFW_intB!X.sMg`Ճa8?%,./vtw)sp#T >:@ WOUO|͜?UәIi)P|38NnbI<^ltTQe8_i#*Ҁ)v8N ]`8YMZ\~PmRZ:Ë 0p6Sx +6|cTs_dFܸľn&裘IVp4X4I^\-͠oŇiٝt0G7G+g_"9aԕq -/K{B1V$8u;KA%@X(85М:OʄUc Zc+3]UU wZ[~,P&$ސ4TP.&#>5}BLb(?Z$zoŤqv|f;[ۏX:k9og0`',g=ub\o x`ącY^.+:S$h. `Z)]Ŧ}y/saeH|MʧJa2ᱵ QK8n"y|E ҲɌS *O '^9QW]DULrfFLSyxJeoIIt,}^ꊋ 5TЙg[K{2 QH~j27`4LbsϺ)y$o76YFamYsA׼: |=w !Bɜ >Q&NGFׂGv]yψZSb -mx^95<#;uzIU ^P4~9p |v;!gz2z|T]ɇ,mlT5c c}-fʍdEVs!TVM$.[Y*fwRPɁOOrz6>:wh %ǏZ^ST50pfqv-f7,޳[04RWE3CjgW]ŒwmGU\RZ{kX`p~շlcMɁ]X,f5~7G[ G6 r9h-Xq5OZ ?@4Z\?xq;k4$ /{/jk?hT;I\2KƁ)CdY]nT1:`a7۷},oao'bZ6y Uv$]էww %cW{aN~R:a'B!;Ipzj 0D%ғ}4@bE }Sh5gq_FxB“ Ӄ4*4״v_.?cx/iJ%TP7kywXB!BOC1u9UDE970凹 L">-SR V[@W1.T1 c N03T /9UЎX|B! N %BuNsj\4i|+9#&,.>,,r|:+]s6m]C1R0ĦK P4tZ/`p=uDB!vHpz4UhBoӾNb0Xpː/[1F'&<00g>GIg1-5@Db.梄0ĥa JxI@oSE{9 _"N>>ct>!B!]C{V{šcES@gU4Uim-q .f@? gc~ Nghu-;wmcuN;w_g+3鮻@F,6/]{!9*ޚswdB!BQ N=ʢ(+/ fJCK}*)ɩbbK;̱#ѽ%|AC(o]_c}M t|)DV7;p|_dԅB!88>s ḄqSO(^(Nsz*՞UO9qG.v o#o})B!B׈S-70d5pTWO=%+eXt;KPPMg?OFt 2++io͹``:5V6.8fc=k+$9!B!# N<fLL-]IJreYyijYy Ϝ`n|loH!Bz^KQ;s˧qġ+a&,q"B%Lhwl\d[ aHnفޯ8|:Y4VJ*B![<7ƆKy-]X,X37s>fwMޱ#466`%%yb<E̦9( t;zc zFlyePB!3N{Ĕݤ4֭7Sb_꽷-12:-ŠZj\Wk6~ !: 9C)}puƯE`C|yB!cizljO VWXۖ}X'nw=O$tUG>EWI]mhK;8~=T4x]:! !B!ENm8A3z>eM-5sڨbxڿFDb1)\Q1Sc#UGץhէo"u`F8a 19@ ЯǎБQB!^4*uNo'0=pp%cNyh},^œ]>{b0bJ])u U}2Z;h_j< N*tmF{wۂ ,^&|J*ǡ=gpЄB! ;8u:OK{5o5u:; q#T{<{&ƨUQy ,uV~}O!,t7@y،6#?pak*:y!!Ϩ?xzB!Bܣ:اL%>g6BQC " fA+Pg3QsQM#,v֬YGgEKf͎*J0+ NWc00,`rEG˃yRwvv7y9 !1=,qfZht3xfu'l;McXCve !B2ɩ:}Mp~%p:΃3%,!*)sCj&1"ޠN*['ȇk,ހ{R5u_F}&쪸nC>ߡ,γ~MjP67GWζ:?룯:]4\^{B!6R2e%uJ/vf/bOcce%&os5LjQ)uR{CJ.UZ; L̑wCrR:qX%^2_GҀq EQۧ?i2II1̠i y0cW>NRt6 SXJgGLj0tww 1lD]Mt]x\U}B!B ,I<6*uN5-&666O$cz5qS:.eAwW‰&pURy|'TFF;áWxy]mn/ȍmaZ+s})I#B!S$fnOfuLj;Qd8GFtL m#jdΪHW8hL+XC!B::GtHQɜUV^" !B!aB!B!FB!B!Ɯ⾤{ԱF/NEr3B!%/k+!b"@T"f1p"B!ĘD;d[I'c:øB{tB!B% N}+ڹGB!B!&dYB!B1'B!B1'B!B1'B!B1'B!B1'B!B1'B!B1'B!B1'B!B1'B!B1'B!B1.|rIENDB`incus-7.0.0/doc/images/ui_console.png000066400000000000000000013630261517523235500175410ustar00rootroot00000000000000PNG  IHDR8zTXtRaw profile type exifxڥgv9cxsf."N*,03QfrjTjn9[bw^T|xޯ_+[zek~~޿]Ur:_ϯz].{]?Ӳu c?_?a?b>_.Do%^LHtĀCu]>#:jA߲G~gG~9ƥwdWsf=fB_zO-tjZ(35Tw'촃}O7b o~/_B +T`F&RY%CdŘ+5iRʩZ9Sιdb/DSRɥZZ5XS͵Z[ͷhZ{v9s@Ï0HfQFmI8̳:˯WYuշ۔Ҏ;ˮ~(̉'|ʩWZW͔,YRޗp0o#E))g2dRΖS`Χ޹3ɨ2_͔-o?͜Q2swY[y3tjݷ˘!Թvy09f{͞E+4N-i֞ e69'ÏFī ZNkY\^spsWua:ur_[!N][^vIdgqlǑ~:g ˜ɢZ۞n\kVF6"^]>V|dƹ *+y=sWXɭ= eޯє71&_HWU{ڀq4ktSjV!+2 wE5a~ޫZ$=]#?"@8J&FLayB -/ N߃_͗CSs#8t$.>=u탾sGӬ˜Ơ>~wW& 4@sTl)hfGhj":=gB.6g *TeعQJp) VN/ҼTjkF8ޝ/ % ovBq>`x7erlا6A8cr=nO:yBCn?5@A.Ab W i U&0OQ@aq!kci,h\n>T#ɷ4ijopq׃8)81oxg<g>h 1)h=`z]j`N1gufMΜ$QMMaZ!=(dC"dCv&Dԉk1*vт*8qy 7Rg 3stGK kEJ}|,bo.$S]U~t$s/lR(Zn9C %ןHW6Er"k3 1/]1CÌ譙wU0I0P6XMVPǑ,0J^fjlN>`+ <߭l, 5pB)D/H@Q: 邌g'ש;XH- 7$fJ$ƔZvipGυŜ2 |ZX;ԽuǪPxgg~-24ceAL ;#xE&lf W +PRm" |@ΑudP8.ӎρ b(Q="G6 jD B}$=7uIՄ$H@3D0bB/1ױ ֟>r53+xkt b/]ĨP`HzΎQPBIΡ~}R:u(J vq`##qƮEeB prU# `P4a<7B7)ʹ/HZT%}K~)gxIEI $0?7 ys{OkhhGb0m>| &c܎O4?xJ>Ǽ I[ ZEPިq-F}n$/ڻp뜍H%Qf$љB"~ua-N`T- 0*P#O)j-pG-i5YT,4aKџ2pwq'(@d8!T#礲!LFNBR{P<%v']F0:WP 2~2& ՈߓBء|^Prǃې &F U!AC=f i5Â7πǢᶰzF ;xGZpr+cY B%b>h3>{l {U o`ij 6MH-uV9:y,b"It=w8/_cAC16ǎv|GV/phz /T=$4Ze0*kѯ4xXj:- T$CEANTTLWf(Y>I P$x0O^k2$mg0bkR`gvF^)hUݐ֟T,,V^Qr+(k4H8%; J/8֡ą{!V TEYZS骁ؾoV@``=, zTׂ9cPFdme+D!d#gDѲP@$8-?a1Js4 >. .;N6V^G#G,srg-a#րȸ|x!NTԊi!{`-%#,|K7 2`@ukSN+4ʲZaHpx1a3mt xg{ עKiĿ0>:lR`s Q c3֔2=2vK tql݅90 l׫I gsWB6 r +Y.qO~ U2.g/q&m9Kl7lDDKVX5 щ*ZN{Ԣ%w0rAmO1Jៈ0x7\I,Ek+EU#K;Ǚ| C; o>{9੊_IV\L.GKztzUBw2^g*o3{V/>O6C 1MY Ј7%9C4.e5D0梌#/7heHsxM0 D ViŃIzwWuմ¥h' f1,JкQe N6eV7H YPKmTayMA|b3.^:,we'.}hêB6j`R&M`vԪ0h+FI) :iK9Yp*Ȏ(v-b(3ohE%5Q66-1Zd3dKK" MÔN'ݛoB F k I>fiMz+h3c$LT>Xeiasi,g~d!fb~~?*}g<~-+[ in}LuDlw2 û]O$0zD"V A]ÐIb^"XtWRWI< -( A SLXUoXzLd Hюh#l)kna|q$N7&4mQ0n>ʚ򹋤N҃9)u"AEi~+5‹D[F]kK=۾ŕ2 ^xUFpU2 aN@gEk%ÁA`d-ymκK bӦB`EByt,w#|RCJo1rH EGSE-Dj*,O-ڛ{M?.SJZ֦|DSPX3)AtV6a4% =줍)DRuf9qN7\Uz 5A5 1XM582iR+=02Xж,4 J;1Zt@͆.9tN?A6n^K'Kh/?x]$d:Ow j =MP L$<*Qz#,тz,p`=3:h{@-$JX! F@(ԓyPiTЎt0EbBFV \z rK'AR. EO6$Ra#=Ҵ 46ʋ Y¦g¾v)H՞É0Ϡ#{͍cb' wrYN , ֯7ٔ,]6,б]WUuNEW}G XpcwWG+$Ț ߮yi 4:K;9>{Uw|umw^8#SVv.q#'4]z/ '3ď\W<~\vY13#.VULx8j: yU[Z_)K\94 "QE 6X~?%r)䪂c ~?ݭU") 8#@xh5qZ'@;&0Izŏ6pqє=r|2dSv MT -з27!0Z5wvv?Z5r#| iTXtXML:com.adobe.xmp -bKGD pHYs  tIME 7+rD IDATxy\TW  2*$$(7Q,+nb~,k\Z\Z\]sB2Tb"U&z10HAF~d~>=|93s aYn0g+Dd5_D5HQ '""""""""Rȉ"r"""""""""HH-R '""""""""Rȉ"r""""rYDDDn r"""""""""HH-R w2|]""""""""rLo&vy9:N_řDbfMaŷ?`9N؈Dp9k4$x5 ju[Y9s#K]NX6yRAAg!bu#*u=z~qݗ9 !wߨ8V$qf͛Y=74bB~o{7ʂd.I;߰A]ɾl|a%X؝1356<e>Oԋrir&܌m#dff2c lmm ~6ls=wSrٳ'!!!Wiٲeҳg+Xl.b""rK/]Q=z2۷1}y:&ǽEOpNR öi UwS ví,dA̢WXw\]/B&>C^܍ XNeg7SZ@ẃq+Y(,~C|ğ>dqZ- aCPݤ1~JEXn8 ;p|8v9Gfn>'J)-i;_X`[! ͶP\tYGkM]^89/3߾" hQGOv~Ee֡Q Gtm U̘Z;_gk0a^`ƈM c1K8{lF%aO:R\ZI}QYˁf\BD`lm9/S}"9oa˾dqC03oZ1t?E}ӵ{4?]<~&YsP~ $`h6lM$~ޝDV ']$) >mz2itr5ȯII;F 3wHȀьǯOd9Dcgdž]uحy7aO2?fc6[*UM[ZƲpz''˻xϿ'GСj9;"X1FNtĹƿ IK˦~[zФl+|LݒhUTqIΓױnˏd#/k|0םmj˕_Kk{9:%mk?aסlϣĖ&!cn;b'}ޭovùӧOc22w\ aՖ cΜ9̝;8 ?~! `""r˩ w!\,l'GIʞS*1O\]`%鳥$-J%;7`(WTVt EdbpS(aO. HRE9N%Pd摔 Nm3_S#;NrW<ła$w6wfpp¿~rS,M" 82)fm`Li -I,<'|Tex^[^ru|u|9y7 ,?; W:-EeP΃ iTѸCOI yk;9^kt#ӻ\ -7tv'c^̋겐u#1҄#Y iX?<:˘~#IN-Em6Ry彸͸NiY:F2zw#QM532iWMVF|*øS=8G3n$hZTͮceفx X$nf3=B88sktbĿg0-,Y4/M'n^6TLfS½23_}K:";Z[w1~uᑤvħO'd~<++{ 9a<|6}DᄏYu-.ڣzLjĬ$$[3 7Dli+47&΀Y #_6Rٴ#G񷃬B]Pj@x?d Zg6EրC:8ytxv`n~Kϻ^tjvUE;і T^ |3- Sj6z6tǿ17*Gywv[8G|p{ x9G0cz=;':yŅY:ĩY:]3DHp~6~^'w#<}[H8MeqwE;Xu#N\ֳܻJYq*5o%#6~]%#~(%Bn4=Ӊ1sc~V;-4ICvz̠Uoy L?FLܫYq;7E12sz(n&?<3{sc:Ӧ́SD%0?li"_HbIlh~UNĘ쭙铉;Uok0]!չw #ɩ#4ӝ=Dj7ڛg&k4w# c\(5++;ᤥ9o Vn#?:5l_o݆"o:tΩ;rw{aGI5ж߅0|)N1,F0a  V~5P|8lAt*Md'y0Y"NkH$c;\r4t?ACi8d,U7ѣ m-OHlIg?]},a6fsH9},GOO ttU @-)ɤ_F )|۷U)@{pv4B)K˿))dmK[س#[Hj9hh65t&J)dL J().FtĿZGR)t2l|Ҏ|Ciex;f\T2Q9\vxPf16*'yX%QdgPכ\VV%[nݛb)`Q_"""r˨!y<П: Q3uc#0t~M]ۨ)u:QqS+`nirUkBU̾'2mN`pSx[`9 X]`)̿L9K+lWלpo!?7915zij7XE~ [e( (b޸^˒!|rs3ԇPV|]Gn}ɶh'ڎNwvgܓc|zx@85ge\Uw#Z~n}O{}}9+%&`nяAݮ.$I_>L kɬ:p J/zdcQ$|Kg3dl: bJ`76H->_aCzAD© -. "CG2BtLQt0GJ/|[XJuB1Ff>dSs 9܁$P z=GmnUz~ FkF<{Uv7+!5C/Y "å\c|Ǜr9idXܹ)Jg聇p.R{\:/^!xxݴ㲲2:l/c_8wܥ?DDDRsuv^N^4za<7/9sfB?J`Ul?Ue?2*AցT_!F2t1;.7z`t`e!?r]Z>6vW=kc8.{xc..B T*!0Mob">mz%^/,bأ,7={W&4OOՁ\bE$ŝMk{ڣQ{]` \l!܌sRe 8YdG}%gd&*W-ĭM,^RtD^n> .78pǓ1^G2^w3aC )ft_#币VW`0cd}%=L^`»V&OF`z1Ma܎xÕw44  ፷ĤDWI?bqudafؔm"%53amߍ >wӤ=^Φd@7ͳKT2ιӣLR]izЪr-G ios;0-l/..P&Vw*4;Cav6WFfx݄f:Woڵcդ}Ų)))dee1tP}-N X/݆==?|_ΝKRmXB^\,ѝ&WQ3ȯ&D~s.wu?nI$JeY$+qvu$r7T/Cfb^G|Eӧ7y{{% 5zzVX3qj_;dprӠug/s5 Vm\m\JRLF>T_ƻq))1={7cqz)Iws]"mӉ~=.VBۺf22g_&mMU'z>d:zbEud^bC:i<١'$BO_5Hb޻{y(Z?IYIw0Er0Is0sX1HڅȸCd{Y?ݻ8?~+G/\dep+M*NfP*x5rWevK~w#5݌gy:F"s.!7 ; $"55kkkZlySq;wёիW_qjaa!WQ :-zY53Ρ,' )I=ԩ@]g7JGĞ?x_WXʇÙϘv\qN\r5Ű a͸KvhJ#d}5֛ŽuZҏ1YW}, Mj{~NCMl c^=7UiTVΘGU&}j;/x^ِOMd27d-%(۾`s7߮c1>y=؜-h2x?wg\ i` \o['sx2އpu?oNN<֔xo:hVg-&tc΀} a"3`0M X ufbhb `9UDoo^Xm |XLޘR`7G3k&zrFDK3H\uB}V&L&ټE_>NȢV#{3cYV Ef2oea$d,BwdB$?5@O)qx+F|B 5s1I/0~}3oGسMXT0Sk,z!PD1o񗙖Τ܏`Bﰆ:Oo񜙴=q|%{sBJ%ыKmKˀ/?w(H%x3+.玒v/jI##^Wvsߝ/'ιwڟ!=lۆu쌛M`ggǰaØ3gseС_|233"//aÆzlmٻw/vݻW-b)O"Zڪbxk$_,#^ S]g}ʹ8l:0v K_旒y 8?W.挎YY<{R*Ik[X:z;b1'/&KC߼;xJ g_')tgB]ʘ7wPYcL"JbC\ooƐY OE8ˎ\,x]:-Ki*0oݓV kE]OCI? -1_ͱd;!|4j*?}W`,e`hWK?HPz$ϤתYVn"%( #l;qR`bC0UG_ ;dž~&fk)Vx^3dK%f}itg4ķsk ?ptY8JAQl})Cį~Ϯgc˅,\a11Q~tQ0o,vg|k Y񟑤`}ųJZ+2b+gLݭzgs̞͚W0&|t`\c Ĭ]H 3.j& 1]t3vBzA pnGS46|jf!D|7⭱D.Ʀ=fA{-1llɜ5,bLmŏ1l 6yh"DM!:Ì`ݑ.k(if<:W3}5SHwb !uE ﾱ[Wvel[6z%W>/pJep'#e ˇt 2 ^t2ֵ 1G. aN\xr3 Xuvvf͚aooS3x{{3n8-[ƌ3>ʥ-ƍUXG{w?SO=Ųeˈb9GGG""r28qnz.΄Ξ=ˉ'hܸqm޿\ zlWݦ&M6[{(P J]. 쇼@AkU;OaW څ2|pC]#+{vDzr&R-? #4/ X&7;MVEpdap w`Kn^ A#!;C {r sSycH)7tʧKO_4tG!-pEXޘ=`h3O,oF(2ܬT#_⍸ pp_y }d^+ܳibg=G"i?._Eh4C`|{9,yo]æv瓜+ȍVHxЃDw^ŏo]L}ؑۦJ:H  5p5s]Bo?@ff&PЮ]Uglgʅ ίN_Uƍqww^r5Uour"r"& R6Dp)FRt*ݱDFUCL:_5ŒƮ-Gsh %d}/`gy&d(>ȍuC DDDL-l8٘-`h삏_-^ jTA!Gb8ZP]\<Ӏg nD#""""7X1iQ&;b$pBtJjq'""""R DDDD?0J(E=dKb=5R9Z@NDDDDDDDD)E DDDDDDDDDj9Z@NDDDDDDDD)E DDDDDDDDDj9Z@NDDDDDDDD)E DDDDDDDDDj9Z@NDDDDDDDD)E DDDDDDDDDj9Z@NDDDDDDDD)E DDDDDDDDDj9Z@NDDDDDDDD)E DDDDDDDDDj9Z@NDDDDDDDDY DwۨNFlmm/[F\[ """"""rN>lb !"""""""""5A_9Z@NDDDDDDDD)E DDDDDDDDDj9Z@NDDDDDDDD)E DDDDDDDDDj9Z@NDDDDDDDD)E DDDDDDDDDj9Z@NDDDDDDDD)E DDDDDDDDDj9KZlQQQ5V(oa۲eؾ};K퓕N|O\ˆM;@mj~gRΒOvx FwɌ欓""""""r6lP :+øΝ;/[ȉHۚ d~k)Cfq)Ê^`?bc q6uǸn:/""""""˅rT5YӲxSԫ gO"7}gnn3z8QC,ގYQmkzxe2YX B`2kGa1[:1""""""KC_~ 㠆36mS^gXA݄~whؠ="--1DCC ~/>KbMгojsK~u3@?ٗp*+5+YJN @֘[Sm^=@=̍P^^"G&XoLkI+>;}X1b3F9C ߛIƈ).It9ύ&)M!hRl~Z_Z'Ռi*6 梨4/Gb (O DE7v>GI<3x%$369ozKԘsz CFN\4n~vC\p%`mmH̲~*.Xn:;|΅8LkZ>=橳w2G\phZΦcڛq(PauIXM: zg#[wQ'ߗsƶoh3mؽv Kލ&vU8C7a}$%ɂA j orc㗄qG; !tolhJ>' cS|#]&1~H0ޕYO0#F&Ӡ7tP΍&&~)º7xNJ"=+$O}xtoٮ$&}g'`3uvmi]r`͛,OH"|u.&AOb,-N|Re30c>a|/>!",: oZ_~ [9ӡ$"6eŠgwJ+1ˠa3_nX9nM15.ˆjт=D(3Sl3F}?9 MGMO@+E[Sȉ\˅qU甋}rڎ`P'|.kZOHsC: _xt'rZr15Ú#oQ{tG+r5ekN3Ė7 n{#57SI^ȸd@pc  Pm3džBJNue F(;Ʊc5vNTXK.X 1u(3ׅ2΢C)6nfp;0bFOH z~8OdpiSv. A3ɉ\˅q.]}zx w6ys%%|uS\~j)3%vvԯ &zvԳ!sߦjkp~E;(-.+"7T9 6M#| 3i`[YINg ;τXju!vo^AF鎝;a\˚F8$xm^rF{|w(b_ړ?gd 9+swRB+DDDDDDoZ\ຂ+Byk[e(w$(w ;!X)6ؕ@Ap;^~MHF:FM9–+?J1L|+qu,_e5!!KfgM |i !u*fID={#šx;H 3 E }2c?slvZ tj $6x78-5أ- DDDDDDٕ)WSn[%qOU':e$*Uyn膩a.KG۱}{ᯞ/Ұ.bqV˖;d6Fx`F `;g϶NxυT.5,(쵛Xς茩M0!=?<չ>6C\L4 FL-rpc16c>`9Z!HJ#-F VlZGŀs ?B^ɤ&(ˡuĚDy?*"""""rw dz%A OΎ|;o'jk^Jow||{zӻ|4b~* zmByn^J k؛4knMz·)錽Mr.%a;ұϷ%-ӤBܛQ?zs#99o oqI?Qqo>SN夤}oAP%鬟;t6r2͕xo永O?zAW|i,^7u?zΞ>Eؽ+s:ޜ!!cNItr"}1|cI@=|3/zƉ``/%yÃ:{ryz]5ƦN4tC:""""""""2pt9fOΜ)9oyk`?:jڣ@NDDDDDDDDؐ 5ui5\E DDne8OA[ĂE """",r""s$͒SYOr)DDDD@ND2&D,5ƼycgaV`ȌI䓯~vGeʺ2b ݥj]@ND2NDcjvw+&ys,N\9ݰl5HQ '""""""""R"Wr,SVbtfiCiz6BLyak$cƈ}?f.N/ҢD *0u]2fpnщ~/ަJӉ4w&7-s)8{2xL&4UC3# Z 3886Xb9DoIb!$Mt+w08sXdzGK<*e~<ćcNR[[<<1 'ۊɛٻbhtG3zJJũz\;zl|Pv{ќ@|I_q kxe[ǻZ3q@iF6WD-ǰ4{msl\ց8+ 8m>ڹrﰖoTyaR~*==~$#mֱYjQ;O`oCi162!2yp7| c[c_qÞ3l{}?).> ~ljFJF҉ČQ_/zlB$F'E Z{s.ߡ^lrwv-ytrn"+vb0jky\Uuq!K DI[EPE-CkFEIsIsR_.ӤX#"Is%%h~ e~><pι~{χ+g0pƭnv%)w |Y=Iç(Yؕ=++5o=O> |w831wK~цclY-r`iA1z8QH,}Sln{+-?9{cop G'ӯ4Χrux. MLbmde`{#FFAn8c҉|~)2Xiz&#VCk`/(m `ߨV|d;'bL#ԥi9/bKV2R#R,""""'s=p$’Q<|;XAi=;о29d˰p905WϚKU@/zN!|eWZ̑<;׶8̺ 9m̍#MLkLTPX̯y=NPz'߉] __btG|?qMju8c?QUX'O;~֧}HgƄ" od޻||7m.}[lOZ)Nsk1'p`?S^VYO63T _)9_e`O? L`^c{É~q a/j-$?3{2-&F]OZZ62c;&Vn=G齖sT9[DDDDDp߱,nwgNPVzY@Ш)dBԘ'Y gBjF%1mșܜq3 Wр8E*7%n[}Dq=zܜcC] +'x}6ǘOsp]kcQbqc8V+N4"A밵ӧ( {zf4n|-=?hlmLŒL ۳< ]43CG=L4嗒р |<\#c1F sZ+""""RKS)H'tYvYgep%`B{#ukNΔѕ Iz&ؗ Vq5b@廿c1Zw5Ϛn%OhDN8N~NgxvvXqf?7/QdNP{xf#Z^Ó5~ Le9x-M.x{[\sn7wu~}a:8c{|ߣ}]3">s2婤,ȰmQr )3Bc6kU DPFVD"""""5Vr K8ߩ$VuɗIx uMO}۷}l<0s5I?_0ss0׾\Ǥq/I|%栳{Ē=>ܹsy7 ʽ xxy䐼l {s;y"Vǧu"ٻE+2ޟMD̅FzsYI69yy&RzeY"z23I\9\c>$o&`6͉$e:9QO~N~wEްw^ʅ{}{79q~q7̳25$&;{)f̍[pwH͹GNũ8@J3~9E~V!(ru(&P{>@xdeSLb={.>c)1|I=x怊bUv ]zTkDׇ=`Ϸm~wc܆BOί?{VdwՎ5]gL/")ss 'JT;h&ujҭ01#>Zd ym*y neݼX3s#go zvK:&b_ZucBH /k]B0f|\2{sޛ"]1?GPNDDDD,6̚M?[{n0jkD'®v cKCg4*̼>|"kAEgʲ\ vxt}gO%<2ȓKU*+Pw5gx7y> ׏_|\Uڟ5 >S`#ѥLR5I}fO23qZYPGF\Tb0=nxy 27MH⏔b܇!/ g8mi4(Rpmݕ|}g>zoSncKf7yW=nԌf~Ta&2+_f[ל,fiݿ mւ6|>RބOnM3oAC6̖s>ԟ=|h)mlZ7mO=fRlSliɛ&4ɓYבw*\#JIq>ݰ5:ؖ9,6XϧS{7N}-< phBY] L9 [urȹ6N=#,Rf,w%c^ mRWS "z~QR=BWBڻ*MY9F/|Gޠi E-eMb2H@ٙ ݒDDDD/&W#!GGǫX,;nW2j502b]tSŒUǴŊ txo6t x3*~w|ƨe=>yƬmeguMh"[(`7`~A j] d"..Q%%%i~X&kA;Oqnk;,G|SzV?KkCVmx#r.w;:uĐWnv 04ז#IZ3r`Q҅_WF0NDNs|=tVN|϶ײ_{ՎȟUjㄋ#(aux?ZmZ:YybLOmLP;vЫW/Ds$ŮSãg3ю )8/vV*̬z?< 1ߚN.Vu}i (!׼i) '""""""""b2w5gVRR򛦟o>e!P_!;;tשDDDDDDDDDr"""""""""H9zH=R@NDDDDDDDD) '""""""""RG ȉ#DDDDDDDDDr"""""""""H9zd{TmʟܹsU """""""rYZ@N iȪURRR%U5Hu٪_ xc,DDDDDDDꉆ#DDDDDDDDDr"""""""""H9zH=R@NDDDDDDDD) '""""""""RG ȉ#DDDDDDDDDr"""""""""H9zH=R@NDDDDDDDD) '""""""""RloFՂH=Q9zH=R@NDDDDDDDD) '""""""""RG ȉ#DDDDDDDDDP`RRRtDDDDUB=hڴ*ADD*kn7nΝ;gΝ:s""""""""rMjCV݉o߾ܹ￟+Wrm{=ʎ>3SI>oZiex1?&5F?:u*}%&&e˖Ѷm[͛Off&}78::^oBCǭ8) '"""""""Рrgz3'xcǎa2xyWj싈>YkFCo߾ۗxV\y衇 88~rc\]ڸ;(<%sXinAt!we^Ml쮨:Ľ?RW~zY*:;ʊr1.spYe="'HVMqt$/ְ]rv3;;Ŧ¼,g_)eDDDDDDDDT\!""l`磏>"""!C@||^N;hpt;_(jA=:E`=xt;>8LCSZٝZB%lڰr0pM-3Ъty 1".l1CB~ɗ%V\L=rxfVMx1sA4=K8&灣e#]ZAfɆex l<O?\^EDDDDDDD5UV/ϼy0L7޸ghu;ku);3o}Y5vgns!qٜǏYgI*{ҿtĦ2;z==KhȢ̓ 45ϝak0\""""""""rŮk=z5SMnxx(؛En]}a ptb @gΏ.\K|X+օVvpx .'w0ص2cw/""""""""g٪ ev<;&F'l"? of9.¥ ?wLzVPԘ )@f7:aFG#k'm~EEDDDDDDD͕~7CأCKv KePjv%Wgs݅ he&w&6m:l *\CgԩpCjj*k1U^Yٹ~P%@{:wyao w>kYK͝gM:+ɽVqzLx{ =\(X5?]͒yĿיs)0д eeu_DDDDDDDDT }3B ̝<$ԭ$<1,li0Ub4[ں9n$~fzuN5#v$ب5^Mhb?D0sz+z+=B5-l{;*DDDت D}^1!nkH9Ƨ93!ͪ&/ /'ĒJ접ğ "ⱅ<ݡ9doM&[ q :5ĥ1_IhδZ jc!?z0627SzvqK'qq]+Ƃi&G޿"a ˖G`:ġދǽ&w_Qf# ,}MgGQIDDD~*%Y3S{.A>Ĥ#@J`kDI#'B'YT'P--]y6opך‚`hU?kM!7^D|<Uښ4^gIxKo/{ f|-1X`s;^NmhȪ\V|쀱L;%ɓB)Zy@^C{[k!H`"#zλrTѓFi# ]FyˆT3=4GHϕoOrC{0k.x&Hۺޮ'y=^]ބ&-I[Ve**2[+9aĂy$w&lGC2rwy"݈ͬ|r7ݞϸ\eudZx $;^CXSS٬<^zWC8y$nNww!v}Nù<JeDCvޕ礭?}k[ѽoNWwuHڜu!9[6`O)UiT]RTg8U՟ʶ7>] ݍ\} k+?wu$2-5QC.4m}now&ˀI$lJ%>zڹݑ,^-e j' M[Kxz0|%FAN](m8@ʾZ„!ף ] y.> IDATj}ңӭکjrwfl9r R#/fw5]Bƒ=صߺ$3wt+ۘw^ =3H"1C{ѮeĜ'""IB{zCzӵf38t-Ɠl ncI"\xnO<ܒIqk#/@ f3X1WNm6`l3Я{K|*w ȝŒq)0 9}I[28d";2EYqYxe#M>WQ G2`.LOđI wźcW/xECC_-$BI8dr&a!ᑁDz,\ X^#i.v}G~5 YŦ-)ɱG2qEޕYF:71 X?!#M_Kڎ]lzg2^ȍ֊y`/8B kIl9w3|"l꿒!l9I[H۸9NXY& a f?>u]e h' kR|W/Zь:=>eLs ΅Ųt O ɟ'hg&fP;ʒ&rc8nbG˙pZ:U%PsO&>ײjt::Gf%ʶx4NwjS)I)Id|ݷ=lD쀦4\6{' ־N[6.pd^O[6=lη,[IͶ mwa"s_#t&&[N/dHYU[0ŲM[;sm7mUrD@kH ȝ8DBZ5Gx+{lҷw_hwU{ ד A4.'al7Q, 5ԝ^Ml@1__w_F!e`۰ pF?$-e6~1|1pkb<ЗX|ϗ'͛c"V1s3S{w"Ȁ}iie_fڏeÀ N1!+X)$2c+X~ɛp4Zl^W^MؕfJX뿸R''jsшXJ.~a $=w/=DgG++'6u:ڽѕJ(`(-Ibrq'mh٦gX(T=_.GȜ$۳n\7d4<*"WNGPw#ɋ`? ˦2-*c@~ OJuDHeֈ@ׇ7ĭV^M*%>tOo$Za{`7d%8J7%Ý._zOj[ v藭7#sFcY쯸\ [,#]FV,'߳ ?{. ͹IsJWު׃8Vcs) 7`}o&?`!5i#Ò6L fOj]ǂәK3?$|8( >#26 aL~/qYDMc_۝pûbfPr"{'G.X6;'sw5qO7X(UWEX0lnK-m] QSX119Nz4,!'"WQX̔%WLˉ42A.6 [jְ@agD`2?F xڵ]f^C L%qm"&B4+!wQ"X ۵ i?IE#O&{a^~gJe&+=1vͬzp^7#`eƊ~kqvrxCXbH&17\\u=nMk8]"n $b3׼hs6gWU}ȮO[y‰0j!=5K_DDQ9:{L> ̈"_ ^Os̜Lͪa E5z7:+ t=ED? XsoB:\%Ү2:ĢY16)dSSh˻7>g6D19{" 3k9l>ՓiQA<MډDF5կNe1UlgM9?9=]B1g$zD 9>@c$+2Ƨ):xFo8m<< b t."XюuND.)"oo/=F' Į8M+pkgϮļ9r,,[z  &vZ:'"䥒9Ícd5v !f=;e&xJdu@_7 yE2=C.:^:C9:% evUum#Jdٞ\TϋLޝe߰WX4UkҌf|ͦܯ NM3C mќR۽cWw /AL\΀4?5P_Ƿ;cA31Xh{^[y+^@HeehYfoDDӨ&y|9 :2f 뼇^/‡1  gٿ>mŐS RWB:\NivyDYkFj] #.DX>+EG(uū}WC) a,\m`δLz c/DtX~e#!|Hܬ̝29?7#G\$bO`0DH@UBdb^HʍE^᜝u}'8?,xue9lMD,_˱ďS3ed_C x⟍'+ׂ`ī}a}AWѯW0>L-^AW k&fA ] }n-FU#=r`77^c.6k"z~QR=BWLt̚CyA̎n0CnyuV$s ,7=vbb6?$TVh+k?v9vH`ɲ5-7sK@0l+o9d} 6rw&s೷I Gn!xBX5-o۱7xٵsp8 g4ǓǬw%cLvtBW ,>R-9  \ƌ>tQs7'Xe!vpZq3o:TOb-"9XyEs[kS5 >S`#ѥLr-O2 ""T1LS54lYYj!dVGwd:nRu\36},G!@P6m+KAYLGkzȉH=9cjC1v}툘`\B=VhE ?omV?0 Y9҆OѬ("""""""r憬pB1 Z H)&-m̰ bȤڴaaycK>7Z/-:f5^k搻RG8+W:9һ2}/&MdɇUC'm4S5k7qk︋Ld͖iADO"k8yUohԷ>fKs>^2[h`!x0mai hӆf^mhq% dӔ20lR>[-|XߘyOVUHzȥ7oܙ~?Z' =)9PbxǀXTٳ%ۇ#YRw]Py!<6Jl^ْ D疱px{SHL`N#zC1gX4% ǜ4;f1S\ADDDDDDD>\StwO9eV[N<EMz?7DQZ-̣f̈́9||XwE1VhSWm`"B@Qg=4V`߾ Mf$YEDDDDDDD搻R6m2Qշx53^aM`s֥)Hdڣg7]' a#igvjD=in1ԍ|{CSgvrtT7XAG7 * |KOO?3i(Rc>ٙ'̂ZN4s ɯ~5@f8ʒ8a|{-N6` Q10u|^XJظL_}DDDDDDDD~{ * HzzyjWzQO⨳^bKЌf7%熫B/7 YGG+{QMCyeBr""""""""7d5== uq`Y3I,`X``9|#oI# p bFl@E7ڴoY^^@W%nvpDDDDDDDD~sCXϷ=.:5/(*AJyWt\S\/hhjEJY׼mid*Fkyi 65- ,T˜@A133g>{tulxҥF9K㯹`Evj<)@0n1cSPlÕ%!f}X 1!sO>6). שS'F}׺ZYFcJߎxr]5\{v0&|\쀍!DX3L(>3 m0`0J9,LQ8x{{ѣGU """7R[cRJ@U U@9ZDr7; J[lV%TŢJkͯ1DD]~DD䖤["""rMU """"""""RuB ȉT!DDDDDDDDDr"""""""""UH9*HR@NDDDDDDDD ) '""""""""RB ȉT!DDDDDDDDDr"""""""""UH9*Hq@`` 7nԙ vUH /n ,[: k1h ~ZADmoL_H5u@TW r]V{ŋYx1 bӦMU3(ih3kbƫϲ D0cе*"""""""? xbzM޽ٻw/'OB< ~5`W=AOUW[& ٳ+,SSz~5%r{(Z)'i[.?%Yw5utH6mK#==.gocIzVYYYdX6efN:fRMbkf),+m[K(?˥X"Wl%ŒNVfD"WYdeySILJ5tҶm"E""""""""ȥw ˫;F撩,| BrF/ o۬mIZ&_}~`}I~o[ƲL~ނ5 &Lڛ1 |pw/1W?rIJH"G|_i`@tv< q[HfN`׉N橌~Êѓّr8y5ߏ%)ˆ DDDDDDDDA` IDATe500(Auذa\VZw^ f} f=-c }) $^L {19XO-CY$ ͎cr3߄%$ώ>4a2B1]:`gbؠeaC<1[MЙXl̤8ID_o33\ } $y3!|z\I9kxa}9d#"""""""r=m 1fϞM`` 6aÆiӦ7Wʾ'o[u(3hٌr ...جXӇ{/Վ%łsgwsGOl{27܋ &LmBApjqKq6 [3|]K=h.#,fgĚe "ڨwu 'p(+{uDK2ydyxp5-'ĽXz3|$Il JK64֠w:ѥe?):|g7[}iiip4h4o\xuguڃ A. W6C}+B{F_~=@0n1cSPlÕ%!fSVv;;dpl4V;'c}ތ%dU5$'ⰖrVpcQCRF6~= hV\!7}>1`R 41xg[`ČdoA'!}Ưl钒 +-d=N`z,Nپ9IhY=pg#<<[Mznذ_~ѣG_6]=z4-bÆ \J2tdXrޥIj6]2H >Qx(:^4e*jvSU-M ?}'ձ ƕ9lvw1ceNtF~:iK~W 3=Ǻգ(}_}&Lm{1͛wƔ3̃j6 Nuhqu_d7WTqީ. tժ˪q"ƌ;oͰYhh?KtP; xl7 #,K6nՃ%W,Y֘ dvKb 2ѪEc7nA+LI𙊻8o#{hѴ1L _JZNC-̞Gē}1y5qVNB?wiiitԉ_6קSNUVDZ؂M[a*9.VGwyˆҪW皺]a.Ⱦ̾&PVhZ9d/f@sɺ} ?O?DhѸ>K\{{6&?Uc7nACy}WJcs36L]c/>b}gX&y0bf>aDT^ 옃E'2Uesҧ|!; ɉ#{^Z ;(g1}/hpy%vPψcC=D·; )i8nF[?N%nȀ'g\6EL?Wܒgi2/gs)"&=\S 9HPƬ"R& S\C6 hŎ/.sN~O^<ޯ-waC6we.gƻc>W|Mx4388,x^xx7_CU$ &m[z;|3Y"\+9t^m 0|IմS@NDhh8TXrt/s(7rlDÆ@AC{I ms3Mp͛ш NS?a?Yɺ39xկ kn4hগtEqi׊VοB9 [N}qڹ!u+YYTqita4tSm8}$çQ_>v228Ǐ©AC.,sUgX:bOG$޳HJN:J]n\9'K?.@I"]sN:M~oo8W59zJ˱ ʺu]]i9zAwOtubg}"ﳏHo;˄3|z:t 9H\>-k8SR~S///222i n~;l'gNnys4:_XיlO{cfH&.úr|8^徊qL]8xhYq6R@թm o4y-bR~CN_ EGawnXLmz½/M-""""7Kxxy߼!RZ?Ǻ9+q̜8m{W@ߊ8Xd';ZRvxdIm/ȹex׬]jqV 9!ύ; -nߍݕ48vJ`?qX+\ԤgJf&|\qxݷ$;;___8zUosQ3tNq./+oD~ycdϝA`j]<ΡBeAkЍmHK8siF C'_n=zc3< w8q~kx:[ؽm&^aKW3 tm_I;͌iRfo4W ȉ]Z&C=jq ︍Ҳ7O#j3qOeU$&;' w㫥D,K&a&;~qdK&='ϖ2f"6_vg=+$&/'- s׼ysL&k֬m֬Ydy7ny1XH^ <3H؜|>y8ogx~Әe'Kg?M'$,$!#lk2*ά=JeM nG\GڞlҶ,&E~x*#Z_!eoYE%ydc'_ӨCьo'&jG:ɬ:i"|L@i 9rhWi7#7?%e+H;'}4jӾtBD/,8SWSaN^%yɷ{plg 5ex+ '",9׼\;n>, , axd,e~RХ77gEd1?]Lnor pnb LK9t<>ZojqVlG ?H?R5nԨQŋ_ѣGYx15j`Utez8cHv582(I[IzqXFДp<>`gXF{t"*yRgwz=Fv7V}W҅\>zq 3ǐ>u}qyX"ބdJ˥fci@dQN;_s~OM%U {ɠ>iCz1,$""""C@U$gdMWY 9/ iH:dStbYI:&oO2V'p7@|Gi AľlwđQ x}VHUe y;6~ue eti@`z78W5Z˾dÆ XV:uD6m뮒>8@FFL* Ɖȍ /`ߜOTCw.9OѮLkE8̛v3u{:KG!21)^# _r>%(s"n;/O#l&3IFĪҿ?S{4t&1/vǾb(vg\=2k["4̀1XNޖH"ތ%!#nx7bx 8t0^˛6\sϓr" |$Il J44֠w:ݰnUHKK#''#GJ5h///|}}47* 'ym"r.-kӥe U-y Ts ȉo P5L:T!DDDDDDDDDr"""""""""Uȵnݚ֭[l-ZOЩS'ԩDEE3'""""""""jkҤ G&00'N/0eV\ƍYhڂ3w' pK.Z& t,K-u}UUȔ)Sضm}aѢEٓ7qFzɢEӧ۶mcƪ)'o0cSXI~ކ[=AOUWTZ{ի6mW_fUXoوbL2p|||5j/\K:aO4!CY .B]I""""""""rUU ^z%cǕW˶j1mE=UH᱊I "I"++Cq)[:l%ŚN5ft_~>͊aS| Yde~&Gpmig.0.wb&tZ-deenM#qr")_:%IOD FgFʇQ{""""""""r#T1iݺ5/h<8pĪ e[<ÈzΜűXW>J}_ɹ%XgD;?̔$]XqXrs2˿|~(v;,% }'P̤b/,F&arǰ&;?sp?f%~K.ɠ.A;撴y5[MEDDDDDDDnj۳gF"00$@I6nHxxxN3o q,h,I^M]kɎcr:v~l$4kp4M/ԳĎ\.{V3T}2G>9_[y~&q #@pN/k,>=}p;Qcp[3,eI%t`0oXs!|z&f^xU-Fu-ƍ !$$E|Ɩ*ф<ߐPJ?~/$|}740wtcqn?8s,I`2crU0`0رce& .K6fkf]I$\pqˮ|Bx qyfUMqX*,'""""""R,X@P:lSS$73dQ0O‰FF\m (8rb xČbrv k[᥃gF`<).X1u<aXaxj 3:zR'zS Np65;xOfc0",M` iKmga+hƿYqeҩcm_fñ,Lsa=IA<$W]DDDDDDD7V&&&2|pF}tFѣG3|pp`VSx6a; g݊Z~n jzgO!Å,\G|B փ`hkn _,V-N~2N[Mf6>Z䉈V-&Oŋ '00(6mT!M`` GI&2y*)kQo/|W#mL4[rq@kΡD:,Mu~ױI>mV, LrI\gmtaF`"? 1DY~XVO3}cY [z. zh'3'W< 4xni&RSSiݺ5X IDAT!!!tܙӤI/^#<޽{{o>c}D >ހ;zwK.999^ǣ_+54L%co{vųzm8_|xhZҾ&=}Ik^VNDDP۶m(ѣoǔ)SӧeٳҠM[CB JP?[ _JZVft5|JEƿ2'5^ۯq&/ٺՏ5<.L{#^NnnԹ+ޅ_Gw ir}F:U'Эv؜wck{kMߚ7pN}܍^^^x]Eޯ%7%]ӗNma~-uǼƥ?nj;i)L`Rm;c-=guTMl2j֬YagϞeƌDFFR~އdb<?~4Ǐ'88:tPvEVVֹW\yW33e[IIK'+3tK"gxc]'韛5I&1M\-yTWDЄ"Fv{KwFwܛ'Tj$;8}n[h֏Hci%OV?HڟzO[u>8ǎldAWvnI77zQtmWHֶCh4b3=.ڷӟN(_2+a)|G4O٘MvplހC;3\5ZX)ci?fѴeӒ`_L!]ǿ}]d""7IHH~-~-4hРw}M6e|<׽Çosĉ˦;qoo5\ԩS犕1`Mc 2X,d:F= !)˯>6 c˄Oj⬞'hzBb_ !|tSSJ9ke#I,ݓ'^ؘF!y7{2w kUv^o/w*Y~ci:STh})_jF#:Ed~J{ (m#P֏/dÎvXD:""rE`^A<ŏ02cѺs>mʉ/3 {pq7wHhyP-kޝ֋mnQFgWՕȖs'wنI6Cx n_3]W84p_a=kLDf}sGׯ_O~{رر__jU_=SaÆUO"r cG%CKO\3| 'eZ/Z#X B+ c8.#]58SƉϿǒ^f?sNtW,gNt_iT'Z:fU ` )|j*|mdbX,G!,|dk l~+=ZvN\QzFld`,Dׁ:2` CwZԫkJzl?䃌 v/-2M!{ cWTo0}3YnāgF@균MO[A7]?YZhq4˗/g888v5ՕJ CDϮ%T_<%I$PhDIJM$EVf:)[bϽ\"P!?LP.':>[IfUҕv(BTҊ8$,?z_ HVԈHH\iI1{^DmI!-=tҶ%r {EURCqŭde%VCYjI'==c߳\wW#~GvK"icJKdPLF]<Қ5/W>a œ:qL>S89p껛/p ~Jގ ؑ~>|8/%倝H}k1~PNj9b~zU0ԏ8[ǒͮ]CCPǹl=ORRwu79`(xq{ӻQɍt>BʾN ,ߞqXÍ<_eWv>$~Ny@>m÷>wf_ڢЁ:-|vC?53}ZSW=VED~WvoBۮ/4UTT]w]<#_ecǎUP@jdc_.]G3? fEd+ K'd,݌W\Of8eB3?BgGuA^Lzi3^AĘ AKa+ͯu1 3SbY2<'.u}߹e>ʮкh݇/<^sY:«N; ԒWڎ ]:p6}%u-3Y-""WhS'~,f{4=?Ŝ:wKrO1~N, ;,].óHڔ]}OGuL#vvr]߀RnL8̎s20 3p\<?B12˙Jb#". ?̆xл5ݎx?ro/;88Gc$Y4{<ӟwSe೩ֺ/CO1yuED:ŏ?~eɶQ%[\Muijթ]}y/>>'OW|<}"A>b)ޜÎ`qM skL_rD0 J|*^vp6Gw84] 3H?ԄNM!XM~z^~Kwrrb„ ;}ߪU?>:uj׮ͳ>{nՅ&uњar5ߔݨA0pirwݳb36'>m\tc0ano$b7Wa  we[ u\t5a O'QYM&.͊5q[YC'1zZ{s;|ycn"l{toNNW;ˏ> C ۗ ۼK=<&lGv2Pcb&đuEDreL1E'K姟NSvIye{/JM ;*>,r8ª Dyog13wN%*LCSgjQaGhK≒cɳwn SV;jܼӆs ێ9@m2kզك#(YVXA-hѢݻwȑ#:M@@iii4nܘ'|K^, IIIԪU4kwe۶mܹZ՗&uZ فF WC}#FG8QXP!pf?X@A1*9iQZgN5`Ppuc`&- PX:[ŹPKG#tl;Sv>nUwtyą-O1{ǰg8"f5i1gEߐS{s~aTŧlIsMpr;$j_I.p"_`K>y!9Bb{_׭osfnnOċժyϒތeSvjzMZ<_8[E|||3[._}*9y9VF<=}'of q71 hVϑ_ KZ ѳ%e]11V <5ϋ&۰#U4uui*lcfxpg0+'R^"nT6㶣{:)&lǟۣ9f+w |.ϓX 2O@oٽlO=B?c^Mjs!y LtED"=p >~VE{:M<+jx3Vz l4c[Y9j+/w;xG&==p,YR3Y^|E>~LBΝ6lFՕ3f iiiղ~ꛇ\6O"?/F?e\<$c bYX:}qǎEIPF͞I|6-yq:zgr,/bheI.ײO<]neؒp^wB_ iY[I+B'yvXW:`Lȷb^g&>ʣ 7UsIG4[xe^`&^V .´Wyl_0 *9 NÑTDlHILLW_hƕsj%JnI껳~0}ǤuԞLkc0Nw?anLleDo. %cycM3*;'#}uĺXMWܓ>v[.僽lb+cb|) Bw'aȿ|WP`_`L~F>1gy};s;̘K_**˹W\ Vܨj 2!T0ʍhKAUB[rhb(#l˷?Z~k3>3`η^{}y?6oo߈7_^cZ3#o6\Ʒ|W_;'cmlc6?'?n6mlc/Wn_x7|a> cV=0mlc6mlc~qo( c~^1lc6mlc6mP&>}2Ƶsmlc6mlc6m1ߚ_g6Z+$*%{MsHf:}6垠gj{s8&ܱP<M3o% W˝v2e0G ;bDrw˙`CQubVg](Ni ZU/>1|ݽxõߎw7$^m{8[/Vd=T~o] ҝd81<7?,0=3!) 5iwgىCHl?Ԙd䶱m<'CwAaRvPsl,guKd'Mp2J/1N`\ItTM,A (Um'6T *&v[9ɯV VNFX0] Z3qBJ.Hɢ Bf쇈66Q. AbP܉R=o y@h.mY+L OX $K.uxx`KF'ckRЬruþj:®*%hN:]e?f +|@4'j>fZ +t71q1GN(2grD`Ki0}gĐ\8Ƀٰ􄨢2XuQcD}~8H.X@#F%~ŏO .ueeb[wjr Њ2'Ķ1SmEJ:5. .[@2߄v r]W IDATT&nt(*Fo#8N-KþB9I_0@צy$o0X|vߩќR n8p"\bē ӟG]<9%Fieַ='=ѣc!^1Nd uAyP:n=搆L漁ߛ14Rv*Έ m6g5x{aPa.,ޡm-~VWNLOAN$a0vŵ%nЅA[FQ]j^䎗6VؚVLVfzr`w@X0I",nm7,ZaUy$u p-L 3h&݊Il(L5]#_hxaLlYnImUؙZ>jד2V؅pҊUTGoրim +Z>p9ǖoajn 4'y3۟mqsĈqT flNly8kI$9:3)~̀Þ;33vL N{Mxd?$|P ,}f4g]lWĦ|WI`llƉAp Z[LC'Z`:W-E#X^Į<E%SDoNj2;܂FZ^-&M`E%0\K| @#x Vsm1S}`"\o[Cyc XPPYG8 >fEnZ"HsZNߛX`s,.8g]0,wJ}#++͈>85lmഃ^t;I@@ZZ>G F&|^zb 2ntb⢈+ eZ$9bZ;=s4,gYb0XbZL^]Kﰻ|)}1}Fҹ Fܖ~%F$-IA c -;w~ɴ˪ 'sZ!h6:dofMEJ޻vRlp\ҴuߠŒEp=xzjU%jĜr>qkF;@ӄ<Z0z ;s%pqnK0t- ݄V9C~TJ.*ܕ⾫>#jud@@ԑmm_̥ "%!c 9Gtߦm +]pQD 6mtC1R0s6to,0g:e9"\1ʟh>1!۪JT:{$AfτgscC]XWၻ?Sd6T9(7]a(VXX]9r׈QRgGE89OIrSYb]|NX-gG܏SiocOq ml㮌KAf_Q|>X O zN9pcx5zfD!?J fĄ 5pSjK:YG6anr{>xY9, M*P5kF&陫-{r}&ǞڡrBNp콵yJw| M%J΢6bQ@ Jt\)0mDk9f0oJif2MvIU•bm`n (ϣVgbD-e3]u@DsGkBi4I躎c "wR tq4"[r;q5B:&;H ěNi?YSSn ܗs;S+\,/ANZ&J} koJRk`@pZS&F?%'ͧdFշ^Ό4b_:V@ynavT9&6'ZTya*xW˴E;sepءc[EU鬬mMSNxc»Gk1 }6kLBMy.ӵñIj[bOO)h>#:,vT8zT %шIf ՑXnq mlYoZٜz J5cw76xuDWxGA+:|X@%{',TO3W]N3wҙ  AAOr rk!'Յ1SiAC)bqõl9 - $̗ qЪQKiste~ۉp#1 (uR_R&=%udg{[s5whZRR&ڒr8 |ƶ$L6b3%, ϥ v~Oq:o}:J695qu629c=W~c4w@vܫֻT"q4bDvˬs[QA/<Qe&_j\RE%cQxyU:ib鼱r ̅lXVOLk=}K+ mQit x+Q*뱃88{t6̤e {qA' Nԕi=sʂf XK]Π"+8.:Gbs(hHl cNV-38UЗݳq mlY53su^*U=!vN9DO1;'*\#=ʽXki ٯ=krZFC %_Fꄡ`,h$ *XϺY~idȩ[5_*_+[If&RX3,Ca3̲_R}`wz}I\Ood$Vͱ*.!Mcex?5eqa!dDV`ޔLBV2ӮY0JTД+eZ9ӥ2U F{Z)ߨlctz{Il R={i=/puVGz[כwިLE7I9u?H)W0Qs.s)CgQ@:g:~u^;u &q_SQ:&|iI^2Vb}:(7&'*l mA+[ޕ@@ל# $A;{k.b$E/bn"uݱGOuq6/ :pWՃJ{e7 gPYwxQB5F"lmn.+g,!x]ZZv}Vb K ΃9>6@n C";V=i PaCǧuMdZG@E93V,=YTyRW( QQfF>K@qSHIAi'c  Drq`\U#]THF\Ϥ=bzL\b+&c{"䄁ISL`f ˞|7et5@*Q1huzrkA~p7-)+qC& Ll{ 'M{ީ{LU`},dq?eDf i,>K Xu8CnDЍ*5-E;CG;O;ܙ' Qh}Ђ%o;,(j`,v2JgDpGW^֥J ;7<>$p %v3+-[Fi0A0%#hL;}8WʮoM;M\V:O>US@LOm>4,:IS7{ l?euyg;+^ ?; (~7UрV U9' 76 9`]4km2-*vuFh5tW#? Й%HCiQ`*'' ^!3hгKUӴ3]_,ܥdDb9~sqT`=}t^ۈ׌=f3j\AS9;baGF\ͪ*Eu#ձR=艬XReRf X0TfmVxm6{puIb-CPCZi(>mLeR <920@da~*Xiž~VP/+*_R0*[Uj/`>G\^m#E*&\Qk uڴÓc&"}25ݪr]x *r(]74v`zy/ ؄©E?aZP82Xk5,ؙj̸Yϋvpp_yKg*|5\J-~P2-V5 !7WŝR0!h{mXMP,YS7&4OkvѰTyɹ[yUKjZw.aq&gҡ[oO:Ϊ9f(c }r& Џr)e|Ձ]2 i_ˎBis{o}sI1235v1he䊬 ~#{0Yz@%fgc!nU*#@FoTJ_fzND1:p;õ=W{}$@=uux,2aO!K̂#?5WnBTഺ'h5{"k$QXU( GCһ @śIvxv6It=Gu ^x5Y˹zrgw^nH6dyTZ98Kԯ5G5NV+Y_zLԑh)(y}q MLUÕ!uyf:<y禷".PE,Y`>?ON߂T?x~Ў2~ hdlڳ4k);iu(20X4,Z.)ӓ6VU' źU"W:DU±"F2 ]Wg3q/n?K~ o]@ DKN k=l]A}}^+c`Wz)LM N,:ڽg]P]a#ƃ{2滘Sʨ$9`7ѾgЦe ҺR(1ׇ\@WlZ#k:EJ֟#rg>qeKh<WHy6fs/b<mg 䀶&+]VOto}!3 Z0J#}y%W7p~vQMotHh4Xqv10o9-e WT,H䳳 JSv!d6{2N.Ӥ3[\N#Ob Rgڞ:OZO֌C]>l(6dD"Ww夒nv;Ds2LY|4܆ѓ=*22*!B|'(NRiQSCUd@EIC {H;~bluԖgbpҁQԠBH@NŶfl̪E:TvaX>&$tnuUm`w'KT)QBBXAn{7?_>q_Em]uAIrѢOfb@-.Pcr̕HD8 CֻGw%J ԁdl aZ1biILz͡-*В-7[k\^P\wSӇJp0) _G+мC$˳6 |Z*T0.'s ր^a#{^nȋXX%1,d,+PaخϛN6𖔲`̮Ge?v IDATP @{g ؝%YG`u/ftj 04#>] \Z.޲{g!* Y[b yvyzEuO׮uB+eA;.'euDJd/Vo?'F s I"w fz`H3(倬98[cX(Ɲ$@t5h`l ZE&Z-/M=dt0 r6dz,=)`) mámݙbK]cY'p;ZVY6?-o(ٻe(EOF0hT\NIhf`kʪ2mQC󓹭= /.?=FMaEjg )Nh[vAAaP3`xbj= q݆+3m+3x 8$ {] 엶8)inr;qw\StKlb*A7 =+ b=\ZT0E`Vig0nОoUjdՃ #NqwBkFD]ևBlz|lA$RAˀgD3֛}ÇEKi<FrrVTܜ7ʹn WMՉu,իWT<`rN92#9ټU"ueeTԑݨ>(2b,U V>2b`@oҚ x߹-„sґ|xir~e /pJi1䶱mjͣ)0)}ԿK sx:*w{#m$t&h\Z#B&`RYպ'p#1*[Vc'*BgdIxᛪaDИ)5V$ߡI|Y.I܀2@pBc%d~-d6& [{t> .x)[:85]|#ogPh5ӒL2([ #jJsŞ\g7dn{p$FC6 63/Zl5E(y$1s\fECЯF6E Ek %Hy-^dneyR eϔ{a#ᵒ=:ȶ 1@ W=.OZK),Ns3ӵE&S<1Z i2)csQJb֏*]+jW;dB;=טKdbizywZ;ː :k^YcrouHo Rroܮ=&DA/D5<'{GT]|Hys识E,f1I0 ¨=F)yf ըܞtFqA%֔[4B:[]+2`Jබ">`BҮȂ&ȯ(o4Wr0 )Sҟv p^q[_<ǀ\@9˭ޜչ"|>ީ(Z |7 :ڰ` Dzn*Jr`ŚbsߟT}G^a>.ށٕd@FZd wP&meW~;@dzb<x CbgN>0"AQHzV#)#̢^yٮ|W\,Gh 3$izU`:Em(%sV0՜[f1}ߊ16g-s@eMBJ1s!X&j\mukjE;yZV'6 mly*Bh#U29En5uIh~6T5' C9y i= Dພ9h͖cj v5~t822S}3qR3\c'M2~*#N)SH?V}#x͐aAKK )#P1WaCBѼ7oXX,1NZRŶ%Ta uy/U-(^hAATA~K mREA3 LF!Cm\Ϙ3D1 M8>O{`QɪjI򝦧 Rtwx_ϿNxlTC,1Q*H1.dѳҋ<kf[=;_hʀY'FSđe'OxgDeܙ̹)Ӣ-t}ґس5֘!snWCک<. ,&a-*GrΜ'"舋%akG@!4Y:$QU1[;J ;:x6YV1*\ ܦeh=|#U"Zk[mcx^GlIN%yRkv]Ma9HK&1(R).&|FOBbb e[7J:B2|BsQ &hH"Qx+T/k)>4'&9bǣ5IKz 98c, #O[K5^7*[UqSۙ1̂ި+%1`fo1Stwdb,T3Ƚ6zcc'5Q3eEXՃGY5P̙>x-EpT9!w ]\(ܧqrd@֡k-[1Aff=F<3{'7~V2@UFavf612p^bOgnLnʵuKNݖI#86TuAFೈ+Tν&PkD7&֥/60nݟПTbk̡5ԑqVge;|Þo&%7XmRDoϕ2T5Ҽm1t3!UzEd\]S@SK#*5(5(KÔIp*08,8ixȰ1uoбig+BTfZ VǦV ~Cm*̧lzf78w PUl.ӑ';i`OI%{t֍[K,N1W'9c뫈7u/gxQckJcD-"ӠK>ȳň:9in( KB/_<̡hDd^1&hDÄ.KMYu]U̷EϮDK0l, k=e^j Y'seɎZ:ƈo3knZt`BCnqoE;ă_~b _:Y7c CN&mQTB9U&R{CϹ  nB'KP2@Abd'O'gkqng4!HlTz Uz>xSor6@C$>Bod<WCÔ *̬PYmx JtN1(6sgf6VQPg>Nn]0.&|]C 9({;gZZ fTIo+z?p1:=± 6ZV?K*{AK|zTbf\FٜNk.j~e@o{LMe3(60<;Ɩ˹Q&|BQ1;ޅY&9ݞ4g+߂+1aE'$`X"=` u01­{'BE ]u{9sX H*,8g6bHRjMRHFY;saԩڵ*,աXHrS,: K;$rlJs: NHrW: !ȎamYX”[t-,Neւe~wk}gqTY*ѬPL1gN}MvK<0q}nk=rˎf4;N(б]~'bq;cտEg.18Wl}NGP9_@:r P,sʚSo)<ފF| mlyp*];mLYͨFScp+ÓE-*#ߟEs?'=*$-5r mIsJ۲6٢ㄬp T~DHUі|7ԩ P&?"$F_&7x1I*-/&OpN.w2b&P #0c2T0",䰀;v/2m쀴q1C@ -4c)E)h>>> ;ڗt^=ࡕ%t%+/(u6ړzXe~ƌjˆIJBi*DN"'ʊe!fϫu\d+>w h k0e=;hVi+HqG--}RYV| - -E`Pn3i*^8 yM8$ɍ6Mk$Cf{E{]t.Ɏ(w˗@PZ=Ebd(e; [wW4g zhժ:`[7@#26Ό#](]V#^SWjj(B$UBE+;>g9Hvmlc/k&Q|7V/2P>0a8]3NEdTSS./#ZA7,0Yakcb$M"Z>W,@Ÿ\px|{io0X j"M0ZWմl"B1+҄q2RP6GW3%!6}XIסi $jЪ-XN#WNfm!Y*ݮnL x0W cp`E?¶RB@ʬuuSwC5z#c_1ZVSK'3XXIP5Qd]10˖sd-@VvҾ3 ybYffmjLDfzP ,(EH?!U\MS8@9K\cјӡ\ 8MT\$ht4.(L >ͮqH-d4vSX^cMojk^MT-fY5[l[o չSXQ v\`}96@nƳBJCM+M:;k"7߻qj̫J=V$`1rhIpe'pܶ>S@1{"3h zI ީsJs7 4" 70²;15@Lj!p }…ox 84Qjɣ_RjCME嬈T#ákwOnf⥄-Nwи;YE:XPz~2EYioi_J%:diZAD*KE6TRJwS-߸"d7w4NSFvy/ǽa2xr9isVBb<2=Jr.B0阚o%P4@+@Yq3j d3)ʭ+xc\42x%Y։f~ip̂4 w8n1X%vD~2VDc6j&E;)~NV[9Ucc_'}b]X,ٵe3L 'lQGfrFF:^\E ;\ gwgRmlcȉǂ}r ~\_;b5Y6l V2Vwtz>%3rYxkLou{qe^FDD]yRX]'SbZqdY ӛeNl%?a|Xvek'{4 IDAT̶H%ja** sTQ׫=ד]v4uhVeq։}'XסɹcuśY_ z p7Xڍ\=+Mrl\Đ $!|g#O^̎-FȺ?`f6 GfYUw'y T'\řQMѻ0(!yuV$;v^U`+Xc4xԚF?0Mct4/Na*,]Hwೃ*BO}mcx.'ۯ2U͇!G2C ~\?}>J;pb{\Oc<|f׆kQL_&e 1D8+YM{5JYQNn =,X=jpPm9Ɇ{ XMh9>֖⹍+ ۋRru/i À/ٽW/{pGZo@H.]E]yvhh2gsy91d`Я/OG~AIޡB0noiY63x{(rL{}K̢cxIab7 3 yMj@u<8&I9sovZ^d% mup_ 4Flo+ڒYm\f@V\ar'Xο… *>U}+B'=PV੏i>7E_HV'Iymi% ?fKyv7Sk׮o;>~ oؽo⻿ko^lc-#kA&4 k7l1۶jŚ,;MC&nQ3rn.b?L,lp͸:e ՆYP;o)]qeU豨 ;dx;uw'pO3SIc{ߪ{PHU,2LaK 2 hBby'əZr|XNaoqX]qL~3N* _@yn꛲ HZVdX5KKKh cTTFxF؍}.1Jo90̉qc7}0wP5vGtҜ4˴rVxUa!4dql2z>%g_x[45zR am (J@/ G %(5`Jȅ̕{"* & LeOVlFPXGet>5/MQA9wM0Xp~7eV0 xd9ڋmb{h =|!FjYX JպMh2*bkoԥ늒XJ r;`1Đ> 8j Sl~B c /ˁ4FvJ%֍*w\hTDtqU@~u_ux{ߋO|җozӛ@<&x~+߈_}7??_Ώg1/ڷ[ށ}buuDaa^O|ORgHctW|ߖ3e:B^38 |jj ʛjG1u,i"kI,"%%EX.;\ &[%n "ȫNաp/My|ŕ?wxAl4 ee5r@[Lw䚌]nye.0=0u" 9nxb|#g۸}Ӥ b8GqX06 WDe ,U<$j@)EQ,&`tX i(1v=%6BpkBS37` $7҈#*w(dԴvCN`7{ ^5~[U; ze)T~; 6\̲.N~Gex#A2\(`lc0kNE$ {٦ռvd͵ϫVOzJ - S+,#+(Y|u[&|WaKe xF''kŢ .\@J;蒬[E(,uԹ7<}&.7 {>_-)\ ?C.tP` [tN \ \ Bpa93M*(BgYYY/'4)0z*ϔB)1AdWj zCJG q_{-\&rv*6Eo7Ȅ>)k|WUkG`2P̉fQ2N\|TQ-RlhXf@},s6?OY>kf]1^wr2~W<^Wݻ}0=!uԯ<+_u/mlȖRCYUPi P5tXӉ{[VRn7ziu@x`Ngd&BLU 8] VDZ&yXXF{`Z{|/ Q׊,{ի:mm50Ƣ6)ƈ't\3@<۸8b6,5 g0l98vM%e>e߾_QmB#zާތ6`Sg+>$'Oԟ6u5pLL^fT*HzpyٰN`"FgΕu=ܖclW.VAh.f6~uaݰ&`l8Xr/aOR{\6e?g})zf̄QbMf\?!쵒)pf6pYufۮXTQVZNذThk$=zfۜq{s\`OpKث.YE#~dV:%ۤ#u/Cc|2Z)=MƤ&tZfK Q!_gI {?Y8(ĒKX[Y¦,_ Rv3@cd!>P4ڃ<.Xl7n, @`r g sGW(fcEfzP()JK t%3os`f,*k`&gndKa}E+z=̪HNS^ xMx<_!?|3^{o=]_i(x= _Q>GG>]oöu>(}ѯq{0>?_ 7O >GG?[nX`TV)3}QS$w{*n4ۤHSYh, w|P_b=ttu3 gc™RW5yWzېTF5q:1u=ώO¥P״q >a%1(ߢޭ<2EdFbm XsV909ޜnZS+4eȶN%'d/Hådme1d\Du+:^&qŸ60ZGHDw8;WΦ[iqgp;\p1 ]U<0\}]Wyspp`-,P R )ugD#VK`جY eFRԗ=u84H`1 g{eᡳ5qy`ZE?2w@9nnU\v >zɞWIwFXYktλ8=DW/#7\#p 7?x߇n S_ok8>C|>~A/ VBj*Tװ``+!a \O!? /OcρsK^r`)|gކٟ|_z9w۾xi_|7b?em|A!9J P]qY\*x_EHjM\=U1;дg5o(trE*&BV ,e1C?4߂Y#(wض WHQ+cFJ-Ιڿ3RM05&ϼ`Ǒ/2!R)G ZRfm<00S%F\bĀ=؁׈6sppv` ow \@xrO7=&@A H0 `XYVSI1'SvZyѬߣ5]#IAARA#nsOGm<% %=pPtm!$)(i't3/ĥR]t@ )1KEBQ?7 Nph-=0p^YƮb&`hi#fhׯA@C]aYG=DE ðo` 4ZDqC֍|= ZKw!KuQ2B6**J:OhJ7]&w&>~mgDoeJZBꐮzn8L Aƃ!>fap{1՗w?ӿQ<?Sx^z?{\mᗿ~y>`"͵**Ij,ejCCBmu0!齰P gK  'V;*i<zD!.\ Ϻpciy̡ M[YqĶj=zWK[ -2 UlϯU [|'ֶGQf=nvqvU ́kZ7`i$mw^nɽ9bh;Xp@ӆہۡ-;Ompmm6{ x^~bǀ= F 4~زH.QU>8 ye nHlT!HL0ͦ&@qIP`UD =>w ˆ=fcVĒQkv5yή,!qi*%˾C~g]<NOsלh f4=Ũ`X ojۗ\hZ P3T ϻ.QUEM13(1kYEN Z?ѢdGbNWcXt\l q|P59@Pjj/=WH)cbR4@pV.C&aR+\p,U0 :1wUO'ltDɳ^Q >Oq߽v~g~??ooz34e/.>}w?~?}3ߍ7#x~b {ыk IDATo5\t e*.sG'+镐}S8gCu tB ]pkW3kN*iI%ǒE " E ؞t:W1Au Sfe %2<סńA!~FZ"'@YX٭n 'sTegϱw<\XkZF1MEuHl;͜&ĉE]Uݞ)2/lEz΃ux3ܒiM4У:,{5V ǁ:]F /qSAgp~2 ;h2b=npc$vy mAWvNKx1^sK'.[G$tk^{K\#.{ d#|DcʹjMd:7XTκuݡesPa_/.ӊ N۳eqٍJ;gB׈qK'`dԡ:yʷNn^MN17\ 1z 461ӪW=Uh^$,tNOlEcQ6LJ7 R,Nk\̬}ę18;>D<柟M-t-6'7 β/-pc ?ϒ$rFD*ӚkMHGu*Y bne~gzDU5k3&W(e!$c zKZp{~~Yl?/@Uz~'~b.O|< O͛ ޗ&>?N\7v D:n >4{7h/$8+ts#NXTn{_U&8'8ȱ$t`ʓ$\ ,W){xˮ\k}NSSM$h` BԠP.`^0 ^+rŇW0pը`P0`14 P@TJ%՝:{9c1>o/~@RunV35;ye'K2S՘V&xtmj ;5U(Dh$S! q`#~l~|zV)VQp>e Ye> CX X =",,Aw4`0Xb85 y'pN{w!X-VbzVQ$P3Hc"VNpPDY%jDOUݞ1'#\h#6bq@HY%RһdwfDDL 8G%ba`*%Ii3I0;B'-<41Gňv%F)-PlYϐ m=Bb̭D-b5%!5O ,VMPb/YwAر@1r XbN:F\8 0ۆﳻps8k^w pߏb/nzsRfgLMf))[V2yȚ$zŤJ/E (Yu!M/άH Bh&d$ϝd;-((d/— rȺi娢⠂d齶\8^OA{AV d=?UtoP-Sw\ Ll"M o6䏛5Gi*@I2"P0a*f:lF>s<$ѧ 0'Ϝx=S75@4dn jyp >_@)5^DuS s͡$-pqa.#<0֎9+5h]{׎tRL,R}ge} 0 0`.` *)lNkkE@ gBxB#bUn찐ۍ4zRt&]\H ƾDȘEdHd%1~vm55IԤZEEXm^m:| Gϗeno) dSjzwkoY q/e:dVM#^;Xc:U[x. ћ& 4Di<'F.!c#T޷H72CvA'5`\bvM˿4(|Pr&zɔյ=X}Fg.8zKnڍbva;v.W=މ]{~_\˟i/`gy@7o.s \}1ܰY g .߉wbNۆB#CLq,O .NV *StCo"X*# wS1 eSgNP9iX,eķ`&QoLx+WgaVXhN`ׇ/͌@ ]@jMIEHHEXu6L@Ch [LUc+Qed?vC&:{Xh1&HTZ$IGCMD2{ ~zu]7RG7A$k"6p 'fxhϽ)Yxǰ^;aq0p; Ff{3[ N3'OWV7_JMs 1DŽ}kn NP3}d>\SI-Ǚy2PCFGX?e 1Q8xM^@lEdS?H{O>|ڧYY%5Ta@} ePxdPy .\;|Hl@DNy]%.5ƱIeCgOb(kT'RF3`6+RW U{ s ($3VP+N#n%[1Vmw_(`dju -Ӟ V4&b.Ɂo^ʒ~R(z '/|gO}*obüWʅػq#Y`xp/^5׼gs6]U,<ܧُGao)g͏`|<1!m7ވ7W2'o랷Ol ٍke^\k/ĥ 򇑅,Մ#WBY{NE̵Y)R'iPE*Qs&L"N&4wğEZٜ3d3i*!ʻ:Z 0GF1ϗ~,\^/Ǒq(`P̍бVq'i^HӴ(Jc."SSkN):EI> f  EɗnL y dMB"kڇ+RDfluY4@?+H3Ɯc9,1D[E";R&a}N0w8zU֚8 YU)D&D#1$ZX+BӹVjy) M+V]y%UER'#K6mD=Idp#z@Q0R?H&1S0(PqOi(P79]fє{mW;'عpKm);270s=Hb&"JЍ$CIPo#r9+:3tT1A3s`SQNJ#G ଳ3\K_«_jtMSczLcϞ=8y6x9pJFnA1cߺQG:>eĥnUnnm%KfefeqI.D\Uo fte!2)qƸ 9sDCȋ~3 ؆&16_ )Eu&4HLb)Xhb`6]LC|"Դ+PƜx%^T-aρ U\sTZ+o׬b Yc}6M8qXw"p66<9'x3_G|qct%,!Vcr gl`4#;BE v 34}5߅ڒ0 > R Sw)50)&;(] d^y{ݜ֍G؛B3EQXj̠3`6X^Dc.=ɟ6-3u3z.KۑϳUf,9g/T3sH ɩ}G:#͔$BBXHC2`(^2ot8۞1^a0FS+ o01 c.p4!/{y!|/ O؁s.|2>kPj81ON@!2{-pY.@,2PKwl}8Rzq{lƧ&ɷ,:P"pY1>Tc(+ߖlm$8Ģ]oA #{l㔙8s݉x1?+"|7(Fq(0cȩ@AĨ!]v(J@ DRJ|/ =cze录sqCbk7'o9.ŐI+a3&!1R\&K2iXQ*3yHBg0V'w;P#Q]/w8W7KF(<I,gmv!y~B4xgwط3)89j v}ፔUݥh"HAb#XZ]992뒗mTX7oh Z40u[gG*ȋX WF;)czLjМ5M9 i`cfr2=ح 3'pܭja TyXO9RDE2wlA$Ql_Ec:u*iSq(PIG%d!jө/ls q`WC _ōK"Vx!\@-(jȱ#-Vp8}#;>/r80># p.ǧ>gyq\4Ɋq}\(; d3elSg``=fUJN-[.s͚OVM`#ho҆E"YnZq#kRyY<8;: ]{g˄^-ܟi[ pxGaMܛ\641>Y񢛣ft3Y([SMZ=lIb@}P&+H-tI^&jO1I8 M>*vX]5CLJr}//z,.]ٵQ')ڑ:o ĕjBM^k*_,fr(ǂ2{T )ǢNUGX MK'hG)D AAX\},}Ϗℙ6@>8cz<ZoCr+I{V4тZ?BBIDj9u@~ET6ZGfJ&^J Τ41&6,laL;*썴O>Pc؜c.)%K$d` rr.6nzۨhqd2Uk7$J)xEB8udS?)kMh EsU&sT.0>=,W=oM> 80Žwk A^DM 4p͵fN.Y]0+aut?_*n8x=⣿8/~qbyEE*f0X c,UWq(,@ch/a IDAT80>{W\KZJkdJ4 ;,-N Y|_8wa׏FEBܦ:E V혅%|\x=gl?7G 5,<<JV >rlDow hll f>:U!$:C=>sef3%(G} _GdY''2a[8[̓3Yɟm4|bU9,w,&]93%`mD+y85$d-i,"]K٢%*JdvRb}n^Am`G3(+t=S昜Xs,]0d^rDAlvl:Vxc5kwC'ncf= O)U T؏mP>Gb)lC-t}l_ 9r?q'܊bSIQl0rl4噣z5nЛ ?`!^BD hZVӠ%UUK HxxfP1UDծ3#6l D͟V[o=lly C1 U܆lzs_:lb}olJGOH\&;TsKPI]rx>Ʌ8:,MLug'a{Psx NP&NP$"C1`~FIf"} qSw4^.!xJlTSRۀ@lIJ7 Ў@)ˆ ^G 9a䄉t?䨔;*7Hݷ]+PhP؂vN9L)rM(tX$@KW!&S˦FRwFDIϫQkT?3˦dZg4Smkk(Rg?uwL1=7 LI0܃5k! :0/ X!6z${eFÀ{55LGi|D œZd a LY9Я)(qT:mr1U`+(TgX[5(>`*+t|'nǾ.Vrq(eJí>`S ہ3=g eM9$a^g,]@`ӡΟGG^vcz<ógӸN^$c)Gj-#+Y2v 8k7CJ(PДe+h'[L @Qnnl O;ͥ/(sUry ڜ)*qY6EM5N;>甴W.I 2)Yl9* ^:u-{+Y5kGwφS(EՇ OHu_ZLYzΥa3m%G&&x$j6hXzS9oc.=V10bQ1 os^䡈<Wl$eij}^o'?t-^ڗ!jQtqć5 &19AGoײEo[0AQF7,M˔H&MI^:`H&cK9xoN}xn#.pXŁ}c>WÃ7Zġ|Ec8ߒVޏ3Y34,N݌M zSz㤍`,a 0zʩ4j|ЉBH1ZFؘ G՞ ӌl. `G*o8ݸ9 `QR:FIơ0m&a1FKyAu$/2%P;PCtbǮZtRFwd>/9·]SyV)b5Z,"K}T,SmANlNdR_YNh T wzX|ζŒ#}ꉒ̆QBDYգd />[phA.}qSOġ{W2V \??l62&yfg.#0P8@&>v*0'`X9Ɔs6$!gD}(,{E(_#-a1TaG}q D-8!1a8_ێyf!aMG"CxDG +TM-)*Tdj,BtةbU|XCT;ͷfN&oZDäk*(ۉ/[@QȨX B0Igrg0cY wmRҚli]F iW"#aE`-$CxqdmY#f׫.Ʉڽ}{E[v5eڽ3B ܴaLξ? 1=ׁNyi6>61[Ώ#tG͓u^LT3Xrc&f_6EL0UYMIkeLĈ-;)طmyMgTO3'grnC ߪ y8Q3Ӝv$lPRC2=ͽگ1SYbX},QI(ߴyN%D#Cz衇^GVm=g2XMi{]8znr'׉DcڴV2 @!g IRdF?#`Jir$ւ#noŋ_ z8 V W<䑧zM6zֆGx /z%9Uz ՝H!6G!_Ý㖔b#J -q ]"%XQrв8 \Q;~%?~zߵQ( a|ף}+C)VVXb)ZZs%(W`.1bn0xLr{=u@%\b ,+35a?ǎM&Mxt7T!d&zeYފ:X3(=C#fj'#eGƭۂ|\t='e&ýB6d; O!«l[rY25@Νm%J _)WbYL; QŐ%m hN 0jiSwCG+5@TM&F:,ݣN1=ׅVy T1!g td8G!HؚfOX 5IK,8PTKIQ p*z1eQ㱎LOnoؓ"&S8] Al\=#L¨6eQLF`evY!"Ia^J`3qTF4JvA<ٚ)3Y; $)?`Ye8.ź6`t~j%H{j<iuo1|g$P xx$f54,l&6}*ս\3Xx~d0 )i[X1CHJ%;7/,Lcޔf+P7h$n(zsY=ʳ2/:ex,,WBv'6Q? `7[7W%pv.*<3'=V&kZKVWp bA1R:MZUŚN+Mb*jI0԰v+Ų IDATu?'?{v{1%6m/jɂ c +!cc\1 PpP`9q _{~{=ۆpX P2 H-&L. Y.D~a=q8a~(š٭G@|(v<$_{߲3( HRFJIزBzqQkQob{Dy']zx9UE5M]Q`qm92D*Pk(Z\.z!d0I\;=J($y}]8]|,0OVFz߀yܓyPlT6%8[]uόW&\x օt-)=LR 듖Nz1L,w`QW] e5ϰy(BFeN%_c9~E $]M5FRϕ mȆ&2bVƦSbs %X.NyI/ɹ/}>wwsne0OpRyWqy\B .Sss yb>߹ո #_Ɨ=C8/!Gˍ.? VoHzi >z؟Ó6< ?I8quԯ+0]#{}M+;rBJ#[bo&nj( [k8FPSjc6nRuZ_7u;;K-APe",-vcԟ\6 V -URj9fPKG .ycf7:q+"pѸ(I-K[0 Vs9$H>qpJqC*Cb)͜rczL2\xk% `< fQ`&2&E Ƈm%vP|?ߴēa4XGnѡfal?ou"[[ru"&I6ГI}2^ktP[[SmK1{?Qc` Ƈtɭ7%j' O^ꭷ.xEG# Kb|mM0*;ɊQN*,ơJ 4d 8sV,U.?FKBJൕaYd,σJ ũPn oyobjH٤]{ۗn'Dznj ^<|>{uqP0i=>֐E(0ujw`T¡b}cwaYMWW7^b+a\|b (ɫKTr XO=$:;iΠL%q(9F3R U:5M6O2`# &t_762@x`u[5uzh (@Dݏ^E!$'gAfB4T9N"d*q A[g`rD`sl:kC6L舘]Е~ZxN"Nk͞lnrlPj<*)x9?8cz<$c}tMvv'8K|YvL208vd -2iM=k&Ԃ|b#i6>ftF:&de4Qd-pRl;4g⊵Sru2f%% 2MvpVyiX@9~Mq;FZǎىmܶU1C2FU+~bV>%Jg aQVWNY04˴7.5;vl|>nvde$rٗ+bV&^H g6樽ffȇR_ƇX>X4gOSW kJࢂߴi?WJNs.Py# bi'`/p8_bPI_CE>\f涹wYyE@Ѭhx wDR0qFB&Z3Iⵠ|mNQ%)kU9> knI%:ۜ=p%~vNjΆ"yw~#C!| 2*+ɀep9D:D"']THOHfÃ#8#纘0ɐ;z%_3BʤS l*7q{|~i~f Zf 'jk2_:h$0Lw z2Ak)smo͘ 0U vYc<!dUsPfN)?YxSf$չaYQy$U#Q%6ŨPQ$Kq>Wb ܖ,OOj, !=9"qA+?\ldnVk (*8kR l޶Z6+xҽ7)^!Nw-Kxg?' 9YOĩ iN^v&P3`Y^sLn aQ)aBb/ju𓯺oݾeQ5Z3sxc3{d>BQOw=V9r7`X|G X # ,`5Si nWVŁ| "83,o=)}PKaS;ݧ5S%k˒)L>_kz3i0]*.{S U`-^\Rc( `8t`i^<3CJX(L]N7Y;Xd%jF]G@$-'t9~D[achЦsSg4Cx`C 0`Jf1f֒_đP{nZ jβ[$7[XXk^ ~N)A$oE&ݔd-cL7ۼi:f-f0N2뒸N#%YKPݷ"ƢN%9K*Tڽy~f#DD]t(YPIvş3ab7n!;jJaGqްi&Y5LƤw\7U*Ry`^+3?n/ʚ$Cw%$l}7Qx[O}+RG0MZF;<WxeoDž/ o̽ĞLCƩ_pu-!:6 IC$C&Ns}7.'=ҌeB#/g?\M[vAL{Iξ]N~v~ `g\;P`5r>zKab0j1R5Y *)Jgoh 2aCαX Xfs3/5!FGCmI8n0֞V5N}?xYBpY{RnZR SҒ}}`T8ȡCV>Lڮ:s]G [7 ;N´WqT@kݗC4271ݱ)QKoRg$Cy-{ȐQ,#t鰀HU@ԭΦp.BքЩΩ3)吊'α Ƚ/>/Ʃ-g\ ;JٳWogݺ;bXOq|g`NZ>I΅.`!- +w/ޏ:@(R餘qmcb?S%R G8`@ e%*d'"Dr,'mSDPLQSng Y ꉟ'.}Ix 60}KťOtt#8ZB*%Ug,iJ3pTVr3cN*r!HlX QnK&;ʤT)Jt+H%8X >kx/#ZCs]fˏξD?mT \os3!X_'h j'aCS[lR ;=|;x-,,/UW]W-c8`v`VaOoz˛pI9,ތK—Sgz|{@ri/o8JfW4(:i8"IגR6NpD,4efNd ʂ`o,ybݵٗKZQDXLİ0Y,AtXK2[̍NH=T{[D]rRoˊP"Ba-[xϒ[ذq+mo#Puo1r&몀'ʼnUg**X`d2XMY|ʒS`Rr!SkxU0@5UsK9.?ҋ[9j@xWnŶ'Nl.Ttd-xP İCr[tj;Ή}L-7 'B XA"SKr*8G_B# GWT+|:DnûrʳY}rȫx+/{*o9r3x #`\֚.`a#qf`W&-SL*lD N7Ss u /H@ YQʌjN2=J3Ȣ_r+ -0V@ؔr+ Ƭ9QमgDkE!9voOuH1l}l0ـlVDB̻UJ Ӏ&szI}S2N-\1Gg e}l 6|oʒ.I (91)ʏ"Ryf6GʈEYډ䁧$- bIǺk_Z,,,.Ýwމ ^x᷄%7s3o;'z6^|mÎӶy7!LѸpX/ LY)D& D/Q4{ f$5#A]oһEq*ZF^hNoᲖ!,14YEl$~4Jv=U.̐%Ir ĉůFRFeZ]c{q Ï%g0Rj#k.>!I8`[jL!dx]jneӢ_[2=j/9 V>^KR&/Om9T3D{ 5\sv}8wЂ8&F~n쏁Yf.Eg,K*\lY/ωz{Q{]5S?DxGƒPK nO.-k~㹯)FC,kL$~9nr)]-VWo#d5Csv<KK'Z22ssȐ C`ej%_!;srF= ΐeYY>ziYb _(ysY39Y]GM]Nu]G?O0DQAw޵8J(V+'PKriɁS/C`Q0 8uڔbJs'b?\~a]6ۂ>ixЛAY4dJߪj'JN^k0u6Q~#;g638%˒SO=^x!*xس+_yb NrB/H=_ޛ4#ݙؙI9Qʟmx`-"X5^|O !fP{%6LrJMȌ;' Z4󤮢ڰ9e33બۦC0.( x4o|H>qJW֔ `F$Tqmv]{ nwI!c")IX&+G8D[DޫEYdƂؚF2 fa15L8g&4x% `rvb2"tIpgX7| f3+cc Z7$)Ci=,=8^?6f ?%6 jS{Ǡ$x祋yNmR T_zƮcf9Y@.2\xᅸKa\q߰fwU!vfझ8 zر oލCXڏx{Ʈ#۰yÛ^zv",g<;{W\{7n߉?ll[ڍj\CNjM)Tbgqa- ^1ٳ.:\tF{]uqyz3e睇\w]8td&MđR? }󿆣("ybpk**r^8KP&\Rq,21\"jD紱'^u"E[5* 5@tO c+ 8=D7f{!Dz&}(6ۓR Z@+NȂ1*c#I>kgRPjK.z9J?d 7v4BL›z\Fsw6aht,F^fF޺odz_\ZIgl IDAT-̕$1,{y>i ?ſr FahRVף< er^=6c+wV56 У89k$2.\o5 0JƓe0g.7#F, {'sR\Ixw}wom)C\Ÿ4CK >킩4R e)L-Ǔ ;x%r v/*qpNhh cMML_LD(ъD:CBR]|׈M.9(K'0 J$Jj^cW*^hmi_kU^RjC7NbiF@T $ $ 99s{Lw${$gX{ y~?{m޼^x!.JW9bwgGߞ_ڄx;g?m=Yk[cx/OuEWc7a;v`OoWӒߌKޱWޞ%l|/tΞނ-a,gEqc6Tv ߁(ir#%hѤ3xA&Ɖ UĔWbJ\wivɪ&OmЀكZ0qd@ϷI@RɋLVk;׉i]G$:ԔT\bRS&\^Qdᔘ=+DR0'8Ҁn*ІR%&ȑ_4|%VG9y6>ɘZ>'!7]RE$Zٿ<<1May6%8+ZKK)tZ:gQNm\sۼj&`<;bHCmAɂZZ:Xњ‰8s,N‰-l n3N!~c΃Rrc nT׿{COv5gHI{I}= ~-8`8ul+/Q-9iuCLb!#mݏ%PNB͌>+?j`V2Tv tz]۶ Jl_i!X n\x̒%/`)%'z\ѻwY'"* MXͻ8w˳BcfgQgxa4Tl ;TnlCtJ`I{q̟6 W6oތAk\=aҒKW};vw֯ǚ5`әؽu+tw`˵{7u'bÙ'RN8]uص}@'=Zv$X]Ә^ӘY`vӿ=Uۊu,SoEI(Zt|B,HUmK"}ٰt(4ʽG|E25ZlK EXG1^5QJGVΠ,86=()B8fBF T9G_301iUhRVS?RQVĐji!Ѐ0G"`Q4gI e,Ad5q,GlFŦ"Nzl5 S6#v(ۗe{,{(D$a-xDHqCksQp[_?b,wՊH@i}qOو2YV1XZ4PmjC؜FEw֏'$t"}HJ[]Th!CXu>]~^[zNZUBf5>]?j=ȭˣ xd,181g9~!󺷠Ӛuz-t$/G);kuu\9@[-%gܴ蚕cNzf2 yyڽ .ލ|,[9UUŰfMu6"%9O_+y&:kNZ;w(N PyܶϹ-ٹCR?G^2X' J<T0$FaW:F)Z/|/UCqZ]wJlQnD!VI֘.E[s]76Gr$Ԥ?e@dm޼Y}CPW6n܈7Im۰o9=;qVi9=Ms ž|3{ .wc۾n'&|[xëz+fO? &795$3ș[m*̝9\Ն9ژZ:Ė^$`A¦7Ӳ+m}`c9@)}JC(Rgk% )W&)ЊiD6R3M0qHXs,w-Dr!=S٪RD FaXຕll7=J8q_X!HuD\EP c 3ֽNd ϏgM@8`]:ARB2IqF Z n*Y$N{J@ P9,@#V *g`ar [ F nz7臹YGㄓN-x=jVCLư 򱫰:xKN=[`ǴaIA2Pg2 p ɽgO!hҤ3Z#nC<,ۺ|2#)Rrl`f.~݋3Y6m_V՘(#[`djt`6#z>D[Yݢ'C&Qo*ÒzRv==֌0[Ҫ=L- `58 x8l"]KBrwZ^6 $ c j[4 ٪*KMXX-*kC0^Pku{43\ZD(8dZ(,DIĹ9-f^ !S[4@0qR5zD gM{T)Xc)rOuKEkNDa#+OmQ[C'Ǫmx7g=閺Nk S˧IHH+14$a# @q붛нa/\,_z Va:h!kv1uEB|Mg9\(܋0(~AEcĕ5) Zxؒ0M8|j*ޑiГN\r%~̓W?%Xz-={3Zk4p_󕸿7Se1z#ְaX8I`p&€f6'}=&1O>rU@AwFp̉9#-*Ԭلt5u}La, p  OTMH'}_4{~97wFc:';J.堺۱=xk0=Cov87olDl8?UDf|d[W<{c \jvV VBe8 a,Gv֘ Nn$ܣtL:~'Kl'OE ٔ +μP"xsɏ!3G"V=hmQk8 ^GYS`cpr?fbӮa9GA5/u)z4 (7$3l@ ƄHIĹH-Xu"fJpAshNHF$T6Za;kp:!7tUʭ ]%;kL)Q7`!17[ dלI4RP6,Kr#\ك2\p$!]: VPJ6+*nϚ" %> `\ҋ18W@ jL)Rkjٷ0.bu߻n'@sfvH*P-Xq3"@`~ VyV )n*Z/dC31,Qm]a0w3nڌoNj0_,"b(8;oE~< q&12?}_s_}߾Ij&'yk76؛`sTM1@a5Hwk p/q 䪹ɞbf.ZTQw%lsOG1K:ek( ,bL̬y&3;iX^-ΦF]=5&[;}b)!h~"|8qS,^8R1%p?u]X\bꔇTLl֭[wT7]zV\yĿ{u .`8x3l'صku|̉ -ImFV~U.>˗L|hxáD.7S0K5ݕ(\N@#bIQ2&—7 @V›BH]'^%3 Qi:AqB,O-=R7Sz8:H\@O1R_RѥᐉQc@:g!rr5DO`{*?53(K򫓀ZosA=@`CjY T^_3^mwM5[Pů\s;pƣ_֬AqqbfMi`y{Z˱zjl ~7v\?w3p`~8;Yi'.Yb鄜 쯺z)]Lg,˦0Zc1Z#s=<.9}h+1mkt5/^d, ^.\H46ܜQEC5Z]^FKOJ (r}0вXoIxjk2dDȅ%Q~ ߣJ@4/3GH^Jx*5F #ƚ eNQXWb#"#J5PӂJV'q}ш4'?2k`=HYXJ>|Uzr]wM7⓿wlw3란M9ݯWL{J p !Uwg62NY][]MP0c' 11W[6iy>?27ɹyo:\U}ۥڱdӘ(j͐f],J0H.r1V2/7~ "pJBe*xug.!spu dG|(JNT% m!&FL1M%h"' oVRH%@MW<3'jA}^MqĆd `-Le<$8dbMhvz#Q., _إnuRߝ]8񴓰;:3'ˀĜH9 t9(_:Sq湇~GQh:CsWyO]+ʰ!Q`,2ɡrϠZsMncO9G򂱘qկ3 ]x ck=WΜwnF'kk{b/'aPm/ຫ9YVԞI*.l͐  lRY~6&*Z?-@A\MΫ[ Hס}.eq"rs6d(r]7<,$Ղj#191HbBaV(F,_-P*N`ӿXK~o^QS\r?\s~QfEU[ST$gЎl* C*t^oBPHfrF9 ^(&qatvRɵܸ̾qWT@kplU_0 i]<`MBC>1rMtq,$I 3gO[AS]bպAs,ET'50?+XxQ2f<5^D %US=Z3;H̩p6fN:M5 (k{0P9 &1ɉId)?>[d?x,0=5~TӠr0CZuKZ<"Ȭd]ePHmOvC}]P2S|‰DĚF4?mn~F(n-ϑ%ڝ9Q<}dZm,Y%en9L)4 bOzJ!3r*UL.R96T' TZ36[ 7,Q+k8gCl*(.dlANlUf<겦 #<"k5Z ȔK U}(#/'dgǞjR`G8,'@0vƖ]'tdȰo8ej #$oω+Xp,j-[,"/p*$|>#L6#>SJ?),w*gt!y(6fѱMg? ~DL𬗜k>'fYy7o ij BF)Ȧ0uB6!p7:$nub&61* W$=oeJ)Y/TM]88)YB -Тl6vfanJ)\W=`п9(4pMri/A8pݬ>M,:!](C|WVHv.Ny'I.9p>N>E-'̱|ǻI5@3LqT%"y,<: 79&)rG!2B-dǍ&$k%6i̓6w@aUReJx<v]$_AY C6(ya2gݑu8L@ܿLuPk6\yd1@aIh oo)P3d``ָ9HV*Ѥ Ը))=L}%I2wbl ^-l2J5Pto Ațxp,4eR2P cjHOPƪU~&o*B^q@Ta܍_WGV,2v l@\(%_&8fD Mf}-ٜhuA>ۇQ!a[-?Ќ![jWk-R a:.KQC6G!ѨQEߵe%ϕJFnU K='su&F"6^:29nrfkje6qO {k2̴`Yk KXjf$\xxSƏW'Wj1v=FzS11d< !뀛\G•p&'vyCQ0fl48 rsu9W?}_pC8&y8qf>U)4{ғGǶMM٥~|ϡ{Q0U~y@&dYv(HQ͎W+84ړUOEp=RЦԄR6eXM Z1MܻtJlf$}N+ < sQ]r &:QD@ﰜԀ# Kàfm$SHo0]sY##k9ڐ1ÏD@COU$瘶Q<;#l-Wm>ȍuM19<&;LGFGQ9r>o4!Q%y2:&6J ǣ^7v*`)حUwKQ:A:q#lM'cE274t4:BduxL"%5k!@#{_j:XL\HX<:d1erIS1OuFPQGn_€q V`t(n7׭,IDL0kJ x"1Y*~_šֱx@{3Xu0EnmR)G]hf),~ﷱtj&qz}VxJ`vnҼ8O,ͰvjS5*VpX0$nz\?&x 8P^l/mw5hc!W?ea3HIzp/ w}_8\,V~X{^l &⬛ R,csF-PEh{( VP|X5%z1{BkN?2iQ +Zظ3~>K[d2'új =2.KCmq"`#53Z [XJx,4%k{@MYY숕0QԔU:q'CxTT"O?ZDZX`y{ K[m)C& @$u\58"5,%4T"LL&!]NJ>̃M!Ffn6"Y_:-an,R1-:N()F()LG) |ڽ"Y/ a 6As<5` d4dC&8_ kH0 ~V:YUqHad-նVd 2mzYCkW,),]+LPreuO<]ޤ]dkwj+AJԞD )Ҽa`/ҧc/-ӑ#YR+^A9v1:Zhfr=Ȟ)W6DJ!x|#ʇ;>"h 3 XW$YTWÕ`Le˞C AB*Np2/;e@<J>r]$z s=WW+@Ik 2I s Koz,YfmqnSSYK)LgSčv?s= #D{8?kX4̥Zhg2ʼi^\`Xā|ws8\,"\0ɠVJ;PNq^wو&vQm+I;˕&VWI$s ;[9*Xh'hQcēs"rxw_E\^DFL :r*P;Fgl@hT OI1.VB9jx5@_-YĺsE\Pz (^:75g%95`\'X2\70/h`ccxɑrJArcrLw(uGq SKЊ`~I9 ,7Fn(0DBNøY\)eLETJ%Srijb[@uP]:(be=VC8 `*TT9ç^sƊ3aMXV-4˺I Qݞ"A=H daj%Tx쵒#@H]='E5`h+&^O {D7eU꒬c |;=!xqd[R</P b_%%k-,Zkީ@ΩsRe%oYq uXli]E "Cu4`Ek f8ǯ8yV_~ nu/Ybz-@ZFLׯCoW'4hU:XښRR?YρX,c( N)9#{ЪUY^ 71C pF&a8#)haH0}8\tqGnY w凰P, CiȏRX>GiJDD޴/L5`S]xMlutr8 &&\δnNf8Frkb qLtGYܔXԭhK#cISxWZ8uqcePcM>C,! л), J1iYcrܯ \ZQׁjɺ:yx'/,7NFV\oToFRD1.:X/TՅyR!fUq|o-Undʾnړ~dkY 8=b߾Cl(՝W .cX?َGXn:|+]"?dզ3l6wZNZh08"9S砏-,/$km 7'hݮ/f Xx)֞7A2FOᨯoZXu5##ß_㉿ ޏ[ Oh0YL`@~W~ާmXc[@fyځnv}d;E}y"DQJ*|Q  $! U<>ȰjDSZH?Z Mb'hć=b>RSV!}4R j~LtH @DdxjUfY6nM2U#Ⱥ}fZaܲNAӑsqr T(DYHm z̘ͻ쏿n!wﺐ+TeTRnS(7&ۡUp^Minku&}leHrj d&@j'H$Lϱ-xs?v?T 2֍ZbG?lm!Zn: #m&Y]28ZXMuAMCq bWgLT' a2d4 *gekkjA|Ab&}I7)ʔٙpX Lv#T`aXqLt'lI;?j[^^:3x`Bȸ{=)â,@P||UB3r0(êR<_Dƕn@lj^J IDATdNmQc&1VE= 1z@[&%4{k-9b)VkyT5זt*C}BFPnUЗJdS့L(*eY_yAMr(6 8%SncX)4*~^" s"q?ZQ]E@mzO-l}X,2ӵLr-40r.} @X(w?X,`rxgx$|/b翌# Bt)}ŀ.,IԲ_>CjH81V+M$nr:MKp5uCHءeXup|{l rܰW/} BYU3PQhH-pғ.()=Ù+526huVilXal6E!v2>-ITB0 )&ָV֯Z9D* fSe Vq:MH *7W90qng-\BP5Z[f-lI D%\XؔzVy4t%Y,$]`4ea&#jTudK A`Fc7E C.-c (P0ׅ>"pB"Y*\v{9 &:ధcOhM19W0'&$fi 6b&ggu3bU!J j5ʯK Iy}YfBQ@)$X'P۝8 V8ֶa"ȌHqFӪo*PXPNwVjZ`gXϩ6U֤&U^]; '᫤9Ibb6S$cdWZP^.+:R:BT3H-YĢH7ySHEqU~X됌ZMktC5\58PY+[K fi-Ų:Y[ 4#yg>=?zp{\b=εYHd-5Ku0;^ai{r,a"?LDtækEjuήAQB7.M< [z:ю1^y0!S,ޫSRi_ @OqǢX:#*5xn}K}칿bdvfb_c`2yX(RhN2RB5$ZQ*-MT;0" l^dF{cYWY-~\Ime* dZj~8"`#,8 ńWxٕC ["H'E (xsS`uЎǢkL:vdH_ b H-T(s©yA>s$O%' .αRh`"8/EnBTuڭ eXJm,iС VZ8yh>9~GC45?7lD=zq* g!SW@*v RR'ѣM$v]=s1l*!4SF2,Fk)Juwlf&1I.}IFHۅɇj ]uGs\k) A'M%K,? T] -4 "h'9G *Х -Qus1Ql)8ƀK+?" BW:I9ZQfJcS? X$$hL&z^|q Y1Z"I0p]G[09X 4dX [Hiwdkžhag8/`f%U[LIZI}R-oAUqHM %38c s%3'+q kD+,"W8djXw]P?&=BQIcz/aGsԕ#p Xpʾ',8X~%5P*`d-dKtEK@'`v,+zo!Ե˿)P# 3cIIvDUk`ӨfDXʬ3*"(bqҲ(]\rcrL1P.lD! A1jaogKŽ l`RL QΤ̵(H'5JnR %٦ܴG:pCAYy~m$cڲo]wLdB3mSYO's~8_?^<+?$ޗKsN1iYcrgG::͆'>#`;QW@B.J|eUNďRojÚni%E ,X)t$ ZjJi` d°VpQ#_=5yp6j`/ Ndf YZ6P~Is;X2_Dk0ti`>' Ȭ?m% TY_T02EՀGU2/[;tW5ƹٰZƠe#-91{I8.qV:K3Iku@ ˶;sªK&+ m*YjsaH~58[=2rQ<(*$5bEIv<;]͓emVW` Ákn,ncjiKJW: I: |d \2IP&n[FFt%E▹zSY c+OXy]Barz?cCQݧիz'{Ő#V@c_ !w#X'e,SFDkm1p)M.2di$Br ? c'E|@$ O7"fu U%-Y͉0 -]n?Q+-k ΅pDB4zeYwwʕ }{peC`ob+Jt]d"Q]zQ#UkHFF[*3@rF#g9RDcZz0uQx5J)|~p;shӦ6ZBFTAs9"5&Ŝx fN9mQO>vR %{UJ5,:9+ڈ9<$ay8ׯeL1_,b q❸?C>(  yGc'4GcWO #M-/ %c}s04lPZfLJJ˰#BIk#,ͪ# ,$R֎LR &֤TrdPRhR PQq5'Z^-(l8B_V[ eVbSFkfqAMNei')i/PHqUZ1c7Ұ_ 0?Xŗ{`+mpɌv\?9ؕ`{' wk;N̄_9ZqGǍ׼0i&ǑFƯq M(㍇ s\2oT Wٖ! bipINUѐGlq="C@_vuLВ] ߁SiQ.bajy<׏+I/:.~QKm Īs+fmAej@JuE"V?#moA#I qBN>ԍT7{(O5ZDbPpB`@^ Y|)Vv10['{}__ϼop^u' ]VÌ2tl [XMa:CPG @Ϩ`nϠXF}K0r ? %V 7)'بkBߡ+`,jif1:Zc/H(HǏoڃ+5X9X(z8Ttq ?;9?9/zX,e51(vg,_<% *Zn'(j)$ҷ wT-v|$wzwCJgrU!,HQ GJC*#hNEDqv,+!I $&Poffcct#š!져믊Ɛʹ|ȹV4qԯ-t`[ 4z|hK2 mhFf1>;.ProXLA#6&09 *w_8;ā|`T5<rdL՟3<W_}5:,|}W3?AuYտcng^w_~r9k1=9mFg9\Oވf=}'ŚUfrswK_9iGo\&yFyHUhHɸ Z;mZ$kDD\#%8=dmӡUS8KeFM:YsAҚ5 ]TuȄH!6P$:b'Q<ß1@U%#"t3aVcYX bU&#(aɼ421@+v!5 k+_;Sp,9Y%|\e+CHVLpVsEwsḿo㊻ uػw/~qK+鬃pR,oMMPF+m+|g>=4gtcN_>yoV,<tٲGj0sd)], %#>k&Z>+WLu5iԫB,kSczWsĭWn<~kBD#qDԋ9yiPPR&#gkTcf;,I!{t58g@m\#\6( TvJ7i\G%EQ[qO}/`.υIrRm!V5:S'> ܹx3_]ve/*Z^~;]ݕ'b6aӺwGmxE/ڄ-݂72[7h&VFR7GG˯ZUqB'geOoUxpIl sz)8J̪X^C 3<2#uM\u#KYf(ڲE L"y&L>1a(9Ye`'+ URCd4V]x¼+Lp6đJG8+cnusD4Vjm'Ng  <1&+UZ&ͣ!?h5-9>w燑"᭿z<ןN{ >eLqkV>_>'z=,]ť,/G3s<Blz0Y28eUϖm%_q۫$UT @;P&4:'%4v4x 8! >1 Cp[\-@4nE3ǯ X>}\O4ZZVҞ.(7Dp\V˝1&-+Mi]K [ %KH] &G |g`B zv\?VDx=愾_=ZϽ.0KP6kI:HZmjr#ĉAtUnІ:issmi\΋SrɵL{J&wѴGU&CgveM?Ν;qA<'><;;!Kނ{/ AACg'U"jj@b[ ?R6TZ$;0cD kq)?ŵ\ D-!2OW})wd)x>$jh\8, X~uF " )2IJ2k( PhMpfćv ^}ѡ'Mɖ\QMBYF<ϾLt|T3<(xcstF 8G/+p'aoÕ nY* 4e.ǩdF xM}يe^EeaPl2W8* HԮT{ oR.Qy`Avv|ڝVv .Q(IQuwf6bd;9ȹK㪏}Uրvƈoh] Zf unhHIt%K@_&>IRJ҉2`3qyyq6[Sq.ڵ >z4ٯv]h O9{vۍصcvx=qN8 \Fڵ 7nWf3< IDATb\WSgLqG(xi4dER<)2N]4# DP9Jӄ(139fyˤ:Q#L:-qj?zjҪ2S9}2kȪt^`LH7[v#=(YV^G"M-I<"jr3 AXcCfi@Sp pzA,JBTTiAj=#dU\ v9CA!C:YkЂ9toeeE,CSb,HU%SD.%'LMrE*1Wtq8_B^8H,Z[oك_G_s{0Dž~c|RT B~$<|R,hD^-[h7'ɝ;ũp8&(I3h$C2h0^BhQk!N5.|8A 1PE`;W)A7<fŜkdP]Yw]ܤb-c9z0X6ZȆN ~*}k3YWKJ"ݒոV{pb[u`0m՜'3Yz pNcSMx)MF? \`brkFsKjX=֓}JX3jt^a5 IqZGXXb)䗱I6 %zg`g ~9&ۣĄؚD?{>*>G!wyK/G?͛w OqWcvv|3ADxk^% qMoz6o|ߢݎwks1.So~DZmoŠ8EwV\Oڄo03+xg~+>O8˞k?-{ wXb7&hbрb"hP4(1(b0?Q4"86 [iݧOGaM]Mc#}oX]z^wMz>n;g~Rh`׮U"}:n#xprg؜ cO#YcC0h_xMePkRb/n62- j!Fcsd69"2)p\g4 h۷#JF6`}MV`DGXJÄ џbDxp/nYqm8Hp|^R{6]lM[$2Y˰ B4 ( &T)!U[ 3bTW$Sΰ`c ˜KTG ĨKR]F/A'k1g%Tzv&a Z؟\lNjQd]UY$`쩑ÿ! X 255be=3sS4 wʜuī`7lNvRzC߁tH1ѢVC5iMtueαkoV"VFiJYdnVsw]8yt>wY_ut NU^6X0#tD1hҒW19B5Ri0~PrԴr킳lM$gOA?S[; n)fK@2#]IMڌ=]b kq}bB\*&rHXuǻh E9SQ9.gvY.]U˯+.!fNXXbRD  C+dxcP%e JI6\vE2Ur& H#U+),l޲^-jb*R6Üh$HO:C<= 5Ӛ) ]J08Y4vl#x68YW_o;;8qYg 'UVꫯ9眃O?v㦛nw_-cߟ^| ͣ`߽ܠA~3`5Fm87Noo:#n:G`4(7+_ =85g߽>t*~C9\kͺ'hU^L(8h1l\dېW~$[3޺$[M jEGGM|gӵv2zEj%fUZ%sN q5 q菈agF1UG6tHe:a,t-ϴF KȒUU+Nͨc"`˺miJR_:VTDV>1Tr)3yX¦V|iyEZB=&VNC*IxP9 ɎO!<BϜ`{uw}}o<]]qcuj5~tobfͺX!%,a(˕hi p M P)HPst\cQOe L QfƠȱCi*HVG=Աċ["P ϛ24Qi!c0Ȗ#Z~XlhYfMqˊ.[ڢ_ m ``4ܗ]V?ef-ru?=HC%xUg ,&0}#EFd5m8B*&`E.%;⽦0a䘥oYO-ZLe꼑)Qh'jMQ*baׁMT2 ,Mmu" ".ƀgnjGtAQܷ&vL\Cc)ϫqkc;XU rr1XƻcdPnrq7SO '/~>y<.Y ,bs87x?næD?n=6_p6.D7}n=wRfJ=>nʍ8g߿wVcgހ6M5cMI?("l­h'kZ;z{#b9*Q)Ti\V$ö"DRBG8 WVƑHR]P(IJ A 9L:`uԚF:x|gqS+l)YQG#?tzXhle^lς]Bc26,To0CL[,ݮA[/Q``9r.s%rso$Ekq+10+)%V2k?T2sQxewܵ jNYjj|pZlo}G!N?ahn.eׇE!pd6no1{c !CXu7b,!pcWG>P0Q(8wM[,Уe8#ͯ|4|_c0+SWڕ֨Id(;)???AYV5s*?$zԻ%1򮑺!vb `BCY cSPrBRj9! `Mg9:wͮWvރ0|drO]LM>a%WjzuRNHvTS XŪ$#&gˑ%SJ ĦxdgQ|SĤ2TłTԲP-3Tu'O [7D<*g# %4#Z]"N đi?),uУ s, C0!!f.؞!#>(3rSΘꎚ}3@:;8ZnM}K_Z9\wuG\}8sqqog?x ݳV~xl85f3䳧◜+?{"nm3=tA/p۷8W⏬ÍnF\Kp۷nĭO/}7._/؄+ 6q37-/~.+jX)onօ:cS0nz?O<{ѥKxF&Gk:\v%YxwuC,!lŲRm͠K(Ч!vs54ZpK95cPM^\ep:˱wY;s `p7 w L#.1 /ӝyQ{W DM]`HKxލok__32Y'3! / &ҙ?:ǚ X9~B#g\!bmlZXjuȺ*Ȅݻ1XZKf.~vqy&~k&)@3-o#kGnZm9c_̆|M% (ёkc B‚E jrm:,9dCߨP0h#},0N t@Z I0LWې;eKȁLeWiD.3uGЈ+:38bfOXp(Pu!E:(!岯r};7 {,N?tرwSOc;6_>O%W0obGK\vE1樆Cy&&:c2WM9 .݀7,qj v6%@ ]Matp|Qx-ܪ$Zذj']7A"IhrX.;i +q-D9bgxB7F[~LEIZ@vO%[{jQUvm5S9 i0^bJ@=Kq0A ¤mt@XF7w^z K]j^Ϳ0}d@s̾QիqyᤓNw܁|cñ뮻gu֏co_]'7>y@ry?qؚSDQk2BRt =`@8lJq(xڑITe-WK-ɵN'ҷұ2)z:ѕ gH#1KRUZ'rA1b-EO8òl+;p@w=?p7w,m ?=KyXȗ#JN}ɤ$4Od3C]f#9NuWb]o-0š L`-(Wȷؼ vb>ߍb E%fK}4?5>q0\]II*igN YƧ>羄^wk0숐R0&(YQZeX8O# FWNk/2+,ތb eKc)ba]Hw =70ȇ%C}SVݞ-$SB8Gټj<" D݃IYYE`\p0gu t!pX39ݕ g rÒh"qJ5k1"q]=ǤB H8=q M89`Ҋ SEymC0Dw->Kl"x8cPc\Qj9. RAB Pp% ]FѹGeɵ@~*PD t;k=wCrf#˵ŜxD˥apg&9S20YjQpA[5eZT-l{>71+۽6Era)o1N>ak*R [McO  ]DكB ZF{f &6mF׮nf-gTJκ yUk;l% lQ7rRhWeD&`f^_]dL\չ\,LEXz0># Eq3IҜCimS`pEIf xJ=hH"Gf\괆e^%bm ]M1=Ǟ<1e6BJyOI@B'>7㤃bDit52EZ1[G*?PoUŕ7tD4:nMs)HI\OZ 6M1=jGf}HEUo&dkY%*%QVb#5Fdr>TVm \eJQ#iD81Hš_Zql %-}!Ժ+֬-{ّðXD؇11BWﾜ(|x]E+9L>]r=]Zý4`H܌˽ΐEzeBTl{ƙf>\V`\g"'111޲:==խSWqF_u{+HA V/0 !˶%HAɕ"xRKƳl)vS xzۥ ,KB5;mMX>(+KD!+YSp8o׏9j:7mEC}\˦mBr]|ĩX0!l!FӌAJI Ѽ=AWme&sR,޾(ʲT'k>dE٤hy`:99X I5M:#l7}B쵯ĽwݍFe!u3x!Ob @HHJ`ZQF-,*܈]EU"h1\xrH 2E -M#KgUĪ@mI"y2[ $]5ߞ~He;¤;ULW%|o|;0z^W.u,\6hy :zp#`4< 9%66)Y`=!IWRqt3b2C"Cr&.hWB,;8h1XM-(TO>{T dDLU914*>6 c9VJB*i,g㢀nѢXW*j*E^%JC5:5] ¨g&.~x$Ȗ&aN$;T2ٌiʷl;M, ʯ|.@+3DsR[Fczϟt0Ygk㺓Dw|lz2qI5 Z_aK@F_EFA69",\>9CRN _b*!;r P XTm#7WN7 u0bMg9dfA椖=sKޣ4rjP\VHZgvOe &(;;06B$9/8ɴ͞vmkhf zgI^Kn"ئZo(͎bWDmJROF H%Oy8^}ޙXn,Vwcmw,_u?+˺#,e39bv,#_@x ~18YFXksS-$֦W`Xp%bY,.,B|6c` w`G"/bC 9G^q Оn6XNo>;b~g\oCyGzLc֠vD"M}b"+1kfƓV'Wk-} %W%I ,x[A\ld)fy ) |T(Kwnez-nJY"- #n1& <(r]ASDb"C2.oBT-d Df_:[Z?jq%ԔO2nN@ <^f;B|%¦ā見Q!>[{U4v|+w4,H1cL1=cxy?qؚFȉ=Cc4b/hih[[XOV^~2^Tiel>HHgyMH#,$舵yAR CJS %4 Ee.n/  đ[*={(9݃} 4jH7ifEJCTB\35mYı;EL6cI6VA-vX3Br.@p%ᠥ]JqsU{5U璥*usmϕ˘ =Ŕ`d^1)26] u*7?{ήĚrʖccqc3.Eb!;2ټ=ΰ8k_BQ:R3A/`袃y=]n-F!۴h&~s1BfWiÒD: =ʰ7V>+hqy*=d#,jt-+Vdc AD[;.'%LZ$VEٙir1aٙa# tK#fuvrc\a!]^!Jʕ'm&z׆c0A/~3_m8Cqpc?swXޙ PD23j= l&ƃ{` C O߀n:!e#Pmȱ\܆]e|]">]G R8;Je"[z Vwz8hf 0{0f6/=OZ~(]L'>uμrK4¢fc6vq\gUpX ˪$\A&Cα=_%| E;қ[+)'YRqrH%}`$[ IHs:mP +B0j05K(f9g-B;LD^I,`4,LqV/veǮԓ仢=HP5+}GlC t(E|385~}_؄>-绱3LP~;6t?<2 @ET, UVj"9r>qG/wܸ2pQnWl^wnlk+ށ8ap=k99ov܍Rc+ʷ)7fz; UAvJ U^+3)VL^qk"Lңd! ѻ+0՘䢩Nnf7TMR@#Ǩb8Z.W&Y+ykKS(V ֜ v0sqqPN]]|\xzҞ[OVQ`fǤ_fcs #kTR/ԂaGo%IFJk2UU0QCsɥ=NZ#zV18ԍ&h{Js~)' 1o4t{ *E&Rn*j>.pt C&q Ke0~*g)aFU ĉV:VҝA - MmЧ UϾ`r-jLqX%'zh5U 8fńZ9 ,"6mfSVHuBNfsJh&P5%wdLUE2 r蝂HT7^`Ŷ?Gb,̙҅b":q dDIaQ,T=C]Y8S7nMhPs Ty۳D5l\pq$\vX6-BljY֮A$vgZmON1=:' ڳɳ -\hhCW-,[PcU%=*fBd2d}q9ƈ(.d!{9:1FMX|$_oa%-Q 1K<΢ Kr!ߎvR,:E9]n'F[Pu*np0o$NմHDDFdkdV)G9{/ P@3lEsGS/֊B4Nqc5+aZE+V>jkц7jT^(U05ns%x؍|WP[p`WdV ,X<:n{i]H *$nE8y{YF&C_zq}bC |1,U9њH[s]̖):p8'{G>6Ѭk?g%bdM Ճ^@w5ٸR4t-4 z;ᐮlux~(|XFl@N kϦzNtevWm0"h#I1 !э!KeE!qBYb%XzNR8dsU~^[!踏r a8N\ϳZƜe)6$U$,z-QJ ;ck q;"2V,zʐcz%khM|OŎw4K4 :BƄtS[ZuY`l:m m_\i且'PXem*@鶦H[3RG%ٵKWmTِX0J܉@#׀s,Kx`0߱ Īw¶X,aȕh|9 XZNr0f0\쫖RW{`d}zÎ; UX>3e4>MYHsP 9,2>'v=5 6BD&qao4{m6ef-#jΏ؈SS+`ϲ [`*^{kwHOQƬdɣ$]a rXd ;)Y&&~,oL = N*Oo1eM1=e&A'J"+qT )snK;vCyD N}#tNz(τDuJ.а8[%#R\H>b 25ϰlwf!݃a2Q6 %4vY7__lgDB)Q % cT4mYacWgM\2ZQquWqed/*,rJM:1HcX2CJoVG-<@6' da0=/M9JNp!]V36NnDFr| ۇpo+nq/fc@.0(rŜN^G$6WQ ߞ>rԡ5{jD? %GA18핿e3G+(7Rem(!R68<ŹUBgdK$-u^u۵`o Enʐczn!KR⭯f!Rs*dx%)=6rAN)STVL?gƈD ?EU݊EC+zuNvӮ*:tۈ5V \@~&X]eF@n;0HA\\c:=0b6anyu7UwQY$wԭddz&ͦkΧ_%Y^%?9JȁZ9Z_K'G GsjۯRwY Z3 sM!N@@}YT]t>q C~1DO}rKCkyqk 3]o`wwMbilǦIm߯5#&jw! ,lƅpšw_?{[qp,X*u(iDh>rh%g~ɈU1Sfz,*L-CV.ckjXJu]j/5:ҨIKDKҔ&B(,C., ߵ>R͢Ң|$Йv%HE$ V)~{*qk{G!l.L .Bs = O܉džőr%0ǤV@$']5o%222da:dxP G`$j]NdnVc2U( t1tKY{WɽݝYyvcʐcz%מ0F7/'@xm2M8pG8? |_)\Yk͊։l5HĔ)VUÐbB >g"̢M+(V*B 2I r!~H )u{Dz-Jq!uA03?[G ȃs d, ?-a:a ;* 5I eJ8lqe,6R8'tj0xk- 2vs1jjt}l52+&')Xf(C.#_#p3{G8`[n@F.Vi&(ףd8h}{ [7ܚHj8X$c̎/ȋ?CGv۾<5ܰn|BǀꝦQ#$F@$ Vm2@z70 ԢB^P\65{ly&"-$T-GқJ2MP5%F(HZ6seXT{[, YJL XvRY@LZcɸtF+_b8yZ-@@S;[ (wT =]l؅ *0Gd)O=i_cɛNͬ!9`:^†QכqEv3ۺ)iCZGCu#6뀓AMUyr) 7=x-/bO CR M g/R`n@x&ԩ'ǛeR3T)bD&Ё.+]lfBuOzSh-3+nm"d[pB3FjPT1k,MD a?JY˶9:!CEshFBŨSӫ3fY3LH5$8f-vφk5kb+ΏLZY4b 4W+8' #b޲U4ourE#\OS%H$T0vvyO]Q< &7956R[c]ԺmGQ(ܞngrixu?9=pƦ@AaX*ؙ/g> |Z_چX YjQN}lm3`rNLHC gdQ&A{n saƐ+Z؂rVYk,i>1Id)ҎVdѴ[02 PȈX4mM N0zǑ}߂\{ھCS2b T87 ~< SH ƅ$|֌j KؒX'a`;l̒2i ]vaup^v1xp@g9f YYHP1PU{ãdiyJdt]jk&`PHczX[ 7$n2+0n/:o*q"(B) LV4)1tv^Hw\0hkךQBkO0Ȁʃ2Z, Z"5WhބqEUhIqGPȶrl@exbI1²P\v+mA]BV@`k#a?E>Ol,H$)K $Cu`s1>DOS2Pi$Zǻ$a1 Z(7HC0c-tw#2Dj8vVn}p+/@(DGyϡF"5``<󔓑^S̀,DIaGtQA1 ߠϣ؅z۶`pNw_,iv< ('5=*Wsz x1D 0$ TSLƌ}ɵOJY$dTݛJBF@}=VsWTL҄lT!C1D%vu%f!_A(ʢ !HX9JԚ%S. `hQg愖[݌T1͌eR:LeYY :xʓN˳Dt)(?b0"m:Ô.lYǛ`ޘ0) 7=xPWea-%34!\0ܻE30 &)P;`>+<_ 8kPi10Pʺ]ٰƪK2p1CGHW+ &:2jgL d:rQ-I1z? XT9 6mKhQVj?$K"w ^LdQ>X&[F1K<ڤC6V:I҃RܲEnkc~4¸Tm']R@7yձY#p̕J W]z9 / 3n.垲mg~Xv9U,#Xe=tBѺg=}oR$:z=lvݍCΗ/ϓ)jͧe؊r6u R'BuIhh $͈]cI*٠̞L0 `â>j +{8i@YīyH"5Hu̟Ja_  h/HƸ-" vE6b$R3PngK7F]t\p>0""fXN#Y^5Xhn{v5Z;9& Œ|ŧȅqu XCa.Ȉ('L@iƑo"]S{q>`5OrczL t䦷r/VtLjf)NdU !Q N禙J3*`j " $jbȸ2Co}WE.;v`-eH]ymĜ RZ0N^!ӘSM[%(HhX@EXS@{EYk6TĖU9D21.tX`+sG}KP/'~+Y4J0)T: >ԢbAEZQF6y5]PlIHl`Ps ׁ?چ/( MDZl2dyߣq1GB^kEYPRt7#1-u: 9&Trg?IHrU Vɥ-RIҩ-n({$R1T)?>Vߴa8oSRE&[4cI8^(]SE#K It31<7h%%3YXWDR0bGBJ^"ٯjkS!3pe+]Oj@j!yjJxגvmZ Sn)˼d8 :*;2Ct, a<521RMCX| 6)P[qӎ][p dRkaU'RL)I `ʿ2Rr]AH@v IDAT3=W8!2ŇbhBC:ϕncsti/?0<`:Dg(Yam;q%LJ\f0:>i{Q5aC/e$嵥 daXGTZSZ,Zb3 @ `9LMUغ_NV<0XRQԌVY'Kx6b D: "RukuxdA1jFZ$$"~UȺͻO6HxQqn-Q=ZjYP|%攃!w[Ж= ՠF%. /rH8I٫)s\ܚ%xSix4/xo{5dದJ b6=w8YJdbz0\26P$AD5Y ֣J#'.Iڑ\"ź!AGMh*`jnj#u^fiǩFL.Ym]@@#Uڃ# p(U~zcϑcEYrP&4AN1 ª@3cG>|7X#8"@z@d ½JpLT˫m͙˾ c2SMm{QOR]c/8kۯpU@)m 5'F p#2NI9r5T#Ҫ͒j~Q$Y%`T%L)kP FǏ&g4cKb_-|H 5m0T-U%RqN$%_ ۇ(SG8,*KD 5&)x2BnM2!2|Ss9[<1v4Jţ|I$$ȍpwalpԵ"ךsyL/xv5R x@WO4:AC"ܺtsoִSi?>Wr"2s~>ܨ8V5ƌҍK2Xсej7z"HYh״SO/œڢPV`"J_ei4Aad$ga+l@礑O[,V@{۷eKƢ.fP.cKz1`S.r>f;*(&m=*|yQ`Cba͎׊ͬ$ĊY(*0BJ?M(H_S@nzLK[ V-=dYOt`{ҎL(C~]֠]ӷۮA(e!65\}U lAmU7ueM QV3ħ,UUXE0 KfTh]L9zaSH8JsI{%6sXi8ɨ$4Utgq@oՃ}Jh]SJKv>e~8o a}5!{on[~U>6uoU$JU: tj|K#D҈`@CDy<#b h MĐΤRUTs6眽c~k[uu9[f1CJhDg{ !pV#5*MB <> K-ȉ+Lu#}OiϻcBVs[-߮:{:,n_͂Gjpiw.ޠ[@XMN͞lS];lhb͡qAtʂr;@nw 968 uyo?ʔx>u{_Eu e콕Tv3Rl/ @E9{sk0{ D1[RZSZy*t5`efFƼ>0XAyyh+>;K:Y =7 Ow }ݎ~5NvƍwڞܓٲPmgkR(2%)A]TP%bٙ, ?GH*Ʌ5Kq~[:iaTTY9ۏas? DdUd +,0 ^r , z %\2gK\)KaUp͵E65GKYi5)gܶ37>*h>he©)A!DDOU1!+J"Y7L&H,>] jS!c\Դ:4cX]kw]K[斋XN?|F )9t[bYxQ}hT@p]\XTqZZz@r{`xq- ?md͞`~ja@VnbՏ.QCjla@q4:zgA̴0~Ibr4ukM)9uSp,zEK$x@|j3DiLJMG(7S3._Qn)U~+g&Q9?LLBcȂ'khK٢JBlnl=P͠+p[6BxqUU++? FV5S; ( 8e&b /CbI. :wfe{9 aPssb5" pj)mjf=hUmڔW0\hȐ8Ɵ6K6 l dIrbog(/_(~FvE:clצh]9E jL6}[0;*˃^ a|@}uP9fdW}gF(/O7vO㟄_5*c|b&wY?ئZ㬵|&J͢Du|1j|(W FF Rw>w}t UcclXF)uq=l۴ V&:E7S#EKfmwҋK =Z+ E#MUZÖ]Nޔ4jkET]sPDM1~k?@o sXnQZT}ftT1W;c\s1Yw~FĴe5CS!ʿ unL {ɶ#g8۵3>˾`nȡA0M;nMجf nf4>ϼ8g4瓏/)S5eTD&\ puP9dPqǝ[vL/Qy1lO 0U;}B: `q=Jdծ;%xAVh;؊FwФ-xk5&.'Sr &VmlŔ(%rRb7 <0rS׽ޟ?*l&ŚH:I\?v)AmgF9l_5+@fO֐uլ`Y77V1%OVRYe>)KGL0&)'YHܘ1H {6ZX"4I?;,\b ,mX$^Éj۾Q sDCV1Kc=~T6 lKW" s ffJ>0!M춴Gp݃6巇I/27Wp } h}!j*[t{C;AXt06s>W2]sj@]¹Ň/^Dȝ*@M3eYRscE*LJsRxǍ f]7s4$eC .-1F1Y]GclSc1;HmQM(]3>5eܻ=$j1^h+5`W,jbԜ#yefZ^X1;%F#xTm`ڶ_̿ v~^#-r#[To}[G3sQCoŸy^y9Ƙ´19f]kw=W#J55U1el6c+1-@Ŋc oI?HGg<ПN6sht$.ёGtVPWNTmfME5|Bwz}cH_wM:~|֩.4w &E)v)KH FUOrvQ 461*,MGV HbAd'qvq '5(!n͛1-ɈWpE@T媖YjBi^ٻlϾr:-$6)`hu*<gЧa4&&:iO?hZ,mE?;tݡo57 [+ԏG2(WO?TmZYÞ&mĄcd-^wID92+VIԤ;22R=]D}q&rz5F i˞I ~R M[lw<y9:A>Y1H[(j0eb/dBֱK+ʼ7fiխm';ft\NK^f|)|Hd@:T**rm;u1snX0==.[PGw]C\G骺 =h` iSEǮ9] 6$Z$d֨ q`,"``pZs4&GZ}~x1a`c8Wc oLojama\d}l3*`)S>xI6I 8йJ n{ə@28CSK!/`K\8| {ɸfy{-XKM,!VSzkTF(PLWn* ])#K:V ͪ 2AбXUaC_d*2~5ZXw-xuPo9Hgu?}80{XiRkOgk~p{kN YLh)g@Y$0@_jdѹ챪k)9VUO^'51&ל̏\irCt+Rыu ÀTQt܀AM FJK:;@nwĂq"5 Yte2V!cz߉y+6xIU:+{qSN\8XeUB;I9[2mr:U5:XMe@,Fr^xH'WB' 1A+i:>U7s4fL!$ΐo7i;* LnQzq Ogi8۝B6 *&~@ٞOy0(Ja%g4yΧ^CUFirZ3, IDATrBk}f02:bT=Sng e(R D(2pMx[ߙ-"4s2_cԁlW^AU*8xɸWvU,x'AorQwK5pqx1>P_2(B2X5Lma4+T+"送VpUSdj̨V&[b7t3`7VsC:)•c{CEd5\ιYY~X82=uPd]؁Z?B|m[g&H4"j.X!J1vvgoPety1+9x,vJ'I@[/Ȭ)e!<>VSמu1$/ -sHj"*\m E6{^Nǖb&Ҿo~Z :Ժk6j M9B{^)s9B{wEcᦄY/_nR]ڽuq`ό Mʺf9eωe?;BeHU$:ea<]ߧiSt h6Y@+*OW?̋ggJFK/'{]dROpYK[ |* 7"7, _m33eQ̠ZE /ًd1f]F:-#J83lC_ZT s]kw=W'z?Z{q^<ĶCvW )*Q U/MG|U*R1Fˑ ħL%{uc;S$C&&٘ I%|>N Wo/U r@c+oP ' T4(%զ~&3$jg-a)@6O(F0N8 \ǯ~㑷#8ƶmN<+DJ"R2_F +\LBr_rĴ+NzcKNGvze=7ӧBhLb\_'?i"P/p6pg<9zd^J.c>?C?jZaJn[|NJ/\HؔE skw)"c$ q=,n)qf 4sDEU~-'ڔ9kM_%LHĸe߹j*2cePb4ɈjYIiu@b 2kfb@FXM/ӑ-1rG"f6 [pZGpb%lcpTbQ-9~/gRzʪ9[ZUHn5I&(\`WJfsnUD=M#Ccމj2F#P"]kwOd7թ|,28lm65XIDU3GSY'9x8QE/a3#htMgi8 NOoKUȐt{))Rfr*8!fʣ֚qU C⒵z;*(fحhQFopekβS`ŗ:<4*憁$+\/ၣs_IkO3c4̖ϹF&6w:FSl(&@R4LlbV!{.@n;Ӌ8-eJ/+0ix*zAf!ynaz|7 {s1hɖud[㩷݆S'NbK,Ru UXk%?܏~!xgTR_sg~WG_xMΘۢ2FY3)[(->pén}^$-׀r g<^ޥ7m!mN/`#&erFs*ՃR: ^ 8ە \CFGX`FcD#Ds^*8 @g\Oh-5&r\^|;G6ctR`5[`+0DJ0C)zG&{!9_ȑv,{ލ' f+U"7،y>S)F2J$ys*FVQs(z+28Varkw'bP6ut\Uc̗98]xނXm<(ө[С"$ʿXҺpSSzIX)r81T6IEjvDڜ+u[& 1Q~3SO56ksƲϾu+q6-gmѣǡV@8 _As.D}M*zCh!h QMO{ X i#hc]HՊOa pvqOZƓqt'pPqaIal3XhѬJe!J1ysh5v>oSK"wdqWڷ^gN?EWҧpQdj8[ U5wم9>XlQU`Ǯۋyɳ!)[^enn=ysnKG%Nt{jq/S߀= -b'%\ԛ\-eЈuR]ΨAϓnh3$KWoԛD48ӝ}>q牧OՋ5[. m%.O{g7%_pvy{\:p+3 (ےo|ͫ7ZbHK=p+cX70gk\dU}/3R4jlV0L@}1k NsA%kvk?gއ͙hO^cx9jz>ٸjwbtxk'&dQ!ޜJ-NC"11=FUÂ)v:kS8wx/503oƟqӧ~|w yCsr5~UzMԨAZcI: Kx629 C6b\tkp ;Gpa-yڥV $B3fǧ5^}xJș_PAWjurY2 DTqj/YBHq_cC5>o&nafC)bqWB|ݵ.R6e=]vJAH,@-=VjÄ̻S@! M4HM놡Qs~SЮ#n=!=8{P̺Bx7%Ȍ nfUX7z ji`AzJ'nY#} dxV@UFf_ib1ЌUYk ټI>U:edo6@RxlkUWgM|FcD6)?bNB }!2kp6BCѭb p[9};U|ԯ݂ qxMx_gS<\]# @RK"ٸ;*g495PWslZӡnqg{ M/'I=x h|3xu7ᎫosuF:3sW_MtV>;a}M\6E %Gq`-a[d'-3N܀:sn:y.wH+G8DC.kՈG+@N/CɅ&ߓQѣ=5J1 2 8gЧ 9NZS)wn  pm5D) y/(`U:[6**O5ab6R$kz*A{5Vâ JXؐNekQe +yTa+V"A|P.A)շQ4l[*V˫3OO/R^K"E+4 MCֹR}rkw'Jy[ެ^ϯ80Y2IP[^^ |p_(\wa[#^Mlaq2ϗm~@:[p;3 V ~.x-o>p9\{'}ܸ8z*p[pW"~8o3GyS||<^Wf,pU2l)7ڼ(M44Ի ɚX@`}. .)<-x+Xqx`u{]neUCѭgFI|jp$Y>>1H 2F7E3E%׈DfF*V!"FXЧJlsvU1~Ԍ36^ ,ء{r6h,:/0%u#zbQZ:,Ҭ3(ii/HWzִghfBW{[0pڵI5`T#cQ(RH$}2E51Stn_9'+Biݵ.v(Dzaw'qvqhlZI6f R 6D v#~?wWp?@Oʽ OKmpL M(#o=#B-#;cr7U[$sm{5L(}Im4I f5T(H0pp#@8⹥E ESF02Myli*o(Ⰳ1DÂƘ`S n?|' 7TwrJ&yVTO!M*S sN(f2PZ,F ː?[}׸[pd Zx?#hH.5a|=I4I<[kdȨ˓y,_J f|4 eX&U; ,Ef̿OdFPbt %D<3_7>{¹E6oN!AZisazӯ?x< ~s{ߗk~oem@'*%շﻏ?Vٺ׏t-t?.`P<M,7F4ۍ㼓7ڵށ!3 ;P&݇ES=]uƸ$haqn_/8۝Xrd 9ӢoԶ8^% *Wp Le5z)$8uźjC[eؗAV*Q{l3`eJ`]JZ6c .6he6i\êr(n3J)fJ9&klAQk~f!^"Xbڱv]#0lCUIe)p 8BE#p٪nÐ;}ܹSaÐKɩ| ¨$ GZB xXpIU/RymN~c%>D!s8*eH4Rj #cөfB"Y. IDAT]Û8Ec]F(dnF܄wXH2`\K7C0mLQu C{]F?Mh\"]S55%%55oɬ8.)YA=9[dXm7?7-´G8ř*[sۇL4eHG_bBvD㏩xTt,j-2Ĝ*v %ݖ V$*aa؍5i"'3+9*Z^#2r)!0>sOж]5@_q=* vr9T2yԱYRByA/%=VjRqN\ Hf2+,Պ{ZZx[GVfVmg#(5:! 0+-1L EPj&UE@"̐񠗓>5 5jp𳚂L+HLA&>X7ATukbH{=L0m{GdWkU.+C2;/yH_ Yi& [=\5>ǵ60X*sop HC8}\S8='a<|ޤS,u;Tzc[afTylyыiC~U6~Zw썠yј6T-i,EV_Y؉7Oxv^R4`9;Sh8 y^PE!Ģ[.nZI8Xy//)|; k>o *ޱ6uׁx _f vyO})e;c/]X3o8/Qy܀sN&rϴZohuKx ?XbB<# \qIG8RxfD S2<;tM F(xn^cX˷Ә '_vbT?=uAUy4!C鐘q@(5F]X%C=.ut޸z6x]y#Nj;.g T gѭ:~ ::=v2/c(pj45CuGӖ)Ȥ[# m96a"4t sT,*^H3J.h:[sǍqs??zǐ]kw]>6+c5VU7X>X?]%Q/8 q޷zG&R/* =UC4Hj dPQ'90.\![yDx5?'oHM&o֍5N{r:I2nYRX нuv[0v[ 2y~lHR34D3Ete+bfO{ꪑUo3ݙy;kp9{=|\=sGp?āVX2 V9 LMn\A:ϓEB"Ou9Ñӹ! 2S}WnȆp?%ǛoN^5/( ziy_vwH,DS\[wnpد|yIj55UgڪCKzleFI sf{m\0#$LBan )f (?5G帪D*3p_cm|ֱd'' mo6>$fK!bv]ǥa`,MB =T΀pS ߭֘:HJӊĖ`ոPeOԃ5@]eRXTǽCrjJ{TmI*h6wP.ra2chSF/oZTL`0ܻL,V.,(ǤOHUgy6{۸Ɣ0N*54.'l kHA.Z|mڲpm7n|Pv_C7e;'}o}WuN-']wN_qxe4 N9].I>/|*~qدp?BCG+M"L4n#964i}|ea ZUfVbF>ʌ:Bm?~hztz:ӈ+YzPs{~t6i0a*Ѭ+Cm z#l]wKXw϶U!+/1[=^*JE(юTaQrJ[-gn *'2YFf5BKNG@+A sѴŬ vzBGˌS+Fxq>2eO a*2ѦƑ+MU5Lp N} b Y1_l!_Iac_ KCj 9{[͞Z.+`ܩnW/˫pfqÂ]j'T eݟ`@I ȼt۞dd57:x&*{c'6!T!k)R̢ .k VT;7X2Ϟ\h߯k{2fpfn=@V7GQr8ϐ9ޅHJo ~uVŶǸO·:p4VϠu@iNz~d⹗%^]O!Zد|7~5oF 3?Ÿcz,Ӟe/y= -!5N7o{9zmٱpNbz#}vi%fg8;/Kzl{ GqԯpPGXm8)0h:[5çmr1&n dCB-;X3L+2mrf"E֝1)qP+̟UB"GWGܛNjqaFc(q@sL(f+[liW۰0 X3oxYm,MʄB!jN4lo跠Jt3\iIN*V}ib[*?6ְ~~9Xݙ:CԡcOGcc͹~ruDžk|嗺ΔaD$>}|IsxcԏsHQYSpRoSzfe'\>!Y'үĂ%\==\8眸7/o_x'>zK8 =VQn;0ф+pJ[3`K3FdK8@čc`58Nqr-v&߻dƽ+T,v[\6 23L֯M kI0vcMl F #3RuD0"-=I$fjkm)GXh _,8NKz|q" Uì'"dh>!x91  m+ּ^&M=wS,oͰ1l-0vxd6UjR6w6*7ci + [w]lxVbє?zawD^q=0nC1c!EKR޽oӳ2EDqd˷jM.b883+[j0l;IN6z#7ဥ/Hɧ?q/`&SBe+GEq$ a Ͻ7FgU^3 J1W}2ڶ6_Utp0QF ^Id"YcElĊ TQ~;{\DS>&e>QU<mvoD C8IDX??>/eTx:His5ZjQbhگRX- }fW_x[ !ڸ{{0v} .X(%~VUN²'߉h4GA_,qz݁ޫfos^܁*tidE& )h[p>Hymr틣 [`0p$*.It9>=RM4`a)otH iaqĴx=RըlT}4z;_D?ljKU@f5pG|);Qj}d}iG)PԤ ZTʊt⊒hhab\CnwC!N;^(u?v C.&kp E w ,MpL/8=!lRjRQ &H&窞!Hq\)ܾw#t'pC8aɏg0L{M^!ﯾĉ8C!R[ ,mk\CV(s-3/&iubHlZT- eP>jcےKpNrp J}1uQ'UI硎 {Uc9a *]8$JvdLsIqxOP-*֌eP~=O'4fUg!n~Spӄz>%/;z;Ͻi}in}ͺOΏ ^l7ވ>;o.9WJKM7x,fFeӧ3ם׿? Sdzm092팣Kk )cy[/a;Vl"ܰ["nL#PmѴi;qWwu+*0Q y T>0_ XZ6;I}?̴ =t;.l*!;g丬Gp{7rkw'"}Ee(1/xqzT?VKլ匀D_ A6+l{; mcX=Iߍ YC g7hMyQ^ nӐCҁkCpo;|?~һqø!`g~YgMۀ,n $ *Xcnf˪]s`̤gmmbu_ŒbЙ ( ]0'l vN>_pO܀:vS qycQ!8@Q[5!K cve4)H:lB <ٝć腸3%Et.;3ol/WP1 &Ϭ0=|իV!U[ D4^=VZ]'b4ϼ]w?ϼ%_}lѥ4h>7zɼb*0nؓs_WOWK[3J2>%\87\)x0-j"[10ЙIgmPJʰ&f۠8<\=r=m21?{ӅL$Z2&UO^Įc:u.p(g!.N!g IDATF;*#QL P4h$M 61('&&΁XZ|;YXV[_=.|Bl;/e*,8LྶjlCJ2 1z(C!?(nO{za.p ' ?ߋ?Y/ur} f(RH.၂ 7 sn.I)?!n2u/<'-fNFsVhe[1xk 7fBz)}:ml"bdU!;:A5e6퓃/{75_*Hnh ZI˓vy 'EhϜpM `|O8|4sduΝƈi&h9~v8yе4\ u⦺؉ඛtdȏJoV`C g͖],+ŰK;ͷb9SvsZTI6o((d E 6\$EQ]-fZ6j+UaYPW{_9<BXrkw'['H T5 >bc}cHJ 5LYck*LK!b񂷛VM鴧`=l>B*"+68~+8tdt ch5xmtl6ʉ#CjۀZ4KR  UAxc#H#31M ueh*K_,tXZ/|7_x'7t/.pI^Eq5L#]++𣏕떓ZcH$ox dV3HeӳI=[98)KP8V |+9Eڇ8~?p隧RDJ2rF]|u5'ȹqzt#jx?Jv#崐U}|_w7`-˯@.Oㅧ;Oߊk%J); (F +:9߹QNi^;9mق@"y- ж]SX;.ff-Ǎ+؀uaf1Ӛ[V" A֨,8{s|SxV, s*sXJ@$݈X;1R['#y|*`![}q W9'paݮJ.15$lUnWW"rkw4և崪N~[VaC֡LK Q/JE/"*_BagY!=ih-nkd鹹{ uh)o[J4@A&xV~鵐5KPa еm{xM}_H'LvbcFFδAMCl/ϰk,;! J&H#p?WRmGwJ*B_{:%.Zx3)؏zM$s`X.]- ?~F% f\vh!^@⡘`xߙ·|_M߂\xC׺0.,s3eU0^Z&FUR߳[.US /||7N,Oz`:^@kk'%||3_SL6cdGJAY*ro2pnFati{!yy (Ώ"Q8~F.aba( aϨklc.f"*V;W _e3 aCF**>g;El`jlJxFܧf]QXS,G [V;j|iXB5`~F%MXgJe/fbL.c\ܴx]kw=QװU]n\ +ycU߾1B"D"!&9Hvy@v1;$?Oa k/y#ߊQJ) *|;yw0U< i'P0 d>( )2n׮N1z % 걈c_v]SIirE4~@ ,ZPG8_B#0*Q {KnNi6荖OJz_6́!dWc"ì=# 1/h%̍f`(i3j$adW/deG> =lmfy .O쩓xÿ.<9T*zI:ǪݿN|7qnď{usUbJe{BQ0I)WSikPv<_ow8_L`X];}É̤ Ȃ(f˝,`Z EX┦0g5ͽzVESuv%SxJP򮮅B啨fRD),s0stm҅/V%Kk EI_IxP֓)/т!΍q ?'VPByB)U$1}Mf&u 2WXC =uXXW[SmN^vB9=w-kw'# ec|{l`c=:V͉,Ct$V!lbLn@,~Iof jdMlFGX4\␶' 3E(4zZ-9UGawziO7I89IlFPt8 hJ+t+EA=hn2BdP.9s xA!pDHVC @ˬYcկp 'L MBڛCaIMTl&[L!Q`DAHj$,0xI).u\\YO$GCGۚ6Sȕ$n!lEv1Whl[wv:zQ?Qb!7 ;tѭ0f;]뉼:]Us"Y7\9mu 0Y|aQ7TCxL_fwewU6ϹeΤN2)dH%!ER/6("TğQވ b@ HH ޓI23v){8gwsg&ٟdfgoYYz?H)mP\%$mѓpU#!ֈ`絛t檖&b5X"_8/I81Y )Wșv#dD㫂t4# "6` \@8M [ 4LLU^ @*olW=@&ـ9ҵW%(-R GR9Y8qʀSnt Bs2G!?jYrL[j՜rtuDJ2 WW;^?ĹOx$'rYQ| "h6/a= Ҿ/Ls蠃u 6xޛ^=) #nAqj՚ y+kM8inƛ~ŘNqT0I|Q2=2zVh)&Ӱ{voۥ:"r1w1`}>u g3)j!g{XlkNIjK*J JNrL-rO$&\H>MfqU9 Y#FBTt 0ݒ]VCFo=?Mţj d1ӝD<f^U"rU 3-h$ㆄWtnym4.% ){ATL5(NjN ā)k9T5G*0g͎] .[ǂnhl< :@czÈq8/(-M V:T30!#R݃D%CuBJ8| Mpt͂E.ED)-7P_.kl"&,Ѥ.XtKUZ|J0HwkȌP0 }<6^%gR!EkonUĔ(b\xoevj_f0u7h@4^6RrZA&=$Vi9eXz]ϣuux)sN|w6m>!4=qG 5|ȵ25$ ۞)(/ifK]z7}cC>u-sZr[S'ʱ.őztq'wcInenh D.BVJYi1Nތzw| p`\bcg6v;밐`jPWܼWy0bN[=H)jJ!UVl{,f6k? Z&y-hn$^gŤb 5flB2W63:Yu  %#BrdCIR -*[a|o6QKeNNDwdqE1P0A[ܦHR5M6g] eJ,Xl5&aN%Dd: BZUAhv4nfSLXN_,{6:m8&X'_g,j |ɸF `^ pvǫߝK^F,FdE[],]63.lbɱ[&jFU 4.yaqj7T9Fi\G)8ebZOYI@0kt 9qTgN= g;oƱ3!C:Œ_'uV2s; al@ {PuY;ـf1E/`!b6ˑU%) OGj]$1Ue|YKDRVk]Qpm%=٨aw,U \A'+\4S.N2H,Make0)`~Du aU'Fs|:BHభIy=2 T&h?*(!AY<)@VFc;8V]sl80d`Fk!9Cy؂K7&ٞL1=Lmm91@D^+c0m )[*(-]X6՘2ty\{Z4m`@>yV8m~b!t[#^͟EPC1bԳ"chQ̤]|y6jLl UGRDd24HG>vL-l\Y8l~2xȬ*kv4fn (v^7lx-e\5Be]YG㑋pʝd𝥛!Á U# ‰< 7]qJ?9"gNCUR˨|+(~.ܴU,c{/zEnb 9G&f*p_™8g+JlM6&eԃgNm`8˽l tuy6h'duzY ӗѡ 3Y 9';W-߉_.b6 %i6UҞ|3m< qWIř&(YsXuN4ApDm~jN!Ǵ:FfĮ)ԠUv"fzjb<( NRpI4]4+X.Sƥzy&ExI1㉗蠞i?8|T!WL:^:ywOf`;ȸ3{uJ*X-6#I'PO hDyVN8_J ߕ&%fQ*91` P(od*\׀"BQ2#w@TS4 D~QvL"Z~RV&jPAeU\Q%VA-kPzgHPC"UX}aTlZ 7 uDº5/ptD&dt h!a>g)#0H봐vОLt߅o }XРdLRQ%[9H\umNl>t. JGV1jf0=lŵ+w;S7=X.zd` Ó` ${h k1 ]1u0u1u1Zyq"88kBlG 0.E^JM}6S(gsR F-XW5$cĺBv!LU[*+u'sm6PIKd3~\'-i*cs*nd; z*H+c~J&P[xy4!L_ɓDڐ6.:#DwZlͬ=)FGKRe_+^{ e.NFbHgdJ-Ԇ(#%NÆXeD:.x)mP$PڀL|H~L1=A!ɽդtXb IDAT&D~tT.A2 ](bUiZ=.cD#eL),S%J$EH Us`A5q =Nj@Y$+'H'@6bf54<Ɇ**BUX"^&s[6MqǤ?6-isWRD1;H}@JPmպ cSJbŔJ< - hբzw |q5= P`͢OM6~jcdz% !EC"9dGN>J2H!Wc;x[G\pkֹU2ep%#CAJC9isY|sW_|ވ|~6oYT+!{W\kw]]ey8*nqmN#jAW#HwNŚIU;"[MK*C`c/☑-e,P-,f٬L+pq~bt FS,{򯸵 Á^00RՍ [bGʴfQYPR ksS@Ԟ"1G=:ƅؕŨBTV1frag%?b>P 0 QF-)hwr{%V,:څhh-0 Ģ螃Nr ׍4-i6Ezcw8=8h֭[qƣ&4d c3pp%B,o4Te3fMm$RAq+Њ&01W?r/#dtbI.F(hu(Ȕ-NmјmґU lcɒ1mJî}R&i߰9q˸mX115_$IeT13)ξrT.rn㍕\5D.4V+ff5ÎD? :]7㑇SmuUsejX@1[`eY/IT|WqLdy%vʼn`d8^|ic`#޴DV 4^M~_DF͉ν< dg68j1  =R:3ؘJ} ߉`һ7nǝ==\ƾ*U=uޒ} 3AG;o\ P2h"l4,i/+D97>q7 ^g6V >NoԒBE;'S$gx+_|>pi3/)5+rc<$/`="!*،p 7!zۖoqh Ұ2Ux|4 rRLFSj"@ bAL,xKWUٲ4` Zչi)c-d^ذŚ;)%c2^0\@}:QѓBXϾ1ʄD//]A/9iv(>A4z'v[k|u"" g4l2勭8Ei GQu .#)n4Q>/״t!hu5^J,Ə})cY@L[V|xUxqV%Z]dnРzTh/J Zm#8Sub0Y-t^rw0:V90X\ Nb8akZjļ<+rGCbnےmW6 hq^8À1bgDUi'%ʒ)5yܚSB%qb2W15+ARN`F.lg>1%FMi2RQK^3PeLdKj$XNDRw4EWj۶3Kc ׬܁-߆kWĶ]ٌ=$HAmvH3Dȑ7|>%!Z52K! ˍIfD+?!3xV@SznR̍tl{څef?!ֶS#5fTec]7pܡ8=C܋̩a@B&oWb[on-s oJ1 Y.䥶1DK5C@cP$HE^4P~qP$7j8F%]ͺUV퐲~A>p&:V(~6&YKBC!K37231pN0$N5]0fqotc 584z A%:}QS&%20R1.$ƴ`ٚGx6LfqzFur RFVl<"br) 7=8Иֈ/_皖@sP8Vl  f, h}4Tm9ܼɴ2Ea]%JKY}n6TLȖJ1 igpZ7e|dH;RK+c=dvKdlZ(pc(zE>&3^>$ nj^2QcLY#憍LLLwB>`t;I.0[%%띟rIsҮr{Έ͚hmB iS0ѡB{lUɼ ?3' n0% Dgc8J,qn9OOkŤ ݆E;(*# x}gJXH1`L!}S@nzL˽y 9zgD&.*D $Cg[ȉ݅5CDh~(VY' Xb,kgdE5gFT`#jid R&GpB@Un#G CIKq8"CWkPfJ됄I I=3-=J/`\WbRcʍp̢Ml~Ibu+vdZ>-tnH+ޙeLbaؒ&MuKF1̨Lghg z@;%!CF Xgc}>Y3N䈁<#̼o;OV+ $!X| y*13y?<ְJ-5f9dI)xDC}۰wuO\VLL Z^'V{xү?|EoWb}X-Xkѻgl-g=E4'N=%gfrvVqJ&)W ==h8 )5IM7/ ǻل%fpޠ[^GȋAjL 6uq9Xm]7\XwqD~]pqp~߀wlgbfAZr4|IZo=~vIv+9$Pcܒdz˟3 ?I<XGnL[“t5=F`VN'=^ޡ5^_zdYbz,gʩqWV2jgRgƬ(=BI{#ܟs v/ɭi05"Evލz6]9TBwr[xI@Xzu2Ijrt6$s\.ld;w0u1K9zE2~NrP1}sDqZ t) xo _f|l[`nl]ކW=+˸a+g4:aX^]r2VK$.47K}0NMԙ9aoX,Qb37М˫O~ ^ pO%} Kp P T>!3z<@ckw89~F`2al^d-yH@ J-0i 6f RFrqX>?erm`0[R!Ma+p7u&T`;)v&ǃuyZX P81YtaVV]"t)uq PUf{# Q*3j`ArN@MıF-###ҷe$"hDAC;bR6Ɗ9%,!۟cʐ/1ѽi@[ְ̑xk6Qւ)PzI%0[`]Ŭ~=A9?'r S196#/ z9,Yw:DY_E>:'DLn\(6*I"#|\IMyј+`1)JP')I$Ɓq"al 35nVe'j֩)hl3 kvÉTB0GEAЬ*8}$ٸ\!F``l:e3߉gR%%`:XfqlwΘ?ޝ3XEthċ*3 t8K\|x[AT$s'w]=%ۈrɑ-L^c)i|~(v3W2kLqc,佴cy-~00fxn**lbɏY!1B2[FsEaUh]6b^eY6>z=Xj1@QC(ět]LSاuuNퟀ,rN0I? MGVqZư}?|瀎+0Ȓ 1%dV1DL#S,un0rXt(gگ)CnzL=Hlj0'ٖh  +Vx@"N)Uu+wPL? Nkj$4\Ubˑߦ`ճe'Alk4ѠB! X}H;zVTNY:ڪ~!4dN|MyM&a; Bk.Ds.7Md"F"D7Ø1CjF:,XԜa]fhTȌDV(U`X-#zBcg߻Bq~[p[4X R(A x.8v;dBNMW}[]F}l<0 'ZIb脥xCy |޹|om`/ƫ1CÈh0h@_sѸ}K5[D}!VI\=0fIgUz='U,T ʖYq#8m̢S]$hAVc3˪J;-! IDAThfZ@ etC H7;b"TӺa)8q˼p2y=&\!]فϽre'<aJD$ټ$ 92e*h;yai׿O84)rmrTG%$㉕4g7El;PkE0ՃΑw:5S>qʐܷrtFkhk6hF=Nh q'LL`I%XR?d=C S|RFiI\vdKTӼrboR+Nl>:oT_ł0!( )]A,peSӿ \~? Z>_ ٬Ǚ3ָoѮHl4G+p,c߱&?\ay>ѲY,Sĩkݏ~1xйǗ?]W|>RTZdDWa@"Bp ~xdh|H>fmqn*V/#>0k6G|-FV#1m O/RroZd)MXX$oFJk?frTuT6 404!2˺Xb.`yǞa+ܯ5쳥o\ {.[Lϱ"2j ZSV,{w ƪ`v/MT8+1$ I ʌjpZ^1 .l w~ ) 7=[ 5$=k DRcQ&YE!Č*&';^ VΏJC)2\ D:5. :rhDT[ɉ Há m;o%'2Cv!dFѢQcKycMj`NGn =Y6$U|HqH|V_rm"9&hS)cD5{s x[_ZtJfjHacNk|H .0~6]͸m΅#EeIF>=àezFP~`&#˗?*W*PFr>Z,^;$ MND<15]ߔS#!b蹱n(AI-Ֆ c #5ߦCq[(Jhe<EQN- lmxۦS@ ѽ⧺Nh-'S{WkIpvY%HF?VNDo"J_V}(@8%XT7O3SR1D<,1dhH$pQԬ96bK&B{@X`+۱5SA]A.l<&9[5.֔)WP&N QXHZ>\T:/ ̂I*N(b\T{rs17T :>}_i@qzBk z3 ,8gW321u.,u!~ފܳ19iiB B5kaS%ZX/ .Y7͛6cwa7\Rѫ~MߌW3o/Ş\&~h(*8'-K?@ NLa[|^_"=:l=]RY&rf zm]bX]FWQq)O>S'8޹llXb?I4 OJ&%bdSY3X(cJ)w0nHXqM 6Н&zWRb?DmkI88 ngg'"=^  Ӱhΐ&{l朂 J {e\ 1=Uz&%0#q-E(U ([>)Rv5:DT Ȭ6 Wj$l;sĭ 4;@iTVXj^[[%Y=EiFC@ ?BD)RV(dr"ЁJF"x o):i*`~­58hfZzh[uwWAnV19eWƞ%JK/PoJ}ңpͷ˯0bb•4dR9\`E~Qxx X?: 8Gtq䏜^| vVwkGJ^I@5dd96밹?ܼ7vݘ45%>e޶˃1϶F,-n.u1| %_N&fH39=杘W9pip $DٙjGuW|R R s(zT?HU e35D& +n;V@OSJ+w*na_wՄ~֑-Rsdˣc5+5H/P n_DÊe͙$sd>xq[҄VIS ~(*VcקG NaXbYrczL@-t&,(ҀRb?qeQ% W뢸Il2 3F(Pb_=&AHbdnA*{Y%c$͎S+E'gx+D*vꜩTPFGt) 7_S2E~*$N5i={'fl~`~mfEM!;ރ~Td*։mc‘G{"GU[`hϵ6 IMxl{ 83ً>rqYdR伮3_bѦlCчa睋{qɗQ?}9^݆%QpH $75Z̈́Aҙgpdg]8}s@މ^x|KuN"V"C I lhs8∣#No#6 &TZ\*qVL69En$9+6@6٬^|Fm;63q@5Dq\d+Xyjl?զjaov6񞣏ZEɡ Q AJ;Nl4I/Z/ܺ[A?Z@SW4GU,2Y̌X]̂ZuHěykojHR¢ɜܡgs6=%㓶Thjg uL1=>#1bok ^Ձ*HYq"+X?? kĸr4$Hy[fc8xV_+nL 8mNLܲPl(6l9Nc 5/Q9P<4¥#cy<󕿃yrd4r_͑Uϳt,hdPܸmIyو?z.>> zx[7mccVRdU1=,@Qf2</BWO XYD| +nǶ, W c׮UM@ ^ [uq `CN\g"g <j&0?&6`qKm, UcbCEuUT,Q%m"K3jJ6+iDuI2ohM#o nk5*{ b}>?|G@&$cD')Ttq@O/k±NSVPB! <$D~Oɶ[(on%#sI/S@۟o>cRԦ%7&BIoFL|8Cp 8O*6Q2ͭG8L-9V }uޏ8'nVc&93"BN$&V$抁`=RY:+/Ӏkɩ>g?GqV̂ٓq1ԽE#X /EN`.cB!}AȨ%,lVBɜ n|on vSrr1CeQX':}9]n'n;KXz8M8'$//0fٱY?*(mĔ }.D.>m7܊JC뱀N:Ll /aORXx(;k7I yf)?w|dS+?ka~bLZΡҪe禠E~H ٤j ع;fpW.߬5aNߙv-_Kvw9kdfU)rvb1H K7 bܷyTѥA":54# L+%֔P,H恒3Kރi 񙅩1#lՎunDŽLQp5 :/n~ M1~1p#2J#)%U[-ܴee J)Vi̕];Ɋe]^EL@ː VU%` RNcBЋ$pR\6a䢩)2jkfhZVX]YHhlIj[gdNÚ{)*е,hanM @h y;$4bah6ӎӚ헄ѐ!hti'5Ir6~Ž_CĈ#>Usݝ: ڲzzEԴCj(Zd<cӖFÊ89`05}WJ+T9]rL WD!dl5DŽTS̐,tH扮c]jN5N^;M`9!9a VmHC!_WXW2 C1[{V8j`(U XC0icYZ\Q[Se\4CNv!Ӟxe5hļ`w6vF#jAS N[**5:jʯ,էUjY{ Y\DnJYp3bAUa B *Ⱥ&+XWW1&$lNĺ_^^hbfY4:+h2pAKzH`[~NޛAdǨ: њV_nju xΗImQ2ߏ ΢5q J4(qbl$LN)8٨E5G΄(`eXgѡb}*X)`ĖCBVxuxxח?gF8aX]7<gp|#|5Dl,}<#P1O7knݱ Y[ufg[c4Ai[T'G8 Mjk5) 2`xrZ@"&Lt_;BOɐk`0?Fd[{#ަ~>C;y(K kK-$g[WMކ.X[sn:3mmcRh'3ODq a:g4`uJuQ" ⑌!l "~D. H=RshCo5^u4YA"1|cb#DA&!]1C9cTT`0߈q"Z38H% >,L.^󎓉DBNu5!#B7۟ 3<7XYܺzψ<tLR.0 {bƅ,_݈w>{'x؏(~G`i^|`e2TQ, uѥ| = +cX[=<O?_'/λ#[Fu1Hd^@ ~i1Mf4{NVBrd} ُ<{u߼!氀f!4ݤfbs$f>N b'%jr+kZ:UkgMvz01V<&Z863}AƪM)AߟHX)in0mlmӞ9$HqQ/Y -7Z5`Hڌć7SY^\T,"ը |D1E)A9V-$\GV꺪a^^(ƵQbD\*66 &w`V5uD0W3ccgj1n!FolLdqKA]t[b!SӅXb҅q<2W܂K S=W`S1`**6 IDATb|ø1Y;oنnنױa[p p[>.ccr}:X`}6leU Ǡzד3`\/ _Q(ϳ\M9Fw}N?:suK lhKbS^ʱ`]6 |ð;p״T (b'wsYmUbƪxƵv1tsLT@&ib=*TXIj2dpnjfkF=Ks4.q,e$.r_f,u!̕دqn(@zxH߆`|G^zOF6kȼr(q$=qJh˪j0c'-FJ|kҾTT2硶p[5N1= @̸5)&R%7 5d>V-4P dSfmUn 0A7_%*hoqaP{sbd`3iH!IFbmx%m_=[A?hb@?40M6]cELD|$KMEʅCrjxgFm]`Sg^/%|bw} 0 "HmpEF"$lCD̗DAHA5.#=.̾8=|[ArzYψp'zi~嘛w1TAʟY+{p7.ȁcO{cp9 W]?Q`c.b1QE;s(g1؇{;A+xc~9GpW._# 8gUZ U(S:t?r@ &I@BKQߛx CqXw{{gƍߺ&)xh%+)Y?Hͬc MJCTY,+S6˂Q d 3pFNXk*+6"aֽP A~ٺ[IFYlu(f\jd+rZ[kHM: Y^M 128Ɲ)'RVV ,ފ`t+WZp\"X- SбaM^qV@ĺq76ס@3ZgS~>)cyd!L2rczLA5v%%zI+D 8GȺs2zIl(D $Q*W6Jr34[g*ZXҾ0J@F^"E/n`\ "[ K*%e-vxpU7+8(}98gÏUi'y]| ne|KqygGo}Rlⲋ/ \'ϋu_Y.m9ԚR fVd%Q % 20G]lـ'~m>;}#:`6[cQ 2Jv,5zcHKPuEfJ2{ݺ}W=(Tl Y29i-~MQeЅ(K3׭Ě=ںmy04pnFAg 3aƒ$56Z]P bn"NE\g@XpVŀXD ()kٱS`IǔL)GvU"!E Hjc0ϻIeuzL,˂~I6vZ@EWN NIdRwD h4NJ g+XXA@]!Z+i L^[4W&W([+:RN\,հPm0؇a9eJ* {w.61'Xl9ڱпae :cgc7v:׷E.0ތ}5==?mK}9Ӳ-eܩ ot|Y+Ѯj/+mU:>%;UI9Huqy>+wݸ[>ڇ(P#Hi1zUl94?eq-q៼ sƙYWiǎ>,} h}XQޱgqPa*-YvcinV3QLdܥ(Ee7ȪؤXۢM%PINE 'OJ=7:6Z5V!Z$%˦Z[ [+ȇK=F1-{4A',& jN RRS.phm x+' ^sPkpgJ-nޱ1PK.b:AEA?<@FcwE]T c %$MUU%Ui%168i ҝ@@&8!$l [y2NJU*c[wlǛ;=< FJo%G9 c x0{)bX=`ƑOR_`F̽)F8yڥ]iT:0\uLbu!_jwlءβ8tFIt֝LI"˔0`b[yl{yc3P w e{QS}.v$%bZ@n63= wG{7|8q3Jif1uG ڼZ8h#f˓ן}%s㫆qѪJVMBm}bMF̧*Hy!|Y-&)`m9 ViCG }L&A$=Ȳiu۠$댪r!]&sWK)%C ţV aSqnk#0Nnm,JVj}M/'cx(ӄX,H6Y=EemrIk&fdGnv-Jv12+(PaM`6s\ikoJ2 (0;.o޷a;AY[pX/ `µ{x2vV~bJck(r0aw1cŤcb?XOQ OEa{޵58::O}QEWx3."~ =>'N z\Ya9cJ훺bG\1KXXspefVF j5=(bs& 9~CEMI&L58b'ۂe$=kO]hWnOƬ `uZ;@'_4wVkpfY%?a"Y.m3S qM #KC`ϼz(.5i xfZ2l oib(}*xT:LwY*JE{'ʍK|M09 7?zqgJ='qmJF몉9+4(_-fη’>\ĂL9ukX嶣7m[ՏZ 2x -"XԵ&O*(a:zg-Z)bzZ"-`#,`0f, àϠq-"f56bpiX]T\d*SvZK*)d ~X&63Q\ƗW\;+P٦\0&ě3KA̴,ce62`#IP[uQpj(լzJ0M6Ft_1*|};cqy׽x߄ߍo Vg'+ }?mc7q3u%c%֊Q68c.3.KSR f ֌cnK#cyyف+ .]\@K] ;J(eYo}5y8!;xq>C/ԋeS6u[f1 K?{"j>dтв0}UB$5{i4ejiI[؇OZgew]q6iqfV+a}2 Mtv"ҦEA ̱ >HSbѿpVYBX `.:\g,iHӐxMs-da*h]Ĩsv1i) | Ai%0 u # X8pf];[b~&E"Dsj= 9'UblB80QG*j[Yu ut2-"}I)Nz *`L1#(:Z ;pW S @ȉJ踺;LuGU+[ | n$ЄMՒ=6`I,HV\); !M,9:-kA'kIW_X2? :!$Yn첬Z҉V^;# $@ #+)2~`–.t(43z FwW^M6l݈#_{9p^xA9ѵkni0qQmrm h:>:-;؉XnMٜKL{G'.zxw GNËG—>s;2ASWT G lLV@ 38jQQnV.,]&mӾ-h -:1,@ =,ԭi}KFo >|u uޤ(r䉮_uÚr T92ooa<=Cg1+8kXmDB`H  h"h_Jۑ"X`)=e6@J#~dh+:ծXHaKT &s ?,jtIP;@De8yE'f]3;I'N cRBʪI^$&*>g}yqyq16nߌ}׷bXY^n{}%c}Musۃxq+OXq'rL45'^b̀%U5"<ѧ}ȓØ&~x ^obU{I|6˳jE01QSL3h$z|U&IRH|ٌ5]H0(`ik$3Flw/kB^Cbf5(hqu͜SWhǸQZ7/((4`/DsH=[ E_'`&P8Lڟ f}BIArtT[L)<šc jd%g(v JӠ(Í zT5ڬ{,#u}OfI:WAdr#0NG s IDAT56z9UPRڳ~ /c̮FƵuP 6͝I3&ERb R Lh-%umY2hm#)^ LbSʣouS ዶmVa:Ǩ1٫ @([2Ջm3 # :"נo5O uu7sS"HP-AӉLMB{R/ԋ'poK/qKVpʅ8CO]ngN?'`XXa%8JE($,p6 "NJ2(L/k,ǺlEEկ7ߍ<©0,GEEXٸ7amq~zy;J{.IazdNr; y.޽i!UɘYTdMEnLJl\ZA|*EqF?rӴ3gϛdF1ڍ/:qr hfШ՝>Su6O/ۜ@h|" _R;Wb.6K!?RגlX}6{sV `{FGZ Ks{PDk{L%R]ut͘QXvhm_x\U5H551-f͏1?n@ܙ/ 6Y%Tx%Փ2xhXnM6?2êrvgZO QV7PFr5볘wTm4Ȫ++rGp8QY`#4UU/)-BAuNG{:̩>cT-EmuGY\%VOKhc&= %@%]\a>6˸ ,n^~Zމg/njh7}qUamG[у/4kzFNb4M$WH4HV3RfkRf1(w6:+,e(znv?:s͙#'xxW"̧Y,VNDȦc˒2MpI[.Z4,>Lϗ/$ qYvUP;IiJ~hW\7^c8=X}ށ;o y_3XHX~ cJ3]jTh @/Y|d $z`M9" 9Hia|TΪ|)kF1XT-#Br,P2Ĉ )1%Z9h>!nŞ1N taZabV ܦՆbrMA@9SB%֐L8k@/,} F|f9PiNMĩnr;g(RgL2.ܮ0r5[1evc!^]5]>Ǯd}lΗqx(W]VgX Qֳq͕K_SjKXضUЌMMX9vG23P=-gfvBnBTHh)-뉘h#k:fVM4v+H F'^JlTlfx Зerc~̏'6vs]`0! 9 PDfɗ:p$LȚYC2h탞ŤCrcX۔wK;߳Ymi,jOK^+9vJ^{Nw.i&`rU!gC̉B`r!hch)ͼj,F%κ Dbjmrē>CBa }60:C|aZs=s"03 Fg?tE} Q4Isʐ!wIR]r5FẒEؼ`TqXCGpx|A^@Y&, 8q8n߯z۷ Wx'Tƭ3%֜4 j4D)1p_tK 8vV b&lCK2|84:A9Ę xa0_LΒAQ)7c#NHkN2L 8O0TGNHZwEpp\jV$TՑ]Kf.<Q,꽚 C>Ŧn 8L%wqC)uI2rk=d%zPq*C !^'_ڵ,#~0x;\c{iu:YbJ,B= /KiE'gCy : TDmH @RKŪ)enEH5Kޢ$L1LIr\}Bn~̏U<}oؽiGXogͶ;DX4/rM `z*Ni:# ]iH&d-߀7[zQ*Ӥ˶8󀖲bS+-{#WXX4X؀)95p*ɓ4A2[;# (ł.ve&2qq+Qz/zK KY+v\?V!FY9#blt"jǬXӞi5&16"YiZa3-mr[VR54+NTzX/a{oK8RNCF1Sabb2_M}Gs(I H\'5Rp9kko @]L \2#J+ F;C$@FFcLYA p]A"d%؋a7!0lRB >Mb2}cN  ŧı4<ے8VneL?=Y7R;hj7+/V9Ga3N''Y.@ Jp”~< #NL* =^Yhm'>cd^ &Kҗ1||vUxatwz #.012ZG{qd02ʷw6y X-aW`W(Xڰ7m8of;r?0~13D>'BxBC\te߄:{SzS zK6ؔ Z0??/D61]Ʀ; ;3lʗKYNj5Q>a9F7TObq4عV_3]_HkPf>U2@o旝fM %|M8Fe0s :(6mi6d- 3Ɗ8AaBG/88>1d sēpO`]ed*mu %(I^cIr:<$Z}.`2 =gJfv.C+!4C 1RA~޴pg3ۈeAh>)^i5>jƊ0R2ܫMcKÛݖ@w;WWN+:)VUt uU:uIf"{-v[41Z0R?#ds-"N!]%D)h M9 \D?;)e8T4@޷neSЦtR}&Bsl qt_|)qݜ. UDִ!![F"KNhڃ'&FL3hZ7, 'Cr47 _m$r,iZw*06ua/=Jҩzu3JZE' U+M?kb[AG"]{ٰD;8sTZH9]0Dӱ]w]VvVbR+Km?S8/౏^cȦ%"J3FH-^]u {M=O q˟|̌7~ G<k_X/xuJC<W:l>+6m݂{o@O#~S"'31V֯G_<rVy+H{Xp܂PWߐ#J 6-e.Nwd5:M\ M]8IҬAx79p\-AmCf^S`0*Rn%. íYhj1 2 .ޅ ٣,J"S+5˰59v?Xk V )gCÄp;c.eeo$rl9 e UqU#,2LmPgfɊ :feƒR]XF ;0_3PqN(f+0F3*acǢ/Lem[֕/ X͍Z;àMoLVKq^`8aBQT|V<64pƦnȝtSԚzmCN218ߙYDl#n2B碔i'tX뤘 2#8QZ(JFQ1؝] ^֧+H'H&NA1N ISN,ŅoK$rwt_2 p"8Cǀ[^!3D[*7,`Q`flޱ]~)οBlڼ[6oņesm^ޅ]SO>"=d m<@K9ne,E(__q!ށw%<|=Mqs,[vn { VB?=}RYy.kp Ib$]B C&$b=ug9~;?#wȘP$zVBk)GՅ#̄sG gTly"],$kNq'H ܶll%%N5ӟTG0LF҈n(u:؀kARV$R[ {kFRU,VLĦ(Z3]nGk('ҔɬIC[))Ʊlm(g ʢwOUing?+C4bkF"+)fRԓ] s@n~̏u>"bk*U!> 6PB}T4'7e2Ai1pˆӂ&K$l@9ƹ11PN`yשF[d--ƏjjP܍ަӤ!Y*CIIlbCE7]H y)M%C$r_o`>7 x/u"6l G )8<ަz}[bE+[p^)p~{~b~1d@01nß߁fzu)õR1 'U):.XZ^U7^ktյ8X?݆^;}?G{qȲ Yw#"JŜ4e(G9eS QФU2 e#?)q_6Jbȹ6˕.D4_ -&hXn; TѴ4@5 Hĕ'N IDATg޶{D,i RmƖ^nu |.l`x0wZ D@CtæCȤ*#pQbO"n[ )؍HLs^΢5aʍ MZ[v3$<}.8ZQt; o$SVN:w( "C.D 9ut\>lMN]vэhZ R36 tpC}&֑$ԔTxug(LvcB փwAYY%1 fчwНUrrq7:&l[iZM"gލ ؆1GuIIfkfXQ(z4{M%ٖ\dCM9Ylj@9VEd Ȳ rSŰi7ep*cv\|!_7n–۱uv="|=xއ1 ѯtzQ>XzXz!0>aM x|׼؃}O?b8Qf=i^{V? 3j,fZ?D˗q6\ۈg/b0NkHh6<-@=_Ke+Q:!I%EYK \pnx7m$>.\ي][1_ơ <;<8><7ոq>p¡}nu4zy_69 Rpѣ KndxZ Q1u`ZWdH[Rj]a_`pG,NьVˋaDEݾ N0VO6[~ xI'yBsj C _kP$ AQ( dqBINbe.h}" tHQ`V&ژ*NEpt 1>[5]#k] `y6QQU({I>_cDiyva⬵3ۈl 01:-8q`tA1[sغ}6l߀wDi 8r0,[A͈!CNeӑ\r1(Qdu`E]y)w'Z2 @858ko{H[k=Ql}11ʡPܢNV3bц Y//®.2ncX=ʰ-b%룗NCúUQA1kzu7aeep}GPD/!0`b2-`}%d#(pV '@s(=  7ŕsJVZ#9a609S$;jn[(ڠX@~}Xã}M/ pP̞8o95ף"όw ՓZ KI!I0!$ #(ɛgc4N*+X1QΦ_PpYk*րd+ltk5v[i@.Dj [d+s1?xI&"i)eU +C%3idڌ0]]D1fJ̲Lt*˷|jnWsypjSnHU\y3LX9AڈE%@q-Q+xbj?Һ60-z)+W[P8ݖ(S J#7,IsO͌kDIi|b%rc~̏ѥ4stΐa#-c6GI͜EVWKe]w?"-#-tbI%hAe6"tJԙ(ظ++`)wϜJZL%PCXւ,ۻ ( eLS5bT%7&|ӒrWUS#Ab2@mE𨾋5 USSf GJؐTb fj)1epD7Xx)DGe,l R J T l#YJ!59ȱEm ߔ 3TMrw>| +OML`yex,a+׊4-Ū5Y-;wu"%^jUXbDƴi6[,#Kx2(|mvh CPr_5i}hF7~n9 7?8"_T8džGSQ Ac&fOjcՆ^hBk fF1׾}{KP:r_(lK2n1[8HPy=wQJcˎmy.\|qwֿ8nxO}pnaOsҡKz7+aeaŨ>q}zi'6ݴ Xqb1jv6.N2;=q8ΪWj;қ+%&ZNq B8gj/BrWM ޲G5hq k>N }5Qrᓓq\@d9D,A(,peO\&Q6dvg@3cxEiF+)LB5kh*;)g'' oZ$;\j܌D|/yKXsS1?o$S.C5qEͧZN[NkX9V>Kجl R Ĭ dP"ż\_jGHUS@iyg+u.̸ȟn{qѱTk7kpn#iJ Q =C֖5 NڵKT1"|`!P ,KrK8U]b<00f QXT*kgw* ;[띨*7 RAy _8|G5Ɓq(V]JhmWU)l[ۣ;nfd&#`tW:|{;<Уx{pidY2 i\|81Ƶes/VUl+qVCrd+qcHft1ۺ;/ 7UxS Gz.MXGu:D4!:[ .m5K;p8k xvxG8U 0,GQ,3zѭ !튥(.XF-%?Q+ xDn$ 8i,p&:@U<\:'Z je¨N?_Jrwz+O|H2")dž5[ĉrGk8VL\Kf( ص4R$q4k9 taN0\Q4Rߣܮ BZx#eu~̏9 S- Q:}T}&ju8+`jf@n{!pR%æeSA1anh#S',,A4 `)3 d#$$-fkIH`6m8FcsQpYy%bP!킍A\$WJw7%W7,,ٶTY׎pVEFnx%  |"Ya9j9b[¦m[w   .O1֟[/ށ]]W}?S\zFsv0O^P%]#pǪr]/`c-uX//pt| GU*X-G(.zI3. ^ƕ!` F/: G<(Pܞ3 :p$ؒ5hRYeu056c%Jf_$]3wRҚ+9.)OBgQk"h$E<3:O4 .uSeXyF(Ġ,11AB6$~6Ѿ|֯T()Β-]ɬ_Ӗa2z]+^ 7oYc~|';Ym F.L6T1T/%tDt-X*bϤ_g ]Gʿ2zve:]1j **2F[+OO1ށHSU#IH>S  (C鳈)c2gQ@qR;@ ,w#P h(=9jFtՎEN&6;eـꆧk@TRDMK&=1Jc-3s|n= ۯ#spzt o?g?1}I;MXv-N<+k/LZaJ$tCP2E>+'Mov._&\r|S[p0D-K/kM8A}O(֋B!wlJ=ÉT7[f #ɧj v\bXqr>*V1qT1U8 |)֐~Kd͞~.(_hg(?A.h VObIt?7}ʱ@9 0V|Z9邎O!6aJ͐s>LI3 Wz`O"'t"w\*csВ{VKc'&Qfۦ oaC'jFT"+=5Op*J̿=$D-`g4`o h/٬%uxTF091?YO3W6ڑ9"p!C>tDVpKiPd-U5Jd[.b5mz5$⬾F6ѝĸ:=$~p$6NjjU%"鷕_~GlART=i)Q)6$8:ejmM 56VDN'_v6Ri(D\V`ӴDLto?qڮeY_IM7 )4cfOe%Wm"dnK|<ς]0%2`_EF~*Wp͟Ý7*v]~ $ מ-/`6HȐldM 1 GMڒ*ap@GYѴd TiR ]?gN~e9`r"nR'%>+m#CNmM{RE?jD[S(Lנ]9|9MyX1:M6G-AѴ- cj <ԃG=l-cf\߿;?;ӿx8y$2uQqC(N-dBa**Aː @hyRG AUlRNUpпHTna6deOM5;WPi HzN/@璕[vL)> ">@]@#.$ώOcǍyUEPMb=eauI0l„ Κ-_LS_3q<8iN>zܑ3g;IF"v)D q vΞ'.̈]b 6S@mjiJ]pK۷KBFcȹ+A.:Ukg3ov)޹x.Ҍ5c}hc"A\V-uX0J]\cD|Rk2I7?MF. G"IIu V4gnH|#Ĭ*p-%[>tN[ۆI2:6OiNL$(w|$վYLͪIqw3R` Qҥ 0&{`QH8J'aQV6dbwZ];]z՛33˶߲ADFJ2D# )n[3%[.0hn~{aE{-+<ɻ![ƒ'7|+NC\~} Hxa戹R3ɞw8v+؟7Au eI)pqLwm_ܶǓ֖ n] NK1U6,jּ Le;YU@˲y5UbT:"Ɔ3mٶUPU=xo$%]{1 aDcx1cxn|GOb r4qeNƽ)aKp'.i2#}wt#wBIR0 "̜mYȁhD>L#(T(ƙ1fFV/^biP&Fsn渉k^fy!<2lPic#ts@n~̏1Qz: K8WBZWFV8̶{RKϵH 5>Dx cN8ji9Κ~v$ZÝìk$lv\@DKM鷭d6HvbMklQ~q/8`YY&?'e'9l:H@&֤ -@©MFk1BvɄ dۃ Xdr‚0.TWdIA4`ˁ:ZJ$OKqq]q7lڬ NWd!ܱ4b5잂 lܺ ׾Fo_brNa*] P"[#\ꈎtHW)K(bݭX!&?T';eD9"tٙ^uPF IbmTٞFf3︍/ DYJ4)ۢ)f_o >&KF$ur*3&0&uUn훾D )ظ )8UG^aK#&71L9&^r|0Bt8@| as]|mv=?񙿺'pGn˲*-w`\@k2ebǴwpeд`cA.)1FY VyqpV WlSYTNc& gVŞj"@ٞrdw%k9a4QlXE_e$^ q-]ԎOZrҭU0BZIՆȱ5iiuگkr}ȜLUVHm8VXuƮK.¯ܿ¯f`ya ˋߥZO r0/JD"ܥw`.-. _X}'?8yx ArͶLZY@)wvXX?$neʥFJ02&囤h $yUɬ%p8,*6s*@H;FΠ9eX,f9efe&'rF$].uH^=l2^%(N#jп@!8z19< .a%Q# mJNaa7yZ ;˺iFptnN;9ͪ%@\^^2XLqӼLj \s;dA>u]r<ױ{ӎZ+{I\R]\1K]hd,R0 JW"+!@]ؙͶW#ʵM#&`*/rv a`k/D{\Q. ll'~w<8N!#J3@:,Rl!IDJ\92%^)xJH 34(QVgyinH(""M.^܄Ã38<s,?$?o9y /3l(lH,u?'QtPޔtvU@ɸ'B$;{kdX1=VcXj ,HryZs =#]"L v-Fr-m+~Iyd1?8eљ:ܑεB]\\dbDVƵCM䕁FV;ʾH+wTt8 UD"/$6@RY?K篪,e˜ҁ97w =EHSfw#ƹx~OՀ&$ML@MⴔTA4Ye^N$5ӶgV֤n$)-19fa](qĚP >7^kP̚@12ɍyVLΣ{<0eo hdHΦJ )ywq?pu 6܈;_џāǟHAZ#Ga煻tq2!d ĂvGP~nc'ڸZG79=_Ľ݁ow]YM{FfB,2adIz]:E4bqXvE|}؎kϚ[qsʁyV(?[Nk|Eb4è gU.=o)Bڥ_M\ԛD_;d],3{ p!ӲL6~bvx1(0O7;o8Yd}L;grTڋIނ9H/.'c| :22Nj^wB 2Ό}V<Gž]VwAqΐ{p< ޺U@(DQAEbP۩5&Q6iN&Qǡqj([$D( ja5{=5|ֽڿ}w>PrӭP1&5vܶZ .qӛBĉm^tNɣ;Q$5!&{dH>n& +FF_-H9T]15kO ܓ4CRõ֩c5!&1kn\O@;W?cdQGĴS (8[5/37`loȟ7jeL.yg*d`Z9:M] 9RDmöFM?Gdz?g_-X_k;%ze/˾%F&;*%2 7ZB:)!qk%7 /Waj$ز'~!~щr'n-VJW8kbc%^ _sRHf<'fu>^5R IQg,si'38?.K~\^])WhΣu[l#2k@j HR`E P~_RcQYT稜=M?S&Ϝ~iyDTn>3md}6Ŀ D+{uNny.%&$n\.hehوSr|Hy!vWB!]Tӏl q,# ]*!؋7L)L@C!۪LɹQc@`S,g@^=Zmp j?%OvdoNM fv;4r-Q&IVt]һk:W{!%%foP7߷-~%&H?)U07-B8OcɳdT:qdaZ2HduHH5{Y4xSns!T//'߂i̥ʗ x'-\[?Q5y@b\P]-RRSP@#ם4K^ s"f'V($Iu<聱$FK: ݸ4jmZr!;툀䂭#$|;"NJ6Nپ;2Mn4u!b۔(i]>6ի+ ۤ⛟6t g\gtZL[`T5DLҢԴ֤3Y 8&!n;=^GM* MPf z,P ȩD\۽Is3J:S~HRc__w~{[4j4x/_se%X]]4Ln(C76vhߵ,E >agp4Nv@ 8eyX,*ؽH&Cqw?rYFnK`hgiQ!gaa_ySLW+6"[pEӝ+kp-8,Dg`&+JegaziT#T&ӌ3ȝU0 XM_J"8D(a!Ĩ7;]-021?9kIE5g9܌pY`%/A֏gݵv5gsܪo C;Tz ߴ9xnc$6I/!jTHxHoՁ,6ϴܤb*%,V`poeh+`۟ ckҺu^Iɏcۍ}[cl MZ^;A9F RΘfu6۬qsr(')Dߧz+K&kF ܸeɋLeo . h 9jHO \\=C^faL2Tmy >s> uߌC48;d,ZtYK/`ɲڵh@Df3`{>рRsO $'@Ɂj_mT-PZl.Z`#83=mbe(?BHKM t{e ))'ulYuqj L@m`Dq,:礡l3]S&IL Y U tܵ'30>!Bٷv-kwQ8dcBj~[U.Ǹ{$=Mw`^ϖJ4VA1},c/M8SM.ԃ\Y"C_mGvu2%CQh<8AS_ki-b$i-\ےhoz5bHf1[E2u˜Aɹ1O IDATMjZi9-T.>PĀ ?ͷ+Ё( =So a7L=>Y(+l@Yp* FB& $s 7lȫg{q~Ks8; 'ފ3 <˾O!McQϖtM7މ .8e0qi@Dd] ƨMF81Ox\(&RWaf`p'b5 >9Ο|rad'CЈ`df=@X43:Qo'vP`(%IY֛sVŞ%H{+SK Pnsʈ\El6D\!Ѣe o6 'S%.b"bɠ'$%)D,m~4gxXLP@];31"J#)n/ $j}` 4?/EUOS^)Ԧ|Dw"i/9)d,E.c_MQK͎m^4e'+4y3Cv ݵvفl7Mޔ.Y) mq#჻~]v!TՉ[ 6DV Fz-4k#'T(cV0Z 1Ͷ$IeXמ0KvAGW:I z WOb%T9|-dhݽBP\H+3eJTXokĘhJ!m0dJ3nFv;fh3 , HC%5\Ă>҉IGH}@ Q"t9lN!Y [_ߋanY6{i~g' \nUcz⫟/q{OkCӜ(bc< sH egdCщbتEr҃c `Y_ RNӆ%K{mſyַ/ye{2ۀq9ZWG[I&[m.KHLXg[4U#`V c1;Eޫ=yTj%φ 4l%N7VNm^9IŤh/8Mj|rjԾ8$Xq$fZW I'Yg6o0y5's Gt|.VIN mJ{:ǂ^V |'Qh۱3BV$ >ς :h*Ya}RaV֕Av ݵv׃椞Do[!oۢ swXR_o{`;j\lkb l5{jƔf#ؔt$.lP5 =qˉUZa[ݗW>Oč\3s=?pp_bqܠːWV["F9iBUwd80^f;|'j-SLn)f! .K/IiޭA jCVY@L)c^sL[XCP"95.pE~~3B l/XL F4`bՠT0dd&1:! z{^$洗e7>MJQnɺ(l7[_Ԇ50&.;I2Hya^Q} P4yn}2T,ٯCnwuvj"P0lc}fTf$Yzdf`Q:@갨}OƖEܡڶe@ߏ/c@N +uJƱyVǡ% mtl´ l˔ J:o%;ETrI8W4ܨ7kkгat,G Uhie/ vAWfaބ%8 ߘrᇅ> ڦWF:BfA9%h֌ I URHL<0T}jm5?F&VY7c{1y '}x/ V\i:)LqR<{uցÈr8_yjΉlf37ZSi1]eH7SΊbx~KKW9_y5̰܀'{렓^)45ǩ‰ldU7@b3,~,b4 .;;T*$-^k+.N-{r- 6 /5kNH̬āNjo2IY0SP|~fF)$ ,8j0>&T]%4/@](~.{V~9xt*kՌk00xIXB;Wk%&im*!WB8CBT8[ @ޓ8KXA,%9.}bhQ@R.$Y!%D{/GW7>xk3ǙXin{4J?lmvLA;"h9g-02q+Łgyc O~<wj5bqG {5 N/nKŬu,Dk9efИ4MI_ {\R-W.Isvz$-M+[jɜ ^:ѣ©ܩ 1ƶv9WA~C8!L_\ Zb#Jͬk؜g_. 5tL[<D]#?A2Q(UY g"xiF%$uie%a+yztjrkw]EomۉbsB]  ӈMWuLN(AqD=AL{2G3kl8+7mO`xOlp-M]W V?bA+KV:b2@CDr W #dyzS% :\' R慌Rrf˦ƾQm]ƲsNMp$ oUПe:nnu`h<جze1K{̢N؇zJHl Xa+r{s9{WGx\)Ji @Q;"oHΎ쵯oxm\>ZS!ofS5tNOb0[THu֝FN(A,c}.5qO3?#lf6tOǵh 6.˖={Fz/ME3ÅffhG)=Ф6Qa$@tي`2Mi /J!mTEr-Ył3r$f (:V*鯊r'wqgb%tcd5C SJܑ e~`ij2,d{/Dar [ݔL8wNeƮ Z=Ȗrc8[^ Zuddn8_ZXr;8$_Y 5&lCQO_]J`u񔒷*UNuݵ^*bLأ[ԤV塡~YoZ)4m^K 5AT [.ֿjd#D @յ6,..B}+8bVZ#>KX,Aj5YV>|$nx];M*C~ǘkӱaՃX85M[{T-ReH ( "T4_y>q(>wӬ3VO-0~ƴ Fj #ALGl1J6 vqz$>V>xԭp~9mq8Y/0)9ٕ >6wBC$S l:6_h' }>?ޒd@[/=ġa-[pcy_^JZ.oKx+;-.ljQqpg'h>@ mi6EG/.ox Z]^y'\cu!Qc\@浓̣\1/Mh9σ,lrg<, 3Џ.i) IΔ^ ؖXd5 4 U:z0br'ᤑ݃T(8%bɇ7Niem°;ΦT3%Лc`qz?o6R}A-j}>]a,\ץ{uײv:;R/~TAxc曳Hv@%. ~ceyƈ׋cǒ] * A gLPf8soD|<Z܈Щ8v=ihJvҽ/@鵮N;Dƈ@A_O:Pa9yM/KStڼw@n P>mTte!iliKc'A%kc\"& -c G xW36<>h`^CdJ{5@Xqŵ%\^1&o]7%.W-S| z;[8jOq*޳{Wq>QܸowUuQq!>> -4ww`ݨcJ7q{ lhI>&=;;&La .RBTiZŲ]h}x ]h}%]o};'fYPFwOX"bG66܇sGrU O\U"b $,Wc&9CD{nJ0vFO)}Oe F\D0Ƹ a)1vQڳ-"dL>ELJ;ɞ-{-o2@`$fO8rs`$;J tdbۮej 6x9O4ͳ}bxH%Z9Z  ]kw_9O8bɀIC3y|AD.d/Tjrd̛gsk*QH|D Z,G_G1SѬLPU^_GTpKY`NNdsI%J28h 1%3%QηEb .DG#8K)#3-$<%_a%-HQm/)x/ ?)ib.Lش5R_:/dFZŨ%ΛroY.&$* ;Rltk ?/@5X+7qlL\/.:_۟d|n) o fLV JuZ{Gb>xEs-nLV#8 )K8iWXqu OW>q:v)6ݹOgv1UKlՂXrO/OwX CwHu4o:'t@ Y>CL+g%ԩDXz4s,yl+8ol#qmF$?2Fk\ݚ1q&  LE@=@;5Y]gJT)@rpv6<.6?:F4(P0TwE9 HvgZi:,g5oBB$ hI;Aҡb/Ҳ:j3O:0v-kw3"T>rtW J|(> &ݥ$DR`)-1v]♎I=\zp]+c(#!L .hq( ;'q)u.qݫ{q^ܷ> WĨ;z YHMN~v 9 `}babn r֢~h3`yl-o ;Ĺ10lBEh5[s'/tm?H]?iYX:fqgyہ\"H|wT3$ -@طAi6di:P1?U P 2-B='6TqB_߁'!zO%!y'/.sBóN(c9TkZY寻?qda2+'\8rZhbͶ+y/?V|ۏ/֋Uۍ^(U48gqV _`8j7'8iWXoe"Ұr0rjҎǛiZ:.]/UQ2?2KEVUaE 9IM16^A؇Ѡ`/C,8|Pnu$ |Ƈq70D "DTe[9ܜBlT%a> >`CWy/X}"8;DG4r&;fw]zIqcaCB k5M3FD Is8(ۦ#*.b_ ,,ʁq:vQ n p>k@po#%KYHo^B$zzXKт>9W3)hSɮb1$YL `,GaaAuNz֩JgeIPuwI,.C38ĒLe:! v1dC%j ALlW$b@Z:|&F|ѺO&A&x^|\~7,V`*t6 SS G<:sbDjJjZZ Zg4`yzK)@RȈq`ffHRw /߿!N/ Nu&imkX.M +jWXc~\ڑ9,wnh?96sIĮ_jHy͉dߤQq(WofgH Q`* L νWYYClKcM1`ÀKؘB WD8z# pޢZ.P^( IDAT[+Wb+&Xf<PRis\V61$iJ W)@{ڊwkwk*E21jD}u2H2-9Gvw;]] rRĬi 1  ,g^rUwq/؉ش لQzmm *!ل1U)wٔҥkmYk@WL]E.@cZ $X[FJTSEY]v,y$D@$ }4@РA#MJe pmݺ1o%aC4[Ő.Ld Jl${CLd3nskv|gx_ (ULXpNϒMo^VɹVf18O܌4[ CK ML|7WI:}@) qǝ;jaU:'FP@V<3d%~_o{> >3A% xgL97 hjr#Ǥvs&:-Haԥ䈾sp, ?'8r@I{7n$fIݖV?L*@̀!9Fr]fD8 gNj,HYDsKyVuX?^ڣTtd2H6ɜ[yFI^U=tmqyC/$I93ߜΛa͠cN D02%\?IA-Bs5dt_5y:#:0(pԉ B,;@nw~%ujM70\ggm D9fin7,k!Py$bvY0>:LӵuC҆eg  {z!tp9َ-IrkTh* e Rq>q $^ ":Xʙs H'7)8V+Z,LD$\fX4s Y`ց+0IȐ@ e8XZ*T,Ş+ڏ(;{C&dzJl&\8h8/87~ xo?~ͯyY2UBԼhX7Ip~g}@f8hk_LMVAmH0P*, O,1Z-OKNpI&)"Vr__[^F|ӏ<<~7{87f=^" %$7*\2'g&$&x^hָ}5#If@wAIDYK@}7hY²,LW[KAdkRXj/eZcbdlݒ%(XX(g0KϺhVLBjc"HkZUgͼ5ָ#Սd߆m 7 .8̰0k:3Āo  0\jVf5Q7u猎Ӊ,Ai[$'Nw]A"VkYEoU{p-||W. xbUn '1J5PLE֐+alrb ɌzskMUEsh\+}&Lu˒)]{cہjQlZPj,i#TceC˯d6V6D)3hp9ܼ8 da˅5鐨J7hk\\#+(ee0qԧ+}j f=8;ĥ94?f0یon\2ji8G\ɲ_%r)ioXNʷ@i9=\]o|y.vca,P\Ocm؃'OY?}8yP*)VIyHPYܹW@pޫv<7qmy ff7pi~qiv7q {2^'`ƜUg%jwZ'dP(' vM9zfBPpI)jj&U?1ɿ@[u⎦caQɪuZVhza d\`uIiFo;˨4mLɞ1Х%㘭nSLjL:1P@0a3ã{mw I3EqtNOz6hjgxjC$BRlX{t9l w]gxMvIY>(P1!KѨǷh)V/_vFwж9jtUa)߹\lbK4JCFPJ+%Xlp]z$fc 21yb% E.C;g9)TjSbr%D*6 T틺e$? J\3je~]AHF/{Τ,p>qx<~x\[`u+ \}#ڹ6qo۷iF}(Pp=2:9q!.!]]'|uxaHlS^ tN`(9pǓT\{p{Q{ %NK,,PNn1jd>@f۷@9|WﻊKFG '[vH{L#,솣 jm > ͺśp3SZrH&q:#iJAWTu[(ɉ ͍l732/S.=qtWXMU}1Bl(Aq[af):ZUY*5}n/|;R,cݵΜzlƩJ94%wӲm%GT1aF|J[ii4&d1@ӷhoAmfv]BE Ga+48k "0]r$К95uU3~Z}GC͂Y9 ï1;UѕNz\?$A ]T<.QqJ AQW,:hXȼk mۗ#/P~(kD/s|썷=q({hzJ(0$vU4S1.5O ]K4r_*4k-#@\rM@fKxc/T<ൿxcAj( xbQ%xsB^oVBB8349'nb/YI} |[bEH YKZ/W_5_#ߍ}m> $|Mcۋu", m>@8 {lvEiR5 \3Ȗp,kPW E#^,)F01HEh?S.[6Ǚ쀱1#y3d\#w1M;R^L]u _)YLjzx)t;ї +EG|_tm i˿Ozqn`Vv}?ܶq]3x4aMYo;r2-}\'W\]kw=+;:X¥օ`G9o1Q)֣-a7L6AXK{YEa\(CQ=F3x/Li=DF6#)HN(UO!Lnp5IԈ\qC/bk't&IZd3CLK^٘%̛97 q]b+k5R1CWG+Wx{~p Vmmpt[8,~`[59g BдiddN4Ngi#T׀4PϺp{WK,t,N߿i\:a[G#r fr`l[&$*:"T+C92rkw•JWeə$8-"3F-JQSCnRLR6nEɬL.4"TpT d < u6|pɽD\6= 9T^K||[YMbSRF(:Щ(y@݌%FR`_Y 9k0j2Pϋ=p|#7ty+6ͪ3%+$ :7 'x{!r%Whl=<X;jr jq߱{TRXa,>Enhg߅h&z GDH@%"$+grؼ`_f9S<}WZPVlQou;fKGĢy<c ]X-Fj zLz Yz 8HA+Ր$< ڭbٮq=ş܍S\|GkAvnPFPoC{L[gxw:,(<,MGd5c㝳K W&)2Y*XBk|iA"C!+CIr `Kjs'0'㵘;kV-.w:+;[;n|L&N<6;1fSP75l'v80O'ickwݵv̯I9m!^Ҧ D aJ=~#(?Ǟ,@Xr /$.Tf"SU T{r2G'\a53m9: u ڙIa9uu8ΑEB=U2oIM1 r"Hu¸ EkX OGZȭ{j\XMXo\Ucg!KWC2L`@߳N5xӵp[nk5x}r14A;/sϔ.T={9~_\Y] H}䪛K&{I҆pDEЭ4HdHi_j8%_]8y/.{͸>ݫ˸ݜO@l:`^x@PΑti |;leS &[O, mf ,&OS2OL1%GZ]|oy0,!=12C}m &=[*;ᢊ cb#q.eCSEƶ_Y2(1W3l)iǧNErT*OH(;`;QwOBS]wKXMd;]kw]0w)8gin70hƍEǭXf%dav[7B(= 5iF! 2V\ʈVHfK9ܲWcwyWIt\-# NzQJ^‚ΚmG&m Kbjr,\E4PT 8(f0}kZa.8mhsMo2C&i @-arÞ%]q}Ǹм qs{Wpգn?Q_NIEhqԞ>9Ɠ9oz!hq=q{j`vkJԃy@ce'_zWݦݍZZ\kWc\1f`5%Sp mT}0p`jd^@Q2%Af".lxU4hʪ*ʍ:{6߅&Jh)ڒ&Zr9 p"18 s&/qs`hq ӿ w-^~+ #G&G]PDmOmHgM"!J \R  ؁wY]բpVMQ795J"i|GI> 8[\q IJc`ג}Mד5|~Y4vt} h !&T]_kH0]->tș+R!T>[IfEJ8 NBltM-%[n5RtIR)E m kZA0<_acƟtK撡2-S9o\q'~XpXm2/`)؄dФuS1Yb-G0Vl6uJϔ"#y͜ 7CPo8khCmV֍`5)ͻƔJeDߔ?8B PM.e1-)I2Жb}4gs-a m樧*1Jw9KjA(a&H.DW0kN#@peڕX& IDAT 5`Xfօ1>$bdXFewi y(@PݵvC*TӪEugי&?ۛiAm6M*]Ksq N;C vԽ"u@йGcgıuo/x ֽ";]Ի$Z!dxE uP%WQj :}ؑtXm=tkr;F$k׬551@lWRx ɘϞ[C4wriv4FiUQ0id\@l`L42<_æR$čs,X%n]kr *h20FFmLR~x+Xe1jXr+a\$n'aٮ6#(5ǩqGЏoqj*tA"S~0IޝE ɝ;gJ=ssg%QSV#4EA8bgw 7!f,wmU:Tdv@6i1W8ڃ"uK-~V,?uGJJe`{O3%Klб phٸfv(5ТqA睴c,'I2H6E[S0L\·L$,j IF Tm"e+V`EqekGYXEoeŗ J#7!wIZ4Ϛ~׏ovzX;>ji/E'07v.Pi^cK`\ G<)]cgrD&6w2hZHU̓\L3$c]47qi1Sn4cDXOs0CL}D Zh _k1Ѣ e u38L"IGʟ'R4t jRbֵLDBk56IO7)-D߱7Bq`4 GwV*ٚa&LtXOXLgI1-c,|mgXsNg,)`&3̥LF1şW(8ۓ`3u!-_yxϽ YSu&`4tЯg Zɍ-"@l*cE*uvOd,p-hpWvD`@0kf8levO<-܃{p.nSh57*Z- {n^`e jAF;h,[/uZzѿ[Hq{l ͈9sU"s2!FYoubqYUĀ*+K-s%{(Ƅ{Cnwu]p/W;t0ϔ2=d+xg!TJښ.@ Ür '8nWlډV\aG)&?gYUhcԶ8%XqENЂXb#$2s_#3>)[$\R„YCϜ4XBE5H@*gP.$50[5{'dpj1Z&tj%0n3v4zgѐ#jfv~T %vbڄ\D)ZB-PzTk9VZ\pPtM{sUhTo{ -ƽm-?i#0J9Xs5{f]&c:Uɜ>Mp[n@4segQ{P۶#s-^3尶l1ђUu$Uҕz*eIxE?>a%VH(ލagHu;A!)[+\-/ \l$laqNEnVn/ۦ8XW&I#4mKp\ʸs;kp?5J|#}=y|orA.o8S E{dQMyZ?MQ8FAM#9wįٟހ$![yS}"9[TZٖFFI%q5޷:o'j{vDؓ/QFp#LBF+G'ŚAkn"5LX_R}P,*X4 ".|R%[%X-3l c;lCOϝ.&7[Tjzvl>vNRJuV[5E δr ho%ZM MDxEIe)MLJ31+^cݵt; !i:D<%p]>8hM3 \_ ѮE$%LY*kb nTBL#ZHm}4(Y)B)bDks4͝hB8+ZHWnjHdhq`*EҤĐ)J[(Þ瀸 '4G@PNg?㩘T賝x@-9 %W>{WW.O܇]XvSg uȦ@AF,*"gD8݆&i{pW{q'Xt>+xk ^k "(r훔V.YXL*-8bGKpT Bf,A39㠙a4C!Vennu<^~)vF 6 fo̤Ava_4eZ%h'Ǵ+H0Af ']kw1cĻLG}+E1CTDC c G&fhD2QeQ䍐MA[h\W\ <]ًq7O(%2ГDGzfRG<4 [9oMtLR|"}{Pli}h^ݚem jY֪ Y̸1 Y@ʐb 7[.f9jRq%@,-n'/ٰ̚7e8ݼXm l*-шVJE ɸL.Yht]%G@8K}W -+O"Pi0[5g ؚFgXkKK,X#/s"(${=OIƂEDAS:?KeW*2 Z/hjR?#kp7ǖT1fMC m;SiL$5ҿ\kxb;#87m(`0$ `HXd poqQ5 n/*[w)=H70 b֒^җٟɦPQ=ʒ|'oF!W/;tsKz٢KDEw*3?~zpq %Z+= ěh5ٷY % ԞnF-'ӿ)쬹_`N~?*R|8y^T f=۱/d* i/̔ & rO/8aEAi#|ĚZ,F6HVzԮMAPHh ip|gވQnM(6" >`ADng kl4 n]SosCofwkt. gel`$ׁ"v]2 m=X^!BMt*~@ im~*f+ Rd]޽ӽ6b^^k zI4<[C Y0^8ʝVT#FXXHcdkRQKe(Ckߧ|bCivP|3.]U:iсT;u)6x0-'D8kǏgYŕLH,@\P@^L'Yb{HӄXB5$w\-QI)%7 E?591fo7琚Xdܲ`"}ƑùøG 6-!αW\EUV1fY4LAN<{Pfxڅ/V|pгCJgHok0mmNOhp|OR|!rܭqoI Qͬ؛ LWrkw`䤦VWsd &U2p-?olpo $zoErSMS)X@(݅ΑSAOX˻JNd֞ex[pJ3̇IFo N;*Q(y A ǘ">bR;0 7?]|:>3 ,M1B/Wc ߟ(bDb=E qP(L* a瘬.Ɠv .QZ_CdJT^*c~В#sVH`:36bO]'Yd ׇ6`K:<8$rVByvKjퟌG-]o~,Kt1X!W8znЕT'L=Gm ҴYAԻ6{GnڶkΟx8Z7lQ.Ί23!kŶzgwĊ-]-q睯_❧8m[УdPTjsAYes[. u=w.kw=|f1E@[ns1@]p-:GTn2"@؇nGݩL7e)y$^:!spQE\;W~ 9LI}b12IdIWRG:p-W׶tkIh=(xt*qEt>fٕZ-Qm gq60O<;t@zD3&Z-f5UPTH[}&]UcL bzꀈӰy" qmwbQmN{qQ7{W E1G>)hyf5q9*5;BgҪzL'sf+q8!8zK rcDgK11 ζ1Rڦ5B0eih8Z&;v/Ltn}VRM+ {kl9D׽?{] >wΩ 9PєTr0T`@1^Z . m,'*3.6riLOlt 1ri0#;Z&jaZXȥJ]Ng{w" mVs˾.e &Njӣ"7g͏1?$+kf$SJꟛO'D?T=`lB-!V}Z3L209ILs^T1nрq-roIL,#IxF=Zs-Ils{EħCVN#N܏98EK Jc0$pa:f}߱:>1*噥đ *fCu/Z#{5I$w'1 tN^;M;qdQ;bzM׈UkH@Pܬ7t@|""(SKƻK^PB^eL`)goz4'"NUҦxHtO 7& RS. poc0)3qQQ$׫IMn*+Y]|氓dĻK/ wNqRI2 =v󊆵dll<gKR_\QZii ' "X-`I8a$"{XvhME@eW "oU1\ bIڱIlfOđ5kƝ"Q8$#c1t4s) jxhc[?Ϟ:6qB(#Mؔa9K8$$O_0UhNYcJcΐc~l|b3 E7m lWmܖeQGCj6s CщLIi붺MI+k ĵ&Um+>NA: Ǿ'yTPryz IDAT!^L._&Z;>ĕ-52I(AU}Tld~(6Xk6cU"[ 5OJrRH*X9>5t$\1#vu1Mm'^.&]]rV$pet\X|=#=!Y*+5EA3J̓7i)+ %<V*@$I\oÍuX2NP@|4zAnJ0z4G]P E2}Fn.fǸ2{baq56/¥!l0^}3(K"lY__E0ِ ( _N]bV5W{YLՋ^sɐ?1 IvHLYE@q:>$%,vem6-lE]gUehN mc7g5U`kt6 2-*5̰ߩ<&ѠzBɇ2=w5  p3OY&Uz@5Juf#41a$ KOG)f'S8ѬcOBR omIDTU8H'F#)NHR t!4F+Q¨*RI[qbjcOm2S)iEUq!aΒ /S(/;,pr_|`acPɀg i8gecLL+ E҂vFDM寸i≯M~C/™n5̘u7#xzhV\, O_\R3$!k}FqbٝEW Up"a0.*}F9Z3g?<0>^tCuKX֏kz"ڽb NiY49?HobJU+k| k2anjh,;gccqϵ5mTq},R;mt 841?#fc/xfܐ %BޙjN*3OtUOGu"!1aGMrILh}VWDcYFLGJ8 62)hf`UIč%WI[$q=˂:ʣf!N\̓~l 6nwّ̤%i'Nc~}&PlOmjtk_PIU$aҲ%$ Sa̧'`(Y `ϥ/ǭ۹Hңc)s9x^5Eu&.KVuy}Yy+$z8hsӂ2$5 <9(Q1| SŽ$q^W\)8h8aDTcZB>+FsDz8r[^o3ZwLlsE%vd\^RF3WdCeTXhՔaK #)Bëٍ-3yVJ.O#5_u6&JsDoyum^t͏1?,3j2 C5ħ&C}BbtJQlHUcpncHo.U:VX+fc*^!AqfXIWe$R5G#qlJqF$z8 O~L4 ýeTZ`6 :4W*HTnL M2up[: zCvII8huR*ftUdXu(V˾ \FstE`J퐂!Ep ^~4tbMJjfw|W_#K [7Q\KY$F&>V g4jcYIX*qsR׆QLYh~ l+$e .9I_L%9n BlYyFc#2쾒Pܳõfŧ?)ww`ꔸMԱsRXaݛڽ`IzZ͒3)֫r*n"qm ^DJnj̹8Y$ڮe bѦr1U;B3wũ 1?Ǔ˷Mda3?tkG+CRә~vmb>8ɰMƛ[^#N,B*D.gȵ𩶕( ]gB12uVeTg")&-h)݇ցyv5J}l" gIOH8Qf%F3d43$E~x&g2 }{(=ƬŬ|[z%IX ,h,AFjP\--:\Tv[}mO+7 ȝͰ)d+눇?/sQwmY܂fLOy֎(ۘ a P v (H$YhI33o+ LTR"mq-:tKۻ&Yw1ɸ8Yt })a/2iEJ3ϑ%4taAJ)Z'T4 5񏧎}ݍ[~kdmѨ|vQ#abZGcB.h*/!X`e″(d${% ^Ŋx-ؘR=k1aV8NkVep ĝ1,m":FL{t,j I`$#,e 2*rYHZ}= { T~Ư_ɟbidA5piA@oQ,\[%A2f۾R|dDIyDqUwnKnm+,{R20jXfv b T\ZpҴŸdIZg C͒B| mӅ7U0ʟt, /kNCDJ'lZadT q 2wrk4iN|SQSŵ)zRo4jhvY1(bb/ pSM< *&vM ".F^6VS:bXzvH,8kg`,A9)%`ӟmY\Ʃu[Y@5hW|;~\! Sݴ61-%PX,g~ YPƂT1iPebUB=DCM$5`1dmFhǚFޮ]fJ`VosaDb]pt˚6EZIueztwhrb܌֌qlc l@3S*qJ)ݗ =~[?$>l:ҒVJIlx _oXQB]ubWlwo{Xz0{Ha8g͏1?<ܣ}EBM6+ϓECy2X)R ;o tI>8|zV R% f=Rxc`p؟QD\(7cΩSMd$1Km4ngu&tUDC1fw,)'E)b>_iELMnq  + iiip5P()K%jх+d:BĘfA@Z^1+d0+CIBIe#bVUŌaA,4|喧ae ln=E%7M7܂hT4CYB$Ke M{ּ&{ŵWS[c:Viƣ%'gy̘kQjH*dmSBh"ULB (af"NZ1^2lXRpF8oΘԄQt“L:iu6-4}s)MڶSm]uQyl g+3%]'ҷJ%*tHdZ嶘v%cc (tm@N  FTʷ n/5_\º~κ!|MJO1͊JL[y0`4FQnm : .5sHjl  ҝm`wwv{ fT-BY6a<=KJ_2ǁTqPdy-g*BabÙqds$ t)UVMLA~ iUu`6{ReaKZwcFg˴*H:,.LGe ‰+Hk~%c~̏'B/D7l5+I P6Ebl4&0 zlk1<0 ',afI4C{mltlIr}RbƼYa&qS"uI8uBCK 6$?8RINIqglcS;z#Z]13G=AZу/vڵ9=aEcfsn291. `[DXf6L`,RخWL~V:`oZeEѽ,a#Δny,Zff6MZox;Zc*(d-ijiIBp(6lֲm`IP˞gQޯ@s0v%$ 2pB*/^~ Рƾ"ì#2 cBdƵb#~a D: F3]㋃9U}PJŘ zyRNB @mw!JhB8DncuhJIQ l nVJ'<-YR̽bHVBDJwaC)]4(!Y'@WfE⌜!He9^2n(l{ mTL{Ӱm~$RS50rsq]Bԅm_@qRH 4C0,KufBJ#[ز2&PѠz㓸wa|na:7L%?)VRps+^]LTߺex}d@ǃ̝_dVIc&X gLZm Zdp1멺y-w-E9ڂ3JT qaO{L3pQ7 "8s.,e7n:4!/fуXdAb\<WR^q P8ܹ)XbY{'@,(}{ҷj&QQ޵4YA-h6RNJuⴞHW;rΎTLz9ePni9[`vW3!1nxm v |^KӘ6lЬJHiYl7a)H!FIQҟ);i|2oGOv_vHQC s5gtEq1X&E&3F#<猳g|[XҔ!t;HV@v'hL ؘ@zQYA+Rthumb}X8.:qͩ3|pybN)q#8娧*q78^W;w[x㍸+pw>n)ʳ/ῼۆ܃˟{_>߈| +qYy_cr:\ z& k͖=iJ.GrTK+5MɈPjLʶIͦoM;f֎&Z72R״9=V3޳SsiebYwf@_UA [@e`u-gp=k/kZS$pt,.߂`|/=MKm+]k؊Ag!{6,qj2EW Y6FZPny'e%0B7ԼajHmO !.:gK +. <8m/a+{TC8k`XKmP&-[Б/_L9|zs.²%Pbw29hZI tκuRZ*&`$i%#p/D %Y7=1hb@IA*a,ς)$ zGkGHk(5Z[s;H 0NUhj+#nN#edH&@gi:}Zr]OiwoH@,>ןȭ$|ͧ$7d!̷Eזz,I ! 0sEj?:͉gRPbfsJ>!R؋b/]ŚhyvhE)M̊h^%<)I\'ZDc\NtOttMہwndQ䴥!-?K/gHK/?q-oyWLJq^?x^2{9핧Xá;nþOѸ/C6 qfxޏDyFrUiӀ& =冉MbDUUئI*4fP[V00 1SĢR^**Vq*mm @!&5>fP!y@I &wA7yd#qw N0jݷ܅5&ǾQb{)Q12PɁLuG$%Gm:\2Ej%Z,*#!dO[X,QR0"V&)۬}?KEYѰ _ _};w>Wue}hp{Fn2C/ӂ `m%# \. ?C@3;dwX,4vwHMA"$OQ(:e)+B"XZJA d#aiKOȉĬ$3tP•Ok1yÉpn?߼.|Qhw]sSkG>$.UIIcǛ1#x7S8uѐΐíFGQ ƘtHY:$$flkTL:G0D y>'4J+zJ'3%~Ab,cq!9e] .?3?c~~Q=zg~gp.{bހ>,?pKߝ w߅;nyy,XW\koww ~Ͻ 7w%{ڻ=ok0q7q30VyݯL^sݸnEv_7vsݓw߇ . V{_gM;?;a c =r=睯Vߵ׾uݓ+[ow;p+'13~VEcEI~QA@bҷ%)L%a9,hX.ڔͳRGMm.C-M,Rz8DhOF)VI\4)I-A;=\!Ymm8Ag _KeQkI s0pf.b &!i l`tЇvڄq!oFև baL㨫t 8 RW3:PktL*`V.|>G Ph r7Sc|:?\ۿx[,.,4kbGoYؽH#mk@p2jN?ϰs(8L{[rey1TP"-Cų^(wf'1X?g׎)V-)Lb,m` -Z)bB$h2Be t70{żj WCT[/UFlV6SXtqڲ wyghS} ^^{Ayx^nͦ}WEG˻p݀>~2vx|}[Mm{}08mgݏJ\KGpp vލK~Wpxbn+ŻqO nkVg߉k؁ְ%{_u.ძݸhn? vރK~Z,.G 0yۭpx `p~ex%o~\]o;޶ۏ¾?+=_X޾",znw?֖E/݃++X=s>d֟gUQ't[z̗կ2}V|FnA$M&4t%o-c &WILowmGmrVzD$o *q0K32Z?(דC4wE)|?KpLfi@i,oSSr}-@:$'0bVf2̽PGEL~-`B `DIzZ9$7sIΛsOi:M% g91$[PP_-to0Zܿ@pH=]zGGg.N[Bt͘{̑Sr" $ N6)!B9a-vJa|*Kr?yƼuP,G 4=KCjAZzj(n:ja ϙdʳ1&.Y IFN5cc9ONQ_Djܦ^[6#{=M.5`!^I77 D/&Jtw\QլZt0}ʠ-<(SS;}aj {eI *Am^$mVIEb:󱦋S#.j?DR.wqU;"wZ;wo?W\qy3d80Vqmێe9^+#z\݆#w^~j\Wb 'b\#?1g+\5`߾x+.ay`K/|ͫq^[8^w\{7&ON~XyGV3p]?}%k{WpXޅx05[qރk/W][xz~;~ݸwawlu8 ,'iAnƒyh1 v;94">tDIKG%Cj+]TIcT並2L 嶟~Hų"2.IhӨ^ONRM"&mTYdb\ʮY@:'J)M0lΦ<.R*F%$fD_N; QDJ{W,Vey+r)h>ⲺCɕGxKvaｽބ_gx~6o}i3HX#440nƢ/nњҥH\JĆMrVvd| яil,ds|1E&'Ks)bt:s $XNuZrą^8/D۟+wkX9+OmkQ`a+-c+XYGh&fk,.<5 <,uso?>9gزEغWո}xrxNc`9Iacr9(^/_S/ ؒ#G2]1XIX4OiVQ-yg:bS-fL_u > 707Ě}E4 j+՛u^2fʃdP܂% !1T? `#V6vT5UW)ŋID)[[ae*VuDiöB%%}.% CdLj D2Y韉XWr]V%^ d$l@FłyoޡkvKIocT26vV*K5lMv-#7*mKhe)[Uq򑁧AWS'Q-)}8ݳľNTfR\+]+cST ȗnḦ́YJV wg…=FK轸\^ D40,b;'BQ>vCF*[$D3YWFG]+ ikGGއe4ш3`_OI[)>}C9ܹsh۹s'.b/,}&U`a |c5-bu l;w[& Yep Vлހ^[;=M|ׂ W [qݾX]܁j|oLm8G]۰mar~_fT@ʹ>FDqF-%)ot" %t!1@}mݻL=}\5AI+XDkǏam| lOW@(1$1B!{޷)>Ol~X1 B8h\'[AB5Me^X'BowW*MUk_֊u YFǙs6f/IPV?C},Ee  e ˶`L}v0{4j)IΏL|n5v=BRF#l-ų?=oY|gpY 732Y|wE&cck&s9MNIꤞ.GJGQkW@[D/A_BC |E.݂79tOK- 2dLt8k0cAuVb+5V?Se܉ y(p|ׁ2%KB `wz|1wղba,W^]CҍeA;F9kZ$)y(`u uIgΰqMj3i-g\XZl&#/=L"2M # ՐȨ< _eoƽދ7̓}[ނ{7՜{\v_z^5߄Xßۏ5l˿Λaybe~ Oŕ_@R¡vc~;v;nmpd8w;s\؎4\u^zoa?^ƶgoŏ o^W;U,o/Uk.5&X]؆][<87׬o| ͛O`ab#_b,_/i(F (y9qжqNןizBT"MD7G4-`L"` 4ĉ`4sJK!+}{us[ yNaME>#T~ )Bck85֛˳;WfI5@O6aQ|{mlB˙r8ov١ ubo4UU[&ÏQ0w֭1_}>BK}Zfm:#3h=O*DDt-T`S`ַ末)ncx._U{!5slfn`- eʴc$dι Qi"#zb$vMr0L݂-kڹn SV,;Wy l Ή.)r_jPin2R.}l9C Xf fήDI;YΊU#!fy}1"P+yCƯzK8H[-PRю60$ֈ7u&͸E3 om`Vd^AƗ yT"f|`FQhu`ZFfҁqPpZ9;qWo}+ny]jۦz+wԫGx;v|^z?ynwݎk@z^՟ػ8qx;zkyǏ?rصß 5~W~.\r2p|?qh#akp+wcw*|llAoo}߾t>w G>Z½x 59rY75q͏kX>~n};þsK ? E$z¡jHW-x9ϤF+bem_ |QGzQ$*?5%E$0>)#[Cḓ$;NwbM[&ia$qPq4uHzOI7?~t/CMtF]*S,P #0G  +f׊Y<\4$ H?WWdvY^8֧Xa7 TXf+mn+^o)?30MetDv>_OHL W$NOW+}{~2ܷ0MQ3K6[S|nRS<3yb Yf毢YUH5uFQZ14;V"N ZE jFĄv֖Y&L@mj'S%/d^ ._4 kJߢG}~$hI,('{ Z!ͅݞB1)$뭊[o~qc~| sϓG/*aM:HAJ)o5Zс6HA4IW]d),Tjo-oBŠ|0@f"ޜ&aD'0=3_+lR}$Xi[a!k3Yjں VtIQ} ]һ)rg[Xtd.l K@%?t)f_8f?j:yv kn k ̀EP&~ӗO~T)*1uF3RMdsVdN 3\^5jP>*E5"V7p%g+9ՀIVt4UM8mZb1*)nR^gyӡXទ YWzdVK4dҦ~$֝)+BmxX\Z/ oqA6M;rc0>9ƶܔJ:ԧzrn7pM혝/' oQ @uaA+ׂ+`s**F+e&NStPj]z;DM2sGCn0t "y}riJlhڌ\GkCWE"4,'**@T&;mI0K"o,er +}s*`;fJ͐%:];SEkKT`Xz`!C\V[rӅEҕԷB׾E$|}.d0v{@5w1^wiۗ^v />uy*9u[ޗX H C KZ[`cjf잁騝֑ L76I}~ iR/mCOf" j5+Fp,+hm:bje&qJ4$E 4n,vԴu1}J [D 3ݯ1b13IEp;^2J#KLS<]51UkP:]%sjԺcBO.xayҚ|յ&{U[@n~̏2k|oT?.M."K(H.l1Q5s9zH o\q&L|)AӉ+_\Ҋe歩IH#E^vDŽ2`)jz-íV\1>u`ͭdy9Fn*,֏(2C^F{Yv[d9‡Vóch@llklls'N 9fe=i@Z88- AZI;C\ͩPf!%ɳDP/Ѡ@<NR~(hRaԳ Jl0pS䡟{$S׺2i/%7b8ҹh Kp` ZmKpAىɇT@Ғ)oUldޕD㻠T\8}3C-Q,7s)zB\$j;vIaXgD]%YRiaK˙ 牔#Tl)lrkYiP}"@:)Ro{_­Cg~WO|v:/ ERV+^'Cz*:%fsSfxXR-ׯcɘL&γG129xy$FW^Kn)"F|)f]b džڤ9d*!DgBh1?FGu,]${#JEQN gDh 0{ P6̫QFN1hݖ-:NZo:!LDU:*& Db˴Auhn&3{RFzcd3*m`Rcĭ8Ռtj <~c790?I ΒɣQ캉:/+~C6Lv~IYW"0)FaD`dsL9u#"uy",="㼎E:+fK } 1n׬Ayh_ G`$Le(6`FY.%9;W82JhF%HJL/i"Nza,lﻕP-P!:irmzOo?_\lE S>Vmxnn[ۿkhfYd=rzQs_3sbif b bcڡcLcN4$6`o484`u6848lĘ84xdcoЉ8|r /}Su˖ǜrc~̏e~Rmd{'*E,_3<04(ɋVºLE$#ci18z &3;E\笚V7*5ɨn8ePK|LbtK KQP;F̈fP M !غI 4w_Si> mok20{~s$!OEfsZueT\g!YDqHWLTXe}ɵpׄBum Lڔtmqy> Ozpm^y۞q~љx_Χ`*:nl1TbFS:wZVLXf(;jVXkuNHZ*Iۡ3S;.- ي4kxawIqݬg$yɯ ϶݅EɼsNNfpX5V.ɞ#m+TUU"'Q,ZϿjMV;lEh_e-T- &>F`$(:7|ӟ>אc~r#Fc5w>XF,a`L )4eZHhPǺ2TAb;4cZ7Vk鳶FfYb bF|J+vsG&FT\Ս JPң!՜, ͸Hm$Q4\ǬބTn=ϻ`+_ոO"]mI DF)aǂj׋k.~idyqw<ʢ5HڝjMd#A\!Ip6*'^SA9=5qljIڛlyHYm珦u#{7y$Rqeue-虁,A5s e8횕A5M~<(|faHxkf{ (5c5fdn?]25hu!h)ks,b[3@^ivjoX#s+%b:l c¦'TlA` b81f3gښNf a.=q:u)n8I.qhߋXSFg:9x_;8x .=#RT1Xp3[5 >XdD X lƅ z$H њy4CwK֖yqϚ:4 B3F^Ɠʄ}#ӆ fPz;j1a߾F8KIg y{Ω%egD%IIUYO{'Vv*g2`xemlr;[^Sh+]/R-ZXgy/ffR>Hp Y+j,a_5P2)((Ąb] Mx30(H02h-a%t́JQ5[)^1}2 eFw?MjB&Z%Þ1Y53;o[[_żIVy'p"f=4 ܂Y!Zcr2xtJ3s2x#=bYa۔̵\mL pzmUNε)P,8MV[M{>Ԓ5?M3Af*I0cqip|cl,O]g,b!7?xtY7TINӡcͫ li&c{;;IRS2W-?Xe,8~S*!w+8BOwBKKtT RfDi_JK8qHTuE &FӿMЧkRv2iB[P+D²]˞Obg^Lu-bmC8=E4Grhj0TD礤70G8ל%σ'Ҕ(yڽcyHYvAgơBm ;. m1eaZ'gXJ%,;Xg+h}[i!9*FOYeczfoC]=VV5gmVuIJ+ HafW A"'M:9z*r^SQJa)Ջk 2ehg,e+ּ,6Z+0؊&{(PTU ALT?A2\3Z|,І~dH,5͕1¨u$Iu>!pq Pb :p]Z*D|iŬ51hM^w --e9{x7ЮHMF ,=`ȩb%D&aZwԟ<+L޽>nobm:C{foVCN+$OE1=iu ҖhqӖ1[ Xb}`LT`ipdD;8<1 <$%:۞Qsc`\5"o_rR2"ޤ Ps&*6YlF ic|<GW& Ҧ̲^k n$YCJPwW~N .ā)r;1ѓі$-^q x8 ۡlP#]Z:&Xp6w tkc0 5OZJnU?2L5 = Jطbhc{CN^Rfrh;KI&P%F uCff)=ߒi u|k ~?F\6qOy/x>+&> HGw^ɳ>g#YL (~8 IDAT4׋~ƙ{b椘d5t쵰7Z&0 giVzsI &@JecĢM5E4%'HGl,e1j8q_X7LʸMR^wRجžcWML7FR;y`,dQfeO@!^8 tіL3Q9@̅* D.Y\!=Oz­-!%0+*YtqK@v1Cӎ`${0o<]e0nuN `G78ʼnL<Al`|,i=,IxtW tIn͔r7 "hs vCgh|iӸ`s!l0 f#g*)nd=rt{oh{;Np 2Q<(:d H&4ێ?S1*=;]j]\TUTwth(GXgZtHuroQ(] 0MJ?'LUA{`9`%"!!Av)bVԼer I]a%?ٜ_x|{|>/߃蟫B?GǮ]@d%wD>W hZvPI*bU bbH'HcӮ"oHBihҍ&HK+Xh@@9{95ƘcεOQ"${G<>{5\so|o^B` аJP 2Z%(Qr)Z{@/3ö;eItDȊCvU Glx2l.kn]bX$Q]lNʀhҎa|VlA]M$«sxp懳7^wE1n -.Jm[9J~Sm' uSRx4L!WT1BL+h JiUV7[4˒S hOύn {nTha,[M+iee&zb9Zgx[MaVPimV$ќ-u~4Cں6f TW?|(0FPQ:lc%5isftQtUsj[}&x˂߱ dlV][oNrs4;Լ:#'qTf dέn%M{m\K\{SOU,Ѭ%>n&`!d];lz1_@Gb!v=܎aHd߽ʤĉn%"8ih9J2#~QU6l;T֦\i{87v\&J ;;^FgV6R2 ݤC 5yZ9T^AAϑbhqO3 ] \2 ߎD$cq APF `Os"8`>5nAnD[DY=UJZeq{ZM=RS{nyNI#s1RQ\b!!$CѢnZ"r ,8 3|\+J3@/kMW;H0QU sJscrG;, :{1¸sov/G\O^A(1C* 0\\ٰ9 pF]>9nD+ؘȳҪkX v ~ײCxks5#+9 000 si]SVmPR*^2b&L1=+դ-2z]I$=:"yQI> Ж@'R$BYdĥuS9xDAj%GMgh&JLJtXhI( "PQ +Gьo]KS@F]^{XDPo4ZAR쮢RP%Om&ʸ-N 僘I=p 5} 5R۾ie}Jf,Caac.8 ٮoJq> +F/g}1Rd(xDf?W p-8Wd[kV4Y'/V"cХY|;Z!sqVeX[ po')M=W'd6 8'Ļ(cyE[Ixdi۬29Zm29ƉP9 ?t6J«j7|Oqo.i+kJu| b& ؅DKN\yq6ؔF 1B*s4b11¸քED ;0¹( FЁ]9U L SIύӮj>2ĺslrN$p[36岗n>չjZWTl ꊵ}&r%w7=i$|a1G`l0*Yv=PzM`X1e<4ZZ~3{÷F\R("#xfnj$u[<[ڡesJf'-F:SFr0ggVcN q!$G޹$)B11^ &L6se dԱA>pē;Xash3}q$1ذ̗(p!cnYFHH0ICP .JyAq.m@@ jX0{_AEՠ)I$.5̸Am\¡=fS`I`+f(eu?ֿ*hY*l=a#xDc"E2 7;XVmEY?"u Zfe*"+'RJ,/ZDwf}GS,Qd?z |J(<Ysc߿|+ƋIŢ¾}X,z:}8&LJϣA[aD%`T\~ ,1%KX䪵0w&%`v,ؤ,6ZmdS#^ J}1ɷbGM[x7aS'4FO;jv lͮ+)`3#KB 𗸕2sw_zcܵPQ,&’뫛ee(]5aqɢZcd\9*c.L8O8p(ܻxk_b;a) 7U0\c jXh۱9DUl6nWQsƵn\ZP\e  ןxfB`zOc "<~~!w饗/Wy睸 p饗>)q z+n{;>3S\ǿ@!(Zrx#Q^gmY.2$!յb-FVIQxET5`0)@gFkFb5 * Ȍ&KcBZA(PpaQTN4Puk+J]ٌ3Mhwmp^r<;&Yp%>?X| 9 h Xmf?9m7fknk4m)/Љ8sАt#vfZc|uU>=7vgBy,ԏ,DKQ@b"$_8"68)nrA߉*,?9_??֭_d{4aFyKXaE<<>Gh{ǏaMk-WXG,\dƹ?e)D:B) ȭ_>z{¾3q۱a`ӷmyW╸꽻x}۟S\8R~1Ŵh49ҥ]͏MI춿'ed[.qUEA11D1ʚ 7CrHH :0uu$,*7S-[%{hТXqEA0l& RK(ۨѦ gdNyNB*~N :LN S 0 %]k1ˆ-M[_ 6Lwv֙M;90fmR%6r$1L~L;Yty0@Δm |g m Dq; ^ƍ'8D &)#4n_};E˺ttHW&Bˣ+\[#Y?),x$x1&&m2̵ɑ\ȲsiPSxٻҍ_E#hET\e(MR_Zz X7þ (.순Kh㇥or>)sNr)zK$V"yrܵ;UVJEc;a;'I`㉞atfZ~\\Xg5eז[ُ$w52&Yuok BȺue`f^[VJM1U+q ,c)^n8D,Lu+vˈafleBSuv"EICA Tkp T T9<\rjCɄU$ޟe܌ 9j0n;%/ci<`Ddڵ(3 ]EJ }=g۫f#()>Hs,ƌ1 Zga!>S1F&BkfOSLGHɸ:0C[pH@QScFM_[ Xk`jx4,.٤h2T%" />+j:U'-Lj2sčm r@Aяj4dqGB$+ fHMs; Pb92#qbQCפ{κ'u9D?AB.nvW@5Z%jy+G5]og 8EWmCJ2Q|%&0Ӌ"k[6EQ:S1c@I[&qhvi偨']$%㕺x 891Sͺ[*cõfVV& *(~ײf5CJX0WDsشv] 6S0;aG\C+۱}v\q} ~:ַc^җUz']5s,W ̽g^^l߹}[6o؊߲؉{,n%<|6˰cN;Y;vb:ö;KiP`ۍa=;Ƕ?\ڛٻwb{SnH+ly{lԇ޷lA}>;wv;qsS0iz);߭+jVH@tG N?cպb5c*H[c8s΅!aXkmY<\|y``Ɲ3& V A+qI|v$UްM a@K]#֭aCbC`5懫Ú,V f9ƱN)~6/cV'Gvh*@&nm<2WzP#n 2R4~N5J-tlPm1UEM]+:Q&65EG"!QHٕ8Ź˃{sggmZafOiGb?v|/×~[ $W82byruX?LNo] ٬O~Nsz":Й<SeNUVzqj|%s^)Y3ڜH`$ub#(Nts+nDrJvPRog&p5 (h޼FO>O?܁*FESэ!4쓳\h=.g37^eqbO+Z2rp>-{-?;@kuq|Mlօg`X2ؤ&\(C 5lաZNhezKM94q1ߣҖkV9m׵53nPb%3^smRˉL4͝F=(C矏n ~zvm˱[o[n}r Kw㆛wMo;_ [~_߁lMs X7v+>;an{iǭك=l6\p`n]a㳀ozv|z`MX8?aa6ߍ?ǽp {6l~8np `ˋ6boM/aҋWi8\pH9[Ԃkhw!yieI^K7W}yA)vQ+m"H@ {AzA= }A"`S3Q- ]V"Ch#N :F5D# E3ly6qzN fb3 "b'z#L8B] N3d?MWwm  ^H^trQ(mq1 k&7n$#oeguS"R1^ k#d:b  sSn7I0`H3؁k}RjZPrl xKOǃ&|AS&H8bڎ,,K C9ovrդqUDq_[Wmbz̸Sr4|*{ظ4ljƁylj&8%[#c)ɇ%hGBe[`_I2rpV׈Wo>GQd'29s L~̜l9rmcffFƠP.N@V^'cB%+oWv*r%xi;1Hʜn! 5Eݠ ]uWmWE[* Ci] m`̠rW%?vX;(ڮ"]* JT`.z`eG'=mXE5PI#I^-D[gTTiƍ;Vz8XKEGƭ0FY[{ H6.ACN\Jg2Q"~xe6_Ԉ@Qki8~v1, <<:Te`J\%Rl72:tON2lkrgT? "LQmk|8@&(j:X60<yV*,V| qUbG/y #s24` o@Ƅ$6P] RQD`Z.ӏvbY``0F\bóOBoFt4 "joʃtq!psOO[Յޚ}8xj~nd& v9NJI ";<Ґ6<5 J܂/1:wRBCl)ĸpjDNK#'""1=4،_f F{m{)e[O(‰w+tvg\ <[Rs>f3--ߨ$V)ǟk#X8XS'B(Ŏܪʭ1Lj uAN}ـ,:f4t,ƥ.qzݲZk%Ǩs ;@6Y`\;3['PEW4A(#޲Oc9܁zΟ: eW)[p֩o~0Aw`#}'ʅ|{ w.g?js8X7ܼۄJQ/7N18,+⚽d1\jiGZE&DMӄ)S.hhYQ 4D-L]6:&썝y"~Zz.`98Vsh =ۘ&&Hg \ L"B#! Q`f=ϙ;9L-xpp= f1GEWTSEK r֢䦳KMF"Q'4+Z)L9ɬt6`;)Y"f1PPy I9 ZXJb q>K!1O[xM:ײζ^Zb]Y} |u1cqh<{ N9d b k6+%\dz} w[Ǹa&L$AIO@CgrtuNB v-'~]z IU]2^[.+! {j,'i,V95,\Nc% LDSDuUYG1HI@$駲1tR;3z~oG%vQm25+Eh]{]9 gt`r& '^V؆VR6_̲Zi9@&^yֶRgյ6Ԯ=QkvRLΰn%mڌmEgF1 lZM9mk!q1WލkY}A ^QVS͌r;Zs;tq1:}g-[XXu]ך6=9眃=yOkd=6=u*]ո7ǎf\[͟00w<{?-mGn•.t|Я[q}܃?z}3˰:vsXža=X=*}.TGnz CKe ~NsXYIľpx%瑱äʢL.|6ZUeXׄ $< ,Js0ˆt83$Nˈz'@X!;6x!'%g,8*`TiKVN[t y1Y4! Q0\!0|%r*~IDLV5D:D J#P 9/ ^]p<'i 'V{Bč|$䕜Uv΄v!Ha {YXA,|o6 `9vKsLsBV53ɳGǟbԗ/y)/a HK|2caȃ{Y(j*ΐQ^!4Iـ1#J)M&ώ7 SJ,,P_pB CZ,h'S]v*Q(F^D Q,:?K'9.5x`raٞPtΚDT\ b[$+m!WW 8͙]mLhO + vҬLߢ;3 65@+s[@|EQc!m4$@U V (~^* B䎩Zh1Ϩ:r7pecЀFzTQkˡ~-fNDX0 «?kCP]8B3uغu+n6\뮻\s .vm8qWK6ל`7ܮěw'=?͸ejK X(Cn`n\]?8 _Y?^g7U|v\K91%x/t=% qqޗ>c+DR ɢ@BD ky'd=F^) ?!?qu%=بYE$#fD`"Id]ݗĉlBLRM8`@QDggʪݶڰ8 1-ԃ/nOj@nKwi5e`/b"d2<jv|{~ Mdm;ኑ1 Y4]',Rk@>-9 *E9(QJO1,_f愱I&bJGdxɔz%kvԴ.s$q$!hE:j2NDLP(U:~̀$L9ىKlL+c-Fғ!߽ßo㘗=7᧯랶ˣEww!F*;-"A-ٳKb7+F4&dUڴ;_kCC$f/տgJߛ.J¾;5̝}\jS[jJdkVr=vDRb$NY+&orD*X# % ,+=Su[ȝ`hKlO;<|74A~8\Zj10%!2+/jo>}igwRA?:;f7Y}vRC, A5Ci+!æ\g֝w*fap`ڶu=cĵ̺1*ne \0jfrTm1 qrgx"mVwآ?&8.-ho~oxsǶm۞8`p?f#6mތM _W+a ^ n,`gb3PƮ{Kwx;^-OnY IDATrY=m;p˧vs~8=XZ /^+CWm=#8c.æ;kN> ,}vvy}Bl/tvx-wMlj9{e8ƤJ^Bc(f׫Q`Ii}yxb9^!%Yc-i&P@&ΔIjRRo+L_,P/u< 3a%oAC8Oo /8v@^5JINEEo"fkX__sQ9m/VrHDgrAA ЎNlSGxDR22{S|v'g<:6Ck Brih'd'v Zه]gn䵸=c7_yp/ff%Nթ:ƃ_6^l׈s3zd-n$ ێu$:y4s ;<ȏ"Ļ[G/$`dRٿ{*-<92 -y\O^X4 OYᬐ}UŎS ԳmD6dJFFWNh&ې8v5cVMo@rWXNBGkqvduA:DA,U<13Q wpl!dRZ6*ŀ6hSѓcYb'N"MO-9R3Dj*'ro./SrBGx)X )u\YH4m`jPxͿ0^xԼKKdR/QG>mMKq;.\s5O,CnzLs'yr̉13frbœ[c8mtT6*쪸"IHAteRp\hqSi aĭ4ҕtwd>DŽY HHREH+,LN# jE݋iqHfhY`b)(3cc,1F<21#.kM -ݥ;m10yl keBo5'MwԲ|0qٛfv$\evY`v {7J綵3,k?3RmoqI"̦Ԡ 3']@`瀷;9aX 1@ P^SNyaUkWcsNiلfݭn+sJQj Uc1's# JSjAУ!Mؼs5=f*\8r7Sc9.=N f❞ؤmSZ3wCMƨL)lw^{{~V0ۢ  ;FG5Uey+\-Woyed@|HSKl w_[?)6d b U)%eVnpbc$xu.D]Ŕ뵋<9Q.HjuEr̫RBJODhJ2Ʊ>;)v✣O)M{wai1`9,$5dPxecr֛dkGqOJg3eXg"J=hq,q_݊o,.bvUf%J8VՌ 'XNuf W:a g=Xs}a~nJكDԉy|ɐczaȝPU8˓>rP"pJH".-Rk}H=bmqxZ% , HQ a$Sꖁ!wI)?  _W65c! 0 PqRO#pqhWrh_ F4H4/a޹eYFN6POێɢuzȉƓ7Ò +xNeA8s37Z:0uXp~zS)nmu+ϛ|nOMgEg'}cD#^ayϿ %V[x66>4l:y~gp>Q^+߫/~>7b0ѨaD%Sr$>JΕTt!se y_ٰ}*iPF{rA.H1~G@QK2zr2IV)`^A$qBR:?^'l!a"B|1%Ή>4MaO" q\8y/Ÿ~>GXЂ2dOz$h-RvcE/q݉Fz0' Q\kb+8J86 TF^7]c6}d&9+k??O)9ifqP |p/|^v+ӟO|5{nNm ơ`078mʼn)Ü)r͊hvƌ<Wъlt'1ϯLlFDmbs>{A@+hq/H3 OG]7 %gѵR$BɊbc`eg[8iGĆI:%9x8}US4!1rOXWq!,R8rTQcg,'sSi]2J\";%;E` ŚrDeU e3%ZSY},7]- %tC qF{.VBoU p 84Sѵi<=_ݢ* TlvǕ?Nrcz<'S8QHZ^m44Ce.$D2;'K"~Iǩ5gP Uc> u(o6Y(vnuz\TJB;&huC3Hɺ%o+ڵRtW|,qk'g zSC}!~(=|#RDEs8KbWICN#JY~;7Ii"6&~ۖI+l!'ar*Aˢ?p*!hnDLYAr9[gOekF)xy|bѯW0t&?sP |p;>wt#p> ?s񙧢冏㑇 CWn_0`\Y1:9͘}!i=]YrtC~;zFrV؉~y^9Z+4=@eb~Of@l1C*+SPpitˎfF`0-ItV}k,d.Vf!'!seDMT33JU3 dĠ!Vqm)4y؀O5qlNcl?UI!3y7"\kkM 9b\q"Tmvm4\;3%`h7ijGiF@u[rhS[#n2)(!(r4) lЌq8ɪm s5I!Iőp ՞Tm2M5)x'_O}xo>.0L3Rj o kׯ^<'N;C2~EgsaHx9Z%.K[Ү|MϱP(;n`7 $A(m)l)0|3RH|H|R~$lX]{;KpwK1 L4[t~7*5d*us\5pb T˚y_YG%.P.Bh8GN,t!qZ֏bfN2ݴ;Drq5g`TqMjsbb͌k> p;_UEŊjϣ.:T]t⹤(Sc M1= b^|D֪e6rdIijoff1ef)Jj1M^w#?Gj֐t ]iǀB蜵 t"V Q{IqAtѱs4T}[m€z:Sde*HqySV+ZY!4Ѻܝ='f&xduKބ/p}b980R D@!,|brq:^}8q J>yfB%.RNpٰ Fxiީ;Qkn٨z vZr`2#ܧ4k:Gs_EҕފÉ&潹(d@$g+p K7m9 6v銰k)(eAIbx>ޚ1iŦLN<^$WLy hgLk5eswښ`HĪXl,-e[o-GBN|~k`ίծg[f;% 43@8c~G WrxXL\vFM1=z 9AWY:=;C(x1iS>}R!$=VMKrݫ-hnDhS eYG)RuuQstj˥jz& O ڈ-͖R XAIoHGCͶ8;gV}vv&9teP.>;R~vb6VO5/8iDt'5x t?3>5.%"aFbI)KƲ6lwr26fhV &$F8$ÑJEm`b=h5VkΘ=@uZy1 u0;Pn]"b`@*"#֮1s3Ypa# ;YߛbczLs}U`MA!O׭@uEi*C.IS6}挴9LZ#6]kB AG Hf^f , bY%J,1e0BP_Q6A=HD2U;0InZ}e.N*^$4Vtam4Ҋ-Ս!Tg]2*_y7 Bn P CP+6p5YYyVR6rr$'@kXl=C&Y~l Ok%}1EIWPj-O5{ R *S,ivՙgB&%bڑQc:V3*v:RƦҪEK*NQQlQ@VBtRТhEE.9}~Zzok$wy/k=YσBz53dY7rI!S`B>mL%σ~Q[n^񁷼fy6ё%6FπAy}IږvV %fC8esqڄe(Q{D>_/ $mmЧ(UJ˵d@TD` i}( d̉7e<5UOTK_*lV2") Q:h^Rςo76`4Nvqb"Fo:1/`9vJFgAdc'^C0tP2HL}%@+ٔZҁs1U9W3^p17EZ׬ _Pv2B;4) 7=xD$+K4^^i8-նCTDq?WL "?N3h]".h-Bdn<ڣZ@47nY;KdY;K#pD0px/v@C9,Wjv'W'Lrd|{t x3Id5"K.4(v~$M,2ڿh'Ч.R yy0l2jrM|D"@)FU _QsYcYV1.dO ˲6eY̥!6< ܬhWR'gKvTy^'}سs2d7N'5ӆ2)+ZDظ$~Uc2_l=C17ړ52`xnCd Lr/2/۾< N&k#t<,X $q1F-Yf:W8LI6ZzHDRHV_+.sz$*~79f3YKvvaqe8꘣polx<ƧؽsW qKzjkGQ/r`*MEãz^Mp}>侬P s"8AI.nP^:L[gQxCS]∈'##<Mũ(|ޭ9|cLFgNh"\t8?%u {Z>Ycl`HꤪUbhMKbK@|W! ,13˰fk;ܾ]DTX&S~|v(wvf jgLCk_4&sLR6'gIavfnv0({$QR3PbRBsI-iJ*”ŜYkCV20#?H6tTY5&o`ɂ&&JvLm*3툎BlBU"R@gRe٬gfu& lD;O^V-u#8b&4aG *W9R4ÐYzcà֒'+`.ђ F^3d]p' 8#Pr-'#AjB'^QEqR`bczLvx$C^G1/=2BƉhW^'M0ޚs1 #i:91nd⢢VVO.NE;fVi7Tl.2dQ]`RdBi,oi؄"m\֜CQCA-)eZn4i-Qd8D>K%ɖ6eˬTF93{]n:Bb,Q9m$ ʃA/H"Ly[\ꞰLKs007>h!ƅn;fB-Tm)-y?U@'ne1|>U>sprJqssfk:yཆuݣpz:#ƭ1 pxS t[Se|*ǾZP'zȧ0{۶ރQ굇jn"xu[kREz'XY~YrP+H%6%J `+N~ mmM 0/VF{A~al`%ύkxCt9C?]XNT 爢O8ٻX6@ٲ `T3Xttdj0,@W@qf;Pbcؖ(㪥4JZlQc/UYk-|}e0aĠm'P`]"pS-#_6vљܨtGE a M1=p#⌭Q'tF$SRʹNYe+oFpmZLn^^XG0a_#f Į@b{z B` J<>?7|=%{~ ccLQ0Sj+$xD"isIs+'r+1*(5LL7#v ~LFPϙ0y9 Tȁp p+ %"d95)SdE\lE"DzɺXaq6Y `:Sy?F7} ʥ?W-DŽJ\Trl2j@J<^_w״R&Z ]0WɈxn̎>A\)FH6?VOu SBTa'ԬH8,)ahauy")m%)LFk@05f$vNd RՖ]2@$tRک@HEYߗL19 !% N1xf?x"D0JZ? ,!kSBa`"ղ@m8e5\Ɋ@O,,ao8!&(#`xh {;glx+fw6maY,ior5D.{(&nC1̸18䁮 kWT㦡tTz~z#" ֚jƀOlR-S' e@^p'Nmbj<ҳLw4f1 7j1y|CJqlh z6Vֲ[ШmzmFϼnSe=NZ/* }.!5s5n*ZjAy7"nYݜ{^Qs\ @gP]6N==0a1jP`Qk^7'qF@yCGqNØ%6+oĚMB<t4J$;HiƅS܏wބ˞ڟތ@8kvW"xۛ ܅"Rˬ.舓 5!&'Nhddd߭ Tx<0k`h7v>0F<8A(Vv ;q wV jO&"ӌ@$ 'lOmq/ ޝ]eNDQ58skK` ͊Qb0e2抬`tliQHCb O-]΂e]W"25 `~=aFcs+dq'LSwko[T>JS50CD;nG.ڞ/+5xL774ݝຎGj1[xtFkr =XuOj`d&ǟ cd Nڀe;"0ȭY[n_|%}#e뮭غu+6+p/}ۢ?֢G{lֶShc9J-Q`7 IDATyl •픕FU/@uXqPKp \IWӹ8^VmgyӪwmx̉`.`2p*m[˙MM~ rXfM̽E/wExQcc <_h yyW]ն6>hSY+NN[|,g=d9*GQ6ZN3)^I);{o|0nsV`(G0#C.o|/ĒKB6z%sr_hJ{*zQ?OܳL;^ 7%MzG񹨬+ r5LN24Adcd∠# W]cf6r&){/x¡nyoMe%-t˘,aQuV 3q2b:xe8AQ:b< 1=EtaDak=Unf<ЇxK0W 蝔:mL32(0`7JO0v -pZ~,ZMK/ ~:rD IJK΃ՋU3BAR^@)fa"Fך笠uHz_ԕ<3cĜ'!`c ypaa`ȅc'0pIqƠ(DLUA`u2"lx@ K6ؚ@UGU#\U$n{0x㦠3F5'v M/}:E 3Jq״ɫ6-fRq.wshqΔ!x[:)X^5?JHU={fS T ٮȉ9xQ0!K'{@77}䓟Q;1n[{,sBoS 쾃sV播Q"=z5"^3HɄU/_{3I `pצe @= W@ 4@̾%UE1(Za˱YZc UŅעVv۔W2s f4_TV- f YTpzt_.؀zli2lk`S\~c50y} mrq6])LFԡ-q$5z%鋚_evNN=2\۪cMa%X| `=8:l@ACI5z^%iA"Vc'Sˆ&hPi0LeON FP.Je=FX@UX\VIT71 nċ1ƍ8`J@;;G mGR+Rߒ'5kպD,)%f] :epDt{{11r.Hը%h~ŰnxDd\+as 7b y w_[|!Bvki+q{jY֏:WKyQaa_b_<Qc.d+ AcN3g ph(߱-)3V4Qr VW'%&jTj׺ OqgNR~*^9I?[^hzXpP.(*Yx瀖&xF2%@('Gt.ϴ< YD%M} X+icRj 7kѰL76%ᖶenɊA1lx̣p`Dlr׌F7Tw V۔(`-[ pNa g%pr]q[q_u+MvuN\yi BIk8p&ps.;?x.8ֽ\qXmM6Z \ ݺ>8髀olƍ܂m߭OvUW߽݃_ߌ:\6̜̀ܩb7MۀSw=eaͻ^|yDžXb6:\W7cT^g1= 60dK/MO,c} 6HAؑr'{h0jp8 ŕhލK\΍8`Q6s`e9rPkI3HYA'2I,Վ6ep݇)~pc0$QRMI4$)JYA:[3IHu" 9 h;܅{1!N6](ksx? 'e,Ef^)diR2TtAO݂}{/9 c11 vIWGR|BְHߕ&@}_~s843 )X)Hnc2s3.y Rt@0"Hy/`She$hpia9aiSiȌn72h2NIe w9}M Ă<\3EC5fIG0:Ѯy[eK en#&;@9Y,89SƒdV(-Ġf N{Ҡ|a/`8Ŋc_΀>cgWXfߕQA="eǑ)+.Ybk\hʞˎUqLk͛;tte{ 7s駟[o~; ~cSK}_WX5،_W .Gp9`7}|@_u5>[___{3\^|*^ <`v-.ոKpug{..>+?'.vg >f;qn•>nuxN\ ~9WSX^pd:w' xw~/ֿ{N[qG3p4E`S{/,L .[O,Nt;!&ym|I5*]#~P5 K2E!ٕWXfdD`d('ce/49Xq Ky8vAd\"58hl7rDaxCU#?4`$scRL:PS#zӡZttCɯ y+J/YڍP"->o#S!`H @֚9Ю L\SrD3T ( s$^O.T'YŚq5Jǀ'&2c:u& Tt(yZCFeōɡdϼuŕmUհܘqua\N&[Y61C :ĕ^\Ɍ( J&T߲+b6Wk}{R\y|'> {xߎ 6Fsq}zXwxm9b?zpmnj;|%fy,`Sv1{vs._| ;9g1;(΢v?8)?< aRAfq:kX (ƅeKMK`~[ PQ|d@v+_%_,W-mfJUBD ֺ],6C&$#%[fd2erHp6z$dVM>l{1WHM *&c Cld2s4X)WFu^̳bSH r`T#c6> LOMu5Z?w)ӂrd2L,1Յ{H.8/LG\lY7BYWJ1z˻/ |_ZVz%r2*66[i$%Q? ,]܁$t'|Phz H4Y/Ɵݥ{5HܔS9VJq$Msd4䘊G R;L meS9J ;\¦anfWVmSBv69ٝ J`+ϋycLVJ'l~.q D6'œ yB'qT4B σK3[V7sM" +'P^lJp#2 h8&U55g$!xˑe!ŋ@.T1"Cx2 fyvvލkߌ˿"pm;b;m[LR0`s+VVjՊcv5(kzE`q՛nƖ>"Sv\ 0 w`n Z +lbVc@p뤰6$^rRx$uoj 9*Ag>0.XbMgH:_UEPEA*@2b'5VmW$J=s XX5;J'% BۢugɚM"LGVVPP>)0jLտU dHO+$5RFF^jzN+:YL@uc;|T$1liQ#u-mيUŨ,J~@%ӕbP j[?"pu#j7E\'Y?(̧` :C ubL:@Jl9C7ްڹ|y;~ I8 XN-\Mq) LVJKD 9sKAcYjow[XI40;S420aCn?=4A"(z6DImjcKQf@0#EVm5,aϜٹV1''.|z:\4.4nuD \,Lfr?O- 5\c I @ +aؒ4-{Ч B'CGQ?RY]+W`HCD(Lb5DgUS3 @y{~6l~{n\qk}K_J2>F\^{3݌;OY6{x_4ygWsی-`nfl/ro;=a7'?}L2ƪ»2 X}X7 ͸s O]WK)7[pصXяSWNvU+jѓxŦODQGta6[5ޏ}1r7B6[rK:Ȳ#p?a2L4*l)Hm% (@pMp@_oI0Uz[[T_ePJ=)6 ZR:K8gH'ZuBU"`L"$rBK(m25&5"KY!gp3ZDMe ՟#R(ʼn.Xɸ(@2a7٣ԃW?e.@]f3JW /%|ʿtj8֜!c@8109GF}amb+cq5L@6̒3Y{S∅X~mڢ+dƐz`0}St|֫ 7 0*cjuzY$c3Y+ۮܛOz>#2 Nrb-?&j z/b T@IZۉ68 IϪ^ ؼ3#T;@$q>#%]Yc }aSeԅ&EF=g! Ezi %=z1;,m=1sy# 1Zς_9=!` 0NH+1ƘC?2 9S;+ٜIk(,>[V|ęg׿xކnsssi*>7ܕ8KpG7/lU݄eћp?߉S17Sc+p݀U߂X~X1؂9}:/ׯƦ/aIou_m9/oƿᵸsUg`%JxW}#־"{ո5S,Hс =FJ^$ פsaUaK Pt&z=0ySH*TI*RQI \3 'Pg d+>7ĸ WYqDZLA>:@t!RAnW] zpYzJuFcE(9xP" IDAT!JUrijRD%GJ5Fqk#6V 0rK2LdGCί~[zRdY ;ʃmM _-T.H\D 1n.H8IK`ke1#mN~r.?\WR\?p@E%"dzC0 vu\M;ac҈u?ʰmֿnåo}w~)mTҳR nRP..SQcf]HsHh'0݉VHJ6 F,iQzKV% O|;%4c9%Mw~m,cQsl``{leҥ ⁎1cΔt*3 l?eLQDlR9›-:$ *αdm-3ܢ;L8h/sl k(vt-5 P荩~!b]IGe=k kf P3Udd(֞gu8row}7V` xmny~a_].%/{>x8;}p?Eoƶ{v)󞛱=w}aP~[6/%ź[}p6?077ጟ=c˧F|ߝe/{>֝{!wNlfl"wlٌwc=ۉ-w{K@Nl7`M'O?w7aW0K~f?kVwb+p:`0{D]&u䄡h%.mɗ?,tƸ(!Z]i):G##hT>M˃, HP:Li-9 mZRADYbD n' f* *}R\ݗU"at"4TG7xoc v}\иcHȮ Yj"tTRƦ p}b$6y2^N\g*xD˅iȭt8j,g=Fve^sTdjRpdB6=9NѤt}I6;Rc \׉tG IF\Bb@˲f;]dȰ;g 0K$x)s,0:m 5+ by ;MyqQ9u5by:}(bpp :'HkR#BGqeH-thF0!_J^)QލuE' 3UKvڏ)݀J]_ۦ:?y5.Xz=^7`o=өddLK&H L@/ }߭Ĩ^C-׵5MT;vU^(Lmo!qx1?FU)ki8nYdeAF%3#Vbb|oc`vy IHC:[{( A5%pn9G 9Ob#~?+`opTcqe-Q97ϋ>>cXԚH΃;BGfԊ+qIe'S%^Dd&f4Lղԧ&Z {VqY?^]Nö;0,V=u֞4ވS0nz@om-p[R\4LJ+Aaco"pG8*8@6wLꎨvJט$.qM؎]fۖ]OqMJ7СL ]ٌYj>:.c-2`Һ!$IN֘*7%+R&|umjՠMҹ2ciZ쵔(DNj4A첓r9&cWΫXVjKM~\?uvR ت]4Am>p2hF*I , $)kx6!Z@,h`޴sV{T풡(=S<;,BK߹7W}Qj2pv8;Hb/fV4rD2l-J|=D]W[^>Q6 f-15(;", E&+rE\T?0"4zVS"_!+!v{7@H\뀱cGD|gZR@т:&:!,?g1믥W3vmor0ǰ,g.PJLN,6Ck_yQ4ǂ;Z~=;ᵡz` ,uݎʱרXsyy}V1B\Bcĺʧ޹o%0EiPkY4}t:(UC8gVQ0;ڙ욀1;&X,5E+rf_!3vN#pn-~:NڏdzN0^R9ql=WPWɎ#k!*y-,%24D,B@_-(!MH$l- zg3)@0a!0!g! p-qj"0 1%qX+$ Ok7 LgskX%:Rcd} «LމcOIDX,䌑Y^*H4q^%M8 wohn(E}oZn7;b9z\03SzQy㗹& `3[S-!i4)Z&4\sGaF:kؠh[-CEY|\c#8Tfi7ӄc7yNUʟg t ~'C`PBSE;#%p3OpwIj$$ "(6bqv^WLGZqZVfO HbVk Pϸ5,r9Ч M]&dI/|"*ʦ9bZMX=y-cz,J?zWb8 zJ6PM^]%/Z'R!G$*Qx$j9ĒoR(УX7@lr*Ҁ + ,[Kp\(<4zKpD[:>FMD0mԜU 14tgp8ԝN⾳h^Aq/L9kN҂pao)È5vtvDŚܷf~ϙApVT۶ƑiWb-]8QqG-fiH_PŞ*Ǔ=ۣ@xobٱ7r RL{sz06Mh`VEQ[>/=ХH(+ػ%'.fxmFWDI-o^'{jk6y'jbҷ]nO^x-sJ*Xrkgmh G(>[RFH="KV)Kѝ>f(˵c Rp:Ь>NBĂ{ŤgE,{#c<1U ͢Rn+H."'[ ݬ\0XQZ4YM6;l'SG\,K3Q\eD5SpCG58ZTZ2")E`}Ĩ ̸(&leKqdWk릍e=t{2rNN1|A;y&;> }6ѾLR|kk5r 13!`C8865<ŀsq߶V8Jc̏$ӌV>ѹqnm󿟕aT(#yAyʶu @\rbB]rكYh(դZ*/3Y*TKC4JED ,tp W.R ; D@w zs֝tv) 7=8DG[2>܋ ɒbѭ9UVh 9-Ɇ\SDN#+*TTl0?渺OH>N(:8ADbfe>N| _3W<̬IJ"dt)CFgja 6z/ůBt`6` i[= jQEh[H<nK;)MozT&"V?r99ĸwrXASC9OE vKN6cry pa Bm7|_ _}A-2)b]#R, s'*5&nhvu"N/3}&cIgKdCLzKTF@}XL Y1neFvlk hZnUzRu!brՈlmU߈]P%qۣ3 |47+, -O **UBŚbs3kd os/=mi" EH?sNXJf1pW!o\љ >Q]j]mvSŊ@zڪҰ̸=`\.kPݓ);'YuF!T1''"[aDZ^1XBc) 7=Ao-`9`ت5S g#;Qc7:I$ZMD*PdLp '؍mf"qiT+U2PG`\yf >z93@IE> m0Þ ̩3g^֭.>9`u$H@r MjMPE-R=$9m[Wu БJF2;;wjj$6ItҶŜH1]rGKM^q@-R5"J2M9q [>:Y}jR ج'#f܋v5S4 J0AM}b^r%6fVѯ:bXmƓa8Ήٍ8XϸexNւ?VTH$֫g<C&=1 |C!\gIb8R53ӠwN!t67Mcx+YX˘q4x!pIP>aN1hfͩaV-k5lTrOY;I8E'yLYEw4R0bu"]<Ȝ]6mEK}eWNO,[hU5s3e(2ڈlD<W8VC[N1=ÂDűa!D6u|F$I*x" SLӎ/@TXx=?j%ӶӗC2Xu',5^_-7ބjMNGL,G5IS^͂[;aN@fG<,yS.xP{l>brm>`wkJ/3䭻L@P |FfȲX䜷To-h%.9ǖʜ`W HN#JM1WBI7~b4ɑQD2CldaZuli>/;P IDAT&-Tym!R߉C3{=ŬB8v K2Rrll s6U^o 8`ȹm/y^17=?@aRpcc=A"fKɰ`f}J~iF`H*Ԯ$]uEHD˪~E!purczL6~0< gTg94hW i}G&ʔ6WEBwF !fO&r :AT4 f`T%*bU.Z01ywo4Ģr`ġvRr1%e` we lSoO;G{t8e"vPMRmQ#&bg<ƕoN͊`-y$_ CC09zTl i0A| X ĩ08U͉qLn&B\ؼ УNzXD3C00# ܉ož*ZèZO4 ?1f) /" 71iێWs *)1f?OۄiܵCPqS5M{W1+HS=KOekp7~=o.ئ:t9!0cQc |r|̀T<˲<ǭy~9ל~݊m6CQ=ʰzXҙA/˰/co` mAR |PH<\T$t)N+Kq,C|{ s<˜c~+ڈ58OLc$}4pb6'J?窔Ŭn|/ң ͷ`=+~sJ ԎF|^3iSvp1R52/Ubg1NМ"27 KʢXL݈H.eK3Pg&CwP #{_~vx|{Zu*98ߺU^c\&-~n""O1000 #!I@,,` i%3K=K\5@:YۿGS:v [7ȵey.Ӈ| xa>f*."N^/_z'?O?v7[`=ͱ|GuCh`8ߐ[ЗI@岏 ;pnZ̋xp@N𷮵 B ӚM6$ςn~ A].R>^4kX=[^_9iv6w RBdN1C{6JCgIVF3KFR`wY{sX 쵚0j}׀=0'8' 98yk͵]s/js BM@Bj¼q1Ú w ع33?rve eg^qt=y1^M'%sjPf`gH556Uݲ3 }+ 8*mj}r9X5' nh_톖WlCBs2IDU\hA5ߧ2c@T'gq-8j')df NRC 2o}J%DcKئMART̓͊'f++*.R7F6Z}@|li? hvpu;;_]O>^$~үW/%\s|vNr~߈Z<_ſOO> ؑ;ô.oabZOG. W ̭bˬfԉ`_vqfge!#Wቢeq40,gz$ Hng=lk=@8ۨNrm¤Q K)Qvl6TI(ܞ45.9IgKOhUNF 䅛-4=^u UY*I V3dp1؀p3NW8=̀qB{Iͽ{5~R&聠`GB2QkrX=5;rL'ţIzͱ9FF[$d.8%+F]fYړA֬(Qaޣ{^[21xIi H9'c\Oo¾#v?ko Aqs?/2v۾㵯zw}->ԧ7{1絧5w/72-(42Wwk$Ŭ5++)ְxr=qp0R6ZtBG= 2a]Vhq%H7'dQB1;Tep8*4<TG kNe䑹{("2x@.^. r: 펭hJI1'coio %V]_2\ׁKC ǃYwުn{08t:vv-;_|3M^ b)Xo*!LSta벺=XI?Ti$MMgՅ{y Smr2DB@8CtL)gU}>Z@u%-:K# &ڿp>ߤ&݋Iw`oD*bp[KGkt`x!~|5iQ\1jemx,,M3)DDʬA$Ck0IKJsjunq}vP+.)-u+:yȌdsU}zcfiVndrisJƞ0C?sQ;:7 ;t/Na~^x?Y|KOΜ֎Hc^#4 VO;r$7>OGmXG(N*I+QZjfH1n##puL. FR$%9;I奱9t|XZg)S d߀ʬfPwqâȷʪ 1>ZcaTLj7|XĹ6NUx?-xZ~2^HL5Y3N*@ځJͳhm2G& )C 3X?ϲx9o)r{]pE#_vu); DÊ6h 4qeZQ&dfI&kS1ʳ- =s ?[W"Iuh X]KV?c@Q&mn8o~S̏Iu>[QigY _ebm/, &p9b:L:׫19@X`>A1Dj*d^2c]-k)>kIrupNS:I+*!apҮFZX5L4 Բ-ejEG&(/{P,b"Idּ31 D̴Hz~\tI\4$i]lhiA?DxǏ/pՐGTJ~~+I1?nhڕy"i&375Q68@QƏLQy}O 2^g: #"FwA\;Ɲg8GB!lbQJCv$wN]kW' J 仅1JU^WmQ=G_ ЮM)Zpv @:] xK:g9<ܯ>n3'.>O=K!pa! Z2U &㜛nT*-UøҀ&aވ{|dkɥM se1c^]=7'W0y#$[zo8U(m0X4#6DeKc_[ta |C굋o sLVk ˒5c}d1K2 :K%19mR )kiE3:q(qlȠ9YțVbŗa|MCLcq%ӚPRg dmӮp2?[ 95 |d@l=V7;Zs҂=zUv[y/n~.=4߀3.?xY=΁r68fm3In56Yd^B0>^%Y#f[-R>ik Ǖ$dlQY%8l ql!.L:t1E-f*cB;+31w;3Dl!uVF2t&ض0Ge$Խ-_7'KLs)trzu-ӹr &m2әKo6JrQGp: Zq1e ۯcE;5/cOvpKx.<-7lAa=6ZYEWz>T@F$-e oi]4gZ]LUr1̃ҏ,XHlzl \7hؐ3c ͤ1hfDI`q͓AvJ(]͒3MbR7e|xxr:c{\ X֦x[Y[VZ:8RwOi-M@n_ͻbݴyH{Z{jSt?IR֘4pgl$ѢX ȦjE:q@\>^L(hPdL3K$3'\*b /D948ߙB:˷M 85P'+T ӎf{/cK$ c$tq#\q?3Ey <4:e\qȣ6t4 (`vZ]nP(V|\}vtkz,MHBV|'# 6SF۝P%\}r9;f'kS8#R䙫[ChU w⾟5WKXu=V/yg?;;Rf/Hx, x` ٻӬ:%aa/hf`^=c5ءofubA9Y*XpBF39p*IZ6 {l%1$<Րc{\іK=T3gkNp5HZbrNQ̒2Mx]xz VK*e:w Pτiͷ_i eǍ!ZcN(rXڨ1%ioGCn0~1^]`ڰڝ{Ǯ1P[K1G&N#nӦYm!6^R%7#qdb9@\NG)ҿa1#E1 v.>w m=}$K>w.UҭdgƸYo3YNܣ!{\'#ۼgrgp$j1'<{fV3Ha>-)0  u2W$=f%%bt%B0TfX)Ĵ"Ncar/%r+]i,ve'[krH^Kq0M=@7p4Es͗3Y1Ăᶨ̲uY,@ό$O\ryZUuf,Z4e6kP0VC4v蘸j[d%;iImgڔ̯K~ <+;"F IzHa0-2dEV BW? zuN K,$m!DՂt<ĺȫZc v|@T֖)]g#J26E 1D袵}lC7RU k5} ) ْzגa!vFLZ g[9?:}vcuurǀ #_w?O|ꫥ,\l%,J/˷3{r fVm3iGѪqs'%Z(IUY&Fs$UېS a\w 22F8LZ52}R{QKj#f\ZhjK2|kO2Nx"G˿_qda,Mnork*/VP7o_4*vC1BH!;Wanq캻ҵwkC-nmRf2[,9+JN1^]S\?Ӥr2ds.;t$J^?Dm1,V[ӶhG>a OuRpsib^L8(ks`C8$>ag蓖>Gz]:.d3xng|{ݠ(Lƍ8>>rmS*$iS mr0̼#̘&<)ܘ!z,~,42>Q rc{lIhخȇїjJϰmH!iicT{V nġ0욱zJ/nn^\l2\wۤxF0IF97V#.NNIY+A&%QJLi V1%7eB&GǞ=.8d#| R:; |!K찜X80E$ )g;ǂ ˎ>=i GxrZAƁ)#JOPebDn]A+$  RMYQ k=%({\ʐ]gNT vfK12&:9x}"6=kjr/)<1 r̳%8pLbS!\3E 2[~4F/{^}mbwTNKOoǼm:/K!Ӻu3( 3}mOdz=ȀP}Eoo'#4 3c0w)>JoKfM]:4Keq2ʎv8p$ݤe zS{gtDf87F`wǶ&Dpo^;DcEr9>r2&-&t\c{<_`, Ysac (9*TM$=_.?{бTPGzFmVdƐ*g O!l_xݬ`$*x^Ǿk%4ڪ [`ًL@2GP6cwI ^S!%S%ˍG%^̭# mTֵdp *T$R<4:՛LzgFeXC (Ÿ>-NG޻H0T+F=a ÷UjmY<{׵4$?B is[9f-6uJ=fk RZMӺ%-#\S܎Xncʭ'-B(2(oStT WsLkk8=x@'Ͷ'o&&l? Y' "o 4xQ 5-P4vl&g ƑI-4Zusϲ1=@VO~=Sssݸr k"Ti.j c<.ѻ>J{xaCIzbrpa{lq*SWx~YдIsյnYu%0'">-Ư%9:X*1)6aUBFHGԜJmmq|b'CʑVݼ.dZ% :XWB1Si(m$IR9<bY4HcV)@2#}4YE=0rIb"=%ңy?UA1Fn-\&MV"cŅOec£!0/t6 Ծh`#-˚K PB[}ZO"{;F3QdYd Hh-/QxKe7|UY\\7Rd4 >)0Ik|s$*OAK2&0K,ipkS.5 ܸl1v_0S(}%z,;NN*Gz΍q&3Gm~;?oZfb+0;+01Z'8%לYLR8χ}{p}t6{(LoyIh!6jnδ&^͌X𒇾~ d8'`~~?r%hc{xִ}LTcI [6Xq|r 0yC1?Uܑ5BK#0v[#oYsAesGm$q wi5A@99# Ph:Ee|ʒ'+X5tMA,= H +ڵcj\QVTKoQ-I:BI(jU#[AGu5걨orcb\ dgBWYN3jv6cjAel+b5N횢UKSA5=/Q4hί N/nwD*Z"~-Q5\;$b`H\YDPVό H), Q\ ba#KR9<u pqR4._O6U4+\HrC/iA3o̠Y+Ul(  ¼´Ωa`wz^Ɉ`K,g³${ff신#m[X熀[n7+]b+4 P".+s9sH)MD׈1 },Zpu,D7M{*J"㱏m㘐UT.^ȉnb),W\v6$RKʃԂ?CO_Yr6(ՈzIv (EL ecq<}SY @ԏwU46 W{ .ι&T~ҡ :OTZ:5*Fyt[e ju#<9fYMHQ3zoQ@AMi d7w.H#oJzN|t_61ZF.Svn-qP U̡uOS ǜuP&dǚ 5T"knNje dW\'KڇڭQw^S/qޒ*}Iʇh[סyUQEJ #o3Ųhy3wX6L>}h?X`'ńpZRw]W^gdb2ax]b:;<0@ZX7梮׬;"qQK\.XM`\סnw71Z.^{ pEŭ㸖8feN:ɤ'kMoMUhwqnt90Wzlrc{lUoKg3~c|yY]6c~c>/ȕ0w'[Vp =Ѿ]R`ل'Tll! is}x?&Ӣ5rq*)uLGT$"$B$u c=~}b,t&|)mq`LB&#STt/Q 2 u*s^TZ3/ Uš0E2^IU :O }9 ԊIкr2װL 9i'7O̯lgYkCBK,džj:J,zTɃq>$EN1k5b~ Ol~yEie6q!hff>-2j`ɈK>#_.xRlĠkߙ1X;3&=[ ^<16|F\\_wZxWu\bM[PNJg\o:S$E=⎩>,U$9,gAc_ur[##'v `m3xcH2)DہS%$ly } a1+ 1c}IFR8^tcMX<* {Z(N*Ikk #H*V_LsD3_I) J|[{rT$Ӭ)(0PN W,-ln* V${o4(qxH^tDf59IIDŋRC]-~.&ugwlFyפc/GjZÀ"3]1yL hGT;}=؀ ,ڃgLO2O*JZAG MQZ8kXи]pmd&u Ĕc F jf+2,:[pecj-'NB5YTѯTM_ORcF4Fºh[|wY$=E4*bʮ Wca*n= )|HΜ=˨x祹`{L %zJQTttA<3׆%vg<%I~p#}7{qAoYg: (v V1zMBWErfLd UCFhٝE^6e[N )-3yz 23 SeΓ`iX\&%;j<T-c%jsW}s#~b0ൡSt`3jژs6c\{~تJ]zL͡YVL1<#_@"c{fDzD b8UĴ(D!w4_ 0VscڇoLphZʞ/لcm.buPJsCލo[n!YSL=,W$<̇xw#]0S)Ɣ2kQd埼Yn`(zʻ ŀ៘ĵ* hr! f_W֬##YK@Zubu~a?mͤDPΟ,>`Wvp9{y߿ƫNtG "N3$ ѝp@$(ֻnܨW ^0ň:V@B a"ZV*߫_x/Uu-2?ͥD/>Ml2[ap}cC DI>g\B?oeM@ )s7NZk)@. Z3z5]ޱ,JduX3AϤeY^`@В*@jLc튡DH/ 8OyW~\N[-Kah4;Xj/WK vbɊa) D5V;!?⯙T%.Nf ٺXJvVE6~.8<"855hUպn- 2AnU7ހ^sz(iD;4 WR-N2 q5u)p8pE$U@p[qןxө{x|+Wϳ{^wÏ~x?Ɲ߂[hދ_|'Cz9wo7)alqcQ8Ć4=fO^tLcxcwpEVsqYB[q/l8jIB~U };q'o?|^v^O~[w>^n gބ?yīqw ;%x?'_*v5y!4>}xG8>|VGo+o${m+ h9l"22M;kPLp=o.j5אuv;MiU," Ua}a'$gԀ\9#ljXwiyMA\E Ia :6Z̶$ b]qs[s0Vwx GmjRzF0$̯G`i}\2т'(_ *72t;Ŀ!AKBFCְYP,K#Vl09(Lq1Ua2.uҀ`NjYTD ;,Bj- 8\‹g5s sTF" S.;ʻ쵦 Ʃ4֕SF;Ue#tXIͷނFivM]OjGNPaOOHX]Ֆe.F32CC^QÐ6%܁7nn ;CÏ{l~}掻ǿe78ߎ;n|?}bހ_^qc'Aߺ38{2(Cz7p\ނqۣ>zMHғC<~I?|%jE ҧ)YK82vfw9LRh=TO[*">5PEPT*x&YMYs3lZl "|!e=H^ H0yp{xTHIB[']iV*\ ؕ 6p77X! ;Wo}Ql@̅'dWv0G:9mҘ^{ ffVK1Rjb~fMI]$-oF3N *]9m?QaČ gXopv1 mb/> cĹ 24Q,`ɭv5W:`կy[w+n'nBD =҈D~0>czW5kL$ֺ{KH Jr%|ĪDQy}^1ѹ,WmfzEaa؀Z΃v<C.+Fc8sp6̛\@}xtZu"oހq1:vѠqEr@]pWӚHhɡ 4] j ă=w݊xөw;+p߂o/O#|%_ƻᑧO=~^wO`3_v;=} ;B9g.O(t^;qBOcM=IB2a<8gl(A+ BBL˄:@ d(4C"Fʒ*[rIb[}aAZ(tA8Z}[ yz@$I՜s| ϿE\c@"W8&c[d%_5 IW=^h OIk].+Kh&/QB0K]Q -uX=gUWI**]:E,W=i-Bæ1nbJ.Ã;k,:y%95R<sh Zy2Biw ~Z ꁢ}" &DY̹]ӱDGc,ȅvDd-f5\"2eg{IT6Gse0qݒֲ%E ɯK?f`ZkVʹ^Y<7Ԁ 9.Fő .{]hMQj]T5>}]7;4>En6uH5@Qc2~GhmZN ۅVG޿7?-o ߏ'x#uk~eǿ'1z/w~O]?7|x䣏#C}U}"W݆GމG3ɿ;gn_ _̓G>O^&kƷq}k*viMMk&pr#3_qGmM ʬ3Df$ RJ*5FW>Ii24Ux%#kp|6ƆZdl4Wb$*8t2Ԏl=ՊSYRh&Mh09eA|Q*}#& ok :Z0Ҍ_lrJ5rJIbB;}o/@E\? ;>LL`x؃ 0v&%/V‡ͱ=*4I\Mk$uJ;O !Y"{4"I!eu%⠒[0tXw y:liē?V.l<f$}2lkPle>LNYR]88 z&sst ^-1զkq`;0N3b4'EбHvzj<se' j~2' Ids ._y[gnw~]x oc|x(nW}l_G!\w]!oo^v}+?NcJ$?K <s;w D)sb"|y$+;'L;c<*7 )xd @['W)m7 ֔!B$% q %°;1Ly% I" -5Ɂ,E!JEU|G (I^V@ (N2VY_Oi23}n Yb ٜym|h,9.'&SF~[逥[ۚF=O+6,o&q6R>uH`B]JS2("-bf2FO}kLX1\SeW]Ã9gŘ8ec? DL؈q z6EhTg D&:DXP^w'> 9 61}zMZy:uX26+·!-9dME?"dd*Edtd)܎v16fIk_/>{oyNcJ X([HgqT7#}\rWu+D _8 9g=z?}~Kހ[ߊ?G|gΞmwԧwv9 B<͸+| ~Gq]w<'أA߃75x}WƓ;3OYj[xǎ$f4娚KshAIR^\< $pE+Sdxi'Nz $|nڮ9@&._eeslK4q0#%/ 0qX)60ҸtNI2h &YcgT^:kPb/^'ʼm0^SZCQXqE}#]f7R$BXVeZv!KIXM29R,\1".Hzǥc%-Y} b%+ {3~X0ެaU@{ۂ+X6K|,6ajdgblxCL}۫Dbک ZVd;ưj,ajUę45@O]LߪeM4{uIdwIWQ:lԌUltk2B ɪٹ0%ZKKBMKO!HWZ'\M])J+62qcׯqw ;N>jփ] vNȮ0X3Ƥ7G#Ծ+)8oo6Ś=ƍ츩uU`ꦑ)S27*Ib4&kؠUNw(ױe2 1j$3ѬfJV;ӌ;ͲW!ղgqc88w;} <ߓxf?۽_W㵯v>pgW4^v~`X̯tϛ' N>?{"21ts߿2T̡ev9 qiA1(H%zsI1Vtenvb ]Ħ"XDmߥz:)rLӚ*Vу6 ՌTuT]8')Sh[T:,aR4!ǰ+ϒRfI *P]apqh9Duʒt^LagX/lϞ&ak\(IeBj]],6ɩIR{D]zrQxEP\RG~8W8)E /vn5Ziҗ]Vgf+׬s5QK WUXU \{nge_vOS񲹎]gij3ksefiph/?,ޮ_|{sfڸF/q g$ŷ2wV#}HW_$ `a?Ў \!BƉܧ-uQ+=,((Bcnr^${{.AAB@Ŧ)0K4Fۋr x򃫥!Ðg~|?=>qW7w?'^v|1ǣOΞí/x͋qpqJ{7 yh[7M_~7έ{;uso7om۞\ BNZ%=Μ=~uk kAWWL!W4+enSt na :+h jsyXMuZh_TW#2G8% :o\gV;I|v8gfVg;Z=[NgY"#d"UD~TЌHBBLy IDATeI0a0i%(:vNI>d0ab0[X"BuҹMJ+PĄپInE֊l35i-J1}nyeWɣNn|]I-ea,OuE=cvB%9ȼ_jVl%ٴ˳:4ܹ3)V9VbV/$ϸM*#B9tP uQ!0w tCD:tt8\O dpR* U+Gd+!َ:Ŵ% (^/HY?X,۫2Xnt5*}!D";UmL !yJScfb1=H^ 1hӾgE:q*nܾ1R\܌@#h+ע3}T MM&IwBhb]%H)q^C;hIl9|q<~{ċn.'Ń?5^|?vO߅Y<x#eػ7/}?`Fα Kyna|ş{'>U+Ur Z7쒎{GrM# 4l#$Z ;|-=PuB[m S} S8(i5m~D8\fQKGޥNoj1YJdVBe[;*)<-)hr먞 t7DUB-i:bTc}4ak"|#>nѬT$3`.lPXHe"^(13y'fA{8֓ g I[n0"n5~S!yjiݸ3xo^D{DMx%Z\V00{+Q$F ^*,ie*%aYL+" i޺RZf^Lj{2WV.;X9<8BB=|5_vq_'gZ1vKҚVi2{g:m{U8ʗ {.{;H8f c뽨CgR5-4(;uXk`!Eĥvc~ne }ǚϾ[fi'613E(/?kvwy&gp}otb`>]hi@i(Ŝ&"7-0'<*١ ~Тc{|~s:nRdVgc`YW9!R_-=_@Կ}3_rxYM*= 5ҵHZNH:xR+"SyvYt%ed7 ;[3Jh" 7y5t3+` c7,0M+.QH> gxt~u{I諮LQӺm]p[JO%àIuIU,>a : FW z4Z*ן[E,'̀K&I j\P \:bȠ$ Z)F+-l.zd[zr(~,υIJm[ʐq,,KfNÙ {Gx@UL8 _LmLq1a|=PW`IG+.1j6TSТ9bh2i1i^s@^%~$gG'ߴ[]C@!CZ('cWkm=,d<տK_t y{LqgG{Kom@ݡذfSUơmhV¨i52h@h+$ pEݐ AP'VFm2R-m$A*ײO )Q0e!Z5iSX笂diJOu[X(JDgE27AHGWmd) #\=C$GL]VW.j!os4ZICfPׯ'^OObhM-sFnp4Bdݼ0;qD4YoU[9пVL7]F,Gd<(6э)!ɷq7U!Kt H֒k04`J#84)S^H<{tG>T &z-iLZq(` 졹6c>7pAfuNIW:o[Y V7bk0XqVM_Y`YЃqTɀGeƩz"sRe;(jWxO9~.y:0Mb^rR|CpEUKҖ1ūDk2<8>r'~=:A(9yf: wOjmH$W oX0%min=wD'A6kFKJt.B,(^Ÿk&LAs3 ^4 \sǰWJ%a.Gwkcsw矿 GCn{lqUQX JG<9VHۓk{M׺)R熦 LA!c+ 1M 0<^qِs-V#,.RlĊ}9WA94ۋ:t27?f7uh>x2 (Jb 4)Fsf%y0{qr$CN>Y]l[f9Njx6dt!߀Khm[VOcT[k jU}w~&-S_FӇLk,Sb.O 5R"!M7dct{m|lx~-@p4a/:E O7sc6Z9E\QBZ2L,D`l~LH7ATK sKÏ? qqSc򞲱Pg%e/5gK9_p-f&uE 4ܩ^?x&΁^MOR۴@3𕑔dd "' C*Txoʴ?eebu X gG]VDG55j+еPkG[dZ% zV*϶VtkޠEmtrCuUx΃,[%[÷VCؓ>\\0X fY{#3Rj .޽C: =A,"sR' Nԉ&1ribщ1,ItμVTD-D#yvèރqg}&Yz-+VPD~Wo:;"vqJ:1!bD9`[S=1ݱ}}'hQD `/:?ø݄mWʬT`EY(7)ߠ4 `K XNHuANFWV #D@eZ̼q`dPR 5Tӵ1>?&8^ -4Fd,|' oFiR5PE.5b;"E_>&:uv9JYcd|\ܹ/⿕Ya1~J>?S??ז?x@~Ev nCs1:,hI`9I*i6u/ ,mT!+#M ѰNaT*ya%ćE'; s9a#GRrXy1h:82RWo]ߗkGzvyp ݑIHF4鹞=mf2vF4S6ZbkJ"@ȏ=8ϠUs(lrˊ܃4T,LϿUܨ,%o$o#.|*yMyRe4PFC3'sj*UŢ$/mfۑJќI E_k:%Ǵ]ꑶDLec|iî!/},+=eZJP5/' .)g45&ָ~(5/;e("(_?^z+?^;nzE. ^Ei0L[#R,FlX Cnkgi\D{mϳw@KIۖx=.لgm+Z$Ղϰ~Gʯ;\nno Ag,yݰ)|vrsF&&zh{C2#]eWsW&4+l\Ґ~\+9Ҽۭ} 8Ug0 M*>3dƩ,f3\x,^  Oߗzeyj8rLt2o(^QFyJJ~FƘ!9ԍ<0H\b"|ڲ #A ƲЙ %˛NP# RDp:P>ͺXraMںrmu_9 J}rzFdYzgtH=&R}bҰlK}'yÉ G׬8I1<8yyoV-UK9U8&6I0k#B$*{:wݧĢZh2cn(Q(W89pcݥScYٚV޳\wM0auQ,MDS1@&Ʈߓ ʃw6=epCfD}p3k+dVwI 1 Kɘ8/ @yؠnx IDATc?̆C"ݬcɊ#M5twۿaG-՝&|\nn-;3H>y )k[lUz.T5k &DI%suPC=9G0J~6;x%N- 1'slzX=,r6S@)?>7w?濓7~_Ad]gRu0@b@ܳ\F'׀ۍxܨyBT˾ZL؉D([mqV)5"`9hL-ɷs=.՘H|_:(LKvhAXp "1?QJϪb:ƽ )-*د)abnQn$Waj8tQ["ņHTIs3 GFNimI/RFc @tj0UŎlii>j/YOndGATæ0X%'q 3L_lj2Y@s'< z͘;TW^gjKFWF$8} 1ʍCj!o5 EE$[u@\Y)ʫ)Ǹ[L VncKܴhŮUKUh}Lj~N#%[Q4*7&dQ!Y3WϪ7ݠo)1x0^7(W\Գ6H*^&Ubýwwi-[ogV3j$E*w XW"/7ɞ5(qރfFX.V/C†%4uMك(s΃Y=: l_?.H'TV"xC-\x:Iwf-jx+Tw6ՅZ;ųeO*Fk|>lo7cvdiu{z򻄸N=1]PR(h;Q!96zL B[\uUbAf42}Smle d ,`iwWztWF=t~Blt7&qܱRTRs:Muf3'< .Cb]] 1{,rT. ' ݅d1?z0}}jŚυpR/3{݌,+Ƈ}/bApFNhl; ׭['AztE_E8\>|Ç/ZyW_um鏵ucKFm 0z*XUq/pH#F`C[K/+S``Ch8 hN/O}aF4rkys|G~-yk9N@()H|Hŷ“ɟbӚ6ğ7ubUW|QRƑ TmW7 iX":}#[\ xuK0Bp(7Wmr4oA2E܎>%Yw;7R'AS?2CLdg^N~ l^1ۿ?3fyvRVbJ$GzF&!ckv|7[0ۗ7:죉YM׬ FTR >;mGtiYĎz81mVi2r˅c `BV~by?XL`pAQuT[̄dfxGs6 NMU*E&?(Hʏ}߇N5-kŨt H(σuDcwH :EF4˳PZݾ_87x]oQvPӧ-Ϳw,wܗ, elZ(D,x݌?je-&qȒ uqy5L N Ēu F%M+ lڥ* }$Am*ju, "&b'A+m4 ۜ{o/?0)))yϩ" H!|2k`Nwj<ɽ xQ"ֵEDׁIDf^Qڙ +{Ӽܝ$GwΪ[$2qKX+ ߱,!ǚͥ>T5Z=֒Pyk\)-޻Lxdx[`vB:PVpͱ Wvuİ8)J0+*VwfmD&QҜM>ȀH| G9(Gyjg+Ii-@`VU }fǑX,<mhc̕ѿ#߿7~$vsͺ^U\dȦە,tW^l?;WiLڈ}rQT},(MQ ӹ,5B>kg +7H>vo; ;{<7^yYa~?TDߥCڵ&K|;֎ZNd NȉQREǐvnݺ1 soG+OxLI::M!'BH1R)OJQ`?L=zA .|%yL2]z>ecV# 4_n K aHFh-㏕׃ +6:a 7Ƕ\P[O/IJQP%jYAa3ÅxY{#grF&]z058eߣ`潃6q5 $f*J.z FG ޣn͚v R8Mb ԞVSǤ\_2^cA=KVp hFPGZK 渚wM|J$c_׋~tUGV<"|=KJR/jjǨ|M!W>ӊzRw~ǹɍSд`64 ksqt&7C Ù||g30%۫hLƃqVt3Ms227aq,m;4.|o[Ub&s ٗd6QcxZ6Вt7rN?۳~ݳirNwVc}~@6Bp `+' }4$CW9i- C,^{{: ׭[oCx/K++%1]Iwe+?zQ)h; bt:%B8jG r^,Og8{]hϐ@):āed⃛;c18v7::t.h+E#{k XQ$[>||©䀃.O>}C]2Ď6L^@j*7cѹ?j9/)(؈}K+8 {E4#DBzV8stk QqtsuHn6?*4;> 6ƑBFAaq|iM#b=Ƽb&%^Hoi:͗7 j(>KC5AJAD }S\4uZKEec-"1*%@ S9fɾ4ʬYgܖjviXҾUt tș8x<0s:ep ^z¶DG>5jV<9gb tۊ'?">k^,>),tm({96z 0j#\nݾ R#o{Ap6\4(rdF#) OrAaB4וk6xcqADXVa=u<W>%u28tdi6rNpDp9n+aJ=Y9W.A"~)}eT HCX v!9'3p@@Lͭgz=c\ r.Eu%"pf&Bc/Gӎ]#'ẗ́F͎tJu3qXYXu${ Rݪ#MݜAJ6O+mCɒʛn(h8 K'IZ՝Od}ȕIUbaXś;o\G~ d`O`jV|&0L-ǬL^g;&@kU &Ztm,K*.D0x9oM~UH6AZi2QTcP1 ͦXЋ`_ ΧXJ}ڽ~cSnd qv^aٟʿQg _y?u)">,2IJ)ƼojF "!w@,Ba%LKa:ǡxZgN<*ѬsucD`uh8"-NZ9ܹa<$ FpA]Rmȸ|4vk!lNerNMHdAh4ܴy9^8m9(WQF!9`QB:XH)KQʪ^kn2,JIq,0$`[i2p8(hQ_ 9x9./ItԵi Sv VJVBʩl &ED N&6֫zH HQ^rѭ9J4 EeLmjבlE#ƱZ)w[{w"Fh%.xU92٘ü3mKy!pXVҠ^#/rJ^}YyӓsYӸOsmtҢX MLCXt2WgT 9)%hYlS{\H{|{ߩ'u K,QZCEH>UɊZZR+2ƀߴ甴kfgUN0 BM\F|„,@Q Zq25x)oJy-0gUVĊUߔenɦyZ<}.%!Q.֤ T1I;!$K\VMTA1iPclG9(oW_7._q|[(丼=yѽ5$&/*8wC5xT{Za|1d's"a^$ʻьeO``qDڤ)g^q:dkRHp߅~x"w Fe/gy.35H}7lץYO<)g@,j^xKr=t@N(S׏h4L16{=ET>b1lE+n ?vtr=ux-_;^\sW Fn0LρaS*pQҐ{8 ,DbI.:&o3炖[)F·7)Z88MJӁh'5۠>xH^t`#Oft,hsW tϜq%Sl`cYhur= q}ǂq|or  |G=ѻ P0zZ SpnAẤMLy+sA@ʡN׍g @Ƽ9Fq5[.1*! e";e@†, 'H}# 0ec212IY͊K eTZ&r1pad3Q푙bh0 LʾZ 'K/~a\Vɝj#CEqB> GcE0Ɏ_ƯPPJ ȨEd8QF%G}$ A|S(٘"tuM#%ڳ'8L%h$su?O#|v1a^Bi` b["(X&ea04-*vM1XZQ2xPֽ<. 5hяĮ4= _XX;5:hI3k/WBp+hI=uݻ's\|3ߴi[cP{ѹ 5o5"Á nP4CtA=/HFьIX7k1n#"@;1K_:5411w-G3R4W s8~3p3yDޗ\q&pvw2OSijfSQ'(,j;Е:%gGr8-Y`/խ&]( S>,+/ՃӰɘ}6ui8XFVFRS- $FUgI!/ 4\OF `JdxU˷V=j܌+N{ڋ5S" 9rg8s kjlĬ}lҙww8x}A(Gs O< (Kզ/z;5ٖxyCJLj'ω;-P gO'fSĢ%IQg~8\~Z1D"`?b^}W͖`ƦLx]IDATr | iy&>EQ ާؙP@L.4^c:.y;9-]uhE IjUv>^DmL3&qq2\X03L.|o߷|w:35cknL^XȂU!AQGNwb s1v9Yx5h2 93$%Zȸ  V *)JD2Xc f札1sgcVWN_f!F;ez%Dъ- {6ZBsC(Zi;|0gج ξ9SMX&gϓF%OZg(´;DZ«9PBcSr@!s7BD Y`|( rwrg8%27˜pJ{&H2v=[O[:`mQ}\5 up72~N*u$ KqfU^АƎJQD˲AgB#{_wx ] pHp;`G%7BD3]P&d惩<8Y`QkTUQ> V5s(؀dVG%gɘjۘiᮺ_9aƁj$y5{똨Wݯ~hBZ A>x|'>LrqҬ 1-4=+FӥDaҥ=X IgV0B{ͨ ?樹ҞW.ƚw`e'Fvj3"cGmo9jg=3s{|j eQ$9 -K#M pf7pUǑc8NCd&.54Vh >)wĈQD&X60|^(6$ cq+R ?"I=ݧCmRT " .N8E@XǐU-\.9W?TKyx-ƣф4&:}`Si' poy{s)fFtbO`3 VX nhM6d$7J2HV[>߬.r電]6Rء; rGU* " u@Vר@#)?qr5F$ zG FQEnݞ+57M{9\Pb\ I Jp*{2[Eݺ}sr}G?""}L޽oJdYM sWL sˋgwAmzt՗ZPec`cnT &i .Jxg>]nj{72RNU@Q6Ze$$TofbZy"F(Vܩ8V(X7.{NF$s}cuF &hkDDU `sXk->J%b<) X݅1[ᙯi&dm:@ ][k-evE7o3 ]U󦥊 J|KL!{iqëiO^sA۷t `/ՒE6BykW)jXݻ'g;(wcD b5F(;tU#o&Rz)5Hl]t-);H=ĈoJsH5*z54ټgZjB`-y%~VUB\=+x @]k䮈2UtM lǐBkXv̅qǞ/P_ L%fPGȀAIͪ(7{ $*Ѕ҉N&*7N}0(ML6\76  (<ۛK>A ~L>@hLD>R/s=c #vkf p+Zrڞnd U]q㕼u+QF@_d+^A)Ea5 ߤI 0?Wưqmivs0_ua^Irȳgy3f["F3P P8  +zC,&INMr(:qTrݺuP [ s܈`)?&7#$1Y8W&w\H b< 4[uHu{"$\7^%(0 ̠`w;(5DO++wH[yr ߋ]:S!eB$Z@ܵ%#|!EͦG@7qӲp>Ug5Mi`Lgۋ5_5 e$Gr{*Ӵإ/8L,Qvve@UD1c  cKn5_?/A,7vy]e"cbD* cfGpS/[2^ΡgYhx*!-6J\i}{(^9 /^xO>yi䧎rТu9/=;LA1nv@[:(Xlsl4-n{6F|x\Zx2d/ +Jv`O6됚[sӝ O%QKkL65Wۦ >1auFG?W6lH.]"`6+d z$)%]ra#2/a"q75}w`] υǶQbp1'0佇g䥳 gÅas33@xxDIX/ O^8Lȶ.aZ ܛ\FI#4\ ¯L}{CIְd#('/ř"^ޫ`RS2Y|Lhߒ.I' +hvi}S㣀6A:3_T7cl48,L/VK8CM XmڤQjaX2rZ+,gVY[R!|֦ԫ+' 04(S~ت9{,|LA8û2h\@`6\uї[.Grp Up53B'2Iy}a>p|/9fUd]xd8JmZU'Aq("n)"^ǵ|F~XDm>vVƿ$O@$^M%M6vD~֮qLXkON8`,79XEH&`)cK7KY9Yd@aqD^|OЎG=!K(y=0o$8CFqԚyRJNj_ ;Ù|og^yz8-cg[J+F5Yk-3+o7U9X n!~wk*OWdf[A}}g W|B2 WcV7[ؘ+k>$pɩ׊P]+bstɱhG+-N;wyEiz4PiEmlchR)a & Q#"Gq@^o'$&]ǔaҁ;` jxș rߘ5*o_ИC39`K;֚UCh=ݒI!$t)4}!U6 A/6bP950C {p.,a=Tʡ-s0-ZwFgOVn`O@:4t-Y%펯R R%iY"`G)Fly Ib۪k8al4(*{TJDSUg_`vtCoC ȎT]Kss{^dri$ a1NʮXߙ%[e@uWTLЭs%C蕱AAHp&/?ay !p[֬+m0i3?ȭkm-.T\@7,@ l=LNlf{ 'nl2;:aYt% g':A À)yÚ1L@$M7[ׄzu9*ŬLbk+f'l# _c h&I|o/033yv800(Hmi .D KcZ &W :@.k#g&GYLvSm"V+E.g8G̊@&0N$0jG;x)D~vpaN9VM%4D^ӎi5U֗jz`clih %p n621꫏K['n/^~_{5xw:/ݭ[nݺu֭[nݺ!;Z`u֭[nݺu֭[nnb֭[nݺu֭[nݺu Zu֭[nݺu֭[nݺu{nݺu֭[nݺu֭[nݞu@[nݺu֭[nݺu֭['h֭[nݺu֭[nݺu Zu֭[nݺu֭[nݺu{voAog{7M֭[nݺu֭[nݺ}[Y֭[nݺu֭[nݺu Zu֭[nݺu֭[nݺu{nݺu֭[nݺu֭[nݞu@[nݺu֭[nݺu֭['h֭[nݺu֭[nݺu Zu֭[nݺu֭[nݺu{nݺu֭[nݺu֭[nݞu@[nݺu֭[nݺu֭['h֭[nݺu֭[nݺu Zu֭[nݺu֭[nݺu{<7^IENDB`incus-7.0.0/doc/images/ui_security_warning.png000066400000000000000000001652621517523235500214740ustar00rootroot00000000000000PNG  IHDRCnO.sBIT|dtEXtSoftwaregnome-screenshot>.tEXtCreation Timefr. 12. mai 2023 kl. 12.06 +0200h IDATx}\TuT31^x`*zsaXnXny]]nR4h/*LqRXM1SIE0egҁ ʝxgΜ=97nvݎB!Mp!BOBB!pIBB!pIBB!pIBB!pIBB!pIBB!pIBB!pIBB!pIBB!pIBB!pIBB!pKF~!BexttZ]E 1?w.B!ͯ˵4^4mD)锢QJ$Bԡ_\fO)oFB!uԡTʚ=}VaB!64eǴǴ2AB!3474imX !B mrl,IZDB!]M+^G|.~WBB!.4^!Knx>KV(B!DwSF\>m>E9jB!nX]6?{y9m !BtU7Mhկ|~>_!Bm"X]h !Bnxxq6B!ͮӇƊ%ƴrJbcZWB!DWC㥏QqJ/.}͗#Bq3fw]^Y٬vpIc0h<\~٢r !BtuѴ6ف)r,z압7Lk!Bӆ+i(=m̫`./fXz[j !Bt24گٸaMͯ[7F7W4+8^ܰ5[-_!f)CcI&<ݺVTr1QU/Bq3t_\myэn݀ҊJJ] }9>VB!7N/nXMU٭[9_ZQIYEl*me\ܰU!BqT,(?|Zݨim\DR(<B!thp7UaZ\ZѴ6 :PB!ӄFKR"6[n՗ۍnTW:l,ImR.!BEWr2?0sl6K.[??'~ʉ$ѥvr\ ]Frg,}jħ9\ў%~Bx1Vj !MCc i{ fA9b|'zQDgbW}(s)e<6?z?mYrA^z>;;47t3w`óe$(|:s'dsXsBqshYj Zϗ\·3'X|nvA-w3IrG F>BQ_k:/. D |6O`!VFm^alb!B-*'~͏!7 Q;<[&p%(K.ۢɎd3a~ yzo|r̂19G/&s G7p/j-&[v N䜣c_\Y[^?m{cHoጛ O»ִbYH.EV}2~3q^ m:mG8E9Z Yk‰b$= Ι0]Z}|=q&O-trժWѼXlhFg X`[$|Jڱ,2sr(`bVg8p$ #'X| aykƙ Z |U'r)4a@{`Aį3Bm|ȶ/H?sXmFE1aO|lB;m-i6r^lqKH7&ioʦ=Y>gTU!:{!o^LL|tyhSɗ8q s0wRNwkBܨ6 hY_vL;[Yαhr*8w,Sj\Fr{!m&rҷ&};۾\I[3k#r_z=t+,rs1;Ծ)fxNU/ZFϮϪv.ݟ'YvoekėѼ%k>jMlj), vL獄̪iI:o=5Rkі[(8i$Gt׃NZ_}]kzl;QE׃DNFEg媥)*;~,K'tzcxm-zNޫcbwmj +4c?ukXT'cxL|W ?mlyFiXIpu:X:.{]l~ZY-ddCv@+N}PNlY<ڬZ8)'Ҿ$MDKpBt-6Ud{2>?3wzWPoGo:s2R,|_o/Ͱ>p&{RcZ/We=֞Y`r(=bN?(ݏMZVs=n{75%wy5~lG>lGE^}TMEf{YY\d? ~վvesTH{Lܥ-7ra*ٷ.T7 YqY_Opl{Rn_N4Q5<ڷ1>)$OWWl({\mLvhq??@]d7_U7s {RT;?z+wȟ?>pw5у9bϿ(kQ({]hso/r,jZeGOD٧85,&9OlEǎO_0׬~~3֚|~wۗ7/y6eO1W;yi_&f4i+ʚ7wbAKo^nUl,Z^ ^L_V;ĽѺzwdCm9a(ef鯖Uo{$B8Zl|iX'׃VwLV|>%552D7~F=Zވ_މ^//8/e1z'y-i֚8#4?/[:.i*7,K nT֣KΑc:{:~ێLKUkDS7}B_XwQʹCxrK/_ 擕Lt߅oNgoD<@E. TZFL3@j^ensߴ|$Ȝz;^ W_=Y^"'[4 !Mß6L=q9?E֛ɉ 8-Y> g/-8eZ&?5lZ'<)ug˓2I=b"L4Q IsԪ#_|IA_.OCP7[Eϴ1 s Fah䪭OGrٖh=:ʹc a/nfS.9|:[a.9MNkVBv'4)K{Z-`aNp`Gl9ilyŭz 2,@Q6Kc'UmAouUoX&nT:x,y%!:/s9 8G̶^ʼE^> w~ğ5ofSD;KDݖ[+h|`9V˩Nꈨժ>e[eZBNil.w?;ns;'ӮD_`/`IgeB6b$1 ytuȕG_S8ƥKؐ T7Y[;Hg֢x<ehߑ&p۷9Qh Y"1`-ak]{mwr*lr3Hx'9uZ![P@@p3GZ4+窗mrҷ;X>ewZFAڧlһt}8sĿ C HrWy JlǷj7?Oi1=cr u;g4Ozp4lz##NfK16Gg6w|sٱ_:,!XY4omw?g$~{B ! `]NѯW;YYoˉw<#9sM21N zk'ƽ'٘,,ᡘz3;o v}-="Bߨ7{ ZlI=E246\(L/_j0Oϥopq_}6Gcmtu3D<5?amv{IsnFmy;YNý?9,leIw;rm2.b&f%4 v l :=ZÆsfy mNd;Y^Y1gպ<6ϥ湴3i %7Ns'ok(/x:Fs>M2 Q<=-^ ܵh{LX4cxk'/x`uawg3y= n32OصesWnr>NƲI``$mI:cU ; A fŦ ˚ mbg9kDafaf k'/ڝ`j}_1_ Z>L~t"ͼu/hz0~F4ml>DV/k"ӝ/5}̚n+6M} 9mZά{kїfػf2}>N7θ;'1\{ IDAT{hL!yt.f˚By+ ƽ?xBN(B!: B!% B!% B!%#B!\F!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BᒄF!BҿMh4DzjwtARQ7'qJ+'>V)hg8V~ "Χ'|ڸ W 9yL/ol6'(8ӶTyKx):fgs$ 3+(u= ۚ_\WE*ѢDa _J޶;jyyŧ38[ܦea׳}D-_ l-9voC[.[~=Y~q=m=4dLy fORs,-$Ճ5=YRoAހiITe ҷAl64*$AcȻ7  7ʋg^"ewwt) ;?2o>TÛcHǠW9pB104*'އ  ( rc1! @`y rF#f8exzFIX>"TV̝>]ݽ8ڥ0H!ɫא` |h.%Oj Y7a:sRX{Y: ~,|ù٘I"tXϦqK6ԄF[0n>#vɀz0NϢ>h&Jz3Y^IRX_'R>'ϖ3f|=z[]oTُ6 X8%3(4Wrlccf^Pxkyݲs@ǐp!g"M|A󶦖mtLx4>;piz?h7I$2?7R#зGA3!chl ~.Z1*:Qnr$p=[Y~}jBG0LNf&y)<3;VzAAu!CIb|r:=5$iG$ #C5@cnǠrP"qI5«f:;vg zzHϰgk"޲ʲhMo? WY>9 0Iv@f|p6y`:B‡oc,NZT?uLt߫'I/)fX=2qh<)EP7(wtZ7O ٿ=>C4a)z gX W9_HTG3f,$ hJeBT.y3aּҰCc:?80!5e8N8>)~]AC9MF01Hnz6XdW5GNnh1eDR_SӬesrFΚ~+:?U.έAW81OfcK< c~55O%q OأՖfG@kR#`IҎ8V`vՆ_uﺨ 7!ֈѓN) ϋ(Aߘ@߃ρ79B`3SԦ!ns!Þ|VϬr XIHyroI?Oɠ7#Rs;e20jH~{M }#Ce{ǫp7Ĝ{N3UdބzVB6$2aU~΂i-n}T&`deraUHqAO@5H ꩥl:xkxBI_o'V^Ψq$?ux5H}>y =># PTkm|?n BjYuРVmc؈T\dٶ$ҧT"Jl)Ec02ߐv_ 7PKI?TL4OX4 @R69=n9;b g|b0|[ɱ aT_j`MzD{GԜ#;6T]& N "xzw`9u@z-90cW]ā"30,[Fb},{x1̱o 7lECg?/6^ ݖA^bC/ʺ/8?vFx-:zMأ@e& T^X (j`:Nj٘*9yD, =HҗN~@قCrJɽ~nބ$;T lcaPðpFѨo޸L%>VJ? 0=T.s`5[0WP2%ovw5sD qkW蓛eLu*m(P?d ]Q$ %odVUQ5狑`YTIG̘{w]TFcryh0ЉL賏{ 37Z@g #h[*@e>9 C =Hx2[se! +%LI^|x&LD`}nӓ._<zY vjSn&w_F4vs޷ E t 'a ,_kJeȓ "a˷2{t>]J߱YomNס2s 1!q:zQOٻ.djֹU@{8o*+BkhfhJ) uy'Jj~͝ !l!S绛KۄFARbyz!nq/RǾͦLӡ⃏⚫tnBp|wo"k0A-`tW;!ϽȬiU 䕹7"F|kYұq*jp#]Z%#yaj༷` ƌcؖxpϝ+d].ZGkz_szzӍgŪ޻]Y<Xt%@F?aZ4=n9A4CIꑃKf!`7Y}zxne@C JFu,=߭-(vHU|vVMQghiO4@嵚XLs#L[n&mr5I"-cZ=79Zh5hrޏTWs6CT7~w9@y1Ne7>֛/1oukHAD`?U?p?Y7á68煮"ӧodנ֖ (@/u뫯PzNڊ,L<^w O5(%][olιz<B˘7JjL91]o3X5 Wchw][4r`40ȻncwA'bM_HlQ<qP0?4pb'BǓ6)Acd50fLtwOyzUo>ea ԁ]-%S孃+D&S,$5SOf| t68j')Jי^E<:hALl@SYZ\1|&Oc?cظ6kbTΐ%S)DÉH/׳ZSi0e$w\''ې=1S "LbM(j3=Jd d`w7O]9`˲1jJ1]Q@;!C j֠Y40acV@ϻk:0S)ucO%t(5 YȒ{Ψ-_el۫&pNό"*޵K9Quz3u{^5 /rS&jۧs-Lx^hz?7c:߮[nMB3ʛOL8"u uy3aKq(3W}]cLƳH#8h=4-[8#Bbtħ'̎dlj|& 6zгo=bQsgmO#zm"}?ߘzG7燑"> `!><I#Rj7|{g1aw㗚K 8s9ť(h0ï{3hx>lN@ZWNKL=h![H^5zB" Q/X0ll={$g̉=tA'q;. +=0x %lhNpe3GAz6}͔{olYCpK Jxzcto<4V^p^YcKm^ O/,6SA6JlS; CT&7 _mcR z0~w BHèXZtXXR6uEAӳߐ[sZ Lq, _+ hz`0x3vǒ;5<$mq`u3j*KfMů]Q/g$n"jk9ރ~۳7f/.D9' 0Ϟ21`8!JF 5yK"ҿ+dqXBg-;x>g9ɂC0~^w1wܘzq ֒:?4?*sUOsSsYP*o'!=V w1G8*-80rTZkmn^M[MI r&|>}Ge߮.f}Ndtpr uuS5~c7nBV3x]B?u!?/OMj) yo8=z%H`WxD[kߓ43.0ce+γjbXZ^^x4(gس ց3ӻؔAcoজ| IDAT|a%h zƄJoΟHt B=Gl<)nX^^>OdL QYLޑ4_U'>SY2sJ뽥s=Q{xa¯ؐG-`-PqK,X+z`˘_@9Zqފ$z|FMcɜ=[wo 7:5nGi!BR'T!BtVB!KB!KB!KB!KB!KB!KB!K[Ke!ƑzJ蒈NG!o_"q{ : 5s;V zڅ߇֢gwtIDc$RO,LaKkQX;AJX۱#NKlBhoZޞ gj_G8бj ᣮef~h=mViфB B^w%"F܀1wg;,B!ڃG̵ ՛SxsFߨ)͚fU Ž}'ln=շgR]{?pdPD%)<<$Ľܔ2]aVnkmRK-)B,jx$"OʌxcP<Ĺ{}뚹d)(:1YbRq ږFH-;m%3`;;|3ID@e?`mj_X]9qo(qI5yB3^@s# 㧑AV-Ia|Jր AI6YJޝ3y4]w]ab>Y"*h['LȧWXǎT wfig$ch"(Ll /5y(Izlz)+wq\$CǎQ7`K Ks=tt2Em}_b&Xy>_jgNx̐Ih] 2Xd=Y'0袭5;m}eo&ǻmxele~a`h_;ɱ;Y!A6djn姓Eu'1VX@JQNT߂'LwѰ];ٺ ̮b.f=j s/L'w_sЌxl`ϑq#{̑,Z#/?|4v,9wXĦ/1?&}HAnԋB !]toBcIs/gY%L:@#-9r%s?7kyKP3x%鋙L `. %WeÑ (@Jq(y98lf~gQ$9Q{=)Φ7IՁzy( HѡhÏIʐAr:{=+\]r&IP)9bhaJNk61+frӿfb8v(z5V2/a=`޷݇aS$?j~okg: cY|L3_Xpҗ>ǒdb>_p[h/d@Ƽt+X,<`lJz>Y(YJ\jl0V +Y0u$xryϾDB`x_ KH`moM\7C!Il|r/rIC1Vp'Ϩ2sV<1CVRmUBnv6'}?*-fԖ;ߏtha/RFzBP2.f: Cvz_~>O!'Lqhp~׷,ǻ?SqD&wF)/õ5p1f|vO2^7o3RzBݛvQy"fmfu@(fKщ ٵ't,2FdBD`e%.%P>c"UY9FpG,hpP>DAAK th=^W)$?Q]F^zmtPK6tc@CI/zXHYMjԂcx^@:#a:@)YSP"{-Jb8)含RʶL:ѶŠqʢ}3~P#}bQg!DbG\sHOlJcҝHRr \ hhՑ90aQ^OY&>cBgѮs>h9UlSw۟NԽǧ<= F)L74mi[)l=7nl[UKdͅx?VwB+bs&߮;Oȿ3md]k8w&?8!Т5ݮ/?+{8Nl'A8<+fѽ8{)2'~~={ lCٮLO1_=x}DuEFZG+R˻},oOǿLӓr; _$Q6%$0rjR^ǮE_R'j%D0_gڑXzO =:n;I^ }Gߔ3 Y9&ePZ'8zUCXuue䚹*,8Cc[ڵm]G=H[sKNhnj}s2\zLY9xgTGK5Rig¬,n/`FVUQ^CSؖȆS9.e-uE=,ۜ'WO쎮ͧ1ĚA%mOQH-4 mpqIrOx%߶ƶUCW+;gq\gag5^ %7 }-;s*6WA7-`2Xǹtl]esܝ5ޱ |~fJPXc֙ʥ4o:@#` I(}~>B)|-\;Slx7s3ק2sC]~5xD6}E$h9!=t>%;mq4H9wfƳ+} cښPF1< 8*8;jrPY+nTH5EhzD^A1닏Yylkuipt-}R &#SS/F(TЃoN;+jmB_yk<TAMehrՄ6yx𵓂2s#K\·rnyčmՐ;ߏUJJ`;u}'mh9~Ҍ?5~[(Pr WdkM w}8!v)5^41jC s޵v'!h{7h=C(Q&+ٺr= I#~zmV- %EfTt˞eڣ(m+*E9M|` ۮ ւ$Aihggc#G򈑕+֣AbÌ6M/EigՈ٤Ja)7f(zw9}r3sl &cb}恱|,e=HR( *~{pmǯa"}ɢїnl\Nzy#d2u&K߇h%_ZZV"vۤdۯ>a FVq 5nI@JDtuN@€x6|sUW6t1ֳ9i+2vy O;/40ZlL{^D?6?b ιNx׫vGweM8|icyr@kj6[@]ia GhM - zb%`j/Um!)Τ,Ocg;(Q-ČK\% HzÛ>|)cW$MN-w6ȀIPdêƬ՛H`h&)lHe#n&:i 5:2t@N/9[c_zQmh~;^UJTE_0_tg1{u@Gy6PC*Mׂsәb16.43<;i4eֲ`w X0zϵ~Lme>aE'G_ږ{g} _&*h[xz{BЬꮗ{̽@)y>*l9ۘP-dP#nÉLG}ba׼[ԏn[!Ŀ~#]CJ=Jg{4)0&++I-7oރB #E@WH;t' ?Q0maaC!!MhB!VB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!KBB!^z+K$إEk6 ,C;O >k2 ^ on{Z>~q#K2s 5_;&!o-l\tψDi ܝ7@vXǚ)lؗOAčG;ikS2ES?d%AAHX@Fk<~=wj84s }<Ή&gEsB1X5?Y7w7Zq )_>O > Lg[ԻL匿,[Ɇ| U'L:TO?_͉r@Ѡ7%"qF,Ϛʆnrzǜ1TOonj(uʑLLҬqt7Br,kgZNw6^©PG$9Fij}BX3㏤4O=׷_=}z?ֳCͿ!{,?ML!p$&yW:3IµieLO"J\4 L?$QN[̨!#yXE؞/覇k zKgؙYsfLWC*aEFMH^x%9Xn: Gu"itqmr40)]24GҞQ6vNrj,e߳uzɧ fB!{A!pXq~¢Aםg{VIGWgh 5-n:%d(_E\o$XB`L#뿱g,W͡r)QG /Ɛu,@FbYVY3C#jQs'2`nA[ B!r9Ja$EBIݠu?fsI̜œ; ke;]>#1I!R=+p&/seoKJu sd?KnVۏ<ݪ^ӮzDbn8FL۶wd!T ]ٵ|sx6k$;ڼf?.c3'݉>#z{yDd;5}l?ԩ~z ;b %{vp-?}?>d# kE\V gal3 ߭/B<$4t1Ykjr"}^K7>q(n͙$]ҺCC$eUqcN+a 1kgs~/WՔ zoQCusֺ0lL&-HGA=Y~ 6ϸ޾ӭPH}zT#-b,R5gލ .2e+I«Nx``8o`%!} LK]ljNB%_F&le/] j\~*e<bq-GwF1wg6y yJxN7y@`H'L*iBЬAWB!B,3aXS!B_5;23o kFH&LLߟ#|@5*=嫲(z H{r -W+}źݧ0[BaùyoQ .g*n=FSy,w;d.<̞Chs}?]Yu޴7xyL|2KZ ZYzUm5 uf҃"?OTPt?[(I 7m+7\ƢQݥ33[f9czh?CϪ_\a[Uwk\MIs҃5x՘kHfx~4goɥK((8e7d6Y$Oj*ݼ31Txm)?FzJKhcbDk5SD3΁wdz:=vgrx0_Bܪ=Zoo;]PФZ,+>~W-l, BԚtw|6KA׊<78]C,RTUкǨ{WDUe;9}յѽfT6habkV׳ ֧r2 RO9q2}]m,x+KpkED<70:[Y;ȿ`Au4 [A)2YAc;ahӉAO'>z]'62m=wҫ*֓иDV�g.Ux3c~3~w5>v :LQcbxj>%_ndO~)(: aK[_[ϲoIXxbͨ#@ peO _~ԯH}3WT: ~ JEWS(%oX*Cxt(E<}MQjՠs"zw~ŔW@ _&b>ؼE' mc4j8^ mKow W1娉o19Ax7ZEsXOLtbol++P 7č| 0C޼0q?|_j?5Yoo aOEbɚ^*†3uBuJd8xol6A/_W)Sl:s z/k^?\QXf :g[fƀO% u%ʶe|UKU;4sO7>%RO[7WC3L 2̧YC`OFl`w x{91Q_2-*_YXSIUn̄aZxuPAnR\J/[חceUn]g5>gfY+f|jy1ء( { :O'xoŤv ޑpFp2}G;uyYԳsUtQL~>Eg?0rusjSO$@9ˏYց V`)l{V ӥY,|=cا1TU1~;e0 c˽1:J~(bd1(i8.].52WhU)1P[KfX8Luo }PCxĴZz*&\CV*Vߢ9pUrvVҢ+ /_w_XqSˌ\E8_*zfG9zeVhjqD8YY azMKCE+]Ĵ]v"?*xyG3!}yuH4tn@ +[7nAO/?[gI4\ޔ6@+MD* oKl:pe%X%-NU:#oZz$5x@еk7֬Y͂2abr/>1UJƚqma%<SEv-^{3ߓi aoE|4ka23߉e m`ΙΒ1w>JoCGs 6D?N7Pr=Cu#$hrs܍; Onaz>w}^-aᄺ!zdfl ,8e {1y*mp7$gd-$_WYJpmNhV("цhߵ @g3s=n$0ĎϨg^՟X=̨GmA}s>/qR DDX{]ķO/8y:M=lrfyb8m:cNCJξ#Ljvd|Ofy8c~4:",~:iYSݗN:<|+K#bQ=ya(b;.z&iEDǐIAkh_Ţm=| . AaM=@+5>FO!5 ! n(tv`2ͅSk|Qk fKq | QZ2s-|zҁ_'T=*\yXsJC7Ɓ .ؑҼ>:Rr#0 zl:9J\Ik*g&U>k3[0}SmM|ׯs5d5Ś8,(Ӄf_PƑ:۶Qy֑`;vTv#@S3}j\Xg46!!  v̦#V<1׬GzCcBv X!,Cj:GUPgUyyc'@RJ}SVy[-%J)hpd)M yr }kcRψ+WQ fnܳA9f:j)[!{ayh ѳs+M1x{BY)Jny}A ZeUht^^^c&+.mwh7zGk8AEg)#uƫ֜`U(BOviCꪟ6c~ >1iw L1DLm$YiӄV/c% u}[a=B~<Wx;M%%eU@pTXfĵyXp{;GnZfJ[+h%>z% ҁV(/jtqcuiQre54;zkj~NLns)vmV*ֵ\Z;)g lܔme$MjǙ*c34~}jnۮXy^5??ۇ-<>`[#3^x׿SjUQT+N8𥉣%: Z? |9(X]1 \?כպE4uދOƦMl^6\뙉ESh_4xj(9 "קƏ-+. G~F0w~V*9[:3iFi0 (iOi}7r"o'ƐM］z?8z1aR,U1nAW#D"QL :fEqf[Gwψ>(cDΏq/& 8}C5_+ &v\#bԪ:T[3u2]N N5Fk z;;TAhNM\\VJ~]tT[xnHKHe/c̽Ԍ`/Z?^wkVtLr&ljVNz:m`ۉ[nՌܣI:Ҍpc#74i5 J$#r>+ (&O=#uefHn?EqXd G|.VCqmA=FP ]xS{~'񽻰ӵ|iaPTJN_D:PL{6/ ][kT["=M|B> gܦ%Z JhEF Mқ;Y'Umkmx/7Dd,a?>?/l|6uKس! r)РuSJzwptpr~y}]ڎaQWZ2g mHEuE n4JtDwmS;h**{s21z?p'˗E[ܛVr36qqvBw,wbe3c䓯PDaP)u#EތjꙀ#Gu>ZBI65 ߶b%os8sy[y"@n.Qh-1"8zldүP!ә73Fr#ʢW1E;RRdBx 4WX SOLΐ/ƙTzk5nӌJ%5 ,Z|WN8UqҌ^ ~g8 Q?KvE.^AiU3Rp! <;|z%Y>o#NǛi;= ,~?3t>! ͨagYnsv֫OEP]cN۴a,`AEAՊ35xd_T' cŪj0}#mj?ƥ#` R!QM;6YfQGiucTBQ6*gx~Nu!u/:"597/v5۰RUAgṎQ,r#?݆VגV ?">! ~q|"~d ~ldƐ0t#j naVna\<s!әiI_\fx5sǻ@fm.e|af:cg%F\/  6Yd=^&+?WQ98?b!"eA& e~27Od̦.q8) -I$DG ?A\h"j k IDAT뼯u]~u](6O:OO7[Y,KӅ@o7=;~y/[X}]I{̝1=Tϟw9,y<561-.\`_nv!4ĝdL;;ȅ<POc:yvx 6,hT\ ,+rA =jR@7k2;\F5]OO}\?xż^AqQX*xw LXfցms,d#nZ}?/DIyE%C\lqEnFY"2*_%0P(;uCW|e !Q8Uɯ*v / m3ccշsOJ+~ŭtI ڷo&48q[2:6jtgP\SU 'yrBwD\7S#X*8!fMnY|:b=f5Qx3g,6CB!eGɀSĤD Bq88#a&硸z=8邃_pOL4Ç xPÆrOL4!so^ϱv5z&4~}$C E/:d0_8B ۋ{ eƦfaE)" !MBfo7±B!& B!BF!BIBB!$Q!ב Ctg|M,RO!67w~f@e]p]6=5o޼}ֱԡQ=Ww%?`:B!n^S'^P4k'yv2P0ߋ0*OTLgxH_ssػnh[OfPdk[+gWEwܷW(B<<]0\h%5Drq^Mv5 =vFdU4Smn\]/Ա7zB!-ւFujYvg`F m.Lx|@uU#o?Ug|Kדૃf6m w!n@xw3W7su3|C2kUJ]z=x鱾L-67߇<ه~.̞yT:@}/?Ez q yjqRZ75 :_q+鑹nXjb*R^ɽV&2y̌w |#٬޸F4RFxu(ֺk OUEFM#u}Na+fT>yZޱ2uttVJѦY$5Tŋ65P>91{ׁޗ?'mgۣ/B+n a)S9jhaϿ,fx$kj=oStŽˇDo6^_c5mLu`]w,9w ږwd=!+A}i8H{T3///Si-;N+r:R'xYqr'u}''Tah^۲Z0g]mXKe,09w9|uLgaxnd=Q*#:t!D scbF M&ܽ2&j**P?GUx{KK3}X0}F B!D7<uK09[ZeEYN[POdpzv*DZU@i;ʼ eg@qCm6Ӎu(cr]C*8xvQ]D~zPUVû#K]nOsB!ւt3Mz7g CR12g8mp+nP!8Y˗zOu\/n.M2kۗBW8kkXn~7_rI}zBFr/=ߟnmqoo_G{=v4܍/4 s;Cϩq͌m.71h#f'ӱfok&漎61=/Mp߻jɿCG|v2ŗcb c={z 6^6xIj[ /PLV+3#]NKYHڸ!YhS!vm95s,: =K/fӵ7Xy:^~?2AJVv- O31/ wh4:6YTg704?nk\3kN.Q)/~=? .h0ThtZF갱ϝ ?5.MíuL5lbXT}1|5SMkYY~>xL1GJYa ϮRà*X@8aq$ SȬY*7d:"tB!ݵDbFמN^.?|g"*_6 `#ki6ӑH2kSDgv|@=u I=?{8[O:/77&[)Tk~[O~4Z+nr|v]ns@89~/xkWUufB!zS^|{=3B!!Q!Bh(B!4IhB!nQJ'Dtnc)Q!E s*kF8=\]QU'LUU\]{B!avߌ$#;~=v'|'!"^Ǜ"<D~{ꮲnƣr_%i?7lE{JZuIiC,[8LO_VJ%<;w {߰רݱzk0^cn׼Cx|:mRyga(}FkZ;Nf6gGj\== 1 t+X~&7iu xQ UVmkysC5,YT*wEQu1RIHkV7o; )w<ǡ?[J^xfGu5½Ϧ3cdt-m$w1*- ϗШq>6`}a?vq*,6OqI4^*v26ݗ{&hq A[ ]vk=gVt)aL;¬)mT0ЈF ߰"רoձz]/>4IdFڀb29v qt0t)/ҏHaވk^;!n)Wu ʹbm*ܔIQ糔] +XK[̉7:}h6WfRQ>[j-%<1Z_G9aiD:ȬU$SvåvhV+}~WqWP-Uen(GM|;56@QP9^fn[8@v/ˠXlsPfuFE @!xl7n7jy1%Izd,mO,޵뜌ŢkZCf!{hn`OWQ[z0LzOS%aXKsx[^wMgv=9G >#:g *5M 3j@F%2u|C>Ϛ9(?弊2p43蒥 OF\-)+﹋驄(ڈBMˆFJ}agzڏ=TZF33ILvª K㉄oU?0a#yd;3+6`YLe=Bp>S?*lƪ/R6Z |BNŘSHm3TG޲gߒq]k>:DHd|2LJaSy ](37z ,~eO)5nxR3S,\G̉k f.1#ZJLr K=4))J'hESXT{=R&?νr*rH+GƑ25Qme~e9xoY4iwp4@1>c=|gC_aiow\RG޳X:6L{`O͂!TnX⏫8SmC̙ݬ^IYOǥ6>Pc$2gq,/@-51jOQYX[͠SPP;^ڎq/L٧ #epkcŔ٢ :ԇ I$a/3Yh {VpH<)ښEdMu,\O>Dּ.EC`YW,b4^L&@edt荄&Ng;tjyP6?r3qi,x<}U'w/o?ghv緭K;b.K'h2sރI@=Y9v +IXM=IM}x?c>)4";W_@gЅ>ȝKQsM!ו|l} <b[kRSڏd&>}OIc0Hv{s D%359w֎ RIgcojj牳ui1FtCtG4Pm#wޥi*E~Ef ' xw(Pe5NF6#~**~W6C?Ƭz$6jo5 ]̞/||:cԳ }CbBlA#f"֩lze?6T?(Uogc[aLX{>Aլb4V$Ro$wVO&U6<51XU;,Ŕ $){X9CF۬挱e|x2Bܨٗ鏯Q/:x2j Q7?ގu "ɱ+*5;زu͒)a(X(),̐Ȍ豢m kpn&kQ&'HLr8mbPMB[A+'ٹib Nv=8Rb}7V}s&+T΀0'2PZ[L_2Yg_^}>bs ~(bf28y?w*c6+Sft| 4$M`8",)U:_F?L^2{0a<̿M;:Zݥd[‚'dd&be^|yygp;SF;J`"&c ϋQMaT]'`Y{D'Ll`bLupC9<4y1P_k]}z$;`dS'o÷[..︨.D*oKt)%fO9i)`UG)Ԝ5fҵߙg~īOD'9%~bPU T;WgO?+ε[﯑@f 3AH:c?Wظ,z9kk{o"ftzr?2Xs?@-ƭ IDAT=h'1xvaڼu3Y45bխ~>d-jNt}8[V{1FԫQAƒE1-CCQIr 5+~ Vά]&9>Lܑyu/ |R#0|{Ch`. O"'X@ZޡU]=f_DԱaIk8G0mV*0"}_̂{ѺOo>&8:<} ;`rdFi, eϙ߰mT&;f)Шg wG5"P%0.Ϊ}5=LR1|˂dn-&aj1#l"5HS !m'6}@i$Qx '*o#4Q# P]9Wrd @@FŴd! ٻ+lqDHp@E1=휫14"`0F S j:y΂ /yeP[ \)UlRMIE2=YJ Tw6&1VΫَn9ˡ tF'x5D" %dw+:{ @p-f˝\5DZA`Oĺs5ؙ,} pn;Iwo5b\gZl>k`Ƃ)8a3}I @Fs1shJsxDTi:@ی >̙X)a-ct'}ᜫZu5:EǸ7<Cu1Rb \WG^k;h'#"j>911^?>o?v k'ץ^ԫso0;V?_PTkڅF%CXGc`, 1N /|o,y+M,\P̘H_);b=gy :Q&_>W2}w)p*5 k8]GwkUPDʰ-^X6E/NBU^UNE&A{+g;I𶽎`C%V:W?" |ʾ: N{P+/0G99b P$[E\~"jNrw>+O1'vc1{=z A7 ?{weE3#Klt;lR̊D-e&*r xFjX:}ts4Vr=\+/"IZT{!Ř^- (c53>u_;Ǒ|Wk*`czuϭlVJUÜ\ yPrO o%UʿX;|R9YQgHkzjioOu6K $JOaEjʿI >lfrʾN {6CBoKnSb}7NhD66gBsyBrr}w %1onL%,Jcov?^at& LㅤvuQˆ1ge=)]~d28n8d_AW4\t*4_*XFOkK2& =Ũw4 H8>BH(g7\b$cRL+֐w,OjsZ}von3fƢ0wm#'s̊mbkP[3 28EGU̧,\PrH 3>`mH{>ό1B2b $J&uu5п/`UZlVj^ڏΗyxX'X0=]}^1x+˘lwv {Sl' y=ɬӯX͕9)<Qdvou:jWqǜ5`sh/g0zO͒9xs}5a ~;{]^:_@M||m?Ww߼00&>|R(?{yW&A~qۤ '/>] ؟W%>pX?>Ay4PTXz&9|Bhp/*mI $pSPhj|/ߏJDKaX |TPtbUq+φ^Jٶ a; /ټ~!dnH7M`2|=|O /vp~vuڏZȢIT~=|>Wkk;.nUBw}2LJC3 O}\Ax}*ԵST6X"C 2wg}q8c.(I͑bj\lw :?O'^p4vVޏ  +8'}v[W~_a>zH^k+U&ƅ᭨XN;v#aHr ;~x_sf2:*݈|0g%4xzDufG5" G㭖)DEБw᝻w}QZRoeCl8m&8}g'2I$(dqHa7X $ؠg `,ofz@;w&%2o?H^c]?pO%/={8{>g **m!wv@gٽoPL>_?ي:s32;lٓi WANpP>%1j,bƄt9YogY_}`-4(؄t$=A޻T~gT x9>P][7+1iJC۩999U3FPƒ%IJuŨXi r@DQt7^`VMp_5)ک*%K6N7I*Ɠ8̈aq^?_)$oK>WTMW]1Jy1f}l}=7UHN&ss){s6h.O5"c6VDU<BcBkU iXIJAAh}cc;> Ysoô1Ǣ*bk L24vZ+:^`~#<˷6ЇsxnJkX׀S3cN+.{SY0S{X0G[f?x\ }WgpHg֏'*Wcٱ8,N!q1ݚש a,YQ=Kc(4Pz2r );݄/1X̖&?b CQ@ɉ:T/{ *jUT ;I½}d:Ͼg֊r͋ƼT[o@8Oeǡ>-ls4R9Ϛ9(?弊2p43~=Ѐ@BۅJedt荄&Ng}>`*(t*?a#[XKݐIna)5V7ƒ<%{}.eM*^z^ϑɤMł[ y:#[R}ISgVsX~>%U}a$8*c n-Yy*¬Gh?gi-?foy(^x5瞽Vۧ-9Rg$SLaTۧ7~"Qt1H%tgJBp߶[o *jAƘI:.ٕHIK€XۯjE-Ɣf ڳ&}JP` !>puٖ;R ܐI{~&YۥOhZٻiTփ!(gcW 1{~[veGTXU aLB,v %TuO%/FdUcy5uO)V0LƪuS@ SXU\wc?W`[wL7!qH$X׍qQ-ejyGORSϘO.ۚ=g?G3iD<9X^@"㒙Agk eOglzV!Xk-DoۇdB}>/j#-ҘČCo3S3̕KKfxUl]&&kSgd ߋ):?GԱ)XFƵm dфcɨ/2Dn <]N&bLg')eADJk;y}cϦ5䖚QimV(Kf/"04[8Yo]}`:Aڌ4BXN}c~`9gP28Ǧg ^2Qjy]L@ɶy^PmQ?zY1Hp4 IOIq0ӆ7qϼqFr,^}H~"`wuzGvr`V=U6<51XU:#t_kmvcy1%@RO!TR &cHy"ZJLVGv}e&wk=G GjV1Xe)/l ON!J͡22f0g̥PʼG}~n_[ Xt{&dPskcncrF=Ɂ}(ӘҨP}y 5A۝ф+k8FUqXQaLFKY bz0F:vffp2Kf$^>H5Q2! B3aۨM8 b#a rlGBM‡޵#`إlr7> VQ1 SH16n"HwFU]=fOhБz !*,+GA!Fxm$$8cN#! P*SvX*G>VJLBc mw/G8 ,WH-pT$dz}@G/=廊f '6@ nomCw;ʾc'xc8Q܀I 1>{0js~ďXTם"7ʨ ;@!AAP]4 邩liWlVRcv]kךn][Ƙ5b*6Soa#l5TD*A$z}S]E|֞>bA/@cñxAXP'[QNab SA6ȬGuu *>&"Ƶ-"2NEKSKP`}Pa5eY8Ue8G-'mM1Ho&Jn\(o]dٮM3y? IǾbfBB) Q2h{1(0Fy6 IDAT$|99ueu9L`|l!Q<1Ƈ?Ğ /=6Ш9[ Ĵ 6ZD&魖*k4Pk>DDTty:3awaClsy bH5 gB 54KN~- 7ϕzl icNODSHFIu9jKtvRs-XW>߾CjQAhWi */\D]w,2&[$+Q %ԧBYg-ēdp!&Sʊ$$>ARbl7urO\Qڕ4Āf}n 5*S}AYZwX)) (@e0Z4йmmV\-wa oOpj4_"㽶]ѻӵE?H nzM.詮bLyWuU6? ڕ&TpRF͹FJ~_PqL8Ɲ8SÝ f˱._r4?[tLO'ܾqO3ǛX).01>L֍G8v!ӥT 'mTonico@/g)^J_ ܸՋD\#YuÄo.tS|z&^\EtA9,D{}mx, n^X܌P ʱ6`}v:Q:F XJ!%|b)%EEoYT.I#l`m_76k&qd?{70'ι7(@Kuhۜǔkꩾܥ,2LqA:ߋnܢ5q[]q:vφִ}%E]Wܮ䮮)nѭhܮpqom_toO*r(2Q˺p_ΖR LʏR?| ]BZJJJ9z x"\w` *(o^M(?(b# }9^&B~jY͔5ˠVř }#A]Cw((7\Sv)` D`^^F9+X9L9q{fϴ~&@cPzs9dBCРRL35h%NoΈSx*:VAOt(Rf-dd5E}Ԛ۾3bO1q@kl]}I5RId&рv?~ܞ5Q^V f-`1wq0u6jOv#~ѓkdF %Sj?]}@=ӼRKC!"9H3AftLsGr wP 4\l(P8VD=XO0_˶ly2ml}eSaF@hvw:vOݸk{JX[_[.n}TW]u+&Ckr@$=ɓ +uSc l{=F`3Pc?? 0 _/loW1,a> 4 >9D^{6%`ӡ};{Fc0!8c%.b2Xg3>Ec>LT1dǴID8K6GبtͺI=l<:^?LDH=پ 64Ň1LX\(g^`XsYMP,"# [8غEP 2vt^:[Kh(~2}bsc->GMPf,bXw[9$<^{[&@Aδg3iDoSxeD(q3^ c2p!lXmm_ 1ڑT^Dnu%h^ zW8iS߾P3ߊ ,U佷7tꚃ3 )@t3ӟ#۾ǟuPEfj(MvZn}fe<{ÄYL-k^Eð^ԎQW Z MJtƻ{oI,X{(> 1>u. gtk.6߽F?MOaQ]7Oj U Lx#=S2hly׋t&, S5)"#c ױ=l<CMD'#㙨{^|:WWPQPt>GwO?}FXא-%Fv\ X}R;U+) /D=GMY0KCMgG~޹Gڭ})Do=^Ůzf/IZMamDҨy/xQnᦝo,ztT,v֗IECEgG*J\dpA9m_#0>yAFXW@1v3HIZvׁ1$3uLP]jB>`$q^DFL/[pّZN,WoLDLs<]J=B!@B![4 !B$hB!nI(B!ܒQ!B%AB!pKF!BB!­/|h?5S3"B9@FB<8ZchcJךa*Tms-UXW,f&ccrB5:??^n\<@=_3KrYYR&mvzg3-8 g1&J=O P> oKl2ߜe =z-%Rxz=? ܔS9XOPq3>G8u:i F;{Z2zݫVR4_ΉhOZg93t7%gY>ȩ &,,x&hׅV,/@5,Ӻk96v kgTN^q)DNǛZj,`òp YtO W`ZPwE2q# ͧ:.ơӍKd2f$6PmnP3O3k]vsUVq0cYq6} uo#3$"ֿuFW} 2`I&sd,L!FJ1)ӉQܳXd}s If'tMz<֬?S=-ݭ&.2uzf/ĺًZF)\V5 ( ڙW^J夲j}23С0)J d+]KevW[r )?ÄًH .}Ƶ<k:=_F"bx7{Eւ D,[ۉXjo>&"ƅcWb;y-!NSq>Z\e,~4D0N9*gh=V4v2U_Q/Tx)-QD|dBM4RRVP`{Sz-5Thû ٸEed}4liĺym?ߢ`X{oGCM t7PY3}:.!~`Gw㹷2i]\> )QOs ݳ|q=ct L%fp|l[_"" J^=뾎5jBp<1mOpaC(?]UFJݞv}䪏psÉ|T ;]_gF?҄zoS?2LqA:\wr:CIR %/r>wy, 2x|Ȧv-=E7[yзcX7c=6<,ytx (8RA'+L>r7}Y}~ꍛ|m7c{=P:!_8LfQLJc 2hwQPor݁lw ]IqR5 Z7cPzs 4]bB3W `6!?4Y9(26S~( :oq1$/!`?SKTHL G;yCL!j}ZnL}=}+[LPWAyBTPB}޻o> uN` JvGIۂHHŶeBz.rClr2vmº< 'HG3wuIU( # s]7Q 9LAA{'29lݐǍc1jU,LV=dq>tH}Ěw7+ p&N&#n݀~z2C)\%㫞D$ﯱ,XzR!]?NQ|ĤxF*a3`2k a__b%V6 a_fϡ.Bّحyc Otρrp_B>Ot&ăyTkb5z4B!wagJ;5XP>Y/q)#;J#՗˥XK LB!"AL*8ZKM|$ۋȈZfcͯOLDLyOy|T!<B!n2B!- B![4 !B$hB!nI(B!ܒQ!B%AB!pKF!BB!- B![4 !B$hB!nI(B!ܒQ!B%AB!pKF!BB!- B![4 !B$hB!nI(B!ܒQ!B%AB!pKF!BB!- B![4 !B$hB!nI(B!ܒf q+h:S!H6*Wo-Sפh]:|ZE+痒svrU=)?LB|xG^j ॠa",j'c70w,fku +ww8+NЇ/Yd>K.PY}iXhJ#{DB!>?%hKn:/tId$Hɶ j .wGЄ:gW5Kso G7gӚ^czK&Pkp~Uد`F6#V ؼ uNNLǺF? 76u 58 _wLD$3wzz@=Sm{9 : 5(w6DĔ GMnh WBLK˘CTBIh)mE!xH_؆樠YBW*UO襂NA=ÚF⾽F IDAT)4Ɠd p (d/y[lߐk380k/E[X90 k Lg1ZJsCg,&l|ujb+b|:E(hcu5@r ,AkVF<CZ۟8?p2?. c` @Zd`Ԓ.B-+l"zD?_^:~d,ep˥V^6l?%#@!0e!+F|5ʷ&Q[K}KlB!êKY7{<;y]%3w^}@_z]3!|9x3&g J9~&Q XG6 &A *O!#%O'rgp 7` %&xpzn)2gn?88TRڋjҮDS M7h0\iBZ.ʹ믾o۟P*Z8+ҋ\^+QHJ%cSnNLfǵi0n0x hkM76\yf`pO%137Ap5p!c_||YWi(W8=ϵBHq.IcG۩vɉ2G  Mp"냱:]Sm;nNV+hS!ZV?7O`S.Qu)Q׮ G:=4Z4%3q8 je3(;iLHxpA6&f/bC>*^t~0'ۧB7vnܢq:([}ɝu]3Ie{/- GO-Uk.=7m@Dՙ+B!QC??Z;RR#vQg%q9v.ŧq:?߈~lg>%g8MZ'[ciGKw4^rP}:2ĠZlo8UۄI7?F?9>8шR#u&ѴKQHH,^C<,5I: 8u> (UPfKlJ\/U>TA *8N\).`zRWH鍦ihmֺTo3kk. &pۮR/|x<#;~Bʰ>RلݩzF*W4A~XõfnZ hh47ɸiQQBL&xB7>Wݹ?}8Ҿ7 337?䪠D6D eKH+̂[wք:cL:?6`21m|?+2Ԁ:h5M! r~[|LbId/hfk~.+ ]g,"$|I}s>F4w)& g9*Qd<\-hB147$p ٺՁFs g"~ VKTUT[UT6hT\ɳ;~&l?K鳨XObYDb}rXMtd>庆Ƙ(,o#(s}k-Kh(:_S|l{9{ TAQsF/Rg&SòPH{7Hn[MrOW KB!>WO?AgB{?jx3C@L=ÿ?Us#Bx4]gwwBt͕iͣ0qӂ;R'o@º˴=/YƺtքBHBp`s=z?4b~F! B![B!x8H(B!ܒQ!B%AB!pKF!BB!- B![4 !B$hB!nI(B!ܒQ!B%AB!pKF!BB!- B![4 !B$hB!nI(B!ܒQ!B/8s:\AcIENDB`incus-7.0.0/doc/images/ui_set_up_certificates.png000066400000000000000000002754261517523235500221300ustar00rootroot00000000000000PNG  IHDRj`tsBIT|dtEXtSoftwaregnome-screenshot>.tEXtCreation Timefr. 12. mai 2023 kl. 12.07 +0200gcP IDATxy\uW2ʘ9(*[@%j"#KUͫV--E+V2 ׊RSrDa~2ᣜ|?|~o ͨC:;v¿q!u^eddtDDDDDD^" jDDDDDDDD섂;FDDDDDDDN( jDDDDDDDD섂;FDDDDDDDN( jDDDDDDDD섂[֭!"""""""e\Ɔ]\;vj'?ϊxRk""""""?܉;67f$<5 fִuDOV4qOV zr>!Zz]lh*k:2do ,;tߥ5Z4k}h}A ɲ-ji%hwWF3_[\c5%|1}` ޟU~;|Bꪑ;Qսy;-O_~wJ1BFFXR?㍫H5?;# ;oƾm`%sZ< _j7ݡ/_W,)xf$b~qc\xY>¶Y At|`<ނSDDDDD*ͱ:*qhԌaݨ4XQ䮌࿖OCnUـhL щ$_%QN;m;Ǐ--ԠٛSX8c>ofSI9r/F911?HO^1x%.;Lj4~+=/<̲^'`_}EՆh;ac7Qч+!sW|7 kd"}0ia$w>1lZ^yEDDDDQ,}2cC_a\]tUXrM 9*)&yS.,;K[bccw_\iKzr˒IϥM0S]T/ж*Yt>Khm),Kގ "G⽏/-%~ʻ˧xFk,?٣i{I}S~HBLU##әOR0&Lw;iRźA ~5s"6;m},%/pe[_o.6ٳj}=drdkDZJ Vb`Lb࿧R퓯\7 W'1;2 =U`f'pK[ϫ"""""Z>׽vQPk 3̴+ Cltg*N*җ6~h%'g9%1?%eG8Ylܻ0%3G&qx|zQ.iBOA }j:ows'9D/jy?/_~$lxّ/N"ą'c_Ng?K!a{:+ފ&zL̮b݁ds8aE) K"n}N''9$m |K+~;- x򵂨r!!yua-&wm&W~5 :a}1.^2-1YChxd2j: 1>_4j^eA܏!@c1z4*xJDDDDDn!U[@ݑO3Kb,W: 4xu'=_8C#4g&yBM4r rzs£gҨ{ݞB-^/v !a i.ʊct 5>~,돔S ?s䄁-_l59q˿hO7g^YHJS =韔/#$4&s|:1~rx>]K2m*ڙb!q0o}dHuҌ㳋OMuٷ? or}è2sk^1Yq~RNٌeS6ޯ@KLi {b$!򡭩.)u! *Rlj:oѝ poc d~ɑi J;ҷKЈ憜=F}5b訑vk\p*,3 ;]g$!8}.~߱u+1 EidNb ̐gZ`#9#rm^&~.RX[9S.#,*ۘ7 ƱLl_,k(?k$K~^NnӕމGo~޼CG߿N8٘5S?u*ܛ@g4$x;M S?;OOgh[a/:.u|̥<6Yl\\ _ʩjpmkLKqc%Ī7<(g 1Ŷ4FO͝~ |cѡiGz7yuoO.+ɀy`s.#fliKɊg`NT3hf (<έ-18ys{m\3Οڃ}(Kx䩊5p:h9˸WS=E)q7@V5gHxvab 0zP[~8¶iGWdt8m*=>u ~']$B 6h cΧybg/$)D?*ʀ,yя3sKՎ 8P|w؈3y8ķbi"tKi;n!w_+0ysKX0v<H˼˝hQV]Hĥ eʴ%LiϏrKx56e99kQ4l˪IFT^V1nQJc)ӅZÒլnK4X;^!iQđ|,xR2Oa3pi{ p;:`4H:t04rǧs;:#?mq[\ؿe?ywtF8>a+CO`~mH_NJ9aKO@ƆevFm`dEX ? Ylql[zе@<Sװ)}YjuL\Yۘ%1LIy0Zch(}Kds֗>S>ܳbfO$R2Dx+3o]YGq _$op81ywylgzކcS3:uem Y=|OM^__"^&0z^4SG9r093 f³C͝(qS:zI<#lx|O2x1=m>FdoN7>a?,c)W,i;/ݭ..ౝ=ͩc!}oHk?5O_ c?a[A)"""""s]AEN|2JHP#rQPS\fT5˪%5G"b߼"H;!"""""rshFM!eYH1:qOLN&t L]mM$R>Mfk+)"""""""QPvkSN̨q ]!搰it"ǛYU1[s078xu?s`;K=9FtFMdG7~f _|2?Ů\ =x^.Sɺi10b.\oe""rZb\~6#&g9˱kNj{؉q}C 7xnQ7ᅙlg!ӊ3-d'צ4qZt b׹4hЀuq{eT{/C4INNt"!MɲN8~%3RN%jq_nc㡋-k.D]<r6杜%WXr%m۶{ĉ,Z޽{ӬY3pppYf<#,\'NpҶm[V\YݗʰekMCn?:= r3̊ȁfq↷GƎx|}憷t)0l߳ފ&q,[G IDAT`(jaC{{A{6& >$>>nYhIײU-vOWp3j> _3f({Խ!zԃ ~!pN>`'8xFYt|P8իCpNq*b]_nig'OSMkO>TK1MMx饗etܹB888/9uSN=wlx'{|w_ gW cXoO_c[xe ,zқ7ZF4 /ocPǽWsݒ4|ۆ_{fz&n>bڌU_gӈsKG.=@\s9\fPN8q>ե 5B !䱾tqhJ Ϣ6ـZu)w"I!Ta&N~9B7yoe]n5\9N hPJtTfq̟/MܡaUpgWAC{Ը! N}O>ߴrJbccIHHpHS\ΝIHH 66ά)"1j0>}_m#勯ꋯꫯHy5v1CwF1X0xHslFZӪIjM% YI]1]ZѤEk|̺+Ld ÿc+6ɽ["W~7C_>M4u22{)S\oSS‘7>52 =נ/MG@<7tiĎ}&>  E](h\˷{d?d7*@gΒ 4h 'r^Ҕ\BA߇bǿ-w‰l_q.l.|x{O$A l.QӣO/}{fTpreff2zhvMfͪ\OfXf ݺuϯe3j0ecJG Om[Aĝ'ʋi("yc" ]϶~ Xk{6,1hu-B_"oNc0& Qd3G8&ݝM񜴍Ma^#Ee1IdbEBO܎m!:<0Ml{γԮVKrRz\ ,,B  ћmHܟ].-X2dZ,017RtA1O#@Fе3yeA;&D=G`g'~˧"~BtOԥU}cBΈ?n+l?K;*С.z؃ w㿀A+|1W,U3}3eO4͎ߖ(uxW>2rn^?73!O&ӮL~w6W^1wk?O.^ý{28׸ =m@~[Ab_fΜITTԵg|-4,HΝb̙Vc/m$2&ުaI Op3-[\`e{ow{({ѯfd7}pf#~)Eo w{s!i(8cp6blpgW:3/[:ŷkUc=:tl01fD,e(1F]M,k #`ۓ+! ,}{JT`۱}\B.Iw_ 3yi- }prpO%wfyPgt0#8: N7y~9ƅЫ?A䓛-۝r_/1q8#Р8Od~~oU~9LgLӌDA~>NoJQn XfMS1wZgM<777222KՓji 3wb9χF1v|)&uv!`'QY8_ ŷClҮ-ֲÝKK@Do&w{AfN 60f_3ƿÈWf`ob۟G_b;z< kJ ѯn 9=cayA9o3Y4wu2o˫Z[=lEf *4F^| TZ8HfgLI2'GѵNXv=Ҿ-s|@7oOxBFA0-rv|D,2K|IjF '0:M ƋؑF |5LgЭEg 'WRa~ܱtpu˘rKw^ѫ'n׮FeW@^-r۝ض}{@,m, #1&&3}t\\\xgؿu6QA9l~Η0 ezxW||3WԇW>Hy+8ofkvDS3WUq{ R)MX<;ޝ"3|ץ;f3^+R6nСCqp(kĈ~m40p4B^f 27V6__CW_Ue3* d-/M8Eh?.&d;YLׅQ6ZsY$F3=x'c7&7UnwlxȼTl?*bok}s{c2^.s-/1b{|,ZmCYiL`b_h8dzTFF5Z3=tם=3%/zjby5ר ZؿB=ϗrMU*ڐWTd "IL.v~fҁ4l ɟΝt[j\qǓK2VdDk6YY[rSʲZ9zK:;v< 9&,=;k/c"ٜȃJ<[vxq.87Ήc1`p6`12_gǢe\m!7)%|8cT9t8]0ވAЯ/GN>5߿uW+"h+VH=V%Po6_|mAMiTd""rs#tW/L2wW%~8sSWz=3gɤ]vhb+;Ye$KKlWI;Mp꓈|tС2^Kٸ<;v/櫇N.quQDDnlJ{-FݾE9l l :){N^ǹ׳%~cՕm,kvUԈ܂qssvJrss#;αrAcW42XދVȰ>pw{RF'eMnL[8yj|:|yWG1xF ;-XZH(eĕdƂXgy0Mɗ>y3i ѫt3'cHܛIיXvƓxn *0FCZ2u+6ԽMR2-sƒpݝ̤aY̋3LK&nGuQDDnO&?4ڬPra=Ú_z`ˌkӃMѵU?F+y<5ګr=H?4n3I:qg <ߋE_z |;Od]M1M;u?+?N7lփ uO(%..=iڧݸ5s,WʕtތvPhsʳKu2=i3;thG@—3rgH𝹞#F}^=FƃX/~yf~ܚ݂ÀwKt˼'ODoDrkޝM<$~]FXr9Fھ4ѧk W2G3cL F:Gi3NgП| ƽOFr2m!XGÈ޸``Cx>vCP~=.^Mpbߟ'!5 uQDDn$|4D?OndCty;Ð]L{{4èV8r '2YrX>C00-$"_zh<^I[pmd@ܦD>E--8Xѯϯ9XTfѭj R5jK[܂ڵk^8@v2W3f6TuGg0TOJPbMi7AD~ݓbEw"ߥaTuDDDDD*Z$r ի ^oBBzѳ{ԣqlN.Sfy85tcKo-AgM7QP#r 0`oUT2`j@XCjϝuţ_Ocf|f̕XޔKk꾉[̝;;w.xxxT[ᄛ0dפX[:6={M a<}5eO?tD̘1mHΝ.ʭk߾}DFF^ͽt%d" |A362J 12q&=͓bʺzxZHв7.MmO<+I> m.VSMDDDDV5"(>|8ǎr=ǎc,_c%jjfG"$bهSISRÖ,0F- >-e?糱|U^_Uۮ&։ae$.e-K#01į=>|k꾉t-lԨQ:u@֬YS̚R۷Ç3tPFuz = ]Y$~j9ւFLxw DlFƵGuI݀~vM [VrJ?"#8;:# 1Fp¸^fV~8?^piLM7[mf\)##㦶rJFMTT'O򅅅̝;H/_~cC1UMwAJF|Smui铈0j(/pssc„ ;vB 9v[ng͍/t4""""""HKD(ڳ&66 6nwKpuu]vՋO>ZOw"Z$bn''O""""""""IDDDDDDw:goь;FDDDDDDDN( jDDDDDDDD섂;FDDDDDDDN( jDDDDDDDD섂;FDDDDDDDN( jDDDDDDDD섂;FDDDDDDDN( jDDDDDDDD섂;FDDDDDDDN( jDDDDDDDD섂;FDDDDDDDN( jDDDDDDDD섂;FDDDDDDDN( jDDDDDDDD섂;FDDDDDDDN( jDDDDDDDD섂g㢬z}־[ִ$E-I`LRjOlh*edfiaX"TbD+&ösoo3(cG1|u]\oFDDDDDDDA(q jDDDDDDDDFDDDDDDDA(q jDDDDDDDDFDDDDDDDA8t"r?Kj( y{{t """"""R }q jDDDDDDDDFDDDDDDDA(q jDDDDDDDDFDDDDDDDA(q jDDDDDDDDFDDDDDDDA(q-]#r eee-]]{''.EDDEGH% iZNYYYK!""bԈTLA8KVrbM_,mGgb :.Pr\j"DDxԈHgϞeDFF퍳3x{{+Fn IDATɂ 8{lK)"@q$۷ '''ڷoO~Xj.\ߟN:iӦ.`,]\'7[=zk߻EG̬q117{󗛽 Ϝ";Ϗx?8|[= 1$O8q;kmq̚H"w  s={6QQQ1"^}fdG3oy`o<n~RؚH}Yعm:>o5e6mt4`0+3xPq_ >-\#3+`w7@-ẇ<mQgQ:.iB >#60Ĝ}lt@[jy0ǟG,i]o+zԈ6mDjj*i* "==fY6x#ߞ>Ù2ϫ{4"uKr|]JaHݗf/z/^}4軶jZJWNϬ;ʾFNdp~5refݽy.Y?&L~SU~X!^^xf{ ߛp?0uKܺ :3?}-]H3IRu𢵗OMd۩J,a~1`&~Kc̕q}Ȅ#y ֲ:짶1s@:T\[SaTs*%WKukZnMkSi՞ǥd'(ߦukm6\v1ңb]I8v,y|x~IwcYBƉ<̈́W1m탷t,K&zp01;KH<~.|G,e#lژV0^ew#Tqކ+)L]_V=~֯^\WPǬƶQqIs//zt{˶Wïbcɬ&QP#rX,;-[о}FӾ}{lرc5g(:!_XӘ0`"]cɇ_Ơ!Id|ٕn",dg_B) Gqfq370֍dFj-QMY>F>ľßl$t7$%f>cX+Nǰ{3v\ow1c6WyINgg؀| |Ҿm5ˆ=l*L#+03+m4Ɗ΢]6;Й I tG&uK!\.+bϔ|ф v2hFc5pF?gSD,.FںV˵qP8KGyөVmă;zˤ&'N#0~42' XwlK 1?gZf_|IW&j@.cR=_9g<۸U{T nmnU뻆wCk7*(p >3#|Y>>`IҺ|"'y->r6)k%zE5˳6t1kB7\Hr,TΟliӇUDVcm;O?[β&2лVMH6v˴~"p jDBgϞ%==-[T2ԍwr8w'IF/|oϙ9s&ڵٳtV_ _RXRZޝ \:{A6+}ػr$?5 V~+Ngo|c' Jfå}Gz/Wm,}tĊ-r;s|_e}m`dqâ12|a]u&G&0d-_29ƒ.5-SGJ' F{%/]DΌLE %RHZ؝pOo! NC‛w !b7 = cg1TAYdF߅?3g3J@+\]Ў- 3h;nfqm |&kU`*vǟ«z[H[%CťY(ۡOpЋІMs‰|l@VIK'7>^9&ҿt 63P(5(!}[QV2gF} IccH%Fۨ6vs$xZaI~o5ȭF.{n ӍA9C9>dpp}$o_2:t(wn։.mfr"z5[[ya{,d8h/k*}I%MDa ܽ ɧ2+.B;6soӎ҂J0"g5B|W2ƧW<%-8rŏL{- p)-q qiySCIaЫd I}묝uqĝظbpo71 OXE3 f>RX7|󫑈(5g7N -!B4*7MUYx  #!fkOsv>(-- PڟUUsce%\(7_[M}&}ʯ;%L cFm여>޴yjJH`g>%8eaㇵ 6>x4Mu框`?I̍zG3}6$hLM<ŶbGF)KqƈHu4G]/$**-*4ڸåZ'ߌ/lXXՐE5 on7%,w}[fFa,8ً03F;7`yΔpq:ƒC|q|{._a9H=_] گ:֞^,iB͒q܀Ẍ&A| ܮ4N nffdʭxLݪVL?);Ag5p.nn\,Jw,X-VT\3uŹm}}77.ϱ&=C?e}ålKcb /c&ˎ2V>wGzѽYBP.R`/oGvU7hD>?T?NqB[EA]#00ϰuv]CXz9:ұ7Z"rfn'l\lŪim:oV}mղcyg˾e̸45ӡr9&.P1k6y+<!7aAA]jҮ]o{I'B65۵kzlBs\Da\, 7/< GH7oR1D_F܄~kr,ѓg+;'"p Rv[˙Gj`NO'8X᱗%Q3v$˩L0'ˏ w )$y W|ێ8Fo#[ y6jyný_o/hfMfd\ 9:$C?ʩ;6q5pAѵcF|;;]"W\5PtNN\MV uIYFF=7psn9y {'+H6J4g m6я bƃdl[k x1Jd$vܯO_Z7r{8?l,y{I$`!{3Ҁ2cۜ eEo\&1~(cLr*JYTZ~l#ipKDUS>yd};,Q629e,?ryedێLrb)wvrYlQ;ˌNcBL۾*?o-dWZSV5"R+3M{^߅9 a1)OT"#iϧKX+ܼ;܋-_˧j=; ,K\ \0=2Op<I+y$q_Q h|^Y ߣ9u/. H3. s>gՕFNg˲Ro-o%lCD!iE qf`-?z إ3gNB;9+0]12z4/ya{&3**+G14>Ohl'3Aw(%R*]=_$ԹKYC*}e/2` J_^̵fF=g ?et |[q_gHTy [ A S?0 \s;MK.!D730=z,cƙ J̧7|$69͚OvpƼIo߀\}}}03sz%i a~aKX'̍G0<}}W~˜* 黩cwՆjb l V,cVp#ylMHOl 6uC0YMӟ4&uY`;4- IKc_ f."\19sIYI X8O0zb~Пgg$b E6Lʺl ]*Tzؓ)[>7NwYn#v0:|!c+&>1~aՑZ7'"З:ҔDDD\ jDЭj"##d08z.~{#_Fgʕ8볇PA5"wڃ]ԈJsԈ܅}Qk"mIXeM=Ct}Ѧx;8`(m8e g~T4$,3HA]gʕfai[@XX5NyRSSygo]ͩŅ\hd2x6e#pڏ) uؑ(,YR-$ϿO_)K,!**;޺›/>%6k7|}(k ZkqCBB}3Wy 9w3Iba] ٹ<^y7=![GBdzH|n\< $DҊO^7|"vFz"""""r"V/}M61vX9sfIr K,!!!ߙ!d"˜1c9}4ڵc߿sq\¹sO4iڵ| iDDDDDDzԈ8ݣ}޽/|i:ẉ>ʳ>{$"""""rQP#Z2O""""""""BAPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ8.@DDDnҖ.ADDťK?0q jDDDDDDDDFDDDDDDDA(q jDDDDDDDDFDDDDDDDA(q jDDDDDDDDFDDDDv {3l#ca`ۥ.BDԈH)">oӨZ`? 40 j4xqہi3 |b%˚#s/g^8!s2oۉ gɊn1QƫD&y]dEUvoA"rPP#"""?vڦ5̬q117Oaw{rޡ,LM#m2fFoRxyqL-/(}1f9L/Gi35&Lf?â[q_p]n6ldy8ԣ8v4Y!Ͱ]Ugzo.u9[3;w˗/?Ҿ}{ZjՈV v0 U;ʡ_yqǎ5; IDAT*`Mq|h.[3]nh4e?:8yb+8ž-?0t#\6ҳpO!ꝍ+m;pui-C=[rOΡFDDDܹspeڷoOjj*Zb̙-]ڝm;ųm-g&]Ό~c;=Nƥ,>=NY:ѯCKW!"-MCDDDA8E0ac„ n,J̩Fƌ=J`3Πy4tdI5~׆L<edOtw?L~DϞ3gvE&oL@ŸgJ3e07SdC95^M}ڰa,QA&Ppm_9=0fv2&4Ϡ'zgd#0254ÿ97yc d"02 '6Dx Ʉ_`_MN ُfܓ>~r5~=^K9,#zezڪ,u6hW& 97pU`>U GX6WifD1ԣFDDD7 sjܰzr e!D=H-}>J귙3"݌\\/eR(ˠYFe0l&ϊ-ld͏!6K3RxF{:J0PV7$GqR@{yRܾ\N?W9y c}s8GŮ:%.g@bs1>-C'&3{Q[Ѽ l)::uy(i  3F^\^dǽ!Y,Ovɩ$]i.s5kk7 b4&7F'B` aD>4'nv~eyg~8?oT7rmq3^"p$""" ҽ{w&L@VHIIa:uՀL1f0|jLJ} .eu "F4©C2tTP7~Yv~}"K;Ew妢zsFIb2ȻkBen`ìLD^b4YwJ%٦9\~_DO}?% F#FB;)6P`5bH#f?O -OzSF{<ѣ|a`86l%} j$"0/.!vv~ʓd\ old҅:6cB^|VUv-\G)Þjf"r(Fkݺ5Æ  77i4O.fjGc Ň* F[BM1ꎫ݊o1*),*W[6<v׽&nvJ:#cѓ^7 'zbrn|^_!9a!?JJ7z:g&8 f P1n ᎫpXdiNC ?S2Q@cc޲COc~qI#?=o7Sxs;71`~b<鼟Oǰ6+;_X\MUyoOg~^(t$#q6{~r֛ǥ/}i}DD{M\d/]Ῥ4ۙi5""" N{ĉ4]ý_7 \â sY1Ε ɸLh7H}Gѥ'f~-vߖUڢ,ƳGP 6+.(psԶkg2-P؄Fqt}qg47N!|qyiU׷]gB{s>4Z7OaU D2̫1Rm=z3EBC, ~m k"W2bȴ3P f &OwV1\4~+mvoCycV !ESחOnJ ;;y "FDDD$66(lN8 q$7bOamϼkż5$I~`<ykͻ"ɌKtm8,4wYnVO]E \;  HqU+ns 4t0|W1ucXuA m&[ cyb Օ [B_2v f[YB+j|4Evm{MPz ޚ޶<Ɣ 0Du"^ϰ%1O\ǣwjzVO}KLhX>ԩ%p6g(FQiկΎɭEDDDDDDDFDDDDDDDA(q jDDDDDDDDFDDDDDDDA(q jDDDDDDDDȭ%HGPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ8.@DnϷt """"""R 5"w!oo.ADDDDDDO""""""""BAPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ85""""""""BAPP#""""""" Ԉ85""""""""¹ q$W\K+s=8;;ҥD!M)++k2DDDZJҴ<)pJViw%m_\ WrK? 5"RٳgY`x{{쌳3DFF`Ξ=euGO cg?,o;Ľ! kYCcbmBb2b҅b0l0BCC),,ddggc۱dgg3eV+ 6 eH=XIn~Z CyzL)Gգ>ʲۻ4j;fl>g^8!s2lδYqg0x8fD oˮoiL֕|KR{| ׿I[yQ߶ED25"¦Mԩ\pիWOҾ}{prr}׏UVqԩ6mj2%X>Ģki Yf~˷Gw4{27}_9^uqf8v[ǘugN[½Tg(n*4Ҷ,cVntrqgCKWRSeu[O{/Wiϫ-""iyn܂ HMM9NNN̞=(FAQQjזiOXRX}*st|$ _ǿgGHG/yz9:2vww8s\2 ||wCU+[[0?#;AmRئMHMM%==!MeAAA5"w Ci,u)dVZ SI셗_Bū}VM+2_JU~gYӈ]ԉ ?^ïF̬7% ߄ɏCv˖8z?d /ތ\xc/!;{n}[wā|;zbxE_$_W:xDק&TLy0~0]?J8>dD׈JkoY6 yȄWkL];6G07]M^xudV:ǒ{wuuFMJ~g&Oԡ*Í,ٯ~bk|1nM֭im#6GǒX!cv*nog~TVu'ah.pN\l@3;SӉn:..ꖈ\= Ԉ\<3f`ƍr--ƍ1cYFgs >;:[oH<^Ĥ InUUO ~=XF&bs v8aT[~VM3rXw-TN;9:4\5}z[Xo0vQwsQפUP5dɾ'Ox,ֿ]Mu lgʙ:qHgi@s? IDAT@΅3ww w%dnv.l>+Y>c,W\lWղak9k["rhujҥtܓ&_4zKiywMAAK._y^eL=0pT}Z{1YHq2ٶk/GА:Ugc4<qÐ: 5P?@kKb &/O $͞H `uuEM^\gfF✽\=nգ@uȑ#TUUqƎwhsLVQI@R/FN8X`9r+{;;=.ܟDӬ [ɚ3aFG[9|F nOkPa?DBOs NgbFE϶WIݧ-?ybN9szsek8Y}aFmT49)|x/f7|9iK/qpjq1K` Cb*#NQsR ljglR'18.$:?[14mH5ugaz /pJjkK(| !xi0sqb0aB0,vn١i/y;mޖt/;d3ҏ~\7 nNFxϯb8;a@:jm"~~O>n`3mN7ޓ  44BxW]%q `uRȾw'2 2c> ޮ mq|%0.'RX_'cu)Σ8W, Z誡gMcLOa.'ۭ$58>a6Q7ĐJ;}_m@) laf8̞f{nx0%(n&uB°>x#9KfGvz{%6enDmxCj"$mݺ)Sp [-nJen- x]NS&tq 7裏u+2pog}V 6Obsd8Oɜdžgv'ݔlޒpOYS+W77 pWFɳcV0wmR>{ŧmš!-/M~oPQ8r7fr/5zȊ_?Of hjT Ho8K[&'!t.a"h<Xi3\S+)ݽƤ⩩a\Ft6d a/ =A '2]Ǥ3]B9[viH؀-Ka`U<lῤjZϿC{\ \67Z=@P~s^4o!Q3@RA g2wHX CB-zDv N6t6HpVoHB|9f 55x&i5)ω)KRX?h^^Fdd0k3mf/7I9 "NsԈ\{=222.n$4ꌌ {\Dj 蹬ڂNJ?!VAqLzfxgG/Dܞ6H`07z&k*73_ȴg]4.8Lc{fjSBVe$]Ty|ǽaN$;c,{񉇘PLf  wRkٙX4,X%y/,$46}u2ѷF& g^r/|Ak0cXx绘,8~n75ND./ Yv l&~]pp=]cd4.azPğv%8{)| ʌ{Ur`]RV,!qdnlZ>FEi&hY, sָtRŢݱm|o*K(|ՉqZm00{ԃu䒖6|' '}fk~_^̪599oa_a~Q nnJTVDa5яG;LT8Fٹϳ$1GӸ%Vxba2$ԑ*Wb==cؘIw:s^O%8(\ku؃;~>sv8i"y9&QՅǫL{R>7pgw%s]xzpmd 'SV1jGvK@ǐbTFR[^\x|?aE䮇ŶmeTYld5uL ޗq G}?X1/Ŕ}J̠-"k^ '~C,$k`F0a $ܓƸ+v<썻Qخ -[t) N#Dž³<!M&[ e>q72Ѯ{u`E^"M@M$z)19gі <+z<6YCݱďł]R x1PQ6rN#!55cR0hyQ x{X%`ߪ90UC1``w~3+w7?kBcm}zsݰa߇tq9& 6~U}UvϾeMeGa﬙owlVN䱟{CS9H!S5W8m]21aftxR2CIQ aI`(yfobY @r$ Öb:{a19l9ZoaKnYǂU?CY wXSGfΘ,gEJ=%SHȢpK \8?kڳTN3bvpcr*_X̤4&=]aF6Y>6V",dZF2 Zu+xhIGi$'1n^)*]e7~d"g;6L'bB`Ug{1O3xP%k!KO1N}'Ul}w ̩ɍ[ϳ{}/9z@ԝ? ]ڞǙG|N~8z/EQB= &~K]CC{a 8o]$mLVv![e)"H(r;|Qðxb؆ t9pwqG~ㄢߘDG-~VyO\ǽ 7ҰL\ |ciu&]v./s׊ ;B878Q߳fbEDD]7CW9#R@5>5"סGy{ｗUVq Lа.#1D|!/k84Ϝ9믿ݻl惄wABMl5lH0#>@Jwxp}h$iڥ]2@4""run}kZ>/|\Q#r!##˗w?[j-(?ՋQ1GCf>*~j|/ú3C)c""-$$&B_ַEH^QEDES_|SRUUuQ _0uT^|E,K =FbYtCkvPƉ׎1R'9{؆ǽ1<1>Y2g2z-RLk oDoD. 7P@/`ƍξ}:u*SLaW(W[(#'ϸX쩧)@$lWPH\ǽe 2_ARE1KQm<9FflC.s3ٰ@Ikly1ߕA.-Ţ~"Aȑ#W|/23f̠ tK3gX|9ߠ Sl`ÔKWDDDDDh2aa:t>~_o_pΜ9_|[oO,!CyqSauƈ9ڌK^JWdӠp&>z5yp <%HW 0Z3` v'<?|:S瓹2tP;~YC~xf?__Z h?;jtʿjuld1}-*eyE3/Ȯo:r%}kYw6uK=㭶&S|y|L'gy?#ƹoɹ,xe^l2,s]ʗL&}h$g_9oE'k5"""ҫNAwOTټ⧌|eg_ܲ¶-96;ͽcW; -k;@?.!Ͻ9*;SOU.8k2fd~ ׳W'g\@7&ڽT!.Kcc2]\AuzX6stm-lzլ eی, gBj߯fO#yj6Ňb# jļ!Ϛ:gH8M{CN_0t9}Z gOkkcĻ.*T~9{y)1kk\٩lY]B}F}#;HU5ֽ{:?7 nni޶RQ0BA?=ًVz~ЯMp߲#q*~|,q~縹`GnHcܑLzYv$L`Zn!z4. JD޾BMI` 8OHq[ \ɲT/'s`6˰XBhsEœ8hGfqLiN<#4j;2 _b]1-,]x‡' xqb`7\|y)c -IytgEZ{b&55$:>!f8 ?{aYСpRY[zkY4b*8>NY]n`,/_k\ak}Vg,,8A},lJ͉ #p䠇۳> #~dN*YY~ZeH5\ԣFDDDԧ8W/&Os-f'0oΏPwx|' =f}] yrh.=7_L&.i`1Ğ>_5umj:f'{;[(Č% Nk;KT(_I]H*6kV~H)>9I~B1rўi2(JKƵo݈𵿡z.y`cqvUyC ڎ3` kpvZլ3$ވ}۩:uDqxz1']\]YOhkd ΃fĶ0(7u{|i^Yk$kf=N49@|z Fbeѵ<wG <}ȍMvX~6jCKm=w "(y)IZ,qX fɄ{SV;]% 0D_S@sHfsj>` DñK%&#&U9 =ApOcƝPhl \-в_En|,[8qt)vK򸐉ĔX{ }fNux] :JMHWFFca{= ÈN/լ3&4_8=LrlX'aԹ $=ijWV = Dod`"7TQwz֓5h ޗ^y|.R1dse8EEsy&?o WɄ?|' $r-RFDDD~}odv!ߦ_W!0_v#:Of~&KyMr^eJL"d&_Bhs@u#no-h 45o/bD}o8e44oY/Jn4`4x/eM͊]aEuN$i@Ν]D?7CFB~ڗ \:72 O½͏uوSH1{ӁǓx)ΗUXpGm3j$ ?Z(Un' ܉lF H—8{~n`^$ C& :jX#Up w8ѧq37 owН'!>_a_wda<[4$>[(,ndF{eض/#Xb+n';nZ̄=s푨Mn܇GE9X15@j'.% rʺ*)ⰻڮruЎX I-u($(VFv)S1`:z{waVqKq,]5Zz;Q[RR̸,G螪.W\$rRFDDDisLJ2R)~ʡT{N}Q/0WTO`unNVLY |o$cT|uZ,Z"=ʭN\ݸ?pbk\n, sָtRŢݱεuݣOdP2=[ztpvI$ܸ؀O|ͽ(.)oF$/76q˟ڗ$;@ӧRۦ!8~i<>n>~=ǂ ߱F0)#Q?̡ h>XTAAHCxm!5Ka9's(zg&C*p8mɹ'ɡxtDffװt]B}D@XrqnRyGp`Z-l;҆5FQ,doty0Nq19?ECY !d nFZŨq,xOJQ%'_n8 &463qBHFx:koDd\Pti-wքs{ӗSZ&b&e1J 66(zgn#mlF>Vu/DXʡX +J5O{ p(˙OJ ax́D5h(Ғ7} qcI:;y!of¬4bF\P5[:"֑#G: " Wқw45#POڬ˫os/e̓Fї:s6N&Tv={ ]NܳCq-}K=jDDDD1$g3kP%=}.y7Rvs6sJNyȘ xqmw>kʛlLy+Lܸmӹ!FAEiL:KY[Iz5Pg2W=LzKի66ءpm}^/댆>! }ޢOGC.ӕX7{G&muޣ@7\nDMgKU-kx1&EFS _m.7JU]'.P#Qf.;F`-.;N1Gcc.|&"""'3 """W qC,YK 8k67K_7ܔJR{'l蝤:cS9DDDڣ5"""%By[&aNH"1 o2jx4DߙNFO3ԟߔ@~m&X^@4%KZ9 S|$u벬~+H hb&ń1ɇsu&22lW=mv (dZZlu>uɺI9/洛ҧ&|gyձ?N쭑uv\>N>@#sOK>G'UwۺpIC IhRzU v=;<8`GDDzzԈ\{iJ&⦬`&"Bx3&3A`g>91'v{`ЛWLb҆Mp"gXۀN2ڗ'1hi3ɛwqY9` 6ȉ֖%d ୡx|rٵz&/3rǬ2?GÛs ^JK~X H@,{xoIvtrݚuW] _-_v3T)HhYu  &1"yKv##6qw[]?_2݌+7E؉9Eٛ_Z,F~?AT8 +$/ʹvmƌusqD:sHc6]~֑F|~#snaRX0i͟Ûb-ā5ճw|r^D\uq!ƚFMX""M@u&QB*qkIly0'$#p3sώQ?J8 y076 Qڽin͚qR %|&ВsQ` 5Ɛ8ҜF kDSY4E6: 'qHs.Q,s;V)ʣ ZɲT/'s`6˰XBhV:ݗ_oY?#eH 9̴U,Jux26zms[ls`v(%0>3ZYً}BˆOkH l_I+ 0عGMZ)凳;؋SI'80E-.>됑2leĪu8,'`".l 1a[0 q|GZ߃kyWk>{XgPaдV9I qi"ecn<<塬t2S3Rӄp+N!k;R* }ö̵2~ (r<Ĥuq}s4{>[{gw{XSu|4Nfzߛ ~[K5%ѿu^ {;vUHO{guv: 7k9%ZjZJXIm O8`ĿnsWP}pl 7NR{ IZ}h cXYCRu=#N5N`群Z˖Ov^NWwn""rP'[A B-tqR7'`DCKzKa ~/xcT[3[Nk48Ugҿ7 Q"1wXŏP5>ɆklUӴVǿ[垄Lumb4A_2a>F}ü=k4{ h'?96cdMwA $U?3M3fCڣ 'F"bl'%n?$.?/EP[G1ZRɞAPAJ;19iH@<ٱ5x[?jkZlq-܂w}kb;ky*_, Za,vR`x0Ȗm+׭]߽?q l^3D!wO{*AUkăQŔ4L/N]>#Aq[\c8(Q#""r3ן-Xy˨p^S66~fAʼnB6nʧi-bEdbr_a//~F 7g(}6eb/x ; iyT0>Џ'>M%Qˆl+_A17=n` ۫{{Z?'@me,v1fxZuo&KٙqJeS;m܄!E&4s)a5ww&9sfbtZt#sN|%`9Q3sk6m,C+׭0^j6+p+{TWW@ \ʲH_P#wG:$b[Cy-aSqB?:x4W6 [1.spoHOOҀ0neAST)?Fpq7db>>u;YҖjfߠ PB{#[ۧiEDnnANj3jy듈PFDDDDDDDK(Q#"""""""%JԈx M-""rD""""%jQ#"""""""%JԈx %jDDDDDDDD5""""""""^B/DPFDDDDDDDK(Q#"""""""%JԈx %jDDD:pr;;<%jDDD=!%%sμ DZ̘#I=uvTR $rJJ:;攨kW+n;&oE]īYs(na{o)G;͉&rRnyDGNFVJǫ>YFDDDD:LA9ľxns{GvnHޣSUL`˿p5Q(I&""^H-jDDDD~ KASz"r އ.RЅD57b2cIe[Gq52"1[D`޶R.Y|<2"ZDdўF۸Ōs >sg>ÌY=f.s\_ˢOiPciSK~TXâ?XDzd֎$X9:͜]Yu{&/㪮p5רE]%_+0x5q;+7ّ۝.`;I\8LU;!^7G}Ys4"fy!fl5O1/kdS-̡O}y32l¯+@9r;w598%J$?;O-(JvH<9TUV܁c|̿r,\T>\_Lމ3AڦqwAe+w( g׈TΊi<؆mF Vg" Ǎe` %g"fpuf(P.ޕ8<'א1UNGIt;k*cC11?r *Kz^k5"""r~^2}̤}%08 <[]L,dR|! (Ｖ,="࿓X_ҮX 0ǗLZ6icq d-ws1M!@wH*+1lA,G134brNˢHY~YK'fQŖ-ڱ#=y57h ~4gj179ϹGu$Oz$#FFFI~;t~}>:C[܀pT{28C1j;>Klp-qYOoel͂K$%1-)8Զp/sC/aSSJ$K۵LTmD{D 3(r$ŗGK NܲIDDDӵ'hO\.ښ4'<1R#×h!1!nH7O搻%3O8C !9H%32q Q,T}k=sSoE4ht[,1.kڥcLTR!*[kԾc1ߏ4"5{JqR+ 2$NWRyG ly~Ix;\ߩ?Ej(}{˱<(yrg\ E!q̨FIac$Rħ :WD~E]sEDDM?I5u62&pNw'jpO`&t,cEM; IL\ۖn\in?s{r XH;k=VDƺ7Vd^˨?Og}X/T4'էqHthtuycX<~V;9|tc+.%L AtK䱁0cOGS2;FLw%""JԈH)]%&0u*,11}16DsȝHݔwneOzFXɤU b?r]r5_ͤT, e.v+8n dR0k-a{UBljN']S>o#>5@GΫԤ%IW+?`%6wyGTVwo]DiDDnyAN:!M)"c2{Տvv0r%A y{;o(Oy@""<US确U1OĦ$Dȯﻏ*%J}ܙCaz$IDu}m""""""""^B/DPFDDDDDDDK(Q#"""""""%JԈx %jDDDDDDDD5""""""""^B/ȍ!HEPFDDDDDDDK(Q#"""""""%JԈx %jDDDDDDDD5""""""""^B/DPFDDDDDDDK(Q#"""""""%JԈx %jDDDDDDDD5""""""""^§̙3t5"""7ADDDD:@]DDDDDDDD5""""""""^B/DPFDDDDDDDK(Q#"""""""%>x.\_~C[m݆]tPD~ԢFDDDD$t_~C7C[4ҙt5"""rs`?:'p\-,Sqbs8~k8į͍ʛ}:.C7㖹G>gMD0-Y<԰<1Pxʛ_WHt`9u%JgWo Vxفg7p5fmGy,ro{TnYxo[.,6%jDDDnvgK}y7WPi#gjJ44\YD![GJY3o5"""7 rM'J抝(<_$k U˖ۮcW2B$Mww=JR- ,:º+I+d^ApSqCI&cgUʺlmΔRoq3d<|!g\ As|ypoA-(ٚyJ:!&EDDnfgUJŌUFD\Z%{[vvv7] \j l8}{W)}?Lj"F$:|}&iK65 Ρ=Q}#uVDml3jM$mJ(&a}}:^r v>tZԈjܶݣz aٗf1$Ɗ98`!7 iu=NCβkk~X{-L'&5|ncѭ[7fN.BtK5a9*|Yɽ10cMZ9G6Ƙeeuvř6Xeey=0Yq\ޘn~]!{3`+{Cm &faim_ӰZ'83 1!LenYhr,{vZ1w[+æD[ρmz[flt [fY Lp|^MϺYmO1e 0`[ .3vVSfb 'S[MYkUkݣMǀcDg݀RHyu_h'bV6IbLbU9 ON!M.q lcs6gƒ=oJ6Tg#\9Hx3sJ-dA9֍?;k?~y7s޹2e`@`rr1 I荹G0xE'%jDDDnfx?PE?OcۉVzUJG֜q$= ?ʇ =ty[t_+#jZvdF'`G0MW+ɡcv#y`/)O^Q_5{ȲcDPBcexk43u#0Rp%_D ZWtV"@#oǵݏbmW;w Q0ڱ%>;O+[{k޽ c>3_լA O.aÜ&88%V_]e)YΈNm@^VW7%餯*#jZv;l`h48'q^ôXna{Zh3b) ]>L])Op=LHH/|2<ӻYJՌ92Y)m*o2> aw}LQ0&0]x|JIj !d:a6bsr\G5@]}-zs ]1g\%k\>zQ&* X9)H3 )bR֌N :9TƽuRyuL!䏛y3KM^x &Gg>0Jq,lҷ磗1FFǕNU?&'A?;9:uHXuR} 6}%d%\+\ABu+ϡO"""702يL!KFH;{2gɡŤ/;uLQgM'"=_D]շ@"/i/b{ Oga<16Hχ҈Y:u |Y,@=u͘ݡ e%8B1B b+(8$)#62h1#0D>ASN|r|QZ廓?4[qwWC$c^&F#ۮY:H}k?7Tdalޔ}RlS8z&UXMŖH oz]kݣ1#>?D@JEl63.)29[^#log2k[_P Ar4{ /KN>cn`|&pl,!~|ܸ\,>~ [r{x O$(8suPn-6/Sd/EzʹX*8rs-"u9fų_n.H#mQ3W3)p3w\saŏØلw33L$L&i#i\^_#% IDATב,8!eag|2^O~ z_oPPS](0^,I³/vð,`*ImӘ.q>ty WK ,$FD?Ca_e0zHEEbOIRj|]"[;Z8xK9V_GJ>7В \=5辽Y[*uVyIp)>|8Zz=Q_2I[?]PzrBOׁ:;dKE!BNF2]ǯ~*R^Dc a謥LUʺ E  :$1H<|Q̱8б`1ΨTRC/ $ip 9걛<"75"""%qKwp"S=qj~؁_-hĿK1r'G0s_fe';x/'H߫)vIM]VKSpT!*BHI8Fne{(SI^/ؼI~"O?L\g؟ZԾkkPA_Q|8s~čJ"A )*E߮P[h1MH_pQYv=+%5YtR۽11yG:eRa0ƻU{ d~kB;A[ߪJ):REG]6O_/)=&j*?=(pSMnLww\ڞx~ΕPtnjµtYx45*2<.cQm0#-5BշnjR¾,̅)MfYB A>/݃08ڜPP'[Ѽ D'Nhwh8[#WPgI}c t5/׃шO}b|-k<Րog1º$^7,w|{$I+<͗*>{D&^j!F| kH3_3Bdn9dnA8˫ڼv}H}&L!GTl$#(Hf*?b+˝ɽ,ѐx睾^FeрyZ67oǐ>>YKZH+^G[g`,.P}(8r;\yNjjphn?s/_vC >&L=j(E*!ӿ?g1bU@kɝv;^T_⾠c/}Iy 5xƕoϙW X@W\pU zkV<,eⴉsmtT6aoW7n{_VͯDm γǾE帩9ہ0AlzoXPޟj_h s/-Zy;19q^M <8O`Hb xj>a,D@IaƑ1dg#lQy$KD\?_l[^v>$ >ʈ{$Fϑbu#I]~/{O<>@<_ěݷu'iO(I9[YUԧs(9v=ڂ9ľw×8-Υ叟яę6.;Px 1:]Į=>3!E2smÇ.tʭ:O୕s.U8jQNR,ϽFΘb'<&l/-_O?3M/oB=,95͸!gۂ lb# %~'cWpgϟښf-"쇱$[p-I}9l{ye -ϴMIL `xd .dilNeokZdgb2J,Y '*.5}oi; tb|*Zz6"{1ٰ.FT*^C[6JNQQnd6r̿9IJ($1_ 8 U.9RvƁLe)bL&EKA[M̸b+(vGy+,68Q'ѿx)3? $uH]H\\,ceT88/tOe+/3am>%'*(;Q‘֧Δlxqb/|l=/&LƼ/|6$mz^$ [J h3XcSWBѾ"ǧ?]}sp4'Gǂ?] X5Qy->ȣ-]Ƹ[Ɏ(31/ DL &Jg0( mr֎4*!E*X: 0)°iӋI[!NɀTGdMFƴLgZKӂ♲"y]8x ;'٤tƿiT[Et'm8 aa(Xq^HbFm=&[ }DĴLw θ'T\j ~! bz0, K& Үk 79a@* /SI}Ѝɸꙸ}v%kNDtx$|<,-Y2׳8[ܛtfOj\X+I-Q@_àhacQNZo2Ks~ D'{ ٖ/b2-Mh EX_I-]Qĺ%)*.buyOD" -kqK/즞aGqXR])Upzv=37Bt2lQf݂l}x'`|6;ClVD1 uA|̊=To/I͚pwm,$&X(8R]9wu:ܷ<tSidu;G9Z\m)dl(d%ߕRm3?QG0 ѽI9OX=2}_81$ { ۅx.bDS D'gQ?fdZO ':~Yꓪ>adnɂ r_xdzx<[3]2 WLn ˝:uCֳƵb[Yk:v2c#oHlIl-RYc;Z?;v^]{S?l!jBr$~~XGz$~āydzoY~EU yfxpyt$"mM op=c-MG_&wH.J&v쥑tܻ OMӯ]ݼ]>>d<6%֎JϸGP`I}}W j6<[_uUVm;0ryk_ ?pwtO'2}LZuW^7L/DGO 'w7 /XKjSoDȯ@I""""ި2` W*-QFDD5"^H-jDDDzSljQ#>LXDDDDpmuvr '~JԈ|||,nGn[@.]ҥKg!""W5""""""""^B/DPFDDDDDDDK(Q#"""""""%JԈx %jDDDDDDDD5""""""""^B/DPFDDDDDDDK(Q#"""""""%JԈx @DDDn3gtv""""JԈBBB:;u}JԈx %jDDDDDDDD5""""""""^B/DPFDDDDDDDKtv""""rc]p:~En!v>>>tҥCZڳ{B}ԢFDDD&w+_0ڭli%jDDDDnrҋx{mVW9h[]#C ig&}_3w)fdDOsxd+д(V$vrǔczN6TǧLZF-F 1"n 珱x)}?Lj"k E\W5|c}(:>uܥ|r& yjVsSjW 9O "~3[{3lKr;̠.iǹi){oC{c5-،5a^q0`ed=G0fkoMr]ו0 7Zv Lp3֘!,ZےEcVv<#IͶFe󇥄g4fчKcqSs,'R,}Lp3sѾ)7;s`%88sAdY d0u3gakؾ7R"wc|}h/kF٪<>xxFِwel*xY%dɁ{ٚ3$s 981 ZL([ ϤQ}_R]3F4*Hol$yl?腳n7㬑d6Edns]ZV]ĺ@t)K4v4ڦ]\`Ó7cF=8u4\)d\"v%ul3fMd̑kgy$BH~wd OLdlRt'0V9$B7G&\bf{Rd8W%H4Hsx4ɣWRta7E I_sxS7SzRxwїkwQQ]iY2SK1^SB &H(1J$m(6bT1F4VIJQKQ!mXt)rۖbPqYUJkseoj?g}8_냬L"&T6"I+x:Pm:&1wCsG`iN; Ysɓzqhɳ/bOo 7ٸG⴫.OE$c&<"&2'6JO ߢ#?YEBm3KZ&Tx;F-8wtjf&O78"BcAj\R96qĄi ҆$~{WIzR# %f\+?-(^ñ̚D\t Ηz5ǵ >^>`K^I> ! _M!k Xn11#8`ZFFO`A^XIF*xUPi FٜX7 2 ]P9km~Q04 `lcW"q+P , r8QL[82YS6]S]/_"GFRMjx9oAnmPr6JW;s?oo y/x!k.IJ*RnFvݹriYDHO¨_3f[7MgS|-;7C󒑱4R4#~=%Tl[ɜtz+yƬXC5QAcJ`PȱId| cu5uVifK)kx댌Tpp2fDo)TkhXp |R{~e'PNq3^7q1]9L‰j>i /T5BPί$eQ;~,+S^~|`!_uoxk ' FJ؈<ʏ,dVo8l]k %!ɟ#8WAB( AHP_O =zAL.%yJ{Pz-tƣ] 觾{;! Ykb h!k(8Y@JssPWJ٥:˗\uN|5\x3,r$%>4$jXa-?~] SeCCʼWPC݉ͬ|uG(Lp]|i'uSW`B@TJ#}4k'1snuOL#!B4ak@^7aTt+~,XB'=@sD-| CmV7qUWAμ"$=KoNj'PT4%3wmd1_&~|tR^;~!3͌:J o%eVRzCzH_dG S~[X .9 Kp;4xv۰*/MXLn_d݅p > *\]‡'0,J g#['}h5wmhhؓO4> *4;Y_?p՗ij3-Kx>Jx6eWcZG›#b1.6d|àմcy0/zY^Vj|3\4$4&=gG IDAT  MZho>)G%_@Nx M*,8sI r['( ƀK.C< #o=aQ#8l+k1aDdd`Xɇ/fzlY{L#%Ym R#{U BOPl! }B!*+U;@kˆDBCC[~ inqZAmlZ= c5~hw; @рo.d@C}or8h@%m-j6s̙zo|>|W{e>Խy|1n3\ְ albLP.y/[h/ۏ,|@WgmOy EOo(hotpN-3HoR}D *T{lct؋#f_|Pw"U>&ze/j[NTmjh;Tዏڇ1oW t[ӗ`zn {aU )_`\17`4;^"v23 !3*"\l#ʭX/Ua[_KD"rJ0}avŊ܈BGϴOq j I+$YKAJIWאUi#鯱GAsnM_ܷRe+2Y|h[&OG dkRmzBCH.3j+%dgllf mtddx'F.ٰ]´y#mNXTY̛KQ ֛筍~ULq5 tӛe|Gx-O<|ah/ #O<*e$/!?4Zgpۀ{(8B-|7o@ݵ6+*,gC0rꛔ䏡AiYR21/A"=jB)X}.oVxe@盂f?fyFrS5hF?gsP7yWxkIZ_Bk ɝ5U0l8 {kM^r7ƓYnoHR,b[ 2I|*BI7m7꟩kl|FZGsv̫ǭf/kXa1^I=QlxF9sdEIaKǬQ8;Û"0'.FMl~&1oCO_.쟇غz'HcuI2B4椤a|1Cx[넍C}~5J P\>=FmV'5P[݈&y,a}UyuŸ[HJ_?;]|P7^a[̲c><1f(>_9x"h;xa&7dgweMtijTcX[By؛g 7>l4$չ'M$" JW8swF%4ˈxRhfPjZ:Nĸ|ؼH)q4_72KQͬܮ0tHXi;Uѩo&vSntqB< tL!Z|rOgA!zNs1i8 O~ϴ.2zl%ي2bWe:Ĵ, !~@8VCEHDf.XnOk8:Up@@Ȑ0)Rb~%>#^8zYz]7',[Wg*4xZEI$HG; 4r9,,NAzBj'sx_5 ㅂx֤ul}>OF񸴚yZ1 L xoCSU4cjc(TŲ͟S( @+ޙjr`_dgB>xE&9o`l)o~OVS?~ JtnRqQPSa#2s|W^Z%BmcV]=!GI!B 477t6B'!B ,9B!jBv,(=w*G9-{:h5;]b:Zuq7+8ݐR !S]LXk=7ƬG~7h[1+t{/61NBOd>l&wh z:#@u [qf(uSgIқ$!~W_EɂɌ~DK`` GF39SMJ\J/uzHo' "(( MK؍$mX Ӽ"S#H~?H Y$EJIdUM+ II-&̧~dlǴ:M? u1PMq&~g0&yD(@ိۃ({y)RDw> !5%3&ahJ`}Q?隅Sۣ=9lT~lB]7q1]F' RBuXQ `xnQ +Wciŧ[Rw`\km[n.VL`iNe֨nSOQ &XkpcUݔf \%" WzʄG==;SOr'jBU8iC]BtJlhF 4tSw,K>,{wW Ϗt- cXw%leݺgU6c{B~s!zӢx#лEc(OBdBq?khӾ-.LLh !$܄IY=F6bHd+|L4???I\gmgGW{hΡGFΌG LD;PKhd?>1t,cI:wj3Οj`p6_{)1nVvbɭJvOƐz U}y.٘{5WJv}2ڱŲC(7Lͅ5F&t` Cmh\FtʅRr'8 !R׿m-U[rrja_yA;U~}h X{:ȰlL `7S8#C70]tvm"9ڙn6m<]MpO̥m WQ2zJ>+M+l iKrZi9<1l5+0楓8"@?DNȥ@-c2E1$t`+lJ/.Ë պ'0asd39!Pm ~LfLE$G:O$u{6c":b]QzVh_?J[#td 3GpOCφIZ߽Ycu^&sdOI$24y1nG5B!L˸Gk)M.eld-:rJOpU$>V񮶱;U;OpnVe'Vc$_u|]_յS2{n> "z&x7Qt ]bW;O@dv#Y]]k􂁪4gcSV6~8q'jxWqkgSIm)%35JMOpb*RZ,&1ibǪjLO&cgYBep[*p\`qGoZILP~>N~9CΓ6HMiXS3-9qdyO׶;9N9gcdzPm?6A<#Y`hSy)9Ιw37̼I` Nw3zMײo(Iy:e\&kZ{ws˜HA^ gY2 #9wӼ903Rl{y 67SF4˹úa2[To/wA+h_?˪Pv6d3!6.-ZXX,nc+2(}0I#{2'8}4ߛMI> !5[v,b{X?3Hۻr5UQ, %ȹ9ǕRv)CLՓm,` tQo~ꏺ_GzCHrͧ<>;c4X).gbe_ƐF%utoX:o+KB5Ql %SYVƇ/% lrvVyT5BPί$eݧ c񆵄;ju)La\8 O)|6l%\-o_*2n8yPSHg=fT.6<龬X-w``gBUD=7fƠNeU#.5e35f=lQY5W&yo6o< `ZJF ߱Tt\ ـNIf"&"z v(sw=O9iOUZKcZ3P~x$/^@?EP86a%zG€?1~OʝÍs鱴ڴlNxFTNʹQ<3ieq3wHK rnPOY:W:: +6FL?(صnGfͩ*y?TUbC,/_VRICL9pWB!wz42)(z*i)/ǡO")m(/~t66Ojܭ !iսKOr0pn hxz%srqkð`ym>]VXl~b12^[:Aߗob~:F>1aQ#;Re%O4Z >ޠ @u[6\1x\qY2˺{**#Gnhb*nOs5]3nʊ3?ܐ:|Yl:~oRYe#">R̘"!gqs5/Q6N_5SmYSPדx٨}DઉMO&1:GbD÷r4SKR۾W՗_`9U#*x򢓠:9ONmv}~װ׻Hb\I"$fs^XŬ_Nbő,==n$B*1~l8N]owӾT_˫&~aMfB)<ŀMkPwRFj|A?_l˱rlLXJ3}GF[ MMm{(/JLv\%g.ņ:J[a$;ˎ ֢Rg@<*w>^'{&IBCǿojO'ߝWmT J3=7V] ݋iк#%%bD^f*w VP) 4ிi~:x;sZDȚLӬߎoxyx;4~< 8PTdLPԑjbL5i:nW9iQ^S(H=Im|վuPx}_?Oͥ؎n$7'I;o,I7`0YX6ang e%{Z?ò5B!G*5QY*jvk74oo#V?!S;iYOkC(Ĉ#wQ:>]xmeV.t.FZ+Xp'>3=l8{ ZȒXn>Zב|'|67whưG`5hEGiQtoggQGANújؒ>Xb JB{;:ؑcȠTƹM%k*tΊأ 9W?j)˧Y6,_ǫD| 9/XU[e6L0_aa9=f gOh/aUXT0bs?HtCK^O]j\T21c % -٤yYn,HJa^#$TĎm"v4#7ݮ_X8rP=Jc~ >I\8hAGXf"Ƿ~h'eQ9KLh hjBY?X(Y01ČMg IS|˽W8sw&g6N$.& {N]5*~6ylXٳmLcCֳq$L"띛_\$o F 1Ľ˞tǭfxt1! g.wu2 Q8=N~웝䭸%<˒y#n[]Iz7T񛩳x;]xߓLmÂ9l=rmv ^פP|08*I I|6GVP:ytXnYAfO!4,/`E'xG3.'j)qDc҂bGyƄgP;r5ק-:,w+]Wgρ\|$i.h(^\ƒ2a$R0qɡ8n3m{~heA}SC:= !Dk/_,! 7ʜ)KJ Gy Gym֖j6''f:i•l=PEu4Dӕ^)*|k:οc⚽yZ1 :I7ۤ]jXKmDԆ9ȔY]O^@yB!8/"yk};Y:hȂh2~}t6B5B!B!jB!B!z !B!BB!B!%$P#B!BKHF!B!@B!B!D/!!B!B^B5B!B!jB!B!z !B!BB!B!%$P#B!BKHF!B!@B!B!D/B!Ľs՞΂B! !΂B!$B!BKHF!B!@B!B!D/!!B!B^B5B!B!jB!B!z !B!BB!B!%$P#B!BKHF!B!@B![ݎru;Ab^QmVӹpKyB.B!,o#iغ7[1錈{ٌ!uz:#-_䑗<81~PJFW$+OB!s-v~h!yF>_tcJ\J/u_d6Y.]"DL8+%$~GuhGc`Db´%tD" 6aA\E@?ncf3aZ2\`pL [Sl d!t` CNNY=PUM~^1E8N>H ^O|$]/ Iv` ~F2zR6{P9>LÝl&b@&bROLX2%-ˬ'?3c>ZJQԛٔWjw懿WhjBܵX#9w[g=]M,܍- G0VvzBAz5SrLeJ qj!/@ ԓq9~;9Dz߇RF'8w'JfTOs|(&6:O⭻(Rkz۽"T^=!BC#|T:^ܰt^]MƁDWkУN5؞*bhuuY(AR<$S>t uǙ̾>o$q3z%(H|J4 (אϟҼ 9 ҡKʤqOLf`ܟux,vsV.;"Hպ0<*=!%Q#BQLRvW_Kӣs)eb]GVUcNp4(>l#_X>1{(TX8 TVpWl*ix4(MY٘48@W_} f^FyXGP|dzCN&8F8r& oG>x VVC_Ly#yl\ry(ǿj3 ۦR)%35Sy2kTDa9؍dMXvX0]Ĥjms^z ?@} R÷VL5dָgR4۩02]w58? fl\4E/rњmLῧFٛYTq|wmL[CYT`wv[O|@ r82>9`l κƒz[f6a&6;] / |6(IS)XGƛhItVA'(Dw@BQX׈Mql_w("5h5˾!}Jڕ gSΆIP OQ CsYPexJ3ξ4[;~4G$2y 9%t;JEGX?$Fi8{6*Uoֵ &3_#4c.sg[5\ٽԕ2_JN:Pk _ _+֛.]7 1E\yyI9PngܷѤ0eצp v ՉJ Nd r1575@^/Lzծ[26.9:{=t9g=Øǭ$E-Y /fp|9c $""r0w'1[喛wPF>&wzKWg{rlq ?3㸜 1*y=<;m\VĩnP"8Fk bǝyFD:QȧU3K ǁrQv_@ܘJ8ψHv'IHef9% &fd0FܨS\2w]?r3{kzf5\,) ,>ݑDIVc4vP&"|_bWCyaq|sq#?s1;1# rs}b |DrG^Vn ~zH@ln]JNLcOywA &v8J1+J(qI"#|CO8M n`{mϤpkA];t;~sЏ~=={(,EqiԳϙr(}vr+yy1~DDDi_Ǐ{AFM]c4w8֍f{j6q73eyWJD`q.N^P-aJz7X~.-@;<րSuގ1/_"SbVԊL^T"؊ MY46bhj/dDgh2JԔTzGf/`H&^vh O:z[1;fqs r}Z>H鎮ΘT-k􇗱`{ &vq{Mq pl Ō87܋𡉶C/66ݜJ~mգ3=cX: cv򿟂A fgH=O̙BV Ix^ֳQa #_7.,mHURU )wcЅ[qW;ܫ7`[*}y:Vfsx%9=0ʆ'x]%_1;kw:EGҸ X<^GeT-#y8VɁVv#"s^ IDATaBvsye% .:o̩v]q9OavJ(U@QE(q[N} =&qhc_FQ3HQOGN @W%W+^V+#q܂oeDvSs~r;;n /s{yO2'秶q,[kr45Q(wpDGܽڙOyhb3ܧpcxR+em裶/X{ĕ͗qOk%ą\lH]{JRnH̽̕i3J:-.eaܳe]hJ}Q*w>oy&u΁"Gus;DukZgH(%tmַЅ׫3H. hc8-;9ڀ-yEJ %:6fRY 5%wAX ֧SE .j8֥oN?fH6UPs8ژ)`s xe{ #.j>hW FkǞ ,%aQsI-d5%u20bJIƋډ03q 9(7j G]8_϶2dpQOgX2l\_8P@gu}1uKj3;RO2ɽr? WcsG=L7PTYCMY?6k")wi7ZUIx?gmk䘫#Zma}6|gdy&rG9\tۧ\R>{4s6wt\'zfLKg˞  &}J蜹n`C>Iwe\vM)H|J*[b`h"ڽ+n:2Hne,ںNeZo0,j^Z]N㡹P@lr|\DyNukuԍ wYGK2"4MgÑx-nMe 7z2q0<T:x f&1Ѯ悖{g *]+(UA{t`Bi}(sk+Xj$4EW ODDdt xe,6M:evI<҅Ww0wwk RZX/_ǁ$ſB5'2#폩,>iI44Y БS[aHX"//t aln_3?~vzNyym6ȫId 'uŴū54{6 CEN|^$nޅKl= f۩LCI H:~v cDqQ0Y Լ 9qċL>jyxpn)f6|] -]0= ݭU2gu|0Sf@Y 2^a Kd9D< $ǖڗ;HbN_Lgkaã66\LhTET0Neät6<0! 6Ǎ 1}څ~ez\!<ۤ8&t DA^&I-5`F%u+R(&r`q2 uz@D-~Hu6ѓW3"kS2x2WDzJ)E h[zE0XW~\ ;q#C Jb{aٿNo~s2F0wDLeSX041 #߼K6IعYZq}Yδm;sXG^ȐD"x 5""""""""^BAPP#"""""""%Ԉx 5""""""""^BAPP#"""""""%Ԉx 5""""""""^BAPP#"""""""%Ԉx 5""""""""^BAPP#"""""""%Ԉx^׺"]N'۷o??n6~&NȐ!CqMEDDDDDQ)Sp}q1͛GEEib&7)SPSSs-"]PI~.4EM]݊-ŖRyjZ+̸ ml:HflK.y`9[FJu6`C~ [ɸYpioahJI[ L8$5g\s""""5"oAxx8~;k`ڵko'<<7xZW_Dx(+<0.1C jVcF)鉐̏ϋFLo7e憵,u̮J*Kٕ51aHF/.(HDDDS'K/D^^䮻R,YB||<ӧO,]ޟӿ-r~[w@_qw41q0z|"On+3wb*] ÖB"fXϤw jri*o?q4!D3m+L~$|MADDD#jQ#r{7ˣ!. ˻*-kNT%>P%sa\ۙsn$al&}?.5yO36z37vے Z̾sk:Oq?-w&E4v2w'q )_oaˎ+%iؿZ.b׹2Bx*@I}|Ԉ\jjj={6 83p@rss={Um`_/[2uAz4. !r)l~y{7y퇠y Gva1o?0[I<7`CTWO?3ƀ7űly_}ɤfl)#V(x;@EWADDDjQP#rzHKKRKtFLm"--^x%DryЂ$\פ"ܕOfn>s Iw0‚ J4^|?%.]lY;QaDæ'⛗Ɇ#MEhy!rÉJLL%yA : ^NASX """rUh餰Wn(`͸|!D.\H`` NL}S i'2n 6z ly ;olYH~4M TFr*>sRߴ<=^V$>2Fėyٹo(;ru(~11\~ܿx#"|] {|Lʨ DžNܠaM(j㈺d ջyKo9k:;jQ#rھ};SLǧ5O]s$>>>ȚJc? m58]K%r-rVgxyEӛ)noe.2~[{mY1 л"FoƶݜJ`N}} 1 DWo#SBGЭa|jx}q&ɛx73ï.liUG]$^Mmu#4DDD*RP#rt=ed])&>>?b1at:rlx1j !1]r?@rK%K_l{æ'I_Uf XP9?py3T:f>bq:%E ?y&7N@ߛZؙ4h:HGČ^כNڒBǾȋ&-]rfw#cml/I9nPP#r6lX+.!=Ksb;３?Bk\4L ;H$LQQ;wԮO!'wUavJ`` xlk7X#m.bGL]U0fDBQ.":w(pKnOA611']~E1\_.~ܿ.Pm filDRߕ}h>m=sI<>0F&2h/M$gk5GN%Sjʤ-^IE\ڹi>㈋LN!Gsx{fc\FW3p/|?Xջlxy#^&^6Lx׭SQ=DGGzjHyy9;w`Сcxb{bw]d`+ƅǂa9/>l_>7RXm؃k[Oe*57 PP. s$n_L*^өh^˨hB3q4;}P 8p WŧF)$-,;c'ש\o3;y& 7Sr*5?YKҬJ5"߳84][Y '  'eyynZ/f. i M.O`d~'ś1C5\1镜Lx{߰_Й1O/`F'gN`nĸg* v{ eŰomdcݿܖ6QiwS;IxJ^atc"""" j, WÌ;r !CfرڵիW3nܸTsI9tLy.EcHϨlu&^jHvm:t́=iS]ֺti=iݤO?n5(&dbVbwum)feجb~mSYIY uk03p@/^|ooe/f?;>OyKS;ƝSyy/V7F2%m9p8*(ݙâ1!}m%'*.; !6f2 ?/6s,5 %dM .$X?#}`Ï4YLƉsTQf0q>/*4W+Uu\Ƥa't-e3&."gRA6 pQ\v➘XfSK\X:4qD>^}U||.M$. c>@njE<\wz2?` FxyTm0([0糥;!E*jOkUU5?gb@mI!y~~~y*)pnkҬ0n"rrWR%`W %2̹:Ab0pǕ5?]Lzs=Ov'|r{d4hB9 b${đ |2vR;/@NtEMuu5CCԵ4( P[_k9^߂K? qmo@:qw&&`NjgŸ|'/*]K۶ hjo=%H$4ԁ+w\]pF=?IDATsOﭧ[7(BCCf;TgKONvv6LfppڍOqT ?wVQ48cxajȜQv2m]:Đqjv:{3+]0 D&Ajr/8ϟt+ٳgK.%,,&,,h^zg$<`N+?p@}Zη\{fǓ]wuY?OΔ)S5kVׯod2OMc$xp(,#4uD*;wij$;q .oB\B*ﮎ!j+{ ?7LgZZ ֻhD\M^Ivc=ƪU:{f|Wmi!әr+x:Ge2x~wuI# (@ YE R D۰ݘO7=WWRl#$,T}wjnsDЍ@}r bo ظj#&NIٶ-n6v=W~NM&ee]JBKIKKc…m0|f222X|9W%OΈ=QLM˪HJ:]-nmq~j0b*#z9JâEؼy3;wdɒ%TWW_0pXXVbСL6 b Xn "$2SG6vl6&ؘ=&]Av>7!7[-ca+l' 礐W_F:7Nf-dﶓ5v6oQVYf`0>ejayrtx0Tzxq<>DtV6C߽^U"o{z#ٻN c֬Y /Ȕ)SgذaP__ϡC(,,7$>>IDDDDDzFL65kְs >髯bڴiw—s8 B}QHb.sn/˵IDDDDDؐba50e={cǎ%Ukǂ9 jDDDDDDOߵPP#"""""""%Ԉx 5""""""""^BAPP#"""""""%Ԉx 5""""""""^BAPP#"""""""%Ԉx 5""""""""^BAPP#"""""""%Ԉx 5""""""""^5۷}tҫ;$''3}t>Lyy9&L 99j= z*ZWS?H]ՠfƌرzŋS^^ӯ.c-Vp8q:8*Jٻ5' µ!`!$${!UDDDDDDDx>}.YWsuSu~KT \-f|q1V+Ct]5Yz5> :|.Œ$:Sp<Ţ1 gQh6̀YGeQ[+CtY>RQQAEEK.eܹ0̙Ν;>|8qqq_T4#`9t22p:̙q_,DNK UNq6g=4Ww -rRZI ' iYwl騠tgSnm]?8EQZYbSWǏ'3[pP-2%*Ac=J8Z7w a@%"""""" բfՌ?{x0a}a޽,^Þ={HOO L8CUE_@WobOߘt6e 2W@i? u~ <<}j)ޕG]A`|B*wTq ݷlB\,"|1 % S21#>WMjǭRDDDDDDD+j, Ǐgǎ,^wDEESOXux?K0)C/J0Hc'CWpb;אfƒ "gPɶOa#!_1cJ<Ƣ771s<})^R#!H Rkg}я~ӛfۜVFb |\IHઢ M 2UT:M5ćY:ܔ߽6 *k i}jժ {AAXw}]3'kH^GO$S{"}b8``oOH-:R^IbLش$5#oi2kyd {,?qu'cb`gmL04Nȵ֭5ݻQF1arssY~=&L`޽?Klxܹ=U1"Iy8PGvV2ھ_?/o)Le`z>DHd$!7:ԅmgޠi9䌁:C(WG#vҷc?t~>M#8j9H}Lbxs!uU71ڈfimJGXNYBN~8şrG8:u۳;F:=Դo/ls8 B}QHb:hLc4y0c#JְPj^ļGc{NU7&ūY鳂qď5;u_TR\Z Ãذ&Z7f6; LOEdʣFP6o1o؟cRlּ^EDDDDDD(44]'''3}txu{' jDDDDDDDD/FDDDDDDDK( jDDDDDDDD/FDDDDDDDK( jDDDDDDDD/FDDDDDDDK( jDDDDDDDD/zc`׺"""""""5׉'ߢ=ݭFDDDDDD[HP$99'6C lϾ 'NGE)ｙŢXuryϧ3ςEWvPcXZcxG ?8@^LqIG3/Ŧ6-0"Up""""""""@X,5̙3={HŒ$j6?.6m6 ȫ9[-jFEXX{eȐ!DEEyWH:M03mj GR,'&7AcVi^J+8ZQ^֋d&oVp8Osn摙eUNe{smY)NgKw o8Ʀc_*'*;}_'Go6g{+g{\#98N6\]{Wd[ ]Dҳ(~󈽱*HբO> 0_zuHӢ1͖%~cٔa'J w'f+01Æ-̟C|21"}`&!X%%dM W%E#3YD$3b<xÃ8qAԕQxr?nh(5M(8E@<^%SM{n!dʏ${g ɚO.GMZw~@Xcnv/"""""""VPc&L@XX,{~=m<P_Fg&6zNYƚi3`Dh[;(FS.&`&z&~OBSHy̘2`M̜0)yd9%n};l>-IK3Fk/t9A񤖴|76]$<O'];ٿJ>>sZ,mBDDDDDDD֭OcDzdKtt47ofԨQ=U a9vL,}t lDY1?}\C۰IՖ<=wHn@UA^KH)#*0B:t% äP-&}tܿN_G߀%]2Fd WV8"""""""ҹn9wwa„ Z3fxQ(U (NlIٺ-mIR~,>` _{.*[w y7c,?QwVN$K?+mS1+f&V? 4+ʃhp boFDDDDDDgu+fԨQgC=Fm^ɚC`yrgsK) lRXuQYU-]{fփpgԞ]+Iwumj> WdBP]!/F]qW… ̈́.'Gg#{ 5M=0D2e1[u*d[eh3\D] (' 6WlZs~=<~=!ѧZi_o P {B 2=n  *$I$IbJAYnׯ(]]]:uj}k׮gYx1˾Si7yc{9V% ܭOqMf4@URA@9Lhn!_oԴ4'`D0̝C`$`=:wX4&?st~R " {&DHi!o CVA$ LJE[#%I$I=0o޼j]DдO_гq#LO\7-3ΰ{ҥK.A$IT'O"$wvD@2DH9!)H$It j4y7B?IjAi!AD,-I$IxIC^}$IӃ.@$I$IU5$I$Iu F$I$NH$I$ I$I:aP#I$IT' j$I$IA$I$IR0$I$I5$I$IubZ &cٵ.k׮պI$I5$I$Iu F$I$NH$I$ I$I:;X, =3IENDB`incus-7.0.0/doc/images/virtual-machines-vs-system-containers.svg000066400000000000000000000427031517523235500247760ustar00rootroot00000000000000 image/svg+xml Hypervisor Host OS kernel Kernel Kernel Kernel Host OS kernel System containers Virtual machines FullOS FullOS FullOS FullOS FullOS FullOS incus-7.0.0/doc/index.md000066400000000000000000000034751517523235500150560ustar00rootroot00000000000000# Incus Incus is a modern, secure and powerful system container and virtual machine manager. % Include content from [../README.md](../README.md) ```{include} ../README.md :start-after: :end-before: ``` ## Security % Include content from [../README.md](../README.md) ```{include} ../README.md :start-after: :end-before: ``` See [Security](security.md) for detailed information. ````{important} % Include content from [../README.md](../README.md) ```{include} ../README.md :start-after: :end-before: ``` ```` ## Project and community Incus is free software and developed under the [Apache 2 license](https://www.apache.org/licenses/LICENSE-2.0). It’s an open source project that warmly welcomes community projects, contributions, suggestions, fixes and constructive feedback. - [Code of Conduct](https://github.com/lxc/incus/blob/main/CODE_OF_CONDUCT.md) - [Contribute to the project](contributing.md) - [Release announcements](https://discuss.linuxcontainers.org/c/news/13) - [Release tarballs](https://github.com/lxc/incus/releases/) - [Get support](support.md) - [Watch tutorials and announcements on YouTube](https://www.youtube.com/@TheZabbly) - [Ask and answer questions on the forum](https://discuss.linuxcontainers.org) ```{toctree} :hidden: :titlesonly: self Getting Started General Client Server Instances Storage Networks Images Projects Clustering API Security Internals Contributing External resources ``` incus-7.0.0/doc/installing.md000066400000000000000000000535001517523235500161050ustar00rootroot00000000000000(installing)= # How to install Incus The easiest way to install Incus is to {ref}`install one of the available packages `, but you can also {ref}`install Incus from the sources `. After installing Incus, make sure you have an `incus-admin` group on your system. Users in this group can interact with Incus. See {ref}`installing-manage-access` for instructions. ## Choose your release % Include content from [support.md](support.md) ```{include} support.md :start-after: :end-before: ``` LTS releases are recommended for production environments, because they benefit from regular bugfix and security updates. However, there are no new features added to an LTS release, nor any kind of behavioral change. To get all the latest features and monthly updates to Incus, use the feature release branch instead. (installing-from-package)= ## Install Incus from a package The Incus daemon only works on Linux. The client tool ([`incus`](incus.md)) is available on most platforms. ### Linux Packages are available for a number of Linux distributions, either in their main repository or through third party repositories. ````{tabs} ```{group-tab} Alpine Incus and all of its dependencies are available in Alpine Linux's edge main and community repository as `incus`. Uncomment the edge main and community repositories in `/etc/apk/repositories` and run: apk update Install Incus with: apk add incus incus-client If running virtual machines, also do: apk add incus-vm Then enable and start the service: rc-update add incusd rc-service incusd start Please report packaging issues [here](https://gitlab.alpinelinux.org/alpine/aports/-/issues). ``` ```{group-tab} Arch Linux Incus and all of its dependencies are available in Arch Linux's main repository as `incus`. Install Incus with: pacman -S incus See also [the Incus documentation page at Arch Linux](https://wiki.archlinux.org/title/Incus) for more details about the installation, configuration, use and troubleshooting. Please report packaging issues [here](https://gitlab.archlinux.org/archlinux/packaging/packages/incus). ``` ```{group-tab} Chimera Linux Incus and its dependencies are available in Chimera Linux's `user` repository as `incus`. Enable the user repository: apk add chimera-repo-user apk update Then add the `incus` package; this will install other dependencies including `incus-client`. Enable the service. apk add incus dinitctl enable incus If running virtual machines, also add the EDK2 firmware. Note that Chimera Linux does not provide complete support for Secure Boot, so virtual machines must be launched with this feature disabled per the example. apk add qemu-edk2-firmware dinitctl restart incus # example, launch virtual machine with secureboot disabled: # incus launch images:debian/12 --vm -c security.secureboot=false Please report packaging issues [here](https://github.com/chimera-linux/cports/issues). ``` ```{group-tab} Debian There are two options currently available to Debian users. 1. Native `incus` and `incus-base` packages Native `incus` and `incus-base` packages are available beginning with the Debian 13 (`trixie`) release. Debian's packaging will track the Incus LTS releases, as these better align with Debian's release cycle and support policies. Monthly Incus feature releases are uploaded to experimental on a best-effort basis, and are intended for use by experienced users. On Debian systems, running `apt install incus` will get Incus installed with all dependencies required for running containers and virtual machines. If you only wish to run containers in Incus, you can run just `apt install incus-base`. If migrating from LXD, also run `apt install incus-extra` to get the `lxd-to-incus` command. 1. Zabbly package repository [Zabbly](https://zabbly.com) provides up to date and supported Incus packages for Debian 13 (`trixie`), 12 (`bookworm`) and 11 (`bullseye`). Those packages contain everything needed to use all Incus features. Up to date installation instructions may be found here: [`https://github.com/zabbly/incus`](https://github.com/zabbly/incus) ``` ```{group-tab} Docker Docker/Podman images of Incus, based on the Zabbly package repository, are available with instructions here: [`ghcr.io/cmspam/incus-docker`](https://ghcr.io/cmspam/incus-docker) ``` ```{group-tab} Fedora Incus and all of its dependencies are available in Fedora. Install Incus with: dnf install incus Please report packaging issues [here](https://bugzilla.redhat.com/). ``` ```{group-tab} Gentoo Incus and all of its dependencies are available in Gentoo's main repository as [`app-containers/incus`](https://packages.gentoo.org/packages/app-containers/incus). Install Incus with: emerge -av app-containers/incus To run virtual machines, also run: emerge -av app-emulation/qemu Note: Installing LTS vs. feature-release will be explained later, when Incus upstream and Gentoo's repository has those releases available. There will be two newly created groups associated to Incus: `incus` to allow basic user access (launch containers), and `incus-admin` for `incus admin` controls. Add your regular users to either, or both, depending on your setup and use cases. After installation, you may want to configure Incus. This is optional though, as the defaults should also just work. - **`openrc`**: Edit `/etc/conf.d/incus` - **`systemd`**: `systemctl edit --full incus.service` Set up `/etc/subuid` and `/etc/subgid`: echo "root:1000000:1000000000" | tee -a /etc/subuid /etc/subgid For more information: {ref}`Idmaps for user namespace ` Start the daemon: - **`openrc`**: `rc-service incus start` - **`systemd`**: `systemctl start incus` Continue in the [Gentoo Wiki](https://wiki.gentoo.org/wiki/Incus). ``` ```{group-tab} NixOS Incus and its dependencies are packaged in NixOS and are configurable through NixOS options. See [`virtualisation.incus`](https://search.nixos.org/options?query=virtualisation.incus) for a complete set of available options. The service can be enabled and started by adding the following to your NixOS configuration. virtualisation.incus.enable = true; Incus initialization can be done manually using `incus admin init`, or through the preseed option in your NixOS configuration. See the NixOS documentation for an example preseed. virtualisation.incus.preseed = {}; Finally, you can add users to the `incus-admin` group to provide non-root access to the Incus socket. In your NixOS configuration: users.users.YOUR_USERNAME.extraGroups = ["incus-admin"]; Instead of giving the users a full Incus daemon access, you can add users to the `incus` group, which will only grant access to the Incus user socket. In your NixOS configuration: users.users.YOUR_USERNAME.extraGroups = ["incus"]; For any NixOS specific issues, please [file an issue](https://github.com/NixOS/nixpkgs/issues/new/choose) in the package repository. ``` ```{group-tab} openSUSE Incus and its dependencies are packaged in both openSUSE Tumbleweed and openSUSE Leap 15.6 and later (this is available through openSUSE Backports, so you can also install the same packages through PackageHub for SUSE Linux Enterprise Server 15 SP6 and later, though no support is provided by SUSE for said packages). Install Incus with: zypper in incus If migrating from LXD, please also install `incus-tools` for `lxd-to-incus`. The default setup should work fine for most users, but if you intend to run many containers on your system you may wish to apply some custom `sysctl` settings [as suggested in the production deployments guide](./reference/server_settings.md). Please report packaging issues [here](https://bugzilla.opensuse.org/). Make sure to mark the bug as being in the "Containers" component, to make sure the right package maintainers see the bug. ``` ```{group-tab} Rocky Linux RPM packages and their dependencies are not yet available from the Extra Packages for Enterprise Linux (EPEL) repository, but via the [`neelc/incus`](https://copr.fedorainfracloud.org/coprs/neelc/incus/) Community Project (COPR) repository for Rocky Linux 9 and 10. On Rocky Linux 9, ensure that the EPEL repository is installed for package dependencies and then install the COPR repository: dnf -y install epel-release dnf copr enable neelc/incus Ensure that the `CodeReady Builder` (`CRB`) is available for other package dependencies: dnf config-manager --enable crb On Rocky Linux 10, ensure that the EPEL repository is installed for package dependencies and then install the COPR repository: dnf -y install epel-release cd /etc/yum.repos.d wget https://copr.fedorainfracloud.org/coprs/neelc/incus/repo/rhel+epel-10/neelc-incus-rhel+epel-10.repo `CRB` is not required on Rocky Linux 10. Then install Incus and optionally, Incus tools: dnf install incus incus-tools Note that this is not an official project of Incus nor Rocky Linux. Please report packaging issues [here](https://github.com/NeilHanlon/incus-rpm/issues). ``` ```{group-tab} Ubuntu There are two options currently available to Ubuntu users. 1. Native `incus` package A native `incus` package is currently available in Ubuntu 24.04 LTS and later. On such systems, just running `apt install incus` will get Incus installed. To run virtual machines, also run `apt install qemu-system`. If migrating from LXD, also run `apt install incus-tools` to get the `lxd-to-incus` command. 1. Zabbly package repository [Zabbly](https://zabbly.com) provides up to date and supported Incus packages for Ubuntu LTS releases (22.04 and 24.04). Those packages contain everything needed to use all Incus features. Up to date installation instructions may be found here: [`https://github.com/zabbly/incus`](https://github.com/zabbly/incus) ``` ```{group-tab} Void Linux Incus and all of its dependencies are available in Void Linux's repository as `incus`. Install Incus with: xbps-install incus incus-client Then enable and start the services with: ln -s /etc/sv/incus /var/service ln -s /etc/sv/incus-user /var/service sv up incus sv up incus-user Please report packaging issues [here](https://github.com/void-linux/void-packages/issues). ``` ```` ### Other operating systems ```{important} The builds for other operating systems include only the client, not the server. ``` ````{tabs} ```{group-tab} macOS **Homebrew** Incus publishes builds of the Incus client for macOS through [Homebrew](https://brew.sh/). To install the feature branch of Incus, run: brew install incus **Colima** Incus is supported as a runtime on [Colima](https://github.com/abiosoft/colima). Install Colima with: brew install colima Start Colima with Incus as runtime with: colima start --runtime incus For any Colima related issues, please [file an issue](https://github.com/abiosoft/colima/issues/new/choose) in the project repository. ``` ```{group-tab} Windows The Incus client on Windows is provided as a [Chocolatey](https://community.chocolatey.org/packages/incus) and [Winget](https://github.com/microsoft/winget-cli) package. To install it using Chocolatey or Winget, follow the instructions below: **Chocolatey** 1. Install Chocolatey by following the [installation instructions](https://docs.chocolatey.org/en-us/choco/setup). 1. Install the Incus client: choco install incus **Winget** 1. Install Winget by following the [installation instructions](https://learn.microsoft.com/en-us/windows/package-manager/winget/#install-winget) 1. Install the Incus client: winget install LinuxContainers.Incus ``` ```` You can also find native builds of the Incus client on [GitHub](https://github.com/lxc/incus/actions): - Incus client for Linux: [`bin.linux.incus.aarch64`](https://github.com/lxc/incus/releases/latest/download/bin.linux.incus.aarch64), [`bin.linux.incus.x86_64`](https://github.com/lxc/incus/releases/latest/download/bin.linux.incus.x86_64) - Incus client for Windows: [`bin.windows.incus.aarch64.exe`](https://github.com/lxc/incus/releases/latest/download/bin.windows.incus.aarch64.exe), [`bin.windows.incus.x86_64.exe`](https://github.com/lxc/incus/releases/latest/download/bin.windows.incus.x86_64.exe) - Incus client for macOS: [`bin.macos.incus.aarch64`](https://github.com/lxc/incus/releases/latest/download/bin.macos.incus.aarch64), [`bin.macos.incus.x86_64`](https://github.com/lxc/incus/releases/latest/download/bin.macos.incus.x86_64) (installing_from_source)= ## Install Incus from source Follow these instructions if you want to build and install Incus from the source code. We recommend having the latest versions of `liblxc` (>= 5.0.0 required) available for Incus development. Additionally, Incus requires a modern Golang (see {ref}`requirements-go`) version to work. ````{tabs} ```{group-tab} Alpine Linux You can get the development resources required to build Incus on your Alpine Linux via the following command: apk add acl-dev autoconf automake eudev-dev gettext-dev go intltool libcap-dev libtool libuv-dev linux-headers lz4-dev tcl-dev sqlite-dev lxc-dev make xz To take advantage of all the necessary features of Incus, you must install additional packages. You can reference the list of packages you need to use specific functions from [LXD package definition in Alpine Linux repository](https://gitlab.alpinelinux.org/alpine/infra/aports/-/blob/master/community/lxd/APKBUILD). Also you can find the package you need with the binary name from [Alpine Linux packages contents filter](https://pkgs.alpinelinux.org/contents). Install the main dependencies: apk add acl attr ca-certificates cgmanager dbus dnsmasq lxc libintl iproute2 nftables netcat-openbsd rsync squashfs-tools shadow-uidmap tar xz Install the extra dependencies for running virtual machines: apk add qemu-system-x86_64 qemu-chardev-spice qemu-hw-usb-redirect qemu-hw-display-virtio-vga qemu-img qemu-ui-spice-core ovmf sgdisk util-linux-misc virtiofsd After preparing the source from a release tarball or git repository, you need follow the below steps to avoid known issues during build time: ****NOTE:**** Some build errors may occur if `/usr/local/include` doesn't exist on the system. Also, due to a [`gettext` issue](https://github.com/gosexy/gettext/issues/1), you may need to set those additional environment variables: export CGO_LDFLAGS="$CGO_LDFLAGS -L/usr/lib -lintl" export CGO_CPPFLAGS="-I/usr/include" ``` ```{group-tab} Debian and Ubuntu Install the build and required runtime dependencies with: sudo apt update sudo apt install acl attr autoconf automake dnsmasq-base git golang-go libacl1-dev libcap-dev liblxc1 lxc-dev libsqlite3-dev libtool libudev-dev liblz4-dev libuv1-dev make pkg-config rsync squashfs-tools tar tcl xz-utils nftables ****NOTE:**** The version of `golang-go` in your version of Debian or Ubuntu may not be sufficient to build Incus (see {ref}`requirements-go`). In such cases, you may need to install a newer Go version [from upstream](https://go.dev/doc/install). There are a few storage drivers for Incus besides the default `dir` driver. Installing these tools adds a bit to initramfs and may slow down your host boot, but are needed if you'd like to use a particular driver: sudo apt install btrfs-progs sudo apt install ceph-common sudo apt install lvm2 thin-provisioning-tools sudo apt install zfsutils-linux To run the test suite, you'll also need: sudo apt install busybox-static curl gettext jq sqlite3 socat bind9-dnsutils ****NOTE:**** If you use the `liblxc-dev` package and get compile time errors when building the `go-lxc` module, ensure that the value for `LXC_DEVEL` is `0` for your `liblxc` build. To check that, look at `/usr/include/lxc/version.h`. If the `LXC_DEVEL` value is `1`, replace it with `0` to work around the problem. It's a packaging bug, and we are aware of it for Ubuntu 22.04/22.10. Ubuntu 23.04/23.10 does not have this problem. ``` ```{group-tab} OpenSUSE You can get the development resources required to build Incus on your OpenSUSE Tumbleweed system via the following command: sudo zypper install autoconf automake git go libacl-devel libcap-devel liblxc1 liblxc-devel sqlite3-devel libtool libudev-devel liblz4-devel libuv-devel make pkg-config tcl In addition, for normal operation, you'll also likely need: sudo zypper install dnsmasq squashfs xz rsync tar attr acl qemu qemu-img qemu-spice qemu-hw-display-virtio-gpu-pci nftables For using NVIDIA GPUs inside containers, you will need the NVIDIA container tools and LXC hooks: sudo zypper install libnvidia-container-tools lxc ``` ```` ```{note} On ARM64 CPUs you need to install AAVMF instead of OVMF for UEFI to work with virtual machines. In some distributions this is done through a separate package. ``` ### From source: Build the latest version These instructions for building from source are suitable for individual developers who want to build the latest version of Incus, or build a specific release of Incus which may not be offered by their Linux distribution. Source builds for integration into Linux distributions are not covered here and may be covered in detail in a separate document in the future. ```bash git clone https://github.com/lxc/incus cd incus ``` This will download the current development tree of Incus and place you in the source tree. Then proceed to the instructions below to actually build and install Incus. ### From source: Build a release The Incus release tarballs bundle a complete dependency tree as well as a local copy of `libraft` and `libcowsql` for Incus' database setup. ```bash tar zxvf incus-6.0.0.tar.gz cd incus-6.0.0 ``` This will unpack the release tarball and place you inside of the source tree. Then proceed to the instructions below to actually build and install Incus. ### Start the build The actual building is done by two separate invocations of the Makefile: `make deps` -- which builds libraries required by Incus -- and `make`, which builds Incus itself. At the end of `make deps`, a message will be displayed which will specify environment variables that should be set prior to invoking `make`. As new versions of Incus are released, these environment variable settings may change, so be sure to use the ones displayed at the end of the `make deps` process, as the ones below (shown for example purposes) may not exactly match what your version of Incus requires: We recommend having at least 2GiB of RAM to allow the build to complete. ```{terminal} :input: make deps ... make[1]: Leaving directory '/root/go/deps/cowsql' # environment Please set the following in your environment (possibly ~/.bashrc) # export CGO_CFLAGS="${CGO_CFLAGS} -I$(go env GOPATH)/deps/cowsql/include/ -I$(go env GOPATH)/deps/raft/include/" # export CGO_LDFLAGS="${CGO_LDFLAGS} -L$(go env GOPATH)/deps/cowsql/.libs/ -L$(go env GOPATH)/deps/raft/.libs/" # export LD_LIBRARY_PATH="$(go env GOPATH)/deps/cowsql/.libs/:$(go env GOPATH)/deps/raft/.libs/:${LD_LIBRARY_PATH}" # export CGO_LDFLAGS_ALLOW="(-Wl,-wrap,pthread_create)|(-Wl,-z,now)" :input: make ``` ### From source: Install Once the build completes, you simply keep the source tree, add the directory referenced by `$(go env GOPATH)/bin` to your shell path, and set the `LD_LIBRARY_PATH` variable printed by `make deps` to your environment. This might look something like this for a `~/.bashrc` file: ```bash export PATH="${PATH}:$(go env GOPATH)/bin" export LD_LIBRARY_PATH="$(go env GOPATH)/deps/cowsql/.libs/:$(go env GOPATH)/deps/raft/.libs/:${LD_LIBRARY_PATH}" ``` Now, the `incusd` and `incus` binaries will be available to you and can be used to set up Incus. The binaries will automatically find and use the dependencies built in `$(go env GOPATH)/deps` thanks to the `LD_LIBRARY_PATH` environment variable. ### Machine setup You'll need sub{u,g}ids for root, so that Incus can create the unprivileged containers: ```bash echo "root:1000000:1000000000" | sudo tee -a /etc/subuid /etc/subgid ``` Now you can run the daemon (the `--group sudo` bit allows everyone in the `sudo` group to talk to Incus; you can create your own group if you want): ```bash sudo -E PATH=${PATH} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} $(go env GOPATH)/bin/incusd --group sudo ``` ```{note} If `newuidmap/newgidmap` tools are present on your system and `/etc/subuid`, `etc/subgid` exist, they must be configured to allow the root user a contiguous range of at least 10M UID/GID. ``` (installing-manage-access)= ## Manage access to Incus Access control for Incus is based on group membership. The root user and all members of the `incus-admin` group can interact with the local daemon. See {ref}`security-daemon-access` for more information. If the `incus-admin` group is missing on your system, create it and restart the Incus daemon. You can then add trusted users to the group. Anyone added to this group will have full control over Incus. Because group membership is normally only applied at login, you might need to either re-open your user session or use the `newgrp incus-admin` command in the shell you're using to talk to Incus. ````{important} % Include content from [../README.md](../README.md) ```{include} ../README.md :start-after: :end-before: ``` ```` (installing-upgrade)= ## Upgrade Incus After upgrading Incus to a newer version, Incus might need to update its database to a new schema. This update happens automatically when the daemon starts up after an Incus upgrade. A backup of the database before the update is stored in the same location as the active database (at `/var/lib/incus/database`). ```{important} After a schema update, older versions of Incus might regard the database as invalid. That means that downgrading Incus might render your Incus installation unusable. In that case, if you need to downgrade, restore the database backup before starting the downgrade. ``` incus-7.0.0/doc/instance-exec.md000066400000000000000000000101531517523235500164640ustar00rootroot00000000000000(run-commands)= # How to run commands in an instance Incus allows to run commands inside an instance using the Incus client, without needing to access the instance through the network. For containers, this always works and is handled directly by Incus. For virtual machines, the `incus-agent` process must be running inside of the virtual machine for this to work. To run commands inside your instance, use the [`incus exec`](incus_exec.md) command. By running a shell command (for example, `/bin/bash`), you can get shell access to your instance. ## Run commands inside your instance To run a single command from the terminal of the host machine, use the [`incus exec`](incus_exec.md) command: incus exec -- For example, enter the following command to update the package list on your container: incus exec debian-container -- apt-get update ### Execution mode Incus can execute commands either interactively or non-interactively. In interactive mode, a pseudo-terminal device (PTS) is used to handle input (stdin) and output (stdout, stderr). This mode is automatically selected by the CLI if connected to a terminal emulator (and not run from a script). To force interactive mode, add either `--force-interactive` or `--mode interactive` to the command. In non-interactive mode, pipes are allocated instead (one for each of stdin, stdout and stderr). This method allows running a command and properly getting separate stdin, stdout and stderr as required by many scripts. To force non-interactive mode, add either `--force-noninteractive` or `--mode non-interactive` to the command. ### User, groups and working directory Incus has a policy not to read data from within the instances or trust anything that can be found in the instance. Therefore, Incus does not parse files like `/etc/passwd`, `/etc/group` or `/etc/nsswitch.conf` to handle user and group resolution. As a result, Incus doesn't know the home directory for the user or the supplementary groups the user is in. By default, Incus runs commands as `root` (UID 0) with the default group (GID 0) and the working directory set to `/root`. You can override the user, group and working directory by specifying absolute values through the following flags: - `--user` - the user ID for running the command - `--group` - the group ID for running the command - `--cwd` - the directory in which the command should run ### Environment You can pass environment variables to an exec session in the following two ways: Set environment variables as instance options : To set the `ENVVAR` environment variable to `VALUE` in the instance, set the `environment.ENVVAR` instance option (see {config:option}`instance-miscellaneous:environment.*`): incus config set environment.ENVVAR=VALUE Pass environment variables to the exec command : To pass an environment variable to the exec command, use the `--env` flag. For example: incus exec --env ENVVAR=VALUE -- In addition, Incus sets the following default values (unless they are passed in one of the ways described above): ```{list-table} :header-rows: 1 * - Variable name - Condition - Value * - `PATH` - \- - Concatenation of: - `/usr/local/sbin` - `/usr/local/bin` - `/usr/sbin` - `/usr/bin` - `/sbin` - `/bin` - `/snap` (if applicable) - `/etc/NIXOS` (if applicable) * - `LANG` - \- - `C.UTF-8` * - `HOME` - running as root (UID 0) - `/root` * - `USER` - running as root (UID 0) - `root` ``` ## Get shell access to your instance If you want to run commands directly in your instance, run a shell command inside it. For example, enter the following command (assuming that the `/bin/bash` command exists in your instance): incus exec -- /bin/bash By default, you are logged in as the `root` user. If you want to log in as a different user, enter the following command: incus exec -- su --login ```{note} Depending on the operating system that you run in your instance, you might need to create a user first. ``` To exit the instance shell, enter `exit` or press `Ctrl`+`d`. incus-7.0.0/doc/instances.md000066400000000000000000000012151517523235500157240ustar00rootroot00000000000000(instances)= # Instances ```{toctree} :titlesonly: explanation/instances.md Create instances Manage instances Configure instances Back up instances Use profiles Use cloud-init Run commands Access the console Access files Add a routed NIC to a VM Troubleshoot errors explanation/instance_config.md Container environment migration ``` incus-7.0.0/doc/internals.md000066400000000000000000000003511517523235500157340ustar00rootroot00000000000000# Internals & debugging ```{toctree} :maxdepth: 1 daemon-behavior Debug Incus Requirements Packaging recommendations environment syscall-interception User namespace setup ``` incus-7.0.0/doc/metrics.md000066400000000000000000000333021517523235500154050ustar00rootroot00000000000000(metrics)= # How to monitor metrics Incus collects metrics for all running instances as well as some internal metrics. These metrics cover the CPU, memory, network, disk and process usage. They are meant to be consumed by Prometheus, and you can use Grafana to display the metrics as graphs. See {ref}`provided-metrics` for lists of available metrics. In a cluster environment, Incus returns only the values for instances running on the server that is being accessed. Therefore, you must scrape each cluster member separately. The instance metrics are updated when calling the `/1.0/metrics` endpoint. To handle multiple scrapers, they are cached for 8 seconds. Fetching metrics is a relatively expensive operation for Incus to perform, so if the impact is too high, consider scraping at a higher than default interval. ## Query the raw data To view the raw data that Incus collects, use the [`incus query`](incus_query.md) command to query the `/1.0/metrics` endpoint: ```{terminal} :input: incus query /1.0/metrics # HELP incus_cpu_seconds_total The total number of CPU time used in seconds. # TYPE incus_cpu_seconds_total counter incus_cpu_seconds_total{cpu="0",mode="system",name="u1",project="default",type="container"} 60.304517 incus_cpu_seconds_total{cpu="0",mode="user",name="u1",project="default",type="container"} 145.647502 incus_cpu_seconds_total{cpu="0",mode="iowait",name="vm",project="default",type="virtual-machine"} 4614.78 incus_cpu_seconds_total{cpu="0",mode="irq",name="vm",project="default",type="virtual-machine"} 0 incus_cpu_seconds_total{cpu="0",mode="idle",name="vm",project="default",type="virtual-machine"} 412762 incus_cpu_seconds_total{cpu="0",mode="nice",name="vm",project="default",type="virtual-machine"} 35.06 incus_cpu_seconds_total{cpu="0",mode="softirq",name="vm",project="default",type="virtual-machine"} 2.41 incus_cpu_seconds_total{cpu="0",mode="steal",name="vm",project="default",type="virtual-machine"} 9.84 incus_cpu_seconds_total{cpu="0",mode="system",name="vm",project="default",type="virtual-machine"} 340.84 incus_cpu_seconds_total{cpu="0",mode="user",name="vm",project="default",type="virtual-machine"} 261.25 # HELP incus_cpu_effective_total The total number of effective CPUs. # TYPE incus_cpu_effective_total gauge incus_cpu_effective_total{name="u1",project="default",type="container"} 4 incus_cpu_effective_total{name="vm",project="default",type="virtual-machine"} 0 # HELP incus_disk_read_bytes_total The total number of bytes read. # TYPE incus_disk_read_bytes_total counter incus_disk_read_bytes_total{device="loop5",name="u1",project="default",type="container"} 2048 incus_disk_read_bytes_total{device="loop3",name="vm",project="default",type="virtual-machine"} 353280 ... ``` ## Set up Prometheus To gather and store the raw metrics, you should set up [Prometheus](https://prometheus.io/). You can then configure it to scrape the metrics through the metrics API endpoint. ### Expose the metrics endpoint To expose the `/1.0/metrics` API endpoint, you must set the address on which it should be available. To do so, you can set either the {config:option}`server-core:core.metrics_address` server configuration option or the {config:option}`server-core:core.https_address` server configuration option. The `core.metrics_address` option is intended for metrics only, while the `core.https_address` option exposes the full API. So if you want to use a different address for the metrics API than for the full API, or if you want to expose only the metrics endpoint but not the full API, you should set the `core.metrics_address` option. For example, to expose the full API on the `8443` port, enter the following command: incus config set core.https_address ":8443" To expose only the metrics API endpoint on the `8444` port, enter the following command: incus config set core.metrics_address ":8444" To expose only the metrics API endpoint on a specific IP address and port, enter a command similar to the following: incus config set core.metrics_address "192.0.2.101:8444" ### Add a metrics certificate to Incus Authentication for the `/1.0/metrics` API endpoint is done through a metrics certificate. A metrics certificate (type `metrics`) is different from a client certificate (type `client`) in that it is meant for metrics only and doesn't work for interaction with instances or any other Incus entities. To create a certificate, enter the following command: openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:secp384r1 -sha384 -keyout metrics.key -nodes -out metrics.crt -days 3650 -subj "/CN=metrics.local" ```{note} The command requires OpenSSL version 1.1.0 or later. ``` Then add this certificate to the list of trusted clients, specifying the type as `metrics`: incus config trust add-certificate metrics.crt --type=metrics If requiring TLS client authentication isn't possible in your environment, the `/1.0/metrics` API endpoint can be made available to unauthenticated clients. While not recommended, this might be acceptable if you have other controls in place to restrict who can reach that API endpoint. To disable the authentication on the metrics API: ```bash # Disable authentication (NOT RECOMMENDED) incus config set core.metrics_authentication false ``` ### Make the metrics certificate available for Prometheus If you run Prometheus on a different machine than your Incus server, you must copy the required certificates to the Prometheus machine: - The metrics certificate (`metrics.crt`) and key (`metrics.key`) that you created - The Incus server certificate (`server.crt`) located in `/var/lib/incus/` Copy these files into a `tls` directory that is accessible to Prometheus, for example, `/etc/prometheus/tls`. See the following example commands: ```bash # Create tls directory mkdir /etc/prometheus/tls/ # Copy newly created certificate and key to tls directory cp metrics.crt metrics.key /etc/prometheus/tls/ # Copy Incus server certificate to tls directory cp /var/lib/incus/server.crt /etc/prometheus/tls/ # Make the files accessible by prometheus chown -R prometheus:prometheus /etc/prometheus/tls ``` ### Configure Prometheus to scrape from Incus Finally, you must add Incus as a target to the Prometheus configuration. To do so, edit `/etc/prometheus/prometheus.yaml` and add a job for Incus. Here's what the configuration needs to look like: ```yaml global: # How frequently to scrape targets by default. The Prometheus default value is 1m. scrape_interval: 15s scrape_configs: - job_name: incus metrics_path: '/1.0/metrics' scheme: 'https' static_configs: - targets: ['foo.example.com:8443'] tls_config: ca_file: 'tls/server.crt' cert_file: 'tls/metrics.crt' key_file: 'tls/metrics.key' # XXX: server_name is required if the target name # is not covered by the certificate (not in the SAN list) server_name: 'foo' ``` ````{note} * The `scrape_interval` is assumed to be 15s by the Grafana Prometheus data source by default. If you decide to use a different `scrape_interval` value, you must change it in both the Prometheus configuration and the Grafana Prometheus data source configuration. Otherwise the Grafana `$__rate_interval` value will be calculated incorrectly and possibly cause a `no data` response in queries using it. * The `server_name` must be specified if the Incus server certificate does not contain the same host name as used in the `targets` list. To verify this, open `server.crt` and check the Subject Alternative Name (SAN) section. For example, assume that `server.crt` has the following content: ```{terminal} :input: openssl x509 -noout -text -in /etc/prometheus/tls/server.crt ... X509v3 Subject Alternative Name: DNS:foo, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1 ... ``` Since the Subject Alternative Name (SAN) list doesn't include the host name provided in the `targets` list (`foo.example.com`), you must override the name used for comparison using the `server_name` directive. ```` Here is an example of a `prometheus.yml` configuration where multiple jobs are used to scrape the metrics of multiple Incus servers: ```yaml global: # How frequently to scrape targets by default. The Prometheus default value is 1m. scrape_interval: 15s scrape_configs: # abydos, langara and orilla are part of a single cluster (called `hdc` here) # initially bootstrapped by abydos which is why all 3 targets # share the same `ca_file` and `server_name`. That `ca_file` corresponds # to the `/var/lib/incus/cluster.crt` file found on every member of # the Incus cluster. # # Note: When the `project` param is provided, it overrides the `default` project # and if not specified will pull all projects. # # Note: each member of the cluster only provide metrics for instances it runs locally # this is why the `incus-hdc` cluster lists 3 targets - job_name: "incus-hdc" metrics_path: '/1.0/metrics' params: project: ['jdoe'] scheme: 'https' static_configs: - targets: - 'abydos.hosts.example.net:8444' - 'langara.hosts.example.net:8444' - 'orilla.hosts.example.net:8444' tls_config: ca_file: 'tls/abydos.crt' cert_file: 'tls/metrics.crt' key_file: 'tls/metrics.key' server_name: 'abydos' # jupiter, mars and saturn are 3 standalone Incus servers. # Note: only the `default` project is used on them, so it is not specified. - job_name: "incus-jupiter" metrics_path: '/1.0/metrics' scheme: 'https' static_configs: - targets: ['jupiter.example.com:9101'] tls_config: ca_file: 'tls/jupiter.crt' cert_file: 'tls/metrics.crt' key_file: 'tls/metrics.key' server_name: 'jupiter' - job_name: "incus-mars" metrics_path: '/1.0/metrics' scheme: 'https' static_configs: - targets: ['mars.example.com:9101'] tls_config: ca_file: 'tls/mars.crt' cert_file: 'tls/metrics.crt' key_file: 'tls/metrics.key' server_name: 'mars' - job_name: "incus-saturn" metrics_path: '/1.0/metrics' scheme: 'https' static_configs: - targets: ['saturn.example.com:9101'] tls_config: ca_file: 'tls/saturn.crt' cert_file: 'tls/metrics.crt' key_file: 'tls/metrics.key' server_name: 'saturn' ``` After editing the configuration, restart Prometheus (for example, `systemctl restart prometheus`) to start scraping. ## Set up a Grafana dashboard To visualize the metrics data, set up [Grafana](https://grafana.com/). Incus provides a [Grafana dashboard](https://grafana.com/grafana/dashboards/19727-incus/) that is configured to display the Incus metrics scraped by Prometheus and log entries from Loki. ```{note} The dashboard requires Grafana 8.4 or later with both Prometheus and Loki configured as data sources in Grafana. It's possible to add a placeholder Loki data source in Grafana to make it possible to import the dashboard in an environment that doesn't have a fully set up Loki server. After import, remove the log sections at the bottom of the dashboard and then remove the placeholder Loki data source. Incus logging to Loki is configured through the [relevant server configuration](server-options-logging). ``` See the Grafana documentation for instructions on installing and signing in: - [Install Grafana](https://grafana.com/docs/grafana/latest/setup-grafana/installation/) - [Sign in to Grafana](https://grafana.com/docs/grafana/latest/setup-grafana/sign-in-to-grafana/) Complete the following steps to import the [Incus dashboard](https://grafana.com/grafana/dashboards/19727-incus/): 1. Configure Prometheus as a data source: 1. Go to {guilabel}`Configuration` > {guilabel}`Data sources`. 1. Click {guilabel}`Add data source`. ![Add data source in Grafana](images/grafana_add_datasource.png) 1. Select {guilabel}`Prometheus`. ![Select Prometheus as the data source](images/grafana_select_prometheus.png) 1. In the {guilabel}`URL` field, enter `http://localhost:9090/` if running Prometheus locally. ![Enter Prometheus URL](images/grafana_configure_datasource.png) 1. Keep the default configuration for the other fields and click {guilabel}`Save & test`. 1. Configure [Loki](https://grafana.com/oss/loki/) as a data source: 1. Go to {guilabel}`Configuration` > {guilabel}`Data sources`. 1. Click {guilabel}`Add data source`. 1. Select {guilabel}`Loki`. 1. In the {guilabel}`URL` field, enter `http://localhost:3100/` if running Loki locally. 1. Keep the default configuration for the other fields and click {guilabel}`Save & test`. 1. Import the Incus dashboard: 1. Go to {guilabel}`Dashboards` > {guilabel}`Browse`. 1. Click {guilabel}`New` and select {guilabel}`Import`. ![Import a dashboard in Grafana](images/grafana_dashboard_import.png) 1. In the {guilabel}`Import via grafana.com` field, enter the dashboard ID `19727`. ![Enter the Incus dashboard ID](images/grafana_dashboard_id.png) 1. Click {guilabel}`Load`. 1. In the {guilabel}`Incus` drop-down menu, select the Prometheus and Loki data sources that you configured. ![Select the Prometheus data source](images/grafana_dashboard_select_datasource.png) 1. Click {guilabel}`Import`. You should now see the Incus dashboard. You can select the project and filter by instances. ![Resource overview in the Incus Grafana dashboard](images/grafana_resources.png) At the bottom of the page, you can see data for each instance. ![Instance data in the Incus Grafana dashboard](images/grafana_instances.png) ```{note} For proper operation of the Loki part of the dashboard, you need to ensure that the `instance` field matches the Prometheus job name. You can change the `instance` field through the `logging.*.target.instance` configuration key. ``` incus-7.0.0/doc/migration.md000066400000000000000000000030641517523235500157320ustar00rootroot00000000000000(migration)= # Migration Incus provides tools and functionality to migrate instances in different contexts. Migrate existing Incus instances between servers : The most basic kind of migration is if you have an Incus instance on one server and want to move it to a different Incus server. For virtual machines, you can do that as a live migration, which means that you can migrate your VM while it is running and there will be no downtime. See {ref}`move-instances` for more information. Migrate physical or virtual machines to Incus instances : If you have an existing machine, either physical or virtual (VM or container), you can use the `incus-migrate` tool to create an Incus instance based on your existing machine. The tool copies the provided partition, disk or image to the Incus storage pool of the provided Incus server, sets up an instance using that storage and allows you to configure additional settings for the new instance. See {ref}`import-machines-to-instances` for more information. Migrate instances from LXC to Incus : If you are using LXC and want to migrate all or some of your LXC containers to an Incus installation on the same machine, you can use the `lxc-to-incus` tool. The tool analyzes the LXC configuration and copies the data and configuration of your existing LXC containers into new Incus containers. See {ref}`migrate-from-lxc` for more information. ```{toctree} :maxdepth: 1 :hidden: Move instances Import existing machines Migrate from LXC ``` incus-7.0.0/doc/networks.md000066400000000000000000000012551517523235500156150ustar00rootroot00000000000000(networking)= # Networking ```{toctree} :maxdepth: 1 /explanation/networks Create and configure a network Configure a network Configure network ACLs Configure network address sets Configure network forwards Configure network integrations Configure network zones Configure Incus as BGP server Display Incus IPAM information /reference/network_bridge /reference/network_ovn /reference/network_external Increase bandwidth ``` incus-7.0.0/doc/packaging.md000066400000000000000000000074661517523235500156770ustar00rootroot00000000000000# Packaging recommendations Below are a few recommendations for packagers of Incus. Following those recommendations should provide a more predictable experience across Linux distributions. ## Packages It's usually a good idea to at least split things into an `incus` and `incus-client` package. The latter allows for installing just the `incus` command line tool without bringing the daemon and its dependencies. Additionally, it may be useful to have an `incus-tools` package with some of the less commonly used tools like `fuidshift`, `lxc-to-incus`, `incus-benchmark` and `incus-migrate`. ## Groups Two groups should be provided: - `incus-admin` which grants access to the `unix.socket` socket and effectively grants full control over Incus. - `incus` which grants access to the `user.socket` socket which provides users with a restricted Incus project. ## Init scripts The following assumes the use of `systemd`. Distributions not using `systemd` should try to stick to a similar naming scheme but will likely see some differences on things like socket activation. - `incus.service` is the main unit that starts and stops the `incusd` daemon. - `incus.socket` is the socket-activation unit for the `incus.service` unit. If present, `incus.service` should not be made to start on its own. - `incus-user.service` is the unit responsible for starting and stopping the `incus-user` daemon. - `incus-user.socket` is the socket-activation unit for the `incus-user.service` unit. If present, `incus-user.service` should not be made to start on its own. - `incus-startup.service` uses the `incusd activateifneeded` command to trigger daemon startup if it is required. It also calls `incusd shutdown` to handle orderly shutdown of instances on host shutdown. ## Binaries The `incusd` and `incus-user` daemons should be kept outside of the user's `PATH`. The same is true of `incus-agent` which needs to be available in the daemon's `PATH` but not be visible to users. The main binary that should be made visible to users is `incus`. On top of those, the following optional binaries may also be made available: - `fuidshift` (should be kept to root only) - `incus-benchmark` - `incus-migrate` - `lxc-to-incus` - `lxd-to-incus` (should be kept to root only) ## Incus agent binaries There are two ways to provide the `incus-agent` binary. ### Single agent setup The simplest way is to have `incus-agent` be available in the `PATH` of `incusd`. In this scenario the agent should be a static build of `incus-agent` for the primary architecture of the system. ### Multiple agent setup Alternatively, it's possible to provide multiple builds of the `incus-agent` binary, offering support for multiple architectures or operating systems. To do that, the `INCUS_AGENT_PATH` environment variable should be set for the `incusd` process and point to a path that includes the `incus-agent` builds. Those builds should be named after the operating system name and architecture. For example `incus-agent.linux.x86_64`, `incus-agent.linux.i686` or `incus-agent.linux.aarch64`. ## Documentation ### Web documentation Incus can serve its own documentation when the network listener is enabled (`core.https_address`). For that to work, the documentation provided in the release tarball should be shipped as part of the package and its path be passed to Incus through the `INCUS_DOCUMENTATION` environment variable. ### Manual pages While we don't specifically write full `manpage` entries for Incus, it is possible to generate those from the CLI. Running `incus manpage --all --format=man /target/path` will generate a separate page for each command/sub-command. This is effectively the same as what's otherwise made available through `--help`, so unless a distribution packaging policy requires all binaries have `manpages`, it's usually best to rely on `--help` and `help` sub-commands. incus-7.0.0/doc/profiles.md000066400000000000000000000107051517523235500155640ustar00rootroot00000000000000(profiles)= # How to use profiles Profiles store a set of configuration options. They can contain instance options, devices and device options. You can apply any number of profiles to an instance. They are applied in the order they are specified, so the last profile to specify a specific key takes precedence. However, instance-specific configuration always overrides the configuration coming from the profiles. ```{note} Profiles can be applied to containers and virtual machines. Therefore, they might contain options and devices that are valid for either type. When applying a profile that contains configuration that is not suitable for the instance type, this configuration is ignored and does not result in an error. ``` If you don't specify any profiles when launching a new instance, the `default` profile is applied automatically. This profile defines a network interface and a root disk. The `default` profile cannot be renamed or removed. ## View profiles Enter the following command to display a list of all available profiles: incus profile list Enter the following command to display the contents of a profile: incus profile show ## Create an empty profile Enter the following command to create an empty profile: incus profile create (profiles-edit)= ## Edit a profile You can either set specific configuration options for a profile or edit the full profile in YAML format. ### Set specific options for a profile To set an instance option for a profile, use the [`incus profile set`](incus_profile_set.md) command. Specify the profile name and the key and value of the instance option: incus profile set = = ... To add and configure an instance device for your profile, use the [`incus profile device add`](incus_profile_device_add.md) command. Specify the profile name, a device name, the device type and maybe device options (depending on the {ref}`device type `): incus profile device add = = ... To configure instance device options for a device that you have added to the profile earlier, use the [`incus profile device set`](incus_profile_device_set.md) command: incus profile device set = = ... ### Edit the full profile Instead of setting each configuration option separately, you can provide all options at once in YAML format. Check the contents of an existing profile or instance configuration for the required markup. For example, the `default` profile might look like this: config: {} description: Default Incus profile devices: eth0: name: eth0 network: incusbr0 type: nic root: path: / pool: default type: disk name: default used_by: Instance options are provided as an array under `config`. Instance devices and instance device options are provided under `devices`. To edit a profile using your standard terminal editor, enter the following command: incus profile edit Alternatively, you can create a YAML file (for example, `profile.yaml`) with the configuration and write the configuration to the profile with the following command: incus profile edit < profile.yaml ## Apply a profile to an instance Enter the following command to apply a profile to an instance: incus profile add ```{tip} Check the configuration after adding the profile: [`incus config show `](incus_config_show.md) You will see that your profile is now listed under `profiles`. However, the configuration options from the profile are not shown under `config` (unless you add the `--expanded` flag). The reason for this behavior is that these options are taken from the profile and not the configuration of the instance. This means that if you edit a profile, the changes are automatically applied to all instances that use the profile. ``` You can also specify profiles when launching an instance by adding the `--profile` flag: incus launch --profile --profile ... ## Remove a profile from an instance Enter the following command to remove a profile from an instance: incus profile remove incus-7.0.0/doc/projects.md000066400000000000000000000003731517523235500155720ustar00rootroot00000000000000(projects)= # Projects ```{toctree} :maxdepth: 1 explanation/projects Create and configure projects Work with different projects Confine projects to users reference/projects ``` incus-7.0.0/doc/reference/000077500000000000000000000000001517523235500153525ustar00rootroot00000000000000incus-7.0.0/doc/reference/cluster_member_config.md000066400000000000000000000010731517523235500222320ustar00rootroot00000000000000(cluster-member-config)= # Cluster member configuration Each cluster member has its own key/value configuration with the following supported namespaces: - `user` (free form key/value for user metadata) - `scheduler` (options related to how the member is automatically targeted by the cluster) The following keys are currently supported: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/devices.md000066400000000000000000000044421517523235500173220ustar00rootroot00000000000000(devices)= # Devices Devices are attached to an instance (see {ref}`instances-configure-devices`) or to a profile (see {ref}`profiles-edit`). They include, for example, network interfaces, mount points, USB and GPU devices. These devices can have instance device options, depending on the type of the instance device. Incus supports the following device types: | ID (database) | Name | Condition | Description | |:--------------|:---------------------------------------|:----------|:--------------------------------| | 0 | [`none`](devices-none) | - | Inheritance blocker | | 1 | [`nic`](devices-nic) | - | Network interface | | 2 | [`disk`](devices-disk) | - | Mount point inside the instance | | 3 | [`unix-char`](devices-unix-char) | container | Unix character device | | 4 | [`unix-block`](devices-unix-block) | container | Unix block device | | 5 | [`usb`](devices-usb) | - | USB device | | 6 | [`gpu`](devices-gpu) | - | GPU device | | 7 | [`infiniband`](devices-infiniband) | container | InfiniBand device | | 8 | [`proxy`](devices-proxy) | container | Proxy device | | 9 | [`unix-hotplug`](devices-unix-hotplug) | container | Unix hotplug device | | 10 | [`tpm`](devices-tpm) | - | TPM device | | 11 | [`pci`](devices-pci) | VM | PCI device | Each instance comes with a set of {ref}`standard-devices`. ```{toctree} :maxdepth: 1 :hidden: ../reference/standard_devices.md ../reference/devices_none.md ../reference/devices_nic.md ../reference/devices_disk.md ../reference/devices_unix_char.md ../reference/devices_unix_block.md ../reference/devices_usb.md ../reference/devices_gpu.md ../reference/devices_infiniband.md ../reference/devices_proxy.md ../reference/devices_unix_hotplug.md ../reference/devices_tpm.md ../reference/devices_pci.md ``` incus-7.0.0/doc/reference/devices_disk.md000066400000000000000000000173431517523235500203400ustar00rootroot00000000000000(devices-disk)= # Type: `disk` ```{note} The `disk` device type is supported for both containers and VMs. It supports hotplugging for both containers and VMs. ``` Disk devices supply additional storage to instances. For containers, they are essentially mount points inside the instance (either as a bind-mount of an existing file or directory on the host, or, if the source is a block device, a regular mount). Virtual machines share host-side mounts or directories through `9p` or `virtiofs` (if available), or as VirtIO disks for block-based disks. ```{warning} The device name affects the serial generated for the device. If the device name exceeds 14 characters for `nvme` and `virtio-blk`, or 30 characters for `virtio-scsi`, Incus will hash the device value to ensure the generated serial remains within supported length constraints. The device name itself is left unchanged. ``` (devices-disk-types)= ## Types of disk devices You can create disk devices from different sources. The value that you specify for the `source` option specifies the type of disk device that is added: Storage volume : The most common type of disk device is a storage volume. To add a storage volume, specify its name as the `source` of the device: incus config device add disk pool= source= [path=] The path is required for file system volumes, but not for block volumes. Alternatively, you can use the [`incus storage volume attach`](incus_storage_volume_attach.md) command to {ref}`storage-attach-volume`. Both commands use the same mechanism to add a storage volume as a disk device. It's possible to attach a sub-path of a custom volume to an instance using the `source=/` syntax. If the sub-path doesn't exist inside the custom volume, it will be created automatically when the device is started. When new directories are created this way, the {config:option}`device-disk-device-conf:initial.uid`, {config:option}`device-disk-device-conf:initial.gid` and {config:option}`device-disk-device-conf:initial.mode` options are used to set their ownership and permissions (defaulting to `0`, `0` and `0711` respectively). Path on the host : You can share a path on your host (either a file system or a block device) to your instance by adding it as a disk device with the host path as the `source`: incus config device add disk source= [path=] The path is required for file systems, but not for block devices. Ceph RBD : Incus can use Ceph to manage an internal file system for the instance, but if you have an existing, externally managed Ceph RBD that you would like to use for an instance, you can add it with the following command: incus config device add disk source=ceph:/ ceph.user_name= ceph.cluster_name= [path=] The path is required for file systems, but not for block devices. CephFS : Incus can use Ceph to manage an internal file system for the instance, but if you have an existing, externally managed Ceph file system that you would like to use for an instance, you can add it with the following command: incus config device add disk source=cephfs:/ ceph.user_name= ceph.cluster_name= path= ISO file : You can add an ISO file as a disk device for a virtual machine. It is added as a ROM device inside the VM. This source type is applicable only to VMs. To add an ISO file, specify its file path as the `source`: incus config device add disk source= VM `cloud-init` : You can generate a `cloud-init` configuration ISO from the {config:option}`instance-cloud-init:cloud-init.vendor-data` and {config:option}`instance-cloud-init:cloud-init.user-data` configuration keys and attach it to a virtual machine. The `cloud-init` that is running inside the VM then detects the drive on boot and applies the configuration. This source type is applicable only to VMs. To add such a device, use the following command: incus config device add disk source=cloud-init:config VM `agent` : You can generate an `agent` configuration ISO which will contain the agent binary, configuration files and installation scripts. This is required for environments where `9p` isn't supported and where an alternative way to load the agent is required. This source type is applicable only to VMs. To add such a device, use the following command: incus config device add disk source=agent:config Tmpfs : You can back a disk device with an in-memory file system by using the `tmpfs:` source. incus config device add disk source=tmpfs: path= [size=] [initial.uid=] [initial.gid=] [initial.mode=] Both `source` and `path` are required. This creates a `tmpfs` mount inside the instance and supports optional properties for size, ownership, and permissions. Tmpfs with overlayfs behavior : If you want the same tmpfs behavior but combined with overlayfs semantics, use `tmpfs-overlay:` as the source. incus config device add disk source=tmpfs-overlay: path= [size=] [initial.uid=] [initial.gid=] [initial.mode=] Both `source` and `path` are required. Additionally, the target `path` must already exist inside the container. This provides an ephemeral in-memory file system with overlayfs handling. (devices-disk-initial-config)= ## Initial volume configuration for instance root disk devices Initial volume configuration allows setting specific configurations for the root disk devices of new instances. These settings are prefixed with `initial.` and are only applied when the instance is created. This method allows creating instances that have unique configurations, independent of the default storage pool settings. For example, you can add an initial volume configuration for `zfs.block_mode` to an existing profile, and this will then take effect for each new instance you create using this profile: incus profile device set initial.zfs.block_mode=true You can also set an initial configuration directly when creating an instance. For example: incus init --device ,initial.zfs.block_mode=true Note that you cannot use initial volume configurations with custom volume options or to set the volume's size. (devices-disk-initial-uid-gid-mode)= ## `initial.uid`, `initial.gid` and `initial.mode` `initial.uid`, `initial.gid` and `initial.mode` apply in three different scenarios: - On root disk devices, they are passed to the storage driver to set the ownership and mode of the instance's root volume at creation time (when supported by the driver). - On `tmpfs:` and `tmpfs-overlay:` disk devices, they are translated into `uid=`, `gid=` and `mode=` mount options for the underlying `tmpfs` mount. - On custom volume disks where the `source` includes a sub-path (for example `source=myvol/sub/path`), they are used as the ownership and mode of any sub-directory that has to be created automatically when the device is started. In all cases, `initial.uid` and `initial.gid` default to `0` and `initial.mode` defaults to `0711` (octal). ## Device options `disk` devices have the following device options: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/devices_gpu.md000066400000000000000000000064101517523235500201720ustar00rootroot00000000000000(devices-gpu)= # Type: `gpu` GPU devices make the specified GPU device or devices appear in the instance. ```{note} For containers, a `gpu` device may match multiple GPUs at once. For VMs, each device can match only a single GPU. ``` The following types of GPUs can be added using the `gputype` device option: - [`physical`](gpu-physical) (container and VM): Passes an entire GPU through into the instance. This value is the default if `gputype` is unspecified. - [`mdev`](gpu-mdev) (VM only): Creates and passes a virtual GPU through into the instance. - [`mig`](gpu-mig) (container only): Creates and passes a MIG (Multi-Instance GPU) through into the instance. - [`sriov`](gpu-sriov) (VM only): Passes a virtual function of an SR-IOV-enabled GPU into the instance. The available device options depend on the GPU type and are listed in the tables in the following sections. (gpu-physical)= ## `gputype`: `physical` ```{note} The `physical` GPU type is supported for both containers and VMs. It supports hotplugging only for containers, not for VMs. ``` A `physical` GPU device passes an entire GPU through into the instance. ### Device options GPU devices of type `physical` have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (gpu-mdev)= ## `gputype`: `mdev` ```{note} The `mdev` GPU type is supported only for VMs. It does not support hotplugging. ``` An `mdev` GPU device creates and passes a virtual GPU through into the instance. You can check the list of available `mdev` profiles by running [`incus info --resources`](incus_info.md). ### Device options GPU devices of type `mdev` have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (gpu-mig)= ## `gputype`: `mig` ```{note} The `mig` GPU type is supported only for containers. It does not support hotplugging. ``` A `mig` GPU device creates and passes a MIG compute instance through into the instance. Currently, this requires NVIDIA MIG instances to be pre-created. ### Device options GPU devices of type `mig` have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` You must set either `mig.uuid` (NVIDIA drivers 470+) or both `mig.ci` and `mig.gi` (old NVIDIA drivers). (gpu-sriov)= ## `gputype`: `sriov` ```{note} The `sriov` GPU type is supported only for VMs. It does not support hotplugging. ``` An `sriov` GPU device passes a virtual function of an SR-IOV-enabled GPU into the instance. ### Device options GPU devices of type `sriov` have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/devices_infiniband.md000066400000000000000000000027061517523235500215040ustar00rootroot00000000000000(devices-infiniband)= # Type: `infiniband` ```{note} The `infiniband` device type is supported for both containers and VMs. It supports hotplugging only for containers, not for VMs. ``` Incus supports two different kinds of network types for InfiniBand devices: - `physical`: Passes a physical device from the host through to the instance. The targeted device will vanish from the host and appear in the instance. - `sriov`: Passes a virtual function of an SR-IOV-enabled physical network device into the instance. ```{note} InfiniBand devices support SR-IOV, but in contrast to other SR-IOV-enabled devices, InfiniBand does not support dynamic device creation in SR-IOV mode. Therefore, you must pre-configure the number of virtual functions by configuring the corresponding kernel module. ``` To create a `physical` `infiniband` device, use the following command: incus config device add infiniband nictype=physical parent= To create an `sriov` `infiniband` device, use the following command: incus config device add infiniband nictype=sriov parent= ## Device options `infiniband` devices have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/devices_nic.md000066400000000000000000000407661517523235500201640ustar00rootroot00000000000000(devices-nic)= # Type: `nic` ```{note} The `nic` device type is supported for both containers and VMs. NICs support hotplugging for both containers and VMs (with the exception of the `ipvlan` NIC type). ``` Network devices, also referred to as *Network Interface Controllers* or *NICs*, supply a connection to a network. Incus supports several different types of network devices (*NIC types*). ```{note} When using a USB network adapter with a VM, mainline QEMU will replace the leading two bytes of a MAC address with `40:`. Those affected by this may want to manually set the `hwaddr` property to a MAC address starting with `40:` to align the host and guest reporting of the MAC. ``` ## `nictype` vs. `network` When adding a network device to an instance, there are two methods to specify the type of device that you want to add: through the `nictype` device option or the `network` device option. These two device options are mutually exclusive, and you can specify only one of them when you create a device. However, note that when you specify the `network` option, the `nictype` option is derived automatically from the network type. `nictype` : When using the `nictype` device option, you can specify a network interface that is not controlled by Incus. Therefore, you must specify all information that Incus needs to use the network interface. When using this method, the `nictype` option must be specified when creating the device, and it cannot be changed later. `network` : When using the `network` device option, the NIC is linked to an existing {ref}`managed network `. In this case, Incus has all required information about the network, and you need to specify only the network name when adding the device. When using this method, Incus derives the `nictype` option automatically. The value is read-only and cannot be changed. Other device options that are inherited from the network are marked with a "yes" in the "Managed" column of the NIC-specific tables of device options. You cannot customize these options directly for the NIC if you're using the `network` method. See {ref}`networks` for more information. ## Available NIC types The following NICs can be added using the `nictype` or `network` options: - [`bridged`](nic-bridged): Uses an existing bridge on the host and creates a virtual device pair to connect the host bridge to the instance. - [`macvlan`](nic-macvlan): Sets up a new network device based on an existing one, but using a different MAC address. - [`sriov`](nic-sriov): Passes a virtual function of an SR-IOV-enabled physical network device into the instance. - [`physical`](nic-physical): Passes a physical device from the host through to the instance. The targeted device will vanish from the host and appear in the instance. The following NICs can be added using only the `network` option: - [`ovn`](nic-ovn): Uses an existing OVN network and creates a virtual device pair to connect the instance to it. The following NICs can be added using only the `nictype` option: - [`ipvlan`](nic-ipvlan): Sets up a new network device based on an existing one, using the same MAC address but a different IP. - [`p2p`](nic-p2p): Creates a virtual device pair, putting one side in the instance and leaving the other side on the host. - [`routed`](nic-routed): Creates a virtual device pair to connect the host to the instance and sets up static routes and proxy ARP/NDP entries to allow the instance to join the network of a designated parent interface. The available device options depend on the NIC type and are listed in the tables in the following sections. (nic-bridged)= ### `nictype`: `bridged` ```{note} You can select this NIC type through the `nictype` option or the `network` option (see {ref}`network-bridge` for information about the managed `bridge` network). ``` A `bridged` NIC uses an existing bridge on the host and creates a virtual device pair to connect the host bridge to the instance. #### Device options NIC devices of type `bridged` have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (nic-macvlan)= ### `nictype`: `macvlan` ```{note} You can select this NIC type through the `nictype` option or the `network` option (see {ref}`network-macvlan` for information about the managed `macvlan` network). ``` A `macvlan` NIC sets up a new network device based on an existing one, but using a different MAC address. If you are using a `macvlan` NIC, communication between the Incus host and the instances is not possible. Both the host and the instances can talk to the gateway, but they cannot communicate directly. #### Device options NIC devices of type `macvlan` have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (nic-sriov)= ### `nictype`: `sriov` ```{note} You can select this NIC type through the `nictype` option or the `network` option (see {ref}`network-sriov` for information about the managed `sriov` network). ``` An `sriov` NIC passes a virtual function of an SR-IOV-enabled physical network device into the instance. An SR-IOV-enabled network device associates a set of virtual functions (VFs) with the single physical function (PF) of the network device. PFs are standard PCIe functions. VFs, on the other hand, are very lightweight PCIe functions that are optimized for data movement. They come with a limited set of configuration capabilities to prevent changing properties of the PF. Given that VFs appear as regular PCIe devices to the system, they can be passed to instances just like a regular physical device. VF allocation : The `sriov` interface type expects to be passed the name of an SR-IOV enabled network device on the system via the `parent` property. Incus then checks for any available VFs on the system. By default, Incus allocates the first free VF it finds. If it detects that either none are enabled or all currently enabled VFs are in use, it bumps the number of supported VFs to the maximum value and uses the first free VF. If all possible VFs are in use or the kernel or card doesn't support incrementing the number of VFs, Incus returns an error. ```{note} If you need Incus to use a specific VF, use a `physical` NIC instead of a `sriov` NIC and set its `parent` option to the VF name. ``` #### Device options NIC devices of type `sriov` have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (nic-ovn)= ### `nictype`: `ovn` ```{note} You can select this NIC type only through the `network` option (see {ref}`network-ovn` for information about the managed `ovn` network). ``` An `ovn` NIC uses an existing OVN network and creates a virtual device pair to connect the instance to it. (devices-nic-hw-acceleration)= SR-IOV hardware acceleration : To use `acceleration=sriov`, you must have a compatible SR-IOV physical NIC that supports the Ethernet switch device driver model (`switchdev`) in your Incus host. Incus assumes that the physical NIC (PF) is configured in `switchdev` mode and connected to the OVN integration OVS bridge, and that it has one or more virtual functions (VFs) active. To achieve this, follow these basic prerequisite setup steps: 1. Set up PF and VF: 1. Activate some VFs on PF (called `enp9s0f0np0` in the following example, with a PCI address of `0000:09:00.0`) and unbind them. 1. Enable `switchdev` mode and `hw-tc-offload` on the PF. 1. Rebind the VFs. ``` echo 4 > /sys/bus/pci/devices/0000:09:00.0/sriov_numvfs for i in $(lspci -nnn | grep "Virtual Function" | cut -d' ' -f1); do echo 0000:$i > /sys/bus/pci/drivers/mlx5_core/unbind; done devlink dev eswitch set pci/0000:09:00.0 mode switchdev ethtool -K enp9s0f0np0 hw-tc-offload on for i in $(lspci -nnn | grep "Virtual Function" | cut -d' ' -f1); do echo 0000:$i > /sys/bus/pci/drivers/mlx5_core/bind; done ``` 1. Set up OVS by enabling hardware offload and adding the PF NIC to the integration bridge (normally called `br-int`): ``` ovs-vsctl set open_vswitch . other_config:hw-offload=true systemctl restart openvswitch-switch ovs-vsctl add-port br-int enp9s0f0np0 ip link set enp9s0f0np0 up ``` VDPA hardware acceleration : To use `acceleration=vdpa`, you must have a compatible VDPA physical NIC. The setup is the same as for SR-IOV hardware acceleration, except that you must also enable the `vhost_vdpa` module and check that you have some available VDPA management devices : ``` modprobe vhost_vdpa && vdpa mgmtdev show ``` #### Device options NIC devices of type `ovn` have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ```{note} Note that using `none` with either `ipv4.address` or `ipv6.address` needs the other protocol to also be disabled. There is currently no way for OVN to disable IP allocation just on IPv4 or IPv6. ``` (nic-physical)= ### `nictype`: `physical` ```{note} - You can select this NIC type through the `nictype` option or the `network` option (see {ref}`network-physical` for information about the managed `physical` network). - You can have only one `physical` NIC for each parent device. ``` A `physical` NIC provides straight physical device pass-through from the host. The targeted device will vanish from the host and appear in the instance (which means that you can have only one `physical` NIC for each targeted device). #### Device options NIC devices of type `physical` have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (nic-ipvlan)= ### `nictype`: `ipvlan` ```{note} - This NIC type is available only for containers, not for virtual machines. - You can select this NIC type only through the `nictype` option. - This NIC type does not support hotplugging. ``` An `ipvlan` NIC sets up a new network device based on an existing one, using the same MAC address but a different IP. If you are using an `ipvlan` NIC, communication between the Incus host and the instances is not possible. Both the host and the instances can talk to the gateway, but they cannot communicate directly. Incus currently supports IPVLAN in L2 and L3S mode. In this mode, the gateway is automatically set by Incus, but the IP addresses must be manually specified using the `ipv4.address` and/or `ipv6.address` options before the container is started. DNS : The name servers must be configured inside the container, because they are not set automatically. To do this, set the following `sysctls`: - When using IPv4 addresses: ``` net.ipv4.conf..forwarding=1 ``` - When using IPv6 addresses: ``` net.ipv6.conf..forwarding=1 net.ipv6.conf..proxy_ndp=1 ``` #### Device options NIC devices of type `ipvlan` have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (nic-p2p)= ### `nictype`: `p2p` ```{note} You can select this NIC type only through the `nictype` option. ``` A `p2p` NIC creates a virtual device pair, putting one side in the instance and leaving the other side on the host. #### Device options NIC devices of type `p2p` have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (nic-routed)= ### `nictype`: `routed` ```{note} You can select this NIC type only through the `nictype` option. ``` A `routed` NIC creates a virtual device pair to connect the host to the instance and sets up static routes and proxy ARP/NDP entries to allow the instance to join the network of a designated parent interface. For containers it uses a virtual Ethernet device pair, and for VMs it uses a TAP device. This NIC type is similar in operation to `ipvlan`, in that it allows an instance to join an external network without needing to configure a bridge and shares the host's MAC address. However, it differs from `ipvlan` because it does not need IPVLAN support in the kernel, and the host and the instance can communicate with each other. This NIC type respects `netfilter` rules on the host and uses the host's routing table to route packets, which can be useful if the host is connected to multiple networks. IP addresses, gateways and routes : You must manually specify the IP addresses (using `ipv4.address` and/or `ipv6.address`) before the instance is started. For containers, the NIC configures the following link-local gateway IPs on the host end and sets them as the default gateways in the container's NIC interface: 169.254.0.1 fe80::1 For VMs, the gateways must be configured manually or via a mechanism like `cloud-init` (see the {ref}`how to guide `). ```{note} If your container image is configured to perform DHCP on the interface, it will likely remove the automatically added configuration. In this case, you must configure the IP addresses and gateways manually or via a mechanism like `cloud-init`. ``` The NIC type configures static routes on the host pointing to the instance's `veth` interface for all of the instance's IPs. Multiple IP addresses : Each NIC device can have multiple IP addresses added to it. However, it might be preferable to use multiple `routed` NIC interfaces instead. In this case, set the `ipv4.gateway` and `ipv6.gateway` values to `none` on any subsequent interfaces to avoid default gateway conflicts. Also consider specifying a different host-side address for these subsequent interfaces using `ipv4.host_address` and/or `ipv6.host_address`. Parent interface : This NIC can operate with and without a `parent` network interface set. : With the `parent` network interface set, proxy ARP/NDP entries of the instance's IPs are added to the parent interface, which allows the instance to join the parent interface's network at layer 2. : To enable this, the following network configuration must be applied on the host via `sysctl`: - When using IPv4 addresses: ``` net.ipv4.conf..forwarding=1 ``` - When using IPv6 addresses: ``` net.ipv6.conf.all.forwarding=1 net.ipv6.conf..forwarding=1 net.ipv6.conf.all.proxy_ndp=1 net.ipv6.conf..proxy_ndp=1 ``` #### Device options NIC devices of type `routed` have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ## `bridged`, `macvlan` or `ipvlan` for connection to physical network The `bridged`, `macvlan` and `ipvlan` interface types can be used to connect to an existing physical network. `macvlan` effectively lets you fork your physical NIC, getting a second interface that is then used by the instance. This method saves you from creating a bridge device and virtual Ethernet device pairs and usually offers better performance than a bridge. The downside to this method is that `macvlan` devices, while able to communicate between themselves and to the outside, cannot talk to their parent device. This means that you can't use `macvlan` if you ever need your instances to talk to the host itself. In such case, a `bridge` device is preferable. A bridge also lets you use MAC filtering and I/O limits, which cannot be applied to a `macvlan` device. `ipvlan` is similar to `macvlan`, with the difference being that the forked device has IPs statically assigned to it and inherits the parent's MAC address on the network. incus-7.0.0/doc/reference/devices_none.md000066400000000000000000000010341517523235500203330ustar00rootroot00000000000000(devices-none)= # Type: `none` ```{note} The `none` device type is supported for both containers and VMs. ``` A `none` device doesn't have any properties and doesn't create anything inside the instance. Its only purpose is to stop inheriting devices that come from profiles. To do so, add a device with the same name as the one that you do not want to inherit, but with the device type `none`. You can add this device either in a profile that is applied after the profile that contains the original device, or directly on the instance. incus-7.0.0/doc/reference/devices_pci.md000066400000000000000000000015471517523235500201600ustar00rootroot00000000000000(devices-pci)= # Type: `pci` ```{note} The `pci` device type is supported for VMs. It does not support hotplugging. ``` PCI devices are used to pass raw PCI devices from the host into a virtual machine. They are mainly intended to be used for specialized single-function PCI cards like sound cards or video capture cards. In theory, you can also use them for more advanced PCI devices like GPUs or network cards, but it's usually more convenient to use the specific device types that Incus provides for these devices ([`gpu` device](devices-gpu) or [`nic` device](devices-nic)). ## Device options `pci` devices have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/devices_proxy.md000066400000000000000000000061601517523235500205620ustar00rootroot00000000000000(devices-proxy)= # Type: `proxy` ```{note} The `proxy` device type is supported for both containers (NAT and non-NAT modes) and VMs (NAT mode only). It supports hotplugging for both containers and VMs. ``` Proxy devices allow forwarding network connections between host and instance. This method makes it possible to forward traffic hitting one of the host's addresses to an address inside the instance, or to do the reverse and have an address in the instance connect through the host. In {ref}`devices-proxy-nat-mode`, a proxy device can be used for TCP and UDP proxying. In non-NAT mode, you can also proxy traffic between Unix sockets (which can be useful to, for example, forward graphical GUI or audio traffic from the container to the host system) or even across protocols (for example, you can have a TCP listener on the host system and forward its traffic to a Unix socket inside a container). The supported connection types are: - `tcp <-> tcp` - `udp <-> udp` - `unix <-> unix` - `tcp <-> unix` - `unix <-> tcp` - `udp <-> tcp` - `tcp <-> udp` - `udp <-> unix` - `unix <-> udp` To add a `proxy` device, use the following command: incus config device add proxy listen=::[-][,] connect=:: bind= (devices-proxy-nat-mode)= ## NAT mode The proxy device also supports a NAT mode (`nat=true`), where packets are forwarded using NAT rather than being proxied through a separate connection. This mode has the benefit that the client address is maintained without the need for the target destination to support the HAProxy PROXY protocol (which is the only way to pass the client address through when using the proxy device in non-NAT mode). However, NAT mode is supported only if the host that the instance is running on is the gateway (which is the case if you're using `incusbr0`, for example). In NAT mode, the supported connection types are: - `tcp <-> tcp` - `udp <-> udp` When configuring a proxy device with `nat=true`, you must ensure that the target instance has a static IP configured on its NIC device. ## Specifying IP addresses Use the following command to configure a static IP for an instance NIC: incus config device set ipv4.address= ipv6.address= To define a static IPv6 address, the parent managed network must have `ipv6.dhcp.stateful` enabled. When defining IPv6 addresses, use the square bracket notation, for example: connect=tcp:[2001:db8::1]:80 You can specify that the connect address should be the IP of the instance by setting the connect IP to the wildcard address (`0.0.0.0` for IPv4 and `[::]` for IPv6). ```{note} The listen address can also use wildcard addresses when using non-NAT mode. However, when using NAT mode, you must specify an IP address on the Incus host. ``` ## Device options `proxy` devices have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/devices_tpm.md000066400000000000000000000021701517523235500201760ustar00rootroot00000000000000(devices-tpm)= # Type: `tpm` ```{note} The `tpm` device type is supported for both containers and VMs. It supports hotplugging only for containers, not for VMs. ``` TPM devices enable access to a {abbr}`TPM (Trusted Platform Module)` emulator. TPM devices can be used to validate the boot process and ensure that no steps in the boot chain have been tampered with, and they can securely generate and store encryption keys. Incus uses a software TPM that supports TPM 2.0. For containers, the main use case is sealing certificates, which means that the keys are stored outside of the container, making it virtually impossible for attackers to retrieve them. For virtual machines, TPM can be used both for sealing certificates and for validating the boot process, which allows using full disk encryption compatible with, for example, Windows BitLocker. ## Device options `tpm` devices have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/devices_unix_block.md000066400000000000000000000017271517523235500215420ustar00rootroot00000000000000(devices-unix-block)= # Type: `unix-block` ```{note} The `unix-block` device type is supported for containers. It supports hotplugging. ``` Unix block devices make the specified block device appear as a device in the instance (under `/dev`). You can read from the device and write to it. ## Device options `unix-block` devices have the following device options: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (devices-unix-block-hotplugging)= ## Hotplugging Hotplugging is enabled if you set `required=false` and specify the `source` option for the device. In this case, the device is automatically passed into the container when it appears on the host, even after the container starts. If the device disappears from the host system, it is removed from the container as well. incus-7.0.0/doc/reference/devices_unix_char.md000066400000000000000000000014451517523235500213620ustar00rootroot00000000000000(devices-unix-char)= # Type: `unix-char` ```{note} The `unix-char` device type is supported for containers. It supports hotplugging. ``` Unix character devices make the specified character device appear as a device in the instance (under `/dev`). You can read from the device and write to it. ## Device options `unix-char` devices have the following device options: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (devices-unix-char-hotplugging)= ## Hotplugging % Include content from [devices_unix_block.md](device_unix_block.md) ```{include} devices_unix_block.md :start-after: Hotplugging ``` incus-7.0.0/doc/reference/devices_unix_hotplug.md000066400000000000000000000013161517523235500221240ustar00rootroot00000000000000(devices-unix-hotplug)= # Type: `unix-hotplug` ```{note} The `unix-hotplug` device type is supported for containers. It supports hotplugging. ``` Unix hotplug devices make the requested Unix device appear as a device in the instance (under `/dev`). If the device exists on the host system, you can read from it and write to it. The implementation depends on `systemd-udev` to be run on the host. ## Device options `unix-hotplug` devices have the following device options: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/devices_usb.md000066400000000000000000000020621517523235500201670ustar00rootroot00000000000000(devices-usb)= # Type: `usb` ```{note} The `usb` device type is supported for both containers and VMs. It supports hotplugging for both containers and VMs. ``` USB devices make the specified USB device appear in the instance. For performance issues, avoid using devices that require high throughput or low latency. For containers, only `libusb` devices (at `/dev/bus/usb`) are passed to the instance. This method works for devices that have user-space drivers. For devices that require dedicated kernel drivers, use a [`unix-char` device](devices-unix-char) or a [`unix-hotplug` device](devices-unix-hotplug) instead. For virtual machines, the entire USB device is passed through, so any USB device is supported. When a device is passed to the instance, it vanishes from the host. ## Device options `usb` devices have the following device options: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/image_format.md000066400000000000000000000142041517523235500203270ustar00rootroot00000000000000(image-format)= # Image format Images contain a root file system and a metadata file that describes the image. They can also contain templates for creating files inside an instance that uses the image. Images can be packaged as either a unified image (single file) or a split image (two files). ## Content Images for containers have the following directory structure: ``` metadata.yaml rootfs/ templates/ ``` Images for VMs have the following directory structure: ``` metadata.yaml rootfs.img templates/ ``` For both instance types, the `templates/` directory is optional. ### Metadata The `metadata.yaml` file contains information that is relevant to running the image in Incus. It includes the following information: ```yaml architecture: x86_64 creation_date: 1424284563 properties: description: Debian 12 Intel 64bit os: Debian release: bookworm 12 templates: ... ``` The `architecture` and `creation_date` fields are mandatory. The `properties` field contains a set of default properties for the image. The `os`, `release`, `name` and `description` fields are commonly used, but are not mandatory. The `templates` field is optional. See {ref}`image_format_templates` for information on how to configure templates. ### Root file system For containers, the `rootfs/` directory contains a full file system tree of the root directory (`/`) in the container. Virtual machines use a single `rootfs.img` file instead of a `rootfs/` directory, that file is expected to be `qcow2` formatted. (image_format_templates)= ### Templates (optional) You can use templates to dynamically create files inside an instance. To do so, configure template rules in the `metadata.yaml` file and place the template files in a `templates/` directory. As a general rule, you should never template a file that is owned by a package or is otherwise expected to be overwritten by normal operation of an instance. #### Template rules For each file that should be generated, create a rule in the `metadata.yaml` file. For example: ```yaml templates: /etc/hosts: when: - create - rename template: hosts.tpl properties: foo: bar /etc/hostname: when: - start template: hostname.tpl /etc/network/interfaces: when: - create template: interfaces.tpl create_only: true /home/foo/setup.sh: when: - create template: setup.sh.tpl create_only: true uid: 1000 gid: 1000 mode: 755 ``` The `when` key can be one or more of: - `create` - run at the time a new instance is created from the image - `copy` - run when an instance is created from an existing one - `start` - run every time the instance is started The `template` key points to the template file in the `templates/` directory. You can pass user-defined template properties to the template file through the `properties` key. Set the `create_only` key if you want Incus to create the file if it doesn't exist, but not overwrite an existing file. The `uid`, `gid` and `mode` keys can be used to control the file ownership and permissions. #### Template files Template files use the [Pongo2](https://pongo2.dev/) format. They always receive the following context: | Variable | Type | Description | |--------------|--------------------------------|-------------------------------------------------------------------------------------| | `trigger` | `string` | Name of the event that triggered the template | | `path` | `string` | Path of the file that uses the template | | `instance` | `map[string]string` | Key/value map of instance properties (name, architecture, privileged and ephemeral) | | `config` | `map[string]string` | Key/value map of the instance's configuration | | `devices` | `map[string]map[string]string` | Key/value map of the devices assigned to the instance | | `properties` | `map[string]string` | Key/value map of the template properties specified in `metadata.yaml` | For convenience, the following functions are exported to the Pongo2 templates: - `config_get("user.foo", "bar")` - Returns the value of `user.foo`, or `"bar"` if not set. ## Image tarballs Incus supports two Incus-specific image formats: a unified tarball and split tarballs. These tarballs can be compressed. Incus supports a wide variety of compression algorithms for tarballs. However, for compatibility purposes, you should use `gzip` or `xz`. (image-format-unified)= ### Unified tarball A unified tarball is a single tarball (usually `*.tar.xz`) that contains the full content of the image, including the metadata, the root file system and optionally the template files. This is the format that Incus itself uses internally when publishing images. It is usually easier to work with; therefore, you should use the unified format when creating Incus-specific images. The image identifier for such images is the SHA-256 of the tarball. (image-format-split)= ### Split tarballs A split image consists of two files. The first is a tarball containing the metadata and optionally the template files (usually `*.tar.xz`). The second is the root file system (container) or root disk (virtual machine). For containers, the second file is most commonly a SquashFS-formatted file system tree, though it can also be a tarball of the same tree. For virtual machines, the second file is always a `qcow2` formatted disk image. Tarballs can be externally compressed (`.tar.xz`, `.tar.gz`, ...) whereas `squashfs` and `qcow2` can be internally compressed through their respective native compression options. This format is designed to allow for easy image building from existing non-Incus rootfs tarballs that are already available. You should also use this format if you want to create images that can be consumed by both Incus and other tools. The image identifier for such images is the SHA-256 of the concatenation of the metadata and data files (in that order). incus-7.0.0/doc/reference/image_servers.md000066400000000000000000000051741517523235500205360ustar00rootroot00000000000000(image-servers)= # Default image server The [`incus`](incus.md) CLI command comes pre-configured with the following default remote image server: `images:` : This server provides unofficial images for a variety of Linux distributions. The images are maintained by the [Linux Containers](https://linuxcontainers.org/) team and are built to be compact and minimal. See [`images.linuxcontainers.org`](https://images.linuxcontainers.org) for an overview of available images. Additional image servers can be added through `incus remote add`. (image-server-types)= ## Image server types Incus supports the following types of remote image servers: Simple streams servers : Pure image servers that use the [simple streams format](https://git.launchpad.net/simplestreams/tree/). No special software is required to run such a server as it's only made of static files. The default `images:` server uses simplestreams. OCI registries : Application container registries that server OCI images. The most common such registry is the `Docker Hub` that can be added with `incus remote add docker https://docker.io --protocol=oci` Public Incus servers : Incus servers that are used solely to serve images and do not run instances themselves. To make an Incus server publicly available over the network on port 8443, set the {config:option}`server-core:core.https_address` configuration option to `:8443` and do not configure any authentication methods (see {ref}`server-expose` for more information). Then set the images that you want to share to `public`. Incus servers : Regular Incus servers that you can manage over a network, and that can also be used as image servers. For security reasons, you should restrict the access to the remote API and configure an authentication method to control access. See {ref}`server-expose` and {ref}`authentication` for more information. (image-server-tooling)= ## Tooling to manage a simplestreams server Incus includes a tool called `incus-simplestreams` which can be used to manage a file system tree using the Simple streams format. It supports importing either a container (`squashfs`) or virtual-machine (`qcow2`) image with `incus-simplestreams add`, list all images available as well as their fingerprints with `incus-simplestreams list` and remove images from the server with `incus-simplestreams remove`. That file system tree must then be placed on a regular web server which supports HTTPS with a valid certificate. When importing an image that doesn't come with an Incus metadata tarball, the `incus-simplestreams generate-metadata` command can be used to generate a new basic metadata tarball from a few questions. incus-7.0.0/doc/reference/instance_options.md000066400000000000000000000505521517523235500212620ustar00rootroot00000000000000(instance-options)= # Instance options Instance options are configuration options that are directly related to the instance. See {ref}`instances-configure-options` for instructions on how to set the instance options. The key/value configuration is namespaced. The following options are available: - {ref}`instance-options-misc` - {ref}`instance-options-boot` - [`cloud-init` configuration](instance-options-cloud-init) - {ref}`instance-options-limits` - {ref}`instance-options-migration` - {ref}`instance-options-nvidia` - {ref}`instance-options-oci` - {ref}`instance-options-raw` - {ref}`instance-options-security` - {ref}`instance-options-snapshots` - {ref}`instance-options-volatile` Note that while a type is defined for each option, all values are stored as strings and should be exported over the REST API as strings (which makes it possible to support any extra values without breaking backward compatibility). (instance-options-misc)= ## Miscellaneous options In addition to the configuration options listed in the following sections, these instance options are supported: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ```{config:option} environment.* instance-miscellaneous :type: "string" :liveupdate: "yes (exec)" :shortdesc: "Environment variables for the instance" You can export key/value environment variables to the instance. These are then set for [`incus exec`](incus_exec.md). ``` (instance-options-boot)= ## Boot-related options The following instance options control the boot-related behavior of the instance: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (instance-options-cloud-init)= ## `cloud-init` configuration The following instance options control the [`cloud-init`](cloud-init) configuration of the instance: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` Support for these options depends on the image that is used and is not guaranteed. If you specify both `cloud-init.user-data` and `cloud-init.vendor-data`, the content of both options is merged. Therefore, make sure that the `cloud-init` configuration you specify in those options does not contain the same keys. (instance-options-limits)= ## Resource limits The following instance options specify resource limits for the instance: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ```{config:option} limits.kernel.* instance-resource-limits :type: "string" :liveupdate: "no" :condition: "container" :shortdesc: "Kernel resources per instance" You can set kernel limits on an instance, for example, you can limit the number of open files. See {ref}`instance-options-limits-kernel` for more information. ``` ### Memory limits in virtual machines Incus supports both increasing and decreasing the memory allocation of virtual machines. Increasing the memory is done through memory hot plug, effectively adding virtual memory sticks to the VM. There is a limit of 16 virtual slots for this, limiting the number of memory increases that can be done without rebooting the VM. ```{note} To avoid high resource usage and compatibility issues with guests, Incus limits memory hotplug to a maximum of 1TiB in clustered environments. On standalone hosts, the default limit matches the total memory amount of the host system. Some CPUs further reduce that amount based on how much memory they are able to address (physical / virtual bits). Exceeding that limit require the instance be stopped, its `memory.limit` updated and then started back up. ``` Decreasing memory is not done through hot remove as that has a high risk of causing guest issues. Instead the memory balloon device is used, causing memory pressure inside the guest and causing memory to be released. This is a pretty slow process, so it is common for a memory reduction to fail to meet the requested value. When that happens, re-applying the lower value will trigger another attempt. As each attempt will cause the effective memory available to the guest to be reduced, it should eventually succeed and lead to the guest having the desired memory limit applied. ### CPU limits You have different options to limit CPU usage: - Set `limits.cpu` to restrict which CPUs the instance can see and use. See {ref}`instance-options-limits-cpu` for how to set this option. - Set `limits.cpu.allowance` to restrict the load an instance can put on the available CPUs. This option is available only for containers. See {ref}`instance-options-limits-cpu-container` for how to set this option. It is possible to set both options at the same time to restrict both which CPUs are visible to the instance and the allowed usage of those instances. However, if you use `limits.cpu.allowance` with a time limit, you should avoid using `limits.cpu` in addition, because that puts a lot of constraints on the scheduler and might lead to less efficient allocations. The CPU limits are implemented through a mix of the `cpuset` and `cpu` cgroup controllers. (instance-options-limits-cpu)= #### CPU pinning `limits.cpu` results in CPU pinning through the `cpuset` controller. You can specify either which CPUs or how many CPUs are visible and available to the instance: - To specify which CPUs to use, set `limits.cpu` to either a set of CPUs (for example, `1,2,3`) or a CPU range (for example, `0-3`). To pin to a single CPU, use the range syntax (for example, `1-1`) to differentiate it from a number of CPUs. - If you specify a number (for example, `4`) of CPUs, Incus will do dynamic load-balancing of all instances that aren't pinned to specific CPUs, trying to spread the load on the machine. Instances are re-balanced every time an instance starts or stops, as well as whenever a CPU is added to the system. ##### CPU limits for virtual machines ```{note} Incus supports live-updating the `limits.cpu` option. However, for virtual machines, this only means that the respective CPUs are hotplugged. Depending on the guest operating system, you might need to either restart the instance or complete some manual actions to bring the new CPUs online. ``` Incus virtual machines default to having just one vCPU allocated, which shows up as matching the host CPU vendor and type, but has a single core and no threads. When `limits.cpu` is set to a single integer, Incus allocates multiple vCPUs and exposes them to the guest as full cores. Those vCPUs are not pinned to specific physical cores on the host. The number of vCPUs can be updated while the VM is running. ```{note} To avoid high resource usage and compatibility issues with guests, Incus limits CPU hotplug to a maximum of 64 cores. VMs needing more than 64 CPU cores will need to be shut down to adjust their `limits.cpu` property. ``` When `limits.cpu` is set to a range or comma-separated list of CPU IDs (as provided by [`incus info --resources`](incus_info.md)), the vCPUs are pinned to those physical cores. In this scenario, Incus checks whether the CPU configuration lines up with a realistic hardware topology and if it does, it replicates that topology in the guest. When doing CPU pinning, it is not possible to change the configuration while the VM is running. For example, if the pinning configuration includes eight threads, with each pair of thread coming from the same core and an even number of cores spread across two CPUs, the guest will show two CPUs, each with two cores and each core with two threads. The NUMA layout is similarly replicated and in this scenario, the guest would most likely end up with two NUMA nodes, one for each CPU socket. In such an environment with multiple NUMA nodes, the memory is similarly divided across NUMA nodes and be pinned accordingly on the host and then exposed to the guest. All this allows for very high performance operations in the guest as the guest scheduler can properly reason about sockets, cores and threads as well as consider NUMA topology when sharing memory or moving processes across NUMA nodes. (instance-options-limits-cpu-container)= #### Allowance and priority (container only) `limits.cpu.allowance` drives either the CFS scheduler quotas when passed a time constraint, or the generic CPU shares mechanism when passed a percentage value: - The time constraint (for example, `20ms/50ms`) is a hard limit. For example, if you want to allow the container to use a maximum of one CPU, set `limits.cpu.allowance` to a value like `100ms/100ms`. The value is relative to one CPU worth of time, so to restrict to two CPUs worth of time, use something like `100ms/50ms` or `200ms/100ms`. - When using a percentage value, the limit is a soft limit that is applied only when under load. It is used to calculate the scheduler priority for the instance, relative to any other instance that is using the same CPU or CPUs. For example, to limit the CPU usage of the container to one CPU when under load, set `limits.cpu.allowance` to `100%`. `limits.cpu.priority` is another factor that is used to compute the scheduler priority score when a number of instances sharing a set of CPUs have the same percentage of CPU assigned to them. (instance-options-limits-hugepages)= ### Huge page limits Incus allows to limit the number of huge pages available to a container through the `limits.hugepage.[size]` key. Architectures often expose multiple huge-page sizes. The available huge-page sizes depend on the architecture. Setting limits for huge pages is especially useful when Incus is configured to intercept the `mount` syscall for the `hugetlbfs` file system in unprivileged containers. When Incus intercepts a `hugetlbfs` `mount` syscall, it mounts the `hugetlbfs` file system for a container with correct `uid` and `gid` values as mount options. This makes it possible to use huge pages from unprivileged containers. However, it is recommended to limit the number of huge pages available to the container through `limits.hugepages.[size]` to stop the container from being able to exhaust the huge pages available to the host. Limiting huge pages is done through the `hugetlb` cgroup controller, which means that the host system must expose the `hugetlb` controller for these limits to apply. (instance-options-limits-kernel)= ### Kernel resource limits For container instances, Incus exposes a generic namespaced key `limits.kernel.*` that can be used to set resource limits. It is generic in the sense that Incus does not perform any validation on the resource that is specified following the `limits.kernel.*` prefix. Incus cannot know about all the possible resources that a given kernel supports. Instead, Incus simply passes down the corresponding resource key after the `limits.kernel.*` prefix and its value to the kernel. The kernel does the appropriate validation. This allows users to specify any supported limit on their system. Some common limits are: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` A full list of all available limits can be found in the manpages for the `getrlimit(2)`/`setrlimit(2)` system calls. To specify a limit within the `limits.kernel.*` namespace, use the resource name in lowercase without the `RLIMIT_` prefix. For example, `RLIMIT_NOFILE` should be specified as `nofile`. A limit is specified as two colon-separated values that are either numeric or the word `unlimited` (for example, `limits.kernel.nofile=1000:2000`). A single value can be used as a shortcut to set both soft and hard limit to the same value (for example, `limits.kernel.nofile=3000`). A resource with no explicitly configured limit will inherit its limit from the process that starts up the container. Note that this inheritance is not enforced by Incus but by the kernel. (instance-options-migration)= ## Migration options The following instance options control the behavior if the instance is {ref}`moved from one Incus server to another `: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (instance-options-nvidia)= ## NVIDIA and CUDA configuration The following instance options specify the NVIDIA and CUDA configuration of the instance: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (instance-options-oci)= ## OCI configuration The following instance options specify the OCI configuration of the instance: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (instance-options-raw)= ## Raw instance configuration overrides The following instance options allow direct interaction with the backend features that Incus itself uses: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ```{important} Setting these `raw.*` keys might break Incus in non-obvious ways. Therefore, you should avoid setting any of these keys. ``` (instance-options-qemu)= ### Override QEMU configuration For VM instances, Incus configures QEMU through a configuration file that is passed to QEMU with the `-readconfig` command-line option. This configuration file is generated for each instance before boot. It can be found at `/run/incus//qemu.conf`. The default configuration works fine for Incus' most common use case: modern UEFI guests with VirtIO devices. In some situations, however, you might need to override the generated configuration. For example: - To run an old guest OS that doesn't support UEFI. - To specify custom virtual devices when VirtIO is not supported by the guest OS. - To add devices that are not supported by Incus before the machines boots. - To remove devices that conflict with the guest OS. To override the configuration, set the `raw.qemu.conf` option. It supports a format similar to `qemu.conf`, with some additions. Since it is a multi-line configuration option, you can use it to modify multiple sections or keys. - To replace a section or key in the generated configuration file, add a section with a different value. For example, use the following section to override the default `virtio-gpu-pci` GPU driver: ``` raw.qemu.conf: |- [device "qemu_gpu"] driver = "qxl-vga" ``` - To remove a section, specify a section without any keys. For example: ``` raw.qemu.conf: |- [device "qemu_gpu"] ``` - To remove a key, specify an empty string as the value. For example: ``` raw.qemu.conf: |- [device "qemu_gpu"] driver = "" ``` - To add a new section, specify a section name that is not present in the configuration file. The configuration file format used by QEMU allows multiple sections with the same name. Here's a piece of the configuration generated by Incus: ``` [global] driver = "ICH9-LPC" property = "disable_s3" value = "1" [global] driver = "ICH9-LPC" property = "disable_s4" value = "0" ``` The first `global` section disabled S3(Suspend to RAM), the second `global` section enabled S4(suspend to disk). In order to disable S4, the second `global` section index needs to be specified: ``` raw.qemu.conf: |- [global][1] value = "1" ``` Section indexes start at 0 (which is the default value when not specified), so the above example would generate the following configuration: ``` [global] driver = "ICH9-LPC" property = "disable_s3" value = "1" [global] driver = "ICH9-LPC" property = "disable_s4" value = "1" ``` ### Override QEMU runtime objects While `raw.qemu` and `raw.qemu.conf` can be used to alter the arguments and configuration file that's passed to QEMU, a lot of devices are now added through QMP instead. This is used by Incus for any device which may need to be re-configured at runtime, effectively anything that can be hot-plugged. Those devices cannot be overridden through the configuration or the command line, but instead additional configuration keys are available to run QMP commands directly. Fixed commands can be provided through the `raw.qemu.early`, `raw.qemu.pre-start` and `raw.qemu.post-start` configuration keys. Those take a JSON encoded list of QMP commands to run. The hooks correspond to: - `early`, run prior to any device having been added by Incus through QMP, after QEMU has started - `pre-start`, run following Incus having added all its devices but prior to the VM being started - `post-start`, run immediately following the VM starting up ### Advanced use For anyone needing dynamic QMP interactions, for example to retrieve the current value of some objects before modifying or generating new objects, it's also possible to attach to those same hooks using a scriptlet. This is done through `raw.qemu.scriptlet`. The scriptlet must define the `qemu_hook(instance, stage)` function. The `instance` arguments is an object representing the VM, whose attributes are those of the `api.Instance` struct. The `stage` argument is the name of the hook (`config`, `early`, `pre-start` or `post-start`), with `config` being run before starting QEMU, and the other hooks defined above. The following commands are exposed to that scriptlet: - `log_info` will log an `INFO` message - `log_warn` will log a `WARNING` message - `log_error` will log an `ERROR` message - `run_qmp` will run an arbitrary QMP command (JSON) and return its output - `run_command` will run the specified command with an optional list of arguments and return its output - `get_qemu_cmdline` will return the list of command-line arguments passed to QEMU - `set_qemu_cmdline` will set them - `get_qemu_conf` will return the QEMU configuration file as a dictionary - `set_qemu_conf` will set it from a dictionary Additionally the following alias commands (internally use `run_command`) are also available to simplify scripts: - `blockdev_add` - `blockdev_del` - `chardev_add` - `chardev_change` - `chardev_remove` - `device_add` - `device_del` - `netdev_add` - `netdev_del` - `object_add` - `object_del` - `qom_get` - `qom_list` - `qom_set` The functions allowing to change QEMU configuration can only be run during the `config` hook. In parallel, the functions running QMP commands cannot be run during the `config` hook. (instance-options-security)= ## Security policies The following instance options control the {ref}`security` policies of the instance: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (instance-options-snapshots)= ## Snapshot scheduling and configuration The following instance options control the creation and expiry of {ref}`instance snapshots `: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (instance-options-snapshots-names)= ### Automatic snapshot names {{snapshot_pattern_detail}} (instance-options-volatile)= ## Volatile internal data The following volatile keys are currently used internally by Incus to store internal data specific to an instance: % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: A network bridge creates a virtual L2 Ethernet switch that instance NICs can connect to, making it possible for them to communicate with each other and the host. Incus bridges can leverage underlying native Linux bridges and Open vSwitch. The `bridge` network type allows to create an L2 bridge that connects the instances that use it together into a single network L2 segment. Bridges created by Incus are managed, which means that in addition to creating the bridge interface itself, Incus also sets up a local `dnsmasq` process to provide DHCP, IPv6 route announcements and DNS services to the network. By default, it also performs NAT for the bridge. See {ref}`network-bridge-firewall` for instructions on how to configure your firewall to work with Incus bridge networks. ```{note} Static DHCP assignments depend on the client using its MAC address as the DHCP identifier. This method prevents conflicting leases when copying an instance, and thus makes statically assigned leases work properly. ``` ## IPv6 prefix size If you're using IPv6 for your bridge network, you should use a prefix size of 64. Larger subnets (i.e., using a prefix smaller than 64) should work properly too, but they aren't typically that useful for {abbr}`SLAAC (Stateless Address Auto-configuration)`. Smaller subnets are in theory possible (when using stateful DHCPv6 for IPv6 allocation), but they aren't properly supported by `dnsmasq` and might cause problems. If you must create a smaller subnet, use static allocation or another standalone router advertisement daemon. (network-bridge-options)= ## Configuration options The following configuration key namespaces are currently supported for the `bridge` network type: - `bgp` (BGP peer configuration) - `bridge` (L2 interface configuration) - `dns` (DNS server and resolution configuration) - `ipv4` (L3 IPv4 configuration) - `ipv6` (L3 IPv6 configuration) - `security` (network ACL configuration) - `raw` (raw configuration file content) - `tunnel` (cross-host tunneling configuration) - `user` (free-form key/value for user metadata) ```{note} {{note_ip_addresses_CIDR}} ``` The following configuration options are available for the `bridge` network type: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ## BGP options These options configure BGP peering for OVN downstream networks: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ```{note} The `bridge.external_interfaces` option supports an extended format allowing the creation of missing VLAN interfaces. The extended format is `//`. When the external interface is added to the list with the extended format, the system will automatically create the interface upon the network's creation and subsequently delete it when the network is terminated. The system verifies that the `` does not already exist. If the interface name is in use with a different parent or VLAN ID, or if the creation of the interface is unsuccessful, the system will revert with an error message. ``` (network-bridge-features)= ## Supported features The following features are supported for the `bridge` network type: - {ref}`network-acls` - {ref}`network-forwards` - {ref}`network-zones` - {ref}`network-bgp` - [How to integrate with `systemd-resolved`](network-bridge-resolved) ```{toctree} :maxdepth: 1 :hidden: Integrate with resolved Configure your firewall ``` incus-7.0.0/doc/reference/network_external.md000066400000000000000000000013131517523235500212650ustar00rootroot00000000000000(network-external)= # External networks External networks use network interfaces that already exist. Therefore, Incus has limited possibility to control them, and Incus features like network ACLs, network forwards and network zones are not supported. The main purpose for using external networks is to provide an uplink network through a parent interface. This external network specifies the presets to use when connecting instances or other networks to a parent interface. Incus supports the following external network types: ```{toctree} :maxdepth: 1 /reference/network_macvlan /reference/network_sriov /reference/network_physical ``` incus-7.0.0/doc/reference/network_macvlan.md000066400000000000000000000027311517523235500210710ustar00rootroot00000000000000(network-macvlan)= # Macvlan network Macvlan is a virtual {abbr}`LAN (Local Area Network)` that you can use if you want to assign several IP addresses to the same network interface, basically splitting up the network interface into several sub-interfaces with their own IP addresses. You can then assign IP addresses based on the randomly generated MAC addresses. The `macvlan` network type allows to specify presets to use when connecting instances to a parent interface. In this case, the instance NICs can simply set the `network` option to the network they connect to without knowing any of the underlying configuration details. ```{note} If you are using a `macvlan` network, communication between the Incus host and the instances is not possible. Both the host and the instances can talk to the gateway, but they cannot communicate directly. ``` (network-macvlan-options)= ## Configuration options The following configuration key namespaces are currently supported for the `macvlan` network type: - `user` (free-form key/value for user metadata) ```{note} {{note_ip_addresses_CIDR}} ``` The following configuration options are available for the `macvlan` network type: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/network_ovn.md000066400000000000000000000057171517523235500202610ustar00rootroot00000000000000(network-ovn)= # OVN network {abbr}`OVN (Open Virtual Network)` is a software-defined networking system that supports virtual network abstraction. You can use it to build your own private cloud. See [`www.ovn.org`](https://www.ovn.org/) for more information. The `ovn` network type allows to create logical networks using the OVN {abbr}`SDN (software-defined networking)`. This kind of network can be useful for labs and multi-tenant environments where the same logical subnets are used in multiple discrete networks. An Incus OVN network can be connected to an existing managed {ref}`network-bridge` or {ref}`network-physical` to gain access to the wider network. By default, all connections from the OVN logical networks are NATed to an IP allocated from the uplink network. See {ref}`network-ovn-setup` for basic instructions for setting up an OVN network. % Include content from [network_bridge.md](network_bridge.md) ```{include} network_bridge.md :start-after: :end-before: ``` (network-ovn-options)= ## Configuration options The following configuration key namespaces are currently supported for the `ovn` network type: - `bridge` (L2 interface configuration) - `dns` (DNS server and resolution configuration) - `ipv4` (L3 IPv4 configuration) - `ipv6` (L3 IPv6 configuration) - `security` (network ACL configuration) - `user` (free-form key/value for user metadata) ```{note} {{note_ip_addresses_CIDR}} ``` The following configuration options are available for the `ovn` network type: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ```{note} The `bridge.external_interfaces` option supports an extended format allowing the creation of missing VLAN interfaces. The extended format is `//`. When the external interface is added to the list with the extended format, the system will automatically create the interface upon the network's creation and subsequently delete it when the network is terminated. The system verifies that the `` does not already exist. If the interface name is in use with a different parent or VLAN ID, or if the creation of the interface is unsuccessful, the system will revert with an error message. ``` (network-ovn-features)= ## Supported features The following features are supported for the `ovn` network type: - {ref}`network-acls` - {ref}`network-forwards` - {ref}`network-integrations` - {ref}`network-zones` - {ref}`network-ovn-peers` - {ref}`network-load-balancers` ```{toctree} :maxdepth: 1 :hidden: Set up OVN Create routing relationships Configure network load balancers ``` incus-7.0.0/doc/reference/network_physical.md000066400000000000000000000062211517523235500212620ustar00rootroot00000000000000(network-physical)= # Physical network The `physical` network type connects to an existing physical network, which can be a network interface or a bridge, and serves as an uplink network for OVN. This network type allows to specify presets to use when connecting OVN networks to a parent interface or to allow an instance to use a physical interface as a NIC. In this case, the instance NICs can simply set the `network`option to the network they connect to without knowing any of the underlying configuration details. (network-physical-options)= ## Configuration options The following configuration key namespaces are currently supported for the `physical` network type: - `bgp` (BGP peer configuration) - `dns` (DNS server and resolution configuration) - `ipv4` (L3 IPv4 configuration) - `ipv6` (L3 IPv6 configuration) - `ovn` (OVN configuration) - `user` (free-form key/value for user metadata) ```{note} {{note_ip_addresses_CIDR}} ``` The following configuration options are available for the `physical` network type: ## BGP options These options configure BGP peering for OVN downstream networks: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ## DNS options These keys control the DNS servers and search domains used by the physical network: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ## IPV4 options These options define the IPv4 configuration for the physical network: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ## IPV6 options These options define the IPv6 configuration for the physical network: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ## OVN options These options apply when using a physical network as an OVN uplink: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ## Common options These apply to all physical networks regardless of other features: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (network-physical-features)= ## Supported features The following features are supported for the `physical` network type: - {ref}`network-bgp` incus-7.0.0/doc/reference/network_sriov.md000066400000000000000000000021411517523235500206050ustar00rootroot00000000000000(network-sriov)= # SR-IOV network {abbr}`SR-IOV (Single root I/O virtualization)` is a hardware standard that allows a single network card port to appear as several virtual network interfaces in a virtualized environment. The `sriov` network type allows to specify presets to use when connecting instances to a parent interface. In this case, the instance NICs can simply set the `network` option to the network they connect to without knowing any of the underlying configuration details. (network-sriov-options)= ## Configuration options The following configuration key namespaces are currently supported for the `sriov` network type: - `user` (free-form key/value for user metadata) ```{note} {{note_ip_addresses_CIDR}} ``` The following configuration options are available for the `sriov` network type: % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/projects.md000066400000000000000000000131541517523235500175310ustar00rootroot00000000000000(ref-projects)= # Project configuration Projects can be configured through a set of key/value configuration options. See {ref}`projects-configure` for instructions on how to set these options. The key/value configuration is namespaced. The following options are available: - {ref}`project-features` - {ref}`project-limits` - {ref}`project-restrictions` - {ref}`project-specific-config` (project-features)= ## Project features The project features define which entities are isolated in the project and which are inherited from the `default` project. If a `feature.*` option is set to `true`, the corresponding entity is isolated in the project. ```{note} When you create a project without explicitly configuring a specific option, this option is set to the initial value given in the following table. However, if you unset one of the `feature.*` options, it does not go back to the initial value, but to the default value. The default value for all `feature.*` options is `false`. ``` % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (project-limits)= ## Project limits Project limits define a hard upper bound for the resources that can be used by the containers and VMs that belong to a project. Depending on the `limits.*` option, the limit applies to the number of entities that are allowed in the project (for example, {config:option}`project-limits:limits.containers` or {config:option}`project-limits:limits.networks`) or to the aggregate value of resource usage for all instances in the project (for example, {config:option}`project-limits:limits.cpu` or {config:option}`project-limits:limits.processes`). In the latter case, the limit usually applies to the {ref}`instance-options-limits` that are configured for each instance (either directly or via a profile), and not to the resources that are actually in use. For example, if you set the project's {config:option}`project-limits:limits.memory` configuration to `50GiB`, the sum of the individual values of all {config:option}`instance-resource-limits:limits.memory` configuration keys defined on the project's instances will be kept under 50 GiB. Similarly, setting the project's {config:option}`project-limits:limits.cpu` configuration key to `100` means that the sum of individual {config:option}`instance-resource-limits:limits.cpu` values will be kept below 100. When using project limits, the following conditions must be fulfilled: - When you set one of the `limits.*` configurations and there is a corresponding configuration for the instance, all instances in the project must have the corresponding configuration defined (either directly or via a profile). See {ref}`instance-options-limits` for the instance configuration options. - The {config:option}`project-limits:limits.cpu` configuration cannot be used if {ref}`instance-options-limits-cpu` is enabled. This means that to use {config:option}`project-limits:limits.cpu` on a project, the {config:option}`instance-resource-limits:limits.cpu` configuration of each instance in the project must be set to a number of CPUs, not a set or a range of CPUs. - The {config:option}`project-limits:limits.memory` configuration must be set to an absolute value, not a percentage. % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (project-restrictions)= ## Project restrictions To prevent the instances of a project from accessing security-sensitive features (such as container nesting or raw LXC configuration), set the {config:option}`project-restricted:restricted` configuration option to `true`. You can then use the various `restricted.*` options to pick individual features that would normally be blocked by {config:option}`project-restricted:restricted` and allow them, so they can be used by the instances of the project. For example, to restrict a project and block all security-sensitive features, but allow container nesting, enter the following commands: incus project set restricted=true incus project set restricted.containers.nesting=allow Each security-sensitive feature has an associated `restricted.*` project configuration option. If you want to allow the usage of a feature, change the value of its `restricted.*` option. Most `restricted.*` configurations are binary switches that can be set to either `block` (the default) or `allow`. However, some options support other values for more fine-grained control. ```{note} You must set the `restricted` configuration to `true` for any of the `restricted.*` options to be effective. If `restricted` is set to `false`, changing a `restricted.*` option has no effect. Setting all `restricted.*` keys to `allow` is equivalent to setting `restricted` itself to `false`. ``` % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` (project-specific-config)= ## Project-specific configuration There are some {ref}`server` options that you can override for a project. In addition, you can add user metadata for a project. % Include content from [../config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/provided_metrics.md000066400000000000000000000143761517523235500212510ustar00rootroot00000000000000(provided-metrics)= # Provided metrics Incus provides a number of instance metrics and internal metrics. See {ref}`metrics` for instructions on how to work with these metrics. ## Instance metrics The following instance metrics are provided: ```{list-table} :header-rows: 1 * - Metric - Description * - `incus_boot_time_seconds` - Unix timestamp when the guest was booted * - `incus_cpu_effective_total` - Total number of effective CPUs * - `incus_cpu_seconds_total{cpu="", mode=""}` - Total number of CPU time used (in seconds) * - `incus_disk_read_bytes_total{device=""}` - Total number of bytes read * - `incus_disk_reads_completed_total{device=""}` - Total number of completed reads * - `incus_disk_written_bytes_total{device=""}` - Total number of bytes written * - `incus_disk_writes_completed_total{device=""}` - Total number of completed writes * - `incus_filesystem_avail_bytes{device="",fstype=""}` - Available space (in bytes) * - `incus_filesystem_free_bytes{device="",fstype=""}` - Free space (in bytes) * - `incus_filesystem_size_bytes{device="",fstype=""}` - Size of the file system (in bytes) * - `incus_memory_Active_anon_bytes` - Amount of anonymous memory on active LRU list * - `incus_memory_Active_bytes` - Amount of memory on active LRU list * - `incus_memory_Active_file_bytes` - Amount of file-backed memory on active LRU list * - `incus_memory_Cached_bytes` - Amount of cached memory * - `incus_memory_Dirty_bytes` - Amount of memory waiting to be written back to the disk * - `incus_memory_HugepagesFree_bytes` - Amount of free memory for `hugetlb` * - `incus_memory_HugepagesTotal_bytes` - Amount of used memory for `hugetlb` * - `incus_memory_Inactive_anon_bytes` - Amount of anonymous memory on inactive LRU list * - `incus_memory_Inactive_bytes` - Amount of memory on inactive LRU list * - `incus_memory_Inactive_file_bytes` - Amount of file-backed memory on inactive LRU list * - `incus_memory_Mapped_bytes` - Amount of mapped memory * - `incus_memory_MemAvailable_bytes` - Amount of available memory * - `incus_memory_MemFree_bytes` - Amount of free memory * - `incus_memory_MemTotal_bytes` - Amount of used memory * - `incus_memory_OOM_kills_total` - The number of out-of-memory kills * - `incus_memory_RSS_bytes` - Amount of anonymous and swap cache memory * - `incus_memory_Shmem_bytes` - Amount of cached file system data that is swap-backed * - `incus_memory_Swap_bytes` - Amount of used swap memory * - `incus_memory_Unevictable_bytes` - Amount of unevictable memory * - `incus_memory_Writeback_bytes` - Amount of memory queued for syncing to disk * - `incus_network_receive_bytes_total{device=""}` - Amount of received bytes on a given interface * - `incus_network_receive_drop_total{device=""}` - Amount of received dropped bytes on a given interface * - `incus_network_receive_errs_total{device=""}` - Amount of received errors on a given interface * - `incus_network_receive_packets_total{device=""}` - Amount of received packets on a given interface * - `incus_network_transmit_bytes_total{device=""}` - Amount of transmitted bytes on a given interface * - `incus_network_transmit_drop_total{device=""}` - Amount of transmitted dropped bytes on a given interface * - `incus_network_transmit_errs_total{device=""}` - Amount of transmitted errors on a given interface * - `incus_network_transmit_packets_total{device=""}` - Amount of transmitted packets on a given interface * - `incus_procs_total` - Number of running processes * - `incus_time_seconds` - Current time from guest in seconds since epoch ``` ## Project metrics The following project metrics are provided: ```{list-table} :header-rows: 1 * - Metric - Description * - `incus_project_resources_total{project="",resource=""}` - Current count of resources in a project * - `incus_project_limit{project="",resource=""}` - Configured limit for a resource in a project (-1 if unlimited) * - `incus_project_usage{project="",resource=""}` - Current usage of a limited resource in a project ``` ## Internal metrics The following internal metrics are provided: ```{list-table} :header-rows: 1 * - Metric - Description * - `incus_go_alloc_bytes_total` - Total number of bytes allocated (even if freed) * - `incus_go_alloc_bytes` - Number of bytes allocated and still in use * - `incus_go_buck_hash_sys_bytes` - Number of bytes used by the profiling bucket hash table * - `incus_go_frees_total` - Total number of frees * - `incus_go_gc_sys_bytes` - Number of bytes used for garbage collection system metadata * - `incus_go_goroutines` - Number of goroutines that currently exist * - `incus_go_heap_alloc_bytes` - Number of heap bytes allocated and still in use * - `incus_go_heap_idle_bytes` - Number of heap bytes waiting to be used * - `incus_go_heap_inuse_bytes` - Number of heap bytes that are in use * - `incus_go_heap_objects` - Number of allocated objects * - `incus_go_heap_released_bytes` - Number of heap bytes released to OS * - `incus_go_heap_sys_bytes` - Number of heap bytes obtained from system * - `incus_go_lookups_total` - Total number of pointer lookups * - `incus_go_mallocs_total` - Total number of `mallocs` * - `incus_go_mcache_inuse_bytes` - Number of bytes in use by `mcache` structures * - `incus_go_mcache_sys_bytes` - Number of bytes used for `mcache` structures obtained from system * - `incus_go_mspan_inuse_bytes` - Number of bytes in use by `mspan` structures * - `incus_go_mspan_sys_bytes` - Number of bytes used for `mspan` structures obtained from system * - `incus_go_next_gc_bytes` - Number of heap bytes when next garbage collection will take place * - `incus_go_other_sys_bytes` - Number of bytes used for other system allocations * - `incus_go_stack_inuse_bytes` - Number of bytes in use by the stack allocator * - `incus_go_stack_sys_bytes` - Number of bytes obtained from system for stack allocator * - `incus_go_sys_bytes` - Number of bytes obtained from system * - `incus_operations_total` - Number of running operations * - `incus_uptime_seconds` - Daemon uptime (in seconds) * - `incus_warnings_total` - Number of active warnings ``` incus-7.0.0/doc/reference/server_settings.md000066400000000000000000000126561517523235500211340ustar00rootroot00000000000000(server-settings)= # Server settings for an Incus production setup To allow your Incus server to run a large number of instances, configure the following settings to avoid hitting server limits. The `Value` column contains the suggested value for each parameter. ## `/etc/security/limits.conf` | Domain | Type | Item | Value | Default | Description | | :--- | :--- | :--- | :--- | :--- | :--- | | `*` | soft | `nofile` | `1048576` | unset | Maximum number of open files | | `*` | hard | `nofile` | `1048576` | unset | Maximum number of open files | | `root` | soft | `nofile` | `1048576` | unset | Maximum number of open files | | `root` | hard | `nofile` | `1048576` | unset | Maximum number of open files | | `*` | soft | `memlock` | `unlimited` | unset | Maximum locked-in-memory address space (KB) | | `*` | hard | `memlock` | `unlimited` | unset | Maximum locked-in-memory address space (KB) | | `root` | soft | `memlock` | `unlimited` | unset | Maximum locked-in-memory address space (KB), only need with `bpf` syscall supervision | | `root` | hard | `memlock` | `unlimited` | unset | Maximum locked-in-memory address space (KB), only need with `bpf` syscall supervision | ## `/etc/sysctl.conf` ```{note} Reboot the server after changing any of these parameters. ``` | Parameter | Value | Default | Description | | :--- | :--- | :--- | :--- | | `fs.aio-max-nr` | `524288` | `65536` | Maximum number of concurrent asynchronous I/O operations (you might need to increase this limit further if you have a lot of workloads that use the AIO subsystem, for example, MySQL) | | `fs.inotify.max_queued_events` | `1048576` | `16384` | Upper limit on the number of events that can be queued to the corresponding `inotify` instance (see [`inotify`](https://man7.org/linux/man-pages/man7/inotify.7.html)) | | `fs.inotify.max_user_instances` | `1048576` | `128` | Upper limit on the number of `inotify` instances that can be created per real user ID (see [`inotify`](https://man7.org/linux/man-pages/man7/inotify.7.html)) | | `fs.inotify.max_user_watches` | `1048576` | `8192` | Upper limit on the number of watches that can be created per real user ID (see [`inotify`](https://man7.org/linux/man-pages/man7/inotify.7.html)) | | `kernel.dmesg_restrict` | `1` | `0` | Whether to deny container access to the messages in the kernel ring buffer (note that this will also deny access to non-root users on the host system) | | `kernel.keys.maxbytes` | `2000000` | `20000` | Maximum size of the key ring that non-root users can use | | `kernel.keys.maxkeys` | `2000` | `200` | Maximum number of keys that a non-root user can use (the value should be higher than the number of instances) | | `net.core.bpf_jit_limit` | `1000000000` | varies | Limit on the size of eBPF JIT allocations (on kernels < 5.15 that are compiled with `CONFIG_BPF_JIT_ALWAYS_ON=y`, this value might limit the amount of instances that can be created) | | `net.ipv4.neigh.default.gc_thresh3` | `8192` | `1024` | Maximum number of entries in the IPv4 ARP table (increase this value if you plan to create over 1024 instances - otherwise, you will get the error `neighbour: ndisc_cache: neighbor table overflow!` when the ARP table gets full and the instances cannot get a network configuration; see [`ip-sysctl`](https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt)) | | `net.ipv6.neigh.default.gc_thresh3` | `8192` | `1024` | Maximum number of entries in IPv6 ARP table (increase this value if you plan to create over 1024 instances - otherwise, you will get the error `neighbour: ndisc_cache: neighbor table overflow!` when the ARP table gets full and the instances cannot get a network configuration; see [`ip-sysctl`](https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt)) | | `vm.max_map_count` | `262144` | `65530` | Maximum number of memory map areas a process may have (memory map areas are used as a side-effect of calling `malloc`, directly by `mmap` and `mprotect`, and also when loading shared libraries) | incus-7.0.0/doc/reference/standard_devices.md000066400000000000000000000016731517523235500212050ustar00rootroot00000000000000(standard-devices)= # Standard devices Incus provides each instance with the basic devices that are required for a standard POSIX system to work. These devices aren't visible in the instance or profile configuration, and they may not be overridden. The standard devices are: | Device | Type of device | |:---------------|:------------------| | `/dev/null` | Character device | | `/dev/zero` | Character device | | `/dev/full` | Character device | | `/dev/console` | Character device | | `/dev/tty` | Character device | | `/dev/random` | Character device | | `/dev/urandom` | Character device | | `/dev/net/tun` | Character device | | `/dev/fuse` | Character device | | `lo` | Network interface | Any other devices must be defined in the instance configuration or in one of the profiles used by the instance. The default profile typically contains a network interface that becomes `eth0` in the instance. incus-7.0.0/doc/reference/storage_btrfs.md000066400000000000000000000110451517523235500205410ustar00rootroot00000000000000(storage-btrfs)= # Btrfs - `btrfs` {abbr}`Btrfs (B-tree file system)` is a local file system based on the {abbr}`COW (copy-on-write)` principle. COW means that data is stored to a different block after it has been modified instead of overwriting the existing data, reducing the risk of data corruption. Unlike other file systems, Btrfs is extent-based, which means that it stores data in contiguous areas of memory. In addition to basic file system features, Btrfs offers RAID and volume management, pooling, snapshots, checksums, compression and other features. To use Btrfs, make sure you have `btrfs-progs` installed on your machine. ## Terminology A Btrfs file system can have *subvolumes*, which are named binary subtrees of the main tree of the file system with their own independent file and directory hierarchy. A *Btrfs snapshot* is a special type of subvolume that captures a specific state of another subvolume. Snapshots can be read-write or read-only. ## `btrfs` driver in Incus The `btrfs` driver in Incus uses a subvolume per instance, image and snapshot. When creating a new entity (for example, launching a new instance), it creates a Btrfs snapshot. Btrfs doesn't natively support storing block devices. Therefore, when using Btrfs for VMs, Incus creates a big file on disk to store the VM. This approach is not very efficient and might cause issues when creating snapshots. Btrfs can be used as a storage backend inside a container in a nested Incus environment. In this case, the parent container itself must use Btrfs. Note, however, that the nested Incus setup does not inherit the Btrfs quotas from the parent (see {ref}`storage-btrfs-quotas` below). (storage-btrfs-quotas)= ### Quotas Btrfs supports storage quotas via qgroups. Btrfs qgroups are hierarchical, but new subvolumes will not automatically be added to the qgroups of their parent subvolumes. This means that users can trivially escape any quotas that are set. Therefore, if strict quotas are needed, you should consider using a different storage driver (for example, ZFS with `refquota` or LVM with Btrfs on top). When using quotas, you must take into account that Btrfs extents are immutable. When blocks are written, they end up in new extents. The old extents remain until all their data is dereferenced or rewritten. This means that a quota can be reached even if the total amount of space used by the current files in the subvolume is smaller than the quota. ```{note} This issue is seen most often when using VMs on Btrfs, due to the random I/O nature of using raw disk image files on top of a Btrfs subvolume. Therefore, you should never use VMs with Btrfs storage pools. If you really need to use VMs with Btrfs storage pools, set the instance root disk's [`size.state`](devices-disk) property to twice the size of the root disk's size. This configuration allows all blocks in the disk image file to be rewritten without reaching the qgroup quota. The [`btrfs.mount_options=compress-force`](storage-btrfs-pool-config) storage pool option can also avoid this scenario, because a side effect of enabling compression is to reduce the maximum extent size such that block rewrites don't cause as much storage to be double-tracked. However, this is a storage pool option, and it therefore affects all volumes on the pool. ``` ## Configuration options The following configuration options are available for storage pools that use the `btrfs` driver and for storage volumes in these pools. (storage-btrfs-pool-config)= ### Storage pool configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` {{volume_configuration}} ### Storage volume configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` [^*]: {{snapshot_pattern_detail}} ### Storage bucket configuration To enable storage buckets for local storage pool drivers and allow applications to access the buckets via the S3 protocol, you must configure the {config:option}`server-core:core.storage_buckets_address` server setting. % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/storage_ceph.md000066400000000000000000000124101517523235500203350ustar00rootroot00000000000000(storage-ceph)= # Ceph RBD - `ceph` [Ceph](https://ceph.io/en/) is an open-source storage platform that stores its data in a storage cluster based on {abbr}`RADOS (Reliable Autonomic Distributed Object Store)`. It is highly scalable and, as a distributed system without a single point of failure, very reliable. Ceph provides different components for block storage and for file systems. Ceph {abbr}`RBD (RADOS Block Device)` is Ceph's block storage component that distributes data and workload across the Ceph cluster. It uses thin provisioning, which means that it is possible to over-commit resources. ## Terminology Ceph uses the term *object* for the data that it stores. The daemon that is responsible for storing and managing data is the *Ceph {abbr}`OSD (Object Storage Daemon)`*. Ceph's storage is divided into *pools*, which are logical partitions for storing objects. They are also referred to as *data pools*, *storage pools* or *OSD pools*. Ceph block devices are also called *RBD images*, and you can create *snapshots* and *clones* of these RBD images. ## `ceph` driver in Incus ```{note} To use the Ceph RBD driver, you must specify it as `ceph`. This is slightly misleading, because it uses only Ceph RBD (block storage) functionality, not full Ceph functionality. For storage volumes with content type `filesystem` (images, containers and custom file-system volumes), the `ceph` driver uses Ceph RBD images with a file system on top (see [`block.filesystem`](storage-ceph-vol-config)). Alternatively, you can use the {ref}`CephFS ` driver to create storage volumes with content type `filesystem`. ``` Unlike other storage drivers, this driver does not set up the storage system but assumes that you already have a Ceph cluster installed. This driver also behaves differently than other drivers in that it provides remote storage. As a result and depending on the internal network, storage access might be a bit slower than for local storage. On the other hand, using remote storage has big advantages in a cluster setup, because all cluster members have access to the same storage pools with the exact same contents, without the need to synchronize storage pools. The `ceph` driver in Incus uses RBD images for images, and snapshots and clones to create instances and snapshots. Incus assumes that it has full control over the OSD storage pool. Therefore, you should never maintain any file system entities that are not owned by Incus in an Incus OSD storage pool, because Incus might delete them. Due to the way copy-on-write works in Ceph RBD, parent RBD images can't be removed until all children are gone. As a result, Incus automatically renames any objects that are removed but still referenced. Such objects are kept with a `zombie_` prefix until all references are gone and the object can safely be removed. ### Limitations The `ceph` driver has the following limitations: Sharing custom volumes between instances : Custom storage volumes with {ref}`content type ` `filesystem` can usually be shared between multiple instances different cluster members. However, because the Ceph RBD driver "simulates" volumes with content type `filesystem` by putting a file system on top of an RBD image, custom storage volumes can only be assigned to a single instance at a time. If you need to share a custom volume with content type `filesystem`, use the {ref}`CephFS ` driver instead. Sharing the OSD storage pool between installations : Sharing the same OSD storage pool between multiple Incus installations is not supported. Using an OSD pool of type "erasure" : To use a Ceph OSD pool of type "erasure", you must create the OSD pool beforehand. You must also create a separate OSD pool of type "replicated" that will be used for storing metadata. This is required because Ceph RBD does not support `omap`. To specify which pool is "erasure coded", set the [`ceph.osd.data_pool_name`](storage-ceph-pool-config) configuration option to the erasure coded pool name and the [`source`](storage-ceph-pool-config) configuration option to the replicated pool name. ## Configuration options The following configuration options are available for storage pools that use the `ceph` driver and for storage volumes in these pools. (storage-ceph-pool-config)= ### Storage pool configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` {{volume_configuration}} (storage-ceph-vol-config)= ### Storage volume configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` [^*]: {{snapshot_pattern_detail}} incus-7.0.0/doc/reference/storage_cephfs.md000066400000000000000000000061701517523235500206740ustar00rootroot00000000000000(storage-cephfs)= # CephFS - `cephfs` % Include content from [storage_ceph.md](storage_ceph.md) ```{include} storage_ceph.md :start-after: :end-before: ``` {abbr}`CephFS (Ceph File System)` is Ceph's file system component that provides a robust, fully-featured POSIX-compliant distributed file system. Internally, it maps files to Ceph objects and stores file metadata (for example, file ownership, directory paths, access permissions) in a separate data pool. ## Terminology % Include content from [storage_ceph.md](storage_ceph.md) ```{include} storage_ceph.md :start-after: :end-before: ``` A *CephFS file system* consists of two OSD storage pools, one for the actual data and one for the file metadata. ## `cephfs` driver in Incus ```{note} The `cephfs` driver can only be used for custom storage volumes with content type `filesystem`. For other storage volumes, use the {ref}`Ceph ` driver. That driver can also be used for custom storage volumes with content type `filesystem`, but it implements them through Ceph RBD images. ``` % Include content from [storage_ceph.md](storage_ceph.md) ```{include} storage_ceph.md :start-after: :end-before: ``` You can either create the CephFS file system that you want to use beforehand and specify it through the [`source`](storage-cephfs-pool-config) option, or specify the [`cephfs.create_missing`](storage-cephfs-pool-config) option to automatically create the file system and the data and metadata OSD pools (with the names given in [`cephfs.data_pool`](storage-cephfs-pool-config) and [`cephfs.meta_pool`](storage-cephfs-pool-config)). % Include content from [storage_ceph.md](storage_ceph.md) ```{include} storage_ceph.md :start-after: :end-before: ``` % Include content from [storage_ceph.md](storage_ceph.md) ```{include} storage_ceph.md :start-after: :end-before: ``` The `cephfs` driver in Incus supports snapshots if snapshots are enabled on the server side. ## Configuration options The following configuration options are available for storage pools that use the `cephfs` driver and for storage volumes in these pools. (storage-cephfs-pool-config)= ### Storage pool configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` {{volume_configuration}} ### Storage volume configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` [^*]: {{snapshot_pattern_detail}} incus-7.0.0/doc/reference/storage_cephobject.md000066400000000000000000000063741517523235500215400ustar00rootroot00000000000000(storage-cephobject)= # Ceph Object - `cephobject` % Include content from [storage_ceph.md](storage_ceph.md) ```{include} storage_ceph.md :start-after: :end-before: ``` [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/) is an object storage interface built on top of [`librados`](https://docs.ceph.com/en/latest/rados/api/librados-intro/) to provide applications with a RESTful gateway to [Ceph Storage Clusters](https://docs.ceph.com/en/latest/rados/). It provides object storage functionality with an interface that is compatible with a large subset of the Amazon S3 RESTful API. ## Terminology % Include content from [storage_ceph.md](storage_ceph.md) ```{include} storage_ceph.md :start-after: :end-before: ``` A *Ceph Object Gateway* consists of several OSD pools and one or more *Ceph Object Gateway daemon* (`radosgw`) processes that provide object gateway functionality. ## `cephobject` driver in Incus ```{note} The `cephobject` driver can only be used for buckets. For storage volumes, use the {ref}`Ceph ` or {ref}`CephFS ` drivers. ``` % Include content from [storage_ceph.md](storage_ceph.md) ```{include} storage_ceph.md :start-after: :end-before: ``` You must set up a `radosgw` environment beforehand and ensure that its HTTP/HTTPS endpoint URL is reachable from the Incus server or servers. See [Manual Deployment](https://docs.ceph.com/en/latest/install/manual-deployment/) for information on how to set up a Ceph cluster and [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/) on how to set up a `radosgw` environment. The `radosgw` URL can be specified at pool creation time using the [`cephobject.radosgw.endpoint`](storage-cephobject-pool-config) option. Incus uses the `radosgw-admin` command to manage buckets. So this command must be available and operational on the Incus servers. % Include content from [storage_ceph.md](storage_ceph.md) ```{include} storage_ceph.md :start-after: :end-before: ``` % Include content from [storage_ceph.md](storage_ceph.md) ```{include} storage_ceph.md :start-after: :end-before: ``` ## Configuration options The following configuration options are available for storage pools that use the `cephobject` driver and for storage buckets in these pools. (storage-cephobject-pool-config)= ### Storage pool configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` ### Storage bucket configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/storage_dir.md000066400000000000000000000042611517523235500202010ustar00rootroot00000000000000(storage-dir)= # Directory - `dir` The directory storage driver is a basic backend that stores its data in a standard file and directory structure. This driver is quick to set up and allows inspecting the files directly on the disk, which can be convenient for testing. However, Incus operations are {ref}`not optimized ` for this driver. ## `dir` driver in Incus The `dir` driver in Incus is fully functional and provides the same set of features as other drivers. However, it is much slower than all the other drivers because it must unpack images and do instant copies of instances, snapshots and images. Unless specified differently during creation (with the `source` configuration option), the data is stored in the `/var/lib/incus/storage-pools/` directory. (storage-dir-quotas)= ### Quotas The `dir` driver supports storage quotas when running on either ext4 or XFS with project quotas enabled at the file system level. ## Configuration options The following configuration options are available for storage pools that use the `dir` driver and for storage volumes in these pools. ### Storage pool configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` {{volume_configuration}} ### Storage volume configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` [^*]: {{snapshot_pattern_detail}} ### Storage bucket configuration To enable storage buckets for local storage pool drivers and allow applications to access the buckets via the S3 protocol, you must configure the {config:option}`server-core:core.storage_buckets_address` server setting. Storage buckets do not have any configuration for `dir` pools. Unlike the other storage pool drivers, the `dir` driver does not support bucket quotas via the `size` setting. incus-7.0.0/doc/reference/storage_drivers.md000066400000000000000000000154201517523235500211000ustar00rootroot00000000000000(storage-drivers)= # Storage drivers Incus supports the following storage drivers for storing images, instances and custom volumes: ```{toctree} :maxdepth: 1 storage_dir storage_btrfs storage_lvm storage_zfs storage_ceph storage_cephfs storage_cephobject storage_linstor storage_truenas ``` See the corresponding pages for driver-specific information and configuration options. (storage-drivers-features)= ## Feature comparison Where possible, Incus uses the advanced features of each storage system to optimize operations. | Feature | Directory | Btrfs | LVM | ZFS | Ceph RBD | CephFS | Ceph Object | LINSTOR | TRUENAS | | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | | {ref}`storage-optimized-image-storage` | no | yes | yes | yes | yes | n/a | n/a | yes | yes | | Optimized instance creation | no | yes | yes | yes | yes | n/a | n/a | yes | yes | | Optimized snapshot creation | no | yes | yes | yes | yes | yes | n/a | yes | yes | | Optimized image transfer | no | yes | no | yes | yes | n/a | n/a | no | no | | {ref}`storage-optimized-volume-transfer` | no | yes | no | yes | yes | n/a | n/a | no | no | | Copy on write | no | yes | yes | yes | yes | yes | n/a | yes | yes | | Block based | no | no | yes | no | yes | no | n/a | yes | yes | | Instant cloning | no | yes | yes | yes | yes | yes | n/a | yes | yes | | Storage driver usable inside a container | yes | yes | no | yes[^1] | no | n/a | n/a | no | no | | Restore from older snapshots (not latest) | yes | yes | yes | no | yes | yes | n/a | no | no | | Storage quotas | yes[^2] | yes | yes | yes | yes | yes | yes | yes | yes | | Available on `incus admin init` | yes | yes | yes | yes | yes | no | no | no | no | | Object storage | yes | yes | yes | yes | no | no | yes | no | no | [^1]: Requires [`zfs.delegate`](storage-zfs-vol-config) to be enabled. [^2]: % Include content from [storage_dir.md](storage_dir.md) ```{include} storage_dir.md :start-after: :end-before: ``` (storage-optimized-image-storage)= ### Optimized image storage All storage drivers except for the directory driver have some kind of optimized image storage format. To make instance creation near instantaneous, Incus clones a pre-made image volume when creating an instance rather than unpacking the image tarball from scratch. To prevent preparing such a volume on a storage pool that might never be used with that image, the volume is generated on demand. Therefore, the first instance takes longer to create than subsequent ones. (storage-optimized-volume-transfer)= ### Optimized volume transfer Btrfs, ZFS and Ceph RBD have an internal send/receive mechanism that allows for optimized volume transfer. Incus uses this optimized transfer when transferring instances and snapshots between storage pools that use the same storage driver, if the storage driver supports optimized transfer and the optimized transfer is actually quicker. Otherwise, Incus uses `rsync` to transfer container and file system volumes, or raw block transfer to transfer virtual machine and custom block volumes. The optimized transfer uses the underlying storage driver's native functionality for transferring data, which is usually faster than using `rsync`. However, the full potential of the optimized transfer becomes apparent when refreshing a copy of an instance or custom volume that uses periodic snapshots. With optimized transfer, Incus bases the refresh on the latest snapshot, which means: - When you take a first snapshot and refresh the copy, the transfer will take roughly the same time as a full copy. Incus transfers the new snapshot and the difference between the snapshot and the main volume. - For subsequent snapshots, the transfer is considerably faster. Incus does not transfer the full new snapshot, but only the difference between the new snapshot and the latest snapshot that already exists on the target. - When refreshing without a new snapshot, Incus transfers only the differences between the main volume and the latest snapshot on the target. This transfer is usually faster than using `rsync` (as long as the latest snapshot is not too outdated). On the other hand, refreshing copies of instances without snapshots (either because the instance doesn't have any snapshots or because the refresh uses the `--instance-only` flag) would actually be slower than using `rsync`. In such cases, the optimized transfer would transfer the difference between the (non-existent) latest snapshot and the main volume, thus the full volume. Therefore, Incus uses `rsync` instead of the optimized transfer for refreshes without snapshots. ## Recommended setup The two best options for use with Incus are ZFS and Btrfs. They have similar functionalities, but ZFS is more reliable. Whenever possible, you should dedicate a full disk or partition to your Incus storage pool. Incus allows to create loop-based storage, but this isn't recommended for production use. See {ref}`storage-location` for more information. The directory backend should be considered as a last resort option. It supports all main Incus features, but is slow and inefficient because it cannot perform instant copies or snapshots. Therefore, it constantly copies the instance's full storage. ## Security considerations Currently, the Linux kernel might silently ignore mount options and not apply them when a block-based file system (for example, `ext4`) is already mounted with different mount options. This means when dedicated disk devices are shared between different storage pools with different mount options set, the second mount might not have the expected mount options. This becomes security relevant when, for example, one storage pool is supposed to provide `acl` support and the second one is supposed to not provide `acl` support. For this reason, it is currently recommended to either have dedicated disk devices per storage pool or to ensure that all storage pools that share the same dedicated disk device use the same mount options. incus-7.0.0/doc/reference/storage_linstor.md000066400000000000000000000173661517523235500211270ustar00rootroot00000000000000(storage-linstor)= # LINSTOR - `linstor` [LINSTOR](https://linbit.com/linstor/) is an open-source software-defined storage solution that is typically used to manage {abbr}`DRBD (Distributed Replicated Block Device)` replicated storage volumes. It provides both highly available and high performance volumes while focusing on operational simplicity. LINSTOR does not manage the underlying storage by itself, and instead relies on other components such as ZFS or LVM to provision block devices. These block devices are then replicated using [DRBD](https://linbit.com/drbd/) to provide fault tolerance and the ability to mount the volumes on any cluster node, regardless of its storage capabilities. Since volumes are replicated using the DRBD kernel module, the data path for the replication is kept entirely on kernel space, reducing its overhead when compared to solutions implemented in user space. ## Terminology A LINSTOR cluster is composed of two main components: *controllers* and *satellites*. The LINSTOR controller manages the database and keeps track of the cluster state and configuration, while satellites provide storage and ability to mount volumes across the cluster. Clients interact only with the controller, which is responsible for orchestrating operations across satellites to fulfill the user's request. LINSTOR takes a somewhat object-oriented approach to its internal concepts. This manifests itself in the hierarchical nature of concepts and the fact that lower level objects can inherit properties from higher level ones. LINSTOR has the concept of a *storage pool*, which describes physical storage that can be consumed by LINSTOR to create volumes. A storage pool defines its backend driver (such as LVM or ZFS), the cluster node in which it exists and properties that can be applied to either the storage pool itself or its backend storage. In LINSTOR, a *resource* is the representation of a storage unit that can be consumed by instances. A resource is most often a DRBD replicated block device, and in that case represents one replica of that device. Resources can be grouped into *resource definitions*, which define common properties that should be inherited by all their child resources. Similarly, *resource groups* define common properties that are applied to their child resource definitions. Resource groups also define placement rules that define how many replicas should be created for a given resource definition, which storage pool should be used, how to spread the replicas among different availability zones, etc. The usual way to interact with LINSTOR is by defining a resource group with the desired properties and then *spawning* resources from it. ## `linstor` driver in Incus ```{note} LINSTOR can only move and mount volumes between its satellite nodes. Therefore, to ensure that all Incus cluster members can access volumes, all Incus nodes must also be LINSTOR satellite nodes. In other words, each node running the `incus` service should also run an `linstor-satellite` service. Note, however, that this does not mean that Incus nodes must also provide storage. It is still possible to use LINSTOR while using separated storage and compute nodes by deploying "diskless" satellites on Incus nodes. Diskless nodes do not provide storage, but are still able to mount DRBD devices and perform IO over the network. ``` Unlike other storage drivers, this driver does not set up the storage system but assumes that you already have a LINSTOR cluster installed. The driver requires the {config:option}`server-miscellaneous:storage.linstor.controller_connection` option to be set to the endpoint of a LINSTOR controller that will be used by Incus. This driver also behaves differently than other drivers in that it can provide both remote and local storage. If a diskful replica of the volume is available on the node, reads and writes can be performed locally to reduce latency (although writes must be synchronously replicated across replicas, so network latency still has an impact). At the same time, a diskless replica performs all IO over the network, enabling volumes to be mounted and used on any node regardless of its physical storage. These hybrid capabilities enable LINSTOR to provide low latency storage while retaining the flexibility of moving volumes across cluster nodes when needed. The `linstor` driver in Incus uses resource groups to manage and spawn resources. The following table describes the mapping between Incus and LINSTOR concepts: | Incus concept | LINSTOR concept | | :--- | :--- | | Storage pool | Resource group | | Volume | Resource definition | | Snapshot | Snapshot | Incus assumes that it has full control over the LINSTOR resource group. Therefore, you should never maintain any entities that are not owned by Incus in an Incus LINSTOR resource group, because Incus might delete them. When managing resources, Incus needs to be able to determine which LINSTOR satellite node corresponds to a given Incus node. By default, Incus assumes that its node names match LINSTOR's (e.g. `incus cluster list` and `linstor node list` show the same node names). When Incus is running as a standalone server (i.e. not clustered), the hostname is used as the node name. If node names between Incus and LINSTOR do not match, the {config:option}`server-miscellaneous:storage.linstor.satellite.name` can be set on each Incus node to the appropriate LINSTOR satellite node name. ### Limitations The `linstor` driver has the following limitations: Sharing custom volumes between instances : Custom storage volumes with {ref}`content type ` `filesystem` can usually be shared between multiple instances different cluster members. However, because the LINSTOR driver "simulates" volumes with content type `filesystem` by putting a file system on top of an DRBD replicated device, custom storage volumes can only be assigned to a single instance at a time. Sharing the resource group between installations : Sharing the same LINSTOR resource group between multiple Incus installations is not supported. Restoring from older snapshots : LINSTOR doesn't support restoring from snapshots other than the latest one. You can, however, create new instances from older snapshots. This method makes it possible to confirm whether a specific snapshot contains what you need. After determining the correct snapshot, you can {ref}`remove the newer snapshots ` so that the snapshot you need is the latest one and you can restore it. Alternatively, you can configure Incus to automatically discard the newer snapshots during restore. To do so, set the [`linstor.remove_snapshots`](storage-linstor-vol-config) configuration for the volume (or the corresponding `volume.linstor.remove_snapshots` configuration on the storage pool for all volumes in the pool). ## Configuration options The following configuration options are available for storage pools that use the `linstor` driver and for storage volumes in these pools. (storage-linstor-pool-config)= ### Storage pool configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` {{volume_configuration}} (storage-linstor-vol-config)= ### Storage volume configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` [^*]: {{snapshot_pattern_detail}} ```{toctree} :maxdepth: 1 :hidden: Setup LINSTOR Driver internals ``` incus-7.0.0/doc/reference/storage_linstor_internals.md000066400000000000000000000263071517523235500232010ustar00rootroot00000000000000(storage-linstor-internals)= # `linstor` driver internals This section describes some of the internal details of the `linstor` driver implementation. Although knowledge of these details is not needed to use the driver, they can be relevant for operators when troubleshooting or even to get a better understanding of the interactions between Incus and LINSTOR. ## Naming objects At the time of writing, LINSTOR does not support renaming any of its objects. This includes resources definitions, resource groups, storage pools and snapshots. Since Incus needs the ability to rename the resources it manages, this limitation requires an alternative way of naming objects in LINSTOR while still being able to relate them to the Incus' database view of the objects. At a first glance, using Incus' database ID to name LINSTOR objects seems like a viable option. Unfortunately, that does not work in {ref}`disaster recovery ` and {ref}`backups ` scenarios. To work around those limitations while accounting for the mentioned scenarios, Incus uses auxiliary properties on LINSTOR resource definitions. The auxiliary properties store metadata about the volume from Incus' perspective. Incus can then query LINSTOR using those auxiliary properties to find the resource definition for a given volume. This makes the resource definition name irrelevant for Incus, which enables the use of a randomly generated value that is combined with the configured `linstor.volume.prefix` value. Visualizing the auxiliary properties in LINSTOR can be done by either adding the `--show-props` flag on the `linstor` command or by using the `resource-definition list-properties` command on a specific resource definition. As shown in the following example: ``` # incus storage volume list linstor +----------------------------+------------------------------------------------------------------+-------------+--------------+---------+----------+ | TYPE | NAME | DESCRIPTION | CONTENT-TYPE | USED BY | LOCATION | +----------------------------+------------------------------------------------------------------+-------------+--------------+---------+----------+ | container | c1 | | filesystem | 1 | | +----------------------------+------------------------------------------------------------------+-------------+--------------+---------+----------+ | custom | vol | | block | 0 | | +----------------------------+------------------------------------------------------------------+-------------+--------------+---------+----------+ | image | d21a26af7d5a95c3aa6e923257a1cb5cd765b102796b92ab111fb29ebfb86137 | | block | 1 | | +----------------------------+------------------------------------------------------------------+-------------+--------------+---------+----------+ | image | e3b67bf05e20c6c977f161a425733b00efe88834914e7fc8dd910c4b51cd5804 | | filesystem | 1 | | +----------------------------+------------------------------------------------------------------+-------------+--------------+---------+----------+ | virtual-machine | v1 | | block | 1 | | +----------------------------+------------------------------------------------------------------+-------------+--------------+---------+----------+ | virtual-machine (snapshot) | v1/snap0 | | block | 0 | | +----------------------------+------------------------------------------------------------------+-------------+--------------+---------+----------+ # linstor resource-definition list --show-props Aux/Incus/name Aux/Incus/content-type Aux/Incus/type ╭─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ ┊ ResourceName ┊ Port ┊ ResourceGroup ┊ Layers ┊ State ┊ Aux/Incus/name ┊ Aux/Incus/content-type ┊ Aux/Incus/type ┊ ╞═════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╡ ┊ incus-volume-4df9f0598bd14d73953151614428d298 ┊ 7004 ┊ linstor ┊ DRBD,STORAGE ┊ ok ┊ incus-volume-v1 ┊ block ┊ virtual-machines ┊ ┊ incus-volume-61be43e12fed4845ab3f2443cccbe50c ┊ 7001 ┊ linstor ┊ DRBD,STORAGE ┊ ok ┊ incus-volume-c1 ┊ filesystem ┊ containers ┊ ┊ incus-volume-d0531e21ce6b4218b9b8996582b9bf31 ┊ 7002 ┊ linstor ┊ DRBD,STORAGE ┊ ok ┊ incus-volume-d21a26af7d5a95c3aa6e923257a1cb5cd765b102796b92ab111fb29ebfb86137 ┊ block ┊ images ┊ ┊ incus-volume-e3e682324ff54fa3a3cf3203d5029366 ┊ 7000 ┊ linstor ┊ DRBD,STORAGE ┊ ok ┊ incus-volume-default_vol ┊ block ┊ custom ┊ ┊ incus-volume-ef2f8cc7bfc148c58a7259f5a201658f ┊ 7003 ┊ linstor ┊ DRBD,STORAGE ┊ ok ┊ incus-volume-e3b67bf05e20c6c977f161a425733b00efe88834914e7fc8dd910c4b51cd5804 ┊ filesystem ┊ images ┊ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ # linstor snapshot list ╭───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ ┊ ResourceName ┊ SnapshotName ┊ NodeNames ┊ Volumes ┊ CreatedOn ┊ State ┊ ╞═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╡ ┊ incus-volume-4df9f0598bd14d73953151614428d298 ┊ incus-volume-19dd0b015d22445f8097ba5e740948de ┊ server01, server02 ┊ 0: 10 GiB, 1: 500 MiB ┊ 2025-03-13 16:01:31 ┊ Successful ┊ ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ # linstor resource-definition list-properties incus-volume-4df9f0598bd14d73953151614428d298 ╭─────────────────────────────────────────────────────────────────────────────────────╮ ┊ Key ┊ Value ┊ ╞═════════════════════════════════════════════════════════════════════════════════════╡ ┊ Aux/Incus/SnapshotName/snap0 ┊ incus-volume-19dd0b015d22445f8097ba5e740948de ┊ ┊ Aux/Incus/content-type ┊ block ┊ ┊ Aux/Incus/name ┊ incus-volume-v1 ┊ ┊ Aux/Incus/type ┊ virtual-machines ┊ ┊ DrbdOptions/Net/allow-two-primaries ┊ yes ┊ ┊ DrbdOptions/Resource/on-no-quorum ┊ io-error ┊ ┊ DrbdOptions/Resource/quorum ┊ majority ┊ ┊ DrbdOptions/auto-verify-alg ┊ crct10dif ┊ ┊ DrbdPrimarySetOn ┊ SERVER01 ┊ ┊ cloned-from ┊ incus-volume-d0531e21ce6b4218b9b8996582b9bf31 ┊ ╰─────────────────────────────────────────────────────────────────────────────────────╯ ``` incus-7.0.0/doc/reference/storage_lvm.md000066400000000000000000000121001517523235500202100ustar00rootroot00000000000000(storage-lvm)= # LVM - `lvm` {abbr}`LVM (Logical Volume Manager)` is a storage management framework rather than a file system. It is used to manage physical storage devices, allowing you to create a number of logical storage volumes that use and virtualize the underlying physical storage devices. Note that it is possible to over-commit the physical storage in the process, to allow flexibility for scenarios where not all available storage is in use at the same time. To use LVM, make sure you have `lvm2` installed on your machine. ## Terminology LVM can combine several physical storage devices into a *volume group*. You can then allocate *logical volumes* of different types from this volume group. One supported volume type is a *thin pool*, which allows over-committing the resources by creating thinly provisioned volumes whose total allowed maximum size is larger than the available physical storage. Another type is a *volume snapshot*, which captures a specific state of a logical volume. ## `lvm` driver in Incus The `lvm` driver in Incus uses logical volumes for images, and volume snapshots for instances and snapshots. Incus assumes that it has full control over the volume group. Therefore, you should not maintain any file system entities that are not owned by Incus in an LVM volume group, because Incus might delete them. However, if you need to reuse an existing volume group (for example, because your setup has only one volume group), you can do so by setting the [`lvm.vg.force_reuse`](storage-lvm-pool-config) configuration. By default, LVM storage pools use an LVM thin pool and create logical volumes for all Incus storage entities (images, instances and custom volumes) in there. This behavior can be changed by setting [`lvm.use_thinpool`](storage-lvm-pool-config) to `false` when you create the pool. In this case, Incus uses "normal" logical volumes for all storage entities that are not snapshots. Note that this entails serious performance and space reductions for the `lvm` driver (close to the `dir` driver both in speed and storage usage). The reason for this is that most storage operations must fall back to using `rsync`, because logical volumes that are not thin pools do not support snapshots of snapshots. In addition, non-thin snapshots take up much more storage space than thin snapshots, because they must reserve space for their maximum size at creation time. Therefore, this option should only be chosen if the use case requires it. For environments with a high instance turnover (for example, continuous integration) you should tweak the backup `retain_min` and `retain_days` settings in `/etc/lvm/lvm.conf` to avoid slowdowns when interacting with Incus. (storage-lvmcluster)= ## `lvmcluster` driver in Incus A second `lvmcluster` driver is available for use within clusters. This relies on the `lvmlockd` and `sanlock` daemons to provide distributed locking over a shared disk or set of disks. It allows using a remote shared block device like a `FiberChannel LUN`, `NVMEoF/NVMEoTCP` disk or `iSCSI` drive as the backing for a LVM storage pool. ```{note} Thin provisioning is incompatible with clustered LVM, so expect higher disk usage. ``` To use this with Incus, you must: - Have a shared block device available on all your cluster members - Install the relevant packages for `lvm`, `lvmlockd` and `sanlock` - Enable `lvmlockd` by setting `use_lvmlockd = 1` in your `/etc/lvm/lvm.conf` - Set a unique (within your cluster) `host_id` value in `/etc/lvm/lvmlocal.conf` - Ensure that both `lvmlockd` and `sanlock` daemons are running ```{warning} `lvmcluster` has the following limitations: - Snapshots for shared custom storage volumes are not supported. - Snapshots of raw block volumes are not supported. - Only `raw` custom volumes support the `security.shared` option. ``` ## Configuration options The following configuration options are available for storage pools that use the `lvm` driver and for storage volumes in these pools. (storage-lvm-pool-config)= ### Storage pool configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` {{volume_configuration}} (storage-lvm-vol-config)= ### Storage volume configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` [^*]: {{snapshot_pattern_detail}} ### Storage bucket configuration To enable storage buckets for local storage pool drivers and allow applications to access the buckets via the S3 protocol, you must configure the {config:option}`server-core:core.storage_buckets_address` server setting. % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/storage_truenas.md000066400000000000000000000100701517523235500210770ustar00rootroot00000000000000(storage-truenas)= # TrueNAS - `truenas` ## The `truenas` storage driver in Incus The `truenas` storage driver enables an Incus node to use a remote TrueNAS storage server to host one or more Incus storage pools. When the node is part of a cluster, all cluster members can access the storage pool simultaneously, making it ideal for use cases such as live migrating virtual machines (VMs) between nodes. The driver operates in a block-based manner, meaning that all Incus volumes are created as ZFS Volume block devices on the remote TrueNAS server. These ZFS Volume block devices are accessed on the local Incus node via iSCSI. Modeled after the existing ZFS driver, the `truenas` driver supports most standard ZFS functionality, but operates on remote TrueNAS servers. For instance, a local VM can be snapshotted and cloned, with the snapshot and clone operations performed on the remote server after synchronizing the local file system. The clone is then activated through iSCSI as necessary. Each storage pool corresponds to a ZFS dataset on a remote TrueNAS host. The dataset is created automatically if it does not exist. The driver uses ZFS features available on the remote host to support efficient image handling, copy operations, and snapshot management without requiring nested ZFS (ZFS-on-ZFS). To reference a remote dataset, the `source` property can be specified in the form: `[:][[/]...][/]` If the path ends with a trailing `/`, the dataset name will be derived from the Incus storage pool name (e.g., `tank/pool1`). ## Requirements The driver relies on the [`truenas_incus_ctl`](https://github.com/truenas/truenas_incus_ctl) tool to interact with the TrueNAS API and perform actions on the remote server. This tool also manages the activation and deactivation of remote ZFS Volumes via `open-iscsi`. If `truenas_incus_ctl` is not installed or available in the system's PATH, the driver will be disabled. To install the required tool, download the latest version (v0.7.2+ is required) from the [`truenas\_incus\_ctl` GitHub page](https://github.com/truenas/truenas_incus_ctl). Additionally, ensure that `open-iscsi` is installed on the system, which can be done using: sudo apt install open-iscsi ## Logging in to the TrueNAS host As an alternative to manually creating an API Key and supplying using the `truenas.api_key` property, you can instead `login` to the remote server using the `truenas_incus_ctl` tool. sudo truenas_incus_ctl config login This will prompt you to provide connection details for the TrueNAS server, including authentication details, and will save the configuration to a local file. After logging in, you can verify the iSCSI setup with: sudo truenas_incus_ctl share iscsi setup --test Once the tool is configured, you can use it to interact with remote datasets and create storage pools: incus storage create truenas source=[host:][/]/[remote-poolname] In this command: * `source` refers to the location on the remote TrueNAS host where the storage pool will be created. * `host` is optional, and can be specified using the `truenas.host` property, or by specifying a configuration with `truenas.config` * If `remote-poolname` is not supplied, it will default to the name of the local pool. ## Configuration options The following configuration options are available for storage pools that use the `truenas` driver and for storage volumes in these pools. (storage-truenas-pool-config)= ### Storage pool configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` {{volume_configuration}} (storage-truenas-vol-config)= ### Storage volume configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/reference/storage_zfs.md000066400000000000000000000154121517523235500202250ustar00rootroot00000000000000(storage-zfs)= # ZFS - `zfs` {abbr}`ZFS (Zettabyte file system)` combines both physical volume management and a file system. A ZFS installation can span across a series of storage devices and is very scalable, allowing you to add disks to expand the available space in the storage pool immediately. ZFS is a block-based file system that protects against data corruption by using checksums to verify, confirm and correct every operation. To run at a sufficient speed, this mechanism requires a powerful environment with a lot of RAM. In addition, ZFS offers snapshots and replication, RAID management, copy-on-write clones, compression and other features. To use ZFS, make sure you have `zfsutils-linux` installed on your machine. ## Terminology ZFS creates logical units based on physical storage devices. These logical units are called *ZFS pools* or *zpools*. Each zpool is then divided into a number of *{spellexception}`datasets`*. These {spellexception}`datasets` can be of different types: - A *{spellexception}`ZFS filesystem`* can be seen as a partition or a mounted file system. - A *ZFS volume* represents a block device. - A *ZFS snapshot* captures a specific state of either a {spellexception}`ZFS filesystem` or a ZFS volume. ZFS snapshots are read-only. - A *ZFS clone* is a writable copy of a ZFS snapshot. ## `zfs` driver in Incus The `zfs` driver in Incus uses {spellexception}`ZFS filesystems` and ZFS volumes for images and custom storage volumes, and ZFS snapshots and clones to create instances from images and for instance and custom volume snapshots. By default, Incus enables compression when creating a ZFS pool. Incus assumes that it has full control over the ZFS pool and {spellexception}`dataset`. Therefore, you should never maintain any {spellexception}`datasets` or file system entities that are not owned by Incus in a ZFS pool or {spellexception}`dataset`, because Incus might delete them. Due to the way copy-on-write works in ZFS, parent {spellexception}`ZFS filesystems` can't be removed until all children are gone. As a result, Incus automatically renames any objects that are removed but still referenced. Such objects are kept at a random `deleted/` path until all references are gone and the object can safely be removed. Note that this method might have ramifications for restoring snapshots. See {ref}`storage-zfs-limitations` below. Incus automatically enables trimming support on all newly created pools on ZFS 0.8 or later. This increases the lifetime of SSDs by allowing better block reuse by the controller, and it also allows to free space on the root file system when using a loop-backed ZFS pool. If you are running a ZFS version earlier than 0.8 and want to enable trimming, upgrade to at least version 0.8. Then use the following commands to make sure that trimming is automatically enabled for the ZFS pool in the future and trim all currently unused space: zpool upgrade ZPOOL-NAME zpool set autotrim=on ZPOOL-NAME zpool trim ZPOOL-NAME (storage-zfs-limitations)= ### Limitations The `zfs` driver has the following limitations: Restoring from older snapshots : ZFS doesn't support restoring from snapshots other than the latest one. You can, however, create new instances from older snapshots. This method makes it possible to confirm whether a specific snapshot contains what you need. After determining the correct snapshot, you can {ref}`remove the newer snapshots ` so that the snapshot you need is the latest one and you can restore it. Alternatively, you can configure Incus to automatically discard the newer snapshots during restore. To do so, set the [`zfs.remove_snapshots`](storage-zfs-vol-config) configuration for the volume (or the corresponding `volume.zfs.remove_snapshots` configuration on the storage pool for all volumes in the pool). Note, however, that if [`zfs.clone_copy`](storage-zfs-pool-config) is set to `true`, instance copies use ZFS snapshots too. In that case, you cannot restore an instance to a snapshot taken before the last copy without having to also delete all its descendants. If this is not an option, you can copy the wanted snapshot into a new instance and then delete the old instance. You will, however, lose any other snapshots the instance might have had. Observing I/O quotas : I/O quotas are unlikely to affect {spellexception}`ZFS filesystems` very much. That's because ZFS is a port of a Solaris module (using SPL) and not a native Linux file system using the Linux VFS API, which is where I/O limits are applied. Feature support in ZFS : Some features, like the use of idmaps or delegation of a ZFS dataset, require ZFS 2.2 or higher and are therefore not widely available yet. ### Quotas ZFS provides two different quota properties: `quota` and `refquota`. `quota` restricts the total size of a {spellexception}`dataset`, including its snapshots and clones. `refquota` restricts only the size of the data in the {spellexception}`dataset`, not its snapshots and clones. By default, Incus uses the `quota` property when you set up a quota for your storage volume. If you want to use the `refquota` property instead, set the [`zfs.use_refquota`](storage-zfs-vol-config) configuration for the volume (or the corresponding `volume.zfs.use_refquota` configuration on the storage pool for all volumes in the pool). You can also set the [`zfs.use_reserve_space`](storage-zfs-vol-config) (or `volume.zfs.use_reserve_space`) configuration to use ZFS `reservation` or `refreservation` along with `quota` or `refquota`. ## Configuration options The following configuration options are available for storage pools that use the `zfs` driver and for storage volumes in these pools. (storage-zfs-pool-config)= ### Storage pool configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` {{volume_configuration}} (storage-zfs-vol-config)= ### Storage volume configuration % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` [^*]: {{snapshot_pattern_detail}} ### Storage bucket configuration To enable storage buckets for local storage pool drivers and allow applications to access the buckets via the S3 protocol, you must configure the {config:option}`server-core:core.storage_buckets_address` server setting. % Include content from [config_options.txt](../config_options.txt) ```{include} ../config_options.txt :start-after: :end-before: ``` incus-7.0.0/doc/remotes.md000066400000000000000000000073461517523235500154260ustar00rootroot00000000000000# How to add remote servers Remote servers are a concept in the Incus command-line client. By default, the command-line client interacts with the local Incus daemon, but you can add other servers or clusters to interact with. One use case for remote servers is to distribute images that can be used to create instances on local servers. See {ref}`image-servers` for more information. You can also add a full Incus server as a remote server to your client. In this case, you can interact with the remote server in the same way as with your local daemon. For example, you can manage instances or update the server configuration on the remote server. ## Authentication To be able to add an Incus server as a remote server, the server's API must be exposed, which means that its {config:option}`server-core:core.https_address` server configuration option must be set. When adding the server, you must then authenticate with it using the chosen method for {ref}`authentication`. See {ref}`server-expose` for more information. ## List configured remotes % Include parts of the content from file [howto/images_remote.md](howto/images_remote.md) ```{include} howto/images_remote.md :start-after: :end-before: ``` ## Add a remote Incus server % Include parts of the content from file [howto/images_remote.md](howto/images_remote.md) ```{include} howto/images_remote.md :start-after: :end-before: ``` ## Select a default remote The Incus command-line client is pre-configured with the `local` remote, which is the local Incus daemon. To select a different remote as the default remote, enter the following command: incus remote switch To see which server is configured as the default remote, enter the following command: incus remote get-default ## Configure a global remote You can configure remotes on a global, per-system basis. These remotes are available for every user of the Incus server for which you add the configuration. Users can override these system remotes (for example, by running [`incus remote rename`](incus_remote_rename.md) or [`incus remote set-url`](incus_remote_set-url.md)), which results in the remote and its associated certificates being copied to the user configuration. To configure a global remote, create or edit a `config.yml` file that is located in `/etc/incus/`. Certificates for the remotes must be stored in the `servercerts` directory in the same location (for example, `/etc/incus/servercerts/`). They must match the remote name (for example, `foo.crt`). It's also possible to provide per-remote client certificates by placing them in the `clientcerts` directory. The similarly must match the remote name (for example, `foo.crt` and `foo.key`). See the following example configuration: ``` remotes: foo: addr: https://192.0.2.4:8443 auth_type: tls project: default protocol: incus public: false bar: addr: https://192.0.2.5:8443 auth_type: tls project: default protocol: incus public: false ``` (remote-keepalive)= ## Enabling `keepalive` For those frequently interacting with a particular remote, it's possible to enable a new `keepalive` mode. When enabled, Incus will maintain a connection with the target server for up to the configured timeout. This can significantly reduce the latency when running many `incus` commands. To enable, edit your `config.yml` (typically in `~/.config/incus`) and change your remote to look like: ``` my-remote: addr: https://192.0.2.5:8443 auth_type: tls project: default protocol: incus public: false keepalive: 30 ``` In this example, a timeout of 30 seconds will be used. incus-7.0.0/doc/requirements.md000066400000000000000000000041201517523235500164560ustar00rootroot00000000000000# Requirements (requirements-go)= ## Go Incus requires Go 1.25 or higher and is only tested with the Golang compiler. We recommend having at least 2GiB of RAM to allow the build to complete. ## Kernel requirements The minimum supported kernel version is 6.12. Incus requires a kernel with support for: * Control Groups (`blkio`, `cpuset`, `devices`, `freezer`, `memory` and `pids`) * Namespaces (`cgroup`, `ipc`, `pid`, `mount`, `net`, `user` and `uts`) * Seccomp * Native Linux AIO ([`io_setup(2)`](https://man7.org/linux/man-pages/man2/io_setup.2.html), etc.) The following optional features also require extra kernel options: * AppArmor * CRIU (exact details to be found with CRIU upstream) * SELinux As well as any other kernel feature required by the LXC version in use. ## LXC Incus requires LXC 6.0.0 or higher with the following build options: * `apparmor` (if using Incus' AppArmor support) (minimum version: 3.0.0) * `seccomp` * `selinux` (if using Incus' SELinux support) LXCFS is strongly recommended to properly report resource consumption inside the container. ## OCI To run OCI containers, Incus currently relies on `skopeo` for registry interactions. ## QEMU For virtual machines, QEMU 8.2 or higher is required. When using `virtiofsd`, only the [Rust rewrite](https://gitlab.com/virtio-fs/virtiofsd) of `virtiofsd` is supported. ## Network Minimum versions for network related tooling: * `nftables`: 1.0.0 * `dnsmasq`: 2.90 When using Incus with OVN networks, the minimum versions of OVS and OVN are: * `openvswitch`: 2.15.0 * `ovn`: 23.03.0 ## Storage Minimum versions for storage drivers: * `zfs`: 2.1.0 * `lvm`: 2.03.11 * `linstor-drbd`: 9.0 * `truenas-incus-ctl`: 0.7.7 ## Additional libraries (and development headers) Incus uses `cowsql` for its database, to build and set it up, you can run `make deps`. Incus itself also uses a number of (usually packaged) C libraries: * `libacl1` * `libcap2` * `libuv1` (for `cowsql`) * `libsqlite3` >= 3.25.0 (for `cowsql`) Make sure you have all these libraries themselves and their development headers (`-dev` packages) installed. incus-7.0.0/doc/rest-api-spec.md000066400000000000000000000015661517523235500164220ustar00rootroot00000000000000# Main API specification
incus-7.0.0/doc/rest-api.md000066400000000000000000000211311517523235500154600ustar00rootroot00000000000000# REST API All communication between Incus and its clients happens using a RESTful API over HTTP. This API is encapsulated over either TLS (for remote operations) or a Unix socket (for local operations). See {ref}`authentication` for information about how to access the API remotely. ```{tip} - For examples on how the API is used, run any command of the Incus client ([`incus`](incus.md)) with the `--debug` flag. The debug information displays the API calls and the return values. - For quickly querying the API, the Incus client provides a [`incus query`](incus_query.md) command. ``` ## API versioning The list of supported major API versions can be retrieved using `GET /`. The reason for a major API bump is if the API breaks backward compatibility. Feature additions done without breaking backward compatibility only result in addition to `api_extensions` which can be used by the client to check if a given feature is supported by the server. ## Return values There are three standard return types: * Standard return value * Background operation * Error ### Standard return value For a standard synchronous operation, the following JSON object is returned: ```js { "type": "sync", "status": "Success", "status_code": 200, "metadata": {} // Extra resource/action specific metadata } ``` HTTP code must be 200. ### Background operation When a request results in a background operation, the HTTP code is set to 202 (Accepted) and the Location HTTP header is set to the operation URL. The body is a JSON object with the following structure: ```js { "type": "async", "status": "OK", "status_code": 100, "operation": "/1.0/instances/", // URL to the background operation "metadata": {} // Operation metadata (see below) } ``` The operation metadata structure looks like: ```js { "id": "a40f5541-5e98-454f-b3b6-8a51ef5dbd3c", // UUID of the operation "class": "websocket", // Class of the operation (task, websocket or token) "created_at": "2015-11-17T22:32:02.226176091-05:00", // When the operation was created "updated_at": "2015-11-17T22:32:02.226176091-05:00", // Last time the operation was updated "status": "Running", // String version of the operation's status "status_code": 103, // Integer version of the operation's status (use this rather than status) "resources": { // Dictionary of resource types (container, snapshots, images) and affected resources "instances": [ "/1.0/instances/test" ] }, "metadata": { // Metadata specific to the operation in question (in this case, exec) "fds": { "0": "2a4a97af81529f6608dca31f03a7b7e47acc0b8dc6514496eb25e325f9e4fa6a", "control": "5b64c661ef313b423b5317ba9cb6410e40b705806c28255f601c0ef603f079a7" } }, "may_cancel": false, // Whether the operation can be canceled (DELETE over REST) "err": "" // The error string should the operation have failed } ``` The body is mostly provided as a user friendly way of seeing what's going on without having to pull the target operation, all information in the body can also be retrieved from the background operation URL. ### Error There are various situations in which something may immediately go wrong, in those cases, the following return value is used: ```js { "type": "error", "error": "Failure", "error_code": 400, "metadata": {} // More details about the error } ``` HTTP code must be one of of 400, 401, 403, 404, 409, 412 or 500. ## Status codes The Incus REST API often has to return status information, be that the reason for an error, the current state of an operation or the state of the various resources it exports. To make it simple to debug, all of those are always doubled. There is a numeric representation of the state which is guaranteed never to change and can be relied on by API clients. Then there is a text version meant to make it easier for people manually using the API to figure out what's happening. In most cases, those will be called status and `status_code`, the former being the user-friendly string representation and the latter the fixed numeric value. The codes are always 3 digits, with the following ranges: * 100 to 199: resource state (started, stopped, ready, ...) * 200 to 399: positive action result * 400 to 599: negative action result * 600 to 999: future use ### List of current status codes | Code | Meaning | | :--- | :--- | | 100 | Operation created | | 101 | Started | | 102 | Stopped | | 103 | Running | | 104 | Canceling | | 105 | Pending | | 106 | Starting | | 107 | Stopping | | 108 | Aborting | | 109 | Freezing | | 110 | Frozen | | 111 | Thawed | | 112 | Error | | 113 | Ready | | 200 | Success | | 400 | Failure | | 401 | Canceled | (rest-api-recursion)= ## Recursion To optimize queries of large lists, recursion is implemented for collections. A `recursion` argument can be passed to a GET query against a collection. The default value is 0 which means that collection member URLs are returned. Setting it to 1 will have those URLs be replaced by the object they point to (typically another JSON object). Recursion is implemented by simply replacing any pointer to an job (URL) by the object itself. (rest-api-filtering)= ## Filtering To filter your results on certain values, filter is implemented for collections. A `filter` argument can be passed to a GET query against a collection. There is no default value for filter which means that all results found will be returned. The following is the language used for the filter argument: ?filter=field_name eq desired_field_assignment The language follows the OData conventions for structuring REST API filtering logic. Logical operators are also supported for filtering: not (`not`), equals (`eq`), not equals (`ne`), and (`and`), or (`or`). Filters are evaluated with left associativity. Values with spaces can be surrounded with quotes. Nesting filtering is also supported. For instance, to filter on a field in a configuration you would pass: ?filter=config.field_name eq desired_field_assignment For filtering on device attributes you would pass: ?filter=devices.device_name.field_name eq desired_field_assignment Here are a few GET query examples of the different filtering methods mentioned above: containers?filter=name eq "my container" and status eq Running containers?filter=config.image.os eq ubuntu or devices.eth0.nictype eq bridged images?filter=Properties.os eq Centos and not UpdateSource.Protocol eq simplestreams ## Asynchronous operations Any operation which may take more than a second to be done must be done in the background, returning a background operation ID to the client. The client will then be able to either poll for a status update or wait for a notification using the long-poll API. ## Notifications A WebSocket-based API is available for notifications, different notification types exist to limit the traffic going to the client. It's recommended that the client always subscribes to the operations notification type before triggering remote operations so that it doesn't have to then poll for their status. ## PUT vs PATCH The Incus API supports both PUT and PATCH to modify existing objects. PUT replaces the entire object with a new definition, it's typically called after the current object state was retrieved through GET. To avoid race conditions, the ETag header should be read from the GET response and sent as If-Match for the PUT request. This will cause Incus to fail the request if the object was modified between GET and PUT. PATCH can be used to modify a single field inside an object by only specifying the property that you want to change. To unset a key, setting it to empty will usually do the trick, but there are cases where PATCH won't work and PUT needs to be used instead. ## API structure Incus has an auto-generated [Swagger](https://swagger.io/) specification describing its API endpoints. The YAML version of this API specification can be found in [`rest-api.yaml`](https://github.com/lxc/incus/blob/main/doc/rest-api.yaml). See {doc}`api` for a convenient web rendering of it. incus-7.0.0/doc/rest-api.yaml000066400000000000000000027022141517523235500160340ustar00rootroot00000000000000definitions: Access: items: $ref: '#/definitions/AccessEntry' title: Access represents everyone that may access a particular resource. type: array x-go-package: github.com/lxc/incus/v7/shared/api AccessEntry: properties: identifier: description: Certificate fingerprint example: 636b69519d27ae3b0e398cb7928043846ce1e3842f0ca7a589993dd913ab8cc9 type: string x-go-name: Identifier provider: description: Which authorization method the certificate uses example: tls, openfga type: string x-go-name: Provider role: description: The role associated with the certificate example: admin, view, operator type: string x-go-name: Role title: AccessEntry represents an entity having access to the resource. type: object x-go-package: github.com/lxc/incus/v7/shared/api BackupTarget: properties: access_key: description: AccessKey is the S3 API access key example: GOOG1234 type: string x-go-name: AccessKey bucket_name: description: BucketName is the name of the S3 bucket. example: my_bucket type: string x-go-name: BucketName path: description: Path is the target path. example: foo/test.tar type: string x-go-name: Path protocol: description: Protocol is the upload protocol. example: S3 type: string x-go-name: Protocol secret_key: description: SecretKey is the S3 API access key example: secret123 type: string x-go-name: SecretKey url: description: URL is the HTTPS URL for the backup example: https://storage.googleapis.com type: string x-go-name: URL title: BackupTarget represents the target storage server for an instance or volume backup. type: object x-go-package: github.com/lxc/incus/v7/shared/api Certificate: description: Certificate represents a certificate properties: certificate: description: The certificate itself, as PEM encoded X509 (or as base64 encoded X509 on POST) example: X509 PEM certificate type: string x-go-name: Certificate description: description: Certificate description example: X509 certificate type: string x-go-name: Description fingerprint: description: SHA256 fingerprint of the certificate example: fd200419b271f1dc2a5591b693cc5774b7f234e1ff8c6b78ad703b6888fe2b69 readOnly: true type: string x-go-name: Fingerprint name: description: Name associated with the certificate example: castiana type: string x-go-name: Name projects: description: List of allowed projects (applies when restricted) example: - default - foo - bar items: type: string type: array x-go-name: Projects restricted: description: Whether to limit the certificate to listed projects example: true type: boolean x-go-name: Restricted type: description: Usage type for the certificate example: client type: string x-go-name: Type type: object x-go-package: github.com/lxc/incus/v7/shared/api CertificateAddToken: properties: addresses: description: The addresses of the server example: - 10.98.30.229:8443 items: type: string type: array x-go-name: Addresses client_name: description: The name of the new client example: user@host type: string x-go-name: ClientName expires_at: description: The token's expiry date. example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: ExpiresAt fingerprint: description: The fingerprint of the network certificate example: 57bb0ff4340b5bb28517e062023101adf788c37846dc8b619eb2c3cb4ef29436 type: string x-go-name: Fingerprint secret: description: The random join secret example: 2b2284d44db32675923fe0d2020477e0e9be11801ff70c435e032b97028c35cd type: string x-go-name: Secret title: CertificateAddToken represents the fields contained within an encoded certificate add token. type: object x-go-package: github.com/lxc/incus/v7/shared/api CertificatePut: description: CertificatePut represents the modifiable fields of a certificate properties: certificate: description: The certificate itself, as PEM encoded X509 (or as base64 encoded X509 on POST) example: X509 PEM certificate type: string x-go-name: Certificate description: description: Certificate description example: X509 certificate type: string x-go-name: Description name: description: Name associated with the certificate example: castiana type: string x-go-name: Name projects: description: List of allowed projects (applies when restricted) example: - default - foo - bar items: type: string type: array x-go-name: Projects restricted: description: Whether to limit the certificate to listed projects example: true type: boolean x-go-name: Restricted type: description: Usage type for the certificate example: client type: string x-go-name: Type type: object x-go-package: github.com/lxc/incus/v7/shared/api CertificatesPost: description: CertificatesPost represents the fields of a new certificate properties: certificate: description: The certificate itself, as PEM encoded X509 (or as base64 encoded X509 on POST) example: X509 PEM certificate type: string x-go-name: Certificate description: description: Certificate description example: X509 certificate type: string x-go-name: Description name: description: Name associated with the certificate example: castiana type: string x-go-name: Name projects: description: List of allowed projects (applies when restricted) example: - default - foo - bar items: type: string type: array x-go-name: Projects restricted: description: Whether to limit the certificate to listed projects example: true type: boolean x-go-name: Restricted token: description: Whether to create a certificate add token example: true type: boolean x-go-name: Token trust_token: description: Trust token (used to add an untrusted client) example: blah type: string x-go-name: TrustToken type: description: Usage type for the certificate example: client type: string x-go-name: Type type: object x-go-package: github.com/lxc/incus/v7/shared/api Cluster: properties: enabled: description: Whether clustering is enabled example: true type: boolean x-go-name: Enabled member_config: description: List of member configuration keys (used during join) example: [] items: $ref: '#/definitions/ClusterMemberConfigKey' type: array x-go-name: MemberConfig server_name: description: Name of the cluster member answering the request example: server01 type: string x-go-name: ServerName title: Cluster represents high-level information about a cluster. type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterCertificatePut: description: ClusterCertificatePut represents the certificate and key pair for all cluster members properties: cluster_certificate: description: The new certificate (X509 PEM encoded) for the cluster example: X509 PEM certificate type: string x-go-name: ClusterCertificate cluster_certificate_key: description: The new certificate key (X509 PEM encoded) for the cluster example: X509 PEM certificate key type: string x-go-name: ClusterCertificateKey type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterGroup: properties: config: description: Cluster group configuration map example: user.mykey: foo type: object x-go-name: Config description: description: The description of the cluster group example: amd64 servers type: string x-go-name: Description members: description: List of members in this group example: - server01 - server02 items: type: string type: array x-go-name: Members name: description: The new name of the cluster group example: group1 type: string x-go-name: Name used_by: description: List of URLs of objects using this cluster group example: - /1.0/cluster/members/server01 - /1.0/project/default items: type: string readOnly: true type: array x-go-name: UsedBy title: ClusterGroup represents a cluster group. type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterGroupPost: properties: name: description: The new name of the cluster group example: group1 type: string x-go-name: Name title: ClusterGroupPost represents the fields required to rename a cluster group. type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterGroupPut: properties: config: description: Cluster group configuration map example: user.mykey: foo type: object x-go-name: Config description: description: The description of the cluster group example: amd64 servers type: string x-go-name: Description members: description: List of members in this group example: - server01 - server02 items: type: string type: array x-go-name: Members title: ClusterGroupPut represents the modifiable fields of a cluster group. type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterGroupsPost: properties: config: description: Cluster group configuration map example: user.mykey: foo type: object x-go-name: Config description: description: The description of the cluster group example: amd64 servers type: string x-go-name: Description members: description: List of members in this group example: - server01 - server02 items: type: string type: array x-go-name: Members name: description: The new name of the cluster group example: group1 type: string x-go-name: Name title: ClusterGroupsPost represents the fields available for a new cluster group. type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterMember: properties: architecture: description: The primary architecture of the cluster member example: x86_64 type: string x-go-name: Architecture config: description: Additional configuration information example: scheduler.instance: all type: object x-go-name: Config database: description: Whether the cluster member is a database server example: true type: boolean x-go-name: Database description: description: Cluster member description example: AMD Epyc 32c/64t type: string x-go-name: Description failure_domain: description: Name of the failure domain for this cluster member example: rack1 type: string x-go-name: FailureDomain groups: description: List of cluster groups this member belongs to example: - group1 - group2 items: type: string type: array x-go-name: Groups message: description: Additional status information example: fully operational type: string x-go-name: Message roles: description: List of roles held by this cluster member example: - database items: type: string type: array x-go-name: Roles server_name: description: Name of the cluster member example: server01 type: string x-go-name: ServerName status: description: Current status example: Online type: string x-go-name: Status url: description: URL at which the cluster member can be reached example: https://10.0.0.1:8443 type: string x-go-name: URL title: ClusterMember represents a member of a cluster. type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterMemberConfigKey: description: |- The Value field is empty when getting clustering information with GET 1.0/cluster, and should be filled by the joining server when performing a PUT 1.0/cluster join request. properties: description: description: A human friendly description key example: '"source" property for storage pool "local"' type: string x-go-name: Description entity: description: The kind of configuration key (network, storage-pool, ...) example: storage-pool type: string x-go-name: Entity key: description: The name of the key example: source type: string x-go-name: Key name: description: The name of the object requiring this key example: local type: string x-go-name: Name value: description: The value on the answering cluster member example: /dev/sdb type: string x-go-name: Value title: |- ClusterMemberConfigKey represents a single config key that a new member of the cluster is required to provide when joining. type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterMemberJoinToken: properties: addresses: description: The addresses of existing online cluster members example: - 10.98.30.229:8443 items: type: string type: array x-go-name: Addresses expires_at: description: The token's expiry date. example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: ExpiresAt fingerprint: description: The fingerprint of the network certificate example: 57bb0ff4340b5bb28517e062023101adf788c37846dc8b619eb2c3cb4ef29436 type: string x-go-name: Fingerprint secret: description: The random join secret. example: 2b2284d44db32675923fe0d2020477e0e9be11801ff70c435e032b97028c35cd type: string x-go-name: Secret server_name: description: The name of the new cluster member example: server02 type: string x-go-name: ServerName title: ClusterMemberJoinToken represents the fields contained within an encoded cluster member join token. type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterMemberPost: properties: server_name: description: The new name of the cluster member example: server02 type: string x-go-name: ServerName title: ClusterMemberPost represents the fields required to rename a cluster member. type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterMemberPut: description: ClusterMemberPut represents the modifiable fields of a cluster member properties: config: description: Additional configuration information example: scheduler.instance: all type: object x-go-name: Config description: description: Cluster member description example: AMD Epyc 32c/64t type: string x-go-name: Description failure_domain: description: Name of the failure domain for this cluster member example: rack1 type: string x-go-name: FailureDomain groups: description: List of cluster groups this member belongs to example: - group1 - group2 items: type: string type: array x-go-name: Groups roles: description: List of roles held by this cluster member example: - database items: type: string type: array x-go-name: Roles type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterMemberState: properties: storage_pools: additionalProperties: $ref: '#/definitions/StoragePoolState' type: object x-go-name: StoragePools sysinfo: $ref: '#/definitions/ClusterMemberSysInfo' title: ClusterMemberState represents the state of a cluster member. type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterMemberStatePost: properties: action: description: The action to be performed. Valid actions are "evacuate" and "restore". example: evacuate type: string x-go-name: Action mode: description: Override the configured evacuation mode. example: stop type: string x-go-name: Mode title: ClusterMemberStatePost represents the fields required to evacuate a cluster member. type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterMemberSysInfo: properties: buffered_ram: format: uint64 type: integer x-go-name: BufferRAM free_ram: format: uint64 type: integer x-go-name: FreeRAM free_swap: format: uint64 type: integer x-go-name: FreeSwap load_averages: items: format: double type: number type: array x-go-name: LoadAverages processes: format: uint16 type: integer x-go-name: Processes shared_ram: format: uint64 type: integer x-go-name: SharedRAM total_ram: format: uint64 type: integer x-go-name: TotalRAM total_swap: format: uint64 type: integer x-go-name: TotalSwap uptime: format: int64 type: integer x-go-name: Uptime title: ClusterMemberSysInfo represents the sysinfo of a cluster member. type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterMembersPost: properties: server_name: description: The name of the new cluster member example: server02 type: string x-go-name: ServerName title: ClusterMembersPost represents the fields required to request a join token to add a member to the cluster. type: object x-go-package: github.com/lxc/incus/v7/shared/api ClusterPut: properties: cluster_address: description: The address of the cluster you wish to join example: 10.0.0.1:8443 type: string x-go-name: ClusterAddress cluster_certificate: description: The expected certificate (X509 PEM encoded) for the cluster example: X509 PEM certificate type: string x-go-name: ClusterCertificate cluster_token: description: The cluster join token for the cluster you're trying to join example: blah type: string x-go-name: ClusterToken enabled: description: Whether clustering is enabled example: true type: boolean x-go-name: Enabled member_config: description: List of member configuration keys (used during join) example: [] items: $ref: '#/definitions/ClusterMemberConfigKey' type: array x-go-name: MemberConfig server_address: description: The local address to use for cluster communication example: 10.0.0.2:8443 type: string x-go-name: ServerAddress server_name: description: Name of the cluster member answering the request example: server01 type: string x-go-name: ServerName title: ClusterPut represents the fields required to bootstrap or join a cluster. type: object x-go-package: github.com/lxc/incus/v7/shared/api ConfigMap: additionalProperties: type: string description: |- ConfigMap type is used to hold incus config. In contrast to plain map[string]string it provides unmarshal methods for JSON and YAML, which gracefully handle numbers and bools. type: object x-go-package: github.com/lxc/incus/v7/shared/api DevicesMap: additionalProperties: additionalProperties: type: string type: object description: |- DevicesMap type is used to hold incus devices configurations. In contrast to plain map[string]map[string]string it provides unmarshal methods for JSON and YAML, which gracefully handle numbers and bools. type: object x-go-package: github.com/lxc/incus/v7/shared/api Event: description: Event represents an event entry (over websocket) properties: location: description: Originating cluster member example: server01 type: string x-go-name: Location metadata: description: JSON encoded metadata (see EventLogging, EventLifecycle or Operation) example: action: instance-started context: {} source: /1.0/instances/c1 type: object x-go-name: Metadata project: description: Project the event belongs to. example: default type: string x-go-name: Project timestamp: description: Time at which the event was sent example: "2021-02-24T19:00:45.452649098-05:00" format: date-time type: string x-go-name: Timestamp type: description: Event type (one of operation, logging or lifecycle) example: lifecycle type: string x-go-name: Type type: object x-go-package: github.com/lxc/incus/v7/shared/api Image: description: Image represents an image properties: aliases: description: List of aliases items: $ref: '#/definitions/ImageAlias' type: array x-go-name: Aliases architecture: description: Architecture example: x86_64 type: string x-go-name: Architecture auto_update: description: Whether the image should auto-update when a new build is available example: true type: boolean x-go-name: AutoUpdate cached: description: Whether the image is an automatically cached remote image example: true type: boolean x-go-name: Cached created_at: description: When the image was originally created example: "2021-03-23T20:00:00-04:00" format: date-time type: string x-go-name: CreatedAt expires_at: description: When the image becomes obsolete example: "2025-03-23T20:00:00-04:00" format: date-time type: string x-go-name: ExpiresAt filename: description: Original filename example: 06b86454720d36b20f94e31c6812e05ec51c1b568cf3a8abd273769d213394bb.rootfs type: string x-go-name: Filename fingerprint: description: Full SHA-256 fingerprint example: 06b86454720d36b20f94e31c6812e05ec51c1b568cf3a8abd273769d213394bb type: string x-go-name: Fingerprint last_used_at: description: Last time the image was used example: "2021-03-22T20:39:00.575185384-04:00" format: date-time type: string x-go-name: LastUsedAt profiles: description: List of profiles to use when creating from this image (if none provided by user) example: - default items: type: string type: array x-go-name: Profiles project: description: Project name example: project1 type: string x-go-name: Project properties: additionalProperties: type: string description: Descriptive properties example: os: Ubuntu release: jammy variant: cloud type: object x-go-name: Properties public: description: Whether the image is available to unauthenticated users example: false type: boolean x-go-name: Public size: description: Size of the image in bytes example: 272237676 format: int64 type: integer x-go-name: Size type: description: Type of image (container or virtual-machine) example: container type: string x-go-name: Type update_source: $ref: '#/definitions/ImageSource' uploaded_at: description: When the image was added to this server example: "2021-03-24T14:18:15.115036787-04:00" format: date-time type: string x-go-name: UploadedAt type: object x-go-package: github.com/lxc/incus/v7/shared/api ImageAlias: description: ImageAlias represents an alias from the alias list of an image properties: description: description: Description of the alias example: Our preferred Ubuntu image type: string x-go-name: Description name: description: Name of the alias example: ubuntu-22.04 type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api ImageAliasesEntry: description: ImageAliasesEntry represents an image alias properties: description: description: Alias description example: Our preferred Ubuntu image type: string x-go-name: Description name: description: Alias name example: ubuntu-22.04 type: string x-go-name: Name target: description: Target fingerprint for the alias example: 06b86454720d36b20f94e31c6812e05ec51c1b568cf3a8abd273769d213394bb type: string x-go-name: Target type: description: Alias type (container or virtual-machine) example: container type: string x-go-name: Type type: object x-go-package: github.com/lxc/incus/v7/shared/api ImageAliasesEntryPost: description: ImageAliasesEntryPost represents the required fields to rename an image alias properties: name: description: Alias name example: ubuntu-22.04 type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api ImageAliasesEntryPut: description: ImageAliasesEntryPut represents the modifiable fields of an image alias properties: description: description: Alias description example: Our preferred Ubuntu image type: string x-go-name: Description target: description: Target fingerprint for the alias example: 06b86454720d36b20f94e31c6812e05ec51c1b568cf3a8abd273769d213394bb type: string x-go-name: Target type: object x-go-package: github.com/lxc/incus/v7/shared/api ImageAliasesPost: description: ImageAliasesPost represents a new image alias properties: description: description: Alias description example: Our preferred Ubuntu image type: string x-go-name: Description name: description: Alias name example: ubuntu-22.04 type: string x-go-name: Name target: description: Target fingerprint for the alias example: 06b86454720d36b20f94e31c6812e05ec51c1b568cf3a8abd273769d213394bb type: string x-go-name: Target type: description: Alias type (container or virtual-machine) example: container type: string x-go-name: Type type: object x-go-package: github.com/lxc/incus/v7/shared/api ImageExportPost: description: ImageExportPost represents the fields required to export an image properties: aliases: description: List of aliases to set on the image items: $ref: '#/definitions/ImageAlias' type: array x-go-name: Aliases certificate: description: Remote server certificate example: X509 PEM certificate type: string x-go-name: Certificate profiles: description: List of profiles to use example: - default items: type: string type: array x-go-name: Profiles project: description: Project name example: project1 type: string x-go-name: Project secret: description: Image receive secret example: RANDOM-STRING type: string x-go-name: Secret target: description: Target server URL example: https://1.2.3.4:8443 type: string x-go-name: Target type: object x-go-package: github.com/lxc/incus/v7/shared/api ImageMetadata: description: ImageMetadata represents image metadata (used in image tarball) properties: architecture: description: Architecture name example: x86_64 type: string x-go-name: Architecture creation_date: description: Image creation data (as UNIX epoch) example: 1620655439 format: int64 type: integer x-go-name: CreationDate expiry_date: description: Image expiry data (as UNIX epoch) example: 1620685757 format: int64 type: integer x-go-name: ExpiryDate properties: additionalProperties: type: string description: Descriptive properties example: os: Ubuntu release: jammy variant: cloud type: object x-go-name: Properties templates: additionalProperties: $ref: '#/definitions/ImageMetadataTemplate' description: Template for files in the image type: object x-go-name: Templates type: object x-go-package: github.com/lxc/incus/v7/shared/api ImageMetadataTemplate: description: ImageMetadataTemplate represents a template entry in image metadata (used in image tarball) properties: create_only: description: Whether to trigger only if the file is missing example: false type: boolean x-go-name: CreateOnly gid: description: The file owner gid. example: "1000" type: string x-go-name: GID mode: description: The file permissions. example: "644" type: string x-go-name: Mode properties: additionalProperties: type: string description: Key/value properties to pass to the template example: foo: bar type: object x-go-name: Properties template: description: The template itself as a valid pongo2 template example: pongo2-template type: string x-go-name: Template uid: description: The file owner uid. example: "1000" type: string x-go-name: UID when: description: When to trigger the template (create, copy or start) example: create items: type: string type: array x-go-name: When type: object x-go-package: github.com/lxc/incus/v7/shared/api ImagePut: description: ImagePut represents the modifiable fields of an image properties: auto_update: description: Whether the image should auto-update when a new build is available example: true type: boolean x-go-name: AutoUpdate expires_at: description: When the image becomes obsolete example: "2025-03-23T20:00:00-04:00" format: date-time type: string x-go-name: ExpiresAt profiles: description: List of profiles to use when creating from this image (if none provided by user) example: - default items: type: string type: array x-go-name: Profiles properties: additionalProperties: type: string description: Descriptive properties example: os: Ubuntu release: jammy variant: cloud type: object x-go-name: Properties public: description: Whether the image is available to unauthenticated users example: false type: boolean x-go-name: Public type: object x-go-package: github.com/lxc/incus/v7/shared/api ImageSource: description: ImageSource represents the source of an image properties: alias: description: Source alias to download from example: jammy type: string x-go-name: Alias certificate: description: Source server certificate (if not trusted by system CA) example: X509 PEM certificate type: string x-go-name: Certificate image_type: description: Type of image (container or virtual-machine) example: container type: string x-go-name: ImageType protocol: description: Source server protocol example: simplestreams type: string x-go-name: Protocol server: description: URL of the source server example: https://images.linuxcontainers.org type: string x-go-name: Server type: object x-go-package: github.com/lxc/incus/v7/shared/api ImagesPost: description: ImagesPost represents the fields available for a new image properties: aliases: description: Aliases to add to the image example: - name: foo - name: bar items: $ref: '#/definitions/ImageAlias' type: array x-go-name: Aliases auto_update: description: Whether the image should auto-update when a new build is available example: true type: boolean x-go-name: AutoUpdate compression_algorithm: description: Compression algorithm to use when turning an instance into an image example: gzip type: string x-go-name: CompressionAlgorithm expires_at: description: When the image becomes obsolete example: "2025-03-23T20:00:00-04:00" format: date-time type: string x-go-name: ExpiresAt filename: description: Original filename of the image example: image.tar.xz type: string x-go-name: Filename format: description: Type of image format example: split type: string x-go-name: Format profiles: description: List of profiles to use when creating from this image (if none provided by user) example: - default items: type: string type: array x-go-name: Profiles properties: additionalProperties: type: string description: Descriptive properties example: os: Ubuntu release: jammy variant: cloud type: object x-go-name: Properties public: description: Whether the image is available to unauthenticated users example: false type: boolean x-go-name: Public source: $ref: '#/definitions/ImagesPostSource' type: object x-go-package: github.com/lxc/incus/v7/shared/api ImagesPostSource: description: ImagesPostSource represents the source of a new image properties: alias: description: Source alias to download from example: jammy type: string x-go-name: Alias certificate: description: Source server certificate (if not trusted by system CA) example: X509 PEM certificate type: string x-go-name: Certificate fingerprint: description: Source image fingerprint (for type "image") example: 8ae945c52bb2f2df51c923b04022312f99bbb72c356251f54fa89ea7cf1df1d0 type: string x-go-name: Fingerprint image_type: description: Type of image (container or virtual-machine) example: container type: string x-go-name: ImageType mode: description: Transfer mode (push or pull) example: pull type: string x-go-name: Mode name: description: Instance name (for type "instance" or "snapshot") example: c1/snap0 type: string x-go-name: Name project: description: Source project name example: project1 type: string x-go-name: Project protocol: description: Source server protocol example: simplestreams type: string x-go-name: Protocol secret: description: Source image server secret token (when downloading private images) example: RANDOM-STRING type: string x-go-name: Secret server: description: URL of the source server example: https://images.linuxcontainers.org type: string x-go-name: Server type: description: Type of image source (instance, snapshot, image or url) example: instance type: string x-go-name: Type url: description: Source URL (for type "url") example: https://some-server.com/some-directory/ type: string x-go-name: URL type: object x-go-package: github.com/lxc/incus/v7/shared/api InitClusterPreseed: properties: cluster_address: description: The address of the cluster you wish to join example: 10.0.0.1:8443 type: string x-go-name: ClusterAddress cluster_certificate: description: The expected certificate (X509 PEM encoded) for the cluster example: X509 PEM certificate type: string x-go-name: ClusterCertificate cluster_certificate_path: description: The path to the cluster certificate example: /tmp/cluster.crt type: string x-go-name: ClusterCertificatePath cluster_token: description: The cluster join token for the cluster you're trying to join example: blah type: string x-go-name: ClusterToken enabled: description: Whether clustering is enabled example: true type: boolean x-go-name: Enabled member_config: description: List of member configuration keys (used during join) example: [] items: $ref: '#/definitions/ClusterMemberConfigKey' type: array x-go-name: MemberConfig server_address: description: The local address to use for cluster communication example: 10.0.0.2:8443 type: string x-go-name: ServerAddress server_name: description: Name of the cluster member answering the request example: server01 type: string x-go-name: ServerName title: InitClusterPreseed represents initialization configuration for the cluster. type: object x-go-package: github.com/lxc/incus/v7/shared/api InitLocalPreseed: properties: certificates: description: Certificates to add example: PEM encoded certificate items: $ref: '#/definitions/CertificatesPost' type: array x-go-name: Certificates cluster_groups: description: |- Cluster groups to add API extension: init_preseed_cluster_groups. items: $ref: '#/definitions/ClusterGroupsPost' type: array x-go-name: ClusterGroups config: description: Server configuration map (refer to doc/server.md) example: core.https_address: :8443 type: object x-go-name: Config networks: description: Networks by project to add example: Network on the "default" project items: $ref: '#/definitions/InitNetworksProjectPost' type: array x-go-name: Networks profiles: description: Profiles to add example: '"default" profile with a root disk device' items: $ref: '#/definitions/InitProfileProjectPost' type: array x-go-name: Profiles projects: description: Projects to add example: '"default" project' items: $ref: '#/definitions/ProjectsPost' type: array x-go-name: Projects storage_pools: description: Storage Pools to add example: local dir storage pool items: $ref: '#/definitions/StoragePoolsPost' type: array x-go-name: StoragePools storage_volumes: description: Storage Volumes to add example: local dir storage volume items: $ref: '#/definitions/InitStorageVolumesProjectPost' type: array x-go-name: StorageVolumes title: InitLocalPreseed represents initialization configuration. type: object x-go-package: github.com/lxc/incus/v7/shared/api InitNetworksProjectPost: properties: Project: description: Project in which the network will reside example: '"default"' type: string config: description: Network configuration map (refer to doc/networks.md) example: ipv4.address: 10.0.0.1/24 ipv4.nat: "true" ipv6.address: none type: object x-go-name: Config description: description: Description of the profile example: My new bridge type: string x-go-name: Description name: description: The name of the new network example: mybr1 type: string x-go-name: Name type: description: The network type (refer to doc/networks.md) example: bridge type: string x-go-name: Type title: InitNetworksProjectPost represents the fields of a new network along with its associated project. type: object x-go-package: github.com/lxc/incus/v7/shared/api InitPreseed: properties: certificates: description: Certificates to add example: PEM encoded certificate items: $ref: '#/definitions/CertificatesPost' type: array x-go-name: Certificates cluster: $ref: '#/definitions/InitClusterPreseed' cluster_groups: description: |- Cluster groups to add API extension: init_preseed_cluster_groups. items: $ref: '#/definitions/ClusterGroupsPost' type: array x-go-name: ClusterGroups config: description: Server configuration map (refer to doc/server.md) example: core.https_address: :8443 type: object x-go-name: Config networks: description: Networks by project to add example: Network on the "default" project items: $ref: '#/definitions/InitNetworksProjectPost' type: array x-go-name: Networks profiles: description: Profiles to add example: '"default" profile with a root disk device' items: $ref: '#/definitions/InitProfileProjectPost' type: array x-go-name: Profiles projects: description: Projects to add example: '"default" project' items: $ref: '#/definitions/ProjectsPost' type: array x-go-name: Projects storage_pools: description: Storage Pools to add example: local dir storage pool items: $ref: '#/definitions/StoragePoolsPost' type: array x-go-name: StoragePools storage_volumes: description: Storage Volumes to add example: local dir storage volume items: $ref: '#/definitions/InitStorageVolumesProjectPost' type: array x-go-name: StorageVolumes title: InitPreseed represents initialization configuration that can be supplied to `init`. type: object x-go-package: github.com/lxc/incus/v7/shared/api InitProfileProjectPost: properties: Project: description: Project in which the profile will reside example: '"default"' type: string config: description: Instance configuration map (refer to doc/instances.md) example: limits.cpu: "4" limits.memory: 4GiB type: object x-go-name: Config description: description: Description of the profile example: Medium size instances type: string x-go-name: Description devices: description: List of devices example: eth0: name: eth0 network: mybr0 type: nic root: path: / pool: default type: disk type: object x-go-name: Devices name: description: The name of the new profile example: foo type: string x-go-name: Name title: InitProfileProjectPost represents the fields of a new profile along with its associated project. type: object x-go-package: github.com/lxc/incus/v7/shared/api InitStorageVolumesProjectPost: properties: Pool: description: Storage pool in which the volume will reside example: '"default"' type: string Project: description: Project in which the volume will reside example: '"default"' type: string config: description: Storage volume configuration map (refer to doc/storage.md) example: size: 50GiB zfs.remove_snapshots: "true" type: object x-go-name: Config content_type: description: Volume content type (filesystem or block) example: filesystem type: string x-go-name: ContentType description: description: Description of the storage volume example: My custom volume type: string x-go-name: Description name: description: Volume name example: foo type: string x-go-name: Name restore: description: Name of a snapshot to restore example: snap0 type: string x-go-name: Restore source: $ref: '#/definitions/StorageVolumeSource' type: description: Volume type (container, custom, image or virtual-machine) example: custom type: string x-go-name: Type title: InitStorageVolumesProjectPost represents the fields of a new storage volume along with its associated pool. type: object x-go-package: github.com/lxc/incus/v7/shared/api Instance: properties: architecture: description: Architecture name example: x86_64 type: string x-go-name: Architecture config: description: Instance configuration (see doc/instances.md) example: security.nesting: "true" type: object x-go-name: Config created_at: description: Instance creation timestamp example: "2021-03-23T20:00:00-04:00" format: date-time type: string x-go-name: CreatedAt description: description: Instance description example: My test instance type: string x-go-name: Description devices: description: Instance devices (see doc/instances.md) example: root: path: / pool: default type: disk type: object x-go-name: Devices disk_only: description: Whether only the instances disk should be restored example: false type: boolean x-go-name: DiskOnly ephemeral: description: Whether the instance is ephemeral (deleted on shutdown) example: false type: boolean x-go-name: Ephemeral expanded_config: description: Expanded configuration (all profiles and local config merged) example: security.nesting: "true" type: object x-go-name: ExpandedConfig expanded_devices: description: Expanded devices (all profiles and local devices merged) example: root: path: / pool: default type: disk type: object x-go-name: ExpandedDevices last_used_at: description: Last start timestamp example: "2021-03-23T20:00:00-04:00" format: date-time type: string x-go-name: LastUsedAt location: description: What cluster member this instance is located on example: server01 type: string x-go-name: Location name: description: Instance name example: foo type: string x-go-name: Name profiles: description: List of profiles applied to the instance example: - default items: type: string type: array x-go-name: Profiles project: description: Instance project name example: foo type: string x-go-name: Project restore: description: If set, instance will be restored to the provided snapshot name example: snap0 type: string x-go-name: Restore stateful: description: Whether the instance currently has saved state on disk example: false type: boolean x-go-name: Stateful status: description: Instance status (see instance_state) example: Running type: string x-go-name: Status status_code: $ref: '#/definitions/StatusCode' type: description: The type of instance (container or virtual-machine) example: container type: string x-go-name: Type title: Instance represents an instance. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceBackup: properties: created_at: description: When the backup was created example: "2021-03-23T16:38:37.753398689-04:00" format: date-time type: string x-go-name: CreatedAt expires_at: description: When the backup expires (gets auto-deleted) example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: ExpiresAt instance_only: description: Whether to ignore snapshots example: false type: boolean x-go-name: InstanceOnly name: description: Backup name example: backup0 type: string x-go-name: Name optimized_storage: description: Whether to use a pool-optimized binary format (instead of plain tarball) example: true type: boolean x-go-name: OptimizedStorage title: InstanceBackup represents an instance backup. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceBackupPost: properties: name: description: New backup name example: backup1 type: string x-go-name: Name title: InstanceBackupPost represents the fields available for the renaming of a instance backup. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceBackupsPost: properties: compression_algorithm: description: What compression algorithm to use example: gzip type: string x-go-name: CompressionAlgorithm expires_at: description: When the backup expires (gets auto-deleted) example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: ExpiresAt instance_only: description: Whether to ignore snapshots example: false type: boolean x-go-name: InstanceOnly name: description: Backup name example: backup0 type: string x-go-name: Name optimized_storage: description: Whether to use a pool-optimized binary format (instead of plain tarball) example: true type: boolean x-go-name: OptimizedStorage root_only: description: Whether to ignore dependent volumes example: false type: boolean x-go-name: RootOnly target: $ref: '#/definitions/BackupTarget' title: InstanceBackupsPost represents the fields available for a new instance backup. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceConsolePost: properties: force: description: Forces a connection to the console example: true type: boolean x-go-name: Force height: description: Console height in rows (console type only) example: 24 format: int64 type: integer x-go-name: Height type: description: Type of console to attach to (console or vga) example: console type: string x-go-name: Type width: description: Console width in columns (console type only) example: 80 format: int64 type: integer x-go-name: Width title: InstanceConsolePost represents an instance console request. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceDebugRepairPost: properties: action: description: The desired repair action. example: rebuild-config-volume type: string x-go-name: Action title: InstanceDebugRepairPost represents an instance repair request. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceExecPost: properties: command: description: Command and its arguments example: - bash items: type: string type: array x-go-name: Command cwd: description: Current working directory for the command example: /home/foo/ type: string x-go-name: Cwd environment: additionalProperties: type: string description: Additional environment to pass to the command example: FOO: BAR type: object x-go-name: Environment group: description: GID of the user to spawn the command as example: 1000 format: uint32 type: integer x-go-name: Group height: description: Terminal height in rows (for interactive) example: 24 format: int64 type: integer x-go-name: Height interactive: description: Whether the command is to be spawned in interactive mode (singled PTY instead of 3 PIPEs) example: true type: boolean x-go-name: Interactive record-output: description: Whether to capture the output for later download (requires non-interactive) type: boolean x-go-name: RecordOutput user: description: UID of the user to spawn the command as example: 1000 format: uint32 type: integer x-go-name: User wait-for-websocket: description: Whether to wait for all websockets to be connected before spawning the command example: true type: boolean x-go-name: WaitForWS width: description: Terminal width in characters (for interactive) example: 80 format: int64 type: integer x-go-name: Width title: InstanceExecPost represents an instance exec request. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceFull: properties: architecture: description: Architecture name example: x86_64 type: string x-go-name: Architecture backups: description: List of backups. items: $ref: '#/definitions/InstanceBackup' type: array x-go-name: Backups config: description: Instance configuration (see doc/instances.md) example: security.nesting: "true" type: object x-go-name: Config created_at: description: Instance creation timestamp example: "2021-03-23T20:00:00-04:00" format: date-time type: string x-go-name: CreatedAt description: description: Instance description example: My test instance type: string x-go-name: Description devices: description: Instance devices (see doc/instances.md) example: root: path: / pool: default type: disk type: object x-go-name: Devices disk_only: description: Whether only the instances disk should be restored example: false type: boolean x-go-name: DiskOnly ephemeral: description: Whether the instance is ephemeral (deleted on shutdown) example: false type: boolean x-go-name: Ephemeral expanded_config: description: Expanded configuration (all profiles and local config merged) example: security.nesting: "true" type: object x-go-name: ExpandedConfig expanded_devices: description: Expanded devices (all profiles and local devices merged) example: root: path: / pool: default type: disk type: object x-go-name: ExpandedDevices last_used_at: description: Last start timestamp example: "2021-03-23T20:00:00-04:00" format: date-time type: string x-go-name: LastUsedAt location: description: What cluster member this instance is located on example: server01 type: string x-go-name: Location name: description: Instance name example: foo type: string x-go-name: Name profiles: description: List of profiles applied to the instance example: - default items: type: string type: array x-go-name: Profiles project: description: Instance project name example: foo type: string x-go-name: Project restore: description: If set, instance will be restored to the provided snapshot name example: snap0 type: string x-go-name: Restore snapshots: description: List of snapshots. items: $ref: '#/definitions/InstanceSnapshot' type: array x-go-name: Snapshots state: $ref: '#/definitions/InstanceState' stateful: description: Whether the instance currently has saved state on disk example: false type: boolean x-go-name: Stateful status: description: Instance status (see instance_state) example: Running type: string x-go-name: Status status_code: $ref: '#/definitions/StatusCode' type: description: The type of instance (container or virtual-machine) example: container type: string x-go-name: Type title: InstanceFull is a combination of Instance, InstanceBackup, InstanceState and InstanceSnapshot. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstancePost: properties: Config: description: Instance configuration file. example: security.nesting: "true" type: object Devices: description: Instance devices. example: root: path: / pool: default type: disk type: object Profiles: description: List of profiles applied to the instance. example: - default items: type: string type: array allow_inconsistent: description: AllowInconsistent allow inconsistent copies when migrating. example: false type: boolean x-go-name: AllowInconsistent instance_only: description: Whether snapshots should be discarded (migration only) example: false type: boolean x-go-name: InstanceOnly live: description: Whether to perform a live migration (migration only) example: false type: boolean x-go-name: Live migration: description: Whether the instance is being migrated to another server example: false type: boolean x-go-name: Migration name: description: New name for the instance example: bar type: string x-go-name: Name pool: description: Target pool for local cross-pool move example: baz type: string x-go-name: Pool project: description: Target project for local cross-project move example: foo type: string x-go-name: Project target: $ref: '#/definitions/InstancePostTarget' title: InstancePost represents the fields required to rename/move an instance. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstancePostTarget: properties: certificate: description: The certificate of the migration target example: X509 PEM certificate type: string x-go-name: Certificate operation: description: The operation URL on the remote target example: https://1.2.3.4:8443/1.0/operations/5e8e1638-5345-4c2d-bac9-2c79c8577292 type: string x-go-name: Operation secrets: additionalProperties: type: string description: Migration websockets credentials example: criu: random-string migration: random-string type: object x-go-name: Websockets title: InstancePostTarget represents the migration target host and operation. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstancePut: properties: architecture: description: Architecture name example: x86_64 type: string x-go-name: Architecture config: description: Instance configuration (see doc/instances.md) example: security.nesting: "true" type: object x-go-name: Config description: description: Instance description example: My test instance type: string x-go-name: Description devices: description: Instance devices (see doc/instances.md) example: root: path: / pool: default type: disk type: object x-go-name: Devices disk_only: description: Whether only the instances disk should be restored example: false type: boolean x-go-name: DiskOnly ephemeral: description: Whether the instance is ephemeral (deleted on shutdown) example: false type: boolean x-go-name: Ephemeral profiles: description: List of profiles applied to the instance example: - default items: type: string type: array x-go-name: Profiles restore: description: If set, instance will be restored to the provided snapshot name example: snap0 type: string x-go-name: Restore stateful: description: Whether the instance currently has saved state on disk example: false type: boolean x-go-name: Stateful title: InstancePut represents the modifiable fields of an instance. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceRebuildPost: properties: source: $ref: '#/definitions/InstanceSource' title: InstanceRebuildPost indicates how to rebuild an instance. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceSnapshot: properties: architecture: description: Architecture name example: x86_64 type: string x-go-name: Architecture config: description: Instance configuration (see doc/instances.md) example: security.nesting: "true" type: object x-go-name: Config created_at: description: Instance creation timestamp example: "2021-03-23T20:00:00-04:00" format: date-time type: string x-go-name: CreatedAt description: description: Instance description example: My description type: string x-go-name: Description devices: description: Instance devices (see doc/instances.md) example: root: path: / pool: default type: disk type: object x-go-name: Devices ephemeral: description: Whether the instance is ephemeral (deleted on shutdown) example: false type: boolean x-go-name: Ephemeral expanded_config: description: Expanded configuration (all profiles and local config merged) example: security.nesting: "true" type: object x-go-name: ExpandedConfig expanded_devices: description: Expanded devices (all profiles and local devices merged) example: root: path: / pool: default type: disk type: object x-go-name: ExpandedDevices expires_at: description: When the snapshot expires (gets auto-deleted) example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: ExpiresAt last_used_at: description: Last start timestamp example: "2021-03-23T20:00:00-04:00" format: date-time type: string x-go-name: LastUsedAt name: description: Snapshot name example: foo type: string x-go-name: Name profiles: description: List of profiles applied to the instance example: - default items: type: string type: array x-go-name: Profiles size: description: Size of the snapshot in bytes example: 143360 format: int64 type: integer x-go-name: Size stateful: description: Whether the instance currently has saved state on disk example: false type: boolean x-go-name: Stateful title: InstanceSnapshot represents an instance snapshot. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceSnapshotPost: properties: live: description: Whether to perform a live migration (requires migration) example: false type: boolean x-go-name: Live migration: description: Whether this is a migration request example: false type: boolean x-go-name: Migration name: description: New name for the snapshot example: foo type: string x-go-name: Name target: $ref: '#/definitions/InstancePostTarget' title: InstanceSnapshotPost represents the fields required to rename/move an instance snapshot. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceSnapshotPut: properties: expires_at: description: When the snapshot expires (gets auto-deleted) example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: ExpiresAt title: InstanceSnapshotPut represents the modifiable fields of an instance snapshot. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceSnapshotsPost: properties: expires_at: description: When the snapshot expires (gets auto-deleted) example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: ExpiresAt name: description: Snapshot name example: snap0 type: string x-go-name: Name stateful: description: Whether the snapshot should include runtime state example: false type: boolean x-go-name: Stateful title: InstanceSnapshotsPost represents the fields available for a new instance snapshot. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceSource: properties: alias: description: Image alias name (for image source) example: ubuntu/22.04 type: string x-go-name: Alias allow_inconsistent: description: Whether to ignore errors when copying (e.g. for volatile files) example: false type: boolean x-go-name: AllowInconsistent base-image: description: Base image fingerprint (for faster migration) example: ed56997f7c5b48e8d78986d2467a26109be6fb9f2d92e8c7b08eb8b6cec7629a type: string x-go-name: BaseImage certificate: description: Certificate (for remote images or migration) example: X509 PEM certificate type: string x-go-name: Certificate fingerprint: description: Image fingerprint (for image source) example: ed56997f7c5b48e8d78986d2467a26109be6fb9f2d92e8c7b08eb8b6cec7629a type: string x-go-name: Fingerprint instance_only: description: Whether the copy should skip the snapshots (for copy) example: false type: boolean x-go-name: InstanceOnly live: description: Whether this is a live migration (for migration) example: false type: boolean x-go-name: Live mode: description: Whether to use pull or push mode (for migration) example: pull type: string x-go-name: Mode operation: description: Remote operation URL (for migration) example: https://1.2.3.4:8443/1.0/operations/1721ae08-b6a8-416a-9614-3f89302466e1 type: string x-go-name: Operation project: description: Source project name (for copy and local image) example: blah type: string x-go-name: Project properties: additionalProperties: type: string description: Image filters (for image source) example: os: Ubuntu release: jammy variant: cloud type: object x-go-name: Properties protocol: description: Protocol name (for remote image) example: simplestreams type: string x-go-name: Protocol refresh: description: Whether this is refreshing an existing instance (for migration and copy) example: false type: boolean x-go-name: Refresh refresh_exclude_older: description: Whether to exclude source snapshots earlier than latest target snapshot example: false type: boolean x-go-name: RefreshExcludeOlder secret: description: Remote server secret (for remote private images) example: RANDOM-STRING type: string x-go-name: Secret secrets: additionalProperties: type: string description: Map of migration websockets (for migration) example: criu: RANDOM-STRING rsync: RANDOM-STRING type: object x-go-name: Websockets server: description: Remote server URL (for remote images) example: https://images.linuxcontainers.org type: string x-go-name: Server source: description: Existing instance name or snapshot (for copy) example: foo/snap0 type: string x-go-name: Source type: description: Source type example: image type: string x-go-name: Type title: InstanceSource represents the creation source for a new instance. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceState: properties: cpu: $ref: '#/definitions/InstanceStateCPU' disk: additionalProperties: $ref: '#/definitions/InstanceStateDisk' description: Disk usage key/value pairs type: object x-go-name: Disk memory: $ref: '#/definitions/InstanceStateMemory' network: additionalProperties: $ref: '#/definitions/InstanceStateNetwork' description: Network usage key/value pairs type: object x-go-name: Network os_info: $ref: '#/definitions/InstanceStateOSInfo' pid: description: PID of the runtime example: 7281 format: int64 type: integer x-go-name: Pid processes: description: Number of processes in the instance example: 50 format: int64 type: integer x-go-name: Processes started_at: description: |- The time that the instance started at API extension: instance_state_started_at. format: date-time type: string x-go-name: StartedAt status: description: Current status (Running, Stopped, Frozen or Error) example: Running type: string x-go-name: Status status_code: $ref: '#/definitions/StatusCode' title: InstanceState represents an instance's state. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceStateCPU: properties: allocated_time: description: CPU time available per second, in nanoseconds example: 4000000000 format: int64 type: integer x-go-name: AllocatedTime usage: description: CPU usage in nanoseconds example: 3637691016 format: int64 type: integer x-go-name: Usage title: InstanceStateCPU represents the cpu information section of an instance's state. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceStateDisk: properties: total: description: Total size in bytes example: 502239232 format: int64 type: integer x-go-name: Total usage: description: Disk usage in bytes example: 502239232 format: int64 type: integer x-go-name: Usage title: InstanceStateDisk represents the disk information section of an instance's state. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceStateMemory: properties: swap_usage: description: SWAP usage in bytes example: 12297557 format: int64 type: integer x-go-name: SwapUsage swap_usage_peak: description: Peak SWAP usage in bytes example: 12297557 format: int64 type: integer x-go-name: SwapUsagePeak total: description: Total memory size in bytes example: 12297557 format: int64 type: integer x-go-name: Total usage: description: Memory usage in bytes example: 73248768 format: int64 type: integer x-go-name: Usage usage_peak: description: Peak memory usage in bytes example: 73785344 format: int64 type: integer x-go-name: UsagePeak title: InstanceStateMemory represents the memory information section of an instance's state. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceStateNetwork: properties: addresses: description: List of IP addresses items: $ref: '#/definitions/InstanceStateNetworkAddress' type: array x-go-name: Addresses counters: $ref: '#/definitions/InstanceStateNetworkCounters' host_name: description: Name of the interface on the host example: vethbbcd39c7 type: string x-go-name: HostName hwaddr: description: MAC address example: 10:66:6a:0c:ee:dd type: string x-go-name: Hwaddr mtu: description: MTU (maximum transmit unit) for the interface example: 1500 format: int64 type: integer x-go-name: Mtu state: description: Administrative state of the interface (up/down) example: up type: string x-go-name: State type: description: Type of interface (broadcast, loopback, point-to-point, ...) example: broadcast type: string x-go-name: Type title: InstanceStateNetwork represents the network information section of an instance's state. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceStateNetworkAddress: description: |- InstanceStateNetworkAddress represents a network address as part of the network section of an instance's state. properties: address: description: IP address example: fd42:4c81:5770:1eaf:1266:6aff:fe0c:eedd type: string x-go-name: Address family: description: Network family (inet or inet6) example: inet6 type: string x-go-name: Family netmask: description: Network mask example: "64" type: string x-go-name: Netmask scope: description: Address scope (local, link or global) example: global type: string x-go-name: Scope type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceStateNetworkCounters: description: |- InstanceStateNetworkCounters represents packet counters as part of the network section of an instance's state. properties: bytes_received: description: Number of bytes received example: 192021 format: int64 type: integer x-go-name: BytesReceived bytes_sent: description: Number of bytes sent example: 10888579 format: int64 type: integer x-go-name: BytesSent errors_received: description: Number of errors received example: 14 format: int64 type: integer x-go-name: ErrorsReceived errors_sent: description: Number of errors sent example: 41 format: int64 type: integer x-go-name: ErrorsSent packets_dropped_inbound: description: Number of inbound packets dropped example: 179 format: int64 type: integer x-go-name: PacketsDroppedInbound packets_dropped_outbound: description: Number of outbound packets dropped example: 541 format: int64 type: integer x-go-name: PacketsDroppedOutbound packets_received: description: Number of packets received example: 1748 format: int64 type: integer x-go-name: PacketsReceived packets_sent: description: Number of packets sent example: 964 format: int64 type: integer x-go-name: PacketsSent type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceStateOSInfo: properties: fqdn: description: FQDN of the instance. example: myhost.mydomain.local type: string x-go-name: FQDN hostname: description: Hostname of the instance. example: myhost type: string x-go-name: Hostname kernel_version: description: Version of the kernel running in the instance. example: 6.1.0-25-amd64 type: string x-go-name: KernelVersion os: description: Operating system running in the instance. example: Debian GNU/Linux type: string x-go-name: OS os_version: description: Version of the operating system. example: 12 (bookworm) type: string x-go-name: OSVersion title: InstanceStateOSInfo represents the operating system information section of an instance's state. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceStatePut: properties: action: description: State change action (start, stop, restart, freeze, unfreeze) example: start type: string x-go-name: Action force: description: Whether to force the action (for stop and restart) example: false type: boolean x-go-name: Force stateful: description: Whether to store the runtime state (for stop) example: false type: boolean x-go-name: Stateful timeout: description: How long to wait (in s) before giving up (when force isn't set) example: 30 format: int64 type: integer x-go-name: Timeout title: InstanceStatePut represents the modifiable fields of an instance's state. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstanceType: title: InstanceType represents the type if instance being returned or requested via the API. type: string x-go-package: github.com/lxc/incus/v7/shared/api InstancesPost: properties: architecture: description: Architecture name example: x86_64 type: string x-go-name: Architecture config: description: Instance configuration (see doc/instances.md) example: security.nesting: "true" type: object x-go-name: Config description: description: Instance description example: My test instance type: string x-go-name: Description devices: description: Instance devices (see doc/instances.md) example: root: path: / pool: default type: disk type: object x-go-name: Devices disk_only: description: Whether only the instances disk should be restored example: false type: boolean x-go-name: DiskOnly ephemeral: description: Whether the instance is ephemeral (deleted on shutdown) example: false type: boolean x-go-name: Ephemeral instance_type: description: Cloud instance type (AWS, GCP, Azure, ...) to emulate with limits example: t1.micro type: string x-go-name: InstanceType name: description: Instance name example: foo type: string x-go-name: Name profiles: description: List of profiles applied to the instance example: - default items: type: string type: array x-go-name: Profiles restore: description: If set, instance will be restored to the provided snapshot name example: snap0 type: string x-go-name: Restore source: $ref: '#/definitions/InstanceSource' start: description: Whether to start the instance after creation example: true type: boolean x-go-name: Start stateful: description: Whether the instance currently has saved state on disk example: false type: boolean x-go-name: Stateful type: $ref: '#/definitions/InstanceType' title: InstancesPost represents the fields available for a new instance. type: object x-go-package: github.com/lxc/incus/v7/shared/api InstancesPut: properties: state: $ref: '#/definitions/InstanceStatePut' title: InstancesPut represents the fields available for a mass update. type: object x-go-package: github.com/lxc/incus/v7/shared/api MetadataConfig: additionalProperties: additionalProperties: $ref: '#/definitions/MetadataConfigGroup' type: object description: MetadataConfig repreents metadata about configuration keys type: object x-go-package: github.com/lxc/incus/v7/shared/api MetadataConfigEntityName: description: MetadataConfigEntityName represents a main API object type example: instance type: string x-go-package: github.com/lxc/incus/v7/shared/api MetadataConfigGroup: description: MetadataConfigGroup represents a group of config keys properties: keys: items: additionalProperties: $ref: '#/definitions/MetadataConfigKey' type: object type: array x-go-name: Keys type: object x-go-package: github.com/lxc/incus/v7/shared/api MetadataConfigGroupName: description: MetadataConfigGroupName represents the name of a group of config keys example: volatile type: string x-go-package: github.com/lxc/incus/v7/shared/api MetadataConfigKey: description: MetadataConfigKey describe a configuration key properties: condition: description: Condition specifies the condition that must be met for the option to be taken into account example: container type: string x-go-name: Condition defaultdesc: description: DefaultDesc specify default value for configuration example: '"`DHCP on eth0`"' type: string x-go-name: Default liveupdate: description: LiveUpdate specifies whether the server must be restarted for the option to be updated example: '"no"' type: string x-go-name: LiveUpdate longdesc: description: LongDesc provides long description for the option example: '"Specify the kernel modules as a comma-separated list."' type: string x-go-name: LongDescription scope: description: Scope defines if option apply to cluster or to the local server example: global type: string x-go-name: Scope shortdesc: description: ShortDesc provides short description for the configuration example: '"Kernel modules to load before starting the instance"' type: string x-go-name: Description type: description: Type specifies the type of the option example: string type: string x-go-name: Type type: object x-go-package: github.com/lxc/incus/v7/shared/api MetadataConfiguration: description: MetadataConfiguration represents a server's exposed configuration metadata properties: configs: $ref: '#/definitions/MetadataConfig' type: object x-go-package: github.com/lxc/incus/v7/shared/api Network: description: Network represents a network properties: config: description: Network configuration map (refer to doc/networks.md) example: ipv4.address: 10.0.0.1/24 ipv4.nat: "true" ipv6.address: none type: object x-go-name: Config description: description: Description of the profile example: My new bridge type: string x-go-name: Description locations: description: Cluster members on which the network has been defined example: - server01 - server02 - server03 items: type: string readOnly: true type: array x-go-name: Locations managed: description: Whether this is a managed network example: true readOnly: true type: boolean x-go-name: Managed name: description: The network name example: mybr0 readOnly: true type: string x-go-name: Name project: description: Project name example: project1 type: string x-go-name: Project status: description: The state of the network (for managed network in clusters) example: Created readOnly: true type: string x-go-name: Status type: description: The network type example: bridge readOnly: true type: string x-go-name: Type used_by: description: List of URLs of objects using this profile example: - /1.0/profiles/default - /1.0/instances/c1 items: type: string readOnly: true type: array x-go-name: UsedBy type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkACL: properties: config: description: ACL configuration map (refer to doc/network-acls.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the ACL example: Web servers type: string x-go-name: Description egress: description: List of egress rules (order independent) items: $ref: '#/definitions/NetworkACLRule' type: array x-go-name: Egress ingress: description: List of ingress rules (order independent) items: $ref: '#/definitions/NetworkACLRule' type: array x-go-name: Ingress name: description: The new name for the ACL example: bar type: string x-go-name: Name project: description: Project name example: project1 type: string x-go-name: Project used_by: description: List of URLs of objects using this profile example: - /1.0/instances/c1 - /1.0/instances/v1 - /1.0/networks/mybr0 items: type: string readOnly: true type: array x-go-name: UsedBy title: NetworkACL used for displaying an ACL. type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkACLPost: properties: name: description: The new name for the ACL example: bar type: string x-go-name: Name title: NetworkACLPost used for renaming an ACL. type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkACLPut: properties: config: description: ACL configuration map (refer to doc/network-acls.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the ACL example: Web servers type: string x-go-name: Description egress: description: List of egress rules (order independent) items: $ref: '#/definitions/NetworkACLRule' type: array x-go-name: Egress ingress: description: List of ingress rules (order independent) items: $ref: '#/definitions/NetworkACLRule' type: array x-go-name: Ingress title: NetworkACLPut used for updating an ACL. type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkACLRule: description: Refer to doc/network-acls.md for details. properties: action: description: Action to perform on rule match example: allow type: string x-go-name: Action description: description: Description of the rule example: Allow DNS queries to Google DNS type: string x-go-name: Description destination: description: Destination address example: 8.8.8.8/32,8.8.4.4/32 type: string x-go-name: Destination destination_port: description: Destination port example: "53" type: string x-go-name: DestinationPort icmp_code: description: ICMP message code (for ICMP protocol) example: "0" type: string x-go-name: ICMPCode icmp_type: description: Type of ICMP message (for ICMP protocol) example: "8" type: string x-go-name: ICMPType protocol: description: Protocol example: udp type: string x-go-name: Protocol source: description: Source address example: '@internal' type: string x-go-name: Source source_port: description: Source port example: "1234" type: string x-go-name: SourcePort state: description: State of the rule example: enabled type: string x-go-name: State title: NetworkACLRule represents a single rule in an ACL ruleset. type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkACLsPost: properties: config: description: ACL configuration map (refer to doc/network-acls.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the ACL example: Web servers type: string x-go-name: Description egress: description: List of egress rules (order independent) items: $ref: '#/definitions/NetworkACLRule' type: array x-go-name: Egress ingress: description: List of ingress rules (order independent) items: $ref: '#/definitions/NetworkACLRule' type: array x-go-name: Ingress name: description: The new name for the ACL example: bar type: string x-go-name: Name title: NetworkACLsPost used for creating an ACL. type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkAddressSet: description: Refer to doc/howto/network_address_sets.md for details. properties: addresses: description: List of addresses in the set example: - 192.0.0.1 - 2001:0db8:1234::1 items: type: string type: array x-go-name: Addresses config: description: Address set configuration map (refer to doc/network-address-sets.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the address set example: Web servers type: string x-go-name: Description name: description: The new name of the address set example: '"bar"' type: string x-go-name: Name project: description: Project name example: project1 type: string x-go-name: Project used_by: description: List of URLs of objects using this profile example: - /1.0/network-acls/foo - /1.0/network-acls/bar - /1.0/network-acls/baz items: type: string readOnly: true type: array x-go-name: UsedBy title: NetworkAddressSet represents an address set. type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkAddressSetPost: properties: name: description: The new name of the address set example: '"bar"' type: string x-go-name: Name title: NetworkAddressSetPost used for renaming an address set. type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkAddressSetPut: properties: addresses: description: List of addresses in the set example: - 192.0.0.1 - 2001:0db8:1234::1 items: type: string type: array x-go-name: Addresses config: description: Address set configuration map (refer to doc/network-address-sets.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the address set example: Web servers type: string x-go-name: Description title: NetworkAddressSetPut used for updating an address set. type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkAddressSetsPost: properties: addresses: description: List of addresses in the set example: - 192.0.0.1 - 2001:0db8:1234::1 items: type: string type: array x-go-name: Addresses config: description: Address set configuration map (refer to doc/network-address-sets.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the address set example: Web servers type: string x-go-name: Description name: description: The new name of the address set example: '"bar"' type: string x-go-name: Name title: NetworkAddressSetsPost used for creating a new address set. type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkAllocations: description: |- NetworkAllocations used for displaying network addresses used by a consuming entity e.g, instance, network forward, load-balancer, network... properties: addresses: description: The network address of the allocation (in CIDR format) example: 192.0.2.1/24 type: string x-go-name: Address hwaddr: description: Hwaddr is the MAC address of the entity consuming the network address type: string x-go-name: Hwaddr nat: description: Whether the entity comes from a network that performs egress source NAT type: boolean x-go-name: NAT type: description: Type of the entity consuming the network address type: string x-go-name: Type used_by: description: Name of the entity consuming the network address type: string x-go-name: UsedBy type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkForward: properties: config: description: Forward configuration map (refer to doc/network-forwards.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the forward listen IP example: My public IP forward type: string x-go-name: Description listen_address: description: The listen address of the forward example: 192.0.2.1 type: string x-go-name: ListenAddress location: description: What cluster member this record was found on example: server01 type: string x-go-name: Location ports: description: Port forwards (optional) items: $ref: '#/definitions/NetworkForwardPort' type: array x-go-name: Ports title: NetworkForward used for displaying an network address forward. type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkForwardPort: description: NetworkForwardPort represents a port specification in a network address forward properties: description: description: Description of the forward port example: My web server forward type: string x-go-name: Description listen_port: description: ListenPort(s) to forward (comma delimited ranges) example: 80,81,8080-8090 type: string x-go-name: ListenPort protocol: description: Protocol for port forward (either tcp or udp) example: tcp type: string x-go-name: Protocol snat: description: SNAT controls whether to apply a matching SNAT rule to new outgoing traffic from the target example: false type: boolean x-go-name: SNAT target_address: description: TargetAddress to forward ListenPorts to example: 198.51.100.2 type: string x-go-name: TargetAddress target_port: description: TargetPort(s) to forward ListenPorts to (allows for many-to-one) example: 80,81,8080-8090 type: string x-go-name: TargetPort type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkForwardPut: description: NetworkForwardPut represents the modifiable fields of a network address forward properties: config: description: Forward configuration map (refer to doc/network-forwards.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the forward listen IP example: My public IP forward type: string x-go-name: Description ports: description: Port forwards (optional) items: $ref: '#/definitions/NetworkForwardPort' type: array x-go-name: Ports type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkForwardsPost: description: NetworkForwardsPost represents the fields of a new network address forward properties: config: description: Forward configuration map (refer to doc/network-forwards.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the forward listen IP example: My public IP forward type: string x-go-name: Description listen_address: description: The listen address of the forward example: 192.0.2.1 type: string x-go-name: ListenAddress ports: description: Port forwards (optional) items: $ref: '#/definitions/NetworkForwardPort' type: array x-go-name: Ports type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkIntegration: properties: config: description: Integration configuration map (refer to doc/network-integrations.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the network integration example: OVN interconnection for region1 type: string x-go-name: Description name: description: The name of the integration example: region1 type: string x-go-name: Name type: description: The type of integration example: ovn type: string x-go-name: Type used_by: description: List of URLs of objects using this network integration example: - /1.0/networks/foo - /1.0/networks/bar items: type: string readOnly: true type: array x-go-name: UsedBy title: NetworkIntegration represents a network integration. type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkIntegrationPost: description: NetworkIntegrationPost represents the fields required to rename a network integration properties: name: description: The new name for the network integration example: region2 type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkIntegrationPut: description: NetworkIntegrationPut represents the modifiable fields of a network integration properties: config: description: Integration configuration map (refer to doc/network-integrations.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the network integration example: OVN interconnection for region1 type: string x-go-name: Description type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkIntegrationsPost: description: NetworkIntegrationsPost represents the fields of a new network integration properties: config: description: Integration configuration map (refer to doc/network-integrations.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the network integration example: OVN interconnection for region1 type: string x-go-name: Description name: description: The name of the integration example: region1 type: string x-go-name: Name type: description: The type of integration example: ovn type: string x-go-name: Type type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkLease: description: NetworkLease represents a DHCP lease properties: address: description: The IP address example: 10.0.0.98 type: string x-go-name: Address hostname: description: The hostname associated with the record example: c1 type: string x-go-name: Hostname hwaddr: description: The MAC address example: 10:66:6a:2c:89:d9 type: string x-go-name: Hwaddr location: description: What cluster member this record was found on example: server01 type: string x-go-name: Location type: description: The type of record (static or dynamic) example: dynamic type: string x-go-name: Type type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkLoadBalancer: description: NetworkLoadBalancer used for displaying a network load balancer properties: backends: description: Backends (optional) items: $ref: '#/definitions/NetworkLoadBalancerBackend' type: array x-go-name: Backends config: description: Load balancer configuration map (refer to doc/network-load-balancers.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the load balancer listen IP example: My public IP load balancer type: string x-go-name: Description listen_address: description: The listen address of the load balancer example: 192.0.2.1 type: string x-go-name: ListenAddress location: description: What cluster member this record was found on example: server01 type: string x-go-name: Location ports: description: Port forwards (optional) items: $ref: '#/definitions/NetworkLoadBalancerPort' type: array x-go-name: Ports type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkLoadBalancerBackend: description: NetworkLoadBalancerBackend represents a target backend specification in a network load balancer properties: description: description: Description of the load balancer backend example: C1 webserver type: string x-go-name: Description name: description: Name of the load balancer backend example: c1-http type: string x-go-name: Name target_address: description: TargetAddress to forward ListenPorts to example: 198.51.100.2 type: string x-go-name: TargetAddress target_port: description: TargetPort(s) to forward ListenPorts to (allows for many-to-one) example: 80,81,8080-8090 type: string x-go-name: TargetPort type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkLoadBalancerPort: description: NetworkLoadBalancerPort represents a port specification in a network load balancer properties: description: description: Description of the load balancer port example: My web server load balancer type: string x-go-name: Description listen_port: description: ListenPort(s) of load balancer (comma delimited ranges) example: 80,81,8080-8090 type: string x-go-name: ListenPort protocol: description: Protocol for load balancer port (either tcp or udp) example: tcp type: string x-go-name: Protocol target_backend: description: TargetBackend backend names to load balance ListenPorts to example: - c1-http - c2-http items: type: string type: array x-go-name: TargetBackend type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkLoadBalancerPut: description: NetworkLoadBalancerPut represents the modifiable fields of a network load balancer properties: backends: description: Backends (optional) items: $ref: '#/definitions/NetworkLoadBalancerBackend' type: array x-go-name: Backends config: description: Load balancer configuration map (refer to doc/network-load-balancers.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the load balancer listen IP example: My public IP load balancer type: string x-go-name: Description ports: description: Port forwards (optional) items: $ref: '#/definitions/NetworkLoadBalancerPort' type: array x-go-name: Ports type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkLoadBalancerState: description: NetworkLoadBalancerState is used for showing current state of a load balancer properties: backend_health: additionalProperties: $ref: '#/definitions/NetworkLoadBalancerStateBackendHealth' type: object x-go-name: BackendHealth type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkLoadBalancerStateBackendHealth: description: NetworkLoadBalancerStateBackendHealth represents the health of a particular load-balancer backend properties: address: type: string x-go-name: Address ports: items: $ref: '#/definitions/NetworkLoadBalancerStateBackendHealthPort' type: array x-go-name: Ports type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkLoadBalancerStateBackendHealthPort: properties: port: format: int64 type: integer x-go-name: Port protocol: type: string x-go-name: Protocol status: type: string x-go-name: Status title: NetworkLoadBalancerStateBackendHealthPort represents the health status of a particular load-balancer backend port. type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkLoadBalancersPost: description: NetworkLoadBalancersPost represents the fields of a new network load balancer properties: backends: description: Backends (optional) items: $ref: '#/definitions/NetworkLoadBalancerBackend' type: array x-go-name: Backends config: description: Load balancer configuration map (refer to doc/network-load-balancers.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the load balancer listen IP example: My public IP load balancer type: string x-go-name: Description listen_address: description: The listen address of the load balancer example: 192.0.2.1 type: string x-go-name: ListenAddress ports: description: Port forwards (optional) items: $ref: '#/definitions/NetworkLoadBalancerPort' type: array x-go-name: Ports type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkPeer: properties: config: description: Peer configuration map (refer to doc/network-peers.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the peer example: Peering with network1 in project1 type: string x-go-name: Description name: description: Name of the peer example: project1-network1 readOnly: true type: string x-go-name: Name status: description: The state of the peering example: Pending readOnly: true type: string x-go-name: Status target_integration: description: Name of the target integration example: ovn-ic1 type: string x-go-name: TargetIntegration target_network: description: Name of the target network example: network1 readOnly: true type: string x-go-name: TargetNetwork target_project: description: Name of the target project example: project1 readOnly: true type: string x-go-name: TargetProject type: description: Type of peer example: local type: string x-go-name: Type used_by: description: List of URLs of objects using this network peering example: - /1.0/network-acls/test - /1.0/network-acls/foo items: type: string readOnly: true type: array x-go-name: UsedBy title: NetworkPeer used for displaying a network peering. type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkPeerPut: description: NetworkPeerPut represents the modifiable fields of a network peering properties: config: description: Peer configuration map (refer to doc/network-peers.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the peer example: Peering with network1 in project1 type: string x-go-name: Description type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkPeersPost: description: NetworkPeersPost represents the fields of a new network peering properties: config: description: Peer configuration map (refer to doc/network-peers.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the peer example: Peering with network1 in project1 type: string x-go-name: Description name: description: Name of the peer example: project1-network1 type: string x-go-name: Name target_integration: description: Name of the target integration example: ovn-ic1 type: string x-go-name: TargetIntegration target_network: description: Name of the target network example: network1 type: string x-go-name: TargetNetwork target_project: description: Name of the target project example: project1 type: string x-go-name: TargetProject type: description: Type of peer example: local type: string x-go-name: Type type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkPost: description: NetworkPost represents the fields required to rename a network properties: name: description: The new name for the network example: mybr1 type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkPut: description: NetworkPut represents the modifiable fields of a network properties: config: description: Network configuration map (refer to doc/networks.md) example: ipv4.address: 10.0.0.1/24 ipv4.nat: "true" ipv6.address: none type: object x-go-name: Config description: description: Description of the profile example: My new bridge type: string x-go-name: Description type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkState: description: NetworkState represents the network state properties: addresses: description: List of addresses items: $ref: '#/definitions/NetworkStateAddress' type: array x-go-name: Addresses bond: $ref: '#/definitions/NetworkStateBond' bridge: $ref: '#/definitions/NetworkStateBridge' counters: $ref: '#/definitions/NetworkStateCounters' hwaddr: description: MAC address example: 10:66:6a:5a:83:57 type: string x-go-name: Hwaddr mtu: description: MTU example: 1500 format: int64 type: integer x-go-name: Mtu ovn: $ref: '#/definitions/NetworkStateOVN' state: description: Link state example: up type: string x-go-name: State type: description: Interface type example: broadcast type: string x-go-name: Type vlan: $ref: '#/definitions/NetworkStateVLAN' type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkStateAddress: description: NetworkStateAddress represents a network address properties: address: description: IP address example: 10.0.0.1 type: string x-go-name: Address family: description: Address family example: inet type: string x-go-name: Family netmask: description: IP netmask (CIDR) example: "24" type: string x-go-name: Netmask scope: description: Address scope example: global type: string x-go-name: Scope type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkStateBond: description: NetworkStateBond represents bond specific state properties: down_delay: description: Delay on link down (ms) example: 0 format: uint64 type: integer x-go-name: DownDelay lower_devices: description: List of devices that are part of the bond example: - eth0 - eth1 items: type: string type: array x-go-name: LowerDevices mii_frequency: description: How often to check for link state (ms) example: 100 format: uint64 type: integer x-go-name: MIIFrequency mii_state: description: Bond link state example: up type: string x-go-name: MIIState mode: description: Bonding mode example: 802.3ad type: string x-go-name: Mode transmit_policy: description: Transmit balancing policy example: layer3+4 type: string x-go-name: TransmitPolicy up_delay: description: Delay on link up (ms) example: 0 format: uint64 type: integer x-go-name: UpDelay type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkStateBridge: description: NetworkStateBridge represents bridge specific state properties: forward_delay: description: Delay on port join (ms) example: 1500 format: uint64 type: integer x-go-name: ForwardDelay id: description: Bridge ID example: 8000.0a0f7c6edbd9 type: string x-go-name: ID stp: description: Whether STP is enabled example: false type: boolean x-go-name: STP upper_devices: description: List of devices that are in the bridge example: - eth0 - eth1 items: type: string type: array x-go-name: UpperDevices vlan_default: description: Default VLAN ID example: 1 format: uint64 type: integer x-go-name: VLANDefault vlan_filtering: description: Whether VLAN filtering is enabled example: false type: boolean x-go-name: VLANFiltering type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkStateCounters: description: NetworkStateCounters represents packet counters properties: bytes_received: description: Number of bytes received example: 250542118 format: int64 type: integer x-go-name: BytesReceived bytes_sent: description: Number of bytes sent example: 17524040140 format: int64 type: integer x-go-name: BytesSent packets_received: description: Number of packets received example: 1182515 format: int64 type: integer x-go-name: PacketsReceived packets_sent: description: Number of packets sent example: 1567934 format: int64 type: integer x-go-name: PacketsSent type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkStateOVN: description: NetworkStateOVN represents OVN specific state properties: chassis: description: OVN network chassis name example: server01 type: string x-go-name: Chassis logical_router: description: OVN logical router name example: incus-net1-lr type: string x-go-name: LogicalRouter logical_switch: description: OVN logical switch name example: incus-net1-ls-int type: string x-go-name: LogicalSwitch uplink_ipv4: description: OVN network uplink ipv4 address example: 10.0.0.1 type: string x-go-name: UplinkIPv4 uplink_ipv6: description: OVN network uplink ipv6 address example: 2001:0000:130F:0000:0000:09C0:876A:130B. type: string x-go-name: UplinkIPv6 type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkStateVLAN: description: NetworkStateVLAN represents VLAN specific state properties: lower_device: description: Parent device example: eth0 type: string x-go-name: LowerDevice vid: description: VLAN ID example: 100 format: uint64 type: integer x-go-name: VID type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkZone: properties: config: description: Zone configuration map (refer to doc/network-zones.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the network zone example: Internal domain type: string x-go-name: Description name: description: The name of the zone (DNS domain name) example: example.net type: string x-go-name: Name project: description: Project name example: project1 type: string x-go-name: Project used_by: description: List of URLs of objects using this network zone example: - /1.0/networks/foo - /1.0/networks/bar items: type: string readOnly: true type: array x-go-name: UsedBy title: NetworkZone represents a network zone (DNS). type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkZonePut: description: NetworkZonePut represents the modifiable fields of a network zone properties: config: description: Zone configuration map (refer to doc/network-zones.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the network zone example: Internal domain type: string x-go-name: Description type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkZoneRecord: properties: config: description: Advanced configuration for the record example: user.mykey: foo type: object x-go-name: Config description: description: Description of the record example: SPF record type: string x-go-name: Description entries: description: Entries in the record items: $ref: '#/definitions/NetworkZoneRecordEntry' type: array x-go-name: Entries name: description: The name of the record example: '@' type: string x-go-name: Name title: NetworkZoneRecord represents a network zone (DNS) record. type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkZoneRecordEntry: description: NetworkZoneRecordEntry represents the fields in a record entry properties: ttl: description: TTL for the entry example: 3600 format: uint64 type: integer x-go-name: TTL type: description: Type of DNS entry example: TXT type: string x-go-name: Type value: description: Value for the record example: v=spf1 mx ~all type: string x-go-name: Value type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkZoneRecordPut: description: NetworkZoneRecordPut represents the modifiable fields of a network zone record properties: config: description: Advanced configuration for the record example: user.mykey: foo type: object x-go-name: Config description: description: Description of the record example: SPF record type: string x-go-name: Description entries: description: Entries in the record items: $ref: '#/definitions/NetworkZoneRecordEntry' type: array x-go-name: Entries type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkZoneRecordsPost: description: NetworkZoneRecordsPost represents the fields of a new network zone record properties: config: description: Advanced configuration for the record example: user.mykey: foo type: object x-go-name: Config description: description: Description of the record example: SPF record type: string x-go-name: Description entries: description: Entries in the record items: $ref: '#/definitions/NetworkZoneRecordEntry' type: array x-go-name: Entries name: description: The record name in the zone example: '@' type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworkZonesPost: description: NetworkZonesPost represents the fields of a new network zone properties: config: description: Zone configuration map (refer to doc/network-zones.md) example: user.mykey: foo type: object x-go-name: Config description: description: Description of the network zone example: Internal domain type: string x-go-name: Description name: description: The name of the zone (DNS domain name) example: example.net type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api NetworksPost: description: NetworksPost represents the fields of a new network properties: config: description: Network configuration map (refer to doc/networks.md) example: ipv4.address: 10.0.0.1/24 ipv4.nat: "true" ipv6.address: none type: object x-go-name: Config description: description: Description of the profile example: My new bridge type: string x-go-name: Description name: description: The name of the new network example: mybr1 type: string x-go-name: Name type: description: The network type (refer to doc/networks.md) example: bridge type: string x-go-name: Type type: object x-go-package: github.com/lxc/incus/v7/shared/api Operation: description: Operation represents a background operation properties: class: description: Type of operation (task, token or websocket) example: websocket type: string x-go-name: Class created_at: description: Operation creation time example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: CreatedAt description: description: Description of the operation example: Executing command type: string x-go-name: Description err: description: Operation error message example: Some error message type: string x-go-name: Err id: description: UUID of the operation example: 6916c8a6-9b7d-4abd-90b3-aedfec7ec7da type: string x-go-name: ID location: description: What cluster member this record was found on example: server01 type: string x-go-name: Location may_cancel: description: Whether the operation can be canceled example: false type: boolean x-go-name: MayCancel metadata: additionalProperties: {} description: Operation specific metadata example: command: - bash environment: HOME: /root LANG: C.UTF-8 PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin TERM: xterm USER: root fds: "0": da3046cf02c0116febf4ef3fe4eaecdf308e720c05e5a9c730ce1a6f15417f66 "1": 05896879d8692607bd6e4a09475667da3b5f6714418ab0ee0e5720b4c57f754b interactive: true type: object x-go-name: Metadata resources: additionalProperties: items: type: string type: array description: Affected resources example: instances: - /1.0/instances/foo type: object x-go-name: Resources status: description: Status name example: Running type: string x-go-name: Status status_code: $ref: '#/definitions/StatusCode' updated_at: description: Operation last change example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: UpdatedAt type: object x-go-package: github.com/lxc/incus/v7/shared/api Profile: description: Profile represents a profile properties: config: description: Instance configuration map (refer to doc/instances.md) example: limits.cpu: "4" limits.memory: 4GiB type: object x-go-name: Config description: description: Description of the profile example: Medium size instances type: string x-go-name: Description devices: description: List of devices example: eth0: name: eth0 network: mybr0 type: nic root: path: / pool: default type: disk type: object x-go-name: Devices name: description: The profile name example: foo readOnly: true type: string x-go-name: Name project: description: Project name example: project1 type: string x-go-name: Project used_by: description: List of URLs of objects using this profile example: - /1.0/instances/c1 - /1.0/instances/v1 items: type: string readOnly: true type: array x-go-name: UsedBy type: object x-go-package: github.com/lxc/incus/v7/shared/api ProfilePost: description: ProfilePost represents the fields required to rename a profile properties: name: description: The new name for the profile example: bar type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api ProfilePut: description: ProfilePut represents the modifiable fields of a profile properties: config: description: Instance configuration map (refer to doc/instances.md) example: limits.cpu: "4" limits.memory: 4GiB type: object x-go-name: Config description: description: Description of the profile example: Medium size instances type: string x-go-name: Description devices: description: List of devices example: eth0: name: eth0 network: mybr0 type: nic root: path: / pool: default type: disk type: object x-go-name: Devices type: object x-go-package: github.com/lxc/incus/v7/shared/api ProfilesPost: description: ProfilesPost represents the fields of a new profile properties: config: description: Instance configuration map (refer to doc/instances.md) example: limits.cpu: "4" limits.memory: 4GiB type: object x-go-name: Config description: description: Description of the profile example: Medium size instances type: string x-go-name: Description devices: description: List of devices example: eth0: name: eth0 network: mybr0 type: nic root: path: / pool: default type: disk type: object x-go-name: Devices name: description: The name of the new profile example: foo type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api Project: description: Project represents a project properties: config: description: Project configuration map (refer to doc/projects.md) example: features.networks: "false" features.profiles: "true" type: object x-go-name: Config description: description: Description of the project example: My new project type: string x-go-name: Description name: description: The project name example: foo readOnly: true type: string x-go-name: Name used_by: description: List of URLs of objects using this project example: - /1.0/images/0e60015346f06627f10580d56ac7fffd9ea775f6d4f25987217d5eed94910a20 - /1.0/instances/c1 - /1.0/networks/mybr0 - /1.0/profiles/default - /1.0/storage-pools/default/volumes/custom/blah items: type: string readOnly: true type: array x-go-name: UsedBy type: object x-go-package: github.com/lxc/incus/v7/shared/api ProjectPost: description: ProjectPost represents the fields required to rename a project properties: name: description: The new name for the project example: bar type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api ProjectPut: description: ProjectPut represents the modifiable fields of a project properties: config: description: Project configuration map (refer to doc/projects.md) example: features.networks: "false" features.profiles: "true" type: object x-go-name: Config description: description: Description of the project example: My new project type: string x-go-name: Description type: object x-go-package: github.com/lxc/incus/v7/shared/api ProjectState: description: ProjectState represents the current running state of a project properties: resources: additionalProperties: $ref: '#/definitions/ProjectStateResource' description: Allocated and used resources example: containers: limit: 10 usage: 4 cpu: limit: 20 usage: 16 readOnly: true type: object x-go-name: Resources type: object x-go-package: github.com/lxc/incus/v7/shared/api ProjectStateResource: description: ProjectStateResource represents the state of a particular resource in a project properties: Limit: description: Limit for the resource (-1 if none) example: 10 format: int64 type: integer Usage: description: Current usage for the resource example: 4 format: int64 type: integer type: object x-go-package: github.com/lxc/incus/v7/shared/api ProjectsPost: description: ProjectsPost represents the fields of a new project properties: config: description: Project configuration map (refer to doc/projects.md) example: features.networks: "false" features.profiles: "true" type: object x-go-name: Config description: description: Description of the project example: My new project type: string x-go-name: Description name: description: The name of the new project example: foo type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api Resources: description: Resources represents the system hardware resources properties: cpu: $ref: '#/definitions/ResourcesCPU' gpu: $ref: '#/definitions/ResourcesGPU' load: $ref: '#/definitions/ResourcesLoad' memory: $ref: '#/definitions/ResourcesMemory' network: $ref: '#/definitions/ResourcesNetwork' pci: $ref: '#/definitions/ResourcesPCI' serial: $ref: '#/definitions/ResourcesSerial' storage: $ref: '#/definitions/ResourcesStorage' system: $ref: '#/definitions/ResourcesSystem' usb: $ref: '#/definitions/ResourcesUSB' type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesCPU: description: ResourcesCPU represents the cpu resources available on the system properties: architecture: description: Architecture name example: x86_64 type: string x-go-name: Architecture sockets: description: List of CPU sockets items: $ref: '#/definitions/ResourcesCPUSocket' type: array x-go-name: Sockets total: description: Total number of CPU threads (from all sockets and cores) example: 1 format: uint64 type: integer x-go-name: Total type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesCPUAddressSizes: properties: physical_bits: format: uint64 type: integer x-go-name: PhysicalBits virtual_bits: format: uint64 type: integer x-go-name: VirtualBits title: ResourcesCPUAddressSizes resprents address size information for a CPU socket. type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesCPUCache: description: ResourcesCPUCache represents a CPU cache properties: level: description: Cache level (usually a number from 1 to 3) example: 1 format: uint64 type: integer x-go-name: Level size: description: Size of the cache (in bytes) example: 32768 format: uint64 type: integer x-go-name: Size type: description: Type of cache (Data, Instruction, Unified, ...) example: Data type: string x-go-name: Type type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesCPUCore: description: ResourcesCPUCore represents a CPU core on the system properties: core: description: Core identifier within the socket example: 0 format: uint64 type: integer x-go-name: Core die: description: What die the CPU is a part of (for chiplet designs) example: 0 format: uint64 type: integer x-go-name: Die flags: description: List of CPU flags example: [] items: type: string type: array x-go-name: Flags frequency: description: Current frequency example: 3500 format: uint64 type: integer x-go-name: Frequency threads: description: List of threads items: $ref: '#/definitions/ResourcesCPUThread' type: array x-go-name: Threads type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesCPUSocket: description: ResourcesCPUSocket represents a CPU socket on the system properties: address_sizes: $ref: '#/definitions/ResourcesCPUAddressSizes' cache: description: List of CPU caches items: $ref: '#/definitions/ResourcesCPUCache' type: array x-go-name: Cache cores: description: List of CPU cores items: $ref: '#/definitions/ResourcesCPUCore' type: array x-go-name: Cores frequency: description: Current CPU frequency (Mhz) example: 3499 format: uint64 type: integer x-go-name: Frequency frequency_minimum: description: Minimum CPU frequency (Mhz) example: 400 format: uint64 type: integer x-go-name: FrequencyMinimum frequency_turbo: description: Maximum CPU frequency (Mhz) example: 3500 format: uint64 type: integer x-go-name: FrequencyTurbo name: description: Product name example: Intel(R) Core(TM) i5-7300U CPU @ 2.60GHz type: string x-go-name: Name socket: description: Socket number example: 0 format: uint64 type: integer x-go-name: Socket vendor: description: Vendor name example: GenuineIntel type: string x-go-name: Vendor type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesCPUThread: description: ResourcesCPUThread represents a CPU thread on the system properties: id: description: Thread ID (used for CPU pinning) example: 0 format: int64 type: integer x-go-name: ID isolated: description: Whether the thread has been isolated (outside of normal scheduling) example: false type: boolean x-go-name: Isolated numa_node: description: NUMA node the thread is a part of example: 0 format: uint64 type: integer x-go-name: NUMANode online: description: Whether the thread is online (enabled) example: true type: boolean x-go-name: Online thread: description: Thread identifier within the core example: 0 format: uint64 type: integer x-go-name: Thread type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesGPU: description: ResourcesGPU represents the GPU resources available on the system properties: cards: description: List of GPUs items: $ref: '#/definitions/ResourcesGPUCard' type: array x-go-name: Cards total: description: Total number of GPUs example: 1 format: uint64 type: integer x-go-name: Total type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesGPUCard: description: ResourcesGPUCard represents a GPU card on the system properties: driver: description: Kernel driver currently associated with the GPU example: i915 type: string x-go-name: Driver driver_version: description: Version of the kernel driver example: 5.8.0-36-generic type: string x-go-name: DriverVersion drm: $ref: '#/definitions/ResourcesGPUCardDRM' mdev: additionalProperties: $ref: '#/definitions/ResourcesGPUCardMdev' description: Map of available mediated device profiles example: null type: object x-go-name: Mdev numa_node: description: NUMA node the GPU is a part of example: 0 format: uint64 type: integer x-go-name: NUMANode nvidia: $ref: '#/definitions/ResourcesGPUCardNvidia' pci_address: description: PCI address example: "0000:00:02.0" type: string x-go-name: PCIAddress product: description: Name of the product example: HD Graphics 620 type: string x-go-name: Product product_id: description: PCI ID of the product example: "5916" type: string x-go-name: ProductID sriov: $ref: '#/definitions/ResourcesGPUCardSRIOV' usb_address: description: USB address (for USB cards) example: "2:7" type: string x-go-name: USBAddress vendor: description: Name of the vendor example: Intel Corporation type: string x-go-name: Vendor vendor_id: description: PCI ID of the vendor example: "8086" type: string x-go-name: VendorID type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesGPUCardDRM: description: ResourcesGPUCardDRM represents the Linux DRM configuration of the GPU properties: card_device: description: Card device number example: "226:0" type: string x-go-name: CardDevice card_name: description: Card device name example: card0 type: string x-go-name: CardName control_device: description: Control device number example: "226:0" type: string x-go-name: ControlDevice control_name: description: Control device name example: controlD64 type: string x-go-name: ControlName id: description: DRM card ID example: 0 format: uint64 type: integer x-go-name: ID render_device: description: Render device number example: 226:128 type: string x-go-name: RenderDevice render_name: description: Render device name example: renderD128 type: string x-go-name: RenderName type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesGPUCardMdev: description: ResourcesGPUCardMdev represents the mediated devices configuration of the GPU properties: api: description: The mechanism used by this device example: vfio-pci type: string x-go-name: API available: description: Number of available devices of this profile example: 2 format: uint64 type: integer x-go-name: Available description: description: Profile description example: 'low_gm_size: 128MB\nhigh_gm_size: 512MB\nfence: 4\nresolution: 1920x1200\nweight: 4' type: string x-go-name: Description devices: description: List of active devices (UUIDs) example: - 42200aac-0977-495c-8c9e-6c51b9092a01 - b4950c00-1437-41d9-88f6-28d61cf9b9ef items: type: string type: array x-go-name: Devices name: description: Profile name example: i915-GVTg_V5_8 type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesGPUCardNvidia: description: ResourcesGPUCardNvidia represents additional information for NVIDIA GPUs properties: architecture: description: Architecture (generation) example: "3.5" type: string x-go-name: Architecture brand: description: Brand name example: GeForce type: string x-go-name: Brand card_device: description: Card device number example: "195:0" type: string x-go-name: CardDevice card_name: description: Card device name example: nvidia0 type: string x-go-name: CardName cuda_version: description: Version of the CUDA API example: "11.0" type: string x-go-name: CUDAVersion model: description: Model name example: GeForce GT 730 type: string x-go-name: Model nvrm_version: description: Version of the NVRM (usually driver version) example: 450.102.04 type: string x-go-name: NVRMVersion uuid: description: GPU UUID example: GPU-6ddadebd-dafe-2db9-f10f-125719770fd3 type: string x-go-name: UUID type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesGPUCardSRIOV: description: ResourcesGPUCardSRIOV represents the SRIOV configuration of the GPU properties: current_vfs: description: Number of VFs currently configured example: 0 format: uint64 type: integer x-go-name: CurrentVFs maximum_vfs: description: Maximum number of supported VFs example: 0 format: uint64 type: integer x-go-name: MaximumVFs vfs: description: List of VFs (as additional GPU devices) example: null items: $ref: '#/definitions/ResourcesGPUCard' type: array x-go-name: VFs type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesLoad: description: ResourcesLoad represents system load information properties: Average1Min: description: Load average in the past minute example: 0.69 format: double type: number Average5Min: description: Load average in the past 5 minutes example: 1.1 format: double type: number Average10Min: description: Load average in the past 10 minutes example: 1.29 format: double type: number Processes: description: The number of active processes example: 1234 format: int64 type: integer type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesMemory: description: ResourcesMemory represents the memory resources available on the system properties: hugepages_size: description: Size of memory huge pages (bytes) example: 2097152 format: uint64 type: integer x-go-name: HugepagesSize hugepages_total: description: Total of memory huge pages (bytes) example: 429284917248 format: uint64 type: integer x-go-name: HugepagesTotal hugepages_used: description: Used memory huge pages (bytes) example: 429284917248 format: uint64 type: integer x-go-name: HugepagesUsed nodes: description: List of NUMA memory nodes example: null items: $ref: '#/definitions/ResourcesMemoryNode' type: array x-go-name: Nodes total: description: Total system memory (bytes) example: 687194767360 format: uint64 type: integer x-go-name: Total used: description: Used system memory (bytes) example: 557450502144 format: uint64 type: integer x-go-name: Used type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesMemoryNode: description: ResourcesMemoryNode represents the node-specific memory resources available on the system properties: hugepages_total: description: Total of memory huge pages (bytes) example: 214536552448 format: uint64 type: integer x-go-name: HugepagesTotal hugepages_used: description: Used memory huge pages (bytes) example: 214536552448 format: uint64 type: integer x-go-name: HugepagesUsed numa_node: description: NUMA node identifier example: 0 format: uint64 type: integer x-go-name: NUMANode total: description: Total system memory (bytes) example: 343597383680 format: uint64 type: integer x-go-name: Total used: description: Used system memory (bytes) example: 264880439296 format: uint64 type: integer x-go-name: Used type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesNetwork: description: ResourcesNetwork represents the network cards available on the system properties: cards: description: List of network cards items: $ref: '#/definitions/ResourcesNetworkCard' type: array x-go-name: Cards total: description: Total number of network cards example: 1 format: uint64 type: integer x-go-name: Total type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesNetworkCard: description: ResourcesNetworkCard represents a network card on the system properties: driver: description: Kernel driver currently associated with the card example: atlantic type: string x-go-name: Driver driver_version: description: Version of the kernel driver example: 5.8.0-36-generic type: string x-go-name: DriverVersion firmware_version: description: Current firmware version example: 3.1.100 type: string x-go-name: FirmwareVersion numa_node: description: NUMA node the card is a part of example: 0 format: uint64 type: integer x-go-name: NUMANode pci_address: description: PCI address (for PCI cards) example: 0000:0d:00.0 type: string x-go-name: PCIAddress ports: description: List of ports on the card items: $ref: '#/definitions/ResourcesNetworkCardPort' type: array x-go-name: Ports product: description: Name of the product example: AQC107 NBase-T/IEEE type: string x-go-name: Product product_id: description: PCI ID of the product example: 87b1 type: string x-go-name: ProductID sriov: $ref: '#/definitions/ResourcesNetworkCardSRIOV' usb_address: description: USB address (for USB cards) example: "2:7" type: string x-go-name: USBAddress vdpa: $ref: '#/definitions/ResourcesNetworkCardVDPA' vendor: description: Name of the vendor example: Aquantia Corp. type: string x-go-name: Vendor vendor_id: description: PCI ID of the vendor example: 1d6a type: string x-go-name: VendorID type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesNetworkCardPort: description: ResourcesNetworkCardPort represents a network port on the system properties: address: description: MAC address example: 00:23:a4:01:01:6f type: string x-go-name: Address auto_negotiation: description: Whether auto negotiation is used example: true type: boolean x-go-name: AutoNegotiation id: description: Port identifier (interface name) example: eth0 type: string x-go-name: ID infiniband: $ref: '#/definitions/ResourcesNetworkCardPortInfiniband' link_detected: description: Whether a link was detected example: true type: boolean x-go-name: LinkDetected link_duplex: description: Duplex type example: full type: string x-go-name: LinkDuplex link_speed: description: Current speed (Mbit/s) example: 10000 format: uint64 type: integer x-go-name: LinkSpeed port: description: Port number example: 0 format: uint64 type: integer x-go-name: Port port_type: description: Current port type example: twisted pair type: string x-go-name: PortType protocol: description: Transport protocol example: ethernet type: string x-go-name: Protocol supported_modes: description: List of supported modes example: - 100baseT/Full - 1000baseT/Full - 2500baseT/Full - 5000baseT/Full - 10000baseT/Full items: type: string type: array x-go-name: SupportedModes supported_ports: description: List of supported port types example: - twisted pair items: type: string type: array x-go-name: SupportedPorts transceiver_type: description: Type of transceiver used example: internal type: string x-go-name: TransceiverType type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesNetworkCardPortInfiniband: description: ResourcesNetworkCardPortInfiniband represents the Linux Infiniband configuration for the port properties: issm_device: description: ISSM device number example: 231:64 type: string x-go-name: IsSMDevice issm_name: description: ISSM device name example: issm0 type: string x-go-name: IsSMName mad_device: description: MAD device number example: "231:0" type: string x-go-name: MADDevice mad_name: description: MAD device name example: umad0 type: string x-go-name: MADName verb_device: description: Verb device number example: 231:192 type: string x-go-name: VerbDevice verb_name: description: Verb device name example: uverbs0 type: string x-go-name: VerbName type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesNetworkCardSRIOV: description: ResourcesNetworkCardSRIOV represents the SRIOV configuration of the network card properties: current_vfs: description: Number of VFs currently configured example: 0 format: uint64 type: integer x-go-name: CurrentVFs maximum_vfs: description: Maximum number of supported VFs example: 0 format: uint64 type: integer x-go-name: MaximumVFs vfs: description: List of VFs (as additional Network devices) example: null items: $ref: '#/definitions/ResourcesNetworkCard' type: array x-go-name: VFs type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesNetworkCardVDPA: description: ResourcesNetworkCardVDPA represents the VDPA configuration of the network card properties: device: description: Device identifier of the VDPA device type: string x-go-name: Device name: description: Name of the VDPA device type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesPCI: description: ResourcesPCI represents the PCI devices available on the system properties: devices: description: List of PCI devices items: $ref: '#/definitions/ResourcesPCIDevice' type: array x-go-name: Devices total: description: Total number of PCI devices example: 1 format: uint64 type: integer x-go-name: Total type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesPCIDevice: description: ResourcesPCIDevice represents a PCI device properties: driver: description: Kernel driver currently associated with the GPU example: mgag200 type: string x-go-name: Driver driver_version: description: Version of the kernel driver example: 5.8.0-36-generic type: string x-go-name: DriverVersion iommu_group: description: IOMMU group number example: 20 format: uint64 type: integer x-go-name: IOMMUGroup numa_node: description: NUMA node the card is a part of example: 0 format: uint64 type: integer x-go-name: NUMANode pci_address: description: PCI address example: "0000:07:03.0" type: string x-go-name: PCIAddress product: description: Name of the product example: MGA G200eW WPCM450 type: string x-go-name: Product product_id: description: PCI ID of the product example: "0532" type: string x-go-name: ProductID vendor: description: Name of the vendor example: Matrox Electronics Systems Ltd. type: string x-go-name: Vendor vendor_id: description: PCI ID of the vendor example: 102b type: string x-go-name: VendorID vpd: $ref: '#/definitions/ResourcesPCIVPD' type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesPCIVPD: description: ResourcesPCIVPD represents VPD entries for a device properties: entries: additionalProperties: type: string description: Vendor provided key/value pairs. example: '{"EC": ""A-5545", "MN": "103C", "V0": "5W PCIeGen2"}' type: object x-go-name: Entries product_name: description: Hardware provided product name. example: HP Ethernet 1Gb 4-port 331i Adapter type: string x-go-name: ProductName type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesSerial: description: ResourcesSerial represents the serial devices available on the system properties: devices: description: List of serial devices items: $ref: '#/definitions/ResourcesSerialDevice' type: array x-go-name: Devices total: description: Total number of serial devices example: 1 format: uint64 type: integer x-go-name: Total type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesSerialDevice: description: ResourcesSerialDevice represents a serial device properties: device: description: Device number (major:minor) example: "188:0" type: string x-go-name: Device device_id: description: Path to /dev/serial/by-id entry example: /dev/serial/by-id/usb-FTDI_FT232R_USB_UART_AB0J1234-if00-port0 type: string x-go-name: DeviceID device_path: description: Path to /dev/serial/by-path entry example: /dev/serial/by-path/pci-0000:00:14.0-usb-0:2:1.0-port0 type: string x-go-name: DevicePath driver: description: kernel driver name (cdc_acm, ftdi_sio, pl2303, cp210x...) example: cdc_acm type: string x-go-name: Driver id: description: Kernel device name (e.g. ttyUSB0, ttyACM0) example: ttyUSB0 type: string x-go-name: ID product: description: USB product name example: Arduino Uno type: string x-go-name: Product product_id: description: USB product ID example: "0043" type: string x-go-name: ProductID vendor: description: USB vendor name example: Arduino LLC type: string x-go-name: Vendor vendor_id: description: USB vendor ID example: "2341" type: string x-go-name: VendorID type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesStorage: description: ResourcesStorage represents the local storage properties: disks: description: List of disks items: $ref: '#/definitions/ResourcesStorageDisk' type: array x-go-name: Disks total: description: Total number of partitions example: 1 format: uint64 type: integer x-go-name: Total type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesStorageDisk: description: ResourcesStorageDisk represents a disk properties: block_size: description: Block size example: 512 format: uint64 type: integer x-go-name: BlockSize device: description: Device number example: "259:0" type: string x-go-name: Device device_id: description: Device by-id identifier example: nvme-eui.0000000001000000e4d25cafae2e4c00 type: string x-go-name: DeviceID device_path: description: Device by-path identifier example: pci-0000:05:00.0-nvme-1 type: string x-go-name: DevicePath firmware_version: description: Current firmware version example: PSF121C type: string x-go-name: FirmwareVersion id: description: ID of the disk (device name) example: nvme0n1 type: string x-go-name: ID model: description: Disk model name example: INTEL SSDPEKKW256G7 type: string x-go-name: Model numa_node: description: NUMA node the disk is a part of example: 0 format: uint64 type: integer x-go-name: NUMANode partitions: description: List of partitions items: $ref: '#/definitions/ResourcesStorageDiskPartition' type: array x-go-name: Partitions pci_address: description: PCI address example: "0000:05:00.0" type: string x-go-name: PCIAddress read_only: description: Whether the disk is read-only example: false type: boolean x-go-name: ReadOnly removable: description: Whether the disk is removable (hot-plug) example: false type: boolean x-go-name: Removable rpm: description: Rotation speed (RPM) example: 0 format: uint64 type: integer x-go-name: RPM serial: description: Serial number example: BTPY63440ARH256D type: string x-go-name: Serial size: description: Total size of the disk (bytes) example: 256060514304 format: uint64 type: integer x-go-name: Size type: description: Storage type example: nvme type: string x-go-name: Type usb_address: description: USB address example: "3:5" type: string x-go-name: USBAddress wwn: description: WWN identifier example: eui.0000000001000000e4d25cafae2e4c00 type: string x-go-name: WWN type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesStorageDiskPartition: description: ResourcesStorageDiskPartition represents a partition on a disk properties: device: description: Device number example: "259:1" type: string x-go-name: Device id: description: ID of the partition (device name) example: nvme0n1p1 type: string x-go-name: ID partition: description: Partition number example: 1 format: uint64 type: integer x-go-name: Partition read_only: description: Whether the partition is read-only example: false type: boolean x-go-name: ReadOnly size: description: Size of the partition (bytes) example: 254933278208 format: uint64 type: integer x-go-name: Size type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesStoragePool: description: ResourcesStoragePool represents the resources available to a given storage pool properties: inodes: $ref: '#/definitions/ResourcesStoragePoolInodes' space: $ref: '#/definitions/ResourcesStoragePoolSpace' type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesStoragePoolInodes: description: ResourcesStoragePoolInodes represents the inodes available to a given storage pool properties: total: description: Total inodes example: 30709993797 format: uint64 type: integer x-go-name: Total used: description: Used inodes example: 23937695 format: uint64 type: integer x-go-name: Used type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesStoragePoolSpace: description: ResourcesStoragePoolSpace represents the space available to a given storage pool properties: total: description: Total disk space (bytes) example: 420100937728 format: uint64 type: integer x-go-name: Total used: description: Used disk space (bytes) example: 343537419776 format: uint64 type: integer x-go-name: Used type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesSystem: description: ResourcesSystem represents the system properties: chassis: $ref: '#/definitions/ResourcesSystemChassis' family: description: System family example: ThinkPad X1 Carbon 5th type: string x-go-name: Family firmware: $ref: '#/definitions/ResourcesSystemFirmware' motherboard: $ref: '#/definitions/ResourcesSystemMotherboard' product: description: System model example: 20HRCTO1WW type: string x-go-name: Product serial: description: System serial number example: PY3DD4X9 type: string x-go-name: Serial sku: description: |- System nanufacturer SKU LENOVO_MT_20HR_BU_Think_FM_ThinkPad X1 Carbon 5th type: string x-go-name: Sku type: description: System type (unknown, physical, virtual-machine, container, ...) example: physical type: string x-go-name: Type uuid: description: System UUID example: 7fa1c0cc-2271-11b2-a85c-aab32a05d71a type: string x-go-name: UUID vendor: description: System vendor example: LENOVO type: string x-go-name: Vendor version: description: System version example: ThinkPad X1 Carbon 5th type: string x-go-name: Version type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesSystemChassis: description: ResourcesSystemChassis represents the system chassis properties: serial: description: Chassis serial number example: PY3DD4X9 type: string x-go-name: Serial type: description: Chassis type example: Notebook type: string x-go-name: Type vendor: description: Chassis vendor example: Lenovo type: string x-go-name: Vendor version: description: Chassis version/revision example: None type: string x-go-name: Version type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesSystemFirmware: description: ResourcesSystemFirmware represents the system firmware properties: date: description: Firmware build date example: 10/14/2020 type: string x-go-name: Date vendor: description: Firmware vendor example: Lenovo type: string x-go-name: Vendor version: description: Firmware version example: N1MET64W (1.49) type: string x-go-name: Version type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesSystemMotherboard: description: ResourcesSystemMotherboard represents the motherboard properties: product: description: Motherboard model example: 20HRCTO1WW type: string x-go-name: Product serial: description: Motherboard serial number example: L3CF4FX003A type: string x-go-name: Serial vendor: description: Motherboard vendor example: Lenovo type: string x-go-name: Vendor version: description: Motherboard version/revision example: None type: string x-go-name: Version type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesUSB: description: ResourcesUSB represents the USB devices available on the system properties: devices: description: List of USB devices items: $ref: '#/definitions/ResourcesUSBDevice' type: array x-go-name: Devices total: description: Total number of USB devices example: 1 format: uint64 type: integer x-go-name: Total type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesUSBDevice: description: ResourcesUSBDevice represents a USB device properties: bus_address: description: USB address (bus) example: 1 format: uint64 type: integer x-go-name: BusAddress device_address: description: USB address (device) example: 3 format: uint64 type: integer x-go-name: DeviceAddress interfaces: description: List of USB interfaces items: $ref: '#/definitions/ResourcesUSBDeviceInterface' type: array x-go-name: Interfaces product: description: Name of the product example: Hermon USB hidmouse Device type: string x-go-name: Product product_id: description: USB ID of the product example: "2221" type: string x-go-name: ProductID serial: description: USB serial number example: DAE005fp type: string x-go-name: Serial speed: description: Transfer speed (Mbit/s) example: 12 format: double type: number x-go-name: Speed vendor: description: Name of the vendor example: ATEN International Co., Ltd type: string x-go-name: Vendor vendor_id: description: USB ID of the vendor example: "0557" type: string x-go-name: VendorID type: object x-go-package: github.com/lxc/incus/v7/shared/api ResourcesUSBDeviceInterface: description: ResourcesUSBDeviceInterface represents a USB device interface properties: class: description: Class of USB interface example: Human Interface Device type: string x-go-name: Class class_id: description: ID of the USB interface class example: 3 format: uint64 type: integer x-go-name: ClassID driver: description: Kernel driver currently associated with the device example: usbhid type: string x-go-name: Driver driver_version: description: Version of the kernel driver example: 5.8.0-36-generic type: string x-go-name: DriverVersion number: description: Interface number example: 0 format: uint64 type: integer x-go-name: Number subclass: description: Sub class of the interface example: Boot Interface Subclass type: string x-go-name: SubClass subclass_id: description: ID of the USB interface sub class example: 1 format: uint64 type: integer x-go-name: SubClassID type: object x-go-package: github.com/lxc/incus/v7/shared/api Server: description: Server represents a server configuration properties: api_extensions: description: List of supported API extensions example: - etag - patch - network - storage items: type: string readOnly: true type: array x-go-name: APIExtensions api_status: description: Support status of the current API (one of "devel", "stable" or "deprecated") example: stable readOnly: true type: string x-go-name: APIStatus api_version: description: API version number example: "1.0" readOnly: true type: string x-go-name: APIVersion auth: description: Whether the client is trusted (one of "trusted" or "untrusted") example: untrusted readOnly: true type: string x-go-name: Auth auth_methods: description: List of supported authentication methods example: - tls items: type: string readOnly: true type: array x-go-name: AuthMethods auth_user_method: description: The current API user login method example: unix readOnly: true type: string x-go-name: AuthUserMethod auth_user_name: description: The current API user identifier example: uid=201105 readOnly: true type: string x-go-name: AuthUserName config: description: Server configuration map (refer to doc/server.md) example: core.https_address: :8443 type: object x-go-name: Config environment: $ref: '#/definitions/ServerEnvironment' public: description: Whether the server is public-only (only public endpoints are implemented) example: false readOnly: true type: boolean x-go-name: Public type: object x-go-package: github.com/lxc/incus/v7/shared/api ServerEnvironment: properties: addresses: description: List of addresses the server is listening on example: - :8443 items: type: string type: array x-go-name: Addresses architectures: description: List of architectures supported by the server example: - x86_64 - i686 items: type: string type: array x-go-name: Architectures certificate: description: Server certificate as PEM encoded X509 example: X509 PEM certificate type: string x-go-name: Certificate certificate_fingerprint: description: Server certificate fingerprint as SHA256 example: fd200419b271f1dc2a5591b693cc5774b7f234e1ff8c6b78ad703b6888fe2b69 type: string x-go-name: CertificateFingerprint driver: description: List of supported instance drivers (separate by " | ") example: lxc | qemu type: string x-go-name: Driver driver_version: description: List of supported instance driver versions (separate by " | ") example: 4.0.7 | 5.2.0 type: string x-go-name: DriverVersion firewall: description: Current firewall driver example: nftables type: string x-go-name: Firewall kernel: description: OS kernel name example: Linux type: string x-go-name: Kernel kernel_architecture: description: OS kernel architecture example: x86_64 type: string x-go-name: KernelArchitecture kernel_features: additionalProperties: type: string description: Map of kernel features that were tested on startup example: netnsid_getifaddrs: "true" seccomp_listener: "true" type: object x-go-name: KernelFeatures kernel_version: description: Kernel version example: 5.4.0-36-generic type: string x-go-name: KernelVersion lxc_features: additionalProperties: type: string description: Map of LXC features that were tested on startup example: cgroup2: "true" devpts_fd: "true" pidfd: "true" type: object x-go-name: LXCFeatures os_name: description: Name of the operating system (Linux distribution) example: Ubuntu type: string x-go-name: OSName os_version: description: Version of the operating system (Linux distribution) example: "22.04" type: string x-go-name: OSVersion project: description: Current project name example: default type: string x-go-name: Project server: description: Server implementation name example: incus type: string x-go-name: Server server_clustered: description: Whether the server is part of a cluster example: false type: boolean x-go-name: ServerClustered server_event_mode: description: |- Mode that the event distribution subsystem is operating in on this server. Either "full-mesh", "hub-server" or "hub-client". example: full-mesh type: string x-go-name: ServerEventMode server_name: description: Server hostname example: castiana type: string x-go-name: ServerName server_pid: description: PID of the daemon example: 1453969 format: int64 type: integer x-go-name: ServerPid server_version: description: Server version example: "4.11" type: string x-go-name: ServerVersion storage: description: List of active storage drivers (separate by " | ") example: dir | zfs type: string x-go-name: Storage storage_supported_drivers: description: List of supported storage drivers items: $ref: '#/definitions/ServerStorageDriverInfo' type: array x-go-name: StorageSupportedDrivers storage_version: description: List of active storage driver versions (separate by " | ") example: 1 | 0.8.4-1ubuntu11 type: string x-go-name: StorageVersion title: ServerEnvironment represents the read-only environment fields of a server configuration. type: object x-go-package: github.com/lxc/incus/v7/shared/api ServerPut: description: ServerPut represents the modifiable fields of a server configuration properties: config: description: Server configuration map (refer to doc/server.md) example: core.https_address: :8443 type: object x-go-name: Config type: object x-go-package: github.com/lxc/incus/v7/shared/api ServerStorageDriverInfo: description: ServerStorageDriverInfo represents the read-only info about a storage driver properties: Name: description: Name of the driver example: zfs type: string Remote: description: Whether the driver has remote volumes example: false type: boolean Version: description: Version of the driver example: 0.8.4-1ubuntu11 type: string type: object x-go-package: github.com/lxc/incus/v7/shared/api ServerUntrusted: description: ServerUntrusted represents a server configuration for an untrusted client properties: api_extensions: description: List of supported API extensions example: - etag - patch - network - storage items: type: string readOnly: true type: array x-go-name: APIExtensions api_status: description: Support status of the current API (one of "devel", "stable" or "deprecated") example: stable readOnly: true type: string x-go-name: APIStatus api_version: description: API version number example: "1.0" readOnly: true type: string x-go-name: APIVersion auth: description: Whether the client is trusted (one of "trusted" or "untrusted") example: untrusted readOnly: true type: string x-go-name: Auth auth_methods: description: List of supported authentication methods example: - tls items: type: string readOnly: true type: array x-go-name: AuthMethods config: description: Server configuration map (refer to doc/server.md) example: core.https_address: :8443 type: object x-go-name: Config public: description: Whether the server is public-only (only public endpoints are implemented) example: false readOnly: true type: boolean x-go-name: Public type: object x-go-package: github.com/lxc/incus/v7/shared/api StatusCode: format: int64 title: StatusCode represents a valid operation and container status. type: integer x-go-package: github.com/lxc/incus/v7/shared/api StorageBucket: description: StorageBucket represents the fields of a storage pool bucket properties: config: description: Storage bucket configuration map example: size: 50GiB type: object x-go-name: Config description: description: Description of the storage bucket example: My custom bucket type: string x-go-name: Description location: description: What cluster member this record was found on example: server01 type: string x-go-name: Location name: description: Bucket name example: foo type: string x-go-name: Name project: description: Project name example: project1 type: string x-go-name: Project s3_url: description: Bucket S3 URL example: https://127.0.0.1:8080/foo type: string x-go-name: S3URL type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageBucketBackup: description: StorageBucketBackup represents the fields available for a new storage bucket backup properties: created_at: description: When the backup was created example: "2021-03-23T16:38:37.753398689-04:00" format: date-time type: string x-go-name: CreatedAt expires_at: description: When the backup expires (gets auto-deleted) example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: ExpiresAt name: description: Backup name example: backup0 type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageBucketBackupPost: description: StorageBucketBackupPost represents the fields available for the renaming of a bucket backup properties: name: description: New backup name example: backup1 type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageBucketBackupsPost: description: StorageBucketBackupsPost represents the fields available for a new storage bucket backup properties: compression_algorithm: description: What compression algorithm to use example: gzip type: string x-go-name: CompressionAlgorithm expires_at: description: When the backup expires (gets auto-deleted) example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: ExpiresAt name: description: Backup name example: backup0 type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageBucketFull: properties: backups: description: List of backups. items: $ref: '#/definitions/StorageBucketBackup' type: array x-go-name: Backups config: description: Storage bucket configuration map example: size: 50GiB type: object x-go-name: Config description: description: Description of the storage bucket example: My custom bucket type: string x-go-name: Description keys: description: List of keys. items: $ref: '#/definitions/StorageBucketKey' type: array x-go-name: Keys location: description: What cluster member this record was found on example: server01 type: string x-go-name: Location name: description: Bucket name example: foo type: string x-go-name: Name project: description: Project name example: project1 type: string x-go-name: Project s3_url: description: Bucket S3 URL example: https://127.0.0.1:8080/foo type: string x-go-name: S3URL title: StorageBucketFull is a combination of StorageBucket, StorageBucketBackup and StorageBucketKey. type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageBucketKey: description: StorageBucketKey represents the fields of a storage pool bucket key properties: access-key: description: Access key example: 33UgkaIBLBIxb7O1 type: string x-go-name: AccessKey description: description: Description of the storage bucket key example: My read-only bucket key type: string x-go-name: Description name: description: Key name example: my-read-only-key type: string x-go-name: Name role: description: Whether the key can perform write actions or not. example: read-only type: string x-go-name: Role secret-key: description: Secret key example: kDQD6AOgwHgaQI1UIJBJpPaiLgZuJbq0 type: string x-go-name: SecretKey type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageBucketKeyPut: description: StorageBucketKeyPut represents the modifiable fields of a storage pool bucket key properties: access-key: description: Access key example: 33UgkaIBLBIxb7O1 type: string x-go-name: AccessKey description: description: Description of the storage bucket key example: My read-only bucket key type: string x-go-name: Description role: description: Whether the key can perform write actions or not. example: read-only type: string x-go-name: Role secret-key: description: Secret key example: kDQD6AOgwHgaQI1UIJBJpPaiLgZuJbq0 type: string x-go-name: SecretKey type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageBucketKeysPost: description: StorageBucketKeysPost represents the fields of a new storage pool bucket key properties: access-key: description: Access key example: 33UgkaIBLBIxb7O1 type: string x-go-name: AccessKey description: description: Description of the storage bucket key example: My read-only bucket key type: string x-go-name: Description name: description: Key name example: my-read-only-key type: string x-go-name: Name role: description: Whether the key can perform write actions or not. example: read-only type: string x-go-name: Role secret-key: description: Secret key example: kDQD6AOgwHgaQI1UIJBJpPaiLgZuJbq0 type: string x-go-name: SecretKey type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageBucketPut: description: StorageBucketPut represents the modifiable fields of a storage pool bucket properties: config: description: Storage bucket configuration map example: size: 50GiB type: object x-go-name: Config description: description: Description of the storage bucket example: My custom bucket type: string x-go-name: Description type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageBucketsPost: description: StorageBucketsPost represents the fields of a new storage pool bucket properties: config: description: Storage bucket configuration map example: size: 50GiB type: object x-go-name: Config description: description: Description of the storage bucket example: My custom bucket type: string x-go-name: Description name: description: Bucket name example: foo type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api StoragePool: properties: config: description: Storage pool configuration map (refer to doc/storage.md) example: volume.block.filesystem: ext4 volume.size: 50GiB type: object x-go-name: Config description: description: Description of the storage pool example: Local SSD pool type: string x-go-name: Description driver: description: Storage pool driver (btrfs, ceph, cephfs, cephobject, dir, lvm, lvmcluster or zfs) example: zfs type: string x-go-name: Driver locations: description: Cluster members on which the storage pool has been defined example: - server01 - server02 - server03 items: type: string readOnly: true type: array x-go-name: Locations name: description: Storage pool name example: local type: string x-go-name: Name status: description: Pool status (Pending, Created, Errored or Unknown) example: Created readOnly: true type: string x-go-name: Status used_by: description: List of URLs of objects using this storage pool example: - /1.0/profiles/default - /1.0/instances/c1 items: type: string type: array x-go-name: UsedBy title: StoragePool represents the fields of a storage pool. type: object x-go-package: github.com/lxc/incus/v7/shared/api StoragePoolPut: properties: config: description: Storage pool configuration map (refer to doc/storage.md) example: volume.block.filesystem: ext4 volume.size: 50GiB type: object x-go-name: Config description: description: Description of the storage pool example: Local SSD pool type: string x-go-name: Description title: StoragePoolPut represents the modifiable fields of a storage pool. type: object x-go-package: github.com/lxc/incus/v7/shared/api StoragePoolState: properties: inodes: $ref: '#/definitions/ResourcesStoragePoolInodes' space: $ref: '#/definitions/ResourcesStoragePoolSpace' title: StoragePoolState represents the state of a storage pool. type: object x-go-package: github.com/lxc/incus/v7/shared/api StoragePoolsPost: description: StoragePoolsPost represents the fields of a new storage pool properties: config: description: Storage pool configuration map (refer to doc/storage.md) example: volume.block.filesystem: ext4 volume.size: 50GiB type: object x-go-name: Config description: description: Description of the storage pool example: Local SSD pool type: string x-go-name: Description driver: description: Storage pool driver (btrfs, ceph, cephfs, cephobject, dir, lvm, lvmcluster or zfs) example: zfs type: string x-go-name: Driver name: description: Storage pool name example: local type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolume: properties: config: description: Storage volume configuration map (refer to doc/storage.md) example: size: 50GiB zfs.remove_snapshots: "true" type: object x-go-name: Config content_type: description: Volume content type (filesystem or block) example: filesystem type: string x-go-name: ContentType created_at: description: Volume creation timestamp example: "2021-03-23T20:00:00-04:00" format: date-time type: string x-go-name: CreatedAt description: description: Description of the storage volume example: My custom volume type: string x-go-name: Description location: description: What cluster member this record was found on example: server01 type: string x-go-name: Location name: description: Volume name example: foo type: string x-go-name: Name project: description: Project containing the volume. example: default type: string x-go-name: Project restore: description: Name of a snapshot to restore example: snap0 type: string x-go-name: Restore type: description: Volume type example: custom type: string x-go-name: Type used_by: description: List of URLs of objects using this storage volume example: - /1.0/instances/blah items: type: string type: array x-go-name: UsedBy title: StorageVolume represents the fields of a storage volume. type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumeBackup: description: StorageVolumeBackup represents a volume backup properties: created_at: description: When the backup was created example: "2021-03-23T16:38:37.753398689-04:00" format: date-time type: string x-go-name: CreatedAt expires_at: description: When the backup expires (gets auto-deleted) example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: ExpiresAt name: description: Backup name example: backup0 type: string x-go-name: Name optimized_storage: description: Whether to use a pool-optimized binary format (instead of plain tarball) example: true type: boolean x-go-name: OptimizedStorage volume_only: description: Whether to ignore snapshots example: false type: boolean x-go-name: VolumeOnly type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumeBackupPost: description: StorageVolumeBackupPost represents the fields available for the renaming of a volume backup properties: name: description: New backup name example: backup1 type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumeBackupsPost: description: StorageVolumeBackupsPost represents the fields available for a new volume backup properties: compression_algorithm: description: What compression algorithm to use example: gzip type: string x-go-name: CompressionAlgorithm expires_at: description: When the backup expires (gets auto-deleted) example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: ExpiresAt name: description: Backup name example: backup0 type: string x-go-name: Name optimized_storage: description: Whether to use a pool-optimized binary format (instead of plain tarball) example: true type: boolean x-go-name: OptimizedStorage target: $ref: '#/definitions/BackupTarget' volume_only: description: Whether to ignore snapshots example: false type: boolean x-go-name: VolumeOnly type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumeBitmap: description: StorageVolumeBitmap represents a volume bitmap properties: busy: description: true if the bitmap is in-use by some operation example: true type: boolean x-go-name: Busy count: description: Number of dirty bytes example: 300 format: int64 type: integer x-go-name: Count granularity: description: Granularity of the dirty bitmap in bytes example: 32768 format: int64 type: integer x-go-name: Granularity inconsistent: description: true if this is a persistent bitmap that was improperly stored example: true type: boolean x-go-name: Inconsistent name: description: Bitmap name example: bitmap0 type: string x-go-name: Name persistent: description: true if the bitmap was stored on disk, is scheduled to be stored on disk, or both example: false type: boolean x-go-name: Persistent recording: description: true if the bitmap is recording new writes from the guest example: false type: boolean x-go-name: Recording type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumeBitmapsPost: description: StorageVolumeBitmapsPost represents the fields available for a new volume bitmap properties: disabled: description: The bitmap is created in the disabled state example: false type: boolean x-go-name: Disabled granularity: description: Granularity of the dirty bitmap in bytes example: 32768 format: int64 type: integer x-go-name: Granularity name: description: Bitmap name example: bitmap0 type: string x-go-name: Name persistent: description: true if the bitmap was stored on disk, is scheduled to be stored on disk, or both example: false type: boolean x-go-name: Persistent type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumeFull: properties: backups: description: List of backups. items: $ref: '#/definitions/StorageVolumeBackup' type: array x-go-name: Backups config: description: Storage volume configuration map (refer to doc/storage.md) example: size: 50GiB zfs.remove_snapshots: "true" type: object x-go-name: Config content_type: description: Volume content type (filesystem or block) example: filesystem type: string x-go-name: ContentType created_at: description: Volume creation timestamp example: "2021-03-23T20:00:00-04:00" format: date-time type: string x-go-name: CreatedAt description: description: Description of the storage volume example: My custom volume type: string x-go-name: Description location: description: What cluster member this record was found on example: server01 type: string x-go-name: Location name: description: Volume name example: foo type: string x-go-name: Name project: description: Project containing the volume. example: default type: string x-go-name: Project restore: description: Name of a snapshot to restore example: snap0 type: string x-go-name: Restore snapshots: description: List of snapshots. items: $ref: '#/definitions/StorageVolumeSnapshot' type: array x-go-name: Snapshots state: $ref: '#/definitions/StorageVolumeState' type: description: Volume type example: custom type: string x-go-name: Type used_by: description: List of URLs of objects using this storage volume example: - /1.0/instances/blah items: type: string type: array x-go-name: UsedBy title: StorageVolumeFull is a combination of StorageVolume, StorageVolumeBackup, StorageVolumeSnapshot and StorageVolumeState. type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumePost: description: StorageVolumePost represents the fields required to rename a storage pool volume properties: migration: description: Initiate volume migration example: false type: boolean x-go-name: Migration name: description: New volume name example: foo type: string x-go-name: Name pool: description: New storage pool example: remote type: string x-go-name: Pool project: description: New project name example: foo type: string x-go-name: Project source: $ref: '#/definitions/StorageVolumeSource' target: $ref: '#/definitions/StorageVolumePostTarget' volume_only: description: Whether snapshots should be discarded (migration only) example: false type: boolean x-go-name: VolumeOnly type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumePostTarget: description: StorageVolumePostTarget represents the migration target host and operation properties: certificate: description: The certificate of the migration target example: X509 PEM certificate type: string x-go-name: Certificate operation: description: Remote operation URL (for migration) example: https://1.2.3.4:8443/1.0/operations/1721ae08-b6a8-416a-9614-3f89302466e1 type: string x-go-name: Operation secrets: additionalProperties: type: string description: Migration websockets credentials example: migration: random-string type: object x-go-name: Websockets type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumePut: description: StorageVolumePut represents the modifiable fields of a storage volume properties: config: description: Storage volume configuration map (refer to doc/storage.md) example: size: 50GiB zfs.remove_snapshots: "true" type: object x-go-name: Config description: description: Description of the storage volume example: My custom volume type: string x-go-name: Description restore: description: Name of a snapshot to restore example: snap0 type: string x-go-name: Restore type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumeSnapshot: description: StorageVolumeSnapshot represents a storage volume snapshot properties: config: description: Storage volume configuration map (refer to doc/storage.md) example: size: 50GiB zfs.remove_snapshots: "true" type: object x-go-name: Config content_type: description: The content type (filesystem or block) example: filesystem type: string x-go-name: ContentType created_at: description: Volume snapshot creation timestamp example: "2021-03-23T20:00:00-04:00" format: date-time type: string x-go-name: CreatedAt description: description: Description of the storage volume example: My custom volume type: string x-go-name: Description expires_at: description: When the snapshot expires (gets auto-deleted) example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: ExpiresAt name: description: Snapshot name example: snap0 type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumeSnapshotPost: description: StorageVolumeSnapshotPost represents the fields required to rename/move a storage volume snapshot properties: migration: description: Initiate volume snapshot migration example: false type: boolean x-go-name: Migration name: description: New snapshot name example: snap1 type: string x-go-name: Name target: $ref: '#/definitions/StorageVolumePostTarget' type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumeSnapshotPut: description: StorageVolumeSnapshotPut represents the modifiable fields of a storage volume properties: description: description: Description of the storage volume example: My custom volume type: string x-go-name: Description expires_at: description: When the snapshot expires (gets auto-deleted) example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: ExpiresAt type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumeSnapshotsPost: description: StorageVolumeSnapshotsPost represents the fields available for a new storage volume snapshot properties: expires_at: description: When the snapshot expires (gets auto-deleted) example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: ExpiresAt name: description: Snapshot name example: snap0 type: string x-go-name: Name type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumeSource: description: StorageVolumeSource represents the creation source for a new storage volume properties: certificate: description: Certificate (for migration) example: X509 PEM certificate type: string x-go-name: Certificate location: description: What cluster member this record was found on example: server01 type: string x-go-name: Location mode: description: Whether to use pull or push mode (for migration) example: pull type: string x-go-name: Mode name: description: Source volume name (for copy) example: foo type: string x-go-name: Name operation: description: Remote operation URL (for migration) example: https://1.2.3.4:8443/1.0/operations/1721ae08-b6a8-416a-9614-3f89302466e1 type: string x-go-name: Operation pool: description: Source storage pool (for copy) example: local type: string x-go-name: Pool project: description: Source project name example: foo type: string x-go-name: Project refresh: description: Whether existing destination volume should be refreshed example: false type: boolean x-go-name: Refresh refresh_exclude_older: description: Whether to exclude source snapshots earlier than latest target snapshot example: false type: boolean x-go-name: RefreshExcludeOlder secrets: additionalProperties: type: string description: Map of migration websockets (for migration) example: rsync: RANDOM-STRING type: object x-go-name: Websockets type: description: Source type (copy or migration) example: copy type: string x-go-name: Type volume_only: description: Whether snapshots should be discarded (for migration) example: false type: boolean x-go-name: VolumeOnly type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumeState: description: StorageVolumeState represents the live state of the volume properties: usage: $ref: '#/definitions/StorageVolumeStateUsage' type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumeStateUsage: description: StorageVolumeStateUsage represents the disk usage of a volume properties: total: description: Storage volume size in bytes example: 5189222192 format: int64 type: integer x-go-name: Total used: description: Used space in bytes example: 1693552640 format: uint64 type: integer x-go-name: Used type: object x-go-package: github.com/lxc/incus/v7/shared/api StorageVolumesPost: description: StorageVolumesPost represents the fields of a new storage pool volume properties: config: description: Storage volume configuration map (refer to doc/storage.md) example: size: 50GiB zfs.remove_snapshots: "true" type: object x-go-name: Config content_type: description: Volume content type (filesystem or block) example: filesystem type: string x-go-name: ContentType description: description: Description of the storage volume example: My custom volume type: string x-go-name: Description name: description: Volume name example: foo type: string x-go-name: Name restore: description: Name of a snapshot to restore example: snap0 type: string x-go-name: Restore source: $ref: '#/definitions/StorageVolumeSource' type: description: Volume type (container, custom, image or virtual-machine) example: custom type: string x-go-name: Type type: object x-go-package: github.com/lxc/incus/v7/shared/api Warning: properties: count: description: The number of times this warning occurred example: 1 format: int64 type: integer x-go-name: Count entity_url: description: The entity affected by this warning example: /1.0/instances/c1?project=default type: string x-go-name: EntityURL first_seen_at: description: The first time this warning occurred example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: FirstSeenAt last_message: description: The warning message example: Couldn't find the CGroup blkio.weight, disk priority will be ignored type: string x-go-name: LastMessage last_seen_at: description: The last time this warning occurred example: "2021-03-23T17:38:37.753398689-04:00" format: date-time type: string x-go-name: LastSeenAt location: description: What cluster member this warning occurred on example: server01 type: string x-go-name: Location project: description: The project the warning occurred in example: default type: string x-go-name: Project severity: description: The severity of this warning example: low type: string x-go-name: Severity status: description: Status of the warning (new, acknowledged, or resolved) example: new type: string x-go-name: Status type: description: Type type of warning example: Couldn't find CGroup type: string x-go-name: Type uuid: description: UUID of the warning example: e9e9da0d-2538-4351-8047-46d4a8ae4dbb type: string x-go-name: UUID title: Warning represents a warning entry. type: object x-go-package: github.com/lxc/incus/v7/shared/api WarningPut: properties: status: description: Status of the warning (new, acknowledged, or resolved) example: new type: string x-go-name: Status title: WarningPut represents the modifiable fields of a warning. type: object x-go-package: github.com/lxc/incus/v7/shared/api info: contact: email: lxc-devel@lists.linuxcontainers.org name: Incus upstream url: https://github.com/lxc/incus description: |- This is the REST API used by all Incus clients. Internal endpoints aren't included in this documentation. The Incus API is available over both a local unix+http and remote https API. Authentication for local users relies on group membership and access to the unix socket. For remote users, the default authentication method is TLS client certificates. license: name: Apache-2.0 url: https://www.apache.org/licenses/LICENSE-2.0 title: Incus external REST API version: "1.0" paths: /: get: description: |- Returns a list of supported API versions (URLs). Internal API endpoints are not reported as those aren't versioned and should only be used by the daemon itself. operationId: api_get produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: - /1.0 items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object summary: Get the supported API endpoints tags: - server /1.0: get: description: Shows the full server environment and configuration. operationId: server_get parameters: - description: Cluster member name example: server01 in: query name: target type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Server environment and configuration schema: description: Sync response properties: metadata: $ref: '#/definitions/Server' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "500": $ref: '#/responses/InternalServerError' summary: Get the server environment and configuration tags: - server patch: consumes: - application/json description: Updates a subset of the server configuration. operationId: server_patch parameters: - description: Cluster member name example: server01 in: query name: target type: string - description: Server configuration in: body name: server required: true schema: $ref: '#/definitions/ServerPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the server configuration tags: - server put: consumes: - application/json description: Updates the entire server configuration. operationId: server_put parameters: - description: Cluster member name example: server01 in: query name: target type: string - description: Server configuration in: body name: server required: true schema: $ref: '#/definitions/ServerPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the server configuration tags: - server /1.0/certificates: get: description: Returns a list of trusted certificates (URLs). operationId: certificates_get parameters: - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/certificates/390fdd27ed5dc2408edc11fe602eafceb6c025ddbad9341dfdcb1056a8dd98b1", "/1.0/certificates/22aee3f051f96abe6d7756892eecabf4b4b22e2ba877840a4ca981e9ea54030a" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the trusted certificates tags: - certificates post: consumes: - application/json description: |- Adds a certificate to the trust store. In this mode, the `token` property is always ignored. operationId: certificates_post parameters: - description: Certificate in: body name: certificate required: true schema: $ref: '#/definitions/CertificatesPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a trusted certificate tags: - certificates /1.0/certificates/{fingerprint}: delete: description: Removes the certificate from the trust store. operationId: certificate_delete parameters: - description: Fingerprint in: path name: fingerprint required: true type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the trusted certificate tags: - certificates get: description: Gets a specific certificate entry from the trust store. operationId: certificate_get parameters: - description: Fingerprint in: path name: fingerprint required: true type: string produces: - application/json responses: "200": description: Certificate schema: description: Sync response properties: metadata: $ref: '#/definitions/Certificate' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the trusted certificate tags: - certificates patch: consumes: - application/json description: Updates a subset of the certificate configuration. operationId: certificate_patch parameters: - description: Fingerprint in: path name: fingerprint required: true type: string - description: Certificate configuration in: body name: certificate required: true schema: $ref: '#/definitions/CertificatePut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the trusted certificate tags: - certificates put: consumes: - application/json description: Updates the entire certificate configuration. operationId: certificate_put parameters: - description: Fingerprint in: path name: fingerprint required: true type: string - description: Certificate configuration in: body name: certificate required: true schema: $ref: '#/definitions/CertificatePut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the trusted certificate tags: - certificates /1.0/certificates?public: post: consumes: - application/json description: |- Adds a certificate to the trust store as an untrusted user. In this mode, the `token` property must be set to the correct value. The `certificate` field can be omitted in which case the TLS client certificate in use for the connection will be retrieved and added to the trust store. The `?public` part of the URL isn't required, it's simply used to separate the two behaviors of this endpoint. operationId: certificates_post_untrusted parameters: - description: Certificate in: body name: certificate required: true schema: $ref: '#/definitions/CertificatesPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a trusted certificate tags: - certificates /1.0/certificates?recursion=1: get: description: Returns a list of trusted certificates (structs). operationId: certificates_get_recursion1 parameters: - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of certificates items: $ref: '#/definitions/Certificate' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the trusted certificates tags: - certificates /1.0/cluster: get: description: Gets the current cluster configuration. operationId: cluster_get produces: - application/json responses: "200": description: Cluster configuration schema: description: Sync response properties: metadata: $ref: '#/definitions/Cluster' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the cluster configuration tags: - cluster put: consumes: - application/json description: Updates the entire cluster configuration. operationId: cluster_put parameters: - description: Cluster configuration in: body name: cluster required: true schema: $ref: '#/definitions/ClusterPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the cluster configuration tags: - cluster /1.0/cluster/certificate: put: consumes: - application/json description: Replaces existing cluster certificate and reloads each cluster member. operationId: clustering_update_cert parameters: - description: Cluster certificate replace request in: body name: cluster required: true schema: $ref: '#/definitions/ClusterCertificatePut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Update the certificate for the cluster tags: - cluster /1.0/cluster/groups: get: description: Returns a list of cluster groups (URLs). operationId: cluster_groups_get produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/cluster/groups/server01", "/1.0/cluster/groups/server02" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the cluster groups tags: - cluster-groups post: consumes: - application/json description: Creates a new cluster group. operationId: cluster_groups_post parameters: - description: Cluster group to create in: body name: cluster required: true schema: $ref: '#/definitions/ClusterGroupsPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Create a cluster group. tags: - cluster /1.0/cluster/groups/{name}: delete: description: Removes the cluster group. operationId: cluster_group_delete parameters: - description: Cluster group name in: path name: name required: true type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the cluster group. tags: - cluster-groups get: description: Gets a specific cluster group. operationId: cluster_group_get parameters: - description: Cluster group name in: path name: name required: true type: string produces: - application/json responses: "200": description: Cluster group schema: description: Sync response properties: metadata: $ref: '#/definitions/ClusterGroup' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the cluster group tags: - cluster-groups patch: consumes: - application/json description: Updates the cluster group configuration. operationId: cluster_group_patch parameters: - description: Cluster group name in: path name: name required: true type: string - description: cluster group configuration in: body name: cluster group required: true schema: $ref: '#/definitions/ClusterGroupPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the cluster group tags: - cluster-groups post: consumes: - application/json description: Renames an existing cluster group. operationId: cluster_group_post parameters: - description: Cluster group name in: path name: name required: true type: string - description: Cluster group rename request in: body name: name required: true schema: $ref: '#/definitions/ClusterGroupPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename the cluster group tags: - cluster-groups put: consumes: - application/json description: Updates the entire cluster group configuration. operationId: cluster_group_put parameters: - description: Cluster group name in: path name: name required: true type: string - description: cluster group configuration in: body name: cluster group required: true schema: $ref: '#/definitions/ClusterGroupPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the cluster group tags: - cluster-groups /1.0/cluster/groups?recursion=1: get: description: Returns a list of cluster groups (structs). operationId: cluster_groups_get_recursion1 produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of cluster groups items: $ref: '#/definitions/ClusterGroup' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the cluster groups tags: - cluster-groups /1.0/cluster/members: get: description: Returns a list of cluster members (URLs). operationId: cluster_members_get parameters: - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/cluster/members/server01", "/1.0/cluster/members/server02" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the cluster members tags: - cluster post: consumes: - application/json description: Requests a join token to add a cluster member. operationId: cluster_members_post parameters: - description: Cluster member add request in: body name: cluster required: true schema: $ref: '#/definitions/ClusterMembersPost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Request a join token tags: - cluster /1.0/cluster/members/{name}: delete: description: Removes the member from the cluster. operationId: cluster_member_delete parameters: - description: Cluster member name in: path name: name required: true type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the cluster member tags: - cluster get: description: Gets a specific cluster member. operationId: cluster_member_get parameters: - description: Cluster member name in: path name: name required: true type: string produces: - application/json responses: "200": description: Cluster member schema: description: Sync response properties: metadata: $ref: '#/definitions/ClusterMember' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the cluster member tags: - cluster patch: consumes: - application/json description: Updates a subset of the cluster member configuration. operationId: cluster_member_patch parameters: - description: Cluster member name in: path name: name required: true type: string - description: Cluster member configuration in: body name: cluster required: true schema: $ref: '#/definitions/ClusterMemberPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the cluster member tags: - cluster post: consumes: - application/json description: Renames an existing cluster member. operationId: cluster_member_post parameters: - description: Cluster member name in: path name: name required: true type: string - description: Cluster member rename request in: body name: cluster required: true schema: $ref: '#/definitions/ClusterMemberPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename the cluster member tags: - cluster put: consumes: - application/json description: Updates the entire cluster member configuration. operationId: cluster_member_put parameters: - description: Cluster member name in: path name: name required: true type: string - description: Cluster member configuration in: body name: cluster required: true schema: $ref: '#/definitions/ClusterMemberPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the cluster member tags: - cluster /1.0/cluster/members/{name}/state: get: description: Gets state of a specific cluster member. operationId: cluster_member_state_get parameters: - description: Cluster member name in: path name: name required: true type: string produces: - application/json responses: "200": description: Cluster member state schema: description: Sync response properties: metadata: $ref: '#/definitions/ClusterMemberState' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get state of the cluster member tags: - cluster post: consumes: - application/json description: Evacuates or restores a cluster member. operationId: cluster_member_state_post parameters: - description: Cluster member name in: path name: name required: true type: string - description: Cluster member state in: body name: cluster required: true schema: $ref: '#/definitions/ClusterMemberStatePost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Evacuate or restore a cluster member tags: - cluster /1.0/cluster/members?recursion=1: get: description: Returns a list of cluster members (structs). operationId: cluster_members_get_recursion1 parameters: - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of cluster members items: $ref: '#/definitions/ClusterMember' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the cluster members tags: - cluster /1.0/events: get: description: Connects to the event API using websocket. operationId: events_get parameters: - description: Project name example: default in: query name: project type: string - description: Event type(s), comma separated (valid types are logging, operation or lifecycle) example: logging,lifecycle in: query name: type type: string - description: Retrieve instances from all projects in: query name: all-projects type: boolean produces: - application/json responses: "200": description: Websocket message (JSON) schema: $ref: '#/definitions/Event' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the event stream tags: - server /1.0/images: get: description: Returns a list of images (URLs). operationId: images_get parameters: - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string - description: Retrieve images from all projects in: query name: all-projects type: boolean produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/images/06b86454720d36b20f94e31c6812e05ec51c1b568cf3a8abd273769d213394bb", "/1.0/images/084dd79dd1360fd25a2479eb46674c2a5ef3022a40fe03c91ab3603e3402b8e1" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the images tags: - images post: consumes: - application/json description: Adds a new image to the image store. operationId: images_post parameters: - description: Project name example: default in: query name: project type: string - description: Image in: body name: image schema: $ref: '#/definitions/ImagesPost' - description: Raw image file in: body name: raw_image - description: Push secret for server to server communication example: RANDOM-STRING in: header name: X-Incus-secret schema: type: string - description: Expected fingerprint when pushing a raw image in: header name: X-Incus-fingerprint schema: type: string - description: List of aliases to assign in: header name: X-Incus-aliases schema: items: type: string type: array - description: Descriptive properties in: header name: X-Incus-properties schema: additionalProperties: type: string type: object - description: Whether the image is available to unauthenticated users in: header name: X-Incus-public schema: type: boolean - description: Original filename of the image in: header name: X-Incus-filename schema: type: string - description: List of profiles to use in: header name: X-Incus-profiles schema: items: type: string type: array produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add an image tags: - images /1.0/images/{fingerprint}: delete: description: Removes the image from the image store. operationId: image_delete parameters: - description: Fingerprint in: path name: fingerprint required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the image tags: - images get: description: Gets a specific image. operationId: image_get parameters: - description: Fingerprint in: path name: fingerprint required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Image schema: description: Sync response properties: metadata: $ref: '#/definitions/Image' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the image tags: - images patch: consumes: - application/json description: Updates a subset of the image definition. operationId: image_patch parameters: - description: Fingerprint in: path name: fingerprint required: true type: string - description: Project name example: default in: query name: project type: string - description: Image configuration in: body name: image required: true schema: $ref: '#/definitions/ImagePut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the image tags: - images put: consumes: - application/json description: Updates the entire image definition. operationId: image_put parameters: - description: Fingerprint in: path name: fingerprint required: true type: string - description: Project name example: default in: query name: project type: string - description: Image configuration in: body name: image required: true schema: $ref: '#/definitions/ImagePut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the image tags: - images /1.0/images/{fingerprint}/export: get: description: |- Download the raw image file(s) from the server. If the image is in split format, a multipart http transfer occurs. operationId: image_export_get parameters: - description: Fingerprint in: path name: fingerprint required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/octet-stream - multipart/form-data responses: "200": description: Raw image data "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the raw image file(s) tags: - images post: description: Gets the server to connect to a remote server and push the image to it. operationId: images_export_post parameters: - description: Fingerprint in: path name: fingerprint required: true type: string - description: Project name example: default in: query name: project type: string - description: Image push request in: body name: image required: true schema: $ref: '#/definitions/ImageExportPost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Make the server push the image to a remote server tags: - images /1.0/images/{fingerprint}/export?public: get: description: |- Download the raw image file(s) of a public image from the server. If the image is in split format, a multipart http transfer occurs. operationId: image_export_get_untrusted parameters: - description: Fingerprint in: path name: fingerprint required: true type: string - description: Project name example: default in: query name: project type: string - description: Secret token to retrieve a private image example: RANDOM-STRING in: query name: secret type: string produces: - application/octet-stream - multipart/form-data responses: "200": description: Raw image data "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the raw image file(s) tags: - images /1.0/images/{fingerprint}/refresh: post: description: |- This causes the server to check the image source server for an updated version of the image and if available to refresh the local copy with the new version. operationId: images_refresh_post parameters: - description: Fingerprint in: path name: fingerprint required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "202": $ref: '#/responses/Operation' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Refresh an image tags: - images /1.0/images/{fingerprint}/secret: post: description: |- This generates a background operation including a secret one time key in its metadata which can be used to fetch this image from an untrusted client. operationId: images_secret_post parameters: - description: Fingerprint in: path name: fingerprint required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "202": $ref: '#/responses/Operation' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Generate secret for retrieval of the image by an untrusted client tags: - images /1.0/images/{fingerprint}?public: get: description: Gets a specific public image. operationId: image_get_untrusted parameters: - description: Fingerprint in: path name: fingerprint required: true type: string - description: Project name example: default in: query name: project type: string - description: Secret token to retrieve a private image example: RANDOM-STRING in: query name: secret type: string produces: - application/json responses: "200": description: Image schema: description: Sync response properties: metadata: $ref: '#/definitions/Image' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the public image tags: - images /1.0/images/aliases: get: description: Returns a list of image aliases (URLs). operationId: images_aliases_get parameters: - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/images/aliases/foo", "/1.0/images/aliases/bar1" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the image aliases tags: - images post: consumes: - application/json description: Creates a new image alias. operationId: images_aliases_post parameters: - description: Project name example: default in: query name: project type: string - description: Image alias in: body name: image alias required: true schema: $ref: '#/definitions/ImageAliasesPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add an image alias tags: - images /1.0/images/aliases/{name}: delete: description: Deletes a specific image alias. operationId: image_alias_delete parameters: - description: Alias name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the image alias tags: - images get: description: Gets a specific image alias. operationId: image_alias_get parameters: - description: Alias name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Image alias schema: description: Sync response properties: metadata: $ref: '#/definitions/ImageAliasesEntry' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the image alias tags: - images patch: consumes: - application/json description: Updates a subset of the image alias configuration. operationId: images_alias_patch parameters: - description: Alias name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Image alias configuration in: body name: image alias required: true schema: $ref: '#/definitions/ImageAliasesEntryPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the image alias tags: - images post: consumes: - application/json description: Renames an existing image alias. operationId: images_alias_post parameters: - description: Alias name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Image alias rename request in: body name: image alias required: true schema: $ref: '#/definitions/ImageAliasesEntryPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename the image alias tags: - images put: consumes: - application/json description: Updates the entire image alias configuration. operationId: images_aliases_put parameters: - description: Alias name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Image alias configuration in: body name: image alias required: true schema: $ref: '#/definitions/ImageAliasesEntryPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the image alias tags: - images /1.0/images/aliases/{name}?public: get: description: |- Gets a specific public image alias. This untrusted endpoint only works for aliases pointing to public images. operationId: image_alias_get_untrusted parameters: - description: Alias name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Image alias schema: description: Sync response properties: metadata: $ref: '#/definitions/ImageAliasesEntry' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the public image alias tags: - images /1.0/images/aliases?recursion=1: get: description: Returns a list of image aliases (structs). operationId: images_aliases_get_recursion1 parameters: - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of image aliases items: $ref: '#/definitions/ImageAliasesEntry' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the image aliases tags: - images /1.0/images?public: get: description: Returns a list of publicly available images (URLs). operationId: images_get_untrusted parameters: - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string - description: Retrieve images from all projects in: query name: all-projects type: boolean produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/images/06b86454720d36b20f94e31c6812e05ec51c1b568cf3a8abd273769d213394bb", "/1.0/images/084dd79dd1360fd25a2479eb46674c2a5ef3022a40fe03c91ab3603e3402b8e1" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the public images tags: - images post: consumes: - application/json description: |- Pushes the data to the target image server. This is meant for server to server communication where a new image entry is prepared on the target server and the source server is provided that URL and a secret token to push the image content over. operationId: images_post_untrusted parameters: - description: Project name example: default in: query name: project type: string - description: Image in: body name: image required: true schema: $ref: '#/definitions/ImagesPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add an image tags: - images /1.0/images?public&recursion=1: get: description: Returns a list of publicly available images (structs). operationId: images_get_recursion1_untrusted parameters: - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string - description: Retrieve images from all projects in: query name: all-projects type: boolean produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of images items: $ref: '#/definitions/Image' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the public images tags: - images /1.0/images?recursion=1: get: description: Returns a list of images (structs). operationId: images_get_recursion1 parameters: - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string - description: Retrieve images from all projects example: default in: query name: all-projects type: boolean produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of images items: $ref: '#/definitions/Image' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the images tags: - images /1.0/instances: get: description: Returns a list of instances (URLs). operationId: instances_get parameters: - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string - description: Retrieve instances from all projects in: query name: all-projects type: boolean produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/instances/foo", "/1.0/instances/bar" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the instances tags: - instances post: consumes: - application/json description: |- Creates a new instance. Depending on the source, this can create an instance from an existing local image, remote image, existing local instance or snapshot, remote migration stream or backup file. operationId: instances_post parameters: - description: Project name example: default in: query name: project type: string - description: Cluster member example: default in: query name: target type: string - description: Instance request in: body name: instance schema: $ref: '#/definitions/InstancesPost' - description: Raw backup file in: body name: raw_backup produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Create a new instance tags: - instances put: consumes: - application/json description: Changes the running state of all instances. operationId: instances_put parameters: - description: Project name example: default in: query name: project type: string - description: State in: body name: state schema: $ref: '#/definitions/InstancesPut' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Bulk instance state update tags: - instances /1.0/instances/{name}: delete: description: |- Deletes a specific instance. This also deletes anything owned by the instance such as snapshots and backups. operationId: instance_delete parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete an instance tags: - instances get: description: Gets a specific instance (basic struct). operationId: instance_get parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Instance schema: description: Sync response properties: metadata: $ref: '#/definitions/Instance' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the instance tags: - instances patch: consumes: - application/json description: Updates a subset of the instance configuration operationId: instance_patch parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Update request in: body name: instance schema: $ref: '#/definitions/InstancePut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Partially update the instance tags: - instances post: consumes: - application/json description: |- Renames, moves an instance between pools or migrates an instance to another server. The returned operation metadata will vary based on what's requested. For rename or move within the same server, this is a simple background operation with progress data. For migration, in the push case, this will similarly be a background operation with progress data, for the pull case, it will be a websocket operation with a number of secrets to be passed to the target server. operationId: instance_post parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Migration request in: body name: migration schema: $ref: '#/definitions/InstancePost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename or move/migrate an instance tags: - instances put: consumes: - application/json description: Updates the instance configuration or trigger a snapshot restore. operationId: instance_put parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Update request in: body name: instance schema: $ref: '#/definitions/InstancePut' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Update the instance tags: - instances /1.0/instances/{name}/access: get: description: Gets the access information for the instance. operationId: instance_access parameters: - description: Instance name in: path name: name required: true type: string - description: Project name in: query name: project type: string produces: - application/json responses: "200": description: Access schema: description: Sync response properties: metadata: $ref: '#/definitions/Access' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get who has access to an instance tags: - instances /1.0/instances/{name}/backups: get: description: Returns a list of instance backups (URLs). operationId: instance_backups_get parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/instances/foo/backups/backup0", "/1.0/instances/foo/backups/backup1" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the backups tags: - instances post: consumes: - application/json description: |- Creates a new backup. If the `Accept` header is set to `application/octet-stream`, this directly streams the backup tarball to the client without any intermediate operation. operationId: instance_backups_post parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Backup request in: body name: backup schema: $ref: '#/definitions/InstanceBackupsPost' produces: - application/json - application/octet-stream responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Create a backup tags: - instances /1.0/instances/{name}/backups/{backup}: delete: consumes: - application/json description: Deletes the instance backup. operationId: instance_backup_delete parameters: - description: Instance name in: path name: name required: true type: string - description: Backup name in: path name: backup required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete a backup tags: - instances get: description: Gets a specific instance backup. operationId: instance_backup_get parameters: - description: Instance name in: path name: name required: true type: string - description: Backup name in: path name: backup required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Instance backup schema: description: Sync response properties: metadata: $ref: '#/definitions/InstanceBackup' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the backup tags: - instances post: consumes: - application/json description: Renames an instance backup. operationId: instance_backup_post parameters: - description: Instance name in: path name: name required: true type: string - description: Backup name in: path name: backup required: true type: string - description: Project name example: default in: query name: project type: string - description: Backup rename in: body name: backup schema: $ref: '#/definitions/InstanceBackupPost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename a backup tags: - instances /1.0/instances/{name}/backups/{backup}/export: get: description: Download the raw backup file(s) from the server. operationId: instance_backup_export parameters: - description: Instance name in: path name: name required: true type: string - description: Backup name in: path name: backup required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/octet-stream responses: "200": description: Raw image data "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the raw backup file(s) tags: - instances /1.0/instances/{name}/backups?recursion=1: get: description: Returns a list of instance backups (structs). operationId: instance_backups_get_recursion1 parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of instance backups items: $ref: '#/definitions/InstanceBackup' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the backups tags: - instances /1.0/instances/{name}/bitmaps: post: consumes: - application/json description: Creates a new bitmap. operationId: instance_bitmaps_post parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Bitmap request in: body name: bitmap schema: $ref: '#/definitions/StorageVolumeBitmapsPost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Create a bitmap tags: - instances /1.0/instances/{name}/console: delete: description: Clears the console log buffer. operationId: instance_console_delete parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Clear the console log tags: - instances get: description: |- Gets the console output for the instance either as text log or as vga screendump. operationId: instance_console_get parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - default: log description: Console type enum: - log - vga example: vga in: query name: type type: string produces: - application/json responses: "200": description: | Console output either as raw console log or as vga screendump in PNG format depending on the `type` parameter provided with the request. "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get console output tags: - instances post: consumes: - application/json description: |- Connects to the console of an instance. The returned operation metadata will contain two websockets, one for data and one for control. operationId: instance_console_post parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Console request in: body name: console schema: $ref: '#/definitions/InstanceConsolePost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Connect to console tags: - instances /1.0/instances/{name}/debug/memory: get: description: |- Returns memory debug information of a running instance. Only supported for VMs. operationId: instance_debug_memory_get parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Memory dump format example: elf in: query name: format type: string responses: "200": description: Success "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get memory debug information of an instance tags: - instances /1.0/instances/{name}/debug/repair: get: description: Runs an internal repair action on the instance. operationId: instance_debug_repair_post parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: State in: body name: state schema: $ref: '#/definitions/InstanceDebugRepairPost' responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Trigger a repair action on the instance. tags: - instances /1.0/instances/{name}/exec: post: consumes: - application/json description: |- Executes a command inside an instance. The returned operation metadata will contain either 2 or 4 websockets. In non-interactive mode, you'll get one websocket for each of stdin, stdout and stderr. In interactive mode, a single bi-directional websocket is used for stdin and stdout/stderr. An additional "control" socket is always added on top which can be used for out of band communications. This allows sending signals and window sizing information through. operationId: instance_exec_post parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Exec request in: body name: exec schema: $ref: '#/definitions/InstanceExecPost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Run a command tags: - instances /1.0/instances/{name}/files: delete: description: Removes the file. operationId: instance_files_delete parameters: - description: Instance name in: path name: name required: true type: string - description: Path to the file example: default in: query name: path type: string - description: Project name example: default in: query name: project type: string - description: Perform recursive deletion example: true in: header name: X-Incus-force schema: type: boolean produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Delete a file tags: - instances get: description: Gets the file content. If it's a directory, a json list of files will be returned instead. operationId: instance_files_get parameters: - description: Instance name in: path name: name required: true type: string - description: Path to the file example: default in: query name: path type: string - description: Project name example: default in: query name: project type: string produces: - application/json - application/octet-stream responses: "200": description: Raw file or directory listing headers: X-Incus-gid: description: File owner GID X-Incus-mode: description: Mode mask X-Incus-modified: description: Last modified date X-Incus-type: description: Type of file (file, symlink or directory) X-Incus-uid: description: File owner UID "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get a file tags: - instances head: description: Gets the file or directory metadata. operationId: instance_files_head parameters: - description: Instance name in: path name: name required: true type: string - description: Path to the file example: default in: query name: path type: string - description: Project name example: default in: query name: project type: string responses: "200": description: Raw file or directory listing headers: X-Incus-gid: description: File owner GID X-Incus-mode: description: Mode mask X-Incus-modified: description: Last modified date X-Incus-type: description: Type of file (file, symlink or directory) X-Incus-uid: description: File owner UID "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get metadata for a file tags: - instances post: consumes: - application/octet-stream description: Creates a new file in the instance. operationId: instance_files_post parameters: - description: Instance name in: path name: name required: true type: string - description: Path to the file example: default in: query name: path type: string - description: Project name example: default in: query name: project type: string - description: Raw file content in: body name: raw_file - description: File owner UID example: 1000 in: header name: X-Incus-uid schema: type: integer - description: File owner GID example: 1000 in: header name: X-Incus-gid schema: type: integer - description: File mode example: 420 in: header name: X-Incus-mode schema: type: integer - description: Type of file (file, symlink or directory) example: file in: header name: X-Incus-type schema: type: string - description: Write mode (overwrite or append) example: overwrite in: header name: X-Incus-write schema: type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Create or replace a file tags: - instances /1.0/instances/{name}/logs: get: description: Returns a list of log files (URLs). operationId: instance_logs_get parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/instances/foo/logs/lxc.log" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get the log files tags: - instances /1.0/instances/{name}/logs/{filename}: delete: description: Removes the log file. operationId: instance_log_delete parameters: - description: Instance name in: path name: name required: true type: string - description: Log file name in: path name: filename required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Delete the log file tags: - instances get: description: Gets the log file. operationId: instance_log_get parameters: - description: Instance name in: path name: name required: true type: string - description: Log file name in: path name: filename required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json - application/octet-stream responses: "200": description: Raw file "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get the log file tags: - instances /1.0/instances/{name}/logs/exec-output: get: description: Returns a list of exec record-output files (URLs). operationId: instance_exec-outputs_get parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/instances/foo/logs/exec-output/exec_d0a89537-0617-4ed6-a79b-c2e88a970965.stdout", "/1.0/instances/foo/logs/exec-output/exec_d0a89537-0617-4ed6-a79b-c2e88a970965.stderr", ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get the exec record-output files tags: - instances /1.0/instances/{name}/logs/exec-output/{filename}: delete: description: Removes the exec record-output file. operationId: instance_exec-output_delete parameters: - description: Instance name in: path name: name required: true type: string - description: Log file name in: path name: filename required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Delete the exec record-output file tags: - instances get: description: Gets the exec-output file. operationId: instance_exec-output_get parameters: - description: Instance name in: path name: name required: true type: string - description: Log file name in: path name: filename required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json - application/octet-stream responses: "200": description: Raw file "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get the exec-output log file tags: - instances /1.0/instances/{name}/metadata: get: description: Gets the image metadata for the instance. operationId: instance_metadata_get parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Image metadata schema: description: Sync response properties: metadata: $ref: '#/definitions/ImageMetadata' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the instance image metadata tags: - instances patch: consumes: - application/json description: Updates a subset of the instance image metadata. operationId: instance_metadata_patch parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Image metadata in: body name: metadata required: true schema: $ref: '#/definitions/ImageMetadata' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the image metadata tags: - instances put: consumes: - application/json description: Updates the instance image metadata. operationId: instance_metadata_put parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Image metadata in: body name: metadata required: true schema: $ref: '#/definitions/ImageMetadata' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the image metadata tags: - instances /1.0/instances/{name}/metadata/templates: delete: description: Removes the template file. operationId: instance_metadata_templates_delete parameters: - description: Instance name in: path name: name required: true type: string - description: Template name example: default in: query name: path type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Delete a template file tags: - instances get: description: |- If no path specified, returns a list of template file names. If a path is specified, returns the file content. operationId: instance_metadata_templates_get parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Template name example: hostname.tpl in: query name: path type: string produces: - application/json - application/octet-stream responses: "200": description: Raw template file or file listing "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get the template file names or a specific tags: - instances post: consumes: - application/octet-stream description: Creates a new image template file for the instance. operationId: instance_metadata_templates_post parameters: - description: Instance name in: path name: name required: true type: string - description: Template name example: default in: query name: path type: string - description: Project name example: default in: query name: project type: string - description: Raw file content in: body name: raw_file produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Create or replace a template file tags: - instances /1.0/instances/{name}/rebuild: post: consumes: - application/octet-stream description: Rebuild an instance using an alternate image or as empty. operationId: instance_rebuild_post parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: InstanceRebuild request in: body name: instance required: true schema: $ref: '#/definitions/InstanceRebuildPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Rebuild an instance tags: - instances /1.0/instances/{name}/sftp: get: description: Upgrades the request to an SFTP connection of the instance's filesystem. operationId: instance_sftp parameters: - description: Instance name in: path name: name required: true type: string produces: - application/json - application/octet-stream responses: "101": description: Switching protocols to SFTP "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get the instance SFTP connection tags: - instances /1.0/instances/{name}/snapshots: get: description: Returns a list of instance snapshots (URLs). operationId: instance_snapshots_get parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/instances/foo/snapshots/snap0", "/1.0/instances/foo/snapshots/snap1" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the snapshots tags: - instances post: consumes: - application/json description: Creates a new snapshot. operationId: instance_snapshots_post parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Snapshot request in: body name: snapshot schema: $ref: '#/definitions/InstanceSnapshotsPost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Create a snapshot tags: - instances /1.0/instances/{name}/snapshots/{snapshot}: delete: consumes: - application/json description: Deletes the instance snapshot. operationId: instance_snapshot_delete parameters: - description: Instance name in: path name: name required: true type: string - description: Snapshot name in: path name: snapshot required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete a snapshot tags: - instances get: description: Gets a specific instance snapshot. operationId: instance_snapshot_get parameters: - description: Instance name in: path name: name required: true type: string - description: Snapshot name in: path name: snapshot required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Instance snapshot schema: description: Sync response properties: metadata: $ref: '#/definitions/InstanceSnapshot' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the snapshot tags: - instances patch: consumes: - application/json description: Updates a subset of the snapshot config. operationId: instance_snapshot_patch parameters: - description: Instance name in: path name: name required: true type: string - description: Snapshot name in: path name: snapshot required: true type: string - description: Project name example: default in: query name: project type: string - description: Snapshot update in: body name: snapshot schema: $ref: '#/definitions/InstanceSnapshotPut' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Partially update snapshot tags: - instances post: consumes: - application/json description: |- Renames or migrates an instance snapshot to another server. The returned operation metadata will vary based on what's requested. For rename or move within the same server, this is a simple background operation with progress data. For migration, in the push case, this will similarly be a background operation with progress data, for the pull case, it will be a websocket operation with a number of secrets to be passed to the target server. operationId: instance_snapshot_post parameters: - description: Instance name in: path name: name required: true type: string - description: Snapshot name in: path name: snapshot required: true type: string - description: Project name example: default in: query name: project type: string - description: Snapshot migration in: body name: snapshot schema: $ref: '#/definitions/InstanceSnapshotPost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename or move/migrate a snapshot tags: - instances put: consumes: - application/json description: Updates the snapshot config. operationId: instance_snapshot_put parameters: - description: Instance name in: path name: name required: true type: string - description: Snapshot name in: path name: snapshot required: true type: string - description: Project name example: default in: query name: project type: string - description: Snapshot update in: body name: snapshot schema: $ref: '#/definitions/InstanceSnapshotPut' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Update snapshot tags: - instances /1.0/instances/{name}/snapshots?recursion=1: get: description: Returns a list of instance snapshots (structs). operationId: instance_snapshots_get_recursion1 parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of instance snapshots items: $ref: '#/definitions/InstanceSnapshot' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the snapshots tags: - instances /1.0/instances/{name}/state: get: description: |- Gets the runtime state of the instance. This is a reasonably expensive call as it causes code to be run inside of the instance to retrieve the resource usage and network information. operationId: instance_state_get parameters: - description: Instance name in: path name: name required: true type: string - description: Project name in: query name: project type: string produces: - application/json responses: "200": description: State schema: description: Sync response properties: metadata: $ref: '#/definitions/InstanceState' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the runtime state tags: - instances put: consumes: - application/json description: Changes the running state of the instance. operationId: instance_state_put parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: State in: body name: state schema: $ref: '#/definitions/InstanceStatePut' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Change the state tags: - instances /1.0/instances/{name}?recursion=1: get: description: |- Gets a specific instance (full struct). recursion=1 also includes information about state, snapshots and backups. operationId: instance_get_recursion1 parameters: - description: Instance name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Instance schema: description: Sync response properties: metadata: $ref: '#/definitions/InstanceFull' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the instance tags: - instances /1.0/instances?recursion=1: get: description: Returns a list of instances (basic structs). operationId: instances_get_recursion1 parameters: - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string - description: Retrieve instances from all projects in: query name: all-projects type: boolean produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of instances items: $ref: '#/definitions/Instance' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the instances tags: - instances /1.0/instances?recursion=2: get: description: |- Returns a list of instances (full structs). The main difference between recursion=1 and recursion=2 is that the latter also includes state and snapshot information allowing for a single API call to return everything needed by most clients. operationId: instances_get_recursion2 parameters: - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string - description: Retrieve instances from all projects in: query name: all-projects type: boolean produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of instances items: $ref: '#/definitions/InstanceFull' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the instances tags: - instances /1.0/metadata/configuration: get: description: Returns the generated metadata configuration in YAML format. operationId: metadata_configuration_get produces: - text/plain responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: The generated metadata configuration type: string status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the metadata configuration /1.0/metrics: get: description: Gets metrics of instances. operationId: metrics_get parameters: - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - text/plain responses: "200": description: Metrics schema: description: Instance metrics type: string "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get metrics tags: - metrics /1.0/network-acls: get: description: Returns a list of network ACLs (URLs). operationId: network_acls_get parameters: - description: Project name example: default in: query name: project type: string - description: Retrieve network ACLs from all projects example: true in: query name: all-projects type: boolean - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/network-acls/foo", "/1.0/network-acls/bar" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network ACLs tags: - network-acls post: consumes: - application/json description: Creates a new network ACL. operationId: network_acls_post parameters: - description: Project name example: default in: query name: project type: string - description: ACL in: body name: acl required: true schema: $ref: '#/definitions/NetworkACLsPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a network ACL tags: - network-acls /1.0/network-acls/{name}: delete: description: Removes the network ACL. operationId: network_acl_delete parameters: - description: ACL name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the network ACL tags: - network-acls get: description: Gets a specific network ACL. operationId: network_acl_get parameters: - description: ACL name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: ACL schema: description: Sync response properties: metadata: $ref: '#/definitions/NetworkACL' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network ACL tags: - network-acls patch: consumes: - application/json description: Updates a subset of the network ACL configuration. operationId: network_acl_patch parameters: - description: ACL name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: ACL configuration in: body name: acl required: true schema: $ref: '#/definitions/NetworkACLPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the network ACL tags: - network-acls post: consumes: - application/json description: Renames an existing network ACL. operationId: network_acl_post parameters: - description: ACL name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: ACL rename request in: body name: acl required: true schema: $ref: '#/definitions/NetworkACLPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename the network ACL tags: - network-acls put: consumes: - application/json description: Updates the entire network ACL configuration. operationId: network_acl_put parameters: - description: ACL name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: ACL configuration in: body name: acl required: true schema: $ref: '#/definitions/NetworkACLPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the network ACL tags: - network-acls /1.0/network-acls/{name}/log: get: description: Gets a specific network ACL log entries. operationId: network_acl_log_get parameters: - description: ACL name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/octet-stream responses: "200": description: Raw log file "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network ACL log tags: - network-acls /1.0/network-acls?recursion=1: get: description: Returns a list of network ACLs (structs). operationId: network_acls_get_recursion1 parameters: - description: Project name example: default in: query name: project type: string - description: Retrieve network ACLs from all projects example: true in: query name: all-projects type: boolean - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of network ACLs items: $ref: '#/definitions/NetworkACL' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network ACLs tags: - network-acls /1.0/network-address-sets: get: description: Returns a list of network address sets (URLs). operationId: network_address_sets_get parameters: - description: Project name example: default in: query name: project type: string - description: Retrieve network address sets from all projects example: true in: query name: all-projects type: boolean - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/network-address-sets/foo", "/1.0/network-address-sets/bar" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network address sets tags: - network-address-sets post: consumes: - application/json description: Creates a new network address set. operationId: network_address_sets_post parameters: - description: Project name example: default in: query name: project type: string - description: address set in: body name: address set required: true schema: $ref: '#/definitions/NetworkAddressSetsPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a network address set tags: - network-address-sets /1.0/network-address-sets/{name}: delete: description: Removes the network address set. operationId: network_address_set_delete parameters: - description: Address set name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the network address set tags: - network-address-sets get: description: Gets a specific network address set. operationId: network_address_set_get parameters: - description: Address set name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: address set schema: description: Sync response properties: metadata: $ref: '#/definitions/NetworkAddressSet' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network address set tags: - network-address-sets patch: consumes: - application/json description: Updates a subset of the network address set configuration. operationId: network_address_set_patch parameters: - description: Address set name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Address set configuration in: body name: address set required: true schema: $ref: '#/definitions/NetworkAddressSetPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the network address set tags: - network-address-sets post: consumes: - application/json description: Renames an existing network address set. operationId: network_address_set_post parameters: - description: Address set name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Address set rename request in: body name: address set required: true schema: $ref: '#/definitions/NetworkAddressSetPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename the network address set tags: - network-address-sets put: consumes: - application/json description: Updates the entire network address set configuration. operationId: network_address_set_put parameters: - description: Address set name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Address set configuration in: body name: address set required: true schema: $ref: '#/definitions/NetworkAddressSetPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the network address set tags: - network-address-sets /1.0/network-address-sets?recursion=1: get: description: Returns a list of network address sets (structs). operationId: network_address_sets_get_recursion1 parameters: - description: Project name example: default in: query name: project type: string - description: Retrieve network address sets from all projects example: true in: query name: all-projects type: boolean - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of network address sets items: $ref: '#/definitions/NetworkAddressSet' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network address sets tags: - network-address-sets /1.0/network-allocations: get: description: Returns a list of network allocations. operationId: network_allocations_get parameters: - description: Project name example: default in: query name: project type: string - description: Retrieve entities from all projects in: query name: all-projects type: boolean produces: - application/json responses: "200": description: API endpoints schema: properties: metadata: description: List of network allocations used by a consuming entity items: $ref: '#/definitions/NetworkAllocations' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network allocations in use (`network`, `network-forward` and `load-balancer` and `instance`) tags: - network-allocations /1.0/network-integrations: get: description: Returns a list of network integrations (URLs). operationId: network_integrations_get parameters: - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/network-integrations/region2", "/1.0/network-integrations/region3" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network integrations tags: - network-integrations post: consumes: - application/json description: Creates a new network integration. operationId: network_integrations_post parameters: - description: integration in: body name: integration required: true schema: $ref: '#/definitions/NetworkIntegrationsPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a network integration tags: - network-integrations /1.0/network-integrations/{integration}: delete: description: Removes the network integration. operationId: network_integration_delete parameters: - description: Integration name in: path name: integration required: true type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the network integration tags: - network-integrations get: description: Gets a specific network integration. operationId: network_integration_get parameters: - description: Integration name in: path name: integration required: true type: string produces: - application/json responses: "200": description: integration schema: description: Sync response properties: metadata: $ref: '#/definitions/NetworkIntegration' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network integration tags: - network-integrations patch: consumes: - application/json description: Updates a subset of the network integration configuration. operationId: network_integration_patch parameters: - description: Integration name in: path name: integration required: true type: string - description: integration configuration in: body name: integration required: true schema: $ref: '#/definitions/NetworkIntegrationPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the network integration tags: - network-integrations post: description: Renames the network integration. operationId: network_integration_post parameters: - description: Integration name in: path name: integration required: true type: string - description: integration configuration in: body name: integration required: true schema: $ref: '#/definitions/NetworkIntegrationPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename the network integration tags: - network-integrations put: consumes: - application/json description: Updates the entire network integration configuration. operationId: network_integration_put parameters: - description: Integration name in: path name: integration required: true type: string - description: integration configuration in: body name: integration required: true schema: $ref: '#/definitions/NetworkIntegrationPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the network integration tags: - network-integrations /1.0/network-integrations?recursion=1: get: description: Returns a list of network integrations (structs). operationId: network_integrations_get_recursion1 parameters: - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of network integrations items: $ref: '#/definitions/NetworkIntegration' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network integrations tags: - network-integrations /1.0/network-zones: get: description: Returns a list of network zones (URLs). operationId: network_zones_get parameters: - description: Project name example: default in: query name: project type: string - description: Retrieve network zones from all projects example: true in: query name: all-projects type: boolean - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/network-zones/example.net", "/1.0/network-zones/example.com" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network zones tags: - network-zones post: consumes: - application/json description: Creates a new network zone. operationId: network_zones_post parameters: - description: Project name example: default in: query name: project type: string - description: zone in: body name: zone required: true schema: $ref: '#/definitions/NetworkZonesPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a network zone tags: - network-zones /1.0/network-zones/{zone}: delete: description: Removes the network zone. operationId: network_zone_delete parameters: - description: Network zone name in: path name: zone required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the network zone tags: - network-zones get: description: Gets a specific network zone. operationId: network_zone_get parameters: - description: Network zone name in: path name: zone required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: zone schema: description: Sync response properties: metadata: $ref: '#/definitions/NetworkZone' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network zone tags: - network-zones patch: consumes: - application/json description: Updates a subset of the network zone configuration. operationId: network_zone_patch parameters: - description: Network zone name in: path name: zone required: true type: string - description: Project name example: default in: query name: project type: string - description: zone configuration in: body name: zone required: true schema: $ref: '#/definitions/NetworkZonePut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the network zone tags: - network-zones put: consumes: - application/json description: Updates the entire network zone configuration. operationId: network_zone_put parameters: - description: Network zone name in: path name: zone required: true type: string - description: Project name example: default in: query name: project type: string - description: zone configuration in: body name: zone required: true schema: $ref: '#/definitions/NetworkZonePut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the network zone tags: - network-zones /1.0/network-zones/{zone}/records: get: description: Returns a list of network zone records (URLs). operationId: network_zone_records_get parameters: - description: Network zone name in: path name: zone required: true type: string - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/network-zones/example.net/records/foo", "/1.0/network-zones/example.net/records/bar" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network zone records tags: - network-zones post: consumes: - application/json description: Creates a new network zone record. operationId: network_zone_records_post parameters: - description: Network zone name in: path name: zone required: true type: string - description: Project name example: default in: query name: project type: string - description: zone in: body name: zone required: true schema: $ref: '#/definitions/NetworkZoneRecordsPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a network zone record tags: - network-zones /1.0/network-zones/{zone}/records/{name}: delete: description: Removes the network zone record. operationId: network_zone_record_delete parameters: - description: Network zone name in: path name: zone required: true type: string - description: Record name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the network zone record tags: - network-zones get: description: Gets a specific network zone record. operationId: network_zone_record_get parameters: - description: Network zone name in: path name: zone required: true type: string - description: Record name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: zone schema: description: Sync response properties: metadata: $ref: '#/definitions/NetworkZoneRecord' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network zone record tags: - network-zones patch: consumes: - application/json description: Updates a subset of the network zone record configuration. operationId: network_zone_record_patch parameters: - description: Network zone name in: path name: zone required: true type: string - description: Record name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: zone record configuration in: body name: zone required: true schema: $ref: '#/definitions/NetworkZoneRecordPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the network zone record tags: - network-zones put: consumes: - application/json description: Updates the entire network zone record configuration. operationId: network_zone_record_put parameters: - description: Network zone name in: path name: zone required: true type: string - description: Record name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: zone record configuration in: body name: zone required: true schema: $ref: '#/definitions/NetworkZoneRecordPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the network zone record tags: - network-zones /1.0/network-zones/{zone}/records?recursion=1: get: description: Returns a list of network zone records (structs). operationId: network_zone_records_get_recursion1 parameters: - description: Network zone name in: path name: zone required: true type: string - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of network zone records items: $ref: '#/definitions/NetworkZoneRecord' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network zone records tags: - network-zones /1.0/network-zones?recursion=1: get: description: Returns a list of network zones (structs). operationId: network_zones_get_recursion1 parameters: - description: Project name example: default in: query name: project type: string - description: Retrieve network zones from all projects example: true in: query name: all-projects type: boolean - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of network zones items: $ref: '#/definitions/NetworkZone' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network zones tags: - network-zones /1.0/networks: get: description: Returns a list of networks (URLs). operationId: networks_get parameters: - description: Project name example: default in: query name: project type: string - description: Retrieve networks from all projects example: true in: query name: all-projects type: boolean - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/networks/mybr0", "/1.0/networks/mybr1" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the networks tags: - networks post: consumes: - application/json description: |- Creates a new network. When clustered, most network types require individual POST for each cluster member prior to a global POST. operationId: networks_post parameters: - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Network in: body name: network required: true schema: $ref: '#/definitions/NetworksPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a network tags: - networks /1.0/networks/{name}: delete: description: Removes the network. operationId: network_delete parameters: - description: Network name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the network tags: - networks get: description: Gets a specific network. operationId: network_get parameters: - description: Network name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: Network schema: description: Sync response properties: metadata: $ref: '#/definitions/Network' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network tags: - networks patch: consumes: - application/json description: Updates a subset of the network configuration. operationId: network_patch parameters: - description: Network name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Network configuration in: body name: network required: true schema: $ref: '#/definitions/NetworkPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the network tags: - networks post: consumes: - application/json description: Renames an existing network. operationId: network_post parameters: - description: Network name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Network rename request in: body name: network required: true schema: $ref: '#/definitions/NetworkPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename the network tags: - networks put: consumes: - application/json description: Updates the entire network configuration. operationId: network_put parameters: - description: Network name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Network configuration in: body name: network required: true schema: $ref: '#/definitions/NetworkPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the network tags: - networks /1.0/networks/{name}/leases: get: description: Returns a list of DHCP leases for the network. operationId: networks_leases_get parameters: - description: Network name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of DHCP leases items: $ref: '#/definitions/NetworkLease' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the DHCP leases tags: - networks /1.0/networks/{name}/state: get: description: Returns the current network state information. operationId: networks_state_get parameters: - description: Network name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: $ref: '#/definitions/NetworkState' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network state tags: - networks /1.0/networks/{networkName}/forwards: get: description: Returns a list of network address forwards (URLs). operationId: network_forwards_get parameters: - description: Network name in: path name: networkName required: true type: string - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/networks/mybr0/forwards/192.0.2.1", "/1.0/networks/mybr0/forwards/192.0.2.2" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network address forwards tags: - network-forwards post: consumes: - application/json description: Creates a new network address forward. operationId: network_forwards_post parameters: - description: Network name in: path name: networkName required: true type: string - description: Project name example: default in: query name: project type: string - description: Forward in: body name: forward required: true schema: $ref: '#/definitions/NetworkForwardsPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a network address forward tags: - network-forwards /1.0/networks/{networkName}/forwards/{listenAddress}: delete: description: Removes the network address forward. operationId: network_forward_delete parameters: - description: Network name in: path name: networkName required: true type: string - description: Listen address in: path name: listenAddress required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the network address forward tags: - network-forwards get: description: Gets a specific network address forward. operationId: network_forward_get parameters: - description: Network name in: path name: networkName required: true type: string - description: Listen address in: path name: listenAddress required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Address forward schema: description: Sync response properties: metadata: $ref: '#/definitions/NetworkForward' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network address forward tags: - network-forwards patch: consumes: - application/json description: Updates a subset of the network address forward configuration. operationId: network_forward_patch parameters: - description: Network name in: path name: networkName required: true type: string - description: Listen address in: path name: listenAddress required: true type: string - description: Project name example: default in: query name: project type: string - description: Address forward configuration in: body name: forward required: true schema: $ref: '#/definitions/NetworkForwardPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the network address forward tags: - network-forwards put: consumes: - application/json description: Updates the entire network address forward configuration. operationId: network_forward_put parameters: - description: Network name in: path name: networkName required: true type: string - description: Listen address in: path name: listenAddress required: true type: string - description: Project name example: default in: query name: project type: string - description: Address forward configuration in: body name: forward required: true schema: $ref: '#/definitions/NetworkForwardPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the network address forward tags: - network-forwards /1.0/networks/{networkName}/forwards?recursion=1: get: description: Returns a list of network address forwards (structs). operationId: network_forward_get_recursion1 parameters: - description: Network name in: path name: networkName required: true type: string - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of network address forwards items: $ref: '#/definitions/NetworkForward' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network address forwards tags: - network-forwards /1.0/networks/{networkName}/load-balancers: get: description: Returns a list of network address load balancers (URLs). operationId: network_load_balancers_get parameters: - description: Network name in: path name: networkName required: true type: string - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/networks/mybr0/load-balancers/192.0.2.1", "/1.0/networks/mybr0/load-balancers/192.0.2.2" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network address of load balancers tags: - network-load-balancers post: consumes: - application/json description: Creates a new network load balancer. operationId: network_load_balancers_post parameters: - description: Network name in: path name: networkName required: true type: string - description: Project name example: default in: query name: project type: string - description: Load Balancer in: body name: load-balancer required: true schema: $ref: '#/definitions/NetworkLoadBalancersPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a network load balancer tags: - network-load-balancers /1.0/networks/{networkName}/load-balancers/{listenAddress}: delete: description: Removes the network address load balancer. operationId: network_load_balancer_delete parameters: - description: Network name in: path name: networkName required: true type: string - description: Listen address in: path name: listenAddress required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the network address load balancer tags: - network-load-balancers get: description: Gets a specific network address load balancer. operationId: network_load_balancer_get parameters: - description: Network name in: path name: networkName required: true type: string - description: Listen address in: path name: listenAddress required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Load Balancer schema: description: Sync response properties: metadata: $ref: '#/definitions/NetworkLoadBalancer' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network address load balancer tags: - network-load-balancers patch: consumes: - application/json description: Updates a subset of the network address load balancer configuration. operationId: network_load_balancer_patch parameters: - description: Network name in: path name: networkName required: true type: string - description: Listen address in: path name: listenAddress required: true type: string - description: Project name example: default in: query name: project type: string - description: Address load balancer configuration in: body name: load-balancer required: true schema: $ref: '#/definitions/NetworkLoadBalancerPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the network address load balancer tags: - network-load-balancers put: consumes: - application/json description: Updates the entire network address load balancer configuration. operationId: network_load_balancer_put parameters: - description: Network name in: path name: networkName required: true type: string - description: Listen address in: path name: listenAddress required: true type: string - description: Project name example: default in: query name: project type: string - description: Address load balancer configuration in: body name: load-balancer required: true schema: $ref: '#/definitions/NetworkLoadBalancerPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the network address load balancer tags: - network-load-balancers /1.0/networks/{networkName}/load-balancers/{listenAddress}/state: get: description: Get the current state of a specific network address load balancer. operationId: network_load_balancer_state_get parameters: - description: Network name in: path name: networkName required: true type: string - description: Listen address in: path name: listenAddress required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Load Balancer state schema: description: Sync response properties: metadata: $ref: '#/definitions/NetworkLoadBalancerState' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network address load balancer state tags: - network-load-balancers /1.0/networks/{networkName}/load-balancers?recursion=1: get: description: Returns a list of network address load balancers (structs). operationId: network_load_balancer_get_recursion1 parameters: - description: Network name in: path name: networkName required: true type: string - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of network address load balancers items: $ref: '#/definitions/NetworkLoadBalancer' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network address load balancers tags: - network-load-balancers /1.0/networks/{networkName}/peers: get: description: Returns a list of network peers (URLs). operationId: network_peers_get parameters: - description: Network name in: path name: networkName required: true type: string - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/networks/mybr0/peers/my-peer-1", "/1.0/networks/mybr0/peers/my-peer-2" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network peers tags: - network-peers post: consumes: - application/json description: Initiates/creates a new network peering. operationId: network_peers_post parameters: - description: Network name in: path name: networkName required: true type: string - description: Project name example: default in: query name: project type: string - description: Peer in: body name: peer required: true schema: $ref: '#/definitions/NetworkPeersPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "202": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a network peer tags: - network-peers /1.0/networks/{networkName}/peers/{peerName}: delete: description: Removes the network peering. operationId: network_peer_delete parameters: - description: Network name in: path name: networkName required: true type: string - description: Peer name in: path name: peerName required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the network peer tags: - network-peers get: description: Gets a specific network peering. operationId: network_peer_get parameters: - description: Network name in: path name: networkName required: true type: string - description: Peer name in: path name: peerName required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Peer schema: description: Sync response properties: metadata: $ref: '#/definitions/NetworkPeer' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network peer tags: - network-peers patch: consumes: - application/json description: Updates a subset of the network peering configuration. operationId: network_peer_patch parameters: - description: Network name in: path name: networkName required: true type: string - description: Peer name in: path name: peerName required: true type: string - description: Project name example: default in: query name: project type: string - description: Peer configuration in: body name: Peer required: true schema: $ref: '#/definitions/NetworkPeerPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the network peer tags: - network-peers put: consumes: - application/json description: Updates the entire network peering configuration. operationId: network_peer_put parameters: - description: Network name in: path name: networkName required: true type: string - description: Peer name in: path name: peerName required: true type: string - description: Project name example: default in: query name: project type: string - description: Peer configuration in: body name: peer required: true schema: $ref: '#/definitions/NetworkPeerPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the network peer tags: - network-peers /1.0/networks/{networkName}/peers?recursion=1: get: description: Returns a list of network peers (structs). operationId: network_peer_get_recursion1 parameters: - description: Network name in: path name: networkName required: true type: string - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of network peers items: $ref: '#/definitions/NetworkPeer' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the network peers tags: - network-peers /1.0/networks?recursion=1: get: description: Returns a list of networks (structs). operationId: networks_get_recursion1 parameters: - description: Project name example: default in: query name: project type: string - description: Retrieve networks from all projects example: true in: query name: all-projects type: boolean - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of networks items: $ref: '#/definitions/Network' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the networks tags: - networks /1.0/operations: get: description: Returns a JSON object of operation type to operation list (URLs). operationId: operations_get parameters: - description: Project name example: default in: query name: project type: string - description: Retrieve operations from all projects in: query name: all-projects type: boolean produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: additionalProperties: items: type: string type: array description: JSON object of operation types to operation URLs example: |- { "running": [ "/1.0/operations/6916c8a6-9b7d-4abd-90b3-aedfec7ec7da" ] } type: object status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the operations tags: - operations /1.0/operations/{id}: delete: description: Cancels the operation if supported. operationId: operation_delete parameters: - description: Operation ID in: path name: id required: true type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Cancel the operation tags: - operations get: description: Gets the operation state. operationId: operation_get parameters: - description: Operation ID in: path name: id required: true type: string produces: - application/json responses: "200": description: Operation schema: description: Sync response properties: metadata: $ref: '#/definitions/Operation' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the operation state tags: - operations /1.0/operations/{id}/wait: get: description: Waits for the operation to reach a final state (or timeout) and retrieve its final state. operationId: operation_wait_get parameters: - description: Operation ID in: path name: id required: true type: string - description: Timeout in seconds (-1 means never) example: -1 in: query name: timeout type: integer produces: - application/json responses: "200": description: Operation schema: description: Sync response properties: metadata: $ref: '#/definitions/Operation' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Wait for the operation tags: - operations /1.0/operations/{id}/wait?public: get: description: |- Waits for the operation to reach a final state (or timeout) and retrieve its final state. When accessed by an untrusted user, the secret token must be provided. operationId: operation_wait_get_untrusted parameters: - description: Operation ID in: path name: id required: true type: string - description: Authentication token example: random-string in: query name: secret type: string - description: Timeout in seconds (-1 means never) example: -1 in: query name: timeout type: integer produces: - application/json responses: "200": description: Operation schema: description: Sync response properties: metadata: $ref: '#/definitions/Operation' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Wait for the operation tags: - operations /1.0/operations/{id}/websocket: get: description: |- Connects to an associated websocket stream for the operation. This should almost never be done directly by a client, instead it's meant for server to server communication with the client only relaying the connection information to the servers. operationId: operation_websocket_get parameters: - description: Operation ID in: path name: id required: true type: string - description: Authentication token example: random-string in: query name: secret type: string produces: - application/json responses: "200": description: Websocket operation messages (dependent on operation) "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the websocket stream tags: - operations /1.0/operations/{id}/websocket?public: get: description: |- Connects to an associated websocket stream for the operation. This should almost never be done directly by a client, instead it's meant for server to server communication with the client only relaying the connection information to the servers. The untrusted endpoint is used by the target server to connect to the source server. Authentication is performed through the secret token. operationId: operation_websocket_get_untrusted parameters: - description: Operation ID in: path name: id required: true type: string - description: Authentication token example: random-string in: query name: secret type: string produces: - application/json responses: "200": description: Websocket operation messages (dependent on operation) "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the websocket stream tags: - operations /1.0/operations?recursion=1: get: description: Returns a list of operations (structs). operationId: operations_get_recursion1 parameters: - description: Project name example: default in: query name: project type: string - description: Retrieve operations from all projects in: query name: all-projects type: boolean produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of operations items: $ref: '#/definitions/Operation' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the operations tags: - operations /1.0/profiles: get: description: Returns a list of profiles (URLs). operationId: profiles_get parameters: - description: Project name example: default in: query name: project type: string - description: Retrieve profiles from all projects example: true in: query name: all-projects type: boolean - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/profiles/default", "/1.0/profiles/foo" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the profiles tags: - profiles post: consumes: - application/json description: Creates a new profile. operationId: profiles_post parameters: - description: Project name example: default in: query name: project type: string - description: Profile in: body name: profile required: true schema: $ref: '#/definitions/ProfilesPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a profile tags: - profiles /1.0/profiles/{name}: delete: description: Removes the profile. operationId: profile_delete parameters: - description: Profile name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the profile tags: - profiles get: description: Gets a specific profile. operationId: profile_get parameters: - description: Profile name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Profile schema: description: Sync response properties: metadata: $ref: '#/definitions/Profile' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the profile tags: - profiles patch: consumes: - application/json description: Updates a subset of the profile configuration. operationId: profile_patch parameters: - description: Profile name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Profile configuration in: body name: profile required: true schema: $ref: '#/definitions/ProfilePut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the profile tags: - profiles post: consumes: - application/json description: Renames an existing profile. operationId: profile_post parameters: - description: Profile name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Profile rename request in: body name: profile required: true schema: $ref: '#/definitions/ProfilePost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename the profile tags: - profiles put: consumes: - application/json description: Updates the entire profile configuration. operationId: profile_put parameters: - description: Profile name in: path name: name required: true type: string - description: Project name example: default in: query name: project type: string - description: Profile configuration in: body name: profile required: true schema: $ref: '#/definitions/ProfilePut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the profile tags: - profiles /1.0/profiles?recursion=1: get: description: Returns a list of profiles (structs). operationId: profiles_get_recursion1 parameters: - description: Project name example: default in: query name: project type: string - description: Retrieve profiles from all projects example: true in: query name: all-projects type: boolean - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of profiles items: $ref: '#/definitions/Profile' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the profiles tags: - profiles /1.0/projects: get: description: Returns a list of projects (URLs). operationId: projects_get parameters: - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/projects/default", "/1.0/projects/foo" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the projects tags: - projects post: consumes: - application/json description: Creates a new project. operationId: projects_post parameters: - description: Project in: body name: project required: true schema: $ref: '#/definitions/ProjectsPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a project tags: - projects /1.0/projects/{name}: delete: description: Removes the project. operationId: project_delete parameters: - description: Project name in: path name: name required: true type: string - description: Delete project and related artifacts in: query name: force type: boolean produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the project tags: - projects get: description: Gets a specific project. operationId: project_get parameters: - description: Project name in: path name: name required: true type: string produces: - application/json responses: "200": description: Project schema: description: Sync response properties: metadata: $ref: '#/definitions/Project' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the project tags: - projects patch: consumes: - application/json description: Updates a subset of the project configuration. operationId: project_patch parameters: - description: Project name in: path name: name required: true type: string - description: Project configuration in: body name: project required: true schema: $ref: '#/definitions/ProjectPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the project tags: - projects post: consumes: - application/json description: Renames an existing project. operationId: project_post parameters: - description: Project name in: path name: name required: true type: string - description: Project rename request in: body name: project required: true schema: $ref: '#/definitions/ProjectPost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename the project tags: - projects put: consumes: - application/json description: Updates the entire project configuration. operationId: project_put parameters: - description: Project name in: path name: name required: true type: string - description: Project configuration in: body name: project required: true schema: $ref: '#/definitions/ProjectPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the project tags: - projects /1.0/projects/{name}/access: get: description: Gets the access information for the project. operationId: project_access parameters: - description: Project name in: path name: name required: true type: string produces: - application/json responses: "200": description: Access schema: description: Sync response properties: metadata: $ref: '#/definitions/Access' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get who has access to a project tags: - projects /1.0/projects/{name}/state: get: description: Gets a specific project resource consumption information. operationId: project_state_get parameters: - description: Project name in: path name: name required: true type: string produces: - application/json responses: "200": description: Project state schema: description: Sync response properties: metadata: $ref: '#/definitions/ProjectState' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the project state tags: - projects /1.0/projects?recursion=1: get: description: Returns a list of projects (structs). operationId: projects_get_recursion1 parameters: - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of projects items: $ref: '#/definitions/Project' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the projects tags: - projects /1.0/resources: get: description: Gets the hardware information profile of the server. operationId: resources_get parameters: - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: Hardware resources schema: description: Sync response properties: metadata: $ref: '#/definitions/Resources' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get system resources information tags: - server /1.0/storage-pools: get: description: Returns a list of storage pools (URLs). operationId: storage_pools_get parameters: - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/storage-pools/local", "/1.0/storage-pools/remote" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage pools tags: - storage post: consumes: - application/json description: |- Creates a new storage pool. When clustered, storage pools require individual POST for each cluster member prior to a global POST. operationId: storage_pools_post parameters: - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage pool in: body name: storage required: true schema: $ref: '#/definitions/StoragePoolsPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a storage pool tags: - storage /1.0/storage-pools/{name}/buckets/{bucketName}: delete: description: Removes the storage bucket. operationId: storage_pool_bucket_delete parameters: - description: Resource name in: path name: name required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the storage bucket tags: - storage patch: consumes: - application/json description: Updates a subset of the storage bucket configuration. operationId: storage_pool_bucket_patch parameters: - description: Resource name in: path name: name required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage bucket configuration in: body name: storage bucket required: true schema: $ref: '#/definitions/StorageBucketPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the storage bucket. tags: - storage put: consumes: - application/json description: Updates the entire storage bucket configuration. operationId: storage_pool_bucket_put parameters: - description: Resource name in: path name: name required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage bucket configuration in: body name: storage bucket required: true schema: $ref: '#/definitions/StorageBucketPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the storage bucket tags: - storage /1.0/storage-pools/{name}/buckets/{bucketName}/keys/{keyName}: delete: description: Removes the storage bucket key. operationId: storage_pool_bucket_key_delete parameters: - description: Resource name in: path name: name required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Storage bucket key name in: path name: keyName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the storage bucket key tags: - storage put: consumes: - application/json description: Updates the entire storage bucket key configuration. operationId: storage_pool_bucket_key_put parameters: - description: Resource name in: path name: name required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Storage bucket key name in: path name: keyName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage bucket key configuration in: body name: storage bucket required: true schema: $ref: '#/definitions/StorageBucketKeyPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the storage bucket key tags: - storage /1.0/storage-pools/{name}/resources: get: description: Gets the usage information for the storage pool. operationId: storage_pool_resources parameters: - description: Resource name in: path name: name required: true type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: Hardware resources schema: description: Sync response properties: metadata: $ref: '#/definitions/ResourcesStoragePool' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get storage pool resources information tags: - storage /1.0/storage-pools/{poolName}: delete: description: Removes the storage pool. operationId: storage_pools_delete parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the storage pool tags: - storage get: description: Gets a specific storage pool. operationId: storage_pool_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: Storage pool schema: description: Sync response properties: metadata: $ref: '#/definitions/StoragePool' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage pool tags: - storage patch: consumes: - application/json description: Updates a subset of the storage pool configuration. operationId: storage_pool_patch parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage pool configuration in: body name: storage pool required: true schema: $ref: '#/definitions/StoragePoolPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the storage pool tags: - storage put: consumes: - application/json description: Updates the entire storage pool configuration. operationId: storage_pool_put parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage pool configuration in: body name: storage pool required: true schema: $ref: '#/definitions/StoragePoolPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the storage pool tags: - storage /1.0/storage-pools/{poolName}/buckets: get: description: Returns a list of storage pool buckets (URLs). operationId: storage_pool_buckets_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Project name example: default in: query name: project type: string - description: Retrieve storage pool buckets from all projects example: true in: query name: all-projects type: boolean - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/storage-pools/default/buckets/foo", "/1.0/storage-pools/default/buckets/bar", ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage pool buckets tags: - storage post: consumes: - application/json description: Creates a new storage pool bucket. operationId: storage_pool_bucket_post parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Project name example: default in: query name: project type: string - description: Bucket in: body name: bucket required: true schema: $ref: '#/definitions/StorageBucketsPost' produces: - application/json responses: "200": $ref: '#/definitions/StorageBucketKey' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a storage pool bucket. tags: - storage /1.0/storage-pools/{poolName}/buckets/{bucketName}: get: description: Gets a specific storage pool bucket. operationId: storage_pool_bucket_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Storage pool bucket schema: description: Sync response properties: metadata: $ref: '#/definitions/StorageBucket' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage pool bucket tags: - storage /1.0/storage-pools/{poolName}/buckets/{bucketName}/backups: get: description: Returns a list of storage bucket backups (URLs). operationId: storage_pool_buckets_backups_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/storage-pools/local/buckets/foo/backups/backup0", "/1.0/storage-pools/local/buckets/foo/backups/backup1" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage bucket backups tags: - storage post: consumes: - application/json description: |- Creates a new storage bucket backup. If the `Accept` header is set to `application/octet-stream`, this directly streams the backup tarball to the client without any intermediate operation. operationId: storage_pool_buckets_backups_post parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage bucket backup in: body name: bucket required: true schema: $ref: '#/definitions/StorageBucketBackupsPost' produces: - application/json - application/octet-stream responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Create a storage bucket backup tags: - storage /1.0/storage-pools/{poolName}/buckets/{bucketName}/backups/{backupName}: delete: consumes: - application/json description: Deletes a new storage bucket backup. operationId: storage_pool_buckets_backup_delete parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Backup name in: path name: backupName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete a storage bucket backup tags: - storage get: description: Gets a specific storage bucket backup. operationId: storage_pool_buckets_backup_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Backup name in: path name: backupName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: Storage bucket backup schema: description: Sync response properties: metadata: $ref: '#/definitions/StorageBucketBackup' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage bucket backup tags: - storage post: consumes: - application/json description: Renames a storage bucket backup. operationId: storage_pool_buckets_backup_post parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Backup name in: path name: backupName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage bucket backup in: body name: bucket rename required: true schema: $ref: '#/definitions/StorageBucketBackupPost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename a storage bucket backup tags: - storage /1.0/storage-pools/{poolName}/buckets/{bucketName}/backups/{backupName}/export: get: description: Download the raw backup file from the server. operationId: storage_pool_buckets_backup_export_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Backup name in: path name: backupName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/octet-stream responses: "200": description: Raw backup data "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the raw backup file tags: - storage /1.0/storage-pools/{poolName}/buckets/{bucketName}/backups?recursion=1: get: description: Returns a list of storage bucket backups (structs). operationId: storage_pool_buckets_backups_get_recursion1 parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of storage bucket backups items: $ref: '#/definitions/StorageBucketBackup' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage bucket backups tags: - storage /1.0/storage-pools/{poolName}/buckets/{bucketName}/keys: get: description: Returns a list of storage pool bucket keys (URLs). operationId: storage_pool_bucket_keys_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/storage-pools/default/buckets/foo/keys/my-read-only-key", "/1.0/storage-pools/default/buckets/bar/keys/admin", ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage pool bucket keys tags: - storage post: consumes: - application/json description: Creates a new storage pool bucket key. operationId: storage_pool_bucket_key_post parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Project name example: default in: query name: project type: string - description: Bucket in: body name: bucket required: true schema: $ref: '#/definitions/StorageBucketKeysPost' produces: - application/json responses: "200": $ref: '#/definitions/StorageBucketKey' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a storage pool bucket key. tags: - storage /1.0/storage-pools/{poolName}/buckets/{bucketName}/keys/{keyName}: get: description: Gets a specific storage pool bucket key. operationId: storage_pool_bucket_key_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Storage bucket key name in: path name: keyName required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Storage pool bucket key schema: description: Sync response properties: metadata: $ref: '#/definitions/StorageBucketKey' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage pool bucket key tags: - storage /1.0/storage-pools/{poolName}/buckets/{bucketName}/keys?recursion=1: get: description: Returns a list of storage pool bucket keys (structs). operationId: storage_pool_bucket_keys_get_recursion1 parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of storage pool bucket keys items: $ref: '#/definitions/StorageBucketKey' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage pool bucket keys tags: - storage /1.0/storage-pools/{poolName}/buckets/{bucketName}?recursion=1: get: description: Gets a specific storage pool bucket with all details (backups and keys). operationId: storage_pool_bucket_get_recursion1 parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage bucket name in: path name: bucketName required: true type: string - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Storage pool bucket schema: description: Sync response properties: metadata: $ref: '#/definitions/StorageBucketFull' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the full storage pool bucket details tags: - storage /1.0/storage-pools/{poolName}/buckets?recursion=1: get: description: Returns a list of storage pool buckets (structs). operationId: storage_pool_buckets_get_recursion1 parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Project name example: default in: query name: project type: string - description: Retrieve storage pool buckets from all projects example: true in: query name: all-projects type: boolean - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of storage pool buckets items: $ref: '#/definitions/StorageBucket' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage pool buckets tags: - storage /1.0/storage-pools/{poolName}/buckets?recursion=2: get: description: Returns a list of storage pool buckets with all details (structs). operationId: storage_pool_buckets_get_recursion2 parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Project name example: default in: query name: project type: string - description: Retrieve storage pool buckets from all projects example: true in: query name: all-projects type: boolean - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of storage pool buckets items: $ref: '#/definitions/StorageBucketFull' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage pool bucket details tags: - storage /1.0/storage-pools/{poolName}/volumes: get: description: Returns a list of storage volumes (URLs). operationId: storage_pool_volumes_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/storage-pools/local/volumes/container/a1", "/1.0/storage-pools/local/volumes/container/a2", "/1.0/storage-pools/local/volumes/custom/backups", "/1.0/storage-pools/local/volumes/custom/images" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volumes tags: - storage post: consumes: - application/json description: |- Creates a new storage volume. Will return an empty sync response on simple volume creation but an operation on copy or migration. operationId: storage_pool_volumes_post parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage volume in: body name: volume required: true schema: $ref: '#/definitions/StorageVolumesPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a storage volume tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}: get: description: Returns a list of storage volumes (URLs) (type specific endpoint). operationId: storage_pool_volumes_type_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/storage-pools/local/volumes/custom/backups", "/1.0/storage-pools/local/volumes/custom/images" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volumes tags: - storage post: consumes: - application/json description: |- Creates a new storage volume (type specific endpoint). Will return an empty sync response on simple volume creation but an operation on copy or migration. operationId: storage_pool_volumes_type_post parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage volume in: body name: volume required: true schema: $ref: '#/definitions/StorageVolumesPost' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Add a storage volume tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}: delete: description: Removes the storage volume. operationId: storage_pool_volume_type_delete parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete the storage volume tags: - storage get: description: Gets a specific storage volume. operationId: storage_pool_volume_type_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: Storage volume schema: description: Sync response properties: metadata: $ref: '#/definitions/StorageVolume' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volume tags: - storage patch: consumes: - application/json description: Updates a subset of the storage volume configuration. operationId: storage_pool_volume_type_patch parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage volume configuration in: body name: storage volume required: true schema: $ref: '#/definitions/StorageVolumePut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the storage volume tags: - storage post: consumes: - application/json description: |- Renames, moves a storage volume between pools or migrates an instance to another server. The returned operation metadata will vary based on what's requested. For rename or move within the same server, this is a simple background operation with progress data. For migration, in the push case, this will similarly be a background operation with progress data, for the pull case, it will be a websocket operation with a number of secrets to be passed to the target server. operationId: storage_pool_volume_type_post parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Migration request in: body name: migration schema: $ref: '#/definitions/StorageVolumePost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename or move/migrate a storage volume tags: - storage put: consumes: - application/json description: Updates the entire storage volume configuration. operationId: storage_pool_volume_type_put parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage volume configuration in: body name: storage volume required: true schema: $ref: '#/definitions/StorageVolumePut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the storage volume tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/backups: get: description: Returns a list of storage volume backups (URLs). operationId: storage_pool_volumes_type_backups_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/storage-pools/local/volumes/custom/foo/backups/backup0", "/1.0/storage-pools/local/volumes/custom/foo/backups/backup1" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volume backups tags: - storage post: consumes: - application/json description: |- Creates a new storage volume backup. If the `Accept` header is set to `application/octet-stream`, this directly streams the backup tarball to the client without any intermediate operation. operationId: storage_pool_volumes_type_backups_post parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage volume backup in: body name: volume required: true schema: $ref: '#/definitions/StorageVolumeBackupsPost' produces: - application/json - application/octet-stream responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Create a storage volume backup tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/backups/{backupName}: delete: consumes: - application/json description: Deletes a new storage volume backup. operationId: storage_pool_volumes_type_backup_delete parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Backup name in: path name: backupName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete a storage volume backup tags: - storage get: description: Gets a specific storage volume backup. operationId: storage_pool_volumes_type_backup_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Backup name in: path name: backupName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: Storage volume backup schema: description: Sync response properties: metadata: $ref: '#/definitions/StorageVolumeBackup' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volume backup tags: - storage post: consumes: - application/json description: Renames a storage volume backup. operationId: storage_pool_volumes_type_backup_post parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Backup name in: path name: backupName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage volume backup in: body name: volume rename required: true schema: $ref: '#/definitions/StorageVolumeSnapshotPost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename a storage volume backup tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/backups/{backupName}/export: get: description: Download the raw backup file from the server. operationId: storage_pool_volumes_type_backup_export_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Backup name in: path name: backupName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/octet-stream responses: "200": description: Raw backup data "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the raw backup file tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/backups?recursion=1: get: description: Returns a list of storage volume backups (structs). operationId: storage_pool_volumes_type_backups_get_recursion1 parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of storage volume backups items: $ref: '#/definitions/StorageVolumeBackup' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volume backups tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/bitmaps: get: description: Gets a specific storage volume bitmaps operationId: storage_pool_volume_type_bitmaps_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: Storage volume bitmaps schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/storage-pools/shared/volumes/custom/foo/bitmaps/bitmap0", "/1.0/storage-pools/shared/volumes/custom/foo/bitmaps/bitmap1" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volume dirty bitmaps tags: - storage post: consumes: - application/json description: Creates a new storage volume bitmap. operationId: storage_pool_volumes_type_bitmaps_post parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage volume bitmap in: body name: volume required: true schema: $ref: '#/definitions/StorageVolumeBitmapsPost' produces: - application/json - application/octet-stream responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Create a storage volume bitmap tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/bitmaps/{bitmapName}: delete: consumes: - application/json description: Deletes a storage volume bitmap. operationId: storage_pool_volumes_type_bitmap_delete parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Bitmap name in: path name: bitmapName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete a storage volume bitmap tags: - storage get: description: Gets a specific storage volume bitmap operationId: storage_pool_volume_type_bitmap_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Bitmap name in: path name: bitmapName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: Storage volume bitmap schema: description: Sync response properties: metadata: $ref: '#/definitions/StorageVolumeBitmap' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volume dirty bitmap tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/bitmaps?recursion=1: get: description: Gets a specific storage volume bitmaps operationId: storage_pool_volume_type_bitmaps_get_recursion1 parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: Storage volume bitmaps schema: description: Sync response properties: metadata: description: List of storage volume bitmaps items: $ref: '#/definitions/StorageVolumeBitmap' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volume dirty bitmaps tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/files: delete: description: Removes the file. operationId: storage_pool_volume_type_files_delete parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Path to the file example: default in: query name: path type: string - description: Project name example: default in: query name: project type: string - description: Perform recursive deletion example: true in: header name: X-Incus-force schema: type: boolean produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Delete a file tags: - storage get: description: Gets the file content. If it's a directory, a json list of files will be returned instead. operationId: storage_pool_volume_type_files_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Path to the file example: default in: query name: path type: string - description: Project name example: default in: query name: project type: string produces: - application/json - application/octet-stream responses: "200": description: Raw file or directory listing headers: X-Incus-gid: description: File owner GID X-Incus-mode: description: Mode mask X-Incus-modified: description: Last modified date X-Incus-type: description: Type of file (file, symlink or directory) X-Incus-uid: description: File owner UID "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get a file tags: - storage head: description: Gets the file or directory metadata. operationId: storage_pool_volume_type_files_head parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Path to the file example: default in: query name: path type: string - description: Project name example: default in: query name: project type: string responses: "200": description: Raw file or directory listing headers: X-Incus-gid: description: File owner GID X-Incus-mode: description: Mode mask X-Incus-modified: description: Last modified date X-Incus-type: description: Type of file (file, symlink or directory) X-Incus-uid: description: File owner UID "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get metadata for a file tags: - storage post: consumes: - application/octet-stream description: Creates a new file in the storage volume. operationId: storage_pool_volume_type_files_post parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Path to the file example: default in: query name: path type: string - description: Project name example: default in: query name: project type: string - description: Raw file content in: body name: raw_file - description: File owner UID example: 1000 in: header name: X-Incus-uid schema: type: integer - description: File owner GID example: 1000 in: header name: X-Incus-gid schema: type: integer - description: File mode example: 420 in: header name: X-Incus-mode schema: type: integer - description: Type of file (file, symlink or directory) example: file in: header name: X-Incus-type schema: type: string - description: Write mode (overwrite or append) example: overwrite in: header name: X-Incus-write schema: type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Create or replace a file tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/nbd: get: description: Upgrades the request to an NBD connection of the storage volume's block device. operationId: storage_pool_volume_type_nbd_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string produces: - application/json - application/octet-stream responses: "101": description: Switching protocols to NBD "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volume NBD connection tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/sftp: get: description: Upgrades the request to an SFTP connection of the storage volume's filesystem. operationId: storage_pool_volume_type_sftp_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string produces: - application/json - application/octet-stream responses: "101": description: Switching protocols to SFTP "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volume SFTP connection tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/snapshots: get: description: Returns a list of storage volume snapshots (URLs). operationId: storage_pool_volumes_type_snapshots_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/storage-pools/local/volumes/custom/foo/snapshots/snap0", "/1.0/storage-pools/local/volumes/custom/foo/snapshots/snap1" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volume snapshots tags: - storage post: consumes: - application/json description: Creates a new storage volume snapshot. operationId: storage_pool_volumes_type_snapshots_post parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage volume snapshot in: body name: volume required: true schema: $ref: '#/definitions/StorageVolumeSnapshotsPost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Create a storage volume snapshot tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/snapshots/{snapshotName}: delete: consumes: - application/json description: Deletes a new storage volume snapshot. operationId: storage_pool_volumes_type_snapshot_delete parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Snapshot name in: path name: snapshotName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Delete a storage volume snapshot tags: - storage get: description: Gets a specific storage volume snapshot. operationId: storage_pool_volumes_type_snapshot_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Snapshot name in: path name: snapshotName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: Storage volume snapshot schema: description: Sync response properties: metadata: $ref: '#/definitions/StorageVolumeSnapshot' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volume snapshot tags: - storage patch: consumes: - application/json description: Updates a subset of the storage volume snapshot configuration. operationId: storage_pool_volumes_type_snapshot_patch parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Snapshot name in: path name: snapshotName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage volume snapshot configuration in: body name: storage volume snapshot required: true schema: $ref: '#/definitions/StorageVolumeSnapshotPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Partially update the storage volume snapshot tags: - storage post: consumes: - application/json description: Renames a storage volume snapshot. operationId: storage_pool_volumes_type_snapshot_post parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Snapshot name in: path name: snapshotName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage volume snapshot in: body name: volume rename required: true schema: $ref: '#/definitions/StorageVolumeSnapshotPost' produces: - application/json responses: "202": $ref: '#/responses/Operation' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Rename a storage volume snapshot tags: - storage put: consumes: - application/json description: Updates the entire storage volume snapshot configuration. operationId: storage_pool_volumes_type_snapshot_put parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Snapshot name in: path name: snapshotName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Storage volume snapshot configuration in: body name: storage volume snapshot required: true schema: $ref: '#/definitions/StorageVolumeSnapshotPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "412": $ref: '#/responses/PreconditionFailed' "500": $ref: '#/responses/InternalServerError' summary: Update the storage volume snapshot tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/snapshots?recursion=1: get: description: Returns a list of storage volume snapshots (structs). operationId: storage_pool_volumes_type_snapshots_get_recursion1 parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of storage volume snapshots items: $ref: '#/definitions/StorageVolumeSnapshot' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volume snapshots tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}/state: get: description: Gets a specific storage volume state (usage data). operationId: storage_pool_volume_type_state_get parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: Storage pool schema: description: Sync response properties: metadata: $ref: '#/definitions/StorageVolumeState' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volume state tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}/{volumeName}?recursion=1: get: description: Gets a specific storage volume with all details (backups, snapshots and state0.. operationId: storage_pool_volume_type_get_recursion1 parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Storage volume name in: path name: volumeName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: Storage volume schema: description: Sync response properties: metadata: $ref: '#/definitions/StorageVolumeFull' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the full storage volume details tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}?recursion=1: get: description: Returns a list of storage volumes (structs) (type specific endpoint). operationId: storage_pool_volumes_type_get_recursion1 parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of storage volumes items: $ref: '#/definitions/StorageVolume' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volumes tags: - storage /1.0/storage-pools/{poolName}/volumes/{type}?recursion=2: get: description: Returns a list of storage volumes (structs) including all details (type specific endpoint). operationId: storage_pool_volumes_type_get_recursion2 parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Storage volume type in: path name: type required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of storage volumes items: $ref: '#/definitions/StorageVolumeFull' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volumes with all details tags: - storage /1.0/storage-pools/{poolName}/volumes?recursion=1: get: description: Returns a list of storage volumes (structs). operationId: storage_pool_volumes_get_recursion1 parameters: - description: Storage pool name in: path name: poolName required: true type: string - description: Project name example: default in: query name: project type: string - description: Cluster member name example: server01 in: query name: target type: string - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of storage volumes items: $ref: '#/definitions/StorageVolume' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage volumes tags: - storage /1.0/storage-pools?recursion=1: get: description: Returns a list of storage pools (structs). operationId: storage_pools_get_recursion1 parameters: - description: Project name example: default in: query name: project type: string - description: Collection filter example: default in: query name: filter type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of storage pools items: $ref: '#/definitions/StoragePool' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Get the storage pools tags: - storage /1.0/warnings: get: description: Returns a list of warnings. operationId: warnings_get parameters: - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: Sync response schema: description: Sync response properties: metadata: description: List of endpoints example: |- [ "/1.0/warnings/39c61a48-cc17-40ae-8248-4f7b4cadedf4", "/1.0/warnings/951779a5-2820-4d96-b01e-88fe820e5310" ] items: type: string type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "500": $ref: '#/responses/InternalServerError' summary: List the warnings tags: - warnings /1.0/warnings/{uuid}: delete: description: Removes the warning. operationId: warning_delete parameters: - description: UUID in: path name: uuid required: true type: string produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "500": $ref: '#/responses/InternalServerError' summary: Delete the warning tags: - warnings get: description: Gets a specific warning. operationId: warning_get parameters: - description: UUID in: path name: uuid required: true type: string produces: - application/json responses: "200": description: Warning schema: description: Sync response properties: metadata: $ref: '#/definitions/Warning' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "404": $ref: '#/responses/NotFound' "500": $ref: '#/responses/InternalServerError' summary: Get the warning tags: - warnings patch: consumes: - application/json description: Updates a subset of the warning status. operationId: warning_patch parameters: - description: UUID in: path name: uuid required: true type: string - description: Warning status in: body name: warning required: true schema: $ref: '#/definitions/WarningPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Partially update the warning tags: - warnings put: consumes: - application/json description: Updates the warning status. operationId: warning_put parameters: - description: UUID in: path name: uuid required: true type: string - description: Warning status in: body name: warning required: true schema: $ref: '#/definitions/WarningPut' produces: - application/json responses: "200": $ref: '#/responses/EmptySyncResponse' "400": $ref: '#/responses/BadRequest' "403": $ref: '#/responses/Forbidden' "500": $ref: '#/responses/InternalServerError' summary: Update the warning tags: - warnings /1.0/warnings?recursion=1: get: description: Returns a list of warnings (structs). operationId: warnings_get_recursion1 parameters: - description: Project name example: default in: query name: project type: string produces: - application/json responses: "200": description: API endpoints schema: description: Sync response properties: metadata: description: List of warnings items: $ref: '#/definitions/Warning' type: array status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "500": $ref: '#/responses/InternalServerError' summary: Get the warnings tags: - warnings /1.0?public: get: description: |- Shows a small subset of the server environment and configuration which is required by untrusted clients to reach a server. The `?public` part of the URL isn't required, it's simply used to separate the two behaviors of this endpoint. operationId: server_get_untrusted produces: - application/json responses: "200": description: Server environment and configuration schema: description: Sync response properties: metadata: $ref: '#/definitions/ServerUntrusted' status: description: Status description example: Success type: string status_code: description: Status code example: 200 type: integer type: description: Response type example: sync type: string type: object "500": $ref: '#/responses/InternalServerError' summary: Get the server environment tags: - server responses: BadRequest: description: Bad Request schema: properties: error: example: bad request type: string x-go-name: Error error_code: example: 400 format: int64 type: integer x-go-name: ErrorCode type: example: error type: string x-go-name: Type type: object EmptySyncResponse: description: Empty sync response schema: properties: status: example: Success type: string x-go-name: Status status_code: example: 200 format: int64 type: integer x-go-name: StatusCode type: example: sync type: string x-go-name: Type type: object Forbidden: description: Forbidden schema: properties: error: example: not authorized type: string x-go-name: Error error_code: example: 403 format: int64 type: integer x-go-name: ErrorCode type: example: error type: string x-go-name: Type type: object InternalServerError: description: Internal Server Error schema: properties: error: example: internal server error type: string x-go-name: Error error_code: example: 500 format: int64 type: integer x-go-name: ErrorCode type: example: error type: string x-go-name: Type type: object NotFound: description: Not found schema: properties: error: example: not found type: string x-go-name: Error error_code: example: 404 format: int64 type: integer x-go-name: ErrorCode type: example: error type: string x-go-name: Type type: object Operation: description: Operation schema: properties: metadata: $ref: '#/definitions/Operation' operation: example: /1.0/operations/66e83638-9dd7-4a26-aef2-5462814869a1 type: string x-go-name: Operation status: example: Operation created type: string x-go-name: Status status_code: example: 100 format: int64 type: integer x-go-name: StatusCode type: example: async type: string x-go-name: Type type: object PreconditionFailed: description: Precondition Failed schema: properties: error: example: precondition failed type: string x-go-name: Error error_code: example: 412 format: int64 type: integer x-go-name: ErrorCode type: example: error type: string x-go-name: Type type: object swagger: "2.0" incus-7.0.0/doc/security.md000066400000000000000000000002561517523235500156100ustar00rootroot00000000000000(security)= # Security ```{toctree} :maxdepth: 1 explanation/security explanation/bpf-tokens authentication authorization Expose Incus to the network incus-7.0.0/doc/server.md000066400000000000000000000006611517523235500152470ustar00rootroot00000000000000(incus-server)= # Incus server ```{toctree} :maxdepth: 1 Migrating from LXD Configure the server /server_config System settings Backups Performance tuning Benchmarking Monitor metrics Recover instances Database /architectures ``` incus-7.0.0/doc/server_config.md000066400000000000000000000111501517523235500165670ustar00rootroot00000000000000(server)= # Server configuration The Incus server can be configured through a set of key/value configuration options. The key/value configuration is namespaced. The following options are available: - {ref}`server-options-core` - {ref}`server-options-acme` - {ref}`server-options-cluster` - {ref}`server-options-images` - {ref}`server-options-logging` - {ref}`server-options-misc` - {ref}`server-options-oidc` - {ref}`server-options-openfga` See {ref}`server-configure` for instructions on how to set the configuration options. ```{note} Options marked with a `global` scope are immediately applied to all cluster members. Options with a `local` scope must be set on a per-member basis. ``` (server-options-core)= ## Core configuration The following server options control the core daemon configuration: % Include content from [config_options.txt](config_options.txt) ```{include} config_options.txt :start-after: :end-before: ``` (server-options-acme)= ## ACME configuration The following server options control the {ref}`ACME ` configuration: % Include content from [config_options.txt](config_options.txt) ```{include} config_options.txt :start-after: :end-before: ``` (server-options-oidc)= ## OpenID Connect configuration The following server options configure external user authentication through {ref}`authentication-openid`: % Include content from [config_options.txt](config_options.txt) ```{include} config_options.txt :start-after: :end-before: ``` (server-options-openfga)= ## OpenFGA configuration The following server options configure external user authorization through {ref}`authorization-openfga`: % Include content from [config_options.txt](config_options.txt) ```{include} config_options.txt :start-after: :end-before: ``` (server-options-cluster)= ## Cluster configuration The following server options control {ref}`clustering`: % Include content from [config_options.txt](config_options.txt) ```{include} config_options.txt :start-after: :end-before: ``` (server-options-images)= ## Images configuration The following server options configure how to handle {ref}`images`: % Include content from [config_options.txt](config_options.txt) ```{include} config_options.txt :start-after: :end-before: ``` (server-options-logging)= ## Logging configuration The logging system now supports multiple configurable targets, each identified by a unique name (e.g., `loki01`, `syslog01`). Each target can be independently configured and assigned specific log types. ### Supported Targets - `loki` - For sending logs to a Grafana Loki server - `syslog` - For sending logs to remote syslog endpoint ### Example configuration ``` logging.loki01.target.type: loki logging.loki01.target.address: https://loki01.int.example.net logging.loki01.target.username: foo logging.loki01.target.password: bar logging.loki01.types: lifecycle,network-acl logging.loki01.lifecycle.types: instance logging.syslog01.target.type: syslog logging.syslog01.target.address: syslog01.int.example.net logging.syslog01.target.facility: security logging.syslog01.types: logging logging.syslog01.logging.level: warning ``` % Include content from [config_options.txt](config_options.txt) ```{include} config_options.txt :start-after: :end-before: ``` (server-options-misc)= ## Miscellaneous options The following server options configure server-specific settings for {ref}`instances`, {ref}`OVN ` integration, {ref}`Backups ` and {ref}`storage`: % Include content from [config_options.txt](config_options.txt) ```{include} config_options.txt :start-after: :end-before: ``` (server-options-user)= ## User options Additional user defined configuration keys are available within the `user.` namespace. User defined configuration keys are always of type `string` and have `global` scope. Note that keys starting with `user.ui.` are used for web UI configuration options and are visible even to unauthenticated users. incus-7.0.0/doc/storage.md000066400000000000000000000006031517523235500154010ustar00rootroot00000000000000(storage)= # Storage ```{toctree} :maxdepth: 1 About storage Manage pools Create an instance in a pool Manage volumes Move or copy a volume Back up a volume Manage buckets reference/storage_drivers ``` incus-7.0.0/doc/substitutions.yaml000066400000000000000000000042231517523235500172400ustar00rootroot00000000000000# Key/value substitutions to use within the Sphinx doc. {note_ip_addresses_CIDR: "Incus uses the [CIDR notation](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) where network subnet information is required, for example, `192.0.2.0/24` or `2001:db8::/32`. This does not apply to cases where a single address is required, for example, local/remote addresses of tunnels, NAT addresses or specific addresses to apply to an instance.", snapshot_expiry_format: "Controls when snapshots are to be deleted (expects an expression like `1M 2H 3d 4w 5m 6y`)", snapshot_pattern_format: "Pongo2 template string that represents the snapshot name (used for scheduled snapshots and unnamed snapshots)", snapshot_pattern_detail: "The `snapshots.pattern` option takes a Pongo2 template string to format the snapshot name.\n\nTo add a time stamp to the snapshot name, use the Pongo2 context variable `creation_date`.\nMake sure to format the date in your template string to avoid forbidden characters in the snapshot name.\nFor example, set `snapshots.pattern` to `{{ creation_date|date:'2006-01-02_15-04-05' }}` to name the snapshots after their time of creation, down to the precision of a second.\n\nAnother way to avoid name collisions is to use the placeholder `%d` in the pattern.\nFor the first snapshot, the placeholder is replaced with `0`.\nFor subsequent snapshots, the existing snapshot names are taken into account to find the highest number at the placeholder's position.\nThis number is then incremented by one for the new name.", snapshot_schedule_format: "Cron expression (` `), a comma-separated list of schedule aliases (`@hourly`, `@daily`, `@midnight`, `@weekly`, `@monthly`, `@annually`, `@yearly`), or empty to disable automatic snapshots (the default)", enable_ID_shifting: "Enable ID shifting overlay (allows attach by multiple isolated instances)", block_filesystem: "File system of the storage volume: `btrfs`, `ext4` or `xfs` (`ext4` if not set)", volume_configuration: "```{tip}\nIn addition to these configurations, you can also set default values for the storage volume configurations. See {ref}`storage-configure-vol-default`.\n```"} incus-7.0.0/doc/support.md000066400000000000000000000016501517523235500154540ustar00rootroot00000000000000# Support Incus maintains different release branches in parallel: - Long term support (LTS) releases: 6.0 - Feature releases: Incus 6.x The Incus 6.0 LTS release will be supported until June 2029. The first 2 years of support will include bug and security fixes as well as minor usability improvements. The remaining 3 years of support (following Incus 7.0 LTS' release) will only feature security updates. Feature releases are pushed out about monthly and contain new features as well as bugfixes. The normal support length for those releases is until the next release comes out. Some Linux distributions might offer longer support for particular feature releases that they decided to ship. % Include content from [../README.md](../README.md) ```{include} ../README.md :start-after: :end-before: ``` incus-7.0.0/doc/syscall-interception.md000066400000000000000000000120731517523235500201140ustar00rootroot00000000000000# System call interception Incus supports intercepting some specific system calls from unprivileged containers. If they're considered to be safe, it executes them with elevated privileges on the host. Doing so comes with a performance impact for the syscall in question and will cause some work for Incus to evaluate the request and if allowed, process it with elevated privileges. Enabling of specific system call interception options is done on a per-container basis through container configuration options. ## Available system calls ### `mknod` / `mknodat` The `mknod` and `mknodat` system calls can be used to create a variety of special files. Most commonly inside containers, they may be called to create block or character devices. Creating such devices isn't allowed in unprivileged containers as this is a very easy way to escalate privileges by allowing direct write access to resources like disks or memory. But there are files which are safe to create. For those, intercepting this syscall may unblock some specific workloads and allow them to run inside an unprivileged containers. The devices which are currently allowed are: - overlayfs whiteout (char 0:0) - `/dev/console` (char 5:1) - `/dev/full` (char 1:7) - `/dev/null` (char 1:3) - `/dev/random` (char 1:8) - `/dev/tty` (char 5:0) - `/dev/urandom` (char 1:9) - `/dev/zero` (char 1:5) All file types other than character devices are currently sent to the kernel as usual, so enabling this feature doesn't change their behavior at all. This can be enabled by setting `security.syscalls.intercept.mknod` to `true`. ### `bpf` The `bpf` system call is used to manage eBPF programs in the kernel. Those can be attached to a variety of kernel subsystems. In general, loading of eBPF programs that are not trusted can be problematic as it can facilitate timing based attacks. Incus' eBPF support is currently restricted to programs managing devices cgroup entries. To enable it, you need to set both `security.syscalls.intercept.bpf` and `security.syscalls.intercept.bpf.devices` to true. ### `mount` The `mount` system call allows for mounting both physical and virtual file systems. By default, unprivileged containers are restricted by the kernel to just a handful of virtual and network file systems. To allow mounting physical file systems, system call interception can be used. Incus offers a variety of options to handle this. `security.syscalls.intercept.mount` is used to control the entire feature and needs to be turned on for any of the other options to work. `security.syscalls.intercept.mount.allowed` allows specifying a list of file systems which can be directly mounted in the container. This is the most dangerous option as it allows the user to feed data that is not trusted at the kernel. This can easily be used to crash the host system or to attack it. It should only ever be used in trusted environments. `security.syscalls.intercept.mount.shift` can be set on top of that so the resulting mount is shifted to the UID/GID map used by the container. This is needed to avoid everything showing up as `nobody`/`nogroup` inside of unprivileged containers. The much safer alternative to those is `security.syscalls.intercept.mount.fuse` which can be set to pairs of file-system name and FUSE handler. When this is set, an attempt at mounting one of the configured file systems will be transparently redirected to instead calling the FUSE equivalent of that file system. As this is all running as the caller, it avoids the entire issue around the kernel attack surface and so is generally considered to be safe, though you should keep in mind that any kind of system call interception makes for an easy way to overload the host system. ### `sched_setscheduler` The `sched_setscheduler` system call is used to manage process priority. Granting this may allow a user to significantly increase the priority of their processes, potentially taking a lot of system resources. It also allows access to schedulers like `SCHED_FIFO` which are generally considered to be flawed and can significantly impact overall system stability. This is why under normal conditions, only the real root user (or global `CAP_SYS_NICE`) would allow its use. ### `setxattr` The `setxattr` system call is used to set extended attributes on files. The attributes which are handled by this currently are: - `trusted.overlay.opaque` (overlayfs directory whiteout) Note that because the mediation must happen on a number of character strings, there is no easy way at present to only intercept the few attributes we care about. As we only allow the attributes above, this may result in breakage for other attributes that would have been previously allowed by the kernel. This can be enabled by setting `security.syscalls.intercept.setxattr` to `true`. ### `sysinfo` The `sysinfo` system call is used by some distributions instead of `/proc/` entries to report on resource usage. In order to provide resource usage information specific to the container, rather than the whole system, this syscall interception mode uses cgroup-based resource usage information to fill in the system call response. incus-7.0.0/doc/third_party.md000066400000000000000000000056441517523235500163000ustar00rootroot00000000000000# Third party tools and integrations Below are a list of common operations tools which feature Incus support, either natively or through a plugin. ## Terraform / OpenTofu [Terraform](https://developer.hashicorp.com/terraform) and [OpenTofu](https://opentofu.org) are infrastructure as code tools which focus on creating the infrastructure itself. For Incus, this means the ability to create projects, profiles, networks, storage volumes and of course instances. In most cases, one will then use Ansible to deploy the workloads themselves once the instances and everything else they need as been put in place. The integration with Incus is done through a [dedicated provider](https://github.com/lxc/terraform-provider-incus). ## Ansible [Ansible](https://www.ansible.com) is an infrastructure as code tool with particular focus on software provisioning and configuration management. It does most of its work by first connecting to the system that it's deploying software on. To do that, it can connect over SSH and a variety of other protocols, one of which is [Incus](https://docs.ansible.com/ansible/latest/collections/community/general/incus_connection.html). That allows for easily deploying software inside of Incus instances without needing to first setup SSH. ## Packer [Packer](https://developer.hashicorp.com/packer) is a tool to generate custom OS images across a wide variety of platforms. A [plugin](https://developer.hashicorp.com/packer/integrations/bketelsen/incus) exists that allows Packer to generate Incus images directly. ## Distrobuilder [Distrobuilder](https://github.com/lxc/distrobuilder) is an image building tool most known for producing the official LXC and Incus images. It consumes YAML definitions for its images and generates LXC container images as well as Incus container and VM images. The focus of Distrobuilder is in producing clean images from scratch, as opposed to repacking existing images. ## GARM [GARM](https://github.com/cloudbase/garm) is the Github Actions Runner Manager which allows for running self-hosted Github runners. It supports a variety of providers for those runners, including [Incus](https://github.com/cloudbase/garm-provider-incus). ## Kubernetes [Kubernetes](https://kubernetes.io), also known as K8s, is an open source system for automating deployment, scaling, and management of containerized applications. [Cluster API](https://cluster-api.sigs.k8s.io) is a Kubernetes sub-project focused on providing declarative APIs and tooling to simplify provisioning, upgrading, and operating multiple Kubernetes clusters. [The Cluster API provider for Incus](https://capn.linuxcontainers.org) is an Infrastructure Provider for Cluster API, which enables deploying Kubernetes clusters on infrastructure operated by Incus. The provider can be used in single-node development environments for evaluation and testing, but also work with multi-node Incus clusters to deploy and manage production Kubernetes clusters. incus-7.0.0/doc/tutorial/000077500000000000000000000000001517523235500152575ustar00rootroot00000000000000incus-7.0.0/doc/tutorial/first_steps.md000066400000000000000000000227521517523235500201560ustar00rootroot00000000000000(first-steps)= # First steps with Incus This tutorial guides you through the first steps with Incus. It covers installing and initializing Incus, creating and configuring some instances, interacting with the instances, and creating snapshots. After going through these steps, you will have a general idea of how to use Incus, and you can start exploring more advanced use cases! ## Install and initialize Incus 1. Install the Incus package Incus is available on most common Linux distributions. For detailed distribution-specific instructions, refer to {ref}`installing`. 1. Allow your user to control Incus Access to Incus in the packages above is controlled through two groups: - `incus` allows basic user access, no configuration and all actions restricted to a per-user project. - `incus-admin` allows full control over Incus. To control Incus without having to run all commands as root, you can add yourself to the `incus-admin` group: sudo adduser $USER incus-admin newgrp incus-admin The `newgrp` step is needed in any terminal that interacts with Incus until you restart your user session. 1. Initialize Incus ```{note} If you are migrating from an existing LXD installation, skip this step and refer to {ref}`server-migrate-lxd` instead. ``` Incus requires some initial setup for networking and storage. This can be done interactively through: incus admin init Or a basic automated configuration can be applied with just: incus admin init --minimal If you want to tune the initialization options, see {ref}`initialize` for more information. ## Launch and inspect instances Incus is image based and can load images from different image servers. In this tutorial, we will use the [official image server](https://images.linuxcontainers.org/). You can list all images that are available on this server with: incus image list images: See {ref}`images` for more information about the images that Incus uses. Now, let's start by launching a few instances. With *instance*, we mean either a container or a virtual machine. See {ref}`containers-and-vms` for information about the difference between the two instance types. For managing instances, we use the Incus command line client `incus`. 1. Launch a container called `first` using the Debian 12 image: incus launch images:debian/12 first ```{note} Launching this container takes a few seconds, because the image must be downloaded and unpacked first. ``` 1. Launch a container called `second` using the same image: incus launch images:debian/12 second ```{note} Launching this container is quicker than launching the first, because the image is already available. ``` 1. Copy the first container into a container called `third`: incus copy first third 1. Launch a VM called `debian-vm` using the Debian 12 image: incus launch images:debian/12 debian-vm --vm ```{note} Even though you are using the same image name to launch the instance, Incus downloads a slightly different image that is compatible with VMs. ``` 1. Check the list of instances that you launched: incus list You will see that all but the third container are running. This is because you created the third container by copying the first, but you didn't start it. You can start the third container with: incus start third 1. Query more information about each instance with: incus info first incus info second incus info third incus info debian-vm 1. We don't need all of these instances for the remainder of the tutorial, so let's clean some of them up: 1. Stop the second container: incus stop second 1. Delete the second container: incus delete second 1. Delete the third container: incus delete third Since this container is running, you get an error message that you must stop it first. Alternatively, you can force-delete it: incus delete third --force See {ref}`instances-create` and {ref}`instances-manage` for more information. ## Configure instances There are several limits and configuration options that you can set for your instances. See {ref}`instance-options` for an overview. Let's create another container with some resource limits: 1. Launch a container and limit it to one vCPU and 192 MiB of RAM: incus launch images:debian/12 limited --config limits.cpu=1 --config limits.memory=192MiB 1. Check the current configuration and compare it to the configuration of the first (unlimited) container: incus config show limited incus config show first 1. Check the amount of free and used memory on the parent system and on the two containers: free -m incus exec first -- free -m incus exec limited -- free -m ```{note} The total amount of memory is identical for the parent system and the first container, because by default, the container inherits the resources from its parent environment. The limited container, on the other hand, has only 192 MiB available. ``` 1. Check the number of CPUs available on the parent system and on the two containers: nproc incus exec first -- nproc incus exec limited -- nproc ```{note} Again, the number is identical for the parent system and the first container, but reduced for the limited container. ``` 1. You can also update the configuration while your container is running: 1. Configure a memory limit for your container: incus config set limited limits.memory=128MiB 1. Check that the configuration has been applied: incus config show limited 1. Check the amount of memory that is available to the container: incus exec limited -- free -m Note that the number has changed. 1. Depending on the instance type and the storage drivers that you use, there are more configuration options that you can specify. For example, you can configure the size of the root disk device for a VM: 1. Check the current size of the root disk device of the Debian VM: ```{terminal} :input: incus exec debian-vm -- df -h Filesystem Size Used Avail Use% Mounted on /dev/root 9.6G 1.4G 8.2G 15% / tmpfs 483M 0 483M 0% /dev/shm tmpfs 193M 604K 193M 1% /run tmpfs 5.0M 0 5.0M 0% /run/lock tmpfs 50M 14M 37M 27% /run/incus_agent /dev/sda15 105M 6.1M 99M 6% /boot/efi ``` 1. Override the size of the root disk device: incus config device override debian-vm root size=30GiB 1. Restart the VM: incus restart debian-vm 1. Check the size of the root disk device again: ```{terminal} :input: incus exec debian-vm -- df -h Filesystem Size Used Avail Use% Mounted on /dev/root 29G 1.4G 28G 5% / tmpfs 483M 0 483M 0% /dev/shm tmpfs 193M 588K 193M 1% /run tmpfs 5.0M 0 5.0M 0% /run/lock tmpfs 50M 14M 37M 27% /run/incus_agent /dev/sda15 105M 6.1M 99M 6% /boot/efi ``` See {ref}`instances-configure` and {ref}`instance-config` for more information. ## Interact with instances You can interact with your instances by running commands in them (including an interactive shell) or accessing the files in the instance. Start by launching an interactive shell in your instance: 1. Run the `bash` command in your container: incus exec first -- bash 1. Enter some commands, for example, display information about the operating system: cat /etc/*release 1. Exit the interactive shell: exit Instead of logging on to the instance and running commands there, you can run commands directly from the host. For example, you can install a command line tool on the instance and run it: incus exec first -- apt-get update incus exec first -- apt-get install sl -y incus exec first -- /usr/games/sl See {ref}`run-commands` for more information. You can also access the files from your instance and interact with them: 1. Pull a file from the container: incus file pull first/etc/hosts . 1. Add an entry to the file: echo "1.2.3.4 my-example" >> hosts 1. Push the file back to the container: incus file push hosts first/etc/hosts 1. Use the same mechanism to access log files: incus file pull first/var/log/syslog - | less ```{note} Press `q` to exit the `less` command. ``` See {ref}`instances-access-files` for more information. ## Manage snapshots You can create a snapshot of your instance, which makes it easy to restore the instance to a previous state. 1. Create a snapshot called "clean": incus snapshot create first clean 1. Confirm that the snapshot has been created: incus list first incus info first ```{note} `incus list` shows the number of snapshots. `incus info` displays information about each snapshot. ``` 1. Break the container: incus exec first -- rm /usr/bin/bash 1. Confirm the breakage: incus exec first -- bash ```{note} You do not get a shell, because you deleted the `bash` command. ``` 1. Restore the container to the state of the snapshot: incus snapshot restore first clean 1. Confirm that everything is back to normal: incus exec first -- bash exit 1. Delete the snapshot: incus snapshot delete first clean See {ref}`instances-snapshots` for more information. incus-7.0.0/doc/userns-idmap.md000066400000000000000000000075031517523235500163520ustar00rootroot00000000000000(userns-idmap)= # Idmaps for user namespace Incus runs safe containers. This is achieved mostly through the use of user namespaces which make it possible to run containers unprivileged, greatly limiting the attack surface. User namespaces work by mapping a set of UIDs and GIDs on the host to a set of UIDs and GIDs in the container. For example, we can define that the host UIDs and GIDs from 100000 to 165535 may be used by Incus and should be mapped to UID/GID 0 through 65535 in the container. As a result a process running as UID 0 in the container will actually be running as UID 100000. Allocations should always be of at least 65536 UIDs and GIDs to cover the POSIX range including root (0) and nobody (65534). ## Kernel support User namespaces require a kernel >= 3.12, Incus will start even on older kernels but will refuse to start containers. ## Allowed ranges On most hosts, Incus will check `/etc/subuid` and `/etc/subgid` for allocations for the `root` user and on first start, set the default profile to use the first 65536 UIDs and GIDs from that range. If the range is shorter than 65536 (which includes no range at all), then Incus will fail to create or start any container until this is corrected. If some but not all of `/etc/subuid`, `/etc/subgid`, `newuidmap` (path lookup) and `newgidmap` (path lookup) can be found on the system, Incus will fail the startup of any container until this is corrected as this shows a broken shadow setup. If none of those files can be found, then Incus will assume a 1000000000 UID/GID range starting at a base UID/GID of 1000000. This is the most common case and is usually the recommended setup when not running on a system which also hosts fully unprivileged containers (where the container runtime itself runs as a user). ## Varying ranges between hosts The source map is sent when moving containers between hosts so that they can be remapped on the receiving host. ## Different idmaps per container Incus supports using different idmaps per container, to further isolate containers from each other. This is controlled with two per-container configuration keys, `security.idmap.isolated` and `security.idmap.size`. Containers with `security.idmap.isolated` will have a unique ID range computed for them among the other containers with `security.idmap.isolated` set (if none is available, setting this key will simply fail). Containers with `security.idmap.size` set will have their ID range set to this size. Isolated containers without this property set default to a ID range of size 65536; this allows for POSIX compliance and a `nobody` user inside the container. To select a specific map, the `security.idmap.base` key will let you override the auto-detection mechanism and tell Incus what host UID/GID you want to use as the base for the container. These properties require a container reboot to take effect. ## Custom idmaps Incus also supports customizing bits of the idmap, e.g. to allow users to bind mount parts of the host's file system into a container without the need for any UID-shifting file system. The per-container configuration key for this is `raw.idmap`, and looks like: both 1000 1000 uid 50-60 500-510 gid 100000-110000 10000-20000 The first line configures both the UID and GID 1000 on the host to map to UID 1000 inside the container (this can be used for example to bind mount a user's home directory into a container). The second and third lines map only the UID or GID ranges into the container, respectively. The second entry per line is the source ID, i.e. the ID on the host, and the third entry is the range inside the container. These ranges must be the same size. This property requires a container reboot to take effect. Remember that you may need to add an entry for the `root` user into `/etc/subid` and/or `/etc/subgid` so the container is allowed to make use of it. incus-7.0.0/go.mod000066400000000000000000000215311517523235500137570ustar00rootroot00000000000000module github.com/lxc/incus/v7 go 1.25.6 require ( github.com/FuturFusion/vsock v0.0.0-20260219213046-d78a7104f821 github.com/LINBIT/golinstor v0.60.0 github.com/adhocore/gronx v1.19.6 github.com/apex/log v1.9.0 github.com/aws/aws-sdk-go-v2 v1.41.7 github.com/aws/aws-sdk-go-v2/credentials v1.19.16 github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.20 github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 github.com/aws/smithy-go v1.25.1 github.com/cenkalti/backoff/v5 v5.0.3 github.com/checkpoint-restore/go-criu/v8 v8.2.0 github.com/cowsql/go-cowsql v1.22.0 github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e github.com/dustinkirkland/golang-petname v0.0.0-20260215035315-f0c533e9ce9b github.com/fatih/color v1.19.0 github.com/flosch/pongo2/v6 v6.0.0 github.com/fvbommel/sortorder v1.1.0 github.com/go-chi/chi/v5 v5.2.5 github.com/go-jose/go-jose/v4 v4.1.4 github.com/go-logr/logr v1.4.3 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/gopacket v1.1.19 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 github.com/gosexy/gettext v0.0.0-20160830220431-74466a0a0c4a github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2 github.com/jaypipes/pcidb v1.1.1 github.com/jochenvg/go-udev v0.0.0-20240801134859-b65ed646224b github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/lxc/go-lxc v0.0.0-20260316180011-3af4ce000ed7 github.com/lxc/incus-os/incus-osd v0.0.0-20260426171132-5b5f02e943ee github.com/mattn/go-colorable v0.1.14 github.com/mattn/go-runewidth v0.0.23 github.com/mattn/go-sqlite3 v1.14.44 github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 github.com/mdlayher/ndp v1.1.0 github.com/mdlayher/netx v0.0.0-20230430222610-7e21880baee8 github.com/mdlayher/vsock v1.2.1 github.com/miekg/dns v1.1.72 github.com/mitchellh/mapstructure v1.5.0 github.com/olekukonko/tablewriter v1.1.4 github.com/opencontainers/runtime-spec v1.3.0 github.com/opencontainers/umoci v0.6.1-0.20251213054154-70fc5ee1f4df github.com/openfga/go-sdk v0.8.0 github.com/osrg/gobgp/v4 v4.5.0 github.com/ovn-kubernetes/libovsdb v0.8.1 github.com/pierrec/lz4/v4 v4.1.26 github.com/pires/go-proxyproto v0.12.0 github.com/pkg/sftp v1.13.10 github.com/pkg/xattr v0.4.12 github.com/shirou/gopsutil/v4 v4.26.4 github.com/sirupsen/logrus v1.9.4 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 github.com/vishvananda/netlink v1.3.1 github.com/zitadel/oidc/v3 v3.47.5 go.starlark.net v0.0.0-20260326113308-fadfc96def35 go.yaml.in/yaml/v4 v4.0.0-rc.4 golang.org/x/crypto v0.50.0 golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.20.0 golang.org/x/sys v0.43.0 golang.org/x/term v0.42.0 golang.org/x/text v0.36.0 golang.org/x/tools v0.44.0 google.golang.org/protobuf v1.36.11 k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 software.sslmate.com/src/go-pkcs12 v0.7.1 ) require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect github.com/Rican7/retry v0.3.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/hub v1.0.2 // indirect github.com/cenkalti/rpc2 v1.0.5 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v1.0.0-rc.4 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect github.com/docker/go-units v0.5.0 // indirect github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 // indirect github.com/eapache/channels v1.1.0 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/fsnotify/fsnotify v1.10.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gaissmai/bart v0.26.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.30.2 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/google/go-querystring v1.2.0 // indirect github.com/google/renameio v1.0.1 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jkeiser/iter v0.0.0-20200628201005-c8aa0ae784d1 // indirect github.com/josharian/native v1.1.0 // indirect github.com/k-sone/critbitgo v1.4.0 // indirect github.com/klauspost/compress v1.18.6 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect github.com/lxc/incus/v6 v6.23.1-0.20260327174201-6acde8bd711a // indirect github.com/mattn/go-isatty v0.0.22 // indirect github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 // indirect github.com/mdlayher/packet v1.1.2 // indirect github.com/mdlayher/socket v0.6.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/muhlemmer/gu v0.3.1 // indirect github.com/muhlemmer/httpforwarded v0.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect github.com/olekukonko/errors v1.3.0 // indirect github.com/olekukonko/ll v0.1.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect github.com/pelletier/go-toml/v2 v2.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.20.1 // indirect github.com/rootless-containers/proto/go-proto v0.0.0-20260207013450-f6ee952d53d9 // indirect github.com/rs/cors v1.11.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/segmentio/fasthash v1.0.3 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/viper v1.21.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect github.com/urfave/cli v1.22.17 // indirect github.com/vbatts/go-mtree v0.7.0 // indirect github.com/vishvananda/netns v0.0.5 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zitadel/logging v0.7.0 // indirect github.com/zitadel/schema v1.3.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel v1.43.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/mod v0.35.0 // indirect golang.org/x/net v0.53.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect google.golang.org/grpc v1.80.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect moul.io/http2curl/v2 v2.3.0 // indirect ) incus-7.0.0/go.sum000066400000000000000000003070371517523235500140140ustar00rootroot00000000000000cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/FuturFusion/vsock v0.0.0-20260219213046-d78a7104f821 h1:t2eOnMiztYbjWnbwQS5U9rvkzHAPaWelKkE+/FV1C18= github.com/FuturFusion/vsock v0.0.0-20260219213046-d78a7104f821/go.mod h1:0atKpUm0hXZMv6+9Kf5crGqV9KKtvkPAElG7/9CCFwU= github.com/LINBIT/golinstor v0.60.0 h1:85QhDoLPQXMmqaZ9TKnGFdymu2FKb1Vj37ANVjBxHAQ= github.com/LINBIT/golinstor v0.60.0/go.mod h1:TXAMGiskT4fY/koTCOt6qqF60uWobDqUHoB8srNeCnY= github.com/Rican7/retry v0.3.0/go.mod h1:CxSDrhAyXmTMeEuRAnArMu1FHu48vtfjLREWqVl7Vw0= github.com/Rican7/retry v0.3.1 h1:scY4IbO8swckzoA/11HgBwaZRJEyY9vaNJshcdhp1Mc= github.com/Rican7/retry v0.3.1/go.mod h1:CxSDrhAyXmTMeEuRAnArMu1FHu48vtfjLREWqVl7Vw0= github.com/adhocore/gronx v1.19.6 h1:5KNVcoR9ACgL9HhEqCm5QXsab/gI4QDIybTAWcXDKDc= github.com/adhocore/gronx v1.19.6/go.mod h1:7oUY1WAU8rEJWmAxXR2DN0JaO4gi9khSgKjiRypqteg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.20 h1:iEd9YuD/T9SrH/7NoMZ3Jz81OLqVfxOa94XZNqpSE9s= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.20/go.mod h1:hHSAgymEQbdCmEDXvNxhXiKJxJOWRJi84Gp34anL858= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 h1:mxuT1xE+dI54NW3RkNjP8DUT5HXqbkiAFvfdyDFwE5c= github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cenkalti/hub v1.0.2 h1:Nqv9TNaA9boeO2wQFW8o87BY3zKthtnzXmWGmJqhAV8= github.com/cenkalti/hub v1.0.2/go.mod h1:8LAFAZcCasb83vfxatMUnZHRoQcffho2ELpHb+kaTJU= github.com/cenkalti/rpc2 v1.0.5 h1:T6l4SS3ja3eaJfRyZrn7Oco/PSx/pr3YK5cjCgLVLTk= github.com/cenkalti/rpc2 v1.0.5/go.mod h1:2yfU5b86vOr16+iY1jN3MvT6Kxc9Nf8j5iZWwUf7iaw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v8 v8.2.0 h1:dsgMgj/eJtZNKn3qn/+Ri0b4bd0uo6o2zt1yd8Nj2NI= github.com/checkpoint-restore/go-criu/v8 v8.2.0/go.mod h1:HVKJ1dK+bowJcFI1MtdL2ECIuY+/AtRMHzD9Lqa4uA4= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v1.0.0-rc.4 h1:M42JrUT4zfZTqtkUwkr0GzmUWbfyO5VO0Q5b3op97T4= github.com/containerd/platforms v1.0.0-rc.4/go.mod h1:lKlMXyLybmBedS/JJm11uDofzI8L2v0J2ZbYvNsbq1A= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cowsql/go-cowsql v1.22.0 h1:NOMuu3RWkkbKtQ3V+ny9ksR4q3a/h4jU54CbY1BEMBM= github.com/cowsql/go-cowsql v1.22.0/go.mod h1:+QzPcM7QRPIBI8XhsKJ47iUtxGY53lsYGX51G1WQ/4s= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e h1:vUmf0yezR0y7jJ5pceLHthLaYf4bA5T14B6q39S4q2Q= github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e/go.mod h1:YTIHhz/QFSYnu/EhlF2SpU2Uk+32abacUYA5ZPljz1A= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeMaEQVy6e8CarIhscYQlNmw5e3G36y7l7Y21Ao= github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1:56wL82FO0bfMU5RvfXoIwSOP2ggqqxT+tAfNEIyxuHw= github.com/dustinkirkland/golang-petname v0.0.0-20260215035315-f0c533e9ce9b h1:qZ21OofI7zneC9dOEqul4FmIWz/YjJJMrf6fL7jrFYQ= github.com/dustinkirkland/golang-petname v0.0.0-20260215035315-f0c533e9ce9b/go.mod h1:8AuBTZBRSFqEYBPYULd+NN474/zZBLP+6WeT5S9xlAc= github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k= github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= github.com/flosch/pongo2/v6 v6.0.0 h1:lsGru8IAzHgIAw6H2m4PCyleO58I40ow6apih0WprMU= github.com/flosch/pongo2/v6 v6.0.0/go.mod h1:CuDpFm47R0uGGE7z13/tTlt1Y6zdxvr2RLT5LJhsHEU= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.10.0 h1:Xx/5Ydg9CeBDX/wi4VJqStNtohYjitZhhlHt4h3St1M= github.com/fsnotify/fsnotify v1.10.0/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQmYw= github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gaissmai/bart v0.26.1 h1:+w4rnLGNlA2GDVn382Tfe3jOsK5vOr5n4KmigJ9lbTo= github.com/gaissmai/bart v0.26.1/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug= github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.30.2 h1:JiFIMtSSHb2/XBUbWM4i/MpeQm9ZK2xqPNk8vgvu5JQ= github.com/go-playground/validator/v10 v10.30.2/go.mod h1:mAf2pIOVXjTEBrwUMGKkCWKKPs9NheYGabeB04txQSc= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0= github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU= github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA= github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/gosexy/gettext v0.0.0-20160830220431-74466a0a0c4a h1:N2b2mb4Gki1SlF3WuhR9P1YHOpl7oy/b+xxX4A3iM2E= github.com/gosexy/gettext v0.0.0-20160830220431-74466a0a0c4a/go.mod h1:IEJaV4/6J0VpoQ33kFCUUP6umRjrcBVEbOva6XCub/Q= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8= github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2 h1:G3irkWwmpl0vH/nn83K2AHqLUZweC7XAONuwXy/w9Co= github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2/go.mod h1:qfvBmyDNp+/liLEYWRvqny/PEz9hGe2Dz833eXILSmo= github.com/jarcoal/httpmock v1.4.1 h1:0Ju+VCFuARfFlhVXFc2HxlcQkfB+Xq12/EotHko+x2A= github.com/jarcoal/httpmock v1.4.1/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0= github.com/jaypipes/pcidb v1.1.1 h1:QmPhpsbmmnCwZmHeYAATxEaoRuiMAJusKYkUncMC0ro= github.com/jaypipes/pcidb v1.1.1/go.mod h1:x27LT2krrUgjf875KxQXKB0Ha/YXLdZRVmw6hH0G7g8= github.com/jeremija/gosubmit v0.2.8 h1:mmSITBz9JxVtu8eqbN+zmmwX7Ij2RidQxhcwRVI4wqA= github.com/jeremija/gosubmit v0.2.8/go.mod h1:Ui+HS073lCFREXBbdfrJzMB57OI/bdxTiLtrDHHhFPI= github.com/jkeiser/iter v0.0.0-20200628201005-c8aa0ae784d1 h1:smvLGU3obGU5kny71BtE/ibR0wIXRUiRFDmSn0Nxz1E= github.com/jkeiser/iter v0.0.0-20200628201005-c8aa0ae784d1/go.mod h1:fP/NdyhRVOv09PLRbVXrSqHhrfQypdZwgE2L4h2U5C8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jochenvg/go-udev v0.0.0-20240801134859-b65ed646224b h1:Pzf7tldbCVqwl3NnOnTamEWdh/rL41fsoYCn2HdHgRA= github.com/jochenvg/go-udev v0.0.0-20240801134859-b65ed646224b/go.mod h1:IBDUGq30U56w969YNPomhMbRje1GrhUsCh7tHdwgLXA= github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/k-sone/critbitgo v1.4.0 h1:l71cTyBGeh6X5ATh6Fibgw3+rtNT80BA0uNNWgkPrbE= github.com/k-sone/critbitgo v1.4.0/go.mod h1:7E6pyoyADnFxlUBEKcnfS49b7SUAQGMK+OAp/UQvo0s= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/lxc/go-lxc v0.0.0-20260316180011-3af4ce000ed7 h1:MXZvjx5IYff3AacumHgmaJqWXyfROImWfJzmMP1ye4o= github.com/lxc/go-lxc v0.0.0-20260316180011-3af4ce000ed7/go.mod h1:3UTWXVcHfgxE7JM4ZUnsy6bDA8L1vuzwJbJRF6dlB90= github.com/lxc/incus-os/incus-osd v0.0.0-20260426171132-5b5f02e943ee h1:fVYJ6NWsCk7K3I+1poS0Jzofqn1valLBZ1Ftf5d8Mrg= github.com/lxc/incus-os/incus-osd v0.0.0-20260426171132-5b5f02e943ee/go.mod h1:ITvI7wOwUbS9QeuFVXlGVgrBa/JEO5HdfO72u6lUlL8= github.com/lxc/incus/v6 v6.23.1-0.20260327174201-6acde8bd711a h1:jquoHV4ziNNU5xDent4Ki9B6YJ8Ybq/yfeaTjubfJyU= github.com/lxc/incus/v6 v6.23.1-0.20260327174201-6acde8bd711a/go.mod h1:efEbxmSexfg8VyYQnBgNQz0dZZLci3s90xcU+VXoCYc= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.44 h1:3VSe+xafpbzsLbdr2AWlAZk9yRHiBhTBakioXaCKTF8= github.com/mattn/go-sqlite3 v1.14.44/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 h1:ql8x//rJsHMjS+qqEag8n3i4azw1QneKh5PieH9UEbY= github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875/go.mod h1:kfOoFJuHWp76v1RgZCb9/gVUc7XdY877S2uVYbNliGc= github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 h1:2oDp6OOhLxQ9JBoUuysVz9UZ9uI6oLUbvAZu0x8o+vE= github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118/go.mod h1:ZFUnHIVchZ9lJoWoEGUg8Q3M4U8aNNWA3CVSUTkW4og= github.com/mdlayher/ndp v1.1.0 h1:QylGKGVtH60sKZUE88+IW5ila1Z/M9/OXhWdsVKuscs= github.com/mdlayher/ndp v1.1.0/go.mod h1:FmgESgemgjl38vuOIyAHWUUL6vQKA/pQNkvXdWsdQFM= github.com/mdlayher/netx v0.0.0-20230430222610-7e21880baee8 h1:HMgSn3c16SXca3M+n6fLK2hXJLd4mhKAsZZh7lQfYmQ= github.com/mdlayher/netx v0.0.0-20230430222610-7e21880baee8/go.mod h1:qhZhwMDNWwZglKfwuWm0U9pCr/YKX1QAEwwJk9qfiTQ= github.com/mdlayher/packet v1.0.0/go.mod h1:eE7/ctqDhoiRhQ44ko5JZU2zxB88g+JH/6jmnjzPjOU= github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY= github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4= github.com/mdlayher/socket v0.2.1/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= github.com/mdlayher/socket v0.6.0 h1:ScZPaAGyO1icQnbFrhPM8mnXyMu9qukC1K4ZoM2IQKU= github.com/mdlayher/socket v0.6.0/go.mod h1:q7vozUAnxSqnjHc12Fik5yUKIzfZ8ITCfMkhOtE9z18= github.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ= github.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnENvE+SE= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/muhlemmer/gu v0.3.1 h1:7EAqmFrW7n3hETvuAdmFmn4hS8W+z3LgKtrnow+YzNM= github.com/muhlemmer/gu v0.3.1/go.mod h1:YHtHR+gxM+bKEIIs7Hmi9sPT3ZDUvTN/i88wQpZkrdM= github.com/muhlemmer/httpforwarded v0.1.0 h1:x4DLrzXdliq8mprgUMR0olDvHGkou5BJsK/vWUetyzY= github.com/muhlemmer/httpforwarded v0.1.0/go.mod h1:yo9czKedo2pdZhoXe+yDkGVbU0TJ0q9oQ90BVoDEtw0= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0= github.com/olekukonko/errors v1.3.0 h1:teJvgLGUEqMzBUms+Dj3/3szNqCG/Jdw9iDbum8fR6U= github.com/olekukonko/errors v1.3.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= github.com/olekukonko/ll v0.1.8 h1:ysHCJRGHYKzmBSdz9w5AySztx7lG8SQY+naTGYUbsz8= github.com/olekukonko/ll v0.1.8/go.mod h1:RPRC6UcscfFZgjo1nulkfMH5IM0QAYim0LfnMvUuozw= github.com/olekukonko/tablewriter v1.1.4 h1:ORUMI3dXbMnRlRggJX3+q7OzQFDdvgbN9nVWj1drm6I= github.com/olekukonko/tablewriter v1.1.4/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/umoci v0.6.1-0.20251213054154-70fc5ee1f4df h1:9hvwN64VeuL1L0Jgp8bxTPmd5IZQoHmeXGWrVqsEhN0= github.com/opencontainers/umoci v0.6.1-0.20251213054154-70fc5ee1f4df/go.mod h1:s6d/s4QJAZTF92hEU6ozuHjE0+VRc6kVe1QIWfvL7KY= github.com/openfga/go-sdk v0.8.0 h1:xwjqxO1v3QU6Tzx5xu33jpOpurqJYxXPM2uiR2QsfCo= github.com/openfga/go-sdk v0.8.0/go.mod h1:s5zZD4NFmf6yQa74iJEAwBZgHx4NaNzoOOVmA6YcY2M= github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= github.com/osrg/gobgp/v4 v4.5.0 h1:1jS4cMxUYSo36UfDyigLOLYEg6Oh+9I2mrnjjgAGCFk= github.com/osrg/gobgp/v4 v4.5.0/go.mod h1:pgu8waqTvZUYl4eQuPrKNOaVwhHv7Zt9YymuzCaX7f8= github.com/ovn-kubernetes/libovsdb v0.8.1 h1:M2J8bcJt5mXCom0HqzfEtuHkT80CTSQRcYG7acT8gf4= github.com/ovn-kubernetes/libovsdb v0.8.1/go.mod h1:ZlnHLzagmLOSvyd9qfxBIZp6wOSOw0IsRsc+6lNUGbU= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.3.0 h1:k59bC/lIZREW0/iVaQR8nDHxVq8OVlIzYCOJf421CaM= github.com/pelletier/go-toml/v2 v2.3.0/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/peterh/liner v1.2.1/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pires/go-proxyproto v0.12.0 h1:TTCxD66dU898tahivkqc3hoceZp7P44FnorWyo9d5vM= github.com/pires/go-proxyproto v0.12.0/go.mod h1:qUvfqUMEoX7T8g0q7TQLDnhMjdTrxnG0hvpMn+7ePNI= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM= github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rootless-containers/proto/go-proto v0.0.0-20260207013450-f6ee952d53d9 h1:3w2GInbYbp08pUeQoM3qI1L4v8htpwHQN9AkfILlUSw= github.com/rootless-containers/proto/go-proto v0.0.0-20260207013450-f6ee952d53d9/go.mod h1:LLjEAc6zmycfeN7/1fxIphWQPjHpTt7ElqT7eVf8e4A= github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil/v4 v4.26.4 h1:B4SXVbcwTyrocPHEmWBC4uCYr4Xcu3MK1TXqbprAOWY= github.com/shirou/gopsutil/v4 v4.26.4/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc= github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8RaCKgVpHZnecvArXvPXcFkM= github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA= github.com/urfave/cli v1.22.17 h1:SYzXoiPfQjHBbkYxbew5prZHS1TOLT3ierW8SYLqtVQ= github.com/urfave/cli v1.22.17/go.mod h1:b0ht0aqgH/6pBYzzxURyrM4xXNgsoT/n2ZzwQiEhNVo= github.com/vbatts/go-mtree v0.7.0 h1:ytmOc3MTRidZiBi9VBCyZ2BHe4fZS47L5v7BVXDWW4E= github.com/vbatts/go-mtree v0.7.0/go.mod h1:EjdpFC+LZy1TXbRGNa1MKKgjQ+7ew3foMFJK8o4/TdY= github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0= github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4= github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zitadel/logging v0.7.0 h1:eugftwMM95Wgqwftsvj81isL0JK/hoScVqp/7iA2adQ= github.com/zitadel/logging v0.7.0/go.mod h1:9A6h9feBF/3u0IhA4uffdzSDY7mBaf7RE78H5sFMINQ= github.com/zitadel/oidc/v3 v3.47.5 h1:cR2z0oqa5XZkwpXQiPCUGqKtndrjHgEXb81y3oXocK4= github.com/zitadel/oidc/v3 v3.47.5/go.mod h1:XxFh0666HRXycyrKmono+3gY0RACpYJLgy4r/+kliKY= github.com/zitadel/schema v1.3.2 h1:gfJvt7dOMfTmxzhscZ9KkapKo3Nei3B6cAxjav+lyjI= github.com/zitadel/schema v1.3.2/go.mod h1:IZmdfF9Wu62Zu6tJJTH3UsArevs3Y4smfJIj3L8fzxw= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.starlark.net v0.0.0-20260326113308-fadfc96def35 h1:VYAqieSOJNxBDX8KJneTAwvdf4J4zRDE2u+UFXtt9h4= go.starlark.net v0.0.0-20260326113308-fadfc96def35/go.mod h1:Iue6g6iirlfLoVi/DYCi5/x0h/bAOuWF3dULTKpt2Vo= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U= go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM= google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbeZuxoSGOX/J+aYM= k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs= moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= software.sslmate.com/src/go-pkcs12 v0.7.1 h1:bxkUPRsvTPNRBZa4M/aSX4PyMOEbq3V8I6hbkG4F4Q8= software.sslmate.com/src/go-pkcs12 v0.7.1/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= incus-7.0.0/grafana/000077500000000000000000000000001517523235500142465ustar00rootroot00000000000000incus-7.0.0/grafana/incus.json000066400000000000000000003167771517523235500163070ustar00rootroot00000000000000{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "Prometheus instance with Incus metrics", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" }, { "name": "DS_LOKI", "label": "Loki", "description": "Loki instance with Incus entries", "type": "datasource", "pluginId": "loki", "pluginName": "Loki" } ], "__elements": {}, "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.4.2" }, { "type": "panel", "id": "logs", "name": "Logs", "version": "" }, { "type": "datasource", "id": "loki", "name": "Loki", "version": "1.0.0" }, { "type": "panel", "id": "piechart", "name": "Pie chart", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "description": "Overview of Incus instances", "editable": false, "fiscalYearStartMonth": 0, "gnetId": 19727, "graphTooltip": 0, "id": null, "links": [ { "icon": "external link", "tags": [], "targetBlank": true, "title": "Documentation", "type": "link", "url": "https://linuxcontainers.org/incus/docs/main/" } ], "liveNow": false, "panels": [ { "collapsed": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 319, "panels": [], "title": "Project overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "decimals": 0, "mappings": [], "unit": "s" }, "overrides": [ { "matcher": { "id": "byName", "options": "Others" }, "properties": [ { "id": "color", "value": { "fixedColor": "#81787be0", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 1 }, "id": 350, "options": { "displayLabels": [], "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false, "values": [ "value" ] }, "pieType": "donut", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "limit": 3, "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "7.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "topk(${top}, sum by (name) (increase(incus_cpu_seconds_total{mode!=\"idle\",job=\"$job\",project=\"$project\"}[$__range])))", "instant": true, "interval": "", "legendFormat": "{{name}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "sum(increase(incus_cpu_seconds_total{mode!=\"idle\",job=\"$job\",project=\"$project\"}[$__range])) - sum(topk(${top}, sum by (name) (increase(incus_cpu_seconds_total{mode!=\"idle\",job=\"$job\",project=\"$project\"}[$__range]))))", "hide": false, "instant": true, "interval": "", "legendFormat": "Others", "refId": "B" } ], "title": "Top ${top} CPU usage", "transparent": true, "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "decimals": 0, "mappings": [], "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Others" }, "properties": [ { "id": "color", "value": { "fixedColor": "#81787be0", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 1 }, "id": 364, "options": { "displayLabels": [], "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false, "values": [ "value" ] }, "pieType": "donut", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "limit": 3, "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "7.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "topk(${top}, incus_memory_MemTotal_bytes{job=\"$job\",project=\"$project\"} - incus_memory_MemFree_bytes - incus_memory_Cached_bytes)", "instant": true, "interval": "", "legendFormat": "{{name}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(incus_memory_MemTotal_bytes{job=\"$job\",project=\"$project\"} - incus_memory_MemFree_bytes - incus_memory_Cached_bytes) -\nsum(topk(${top}, incus_memory_MemTotal_bytes{job=\"$job\",project=\"$project\"} - incus_memory_MemFree_bytes - incus_memory_Cached_bytes))", "hide": false, "instant": true, "interval": "", "legendFormat": "Others", "refId": "B" } ], "title": "Top ${top} memory usage", "transparent": true, "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "decimals": 0, "mappings": [], "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Others" }, "properties": [ { "id": "color", "value": { "fixedColor": "#81787be0", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 1 }, "id": 351, "options": { "displayLabels": [], "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false, "values": [ "value" ] }, "pieType": "donut", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "limit": 3, "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "7.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "topk(${top}, incus_filesystem_size_bytes{job=\"$job\",project=\"$project\",mountpoint=\"/\"} - incus_filesystem_avail_bytes)", "hide": false, "instant": true, "interval": "", "legendFormat": "{{name}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(incus_filesystem_size_bytes{job=\"$job\",project=\"$project\",mountpoint=\"/\"} - incus_filesystem_avail_bytes) - \nsum(topk(${top}, incus_filesystem_size_bytes{job=\"$job\",project=\"$project\",mountpoint=\"/\"} - incus_filesystem_avail_bytes))", "hide": false, "interval": "", "legendFormat": "Others", "refId": "B" } ], "title": "Top ${top} rootfs usage", "transparent": true, "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total disk reads during the time interval", "fieldConfig": { "defaults": { "color": { "fixedColor": "semi-dark-blue", "mode": "fixed" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 18, "y": 1 }, "id": 360, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(increase(incus_disk_read_bytes_total{job=\"$job\",project=\"$project\"}[$__range]))", "hide": false, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Disk reads", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total disk writes during the time interval", "fieldConfig": { "defaults": { "color": { "fixedColor": "semi-dark-green", "mode": "fixed" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 21, "y": 1 }, "id": 361, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(increase(incus_disk_written_bytes_total{job=\"$job\",project=\"$project\"}[$__range]))", "hide": false, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Disk writes", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total network RX traffic during the time interval", "fieldConfig": { "defaults": { "color": { "fixedColor": "semi-dark-blue", "mode": "fixed" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "decbits" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 18, "y": 5 }, "id": 363, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(increase(incus_network_receive_bytes_total{job=\"$job\",project=\"$project\"}[$__range]))*8", "hide": false, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Network RX", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total network TX traffic during the time interval", "fieldConfig": { "defaults": { "color": { "fixedColor": "semi-dark-green", "mode": "fixed" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "decbits" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 21, "y": 5 }, "id": 362, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(increase(incus_network_transmit_bytes_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__range]))*8", "hide": false, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Network TX", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "decimals": 0, "mappings": [], "unit": "bps" }, "overrides": [ { "matcher": { "id": "byName", "options": "Others" }, "properties": [ { "id": "color", "value": { "fixedColor": "#81787be0", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 9 }, "id": 417, "options": { "displayLabels": [], "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false, "values": [ "value" ] }, "pieType": "donut", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "limit": 3, "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "7.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "topk(${top}, sum by (name) (rate(incus_network_transmit_bytes_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))*8)", "instant": true, "interval": "", "legendFormat": "{{name}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(incus_network_transmit_bytes_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval])*8) -\nsum(topk(${top}, sum by (name) (rate(incus_network_transmit_bytes_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))*8))", "hide": false, "instant": true, "interval": "", "legendFormat": "Others", "refId": "B" } ], "title": "Top ${top} network TX traffic usage", "transparent": true, "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "decimals": 0, "mappings": [], "unit": "bps" }, "overrides": [ { "matcher": { "id": "byName", "options": "Others" }, "properties": [ { "id": "color", "value": { "fixedColor": "#81787be0", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 9 }, "id": 468, "options": { "displayLabels": [], "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false, "values": [ "value" ] }, "pieType": "donut", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "limit": 3, "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "7.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "topk(${top}, sum by (name) (rate(incus_network_receive_bytes_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))*8)", "instant": true, "interval": "", "legendFormat": "{{name}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(incus_network_receive_bytes_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval])*8) -\nsum(topk(${top}, sum by (name) (rate(incus_network_receive_bytes_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))*8))", "hide": false, "instant": true, "interval": "", "legendFormat": "Others", "refId": "B" } ], "title": "Top ${top} network RX traffic usage", "transparent": true, "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "decimals": 0, "mappings": [], "unit": "pps" }, "overrides": [ { "matcher": { "id": "byName", "options": "Others" }, "properties": [ { "id": "color", "value": { "fixedColor": "#81787be0", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 9 }, "id": 469, "options": { "displayLabels": [], "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false, "values": [ "value" ] }, "pieType": "donut", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "limit": 3, "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "7.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "topk(${top}, sum by (name) (rate(incus_network_transmit_packets_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))*8)", "instant": true, "interval": "", "legendFormat": "{{name}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(incus_network_transmit_packets_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval])*8) -\nsum(topk(${top}, sum by (name) (rate(incus_network_transmit_packets_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))*8))", "hide": false, "instant": true, "interval": "", "legendFormat": "Others", "refId": "B" } ], "title": "Top ${top} network TX packets/s usage", "transparent": true, "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "decimals": 0, "mappings": [], "unit": "pps" }, "overrides": [ { "matcher": { "id": "byName", "options": "Others" }, "properties": [ { "id": "color", "value": { "fixedColor": "#81787be0", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 9 }, "id": 470, "options": { "displayLabels": [], "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false, "values": [ "value" ] }, "pieType": "donut", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "limit": 3, "values": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "7.4.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "topk(${top}, sum by (name) (rate(incus_network_receive_packets_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))*8)", "instant": true, "interval": "", "legendFormat": "{{name}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(incus_network_receive_packets_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval])*8) -\nsum(topk(${top}, sum by (name) (rate(incus_network_receive_packets_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))*8))", "hide": false, "instant": true, "interval": "", "legendFormat": "Others", "refId": "B" } ], "title": "Top ${top} network RX packets/s usage", "transparent": true, "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "CPU usage overview for the whole project", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percent" }, "overrides": [ { "matcher": { "id": "byName", "options": "system" }, "properties": [ { "id": "color", "value": { "fixedColor": "#508642", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "user" }, "properties": [ { "id": "color", "value": { "fixedColor": "#1F60C4", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 15 }, "id": 328, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(incus_cpu_seconds_total{mode!=\"idle\",job=\"$job\",project=\"$project\"}[$__rate_interval])) by (mode) * 100", "hide": false, "interval": "", "legendFormat": "{{mode}}", "refId": "C" } ], "title": "Project CPU Usage", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Memory usage overview for the whole project", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "SWAP Used" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "RAM Total" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E0F9D7", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 10 }, { "id": "custom.stacking", "value": { "group": false, "mode": "normal" } } ] }, { "matcher": { "id": "byName", "options": "RAM Cache" }, "properties": [ { "id": "color", "value": { "fixedColor": "#052B51", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "RAM Free" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } }, { "id": "custom.gradientMode", "value": "opacity" } ] } ] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 15 }, "id": 354, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(incus_memory_MemTotal_bytes{job=\"$job\",project=\"$project\"})", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "RAM Total", "refId": "A", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(incus_memory_MemTotal_bytes{job=\"$job\",project=\"$project\"} - incus_memory_MemFree_bytes - incus_memory_Cached_bytes)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "RAM Used", "refId": "B", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(incus_memory_Cached_bytes{job=\"$job\",project=\"$project\"})", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "RAM Cache", "refId": "C", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(incus_memory_MemFree_bytes{job=\"$job\",project=\"$project\"})", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "RAM Free", "refId": "D", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(incus_memory_Swap_bytes{job=\"$job\",project=\"$project\"})", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "SWAP Used", "refId": "E", "step": 240 } ], "title": "Project Memory Usage", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of running processes for the whole project", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 25 }, "id": 344, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(incus_procs_total{job=\"$job\",project=\"$project\"})", "interval": "", "legendFormat": "Processes", "refId": "A" } ], "title": "Project Running Processes", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Disk space used for the whole project", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "custom.fillOpacity", "value": 0 }, { "id": "color", "value": { "fixedColor": "semi-dark-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 25 }, "id": 326, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(incus_filesystem_size_bytes{mountpoint=\"/\",job=\"$job\",project=\"$project\"} - incus_filesystem_avail_bytes)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Disk used", "refId": "A", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(incus_filesystem_size_bytes{mountpoint=\"/\",job=\"$job\",project=\"$project\"})", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "Disk quota", "refId": "B", "step": 240 } ], "title": "Project Disk Space Usage (rootfs)", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "The number of bytes read or written per second for the whole project", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "Bps" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "color", "value": { "fixedColor": "#3274D9", "mode": "fixed" } }, { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 34 }, "id": 338, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(rate(incus_disk_written_bytes_total{job=\"$job\",project=\"$project\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Written bytes", "refId": "A", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(rate(incus_disk_read_bytes_total{job=\"$job\",project=\"$project\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Read bytes", "refId": "B", "step": 240 } ], "title": "Project Disk R/W Data", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "The number (after merges) of I/O requests completed per second for the whole project", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "iops" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-blue", "mode": "fixed" } }, { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 34 }, "id": 336, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(rate(incus_disk_writes_completed_total{job=\"$job\",project=\"$project\"}[$__rate_interval]))", "interval": "", "intervalFactor": 1, "legendFormat": "Writes completed", "refId": "A", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(rate(incus_disk_reads_completed_total{job=\"$job\",project=\"$project\"}[$__rate_interval]))", "interval": "", "intervalFactor": 1, "legendFormat": "Reads completed", "refId": "B", "step": 240 } ], "title": "Project Disk IO/s", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Network traffic overview for the whole project", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bps" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "color", "value": { "fixedColor": "#3274D9", "mode": "fixed" } }, { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 43 }, "id": 322, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(rate(incus_network_transmit_bytes_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))*8", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "TX", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(rate(incus_network_receive_bytes_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))*8", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "RX", "refId": "B" } ], "title": "Project Network Traffic", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Network traffic overview for the whole project", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "pps" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-blue", "mode": "fixed" } }, { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 43 }, "id": 365, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(incus_network_transmit_packets_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "TX", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(incus_network_receive_packets_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "RX", "refId": "B" } ], "title": "Project Network Packets/s", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Network errors and drop overview for the whole project", "fieldConfig": { "defaults": { "color": { "fixedColor": "dark-blue", "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "pps" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "A" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] }, { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 49 }, "id": 366, "options": { "legend": { "calcs": [ "max" ], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(rate(incus_network_receive_errs_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "RX errors", "refId": "A", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(rate(incus_network_transmit_errs_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "TX errors", "refId": "B", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(rate(incus_network_receive_drop_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "RX drop", "refId": "C", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(rate(incus_network_transmit_drop_total{device!=\"lo\",job=\"$job\",project=\"$project\"}[$__rate_interval]))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "TX drop", "refId": "D", "step": 240 } ], "title": "Project Network Errors & Drops", "transparent": true, "type": "timeseries" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 54 }, "id": 6, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Basic CPU info", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "percent" }, "overrides": [ { "matcher": { "id": "byName", "options": "system" }, "properties": [ { "id": "color", "value": { "fixedColor": "#508642", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "user" }, "properties": [ { "id": "color", "value": { "fixedColor": "#1F60C4", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 55 }, "id": 353, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(rate(incus_cpu_seconds_total{mode!=\"idle\",job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])) by (mode) * 100", "hide": false, "interval": "", "legendFormat": "{{mode}}", "refId": "C" } ], "title": "CPU Usage", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Basic memory usage", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "SWAP Used" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "RAM Total" }, "properties": [ { "id": "color", "value": { "fixedColor": "#E0F9D7", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 10 }, { "id": "custom.stacking", "value": { "group": false, "mode": "normal" } } ] }, { "matcher": { "id": "byName", "options": "RAM Cache" }, "properties": [ { "id": "color", "value": { "fixedColor": "#052B51", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "RAM Free" }, "properties": [ { "id": "color", "value": { "fixedColor": "#7EB26D", "mode": "fixed" } }, { "id": "custom.gradientMode", "value": "opacity" } ] } ] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 55 }, "id": 324, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(incus_memory_MemTotal_bytes{job=\"$job\",project=\"$project\",name=\"$name\"})", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "RAM Total", "refId": "A", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(incus_memory_MemTotal_bytes{job=\"$job\",project=\"$project\",name=\"$name\"} - incus_memory_MemFree_bytes - incus_memory_Cached_bytes)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "RAM Used", "refId": "B", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(incus_memory_Cached_bytes{job=\"$job\",project=\"$project\",name=\"$name\"})", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "RAM Cache", "refId": "C", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(incus_memory_MemFree_bytes{job=\"$job\",project=\"$project\",name=\"$name\"})", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "RAM Free", "refId": "D", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(incus_memory_Swap_bytes{job=\"$job\",project=\"$project\",name=\"$name\"})", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "SWAP Used", "refId": "E", "step": 240 } ], "title": "Memory Usage", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of running processes in the instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 65 }, "id": 342, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "incus_procs_total{job=\"$job\",project=\"$project\",name=\"$name\"}", "interval": "", "legendFormat": "Processes", "refId": "A" } ], "title": "Running Processes", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Disk space used of all filesystems mounted", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "max": 100, "min": 0, "thresholds": { "mode": "percentage", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 65 }, "id": 14, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "100 - ((incus_filesystem_avail_bytes{job=\"$job\",project=\"$project\",name=\"$name\"} * 100) / incus_filesystem_size_bytes)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{mountpoint}}", "refId": "A", "step": 240 } ], "title": "Disk Space Used", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "The number of bytes read from or written to the device per second", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "Bps" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 75 }, "id": 334, "links": [], "options": { "legend": { "calcs": [ "min", "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "rate(incus_disk_written_bytes_total{job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{device}} - Written bytes", "refId": "A", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "rate(incus_disk_read_bytes_total{job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{device}} - Read bytes", "refId": "B", "step": 240 } ], "title": "Disk R/W Data", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "The number (after merges) of I/O requests completed per second for the device", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "iops" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 75 }, "id": 358, "links": [], "options": { "legend": { "calcs": [ "min", "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(incus_disk_writes_completed_total{job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])", "interval": "", "intervalFactor": 1, "legendFormat": "{{device}} - Writes completed", "refId": "A", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "rate(incus_disk_reads_completed_total{job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])", "interval": "", "intervalFactor": 1, "legendFormat": "{{device}} - Reads completed", "refId": "B", "step": 240 } ], "title": "Disk IO/s", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Basic network info per interface", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "bps" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-blue", "mode": "fixed" } }, { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 85 }, "id": 356, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "rate(incus_network_transmit_bytes_total{device!=\"lo\",job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])*8", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{device}} - TX", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "rate(incus_network_receive_bytes_total{device!=\"lo\",job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])*8", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{device}} - RX", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "rate(incus_network_transmit_bytes_total{device=\"lo\",job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])*8", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "TX {{device}}", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "rate(incus_network_receive_bytes_total{device=\"lo\",job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])*8", "hide": true, "interval": "", "intervalFactor": 1, "legendFormat": "RX {{device}}", "refId": "D" } ], "title": "Network Traffic", "transformations": [], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "pps" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-blue", "mode": "fixed" } }, { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 85 }, "id": 357, "links": [], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "rate(incus_network_transmit_packets_total{device!=\"lo\",job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "TX {{device}}", "refId": "A", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "rate(incus_network_receive_packets_total{device!=\"lo\",job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "RX {{device}}", "refId": "B", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "rate(incus_network_transmit_packets_total{device=\"lo\",job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])", "format": "time_series", "hide": true, "interval": "", "intervalFactor": 1, "legendFormat": "TX {{device}}", "refId": "C", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "rate(incus_network_receive_packets_total{device=\"lo\",job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])", "format": "time_series", "hide": true, "interval": "", "intervalFactor": 1, "legendFormat": "RX {{device}}", "refId": "D", "step": 240 } ], "title": "Network Packets/s", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 40, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 1 } ] }, "unit": "pps" }, "overrides": [] }, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 90 }, "id": 346, "links": [], "options": { "legend": { "calcs": [ "max" ], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "8.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(incus_network_receive_errs_total{device!=\"lo\",job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{device}} - RX errors", "refId": "A", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(incus_network_transmit_errs_total{device!=\"lo\",job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{device}} - TX errors", "refId": "B", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(incus_network_receive_drop_total{device!=\"lo\",job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{device}} - RX drop", "refId": "C", "step": 240 }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(incus_network_transmit_drop_total{device!=\"lo\",job=\"$job\",project=\"$project\",name=\"$name\"}[$__rate_interval])", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{device}} - TX drop", "refId": "D", "step": 240 } ], "title": "Network Errors & Drops", "transparent": true, "type": "timeseries" } ], "repeat": "name", "title": "Instance: $name", "type": "row" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 90 }, "id": 521, "panels": [], "title": "Loki logs", "type": "row" }, { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 91 }, "id": 572, "options": { "dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, "showLabels": false, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": false }, "targets": [ { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "editorMode": "code", "expr": "{app=\"incus\", type=\"lifecycle\", instance=\"$job\", project=~\"|$project\"}", "queryType": "range", "refId": "A" } ], "title": "Lifecycle event logs", "type": "logs" }, { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 98 }, "id": 573, "options": { "dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, "showLabels": false, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": false }, "targets": [ { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "editorMode": "code", "expr": "{app=\"incus\", type=\"logging\", instance=\"$job\", project=~\"|$project\"}", "queryType": "range", "refId": "A" } ], "title": "Event logs", "type": "logs" } ], "refresh": "", "schemaVersion": 39, "tags": [], "templating": { "list": [ { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(incus_procs_total, job)", "hide": 0, "includeAll": false, "label": "Job", "multi": false, "name": "job", "options": [], "query": { "query": "label_values(incus_procs_total, job)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(incus_procs_total{job=\"$job\"}, project)", "description": "Name of the project", "hide": 0, "includeAll": false, "label": "Project", "multi": false, "name": "project", "options": [], "query": { "query": "label_values(incus_procs_total{job=\"$job\"}, project)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": "", "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(incus_procs_total{job=\"$job\",project=\"$project\"}, name)", "description": "Name of the instance", "hide": 0, "includeAll": true, "label": "Instance", "multi": true, "name": "name", "options": [], "query": { "query": "label_values(incus_procs_total{job=\"$job\",project=\"$project\"}, name)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": { "selected": false, "text": "5", "value": "5" }, "description": "Number of instances to include in pie charts", "hide": 0, "includeAll": false, "label": "Top", "multi": false, "name": "top", "options": [ { "selected": false, "text": "3", "value": "3" }, { "selected": false, "text": "4", "value": "4" }, { "selected": true, "text": "5", "value": "5" }, { "selected": false, "text": "10", "value": "10" }, { "selected": false, "text": "15", "value": "15" }, { "selected": false, "text": "20", "value": "20" } ], "query": "3, 4, 5, 10, 15, 20", "queryValue": "", "skipUrlSync": false, "type": "custom" } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": { "refresh_intervals": [ "1m", "5m", "15m", "30m", "1h", "2h", "1d" ] }, "timezone": "", "title": "Incus", "uid": "bGY-LSB7k", "version": 4, "weekStart": "" } incus-7.0.0/internal/000077500000000000000000000000001517523235500144635ustar00rootroot00000000000000incus-7.0.0/internal/eagain/000077500000000000000000000000001517523235500157075ustar00rootroot00000000000000incus-7.0.0/internal/eagain/file_linux.go000066400000000000000000000020161517523235500203730ustar00rootroot00000000000000//go:build linux package eagain import ( "errors" "io" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" ) // Reader represents an io.Reader that handles EAGAIN. type Reader struct { Reader io.Reader } // Read behaves like io.Reader.Read but will retry on EAGAIN. func (er Reader) Read(p []byte) (int, error) { again: n, err := er.Reader.Read(p) if err == nil { return n, nil } // keep retrying on EAGAIN errno, ok := linux.GetErrno(err) if ok && (errors.Is(errno, unix.EAGAIN) || errors.Is(errno, unix.EINTR)) { goto again } return n, err } // Writer represents an io.Writer that handles EAGAIN. type Writer struct { Writer io.Writer } // Write behaves like io.Writer.Write but will retry on EAGAIN. func (ew Writer) Write(p []byte) (int, error) { again: n, err := ew.Writer.Write(p) if err == nil { return n, nil } // keep retrying on EAGAIN errno, ok := linux.GetErrno(err) if ok && (errors.Is(errno, unix.EAGAIN) || errors.Is(errno, unix.EINTR)) { goto again } return n, err } incus-7.0.0/internal/filter/000077500000000000000000000000001517523235500157505ustar00rootroot00000000000000incus-7.0.0/internal/filter/clause.go000066400000000000000000000045561517523235500175650ustar00rootroot00000000000000package filter import ( "errors" "regexp" "slices" "strings" ) // Clause is a single filter clause in a filter string. type Clause struct { PrevLogical string Not bool Field string Operator string Value string } // ClauseSet is a set of clauses. There are configurable functions that can be used to // perform unique parsing of the clauses. type ClauseSet struct { Clauses []Clause Ops OperatorSet ParseInt func(Clause) (int64, error) ParseUint func(Clause) (uint64, error) ParseString func(Clause) (string, error) ParseBool func(Clause) (bool, error) ParseRegexp func(Clause) (*regexp.Regexp, error) ParseStringSlice func(Clause) ([]string, error) } // Parse a user-provided filter string. func Parse(s string, op OperatorSet) (*ClauseSet, error) { if !op.isValid() { return nil, errors.New("Invalid operator set") } clauses := []Clause{} parts := strings.Fields(s) index := 0 prevLogical := op.And for index < len(parts) { clause := Clause{} if strings.EqualFold(parts[index], op.Negate) { clause.Not = true index++ if index == len(parts) { return nil, errors.New("incomplete not clause") } } else { clause.Not = false } clause.Field = parts[index] index++ if index == len(parts) { return nil, errors.New("clause has no operator") } clause.Operator = parts[index] index++ if index == len(parts) { return nil, errors.New("clause has no value") } value := parts[index] // support strings with spaces that are quoted for _, symbol := range op.Quote { if strings.HasPrefix(value, symbol) { value = value[1:] for { index++ if index == len(parts) { return nil, errors.New("unterminated quote") } if strings.HasSuffix(parts[index], symbol) { break } value += " " + parts[index] } end := parts[index] value += " " + end[0:len(end)-1] } } clause.Value = value index++ clause.PrevLogical = prevLogical if index < len(parts) { prevLogical = parts[index] if !slices.Contains([]string{op.And, op.Or}, prevLogical) { return nil, errors.New("invalid clause composition") } index++ if index == len(parts) { return nil, errors.New("unterminated compound clause") } } clauses = append(clauses, clause) } return &ClauseSet{Clauses: clauses, Ops: op}, nil } incus-7.0.0/internal/filter/clause_test.go000066400000000000000000000027601517523235500206170ustar00rootroot00000000000000package filter_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/filter" ) func TestParse_Error(t *testing.T) { cases := map[string]string{ "not": "incomplete not clause", "foo": "clause has no operator", "not foo": "clause has no operator", "foo eq": "clause has no value", "foo eq \"bar": "unterminated quote", "foo eq bar and": "unterminated compound clause", "foo eq \"bar egg\" and": "unterminated compound clause", "foo eq bar xxx": "invalid clause composition", } for s, message := range cases { t.Run(s, func(t *testing.T) { clauses, err := filter.Parse(s, filter.QueryOperatorSet()) assert.Nil(t, clauses) assert.EqualError(t, err, message) }) } } func TestParse(t *testing.T) { clauses, err := filter.Parse("foo eq \"bar egg\" or not baz eq yuk", filter.QueryOperatorSet()) require.NoError(t, err) assert.Len(t, clauses.Clauses, 2) clause1 := clauses.Clauses[0] clause2 := clauses.Clauses[1] assert.False(t, clause1.Not) assert.Equal(t, "and", clause1.PrevLogical) assert.Equal(t, "foo", clause1.Field) assert.Equal(t, "eq", clause1.Operator) assert.Equal(t, "bar egg", clause1.Value) assert.True(t, clause2.Not) assert.Equal(t, "baz", clause2.Field) assert.Equal(t, "or", clause2.PrevLogical) assert.Equal(t, "eq", clause2.Operator) assert.Equal(t, "yuk", clause2.Value) } incus-7.0.0/internal/filter/doc.go000066400000000000000000000000421517523235500170400ustar00rootroot00000000000000// API filtering. package filter incus-7.0.0/internal/filter/match.go000066400000000000000000000156161517523235500174040ustar00rootroot00000000000000package filter import ( "encoding/json" "errors" "fmt" "reflect" "regexp" "strconv" "strings" ) const stringMultiValueDelimiter = "," // Match returns true if the given object matches the given filter. func Match(obj any, set ClauseSet) (bool, error) { if set.ParseInt == nil { set.ParseInt = DefaultParseInt } if set.ParseUint == nil { set.ParseUint = DefaultParseUint } if set.ParseString == nil { set.ParseString = DefaultParseString } if set.ParseBool == nil { set.ParseBool = DefaultParseBool } if set.ParseRegexp == nil { set.ParseRegexp = DefaultParseRegexp } if set.ParseStringSlice == nil { set.ParseStringSlice = DefaultParseStringSlice } match := true for _, clause := range set.Clauses { value := ValueOf(obj, clause.Field) clauseMatch, err := set.match(clause, value) if err != nil { return false, err } // Finish out logic if clause.Not { clauseMatch = !clauseMatch } switch clause.PrevLogical { case set.Ops.And: match = match && clauseMatch case set.Ops.Or: match = match || clauseMatch default: return false, errors.New("unexpected clause operator") } } return match, nil } // DefaultParseInt converts the value of the clause to int64. func DefaultParseInt(c Clause) (int64, error) { return strconv.ParseInt(c.Value, 10, 0) } // DefaultParseUint converts the value of the clause to Uint64. func DefaultParseUint(c Clause) (uint64, error) { return strconv.ParseUint(c.Value, 10, 0) } // DefaultParseString converts the value of the clause to string. func DefaultParseString(c Clause) (string, error) { return c.Value, nil } // DefaultParseBool converts the value of the clause to boolean. func DefaultParseBool(c Clause) (bool, error) { return strconv.ParseBool(c.Value) } // DefaultParseRegexp converts the value of the clause to regexp. func DefaultParseRegexp(c Clause) (*regexp.Regexp, error) { regexpValue := c.Value if !strings.Contains(regexpValue, "^") && !strings.Contains(regexpValue, "$") { regexpValue = "^" + regexpValue + "$" } return regexp.Compile("(?i)" + regexpValue) } // DefaultParseStringSlice converts the value of the clause to a slice of string. func DefaultParseStringSlice(c Clause) ([]string, error) { var val []string err := json.Unmarshal([]byte(c.Value), &val) if err != nil { return nil, err } return val, nil } func (s ClauseSet) match(c Clause, objValue any) (bool, error) { var valueStr string var valueRegexp *regexp.Regexp var valueInt int64 var valueUint uint64 var valueBool bool var valueSlice []string var err error // If 'value' is type of string try to test value as a regexp. valInfo := reflect.ValueOf(objValue) kind := valInfo.Kind() switch kind { case reflect.String: valueStr, err = s.ParseString(c) if !strings.Contains(valueStr, ",") { valueRegexp, _ = s.ParseRegexp(c) if valueRegexp != nil { valueStr = "" } } case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: valueInt, err = s.ParseInt(c) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: valueUint, err = s.ParseUint(c) case reflect.Bool: valueBool, err = s.ParseBool(c) case reflect.Slice: if reflect.TypeOf(objValue).Elem().Kind() != reflect.String { return false, fmt.Errorf("Invalid slice type %q for field %q", reflect.TypeOf(objValue).Elem().Kind(), c.Field) } valueSlice, err = s.ParseStringSlice(c) default: return false, fmt.Errorf("Invalid type %q for field %q", kind.String(), c.Field) } if err != nil { return false, fmt.Errorf("Failed to parse value: %w", err) } switch c.Operator { case s.Ops.Equals: if valueRegexp != nil { return valueRegexp.MatchString(objValue.(string)), nil } switch val := objValue.(type) { case string: // Comparison is case insensitive. for _, curValue := range strings.Split(valueStr, stringMultiValueDelimiter) { if strings.EqualFold(val, curValue) { return true, nil } } return strings.EqualFold(val, valueStr), nil case int, int8, int16, int32, int64: return objValue == valueInt, nil case uint, uint8, uint16, uint32, uint64: return objValue == valueUint, nil case bool: return objValue == valueBool, nil case []string: match := func() bool { if len(objValue.([]string)) != len(valueSlice) { return false } //revive:disable-next-line:unchecked-type-assertion for k, v := range objValue.([]string) { if valueSlice[k] != v { return false } } return true }() return match, nil } case s.Ops.NotEquals: if valueRegexp != nil { return !valueRegexp.MatchString(objValue.(string)), nil } switch val := objValue.(type) { case string: // Comparison is case insensitive. for _, curValue := range strings.Split(valueStr, stringMultiValueDelimiter) { if !strings.EqualFold(val, curValue) { return true, nil } } return !strings.EqualFold(val, valueStr), nil case int, int8, int16, int32, int64: return objValue != valueInt, nil case uint, uint8, uint16, uint32, uint64: return objValue != valueUint, nil case bool: return objValue != valueBool, nil case []string: match := func() bool { if len(objValue.([]string)) != len(valueSlice) { return false } //revive:disable-next-line:unchecked-type-assertion for k, v := range objValue.([]string) { if valueSlice[k] != v { return false } } return true }() return !match, nil } case s.Ops.GreaterThan: switch objValue.(type) { case string, bool, []string: return false, fmt.Errorf("Invalid operator %q for field %q", c.Operator, c.Field) case int, int8, int16, int32, int64: return valInfo.Int() > valueInt, nil case uint, uint8, uint16, uint32, uint64: return valInfo.Uint() > valueUint, nil } case s.Ops.LessThan: switch objValue.(type) { case string, bool, []string: return false, fmt.Errorf("Invalid operator %q for field %q", c.Operator, c.Field) case int, int8, int16, int32, int64: return valInfo.Int() < valueInt, nil case uint, uint8, uint16, uint32, uint64: return valInfo.Uint() < valueUint, nil } case s.Ops.GreaterEqual: switch objValue.(type) { case string, bool, []string: return false, fmt.Errorf("Invalid operator %q for field %q", c.Operator, c.Field) case int, int8, int16, int32, int64: return valInfo.Int() >= valueInt, nil case uint, uint8, uint16, uint32, uint64: return valInfo.Uint() >= valueUint, nil } case s.Ops.LessEqual: switch objValue.(type) { case string, bool, []string: return false, fmt.Errorf("Invalid operator %q for field %q", c.Operator, c.Field) case int, int8, int16, int32, int64: return valInfo.Int() <= valueInt, nil case uint, uint8, uint16, uint32, uint64: return valInfo.Uint() <= valueUint, nil } default: return false, errors.New("Unsupported operation") } return false, fmt.Errorf("Unsupported filter type %q for field %q", kind.String(), c.Field) } incus-7.0.0/internal/filter/match_test.go000066400000000000000000000041631517523235500204360ustar00rootroot00000000000000package filter_test import ( "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/shared/api" ) func TestMatch_Instance(t *testing.T) { instance := api.Instance{ InstancePut: api.InstancePut{ Architecture: "x86_64", Config: map[string]string{ "image.os": "BusyBox", }, Stateful: false, }, CreatedAt: time.Date(2020, 1, 29, 11, 10, 32, 0, time.UTC), Name: "c1", ExpandedConfig: map[string]string{ "image.os": "BusyBox", }, ExpandedDevices: map[string]map[string]string{ "root": { "path": "/", "pool": "default", "type": "disk", }, }, Status: "Running", } cases := map[string]any{ "architecture eq x86_64": true, "architecture eq i686": false, "name eq c1 and status eq Running": true, "config.image.os eq BusyBox and expanded_devices.root.path eq /": true, "name eq c2 or status eq Running": true, "name eq c2 or name eq c3": false, "status eq Running,Stopped": true, "name eq c2,c3": false, } for s := range cases { t.Run(s, func(t *testing.T) { f, err := filter.Parse(s, filter.QueryOperatorSet()) require.NoError(t, err) match, err := filter.Match(instance, *f) require.NoError(t, err) assert.Equal(t, cases[s], match) }) } } func TestMatch_Image(t *testing.T) { image := api.Image{ ImagePut: api.ImagePut{ Public: true, Properties: map[string]string{ "os": "Ubuntu", }, }, Architecture: "i686", } cases := map[string]any{ "properties.os eq Ubuntu": true, "architecture eq x86_64": false, } for s := range cases { t.Run(s, func(t *testing.T) { f, err := filter.Parse(s, filter.QueryOperatorSet()) require.NoError(t, err) match, err := filter.Match(image, *f) require.NoError(t, err) assert.Equal(t, cases[s], match) }) } } incus-7.0.0/internal/filter/operator.go000066400000000000000000000015011517523235500201270ustar00rootroot00000000000000package filter // OperatorSet is represents the types of operators and symbols that a filter can support. type OperatorSet struct { And string Or string Equals string NotEquals string GreaterThan string LessThan string GreaterEqual string LessEqual string Negate string Quote []string } // isValid ensures the OperatorSet has valid fields for the minimum supported operators. func (o *OperatorSet) isValid() bool { return o.And != "" && o.Or != "" && o.Equals != "" && o.NotEquals != "" && o.Negate != "" && len(o.Quote) > 0 } // QueryOperatorSet returns the default operator set for REST API queries. func QueryOperatorSet() OperatorSet { return OperatorSet{ And: "and", Or: "or", Equals: "eq", NotEquals: "ne", Negate: "not", Quote: []string{"\""}, } } incus-7.0.0/internal/filter/value.go000066400000000000000000000033321517523235500174140ustar00rootroot00000000000000package filter import ( "reflect" "strings" "github.com/lxc/incus/v7/shared/api" ) // DotPrefixMatch finds the shortest unambiguous identifier for a given namespace. func DotPrefixMatch(short string, full string) bool { fullMembs := strings.Split(full, ".") shortMembs := strings.Split(short, ".") if len(fullMembs) != len(shortMembs) { return false } for i := range fullMembs { if !strings.HasPrefix(fullMembs[i], shortMembs[i]) { return false } } return true } // ValueOf returns the value of the given field. func ValueOf(obj any, field string) any { value := reflect.ValueOf(obj) typ := value.Type() parts := strings.Split(field, ".") key := parts[0] rest := strings.Join(parts[1:], ".") if value.Kind() == reflect.Map { switch reflect.TypeOf(obj).Elem().Kind() { case reflect.String: m := map[string]string{} switch mm := value.Interface().(type) { case map[string]string: m = mm case api.ConfigMap: m = mm } for k, v := range m { if DotPrefixMatch(field, k) { return v } } return m[field] case reflect.Map: for _, entry := range value.MapKeys() { if entry.Interface() != key { continue } m := value.MapIndex(entry) return ValueOf(m.Interface(), rest) } return nil default: return nil } } for i := range value.NumField() { fieldValue := value.Field(i) fieldType := typ.Field(i) yaml := fieldType.Tag.Get("yaml") if yaml == ",inline" { v := ValueOf(fieldValue.Interface(), field) if v != nil { return v } } yamlKey, _, _ := strings.Cut(yaml, ",") if yamlKey == key { v := fieldValue.Interface() if len(parts) == 1 { return v } return ValueOf(v, rest) } } return nil } incus-7.0.0/internal/filter/value_test.go000066400000000000000000000026651517523235500204630ustar00rootroot00000000000000package filter_test import ( "testing" "time" "github.com/stretchr/testify/assert" "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/shared/api" ) func TestDotPrefixMatch(t *testing.T) { pass := true pass = pass && filter.DotPrefixMatch("s.privileged", "security.privileged") pass = pass && filter.DotPrefixMatch("u.blah", "user.blah") if !pass { t.Error("failed prefix matching") } } func TestValueOf_Instance(t *testing.T) { date := time.Date(2020, 1, 29, 11, 10, 32, 0, time.UTC) instance := api.Instance{ InstancePut: api.InstancePut{ Architecture: "x86_64", Config: map[string]string{ "image.os": "BusyBox", }, Stateful: false, }, CreatedAt: date, Name: "c1", ExpandedConfig: map[string]string{ "image.os": "BusyBox", }, ExpandedDevices: map[string]map[string]string{ "root": { "path": "/", "pool": "default", "type": "disk", }, }, Status: "Running", } cases := map[string]any{} cases["architecture"] = "x86_64" cases["created_at"] = date cases["config.image.os"] = "BusyBox" cases["name"] = "c1" cases["expanded_config.image.os"] = "BusyBox" cases["expanded_config.im.os"] = "BusyBox" cases["expanded_devices.root.pool"] = "default" cases["status"] = "Running" cases["stateful"] = false for field := range cases { t.Run(field, func(t *testing.T) { value := filter.ValueOf(instance, field) assert.Equal(t, cases[field], value) }) } } incus-7.0.0/internal/i18n/000077500000000000000000000000001517523235500152425ustar00rootroot00000000000000incus-7.0.0/internal/i18n/i18n.go000066400000000000000000000001721517523235500163500ustar00rootroot00000000000000//go:build !linux || !cgo package i18n // G returns the translated string func G(msgid string) string { return msgid } incus-7.0.0/internal/i18n/i18n_linux.go000066400000000000000000000003651517523235500175730ustar00rootroot00000000000000//go:build linux && cgo package i18n import ( "github.com/gosexy/gettext" ) // G returns the translated string. func G(msgid string) string { return gettext.DGettext("incus", msgid) } func init() { gettext.SetLocale(gettext.LC_ALL, "") } incus-7.0.0/internal/incusos/000077500000000000000000000000001517523235500161465ustar00rootroot00000000000000incus-7.0.0/internal/incusos/api_services.go000066400000000000000000000010441517523235500211500ustar00rootroot00000000000000package incusos import ( "net/http" ) // IsServiceEnabled checks if the provided service is currently enabled. func (c *Client) IsServiceEnabled(name string) (bool, error) { // Get the data. resp, err := c.query(http.MethodGet, "/services/"+name) if err != nil { return false, err } // Parse the response. type srv struct { Config struct { Enabled bool `json:"enabled"` } `json:"config"` } service := &srv{} err = resp.MetadataAsStruct(service) if err != nil { return false, err } return service.Config.Enabled, nil } incus-7.0.0/internal/incusos/api_system.go000066400000000000000000000015341517523235500206550ustar00rootroot00000000000000package incusos import ( "errors" "net/http" osapi "github.com/lxc/incus-os/incus-osd/api" ) // GetSystemNetwork returns the IncusOS network configuration and state. func (c *Client) GetSystemNetwork() (*osapi.SystemNetwork, error) { // Get the data. resp, err := c.query(http.MethodGet, "/system/network") if err != nil { return nil, err } // Parse the response. ns := &osapi.SystemNetwork{} err = resp.MetadataAsStruct(ns) if err != nil { return nil, err } return ns, nil } // TriggerSystemUpdateCheck asks IncusOS to check for and apply any pending update. func (c *Client) TriggerSystemUpdateCheck() error { // Get the data. resp, err := c.query(http.MethodPost, "/system/update/:check") if err != nil { return err } if resp.StatusCode != http.StatusOK { return errors.New("Failed to check for updates") } return nil } incus-7.0.0/internal/incusos/client.go000066400000000000000000000022601517523235500177530ustar00rootroot00000000000000package incusos import ( "context" "encoding/json" "errors" "net" "net/http" "github.com/lxc/incus/v7/shared/api" ) // Client represents an IncusOS API client. type Client struct { http *http.Client } // NewClient instantiates a new IncusOS API client. func NewClient() (*Client, error) { c := &Client{} c.http = &http.Client{ Transport: &http.Transport{ DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { return net.Dial("unix", "/run/incus-os/unix.socket") }, }, } return c, nil } func (c *Client) query(method string, path string) (*api.Response, error) { // Prepare the request. req, err := http.NewRequest(method, "http://incus-os/1.0"+path, nil) if err != nil { return nil, err } // Query the OS network state. resp, err := c.http.Do(req) if err != nil { return nil, err } defer resp.Body.Close() // Convert to an Incus response struct. apiResp := &api.Response{} err = json.NewDecoder(resp.Body).Decode(apiResp) if err != nil { return nil, err } // Quick validation. if apiResp.Type != "sync" || apiResp.StatusCode != http.StatusOK { return nil, errors.New("Bad response from IncusOS") } return apiResp, nil } incus-7.0.0/internal/instance/000077500000000000000000000000001517523235500162675ustar00rootroot00000000000000incus-7.0.0/internal/instance/action.go000066400000000000000000000005051517523235500200730ustar00rootroot00000000000000package instance // InstanceAction indicates the type of action being performed. type InstanceAction string // InstanceAction types. const ( Stop InstanceAction = "stop" Start InstanceAction = "start" Restart InstanceAction = "restart" Freeze InstanceAction = "freeze" Unfreeze InstanceAction = "unfreeze" ) incus-7.0.0/internal/instance/config.go000066400000000000000000001710661517523235500200760ustar00rootroot00000000000000package instance import ( "errors" "fmt" "strconv" "strings" "time" scriptletLoad "github.com/lxc/incus/v7/internal/server/scriptlet/load" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/validate" ) // IsUserConfig returns true if the config key is a user configuration. func IsUserConfig(key string) bool { return strings.HasPrefix(key, "user.") } // ConfigVolatilePrefix indicates the prefix used for volatile config keys. const ConfigVolatilePrefix = "volatile." // HugePageSizeKeys is a list of known hugepage size configuration keys. var HugePageSizeKeys = [...]string{"limits.hugepages.64KB", "limits.hugepages.1MB", "limits.hugepages.2MB", "limits.hugepages.1GB"} // HugePageSizeSuffix contains the list of known hugepage size suffixes. var HugePageSizeSuffix = [...]string{"64KB", "1MB", "2MB", "1GB"} // InstanceConfigKeysAny is a map of config key to validator. (keys applying to containers AND virtual machines). var InstanceConfigKeysAny = map[string]func(value string) error{ // gendoc:generate(entity=instance, group=boot, key=boot.autorestart) // If set to `true` will attempt up to 10 restarts over a 1 minute period upon unexpected instance exit. // --- // type: bool // liveupdate: no // shortdesc: Whether to automatically restart an instance on unexpected exit "boot.autorestart": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=boot, key=boot.autostart) // If unset or set to `last-state`, restores the last state. // --- // type: bool // liveupdate: no // shortdesc: Whether to always start the instance when the daemon starts "boot.autostart": validate.Optional(validate.Or(validate.IsBool, validate.IsOneOf("last-state"))), // gendoc:generate(entity=instance, group=boot, key=boot.autostart.delay) // The number of seconds to wait after the instance started before starting the next one. // --- // type: integer // defaultdesc: 0 // liveupdate: no // shortdesc: Delay after starting the instance "boot.autostart.delay": validate.Optional(validate.IsInt64), // gendoc:generate(entity=instance, group=boot, key=boot.autostart.priority) // The instance with the highest value is started first. // Instances without a priority set will be started (with some parallelism) ahead of // instances with a priority set. // --- // type: integer // liveupdate: no // shortdesc: What order to start the instances in "boot.autostart.priority": validate.Optional(validate.IsInt64), // gendoc:generate(entity=instance, group=boot, key=boot.stop.priority) // The instance with the highest value is shut down first. // --- // type: integer // defaultdesc: 0 // liveupdate: no // shortdesc: What order to shut down the instances in "boot.stop.priority": validate.Optional(validate.IsInt64), // gendoc:generate(entity=instance, group=boot, key=boot.host_shutdown_action) // Action to take on host shut down // // Valid values are: `stop`, `force-stop` or `stateful-stop` // --- // type: string // defaultdesc: stop // liveupdate: yes // shortdesc: What action to take on the instance when the host is shut down "boot.host_shutdown_action": validate.Optional(validate.IsOneOf("stop", "force-stop", "stateful-stop")), // gendoc:generate(entity=instance, group=boot, key=boot.host_shutdown_timeout) // Number of seconds to wait for the instance to shut down before it is force-stopped. // --- // type: integer // defaultdesc: 30 // liveupdate: yes // shortdesc: How long to wait for the instance to shut down "boot.host_shutdown_timeout": validate.Optional(validate.IsInt64), // gendoc:generate(entity=instance, group=cloud-init, key=cloud-init.network-config) // The content is used as seed value for `cloud-init`. // --- // type: string // defaultdesc: `DHCP on eth0` // liveupdate: no // condition: If supported by image // shortdesc: Network configuration for `cloud-init` "cloud-init.network-config": validate.Optional(validate.IsYAML), // gendoc:generate(entity=instance, group=cloud-init, key=cloud-init.user-data) // The content is used as seed value for `cloud-init`. // --- // type: string // defaultdesc: `#cloud-config` // liveupdate: no // condition: If supported by image // shortdesc: User data for `cloud-init` "cloud-init.user-data": validate.Optional(validate.IsCloudInitUserData), // gendoc:generate(entity=instance, group=cloud-init, key=cloud-init.vendor-data) // The content is used as seed value for `cloud-init`. // --- // type: string // defaultdesc: `#cloud-config` // liveupdate: no // condition: If supported by image // shortdesc: Vendor data for `cloud-init` "cloud-init.vendor-data": validate.Optional(validate.IsCloudInitUserData), // gendoc:generate(entity=instance, group=cloud-init, key=user.network-config) // // --- // type: string // defaultdesc: `DHCP on eth0` // liveupdate: no // condition: If supported by image // shortdesc: Legacy version of `cloud-init.network-config` // gendoc:generate(entity=instance, group=cloud-init, key=user.user-data) // // --- // type: string // defaultdesc: `#cloud-config` // liveupdate: no // condition: If supported by image // shortdesc: Legacy version of `cloud-init.user-data` // gendoc:generate(entity=instance, group=cloud-init, key=user.vendor-data) // // --- // type: string // defaultdesc: `#cloud-config` // liveupdate: no // condition: If supported by image // shortdesc: Legacy version of `cloud-init.vendor-data` // gendoc:generate(entity=instance, group=miscellaneous, key=cluster.evacuate) // The `cluster.evacuate` provides control over how instances are handled when a cluster member is being // evacuated. // // Available Modes: // - `auto` *(default)*: The system will automatically decide the best evacuation method based on the // instance's type and configured devices: // + If any device is not suitable for migration, the instance will not be migrated (only stopped). // + Live migration will be used only for virtual machines with the `migration.stateful` setting // enabled and for which all its devices can be migrated as well. // - `live-migrate`: Instances are live-migrated to another server. This means the instance remains running // and operational during the migration process, ensuring minimal disruption. // - `migrate`: In this mode, instances are migrated to another server in the cluster. The migration // process will not be live, meaning there will be a brief downtime for the instance during the // migration. // - `stop`: Instances are not migrated. Instead, they are stopped on the current server. // - `stateful-stop`: Instances are not migrated. Instead, they are stopped on the current server // but with their runtime state (memory) stored on disk for resuming on restore. // - `force-stop`: Instances are not migrated. Instead, they are forcefully stopped. // // See {ref}`cluster-evacuate` for more information. // --- // type: string // defaultdesc: `auto` // liveupdate: no // shortdesc: What to do when evacuating the instance "cluster.evacuate": validate.Optional(validate.IsOneOf("auto", "migrate", "live-migrate", "stop", "stateful-stop", "force-stop")), // gendoc:generate(entity=instance, group=resource-limits, key=limits.cpu) // A number or a specific range of CPUs to expose to the instance. // // See {ref}`instance-options-limits-cpu` for more information. // --- // type: string // defaultdesc: 1 (VMs) // liveupdate: yes // shortdesc: Which CPUs to expose to the instance "limits.cpu": validate.Optional(validate.IsValidCPUSet), // gendoc:generate(entity=instance, group=resource-limits, key=limits.cpu.nodes) // A comma-separated list of NUMA node IDs or ranges to place the instance CPUs on. // Alternatively, the value `balanced` may be used to have Incus pick the least busy NUMA node on startup. // // See {ref}`instance-options-limits-cpu-container` for more information. // --- // type: string // liveupdate: yes // shortdesc: Which NUMA nodes to place the instance CPUs on "limits.cpu.nodes": validate.Optional(validate.Or(validate.IsValidCPUSet, validate.IsOneOf("0", "balanced"))), // gendoc:generate(entity=instance, group=resource-limits, key=limits.disk.priority) // Controls how much priority to give to the instance's I/O requests when under load. // // Specify an integer between 0 and 10. // --- // type: integer // defaultdesc: `5` (medium) // liveupdate: yes // shortdesc: Priority of the instance's I/O requests "limits.disk.priority": validate.Optional(validate.IsPriority), // gendoc:generate(entity=instance, group=resource-limits, key=limits.memory) // Percentage of the host's memory or a fixed value in bytes. // Various suffixes are supported. // // See {ref}`instances-limit-units` for details. // --- // type: string // defaultdesc: `1GiB` (VMs) // liveupdate: yes // shortdesc: Usage limit for the host's memory "limits.memory": func(value string) error { if value == "" { return nil } if strings.HasSuffix(value, "%") { num, err := strconv.ParseInt(strings.TrimSuffix(value, "%"), 10, 64) if err != nil { return err } if num == 0 { return errors.New("Memory limit can't be 0%") } return nil } num, err := units.ParseByteSizeString(value) if err != nil { return err } if num == 0 { return errors.New("Memory limit can't be 0") } return nil }, // gendoc:generate(entity=instance, group=resource-limits, key=limits.memory.oom_priority) // Specify an integer between -1000 and 1000. // A negative value makes the instance less likely to be killed by the Out Of Memory killer, // while a positive value makes it more likely to be killed. // The default value of 0 means no adjustment to the Out Of Memory score. // --- // type: integer // defaultdesc: `0` // liveupdate: yes // shortdesc: Out Of Memory killer priority adjustment for the instance "limits.memory.oom_priority": validate.Optional(validate.IsOOMPriority), // gendoc:generate(entity=instance, group=migration, key=migration.stateful) // Enabling this option prevents the use of some features that are incompatible with it. // --- // type: bool // defaultdesc: `false` // liveupdate: no // shortdesc: Whether to allow for stateful stop/start and snapshots "migration.stateful": validate.Optional(validate.IsBool), // Caller is responsible for full validation of any raw.* value. // gendoc:generate(entity=instance, group=raw, key=raw.apparmor) // The specified entries are appended to the generated profile. // --- // type: blob // liveupdate: yes // shortdesc: AppArmor profile entries "raw.apparmor": validate.IsAny, // gendoc:generate(entity=instance, group=raw, key=raw.idmap) // For example: `both 1000 1000` // --- // type: blob // liveupdate: no // condition: unprivileged container // shortdesc: Raw idmap configuration "raw.idmap": validate.IsAny, // gendoc:generate(entity=instance, group=security, key=security.guestapi) // See {ref}`dev-incus` for more information. // --- // type: bool // defaultdesc: `true` // liveupdate: no // shortdesc: Whether `/dev/incus` is present in the instance "security.guestapi": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.protection.delete) // // --- // type: bool // defaultdesc: `false` // liveupdate: yes // shortdesc: Prevents the instance from being deleted "security.protection.delete": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=snapshots, key=snapshots.schedule) // Specify either a cron expression (` `), a comma-and-space-separated list of schedule aliases (`@startup`, `@hourly`, `@daily`, `@midnight`, `@weekly`, `@monthly`, `@annually`, `@yearly`), or leave empty to disable automatic snapshots. // // Note that unlike most other configuration keys, this one must be comma-and-space-separated and not just comma-separated as cron expression can themselves contain commas. // // --- // type: string // defaultdesc: empty // liveupdate: no // shortdesc: Schedule for automatic instance snapshots "snapshots.schedule": validate.Optional(validate.IsCron([]string{"@hourly", "@daily", "@midnight", "@weekly", "@monthly", "@annually", "@yearly", "@startup", "@never"})), // gendoc:generate(entity=instance, group=snapshots, key=snapshots.schedule.stopped) // // --- // type: bool // defaultdesc: `false` // liveupdate: no // shortdesc: Whether to automatically snapshot stopped instances "snapshots.schedule.stopped": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=snapshots, key=snapshots.pattern) // Specify a Pongo2 template string that represents the snapshot name. // This template is used for scheduled snapshots and for unnamed snapshots. // // See {ref}`instance-options-snapshots-names` for more information. // --- // type: string // defaultdesc: `snap%d` // liveupdate: no // shortdesc: Template for the snapshot name "snapshots.pattern": validate.IsAny, // gendoc:generate(entity=instance, group=snapshots, key=snapshots.expiry) // Specify an expression like `1M 2H 3d 4w 5m 6y`. // --- // type: string // liveupdate: no // shortdesc: When snapshots are to be deleted "snapshots.expiry": func(value string) error { // Validate expression _, err := GetExpiry(time.Time{}, value) return err }, // gendoc:generate(entity=instance, group=snapshots, key=snapshots.expiry.manual) // Specify an expression like `1M 2H 3d 4w 5m 6y`. // --- // type: string // liveupdate: no // shortdesc: When snapshots are to be deleted (for those not created through scheduling) "snapshots.expiry.manual": func(value string) error { // Validate expression _, err := GetExpiry(time.Time{}, value) return err }, // Volatile keys. // gendoc:generate(entity=instance, group=volatile, key=volatile.apply_template) // The template with the given name is triggered upon next startup. // --- // type: string // shortdesc: Template hook "volatile.apply_template": validate.IsAny, // gendoc:generate(entity=instance, group=volatile, key=volatile.base_image) // The hash of the image that the instance was created from (empty if the instance was not created from an image). // --- // type: string // shortdesc: Hash of the base image "volatile.base_image": validate.IsAny, // gendoc:generate(entity=instance, group=volatile, key=volatile.cloud_init.instance-id) // // --- // type: string // shortdesc: `instance-id` (UUID) exposed to `cloud-init` "volatile.cloud-init.instance-id": validate.Optional(validate.IsUUID), // gendoc:generate(entity=instance, group=volatile, key=volatile.cluster.group) // The cluster group(s) that the instance was restricted to at creation time. // This is used during re-scheduling events like an evacuation to keep the instance within the requested set. // --- // type: string // shortdesc: The original cluster group for the instance "volatile.cluster.group": validate.IsAny, // gendoc:generate(entity=instance, group=volatile, key=volatile.cpu.nodes) // The NUMA node that was selected for the instance. // --- // type: string // shortdesc: Instance NUMA node "volatile.cpu.nodes": validate.Optional(validate.Or(validate.IsValidCPUSet, validate.IsOneOf("0", "balanced"))), // gendoc:generate(entity=instance, group=volatile, key=volatile.evacuate.origin) // The cluster member that the instance lived on before evacuation. // --- // type: string // shortdesc: The origin of the evacuated instance "volatile.evacuate.origin": validate.IsAny, // gendoc:generate(entity=instance, group=volatile, key=volatile.last_state.power) // // --- // type: string // shortdesc: Instance state as of last host shutdown "volatile.last_state.power": validate.IsAny, // gendoc:generate(entity=instance, group=volatile, key=volatile.last_state.ready) // // --- // type: string // shortdesc: Instance marked itself as ready "volatile.last_state.ready": validate.IsBool, // gendoc:generate(entity=instance, group=volatile, key=volatile.rebalance.last_move) // // --- // type: integer // shortdesc: Timestamp of last move by automatic live-migration "volatile.rebalance.last_move": validate.Optional(validate.IsInt64), // gendoc:generate(entity=instance, group=volatile, key=volatile.uuid) // The instance UUID is globally unique across all servers and projects. // --- // type: string // shortdesc: Instance UUID "volatile.uuid": validate.Optional(validate.IsUUID), // gendoc:generate(entity=instance, group=volatile, key=volatile.uuid.generation) // The instance generation UUID changes whenever the instance's place in time moves backwards. // It is globally unique across all servers and projects. // --- // type: string // shortdesc: Instance generation UUID "volatile.uuid.generation": validate.Optional(validate.IsUUID), } // InstanceConfigKeysContainer is a map of config key to validator. (keys applying to containers only). var InstanceConfigKeysContainer = map[string]func(value string) error{ // gendoc:generate(entity=instance, group=resource-limits, key=limits.cpu.allowance) // To control how much of the CPU can be used, specify either a percentage (`50%`) for a soft limit // or a chunk of time (`25ms/100ms`) for a hard limit. // // See {ref}`instance-options-limits-cpu-container` for more information. // --- // type: string // defaultdesc: 100% // liveupdate: yes // condition: container // shortdesc: How much of the CPU can be used "limits.cpu.allowance": func(value string) error { if value == "" { return nil } if strings.HasSuffix(value, "%") { // Percentage based allocation _, err := strconv.Atoi(strings.TrimSuffix(value, "%")) if err != nil { return err } return nil } // Time based allocation fields := strings.SplitN(value, "/", 2) if len(fields) != 2 { return fmt.Errorf("Invalid allowance: %s", value) } _, err := strconv.Atoi(strings.TrimSuffix(fields[0], "ms")) if err != nil { return err } _, err = strconv.Atoi(strings.TrimSuffix(fields[1], "ms")) if err != nil { return err } return nil }, // gendoc:generate(entity=instance, group=resource-limits, key=limits.cpu.priority) // When overcommitting resources, specify the CPU scheduling priority compared to other instances that share the same CPUs. // Specify an integer between 0 and 10. // // See {ref}`instance-options-limits-cpu-container` for more information. // --- // type: integer // defaultdesc: `10` (maximum) // liveupdate: yes // condition: container // shortdesc: CPU scheduling priority compared to other instances "limits.cpu.priority": validate.Optional(validate.IsPriority), // gendoc:generate(entity=instance, group=resource-limits, key=limits.hugepages.64KB) // Fixed value (in bytes) to limit the number of 64 KB huge pages. // Various suffixes are supported (see {ref}`instances-limit-units`). // // See {ref}`instance-options-limits-hugepages` for more information. // --- // type: string // liveupdate: yes // condition: container // shortdesc: Limit for the number of 64 KB huge pages "limits.hugepages.64KB": validate.Optional(validate.IsSize), // gendoc:generate(entity=instance, group=resource-limits, key=limits.hugepages.1MB) // Fixed value (in bytes) to limit the number of 1 MB huge pages. // Various suffixes are supported (see {ref}`instances-limit-units`). // // See {ref}`instance-options-limits-hugepages` for more information. // --- // type: string // liveupdate: yes // condition: container // shortdesc: Limit for the number of 1 MB huge pages "limits.hugepages.1MB": validate.Optional(validate.IsSize), // gendoc:generate(entity=instance, group=resource-limits, key=limits.hugepages.2MB) // Fixed value (in bytes) to limit the number of 2 MB huge pages. // Various suffixes are supported (see {ref}`instances-limit-units`). // // See {ref}`instance-options-limits-hugepages` for more information. // --- // type: string // liveupdate: yes // condition: container // shortdesc: Limit for the number of 2 MB huge pages "limits.hugepages.2MB": validate.Optional(validate.IsSize), // gendoc:generate(entity=instance, group=resource-limits, key=limits.hugepages.1GB) // Fixed value (in bytes) to limit the number of 1 GB huge pages. // Various suffixes are supported (see {ref}`instances-limit-units`). // // See {ref}`instance-options-limits-hugepages` for more information. // --- // type: string // liveupdate: yes // condition: container // shortdesc: Limit for the number of 1 GB huge pages "limits.hugepages.1GB": validate.Optional(validate.IsSize), // gendoc:generate(entity=instance, group=resource-limits, key=limits.memory.enforce) // If the instance's memory limit is `hard`, the instance cannot exceed its limit. // If it is `soft`, the instance can exceed its memory limit when extra host memory is available. // --- // type: string // defaultdesc: `hard` // liveupdate: yes // condition: container // shortdesc: Whether the memory limit is `hard` or `soft` "limits.memory.enforce": validate.Optional(validate.IsOneOf("soft", "hard")), // gendoc:generate(entity=instance, group=resource-limits, key=limits.memory.swap) // When set to `true` or `false`, it controls whether the container is likely to get some of // its memory swapped by the kernel. Alternatively, it can be set to a bytes value which will // then allow the container to make use of additional memory through swap. // --- // type: string // defaultdesc: `true` // liveupdate: yes // condition: container // shortdesc: Control swap usage by the instance "limits.memory.swap": validate.Optional(validate.Or(validate.IsBool, validate.IsSize)), // gendoc:generate(entity=instance, group=resource-limits, key=limits.memory.swap.priority) // Specify an integer between 0 and 10. // The higher the value, the less likely the instance is to be swapped to disk. // --- // type: integer // defaultdesc: `10` (maximum) // liveupdate: yes // condition: container // shortdesc: Prevents the instance from being swapped to disk "limits.memory.swap.priority": validate.Optional(validate.IsPriority), // gendoc:generate(entity=instance, group=resource-limits, key=limits.processes) // If left empty, no limit is set. // --- // type: integer // defaultdesc: empty // liveupdate: yes // condition: container // shortdesc: Maximum number of processes that can run in the instance "limits.processes": validate.Optional(validate.IsInt64), // gendoc:generate(entity=instance, group=miscellaneous, key=linux.kernel_modules) // Specify the kernel modules as a comma-separated list. // --- // type: string // liveupdate: yes // condition: container // shortdesc: Kernel modules to load before starting the instance "linux.kernel_modules": validate.IsAny, // gendoc:generate(entity=instance, group=migration, key=migration.incremental.memory) // Using incremental memory transfer of the instance's memory can reduce downtime. // --- // type: bool // defaultdesc: `false` // liveupdate: yes // condition: container // shortdesc: Whether to use incremental memory transfer "migration.incremental.memory": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=migration, key=migration.incremental.memory.iterations) // // --- // type: integer // defaultdesc: `10` // liveupdate: yes // condition: container // shortdesc: Maximum number of transfer operations to go through before stopping the instance "migration.incremental.memory.iterations": validate.Optional(validate.IsUint32), // gendoc:generate(entity=instance, group=migration, key=migration.incremental.memory.goal) // // --- // type: integer // defaultdesc: `70` // liveupdate: yes // condition: container // shortdesc: Percentage of memory to have in sync before stopping the instance "migration.incremental.memory.goal": validate.Optional(validate.IsUint32), // gendoc:generate(entity=instance, group=nvidia, key=nvidia.runtime) // // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: container // shortdesc: Whether to pass the host NVIDIA and CUDA runtime libraries into the instance "nvidia.runtime": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=nvidia, key=nvidia.driver.capabilities) // The specified driver capabilities are used to set `libnvidia-container NVIDIA_DRIVER_CAPABILITIES`. // --- // type: string // defaultdesc: `compute,utility` // liveupdate: no // condition: container // shortdesc: What driver capabilities the instance needs "nvidia.driver.capabilities": validate.IsAny, // gendoc:generate(entity=instance, group=nvidia, key=nvidia.require.cuda) // The specified version expression is used to set `libnvidia-container NVIDIA_REQUIRE_CUDA`. // --- // type: string // liveupdate: no // condition: container // shortdesc: Required CUDA version "nvidia.require.cuda": validate.IsAny, // gendoc:generate(entity=instance, group=nvidia, key=nvidia.require.driver) // The specified version expression is used to set `libnvidia-container NVIDIA_REQUIRE_DRIVER`. // --- // type: string // liveupdate: no // condition: container // shortdesc: Required driver version "nvidia.require.driver": validate.IsAny, // gendoc:generate(entity=instance, group=oci, key=oci.entrypoint) // Override the entry point of an OCI container. // --- // type: string // liveupdate: no // condition: OCI container // shortdesc: OCI container entry point "oci.entrypoint": validate.IsAny, // gendoc:generate(entity=instance, group=oci, key=oci.cwd) // Override the working directory of an OCI container. // --- // type: string // liveupdate: no // condition: OCI container // shortdesc: OCI container working directory "oci.cwd": validate.Optional(validate.IsAbsFilePath), // gendoc:generate(entity=instance, group=oci, key=oci.gid) // Override the GID of the process run in an OCI container. // --- // type: string // liveupdate: no // condition: OCI container // shortdesc: OCI container GID "oci.gid": validate.Optional(validate.IsUint32), // gendoc:generate(entity=instance, group=oci, key=oci.uid) // Override the UID of the process run in an OCI container. // --- // type: string // liveupdate: no // condition: OCI container // shortdesc: OCI container UID "oci.uid": validate.Optional(validate.IsUint32), // Caller is responsible for full validation of any raw.* value. // gendoc:generate(entity=instance, group=raw, key=raw.lxc) // // --- // type: blob // liveupdate: no // condition: container // shortdesc: Raw LXC configuration to be appended to the generated one "raw.lxc": validate.IsAny, // gendoc:generate(entity=instance, group=raw, key=raw.seccomp) // // --- // type: blob // liveupdate: no // condition: container // shortdesc: Raw Seccomp configuration "raw.seccomp": validate.IsAny, // gendoc:generate(entity=instance, group=security, key=security.bpffs.delegate_cmds) // See {ref}`bpf-tokens` for more information. // // --- // type: string // liveupdate: no // condition: unprivileged container // shortdesc: What BPF command types to delegate "security.bpffs.delegate_cmds": validate.Optional(validate.IsListOf(validate.IsAny)), // gendoc:generate(entity=instance, group=security, key=security.bpffs.delegate_maps) // See {ref}`bpf-tokens` for more information. // // --- // type: string // liveupdate: no // condition: unprivileged container // shortdesc: What BPF map types to delegate "security.bpffs.delegate_maps": validate.Optional(validate.IsListOf(validate.IsAny)), // gendoc:generate(entity=instance, group=security, key=security.bpffs.delegate_progs) // See {ref}`bpf-tokens` for more information. // // --- // type: string // liveupdate: no // condition: unprivileged container // shortdesc: What BPF program types to delegate "security.bpffs.delegate_progs": validate.Optional(validate.IsListOf(validate.IsAny)), // gendoc:generate(entity=instance, group=security, key=security.bpffs.delegate_attachs) // See {ref}`bpf-tokens` for more information. // // --- // type: string // liveupdate: no // condition: unprivileged container // shortdesc: What BPF attach types to delegate "security.bpffs.delegate_attachs": validate.Optional(validate.IsListOf(validate.IsAny)), // gendoc:generate(entity=instance, group=security, key=security.bpffs.path) // The specified path must exist in the container. // The BPF file system is only mounted if any of the `security.bpffs.delegate_*` options are set. // See {ref}`bpf-tokens` for more information. // // --- // type: string // defaultdesc: `/sys/fs/bpf` // liveupdate: no // condition: unprivileged container // shortdesc: The path to mount the BPF file system at "security.bpffs.path": validate.Optional(validate.IsAbsFilePath), // gendoc:generate(entity=instance, group=security, key=security.guestapi.images) // // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: container // shortdesc: Controls the availability of the `/1.0/images` API over `guestapi` "security.guestapi.images": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.idmap.base) // Setting this option overrides auto-detection. // --- // type: integer // liveupdate: no // condition: unprivileged container // shortdesc: The base host ID to use for the allocation "security.idmap.base": validate.Optional(validate.IsUint32), // gendoc:generate(entity=instance, group=security, key=security.idmap.isolated) // If specified, the idmap used for this instance is unique among instances that have this option set. // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: unprivileged container // shortdesc: Whether to use a unique idmap for this instance "security.idmap.isolated": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.idmap.size) // // --- // type: integer // liveupdate: no // condition: unprivileged container // shortdesc: The size of the idmap to use "security.idmap.size": validate.Optional(validate.IsUint32), // gendoc:generate(entity=instance, group=security, key=security.nesting) // // --- // type: bool // defaultdesc: `false` // liveupdate: yes // condition: container // shortdesc: Whether to support running Incus (nested) inside the instance "security.nesting": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.privileged) // // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: container // shortdesc: Whether to run the instance in privileged mode "security.privileged": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.protection.shift) // Set this option to `true` to prevent the instance's file system from being UID/GID shifted on startup. // --- // type: bool // defaultdesc: `false` // liveupdate: yes // condition: container // shortdesc: Whether to protect the file system from being UID/GID shifted "security.protection.shift": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.syscalls.allow) // A `\n`-separated list of syscalls to allow. // This list must be mutually exclusive with `security.syscalls.deny*`. // --- // type: string // liveupdate: no // condition: container // shortdesc: List of syscalls to allow "security.syscalls.allow": validate.IsAny, // Legacy configuration keys (old names). "security.syscalls.blacklist_default": validate.Optional(validate.IsBool), "security.syscalls.blacklist_compat": validate.Optional(validate.IsBool), "security.syscalls.blacklist": validate.IsAny, // gendoc:generate(entity=instance, group=security, key=security.syscalls.deny_default) // // --- // type: bool // defaultdesc: `true` // liveupdate: no // condition: container // shortdesc: Whether to enable the default syscall deny "security.syscalls.deny_default": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.syscalls.deny_compat) // On `x86_64`, this option controls whether to block `compat_*` syscalls. // On other architectures, the option is ignored. // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: container // shortdesc: Whether to block `compat_*` syscalls (`x86_64` only) "security.syscalls.deny_compat": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.syscalls.deny) // A `\n`-separated list of syscalls to deny. // This list must be mutually exclusive with `security.syscalls.allow`. // --- // type: string // liveupdate: no // condition: container // shortdesc: List of syscalls to deny "security.syscalls.deny": validate.IsAny, // gendoc:generate(entity=instance, group=security, key=security.syscalls.intercept.bpf) // // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: container // shortdesc: Whether to handle the `bpf()` system call "security.syscalls.intercept.bpf": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.syscalls.intercept.bpf.devices) // This option controls whether to allow BPF programs for the devices cgroup in the unified hierarchy to be loaded. // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: container // shortdesc: Whether to allow BPF programs "security.syscalls.intercept.bpf.devices": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.syscalls.intercept.mknod) // These system calls allow creation of a limited subset of char/block devices. // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: container // shortdesc: Whether to handle the `mknod` and `mknodat` system calls "security.syscalls.intercept.mknod": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.syscalls.intercept.mount) // // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: container // shortdesc: Whether to handle the `mount` system call "security.syscalls.intercept.mount": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.syscalls.intercept.mount.allowed) // Specify a comma-separated list of file systems that are safe to mount for processes inside the instance. // --- // type: string // liveupdate: yes // condition: container // shortdesc: File systems that can be mounted "security.syscalls.intercept.mount.allowed": validate.IsAny, // gendoc:generate(entity=instance, group=security, key=security.syscalls.intercept.mount.fuse) // Specify the mounts of a given file system that should be redirected to their FUSE implementation (for example, `ext4=fuse2fs`). // --- // type: string // liveupdate: yes // condition: container // shortdesc: File system that should be redirected to FUSE implementation "security.syscalls.intercept.mount.fuse": validate.IsAny, // gendoc:generate(entity=instance, group=security, key=security.syscalls.intercept.mount.shift) // // --- // type: bool // defaultdesc: `false` // liveupdate: yes // condition: container // shortdesc: Whether to use idmapped mounts for syscall interception "security.syscalls.intercept.mount.shift": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.syscalls.intercept.sched_setscheduler) // This system call allows increasing process priority. // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: container // shortdesc: Whether to handle the `sched_setscheduler` system call "security.syscalls.intercept.sched_setscheduler": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.syscalls.intercept.setxattr) // This system call allows setting a limited subset of restricted extended attributes. // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: container // shortdesc: Whether to handle the `setxattr` system call "security.syscalls.intercept.setxattr": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.syscalls.intercept.sysinfo) // This system call can be used to get cgroup-based resource usage information. // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: container // shortdesc: Whether to handle the `sysinfo` system call "security.syscalls.intercept.sysinfo": validate.Optional(validate.IsBool), "security.syscalls.whitelist": validate.IsAny, // gendoc:generate(entity=instance, group=volatile, key=volatile.container.oci) // // --- // type: bool // defaultdesc: `false` // shortdesc: Whether the container is an OCI application container "volatile.container.oci": validate.IsBool, // gendoc:generate(entity=instance, group=volatile, key=volatile.last_state.idmap) // // --- // type: string // shortdesc: Serialized instance UID/GID map "volatile.last_state.idmap": validate.IsAny, // gendoc:generate(entity=instance, group=volatile, key=volatile.idmap.base) // // --- // type: integer // shortdesc: The first ID in the instance's primary idmap range "volatile.idmap.base": validate.IsAny, // gendoc:generate(entity=instance, group=volatile, key=volatile.idmap.current) // // --- // type: string // shortdesc: The idmap currently in use by the instance "volatile.idmap.current": validate.IsAny, // gendoc:generate(entity=instance, group=volatile, key=volatile.idmap.next) // // --- // type: string // shortdesc: The idmap to use the next time the instance starts "volatile.idmap.next": validate.IsAny, } // InstanceConfigKeysVM is a map of config key to validator. (keys applying to VM only). var InstanceConfigKeysVM = map[string]func(value string) error{ // gendoc:generate(entity=instance, group=resource-limits, key=limits.memory.hotplug) // If this option is set to `false`, disable memory hotplug entirely. // Alternatively, it can be set to a bytes value which will define an upper limit for hotplugged memory. // The value must be greater than or equal to limits.memory. // --- // type: string // defaultdesc: `true` // liveupdate: yes // condition: virtual machine // shortdesc: Control upper limit for hotplugged memory or disable memory hotplug. "limits.memory.hotplug": validate.Optional(validate.Or(validate.IsBool, validate.IsSize)), // gendoc:generate(entity=instance, group=resource-limits, key=limits.memory.hugepages) // If this option is set to `false`, regular system memory is used. // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: virtual machine // shortdesc: Whether to back the instance using huge pages "limits.memory.hugepages": validate.Optional(validate.IsBool), // Caller is responsible for full validation of any raw.* value. // gendoc:generate(entity=instance, group=raw, key=raw.qemu) // // --- // type: blob // liveupdate: no // condition: virtual machine // shortdesc: Raw QEMU configuration to be appended to the generated command line "raw.qemu": validate.IsAny, // gendoc:generate(entity=instance, group=raw, key=raw.qemu.conf) // See {ref}`instance-options-qemu` for more information. // --- // type: blob // liveupdate: no // condition: virtual machine // shortdesc: Addition/override to the generated `qemu.conf` file "raw.qemu.conf": validate.IsAny, // gendoc:generate(entity=instance, group=raw, key=raw.qemu.qmp.early) // // --- // type: blob // liveupdate: no // condition: virtual machine // shortdesc: QMP commands to run before Incus QEMU initialization "raw.qemu.qmp.early": validate.IsAny, // gendoc:generate(entity=instance, group=raw, key=raw.qemu.qmp.post-start) // // --- // type: blob // liveupdate: no // condition: virtual machine // shortdesc: QMP commands to run after the VM has started "raw.qemu.qmp.post-start": validate.IsAny, // gendoc:generate(entity=instance, group=raw, key=raw.qemu.qmp.pre-start) // // --- // type: blob // liveupdate: no // condition: virtual machine // shortdesc: QMP commands to run after Incus QEMU initialization and before the VM has started "raw.qemu.qmp.pre-start": validate.IsAny, // gendoc:generate(entity=instance, group=raw, key=raw.qemu.scriptlet) // // --- // type: string // liveupdate: no // condition: virtual machine // shortdesc: QEMU scriptlet to run at early, pre-start and post-start stages "raw.qemu.scriptlet": validate.Optional(scriptletLoad.QEMUValidate), // gendoc:generate(entity=instance, group=security, key=security.agent.metrics) // // --- // type: bool // defaultdesc: `true` // liveupdate: no // condition: virtual machine // shortdesc: Whether the `incus-agent` is queried for state information and metrics "security.agent.metrics": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.csm) // When enabling this option, set {config:option}`instance-security:security.secureboot` to `false`. // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: virtual machine // shortdesc: Whether to use a firmware that supports UEFI-incompatible operating systems "security.csm": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.iommu) // // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: virtual machine // shortdesc: Whether to enable virtual IOMMU, useful for device passthrough and nesting "security.iommu": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.secureboot) // When disabling this option, consider enabling {config:option}`instance-security:security.csm`. // --- // type: bool // defaultdesc: `true` // liveupdate: no // condition: virtual machine // shortdesc: Whether UEFI secure boot is enforced with the default Microsoft keys "security.secureboot": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.sev) // // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: virtual machine // shortdesc: Whether AMD SEV (Secure Encrypted Virtualization) is enabled for this VM "security.sev": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.sev.policy.es) // // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: virtual machine // shortdesc: Whether AMD SEV-ES (SEV Encrypted State) is enabled for this VM "security.sev.policy.es": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=security, key=security.sev.session.dh) // // --- // type: string // defaultdesc: `true` // liveupdate: no // condition: virtual machine // shortdesc: The guest owner's `base64`-encoded Diffie-Hellman key "security.sev.session.dh": validate.Optional(validate.IsAny), // gendoc:generate(entity=instance, group=security, key=security.sev.session.data) // // --- // type: string // defaultdesc: `true` // liveupdate: no // condition: virtual machine // shortdesc: The guest owner's `base64`-encoded session blob "security.sev.session.data": validate.Optional(validate.IsAny), // gendoc:generate(entity=instance, group=miscellaneous, key=agent.nic_config) // For containers, the name and MTU of the default network interfaces is used for the instance devices. // For virtual machines, set this option to `true` to set the name and MTU of the default network interfaces to be the same as the instance devices. // --- // type: bool // defaultdesc: `false` // liveupdate: no // condition: virtual machine // shortdesc: Whether to use the name and MTU of the default network interfaces "agent.nic_config": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=volatile, key=volatile.apply_nvram) // // --- // type: bool // shortdesc: Whether to regenerate VM NVRAM the next time the instance starts "volatile.apply_nvram": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=volatile, key=volatile.vm.boot_state) // // --- // type: string // shortdesc: JSON encoded VM properties used during live migration and other state restoration. "volatile.vm.boot_state": validate.Optional(validate.IsAny), // gendoc:generate(entity=instance, group=volatile, key=volatile.vm.needs_reset) // // --- // type: bool // shortdesc: Indicates that the VM needs a full reset on next reboot "volatile.vm.needs_reset": validate.Optional(validate.IsBool), // gendoc:generate(entity=instance, group=volatile, key=volatile.vm.rtc_adjustment) // Real Time Clock adjustment time to allow virtual machines to run on a different base than the host. // --- // type: int64 // shortdesc: Real Time Clock change adjustment "volatile.vm.rtc_adjustment": validate.Optional(validate.IsInt64), // gendoc:generate(entity=instance, group=volatile, key=volatile.vm.rtc_offset) // Real Time Clock offset to allow virtual machines to run on a different base than the host. // --- // type: int64 // shortdesc: Real Time Clock change offset "volatile.vm.rtc_offset": validate.Optional(validate.IsInt64), // gendoc:generate(entity=instance, group=volatile, key=volatile.vsock_id) // // --- // type: string // shortdesc: Instance `vsock ID` used as of last start "volatile.vsock_id": validate.Optional(validate.IsInt64), // Deprecated keys, keep in map to allow transition. "volatile.vm.definition": validate.Optional(validate.IsAny), "volatile.vm.hotplug.memory": validate.Optional(validate.IsAny), } // ConfigKeyChecker returns a function that will check whether or not // a provide value is valid for the associate config key. Returns an // error if the key is not known. The checker function only performs // syntactic checking of the value, semantic and usage checking must // be done by the caller. User defined keys are always considered to // be valid, e.g. user.* and environment.* keys. func ConfigKeyChecker(key string, instanceType api.InstanceType) (func(value string) error, error) { f, ok := InstanceConfigKeysAny[key] if ok { return f, nil } if instanceType == api.InstanceTypeAny || instanceType == api.InstanceTypeContainer { f, ok := InstanceConfigKeysContainer[key] if ok { return f, nil } } if instanceType == api.InstanceTypeAny || instanceType == api.InstanceTypeVM { f, ok := InstanceConfigKeysVM[key] if ok { return f, nil } } if strings.HasPrefix(key, ConfigVolatilePrefix) { // gendoc:generate(entity=instance, group=volatile, key=volatile..apply_quota) // The disk quota is applied the next time the instance starts. // --- // type: string // shortdesc: Disk quota if strings.HasSuffix(key, ".apply_quota") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..ceph_rbd) // // --- // type: string // shortdesc: RBD device path for Ceph disk devices if strings.HasSuffix(key, ".ceph_rbd") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..host_name) // // --- // type: string // shortdesc: Network device name on the host if strings.HasSuffix(key, ".host_name") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..hwaddr) // The network device MAC address is used when no `hwaddr` property is set on the device itself. // --- // type: string // shortdesc: Network device MAC address if strings.HasSuffix(key, ".hwaddr") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..io.bus) // The IO bus stores the actual IO bus being used, checked in case `io.bus=auto`. // --- // type: string // shortdesc: IO bus in use if strings.HasSuffix(key, ".io.bus") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..mig.uuid) // The NVIDIA MIG instance UUID. // --- // type: string // shortdesc: MIG instance UUID if strings.HasSuffix(key, ".mig.uuid") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..name) // The network interface name inside of the instance when no `name` property is set on the device itself. // --- // type: string // shortdesc: Network interface name inside of the instance if strings.HasSuffix(key, ".name") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..vgpu.uuid) // The NVIDIA virtual GPU instance UUID. // --- // type: string // shortdesc: virtual GPU instance UUID if strings.HasSuffix(key, ".vgpu.uuid") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.created) // Possible values are `true` or `false`. // --- // type: string // shortdesc: Whether the network device physical device was created if strings.HasSuffix(key, ".last_state.created") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.hwaddr) // The original MAC that was used when moving a physical device into an instance. // --- // type: string // shortdesc: Network device original MAC if strings.HasSuffix(key, ".last_state.hwaddr") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.ip_addresses) // Comma-separated list of the last used IP addresses of the network device. // --- // type: string // shortdesc: Last used IP addresses if strings.HasSuffix(key, ".last_state.ip_addresses") { return validate.IsListOf(validate.IsNetworkAddress), nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.mtu) // The original MTU that was used when moving a physical device into an instance. // --- // type: string // shortdesc: Network device original MTU if strings.HasSuffix(key, ".last_state.mtu") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.pci.driver) // The original host driver for the PCI device. // --- // type: string // shortdesc: PCI original host driver if strings.HasSuffix(key, ".last_state.pci.driver") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.pci.parent) // The parent host device used when allocating a PCI device to an instance. // --- // type: string // shortdesc: PCI parent host device if strings.HasSuffix(key, ".last_state.pci.parent") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.pci.slot.name) // The parent host device PCI slot name. // --- // type: string // shortdesc: PCI parent slot name if strings.HasSuffix(key, ".last_state.pci.slot.name") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.usb.bus) // The original USB bus address. // --- // type: string // shortdesc: USB bus address if strings.HasSuffix(key, ".last_state.usb.bus") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.usb.device) // The original USB device identifier. // --- // type: string // shortdesc: USB device identifier if strings.HasSuffix(key, ".last_state.usb.device") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.vdpa.name) // The VDPA device name used when moving a VDPA device file descriptor into an instance. // --- // type: string // shortdesc: VDPA device name if strings.HasSuffix(key, ".last_state.vdpa.name") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.vf.hwaddr) // The original MAC used when moving a VF into an instance. // --- // type: string // shortdesc: SR-IOV virtual function original MAC if strings.HasSuffix(key, ".last_state.vf.hwaddr") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.vf.id) // The ID used when moving a VF into an instance. // --- // type: string // shortdesc: SR-IOV virtual function ID if strings.HasSuffix(key, ".last_state.vf.id") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.vf.parent) // The parent host device used when allocating a VF into an instance. // --- // type: string // shortdesc: SR-IOV parent host device if strings.HasSuffix(key, ".last_state.vf.parent") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.vf.spoofcheck) // The original spoof check setting used when moving a VF into an instance. // --- // type: string // shortdesc: SR-IOV virtual function original spoof check setting if strings.HasSuffix(key, ".last_state.vf.spoofcheck") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.vf.trusted) // The original trusted setting used when moving a VF into an instance. // --- // type: string // shortdesc: SR-IOV virtual function original trusted setting if strings.HasSuffix(key, ".last_state.vf.trusted") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=volatile, key=volatile..last_state.vf.vlan) // The original VLAN used when moving a VF into an instance. // --- // type: string // shortdesc: SR-IOV virtual function original VLAN if strings.HasSuffix(key, ".last_state.vf.vlan") { return validate.IsAny, nil } } // gendoc:generate(entity=instance, group=miscellaneous, key=environment.*) // Extra environment variables to set on boot and during exec. // --- // type: string // liveupdate: yes // shortdesc: Free-form environment key/value if strings.HasPrefix(key, "environment.") { return func(val string) error { if strings.Contains(val, "\n") { return errors.New("Environment variables cannot contain line breaks") } return nil }, nil } // gendoc:generate(entity=instance, group=miscellaneous, key=user.*) // User keys can be used in search. // --- // type: string // liveupdate: yes // shortdesc: Free-form user key/value storage if strings.HasPrefix(key, "user.") { return validate.IsAny, nil } if strings.HasPrefix(key, "image.") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=miscellaneous, key=smbios11.*) // `SMBIOS Type 11` configuration keys. // --- // type: string // liveupdate: yes // shortdesc: Free-form `SMBIOS Type 11` key/value if strings.HasPrefix(key, "smbios11.") && instanceType == api.InstanceTypeAny || instanceType == api.InstanceTypeVM { return validate.IsAny, nil } if strings.HasPrefix(key, "limits.kernel.") { // gendoc:generate(entity=kernel, group=limits, key=limits.kernel.as) // // --- // type: string // resource: `RLIMIT_AS` // shortdesc: Maximum size of the process's virtual memory if strings.HasSuffix(key, ".as") { return validate.IsAny, nil } // gendoc:generate(entity=kernel, group=limits, key=limits.kernel.core) // // --- // type: string // resource: `RLIMIT_CORE` // shortdesc: Maximum size of the process's core dump file if strings.HasSuffix(key, ".core") { return validate.IsAny, nil } // gendoc:generate(entity=kernel, group=limits, key=limits.kernel.cpu) // // --- // type: string // resource: `RLIMIT_CPU` // shortdesc: Limit in seconds on the amount of CPU time the process can consume if strings.HasSuffix(key, ".cpu") { return validate.IsAny, nil } // gendoc:generate(entity=kernel, group=limits, key=limits.kernel.data) // // --- // type: string // resource: `RLIMIT_DATA` // shortdesc: Maximum size of the process's data segment if strings.HasSuffix(key, ".data") { return validate.IsAny, nil } // gendoc:generate(entity=kernel, group=limits, key=limits.kernel.fsize) // // --- // type: string // resource: `RLIMIT_FSIZE` // shortdesc: Maximum size of files the process may create if strings.HasSuffix(key, ".fsize") { return validate.IsAny, nil } // gendoc:generate(entity=kernel, group=limits, key=limits.kernel.locks) // // --- // type: string // resource: `RLIMIT_LOCKS` // shortdesc: Limit on the number of file locks that this process may establish if strings.HasSuffix(key, ".locks") { return validate.IsAny, nil } // gendoc:generate(entity=kernel, group=limits, key=limits.kernel.memlock) // // --- // type: string // resource: `RLIMIT_MEMLOCK` // shortdesc: Limit on the number of bytes of memory that the process may lock in RAM if strings.HasSuffix(key, ".memlock") { return validate.IsAny, nil } // gendoc:generate(entity=kernel, group=limits, key=limits.kernel.nice) // // --- // type: string // resource: `RLIMIT_NICE` // shortdesc: Maximum value to which the process's nice value can be raised if strings.HasSuffix(key, ".nice") { return validate.IsAny, nil } // gendoc:generate(entity=kernel, group=limits, key=limits.kernel.nofile) // // --- // type: string // resource: `RLIMIT_NOFILE` // shortdesc: Maximum number of open files for the process if strings.HasSuffix(key, ".nofile") { return validate.IsAny, nil } // gendoc:generate(entity=kernel, group=limits, key=limits.kernel.nproc) // // --- // type: string // resource: `RLIMIT_NPROC` // shortdesc: Maximum number of processes that can be created for the user of the calling process if strings.HasSuffix(key, ".nproc") { return validate.IsAny, nil } // gendoc:generate(entity=kernel, group=limits, key=limits.kernel.rtprio) // // --- // type: string // resource: `RLIMIT_RTPRIO` // shortdesc: Maximum value on the real-time-priority that may be set for this process if strings.HasSuffix(key, ".rtprio") { return validate.IsAny, nil } // gendoc:generate(entity=kernel, group=limits, key=limits.kernel.sigpending) // // --- // type: string // resource: `RLIMIT_SIGPENDING` // shortdesc: Limit on the number of bytes of memory that the process may lock in RAM if strings.HasSuffix(key, ".sigpending") { return validate.IsAny, nil } if len(key) > len("limits.kernel.") { return validate.IsAny, nil } } if (instanceType == api.InstanceTypeAny || instanceType == api.InstanceTypeContainer) && strings.HasPrefix(key, "linux.sysctl.") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=miscellaneous, key=systemd.credential.*) // Systemd credential key/value pair passed as a read-only bind mount in containers and as `SMBIOS Type 11` data in virtual machines. // --- // type: string // liveupdate: yes // shortdesc: Systemd credential key/value if strings.HasPrefix(key, "systemd.credential.") { return validate.IsAny, nil } // gendoc:generate(entity=instance, group=miscellaneous, key=systemd.credential-binary.*) // Systemd credential key/value pair passed as a read-only bind mount in containers and as `SMBIOS Type 11` data in virtual machines. The value is Base64 encoded. // --- // type: string // liveupdate: yes // shortdesc: Systemd credential key/value, where value is Base64 encoded if strings.HasPrefix(key, "systemd.credential-binary.") { return validate.IsBase64, nil } return nil, fmt.Errorf("Unknown configuration key: %s", key) } // InstanceIncludeWhenCopying is used to decide whether to include a config item or not when copying an instance. // The remoteCopy argument indicates if the copy is remote (i.e between servers) as this affects the keys kept. func InstanceIncludeWhenCopying(configKey string, remoteCopy bool) bool { if configKey == "volatile.apply_nvram" { return true // Include volatile.apply_nvram to also reset the NVRAM in copied instances. } if configKey == "volatile.base_image" { return true // Include volatile.base_image always as it can help optimize copies. } if configKey == "volatile.last_state.idmap" && !remoteCopy { return true // Include volatile.last_state.idmap when doing local copy to avoid needless remapping. } if strings.HasPrefix(configKey, ConfigVolatilePrefix) { return false // Exclude all other volatile keys. } return true // Keep all other keys. } incus-7.0.0/internal/instance/device.go000066400000000000000000000032471517523235500200630ustar00rootroot00000000000000package instance import ( "errors" "strings" ) // IsRootDiskDevice returns true if the given device representation is configured as root disk for // an instance. It typically get passed a specific entry of api.Instance.Devices. func IsRootDiskDevice(device map[string]string) bool { // Root disk devices also need a non-empty "pool" property, but we can't check that here // because this function is used with clients talking to older servers where there was no // concept of a storage pool, and also it is used for migrating from old to new servers. // The validation of the non-empty "pool" property is done inside the disk device itself. if device["type"] == "disk" && device["path"] == "/" && device["source"] == "" { return true } return false } // ErrNoRootDisk means there is no root disk device found. var ErrNoRootDisk = errors.New("No root device could be found") // GetRootDiskDevice returns the instance device that is configured as root disk. // Returns the device name and device config map. func GetRootDiskDevice(devices map[string]map[string]string) (string, map[string]string, error) { var devName string var dev map[string]string for n, d := range devices { if IsRootDiskDevice(d) { if devName != "" { return "", nil, errors.New("More than one root device found") } devName = n dev = d } } if devName != "" { return devName, dev, nil } return "", nil, ErrNoRootDisk } // SplitVolumeSource splits the volume name and any provided sub-path. func SplitVolumeSource(source string) (string, string) { volFields := strings.SplitN(source, "/", 2) if len(volFields) == 1 { return volFields[0], "" } return volFields[0], volFields[1] } incus-7.0.0/internal/instance/expiry.go000066400000000000000000000027051517523235500201420ustar00rootroot00000000000000package instance import ( "errors" "regexp" "strconv" "strings" "time" ) // ErrInvalidExpiry is returned if the provided expiry cannot be parsed. var ErrInvalidExpiry = errors.New("Invalid expiry expression") // GetExpiry returns the expiry date based on the reference date and a length of time. // The length of time format is "(S|M|H|d|w|m|y)", and can contain multiple such fields, e.g. // "1d 3H" (1 day and 3 hours). func GetExpiry(refDate time.Time, s string) (time.Time, error) { expr := strings.TrimSpace(s) if expr == "" { return time.Time{}, nil } re, err := regexp.Compile(`^(\d+)(S|M|H|d|w|m|y)$`) if err != nil { return time.Time{}, err } expiry := map[string]int{ "S": 0, "M": 0, "H": 0, "d": 0, "w": 0, "m": 0, "y": 0, } values := strings.Split(expr, " ") if len(values) == 0 { return time.Time{}, nil } for _, value := range values { fields := re.FindStringSubmatch(value) if fields == nil { return time.Time{}, ErrInvalidExpiry } if expiry[fields[2]] > 0 { // We don't allow fields to be set multiple times return time.Time{}, ErrInvalidExpiry } val, err := strconv.Atoi(fields[1]) if err != nil { return time.Time{}, err } expiry[fields[2]] = val } t := refDate.AddDate(expiry["y"], expiry["m"], expiry["d"]+expiry["w"]*7).Add( time.Hour*time.Duration(expiry["H"]) + time.Minute*time.Duration(expiry["M"]) + time.Second*time.Duration(expiry["S"])) return t, nil } incus-7.0.0/internal/instance/snapshot.go000066400000000000000000000004451517523235500204600ustar00rootroot00000000000000package instance import ( "strings" ) // SnapshotDelimiter is used to separate instance name from snapshot name. const SnapshotDelimiter = "/" // IsSnapshot checks if provided name is a snapshot name. func IsSnapshot(name string) bool { return strings.Contains(name, SnapshotDelimiter) } incus-7.0.0/internal/instancewriter/000077500000000000000000000000001517523235500175245ustar00rootroot00000000000000incus-7.0.0/internal/instancewriter/instance_file_info.go000066400000000000000000000015561517523235500237000ustar00rootroot00000000000000package instancewriter import ( "archive/tar" "os" "time" ) // FileInfo static file implementation of os.FileInfo. type FileInfo struct { FileName string FileSize int64 FileMode os.FileMode FileModTime time.Time } // Name of file. func (f *FileInfo) Name() string { return f.FileName } // Size of file. func (f *FileInfo) Size() int64 { return f.FileSize } // Mode of file. func (f *FileInfo) Mode() os.FileMode { return f.FileMode } // ModTime of file. func (f *FileInfo) ModTime() time.Time { return f.FileModTime } // IsDir is file a directory. func (f *FileInfo) IsDir() bool { return false } // Sys returns further unix attributes for a file owned by root. func (f *FileInfo) Sys() any { return &tar.Header{ Uid: 0, Gid: 0, Uname: "root", Gname: "root", AccessTime: time.Now(), ChangeTime: time.Now(), } } incus-7.0.0/internal/instancewriter/instance_raw_writer.go000066400000000000000000000015501517523235500241250ustar00rootroot00000000000000package instancewriter import ( "io" "os" ) // InstanceRawWriter provides an InstanceWriter implementation that copies the contents of a file to another. type InstanceRawWriter struct { rawWriter io.WriteCloser } // NewInstanceRawWriter returns an InstanceRawWriter for the provided target file. func NewInstanceRawWriter(writer io.WriteCloser) *InstanceRawWriter { return &InstanceRawWriter{rawWriter: writer} } // ResetHardLinkMap is a no-op. func (crw *InstanceRawWriter) ResetHardLinkMap() {} // WriteFile is a no-op. func (crw *InstanceRawWriter) WriteFile(name string, srcPath string, fi os.FileInfo, ignoreGrowth bool) error { return nil } // WriteFileFromReader streams a file into the target file. func (crw *InstanceRawWriter) WriteFileFromReader(src io.Reader, fi os.FileInfo) error { _, err := io.CopyN(crw.rawWriter, src, fi.Size()) return err } incus-7.0.0/internal/instancewriter/instance_tar_writer.go000066400000000000000000000135601517523235500241260ustar00rootroot00000000000000//go:build linux && cgo package instancewriter import ( "archive/tar" "fmt" "io" "os" "strings" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) // InstanceTarWriter provides an InstanceWriter implementation that handles ID shifting and hardlink tracking. type InstanceTarWriter struct { tarWriter *tar.Writer idmapSet *idmap.Set linkMap map[uint64]string } // NewInstanceTarWriter returns an InstanceTarWriter for the provided target Writer and id map. func NewInstanceTarWriter(writer io.Writer, idmapSet *idmap.Set) *InstanceTarWriter { ctw := &InstanceTarWriter{} ctw.tarWriter = tar.NewWriter(writer) ctw.idmapSet = idmapSet ctw.linkMap = map[uint64]string{} return ctw } // ResetHardLinkMap resets the hard link map. Use when copying multiple instances (or snapshots) into a tarball. // So that the hard link map doesn't work across different instances/snapshots. func (ctw *InstanceTarWriter) ResetHardLinkMap() { ctw.linkMap = map[uint64]string{} } // WriteFile adds a file to the tarball with the specified name using the srcPath file as the contents of the file. // The ignoreGrowth argument indicates whether to error if the srcPath file increases in size beyond the size in fi // during the write. If false the write will return an error. If true, no error is returned, instead only the size // specified in fi is written to the tarball. This can be used when you don't need a consistent copy of the file. func (ctw *InstanceTarWriter) WriteFile(name string, srcPath string, fi os.FileInfo, ignoreGrowth bool) error { var err error var major, minor uint32 var nlink int var ino uint64 link := "" if fi.Mode()&os.ModeSymlink == os.ModeSymlink { link, err = os.Readlink(srcPath) if err != nil { return fmt.Errorf("Failed to resolve symlink for %q: %w", srcPath, err) } } // Sockets cannot be stored in tarballs, just skip them (consistent with tar). if fi.Mode()&os.ModeSocket == os.ModeSocket { return nil } hdr, err := tar.FileInfoHeader(fi, link) if err != nil { return fmt.Errorf("Failed to create tar info header: %w", err) } hdr.Name = name if fi.IsDir() || fi.Mode()&os.ModeSymlink == os.ModeSymlink { hdr.Size = 0 } else { hdr.Size = fi.Size() } // Get file stat. var stat unix.Stat_t err = unix.Lstat(srcPath, &stat) if err != nil { return fmt.Errorf("Failed to get file stat: %w", err) } hdr.Uid = int(stat.Uid) hdr.Gid = int(stat.Gid) ino = stat.Ino nlink = int(stat.Nlink) if stat.Mode&unix.S_IFBLK != 0 || stat.Mode&unix.S_IFCHR != 0 { major = unix.Major(uint64(stat.Rdev)) minor = unix.Minor(uint64(stat.Rdev)) } // Unshift the id under rootfs/ for unpriv containers. if strings.HasPrefix(hdr.Name, "rootfs") && ctw.idmapSet != nil { hUID, hGID := ctw.idmapSet.ShiftFromNS(int64(hdr.Uid), int64(hdr.Gid)) hdr.Uid = int(hUID) hdr.Gid = int(hGID) if hdr.Uid == -1 || hdr.Gid == -1 { return nil } } hdr.Devmajor = int64(major) hdr.Devminor = int64(minor) // If it's a hardlink we've already seen use the old name. if fi.Mode().IsRegular() && nlink > 1 { firstPath, found := ctw.linkMap[ino] if found { hdr.Typeflag = tar.TypeLink hdr.Linkname = firstPath hdr.Size = 0 } else { ctw.linkMap[ino] = hdr.Name } } // Handle xattrs (for real files only). if link == "" { xattrs, err := linux.GetAllXattr(srcPath) if err != nil { return fmt.Errorf("Failed to read xattr for %q: %w", srcPath, err) } hdr.PAXRecords = make(map[string]string, len(xattrs)) for key, val := range xattrs { if key == "system.posix_acl_access" && ctw.idmapSet != nil { aclAccess, err := idmap.UnshiftACL(val, ctw.idmapSet) if err != nil { logger.Debugf("Failed to unshift ACL access permissions of %q: %v", srcPath, err) continue } val = aclAccess } else if key == "system.posix_acl_default" && ctw.idmapSet != nil { aclDefault, err := idmap.UnshiftACL(val, ctw.idmapSet) if err != nil { logger.Debugf("Failed to unshift ACL default permissions of %q: %v", srcPath, err) continue } val = aclDefault } else if key == "security.capability" && ctw.idmapSet != nil { vfsCaps, err := idmap.UnshiftCaps(val, ctw.idmapSet) if err != nil { logger.Debugf("Failed to unshift VFS capabilities of %q: %v", srcPath, err) continue } val = vfsCaps } hdr.PAXRecords["SCHILY.xattr."+key] = val } } err = ctw.tarWriter.WriteHeader(hdr) if err != nil { return fmt.Errorf("Failed to write tar header: %w", err) } if hdr.Typeflag == tar.TypeReg { f, err := os.Open(srcPath) if err != nil { return fmt.Errorf("Failed to open file %q: %w", srcPath, err) } defer func() { _ = f.Close() }() r := io.Reader(f) if ignoreGrowth { r = io.LimitReader(r, fi.Size()) } _, err = util.SafeCopy(ctw.tarWriter, r) if err != nil { return fmt.Errorf("Failed to copy file content %q: %w", srcPath, err) } err = f.Close() if err != nil { return fmt.Errorf("Failed to close file %q: %w", srcPath, err) } } return nil } // WriteFileFromReader streams a file into the tarball using the src reader. // A manually generated os.FileInfo should be supplied so that the tar header can be added before streaming starts. func (ctw *InstanceTarWriter) WriteFileFromReader(src io.Reader, fi os.FileInfo) error { hdr, err := tar.FileInfoHeader(fi, "") if err != nil { return fmt.Errorf("Failed to create tar info header: %w", err) } err = ctw.tarWriter.WriteHeader(hdr) if err != nil { return fmt.Errorf("Failed to write tar header: %w", err) } _, err = util.SafeCopy(ctw.tarWriter, src) return err } // Close finishes writing the tarball. func (ctw *InstanceTarWriter) Close() error { err := ctw.tarWriter.Close() if err != nil { return fmt.Errorf("Failed to close tar writer: %w", err) } return nil } incus-7.0.0/internal/instancewriter/instance_writer_interface.go000066400000000000000000000004451517523235500252760ustar00rootroot00000000000000package instancewriter import ( "io" "os" ) // InstanceWriter is the instance writer interface. type InstanceWriter interface { ResetHardLinkMap() WriteFile(name string, srcPath string, fi os.FileInfo, ignoreGrowth bool) error WriteFileFromReader(src io.Reader, fi os.FileInfo) error } incus-7.0.0/internal/io/000077500000000000000000000000001517523235500150725ustar00rootroot00000000000000incus-7.0.0/internal/io/bytesreadcloser.go000066400000000000000000000005401517523235500206120ustar00rootroot00000000000000package io import ( "bytes" ) // BytesReadCloser is a basic in-memory reader with a closer interface. type BytesReadCloser struct { Buf *bytes.Buffer } // Read just returns the buffer. func (r BytesReadCloser) Read(b []byte) (n int, err error) { return r.Buf.Read(b) } // Close is a no-op. func (r BytesReadCloser) Close() error { return nil } incus-7.0.0/internal/io/filesystem.go000066400000000000000000000004231517523235500176040ustar00rootroot00000000000000package io import ( "os" ) // GetPathMode returns a os.FileMode for the provided path. func GetPathMode(path string) (os.FileMode, error) { fi, err := os.Stat(path) if err != nil { return os.FileMode(0o000), err } mode, _, _ := GetOwnerMode(fi) return mode, nil } incus-7.0.0/internal/io/filesystem_unix.go000066400000000000000000000005361517523235500206540ustar00rootroot00000000000000//go:build !windows package io import ( "os" "syscall" ) // GetOwnerMode returns the file mode, owner UID, and owner GID for the given file. func GetOwnerMode(fInfo os.FileInfo) (os.FileMode, int, int) { mode := fInfo.Mode() uid := int(fInfo.Sys().(*syscall.Stat_t).Uid) gid := int(fInfo.Sys().(*syscall.Stat_t).Gid) return mode, uid, gid } incus-7.0.0/internal/io/filesystem_windows.go000066400000000000000000000002201517523235500213510ustar00rootroot00000000000000//go:build windows package io import ( "os" ) func GetOwnerMode(fInfo os.FileInfo) (os.FileMode, int, int) { return fInfo.Mode(), -1, -1 } incus-7.0.0/internal/io/quotawriter.go000066400000000000000000000013311517523235500200050ustar00rootroot00000000000000package io import ( "fmt" "io" ) // QuotaWriter returns an error once a given write quota gets exceeded. type QuotaWriter struct { writer io.Writer quota int64 n int64 } // NewQuotaWriter returns a new QuotaWriter wrapping the given writer. // // If the given quota is negative, then no quota is applied. func NewQuotaWriter(writer io.Writer, quota int64) *QuotaWriter { return &QuotaWriter{ writer: writer, quota: quota, } } // Write implements the Writer interface. func (w *QuotaWriter) Write(p []byte) (n int, err error) { if w.quota >= 0 { w.n += int64(len(p)) if w.n > w.quota { return 0, fmt.Errorf("reached %d bytes, exceeding quota of %d", w.n, w.quota) } } return w.writer.Write(p) } incus-7.0.0/internal/io/readseeker.go000066400000000000000000000010601517523235500175300ustar00rootroot00000000000000package io import ( "io" ) type readSeeker struct { io.Reader io.Seeker } // NewReadSeeker combines provided io.Reader and io.Seeker into a new io.ReadSeeker. func NewReadSeeker(reader io.Reader, seeker io.Seeker) io.ReadSeeker { return &readSeeker{Reader: reader, Seeker: seeker} } // Read reads data from the reader. func (r *readSeeker) Read(p []byte) (n int, err error) { return r.Reader.Read(p) } // Seek seeks to the specified offset. func (r *readSeeker) Seek(offset int64, whence int) (int64, error) { return r.Seeker.Seek(offset, whence) } incus-7.0.0/internal/io/writer.go000066400000000000000000000005731517523235500167420ustar00rootroot00000000000000package io import ( "bytes" "io" "github.com/lxc/incus/v7/shared/util" ) // WriteAll copies content of data to specified writer. func WriteAll(w io.Writer, data []byte) error { buf := bytes.NewBuffer(data) toWrite := int64(buf.Len()) for { n, err := util.SafeCopy(w, buf) if err != nil { return err } toWrite -= n if toWrite <= 0 { return nil } } } incus-7.0.0/internal/iprange/000077500000000000000000000000001517523235500161105ustar00rootroot00000000000000incus-7.0.0/internal/iprange/range.go000066400000000000000000000011331517523235500175310ustar00rootroot00000000000000package iprange import ( "bytes" "fmt" "net" ) // Range defines a range of IP addresses. // Optionally just set Start to indicate a single IP. type Range struct { Start net.IP End net.IP } // ContainsIP tests whether a supplied IP falls within the IPRange. func (r *Range) ContainsIP(ip net.IP) bool { if r.End == nil { // the range is only a single IP return r.Start.Equal(ip) } return bytes.Compare(ip, r.Start) >= 0 && bytes.Compare(ip, r.End) <= 0 } func (r *Range) String() string { if r.End == nil { return r.Start.String() } return fmt.Sprintf("%v-%v", r.Start, r.End) } incus-7.0.0/internal/iprange/range_test.go000066400000000000000000000053031517523235500205730ustar00rootroot00000000000000package iprange_test import ( "net" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/lxc/incus/v7/internal/iprange" ) // parseRange is a custom parse function to not depend on other packages. func parseRange(rangeString string) iprange.Range { ips := strings.Split(rangeString, "-") start := net.ParseIP(ips[0]) var end net.IP if len(ips) == 2 { end = net.ParseIP(ips[1]) } return iprange.Range{ Start: start, End: end, } } func TestRange_ContainsIP(t *testing.T) { type containsIPTest struct { name string rangeString string // a string of format - optionally just an testIP string expected bool } tests := []containsIPTest{ { name: "ip below range", rangeString: "10.10.0.0-10.16.0.0", testIP: "10.0.0.1", expected: false, }, { name: "ip is lower bound", rangeString: "10.10.0.0-10.16.0.0", testIP: "10.10.0.0", expected: true, }, { name: "ip in range", rangeString: "10.10.0.0-10.16.0.0", testIP: "10.12.59.1", expected: true, }, { name: "ip is upper bound", rangeString: "10.10.0.0-10.16.0.0", testIP: "10.16.0.0", expected: true, }, { name: "ip above range", rangeString: "10.10.0.0-10.16.0.0", testIP: "10.23.59.1", expected: false, }, { name: "range has no end and ip is below range", rangeString: "10.10.0.1", testIP: "10.2.59.1", expected: false, }, { name: "range has no end and ip is in range", rangeString: "10.10.0.1", testIP: "10.10.0.1", expected: true, }, { name: "range has no end and ip is above range", rangeString: "10.10.0.1", testIP: "10.23.59.1", expected: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { // arrange r := parseRange(test.rangeString) testIP := net.ParseIP(test.testIP) // act isContained := r.ContainsIP(testIP) // assert assert.Equal(t, test.expected, isContained) }) } } func TestRange_String(t *testing.T) { type stringTest struct { name string rangeString string // a string of format - optionally just an expected string } tests := []stringTest{ { name: "start and end", rangeString: "10.10.0.0-10.16.0.5", expected: "10.10.0.0-10.16.0.5", }, { name: "start only", rangeString: "10.10.0.0", expected: "10.10.0.0", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { // arrange r := parseRange(test.rangeString) // act s := r.String() // assert assert.Equal(t, test.expected, s) }) } } incus-7.0.0/internal/jmap/000077500000000000000000000000001517523235500154125ustar00rootroot00000000000000incus-7.0.0/internal/jmap/map.go000066400000000000000000000025421517523235500165210ustar00rootroot00000000000000package jmap import ( "fmt" ) // Map represents a simple JSON map. type Map map[string]any // GetString retrieves a value from the map as a string. func (m Map) GetString(key string) (string, error) { val, ok := m[key] if !ok { return "", fmt.Errorf("Response was missing `%s`", key) } strVal, ok := val.(string) if !ok { return "", fmt.Errorf("`%s` was not a string", key) } return strVal, nil } // GetMap retrieves a value from the map as a map. func (m Map) GetMap(key string) (Map, error) { val, ok := m[key] if !ok { return nil, fmt.Errorf("Response was missing `%s`", key) } mapVal, ok := val.(map[string]any) if !ok { return nil, fmt.Errorf("`%s` was not a map, got %T", key, m[key]) } return mapVal, nil } // GetInt retrieves a value from the map as an int. func (m Map) GetInt(key string) (int, error) { val, ok := m[key] if !ok { return -1, fmt.Errorf("Response was missing `%s`", key) } floatVal, ok := val.(float64) if !ok { return -1, fmt.Errorf("`%s` was not an int", key) } return int(floatVal), nil } // GetBool retrieves a value from the map as a bool. func (m Map) GetBool(key string) (bool, error) { val, ok := m[key] if !ok { return false, fmt.Errorf("Response was missing `%s`", key) } boolVal, ok := val.(bool) if !ok { return false, fmt.Errorf("`%s` was not a bool", key) } return boolVal, nil } incus-7.0.0/internal/jmap/map_test.go000066400000000000000000000120721517523235500175570ustar00rootroot00000000000000package jmap import ( "testing" "github.com/stretchr/testify/assert" ) // --- GetString --- func TestMap_GetString_Valid(t *testing.T) { // GIVEN m := Map{"name": "incus"} // WHEN got, err := m.GetString("name") // THEN assert.NoError(t, err) assert.Equal(t, "incus", got) } func TestMap_GetString_EmptyString(t *testing.T) { // GIVEN m := Map{"name": ""} // WHEN got, err := m.GetString("name") // THEN assert.NoError(t, err) assert.Equal(t, "", got) } func TestMap_GetString_MissingKey(t *testing.T) { // GIVEN m := Map{} // WHEN got, err := m.GetString("name") // THEN assert.Error(t, err) assert.Equal(t, "", got) assert.Equal(t, "Response was missing `name`", err.Error()) } func TestMap_GetString_WrongTypeInt(t *testing.T) { // GIVEN m := Map{"name": 42} // WHEN got, err := m.GetString("name") // THEN assert.Error(t, err) assert.Equal(t, "", got) assert.Equal(t, "`name` was not a string", err.Error()) } func TestMap_GetString_WrongTypeBool(t *testing.T) { // GIVEN m := Map{"name": true} // WHEN got, err := m.GetString("name") // THEN assert.Error(t, err) assert.Equal(t, "", got) assert.Equal(t, "`name` was not a string", err.Error()) } // --- GetMap --- func TestMap_GetMap_Valid(t *testing.T) { // GIVEN m := Map{"inner": map[string]any{"name": "incus"}} // WHEN got, err := m.GetMap("inner") // THEN assert.NoError(t, err) assert.Equal(t, Map{"name": "incus"}, got) } func TestMap_GetMap_EmptyMap(t *testing.T) { // GIVEN m := Map{"inner": map[string]any{}} // WHEN got, err := m.GetMap("inner") // THEN assert.NoError(t, err) assert.Equal(t, Map{}, got) } func TestMap_GetMap_MissingKey(t *testing.T) { // GIVEN m := Map{} // WHEN got, err := m.GetMap("inner") // THEN assert.Error(t, err) assert.Nil(t, got) assert.Equal(t, "Response was missing `inner`", err.Error()) } func TestMap_GetMap_WrongTypeString(t *testing.T) { // GIVEN m := Map{"inner": "incus"} // WHEN got, err := m.GetMap("inner") // THEN assert.Error(t, err) assert.Nil(t, got) assert.Equal(t, "`inner` was not a map, got string", err.Error()) } func TestMap_GetMap_WrongTypeInt(t *testing.T) { // GIVEN m := Map{"inner": 123} // WHEN got, err := m.GetMap("inner") // THEN assert.Error(t, err) assert.Nil(t, got) assert.Equal(t, "`inner` was not a map, got int", err.Error()) } // --- GetInt --- func TestMap_GetInt_Valid(t *testing.T) { // GIVEN m := Map{"num": 42.0} // WHEN got, err := m.GetInt("num") // THEN assert.NoError(t, err) assert.Equal(t, 42, got) } func TestMap_GetInt_Negative(t *testing.T) { // GIVEN m := Map{"num": -10.0} // WHEN got, err := m.GetInt("num") // THEN assert.NoError(t, err) assert.Equal(t, -10, got) } func TestMap_GetInt_Zero(t *testing.T) { // GIVEN m := Map{"num": 0.0} // WHEN got, err := m.GetInt("num") // THEN assert.NoError(t, err) assert.Equal(t, 0, got) } func TestMap_GetInt_FloatTruncated(t *testing.T) { // GIVEN m := Map{"num": 42.7} // WHEN got, err := m.GetInt("num") // THEN assert.NoError(t, err) assert.Equal(t, 42, got) } func TestMap_GetInt_MissingKey(t *testing.T) { // GIVEN m := Map{} // WHEN got, err := m.GetInt("num") // THEN assert.Error(t, err) assert.Equal(t, -1, got) assert.Equal(t, "Response was missing `num`", err.Error()) } func TestMap_GetInt_WrongTypeString(t *testing.T) { // GIVEN m := Map{"num": "123"} // WHEN got, err := m.GetInt("num") // THEN assert.Error(t, err) assert.Equal(t, -1, got) assert.Equal(t, "`num` was not an int", err.Error()) } func TestMap_GetInt_WrongTypeBool(t *testing.T) { // GIVEN m := Map{"num": true} // WHEN got, err := m.GetInt("num") // THEN assert.Error(t, err) assert.Equal(t, -1, got) assert.Equal(t, "`num` was not an int", err.Error()) } // --- GetBool --- func TestMap_GetBool_True(t *testing.T) { // GIVEN m := Map{"flag": true} // WHEN got, err := m.GetBool("flag") // THEN assert.NoError(t, err) assert.Equal(t, true, got) } func TestMap_GetBool_False(t *testing.T) { // GIVEN m := Map{"flag": false} // WHEN got, err := m.GetBool("flag") // THEN assert.NoError(t, err) assert.Equal(t, false, got) } func TestMap_GetBool_MissingKey(t *testing.T) { // GIVEN m := Map{} // WHEN got, err := m.GetBool("flag") // THEN assert.Error(t, err) assert.Equal(t, false, got) assert.Equal(t, "Response was missing `flag`", err.Error()) } func TestMap_GetBool_WrongTypeString(t *testing.T) { // GIVEN m := Map{"flag": "true"} // WHEN got, err := m.GetBool("flag") // THEN assert.Error(t, err) assert.Equal(t, false, got) assert.Equal(t, "`flag` was not a bool", err.Error()) } func TestMap_GetBool_WrongTypeInt(t *testing.T) { // GIVEN m := Map{"flag": 1} // WHEN got, err := m.GetBool("flag") // THEN assert.Error(t, err) assert.Equal(t, false, got) assert.Equal(t, "`flag` was not a bool", err.Error()) } func TestMap_GetBool_WrongTypeIntZero(t *testing.T) { // GIVEN m := Map{"flag": 0} // WHEN got, err := m.GetBool("flag") // THEN assert.Error(t, err) assert.Equal(t, false, got) assert.Equal(t, "`flag` was not a bool", err.Error()) } incus-7.0.0/internal/linux/000077500000000000000000000000001517523235500156225ustar00rootroot00000000000000incus-7.0.0/internal/linux/cgo.go000066400000000000000000000007221517523235500167220ustar00rootroot00000000000000//go:build linux && cgo package linux // #cgo CFLAGS: -std=gnu11 -Wvla -Werror -fvisibility=hidden -Winit-self // #cgo CFLAGS: -Wformat=2 -Wshadow -Wendif-labels -fasynchronous-unwind-tables // #cgo CFLAGS: -pipe --param=ssp-buffer-size=4 -g -Wunused // #cgo CFLAGS: -Werror=implicit-function-declaration // #cgo CFLAGS: -Werror=return-type -Wendif-labels -Werror=overflow // #cgo CFLAGS: -Wnested-externs -fexceptions // #cgo LDFLAGS: -lutil -lpthread import "C" incus-7.0.0/internal/linux/discard.go000066400000000000000000000143211517523235500175630ustar00rootroot00000000000000package linux import ( "errors" "fmt" "io" "os" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/subprocess" ) // ClearBlock fully resets a block device or disk file using the most efficient mechanism available. // For files, it will truncate them down to zero and back to their original size. // For blocks, it will attempt a variety of discard options, validating the result with marker files and eventually fallback to full zero-ing. // // An offset can be specified to only reset a part of a device. func ClearBlock(blockPath string, blockOffset int64) error { logger.Debug("Clearing block device", logger.Ctx{"dev": blockPath, "offset": blockOffset}) // Open the block device for checking. fd, err := os.OpenFile(blockPath, os.O_RDWR, 0o644) if err != nil { if os.IsNotExist(err) { // If the file is missing, there is nothing to clear. return nil } return err } defer fd.Close() // Get the size of the file/block. size, err := fd.Seek(0, io.SeekEnd) if err != nil { return err } // Get all the stat data. st, err := fd.Stat() if err != nil { return err } if !IsBlockdev(st.Mode()) { // For files, truncate them. err := fd.Truncate(blockOffset) if err != nil { return err } err = fd.Truncate(size) if err != nil { return err } logger.Debug("Cleared block device", logger.Ctx{"dev": blockPath, "offset": blockOffset, "method": "truncate"}) return nil } // Blocks are trickier to reset with options varying based on disk features. // We use a set of 3 markers to validate whether it was reset. marker := []byte("INCUS") markerLength := int64(len(marker)) markerOffsetStart := blockOffset markerOffsetMiddle := blockOffset + ((size - blockOffset) / 2) markerOffsetEnd := size - markerLength if markerOffsetStart+markerLength > size { // No markers can fit. markerOffsetStart = -1 markerOffsetMiddle = -1 markerOffsetEnd = -1 } else { if markerOffsetMiddle <= markerOffsetStart+markerLength { // Middle marker goes over start marker. markerOffsetMiddle = -1 } if markerOffsetEnd <= markerOffsetMiddle+markerLength { // End marker goes over middle marker. markerOffsetEnd = -1 } if markerOffsetEnd <= markerOffsetStart+markerLength { // End marker goes over start marker. markerOffsetEnd = -1 } } writeMarkers := func(fd *os.File) error { for _, offset := range []int64{markerOffsetStart, markerOffsetMiddle, markerOffsetEnd} { if offset < 0 { continue } // Write the marker at the set offset. n, err := fd.WriteAt(marker, offset) if err != nil { return err } if n != int(markerLength) { return fmt.Errorf("Only managed to write %d bytes out of %d of the %d offset marker", n, markerLength, offset) } } return nil } checkMarkers := func(fd *os.File) (int, error) { found := 0 for _, offset := range []int64{markerOffsetStart, markerOffsetMiddle, markerOffsetEnd} { if offset < 0 { found++ continue } buf := make([]byte, markerLength) // Read the marker from the offset. n, err := fd.ReadAt(buf, offset) if err != nil { return found, err } if n != int(markerLength) { return found, fmt.Errorf("Only managed to read %d bytes out of %d of the %d offset marker", n, markerLength, offset) } // Check if we found it. if string(buf) == string(marker) { found++ } } return found, nil } // Write and check an initial set of markers. err = writeMarkers(fd) if err != nil { return err } found, err := checkMarkers(fd) if err != nil { return err } if found != 3 { return errors.New("Some of our initial markers weren't written properly") } // Start clearing the block. _ = fd.Close() // Attempt a secure discard run. _, err = subprocess.RunCommand("blkdiscard", "-f", "-o", fmt.Sprintf("%d", blockOffset), "-s", blockPath) if err == nil { // Check if the markers are gone. fd, err := os.Open(blockPath) if err != nil { return err } defer fd.Close() found, err = checkMarkers(fd) if err != nil { return err } if found == 0 { logger.Debug("Cleared block device", logger.Ctx{"dev": blockPath, "offset": blockOffset, "method": "secure-discard"}) // All markers are gone, secure discard succeeded. return nil } // Some markers were found, go to the next clearing option. _ = fd.Close() } // Attempt a regular discard run. _, err = subprocess.RunCommand("blkdiscard", "-f", "-o", fmt.Sprintf("%d", blockOffset), blockPath) if err == nil { // Check if the markers are gone. fd, err := os.Open(blockPath) if err != nil { return err } defer fd.Close() found, err = checkMarkers(fd) if err != nil { return err } if found == 0 { logger.Debug("Cleared block device", logger.Ctx{"dev": blockPath, "offset": blockOffset, "method": "discard"}) // All markers are gone, regular discard succeeded. return nil } // Some markers were found, go to the next clearing option. _ = fd.Close() } // Attempt device zero-ing. _, err = subprocess.RunCommand("blkdiscard", "-f", "-o", fmt.Sprintf("%d", blockOffset), "-z", blockPath) if err == nil { // Check if the markers are gone. fd, err := os.Open(blockPath) if err != nil { return err } defer fd.Close() found, err = checkMarkers(fd) if err != nil { return err } if found == 0 { logger.Debug("Cleared block device", logger.Ctx{"dev": blockPath, "offset": blockOffset, "method": "zero-discard"}) // All markers are gone, device zero-ing succeeded. return nil } // Some markers were found, go to the next clearing option. _ = fd.Close() } // All fast discard attempts have failed, proceed with manual zero-ing. zero, err := os.Open("/dev/zero") if err != nil { return err } defer zero.Close() fd, err = os.OpenFile(blockPath, os.O_WRONLY, 0o644) if err != nil { return err } defer fd.Close() _, err = fd.Seek(blockOffset, 0) if err != nil { return err } n, err := io.CopyN(fd, zero, size-blockOffset) if err != nil { return err } if n != (size - blockOffset) { return fmt.Errorf("Only managed to reset %d bytes out of %d", n, size) } logger.Debug("Cleared block device", logger.Ctx{"dev": blockPath, "offset": blockOffset, "method": "zero-overwrite"}) return nil } incus-7.0.0/internal/linux/error.go000066400000000000000000000026711517523235500173100ustar00rootroot00000000000000package linux import ( "errors" "os" "os/exec" "golang.org/x/sys/unix" ) // GetErrno checks if the Go error is a kernel errno. func GetErrno(err error) (errno error, iserrno bool) { var sysErr *os.SyscallError if errors.As(err, &sysErr) { return sysErr.Err, true } var pathErr *os.PathError if errors.As(err, &pathErr) { return pathErr.Err, true } var tmpErrno unix.Errno if errors.As(err, &tmpErrno) { return tmpErrno, true } return nil, false } // ExitStatus extracts the exit status from the error returned by exec.Cmd. // If a nil err is provided then an exit status of 0 is returned along with the nil error. // If a valid exit status can be extracted from err then it is returned along with a nil error. // If no valid exit status can be extracted then a -1 exit status is returned along with the err provided. func ExitStatus(err error) (int, error) { if err == nil { return 0, err // No error exit status. } var exitErr *exec.ExitError // Detect and extract ExitError to check the embedded exit status. if errors.As(err, &exitErr) { // If the process was signaled, extract the signal. status, isWaitStatus := exitErr.Sys().(unix.WaitStatus) if isWaitStatus && status.Signaled() { return 128 + int(status.Signal()), nil // 128 + n == Fatal error signal "n" } // Otherwise capture the exit status from the command. return exitErr.ExitCode(), nil } return -1, err // Not able to extract an exit status. } incus-7.0.0/internal/linux/filesystem.go000066400000000000000000000200761517523235500203420ustar00rootroot00000000000000//go:build linux package linux import ( "bufio" "errors" "fmt" "os" "path/filepath" "strings" "syscall" "github.com/pkg/xattr" "golang.org/x/sys/unix" ) // Filesystem magic numbers. const ( FilesystemSuperMagicZfs = 0x2fc12fc1 ) // StatVFS retrieves Virtual File System (VFS) info about a path. func StatVFS(path string) (*unix.Statfs_t, error) { var st unix.Statfs_t err := unix.Statfs(path, &st) if err != nil { return nil, err } return &st, nil } // DetectFilesystem returns the filesystem on which the passed-in path sits. func DetectFilesystem(path string) (string, error) { fs, err := StatVFS(path) if err != nil { return "", err } return FSTypeToName(int32(fs.Type)) } // IsNFS returns true if the path exists and is on a NFS mount. func IsNFS(path string) bool { backingFs, err := DetectFilesystem(path) if err != nil { return false } return backingFs == "nfs" } // FSTypeToName returns the name of the given fs type. // The fsType is from the Type field of unix.Statfs_t. We use int32 so that this function behaves the same on both // 32bit and 64bit platforms by requiring any 64bit FS types to be overflowed before being passed in. They will // then be compared with equally overflowed FS type constant values. func FSTypeToName(fsType int32) (string, error) { // This function is needed to allow FS type constants that overflow an int32 to be overflowed without a // compile error on 32bit platforms. This allows us to use any 64bit constants from the unix package on // both 64bit and 32bit platforms without having to define the constant in its rolled over form on 32bit. to32 := func(fsType int64) int32 { return int32(fsType) } switch fsType { case to32(unix.BTRFS_SUPER_MAGIC): // BTRFS' constant required overflowing to an int32. return "btrfs", nil case unix.TMPFS_MAGIC: return "tmpfs", nil case unix.EXT4_SUPER_MAGIC: return "ext4", nil case unix.XFS_SUPER_MAGIC: return "xfs", nil case unix.NFS_SUPER_MAGIC: return "nfs", nil case FilesystemSuperMagicZfs: return "zfs", nil } return fmt.Sprintf("0x%x", fsType), nil } func hasMountEntry(name string) int { // In case someone uses symlinks we need to look for the actual // mountpoint. actualPath, err := filepath.EvalSymlinks(name) if err != nil { return -1 } f, err := os.Open("/proc/self/mountinfo") if err != nil { return -1 } defer func() { _ = f.Close() }() scanner := bufio.NewScanner(f) for scanner.Scan() { line := scanner.Text() tokens := strings.Fields(line) if len(tokens) < 5 { return -1 } cleanPath := filepath.Clean(tokens[4]) if cleanPath == actualPath { return 1 } } return 0 } // IsMountPoint returns true if path is a mount point. func IsMountPoint(path string) bool { // If we find a mount entry, it is obviously a mount point. ret := hasMountEntry(path) if ret == 1 { return true } // Get the stat details. stat, err := os.Stat(path) if err != nil { return false } rootStat, err := os.Lstat(path + "/..") if err != nil { return false } // If the directory has the same device as parent, then it's not a mountpoint. if stat.Sys().(*syscall.Stat_t).Dev == rootStat.Sys().(*syscall.Stat_t).Dev { return false } // Btrfs annoyingly uses a different Dev id for different subvolumes on the same mount. // So for btrfs, we require a matching mount entry in mountinfo. fs, _ := DetectFilesystem(path) return fs != "btrfs" } // SyncFS will force a filesystem sync for the filesystem backing the provided path. func SyncFS(path string) error { // Get us a file descriptor. fsFile, err := os.Open(path) if err != nil { return err } defer func() { _ = fsFile.Close() }() // Call SyncFS. return unix.Syncfs(int(fsFile.Fd())) } // PathNameEncode encodes a path string to be used as part of a file name. // The encoding scheme replaces "-" with "--" and then "/" with "-". func PathNameEncode(text string) string { return strings.ReplaceAll(strings.ReplaceAll(text, "-", "--"), "/", "-") } // PathNameDecode decodes a string containing an encoded path back to its original form. // The decoding scheme converts "-" back to "/" and "--" back to "-". func PathNameDecode(text string) string { // This converts "--" to the null character "\0" first, to allow remaining "-" chars to be // converted back to "/" before making a final pass to convert "\0" back to original "-". return strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(text, "--", "\000"), "-", "/"), "\000", "-") } // mountOption represents an individual mount option. type mountOption struct { capture bool flag uintptr } // mountFlagTypes represents a list of possible mount flags. var mountFlagTypes = map[string]mountOption{ "async": {false, unix.MS_SYNCHRONOUS}, "atime": {false, unix.MS_NOATIME}, "bind": {true, unix.MS_BIND}, "defaults": {true, 0}, "dev": {false, unix.MS_NODEV}, "diratime": {false, unix.MS_NODIRATIME}, "dirsync": {true, unix.MS_DIRSYNC}, "exec": {false, unix.MS_NOEXEC}, "lazytime": {true, unix.MS_LAZYTIME}, "mand": {true, unix.MS_MANDLOCK}, "noatime": {true, unix.MS_NOATIME}, "nodev": {true, unix.MS_NODEV}, "nodiratime": {true, unix.MS_NODIRATIME}, "noexec": {true, unix.MS_NOEXEC}, "nomand": {false, unix.MS_MANDLOCK}, "norelatime": {false, unix.MS_RELATIME}, "nostrictatime": {false, unix.MS_STRICTATIME}, "nosuid": {true, unix.MS_NOSUID}, "rbind": {true, unix.MS_BIND | unix.MS_REC}, "relatime": {true, unix.MS_RELATIME}, "remount": {true, unix.MS_REMOUNT}, "ro": {true, unix.MS_RDONLY}, "rw": {false, unix.MS_RDONLY}, "strictatime": {true, unix.MS_STRICTATIME}, "suid": {false, unix.MS_NOSUID}, "sync": {true, unix.MS_SYNCHRONOUS}, } // ResolveMountOptions resolves the provided mount options. func ResolveMountOptions(options []string) (uintptr, string) { mountFlags := uintptr(0) var mountOptions []string for i := range options { do, ok := mountFlagTypes[options[i]] if !ok { mountOptions = append(mountOptions, options[i]) continue } if do.capture { mountFlags |= do.flag } else { mountFlags &= ^do.flag } } return mountFlags, strings.Join(mountOptions, ",") } // GetAllXattr retrieves all extended attributes associated with a file, directory or symbolic link. func GetAllXattr(path string) (map[string]string, error) { xattrNames, err := xattr.LList(path) if err != nil { // Some filesystems don't support llistxattr() for various reasons. // Interpret this as a set of no xattrs, instead of an error. if errors.Is(err, unix.EOPNOTSUPP) { return nil, nil } return nil, fmt.Errorf("Failed getting extended attributes from %q: %w", path, err) } xattrs := make(map[string]string, len(xattrNames)) for _, xattrName := range xattrNames { value, err := xattr.LGet(path, xattrName) if err != nil { return nil, fmt.Errorf("Failed getting %q extended attribute from %q: %w", xattrName, path, err) } xattrs[xattrName] = string(value) } return xattrs, nil } // IsBlockdev checks if the provided file is a block device. func IsBlockdev(fm os.FileMode) bool { return ((fm&os.ModeDevice != 0) && (fm&os.ModeCharDevice == 0)) } // IsBlockdevPath checks if the provided path is a block device. func IsBlockdevPath(pathName string) bool { sb, err := os.Stat(pathName) if err != nil { return false } fm := sb.Mode() return ((fm&os.ModeDevice != 0) && (fm&os.ModeCharDevice == 0)) } // GetMountinfo tracks down the mount entry for the path and returns all MountInfo fields. func GetMountinfo(path string) ([]string, error) { stat := &unix.Statx_t{} err := unix.Statx(0, path, 0, 0, stat) if err != nil { return nil, err } f, err := os.Open("/proc/self/mountinfo") if err != nil { return nil, err } defer func() { _ = f.Close() }() scanner := bufio.NewScanner(f) for scanner.Scan() { line := scanner.Text() tokens := strings.Fields(line) if len(tokens) < 5 { continue } if tokens[0] == fmt.Sprintf("%d", stat.Mnt_id) { return tokens, nil } } return nil, errors.New("No mountinfo entry found") } incus-7.0.0/internal/linux/ioctls.go000066400000000000000000000012121517523235500174420ustar00rootroot00000000000000package linux /* #include #include #include #define ZFS_MAX_DATASET_NAME_LEN 256 #define BLKZNAME _IOR(0x12, 125, char[ZFS_MAX_DATASET_NAME_LEN]) */ import "C" const ( // IoctlBtrfsSetReceivedSubvol matches BTRFS_IOC_SET_RECEIVED_SUBVOL. IoctlBtrfsSetReceivedSubvol = C.BTRFS_IOC_SET_RECEIVED_SUBVOL // IoctlHIDIOCGrawInfo matches HIDIOCGRAWINFO. IoctlHIDIOCGrawInfo = C.HIDIOCGRAWINFO // IoctlVhostVsockSetGuestCid matches VHOST_VSOCK_SET_GUEST_CID. IoctlVhostVsockSetGuestCid = C.VHOST_VSOCK_SET_GUEST_CID // IoctlBlkZname matches BLKZNAME (ZFS specific). IoctlBlkZname = C.BLKZNAME ) incus-7.0.0/internal/linux/kernel.go000066400000000000000000000035631517523235500174400ustar00rootroot00000000000000//go:build linux package linux import ( "fmt" "reflect" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) // LoadModule loads the kernel module with the given name, by invoking // modprobe. This respects any modprobe configuration on the system. func LoadModule(module string) error { if util.PathExists(fmt.Sprintf("/sys/module/%s", module)) { return nil } _, err := subprocess.RunCommand("modprobe", "-b", module) return err } // Utsname returns the same info as unix.Utsname, as strings. type Utsname struct { Sysname string Nodename string Release string Version string Machine string Domainname string } // Uname returns Utsname as strings. func Uname() (*Utsname, error) { /* * Based on: https://groups.google.com/forum/#!topic/golang-nuts/Jel8Bb-YwX8 * there is really no better way to do this, which is * unfortunate. Also, we ditch the more accepted CharsToString * version in that thread, since it doesn't seem as portable, * viz. github issue #206. */ uname := unix.Utsname{} err := unix.Uname(&uname) if err != nil { return nil, err } return &Utsname{ Sysname: intArrayToString(uname.Sysname), Nodename: intArrayToString(uname.Nodename), Release: intArrayToString(uname.Release), Version: intArrayToString(uname.Version), Machine: intArrayToString(uname.Machine), Domainname: intArrayToString(uname.Domainname), }, nil } func intArrayToString(arr any) string { slice := reflect.ValueOf(arr) s := "" for i := range slice.Len() { val := slice.Index(i) valInt := int64(-1) switch val.Kind() { case reflect.Int: case reflect.Int8: valInt = int64(val.Int()) case reflect.Uint: case reflect.Uint8: valInt = int64(val.Uint()) default: continue } if valInt == 0 { break } s += string(byte(valInt)) } return s } incus-7.0.0/internal/linux/memfd.go000066400000000000000000000015641517523235500172470ustar00rootroot00000000000000package linux import ( "os" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/shared/revert" ) // CreateMemfd creates a new memfd for the provided byte slice. func CreateMemfd(content []byte) (*os.File, error) { reverter := revert.New() defer reverter.Fail() // Create the memfd. fd, err := unix.MemfdCreate("memfd", unix.MFD_CLOEXEC) if err != nil { return nil, err } reverter.Add(func() { _ = unix.Close(fd) }) // Set its size. err = unix.Ftruncate(fd, int64(len(content))) if err != nil { return nil, err } // Prepare the storage. data, err := unix.Mmap(fd, 0, len(content), unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED) if err != nil { return nil, err } // Write the content. copy(data, content) // Cleanup. err = unix.Munmap(data) if err != nil { return nil, err } reverter.Success() return os.NewFile(uintptr(fd), "memfd"), nil } incus-7.0.0/internal/linux/memory.go000066400000000000000000000021001517523235500174520ustar00rootroot00000000000000package linux import ( "bufio" "fmt" "os" "strings" "github.com/lxc/incus/v7/shared/units" ) // DeviceTotalMemory returns the total amount of memory on the system (in bytes). func DeviceTotalMemory() (int64, error) { return GetMeminfo("MemTotal") } // GetMeminfo parses /proc/meminfo for the specified field. func GetMeminfo(field string) (int64, error) { // Open /proc/meminfo f, err := os.Open("/proc/meminfo") if err != nil { return -1, err } defer func() { _ = f.Close() }() // Read it line by line scan := bufio.NewScanner(f) for scan.Scan() { line := scan.Text() // We only care about MemTotal if !strings.HasPrefix(line, field+":") { continue } // Extract the before last (value) and last (unit) fields fields := strings.Split(line, " ") value := fields[len(fields)-2] + fields[len(fields)-1] // Feed the result to units.ParseByteSizeString to get an int value valueBytes, err := units.ParseByteSizeString(value) if err != nil { return -1, err } return valueBytes, nil } return -1, fmt.Errorf("Couldn't find %s", field) } incus-7.0.0/internal/linux/netlink.go000066400000000000000000000047611517523235500176250ustar00rootroot00000000000000//go:build linux package linux import ( "fmt" "net" "syscall" "unsafe" ) // NetlinkInterface returns a net.Interface extended to also contain its addresses. type NetlinkInterface struct { net.Interface Addresses []net.Addr } // NetlinkInterfaces performs a RTM_GETADDR call to get both. func NetlinkInterfaces() ([]NetlinkInterface, error) { // Grab the interface list. ifaces, err := net.Interfaces() if err != nil { return nil, err } // Initialize result slice. netlinkIfaces := make([]NetlinkInterface, 0, len(ifaces)) for _, iface := range ifaces { netlinkIfaces = append(netlinkIfaces, NetlinkInterface{iface, make([]net.Addr, 0)}) } // Turn it into a map. ifaceMap := make(map[int]*NetlinkInterface, len(ifaces)) for k, v := range netlinkIfaces { ifaceMap[v.Index] = &netlinkIfaces[k] //nolint:typecheck } // Make the netlink call. rib, err := syscall.NetlinkRIB(syscall.RTM_GETADDR, syscall.AF_UNSPEC) if err != nil { return nil, fmt.Errorf("Failed to query RTM_GETADDR: %v", err) } messages, err := syscall.ParseNetlinkMessage(rib) if err != nil { return nil, fmt.Errorf("Failed to parse RTM_GETADDR: %v", err) } for _, m := range messages { if m.Header.Type == syscall.RTM_NEWADDR { addrMessage := (*syscall.IfAddrmsg)(unsafe.Pointer(&m.Data[0])) addrAttrs, err := syscall.ParseNetlinkRouteAttr(&m) if err != nil { return nil, fmt.Errorf("Failed to parse route attribute: %v", err) } ifi, ok := ifaceMap[int(addrMessage.Index)] if ok { ifi.Addresses = append(ifi.Addresses, newAddr(addrMessage, addrAttrs)) } } } return netlinkIfaces, nil } // Variation of function of the same name from within Go source. func newAddr(ifam *syscall.IfAddrmsg, attrs []syscall.NetlinkRouteAttr) net.Addr { var ipPointToPoint bool // Seems like we need to make sure whether the IP interface // stack consists of IP point-to-point numbered or unnumbered // addressing. for _, a := range attrs { if a.Attr.Type == syscall.IFA_LOCAL { ipPointToPoint = true break } } for _, a := range attrs { if ipPointToPoint && a.Attr.Type == syscall.IFA_ADDRESS { continue } switch ifam.Family { case syscall.AF_INET: return &net.IPNet{IP: net.IPv4(a.Value[0], a.Value[1], a.Value[2], a.Value[3]), Mask: net.CIDRMask(int(ifam.Prefixlen), 8*net.IPv4len)} case syscall.AF_INET6: ifa := &net.IPNet{IP: make(net.IP, net.IPv6len), Mask: net.CIDRMask(int(ifam.Prefixlen), 8*net.IPv6len)} copy(ifa.IP, a.Value[:]) return ifa } } return nil } incus-7.0.0/internal/linux/poll.go000066400000000000000000000063321517523235500171230ustar00rootroot00000000000000package linux import ( "context" "errors" "io" "os" "time" "golang.org/x/sys/unix" ) // GetPollRevents poll for events on provided fd. func GetPollRevents(fd int, timeout int, flags int) (int, int, error) { pollFd := unix.PollFd{ Fd: int32(fd), Events: int16(flags), Revents: 0, } pollFds := []unix.PollFd{pollFd} again: n, err := unix.Poll(pollFds, timeout) if err != nil { if errors.Is(err, unix.EAGAIN) || errors.Is(err, unix.EINTR) { goto again } return -1, -1, err } return n, int(pollFds[0].Revents), err } // NewExecWrapper returns a new ReadWriteCloser wrapper for an os.File. // The ctx is used to indicate when the executed process has ended, at which point any further Read calls will // return io.EOF rather than potentially blocking on the poll syscall if the process is a shell that still has // background processes running that are not producing any output. func NewExecWrapper(ctx context.Context, f *os.File) io.ReadWriteCloser { return &execWrapper{ ctx: ctx, f: f, } } // execWrapper implements a ReadWriteCloser wrapper for an os.File connected to a PTY. type execWrapper struct { f *os.File ctx context.Context finishDeadline time.Time } // Read uses the poll syscall with a timeout of 1s to check if there is any data to read. // This avoids potentially blocking in the poll syscall in situations where the process is a shell that has // background processes that are not producing any output. // If the ctx has been cancelled before the poll starts then io.EOF error is returned. func (w *execWrapper) Read(p []byte) (int, error) { rawConn, err := w.f.SyscallConn() if err != nil { return 0, err } var opErr error var n int err = rawConn.Read(func(fd uintptr) bool { for { // Call poll() with 1s timeout, this prevents blocking if a shell process exits leaving // background processes running that are not outputting anything. _, revents, err := GetPollRevents(int(fd), 1000, (unix.POLLIN | unix.POLLPRI | unix.POLLERR | unix.POLLNVAL | unix.POLLHUP | unix.POLLRDHUP)) switch { case err != nil: opErr = err case revents&unix.POLLERR > 0: opErr = errors.New("Got POLLERR event") case revents&unix.POLLNVAL > 0: opErr = errors.New("Got POLLNVAL event") case revents&(unix.POLLIN|unix.POLLPRI) > 0: // If there is something to read then read it. n, opErr = unix.Read(int(fd), p) if opErr == nil && w.ctx.Err() != nil { if w.finishDeadline.IsZero() { // When the parent process finishes set a deadline to complete // future reads by. w.finishDeadline = time.Now().Add(time.Second) } else if time.Now().After(w.finishDeadline) { // If there is still output being received after the parent // process has finished then return EOF to prevent background // processes from keeping the reads ongoing. opErr = io.EOF } } case w.ctx.Err() != nil: // Nothing to read after process exited then return EOF. opErr = io.EOF default: continue } return true } }) if err != nil { return n, err } return n, opErr } func (w *execWrapper) Write(p []byte) (int, error) { return w.f.Write(p) } func (w *execWrapper) Close() error { return w.f.Close() } incus-7.0.0/internal/linux/pty.go000066400000000000000000000070431517523235500167710ustar00rootroot00000000000000package linux import ( "errors" "fmt" "os" "unsafe" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/shared/revert" ) // OpenPtyInDevpts creates a new PTS pair, configures them and returns them. func OpenPtyInDevpts(devpts_fd int, uid, gid int64) (*os.File, *os.File, error) { reverter := revert.New() defer reverter.Fail() var fd int var ptx *os.File var err error // Create a PTS pair. if devpts_fd >= 0 { fd, err = unix.Openat(devpts_fd, "ptmx", unix.O_RDWR|unix.O_CLOEXEC|unix.O_NOCTTY, 0) } else { fd, err = unix.Openat(-1, "/dev/ptmx", unix.O_RDWR|unix.O_CLOEXEC|unix.O_NOCTTY, 0) } if err != nil { return nil, nil, err } ptx = os.NewFile(uintptr(fd), "/dev/pts/ptmx") reverter.Add(func() { _ = ptx.Close() }) // Unlock the ptx and pty. val := 0 _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(ptx.Fd()), unix.TIOCSPTLCK, uintptr(unsafe.Pointer(&val))) if errno != 0 { return nil, nil, unix.Errno(errno) } var pty *os.File ptyFd, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(ptx.Fd()), unix.TIOCGPTPEER, uintptr(unix.O_NOCTTY|unix.O_CLOEXEC|os.O_RDWR)) // We can only fallback to looking up the fd in /dev/pts when we aren't dealing with the container's devpts instance. if errno == 0 { // Get the pty side. id := 0 _, _, errno = unix.Syscall(unix.SYS_IOCTL, uintptr(ptx.Fd()), unix.TIOCGPTN, uintptr(unsafe.Pointer(&id))) if errno != 0 { return nil, nil, unix.Errno(errno) } pty = os.NewFile(ptyFd, fmt.Sprintf("/dev/pts/%d", id)) } else { if devpts_fd >= 0 { return nil, nil, errors.New("TIOCGPTPEER required but not available") } // Get the pty side. id := 0 _, _, errno = unix.Syscall(unix.SYS_IOCTL, uintptr(ptx.Fd()), unix.TIOCGPTN, uintptr(unsafe.Pointer(&id))) if errno != 0 { return nil, nil, unix.Errno(errno) } // Open the pty. pty, err = os.OpenFile(fmt.Sprintf("/dev/pts/%d", id), unix.O_NOCTTY|unix.O_CLOEXEC|os.O_RDWR, 0) if err != nil { return nil, nil, err } } reverter.Add(func() { _ = pty.Close() }) // Configure both sides for _, entry := range []*os.File{ptx, pty} { // Get termios. t, err := unix.IoctlGetTermios(int(entry.Fd()), unix.TCGETS) if err != nil { return nil, nil, err } // Set flags. t.Cflag |= unix.IMAXBEL t.Cflag |= unix.IUTF8 t.Cflag |= unix.BRKINT t.Cflag |= unix.IXANY t.Cflag |= unix.HUPCL // Set termios. err = unix.IoctlSetTermios(int(entry.Fd()), unix.TCSETS, t) if err != nil { return nil, nil, err } // Set the default window size. sz := &unix.Winsize{ Col: 80, Row: 25, } err = unix.IoctlSetWinsize(int(entry.Fd()), unix.TIOCSWINSZ, sz) if err != nil { return nil, nil, err } // Set CLOEXEC. _, _, errno = unix.Syscall(unix.SYS_FCNTL, uintptr(entry.Fd()), unix.F_SETFD, unix.FD_CLOEXEC) if errno != 0 { return nil, nil, unix.Errno(errno) } } // Fix the ownership of the pty side. err = unix.Fchown(int(pty.Fd()), int(uid), int(gid)) if err != nil { return nil, nil, err } reverter.Success() return ptx, pty, nil } // OpenPty creates a new PTS pair, configures them and returns them. func OpenPty(uid, gid int64) (*os.File, *os.File, error) { return OpenPtyInDevpts(-1, uid, gid) } // SetPtySize issues the correct ioctl to resize a pty. func SetPtySize(fd int, width int, height int) (err error) { var dimensions [4]uint16 dimensions[0] = uint16(height) dimensions[1] = uint16(width) _, _, errno := unix.Syscall6(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.TIOCSWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0) if errno != 0 { return errno } return nil } incus-7.0.0/internal/linux/socket_linux_cgo.go000066400000000000000000000044601517523235500215140ustar00rootroot00000000000000//go:build linux && cgo package linux /* #ifndef _GNU_SOURCE #define _GNU_SOURCE 1 #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../../shared/cgo/process_utils.h" #include "../../shared/cgo/syscall_wrappers.h" #define ABSTRACT_UNIX_SOCK_LEN sizeof(((struct sockaddr_un *)0)->sun_path) static int read_pid(int fd) { ssize_t ret; pid_t n = -1; again: ret = read(fd, &n, sizeof(n)); if (ret < 0 && errno == EINTR) goto again; if (ret < 0) return -1; return n; } */ import "C" import ( "fmt" "os" "golang.org/x/sys/unix" _ "github.com/lxc/incus/v7/shared/cgo" // Used by cgo ) const ABSTRACT_UNIX_SOCK_LEN int = C.ABSTRACT_UNIX_SOCK_LEN func ReadPid(r *os.File) int { return int(C.read_pid(C.int(r.Fd()))) } func unCloexec(fd int) error { var err error = nil flags, _, errno := unix.Syscall(unix.SYS_FCNTL, uintptr(fd), unix.F_GETFD, 0) if errno != 0 { err = errno return err } flags &^= unix.FD_CLOEXEC _, _, errno = unix.Syscall(unix.SYS_FCNTL, uintptr(fd), unix.F_SETFD, flags) if errno != 0 { err = errno } return err } func PidFdOpen(Pid int, Flags uint32) (*os.File, error) { pidFd, errno := C.incus_pidfd_open(C.int(Pid), C.uint32_t(Flags)) if errno != nil { return nil, errno } errno = unCloexec(int(pidFd)) if errno != nil { return nil, errno } return os.NewFile(uintptr(pidFd), fmt.Sprintf("%d", Pid)), nil } func PidfdSendSignal(Pidfd int, Signal int, Flags uint32) error { ret, errno := C.incus_pidfd_send_signal(C.int(Pidfd), C.int(Signal), nil, C.uint32_t(Flags)) if ret != 0 { return errno } return nil } const ( // CLOSE_RANGE_UNSHARE matches CLOSE_RANGE_UNSHARE flag. CLOSE_RANGE_UNSHARE uint32 = C.CLOSE_RANGE_UNSHARE // CLOSE_RANGE_CLOEXEC matches CLOSE_RANGE_CLOEXEC flag. CLOSE_RANGE_CLOEXEC uint32 = C.CLOSE_RANGE_CLOEXEC ) func CloseRange(FirstFd uint32, LastFd uint32, Flags uint32) error { ret, errno := C.incus_close_range(C.uint32_t(FirstFd), C.uint32_t(LastFd), C.uint32_t(Flags)) if ret != 0 { if errno != unix.ENOSYS && errno != unix.EINVAL { return errno } } return nil } incus-7.0.0/internal/linux/socket_linux_notcgo.go000066400000000000000000000001201517523235500222220ustar00rootroot00000000000000//go:build linux && !cgo package linux const ABSTRACT_UNIX_SOCK_LEN int = 107 incus-7.0.0/internal/linux/storage.go000066400000000000000000000021501517523235500176130ustar00rootroot00000000000000//go:build linux package linux import ( "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" ) // AvailableStorageDrivers returns a list of storage drivers that are available. func AvailableStorageDrivers(path string, supportedDrivers []api.ServerStorageDriverInfo, poolType util.PoolType) []string { backingFs, err := DetectFilesystem(path) if err != nil { backingFs = "dir" } drivers := make([]string, 0, len(supportedDrivers)) // Check available backends. for _, driver := range supportedDrivers { if poolType == util.PoolTypeRemote && !driver.Remote { continue } if poolType == util.PoolTypeLocal && driver.Remote { continue } if poolType == util.PoolTypeAny && (driver.Name == "cephfs" || driver.Name == "cephobject") { continue } if driver.Name == "dir" { drivers = append(drivers, driver.Name) continue } // btrfs can work in user namespaces too. (If source=/some/path/on/btrfs is used.) if RunningInUserNS() && (backingFs != "btrfs" || driver.Name != "btrfs") { continue } drivers = append(drivers, driver.Name) } return drivers } incus-7.0.0/internal/linux/systemd.go000066400000000000000000000024671517523235500176520ustar00rootroot00000000000000package linux import ( "fmt" "net" "os" "strconv" "golang.org/x/sys/unix" ) // GetSystemdListeners returns the socket-activated network listeners, if any. // // The 'start' parameter must be SystemdListenFDsStart, except in unit tests, // see the docstring of SystemdListenFDsStart below. func GetSystemdListeners(start int) []net.Listener { defer func() { _ = os.Unsetenv("LISTEN_PID") _ = os.Unsetenv("LISTEN_FDS") }() pid, err := strconv.Atoi(os.Getenv("LISTEN_PID")) if err != nil { return nil } if pid != os.Getpid() { return nil } fds, err := strconv.Atoi(os.Getenv("LISTEN_FDS")) if err != nil { return nil } listeners := []net.Listener{} for i := start; i < start+fds; i++ { unix.CloseOnExec(i) file := os.NewFile(uintptr(i), fmt.Sprintf("inherited-fd%d", i)) listener, err := net.FileListener(file) if err != nil { continue } listeners = append(listeners, listener) } return listeners } // SystemdListenFDsStart is the number of the first file descriptor that might // have been opened by systemd when socket activation is enabled. It's always 3 // in real-world usage (i.e. the first file descriptor opened after stdin, // stdout and stderr), so this constant should always be the value passed to // GetListeners, except for unit tests. const SystemdListenFDsStart = 3 incus-7.0.0/internal/linux/ucred.go000066400000000000000000000010451517523235500172530ustar00rootroot00000000000000package linux import ( "net" "golang.org/x/sys/unix" ) // GetUcred returns the credentials from the remote end of a unix socket. func GetUcred(conn *net.UnixConn) (*unix.Ucred, error) { rawConn, err := conn.SyscallConn() if err != nil { return nil, err } var ucred *unix.Ucred var ucredErr error err = rawConn.Control(func(fd uintptr) { ucred, ucredErr = unix.GetsockoptUcred(int(fd), unix.SOL_SOCKET, unix.SO_PEERCRED) }) if err != nil { return nil, err } if ucredErr != nil { return nil, ucredErr } return ucred, nil } incus-7.0.0/internal/linux/userns.go000066400000000000000000000007071517523235500174740ustar00rootroot00000000000000package linux import ( "bufio" "fmt" "os" ) func RunningInUserNS() bool { file, err := os.Open("/proc/self/uid_map") if err != nil { return false } defer func() { _ = file.Close() }() buf := bufio.NewReader(file) l, _, err := buf.ReadLine() if err != nil { return false } line := string(l) var a, b, c int64 _, _ = fmt.Sscanf(line, "%d %d %d", &a, &b, &c) if a == 0 && b == 0 && c == 4294967295 { return false } return true } incus-7.0.0/internal/migration/000077500000000000000000000000001517523235500164545ustar00rootroot00000000000000incus-7.0.0/internal/migration/migrate.pb.go000066400000000000000000001056611517523235500210440ustar00rootroot00000000000000// silence the protobuf compiler warning by setting the default // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 // protoc v3.12.4 // source: internal/migration/migrate.proto package migration import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" unsafe "unsafe" ) const ( // Verify that this generated code is sufficiently up-to-date. _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) // Verify that runtime/protoimpl is sufficiently up-to-date. _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) type MigrationFSType int32 const ( MigrationFSType_RSYNC MigrationFSType = 0 MigrationFSType_BTRFS MigrationFSType = 1 MigrationFSType_ZFS MigrationFSType = 2 MigrationFSType_RBD MigrationFSType = 3 MigrationFSType_BLOCK_AND_RSYNC MigrationFSType = 4 MigrationFSType_LINSTOR MigrationFSType = 5 ) // Enum value maps for MigrationFSType. var ( MigrationFSType_name = map[int32]string{ 0: "RSYNC", 1: "BTRFS", 2: "ZFS", 3: "RBD", 4: "BLOCK_AND_RSYNC", 5: "LINSTOR", } MigrationFSType_value = map[string]int32{ "RSYNC": 0, "BTRFS": 1, "ZFS": 2, "RBD": 3, "BLOCK_AND_RSYNC": 4, "LINSTOR": 5, } ) func (x MigrationFSType) Enum() *MigrationFSType { p := new(MigrationFSType) *p = x return p } func (x MigrationFSType) String() string { return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } func (MigrationFSType) Descriptor() protoreflect.EnumDescriptor { return file_internal_migration_migrate_proto_enumTypes[0].Descriptor() } func (MigrationFSType) Type() protoreflect.EnumType { return &file_internal_migration_migrate_proto_enumTypes[0] } func (x MigrationFSType) Number() protoreflect.EnumNumber { return protoreflect.EnumNumber(x) } // Deprecated: Do not use. func (x *MigrationFSType) UnmarshalJSON(b []byte) error { num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) if err != nil { return err } *x = MigrationFSType(num) return nil } // Deprecated: Use MigrationFSType.Descriptor instead. func (MigrationFSType) EnumDescriptor() ([]byte, []int) { return file_internal_migration_migrate_proto_rawDescGZIP(), []int{0} } type CRIUType int32 const ( CRIUType_CRIU_RSYNC CRIUType = 0 CRIUType_PHAUL CRIUType = 1 CRIUType_NONE CRIUType = 2 CRIUType_VM_QEMU CRIUType = 3 ) // Enum value maps for CRIUType. var ( CRIUType_name = map[int32]string{ 0: "CRIU_RSYNC", 1: "PHAUL", 2: "NONE", 3: "VM_QEMU", } CRIUType_value = map[string]int32{ "CRIU_RSYNC": 0, "PHAUL": 1, "NONE": 2, "VM_QEMU": 3, } ) func (x CRIUType) Enum() *CRIUType { p := new(CRIUType) *p = x return p } func (x CRIUType) String() string { return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } func (CRIUType) Descriptor() protoreflect.EnumDescriptor { return file_internal_migration_migrate_proto_enumTypes[1].Descriptor() } func (CRIUType) Type() protoreflect.EnumType { return &file_internal_migration_migrate_proto_enumTypes[1] } func (x CRIUType) Number() protoreflect.EnumNumber { return protoreflect.EnumNumber(x) } // Deprecated: Do not use. func (x *CRIUType) UnmarshalJSON(b []byte) error { num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) if err != nil { return err } *x = CRIUType(num) return nil } // Deprecated: Use CRIUType.Descriptor instead. func (CRIUType) EnumDescriptor() ([]byte, []int) { return file_internal_migration_migrate_proto_rawDescGZIP(), []int{1} } type IDMapType struct { state protoimpl.MessageState `protogen:"open.v1"` Isuid *bool `protobuf:"varint,1,req,name=isuid" json:"isuid,omitempty"` Isgid *bool `protobuf:"varint,2,req,name=isgid" json:"isgid,omitempty"` Hostid *int32 `protobuf:"varint,3,req,name=hostid" json:"hostid,omitempty"` Nsid *int32 `protobuf:"varint,4,req,name=nsid" json:"nsid,omitempty"` Maprange *int32 `protobuf:"varint,5,req,name=maprange" json:"maprange,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *IDMapType) Reset() { *x = IDMapType{} mi := &file_internal_migration_migrate_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *IDMapType) String() string { return protoimpl.X.MessageStringOf(x) } func (*IDMapType) ProtoMessage() {} func (x *IDMapType) ProtoReflect() protoreflect.Message { mi := &file_internal_migration_migrate_proto_msgTypes[0] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use IDMapType.ProtoReflect.Descriptor instead. func (*IDMapType) Descriptor() ([]byte, []int) { return file_internal_migration_migrate_proto_rawDescGZIP(), []int{0} } func (x *IDMapType) GetIsuid() bool { if x != nil && x.Isuid != nil { return *x.Isuid } return false } func (x *IDMapType) GetIsgid() bool { if x != nil && x.Isgid != nil { return *x.Isgid } return false } func (x *IDMapType) GetHostid() int32 { if x != nil && x.Hostid != nil { return *x.Hostid } return 0 } func (x *IDMapType) GetNsid() int32 { if x != nil && x.Nsid != nil { return *x.Nsid } return 0 } func (x *IDMapType) GetMaprange() int32 { if x != nil && x.Maprange != nil { return *x.Maprange } return 0 } type Config struct { state protoimpl.MessageState `protogen:"open.v1"` Key *string `protobuf:"bytes,1,req,name=key" json:"key,omitempty"` Value *string `protobuf:"bytes,2,req,name=value" json:"value,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *Config) Reset() { *x = Config{} mi := &file_internal_migration_migrate_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *Config) String() string { return protoimpl.X.MessageStringOf(x) } func (*Config) ProtoMessage() {} func (x *Config) ProtoReflect() protoreflect.Message { mi := &file_internal_migration_migrate_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use Config.ProtoReflect.Descriptor instead. func (*Config) Descriptor() ([]byte, []int) { return file_internal_migration_migrate_proto_rawDescGZIP(), []int{1} } func (x *Config) GetKey() string { if x != nil && x.Key != nil { return *x.Key } return "" } func (x *Config) GetValue() string { if x != nil && x.Value != nil { return *x.Value } return "" } type Device struct { state protoimpl.MessageState `protogen:"open.v1"` Name *string `protobuf:"bytes,1,req,name=name" json:"name,omitempty"` Config []*Config `protobuf:"bytes,2,rep,name=config" json:"config,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *Device) Reset() { *x = Device{} mi := &file_internal_migration_migrate_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *Device) String() string { return protoimpl.X.MessageStringOf(x) } func (*Device) ProtoMessage() {} func (x *Device) ProtoReflect() protoreflect.Message { mi := &file_internal_migration_migrate_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use Device.ProtoReflect.Descriptor instead. func (*Device) Descriptor() ([]byte, []int) { return file_internal_migration_migrate_proto_rawDescGZIP(), []int{2} } func (x *Device) GetName() string { if x != nil && x.Name != nil { return *x.Name } return "" } func (x *Device) GetConfig() []*Config { if x != nil { return x.Config } return nil } type Snapshot struct { state protoimpl.MessageState `protogen:"open.v1"` Name *string `protobuf:"bytes,1,req,name=name" json:"name,omitempty"` LocalConfig []*Config `protobuf:"bytes,2,rep,name=localConfig" json:"localConfig,omitempty"` Profiles []string `protobuf:"bytes,3,rep,name=profiles" json:"profiles,omitempty"` Ephemeral *bool `protobuf:"varint,4,req,name=ephemeral" json:"ephemeral,omitempty"` LocalDevices []*Device `protobuf:"bytes,5,rep,name=localDevices" json:"localDevices,omitempty"` Architecture *int32 `protobuf:"varint,6,req,name=architecture" json:"architecture,omitempty"` Stateful *bool `protobuf:"varint,7,req,name=stateful" json:"stateful,omitempty"` CreationDate *int64 `protobuf:"varint,8,opt,name=creation_date,json=creationDate" json:"creation_date,omitempty"` LastUsedDate *int64 `protobuf:"varint,9,opt,name=last_used_date,json=lastUsedDate" json:"last_used_date,omitempty"` ExpiryDate *int64 `protobuf:"varint,10,opt,name=expiry_date,json=expiryDate" json:"expiry_date,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *Snapshot) Reset() { *x = Snapshot{} mi := &file_internal_migration_migrate_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *Snapshot) String() string { return protoimpl.X.MessageStringOf(x) } func (*Snapshot) ProtoMessage() {} func (x *Snapshot) ProtoReflect() protoreflect.Message { mi := &file_internal_migration_migrate_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use Snapshot.ProtoReflect.Descriptor instead. func (*Snapshot) Descriptor() ([]byte, []int) { return file_internal_migration_migrate_proto_rawDescGZIP(), []int{3} } func (x *Snapshot) GetName() string { if x != nil && x.Name != nil { return *x.Name } return "" } func (x *Snapshot) GetLocalConfig() []*Config { if x != nil { return x.LocalConfig } return nil } func (x *Snapshot) GetProfiles() []string { if x != nil { return x.Profiles } return nil } func (x *Snapshot) GetEphemeral() bool { if x != nil && x.Ephemeral != nil { return *x.Ephemeral } return false } func (x *Snapshot) GetLocalDevices() []*Device { if x != nil { return x.LocalDevices } return nil } func (x *Snapshot) GetArchitecture() int32 { if x != nil && x.Architecture != nil { return *x.Architecture } return 0 } func (x *Snapshot) GetStateful() bool { if x != nil && x.Stateful != nil { return *x.Stateful } return false } func (x *Snapshot) GetCreationDate() int64 { if x != nil && x.CreationDate != nil { return *x.CreationDate } return 0 } func (x *Snapshot) GetLastUsedDate() int64 { if x != nil && x.LastUsedDate != nil { return *x.LastUsedDate } return 0 } func (x *Snapshot) GetExpiryDate() int64 { if x != nil && x.ExpiryDate != nil { return *x.ExpiryDate } return 0 } type RsyncFeatures struct { state protoimpl.MessageState `protogen:"open.v1"` Xattrs *bool `protobuf:"varint,1,opt,name=xattrs" json:"xattrs,omitempty"` Delete *bool `protobuf:"varint,2,opt,name=delete" json:"delete,omitempty"` Compress *bool `protobuf:"varint,3,opt,name=compress" json:"compress,omitempty"` Bidirectional *bool `protobuf:"varint,4,opt,name=bidirectional" json:"bidirectional,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *RsyncFeatures) Reset() { *x = RsyncFeatures{} mi := &file_internal_migration_migrate_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *RsyncFeatures) String() string { return protoimpl.X.MessageStringOf(x) } func (*RsyncFeatures) ProtoMessage() {} func (x *RsyncFeatures) ProtoReflect() protoreflect.Message { mi := &file_internal_migration_migrate_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use RsyncFeatures.ProtoReflect.Descriptor instead. func (*RsyncFeatures) Descriptor() ([]byte, []int) { return file_internal_migration_migrate_proto_rawDescGZIP(), []int{4} } func (x *RsyncFeatures) GetXattrs() bool { if x != nil && x.Xattrs != nil { return *x.Xattrs } return false } func (x *RsyncFeatures) GetDelete() bool { if x != nil && x.Delete != nil { return *x.Delete } return false } func (x *RsyncFeatures) GetCompress() bool { if x != nil && x.Compress != nil { return *x.Compress } return false } func (x *RsyncFeatures) GetBidirectional() bool { if x != nil && x.Bidirectional != nil { return *x.Bidirectional } return false } type ZfsFeatures struct { state protoimpl.MessageState `protogen:"open.v1"` Compress *bool `protobuf:"varint,1,opt,name=compress" json:"compress,omitempty"` MigrationHeader *bool `protobuf:"varint,2,opt,name=migration_header,json=migrationHeader" json:"migration_header,omitempty"` HeaderZvols *bool `protobuf:"varint,3,opt,name=header_zvols,json=headerZvols" json:"header_zvols,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *ZfsFeatures) Reset() { *x = ZfsFeatures{} mi := &file_internal_migration_migrate_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *ZfsFeatures) String() string { return protoimpl.X.MessageStringOf(x) } func (*ZfsFeatures) ProtoMessage() {} func (x *ZfsFeatures) ProtoReflect() protoreflect.Message { mi := &file_internal_migration_migrate_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use ZfsFeatures.ProtoReflect.Descriptor instead. func (*ZfsFeatures) Descriptor() ([]byte, []int) { return file_internal_migration_migrate_proto_rawDescGZIP(), []int{5} } func (x *ZfsFeatures) GetCompress() bool { if x != nil && x.Compress != nil { return *x.Compress } return false } func (x *ZfsFeatures) GetMigrationHeader() bool { if x != nil && x.MigrationHeader != nil { return *x.MigrationHeader } return false } func (x *ZfsFeatures) GetHeaderZvols() bool { if x != nil && x.HeaderZvols != nil { return *x.HeaderZvols } return false } type BtrfsFeatures struct { state protoimpl.MessageState `protogen:"open.v1"` MigrationHeader *bool `protobuf:"varint,1,opt,name=migration_header,json=migrationHeader" json:"migration_header,omitempty"` HeaderSubvolumes *bool `protobuf:"varint,2,opt,name=header_subvolumes,json=headerSubvolumes" json:"header_subvolumes,omitempty"` HeaderSubvolumeUuids *bool `protobuf:"varint,3,opt,name=header_subvolume_uuids,json=headerSubvolumeUuids" json:"header_subvolume_uuids,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *BtrfsFeatures) Reset() { *x = BtrfsFeatures{} mi := &file_internal_migration_migrate_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *BtrfsFeatures) String() string { return protoimpl.X.MessageStringOf(x) } func (*BtrfsFeatures) ProtoMessage() {} func (x *BtrfsFeatures) ProtoReflect() protoreflect.Message { mi := &file_internal_migration_migrate_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use BtrfsFeatures.ProtoReflect.Descriptor instead. func (*BtrfsFeatures) Descriptor() ([]byte, []int) { return file_internal_migration_migrate_proto_rawDescGZIP(), []int{6} } func (x *BtrfsFeatures) GetMigrationHeader() bool { if x != nil && x.MigrationHeader != nil { return *x.MigrationHeader } return false } func (x *BtrfsFeatures) GetHeaderSubvolumes() bool { if x != nil && x.HeaderSubvolumes != nil { return *x.HeaderSubvolumes } return false } func (x *BtrfsFeatures) GetHeaderSubvolumeUuids() bool { if x != nil && x.HeaderSubvolumeUuids != nil { return *x.HeaderSubvolumeUuids } return false } type DependentVolume struct { state protoimpl.MessageState `protogen:"open.v1"` Name *string `protobuf:"bytes,1,req,name=name" json:"name,omitempty"` Pool *string `protobuf:"bytes,2,req,name=pool" json:"pool,omitempty"` ContentType *string `protobuf:"bytes,3,req,name=contentType" json:"contentType,omitempty"` Fs *MigrationFSType `protobuf:"varint,4,req,name=fs,enum=migration.MigrationFSType" json:"fs,omitempty"` RsyncFeatures *RsyncFeatures `protobuf:"bytes,5,opt,name=rsyncFeatures" json:"rsyncFeatures,omitempty"` ZfsFeatures *ZfsFeatures `protobuf:"bytes,6,opt,name=zfsFeatures" json:"zfsFeatures,omitempty"` BtrfsFeatures *BtrfsFeatures `protobuf:"bytes,7,opt,name=btrfsFeatures" json:"btrfsFeatures,omitempty"` VolumeSize *int64 `protobuf:"varint,8,opt,name=volumeSize" json:"volumeSize,omitempty"` Snapshots []*Snapshot `protobuf:"bytes,9,rep,name=snapshots" json:"snapshots,omitempty"` DeviceName *string `protobuf:"bytes,10,opt,name=deviceName" json:"deviceName,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *DependentVolume) Reset() { *x = DependentVolume{} mi := &file_internal_migration_migrate_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *DependentVolume) String() string { return protoimpl.X.MessageStringOf(x) } func (*DependentVolume) ProtoMessage() {} func (x *DependentVolume) ProtoReflect() protoreflect.Message { mi := &file_internal_migration_migrate_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use DependentVolume.ProtoReflect.Descriptor instead. func (*DependentVolume) Descriptor() ([]byte, []int) { return file_internal_migration_migrate_proto_rawDescGZIP(), []int{7} } func (x *DependentVolume) GetName() string { if x != nil && x.Name != nil { return *x.Name } return "" } func (x *DependentVolume) GetPool() string { if x != nil && x.Pool != nil { return *x.Pool } return "" } func (x *DependentVolume) GetContentType() string { if x != nil && x.ContentType != nil { return *x.ContentType } return "" } func (x *DependentVolume) GetFs() MigrationFSType { if x != nil && x.Fs != nil { return *x.Fs } return MigrationFSType_RSYNC } func (x *DependentVolume) GetRsyncFeatures() *RsyncFeatures { if x != nil { return x.RsyncFeatures } return nil } func (x *DependentVolume) GetZfsFeatures() *ZfsFeatures { if x != nil { return x.ZfsFeatures } return nil } func (x *DependentVolume) GetBtrfsFeatures() *BtrfsFeatures { if x != nil { return x.BtrfsFeatures } return nil } func (x *DependentVolume) GetVolumeSize() int64 { if x != nil && x.VolumeSize != nil { return *x.VolumeSize } return 0 } func (x *DependentVolume) GetSnapshots() []*Snapshot { if x != nil { return x.Snapshots } return nil } func (x *DependentVolume) GetDeviceName() string { if x != nil && x.DeviceName != nil { return *x.DeviceName } return "" } type MigrationHeader struct { state protoimpl.MessageState `protogen:"open.v1"` Fs *MigrationFSType `protobuf:"varint,1,req,name=fs,enum=migration.MigrationFSType" json:"fs,omitempty"` Criu *CRIUType `protobuf:"varint,2,opt,name=criu,enum=migration.CRIUType" json:"criu,omitempty"` Idmap []*IDMapType `protobuf:"bytes,3,rep,name=idmap" json:"idmap,omitempty"` SnapshotNames []string `protobuf:"bytes,4,rep,name=snapshotNames" json:"snapshotNames,omitempty"` Snapshots []*Snapshot `protobuf:"bytes,5,rep,name=snapshots" json:"snapshots,omitempty"` Predump *bool `protobuf:"varint,7,opt,name=predump" json:"predump,omitempty"` RsyncFeatures *RsyncFeatures `protobuf:"bytes,8,opt,name=rsyncFeatures" json:"rsyncFeatures,omitempty"` Refresh *bool `protobuf:"varint,9,opt,name=refresh" json:"refresh,omitempty"` ZfsFeatures *ZfsFeatures `protobuf:"bytes,10,opt,name=zfsFeatures" json:"zfsFeatures,omitempty"` VolumeSize *int64 `protobuf:"varint,11,opt,name=volumeSize" json:"volumeSize,omitempty"` BtrfsFeatures *BtrfsFeatures `protobuf:"bytes,12,opt,name=btrfsFeatures" json:"btrfsFeatures,omitempty"` IndexHeaderVersion *uint32 `protobuf:"varint,13,opt,name=indexHeaderVersion" json:"indexHeaderVersion,omitempty"` DependentVolumes []*DependentVolume `protobuf:"bytes,14,rep,name=dependentVolumes" json:"dependentVolumes,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *MigrationHeader) Reset() { *x = MigrationHeader{} mi := &file_internal_migration_migrate_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *MigrationHeader) String() string { return protoimpl.X.MessageStringOf(x) } func (*MigrationHeader) ProtoMessage() {} func (x *MigrationHeader) ProtoReflect() protoreflect.Message { mi := &file_internal_migration_migrate_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use MigrationHeader.ProtoReflect.Descriptor instead. func (*MigrationHeader) Descriptor() ([]byte, []int) { return file_internal_migration_migrate_proto_rawDescGZIP(), []int{8} } func (x *MigrationHeader) GetFs() MigrationFSType { if x != nil && x.Fs != nil { return *x.Fs } return MigrationFSType_RSYNC } func (x *MigrationHeader) GetCriu() CRIUType { if x != nil && x.Criu != nil { return *x.Criu } return CRIUType_CRIU_RSYNC } func (x *MigrationHeader) GetIdmap() []*IDMapType { if x != nil { return x.Idmap } return nil } func (x *MigrationHeader) GetSnapshotNames() []string { if x != nil { return x.SnapshotNames } return nil } func (x *MigrationHeader) GetSnapshots() []*Snapshot { if x != nil { return x.Snapshots } return nil } func (x *MigrationHeader) GetPredump() bool { if x != nil && x.Predump != nil { return *x.Predump } return false } func (x *MigrationHeader) GetRsyncFeatures() *RsyncFeatures { if x != nil { return x.RsyncFeatures } return nil } func (x *MigrationHeader) GetRefresh() bool { if x != nil && x.Refresh != nil { return *x.Refresh } return false } func (x *MigrationHeader) GetZfsFeatures() *ZfsFeatures { if x != nil { return x.ZfsFeatures } return nil } func (x *MigrationHeader) GetVolumeSize() int64 { if x != nil && x.VolumeSize != nil { return *x.VolumeSize } return 0 } func (x *MigrationHeader) GetBtrfsFeatures() *BtrfsFeatures { if x != nil { return x.BtrfsFeatures } return nil } func (x *MigrationHeader) GetIndexHeaderVersion() uint32 { if x != nil && x.IndexHeaderVersion != nil { return *x.IndexHeaderVersion } return 0 } func (x *MigrationHeader) GetDependentVolumes() []*DependentVolume { if x != nil { return x.DependentVolumes } return nil } type MigrationControl struct { state protoimpl.MessageState `protogen:"open.v1"` Success *bool `protobuf:"varint,1,req,name=success" json:"success,omitempty"` // optional failure message if sending a failure Message *string `protobuf:"bytes,2,opt,name=message" json:"message,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *MigrationControl) Reset() { *x = MigrationControl{} mi := &file_internal_migration_migrate_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *MigrationControl) String() string { return protoimpl.X.MessageStringOf(x) } func (*MigrationControl) ProtoMessage() {} func (x *MigrationControl) ProtoReflect() protoreflect.Message { mi := &file_internal_migration_migrate_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use MigrationControl.ProtoReflect.Descriptor instead. func (*MigrationControl) Descriptor() ([]byte, []int) { return file_internal_migration_migrate_proto_rawDescGZIP(), []int{9} } func (x *MigrationControl) GetSuccess() bool { if x != nil && x.Success != nil { return *x.Success } return false } func (x *MigrationControl) GetMessage() string { if x != nil && x.Message != nil { return *x.Message } return "" } type MigrationSync struct { state protoimpl.MessageState `protogen:"open.v1"` FinalPreDump *bool `protobuf:"varint,1,req,name=finalPreDump" json:"finalPreDump,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *MigrationSync) Reset() { *x = MigrationSync{} mi := &file_internal_migration_migrate_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } func (x *MigrationSync) String() string { return protoimpl.X.MessageStringOf(x) } func (*MigrationSync) ProtoMessage() {} func (x *MigrationSync) ProtoReflect() protoreflect.Message { mi := &file_internal_migration_migrate_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } return mi.MessageOf(x) } // Deprecated: Use MigrationSync.ProtoReflect.Descriptor instead. func (*MigrationSync) Descriptor() ([]byte, []int) { return file_internal_migration_migrate_proto_rawDescGZIP(), []int{10} } func (x *MigrationSync) GetFinalPreDump() bool { if x != nil && x.FinalPreDump != nil { return *x.FinalPreDump } return false } var File_internal_migration_migrate_proto protoreflect.FileDescriptor const file_internal_migration_migrate_proto_rawDesc = "" + "\n" + " internal/migration/migrate.proto\x12\tmigration\"\x7f\n" + "\tIDMapType\x12\x14\n" + "\x05isuid\x18\x01 \x02(\bR\x05isuid\x12\x14\n" + "\x05isgid\x18\x02 \x02(\bR\x05isgid\x12\x16\n" + "\x06hostid\x18\x03 \x02(\x05R\x06hostid\x12\x12\n" + "\x04nsid\x18\x04 \x02(\x05R\x04nsid\x12\x1a\n" + "\bmaprange\x18\x05 \x02(\x05R\bmaprange\"0\n" + "\x06Config\x12\x10\n" + "\x03key\x18\x01 \x02(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x02(\tR\x05value\"G\n" + "\x06Device\x12\x12\n" + "\x04name\x18\x01 \x02(\tR\x04name\x12)\n" + "\x06config\x18\x02 \x03(\v2\x11.migration.ConfigR\x06config\"\xf0\x02\n" + "\bSnapshot\x12\x12\n" + "\x04name\x18\x01 \x02(\tR\x04name\x123\n" + "\vlocalConfig\x18\x02 \x03(\v2\x11.migration.ConfigR\vlocalConfig\x12\x1a\n" + "\bprofiles\x18\x03 \x03(\tR\bprofiles\x12\x1c\n" + "\tephemeral\x18\x04 \x02(\bR\tephemeral\x125\n" + "\flocalDevices\x18\x05 \x03(\v2\x11.migration.DeviceR\flocalDevices\x12\"\n" + "\farchitecture\x18\x06 \x02(\x05R\farchitecture\x12\x1a\n" + "\bstateful\x18\a \x02(\bR\bstateful\x12#\n" + "\rcreation_date\x18\b \x01(\x03R\fcreationDate\x12$\n" + "\x0elast_used_date\x18\t \x01(\x03R\flastUsedDate\x12\x1f\n" + "\vexpiry_date\x18\n" + " \x01(\x03R\n" + "expiryDate\"\x81\x01\n" + "\rrsyncFeatures\x12\x16\n" + "\x06xattrs\x18\x01 \x01(\bR\x06xattrs\x12\x16\n" + "\x06delete\x18\x02 \x01(\bR\x06delete\x12\x1a\n" + "\bcompress\x18\x03 \x01(\bR\bcompress\x12$\n" + "\rbidirectional\x18\x04 \x01(\bR\rbidirectional\"w\n" + "\vzfsFeatures\x12\x1a\n" + "\bcompress\x18\x01 \x01(\bR\bcompress\x12)\n" + "\x10migration_header\x18\x02 \x01(\bR\x0fmigrationHeader\x12!\n" + "\fheader_zvols\x18\x03 \x01(\bR\vheaderZvols\"\x9d\x01\n" + "\rbtrfsFeatures\x12)\n" + "\x10migration_header\x18\x01 \x01(\bR\x0fmigrationHeader\x12+\n" + "\x11header_subvolumes\x18\x02 \x01(\bR\x10headerSubvolumes\x124\n" + "\x16header_subvolume_uuids\x18\x03 \x01(\bR\x14headerSubvolumeUuids\"\xb4\x03\n" + "\x0fDependentVolume\x12\x12\n" + "\x04name\x18\x01 \x02(\tR\x04name\x12\x12\n" + "\x04pool\x18\x02 \x02(\tR\x04pool\x12 \n" + "\vcontentType\x18\x03 \x02(\tR\vcontentType\x12*\n" + "\x02fs\x18\x04 \x02(\x0e2\x1a.migration.MigrationFSTypeR\x02fs\x12>\n" + "\rrsyncFeatures\x18\x05 \x01(\v2\x18.migration.rsyncFeaturesR\rrsyncFeatures\x128\n" + "\vzfsFeatures\x18\x06 \x01(\v2\x16.migration.zfsFeaturesR\vzfsFeatures\x12>\n" + "\rbtrfsFeatures\x18\a \x01(\v2\x18.migration.btrfsFeaturesR\rbtrfsFeatures\x12\x1e\n" + "\n" + "volumeSize\x18\b \x01(\x03R\n" + "volumeSize\x121\n" + "\tsnapshots\x18\t \x03(\v2\x13.migration.SnapshotR\tsnapshots\x12\x1e\n" + "\n" + "deviceName\x18\n" + " \x01(\tR\n" + "deviceName\"\xf1\x04\n" + "\x0fMigrationHeader\x12*\n" + "\x02fs\x18\x01 \x02(\x0e2\x1a.migration.MigrationFSTypeR\x02fs\x12'\n" + "\x04criu\x18\x02 \x01(\x0e2\x13.migration.CRIUTypeR\x04criu\x12*\n" + "\x05idmap\x18\x03 \x03(\v2\x14.migration.IDMapTypeR\x05idmap\x12$\n" + "\rsnapshotNames\x18\x04 \x03(\tR\rsnapshotNames\x121\n" + "\tsnapshots\x18\x05 \x03(\v2\x13.migration.SnapshotR\tsnapshots\x12\x18\n" + "\apredump\x18\a \x01(\bR\apredump\x12>\n" + "\rrsyncFeatures\x18\b \x01(\v2\x18.migration.rsyncFeaturesR\rrsyncFeatures\x12\x18\n" + "\arefresh\x18\t \x01(\bR\arefresh\x128\n" + "\vzfsFeatures\x18\n" + " \x01(\v2\x16.migration.zfsFeaturesR\vzfsFeatures\x12\x1e\n" + "\n" + "volumeSize\x18\v \x01(\x03R\n" + "volumeSize\x12>\n" + "\rbtrfsFeatures\x18\f \x01(\v2\x18.migration.btrfsFeaturesR\rbtrfsFeatures\x12.\n" + "\x12indexHeaderVersion\x18\r \x01(\rR\x12indexHeaderVersion\x12F\n" + "\x10dependentVolumes\x18\x0e \x03(\v2\x1a.migration.DependentVolumeR\x10dependentVolumes\"F\n" + "\x10MigrationControl\x12\x18\n" + "\asuccess\x18\x01 \x02(\bR\asuccess\x12\x18\n" + "\amessage\x18\x02 \x01(\tR\amessage\"3\n" + "\rMigrationSync\x12\"\n" + "\ffinalPreDump\x18\x01 \x02(\bR\ffinalPreDump*[\n" + "\x0fMigrationFSType\x12\t\n" + "\x05RSYNC\x10\x00\x12\t\n" + "\x05BTRFS\x10\x01\x12\a\n" + "\x03ZFS\x10\x02\x12\a\n" + "\x03RBD\x10\x03\x12\x13\n" + "\x0fBLOCK_AND_RSYNC\x10\x04\x12\v\n" + "\aLINSTOR\x10\x05*<\n" + "\bCRIUType\x12\x0e\n" + "\n" + "CRIU_RSYNC\x10\x00\x12\t\n" + "\x05PHAUL\x10\x01\x12\b\n" + "\x04NONE\x10\x02\x12\v\n" + "\aVM_QEMU\x10\x03B\x14Z\x12internal/migration" var ( file_internal_migration_migrate_proto_rawDescOnce sync.Once file_internal_migration_migrate_proto_rawDescData []byte ) func file_internal_migration_migrate_proto_rawDescGZIP() []byte { file_internal_migration_migrate_proto_rawDescOnce.Do(func() { file_internal_migration_migrate_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_internal_migration_migrate_proto_rawDesc), len(file_internal_migration_migrate_proto_rawDesc))) }) return file_internal_migration_migrate_proto_rawDescData } var file_internal_migration_migrate_proto_enumTypes = make([]protoimpl.EnumInfo, 2) var file_internal_migration_migrate_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_internal_migration_migrate_proto_goTypes = []any{ (MigrationFSType)(0), // 0: migration.MigrationFSType (CRIUType)(0), // 1: migration.CRIUType (*IDMapType)(nil), // 2: migration.IDMapType (*Config)(nil), // 3: migration.Config (*Device)(nil), // 4: migration.Device (*Snapshot)(nil), // 5: migration.Snapshot (*RsyncFeatures)(nil), // 6: migration.rsyncFeatures (*ZfsFeatures)(nil), // 7: migration.zfsFeatures (*BtrfsFeatures)(nil), // 8: migration.btrfsFeatures (*DependentVolume)(nil), // 9: migration.DependentVolume (*MigrationHeader)(nil), // 10: migration.MigrationHeader (*MigrationControl)(nil), // 11: migration.MigrationControl (*MigrationSync)(nil), // 12: migration.MigrationSync } var file_internal_migration_migrate_proto_depIdxs = []int32{ 3, // 0: migration.Device.config:type_name -> migration.Config 3, // 1: migration.Snapshot.localConfig:type_name -> migration.Config 4, // 2: migration.Snapshot.localDevices:type_name -> migration.Device 0, // 3: migration.DependentVolume.fs:type_name -> migration.MigrationFSType 6, // 4: migration.DependentVolume.rsyncFeatures:type_name -> migration.rsyncFeatures 7, // 5: migration.DependentVolume.zfsFeatures:type_name -> migration.zfsFeatures 8, // 6: migration.DependentVolume.btrfsFeatures:type_name -> migration.btrfsFeatures 5, // 7: migration.DependentVolume.snapshots:type_name -> migration.Snapshot 0, // 8: migration.MigrationHeader.fs:type_name -> migration.MigrationFSType 1, // 9: migration.MigrationHeader.criu:type_name -> migration.CRIUType 2, // 10: migration.MigrationHeader.idmap:type_name -> migration.IDMapType 5, // 11: migration.MigrationHeader.snapshots:type_name -> migration.Snapshot 6, // 12: migration.MigrationHeader.rsyncFeatures:type_name -> migration.rsyncFeatures 7, // 13: migration.MigrationHeader.zfsFeatures:type_name -> migration.zfsFeatures 8, // 14: migration.MigrationHeader.btrfsFeatures:type_name -> migration.btrfsFeatures 9, // 15: migration.MigrationHeader.dependentVolumes:type_name -> migration.DependentVolume 16, // [16:16] is the sub-list for method output_type 16, // [16:16] is the sub-list for method input_type 16, // [16:16] is the sub-list for extension type_name 16, // [16:16] is the sub-list for extension extendee 0, // [0:16] is the sub-list for field type_name } func init() { file_internal_migration_migrate_proto_init() } func file_internal_migration_migrate_proto_init() { if File_internal_migration_migrate_proto != nil { return } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_internal_migration_migrate_proto_rawDesc), len(file_internal_migration_migrate_proto_rawDesc)), NumEnums: 2, NumMessages: 11, NumExtensions: 0, NumServices: 0, }, GoTypes: file_internal_migration_migrate_proto_goTypes, DependencyIndexes: file_internal_migration_migrate_proto_depIdxs, EnumInfos: file_internal_migration_migrate_proto_enumTypes, MessageInfos: file_internal_migration_migrate_proto_msgTypes, }.Build() File_internal_migration_migrate_proto = out.File file_internal_migration_migrate_proto_goTypes = nil file_internal_migration_migrate_proto_depIdxs = nil } incus-7.0.0/internal/migration/migrate.proto000066400000000000000000000051011517523235500211660ustar00rootroot00000000000000// silence the protobuf compiler warning by setting the default syntax = "proto2"; option go_package = "internal/migration"; package migration; enum MigrationFSType { RSYNC = 0; BTRFS = 1; ZFS = 2; RBD = 3; BLOCK_AND_RSYNC = 4; LINSTOR = 5; } enum CRIUType { CRIU_RSYNC = 0; PHAUL = 1; NONE = 2; VM_QEMU = 3; } message IDMapType { required bool isuid = 1; required bool isgid = 2; required int32 hostid = 3; required int32 nsid = 4; required int32 maprange = 5; } message Config { required string key = 1; required string value = 2; } message Device { required string name = 1; repeated Config config = 2; } message Snapshot { required string name = 1; repeated Config localConfig = 2; repeated string profiles = 3; required bool ephemeral = 4; repeated Device localDevices = 5; required int32 architecture = 6; required bool stateful = 7; optional int64 creation_date = 8; optional int64 last_used_date = 9; optional int64 expiry_date = 10; } message rsyncFeatures { optional bool xattrs = 1; optional bool delete = 2; optional bool compress = 3; optional bool bidirectional = 4; } message zfsFeatures { optional bool compress = 1; optional bool migration_header = 2; optional bool header_zvols = 3; } message btrfsFeatures { optional bool migration_header = 1; optional bool header_subvolumes = 2; optional bool header_subvolume_uuids = 3; } message DependentVolume { required string name = 1; required string pool = 2; required string contentType = 3; required MigrationFSType fs = 4; optional rsyncFeatures rsyncFeatures = 5; optional zfsFeatures zfsFeatures = 6; optional btrfsFeatures btrfsFeatures = 7; optional int64 volumeSize = 8; repeated Snapshot snapshots = 9; optional string deviceName = 10; } message MigrationHeader { required MigrationFSType fs = 1; optional CRIUType criu = 2; repeated IDMapType idmap = 3; repeated string snapshotNames = 4; repeated Snapshot snapshots = 5; optional bool predump = 7; optional rsyncFeatures rsyncFeatures = 8; optional bool refresh = 9; optional zfsFeatures zfsFeatures = 10; optional int64 volumeSize = 11; optional btrfsFeatures btrfsFeatures = 12; optional uint32 indexHeaderVersion = 13; repeated DependentVolume dependentVolumes = 14; } message MigrationControl { required bool success = 1; /* optional failure message if sending a failure */ optional string message = 2; } message MigrationSync { required bool finalPreDump = 1; } incus-7.0.0/internal/migration/utils.go000066400000000000000000000067721517523235500201570ustar00rootroot00000000000000package migration // BTRFSFeatureMigrationHeader indicates a migration header will be sent/recv in data channel after index header. const BTRFSFeatureMigrationHeader = "migration_header" // BTRFSFeatureSubvolumes indicates migration can send/recv subvolumes. const BTRFSFeatureSubvolumes = "header_subvolumes" // BTRFSFeatureSubvolumeUUIDs indicates that the header will include subvolume UUIDs. const BTRFSFeatureSubvolumeUUIDs = "header_subvolume_uuids" // ZFSFeatureMigrationHeader indicates a migration header will be sent/recv in data channel after index header. const ZFSFeatureMigrationHeader = "migration_header" // ZFSFeatureZvolFilesystems indicates migration can send/recv zvols. const ZFSFeatureZvolFilesystems = "header_zvol_filesystems" // GetRsyncFeaturesSlice returns a slice of strings representing the supported RSYNC features. func (m *MigrationHeader) GetRsyncFeaturesSlice() []string { features := []string{} if m == nil { return features } if m.RsyncFeatures != nil { if m.RsyncFeatures.Xattrs != nil && *m.RsyncFeatures.Xattrs { features = append(features, "xattrs") } if m.RsyncFeatures.Delete != nil && *m.RsyncFeatures.Delete { features = append(features, "delete") } if m.RsyncFeatures.Compress != nil && *m.RsyncFeatures.Compress { features = append(features, "compress") } if m.RsyncFeatures.Bidirectional != nil && *m.RsyncFeatures.Bidirectional { features = append(features, "bidirectional") } } return features } // GetZfsFeaturesSlice returns a slice of strings representing the supported ZFS features. func (m *MigrationHeader) GetZfsFeaturesSlice() []string { features := []string{} if m == nil { return features } if m.ZfsFeatures != nil { if m.ZfsFeatures.Compress != nil && *m.ZfsFeatures.Compress { features = append(features, "compress") } if m.ZfsFeatures.MigrationHeader != nil && *m.ZfsFeatures.MigrationHeader { features = append(features, ZFSFeatureMigrationHeader) } if m.ZfsFeatures.HeaderZvols != nil && *m.ZfsFeatures.HeaderZvols { features = append(features, ZFSFeatureZvolFilesystems) } } return features } // GetBtrfsFeaturesSlice returns a slice of strings representing the supported BTRFS features. func (m *MigrationHeader) GetBtrfsFeaturesSlice() []string { features := []string{} if m == nil { return features } if m.BtrfsFeatures != nil { if m.BtrfsFeatures.MigrationHeader != nil && *m.BtrfsFeatures.MigrationHeader { features = append(features, BTRFSFeatureMigrationHeader) } if m.BtrfsFeatures.HeaderSubvolumes != nil && *m.BtrfsFeatures.HeaderSubvolumes { features = append(features, BTRFSFeatureSubvolumes) } if m.BtrfsFeatures.HeaderSubvolumeUuids != nil && *m.BtrfsFeatures.HeaderSubvolumeUuids { features = append(features, BTRFSFeatureSubvolumeUUIDs) } } return features } // GetSnapshotConfigValue retrieves the value associated with the given key from the snapshot LocalConfig. func GetSnapshotConfigValue(snapshot *Snapshot, key string) string { var value string for _, c := range snapshot.LocalConfig { if c.GetKey() != key { continue } value = c.GetValue() } return value } // SetSnapshotConfigValue stores the given value for the specified key in the snapshot LocalConfig. func SetSnapshotConfigValue(snapshot *Snapshot, key string, value string) { for _, c := range snapshot.LocalConfig { if c.GetKey() != key { continue } c.Value = &value return } config := Config{Key: &key, Value: &value} snapshot.LocalConfig = append(snapshot.LocalConfig, &config) } incus-7.0.0/internal/migration/wsproto.go000066400000000000000000000026301517523235500205210ustar00rootroot00000000000000package migration import ( "errors" "io" "github.com/gorilla/websocket" "google.golang.org/protobuf/proto" internalIO "github.com/lxc/incus/v7/internal/io" ) // ProtoRecv gets a protobuf message from a websocket. func ProtoRecv(ws *websocket.Conn, msg proto.Message) error { if ws == nil { return errors.New("Empty websocket connection") } mt, r, err := ws.NextReader() if err != nil { return err } if mt != websocket.BinaryMessage { return errors.New("Only binary messages allowed") } buf, err := io.ReadAll(r) if err != nil { return err } err = proto.Unmarshal(buf, msg) if err != nil { return err } return nil } // ProtoSend sends a protobuf message over a websocket. func ProtoSend(ws *websocket.Conn, msg proto.Message) error { if ws == nil { return errors.New("Empty websocket connection") } w, err := ws.NextWriter(websocket.BinaryMessage) if err != nil { return err } defer func() { _ = w.Close() }() data, err := proto.Marshal(msg) if err != nil { return err } err = internalIO.WriteAll(w, data) if err != nil { return err } return w.Close() } // ProtoSendControl sends a migration control message over a websocket. func ProtoSendControl(ws *websocket.Conn, err error) { message := "" if err != nil { message = err.Error() } msg := MigrationControl{ Success: proto.Bool(err == nil), Message: proto.String(message), } _ = ProtoSend(ws, &msg) } incus-7.0.0/internal/netutils/000077500000000000000000000000001517523235500163325ustar00rootroot00000000000000incus-7.0.0/internal/netutils/cgo.go000066400000000000000000000006631517523235500174360ustar00rootroot00000000000000//go:build linux && cgo package netutils // #cgo CFLAGS: -std=gnu11 -Wvla -Werror -fvisibility=hidden -Winit-self // #cgo CFLAGS: -Wformat=2 -Wshadow -Wendif-labels -fasynchronous-unwind-tables // #cgo CFLAGS: -pipe --param=ssp-buffer-size=4 -g -Wunused // #cgo CFLAGS: -Werror=implicit-function-declaration // #cgo CFLAGS: -Werror=return-type -Wendif-labels -Werror=overflow // #cgo CFLAGS: -Wnested-externs -fexceptions import "C" incus-7.0.0/internal/netutils/netns_getifaddrs.c000066400000000000000000000277571517523235500220430ustar00rootroot00000000000000 #ifndef _GNU_SOURCE #define _GNU_SOURCE 1 #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../../shared/cgo/compiler.h" #include "network.c" struct netns_ifaddrs { struct netns_ifaddrs *ifa_next; // Can - but shouldn't be - NULL. char *ifa_name; // This field is not present struct ifaddrs int ifa_ifindex; // This field is not present struct ifaddrs int ifa_ifindex_peer; unsigned ifa_flags; // This field is not present struct ifaddrs int ifa_mtu; // This field is not present struct ifaddrs int ifa_prefixlen; struct sockaddr *ifa_addr; struct sockaddr *ifa_netmask; union { struct sockaddr *ifu_broadaddr; struct sockaddr *ifu_dstaddr; } ifa_ifu; // If you don't know what this is for don't touch it. int ifa_stats_type; struct rtnl_link_stats64 ifa_stats64; }; #define __ifa_broadaddr ifa_ifu.ifu_broadaddr #define __ifa_dstaddr ifa_ifu.ifu_dstaddr // getifaddrs() reports hardware addresses with PF_PACKET that implies // struct sockaddr_ll. But e.g. Infiniband socket address length is // longer than sockaddr_ll.ssl_addr[8] can hold. Use this hack struct // to extend ssl_addr - callers should be able to still use it. struct sockaddr_ll_hack { unsigned short sll_family, sll_protocol; int sll_ifindex; unsigned short sll_hatype; unsigned char sll_pkttype, sll_halen; unsigned char sll_addr[24]; }; union sockany { struct sockaddr sa; struct sockaddr_ll_hack ll; struct sockaddr_in v4; struct sockaddr_in6 v6; }; struct ifaddrs_storage { struct netns_ifaddrs ifa; struct ifaddrs_storage *hash_next; union sockany addr, netmask, ifu; unsigned int index; char name[IFNAMSIZ + 1]; }; struct ifaddrs_ctx { struct ifaddrs_storage *first; struct ifaddrs_storage *last; struct ifaddrs_storage *hash[IFADDRS_HASH_SIZE]; }; static void netns_freeifaddrs(struct netns_ifaddrs *ifp) { struct netns_ifaddrs *n; while (ifp) { n = ifp->ifa_next; free(ifp); ifp = n; } } static void copy_addr(struct sockaddr **r, int af, union sockany *sa, void *addr, size_t addrlen, int ifindex) { uint8_t *dst; size_t len; switch (af) { case AF_INET: dst = (uint8_t *)&sa->v4.sin_addr; len = 4; break; case AF_INET6: dst = (uint8_t *)&sa->v6.sin6_addr; len = 16; if (__IN6_IS_ADDR_LINKLOCAL(addr) || __IN6_IS_ADDR_MC_LINKLOCAL(addr)) sa->v6.sin6_scope_id = ifindex; break; default: return; } if (addrlen < len) return; sa->sa.sa_family = af; memcpy(dst, addr, len); *r = &sa->sa; } static void gen_netmask(struct sockaddr **r, int af, union sockany *sa, int prefixlen) { uint8_t addr[16] = {0}; int i; if ((size_t)prefixlen > 8 * sizeof(addr)) prefixlen = 8 * sizeof(addr); i = prefixlen / 8; memset(addr, 0xff, i); if ((size_t)i < sizeof(addr)) addr[i++] = 0xff << (8 - (prefixlen % 8)); copy_addr(r, af, sa, addr, sizeof(addr), 0); } static void copy_lladdr(struct sockaddr **r, union sockany *sa, void *addr, size_t addrlen, int ifindex, unsigned short hatype) { if (addrlen > sizeof(sa->ll.sll_addr)) return; sa->ll.sll_family = AF_PACKET; sa->ll.sll_ifindex = ifindex; sa->ll.sll_hatype = hatype; sa->ll.sll_halen = addrlen; memcpy(sa->ll.sll_addr, addr, addrlen); *r = &sa->sa; } static int nl_msg_to_ifaddr(void *pctx, bool *netnsid_aware, struct nlmsghdr *h) { struct ifaddrs_storage *ifs, *ifs0; struct rtattr *rta; int stats_len = 0; struct ifinfomsg *ifi = __NLMSG_DATA(h); struct ifaddrmsg *ifa = __NLMSG_DATA(h); struct ifaddrs_ctx *ctx = pctx; if (h->nlmsg_type == RTM_NEWLINK) { for (rta = __NLMSG_RTA(h, sizeof(*ifi)); __NLMSG_RTAOK(rta, h); rta = __RTA_NEXT(rta)) { if (rta->rta_type != IFLA_STATS64) continue; stats_len = __RTA_DATALEN(rta); break; } } else { for (ifs0 = ctx->hash[ifa->ifa_index % IFADDRS_HASH_SIZE]; ifs0; ifs0 = ifs0->hash_next) if (ifs0->index == ifa->ifa_index) break; if (!ifs0) return 0; } ifs = calloc(1, sizeof(struct ifaddrs_storage) + stats_len); if (!ifs) { errno = ENOMEM; return -1; } if (h->nlmsg_type == RTM_NEWLINK) { ifs->index = ifi->ifi_index; ifs->ifa.ifa_ifindex = ifi->ifi_index; ifs->ifa.ifa_flags = ifi->ifi_flags; for (rta = __NLMSG_RTA(h, sizeof(*ifi)); __NLMSG_RTAOK(rta, h); rta = __RTA_NEXT(rta)) { switch (rta->rta_type) { case IFLA_IFNAME: if (__RTA_DATALEN(rta) < sizeof(ifs->name)) { memcpy(ifs->name, __RTA_DATA(rta), __RTA_DATALEN(rta)); ifs->ifa.ifa_name = ifs->name; } break; case IFLA_ADDRESS: copy_lladdr(&ifs->ifa.ifa_addr, &ifs->addr, __RTA_DATA(rta), __RTA_DATALEN(rta), ifi->ifi_index, ifi->ifi_type); break; case IFLA_BROADCAST: copy_lladdr(&ifs->ifa.__ifa_broadaddr, &ifs->ifu, __RTA_DATA(rta), __RTA_DATALEN(rta), ifi->ifi_index, ifi->ifi_type); break; case IFLA_STATS64: ifs->ifa.ifa_stats_type = IFLA_STATS64; memcpy(&ifs->ifa.ifa_stats64, __RTA_DATA(rta), __RTA_DATALEN(rta)); break; case IFLA_MTU: memcpy(&ifs->ifa.ifa_mtu, __RTA_DATA(rta), sizeof(int)); break; case IFLA_TARGET_NETNSID: *netnsid_aware = true; break; case IFLA_LINK: if (__RTA_DATALEN(rta)) memcpy(&ifs->ifa.ifa_ifindex_peer, __RTA_DATA(rta), __RTA_DATALEN(rta)); break; } } if (ifs->ifa.ifa_name) { unsigned int bucket = ifs->index % IFADDRS_HASH_SIZE; ifs->hash_next = ctx->hash[bucket]; ctx->hash[bucket] = ifs; } } else { ifs->ifa.ifa_name = ifs0->ifa.ifa_name; ifs->ifa.ifa_mtu = ifs0->ifa.ifa_mtu; ifs->ifa.ifa_ifindex = ifs0->ifa.ifa_ifindex; ifs->ifa.ifa_flags = ifs0->ifa.ifa_flags; for (rta = __NLMSG_RTA(h, sizeof(*ifa)); __NLMSG_RTAOK(rta, h); rta = __RTA_NEXT(rta)) { switch (rta->rta_type) { case IFA_ADDRESS: // If ifa_addr is already set we, received an // IFA_LOCAL before so treat this as destination // address. if (ifs->ifa.ifa_addr) copy_addr(&ifs->ifa.__ifa_dstaddr, ifa->ifa_family, &ifs->ifu, __RTA_DATA(rta), __RTA_DATALEN(rta), ifa->ifa_index); else copy_addr(&ifs->ifa.ifa_addr, ifa->ifa_family, &ifs->addr, __RTA_DATA(rta), __RTA_DATALEN(rta), ifa->ifa_index); break; case IFA_BROADCAST: copy_addr(&ifs->ifa.__ifa_broadaddr, ifa->ifa_family, &ifs->ifu, __RTA_DATA(rta), __RTA_DATALEN(rta), ifa->ifa_index); break; case IFA_LOCAL: // If ifa_addr is set and we get IFA_LOCAL, // assume we have a point-to-point network. Move // address to correct field. if (ifs->ifa.ifa_addr) { ifs->ifu = ifs->addr; ifs->ifa.__ifa_dstaddr = &ifs->ifu.sa; memset(&ifs->addr, 0, sizeof(ifs->addr)); } copy_addr(&ifs->ifa.ifa_addr, ifa->ifa_family, &ifs->addr, __RTA_DATA(rta), __RTA_DATALEN(rta), ifa->ifa_index); break; case IFA_LABEL: if (__RTA_DATALEN(rta) < sizeof(ifs->name)) { memcpy(ifs->name, __RTA_DATA(rta), __RTA_DATALEN(rta)); ifs->ifa.ifa_name = ifs->name; } break; case IFA_TARGET_NETNSID: *netnsid_aware = true; break; } } if (ifs->ifa.ifa_addr) { gen_netmask(&ifs->ifa.ifa_netmask, ifa->ifa_family, &ifs->netmask, ifa->ifa_prefixlen); ifs->ifa.ifa_prefixlen = ifa->ifa_prefixlen; } } if (ifs->ifa.ifa_name) { if (!ctx->first) ctx->first = ifs; if (ctx->last) ctx->last->ifa.ifa_next = &ifs->ifa; ctx->last = ifs; } else { free(ifs); } return 0; } #define NLMSG_TAIL(nmsg) \ ((struct rtattr *)(((void *)(nmsg)) + \ __NETLINK_ALIGN((nmsg)->nlmsg_len))) static int __netlink_recv(int fd, unsigned int seq, int type, int af, __s32 netns_id, bool *netnsid_aware, int (*cb)(void *ctx, bool *netnsid_aware, struct nlmsghdr *h), void *ctx) { int r, property, ret; char *buf; struct nlmsghdr *hdr; struct ifinfomsg *ifi_msg; struct ifaddrmsg *ifa_msg; union { uint8_t buf[8192]; struct { struct nlmsghdr nlh; struct rtgenmsg g; } req; struct nlmsghdr reply; } u; char getlink_buf[__NETLINK_ALIGN(sizeof(struct nlmsghdr)) + __NETLINK_ALIGN(sizeof(struct ifinfomsg)) + __NETLINK_ALIGN(1024)] = {0}; char getaddr_buf[__NETLINK_ALIGN(sizeof(struct nlmsghdr)) + __NETLINK_ALIGN(sizeof(struct ifaddrmsg)) + __NETLINK_ALIGN(1024)] = {0}; if (type == RTM_GETLINK) { buf = getlink_buf; hdr = (struct nlmsghdr *)buf; hdr->nlmsg_len = NLMSG_LENGTH(sizeof(*ifi_msg)); ifi_msg = (struct ifinfomsg *)__NLMSG_DATA(hdr); ifi_msg->ifi_family = af; property = IFLA_TARGET_NETNSID; } else if (type == RTM_GETADDR) { buf = getaddr_buf; hdr = (struct nlmsghdr *)buf; hdr->nlmsg_len = NLMSG_LENGTH(sizeof(*ifa_msg)); ifa_msg = (struct ifaddrmsg *)__NLMSG_DATA(hdr); ifa_msg->ifa_family = af; property = IFA_TARGET_NETNSID; } else { errno = EINVAL; return -1; } hdr->nlmsg_type = type; hdr->nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST; hdr->nlmsg_pid = 0; hdr->nlmsg_seq = seq; if (netns_id >= 0) addattr(hdr, 1024, property, &netns_id, sizeof(netns_id)); r = __netlink_send(fd, hdr); if (r < 0) return -1; for (;;) { r = recv(fd, u.buf, sizeof(u.buf), MSG_DONTWAIT); if (r <= 0) return -1; for (hdr = &u.reply; __NLMSG_OK(hdr, (void *)&u.buf[r]); hdr = __NLMSG_NEXT(hdr)) { if (hdr->nlmsg_type == NLMSG_DONE) return 0; if (hdr->nlmsg_type == NLMSG_ERROR) { errno = EINVAL; return -1; } ret = cb(ctx, netnsid_aware, hdr); if (ret) return ret; } } } static int __rtnl_enumerate(int link_af, int addr_af, __s32 netns_id, bool *netnsid_aware, int (*cb)(void *ctx, bool *netnsid_aware, struct nlmsghdr *h), void *ctx) { int fd, r, saved_errno; bool getaddr_netnsid_aware = false, getlink_netnsid_aware = false; fd = socket(PF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, NETLINK_ROUTE); if (fd < 0) return -1; r = setsockopt(fd, SOL_NETLINK, NETLINK_GET_STRICT_CHK, &(int){1}, sizeof(int)); if (r < 0 && netns_id >= 0) { close(fd); *netnsid_aware = false; return -1; } r = __netlink_recv(fd, 1, RTM_GETLINK, link_af, netns_id, &getlink_netnsid_aware, cb, ctx); if (!r) r = __netlink_recv(fd, 2, RTM_GETADDR, addr_af, netns_id, &getaddr_netnsid_aware, cb, ctx); saved_errno = errno; close(fd); errno = saved_errno; if (getaddr_netnsid_aware && getlink_netnsid_aware) *netnsid_aware = true; else *netnsid_aware = false; return r; } __unused static int netns_getifaddrs(struct netns_ifaddrs **ifap, __s32 netns_id, bool *netnsid_aware) { int r, saved_errno; struct ifaddrs_ctx _ctx; struct ifaddrs_ctx *ctx = &_ctx; memset(ctx, 0, sizeof *ctx); r = __rtnl_enumerate(AF_UNSPEC, AF_UNSPEC, netns_id, netnsid_aware, nl_msg_to_ifaddr, ctx); saved_errno = errno; if (r < 0) netns_freeifaddrs(&ctx->first->ifa); else *ifap = &ctx->first->ifa; errno = saved_errno; return r; } // Get a pointer to the address structure from a sockaddr. __unused static void *get_addr_ptr(struct sockaddr *sockaddr_ptr) { if (sockaddr_ptr->sa_family == AF_INET) return &((struct sockaddr_in *)sockaddr_ptr)->sin_addr; if (sockaddr_ptr->sa_family == AF_INET6) return &((struct sockaddr_in6 *)sockaddr_ptr)->sin6_addr; return NULL; } __unused static char *get_packet_address(struct sockaddr *sockaddr_ptr, char *buf, size_t buflen) { char *slider = buf; unsigned char *m = ((struct sockaddr_ll *)sockaddr_ptr)->sll_addr; unsigned char n = ((struct sockaddr_ll *)sockaddr_ptr)->sll_halen; for (unsigned char i = 0; i < n; i++) { int ret; ret = snprintf(slider, buflen, "%02x%s", m[i], (i + 1) < n ? ":" : ""); if (ret < 0 || (size_t)ret >= buflen) return NULL; buflen -= ret; slider = (slider + ret); } return buf; } incus-7.0.0/internal/netutils/network.c000066400000000000000000000144571517523235500202020ustar00rootroot00000000000000 #ifndef _GNU_SOURCE #define _GNU_SOURCE 1 #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../../shared/cgo/compiler.h" #include "../../shared/cgo/macro.h" #ifndef NETNS_RTA #define NETNS_RTA(r) \ ((struct rtattr *)(((char *)(r)) + NLMSG_ALIGN(sizeof(struct rtgenmsg)))) #endif #define IFADDRS_HASH_SIZE 64 #define __NETLINK_ALIGN(len) (((len) + 3) & ~3) #define __NLMSG_OK(nlh, end) \ ((char *)(end) - (char *)(nlh) >= sizeof(struct nlmsghdr)) #define __NLMSG_NEXT(nlh) \ (struct nlmsghdr *)((char *)(nlh) + __NETLINK_ALIGN((nlh)->nlmsg_len)) #define __NLMSG_DATA(nlh) ((void *)((char *)(nlh) + sizeof(struct nlmsghdr))) #define __NLMSG_DATAEND(nlh) ((char *)(nlh) + (nlh)->nlmsg_len) #define __NLMSG_RTA(nlh, len) \ ((void *)((char *)(nlh) + sizeof(struct nlmsghdr) + \ __NETLINK_ALIGN(len))) #define __RTA_DATALEN(rta) ((rta)->rta_len - sizeof(struct rtattr)) #define __RTA_NEXT(rta) \ (struct rtattr *)((char *)(rta) + __NETLINK_ALIGN((rta)->rta_len)) #define __RTA_OK(nlh, end) \ ((char *)(end) - (char *)(rta) >= sizeof(struct rtattr)) #define __NLMSG_RTAOK(rta, nlh) __RTA_OK(rta, __NLMSG_DATAEND(nlh)) #define __IN6_IS_ADDR_LINKLOCAL(a) \ ((((uint8_t *)(a))[0]) == 0xfe && (((uint8_t *)(a))[1] & 0xc0) == 0x80) #define __IN6_IS_ADDR_MC_LINKLOCAL(a) \ (IN6_IS_ADDR_MULTICAST(a) && ((((uint8_t *)(a))[1] & 0xf) == 0x2)) #define __RTA_DATA(rta) ((void *)((char *)(rta) + sizeof(struct rtattr))) #define NLMSG_TAIL(nmsg) \ ((struct rtattr *)(((void *)(nmsg)) + \ __NETLINK_ALIGN((nmsg)->nlmsg_len))) static int netlink_open(int protocol) { int fd, ret; socklen_t socklen; struct sockaddr_nl local; int sndbuf = 32768; int rcvbuf = 32768; int err = -1; fd = socket(AF_NETLINK, SOCK_RAW, protocol); if (fd < 0) return -1; ret = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf)); if (ret < 0) goto out; ret = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)); if (ret < 0) goto out; memset(&local, 0, sizeof(local)); local.nl_family = AF_NETLINK; local.nl_groups = 0; ret = bind(fd, (struct sockaddr *)&local, sizeof(local)); if (ret < 0) goto out; socklen = sizeof(local); ret = getsockname(fd, (struct sockaddr *)&local, &socklen); if (ret < 0) goto out; errno = -EINVAL; if (socklen != sizeof(local)) goto out; errno = -EINVAL; if (local.nl_family != AF_NETLINK) goto out; return fd; out: close(fd); return err; } static int netlink_recv(int fd, struct nlmsghdr *nlmsghdr) { int ret; struct sockaddr_nl nladdr; struct iovec iov = { .iov_base = nlmsghdr, .iov_len = nlmsghdr->nlmsg_len, }; struct msghdr msg = { .msg_name = &nladdr, .msg_namelen = sizeof(nladdr), .msg_iov = &iov, .msg_iovlen = 1, }; memset(&nladdr, 0, sizeof(nladdr)); nladdr.nl_family = AF_NETLINK; nladdr.nl_pid = 0; nladdr.nl_groups = 0; again: ret = recvmsg(fd, &msg, 0); if (ret < 0) { if (errno == EINTR) goto again; return -1; } if (!ret) return 0; if (msg.msg_flags & MSG_TRUNC && ((__u32)ret == nlmsghdr->nlmsg_len)) { errno = EMSGSIZE; ret = -1; } return ret; } static int __netlink_send(int fd, struct nlmsghdr *nlmsghdr) { int ret; struct sockaddr_nl nladdr; struct iovec iov = { .iov_base = nlmsghdr, .iov_len = nlmsghdr->nlmsg_len, }; struct msghdr msg = { .msg_name = &nladdr, .msg_namelen = sizeof(nladdr), .msg_iov = &iov, .msg_iovlen = 1, }; memset(&nladdr, 0, sizeof(nladdr)); nladdr.nl_family = AF_NETLINK; nladdr.nl_pid = 0; nladdr.nl_groups = 0; ret = sendmsg(fd, &msg, MSG_NOSIGNAL); if (ret < 0) return -1; return ret; } static int netlink_transaction(int fd, struct nlmsghdr *request, struct nlmsghdr *answer) { int ret; ret = __netlink_send(fd, request); if (ret < 0) return -1; ret = netlink_recv(fd, answer); if (ret < 0) return -1; ret = 0; if (answer->nlmsg_type == NLMSG_ERROR) { struct nlmsgerr *err = (struct nlmsgerr *)__NLMSG_DATA(answer); errno = -err->error; if (err->error < 0) ret = -1; } return ret; } static int parse_rtattr(struct rtattr *tb[], int max, struct rtattr *rta, int len) { memset(tb, 0, sizeof(struct rtattr *) * (max + 1)); while (RTA_OK(rta, len)) { unsigned short type = rta->rta_type; if ((type <= max) && (!tb[type])) tb[type] = rta; rta = RTA_NEXT(rta, len); } return 0; } static __s32 rta_getattr_s32(const struct rtattr *rta) { return *(__s32 *)RTA_DATA(rta); } static int addattr(struct nlmsghdr *n, size_t maxlen, int type, const void *data, size_t alen) { int len = RTA_LENGTH(alen); struct rtattr *rta; if (NLMSG_ALIGN(n->nlmsg_len) + RTA_ALIGN(len) > maxlen) return -1; rta = NLMSG_TAIL(n); rta->rta_type = type; rta->rta_len = len; if (alen) memcpy(RTA_DATA(rta), data, alen); n->nlmsg_len = NLMSG_ALIGN(n->nlmsg_len) + RTA_ALIGN(len); return 0; } __unused static __s32 netns_get_nsid(__s32 netns_fd) { int fd, ret; ssize_t len; char buf[NLMSG_ALIGN(sizeof(struct nlmsghdr)) + NLMSG_ALIGN(sizeof(struct rtgenmsg)) + NLMSG_ALIGN(1024)]; struct rtattr *tb[__LXC_NETNSA_MAX + 1]; struct nlmsghdr *hdr; struct rtgenmsg *msg; int saved_errno; fd = netlink_open(NETLINK_ROUTE); if (fd < 0) return -1; memset(buf, 0, sizeof(buf)); hdr = (struct nlmsghdr *)buf; msg = (struct rtgenmsg *)__NLMSG_DATA(hdr); hdr->nlmsg_len = NLMSG_LENGTH(sizeof(*msg)); hdr->nlmsg_type = RTM_GETNSID; hdr->nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; hdr->nlmsg_pid = 0; hdr->nlmsg_seq = RTM_GETNSID; msg->rtgen_family = AF_UNSPEC; addattr(hdr, 1024, __LXC_NETNSA_FD, &netns_fd, sizeof(__s32)); ret = netlink_transaction(fd, hdr, hdr); saved_errno = errno; close(fd); errno = saved_errno; if (ret < 0) return -1; msg = __NLMSG_DATA(hdr); len = hdr->nlmsg_len - NLMSG_SPACE(sizeof(*msg)); if (len < 0) return -1; parse_rtattr(tb, __LXC_NETNSA_MAX, NETNS_RTA(msg), len); if (tb[__LXC_NETNSA_NSID]) return rta_getattr_s32(tb[__LXC_NETNSA_NSID]); return -1; } incus-7.0.0/internal/netutils/network.go000066400000000000000000000000211517523235500203430ustar00rootroot00000000000000package netutils incus-7.0.0/internal/netutils/network_linux_cgo.go000066400000000000000000000210011517523235500224130ustar00rootroot00000000000000//go:build linux && cgo package netutils /* #include "unixfd.h" #include "netns_getifaddrs.c" */ import "C" import ( "errors" "fmt" "io" "net" "os" "strings" "unsafe" "github.com/lxc/incus/v7/shared/api" ) // Allow the caller to set expectations. // UnixFdsAcceptExact will only succeed if the exact amount of fds has been // received (unless combined with UNIX_FDS_ACCEPT_NONE). const UnixFdsAcceptExact uint = C.UNIX_FDS_ACCEPT_EXACT // UnixFdsAcceptLess will also succeed if less than the requested number of fd // has been received. // If the UNIX_FDS_ACCEPT_NONE flag is not raised than at least one fd must be // received. const UnixFdsAcceptLess uint = C.UNIX_FDS_ACCEPT_LESS // UnixFdsAcceptMore will also succeed if more than the requested number of fds // have been received. Any additional fds will be silently closed. // If the UNIX_FDS_ACCEPT_NONE flag is not raised than at least one fd must be // received. const UnixFdsAcceptMore uint = C.UNIX_FDS_ACCEPT_MORE // UnixFdsAcceptNone can be specified with any of the above flags and indicates // that the caller will accept no file descriptors to be received. const UnixFdsAcceptNone uint = C.UNIX_FDS_ACCEPT_NONE // UnixFdsAcceptMask is the value of all the above flags or-ed together. const UnixFdsAcceptMask uint = C.UNIX_FDS_ACCEPT_MASK // Allow the callee to report back what happened. Only one of those will ever // be set. // UnixFdsReceivedExact indicates that the exact number of fds was received. const UnixFdsReceivedExact uint = C.UNIX_FDS_RECEIVED_EXACT // UnixFdsReceivedLess indicates that less than the requested number of fd has // been received. const UnixFdsReceivedLess uint = C.UNIX_FDS_RECEIVED_LESS // UnixFdsReceivedMore indicates that more than the requested number of fd has // been received. const UnixFdsReceivedMore uint = C.UNIX_FDS_RECEIVED_MORE // UnixFdsReceivedNone indicates that no fds have been received. const UnixFdsReceivedNone uint = C.UNIX_FDS_RECEIVED_NONE // NetnsGetifaddrs returns a map of InstanceStateNetwork for a particular process. func NetnsGetifaddrs(initPID int32, hostInterfaces []net.Interface) (map[string]api.InstanceStateNetwork, error) { var netnsidAware C.bool var ifaddrs *C.struct_netns_ifaddrs var netnsID C.__s32 if initPID > 0 { f, err := os.Open(fmt.Sprintf("/proc/%d/ns/net", initPID)) if err != nil { return nil, err } defer func() { _ = f.Close() }() netnsID = C.netns_get_nsid(C.__s32(f.Fd())) if netnsID < 0 { return nil, errors.New("Failed to retrieve network namespace id") } } else { netnsID = -1 } ret := C.netns_getifaddrs(&ifaddrs, netnsID, &netnsidAware) if ret < 0 { return nil, errors.New("Failed to retrieve network interfaces and addresses") } defer C.netns_freeifaddrs(ifaddrs) if netnsID >= 0 && !netnsidAware { return nil, errors.New("Netlink requests are not fully network namespace id aware") } // We're using the interface name as key here but we should really // switch to the ifindex at some point to handle ip aliasing correctly. networks := map[string]api.InstanceStateNetwork{} for addr := ifaddrs; addr != nil; addr = addr.ifa_next { var address [C.INET6_ADDRSTRLEN]C.char addNetwork, networkExists := networks[C.GoString(addr.ifa_name)] if !networkExists { addNetwork = api.InstanceStateNetwork{ Addresses: []api.InstanceStateNetworkAddress{}, Counters: api.InstanceStateNetworkCounters{}, } } // Interface flags netState := "down" netType := "unknown" if (addr.ifa_flags & C.IFF_BROADCAST) > 0 { netType = "broadcast" } if (addr.ifa_flags & C.IFF_LOOPBACK) > 0 { netType = "loopback" } if (addr.ifa_flags & C.IFF_POINTOPOINT) > 0 { netType = "point-to-point" } if (addr.ifa_flags & C.IFF_UP) > 0 { netState = "up" } addNetwork.State = netState addNetwork.Type = netType addNetwork.Mtu = int(addr.ifa_mtu) if initPID != 0 && int(addr.ifa_ifindex_peer) > 0 { for _, hostInterface := range hostInterfaces { if hostInterface.Index == int(addr.ifa_ifindex_peer) { addNetwork.HostName = hostInterface.Name break } } } // Addresses if addr.ifa_addr != nil && (addr.ifa_addr.sa_family == C.AF_INET || addr.ifa_addr.sa_family == C.AF_INET6) { family := "inet" if addr.ifa_addr.sa_family == C.AF_INET6 { family = "inet6" } addrPtr := C.get_addr_ptr(addr.ifa_addr) if addrPtr == nil { return nil, errors.New("Failed to retrieve valid address pointer") } addressStr := C.inet_ntop(C.int(addr.ifa_addr.sa_family), addrPtr, &address[0], C.INET6_ADDRSTRLEN) if addressStr == nil { return nil, errors.New("Failed to retrieve address string") } if addNetwork.Addresses == nil { addNetwork.Addresses = []api.InstanceStateNetworkAddress{} } goAddrString := C.GoString(addressStr) scope := "global" if strings.HasPrefix(goAddrString, "127") { scope = "local" } if goAddrString == "::1" { scope = "local" } if strings.HasPrefix(goAddrString, "169.254") { scope = "link" } if strings.HasPrefix(goAddrString, "fe80:") { scope = "link" } address := api.InstanceStateNetworkAddress{} address.Family = family address.Address = goAddrString address.Netmask = fmt.Sprintf("%d", int(addr.ifa_prefixlen)) address.Scope = scope addNetwork.Addresses = append(addNetwork.Addresses, address) } else if addr.ifa_addr != nil && addr.ifa_addr.sa_family == C.AF_PACKET { if (addr.ifa_flags & C.IFF_LOOPBACK) == 0 { var buf [1024]C.char hwaddr := C.get_packet_address(addr.ifa_addr, &buf[0], 1024) if hwaddr == nil { return nil, errors.New("Failed to retrieve hardware address") } addNetwork.Hwaddr = C.GoString(hwaddr) } } if addr.ifa_stats_type == C.IFLA_STATS64 { addNetwork.Counters.BytesReceived = int64(addr.ifa_stats64.rx_bytes) addNetwork.Counters.BytesSent = int64(addr.ifa_stats64.tx_bytes) addNetwork.Counters.PacketsReceived = int64(addr.ifa_stats64.rx_packets) addNetwork.Counters.PacketsSent = int64(addr.ifa_stats64.tx_packets) addNetwork.Counters.ErrorsReceived = int64(addr.ifa_stats64.rx_errors) addNetwork.Counters.ErrorsSent = int64(addr.ifa_stats64.tx_errors) addNetwork.Counters.PacketsDroppedInbound = int64(addr.ifa_stats64.rx_dropped) addNetwork.Counters.PacketsDroppedOutbound = int64(addr.ifa_stats64.tx_dropped) } ifName := C.GoString(addr.ifa_name) networks[ifName] = addNetwork } return networks, nil } // AbstractUnixSendFd sends a Unix file descriptor over a Unix socket. func AbstractUnixSendFd(sockFD int, sendFD int) error { fd := C.int(sendFD) skFd := C.int(sockFD) ret := C.lxc_abstract_unix_send_fds(skFd, &fd, C.int(1), nil, C.size_t(0)) if ret < 0 { return errors.New("Failed to send file descriptor via abstract unix socket") } return nil } // AbstractUnixReceiveFd receives a Unix file descriptor from a Unix socket. func AbstractUnixReceiveFd(sockFD int, flags uint) (*os.File, error) { skFd := C.int(sockFD) fds := C.struct_unix_fds{} fds.fd_count_max = 1 fds.flags = C.__u32(flags) ret := C.lxc_abstract_unix_recv_fds(skFd, &fds, nil, C.size_t(0)) if ret < 0 { return nil, errors.New("Failed to receive file descriptor via abstract unix socket") } if fds.fd_count_max != fds.fd_count_ret { return nil, errors.New("Failed to receive file descriptor via abstract unix socket") } file := os.NewFile(uintptr(fds.fd[0]), "") return file, nil } // AbstractUnixReceiveFdData is a low level function to receive a file descriptor over a unix socket. func AbstractUnixReceiveFdData(sockFD int, numFds int, flags uint, iov unsafe.Pointer, iovLen int32) (uint64, []C.int, error) { fds := C.struct_unix_fds{} if numFds >= C.KERNEL_SCM_MAX_FD { return 0, []C.int{-C.EBADF}, errors.New("Excessive number of file descriptors requested") } fds.fd_count_max = C.__u32(numFds) fds.flags = C.__u32(flags) skFd := C.int(sockFD) ret, errno := C.lxc_abstract_unix_recv_fds_iov(skFd, &fds, (*C.struct_iovec)(iov), C.size_t(iovLen)) if ret < 0 { return 0, []C.int{-C.EBADF}, fmt.Errorf("Failed to receive file descriptor via abstract unix socket: errno=%d", errno) } if ret == 0 { return 0, []C.int{-C.EBADF}, io.EOF } if fds.fd_count_ret == 0 { return 0, []C.int{-C.EBADF}, io.EOF } cfd := make([]C.int, numFds) // Transfer the file descriptors. for i := C.__u32(0); i < fds.fd_count_ret; i++ { cfd[i] = fds.fd[i] } // Make sure that when we received less fds than we intended any // additional entries are negative. for i := fds.fd_count_ret; i < C.__u32(numFds); i++ { cfd[i] = -1 } return uint64(ret), cfd, nil } incus-7.0.0/internal/netutils/unixfd.c000066400000000000000000000122711517523235500177760ustar00rootroot00000000000000 #ifndef _GNU_SOURCE #define _GNU_SOURCE 1 #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "unixfd.h" #include "../../shared/cgo/memory_utils.h" int lxc_abstract_unix_send_fds(int fd, int *sendfds, int num_sendfds, void *data, size_t size) { __do_free char *cmsgbuf = NULL; struct msghdr msg; struct iovec iov; struct cmsghdr *cmsg = NULL; char buf[1] = {0}; size_t cmsgbufsize = CMSG_SPACE(num_sendfds * sizeof(int)); memset(&msg, 0, sizeof(msg)); memset(&iov, 0, sizeof(iov)); cmsgbuf = malloc(cmsgbufsize); if (!cmsgbuf) return -1; msg.msg_control = cmsgbuf; msg.msg_controllen = cmsgbufsize; cmsg = CMSG_FIRSTHDR(&msg); cmsg->cmsg_level = SOL_SOCKET; cmsg->cmsg_type = SCM_RIGHTS; cmsg->cmsg_len = CMSG_LEN(num_sendfds * sizeof(int)); msg.msg_controllen = cmsg->cmsg_len; memcpy(CMSG_DATA(cmsg), sendfds, num_sendfds * sizeof(int)); iov.iov_base = data ? data : buf; iov.iov_len = data ? size : sizeof(buf); msg.msg_iov = &iov; msg.msg_iovlen = 1; return sendmsg(fd, &msg, MSG_NOSIGNAL); } ssize_t lxc_abstract_unix_recv_fds_iov(int fd, struct unix_fds *ret_fds, struct iovec *ret_iov, size_t size_ret_iov) { __do_free char *cmsgbuf = NULL; ssize_t ret; struct msghdr msg = {}; struct cmsghdr *cmsg = NULL; size_t cmsgbufsize = CMSG_SPACE(sizeof(struct ucred)) + CMSG_SPACE(ret_fds->fd_count_max * sizeof(int)); if (ret_fds->flags & ~UNIX_FDS_ACCEPT_MASK) return ret_errno(EINVAL); if (hweight32((ret_fds->flags & ~UNIX_FDS_ACCEPT_NONE)) > 1) return ret_errno(EINVAL); if (ret_fds->fd_count_max >= KERNEL_SCM_MAX_FD) return ret_errno(EINVAL); if (ret_fds->fd_count_ret != 0) return ret_errno(EINVAL); cmsgbuf = zalloc(cmsgbufsize); if (!cmsgbuf) return ret_errno(ENOMEM); msg.msg_control = cmsgbuf; msg.msg_controllen = cmsgbufsize; msg.msg_iov = ret_iov; msg.msg_iovlen = size_ret_iov; again: ret = recvmsg(fd, &msg, MSG_CMSG_CLOEXEC); if (ret < 0) { if (errno == EINTR) goto again; return -errno; } if (ret == 0) return 0; /* If SO_PASSCRED is set we will always get a ucred message. */ for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) { if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) { __u32 idx; /* * This causes some compilers to complain about * increased alignment requirements but I haven't found * a better way to deal with this yet. Suggestions * welcome! */ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-align" int *fds_raw = (int *)CMSG_DATA(cmsg); #pragma GCC diagnostic pop __u32 num_raw = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(int); /* * We received an insane amount of file descriptors * which exceeds the kernel limit we know about so * close them and return an error. */ if (num_raw >= KERNEL_SCM_MAX_FD) { for (idx = 0; idx < num_raw; idx++) close(fds_raw[idx]); return -EFBIG; } if (msg.msg_flags & MSG_CTRUNC) { for (idx = 0; idx < num_raw; idx++) close(fds_raw[idx]); return -EFBIG; } if (ret_fds->fd_count_max > num_raw) { if (!(ret_fds->flags & UNIX_FDS_ACCEPT_LESS)) { for (idx = 0; idx < num_raw; idx++) close(fds_raw[idx]); return -EINVAL; } /* * Make sure any excess entries in the fd array * are set to -EBADF so our cleanup functions * can safely be called. */ for (idx = num_raw; idx < ret_fds->fd_count_max; idx++) ret_fds->fd[idx] = -EBADF; ret_fds->flags |= UNIX_FDS_RECEIVED_LESS; } else if (ret_fds->fd_count_max < num_raw) { if (!(ret_fds->flags & UNIX_FDS_ACCEPT_MORE)) { for (idx = 0; idx < num_raw; idx++) close(fds_raw[idx]); return -EINVAL; } /* Make sure we close any excess fds we received. */ for (idx = ret_fds->fd_count_max; idx < num_raw; idx++) close(fds_raw[idx]); /* Cap the number of received file descriptors. */ num_raw = ret_fds->fd_count_max; ret_fds->flags |= UNIX_FDS_RECEIVED_MORE; } else { ret_fds->flags |= UNIX_FDS_RECEIVED_EXACT; } if (hweight32((ret_fds->flags & ~UNIX_FDS_ACCEPT_MASK)) > 1) { for (idx = 0; idx < num_raw; idx++) close(fds_raw[idx]); return -EINVAL; } memcpy(ret_fds->fd, CMSG_DATA(cmsg), num_raw * sizeof(int)); ret_fds->fd_count_ret = num_raw; break; } } if (ret_fds->fd_count_ret == 0) { ret_fds->flags |= UNIX_FDS_RECEIVED_NONE; /* We expected to receive file descriptors. */ if ((ret_fds->flags & UNIX_FDS_ACCEPT_MASK) && !(ret_fds->flags & UNIX_FDS_ACCEPT_NONE)) return -EINVAL; } return ret; } ssize_t lxc_abstract_unix_recv_fds(int fd, struct unix_fds *ret_fds, void *ret_data, size_t size_ret_data) { char buf[1] = {}; struct iovec iov = { .iov_base = ret_data ? ret_data : buf, .iov_len = ret_data ? size_ret_data : sizeof(buf), }; ssize_t ret; ret = lxc_abstract_unix_recv_fds_iov(fd, ret_fds, &iov, 1); if (ret < 0) return ret; return ret; } incus-7.0.0/internal/netutils/unixfd.h000066400000000000000000000025031517523235500200000ustar00rootroot00000000000000 #ifndef NETUTILS_UNIXFD_H #define NETUTILS_UNIXFD_H #include #include #include #define KERNEL_SCM_MAX_FD 253 /* Allow the caller to set expectations. */ #define UNIX_FDS_ACCEPT_EXACT ((__u32)(1 << 0)) /* default */ #define UNIX_FDS_ACCEPT_LESS ((__u32)(1 << 1)) #define UNIX_FDS_ACCEPT_MORE ((__u32)(1 << 2)) /* wipe any extra fds */ #define UNIX_FDS_ACCEPT_NONE ((__u32)(1 << 3)) #define UNIX_FDS_ACCEPT_MASK (UNIX_FDS_ACCEPT_EXACT | UNIX_FDS_ACCEPT_LESS | UNIX_FDS_ACCEPT_MORE | UNIX_FDS_ACCEPT_NONE) /* Allow the callee to disappoint them. */ #define UNIX_FDS_RECEIVED_EXACT ((__u32)(1 << 16)) #define UNIX_FDS_RECEIVED_LESS ((__u32)(1 << 17)) #define UNIX_FDS_RECEIVED_MORE ((__u32)(1 << 18)) #define UNIX_FDS_RECEIVED_NONE ((__u32)(1 << 19)) struct unix_fds { __u32 fd_count_max; __u32 fd_count_ret; __u32 flags; __s32 fd[KERNEL_SCM_MAX_FD]; } __attribute__((aligned(8))); extern int lxc_abstract_unix_send_fds(int fd, int *sendfds, int num_sendfds, void *data, size_t size); extern ssize_t lxc_abstract_unix_recv_fds_iov(int fd, struct unix_fds *ret_fds, struct iovec *ret_iov, size_t size_ret_iov); extern ssize_t lxc_abstract_unix_recv_fds(int fd, struct unix_fds *ret_fds, void *ret_data, size_t size_ret_data); #endif // NETUTILS_UNIXFD_H incus-7.0.0/internal/ports/000077500000000000000000000000001517523235500156325ustar00rootroot00000000000000incus-7.0.0/internal/ports/ports.go000066400000000000000000000004461517523235500173340ustar00rootroot00000000000000package ports // Default ports for common services. const ( BGPDefaultPort = 179 DNSDefaultPort = 53 HTTPDebugDefaultPort = 8080 HTTPSDefaultPort = 8443 HTTPSMetricsDefaultPort = 9100 HTTPSStorageBucketsDefaultPort = 9000 ) incus-7.0.0/internal/recover/000077500000000000000000000000001517523235500161305ustar00rootroot00000000000000incus-7.0.0/internal/recover/struct.go000066400000000000000000000023671517523235500200130ustar00rootroot00000000000000package recover import ( "github.com/lxc/incus/v7/shared/api" ) // ValidatePost is used to initiate a recovery validation scan. type ValidatePost struct { Pools []api.StoragePoolsPost `json:"pools" yaml:"pools"` } // ValidateVolume provides info about a missing volume that the recovery validation scan found. type ValidateVolume struct { Name string `json:"name" yaml:"name"` // Name of volume. Type string `json:"type" yaml:"type"` // Same as Type from StorageVolumesPost (container, custom or virtual-machine). SnapshotCount int `json:"snapshotCount" yaml:"snapshotCount"` // Count of snapshots found for volume. Project string `json:"project" yaml:"project"` // Project the volume belongs to. Pool string `json:"pool" yaml:"pool"` // Pool the volume belongs to. } // ValidateResult returns the result of the validation scan. type ValidateResult struct { UnknownVolumes []ValidateVolume // Volumes that could be imported. DependencyErrors []string // Errors that are preventing import from proceeding. } // ImportPost is used to initiate a recovert import. type ImportPost struct { Pools []api.StoragePoolsPost `json:"pools" yaml:"pools"` } incus-7.0.0/internal/rsync/000077500000000000000000000000001517523235500156215ustar00rootroot00000000000000incus-7.0.0/internal/rsync/rsync.go000066400000000000000000000234741517523235500173200ustar00rootroot00000000000000package rsync import ( "bytes" "errors" "fmt" "io" "net" "os" "os/exec" "slices" "time" "github.com/google/uuid" "github.com/lxc/incus/v7/internal/linux" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) // Debug controls additional debugging in rsync output. var Debug bool // RunWrapper is an optional function that's used to wrap rsync, useful for confinement like AppArmor. var RunWrapper func(cmd *exec.Cmd, source string, destination string) (func(), error) // rsync is a wrapper for the rsync command which will respect RunWrapper. func rsync(args ...string) (string, error) { if len(args) < 2 { return "", errors.New("rsync call expects a minimum of two arguments (source and destination)") } // Setup the command. cmd := exec.Command("rsync", args...) var stderr bytes.Buffer cmd.Stderr = &stderr var stdout bytes.Buffer cmd.Stdout = &stdout // Call the wrapper if defined. if RunWrapper != nil { source := args[len(args)-2] destination := args[len(args)-1] cleanup, err := RunWrapper(cmd, source, destination) if err != nil { return "", err } defer cleanup() } // Run the command. err := cmd.Run() if err != nil { return stdout.String(), subprocess.NewRunError("rsync", args, err, &stdout, &stderr) } return stdout.String(), nil } // LocalCopy copies a directory using rsync (with the --devices option). func LocalCopy(source string, dest string, bwlimit string, xattrs bool, rsyncArgs ...string) (string, error) { err := os.MkdirAll(dest, 0o755) if err != nil { return "", err } rsyncVerbosity := "-q" if Debug { rsyncVerbosity = "-vi" } args := []string{ "-a", "-HA", "--sparse", "--devices", "--delete", "--numeric-ids", // Checks for file modifications on nanoseconds granularity. "--modify-window=-1", } if xattrs { args = append(args, "--xattrs", "--filter=-x security.selinux") } if bwlimit != "" { args = append(args, "--bwlimit", bwlimit) } if len(rsyncArgs) > 0 { args = append(args, rsyncArgs...) } args = append(args, rsyncVerbosity, internalUtil.AddSlash(source), dest) msg, err := rsync(args...) if err != nil { var exitError *exec.ExitError ok := errors.As(err, &exitError) if ok { if exitError.ExitCode() == 24 { return msg, nil } } return msg, err } return msg, nil } func sendSetup(name string, path string, bwlimit string, execPath string, features []string, rsyncArgs ...string) (*exec.Cmd, net.Conn, io.ReadCloser, error) { /* * The way rsync works, it invokes a subprocess that does the actual * talking (given to it by a -E argument). Since there isn't an easy * way for us to capture this process' stdin/stdout, we just use netcat * and write to/from a unix socket. * * In principle we don't need this socket. It seems to me that some * clever invocation of rsync --server --sender and usage of that * process' stdin/stdout could work around the need for this socket, * but I couldn't get it to work. Another option would be to look at * the spawned process' first child and read/write from its * stdin/stdout, but that also seemed messy. In any case, this seems to * work just fine. */ auds := fmt.Sprintf("@incusd/%s", uuid.New().String()) // We simply copy a part of the uuid if it's longer than the allowed // maximum. That should be safe enough for our purposes. if len(auds) > linux.ABSTRACT_UNIX_SOCK_LEN-1 { auds = auds[:linux.ABSTRACT_UNIX_SOCK_LEN-1] } l, err := net.Listen("unix", auds) if err != nil { return nil, nil, nil, err } defer func() { _ = l.Close() }() /* * Here, the path /tmp/foo is ignored. Since we specify localhost, * rsync thinks we are syncing to a remote host (in this case, the * other end of the incus websocket), and so the path specified on the * --server instance of rsync takes precedence. */ rsyncCmd := fmt.Sprintf("%s netcat %s %s --", execPath, auds, name) args := []string{ "-ar", "--devices", "--numeric-ids", "--partial", "--sparse", } if bwlimit != "" { args = append(args, "--bwlimit", bwlimit) } if len(features) > 0 { args = append(args, rsyncFeatureArgs(features)...) } if len(rsyncArgs) > 0 { args = append(args, rsyncArgs...) } args = append(args, []string{ path, "localhost:/tmp/foo", "-e", rsyncCmd, }...) cmd := exec.Command("rsync", args...) // Call the wrapper if defined. if RunWrapper != nil { cleanup, err := RunWrapper(cmd, path, "") if err != nil { return nil, nil, nil, err } defer cleanup() } stderr, err := cmd.StderrPipe() if err != nil { return nil, nil, nil, err } err = cmd.Start() if err != nil { return nil, nil, nil, err } var conn *net.Conn chConn := make(chan *net.Conn, 1) go func() { conn, err := l.Accept() if err != nil { chConn <- nil return } chConn <- &conn }() select { case conn = <-chConn: if conn == nil { output, _ := io.ReadAll(stderr) _ = cmd.Process.Kill() _ = cmd.Wait() return nil, nil, nil, fmt.Errorf("Failed to connect to rsync socket (%s)", string(output)) } case <-time.After(10 * time.Second): output, _ := io.ReadAll(stderr) _ = cmd.Process.Kill() _ = cmd.Wait() return nil, nil, nil, fmt.Errorf("rsync failed to spawn after 10s (%s)", string(output)) } return cmd, *conn, stderr, nil } // Send sets up the sending half of an rsync, to recursively send the // directory pointed to by path over the websocket. func Send(name string, path string, conn io.ReadWriteCloser, tracker *ioprogress.ProgressTracker, features []string, bwlimit string, execPath string, rsyncArgs ...string) error { cmd, netcatConn, stderr, err := sendSetup(name, path, bwlimit, execPath, features, rsyncArgs...) if err != nil { return err } // Setup progress tracker. readNetcatPipe := io.ReadCloser(netcatConn) if tracker != nil { readNetcatPipe = &ioprogress.ProgressReader{ ReadCloser: netcatConn, Tracker: tracker, } } // Forward from netcat to target. chCopyNetcat := make(chan error, 1) go func() { _, err := util.SafeCopy(conn, readNetcatPipe) chCopyNetcat <- err _ = readNetcatPipe.Close() _ = netcatConn.Close() _ = conn.Close() // sends barrier message. }() // Forward from target to netcat. writeNetcatPipe := io.WriteCloser(netcatConn) chCopyTarget := make(chan error, 1) go func() { _, err := util.SafeCopy(writeNetcatPipe, conn) chCopyTarget <- err _ = writeNetcatPipe.Close() }() // Wait for rsync to complete. output, err := io.ReadAll(stderr) if err != nil { _ = cmd.Process.Kill() logger.Errorf("Rsync stderr read failed: %s: %v", path, err) } err = cmd.Wait() errs := []error{} chCopyNetcatErr := <-chCopyNetcat chCopyTargetErr := <-chCopyTarget if err != nil { errs = append(errs, err) // Try to get more info about the error. if chCopyNetcatErr != nil { errs = append(errs, chCopyNetcatErr) } if chCopyTargetErr != nil { errs = append(errs, chCopyTargetErr) } } if len(errs) > 0 { return fmt.Errorf("Rsync send failed: %s, %s: %v (%s)", name, path, errs, string(output)) } return nil } // Recv sets up the receiving half of the websocket to rsync (the other // half set up by rsync.Send), putting the contents in the directory specified // by path. func Recv(path string, conn io.ReadWriteCloser, tracker *ioprogress.ProgressTracker, features []string) error { args := []string{ "--server", "-vlogDtpre.iLsfx", "--numeric-ids", "--devices", "--partial", "--sparse", // This flag is only required on the receiving end. // Checks for file modifications on nanoseconds granularity. "--modify-window=-1", } if len(features) > 0 { args = append(args, rsyncFeatureArgs(features)...) } args = append(args, []string{".", path}...) cmd := exec.Command("rsync", args...) // Call the wrapper if defined. if RunWrapper != nil { cleanup, err := RunWrapper(cmd, "", path) if err != nil { return err } defer cleanup() } // Forward from rsync to source. stdout, err := cmd.StdoutPipe() if err != nil { return err } chCopyRsync := make(chan error, 1) go func() { _, err := util.SafeCopy(conn, stdout) _ = stdout.Close() _ = conn.Close() // sends barrier message. chCopyRsync <- err }() // Forward from source to rsync. stdin, err := cmd.StdinPipe() if err != nil { return err } readSourcePipe := io.ReadCloser(conn) if tracker != nil { readSourcePipe = &ioprogress.ProgressReader{ ReadCloser: conn, Tracker: tracker, } } chCopySource := make(chan error, 1) go func() { _, err := util.SafeCopy(stdin, readSourcePipe) _ = stdin.Close() chCopySource <- err }() stderr, err := cmd.StderrPipe() if err != nil { _ = cmd.Process.Kill() logger.Errorf("Rsync stderr read failed: %s: %v", path, err) } err = cmd.Start() if err != nil { return err } output, err := io.ReadAll(stderr) if err != nil { logger.Errorf("Rsync stderr read failed: %s: %v", path, err) } err = cmd.Wait() errs := []error{} chCopyRsyncErr := <-chCopyRsync chCopySourceErr := <-chCopySource if err != nil { errs = append(errs, err) // Try to get more info about the error. if chCopyRsyncErr != nil { errs = append(errs, chCopyRsyncErr) } if chCopySourceErr != nil { errs = append(errs, chCopySourceErr) } } if len(errs) > 0 { return fmt.Errorf("Rsync receive failed: %s: %v (%s)", path, errs, string(output)) } return nil } func rsyncFeatureArgs(features []string) []string { args := []string{} if slices.Contains(features, "xattrs") { args = append(args, "--xattrs", "--filter=-x security.selinux") } if slices.Contains(features, "delete") { args = append(args, "--delete") } if slices.Contains(features, "compress") { args = append(args, "--compress") args = append(args, "--compress-level=2") } return args } incus-7.0.0/internal/server/000077500000000000000000000000001517523235500157715ustar00rootroot00000000000000incus-7.0.0/internal/server/acme/000077500000000000000000000000001517523235500166765ustar00rootroot00000000000000incus-7.0.0/internal/server/acme/acme.go000066400000000000000000000067111517523235500201370ustar00rootroot00000000000000package acme import ( "context" "crypto/tls" "crypto/x509" "fmt" "os" "time" "github.com/lxc/incus/v7/internal/server/state" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" incustls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) // ClusterCertFilename describes the filename of the new certificate which is stored in case it // cannot be distributed in a cluster due to offline members. Incus will try to distribute this // certificate at a later stage. const ClusterCertFilename = "cluster.crt.new" // CertKeyPair describes a certificate and its private key. type CertKeyPair struct { Certificate []byte `json:"-"` PrivateKey []byte `json:"-"` } // UpdateCertificate updates the certificate. func UpdateCertificate(s *state.State, challengeType string, clustered bool, domain string, email string, caURL string, force bool) (*CertKeyPair, error) { clusterCertFilename := internalUtil.VarPath(ClusterCertFilename) l := logger.AddContext(logger.Ctx{"domain": domain, "caURL": caURL, "challenge": challengeType}) // If clusterCertFilename exists, it means that a previously issued certificate couldn't be // distributed to all cluster members and was therefore kept back. In this case, don't issue // a new certificate but return the previously issued one. if !force && clustered && util.PathExists(clusterCertFilename) { keyFilename := internalUtil.VarPath("cluster.key") clusterCert, err := os.ReadFile(clusterCertFilename) if err != nil { return nil, fmt.Errorf("Failed reading cluster certificate file: %w", err) } key, err := os.ReadFile(keyFilename) if err != nil { return nil, fmt.Errorf("Failed reading cluster key file: %w", err) } keyPair, err := tls.X509KeyPair(clusterCert, key) if err != nil { return nil, fmt.Errorf("Failed to get keypair: %w", err) } cert, err := x509.ParseCertificate(keyPair.Certificate[0]) if err != nil { return nil, fmt.Errorf("Failed to parse certificate: %w", err) } if !incustls.CertificateNeedsUpdate(domain, cert, 30*24*time.Hour) { return &CertKeyPair{ Certificate: clusterCert, PrivateKey: key, }, nil } } if util.PathExists(clusterCertFilename) { _ = os.Remove(clusterCertFilename) } // Load the certificate. certInfo, err := internalUtil.LoadCert(s.OS.VarDir) if err != nil { return nil, fmt.Errorf("Failed to load certificate and key file: %w", err) } cert, err := x509.ParseCertificate(certInfo.KeyPair().Certificate[0]) if err != nil { return nil, fmt.Errorf("Failed to parse certificate: %w", err) } if !force && !incustls.CertificateNeedsUpdate(domain, cert, 30*24*time.Hour) { l.Debug("Skipping certificate renewal as it is still valid for more than 30 days") return nil, nil } port := s.GlobalConfig.ACMEHTTP() provider, environment, resolvers := s.GlobalConfig.ACMEDNS() proxy := s.GlobalConfig.ProxyHTTPS() tmpDir, err := os.MkdirTemp("", "lego") if err != nil { return nil, fmt.Errorf("Failed to create temporary directory: %w", err) } defer func() { err := os.RemoveAll(tmpDir) if err != nil { logger.Warn("Failed to remove temporary directory", logger.Ctx{"err": err}) } }() certBytes, keyBytes, err := incustls.RunACMEChallenge(context.TODO(), tmpDir, caURL, domain, email, challengeType, provider, port, proxy, resolvers, environment) if err != nil { return nil, err } return &CertKeyPair{ Certificate: certBytes, PrivateKey: keyBytes, }, nil } incus-7.0.0/internal/server/apparmor/000077500000000000000000000000001517523235500176125ustar00rootroot00000000000000incus-7.0.0/internal/server/apparmor/apparmor.go000066400000000000000000000126241517523235500217670ustar00rootroot00000000000000package apparmor import ( "crypto/sha256" "errors" "fmt" "io" "io/fs" "os" "path/filepath" "strings" "github.com/lxc/incus/v7/internal/server/sys" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) const ( cmdLoad = "r" cmdUnload = "R" cmdParse = "Q" ) var ( aaCacheDir string aaPath = internalUtil.VarPath("security", "apparmor") aaVersion *version.DottedVersion ) // Init performs initial version and feature detection. func Init() error { // Fill in aaVersion. out, err := subprocess.RunCommand("apparmor_parser", "--version") if err != nil { return err } fields := strings.Fields(strings.Split(out, "\n")[0]) parsedVersion, err := version.Parse(fields[len(fields)-1]) if err != nil { return err } aaVersion = parsedVersion // Fill in aaCacheDir. basePath := filepath.Join(aaPath, "cache") output, err := subprocess.RunCommand("apparmor_parser", "-L", basePath, "--print-cache-dir") if err != nil { return err } aaCacheDir = strings.TrimSpace(output) return nil } // runApparmor runs the relevant AppArmor command. func runApparmor(sysOS *sys.OS, command string, name string) error { if !sysOS.AppArmorAvailable { return nil } _, err := subprocess.RunCommand("apparmor_parser", []string{ fmt.Sprintf("-%sWL", command), filepath.Join(aaPath, "cache"), filepath.Join(aaPath, "profiles", name), }...) if err != nil { return err } return nil } // createNamespace creates a new AppArmor namespace. func createNamespace(sysOS *sys.OS, name string) error { if !sysOS.AppArmorAvailable { return nil } if !sysOS.AppArmorStacking || sysOS.AppArmorStacked { return nil } p := filepath.Join("/sys/kernel/security/apparmor/policy/namespaces", name) err := os.Mkdir(p, 0o755) if err != nil && !os.IsExist(err) { return err } return nil } // deleteNamespace destroys an AppArmor namespace. func deleteNamespace(sysOS *sys.OS, name string) error { if !sysOS.AppArmorAvailable { return nil } if !sysOS.AppArmorStacking || sysOS.AppArmorStacked { return nil } p := filepath.Join("/sys/kernel/security/apparmor/policy/namespaces", name) err := os.Remove(p) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } return nil } // hasProfile checks if the profile is already loaded. func hasProfile(sysOS *sys.OS, name string) (bool, error) { mangled := strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(name, "/", "."), "<", ""), ">", "") profilesPath := "/sys/kernel/security/apparmor/policy/profiles" if util.PathExists(profilesPath) { entries, err := os.ReadDir(profilesPath) if err != nil { return false, err } for _, entry := range entries { fields := strings.Split(entry.Name(), ".") if mangled == strings.Join(fields[0:len(fields)-1], ".") { return true, nil } } } return false, nil } // parseProfile parses the profile without loading it into the kernel. func parseProfile(sysOS *sys.OS, name string) error { if !sysOS.AppArmorAvailable { return nil } return runApparmor(sysOS, cmdParse, name) } // loadProfile loads the AppArmor profile into the kernel. func loadProfile(sysOS *sys.OS, name string) error { if !sysOS.AppArmorAdmin { return nil } return runApparmor(sysOS, cmdLoad, name) } // unloadProfile removes the profile from the kernel. func unloadProfile(sysOS *sys.OS, fullName string, name string) error { if !sysOS.AppArmorAvailable { return nil } ok, err := hasProfile(sysOS, fullName) if err != nil { return err } if !ok { return nil } return runApparmor(sysOS, cmdUnload, name) } // deleteProfile unloads and delete profile and cache for a profile. func deleteProfile(sysOS *sys.OS, fullName string, name string) error { if !sysOS.AppArmorAvailable || !sysOS.AppArmorAdmin { return nil } if aaCacheDir == "" { return errors.New("Couldn't identify AppArmor cache directory") } err := unloadProfile(sysOS, fullName, name) if err != nil { return err } err = os.Remove(filepath.Join(aaCacheDir, name)) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove %s: %w", filepath.Join(aaCacheDir, name), err) } err = os.Remove(filepath.Join(aaPath, "profiles", name)) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove %s: %w", filepath.Join(aaPath, "profiles", name), err) } return nil } // parserSupports checks if the parser supports a particular feature. func parserSupports(sysOS *sys.OS, feature string) (bool, error) { if !sysOS.AppArmorAvailable { return false, nil } if aaVersion == nil { return false, errors.New("Couldn't identify AppArmor version") } if feature == "unix" { return true, nil } if feature == "userns" { minVer, err := version.NewDottedVersion("4.0.0") if err != nil { return false, err } return aaVersion.Compare(minVer) >= 0, nil } return false, nil } // profileName handles generating valid profile names. func profileName(prefix string, name string) string { separators := 1 if len(prefix) > 0 { separators = 2 } // Max length in AppArmor is 253 chars. if len(name)+len(prefix)+3+separators >= 253 { hash256 := sha256.New() _, _ = io.WriteString(hash256, name) name = fmt.Sprintf("%x", hash256.Sum(nil)) } if len(prefix) > 0 { return fmt.Sprintf("incus_%s-%s", prefix, name) } return fmt.Sprintf("incus-%s", name) } incus-7.0.0/internal/server/apparmor/archive.go000066400000000000000000000065131517523235500215670ustar00rootroot00000000000000package apparmor import ( "fmt" "os" "os/exec" "path/filepath" "strings" "github.com/google/uuid" "github.com/lxc/incus/v7/internal/server/sys" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/revert" ) // ArchiveWrapper is used as a RunWrapper in the rsync package. func ArchiveWrapper(sysOS *sys.OS, cmd *exec.Cmd, output string, allowedCmds []string) (func(), error) { if !sysOS.AppArmorAvailable { return func() {}, nil } reverter := revert.New() defer reverter.Fail() // Load the profile. profileName, err := archiveProfileLoad(sysOS, output, allowedCmds) if err != nil { return nil, fmt.Errorf("Failed to load apparmor profile: %w", err) } reverter.Add(func() { _ = deleteProfile(sysOS, profileName, profileName) }) // Resolve aa-exec. execPath, err := exec.LookPath("aa-exec") if err != nil { return nil, err } // Override the command. newArgs := []string{"aa-exec", "-p", profileName} newArgs = append(newArgs, cmd.Args...) cmd.Args = newArgs cmd.Path = execPath // All done, setup a cleanup function and disarm reverter. cleanup := func() { _ = deleteProfile(sysOS, profileName, profileName) } reverter.Success() return cleanup, nil } func archiveProfileLoad(sysOS *sys.OS, output string, allowedCommandPaths []string) (string, error) { reverter := revert.New() defer reverter.Fail() // Generate a temporary profile name. name := profileName("archive", uuid.New().String()) profilePath := filepath.Join(aaPath, "profiles", name) // Generate the profile content, err := archiveProfile(name, output, allowedCommandPaths) if err != nil { return "", err } // Write it to disk. err = os.WriteFile(profilePath, []byte(content), 0o600) if err != nil { return "", err } reverter.Add(func() { os.Remove(profilePath) }) // Load it. err = loadProfile(sysOS, name) if err != nil { return "", err } reverter.Success() return name, nil } // archiveProfile generates the AppArmor profile template from the given destination path. func archiveProfile(name string, outputPath string, allowedCommandPaths []string) (string, error) { // Attempt to deref all paths. outputPathFull, err := filepath.EvalSymlinks(outputPath) if err != nil { outputPathFull = outputPath // Use requested path if cannot resolve it. } backupsPath := internalUtil.VarPath("backups") backupsPathFull, err := filepath.EvalSymlinks(backupsPath) if err == nil { backupsPath = backupsPathFull } imagesPath := internalUtil.VarPath("images") imagesPathFull, err := filepath.EvalSymlinks(imagesPath) if err == nil { imagesPath = imagesPathFull } derefCommandPaths := make([]string, len(allowedCommandPaths)) for i, cmd := range allowedCommandPaths { cmdPath, err := exec.LookPath(cmd) if err == nil { cmd = cmdPath } cmdFull, err := filepath.EvalSymlinks(cmd) if err == nil { derefCommandPaths[i] = cmdFull } else { derefCommandPaths[i] = cmd } } // Render the profile. var sb *strings.Builder = &strings.Builder{} err = archiveProfileTpl.Execute(sb, map[string]any{ "name": name, "outputPath": outputPathFull, // Use deferenced path in AppArmor profile. "backupsPath": backupsPath, "imagesPath": imagesPath, "allowedCommandPaths": derefCommandPaths, }) if err != nil { return "", err } return sb.String(), nil } incus-7.0.0/internal/server/apparmor/archive.profile.go000066400000000000000000000012141517523235500232170ustar00rootroot00000000000000package apparmor import ( "text/template" ) var archiveProfileTpl = template.Must(template.New("archiveProfile").Parse(`#include profile "{{.name}}" { #include #include {{range $index, $element := .allowedCommandPaths}} {{$element}} mixr, {{- end }} {{ .outputPath }}/ rw, {{ .outputPath }}/** rwl, {{ .backupsPath }}/** rw, {{ .imagesPath }}/** r, signal (receive) set=("term"), # Capabilities capability chown, capability dac_override, capability dac_read_search, capability fowner, capability fsetid, capability mknod, capability setfcap, } `)) incus-7.0.0/internal/server/apparmor/instance.go000066400000000000000000000205151517523235500217500ustar00rootroot00000000000000package apparmor import ( "errors" "fmt" "io/fs" "os" "path/filepath" "strings" "github.com/lxc/incus/v7/internal/server/instance/drivers/edk2" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/project" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/internal/server/sys" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/util" ) // Internal copy of the instance interface. type instance interface { Project() api.Project Name() string ID() int ExpandedConfig() map[string]string Type() instancetype.Type LogPath() string RunPath() string Path() string DevicesPath() string IsPrivileged() bool } // InstanceProfileName returns the instance's AppArmor profile name. func InstanceProfileName(inst instance) string { path := internalUtil.VarPath("") name := fmt.Sprintf("%s_<%s>", project.Instance(inst.Project().Name, inst.Name()), path) return profileName("", name) } // InstanceNamespaceName returns the instance's AppArmor namespace. func InstanceNamespaceName(inst instance) string { // Unlike in profile names, / isn't an allowed character so replace with a -. path := strings.ReplaceAll(strings.Trim(internalUtil.VarPath(""), "/"), "/", "-") name := fmt.Sprintf("%s_<%s>", project.Instance(inst.Project().Name, inst.Name()), path) return profileName("", name) } // instanceProfileFilename returns the name of the on-disk profile name. func instanceProfileFilename(inst instance) string { name := project.Instance(inst.Project().Name, inst.Name()) return profileName("", name) } // InstanceLoad ensures that the instances's policy is loaded into the kernel so the it can boot. func InstanceLoad(sysOS *sys.OS, inst instance, extraBinaries []string) error { if inst.Type() == instancetype.Container { err := createNamespace(sysOS, InstanceNamespaceName(inst)) if err != nil { return err } } err := instanceProfileGenerate(sysOS, inst, extraBinaries) if err != nil { return err } err = loadProfile(sysOS, instanceProfileFilename(inst)) if err != nil { return err } return nil } // InstanceUnload ensures that the instances's policy namespace is unloaded to free kernel memory. // This does not delete the policy from disk or cache. func InstanceUnload(sysOS *sys.OS, inst instance) error { if inst.Type() == instancetype.Container { err := deleteNamespace(sysOS, InstanceNamespaceName(inst)) if err != nil { return err } } err := unloadProfile(sysOS, InstanceProfileName(inst), instanceProfileFilename(inst)) if err != nil { return err } return nil } // InstanceValidate generates the instance profile file and validates it. func InstanceValidate(sysOS *sys.OS, inst instance, extraBinaries []string) error { err := instanceProfileGenerate(sysOS, inst, extraBinaries) if err != nil { return err } return parseProfile(sysOS, instanceProfileFilename(inst)) } // InstanceDelete removes the policy from cache/disk. func InstanceDelete(sysOS *sys.OS, inst instance) error { return deleteProfile(sysOS, InstanceProfileName(inst), instanceProfileFilename(inst)) } // instanceProfileGenerate generates instance apparmor profile policy file. func instanceProfileGenerate(sysOS *sys.OS, inst instance, extraBinaries []string) error { /* In order to avoid forcing a profile parse (potentially slow) on * every container start, let's use AppArmor's binary policy cache, * which checks mtime of the files to figure out if the policy needs to * be regenerated. * * Since it uses mtimes, we shouldn't just always write out our local * AppArmor template; instead we should check to see whether the * template is the same as ours. If it isn't we should write our * version out so that the new changes are reflected and we definitely * force a recompile. */ profile := filepath.Join(aaPath, "profiles", instanceProfileFilename(inst)) content, err := os.ReadFile(profile) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } updated, err := instanceProfile(sysOS, inst, extraBinaries) if err != nil { return err } if string(content) != string(updated) { err = os.WriteFile(profile, []byte(updated), 0o600) if err != nil { return err } } return nil } // instanceProfile generates the AppArmor profile template from the given instance. func instanceProfile(sysOS *sys.OS, inst instance, extraBinaries []string) (string, error) { // Prepare raw.apparmor. rawContent := "" rawApparmor, ok := inst.ExpandedConfig()["raw.apparmor"] if ok { for _, line := range strings.Split(strings.Trim(rawApparmor, "\n"), "\n") { rawContent += fmt.Sprintf(" %s\n", line) } } // Check for features. unixSupported, err := parserSupports(sysOS, "unix") if err != nil { return "", err } usernsSupported, err := parserSupports(sysOS, "userns") if err != nil { return "", err } // Deref the extra binaries. for i, entry := range extraBinaries { fullPath, err := filepath.EvalSymlinks(entry) if err != nil { continue } extraBinaries[i] = fullPath } // Render the profile. var sb *strings.Builder = &strings.Builder{} if inst.Type() == instancetype.Container { err = lxcProfileTpl.Execute(sb, map[string]any{ "extra_binaries": extraBinaries, "feature_stacking": sysOS.AppArmorStacking && !sysOS.AppArmorStacked, "feature_unix": unixSupported, "feature_userns": usernsSupported, "kernel_binfmt": util.IsFalseOrEmpty(inst.ExpandedConfig()["security.privileged"]), "name": InstanceProfileName(inst), "namespace": InstanceNamespaceName(inst), "nesting": util.IsTrue(inst.ExpandedConfig()["security.nesting"]), "raw": rawContent, "unprivileged": util.IsFalseOrEmpty(inst.ExpandedConfig()["security.privileged"]) || sysOS.RunningInUserNS, "zfs_delegation": !inst.IsPrivileged() && storageDrivers.ZFSSupportsDelegation() && util.PathExists("/dev/zfs"), }) if err != nil { return "", err } } else { // AppArmor requires deref of all paths. path, err := filepath.EvalSymlinks(inst.Path()) if err != nil { return "", err } var edk2Paths []string edk2Path, err := edk2.GetenvEdk2Path() if err != nil { return "", err } if edk2Path != "" { edk2Path, err := filepath.EvalSymlinks(edk2Path) if err != nil { return "", err } edk2Paths = append(edk2Paths, edk2Path) } else { arch, err := osarch.ArchitectureGetLocalID() if err == nil { for _, installation := range edk2.GetArchitectureInstallations(arch) { if util.PathExists(installation.Path) { edk2Path, err := filepath.EvalSymlinks(installation.Path) if err != nil { return "", err } edk2Paths = append(edk2Paths, edk2Path) } } } } agentPath := "" if os.Getenv("INCUS_AGENT_PATH") != "" { agentPath, err = filepath.EvalSymlinks(os.Getenv("INCUS_AGENT_PATH")) if err != nil { return "", err } } execPath := localUtil.GetExecPath() execPathFull, err := filepath.EvalSymlinks(execPath) if err == nil { execPath = execPathFull } logPath := inst.LogPath() logPathFull, err := filepath.EvalSymlinks(logPath) if err == nil { logPath = logPathFull } // Extra (read-only) config paths. extraConfig := []string{} if util.PathExists("/etc/ceph") { extraConfig = append(extraConfig, "/etc/ceph") // See if default config points to another path. if util.PathExists("/etc/ceph/ceph.conf") { target, err := filepath.EvalSymlinks("/etc/ceph/ceph.conf") if err == nil && target != "/etc/ceph/ceph.conf" { extraConfig = append(extraConfig, filepath.Dir(target)) } } } err = qemuProfileTpl.Execute(sb, map[string]any{ "devicesPath": inst.DevicesPath(), "exePath": execPath, "extra_config": extraConfig, "extra_binaries": extraBinaries, "libraryPath": strings.Split(os.Getenv("LD_LIBRARY_PATH"), ":"), "logPath": logPath, "runPath": inst.RunPath(), "id": inst.ID(), "name": InstanceProfileName(inst), "path": path, "raw": rawContent, "edk2Paths": edk2Paths, "agentPath": agentPath, }) if err != nil { return "", err } } return sb.String(), nil } incus-7.0.0/internal/server/apparmor/instance_forkproxy.go000066400000000000000000000104461517523235500240750ustar00rootroot00000000000000package apparmor import ( "errors" "fmt" "io/fs" "os" "path/filepath" "slices" "strings" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/sys" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/util" ) // Internal copy of the device interface. type device interface { Config() deviceConfig.Device Name() string } // forkproxyProfile generates the AppArmor profile template from the given network. func forkproxyProfile(sysOS *sys.OS, inst instance, dev device) (string, error) { // Add any socket used by forkproxy. sockets := []string{} fields := strings.SplitN(dev.Config()["listen"], ":", 2) if fields[0] == "unix" && !strings.HasPrefix(fields[1], "@") { sockets = append(sockets, fields[1]) } fields = strings.SplitN(dev.Config()["connect"], ":", 2) if fields[0] == "unix" && !strings.HasPrefix(fields[1], "@") { sockets = append(sockets, fields[1]) } // AppArmor requires deref of all paths. for k := range sockets { // Skip non-existing because of the additional entry for the host side. if !util.PathExists(sockets[k]) { continue } v, err := filepath.EvalSymlinks(sockets[k]) if err != nil { return "", err } if !slices.Contains(sockets, v) { sockets = append(sockets, v) } } execPath := localUtil.GetExecPath() execPathFull, err := filepath.EvalSymlinks(execPath) if err == nil { execPath = execPathFull } // Render the profile. var sb *strings.Builder = &strings.Builder{} err = forkproxyProfileTpl.Execute(sb, map[string]any{ "name": ForkproxyProfileName(inst, dev), "varPath": internalUtil.VarPath(""), "exePath": execPath, "logPath": inst.LogPath(), "libraryPath": strings.Split(os.Getenv("LD_LIBRARY_PATH"), ":"), "sockets": sockets, }) if err != nil { return "", err } return sb.String(), nil } // ForkproxyProfileName returns the AppArmor profile name. func ForkproxyProfileName(inst instance, dev device) string { path := internalUtil.VarPath("") name := fmt.Sprintf("%s_%s_<%s>", dev.Name(), project.Instance(inst.Project().Name, inst.Name()), path) return profileName("forkproxy", name) } // forkproxyProfileFilename returns the name of the on-disk profile name. func forkproxyProfileFilename(inst instance, dev device) string { name := fmt.Sprintf("%s_%s", dev.Name(), project.Instance(inst.Project().Name, inst.Name())) return profileName("forkproxy", name) } // ForkproxyLoad ensures that the instances's policy is loaded into the kernel so the it can boot. func ForkproxyLoad(sysOS *sys.OS, inst instance, dev device) error { /* In order to avoid forcing a profile parse (potentially slow) on * every container start, let's use AppArmor's binary policy cache, * which checks mtime of the files to figure out if the policy needs to * be regenerated. * * Since it uses mtimes, we shouldn't just always write out our local * AppArmor template; instead we should check to see whether the * template is the same as ours. If it isn't we should write our * version out so that the new changes are reflected and we definitely * force a recompile. */ profile := filepath.Join(aaPath, "profiles", forkproxyProfileFilename(inst, dev)) content, err := os.ReadFile(profile) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } updated, err := forkproxyProfile(sysOS, inst, dev) if err != nil { return err } if string(content) != string(updated) { err = os.WriteFile(profile, []byte(updated), 0o600) if err != nil { return err } } err = loadProfile(sysOS, forkproxyProfileFilename(inst, dev)) if err != nil { return err } return nil } // ForkproxyUnload ensures that the instances's policy namespace is unloaded to free kernel memory. // This does not delete the policy from disk or cache. func ForkproxyUnload(sysOS *sys.OS, inst instance, dev device) error { return unloadProfile(sysOS, ForkproxyProfileName(inst, dev), forkproxyProfileFilename(inst, dev)) } // ForkproxyDelete removes the policy from cache/disk. func ForkproxyDelete(sysOS *sys.OS, inst instance, dev device) error { return deleteProfile(sysOS, ForkproxyProfileName(inst, dev), forkproxyProfileFilename(inst, dev)) } incus-7.0.0/internal/server/apparmor/instance_forkproxy.profile.go000066400000000000000000000032251517523235500255310ustar00rootroot00000000000000package apparmor import ( "text/template" ) var forkproxyProfileTpl = template.Must(template.New("forkproxyProfile").Parse(`#include profile "{{ .name }}" flags=(attach_disconnected,mediate_deleted) { #include # Capabilities capability chown, capability dac_read_search, capability dac_override, capability fowner, capability fsetid, capability kill, capability net_bind_service, capability setgid, capability setuid, capability sys_admin, capability sys_chroot, capability sys_ptrace, # Network access network inet dgram, network inet6 dgram, network inet stream, network inet6 stream, network unix stream, # Forkproxy operation {{ .logPath }}/** rw, @{PROC}/** rw, / rw, ptrace (read), ptrace (trace), /etc/machine-id r, /run/systemd/resolve/stub-resolv.conf r, /run/{resolvconf,NetworkManager,systemd/resolve,connman,netconfig}/resolv.conf r, /usr/lib/systemd/resolv.conf r, # Allow /dev/shm and /dev/dri access (for X11/Wayland) /dev/dri/** rwkl, /dev/shm/** rwkl, # Needed for the fork sub-commands {{ .exePath }} mr, @{PROC}/@{pid}/cmdline r, /{etc,lib,usr/lib}/os-release r, {{if .sockets -}} {{range $index, $element := .sockets}} {{$element}} rw, {{- end }} {{- end }} # Things that we definitely don't need deny @{PROC}/@{pid}/cgroup r, deny /sys/module/apparmor/parameters/enabled r, deny /sys/kernel/mm/transparent_hugepage/hpage_pmd_size r, deny /sys/devices/virtual/dmi/id/product_uuid r, {{if .libraryPath }} # Entries from LD_LIBRARY_PATH {{range $index, $element := .libraryPath}} {{$element}}/** mr, {{- end }} {{- end }} } `)) incus-7.0.0/internal/server/apparmor/instance_lxc.profile.go000066400000000000000000000517001517523235500242550ustar00rootroot00000000000000package apparmor import ( "text/template" ) var lxcProfileTpl = template.Must(template.New("lxcProfile").Parse(`#include profile "{{ .name }}" flags=(attach_disconnected,mediate_deleted) { ### Base profile capability, dbus, file, network, umount, {{- if .feature_userns }} userns, {{- end }} # Extra binaries {{- range $index, $element := .extra_binaries }} {{ $element }} mrix, {{- end }} # Hide common denials deny mount options=(ro,remount) -> /, deny mount options=(ro,remount,silent) -> /, # Allow normal signal handling signal (receive), signal peer=@{profile_name}, # Allow normal process handling ptrace (readby), ptrace (tracedby), ptrace peer=@{profile_name}, # Handle binfmt mount fstype=binfmt_misc -> /proc/sys/fs/binfmt_misc/, {{- if not .kernel_binfmt }} deny /proc/sys/fs/binfmt_misc/{,**} rwklx, {{- end }} {{- if .zfs_delegation }} # Handle binfmt mount fstype=zfs, {{- end }} # Handle cgroupfs mount options=(ro,nosuid,nodev,noexec,remount,strictatime) -> /sys/fs/cgroup/, # Handle configfs mount fstype=configfs -> /sys/kernel/config/, deny /sys/kernel/config/{,**} rwklx, # Handle debugfs mount fstype=debugfs -> /sys/kernel/debug/, deny /sys/kernel/debug/{,**} rwklx, # Handle efivarfs mount fstype=efivarfs -> /sys/firmware/efi/efivars/, deny /sys/firmware/efi/efivars/{,**} rwklx, # Handle tracefs mount fstype=tracefs -> /sys/kernel/tracing/, deny /sys/kernel/tracing/{,**} rwklx, # Handle fuse mount fstype=fuse, mount fstype=fuse.*, mount fstype=fusectl -> /sys/fs/fuse/connections/, # Handle hugetlbfs mount fstype=hugetlbfs, # Handle mqueue mount fstype=mqueue, # Handle proc mount fstype=proc -> /proc/, deny /proc/bus/** wklx, deny /proc/kcore rwklx, deny /proc/sysrq-trigger rwklx, deny /proc/acpi/** rwklx, # Handle securityfs (access handled separately) mount fstype=securityfs -> /sys/kernel/security/, # Handle sysfs (access handled below) mount fstype=sysfs -> /sys/, mount options=(rw,nosuid,nodev,noexec,remount) -> /sys/, # Handle /run remounts. mount options=(rw,nosuid,nodev,remount) -> /run/, # Handle ramfs (same as tmpfs) mount fstype=ramfs, # Handle tmpfs mount fstype=tmpfs, # Handle devpts mount fstype=devpts, # Allow limited modification of mount propagation mount options=(rw,slave) -> /, mount options=(rw,rslave) -> /, mount options=(rw,shared) -> /, mount options=(rw,rshared) -> /, mount options=(rw,private) -> /, mount options=(rw,rprivate) -> /, mount options=(rw,unbindable) -> /, mount options=(rw,runbindable) -> /, # Allow various ro-bind-*re*-mounts of anything except /proc, /sys and /dev/.lxc mount options=(ro,remount,bind) /[^spd]*{,/**}, mount options=(ro,remount,bind) /d[^e]*{,/**}, mount options=(ro,remount,bind) /de[^v]*{,/**}, mount options=(ro,remount,bind) /dev/.[^l]*{,/**}, mount options=(ro,remount,bind) /dev/.l[^x]*{,/**}, mount options=(ro,remount,bind) /dev/.lx[^c]*{,/**}, mount options=(ro,remount,bind) /dev/.lxc?*{,/**}, mount options=(ro,remount,bind) /dev/[^.]*{,/**}, mount options=(ro,remount,bind) /dev?*{,/**}, mount options=(ro,remount,bind) /p[^r]*{,/**}, mount options=(ro,remount,bind) /pr[^o]*{,/**}, mount options=(ro,remount,bind) /pro[^c]*{,/**}, mount options=(ro,remount,bind) /proc?*{,/**}, mount options=(ro,remount,bind) /s[^y]*{,/**}, mount options=(ro,remount,bind) /sy[^s]*{,/**}, mount options=(ro,remount,bind) /sys?*{,/**}, mount options=(ro,remount,bind,nodev) /[^spd]*{,/**}, mount options=(ro,remount,bind,nodev) /d[^e]*{,/**}, mount options=(ro,remount,bind,nodev) /de[^v]*{,/**}, mount options=(ro,remount,bind,nodev) /dev/.[^l]*{,/**}, mount options=(ro,remount,bind,nodev) /dev/.l[^x]*{,/**}, mount options=(ro,remount,bind,nodev) /dev/.lx[^c]*{,/**}, mount options=(ro,remount,bind,nodev) /dev/.lxc?*{,/**}, mount options=(ro,remount,bind,nodev) /dev/[^.]*{,/**}, mount options=(ro,remount,bind,nodev) /dev?*{,/**}, mount options=(ro,remount,bind,nodev) /p[^r]*{,/**}, mount options=(ro,remount,bind,nodev) /pr[^o]*{,/**}, mount options=(ro,remount,bind,nodev) /pro[^c]*{,/**}, mount options=(ro,remount,bind,nodev) /proc?*{,/**}, mount options=(ro,remount,bind,nodev) /s[^y]*{,/**}, mount options=(ro,remount,bind,nodev) /sy[^s]*{,/**}, mount options=(ro,remount,bind,nodev) /sys?*{,/**}, mount options=(ro,remount,bind,noexec) /[^spd]*{,/**}, mount options=(ro,remount,bind,noexec) /d[^e]*{,/**}, mount options=(ro,remount,bind,noexec) /de[^v]*{,/**}, mount options=(ro,remount,bind,noexec) /dev/.[^l]*{,/**}, mount options=(ro,remount,bind,noexec) /dev/.l[^x]*{,/**}, mount options=(ro,remount,bind,noexec) /dev/.lx[^c]*{,/**}, mount options=(ro,remount,bind,noexec) /dev/.lxc?*{,/**}, mount options=(ro,remount,bind,noexec) /dev/[^.]*{,/**}, mount options=(ro,remount,bind,noexec) /dev?*{,/**}, mount options=(ro,remount,bind,noexec) /p[^r]*{,/**}, mount options=(ro,remount,bind,noexec) /pr[^o]*{,/**}, mount options=(ro,remount,bind,noexec) /pro[^c]*{,/**}, mount options=(ro,remount,bind,noexec) /proc?*{,/**}, mount options=(ro,remount,bind,noexec) /s[^y]*{,/**}, mount options=(ro,remount,bind,noexec) /sy[^s]*{,/**}, mount options=(ro,remount,bind,noexec) /sys?*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /[^spd]*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /d[^e]*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /de[^v]*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /dev/.[^l]*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /dev/.l[^x]*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /dev/.lx[^c]*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /dev/.lxc?*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /dev/[^.]*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /dev?*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /p[^r]*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /pr[^o]*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /pro[^c]*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /proc?*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /s[^y]*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /sy[^s]*{,/**}, mount options=(ro,remount,bind,noexec,nodev) /sys?*{,/**}, mount options=(ro,remount,bind,noatime) /[^spd]*{,/**}, mount options=(ro,remount,bind,noatime) /d[^e]*{,/**}, mount options=(ro,remount,bind,noatime) /de[^v]*{,/**}, mount options=(ro,remount,bind,noatime) /dev/.[^l]*{,/**}, mount options=(ro,remount,bind,noatime) /dev/.l[^x]*{,/**}, mount options=(ro,remount,bind,noatime) /dev/.lx[^c]*{,/**}, mount options=(ro,remount,bind,noatime) /dev/.lxc?*{,/**}, mount options=(ro,remount,bind,noatime) /dev/[^.]*{,/**}, mount options=(ro,remount,bind,noatime) /dev?*{,/**}, mount options=(ro,remount,bind,noatime) /p[^r]*{,/**}, mount options=(ro,remount,bind,noatime) /pr[^o]*{,/**}, mount options=(ro,remount,bind,noatime) /pro[^c]*{,/**}, mount options=(ro,remount,bind,noatime) /proc?*{,/**}, mount options=(ro,remount,bind,noatime) /s[^y]*{,/**}, mount options=(ro,remount,bind,noatime) /sy[^s]*{,/**}, mount options=(ro,remount,bind,noatime) /sys?*{,/**}, mount options=(ro,remount,bind,nosuid) /[^spd]*{,/**}, mount options=(ro,remount,bind,nosuid) /d[^e]*{,/**}, mount options=(ro,remount,bind,nosuid) /de[^v]*{,/**}, mount options=(ro,remount,bind,nosuid) /dev/.[^l]*{,/**}, mount options=(ro,remount,bind,nosuid) /dev/.l[^x]*{,/**}, mount options=(ro,remount,bind,nosuid) /dev/.lx[^c]*{,/**}, mount options=(ro,remount,bind,nosuid) /dev/.lxc?*{,/**}, mount options=(ro,remount,bind,nosuid) /dev/[^.]*{,/**}, mount options=(ro,remount,bind,nosuid) /dev?*{,/**}, mount options=(ro,remount,bind,nosuid) /p[^r]*{,/**}, mount options=(ro,remount,bind,nosuid) /pr[^o]*{,/**}, mount options=(ro,remount,bind,nosuid) /pro[^c]*{,/**}, mount options=(ro,remount,bind,nosuid) /proc?*{,/**}, mount options=(ro,remount,bind,nosuid) /s[^y]*{,/**}, mount options=(ro,remount,bind,nosuid) /sy[^s]*{,/**}, mount options=(ro,remount,bind,nosuid) /sys?*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /[^spd]*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /d[^e]*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /de[^v]*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /dev/.[^l]*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /dev/.l[^x]*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /dev/.lx[^c]*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /dev/.lxc?*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /dev/[^.]*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /dev?*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /p[^r]*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /pr[^o]*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /pro[^c]*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /proc?*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /s[^y]*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /sy[^s]*{,/**}, mount options=(ro,remount,bind,nosuid,nodev) /sys?*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /[^spd]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /d[^e]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /de[^v]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /dev/.[^l]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /dev/.l[^x]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /dev/.lx[^c]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /dev/.lxc?*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /dev/[^.]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /dev?*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /p[^r]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /pr[^o]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /pro[^c]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /proc?*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /s[^y]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /sy[^s]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec) /sys?*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /[^spd]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /d[^e]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /de[^v]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /dev/.[^l]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /dev/.l[^x]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /dev/.lx[^c]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /dev/.lxc?*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /dev/[^.]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /dev?*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /p[^r]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /pr[^o]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /pro[^c]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /proc?*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /s[^y]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /sy[^s]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,nodev) /sys?*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /[^spd]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /d[^e]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /de[^v]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /dev/.[^l]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /dev/.l[^x]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /dev/.lx[^c]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /dev/.lxc?*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /dev/[^.]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /dev?*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /p[^r]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /pr[^o]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /pro[^c]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /proc?*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /s[^y]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /sy[^s]*{,/**}, mount options=(ro,remount,bind,nosuid,noexec,strictatime) /sys?*{,/**}, # Allow bind-mounts of anything except /proc, /sys and /dev/.lxc mount options=(rw,bind) /[^spd]*{,/**}, mount options=(rw,bind) /d[^e]*{,/**}, mount options=(rw,bind) /de[^v]*{,/**}, mount options=(rw,bind) /dev/.[^l]*{,/**}, mount options=(rw,bind) /dev/.l[^x]*{,/**}, mount options=(rw,bind) /dev/.lx[^c]*{,/**}, mount options=(rw,bind) /dev/.lxc?*{,/**}, mount options=(rw,bind) /dev/[^.]*{,/**}, mount options=(rw,bind) /dev?*{,/**}, mount options=(rw,bind) /p[^r]*{,/**}, mount options=(rw,bind) /pr[^o]*{,/**}, mount options=(rw,bind) /pro[^c]*{,/**}, mount options=(rw,bind) /proc?*{,/**}, mount options=(rw,bind) /s[^y]*{,/**}, mount options=(rw,bind) /sy[^s]*{,/**}, mount options=(rw,bind) /sys?*{,/**}, # Allow rbind-mounts of anything except /, /dev, /proc and /sys mount options=(rw,rbind) /[^spd]*{,/**}, mount options=(rw,rbind) /d[^e]*{,/**}, mount options=(rw,rbind) /de[^v]*{,/**}, mount options=(rw,rbind) /dev?*{,/**}, mount options=(rw,rbind) /p[^r]*{,/**}, mount options=(rw,rbind) /pr[^o]*{,/**}, mount options=(rw,rbind) /pro[^c]*{,/**}, mount options=(rw,rbind) /proc?*{,/**}, mount options=(rw,rbind) /s[^y]*{,/**}, mount options=(rw,rbind) /sy[^s]*{,/**}, mount options=(rw,rbind) /sys?*{,/**}, # Allow moving mounts except for /proc, /sys and /dev/.lxc mount options=(rw,move) /[^spd]*{,/**}, mount options=(rw,move) /d[^e]*{,/**}, mount options=(rw,move) /de[^v]*{,/**}, mount options=(rw,move) /dev/.[^l]*{,/**}, mount options=(rw,move) /dev/.l[^x]*{,/**}, mount options=(rw,move) /dev/.lx[^c]*{,/**}, mount options=(rw,move) /dev/.lxc?*{,/**}, mount options=(rw,move) /dev/[^.]*{,/**}, mount options=(rw,move) /dev?*{,/**}, mount options=(rw,move) /p[^r]*{,/**}, mount options=(rw,move) /pr[^o]*{,/**}, mount options=(rw,move) /pro[^c]*{,/**}, mount options=(rw,move) /proc?*{,/**}, mount options=(rw,move) /s[^y]*{,/**}, mount options=(rw,move) /sy[^s]*{,/**}, mount options=(rw,move) /sys?*{,/**}, {{- if not .nesting }} # Block dangerous paths under /proc/sys deny /proc/sys/[^fknu]*{,/**} wklx, deny /proc/sys/f[^s]*{,/**} wklx, deny /proc/sys/fs/[^b]*{,/**} wklx, deny /proc/sys/fs/b[^i]*{,/**} wklx, deny /proc/sys/fs/bi[^n]*{,/**} wklx, deny /proc/sys/fs/bin[^f]*{,/**} wklx, deny /proc/sys/fs/binf[^m]*{,/**} wklx, deny /proc/sys/fs/binfm[^t]*{,/**} wklx, deny /proc/sys/fs/binfmt[^_]*{,/**} wklx, deny /proc/sys/fs/binfmt_[^m]*{,/**} wklx, deny /proc/sys/fs/binfmt_m[^i]*{,/**} wklx, deny /proc/sys/fs/binfmt_mi[^s]*{,/**} wklx, deny /proc/sys/fs/binfmt_mis[^c]*{,/**} wklx, deny /proc/sys/fs/binfmt_misc?*{,/**} wklx, deny /proc/sys/fs?*{,/**} wklx, deny /proc/sys/k[^e]*{,/**} wklx, deny /proc/sys/ke[^r]*{,/**} wklx, deny /proc/sys/ker[^n]*{,/**} wklx, deny /proc/sys/kern[^e]*{,/**} wklx, deny /proc/sys/kerne[^l]*{,/**} wklx, deny /proc/sys/kernel/[^smhd]*{,/**} wklx, deny /proc/sys/kernel/d[^o]*{,/**} wklx, deny /proc/sys/kernel/do[^m]*{,/**} wklx, deny /proc/sys/kernel/dom[^a]*{,/**} wklx, deny /proc/sys/kernel/doma[^i]*{,/**} wklx, deny /proc/sys/kernel/domai[^n]*{,/**} wklx, deny /proc/sys/kernel/domain[^n]*{,/**} wklx, deny /proc/sys/kernel/domainn[^a]*{,/**} wklx, deny /proc/sys/kernel/domainna[^m]*{,/**} wklx, deny /proc/sys/kernel/domainnam[^e]*{,/**} wklx, deny /proc/sys/kernel/domainname?*{,/**} wklx, deny /proc/sys/kernel/h[^o]*{,/**} wklx, deny /proc/sys/kernel/ho[^s]*{,/**} wklx, deny /proc/sys/kernel/hos[^t]*{,/**} wklx, deny /proc/sys/kernel/host[^n]*{,/**} wklx, deny /proc/sys/kernel/hostn[^a]*{,/**} wklx, deny /proc/sys/kernel/hostna[^m]*{,/**} wklx, deny /proc/sys/kernel/hostnam[^e]*{,/**} wklx, deny /proc/sys/kernel/hostname?*{,/**} wklx, deny /proc/sys/kernel/m[^s]*{,/**} wklx, deny /proc/sys/kernel/ms[^g]*{,/**} wklx, deny /proc/sys/kernel/msg*/** wklx, deny /proc/sys/kernel/s[^he]*{,/**} wklx, deny /proc/sys/kernel/se[^m]*{,/**} wklx, deny /proc/sys/kernel/sem*/** wklx, deny /proc/sys/kernel/sh[^m]*{,/**} wklx, deny /proc/sys/kernel/shm*/** wklx, deny /proc/sys/kernel?*{,/**} wklx, deny /proc/sys/n[^e]*{,/**} wklx, deny /proc/sys/ne[^t]*{,/**} wklx, deny /proc/sys/net?*{,/**} wklx, deny /proc/sys/u[^s]*{,/**} wklx, deny /proc/sys/us[^e]*{,/**} wklx, deny /proc/sys/use[^r]*{,/**} wklx, deny /proc/sys/user?*{,/**} wklx, # Block dangerous paths under /sys deny /sys/[^fdck]*{,/**} wklx, deny /sys/c[^l]*{,/**} wklx, deny /sys/cl[^a]*{,/**} wklx, deny /sys/cla[^s]*{,/**} wklx, deny /sys/clas[^s]*{,/**} wklx, deny /sys/class/[^n]*{,/**} wklx, deny /sys/class/n[^e]*{,/**} wklx, deny /sys/class/ne[^t]*{,/**} wklx, deny /sys/class/net?*{,/**} wklx, deny /sys/class?*{,/**} wklx, deny /sys/d[^e]*{,/**} wklx, deny /sys/de[^v]*{,/**} wklx, deny /sys/dev[^i]*{,/**} wklx, deny /sys/devi[^c]*{,/**} wklx, deny /sys/devic[^e]*{,/**} wklx, deny /sys/device[^s]*{,/**} wklx, deny /sys/devices/[^v]*{,/**} wklx, deny /sys/devices/v[^i]*{,/**} wklx, deny /sys/devices/vi[^r]*{,/**} wklx, deny /sys/devices/vir[^t]*{,/**} wklx, deny /sys/devices/virt[^u]*{,/**} wklx, deny /sys/devices/virtu[^a]*{,/**} wklx, deny /sys/devices/virtua[^l]*{,/**} wklx, deny /sys/devices/virtual/[^n]*{,/**} wklx, deny /sys/devices/virtual/n[^e]*{,/**} wklx, deny /sys/devices/virtual/ne[^t]*{,/**} wklx, deny /sys/devices/virtual/net?*{,/**} wklx, deny /sys/devices/virtual?*{,/**} wklx, deny /sys/devices?*{,/**} wklx, deny /sys/f[^s]*{,/**} wklx, deny /sys/fs/[^bc]*{,/**} wklx, deny /sys/fs/b[^p]*{,/**} wklx, deny /sys/fs/bp[^f]*{,/**} wklx, deny /sys/fs/bpf?*{,/**} wklx, deny /sys/fs/c[^g]*{,/**} wklx, deny /sys/fs/cg[^r]*{,/**} wklx, deny /sys/fs/cgr[^o]*{,/**} wklx, deny /sys/fs/cgro[^u]*{,/**} wklx, deny /sys/fs/cgrou[^p]*{,/**} wklx, deny /sys/fs/cgroup?*{,/**} wklx, deny /sys/fs?*{,/**} wklx, {{- end }} {{- if .feature_unix }} ### Feature: unix # Allow receive via unix sockets from anywhere unix (receive), # Allow all unix in the container unix peer=(label=@{profile_name}), {{- end }} ### Feature: cgroup namespace mount fstype=cgroup -> /sys/fs/cgroup/**, mount fstype=cgroup2 -> /sys/fs/cgroup/**, {{- if .feature_stacking }} {{- if not .nesting }} ### Feature: apparmor stacking deny /sys/k[^e]*{,/**} wklx, deny /sys/ke[^r]*{,/**} wklx, deny /sys/ker[^n]*{,/**} wklx, deny /sys/kern[^e]*{,/**} wklx, deny /sys/kerne[^l]*{,/**} wklx, deny /sys/kernel/[^s]*{,/**} wklx, deny /sys/kernel/s[^e]*{,/**} wklx, deny /sys/kernel/se[^c]*{,/**} wklx, deny /sys/kernel/sec[^u]*{,/**} wklx, deny /sys/kernel/secu[^r]*{,/**} wklx, deny /sys/kernel/secur[^i]*{,/**} wklx, deny /sys/kernel/securi[^t]*{,/**} wklx, deny /sys/kernel/securit[^y]*{,/**} wklx, deny /sys/kernel/security/[^a]*{,/**} wklx, deny /sys/kernel/security/a[^p]*{,/**} wklx, deny /sys/kernel/security/ap[^p]*{,/**} wklx, deny /sys/kernel/security/app[^a]*{,/**} wklx, deny /sys/kernel/security/appa[^r]*{,/**} wklx, deny /sys/kernel/security/appar[^m]*{,/**} wklx, deny /sys/kernel/security/apparm[^o]*{,/**} wklx, deny /sys/kernel/security/apparmo[^r]*{,/**} wklx, deny /sys/kernel/security/apparmor?*{,/**} wklx, deny /sys/kernel/security?*{,/**} wklx, deny /sys/kernel?*{,/**} wklx, {{- end }} change_profile -> ":{{ .namespace }}:*", change_profile -> ":{{ .namespace }}://*", {{- else }} ### Feature: apparmor stacking (not present) {{- if not .nesting }} deny /sys/k*{,/**} wklx, {{- end }} {{- end }} {{- if .nesting }} ### Configuration: nesting pivot_root, # Allow sending signals and tracing children namespaces ptrace, signal, # Prevent access to hidden proc/sys mounts deny /dev/.lxc/proc/** rw, deny /dev/.lxc/sys/** rw, # Allow mounting proc and sysfs in the container mount fstype=proc -> /usr/lib/*/lxc/**, mount fstype=sysfs -> /usr/lib/*/lxc/**, # Allow nested Incus mount none -> /var/lib/incus/shmounts/, mount /var/lib/incus/shmounts/ -> /var/lib/incus/shmounts/, mount options=bind /var/lib/incus/shmounts/** -> /var/lib/incus/**, # FIXME: There doesn't seem to be a way to ask for: # mount options=(ro,nosuid,nodev,noexec,remount,bind), # as we always get mount to $cdir/proc/sys with those flags denied # So allow all mounts until that is straightened out: mount, {{- if not .feature_stacking }} change_profile -> "{{ .name }}", {{- end }} {{- end }} {{- if .unprivileged }} ### Configuration: unprivileged containers pivot_root, mount, {{- end }} {{- if .raw }} ### Configuration: raw.apparmor {{ .raw }} {{- end }} } `)) incus-7.0.0/internal/server/apparmor/instance_qemu.profile.go000066400000000000000000000061651517523235500244430ustar00rootroot00000000000000package apparmor import ( "text/template" ) var qemuProfileTpl = template.Must(template.New("qemuProfile").Parse(`#include profile "{{ .name }}" flags=(attach_disconnected,mediate_deleted) { #include #include #include capability dac_override, capability dac_read_search, capability ipc_lock, capability setgid, capability setuid, capability sys_chroot, capability sys_rawio, capability sys_resource, # Needed by qemu /dev/hugepages/** rw, /dev/kvm rw, /dev/net/tun rw, /dev/ptmx rw, /dev/sev rw, /dev/vfio/** rw, /dev/vhost-net rw, /dev/vhost-vsock rw, /etc/machine-id r, /run/udev/data/* r, @{PROC}/sys/vm/max_map_count r, @{PROC}/@{pid}/cpuset r, @{PROC}/@{pid}/gid_map r, @{PROC}/@{pid}/uid_map r, @{PROC}/@{pid}/task/*/comm rw, /sys/bus/ r, /sys/bus/nd/devices/ r, /sys/bus/usb/devices/ r, /sys/bus/usb/devices/** r, /sys/class/ r, /sys/devices/** r, /sys/module/vhost/** r, /tmp/incus_sev_* r, {{- range $index, $element := .edk2Paths }} {{ $element }}/** kr, {{- end }} /usr/share/qemu/** kr, /usr/share/seabios/** kr, /etc/nsswitch.conf r, /etc/passwd r, /etc/group r, @{PROC}/version r, # Extra config paths {{- range $index, $element := .extra_config }} {{ $element }}/** kr, {{- end }} # Extra binaries {{- range $index, $element := .extra_binaries }} {{ $element }} mrix, {{- end }} # Used by qemu for live migration NBD server and client unix (bind, listen, accept, send, receive, connect) type=stream, # Instance specific paths {{ .logPath }}/** rwk, {{ .runPath }}/** rwk, {{ .path }}/** rwk, {{ .devicesPath }}/** rwk, /tmp/incus_screenshot_{{ .id }} rwk, # Needed for the fork sub-commands {{ .exePath }} mr, @{PROC}/@{pid}/cmdline r, /{etc,lib,usr/lib}/os-release r, # Things that we definitely don't need deny @{PROC}/@{pid}/cgroup r, deny /sys/module/apparmor/parameters/enabled r, deny /sys/kernel/mm/transparent_hugepage/hpage_pmd_size r, deny /etc/gss/mech.d/ r, deny /etc/ssl/openssl.cnf r, {{if .agentPath -}} {{ .agentPath }}/ r, {{ .agentPath }}/* r, {{- end }} {{if .libraryPath -}} # Entries from LD_LIBRARY_PATH {{range $index, $element := .libraryPath}} {{$element}}/** mr, {{- end }} {{- end }} {{- if .raw }} ### Configuration: raw.apparmor {{ .raw }} {{- end }} } `)) incus-7.0.0/internal/server/apparmor/network.go000066400000000000000000000036711517523235500216410ustar00rootroot00000000000000package apparmor import ( "errors" "io/fs" "os" "path/filepath" "github.com/lxc/incus/v7/internal/server/sys" ) // Internal copy of the network interface. type network interface { Config() map[string]string Name() string } // NetworkLoad ensures that the network's profiles are loaded into the kernel. func NetworkLoad(sysOS *sys.OS, n network) error { /* In order to avoid forcing a profile parse (potentially slow) on * every network start, let's use AppArmor's binary policy cache, * which checks mtime of the files to figure out if the policy needs to * be regenerated. * * Since it uses mtimes, we shouldn't just always write out our local * AppArmor template; instead we should check to see whether the * template is the same as ours. If it isn't we should write our * version out so that the new changes are reflected and we definitely * force a recompile. */ // dnsmasq profile := filepath.Join(aaPath, "profiles", dnsmasqProfileFilename(n)) content, err := os.ReadFile(profile) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } updated, err := dnsmasqProfile(sysOS, n) if err != nil { return err } if string(content) != string(updated) { err = os.WriteFile(profile, []byte(updated), 0o600) if err != nil { return err } } err = loadProfile(sysOS, dnsmasqProfileFilename(n)) if err != nil { return err } return nil } // NetworkUnload ensures that the network's profiles are unloaded to free kernel memory. // This does not delete the policy from disk or cache. func NetworkUnload(sysOS *sys.OS, n network) error { // dnsmasq err := unloadProfile(sysOS, DnsmasqProfileName(n), dnsmasqProfileFilename(n)) if err != nil { return err } return nil } // NetworkDelete removes the profiles from cache/disk. func NetworkDelete(sysOS *sys.OS, n network) error { err := deleteProfile(sysOS, DnsmasqProfileName(n), dnsmasqProfileFilename(n)) if err != nil { return err } return nil } incus-7.0.0/internal/server/apparmor/network_dnsmasq.go000066400000000000000000000020051517523235500233550ustar00rootroot00000000000000package apparmor import ( "fmt" "strings" "github.com/lxc/incus/v7/internal/server/sys" internalUtil "github.com/lxc/incus/v7/internal/util" ) // dnsmasqProfile generates the AppArmor profile template from the given network. func dnsmasqProfile(sysOS *sys.OS, n network) (string, error) { // Render the profile. var sb *strings.Builder = &strings.Builder{} err := dnsmasqProfileTpl.Execute(sb, map[string]any{ "name": DnsmasqProfileName(n), "networkName": n.Name(), "logPath": internalUtil.LogPath(""), "varPath": internalUtil.VarPath(""), }) if err != nil { return "", err } return sb.String(), nil } // DnsmasqProfileName returns the AppArmor profile name. func DnsmasqProfileName(n network) string { path := internalUtil.VarPath("") name := fmt.Sprintf("%s_<%s>", n.Name(), path) return profileName("dnsmasq", name) } // dnsmasqProfileFilename returns the name of the on-disk profile name. func dnsmasqProfileFilename(n network) string { return profileName("dnsmasq", n.Name()) } incus-7.0.0/internal/server/apparmor/network_dnsmasq.profile.go000066400000000000000000000033241517523235500250210ustar00rootroot00000000000000package apparmor import ( "text/template" ) var dnsmasqProfileTpl = template.Must(template.New("dnsmasqProfile").Parse(`#include profile "{{ .name }}" flags=(attach_disconnected,mediate_deleted) { #include #include #include # Capabilities capability chown, capability net_bind_service, capability setgid, capability setuid, capability dac_override, capability dac_read_search, capability net_admin, # for DHCP server capability net_raw, # for DHCP server ping checks # Network access network inet raw, network inet6 raw, network unix stream, network unix dgram, # Network-specific paths {{ .varPath }}/networks/{{ .networkName }}/dnsmasq.hosts/{,*} r, {{ .varPath }}/networks/{{ .networkName }}/dnsmasq.leases rw, {{ .varPath }}/networks/{{ .networkName }}/dnsmasq.raw r, # Allow to restart dnsmasq signal (receive) set=("hup","kill"), # Logging path {{ .logPath }}/dnsmasq.{{ .networkName }}.log rw, # Additional system files @{PROC}/sys/net/ipv6/conf/*/mtu r, @{PROC}/@{pid}/fd/ r, /etc/localtime r, /usr/share/zoneinfo/** r, # System configuration access /etc/gai.conf r, /etc/group r, /etc/host.conf r, /etc/hosts r, /etc/nsswitch.conf r, /etc/passwd r, /etc/protocols r, /etc/resolv.conf r, /etc/resolvconf/run/resolv.conf r, /run/{resolvconf,NetworkManager,systemd/resolve,connman,netconfig}/resolv.conf r, /run/systemd/resolve/stub-resolv.conf r, /mnt/wsl/resolv.conf r, # The binary itself (for nesting) /{,usr/}sbin/dnsmasq mr, } `)) incus-7.0.0/internal/server/apparmor/qemuimg.go000066400000000000000000000107311517523235500216070ustar00rootroot00000000000000package apparmor import ( "bytes" "context" "errors" "fmt" "io" "io/fs" "os" "os/exec" "path/filepath" "strconv" "strings" "github.com/lxc/incus/v7/internal/server/sys" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/subprocess" ) type nullWriteCloser struct { io.Writer } func (nwc *nullWriteCloser) Close() error { return nil } type writerFunc func([]byte) (int, error) func (w writerFunc) Write(b []byte) (n int, err error) { return w(b) } func handleWriter(out io.Writer, hand func(int64, int64)) io.Writer { var current int64 return writerFunc(func(b []byte) (int, error) { n, _ := out.Write(b) ss := strings.Split(strings.Trim(string(b), "(%) \t\n\v\f\r"), "/") f, err := strconv.ParseFloat(ss[0], 64) if err != nil { return n, nil } percent := int64(f) if percent != current { current = percent hand(percent, 0) } return n, nil }) } // QemuImg runs qemu-img with an AppArmor profile based on the imgPath and dstPath supplied. // The first element of the cmd slice is expected to be a priority limiting command (such as nice or prlimit) and // will be added as an allowed command to the AppArmor profile. The remaining elements of the cmd slice are // expected to be the qemu-img command and its arguments. func QemuImg(sysOS *sys.OS, cmd []string, imgPath string, dstPath string, tracker *ioprogress.ProgressTracker) (string, error) { // It is assumed that command starts with a program which sets resource limits, like prlimit or nice allowedCmds := []string{"qemu-img", cmd[0]} allowedCmdPaths := []string{} for _, c := range allowedCmds { cmdPath, err := exec.LookPath(c) if err != nil { return "", fmt.Errorf("Failed to find executable %q: %w", c, err) } cmdFullPath, err := filepath.EvalSymlinks(cmdPath) if err == nil { cmdPath = cmdFullPath } allowedCmdPaths = append(allowedCmdPaths, cmdPath) } // Attempt to deref all paths. imgFullPath, err := filepath.EvalSymlinks(imgPath) if err == nil { imgPath = imgFullPath } if dstPath != "" { dstFullPath, err := filepath.EvalSymlinks(dstPath) if err == nil { dstPath = dstFullPath } } profileName, err := qemuImgProfileLoad(sysOS, imgPath, dstPath, allowedCmdPaths) if err != nil { return "", fmt.Errorf("Failed to load qemu-img profile: %w", err) } defer func() { _ = deleteProfile(sysOS, profileName, profileName) }() var buffer bytes.Buffer var output bytes.Buffer var writer io.Writer = &output if tracker != nil && tracker.Handler != nil { writer = handleWriter(&output, tracker.Handler) } p := subprocess.NewProcessWithFds(cmd[0], cmd[1:], nil, &nullWriteCloser{writer}, &nullWriteCloser{&buffer}) p.SetApparmor(profileName) err = p.Start(context.Background()) if err != nil { return "", fmt.Errorf("Failed running qemu-img: %w", err) } _, err = p.Wait(context.Background()) if err != nil { return "", subprocess.NewRunError(cmd[0], cmd[1:], err, nil, &buffer) } return output.String(), nil } // qemuImgProfileLoad ensures that the qemu-img's policy is loaded into the kernel. func qemuImgProfileLoad(sysOS *sys.OS, imgPath string, dstPath string, allowedCmdPaths []string) (string, error) { name := fmt.Sprintf("<%s>_<%s>", strings.ReplaceAll(strings.Trim(imgPath, "/"), "/", "-"), strings.ReplaceAll(strings.Trim(dstPath, "/"), "/", "-")) profileName := profileName("qemu-img", name) profilePath := filepath.Join(aaPath, "profiles", profileName) content, err := os.ReadFile(profilePath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return "", err } updated, err := qemuImgProfile(profileName, imgPath, dstPath, allowedCmdPaths) if err != nil { return "", err } if string(content) != string(updated) { err = os.WriteFile(profilePath, []byte(updated), 0o600) if err != nil { return "", err } } err = loadProfile(sysOS, profileName) if err != nil { return "", err } return profileName, nil } // qemuImgProfile generates the AppArmor profile template from the given destination path. func qemuImgProfile(profileName string, imgPath string, dstPath string, allowedCmdPaths []string) (string, error) { // Render the profile. var sb *strings.Builder = &strings.Builder{} err := qemuImgProfileTpl.Execute(sb, map[string]any{ "name": profileName, "pathToImg": imgPath, "dstPath": dstPath, "allowedCmdPaths": allowedCmdPaths, "libraryPath": strings.Split(os.Getenv("LD_LIBRARY_PATH"), ":"), }) if err != nil { return "", err } return sb.String(), nil } incus-7.0.0/internal/server/apparmor/qemuimg.profile.go000066400000000000000000000014701517523235500232460ustar00rootroot00000000000000package apparmor import ( "text/template" ) var qemuImgProfileTpl = template.Must(template.New("qemuImgProfile").Parse(`#include profile "{{ .name }}" flags=(attach_disconnected,mediate_deleted) { #include capability dac_override, capability dac_read_search, capability ipc_lock, /proc/sys/vm/max_map_count r, /sys/devices/**/block/*/queue/max_segments r, /sys/devices/**/block/*/queue/zoned r, /sys/devices/system/node/ r, /sys/devices/system/node/** r, {{range $index, $element := .allowedCmdPaths}} {{$element}} mixr, {{- end }} {{ .pathToImg }} rk, {{- if .dstPath }} {{ .dstPath }} rwk, {{- end }} {{if .libraryPath -}} # Entries from LD_LIBRARY_PATH {{range $index, $element := .libraryPath}} {{$element}}/** mr, {{- end }} {{- end }} } `)) incus-7.0.0/internal/server/apparmor/qemuimg_test.go000066400000000000000000000013021517523235500226400ustar00rootroot00000000000000package apparmor import ( "bytes" "fmt" "testing" ) func TestHandleWriter(t *testing.T) { status := []int64{} var buffer bytes.Buffer out := &nullWriteCloser{handleWriter(&buffer, func(percent int64, _ int64) { status = append(status, percent) })} for i := range 101 { for j := range 100 { n, err := fmt.Fprintf(out, "\t (%02d.%02d/100%s)\r", i, j, "%") if err != nil { t.Fatal(err, n) } if i == 100 { break } } } if len(status) != 100 { t.Fatal(status) } for i := int64(1); i < 101; i++ { if status[i-1] != i { t.Fatal(status[i], i) } } output := buffer.String() // Do not check output carefully if len(output) == 0 { t.Fatal(output) } } incus-7.0.0/internal/server/apparmor/rsync.go000066400000000000000000000102311517523235500212740ustar00rootroot00000000000000package apparmor import ( "fmt" "os" "os/exec" "path/filepath" "strings" "text/template" "github.com/google/uuid" "github.com/lxc/incus/v7/internal/server/sys" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/revert" ) var rsyncProfileTpl = template.Must(template.New("rsyncProfile").Parse(`#include profile "{{ .name }}" flags=(attach_disconnected,mediate_deleted) { #include capability chown, capability dac_override, capability dac_read_search, capability fowner, capability fsetid, capability mknod, capability setfcap, unix (connect, send, receive) type=stream, @{PROC}/@{pid}/cmdline r, @{PROC}/@{pid}/cpuset r, /{etc,lib,usr/lib}/os-release r, {{ .logPath }}/*/netcat.log rw, /run/{resolvconf,NetworkManager,systemd/resolve,connman,netconfig}/resolv.conf r, /run/systemd/resolve/stub-resolv.conf r, {{- if .sourcePath }} {{ .sourcePath }}/** r, {{ .sourcePath }}/ r, {{- end }} {{- if .dstPath }} {{ .dstPath }}/** rwkl, {{ .dstPath }}/ rwkl, {{- end }} {{ .execPath }} mixr, {{if .libraryPath -}} # Entries from LD_LIBRARY_PATH {{range $index, $element := .libraryPath}} {{$element}}/** mr, {{- end }} {{- end }} # The binary itself (for nesting) /{,usr/}bin/rsync mr, # Silence denials on files that aren't required. deny /etc/ssl/openssl.cnf r, deny /sys/devices/virtual/dmi/id/product_uuid r, deny /sys/kernel/mm/transparent_hugepage/hpage_pmd_size r, } `)) // RsyncWrapper is used as a RunWrapper in the rsync package. func RsyncWrapper(sysOS *sys.OS, cmd *exec.Cmd, sourcePath string, dstPath string) (func(), error) { if !sysOS.AppArmorAvailable { return func() {}, nil } reverter := revert.New() defer reverter.Fail() // Attempt to deref all paths. if sourcePath != "" { fullPath, err := filepath.EvalSymlinks(sourcePath) if err == nil { sourcePath = fullPath } } if dstPath != "" { fullPath, err := filepath.EvalSymlinks(dstPath) if err == nil { dstPath = fullPath } } // Load the profile. profileName, err := rsyncProfileLoad(sysOS, sourcePath, dstPath) if err != nil { return nil, fmt.Errorf("Failed to load rsync profile: %w", err) } reverter.Add(func() { _ = deleteProfile(sysOS, profileName, profileName) }) // Resolve aa-exec. execPath, err := exec.LookPath("aa-exec") if err != nil { return nil, err } // Override the command. newArgs := []string{"aa-exec", "-p", profileName} newArgs = append(newArgs, cmd.Args...) cmd.Args = newArgs cmd.Path = execPath // All done, setup a cleanup function and disarm reverter. cleanup := func() { _ = deleteProfile(sysOS, profileName, profileName) } reverter.Success() return cleanup, nil } func rsyncProfileLoad(sysOS *sys.OS, sourcePath string, dstPath string) (string, error) { reverter := revert.New() defer reverter.Fail() // Generate a temporary profile name. name := profileName("rsync", uuid.New().String()) profilePath := filepath.Join(aaPath, "profiles", name) // Generate the profile content, err := rsyncProfile(sysOS, name, sourcePath, dstPath) if err != nil { return "", err } // Write it to disk. err = os.WriteFile(profilePath, []byte(content), 0o600) if err != nil { return "", err } reverter.Add(func() { os.Remove(profilePath) }) // Load it. err = loadProfile(sysOS, name) if err != nil { return "", err } reverter.Success() return name, nil } // rsyncProfile generates the AppArmor profile template from the given destination path. func rsyncProfile(sysOS *sys.OS, name string, sourcePath string, dstPath string) (string, error) { // Render the profile. logPath := internalUtil.LogPath("") // Fully deref the executable path. execPath := sysOS.ExecPath fullPath, err := filepath.EvalSymlinks(execPath) if err == nil { execPath = fullPath } var sb *strings.Builder = &strings.Builder{} err = rsyncProfileTpl.Execute(sb, map[string]any{ "name": name, "execPath": execPath, "sourcePath": sourcePath, "dstPath": dstPath, "logPath": logPath, "libraryPath": strings.Split(os.Getenv("LD_LIBRARY_PATH"), ":"), }) if err != nil { return "", err } return sb.String(), nil } incus-7.0.0/internal/server/auth/000077500000000000000000000000001517523235500167325ustar00rootroot00000000000000incus-7.0.0/internal/server/auth/authorization.go000066400000000000000000000174131517523235500221670ustar00rootroot00000000000000package auth import ( "context" "errors" "fmt" "net/http" "github.com/lxc/incus/v7/internal/server/certificate" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) const ( // DriverTLS is the default TLS authorization driver. It is not compatible with OIDC or Candid authentication. DriverTLS string = "tls" // DriverOpenFGA provides fine-grained authorization. It is compatible with any authentication method. DriverOpenFGA string = "openfga" // DriverScriptlet provides scriptlet-based authorization. It is compatible with any authentication method. DriverScriptlet string = "scriptlet" ) // ErrUnknownDriver is the "Unknown driver" error. var ErrUnknownDriver = errors.New("Unknown driver") var authorizers = map[string]func() authorizer{ DriverTLS: func() authorizer { return &TLS{} }, DriverOpenFGA: func() authorizer { return &FGA{} }, DriverScriptlet: func() authorizer { return &Scriptlet{} }, } type authorizer interface { Authorizer init(driverName string, logger logger.Logger) error load(ctx context.Context, certificateCache *certificate.Cache, opts Opts) error } // PermissionChecker is a type alias for a function that returns whether a user has required permissions on an object. // It is returned by Authorizer.GetPermissionChecker. type PermissionChecker func(object Object) bool // Authorizer is the primary external API for this package. type Authorizer interface { Driver() string StopService(ctx context.Context) error ApplyPatch(ctx context.Context, name string) error CheckPermission(ctx context.Context, r *http.Request, object Object, entitlement Entitlement) error GetPermissionChecker(ctx context.Context, r *http.Request, entitlement Entitlement, objectType ObjectType) (PermissionChecker, error) AddProject(ctx context.Context, projectID int64, projectName string) error DeleteProject(ctx context.Context, projectID int64, projectName string) error RenameProject(ctx context.Context, projectID int64, oldName string, newName string) error AddCertificate(ctx context.Context, fingerprint string) error DeleteCertificate(ctx context.Context, fingerprint string) error AddStoragePool(ctx context.Context, storagePoolName string) error DeleteStoragePool(ctx context.Context, storagePoolName string) error AddImage(ctx context.Context, projectName string, fingerprint string) error DeleteImage(ctx context.Context, projectName string, fingerprint string) error AddImageAlias(ctx context.Context, projectName string, imageAliasName string) error DeleteImageAlias(ctx context.Context, projectName string, imageAliasName string) error RenameImageAlias(ctx context.Context, projectName string, oldAliasName string, newAliasName string) error AddInstance(ctx context.Context, projectName string, instanceName string) error DeleteInstance(ctx context.Context, projectName string, instanceName string) error RenameInstance(ctx context.Context, projectName string, oldInstanceName string, newInstanceName string) error AddNetwork(ctx context.Context, projectName string, networkName string) error DeleteNetwork(ctx context.Context, projectName string, networkName string) error RenameNetwork(ctx context.Context, projectName string, oldNetworkName string, newNetworkName string) error AddNetworkZone(ctx context.Context, projectName string, networkZoneName string) error DeleteNetworkZone(ctx context.Context, projectName string, networkZoneName string) error AddNetworkIntegration(ctx context.Context, networkIntegrationName string) error DeleteNetworkIntegration(ctx context.Context, networkIntegrationName string) error RenameNetworkIntegration(ctx context.Context, oldNetworkIntegrationName string, newNetworkIntegrationName string) error AddNetworkACL(ctx context.Context, projectName string, networkACLName string) error DeleteNetworkACL(ctx context.Context, projectName string, networkACLName string) error RenameNetworkACL(ctx context.Context, projectName string, oldNetworkACLName string, newNetworkACLName string) error AddNetworkAddressSet(ctx context.Context, projectName string, networkAddressSetName string) error DeleteNetworkAddressSet(ctx context.Context, projectName string, networkAddressSetName string) error RenameNetworkAddressSet(ctx context.Context, projectName string, oldNetworkAddressSetName string, newNetworkAddressSetName string) error AddProfile(ctx context.Context, projectName string, profileName string) error DeleteProfile(ctx context.Context, projectName string, profileName string) error RenameProfile(ctx context.Context, projectName string, oldProfileName string, newProfileName string) error AddStoragePoolVolume(ctx context.Context, projectName string, storagePoolName string, storageVolumeType string, storageVolumeName string, storageVolumeLocation string) error DeleteStoragePoolVolume(ctx context.Context, projectName string, storagePoolName string, storageVolumeType string, storageVolumeName string, storageVolumeLocation string) error RenameStoragePoolVolume(ctx context.Context, projectName string, storagePoolName string, storageVolumeType string, oldStorageVolumeName string, newStorageVolumeName string, storageVolumeLocation string) error AddStorageBucket(ctx context.Context, projectName string, storagePoolName string, storageBucketName string, storageBucketLocation string) error DeleteStorageBucket(ctx context.Context, projectName string, storagePoolName string, storageBucketName string, storageBucketLocation string) error GetInstanceAccess(ctx context.Context, projectName string, instanceName string) (*api.Access, error) GetProjectAccess(ctx context.Context, projectName string) (*api.Access, error) } // Opts is used as part of the LoadAuthorizer function so that only the relevant configuration fields are passed into a // particular driver. type Opts struct { config map[string]any projectsGetFunc func(ctx context.Context) (map[int64]string, error) resourcesFunc func() (*Resources, error) } // Resources represents a set of current API resources as Object slices for use when loading an Authorizer. type Resources struct { CertificateObjects []Object StoragePoolObjects []Object ProjectObjects []Object ImageObjects []Object ImageAliasObjects []Object InstanceObjects []Object NetworkObjects []Object NetworkACLObjects []Object NetworkAddressSetObjects []Object NetworkIntegrationObjects []Object NetworkZoneObjects []Object ProfileObjects []Object StoragePoolVolumeObjects []Object StorageBucketObjects []Object } // WithConfig can be passed into LoadAuthorizer to pass in driver specific configuration. func WithConfig(c map[string]any) func(*Opts) { return func(o *Opts) { o.config = c } } // WithProjectsGetFunc should be passed into LoadAuthorizer when DriverRBAC is used. func WithProjectsGetFunc(f func(ctx context.Context) (map[int64]string, error)) func(*Opts) { return func(o *Opts) { o.projectsGetFunc = f } } // WithResourcesFunc should be passed into LoadAuthorizer when DriverOpenFGA is used. func WithResourcesFunc(f func() (*Resources, error)) func(*Opts) { return func(o *Opts) { o.resourcesFunc = f } } // LoadAuthorizer instantiates, configures, and initializes an Authorizer. func LoadAuthorizer(ctx context.Context, driver string, logger logger.Logger, certificateCache *certificate.Cache, options ...func(opts *Opts)) (Authorizer, error) { opts := &Opts{} for _, o := range options { o(opts) } driverFunc, ok := authorizers[driver] if !ok { return nil, ErrUnknownDriver } d := driverFunc() err := d.init(driver, logger) if err != nil { return nil, fmt.Errorf("Failed to initialize authorizer: %w", err) } err = d.load(ctx, certificateCache, *opts) if err != nil { return nil, fmt.Errorf("Failed to load authorizer: %w", err) } return d, nil } incus-7.0.0/internal/server/auth/authorization_objects.go000066400000000000000000000326411517523235500237000ustar00rootroot00000000000000package auth import ( "errors" "fmt" "net/http" "net/url" "strings" "github.com/gorilla/mux" "github.com/lxc/incus/v7/internal/version" ) // Object is a string alias that represents an authorization object. These are formatted strings that // uniquely identify an API resource, and can be constructed/deconstructed reliably. // An Object is always of the form : where the identifier is a "/" delimited path containing elements that // uniquely identify a resource. If the resource is defined at the project level, the first element of this path is always the project. // Some example objects would be: // - `instance:default/c1`: Instance object in project "default" and name "c1". // - `storage_pool:local`: Storage pool object with name "local". // - `storage_volume:default/local/custom/vol1`: Storage volume object in project "default", storage pool "local", type "custom", and name "vol1". type Object string const ( // objectTypeDelimiter is the string which separates the ObjectType from the remaining elements. Object types are // statically defined and do not contain this character, so we can extract the object type from an object by splitting // the string at this character. objectTypeDelimiter = ":" // objectElementDelimiter is the string which separates the elements of an object that make it a uniquely identifiable // resource. This was chosen because the character is not allowed in the majority of Incus resource names. Nevertheless // it is still necessary to escape this character in order to reliably construct/deconstruct an Object. objectElementDelimiter = "/" ) // String implements fmt.Stringer for Object. func (o Object) String() string { return string(o) } // Type returns the ObjectType of the Object. func (o Object) Type() ObjectType { t, _, _ := strings.Cut(o.String(), objectTypeDelimiter) return ObjectType(t) } // Project returns the project of the Object if present. func (o Object) Project() string { project, _ := o.projectAndElements() return project } // Elements returns the elements that uniquely identify the authorization Object. func (o Object) Elements() []string { _, elements := o.projectAndElements() return elements } func (o Object) projectAndElements() (string, []string) { validator := objectValidators[o.Type()] _, identifier, _ := strings.Cut(o.String(), objectTypeDelimiter) var projectName string escapedObjectComponents := strings.SplitN(identifier, objectElementDelimiter, -1) components := make([]string, 0, len(escapedObjectComponents)) for i, escapedComponent := range escapedObjectComponents { if validator.requireProject && i == 0 { projectName = unescape(escapedComponent) continue } components = append(components, unescape(escapedComponent)) } return projectName, components } func (o Object) validate() error { objectType := o.Type() v, ok := objectValidators[objectType] if !ok { return fmt.Errorf("Missing validator for object of type %q", objectType) } projectName, identifierElements := o.projectAndElements() if v.requireProject && projectName == "" { return fmt.Errorf("Authorization objects of type %q require a project", objectType) } if len(identifierElements) < v.minIdentifierElements { return fmt.Errorf("Authorization objects of type %q require at least %d components to be uniquely identifiable", objectType, v.minIdentifierElements) } if len(identifierElements) > v.maxIdentifierElements { return fmt.Errorf("Authorization objects of type %q require at most %d components to be uniquely identifiable", objectType, v.maxIdentifierElements) } return nil } // objectValidator contains fields that can be used to determine if a string is a valid Object. type objectValidator struct { minIdentifierElements int maxIdentifierElements int requireProject bool } var objectValidators = map[ObjectType]objectValidator{ ObjectTypeUser: {minIdentifierElements: 1, maxIdentifierElements: 1, requireProject: false}, ObjectTypeServer: {minIdentifierElements: 1, maxIdentifierElements: 1, requireProject: false}, ObjectTypeCertificate: {minIdentifierElements: 1, maxIdentifierElements: 1, requireProject: false}, ObjectTypeStoragePool: {minIdentifierElements: 1, maxIdentifierElements: 1, requireProject: false}, ObjectTypeProject: {minIdentifierElements: 0, maxIdentifierElements: 0, requireProject: true}, ObjectTypeImage: {minIdentifierElements: 1, maxIdentifierElements: 1, requireProject: true}, ObjectTypeImageAlias: {minIdentifierElements: 1, maxIdentifierElements: 1, requireProject: true}, ObjectTypeInstance: {minIdentifierElements: 1, maxIdentifierElements: 1, requireProject: true}, ObjectTypeNetwork: {minIdentifierElements: 1, maxIdentifierElements: 1, requireProject: true}, ObjectTypeNetworkACL: {minIdentifierElements: 1, maxIdentifierElements: 1, requireProject: true}, ObjectTypeNetworkAddressSet: {minIdentifierElements: 1, maxIdentifierElements: 1, requireProject: true}, ObjectTypeNetworkIntegration: {minIdentifierElements: 1, maxIdentifierElements: 1, requireProject: false}, ObjectTypeNetworkZone: {minIdentifierElements: 1, maxIdentifierElements: 1, requireProject: true}, ObjectTypeProfile: {minIdentifierElements: 1, maxIdentifierElements: 1, requireProject: true}, ObjectTypeStorageBucket: {minIdentifierElements: 2, maxIdentifierElements: 3, requireProject: true}, ObjectTypeStorageVolume: {minIdentifierElements: 3, maxIdentifierElements: 4, requireProject: true}, } // NewObject returns an Object of the given type. The passed in arguments must be in the correct // order (as found in the URL for the resource). This function will error if an invalid object type is // given, or if the correct number of arguments is not passed in. func NewObject(objectType ObjectType, projectName string, identifierElements ...string) (Object, error) { v, ok := objectValidators[objectType] if !ok { return "", fmt.Errorf("Missing validator for object of type %q", objectType) } if v.requireProject && projectName == "" { return "", fmt.Errorf("Authorization objects of type %q require a project", objectType) } if len(identifierElements) < v.minIdentifierElements { return "", fmt.Errorf("Authorization objects of type %q require at least %d components to be uniquely identifiable", objectType, v.minIdentifierElements) } if len(identifierElements) > v.maxIdentifierElements { return "", fmt.Errorf("Authorization objects of type %q require at most %d components to be uniquely identifiable", objectType, v.maxIdentifierElements) } builder := strings.Builder{} builder.WriteString(string(objectType)) builder.WriteString(objectTypeDelimiter) if v.requireProject { builder.WriteString(escape(projectName)) if len(identifierElements) > 0 { builder.WriteString(objectElementDelimiter) } } for i, c := range identifierElements { builder.WriteString(escape(c)) if i != len(identifierElements)-1 { builder.WriteString(objectElementDelimiter) } } return Object(builder.String()), nil } // ObjectFromRequest returns an object created from the request by evaluating the given mux vars. // Mux vars must be provided in the order that they are found in the endpoint path. If the object // requires a project name, this is taken from the project query parameter unless the URL begins // with /1.0/projects. func ObjectFromRequest(r *http.Request, objectType ObjectType, expandProject func(string) string, expandFingerprint func(string, string) string, expandVolumeLocation func(string, string, string, string) string, muxVars ...string) (Object, error) { // Shortcut for server objects which don't require any arguments. if objectType == ObjectTypeServer { return ObjectServer(), nil } values, err := url.ParseQuery(r.URL.RawQuery) if err != nil { return "", err } projectName := values.Get("project") if projectName == "" { projectName = "default" } else if projectName != "default" { projectName = expandProject(projectName) } location := values.Get("target") muxValues := make([]string, 0, len(muxVars)) vars := mux.Vars(r) for _, muxVar := range muxVars { var err error var muxValue string if muxVar == "location" { // Special handling for the location which is not present as a real mux var. if location != "" { muxValue = location } else if objectType == ObjectTypeStorageVolume { muxValue = expandVolumeLocation(projectName, vars["poolName"], vars["type"], vars["volumeName"]) } if muxValue == "" { continue } } else { muxValue, err = url.PathUnescape(vars[muxVar]) if err != nil { return "", fmt.Errorf("Failed to unescape mux var %q for object type %q: %w", muxVar, objectType, err) } if muxValue == "" { return "", fmt.Errorf("Mux var %q not found for object type %q", muxVar, objectType) } // Expand fingerprints. if muxVar == "fingerprint" { muxValue = expandFingerprint(projectName, muxValue) } } muxValues = append(muxValues, muxValue) } // If using projects API we want to pass in the mux var, not the query parameter. if objectType == ObjectTypeProject && strings.HasPrefix(r.URL.Path, fmt.Sprintf("/%s/projects", version.APIVersion)) { if len(muxValues) == 0 { return "", errors.New("Missing project name path variable") } return ObjectProject(muxValues[0]), nil } return NewObject(objectType, projectName, muxValues...) } // ObjectFromString parses a string into an Object. It returns an error if the string is not valid. func ObjectFromString(objectstr string) (Object, error) { o := Object(objectstr) err := o.validate() if err != nil { return "", err } return o, nil } // ObjectUser represents a user. func ObjectUser(userName string) Object { object, _ := NewObject(ObjectTypeUser, "", userName) return object } // ObjectServer represents a server. func ObjectServer() Object { object, _ := NewObject(ObjectTypeServer, "", "incus") return object } // ObjectCertificate represents a certificate. func ObjectCertificate(fingerprint string) Object { object, _ := NewObject(ObjectTypeCertificate, "", fingerprint) return object } // ObjectStoragePool represents a storage pool. func ObjectStoragePool(storagePoolName string) Object { object, _ := NewObject(ObjectTypeStoragePool, "", storagePoolName) return object } // ObjectProject represents a project. func ObjectProject(projectName string) Object { object, _ := NewObject(ObjectTypeProject, projectName) return object } // ObjectImage represents an image. func ObjectImage(projectName string, imageFingerprint string) Object { object, _ := NewObject(ObjectTypeImage, projectName, imageFingerprint) return object } // ObjectImageAlias represents an image alias. func ObjectImageAlias(projectName string, aliasName string) Object { object, _ := NewObject(ObjectTypeImageAlias, projectName, aliasName) return object } // ObjectInstance represents an instance. func ObjectInstance(projectName string, instanceName string) Object { object, _ := NewObject(ObjectTypeInstance, projectName, instanceName) return object } // ObjectNetwork represents a network. func ObjectNetwork(projectName string, networkName string) Object { object, _ := NewObject(ObjectTypeNetwork, projectName, networkName) return object } // ObjectNetworkACL represents a network ACL. func ObjectNetworkACL(projectName string, networkACLName string) Object { object, _ := NewObject(ObjectTypeNetworkACL, projectName, networkACLName) return object } // ObjectNetworkAddressSet represents a network address set. func ObjectNetworkAddressSet(projectName string, networkAddressSetName string) Object { object, _ := NewObject(ObjectTypeNetworkAddressSet, projectName, networkAddressSetName) return object } // ObjectNetworkIntegration represents a network integration. func ObjectNetworkIntegration(networkIntegrationName string) Object { object, _ := NewObject(ObjectTypeNetworkIntegration, "", networkIntegrationName) return object } // ObjectNetworkZone represents a network zone. func ObjectNetworkZone(projectName string, networkZoneName string) Object { object, _ := NewObject(ObjectTypeNetworkZone, projectName, networkZoneName) return object } // ObjectProfile represents a profile. func ObjectProfile(projectName string, profileName string) Object { object, _ := NewObject(ObjectTypeProfile, projectName, profileName) return object } // ObjectStorageBucket represents a storage bucket. func ObjectStorageBucket(projectName string, poolName string, bucketName string, location string) Object { var object Object if location != "" { object, _ = NewObject(ObjectTypeStorageBucket, projectName, poolName, bucketName, location) } else { object, _ = NewObject(ObjectTypeStorageBucket, projectName, poolName, bucketName) } return object } // ObjectStorageVolume represents a storage volume. func ObjectStorageVolume(projectName string, poolName string, volumeType string, volumeName string, location string) Object { var object Object if location != "" { object, _ = NewObject(ObjectTypeStorageVolume, projectName, poolName, volumeType, volumeName, location) } else { object, _ = NewObject(ObjectTypeStorageVolume, projectName, poolName, volumeType, volumeName) } return object } // escape escapes only the forward slash character as this is used as a delimiter. Everything else is allowed. func escape(s string) string { return strings.ReplaceAll(s, "/", "%2F") } // unescape replaces only the escaped forward slashes. func unescape(s string) string { return strings.ReplaceAll(s, "%2F", "/") } incus-7.0.0/internal/server/auth/authorization_objects_test.go000066400000000000000000000124371517523235500247400ustar00rootroot00000000000000package auth import ( "fmt" "testing" "github.com/stretchr/testify/suite" "github.com/lxc/incus/v7/shared/tls/tlstest" ) type objectSuite struct { suite.Suite } func TestObjectSuite(t *testing.T) { suite.Run(t, &objectSuite{}) } func (s *objectSuite) TestObjectCertificate() { s.Assert().NotPanics(func() { fingerprint := tlstest.TestingKeyPair(s.T()).Fingerprint() o := ObjectCertificate(fingerprint) s.Equal(fmt.Sprintf("certificate:%s", fingerprint), string(o)) }) } func (s *objectSuite) TestObjectImage() { s.Assert().NotPanics(func() { fingerprint := tlstest.TestingKeyPair(s.T()).Fingerprint() o := ObjectImage("default", fingerprint) s.Equal(fmt.Sprintf("image:default/%s", fingerprint), string(o)) }) } func (s *objectSuite) TestObjectImageAlias() { s.Assert().NotPanics(func() { o := ObjectImageAlias("default", "image_alias_name") s.Equal("image_alias:default/image_alias_name", string(o)) }) } func (s *objectSuite) TestObjectInstance() { s.Assert().NotPanics(func() { o := ObjectInstance("default", "instance_name") s.Equal("instance:default/instance_name", string(o)) }) } func (s *objectSuite) TestObjectNetwork() { s.Assert().NotPanics(func() { o := ObjectNetwork("default", "network_name") s.Equal("network:default/network_name", string(o)) }) } func (s *objectSuite) TestObjectNetworkACL() { s.Assert().NotPanics(func() { o := ObjectNetworkACL("default", "network_acl_name") s.Equal("network_acl:default/network_acl_name", string(o)) }) } func (s *objectSuite) TestObjectNetworkAddressSet() { s.Assert().NotPanics(func() { o := ObjectNetworkAddressSet("default", "network_address_set_name") s.Equal("network_address_set:default/network_address_set_name", string(o)) }) } func (s *objectSuite) TestObjectNetworkZone() { s.Assert().NotPanics(func() { o := ObjectNetworkZone("default", "network_zone_name") s.Equal("network_zone:default/network_zone_name", string(o)) }) } func (s *objectSuite) TestObjectProfile() { s.Assert().NotPanics(func() { o := ObjectProfile("default", "profile_name") s.Equal("profile:default/profile_name", string(o)) }) } func (s *objectSuite) TestObjectProject() { s.Assert().NotPanics(func() { o := ObjectProject("default") s.Equal("project:default", string(o)) }) } func (s *objectSuite) TestObjectServer() { s.Assert().NotPanics(func() { o := ObjectServer() s.Equal("server:incus", string(o)) }) } func (s *objectSuite) TestObjectStorageBucket() { s.Assert().NotPanics(func() { o := ObjectStorageBucket("default", "pool_name", "storage_bucket_name", "") s.Equal("storage_bucket:default/pool_name/storage_bucket_name", string(o)) }) s.Assert().NotPanics(func() { o := ObjectStorageBucket("default", "pool_name", "storage_bucket_name", "location") s.Equal("storage_bucket:default/pool_name/storage_bucket_name/location", string(o)) }) } func (s *objectSuite) TestObjectStoragePool() { s.Assert().NotPanics(func() { o := ObjectStoragePool("pool_name") s.Equal("storage_pool:pool_name", string(o)) }) } func (s *objectSuite) TestObjectStorageVolume() { s.Assert().NotPanics(func() { o := ObjectStorageVolume("default", "pool_name", "volume_type", "volume_name", "") s.Equal("storage_volume:default/pool_name/volume_type/volume_name", string(o)) }) s.Assert().NotPanics(func() { o := ObjectStorageVolume("default", "pool_name", "volume_type", "volume_name", "location") s.Equal("storage_volume:default/pool_name/volume_type/volume_name/location", string(o)) }) } func (s *objectSuite) TestObjectUser() { s.Assert().NotPanics(func() { o := ObjectUser("username") s.Equal("user:username", string(o)) }) } func (s *objectSuite) TestObjectFromString() { tests := []struct { in string out Object err error }{ { in: "server:incus", out: Object("server:incus"), }, { in: "certificate:weaowiejfoiawefpajewfpoawjfepojawef", out: Object("certificate:weaowiejfoiawefpajewfpoawjfepojawef"), }, { in: "storage_pool:local", out: Object("storage_pool:local"), }, { in: "project:default", out: Object("project:default"), }, { in: "profile:default/default", out: Object("profile:default/default"), }, { in: "image:default/eoaiwenfoaiwnefoianwef", out: Object("image:default/eoaiwenfoaiwnefoianwef"), }, { in: "image_alias:default/windows11", out: Object("image_alias:default/windows11"), }, { in: "network:default/incusbr0", out: Object("network:default/incusbr0"), }, { in: "network_acl:default/acl1", out: Object("network_acl:default/acl1"), }, { in: "network_address_set:default/as1", out: Object("network_address_set:default/as1"), }, { in: "network_zone:default/example.com", out: Object("network_zone:default/example.com"), }, { in: "storage_volume:default/local/custom/vol1", out: Object("storage_volume:default/local/custom/vol1"), }, { in: "storage_bucket:default/local/bucket1", out: Object("storage_bucket:default/local/bucket1"), }, } for _, tt := range tests { o, err := ObjectFromString(tt.in) s.Equal(tt.err, err) s.Equal(tt.out, o) } } // Objects shouldn't continuously path escape. func (s *objectSuite) TestRemake() { o := ObjectProject("contains/forward/slashes") oSquared, err := ObjectFromString(o.String()) s.Nil(err) s.Equal(o.String(), oSquared.String()) } incus-7.0.0/internal/server/auth/authorization_types.go000066400000000000000000000104731517523235500234120ustar00rootroot00000000000000package auth // Entitlement is a type representation of a permission as it applies to a particular ObjectType. type Entitlement string const ( // Entitlements that apply to all resources. EntitlementCanEdit Entitlement = "can_edit" EntitlementCanView Entitlement = "can_view" // Server entitlements. EntitlementCanCreateCertificates Entitlement = "can_create_certificates" EntitlementCanCreateNetworkIntegrations Entitlement = "can_create_network_integrations" EntitlementCanCreateProjects Entitlement = "can_create_projects" EntitlementCanCreateStoragePools Entitlement = "can_create_storage_pools" EntitlementCanOverrideClusterTargetRestriction Entitlement = "can_override_cluster_target_restriction" EntitlementCanViewMetrics Entitlement = "can_view_metrics" EntitlementCanViewPrivilegedEvents Entitlement = "can_view_privileged_events" EntitlementCanViewResources Entitlement = "can_view_resources" EntitlementCanViewSensitive Entitlement = "can_view_sensitive" // Project entitlements. EntitlementCanCreateImageAliases Entitlement = "can_create_image_aliases" EntitlementCanCreateImages Entitlement = "can_create_images" EntitlementCanCreateInstances Entitlement = "can_create_instances" EntitlementCanCreateNetworkACLs Entitlement = "can_create_network_acls" EntitlementCanCreateNetworkAddressSets Entitlement = "can_create_network_address_sets" EntitlementCanCreateNetworks Entitlement = "can_create_networks" EntitlementCanCreateNetworkZones Entitlement = "can_create_network_zones" EntitlementCanCreateProfiles Entitlement = "can_create_profiles" EntitlementCanCreateStorageBuckets Entitlement = "can_create_storage_buckets" EntitlementCanCreateStorageVolumes Entitlement = "can_create_storage_volumes" EntitlementCanViewEvents Entitlement = "can_view_events" EntitlementCanViewOperations Entitlement = "can_view_operations" // Instance entitlements. EntitlementCanAccessConsole Entitlement = "can_access_console" EntitlementCanExec Entitlement = "can_exec" EntitlementCanUpdateState Entitlement = "can_update_state" // Instance and storage volume entitlements. EntitlementCanAccessFiles Entitlement = "can_access_files" EntitlementCanConnectNBD Entitlement = "can_connect_nbd" EntitlementCanConnectSFTP Entitlement = "can_connect_sftp" EntitlementCanManageBackups Entitlement = "can_manage_backups" EntitlementCanManageSnapshots Entitlement = "can_manage_snapshots" ) // ObjectType is a type of resource within Incus. type ObjectType string const ( // ObjectTypeUser represents a user. ObjectTypeUser ObjectType = "user" // ObjectTypeServer represents a server. ObjectTypeServer ObjectType = "server" // ObjectTypeCertificate represents a certificate. ObjectTypeCertificate ObjectType = "certificate" // ObjectTypeStoragePool represents a storage pool. ObjectTypeStoragePool ObjectType = "storage_pool" // ObjectTypeProject represents a project. ObjectTypeProject ObjectType = "project" // ObjectTypeImage represents an image. ObjectTypeImage ObjectType = "image" // ObjectTypeImageAlias represents an image alias. ObjectTypeImageAlias ObjectType = "image_alias" // ObjectTypeInstance represents an instance. ObjectTypeInstance ObjectType = "instance" // ObjectTypeNetwork represents a network. ObjectTypeNetwork ObjectType = "network" // ObjectTypeNetworkACL represents a network ACL. ObjectTypeNetworkACL ObjectType = "network_acl" // ObjectTypeNetworkAddressSet represents a network address set. ObjectTypeNetworkAddressSet ObjectType = "network_address_set" // ObjectTypeNetworkIntegration represents a network integration. ObjectTypeNetworkIntegration ObjectType = "network_integration" // ObjectTypeNetworkZone represents a network zone. ObjectTypeNetworkZone ObjectType = "network_zone" // ObjectTypeProfile represents a profile. ObjectTypeProfile ObjectType = "profile" // ObjectTypeStorageBucket represents a storage bucket. ObjectTypeStorageBucket ObjectType = "storage_bucket" // ObjectTypeStorageVolume represents a storage volume. ObjectTypeStorageVolume ObjectType = "storage_volume" ) const ( relationServer = "server" relationProject = "project" ) incus-7.0.0/internal/server/auth/common/000077500000000000000000000000001517523235500202225ustar00rootroot00000000000000incus-7.0.0/internal/server/auth/common/common.go000066400000000000000000000003441517523235500220420ustar00rootroot00000000000000package common // RequestDetails is a type representing an authorization request. type RequestDetails struct { Username string Protocol string IsAllProjectsRequest bool ProjectName string } incus-7.0.0/internal/server/auth/driver_common.go000066400000000000000000000235121517523235500221270ustar00rootroot00000000000000package auth import ( "context" "errors" "fmt" "net/http" "net/url" "github.com/lxc/incus/v7/internal/server/auth/common" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) type commonAuthorizer struct { driverName string logger logger.Logger } func (c *commonAuthorizer) init(driverName string, l logger.Logger) error { if l == nil { return errors.New("Cannot initialize authorizer: nil logger provided") } l = l.AddContext(logger.Ctx{"driver": driverName}) c.driverName = driverName c.logger = l return nil } type requestDetails struct { common.RequestDetails forwardedUsername string forwardedProtocol string } func (r *requestDetails) isInternalOrUnix() bool { if r.Protocol == "unix" { return true } if r.Protocol == "cluster" && (r.forwardedProtocol == "unix" || r.forwardedProtocol == "cluster" || r.forwardedProtocol == "") { return true } return false } func (r *requestDetails) username() string { if r.Protocol == "cluster" && r.forwardedUsername != "" { return r.forwardedUsername } return r.Username } func (r *requestDetails) authenticationProtocol() string { if r.Protocol == "cluster" { return r.forwardedProtocol } return r.Protocol } func (r *requestDetails) actualDetails() *common.RequestDetails { return &common.RequestDetails{ Username: r.username(), Protocol: r.authenticationProtocol(), IsAllProjectsRequest: r.IsAllProjectsRequest, ProjectName: r.ProjectName, } } func (c *commonAuthorizer) requestDetails(r *http.Request) (*requestDetails, error) { if r == nil { return nil, errors.New("Cannot inspect nil request") } else if r.URL == nil { return nil, errors.New("Request URL is not set") } val := r.Context().Value(request.CtxUsername) if val == nil { return nil, errors.New("Username not present in request context") } username, ok := val.(string) if !ok { return nil, errors.New("Request context username has incorrect type") } val = r.Context().Value(request.CtxProtocol) if val == nil { return nil, errors.New("Protocol not present in request context") } protocol, ok := val.(string) if !ok { return nil, errors.New("Request context protocol has incorrect type") } var forwardedUsername string val = r.Context().Value(request.CtxForwardedUsername) if val != nil { forwardedUsername, ok = val.(string) if !ok { return nil, errors.New("Request context forwarded username has incorrect type") } } var forwardedProtocol string val = r.Context().Value(request.CtxForwardedProtocol) if val != nil { forwardedProtocol, ok = val.(string) if !ok { return nil, errors.New("Request context forwarded username has incorrect type") } } values, err := url.ParseQuery(r.URL.RawQuery) if err != nil { return nil, fmt.Errorf("Failed to parse request query parameters: %w", err) } return &requestDetails{ RequestDetails: common.RequestDetails{ Username: username, Protocol: protocol, IsAllProjectsRequest: util.IsTrue(values.Get("all-projects")), ProjectName: request.ProjectParam(r), }, forwardedUsername: forwardedUsername, forwardedProtocol: forwardedProtocol, }, nil } func (c *commonAuthorizer) Driver() string { return c.driverName } // StopService is a no-op. func (c *commonAuthorizer) StopService(ctx context.Context) error { return nil } // ApplyPatch is a no-op. func (c *commonAuthorizer) ApplyPatch(ctx context.Context, name string) error { return nil } // AddProject is a no-op. func (c *commonAuthorizer) AddProject(ctx context.Context, projectID int64, name string) error { return nil } // DeleteProject is a no-op. func (c *commonAuthorizer) DeleteProject(ctx context.Context, projectID int64, name string) error { return nil } // RenameProject is a no-op. func (c *commonAuthorizer) RenameProject(ctx context.Context, projectID int64, oldName string, newName string) error { return nil } // AddCertificate is a no-op. func (c *commonAuthorizer) AddCertificate(ctx context.Context, fingerprint string) error { return nil } // DeleteCertificate is a no-op. func (c *commonAuthorizer) DeleteCertificate(ctx context.Context, fingerprint string) error { return nil } // AddStoragePool is a no-op. func (c *commonAuthorizer) AddStoragePool(ctx context.Context, storagePoolName string) error { return nil } // DeleteStoragePool is a no-op. func (c *commonAuthorizer) DeleteStoragePool(ctx context.Context, storagePoolName string) error { return nil } // AddImage is a no-op. func (c *commonAuthorizer) AddImage(ctx context.Context, projectName string, fingerprint string) error { return nil } // DeleteImage is a no-op. func (c *commonAuthorizer) DeleteImage(ctx context.Context, projectName string, fingerprint string) error { return nil } // AddImageAlias is a no-op. func (c *commonAuthorizer) AddImageAlias(ctx context.Context, projectName string, imageAliasName string) error { return nil } // DeleteImageAlias is a no-op. func (c *commonAuthorizer) DeleteImageAlias(ctx context.Context, projectName string, imageAliasName string) error { return nil } // RenameImageAlias is a no-op. func (c *commonAuthorizer) RenameImageAlias(ctx context.Context, projectName string, oldAliasName string, newAliasName string) error { return nil } // AddInstance is a no-op. func (c *commonAuthorizer) AddInstance(ctx context.Context, projectName string, instanceName string) error { return nil } // DeleteInstance is a no-op. func (c *commonAuthorizer) DeleteInstance(ctx context.Context, projectName string, instanceName string) error { return nil } // RenameInstance is a no-op. func (c *commonAuthorizer) RenameInstance(ctx context.Context, projectName string, oldInstanceName string, newInstanceName string) error { return nil } // AddNetwork is a no-op. func (c *commonAuthorizer) AddNetwork(ctx context.Context, projectName string, networkName string) error { return nil } // DeleteNetwork is a no-op. func (c *commonAuthorizer) DeleteNetwork(ctx context.Context, projectName string, networkName string) error { return nil } // RenameNetwork is a no-op. func (c *commonAuthorizer) RenameNetwork(ctx context.Context, projectName string, oldNetworkName string, newNetworkName string) error { return nil } // AddNetworkZone is a no-op. func (c *commonAuthorizer) AddNetworkZone(ctx context.Context, projectName string, networkZoneName string) error { return nil } // DeleteNetworkZone is a no-op. func (c *commonAuthorizer) DeleteNetworkZone(ctx context.Context, projectName string, networkZoneName string) error { return nil } // AddNetworkIntegration is a no-op. func (c *commonAuthorizer) AddNetworkIntegration(ctx context.Context, networkIntegrationName string) error { return nil } // DeleteNetworkIntegration is a no-op. func (c *commonAuthorizer) DeleteNetworkIntegration(ctx context.Context, networkIntegrationName string) error { return nil } // RenameNetworkIntegration is a no-op. func (c *commonAuthorizer) RenameNetworkIntegration(ctx context.Context, oldNetworkIntegrationName string, newNetworkIntegrationName string) error { return nil } // AddNetworkACL is a no-op. func (c *commonAuthorizer) AddNetworkACL(ctx context.Context, projectName string, networkACLName string) error { return nil } // DeleteNetworkACL is a no-op. func (c *commonAuthorizer) DeleteNetworkACL(ctx context.Context, projectName string, networkACLName string) error { return nil } // RenameNetworkACL is a no-op. func (c *commonAuthorizer) RenameNetworkACL(ctx context.Context, projectName string, oldNetworkACLName string, newNetworkACLName string) error { return nil } // AddNetworkAddressSet is a no-op. func (c *commonAuthorizer) AddNetworkAddressSet(ctx context.Context, projectName string, networkAddressSetName string) error { return nil } // DeleteNetworkAddressSet is a no-op. func (c *commonAuthorizer) DeleteNetworkAddressSet(ctx context.Context, projectName string, networkAddressSetName string) error { return nil } // RenameNetworkAddressSet is a no-op. func (c *commonAuthorizer) RenameNetworkAddressSet(ctx context.Context, projectName string, oldNetworkAddressSetName string, newNetworkAddressSetName string) error { return nil } // AddProfile is a no-op. func (c *commonAuthorizer) AddProfile(ctx context.Context, projectName string, profileName string) error { return nil } // DeleteProfile is a no-op. func (c *commonAuthorizer) DeleteProfile(ctx context.Context, projectName string, profileName string) error { return nil } // RenameProfile is a no-op. func (c *commonAuthorizer) RenameProfile(ctx context.Context, projectName string, oldProfileName string, newProfileName string) error { return nil } // AddStoragePoolVolume is a no-op. func (c *commonAuthorizer) AddStoragePoolVolume(ctx context.Context, projectName string, storagePoolName string, storageVolumeType string, storageVolumeName string, storageVolumeLocation string) error { return nil } // DeleteStoragePoolVolume is a no-op. func (c *commonAuthorizer) DeleteStoragePoolVolume(ctx context.Context, projectName string, storagePoolName string, storageVolumeType string, storageVolumeName string, storageVolumeLocation string) error { return nil } // RenameStoragePoolVolume is a no-op. func (c *commonAuthorizer) RenameStoragePoolVolume(ctx context.Context, projectName string, storagePoolName string, storageVolumeType string, oldStorageVolumeName string, newStorageVolumeName string, storageVolumeLocation string) error { return nil } // AddStorageBucket is a no-op. func (c *commonAuthorizer) AddStorageBucket(ctx context.Context, projectName string, storagePoolName string, storageBucketName string, storageBucketLocation string) error { return nil } // DeleteStorageBucket is a no-op. func (c *commonAuthorizer) DeleteStorageBucket(ctx context.Context, projectName string, storagePoolName string, storageBucketName string, storageBucketLocation string) error { return nil } incus-7.0.0/internal/server/auth/driver_openfga.go000066400000000000000000001152151517523235500222600ustar00rootroot00000000000000package auth import ( "context" "encoding/json" "errors" "fmt" "net/http" "slices" "sync" "time" openfga "github.com/openfga/go-sdk" "github.com/openfga/go-sdk/client" "github.com/openfga/go-sdk/credentials" "github.com/lxc/incus/v7/internal/server/certificate" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) // FGA represents an OpenFGA authorizer. type FGA struct { commonAuthorizer tls *TLS apiURL string apiToken string storeID string onlineMu sync.Mutex online bool shutdownCtx context.Context shutdownCancel context.CancelFunc client *client.OpenFgaClient } func (f *FGA) configure(opts Opts) error { if opts.config == nil { return errors.New("Missing OpenFGA config") } val, ok := opts.config["openfga.api.token"] if !ok || val == nil { return errors.New("Missing OpenFGA API token") } f.apiToken, ok = val.(string) if !ok { return fmt.Errorf("Expected a string for configuration key %q, got: %T", "openfga.api.token", val) } val, ok = opts.config["openfga.api.url"] if !ok || val == nil { return errors.New("Missing OpenFGA API URL") } f.apiURL, ok = val.(string) if !ok { return fmt.Errorf("Expected a string for configuration key %q, got: %T", "openfga.api.url", val) } val, ok = opts.config["openfga.store.id"] if !ok || val == nil { return errors.New("Missing OpenFGA store ID") } f.storeID, ok = val.(string) if !ok { return fmt.Errorf("Expected a string for configuration key %q, got: %T", "openfga.store.id", val) } return nil } func (f *FGA) load(ctx context.Context, certificateCache *certificate.Cache, opts Opts) error { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() err := f.configure(opts) if err != nil { return err } f.tls = &TLS{} err = f.tls.load(ctx, certificateCache, opts) if err != nil { return err } conf := client.ClientConfiguration{ ApiUrl: f.apiURL, StoreId: f.storeID, Credentials: &credentials.Credentials{ Method: credentials.CredentialsMethodApiToken, Config: &credentials.Config{ ApiToken: f.apiToken, }, }, } f.client, err = client.NewSdkClient(&conf) if err != nil { return fmt.Errorf("Failed to create OpenFGA client: %w", err) } f.shutdownCtx, f.shutdownCancel = context.WithCancel(context.Background()) // Connect in the background. go func(ctx context.Context, certificateCache *certificate.Cache, opts Opts) { first := true for { // Attempt a connection. err := f.connect(ctx, certificateCache, opts) if err == nil { if !first { logger.Warn("Connection with OpenFGA established") } f.onlineMu.Lock() defer f.onlineMu.Unlock() f.online = true return } // Handle re-tries. if first { logger.Warn("Unable to connect to the OpenFGA server, will retry every 30s", logger.Ctx{"err": err}) first = false } select { case <-time.After(30 * time.Second): continue case <-f.shutdownCtx.Done(): return } } }(f.shutdownCtx, certificateCache, opts) return nil } // StopService stops the authorizer gracefully. func (f *FGA) StopService(ctx context.Context) error { // Cancel any background routine. f.shutdownCancel() return nil } // ApplyPatch is called when an applicable server patch is run, this triggers a model re-upload. func (f *FGA) ApplyPatch(ctx context.Context, name string) error { // Always refresh the model. logger.Info("Refreshing the OpenFGA model") err := f.refreshModel(ctx) if err != nil { return err } if name == "auth_openfga_viewer" { // Add the public access permission if not set. resp, err := f.client.Check(ctx).Body(client.ClientCheckRequest{ User: "user:*", Relation: "authenticated", Object: ObjectServer().String(), }).Execute() if err != nil { return err } if !resp.GetAllowed() { err = f.sendTuples(ctx, []client.ClientTupleKey{ {User: "user:*", Relation: "authenticated", Object: ObjectServer().String()}, }, nil) if err != nil { return err } // Attempt to clear the former version of this permission. _ = f.sendTuples(ctx, nil, []client.ClientTupleKeyWithoutCondition{ {User: "user:*", Relation: "viewer", Object: ObjectServer().String()}, }) } } return nil } func (f *FGA) refreshModel(ctx context.Context) error { var builtinAuthorizationModel client.ClientWriteAuthorizationModelRequest err := json.Unmarshal([]byte(authModel), &builtinAuthorizationModel) if err != nil { return fmt.Errorf("Failed to unmarshal built in authorization model: %w", err) } _, err = f.client.WriteAuthorizationModel(ctx).Body(builtinAuthorizationModel).Execute() if err != nil { return fmt.Errorf("Failed to write the authorization model: %w", err) } return nil } func (f *FGA) connect(ctx context.Context, certificateCache *certificate.Cache, opts Opts) error { // Load current authorization model. readModelResponse, err := f.client.ReadLatestAuthorizationModel(ctx).Execute() if err != nil { return fmt.Errorf("Failed to read pre-existing OpenFGA model: %w", err) } // Check if we need to upload an initial model. if readModelResponse.AuthorizationModel == nil { logger.Info("Upload initial OpenFGA model") // Upload the model itself. err := f.refreshModel(ctx) if err != nil { return fmt.Errorf("Failed to load initial model: %w", err) } // Allow basic authenticated access. err = f.sendTuples(ctx, []client.ClientTupleKey{ {User: "user:*", Relation: "authenticated", Object: ObjectServer().String()}, }, nil) if err != nil { return err } } if opts.resourcesFunc != nil { // Start resource sync routine. go func(resourcesFunc func() (*Resources, error)) { for { resources, err := resourcesFunc() if err == nil { // resources will be nil on cluster members that shouldn't be performing updates. if resources != nil { err := f.syncResources(f.shutdownCtx, *resources) if err != nil { logger.Error("Failed background OpenFGA resource sync", logger.Ctx{"err": err}) } } } else { logger.Error("Failed getting local OpenFGA resources", logger.Ctx{"err": err}) } select { case <-time.After(time.Hour): continue case <-f.shutdownCtx.Done(): return } } }(opts.resourcesFunc) } return nil } // CheckPermission returns an error if the user does not have the given Entitlement on the given Object. func (f *FGA) CheckPermission(ctx context.Context, r *http.Request, object Object, entitlement Entitlement) error { logCtx := logger.Ctx{"object": object, "entitlement": entitlement, "url": r.URL.String(), "method": r.Method} ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() details, err := f.requestDetails(r) if err != nil { return api.StatusErrorf(http.StatusForbidden, "Failed to extract request details: %v", err) } if details.isInternalOrUnix() { return nil } // Use the TLS driver if the user authenticated with TLS. if details.authenticationProtocol() == api.AuthenticationMethodTLS { return f.tls.CheckPermission(ctx, r, object, entitlement) } // If offline, return a clear error to the user. f.onlineMu.Lock() defer f.onlineMu.Unlock() if !f.online { return api.StatusErrorf(http.StatusForbidden, "The authorization server is currently offline, please try again later") } username := details.username() logCtx["username"] = username logCtx["protocol"] = details.Protocol objectUser := ObjectUser(username) body := client.ClientCheckRequest{ User: objectUser.String(), Relation: string(entitlement), Object: object.String(), } f.logger.Debug("Checking OpenFGA relation", logCtx) resp, err := f.client.Check(ctx).Body(body).Execute() if err != nil { return fmt.Errorf("Failed to check OpenFGA relation: %w", err) } if !resp.GetAllowed() { return api.StatusErrorf(http.StatusForbidden, "User does not have entitlement %q on object %q", entitlement, object) } return nil } // GetPermissionChecker returns a function that can be used to check whether a user has the required entitlement on an authorization object. func (f *FGA) GetPermissionChecker(ctx context.Context, r *http.Request, entitlement Entitlement, objectType ObjectType) (PermissionChecker, error) { allowFunc := func(b bool) func(Object) bool { return func(Object) bool { return b } } logCtx := logger.Ctx{"object_type": objectType, "entitlement": entitlement, "url": r.URL.String(), "method": r.Method} details, err := f.requestDetails(r) if err != nil { return nil, api.StatusErrorf(http.StatusForbidden, "Failed to extract request details: %v", err) } if details.isInternalOrUnix() { return allowFunc(true), nil } // Use the TLS driver if the user authenticated with TLS. if details.authenticationProtocol() == api.AuthenticationMethodTLS { return f.tls.GetPermissionChecker(ctx, r, entitlement, objectType) } username := details.username() logCtx["username"] = username logCtx["protocol"] = details.Protocol f.logger.Debug("Listing related objects for user", logCtx) resp, err := f.client.ListObjects(ctx).Body(client.ClientListObjectsRequest{ User: ObjectUser(username).String(), Relation: string(entitlement), Type: string(objectType), }).Execute() if err != nil { return nil, fmt.Errorf("Failed to OpenFGA objects of type %q with relation %q for user %q: %w", objectType, entitlement, username, err) } objects := resp.GetObjects() return func(object Object) bool { return slices.Contains(objects, object.String()) }, nil } // AddProject adds a project to the authorizer. func (f *FGA) AddProject(ctx context.Context, _ int64, projectName string) error { writes := []client.ClientTupleKey{ { User: ObjectServer().String(), Relation: relationServer, Object: ObjectProject(projectName).String(), }, { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectProfile(projectName, "default").String(), }, } return f.updateTuples(ctx, writes, nil) } // DeleteProject deletes a project from the authorizer. func (f *FGA) DeleteProject(ctx context.Context, _ int64, projectName string) error { // Only empty projects can be deleted, so we don't need to worry about any tuples with this project as a parent. deletions := []client.ClientTupleKeyWithoutCondition{ { // Remove the default profile User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectProfile(projectName, "default").String(), }, { User: ObjectServer().String(), Relation: relationServer, Object: ObjectProject(projectName).String(), }, } return f.updateTuples(ctx, nil, deletions) } // RenameProject renames a project in the authorizer. func (f *FGA) RenameProject(ctx context.Context, _ int64, oldName string, newName string) error { writes := []client.ClientTupleKey{ { User: ObjectServer().String(), Relation: relationServer, Object: ObjectProject(newName).String(), }, { User: ObjectProject(newName).String(), Relation: relationProject, Object: ObjectProfile(newName, "default").String(), }, } // Only empty projects can be renamed, so we don't need to worry about any tuples with this project as a parent. deletions := []client.ClientTupleKeyWithoutCondition{ { // Remove the default profile User: ObjectProject(oldName).String(), Relation: relationProject, Object: ObjectProfile(oldName, "default").String(), }, { User: ObjectServer().String(), Relation: relationServer, Object: ObjectProject(oldName).String(), }, } return f.updateTuples(ctx, writes, deletions) } // AddCertificate adds a certificate to the authorizer. func (f *FGA) AddCertificate(ctx context.Context, fingerprint string) error { writes := []client.ClientTupleKey{ { User: ObjectServer().String(), Relation: relationServer, Object: ObjectCertificate(fingerprint).String(), }, } return f.updateTuples(ctx, writes, nil) } // DeleteCertificate deletes a certificate from the authorizer. func (f *FGA) DeleteCertificate(ctx context.Context, fingerprint string) error { deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectServer().String(), Relation: relationServer, Object: ObjectCertificate(fingerprint).String(), }, } return f.updateTuples(ctx, nil, deletions) } // AddStoragePool adds a storage pool to the authorizer. func (f *FGA) AddStoragePool(ctx context.Context, storagePoolName string) error { writes := []client.ClientTupleKey{ { User: ObjectServer().String(), Relation: relationServer, Object: ObjectStoragePool(storagePoolName).String(), }, } return f.updateTuples(ctx, writes, nil) } // DeleteStoragePool deletes a storage pool from the authorizer. func (f *FGA) DeleteStoragePool(ctx context.Context, storagePoolName string) error { deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectServer().String(), Relation: relationServer, Object: ObjectStoragePool(storagePoolName).String(), }, } return f.updateTuples(ctx, nil, deletions) } // AddImage adds an image to the authorizer. func (f *FGA) AddImage(ctx context.Context, projectName string, fingerprint string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectImage(projectName, fingerprint).String(), }, } return f.updateTuples(ctx, writes, nil) } // DeleteImage deletes an image from the authorizer. func (f *FGA) DeleteImage(ctx context.Context, projectName string, fingerprint string) error { deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectImage(projectName, fingerprint).String(), }, } return f.updateTuples(ctx, nil, deletions) } // AddImageAlias adds an image alias to the authorizer. func (f *FGA) AddImageAlias(ctx context.Context, projectName string, imageAliasName string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectImageAlias(projectName, imageAliasName).String(), }, } return f.updateTuples(ctx, writes, nil) } // DeleteImageAlias deletes an image alias from the authorizer. func (f *FGA) DeleteImageAlias(ctx context.Context, projectName string, imageAliasName string) error { deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectImageAlias(projectName, imageAliasName).String(), }, } return f.updateTuples(ctx, nil, deletions) } // RenameImageAlias renames an image alias in the authorizer. func (f *FGA) RenameImageAlias(ctx context.Context, projectName string, oldAliasName string, newAliasName string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectImageAlias(projectName, newAliasName).String(), }, } deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectImageAlias(projectName, oldAliasName).String(), }, } return f.updateTuples(ctx, writes, deletions) } // AddInstance adds an instance to the authorizer. func (f *FGA) AddInstance(ctx context.Context, projectName string, instanceName string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectInstance(projectName, instanceName).String(), }, } return f.updateTuples(ctx, writes, nil) } // DeleteInstance deletes an instance from the authorizer. func (f *FGA) DeleteInstance(ctx context.Context, projectName string, instanceName string) error { deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectInstance(projectName, instanceName).String(), }, } return f.updateTuples(ctx, nil, deletions) } // RenameInstance renames an instance in the authorizer. func (f *FGA) RenameInstance(ctx context.Context, projectName string, oldInstanceName string, newInstanceName string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectInstance(projectName, newInstanceName).String(), }, } deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectInstance(projectName, oldInstanceName).String(), }, } return f.updateTuples(ctx, writes, deletions) } // AddNetwork adds a network to the authorizer. func (f *FGA) AddNetwork(ctx context.Context, projectName string, networkName string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectNetwork(projectName, networkName).String(), }, } return f.updateTuples(ctx, writes, nil) } // DeleteNetwork deletes a network from the authorizer. func (f *FGA) DeleteNetwork(ctx context.Context, projectName string, networkName string) error { deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectNetwork(projectName, networkName).String(), }, } return f.updateTuples(ctx, nil, deletions) } // RenameNetwork renames a network in the authorizer. func (f *FGA) RenameNetwork(ctx context.Context, projectName string, oldNetworkName string, newNetworkName string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectNetwork(projectName, newNetworkName).String(), }, } deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectNetwork(projectName, oldNetworkName).String(), }, } return f.updateTuples(ctx, writes, deletions) } // AddNetworkZone adds a network zone in the authorizer. func (f *FGA) AddNetworkZone(ctx context.Context, projectName string, networkZoneName string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectNetworkZone(projectName, networkZoneName).String(), }, } return f.updateTuples(ctx, writes, nil) } // DeleteNetworkZone deletes a network zone from the authorizer. func (f *FGA) DeleteNetworkZone(ctx context.Context, projectName string, networkZoneName string) error { deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectNetworkZone(projectName, networkZoneName).String(), }, } return f.updateTuples(ctx, nil, deletions) } // AddNetworkIntegration adds a network integration to the authorizer. func (f *FGA) AddNetworkIntegration(ctx context.Context, networkIntegrationName string) error { writes := []client.ClientTupleKey{ { User: ObjectServer().String(), Relation: relationServer, Object: ObjectNetworkIntegration(networkIntegrationName).String(), }, } return f.updateTuples(ctx, writes, nil) } // DeleteNetworkIntegration deletes a network integration from the authorizer. func (f *FGA) DeleteNetworkIntegration(ctx context.Context, networkIntegrationName string) error { deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectServer().String(), Relation: relationServer, Object: ObjectNetworkIntegration(networkIntegrationName).String(), }, } return f.updateTuples(ctx, nil, deletions) } // RenameNetworkIntegration renames a network integration in the authorizer. func (f *FGA) RenameNetworkIntegration(ctx context.Context, oldNetworkIntegrationName string, newNetworkIntegrationName string) error { writes := []client.ClientTupleKey{ { User: ObjectServer().String(), Relation: relationServer, Object: ObjectNetworkIntegration(newNetworkIntegrationName).String(), }, } deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectServer().String(), Relation: relationServer, Object: ObjectNetworkIntegration(oldNetworkIntegrationName).String(), }, } return f.updateTuples(ctx, writes, deletions) } // AddNetworkACL adds a network ACL in the authorizer. func (f *FGA) AddNetworkACL(ctx context.Context, projectName string, networkACLName string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectNetworkACL(projectName, networkACLName).String(), }, } return f.updateTuples(ctx, writes, nil) } // DeleteNetworkACL deletes a network ACL from the authorizer. func (f *FGA) DeleteNetworkACL(ctx context.Context, projectName string, networkACLName string) error { deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectNetworkACL(projectName, networkACLName).String(), }, } return f.updateTuples(ctx, nil, deletions) } // RenameNetworkACL renames a network ACL in the authorizer. func (f *FGA) RenameNetworkACL(ctx context.Context, projectName string, oldNetworkACLName string, newNetworkACLName string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectNetworkACL(projectName, newNetworkACLName).String(), }, } deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectNetworkACL(projectName, oldNetworkACLName).String(), }, } return f.updateTuples(ctx, writes, deletions) } // AddNetworkAddressSet adds a network address set to the authorization model. func (f *FGA) AddNetworkAddressSet(ctx context.Context, projectName string, networkAddressSetName string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectNetworkAddressSet(projectName, networkAddressSetName).String(), }, } return f.updateTuples(ctx, writes, nil) } // DeleteNetworkAddressSet removes a network address set from the authorization model. func (f *FGA) DeleteNetworkAddressSet(ctx context.Context, projectName string, networkAddressSetName string) error { deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectNetworkAddressSet(projectName, networkAddressSetName).String(), }, } return f.updateTuples(ctx, nil, deletions) } // RenameNetworkAddressSet renames an existing network address set in the authorization model. func (f *FGA) RenameNetworkAddressSet(ctx context.Context, projectName string, oldNetworkAddressSetName string, newNetworkAddressSetName string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectNetworkAddressSet(projectName, newNetworkAddressSetName).String(), }, } deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectNetworkAddressSet(projectName, oldNetworkAddressSetName).String(), }, } return f.updateTuples(ctx, writes, deletions) } // AddProfile is a no-op. func (f *FGA) AddProfile(ctx context.Context, projectName string, profileName string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectProfile(projectName, profileName).String(), }, } return f.updateTuples(ctx, writes, nil) } // DeleteProfile deletes a profile from the authorizer. func (f *FGA) DeleteProfile(ctx context.Context, projectName string, profileName string) error { deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectProfile(projectName, profileName).String(), }, } return f.updateTuples(ctx, nil, deletions) } // RenameProfile renames a profile in the authorizer. func (f *FGA) RenameProfile(ctx context.Context, projectName string, oldProfileName string, newProfileName string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectProfile(projectName, newProfileName).String(), }, } deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectProfile(projectName, oldProfileName).String(), }, } return f.updateTuples(ctx, writes, deletions) } // AddStoragePoolVolume adds a storage volume to the authorizer. func (f *FGA) AddStoragePoolVolume(ctx context.Context, projectName string, storagePoolName string, storageVolumeType string, storageVolumeName string, storageVolumeLocation string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectStorageVolume(projectName, storagePoolName, storageVolumeType, storageVolumeName, storageVolumeLocation).String(), }, } return f.updateTuples(ctx, writes, nil) } // DeleteStoragePoolVolume deletes a storage volume from the authorizer. func (f *FGA) DeleteStoragePoolVolume(ctx context.Context, projectName string, storagePoolName string, storageVolumeType string, storageVolumeName string, storageVolumeLocation string) error { deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectStorageVolume(projectName, storagePoolName, storageVolumeType, storageVolumeName, storageVolumeLocation).String(), }, } return f.updateTuples(ctx, nil, deletions) } // RenameStoragePoolVolume renames a storage volume in the authorizer. func (f *FGA) RenameStoragePoolVolume(ctx context.Context, projectName string, storagePoolName string, storageVolumeType string, oldStorageVolumeName string, newStorageVolumeName string, storageVolumeLocation string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectStorageVolume(projectName, storagePoolName, storageVolumeType, newStorageVolumeName, storageVolumeLocation).String(), }, } deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectStorageVolume(projectName, storagePoolName, storageVolumeType, oldStorageVolumeName, storageVolumeLocation).String(), }, } return f.updateTuples(ctx, writes, deletions) } // AddStorageBucket adds a storage bucket to the authorizer. func (f *FGA) AddStorageBucket(ctx context.Context, projectName string, storagePoolName string, storageBucketName string, storageBucketLocation string) error { writes := []client.ClientTupleKey{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectStorageBucket(projectName, storagePoolName, storageBucketName, storageBucketLocation).String(), }, } return f.updateTuples(ctx, writes, nil) } // DeleteStorageBucket deletes a storage bucket from the authorizer. func (f *FGA) DeleteStorageBucket(ctx context.Context, projectName string, storagePoolName string, storageBucketName string, storageBucketLocation string) error { deletions := []client.ClientTupleKeyWithoutCondition{ { User: ObjectProject(projectName).String(), Relation: relationProject, Object: ObjectStorageBucket(projectName, storagePoolName, storageBucketName, storageBucketLocation).String(), }, } return f.updateTuples(ctx, nil, deletions) } // updateTuples sends an object update to OpenFGA if it's currently online. func (f *FGA) updateTuples(ctx context.Context, writes []client.ClientTupleKey, deletions []client.ClientTupleKeyWithoutCondition) error { // If offline, skip updating as a full sync will happen after connection. f.onlineMu.Lock() defer f.onlineMu.Unlock() if !f.online { return nil } if len(writes) == 0 && len(deletions) == 0 { return nil } return f.sendTuples(ctx, writes, deletions) } // sendTuples directly sends the write/deletion tuples to OpenFGA. func (f *FGA) sendTuples(ctx context.Context, writes []client.ClientTupleKey, deletions []client.ClientTupleKeyWithoutCondition) error { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() opts := client.ClientWriteOptions{ Transaction: &client.TransactionOptions{ Disable: true, MaxParallelRequests: 5, MaxPerChunk: 50, }, } body := client.ClientWriteRequest{} if writes != nil { body.Writes = writes } else { body.Writes = []client.ClientTupleKey{} } if deletions != nil { body.Deletes = deletions } else { body.Deletes = []openfga.TupleKeyWithoutCondition{} } clientWriteResponse, err := f.client.Write(ctx).Options(opts).Body(body).Execute() if err != nil { return fmt.Errorf("Failed to write to OpenFGA store: %w", err) } for _, write := range clientWriteResponse.Writes { if write.Error != nil { return fmt.Errorf("Failed to write tuple to OpenFGA store (user: %q; relation: %q; object: %q): %w", write.TupleKey.User, write.TupleKey.Relation, write.TupleKey.Object, write.Error) } } for _, deletion := range clientWriteResponse.Deletes { if deletion.Error != nil { return fmt.Errorf("Failed to delete tuple from OpenFGA store (user: %q; relation: %q; object: %q): %w", deletion.TupleKey.User, deletion.TupleKey.Relation, deletion.TupleKey.Object, deletion.Error) } } return nil } func (f *FGA) projectObjects(ctx context.Context, projectName string) ([]string, error) { objectTypes := []ObjectType{ ObjectTypeInstance, ObjectTypeImage, ObjectTypeImageAlias, ObjectTypeNetwork, ObjectTypeNetworkACL, ObjectTypeNetworkAddressSet, ObjectTypeNetworkZone, ObjectTypeProfile, ObjectTypeStorageVolume, ObjectTypeStorageBucket, } var allObjects []string projectObjectString := ObjectProject(projectName).String() for _, objectType := range objectTypes { resp, err := f.client.ListObjects(ctx).Body(client.ClientListObjectsRequest{ User: projectObjectString, Relation: relationProject, Type: string(objectType), }).Execute() if err != nil { return nil, err } allObjects = append(allObjects, resp.GetObjects()...) } return allObjects, nil } func (f *FGA) syncResources(ctx context.Context, resources Resources) error { var writes []client.ClientTupleKey var deletions []client.ClientTupleKeyWithoutCondition // Helper function for diffing local objects with those in OpenFGA. These are appended to the writes and deletions // slices as appropriate. If the given relation is relationProject, we need to construct a project object for the // "user" field. The project is calculated from the object we are inspecting. diffObjects := func(relation string, remoteObjectStrs []string, localObjects []Object) error { user := ObjectServer().String() for _, localObject := range localObjects { if !slices.Contains(remoteObjectStrs, localObject.String()) { if relation == relationProject { user = ObjectProject(localObject.Project()).String() } writes = append(writes, client.ClientTupleKey{ User: user, Relation: relation, Object: localObject.String(), }) } } for _, remoteObjectStr := range remoteObjectStrs { remoteObject, err := ObjectFromString(remoteObjectStr) if err != nil { return err } if !slices.Contains(localObjects, remoteObject) { if relation == relationProject { user = ObjectProject(remoteObject.Project()).String() } deletions = append(deletions, client.ClientTupleKeyWithoutCondition{ User: user, Relation: relation, Object: remoteObject.String(), }) } } return nil } // List the certificates we have added to OpenFGA already. certificatesResp, err := f.client.ListObjects(ctx).Body(client.ClientListObjectsRequest{ User: ObjectServer().String(), Relation: relationServer, Type: string(ObjectTypeCertificate), }).Execute() if err != nil { return err } // Compare with local certificates. err = diffObjects(relationServer, certificatesResp.GetObjects(), resources.CertificateObjects) if err != nil { return err } // List the network integrations we have added to OpenFGA already. networkIntegrationsResp, err := f.client.ListObjects(ctx).Body(client.ClientListObjectsRequest{ User: ObjectServer().String(), Relation: relationServer, Type: string(ObjectTypeNetworkIntegration), }).Execute() if err != nil { return err } // Compare with local network integrations. err = diffObjects(relationServer, networkIntegrationsResp.GetObjects(), resources.NetworkIntegrationObjects) if err != nil { return err } // List the storage pools we have added to OpenFGA already. storagePoolsResp, err := f.client.ListObjects(ctx).Body(client.ClientListObjectsRequest{ User: ObjectServer().String(), Relation: relationServer, Type: string(ObjectTypeStoragePool), }).Execute() if err != nil { return err } // Compare with local storage pools. err = diffObjects(relationServer, storagePoolsResp.GetObjects(), resources.StoragePoolObjects) if err != nil { return err } // List the projects we have added to OpenFGA already. projectsResp, err := f.client.ListObjects(ctx).Body(client.ClientListObjectsRequest{ User: ObjectServer().String(), Relation: relationServer, Type: string(ObjectTypeProject), }).Execute() if err != nil { return err } // Compare with local projects. remoteProjectObjectStrs := projectsResp.GetObjects() err = diffObjects(relationServer, remoteProjectObjectStrs, resources.ProjectObjects) if err != nil { return err } // Get a slice of project level resources for all projects. var remoteProjectResourceObjectStrs []string for _, remoteProjectObjectStr := range remoteProjectObjectStrs { remoteProjectObject, err := ObjectFromString(remoteProjectObjectStr) if err != nil { return err } // project level resources just for this project. remoteProjectResources, err := f.projectObjects(ctx, remoteProjectObject.Project()) if err != nil { return err } remoteProjectResourceObjectStrs = append(remoteProjectResourceObjectStrs, remoteProjectResources...) } // Compose a slice of all project level objects from the given Resources. localProjectObjects := append(resources.ImageObjects, resources.ImageAliasObjects...) localProjectObjects = append(localProjectObjects, resources.InstanceObjects...) localProjectObjects = append(localProjectObjects, resources.NetworkObjects...) localProjectObjects = append(localProjectObjects, resources.NetworkZoneObjects...) localProjectObjects = append(localProjectObjects, resources.NetworkACLObjects...) localProjectObjects = append(localProjectObjects, resources.NetworkAddressSetObjects...) localProjectObjects = append(localProjectObjects, resources.ProfileObjects...) localProjectObjects = append(localProjectObjects, resources.StoragePoolVolumeObjects...) localProjectObjects = append(localProjectObjects, resources.StorageBucketObjects...) // Perform a diff on the project resource objects. err = diffObjects(relationProject, remoteProjectResourceObjectStrs, localProjectObjects) if err != nil { return err } // Perform any necessary writes and deletions against the OpenFGA server. return f.updateTuples(ctx, writes, deletions) } // GetInstanceAccess returns the list of entities who have access to the instance. func (f *FGA) GetInstanceAccess(ctx context.Context, projectName string, instanceName string) (*api.Access, error) { // Get all the entries from OpenFGA. entries := map[string]string{} userFilters := []openfga.UserTypeFilter{{Type: "user"}} relations := []string{"admin", "operator", "user", "viewer"} for _, relation := range relations { resp, err := f.client.ListUsers(ctx).Body(client.ClientListUsersRequest{ Object: openfga.FgaObject{ Type: "instance", Id: fmt.Sprintf("%s/%s", projectName, instanceName), }, Relation: relation, UserFilters: userFilters, }).Execute() if err != nil { var fgaAPIErr openfga.FgaApiValidationError ok := errors.As(err, &fgaAPIErr) if !ok || fgaAPIErr.ResponseCode() != openfga.ERRORCODE_RELATION_NOT_FOUND { var fgaNotFoundErr openfga.FgaApiNotFoundError ok := errors.As(err, &fgaNotFoundErr) if ok && fgaNotFoundErr.ResponseCode() == openfga.NOTFOUNDERRORCODE_UNDEFINED_ENDPOINT { return nil, errors.New("OpenFGA server doesn't support listing users") } return nil, fmt.Errorf("Failed to list objects with relation %q: %w: %T", relation, err, err) } } for _, user := range resp.GetUsers() { obj := user.GetObject() if obj.Id == "" { continue } _, ok := entries[obj.Id] if !ok { entries[obj.Id] = relation } } } // Convert to our access records. access := api.Access{} for user, relation := range entries { access = append(access, api.AccessEntry{ Identifier: user, Role: relation, Provider: "openfga", }) } return &access, nil } // GetProjectAccess returns the list of entities who have access to the project. func (f *FGA) GetProjectAccess(ctx context.Context, projectName string) (*api.Access, error) { // Get all the entries from OpenFGA. entries := map[string]string{} userFilters := []openfga.UserTypeFilter{{Type: "user"}} relations := []string{"admin", "operator", "user", "viewer"} for _, relation := range relations { resp, err := f.client.ListUsers(ctx).Body(client.ClientListUsersRequest{ Object: openfga.FgaObject{ Type: "project", Id: projectName, }, Relation: relation, UserFilters: userFilters, }).Execute() if err != nil { var fgaAPIErr openfga.FgaApiValidationError ok := errors.As(err, &fgaAPIErr) if !ok || fgaAPIErr.ResponseCode() != openfga.ERRORCODE_RELATION_NOT_FOUND { var fgaNotFoundErr openfga.FgaApiNotFoundError ok := errors.As(err, &fgaNotFoundErr) if ok && fgaNotFoundErr.ResponseCode() == openfga.NOTFOUNDERRORCODE_UNDEFINED_ENDPOINT { return nil, errors.New("OpenFGA server doesn't support listing users") } return nil, fmt.Errorf("Failed to list objects with relation %q: %w: %T", relation, err, err) } } for _, user := range resp.GetUsers() { obj := user.GetObject() if obj.Id == "" { continue } _, ok := entries[obj.Id] if !ok { entries[obj.Id] = relation } } } // Convert to our access records. access := api.Access{} for user, relation := range entries { access = append(access, api.AccessEntry{ Identifier: user, Role: relation, Provider: "openfga", }) } return &access, nil } incus-7.0.0/internal/server/auth/driver_openfga_model.go000066400000000000000000000437231517523235500234440ustar00rootroot00000000000000package auth // Code generated by Makefile; DO NOT EDIT. var authModel = `{"schema_version":"1.1","type_definitions":[{"type":"user"},{"metadata":{"relations":{"member":{"directly_related_user_types":[{"type":"user"}]}}},"relations":{"member":{"this":{}}},"type":"group"},{"metadata":{"relations":{"can_edit":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view":{},"server":{"directly_related_user_types":[{"type":"server"}]}}},"relations":{"can_edit":{"union":{"child":[{"this":{}},{"tupleToUserset":{"computedUserset":{"relation":"admin"},"tupleset":{"relation":"server"}}}]}},"can_view":{"tupleToUserset":{"computedUserset":{"relation":"viewer"},"tupleset":{"relation":"server"}}},"server":{"this":{}}},"type":"certificate"},{"metadata":{"relations":{"can_edit":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"project":{"directly_related_user_types":[{"type":"project"}]}}},"relations":{"can_edit":{"union":{"child":[{"this":{}},{"tupleToUserset":{"computedUserset":{"relation":"operator"},"tupleset":{"relation":"project"}}}]}},"can_view":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"can_edit"}},{"tupleToUserset":{"computedUserset":{"relation":"viewer"},"tupleset":{"relation":"project"}}}]}},"project":{"this":{}}},"type":"image"},{"metadata":{"relations":{"can_edit":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"project":{"directly_related_user_types":[{"type":"project"}]}}},"relations":{"can_edit":{"union":{"child":[{"this":{}},{"tupleToUserset":{"computedUserset":{"relation":"operator"},"tupleset":{"relation":"project"}}}]}},"can_view":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"can_edit"}},{"tupleToUserset":{"computedUserset":{"relation":"viewer"},"tupleset":{"relation":"project"}}}]}},"project":{"this":{}}},"type":"image_alias"},{"metadata":{"relations":{"admin":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_access_console":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_access_files":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_connect_nbd":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_connect_sftp":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_edit":{},"can_exec":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_manage_backups":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_manage_snapshots":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_update_state":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view":{},"operator":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"project":{"directly_related_user_types":[{"type":"project"}]},"user":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"viewer":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]}}},"relations":{"admin":{"union":{"child":[{"this":{}},{"tupleToUserset":{"computedUserset":{"relation":"admin"},"tupleset":{"relation":"project"}}}]}},"can_access_console":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"user"}}]}},"can_access_files":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"user"}}]}},"can_connect_nbd":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"user"}}]}},"can_connect_sftp":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"user"}}]}},"can_edit":{"computedUserset":{"relation":"operator"}},"can_exec":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"user"}}]}},"can_manage_backups":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}}]}},"can_manage_snapshots":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}}]}},"can_update_state":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}}]}},"can_view":{"computedUserset":{"relation":"viewer"}},"operator":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"admin"}},{"tupleToUserset":{"computedUserset":{"relation":"operator"},"tupleset":{"relation":"project"}}}]}},"project":{"this":{}},"user":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}},{"tupleToUserset":{"computedUserset":{"relation":"user"},"tupleset":{"relation":"project"}}}]}},"viewer":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"user"}},{"tupleToUserset":{"computedUserset":{"relation":"viewer"},"tupleset":{"relation":"project"}}}]}}},"type":"instance"},{"metadata":{"relations":{"can_edit":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"project":{"directly_related_user_types":[{"type":"project"}]}}},"relations":{"can_edit":{"union":{"child":[{"this":{}},{"tupleToUserset":{"computedUserset":{"relation":"operator"},"tupleset":{"relation":"project"}}}]}},"can_view":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"can_edit"}},{"tupleToUserset":{"computedUserset":{"relation":"viewer"},"tupleset":{"relation":"project"}}}]}},"project":{"this":{}}},"type":"network"},{"metadata":{"relations":{"can_edit":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"project":{"directly_related_user_types":[{"type":"project"}]}}},"relations":{"can_edit":{"union":{"child":[{"this":{}},{"tupleToUserset":{"computedUserset":{"relation":"operator"},"tupleset":{"relation":"project"}}}]}},"can_view":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"can_edit"}},{"tupleToUserset":{"computedUserset":{"relation":"viewer"},"tupleset":{"relation":"project"}}}]}},"project":{"this":{}}},"type":"network_acl"},{"metadata":{"relations":{"can_edit":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"project":{"directly_related_user_types":[{"type":"project"}]}}},"relations":{"can_edit":{"union":{"child":[{"this":{}},{"tupleToUserset":{"computedUserset":{"relation":"operator"},"tupleset":{"relation":"project"}}}]}},"can_view":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"can_edit"}},{"tupleToUserset":{"computedUserset":{"relation":"viewer"},"tupleset":{"relation":"project"}}}]}},"project":{"this":{}}},"type":"network_address_set"},{"metadata":{"relations":{"can_edit":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view":{},"server":{"directly_related_user_types":[{"type":"server"}]}}},"relations":{"can_edit":{"union":{"child":[{"this":{}},{"tupleToUserset":{"computedUserset":{"relation":"admin"},"tupleset":{"relation":"server"}}}]}},"can_view":{"tupleToUserset":{"computedUserset":{"relation":"viewer"},"tupleset":{"relation":"server"}}},"server":{"this":{}}},"type":"network_integration"},{"metadata":{"relations":{"can_edit":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"project":{"directly_related_user_types":[{"type":"project"}]}}},"relations":{"can_edit":{"union":{"child":[{"this":{}},{"tupleToUserset":{"computedUserset":{"relation":"operator"},"tupleset":{"relation":"project"}}}]}},"can_view":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"can_edit"}},{"tupleToUserset":{"computedUserset":{"relation":"viewer"},"tupleset":{"relation":"project"}}}]}},"project":{"this":{}}},"type":"network_zone"},{"metadata":{"relations":{"can_edit":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"project":{"directly_related_user_types":[{"type":"project"}]}}},"relations":{"can_edit":{"union":{"child":[{"this":{}},{"tupleToUserset":{"computedUserset":{"relation":"operator"},"tupleset":{"relation":"project"}}}]}},"can_view":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"can_edit"}},{"tupleToUserset":{"computedUserset":{"relation":"viewer"},"tupleset":{"relation":"project"}}}]}},"project":{"this":{}}},"type":"profile"},{"metadata":{"relations":{"admin":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_create_image_aliases":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_create_images":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_create_instances":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_create_network_acls":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_create_network_address_sets":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_create_network_zones":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_create_networks":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_create_profiles":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_create_storage_buckets":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_create_storage_volumes":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_edit":{},"can_view":{},"can_view_events":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view_operations":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"operator":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"server":{"directly_related_user_types":[{"type":"server"}]},"user":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"viewer":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]}}},"relations":{"admin":{"union":{"child":[{"this":{}},{"tupleToUserset":{"computedUserset":{"relation":"admin"},"tupleset":{"relation":"server"}}}]}},"can_create_image_aliases":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}}]}},"can_create_images":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}}]}},"can_create_instances":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}}]}},"can_create_network_acls":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}}]}},"can_create_network_address_sets":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}}]}},"can_create_network_zones":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}}]}},"can_create_networks":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}}]}},"can_create_profiles":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}}]}},"can_create_storage_buckets":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}}]}},"can_create_storage_volumes":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}}]}},"can_edit":{"computedUserset":{"relation":"admin"}},"can_view":{"computedUserset":{"relation":"viewer"}},"can_view_events":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"user"}}]}},"can_view_operations":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"user"}}]}},"operator":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"admin"}},{"tupleToUserset":{"computedUserset":{"relation":"operator"},"tupleset":{"relation":"server"}}}]}},"server":{"this":{}},"user":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}},{"tupleToUserset":{"computedUserset":{"relation":"user"},"tupleset":{"relation":"server"}}}]}},"viewer":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"user"}},{"tupleToUserset":{"computedUserset":{"relation":"viewer"},"tupleset":{"relation":"server"}}}]}}},"type":"project"},{"metadata":{"relations":{"admin":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"authenticated":{"directly_related_user_types":[{"type":"user","wildcard":{}}]},"can_create_certificates":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_create_network_integrations":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_create_projects":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_create_storage_pools":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_edit":{},"can_override_cluster_target_restriction":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view":{},"can_view_metrics":{},"can_view_privileged_events":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view_resources":{},"can_view_sensitive":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"operator":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"user":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"viewer":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]}}},"relations":{"admin":{"this":{}},"authenticated":{"this":{}},"can_create_certificates":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"admin"}}]}},"can_create_network_integrations":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"admin"}}]}},"can_create_projects":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"admin"}}]}},"can_create_storage_pools":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"admin"}}]}},"can_edit":{"computedUserset":{"relation":"admin"}},"can_override_cluster_target_restriction":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"admin"}}]}},"can_view":{"computedUserset":{"relation":"authenticated"}},"can_view_metrics":{"computedUserset":{"relation":"authenticated"}},"can_view_privileged_events":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"admin"}}]}},"can_view_resources":{"computedUserset":{"relation":"authenticated"}},"can_view_sensitive":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"viewer"}}]}},"operator":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"admin"}}]}},"user":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"operator"}}]}},"viewer":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"user"}}]}}},"type":"server"},{"metadata":{"relations":{"can_edit":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"project":{"directly_related_user_types":[{"type":"project"}]}}},"relations":{"can_edit":{"union":{"child":[{"this":{}},{"tupleToUserset":{"computedUserset":{"relation":"operator"},"tupleset":{"relation":"project"}}}]}},"can_view":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"can_edit"}},{"tupleToUserset":{"computedUserset":{"relation":"viewer"},"tupleset":{"relation":"project"}}}]}},"project":{"this":{}}},"type":"storage_bucket"},{"metadata":{"relations":{"can_edit":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view":{},"server":{"directly_related_user_types":[{"type":"server"}]}}},"relations":{"can_edit":{"union":{"child":[{"this":{}},{"tupleToUserset":{"computedUserset":{"relation":"admin"},"tupleset":{"relation":"server"}}}]}},"can_view":{"tupleToUserset":{"computedUserset":{"relation":"authenticated"},"tupleset":{"relation":"server"}}},"server":{"this":{}}},"type":"storage_pool"},{"metadata":{"relations":{"can_access_files":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_connect_nbd":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_connect_sftp":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_edit":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_manage_backups":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_manage_snapshots":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"can_view":{"directly_related_user_types":[{"type":"user"},{"relation":"member","type":"group"}]},"project":{"directly_related_user_types":[{"type":"project"}]}}},"relations":{"can_access_files":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"can_edit"}}]}},"can_connect_nbd":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"can_edit"}}]}},"can_connect_sftp":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"can_edit"}}]}},"can_edit":{"union":{"child":[{"this":{}},{"tupleToUserset":{"computedUserset":{"relation":"operator"},"tupleset":{"relation":"project"}}}]}},"can_manage_backups":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"can_edit"}}]}},"can_manage_snapshots":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"can_edit"}}]}},"can_view":{"union":{"child":[{"this":{}},{"computedUserset":{"relation":"can_edit"}},{"tupleToUserset":{"computedUserset":{"relation":"viewer"},"tupleset":{"relation":"project"}}}]}},"project":{"this":{}}},"type":"storage_volume"}]}` incus-7.0.0/internal/server/auth/driver_openfga_model.openfga000066400000000000000000000130461517523235500244510ustar00rootroot00000000000000model schema 1.1 type user type group relations define member: [user] type certificate relations define server: [server] define can_edit: [user, group#member] or admin from server define can_view: viewer from server type image relations define project: [project] define can_edit: [user, group#member] or operator from project define can_view: [user, group#member] or can_edit or viewer from project type image_alias relations define project: [project] define can_edit: [user, group#member] or operator from project define can_view: [user, group#member] or can_edit or viewer from project type instance relations define project: [project] define admin: [user, group#member] or admin from project define operator: [user, group#member] or admin or operator from project define user: [user, group#member] or operator or user from project define viewer: [user, group#member] or user or viewer from project define can_access_console: [user, group#member] or user define can_access_files: [user, group#member] or user define can_connect_nbd: [user, group#member] or user define can_connect_sftp: [user, group#member] or user define can_edit: operator define can_exec: [user, group#member] or user define can_manage_backups: [user, group#member] or operator define can_manage_snapshots: [user, group#member] or operator define can_update_state: [user, group#member] or operator define can_view: viewer type network relations define project: [project] define can_edit: [user, group#member] or operator from project define can_view: [user, group#member] or can_edit or viewer from project type network_acl relations define project: [project] define can_edit: [user, group#member] or operator from project define can_view: [user, group#member] or can_edit or viewer from project type network_address_set relations define project: [project] define can_edit: [user, group#member] or operator from project define can_view: [user, group#member] or can_edit or viewer from project type network_integration relations define server: [server] define can_edit: [user, group#member] or admin from server define can_view: viewer from server type network_zone relations define project: [project] define can_edit: [user, group#member] or operator from project define can_view: [user, group#member] or can_edit or viewer from project type profile relations define project: [project] define can_edit: [user, group#member] or operator from project define can_view: [user, group#member] or can_edit or viewer from project type project relations define server: [server] define admin: [user, group#member] or admin from server define operator: [user, group#member] or admin or operator from server define user: [user, group#member] or operator or user from server define viewer: [user, group#member] or user or viewer from server define can_create_image_aliases: [user, group#member] or operator define can_create_images: [user, group#member] or operator define can_create_instances: [user, group#member] or operator define can_create_network_acls: [user, group#member] or operator define can_create_network_address_sets: [user, group#member] or operator define can_create_networks: [user, group#member] or operator define can_create_network_zones: [user, group#member] or operator define can_create_profiles: [user, group#member] or operator define can_create_storage_buckets: [user, group#member] or operator define can_create_storage_volumes: [user, group#member] or operator define can_edit: admin define can_view_events: [user, group#member] or user define can_view_operations: [user, group#member] or user define can_view: viewer type server relations define admin: [user, group#member] define operator: [user, group#member] or admin define user: [user, group#member] or operator define viewer: [user, group#member] or user define authenticated: [user:*] define can_create_certificates: [user, group#member] or admin define can_create_network_integrations: [user, group#member] or admin define can_create_projects: [user, group#member] or admin define can_create_storage_pools: [user, group#member] or admin define can_edit: admin define can_override_cluster_target_restriction: [user, group#member] or admin define can_view_privileged_events: [user, group#member] or admin define can_view_metrics: authenticated define can_view_resources: authenticated define can_view_sensitive: [user, group#member] or viewer define can_view: authenticated type storage_bucket relations define project: [project] define can_edit: [user, group#member] or operator from project define can_view: [user, group#member] or can_edit or viewer from project type storage_pool relations define server: [server] define can_edit: [user, group#member] or admin from server define can_view: authenticated from server type storage_volume relations define project: [project] define can_edit: [user, group#member] or operator from project define can_manage_backups: [user, group#member] or can_edit define can_manage_snapshots: [user, group#member] or can_edit define can_view: [user, group#member] or can_edit or viewer from project define can_access_files: [user, group#member] or can_edit define can_connect_nbd: [user, group#member] or can_edit define can_connect_sftp: [user, group#member] or can_edit incus-7.0.0/internal/server/auth/driver_scriptlet.go000066400000000000000000000066041517523235500226530ustar00rootroot00000000000000package auth import ( "context" "crypto/x509" "net/http" "github.com/lxc/incus/v7/internal/server/certificate" authScriptlet "github.com/lxc/incus/v7/internal/server/scriptlet/auth" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) // Scriptlet represents a scriptlet authorizer. type Scriptlet struct { commonAuthorizer certificates *certificate.Cache } // CheckPermission returns an error if the user does not have the given Entitlement on the given Object. func (s *Scriptlet) CheckPermission(ctx context.Context, r *http.Request, object Object, entitlement Entitlement) error { details, err := s.requestDetails(r) if err != nil { return api.StatusErrorf(http.StatusForbidden, "Failed to extract request details: %v", err) } if details.isInternalOrUnix() { return nil } actualDetails := details.actualDetails() peerCertificates := []*x509.Certificate{} var apiCert *api.CertificatePut if r.TLS != nil { peerCertificates = r.TLS.PeerCertificates if s.certificates != nil { apiCert = s.certificates.GetAPICertificate(actualDetails.Username) } } authorized, err := authScriptlet.AuthorizationRun(logger.Log, actualDetails, peerCertificates, apiCert, object.String(), string(entitlement)) if err != nil { return api.StatusErrorf(http.StatusForbidden, "Authorization scriptlet execution failed with error: %v", err) } if authorized { return nil } return api.StatusErrorf(http.StatusForbidden, "Permission denied") } // GetInstanceAccess returns the list of entities who have access to the instance. func (s *Scriptlet) GetInstanceAccess(ctx context.Context, projectName string, instanceName string) (*api.Access, error) { return authScriptlet.GetInstanceAccessRun(logger.Log, projectName, instanceName) } // GetPermissionChecker returns a function that can be used to check whether a user has the required entitlement on an authorization object. func (s *Scriptlet) GetPermissionChecker(ctx context.Context, r *http.Request, entitlement Entitlement, objectType ObjectType) (PermissionChecker, error) { allowFunc := func(b bool) func(Object) bool { return func(Object) bool { return b } } details, err := s.requestDetails(r) if err != nil { return nil, api.StatusErrorf(http.StatusForbidden, "Failed to extract request details: %v", err) } if details.isInternalOrUnix() { return allowFunc(true), nil } actualDetails := details.actualDetails() peerCertificates := []*x509.Certificate{} var apiCert *api.CertificatePut if r.TLS != nil { peerCertificates = r.TLS.PeerCertificates if s.certificates != nil { apiCert = s.certificates.GetAPICertificate(actualDetails.Username) } } permissionChecker := func(o Object) bool { authorized, err := authScriptlet.AuthorizationRun(logger.Log, actualDetails, peerCertificates, apiCert, o.String(), string(entitlement)) if err != nil { logger.Error("Authorization scriptlet execution failed", logger.Ctx{"err": err}) return false } return authorized } return permissionChecker, nil } // GetProjectAccess returns the list of entities who have access to the project. func (s *Scriptlet) GetProjectAccess(ctx context.Context, projectName string) (*api.Access, error) { return authScriptlet.GetProjectAccessRun(logger.Log, projectName) } func (s *Scriptlet) load(ctx context.Context, certificateCache *certificate.Cache, opts Opts) error { s.certificates = certificateCache return nil } incus-7.0.0/internal/server/auth/driver_tls.go000066400000000000000000000234411517523235500214420ustar00rootroot00000000000000package auth import ( "context" "errors" "net/http" "slices" "github.com/lxc/incus/v7/internal/server/certificate" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) // TLS represents a TLS authorizer. type TLS struct { commonAuthorizer certificates *certificate.Cache } func (t *TLS) load(ctx context.Context, certificateCache *certificate.Cache, opts Opts) error { if certificateCache == nil { return errors.New("TLS authorization driver requires a certificate cache") } t.certificates = certificateCache return nil } // CheckPermission returns an error if the user does not have the given Entitlement on the given Object. func (t *TLS) CheckPermission(ctx context.Context, r *http.Request, object Object, entitlement Entitlement) error { details, err := t.requestDetails(r) if err != nil { return api.StatusErrorf(http.StatusForbidden, "Failed to extract request details: %v", err) } if details.isInternalOrUnix() { return nil } authenticationProtocol := details.authenticationProtocol() if authenticationProtocol != api.AuthenticationMethodTLS { // Return nil. If the server has been configured with an authentication method but no associated authorization driver, // the default is to give these authenticated users admin privileges. return nil } certType, isNotRestricted, projectNames, err := t.certificateDetails(details.username()) if err != nil { return err } if isNotRestricted || (certType == certificate.TypeMetrics && entitlement == EntitlementCanViewMetrics) { return nil } if details.IsAllProjectsRequest { // Only admins (users with non-restricted certs) can use the all-projects parameter. return api.StatusErrorf(http.StatusForbidden, "Certificate is restricted") } // Check server level object types switch object.Type() { case ObjectTypeServer: if entitlement == EntitlementCanView || entitlement == EntitlementCanViewResources || entitlement == EntitlementCanViewMetrics { return nil } return api.StatusErrorf(http.StatusForbidden, "Certificate is restricted") case ObjectTypeStoragePool, ObjectTypeCertificate: if entitlement == EntitlementCanView { return nil } return api.StatusErrorf(http.StatusForbidden, "Certificate is restricted") } // Don't allow project modifications. if object.Type() == ObjectTypeProject && entitlement == EntitlementCanEdit { return api.StatusErrorf(http.StatusForbidden, "Certificate is restricted") } // Check project level permissions against the certificates project list. projectName := object.Project() if slices.Contains(projectNames, projectName) { return nil } // Also allow read-only access to inherited resources. if object.Project() == api.ProjectDefaultName && entitlement == EntitlementCanView && slices.Contains([]ObjectType{ObjectTypeImage, ObjectTypeProfile, ObjectTypeStorageVolume, ObjectTypeStorageBucket, ObjectTypeNetwork, ObjectTypeNetworkZone}, object.Type()) { return nil } return api.StatusErrorf(http.StatusForbidden, "User does not have permission for project %q", projectName) } // GetPermissionChecker returns a function that can be used to check whether a user has the required entitlement on an authorization object. func (t *TLS) GetPermissionChecker(ctx context.Context, r *http.Request, entitlement Entitlement, objectType ObjectType) (PermissionChecker, error) { allowFunc := func(b bool) func(Object) bool { return func(Object) bool { return b } } details, err := t.requestDetails(r) if err != nil { return nil, api.StatusErrorf(http.StatusForbidden, "Failed to extract request details: %v", err) } if details.isInternalOrUnix() { return allowFunc(true), nil } authenticationProtocol := details.authenticationProtocol() if authenticationProtocol != api.AuthenticationMethodTLS { // Allow all. If the server has been configured with an authentication method but no associated authorization driver, // the default is to give these authenticated users admin privileges. return allowFunc(true), nil } certType, isNotRestricted, projectNames, err := t.certificateDetails(details.username()) if err != nil { return nil, err } if isNotRestricted { return allowFunc(true), nil } // Handle project-restricted metrics access. if certType == certificate.TypeMetrics && entitlement == EntitlementCanViewMetrics { return func(o Object) bool { return slices.Contains(projectNames, o.Project()) }, nil } // Check server level object types switch objectType { case ObjectTypeServer: if entitlement == EntitlementCanView || entitlement == EntitlementCanViewResources || entitlement == EntitlementCanViewMetrics { return allowFunc(true), nil } return allowFunc(false), nil case ObjectTypeStoragePool, ObjectTypeCertificate: if entitlement == EntitlementCanView { return allowFunc(true), nil } return allowFunc(false), nil } // Error if user does not have access to the project (unless we're getting projects, where we want to filter the results). if !details.IsAllProjectsRequest && !slices.Contains(projectNames, details.ProjectName) && objectType != ObjectTypeProject { return allowFunc(false), nil } // Filter objects by project. return func(object Object) bool { // Allow if the project is in the allowed set. if slices.Contains(projectNames, object.Project()) { return true } // Also allow read-only access to inherited resources. if object.Project() != api.ProjectDefaultName { return false } if entitlement != EntitlementCanView { return false } if !slices.Contains([]ObjectType{ObjectTypeImage, ObjectTypeProfile, ObjectTypeStorageVolume, ObjectTypeStorageBucket, ObjectTypeNetwork, ObjectTypeNetworkZone}, objectType) { return false } return true }, nil } // certificateDetails returns the certificate type, a boolean indicating if the certificate is *not* restricted, a slice of // project names for this certificate, or an error if the certificate could not be found. func (t *TLS) certificateDetails(fingerprint string) (certificate.Type, bool, []string, error) { certs, projects := t.certificates.GetCertificatesAndProjects() clientCerts := certs[certificate.TypeClient] _, ok := clientCerts[fingerprint] if ok { projectNames, ok := projects[fingerprint] if !ok { // Certificate is not restricted. return certificate.TypeClient, true, nil, nil } return certificate.TypeClient, false, projectNames, nil } // If not a client cert, could be a metrics cert. Only need to check one entitlement. metricCerts := certs[certificate.TypeMetrics] _, ok = metricCerts[fingerprint] if ok { projectNames, ok := projects[fingerprint] if !ok { // Certificate is not restricted. return certificate.TypeClient, true, nil, nil } return certificate.TypeMetrics, false, projectNames, nil } // If we're in a CA environment, it's possible for a certificate to be trusted despite not being present in the trust store. // We rely on the validation of the certificate (and its potential revocation) having been done in CheckTrustState. if util.PathExists(internalUtil.VarPath("server.ca")) { return certificate.TypeClient, true, nil, nil } return -1, false, nil, api.StatusErrorf(http.StatusForbidden, "Client certificate not found") } // GetInstanceAccess returns the list of entities who have access to the instance. func (t *TLS) GetInstanceAccess(ctx context.Context, projectName string, instanceName string) (*api.Access, error) { var access api.Access certificates, projects := t.certificates.GetCertificatesAndProjects() // client (type = 1) clientCertificates := certificates[1] for fingerprint := range clientCertificates { certificateProjects := projects[fingerprint] // Unrestricted if certificateProjects == nil { access = append(access, api.AccessEntry{ Identifier: fingerprint, Role: "admin", Provider: "tls", }) } // Restricted if slices.Contains(certificateProjects, projectName) { access = append(access, api.AccessEntry{ Identifier: fingerprint, Role: "operator", Provider: "tls", }) } } // metric (type = 3) metricCertificates := certificates[3] for fingerprint := range metricCertificates { certificateProjects := projects[fingerprint] // Unrestricted if certificateProjects == nil { access = append(access, api.AccessEntry{ Identifier: fingerprint, Role: "view", Provider: "tls", }) } // Restricted if slices.Contains(certificateProjects, projectName) { access = append(access, api.AccessEntry{ Identifier: fingerprint, Role: "view", Provider: "tls", }) } } return &access, nil } // GetProjectAccess returns the list of entities who have access to the project. func (t *TLS) GetProjectAccess(ctx context.Context, projectName string) (*api.Access, error) { var access api.Access certificates, projects := t.certificates.GetCertificatesAndProjects() clientCerts := certificates[certificate.TypeClient] for fingerprint := range clientCerts { certificateProjects := projects[fingerprint] if certificateProjects == nil { access = append(access, api.AccessEntry{ Identifier: fingerprint, Role: "admin", Provider: "tls", }) } if slices.Contains(certificateProjects, projectName) { access = append(access, api.AccessEntry{ Identifier: fingerprint, Role: "operator", Provider: "tls", }) } } for fingerprint := range projects { certificateProjects := projects[fingerprint] if certificateProjects == nil { access = append(access, api.AccessEntry{ Identifier: fingerprint, Role: "view", Provider: "tls", }) } if slices.Contains(certificateProjects, projectName) { access = append(access, api.AccessEntry{ Identifier: fingerprint, Role: "view", Provider: "tls", }) } } return &access, nil } incus-7.0.0/internal/server/auth/oidc/000077500000000000000000000000001517523235500176505ustar00rootroot00000000000000incus-7.0.0/internal/server/auth/oidc/oidc.go000066400000000000000000000266471517523235500211340ustar00rootroot00000000000000package oidc import ( "context" "errors" "fmt" "net/http" "net/netip" "slices" "strings" "time" "github.com/google/uuid" "github.com/zitadel/oidc/v3/pkg/client" "github.com/zitadel/oidc/v3/pkg/client/rp" httphelper "github.com/zitadel/oidc/v3/pkg/http" "github.com/zitadel/oidc/v3/pkg/oidc" "github.com/zitadel/oidc/v3/pkg/op" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) // Verifier holds all information needed to verify an access token offline. type Verifier struct { accessTokenVerifier *op.AccessTokenVerifier clientID string issuer string scopes []string audience string claim string cookieKey []byte } // AuthError represents an authentication error. type AuthError struct { Err error } func (e AuthError) Error() string { return fmt.Sprintf("Failed to authenticate: %s", e.Err.Error()) } func (e AuthError) Unwrap() error { return e.Err } // Auth extracts the token, validates it and returns the user information. func (o *Verifier) Auth(ctx context.Context, w http.ResponseWriter, r *http.Request) (string, error) { var token string auth := r.Header.Get("Authorization") if auth != "" { // When a client wants to authenticate, it needs to set the Authorization HTTP header like this: // Authorization Bearer // If set correctly, Incus will attempt to verify the access token, and grant access if it's valid. // If the verification fails, Incus will return an InvalidToken error. The client should then either use its refresh token to get a new valid access token, or log in again. // If the Authorization header is missing, Incus returns an AuthenticationRequired error. // Both returned errors contain information which are needed for the client to authenticate. parts := strings.Split(auth, "Bearer ") if len(parts) != 2 { return "", &AuthError{errors.New("Bad authorization token, expected a Bearer token")} } token = parts[1] } else { // When not using a Bearer token, fetch the equivalent from a cookie and move on with it. cookie, err := r.Cookie("oidc_access") if err != nil { return "", &AuthError{err} } token = cookie.Value } if o.accessTokenVerifier == nil { var err error o.accessTokenVerifier, err = getAccessTokenVerifier(o.issuer) if err != nil { return "", &AuthError{err} } } claims, err := o.VerifyAccessToken(ctx, r, token) if err != nil { // See if we can refresh the access token. cookie, cookieErr := r.Cookie("oidc_refresh") if cookieErr != nil { return "", &AuthError{err} } // Get the provider. provider, err := o.getProvider(r) if err != nil { return "", &AuthError{err} } // Attempt the refresh. tokens, err := rp.RefreshTokens[*oidc.IDTokenClaims](context.TODO(), provider, cookie.Value, "", "") if err != nil { return "", &AuthError{err} } // Validate the refreshed token. claims, err = o.VerifyAccessToken(ctx, r, tokens.AccessToken) if err != nil { return "", &AuthError{err} } // If we have a ResponseWriter, refresh the cookies. if w != nil { // Update the access token cookie. accessCookie := http.Cookie{ Name: "oidc_access", Value: tokens.AccessToken, Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteStrictMode, } http.SetCookie(w, &accessCookie) // Update the refresh token cookie. if tokens.RefreshToken != "" { refreshCookie := http.Cookie{ Name: "oidc_refresh", Value: tokens.RefreshToken, Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteStrictMode, } http.SetCookie(w, &refreshCookie) } } } if o.claim != "" { claim := claims.Claims[o.claim] username, ok := claim.(string) if claim == nil || !ok || username == "" { return "", fmt.Errorf("OIDC user is missing required claim %q", o.claim) } return username, nil } user, ok := claims.Claims["email"] if ok && user != nil && user.(string) != "" { return user.(string), nil } return claims.Subject, nil } func (o *Verifier) Login(w http.ResponseWriter, r *http.Request) { // Get the provider. provider, err := o.getProvider(r) if err != nil { logger.Error("Failed to get OIDC provider", logger.Ctx{"err": err}) http.Error(w, err.Error(), http.StatusInternalServerError) return } handler := rp.AuthURLHandler(func() string { return uuid.New().String() }, provider, rp.WithURLParam("audience", o.audience)) handler(w, r) } func (o *Verifier) Logout(w http.ResponseWriter, r *http.Request) { // Attempt to get the provider. provider, _ := o.getProvider(r) // Attempt to get the token. var token string cookie, err := r.Cookie("oidc_id") if err == nil { token = cookie.Value } // Attempt to end the OIDC session. if provider != nil && token != "" { _, _ = rp.EndSession(r.Context(), provider, token, fmt.Sprintf("https://%s", r.Host), "", "", nil) } // Access token. accessCookie := http.Cookie{ Name: "oidc_access", Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteStrictMode, Expires: time.Unix(0, 0), } http.SetCookie(w, &accessCookie) // ID token. idCookie := http.Cookie{ Name: "oidc_id", Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteStrictMode, Expires: time.Unix(0, 0), } http.SetCookie(w, &idCookie) // Refresh token. refreshCookie := http.Cookie{ Name: "oidc_refresh", Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteStrictMode, Expires: time.Unix(0, 0), } http.SetCookie(w, &refreshCookie) } func (o *Verifier) Callback(w http.ResponseWriter, r *http.Request) { // Get the provider. provider, err := o.getProvider(r) if err != nil { logger.Error("Failed to get OIDC provider", logger.Ctx{"err": err}) http.Error(w, err.Error(), http.StatusInternalServerError) return } handler := rp.CodeExchangeHandler(func(w http.ResponseWriter, r *http.Request, tokens *oidc.Tokens[*oidc.IDTokenClaims], state string, rp rp.RelyingParty) { // Access token. accessCookie := http.Cookie{ Name: "oidc_access", Value: tokens.AccessToken, Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteStrictMode, } http.SetCookie(w, &accessCookie) // Refresh token. if tokens.RefreshToken != "" { refreshCookie := http.Cookie{ Name: "oidc_refresh", Value: tokens.RefreshToken, Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteStrictMode, } http.SetCookie(w, &refreshCookie) } // ID token. if tokens.IDToken != "" { idCookie := http.Cookie{ Name: "oidc_id", Value: tokens.IDToken, Path: "/", Secure: true, HttpOnly: true, SameSite: http.SameSiteStrictMode, } http.SetCookie(w, &idCookie) } // Send to the UI. // NOTE: Once the UI does the redirection on its own, we may be able to use the referer here instead. http.Redirect(w, r, "/ui/", http.StatusMovedPermanently) }, provider) handler(w, r) } // VerifyAccessToken is a wrapper around op.VerifyAccessToken which avoids having to deal with Go generics elsewhere. It validates the access token (issuer, signature and expiration). func (o *Verifier) VerifyAccessToken(ctx context.Context, r *http.Request, token string) (*oidc.AccessTokenClaims, error) { var err error if o.accessTokenVerifier == nil { o.accessTokenVerifier, err = getAccessTokenVerifier(o.issuer) if err != nil { return nil, err } } claims, err := op.VerifyAccessToken[*oidc.AccessTokenClaims](ctx, token, o.accessTokenVerifier) if err != nil { return nil, err } // Check that the token includes the configured audience. audience := claims.GetAudience() if o.audience != "" && !slices.Contains(audience, o.audience) { return nil, errors.New("Provided OIDC token doesn't allow the configured audience") } // Check if we have a subnet restriction. err = o.validateSubnet(r, claims.Claims["incus.allowed_subnets"]) if err != nil { return nil, err } return claims, nil } func (o *Verifier) validateSubnet(r *http.Request, claim any) error { // If the claim is missing, allow access. if claim == nil { return nil } subnets, ok := claim.([]any) if !ok { // Invalid claim type. return errors.New("Bad type for incus.allowed_subnets OIDC claim") } if len(subnets) == 0 { // User isn't allowed from any subnet. return errors.New("Client isn't allowed to connect from its current network") } // Check if the requestor is allowed access. requestor := request.CreateRequestor(r) found := false for _, subnet := range subnets { subnetStr, ok := subnet.(string) if !ok { continue } subnetCIDR, err := netip.ParsePrefix(subnetStr) if err != nil { return fmt.Errorf("Bad subnet in incus.allowed_subnets claim %q: %w", subnet, err) } clientIP, err := netip.ParseAddr(requestor.Address) if err != nil { return fmt.Errorf("Bad client address %q: %w", requestor.Address, err) } if subnetCIDR.Contains(clientIP) { found = true break } } if !found { return errors.New("Client isn't allowed to connect from its current network") } return nil } // WriteHeaders writes the OIDC configuration as HTTP headers so the client can initatiate the device code flow. func (o *Verifier) WriteHeaders(w http.ResponseWriter) error { w.Header().Set("X-Incus-OIDC-audience", o.audience) w.Header().Set("X-Incus-OIDC-clientid", o.clientID) w.Header().Set("X-Incus-OIDC-issuer", o.issuer) w.Header().Set("X-Incus-OIDC-scopes", strings.Join(o.scopes, ",")) return nil } // IsRequest checks if the request is using OIDC authentication. func (o *Verifier) IsRequest(r *http.Request) bool { if r.Header.Get("Authorization") != "" { return true } cookie, err := r.Cookie("oidc_access") if err == nil && cookie != nil { return true } return false } func (o *Verifier) getProvider(r *http.Request) (rp.RelyingParty, error) { cookieHandler := httphelper.NewCookieHandler(o.cookieKey, o.cookieKey, httphelper.WithUnsecure()) options := []rp.Option{ rp.WithCookieHandler(cookieHandler), rp.WithVerifierOpts(rp.WithIssuedAtOffset(5 * time.Second)), rp.WithPKCE(cookieHandler), } provider, err := rp.NewRelyingPartyOIDC(context.TODO(), o.issuer, o.clientID, "", fmt.Sprintf("https://%s/oidc/callback", r.Host), o.scopes, options...) if err != nil { return nil, err } return provider, nil } // getAccessTokenVerifier calls the OIDC discovery endpoint in order to get the issuer's remote keys which are needed to create an access token verifier. func getAccessTokenVerifier(issuer string) (*op.AccessTokenVerifier, error) { discoveryConfig, err := client.Discover(context.TODO(), issuer, http.DefaultClient) if err != nil { return nil, fmt.Errorf("Failed calling OIDC discovery endpoint: %w", err) } keySet := rp.NewRemoteKeySet(http.DefaultClient, discoveryConfig.JwksURI) return op.NewAccessTokenVerifier(issuer, keySet), nil } // NewVerifier returns a Verifier. func NewVerifier(issuer string, clientid string, scope string, audience string, claim string) (*Verifier, error) { cookieKey, err := uuid.New().MarshalBinary() if err != nil { return nil, fmt.Errorf("Failed to create UUID: %w", err) } scopes := util.SplitNTrimSpace(scope, ",", -1, false) verifier := &Verifier{issuer: issuer, clientID: clientid, scopes: scopes, audience: audience, cookieKey: cookieKey, claim: claim} verifier.accessTokenVerifier, _ = getAccessTokenVerifier(issuer) return verifier, nil } incus-7.0.0/internal/server/backup/000077500000000000000000000000001517523235500172365ustar00rootroot00000000000000incus-7.0.0/internal/server/backup/backup_bucket.go000066400000000000000000000076021517523235500223740ustar00rootroot00000000000000package backup import ( "context" "os" "time" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" ) // BucketBackup represents a bucket backup. type BucketBackup struct { CommonBackup projectName string poolName string bucketName string } // NewBucketBackup instantiates a new BucketBackup struct. func NewBucketBackup(s *state.State, projectName, poolName, bucketName string, ID int, name string, creationDate, expiryDate time.Time) *BucketBackup { return &BucketBackup{ CommonBackup: CommonBackup{ state: s, id: ID, name: name, creationDate: creationDate, expiryDate: expiryDate, }, projectName: projectName, poolName: poolName, bucketName: bucketName, } } // Delete removes a bucket backup. func (b *BucketBackup) Delete() error { backupPath := internalUtil.VarPath("backups", "buckets", b.poolName, project.StorageBucket(b.projectName, b.name)) // Delete the on-disk data. if util.PathExists(backupPath) { err := os.RemoveAll(backupPath) if err != nil { return err } } // Check if we can remove the bucket directory. backupsPath := internalUtil.VarPath("backups", "buckets", b.poolName, project.StorageBucket(b.projectName, b.bucketName)) empty, _ := internalUtil.PathIsEmpty(backupsPath) if empty { err := os.Remove(backupsPath) if err != nil { return err } } // Remove the database record. err := b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.DeleteStoragePoolBucketBackup(ctx, b.name) }) if err != nil { return err } return nil } // Rename renames a bucket backup. func (b *BucketBackup) Rename(newName string) error { oldBackupPath := internalUtil.VarPath("backups", "buckets", b.poolName, project.StorageBucket(b.projectName, b.name)) newBackupPath := internalUtil.VarPath("backups", "buckets", b.poolName, project.StorageBucket(b.projectName, newName)) // Extract the old and new parent backup paths from the old and new backup names rather than use // bucket.Name() as this may be in flux if the bucket itself is being renamed, whereas the relevant // bucket name is encoded into the backup names. oldParentName, _, _ := api.GetParentAndSnapshotName(b.name) oldParentBackupsPath := internalUtil.VarPath("backups", "buckets", b.poolName, project.StorageBucket(b.projectName, oldParentName)) newParentName, _, _ := api.GetParentAndSnapshotName(newName) newParentBackupsPath := internalUtil.VarPath("backups", "buckets", b.poolName, project.StorageBucket(b.projectName, newParentName)) reverter := revert.New() defer reverter.Fail() // Create the new backup path if doesn't exist. if !util.PathExists(newParentBackupsPath) { err := os.MkdirAll(newParentBackupsPath, 0o700) if err != nil { return err } } // Rename the backup directory. err := os.Rename(oldBackupPath, newBackupPath) if err != nil { return err } reverter.Add(func() { _ = os.Rename(newBackupPath, oldBackupPath) }) // Check if we can remove the old parent directory. empty, _ := internalUtil.PathIsEmpty(oldParentBackupsPath) if empty { err := os.Remove(oldParentBackupsPath) if err != nil { return err } } // Rename the database record. err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RenameBucketBackup(ctx, b.name, newName) }) if err != nil { return err } reverter.Success() return nil } // Render returns a BucketBackup struct of the backup. func (b *BucketBackup) Render() *api.StorageBucketBackup { return &api.StorageBucketBackup{ Name: b.name, CreatedAt: b.creationDate, ExpiresAt: b.expiryDate, } } incus-7.0.0/internal/server/backup/backup_common.go000066400000000000000000000021241517523235500224010ustar00rootroot00000000000000package backup import ( "time" "github.com/lxc/incus/v7/internal/server/state" ) // WorkingDirPrefix is used when temporary working directories are needed. const WorkingDirPrefix = "incus_backup" // CommonBackup represents a common backup. type CommonBackup struct { state *state.State id int name string creationDate time.Time expiryDate time.Time optimizedStorage bool compressionAlgorithm string } // Name returns the name of the backup. func (b *CommonBackup) Name() string { return b.name } // CompressionAlgorithm returns the compression used for the tarball. func (b *CommonBackup) CompressionAlgorithm() string { return b.compressionAlgorithm } // SetCompressionAlgorithm sets the tarball compression. func (b *CommonBackup) SetCompressionAlgorithm(compression string) { b.compressionAlgorithm = compression } // OptimizedStorage returns whether the backup is to be performed using // optimization supported by the storage driver. func (b *CommonBackup) OptimizedStorage() bool { return b.optimizedStorage } incus-7.0.0/internal/server/backup/backup_config_utils.go000066400000000000000000000121551517523235500236030ustar00rootroot00000000000000package backup import ( "context" "errors" "os" "path/filepath" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/backup/config" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/osarch" ) // ConfigToInstanceDBArgs converts the instance config in the backup config to DB InstanceArgs. func ConfigToInstanceDBArgs(state *state.State, c *config.Config, projectName string, applyProfiles bool) (*db.InstanceArgs, error) { if c.Container == nil { return nil, nil } arch, _ := osarch.ArchitectureID(c.Container.Architecture) instanceType, _ := instancetype.New(c.Container.Type) inst := &db.InstanceArgs{ Project: projectName, Architecture: arch, BaseImage: c.Container.Config["volatile.base_image"], Config: c.Container.Config, CreationDate: c.Container.CreatedAt, Type: instanceType, Description: c.Container.Description, Devices: deviceConfig.NewDevices(c.Container.Devices), Ephemeral: c.Container.Ephemeral, LastUsedDate: c.Container.LastUsedAt, Name: c.Container.Name, Stateful: c.Container.Stateful, } if applyProfiles { err := state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { inst.Profiles = make([]api.Profile, 0, len(c.Container.Profiles)) profiles, err := cluster.GetProfilesIfEnabled(ctx, tx.Tx(), projectName, c.Container.Profiles) if err != nil { return err } // Get all the profile configs. profileConfigs, err := cluster.GetAllProfileConfigs(ctx, tx.Tx()) if err != nil { return err } // Get all the profile devices. profileDevices, err := cluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return err } for _, profile := range profiles { apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices) if err != nil { return err } inst.Profiles = append(inst.Profiles, *apiProfile) } return nil }) if err != nil { return nil, err } } return inst, nil } // ParseConfigYamlFile decodes the YAML file at path specified into a Config. func ParseConfigYamlFile(path string) (*config.Config, error) { data, err := os.ReadFile(path) if err != nil { return nil, err } backupConf := config.Config{} err = yaml.Load(data, &backupConf) if err != nil { return nil, err } // Default to container if type not specified in backup config. if backupConf.Container != nil && backupConf.Container.Type == "" { backupConf.Container.Type = string(api.InstanceTypeContainer) } return &backupConf, nil } // updateRootDevicePool updates the root disk device in the supplied list of devices to the pool // specified. Returns true if a root disk device has been found and updated otherwise false. func updateRootDevicePool(devices map[string]map[string]string, poolName string) bool { if devices != nil { devName, _, err := instance.GetRootDiskDevice(devices) if err == nil { devices[devName]["pool"] = poolName return true } } return false } // UpdateInstanceConfig updates the instance's backup.yaml configuration file. func UpdateInstanceConfig(c *db.Cluster, b Info, mountPath string) error { backupFilePath := filepath.Join(mountPath, "backup.yaml") // Read in the backup.yaml file. backup, err := ParseConfigYamlFile(backupFilePath) if err != nil { return err } // Update instance information in the backup.yaml. if backup.Container != nil { backup.Container.Name = b.Name backup.Container.Project = b.Project } // Update volume information in the backup.yaml. if backup.Volume != nil { backup.Volume.Name = b.Name backup.Volume.Project = b.Project } var pool *api.StoragePool err = c.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Load the storage pool. _, pool, _, err = tx.GetStoragePool(ctx, b.Pool) return err }) if err != nil { return err } rootDiskDeviceFound := false // Change the pool in the backup.yaml. backup.Pool = pool if backup.Container != nil && updateRootDevicePool(backup.Container.Devices, pool.Name) { rootDiskDeviceFound = true } if backup.Container != nil && updateRootDevicePool(backup.Container.ExpandedDevices, pool.Name) { rootDiskDeviceFound = true } for _, snapshot := range backup.Snapshots { updateRootDevicePool(snapshot.Devices, pool.Name) updateRootDevicePool(snapshot.ExpandedDevices, pool.Name) } if !rootDiskDeviceFound { return errors.New("No root device could be found") } // Write updated backup.yaml file. file, err := os.Create(backupFilePath) if err != nil { return err } defer func() { _ = file.Close() }() data, err := yaml.Dump(&backup, yaml.V2) if err != nil { return err } _, err = file.Write(data) if err != nil { return err } err = file.Sync() if err != nil { return err } return file.Close() } incus-7.0.0/internal/server/backup/backup_info.go000066400000000000000000000115351517523235500220520ustar00rootroot00000000000000package backup import ( "fmt" "io" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/internal/server/backup/config" "github.com/lxc/incus/v7/internal/server/sys" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/api" ) // Type indicates the type of backup. type Type string // TypeUnknown defines the backup type value for unknown backups. const TypeUnknown = Type("") // TypeContainer defines the backup type value for a container. const TypeContainer = Type("container") // TypeVM defines the backup type value for a virtual-machine. const TypeVM = Type("virtual-machine") // TypeCustom defines the backup type value for a custom volume. const TypeCustom = Type("custom") // TypeBucket defines the backup type value for a storage bucket. const TypeBucket = Type("bucket") // DefaultBackupPrefix is the default path prefix used for volume export/import. const DefaultBackupPrefix = "backup" const backupIndexPath = "backup/index.yaml" // InstanceTypeToBackupType converts instance type to backup type. func InstanceTypeToBackupType(instanceType api.InstanceType) Type { switch instanceType { case api.InstanceTypeContainer: return TypeContainer case api.InstanceTypeVM: return TypeVM } return TypeUnknown } // Info represents exported backup information. type Info struct { Project string `json:"-" yaml:"-"` // Project is set during import based on current project. Name string `json:"name" yaml:"name"` Backend string `json:"backend" yaml:"backend"` Pool string `json:"pool" yaml:"pool"` Snapshots []string `json:"snapshots,omitempty" yaml:"snapshots,omitempty"` OptimizedStorage *bool `json:"optimized,omitempty" yaml:"optimized,omitempty"` // Optional field to handle older optimized backups that don't have this field. OptimizedHeader *bool `json:"optimized_header,omitempty" yaml:"optimized_header,omitempty"` // Optional field to handle older optimized backups that don't have this field. Type Type `json:"type,omitempty" yaml:"type,omitempty"` // Type of backup. Config *config.Config `json:"config,omitempty" yaml:"config,omitempty"` // Equivalent of backup.yaml but embedded in index for quick retrieval. } // GetInfo extracts backup information from a given ReadSeeker. func GetInfo(r io.ReadSeeker, sysOS *sys.OS, outputPath string) (*Info, error) { result := Info{} hasIndexFile := false hasOptimizedHeader := false // Define some bools used to create points for OptimizedStorage field. optimizedStorageFalse := false optimizedHeaderFalse := false // Extract. tr, cancelFunc, err := TarReader(r, sysOS, outputPath) if err != nil { return nil, err } defer cancelFunc() for { hdr, err := tr.Next() if err == io.EOF { break // End of archive. } if err != nil { return nil, fmt.Errorf("Error reading backup file info: %w", err) } if hdr.Name == backupIndexPath { loader, err := yaml.NewLoader(localUtil.MaxBytesReader(tr, 1024*1024)) if err != nil { return nil, err } err = loader.Load(&result) if err != nil { return nil, err } hasIndexFile = true // Default to container if index doesn't specify instance type. if result.Type == TypeUnknown { result.Type = TypeContainer } // Default to no optimized header if not specified. if result.OptimizedHeader == nil { result.OptimizedHeader = &optimizedHeaderFalse } if result.OptimizedStorage != nil { hasOptimizedHeader = true } else { // Default to non-optimized if not specified and continue reading to see if // optimized container.bin file present. result.OptimizedStorage = &optimizedStorageFalse } } // Load old backup data. if result.Config == nil && hdr.Name == "backup/container/backup.yaml" { loader, err := yaml.NewLoader(localUtil.MaxBytesReader(tr, 1024*1024)) if err != nil { return nil, err } err = loader.Load(&result.Config) if err != nil { return nil, err } } // If the tarball contains a binary dump of the container, then this is an optimized backup. // This check is only for legacy backups before we introduced the Type and OptimizedStorage fields // in index.yaml, so there is no need to perform this type of check for other types of backups that // have always had these fields populated. if hdr.Name == "backup/container.bin" { optimizedStorageTrue := true result.OptimizedStorage = &optimizedStorageTrue hasOptimizedHeader = true } // Check if we're done processing what we need. if hasIndexFile && hasOptimizedHeader && result.Config != nil { break } } cancelFunc() // Done reading archive. if !hasIndexFile { return nil, fmt.Errorf("Backup is missing at %q", backupIndexPath) } return &result, nil } incus-7.0.0/internal/server/backup/backup_instance.go000066400000000000000000000120271517523235500227200ustar00rootroot00000000000000package backup import ( "context" "os" "strings" "time" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) // Instance represents the backup relevant subset of an instance. // This is used rather than instance.Instance to avoid import loops. type Instance interface { Name() string Project() api.Project Operation() *operations.Operation } // InstanceBackup represents an instance backup. type InstanceBackup struct { CommonBackup instance Instance instanceOnly bool rootOnly bool } // NewInstanceBackup instantiates a new InstanceBackup struct. func NewInstanceBackup(s *state.State, inst Instance, ID int, name string, creationDate time.Time, expiryDate time.Time, instanceOnly bool, rootOnly bool, optimizedStorage bool) *InstanceBackup { return &InstanceBackup{ CommonBackup: CommonBackup{ state: s, id: ID, name: name, creationDate: creationDate, expiryDate: expiryDate, optimizedStorage: optimizedStorage, }, instance: inst, instanceOnly: instanceOnly, rootOnly: rootOnly, } } // InstanceOnly returns whether only the instance itself is to be backed up (without snapshots). func (b *InstanceBackup) InstanceOnly() bool { return b.instanceOnly } // RootOnly returns whether only the instance itself is to be backed up (without dependent volumes). func (b *InstanceBackup) RootOnly() bool { return b.rootOnly } // Instance returns the instance to be backed up. func (b *InstanceBackup) Instance() Instance { return b.instance } // Rename renames an instance backup. func (b *InstanceBackup) Rename(newName string) error { oldBackupPath := internalUtil.VarPath("backups", "instances", project.Instance(b.instance.Project().Name, b.name)) newBackupPath := internalUtil.VarPath("backups", "instances", project.Instance(b.instance.Project().Name, newName)) // Extract the old and new parent backup paths from the old and new backup names rather than use // instance.Name() as this may be in flux if the instance itself is being renamed, whereas the relevant // instance name is encoded into the backup names. oldParentName, _, _ := api.GetParentAndSnapshotName(b.name) oldParentBackupsPath := internalUtil.VarPath("backups", "instances", project.Instance(b.instance.Project().Name, oldParentName)) newParentName, _, _ := api.GetParentAndSnapshotName(newName) newParentBackupsPath := internalUtil.VarPath("backups", "instances", project.Instance(b.instance.Project().Name, newParentName)) // Create the new backup path if doesn't exist. if !util.PathExists(newParentBackupsPath) { err := os.MkdirAll(newParentBackupsPath, 0o700) if err != nil { return err } } // Rename the backup directory. err := os.Rename(oldBackupPath, newBackupPath) if err != nil { return err } // Check if we can remove the old parent directory. empty, _ := internalUtil.PathIsEmpty(oldParentBackupsPath) if empty { err := os.Remove(oldParentBackupsPath) if err != nil { return err } } // Rename the database record. err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RenameInstanceBackup(ctx, b.name, newName) }) if err != nil { return err } oldName := b.name b.name = newName b.state.Events.SendLifecycle(b.instance.Project().Name, lifecycle.InstanceBackupRenamed.Event(b.name, b.instance, map[string]any{"old_name": oldName})) return nil } // Delete removes an instance backup. func (b *InstanceBackup) Delete() error { backupPath := internalUtil.VarPath("backups", "instances", project.Instance(b.instance.Project().Name, b.name)) // Delete the on-disk data. if util.PathExists(backupPath) { err := os.RemoveAll(backupPath) if err != nil { return err } } // Check if we can remove the instance directory. backupsPath := internalUtil.VarPath("backups", "instances", project.Instance(b.instance.Project().Name, b.instance.Name())) empty, _ := internalUtil.PathIsEmpty(backupsPath) if empty { err := os.Remove(backupsPath) if err != nil { return err } } // Remove the database record. err := b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.DeleteInstanceBackup(ctx, b.name) }) if err != nil { return err } b.state.Events.SendLifecycle(b.instance.Project().Name, lifecycle.InstanceBackupDeleted.Event(b.name, b.instance, nil)) return nil } // Render returns an InstanceBackup struct of the backup. func (b *InstanceBackup) Render() *api.InstanceBackup { return &api.InstanceBackup{ Name: strings.SplitN(b.name, "/", 2)[1], CreatedAt: b.creationDate, ExpiresAt: b.expiryDate, InstanceOnly: b.instanceOnly, OptimizedStorage: b.optimizedStorage, } } incus-7.0.0/internal/server/backup/backup_utils.go000066400000000000000000000047501517523235500222600ustar00rootroot00000000000000package backup import ( "archive/tar" "context" "errors" "fmt" "io" "net/http" "net/url" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/lxc/incus/v7/internal/server/storage/s3util" "github.com/lxc/incus/v7/internal/server/sys" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" localtls "github.com/lxc/incus/v7/shared/tls" ) // TarReader rewinds backup file handle r and returns new tar reader and process cleanup function. func TarReader(r io.ReadSeeker, sysOS *sys.OS, outputPath string) (*tar.Reader, context.CancelFunc, error) { _, err := r.Seek(0, io.SeekStart) if err != nil { return nil, nil, err } _, _, unpacker, err := archive.DetectCompressionFile(r) if err != nil { return nil, nil, err } if unpacker == nil { return nil, nil, errors.New("Unsupported backup compression") } tr, cancelFunc, err := archive.CompressedTarReader(context.Background(), r, unpacker, outputPath) if err != nil { return nil, nil, err } return tr, cancelFunc, nil } // Upload handles backup uploads. func Upload(reader *io.PipeReader, req *api.BackupTarget) error { // We want to close the reader as soon as something bad occurs, ensuring that we don't hang on a // pipe that's unable to consume anything. defer func() { _ = reader.Close() }() if req.Protocol != "s3" { return fmt.Errorf("Unsupported backup target protocol %q", req.Protocol) } // Set up an S3 client. uri, err := url.Parse(req.URL) if err != nil { return err } // Get a basic TLS client. tlsConfig := localtls.InitTLSConfig() // Setup the transport. ts := &http.Transport{ MaxIdleConns: 10, IdleConnTimeout: 30 * time.Second, DisableCompression: true, TLSClientConfig: tlsConfig, } cfg := aws.Config{ Region: s3util.RegionFromURL(uri), Credentials: credentials.NewStaticCredentialsProvider(req.AccessKey, req.SecretKey, ""), HTTPClient: &http.Client{Transport: ts}, } client := s3.NewFromConfig(cfg, func(o *s3.Options) { o.BaseEndpoint = aws.String(fmt.Sprintf("%s://%s", uri.Scheme, uri.Host)) o.UsePathStyle = true }) uploader := transfermanager.New(client) _, err = uploader.UploadObject(context.Background(), &transfermanager.UploadObjectInput{ Bucket: aws.String(req.BucketName), Key: aws.String(req.Path), Body: reader, }) if err != nil { return err } return nil } incus-7.0.0/internal/server/backup/backup_volume.go000066400000000000000000000107571517523235500224330ustar00rootroot00000000000000package backup import ( "context" "os" "strings" "time" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" ) // VolumeBackup represents a custom volume backup. type VolumeBackup struct { CommonBackup projectName string poolName string volumeName string volumeOnly bool } // NewVolumeBackup instantiates a new VolumeBackup struct. func NewVolumeBackup(state *state.State, projectName, poolName, volumeName string, ID int, name string, creationDate, expiryDate time.Time, volumeOnly, optimizedStorage bool) *VolumeBackup { return &VolumeBackup{ CommonBackup: CommonBackup{ state: state, id: ID, name: name, creationDate: creationDate, expiryDate: expiryDate, optimizedStorage: optimizedStorage, }, projectName: projectName, poolName: poolName, volumeName: volumeName, volumeOnly: volumeOnly, } } // VolumeOnly returns whether only the volume itself is to be backed up. func (b *VolumeBackup) VolumeOnly() bool { return b.volumeOnly } // OptimizedStorage returns whether the backup is to be performed using optimization format of the storage driver. func (b *VolumeBackup) OptimizedStorage() bool { return b.optimizedStorage } // Rename renames a volume backup. func (b *VolumeBackup) Rename(newName string) error { oldBackupPath := internalUtil.VarPath("backups", "custom", b.poolName, project.StorageVolume(b.projectName, b.name)) newBackupPath := internalUtil.VarPath("backups", "custom", b.poolName, project.StorageVolume(b.projectName, newName)) // Extract the old and new parent backup paths from the old and new backup names rather than use // instance.Name() as this may be in flux if the instance itself is being renamed, whereas the relevant // instance name is encoded into the backup names. oldParentName, _, _ := api.GetParentAndSnapshotName(b.name) oldParentBackupsPath := internalUtil.VarPath("backups", "custom", b.poolName, project.StorageVolume(b.projectName, oldParentName)) newParentName, _, _ := api.GetParentAndSnapshotName(newName) newParentBackupsPath := internalUtil.VarPath("backups", "custom", b.poolName, project.StorageVolume(b.projectName, newParentName)) reverter := revert.New() defer reverter.Fail() // Create the new backup path if doesn't exist. if !util.PathExists(newParentBackupsPath) { err := os.MkdirAll(newParentBackupsPath, 0o700) if err != nil { return err } } // Rename the backup directory. err := os.Rename(oldBackupPath, newBackupPath) if err != nil { return err } reverter.Add(func() { _ = os.Rename(newBackupPath, oldBackupPath) }) // Check if we can remove the old parent directory. empty, _ := internalUtil.PathIsEmpty(oldParentBackupsPath) if empty { err := os.Remove(oldParentBackupsPath) if err != nil { return err } } // Rename the database record. err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RenameVolumeBackup(ctx, b.name, newName) }) if err != nil { return err } reverter.Success() return nil } // Delete removes a volume backup. func (b *VolumeBackup) Delete() error { backupPath := internalUtil.VarPath("backups", "custom", b.poolName, project.StorageVolume(b.projectName, b.name)) // Delete the on-disk data. if util.PathExists(backupPath) { err := os.RemoveAll(backupPath) if err != nil { return err } } // Check if we can remove the volume directory. backupsPath := internalUtil.VarPath("backups", "custom", b.poolName, project.StorageVolume(b.projectName, b.volumeName)) empty, _ := internalUtil.PathIsEmpty(backupsPath) if empty { err := os.Remove(backupsPath) if err != nil { return err } } // Remove the database record. err := b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.DeleteStoragePoolVolumeBackup(ctx, b.name) }) if err != nil { return err } return nil } // Render returns a VolumeBackup struct of the backup. func (b *VolumeBackup) Render() *api.StorageVolumeBackup { return &api.StorageVolumeBackup{ Name: strings.SplitN(b.name, "/", 2)[1], CreatedAt: b.creationDate, ExpiresAt: b.expiryDate, VolumeOnly: b.volumeOnly, OptimizedStorage: b.optimizedStorage, } } incus-7.0.0/internal/server/backup/config/000077500000000000000000000000001517523235500205035ustar00rootroot00000000000000incus-7.0.0/internal/server/backup/config/backup_config.go000066400000000000000000000016271517523235500236320ustar00rootroot00000000000000package config import ( "github.com/lxc/incus/v7/shared/api" ) // Config represents the config of a backup that can be stored in a backup.yaml file (or embedded in index.yaml). type Config struct { Container *api.Instance `yaml:"container,omitempty"` // Used by VM backups too. Snapshots []*api.InstanceSnapshot `yaml:"snapshots,omitempty"` Pool *api.StoragePool `yaml:"pool,omitempty"` Profiles []*api.Profile `yaml:"profiles,omitempty"` Volume *api.StorageVolume `yaml:"volume,omitempty"` VolumeSnapshots []*api.StorageVolumeSnapshot `yaml:"volume_snapshots,omitempty"` DependentVolumes []*Config `yaml:"dependent_volumes,omitempty"` Bucket *api.StorageBucket `yaml:"bucket,omitempty"` BucketKeys []*api.StorageBucketKey `yaml:"bucket_keys,omitempty"` } incus-7.0.0/internal/server/bgp/000077500000000000000000000000001517523235500165415ustar00rootroot00000000000000incus-7.0.0/internal/server/bgp/debug.go000066400000000000000000000041241517523235500201570ustar00rootroot00000000000000package bgp // DebugInfo represents the internal debug state of the BGP server. type DebugInfo struct { Server DebugInfoServer `json:"server" yaml:"server"` Prefixes []DebugInfoPrefix `json:"prefixes" yaml:"prefixes"` Peers []DebugInfoPeer `json:"peers" yaml:"peers"` } // DebugInfoServer exposes the shared listener configuration. type DebugInfoServer struct { Address string `json:"address" yaml:"address"` ASN uint32 `json:"asn" yaml:"asn"` RouterID string `json:"router_id" yaml:"router_id"` Running bool `json:"running" yaml:"running"` } // DebugInfoPrefix exposes details on a single BGP prefix. type DebugInfoPrefix struct { Owner string `json:"owner" yaml:"owner"` Prefix string `json:"prefix" yaml:"prefix"` Nexthop string `json:"nexthop" yaml:"nexthop"` } // DebugInfoPeer exposes details on a single BGP peer. type DebugInfoPeer struct { Address string `json:"address" yaml:"address"` ASN uint32 `json:"asn" yaml:"asn"` Password string `json:"password" yaml:"password"` Count int `json:"count" yaml:"count"` HoldTime uint64 `json:"holdtime" yaml:"holdtime"` } // Debug returns a dump of the current configuration. func (s *Server) Debug() DebugInfo { // Locking. s.mu.Lock() defer s.mu.Unlock() return s.debug() } func (s *Server) debug() DebugInfo { debug := DebugInfo{} // Fill in server state. debug.Server.Running = s.bgp != nil debug.Server.ASN = s.asn debug.Server.Address = s.address debug.Server.RouterID = s.routerID.String() // Fill in the peers. debug.Peers = []DebugInfoPeer{} for _, peer := range s.peers { entry := DebugInfoPeer{} entry.Address = peer.address.String() entry.ASN = peer.asn entry.Password = peer.password entry.Count = peer.count entry.HoldTime = peer.holdtime debug.Peers = append(debug.Peers, entry) } // Fill in the prefixes. debug.Prefixes = []DebugInfoPrefix{} for _, path := range s.paths { entry := DebugInfoPrefix{} entry.Prefix = path.prefix.String() entry.Owner = path.owner entry.Nexthop = path.nexthop.String() debug.Prefixes = append(debug.Prefixes, entry) } return debug } incus-7.0.0/internal/server/bgp/errors.go000066400000000000000000000006731517523235500204120ustar00rootroot00000000000000package bgp import ( "errors" ) // ErrPrefixNotFound is returned when a user provided prefix couldn't be found. var ErrPrefixNotFound = errors.New("Prefix not found") // ErrPeerNotFound is returned when a user provided peer couldn't be found. var ErrPeerNotFound = errors.New("Peer not found") // ErrBadRouterID is returned when an invalid router-id is provided. var ErrBadRouterID = errors.New("Invalid router-id (must be IPv4 address") incus-7.0.0/internal/server/bgp/server.go000066400000000000000000000312501517523235500203770ustar00rootroot00000000000000package bgp import ( "context" "errors" "fmt" "log/slog" "maps" "net" "strconv" "sync" "github.com/google/uuid" bgpAPI "github.com/osrg/gobgp/v4/api" bgpAPIutil "github.com/osrg/gobgp/v4/pkg/apiutil" bgpPacket "github.com/osrg/gobgp/v4/pkg/packet/bgp" bgpServer "github.com/osrg/gobgp/v4/pkg/server" "github.com/lxc/incus/v7/internal/ports" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" ) // Server represents a BGP server instance. type Server struct { bgp *bgpServer.BgpServer // Internal state (to handle reconfiguration) address string asn uint32 routerID net.IP paths map[string]path peers map[string]peer mu sync.Mutex } type path struct { owner string prefix net.IPNet nexthop net.IP } type peer struct { address net.IP asn uint32 password string holdtime uint64 count int } // NewServer returns a new server instance. func NewServer() *Server { // Setup new struct. s := &Server{ paths: map[string]path{}, peers: map[string]peer{}, } return s } func (s *Server) setup() { } // Start sets up the BGP listener. func (s *Server) start(address string, asn uint32, routerID net.IP) error { // If routerID is nil, fill with our best guess. if routerID == nil || routerID.To4() == nil { return ErrBadRouterID } // Check if already running if s.bgp != nil { return errors.New("BGP listener is already running") } // Setup the logger. logHandler := logger.NewSlogHandler("bgp:", logger.Warn) logLevel := &slog.LevelVar{} logLevel.Set(slog.LevelWarn) // Spawn the BGP goroutines. s.bgp = bgpServer.NewBgpServer(bgpServer.LoggerOption(slog.New(logHandler), logLevel)) go s.bgp.Serve() // Get the address and port. addrHost, addrPort, err := net.SplitHostPort(address) if err != nil { addrHost = address addrPort = fmt.Sprintf("%d", ports.BGPDefaultPort) } if addrHost == "" { addrHost = "::" } addrPortInt, err := strconv.ParseInt(addrPort, 10, 32) if err != nil { return err } // Setup the listener configuration. conf := &bgpAPI.Global{ RouterId: routerID.String(), Asn: asn, // Always setup for IPv4 and IPv6. Families: []uint32{0, 1}, // Listen address. ListenAddresses: []string{addrHost}, ListenPort: int32(addrPortInt), } // Start the listener. err = s.bgp.StartBgp(context.Background(), &bgpAPI.StartBgpRequest{Global: conf}) if err != nil { return err } // Copy the path list oldPaths := map[string]path{} maps.Copy(oldPaths, s.paths) // Add existing paths. s.paths = map[string]path{} for _, path := range oldPaths { err := s.addPrefix(path.prefix, path.nexthop, path.owner) if err != nil { return err } } // Copy the peer list. oldPeers := map[string]peer{} maps.Copy(oldPeers, s.peers) // Add existing peers. s.peers = map[string]peer{} for _, peer := range oldPeers { err := s.addPeer(peer.address, peer.asn, peer.password, peer.holdtime) if err != nil { return err } } // Record the address. s.address = address s.asn = asn s.routerID = routerID return nil } // Stop tears down the BGP listener. func (s *Server) stop() error { // Skip if no instance. if s.bgp == nil { return nil } // Save the peer list. oldPeers := map[string]peer{} maps.Copy(oldPeers, s.peers) // Remove all the peers. for _, peer := range s.peers { err := s.removePeer(peer.address) if err != nil { return err } } // Restore peer list. s.peers = oldPeers // Stop the listener. err := s.bgp.StopBgp(context.Background(), &bgpAPI.StopBgpRequest{}) if err != nil { return err } // Mark the daemon as down. s.address = "" s.asn = 0 s.routerID = nil s.bgp = nil return nil } // Configure updates the listener with a new configuration.. func (s *Server) Configure(address string, asn uint32, routerID net.IP) error { // Locking. s.mu.Lock() defer s.mu.Unlock() return s.configure(address, asn, routerID) } func (s *Server) configure(address string, asn uint32, routerID net.IP) error { // Store current configuration for reverting. oldAddress := s.address oldASN := s.asn oldRouterID := s.routerID // Setup reverter. reverter := revert.New() defer reverter.Fail() // Stop the listener. err := s.stop() if err != nil { return fmt.Errorf("Failed to stop current listener: %w", err) } // Check if we should start. if address != "" && asn > 0 && routerID != nil { // Restore old address on failure. reverter.Add(func() { _ = s.start(oldAddress, oldASN, oldRouterID) }) // Start the listener with the new address. err = s.start(address, asn, routerID) if err != nil { return fmt.Errorf("Failed to start new listener: %w", err) } } // All done. reverter.Success() return nil } // AddPrefix adds a new prefix to the BGP server. func (s *Server) AddPrefix(subnet net.IPNet, nexthop net.IP, owner string) error { // Locking. s.mu.Lock() defer s.mu.Unlock() return s.addPrefix(subnet, nexthop, owner) } func (s *Server) addPrefix(subnet net.IPNet, nexthop net.IP, owner string) error { // Check for an existing entry. for _, path := range s.paths { if path.owner != owner || path.prefix.String() != subnet.String() || path.nexthop.String() != nexthop.String() { continue } return nil } // Prepare the prefix. prefixLen, _ := subnet.Mask.Size() prefix := subnet.IP.String() nlri := &bgpAPI.NLRI{Nlri: &bgpAPI.NLRI_Prefix{Prefix: &bgpAPI.IPAddressPrefix{ Prefix: prefix, PrefixLen: uint32(prefixLen), }}} aOrigin := &bgpAPI.Attribute_Origin{Origin: &bgpAPI.OriginAttribute{ Origin: 0, }} // Add the prefix to the server. var pathUUID string if s.bgp != nil { if subnet.IP.To4() != nil { // IPv4 prefix. family := &bgpAPI.Family{ Afi: bgpAPI.Family_AFI_IP, Safi: bgpAPI.Family_SAFI_UNICAST, } aNextHop := &bgpAPI.Attribute_NextHop{NextHop: &bgpAPI.NextHopAttribute{ NextHop: nexthop.String(), }} path := &bgpAPI.Path{ Family: family, Nlri: nlri, Pattrs: []*bgpAPI.Attribute{ { Attr: aOrigin, }, { Attr: aNextHop, }, }, } utilNlri, err := bgpAPIutil.GetNativeNlri(path) if err != nil { return err } utilAttrs, err := bgpAPIutil.GetNativePathAttributes(path) if err != nil { return err } utilPath := &bgpAPIutil.Path{ Family: bgpPacket.NewFamily(bgpPacket.AFI_IP, bgpPacket.SAFI_UNICAST), Nlri: utilNlri, Attrs: utilAttrs, } resp, err := s.bgp.AddPath(bgpAPIutil.AddPathRequest{ Paths: []*bgpAPIutil.Path{utilPath}, }) if err != nil { return err } if len(resp) != 1 { return errors.New("Expected single response from AddPath") } pathUUID = string(resp[0].UUID.String()) } else { // IPv6 prefix. family := &bgpAPI.Family{ Afi: bgpAPI.Family_AFI_IP6, Safi: bgpAPI.Family_SAFI_UNICAST, } v6Attrs := &bgpAPI.Attribute_MpReach{MpReach: &bgpAPI.MpReachNLRIAttribute{ Family: family, NextHops: []string{nexthop.String()}, Nlris: []*bgpAPI.NLRI{nlri}, }} path := &bgpAPI.Path{ Family: family, Nlri: nlri, Pattrs: []*bgpAPI.Attribute{ { Attr: aOrigin, }, { Attr: v6Attrs, }, }, } utilNlri, err := bgpAPIutil.GetNativeNlri(path) if err != nil { return err } utilAttrs, err := bgpAPIutil.GetNativePathAttributes(path) if err != nil { return err } utilPath := &bgpAPIutil.Path{ Family: bgpPacket.NewFamily(bgpPacket.AFI_IP6, bgpPacket.SAFI_UNICAST), Nlri: utilNlri, Attrs: utilAttrs, } resp, err := s.bgp.AddPath(bgpAPIutil.AddPathRequest{ Paths: []*bgpAPIutil.Path{utilPath}, }) if err != nil { return err } if len(resp) != 1 { return errors.New("Expected single response from AddPath") } pathUUID = string(resp[0].UUID.String()) } } else { // Generate a dummy UUID. pathUUID = uuid.New().String() } // Add path to the map. s.paths[pathUUID] = path{ prefix: subnet, nexthop: nexthop, owner: owner, } return nil } // RemovePrefixByOwner removes all prefixes for the provided owner. func (s *Server) RemovePrefixByOwner(owner string) error { // Locking. s.mu.Lock() defer s.mu.Unlock() // Make a copy of the paths dict to safely iterate (path removal mutates it). paths := map[string]path{} maps.Copy(paths, s.paths) // Iterate through the paths and remove them from the server. for pathUUID, path := range paths { if path.owner == owner { err := s.removePrefixByUUID(pathUUID) if err != nil { return err } } } return nil } // RemovePrefix removes a prefix from the BGP server. func (s *Server) RemovePrefix(subnet net.IPNet, nexthop net.IP) error { // Locking. s.mu.Lock() defer s.mu.Unlock() return s.removePrefix(subnet, nexthop) } func (s *Server) removePrefix(subnet net.IPNet, nexthop net.IP) error { found := false for pathUUID, path := range s.paths { if path.prefix.String() != subnet.String() || path.nexthop.String() != nexthop.String() { continue } found = true // Remove the prefix. err := s.removePrefixByUUID(pathUUID) if err != nil { return err } } if !found { return ErrPrefixNotFound } return nil } func (s *Server) removePrefixByUUID(pathUUID string) error { // Remove it from the BGP server. if s.bgp != nil { nativeUUID, err := uuid.Parse(pathUUID) if err != nil { return err } err = s.bgp.DeletePath(bgpAPIutil.DeletePathRequest{UUIDs: []uuid.UUID{nativeUUID}}) if err != nil && err.Error() != "can't find a specified path" { return err } } // Remove the path from the map. delete(s.paths, pathUUID) return nil } // AddPeer adds a new BGP peer. func (s *Server) AddPeer(address net.IP, asn uint32, password string, holdTime uint64) error { // Locking. s.mu.Lock() defer s.mu.Unlock() return s.addPeer(address, asn, password, holdTime) } func (s *Server) addPeer(address net.IP, asn uint32, password string, holdTime uint64) error { // Look for an existing peer. bgpPeer, bgpPeerExists := s.peers[address.String()] if bgpPeerExists { if bgpPeer.asn != asn { return fmt.Errorf("Peer %q already used but with differing ASN (%d vs %d)", address, asn, bgpPeer.asn) } if bgpPeer.password != password { return fmt.Errorf("Peer %q already used but with a different password", address) } // Reuse the existing entry. bgpPeer.count++ s.peers[address.String()] = bgpPeer return nil } // Setup the configuration. n := &bgpAPI.Peer{ // Peer information. Conf: &bgpAPI.PeerConf{ NeighborAddress: address.String(), PeerAsn: uint32(asn), AuthPassword: password, }, // Allow for 120s offline before route removal. GracefulRestart: &bgpAPI.GracefulRestart{ Enabled: true, RestartTime: 3600, }, // Always allow for the maximum multihop. EbgpMultihop: &bgpAPI.EbgpMultihop{ Enabled: true, MultihopTtl: 255, }, } // Add hold time if configured. if holdTime > 0 { n.Timers = &bgpAPI.Timers{ Config: &bgpAPI.TimersConfig{ HoldTime: holdTime, }, } } // Setup peer for dual-stack. n.AfiSafis = make([]*bgpAPI.AfiSafi, 0) for _, f := range []string{"ipv4-unicast", "ipv6-unicast"} { rf, err := bgpPacket.GetFamily(f) if err != nil { return err } afi := rf.Afi() safi := rf.Safi() family := &bgpAPI.Family{ Afi: bgpAPI.Family_Afi(afi), Safi: bgpAPI.Family_Safi(safi), } n.AfiSafis = append(n.AfiSafis, &bgpAPI.AfiSafi{ MpGracefulRestart: &bgpAPI.MpGracefulRestart{ Config: &bgpAPI.MpGracefulRestartConfig{ Enabled: true, }, }, Config: &bgpAPI.AfiSafiConfig{Family: family}, }) } // Add the peer. if s.bgp != nil { err := s.bgp.AddPeer(context.Background(), &bgpAPI.AddPeerRequest{Peer: n}) if err != nil { return err } } // Add the peer to the list. if bgpPeerExists { bgpPeer.count++ s.peers[address.String()] = bgpPeer } else { s.peers[address.String()] = peer{ address: address, asn: asn, password: password, holdtime: holdTime, count: 1, } } return nil } // RemovePeer removes a prefix from the BGP server. func (s *Server) RemovePeer(address net.IP) error { // Locking. s.mu.Lock() defer s.mu.Unlock() return s.removePeer(address) } func (s *Server) removePeer(address net.IP) error { // Find the peer. bgpPeer, bgpPeerExists := s.peers[address.String()] if !bgpPeerExists { return ErrPeerNotFound } // Remove the peer from the BGP server. if s.bgp != nil && bgpPeer.count == 1 { err := s.bgp.DeletePeer(context.Background(), &bgpAPI.DeletePeerRequest{Address: address.String()}) if err != nil { return err } } // Update peer list. if bgpPeer.count == 1 { // Delete the peer. delete(s.peers, address.String()) } else { // Decrease refcount. bgpPeer.count-- s.peers[address.String()] = bgpPeer } return nil } incus-7.0.0/internal/server/certificate/000077500000000000000000000000001517523235500202535ustar00rootroot00000000000000incus-7.0.0/internal/server/certificate/cache.go000066400000000000000000000064361517523235500216560ustar00rootroot00000000000000package certificate import ( "crypto/x509" "encoding/pem" "sync" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) // Cache represents an thread-safe in-memory cache of the certificates in the database. type Cache struct { apiCertificates map[string]api.CertificatePut certificates map[string]*x509.Certificate mu sync.RWMutex } // SetCertificates sets the certificates on the Cache. func (c *Cache) SetCertificates(certificates []*api.Certificate) { c.mu.Lock() defer c.mu.Unlock() c.apiCertificates = make(map[string]api.CertificatePut, len(certificates)) c.certificates = make(map[string]*x509.Certificate, len(certificates)) for _, certificate := range certificates { c.apiCertificates[certificate.Fingerprint] = certificate.CertificatePut certBlock, _ := pem.Decode([]byte(certificate.Certificate)) if certBlock == nil { logger.Warn("Failed decoding certificate", logger.Ctx{"name": certificate.Name}) continue } cert, err := x509.ParseCertificate(certBlock.Bytes) if err != nil { logger.Warn("Failed parsing certificate", logger.Ctx{"name": certificate.Name, "err": err}) continue } c.certificates[certificate.Fingerprint] = cert } } // GetCertificatesAndProjects returns certificate and project maps. func (c *Cache) GetCertificatesAndProjects() (map[Type]map[string]x509.Certificate, map[string][]string) { c.mu.RLock() defer c.mu.RUnlock() certificates := map[Type]map[string]x509.Certificate{} projects := map[string][]string{} for fingerprint, certificate := range c.apiCertificates { certType, err := FromAPIType(certificate.Type) if err != nil { logger.Warn("Failed getting certificate type", logger.Ctx{"name": certificate.Name, "err": err}) continue } cert, ok := c.certificates[fingerprint] if !ok { logger.Warn("Certificate data not found", logger.Ctx{"name": certificate.Name}) continue } _, ok = certificates[certType] if !ok { certificates[certType] = map[string]x509.Certificate{} } certificates[certType][fingerprint] = *cert if certificate.Restricted { projects[fingerprint] = make([]string, len(certificate.Projects)) copy(projects[fingerprint], certificate.Projects) } } return certificates, projects } // GetCertificates returns a certificate map. func (c *Cache) GetCertificates() map[Type]map[string]x509.Certificate { certificates, _ := c.GetCertificatesAndProjects() return certificates } // GetProjects returns a project map. func (c *Cache) GetProjects() map[string][]string { c.mu.RLock() defer c.mu.RUnlock() projects := map[string][]string{} for fingerprint, certificate := range c.apiCertificates { if certificate.Restricted { projects[fingerprint] = make([]string, len(certificate.Projects)) copy(projects[fingerprint], certificate.Projects) } } return projects } // GetAPICertificate returns a read-only copy of the API certificate associated to the given // fingerprint. func (c *Cache) GetAPICertificate(fingerprint string) *api.CertificatePut { c.mu.RLock() defer c.mu.RUnlock() certificate, ok := c.apiCertificates[fingerprint] if !ok { return nil } newCertificate := certificate newCertificate.Projects = make([]string, len(certificate.Projects)) copy(newCertificate.Projects, certificate.Projects) return &newCertificate } incus-7.0.0/internal/server/certificate/type.go000066400000000000000000000013501517523235500215620ustar00rootroot00000000000000package certificate import ( "errors" "github.com/lxc/incus/v7/shared/api" ) // Type indicates the type of the certificate. type Type int // TypeClient indicates a client certificate type. const TypeClient = Type(1) // TypeServer indicates a server certificate type. const TypeServer = Type(2) // TypeMetrics indicates a metrics certificate type. const TypeMetrics = Type(3) // FromAPIType converts an API type to the equivalent Type. func FromAPIType(apiType string) (Type, error) { switch apiType { case api.CertificateTypeClient: return TypeClient, nil case api.CertificateTypeServer: return TypeServer, nil case api.CertificateTypeMetrics: return TypeMetrics, nil } return -1, errors.New("Invalid certificate type") } incus-7.0.0/internal/server/cgroup/000077500000000000000000000000001517523235500172705ustar00rootroot00000000000000incus-7.0.0/internal/server/cgroup/abstraction.go000066400000000000000000000412111517523235500221270ustar00rootroot00000000000000package cgroup import ( "bufio" "bytes" "errors" "fmt" "io/fs" "os" "slices" "strconv" "strings" "github.com/lxc/incus/v7/internal/linux" ) // CGroup represents the main cgroup abstraction. type CGroup struct { rw ReadWriter } // SetMaxProcesses applies a limit to the number of processes. func (cg *CGroup) SetMaxProcesses(limit int64) error { if !cgControllers["pids"] { return ErrControllerMissing } if limit == -1 { return cg.rw.Set("pids", "pids.max", "max") } return cg.rw.Set("pids", "pids.max", fmt.Sprintf("%d", limit)) } // GetMemorySoftLimit returns the soft limit for memory. func (cg *CGroup) GetMemorySoftLimit() (int64, error) { if !cgControllers["memory"] { return -1, ErrControllerMissing } val, err := cg.rw.Get("memory", "memory.high") if err != nil { return -1, err } n, err := strconv.ParseInt(val, 10, 64) if err != nil { return -1, fmt.Errorf("Failed parsing %q: %w", val, err) } return n, nil } // SetMemorySoftLimit set the soft limit for memory. func (cg *CGroup) SetMemorySoftLimit(limit int64) error { if !cgControllers["memory"] { return ErrControllerMissing } if limit == -1 { return cg.rw.Set("memory", "memory.high", "max") } return cg.rw.Set("memory", "memory.high", fmt.Sprintf("%d", limit)) } // GetMemoryLimit return the hard limit for memory. func (cg *CGroup) GetMemoryLimit() (int64, error) { if !cgControllers["memory"] { return -1, ErrControllerMissing } val, err := cg.rw.Get("memory", "memory.max") if err != nil { return -1, err } n, err := strconv.ParseInt(val, 10, 64) if err != nil { return -1, fmt.Errorf("Failed parsing %q: %w", val, err) } return n, nil } // GetEffectiveMemoryLimit return the effective hard limit for memory. // Returns the cgroup memory limit, or if the cgroup memory limit couldn't be determined or is larger than the // total system memory, then the total system memory is returned. func (cg *CGroup) GetEffectiveMemoryLimit() (int64, error) { memoryTotal, err := linux.DeviceTotalMemory() if err != nil { return -1, fmt.Errorf("Failed getting total memory: %q", err) } memoryLimit, err := cg.GetMemoryLimit() if err != nil || memoryLimit > memoryTotal { return memoryTotal, nil } return memoryLimit, nil } // SetMemoryLimit sets the hard limit for memory. func (cg *CGroup) SetMemoryLimit(limit int64) error { if !cgControllers["memory"] { return ErrControllerMissing } if limit == -1 { return cg.rw.Set("memory", "memory.max", "max") } return cg.rw.Set("memory", "memory.max", fmt.Sprintf("%d", limit)) } // GetMemoryUsage returns the current use of memory. func (cg *CGroup) GetMemoryUsage() (int64, error) { if !cgControllers["memory"] { return -1, ErrControllerMissing } val, err := cg.rw.Get("memory", "memory.current") if err != nil { return -1, err } n, err := strconv.ParseInt(val, 10, 64) if err != nil { return -1, fmt.Errorf("Failed parsing %q: %w", val, err) } return n, nil } // GetProcessesUsage returns the current number of pids. func (cg *CGroup) GetProcessesUsage() (int64, error) { if !cgControllers["pids"] { return -1, ErrControllerMissing } val, err := cg.rw.Get("pids", "pids.current") if err != nil { return -1, err } n, err := strconv.ParseInt(val, 10, 64) if err != nil { return -1, fmt.Errorf("Failed parsing %q: %w", val, err) } return n, nil } // SetMemorySwapLimit sets the hard limit for swap. func (cg *CGroup) SetMemorySwapLimit(limit int64) error { if !cgControllers["memory"] { return ErrControllerMissing } if limit == -1 { return cg.rw.Set("memory", "memory.swap.max", "max") } return cg.rw.Set("memory", "memory.swap.max", fmt.Sprintf("%d", limit)) } // GetCPUAcctUsageAll returns the user and system CPU times of each CPU thread in ns used by processes. func (cg *CGroup) GetCPUAcctUsageAll() (map[int64]CPUStats, error) { out := map[int64]CPUStats{} if !cgControllers["cpu"] { return nil, ErrControllerMissing } val, err := cg.rw.Get("cpu", "cpu.stat") if err != nil { return nil, err } stats := CPUStats{} scanner := bufio.NewScanner(strings.NewReader(val)) for scanner.Scan() { fields := strings.Fields(scanner.Text()) switch fields[0] { case "user_usec": val, err := strconv.ParseInt(fields[1], 10, 64) if err != nil { return nil, fmt.Errorf("Failed parsing %q: %w", val, err) } // Convert usec to nsec stats.User = val * 1000 case "system_usec": val, err := strconv.ParseInt(fields[1], 10, 64) if err != nil { return nil, fmt.Errorf("Failed parsing %q: %w", val, err) } // Convert usec to nsec stats.System = val * 1000 } } // Use CPU ID 0 here as cgroup v2 doesn't show the usage of separate CPUs. out[0] = stats return out, nil } // GetCPUAcctUsage returns the total CPU time in ns used by processes. func (cg *CGroup) GetCPUAcctUsage() (int64, error) { if !cgControllers["cpu"] { return -1, ErrControllerMissing } stats, err := cg.rw.Get("cpu", "cpu.stat") if err != nil { return -1, err } scanner := bufio.NewScanner(strings.NewReader(stats)) for scanner.Scan() { fields := strings.Fields(scanner.Text()) if fields[0] != "usage_usec" { continue } val, err := strconv.ParseInt(fields[1], 10, 64) if err != nil { return -1, fmt.Errorf("Failed parsing %q: %w", val, err) } // Convert usec to nsec return val * 1000, nil } return -1, errors.New("Failed getting usage_usec") } // GetEffectiveCPUs returns the total number of effective CPUs. func (cg *CGroup) GetEffectiveCPUs() (int, error) { set, err := cg.GetEffectiveCpuset() if err != nil { return -1, err } return parseCPUSet(set) } // parseCPUSet parses a cpuset string and returns the number of CPUs. func parseCPUSet(set string) (int, error) { var out int fields := strings.Split(strings.TrimSpace(set), ",") for _, value := range fields { // Parse non-range values. if !strings.Contains(value, "-") { _, err := strconv.Atoi(value) if err != nil { return -1, fmt.Errorf("Failed parsing %q: %w", value, err) } out++ continue } // Parse ranges (should be made of two elements only). valueFields := strings.Split(value, "-") if len(valueFields) != 2 { return -1, fmt.Errorf("Failed parsing %q: Invalid range format", value) } startRange, err := strconv.Atoi(valueFields[0]) if err != nil { return -1, fmt.Errorf("Failed parsing %q: %w", valueFields[0], err) } endRange, err := strconv.Atoi(valueFields[1]) if err != nil { return -1, fmt.Errorf("Failed parsing %q: %w", valueFields[1], err) } for i := startRange; i <= endRange; i++ { out++ } } if out == 0 { return -1, fmt.Errorf("Failed parsing %q", set) } return out, nil } // GetMemoryMaxUsage returns the record high for memory usage. func (cg *CGroup) GetMemoryMaxUsage() (int64, error) { return -1, ErrControllerMissing } // GetMemorySwapMaxUsage returns the record high for swap usage. func (cg *CGroup) GetMemorySwapMaxUsage() (int64, error) { return -1, ErrControllerMissing } // SetMemorySwappiness sets swappiness paramet of vmscan. func (cg *CGroup) SetMemorySwappiness(limit int64) error { return ErrControllerMissing } // GetMemorySwapLimit returns the hard limit on swap usage. func (cg *CGroup) GetMemorySwapLimit() (int64, error) { if !cgControllers["memory"] { return -1, ErrControllerMissing } val, err := cg.rw.Get("memory", "memory.swap.max") if err != nil { return -1, err } if val == "max" { return linux.GetMeminfo("SwapTotal") } n, err := strconv.ParseInt(val, 10, 64) if err != nil { return -1, fmt.Errorf("Failed parsing %q: %w", val, err) } return n, nil } // GetMemorySwapUsage return current usage of swap. func (cg *CGroup) GetMemorySwapUsage() (int64, error) { if !cgControllers["memory"] { return -1, ErrControllerMissing } val, err := cg.rw.Get("memory", "memory.swap.current") if err != nil { return -1, err } n, err := strconv.ParseInt(val, 10, 64) if err != nil { return -1, fmt.Errorf("Failed parsing %q: %w", val, err) } return n, nil } // GetBlkioWeight returns the currently allowed range of weights. func (cg *CGroup) GetBlkioWeight() (int64, error) { if !cgControllers["io"] { return -1, ErrControllerMissing } val, err := cg.rw.Get("io", "io.weight") if err != nil { return -1, err } n, err := strconv.ParseInt(val, 10, 64) if err != nil { return -1, fmt.Errorf("Failed parsing %q: %w", val, err) } return n, nil } // SetBlkioWeight sets the currently allowed range of weights. func (cg *CGroup) SetBlkioWeight(limit int64) error { if !cgControllers["io"] { return ErrControllerMissing } return cg.rw.Set("io", "io.weight", fmt.Sprintf("%d", limit)) } // SetBlkioLimit sets the specified read or write limit for a device. func (cg *CGroup) SetBlkioLimit(dev string, oType string, uType string, limit int64) error { if !slices.Contains([]string{"read", "write"}, oType) { return fmt.Errorf("Invalid I/O operation type: %s", oType) } if !slices.Contains([]string{"iops", "bps"}, uType) { return fmt.Errorf("Invalid I/O limit type: %s", uType) } if !cgControllers["io"] { return ErrControllerMissing } var op string switch oType { case "read": op = fmt.Sprintf("r%s", uType) case "write": op = fmt.Sprintf("w%s", uType) } return cg.rw.Set("io", "io.max", fmt.Sprintf("%s %s=%d", dev, op, limit)) } // SetCPUShare sets the weight of each group in the same hierarchy. func (cg *CGroup) SetCPUShare(limit int64) error { if !cgControllers["cpu"] { return ErrControllerMissing } return cg.rw.Set("cpu", "cpu.weight", fmt.Sprintf("%d", limit)) } // SetCPUCfsLimit sets the quota and duration in ms for each scheduling period. func (cg *CGroup) SetCPUCfsLimit(limitPeriod int64, limitQuota int64) error { if !cgControllers["cpu"] { return ErrControllerMissing } if limitPeriod == -1 && limitQuota == -1 { return cg.rw.Set("cpu", "cpu.max", "max") } return cg.rw.Set("cpu", "cpu.max", fmt.Sprintf("%d %d", limitQuota, limitPeriod)) } // GetCPUCfsLimit gets the quota and duration in ms for each scheduling period. func (cg *CGroup) GetCPUCfsLimit() (int64, int64, error) { if !cgControllers["cpu"] { return -1, -1, ErrControllerMissing } cpuMax, err := cg.rw.Get("cpu", "cpu.max") if err != nil { return -1, -1, err } cpuMaxFields := strings.Split(cpuMax, " ") if len(cpuMaxFields) != 2 { return -1, -1, errors.New("Couldn't parse CFS limits") } if cpuMaxFields[0] == "max" { return -1, -1, nil } limitQuota, err := strconv.ParseInt(cpuMaxFields[0], 10, 64) if err != nil { return -1, -1, err } limitPeriod, err := strconv.ParseInt(cpuMaxFields[1], 10, 64) if err != nil { return -1, -1, err } return limitPeriod, limitQuota, nil } // SetHugepagesLimit applies a limit to the number of processes. func (cg *CGroup) SetHugepagesLimit(pageType string, limit int64) error { if !cgControllers["hugetlb"] { return ErrControllerMissing } if limit == -1 { // Apply the overall limit. err := cg.rw.Set("hugetlb", fmt.Sprintf("hugetlb.%s.max", pageType), "max") if err != nil { return err } // Apply the reserved limit. err = cg.rw.Set("hugetlb", fmt.Sprintf("hugetlb.%s.rsvd.max", pageType), "max") if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } return nil } // Apply the overall limit. err := cg.rw.Set("hugetlb", fmt.Sprintf("hugetlb.%s.max", pageType), fmt.Sprintf("%d", limit)) if err != nil { return err } // Apply the reserved limit. err = cg.rw.Set("hugetlb", fmt.Sprintf("hugetlb.%s.rsvd.max", pageType), fmt.Sprintf("%d", limit)) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } return nil } // GetEffectiveCpuset returns the current set of CPUs for the cgroup. func (cg *CGroup) GetEffectiveCpuset() (string, error) { if !cgControllers["cpuset"] { return "", ErrControllerMissing } return cg.rw.Get("cpuset", "cpuset.cpus.effective") } // GetCpuset returns the current set of CPUs for the cgroup. func (cg *CGroup) GetCpuset() (string, error) { if !cgControllers["cpuset"] { return "", ErrControllerMissing } return cg.rw.Get("cpuset", "cpuset.cpus") } // SetCpuset set the currently allowed set of CPUs for the cgroups. func (cg *CGroup) SetCpuset(limit string) error { if !cgControllers["cpuset"] { return ErrControllerMissing } return cg.rw.Set("cpuset", "cpuset.cpus", limit) } // GetMemoryStats returns memory stats. func (cg *CGroup) GetMemoryStats() (map[string]uint64, error) { out := make(map[string]uint64) if !cgControllers["memory"] { return nil, ErrControllerMissing } stats, err := cg.rw.Get("memory", "memory.stat") if err != nil { return nil, err } for _, stat := range strings.Split(stats, "\n") { field := strings.Split(stat, " ") switch field[0] { case "total_active_anon", "active_anon": out["active_anon"], _ = strconv.ParseUint(field[1], 10, 64) case "total_active_file", "active_file": out["active_file"], _ = strconv.ParseUint(field[1], 10, 64) case "total_inactive_anon", "inactive_anon": out["inactive_anon"], _ = strconv.ParseUint(field[1], 10, 64) case "total_inactive_file", "inactive_file": out["inactive_file"], _ = strconv.ParseUint(field[1], 10, 64) case "total_unevictable", "unevictable": out["unevictable"], _ = strconv.ParseUint(field[1], 10, 64) case "total_writeback", "file_writeback": out["writeback"], _ = strconv.ParseUint(field[1], 10, 64) case "total_dirty", "file_dirty": out["dirty"], _ = strconv.ParseUint(field[1], 10, 64) case "total_mapped_file", "file_mapped": out["mapped"], _ = strconv.ParseUint(field[1], 10, 64) case "total_rss": // v1 only out["rss"], _ = strconv.ParseUint(field[1], 10, 64) case "total_shmem", "shmem": out["shmem"], _ = strconv.ParseUint(field[1], 10, 64) case "total_cache", "file": out["cache"], _ = strconv.ParseUint(field[1], 10, 64) } } // Calculated values out["active"] = out["active_anon"] + out["active_file"] out["inactive"] = out["inactive_anon"] + out["inactive_file"] return out, nil } // GetOOMKills returns the number of oom kills. func (cg *CGroup) GetOOMKills() (int64, error) { if !cgControllers["memory"] { return -1, ErrControllerMissing } stats, err := cg.rw.Get("memory", "memory.events") if err != nil { return -1, err } for _, stat := range strings.Split(stats, "\n") { field := strings.Split(stat, " ") // skip incorrect lines if len(field) != 2 { continue } switch field[0] { case "oom_kill": out, _ := strconv.ParseInt(field[1], 10, 64) return out, nil } } return -1, errors.New("Failed getting oom_kill") } // GetIOStats returns disk stats. func (cg *CGroup) GetIOStats() (map[string]*IOStats, error) { partitions, err := os.ReadFile("/proc/partitions") if err != nil { return nil, fmt.Errorf("Failed to read /proc/partitions: %w", err) } // partMap maps major:minor to device names, e.g. 259:0 -> nvme0n1 partMap := make(map[string]string) scanner := bufio.NewScanner(bytes.NewReader(partitions)) for scanner.Scan() { line := scanner.Text() if line == "" { continue } fields := strings.Fields(line) // Ignore the header if fields[0] == "major" { continue } partMap[fmt.Sprintf("%s:%s", fields[0], fields[1])] = fields[3] } // ioMap contains io stats for each device ioMap := make(map[string]*IOStats) if !cgControllers["io"] { return nil, ErrControllerMissing } val, err := cg.rw.Get("io", "io.stat") if err != nil { return nil, fmt.Errorf("Failed getting io.stat: %w", err) } scanner = bufio.NewScanner(strings.NewReader(val)) for scanner.Scan() { var devID string ioStats := &IOStats{} for _, statPart := range strings.Split(scanner.Text(), " ") { // If the stat part is empty, skip it. if statPart == "" { continue } // Skip unknown devices. if statPart == "(unknown)" { devID = "" continue } if strings.Contains(statPart, ":") { // Store the last dev ID as this works around a kernel bug where multiple dev IDs could appear on a single line. devID = statPart continue } // Skip loop devices (major dev ID 7) as they are irrelevant. if strings.HasPrefix(devID, "7:") { continue } // Parse the stat value. statName, statValueStr, found := strings.Cut(statPart, "=") if !found { return nil, fmt.Errorf("Failed extracting io.stat %q (from %q)", statPart, scanner.Text()) } statValue, err := strconv.ParseUint(statValueStr, 10, 64) if err != nil { return nil, fmt.Errorf("Failed parsing io.stat %q %q (from %q): %w", statName, statValueStr, scanner.Text(), err) } switch statName { case "rbytes": ioStats.ReadBytes = statValue case "wbytes": ioStats.WrittenBytes = statValue case "rios": ioStats.ReadsCompleted = statValue case "wios": ioStats.WritesCompleted = statValue } } ioMap[partMap[devID]] = ioStats } return ioMap, nil } incus-7.0.0/internal/server/cgroup/cgroup_cpu.go000066400000000000000000000037341517523235500217740ustar00rootroot00000000000000package cgroup import ( "fmt" "strconv" "strings" ) // DeviceSchedRebalance channel for scheduling a CPU rebalance. var DeviceSchedRebalance = make(chan []string, 2) // TaskSchedulerTrigger triggers a CPU rebalance. func TaskSchedulerTrigger(srcType string, srcName string, srcStatus string) { // Spawn a go routine which then triggers the scheduler select { case DeviceSchedRebalance <- []string{srcType, srcName, srcStatus}: default: // Channel is full, drop the event } } // ParseCPU parses CPU allowances. func ParseCPU(cpuAllowance string, cpuPriority string) (int64, int64, int64, error) { var err error maxShares := int64(100) // Parse priority cpuShares := int64(0) cpuPriorityInt := 10 if cpuPriority != "" { cpuPriorityInt, err = strconv.Atoi(cpuPriority) if err != nil { return -1, -1, -1, err } } cpuShares -= int64(10 - cpuPriorityInt) // Parse allowance cpuCfsQuota := int64(-1) cpuCfsPeriod := int64(-1) if cpuAllowance != "" { if strings.HasSuffix(cpuAllowance, "%") { // Percentage based allocation percent, err := strconv.Atoi(strings.TrimSuffix(cpuAllowance, "%")) if err != nil { return -1, -1, -1, err } cpuShares += int64(float64(maxShares) / float64(100) * float64(percent)) } else { // Time based allocation fields := strings.SplitN(cpuAllowance, "/", 2) if len(fields) != 2 { return -1, -1, -1, fmt.Errorf("Invalid allowance: %s", cpuAllowance) } quota, err := strconv.Atoi(strings.TrimSuffix(fields[0], "ms")) if err != nil { return -1, -1, -1, err } period, err := strconv.Atoi(strings.TrimSuffix(fields[1], "ms")) if err != nil { return -1, -1, -1, err } // Set limit in ms cpuCfsQuota = int64(quota * 1000) cpuCfsPeriod = int64(period * 1000) cpuShares += maxShares } } else { // Default is 100% cpuShares += maxShares } // Deal with a potential negative score if cpuShares < 0 { cpuShares = 0 } return cpuShares, cpuCfsQuota, cpuCfsPeriod, nil } incus-7.0.0/internal/server/cgroup/errors.go000066400000000000000000000005501517523235500211330ustar00rootroot00000000000000package cgroup import ( "errors" ) // ErrControllerMissing indicates that the requested controller isn't setup on the system. var ErrControllerMissing = errors.New("Cgroup controller is missing") // ErrUnknownVersion indicates that a version other than those supported was detected during init. var ErrUnknownVersion = errors.New("Unknown cgroup version") incus-7.0.0/internal/server/cgroup/file.go000066400000000000000000000027011517523235500205360ustar00rootroot00000000000000package cgroup import ( "fmt" "os" "path/filepath" "strings" ) // NewFileReadWriter returns a CGroup instance using the filesystem as its backend. func NewFileReadWriter(pid int) (*CGroup, error) { // Setup the read/writer struct. rw := fileReadWriter{} // Get the cgroup paths. controllers, err := os.ReadFile(fmt.Sprintf("/proc/%d/cgroup", pid)) if err != nil { return nil, err } for _, line := range strings.Split(string(controllers), "\n") { // Skip empty lines. line = strings.TrimSpace(line) if line == "" { continue } // Extract the fields. fields := strings.Split(line, ":") // Check for the unified cgroup. if fields[0] != "0" { continue } path := filepath.Join("/sys/fs/cgroup", fields[2]) if strings.HasSuffix(fields[2], "/init.scope") { path = filepath.Dir(path) } rw.path = path break } cg, err := New(&rw) if err != nil { return nil, err } return cg, nil } type fileReadWriter struct { path string } // Get reads the value of a cgroup key. func (rw *fileReadWriter) Get(controller string, key string) (string, error) { path := filepath.Join(rw.path, key) value, err := os.ReadFile(path) if err != nil { return "", err } return strings.TrimSpace(string(value)), nil } // Set writes a value to a cgroup key. func (rw *fileReadWriter) Set(controller string, key string, value string) error { path := filepath.Join(rw.path, key) return os.WriteFile(path, []byte(value), 0o600) } incus-7.0.0/internal/server/cgroup/init.go000066400000000000000000000067131517523235500205710ustar00rootroot00000000000000package cgroup import ( "bufio" "errors" "io/fs" "os" "path/filepath" "strings" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/warningtype" "github.com/lxc/incus/v7/shared/logger" ) var cgControllers = map[string]bool{} // Resource is a generic type used to abstract resource control features. type Resource int const ( // BlkioWeight resource control. BlkioWeight Resource = iota // CPU resource control. CPU // CPUSet resource control. CPUSet // Hugetlb resource control. Hugetlb // IO resource control. IO // Memory resource control. Memory // Pids resource control. Pids ) // Supports indicates whether or not a given cgroup resource is controllable. func Supports(resource Resource) bool { switch resource { case CPU: return cgControllers["cpu"] case CPUSet: return cgControllers["cpuset"] case Hugetlb: return cgControllers["hugetlb"] case IO: return cgControllers["io"] case Memory: return cgControllers["memory"] case Pids: return cgControllers["pids"] } return false } // Warnings returns a list of CGroup warnings. func Warnings() []cluster.Warning { warnings := []cluster.Warning{} if !Supports(CPU) { warnings = append(warnings, cluster.Warning{ TypeCode: warningtype.MissingCGroupCPUController, LastMessage: "CPU time limits will be ignored", }) } if !Supports(CPUSet) { warnings = append(warnings, cluster.Warning{ TypeCode: warningtype.MissingCGroupCPUController, LastMessage: "CPU pinning will be ignored", }) } if !Supports(Hugetlb) { warnings = append(warnings, cluster.Warning{ TypeCode: warningtype.MissingCGroupHugetlbController, LastMessage: "hugepage limits will be ignored", }) } if !Supports(IO) { warnings = append(warnings, cluster.Warning{ TypeCode: warningtype.MissingCGroupBlkio, LastMessage: "disk I/O limits will be ignored", }) } if !Supports(Memory) { warnings = append(warnings, cluster.Warning{ TypeCode: warningtype.MissingCGroupMemoryController, LastMessage: "memory limits will be ignored", }) } if !Supports(Pids) { warnings = append(warnings, cluster.Warning{ TypeCode: warningtype.MissingCGroupPidsController, LastMessage: "process limits will be ignored", }) } return warnings } // Init initializes cgroups. func Init() { // Go through the list of resource controllers for Incus. selfCg, err := os.Open("/proc/self/cgroup") if err != nil { if errors.Is(err, fs.ErrNotExist) { logger.Warnf("System doesn't appear to support CGroups") } else { logger.Errorf("Unable to load list of cgroups: %v", err) } return } defer func() { _ = selfCg.Close() }() // Go through the file line by line. scanSelfCg := bufio.NewScanner(selfCg) for scanSelfCg.Scan() { line := strings.TrimSpace(scanSelfCg.Text()) fields := strings.SplitN(line, ":", 3) // Ignore all V1 controllers. if fields[1] != "" { continue } // Parse V2 controllers. dedicatedPath := filepath.Join(cgPath, "cgroup.controllers") controllers, err := os.Open(dedicatedPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { logger.Errorf("Unable to load cgroup.controllers") return } if err == nil { scanControllers := bufio.NewScanner(controllers) for scanControllers.Scan() { line := strings.TrimSpace(scanControllers.Text()) for _, entry := range strings.Split(line, " ") { cgControllers[entry] = true } } } _ = controllers.Close() } } incus-7.0.0/internal/server/cgroup/load.go000066400000000000000000000004311517523235500205340ustar00rootroot00000000000000package cgroup import ( "errors" ) // New setups a new CGroup abstraction using the provided read/writer. func New(rw ReadWriter) (*CGroup, error) { if rw == nil { return nil, errors.New("A CGroup read/writer is required") } cg := CGroup{} cg.rw = rw return &cg, nil } incus-7.0.0/internal/server/cgroup/types.go000066400000000000000000000007471517523235500207730ustar00rootroot00000000000000package cgroup var cgPath = "/sys/fs/cgroup" // The ReadWriter interface is used to read/write cgroup data. type ReadWriter interface { Get(controller string, key string) (string, error) Set(controller string, key string, value string) error } // IOStats represent IO stats. type IOStats struct { ReadBytes uint64 ReadsCompleted uint64 WrittenBytes uint64 WritesCompleted uint64 } // CPUStats represent CPU stats. type CPUStats struct { User int64 System int64 } incus-7.0.0/internal/server/cluster/000077500000000000000000000000001517523235500174525ustar00rootroot00000000000000incus-7.0.0/internal/server/cluster/config/000077500000000000000000000000001517523235500207175ustar00rootroot00000000000000incus-7.0.0/internal/server/cluster/config/config.go000066400000000000000000001214351517523235500225210ustar00rootroot00000000000000package config import ( "context" "errors" "fmt" "maps" "strconv" "strings" "time" "github.com/sirupsen/logrus" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/config" "github.com/lxc/incus/v7/internal/server/db" scriptletLoad "github.com/lxc/incus/v7/internal/server/scriptlet/load" "github.com/lxc/incus/v7/shared/validate" ) // Config holds cluster-wide configuration values. type Config struct { tx *db.ClusterTx // DB transaction the values in this config are bound to. m config.Map // Low-level map holding the config values. } // Load loads a new Config object with the current cluster configuration // values fetched from the database. func Load(ctx context.Context, tx *db.ClusterTx) (*Config, error) { // Load current raw values from the database, any error is fatal. values, err := tx.Config(ctx) if err != nil { return nil, fmt.Errorf("cannot fetch node config from database: %w", err) } m, err := config.SafeLoad(ConfigSchema, values) if err != nil { return nil, fmt.Errorf("failed to load node config: %w", err) } return &Config{tx: tx, m: m}, nil } // BackupsCompressionAlgorithm returns the compression algorithm to use for backups. func (c *Config) BackupsCompressionAlgorithm() string { return c.m.GetString("backups.compression_algorithm") } // MetricsAuthentication checks whether metrics API requires authentication. func (c *Config) MetricsAuthentication() bool { return c.m.GetBool("core.metrics_authentication") } // BGPASN returns the BGP ASN setting. func (c *Config) BGPASN() int64 { return c.m.GetInt64("core.bgp_asn") } // HTTPSAllowedHeaders returns the relevant CORS setting. func (c *Config) HTTPSAllowedHeaders() string { return c.m.GetString("core.https_allowed_headers") } // HTTPSAllowedMethods returns the relevant CORS setting. func (c *Config) HTTPSAllowedMethods() string { return c.m.GetString("core.https_allowed_methods") } // HTTPSAllowedOrigin returns the relevant CORS setting. func (c *Config) HTTPSAllowedOrigin() string { return c.m.GetString("core.https_allowed_origin") } // HTTPSAllowedCredentials returns the relevant CORS setting. func (c *Config) HTTPSAllowedCredentials() bool { return c.m.GetBool("core.https_allowed_credentials") } // TrustCACertificates returns whether client certificates are checked // against a CA. func (c *Config) TrustCACertificates() bool { return c.m.GetBool("core.trust_ca_certificates") } // ProxyHTTPS returns the configured HTTPS proxy, if any. func (c *Config) ProxyHTTPS() string { return c.m.GetString("core.proxy_https") } // ProxyHTTP returns the configured HTTP proxy, if any. func (c *Config) ProxyHTTP() string { return c.m.GetString("core.proxy_http") } // ProxyIgnoreHosts returns the configured ignore-hosts proxy setting, if any. func (c *Config) ProxyIgnoreHosts() string { return c.m.GetString("core.proxy_ignore_hosts") } // HTTPSTrustedProxy returns the configured HTTPS trusted proxy setting, if any. func (c *Config) HTTPSTrustedProxy() string { return c.m.GetString("core.https_trusted_proxy") } // OfflineThreshold returns the configured heartbeat threshold, i.e. the // number of seconds before after which an unresponsive node is considered // offline.. func (c *Config) OfflineThreshold() time.Duration { n := c.m.GetInt64("cluster.offline_threshold") return time.Duration(n) * time.Second } // ImagesMinimalReplica returns the numbers of nodes for cluster images replication. func (c *Config) ImagesMinimalReplica() int64 { return c.m.GetInt64("cluster.images_minimal_replica") } // MaxVoters returns the maximum number of members in a cluster that will be // assigned the voter role. func (c *Config) MaxVoters() int64 { return c.m.GetInt64("cluster.max_voters") } // MaxStandBy returns the maximum number of standby members in a cluster that // will be assigned the stand-by role. func (c *Config) MaxStandBy() int64 { return c.m.GetInt64("cluster.max_standby") } // ClusterRebalanceBatch returns maximum number of instances to move during one re-balancing run. func (c *Config) ClusterRebalanceBatch() int64 { return c.m.GetInt64("cluster.rebalance.batch") } // ClusterRebalanceCooldown returns amount of time during which an instance will not be moved again. func (c *Config) ClusterRebalanceCooldown() string { return c.m.GetString("cluster.rebalance.cooldown") } // ClusterRebalanceInterval returns the interval at which to evaluate re-balanicng. func (c *Config) ClusterRebalanceInterval() int64 { return c.m.GetInt64("cluster.rebalance.interval") } // ClusterRebalanceThreshold returns load difference between most and least busy server // needed to trigger a migration. func (c *Config) ClusterRebalanceThreshold() int64 { return c.m.GetInt64("cluster.rebalance.threshold") } // NetworkOVNIntegrationBridge returns the integration OVS bridge to use for OVN networks. func (c *Config) NetworkOVNIntegrationBridge() string { return c.m.GetString("network.ovn.integration_bridge") } // NetworkOVNNorthboundConnection returns the OVN northbound database connection string for OVN networks. func (c *Config) NetworkOVNNorthboundConnection() string { return c.m.GetString("network.ovn.northbound_connection") } // NetworkOVNSSL returns all three SSL configuration keys needed for a connection. func (c *Config) NetworkOVNSSL() (string, string, string) { return c.m.GetString("network.ovn.ca_cert"), c.m.GetString("network.ovn.client_cert"), c.m.GetString("network.ovn.client_key") } // LinstorControllerConnection returns the Linstor controller connection string. func (c *Config) LinstorControllerConnection() string { return c.m.GetString("storage.linstor.controller_connection") } // LinstorSSL returns all three SSL configuration keys needed for a Linstor controller connection. func (c *Config) LinstorSSL() (string, string, string) { return c.m.GetString("storage.linstor.ca_cert"), c.m.GetString("storage.linstor.client_cert"), c.m.GetString("storage.linstor.client_key") } // ShutdownAction returns the action to perform when the server is being shut down. func (c *Config) ShutdownAction() string { return c.m.GetString("core.shutdown_action") } // ShutdownTimeout returns the number of minutes to wait for running operation to complete // before the server shuts down. func (c *Config) ShutdownTimeout() time.Duration { n := c.m.GetInt64("core.shutdown_timeout") return time.Duration(n) * time.Minute } // ImagesDefaultArchitecture returns the default architecture. func (c *Config) ImagesDefaultArchitecture() string { return c.m.GetString("images.default_architecture") } // ImagesCompressionAlgorithm returns the compression algorithm to use for images. func (c *Config) ImagesCompressionAlgorithm() string { return c.m.GetString("images.compression_algorithm") } // ImagesAutoUpdateCached returns whether or not to auto update cached images. func (c *Config) ImagesAutoUpdateCached() bool { return c.m.GetBool("images.auto_update_cached") } // ImagesAutoUpdateIntervalHours returns interval in hours at which to look for update to cached images. func (c *Config) ImagesAutoUpdateIntervalHours() int64 { return c.m.GetInt64("images.auto_update_interval") } // ImagesRemoteCacheExpiryDays returns the number of days after which an unused cached remote image will be flushed. func (c *Config) ImagesRemoteCacheExpiryDays() int64 { return c.m.GetInt64("images.remote_cache_expiry") } // InstancesNICHostname returns hostname mode to use for instance NICs. func (c *Config) InstancesNICHostname() string { return c.m.GetString("instances.nic.host_name") } // InstancesPlacementScriptlet returns the instances placement scriptlet source code. func (c *Config) InstancesPlacementScriptlet() string { return c.m.GetString("instances.placement.scriptlet") } // AuthorizationScriptlet returns the authorization scriptlet source code. func (c *Config) AuthorizationScriptlet() string { return c.m.GetString("authorization.scriptlet") } // InstancesLXCFSPerInstance returns whether LXCFS should be run on a per-instance basis. func (c *Config) InstancesLXCFSPerInstance() bool { return c.m.GetBool("instances.lxcfs.per_instance") } // LokiServer returns all the Loki settings needed to connect to a server. func (c *Config) LokiServer() (string, string, string, string, string, string, []string, []string) { var types []string var labels []string if c.m.GetString("loki.types") != "" { types = strings.Split(c.m.GetString("loki.types"), ",") } if c.m.GetString("loki.labels") != "" { labels = strings.Split(c.m.GetString("loki.labels"), ",") } return c.m.GetString("loki.api.url"), c.m.GetString("loki.auth.username"), c.m.GetString("loki.auth.password"), c.m.GetString("loki.api.ca_cert"), c.m.GetString("loki.instance"), c.m.GetString("loki.loglevel"), labels, types } // ACME returns all ACME settings needed for certificate renewal. func (c *Config) ACME() (string, string, string, bool, string) { return c.m.GetString("acme.domain"), c.m.GetString("acme.email"), c.m.GetString("acme.ca_url"), c.m.GetBool("acme.agree_tos"), c.m.GetString("acme.challenge") } // ACMEDNS returns all ACME DNS settings needed for DNS-01 challenge. func (c *Config) ACMEDNS() (string, []string, []string) { var environment []string var resolvers []string if c.m.GetString("acme.provider.environment") != "" { lines := strings.Split(strings.TrimSpace(c.m.GetString("acme.provider.environment")), "\n") for _, line := range lines { if len(strings.TrimSpace(line)) == 0 { continue } parts := strings.SplitN(line, "=", 2) if len(parts) != 2 { logrus.Warnf("Malformed line in config string: %q", line) continue } key := strings.TrimSpace(parts[0]) value := strings.TrimSpace(parts[1]) environment = append(environment, strings.Join([]string{key, value}, "=")) } } if c.m.GetString("acme.provider.resolvers") != "" { resolvers = strings.Split(c.m.GetString("acme.provider.resolvers"), ",") } return c.m.GetString("acme.provider"), environment, resolvers } // ACMEHTTP returns all ACME HTTP settings needed for HTTP-01 challenge. func (c *Config) ACMEHTTP() string { return c.m.GetString("acme.http.port") } // ClusterJoinTokenExpiry returns the cluster join token expiry. func (c *Config) ClusterJoinTokenExpiry() string { return c.m.GetString("cluster.join_token_expiry") } // RemoteTokenExpiry returns the time after which a remote add token expires. func (c *Config) RemoteTokenExpiry() string { return c.m.GetString("core.remote_token_expiry") } // OIDCServer returns all the OpenID Connect settings needed to connect to a server. func (c *Config) OIDCServer() (string, string, string, string, string) { return c.m.GetString("oidc.issuer"), c.m.GetString("oidc.client.id"), c.m.GetString("oidc.scopes"), c.m.GetString("oidc.audience"), c.m.GetString("oidc.claim") } // ClusterHealingThreshold returns the configured healing threshold, i.e. the // number of seconds after which an offline node will be evacuated automatically. If the config key // is set but its value is lower than cluster.offline_threshold it returns // the value of cluster.offline_threshold instead. If this feature is disabled, it returns 0. func (c *Config) ClusterHealingThreshold() time.Duration { n := c.m.GetInt64("cluster.healing_threshold") if n == 0 { return 0 } healingThreshold := time.Duration(n) * time.Second offlineThreshold := c.OfflineThreshold() if healingThreshold < offlineThreshold { return offlineThreshold } return healingThreshold } // OpenFGA returns all OpenFGA settings need to interact with an OpenFGA server. func (c *Config) OpenFGA() (apiURL string, apiToken string, storeID string) { return c.m.GetString("openfga.api.url"), c.m.GetString("openfga.api.token"), c.m.GetString("openfga.store.id") } // NetworkHWAddrPattern returns the MAC address pattern used in the cluster. func (c *Config) NetworkHWAddrPattern() string { return c.m.GetString("network.hwaddr_pattern") } // Loggers returns a map where the key is the logger name and the value is its type. func (c *Config) Loggers() (map[string]string, error) { result := make(map[string]string) // Backward compatibility with old Loki config keys if c.m.GetString("loki.api.url") != "" { result["loki"] = "loki" } for k, v := range c.m.Dump() { if !strings.HasPrefix(k, "logging.") { continue } fields := strings.Split(k, ".") if len(fields) < 3 { return nil, fmt.Errorf("%s is not a valid logging config key", k) } loggingKey := strings.Join(fields[2:], ".") if loggingKey != "target.type" { continue } loggerName := fields[1] _, exists := result[loggerName] if !exists { result[loggerName] = v } } return result, nil } // LoggingCommonConfig returns the logging configuration common to all types of loggers. func (c *Config) LoggingCommonConfig(loggerName string) (string, string, string, string) { if loggerName == "loki" && c.m.GetString("loki.api.url") != "" { return "", "", c.m.GetString("loki.loglevel"), c.m.GetString("loki.types") } prefix := fmt.Sprintf("logging.%s", loggerName) lifecycleProjectsKey := fmt.Sprintf("%s.%s", prefix, "lifecycle.projects") lifecycleTypesKey := fmt.Sprintf("%s.%s", prefix, "lifecycle.types") loggingLevelKey := fmt.Sprintf("%s.%s", prefix, "logging.level") typesKey := fmt.Sprintf("%s.%s", prefix, "types") return c.m.GetString(lifecycleProjectsKey), c.m.GetString(lifecycleTypesKey), c.m.GetString(loggingLevelKey), c.m.GetString(typesKey) } // LoggingConfigForSyslog returns the logging configuration for the syslog logger type. func (c *Config) LoggingConfigForSyslog(loggerName string) (string, string) { prefix := fmt.Sprintf("logging.%s", loggerName) addressKey := fmt.Sprintf("%s.%s", prefix, "target.address") facilityKey := fmt.Sprintf("%s.%s", prefix, "target.facility") return c.m.GetString(addressKey), c.m.GetString(facilityKey) } // LoggingConfigForLoki returns all the Loki settings needed to connect to a server. func (c *Config) LoggingConfigForLoki(loggerName string) (string, string, string, string, string, string, int) { if loggerName == "loki" && c.m.GetString("loki.api.url") != "" { return c.m.GetString("loki.api.url"), c.m.GetString("loki.auth.username"), c.m.GetString("loki.auth.password"), c.m.GetString("loki.api.ca_cert"), c.m.GetString("loki.instance"), c.m.GetString("loki.labels"), 3 } prefix := fmt.Sprintf("logging.%s", loggerName) addressKey := fmt.Sprintf("%s.%s", prefix, "target.address") usernameKey := fmt.Sprintf("%s.%s", prefix, "target.username") passwordKey := fmt.Sprintf("%s.%s", prefix, "target.password") caCertKey := fmt.Sprintf("%s.%s", prefix, "target.ca_cert") instanceKey := fmt.Sprintf("%s.%s", prefix, "target.instance") labelsKey := fmt.Sprintf("%s.%s", prefix, "target.labels") retryKey := fmt.Sprintf("%s.%s", prefix, "target.retry") return c.m.GetString(addressKey), c.m.GetString(usernameKey), c.m.GetString(passwordKey), c.m.GetString(caCertKey), c.m.GetString(instanceKey), c.m.GetString(labelsKey), int(c.m.GetInt64(retryKey)) } // LoggingConfigForWebhook returns the logging configuration for the webhook logger type. func (c *Config) LoggingConfigForWebhook(loggerName string) (string, string, string, string, int) { prefix := fmt.Sprintf("logging.%s", loggerName) addressKey := fmt.Sprintf("%s.%s", prefix, "target.address") usernameKey := fmt.Sprintf("%s.%s", prefix, "target.username") passwordKey := fmt.Sprintf("%s.%s", prefix, "target.password") caCertKey := fmt.Sprintf("%s.%s", prefix, "target.ca_cert") retryKey := fmt.Sprintf("%s.%s", prefix, "target.retry") return c.m.GetString(addressKey), c.m.GetString(usernameKey), c.m.GetString(passwordKey), c.m.GetString(caCertKey), int(c.m.GetInt64(retryKey)) } // Dump current configuration keys and their values. Keys with values matching // their defaults are omitted. func (c *Config) Dump() map[string]string { return c.m.Dump() } // Replace the current configuration with the given values. // // Return what has actually changed. func (c *Config) Replace(values map[string]string) (map[string]string, error) { return c.update(values) } // Patch changes only the configuration keys in the given map. // // Return what has actually changed. func (c *Config) Patch(patch map[string]string) (map[string]string, error) { values := c.Dump() // Use current values as defaults maps.Copy(values, patch) return c.update(values) } func (c *Config) update(values map[string]string) (map[string]string, error) { changed, err := c.m.Change(values) if err != nil { return nil, err } err = c.tx.UpdateClusterConfig(changed) if err != nil { return nil, fmt.Errorf("cannot persist configuration changes: %w", err) } return changed, nil } // ConfigSchema defines available server configuration keys. var ConfigSchema = config.Schema{ // gendoc:generate(entity=server, group=acme, key=acme.ca_url) // // --- // type: string // scope: global // defaultdesc: `https://acme-v02.api.letsencrypt.org/directory` // shortdesc: URL to the directory resource of the ACME service "acme.ca_url": {Default: "https://acme-v02.api.letsencrypt.org/directory"}, // gendoc:generate(entity=server, group=acme, key=acme.domain) // // --- // type: string // scope: global // shortdesc: Domain for which the certificate is issued "acme.domain": {}, // gendoc:generate(entity=server, group=acme, key=acme.email) // // --- // type: string // scope: global // shortdesc: Email address used for the account registration "acme.email": {}, // gendoc:generate(entity=server, group=acme, key=acme.agree_tos) // // --- // type: bool // scope: global // defaultdesc: `false` // shortdesc: Agree to ACME terms of service "acme.agree_tos": {Type: config.Bool, Default: "false"}, // gendoc:generate(entity=server, group=acme, key=acme.challenge) // Possible values are `DNS-01` and `HTTP-01`. // --- // type: string // scope: global // defaultdesc: `HTTP-01` // shortdesc: ACME challenge type to use "acme.challenge": {Type: config.String, Default: "HTTP-01", Validator: validate.Optional(validate.IsOneOf("DNS-01", "HTTP-01"))}, // gendoc:generate(entity=server, group=acme, key=acme.provider) // // --- // type: string // scope: global // defaultdesc: `` // shortdesc: Backend provider for the challenge (used by DNS-01) "acme.provider": {Type: config.String, Default: ""}, // gendoc:generate(entity=server, group=acme, key=acme.provider.environment) // // --- // type: string // scope: global // defaultdesc: `` // shortdesc: Environment variables to set during the challenge (used by DNS-01) "acme.provider.environment": {Type: config.String, Default: ""}, // gendoc:generate(entity=server, group=acme, key=acme.provider.resolvers) // DNS resolvers to use for performing (recursive) `CNAME` resolving and apex domain determination during DNS-01 challenge. // --- // type: string // scope: global // defaultdesc: `` // shortdesc: Comma-separated list of DNS resolvers (used by DNS-01) "acme.provider.resolvers": {Type: config.String, Default: ""}, // gendoc:generate(entity=server, group=acme, key=acme.http.port) // Set the port and interface to use for HTTP-01 based challenges to listen on // --- // type: string // scope: global // defaultdesc: `:80` // shortdesc: Port and interface for HTTP server (used by HTTP-01) "acme.http.port": {Default: ":80", Validator: validate.Optional(validate.IsListenAddress(true, true, false))}, // gendoc:generate(entity=server, group=miscellaneous, key=authorization.scriptlet) // When using scriptlet-based authorization, this option stores the scriptlet. // --- // type: string // scope: global // shortdesc: Authorization scriptlet "authorization.scriptlet": {Validator: validate.Optional(scriptletLoad.AuthorizationValidate)}, // gendoc:generate(entity=server, group=miscellaneous, key=backups.compression_algorithm) // Possible values are `bzip2`, `gzip`, `lz4`, `lzma`, `xz`, `zstd` or `none`. // --- // type: string // scope: global // defaultdesc: `gzip` // shortdesc: Compression algorithm to use for backups "backups.compression_algorithm": {Default: "gzip", Validator: validate.IsCompressionAlgorithm}, // gendoc:generate(entity=server, group=cluster, key=cluster.offline_threshold) // Specify the number of seconds after which an unresponsive member is considered offline. // --- // type: integer // scope: global // defaultdesc: `20` // shortdesc: Threshold when an unresponsive member is considered offline "cluster.offline_threshold": {Type: config.Int64, Default: offlineThresholdDefault(), Validator: offlineThresholdValidator}, // gendoc:generate(entity=server, group=cluster, key=cluster.images_minimal_replica) // Specify the minimal number of cluster members that keep a copy of a particular image. // Set this option to `1` for no replication, or to `-1` to replicate images on all members. // --- // type: integer // scope: global // defaultdesc: `3` // shortdesc: Number of cluster members that replicate an image "cluster.images_minimal_replica": {Type: config.Int64, Default: "3", Validator: imageMinimalReplicaValidator}, // gendoc:generate(entity=server, group=cluster, key=cluster.healing_threshold) // Specify the number of seconds after which an offline cluster member is to be evacuated. // To disable evacuating offline members, set this option to `0`. // --- // type: integer // scope: global // defaultdesc: `0` // shortdesc: Threshold when to evacuate an offline cluster member "cluster.healing_threshold": {Type: config.Int64, Default: "0"}, // gendoc:generate(entity=server, group=cluster, key=cluster.join_token_expiry) // // --- // type: string // scope: global // defaultdesc: `3H` // shortdesc: Time after which a cluster join token expires "cluster.join_token_expiry": {Type: config.String, Default: "3H", Validator: expiryValidator}, // gendoc:generate(entity=server, group=cluster, key=cluster.max_voters) // Specify the maximum number of cluster members that are assigned the database voter role. // This must be an odd number >= `3`. // --- // type: integer // scope: global // defaultdesc: `3` // shortdesc: Number of database voter members "cluster.max_voters": {Type: config.Int64, Default: "3", Validator: maxVotersValidator}, // gendoc:generate(entity=server, group=cluster, key=cluster.max_standby) // Specify the maximum number of cluster members that are assigned the database stand-by role. // This must be a number between `0` and `5`. // --- // type: integer // scope: global // defaultdesc: `2` // shortdesc: Number of database stand-by members "cluster.max_standby": {Type: config.Int64, Default: "2", Validator: maxStandByValidator}, // gendoc:generate(entity=server, group=cluster, key=cluster.rebalance.batch) // // --- // type: integer // scope: global // defaultdesc: `1` // shortdesc: Maximum number of instances to move during one re-balancing run "cluster.rebalance.batch": {Type: config.Int64, Default: "1"}, // gendoc:generate(entity=server, group=cluster, key=cluster.rebalance.cooldown) // // --- // type: string // scope: global // defaultdesc: `6H` // shortdesc: Amount of time during which an instance will not be moved again "cluster.rebalance.cooldown": {Type: config.String, Default: "6H", Validator: validate.Optional(expiryValidator)}, // gendoc:generate(entity=server, group=cluster, key=cluster.rebalance.interval) // // --- // type: integer // scope: global // defaultdesc: `0` // shortdesc: How often (in minutes) to consider re-balancing things. 0 to disable (default) "cluster.rebalance.interval": {Type: config.Int64, Default: "0"}, // gendoc:generate(entity=server, group=cluster, key=cluster.rebalance.threshold) // // --- // type: integer // scope: global // defaultdesc: `20` // shortdesc: Percentage load difference between most and least busy server needed to trigger a migration "cluster.rebalance.threshold": {Type: config.Int64, Default: "20", Validator: validate.Optional(rebalanceThresholdValidator)}, // gendoc:generate(entity=server, group=core, key=core.metrics_authentication) // // --- // type: bool // scope: global // defaultdesc: `true` // shortdesc: Whether to enforce authentication on the metrics endpoint "core.metrics_authentication": {Type: config.Bool, Default: "true"}, // gendoc:generate(entity=server, group=core, key=core.bgp_asn) // // --- // type: string // scope: global // shortdesc: BGP Autonomous System Number for the local server "core.bgp_asn": {Type: config.Int64, Default: "0", Validator: validate.Optional(validate.IsInRange(0, 4294967294))}, // gendoc:generate(entity=server, group=core, key=core.https_allowed_headers) // // --- // type: string // scope: global // shortdesc: `Access-Control-Allow-Headers` HTTP header value "core.https_allowed_headers": {}, // gendoc:generate(entity=server, group=core, key=core.https_allowed_methods) // // --- // type: string // scope: global // shortdesc: `Access-Control-Allow-Methods` HTTP header value "core.https_allowed_methods": {}, // gendoc:generate(entity=server, group=core, key=core.https_allowed_origin) // // --- // type: string // scope: global // shortdesc: `Access-Control-Allow-Origin` HTTP header value "core.https_allowed_origin": {}, // gendoc:generate(entity=server, group=core, key=core.https_allowed_credentials) // If enabled, the `Access-Control-Allow-Credentials` HTTP header value is set to `true`. // --- // type: bool // scope: global // defaultdesc: `false` // shortdesc: Whether to set `Access-Control-Allow-Credentials` "core.https_allowed_credentials": {Type: config.Bool, Default: "false"}, // gendoc:generate(entity=server, group=core, key=core.https_trusted_proxy) // Specify a comma-separated list of IP addresses of trusted servers that provide the client's address through the proxy connection header. // --- // type: string // scope: global // shortdesc: Trusted servers to provide the client's address "core.https_trusted_proxy": {}, // gendoc:generate(entity=server, group=core, key=core.proxy_http) // If this option is not specified, the daemon falls back to the `HTTP_PROXY` environment variable (if set). // --- // type: string // scope: global // shortdesc: HTTP proxy to use "core.proxy_http": {}, // gendoc:generate(entity=server, group=core, key=core.proxy_https) // If this option is not specified, the daemon falls back to the `HTTPS_PROXY` environment variable (if set). // --- // type: string // scope: global // shortdesc: HTTPS proxy to use "core.proxy_https": {}, // gendoc:generate(entity=server, group=core, key=core.proxy_ignore_hosts) // Specify this option in a similar format to `NO_PROXY` (for example, `1.2.3.4,1.2.3.5`) // // If this option is not specified, the daemon falls back to the `NO_PROXY` environment variable (if set). // --- // type: string // scope: global // shortdesc: Hosts that don't need the proxy "core.proxy_ignore_hosts": {}, // gendoc:generate(entity=server, group=core, key=core.remote_token_expiry) // // --- // type: string // scope: global // defaultdesc: no expiry // shortdesc: Time after which a remote add token expires "core.remote_token_expiry": {Type: config.String, Validator: validate.Optional(expiryValidator)}, // gendoc:generate(entity=server, group=core, key=core.shutdown_action) // Specify the action to take when the daemon is being shut down. // Supported values are `shutdown` (stop all instances) and `evacuate` (attempt to evacuate the clustered server). // --- // type: string // scope: global // defaultdesc: `shutdown` // shortdesc: Action to perform on server shutdown "core.shutdown_action": {Type: config.String, Default: "shutdown", Validator: validate.IsOneOf("shutdown", "evacuate")}, // gendoc:generate(entity=server, group=core, key=core.shutdown_timeout) // Specify the number of minutes to wait for running operations to complete before the daemon shuts down. // --- // type: integer // scope: global // defaultdesc: `5` // shortdesc: How long to wait before shutdown "core.shutdown_timeout": {Type: config.Int64, Default: "5"}, // gendoc:generate(entity=server, group=core, key=core.trust_ca_certificates) // // --- // type: bool // scope: global // defaultdesc: `false` // shortdesc: Whether to automatically trust clients signed by the CA "core.trust_ca_certificates": {Type: config.Bool, Default: "false"}, // gendoc:generate(entity=server, group=images, key=images.auto_update_cached) // // --- // type: bool // scope: global // defaultdesc: `true` // shortdesc: Whether to automatically update cached images "images.auto_update_cached": {Type: config.Bool, Default: "true"}, // gendoc:generate(entity=server, group=images, key=images.auto_update_interval) // Specify the interval in hours. // To disable looking for updates to cached images, set this option to `0`. // --- // type: integer // scope: global // defaultdesc: `6` // shortdesc: Interval at which to look for updates to cached images "images.auto_update_interval": {Type: config.Int64, Default: "6"}, // gendoc:generate(entity=server, group=images, key=images.compression_algorithm) // Possible values are `bzip2`, `gzip`, `lz4`, `lzma`, `xz`, `zstd` or `none`. // --- // type: string // scope: global // defaultdesc: `gzip` // shortdesc: Compression algorithm to use for new images "images.compression_algorithm": {Default: "gzip", Validator: validate.IsCompressionAlgorithm}, // gendoc:generate(entity=server, group=images, key=images.default_architecture) // // --- // type: string // shortdesc: Default architecture to use in a mixed-architecture cluster "images.default_architecture": {Validator: validate.Optional(validate.IsArchitecture)}, // gendoc:generate(entity=server, group=images, key=images.remote_cache_expiry) // Specify the number of days after which the unused cached image expires. // --- // type: integer // scope: global // defaultdesc: `10` // shortdesc: When an unused cached remote image is flushed "images.remote_cache_expiry": {Type: config.Int64, Default: "10"}, // gendoc:generate(entity=server, group=miscellaneous, key=instances.lxcfs.per_instance) // LXCFS is used to provide overlays for common `/proc` and `/sys` // files which reflect the resource limits applied to the container. // // It normally operates through a single file system mount on the host which is then shared by all containers. // This is very efficient but comes with the downside that a crash of LXCFS will break all containers. // // With this option, it's now possible to run a LXCFS instance per // container instead, using more system resources but reducing the impact // of a crash. // --- // type: bool // scope: global // defaultdesc: `false` // shortdesc: Whether to run LXCFS on a per-instance basis "instances.lxcfs.per_instance": {Type: config.Bool, Validator: validate.Optional(validate.IsBool)}, // gendoc:generate(entity=server, group=miscellaneous, key=instances.nic.host_name) // Possible values are `random` and `mac`. // // If set to `random`, use the random host interface name as the host name. // If set to `mac`, generate a host name in the form `inc` (MAC without leading two digits). // --- // type: string // scope: global // defaultdesc: `random` // shortdesc: How to set the host name for a NIC "instances.nic.host_name": {Validator: validate.Optional(validate.IsOneOf("random", "mac"))}, // gendoc:generate(entity=server, group=miscellaneous, key=instances.placement.scriptlet) // When using custom automatic instance placement logic, this option stores the scriptlet. // See {ref}`clustering-instance-placement-scriptlet` for more information. // --- // type: string // scope: global // shortdesc: Instance placement scriptlet for automatic instance placement "instances.placement.scriptlet": {Validator: validate.Optional(scriptletLoad.InstancePlacementValidate)}, // gendoc:generate(entity=server, group=loki, key=loki.auth.username) // // --- // type: string // scope: global // shortdesc: User name used for Loki authentication "loki.auth.username": {Deprecated: "Use 'logging.*.target.username' instead"}, // gendoc:generate(entity=server, group=loki, key=loki.auth.password) // // --- // type: string // scope: global // shortdesc: Password used for Loki authentication "loki.auth.password": {Deprecated: "Use 'logging.*.target.password' instead"}, // gendoc:generate(entity=server, group=loki, key=loki.api.ca_cert) // // --- // type: string // scope: global // shortdesc: CA certificate for the Loki server "loki.api.ca_cert": {Deprecated: "Use 'logging.*.target.ca_cert' instead"}, // gendoc:generate(entity=server, group=loki, key=loki.api.url) // Specify the protocol, name or IP and port. For example `https://loki.example.com:3100`. Incus will automatically add the `/loki/api/v1/push` suffix so there's no need to add it here. // --- // type: string // scope: global // shortdesc: URL to the Loki server "loki.api.url": {Deprecated: "Use 'logging.*.target.address' instead"}, // gendoc:generate(entity=server, group=loki, key=loki.instance) // This allows replacing the default instance value (server host name) by a more relevant value like a cluster identifier. // --- // type: string // scope: global // defaultdesc: Local server host name or cluster member name // shortdesc: Name to use as the instance field in Loki events. "loki.instance": {Deprecated: "Use 'logging.*.target.instance' instead"}, // gendoc:generate(entity=server, group=loki, key=loki.labels) // Specify a comma-separated list of values that should be used as labels for a Loki log entry. // --- // type: string // scope: global // shortdesc: Labels for a Loki log entry "loki.labels": {Deprecated: "Use 'logging.*.target.labels' instead"}, // gendoc:generate(entity=server, group=loki, key=loki.loglevel) // // --- // type: string // scope: global // defaultdesc: `info` // shortdesc: Minimum log level to send to the Loki server "loki.loglevel": {Validator: config.LogLevelValidator, Default: logrus.InfoLevel.String(), Deprecated: "Use 'logging.*.logging.level' instead"}, // gendoc:generate(entity=server, group=loki, key=loki.types) // Specify a comma-separated list of events to send to the Loki server. // The events can be any combination of `lifecycle`, `logging`, and `network-acl`. // --- // type: string // scope: global // defaultdesc: `lifecycle,logging` // shortdesc: Events to send to the Loki server "loki.types": {Validator: validate.Optional(validate.IsListOf(validate.IsOneOf("lifecycle", "logging", "network-acl"))), Default: "lifecycle,logging", Deprecated: "Use 'logging.*.types' instead"}, // gendoc:generate(entity=server, group=network, key=network.hwaddr_pattern) // Specify a MAC address template, e.g. `10:66:6a:xx:xx:xx`, to use within the cluster. // Every `x` in the template will be replaced by a random character in `0`–`f`. // Beware of the birthday paradox! A single `xx` block leads to a 10% collision probability with only 8 addresses; for a double `xx:xx` block, 118 addresses; for a triple `xx:xx:xx` block, 1881; for a quadruple `xx:xx:xx:xx` block, 30084. We provide absolutely no guardrail against that. // --- // type: string // scope: global // defaultdesc: `10:66:6a:xx:xx:xx` // shortdesc: MAC address template "network.hwaddr_pattern": {Default: "10:66:6a:xx:xx:xx", Validator: validate.Optional(validate.IsMACPattern)}, // gendoc:generate(entity=server, group=openfga, key=openfga.api.token) // // --- // type: string // scope: global // shortdesc: API token of the OpenFGA server "openfga.api.token": {}, // gendoc:generate(entity=server, group=openfga, key=openfga.api.url) // // --- // type: string // scope: global // shortdesc: URL of the OpenFGA server "openfga.api.url": {}, // gendoc:generate(entity=server, group=openfga, key=openfga.store.id) // // --- // type: string // scope: global // shortdesc: ID of the OpenFGA permission store "openfga.store.id": {}, // gendoc:generate(entity=server, group=oidc, key=oidc.client.id) // // --- // type: string // scope: global // shortdesc: OpenID Connect client ID "oidc.client.id": {}, // gendoc:generate(entity=server, group=oidc, key=oidc.issuer) // // --- // type: string // scope: global // shortdesc: OpenID Connect Discovery URL for the provider "oidc.issuer": {}, // gendoc:generate(entity=server, group=oidc, key=oidc.scopes) // // --- // type: string // scope: global // shortdesc: Comma separated list of OpenID Connect scopes "oidc.scopes": {Default: "openid, offline_access"}, // gendoc:generate(entity=server, group=oidc, key=oidc.audience) // This value is required by some providers. // --- // type: string // scope: global // shortdesc: Expected audience value for the application "oidc.audience": {}, // gendoc:generate(entity=server, group=oidc, key=oidc.claim) // Note that the claim must be contained in the access token. // --- // type: string // scope: global // shortdesc: OpenID Connect claim to use as the username "oidc.claim": {}, // OVN networking global keys. // gendoc:generate(entity=server, group=miscellaneous, key=network.ovn.integration_bridge) // // --- // type: string // scope: global // defaultdesc: `br-int` // shortdesc: OVS integration bridge to use for OVN networks "network.ovn.integration_bridge": {Default: "br-int"}, // gendoc:generate(entity=server, group=miscellaneous, key=network.ovn.northbound_connection) // // --- // type: string // scope: global // defaultdesc: `unix:/run/ovn/ovnnb_db.sock` // shortdesc: OVN northbound database connection string "network.ovn.northbound_connection": {Default: "unix:/run/ovn/ovnnb_db.sock"}, // gendoc:generate(entity=server, group=miscellaneous, key=network.ovn.ca_cert) // // --- // type: string // scope: global // defaultdesc: Content of `/etc/ovn/ovn-central.crt` if present // shortdesc: OVN SSL certificate authority "network.ovn.ca_cert": {Default: ""}, // gendoc:generate(entity=server, group=miscellaneous, key=network.ovn.client_cert) // // --- // type: string // scope: global // defaultdesc: Content of `/etc/ovn/cert_host` if present // shortdesc: OVN SSL client certificate "network.ovn.client_cert": {Default: ""}, // gendoc:generate(entity=server, group=miscellaneous, key=network.ovn.client_key) // // --- // type: string // scope: global // defaultdesc: Content of `/etc/ovn/key_host` if present // shortdesc: OVN SSL client key "network.ovn.client_key": {Default: ""}, // gendoc:generate(entity=server, group=miscellaneous, key=storage.linstor.controller_connection) // // --- // type: string // scope: global // shortdesc: LINSTOR controller connection string "storage.linstor.controller_connection": {Default: ""}, // gendoc:generate(entity=server, group=miscellaneous, key=storage.linstor.ca_cert) // // --- // type: string // scope: global // shortdesc: LINSTOR SSL certificate authority "storage.linstor.ca_cert": {Default: ""}, // gendoc:generate(entity=server, group=miscellaneous, key=storage.linstor.client_cert) // // --- // type: string // scope: global // shortdesc: LINSTOR SSL client certificate "storage.linstor.client_cert": {Default: ""}, // gendoc:generate(entity=server, group=miscellaneous, key=storage.linstor.client_key) // // --- // type: string // scope: global // shortdesc: LINSTOR SSL client key "storage.linstor.client_key": {Default: ""}, } func expiryValidator(value string) error { _, err := internalInstance.GetExpiry(time.Time{}, value) if err != nil { return err } return nil } func offlineThresholdDefault() string { return strconv.Itoa(db.DefaultOfflineThreshold) } func offlineThresholdValidator(value string) error { minThreshold := 10 // Ensure that the given value is greater than the heartbeat interval, // which is the lower bound granularity of the offline check. threshold, err := strconv.Atoi(value) if err != nil { return errors.New("Offline threshold is not a number") } if threshold <= minThreshold { return fmt.Errorf("Value must be greater than '%d'", minThreshold) } return nil } func imageMinimalReplicaValidator(value string) error { count, err := strconv.Atoi(value) if err != nil { return errors.New("Minimal image replica count is not a number") } if count < 1 && count != -1 { return errors.New("Invalid value for image replica count") } return nil } func maxVotersValidator(value string) error { n, err := strconv.Atoi(value) if err != nil { return errors.New("Value is not a number") } if n < 3 || n%2 != 1 { return errors.New("Value must be an odd number equal to or higher than 3") } return nil } func maxStandByValidator(value string) error { n, err := strconv.Atoi(value) if err != nil { return errors.New("Value is not a number") } if n < 0 || n > 5 { return errors.New("Value must be between 0 and 5") } return nil } func rebalanceThresholdValidator(value string) error { n, err := strconv.Atoi(value) if err != nil { return errors.New("Value is not a number") } if n < 10 || n > 100 { return errors.New("Value must be between 10 and 100") } return nil } incus-7.0.0/internal/server/cluster/config/config_test.go000066400000000000000000000072251517523235500235600ustar00rootroot00000000000000package config_test import ( "context" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" clusterConfig "github.com/lxc/incus/v7/internal/server/cluster/config" "github.com/lxc/incus/v7/internal/server/db" ) // The server configuration is initially empty. func TestConfigLoad_Initial(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() config, err := clusterConfig.Load(context.Background(), tx) require.NoError(t, err) assert.Equal(t, map[string]string{}, config.Dump()) assert.Equal(t, float64(20), config.OfflineThreshold().Seconds()) } // If the database contains invalid keys, they are ignored. func TestConfigLoad_IgnoreInvalidKeys(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() err := tx.UpdateClusterConfig(map[string]string{ "foo": "garbage", "core.proxy_http": "foo.bar", }) require.NoError(t, err) config, err := clusterConfig.Load(context.Background(), tx) require.NoError(t, err) values := map[string]string{"core.proxy_http": "foo.bar"} assert.Equal(t, values, config.Dump()) } // Triggers can be specified to execute custom code on config key changes. func TestConfigLoad_Triggers(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() config, err := clusterConfig.Load(context.Background(), tx) require.NoError(t, err) assert.Equal(t, map[string]string{}, config.Dump()) } // Offline threshold must be greater than the heartbeat interval. func TestConfigLoad_OfflineThresholdValidator(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() config, err := clusterConfig.Load(context.Background(), tx) require.NoError(t, err) _, err = config.Patch(map[string]string{"cluster.offline_threshold": "2"}) require.EqualError(t, err, "cannot set 'cluster.offline_threshold' to '2': Value must be greater than '10'") } // Max number of voters must be odd. func TestConfigLoad_MaxVotersValidator(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() config, err := clusterConfig.Load(context.Background(), tx) require.NoError(t, err) _, err = config.Patch(map[string]string{"cluster.max_voters": "4"}) require.EqualError(t, err, "cannot set 'cluster.max_voters' to '4': Value must be an odd number equal to or higher than 3") } // If some previously set values are missing from the ones passed to Replace(), // they are deleted from the configuration. func TestConfig_ReplaceDeleteValues(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() config, err := clusterConfig.Load(context.Background(), tx) require.NoError(t, err) changed, err := config.Replace(map[string]string{"core.proxy_http": "foo.bar"}) assert.NoError(t, err) assert.Equal(t, map[string]string{"core.proxy_http": "foo.bar"}, changed) _, err = config.Replace(map[string]string{}) assert.NoError(t, err) assert.Equal(t, "", config.ProxyHTTP()) values, err := tx.Config(context.Background()) require.NoError(t, err) assert.Equal(t, map[string]string{}, values) } // If some previously set values are missing from the ones passed to Patch(), // they are kept as they are. func TestConfig_PatchKeepsValues(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() config, err := clusterConfig.Load(context.Background(), tx) require.NoError(t, err) _, err = config.Replace(map[string]string{"core.proxy_http": "foo.bar"}) assert.NoError(t, err) _, err = config.Patch(map[string]string{}) assert.NoError(t, err) assert.Equal(t, "foo.bar", config.ProxyHTTP()) values, err := tx.Config(context.Background()) require.NoError(t, err) assert.Equal(t, map[string]string{"core.proxy_http": "foo.bar"}, values) } incus-7.0.0/internal/server/cluster/connect.go000066400000000000000000000300641517523235500214350ustar00rootroot00000000000000package cluster import ( "context" "errors" "fmt" "net" "net/http" "net/url" "time" incus "github.com/lxc/incus/v7/client" clusterRequest "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" localtls "github.com/lxc/incus/v7/shared/tls" ) // Set references. func init() { storagePools.ConnectIfInstanceIsRemote = ConnectIfInstanceIsRemote } // Connect is a convenience around incus.ConnectIncus that configures the client // with the correct parameters for node-to-node communication. // // If 'notify' switch is true, then the user agent will be set to the special // to the UserAgentNotifier value, which can be used in some cases to distinguish // between a regular client request and an internal cluster request. func Connect(address string, networkCert *localtls.CertInfo, serverCert *localtls.CertInfo, r *http.Request, notify bool) (incus.InstanceServer, error) { // Wait for a connection to the events API first for non-notify connections. if !notify { ctx, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second) defer cancel() err := EventListenerWait(ctx, address) if err != nil { return nil, err } } args := &incus.ConnectionArgs{ IdenticalCertificate: true, TLSServerCert: string(networkCert.PublicKey()), TLSClientCert: string(serverCert.PublicKey()), TLSClientKey: string(serverCert.PrivateKey()), SkipGetServer: true, SkipGetEvents: true, UserAgent: version.UserAgent, } if notify { args.UserAgent = clusterRequest.UserAgentNotifier } // Always set a proxy function to have cluster traffic bypass any configured HTTP proxy. proxy := func(req *http.Request) (*url.URL, error) { // If dealing with a user request, proxy through the requestor. if r != nil { ctx := r.Context() val, ok := ctx.Value(request.CtxUsername).(string) if ok { req.Header.Add(request.HeaderForwardedUsername, val) } val, ok = ctx.Value(request.CtxProtocol).(string) if ok { req.Header.Add(request.HeaderForwardedProtocol, val) } req.Header.Add(request.HeaderForwardedAddress, r.RemoteAddr) } return nil, nil } args.Proxy = proxy // Connect to the target server. url := fmt.Sprintf("https://%s", address) return incus.ConnectIncus(url, args) } // ConnectIfInstanceIsRemote figures out the address of the cluster member which is running the instance with the // given name in the specified project. If it's not the local member will connect to it and return the connected // client (configured with the specified project), otherwise it will just return nil. func ConnectIfInstanceIsRemote(s *state.State, projectName string, instName string, r *http.Request) (incus.InstanceServer, error) { // No need to connect if not clustered. if !s.ServerClustered { return nil, nil } var address string // Cluster member address. err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error address, err = tx.GetNodeAddressOfInstance(ctx, projectName, instName) return err }) if err != nil { return nil, err } if address == "" { return nil, nil // The instance is running on this local member, no need to connect. } client, err := Connect(address, s.Endpoints.NetworkCert(), s.ServerCert(), r, false) if err != nil { return nil, err } client = client.UseProject(projectName) return client, nil } // ConnectIfVolumeIsRemote figures out the address of the cluster member on which the volume with the given name is // defined. If it's not the local cluster member it will connect to it and return the connected client, otherwise // it just returns nil. If there is more than one cluster member with a matching volume name, an error is returned. func ConnectIfVolumeIsRemote(s *state.State, poolName string, projectName string, volumeName string, volumeType int, networkCert *localtls.CertInfo, serverCert *localtls.CertInfo, r *http.Request) (incus.InstanceServer, error) { localNodeID := s.DB.Cluster.GetNodeID() var err error var nodes []db.NodeInfo var poolID int64 err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { poolID, err = tx.GetStoragePoolID(ctx, poolName) if err != nil { return err } nodes, err = tx.GetStorageVolumeNodes(ctx, poolID, projectName, volumeName, volumeType) if err != nil { return err } return nil }) if err != nil && !errors.Is(err, db.ErrNoClusterMember) { return nil, err } // If volume uses a remote storage driver and so has no explicit cluster member, then we need to check // whether it is exclusively attached to remote instance, and if so then we need to forward the request to // the node whereit is currently used. This avoids conflicting with another member when using it locally. if errors.Is(err, db.ErrNoClusterMember) { // GetStoragePoolVolume returns a volume with an empty Location field for remote drivers. var dbVolume *db.StorageVolume err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, volumeName, true) return err }) if err != nil { return nil, err } // Find if volume is attached to a remote instance. var remoteInstance *db.InstanceArgs err = storagePools.VolumeUsedByInstanceDevices(s, poolName, projectName, &dbVolume.StorageVolume, true, func(dbInst db.InstanceArgs, project api.Project, usedByDevices []string) error { if dbInst.Node == s.ServerName { remoteInstance = nil return db.ErrInstanceListStop // Stop the search if the volume is attached to the local system. } remoteInstance = &dbInst return nil }) if err != nil && !errors.Is(err, db.ErrInstanceListStop) { return nil, err } if remoteInstance != nil { var instNode db.NodeInfo err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { instNode, err = tx.GetNodeByName(ctx, remoteInstance.Node) return err }) if err != nil { return nil, fmt.Errorf("Failed getting cluster member info for %q: %w", remoteInstance.Node, err) } // Replace node list with instance's cluster member node (which might be local member). nodes = []db.NodeInfo{instNode} } else { // Volume isn't exclusively attached to an instance. Use local cluster member. return nil, nil } } nodeCount := len(nodes) if nodeCount > 1 { return nil, fmt.Errorf("More than one cluster member has a volume named %q. Please target a specific member", volumeName) } else if nodeCount < 1 { // Should never get here. return nil, fmt.Errorf("Volume %q has empty cluster member list", volumeName) } node := nodes[0] if node.ID == localNodeID { // Use local cluster member if volume belongs to this local member. return nil, nil } // Connect to remote cluster member. client, err := Connect(node.Address, networkCert, serverCert, r, false) if err != nil { return nil, err } client = client.UseProject(projectName) return client, nil } // SetupTrust is a convenience around InstanceServer.CreateCertificate that adds the given server certificate to // the trusted pool of the cluster at the given address, using the given token. The certificate is added as // type CertificateTypeServer to allow intra-member communication. If a certificate with the same fingerprint // already exists with a different name or type, then no error is returned. func SetupTrust(serverCert *localtls.CertInfo, serverName string, targetAddress string, targetCert string, targetToken string) error { // Connect to the target cluster node. args := &incus.ConnectionArgs{ TLSServerCert: targetCert, UserAgent: version.UserAgent, SkipGetEvents: true, SkipGetServer: true, } // Always set a proxy function to have cluster traffic bypass any configured HTTP proxy. proxy := func(req *http.Request) (*url.URL, error) { return nil, nil } args.Proxy = proxy target, err := incus.ConnectIncus(fmt.Sprintf("https://%s", targetAddress), args) if err != nil { return fmt.Errorf("Failed to connect to target cluster node %q: %w", targetAddress, err) } cert, err := localtls.GenerateTrustCertificate(serverCert, serverName) if err != nil { return fmt.Errorf("Failed generating trust certificate: %w", err) } post := api.CertificatesPost{ CertificatePut: cert.CertificatePut, TrustToken: targetToken, } err = target.CreateCertificate(post) if err != nil && !api.StatusErrorCheck(err, http.StatusConflict) { return fmt.Errorf("Failed to add server cert to cluster: %w", err) } return nil } // UpdateTrust ensures that the supplied certificate is stored in the target trust store with the correct name // and type to ensure correct cluster operation. Should be called after SetupTrust. If a certificate with the same // fingerprint is already in the trust store, but is of the wrong type or name then the existing certificate is // updated to the correct type and name. If the existing certificate is the correct type but the wrong name then an // error is returned. And if the existing certificate is the correct type and name then nothing more is done. func UpdateTrust(serverCert *localtls.CertInfo, serverName string, targetAddress string, targetCert string) error { // Connect to the target cluster node. args := &incus.ConnectionArgs{ TLSClientCert: string(serverCert.PublicKey()), TLSClientKey: string(serverCert.PrivateKey()), TLSServerCert: targetCert, UserAgent: version.UserAgent, SkipGetEvents: true, SkipGetServer: true, } // Always set a proxy function to have cluster traffic bypass any configured HTTP proxy. proxy := func(req *http.Request) (*url.URL, error) { return nil, nil } args.Proxy = proxy target, err := incus.ConnectIncus(fmt.Sprintf("https://%s", targetAddress), args) if err != nil { return fmt.Errorf("Failed to connect to target cluster node %q: %w", targetAddress, err) } cert, err := localtls.GenerateTrustCertificate(serverCert, serverName) if err != nil { return fmt.Errorf("Failed generating trust certificate: %w", err) } existingCert, _, err := target.GetCertificate(cert.Fingerprint) if err != nil { return fmt.Errorf("Failed getting existing certificate: %w", err) } if existingCert.Name != serverName && existingCert.Type == api.CertificateTypeServer { // Don't alter an existing server certificate that has our fingerprint but not our name. // Something is wrong as this shouldn't happen. return fmt.Errorf("Existing server certificate with different name %q already in trust store", existingCert.Name) } else if existingCert.Name != serverName && existingCert.Type != api.CertificateTypeServer { // Ensure that if a client certificate already exists that matches our fingerprint, that it // has the correct name and type for cluster operation, to allow us to associate member // server names to certificate names. err = target.UpdateCertificate(cert.Fingerprint, cert.CertificatePut, "") if err != nil { return fmt.Errorf("Failed updating certificate name and type in trust store: %w", err) } } return nil } // HasConnectivity probes the member with the given address for connectivity. func HasConnectivity(networkCert *localtls.CertInfo, serverCert *localtls.CertInfo, address string, apiCheck bool) bool { // Check if the main server endpoint is functional. if apiCheck { c, err := Connect(address, networkCert, serverCert, nil, true) if err != nil { return false } _, _, err = c.GetServer() return err == nil } // Get the transport. transport, cleanup, err := tlsTransport(networkCert, serverCert) if err != nil { return false } defer cleanup() ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() var conn net.Conn conn, err = transport.DialTLSContext(ctx, "tcp", address) if err == nil { _ = conn.Close() return true } return false } incus-7.0.0/internal/server/cluster/events.go000066400000000000000000000333771517523235500213220ustar00rootroot00000000000000package cluster import ( "context" "errors" "slices" "sync" "time" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/events" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" localtls "github.com/lxc/incus/v7/shared/tls" ) // eventHubMinHosts is the minimum number of members that must have the event-hub role to trigger switching into // event-hub mode (where cluster members will only connect to event-hub members rather than all members when // operating in the normal full-mesh mode). const eventHubMinHosts = 2 // EventMode indicates the event distribution mode. type EventMode string // EventModeFullMesh is when every cluster member connects to every other cluster member to pull events. const EventModeFullMesh EventMode = "full-mesh" // EventModeHubServer is when the cluster is operating in event-hub mode and this server is designated as a hub // server, meaning that it will only connect to the other event-hub members and not other members. const EventModeHubServer EventMode = "hub-server" // EventModeHubClient is when the cluster is operating in event-hub mode and this member is designated as a hub // client, meaning that it is expected to connect to the event-hub members. const EventModeHubClient EventMode = "hub-client" // eventListenerClient stores both the event listener and its associated client. type eventListenerClient struct { *incus.EventListener client incus.InstanceServer hubPushCancel context.CancelFunc } // Disconnect disconnects both the listener and the client. func (lc *eventListenerClient) Disconnect() { if lc.hubPushCancel != nil { lc.hubPushCancel() } lc.EventListener.Disconnect() lc.client.Disconnect() } // SetEventMode applies the specified eventMode of the local server to the listener. // If the eventMode is EventModeHubClient then a go routine is started that consumes events from eventHubPushCh and // pushes them to the remote server. If the eventMode is anything else then the go routine is stopped if running. func (lc *eventListenerClient) SetEventMode(eventMode EventMode, eventHubPushCh chan api.Event) { if eventMode == EventModeHubClient { if lc.hubPushCancel != nil || !lc.IsActive() { return } ctx, cancel := context.WithCancel(context.Background()) go func() { lc.hubPushCancel = cancel info, _ := lc.client.GetConnectionInfo() logger.Info("Event hub client started", logger.Ctx{"remote": info.URL}) defer logger.Info("Event hub client stopped", logger.Ctx{"remote": info.URL}) defer func() { cancel() lc.hubPushCancel = nil }() for { select { case event, more := <-eventHubPushCh: if !more { return } err := lc.client.SendEvent(event) if err != nil { // Send failed, something is wrong with this hub server. lc.Disconnect() // Disconnect listener and client. // Try and put event back onto event hub push queue for consumption // by another consumer. ctx, cancel := context.WithTimeout(context.Background(), eventHubPushChTimeout) defer cancel() select { case eventHubPushCh <- event: case <-ctx.Done(): // Don't block if all consumers are slow/down. } return } case <-ctx.Done(): return } } }() } else if lc.hubPushCancel != nil { lc.hubPushCancel() lc.hubPushCancel = nil } } var ( eventMode = EventModeFullMesh eventHubAddresses []string eventHubPushCh = make(chan api.Event, 10) // Buffer size to accommodate slow consumers before dropping events. eventHubPushChTimeout = time.Duration(time.Second) listeners = map[string]*eventListenerClient{} listenersUnavailable = map[string]bool{} listenersNotify = map[chan struct{}][]string{} listenersLock sync.Mutex listenersUpdateLock sync.Mutex ) // ServerEventMode returns the event distribution mode that this local server is operating in. func ServerEventMode() EventMode { listenersLock.Lock() defer listenersLock.Unlock() return eventMode } // RoleInSlice returns whether or not the rule is within the roles list. func RoleInSlice(role db.ClusterRole, roles []db.ClusterRole) bool { return slices.Contains(roles, role) } // EventListenerWait waits for there to be listener connected to the specified address, or one of the event hubs // if operating in event hub mode. func EventListenerWait(ctx context.Context, address string) error { // Check if there is already a listener. listenersLock.Lock() listener, found := listeners[address] if found && listener.IsActive() { listenersLock.Unlock() return nil } if listenersUnavailable[address] { listenersLock.Unlock() return errors.New("Server isn't ready yet") } listenAddresses := []string{address} // Check if operating in event hub mode and if one of the event hub connections is available. // If so then we are ready to receive events from all members. if eventMode != EventModeFullMesh { for _, eventHubAddress := range eventHubAddresses { listener, found := listeners[eventHubAddress] if found && listener.IsActive() { listenersLock.Unlock() return nil } listenAddresses = append(listenAddresses, eventHubAddress) } } // If not setup a notification for when the desired address or any of the event hubs connect. connected := make(chan struct{}) listenersNotify[connected] = listenAddresses listenersLock.Unlock() defer func() { listenersLock.Lock() delete(listenersNotify, connected) listenersLock.Unlock() }() // Wait for the connected channel to be closed (indicating a new listener has been connected), and return. select { case <-connected: return nil case <-ctx.Done(): if ctx.Err() != nil { return errors.New("Missing event connection with target cluster member") } return nil } } // hubAddresses returns the addresses of members with event-hub role, and the event mode of the server. // The event mode will only be hub-server or hub-client if at least eventHubMinHosts have an event-hub role. // Otherwise the mode will be full-mesh. func hubAddresses(localAddress string, members map[int64]APIHeartbeatMember) ([]string, EventMode) { var hubAddresses []string var localHasHubRole bool // Do a first pass of members to count the members with event-hub role, and whether we are a hub server. for _, member := range members { if RoleInSlice(db.ClusterRoleEventHub, member.Roles) { hubAddresses = append(hubAddresses, member.Address) if member.Address == localAddress { localHasHubRole = true } } } eventMode := EventModeFullMesh if len(hubAddresses) >= eventHubMinHosts { if localHasHubRole { eventMode = EventModeHubServer } else { eventMode = EventModeHubClient } } return hubAddresses, eventMode } // EventsUpdateListeners refreshes the cluster event listener connections. func EventsUpdateListeners(s *state.State, hbMembers map[int64]APIHeartbeatMember, inject events.InjectFunc) { listenersUpdateLock.Lock() defer listenersUpdateLock.Unlock() // If no heartbeat members provided, populate from global database. if hbMembers == nil { var err error var members []db.NodeInfo var offlineThreshold time.Duration err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { members, err = tx.GetNodes(ctx) if err != nil { return err } offlineThreshold, err = tx.GetNodeOfflineThreshold(ctx) if err != nil { return err } return nil }) if err != nil { logger.Warn("Failed to get current cluster members", logger.Ctx{"err": err}) return } hbMembers = make(map[int64]APIHeartbeatMember, len(members)) for _, member := range members { hbMembers[member.ID] = APIHeartbeatMember{ ID: member.ID, Name: member.Name, Address: member.Address, LastHeartbeat: member.Heartbeat, Online: !member.IsOffline(offlineThreshold), Roles: member.Roles, } } } localAddress := s.Endpoints.NetworkAddress() hubAddresses, localEventMode := hubAddresses(localAddress, hbMembers) keepListeners := make(map[string]struct{}) wg := sync.WaitGroup{} for _, hbMember := range hbMembers { // Don't bother trying to connect to ourselves or offline members. if hbMember.Name == s.ServerName || !hbMember.Online { continue } if localEventMode != EventModeFullMesh && !RoleInSlice(db.ClusterRoleEventHub, hbMember.Roles) { continue // Skip non-event-hub members if we are operating in event-hub mode. } listenersLock.Lock() listener, ok := listeners[hbMember.Address] // If the member already has a listener associated to it, check that the listener is still active. // If it is, just move on to next member, but if not then we'll try to connect again. if ok { if listener.IsActive() { keepListeners[hbMember.Address] = struct{}{} // Add to current listeners list. listener.SetEventMode(localEventMode, eventHubPushCh) listenersLock.Unlock() continue } // Disconnect and delete listener, but don't delete any listenersNotify entry as there // might be something waiting for a future connection. listener.Disconnect() delete(listeners, hbMember.Address) listenersLock.Unlock() // Log after releasing listenersLock to avoid deadlock on listenersLock with EventHubPush. logger.Debug("Removed inactive member event listener client", logger.Ctx{"address": hbMember.Address}) } else { listenersLock.Unlock() } keepListeners[hbMember.Address] = struct{}{} // Add to current listeners list. // Connect to remote concurrently and add to active listeners if successful. wg.Add(1) go func(m APIHeartbeatMember) { defer wg.Done() l := logger.AddContext(logger.Ctx{"local": localAddress, "remote": m.Address}) if !HasConnectivity(s.Endpoints.NetworkCert(), s.ServerCert(), m.Address, true) { listenersLock.Lock() listenersUnavailable[m.Address] = true listenersLock.Unlock() return } listener, err := eventsConnect(m.Address, s.Endpoints.NetworkCert(), s.ServerCert()) if err != nil { l.Warn("Failed adding member event listener client", logger.Ctx{"err": err}) return } _, _ = listener.AddHandler(nil, func(event api.Event) { // Inject event received via pull as forwarded so that its not forwarded again // onto other members. inject(event, events.EventSourcePull) }) listener.SetEventMode(localEventMode, eventHubPushCh) listenersLock.Lock() listeners[m.Address] = listener listenersUnavailable[m.Address] = false // Indicate to any notifiers waiting for this member's address that it is connected. for connected, notifyAddresses := range listenersNotify { if slices.Contains(notifyAddresses, m.Address) { close(connected) delete(listenersNotify, connected) } } listenersLock.Unlock() // Log after releasing listenersLock to avoid deadlock on listenersLock with EventHubPush. l.Debug("Added member event listener client") }(hbMember) } wg.Wait() // Disconnect and delete any out of date listeners and their notifiers. var removedAddresses []string listenersLock.Lock() for address, listener := range listeners { _, found := keepListeners[address] if !found { listener.Disconnect() delete(listeners, address) // Record address removed, but don't log it here as this could cause a deadlock on // listenersLock with EventHubPush removedAddresses = append(removedAddresses, address) } } // Store event hub addresses in global slice late in the function after all event connections have been // opened above. This way the reported state by this server won't be updated until its ready. eventHubAddresses = hubAddresses eventMode = localEventMode listenersLock.Unlock() // Log the listeners removed after releasing listenersLock. for _, removedAddress := range removedAddresses { logger.Debug("Removed old member event listener client", logger.Ctx{"address": removedAddress}) } if len(hbMembers) > 1 && len(keepListeners) <= 0 { logger.Warn("No active cluster event listener clients") } } // Establish a client connection to get events from the given node. func eventsConnect(address string, networkCert *localtls.CertInfo, serverCert *localtls.CertInfo) (*eventListenerClient, error) { client, err := Connect(address, networkCert, serverCert, nil, true) if err != nil { return nil, err } reverter := revert.New() reverter.Add(func() { client.Disconnect() }) listener, err := client.GetEventsAllProjects() if err != nil { return nil, err } reverter.Success() lc := &eventListenerClient{ EventListener: listener, client: client, } return lc, nil } // EventHubPush pushes the event to the event hub members if local server is an event-hub client. func EventHubPush(event api.Event) { listenersLock.Lock() // If the local server isn't an event-hub client, then we don't need to push messages as the other // members should be connected to us via a pull event listener and so will receive the event that way. // Also if there are no listeners available then there's no point in pushing to the eventHubPushCh as it // will have no consumers reading from it (this allows somewhat graceful handling of the situation where // all event-hub members are down by dropping events rather than slowing down the local system). if eventMode != EventModeHubClient || len(listeners) <= 0 { listenersLock.Unlock() return } listenersLock.Unlock() // Run in a go routine so as not to delay caller of this function as we try and deliver it. go func() { ctx, cancel := context.WithTimeout(context.Background(), eventHubPushChTimeout) defer cancel() select { case eventHubPushCh <- event: case <-ctx.Done(): // Don't block if all consumers are slow/down. } }() } incus-7.0.0/internal/server/cluster/gateway.go000066400000000000000000000777511517523235500214630ustar00rootroot00000000000000package cluster import ( "bufio" "context" "crypto/x509" "encoding/json" "errors" "fmt" "net" "net/http" "net/url" "os" "path/filepath" "strconv" "sync" "time" dqlite "github.com/cowsql/go-cowsql" client "github.com/cowsql/go-cowsql/client" "github.com/lxc/incus/v7/internal/server/certificate" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/tcp" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) // NewGateway creates a new Gateway for managing access to the dqlite cluster. // // When a new gateway is created, the node-level database is queried to check // what kind of role this node plays and if it's exposed over the network. It // will initialize internal data structures accordingly, for example starting a // local dqlite server if this node is a database node. // // After creation, the Daemon is expected to expose whatever http handlers the // HandlerFuncs method returns and to access the dqlite cluster using the // dialer returned by the DialFunc method. func NewGateway(shutdownCtx context.Context, db *db.Node, networkCert *localtls.CertInfo, stateFunc func() *state.State, options ...Option) (*Gateway, error) { ctx, cancel := context.WithCancel(context.Background()) o := newOptions() for _, option := range options { option(o) } gateway := &Gateway{ shutdownCtx: shutdownCtx, db: db, networkCert: networkCert, options: o, ctx: ctx, cancel: cancel, upgradeCh: make(chan struct{}), acceptCh: make(chan net.Conn), store: &dqliteNodeStore{}, state: stateFunc, } err := gateway.init(false) if err != nil { return nil, err } return gateway, nil } // HeartbeatHook represents a function that can be called as the heartbeat hook. type HeartbeatHook func(heartbeatData *APIHeartbeat, isLeader bool, unavailableMembers []string) // HeartbeatHandler represents a function that can be called when a heartbeat request arrives. type HeartbeatHandler func(w http.ResponseWriter, r *http.Request, isLeader bool, hbData *APIHeartbeat) // Gateway mediates access to the dqlite cluster using a gRPC SQL client, and // possibly runs a dqlite replica on this member (if we're configured to do so). type Gateway struct { db *db.Node networkCert *localtls.CertInfo options *options // The raft instance to use for creating the dqlite driver. It's nil if // this member is not supposed to be part of the raft cluster. info *db.RaftNode // The gRPC server exposing the dqlite driver created by this // gateway. It's nil if this member is not supposed to be part of the // raft cluster. server *dqlite.Node acceptCh chan net.Conn stopCh chan struct{} // A dialer that will connect to the dqlite server using a loopback // net.Conn. It's non-nil when clustering is not enabled on this member // and so we don't expose any dqlite or raft network endpoint, // but still we want to use dqlite as backend for the "cluster" // database, to minimize the difference between code paths in // clustering and non-clustering modes. memoryDial client.DialFunc // Used when shutting down the daemon to cancel any ongoing gRPC // dialing attempt. shutdownCtx context.Context ctx context.Context cancel context.CancelFunc // Used to unblock nodes that are waiting for other nodes to upgrade // their version. upgradeCh chan struct{} // Used to track whether we already triggered an upgrade because we // detected a peer with an higher version. upgradeTriggered bool // Used for the heartbeat handler Cluster *db.Cluster HeartbeatNodeHook HeartbeatHook HeartbeatOfflineThreshold time.Duration heartbeatCancel context.CancelFunc heartbeatCancelLock sync.Mutex HeartbeatLock sync.Mutex // NodeStore wrapper. store *dqliteNodeStore lock sync.RWMutex // Abstract unix socket that the local dqlite task is listening to. bindAddress string // State function. state func() *state.State } // Current dqlite protocol version. const dqliteVersion = 1 // Set the dqlite version header. func setDqliteVersionHeader(request *http.Request) { request.Header.Set("X-Dqlite-Version", fmt.Sprintf("%d", dqliteVersion)) } // HandlerFuncs returns the HTTP handlers that should be added to the REST API // endpoint in order to handle database-related requests. // // There are two handlers, one for the /internal/raft endpoint and the other // for /internal/db, which handle respectively raft and gRPC-SQL requests. // // These handlers might return 404, either because this server is a // non-clustered member not available over the network or because it is not a // database node part of the dqlite cluster. func (g *Gateway) HandlerFuncs(heartbeatHandler HeartbeatHandler, trustedCerts func() (map[certificate.Type]map[string]x509.Certificate, error)) map[string]http.HandlerFunc { database := func(w http.ResponseWriter, r *http.Request) { g.lock.RLock() certs, err := trustedCerts() if err != nil { g.lock.RUnlock() http.Error(w, "403 invalid client certificate", http.StatusForbidden) return } if !tlsCheckCert(r, g.networkCert, g.state().ServerCert(), certs) { g.lock.RUnlock() http.Error(w, "403 invalid client certificate", http.StatusForbidden) return } g.lock.RUnlock() // Compare the dqlite version of the connecting client // with our own one. versionHeader := r.Header.Get("X-Dqlite-Version") if versionHeader == "" { // No version header means an old pre dqlite 1.0 client. versionHeader = "0" } version, err := strconv.Atoi(versionHeader) if err != nil { http.Error(w, "400 invalid dqlite version", http.StatusBadRequest) return } if version != dqliteVersion { if version > dqliteVersion { g.lock.Lock() if !g.upgradeTriggered { err = triggerUpdate(g.state()) if err == nil { g.upgradeTriggered = true } } g.lock.Unlock() http.Error(w, "503 unsupported dqlite version", http.StatusServiceUnavailable) } else { http.Error(w, "426 dqlite version too old ", http.StatusUpgradeRequired) } return } // Handle heartbeats (these normally come from leader, but can come from joining nodes too). if r.Method == "PUT" { if g.shutdownCtx.Err() != nil { logger.Warn("Rejecting heartbeat request as shutting down") http.Error(w, "503 Shutting down", http.StatusServiceUnavailable) return } var heartbeatData APIHeartbeat err := json.NewDecoder(r.Body).Decode(&heartbeatData) if err != nil { logger.Error("Failed decoding heartbeat", logger.Ctx{"err": err}) http.Error(w, "400 Failed decoding heartbeat", http.StatusBadRequest) return } g.lock.RLock() isLeader, err := g.isLeader() g.lock.RUnlock() if err != nil { logger.Error("Failed checking if leader", logger.Ctx{"err": err}) http.Error(w, "500 Failed checking if leader", http.StatusInternalServerError) return } if heartbeatHandler == nil { logger.Error("No heartbeat handler", logger.Ctx{"err": err}) return } heartbeatHandler(w, r, isLeader, &heartbeatData) return } // Handle database upgrade notifications. if r.Method == "PATCH" { select { case g.upgradeCh <- struct{}{}: default: } return } // From here on we require that this node is part of the raft // cluster. g.lock.RLock() if g.server == nil || g.memoryDial != nil { g.lock.RUnlock() http.NotFound(w, r) return } g.lock.RUnlock() // NOTE: this is kept for backward compatibility when upgrading // a cluster with version <= 4.2. // // Once all nodes are on >= 4.3 this code is effectively // unused. if r.Method == "HEAD" { g.lock.RLock() defer g.lock.RUnlock() // We can safely know about current leader only if we are a voter. if g.info.Role != db.RaftVoter { http.NotFound(w, r) return } client, err := g.getClient() if err != nil { http.Error(w, "500 failed to get dqlite client", http.StatusInternalServerError) return } defer func() { _ = client.Close() }() ctx, cancel := context.WithTimeout(g.ctx, 3*time.Second) defer cancel() leader, err := client.Leader(ctx) if err != nil { http.Error(w, "500 failed to get leader address", http.StatusInternalServerError) return } if leader == nil || leader.ID != g.info.ID { http.Error(w, "503 not leader", http.StatusServiceUnavailable) return } return } // Handle leader address requests. if r.Method == "GET" { leader, err := g.LeaderAddress() if err != nil { http.Error(w, "500 no elected leader", http.StatusInternalServerError) return } _ = localUtil.WriteJSON(w, map[string]string{"leader": leader}, nil) return } if r.Header.Get("Upgrade") != "dqlite" { http.Error(w, "Missing or invalid upgrade header", http.StatusBadRequest) return } hijacker, ok := w.(http.Hijacker) if !ok { http.Error(w, "Webserver doesn't support hijacking", http.StatusInternalServerError) return } conn, _, err := hijacker.Hijack() if err != nil { http.Error(w, fmt.Errorf("Failed to hijack connection: %w", err).Error(), http.StatusInternalServerError) return } err = response.Upgrade(conn, "dqlite") if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) _ = conn.Close() return } g.acceptCh <- conn } return map[string]http.HandlerFunc{ databaseEndpoint: database, } } // WaitUpgradeNotification waits for a notification from another node that all // nodes in the cluster should now have been upgraded and have matching schema // and API versions. func (g *Gateway) WaitUpgradeNotification() { select { case <-g.upgradeCh: case <-time.After(time.Minute): } } // IsDqliteNode returns true if this gateway is running a dqlite node. func (g *Gateway) IsDqliteNode() bool { g.lock.RLock() defer g.lock.RUnlock() if g.info != nil { if g.server == nil { panic("gateway has node identity but no dqlite server") } return true } if g.server != nil { panic("gateway dqlite server but no node identity") } return true } // DialFunc returns a dial function that can be used to connect to one of the // dqlite nodes. func (g *Gateway) DialFunc() client.DialFunc { return func(ctx context.Context, address string) (net.Conn, error) { g.lock.RLock() defer g.lock.RUnlock() // Memory connection. if g.memoryDial != nil { return g.memoryDial(ctx, address) } conn, err := dqliteNetworkDial(ctx, "dqlite", address, g) if err != nil { return nil, err } // We successfully established a connection with the leader. Maybe the // leader is ourselves, and we were recently elected. In that case // trigger a full heartbeat now: it will be a no-op if we aren't // actually leaders. go g.heartbeat(g.ctx, heartbeatInitial) return conn, nil } } // Dial function for establishing raft connections. func (g *Gateway) raftDial() client.DialFunc { return func(ctx context.Context, address string) (net.Conn, error) { nodeAddress, err := g.nodeAddress(address) if err != nil { return nil, err } conn, err := dqliteNetworkDial(ctx, "raft", nodeAddress, g) if err != nil { return nil, err } listener, err := net.Listen("unix", "") if err != nil { return nil, fmt.Errorf("Failed to create unix listener: %w", err) } goUnix, err := net.Dial("unix", listener.Addr().String()) if err != nil { return nil, fmt.Errorf("Failed to connect to unix listener: %w", err) } cUnix, err := listener.Accept() if err != nil { return nil, fmt.Errorf("Failed to connect to unix listener: %w", err) } _ = listener.Close() go dqliteProxy("raft", g.stopCh, conn, goUnix) return cUnix, nil } } // Context returns a cancellation context to pass to dqlite.NewDriver as // option. // // This context gets cancelled by Gateway.Kill() and at that point any // connection failure won't be retried. func (g *Gateway) Context() context.Context { return g.ctx } // NodeStore returns a dqlite server store that can be used to lookup the // addresses of known database nodes. func (g *Gateway) NodeStore() client.NodeStore { return g.store } // Kill is an API that the daemon calls before it actually shuts down and calls // Shutdown(). It will abort any ongoing or new attempt to establish a SQL gRPC // connection with the dialer (typically for running some pre-shutdown // queries). func (g *Gateway) Kill() { logger.Debug("Cancel ongoing or future gRPC connection attempts") g.cancel() } // TransferLeadership attempts to transfer leadership to another node. func (g *Gateway) TransferLeadership() error { client, err := g.getClient() if err != nil { return err } defer func() { _ = client.Close() }() // Try to find a voter that is also online. servers, err := client.Cluster(context.Background()) if err != nil { return err } var id uint64 for _, server := range servers { if server.ID == g.info.ID || server.Role != db.RaftVoter { continue } address, err := g.nodeAddress(server.Address) if err != nil { return err } if !HasConnectivity(g.networkCert, g.state().ServerCert(), address, false) { continue } id = server.ID break } if id == 0 { return errors.New("No online voter found") } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() return client.Transfer(ctx, id) } // DemoteOfflineNode force demoting an offline node. func (g *Gateway) DemoteOfflineNode(raftID uint64) error { cli, err := g.getClient() if err != nil { return fmt.Errorf("Connect to local dqlite node: %w", err) } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() err = cli.Assign(ctx, raftID, db.RaftSpare) if err != nil { return err } return nil } // Shutdown this gateway, stopping the gRPC server and possibly the raft factory. func (g *Gateway) Shutdown() error { logger.Debugf("Stop database gateway") var err error if g.server != nil { if g.info.Role == db.RaftVoter { g.Sync() } err = g.server.Close() close(g.stopCh) // Unset the memory dial, since Shutdown() is also called for // switching between in-memory and network mode. g.lock.Lock() g.memoryDial = nil g.lock.Unlock() } return err } // Sync dumps the content of the database to disk. This is useful for // inspection purposes, and it's also needed by the activateifneeded command so // it can inspect the database in order to decide whether to activate the // daemon or not. func (g *Gateway) Sync() { g.lock.RLock() defer g.lock.RUnlock() if g.server == nil || g.info.Role != db.RaftVoter { return } client, err := g.getClient() if err != nil { logger.Warnf("Failed to get client: %v", err) return } defer func() { _ = client.Close() }() files, err := client.Dump(context.Background(), "db.bin") if err != nil { // Just log a warning, since this is not fatal. logger.Warnf("Failed get database dump: %v", err) return } dir := filepath.Join(g.db.Dir(), "global") for _, file := range files { path := filepath.Join(dir, file.Name) err := os.WriteFile(path, file.Data, 0o600) if err != nil { logger.Warnf("Failed to dump database file %s: %v", file.Name, err) } } } func (g *Gateway) getClient() (*client.Client, error) { return client.New(context.Background(), g.bindAddress) } // Reset the gateway, shutting it down. // // This is used when disabling clustering on a node. func (g *Gateway) Reset(networkCert *localtls.CertInfo) error { err := g.Shutdown() if err != nil { return err } err = os.RemoveAll(filepath.Join(g.db.Dir(), "global")) if err != nil { return err } err = g.db.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { return tx.ReplaceRaftNodes(nil) }) if err != nil { return err } g.networkCert = networkCert return nil } // ErrNodeIsNotClustered indicates the node is not clustered. var ErrNodeIsNotClustered error = errors.New("Server is not clustered") // LeaderAddress returns the address of the current raft leader. func (g *Gateway) LeaderAddress() (string, error) { g.lock.RLock() defer g.lock.RUnlock() // If we aren't clustered, return an error. if g.memoryDial != nil { return "", ErrNodeIsNotClustered } // If this is a voter node, return the address of the current leader, or wait a bit until one is elected. if g.server != nil && g.info.Role == db.RaftVoter { ctx, cancel := context.WithTimeout(g.ctx, 5*time.Second) defer cancel() for { client, err := g.getClient() if err != nil { return "", fmt.Errorf("Failed to get dqlite client: %w", err) } leader, err := client.Leader(ctx) if err != nil { _ = client.Close() return "", fmt.Errorf("Failed to get leader address: %w", err) } if leader != nil && leader.Address != "" { _ = client.Close() return leader.Address, nil } _ = client.Close() select { case <-ctx.Done(): return "", ctx.Err() case <-time.After(time.Second): continue } } } addresses := []string{} err := g.db.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { nodes, err := tx.GetRaftNodes(ctx) if err != nil { return err } for _, node := range nodes { if node.Role != db.RaftVoter { continue } addresses = append(addresses, node.Address) } return nil }) if err != nil { return "", fmt.Errorf("Failed to fetch raft nodes addresses: %w", err) } if len(addresses) == 0 { // This should never happen because the raft_nodes table should // be never empty for a clustered node, but check it for good // measure. return "", errors.New("No raft node known") } transport, cleanup, err := tlsTransport(g.networkCert, g.state().ServerCert()) if err != nil { return "", err } defer cleanup() for _, address := range addresses { timeout := 2 * time.Second client := &http.Client{ Transport: transport, Timeout: timeout, } url := fmt.Sprintf("https://%s%s", address, databaseEndpoint) request, err := http.NewRequest("GET", url, nil) if err != nil { return "", err } setDqliteVersionHeader(request) // Use 1s later timeout to give HTTP client chance timeout with // more useful info. ctx, cancel := context.WithTimeout(g.ctx, timeout+time.Second) defer cancel() request = request.WithContext(ctx) response, err := client.Do(request) if err != nil { logger.Debugf("Failed to fetch leader address from %s", address) continue } if response.StatusCode != http.StatusOK { logger.Debugf("Request for leader address from %s failed", address) continue } info := map[string]string{} err = json.NewDecoder(response.Body).Decode(&info) if err != nil { logger.Debugf("Failed to parse leader address from %s", address) continue } leader := info["leader"] if leader == "" { logger.Debugf("Raft node %s returned no leader address", address) continue } return leader, nil } return "", errors.New("RAFT cluster is unavailable") } // NetworkUpdateCert sets a new network certificate for the gateway // Use with Endpoints.NetworkUpdateCert() to fully update the API endpoint. func (g *Gateway) NetworkUpdateCert(cert *localtls.CertInfo) { g.lock.Lock() defer g.lock.Unlock() g.networkCert = cert } // Initialize the gateway, creating a new raft factory and gRPC server (if this // node is a database node), and a gRPC dialer. // @bootstrap should only be true when turning a non-clustered server into // the first (and leader) member of a new cluster. func (g *Gateway) init(bootstrap bool) error { logger.Debugf("Initializing database gateway") g.stopCh = make(chan struct{}) info, err := loadInfo(g.db, g.networkCert) if err != nil { return fmt.Errorf("Failed to create raft factory: %w", err) } dir := filepath.Join(g.db.Dir(), "global") if util.PathExists(filepath.Join(dir, "logs.db")) { return errors.New("Unsupported upgrade path, please first upgrade to LXD 4.0") } // If the resulting raft instance is not nil, it means that this node // should serve as database node, so create a dqlite driver possibly // exposing it over the network. if info != nil { // Use the autobind feature of abstract unix sockets to get a // random unused address. listener, err := net.Listen("unix", "") if err != nil { return fmt.Errorf("Failed to autobind unix socket: %w", err) } g.bindAddress = listener.Addr().String() _ = listener.Close() options := []dqlite.Option{ dqlite.WithBindAddress(g.bindAddress), } if info.Address == "1" { if info.ID != 1 { panic("unexpected server ID") } g.memoryDial = dqliteMemoryDial(g.bindAddress) g.store.inMemory = client.NewInmemNodeStore() err = g.store.Set(context.Background(), []client.NodeInfo{info.NodeInfo}) if err != nil { return fmt.Errorf("Failed setting node info in store: %w", err) } } else { go runDqliteProxy(g.stopCh, g.bindAddress, g.acceptCh) g.store.inMemory = nil options = append(options, dqlite.WithDialFunc(g.raftDial())) } server, err := dqlite.New( info.ID, info.Address, dir, options..., ) if err != nil { return fmt.Errorf("Failed to create dqlite server: %w", err) } // Force the correct configuration into the bootstrap node, this is needed // when the raft node already has log entries, in which case a regular // bootstrap fails, resulting in the node containing outdated configuration. if bootstrap { logger.Debugf("Bootstrap database gateway ID:%v Address:%v", info.ID, info.Address) cluster := []dqlite.NodeInfo{ {ID: uint64(info.ID), Address: info.Address}, } err = server.Recover(cluster) if err != nil { return fmt.Errorf("Failed to recover database state: %w", err) } } err = server.Start() if err != nil { return fmt.Errorf("Failed to start dqlite server: %w", err) } g.lock.Lock() g.server = server g.info = info g.lock.Unlock() } else { g.lock.Lock() g.server = nil g.info = nil g.store.inMemory = nil g.lock.Unlock() } g.lock.Lock() g.store.onDisk = client.NewNodeStore( g.db.DB(), "main", "raft_nodes", "address") g.lock.Unlock() return nil } // WaitLeadership waits for the raft node to become leader. func (g *Gateway) WaitLeadership() error { n := 80 sleep := 250 * time.Millisecond for range n { g.lock.RLock() isLeader, err := g.isLeader() if err != nil { g.lock.RUnlock() return err } if isLeader { g.lock.RUnlock() return nil } g.lock.RUnlock() time.Sleep(sleep) } return fmt.Errorf("RAFT node did not self-elect within %s", time.Duration(n)*sleep) } func (g *Gateway) isLeader() (bool, error) { if g.server == nil || g.info.Role != db.RaftVoter { return false, nil } client, err := g.getClient() if err != nil { return false, fmt.Errorf("Failed to get dqlite client: %w", err) } defer func() { _ = client.Close() }() ctx, cancel := context.WithTimeout(g.ctx, 3*time.Second) defer cancel() leader, err := client.Leader(ctx) if err != nil { return false, fmt.Errorf("Failed to get leader address: %w", err) } return leader != nil && leader.ID == g.info.ID, nil } // ErrNotLeader signals that a node not the leader. var ErrNotLeader = errors.New("Not leader") // Return information about the cluster members that a currently part of the raft // cluster, as configured in the raft log. It returns an error if this node is // not the leader. func (g *Gateway) currentRaftNodes() ([]db.RaftNode, error) { g.lock.RLock() defer g.lock.RUnlock() if g.info == nil || g.info.Role != db.RaftVoter { return nil, ErrNotLeader } isLeader, err := g.isLeader() if err != nil { return nil, err } if !isLeader { return nil, ErrNotLeader } client, err := g.getClient() if err != nil { return nil, err } defer func() { _ = client.Close() }() servers, err := client.Cluster(context.Background()) if err != nil { return nil, err } raftNodes := make([]db.RaftNode, 0, len(servers)) for i, server := range servers { address, err := g.nodeAddress(server.Address) if err != nil { return nil, fmt.Errorf("Failed to fetch raft server address: %w", err) } servers[i].Address = address raftNode := db.RaftNode{NodeInfo: servers[i]} raftNodes = append(raftNodes, raftNode) } // Get the names of the raft nodes from the global database. if g.Cluster != nil { err = g.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { members, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } membersByAddress := make(map[string]db.NodeInfo, len(members)) for _, member := range members { membersByAddress[member.Address] = member } for i, server := range servers { member, found := membersByAddress[server.Address] if !found { logger.Warn("Cluster member info not found", logger.Ctx{"address": server.Address}) } raftNodes[i].Name = member.Name } return nil }) if err != nil { logger.Warn("Failed getting raft nodes", logger.Ctx{"err": err}) } } return raftNodes, nil } // Translate a raft address to a node address. They are always the same except // for the bootstrap node, which has address "1". func (g *Gateway) nodeAddress(raftAddress string) (string, error) { if raftAddress != "1" && raftAddress != "0" { return raftAddress, nil } var address string err := g.db.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { var err error address, err = tx.GetRaftNodeAddress(ctx, 1) if err != nil { if !response.IsNotFoundError(err) { return fmt.Errorf("Failed to fetch raft server address: %w", err) } // Use the initial address as fallback. This is an edge // case that happens when listing members on a // non-clustered node. address = raftAddress } return nil }) if err != nil { return "", err } return address, nil } func dqliteNetworkDial(ctx context.Context, name string, addr string, g *Gateway) (net.Conn, error) { transport, cleanup, err := tlsTransport(g.networkCert, g.state().ServerCert()) if err != nil { return nil, err } defer cleanup() path := fmt.Sprintf("https://%s%s", addr, databaseEndpoint) // Establish the connection request := &http.Request{ Method: "POST", Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, Header: make(http.Header), Host: addr, } request.URL, err = url.Parse(path) if err != nil { return nil, err } request.Header.Set("Upgrade", "dqlite") setDqliteVersionHeader(request) request = request.WithContext(ctx) reverter := revert.New() defer reverter.Fail() conn, err := transport.DialTLSContext(ctx, "tcp", addr) if err != nil { return nil, err } reverter.Add(func() { _ = conn.Close() }) l := logger.AddContext(logger.Ctx{"name": name, "local": conn.LocalAddr(), "remote": conn.RemoteAddr()}) l.Debug("Dqlite connected outbound") remoteTCP, err := tcp.ExtractConn(conn) if err != nil { l.Warn("Failed extracting TCP connection from remote connection", logger.Ctx{"err": err}) } else { err := tcp.SetTimeouts(remoteTCP, time.Second*30) if err != nil { l.Warn("Failed setting TCP timeouts on remote connection", logger.Ctx{"err": err}) } } err = request.Write(conn) if err != nil { return nil, fmt.Errorf("Failed sending HTTP request to %q: %w", request.URL, err) } response, err := http.ReadResponse(bufio.NewReader(conn), request) if err != nil { return nil, fmt.Errorf("Failed to read response: %w", err) } // If the remote server has detected that we are out of date, let's // trigger an upgrade. if response.StatusCode == http.StatusUpgradeRequired { g.lock.Lock() defer g.lock.Unlock() if !g.upgradeTriggered { err = triggerUpdate(g.state()) if err == nil { g.upgradeTriggered = true } } return nil, errors.New("Upgrade needed") } if response.StatusCode != http.StatusSwitchingProtocols { return nil, fmt.Errorf("Dialing failed: expected status code 101 got %d", response.StatusCode) } if response.Header.Get("Upgrade") != "dqlite" { return nil, errors.New("Missing or unexpected Upgrade header in response") } reverter.Success() return conn, nil } // Create a dial function that connects to the local dqlite. func dqliteMemoryDial(bindAddress string) client.DialFunc { return func(ctx context.Context, address string) (net.Conn, error) { return net.Dial("unix", bindAddress) } } // The API endpoint path that gets routed to a dqlite server handler for // performing SQL queries against the dqlite server running on this node. const databaseEndpoint = "/internal/database" // DqliteLog redirects dqlite's logs to our own logger. func DqliteLog(l client.LogLevel, format string, a ...any) { format = fmt.Sprintf("Dqlite: %s", format) switch l { case client.LogDebug: logger.Debugf(format, a...) case client.LogInfo: logger.Debugf(format, a...) case client.LogWarn: logger.Debugf(format, a...) case client.LogError: logger.Errorf(format, a...) } } // Copy incoming TLS streams from upgraded HTTPS connections into Unix sockets // connected to the dqlite task. func runDqliteProxy(stopCh chan struct{}, bindAddress string, acceptCh chan net.Conn) { for { remote := <-acceptCh local, err := net.Dial("unix", bindAddress) if err != nil { continue } go dqliteProxy("dqlite", stopCh, remote, local) } } // Copies data between a remote TLS network connection and a local unix socket. // Accepts name argument that can be used to identify the connection in the logs. func dqliteProxy(name string, stopCh chan struct{}, remote net.Conn, local net.Conn) { l := logger.AddContext(logger.Ctx{"name": name, "local": remote.LocalAddr(), "remote": remote.RemoteAddr()}) l.Debug("Dqlite proxy started") defer l.Debug("Dqlite proxy stopped") remoteTCP, err := tcp.ExtractConn(remote) if err != nil { l.Warn("Failed extracting TCP connection from remote connection", logger.Ctx{"err": err}) } else { err := tcp.SetTimeouts(remoteTCP, time.Second*30) if err != nil { l.Warn("Failed setting TCP timeouts on remote connection", logger.Ctx{"err": err}) } } remoteToLocal := make(chan error) localToRemote := make(chan error) // Start copying data back and forth until either the client or the // server get closed or hit an error. go func() { _, err := util.SafeCopy(local, remote) remoteToLocal <- err }() go func() { _, err := util.SafeCopy(remote, local) localToRemote <- err }() errs := make([]error, 2) select { case <-stopCh: // Force closing, ignore errors. _ = remote.Close() _ = local.Close() <-remoteToLocal <-localToRemote case err := <-remoteToLocal: if err != nil { errs[0] = fmt.Errorf("remote -> local: %w", err) } _ = local.(*net.UnixConn).CloseRead() err = <-localToRemote if err != nil { errs[1] = fmt.Errorf("local -> remote: %w", err) } _ = remote.Close() _ = local.Close() case err := <-localToRemote: if err != nil { errs[0] = fmt.Errorf("local -> remote: %w", err) } _ = remoteTCP.CloseRead() err = <-remoteToLocal if err != nil { errs[1] = fmt.Errorf("remote -> local: %w", err) } _ = local.Close() _ = remote.Close() } if errs[0] != nil || errs[1] != nil { err := dqliteProxyError{first: errs[0], second: errs[1]} l.Debug("Dqlite proxy failed", logger.Ctx{"err": err}) } } type dqliteProxyError struct { first error second error } func (e dqliteProxyError) Error() string { msg := "" if e.first != nil { msg += "first: " + e.first.Error() } if e.second != nil { if e.first != nil { msg += " " } msg += "second: " + e.second.Error() } return msg } // Conditionally uses the in-memory or the on-disk server store. type dqliteNodeStore struct { inMemory client.NodeStore onDisk client.NodeStore } func (s *dqliteNodeStore) Get(ctx context.Context) ([]client.NodeInfo, error) { if s.inMemory != nil { return s.inMemory.Get(ctx) } return s.onDisk.Get(ctx) } func (s *dqliteNodeStore) Set(ctx context.Context, servers []client.NodeInfo) error { if s.inMemory != nil { return s.inMemory.Set(ctx, servers) } return s.onDisk.Set(ctx, servers) } incus-7.0.0/internal/server/cluster/gateway_export_test.go000066400000000000000000000013201517523235500240760ustar00rootroot00000000000000package cluster import ( "github.com/lxc/incus/v7/internal/server/db" localtls "github.com/lxc/incus/v7/shared/tls" ) // IsLeader returns true if this node is the leader. func (g *Gateway) IsLeader() (bool, error) { return g.isLeader() } // ServerCert returns the gateway's internal TLS server certificate information. func (g *Gateway) ServerCert() *localtls.CertInfo { return g.networkCert } // NetworkCert returns the gateway's internal TLS NetworkCert certificate information. func (g *Gateway) NetworkCert() *localtls.CertInfo { return g.networkCert } // RaftNodes returns the nodes currently part of the raft cluster. func (g *Gateway) RaftNodes() ([]db.RaftNode, error) { return g.currentRaftNodes() } incus-7.0.0/internal/server/cluster/gateway_test.go000066400000000000000000000125241517523235500225050ustar00rootroot00000000000000package cluster_test import ( "context" "crypto/tls" "crypto/x509" "fmt" "net/http" "net/http/httptest" "os" "path/filepath" "testing" "github.com/cowsql/go-cowsql/driver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/certificate" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/state" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/tls/tlstest" ) func trustedCerts() (map[certificate.Type]map[string]x509.Certificate, error) { return nil, nil } // Basic creation and shutdown. By default, the gateway runs an in-memory gRPC // server. func TestGateway_Single(t *testing.T) { node, cleanup := db.NewTestNode(t) defer cleanup() cert := tlstest.TestingKeyPair(t) s := &state.State{ ServerCert: func() *localtls.CertInfo { return cert }, } gateway := newGateway(t, node, cert, s) defer func() { _ = gateway.Shutdown() }() handlerFuncs := gateway.HandlerFuncs(nil, trustedCerts) assert.Len(t, handlerFuncs, 1) for endpoint, f := range handlerFuncs { c, err := x509.ParseCertificate(cert.KeyPair().Certificate[0]) require.NoError(t, err) w := httptest.NewRecorder() r := &http.Request{} r.Header = http.Header{} r.Header.Set("X-Dqlite-Version", "1") r.TLS = &tls.ConnectionState{ PeerCertificates: []*x509.Certificate{c}, } f(w, r) assert.Equal(t, 404, w.Code, endpoint) } dial := gateway.DialFunc() netConn, err := dial(context.Background(), "") assert.NoError(t, err) assert.NotNil(t, netConn) require.NoError(t, netConn.Close()) leader, err := gateway.LeaderAddress() assert.Equal(t, "", leader) assert.EqualError(t, err, cluster.ErrNodeIsNotClustered.Error()) driver, err := driver.New( gateway.NodeStore(), driver.WithDialFunc(gateway.DialFunc()), ) require.NoError(t, err) conn, err := driver.Open("test.db") require.NoError(t, err) require.NoError(t, conn.Close()) } // If there's a network address configured, we expose the dqlite endpoint with // an HTTP handler. func TestGateway_SingleWithNetworkAddress(t *testing.T) { node, cleanup := db.NewTestNode(t) defer cleanup() cert := tlstest.TestingKeyPair(t) mux := http.NewServeMux() server := newServer(cert, mux) defer server.Close() address := server.Listener.Addr().String() setRaftRole(t, node, address) s := &state.State{ ServerCert: func() *localtls.CertInfo { return cert }, } gateway := newGateway(t, node, cert, s) defer func() { _ = gateway.Shutdown() }() for path, handler := range gateway.HandlerFuncs(nil, trustedCerts) { mux.HandleFunc(path, handler) } driver, err := driver.New( gateway.NodeStore(), driver.WithDialFunc(gateway.DialFunc()), ) require.NoError(t, err) conn, err := driver.Open("test.db") require.NoError(t, err) require.NoError(t, conn.Close()) leader, err := gateway.LeaderAddress() require.NoError(t, err) assert.Equal(t, address, leader) } // When networked, the grpc and raft endpoints requires the cluster // certificate. func TestGateway_NetworkAuth(t *testing.T) { node, cleanup := db.NewTestNode(t) defer cleanup() cert := tlstest.TestingKeyPair(t) mux := http.NewServeMux() server := newServer(cert, mux) defer server.Close() address := server.Listener.Addr().String() setRaftRole(t, node, address) s := &state.State{ ServerCert: func() *localtls.CertInfo { return cert }, } gateway := newGateway(t, node, cert, s) defer func() { _ = gateway.Shutdown() }() for path, handler := range gateway.HandlerFuncs(nil, trustedCerts) { mux.HandleFunc(path, handler) } // Make a request using a certificate different than the cluster one. certAlt := tlstest.TestingAltKeyPair(t) config, err := cluster.TLSClientConfig(certAlt, certAlt) config.InsecureSkipVerify = true // Skip client-side verification require.NoError(t, err) client := &http.Client{Transport: &http.Transport{TLSClientConfig: config}} for path := range gateway.HandlerFuncs(nil, trustedCerts) { url := fmt.Sprintf("https://%s%s", address, path) response, err := client.Head(url) require.NoError(t, err) assert.Equal(t, http.StatusForbidden, response.StatusCode) } } // RaftNodes returns all nodes of the cluster. func TestGateway_RaftNodesNotLeader(t *testing.T) { node, cleanup := db.NewTestNode(t) defer cleanup() cert := tlstest.TestingKeyPair(t) mux := http.NewServeMux() server := newServer(cert, mux) defer server.Close() address := server.Listener.Addr().String() setRaftRole(t, node, address) s := &state.State{ ServerCert: func() *localtls.CertInfo { return cert }, } gateway := newGateway(t, node, cert, s) defer func() { _ = gateway.Shutdown() }() nodes, err := gateway.RaftNodes() require.NoError(t, err) assert.Len(t, nodes, 1) assert.Equal(t, nodes[0].ID, uint64(1)) assert.Equal(t, nodes[0].Address, address) } // Create a new test Gateway with the given parameters, and ensure no error happens. func newGateway(t *testing.T, node *db.Node, networkCert *localtls.CertInfo, s *state.State) *cluster.Gateway { require.NoError(t, os.Mkdir(filepath.Join(node.Dir(), "global"), 0o755)) stateFunc := func() *state.State { return s } gateway, err := cluster.NewGateway(context.Background(), node, networkCert, stateFunc, cluster.Latency(0.2), cluster.LogLevel("TRACE")) require.NoError(t, err) return gateway } incus-7.0.0/internal/server/cluster/heartbeat.go000066400000000000000000000450441517523235500217470ustar00rootroot00000000000000package cluster import ( "bytes" "context" "encoding/json" "errors" "fmt" "math/rand" "net/http" "sync" "time" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/internal/server/db/warningtype" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/task" "github.com/lxc/incus/v7/internal/server/warnings" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" localtls "github.com/lxc/incus/v7/shared/tls" ) type heartbeatMode int const ( heartbeatNormal heartbeatMode = iota heartbeatImmediate heartbeatInitial ) func (m *heartbeatMode) name() string { switch *m { case heartbeatNormal: return "normal" case heartbeatImmediate: return "immediate" case heartbeatInitial: return "initial" default: return "unknown" } } // APIHeartbeatMember contains specific cluster node info. type APIHeartbeatMember struct { ID int64 // ID field value in nodes table. Address string // Host and Port of node. Name string // Name of cluster member. RaftID uint64 // ID field value in raft_nodes table, zero if non-raft node. RaftRole int // Node role in the raft cluster, from the raft_nodes table LastHeartbeat time.Time // Last time we received a successful response from node. Online bool // Calculated from offline threshold and LastHeatbeat time. Roles []db.ClusterRole // Supplementary non-database roles the member has. updated bool // Has node been updated during this heartbeat run. Not sent to nodes. } // APIHeartbeatVersion contains max versions for all nodes in cluster. type APIHeartbeatVersion struct { Schema int APIExtensions int MinAPIExtensions int } // NewAPIHearbeat returns initialized APIHeartbeat. func NewAPIHearbeat(cluster *db.Cluster) *APIHeartbeat { return &APIHeartbeat{ cluster: cluster, } } // APIHeartbeat contains data sent to nodes in heartbeat. type APIHeartbeat struct { sync.Mutex // Used to control access to Members maps. cluster *db.Cluster Members map[int64]APIHeartbeatMember Version APIHeartbeatVersion Time time.Time // Indicates if heartbeat contains a fresh set of node states. // This can be used to indicate to the receiving node that the state is fresh enough to // trigger node refresh activities. FullStateList bool } // Update updates an existing APIHeartbeat struct with the raft and all node states supplied. // If allNodes provided is an empty set then this is considered a non-full state list. func (hbState *APIHeartbeat) Update(fullStateList bool, raftNodes []db.RaftNode, allNodes []db.NodeInfo, offlineThreshold time.Duration) { var maxSchemaVersion, maxAPIExtensionsVersion, minAPIExtensionsVersion int if hbState.Members == nil { hbState.Members = make(map[int64]APIHeartbeatMember) } // If we've been supplied a fresh set of node states, this is a full state list. hbState.FullStateList = fullStateList // Convert raftNodes to a map keyed on address for lookups later. raftNodeMap := make(map[string]db.RaftNode, len(raftNodes)) for _, raftNode := range raftNodes { raftNodeMap[raftNode.Address] = raftNode } // Add nodes (overwrites any nodes with same ID in map with fresh data). for _, node := range allNodes { member := APIHeartbeatMember{ ID: node.ID, Address: node.Address, Name: node.Name, LastHeartbeat: node.Heartbeat, Online: !node.IsOffline(offlineThreshold), Roles: node.Roles, } raftNode, exists := raftNodeMap[member.Address] if exists { member.RaftID = raftNode.ID member.RaftRole = int(raftNode.Role) delete(raftNodeMap, member.Address) // Used to check any remaining later. } // Add to the members map using the node ID (not the Raft Node ID). hbState.Members[node.ID] = member // Keep a record of highest APIExtensions and Schema version seen in all nodes. if node.APIExtensions > maxAPIExtensionsVersion { maxAPIExtensionsVersion = node.APIExtensions } if minAPIExtensionsVersion == 0 || node.APIExtensions < minAPIExtensionsVersion { minAPIExtensionsVersion = node.APIExtensions } if node.Schema > maxSchemaVersion { maxSchemaVersion = node.Schema } } hbState.Version = APIHeartbeatVersion{ Schema: maxSchemaVersion, APIExtensions: maxAPIExtensionsVersion, MinAPIExtensions: minAPIExtensionsVersion, } if len(raftNodeMap) > 0 && hbState.cluster != nil { _ = hbState.cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { for addr, raftNode := range raftNodeMap { _, err := tx.GetPendingNodeByAddress(ctx, addr) if err != nil { logger.Errorf("Unaccounted raft node(s) not found in 'nodes' table for heartbeat: %+v", raftNode) } } return nil }) } } // Send sends heartbeat requests to the nodes supplied and updates heartbeat state. func (hbState *APIHeartbeat) Send(ctx context.Context, networkCert *localtls.CertInfo, serverCert *localtls.CertInfo, localAddress string, nodes []db.NodeInfo, spreadDuration time.Duration) { heartbeatsWg := sync.WaitGroup{} sendHeartbeat := func(nodeID int64, name string, address string, spreadDuration time.Duration, heartbeatData *APIHeartbeat) { defer heartbeatsWg.Done() if spreadDuration > 0 { // Spread in time by waiting up to 3s less than the interval. spreadDurationMs := int(spreadDuration.Milliseconds()) spreadRange := spreadDurationMs - 3000 if spreadRange > 0 { select { case <-time.After(time.Duration(rand.Intn(spreadRange)) * time.Millisecond): case <-ctx.Done(): // Proceed immediately to heartbeat of member if asked to. } } } // Update timestamp to current, used for time skew detection heartbeatData.Time = time.Now().UTC() // Don't use ctx here, as we still want to finish off the request if the ctx has been cancelled. err := HeartbeatNode(context.Background(), address, networkCert, serverCert, heartbeatData) if err == nil { heartbeatData.Lock() // Ensure only update nodes that exist in Members already. hbNode, existing := hbState.Members[nodeID] if !existing { return } hbNode.LastHeartbeat = time.Now() hbNode.Online = true hbNode.updated = true heartbeatData.Members[nodeID] = hbNode heartbeatData.Unlock() logger.Debug("Successful heartbeat", logger.Ctx{"remote": address}) err = warnings.ResolveWarningsByLocalNodeAndProjectAndTypeAndEntity(hbState.cluster, "", warningtype.OfflineClusterMember, cluster.TypeNode, int(nodeID)) if err != nil { logger.Warn("Failed to resolve warning", logger.Ctx{"err": err}) } } else { logger.Warn("Cluster member isn't responding", logger.Ctx{"name": name}) if ctx.Err() == nil { err = hbState.cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpsertWarningLocalNode(ctx, "", cluster.TypeNode, int(nodeID), warningtype.OfflineClusterMember, err.Error()) }) if err != nil { logger.Warn("Failed to create warning", logger.Ctx{"err": err}) } } } } for _, node := range nodes { // Special case for the local member - just record the time now. if node.Address == localAddress { hbState.Lock() hbNode := hbState.Members[node.ID] hbNode.LastHeartbeat = time.Now() hbNode.Online = true hbNode.updated = true hbState.Members[node.ID] = hbNode hbState.Unlock() continue } // Parallelize the rest. heartbeatsWg.Add(1) go sendHeartbeat(node.ID, node.Name, node.Address, spreadDuration, hbState) } heartbeatsWg.Wait() } // HeartbeatTask returns a task function that performs leader-initiated heartbeat // checks against all cluster members in the cluster. // // It will update the heartbeat timestamp column of the nodes table // accordingly, and also notify them of the current list of database nodes. func HeartbeatTask(gateway *Gateway) (task.Func, task.Schedule) { // Since the database APIs are blocking we need to wrap the core logic // and run it in a goroutine, so we can abort as soon as the context expires. heartbeatWrapper := func(ctx context.Context) { if gateway.HearbeatCancelFunc() == nil { ch := make(chan struct{}) go func() { gateway.heartbeat(ctx, heartbeatNormal) close(ch) }() select { case <-ch: case <-ctx.Done(): } } } schedule := func() (time.Duration, error) { return task.Every(gateway.heartbeatInterval())() } return heartbeatWrapper, schedule } // heartbeatInterval returns heartbeat interval to use. func (g *Gateway) heartbeatInterval() time.Duration { threshold := g.HeartbeatOfflineThreshold if threshold <= 0 { threshold = time.Duration(db.DefaultOfflineThreshold) * time.Second } return threshold / 2 } // HearbeatCancelFunc returns the function that can be used to cancel an ongoing heartbeat. // Returns nil if no ongoing heartbeat. func (g *Gateway) HearbeatCancelFunc() func() { g.heartbeatCancelLock.Lock() defer g.heartbeatCancelLock.Unlock() return g.heartbeatCancel } // HeartbeatRestart restarts cancels any ongoing heartbeat and restarts it. // If there is no ongoing heartbeat then this is a no-op. // Returns true if new heartbeat round was started. func (g *Gateway) HeartbeatRestart() bool { heartbeatCancel := g.HearbeatCancelFunc() // There is a cancellable heartbeat round ongoing. if heartbeatCancel != nil { g.heartbeatCancel() // Request ongoing heartbeat round cancel itself. // Start a new heartbeat round async that will run as soon as ongoing heartbeat round exits. go g.heartbeat(g.ctx, heartbeatImmediate) return true } return false } func (g *Gateway) heartbeat(ctx context.Context, mode heartbeatMode) { if g.Cluster == nil || g.server == nil || g.memoryDial != nil { // We're not a raft node or we're not clustered return } // Avoid concurrent heartbeat loops. // This is possible when both the regular task and the out of band heartbeat round from a dqlite // connection or notification restart both kick in at the same time. g.HeartbeatLock.Lock() defer g.HeartbeatLock.Unlock() // Acquire the cancellation lock and populate it so that this heartbeat round can be cancelled if a // notification cancellation request arrives during the round. Also setup a defer so that the cancellation // function is set to nil when this function ends to indicate there is no ongoing heartbeat round. g.heartbeatCancelLock.Lock() ctx, g.heartbeatCancel = context.WithCancel(ctx) g.heartbeatCancelLock.Unlock() defer func() { heartbeatCancel := g.HearbeatCancelFunc() if heartbeatCancel != nil { g.heartbeatCancel() g.heartbeatCancel = nil } }() raftNodes, err := g.currentRaftNodes() if err != nil { if errors.Is(err, ErrNotLeader) { return } logger.Error("Failed to get current raft members", logger.Ctx{"err": err}) return } // Address of this node. var localClusterAddress string s := g.state() if s.LocalConfig != nil { localClusterAddress = s.LocalConfig.ClusterAddress() } var members []db.NodeInfo err = g.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { members, err = tx.GetNodes(ctx) if err != nil { return err } return nil }) if err != nil { logger.Warn("Failed to get current cluster members", logger.Ctx{"err": err}) return } modeStr := mode.name() if mode != heartbeatNormal { // Log unscheduled heartbeats with a higher level than normal heartbeats. logger.Info("Starting instant heartbeat round", logger.Ctx{"mode": modeStr}) } else { // Don't spam the normal log with regular heartbeat messages. logger.Debug("Starting heartbeat round", logger.Ctx{"mode": modeStr}) } // Replace the local raft_nodes table immediately because it // might miss a row containing ourselves, since we might have // been elected leader before the former leader had chance to // send us a fresh update through the heartbeat pool. logger.Debug("Heartbeat updating local raft members", logger.Ctx{"members": raftNodes}) err = g.db.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { return tx.ReplaceRaftNodes(raftNodes) }) if err != nil { logger.Warn("Failed to replace local raft members", logger.Ctx{"err": err, "mode": modeStr}) return } if localClusterAddress == "" { logger.Error("No local address set, aborting heartbeat round", logger.Ctx{"mode": modeStr}) return } startTime := time.Now() heartbeatInterval := g.heartbeatInterval() // Cumulative set of node states (will be written back to database once done). hbState := NewAPIHearbeat(g.Cluster) // If we are doing a normal heartbeat round then spread the requests over the heartbeatInterval in order // to reduce load on the cluster. spreadDuration := time.Duration(0) if mode == heartbeatNormal { spreadDuration = heartbeatInterval } serverCert := g.state().ServerCert() // If this leader node hasn't sent a heartbeat recently, then its node state records // are likely out of date, this can happen when a node becomes a leader. // Send stale set to all nodes in database to get a fresh set of active nodes. if mode == heartbeatInitial { hbState.Update(false, raftNodes, members, g.HeartbeatOfflineThreshold) hbState.Send(ctx, g.networkCert, serverCert, localClusterAddress, members, spreadDuration) // We have the latest set of node states now, lets send that state set to all nodes. hbState.FullStateList = true hbState.Send(ctx, g.networkCert, serverCert, localClusterAddress, members, spreadDuration) } else { hbState.Update(true, raftNodes, members, g.HeartbeatOfflineThreshold) hbState.Send(ctx, g.networkCert, serverCert, localClusterAddress, members, spreadDuration) } // Check if context has been cancelled. ctxErr := ctx.Err() // Look for any new node which appeared since sending last heartbeat. if ctxErr == nil { var currentMembers []db.NodeInfo err = g.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error currentMembers, err = tx.GetNodes(ctx) if err != nil { return err } return nil }) if err != nil { logger.Warn("Failed to get current cluster members", logger.Ctx{"err": err, "mode": modeStr}) return } newMembers := []db.NodeInfo{} for _, currentMember := range currentMembers { existing := false for _, member := range members { if member.Address == currentMember.Address && member.ID == currentMember.ID { existing = true break } } if !existing { // We found a new node members = append(members, currentMember) newMembers = append(newMembers, currentMember) } } // If any new nodes found, send heartbeat to just them (with full node state). if len(newMembers) > 0 { hbState.Update(true, raftNodes, members, g.HeartbeatOfflineThreshold) hbState.Send(ctx, g.networkCert, serverCert, localClusterAddress, newMembers, 0) } } // Initialize slice to indicate to HeartbeatNodeHook that its being called from leader. unavailableMembers := make([]string, 0) err = query.Retry(ctx, func(ctx context.Context) error { // Durating cluster member fluctuations/upgrades the cluster can become unavailable so check here. if g.Cluster == nil { return errors.New("Cluster unavailable") } return g.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { for _, node := range hbState.Members { if !node.updated { // If member has not been updated during this heartbeat round it means // they are currently unreachable or rejecting heartbeats due to being // in the process of shutting down. Either way we do not want to use this // member as a candidate for role promotion. unavailableMembers = append(unavailableMembers, node.Address) continue } err := tx.SetNodeHeartbeat(node.Address, node.LastHeartbeat) if err != nil && !response.IsNotFoundError(err) { return fmt.Errorf("Failed updating heartbeat time for member %q: %w", node.Address, err) } } return nil }) }) if err != nil { logger.Error("Failed updating cluster heartbeats", logger.Ctx{"err": err}) return } // If the context has been cancelled, return prematurely after saving the members we did manage to ping. if ctxErr != nil { logger.Warn("Aborting heartbeat round", logger.Ctx{"err": ctxErr, "mode": modeStr}) return } // If full node state was sent and node refresh task is specified. if g.HeartbeatNodeHook != nil { g.HeartbeatNodeHook(hbState, true, unavailableMembers) } duration := time.Since(startTime) if duration > heartbeatInterval { logger.Warn("Cluster heartbeat took too long", logger.Ctx{"duration": duration, "interval": heartbeatInterval}) } if mode != heartbeatNormal { // Log unscheduled heartbeats with a higher level than normal heartbeats. logger.Info("Completed instant heartbeat round", logger.Ctx{"duration": duration}) } else { // Don't spam the normal log with regular heartbeat messages. logger.Debug("Completed heartbeat round", logger.Ctx{"duration": duration}) } } // HeartbeatNode performs a single heartbeat request against the node with the given address. func HeartbeatNode(taskCtx context.Context, address string, networkCert *localtls.CertInfo, serverCert *localtls.CertInfo, heartbeatData *APIHeartbeat) error { logger.Debug("Sending heartbeat request", logger.Ctx{"address": address}) timeout := 2 * time.Second url := fmt.Sprintf("https://%s%s", address, databaseEndpoint) transport, cleanup, err := tlsTransport(networkCert, serverCert) if err != nil { return err } defer cleanup() client := &http.Client{ Transport: transport, Timeout: timeout, } buffer := bytes.Buffer{} heartbeatData.Lock() err = json.NewEncoder(&buffer).Encode(heartbeatData) heartbeatData.Unlock() if err != nil { return err } request, err := http.NewRequest("PUT", url, bytes.NewReader(buffer.Bytes())) if err != nil { return err } setDqliteVersionHeader(request) // Use 1s later timeout to give HTTP client chance timeout with more useful info. ctx, cancel := context.WithTimeout(taskCtx, timeout+time.Second) defer cancel() request = request.WithContext(ctx) request.Close = true // Immediately close the connection after the request is done response, err := client.Do(request) if err != nil { return fmt.Errorf("Failed to send heartbeat request: %w", err) } defer func() { _ = response.Body.Close() }() if response.StatusCode != http.StatusOK { return fmt.Errorf("Heartbeat request failed with status: %w", api.StatusErrorf(response.StatusCode, "%s", response.Status)) } return nil } incus-7.0.0/internal/server/cluster/heartbeat_test.go000066400000000000000000000162041517523235500230020ustar00rootroot00000000000000package cluster_test import ( "context" "net/http" "net/http/httptest" "testing" "time" "github.com/cowsql/go-cowsql/driver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/cluster" clusterConfig "github.com/lxc/incus/v7/internal/server/cluster/config" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/osarch" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/tls/tlstest" ) // After a heartbeat request is completed, the leader updates the heartbeat // timestamp column, and the serving node updates its cache of raft nodes. func TestHeartbeat(t *testing.T) { f := heartbeatFixture{t: t} defer f.Cleanup() f.Bootstrap() f.Grow() f.Grow() time.Sleep(1 * time.Second) // Wait for join notification triggered heartbeats to complete. leader := f.Leader() leaderState := f.State(leader) // Artificially mark all nodes as down err := leaderState.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { members, err := tx.GetNodes(ctx) require.NoError(t, err) for _, member := range members { err := tx.SetNodeHeartbeat(member.Address, time.Now().Add(-time.Minute)) require.NoError(t, err) } return nil }) require.NoError(t, err) // Perform the heartbeat requests. leader.Cluster = leaderState.DB.Cluster heartbeat, _ := cluster.HeartbeatTask(leader) ctx := context.Background() heartbeat(ctx) // The heartbeat timestamps of all nodes got updated err = leaderState.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { members, err := tx.GetNodes(ctx) require.NoError(t, err) offlineThreshold, err := tx.GetNodeOfflineThreshold(ctx) require.NoError(t, err) for _, member := range members { assert.False(t, member.IsOffline(offlineThreshold)) } return nil }) require.NoError(t, err) } // Helper for testing heartbeat-related code. type heartbeatFixture struct { t *testing.T gateways map[int]*cluster.Gateway // node index to gateway states map[*cluster.Gateway]*state.State // gateway to its state handle servers map[*cluster.Gateway]*httptest.Server // gateway to its HTTP server cleanups []func() } // Bootstrap the first node of the cluster. func (f *heartbeatFixture) Bootstrap() *cluster.Gateway { f.t.Logf("create bootstrap node for test cluster") state, gateway, _ := f.node() err := cluster.Bootstrap(state, gateway, "buzz") require.NoError(f.t, err) return gateway } // Grow adds a new node to the cluster. func (f *heartbeatFixture) Grow() *cluster.Gateway { // Figure out the current leader f.t.Logf("adding another node to the test cluster") target := f.Leader() targetState := f.states[target] state, gateway, address := f.node() name := address nodes, err := cluster.Accept( targetState, target, name, address, cluster.SchemaVersion, len(version.APIExtensions), osarch.ARCH_64BIT_INTEL_X86) require.NoError(f.t, err) err = cluster.Join(state, gateway, target.NetworkCert(), target.ServerCert(), name, nodes) require.NoError(f.t, err) return gateway } // Return the leader gateway in the cluster. func (f *heartbeatFixture) Leader() *cluster.Gateway { timeout := time.Second ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() for { for _, gateway := range f.gateways { isLeader, err := gateway.IsLeader() if err != nil { f.t.Errorf("failed to check leadership: %v", err) } if isLeader { return gateway } } select { case <-ctx.Done(): f.t.Errorf("no leader was elected within %s", timeout) default: } // Wait a bit for election to take place time.Sleep(10 * time.Millisecond) } } // Return a follower gateway in the cluster. func (f *heartbeatFixture) Follower() *cluster.Gateway { timeout := time.Second ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() for { for _, gateway := range f.gateways { isLeader, err := gateway.IsLeader() if err != nil { f.t.Errorf("failed to check leadership: %v", err) } if !isLeader { return gateway } } select { case <-ctx.Done(): f.t.Errorf("no node running as follower") default: } // Wait a bit for election to take place time.Sleep(10 * time.Millisecond) } } // Return the cluster index of the given gateway. func (f *heartbeatFixture) Index(gateway *cluster.Gateway) int { for i := range f.gateways { if f.gateways[i] == gateway { return i } } return -1 } // Return the state associated with the given gateway. func (f *heartbeatFixture) State(gateway *cluster.Gateway) *state.State { return f.states[gateway] } // Return the HTTP server associated with the given gateway. func (f *heartbeatFixture) Server(gateway *cluster.Gateway) *httptest.Server { return f.servers[gateway] } // Creates a new node, without either bootstrapping or joining it. // // Return the associated gateway and network address. func (f *heartbeatFixture) node() (*state.State, *cluster.Gateway, string) { if f.gateways == nil { f.gateways = make(map[int]*cluster.Gateway) f.states = make(map[*cluster.Gateway]*state.State) f.servers = make(map[*cluster.Gateway]*httptest.Server) } state, cleanup := state.NewTestState(f.t) f.cleanups = append(f.cleanups, cleanup) serverCert := tlstest.TestingKeyPair(f.t) state.ServerCert = func() *localtls.CertInfo { return serverCert } gateway := newGateway(f.t, state.DB.Node, serverCert, state) f.cleanups = append(f.cleanups, func() { _ = gateway.Shutdown() }) mux := http.NewServeMux() server := newServer(serverCert, mux) for path, handler := range gateway.HandlerFuncs(nil, trustedCerts) { mux.HandleFunc(path, handler) } address := server.Listener.Addr().String() mf := &membershipFixtures{t: f.t, state: state} mf.ClusterAddress(address) var err error require.NoError(f.t, state.DB.Cluster.Close()) store := gateway.NodeStore() dial := gateway.DialFunc() state.DB.Cluster, err = db.OpenCluster(context.Background(), "db.bin", store, address, "/unused/db/dir", 5*time.Second, driver.WithDialFunc(dial)) require.NoError(f.t, err) err = state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { state.GlobalConfig, err = clusterConfig.Load(ctx, tx) if err != nil { return err } // Get the local node (will be used if clustered). state.ServerName, err = tx.GetLocalNodeName(ctx) if err != nil { return err } return nil }) require.NoError(f.t, err) err = state.DB.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { state.LocalConfig, err = node.ConfigLoad(ctx, tx) return err }) require.NoError(f.t, err) f.gateways[len(f.gateways)] = gateway f.states[gateway] = state f.servers[gateway] = server return state, gateway, address } func (f *heartbeatFixture) Cleanup() { // Run the cleanups in reverse order for i := len(f.cleanups) - 1; i >= 0; i-- { f.cleanups[i]() } for _, server := range f.servers { server.Close() } } incus-7.0.0/internal/server/cluster/info.go000066400000000000000000000023471517523235500207420ustar00rootroot00000000000000package cluster import ( "context" "os" "path/filepath" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/shared/logger" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) // Load information about the dqlite node associated with this cluster member. func loadInfo(database *db.Node, cert *localtls.CertInfo) (*db.RaftNode, error) { // Figure out if we actually need to act as dqlite node. var info *db.RaftNode err := database.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { var err error info, err = node.DetermineRaftNode(ctx, tx) return err }) if err != nil { return nil, err } // If we're not part of the dqlite cluster, there's nothing to do. if info == nil { return nil, nil } if info.Address == "" { // This is a standalone node not exposed to the network. info.Address = "1" } logger.Info("Starting database node", logger.Ctx{"id": info.ID, "local": info.Address, "role": info.Role}) // Data directory dir := filepath.Join(database.Dir(), "global") if !util.PathExists(dir) { err := os.Mkdir(dir, 0o750) if err != nil { return nil, err } } return info, nil } incus-7.0.0/internal/server/cluster/member_state.go000066400000000000000000000057451517523235500224630ustar00rootroot00000000000000package cluster import ( "context" "fmt" "os" "strconv" "strings" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) // getLoadAvgs returns the host's load averages from /proc/loadavg. func getLoadAvgs() ([]float64, error) { loadAvgs := make([]float64, 3) loadAvgsBuf, err := os.ReadFile("/proc/loadavg") if err != nil { return nil, err } loadAvgFields := strings.Fields(string(loadAvgsBuf)) loadAvgs[0], err = strconv.ParseFloat(loadAvgFields[0], 64) if err != nil { return nil, err } loadAvgs[1], err = strconv.ParseFloat(loadAvgFields[1], 64) if err != nil { return nil, err } loadAvgs[2], err = strconv.ParseFloat(loadAvgFields[2], 64) if err != nil { return nil, err } return loadAvgs, nil } // MemberState retrieves state information about the cluster member. func MemberState(ctx context.Context, s *state.State, memberName string) (*api.ClusterMemberState, error) { var err error var memberState api.ClusterMemberState // Get system info. info := unix.Sysinfo_t{} err = unix.Sysinfo(&info) if err != nil { logger.Warn("Failed getting sysinfo", logger.Ctx{"err": err}) return nil, err } // Account for different representations of Sysinfo_t on different architectures. memberState.SysInfo.Uptime = int64(info.Uptime) memberState.SysInfo.TotalRAM = uint64(info.Totalram) memberState.SysInfo.SharedRAM = uint64(info.Sharedram) memberState.SysInfo.BufferRAM = uint64(info.Bufferram) memberState.SysInfo.FreeRAM = uint64(info.Freeram) memberState.SysInfo.TotalSwap = uint64(info.Totalswap) memberState.SysInfo.FreeSwap = uint64(info.Freeswap) memberState.SysInfo.Processes = info.Procs memberState.SysInfo.LoadAverages, err = getLoadAvgs() if err != nil { return nil, fmt.Errorf("Failed getting load averages: %w", err) } // Get storage pool states. stateCreated := db.StoragePoolCreated var pools map[int64]api.StoragePool var poolMembers map[int64]map[int64]db.StoragePoolNode err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { pools, poolMembers, err = tx.GetStoragePools(ctx, &stateCreated) return err }) if err != nil { return nil, fmt.Errorf("Failed loading storage pools: %w", err) } memberState.StoragePools = make(map[string]api.StoragePoolState, len(pools)) for poolID := range pools { pool, err := storagePools.LoadByRecord(s, poolID, pools[poolID], poolMembers[poolID]) if err != nil { return nil, fmt.Errorf("Failed loading storage pool %q: %w", pools[poolID].Name, err) } res, err := pool.GetResources() if err != nil { return nil, fmt.Errorf("Failed getting storage pool resources %q: %w", pools[poolID].Name, err) } memberState.StoragePools[pools[poolID].Name] = api.StoragePoolState{ ResourcesStoragePool: *res, } } return &memberState, nil } incus-7.0.0/internal/server/cluster/membership.go000066400000000000000000001144511517523235500221420ustar00rootroot00000000000000package cluster import ( "context" "crypto/x509" "encoding/pem" "errors" "fmt" "os" "path/filepath" "slices" "sync" "time" "github.com/cowsql/go-cowsql/app" "github.com/cowsql/go-cowsql/client" "github.com/lxc/incus/v7/internal/server/certificate" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/internal/server/state" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/logger" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) // errClusterBusy is returned by dqlite if attempting attempting to join a cluster at the same time as a role-change. // This error tells us we can retry and probably join the cluster or fail due to something else. // The error code here is SQLITE_BUSY. var errClusterBusy = errors.New("A configuration change is already in progress (5)") // Bootstrap turns a non-clustered server into the first (and leader) // member of a new cluster. // // This instance must already have its cluster.https_address set and be listening // on the associated network address. func Bootstrap(state *state.State, gateway *Gateway, serverName string) error { // Check parameters if serverName == "" { return errors.New("Server name must not be empty") } err := membershipCheckNoLeftoverClusterCert(state.OS.VarDir) if err != nil { return err } var localClusterAddress string err = state.DB.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { // Fetch current network address and raft nodes config, err := node.ConfigLoad(ctx, tx) if err != nil { return fmt.Errorf("Failed to fetch node configuration: %w", err) } localClusterAddress = config.ClusterAddress() // Make sure node-local database state is in order. err = membershipCheckNodeStateForBootstrapOrJoin(ctx, tx, localClusterAddress) if err != nil { return err } // Add ourselves as first raft node err = tx.CreateFirstRaftNode(localClusterAddress, serverName) if err != nil { return fmt.Errorf("Failed to insert first raft node: %w", err) } return nil }) if err != nil { return err } // Update our own entry in the nodes table. err = state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Make sure cluster database state is in order. err := membershipCheckClusterStateForBootstrapOrJoin(ctx, tx) if err != nil { return err } // Add ourselves to the nodes table. err = tx.BootstrapNode(serverName, localClusterAddress) if err != nil { return fmt.Errorf("Failed updating cluster member: %w", err) } err = EnsureServerCertificateTrusted(serverName, state.ServerCert(), tx) if err != nil { return fmt.Errorf("Failed ensuring server certificate is trusted: %w", err) } return nil }) if err != nil { return err } // Reload the trusted certificate cache to enable the certificate we just added to the local trust store // to be used when validating endpoint connections. This will allow Dqlite to connect to ourselves. state.UpdateCertificateCache() // Shutdown the gateway. This will trash any dqlite connection against // our in-memory dqlite driver and shutdown the associated raft // instance. We also lock regular access to the cluster database since // we don't want any other database code to run while we're // reconfiguring raft. err = state.DB.Cluster.EnterExclusive() if err != nil { return fmt.Errorf("Failed to acquire cluster database lock: %w", err) } err = gateway.Shutdown() if err != nil { return fmt.Errorf("Failed to shutdown gRPC SQL gateway: %w", err) } // The cluster CA certificate is a symlink against the regular server CA certificate. if util.PathExists(filepath.Join(state.OS.VarDir, "server.ca")) { err := os.Symlink("server.ca", filepath.Join(state.OS.VarDir, "cluster.ca")) if err != nil { return fmt.Errorf("Failed to symlink server CA cert to cluster CA cert: %w", err) } } // Generate a new cluster certificate. clusterCert, err := internalUtil.LoadClusterCert(state.OS.VarDir) if err != nil { return fmt.Errorf("Failed to create cluster cert: %w", err) } // If endpoint listeners are active, apply new cluster certificate. if state.Endpoints != nil { gateway.networkCert = clusterCert state.Endpoints.NetworkUpdateCert(clusterCert) } // Re-initialize the gateway. This will create a new raft factory an // dqlite driver instance, which will be exposed over gRPC by the // gateway handlers. err = gateway.init(true) if err != nil { return fmt.Errorf("Failed to re-initialize gRPC SQL gateway: %w", err) } err = gateway.WaitLeadership() if err != nil { return err } // Make sure we can actually connect to the cluster database through // the network endpoint. This also releases the previously acquired // lock and makes the Go SQL pooling system invalidate the old // connection, so new queries will be executed over the new network // connection. err = state.DB.Cluster.ExitExclusive(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } return nil }) if err != nil { return fmt.Errorf("Cluster database initialization failed: %w", err) } return nil } // EnsureServerCertificateTrusted adds the serverCert to the DB trusted certificates store using the serverName. // If a certificate with the same fingerprint is already in the trust store, but is of the wrong type or name then // the existing certificate is updated to the correct type and name. If the existing certificate is the correct // type but the wrong name then an error is returned. And if the existing certificate is the correct type and name // then nothing more is done. func EnsureServerCertificateTrusted(serverName string, serverCert *localtls.CertInfo, tx *db.ClusterTx) error { // Parse our server certificate and prepare to add it to DB trust store. serverCertx509, err := x509.ParseCertificate(serverCert.KeyPair().Certificate[0]) if err != nil { return err } fingerprint := localtls.CertFingerprint(serverCertx509) dbCert := cluster.Certificate{ Fingerprint: fingerprint, Type: certificate.TypeServer, // Server type for intra-member communication. Name: serverName, Certificate: string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverCertx509.Raw})), } // Add our server cert to the DB trust store (so when other members join this cluster they will be // able to trust intra-cluster requests from this member). ctx := context.Background() existingCert, _ := cluster.GetCertificate(ctx, tx.Tx(), dbCert.Fingerprint) if existingCert != nil { if existingCert.Name != dbCert.Name && existingCert.Type == certificate.TypeServer { // Don't alter an existing server certificate that has our fingerprint but not our name. // Something is wrong as this shouldn't happen. return fmt.Errorf("Existing server certificate with different name %q already in trust store", existingCert.Name) } else if existingCert.Name != dbCert.Name && existingCert.Type != certificate.TypeServer { // Ensure that if a client certificate already exists that matches our fingerprint, that it // has the correct name and type for cluster operation, to allow us to associate member // server names to certificate names. err = cluster.UpdateCertificate(ctx, tx.Tx(), dbCert.Fingerprint, dbCert) if err != nil { return fmt.Errorf("Failed updating certificate name and type in trust store: %w", err) } } } else { _, err = cluster.CreateCertificate(ctx, tx.Tx(), dbCert) if err != nil { return fmt.Errorf("Failed adding server certificate to trust store: %w", err) } } return nil } // Accept a new node and add it to the cluster. // // This instance must already be clustered. // // Return an updated list raft database nodes (possibly including the newly // accepted node). func Accept(state *state.State, gateway *Gateway, name, address string, schema, api, arch int) ([]db.RaftNode, error) { // Check parameters if name == "" { return nil, errors.New("Member name must not be empty") } if address == "" { return nil, errors.New("Member address must not be empty") } // Insert the new node into the nodes table. var id int64 err := state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Check that the node can be accepted with these parameters. err := membershipCheckClusterStateForAccept(ctx, tx, name, address, schema, api) if err != nil { return err } // Add the new node. id, err = tx.CreateNodeWithArch(name, address, arch) if err != nil { return fmt.Errorf("Failed to insert new node into the database: %w", err) } // Mark the node as pending, so it will be skipped when // performing heartbeats or sending cluster // notifications. err = tx.SetNodePendingFlag(id, true) if err != nil { return fmt.Errorf("Failed to mark the new node as pending: %w", err) } return nil }) if err != nil { return nil, err } // Possibly insert the new node into the raft_nodes table (if we have // less than 3 database nodes). nodes, err := gateway.currentRaftNodes() if err != nil { return nil, fmt.Errorf("Failed to get raft nodes from the log: %w", err) } count := len(nodes) // Existing nodes voters := 0 standbys := 0 for _, node := range nodes { switch node.Role { case db.RaftVoter: voters++ case db.RaftStandBy: standbys++ } } node := db.RaftNode{ NodeInfo: client.NodeInfo{ ID: uint64(id), Address: address, Role: db.RaftSpare, }, Name: name, } if count > 1 && voters < int(state.GlobalConfig.MaxVoters()) { node.Role = db.RaftVoter } else if standbys < int(state.GlobalConfig.MaxStandBy()) { node.Role = db.RaftStandBy } nodes = append(nodes, node) return nodes, nil } // Join makes a non-clustered server join an existing cluster. // // It's assumed that Accept() was previously called against the leader node, // which handed the raft server ID. // // The cert parameter must contain the keypair/CA material of the cluster being // joined. func Join(state *state.State, gateway *Gateway, networkCert *localtls.CertInfo, serverCert *localtls.CertInfo, name string, raftNodes []db.RaftNode) error { // Check parameters if name == "" { return errors.New("Member name must not be empty") } var localClusterAddress string err := state.DB.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { // Fetch current network address and raft nodes config, err := node.ConfigLoad(ctx, tx) if err != nil { return fmt.Errorf("Failed to fetch node configuration: %w", err) } localClusterAddress = config.ClusterAddress() // Make sure node-local database state is in order. err = membershipCheckNodeStateForBootstrapOrJoin(ctx, tx, localClusterAddress) if err != nil { return err } // Set the raft nodes list to the one that was returned by Accept(). err = tx.ReplaceRaftNodes(raftNodes) if err != nil { return fmt.Errorf("Failed to set raft nodes: %w", err) } return nil }) if err != nil { return err } // Get the local config keys for the cluster pools and networks. It // assumes that the local storage pools and networks match the cluster // networks, if not an error will be returned. Also get any outstanding // operation, typically there will be just one, created by the POST // /cluster/nodes request which triggered this code. var pools map[string]map[string]string var networks map[string]map[string]string var operations []cluster.Operation err = state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { pools, err = tx.GetStoragePoolsLocalConfig(ctx) if err != nil { return err } networks, err = tx.GetNetworksLocalConfig(ctx) if err != nil { return err } nodeID := tx.GetNodeID() filter := cluster.OperationFilter{NodeID: &nodeID} operations, err = cluster.GetOperations(ctx, tx.Tx(), filter) if err != nil { return err } return nil }) if err != nil { return err } // Lock regular access to the cluster database since we don't want any // other database code to run while we're reconfiguring raft. err = state.DB.Cluster.EnterExclusive() if err != nil { return fmt.Errorf("Failed to acquire cluster database lock: %w", err) } // Shutdown the gateway and wipe any raft data. This will trash any // gRPC SQL connection against our in-memory dqlite driver and shutdown // the associated raft instance. err = gateway.Shutdown() if err != nil { return fmt.Errorf("Failed to shutdown gRPC SQL gateway: %w", err) } err = os.RemoveAll(state.OS.GlobalDatabaseDir()) if err != nil { return fmt.Errorf("Failed to remove existing raft data: %w", err) } // Re-initialize the gateway. This will create a new raft factory an // dqlite driver instance, which will be exposed over gRPC by the // gateway handlers. gateway.networkCert = networkCert err = gateway.init(false) if err != nil { return fmt.Errorf("Failed to re-initialize gRPC SQL gateway: %w", err) } // If we are listed among the database nodes, join the raft cluster. var info *db.RaftNode for _, node := range raftNodes { if node.Address == localClusterAddress { info = &node } } if info == nil { panic("Joining member not found") } logger.Info("Joining dqlite raft cluster", logger.Ctx{"id": info.ID, "local": info.Address, "role": info.Role}) ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() client, err := client.FindLeader( ctx, gateway.NodeStore(), client.WithDialFunc(gateway.raftDial()), client.WithLogFunc(DqliteLog), ) if err != nil { return fmt.Errorf("Failed to connect to cluster leader: %w", err) } defer func() { _ = client.Close() }() logger.Info("Adding node to cluster", logger.Ctx{"id": info.ID, "local": info.Address, "role": info.Role}) ctx, cancel = context.WithTimeout(context.Background(), time.Minute) defer cancel() // Repeatedly try to join in case the cluster is busy with a role-change. joined := false for !joined { select { case <-ctx.Done(): return fmt.Errorf("Failed to join cluster: %w", ctx.Err()) default: err = client.Add(ctx, info.NodeInfo) if err != nil && err.Error() == errClusterBusy.Error() { // If the cluster is busy with a role change, sleep a second and then keep trying to join. time.Sleep(1 * time.Second) continue } if err != nil { return fmt.Errorf("Failed to join cluster: %w", err) } joined = true } } // Make sure we can actually connect to the cluster database through // the network endpoint. This also releases the previously acquired // lock and makes the Go SQL pooling system invalidate the old // connection, so new queries will be executed over the new gRPC // network connection. Also, update the storage_pools and networks // tables with our local configuration. logger.Info("Migrate local data to cluster database") err = state.DB.Cluster.ExitExclusive(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { node, err := tx.GetPendingNodeByAddress(ctx, localClusterAddress) if err != nil { return fmt.Errorf("Failed to get ID of joining node: %w", err) } state.DB.Cluster.NodeID(node.ID) tx.NodeID(node.ID) // Storage pools. ids, err := tx.GetNonPendingStoragePoolsNamesToIDs(ctx) if err != nil { return fmt.Errorf("Failed to get cluster storage pool IDs: %w", err) } for name, id := range ids { err := tx.UpdateStoragePoolAfterNodeJoin(id, node.ID) if err != nil { return fmt.Errorf("Failed to add joining node's to the pool: %w", err) } driver, err := tx.GetStoragePoolDriver(ctx, id) if err != nil { return fmt.Errorf("Failed to get storage pool driver: %w", err) } // For all pools we add the config provided by the joining node. config, ok := pools[name] if !ok { return fmt.Errorf("Joining member has no config for pool %s", name) } err = tx.CreateStoragePoolConfig(id, node.ID, config) if err != nil { return fmt.Errorf("Failed to add joining node's pool config: %w", err) } if slices.Contains(db.StorageRemoteDriverNames(), driver) { // For remote pools we have to create volume entries for the joining node. err := tx.UpdateRemoteStoragePoolAfterNodeJoin(ctx, id, node.ID) if err != nil { return fmt.Errorf("Failed to create remote volumes for joining node: %w", err) } } } // Networks. netids, err := tx.GetNonPendingNetworkIDs(ctx) if err != nil { return fmt.Errorf("Failed to get cluster network IDs: %w", err) } for _, network := range netids { for name, id := range network { config, ok := networks[name] if !ok { // Not all networks are present as virtual networks (OVN) don't need entries. continue } err := tx.NetworkNodeJoin(id, node.ID) if err != nil { return fmt.Errorf("Failed to add joining node's to the network: %w", err) } err = tx.CreateNetworkConfig(id, node.ID, config) if err != nil { return fmt.Errorf("Failed to add joining node's network config: %w", err) } } } // Migrate outstanding operations. for _, operation := range operations { op := cluster.Operation{ UUID: operation.UUID, Type: operation.Type, NodeID: tx.GetNodeID(), } _, err := cluster.CreateOrReplaceOperation(ctx, tx.Tx(), op) if err != nil { return fmt.Errorf("Failed to migrate operation %s: %w", operation.UUID, err) } } // Remove the pending flag for ourselves // notifications. err = tx.SetNodePendingFlag(node.ID, false) if err != nil { return fmt.Errorf("Failed to unmark the node as pending: %w", err) } // Set last heartbeat time to now, as member is clearly online as it just successfully joined, // that way when we send the notification to all members below it will consider this member online. err = tx.SetNodeHeartbeat(node.Address, time.Now().UTC()) if err != nil { return fmt.Errorf("Failed setting last heartbeat time for member: %w", err) } return nil }) if err != nil { return fmt.Errorf("Cluster database initialization failed: %w", err) } // Generate partial heartbeat request containing just a raft node list. if state.Endpoints != nil { NotifyHeartbeat(state, gateway) } return nil } // NotifyHeartbeat attempts to send a heartbeat to all other members to notify them of a new or changed member. func NotifyHeartbeat(state *state.State, gateway *Gateway) { // If a heartbeat round is already running (and implicitly this means we are the leader), then cancel it // so we can distribute the fresh member state info. heartbeatCancel := gateway.HearbeatCancelFunc() if heartbeatCancel != nil { heartbeatCancel() // Wait for heartbeat to finish and then release. // Ignore staticcheck "SA2001: empty critical section" because we want to wait for the lock. gateway.HeartbeatLock.Lock() gateway.HeartbeatLock.Unlock() //nolint:staticcheck } hbState := NewAPIHearbeat(state.DB.Cluster) hbState.Time = time.Now().UTC() var err error var raftNodes []db.RaftNode var localClusterAddress string err = state.DB.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { raftNodes, err = tx.GetRaftNodes(ctx) if err != nil { return err } config, err := node.ConfigLoad(ctx, tx) if err != nil { return err } localClusterAddress = config.ClusterAddress() return nil }) if err != nil { logger.Warn("Failed to get current raft members", logger.Ctx{"err": err, "local": localClusterAddress}) return } var members []db.NodeInfo err = state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { members, err = tx.GetNodes(ctx) if err != nil { return err } return nil }) if err != nil { logger.Warn("Failed to get current cluster members", logger.Ctx{"err": err, "local": localClusterAddress}) return } // Setup a full-state notification heartbeat. hbState.Update(true, raftNodes, members, gateway.HeartbeatOfflineThreshold) var wg sync.WaitGroup // Refresh local event listeners. wg.Add(1) go func() { EventsUpdateListeners(state, hbState.Members, state.Events.Inject) wg.Done() }() // Notify all other members of the change in membership. logger.Info("Notifying cluster members of local role change") for _, member := range members { if member.Address == localClusterAddress { continue } wg.Add(1) go func(address string) { _ = HeartbeatNode(context.Background(), address, state.Endpoints.NetworkCert(), state.ServerCert(), hbState) wg.Done() }(member.Address) } // Wait until all members have been notified (or at least have had a change to be notified). wg.Wait() } // Rebalance adjusts raft node roles to maintain cluster membership limits. // // - Incus nodes with the 'database-client' role are mapped to the raft // 'spare' role during rebalancing. // - If we are below membershipMaxRaftVoters, promote a node to 'voter'. // - If we are below membershipMaxStandBys, promote a node to 'standby'. // // Returns: // - the address of the node that was promoted or demoted (if any). // - and the full list of raft nodes after rebalancing. func Rebalance(state *state.State, gateway *Gateway, unavailableMembers []string) (string, []db.RaftNode, error) { // If we're a standalone node, do nothing. if gateway.memoryDial != nil { return "", nil, nil } nodes, err := gateway.currentRaftNodes() if err != nil { return "", nil, fmt.Errorf("Get current raft nodes: %w", err) } var members []db.NodeInfo err = state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { members, err = tx.GetNodes(ctx) if err != nil { return err } return nil }) if err != nil { return "", nil, fmt.Errorf("Get current members: %w", err) } // Prepare a map that stores node information by address. membersInfo := map[string]db.NodeInfo{} for _, member := range members { membersInfo[member.Address] = member } for i, n := range nodes { // If no member has this address, continue searching. This should not happen. member, ok := membersInfo[n.Address] if !ok { continue } // Check if the node has the 'database-client' role. if !slices.Contains(member.Roles, db.ClusterRoleDatabaseClient) { continue } // If the node already has the 'spare' role, do nothing. if n.Role == db.RaftSpare { continue } nodes[i].Role = db.RaftSpare return n.Address, nodes, nil } roles, err := newRolesChanges(state, gateway, nodes, unavailableMembers) if err != nil { return "", nil, err } role, candidates := roles.Adjust(gateway.info.ID) if role == -1 { // No node to process. return "", nodes, nil } // Check if we have a spare node that we can promote to the missing role. candidateAddress := "" for _, candidate := range candidates { // If no member has this address, continue searching. This should not happen. member, ok := membersInfo[candidate.Address] if !ok { continue } // Exclude nodes with the "database-client" role from candidates. if slices.Contains(member.Roles, db.ClusterRoleDatabaseClient) { continue } candidateAddress = candidate.Address } for i, node := range nodes { if node.Address == candidateAddress { nodes[i].Role = role break } } return candidateAddress, nodes, nil } // Assign a new role to the local dqlite node. func Assign(state *state.State, gateway *Gateway, nodes []db.RaftNode) error { // Figure out our own address. address := "" err := state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error address, err = tx.GetLocalNodeAddress(ctx) if err != nil { return fmt.Errorf("Failed to fetch the address of this cluster member: %w", err) } return nil }) if err != nil { return err } // Ensure we actually have an address. if address == "" { return errors.New("Cluster member is not exposed on the network") } // Figure out our node identity. var info *db.RaftNode for i, node := range nodes { if node.Address == address { info = &nodes[i] } } // Ensure that our address was actually included in the given list of raft nodes. if info == nil { return errors.New("This member is not included in the given list of database nodes") } // Replace our local list of raft nodes with the given one (which // includes ourselves). err = state.DB.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { err = tx.ReplaceRaftNodes(nodes) if err != nil { return fmt.Errorf("Failed to set raft nodes: %w", err) } return nil }) if err != nil { return err } var transactor func(context.Context, func(ctx context.Context, tx *db.ClusterTx) error) error // If we are already running a dqlite node, it means we have cleanly // joined the cluster before, using the roles support API. In that case // there's no need to restart the gateway and we can just change our // dqlite role. if gateway.IsDqliteNode() { transactor = state.DB.Cluster.Transaction goto assign } // If we get here it means that we are an upgraded node from cluster // without roles support, or we didn't cleanly join the cluster. Either // way, we don't have a dqlite node running, so we need to restart the // gateway. // Lock regular access to the cluster database since we don't want any // other database code to run while we're reconfiguring raft. err = state.DB.Cluster.EnterExclusive() if err != nil { return fmt.Errorf("Failed to acquire cluster database lock: %w", err) } transactor = state.DB.Cluster.ExitExclusive // Wipe all existing raft data, for good measure (perhaps they were // somehow leftover). err = os.RemoveAll(state.OS.GlobalDatabaseDir()) if err != nil { return fmt.Errorf("Failed to remove existing raft data: %w", err) } // Re-initialize the gateway. This will create a new raft factory an // dqlite driver instance, which will be exposed over gRPC by the // gateway handlers. err = gateway.init(false) if err != nil { return fmt.Errorf("Failed to re-initialize gRPC SQL gateway: %w", err) } assign: logger.Info("Changing local database role", logger.Ctx{"role": info.Role}) ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() client, err := client.FindLeader(ctx, gateway.NodeStore(), client.WithDialFunc(gateway.raftDial())) if err != nil { return fmt.Errorf("Connect to cluster leader: %w", err) } defer func() { _ = client.Close() }() // Figure out our current role. role := db.RaftRole(-1) cluster, err := client.Cluster(ctx) if err != nil { return fmt.Errorf("Fetch current cluster configuration: %w", err) } for _, server := range cluster { if server.ID == info.ID { role = server.Role break } } if role == -1 { return fmt.Errorf("Node %s does not belong to the current raft configuration", address) } // If we're stepping back from voter to spare, let's first transition // to stand-by first and wait for the configuration change to be // notified to us. This prevent us from thinking we're still voters and // potentially disrupt the cluster. if role == db.RaftVoter && info.Role == db.RaftSpare { err = client.Assign(ctx, info.ID, db.RaftStandBy) if err != nil { return fmt.Errorf("Failed to step back to stand-by: %w", err) } local, err := gateway.getClient() if err != nil { return fmt.Errorf("Failed to get local dqlite client: %w", err) } notified := false for range 10 { time.Sleep(500 * time.Millisecond) servers, err := local.Cluster(context.Background()) if err != nil { return fmt.Errorf("Failed to get current cluster: %w", err) } for _, server := range servers { if server.ID != info.ID { continue } if server.Role == db.RaftStandBy { notified = true break } } if notified { break } } if !notified { return errors.New("Timeout waiting for configuration change notification") } } // Give the Assign operation a bit more budget in case we're promoting // to voter, since that might require a snapshot transfer. if info.Role == db.RaftVoter { ctx, cancel = context.WithTimeout(context.Background(), 20*time.Second) defer cancel() } err = client.Assign(ctx, info.ID, info.Role) if err != nil { return fmt.Errorf("Failed to assign role: %w", err) } gateway.info = info // Unlock regular access to our cluster database. err = transactor(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return nil }) if err != nil { return fmt.Errorf("Cluster database initialization failed: %w", err) } // Generate partial heartbeat request containing just a raft node list. if state.Endpoints != nil { NotifyHeartbeat(state, gateway) } return nil } // Leave a cluster. // // If the force flag is true, the node will leave even if it still has // containers and images. // // The node will only leave the raft cluster, and won't be removed from the // database. That's done by Purge(). // // Upon success, return the address of the leaving node. // // This function must be called by the cluster leader. func Leave(s *state.State, gateway *Gateway, name string, force bool, pending bool) (string, error) { logger.Debugf("Make node %s leave the cluster", name) // Check if the node can be deleted and track its address. var address string err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get the node (if it doesn't exists an error is returned). var node db.NodeInfo var err error if pending { node, err = tx.GetPendingNodeByName(ctx, name) if err != nil { return fmt.Errorf("Failed to get member %q: %w", name, err) } } else { node, err = tx.GetNodeByName(ctx, name) if err != nil { return fmt.Errorf("Failed to get member %q: %w", name, err) } } // Check that the node is eligeable for leaving. if !force { err := membershipCheckClusterStateForLeave(ctx, tx, node.ID) if err != nil { return err } } address = node.Address return nil }) if err != nil { return "", err } nodes, err := gateway.currentRaftNodes() if err != nil { return "", err } var info *db.RaftNode // Raft node to remove, if any. for i, node := range nodes { if node.Address == address { info = &nodes[i] break } } if info == nil { // The node was not part of the raft cluster, nothing left to do. return address, nil } // Get the address of another database node, logger.Info( "Remove node from dqlite raft cluster", logger.Ctx{"id": info.ID, "address": info.Address}) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() client, err := gateway.getClient() if err != nil { return "", fmt.Errorf("Failed to connect to cluster leader: %w", err) } defer func() { _ = client.Close() }() err = client.Remove(ctx, info.ID) if err != nil { return "", fmt.Errorf("Failed to leave the cluster: %w", err) } return address, nil } // Handover looks for a non-voter member that can be promoted to replace a the // member with the given address, which is shutting down. It returns the // address of such member along with an updated list of nodes, with the ne role // set. // // It should be called only by the current leader. func Handover(state *state.State, gateway *Gateway, address string) (string, []db.RaftNode, error) { nodes, err := gateway.currentRaftNodes() if err != nil { return "", nil, fmt.Errorf("Get current raft nodes: %w", err) } var nodeID uint64 for _, node := range nodes { if node.Address == address { nodeID = node.ID } } if nodeID == 0 { return "", nil, fmt.Errorf("No dqlite node has address %s: %w", address, err) } roles, err := newRolesChanges(state, gateway, nodes, nil) if err != nil { return "", nil, err } role, candidates := roles.Handover(nodeID) if role == -1 { return "", nil, nil } for i, node := range nodes { if node.Address == candidates[0].Address { nodes[i].Role = role return node.Address, nodes, nil } } return "", nil, nil } // Build an app.RolesChanges object fed with the current cluster state. func newRolesChanges(state *state.State, gateway *Gateway, nodes []db.RaftNode, unavailableMembers []string) (*app.RolesChanges, error) { var domains map[string]uint64 err := state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error domains, err = tx.GetNodesFailureDomains(ctx) if err != nil { return fmt.Errorf("Load failure domains: %w", err) } return nil }) if err != nil { return nil, err } cluster := map[client.NodeInfo]*client.NodeMetadata{} for _, node := range nodes { if !slices.Contains(unavailableMembers, node.Address) && HasConnectivity(gateway.networkCert, gateway.state().ServerCert(), node.Address, false) { cluster[node.NodeInfo] = &client.NodeMetadata{ FailureDomain: domains[node.Address], } } else { cluster[node.NodeInfo] = nil } } roles := &app.RolesChanges{ Config: app.RolesConfig{ Voters: int(state.GlobalConfig.MaxVoters()), StandBys: int(state.GlobalConfig.MaxStandBy()), }, State: cluster, } return roles, nil } // Purge removes a node entirely from the cluster database. func Purge(c *db.Cluster, name string, pending bool) error { logger.Debugf("Remove node %s from the database", name) return c.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get the node (if it doesn't exists an error is returned). var node db.NodeInfo var err error if pending { node, err = tx.GetPendingNodeByName(ctx, name) if err != nil { return fmt.Errorf("Failed to get member %q: %w", name, err) } } else { node, err = tx.GetNodeByName(ctx, name) if err != nil { return fmt.Errorf("Failed to get member %q: %w", name, err) } } err = tx.ClearNode(ctx, node.ID) if err != nil { return fmt.Errorf("Failed to clear member %q: %w", name, err) } err = tx.RemoveNode(node.ID) if err != nil { return fmt.Errorf("Failed to remove member %q: %w", name, err) } err = cluster.DeleteCertificates(context.Background(), tx.Tx(), name, certificate.TypeServer) if err != nil { return fmt.Errorf("Failed to remove member %q certificate from trust store: %w", name, err) } return nil }) } // Count is a convenience for checking the current number of nodes in the // cluster. func Count(state *state.State) (int, error) { var count int err := state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error count, err = tx.GetNodesCount(ctx) return err }) return count, err } // Enabled is a convenience that returns true if clustering is enabled on this // node. func Enabled(node *db.Node) (bool, error) { enabled := false err := node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { addresses, err := tx.GetRaftNodeAddresses(ctx) if err != nil { return err } enabled = len(addresses) > 0 return nil }) return enabled, err } // Check that node-related preconditions are met for bootstrapping or joining a // cluster. func membershipCheckNodeStateForBootstrapOrJoin(ctx context.Context, tx *db.NodeTx, address string) error { nodes, err := tx.GetRaftNodes(ctx) if err != nil { return fmt.Errorf("Failed to fetch current raft nodes: %w", err) } hasClusterAddress := address != "" hasRaftNodes := len(nodes) > 0 // Ensure that we're not in an inconsistent situation, where no cluster address is set, but still there // are entries in the raft_nodes table. if !hasClusterAddress && hasRaftNodes { return errors.New("Inconsistent state: found leftover entries in raft_nodes") } if !hasClusterAddress { return errors.New("No cluster.https_address config is set on this member") } if hasRaftNodes { return errors.New("The member is already part of a cluster") } return nil } // Check that cluster-related preconditions are met for bootstrapping or // joining a cluster. func membershipCheckClusterStateForBootstrapOrJoin(ctx context.Context, tx *db.ClusterTx) error { members, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } if len(members) != 1 { return errors.New("Inconsistent state: Found leftover entries in cluster members") } return nil } // Check that cluster-related preconditions are met for accepting a new node. func membershipCheckClusterStateForAccept(ctx context.Context, tx *db.ClusterTx, name string, address string, schema int, api int) error { members, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } if len(members) == 1 && members[0].Address == "0.0.0.0" { return errors.New("Clustering isn't enabled") } for _, member := range members { if member.Name == name { return fmt.Errorf("The cluster already has a member with name: %s", name) } if member.Address == address { return fmt.Errorf("The cluster already has a member with address: %s", address) } if member.Schema != schema { return fmt.Errorf("The joining server version doesn't match (expected %s with DB schema %v)", version.Version, schema) } if member.APIExtensions != api { return fmt.Errorf("The joining server version doesn't match (expected %s with API count %v)", version.Version, api) } } return nil } // Check that cluster-related preconditions are met for leaving a cluster. func membershipCheckClusterStateForLeave(ctx context.Context, tx *db.ClusterTx, nodeID int64) error { // Check that it has no containers or images. message, err := tx.NodeIsEmpty(ctx, nodeID) if err != nil { return err } if message != "" { return errors.New(message) } // Check that it's not the last member. members, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } if len(members) == 1 { return errors.New("Member is the only member in the cluster") } return nil } // Check that there is no left-over cluster certificate in the var dir of this server. func membershipCheckNoLeftoverClusterCert(dir string) error { // Ensure that there's no leftover cluster certificate. for _, basename := range []string{"cluster.crt", "cluster.key", "cluster.ca"} { if util.PathExists(filepath.Join(dir, basename)) { return errors.New("Inconsistent state: found leftover cluster certificate") } } return nil } // SchemaVersion holds the version of the cluster database schema. var SchemaVersion = cluster.SchemaVersion incus-7.0.0/internal/server/cluster/membership_test.go000066400000000000000000000353061517523235500232020ustar00rootroot00000000000000package cluster_test import ( "context" "crypto/x509" "fmt" "net/http" "os" "path/filepath" "testing" "time" "github.com/cowsql/go-cowsql/driver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/certificate" "github.com/lxc/incus/v7/internal/server/cluster" clusterConfig "github.com/lxc/incus/v7/internal/server/cluster/config" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/osarch" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/tls/tlstest" "github.com/lxc/incus/v7/shared/util" ) func init() { db.StorageRemoteDriverNames = func() []string { // Tests only use ceph. return []string{"ceph"} } } func TestBootstrap_UnmetPreconditions(t *testing.T) { cases := []struct { setup func(*membershipFixtures) error string }{ { func(f *membershipFixtures) { f.ClusterAddress("1.2.3.4:666") f.RaftNode("5.6.7.8:666") filename := filepath.Join(f.state.OS.VarDir, "cluster.crt") _ = os.WriteFile(filename, []byte{}, 0o644) }, "Inconsistent state: found leftover cluster certificate", }, { func(*membershipFixtures) {}, "No cluster.https_address config is set on this member", }, { func(f *membershipFixtures) { f.ClusterAddress("1.2.3.4:666") f.RaftNode("5.6.7.8:666") }, "The member is already part of a cluster", }, { func(f *membershipFixtures) { f.RaftNode("5.6.7.8:666") }, "Inconsistent state: found leftover entries in raft_nodes", }, { func(f *membershipFixtures) { f.ClusterAddress("1.2.3.4:666") f.ClusterNode("5.6.7.8:666") }, "Inconsistent state: Found leftover entries in cluster members", }, } for _, c := range cases { t.Run(c.error, func(t *testing.T) { state, cleanup := state.NewTestState(t) defer cleanup() c.setup(&membershipFixtures{t: t, state: state}) serverCert := tlstest.TestingKeyPair(t) state.ServerCert = func() *localtls.CertInfo { return serverCert } gateway := newGateway(t, state.DB.Node, serverCert, state) defer func() { _ = gateway.Shutdown() }() err := cluster.Bootstrap(state, gateway, "buzz") assert.EqualError(t, err, c.error) }) } } func TestBootstrap(t *testing.T) { state, cleanup := state.NewTestState(t) defer cleanup() serverCert := tlstest.TestingKeyPair(t) state.ServerCert = func() *localtls.CertInfo { return serverCert } gateway := newGateway(t, state.DB.Node, serverCert, state) defer func() { _ = gateway.Shutdown() }() mux := http.NewServeMux() server := newServer(serverCert, mux) defer server.Close() address := server.Listener.Addr().String() f := &membershipFixtures{t: t, state: state} f.ClusterAddress(address) err := cluster.Bootstrap(state, gateway, "buzz") require.NoError(t, err) // The node-local database has now an entry in the raft_nodes table err = state.DB.Node.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { nodes, err := tx.GetRaftNodes(ctx) require.NoError(t, err) require.Len(t, nodes, 1) assert.Equal(t, uint64(1), nodes[0].ID) assert.Equal(t, address, nodes[0].Address) return nil }) require.NoError(t, err) // The cluster database has now an entry in the nodes table err = state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { members, err := tx.GetNodes(ctx) require.NoError(t, err) require.Len(t, members, 1) assert.Equal(t, "buzz", members[0].Name) assert.Equal(t, address, members[0].Address) return nil }) require.NoError(t, err) // The cluster certificate is in place. assert.True(t, util.PathExists(filepath.Join(state.OS.VarDir, "cluster.crt"))) // The dqlite driver is now exposed over the network. for path, handler := range gateway.HandlerFuncs(nil, trustedCerts) { mux.HandleFunc(path, handler) } count, err := cluster.Count(state) require.NoError(t, err) assert.Equal(t, 1, count) enabled, err := cluster.Enabled(state.DB.Node) require.NoError(t, err) assert.True(t, enabled) } // If pre-conditions are not met, a descriptive error is returned. func TestAccept_UnmetPreconditions(t *testing.T) { cases := []struct { name string address string schema int api int setup func(*membershipFixtures) error string }{ { "buzz", "1.2.3.4:666", cluster.SchemaVersion, len(version.APIExtensions), func(f *membershipFixtures) {}, "Clustering isn't enabled", }, { "rusp", "1.2.3.4:666", cluster.SchemaVersion, len(version.APIExtensions), func(f *membershipFixtures) { f.ClusterNode("5.6.7.8:666") }, "The cluster already has a member with name: rusp", }, { "buzz", "5.6.7.8:666", cluster.SchemaVersion, len(version.APIExtensions), func(f *membershipFixtures) { f.ClusterNode("5.6.7.8:666") }, "The cluster already has a member with address: 5.6.7.8:666", }, { "buzz", "1.2.3.4:666", cluster.SchemaVersion - 1, len(version.APIExtensions), func(f *membershipFixtures) { f.ClusterNode("5.6.7.8:666") }, fmt.Sprintf("The joining server version doesn't match (expected %s with DB schema %d)", version.Version, cluster.SchemaVersion-1), }, { "buzz", "1.2.3.4:666", cluster.SchemaVersion, len(version.APIExtensions) - 1, func(f *membershipFixtures) { f.ClusterNode("5.6.7.8:666") }, fmt.Sprintf("The joining server version doesn't match (expected %s with API count %d)", version.Version, len(version.APIExtensions)-1), }, } for _, c := range cases { t.Run(c.error, func(t *testing.T) { state, cleanup := state.NewTestState(t) defer cleanup() serverCert := tlstest.TestingKeyPair(t) state.ServerCert = func() *localtls.CertInfo { return serverCert } gateway := newGateway(t, state.DB.Node, serverCert, state) defer func() { _ = gateway.Shutdown() }() c.setup(&membershipFixtures{t: t, state: state}) _, err := cluster.Accept(state, gateway, c.name, c.address, c.schema, c.api, osarch.ARCH_64BIT_INTEL_X86) assert.EqualError(t, err, c.error) }) } } // When a node gets accepted, it gets included in the raft nodes. func TestAccept(t *testing.T) { state, cleanup := state.NewTestState(t) defer cleanup() serverCert := tlstest.TestingKeyPair(t) state.ServerCert = func() *localtls.CertInfo { return serverCert } gateway := newGateway(t, state.DB.Node, serverCert, state) defer func() { _ = gateway.Shutdown() }() f := &membershipFixtures{t: t, state: state} f.RaftNode("1.2.3.4:666") f.ClusterNode("1.2.3.4:666") err := state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error state.GlobalConfig, err = clusterConfig.Load(ctx, tx) if err != nil { return err } // Get the local node (will be used if clustered). state.ServerName, err = tx.GetLocalNodeName(ctx) if err != nil { return err } return nil }) require.NoError(t, err) nodes, err := cluster.Accept( state, gateway, "buzz", "5.6.7.8:666", cluster.SchemaVersion, len(version.APIExtensions), osarch.ARCH_64BIT_INTEL_X86) assert.NoError(t, err) assert.Len(t, nodes, 2) assert.Equal(t, uint64(1), nodes[0].ID) assert.Equal(t, uint64(3), nodes[1].ID) assert.Equal(t, "1.2.3.4:666", nodes[0].Address) assert.Equal(t, "5.6.7.8:666", nodes[1].Address) } func TestJoin(t *testing.T) { // Setup a target node running as leader of a cluster. targetCert := tlstest.TestingKeyPair(t) targetMux := http.NewServeMux() targetServer := newServer(targetCert, targetMux) defer targetServer.Close() targetState, cleanup := state.NewTestState(t) defer cleanup() targetState.ServerCert = func() *localtls.CertInfo { return targetCert } targetGateway := newGateway(t, targetState.DB.Node, targetCert, targetState) defer func() { _ = targetGateway.Shutdown() }() altServerCert := tlstest.TestingAltKeyPair(t) trustedAltServerCert, _ := x509.ParseCertificate(altServerCert.KeyPair().Certificate[0]) trustedCerts := func() (map[certificate.Type]map[string]x509.Certificate, error) { return map[certificate.Type]map[string]x509.Certificate{ certificate.TypeServer: { altServerCert.Fingerprint(): *trustedAltServerCert, }, }, nil } for path, handler := range targetGateway.HandlerFuncs(nil, trustedCerts) { targetMux.HandleFunc(path, handler) } targetAddress := targetServer.Listener.Addr().String() require.NoError(t, targetState.DB.Cluster.Close()) targetStore := targetGateway.NodeStore() targetDialFunc := targetGateway.DialFunc() var err error targetState.DB.Cluster, err = db.OpenCluster(context.Background(), "db.bin", targetStore, targetAddress, "/unused/db/dir", 10*time.Second, driver.WithDialFunc(targetDialFunc)) targetState.ServerCert = func() *localtls.CertInfo { return targetCert } require.NoError(t, err) err = targetState.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { targetState.GlobalConfig, err = clusterConfig.Load(ctx, tx) if err != nil { return err } // Get the local node (will be used if clustered). targetState.ServerName, err = tx.GetLocalNodeName(ctx) if err != nil { return err } return nil }) require.NoError(t, err) // PreparedStmts is a global variable and will be overwritten by the OpenCluster call below, so save it here. targetStmts := dbCluster.PreparedStmts targetF := &membershipFixtures{t: t, state: targetState} targetF.ClusterAddress(targetAddress) err = cluster.Bootstrap(targetState, targetGateway, "buzz") require.NoError(t, err) err = targetState.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, err = tx.GetNetworks(ctx, api.ProjectDefaultName) return err }) require.NoError(t, err) // Setup a joining node mux := http.NewServeMux() server := newServer(targetCert, mux) defer server.Close() state, cleanup := state.NewTestState(t) defer cleanup() state.ServerCert = func() *localtls.CertInfo { return altServerCert } gateway := newGateway(t, state.DB.Node, targetCert, state) defer func() { _ = gateway.Shutdown() }() for path, handler := range gateway.HandlerFuncs(nil, trustedCerts) { mux.HandleFunc(path, handler) } address := server.Listener.Addr().String() require.NoError(t, state.DB.Cluster.Close()) store := gateway.NodeStore() dialFunc := gateway.DialFunc() state.DB.Cluster, err = db.OpenCluster(context.Background(), "db.bin", store, address, "/unused/db/dir", 5*time.Second, driver.WithDialFunc(dialFunc)) require.NoError(t, err) err = state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { state.GlobalConfig, err = clusterConfig.Load(ctx, tx) if err != nil { return err } // Get the local node (will be used if clustered). state.ServerName, err = tx.GetLocalNodeName(ctx) if err != nil { return err } return nil }) require.NoError(t, err) // Save the other instance of PreparedStmts here. sourceStmts := dbCluster.PreparedStmts f := &membershipFixtures{t: t, state: state} f.ClusterAddress(address) // Accept the joining node. dbCluster.PreparedStmts = targetStmts raftNodes, err := cluster.Accept( targetState, targetGateway, "rusp", address, cluster.SchemaVersion, len(version.APIExtensions), osarch.ARCH_64BIT_INTEL_X86) require.NoError(t, err) // Actually join the cluster. dbCluster.PreparedStmts = sourceStmts err = cluster.Join(state, gateway, targetCert, altServerCert, "rusp", raftNodes) require.NoError(t, err) // The leader now returns an updated list of raft nodes. // The new node is not included to ensure distributed consensus. raftNodes, err = targetGateway.RaftNodes() require.NoError(t, err) assert.Len(t, raftNodes, 2) assert.Equal(t, uint64(1), raftNodes[0].ID) assert.Equal(t, targetAddress, raftNodes[0].Address) assert.Equal(t, db.RaftVoter, raftNodes[0].Role) assert.Equal(t, uint64(2), raftNodes[1].ID) assert.Equal(t, address, raftNodes[1].Address) assert.Equal(t, db.RaftStandBy, raftNodes[1].Role) // The Count function returns the number of nodes. count, err := cluster.Count(state) require.NoError(t, err) assert.Equal(t, 2, count) // Leave the cluster. leaving, err := cluster.Leave(state, targetGateway, "rusp", false /* force */, false) require.NoError(t, err) assert.Equal(t, address, leaving) dbCluster.PreparedStmts = targetStmts err = cluster.Purge(targetState.DB.Cluster, "rusp", false) require.NoError(t, err) // The node has gone from the cluster db. err = targetState.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { members, err := tx.GetNodes(ctx) require.NoError(t, err) assert.Len(t, members, 1) return nil }) require.NoError(t, err) // The node has gone from the raft cluster. members, err := targetGateway.RaftNodes() require.NoError(t, err) assert.Len(t, members, 1) } // Helper for setting fixtures for Bootstrap tests. type membershipFixtures struct { t *testing.T state *state.State } // Set core.https_address to the given value. func (h *membershipFixtures) CoreAddress(address string) { err := h.state.DB.Node.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { config := map[string]string{ "core.https_address": address, } return tx.UpdateConfig(config) }) require.NoError(h.t, err) } // Set cluster.https_address to the given value. func (h *membershipFixtures) ClusterAddress(address string) { err := h.state.DB.Node.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { config := map[string]string{ "cluster.https_address": address, } return tx.UpdateConfig(config) }) require.NoError(h.t, err) } // Add the given address to the raft_nodes table. func (h *membershipFixtures) RaftNode(address string) { err := h.state.DB.Node.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { _, err := tx.CreateRaftNode(address, "rusp") return err }) require.NoError(h.t, err) } // Get the current list of the raft nodes in the raft_nodes table. func (h *membershipFixtures) RaftNodes() []db.RaftNode { var nodes []db.RaftNode err := h.state.DB.Node.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { var err error nodes, err = tx.GetRaftNodes(ctx) return err }) require.NoError(h.t, err) return nodes } // Add the given address to the nodes table of the cluster database. func (h *membershipFixtures) ClusterNode(address string) { err := h.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, err := tx.CreateNode("rusp", address) return err }) require.NoError(h.t, err) } incus-7.0.0/internal/server/cluster/notify.go000066400000000000000000000066531517523235500213230ustar00rootroot00000000000000package cluster import ( "context" "fmt" "sync" "time" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/logger" localtls "github.com/lxc/incus/v7/shared/tls" ) // Notifier is a function that invokes the given function against each node in // the cluster excluding the invoking one. type Notifier func(hook func(incus.InstanceServer) error) error // NotifierPolicy can be used to tweak the behavior of NewNotifier in case of // some nodes are down. type NotifierPolicy int // Possible notification policies. const ( NotifyAll NotifierPolicy = iota // Requires that all nodes are up. NotifyAlive // Only notifies nodes that are alive NotifyTryAll // Attempt to notify all nodes regardless of state. ) // NewNotifier builds a Notifier that can be used to notify other peers using // the given policy. func NewNotifier(state *state.State, networkCert *localtls.CertInfo, serverCert *localtls.CertInfo, policy NotifierPolicy) (Notifier, error) { localClusterAddress := state.LocalConfig.ClusterAddress() // Fast-track the case where we're not clustered at all. if localClusterAddress == "" { nullNotifier := func(func(incus.InstanceServer) error) error { return nil } return nullNotifier, nil } var err error var members []db.NodeInfo var offlineThreshold time.Duration err = state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { offlineThreshold, err = tx.GetNodeOfflineThreshold(ctx) if err != nil { return err } members, err = tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } return nil }) if err != nil { return nil, err } peers := []string{} for _, member := range members { if member.Address == localClusterAddress || member.Address == "0.0.0.0" { continue // Exclude ourselves } if member.IsOffline(offlineThreshold) { // Even if the heartbeat timestamp is not recent // enough, let's try to connect to the node, just in // case the heartbeat is lagging behind for some reason // and the node is actually up. if !HasConnectivity(networkCert, serverCert, member.Address, true) { switch policy { case NotifyAll: return nil, fmt.Errorf("peer node %s is down", member.Address) case NotifyAlive: continue // Just skip this node case NotifyTryAll: } } } peers = append(peers, member.Address) } notifier := func(hook func(incus.InstanceServer) error) error { errs := make([]error, len(peers)) wg := sync.WaitGroup{} wg.Add(len(peers)) for i, address := range peers { logger.Debugf("Notify node %s of state changes", address) go func(i int, address string) { defer wg.Done() client, err := Connect(address, networkCert, serverCert, nil, true) if err != nil { errs[i] = fmt.Errorf("failed to connect to peer %s: %w", address, err) return } err = hook(client) if err != nil { errs[i] = fmt.Errorf("failed to notify peer %s: %w", address, err) } }(i, address) } wg.Wait() // TODO: aggregate all errors? for i, err := range errs { if err != nil { if localtls.IsConnectionError(err) && policy == NotifyAlive { logger.Warnf("Could not notify node %s", peers[i]) continue } return err } } return nil } return notifier, nil } incus-7.0.0/internal/server/cluster/notify_test.go000066400000000000000000000146661517523235500223650ustar00rootroot00000000000000package cluster_test import ( "context" "net/http" "net/http/httptest" "slices" "strconv" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/api" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/tls/tlstest" ) // The returned notifier connects to all nodes. func TestNewNotifier(t *testing.T) { state, cleanup := state.NewTestState(t) defer cleanup() cert := tlstest.TestingKeyPair(t) f := notifyFixtures{t: t, state: state} defer f.Nodes(cert, 3)() // Populate state.LocalConfig after nodes created above. var err error var nodeConfig *node.Config err = state.DB.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { nodeConfig, err = node.ConfigLoad(ctx, tx) return err }) require.NoError(t, err) state.LocalConfig = nodeConfig notifier, err := cluster.NewNotifier(state, cert, cert, cluster.NotifyAll) require.NoError(t, err) peers := make(chan string, 2) hook := func(client incus.InstanceServer) error { server, _, err := client.GetServer() require.NoError(t, err) peers <- server.Config["cluster.https_address"] return nil } assert.NoError(t, notifier(hook)) addresses := make([]string, 2) for i := range addresses { select { case addresses[i] = <-peers: default: } } require.NoError(t, err) for i := range addresses { assert.True(t, slices.Contains(addresses, f.Address(i+1))) } } // Creating a new notifier fails if the policy is set to NotifyAll and one of // the nodes is down. func TestNewNotify_NotifyAllError(t *testing.T) { state, cleanup := state.NewTestState(t) defer cleanup() cert := tlstest.TestingKeyPair(t) f := notifyFixtures{t: t, state: state} defer f.Nodes(cert, 3)() f.Down(1) // Populate state.LocalConfig after nodes created above. var err error var nodeConfig *node.Config err = state.DB.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { nodeConfig, err = node.ConfigLoad(ctx, tx) return err }) require.NoError(t, err) state.LocalConfig = nodeConfig notifier, err := cluster.NewNotifier(state, cert, cert, cluster.NotifyAll) assert.Nil(t, notifier) require.Error(t, err) assert.Regexp(t, "peer node .+ is down", err.Error()) } // Creating a new notifier does not fail if the policy is set to NotifyAlive // and one of the nodes is down, however dead nodes are ignored. func TestNewNotify_NotifyAlive(t *testing.T) { state, cleanup := state.NewTestState(t) defer cleanup() cert := tlstest.TestingKeyPair(t) f := notifyFixtures{t: t, state: state} defer f.Nodes(cert, 3)() f.Down(1) // Populate state.LocalConfig after nodes created above. var err error var nodeConfig *node.Config err = state.DB.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { nodeConfig, err = node.ConfigLoad(ctx, tx) return err }) require.NoError(t, err) state.LocalConfig = nodeConfig notifier, err := cluster.NewNotifier(state, cert, cert, cluster.NotifyAlive) assert.NoError(t, err) i := 0 hook := func(client incus.InstanceServer) error { i++ return nil } assert.NoError(t, notifier(hook)) assert.Equal(t, 1, i) } // Helper for setting fixtures for Notify tests. type notifyFixtures struct { t *testing.T state *state.State servers []*httptest.Server } // Spawn the given number of fake nodes, save in them in the database and // return a cleanup function. // // The address of the first node spawned will be saved as local // cluster.https_address. func (h *notifyFixtures) Nodes(cert *localtls.CertInfo, n int) func() { servers := make([]*httptest.Server, n) for i := range n { servers[i] = newRestServer(cert) } // Insert new entries in the nodes table of the cluster database. err := h.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { for i := range n { name := strconv.Itoa(i) address := servers[i].Listener.Addr().String() var err error if i == 0 { err = tx.BootstrapNode(name, address) } else { _, err = tx.CreateNode(name, address) } require.NoError(h.t, err) } return nil }) require.NoError(h.t, err) // Set the address in the config table of the node database. err = h.state.DB.Node.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { config, err := node.ConfigLoad(ctx, tx) require.NoError(h.t, err) address := servers[0].Listener.Addr().String() values := map[string]string{"cluster.https_address": address} _, err = config.Patch(values) require.NoError(h.t, err) return nil }) require.NoError(h.t, err) cleanup := func() { for _, server := range servers { server.Close() } } h.servers = servers return cleanup } // Return the network address of the i-th node. func (h *notifyFixtures) Address(i int) string { var address string err := h.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { members, err := tx.GetNodes(ctx) require.NoError(h.t, err) address = members[i].Address return nil }) require.NoError(h.t, err) return address } // Mark the i'th node as down. func (h *notifyFixtures) Down(i int) { err := h.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { members, err := tx.GetNodes(ctx) require.NoError(h.t, err) err = tx.SetNodeHeartbeat(members[i].Address, time.Now().Add(-time.Minute)) require.NoError(h.t, err) return nil }) require.NoError(h.t, err) h.servers[i].Close() } // Returns a minimal stub for the REST API server, just realistic // enough to make incus.ConnectIncus succeed. func newRestServer(cert *localtls.CertInfo) *httptest.Server { mux := http.NewServeMux() server := httptest.NewUnstartedServer(mux) server.TLS = localUtil.ServerTLSConfig(cert) server.StartTLS() mux.HandleFunc("/1.0/", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") config := map[string]string{"cluster.https_address": server.Listener.Addr().String()} metadata := api.ServerPut{Config: config} _ = localUtil.WriteJSON(w, api.ResponseRaw{Metadata: metadata}, nil) }) return server } incus-7.0.0/internal/server/cluster/options.go000066400000000000000000000017511517523235500215000ustar00rootroot00000000000000package cluster // Option to be passed to NewGateway to customize the resulting instance. type Option func(*options) // LogLevel sets the logging level for messages emitted by dqlite and raft. func LogLevel(level string) Option { return func(options *options) { options.logLevel = level } } // Latency is a coarse grain measure of how fast/reliable network links // are. This is used to tweak the various timeouts parameters of the raft // algorithm. See the raft.Config structure for more details. A value of 1.0 // means use the default values from hashicorp's raft package. Values closer to // 0 reduce the values of the various timeouts (useful when running unit tests // in-memory). func Latency(latency float64) Option { return func(options *options) { options.latency = latency } } // Create a options instance with default values. func newOptions() *options { return &options{ latency: 1.0, logLevel: "ERROR", } } type options struct { latency float64 logLevel string } incus-7.0.0/internal/server/cluster/raft_test.go000066400000000000000000000024001517523235500217700ustar00rootroot00000000000000package cluster_test import ( "context" "net/http" "net/http/httptest" "testing" "github.com/cowsql/go-cowsql/client" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/util" localtls "github.com/lxc/incus/v7/shared/tls" ) // Set the cluster.https_address config key to the given address, and insert the // address into the raft_nodes table. // // This effectively makes the node act as a database raft node. func setRaftRole(t *testing.T, database *db.Node, address string) client.NodeStore { require.NoError(t, database.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { err := tx.UpdateConfig(map[string]string{"cluster.https_address": address}) if err != nil { return err } _, err = tx.CreateRaftNode(address, "test") return err })) store := client.NewNodeStore(database.DB(), "main", "raft_nodes", "address") return store } // Create a new test HTTP server configured with the given TLS certificate and // using the given handler. func newServer(cert *localtls.CertInfo, handler http.Handler) *httptest.Server { server := httptest.NewUnstartedServer(handler) server.TLS = util.ServerTLSConfig(cert) server.StartTLS() return server } incus-7.0.0/internal/server/cluster/recover.go000066400000000000000000000135421517523235500214530ustar00rootroot00000000000000package cluster import ( "context" "errors" "fmt" "os" "path/filepath" "time" dqlite "github.com/cowsql/go-cowsql" client "github.com/cowsql/go-cowsql/client" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/node" ) // ListDatabaseNodes returns a list of database node names. func ListDatabaseNodes(database *db.Node) ([]string, error) { nodes := []db.RaftNode{} err := database.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { var err error nodes, err = tx.GetRaftNodes(ctx) return err }) if err != nil { return nil, fmt.Errorf("Failed to list database nodes: %w", err) } addresses := make([]string, 0) for _, node := range nodes { if node.Role != db.RaftVoter { continue } addresses = append(addresses, node.Address) } return addresses, nil } // Recover attempts data recovery on the cluster database. func Recover(database *db.Node) error { // Figure out if we actually act as dqlite node. var info *db.RaftNode err := database.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { var err error info, err = node.DetermineRaftNode(ctx, tx) return err }) if err != nil { return fmt.Errorf("Failed to determine node role: %w", err) } // If we're not a database node, return an error. if info == nil { return errors.New("This server has no database role") } // If this is a standalone node not exposed to the network, return an // error. if info.Address == "" { return errors.New("This server is not clustered") } dir := filepath.Join(database.Dir(), "global") server, err := dqlite.New( uint64(info.ID), info.Address, dir, ) if err != nil { return fmt.Errorf("Failed to create dqlite server: %w", err) } cluster := []dqlite.NodeInfo{ {ID: uint64(info.ID), Address: info.Address}, } err = server.Recover(cluster) if err != nil { return fmt.Errorf("Failed to recover database state: %w", err) } // Update the list of raft nodes. err = database.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { nodes := []db.RaftNode{ { NodeInfo: client.NodeInfo{ ID: info.ID, Address: info.Address, }, Name: info.Name, }, } return tx.ReplaceRaftNodes(nodes) }) if err != nil { return fmt.Errorf("Failed to update database nodes: %w", err) } return nil } // updateLocalAddress updates the cluster.https_address for this node. func updateLocalAddress(database *db.Node, address string) error { err := database.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { var err error config, err := node.ConfigLoad(ctx, tx) if err != nil { return err } newConfig := map[string]string{"cluster.https_address": address} _, err = config.Patch(newConfig) if err != nil { return err } return nil }) if err != nil { return fmt.Errorf("Failed to update node configuration: %w", err) } return nil } // Reconfigure replaces the entire cluster configuration. // Addresses and node roles may be updated. Node IDs are read-only. func Reconfigure(database *db.Node, raftNodes []db.RaftNode) error { var info *db.RaftNode err := database.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { var err error info, err = node.DetermineRaftNode(ctx, tx) return err }) if err != nil { return fmt.Errorf("Failed to determine cluster member raft role: %w", err) } if info == nil { return errors.New("This cluster member has no raft role") } localAddress := info.Address nodes := make([]client.NodeInfo, 0, len(raftNodes)) for _, raftNode := range raftNodes { nodes = append(nodes, raftNode.NodeInfo) // Get the new address for this node. if raftNode.ID == info.ID { localAddress = raftNode.Address } } // Update cluster.https_address if changed. if localAddress != info.Address { err := updateLocalAddress(database, localAddress) if err != nil { return err } } dir := filepath.Join(database.Dir(), "global") // Replace cluster configuration in dqlite. err = dqlite.ReconfigureMembershipExt(dir, nodes) if err != nil { return fmt.Errorf("Failed to recover database state: %w", err) } // Replace cluster configuration in local raft_nodes database. err = database.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { return tx.ReplaceRaftNodes(raftNodes) }) if err != nil { return err } // Create patch file for global nodes database. content := "" for _, node := range nodes { content += fmt.Sprintf("UPDATE nodes SET address = %q WHERE id = %d;\n", node.Address, node.ID) } if len(content) > 0 { filePath := filepath.Join(database.Dir(), "patch.global.sql") file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { return err } defer func() { _ = file.Close() }() _, err = file.Write([]byte(content)) if err != nil { return err } err = file.Close() if err != nil { return err } } return nil } // RemoveRaftNode removes a raft node from the raft configuration. func RemoveRaftNode(gateway *Gateway, address string) error { nodes, err := gateway.currentRaftNodes() if err != nil { return fmt.Errorf("Failed to get current raft nodes: %w", err) } var id uint64 for _, node := range nodes { if node.Address == address { id = node.ID break } } if id == 0 { return fmt.Errorf("No raft node with address %q", address) } ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() client, err := client.FindLeader( ctx, gateway.NodeStore(), client.WithDialFunc(gateway.raftDial()), client.WithLogFunc(DqliteLog), ) if err != nil { return fmt.Errorf("Failed to connect to cluster leader: %w", err) } defer func() { _ = client.Close() }() err = client.Remove(ctx, id) if err != nil { return fmt.Errorf("Failed to remove node: %w", err) } return nil } incus-7.0.0/internal/server/cluster/request/000077500000000000000000000000001517523235500211425ustar00rootroot00000000000000incus-7.0.0/internal/server/cluster/request/clienttype.go000066400000000000000000000026031517523235500236520ustar00rootroot00000000000000package request // UserAgentNotifier used to distinguish between a regular client request and an internal cluster request when // notifying other nodes of a cluster change. const UserAgentNotifier = "incus-cluster-notifier" // UserAgentClient used to distinguish between a regular client request and an internal cluster request when // performing a regular API interaction as an internal client. const UserAgentClient = "incus-cluster-client" // UserAgentJoiner used to distinguish between a regular client request and an internal cluster request when // joining a node to a cluster. const UserAgentJoiner = "incus-cluster-joiner" // ClientType indicates which sort of client type is being used. type ClientType string // ClientTypeNotifier cluster notification client. const ClientTypeNotifier ClientType = "notifier" // ClientTypeJoiner cluster joiner client. const ClientTypeJoiner ClientType = "joiner" // ClientTypeNormal normal client. const ClientTypeNormal ClientType = "normal" // ClientTypeInternal cluster internal client. const ClientTypeInternal ClientType = "internal" // UserAgentClientType converts user agent to client type. func UserAgentClientType(userAgent string) ClientType { switch userAgent { case UserAgentNotifier: return ClientTypeNotifier case UserAgentJoiner: return ClientTypeJoiner case UserAgentClient: return ClientTypeInternal } return ClientTypeNormal } incus-7.0.0/internal/server/cluster/resolve.go000066400000000000000000000015071517523235500214630ustar00rootroot00000000000000package cluster import ( "context" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/state" ) // ResolveTarget is a convenience for resolving a target member name to address. // It returns the address of the given member, or the empty string if the given member is the local one. func ResolveTarget(ctx context.Context, s *state.State, targetMember string) (string, error) { // Avoid starting a transaction if the requested target is this local server. if targetMember == s.ServerName { return "", nil } var memberAddress string err := s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { member, err := tx.GetNodeByName(ctx, targetMember) if err != nil { return err } memberAddress = member.Address return nil }) return memberAddress, err } incus-7.0.0/internal/server/cluster/tls.go000066400000000000000000000115461517523235500206120ustar00rootroot00000000000000package cluster import ( "context" "crypto/tls" "crypto/x509" "encoding/pem" "errors" "fmt" "net" "net/http" "time" "github.com/lxc/incus/v7/internal/server/certificate" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/logger" localtls "github.com/lxc/incus/v7/shared/tls" ) // Return a TLS configuration suitable for establishing intra-member network connections using the server cert. func tlsClientConfig(networkCert *localtls.CertInfo, serverCert *localtls.CertInfo) (*tls.Config, error) { if networkCert == nil { return nil, errors.New("Invalid networkCert") } if serverCert == nil { return nil, errors.New("Invalid serverCert") } keypair := serverCert.KeyPair() config := localtls.InitTLSConfig() config.Certificates = []tls.Certificate{keypair} config.RootCAs = x509.NewCertPool() ca := serverCert.CA() if ca != nil { config.RootCAs.AddCert(ca) } // Since the same cluster keypair is used both as server and as client // cert, let's add it to the CA pool to make it trusted. networkKeypair := networkCert.KeyPair() netCert, err := x509.ParseCertificate(networkKeypair.Certificate[0]) if err != nil { return nil, err } netCert.IsCA = true netCert.KeyUsage = x509.KeyUsageCertSign config.RootCAs.AddCert(netCert) // Always use network certificate's DNS name rather than server cert, so that it matches. if len(netCert.DNSNames) > 0 { config.ServerName = netCert.DNSNames[0] } return config, nil } // tlsCheckCert checks certificate access, returns true if certificate is trusted. func tlsCheckCert(r *http.Request, networkCert *localtls.CertInfo, serverCert *localtls.CertInfo, trustedCerts map[certificate.Type]map[string]x509.Certificate) bool { _, err := x509.ParseCertificate(networkCert.KeyPair().Certificate[0]) if err != nil { // Since we have already loaded this certificate, typically // using LoadX509KeyPair, an error should never happen, but // check for good measure. panic(fmt.Sprintf("Invalid keypair material: %v", err)) } if r.TLS == nil { return false } for _, i := range r.TLS.PeerCertificates { // Trust our own server certificate. This allows Dqlite to start with a connection back to this // member before the database is available. It also allows us to switch the server certificate to // the network certificate during cluster upgrade to per-server certificates, and it be trusted. trustedServerCert, _ := x509.ParseCertificate(serverCert.KeyPair().Certificate[0]) trusted, _ := localUtil.CheckTrustState(*i, map[string]x509.Certificate{serverCert.Fingerprint(): *trustedServerCert}, networkCert, false) if trusted { return true } // Check the trusted server certificates list provided. trusted, _ = localUtil.CheckTrustState(*i, trustedCerts[certificate.TypeServer], networkCert, false) if trusted { return true } logger.Errorf("Invalid client certificate %v (%v) from %v", i.Subject, localtls.CertFingerprint(i), r.RemoteAddr) } return false } // Return an http.Transport configured using the given configuration and a // cleanup function to use to close all connections the transport has been // used. func tlsTransport(networkCert *localtls.CertInfo, serverCert *localtls.CertInfo) (*http.Transport, func(), error) { config, err := tlsClientConfig(networkCert, serverCert) if err != nil { return nil, nil, err } // Set InsecureSkipVerify as we're doing our own certificate validation below. config.InsecureSkipVerify = true transport := &http.Transport{ TLSClientConfig: config, DisableKeepAlives: true, MaxIdleConns: 0, ExpectContinueTimeout: time.Second * 30, ResponseHeaderTimeout: time.Second * 3600, TLSHandshakeTimeout: time.Second * 5, } transport.DialTLSContext = func(ctx context.Context, network string, addr string) (net.Conn, error) { // Establish the TCP connection. conn, err := net.Dial("tcp", addr) if err != nil { return nil, fmt.Errorf("Failed connecting to HTTPS endpoint %q: %w", addr, err) } // Get TLS connection. tlsConn := tls.Client(conn, config) // Validate the connection err = tlsConn.Handshake() if err != nil { _ = conn.Close() return nil, err } // Look for an exact match with the certificate provided. // But ignore any other issue (validity, scope, ...). cs := tlsConn.ConnectionState() if len(cs.PeerCertificates) < 1 { return nil, errors.New("Couldn't validate peer certificate") } certBlock, _ := pem.Decode([]byte(networkCert.PublicKey())) if certBlock == nil { return nil, errors.New("Invalid remote certificate") } expectedRemoteCert, err := x509.ParseCertificate(certBlock.Bytes) if err != nil { return nil, err } if !cs.PeerCertificates[0].Equal(expectedRemoteCert) { return nil, errors.New("Remote certificate differs from expected") } return tlsConn, nil } return transport, transport.CloseIdleConnections, nil } incus-7.0.0/internal/server/cluster/tls_export_test.go000066400000000000000000000002071517523235500232420ustar00rootroot00000000000000package cluster // TLSClientConfig is used to generate TLS client configurations in unit tests. var TLSClientConfig = tlsClientConfig incus-7.0.0/internal/server/cluster/upgrade.go000066400000000000000000000137621517523235500214410ustar00rootroot00000000000000package cluster import ( "context" "errors" "fmt" "math/rand" "net/http" "os" "time" "github.com/cowsql/go-cowsql/client" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/subprocess" localtls "github.com/lxc/incus/v7/shared/tls" ) // NotifyUpgradeCompleted sends a notification to all other nodes in the // cluster that any possible pending database update has been applied, and any // nodes which was waiting for this node to be upgraded should re-check if it's // okay to move forward. func NotifyUpgradeCompleted(state *state.State, networkCert *localtls.CertInfo, serverCert *localtls.CertInfo) error { notifier, err := NewNotifier(state, networkCert, serverCert, NotifyTryAll) if err != nil { return err } return notifier(func(client incus.InstanceServer) error { info, err := client.GetConnectionInfo() if err != nil { return fmt.Errorf("failed to get connection info: %w", err) } url := fmt.Sprintf("%s%s", info.Addresses[0], databaseEndpoint) request, err := http.NewRequest("PATCH", url, nil) if err != nil { return fmt.Errorf("failed to create database notify upgrade request: %w", err) } setDqliteVersionHeader(request) httpClient, err := client.GetHTTPClient() if err != nil { return fmt.Errorf("failed to get HTTP client: %w", err) } httpClient.Timeout = 5 * time.Second response, err := httpClient.Do(request) if err != nil { return fmt.Errorf("failed to notify node about completed upgrade: %w", err) } if response.StatusCode != http.StatusOK { return fmt.Errorf("database upgrade notification failed: %s", response.Status) } return nil }) } // MaybeUpdate Check this node's version and possibly run INCUS_CLUSTER_UPDATE. func MaybeUpdate(s *state.State) error { shouldUpdate := false enabled, err := Enabled(s.DB.Node) if err != nil { return fmt.Errorf("Failed to check clustering is enabled: %w", err) } if !enabled { return nil } if s.DB.Cluster == nil { return errors.New("Failed checking cluster update, state not initialized yet") } err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { outdated, err := tx.NodeIsOutdated(ctx) if err != nil { return err } shouldUpdate = outdated return nil }) if err != nil { // Just log the error and return. return fmt.Errorf("Failed to check if this node is out-of-date: %w", err) } if !shouldUpdate { logger.Debugf("Cluster node is up-to-date") return nil } return triggerUpdate(s) } func triggerUpdate(s *state.State) error { logger.Warn("Member is out-of-date with respect to other cluster members") // If on IncusOS, start by trying an automatic update. if s.OS.IncusOS != nil { err := s.OS.IncusOS.TriggerSystemUpdateCheck() if err != nil { return err } } updateExecutable := os.Getenv("INCUS_CLUSTER_UPDATE") if updateExecutable == "" { logger.Debug("No INCUS_CLUSTER_UPDATE variable set, skipping auto-update") return nil } // Wait a random amount of seconds (up to 30) in order to avoid // restarting all cluster members at the same time, and make the // upgrade more graceful. wait := time.Duration(rand.Intn(30)) * time.Second logger.Info("Triggering cluster auto-update soon", logger.Ctx{"wait": wait, "updateExecutable": updateExecutable}) time.Sleep(wait) logger.Info("Triggering cluster auto-update now") _, err := subprocess.RunCommand(updateExecutable) if err != nil { logger.Error("Triggering cluster update failed", logger.Ctx{"err": err}) return err } logger.Info("Triggering cluster auto-update succeeded") return nil } // UpgradeMembersWithoutRole assigns the Spare raft role to all cluster members that are not currently part of the // raft configuration. It's used for upgrading a cluster from a version without roles support. func UpgradeMembersWithoutRole(gateway *Gateway, members []db.NodeInfo) error { nodes, err := gateway.currentRaftNodes() if errors.Is(err, ErrNotLeader) { return nil } if err != nil { return fmt.Errorf("Failed to get current raft members: %w", err) } // Convert raft node list to map keyed on ID. raftNodeIDs := map[uint64]bool{} for _, node := range nodes { raftNodeIDs[node.ID] = true } dqliteClient, err := gateway.getClient() if err != nil { return fmt.Errorf("Failed to connect to local dqlite member: %w", err) } defer func() { _ = dqliteClient.Close() }() // Check that each member is present in the raft configuration, and add it if not. for _, member := range members { found := false for _, node := range nodes { if member.ID == 1 && node.ID == 1 || member.Address == node.Address { found = true break } } if found { continue } // Try to use the same ID as the node, but it might not be possible if it's use. id := uint64(member.ID) _, ok := raftNodeIDs[id] if ok { for _, other := range members { _, ok := raftNodeIDs[uint64(other.ID)] if !ok { id = uint64(other.ID) // Found unused raft ID for member. break } } // This can't really happen (but has in the past) since there are always at least as many // members as there are nodes, and all of them have different IDs. if id == uint64(member.ID) { logger.Error("No available raft ID for cluster member", logger.Ctx{"memberID": member.ID, "members": members, "raftMembers": nodes}) return fmt.Errorf("No available raft ID for cluster member ID %d", member.ID) } } raftNodeIDs[id] = true info := db.RaftNode{ NodeInfo: client.NodeInfo{ ID: id, Address: member.Address, Role: db.RaftSpare, }, Name: "", } logger.Info("Add spare dqlite node", logger.Ctx{"id": info.ID, "address": info.Address}) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() err = dqliteClient.Add(ctx, info.NodeInfo) if err != nil { return fmt.Errorf("Failed to add dqlite member: %w", err) } } return nil } incus-7.0.0/internal/server/cluster/upgrade_test.go000066400000000000000000000126171517523235500224760ustar00rootroot00000000000000package cluster_test import ( "context" "errors" "fmt" "io/fs" "net/http" "os" "path/filepath" "sync" "testing" "time" "github.com/cowsql/go-cowsql/client" "github.com/cowsql/go-cowsql/driver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/internal/server/state" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/tls/tlstest" ) // A node can unblock other nodes that were waiting for a cluster upgrade to // complete. func TestNotifyUpgradeCompleted(t *testing.T) { f := heartbeatFixture{t: t} defer f.Cleanup() gateway0 := f.Bootstrap() gateway1 := f.Grow() wg := sync.WaitGroup{} wg.Add(1) go func() { gateway1.WaitUpgradeNotification() wg.Done() }() state0 := f.State(gateway0) // Populate state.LocalConfig after nodes created above. var err error var nodeConfig *node.Config err = state0.DB.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { nodeConfig, err = node.ConfigLoad(ctx, tx) return err }) require.NoError(t, err) state0.LocalConfig = nodeConfig serverCert0 := gateway0.ServerCert() err = cluster.NotifyUpgradeCompleted(state0, serverCert0, serverCert0) require.NoError(t, err) wg.Wait() } // The task function checks if the node is out of date and runs whatever is in // INCUS_CLUSTER_UPDATE if so. func TestMaybeUpdate_Upgrade(t *testing.T) { dir, err := os.MkdirTemp("", "") require.NoError(t, err) defer func() { _ = os.RemoveAll(dir) }() // Create a stub upgrade script that just touches a stamp file. stamp := filepath.Join(dir, "stamp") script := filepath.Join(dir, "cluster-upgrade") data := fmt.Appendf(nil, "#!/bin/sh\ntouch %s\n", stamp) err = os.WriteFile(script, data, 0o755) require.NoError(t, err) state, cleanup := state.NewTestState(t) defer cleanup() _ = state.DB.Node.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { nodes := []db.RaftNode{ {NodeInfo: client.NodeInfo{ID: 1, Address: "0.0.0.0:666"}}, {NodeInfo: client.NodeInfo{ID: 2, Address: "1.2.3.4:666"}}, } err := tx.ReplaceRaftNodes(nodes) require.NoError(t, err) return nil }) _ = state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { id, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) node, err := tx.GetNodeByName(ctx, "buzz") require.NoError(t, err) version := node.Version() version[0]++ err = tx.SetNodeVersion(id, version) require.NoError(t, err) return nil }) _ = os.Setenv("INCUS_CLUSTER_UPDATE", script) defer func() { _ = os.Unsetenv("INCUS_CLUSTER_UPDATE") }() _ = cluster.MaybeUpdate(state) _, err = os.Stat(stamp) require.NoError(t, err) } // If the node is up-to-date, nothing is done. func TestMaybeUpdate_NothingToDo(t *testing.T) { dir, err := os.MkdirTemp("", "") require.NoError(t, err) defer func() { _ = os.RemoveAll(dir) }() // Create a stub upgrade script that just touches a stamp file. stamp := filepath.Join(dir, "stamp") script := filepath.Join(dir, "cluster-upgrade") data := fmt.Appendf(nil, "#!/bin/sh\ntouch %s\n", stamp) err = os.WriteFile(script, data, 0o755) require.NoError(t, err) state, cleanup := state.NewTestState(t) defer cleanup() _ = os.Setenv("INCUS_CLUSTER_UPDATE", script) defer func() { _ = os.Unsetenv("INCUS_CLUSTER_UPDATE") }() _ = cluster.MaybeUpdate(state) _, err = os.Stat(stamp) require.True(t, errors.Is(err, fs.ErrNotExist)) } func TestUpgradeMembersWithoutRole(t *testing.T) { state, cleanup := state.NewTestState(t) defer cleanup() serverCert := tlstest.TestingKeyPair(t) mux := http.NewServeMux() server := newServer(serverCert, mux) defer server.Close() address := server.Listener.Addr().String() setRaftRole(t, state.DB.Node, address) state.ServerCert = func() *localtls.CertInfo { return serverCert } gateway := newGateway(t, state.DB.Node, serverCert, state) defer func() { _ = gateway.Shutdown() }() for path, handler := range gateway.HandlerFuncs(nil, trustedCerts) { mux.HandleFunc(path, handler) } var err error require.NoError(t, state.DB.Cluster.Close()) store := gateway.NodeStore() dial := gateway.DialFunc() state.DB.Cluster, err = db.OpenCluster(context.Background(), "db.bin", store, address, "/unused/db/dir", 5*time.Second, driver.WithDialFunc(dial)) require.NoError(t, err) gateway.Cluster = state.DB.Cluster // Add a couple of members to the database. var members []db.NodeInfo err = state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, err := tx.CreateNode("foo", "1.2.3.4") require.NoError(t, err) _, err = tx.CreateNode("bar", "5.6.7.8") require.NoError(t, err) members, err = tx.GetNodes(ctx) require.NoError(t, err) return nil }) require.NoError(t, err) err = cluster.UpgradeMembersWithoutRole(gateway, members) require.NoError(t, err) // The members have been added to the raft configuration. nodes, err := gateway.RaftNodes() require.NoError(t, err) assert.Len(t, nodes, 3) assert.Equal(t, uint64(1), nodes[0].ID) assert.Equal(t, address, nodes[0].Address) assert.Equal(t, uint64(2), nodes[1].ID) assert.Equal(t, "1.2.3.4", nodes[1].Address) assert.Equal(t, uint64(3), nodes[2].ID) assert.Equal(t, "5.6.7.8", nodes[2].Address) } incus-7.0.0/internal/server/config/000077500000000000000000000000001517523235500172365ustar00rootroot00000000000000incus-7.0.0/internal/server/config/errors.go000066400000000000000000000037271517523235500211120ustar00rootroot00000000000000package config import ( "fmt" "sort" "strings" ) // configurationError is generated when trying to set a config key to an erroneous value. type configurationError struct { configKey string erroneousValue any reason string } // ConfigurationError implements the error interface. func (e configurationError) Error() string { message := fmt.Sprintf("cannot set '%s'", e.configKey) if e.erroneousValue != nil { message += fmt.Sprintf(" to '%v'", e.erroneousValue) } return message + fmt.Sprintf(": %s", e.reason) } // ErrorList is a list of configuration Errors occurred during Load() or Map.Change(). type ErrorList struct { errors []configurationError } // ErrorList implements the error interface. func (l *ErrorList) Error() string { errorCount := l.Len() if errorCount == 0 { return "no errors" } errorMessage := strings.Builder{} firstError := l.errors[0].Error() errorMessage.WriteString(firstError) if errorCount > 1 { errorMessage.WriteString(fmt.Sprintf(" (and %d more errors)", errorCount-1)) } return errorMessage.String() } // Len returns the amount of errors contained in the list. This is needed to implement the sort Interface. func (l *ErrorList) Len() int { return len(l.errors) } // Swap swaps two errors at two indices. This is needed to implement the sort Interface. func (l *ErrorList) Swap(i, j int) { l.errors[i], l.errors[j] = l.errors[j], l.errors[i] } // Less defines an ordering of errors inside the error list. This is needed to implement the sort Interface. func (l *ErrorList) Less(i, j int) bool { return l.errors[i].configKey < l.errors[j].configKey } // sort sorts an ErrorList. *ConfigurationError entries are sorted by key name. func (l *ErrorList) sort() { sort.Sort(l) } // add adds an ConfigurationError with given key name, value and reason. func (l *ErrorList) add(configKey string, erroneousValue any, errorReason string) { l.errors = append(l.errors, configurationError{configKey, erroneousValue, errorReason}) } incus-7.0.0/internal/server/config/errors_internal_test.go000066400000000000000000000023731517523235500240410ustar00rootroot00000000000000package config import ( "testing" "github.com/stretchr/testify/assert" ) func Test_ErrorList_Error_NoErrors(t *testing.T) { // arrange errors := &ErrorList{} // assert assert.EqualError(t, errors, "no errors") } func Test_ErrorList_Error_OneError(t *testing.T) { // arrange errors := &ErrorList{} errors.add("qux", "zzz", "bean") // assert assert.EqualError(t, errors, "cannot set 'qux' to 'zzz': bean") } func Test_ErrorList_Error_TwoErrorsSorted(t *testing.T) { // arrange errors := &ErrorList{} errors.add("foo", "xxx", "boom") errors.add("bar", "yyy", "ugh") // act errors.sort() // assert assert.EqualError(t, errors, "cannot set 'bar' to 'yyy': ugh (and 1 more errors)") } func Test_ErrorList_Error_TwoErrorsUnsorted(t *testing.T) { // arrange errors := &ErrorList{} errors.add("foo", "xxx", "boom") errors.add("bar", "yyy", "ugh") // assert assert.EqualError(t, errors, "cannot set 'foo' to 'xxx': boom (and 1 more errors)") } func Test_ErrorList_Error_MoreThanTwoErrorsUnsorted(t *testing.T) { // arrange errors := &ErrorList{} errors.add("foo", "xxx", "boom") errors.add("bar", "yyy", "ugh") errors.add("qux", "zzz", "bean") // assert assert.EqualError(t, errors, "cannot set 'foo' to 'xxx': boom (and 2 more errors)") } incus-7.0.0/internal/server/config/logging.go000066400000000000000000000113121517523235500212110ustar00rootroot00000000000000package config import ( "fmt" "strings" "github.com/sirupsen/logrus" "github.com/lxc/incus/v7/shared/validate" ) // IsLoggingConfig reports whether the config key is for a logging configuration. func IsLoggingConfig(key string) bool { return strings.HasPrefix(key, "logging.") } // GetLoggingRuleForKey returns the rule for the specified logging config key. func GetLoggingRuleForKey(key string) (Key, error) { fields := strings.Split(key, ".") if len(fields) < 3 { return Key{}, fmt.Errorf("%s is not a valid logging config key", key) } loggingKey := strings.Join(fields[2:], ".") switch loggingKey { case "target.address": // gendoc:generate(entity=server, group=logging, key=logging.NAME.target.address) // Specify the protocol, name or IP and port. For example `tcp://syslog01.int.example.net:514`. // --- // type: string // scope: global // shortdesc: Address of the logger return Key{}, nil case "target.username": // gendoc:generate(entity=server, group=logging, key=logging.NAME.target.username) // // --- // type: string // scope: global // shortdesc: User name used for authentication return Key{}, nil case "target.password": // gendoc:generate(entity=server, group=logging, key=logging.NAME.target.password) // // --- // type: string // scope: global // shortdesc: Password used for authentication return Key{}, nil case "target.ca_cert": // gendoc:generate(entity=server, group=logging, key=logging.NAME.target.ca_cert) // // --- // type: string // scope: global // shortdesc: CA certificate for the server return Key{}, nil case "target.instance": // gendoc:generate(entity=server, group=logging, key=logging.NAME.target.instance) // This allows replacing the default instance value (server host name) by a more relevant value like a cluster identifier. // --- // type: string // scope: global // defaultdesc: Local server host name or cluster member name // shortdesc: Name to use as the instance field in Loki events. return Key{}, nil case "target.labels": // gendoc:generate(entity=server, group=logging, key=logging.NAME.target.labels) // Specify a comma-separated list of values that should be used as labels for a Loki log entry. // --- // type: string // scope: global // shortdesc: Labels for a Loki log entry return Key{}, nil case "target.facility": // gendoc:generate(entity=server, group=logging, key=logging.NAME.target.facility) // // --- // type: string // scope: global // shortdesc: The syslog facility defines the category of the log message return Key{Default: "daemon"}, nil case "target.type": // gendoc:generate(entity=server, group=logging, key=logging.NAME.target.type) // // --- // type: string // scope: global // shortdesc: The type of the logger. One of `loki`, `syslog` or `webhook`. return Key{Validator: validate.Optional(validate.IsListOf(validate.IsOneOf("syslog", "loki", "webhook")))}, nil case "target.retry": // gendoc:generate(entity=server, group=logging, key=logging.NAME.target.retry) // // --- // type: integer // scope: global // shortdesc: number of delivery retries, default 3 return Key{Validator: validate.Optional(), Default: "3"}, nil case "types": // gendoc:generate(entity=server, group=logging, key=logging.NAME.types) // Specify a comma-separated list of events to send to the logger. // The events can be any combination of `lifecycle`, `logging`, and `network-acl`. // --- // type: string // scope: global // defaultdesc: `lifecycle,logging` // shortdesc: Events to send to the logger return Key{Validator: validate.Optional(validate.IsListOf(validate.IsOneOf("lifecycle", "logging", "network-acl"))), Default: "lifecycle,logging"}, nil case "logging.level": // gendoc:generate(entity=server, group=logging, key=logging.NAME.logging.level) // // --- // type: string // scope: global // defaultdesc: `info` // shortdesc: Minimum log level to send to the logger return Key{Validator: LogLevelValidator, Default: logrus.InfoLevel.String()}, nil case "lifecycle.types": // gendoc:generate(entity=server, group=logging, key=logging.NAME.lifecycle.types) // // --- // type: string // scope: global // shortdesc: E.g., `instance`, comma separate, empty means all return Key{Validator: validate.Optional(validate.IsAny)}, nil case "lifecycle.projects": // gendoc:generate(entity=server, group=logging, key=logging.NAME.lifecycle.projects) // // --- // type: string // scope: global // shortdesc: Comma separate list of projects, empty means all return Key{Validator: validate.Optional(validate.IsAny)}, nil } return Key{}, fmt.Errorf("%s is not a valid logging config key", key) } incus-7.0.0/internal/server/config/map.go000066400000000000000000000152451517523235500203510ustar00rootroot00000000000000package config import ( "errors" "fmt" "reflect" "sort" "strconv" "strings" "unicode" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/shared/util" ) // Map is a structured map of config keys to config values. // // Each legal key is declared in a config Schema using a Key object. type Map struct { schema Schema values map[string]string // Key/value pairs stored in the map. } // Load creates a new configuration Map with the given schema and initial // values. It is meant to be called with a set of initial values that were set // at a previous time and persisted to some storage like a database. // // If one or more keys fail to be loaded, return an ErrorList describing what // went wrong. Non-failing keys are still loaded in the returned Map. func Load(schema Schema, values map[string]string) (Map, error) { m := Map{ schema: schema, } // Populate the initial values. _, err := m.update(values) return m, err } // Change the values of this configuration Map. // // Return a map of key/value pairs that were actually changed. If some keys // fail to apply, details are included in the returned ErrorList. func (m *Map) Change(changes map[string]string) (map[string]string, error) { values := make(map[string]string, len(m.schema)) errList := &ErrorList{} for name, change := range changes { // Ensure that we were actually passed a string. s := reflect.ValueOf(change) if s.Kind() != reflect.String { errList.add(name, nil, fmt.Sprintf("invalid type %s", s.Kind())) continue } values[name] = change } if errList.Len() > 0 { return nil, errList } // Any key not explicitly set, is considered unset. for name, key := range m.schema { _, ok := values[name] if !ok { values[name] = key.Default } } names, err := m.update(values) changed := map[string]string{} for _, name := range names { changed[name] = m.GetRaw(name) } return changed, err } // Dump the current configuration held by this Map. // // Keys that match their default value will not be included in the dump. func (m *Map) Dump() map[string]string { values := map[string]string{} for name, value := range m.values { key, ok := m.schema[name] if ok { // Schema key value := m.GetRaw(name) if value != key.Default { values[name] = value } } else if internalInstance.IsUserConfig(name) { // User key, just include it as is values[name] = value } } return values } // GetRaw returns the value of the given key, which must be of type String. func (m *Map) GetRaw(name string) string { value, ok := m.values[name] // User key? if internalInstance.IsUserConfig(name) { return value } if IsLoggingConfig(name) { if !ok { key, err := GetLoggingRuleForKey(name) if err != nil { panic(err) } value = key.Default } return value } // Schema key key := m.schema.mustGetKey(name) if !ok { value = key.Default } return value } // GetString returns the value of the given key, which must be of type String. func (m *Map) GetString(name string) string { if !internalInstance.IsUserConfig(name) && !IsLoggingConfig(name) { m.schema.assertKeyType(name, String) } return m.GetRaw(name) } // GetBool returns the value of the given key, which must be of type Bool. func (m *Map) GetBool(name string) bool { m.schema.assertKeyType(name, Bool) return util.IsTrue(m.GetRaw(name)) } // GetInt64 returns the value of the given key, which must be of type Int64. func (m *Map) GetInt64(name string) int64 { if !IsLoggingConfig(name) { m.schema.assertKeyType(name, Int64) } n, err := strconv.ParseInt(m.GetRaw(name), 10, 64) if err != nil { panic(fmt.Sprintf("cannot convert to int64: %v", err)) } return n } // Update the current values in the map using the newly provided ones. Return a // list of key names that were actually changed and an ErrorList with possible // errors. func (m *Map) update(values map[string]string) ([]string, error) { // Detect if this is the first time we're setting values. This happens // when Load is called. initial := m.values == nil if initial { m.values = make(map[string]string, len(values)) } // Update our keys with the values from the given map, and keep track // of which keys actually changed their value. errList := &ErrorList{} names := []string{} for name, value := range values { changed, err := m.set(name, value, initial) if err != nil { errList.add(name, value, err.Error()) continue } if changed { names = append(names, name) } } sort.Strings(names) var err error if errList.Len() > 0 { errList.sort() err = errList } return names, err } // Set or change an individual key. Empty string means delete this value and // effectively revert it to the default. Return a boolean indicating whether // the value has changed, and error if something went wrong. func (m *Map) set(name string, value string, initial bool) (bool, error) { // Bypass schema for user.* keys if internalInstance.IsUserConfig(name) { for _, r := range strings.TrimPrefix(name, "user.") { // Only allow letters, digits, and punctuation characters. if !unicode.In(r, unicode.Letter, unicode.Digit, unicode.Punct) { return false, errors.New("Invalid key name") } } current, ok := m.values[name] if ok && value == current { // Value is unchanged return false, nil } if value == "" { delete(m.values, name) } else { m.values[name] = value } return true, nil } if IsLoggingConfig(name) { rule, err := GetLoggingRuleForKey(name) if err != nil { return false, err } m.schema[name] = rule } key, ok := m.schema[name] if !ok { return false, errors.New("unknown key") } // When unsetting a config key, the value argument will be empty. // This ensures that the default value is set if the provided value is empty. if value == "" { value = key.Default } err := key.validate(value) if err != nil { return false, err } // Normalize boolan values, so the comparison below works fine. current := m.GetRaw(name) if key.Type == Bool { value = normalizeBool(value) current = normalizeBool(current) } // Compare the new value with the current one, and return now if they // are equal. if value == current { return false, nil } // Trigger the Setter if this is not an initial load and the key's // schema has declared it. if !initial && key.Setter != nil { value, err = key.Setter(value) if err != nil { return false, err } } if value == "" { delete(m.values, name) } else { m.values[name] = value } return true, nil } // Normalize a boolean value, converting it to the string "true" or "false". func normalizeBool(value string) string { if util.IsTrue(value) { return "true" } return "false" } incus-7.0.0/internal/server/config/map_test.go000066400000000000000000000164071517523235500214110ustar00rootroot00000000000000package config_test import ( "errors" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/config" ) // Loading a config Map initializes it with the given values. func TestLoad(t *testing.T) { schema := config.Schema{ "foo": {}, "bar": {Setter: failingSetter}, "egg": {Type: config.Bool}, } cases := []struct { title string values map[string]string // Initial values result map[string]string // Expected values after loading }{ { `plain load of regular key`, map[string]string{"foo": "hello"}, map[string]string{"foo": "hello"}, }, { `key setter is ignored upon loading`, map[string]string{"bar": "hello"}, map[string]string{"bar": "hello"}, }, { `bool true values are normalized`, map[string]string{"egg": "yes"}, map[string]string{"egg": "true"}, }, { `multiple values are all loaded`, map[string]string{"foo": "x", "bar": "yuk", "egg": "1"}, map[string]string{"foo": "x", "bar": "yuk", "egg": "true"}, }, } for _, c := range cases { t.Run(c.title, func(t *testing.T) { m, err := config.Load(schema, c.values) require.NoError(t, err) for name, value := range c.result { assert.Equal(t, value, m.GetRaw(name)) } }) } } // If some keys fail to load, an ErrorList with the offending issues is // returned. func TestLoad_Error(t *testing.T) { cases := []struct { title string schema config.Schema // Test schema to use values map[string]string // Initial values message string // Expected error message }{ { `schema has no key with the given name`, config.Schema{}, map[string]string{"bar": ""}, "cannot set 'bar' to '': unknown key", }, { `validation fails`, config.Schema{"foo": {Type: config.Bool}}, map[string]string{"foo": "yyy"}, "cannot set 'foo' to 'yyy': invalid boolean", }, { `only the first of multiple errors is shown (in key name order)`, config.Schema{"foo": {Type: config.Bool}}, map[string]string{"foo": "yyy", "bar": ""}, "cannot set 'bar' to '': unknown key (and 1 more errors)", }, } for _, c := range cases { t.Run(c.title, func(t *testing.T) { _, err := config.Load(c.schema, c.values) assert.EqualError(t, err, c.message) }) } } // Changing a config Map mutates the initial values. func TestChange(t *testing.T) { schema := config.Schema{ "foo": {}, "bar": {Setter: upperCase}, "egg": {Type: config.Bool}, "yuk": {Type: config.Bool, Default: "true"}, } values := map[string]string{ // Initial values "foo": "hello", "bar": "x", } cases := []struct { title string values map[string]string // New values result map[string]string // Expected values after change }{ { `plain change of regular key`, map[string]string{"foo": "world"}, map[string]string{"foo": "world"}, }, { `key setter is honored`, map[string]string{"bar": "y"}, map[string]string{"bar": "Y"}, }, { `bool true values are normalized`, map[string]string{"egg": "yes"}, map[string]string{"egg": "true"}, }, { `bool false values are normalized`, map[string]string{"yuk": "0"}, map[string]string{"yuk": "false"}, }, { `multiple values are all mutated`, map[string]string{"foo": "x", "bar": "hey", "egg": "0"}, map[string]string{"foo": "x", "bar": "HEY", "egg": ""}, }, } for _, c := range cases { t.Run(c.title, func(t *testing.T) { m, err := config.Load(schema, values) require.NoError(t, err) _, err = m.Change(c.values) require.NoError(t, err) for name, value := range c.result { assert.Equal(t, value, m.GetRaw(name)) } }) } } // A map of changed key/value pairs is returned. func TestMap_ChangeReturnsChangedKeys(t *testing.T) { schema := config.Schema{ "foo": {Type: config.Bool}, "bar": {Default: "egg"}, } values := map[string]string{"foo": "true"} // Initial values cases := []struct { title string changes map[string]string // New values changed map[string]string // Keys that should have actually changed }{ { `plain single change`, map[string]string{"foo": "no"}, map[string]string{"foo": "false"}, }, { `unchanged boolean value, even if it's spelled 'yes' and not 'true'`, map[string]string{"foo": "yes"}, map[string]string{}, }, { `unset value`, map[string]string{"foo": ""}, map[string]string{"foo": "false"}, }, { `unchanged value, since it matches the default`, map[string]string{"foo": "true", "bar": "egg"}, map[string]string{}, }, { `multiple changes`, map[string]string{"foo": "false", "bar": "baz"}, map[string]string{"foo": "false", "bar": "baz"}, }, } for _, c := range cases { t.Run(c.title, func(t *testing.T) { m, err := config.Load(schema, values) assert.NoError(t, err) changed, err := m.Change(c.changes) require.NoError(t, err) assert.Equal(t, c.changed, changed) }) } } // If some keys fail to load, an ErrorList with the offending issues is // returned. func TestMap_ChangeError(t *testing.T) { schema := config.Schema{ "foo": {Type: config.Bool}, "egg": {Setter: failingSetter}, } cases := []struct { title string changes map[string]string message string }{ { `schema has no key with the given name`, map[string]string{"xxx": ""}, "cannot set 'xxx' to '': unknown key", }, { `validation fails`, map[string]string{"foo": "yyy"}, "cannot set 'foo' to 'yyy': invalid boolean", }, { `custom setter fails`, map[string]string{"egg": "xxx"}, "cannot set 'egg' to 'xxx': boom", }, } for _, c := range cases { t.Run(c.message, func(t *testing.T) { m, err := config.Load(schema, nil) assert.NoError(t, err) _, err = m.Change(c.changes) assert.EqualError(t, err, c.message) }) } } // A Map dump contains only values that differ from their default. func TestMap_Dump(t *testing.T) { schema := config.Schema{ "foo": {}, "bar": {Default: "x"}, } values := map[string]string{ "foo": "hello", "bar": "x", } m, err := config.Load(schema, values) assert.NoError(t, err) dump := map[string]string{ "foo": "hello", } assert.Equal(t, dump, m.Dump()) } // The various GetXXX methods return typed values. func TestMap_Getters(t *testing.T) { schema := config.Schema{ "foo": {}, "bar": {Type: config.Bool}, "egg": {Type: config.Int64}, } values := map[string]string{ "foo": "hello", "bar": "true", "egg": "123", } m, err := config.Load(schema, values) assert.NoError(t, err) assert.Equal(t, "hello", m.GetString("foo")) assert.Equal(t, true, m.GetBool("bar")) assert.Equal(t, int64(123), m.GetInt64("egg")) } // The various GetXXX methods panic if they are used with the wrong key name or // type. func TestMap_GettersPanic(t *testing.T) { schema := config.Schema{ "foo": {}, "bar": {Type: config.Bool}, } m, err := config.Load(schema, nil) assert.NoError(t, err) assert.Panics(t, func() { m.GetRaw("egg") }) assert.Panics(t, func() { m.GetString("bar") }) assert.Panics(t, func() { m.GetBool("foo") }) assert.Panics(t, func() { m.GetInt64("foo") }) } // A Key setter that always fail. func failingSetter(string) (string, error) { return "", errors.New("boom") } // A Key setter that uppercases the value. func upperCase(v string) (string, error) { return strings.ToUpper(v), nil } incus-7.0.0/internal/server/config/safe.go000066400000000000000000000012041517523235500205000ustar00rootroot00000000000000package config import ( "errors" "fmt" "github.com/lxc/incus/v7/shared/logger" ) // SafeLoad is a wrapper around Load() that does not error when invalid keys // are found, and just logs warnings instead. Other kinds of errors are still // returned. func SafeLoad(schema Schema, values map[string]string) (Map, error) { m, err := Load(schema, values) if err != nil { var errs *ErrorList ok := errors.As(err, &errs) if !ok { return m, err } for _, e := range errs.errors { message := fmt.Sprintf("Invalid configuration key: %s", e.reason) logger.Error(message, logger.Ctx{"key": e.configKey}) } } return m, nil } incus-7.0.0/internal/server/config/safe_test.go000066400000000000000000000010111517523235500215330ustar00rootroot00000000000000package config_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/config" ) // If the given values contain invalid keys, they are ignored. func TestSafeLoad_IgnoreInvalidKeys(t *testing.T) { schema := config.Schema{"bar": {}} values := map[string]string{ "foo": "garbage", "bar": "x", } m, err := config.SafeLoad(schema, values) require.NoError(t, err) assert.Equal(t, map[string]string{"bar": "x"}, m.Dump()) } incus-7.0.0/internal/server/config/schema.go000066400000000000000000000060661517523235500210350ustar00rootroot00000000000000package config import ( "errors" "fmt" "slices" "sort" "strconv" "strings" ) // Schema defines the available keys of a config Map, along with the types // and options for their values, expressed using Key objects. type Schema map[string]Key // Keys returns all keys defined in the schema. func (s Schema) Keys() []string { keys := make([]string, len(s)) i := 0 for key := range s { keys[i] = key i++ } sort.Strings(keys) return keys } // Defaults returns a map of all key names in the schema along with their default // values. func (s Schema) Defaults() map[string]string { values := make(map[string]string, len(s)) for name, key := range s { values[name] = key.Default } return values } // Get the Key associated with the given name, or panic. func (s Schema) mustGetKey(name string) Key { key, ok := s[name] if !ok { panic(fmt.Sprintf("attempt to access unknown key '%s'", name)) } return key } // Assert that the Key with the given name as the given type. Panic if no Key // with such name exists, or if it does not match the tiven type. func (s Schema) assertKeyType(name string, code Type) { key := s.mustGetKey(name) if key.Type != code { panic(fmt.Sprintf("key '%s' has type code %d, not %d", name, key.Type, code)) } } // Key defines the type of the value of a particular config key, along with // other knobs such as default, validator, etc. type Key struct { Type Type // Type of the value. It defaults to String. Default string // If the key is not set in a Map, use this value instead. Deprecated string // Optional message to set if this config value is deprecated. // Optional function used to validate the values. It's called by Map // all the times the value associated with this Key is going to be // changed. Validator func(string) error // Optional function to manipulate a value before it's actually saved // in a Map. It's called only by Map.Change(), and not by Load() since // values passed to Load() are supposed to have been previously // processed. Setter func(string) (string, error) } // Type is a numeric code indetifying a node value type. type Type int // Possible Value types. const ( String Type = iota Bool Int64 ) // Tells if the given value can be assigned to this particular Value instance. func (v *Key) validate(value string) error { validator := v.Validator if validator == nil { // Dummy validator validator = func(string) error { return nil } } // Handle unsetting if value == "" { return validator(v.Default) } switch v.Type { case String: case Bool: if !slices.Contains(booleans, strings.ToLower(value)) { return errors.New("invalid boolean") } case Int64: _, err := strconv.ParseInt(value, 10, 64) if err != nil { return errors.New("invalid integer") } default: panic(fmt.Sprintf("unexpected value type: %d", v.Type)) } if v.Deprecated != "" && value != v.Default { return fmt.Errorf("deprecated: %s", v.Deprecated) } // Run external validation function return validator(value) } var booleans = []string{"true", "false", "1", "0", "yes", "no", "on", "off"} incus-7.0.0/internal/server/config/schema_internal_test.go000066400000000000000000000031621517523235500237620ustar00rootroot00000000000000package config import ( "errors" "testing" "github.com/stretchr/testify/assert" ) // Exercise valid values. func TestKey_validate(t *testing.T) { for _, c := range validateCases { t.Run(c.value, func(t *testing.T) { assert.NoError(t, c.node.validate(c.value)) }) } } // Test cases for TestKey_validate. var validateCases = []struct { node Key value string }{ {Key{}, "hello"}, {Key{Type: Bool}, "yes"}, {Key{Type: Bool}, "0"}, {Key{Type: Int64}, "666"}, {Key{Type: Int64}, "666"}, {Key{Type: Bool}, ""}, {Key{Validator: isNotEmptyString, Default: "foo"}, ""}, } // Validator that returns an error if the value is not the empty string. func isNotEmptyString(value string) error { if value == "" { return errors.New("empty value not valid") } return nil } // Exercise all possible validation errors. func TestKey_validateError(t *testing.T) { for _, c := range validateErrorCases { t.Run(c.message, func(t *testing.T) { err := c.node.validate(c.value) assert.EqualError(t, err, c.message) }) } } // Test cases for TestKey_validateError. var validateErrorCases = []struct { node Key value string message string }{ {Key{Type: Int64}, "1.2", "invalid integer"}, {Key{Type: Bool}, "yyy", "invalid boolean"}, {Key{Validator: func(string) error { return errors.New("ugh") }}, "", "ugh"}, {Key{Deprecated: "don't use this"}, "foo", "deprecated: don't use this"}, } // If a value has an expected kind code, a panic is thrown. func TestKey_UnexpectedKind(t *testing.T) { value := Key{Type: 999} f := func() { _ = value.validate("foo") } assert.PanicsWithValue(t, "unexpected value type: 999", f) } incus-7.0.0/internal/server/config/schema_test.go000066400000000000000000000010071517523235500220620ustar00rootroot00000000000000package config_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/lxc/incus/v7/internal/server/config" ) func TestSchema_Defaults(t *testing.T) { schema := config.Schema{ "foo": {}, "bar": {Default: "x"}, } values := map[string]string{"foo": "", "bar": "x"} assert.Equal(t, values, schema.Defaults()) } func TestSchema_Keys(t *testing.T) { schema := config.Schema{ "foo": {}, "bar": {Default: "x"}, } keys := []string{"bar", "foo"} assert.Equal(t, keys, schema.Keys()) } incus-7.0.0/internal/server/config/validators.go000066400000000000000000000010421517523235500217320ustar00rootroot00000000000000package config import ( "os/exec" "github.com/sirupsen/logrus" ) // AvailableExecutable checks that the given value is the name of an executable // file, in PATH. func AvailableExecutable(value string) error { if value == "none" { return nil } _, err := exec.LookPath(value) return err } // LogLevelValidator checks whether the provided value is a valid logging level. func LogLevelValidator(value string) error { if value == "" { return nil } _, err := logrus.ParseLevel(value) if err != nil { return err } return nil } incus-7.0.0/internal/server/config/validators_test.go000066400000000000000000000005441517523235500227770ustar00rootroot00000000000000package config_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/lxc/incus/v7/internal/server/config" ) func TestAvailableExecutable(t *testing.T) { assert.NoError(t, config.AvailableExecutable("ls")) assert.NoError(t, config.AvailableExecutable("none")) assert.Error(t, config.AvailableExecutable("somenonexistingbin")) } incus-7.0.0/internal/server/daemon/000077500000000000000000000000001517523235500172345ustar00rootroot00000000000000incus-7.0.0/internal/server/daemon/daemon.go000066400000000000000000000004051517523235500210250ustar00rootroot00000000000000package daemon // Debug indicates if daemon is started with debug mode. var Debug bool // Verbose indicates if daemon is started with verbose mode. var Verbose bool // SharedMountsSetup indicates if daemon has setup shared mounts. var SharedMountsSetup bool incus-7.0.0/internal/server/db/000077500000000000000000000000001517523235500163565ustar00rootroot00000000000000incus-7.0.0/internal/server/db/backups.go000066400000000000000000000534721517523235500203500ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "database/sql" "errors" "fmt" "net/http" "time" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) // InstanceBackup is a value object holding all db-related details about an instance backup. type InstanceBackup struct { ID int InstanceID int Name string CreationDate time.Time ExpiryDate time.Time InstanceOnly bool RootOnly bool OptimizedStorage bool CompressionAlgorithm string } // StoragePoolVolumeBackup is a value object holding all db-related details about a storage volume backup. type StoragePoolVolumeBackup struct { ID int VolumeID int64 Name string CreationDate time.Time ExpiryDate time.Time VolumeOnly bool OptimizedStorage bool CompressionAlgorithm string } // StoragePoolBucketBackup is a value object holding all db-related details about a storage bucket backup. type StoragePoolBucketBackup struct { ID int BucketID int64 Name string CreationDate time.Time ExpiryDate time.Time CompressionAlgorithm string } // Returns the ID of the instance backup with the given name. func (c *ClusterTx) getInstanceBackupID(ctx context.Context, name string) (int, error) { q := "SELECT id FROM instances_backups WHERE name=?" id := -1 arg1 := []any{name} arg2 := []any{&id} err := dbQueryRowScan(ctx, c, q, arg1, arg2) if errors.Is(err, sql.ErrNoRows) { return -1, api.StatusErrorf(http.StatusNotFound, "Instance backup not found") } return id, err } // GetInstanceBackup returns the backup with the given name. func (c *ClusterTx) GetInstanceBackup(ctx context.Context, projectName string, name string) (InstanceBackup, error) { args := InstanceBackup{} args.Name = name instanceOnlyInt := -1 rootOnlyInt := -1 optimizedStorageInt := -1 q := ` SELECT instances_backups.id, instances_backups.instance_id, instances_backups.creation_date, instances_backups.expiry_date, instances_backups.container_only, instances_backups.root_only, instances_backups.optimized_storage FROM instances_backups JOIN instances ON instances.id=instances_backups.instance_id JOIN projects ON projects.id=instances.project_id WHERE projects.name=? AND instances_backups.name=? ` arg1 := []any{projectName, name} arg2 := []any{ &args.ID, &args.InstanceID, &args.CreationDate, &args.ExpiryDate, &instanceOnlyInt, &rootOnlyInt, &optimizedStorageInt, } err := dbQueryRowScan(ctx, c, q, arg1, arg2) if err != nil { if errors.Is(err, sql.ErrNoRows) { return args, api.StatusErrorf(http.StatusNotFound, "Instance backup not found") } return args, err } if instanceOnlyInt == 1 { args.InstanceOnly = true } if rootOnlyInt == 1 { args.RootOnly = true } if optimizedStorageInt == 1 { args.OptimizedStorage = true } return args, nil } // GetInstanceBackupWithID returns the backup with the given ID. func (c *ClusterTx) GetInstanceBackupWithID(ctx context.Context, backupID int) (InstanceBackup, error) { args := InstanceBackup{} args.ID = backupID instanceOnlyInt := -1 rootOnlyInt := -1 optimizedStorageInt := -1 q := ` SELECT instances_backups.name, instances_backups.instance_id, instances_backups.creation_date, instances_backups.expiry_date, instances_backups.container_only, instances_backups.root_only, instances_backups.optimized_storage FROM instances_backups JOIN instances ON instances.id=instances_backups.instance_id JOIN projects ON projects.id=instances.project_id WHERE instances_backups.id=? ` arg1 := []any{backupID} arg2 := []any{ &args.Name, &args.InstanceID, &args.CreationDate, &args.ExpiryDate, &instanceOnlyInt, &rootOnlyInt, &optimizedStorageInt, } err := dbQueryRowScan(ctx, c, q, arg1, arg2) if err != nil { if errors.Is(err, sql.ErrNoRows) { return args, api.StatusErrorf(http.StatusNotFound, "Instance backup not found") } return args, err } if instanceOnlyInt == 1 { args.InstanceOnly = true } if rootOnlyInt == 1 { args.RootOnly = true } if optimizedStorageInt == 1 { args.OptimizedStorage = true } return args, nil } // GetInstanceBackups returns the names of all backups of the instance with the // given name. func (c *ClusterTx) GetInstanceBackups(ctx context.Context, projectName string, name string) ([]string, error) { var result []string q := `SELECT instances_backups.name FROM instances_backups JOIN instances ON instances_backups.instance_id=instances.id JOIN projects ON projects.id=instances.project_id WHERE projects.name=? AND instances.name=?` inargs := []any{projectName, name} outfmt := []any{name} dbResults, err := queryScan(ctx, c, q, inargs, outfmt) if err != nil { return nil, err } for _, r := range dbResults { result = append(result, r[0].(string)) } return result, nil } // CreateInstanceBackup creates a new backup. func (c *ClusterTx) CreateInstanceBackup(ctx context.Context, args InstanceBackup) error { _, err := c.getInstanceBackupID(ctx, args.Name) if err == nil { return ErrAlreadyDefined } instanceOnlyInt := 0 if args.InstanceOnly { instanceOnlyInt = 1 } rootOnlyInt := 0 if args.RootOnly { rootOnlyInt = 1 } optimizedStorageInt := 0 if args.OptimizedStorage { optimizedStorageInt = 1 } str := "INSERT INTO instances_backups (instance_id, name, creation_date, expiry_date, container_only, root_only, optimized_storage) VALUES (?, ?, ?, ?, ?, ?, ?)" stmt, err := c.tx.Prepare(str) if err != nil { return err } defer func() { _ = stmt.Close() }() result, err := stmt.Exec(args.InstanceID, args.Name, args.CreationDate.Unix(), args.ExpiryDate.Unix(), instanceOnlyInt, rootOnlyInt, optimizedStorageInt) if err != nil { return err } _, err = result.LastInsertId() if err != nil { return fmt.Errorf("Error inserting %q into database", args.Name) } return nil } // DeleteInstanceBackup removes the instance backup with the given name from the database. func (c *ClusterTx) DeleteInstanceBackup(ctx context.Context, name string) error { id, err := c.getInstanceBackupID(ctx, name) if err != nil { return err } _, err = c.tx.ExecContext(ctx, "DELETE FROM instances_backups WHERE id=?", id) if err != nil { return err } return nil } // RenameInstanceBackup renames an instance backup from the given current name // to the new one. func (c *ClusterTx) RenameInstanceBackup(ctx context.Context, oldName, newName string) error { str := "UPDATE instances_backups SET name = ? WHERE name = ?" stmt, err := c.tx.PrepareContext(ctx, str) if err != nil { return err } defer func() { _ = stmt.Close() }() logger.Debug( "Calling SQL Query", logger.Ctx{ "query": "UPDATE instances_backups SET name = ? WHERE name = ?", "oldName": oldName, "newName": newName, }) _, err = stmt.ExecContext(ctx, newName, oldName) if err != nil { return err } return nil } // GetExpiredInstanceBackups returns a list of expired instance backups. func (c *ClusterTx) GetExpiredInstanceBackups(ctx context.Context) ([]InstanceBackup, error) { var result []InstanceBackup var name string var expiryDate string var instanceID int q := `SELECT instances_backups.name, instances_backups.expiry_date, instances_backups.instance_id FROM instances_backups` outfmt := []any{name, expiryDate, instanceID} dbResults, err := queryScan(ctx, c, q, nil, outfmt) if err != nil { return nil, err } for _, r := range dbResults { timestamp := r[1] var backupExpiry time.Time err = backupExpiry.UnmarshalText([]byte(timestamp.(string))) if err != nil { return []InstanceBackup{}, err } // Since zero time causes some issues due to timezones, we check the // unix timestamp instead of IsZero(). if backupExpiry.Unix() <= 0 { // Backup doesn't expire continue } // Backup has expired if time.Now().Unix()-backupExpiry.Unix() >= 0 { result = append(result, InstanceBackup{ Name: r[0].(string), InstanceID: r[2].(int), ExpiryDate: backupExpiry, }) } } return result, nil } // GetExpiredStorageVolumeBackups returns a list of expired storage volume backups. func (c *ClusterTx) GetExpiredStorageVolumeBackups(ctx context.Context) ([]StoragePoolVolumeBackup, error) { var backups []StoragePoolVolumeBackup q := `SELECT storage_volumes_backups.name, storage_volumes_backups.expiry_date, storage_volumes_backups.storage_volume_id FROM storage_volumes_backups` err := query.Scan(ctx, c.Tx(), q, func(scan func(dest ...any) error) error { var b StoragePoolVolumeBackup var expiryTime sql.NullTime err := scan(&b.Name, &expiryTime, &b.VolumeID) if err != nil { return err } b.ExpiryDate = expiryTime.Time // Convert nulls to zero. // Since zero time causes some issues due to timezones, we check the // unix timestamp instead of IsZero(). if b.ExpiryDate.Unix() <= 0 { // Backup doesn't expire return nil } // Backup has expired if time.Now().Unix()-b.ExpiryDate.Unix() >= 0 { backups = append(backups, b) } return nil }) if err != nil { return nil, err } return backups, nil } // GetStoragePoolVolumeBackups returns a list of volume backups. func (c *ClusterTx) GetStoragePoolVolumeBackups(ctx context.Context, projectName string, volumeName string, poolID int64) ([]StoragePoolVolumeBackup, error) { q := ` SELECT backups.id, backups.storage_volume_id, backups.name, backups.creation_date, backups.expiry_date, backups.volume_only, backups.optimized_storage FROM storage_volumes_backups AS backups JOIN storage_volumes ON storage_volumes.id=backups.storage_volume_id JOIN projects ON projects.id=storage_volumes.project_id WHERE projects.name=? AND storage_volumes.name=? AND storage_volumes.storage_pool_id=? ORDER BY backups.id ` var backups []StoragePoolVolumeBackup err := query.Scan(ctx, c.tx, q, func(scan func(dest ...any) error) error { var b StoragePoolVolumeBackup var expiryTime sql.NullTime err := scan(&b.ID, &b.VolumeID, &b.Name, &b.CreationDate, &expiryTime, &b.VolumeOnly, &b.OptimizedStorage) if err != nil { return err } b.ExpiryDate = expiryTime.Time // Convert nulls to zero. backups = append(backups, b) return nil }, projectName, volumeName, poolID) if err != nil { return nil, err } return backups, nil } // GetStoragePoolVolumeBackupsNames returns the names of all backups of the storage volume with the given name. func (c *ClusterTx) GetStoragePoolVolumeBackupsNames(ctx context.Context, projectName string, volumeName string, poolID int64) ([]string, error) { var result []string q := `SELECT storage_volumes_backups.name FROM storage_volumes_backups JOIN storage_volumes ON storage_volumes_backups.storage_volume_id=storage_volumes.id JOIN projects ON projects.id=storage_volumes.project_id WHERE projects.name=? AND storage_volumes.name=? ORDER BY storage_volumes_backups.id` inargs := []any{projectName, volumeName} outfmt := []any{volumeName} dbResults, err := queryScan(ctx, c, q, inargs, outfmt) if err != nil { return nil, err } for _, r := range dbResults { result = append(result, r[0].(string)) } return result, nil } // CreateStoragePoolVolumeBackup creates a new storage volume backup. func (c *ClusterTx) CreateStoragePoolVolumeBackup(ctx context.Context, args StoragePoolVolumeBackup) error { _, err := c.getStoragePoolVolumeBackupID(ctx, args.Name) if err == nil { return ErrAlreadyDefined } volumeOnlyInt := 0 if args.VolumeOnly { volumeOnlyInt = 1 } optimizedStorageInt := 0 if args.OptimizedStorage { optimizedStorageInt = 1 } str := "INSERT INTO storage_volumes_backups (storage_volume_id, name, creation_date, expiry_date, volume_only, optimized_storage) VALUES (?, ?, ?, ?, ?, ?)" stmt, err := c.tx.Prepare(str) if err != nil { return err } defer func() { _ = stmt.Close() }() result, err := stmt.Exec(args.VolumeID, args.Name, args.CreationDate.Unix(), args.ExpiryDate.Unix(), volumeOnlyInt, optimizedStorageInt) if err != nil { return err } _, err = result.LastInsertId() if err != nil { return fmt.Errorf("Error inserting %q into database", args.Name) } return nil } // Returns the ID of the storage volume backup with the given name. func (c *ClusterTx) getStoragePoolVolumeBackupID(ctx context.Context, name string) (int, error) { q := "SELECT id FROM storage_volumes_backups WHERE name=?" id := -1 arg1 := []any{name} arg2 := []any{&id} err := dbQueryRowScan(ctx, c, q, arg1, arg2) if errors.Is(err, sql.ErrNoRows) { return -1, api.StatusErrorf(http.StatusNotFound, "Storage volume backup not found") } return id, err } // DeleteStoragePoolVolumeBackup removes the storage volume backup with the given name from the database. func (c *ClusterTx) DeleteStoragePoolVolumeBackup(ctx context.Context, name string) error { id, err := c.getStoragePoolVolumeBackupID(ctx, name) if err != nil { return err } _, err = c.tx.ExecContext(ctx, "DELETE FROM storage_volumes_backups WHERE id=?", id) if err != nil { return err } return nil } // GetStoragePoolVolumeBackup returns the volume backup with the given name. func (c *ClusterTx) GetStoragePoolVolumeBackup(ctx context.Context, projectName string, poolName string, backupName string) (StoragePoolVolumeBackup, error) { args := StoragePoolVolumeBackup{} q := ` SELECT backups.id, backups.storage_volume_id, backups.name, backups.creation_date, backups.expiry_date, backups.volume_only, backups.optimized_storage FROM storage_volumes_backups AS backups JOIN storage_volumes ON storage_volumes.id=backups.storage_volume_id JOIN projects ON projects.id=storage_volumes.project_id WHERE projects.name=? AND backups.name=? ` arg1 := []any{projectName, backupName} outfmt := []any{&args.ID, &args.VolumeID, &args.Name, &args.CreationDate, &args.ExpiryDate, &args.VolumeOnly, &args.OptimizedStorage} err := dbQueryRowScan(ctx, c, q, arg1, outfmt) if err != nil { if errors.Is(err, sql.ErrNoRows) { return args, api.StatusErrorf(http.StatusNotFound, "Storage volume backup not found") } return args, err } return args, nil } // GetStoragePoolVolumeBackupWithID returns the volume backup with the given ID. func (c *ClusterTx) GetStoragePoolVolumeBackupWithID(ctx context.Context, backupID int) (StoragePoolVolumeBackup, error) { args := StoragePoolVolumeBackup{} q := ` SELECT backups.id, backups.storage_volume_id, backups.name, backups.creation_date, backups.expiry_date, backups.volume_only, backups.optimized_storage FROM storage_volumes_backups AS backups JOIN storage_volumes ON storage_volumes.id=backups.storage_volume_id JOIN projects ON projects.id=storage_volumes.project_id WHERE backups.id=? ` arg1 := []any{backupID} outfmt := []any{&args.ID, &args.VolumeID, &args.Name, &args.CreationDate, &args.ExpiryDate, &args.VolumeOnly, &args.OptimizedStorage} err := dbQueryRowScan(ctx, c, q, arg1, outfmt) if err != nil { if errors.Is(err, sql.ErrNoRows) { return args, api.StatusErrorf(http.StatusNotFound, "Storage volume backup not found") } return args, err } return args, nil } // RenameVolumeBackup renames a volume backup from the given current name // to the new one. func (c *ClusterTx) RenameVolumeBackup(ctx context.Context, oldName, newName string) error { str := "UPDATE storage_volumes_backups SET name = ? WHERE name = ?" stmt, err := c.tx.Prepare(str) if err != nil { return err } defer func() { _ = stmt.Close() }() logger.Debug( "Calling SQL Query", logger.Ctx{ "query": "UPDATE storage_volumes_backups SET name = ? WHERE name = ?", "oldName": oldName, "newName": newName, }) _, err = stmt.Exec(newName, oldName) if err != nil { return err } return nil } // GetStoragePoolBucketBackups returns a list of bucket backups. func (c *ClusterTx) GetStoragePoolBucketBackups(ctx context.Context, projectName string, bucketName string, poolID int64) ([]StoragePoolBucketBackup, error) { q := ` SELECT backups.id, backups.storage_bucket_id, backups.name, backups.creation_date, backups.expiry_date FROM storage_buckets_backups AS backups JOIN storage_buckets ON storage_buckets.id=backups.storage_bucket_id JOIN projects ON projects.id=storage_buckets.project_id WHERE projects.name=? AND storage_buckets.name=? AND storage_buckets.storage_pool_id=? ORDER BY backups.id ` var backups []StoragePoolBucketBackup err := query.Scan(ctx, c.tx, q, func(scan func(dest ...any) error) error { var b StoragePoolBucketBackup var expiryTime sql.NullTime err := scan(&b.ID, &b.BucketID, &b.Name, &b.CreationDate, &expiryTime) if err != nil { return err } b.ExpiryDate = expiryTime.Time // Convert nulls to zero. backups = append(backups, b) return nil }, projectName, bucketName, poolID) if err != nil { return nil, err } return backups, nil } // GetStoragePoolBucketBackupsName returns the names of all backups of the storage bucket with the given name. func (c *ClusterTx) GetStoragePoolBucketBackupsName(ctx context.Context, projectName string, bucketName string) ([]string, error) { var result []string q := `SELECT storage_buckets_backups.name FROM storage_buckets_backups JOIN storage_buckets ON storage_buckets_backups.storage_bucket_id=storage_buckets.id JOIN projects ON projects.id=storage_buckets.project_id WHERE projects.name=? AND storage_buckets.name=? ORDER BY storage_buckets_backups.id` inargs := []any{projectName, bucketName} outfmt := []any{bucketName} dbResults, err := queryScan(ctx, c, q, inargs, outfmt) if err != nil { return nil, err } for _, r := range dbResults { result = append(result, r[0].(string)) } return result, nil } // CreateStoragePoolBucketBackup creates a new storage bucket backup. func (c *ClusterTx) CreateStoragePoolBucketBackup(ctx context.Context, args StoragePoolBucketBackup) error { _, err := c.getStoragePoolBucketBackupID(ctx, args.Name) if err == nil { return ErrAlreadyDefined } str := "INSERT INTO storage_buckets_backups (storage_bucket_id, name, creation_date, expiry_date) VALUES (?, ?, ?, ?)" stmt, err := c.tx.Prepare(str) if err != nil { return err } defer func() { _ = stmt.Close() }() result, err := stmt.Exec(args.BucketID, args.Name, args.CreationDate.Unix(), args.ExpiryDate.Unix()) if err != nil { return err } _, err = result.LastInsertId() if err != nil { return fmt.Errorf("Error inserting %q into database", args.Name) } return nil } // Returns the ID of the storage bucket backup with the given name. func (c *ClusterTx) getStoragePoolBucketBackupID(ctx context.Context, name string) (int, error) { q := "SELECT id FROM storage_buckets_backups WHERE name=?" id := -1 arg1 := []any{name} arg2 := []any{&id} err := dbQueryRowScan(ctx, c, q, arg1, arg2) if errors.Is(err, sql.ErrNoRows) { return -1, api.StatusErrorf(http.StatusNotFound, "Storage volume backup not found") } return id, err } // DeleteStoragePoolBucketBackup removes the storage bucket backup with the given name from the database. func (c *ClusterTx) DeleteStoragePoolBucketBackup(ctx context.Context, name string) error { id, err := c.getStoragePoolBucketBackupID(ctx, name) if err != nil { return err } _, err = c.tx.ExecContext(ctx, "DELETE FROM storage_buckets_backups WHERE id=?", id) if err != nil { return err } return nil } // GetStoragePoolBucketBackup returns the bucket backup with the given name. func (c *ClusterTx) GetStoragePoolBucketBackup(ctx context.Context, projectName string, poolName string, backupName string) (StoragePoolBucketBackup, error) { args := StoragePoolBucketBackup{} q := ` SELECT backups.id, backups.storage_bucket_id, backups.name, backups.creation_date, backups.expiry_date FROM storage_buckets_backups AS backups JOIN storage_buckets ON storage_buckets.id=backups.storage_bucket_id JOIN projects ON projects.id=storage_buckets.project_id WHERE projects.name=? AND backups.name=? ` arg1 := []any{projectName, backupName} outfmt := []any{&args.ID, &args.BucketID, &args.Name, &args.CreationDate, &args.ExpiryDate} err := dbQueryRowScan(ctx, c, q, arg1, outfmt) if err != nil { if errors.Is(err, sql.ErrNoRows) { return args, api.StatusErrorf(http.StatusNotFound, "Storage bucket backup not found") } return args, err } return args, nil } // GetExpiredStorageBucketBackups returns a list of expired storage bucket backups. func (c *ClusterTx) GetExpiredStorageBucketBackups(ctx context.Context) ([]StoragePoolBucketBackup, error) { var backups []StoragePoolBucketBackup q := ` SELECT storage_buckets_backups.id, storage_buckets_backups.name, storage_buckets_backups.expiry_date, storage_buckets_backups.storage_bucket_id FROM storage_buckets_backups` err := query.Scan(ctx, c.Tx(), q, func(scan func(dest ...any) error) error { var b StoragePoolBucketBackup var expiryTime sql.NullTime err := scan(&b.ID, &b.Name, &expiryTime, &b.BucketID) if err != nil { return err } b.ExpiryDate = expiryTime.Time // Convert nulls to zero. // Since zero time causes some issues due to timezones, we check the // unix timestamp instead of IsZero(). if b.ExpiryDate.Unix() <= 0 { // Backup doesn't expire return nil } // Backup has expired if time.Now().Unix()-b.ExpiryDate.Unix() >= 0 { backups = append(backups, b) } return nil }) if err != nil { return nil, err } return backups, nil } // RenameBucketBackup renames a bucket backup from the given current name to the new one. func (c *ClusterTx) RenameBucketBackup(ctx context.Context, oldName, newName string) error { str := "UPDATE storage_buckets_backups SET name = ? WHERE name = ?" stmt, err := c.tx.Prepare(str) if err != nil { return err } defer func() { _ = stmt.Close() }() logger.Debug( "Calling SQL Query", logger.Ctx{ "query": "UPDATE storage_buckets_backups SET name = ? WHERE name = ?", "oldName": oldName, "newName": newName, }) _, err = stmt.Exec(newName, oldName) if err != nil { return err } return nil } incus-7.0.0/internal/server/db/certificates.go000066400000000000000000000044471517523235500213630ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "github.com/lxc/incus/v7/internal/server/certificate" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/query" ) // UpdateCertificate updates a certificate in the db. func (db *DB) UpdateCertificate(ctx context.Context, fingerprint string, cert cluster.Certificate, projectNames []string) error { err := db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error { id, err := cluster.GetCertificateID(ctx, tx.Tx(), fingerprint) if err != nil { return err } err = cluster.UpdateCertificate(ctx, tx.Tx(), fingerprint, cert) if err != nil { return err } return cluster.UpdateCertificateProjects(ctx, tx.Tx(), int(id), projectNames) }) return err } // GetCertificates returns all available local certificates. func (n *NodeTx) GetCertificates(ctx context.Context) ([]cluster.Certificate, error) { type cert struct { fingerprint string certType certificate.Type name string certificate string } sql := "SELECT fingerprint, type, name, certificate FROM certificates" dbCerts := []cert{} err := query.Scan(ctx, n.tx, sql, func(scan func(dest ...any) error) error { dbCert := cert{} err := scan(&dbCert.fingerprint, &dbCert.certType, &dbCert.name, &dbCert.certificate) if err != nil { return err } dbCerts = append(dbCerts, dbCert) return nil }) if err != nil { return nil, err } certs := make([]cluster.Certificate, 0, len(dbCerts)) for _, dbCert := range dbCerts { certs = append(certs, cluster.Certificate{ Fingerprint: dbCert.fingerprint, Type: dbCert.certType, Name: dbCert.name, Certificate: dbCert.certificate, }) } return certs, nil } // ReplaceCertificates removes all existing certificates from the local certificates table and replaces them with // the ones provided. func (n *NodeTx) ReplaceCertificates(certs []cluster.Certificate) error { _, err := n.tx.Exec("DELETE FROM certificates") if err != nil { return err } sql := "INSERT INTO certificates (fingerprint, type, name, certificate) VALUES(?,?,?,?)" for _, cert := range certs { _, err = n.tx.Exec(sql, cert.Fingerprint, cert.Type, cert.Name, cert.Certificate) if err != nil { return err } } return nil } incus-7.0.0/internal/server/db/certificates_test.go000066400000000000000000000012051517523235500224070ustar00rootroot00000000000000//go:build linux && cgo && !agent package db_test import ( "context" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" ) func TestGetCertificate(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() ctx := context.Background() _, err := cluster.CreateCertificate(ctx, tx.Tx(), cluster.Certificate{Fingerprint: "foobar"}) require.NoError(t, err) cert, err := cluster.GetCertificate(ctx, tx.Tx(), "foobar") require.NoError(t, err) assert.Equal(t, cert.Fingerprint, "foobar") } incus-7.0.0/internal/server/db/cluster.go000066400000000000000000000111221517523235500203630ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "errors" "fmt" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // ClusterGroupToAPI is a convenience to convert a ClusterGroup db struct into // an API cluster group struct. func ClusterGroupToAPI(clusterGroup *cluster.ClusterGroup, nodes []string) *api.ClusterGroup { c := &api.ClusterGroup{ ClusterGroupPut: api.ClusterGroupPut{ Description: clusterGroup.Description, Members: nodes, }, ClusterGroupPost: api.ClusterGroupPost{ Name: clusterGroup.Name, }, } return c } // GetClusterGroupNodes returns a list of nodes of the given cluster group. func (c *ClusterTx) GetClusterGroupNodes(ctx context.Context, groupName string) ([]string, error) { q := `SELECT nodes.name FROM nodes_cluster_groups JOIN nodes ON nodes.id = nodes_cluster_groups.node_id JOIN cluster_groups ON cluster_groups.id = nodes_cluster_groups.group_id WHERE cluster_groups.name = ?` return query.SelectStrings(ctx, c.tx, q, groupName) } // GetClusterGroupURIs returns all available ClusterGroup URIs. // generator: ClusterGroup URIs func (c *ClusterTx) GetClusterGroupURIs(ctx context.Context, filter cluster.ClusterGroupFilter) ([]string, error) { var args []any var sql string if filter.Name != nil && filter.ID == nil { sql = `SELECT cluster_groups.name FROM cluster_groups WHERE cluster_groups.name = ? ORDER BY cluster_groups.name ` args = []any{ filter.Name, } } else if filter.ID == nil && filter.Name == nil { sql = `SELECT cluster_groups.name FROM cluster_groups ORDER BY cluster_groups.name` args = []any{} } else { return nil, errors.New("No statement exists for the given Filter") } names, err := query.SelectStrings(ctx, c.tx, sql, args...) if err != nil { return nil, err } uris := make([]string, len(names)) for i, name := range names { uris[i] = api.NewURL().Path(version.APIVersion, "cluster", "groups", name).String() } return uris, nil } // AddNodeToClusterGroup adds a given node to the given cluster group. func (c *ClusterTx) AddNodeToClusterGroup(ctx context.Context, groupName string, nodeName string) error { groupID, err := cluster.GetClusterGroupID(ctx, c.tx, groupName) if err != nil { return fmt.Errorf("Failed to get cluster group ID: %w", err) } nodeInfo, err := c.GetNodeByName(ctx, nodeName) if err != nil { return fmt.Errorf("Failed to get node info: %w", err) } _, err = c.tx.Exec(`INSERT INTO nodes_cluster_groups (node_id, group_id) VALUES(?, ?)`, nodeInfo.ID, groupID) if err != nil { return err } return nil } // RemoveNodeFromClusterGroup removes a given node from the given group name. func (c *ClusterTx) RemoveNodeFromClusterGroup(ctx context.Context, groupName string, nodeName string) error { groupID, err := cluster.GetClusterGroupID(ctx, c.tx, groupName) if err != nil { return fmt.Errorf("Failed to get cluster group ID: %w", err) } nodeInfo, err := c.GetNodeByName(ctx, nodeName) if err != nil { return fmt.Errorf("Failed to get node info: %w", err) } _, err = c.tx.Exec(`DELETE FROM nodes_cluster_groups WHERE node_id = ? AND group_id = ?`, nodeInfo.ID, groupID) if err != nil { return err } return nil } // GetClusterGroupsWithNode returns a list of cluster group names the given node belongs to. func (c *ClusterTx) GetClusterGroupsWithNode(ctx context.Context, nodeName string) ([]string, error) { q := `SELECT cluster_groups.name FROM nodes_cluster_groups JOIN cluster_groups ON cluster_groups.id = nodes_cluster_groups.group_id JOIN nodes ON nodes.id = nodes_cluster_groups.node_id WHERE nodes.name = ?` return query.SelectStrings(ctx, c.tx, q, nodeName) } // GetClusterGroupMemberInstances retrieves instances hosted on this member that are part of the specified cluster group. func (c *ClusterTx) GetClusterGroupMemberInstances(ctx context.Context, clusterGroup *cluster.ClusterGroup, clusterGroupMember string) ([]cluster.Instance, error) { filteredInstances := []cluster.Instance{} instances, err := cluster.GetInstances(ctx, c.Tx(), cluster.InstanceFilter{Node: &clusterGroupMember}) if err != nil { return nil, err } for _, instance := range instances { config, err := cluster.GetInstanceConfig(ctx, c.Tx(), instance.ID) if err != nil { return nil, err } // Check if the instance is assigned to the specified cluster group. group := config["volatile.cluster.group"] if group != clusterGroup.Name { continue } filteredInstances = append(filteredInstances, instance) } return filteredInstances, nil } incus-7.0.0/internal/server/db/cluster/000077500000000000000000000000001517523235500200375ustar00rootroot00000000000000incus-7.0.0/internal/server/db/cluster/certificate_projects.go000066400000000000000000000023241517523235500245620ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster // Code generation directives. // //generate-database:mapper target certificate_projects.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e certificate_project objects //generate-database:mapper stmt -e certificate_project objects-by-CertificateID //generate-database:mapper stmt -e certificate_project create struct=CertificateProject //generate-database:mapper stmt -e certificate_project delete-by-CertificateID // //generate-database:mapper method -i -e certificate_project GetMany struct=Certificate //generate-database:mapper method -i -e certificate_project DeleteMany struct=Certificate //generate-database:mapper method -i -e certificate_project Create struct=Certificate //generate-database:mapper method -i -e certificate_project Update struct=Certificate // CertificateProject is an association table struct that associates // Certificates to Projects. type CertificateProject struct { CertificateID int `db:"primary=yes"` ProjectID int } // CertificateProjectFilter specifies potential query parameter fields. type CertificateProjectFilter struct { CertificateID *int ProjectID *int } incus-7.0.0/internal/server/db/cluster/certificate_projects.interface.mapper.go000066400000000000000000000021141517523235500300010ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // CertificateProjectGenerated is an interface of generated methods for CertificateProject. type CertificateProjectGenerated interface { // GetCertificateProjects returns all available Projects for the Certificate. // generator: certificate_project GetMany GetCertificateProjects(ctx context.Context, db tx, certificateID int) ([]Project, error) // DeleteCertificateProjects deletes the certificate_project matching the given key parameters. // generator: certificate_project DeleteMany DeleteCertificateProjects(ctx context.Context, db tx, certificateID int) error // CreateCertificateProjects adds a new certificate_project to the database. // generator: certificate_project Create CreateCertificateProjects(ctx context.Context, db tx, objects []CertificateProject) error // UpdateCertificateProjects updates the certificate_project matching the given key parameters. // generator: certificate_project Update UpdateCertificateProjects(ctx context.Context, db tx, certificateID int, projectNames []string) error } incus-7.0.0/internal/server/db/cluster/certificate_projects.mapper.go000066400000000000000000000140711517523235500260470ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "fmt" "strings" ) var certificateProjectObjects = RegisterStmt(` SELECT certificates_projects.certificate_id, certificates_projects.project_id FROM certificates_projects ORDER BY certificates_projects.certificate_id `) var certificateProjectObjectsByCertificateID = RegisterStmt(` SELECT certificates_projects.certificate_id, certificates_projects.project_id FROM certificates_projects WHERE ( certificates_projects.certificate_id = ? ) ORDER BY certificates_projects.certificate_id `) var certificateProjectCreate = RegisterStmt(` INSERT INTO certificates_projects (certificate_id, project_id) VALUES (?, ?) `) var certificateProjectDeleteByCertificateID = RegisterStmt(` DELETE FROM certificates_projects WHERE certificate_id = ? `) // certificateProjectColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the CertificateProject entity. func certificateProjectColumns() string { return "certificates_projects.certificate_id, certificates_projects.project_id" } // getCertificateProjects can be used to run handwritten sql.Stmts to return a slice of objects. func getCertificateProjects(ctx context.Context, stmt *sql.Stmt, args ...any) ([]CertificateProject, error) { objects := make([]CertificateProject, 0) dest := func(scan func(dest ...any) error) error { c := CertificateProject{} err := scan(&c.CertificateID, &c.ProjectID) if err != nil { return err } objects = append(objects, c) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"certificates_projects\" table: %w", err) } return objects, nil } // getCertificateProjectsRaw can be used to run handwritten query strings to return a slice of objects. func getCertificateProjectsRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]CertificateProject, error) { objects := make([]CertificateProject, 0) dest := func(scan func(dest ...any) error) error { c := CertificateProject{} err := scan(&c.CertificateID, &c.ProjectID) if err != nil { return err } objects = append(objects, c) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"certificates_projects\" table: %w", err) } return objects, nil } // GetCertificateProjects returns all available Projects for the Certificate. // generator: certificate_project GetMany func GetCertificateProjects(ctx context.Context, db tx, certificateID int) (_ []Project, _err error) { defer func() { _err = mapErr(_err, "Certificate_project") }() var err error // Result slice. objects := make([]CertificateProject, 0) sqlStmt, err := Stmt(db, certificateProjectObjectsByCertificateID) if err != nil { return nil, fmt.Errorf("Failed to get \"certificateProjectObjectsByCertificateID\" prepared statement: %w", err) } args := []any{certificateID} // Select. objects, err = getCertificateProjects(ctx, sqlStmt, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"certificates_projects\" table: %w", err) } result := make([]Project, len(objects)) for i, object := range objects { project, err := GetProjects(ctx, db, ProjectFilter{ID: &object.ProjectID}) if err != nil { return nil, err } result[i] = project[0] } return result, nil } // DeleteCertificateProjects deletes the certificate_project matching the given key parameters. // generator: certificate_project DeleteMany func DeleteCertificateProjects(ctx context.Context, db tx, certificateID int) (_err error) { defer func() { _err = mapErr(_err, "Certificate_project") }() stmt, err := Stmt(db, certificateProjectDeleteByCertificateID) if err != nil { return fmt.Errorf("Failed to get \"certificateProjectDeleteByCertificateID\" prepared statement: %w", err) } result, err := stmt.Exec(int(certificateID)) if err != nil { return fmt.Errorf("Delete \"certificates_projects\" entry failed: %w", err) } _, err = result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } return nil } // CreateCertificateProjects adds a new certificate_project to the database. // generator: certificate_project Create func CreateCertificateProjects(ctx context.Context, db tx, objects []CertificateProject) (_err error) { defer func() { _err = mapErr(_err, "Certificate_project") }() for _, object := range objects { args := make([]any, 2) // Populate the statement arguments. args[0] = object.CertificateID args[1] = object.ProjectID // Prepared statement to use. stmt, err := Stmt(db, certificateProjectCreate) if err != nil { return fmt.Errorf("Failed to get \"certificateProjectCreate\" prepared statement: %w", err) } // Execute the statement. _, err = stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return ErrConflict } if err != nil { return fmt.Errorf("Failed to create \"certificates_projects\" entry: %w", err) } } return nil } // UpdateCertificateProjects updates the certificate_project matching the given key parameters. // generator: certificate_project Update func UpdateCertificateProjects(ctx context.Context, db tx, certificateID int, projectNames []string) (_err error) { defer func() { _err = mapErr(_err, "Certificate_project") }() // Delete current entry. err := DeleteCertificateProjects(ctx, db, certificateID) if err != nil { return err } // Get new entry IDs. certificateProjects := make([]CertificateProject, 0, len(projectNames)) for _, entry := range projectNames { refID, err := GetProjectID(ctx, db, entry) if err != nil { return err } certificateProjects = append(certificateProjects, CertificateProject{CertificateID: certificateID, ProjectID: int(refID)}) } err = CreateCertificateProjects(ctx, db, certificateProjects) if err != nil { return err } return nil } incus-7.0.0/internal/server/db/cluster/certificates.go000066400000000000000000000112411517523235500230320ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "context" "database/sql" "errors" "fmt" "net/http" "github.com/lxc/incus/v7/internal/server/certificate" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/shared/api" ) // Code generation directives. // //generate-database:mapper target certificates.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e certificate objects //generate-database:mapper stmt -e certificate objects-by-ID //generate-database:mapper stmt -e certificate objects-by-Fingerprint //generate-database:mapper stmt -e certificate id //generate-database:mapper stmt -e certificate create struct=Certificate //generate-database:mapper stmt -e certificate delete-by-Fingerprint //generate-database:mapper stmt -e certificate delete-by-Name-and-Type //generate-database:mapper stmt -e certificate update struct=Certificate // //generate-database:mapper method -i -e certificate GetMany //generate-database:mapper method -i -e certificate GetOne //generate-database:mapper method -i -e certificate ID struct=Certificate //generate-database:mapper method -i -e certificate Exists struct=Certificate //generate-database:mapper method -i -e certificate Create struct=Certificate //generate-database:mapper method -i -e certificate DeleteOne-by-Fingerprint //generate-database:mapper method -i -e certificate DeleteMany-by-Name-and-Type //generate-database:mapper method -i -e certificate Update struct=Certificate // Certificate is here to pass the certificates content from the database around. type Certificate struct { ID int Fingerprint string `db:"primary=yes"` Type certificate.Type Name string Certificate string Restricted bool Description string } // CertificateFilter specifies potential query parameter fields. type CertificateFilter struct { ID *int Fingerprint *string Name *string Type *certificate.Type } // ToAPIType returns the API equivalent type. func (cert *Certificate) ToAPIType() string { switch cert.Type { case certificate.TypeClient: return api.CertificateTypeClient case certificate.TypeServer: return api.CertificateTypeServer case certificate.TypeMetrics: return api.CertificateTypeMetrics } return api.CertificateTypeUnknown } // ToAPI converts the database Certificate struct to an api.Certificate // entry filling fields from the database as necessary. func (cert *Certificate) ToAPI(ctx context.Context, tx *sql.Tx) (*api.Certificate, error) { resp := api.Certificate{} resp.Fingerprint = cert.Fingerprint resp.Certificate = cert.Certificate resp.Name = cert.Name resp.Restricted = cert.Restricted resp.Type = cert.ToAPIType() resp.Description = cert.Description if tx != nil { projects, err := GetCertificateProjects(ctx, tx, cert.ID) if err != nil { return nil, err } resp.Projects = make([]string, len(projects)) for i, p := range projects { resp.Projects[i] = p.Name } } return &resp, nil } // GetCertificateByFingerprintPrefix gets an CertBaseInfo object from the database. // The argument fingerprint will be queried with a LIKE query, means you can // pass a shortform and will get the full fingerprint. // There can never be more than one certificate with a given fingerprint, as it is // enforced by a UNIQUE constraint in the schema. func GetCertificateByFingerprintPrefix(ctx context.Context, tx *sql.Tx, fingerprintPrefix string) (*Certificate, error) { var err error var cert *Certificate sql := ` SELECT certificates.fingerprint FROM certificates WHERE certificates.fingerprint LIKE ? ORDER BY certificates.fingerprint ` fingerprints, err := query.SelectStrings(ctx, tx, sql, fingerprintPrefix+"%") if err != nil { return nil, fmt.Errorf("Failed to fetch certificates fingerprints matching prefix %q: %w", fingerprintPrefix, err) } if len(fingerprints) > 1 { return nil, errors.New("More than one certificate matches") } if len(fingerprints) == 0 { return nil, api.StatusErrorf(http.StatusNotFound, "Certificate not found") } cert, err = GetCertificate(ctx, tx, fingerprints[0]) if err != nil { return nil, err } return cert, nil } // CreateCertificateWithProjects stores a CertInfo object in the db, and associates it to a list of project names. // It will ignore the ID field from the CertInfo. func CreateCertificateWithProjects(ctx context.Context, tx *sql.Tx, cert Certificate, projectNames []string) (int64, error) { var id int64 var err error id, err = CreateCertificate(ctx, tx, cert) if err != nil { return -1, err } err = UpdateCertificateProjects(ctx, tx, int(id), projectNames) if err != nil { return -1, err } return id, err } incus-7.0.0/internal/server/db/cluster/certificates.interface.mapper.go000066400000000000000000000034631517523235500262630ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "context" "github.com/lxc/incus/v7/internal/server/certificate" ) // CertificateGenerated is an interface of generated methods for Certificate. type CertificateGenerated interface { // GetCertificates returns all available certificates. // generator: certificate GetMany GetCertificates(ctx context.Context, db dbtx, filters ...CertificateFilter) ([]Certificate, error) // GetCertificate returns the certificate with the given key. // generator: certificate GetOne GetCertificate(ctx context.Context, db dbtx, fingerprint string) (*Certificate, error) // GetCertificateID return the ID of the certificate with the given key. // generator: certificate ID GetCertificateID(ctx context.Context, db tx, fingerprint string) (int64, error) // CertificateExists checks if a certificate with the given key exists. // generator: certificate Exists CertificateExists(ctx context.Context, db dbtx, fingerprint string) (bool, error) // CreateCertificate adds a new certificate to the database. // generator: certificate Create CreateCertificate(ctx context.Context, db dbtx, object Certificate) (int64, error) // DeleteCertificate deletes the certificate matching the given key parameters. // generator: certificate DeleteOne-by-Fingerprint DeleteCertificate(ctx context.Context, db dbtx, fingerprint string) error // DeleteCertificates deletes the certificate matching the given key parameters. // generator: certificate DeleteMany-by-Name-and-Type DeleteCertificates(ctx context.Context, db dbtx, name string, certificateType certificate.Type) error // UpdateCertificate updates the certificate matching the given key parameters. // generator: certificate Update UpdateCertificate(ctx context.Context, db tx, fingerprint string, object Certificate) error } incus-7.0.0/internal/server/db/cluster/certificates.mapper.go000066400000000000000000000271771517523235500243340ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" "github.com/lxc/incus/v7/internal/server/certificate" ) var certificateObjects = RegisterStmt(` SELECT certificates.id, certificates.fingerprint, certificates.type, certificates.name, certificates.certificate, certificates.restricted, certificates.description FROM certificates ORDER BY certificates.fingerprint `) var certificateObjectsByID = RegisterStmt(` SELECT certificates.id, certificates.fingerprint, certificates.type, certificates.name, certificates.certificate, certificates.restricted, certificates.description FROM certificates WHERE ( certificates.id = ? ) ORDER BY certificates.fingerprint `) var certificateObjectsByFingerprint = RegisterStmt(` SELECT certificates.id, certificates.fingerprint, certificates.type, certificates.name, certificates.certificate, certificates.restricted, certificates.description FROM certificates WHERE ( certificates.fingerprint = ? ) ORDER BY certificates.fingerprint `) var certificateID = RegisterStmt(` SELECT certificates.id FROM certificates WHERE certificates.fingerprint = ? `) var certificateCreate = RegisterStmt(` INSERT INTO certificates (fingerprint, type, name, certificate, restricted, description) VALUES (?, ?, ?, ?, ?, ?) `) var certificateDeleteByFingerprint = RegisterStmt(` DELETE FROM certificates WHERE fingerprint = ? `) var certificateDeleteByNameAndType = RegisterStmt(` DELETE FROM certificates WHERE name = ? AND type = ? `) var certificateUpdate = RegisterStmt(` UPDATE certificates SET fingerprint = ?, type = ?, name = ?, certificate = ?, restricted = ?, description = ? WHERE id = ? `) // certificateColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the Certificate entity. func certificateColumns() string { return "certificates.id, certificates.fingerprint, certificates.type, certificates.name, certificates.certificate, certificates.restricted, certificates.description" } // getCertificates can be used to run handwritten sql.Stmts to return a slice of objects. func getCertificates(ctx context.Context, stmt *sql.Stmt, args ...any) ([]Certificate, error) { objects := make([]Certificate, 0) dest := func(scan func(dest ...any) error) error { c := Certificate{} err := scan(&c.ID, &c.Fingerprint, &c.Type, &c.Name, &c.Certificate, &c.Restricted, &c.Description) if err != nil { return err } objects = append(objects, c) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"certificates\" table: %w", err) } return objects, nil } // getCertificatesRaw can be used to run handwritten query strings to return a slice of objects. func getCertificatesRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]Certificate, error) { objects := make([]Certificate, 0) dest := func(scan func(dest ...any) error) error { c := Certificate{} err := scan(&c.ID, &c.Fingerprint, &c.Type, &c.Name, &c.Certificate, &c.Restricted, &c.Description) if err != nil { return err } objects = append(objects, c) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"certificates\" table: %w", err) } return objects, nil } // GetCertificates returns all available certificates. // generator: certificate GetMany func GetCertificates(ctx context.Context, db dbtx, filters ...CertificateFilter) (_ []Certificate, _err error) { defer func() { _err = mapErr(_err, "Certificate") }() var err error // Result slice. objects := make([]Certificate, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, certificateObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"certificateObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.ID != nil && filter.Fingerprint == nil && filter.Name == nil && filter.Type == nil { args = append(args, []any{filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, certificateObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"certificateObjectsByID\" prepared statement: %w", err) } break } query, err := StmtString(certificateObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"certificateObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Fingerprint != nil && filter.ID == nil && filter.Name == nil && filter.Type == nil { args = append(args, []any{filter.Fingerprint}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, certificateObjectsByFingerprint) if err != nil { return nil, fmt.Errorf("Failed to get \"certificateObjectsByFingerprint\" prepared statement: %w", err) } break } query, err := StmtString(certificateObjectsByFingerprint) if err != nil { return nil, fmt.Errorf("Failed to get \"certificateObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.Fingerprint == nil && filter.Name == nil && filter.Type == nil { return nil, fmt.Errorf("Cannot filter on empty CertificateFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getCertificates(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getCertificatesRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"certificates\" table: %w", err) } return objects, nil } // GetCertificate returns the certificate with the given key. // generator: certificate GetOne func GetCertificate(ctx context.Context, db dbtx, fingerprint string) (_ *Certificate, _err error) { defer func() { _err = mapErr(_err, "Certificate") }() filter := CertificateFilter{} filter.Fingerprint = &fingerprint objects, err := GetCertificates(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"certificates\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"certificates\" entry matches") } } // GetCertificateID return the ID of the certificate with the given key. // generator: certificate ID func GetCertificateID(ctx context.Context, db tx, fingerprint string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Certificate") }() stmt, err := Stmt(db, certificateID) if err != nil { return -1, fmt.Errorf("Failed to get \"certificateID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, fingerprint) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"certificates\" ID: %w", err) } return id, nil } // CertificateExists checks if a certificate with the given key exists. // generator: certificate Exists func CertificateExists(ctx context.Context, db dbtx, fingerprint string) (_ bool, _err error) { defer func() { _err = mapErr(_err, "Certificate") }() stmt, err := Stmt(db, certificateID) if err != nil { return false, fmt.Errorf("Failed to get \"certificateID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, fingerprint) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("Failed to get \"certificates\" ID: %w", err) } return true, nil } // CreateCertificate adds a new certificate to the database. // generator: certificate Create func CreateCertificate(ctx context.Context, db dbtx, object Certificate) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Certificate") }() args := make([]any, 6) // Populate the statement arguments. args[0] = object.Fingerprint args[1] = object.Type args[2] = object.Name args[3] = object.Certificate args[4] = object.Restricted args[5] = object.Description // Prepared statement to use. stmt, err := Stmt(db, certificateCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"certificateCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"certificates\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"certificates\" entry ID: %w", err) } return id, nil } // DeleteCertificate deletes the certificate matching the given key parameters. // generator: certificate DeleteOne-by-Fingerprint func DeleteCertificate(ctx context.Context, db dbtx, fingerprint string) (_err error) { defer func() { _err = mapErr(_err, "Certificate") }() stmt, err := Stmt(db, certificateDeleteByFingerprint) if err != nil { return fmt.Errorf("Failed to get \"certificateDeleteByFingerprint\" prepared statement: %w", err) } result, err := stmt.Exec(fingerprint) if err != nil { return fmt.Errorf("Delete \"certificates\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d Certificate rows instead of 1", n) } return nil } // DeleteCertificates deletes the certificate matching the given key parameters. // generator: certificate DeleteMany-by-Name-and-Type func DeleteCertificates(ctx context.Context, db dbtx, name string, certificateType certificate.Type) (_err error) { defer func() { _err = mapErr(_err, "Certificate") }() stmt, err := Stmt(db, certificateDeleteByNameAndType) if err != nil { return fmt.Errorf("Failed to get \"certificateDeleteByNameAndType\" prepared statement: %w", err) } result, err := stmt.Exec(name, certificateType) if err != nil { return fmt.Errorf("Delete \"certificates\": %w", err) } _, err = result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } return nil } // UpdateCertificate updates the certificate matching the given key parameters. // generator: certificate Update func UpdateCertificate(ctx context.Context, db tx, fingerprint string, object Certificate) (_err error) { defer func() { _err = mapErr(_err, "Certificate") }() id, err := GetCertificateID(ctx, db, fingerprint) if err != nil { return err } stmt, err := Stmt(db, certificateUpdate) if err != nil { return fmt.Errorf("Failed to get \"certificateUpdate\" prepared statement: %w", err) } result, err := stmt.Exec(object.Fingerprint, object.Type, object.Name, object.Certificate, object.Restricted, object.Description, id) if err != nil { return fmt.Errorf("Update \"certificates\" entry failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } return nil } incus-7.0.0/internal/server/db/cluster/cluster_groups.go000066400000000000000000000045071517523235500234540ustar00rootroot00000000000000package cluster import ( "context" "database/sql" "github.com/lxc/incus/v7/shared/api" ) // Code generation directives. // //generate-database:mapper target cluster_groups.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e cluster_group objects table=cluster_groups //generate-database:mapper stmt -e cluster_group objects-by-Name table=cluster_groups //generate-database:mapper stmt -e cluster_group id table=cluster_groups //generate-database:mapper stmt -e cluster_group create table=cluster_groups //generate-database:mapper stmt -e cluster_group rename table=cluster_groups //generate-database:mapper stmt -e cluster_group delete-by-Name table=cluster_groups //generate-database:mapper stmt -e cluster_group update table=cluster_groups // //generate-database:mapper method -i -e cluster_group GetMany references=Config table=cluster_groups //generate-database:mapper method -i -e cluster_group GetOne table=cluster_groups //generate-database:mapper method -i -e cluster_group ID table=cluster_groups //generate-database:mapper method -i -e cluster_group Exists table=cluster_groups //generate-database:mapper method -i -e cluster_group Rename table=cluster_groups //generate-database:mapper method -i -e cluster_group Create references=Config table=cluster_groups //generate-database:mapper method -i -e cluster_group Update references=Config table=cluster_groups //generate-database:mapper method -i -e cluster_group DeleteOne-by-Name table=cluster_groups // ClusterGroup is a value object holding db-related details about a cluster group. type ClusterGroup struct { ID int Name string Description string `db:"coalesce=''"` Nodes []string `db:"ignore"` } // ClusterGroupFilter specifies potential query parameter fields. type ClusterGroupFilter struct { ID *int Name *string } // ToAPI returns an API entry. func (c *ClusterGroup) ToAPI(ctx context.Context, tx *sql.Tx) (*api.ClusterGroup, error) { // Get the config. config, err := GetClusterGroupConfig(ctx, tx, c.ID) if err != nil { return nil, err } result := api.ClusterGroup{ ClusterGroupPut: api.ClusterGroupPut{ Config: config, Description: c.Description, Members: c.Nodes, }, ClusterGroupPost: api.ClusterGroupPost{ Name: c.Name, }, } return &result, nil } incus-7.0.0/internal/server/db/cluster/cluster_groups.interface.mapper.go000066400000000000000000000046141517523235500266750ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // ClusterGroupGenerated is an interface of generated methods for ClusterGroup. type ClusterGroupGenerated interface { // GetClusterGroupConfig returns all available ClusterGroup Config // generator: cluster_group GetMany GetClusterGroupConfig(ctx context.Context, db tx, clusterGroupID int, filters ...ConfigFilter) (map[string]string, error) // GetClusterGroups returns all available cluster_groups. // generator: cluster_group GetMany GetClusterGroups(ctx context.Context, db dbtx, filters ...ClusterGroupFilter) ([]ClusterGroup, error) // GetClusterGroup returns the cluster_group with the given key. // generator: cluster_group GetOne GetClusterGroup(ctx context.Context, db dbtx, name string) (*ClusterGroup, error) // GetClusterGroupID return the ID of the cluster_group with the given key. // generator: cluster_group ID GetClusterGroupID(ctx context.Context, db tx, name string) (int64, error) // ClusterGroupExists checks if a cluster_group with the given key exists. // generator: cluster_group Exists ClusterGroupExists(ctx context.Context, db dbtx, name string) (bool, error) // RenameClusterGroup renames the cluster_group matching the given key parameters. // generator: cluster_group Rename RenameClusterGroup(ctx context.Context, db dbtx, name string, to string) error // CreateClusterGroupConfig adds new cluster_group Config to the database. // generator: cluster_group Create CreateClusterGroupConfig(ctx context.Context, db dbtx, clusterGroupID int64, config map[string]string) error // CreateClusterGroup adds a new cluster_group to the database. // generator: cluster_group Create CreateClusterGroup(ctx context.Context, db dbtx, object ClusterGroup) (int64, error) // UpdateClusterGroupConfig updates the cluster_group Config matching the given key parameters. // generator: cluster_group Update UpdateClusterGroupConfig(ctx context.Context, db tx, clusterGroupID int64, config map[string]string) error // UpdateClusterGroup updates the cluster_group matching the given key parameters. // generator: cluster_group Update UpdateClusterGroup(ctx context.Context, db tx, name string, object ClusterGroup) error // DeleteClusterGroup deletes the cluster_group matching the given key parameters. // generator: cluster_group DeleteOne-by-Name DeleteClusterGroup(ctx context.Context, db dbtx, name string) error } incus-7.0.0/internal/server/db/cluster/cluster_groups.mapper.go000066400000000000000000000267531517523235500247460ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var clusterGroupObjects = RegisterStmt(` SELECT cluster_groups.id, cluster_groups.name, coalesce(cluster_groups.description, '') FROM cluster_groups ORDER BY cluster_groups.name `) var clusterGroupObjectsByName = RegisterStmt(` SELECT cluster_groups.id, cluster_groups.name, coalesce(cluster_groups.description, '') FROM cluster_groups WHERE ( cluster_groups.name = ? ) ORDER BY cluster_groups.name `) var clusterGroupID = RegisterStmt(` SELECT cluster_groups.id FROM cluster_groups WHERE cluster_groups.name = ? `) var clusterGroupCreate = RegisterStmt(` INSERT INTO cluster_groups (name, description) VALUES (?, ?) `) var clusterGroupRename = RegisterStmt(` UPDATE cluster_groups SET name = ? WHERE name = ? `) var clusterGroupDeleteByName = RegisterStmt(` DELETE FROM cluster_groups WHERE name = ? `) var clusterGroupUpdate = RegisterStmt(` UPDATE cluster_groups SET name = ?, description = ? WHERE id = ? `) // clusterGroupColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the ClusterGroup entity. func clusterGroupColumns() string { return "cluster_groups.id, cluster_groups.name, coalesce(cluster_groups.description, '')" } // getClusterGroups can be used to run handwritten sql.Stmts to return a slice of objects. func getClusterGroups(ctx context.Context, stmt *sql.Stmt, args ...any) ([]ClusterGroup, error) { objects := make([]ClusterGroup, 0) dest := func(scan func(dest ...any) error) error { c := ClusterGroup{} err := scan(&c.ID, &c.Name, &c.Description) if err != nil { return err } objects = append(objects, c) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"cluster_groups\" table: %w", err) } return objects, nil } // getClusterGroupsRaw can be used to run handwritten query strings to return a slice of objects. func getClusterGroupsRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]ClusterGroup, error) { objects := make([]ClusterGroup, 0) dest := func(scan func(dest ...any) error) error { c := ClusterGroup{} err := scan(&c.ID, &c.Name, &c.Description) if err != nil { return err } objects = append(objects, c) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"cluster_groups\" table: %w", err) } return objects, nil } // GetClusterGroups returns all available cluster_groups. // generator: cluster_group GetMany func GetClusterGroups(ctx context.Context, db dbtx, filters ...ClusterGroupFilter) (_ []ClusterGroup, _err error) { defer func() { _err = mapErr(_err, "Cluster_group") }() var err error // Result slice. objects := make([]ClusterGroup, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, clusterGroupObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"clusterGroupObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.Name != nil && filter.ID == nil { args = append(args, []any{filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, clusterGroupObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"clusterGroupObjectsByName\" prepared statement: %w", err) } break } query, err := StmtString(clusterGroupObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"clusterGroupObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.Name == nil { return nil, fmt.Errorf("Cannot filter on empty ClusterGroupFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getClusterGroups(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getClusterGroupsRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"cluster_groups\" table: %w", err) } return objects, nil } // GetClusterGroupConfig returns all available ClusterGroup Config // generator: cluster_group GetMany func GetClusterGroupConfig(ctx context.Context, db tx, clusterGroupID int, filters ...ConfigFilter) (_ map[string]string, _err error) { defer func() { _err = mapErr(_err, "Cluster_group") }() clusterGroupConfig, err := GetConfig(ctx, db, "cluster_groups", "cluster_group", filters...) if err != nil { return nil, err } config, ok := clusterGroupConfig[clusterGroupID] if !ok { config = map[string]string{} } return config, nil } // GetClusterGroup returns the cluster_group with the given key. // generator: cluster_group GetOne func GetClusterGroup(ctx context.Context, db dbtx, name string) (_ *ClusterGroup, _err error) { defer func() { _err = mapErr(_err, "Cluster_group") }() filter := ClusterGroupFilter{} filter.Name = &name objects, err := GetClusterGroups(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"cluster_groups\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"cluster_groups\" entry matches") } } // GetClusterGroupID return the ID of the cluster_group with the given key. // generator: cluster_group ID func GetClusterGroupID(ctx context.Context, db tx, name string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Cluster_group") }() stmt, err := Stmt(db, clusterGroupID) if err != nil { return -1, fmt.Errorf("Failed to get \"clusterGroupID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"cluster_groups\" ID: %w", err) } return id, nil } // ClusterGroupExists checks if a cluster_group with the given key exists. // generator: cluster_group Exists func ClusterGroupExists(ctx context.Context, db dbtx, name string) (_ bool, _err error) { defer func() { _err = mapErr(_err, "Cluster_group") }() stmt, err := Stmt(db, clusterGroupID) if err != nil { return false, fmt.Errorf("Failed to get \"clusterGroupID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("Failed to get \"cluster_groups\" ID: %w", err) } return true, nil } // RenameClusterGroup renames the cluster_group matching the given key parameters. // generator: cluster_group Rename func RenameClusterGroup(ctx context.Context, db dbtx, name string, to string) (_err error) { defer func() { _err = mapErr(_err, "Cluster_group") }() stmt, err := Stmt(db, clusterGroupRename) if err != nil { return fmt.Errorf("Failed to get \"clusterGroupRename\" prepared statement: %w", err) } result, err := stmt.Exec(to, name) if err != nil { return fmt.Errorf("Rename ClusterGroup failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows failed: %w", err) } if n != 1 { return fmt.Errorf("Query affected %d rows instead of 1", n) } return nil } // CreateClusterGroup adds a new cluster_group to the database. // generator: cluster_group Create func CreateClusterGroup(ctx context.Context, db dbtx, object ClusterGroup) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Cluster_group") }() args := make([]any, 2) // Populate the statement arguments. args[0] = object.Name args[1] = object.Description // Prepared statement to use. stmt, err := Stmt(db, clusterGroupCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"clusterGroupCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"cluster_groups\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"cluster_groups\" entry ID: %w", err) } return id, nil } // CreateClusterGroupConfig adds new cluster_group Config to the database. // generator: cluster_group Create func CreateClusterGroupConfig(ctx context.Context, db dbtx, clusterGroupID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Cluster_group") }() referenceID := int(clusterGroupID) for key, value := range config { insert := Config{ ReferenceID: referenceID, Key: key, Value: value, } err := CreateConfig(ctx, db, "cluster_groups", "cluster_group", insert) if err != nil { return fmt.Errorf("Insert Config failed for ClusterGroup: %w", err) } } return nil } // UpdateClusterGroup updates the cluster_group matching the given key parameters. // generator: cluster_group Update func UpdateClusterGroup(ctx context.Context, db tx, name string, object ClusterGroup) (_err error) { defer func() { _err = mapErr(_err, "Cluster_group") }() id, err := GetClusterGroupID(ctx, db, name) if err != nil { return err } stmt, err := Stmt(db, clusterGroupUpdate) if err != nil { return fmt.Errorf("Failed to get \"clusterGroupUpdate\" prepared statement: %w", err) } result, err := stmt.Exec(object.Name, object.Description, id) if err != nil { return fmt.Errorf("Update \"cluster_groups\" entry failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } return nil } // UpdateClusterGroupConfig updates the cluster_group Config matching the given key parameters. // generator: cluster_group Update func UpdateClusterGroupConfig(ctx context.Context, db tx, clusterGroupID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Cluster_group") }() err := UpdateConfig(ctx, db, "cluster_groups", "cluster_group", int(clusterGroupID), config) if err != nil { return fmt.Errorf("Replace Config for ClusterGroup failed: %w", err) } return nil } // DeleteClusterGroup deletes the cluster_group matching the given key parameters. // generator: cluster_group DeleteOne-by-Name func DeleteClusterGroup(ctx context.Context, db dbtx, name string) (_err error) { defer func() { _err = mapErr(_err, "Cluster_group") }() stmt, err := Stmt(db, clusterGroupDeleteByName) if err != nil { return fmt.Errorf("Failed to get \"clusterGroupDeleteByName\" prepared statement: %w", err) } result, err := stmt.Exec(name) if err != nil { return fmt.Errorf("Delete \"cluster_groups\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d ClusterGroup rows instead of 1", n) } return nil } incus-7.0.0/internal/server/db/cluster/config.go000066400000000000000000000016721517523235500216410ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster // Code generation directives. // //generate-database:mapper target config.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e config objects //generate-database:mapper stmt -e config create struct=Config //generate-database:mapper stmt -e config delete // //generate-database:mapper method -i -e config GetMany //generate-database:mapper method -i -e config Create struct=Config //generate-database:mapper method -i -e config Update struct=Config //generate-database:mapper method -i -e config DeleteMany // Config is a reference struct representing one configuration entry of another entity. type Config struct { ID int `db:"primary=yes"` ReferenceID int Key string Value string } // ConfigFilter specifies potential query parameter fields. type ConfigFilter struct { Key *string Value *string } incus-7.0.0/internal/server/db/cluster/config.interface.mapper.go000066400000000000000000000020611517523235500250540ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // ConfigGenerated is an interface of generated methods for Config. type ConfigGenerated interface { // GetConfig returns all available config. // generator: config GetMany GetConfig(ctx context.Context, db dbtx, parentTablePrefix string, parentColumnPrefix string, filters ...ConfigFilter) (map[int]map[string]string, error) // CreateConfig adds a new config to the database. // generator: config Create CreateConfig(ctx context.Context, db dbtx, parentTablePrefix string, parentColumnPrefix string, object Config) error // UpdateConfig updates the config matching the given key parameters. // generator: config Update UpdateConfig(ctx context.Context, db dbtx, parentTablePrefix string, parentColumnPrefix string, referenceID int, config map[string]string) error // DeleteConfig deletes the config matching the given key parameters. // generator: config DeleteMany DeleteConfig(ctx context.Context, db dbtx, parentTablePrefix string, parentColumnPrefix string, referenceID int) error } incus-7.0.0/internal/server/db/cluster/config.mapper.go000066400000000000000000000142431517523235500231220ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "fmt" "strings" ) const configObjects = `SELECT %s_config.id, %s_config.%s_id, %s_config.key, %s_config.value FROM %s_config ORDER BY %s_config.id` const configCreate = `INSERT INTO %s_config (%s_id, key, value) VALUES (?, ?, ?)` const configDelete = `DELETE FROM %s_config WHERE %s_id = ?` // configColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the Config entity. func configColumns() string { return "%s_config.id, %s_config.%s_id, %s_config.key, %s_config.value" } // getConfig can be used to run handwritten sql.Stmts to return a slice of objects. func getConfig(ctx context.Context, stmt *sql.Stmt, parent string, args ...any) ([]Config, error) { objects := make([]Config, 0) dest := func(scan func(dest ...any) error) error { c := Config{} err := scan(&c.ID, &c.ReferenceID, &c.Key, &c.Value) if err != nil { return err } objects = append(objects, c) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"%s_config\" table: %w", parent, err) } return objects, nil } // getConfigRaw can be used to run handwritten query strings to return a slice of objects. func getConfigRaw(ctx context.Context, db dbtx, sql string, parent string, args ...any) ([]Config, error) { objects := make([]Config, 0) dest := func(scan func(dest ...any) error) error { c := Config{} err := scan(&c.ID, &c.ReferenceID, &c.Key, &c.Value) if err != nil { return err } objects = append(objects, c) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"%s_config\" table: %w", parent, err) } return objects, nil } // GetConfig returns all available config. // generator: config GetMany func GetConfig(ctx context.Context, db dbtx, parentTablePrefix string, parentColumnPrefix string, filters ...ConfigFilter) (_ map[int]map[string]string, _err error) { defer func() { _err = mapErr(_err, "Config") }() var err error // Result slice. objects := make([]Config, 0) configObjectsLocal := strings.ReplaceAll(configObjects, "%s_id", fmt.Sprintf("%s_id", parentColumnPrefix)) fillParent := make([]any, strings.Count(configObjectsLocal, "%s")) for i := range fillParent { fillParent[i] = parentTablePrefix } queryStr := fmt.Sprintf(configObjectsLocal, fillParent...) queryParts := strings.SplitN(queryStr, "ORDER BY", 2) args := []any{} for i, filter := range filters { var cond string if i == 0 { cond = " WHERE ( %s )" } else { cond = " OR ( %s )" } entries := []string{} if filter.Key != nil { entries = append(entries, "key = ?") args = append(args, filter.Key) } if filter.Value != nil { entries = append(entries, "value = ?") args = append(args, filter.Value) } if len(entries) == 0 { return nil, fmt.Errorf("Cannot filter on empty ConfigFilter") } queryParts[0] += fmt.Sprintf(cond, strings.Join(entries, " AND ")) } queryStr = strings.Join(queryParts, " ORDER BY") // Select. objects, err = getConfigRaw(ctx, db, queryStr, parentTablePrefix, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"%s_config\" table: %w", parentTablePrefix, err) } resultMap := map[int]map[string]string{} for _, object := range objects { _, ok := resultMap[object.ReferenceID] if !ok { resultMap[object.ReferenceID] = map[string]string{} } resultMap[object.ReferenceID][object.Key] = object.Value } return resultMap, nil } // CreateConfig adds a new config to the database. // generator: config Create func CreateConfig(ctx context.Context, db dbtx, parentTablePrefix string, parentColumnPrefix string, object Config) (_err error) { defer func() { _err = mapErr(_err, "Config") }() // An empty value means we are unsetting this key, so just return. if object.Value == "" { return nil } configCreateLocal := strings.ReplaceAll(configCreate, "%s_id", fmt.Sprintf("%s_id", parentColumnPrefix)) fillParent := make([]any, strings.Count(configCreateLocal, "%s")) for i := range fillParent { fillParent[i] = parentTablePrefix } queryStr := fmt.Sprintf(configCreateLocal, fillParent...) _, err := db.ExecContext(ctx, queryStr, object.ReferenceID, object.Key, object.Value) if err != nil { return fmt.Errorf("Insert failed for \"%s_config\" table: %w", parentTablePrefix, err) } return nil } // UpdateConfig updates the config matching the given key parameters. // generator: config Update func UpdateConfig(ctx context.Context, db dbtx, parentTablePrefix string, parentColumnPrefix string, referenceID int, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Config") }() // Delete current entry. err := DeleteConfig(ctx, db, parentTablePrefix, parentColumnPrefix, referenceID) if err != nil { return err } // Insert new entries. for key, value := range config { object := Config{ ReferenceID: referenceID, Key: key, Value: value, } err = CreateConfig(ctx, db, parentTablePrefix, parentColumnPrefix, object) if err != nil { return err } } return nil } // DeleteConfig deletes the config matching the given key parameters. // generator: config DeleteMany func DeleteConfig(ctx context.Context, db dbtx, parentTablePrefix string, parentColumnPrefix string, referenceID int) (_err error) { defer func() { _err = mapErr(_err, "Config") }() configDeleteLocal := strings.ReplaceAll(configDelete, "%s_id", fmt.Sprintf("%s_id", parentColumnPrefix)) fillParent := make([]any, strings.Count(configDeleteLocal, "%s")) for i := range fillParent { fillParent[i] = parentTablePrefix } queryStr := fmt.Sprintf(configDeleteLocal, fillParent...) result, err := db.ExecContext(ctx, queryStr, referenceID) if err != nil { return fmt.Errorf("Delete entry for \"%s_config\" failed: %w", parentTablePrefix, err) } _, err = result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } return nil } incus-7.0.0/internal/server/db/cluster/devices.go000066400000000000000000000067321517523235500220200ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "fmt" ) // Code generation directives. // //generate-database:mapper target devices.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e device objects //generate-database:mapper stmt -e device create struct=Device //generate-database:mapper stmt -e device delete // //generate-database:mapper method -i -e device GetMany //generate-database:mapper method -i -e device Create struct=Device //generate-database:mapper method -i -e device Update struct=Device //generate-database:mapper method -i -e device DeleteMany // DeviceType represents the types of supported devices. type DeviceType int // Device is a reference struct representing another entity's device. type Device struct { ID int ReferenceID int Name string Type DeviceType Config map[string]string } // DeviceFilter specifies potential query parameter fields. type DeviceFilter struct { Name *string Type *DeviceType Config *ConfigFilter } // Supported device types. const ( TypeNone = DeviceType(0) TypeNIC = DeviceType(1) TypeDisk = DeviceType(2) TypeUnixChar = DeviceType(3) TypeUnixBlock = DeviceType(4) TypeUSB = DeviceType(5) TypeGPU = DeviceType(6) TypeInfiniband = DeviceType(7) TypeProxy = DeviceType(8) TypeUnixHotplug = DeviceType(9) TypeTPM = DeviceType(10) TypePCI = DeviceType(11) ) func (t DeviceType) String() string { switch t { case TypeNone: return "none" case TypeNIC: return "nic" case TypeDisk: return "disk" case TypeUnixChar: return "unix-char" case TypeUnixBlock: return "unix-block" case TypeUSB: return "usb" case TypeGPU: return "gpu" case TypeInfiniband: return "infiniband" case TypeProxy: return "proxy" case TypeUnixHotplug: return "unix-hotplug" case TypeTPM: return "tpm" case TypePCI: return "pci" } return "" } // NewDeviceType determines the device type from the given string, if supported. func NewDeviceType(t string) (DeviceType, error) { switch t { case "none": return TypeNone, nil case "nic": return TypeNIC, nil case "disk": return TypeDisk, nil case "unix-char": return TypeUnixChar, nil case "unix-block": return TypeUnixBlock, nil case "usb": return TypeUSB, nil case "gpu": return TypeGPU, nil case "infiniband": return TypeInfiniband, nil case "proxy": return TypeProxy, nil case "unix-hotplug": return TypeUnixHotplug, nil case "tpm": return TypeTPM, nil case "pci": return TypePCI, nil default: return -1, fmt.Errorf("Invalid device type %q", t) } } // DevicesToAPI takes a map of devices and converts them to API format. func DevicesToAPI(devices map[string]Device) map[string]map[string]string { config := map[string]map[string]string{} for _, d := range devices { if d.Config == nil { d.Config = map[string]string{} } config[d.Name] = d.Config config[d.Name]["type"] = d.Type.String() } return config } // APIToDevices takes an API format devices map and converts it to a map of db.Device. func APIToDevices(apiDevices map[string]map[string]string) (map[string]Device, error) { devices := map[string]Device{} for name, config := range apiDevices { newType, err := NewDeviceType(config["type"]) if err != nil { return nil, err } device := Device{ Name: name, Type: newType, Config: config, } devices[name] = device } return devices, nil } incus-7.0.0/internal/server/db/cluster/devices.interface.mapper.go000066400000000000000000000021141517523235500252300ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // DeviceGenerated is an interface of generated methods for Device. type DeviceGenerated interface { // GetDevices returns all available devices for the parent entity. // generator: device GetMany GetDevices(ctx context.Context, db tx, parentTablePrefix string, parentColumnPrefix string, filters ...DeviceFilter) (map[int][]Device, error) // CreateDevices adds a new device to the database. // generator: device Create CreateDevices(ctx context.Context, db tx, parentTablePrefix string, parentColumnPrefix string, objects map[string]Device) error // UpdateDevices updates the device matching the given key parameters. // generator: device Update UpdateDevices(ctx context.Context, db tx, parentTablePrefix string, parentColumnPrefix string, referenceID int, devices map[string]Device) error // DeleteDevices deletes the device matching the given key parameters. // generator: device DeleteMany DeleteDevices(ctx context.Context, db tx, parentTablePrefix string, parentColumnPrefix string, referenceID int) error } incus-7.0.0/internal/server/db/cluster/devices.mapper.go000066400000000000000000000163061517523235500233010ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "fmt" "strings" ) const deviceObjects = `SELECT %s_devices.id, %s_devices.%s_id, %s_devices.name, %s_devices.type FROM %s_devices ORDER BY %s_devices.name` const deviceCreate = `INSERT INTO %s_devices (%s_id, name, type) VALUES (?, ?, ?)` const deviceDelete = `DELETE FROM %s_devices WHERE %s_id = ?` // deviceColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the Device entity. func deviceColumns() string { return "%s_devices.id, %s_devices.%s_id, %s_devices.name, %s_devices.type, %s_devices.config" } // getDevices can be used to run handwritten sql.Stmts to return a slice of objects. func getDevices(ctx context.Context, stmt *sql.Stmt, parent string, args ...any) ([]Device, error) { objects := make([]Device, 0) dest := func(scan func(dest ...any) error) error { d := Device{} err := scan(&d.ID, &d.ReferenceID, &d.Name, &d.Type) if err != nil { return err } objects = append(objects, d) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"%s_devices\" table: %w", parent, err) } return objects, nil } // getDevicesRaw can be used to run handwritten query strings to return a slice of objects. func getDevicesRaw(ctx context.Context, db dbtx, sql string, parent string, args ...any) ([]Device, error) { objects := make([]Device, 0) dest := func(scan func(dest ...any) error) error { d := Device{} err := scan(&d.ID, &d.ReferenceID, &d.Name, &d.Type) if err != nil { return err } objects = append(objects, d) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"%s_devices\" table: %w", parent, err) } return objects, nil } // GetDevices returns all available devices for the parent entity. // generator: device GetMany func GetDevices(ctx context.Context, db tx, parentTablePrefix string, parentColumnPrefix string, filters ...DeviceFilter) (_ map[int][]Device, _err error) { defer func() { _err = mapErr(_err, "Device") }() var err error // Result slice. objects := make([]Device, 0) deviceObjectsLocal := strings.ReplaceAll(deviceObjects, "%s_id", fmt.Sprintf("%s_id", parentColumnPrefix)) fillParent := make([]any, strings.Count(deviceObjectsLocal, "%s")) for i := range fillParent { fillParent[i] = parentTablePrefix } queryStr := fmt.Sprintf(deviceObjectsLocal, fillParent...) queryParts := strings.SplitN(queryStr, "ORDER BY", 2) args := []any{} for i, filter := range filters { var cond string if i == 0 { cond = " WHERE ( %s )" } else { cond = " OR ( %s )" } entries := []string{} if filter.Name != nil { entries = append(entries, "name = ?") args = append(args, filter.Name) } if filter.Type != nil { entries = append(entries, "type = ?") args = append(args, filter.Type) } if len(entries) == 0 { return nil, fmt.Errorf("Cannot filter on empty DeviceFilter") } queryParts[0] += fmt.Sprintf(cond, strings.Join(entries, " AND ")) } queryStr = strings.Join(queryParts, " ORDER BY") // Select. objects, err = getDevicesRaw(ctx, db, queryStr, parentTablePrefix, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"%s_devices\" table: %w", parentTablePrefix, err) } configFilters := []ConfigFilter{} for _, f := range filters { filter := f.Config if filter != nil { if filter.Key == nil && filter.Value == nil { return nil, fmt.Errorf("Cannot filter on empty ConfigFilter") } configFilters = append(configFilters, *filter) } } config, err := GetConfig(ctx, db, parentTablePrefix+"_devices", parentColumnPrefix+"_device", configFilters...) if err != nil { return nil, err } for i := range objects { _, ok := config[objects[i].ID] if !ok { objects[i].Config = map[string]string{} } else { objects[i].Config = config[objects[i].ID] } } resultMap := map[int][]Device{} for _, object := range objects { _, ok := resultMap[object.ReferenceID] if !ok { resultMap[object.ReferenceID] = []Device{} } resultMap[object.ReferenceID] = append(resultMap[object.ReferenceID], object) } return resultMap, nil } // CreateDevices adds a new device to the database. // generator: device Create func CreateDevices(ctx context.Context, db tx, parentTablePrefix string, parentColumnPrefix string, objects map[string]Device) (_err error) { defer func() { _err = mapErr(_err, "Device") }() deviceCreateLocal := strings.ReplaceAll(deviceCreate, "%s_id", fmt.Sprintf("%s_id", parentColumnPrefix)) fillParent := make([]any, strings.Count(deviceCreateLocal, "%s")) for i := range fillParent { fillParent[i] = parentTablePrefix } queryStr := fmt.Sprintf(deviceCreateLocal, fillParent...) for _, object := range objects { result, err := db.ExecContext(ctx, queryStr, object.ReferenceID, object.Name, object.Type) if err != nil { return fmt.Errorf("Insert failed for \"%s_devices\" table: %w", parentTablePrefix, err) } id, err := result.LastInsertId() if err != nil { return fmt.Errorf("Failed to fetch ID: %w", err) } referenceID := int(id) for key, value := range object.Config { insert := Config{ ReferenceID: referenceID, Key: key, Value: value, } err = CreateConfig(ctx, db, parentTablePrefix+"_devices", parentColumnPrefix+"_device", insert) if err != nil { return fmt.Errorf("Insert Config failed for Device: %w", err) } } } return nil } // UpdateDevices updates the device matching the given key parameters. // generator: device Update func UpdateDevices(ctx context.Context, db tx, parentTablePrefix string, parentColumnPrefix string, referenceID int, devices map[string]Device) (_err error) { defer func() { _err = mapErr(_err, "Device") }() // Delete current entry. err := DeleteDevices(ctx, db, parentTablePrefix, parentColumnPrefix, referenceID) if err != nil { return err } // Insert new entries. for key, object := range devices { object.ReferenceID = referenceID devices[key] = object } err = CreateDevices(ctx, db, parentTablePrefix, parentColumnPrefix, devices) if err != nil { return err } return nil } // DeleteDevices deletes the device matching the given key parameters. // generator: device DeleteMany func DeleteDevices(ctx context.Context, db tx, parentTablePrefix string, parentColumnPrefix string, referenceID int) (_err error) { defer func() { _err = mapErr(_err, "Device") }() deviceDeleteLocal := strings.ReplaceAll(deviceDelete, "%s_id", fmt.Sprintf("%s_id", parentColumnPrefix)) fillParent := make([]any, strings.Count(deviceDeleteLocal, "%s")) for i := range fillParent { fillParent[i] = parentTablePrefix } queryStr := fmt.Sprintf(deviceDeleteLocal, fillParent...) result, err := db.ExecContext(ctx, queryStr, referenceID) if err != nil { return fmt.Errorf("Delete entry for \"%s_device\" failed: %w", parentTablePrefix, err) } _, err = result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } return nil } incus-7.0.0/internal/server/db/cluster/entities.go000066400000000000000000000132351517523235500222160ustar00rootroot00000000000000package cluster import ( "fmt" "net/url" "strings" "github.com/lxc/incus/v7/internal/version" ) // Numeric type codes identifying different kind of entities. const ( TypeContainer = 0 TypeImage = 1 TypeProfile = 2 TypeProject = 3 TypeCertificate = 4 TypeInstance = 5 TypeInstanceBackup = 6 TypeInstanceSnapshot = 7 TypeNetwork = 8 TypeNetworkACL = 9 TypeNode = 10 TypeOperation = 11 TypeStoragePool = 12 TypeStorageVolume = 13 TypeStorageVolumeBackup = 14 TypeStorageVolumeSnapshot = 15 TypeWarning = 16 TypeClusterGroup = 17 TypeStorageBucket = 18 ) // EntityNames associates an entity code to its name. var EntityNames = map[int]string{ TypeCertificate: "certificate", TypeClusterGroup: "cluster group", TypeContainer: "container", TypeImage: "image", TypeInstanceBackup: "instance backup", TypeInstance: "instance", TypeInstanceSnapshot: "instance snapshot", TypeNetworkACL: "network acl", TypeNetwork: "network", TypeNode: "node", TypeOperation: "operation", TypeProfile: "profile", TypeProject: "project", TypeStorageBucket: "storage bucket", TypeStoragePool: "storage pool", TypeStorageVolumeBackup: "storage volume backup", TypeStorageVolumeSnapshot: "storage volume snapshot", TypeStorageVolume: "storage volume", TypeWarning: "warning", } // EntityTypes associates an entity name to its type code. var EntityTypes = map[string]int{} // EntityURIs associates an entity code to its URI pattern. var EntityURIs = map[int]string{ TypeCertificate: "/" + version.APIVersion + "/certificates/%s", TypeClusterGroup: "/" + version.APIVersion + "/cluster/groups/%s", TypeContainer: "/" + version.APIVersion + "/containers/%s?project=%s", TypeImage: "/" + version.APIVersion + "/images/%s?project=%s", TypeInstanceBackup: "/" + version.APIVersion + "/instances/%s/backups/%s?project=%s", TypeInstanceSnapshot: "/" + version.APIVersion + "/instances/%s/snapshots/%s?project=%s", TypeInstance: "/" + version.APIVersion + "/instances/%s?project=%s", TypeNetworkACL: "/" + version.APIVersion + "/network-acls/%s?project=%s", TypeNetwork: "/" + version.APIVersion + "/networks/%s?project=%s", TypeNode: "/" + version.APIVersion + "/cluster/members/%s", TypeOperation: "/" + version.APIVersion + "/operations/%s", TypeProfile: "/" + version.APIVersion + "/profiles/%s?project=%s", TypeProject: "/" + version.APIVersion + "/projects/%s", TypeStorageBucket: "/" + version.APIVersion + "/storage-pools/%s/buckets/%s?project=%s", TypeStoragePool: "/" + version.APIVersion + "/storage-pools/%s", TypeStorageVolumeBackup: "/" + version.APIVersion + "/storage-pools/%s/volumes/%s/%s/backups/%s?project=%s", TypeStorageVolumeSnapshot: "/" + version.APIVersion + "/storage-pools/%s/volumes/%s/%s/snapshots/%s?project=%s", TypeStorageVolume: "/" + version.APIVersion + "/storage-pools/%s/volumes/%s/%s?project=%s", TypeWarning: "/" + version.APIVersion + "/warnings/%s", } func init() { for code, name := range EntityNames { EntityTypes[name] = code } } // URLToEntityType parses a raw URL string and returns the entity type, the project, the location and the path arguments. The // returned project is set to "default" if it is not present (unless the entity type is TypeProject, in which case it is // set to the value of the path parameter). An error is returned if the URL is not recognised. func URLToEntityType(rawURL string) (int, string, string, []string, error) { u, err := url.Parse(rawURL) if err != nil { return -1, "", "", nil, fmt.Errorf("Failed to parse url %q into an entity type: %w", rawURL, err) } // We need to space separate the path because fmt.Sscanf uses this as a delimiter. spaceSeparatedURLPath := strings.ReplaceAll(u.Path, "/", " / ") for entityType, entityURI := range EntityURIs { entityPath, _, _ := strings.Cut(entityURI, "?") // Skip if we don't have the same number of slashes. if strings.Count(entityPath, "/") != strings.Count(u.Path, "/") { continue } spaceSeparatedEntityPath := strings.ReplaceAll(entityPath, "/", " / ") // Make an []any for the number of expected path arguments and set each value in the slice to a *string. nPathArgs := strings.Count(spaceSeparatedEntityPath, "%s") pathArgsAny := make([]any, 0, nPathArgs) for range nPathArgs { var pathComponentStr string pathArgsAny = append(pathArgsAny, &pathComponentStr) } // Scan the given URL into the entity URL. If we found all the expected path arguments and there // are no errors we have a match. nFound, err := fmt.Sscanf(spaceSeparatedURLPath, spaceSeparatedEntityPath, pathArgsAny...) if nFound == nPathArgs && err == nil { pathArgs := make([]string, 0, nPathArgs) for _, pathArgAny := range pathArgsAny { pathArgPtr := pathArgAny.(*string) pathArgs = append(pathArgs, *pathArgPtr) } projectName := u.Query().Get("project") if projectName == "" { projectName = "default" } location := u.Query().Get("target") if entityType == TypeProject { return TypeProject, pathArgs[0], location, pathArgs, nil } return entityType, projectName, location, pathArgs, nil } } return -1, "", "", nil, fmt.Errorf("Unknown entity URL %q", u.String()) } incus-7.0.0/internal/server/db/cluster/entities_test.go000066400000000000000000000140501517523235500232510ustar00rootroot00000000000000package cluster import ( "testing" "github.com/stretchr/testify/assert" ) func TestURLToEntityType(t *testing.T) { tests := []struct { name string rawURL string expectedEntityType int expectedProject string expectedLocation string expectedPathArgs []string expectedErr error }{ { name: "images", rawURL: "/1.0/images/fwirnoaiwnerfoiawnef", expectedEntityType: TypeImage, expectedProject: "default", expectedPathArgs: []string{"fwirnoaiwnerfoiawnef"}, expectedErr: nil, }, { name: "profiles", rawURL: "/1.0/profiles/my-profile?project=my-project", expectedEntityType: TypeProfile, expectedProject: "my-project", expectedPathArgs: []string{"my-profile"}, expectedErr: nil, }, { name: "projects", rawURL: "/1.0/projects/my-project", expectedEntityType: TypeProject, expectedProject: "my-project", expectedPathArgs: []string{"my-project"}, expectedErr: nil, }, { name: "certificates", rawURL: "/1.0/certificates/foawienfoawnefkanwelfknsfl", expectedEntityType: TypeCertificate, expectedProject: "default", expectedPathArgs: []string{"foawienfoawnefkanwelfknsfl"}, expectedErr: nil, }, { name: "instances", rawURL: "/1.0/instances/my-instance", expectedEntityType: TypeInstance, expectedProject: "default", expectedPathArgs: []string{"my-instance"}, expectedErr: nil, }, { name: "instance backup", rawURL: "/1.0/instances/my-instance/backups/my-backup?project=my-project", expectedEntityType: TypeInstanceBackup, expectedProject: "my-project", expectedPathArgs: []string{"my-instance", "my-backup"}, expectedErr: nil, }, { name: "instance snapshot", rawURL: "/1.0/instances/my-instance/snapshots/my-snapshot", expectedEntityType: TypeInstanceSnapshot, expectedProject: "default", expectedPathArgs: []string{"my-instance", "my-snapshot"}, expectedErr: nil, }, { name: "networks", rawURL: "/1.0/networks/my-network?project=my-project", expectedEntityType: TypeNetwork, expectedProject: "my-project", expectedPathArgs: []string{"my-network"}, expectedErr: nil, }, { name: "network acls", rawURL: "/1.0/network-acls/my-network-acl", expectedEntityType: TypeNetworkACL, expectedProject: "default", expectedPathArgs: []string{"my-network-acl"}, expectedErr: nil, }, { name: "cluster members", rawURL: "/1.0/cluster/members/node01", expectedEntityType: TypeNode, expectedProject: "default", expectedPathArgs: []string{"node01"}, expectedErr: nil, }, { name: "operation", rawURL: "/1.0/operations/3e75d1bf-30ed-45ce-9e02-267fa7338eb4", expectedEntityType: TypeOperation, expectedProject: "default", expectedPathArgs: []string{"3e75d1bf-30ed-45ce-9e02-267fa7338eb4"}, expectedErr: nil, }, { name: "storage pools", rawURL: "/1.0/storage-pools/my-storage-pool", expectedEntityType: TypeStoragePool, expectedProject: "default", expectedPathArgs: []string{"my-storage-pool"}, expectedErr: nil, }, { name: "storage volumes", rawURL: "/1.0/storage-pools/my-storage-pool/volumes/custom/my-storage-volume?project=my-project", expectedEntityType: TypeStorageVolume, expectedProject: "my-project", expectedPathArgs: []string{"my-storage-pool", "custom", "my-storage-volume"}, expectedErr: nil, }, { name: "storage volumes", rawURL: "/1.0/storage-pools/my-storage-pool/volumes/custom/my-storage-volume?project=my-project&target=foo", expectedEntityType: TypeStorageVolume, expectedProject: "my-project", expectedLocation: "foo", expectedPathArgs: []string{"my-storage-pool", "custom", "my-storage-volume"}, expectedErr: nil, }, { name: "storage volume backups", rawURL: "/1.0/storage-pools/my-storage-pool/volumes/custom/my-storage-volume/backups/my-backup?project=my-project", expectedEntityType: TypeStorageVolumeBackup, expectedProject: "my-project", expectedPathArgs: []string{"my-storage-pool", "custom", "my-storage-volume", "my-backup"}, expectedErr: nil, }, { name: "storage volume snapshots", rawURL: "/1.0/storage-pools/my-storage-pool/volumes/custom/my-storage-volume/snapshots/my-snapshot?project=my-project", expectedEntityType: TypeStorageVolumeSnapshot, expectedProject: "my-project", expectedPathArgs: []string{"my-storage-pool", "custom", "my-storage-volume", "my-snapshot"}, expectedErr: nil, }, { name: "warnings", rawURL: "/1.0/warnings/3e75d1bf-30ed-45ce-9e02-267fa7338eb4", expectedEntityType: TypeWarning, expectedProject: "default", expectedPathArgs: []string{"3e75d1bf-30ed-45ce-9e02-267fa7338eb4"}, expectedErr: nil, }, { name: "cluster groups", rawURL: "/1.0/cluster/groups/my-cluster-group", expectedEntityType: TypeClusterGroup, expectedProject: "default", expectedPathArgs: []string{"my-cluster-group"}, expectedErr: nil, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { actualEntityType, actualProject, actualLocation, actualPathArgs, actualErr := URLToEntityType(tt.rawURL) assert.Equal(t, tt.expectedEntityType, actualEntityType) assert.Equal(t, tt.expectedProject, actualProject) assert.Equal(t, tt.expectedLocation, actualLocation) for i, pathArg := range actualPathArgs { assert.Equal(t, tt.expectedPathArgs[i], pathArg) } assert.Equal(t, tt.expectedErr, actualErr) }) } } incus-7.0.0/internal/server/db/cluster/generate.go000066400000000000000000000001351517523235500221570ustar00rootroot00000000000000package cluster //go:generate generate-database db mapper generate -b mapper_boilerplate.go incus-7.0.0/internal/server/db/cluster/images.go000066400000000000000000000027531517523235500216420ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "database/sql" "time" ) // Code generation directives. // //generate-database:mapper target images.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e image objects //generate-database:mapper stmt -e image objects-by-ID //generate-database:mapper stmt -e image objects-by-Project //generate-database:mapper stmt -e image objects-by-Project-and-Cached //generate-database:mapper stmt -e image objects-by-Project-and-Public //generate-database:mapper stmt -e image objects-by-Fingerprint //generate-database:mapper stmt -e image objects-by-Cached //generate-database:mapper stmt -e image objects-by-AutoUpdate // //generate-database:mapper method -i -e image GetMany //generate-database:mapper method -i -e image GetOne // Image is a value object holding db-related details about an image. type Image struct { ID int Project string `db:"primary=yes&join=projects.name"` Fingerprint string `db:"primary=yes"` Type int Filename string Size int64 Public bool Architecture int CreationDate sql.NullTime ExpiryDate sql.NullTime UploadDate time.Time Cached bool LastUseDate sql.NullTime AutoUpdate bool } // ImageFilter can be used to filter results yielded by GetImages. type ImageFilter struct { ID *int Project *string Fingerprint *string Public *bool Cached *bool AutoUpdate *bool } incus-7.0.0/internal/server/db/cluster/images.interface.mapper.go000066400000000000000000000007611517523235500250610ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // ImageGenerated is an interface of generated methods for Image. type ImageGenerated interface { // GetImages returns all available images. // generator: image GetMany GetImages(ctx context.Context, db dbtx, filters ...ImageFilter) ([]Image, error) // GetImage returns the image with the given key. // generator: image GetOne GetImage(ctx context.Context, db dbtx, project string, fingerprint string) (*Image, error) } incus-7.0.0/internal/server/db/cluster/images.mapper.go000066400000000000000000000317021517523235500231210ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var imageObjects = RegisterStmt(` SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update FROM images JOIN projects ON images.project_id = projects.id ORDER BY projects.id, images.fingerprint `) var imageObjectsByID = RegisterStmt(` SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update FROM images JOIN projects ON images.project_id = projects.id WHERE ( images.id = ? ) ORDER BY projects.id, images.fingerprint `) var imageObjectsByProject = RegisterStmt(` SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update FROM images JOIN projects ON images.project_id = projects.id WHERE ( project = ? ) ORDER BY projects.id, images.fingerprint `) var imageObjectsByProjectAndCached = RegisterStmt(` SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update FROM images JOIN projects ON images.project_id = projects.id WHERE ( project = ? AND images.cached = ? ) ORDER BY projects.id, images.fingerprint `) var imageObjectsByProjectAndPublic = RegisterStmt(` SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update FROM images JOIN projects ON images.project_id = projects.id WHERE ( project = ? AND images.public = ? ) ORDER BY projects.id, images.fingerprint `) var imageObjectsByFingerprint = RegisterStmt(` SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update FROM images JOIN projects ON images.project_id = projects.id WHERE ( images.fingerprint = ? ) ORDER BY projects.id, images.fingerprint `) var imageObjectsByCached = RegisterStmt(` SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update FROM images JOIN projects ON images.project_id = projects.id WHERE ( images.cached = ? ) ORDER BY projects.id, images.fingerprint `) var imageObjectsByAutoUpdate = RegisterStmt(` SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update FROM images JOIN projects ON images.project_id = projects.id WHERE ( images.auto_update = ? ) ORDER BY projects.id, images.fingerprint `) // imageColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the Image entity. func imageColumns() string { return "images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update" } // getImages can be used to run handwritten sql.Stmts to return a slice of objects. func getImages(ctx context.Context, stmt *sql.Stmt, args ...any) ([]Image, error) { objects := make([]Image, 0) dest := func(scan func(dest ...any) error) error { i := Image{} err := scan(&i.ID, &i.Project, &i.Fingerprint, &i.Type, &i.Filename, &i.Size, &i.Public, &i.Architecture, &i.CreationDate, &i.ExpiryDate, &i.UploadDate, &i.Cached, &i.LastUseDate, &i.AutoUpdate) if err != nil { return err } objects = append(objects, i) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"images\" table: %w", err) } return objects, nil } // getImagesRaw can be used to run handwritten query strings to return a slice of objects. func getImagesRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]Image, error) { objects := make([]Image, 0) dest := func(scan func(dest ...any) error) error { i := Image{} err := scan(&i.ID, &i.Project, &i.Fingerprint, &i.Type, &i.Filename, &i.Size, &i.Public, &i.Architecture, &i.CreationDate, &i.ExpiryDate, &i.UploadDate, &i.Cached, &i.LastUseDate, &i.AutoUpdate) if err != nil { return err } objects = append(objects, i) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"images\" table: %w", err) } return objects, nil } // GetImages returns all available images. // generator: image GetMany func GetImages(ctx context.Context, db dbtx, filters ...ImageFilter) (_ []Image, _err error) { defer func() { _err = mapErr(_err, "Image") }() var err error // Result slice. objects := make([]Image, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, imageObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.Project != nil && filter.Public != nil && filter.ID == nil && filter.Fingerprint == nil && filter.Cached == nil && filter.AutoUpdate == nil { args = append(args, []any{filter.Project, filter.Public}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, imageObjectsByProjectAndPublic) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjectsByProjectAndPublic\" prepared statement: %w", err) } break } query, err := StmtString(imageObjectsByProjectAndPublic) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.Cached != nil && filter.ID == nil && filter.Fingerprint == nil && filter.Public == nil && filter.AutoUpdate == nil { args = append(args, []any{filter.Project, filter.Cached}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, imageObjectsByProjectAndCached) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjectsByProjectAndCached\" prepared statement: %w", err) } break } query, err := StmtString(imageObjectsByProjectAndCached) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.ID == nil && filter.Fingerprint == nil && filter.Public == nil && filter.Cached == nil && filter.AutoUpdate == nil { args = append(args, []any{filter.Project}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, imageObjectsByProject) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjectsByProject\" prepared statement: %w", err) } break } query, err := StmtString(imageObjectsByProject) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID != nil && filter.Project == nil && filter.Fingerprint == nil && filter.Public == nil && filter.Cached == nil && filter.AutoUpdate == nil { args = append(args, []any{filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, imageObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjectsByID\" prepared statement: %w", err) } break } query, err := StmtString(imageObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Fingerprint != nil && filter.ID == nil && filter.Project == nil && filter.Public == nil && filter.Cached == nil && filter.AutoUpdate == nil { args = append(args, []any{filter.Fingerprint}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, imageObjectsByFingerprint) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjectsByFingerprint\" prepared statement: %w", err) } break } query, err := StmtString(imageObjectsByFingerprint) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Cached != nil && filter.ID == nil && filter.Project == nil && filter.Fingerprint == nil && filter.Public == nil && filter.AutoUpdate == nil { args = append(args, []any{filter.Cached}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, imageObjectsByCached) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjectsByCached\" prepared statement: %w", err) } break } query, err := StmtString(imageObjectsByCached) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.AutoUpdate != nil && filter.ID == nil && filter.Project == nil && filter.Fingerprint == nil && filter.Public == nil && filter.Cached == nil { args = append(args, []any{filter.AutoUpdate}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, imageObjectsByAutoUpdate) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjectsByAutoUpdate\" prepared statement: %w", err) } break } query, err := StmtString(imageObjectsByAutoUpdate) if err != nil { return nil, fmt.Errorf("Failed to get \"imageObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.Project == nil && filter.Fingerprint == nil && filter.Public == nil && filter.Cached == nil && filter.AutoUpdate == nil { return nil, fmt.Errorf("Cannot filter on empty ImageFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getImages(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getImagesRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"images\" table: %w", err) } return objects, nil } // GetImage returns the image with the given key. // generator: image GetOne func GetImage(ctx context.Context, db dbtx, project string, fingerprint string) (_ *Image, _err error) { defer func() { _err = mapErr(_err, "Image") }() filter := ImageFilter{} filter.Project = &project filter.Fingerprint = &fingerprint objects, err := GetImages(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"images\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"images\" entry matches") } } incus-7.0.0/internal/server/db/cluster/instance_profiles.go000066400000000000000000000043071517523235500241010ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "context" "database/sql" "fmt" ) // Code generation directives. // //generate-database:mapper target instance_profiles.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e instance_profile objects //generate-database:mapper stmt -e instance_profile objects-by-ProfileID //generate-database:mapper stmt -e instance_profile objects-by-InstanceID //generate-database:mapper stmt -e instance_profile create //generate-database:mapper stmt -e instance_profile delete-by-InstanceID // //generate-database:mapper method -i -e instance_profile GetMany struct=Profile //generate-database:mapper method -i -e instance_profile GetMany struct=Instance //generate-database:mapper method -i -e instance_profile Create struct=Instance //generate-database:mapper method -i -e instance_profile DeleteMany struct=Instance // InstanceProfile is an association table struct that associates Instances // to Profiles. type InstanceProfile struct { InstanceID int `db:"primary=yes&order=yes"` ProfileID int ApplyOrder int `db:"order=yes"` } // InstanceProfileFilter specifies potential query parameter fields. type InstanceProfileFilter struct { InstanceID *int ProfileID *int } // UpdateInstanceProfiles updates the profiles of an instance in the order they are given. func UpdateInstanceProfiles(ctx context.Context, tx *sql.Tx, instanceID int, projectName string, profiles []string) error { err := DeleteInstanceProfiles(ctx, tx, instanceID) if err != nil { return err } project := projectName enabled, err := ProjectHasProfiles(ctx, tx, project) if err != nil { return fmt.Errorf("Check if project has profiles: %w", err) } if !enabled { project = "default" } applyOrder := 1 stmt, err := Stmt(tx, instanceProfileCreate) if err != nil { return fmt.Errorf("Failed to get \"instanceProfileCreate\" prepared statement: %w", err) } for _, name := range profiles { profileID, err := GetProfileID(ctx, tx, project, name) if err != nil { return err } _, err = stmt.Exec(instanceID, profileID, applyOrder) if err != nil { return err } applyOrder = applyOrder + 1 } return nil } incus-7.0.0/internal/server/db/cluster/instance_profiles.interface.mapper.go000066400000000000000000000017511517523235500273230ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // InstanceProfileGenerated is an interface of generated methods for InstanceProfile. type InstanceProfileGenerated interface { // GetProfileInstances returns all available Instances for the Profile. // generator: instance_profile GetMany GetProfileInstances(ctx context.Context, db tx, profileID int) ([]Instance, error) // GetInstanceProfiles returns all available Profiles for the Instance. // generator: instance_profile GetMany GetInstanceProfiles(ctx context.Context, db tx, instanceID int) ([]Profile, error) // CreateInstanceProfiles adds a new instance_profile to the database. // generator: instance_profile Create CreateInstanceProfiles(ctx context.Context, db tx, objects []InstanceProfile) error // DeleteInstanceProfiles deletes the instance_profile matching the given key parameters. // generator: instance_profile DeleteMany DeleteInstanceProfiles(ctx context.Context, db tx, instanceID int) error } incus-7.0.0/internal/server/db/cluster/instance_profiles.mapper.go000066400000000000000000000147121517523235500253650ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "fmt" "strings" ) var instanceProfileObjects = RegisterStmt(` SELECT instances_profiles.instance_id, instances_profiles.profile_id, instances_profiles.apply_order FROM instances_profiles ORDER BY instances_profiles.instance_id, instances_profiles.apply_order `) var instanceProfileObjectsByProfileID = RegisterStmt(` SELECT instances_profiles.instance_id, instances_profiles.profile_id, instances_profiles.apply_order FROM instances_profiles WHERE ( instances_profiles.profile_id = ? ) ORDER BY instances_profiles.instance_id, instances_profiles.apply_order `) var instanceProfileObjectsByInstanceID = RegisterStmt(` SELECT instances_profiles.instance_id, instances_profiles.profile_id, instances_profiles.apply_order FROM instances_profiles WHERE ( instances_profiles.instance_id = ? ) ORDER BY instances_profiles.instance_id, instances_profiles.apply_order `) var instanceProfileCreate = RegisterStmt(` INSERT INTO instances_profiles (instance_id, profile_id, apply_order) VALUES (?, ?, ?) `) var instanceProfileDeleteByInstanceID = RegisterStmt(` DELETE FROM instances_profiles WHERE instance_id = ? `) // GetProfileInstances returns all available Instances for the Profile. // generator: instance_profile GetMany func GetProfileInstances(ctx context.Context, db tx, profileID int) (_ []Instance, _err error) { defer func() { _err = mapErr(_err, "Instance_profile") }() var err error // Result slice. objects := make([]InstanceProfile, 0) sqlStmt, err := Stmt(db, instanceProfileObjectsByProfileID) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceProfileObjectsByProfileID\" prepared statement: %w", err) } args := []any{profileID} // Select. objects, err = getInstanceProfiles(ctx, sqlStmt, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"instances_profiles\" table: %w", err) } result := make([]Instance, len(objects)) for i, object := range objects { instance, err := GetInstances(ctx, db, InstanceFilter{ID: &object.InstanceID}) if err != nil { return nil, err } result[i] = instance[0] } return result, nil } // instanceProfileColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the InstanceProfile entity. func instanceProfileColumns() string { return "instances_profiles.instance_id, instances_profiles.profile_id, instances_profiles.apply_order" } // getInstanceProfiles can be used to run handwritten sql.Stmts to return a slice of objects. func getInstanceProfiles(ctx context.Context, stmt *sql.Stmt, args ...any) ([]InstanceProfile, error) { objects := make([]InstanceProfile, 0) dest := func(scan func(dest ...any) error) error { i := InstanceProfile{} err := scan(&i.InstanceID, &i.ProfileID, &i.ApplyOrder) if err != nil { return err } objects = append(objects, i) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"instances_profiles\" table: %w", err) } return objects, nil } // getInstanceProfilesRaw can be used to run handwritten query strings to return a slice of objects. func getInstanceProfilesRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]InstanceProfile, error) { objects := make([]InstanceProfile, 0) dest := func(scan func(dest ...any) error) error { i := InstanceProfile{} err := scan(&i.InstanceID, &i.ProfileID, &i.ApplyOrder) if err != nil { return err } objects = append(objects, i) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"instances_profiles\" table: %w", err) } return objects, nil } // GetInstanceProfiles returns all available Profiles for the Instance. // generator: instance_profile GetMany func GetInstanceProfiles(ctx context.Context, db tx, instanceID int) (_ []Profile, _err error) { defer func() { _err = mapErr(_err, "Instance_profile") }() var err error // Result slice. objects := make([]InstanceProfile, 0) sqlStmt, err := Stmt(db, instanceProfileObjectsByInstanceID) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceProfileObjectsByInstanceID\" prepared statement: %w", err) } args := []any{instanceID} // Select. objects, err = getInstanceProfiles(ctx, sqlStmt, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"instances_profiles\" table: %w", err) } result := make([]Profile, len(objects)) for i, object := range objects { profile, err := GetProfiles(ctx, db, ProfileFilter{ID: &object.ProfileID}) if err != nil { return nil, err } result[i] = profile[0] } return result, nil } // CreateInstanceProfiles adds a new instance_profile to the database. // generator: instance_profile Create func CreateInstanceProfiles(ctx context.Context, db tx, objects []InstanceProfile) (_err error) { defer func() { _err = mapErr(_err, "Instance_profile") }() for _, object := range objects { args := make([]any, 3) // Populate the statement arguments. args[0] = object.InstanceID args[1] = object.ProfileID args[2] = object.ApplyOrder // Prepared statement to use. stmt, err := Stmt(db, instanceProfileCreate) if err != nil { return fmt.Errorf("Failed to get \"instanceProfileCreate\" prepared statement: %w", err) } // Execute the statement. _, err = stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return ErrConflict } if err != nil { return fmt.Errorf("Failed to create \"instances_profiles\" entry: %w", err) } } return nil } // DeleteInstanceProfiles deletes the instance_profile matching the given key parameters. // generator: instance_profile DeleteMany func DeleteInstanceProfiles(ctx context.Context, db tx, instanceID int) (_err error) { defer func() { _err = mapErr(_err, "Instance_profile") }() stmt, err := Stmt(db, instanceProfileDeleteByInstanceID) if err != nil { return fmt.Errorf("Failed to get \"instanceProfileDeleteByInstanceID\" prepared statement: %w", err) } result, err := stmt.Exec(int(instanceID)) if err != nil { return fmt.Errorf("Delete \"instances_profiles\" entry failed: %w", err) } _, err = result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } return nil } incus-7.0.0/internal/server/db/cluster/instances.go000066400000000000000000000132131517523235500223550ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "context" "database/sql" "time" "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/osarch" ) // Code generation directives. // //generate-database:mapper target instances.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e instance objects //generate-database:mapper stmt -e instance objects-by-ID //generate-database:mapper stmt -e instance objects-by-Project //generate-database:mapper stmt -e instance objects-by-Project-and-Type //generate-database:mapper stmt -e instance objects-by-Project-and-Type-and-Node //generate-database:mapper stmt -e instance objects-by-Project-and-Type-and-Node-and-Name //generate-database:mapper stmt -e instance objects-by-Project-and-Type-and-Name //generate-database:mapper stmt -e instance objects-by-Project-and-Name //generate-database:mapper stmt -e instance objects-by-Project-and-Name-and-Node //generate-database:mapper stmt -e instance objects-by-Project-and-Node //generate-database:mapper stmt -e instance objects-by-Type //generate-database:mapper stmt -e instance objects-by-Type-and-Name //generate-database:mapper stmt -e instance objects-by-Type-and-Name-and-Node //generate-database:mapper stmt -e instance objects-by-Type-and-Node //generate-database:mapper stmt -e instance objects-by-Node //generate-database:mapper stmt -e instance objects-by-Node-and-Name //generate-database:mapper stmt -e instance objects-by-Name //generate-database:mapper stmt -e instance id //generate-database:mapper stmt -e instance create //generate-database:mapper stmt -e instance rename //generate-database:mapper stmt -e instance delete-by-Project-and-Name //generate-database:mapper stmt -e instance update // //generate-database:mapper method -i -e instance GetMany references=Config,Device //generate-database:mapper method -i -e instance GetOne //generate-database:mapper method -i -e instance ID //generate-database:mapper method -i -e instance Exists //generate-database:mapper method -i -e instance Create references=Config,Device //generate-database:mapper method -i -e instance Rename //generate-database:mapper method -i -e instance DeleteOne-by-Project-and-Name //generate-database:mapper method -i -e instance Update references=Config,Device // Instance is a value object holding db-related details about an instance. type Instance struct { ID int Project string `db:"primary=yes&join=projects.name"` Name string `db:"primary=yes"` Node string `db:"join=nodes.name"` Type instancetype.Type Snapshot bool `db:"ignore"` Architecture int Ephemeral bool CreationDate time.Time Stateful bool LastUseDate sql.NullTime Description string `db:"coalesce=''"` ExpiryDate sql.NullTime } // InstanceFilter specifies potential query parameter fields. type InstanceFilter struct { ID *int Project *string Name *string Node *string Type *instancetype.Type } // ToAPI converts the database Instance to API type. func (i *Instance) ToAPI(ctx context.Context, tx *sql.Tx, instanceDevices map[int][]Device, profileConfigs map[int]map[string]string, profileDevices map[int][]Device) (*api.Instance, error) { profiles, err := GetInstanceProfiles(ctx, tx, i.ID) if err != nil { return nil, err } if profileConfigs == nil { profileConfigs, err = GetAllProfileConfigs(ctx, tx) if err != nil { return nil, err } } if profileDevices == nil { profileDevices, err = GetAllProfileDevices(ctx, tx) if err != nil { return nil, err } } apiProfiles := make([]api.Profile, 0, len(profiles)) profileNames := make([]string, 0, len(profiles)) for _, p := range profiles { apiProfile, err := p.ToAPI(ctx, tx, profileConfigs, profileDevices) if err != nil { return nil, err } apiProfiles = append(apiProfiles, *apiProfile) profileNames = append(profileNames, p.Name) } var devices map[string]Device if instanceDevices != nil { devices = map[string]Device{} for _, dev := range instanceDevices[i.ID] { devices[dev.Name] = dev } } else { devices, err = GetInstanceDevices(ctx, tx, i.ID) if err != nil { return nil, err } } apiDevices := DevicesToAPI(devices) expandedDevices := ExpandInstanceDevices(config.NewDevices(apiDevices), apiProfiles) config, err := GetInstanceConfig(ctx, tx, i.ID) if err != nil { return nil, err } expandedConfig := ExpandInstanceConfig(config, apiProfiles) archName, err := osarch.ArchitectureName(i.Architecture) if err != nil { return nil, err } return &api.Instance{ InstancePut: api.InstancePut{ Architecture: archName, Config: config, Devices: apiDevices, Ephemeral: i.Ephemeral, Profiles: profileNames, Stateful: i.Stateful, Description: i.Description, }, CreatedAt: i.CreationDate, ExpandedConfig: expandedConfig, ExpandedDevices: expandedDevices.CloneNative(), Name: i.Name, LastUsedAt: i.LastUseDate.Time, Location: i.Node, Type: i.Type.String(), Project: i.Project, }, nil } // GetAllInstanceConfigs returns a map of all instance configurations, keyed by database ID. func GetAllInstanceConfigs(ctx context.Context, tx *sql.Tx) (map[int]map[string]string, error) { return GetConfig(ctx, tx, "instances", "instance") } // GetAllInstanceDevices returns a map of all instance devices, keyed by database ID. func GetAllInstanceDevices(ctx context.Context, tx *sql.Tx) (map[int][]Device, error) { return GetDevices(ctx, tx, "instances", "instance") } incus-7.0.0/internal/server/db/cluster/instances.interface.mapper.go000066400000000000000000000056041517523235500256040ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // InstanceGenerated is an interface of generated methods for Instance. type InstanceGenerated interface { // GetInstanceConfig returns all available Instance Config // generator: instance GetMany GetInstanceConfig(ctx context.Context, db tx, instanceID int, filters ...ConfigFilter) (map[string]string, error) // GetInstanceDevices returns all available Instance Devices // generator: instance GetMany GetInstanceDevices(ctx context.Context, db tx, instanceID int, filters ...DeviceFilter) (map[string]Device, error) // GetInstances returns all available instances. // generator: instance GetMany GetInstances(ctx context.Context, db dbtx, filters ...InstanceFilter) ([]Instance, error) // GetInstance returns the instance with the given key. // generator: instance GetOne GetInstance(ctx context.Context, db dbtx, project string, name string) (*Instance, error) // GetInstanceID return the ID of the instance with the given key. // generator: instance ID GetInstanceID(ctx context.Context, db tx, project string, name string) (int64, error) // InstanceExists checks if a instance with the given key exists. // generator: instance Exists InstanceExists(ctx context.Context, db dbtx, project string, name string) (bool, error) // CreateInstanceConfig adds new instance Config to the database. // generator: instance Create CreateInstanceConfig(ctx context.Context, db dbtx, instanceID int64, config map[string]string) error // CreateInstanceDevices adds new instance Devices to the database. // generator: instance Create CreateInstanceDevices(ctx context.Context, db tx, instanceID int64, devices map[string]Device) error // CreateInstance adds a new instance to the database. // generator: instance Create CreateInstance(ctx context.Context, db dbtx, object Instance) (int64, error) // RenameInstance renames the instance matching the given key parameters. // generator: instance Rename RenameInstance(ctx context.Context, db dbtx, project string, name string, to string) error // DeleteInstance deletes the instance matching the given key parameters. // generator: instance DeleteOne-by-Project-and-Name DeleteInstance(ctx context.Context, db dbtx, project string, name string) error // UpdateInstanceConfig updates the instance Config matching the given key parameters. // generator: instance Update UpdateInstanceConfig(ctx context.Context, db tx, instanceID int64, config map[string]string) error // UpdateInstanceDevices updates the instance Device matching the given key parameters. // generator: instance Update UpdateInstanceDevices(ctx context.Context, db tx, instanceID int64, devices map[string]Device) error // UpdateInstance updates the instance matching the given key parameters. // generator: instance Update UpdateInstance(ctx context.Context, db tx, project string, name string, object Instance) error } incus-7.0.0/internal/server/db/cluster/instances.mapper.go000066400000000000000000001045721517523235500236510ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var instanceObjects = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id ORDER BY projects.id, instances.name `) var instanceObjectsByID = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( instances.id = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByProject = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( project = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByProjectAndType = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( project = ? AND instances.type = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByProjectAndTypeAndNode = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( project = ? AND instances.type = ? AND node = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByProjectAndTypeAndNodeAndName = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( project = ? AND instances.type = ? AND node = ? AND instances.name = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByProjectAndTypeAndName = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( project = ? AND instances.type = ? AND instances.name = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByProjectAndName = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( project = ? AND instances.name = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByProjectAndNameAndNode = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( project = ? AND instances.name = ? AND node = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByProjectAndNode = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( project = ? AND node = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByType = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( instances.type = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByTypeAndName = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( instances.type = ? AND instances.name = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByTypeAndNameAndNode = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( instances.type = ? AND instances.name = ? AND node = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByTypeAndNode = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( instances.type = ? AND node = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByNode = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( node = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByNodeAndName = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( node = ? AND instances.name = ? ) ORDER BY projects.id, instances.name `) var instanceObjectsByName = RegisterStmt(` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id WHERE ( instances.name = ? ) ORDER BY projects.id, instances.name `) var instanceID = RegisterStmt(` SELECT instances.id FROM instances JOIN projects ON instances.project_id = projects.id WHERE projects.name = ? AND instances.name = ? `) var instanceCreate = RegisterStmt(` INSERT INTO instances (project_id, name, node_id, type, architecture, ephemeral, creation_date, stateful, last_use_date, description, expiry_date) VALUES ((SELECT projects.id FROM projects WHERE projects.name = ?), ?, (SELECT nodes.id FROM nodes WHERE nodes.name = ?), ?, ?, ?, ?, ?, ?, ?, ?) `) var instanceRename = RegisterStmt(` UPDATE instances SET name = ? WHERE project_id = (SELECT projects.id FROM projects WHERE projects.name = ?) AND name = ? `) var instanceDeleteByProjectAndName = RegisterStmt(` DELETE FROM instances WHERE project_id = (SELECT projects.id FROM projects WHERE projects.name = ?) AND name = ? `) var instanceUpdate = RegisterStmt(` UPDATE instances SET project_id = (SELECT projects.id FROM projects WHERE projects.name = ?), name = ?, node_id = (SELECT nodes.id FROM nodes WHERE nodes.name = ?), type = ?, architecture = ?, ephemeral = ?, creation_date = ?, stateful = ?, last_use_date = ?, description = ?, expiry_date = ? WHERE id = ? `) // instanceColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the Instance entity. func instanceColumns() string { return "instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date" } // getInstances can be used to run handwritten sql.Stmts to return a slice of objects. func getInstances(ctx context.Context, stmt *sql.Stmt, args ...any) ([]Instance, error) { objects := make([]Instance, 0) dest := func(scan func(dest ...any) error) error { i := Instance{} err := scan(&i.ID, &i.Project, &i.Name, &i.Node, &i.Type, &i.Architecture, &i.Ephemeral, &i.CreationDate, &i.Stateful, &i.LastUseDate, &i.Description, &i.ExpiryDate) if err != nil { return err } objects = append(objects, i) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"instances\" table: %w", err) } return objects, nil } // getInstancesRaw can be used to run handwritten query strings to return a slice of objects. func getInstancesRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]Instance, error) { objects := make([]Instance, 0) dest := func(scan func(dest ...any) error) error { i := Instance{} err := scan(&i.ID, &i.Project, &i.Name, &i.Node, &i.Type, &i.Architecture, &i.Ephemeral, &i.CreationDate, &i.Stateful, &i.LastUseDate, &i.Description, &i.ExpiryDate) if err != nil { return err } objects = append(objects, i) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"instances\" table: %w", err) } return objects, nil } // GetInstances returns all available instances. // generator: instance GetMany func GetInstances(ctx context.Context, db dbtx, filters ...InstanceFilter) (_ []Instance, _err error) { defer func() { _err = mapErr(_err, "Instance") }() var err error // Result slice. objects := make([]Instance, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, instanceObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.Project != nil && filter.Type != nil && filter.Node != nil && filter.Name != nil && filter.ID == nil { args = append(args, []any{filter.Project, filter.Type, filter.Node, filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByProjectAndTypeAndNodeAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByProjectAndTypeAndNodeAndName\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByProjectAndTypeAndNodeAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.Type != nil && filter.Node != nil && filter.ID == nil && filter.Name == nil { args = append(args, []any{filter.Project, filter.Type, filter.Node}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByProjectAndTypeAndNode) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByProjectAndTypeAndNode\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByProjectAndTypeAndNode) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.Type != nil && filter.Name != nil && filter.ID == nil && filter.Node == nil { args = append(args, []any{filter.Project, filter.Type, filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByProjectAndTypeAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByProjectAndTypeAndName\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByProjectAndTypeAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Type != nil && filter.Name != nil && filter.Node != nil && filter.ID == nil && filter.Project == nil { args = append(args, []any{filter.Type, filter.Name, filter.Node}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByTypeAndNameAndNode) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByTypeAndNameAndNode\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByTypeAndNameAndNode) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.Name != nil && filter.Node != nil && filter.ID == nil && filter.Type == nil { args = append(args, []any{filter.Project, filter.Name, filter.Node}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByProjectAndNameAndNode) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByProjectAndNameAndNode\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByProjectAndNameAndNode) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.Type != nil && filter.ID == nil && filter.Name == nil && filter.Node == nil { args = append(args, []any{filter.Project, filter.Type}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByProjectAndType) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByProjectAndType\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByProjectAndType) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Type != nil && filter.Node != nil && filter.ID == nil && filter.Project == nil && filter.Name == nil { args = append(args, []any{filter.Type, filter.Node}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByTypeAndNode) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByTypeAndNode\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByTypeAndNode) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Type != nil && filter.Name != nil && filter.ID == nil && filter.Project == nil && filter.Node == nil { args = append(args, []any{filter.Type, filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByTypeAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByTypeAndName\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByTypeAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.Node != nil && filter.ID == nil && filter.Name == nil && filter.Type == nil { args = append(args, []any{filter.Project, filter.Node}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByProjectAndNode) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByProjectAndNode\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByProjectAndNode) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.Name != nil && filter.ID == nil && filter.Node == nil && filter.Type == nil { args = append(args, []any{filter.Project, filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByProjectAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByProjectAndName\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByProjectAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Node != nil && filter.Name != nil && filter.ID == nil && filter.Project == nil && filter.Type == nil { args = append(args, []any{filter.Node, filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByNodeAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByNodeAndName\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByNodeAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Type != nil && filter.ID == nil && filter.Project == nil && filter.Name == nil && filter.Node == nil { args = append(args, []any{filter.Type}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByType) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByType\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByType) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.ID == nil && filter.Name == nil && filter.Node == nil && filter.Type == nil { args = append(args, []any{filter.Project}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByProject) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByProject\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByProject) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Node != nil && filter.ID == nil && filter.Project == nil && filter.Name == nil && filter.Type == nil { args = append(args, []any{filter.Node}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByNode) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByNode\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByNode) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Name != nil && filter.ID == nil && filter.Project == nil && filter.Node == nil && filter.Type == nil { args = append(args, []any{filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByName\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID != nil && filter.Project == nil && filter.Name == nil && filter.Node == nil && filter.Type == nil { args = append(args, []any{filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjectsByID\" prepared statement: %w", err) } break } query, err := StmtString(instanceObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.Project == nil && filter.Name == nil && filter.Node == nil && filter.Type == nil { return nil, fmt.Errorf("Cannot filter on empty InstanceFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getInstances(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getInstancesRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"instances\" table: %w", err) } return objects, nil } // GetInstanceDevices returns all available Instance Devices // generator: instance GetMany func GetInstanceDevices(ctx context.Context, db tx, instanceID int, filters ...DeviceFilter) (_ map[string]Device, _err error) { defer func() { _err = mapErr(_err, "Instance") }() instanceDevices, err := GetDevices(ctx, db, "instances", "instance", filters...) if err != nil { return nil, err } devices := map[string]Device{} for _, ref := range instanceDevices[instanceID] { _, ok := devices[ref.Name] if !ok { devices[ref.Name] = ref } else { return nil, fmt.Errorf("Found duplicate Device with name %q", ref.Name) } } return devices, nil } // GetInstanceConfig returns all available Instance Config // generator: instance GetMany func GetInstanceConfig(ctx context.Context, db tx, instanceID int, filters ...ConfigFilter) (_ map[string]string, _err error) { defer func() { _err = mapErr(_err, "Instance") }() instanceConfig, err := GetConfig(ctx, db, "instances", "instance", filters...) if err != nil { return nil, err } config, ok := instanceConfig[instanceID] if !ok { config = map[string]string{} } return config, nil } // GetInstance returns the instance with the given key. // generator: instance GetOne func GetInstance(ctx context.Context, db dbtx, project string, name string) (_ *Instance, _err error) { defer func() { _err = mapErr(_err, "Instance") }() filter := InstanceFilter{} filter.Project = &project filter.Name = &name objects, err := GetInstances(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"instances\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"instances\" entry matches") } } // GetInstanceID return the ID of the instance with the given key. // generator: instance ID func GetInstanceID(ctx context.Context, db tx, project string, name string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Instance") }() stmt, err := Stmt(db, instanceID) if err != nil { return -1, fmt.Errorf("Failed to get \"instanceID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, project, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"instances\" ID: %w", err) } return id, nil } // InstanceExists checks if a instance with the given key exists. // generator: instance Exists func InstanceExists(ctx context.Context, db dbtx, project string, name string) (_ bool, _err error) { defer func() { _err = mapErr(_err, "Instance") }() stmt, err := Stmt(db, instanceID) if err != nil { return false, fmt.Errorf("Failed to get \"instanceID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, project, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("Failed to get \"instances\" ID: %w", err) } return true, nil } // CreateInstance adds a new instance to the database. // generator: instance Create func CreateInstance(ctx context.Context, db dbtx, object Instance) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Instance") }() args := make([]any, 11) // Populate the statement arguments. args[0] = object.Project args[1] = object.Name args[2] = object.Node args[3] = object.Type args[4] = object.Architecture args[5] = object.Ephemeral args[6] = object.CreationDate args[7] = object.Stateful args[8] = object.LastUseDate args[9] = object.Description args[10] = object.ExpiryDate // Prepared statement to use. stmt, err := Stmt(db, instanceCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"instanceCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"instances\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"instances\" entry ID: %w", err) } return id, nil } // CreateInstanceDevices adds new instance Devices to the database. // generator: instance Create func CreateInstanceDevices(ctx context.Context, db tx, instanceID int64, devices map[string]Device) (_err error) { defer func() { _err = mapErr(_err, "Instance") }() for key, device := range devices { device.ReferenceID = int(instanceID) devices[key] = device } err := CreateDevices(ctx, db, "instances", "instance", devices) if err != nil { return fmt.Errorf("Insert Device failed for Instance: %w", err) } return nil } // CreateInstanceConfig adds new instance Config to the database. // generator: instance Create func CreateInstanceConfig(ctx context.Context, db dbtx, instanceID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Instance") }() referenceID := int(instanceID) for key, value := range config { insert := Config{ ReferenceID: referenceID, Key: key, Value: value, } err := CreateConfig(ctx, db, "instances", "instance", insert) if err != nil { return fmt.Errorf("Insert Config failed for Instance: %w", err) } } return nil } // RenameInstance renames the instance matching the given key parameters. // generator: instance Rename func RenameInstance(ctx context.Context, db dbtx, project string, name string, to string) (_err error) { defer func() { _err = mapErr(_err, "Instance") }() stmt, err := Stmt(db, instanceRename) if err != nil { return fmt.Errorf("Failed to get \"instanceRename\" prepared statement: %w", err) } result, err := stmt.Exec(to, project, name) if err != nil { return fmt.Errorf("Rename Instance failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows failed: %w", err) } if n != 1 { return fmt.Errorf("Query affected %d rows instead of 1", n) } return nil } // DeleteInstance deletes the instance matching the given key parameters. // generator: instance DeleteOne-by-Project-and-Name func DeleteInstance(ctx context.Context, db dbtx, project string, name string) (_err error) { defer func() { _err = mapErr(_err, "Instance") }() stmt, err := Stmt(db, instanceDeleteByProjectAndName) if err != nil { return fmt.Errorf("Failed to get \"instanceDeleteByProjectAndName\" prepared statement: %w", err) } result, err := stmt.Exec(project, name) if err != nil { return fmt.Errorf("Delete \"instances\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d Instance rows instead of 1", n) } return nil } // UpdateInstance updates the instance matching the given key parameters. // generator: instance Update func UpdateInstance(ctx context.Context, db tx, project string, name string, object Instance) (_err error) { defer func() { _err = mapErr(_err, "Instance") }() id, err := GetInstanceID(ctx, db, project, name) if err != nil { return err } stmt, err := Stmt(db, instanceUpdate) if err != nil { return fmt.Errorf("Failed to get \"instanceUpdate\" prepared statement: %w", err) } result, err := stmt.Exec(object.Project, object.Name, object.Node, object.Type, object.Architecture, object.Ephemeral, object.CreationDate, object.Stateful, object.LastUseDate, object.Description, object.ExpiryDate, id) if err != nil { return fmt.Errorf("Update \"instances\" entry failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } return nil } // UpdateInstanceDevices updates the instance Device matching the given key parameters. // generator: instance Update func UpdateInstanceDevices(ctx context.Context, db tx, instanceID int64, devices map[string]Device) (_err error) { defer func() { _err = mapErr(_err, "Instance") }() err := UpdateDevices(ctx, db, "instances", "instance", int(instanceID), devices) if err != nil { return fmt.Errorf("Replace Device for Instance failed: %w", err) } return nil } // UpdateInstanceConfig updates the instance Config matching the given key parameters. // generator: instance Update func UpdateInstanceConfig(ctx context.Context, db tx, instanceID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Instance") }() err := UpdateConfig(ctx, db, "instances", "instance", int(instanceID), config) if err != nil { return fmt.Errorf("Replace Config for Instance failed: %w", err) } return nil } incus-7.0.0/internal/server/db/cluster/mapper_boilerplate.go000066400000000000000000000115421517523235500242370ustar00rootroot00000000000000// Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "encoding/json" "errors" "fmt" ) type tx interface { //nolint:unused dbtx Commit() error Rollback() error } type dbtx interface { ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) PrepareContext(ctx context.Context, query string) (*sql.Stmt, error) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row } type preparer interface { Prepare(query string) (*sql.Stmt, error) } // RegisterStmt register a SQL statement. // // Registered statements will be prepared upfront and reused, to speed up // execution. // // Return a unique registration code. func RegisterStmt(sqlStmt string) int { code := len(stmts) stmts[code] = sqlStmt return code } // PrepareStmts prepares all registered statements and returns an index from // statement code to prepared statement object. func PrepareStmts(db preparer, skipErrors bool) (map[int]*sql.Stmt, error) { index := map[int]*sql.Stmt{} for code, sqlStmt := range stmts { stmt, err := db.Prepare(sqlStmt) if err != nil && !skipErrors { return nil, fmt.Errorf("%q: %w", sqlStmt, err) } index[code] = stmt } return index, nil } var stmts = map[int]string{} // Statement code to statement SQL text. // PreparedStmts is a placeholder for transitioning to package-scoped transaction functions. var PreparedStmts = map[int]*sql.Stmt{} // Stmt prepares the in-memory prepared statement for the transaction. func Stmt(db dbtx, code int) (*sql.Stmt, error) { stmt, ok := PreparedStmts[code] if !ok { return nil, fmt.Errorf("No prepared statement registered with code %d", code) } tx, ok := db.(*sql.Tx) if ok { return tx.Stmt(stmt), nil } return stmt, nil } // StmtString returns the in-memory query string with the given code. func StmtString(code int) (string, error) { stmt, ok := stmts[code] if !ok { return "", fmt.Errorf("No prepared statement registered with code %d", code) } return stmt, nil } var ( // ErrNotFound is the error returned, if the entity is not found in the DB. ErrNotFound = errors.New("Not found") // ErrConflict is the error returned, if the adding or updating an entity // causes a conflict with an existing entity. ErrConflict = errors.New("Conflict") ) var mapErr = defaultMapErr func defaultMapErr(err error, entity string) error { return err } // Marshaler is the interface that wraps the MarshalDB method, which converts // the underlying type into a string representation suitable for persistence in // the database. type Marshaler interface { MarshalDB() (string, error) } // Unmarshaler is the interface that wraps the UnmarshalDB method, which converts // a string representation retrieved from the database into the underlying type. type Unmarshaler interface { UnmarshalDB(string) error } func marshal(v any) (string, error) { marshaller, ok := v.(Marshaler) if !ok { return "", errors.New("Cannot marshal data, type does not implement DBMarshaler") } return marshaller.MarshalDB() } func unmarshal(data string, v any) error { if v == nil { return errors.New("Cannot unmarshal data into nil value") } unmarshaler, ok := v.(Unmarshaler) if !ok { return errors.New("Cannot marshal data, type does not implement DBUnmarshaler") } return unmarshaler.UnmarshalDB(data) } func marshalJSON(v any) (string, error) { marshalled, err := json.Marshal(v) if err != nil { return "", err } return string(marshalled), nil } func unmarshalJSON(data string, v any) error { return json.Unmarshal([]byte(data), v) } // dest is a function that is expected to return the objects to pass to the // 'dest' argument of sql.Rows.Scan(). It is invoked by SelectObjects once per // yielded row, and it will be passed the index of the row being scanned. type dest func(scan func(dest ...any) error) error // selectObjects executes a statement which must yield rows with a specific // columns schema. It invokes the given Dest hook for each yielded row. func selectObjects(ctx context.Context, stmt *sql.Stmt, rowFunc dest, args ...any) error { rows, err := stmt.QueryContext(ctx, args...) if err != nil { return err } defer func() { _ = rows.Close() }() for rows.Next() { err = rowFunc(rows.Scan) if err != nil { return err } } return rows.Err() } // scan runs a query with inArgs and provides the rowFunc with the scan function for each row. // It handles closing the rows and errors from the result set. func scan(ctx context.Context, db dbtx, sqlStmt string, rowFunc dest, inArgs ...any) error { rows, err := db.QueryContext(ctx, sqlStmt, inArgs...) if err != nil { return err } defer func() { _ = rows.Close() }() for rows.Next() { err = rowFunc(rows.Scan) if err != nil { return err } } return rows.Err() } incus-7.0.0/internal/server/db/cluster/mapper_errors.go000066400000000000000000000006601517523235500232500ustar00rootroot00000000000000package cluster import ( "errors" "net/http" "github.com/lxc/incus/v7/shared/api" ) func init() { mapErr = clusterMapErr } func clusterMapErr(err error, entity string) error { if errors.Is(err, ErrNotFound) { return api.StatusErrorf(http.StatusNotFound, "%s not found", entity) } if errors.Is(err, ErrConflict) { return api.StatusErrorf(http.StatusConflict, "This %q entry already exists", entity) } return err } incus-7.0.0/internal/server/db/cluster/networks_acls.go000066400000000000000000000103271517523235500232470ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "context" "fmt" "net/http" "github.com/lxc/incus/v7/shared/api" ) // Code generation directives. //generate-database:mapper target networks_acls.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // // Statements: //generate-database:mapper stmt -e NetworkACL objects table=networks_acls //generate-database:mapper stmt -e NetworkACL objects-by-ID table=networks_acls //generate-database:mapper stmt -e NetworkACL objects-by-Name table=networks_acls //generate-database:mapper stmt -e NetworkACL objects-by-Project table=networks_acls //generate-database:mapper stmt -e NetworkACL objects-by-Project-and-Name table=networks_acls //generate-database:mapper stmt -e NetworkACL id table=networks_acls //generate-database:mapper stmt -e NetworkACL create table=networks_acls //generate-database:mapper stmt -e NetworkACL rename table=networks_acls //generate-database:mapper stmt -e NetworkACL update table=networks_acls //generate-database:mapper stmt -e NetworkACL delete-by-ID table=networks_acls // // Methods: //generate-database:mapper method -i -e NetworkACL GetMany references=Config table=networks_acls //generate-database:mapper method -i -e NetworkACL GetOne table=networks_acls //generate-database:mapper method -i -e NetworkACL Exists table=networks_acls //generate-database:mapper method -i -e NetworkACL Create references=Config table=networks_acls //generate-database:mapper method -i -e NetworkACL ID table=networks_acls //generate-database:mapper method -i -e NetworkACL Rename table=networks_acls //generate-database:mapper method -i -e NetworkACL Update references=Config table=networks_acls //generate-database:mapper method -i -e NetworkACL DeleteOne-by-ID table=networks_acls // NetworkACL is a value object holding db-related details about a network ACL. type NetworkACL struct { ID int `db:"order=yes"` ProjectID int `db:"omit=create,update"` Project string `db:"primary=yes&join=projects.name"` Name string `db:"primary=yes"` Description string Ingress []api.NetworkACLRule `db:"marshal=json"` Egress []api.NetworkACLRule `db:"marshal=json"` } // NetworkACLFilter specifies potential query parameter fields. type NetworkACLFilter struct { ID *int Name *string Project *string } // ToAPI converts the DB record into the shared/api form. func (n *NetworkACL) ToAPI(ctx context.Context, db tx) (*api.NetworkACL, error) { cfg, err := GetNetworkACLConfig(ctx, db, n.ID) if err != nil { return nil, err } out := api.NetworkACL{ NetworkACLPost: api.NetworkACLPost{ Name: n.Name, }, NetworkACLPut: api.NetworkACLPut{ Description: n.Description, Config: cfg, Ingress: n.Ingress, Egress: n.Egress, }, } return &out, nil } // GetNetworkACLAPI returns the Network ACL API struct for the ACL with the given name in the given project. func GetNetworkACLAPI(ctx context.Context, db tx, projectName string, name string) (int, *api.NetworkACL, error) { acls, err := GetNetworkACLs(ctx, db, NetworkACLFilter{Project: &projectName, Name: &name}) if err != nil { return -1, nil, err } if len(acls) == 0 { return -1, nil, api.StatusErrorf(http.StatusNotFound, "Network ACL not found") } acl := acls[0] apiACL, err := acl.ToAPI(ctx, db) if err != nil { return -1, nil, fmt.Errorf("Failed loading config: %w", err) } return acl.ID, apiACL, nil } // UpdateNetworkACLAPI updates the Network ACL with the given ID using the provided API struct. func UpdateNetworkACLAPI(ctx context.Context, db tx, id int64, put *api.NetworkACLPut) error { // Fetch existing to recover project and name. idInt := int(id) acls, err := GetNetworkACLs(ctx, db, NetworkACLFilter{ID: &idInt}) if err != nil { return err } if len(acls) == 0 { return api.StatusErrorf(http.StatusNotFound, "Network ACL not found") } curr := acls[0] upd := NetworkACL{ Project: curr.Project, Name: curr.Name, Description: put.Description, Ingress: put.Ingress, Egress: put.Egress, } err = UpdateNetworkACL(ctx, db, curr.Project, curr.Name, upd) if err != nil { return err } err = UpdateNetworkACLConfig(ctx, db, id, put.Config) if err != nil { return err } return nil } incus-7.0.0/internal/server/db/cluster/networks_acls.interface.mapper.go000066400000000000000000000045221517523235500264710ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // NetworkACLGenerated is an interface of generated methods for NetworkACL. type NetworkACLGenerated interface { // GetNetworkACLConfig returns all available NetworkACL Config // generator: NetworkACL GetMany GetNetworkACLConfig(ctx context.Context, db tx, networkACLID int, filters ...ConfigFilter) (map[string]string, error) // GetNetworkACLs returns all available NetworkACLs. // generator: NetworkACL GetMany GetNetworkACLs(ctx context.Context, db dbtx, filters ...NetworkACLFilter) ([]NetworkACL, error) // GetNetworkACL returns the NetworkACL with the given key. // generator: NetworkACL GetOne GetNetworkACL(ctx context.Context, db dbtx, project string, name string) (*NetworkACL, error) // NetworkACLExists checks if a NetworkACL with the given key exists. // generator: NetworkACL Exists NetworkACLExists(ctx context.Context, db dbtx, project string, name string) (bool, error) // CreateNetworkACLConfig adds new NetworkACL Config to the database. // generator: NetworkACL Create CreateNetworkACLConfig(ctx context.Context, db dbtx, networkACLID int64, config map[string]string) error // CreateNetworkACL adds a new NetworkACL to the database. // generator: NetworkACL Create CreateNetworkACL(ctx context.Context, db dbtx, object NetworkACL) (int64, error) // GetNetworkACLID return the ID of the NetworkACL with the given key. // generator: NetworkACL ID GetNetworkACLID(ctx context.Context, db tx, project string, name string) (int64, error) // RenameNetworkACL renames the NetworkACL matching the given key parameters. // generator: NetworkACL Rename RenameNetworkACL(ctx context.Context, db dbtx, project string, name string, to string) error // UpdateNetworkACLConfig updates the NetworkACL Config matching the given key parameters. // generator: NetworkACL Update UpdateNetworkACLConfig(ctx context.Context, db tx, networkACLID int64, config map[string]string) error // UpdateNetworkACL updates the NetworkACL matching the given key parameters. // generator: NetworkACL Update UpdateNetworkACL(ctx context.Context, db tx, project string, name string, object NetworkACL) error // DeleteNetworkACL deletes the NetworkACL matching the given key parameters. // generator: NetworkACL DeleteOne-by-ID DeleteNetworkACL(ctx context.Context, db dbtx, id int) error } incus-7.0.0/internal/server/db/cluster/networks_acls.mapper.go000066400000000000000000000403701517523235500245330ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var networkACLObjects = RegisterStmt(` SELECT networks_acls.id, networks_acls.project_id, projects.name AS project, networks_acls.name, networks_acls.description, networks_acls.ingress, networks_acls.egress FROM networks_acls JOIN projects ON networks_acls.project_id = projects.id ORDER BY networks_acls.id `) var networkACLObjectsByID = RegisterStmt(` SELECT networks_acls.id, networks_acls.project_id, projects.name AS project, networks_acls.name, networks_acls.description, networks_acls.ingress, networks_acls.egress FROM networks_acls JOIN projects ON networks_acls.project_id = projects.id WHERE ( networks_acls.id = ? ) ORDER BY networks_acls.id `) var networkACLObjectsByName = RegisterStmt(` SELECT networks_acls.id, networks_acls.project_id, projects.name AS project, networks_acls.name, networks_acls.description, networks_acls.ingress, networks_acls.egress FROM networks_acls JOIN projects ON networks_acls.project_id = projects.id WHERE ( networks_acls.name = ? ) ORDER BY networks_acls.id `) var networkACLObjectsByProject = RegisterStmt(` SELECT networks_acls.id, networks_acls.project_id, projects.name AS project, networks_acls.name, networks_acls.description, networks_acls.ingress, networks_acls.egress FROM networks_acls JOIN projects ON networks_acls.project_id = projects.id WHERE ( project = ? ) ORDER BY networks_acls.id `) var networkACLObjectsByProjectAndName = RegisterStmt(` SELECT networks_acls.id, networks_acls.project_id, projects.name AS project, networks_acls.name, networks_acls.description, networks_acls.ingress, networks_acls.egress FROM networks_acls JOIN projects ON networks_acls.project_id = projects.id WHERE ( project = ? AND networks_acls.name = ? ) ORDER BY networks_acls.id `) var networkACLID = RegisterStmt(` SELECT networks_acls.id FROM networks_acls JOIN projects ON networks_acls.project_id = projects.id WHERE projects.name = ? AND networks_acls.name = ? `) var networkACLCreate = RegisterStmt(` INSERT INTO networks_acls (project_id, name, description, ingress, egress) VALUES ((SELECT projects.id FROM projects WHERE projects.name = ?), ?, ?, ?, ?) `) var networkACLRename = RegisterStmt(` UPDATE networks_acls SET name = ? WHERE project_id = (SELECT projects.id FROM projects WHERE projects.name = ?) AND name = ? `) var networkACLUpdate = RegisterStmt(` UPDATE networks_acls SET project_id = (SELECT projects.id FROM projects WHERE projects.name = ?), name = ?, description = ?, ingress = ?, egress = ? WHERE id = ? `) var networkACLDeleteByID = RegisterStmt(` DELETE FROM networks_acls WHERE id = ? `) // networkACLColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the NetworkACL entity. func networkACLColumns() string { return "networks_acls.id, networks_acls.project_id, projects.name AS project, networks_acls.name, networks_acls.description, networks_acls.ingress, networks_acls.egress" } // getNetworkACLs can be used to run handwritten sql.Stmts to return a slice of objects. func getNetworkACLs(ctx context.Context, stmt *sql.Stmt, args ...any) ([]NetworkACL, error) { objects := make([]NetworkACL, 0) dest := func(scan func(dest ...any) error) error { n := NetworkACL{} var ingressStr string var egressStr string err := scan(&n.ID, &n.ProjectID, &n.Project, &n.Name, &n.Description, &ingressStr, &egressStr) if err != nil { return err } err = unmarshalJSON(ingressStr, &n.Ingress) if err != nil { return err } err = unmarshalJSON(egressStr, &n.Egress) if err != nil { return err } objects = append(objects, n) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_acls\" table: %w", err) } return objects, nil } // getNetworkACLsRaw can be used to run handwritten query strings to return a slice of objects. func getNetworkACLsRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]NetworkACL, error) { objects := make([]NetworkACL, 0) dest := func(scan func(dest ...any) error) error { n := NetworkACL{} var ingressStr string var egressStr string err := scan(&n.ID, &n.ProjectID, &n.Project, &n.Name, &n.Description, &ingressStr, &egressStr) if err != nil { return err } err = unmarshalJSON(ingressStr, &n.Ingress) if err != nil { return err } err = unmarshalJSON(egressStr, &n.Egress) if err != nil { return err } objects = append(objects, n) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_acls\" table: %w", err) } return objects, nil } // GetNetworkACLs returns all available NetworkACLs. // generator: NetworkACL GetMany func GetNetworkACLs(ctx context.Context, db dbtx, filters ...NetworkACLFilter) (_ []NetworkACL, _err error) { defer func() { _err = mapErr(_err, "NetworkACL") }() var err error // Result slice. objects := make([]NetworkACL, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, networkACLObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"networkACLObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.Project != nil && filter.Name != nil && filter.ID == nil { args = append(args, []any{filter.Project, filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkACLObjectsByProjectAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkACLObjectsByProjectAndName\" prepared statement: %w", err) } break } query, err := StmtString(networkACLObjectsByProjectAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkACLObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.ID == nil && filter.Name == nil { args = append(args, []any{filter.Project}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkACLObjectsByProject) if err != nil { return nil, fmt.Errorf("Failed to get \"networkACLObjectsByProject\" prepared statement: %w", err) } break } query, err := StmtString(networkACLObjectsByProject) if err != nil { return nil, fmt.Errorf("Failed to get \"networkACLObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Name != nil && filter.ID == nil && filter.Project == nil { args = append(args, []any{filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkACLObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkACLObjectsByName\" prepared statement: %w", err) } break } query, err := StmtString(networkACLObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkACLObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID != nil && filter.Name == nil && filter.Project == nil { args = append(args, []any{filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkACLObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkACLObjectsByID\" prepared statement: %w", err) } break } query, err := StmtString(networkACLObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkACLObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.Name == nil && filter.Project == nil { return nil, fmt.Errorf("Cannot filter on empty NetworkACLFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getNetworkACLs(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getNetworkACLsRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_acls\" table: %w", err) } return objects, nil } // GetNetworkACLConfig returns all available NetworkACL Config // generator: NetworkACL GetMany func GetNetworkACLConfig(ctx context.Context, db tx, networkACLID int, filters ...ConfigFilter) (_ map[string]string, _err error) { defer func() { _err = mapErr(_err, "NetworkACL") }() networkACLConfig, err := GetConfig(ctx, db, "networks_acls", "network_acl", filters...) if err != nil { return nil, err } config, ok := networkACLConfig[networkACLID] if !ok { config = map[string]string{} } return config, nil } // GetNetworkACL returns the NetworkACL with the given key. // generator: NetworkACL GetOne func GetNetworkACL(ctx context.Context, db dbtx, project string, name string) (_ *NetworkACL, _err error) { defer func() { _err = mapErr(_err, "NetworkACL") }() filter := NetworkACLFilter{} filter.Project = &project filter.Name = &name objects, err := GetNetworkACLs(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_acls\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"networks_acls\" entry matches") } } // NetworkACLExists checks if a NetworkACL with the given key exists. // generator: NetworkACL Exists func NetworkACLExists(ctx context.Context, db dbtx, project string, name string) (_ bool, _err error) { defer func() { _err = mapErr(_err, "NetworkACL") }() stmt, err := Stmt(db, networkACLID) if err != nil { return false, fmt.Errorf("Failed to get \"networkACLID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, project, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("Failed to get \"networks_acls\" ID: %w", err) } return true, nil } // CreateNetworkACL adds a new NetworkACL to the database. // generator: NetworkACL Create func CreateNetworkACL(ctx context.Context, db dbtx, object NetworkACL) (_ int64, _err error) { defer func() { _err = mapErr(_err, "NetworkACL") }() args := make([]any, 5) // Populate the statement arguments. args[0] = object.Project args[1] = object.Name args[2] = object.Description marshaledIngress, err := marshalJSON(object.Ingress) if err != nil { return -1, err } args[3] = marshaledIngress marshaledEgress, err := marshalJSON(object.Egress) if err != nil { return -1, err } args[4] = marshaledEgress // Prepared statement to use. stmt, err := Stmt(db, networkACLCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"networkACLCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"networks_acls\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"networks_acls\" entry ID: %w", err) } return id, nil } // CreateNetworkACLConfig adds new NetworkACL Config to the database. // generator: NetworkACL Create func CreateNetworkACLConfig(ctx context.Context, db dbtx, networkACLID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "NetworkACL") }() referenceID := int(networkACLID) for key, value := range config { insert := Config{ ReferenceID: referenceID, Key: key, Value: value, } err := CreateConfig(ctx, db, "networks_acls", "network_acl", insert) if err != nil { return fmt.Errorf("Insert Config failed for NetworkACL: %w", err) } } return nil } // GetNetworkACLID return the ID of the NetworkACL with the given key. // generator: NetworkACL ID func GetNetworkACLID(ctx context.Context, db tx, project string, name string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "NetworkACL") }() stmt, err := Stmt(db, networkACLID) if err != nil { return -1, fmt.Errorf("Failed to get \"networkACLID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, project, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"networks_acls\" ID: %w", err) } return id, nil } // RenameNetworkACL renames the NetworkACL matching the given key parameters. // generator: NetworkACL Rename func RenameNetworkACL(ctx context.Context, db dbtx, project string, name string, to string) (_err error) { defer func() { _err = mapErr(_err, "NetworkACL") }() stmt, err := Stmt(db, networkACLRename) if err != nil { return fmt.Errorf("Failed to get \"networkACLRename\" prepared statement: %w", err) } result, err := stmt.Exec(to, project, name) if err != nil { return fmt.Errorf("Rename NetworkACL failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows failed: %w", err) } if n != 1 { return fmt.Errorf("Query affected %d rows instead of 1", n) } return nil } // UpdateNetworkACL updates the NetworkACL matching the given key parameters. // generator: NetworkACL Update func UpdateNetworkACL(ctx context.Context, db tx, project string, name string, object NetworkACL) (_err error) { defer func() { _err = mapErr(_err, "NetworkACL") }() id, err := GetNetworkACLID(ctx, db, project, name) if err != nil { return err } stmt, err := Stmt(db, networkACLUpdate) if err != nil { return fmt.Errorf("Failed to get \"networkACLUpdate\" prepared statement: %w", err) } marshaledIngress, err := marshalJSON(object.Ingress) if err != nil { return err } marshaledEgress, err := marshalJSON(object.Egress) if err != nil { return err } result, err := stmt.Exec(object.Project, object.Name, object.Description, marshaledIngress, marshaledEgress, id) if err != nil { return fmt.Errorf("Update \"networks_acls\" entry failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } return nil } // UpdateNetworkACLConfig updates the NetworkACL Config matching the given key parameters. // generator: NetworkACL Update func UpdateNetworkACLConfig(ctx context.Context, db tx, networkACLID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "NetworkACL") }() err := UpdateConfig(ctx, db, "networks_acls", "network_acl", int(networkACLID), config) if err != nil { return fmt.Errorf("Replace Config for NetworkACL failed: %w", err) } return nil } // DeleteNetworkACL deletes the NetworkACL matching the given key parameters. // generator: NetworkACL DeleteOne-by-ID func DeleteNetworkACL(ctx context.Context, db dbtx, id int) (_err error) { defer func() { _err = mapErr(_err, "NetworkACL") }() stmt, err := Stmt(db, networkACLDeleteByID) if err != nil { return fmt.Errorf("Failed to get \"networkACLDeleteByID\" prepared statement: %w", err) } result, err := stmt.Exec(id) if err != nil { return fmt.Errorf("Delete \"networks_acls\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d NetworkACL rows instead of 1", n) } return nil } incus-7.0.0/internal/server/db/cluster/networks_address_sets.go000066400000000000000000000064441517523235500250150ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "context" "database/sql" "github.com/lxc/incus/v7/shared/api" ) // Code generation directives. // //generate-database:mapper target networks_address_sets.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e network_address_set objects table=networks_address_sets //generate-database:mapper stmt -e network_address_set objects-by-ID table=networks_address_sets //generate-database:mapper stmt -e network_address_set objects-by-Name table=networks_address_sets //generate-database:mapper stmt -e network_address_set objects-by-Project table=networks_address_sets //generate-database:mapper stmt -e network_address_set objects-by-Project-and-Name table=networks_address_sets //generate-database:mapper stmt -e network_address_set id table=networks_address_sets //generate-database:mapper stmt -e network_address_set create struct=NetworkAddressSet table=networks_address_sets //generate-database:mapper stmt -e network_address_set rename table=networks_address_sets //generate-database:mapper stmt -e network_address_set update struct=NetworkAddressSet table=networks_address_sets //generate-database:mapper stmt -e network_address_set delete-by-Project-and-Name table=networks_address_sets // //generate-database:mapper method -i -e network_address_set ID struct=NetworkAddressSet table=networks_address_sets //generate-database:mapper method -i -e network_address_set Exists struct=NetworkAddressSet table=networks_address_sets //generate-database:mapper method -i -e network_address_set GetMany references=Config table=networks_address_sets //generate-database:mapper method -i -e network_address_set GetOne struct=NetworkAddressSet table=networks_address_sets //generate-database:mapper method -i -e network_address_set Create references=Config table=networks_address_sets //generate-database:mapper method -i -e network_address_set Rename table=networks_address_sets //generate-database:mapper method -i -e network_address_set Update struct=NetworkAddressSet references=Config table=networks_address_sets //generate-database:mapper method -i -e network_address_set DeleteOne-by-Project-and-Name table=networks_address_sets // NetworkAddressSet is a value object holding db-related details about a network address_set. type NetworkAddressSet struct { ID int ProjectID int `db:"omit=create,update"` Project string `db:"primary=yes&join=projects.name"` Name string `db:"primary=yes"` Description string `db:"coalesce=''"` Addresses []string `db:"marshal=json"` } // NetworkAddressSetFilter specifies potential query parameter fields. type NetworkAddressSetFilter struct { ID *int Name *string Project *string } // ToAPI converts the DB records to an API record. func (n *NetworkAddressSet) ToAPI(ctx context.Context, tx *sql.Tx) (*api.NetworkAddressSet, error) { // Get the config. config, err := GetNetworkAddressSetConfig(ctx, tx, n.ID) if err != nil { return nil, err } // Fill in the struct. resp := api.NetworkAddressSet{ NetworkAddressSetPost: api.NetworkAddressSetPost{ Name: n.Name, }, NetworkAddressSetPut: api.NetworkAddressSetPut{ Addresses: n.Addresses, Description: n.Description, Config: config, }, } return &resp, nil } incus-7.0.0/internal/server/db/cluster/networks_address_sets.interface.mapper.go000066400000000000000000000054401517523235500302320ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // NetworkAddressSetGenerated is an interface of generated methods for NetworkAddressSet. type NetworkAddressSetGenerated interface { // GetNetworkAddressSetID return the ID of the network_address_set with the given key. // generator: network_address_set ID GetNetworkAddressSetID(ctx context.Context, db tx, project string, name string) (int64, error) // NetworkAddressSetExists checks if a network_address_set with the given key exists. // generator: network_address_set Exists NetworkAddressSetExists(ctx context.Context, db dbtx, project string, name string) (bool, error) // GetNetworkAddressSetConfig returns all available NetworkAddressSet Config // generator: network_address_set GetMany GetNetworkAddressSetConfig(ctx context.Context, db tx, networkAddressSetID int, filters ...ConfigFilter) (map[string]string, error) // GetNetworkAddressSets returns all available network_address_sets. // generator: network_address_set GetMany GetNetworkAddressSets(ctx context.Context, db dbtx, filters ...NetworkAddressSetFilter) ([]NetworkAddressSet, error) // GetNetworkAddressSet returns the network_address_set with the given key. // generator: network_address_set GetOne GetNetworkAddressSet(ctx context.Context, db dbtx, project string, name string) (*NetworkAddressSet, error) // CreateNetworkAddressSetConfig adds new network_address_set Config to the database. // generator: network_address_set Create CreateNetworkAddressSetConfig(ctx context.Context, db dbtx, networkAddressSetID int64, config map[string]string) error // CreateNetworkAddressSet adds a new network_address_set to the database. // generator: network_address_set Create CreateNetworkAddressSet(ctx context.Context, db dbtx, object NetworkAddressSet) (int64, error) // RenameNetworkAddressSet renames the network_address_set matching the given key parameters. // generator: network_address_set Rename RenameNetworkAddressSet(ctx context.Context, db dbtx, project string, name string, to string) error // UpdateNetworkAddressSetConfig updates the network_address_set Config matching the given key parameters. // generator: network_address_set Update UpdateNetworkAddressSetConfig(ctx context.Context, db tx, networkAddressSetID int64, config map[string]string) error // UpdateNetworkAddressSet updates the network_address_set matching the given key parameters. // generator: network_address_set Update UpdateNetworkAddressSet(ctx context.Context, db tx, project string, name string, object NetworkAddressSet) error // DeleteNetworkAddressSet deletes the network_address_set matching the given key parameters. // generator: network_address_set DeleteOne-by-Project-and-Name DeleteNetworkAddressSet(ctx context.Context, db dbtx, project string, name string) error } incus-7.0.0/internal/server/db/cluster/networks_address_sets.mapper.go000066400000000000000000000430501517523235500262720ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var networkAddressSetObjects = RegisterStmt(` SELECT networks_address_sets.id, networks_address_sets.project_id, projects.name AS project, networks_address_sets.name, coalesce(networks_address_sets.description, ''), networks_address_sets.addresses FROM networks_address_sets JOIN projects ON networks_address_sets.project_id = projects.id ORDER BY projects.id, networks_address_sets.name `) var networkAddressSetObjectsByID = RegisterStmt(` SELECT networks_address_sets.id, networks_address_sets.project_id, projects.name AS project, networks_address_sets.name, coalesce(networks_address_sets.description, ''), networks_address_sets.addresses FROM networks_address_sets JOIN projects ON networks_address_sets.project_id = projects.id WHERE ( networks_address_sets.id = ? ) ORDER BY projects.id, networks_address_sets.name `) var networkAddressSetObjectsByName = RegisterStmt(` SELECT networks_address_sets.id, networks_address_sets.project_id, projects.name AS project, networks_address_sets.name, coalesce(networks_address_sets.description, ''), networks_address_sets.addresses FROM networks_address_sets JOIN projects ON networks_address_sets.project_id = projects.id WHERE ( networks_address_sets.name = ? ) ORDER BY projects.id, networks_address_sets.name `) var networkAddressSetObjectsByProject = RegisterStmt(` SELECT networks_address_sets.id, networks_address_sets.project_id, projects.name AS project, networks_address_sets.name, coalesce(networks_address_sets.description, ''), networks_address_sets.addresses FROM networks_address_sets JOIN projects ON networks_address_sets.project_id = projects.id WHERE ( project = ? ) ORDER BY projects.id, networks_address_sets.name `) var networkAddressSetObjectsByProjectAndName = RegisterStmt(` SELECT networks_address_sets.id, networks_address_sets.project_id, projects.name AS project, networks_address_sets.name, coalesce(networks_address_sets.description, ''), networks_address_sets.addresses FROM networks_address_sets JOIN projects ON networks_address_sets.project_id = projects.id WHERE ( project = ? AND networks_address_sets.name = ? ) ORDER BY projects.id, networks_address_sets.name `) var networkAddressSetID = RegisterStmt(` SELECT networks_address_sets.id FROM networks_address_sets JOIN projects ON networks_address_sets.project_id = projects.id WHERE projects.name = ? AND networks_address_sets.name = ? `) var networkAddressSetCreate = RegisterStmt(` INSERT INTO networks_address_sets (project_id, name, description, addresses) VALUES ((SELECT projects.id FROM projects WHERE projects.name = ?), ?, ?, ?) `) var networkAddressSetRename = RegisterStmt(` UPDATE networks_address_sets SET name = ? WHERE project_id = (SELECT projects.id FROM projects WHERE projects.name = ?) AND name = ? `) var networkAddressSetUpdate = RegisterStmt(` UPDATE networks_address_sets SET project_id = (SELECT projects.id FROM projects WHERE projects.name = ?), name = ?, description = ?, addresses = ? WHERE id = ? `) var networkAddressSetDeleteByProjectAndName = RegisterStmt(` DELETE FROM networks_address_sets WHERE project_id = (SELECT projects.id FROM projects WHERE projects.name = ?) AND name = ? `) // GetNetworkAddressSetID return the ID of the network_address_set with the given key. // generator: network_address_set ID func GetNetworkAddressSetID(ctx context.Context, db tx, project string, name string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Network_address_set") }() stmt, err := Stmt(db, networkAddressSetID) if err != nil { return -1, fmt.Errorf("Failed to get \"networkAddressSetID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, project, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"networks_address_sets\" ID: %w", err) } return id, nil } // NetworkAddressSetExists checks if a network_address_set with the given key exists. // generator: network_address_set Exists func NetworkAddressSetExists(ctx context.Context, db dbtx, project string, name string) (_ bool, _err error) { defer func() { _err = mapErr(_err, "Network_address_set") }() stmt, err := Stmt(db, networkAddressSetID) if err != nil { return false, fmt.Errorf("Failed to get \"networkAddressSetID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, project, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("Failed to get \"networks_address_sets\" ID: %w", err) } return true, nil } // networkAddressSetColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the NetworkAddressSet entity. func networkAddressSetColumns() string { return "networks_address_sets.id, networks_address_sets.project_id, projects.name AS project, networks_address_sets.name, coalesce(networks_address_sets.description, ''), networks_address_sets.addresses" } // getNetworkAddressSets can be used to run handwritten sql.Stmts to return a slice of objects. func getNetworkAddressSets(ctx context.Context, stmt *sql.Stmt, args ...any) ([]NetworkAddressSet, error) { objects := make([]NetworkAddressSet, 0) dest := func(scan func(dest ...any) error) error { n := NetworkAddressSet{} var addressesStr string err := scan(&n.ID, &n.ProjectID, &n.Project, &n.Name, &n.Description, &addressesStr) if err != nil { return err } err = unmarshalJSON(addressesStr, &n.Addresses) if err != nil { return err } objects = append(objects, n) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_address_sets\" table: %w", err) } return objects, nil } // getNetworkAddressSetsRaw can be used to run handwritten query strings to return a slice of objects. func getNetworkAddressSetsRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]NetworkAddressSet, error) { objects := make([]NetworkAddressSet, 0) dest := func(scan func(dest ...any) error) error { n := NetworkAddressSet{} var addressesStr string err := scan(&n.ID, &n.ProjectID, &n.Project, &n.Name, &n.Description, &addressesStr) if err != nil { return err } err = unmarshalJSON(addressesStr, &n.Addresses) if err != nil { return err } objects = append(objects, n) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_address_sets\" table: %w", err) } return objects, nil } // GetNetworkAddressSets returns all available network_address_sets. // generator: network_address_set GetMany func GetNetworkAddressSets(ctx context.Context, db dbtx, filters ...NetworkAddressSetFilter) (_ []NetworkAddressSet, _err error) { defer func() { _err = mapErr(_err, "Network_address_set") }() var err error // Result slice. objects := make([]NetworkAddressSet, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, networkAddressSetObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"networkAddressSetObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.Project != nil && filter.Name != nil && filter.ID == nil { args = append(args, []any{filter.Project, filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkAddressSetObjectsByProjectAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkAddressSetObjectsByProjectAndName\" prepared statement: %w", err) } break } query, err := StmtString(networkAddressSetObjectsByProjectAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkAddressSetObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.ID == nil && filter.Name == nil { args = append(args, []any{filter.Project}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkAddressSetObjectsByProject) if err != nil { return nil, fmt.Errorf("Failed to get \"networkAddressSetObjectsByProject\" prepared statement: %w", err) } break } query, err := StmtString(networkAddressSetObjectsByProject) if err != nil { return nil, fmt.Errorf("Failed to get \"networkAddressSetObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Name != nil && filter.ID == nil && filter.Project == nil { args = append(args, []any{filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkAddressSetObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkAddressSetObjectsByName\" prepared statement: %w", err) } break } query, err := StmtString(networkAddressSetObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkAddressSetObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID != nil && filter.Name == nil && filter.Project == nil { args = append(args, []any{filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkAddressSetObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkAddressSetObjectsByID\" prepared statement: %w", err) } break } query, err := StmtString(networkAddressSetObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkAddressSetObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.Name == nil && filter.Project == nil { return nil, fmt.Errorf("Cannot filter on empty NetworkAddressSetFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getNetworkAddressSets(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getNetworkAddressSetsRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_address_sets\" table: %w", err) } return objects, nil } // GetNetworkAddressSetConfig returns all available NetworkAddressSet Config // generator: network_address_set GetMany func GetNetworkAddressSetConfig(ctx context.Context, db tx, networkAddressSetID int, filters ...ConfigFilter) (_ map[string]string, _err error) { defer func() { _err = mapErr(_err, "Network_address_set") }() networkAddressSetConfig, err := GetConfig(ctx, db, "networks_address_sets", "network_address_set", filters...) if err != nil { return nil, err } config, ok := networkAddressSetConfig[networkAddressSetID] if !ok { config = map[string]string{} } return config, nil } // GetNetworkAddressSet returns the network_address_set with the given key. // generator: network_address_set GetOne func GetNetworkAddressSet(ctx context.Context, db dbtx, project string, name string) (_ *NetworkAddressSet, _err error) { defer func() { _err = mapErr(_err, "Network_address_set") }() filter := NetworkAddressSetFilter{} filter.Project = &project filter.Name = &name objects, err := GetNetworkAddressSets(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_address_sets\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"networks_address_sets\" entry matches") } } // CreateNetworkAddressSet adds a new network_address_set to the database. // generator: network_address_set Create func CreateNetworkAddressSet(ctx context.Context, db dbtx, object NetworkAddressSet) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Network_address_set") }() args := make([]any, 4) // Populate the statement arguments. args[0] = object.Project args[1] = object.Name args[2] = object.Description marshaledAddresses, err := marshalJSON(object.Addresses) if err != nil { return -1, err } args[3] = marshaledAddresses // Prepared statement to use. stmt, err := Stmt(db, networkAddressSetCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"networkAddressSetCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"networks_address_sets\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"networks_address_sets\" entry ID: %w", err) } return id, nil } // CreateNetworkAddressSetConfig adds new network_address_set Config to the database. // generator: network_address_set Create func CreateNetworkAddressSetConfig(ctx context.Context, db dbtx, networkAddressSetID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Network_address_set") }() referenceID := int(networkAddressSetID) for key, value := range config { insert := Config{ ReferenceID: referenceID, Key: key, Value: value, } err := CreateConfig(ctx, db, "networks_address_sets", "network_address_set", insert) if err != nil { return fmt.Errorf("Insert Config failed for NetworkAddressSet: %w", err) } } return nil } // RenameNetworkAddressSet renames the network_address_set matching the given key parameters. // generator: network_address_set Rename func RenameNetworkAddressSet(ctx context.Context, db dbtx, project string, name string, to string) (_err error) { defer func() { _err = mapErr(_err, "Network_address_set") }() stmt, err := Stmt(db, networkAddressSetRename) if err != nil { return fmt.Errorf("Failed to get \"networkAddressSetRename\" prepared statement: %w", err) } result, err := stmt.Exec(to, project, name) if err != nil { return fmt.Errorf("Rename NetworkAddressSet failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows failed: %w", err) } if n != 1 { return fmt.Errorf("Query affected %d rows instead of 1", n) } return nil } // UpdateNetworkAddressSet updates the network_address_set matching the given key parameters. // generator: network_address_set Update func UpdateNetworkAddressSet(ctx context.Context, db tx, project string, name string, object NetworkAddressSet) (_err error) { defer func() { _err = mapErr(_err, "Network_address_set") }() id, err := GetNetworkAddressSetID(ctx, db, project, name) if err != nil { return err } stmt, err := Stmt(db, networkAddressSetUpdate) if err != nil { return fmt.Errorf("Failed to get \"networkAddressSetUpdate\" prepared statement: %w", err) } marshaledAddresses, err := marshalJSON(object.Addresses) if err != nil { return err } result, err := stmt.Exec(object.Project, object.Name, object.Description, marshaledAddresses, id) if err != nil { return fmt.Errorf("Update \"networks_address_sets\" entry failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } return nil } // UpdateNetworkAddressSetConfig updates the network_address_set Config matching the given key parameters. // generator: network_address_set Update func UpdateNetworkAddressSetConfig(ctx context.Context, db tx, networkAddressSetID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Network_address_set") }() err := UpdateConfig(ctx, db, "networks_address_sets", "network_address_set", int(networkAddressSetID), config) if err != nil { return fmt.Errorf("Replace Config for NetworkAddressSet failed: %w", err) } return nil } // DeleteNetworkAddressSet deletes the network_address_set matching the given key parameters. // generator: network_address_set DeleteOne-by-Project-and-Name func DeleteNetworkAddressSet(ctx context.Context, db dbtx, project string, name string) (_err error) { defer func() { _err = mapErr(_err, "Network_address_set") }() stmt, err := Stmt(db, networkAddressSetDeleteByProjectAndName) if err != nil { return fmt.Errorf("Failed to get \"networkAddressSetDeleteByProjectAndName\" prepared statement: %w", err) } result, err := stmt.Exec(project, name) if err != nil { return fmt.Errorf("Delete \"networks_address_sets\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d NetworkAddressSet rows instead of 1", n) } return nil } incus-7.0.0/internal/server/db/cluster/networks_forwards.go000066400000000000000000000053171517523235500241570ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "context" "database/sql" "github.com/lxc/incus/v7/shared/api" ) // Code generation directives. // //generate-database:mapper target networks_forwards.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e network_forward objects table=networks_forwards //generate-database:mapper stmt -e network_forward objects-by-NetworkID table=networks_forwards //generate-database:mapper stmt -e network_forward objects-by-NetworkID-and-ListenAddress table=networks_forwards //generate-database:mapper stmt -e network_forward id table=networks_forwards //generate-database:mapper stmt -e network_forward create table=networks_forwards //generate-database:mapper stmt -e network_forward update table=networks_forwards //generate-database:mapper stmt -e network_forward delete-by-NetworkID-and-ID table=networks_forwards // //generate-database:mapper method -i -e network_forward GetMany references=Config table=networks_forwards //generate-database:mapper method -i -e network_forward GetOne table=networks_forwards //generate-database:mapper method -i -e network_forward ID table=networks_forwards //generate-database:mapper method -i -e network_forward Create references=Config table=networks_forwards //generate-database:mapper method -i -e network_forward Update references=Config table=networks_forwards //generate-database:mapper method -i -e network_forward DeleteOne-by-NetworkID-and-ID table=networks_forwards // NetworkForward is the generated entity backing the networks_forwards table. type NetworkForward struct { ID int64 NetworkID int64 `db:"primary=yes&column=network_id"` NodeID sql.NullInt64 `db:"column=node_id&nullable=true"` Location *string `db:"leftjoin=nodes.name&omit=create,update"` ListenAddress string `db:"primary=yes"` Description string Ports []api.NetworkForwardPort `db:"marshal=json"` } // NetworkForwardFilter defines the optional WHERE-clause fields. type NetworkForwardFilter struct { ID *int64 NetworkID *int64 NodeID *int64 ListenAddress *string } // ToAPI converts the DB record into the external API type. func (n *NetworkForward) ToAPI(ctx context.Context, tx *sql.Tx) (*api.NetworkForward, error) { // Get the config. cfg, err := GetNetworkForwardConfig(ctx, tx, int(n.ID)) if err != nil { return nil, err } // Fill in the struct. out := api.NetworkForward{ NetworkForwardPut: api.NetworkForwardPut{ Description: n.Description, Config: cfg, Ports: n.Ports, }, ListenAddress: n.ListenAddress, } if n.Location != nil { out.Location = *n.Location } return &out, nil } incus-7.0.0/internal/server/db/cluster/networks_forwards.interface.mapper.go000066400000000000000000000043131517523235500273740ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // NetworkForwardGenerated is an interface of generated methods for NetworkForward. type NetworkForwardGenerated interface { // GetNetworkForwardConfig returns all available NetworkForward Config // generator: network_forward GetMany GetNetworkForwardConfig(ctx context.Context, db tx, networkForwardID int, filters ...ConfigFilter) (map[string]string, error) // GetNetworkForwards returns all available network_forwards. // generator: network_forward GetMany GetNetworkForwards(ctx context.Context, db dbtx, filters ...NetworkForwardFilter) ([]NetworkForward, error) // GetNetworkForward returns the network_forward with the given key. // generator: network_forward GetOne GetNetworkForward(ctx context.Context, db dbtx, networkID int64, listenAddress string) (*NetworkForward, error) // GetNetworkForwardID return the ID of the network_forward with the given key. // generator: network_forward ID GetNetworkForwardID(ctx context.Context, db tx, networkID int64, listenAddress string) (int64, error) // CreateNetworkForwardConfig adds new network_forward Config to the database. // generator: network_forward Create CreateNetworkForwardConfig(ctx context.Context, db dbtx, networkForwardID int64, config map[string]string) error // CreateNetworkForward adds a new network_forward to the database. // generator: network_forward Create CreateNetworkForward(ctx context.Context, db dbtx, object NetworkForward) (int64, error) // UpdateNetworkForwardConfig updates the network_forward Config matching the given key parameters. // generator: network_forward Update UpdateNetworkForwardConfig(ctx context.Context, db tx, networkForwardID int64, config map[string]string) error // UpdateNetworkForward updates the network_forward matching the given key parameters. // generator: network_forward Update UpdateNetworkForward(ctx context.Context, db tx, networkID int64, listenAddress string, object NetworkForward) error // DeleteNetworkForward deletes the network_forward matching the given key parameters. // generator: network_forward DeleteOne-by-NetworkID-and-ID DeleteNetworkForward(ctx context.Context, db dbtx, networkID int64, id int64) error } incus-7.0.0/internal/server/db/cluster/networks_forwards.mapper.go000066400000000000000000000324531517523235500254430ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var networkForwardObjects = RegisterStmt(` SELECT networks_forwards.id, networks_forwards.network_id, networks_forwards.node_id, nodes.name AS location, networks_forwards.listen_address, networks_forwards.description, networks_forwards.ports FROM networks_forwards LEFT JOIN nodes ON networks_forwards.node_id = nodes.id ORDER BY networks_forwards.network_id, networks_forwards.listen_address `) var networkForwardObjectsByNetworkID = RegisterStmt(` SELECT networks_forwards.id, networks_forwards.network_id, networks_forwards.node_id, nodes.name AS location, networks_forwards.listen_address, networks_forwards.description, networks_forwards.ports FROM networks_forwards LEFT JOIN nodes ON networks_forwards.node_id = nodes.id WHERE ( networks_forwards.network_id = ? ) ORDER BY networks_forwards.network_id, networks_forwards.listen_address `) var networkForwardObjectsByNetworkIDAndListenAddress = RegisterStmt(` SELECT networks_forwards.id, networks_forwards.network_id, networks_forwards.node_id, nodes.name AS location, networks_forwards.listen_address, networks_forwards.description, networks_forwards.ports FROM networks_forwards LEFT JOIN nodes ON networks_forwards.node_id = nodes.id WHERE ( networks_forwards.network_id = ? AND networks_forwards.listen_address = ? ) ORDER BY networks_forwards.network_id, networks_forwards.listen_address `) var networkForwardID = RegisterStmt(` SELECT networks_forwards.id FROM networks_forwards WHERE networks_forwards.network_id = ? AND networks_forwards.listen_address = ? `) var networkForwardCreate = RegisterStmt(` INSERT INTO networks_forwards (network_id, node_id, listen_address, description, ports) VALUES (?, ?, ?, ?, ?) `) var networkForwardUpdate = RegisterStmt(` UPDATE networks_forwards SET network_id = ?, node_id = ?, listen_address = ?, description = ?, ports = ? WHERE id = ? `) var networkForwardDeleteByNetworkIDAndID = RegisterStmt(` DELETE FROM networks_forwards WHERE network_id = ? AND id = ? `) // networkForwardColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the NetworkForward entity. func networkForwardColumns() string { return "networks_forwards.id, networks_forwards.network_id, networks_forwards.node_id, nodes.name AS location, networks_forwards.listen_address, networks_forwards.description, networks_forwards.ports" } // getNetworkForwards can be used to run handwritten sql.Stmts to return a slice of objects. func getNetworkForwards(ctx context.Context, stmt *sql.Stmt, args ...any) ([]NetworkForward, error) { objects := make([]NetworkForward, 0) dest := func(scan func(dest ...any) error) error { n := NetworkForward{} var portsStr string err := scan(&n.ID, &n.NetworkID, &n.NodeID, &n.Location, &n.ListenAddress, &n.Description, &portsStr) if err != nil { return err } err = unmarshalJSON(portsStr, &n.Ports) if err != nil { return err } objects = append(objects, n) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_forwards\" table: %w", err) } return objects, nil } // getNetworkForwardsRaw can be used to run handwritten query strings to return a slice of objects. func getNetworkForwardsRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]NetworkForward, error) { objects := make([]NetworkForward, 0) dest := func(scan func(dest ...any) error) error { n := NetworkForward{} var portsStr string err := scan(&n.ID, &n.NetworkID, &n.NodeID, &n.Location, &n.ListenAddress, &n.Description, &portsStr) if err != nil { return err } err = unmarshalJSON(portsStr, &n.Ports) if err != nil { return err } objects = append(objects, n) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_forwards\" table: %w", err) } return objects, nil } // GetNetworkForwards returns all available network_forwards. // generator: network_forward GetMany func GetNetworkForwards(ctx context.Context, db dbtx, filters ...NetworkForwardFilter) (_ []NetworkForward, _err error) { defer func() { _err = mapErr(_err, "Network_forward") }() var err error // Result slice. objects := make([]NetworkForward, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, networkForwardObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"networkForwardObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.NetworkID != nil && filter.ListenAddress != nil && filter.ID == nil && filter.NodeID == nil { args = append(args, []any{filter.NetworkID, filter.ListenAddress}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkForwardObjectsByNetworkIDAndListenAddress) if err != nil { return nil, fmt.Errorf("Failed to get \"networkForwardObjectsByNetworkIDAndListenAddress\" prepared statement: %w", err) } break } query, err := StmtString(networkForwardObjectsByNetworkIDAndListenAddress) if err != nil { return nil, fmt.Errorf("Failed to get \"networkForwardObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.NetworkID != nil && filter.ID == nil && filter.NodeID == nil && filter.ListenAddress == nil { args = append(args, []any{filter.NetworkID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkForwardObjectsByNetworkID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkForwardObjectsByNetworkID\" prepared statement: %w", err) } break } query, err := StmtString(networkForwardObjectsByNetworkID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkForwardObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.NetworkID == nil && filter.NodeID == nil && filter.ListenAddress == nil { return nil, fmt.Errorf("Cannot filter on empty NetworkForwardFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getNetworkForwards(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getNetworkForwardsRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_forwards\" table: %w", err) } return objects, nil } // GetNetworkForwardConfig returns all available NetworkForward Config // generator: network_forward GetMany func GetNetworkForwardConfig(ctx context.Context, db tx, networkForwardID int, filters ...ConfigFilter) (_ map[string]string, _err error) { defer func() { _err = mapErr(_err, "Network_forward") }() networkForwardConfig, err := GetConfig(ctx, db, "networks_forwards", "network_forward", filters...) if err != nil { return nil, err } config, ok := networkForwardConfig[networkForwardID] if !ok { config = map[string]string{} } return config, nil } // GetNetworkForward returns the network_forward with the given key. // generator: network_forward GetOne func GetNetworkForward(ctx context.Context, db dbtx, networkID int64, listenAddress string) (_ *NetworkForward, _err error) { defer func() { _err = mapErr(_err, "Network_forward") }() filter := NetworkForwardFilter{} filter.NetworkID = &networkID filter.ListenAddress = &listenAddress objects, err := GetNetworkForwards(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_forwards\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"networks_forwards\" entry matches") } } // GetNetworkForwardID return the ID of the network_forward with the given key. // generator: network_forward ID func GetNetworkForwardID(ctx context.Context, db tx, networkID int64, listenAddress string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Network_forward") }() stmt, err := Stmt(db, networkForwardID) if err != nil { return -1, fmt.Errorf("Failed to get \"networkForwardID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, networkID, listenAddress) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"networks_forwards\" ID: %w", err) } return id, nil } // CreateNetworkForward adds a new network_forward to the database. // generator: network_forward Create func CreateNetworkForward(ctx context.Context, db dbtx, object NetworkForward) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Network_forward") }() args := make([]any, 5) // Populate the statement arguments. args[0] = object.NetworkID args[1] = object.NodeID args[2] = object.ListenAddress args[3] = object.Description marshaledPorts, err := marshalJSON(object.Ports) if err != nil { return -1, err } args[4] = marshaledPorts // Prepared statement to use. stmt, err := Stmt(db, networkForwardCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"networkForwardCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"networks_forwards\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"networks_forwards\" entry ID: %w", err) } return id, nil } // CreateNetworkForwardConfig adds new network_forward Config to the database. // generator: network_forward Create func CreateNetworkForwardConfig(ctx context.Context, db dbtx, networkForwardID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Network_forward") }() referenceID := int(networkForwardID) for key, value := range config { insert := Config{ ReferenceID: referenceID, Key: key, Value: value, } err := CreateConfig(ctx, db, "networks_forwards", "network_forward", insert) if err != nil { return fmt.Errorf("Insert Config failed for NetworkForward: %w", err) } } return nil } // UpdateNetworkForward updates the network_forward matching the given key parameters. // generator: network_forward Update func UpdateNetworkForward(ctx context.Context, db tx, networkID int64, listenAddress string, object NetworkForward) (_err error) { defer func() { _err = mapErr(_err, "Network_forward") }() id, err := GetNetworkForwardID(ctx, db, networkID, listenAddress) if err != nil { return err } stmt, err := Stmt(db, networkForwardUpdate) if err != nil { return fmt.Errorf("Failed to get \"networkForwardUpdate\" prepared statement: %w", err) } marshaledPorts, err := marshalJSON(object.Ports) if err != nil { return err } result, err := stmt.Exec(object.NetworkID, object.NodeID, object.ListenAddress, object.Description, marshaledPorts, id) if err != nil { return fmt.Errorf("Update \"networks_forwards\" entry failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } return nil } // UpdateNetworkForwardConfig updates the network_forward Config matching the given key parameters. // generator: network_forward Update func UpdateNetworkForwardConfig(ctx context.Context, db tx, networkForwardID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Network_forward") }() err := UpdateConfig(ctx, db, "networks_forwards", "network_forward", int(networkForwardID), config) if err != nil { return fmt.Errorf("Replace Config for NetworkForward failed: %w", err) } return nil } // DeleteNetworkForward deletes the network_forward matching the given key parameters. // generator: network_forward DeleteOne-by-NetworkID-and-ID func DeleteNetworkForward(ctx context.Context, db dbtx, networkID int64, id int64) (_err error) { defer func() { _err = mapErr(_err, "Network_forward") }() stmt, err := Stmt(db, networkForwardDeleteByNetworkIDAndID) if err != nil { return fmt.Errorf("Failed to get \"networkForwardDeleteByNetworkIDAndID\" prepared statement: %w", err) } result, err := stmt.Exec(networkID, id) if err != nil { return fmt.Errorf("Delete \"networks_forwards\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d NetworkForward rows instead of 1", n) } return nil } incus-7.0.0/internal/server/db/cluster/networks_integrations.go000066400000000000000000000052711517523235500250350ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "context" "database/sql" "github.com/lxc/incus/v7/shared/api" ) // Code generation directives. // //generate-database:mapper target networks_integrations.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e network_integration objects //generate-database:mapper stmt -e network_integration objects-by-Name //generate-database:mapper stmt -e network_integration objects-by-ID //generate-database:mapper stmt -e network_integration create struct=NetworkIntegration //generate-database:mapper stmt -e network_integration id //generate-database:mapper stmt -e network_integration rename //generate-database:mapper stmt -e network_integration update struct=NetworkIntegration //generate-database:mapper stmt -e network_integration delete-by-Name // //generate-database:mapper method -i -e network_integration GetMany references=Config //generate-database:mapper method -i -e network_integration GetOne struct=NetworkIntegration //generate-database:mapper method -i -e network_integration Exists struct=NetworkIntegration //generate-database:mapper method -i -e network_integration Create references=Config //generate-database:mapper method -i -e network_integration ID struct=NetworkIntegration //generate-database:mapper method -i -e network_integration Rename //generate-database:mapper method -i -e network_integration DeleteOne-by-Name //generate-database:mapper method -i -e network_integration Update struct=NetworkIntegration references=Config const ( // NetworkIntegrationTypeOVN represents an OVN network integration. NetworkIntegrationTypeOVN = iota ) // NetworkIntegrationTypeNames is a map between DB type to their string representation. var NetworkIntegrationTypeNames = map[int]string{ NetworkIntegrationTypeOVN: "ovn", } // NetworkIntegration is a value object holding db-related details about a network integration. type NetworkIntegration struct { ID int Name string Description string Type int } // ToAPI converts the DB records to an API record. func (n *NetworkIntegration) ToAPI(ctx context.Context, tx *sql.Tx) (*api.NetworkIntegration, error) { // Get the config. config, err := GetNetworkIntegrationConfig(ctx, tx, n.ID) if err != nil { return nil, err } // Fill in the struct. resp := api.NetworkIntegration{ Name: n.Name, Type: NetworkIntegrationTypeNames[n.Type], NetworkIntegrationPut: api.NetworkIntegrationPut{ Description: n.Description, Config: config, }, } return &resp, nil } // NetworkIntegrationFilter specifies potential query parameter fields. type NetworkIntegrationFilter struct { ID *int Name *string } incus-7.0.0/internal/server/db/cluster/networks_integrations.interface.mapper.go000066400000000000000000000053261517523235500302600ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // NetworkIntegrationGenerated is an interface of generated methods for NetworkIntegration. type NetworkIntegrationGenerated interface { // GetNetworkIntegrationConfig returns all available NetworkIntegration Config // generator: network_integration GetMany GetNetworkIntegrationConfig(ctx context.Context, db tx, networkIntegrationID int, filters ...ConfigFilter) (map[string]string, error) // GetNetworkIntegrations returns all available network_integrations. // generator: network_integration GetMany GetNetworkIntegrations(ctx context.Context, db dbtx, filters ...NetworkIntegrationFilter) ([]NetworkIntegration, error) // GetNetworkIntegration returns the network_integration with the given key. // generator: network_integration GetOne GetNetworkIntegration(ctx context.Context, db dbtx, name string) (*NetworkIntegration, error) // NetworkIntegrationExists checks if a network_integration with the given key exists. // generator: network_integration Exists NetworkIntegrationExists(ctx context.Context, db dbtx, name string) (bool, error) // CreateNetworkIntegrationConfig adds new network_integration Config to the database. // generator: network_integration Create CreateNetworkIntegrationConfig(ctx context.Context, db dbtx, networkIntegrationID int64, config map[string]string) error // CreateNetworkIntegration adds a new network_integration to the database. // generator: network_integration Create CreateNetworkIntegration(ctx context.Context, db dbtx, object NetworkIntegration) (int64, error) // GetNetworkIntegrationID return the ID of the network_integration with the given key. // generator: network_integration ID GetNetworkIntegrationID(ctx context.Context, db tx, name string) (int64, error) // RenameNetworkIntegration renames the network_integration matching the given key parameters. // generator: network_integration Rename RenameNetworkIntegration(ctx context.Context, db dbtx, name string, to string) error // DeleteNetworkIntegration deletes the network_integration matching the given key parameters. // generator: network_integration DeleteOne-by-Name DeleteNetworkIntegration(ctx context.Context, db dbtx, name string) error // UpdateNetworkIntegrationConfig updates the network_integration Config matching the given key parameters. // generator: network_integration Update UpdateNetworkIntegrationConfig(ctx context.Context, db tx, networkIntegrationID int64, config map[string]string) error // UpdateNetworkIntegration updates the network_integration matching the given key parameters. // generator: network_integration Update UpdateNetworkIntegration(ctx context.Context, db tx, name string, object NetworkIntegration) error } incus-7.0.0/internal/server/db/cluster/networks_integrations.mapper.go000066400000000000000000000330131517523235500263130ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var networkIntegrationObjects = RegisterStmt(` SELECT networks_integrations.id, networks_integrations.name, networks_integrations.description, networks_integrations.type FROM networks_integrations ORDER BY networks_integrations.name `) var networkIntegrationObjectsByName = RegisterStmt(` SELECT networks_integrations.id, networks_integrations.name, networks_integrations.description, networks_integrations.type FROM networks_integrations WHERE ( networks_integrations.name = ? ) ORDER BY networks_integrations.name `) var networkIntegrationObjectsByID = RegisterStmt(` SELECT networks_integrations.id, networks_integrations.name, networks_integrations.description, networks_integrations.type FROM networks_integrations WHERE ( networks_integrations.id = ? ) ORDER BY networks_integrations.name `) var networkIntegrationCreate = RegisterStmt(` INSERT INTO networks_integrations (name, description, type) VALUES (?, ?, ?) `) var networkIntegrationID = RegisterStmt(` SELECT networks_integrations.id FROM networks_integrations WHERE networks_integrations.name = ? `) var networkIntegrationRename = RegisterStmt(` UPDATE networks_integrations SET name = ? WHERE name = ? `) var networkIntegrationUpdate = RegisterStmt(` UPDATE networks_integrations SET name = ?, description = ?, type = ? WHERE id = ? `) var networkIntegrationDeleteByName = RegisterStmt(` DELETE FROM networks_integrations WHERE name = ? `) // networkIntegrationColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the NetworkIntegration entity. func networkIntegrationColumns() string { return "networks_integrations.id, networks_integrations.name, networks_integrations.description, networks_integrations.type" } // getNetworkIntegrations can be used to run handwritten sql.Stmts to return a slice of objects. func getNetworkIntegrations(ctx context.Context, stmt *sql.Stmt, args ...any) ([]NetworkIntegration, error) { objects := make([]NetworkIntegration, 0) dest := func(scan func(dest ...any) error) error { n := NetworkIntegration{} err := scan(&n.ID, &n.Name, &n.Description, &n.Type) if err != nil { return err } objects = append(objects, n) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_integrations\" table: %w", err) } return objects, nil } // getNetworkIntegrationsRaw can be used to run handwritten query strings to return a slice of objects. func getNetworkIntegrationsRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]NetworkIntegration, error) { objects := make([]NetworkIntegration, 0) dest := func(scan func(dest ...any) error) error { n := NetworkIntegration{} err := scan(&n.ID, &n.Name, &n.Description, &n.Type) if err != nil { return err } objects = append(objects, n) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_integrations\" table: %w", err) } return objects, nil } // GetNetworkIntegrations returns all available network_integrations. // generator: network_integration GetMany func GetNetworkIntegrations(ctx context.Context, db dbtx, filters ...NetworkIntegrationFilter) (_ []NetworkIntegration, _err error) { defer func() { _err = mapErr(_err, "Network_integration") }() var err error // Result slice. objects := make([]NetworkIntegration, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, networkIntegrationObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"networkIntegrationObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.Name != nil && filter.ID == nil { args = append(args, []any{filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkIntegrationObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkIntegrationObjectsByName\" prepared statement: %w", err) } break } query, err := StmtString(networkIntegrationObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkIntegrationObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID != nil && filter.Name == nil { args = append(args, []any{filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkIntegrationObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkIntegrationObjectsByID\" prepared statement: %w", err) } break } query, err := StmtString(networkIntegrationObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkIntegrationObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.Name == nil { return nil, fmt.Errorf("Cannot filter on empty NetworkIntegrationFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getNetworkIntegrations(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getNetworkIntegrationsRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_integrations\" table: %w", err) } return objects, nil } // GetNetworkIntegrationConfig returns all available NetworkIntegration Config // generator: network_integration GetMany func GetNetworkIntegrationConfig(ctx context.Context, db tx, networkIntegrationID int, filters ...ConfigFilter) (_ map[string]string, _err error) { defer func() { _err = mapErr(_err, "Network_integration") }() networkIntegrationConfig, err := GetConfig(ctx, db, "networks_integrations", "network_integration", filters...) if err != nil { return nil, err } config, ok := networkIntegrationConfig[networkIntegrationID] if !ok { config = map[string]string{} } return config, nil } // GetNetworkIntegration returns the network_integration with the given key. // generator: network_integration GetOne func GetNetworkIntegration(ctx context.Context, db dbtx, name string) (_ *NetworkIntegration, _err error) { defer func() { _err = mapErr(_err, "Network_integration") }() filter := NetworkIntegrationFilter{} filter.Name = &name objects, err := GetNetworkIntegrations(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_integrations\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"networks_integrations\" entry matches") } } // NetworkIntegrationExists checks if a network_integration with the given key exists. // generator: network_integration Exists func NetworkIntegrationExists(ctx context.Context, db dbtx, name string) (_ bool, _err error) { defer func() { _err = mapErr(_err, "Network_integration") }() stmt, err := Stmt(db, networkIntegrationID) if err != nil { return false, fmt.Errorf("Failed to get \"networkIntegrationID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("Failed to get \"networks_integrations\" ID: %w", err) } return true, nil } // CreateNetworkIntegration adds a new network_integration to the database. // generator: network_integration Create func CreateNetworkIntegration(ctx context.Context, db dbtx, object NetworkIntegration) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Network_integration") }() args := make([]any, 3) // Populate the statement arguments. args[0] = object.Name args[1] = object.Description args[2] = object.Type // Prepared statement to use. stmt, err := Stmt(db, networkIntegrationCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"networkIntegrationCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"networks_integrations\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"networks_integrations\" entry ID: %w", err) } return id, nil } // CreateNetworkIntegrationConfig adds new network_integration Config to the database. // generator: network_integration Create func CreateNetworkIntegrationConfig(ctx context.Context, db dbtx, networkIntegrationID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Network_integration") }() referenceID := int(networkIntegrationID) for key, value := range config { insert := Config{ ReferenceID: referenceID, Key: key, Value: value, } err := CreateConfig(ctx, db, "networks_integrations", "network_integration", insert) if err != nil { return fmt.Errorf("Insert Config failed for NetworkIntegration: %w", err) } } return nil } // GetNetworkIntegrationID return the ID of the network_integration with the given key. // generator: network_integration ID func GetNetworkIntegrationID(ctx context.Context, db tx, name string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Network_integration") }() stmt, err := Stmt(db, networkIntegrationID) if err != nil { return -1, fmt.Errorf("Failed to get \"networkIntegrationID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"networks_integrations\" ID: %w", err) } return id, nil } // RenameNetworkIntegration renames the network_integration matching the given key parameters. // generator: network_integration Rename func RenameNetworkIntegration(ctx context.Context, db dbtx, name string, to string) (_err error) { defer func() { _err = mapErr(_err, "Network_integration") }() stmt, err := Stmt(db, networkIntegrationRename) if err != nil { return fmt.Errorf("Failed to get \"networkIntegrationRename\" prepared statement: %w", err) } result, err := stmt.Exec(to, name) if err != nil { return fmt.Errorf("Rename NetworkIntegration failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows failed: %w", err) } if n != 1 { return fmt.Errorf("Query affected %d rows instead of 1", n) } return nil } // DeleteNetworkIntegration deletes the network_integration matching the given key parameters. // generator: network_integration DeleteOne-by-Name func DeleteNetworkIntegration(ctx context.Context, db dbtx, name string) (_err error) { defer func() { _err = mapErr(_err, "Network_integration") }() stmt, err := Stmt(db, networkIntegrationDeleteByName) if err != nil { return fmt.Errorf("Failed to get \"networkIntegrationDeleteByName\" prepared statement: %w", err) } result, err := stmt.Exec(name) if err != nil { return fmt.Errorf("Delete \"networks_integrations\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d NetworkIntegration rows instead of 1", n) } return nil } // UpdateNetworkIntegration updates the network_integration matching the given key parameters. // generator: network_integration Update func UpdateNetworkIntegration(ctx context.Context, db tx, name string, object NetworkIntegration) (_err error) { defer func() { _err = mapErr(_err, "Network_integration") }() id, err := GetNetworkIntegrationID(ctx, db, name) if err != nil { return err } stmt, err := Stmt(db, networkIntegrationUpdate) if err != nil { return fmt.Errorf("Failed to get \"networkIntegrationUpdate\" prepared statement: %w", err) } result, err := stmt.Exec(object.Name, object.Description, object.Type, id) if err != nil { return fmt.Errorf("Update \"networks_integrations\" entry failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } return nil } // UpdateNetworkIntegrationConfig updates the network_integration Config matching the given key parameters. // generator: network_integration Update func UpdateNetworkIntegrationConfig(ctx context.Context, db tx, networkIntegrationID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Network_integration") }() err := UpdateConfig(ctx, db, "networks_integrations", "network_integration", int(networkIntegrationID), config) if err != nil { return fmt.Errorf("Replace Config for NetworkIntegration failed: %w", err) } return nil } incus-7.0.0/internal/server/db/cluster/networks_load_balancers.go000066400000000000000000000054201517523235500252540ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "context" "database/sql" "github.com/lxc/incus/v7/shared/api" ) // Code generation directives. // //generate-database:mapper target networks_load_balancers.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e network_load_balancer objects table=networks_load_balancers //generate-database:mapper stmt -e network_load_balancer objects-by-NetworkID table=networks_load_balancers //generate-database:mapper stmt -e network_load_balancer objects-by-NetworkID-and-ListenAddress table=networks_load_balancers //generate-database:mapper stmt -e network_load_balancer id table=networks_load_balancers //generate-database:mapper stmt -e network_load_balancer create table=networks_load_balancers //generate-database:mapper stmt -e network_load_balancer update table=networks_load_balancers //generate-database:mapper stmt -e network_load_balancer delete-by-NetworkID-and-ID table=networks_load_balancers // //generate-database:mapper method -i -e network_load_balancer GetMany references=Config table=networks_load_balancers //generate-database:mapper method -i -e network_load_balancer GetOne table=networks_load_balancers //generate-database:mapper method -i -e network_load_balancer ID table=networks_load_balancers //generate-database:mapper method -i -e network_load_balancer Create references=Config table=networks_load_balancers //generate-database:mapper method -i -e network_load_balancer Update references=Config table=networks_load_balancers //generate-database:mapper method -i -e network_load_balancer DeleteOne-by-NetworkID-and-ID table=networks_load_balancers // NetworkLoadBalancer is the generated entity backing the networks_load_balancers table. type NetworkLoadBalancer struct { ID int64 NetworkID int64 `db:"primary=yes&column=network_id"` ListenAddress string `db:"primary=yes"` Description string Backends []api.NetworkLoadBalancerBackend `db:"marshal=json"` Ports []api.NetworkLoadBalancerPort `db:"marshal=json"` } // NetworkLoadBalancerFilter defines the optional WHERE-clause fields. type NetworkLoadBalancerFilter struct { ID *int64 NetworkID *int64 ListenAddress *string } // ToAPI converts the DB record into the external API type. func (n *NetworkLoadBalancer) ToAPI(ctx context.Context, tx *sql.Tx) (*api.NetworkLoadBalancer, error) { // Get the config. cfg, err := GetNetworkLoadBalancerConfig(ctx, tx, int(n.ID)) if err != nil { return nil, err } out := api.NetworkLoadBalancer{ NetworkLoadBalancerPut: api.NetworkLoadBalancerPut{ Description: n.Description, Config: cfg, Backends: n.Backends, Ports: n.Ports, }, ListenAddress: n.ListenAddress, } return &out, nil } incus-7.0.0/internal/server/db/cluster/networks_load_balancers.interface.mapper.go000066400000000000000000000047071517523235500305050ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // NetworkLoadBalancerGenerated is an interface of generated methods for NetworkLoadBalancer. type NetworkLoadBalancerGenerated interface { // GetNetworkLoadBalancerConfig returns all available NetworkLoadBalancer Config // generator: network_load_balancer GetMany GetNetworkLoadBalancerConfig(ctx context.Context, db tx, networkLoadBalancerID int, filters ...ConfigFilter) (map[string]string, error) // GetNetworkLoadBalancers returns all available network_load_balancers. // generator: network_load_balancer GetMany GetNetworkLoadBalancers(ctx context.Context, db dbtx, filters ...NetworkLoadBalancerFilter) ([]NetworkLoadBalancer, error) // GetNetworkLoadBalancer returns the network_load_balancer with the given key. // generator: network_load_balancer GetOne GetNetworkLoadBalancer(ctx context.Context, db dbtx, networkID int64, listenAddress string) (*NetworkLoadBalancer, error) // GetNetworkLoadBalancerID return the ID of the network_load_balancer with the given key. // generator: network_load_balancer ID GetNetworkLoadBalancerID(ctx context.Context, db tx, networkID int64, listenAddress string) (int64, error) // CreateNetworkLoadBalancerConfig adds new network_load_balancer Config to the database. // generator: network_load_balancer Create CreateNetworkLoadBalancerConfig(ctx context.Context, db dbtx, networkLoadBalancerID int64, config map[string]string) error // CreateNetworkLoadBalancer adds a new network_load_balancer to the database. // generator: network_load_balancer Create CreateNetworkLoadBalancer(ctx context.Context, db dbtx, object NetworkLoadBalancer) (int64, error) // UpdateNetworkLoadBalancerConfig updates the network_load_balancer Config matching the given key parameters. // generator: network_load_balancer Update UpdateNetworkLoadBalancerConfig(ctx context.Context, db tx, networkLoadBalancerID int64, config map[string]string) error // UpdateNetworkLoadBalancer updates the network_load_balancer matching the given key parameters. // generator: network_load_balancer Update UpdateNetworkLoadBalancer(ctx context.Context, db tx, networkID int64, listenAddress string, object NetworkLoadBalancer) error // DeleteNetworkLoadBalancer deletes the network_load_balancer matching the given key parameters. // generator: network_load_balancer DeleteOne-by-NetworkID-and-ID DeleteNetworkLoadBalancer(ctx context.Context, db dbtx, networkID int64, id int64) error } incus-7.0.0/internal/server/db/cluster/networks_load_balancers.mapper.go000066400000000000000000000343611517523235500265450ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var networkLoadBalancerObjects = RegisterStmt(` SELECT networks_load_balancers.id, networks_load_balancers.network_id, networks_load_balancers.listen_address, networks_load_balancers.description, networks_load_balancers.backends, networks_load_balancers.ports FROM networks_load_balancers ORDER BY networks_load_balancers.network_id, networks_load_balancers.listen_address `) var networkLoadBalancerObjectsByNetworkID = RegisterStmt(` SELECT networks_load_balancers.id, networks_load_balancers.network_id, networks_load_balancers.listen_address, networks_load_balancers.description, networks_load_balancers.backends, networks_load_balancers.ports FROM networks_load_balancers WHERE ( networks_load_balancers.network_id = ? ) ORDER BY networks_load_balancers.network_id, networks_load_balancers.listen_address `) var networkLoadBalancerObjectsByNetworkIDAndListenAddress = RegisterStmt(` SELECT networks_load_balancers.id, networks_load_balancers.network_id, networks_load_balancers.listen_address, networks_load_balancers.description, networks_load_balancers.backends, networks_load_balancers.ports FROM networks_load_balancers WHERE ( networks_load_balancers.network_id = ? AND networks_load_balancers.listen_address = ? ) ORDER BY networks_load_balancers.network_id, networks_load_balancers.listen_address `) var networkLoadBalancerID = RegisterStmt(` SELECT networks_load_balancers.id FROM networks_load_balancers WHERE networks_load_balancers.network_id = ? AND networks_load_balancers.listen_address = ? `) var networkLoadBalancerCreate = RegisterStmt(` INSERT INTO networks_load_balancers (network_id, listen_address, description, backends, ports) VALUES (?, ?, ?, ?, ?) `) var networkLoadBalancerUpdate = RegisterStmt(` UPDATE networks_load_balancers SET network_id = ?, listen_address = ?, description = ?, backends = ?, ports = ? WHERE id = ? `) var networkLoadBalancerDeleteByNetworkIDAndID = RegisterStmt(` DELETE FROM networks_load_balancers WHERE network_id = ? AND id = ? `) // networkLoadBalancerColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the NetworkLoadBalancer entity. func networkLoadBalancerColumns() string { return "networks_load_balancers.id, networks_load_balancers.network_id, networks_load_balancers.listen_address, networks_load_balancers.description, networks_load_balancers.backends, networks_load_balancers.ports" } // getNetworkLoadBalancers can be used to run handwritten sql.Stmts to return a slice of objects. func getNetworkLoadBalancers(ctx context.Context, stmt *sql.Stmt, args ...any) ([]NetworkLoadBalancer, error) { objects := make([]NetworkLoadBalancer, 0) dest := func(scan func(dest ...any) error) error { n := NetworkLoadBalancer{} var backendsStr string var portsStr string err := scan(&n.ID, &n.NetworkID, &n.ListenAddress, &n.Description, &backendsStr, &portsStr) if err != nil { return err } err = unmarshalJSON(backendsStr, &n.Backends) if err != nil { return err } err = unmarshalJSON(portsStr, &n.Ports) if err != nil { return err } objects = append(objects, n) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_load_balancers\" table: %w", err) } return objects, nil } // getNetworkLoadBalancersRaw can be used to run handwritten query strings to return a slice of objects. func getNetworkLoadBalancersRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]NetworkLoadBalancer, error) { objects := make([]NetworkLoadBalancer, 0) dest := func(scan func(dest ...any) error) error { n := NetworkLoadBalancer{} var backendsStr string var portsStr string err := scan(&n.ID, &n.NetworkID, &n.ListenAddress, &n.Description, &backendsStr, &portsStr) if err != nil { return err } err = unmarshalJSON(backendsStr, &n.Backends) if err != nil { return err } err = unmarshalJSON(portsStr, &n.Ports) if err != nil { return err } objects = append(objects, n) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_load_balancers\" table: %w", err) } return objects, nil } // GetNetworkLoadBalancers returns all available network_load_balancers. // generator: network_load_balancer GetMany func GetNetworkLoadBalancers(ctx context.Context, db dbtx, filters ...NetworkLoadBalancerFilter) (_ []NetworkLoadBalancer, _err error) { defer func() { _err = mapErr(_err, "Network_load_balancer") }() var err error // Result slice. objects := make([]NetworkLoadBalancer, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, networkLoadBalancerObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"networkLoadBalancerObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.NetworkID != nil && filter.ListenAddress != nil && filter.ID == nil { args = append(args, []any{filter.NetworkID, filter.ListenAddress}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkLoadBalancerObjectsByNetworkIDAndListenAddress) if err != nil { return nil, fmt.Errorf("Failed to get \"networkLoadBalancerObjectsByNetworkIDAndListenAddress\" prepared statement: %w", err) } break } query, err := StmtString(networkLoadBalancerObjectsByNetworkIDAndListenAddress) if err != nil { return nil, fmt.Errorf("Failed to get \"networkLoadBalancerObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.NetworkID != nil && filter.ID == nil && filter.ListenAddress == nil { args = append(args, []any{filter.NetworkID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkLoadBalancerObjectsByNetworkID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkLoadBalancerObjectsByNetworkID\" prepared statement: %w", err) } break } query, err := StmtString(networkLoadBalancerObjectsByNetworkID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkLoadBalancerObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.NetworkID == nil && filter.ListenAddress == nil { return nil, fmt.Errorf("Cannot filter on empty NetworkLoadBalancerFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getNetworkLoadBalancers(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getNetworkLoadBalancersRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_load_balancers\" table: %w", err) } return objects, nil } // GetNetworkLoadBalancerConfig returns all available NetworkLoadBalancer Config // generator: network_load_balancer GetMany func GetNetworkLoadBalancerConfig(ctx context.Context, db tx, networkLoadBalancerID int, filters ...ConfigFilter) (_ map[string]string, _err error) { defer func() { _err = mapErr(_err, "Network_load_balancer") }() networkLoadBalancerConfig, err := GetConfig(ctx, db, "networks_load_balancers", "network_load_balancer", filters...) if err != nil { return nil, err } config, ok := networkLoadBalancerConfig[networkLoadBalancerID] if !ok { config = map[string]string{} } return config, nil } // GetNetworkLoadBalancer returns the network_load_balancer with the given key. // generator: network_load_balancer GetOne func GetNetworkLoadBalancer(ctx context.Context, db dbtx, networkID int64, listenAddress string) (_ *NetworkLoadBalancer, _err error) { defer func() { _err = mapErr(_err, "Network_load_balancer") }() filter := NetworkLoadBalancerFilter{} filter.NetworkID = &networkID filter.ListenAddress = &listenAddress objects, err := GetNetworkLoadBalancers(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_load_balancers\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"networks_load_balancers\" entry matches") } } // GetNetworkLoadBalancerID return the ID of the network_load_balancer with the given key. // generator: network_load_balancer ID func GetNetworkLoadBalancerID(ctx context.Context, db tx, networkID int64, listenAddress string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Network_load_balancer") }() stmt, err := Stmt(db, networkLoadBalancerID) if err != nil { return -1, fmt.Errorf("Failed to get \"networkLoadBalancerID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, networkID, listenAddress) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"networks_load_balancers\" ID: %w", err) } return id, nil } // CreateNetworkLoadBalancer adds a new network_load_balancer to the database. // generator: network_load_balancer Create func CreateNetworkLoadBalancer(ctx context.Context, db dbtx, object NetworkLoadBalancer) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Network_load_balancer") }() args := make([]any, 5) // Populate the statement arguments. args[0] = object.NetworkID args[1] = object.ListenAddress args[2] = object.Description marshaledBackends, err := marshalJSON(object.Backends) if err != nil { return -1, err } args[3] = marshaledBackends marshaledPorts, err := marshalJSON(object.Ports) if err != nil { return -1, err } args[4] = marshaledPorts // Prepared statement to use. stmt, err := Stmt(db, networkLoadBalancerCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"networkLoadBalancerCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"networks_load_balancers\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"networks_load_balancers\" entry ID: %w", err) } return id, nil } // CreateNetworkLoadBalancerConfig adds new network_load_balancer Config to the database. // generator: network_load_balancer Create func CreateNetworkLoadBalancerConfig(ctx context.Context, db dbtx, networkLoadBalancerID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Network_load_balancer") }() referenceID := int(networkLoadBalancerID) for key, value := range config { insert := Config{ ReferenceID: referenceID, Key: key, Value: value, } err := CreateConfig(ctx, db, "networks_load_balancers", "network_load_balancer", insert) if err != nil { return fmt.Errorf("Insert Config failed for NetworkLoadBalancer: %w", err) } } return nil } // UpdateNetworkLoadBalancer updates the network_load_balancer matching the given key parameters. // generator: network_load_balancer Update func UpdateNetworkLoadBalancer(ctx context.Context, db tx, networkID int64, listenAddress string, object NetworkLoadBalancer) (_err error) { defer func() { _err = mapErr(_err, "Network_load_balancer") }() id, err := GetNetworkLoadBalancerID(ctx, db, networkID, listenAddress) if err != nil { return err } stmt, err := Stmt(db, networkLoadBalancerUpdate) if err != nil { return fmt.Errorf("Failed to get \"networkLoadBalancerUpdate\" prepared statement: %w", err) } marshaledBackends, err := marshalJSON(object.Backends) if err != nil { return err } marshaledPorts, err := marshalJSON(object.Ports) if err != nil { return err } result, err := stmt.Exec(object.NetworkID, object.ListenAddress, object.Description, marshaledBackends, marshaledPorts, id) if err != nil { return fmt.Errorf("Update \"networks_load_balancers\" entry failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } return nil } // UpdateNetworkLoadBalancerConfig updates the network_load_balancer Config matching the given key parameters. // generator: network_load_balancer Update func UpdateNetworkLoadBalancerConfig(ctx context.Context, db tx, networkLoadBalancerID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Network_load_balancer") }() err := UpdateConfig(ctx, db, "networks_load_balancers", "network_load_balancer", int(networkLoadBalancerID), config) if err != nil { return fmt.Errorf("Replace Config for NetworkLoadBalancer failed: %w", err) } return nil } // DeleteNetworkLoadBalancer deletes the network_load_balancer matching the given key parameters. // generator: network_load_balancer DeleteOne-by-NetworkID-and-ID func DeleteNetworkLoadBalancer(ctx context.Context, db dbtx, networkID int64, id int64) (_err error) { defer func() { _err = mapErr(_err, "Network_load_balancer") }() stmt, err := Stmt(db, networkLoadBalancerDeleteByNetworkIDAndID) if err != nil { return fmt.Errorf("Failed to get \"networkLoadBalancerDeleteByNetworkIDAndID\" prepared statement: %w", err) } result, err := stmt.Exec(networkID, id) if err != nil { return fmt.Errorf("Delete \"networks_load_balancers\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d NetworkLoadBalancer rows instead of 1", n) } return nil } incus-7.0.0/internal/server/db/cluster/networks_peers.go000066400000000000000000000141601517523235500234420ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "context" "database/sql" "errors" "fmt" "github.com/lxc/incus/v7/shared/api" ) // Code generation directives. // //generate-database:mapper target networks_peers.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e network_peer objects //generate-database:mapper stmt -e network_peer objects-by-Name //generate-database:mapper stmt -e network_peer objects-by-ID //generate-database:mapper stmt -e network_peer objects-by-NetworkID //generate-database:mapper stmt -e network_peer objects-by-TargetNetworkID //generate-database:mapper stmt -e network_peer objects-by-NetworkID-and-Name //generate-database:mapper stmt -e network_peer objects-by-NetworkID-and-ID //generate-database:mapper stmt -e network_peer objects-by-NetworkID-and-TargetNetworkProject-and-TargetNetworkName //generate-database:mapper stmt -e network_peer objects-by-Type-and-TargetNetworkProject-and-TargetNetworkName //generate-database:mapper stmt -e network_peer create struct=NetworkPeer //generate-database:mapper stmt -e network_peer id //generate-database:mapper stmt -e network_peer update struct=NetworkPeer //generate-database:mapper stmt -e network_peer delete-by-NetworkID-and-ID // //generate-database:mapper method -i -e network_peer GetMany references=Config //generate-database:mapper method -i -e network_peer GetOne struct=NetworkPeer //generate-database:mapper method -i -e network_peer Exists struct=NetworkPeer //generate-database:mapper method -i -e network_peer Create references=Config //generate-database:mapper method -i -e network_peer ID struct=NetworkPeer //generate-database:mapper method -i -e network_peer DeleteOne-by-NetworkID-and-ID //generate-database:mapper method -i -e network_peer Update struct=NetworkPeer references=Config const ( // NetworkPeerTypeLocal represents a local peer connection. NetworkPeerTypeLocal = iota // NetworkPeerTypeRemote represents a remote peer connection. NetworkPeerTypeRemote ) // NetworkPeerTypeNames maps peer types (integers) to their API representation (string). var NetworkPeerTypeNames = map[int]string{ NetworkPeerTypeLocal: "local", NetworkPeerTypeRemote: "remote", } // NetworkPeerTypes maps peer strings to their internal representation (integers). var NetworkPeerTypes = map[string]int{ NetworkPeerTypeNames[NetworkPeerTypeLocal]: NetworkPeerTypeLocal, NetworkPeerTypeNames[NetworkPeerTypeRemote]: NetworkPeerTypeRemote, } // NetworkPeer is a value object holding db-related details about a network peer. // Fields correspond to the columns in the networks_peers table. // generate-database will create CRUD methods and config helpers automatically. type NetworkPeer struct { ID int64 NetworkID int64 `db:"primary=yes&column=network_id"` Name string `db:"primary=yes"` Description string Type int TargetNetworkProject sql.NullString TargetNetworkName sql.NullString TargetNetworkIntegrationID sql.NullInt64 TargetNetworkID sql.NullInt64 } // NetworkPeerFilter specifies potential query parameter fields. type NetworkPeerFilter struct { ID *int64 NetworkID *int64 Name *string Type *int TargetNetworkProject *string TargetNetworkName *string TargetNetworkIntegrationID *int64 TargetNetworkID *int64 } // NetworkPeerConnection represents a peer connection. type NetworkPeerConnection struct { NetworkName string PeerName string } // ToAPI converts the database NetworkPeer to API type. func (n *NetworkPeer) ToAPI(ctx context.Context, tx *sql.Tx) (*api.NetworkPeer, error) { configMap, err := GetNetworkPeerConfig(ctx, tx, int(n.ID)) if err != nil { return nil, err } resp := api.NetworkPeer{ NetworkPeerPut: api.NetworkPeerPut{ Description: n.Description, Config: configMap, }, Name: n.Name, TargetProject: n.TargetNetworkProject.String, TargetNetwork: n.TargetNetworkName.String, Type: NetworkPeerTypeNames[n.Type], UsedBy: []string{}, } if n.TargetNetworkID.Valid { // This is a workaround until networks themselves are ported over to the generator. dest := func(scan func(dest ...any) error) error { err := scan(&resp.TargetNetwork, &resp.TargetProject) if err != nil { return err } return nil } err := scan(ctx, tx, "SELECT networks.name, projects.name FROM networks JOIN projects ON networks.project_id=projects.id WHERE networks.id=?", dest, n.TargetNetworkID) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks\" table: %w", err) } } // Get the target integration name if needed. if n.Type == NetworkPeerTypeRemote { idInt := int(n.TargetNetworkIntegrationID.Int64) integrations, err := GetNetworkIntegrations(ctx, tx, NetworkIntegrationFilter{ID: &idInt}) if err != nil { return nil, err } if len(integrations) != 1 { return nil, errors.New("Couldn't find network integration") } resp.TargetIntegration = integrations[0].Name resp.Status = api.NetworkStatusCreated } else { // Peer has mutual peering from target network. if n.TargetNetworkName.String != "" && n.TargetNetworkProject.String != "" { if n.TargetNetworkID.Valid { // Peer is in a conflicting state with both the peer network ID and net/project names set. // Peer net/project names should only be populated before the peer is linked with a peer network ID. resp.Status = api.NetworkStatusErrored } else { // Peer isn't linked to a mutual peer on the target network yet but has joining details. resp.Status = api.NetworkStatusPending } } else { if n.TargetNetworkID.Valid { // Peer is linked to an mutual peer on the target network. resp.Status = api.NetworkStatusCreated } else { // Peer isn't linked to a mutual peer on the target network yet and has no joining details. // Perhaps it was formerly joined (and had its joining details cleared) and subsequently // the target peer removed its peering entry. resp.Status = api.NetworkStatusErrored } } } return &resp, nil } incus-7.0.0/internal/server/db/cluster/networks_peers.interface.mapper.go000066400000000000000000000043561517523235500266720ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // NetworkPeerGenerated is an interface of generated methods for NetworkPeer. type NetworkPeerGenerated interface { // GetNetworkPeerConfig returns all available NetworkPeer Config // generator: network_peer GetMany GetNetworkPeerConfig(ctx context.Context, db tx, networkPeerID int, filters ...ConfigFilter) (map[string]string, error) // GetNetworkPeers returns all available network_peers. // generator: network_peer GetMany GetNetworkPeers(ctx context.Context, db dbtx, filters ...NetworkPeerFilter) ([]NetworkPeer, error) // GetNetworkPeer returns the network_peer with the given key. // generator: network_peer GetOne GetNetworkPeer(ctx context.Context, db dbtx, networkID int64, name string) (*NetworkPeer, error) // NetworkPeerExists checks if a network_peer with the given key exists. // generator: network_peer Exists NetworkPeerExists(ctx context.Context, db dbtx, networkID int64, name string) (bool, error) // CreateNetworkPeerConfig adds new network_peer Config to the database. // generator: network_peer Create CreateNetworkPeerConfig(ctx context.Context, db dbtx, networkPeerID int64, config map[string]string) error // CreateNetworkPeer adds a new network_peer to the database. // generator: network_peer Create CreateNetworkPeer(ctx context.Context, db dbtx, object NetworkPeer) (int64, error) // GetNetworkPeerID return the ID of the network_peer with the given key. // generator: network_peer ID GetNetworkPeerID(ctx context.Context, db tx, networkID int64, name string) (int64, error) // DeleteNetworkPeer deletes the network_peer matching the given key parameters. // generator: network_peer DeleteOne-by-NetworkID-and-ID DeleteNetworkPeer(ctx context.Context, db dbtx, networkID int64, id int64) error // UpdateNetworkPeerConfig updates the network_peer Config matching the given key parameters. // generator: network_peer Update UpdateNetworkPeerConfig(ctx context.Context, db tx, networkPeerID int64, config map[string]string) error // UpdateNetworkPeer updates the network_peer matching the given key parameters. // generator: network_peer Update UpdateNetworkPeer(ctx context.Context, db tx, networkID int64, name string, object NetworkPeer) error } incus-7.0.0/internal/server/db/cluster/networks_peers.mapper.go000066400000000000000000000552641517523235500247370ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var networkPeerObjects = RegisterStmt(` SELECT networks_peers.id, networks_peers.network_id, networks_peers.name, networks_peers.description, networks_peers.type, networks_peers.target_network_project, networks_peers.target_network_name, networks_peers.target_network_integration_id, networks_peers.target_network_id FROM networks_peers ORDER BY networks_peers.network_id, networks_peers.name `) var networkPeerObjectsByName = RegisterStmt(` SELECT networks_peers.id, networks_peers.network_id, networks_peers.name, networks_peers.description, networks_peers.type, networks_peers.target_network_project, networks_peers.target_network_name, networks_peers.target_network_integration_id, networks_peers.target_network_id FROM networks_peers WHERE ( networks_peers.name = ? ) ORDER BY networks_peers.network_id, networks_peers.name `) var networkPeerObjectsByID = RegisterStmt(` SELECT networks_peers.id, networks_peers.network_id, networks_peers.name, networks_peers.description, networks_peers.type, networks_peers.target_network_project, networks_peers.target_network_name, networks_peers.target_network_integration_id, networks_peers.target_network_id FROM networks_peers WHERE ( networks_peers.id = ? ) ORDER BY networks_peers.network_id, networks_peers.name `) var networkPeerObjectsByNetworkID = RegisterStmt(` SELECT networks_peers.id, networks_peers.network_id, networks_peers.name, networks_peers.description, networks_peers.type, networks_peers.target_network_project, networks_peers.target_network_name, networks_peers.target_network_integration_id, networks_peers.target_network_id FROM networks_peers WHERE ( networks_peers.network_id = ? ) ORDER BY networks_peers.network_id, networks_peers.name `) var networkPeerObjectsByTargetNetworkID = RegisterStmt(` SELECT networks_peers.id, networks_peers.network_id, networks_peers.name, networks_peers.description, networks_peers.type, networks_peers.target_network_project, networks_peers.target_network_name, networks_peers.target_network_integration_id, networks_peers.target_network_id FROM networks_peers WHERE ( networks_peers.target_network_id = ? ) ORDER BY networks_peers.network_id, networks_peers.name `) var networkPeerObjectsByNetworkIDAndName = RegisterStmt(` SELECT networks_peers.id, networks_peers.network_id, networks_peers.name, networks_peers.description, networks_peers.type, networks_peers.target_network_project, networks_peers.target_network_name, networks_peers.target_network_integration_id, networks_peers.target_network_id FROM networks_peers WHERE ( networks_peers.network_id = ? AND networks_peers.name = ? ) ORDER BY networks_peers.network_id, networks_peers.name `) var networkPeerObjectsByNetworkIDAndID = RegisterStmt(` SELECT networks_peers.id, networks_peers.network_id, networks_peers.name, networks_peers.description, networks_peers.type, networks_peers.target_network_project, networks_peers.target_network_name, networks_peers.target_network_integration_id, networks_peers.target_network_id FROM networks_peers WHERE ( networks_peers.network_id = ? AND networks_peers.id = ? ) ORDER BY networks_peers.network_id, networks_peers.name `) var networkPeerObjectsByNetworkIDAndTargetNetworkProjectAndTargetNetworkName = RegisterStmt(` SELECT networks_peers.id, networks_peers.network_id, networks_peers.name, networks_peers.description, networks_peers.type, networks_peers.target_network_project, networks_peers.target_network_name, networks_peers.target_network_integration_id, networks_peers.target_network_id FROM networks_peers WHERE ( networks_peers.network_id = ? AND networks_peers.target_network_project = ? AND networks_peers.target_network_name = ? ) ORDER BY networks_peers.network_id, networks_peers.name `) var networkPeerObjectsByTypeAndTargetNetworkProjectAndTargetNetworkName = RegisterStmt(` SELECT networks_peers.id, networks_peers.network_id, networks_peers.name, networks_peers.description, networks_peers.type, networks_peers.target_network_project, networks_peers.target_network_name, networks_peers.target_network_integration_id, networks_peers.target_network_id FROM networks_peers WHERE ( networks_peers.type = ? AND networks_peers.target_network_project = ? AND networks_peers.target_network_name = ? ) ORDER BY networks_peers.network_id, networks_peers.name `) var networkPeerCreate = RegisterStmt(` INSERT INTO networks_peers (network_id, name, description, type, target_network_project, target_network_name, target_network_integration_id, target_network_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?) `) var networkPeerID = RegisterStmt(` SELECT networks_peers.id FROM networks_peers WHERE networks_peers.network_id = ? AND networks_peers.name = ? `) var networkPeerUpdate = RegisterStmt(` UPDATE networks_peers SET network_id = ?, name = ?, description = ?, type = ?, target_network_project = ?, target_network_name = ?, target_network_integration_id = ?, target_network_id = ? WHERE id = ? `) var networkPeerDeleteByNetworkIDAndID = RegisterStmt(` DELETE FROM networks_peers WHERE network_id = ? AND id = ? `) // networkPeerColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the NetworkPeer entity. func networkPeerColumns() string { return "networks_peers.id, networks_peers.network_id, networks_peers.name, networks_peers.description, networks_peers.type, networks_peers.target_network_project, networks_peers.target_network_name, networks_peers.target_network_integration_id, networks_peers.target_network_id" } // getNetworkPeers can be used to run handwritten sql.Stmts to return a slice of objects. func getNetworkPeers(ctx context.Context, stmt *sql.Stmt, args ...any) ([]NetworkPeer, error) { objects := make([]NetworkPeer, 0) dest := func(scan func(dest ...any) error) error { n := NetworkPeer{} err := scan(&n.ID, &n.NetworkID, &n.Name, &n.Description, &n.Type, &n.TargetNetworkProject, &n.TargetNetworkName, &n.TargetNetworkIntegrationID, &n.TargetNetworkID) if err != nil { return err } objects = append(objects, n) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_peers\" table: %w", err) } return objects, nil } // getNetworkPeersRaw can be used to run handwritten query strings to return a slice of objects. func getNetworkPeersRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]NetworkPeer, error) { objects := make([]NetworkPeer, 0) dest := func(scan func(dest ...any) error) error { n := NetworkPeer{} err := scan(&n.ID, &n.NetworkID, &n.Name, &n.Description, &n.Type, &n.TargetNetworkProject, &n.TargetNetworkName, &n.TargetNetworkIntegrationID, &n.TargetNetworkID) if err != nil { return err } objects = append(objects, n) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_peers\" table: %w", err) } return objects, nil } // GetNetworkPeers returns all available network_peers. // generator: network_peer GetMany func GetNetworkPeers(ctx context.Context, db dbtx, filters ...NetworkPeerFilter) (_ []NetworkPeer, _err error) { defer func() { _err = mapErr(_err, "Network_peer") }() var err error // Result slice. objects := make([]NetworkPeer, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, networkPeerObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.Type != nil && filter.TargetNetworkProject != nil && filter.TargetNetworkName != nil && filter.ID == nil && filter.NetworkID == nil && filter.Name == nil && filter.TargetNetworkIntegrationID == nil && filter.TargetNetworkID == nil { args = append(args, []any{filter.Type, filter.TargetNetworkProject, filter.TargetNetworkName}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkPeerObjectsByTypeAndTargetNetworkProjectAndTargetNetworkName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjectsByTypeAndTargetNetworkProjectAndTargetNetworkName\" prepared statement: %w", err) } break } query, err := StmtString(networkPeerObjectsByTypeAndTargetNetworkProjectAndTargetNetworkName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.NetworkID != nil && filter.TargetNetworkProject != nil && filter.TargetNetworkName != nil && filter.ID == nil && filter.Name == nil && filter.Type == nil && filter.TargetNetworkIntegrationID == nil && filter.TargetNetworkID == nil { args = append(args, []any{filter.NetworkID, filter.TargetNetworkProject, filter.TargetNetworkName}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkPeerObjectsByNetworkIDAndTargetNetworkProjectAndTargetNetworkName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjectsByNetworkIDAndTargetNetworkProjectAndTargetNetworkName\" prepared statement: %w", err) } break } query, err := StmtString(networkPeerObjectsByNetworkIDAndTargetNetworkProjectAndTargetNetworkName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.NetworkID != nil && filter.Name != nil && filter.ID == nil && filter.Type == nil && filter.TargetNetworkProject == nil && filter.TargetNetworkName == nil && filter.TargetNetworkIntegrationID == nil && filter.TargetNetworkID == nil { args = append(args, []any{filter.NetworkID, filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkPeerObjectsByNetworkIDAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjectsByNetworkIDAndName\" prepared statement: %w", err) } break } query, err := StmtString(networkPeerObjectsByNetworkIDAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.NetworkID != nil && filter.ID != nil && filter.Name == nil && filter.Type == nil && filter.TargetNetworkProject == nil && filter.TargetNetworkName == nil && filter.TargetNetworkIntegrationID == nil && filter.TargetNetworkID == nil { args = append(args, []any{filter.NetworkID, filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkPeerObjectsByNetworkIDAndID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjectsByNetworkIDAndID\" prepared statement: %w", err) } break } query, err := StmtString(networkPeerObjectsByNetworkIDAndID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.TargetNetworkID != nil && filter.ID == nil && filter.NetworkID == nil && filter.Name == nil && filter.Type == nil && filter.TargetNetworkProject == nil && filter.TargetNetworkName == nil && filter.TargetNetworkIntegrationID == nil { args = append(args, []any{filter.TargetNetworkID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkPeerObjectsByTargetNetworkID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjectsByTargetNetworkID\" prepared statement: %w", err) } break } query, err := StmtString(networkPeerObjectsByTargetNetworkID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.NetworkID != nil && filter.ID == nil && filter.Name == nil && filter.Type == nil && filter.TargetNetworkProject == nil && filter.TargetNetworkName == nil && filter.TargetNetworkIntegrationID == nil && filter.TargetNetworkID == nil { args = append(args, []any{filter.NetworkID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkPeerObjectsByNetworkID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjectsByNetworkID\" prepared statement: %w", err) } break } query, err := StmtString(networkPeerObjectsByNetworkID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Name != nil && filter.ID == nil && filter.NetworkID == nil && filter.Type == nil && filter.TargetNetworkProject == nil && filter.TargetNetworkName == nil && filter.TargetNetworkIntegrationID == nil && filter.TargetNetworkID == nil { args = append(args, []any{filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkPeerObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjectsByName\" prepared statement: %w", err) } break } query, err := StmtString(networkPeerObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID != nil && filter.NetworkID == nil && filter.Name == nil && filter.Type == nil && filter.TargetNetworkProject == nil && filter.TargetNetworkName == nil && filter.TargetNetworkIntegrationID == nil && filter.TargetNetworkID == nil { args = append(args, []any{filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkPeerObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjectsByID\" prepared statement: %w", err) } break } query, err := StmtString(networkPeerObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkPeerObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.NetworkID == nil && filter.Name == nil && filter.Type == nil && filter.TargetNetworkProject == nil && filter.TargetNetworkName == nil && filter.TargetNetworkIntegrationID == nil && filter.TargetNetworkID == nil { return nil, fmt.Errorf("Cannot filter on empty NetworkPeerFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getNetworkPeers(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getNetworkPeersRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_peers\" table: %w", err) } return objects, nil } // GetNetworkPeerConfig returns all available NetworkPeer Config // generator: network_peer GetMany func GetNetworkPeerConfig(ctx context.Context, db tx, networkPeerID int, filters ...ConfigFilter) (_ map[string]string, _err error) { defer func() { _err = mapErr(_err, "Network_peer") }() networkPeerConfig, err := GetConfig(ctx, db, "networks_peers", "network_peer", filters...) if err != nil { return nil, err } config, ok := networkPeerConfig[networkPeerID] if !ok { config = map[string]string{} } return config, nil } // GetNetworkPeer returns the network_peer with the given key. // generator: network_peer GetOne func GetNetworkPeer(ctx context.Context, db dbtx, networkID int64, name string) (_ *NetworkPeer, _err error) { defer func() { _err = mapErr(_err, "Network_peer") }() filter := NetworkPeerFilter{} filter.NetworkID = &networkID filter.Name = &name objects, err := GetNetworkPeers(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_peers\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"networks_peers\" entry matches") } } // NetworkPeerExists checks if a network_peer with the given key exists. // generator: network_peer Exists func NetworkPeerExists(ctx context.Context, db dbtx, networkID int64, name string) (_ bool, _err error) { defer func() { _err = mapErr(_err, "Network_peer") }() stmt, err := Stmt(db, networkPeerID) if err != nil { return false, fmt.Errorf("Failed to get \"networkPeerID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, networkID, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("Failed to get \"networks_peers\" ID: %w", err) } return true, nil } // CreateNetworkPeer adds a new network_peer to the database. // generator: network_peer Create func CreateNetworkPeer(ctx context.Context, db dbtx, object NetworkPeer) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Network_peer") }() args := make([]any, 8) // Populate the statement arguments. args[0] = object.NetworkID args[1] = object.Name args[2] = object.Description args[3] = object.Type args[4] = object.TargetNetworkProject args[5] = object.TargetNetworkName args[6] = object.TargetNetworkIntegrationID args[7] = object.TargetNetworkID // Prepared statement to use. stmt, err := Stmt(db, networkPeerCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"networkPeerCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"networks_peers\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"networks_peers\" entry ID: %w", err) } return id, nil } // CreateNetworkPeerConfig adds new network_peer Config to the database. // generator: network_peer Create func CreateNetworkPeerConfig(ctx context.Context, db dbtx, networkPeerID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Network_peer") }() referenceID := int(networkPeerID) for key, value := range config { insert := Config{ ReferenceID: referenceID, Key: key, Value: value, } err := CreateConfig(ctx, db, "networks_peers", "network_peer", insert) if err != nil { return fmt.Errorf("Insert Config failed for NetworkPeer: %w", err) } } return nil } // GetNetworkPeerID return the ID of the network_peer with the given key. // generator: network_peer ID func GetNetworkPeerID(ctx context.Context, db tx, networkID int64, name string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Network_peer") }() stmt, err := Stmt(db, networkPeerID) if err != nil { return -1, fmt.Errorf("Failed to get \"networkPeerID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, networkID, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"networks_peers\" ID: %w", err) } return id, nil } // DeleteNetworkPeer deletes the network_peer matching the given key parameters. // generator: network_peer DeleteOne-by-NetworkID-and-ID func DeleteNetworkPeer(ctx context.Context, db dbtx, networkID int64, id int64) (_err error) { defer func() { _err = mapErr(_err, "Network_peer") }() stmt, err := Stmt(db, networkPeerDeleteByNetworkIDAndID) if err != nil { return fmt.Errorf("Failed to get \"networkPeerDeleteByNetworkIDAndID\" prepared statement: %w", err) } result, err := stmt.Exec(networkID, id) if err != nil { return fmt.Errorf("Delete \"networks_peers\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d NetworkPeer rows instead of 1", n) } return nil } // UpdateNetworkPeer updates the network_peer matching the given key parameters. // generator: network_peer Update func UpdateNetworkPeer(ctx context.Context, db tx, networkID int64, name string, object NetworkPeer) (_err error) { defer func() { _err = mapErr(_err, "Network_peer") }() id, err := GetNetworkPeerID(ctx, db, networkID, name) if err != nil { return err } stmt, err := Stmt(db, networkPeerUpdate) if err != nil { return fmt.Errorf("Failed to get \"networkPeerUpdate\" prepared statement: %w", err) } result, err := stmt.Exec(object.NetworkID, object.Name, object.Description, object.Type, object.TargetNetworkProject, object.TargetNetworkName, object.TargetNetworkIntegrationID, object.TargetNetworkID, id) if err != nil { return fmt.Errorf("Update \"networks_peers\" entry failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } return nil } // UpdateNetworkPeerConfig updates the network_peer Config matching the given key parameters. // generator: network_peer Update func UpdateNetworkPeerConfig(ctx context.Context, db tx, networkPeerID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Network_peer") }() err := UpdateConfig(ctx, db, "networks_peers", "network_peer", int(networkPeerID), config) if err != nil { return fmt.Errorf("Replace Config for NetworkPeer failed: %w", err) } return nil } incus-7.0.0/internal/server/db/cluster/networks_zones.go000066400000000000000000000052651517523235500234700ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "context" "github.com/lxc/incus/v7/shared/api" ) // Code generation directives. // //generate-database:mapper target networks_zones.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // // Statements: //generate-database:mapper stmt -e NetworkZone objects table=networks_zones //generate-database:mapper stmt -e NetworkZone objects-by-ID table=networks_zones //generate-database:mapper stmt -e NetworkZone objects-by-Name table=networks_zones //generate-database:mapper stmt -e NetworkZone objects-by-Project table=networks_zones //generate-database:mapper stmt -e NetworkZone objects-by-Project-and-Name table=networks_zones //generate-database:mapper stmt -e NetworkZone id table=networks_zones //generate-database:mapper stmt -e NetworkZone create table=networks_zones //generate-database:mapper stmt -e NetworkZone rename table=networks_zones //generate-database:mapper stmt -e NetworkZone update table=networks_zones //generate-database:mapper stmt -e NetworkZone delete-by-ID table=networks_zones // // Methods: //generate-database:mapper method -i -e NetworkZone GetMany references=Config table=networks_zones //generate-database:mapper method -i -e NetworkZone GetOne table=networks_zones //generate-database:mapper method -i -e NetworkZone Exists table=networks_zones //generate-database:mapper method -i -e NetworkZone Create references=Config table=networks_zones //generate-database:mapper method -i -e NetworkZone ID table=networks_zones //generate-database:mapper method -i -e NetworkZone Rename table=networks_zones //generate-database:mapper method -i -e NetworkZone Update references=Config table=networks_zones //generate-database:mapper method -i -e NetworkZone DeleteOne-by-ID table=networks_zones // NetworkZone is a value object holding db-related details about a network zone (DNS). type NetworkZone struct { ID int `db:"order=yes"` ProjectID int `db:"omit=create,update"` Project string `db:"primary=yes&join=projects.name"` Name string `db:"primary=yes"` Description string } // NetworkZoneFilter specifies potential query parameter fields. type NetworkZoneFilter struct { ID *int Name *string Project *string } // ToAPI converts the DB records to an API record. func (n *NetworkZone) ToAPI(ctx context.Context, db tx) (*api.NetworkZone, error) { // Get the config. config, err := GetNetworkZoneConfig(ctx, db, n.ID) if err != nil { return nil, err } // Fill in the struct. out := api.NetworkZone{ Name: n.Name, Project: n.Project, NetworkZonePut: api.NetworkZonePut{ Description: n.Description, Config: config, }, } return &out, nil } incus-7.0.0/internal/server/db/cluster/networks_zones.interface.mapper.go000066400000000000000000000046111517523235500267040ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // NetworkZoneGenerated is an interface of generated methods for NetworkZone. type NetworkZoneGenerated interface { // GetNetworkZoneConfig returns all available NetworkZone Config // generator: NetworkZone GetMany GetNetworkZoneConfig(ctx context.Context, db tx, networkZoneID int, filters ...ConfigFilter) (map[string]string, error) // GetNetworkZones returns all available NetworkZones. // generator: NetworkZone GetMany GetNetworkZones(ctx context.Context, db dbtx, filters ...NetworkZoneFilter) ([]NetworkZone, error) // GetNetworkZone returns the NetworkZone with the given key. // generator: NetworkZone GetOne GetNetworkZone(ctx context.Context, db dbtx, project string, name string) (*NetworkZone, error) // NetworkZoneExists checks if a NetworkZone with the given key exists. // generator: NetworkZone Exists NetworkZoneExists(ctx context.Context, db dbtx, project string, name string) (bool, error) // CreateNetworkZoneConfig adds new NetworkZone Config to the database. // generator: NetworkZone Create CreateNetworkZoneConfig(ctx context.Context, db dbtx, networkZoneID int64, config map[string]string) error // CreateNetworkZone adds a new NetworkZone to the database. // generator: NetworkZone Create CreateNetworkZone(ctx context.Context, db dbtx, object NetworkZone) (int64, error) // GetNetworkZoneID return the ID of the NetworkZone with the given key. // generator: NetworkZone ID GetNetworkZoneID(ctx context.Context, db tx, project string, name string) (int64, error) // RenameNetworkZone renames the NetworkZone matching the given key parameters. // generator: NetworkZone Rename RenameNetworkZone(ctx context.Context, db dbtx, project string, name string, to string) error // UpdateNetworkZoneConfig updates the NetworkZone Config matching the given key parameters. // generator: NetworkZone Update UpdateNetworkZoneConfig(ctx context.Context, db tx, networkZoneID int64, config map[string]string) error // UpdateNetworkZone updates the NetworkZone matching the given key parameters. // generator: NetworkZone Update UpdateNetworkZone(ctx context.Context, db tx, project string, name string, object NetworkZone) error // DeleteNetworkZone deletes the NetworkZone matching the given key parameters. // generator: NetworkZone DeleteOne-by-ID DeleteNetworkZone(ctx context.Context, db dbtx, id int) error } incus-7.0.0/internal/server/db/cluster/networks_zones.mapper.go000066400000000000000000000363531517523235500247550ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var networkZoneObjects = RegisterStmt(` SELECT networks_zones.id, networks_zones.project_id, projects.name AS project, networks_zones.name, networks_zones.description FROM networks_zones JOIN projects ON networks_zones.project_id = projects.id ORDER BY networks_zones.id `) var networkZoneObjectsByID = RegisterStmt(` SELECT networks_zones.id, networks_zones.project_id, projects.name AS project, networks_zones.name, networks_zones.description FROM networks_zones JOIN projects ON networks_zones.project_id = projects.id WHERE ( networks_zones.id = ? ) ORDER BY networks_zones.id `) var networkZoneObjectsByName = RegisterStmt(` SELECT networks_zones.id, networks_zones.project_id, projects.name AS project, networks_zones.name, networks_zones.description FROM networks_zones JOIN projects ON networks_zones.project_id = projects.id WHERE ( networks_zones.name = ? ) ORDER BY networks_zones.id `) var networkZoneObjectsByProject = RegisterStmt(` SELECT networks_zones.id, networks_zones.project_id, projects.name AS project, networks_zones.name, networks_zones.description FROM networks_zones JOIN projects ON networks_zones.project_id = projects.id WHERE ( project = ? ) ORDER BY networks_zones.id `) var networkZoneObjectsByProjectAndName = RegisterStmt(` SELECT networks_zones.id, networks_zones.project_id, projects.name AS project, networks_zones.name, networks_zones.description FROM networks_zones JOIN projects ON networks_zones.project_id = projects.id WHERE ( project = ? AND networks_zones.name = ? ) ORDER BY networks_zones.id `) var networkZoneID = RegisterStmt(` SELECT networks_zones.id FROM networks_zones JOIN projects ON networks_zones.project_id = projects.id WHERE projects.name = ? AND networks_zones.name = ? `) var networkZoneCreate = RegisterStmt(` INSERT INTO networks_zones (project_id, name, description) VALUES ((SELECT projects.id FROM projects WHERE projects.name = ?), ?, ?) `) var networkZoneRename = RegisterStmt(` UPDATE networks_zones SET name = ? WHERE project_id = (SELECT projects.id FROM projects WHERE projects.name = ?) AND name = ? `) var networkZoneUpdate = RegisterStmt(` UPDATE networks_zones SET project_id = (SELECT projects.id FROM projects WHERE projects.name = ?), name = ?, description = ? WHERE id = ? `) var networkZoneDeleteByID = RegisterStmt(` DELETE FROM networks_zones WHERE id = ? `) // networkZoneColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the NetworkZone entity. func networkZoneColumns() string { return "networks_zones.id, networks_zones.project_id, projects.name AS project, networks_zones.name, networks_zones.description" } // getNetworkZones can be used to run handwritten sql.Stmts to return a slice of objects. func getNetworkZones(ctx context.Context, stmt *sql.Stmt, args ...any) ([]NetworkZone, error) { objects := make([]NetworkZone, 0) dest := func(scan func(dest ...any) error) error { n := NetworkZone{} err := scan(&n.ID, &n.ProjectID, &n.Project, &n.Name, &n.Description) if err != nil { return err } objects = append(objects, n) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_zones\" table: %w", err) } return objects, nil } // getNetworkZonesRaw can be used to run handwritten query strings to return a slice of objects. func getNetworkZonesRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]NetworkZone, error) { objects := make([]NetworkZone, 0) dest := func(scan func(dest ...any) error) error { n := NetworkZone{} err := scan(&n.ID, &n.ProjectID, &n.Project, &n.Name, &n.Description) if err != nil { return err } objects = append(objects, n) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_zones\" table: %w", err) } return objects, nil } // GetNetworkZones returns all available NetworkZones. // generator: NetworkZone GetMany func GetNetworkZones(ctx context.Context, db dbtx, filters ...NetworkZoneFilter) (_ []NetworkZone, _err error) { defer func() { _err = mapErr(_err, "NetworkZone") }() var err error // Result slice. objects := make([]NetworkZone, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, networkZoneObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.Project != nil && filter.Name != nil && filter.ID == nil { args = append(args, []any{filter.Project, filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkZoneObjectsByProjectAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneObjectsByProjectAndName\" prepared statement: %w", err) } break } query, err := StmtString(networkZoneObjectsByProjectAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.ID == nil && filter.Name == nil { args = append(args, []any{filter.Project}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkZoneObjectsByProject) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneObjectsByProject\" prepared statement: %w", err) } break } query, err := StmtString(networkZoneObjectsByProject) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Name != nil && filter.ID == nil && filter.Project == nil { args = append(args, []any{filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkZoneObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneObjectsByName\" prepared statement: %w", err) } break } query, err := StmtString(networkZoneObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID != nil && filter.Name == nil && filter.Project == nil { args = append(args, []any{filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkZoneObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneObjectsByID\" prepared statement: %w", err) } break } query, err := StmtString(networkZoneObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.Name == nil && filter.Project == nil { return nil, fmt.Errorf("Cannot filter on empty NetworkZoneFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getNetworkZones(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getNetworkZonesRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_zones\" table: %w", err) } return objects, nil } // GetNetworkZoneConfig returns all available NetworkZone Config // generator: NetworkZone GetMany func GetNetworkZoneConfig(ctx context.Context, db tx, networkZoneID int, filters ...ConfigFilter) (_ map[string]string, _err error) { defer func() { _err = mapErr(_err, "NetworkZone") }() networkZoneConfig, err := GetConfig(ctx, db, "networks_zones", "network_zone", filters...) if err != nil { return nil, err } config, ok := networkZoneConfig[networkZoneID] if !ok { config = map[string]string{} } return config, nil } // GetNetworkZone returns the NetworkZone with the given key. // generator: NetworkZone GetOne func GetNetworkZone(ctx context.Context, db dbtx, project string, name string) (_ *NetworkZone, _err error) { defer func() { _err = mapErr(_err, "NetworkZone") }() filter := NetworkZoneFilter{} filter.Project = &project filter.Name = &name objects, err := GetNetworkZones(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_zones\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"networks_zones\" entry matches") } } // NetworkZoneExists checks if a NetworkZone with the given key exists. // generator: NetworkZone Exists func NetworkZoneExists(ctx context.Context, db dbtx, project string, name string) (_ bool, _err error) { defer func() { _err = mapErr(_err, "NetworkZone") }() stmt, err := Stmt(db, networkZoneID) if err != nil { return false, fmt.Errorf("Failed to get \"networkZoneID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, project, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("Failed to get \"networks_zones\" ID: %w", err) } return true, nil } // CreateNetworkZone adds a new NetworkZone to the database. // generator: NetworkZone Create func CreateNetworkZone(ctx context.Context, db dbtx, object NetworkZone) (_ int64, _err error) { defer func() { _err = mapErr(_err, "NetworkZone") }() args := make([]any, 3) // Populate the statement arguments. args[0] = object.Project args[1] = object.Name args[2] = object.Description // Prepared statement to use. stmt, err := Stmt(db, networkZoneCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"networkZoneCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"networks_zones\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"networks_zones\" entry ID: %w", err) } return id, nil } // CreateNetworkZoneConfig adds new NetworkZone Config to the database. // generator: NetworkZone Create func CreateNetworkZoneConfig(ctx context.Context, db dbtx, networkZoneID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "NetworkZone") }() referenceID := int(networkZoneID) for key, value := range config { insert := Config{ ReferenceID: referenceID, Key: key, Value: value, } err := CreateConfig(ctx, db, "networks_zones", "network_zone", insert) if err != nil { return fmt.Errorf("Insert Config failed for NetworkZone: %w", err) } } return nil } // GetNetworkZoneID return the ID of the NetworkZone with the given key. // generator: NetworkZone ID func GetNetworkZoneID(ctx context.Context, db tx, project string, name string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "NetworkZone") }() stmt, err := Stmt(db, networkZoneID) if err != nil { return -1, fmt.Errorf("Failed to get \"networkZoneID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, project, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"networks_zones\" ID: %w", err) } return id, nil } // RenameNetworkZone renames the NetworkZone matching the given key parameters. // generator: NetworkZone Rename func RenameNetworkZone(ctx context.Context, db dbtx, project string, name string, to string) (_err error) { defer func() { _err = mapErr(_err, "NetworkZone") }() stmt, err := Stmt(db, networkZoneRename) if err != nil { return fmt.Errorf("Failed to get \"networkZoneRename\" prepared statement: %w", err) } result, err := stmt.Exec(to, project, name) if err != nil { return fmt.Errorf("Rename NetworkZone failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows failed: %w", err) } if n != 1 { return fmt.Errorf("Query affected %d rows instead of 1", n) } return nil } // UpdateNetworkZone updates the NetworkZone matching the given key parameters. // generator: NetworkZone Update func UpdateNetworkZone(ctx context.Context, db tx, project string, name string, object NetworkZone) (_err error) { defer func() { _err = mapErr(_err, "NetworkZone") }() id, err := GetNetworkZoneID(ctx, db, project, name) if err != nil { return err } stmt, err := Stmt(db, networkZoneUpdate) if err != nil { return fmt.Errorf("Failed to get \"networkZoneUpdate\" prepared statement: %w", err) } result, err := stmt.Exec(object.Project, object.Name, object.Description, id) if err != nil { return fmt.Errorf("Update \"networks_zones\" entry failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } return nil } // UpdateNetworkZoneConfig updates the NetworkZone Config matching the given key parameters. // generator: NetworkZone Update func UpdateNetworkZoneConfig(ctx context.Context, db tx, networkZoneID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "NetworkZone") }() err := UpdateConfig(ctx, db, "networks_zones", "network_zone", int(networkZoneID), config) if err != nil { return fmt.Errorf("Replace Config for NetworkZone failed: %w", err) } return nil } // DeleteNetworkZone deletes the NetworkZone matching the given key parameters. // generator: NetworkZone DeleteOne-by-ID func DeleteNetworkZone(ctx context.Context, db dbtx, id int) (_err error) { defer func() { _err = mapErr(_err, "NetworkZone") }() stmt, err := Stmt(db, networkZoneDeleteByID) if err != nil { return fmt.Errorf("Failed to get \"networkZoneDeleteByID\" prepared statement: %w", err) } result, err := stmt.Exec(id) if err != nil { return fmt.Errorf("Delete \"networks_zones\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d NetworkZone rows instead of 1", n) } return nil } incus-7.0.0/internal/server/db/cluster/networks_zones_records.go000066400000000000000000000062201517523235500252010ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "context" "github.com/lxc/incus/v7/shared/api" ) // Code generation directives. // //generate-database:mapper target networks_zones_records.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // // Statements: //generate-database:mapper stmt -e NetworkZoneRecord objects table=networks_zones_records //generate-database:mapper stmt -e NetworkZoneRecord objects-by-ID table=networks_zones_records //generate-database:mapper stmt -e NetworkZoneRecord objects-by-Name table=networks_zones_records //generate-database:mapper stmt -e NetworkZoneRecord objects-by-NetworkZoneID table=networks_zones_records //generate-database:mapper stmt -e NetworkZoneRecord objects-by-NetworkZoneID-and-Name table=networks_zones_records //generate-database:mapper stmt -e NetworkZoneRecord objects-by-NetworkZoneID-and-ID table=networks_zones_records //generate-database:mapper stmt -e NetworkZoneRecord id table=networks_zones_records //generate-database:mapper stmt -e NetworkZoneRecord create table=networks_zones_records //generate-database:mapper stmt -e NetworkZoneRecord rename table=networks_zones_records //generate-database:mapper stmt -e NetworkZoneRecord update table=networks_zones_records //generate-database:mapper stmt -e NetworkZoneRecord delete-by-NetworkZoneID-and-ID table=networks_zones_records // // Methods: //generate-database:mapper method -i -e NetworkZoneRecord GetMany references=Config table=networks_zones_records //generate-database:mapper method -i -e NetworkZoneRecord GetOne table=networks_zones_records //generate-database:mapper method -i -e NetworkZoneRecord Exists table=networks_zones_records //generate-database:mapper method -i -e NetworkZoneRecord Create references=Config table=networks_zones_records //generate-database:mapper method -i -e NetworkZoneRecord ID table=networks_zones_records //generate-database:mapper method -i -e NetworkZoneRecord Rename table=networks_zones_records //generate-database:mapper method -i -e NetworkZoneRecord Update references=Config table=networks_zones_records //generate-database:mapper method -i -e NetworkZoneRecord DeleteOne-by-NetworkZoneID-and-ID table=networks_zones_records // NetworkZoneRecord is a value object holding db-related details about a DNS record in a network zone. type NetworkZoneRecord struct { ID int `db:"order=yes"` NetworkZoneID int `db:"primary=yes"` Name string `db:"primary=yes"` Description string Entries []api.NetworkZoneRecordEntry `db:"marshal=json"` } // NetworkZoneRecordFilter defines the optional WHERE-clause fields. type NetworkZoneRecordFilter struct { ID *int Name *string NetworkZoneID *int } // ToAPI converts the DB record into external API type. func (r *NetworkZoneRecord) ToAPI(ctx context.Context, db tx) (*api.NetworkZoneRecord, error) { config, err := GetNetworkZoneRecordConfig(ctx, db, r.ID) if err != nil { return nil, err } out := api.NetworkZoneRecord{ Name: r.Name, NetworkZoneRecordPut: api.NetworkZoneRecordPut{ Description: r.Description, Entries: r.Entries, Config: config, }, } return &out, nil } incus-7.0.0/internal/server/db/cluster/networks_zones_records.interface.mapper.go000066400000000000000000000054071517523235500304310ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // NetworkZoneRecordGenerated is an interface of generated methods for NetworkZoneRecord. type NetworkZoneRecordGenerated interface { // GetNetworkZoneRecordConfig returns all available NetworkZoneRecord Config // generator: NetworkZoneRecord GetMany GetNetworkZoneRecordConfig(ctx context.Context, db tx, networkZoneRecordID int, filters ...ConfigFilter) (map[string]string, error) // GetNetworkZoneRecords returns all available NetworkZoneRecords. // generator: NetworkZoneRecord GetMany GetNetworkZoneRecords(ctx context.Context, db dbtx, filters ...NetworkZoneRecordFilter) ([]NetworkZoneRecord, error) // GetNetworkZoneRecord returns the NetworkZoneRecord with the given key. // generator: NetworkZoneRecord GetOne GetNetworkZoneRecord(ctx context.Context, db dbtx, networkZoneID int, name string) (*NetworkZoneRecord, error) // NetworkZoneRecordExists checks if a NetworkZoneRecord with the given key exists. // generator: NetworkZoneRecord Exists NetworkZoneRecordExists(ctx context.Context, db dbtx, networkZoneID int, name string) (bool, error) // CreateNetworkZoneRecordConfig adds new NetworkZoneRecord Config to the database. // generator: NetworkZoneRecord Create CreateNetworkZoneRecordConfig(ctx context.Context, db dbtx, networkZoneRecordID int64, config map[string]string) error // CreateNetworkZoneRecord adds a new NetworkZoneRecord to the database. // generator: NetworkZoneRecord Create CreateNetworkZoneRecord(ctx context.Context, db dbtx, object NetworkZoneRecord) (int64, error) // GetNetworkZoneRecordID return the ID of the NetworkZoneRecord with the given key. // generator: NetworkZoneRecord ID GetNetworkZoneRecordID(ctx context.Context, db tx, networkZoneID int, name string) (int64, error) // RenameNetworkZoneRecord renames the NetworkZoneRecord matching the given key parameters. // generator: NetworkZoneRecord Rename RenameNetworkZoneRecord(ctx context.Context, db dbtx, networkZoneID int, name string, to string) error // UpdateNetworkZoneRecordConfig updates the NetworkZoneRecord Config matching the given key parameters. // generator: NetworkZoneRecord Update UpdateNetworkZoneRecordConfig(ctx context.Context, db tx, networkZoneRecordID int64, config map[string]string) error // UpdateNetworkZoneRecord updates the NetworkZoneRecord matching the given key parameters. // generator: NetworkZoneRecord Update UpdateNetworkZoneRecord(ctx context.Context, db tx, networkZoneID int, name string, object NetworkZoneRecord) error // DeleteNetworkZoneRecord deletes the NetworkZoneRecord matching the given key parameters. // generator: NetworkZoneRecord DeleteOne-by-NetworkZoneID-and-ID DeleteNetworkZoneRecord(ctx context.Context, db dbtx, networkZoneID int, id int) error } incus-7.0.0/internal/server/db/cluster/networks_zones_records.mapper.go000066400000000000000000000440521517523235500264710ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var networkZoneRecordObjects = RegisterStmt(` SELECT networks_zones_records.id, networks_zones_records.network_zone_id, networks_zones_records.name, networks_zones_records.description, networks_zones_records.entries FROM networks_zones_records ORDER BY networks_zones_records.id `) var networkZoneRecordObjectsByID = RegisterStmt(` SELECT networks_zones_records.id, networks_zones_records.network_zone_id, networks_zones_records.name, networks_zones_records.description, networks_zones_records.entries FROM networks_zones_records WHERE ( networks_zones_records.id = ? ) ORDER BY networks_zones_records.id `) var networkZoneRecordObjectsByName = RegisterStmt(` SELECT networks_zones_records.id, networks_zones_records.network_zone_id, networks_zones_records.name, networks_zones_records.description, networks_zones_records.entries FROM networks_zones_records WHERE ( networks_zones_records.name = ? ) ORDER BY networks_zones_records.id `) var networkZoneRecordObjectsByNetworkZoneID = RegisterStmt(` SELECT networks_zones_records.id, networks_zones_records.network_zone_id, networks_zones_records.name, networks_zones_records.description, networks_zones_records.entries FROM networks_zones_records WHERE ( networks_zones_records.network_zone_id = ? ) ORDER BY networks_zones_records.id `) var networkZoneRecordObjectsByNetworkZoneIDAndName = RegisterStmt(` SELECT networks_zones_records.id, networks_zones_records.network_zone_id, networks_zones_records.name, networks_zones_records.description, networks_zones_records.entries FROM networks_zones_records WHERE ( networks_zones_records.network_zone_id = ? AND networks_zones_records.name = ? ) ORDER BY networks_zones_records.id `) var networkZoneRecordObjectsByNetworkZoneIDAndID = RegisterStmt(` SELECT networks_zones_records.id, networks_zones_records.network_zone_id, networks_zones_records.name, networks_zones_records.description, networks_zones_records.entries FROM networks_zones_records WHERE ( networks_zones_records.network_zone_id = ? AND networks_zones_records.id = ? ) ORDER BY networks_zones_records.id `) var networkZoneRecordID = RegisterStmt(` SELECT networks_zones_records.id FROM networks_zones_records WHERE networks_zones_records.network_zone_id = ? AND networks_zones_records.name = ? `) var networkZoneRecordCreate = RegisterStmt(` INSERT INTO networks_zones_records (network_zone_id, name, description, entries) VALUES (?, ?, ?, ?) `) var networkZoneRecordRename = RegisterStmt(` UPDATE networks_zones_records SET name = ? WHERE network_zone_id = ? AND name = ? `) var networkZoneRecordUpdate = RegisterStmt(` UPDATE networks_zones_records SET network_zone_id = ?, name = ?, description = ?, entries = ? WHERE id = ? `) var networkZoneRecordDeleteByNetworkZoneIDAndID = RegisterStmt(` DELETE FROM networks_zones_records WHERE network_zone_id = ? AND id = ? `) // networkZoneRecordColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the NetworkZoneRecord entity. func networkZoneRecordColumns() string { return "networks_zones_records.id, networks_zones_records.network_zone_id, networks_zones_records.name, networks_zones_records.description, networks_zones_records.entries" } // getNetworkZoneRecords can be used to run handwritten sql.Stmts to return a slice of objects. func getNetworkZoneRecords(ctx context.Context, stmt *sql.Stmt, args ...any) ([]NetworkZoneRecord, error) { objects := make([]NetworkZoneRecord, 0) dest := func(scan func(dest ...any) error) error { n := NetworkZoneRecord{} var entriesStr string err := scan(&n.ID, &n.NetworkZoneID, &n.Name, &n.Description, &entriesStr) if err != nil { return err } err = unmarshalJSON(entriesStr, &n.Entries) if err != nil { return err } objects = append(objects, n) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_zones_records\" table: %w", err) } return objects, nil } // getNetworkZoneRecordsRaw can be used to run handwritten query strings to return a slice of objects. func getNetworkZoneRecordsRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]NetworkZoneRecord, error) { objects := make([]NetworkZoneRecord, 0) dest := func(scan func(dest ...any) error) error { n := NetworkZoneRecord{} var entriesStr string err := scan(&n.ID, &n.NetworkZoneID, &n.Name, &n.Description, &entriesStr) if err != nil { return err } err = unmarshalJSON(entriesStr, &n.Entries) if err != nil { return err } objects = append(objects, n) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_zones_records\" table: %w", err) } return objects, nil } // GetNetworkZoneRecords returns all available NetworkZoneRecords. // generator: NetworkZoneRecord GetMany func GetNetworkZoneRecords(ctx context.Context, db dbtx, filters ...NetworkZoneRecordFilter) (_ []NetworkZoneRecord, _err error) { defer func() { _err = mapErr(_err, "NetworkZoneRecord") }() var err error // Result slice. objects := make([]NetworkZoneRecord, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, networkZoneRecordObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneRecordObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.NetworkZoneID != nil && filter.Name != nil && filter.ID == nil { args = append(args, []any{filter.NetworkZoneID, filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkZoneRecordObjectsByNetworkZoneIDAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneRecordObjectsByNetworkZoneIDAndName\" prepared statement: %w", err) } break } query, err := StmtString(networkZoneRecordObjectsByNetworkZoneIDAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneRecordObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.NetworkZoneID != nil && filter.ID != nil && filter.Name == nil { args = append(args, []any{filter.NetworkZoneID, filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkZoneRecordObjectsByNetworkZoneIDAndID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneRecordObjectsByNetworkZoneIDAndID\" prepared statement: %w", err) } break } query, err := StmtString(networkZoneRecordObjectsByNetworkZoneIDAndID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneRecordObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.NetworkZoneID != nil && filter.ID == nil && filter.Name == nil { args = append(args, []any{filter.NetworkZoneID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkZoneRecordObjectsByNetworkZoneID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneRecordObjectsByNetworkZoneID\" prepared statement: %w", err) } break } query, err := StmtString(networkZoneRecordObjectsByNetworkZoneID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneRecordObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Name != nil && filter.ID == nil && filter.NetworkZoneID == nil { args = append(args, []any{filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkZoneRecordObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneRecordObjectsByName\" prepared statement: %w", err) } break } query, err := StmtString(networkZoneRecordObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneRecordObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID != nil && filter.Name == nil && filter.NetworkZoneID == nil { args = append(args, []any{filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, networkZoneRecordObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneRecordObjectsByID\" prepared statement: %w", err) } break } query, err := StmtString(networkZoneRecordObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"networkZoneRecordObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.Name == nil && filter.NetworkZoneID == nil { return nil, fmt.Errorf("Cannot filter on empty NetworkZoneRecordFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getNetworkZoneRecords(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getNetworkZoneRecordsRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_zones_records\" table: %w", err) } return objects, nil } // GetNetworkZoneRecordConfig returns all available NetworkZoneRecord Config // generator: NetworkZoneRecord GetMany func GetNetworkZoneRecordConfig(ctx context.Context, db tx, networkZoneRecordID int, filters ...ConfigFilter) (_ map[string]string, _err error) { defer func() { _err = mapErr(_err, "NetworkZoneRecord") }() networkZoneRecordConfig, err := GetConfig(ctx, db, "networks_zones_records", "network_zone_record", filters...) if err != nil { return nil, err } config, ok := networkZoneRecordConfig[networkZoneRecordID] if !ok { config = map[string]string{} } return config, nil } // GetNetworkZoneRecord returns the NetworkZoneRecord with the given key. // generator: NetworkZoneRecord GetOne func GetNetworkZoneRecord(ctx context.Context, db dbtx, networkZoneID int, name string) (_ *NetworkZoneRecord, _err error) { defer func() { _err = mapErr(_err, "NetworkZoneRecord") }() filter := NetworkZoneRecordFilter{} filter.NetworkZoneID = &networkZoneID filter.Name = &name objects, err := GetNetworkZoneRecords(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"networks_zones_records\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"networks_zones_records\" entry matches") } } // NetworkZoneRecordExists checks if a NetworkZoneRecord with the given key exists. // generator: NetworkZoneRecord Exists func NetworkZoneRecordExists(ctx context.Context, db dbtx, networkZoneID int, name string) (_ bool, _err error) { defer func() { _err = mapErr(_err, "NetworkZoneRecord") }() stmt, err := Stmt(db, networkZoneRecordID) if err != nil { return false, fmt.Errorf("Failed to get \"networkZoneRecordID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, networkZoneID, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("Failed to get \"networks_zones_records\" ID: %w", err) } return true, nil } // CreateNetworkZoneRecord adds a new NetworkZoneRecord to the database. // generator: NetworkZoneRecord Create func CreateNetworkZoneRecord(ctx context.Context, db dbtx, object NetworkZoneRecord) (_ int64, _err error) { defer func() { _err = mapErr(_err, "NetworkZoneRecord") }() args := make([]any, 4) // Populate the statement arguments. args[0] = object.NetworkZoneID args[1] = object.Name args[2] = object.Description marshaledEntries, err := marshalJSON(object.Entries) if err != nil { return -1, err } args[3] = marshaledEntries // Prepared statement to use. stmt, err := Stmt(db, networkZoneRecordCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"networkZoneRecordCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"networks_zones_records\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"networks_zones_records\" entry ID: %w", err) } return id, nil } // CreateNetworkZoneRecordConfig adds new NetworkZoneRecord Config to the database. // generator: NetworkZoneRecord Create func CreateNetworkZoneRecordConfig(ctx context.Context, db dbtx, networkZoneRecordID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "NetworkZoneRecord") }() referenceID := int(networkZoneRecordID) for key, value := range config { insert := Config{ ReferenceID: referenceID, Key: key, Value: value, } err := CreateConfig(ctx, db, "networks_zones_records", "network_zone_record", insert) if err != nil { return fmt.Errorf("Insert Config failed for NetworkZoneRecord: %w", err) } } return nil } // GetNetworkZoneRecordID return the ID of the NetworkZoneRecord with the given key. // generator: NetworkZoneRecord ID func GetNetworkZoneRecordID(ctx context.Context, db tx, networkZoneID int, name string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "NetworkZoneRecord") }() stmt, err := Stmt(db, networkZoneRecordID) if err != nil { return -1, fmt.Errorf("Failed to get \"networkZoneRecordID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, networkZoneID, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"networks_zones_records\" ID: %w", err) } return id, nil } // RenameNetworkZoneRecord renames the NetworkZoneRecord matching the given key parameters. // generator: NetworkZoneRecord Rename func RenameNetworkZoneRecord(ctx context.Context, db dbtx, networkZoneID int, name string, to string) (_err error) { defer func() { _err = mapErr(_err, "NetworkZoneRecord") }() stmt, err := Stmt(db, networkZoneRecordRename) if err != nil { return fmt.Errorf("Failed to get \"networkZoneRecordRename\" prepared statement: %w", err) } result, err := stmt.Exec(to, networkZoneID, name) if err != nil { return fmt.Errorf("Rename NetworkZoneRecord failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows failed: %w", err) } if n != 1 { return fmt.Errorf("Query affected %d rows instead of 1", n) } return nil } // UpdateNetworkZoneRecord updates the NetworkZoneRecord matching the given key parameters. // generator: NetworkZoneRecord Update func UpdateNetworkZoneRecord(ctx context.Context, db tx, networkZoneID int, name string, object NetworkZoneRecord) (_err error) { defer func() { _err = mapErr(_err, "NetworkZoneRecord") }() id, err := GetNetworkZoneRecordID(ctx, db, networkZoneID, name) if err != nil { return err } stmt, err := Stmt(db, networkZoneRecordUpdate) if err != nil { return fmt.Errorf("Failed to get \"networkZoneRecordUpdate\" prepared statement: %w", err) } marshaledEntries, err := marshalJSON(object.Entries) if err != nil { return err } result, err := stmt.Exec(object.NetworkZoneID, object.Name, object.Description, marshaledEntries, id) if err != nil { return fmt.Errorf("Update \"networks_zones_records\" entry failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } return nil } // UpdateNetworkZoneRecordConfig updates the NetworkZoneRecord Config matching the given key parameters. // generator: NetworkZoneRecord Update func UpdateNetworkZoneRecordConfig(ctx context.Context, db tx, networkZoneRecordID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "NetworkZoneRecord") }() err := UpdateConfig(ctx, db, "networks_zones_records", "network_zone_record", int(networkZoneRecordID), config) if err != nil { return fmt.Errorf("Replace Config for NetworkZoneRecord failed: %w", err) } return nil } // DeleteNetworkZoneRecord deletes the NetworkZoneRecord matching the given key parameters. // generator: NetworkZoneRecord DeleteOne-by-NetworkZoneID-and-ID func DeleteNetworkZoneRecord(ctx context.Context, db dbtx, networkZoneID int, id int) (_err error) { defer func() { _err = mapErr(_err, "NetworkZoneRecord") }() stmt, err := Stmt(db, networkZoneRecordDeleteByNetworkZoneIDAndID) if err != nil { return fmt.Errorf("Failed to get \"networkZoneRecordDeleteByNetworkZoneIDAndID\" prepared statement: %w", err) } result, err := stmt.Exec(networkZoneID, id) if err != nil { return fmt.Errorf("Delete \"networks_zones_records\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d NetworkZoneRecord rows instead of 1", n) } return nil } incus-7.0.0/internal/server/db/cluster/nodes.go000066400000000000000000000007071517523235500215020ustar00rootroot00000000000000package cluster // Code generation directives. // //generate-database:mapper target nodes.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e node id // //generate-database:mapper method -i -e node ID // Node represents a cluster member. type Node struct { ID int Name string } // NodeFilter specifies potential query parameter fields. type NodeFilter struct { Name *string } incus-7.0.0/internal/server/db/cluster/nodes.interface.mapper.go000066400000000000000000000004751517523235500247260ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // NodeGenerated is an interface of generated methods for Node. type NodeGenerated interface { // GetNodeID return the ID of the node with the given key. // generator: node ID GetNodeID(ctx context.Context, db tx, name string) (int64, error) } incus-7.0.0/internal/server/db/cluster/nodes.mapper.go000066400000000000000000000015271517523235500227660ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" ) var nodeID = RegisterStmt(` SELECT nodes.id FROM nodes WHERE nodes.name = ? `) // GetNodeID return the ID of the node with the given key. // generator: node ID func GetNodeID(ctx context.Context, db tx, name string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Node") }() stmt, err := Stmt(db, nodeID) if err != nil { return -1, fmt.Errorf("Failed to get \"nodeID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"nodes\" ID: %w", err) } return id, nil } incus-7.0.0/internal/server/db/cluster/nodes_cluster_groups.go000066400000000000000000000024671517523235500246470ustar00rootroot00000000000000package cluster // Code generation directives. // //generate-database:mapper target nodes_cluster_groups.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e node_cluster_group objects table=nodes_cluster_groups //generate-database:mapper stmt -e node_cluster_group objects-by-GroupID table=nodes_cluster_groups //generate-database:mapper stmt -e node_cluster_group id table=nodes_cluster_groups //generate-database:mapper stmt -e node_cluster_group create table=nodes_cluster_groups //generate-database:mapper stmt -e node_cluster_group delete-by-GroupID table=nodes_cluster_groups // //generate-database:mapper method -e node_cluster_group GetMany //generate-database:mapper method -e node_cluster_group Create //generate-database:mapper method -e node_cluster_group Exists //generate-database:mapper method -e node_cluster_group ID //generate-database:mapper method -e node_cluster_group DeleteOne-by-GroupID // NodeClusterGroup associates a node to a cluster group. type NodeClusterGroup struct { GroupID int `db:"primary=yes"` Node string `db:"join=nodes.name"` NodeID int `db:"omit=create,objects,objects-by-GroupID"` } // NodeClusterGroupFilter specifies potential query parameter fields. type NodeClusterGroupFilter struct { GroupID *int } incus-7.0.0/internal/server/db/cluster/nodes_cluster_groups.interface.mapper.go000066400000000000000000000000631517523235500300570ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster incus-7.0.0/internal/server/db/cluster/nodes_cluster_groups.mapper.go000066400000000000000000000175341517523235500261330ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var nodeClusterGroupObjects = RegisterStmt(` SELECT nodes_cluster_groups.group_id, nodes.name AS node FROM nodes_cluster_groups JOIN nodes ON nodes_cluster_groups.node_id = nodes.id ORDER BY nodes_cluster_groups.group_id `) var nodeClusterGroupObjectsByGroupID = RegisterStmt(` SELECT nodes_cluster_groups.group_id, nodes.name AS node FROM nodes_cluster_groups JOIN nodes ON nodes_cluster_groups.node_id = nodes.id WHERE ( nodes_cluster_groups.group_id = ? ) ORDER BY nodes_cluster_groups.group_id `) var nodeClusterGroupID = RegisterStmt(` SELECT nodes_cluster_groups.id FROM nodes_cluster_groups WHERE nodes_cluster_groups.group_id = ? `) var nodeClusterGroupCreate = RegisterStmt(` INSERT INTO nodes_cluster_groups (group_id, node_id) VALUES (?, (SELECT nodes.id FROM nodes WHERE nodes.name = ?)) `) var nodeClusterGroupDeleteByGroupID = RegisterStmt(` DELETE FROM nodes_cluster_groups WHERE group_id = ? `) // nodeClusterGroupColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the NodeClusterGroup entity. func nodeClusterGroupColumns() string { return "nodes_clusters_groups.group_id, nodes.name AS node" } // getNodeClusterGroups can be used to run handwritten sql.Stmts to return a slice of objects. func getNodeClusterGroups(ctx context.Context, stmt *sql.Stmt, args ...any) ([]NodeClusterGroup, error) { objects := make([]NodeClusterGroup, 0) dest := func(scan func(dest ...any) error) error { n := NodeClusterGroup{} err := scan(&n.GroupID, &n.Node) if err != nil { return err } objects = append(objects, n) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"nodes_clusters_groups\" table: %w", err) } return objects, nil } // getNodeClusterGroupsRaw can be used to run handwritten query strings to return a slice of objects. func getNodeClusterGroupsRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]NodeClusterGroup, error) { objects := make([]NodeClusterGroup, 0) dest := func(scan func(dest ...any) error) error { n := NodeClusterGroup{} err := scan(&n.GroupID, &n.Node) if err != nil { return err } objects = append(objects, n) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"nodes_clusters_groups\" table: %w", err) } return objects, nil } // GetNodeClusterGroups returns all available node_cluster_groups. // generator: node_cluster_group GetMany func GetNodeClusterGroups(ctx context.Context, db dbtx, filters ...NodeClusterGroupFilter) (_ []NodeClusterGroup, _err error) { defer func() { _err = mapErr(_err, "Node_cluster_group") }() var err error // Result slice. objects := make([]NodeClusterGroup, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, nodeClusterGroupObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"nodeClusterGroupObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.GroupID != nil { args = append(args, []any{filter.GroupID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, nodeClusterGroupObjectsByGroupID) if err != nil { return nil, fmt.Errorf("Failed to get \"nodeClusterGroupObjectsByGroupID\" prepared statement: %w", err) } break } query, err := StmtString(nodeClusterGroupObjectsByGroupID) if err != nil { return nil, fmt.Errorf("Failed to get \"nodeClusterGroupObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.GroupID == nil { return nil, fmt.Errorf("Cannot filter on empty NodeClusterGroupFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getNodeClusterGroups(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getNodeClusterGroupsRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"nodes_clusters_groups\" table: %w", err) } return objects, nil } // CreateNodeClusterGroup adds a new node_cluster_group to the database. // generator: node_cluster_group Create func CreateNodeClusterGroup(ctx context.Context, db dbtx, object NodeClusterGroup) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Node_cluster_group") }() args := make([]any, 2) // Populate the statement arguments. args[0] = object.GroupID args[1] = object.Node // Prepared statement to use. stmt, err := Stmt(db, nodeClusterGroupCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"nodeClusterGroupCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"nodes_clusters_groups\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"nodes_clusters_groups\" entry ID: %w", err) } return id, nil } // NodeClusterGroupExists checks if a node_cluster_group with the given key exists. // generator: node_cluster_group Exists func NodeClusterGroupExists(ctx context.Context, db dbtx, groupID int) (_ bool, _err error) { defer func() { _err = mapErr(_err, "Node_cluster_group") }() stmt, err := Stmt(db, nodeClusterGroupID) if err != nil { return false, fmt.Errorf("Failed to get \"nodeClusterGroupID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, groupID) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("Failed to get \"nodes_clusters_groups\" ID: %w", err) } return true, nil } // GetNodeClusterGroupID return the ID of the node_cluster_group with the given key. // generator: node_cluster_group ID func GetNodeClusterGroupID(ctx context.Context, db tx, groupID int) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Node_cluster_group") }() stmt, err := Stmt(db, nodeClusterGroupID) if err != nil { return -1, fmt.Errorf("Failed to get \"nodeClusterGroupID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, groupID) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"nodes_clusters_groups\" ID: %w", err) } return id, nil } // DeleteNodeClusterGroup deletes the node_cluster_group matching the given key parameters. // generator: node_cluster_group DeleteOne-by-GroupID func DeleteNodeClusterGroup(ctx context.Context, db dbtx, groupID int) (_err error) { defer func() { _err = mapErr(_err, "Node_cluster_group") }() stmt, err := Stmt(db, nodeClusterGroupDeleteByGroupID) if err != nil { return fmt.Errorf("Failed to get \"nodeClusterGroupDeleteByGroupID\" prepared statement: %w", err) } result, err := stmt.Exec(groupID) if err != nil { return fmt.Errorf("Delete \"nodes_clusters_groups\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d NodeClusterGroup rows instead of 1", n) } return nil } incus-7.0.0/internal/server/db/cluster/open.go000066400000000000000000000206441517523235500213350ustar00rootroot00000000000000package cluster import ( "context" "database/sql" "errors" "fmt" "path/filepath" "strings" "sync/atomic" driver "github.com/cowsql/go-cowsql/driver" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/internal/server/db/schema" daemonUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" ) // Open the cluster database object. // // The name argument is the name of the cluster database. It defaults to // 'db.bin', but can be overwritten for testing. // // The dialer argument is a function that returns a gRPC dialer that can be // used to connect to a database node using the gRPC SQL package. func Open(name string, store driver.NodeStore, options ...driver.Option) (*sql.DB, error) { driver, err := driver.New(store, options...) if err != nil { return nil, fmt.Errorf("Failed to create dqlite driver: %w", err) } driverName := dqliteDriverName() sql.Register(driverName, driver) // Create the cluster db. This won't immediately establish any network // connection, that will happen only when a db transaction is started // (see the database/sql connection pooling code for more details). if name == "" { name = "db.bin" } db, err := sql.Open(driverName, name) if err != nil { return nil, fmt.Errorf("cannot open cluster database: %w", err) } return db, nil } // EnsureSchema applies all relevant schema updates to the cluster database. // // Before actually doing anything, this function will make sure that all nodes // in the cluster have a schema version and a number of API extensions that // match our one. If it's not the case, we either return an error (if some // nodes have version greater than us and we need to be upgraded), or return // false and no error (if some nodes have a lower version, and we need to wait // till they get upgraded and restarted). func EnsureSchema(db *sql.DB, address string, dir string) (bool, error) { someNodesAreBehind := false apiExtensions := version.APIExtensionsCount() backupDone := false hook := func(ctx context.Context, schemaVersion int, tx *sql.Tx) error { // Check if this is a fresh instance. isUpdate, err := schema.DoesSchemaTableExist(ctx, tx) if err != nil { return fmt.Errorf("Failed to check if schema table exists: %w", err) } if !isUpdate { return nil } // Check if we're clustered clustered := true n, err := selectUnclusteredNodesCount(ctx, tx) if err != nil { return fmt.Errorf("Failed to fetch standalone member count: %w", err) } if n > 1 { // This should never happen, since we only add cluster members with valid addresses. return errors.New("Found more than one cluster member with a standalone address (0.0.0.0)") } else if n == 1 { clustered = false } // If we're not clustered, backup the local cluster database directory // before performing any schema change. This makes sense only in the // non-clustered case, because otherwise the directory would be // re-populated by replication. if !clustered && !backupDone { logger.Infof("Updating the global schema. Backup made as \"global.bak\"") err := internalUtil.DirCopy( filepath.Join(dir, "global"), filepath.Join(dir, "global.bak"), ) if err != nil { return fmt.Errorf("Failed to backup global database: %w", err) } backupDone = true } if schemaVersion == -1 { logger.Debugf("Running pre-update queries from file for global DB schema") } else { logger.Debugf("Updating global DB schema from %d to %d", schemaVersion, schemaVersion+1) } return nil } check := func(ctx context.Context, current int, tx *sql.Tx) error { // If we're bootstrapping a fresh schema, skip any check, since // it's safe to assume we are the only node. if current == 0 { return nil } // Check if we're clustered n, err := selectUnclusteredNodesCount(ctx, tx) if err != nil { return fmt.Errorf("Failed to fetch standalone member count: %w", err) } if n > 1 { // This should never happen, since we only add nodes with valid addresses. return errors.New("Found more than one cluster member with a standalone address (0.0.0.0)") } else if n == 1 { address = "0.0.0.0" // We're not clustered } // Update the schema and api_extension columns of ourselves. err = updateNodeVersion(tx, address, apiExtensions) if err != nil { return fmt.Errorf("Failed to update cluster member version info: %w", err) } err = checkClusterIsUpgradable(ctx, tx, [2]int{len(updates), apiExtensions}) if errors.Is(err, errSomeNodesAreBehind) { someNodesAreBehind = true return schema.ErrGracefulAbort } return err } schema := Schema() schema.File(filepath.Join(dir, "patch.global.sql")) // Optional custom queries schema.Check(check) schema.Hook(hook) var initial int err := query.Retry(context.TODO(), func(_ context.Context) error { var err error initial, err = schema.Ensure(db) return err }) if someNodesAreBehind { return false, nil } if err != nil { return false, err } // When creating a database from scratch, insert an entry for node // 1. This is needed for referential integrity with other tables. Also, // create a default profile. if initial == 0 { arch, err := osarch.ArchitectureGetLocalID() if err != nil { return false, err } err = query.Transaction(context.TODO(), db, func(ctx context.Context, tx *sql.Tx) error { stmt := ` INSERT INTO nodes(id, name, address, schema, api_extensions, arch, description) VALUES(1, 'none', '0.0.0.0', ?, ?, ?, '') ` _, err = tx.Exec(stmt, SchemaVersion, apiExtensions, arch) if err != nil { return err } // Default project var defaultProjectStmt strings.Builder _, _ = defaultProjectStmt.WriteString("INSERT INTO projects (name, description) VALUES ('default', 'Default Incus project');") // Enable all features for default project. for featureName := range ProjectFeatures { _, _ = defaultProjectStmt.WriteString(fmt.Sprintf("INSERT INTO projects_config (project_id, key, value) VALUES (1, '%s', 'true');", featureName)) } _, err = tx.Exec(defaultProjectStmt.String()) if err != nil { return err } // Default profile stmt = ` INSERT INTO profiles (name, description, project_id) VALUES ('default', 'Default Incus profile', 1) ` _, err = tx.Exec(stmt) if err != nil { return err } // Default cluster group stmt = ` INSERT INTO cluster_groups (name, description) VALUES ('default', 'Default cluster group'); INSERT INTO nodes_cluster_groups (node_id, group_id) VALUES(1, 1); ` _, err = tx.Exec(stmt) if err != nil { return err } return nil }) if err != nil { return false, err } } return true, err } // Generate a new name for the dqlite driver registration. We need it to be // unique for testing, see below. func dqliteDriverName() string { defer atomic.AddUint64(&dqliteDriverSerial, 1) return fmt.Sprintf("dqlite-%d", dqliteDriverSerial) } // Monotonic serial number for registering new instances of dqlite.Driver // using the database/sql stdlib package. This is needed since there's no way // to unregister drivers, and in unit tests more than one driver gets // registered. var dqliteDriverSerial uint64 func checkClusterIsUpgradable(ctx context.Context, tx *sql.Tx, target [2]int) error { // Get the current versions in the nodes table. versions, err := selectNodesVersions(ctx, tx) if err != nil { return fmt.Errorf("failed to fetch current nodes versions: %w", err) } for _, version := range versions { // Compare schema versions only. n, err := daemonUtil.CompareVersions(target, version, false) if err != nil { return err } switch n { case 0: // Versions are equal, there's hope for the // update. Let's check the next node. continue case 1: // Our version is bigger, we should stop here // and wait for other nodes to be upgraded and // restarted. return errSomeNodesAreBehind case 2: // Another node has a version greater than ours // and presumably is waiting for other nodes // to upgrade. Let's error out and shutdown // since we need a greater version. return errors.New("This cluster member's version is behind, please upgrade") default: panic("Unexpected return value from compareVersions") } } return nil } var errSomeNodesAreBehind = errors.New("Some cluster members are behind this cluster member's version") incus-7.0.0/internal/server/db/cluster/open_test.go000066400000000000000000000131411517523235500223660ustar00rootroot00000000000000package cluster_test import ( "context" "database/sql" "errors" "fmt" "io/fs" "os" "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/osarch" ) // If the node is not clustered, the schema updates works normally. func TestEnsureSchema_NoClustered(t *testing.T) { dir, cleanup := newDir(t) defer cleanup() assert.NoError(t, os.Mkdir(filepath.Join(dir, "global"), 0o711)) db := newDB(t) addNode(t, db, "0.0.0.0", 1, 1) ready, err := cluster.EnsureSchema(db, "1.2.3.4:666", dir) assert.True(t, ready) assert.NoError(t, err) } // Exercise EnsureSchema failures when the cluster can't be upgraded right now. func TestEnsureSchema_ClusterNotUpgradable(t *testing.T) { schema := cluster.SchemaVersion apiExtensions := len(version.APIExtensions) cases := []struct { title string setup func(*testing.T, *sql.DB) ready bool error string }{ { `a node's schema version is behind`, func(t *testing.T, db *sql.DB) { addNode(t, db, "1", schema, apiExtensions) addNode(t, db, "2", schema-1, apiExtensions) }, false, // The schema was not updated "", // No error is returned }, { `a node's number of API extensions is behind`, func(t *testing.T, db *sql.DB) { addNode(t, db, "1", schema, apiExtensions) addNode(t, db, "2", schema, apiExtensions-1) }, true, // The schema was not updated "", // No error is returned }, { `this node's schema is behind`, func(t *testing.T, db *sql.DB) { addNode(t, db, "1", schema, apiExtensions) addNode(t, db, "2", schema+1, apiExtensions) }, false, "This cluster member's version is behind, please upgrade", }, { `this node's number of API extensions is behind`, func(t *testing.T, db *sql.DB) { addNode(t, db, "1", schema, apiExtensions) addNode(t, db, "2", schema, apiExtensions+1) }, true, "", }, { `inconsistent schema version and API extensions number`, func(t *testing.T, db *sql.DB) { addNode(t, db, "1", schema, apiExtensions) addNode(t, db, "2", schema+1, apiExtensions-1) }, false, "This cluster member's version is behind, please upgrade", }, } for _, c := range cases { t.Run(c.title, func(t *testing.T) { db := newDB(t) c.setup(t, db) ready, err := cluster.EnsureSchema(db, "1", "/unused/db/dir") assert.Equal(t, c.ready, ready) if c.error == "" { assert.NoError(t, err) } else { assert.EqualError(t, err, c.error) } }) } } // Regardless of whether the schema could actually be upgraded or not, the // version of this node gets updated. func TestEnsureSchema_UpdateNodeVersion(t *testing.T) { schema := cluster.SchemaVersion apiExtensions := len(version.APIExtensions) cases := []struct { setup func(*testing.T, *sql.DB) ready bool }{ { func(t *testing.T, db *sql.DB) {}, true, }, { func(t *testing.T, db *sql.DB) { // Add a node which is behind. addNode(t, db, "2", schema, apiExtensions-1) }, true, }, } for _, c := range cases { t.Run(fmt.Sprintf("%v", c.ready), func(t *testing.T) { db := newDB(t) // Add ourselves with an older schema version and API // extensions number. addNode(t, db, "1", schema-1, apiExtensions-1) // Ensure the schema. ready, err := cluster.EnsureSchema(db, "1", "/unused/db/dir") assert.NoError(t, err) assert.Equal(t, c.ready, ready) // Check that the nodes table was updated with our new // schema version and API extensions number. assertNode(t, db, "1", schema, apiExtensions) }) } } // Create a new in-memory SQLite database with a fresh cluster schema. func newDB(t *testing.T) *sql.DB { db, err := sql.Open("sqlite3", ":memory:") require.NoError(t, err) createTableSchema := ` CREATE TABLE schema ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, version INTEGER NOT NULL, updated_at DATETIME NOT NULL, UNIQUE (version) ); ` _, err = db.Exec(createTableSchema + cluster.FreshSchema()) require.NoError(t, err) return db } // Add a new node with the given address, schema version and number of api extensions. func addNode(t *testing.T, db *sql.DB, address string, schema int, apiExtensions int) { err := query.Transaction(context.TODO(), db, func(ctx context.Context, tx *sql.Tx) error { stmt := ` INSERT INTO nodes(name, address, schema, api_extensions, arch, description) VALUES (?, ?, ?, ?, ?, '') ` name := fmt.Sprintf("node at %s", address) _, err := tx.Exec(stmt, name, address, schema, apiExtensions, osarch.ARCH_64BIT_INTEL_X86) return err }) require.NoError(t, err) } // Assert that the node with the given address has the given schema version and API // extensions number. func assertNode(t *testing.T, db *sql.DB, address string, schema int, apiExtensions int) { err := query.Transaction(context.TODO(), db, func(ctx context.Context, tx *sql.Tx) error { where := "address=? AND schema=? AND api_extensions=?" n, err := query.Count(ctx, tx, "nodes", where, address, schema, apiExtensions) assert.Equal(t, 1, n, "node does not have expected version") return err }) require.NoError(t, err) } // Return a new temporary directory. func newDir(t *testing.T) (string, func()) { t.Helper() dir, err := os.MkdirTemp("", "dqlite-replication-test-") assert.NoError(t, err) cleanup := func() { _, err := os.Stat(dir) if err != nil { assert.True(t, errors.Is(err, fs.ErrNotExist)) } else { assert.NoError(t, os.RemoveAll(dir)) } } return dir, cleanup } incus-7.0.0/internal/server/db/cluster/operations.go000066400000000000000000000034051517523235500225530ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "github.com/lxc/incus/v7/internal/server/db/operationtype" ) // Code generation directives. // //generate-database:mapper target operations.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e operation objects //generate-database:mapper stmt -e operation objects-by-NodeID //generate-database:mapper stmt -e operation objects-by-ID //generate-database:mapper stmt -e operation objects-by-UUID //generate-database:mapper stmt -e operation create-or-replace //generate-database:mapper stmt -e operation delete-by-UUID //generate-database:mapper stmt -e operation delete-by-NodeID // //generate-database:mapper method -i -e operation GetMany //generate-database:mapper method -i -e operation CreateOrReplace //generate-database:mapper method -i -e operation DeleteOne-by-UUID //generate-database:mapper method -i -e operation DeleteMany-by-NodeID // Operation holds information about a single operation running on a member in the cluster. type Operation struct { ID int64 `db:"primary=yes"` // Stable database identifier UUID string `db:"primary=yes"` // User-visible identifier NodeAddress string `db:"join=nodes.address&omit=create-or-replace"` // Address of the node the operation is running on ProjectID *int64 // ID of the project for the operation. NodeID int64 // ID of the node the operation is running on Type operationtype.Type // Type of the operation } // OperationFilter specifies potential query parameter fields. type OperationFilter struct { ID *int64 NodeID *int64 UUID *string } incus-7.0.0/internal/server/db/cluster/operations.interface.mapper.go000066400000000000000000000016531517523235500260000ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // OperationGenerated is an interface of generated methods for Operation. type OperationGenerated interface { // GetOperations returns all available operations. // generator: operation GetMany GetOperations(ctx context.Context, db dbtx, filters ...OperationFilter) ([]Operation, error) // CreateOrReplaceOperation adds a new operation to the database. // generator: operation CreateOrReplace CreateOrReplaceOperation(ctx context.Context, db dbtx, object Operation) (int64, error) // DeleteOperation deletes the operation matching the given key parameters. // generator: operation DeleteOne-by-UUID DeleteOperation(ctx context.Context, db dbtx, uuid string) error // DeleteOperations deletes the operation matching the given key parameters. // generator: operation DeleteMany-by-NodeID DeleteOperations(ctx context.Context, db dbtx, nodeID int64) error } incus-7.0.0/internal/server/db/cluster/operations.mapper.go000066400000000000000000000217701517523235500240430ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var operationObjects = RegisterStmt(` SELECT operations.id, operations.uuid, nodes.address AS node_address, operations.project_id, operations.node_id, operations.type FROM operations JOIN nodes ON operations.node_id = nodes.id ORDER BY operations.id, operations.uuid `) var operationObjectsByNodeID = RegisterStmt(` SELECT operations.id, operations.uuid, nodes.address AS node_address, operations.project_id, operations.node_id, operations.type FROM operations JOIN nodes ON operations.node_id = nodes.id WHERE ( operations.node_id = ? ) ORDER BY operations.id, operations.uuid `) var operationObjectsByID = RegisterStmt(` SELECT operations.id, operations.uuid, nodes.address AS node_address, operations.project_id, operations.node_id, operations.type FROM operations JOIN nodes ON operations.node_id = nodes.id WHERE ( operations.id = ? ) ORDER BY operations.id, operations.uuid `) var operationObjectsByUUID = RegisterStmt(` SELECT operations.id, operations.uuid, nodes.address AS node_address, operations.project_id, operations.node_id, operations.type FROM operations JOIN nodes ON operations.node_id = nodes.id WHERE ( operations.uuid = ? ) ORDER BY operations.id, operations.uuid `) var operationCreateOrReplace = RegisterStmt(` INSERT OR REPLACE INTO operations (uuid, project_id, node_id, type) VALUES (?, ?, ?, ?) `) var operationDeleteByUUID = RegisterStmt(` DELETE FROM operations WHERE uuid = ? `) var operationDeleteByNodeID = RegisterStmt(` DELETE FROM operations WHERE node_id = ? `) // operationColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the Operation entity. func operationColumns() string { return "operations.id, operations.uuid, nodes.address AS node_address, operations.project_id, operations.node_id, operations.type" } // getOperations can be used to run handwritten sql.Stmts to return a slice of objects. func getOperations(ctx context.Context, stmt *sql.Stmt, args ...any) ([]Operation, error) { objects := make([]Operation, 0) dest := func(scan func(dest ...any) error) error { o := Operation{} err := scan(&o.ID, &o.UUID, &o.NodeAddress, &o.ProjectID, &o.NodeID, &o.Type) if err != nil { return err } objects = append(objects, o) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"operations\" table: %w", err) } return objects, nil } // getOperationsRaw can be used to run handwritten query strings to return a slice of objects. func getOperationsRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]Operation, error) { objects := make([]Operation, 0) dest := func(scan func(dest ...any) error) error { o := Operation{} err := scan(&o.ID, &o.UUID, &o.NodeAddress, &o.ProjectID, &o.NodeID, &o.Type) if err != nil { return err } objects = append(objects, o) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"operations\" table: %w", err) } return objects, nil } // GetOperations returns all available operations. // generator: operation GetMany func GetOperations(ctx context.Context, db dbtx, filters ...OperationFilter) (_ []Operation, _err error) { defer func() { _err = mapErr(_err, "Operation") }() var err error // Result slice. objects := make([]Operation, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, operationObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"operationObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.UUID != nil && filter.ID == nil && filter.NodeID == nil { args = append(args, []any{filter.UUID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, operationObjectsByUUID) if err != nil { return nil, fmt.Errorf("Failed to get \"operationObjectsByUUID\" prepared statement: %w", err) } break } query, err := StmtString(operationObjectsByUUID) if err != nil { return nil, fmt.Errorf("Failed to get \"operationObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.NodeID != nil && filter.ID == nil && filter.UUID == nil { args = append(args, []any{filter.NodeID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, operationObjectsByNodeID) if err != nil { return nil, fmt.Errorf("Failed to get \"operationObjectsByNodeID\" prepared statement: %w", err) } break } query, err := StmtString(operationObjectsByNodeID) if err != nil { return nil, fmt.Errorf("Failed to get \"operationObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID != nil && filter.NodeID == nil && filter.UUID == nil { args = append(args, []any{filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, operationObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"operationObjectsByID\" prepared statement: %w", err) } break } query, err := StmtString(operationObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"operationObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.NodeID == nil && filter.UUID == nil { return nil, fmt.Errorf("Cannot filter on empty OperationFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getOperations(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getOperationsRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"operations\" table: %w", err) } return objects, nil } // CreateOrReplaceOperation adds a new operation to the database. // generator: operation CreateOrReplace func CreateOrReplaceOperation(ctx context.Context, db dbtx, object Operation) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Operation") }() args := make([]any, 4) // Populate the statement arguments. args[0] = object.UUID args[1] = object.ProjectID args[2] = object.NodeID args[3] = object.Type // Prepared statement to use. stmt, err := Stmt(db, operationCreateOrReplace) if err != nil { return -1, fmt.Errorf("Failed to get \"operationCreateOrReplace\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"operations\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"operations\" entry ID: %w", err) } return id, nil } // DeleteOperation deletes the operation matching the given key parameters. // generator: operation DeleteOne-by-UUID func DeleteOperation(ctx context.Context, db dbtx, uuid string) (_err error) { defer func() { _err = mapErr(_err, "Operation") }() stmt, err := Stmt(db, operationDeleteByUUID) if err != nil { return fmt.Errorf("Failed to get \"operationDeleteByUUID\" prepared statement: %w", err) } result, err := stmt.Exec(uuid) if err != nil { return fmt.Errorf("Delete \"operations\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d Operation rows instead of 1", n) } return nil } // DeleteOperations deletes the operation matching the given key parameters. // generator: operation DeleteMany-by-NodeID func DeleteOperations(ctx context.Context, db dbtx, nodeID int64) (_err error) { defer func() { _err = mapErr(_err, "Operation") }() stmt, err := Stmt(db, operationDeleteByNodeID) if err != nil { return fmt.Errorf("Failed to get \"operationDeleteByNodeID\" prepared statement: %w", err) } result, err := stmt.Exec(nodeID) if err != nil { return fmt.Errorf("Delete \"operations\": %w", err) } _, err = result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } return nil } incus-7.0.0/internal/server/db/cluster/profiles.go000066400000000000000000000120751517523235500222160ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "context" "database/sql" "maps" "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/shared/api" ) // Code generation directives. // //generate-database:mapper target profiles.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e profile objects //generate-database:mapper stmt -e profile objects-by-ID //generate-database:mapper stmt -e profile objects-by-Name //generate-database:mapper stmt -e profile objects-by-Project //generate-database:mapper stmt -e profile objects-by-Project-and-Name //generate-database:mapper stmt -e profile id //generate-database:mapper stmt -e profile create //generate-database:mapper stmt -e profile rename //generate-database:mapper stmt -e profile update //generate-database:mapper stmt -e profile delete-by-Project-and-Name // //generate-database:mapper method -i -e profile ID //generate-database:mapper method -i -e profile Exists //generate-database:mapper method -i -e profile GetMany references=Config,Device //generate-database:mapper method -i -e profile GetOne //generate-database:mapper method -i -e profile Create references=Config,Device //generate-database:mapper method -i -e profile Rename //generate-database:mapper method -i -e profile Update references=Config,Device //generate-database:mapper method -i -e profile DeleteOne-by-Project-and-Name // Profile is a value object holding db-related details about a profile. type Profile struct { ID int ProjectID int `db:"omit=create,update"` Project string `db:"primary=yes&join=projects.name"` Name string `db:"primary=yes"` Description string `db:"coalesce=''"` } // ProfileFilter specifies potential query parameter fields. type ProfileFilter struct { ID *int Project *string Name *string } // ToAPI returns a cluster Profile as an API struct. func (p *Profile) ToAPI(ctx context.Context, tx *sql.Tx, profileConfigs map[int]map[string]string, profileDevices map[int][]Device) (*api.Profile, error) { var err error var dbConfig map[string]string if profileConfigs != nil { dbConfig = profileConfigs[p.ID] if dbConfig == nil { dbConfig = map[string]string{} } } else { dbConfig, err = GetProfileConfig(ctx, tx, p.ID) if err != nil { return nil, err } } var dbDevices map[string]Device if profileDevices != nil { dbDevices = map[string]Device{} for _, dev := range profileDevices[p.ID] { dbDevices[dev.Name] = dev } } else { dbDevices, err = GetProfileDevices(ctx, tx, p.ID) if err != nil { return nil, err } } profile := &api.Profile{ Name: p.Name, ProfilePut: api.ProfilePut{ Description: p.Description, Config: dbConfig, Devices: DevicesToAPI(dbDevices), }, Project: p.Project, } return profile, nil } // GetProfilesIfEnabled returns the profiles from the given project, or the // default project if "features.profiles" is not set. func GetProfilesIfEnabled(ctx context.Context, tx *sql.Tx, projectName string, names []string) ([]Profile, error) { enabled, err := ProjectHasProfiles(ctx, tx, projectName) if err != nil { return nil, err } if !enabled { projectName = "default" } profiles := make([]Profile, 0, len(names)) for _, name := range names { profile, err := GetProfile(ctx, tx, projectName, name) if err != nil { return nil, err } profiles = append(profiles, *profile) } return profiles, nil } // ExpandInstanceConfig expands the given instance config with the config // values of the given profiles. func ExpandInstanceConfig(config map[string]string, profiles []api.Profile) map[string]string { expandedConfig := map[string]string{} // Apply all the profiles profileConfigs := make([]map[string]string, len(profiles)) for i, profile := range profiles { profileConfigs[i] = profile.Config } for i := range profileConfigs { maps.Copy(expandedConfig, profileConfigs[i]) } // Stick the given config on top maps.Copy(expandedConfig, config) return expandedConfig } // ExpandInstanceDevices expands the given instance devices with the devices // defined in the given profiles. func ExpandInstanceDevices(devices config.Devices, profiles []api.Profile) config.Devices { expandedDevices := config.Devices{} // Apply all the profiles profileDevices := make([]config.Devices, len(profiles)) for i, profile := range profiles { profileDevices[i] = config.NewDevices(profile.Devices) } for i := range profileDevices { maps.Copy(expandedDevices, profileDevices[i]) } // Stick the given devices on top maps.Copy(expandedDevices, devices) return expandedDevices } // GetAllProfileConfigs returns a map of all profile configurations, keyed by database ID. func GetAllProfileConfigs(ctx context.Context, tx *sql.Tx) (map[int]map[string]string, error) { return GetConfig(ctx, tx, "profiles", "profile") } // GetAllProfileDevices returns a map of all profile devices, keyed by database ID. func GetAllProfileDevices(ctx context.Context, tx *sql.Tx) (map[int][]Device, error) { return GetDevices(ctx, tx, "profiles", "profile") } incus-7.0.0/internal/server/db/cluster/profiles.interface.mapper.go000066400000000000000000000054761517523235500254470ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // ProfileGenerated is an interface of generated methods for Profile. type ProfileGenerated interface { // GetProfileID return the ID of the profile with the given key. // generator: profile ID GetProfileID(ctx context.Context, db tx, project string, name string) (int64, error) // ProfileExists checks if a profile with the given key exists. // generator: profile Exists ProfileExists(ctx context.Context, db dbtx, project string, name string) (bool, error) // GetProfileConfig returns all available Profile Config // generator: profile GetMany GetProfileConfig(ctx context.Context, db tx, profileID int, filters ...ConfigFilter) (map[string]string, error) // GetProfileDevices returns all available Profile Devices // generator: profile GetMany GetProfileDevices(ctx context.Context, db tx, profileID int, filters ...DeviceFilter) (map[string]Device, error) // GetProfiles returns all available profiles. // generator: profile GetMany GetProfiles(ctx context.Context, db dbtx, filters ...ProfileFilter) ([]Profile, error) // GetProfile returns the profile with the given key. // generator: profile GetOne GetProfile(ctx context.Context, db dbtx, project string, name string) (*Profile, error) // CreateProfileConfig adds new profile Config to the database. // generator: profile Create CreateProfileConfig(ctx context.Context, db dbtx, profileID int64, config map[string]string) error // CreateProfileDevices adds new profile Devices to the database. // generator: profile Create CreateProfileDevices(ctx context.Context, db tx, profileID int64, devices map[string]Device) error // CreateProfile adds a new profile to the database. // generator: profile Create CreateProfile(ctx context.Context, db dbtx, object Profile) (int64, error) // RenameProfile renames the profile matching the given key parameters. // generator: profile Rename RenameProfile(ctx context.Context, db dbtx, project string, name string, to string) error // UpdateProfileConfig updates the profile Config matching the given key parameters. // generator: profile Update UpdateProfileConfig(ctx context.Context, db tx, profileID int64, config map[string]string) error // UpdateProfileDevices updates the profile Device matching the given key parameters. // generator: profile Update UpdateProfileDevices(ctx context.Context, db tx, profileID int64, devices map[string]Device) error // UpdateProfile updates the profile matching the given key parameters. // generator: profile Update UpdateProfile(ctx context.Context, db tx, project string, name string, object Profile) error // DeleteProfile deletes the profile matching the given key parameters. // generator: profile DeleteOne-by-Project-and-Name DeleteProfile(ctx context.Context, db dbtx, project string, name string) error } incus-7.0.0/internal/server/db/cluster/profiles.mapper.go000066400000000000000000000403731517523235500235030ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var profileObjects = RegisterStmt(` SELECT profiles.id, profiles.project_id, projects.name AS project, profiles.name, coalesce(profiles.description, '') FROM profiles JOIN projects ON profiles.project_id = projects.id ORDER BY projects.id, profiles.name `) var profileObjectsByID = RegisterStmt(` SELECT profiles.id, profiles.project_id, projects.name AS project, profiles.name, coalesce(profiles.description, '') FROM profiles JOIN projects ON profiles.project_id = projects.id WHERE ( profiles.id = ? ) ORDER BY projects.id, profiles.name `) var profileObjectsByName = RegisterStmt(` SELECT profiles.id, profiles.project_id, projects.name AS project, profiles.name, coalesce(profiles.description, '') FROM profiles JOIN projects ON profiles.project_id = projects.id WHERE ( profiles.name = ? ) ORDER BY projects.id, profiles.name `) var profileObjectsByProject = RegisterStmt(` SELECT profiles.id, profiles.project_id, projects.name AS project, profiles.name, coalesce(profiles.description, '') FROM profiles JOIN projects ON profiles.project_id = projects.id WHERE ( project = ? ) ORDER BY projects.id, profiles.name `) var profileObjectsByProjectAndName = RegisterStmt(` SELECT profiles.id, profiles.project_id, projects.name AS project, profiles.name, coalesce(profiles.description, '') FROM profiles JOIN projects ON profiles.project_id = projects.id WHERE ( project = ? AND profiles.name = ? ) ORDER BY projects.id, profiles.name `) var profileID = RegisterStmt(` SELECT profiles.id FROM profiles JOIN projects ON profiles.project_id = projects.id WHERE projects.name = ? AND profiles.name = ? `) var profileCreate = RegisterStmt(` INSERT INTO profiles (project_id, name, description) VALUES ((SELECT projects.id FROM projects WHERE projects.name = ?), ?, ?) `) var profileRename = RegisterStmt(` UPDATE profiles SET name = ? WHERE project_id = (SELECT projects.id FROM projects WHERE projects.name = ?) AND name = ? `) var profileUpdate = RegisterStmt(` UPDATE profiles SET project_id = (SELECT projects.id FROM projects WHERE projects.name = ?), name = ?, description = ? WHERE id = ? `) var profileDeleteByProjectAndName = RegisterStmt(` DELETE FROM profiles WHERE project_id = (SELECT projects.id FROM projects WHERE projects.name = ?) AND name = ? `) // GetProfileID return the ID of the profile with the given key. // generator: profile ID func GetProfileID(ctx context.Context, db tx, project string, name string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Profile") }() stmt, err := Stmt(db, profileID) if err != nil { return -1, fmt.Errorf("Failed to get \"profileID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, project, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"profiles\" ID: %w", err) } return id, nil } // ProfileExists checks if a profile with the given key exists. // generator: profile Exists func ProfileExists(ctx context.Context, db dbtx, project string, name string) (_ bool, _err error) { defer func() { _err = mapErr(_err, "Profile") }() stmt, err := Stmt(db, profileID) if err != nil { return false, fmt.Errorf("Failed to get \"profileID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, project, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("Failed to get \"profiles\" ID: %w", err) } return true, nil } // profileColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the Profile entity. func profileColumns() string { return "profiles.id, profiles.project_id, projects.name AS project, profiles.name, coalesce(profiles.description, '')" } // getProfiles can be used to run handwritten sql.Stmts to return a slice of objects. func getProfiles(ctx context.Context, stmt *sql.Stmt, args ...any) ([]Profile, error) { objects := make([]Profile, 0) dest := func(scan func(dest ...any) error) error { p := Profile{} err := scan(&p.ID, &p.ProjectID, &p.Project, &p.Name, &p.Description) if err != nil { return err } objects = append(objects, p) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"profiles\" table: %w", err) } return objects, nil } // getProfilesRaw can be used to run handwritten query strings to return a slice of objects. func getProfilesRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]Profile, error) { objects := make([]Profile, 0) dest := func(scan func(dest ...any) error) error { p := Profile{} err := scan(&p.ID, &p.ProjectID, &p.Project, &p.Name, &p.Description) if err != nil { return err } objects = append(objects, p) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"profiles\" table: %w", err) } return objects, nil } // GetProfiles returns all available profiles. // generator: profile GetMany func GetProfiles(ctx context.Context, db dbtx, filters ...ProfileFilter) (_ []Profile, _err error) { defer func() { _err = mapErr(_err, "Profile") }() var err error // Result slice. objects := make([]Profile, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, profileObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"profileObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.Project != nil && filter.Name != nil && filter.ID == nil { args = append(args, []any{filter.Project, filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, profileObjectsByProjectAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"profileObjectsByProjectAndName\" prepared statement: %w", err) } break } query, err := StmtString(profileObjectsByProjectAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"profileObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.ID == nil && filter.Name == nil { args = append(args, []any{filter.Project}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, profileObjectsByProject) if err != nil { return nil, fmt.Errorf("Failed to get \"profileObjectsByProject\" prepared statement: %w", err) } break } query, err := StmtString(profileObjectsByProject) if err != nil { return nil, fmt.Errorf("Failed to get \"profileObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Name != nil && filter.ID == nil && filter.Project == nil { args = append(args, []any{filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, profileObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"profileObjectsByName\" prepared statement: %w", err) } break } query, err := StmtString(profileObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"profileObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID != nil && filter.Project == nil && filter.Name == nil { args = append(args, []any{filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, profileObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"profileObjectsByID\" prepared statement: %w", err) } break } query, err := StmtString(profileObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"profileObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.Project == nil && filter.Name == nil { return nil, fmt.Errorf("Cannot filter on empty ProfileFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getProfiles(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getProfilesRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"profiles\" table: %w", err) } return objects, nil } // GetProfileDevices returns all available Profile Devices // generator: profile GetMany func GetProfileDevices(ctx context.Context, db tx, profileID int, filters ...DeviceFilter) (_ map[string]Device, _err error) { defer func() { _err = mapErr(_err, "Profile") }() profileDevices, err := GetDevices(ctx, db, "profiles", "profile", filters...) if err != nil { return nil, err } devices := map[string]Device{} for _, ref := range profileDevices[profileID] { _, ok := devices[ref.Name] if !ok { devices[ref.Name] = ref } else { return nil, fmt.Errorf("Found duplicate Device with name %q", ref.Name) } } return devices, nil } // GetProfileConfig returns all available Profile Config // generator: profile GetMany func GetProfileConfig(ctx context.Context, db tx, profileID int, filters ...ConfigFilter) (_ map[string]string, _err error) { defer func() { _err = mapErr(_err, "Profile") }() profileConfig, err := GetConfig(ctx, db, "profiles", "profile", filters...) if err != nil { return nil, err } config, ok := profileConfig[profileID] if !ok { config = map[string]string{} } return config, nil } // GetProfile returns the profile with the given key. // generator: profile GetOne func GetProfile(ctx context.Context, db dbtx, project string, name string) (_ *Profile, _err error) { defer func() { _err = mapErr(_err, "Profile") }() filter := ProfileFilter{} filter.Project = &project filter.Name = &name objects, err := GetProfiles(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"profiles\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"profiles\" entry matches") } } // CreateProfile adds a new profile to the database. // generator: profile Create func CreateProfile(ctx context.Context, db dbtx, object Profile) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Profile") }() args := make([]any, 3) // Populate the statement arguments. args[0] = object.Project args[1] = object.Name args[2] = object.Description // Prepared statement to use. stmt, err := Stmt(db, profileCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"profileCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"profiles\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"profiles\" entry ID: %w", err) } return id, nil } // CreateProfileDevices adds new profile Devices to the database. // generator: profile Create func CreateProfileDevices(ctx context.Context, db tx, profileID int64, devices map[string]Device) (_err error) { defer func() { _err = mapErr(_err, "Profile") }() for key, device := range devices { device.ReferenceID = int(profileID) devices[key] = device } err := CreateDevices(ctx, db, "profiles", "profile", devices) if err != nil { return fmt.Errorf("Insert Device failed for Profile: %w", err) } return nil } // CreateProfileConfig adds new profile Config to the database. // generator: profile Create func CreateProfileConfig(ctx context.Context, db dbtx, profileID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Profile") }() referenceID := int(profileID) for key, value := range config { insert := Config{ ReferenceID: referenceID, Key: key, Value: value, } err := CreateConfig(ctx, db, "profiles", "profile", insert) if err != nil { return fmt.Errorf("Insert Config failed for Profile: %w", err) } } return nil } // RenameProfile renames the profile matching the given key parameters. // generator: profile Rename func RenameProfile(ctx context.Context, db dbtx, project string, name string, to string) (_err error) { defer func() { _err = mapErr(_err, "Profile") }() stmt, err := Stmt(db, profileRename) if err != nil { return fmt.Errorf("Failed to get \"profileRename\" prepared statement: %w", err) } result, err := stmt.Exec(to, project, name) if err != nil { return fmt.Errorf("Rename Profile failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows failed: %w", err) } if n != 1 { return fmt.Errorf("Query affected %d rows instead of 1", n) } return nil } // UpdateProfile updates the profile matching the given key parameters. // generator: profile Update func UpdateProfile(ctx context.Context, db tx, project string, name string, object Profile) (_err error) { defer func() { _err = mapErr(_err, "Profile") }() id, err := GetProfileID(ctx, db, project, name) if err != nil { return err } stmt, err := Stmt(db, profileUpdate) if err != nil { return fmt.Errorf("Failed to get \"profileUpdate\" prepared statement: %w", err) } result, err := stmt.Exec(object.Project, object.Name, object.Description, id) if err != nil { return fmt.Errorf("Update \"profiles\" entry failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } return nil } // UpdateProfileDevices updates the profile Device matching the given key parameters. // generator: profile Update func UpdateProfileDevices(ctx context.Context, db tx, profileID int64, devices map[string]Device) (_err error) { defer func() { _err = mapErr(_err, "Profile") }() err := UpdateDevices(ctx, db, "profiles", "profile", int(profileID), devices) if err != nil { return fmt.Errorf("Replace Device for Profile failed: %w", err) } return nil } // UpdateProfileConfig updates the profile Config matching the given key parameters. // generator: profile Update func UpdateProfileConfig(ctx context.Context, db tx, profileID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Profile") }() err := UpdateConfig(ctx, db, "profiles", "profile", int(profileID), config) if err != nil { return fmt.Errorf("Replace Config for Profile failed: %w", err) } return nil } // DeleteProfile deletes the profile matching the given key parameters. // generator: profile DeleteOne-by-Project-and-Name func DeleteProfile(ctx context.Context, db dbtx, project string, name string) (_err error) { defer func() { _err = mapErr(_err, "Profile") }() stmt, err := Stmt(db, profileDeleteByProjectAndName) if err != nil { return fmt.Errorf("Failed to get \"profileDeleteByProjectAndName\" prepared statement: %w", err) } result, err := stmt.Exec(project, name) if err != nil { return fmt.Errorf("Delete \"profiles\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d Profile rows instead of 1", n) } return nil } incus-7.0.0/internal/server/db/cluster/projects.go000066400000000000000000000145361517523235500222300ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "context" "database/sql" "fmt" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) // Code generation directives. // //generate-database:mapper target projects.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e project objects //generate-database:mapper stmt -e project objects-by-Name //generate-database:mapper stmt -e project objects-by-ID //generate-database:mapper stmt -e project create struct=Project //generate-database:mapper stmt -e project id //generate-database:mapper stmt -e project rename //generate-database:mapper stmt -e project update struct=Project //generate-database:mapper stmt -e project delete-by-Name // //generate-database:mapper method -i -e project GetMany references=Config //generate-database:mapper method -i -e project GetOne struct=Project //generate-database:mapper method -i -e project Exists struct=Project //generate-database:mapper method -i -e project Create references=Config //generate-database:mapper method -i -e project ID struct=Project //generate-database:mapper method -i -e project Rename //generate-database:mapper method -i -e project DeleteOne-by-Name // ProjectFeature indicates the behaviour of a project feature. type ProjectFeature struct { // DefaultEnabled // Whether the feature should be enabled by default on new projects. DefaultEnabled bool // CanEnableNonEmpty // Whether or not the feature can be changed to enabled on a non-empty project. CanEnableNonEmpty bool } // ProjectFeatures lists available project features and their behaviours. var ProjectFeatures = map[string]ProjectFeature{ "features.images": { DefaultEnabled: true, }, "features.profiles": { DefaultEnabled: true, }, "features.storage.volumes": { DefaultEnabled: true, }, "features.storage.buckets": { DefaultEnabled: true, }, "features.networks": {}, "features.networks.zones": { CanEnableNonEmpty: true, }, } // Project represents a project. type Project struct { ID int Description string Name string `db:"omit=update"` } // ProjectFilter specifies potential query parameter fields. type ProjectFilter struct { ID *int Name *string `db:"omit=update"` // If non-empty, return only the project with this name. } // ToAPI converts the database Project struct to an api.Project entry. func (p *Project) ToAPI(ctx context.Context, tx *sql.Tx) (*api.Project, error) { apiProject := &api.Project{ ProjectPut: api.ProjectPut{ Description: p.Description, }, Name: p.Name, } var err error apiProject.Config, err = GetProjectConfig(ctx, tx, p.ID) if err != nil { return nil, fmt.Errorf("Failed loading project config: %w", err) } return apiProject, nil } // ProjectHasProfiles is a helper to check if a project has the profiles // feature enabled. func ProjectHasProfiles(ctx context.Context, tx *sql.Tx, name string) (bool, error) { stmt := ` SELECT projects_config.value FROM projects_config JOIN projects ON projects.id=projects_config.project_id WHERE projects.name=? AND projects_config.key='features.profiles' ` values, err := query.SelectStrings(ctx, tx, stmt, name) if err != nil { return false, fmt.Errorf("Fetch project config: %w", err) } if len(values) == 0 { return false, nil } return util.IsTrue(values[0]), nil } // GetProjectNames returns the names of all availablprojects. func GetProjectNames(ctx context.Context, tx *sql.Tx) ([]string, error) { stmt := "SELECT name FROM projects" names, err := query.SelectStrings(ctx, tx, stmt) if err != nil { return nil, fmt.Errorf("Fetch project names: %w", err) } return names, nil } // GetProjectIDsToNames returns a map associating each prect ID to its // project name. func GetProjectIDsToNames(ctx context.Context, tx *sql.Tx) (map[int64]string, error) { stmt := "SELECT id, name FROM projects" rows, err := tx.QueryContext(ctx, stmt) if err != nil { return nil, err } defer func() { _ = rows.Close() }() result := map[int64]string{} for i := 0; rows.Next(); i++ { var id int64 var name string err := rows.Scan(&id, &name) if err != nil { return nil, err } result[id] = name } err = rows.Err() if err != nil { return nil, err } return result, nil } // ProjectHasImages is a helper to check if a project has the images // feature enabled. func ProjectHasImages(ctx context.Context, tx *sql.Tx, name string) (bool, error) { project, err := GetProject(ctx, tx, name) if err != nil { return false, fmt.Errorf("fetch project: %w", err) } config, err := GetProjectConfig(ctx, tx, project.ID) if err != nil { return false, err } enabled := util.IsTrue(config["features.images"]) return enabled, nil } // UpdateProject updates the project matching the given key parameters. func UpdateProject(ctx context.Context, tx *sql.Tx, name string, object api.ProjectPut) error { id, err := GetProjectID(ctx, tx, name) if err != nil { return fmt.Errorf("Fetch project ID: %w", err) } stmt, err := Stmt(tx, projectUpdate) if err != nil { return fmt.Errorf("Failed to get \"projectUpdate\" prepared statement: %w", err) } result, err := stmt.Exec(object.Description, id) if err != nil { return fmt.Errorf("Update project: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } // Clear config. _, err = tx.Exec(` DELETE FROM projects_config WHERE projects_config.project_id = ? `, id) if err != nil { return fmt.Errorf("Delete project config: %w", err) } err = UpdateConfig(ctx, tx, "projects", "project", int(id), object.Config) if err != nil { return fmt.Errorf("Insert config for project: %w", err) } return nil } // InitProjectWithoutImages populates the images_profiles table with // all images from the default project when a project is created with // features.images=false. func InitProjectWithoutImages(ctx context.Context, tx *sql.Tx, project string) error { defaultProfileID, err := GetProfileID(ctx, tx, project, "default") if err != nil { return fmt.Errorf("Fetch project ID: %w", err) } stmt := `INSERT INTO images_profiles (image_id, profile_id) SELECT images.id, ? FROM images WHERE project_id=1` _, err = tx.Exec(stmt, defaultProfileID) return err } incus-7.0.0/internal/server/db/cluster/projects.interface.mapper.go000066400000000000000000000033271517523235500254460ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // ProjectGenerated is an interface of generated methods for Project. type ProjectGenerated interface { // GetProjectConfig returns all available Project Config // generator: project GetMany GetProjectConfig(ctx context.Context, db tx, projectID int, filters ...ConfigFilter) (map[string]string, error) // GetProjects returns all available projects. // generator: project GetMany GetProjects(ctx context.Context, db dbtx, filters ...ProjectFilter) ([]Project, error) // GetProject returns the project with the given key. // generator: project GetOne GetProject(ctx context.Context, db dbtx, name string) (*Project, error) // ProjectExists checks if a project with the given key exists. // generator: project Exists ProjectExists(ctx context.Context, db dbtx, name string) (bool, error) // CreateProjectConfig adds new project Config to the database. // generator: project Create CreateProjectConfig(ctx context.Context, db dbtx, projectID int64, config map[string]string) error // CreateProject adds a new project to the database. // generator: project Create CreateProject(ctx context.Context, db dbtx, object Project) (int64, error) // GetProjectID return the ID of the project with the given key. // generator: project ID GetProjectID(ctx context.Context, db tx, name string) (int64, error) // RenameProject renames the project matching the given key parameters. // generator: project Rename RenameProject(ctx context.Context, db dbtx, name string, to string) error // DeleteProject deletes the project matching the given key parameters. // generator: project DeleteOne-by-Name DeleteProject(ctx context.Context, db dbtx, name string) error } incus-7.0.0/internal/server/db/cluster/projects.mapper.go000066400000000000000000000242731517523235500235120ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var projectObjects = RegisterStmt(` SELECT projects.id, projects.description, projects.name FROM projects ORDER BY projects.name `) var projectObjectsByName = RegisterStmt(` SELECT projects.id, projects.description, projects.name FROM projects WHERE ( projects.name = ? ) ORDER BY projects.name `) var projectObjectsByID = RegisterStmt(` SELECT projects.id, projects.description, projects.name FROM projects WHERE ( projects.id = ? ) ORDER BY projects.name `) var projectCreate = RegisterStmt(` INSERT INTO projects (description, name) VALUES (?, ?) `) var projectID = RegisterStmt(` SELECT projects.id FROM projects WHERE projects.name = ? `) var projectRename = RegisterStmt(` UPDATE projects SET name = ? WHERE name = ? `) var projectUpdate = RegisterStmt(` UPDATE projects SET description = ? WHERE id = ? `) var projectDeleteByName = RegisterStmt(` DELETE FROM projects WHERE name = ? `) // projectColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the Project entity. func projectColumns() string { return "projects.id, projects.description, projects.name" } // getProjects can be used to run handwritten sql.Stmts to return a slice of objects. func getProjects(ctx context.Context, stmt *sql.Stmt, args ...any) ([]Project, error) { objects := make([]Project, 0) dest := func(scan func(dest ...any) error) error { p := Project{} err := scan(&p.ID, &p.Description, &p.Name) if err != nil { return err } objects = append(objects, p) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"projects\" table: %w", err) } return objects, nil } // getProjectsRaw can be used to run handwritten query strings to return a slice of objects. func getProjectsRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]Project, error) { objects := make([]Project, 0) dest := func(scan func(dest ...any) error) error { p := Project{} err := scan(&p.ID, &p.Description, &p.Name) if err != nil { return err } objects = append(objects, p) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"projects\" table: %w", err) } return objects, nil } // GetProjects returns all available projects. // generator: project GetMany func GetProjects(ctx context.Context, db dbtx, filters ...ProjectFilter) (_ []Project, _err error) { defer func() { _err = mapErr(_err, "Project") }() var err error // Result slice. objects := make([]Project, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, projectObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"projectObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.Name != nil && filter.ID == nil { args = append(args, []any{filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, projectObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"projectObjectsByName\" prepared statement: %w", err) } break } query, err := StmtString(projectObjectsByName) if err != nil { return nil, fmt.Errorf("Failed to get \"projectObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID != nil && filter.Name == nil { args = append(args, []any{filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, projectObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"projectObjectsByID\" prepared statement: %w", err) } break } query, err := StmtString(projectObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"projectObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.Name == nil { return nil, fmt.Errorf("Cannot filter on empty ProjectFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getProjects(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getProjectsRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"projects\" table: %w", err) } return objects, nil } // GetProjectConfig returns all available Project Config // generator: project GetMany func GetProjectConfig(ctx context.Context, db tx, projectID int, filters ...ConfigFilter) (_ map[string]string, _err error) { defer func() { _err = mapErr(_err, "Project") }() projectConfig, err := GetConfig(ctx, db, "projects", "project", filters...) if err != nil { return nil, err } config, ok := projectConfig[projectID] if !ok { config = map[string]string{} } return config, nil } // GetProject returns the project with the given key. // generator: project GetOne func GetProject(ctx context.Context, db dbtx, name string) (_ *Project, _err error) { defer func() { _err = mapErr(_err, "Project") }() filter := ProjectFilter{} filter.Name = &name objects, err := GetProjects(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"projects\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"projects\" entry matches") } } // ProjectExists checks if a project with the given key exists. // generator: project Exists func ProjectExists(ctx context.Context, db dbtx, name string) (_ bool, _err error) { defer func() { _err = mapErr(_err, "Project") }() stmt, err := Stmt(db, projectID) if err != nil { return false, fmt.Errorf("Failed to get \"projectID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("Failed to get \"projects\" ID: %w", err) } return true, nil } // CreateProject adds a new project to the database. // generator: project Create func CreateProject(ctx context.Context, db dbtx, object Project) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Project") }() args := make([]any, 2) // Populate the statement arguments. args[0] = object.Description args[1] = object.Name // Prepared statement to use. stmt, err := Stmt(db, projectCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"projectCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"projects\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"projects\" entry ID: %w", err) } return id, nil } // CreateProjectConfig adds new project Config to the database. // generator: project Create func CreateProjectConfig(ctx context.Context, db dbtx, projectID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Project") }() referenceID := int(projectID) for key, value := range config { insert := Config{ ReferenceID: referenceID, Key: key, Value: value, } err := CreateConfig(ctx, db, "projects", "project", insert) if err != nil { return fmt.Errorf("Insert Config failed for Project: %w", err) } } return nil } // GetProjectID return the ID of the project with the given key. // generator: project ID func GetProjectID(ctx context.Context, db tx, name string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Project") }() stmt, err := Stmt(db, projectID) if err != nil { return -1, fmt.Errorf("Failed to get \"projectID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"projects\" ID: %w", err) } return id, nil } // RenameProject renames the project matching the given key parameters. // generator: project Rename func RenameProject(ctx context.Context, db dbtx, name string, to string) (_err error) { defer func() { _err = mapErr(_err, "Project") }() stmt, err := Stmt(db, projectRename) if err != nil { return fmt.Errorf("Failed to get \"projectRename\" prepared statement: %w", err) } result, err := stmt.Exec(to, name) if err != nil { return fmt.Errorf("Rename Project failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows failed: %w", err) } if n != 1 { return fmt.Errorf("Query affected %d rows instead of 1", n) } return nil } // DeleteProject deletes the project matching the given key parameters. // generator: project DeleteOne-by-Name func DeleteProject(ctx context.Context, db dbtx, name string) (_err error) { defer func() { _err = mapErr(_err, "Project") }() stmt, err := Stmt(db, projectDeleteByName) if err != nil { return fmt.Errorf("Failed to get \"projectDeleteByName\" prepared statement: %w", err) } result, err := stmt.Exec(name) if err != nil { return fmt.Errorf("Delete \"projects\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d Project rows instead of 1", n) } return nil } incus-7.0.0/internal/server/db/cluster/query.go000066400000000000000000000034731517523235500215420ustar00rootroot00000000000000package cluster import ( "context" "database/sql" "fmt" "github.com/lxc/incus/v7/internal/server/db/query" ) // Update the schema and api_extensions columns of the row in the nodes table // that matches the given id. // // If not such row is found, an error is returned. func updateNodeVersion(tx *sql.Tx, address string, apiExtensions int) error { stmt := "UPDATE nodes SET schema=?, api_extensions=? WHERE address=?" result, err := tx.Exec(stmt, len(updates), apiExtensions, address) if err != nil { return err } n, err := result.RowsAffected() if err != nil { return err } if n != 1 { return fmt.Errorf("updated %d rows instead of 1", n) } return nil } // Return the number of rows in the nodes table that have their address column // set to '0.0.0.0'. func selectUnclusteredNodesCount(ctx context.Context, tx *sql.Tx) (int, error) { return query.Count(ctx, tx, "nodes", "address='0.0.0.0'") } // Return a slice of binary integer tuples. Each tuple contains the schema // version and number of api extensions of a node in the cluster. func selectNodesVersions(ctx context.Context, tx *sql.Tx) ([][2]int, error) { versions := [][2]int{} stmt, err := tx.Prepare("SELECT schema, api_extensions FROM nodes WHERE state=0") if err != nil { // In order to make cluster updates work, let's check for "pending" as well as that's the column's previous name. stmt, err = tx.Prepare("SELECT schema, api_extensions FROM nodes WHERE pending=0") if err != nil { return nil, err } } defer func() { _ = stmt.Close() }() err = query.SelectObjects(ctx, stmt, func(scan func(dest ...any) error) error { version := [2]int{} err := scan(&version[0], &version[1]) if err != nil { return err } versions = append(versions, version) return nil }) if err != nil { return nil, err } return versions, nil } incus-7.0.0/internal/server/db/cluster/schema.go000066400000000000000000000616541517523235500216420ustar00rootroot00000000000000package cluster // DO NOT EDIT BY HAND // // This code was generated by the schema.DotGo function. If you need to // modify the database schema, please add a new schema update to update.go // and the run 'make update-schema'. const freshSchema = ` CREATE TABLE certificates ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, fingerprint TEXT NOT NULL, type INTEGER NOT NULL, name TEXT NOT NULL, certificate TEXT NOT NULL, restricted INTEGER NOT NULL DEFAULT 0, description TEXT NOT NULL DEFAULT "", UNIQUE (fingerprint) ); CREATE TABLE "certificates_projects" ( certificate_id INTEGER NOT NULL, project_id INTEGER NOT NULL, FOREIGN KEY (certificate_id) REFERENCES certificates (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE, UNIQUE (certificate_id, project_id) ); CREATE TABLE "cluster_groups" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, UNIQUE (name) ); CREATE TABLE cluster_groups_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, cluster_group_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (cluster_group_id, key), FOREIGN KEY (cluster_group_id) REFERENCES cluster_groups (id) ON DELETE CASCADE ); CREATE TABLE config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (key) ); CREATE TABLE "images" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, fingerprint TEXT NOT NULL, filename TEXT NOT NULL, size INTEGER NOT NULL, public INTEGER NOT NULL DEFAULT 0, architecture INTEGER NOT NULL, creation_date DATETIME, expiry_date DATETIME, upload_date DATETIME NOT NULL, cached INTEGER NOT NULL DEFAULT 0, last_use_date DATETIME, auto_update INTEGER NOT NULL DEFAULT 0, project_id INTEGER NOT NULL, type INTEGER NOT NULL DEFAULT 0, UNIQUE (project_id, fingerprint), FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE ); CREATE TABLE "images_aliases" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, image_id INTEGER NOT NULL, description TEXT NOT NULL, project_id INTEGER NOT NULL, UNIQUE (project_id, name), FOREIGN KEY (image_id) REFERENCES "images" (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE ); CREATE INDEX images_aliases_project_id_idx ON images_aliases (project_id); CREATE TABLE "images_nodes" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, node_id INTEGER NOT NULL, UNIQUE (image_id, node_id), FOREIGN KEY (image_id) REFERENCES "images" (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE ); CREATE TABLE "images_profiles" ( image_id INTEGER NOT NULL, profile_id INTEGER NOT NULL, FOREIGN KEY (image_id) REFERENCES "images" (id) ON DELETE CASCADE, FOREIGN KEY (profile_id) REFERENCES "profiles" (id) ON DELETE CASCADE, UNIQUE (image_id, profile_id) ); CREATE INDEX images_project_id_idx ON images (project_id); CREATE TABLE "images_properties" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, type INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (image_id) REFERENCES "images" (id) ON DELETE CASCADE ); CREATE TABLE "images_source" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, server TEXT NOT NULL, protocol INTEGER NOT NULL, certificate TEXT NOT NULL, alias TEXT NOT NULL, FOREIGN KEY (image_id) REFERENCES "images" (id) ON DELETE CASCADE ); CREATE TABLE "instances" ( id INTEGER primary key AUTOINCREMENT NOT NULL, node_id INTEGER NOT NULL, name TEXT NOT NULL, architecture INTEGER NOT NULL, type INTEGER NOT NULL, ephemeral INTEGER NOT NULL DEFAULT 0, creation_date DATETIME NOT NULL DEFAULT 0, stateful INTEGER NOT NULL DEFAULT 0, last_use_date DATETIME, description TEXT NOT NULL, project_id INTEGER NOT NULL, expiry_date DATETIME, UNIQUE (project_id, name), FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE ); CREATE TABLE "instances_backups" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, instance_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, creation_date DATETIME, expiry_date DATETIME, container_only INTEGER NOT NULL default 0, optimized_storage INTEGER NOT NULL default 0, root_only INTEGER NOT NULL DEFAULT 0, FOREIGN KEY (instance_id) REFERENCES "instances" (id) ON DELETE CASCADE, UNIQUE (instance_id, name) ); CREATE TABLE "instances_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, instance_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (instance_id) REFERENCES "instances" (id) ON DELETE CASCADE, UNIQUE (instance_id, key) ); CREATE TABLE "instances_devices" ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_id INTEGER NOT NULL, name TEXT NOT NULL, type INTEGER NOT NULL default 0, FOREIGN KEY (instance_id) REFERENCES "instances" (id) ON DELETE CASCADE, UNIQUE (instance_id, name) ); CREATE TABLE "instances_devices_config" ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (instance_device_id) REFERENCES "instances_devices" (id) ON DELETE CASCADE, UNIQUE (instance_device_id, key) ); CREATE INDEX instances_node_id_idx ON instances (node_id); CREATE TABLE "instances_profiles" ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_id INTEGER NOT NULL, profile_id INTEGER NOT NULL, apply_order INTEGER NOT NULL default 0, UNIQUE (instance_id, profile_id), FOREIGN KEY (instance_id) REFERENCES "instances" (id) ON DELETE CASCADE, FOREIGN KEY (profile_id) REFERENCES "profiles"(id) ON DELETE CASCADE ); CREATE INDEX instances_project_id_and_name_idx ON instances (project_id, name); CREATE INDEX instances_project_id_and_node_id_and_name_idx ON instances (project_id, node_id, name); CREATE INDEX instances_project_id_and_node_id_idx ON instances (project_id, node_id); CREATE INDEX instances_project_id_idx ON instances (project_id); CREATE TABLE "instances_snapshots" ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_id INTEGER NOT NULL, name TEXT NOT NULL, creation_date DATETIME NOT NULL DEFAULT 0, stateful INTEGER NOT NULL DEFAULT 0, description TEXT NOT NULL, expiry_date DATETIME, UNIQUE (instance_id, name), FOREIGN KEY (instance_id) REFERENCES "instances" (id) ON DELETE CASCADE ); CREATE TABLE "instances_snapshots_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, instance_snapshot_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (instance_snapshot_id) REFERENCES "instances_snapshots" (id) ON DELETE CASCADE, UNIQUE (instance_snapshot_id, key) ); CREATE TABLE "instances_snapshots_devices" ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_snapshot_id INTEGER NOT NULL, name TEXT NOT NULL, type INTEGER NOT NULL default 0, FOREIGN KEY (instance_snapshot_id) REFERENCES "instances_snapshots" (id) ON DELETE CASCADE, UNIQUE (instance_snapshot_id, name) ); CREATE TABLE "instances_snapshots_devices_config" ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_snapshot_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (instance_snapshot_device_id) REFERENCES "instances_snapshots_devices" (id) ON DELETE CASCADE, UNIQUE (instance_snapshot_device_id, key) ); CREATE TABLE "networks" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, state INTEGER NOT NULL DEFAULT 0, type INTEGER NOT NULL DEFAULT 0, UNIQUE (project_id, name), FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE ); CREATE TABLE "networks_acls" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, ingress TEXT NOT NULL, egress TEXT NOT NULL, UNIQUE (project_id, name), FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE ); CREATE TABLE "networks_acls_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_acl_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_acl_id, key), FOREIGN KEY (network_acl_id) REFERENCES "networks_acls" (id) ON DELETE CASCADE ); CREATE TABLE "networks_address_sets" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, name TEXT NOT NULL, addresses TEXT NOT NULL, description TEXT, UNIQUE (name) UNIQUE (project_id, name), FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE ); CREATE TABLE "networks_address_sets_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_address_set_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_address_set_id, key), FOREIGN KEY (network_address_set_id) REFERENCES networks_address_sets (id) ON DELETE CASCADE ); CREATE TABLE "networks_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_id, node_id, key), FOREIGN KEY (network_id) REFERENCES "networks" (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE ); CREATE TABLE "networks_forwards" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER, listen_address TEXT NOT NULL, description TEXT NOT NULL, ports TEXT NOT NULL, UNIQUE (network_id, node_id, listen_address), FOREIGN KEY (network_id) REFERENCES "networks" (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE ); CREATE TABLE "networks_forwards_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_forward_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_forward_id, key), FOREIGN KEY (network_forward_id) REFERENCES "networks_forwards" (id) ON DELETE CASCADE ); CREATE TABLE networks_integrations ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, type INTEGER NOT NULL, UNIQUE (name) ); CREATE TABLE networks_integrations_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_integration_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_integration_id, key), FOREIGN KEY (network_integration_id) REFERENCES networks_integrations (id) ON DELETE CASCADE ); CREATE TABLE "networks_load_balancers" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER, listen_address TEXT NOT NULL, description TEXT NOT NULL, backends TEXT NOT NULL, ports TEXT NOT NULL, UNIQUE (network_id, node_id, listen_address), FOREIGN KEY (network_id) REFERENCES "networks" (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE ); CREATE TABLE "networks_load_balancers_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_load_balancer_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_load_balancer_id, key), FOREIGN KEY (network_load_balancer_id) REFERENCES "networks_load_balancers" (id) ON DELETE CASCADE ); CREATE TABLE "networks_nodes" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER NOT NULL, state INTEGER NOT NULL DEFAULT 0, UNIQUE (network_id, node_id), FOREIGN KEY (network_id) REFERENCES "networks" (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE ); CREATE TABLE "networks_peers" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, target_network_project TEXT NULL, target_network_name TEXT NULL, target_network_id INTEGER NULL, type INTEGER NOT NULL DEFAULT 0, target_network_integration_id INTEGER DEFAULT NULL REFERENCES networks_integrations (id) ON DELETE CASCADE, UNIQUE (network_id, name), UNIQUE (network_id, target_network_project, target_network_name), UNIQUE (network_id, target_network_id), FOREIGN KEY (network_id) REFERENCES "networks" (id) ON DELETE CASCADE ); CREATE TABLE "networks_peers_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_peer_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_peer_id, key), FOREIGN KEY (network_peer_id) REFERENCES "networks_peers" (id) ON DELETE CASCADE ); CREATE UNIQUE INDEX networks_unique_network_id_node_id_key ON "networks_config" (network_id, IFNULL(node_id, -1), key); CREATE TABLE "networks_zones" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, UNIQUE (name), FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE ); CREATE TABLE "networks_zones_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_zone_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_zone_id, key), FOREIGN KEY (network_zone_id) REFERENCES "networks_zones" (id) ON DELETE CASCADE ); CREATE TABLE "networks_zones_records" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_zone_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, entries TEXT NOT NULL, UNIQUE (network_zone_id, name), FOREIGN KEY (network_zone_id) REFERENCES networks_zones (id) ON DELETE CASCADE ); CREATE TABLE "networks_zones_records_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_zone_record_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_zone_record_id, key), FOREIGN KEY (network_zone_record_id) REFERENCES "networks_zones_records" (id) ON DELETE CASCADE ); CREATE TABLE "nodes" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, address TEXT NOT NULL, schema INTEGER NOT NULL, api_extensions INTEGER NOT NULL, heartbeat DATETIME DEFAULT CURRENT_TIMESTAMP, state INTEGER NOT NULL DEFAULT 0, arch INTEGER NOT NULL DEFAULT 0 CHECK (arch > 0), failure_domain_id INTEGER DEFAULT NULL REFERENCES nodes_failure_domains (id) ON DELETE SET NULL, UNIQUE (name), UNIQUE (address) ); CREATE TABLE "nodes_cluster_groups" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, node_id INTEGER NOT NULL, group_id INTEGER NOT NULL, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE, FOREIGN KEY (group_id) REFERENCES cluster_groups (id) ON DELETE CASCADE, UNIQUE (node_id, group_id) ); CREATE TABLE "nodes_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, node_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE, UNIQUE (node_id, key) ); CREATE TABLE nodes_failure_domains ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, UNIQUE (name) ); CREATE TABLE "nodes_roles" ( node_id INTEGER NOT NULL, role INTEGER NOT NULL, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE, UNIQUE (node_id, role) ); CREATE TABLE "operations" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, uuid TEXT NOT NULL, node_id TEXT NOT NULL, type INTEGER NOT NULL DEFAULT 0, project_id INTEGER, UNIQUE (uuid), FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE ); CREATE TABLE "profiles" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, project_id INTEGER NOT NULL, UNIQUE (project_id, name), FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE ); CREATE TABLE "profiles_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (profile_id, key), FOREIGN KEY (profile_id) REFERENCES "profiles"(id) ON DELETE CASCADE ); CREATE TABLE "profiles_devices" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_id INTEGER NOT NULL, name TEXT NOT NULL, type INTEGER NOT NULL default 0, UNIQUE (profile_id, name), FOREIGN KEY (profile_id) REFERENCES "profiles" (id) ON DELETE CASCADE ); CREATE TABLE "profiles_devices_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (profile_device_id, key), FOREIGN KEY (profile_device_id) REFERENCES "profiles_devices" (id) ON DELETE CASCADE ); CREATE INDEX profiles_project_id_idx ON profiles (project_id); CREATE TABLE "projects" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, UNIQUE (name) ); CREATE TABLE "projects_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE, UNIQUE (project_id, key) ); CREATE TABLE "storage_buckets" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER, description TEXT NOT NULL, project_id INTEGER NOT NULL, UNIQUE (node_id, name), FOREIGN KEY (storage_pool_id) REFERENCES "storage_pools" (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE ); CREATE TABLE "storage_buckets_backups" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_bucket_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, creation_date DATETIME, expiry_date DATETIME, FOREIGN KEY (storage_bucket_id) REFERENCES "storage_buckets" (id) ON DELETE CASCADE, UNIQUE (storage_bucket_id, name) ); CREATE TABLE "storage_buckets_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_bucket_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (storage_bucket_id, key), FOREIGN KEY (storage_bucket_id) REFERENCES "storage_buckets" (id) ON DELETE CASCADE ); CREATE TABLE "storage_buckets_keys" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_bucket_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, access_key TEXT NOT NULL, secret_key TEXT NOT NULL, role TEXT NOT NULL, UNIQUE (storage_bucket_id, name), FOREIGN KEY (storage_bucket_id) REFERENCES "storage_buckets" (id) ON DELETE CASCADE ); CREATE UNIQUE INDEX storage_buckets_unique_storage_pool_id_node_id_name ON "storage_buckets" (storage_pool_id, IFNULL(node_id, -1), name); CREATE TABLE "storage_pools" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, driver TEXT NOT NULL, description TEXT NOT NULL, state INTEGER NOT NULL DEFAULT 0, UNIQUE (name) ); CREATE TABLE "storage_pools_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (storage_pool_id, node_id, key), FOREIGN KEY (storage_pool_id) REFERENCES "storage_pools" (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE ); CREATE TABLE "storage_pools_nodes" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER NOT NULL, state INTEGER NOT NULL DEFAULT 0, UNIQUE (storage_pool_id, node_id), FOREIGN KEY (storage_pool_id) REFERENCES "storage_pools" (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE ); CREATE UNIQUE INDEX storage_pools_unique_storage_pool_id_node_id_key ON storage_pools_config (storage_pool_id, IFNULL(node_id, -1), key); CREATE TABLE "storage_volumes" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER, type INTEGER NOT NULL, description TEXT NOT NULL, project_id INTEGER NOT NULL, content_type INTEGER NOT NULL DEFAULT 0, creation_date DATETIME NOT NULL DEFAULT "0001-01-01T00:00:00Z", UNIQUE (storage_pool_id, node_id, project_id, name, type), FOREIGN KEY (storage_pool_id) REFERENCES "storage_pools" (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE ); CREATE VIEW storage_volumes_all ( id, name, storage_pool_id, node_id, type, description, project_id, content_type, creation_date) AS SELECT id, name, storage_pool_id, node_id, type, description, project_id, content_type, creation_date FROM storage_volumes UNION SELECT storage_volumes_snapshots.id, printf('%s/%s', storage_volumes.name, storage_volumes_snapshots.name), storage_volumes.storage_pool_id, storage_volumes.node_id, storage_volumes.type, storage_volumes_snapshots.description, storage_volumes.project_id, storage_volumes.content_type, storage_volumes_snapshots.creation_date FROM storage_volumes JOIN storage_volumes_snapshots ON storage_volumes.id = storage_volumes_snapshots.storage_volume_id; CREATE TABLE "storage_volumes_backups" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, creation_date DATETIME, expiry_date DATETIME, volume_only INTEGER NOT NULL default 0, optimized_storage INTEGER NOT NULL default 0, FOREIGN KEY (storage_volume_id) REFERENCES "storage_volumes" (id) ON DELETE CASCADE, UNIQUE (storage_volume_id, name) ); CREATE TRIGGER storage_volumes_check_id BEFORE INSERT ON storage_volumes WHEN NEW.id IN (SELECT id FROM storage_volumes_snapshots) BEGIN SELECT RAISE(FAIL, "invalid ID"); END; CREATE TABLE "storage_volumes_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (storage_volume_id, key), FOREIGN KEY (storage_volume_id) REFERENCES "storage_volumes" (id) ON DELETE CASCADE ); CREATE TABLE "storage_volumes_snapshots" ( id INTEGER NOT NULL, storage_volume_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, expiry_date DATETIME, creation_date DATETIME NOT NULL DEFAULT "0001-01-01T00:00:00Z", UNIQUE (id), UNIQUE (storage_volume_id, name), FOREIGN KEY (storage_volume_id) REFERENCES "storage_volumes" (id) ON DELETE CASCADE ); CREATE TRIGGER storage_volumes_snapshots_check_id BEFORE INSERT ON storage_volumes_snapshots WHEN NEW.id IN (SELECT id FROM storage_volumes) BEGIN SELECT RAISE(FAIL, "invalid ID"); END; CREATE TABLE "storage_volumes_snapshots_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_snapshot_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (storage_volume_snapshot_id) REFERENCES "storage_volumes_snapshots" (id) ON DELETE CASCADE, UNIQUE (storage_volume_snapshot_id, key) ); CREATE UNIQUE INDEX storage_volumes_unique_storage_pool_id_node_id_project_id_name_type ON "storage_volumes" (storage_pool_id, IFNULL(node_id, -1), project_id, name, type); CREATE TABLE "warnings" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, node_id INTEGER, project_id INTEGER, entity_type_code INTEGER, entity_id INTEGER, uuid TEXT NOT NULL, type_code INTEGER NOT NULL, status INTEGER NOT NULL, first_seen_date DATETIME NOT NULL, last_seen_date DATETIME NOT NULL, updated_date DATETIME, last_message TEXT NOT NULL, count INTEGER NOT NULL, UNIQUE (uuid), FOREIGN KEY (node_id) REFERENCES "nodes"(id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE ); CREATE UNIQUE INDEX warnings_unique_node_id_project_id_entity_type_code_entity_id_type_code ON warnings(IFNULL(node_id, -1), IFNULL(project_id, -1), entity_type_code, entity_id, type_code); INSERT INTO schema (version, updated_at) VALUES (77, strftime("%s")) ` incus-7.0.0/internal/server/db/cluster/snapshots.go000066400000000000000000000054001517523235500224070ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "database/sql" "time" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" ) // Code generation directives. // //generate-database:mapper target snapshots.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e instance_snapshot objects //generate-database:mapper stmt -e instance_snapshot objects-by-ID //generate-database:mapper stmt -e instance_snapshot objects-by-Project-and-Instance //generate-database:mapper stmt -e instance_snapshot objects-by-Project-and-Instance-and-Name //generate-database:mapper stmt -e instance_snapshot id //generate-database:mapper stmt -e instance_snapshot create references=Config,Devices //generate-database:mapper stmt -e instance_snapshot rename //generate-database:mapper stmt -e instance_snapshot delete-by-Project-and-Instance-and-Name // //generate-database:mapper method -i -e instance_snapshot GetMany references=Config,Device //generate-database:mapper method -i -e instance_snapshot GetOne //generate-database:mapper method -i -e instance_snapshot ID //generate-database:mapper method -i -e instance_snapshot Exists //generate-database:mapper method -i -e instance_snapshot Create references=Config,Device //generate-database:mapper method -i -e instance_snapshot Rename //generate-database:mapper method -i -e instance_snapshot DeleteOne-by-Project-and-Instance-and-Name // InstanceSnapshot is a value object holding db-related details about a snapshot. type InstanceSnapshot struct { ID int Project string `db:"primary=yes&join=projects.name&joinon=instances.project_id"` Instance string `db:"primary=yes&join=instances.name"` Name string `db:"primary=yes"` CreationDate time.Time Stateful bool Description string `db:"coalesce=''"` ExpiryDate sql.NullTime } // InstanceSnapshotFilter specifies potential query parameter fields. type InstanceSnapshotFilter struct { ID *int Project *string Instance *string Name *string } // ToInstance converts an instance snapshot to a database Instance, filling in extra fields from the parent instance. func (s *InstanceSnapshot) ToInstance(parentName string, parentNode string, parentType instancetype.Type, parentArch int) Instance { return Instance{ ID: s.ID, Project: s.Project, Name: parentName + internalInstance.SnapshotDelimiter + s.Name, Node: parentNode, Type: parentType, Snapshot: true, Architecture: parentArch, Ephemeral: false, CreationDate: s.CreationDate, Stateful: s.Stateful, LastUseDate: sql.NullTime{}, Description: s.Description, ExpiryDate: s.ExpiryDate, } } incus-7.0.0/internal/server/db/cluster/snapshots.interface.mapper.go000066400000000000000000000054541517523235500256420ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // InstanceSnapshotGenerated is an interface of generated methods for InstanceSnapshot. type InstanceSnapshotGenerated interface { // GetInstanceSnapshotConfig returns all available InstanceSnapshot Config // generator: instance_snapshot GetMany GetInstanceSnapshotConfig(ctx context.Context, db tx, instanceSnapshotID int, filters ...ConfigFilter) (map[string]string, error) // GetInstanceSnapshotDevices returns all available InstanceSnapshot Devices // generator: instance_snapshot GetMany GetInstanceSnapshotDevices(ctx context.Context, db tx, instanceSnapshotID int, filters ...DeviceFilter) (map[string]Device, error) // GetInstanceSnapshots returns all available instance_snapshots. // generator: instance_snapshot GetMany GetInstanceSnapshots(ctx context.Context, db dbtx, filters ...InstanceSnapshotFilter) ([]InstanceSnapshot, error) // GetInstanceSnapshot returns the instance_snapshot with the given key. // generator: instance_snapshot GetOne GetInstanceSnapshot(ctx context.Context, db dbtx, project string, instance string, name string) (*InstanceSnapshot, error) // GetInstanceSnapshotID return the ID of the instance_snapshot with the given key. // generator: instance_snapshot ID GetInstanceSnapshotID(ctx context.Context, db tx, project string, instance string, name string) (int64, error) // InstanceSnapshotExists checks if a instance_snapshot with the given key exists. // generator: instance_snapshot Exists InstanceSnapshotExists(ctx context.Context, db dbtx, project string, instance string, name string) (bool, error) // CreateInstanceSnapshotConfig adds new instance_snapshot Config to the database. // generator: instance_snapshot Create CreateInstanceSnapshotConfig(ctx context.Context, db dbtx, instanceSnapshotID int64, config map[string]string) error // CreateInstanceSnapshotDevices adds new instance_snapshot Devices to the database. // generator: instance_snapshot Create CreateInstanceSnapshotDevices(ctx context.Context, db tx, instanceSnapshotID int64, devices map[string]Device) error // CreateInstanceSnapshot adds a new instance_snapshot to the database. // generator: instance_snapshot Create CreateInstanceSnapshot(ctx context.Context, db dbtx, object InstanceSnapshot) (int64, error) // RenameInstanceSnapshot renames the instance_snapshot matching the given key parameters. // generator: instance_snapshot Rename RenameInstanceSnapshot(ctx context.Context, db dbtx, project string, instance string, name string, to string) error // DeleteInstanceSnapshot deletes the instance_snapshot matching the given key parameters. // generator: instance_snapshot DeleteOne-by-Project-and-Instance-and-Name DeleteInstanceSnapshot(ctx context.Context, db dbtx, project string, instance string, name string) error } incus-7.0.0/internal/server/db/cluster/snapshots.mapper.go000066400000000000000000000415661517523235500237070ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var instanceSnapshotObjects = RegisterStmt(` SELECT instances_snapshots.id, projects.name AS project, instances.name AS instance, instances_snapshots.name, instances_snapshots.creation_date, instances_snapshots.stateful, coalesce(instances_snapshots.description, ''), instances_snapshots.expiry_date FROM instances_snapshots JOIN projects ON instances.project_id = projects.id JOIN instances ON instances_snapshots.instance_id = instances.id ORDER BY projects.id, instances.id, instances_snapshots.name `) var instanceSnapshotObjectsByID = RegisterStmt(` SELECT instances_snapshots.id, projects.name AS project, instances.name AS instance, instances_snapshots.name, instances_snapshots.creation_date, instances_snapshots.stateful, coalesce(instances_snapshots.description, ''), instances_snapshots.expiry_date FROM instances_snapshots JOIN projects ON instances.project_id = projects.id JOIN instances ON instances_snapshots.instance_id = instances.id WHERE ( instances_snapshots.id = ? ) ORDER BY projects.id, instances.id, instances_snapshots.name `) var instanceSnapshotObjectsByProjectAndInstance = RegisterStmt(` SELECT instances_snapshots.id, projects.name AS project, instances.name AS instance, instances_snapshots.name, instances_snapshots.creation_date, instances_snapshots.stateful, coalesce(instances_snapshots.description, ''), instances_snapshots.expiry_date FROM instances_snapshots JOIN projects ON instances.project_id = projects.id JOIN instances ON instances_snapshots.instance_id = instances.id WHERE ( project = ? AND instance = ? ) ORDER BY projects.id, instances.id, instances_snapshots.name `) var instanceSnapshotObjectsByProjectAndInstanceAndName = RegisterStmt(` SELECT instances_snapshots.id, projects.name AS project, instances.name AS instance, instances_snapshots.name, instances_snapshots.creation_date, instances_snapshots.stateful, coalesce(instances_snapshots.description, ''), instances_snapshots.expiry_date FROM instances_snapshots JOIN projects ON instances.project_id = projects.id JOIN instances ON instances_snapshots.instance_id = instances.id WHERE ( project = ? AND instance = ? AND instances_snapshots.name = ? ) ORDER BY projects.id, instances.id, instances_snapshots.name `) var instanceSnapshotID = RegisterStmt(` SELECT instances_snapshots.id FROM instances_snapshots JOIN projects ON instances.project_id = projects.id JOIN instances ON instances_snapshots.instance_id = instances.id WHERE projects.name = ? AND instances.name = ? AND instances_snapshots.name = ? `) var instanceSnapshotCreate = RegisterStmt(` INSERT INTO instances_snapshots (instance_id, name, creation_date, stateful, description, expiry_date) VALUES ((SELECT instances.id FROM instances JOIN projects ON instances.project_id = projects.id WHERE projects.name = ? AND instances.name = ?), ?, ?, ?, ?, ?) `) var instanceSnapshotRename = RegisterStmt(` UPDATE instances_snapshots SET name = ? WHERE instance_id = (SELECT instances.id FROM instances JOIN projects ON instances.project_id = projects.id WHERE projects.name = ? AND instances.name = ?) AND name = ? `) var instanceSnapshotDeleteByProjectAndInstanceAndName = RegisterStmt(` DELETE FROM instances_snapshots WHERE instance_id = (SELECT instances.id FROM instances JOIN projects ON instances.project_id = projects.id WHERE projects.name = ? AND instances.name = ?) AND name = ? `) // instanceSnapshotColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the InstanceSnapshot entity. func instanceSnapshotColumns() string { return "instances_snapshots.id, projects.name AS project, instances.name AS instance, instances_snapshots.name, instances_snapshots.creation_date, instances_snapshots.stateful, coalesce(instances_snapshots.description, ''), instances_snapshots.expiry_date" } // getInstanceSnapshots can be used to run handwritten sql.Stmts to return a slice of objects. func getInstanceSnapshots(ctx context.Context, stmt *sql.Stmt, args ...any) ([]InstanceSnapshot, error) { objects := make([]InstanceSnapshot, 0) dest := func(scan func(dest ...any) error) error { i := InstanceSnapshot{} err := scan(&i.ID, &i.Project, &i.Instance, &i.Name, &i.CreationDate, &i.Stateful, &i.Description, &i.ExpiryDate) if err != nil { return err } objects = append(objects, i) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"instances_snapshots\" table: %w", err) } return objects, nil } // getInstanceSnapshotsRaw can be used to run handwritten query strings to return a slice of objects. func getInstanceSnapshotsRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]InstanceSnapshot, error) { objects := make([]InstanceSnapshot, 0) dest := func(scan func(dest ...any) error) error { i := InstanceSnapshot{} err := scan(&i.ID, &i.Project, &i.Instance, &i.Name, &i.CreationDate, &i.Stateful, &i.Description, &i.ExpiryDate) if err != nil { return err } objects = append(objects, i) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"instances_snapshots\" table: %w", err) } return objects, nil } // GetInstanceSnapshots returns all available instance_snapshots. // generator: instance_snapshot GetMany func GetInstanceSnapshots(ctx context.Context, db dbtx, filters ...InstanceSnapshotFilter) (_ []InstanceSnapshot, _err error) { defer func() { _err = mapErr(_err, "Instance_snapshot") }() var err error // Result slice. objects := make([]InstanceSnapshot, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, instanceSnapshotObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceSnapshotObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.Project != nil && filter.Instance != nil && filter.Name != nil && filter.ID == nil { args = append(args, []any{filter.Project, filter.Instance, filter.Name}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceSnapshotObjectsByProjectAndInstanceAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceSnapshotObjectsByProjectAndInstanceAndName\" prepared statement: %w", err) } break } query, err := StmtString(instanceSnapshotObjectsByProjectAndInstanceAndName) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceSnapshotObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.Instance != nil && filter.ID == nil && filter.Name == nil { args = append(args, []any{filter.Project, filter.Instance}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceSnapshotObjectsByProjectAndInstance) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceSnapshotObjectsByProjectAndInstance\" prepared statement: %w", err) } break } query, err := StmtString(instanceSnapshotObjectsByProjectAndInstance) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceSnapshotObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID != nil && filter.Project == nil && filter.Instance == nil && filter.Name == nil { args = append(args, []any{filter.ID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, instanceSnapshotObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceSnapshotObjectsByID\" prepared statement: %w", err) } break } query, err := StmtString(instanceSnapshotObjectsByID) if err != nil { return nil, fmt.Errorf("Failed to get \"instanceSnapshotObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.Project == nil && filter.Instance == nil && filter.Name == nil { return nil, fmt.Errorf("Cannot filter on empty InstanceSnapshotFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getInstanceSnapshots(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getInstanceSnapshotsRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"instances_snapshots\" table: %w", err) } return objects, nil } // GetInstanceSnapshotDevices returns all available InstanceSnapshot Devices // generator: instance_snapshot GetMany func GetInstanceSnapshotDevices(ctx context.Context, db tx, instanceSnapshotID int, filters ...DeviceFilter) (_ map[string]Device, _err error) { defer func() { _err = mapErr(_err, "Instance_snapshot") }() instanceSnapshotDevices, err := GetDevices(ctx, db, "instances_snapshots", "instance_snapshot", filters...) if err != nil { return nil, err } devices := map[string]Device{} for _, ref := range instanceSnapshotDevices[instanceSnapshotID] { _, ok := devices[ref.Name] if !ok { devices[ref.Name] = ref } else { return nil, fmt.Errorf("Found duplicate Device with name %q", ref.Name) } } return devices, nil } // GetInstanceSnapshotConfig returns all available InstanceSnapshot Config // generator: instance_snapshot GetMany func GetInstanceSnapshotConfig(ctx context.Context, db tx, instanceSnapshotID int, filters ...ConfigFilter) (_ map[string]string, _err error) { defer func() { _err = mapErr(_err, "Instance_snapshot") }() instanceSnapshotConfig, err := GetConfig(ctx, db, "instances_snapshots", "instance_snapshot", filters...) if err != nil { return nil, err } config, ok := instanceSnapshotConfig[instanceSnapshotID] if !ok { config = map[string]string{} } return config, nil } // GetInstanceSnapshot returns the instance_snapshot with the given key. // generator: instance_snapshot GetOne func GetInstanceSnapshot(ctx context.Context, db dbtx, project string, instance string, name string) (_ *InstanceSnapshot, _err error) { defer func() { _err = mapErr(_err, "Instance_snapshot") }() filter := InstanceSnapshotFilter{} filter.Project = &project filter.Instance = &instance filter.Name = &name objects, err := GetInstanceSnapshots(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"instances_snapshots\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"instances_snapshots\" entry matches") } } // GetInstanceSnapshotID return the ID of the instance_snapshot with the given key. // generator: instance_snapshot ID func GetInstanceSnapshotID(ctx context.Context, db tx, project string, instance string, name string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Instance_snapshot") }() stmt, err := Stmt(db, instanceSnapshotID) if err != nil { return -1, fmt.Errorf("Failed to get \"instanceSnapshotID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, project, instance, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"instances_snapshots\" ID: %w", err) } return id, nil } // InstanceSnapshotExists checks if a instance_snapshot with the given key exists. // generator: instance_snapshot Exists func InstanceSnapshotExists(ctx context.Context, db dbtx, project string, instance string, name string) (_ bool, _err error) { defer func() { _err = mapErr(_err, "Instance_snapshot") }() stmt, err := Stmt(db, instanceSnapshotID) if err != nil { return false, fmt.Errorf("Failed to get \"instanceSnapshotID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, project, instance, name) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("Failed to get \"instances_snapshots\" ID: %w", err) } return true, nil } // CreateInstanceSnapshot adds a new instance_snapshot to the database. // generator: instance_snapshot Create func CreateInstanceSnapshot(ctx context.Context, db dbtx, object InstanceSnapshot) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Instance_snapshot") }() args := make([]any, 7) // Populate the statement arguments. args[0] = object.Project args[1] = object.Instance args[2] = object.Name args[3] = object.CreationDate args[4] = object.Stateful args[5] = object.Description args[6] = object.ExpiryDate // Prepared statement to use. stmt, err := Stmt(db, instanceSnapshotCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"instanceSnapshotCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil && strings.HasPrefix(err.Error(), "UNIQUE constraint failed:") { return -1, ErrConflict } if err != nil { return -1, fmt.Errorf("Failed to create \"instances_snapshots\" entry: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch \"instances_snapshots\" entry ID: %w", err) } return id, nil } // CreateInstanceSnapshotDevices adds new instance_snapshot Devices to the database. // generator: instance_snapshot Create func CreateInstanceSnapshotDevices(ctx context.Context, db tx, instanceSnapshotID int64, devices map[string]Device) (_err error) { defer func() { _err = mapErr(_err, "Instance_snapshot") }() for key, device := range devices { device.ReferenceID = int(instanceSnapshotID) devices[key] = device } err := CreateDevices(ctx, db, "instances_snapshots", "instance_snapshot", devices) if err != nil { return fmt.Errorf("Insert Device failed for InstanceSnapshot: %w", err) } return nil } // CreateInstanceSnapshotConfig adds new instance_snapshot Config to the database. // generator: instance_snapshot Create func CreateInstanceSnapshotConfig(ctx context.Context, db dbtx, instanceSnapshotID int64, config map[string]string) (_err error) { defer func() { _err = mapErr(_err, "Instance_snapshot") }() referenceID := int(instanceSnapshotID) for key, value := range config { insert := Config{ ReferenceID: referenceID, Key: key, Value: value, } err := CreateConfig(ctx, db, "instances_snapshots", "instance_snapshot", insert) if err != nil { return fmt.Errorf("Insert Config failed for InstanceSnapshot: %w", err) } } return nil } // RenameInstanceSnapshot renames the instance_snapshot matching the given key parameters. // generator: instance_snapshot Rename func RenameInstanceSnapshot(ctx context.Context, db dbtx, project string, instance string, name string, to string) (_err error) { defer func() { _err = mapErr(_err, "Instance_snapshot") }() stmt, err := Stmt(db, instanceSnapshotRename) if err != nil { return fmt.Errorf("Failed to get \"instanceSnapshotRename\" prepared statement: %w", err) } result, err := stmt.Exec(to, project, instance, name) if err != nil { return fmt.Errorf("Rename InstanceSnapshot failed: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows failed: %w", err) } if n != 1 { return fmt.Errorf("Query affected %d rows instead of 1", n) } return nil } // DeleteInstanceSnapshot deletes the instance_snapshot matching the given key parameters. // generator: instance_snapshot DeleteOne-by-Project-and-Instance-and-Name func DeleteInstanceSnapshot(ctx context.Context, db dbtx, project string, instance string, name string) (_err error) { defer func() { _err = mapErr(_err, "Instance_snapshot") }() stmt, err := Stmt(db, instanceSnapshotDeleteByProjectAndInstanceAndName) if err != nil { return fmt.Errorf("Failed to get \"instanceSnapshotDeleteByProjectAndInstanceAndName\" prepared statement: %w", err) } result, err := stmt.Exec(project, instance, name) if err != nil { return fmt.Errorf("Delete \"instances_snapshots\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d InstanceSnapshot rows instead of 1", n) } return nil } incus-7.0.0/internal/server/db/cluster/update.go000066400000000000000000004757011517523235500216660ustar00rootroot00000000000000package cluster import ( "context" "database/sql" "fmt" "strconv" "strings" "time" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/internal/server/db/schema" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" ) // Schema for the cluster database. func Schema() *schema.Schema { schema := schema.NewFromMap(updates) schema.Fresh(freshSchema) return schema } // FreshSchema returns the fresh schema definition of the global database. func FreshSchema() string { return freshSchema } // SchemaDotGo refreshes the schema.go file in this package, using the updates // defined here. func SchemaDotGo() error { return schema.DotGo(updates, "schema") } // SchemaVersion is the current version of the cluster database schema. var SchemaVersion = len(updates) var updates = map[int]schema.Update{ 1: updateFromV0, 2: updateFromV1, 3: updateFromV2, 4: updateFromV3, 5: updateFromV4, 6: updateFromV5, 7: updateFromV6, 8: updateFromV7, 9: updateFromV8, 10: updateFromV9, 11: updateFromV10, 12: updateFromV11, 13: updateFromV12, 14: updateFromV13, 15: updateFromV14, 16: updateFromV15, 17: updateFromV16, 18: updateFromV17, 19: updateFromV18, 20: updateFromV19, 21: updateFromV20, 22: updateFromV21, 23: updateFromV22, 24: updateFromV23, 25: updateFromV24, 26: updateFromV25, 27: updateFromV26, 28: updateFromV27, 29: updateFromV28, 30: updateFromV29, 31: updateFromV30, 32: updateFromV31, 33: updateFromV32, 34: updateFromV33, 35: updateFromV34, 36: updateFromV35, 37: updateFromV36, 38: updateFromV37, 39: updateFromV38, 40: updateFromV39, 41: updateFromV40, 42: updateFromV41, 43: updateFromV42, 44: updateFromV43, 45: updateFromV44, 46: updateFromV45, 47: updateFromV46, 48: updateFromV47, 49: updateFromV48, 50: updateFromV49, 51: updateFromV50, 52: updateFromV51, 53: updateFromV52, 54: updateFromV53, 55: updateFromV54, 56: updateFromV55, 57: updateFromV56, 58: updateFromV57, 59: updateFromV58, 60: updateFromV59, 61: updateFromV60, 62: updateFromV61, 63: updateFromV62, 64: updateFromV63, 65: updateFromV64, 66: updateFromV65, 67: updateFromV66, 68: updateFromV67, 69: updateFromV68, 70: updateFromV69, 71: updateFromV70, 72: updateFromV71, 73: updateFromV72, 74: updateFromV73, 75: updateFromV74, 76: updateFromV75, 77: updateFromV76, } func updateFromV76(ctx context.Context, tx *sql.Tx) error { stmts := ` ALTER TABLE instances_backups ADD COLUMN root_only INTEGER NOT NULL DEFAULT 0; ` _, err := tx.Exec(stmts) return err } func updateFromV75(ctx context.Context, tx *sql.Tx) error { q := ` CREATE TABLE "networks_address_sets" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, name TEXT NOT NULL, addresses TEXT NOT NULL, description TEXT, UNIQUE (name) UNIQUE (project_id, name), FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE ); CREATE TABLE "networks_address_sets_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_address_set_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_address_set_id, key), FOREIGN KEY (network_address_set_id) REFERENCES networks_address_sets (id) ON DELETE CASCADE ); ` _, err := tx.Exec(q) if err != nil { return fmt.Errorf("Failed creating networks_address_sets and networks_address_sets_external_ids tables: %w", err) } return nil } // updateFromV74 removes the index preventing the same integration to be used multiple times. func updateFromV74(ctx context.Context, tx *sql.Tx) error { q := `DROP INDEX IF EXISTS networks_peers_unique_network_id_target_network_integration_id;` _, err := tx.Exec(q) if err != nil { return fmt.Errorf("Failed dropping network peer index: %w", err) } return nil } // updateFromV73 adds a config table to cluster groups. func updateFromV73(ctx context.Context, tx *sql.Tx) error { q := ` CREATE TABLE cluster_groups_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, cluster_group_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (cluster_group_id, key), FOREIGN KEY (cluster_group_id) REFERENCES cluster_groups (id) ON DELETE CASCADE ); ` _, err := tx.Exec(q) if err != nil { return fmt.Errorf("Failed adding cluster group config table: %w", err) } return nil } // updateFromV72 removes the openfga.store.model_id server config key. func updateFromV72(ctx context.Context, tx *sql.Tx) error { q := `DELETE FROM config WHERE key='openfga.store.model_id';` _, err := tx.Exec(q) if err != nil { return fmt.Errorf("Failed adding network integration support: %w", err) } return nil } // updateFromV71 adds network integration support. func updateFromV71(ctx context.Context, tx *sql.Tx) error { q := ` CREATE TABLE networks_integrations ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, type INTEGER NOT NULL, UNIQUE (name) ); CREATE TABLE networks_integrations_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_integration_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_integration_id, key), FOREIGN KEY (network_integration_id) REFERENCES networks_integrations (id) ON DELETE CASCADE ); ALTER TABLE networks_peers ADD COLUMN type INTEGER NOT NULL DEFAULT 0; ALTER TABLE networks_peers ADD COLUMN target_network_integration_id INTEGER DEFAULT NULL REFERENCES networks_integrations (id) ON DELETE CASCADE; CREATE UNIQUE INDEX networks_peers_unique_network_id_target_network_integration_id ON "networks_peers" (network_id, target_network_integration_id); ` _, err := tx.Exec(q) if err != nil { return fmt.Errorf("Failed adding network integration support: %w", err) } return nil } func updateFromV70(ctx context.Context, tx *sql.Tx) error { q := ` CREATE TABLE "storage_buckets_backups" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_bucket_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, creation_date DATETIME, expiry_date DATETIME, FOREIGN KEY (storage_bucket_id) REFERENCES "storage_buckets" (id) ON DELETE CASCADE, UNIQUE (storage_bucket_id, name) ); ` _, err := tx.Exec(q) if err != nil { return fmt.Errorf("Failed adding storage bucket backup table: %w", err) } return nil } // updateFromV69 adds description column to certificate. func updateFromV69(ctx context.Context, tx *sql.Tx) error { q := ` ALTER TABLE certificates ADD COLUMN description TEXT NOT NULL DEFAULT ""; ` _, err := tx.Exec(q) if err != nil { return fmt.Errorf("Failed adding description column to certificate: %w", err) } return nil } // updateFromV68 fixes unique index for record name to make it zone specific. func updateFromV68(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE networks_zones_records_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_zone_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, entries TEXT NOT NULL, UNIQUE (network_zone_id, name), FOREIGN KEY (network_zone_id) REFERENCES networks_zones (id) ON DELETE CASCADE ); CREATE TABLE networks_zones_records_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_zone_record_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_zone_record_id, key), FOREIGN KEY (network_zone_record_id) REFERENCES networks_zones_records_new (id) ON DELETE CASCADE ); INSERT INTO "networks_zones_records_new" SELECT * FROM "networks_zones_records"; INSERT INTO "networks_zones_records_config_new" SELECT * FROM "networks_zones_records_config"; DROP TABLE "networks_zones_records"; ALTER TABLE "networks_zones_records_new" RENAME TO "networks_zones_records"; DROP TABLE "networks_zones_records_config"; ALTER TABLE "networks_zones_records_config_new" RENAME TO "networks_zones_records_config"; `) if err != nil { return fmt.Errorf("Failed altering network_zones_records schema: %w", err) } return nil } // updateFromV67 adds features.networks.zones=true to any project that has features.networks=true. func updateFromV67(ctx context.Context, tx *sql.Tx) error { // Find projects that have features.networks=true. rows, err := tx.QueryContext(ctx, ` SELECT projects.id FROM projects JOIN projects_config ON projects_config.project_id = projects.id AND projects_config.key = "features.networks" AND projects_config.value = "true" `) if err != nil { return fmt.Errorf("Failed finding projects with features.networks=true: %w", err) } defer func() { _ = rows.Close() }() var projectIDs []int64 for rows.Next() { var projectID int64 err := rows.Scan(&projectID) if err != nil { return fmt.Errorf("Failed scanning project ID row: %w", err) } projectIDs = append(projectIDs, projectID) } _ = rows.Close() // Add features.networks.zones=true to any project that has features.networks=true. for _, projectID := range projectIDs { _, err = tx.Exec(`INSERT OR REPLACE INTO projects_config (project_id,key,value) VALUES(?,?,?);`, projectID, "features.networks.zones", "true") if err != nil { return fmt.Errorf("Failed adding features.networks.zones=true to project ID %d: %w", projectID, err) } logger.Info("Added features.networks.zones=true on project with features.networks=true", logger.Ctx{"projectID": projectID}) } return nil } // updateFromV66 adds creation_date column to storage_volumes and storage_volumes_snapshots tables. func updateFromV66(ctx context.Context, tx *sql.Tx) error { q := ` ALTER TABLE storage_volumes ADD COLUMN creation_date DATETIME NOT NULL DEFAULT "0001-01-01T00:00:00Z"; ALTER TABLE storage_volumes_snapshots ADD COLUMN creation_date DATETIME NOT NULL DEFAULT "0001-01-01T00:00:00Z"; DROP VIEW storage_volumes_all; CREATE VIEW storage_volumes_all ( id, name, storage_pool_id, node_id, type, description, project_id, content_type, creation_date) AS SELECT id, name, storage_pool_id, node_id, type, description, project_id, content_type, creation_date FROM storage_volumes UNION SELECT storage_volumes_snapshots.id, printf('%s/%s', storage_volumes.name, storage_volumes_snapshots.name), storage_volumes.storage_pool_id, storage_volumes.node_id, storage_volumes.type, storage_volumes_snapshots.description, storage_volumes.project_id, storage_volumes.content_type, storage_volumes_snapshots.creation_date FROM storage_volumes JOIN storage_volumes_snapshots ON storage_volumes.id = storage_volumes_snapshots.storage_volume_id; ` _, err := tx.Exec(q) if err != nil { return fmt.Errorf("Failed adding creation_date column to storage volumes: %w", err) } return nil } // updateFromV65 fixes typo in cephobject.radosgw.endpoint* settings. func updateFromV65(ctx context.Context, tx *sql.Tx) error { q := ` UPDATE storage_pools_config SET key = REPLACE(key, "cephobject.radosgsw.endpoint", "cephobject.radosgw.endpoint") WHERE key IN ("cephobject.radosgsw.endpoint", "cephobject.radosgsw.endpoint_cert_file") ` _, err := tx.Exec(q) if err != nil { return fmt.Errorf("Failed replacing storage pool config cephobject.radosgsw.endpoint* with cephobject.radosgw.endpoint*: %w", err) } return nil } // updatefromV64 updates nodes_cluster_groups to include an ID field so that it works well with generate-database. func updateFromV64(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE "nodes_cluster_groups_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, node_id INTEGER NOT NULL, group_id INTEGER NOT NULL, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE, FOREIGN KEY (group_id) REFERENCES cluster_groups (id) ON DELETE CASCADE, UNIQUE (node_id, group_id) ); INSERT INTO nodes_cluster_groups_new (node_id, group_id) SELECT node_id, group_id FROM nodes_cluster_groups; DROP TABLE nodes_cluster_groups; ALTER TABLE nodes_cluster_groups_new RENAME TO nodes_cluster_groups; `) if err != nil { return fmt.Errorf("Failed altering nodes_cluster_groups table: %w", err) } return nil } // updateFromV63 creates the storage buckets tables and adds features.storage.buckets=true to all projects that // have features.storage.volumes=true. func updateFromV63(ctx context.Context, tx *sql.Tx) error { // Find all projects that have features.storage.volumes=true and add features.storage.buckets=true. rows, err := tx.QueryContext(ctx, `SELECT project_id FROM projects_config WHERE key = "features.storage.volumes" AND value = "true"`) if err != nil { return fmt.Errorf("Failed getting projects with features.storage.volumes=true: %w", err) } defer func() { _ = rows.Close() }() var projectIDs []int64 for rows.Next() { var projectID int64 err = rows.Scan(&projectID) if err != nil { return fmt.Errorf("Failed scanning project ID row: %w", err) } projectIDs = append(projectIDs, projectID) } err = rows.Err() if err != nil { return fmt.Errorf("Got a row error getting projects with features.storage.volumes=true: %w", err) } for _, projectID := range projectIDs { _, err = tx.Exec(`INSERT OR REPLACE INTO projects_config (project_id,key,value) VALUES(?,?,?);`, projectID, "features.storage.buckets", "true") if err != nil { return fmt.Errorf("Failed adding features.storage.buckets=true to projects: %w", err) } } // Create storage buckets tables. _, err = tx.Exec(` CREATE TABLE IF NOT EXISTS "storage_buckets" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER, description TEXT NOT NULL, project_id INTEGER NOT NULL, UNIQUE (node_id, name), FOREIGN KEY (storage_pool_id) REFERENCES "storage_pools" (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE ); CREATE UNIQUE INDEX storage_buckets_unique_storage_pool_id_node_id_name ON "storage_buckets" (storage_pool_id, IFNULL(node_id, -1), name); CREATE TABLE "storage_buckets_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_bucket_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (storage_bucket_id, key), FOREIGN KEY (storage_bucket_id) REFERENCES "storage_buckets" (id) ON DELETE CASCADE ); CREATE TABLE "storage_buckets_keys" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_bucket_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, access_key TEXT NOT NULL, secret_key TEXT NOT NULL, role TEXT NOT NULL, UNIQUE (storage_bucket_id, name), FOREIGN KEY (storage_bucket_id) REFERENCES "storage_buckets" (id) ON DELETE CASCADE ); `) if err != nil { return fmt.Errorf("Failed adding storage bucket tables: %w", err) } return nil } // updateFromV62 adds unique index to storage_volumes that prevents duplicate volumes when using remote storage // pool where the node_id column is NULL. // Also ensures that the default project has features.networks set to true. func updateFromV62(ctx context.Context, tx *sql.Tx) error { // Find the default project ID, and what it has features.networks config key set to (if at all). rows := tx.QueryRowContext(ctx, ` SELECT projects.id, IFNULL(projects_config.key, "") as key, IFNULL(projects_config.value, "") as value FROM projects LEFT JOIN projects_config ON projects_config.project_id = projects.id AND projects_config.key = "features.networks" WHERE projects.name = "default" `) var defaultProjectID int64 var featureKey, featureValue string err := rows.Scan(&defaultProjectID, &featureKey, &featureValue) if err != nil { return fmt.Errorf("Failed scanning default project row: %w", err) } // If the features.networks key is missing or not set to true, insert/replace the correct row. if featureKey == "" || featureValue != "true" { _, err = tx.Exec(`INSERT OR REPLACE INTO projects_config (project_id,key,value) VALUES(?,?,?);`, defaultProjectID, "features.networks", "true") if err != nil { return fmt.Errorf("Failed adding features.networks=true to default project: %w", err) } } // Create unique index on storage_volumes that protects against duplicate volumes when using remote // storage pool where the node_id field is NULL (which the current unique index doesn't protect against). _, err = tx.Exec(`CREATE UNIQUE INDEX storage_volumes_unique_storage_pool_id_node_id_project_id_name_type ON "storage_volumes" (storage_pool_id, IFNULL(node_id, -1), project_id, name, type);`) if err != nil { return fmt.Errorf("Failed adding storage volumes unique index: %w", err) } return nil } // updateFromV61 converts config value fields to NOT NULL and config key fields to TEXT (from VARCHAR). func updateFromV61(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE "instances_config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, instance_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (instance_id) REFERENCES "instances" (id) ON DELETE CASCADE, UNIQUE (instance_id, key) ); INSERT INTO "instances_config_new" SELECT * FROM "instances_config"; DROP TABLE "instances_config"; ALTER TABLE "instances_config_new" RENAME TO "instances_config"; CREATE TABLE "instances_devices_config_new" ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (instance_device_id) REFERENCES "instances_devices" (id) ON DELETE CASCADE, UNIQUE (instance_device_id, key) ); INSERT INTO "instances_devices_config_new" SELECT * FROM "instances_devices_config"; DROP TABLE "instances_devices_config"; ALTER TABLE "instances_devices_config_new" RENAME TO "instances_devices_config"; CREATE TABLE "instances_snapshots_config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, instance_snapshot_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (instance_snapshot_id) REFERENCES "instances_snapshots" (id) ON DELETE CASCADE, UNIQUE (instance_snapshot_id, key) ); INSERT INTO "instances_snapshots_config_new" SELECT * FROM "instances_snapshots_config"; DROP TABLE "instances_snapshots_config"; ALTER TABLE "instances_snapshots_config_new" RENAME TO "instances_snapshots_config"; CREATE TABLE "instances_snapshots_devices_config_new" ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_snapshot_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (instance_snapshot_device_id) REFERENCES "instances_snapshots_devices" (id) ON DELETE CASCADE, UNIQUE (instance_snapshot_device_id, key) ); INSERT INTO "instances_snapshots_devices_config_new" SELECT * FROM "instances_snapshots_devices_config"; DROP TABLE "instances_snapshots_devices_config"; ALTER TABLE "instances_snapshots_devices_config_new" RENAME TO "instances_snapshots_devices_config"; CREATE TABLE "networks_acls_config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_acl_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_acl_id, key), FOREIGN KEY (network_acl_id) REFERENCES "networks_acls" (id) ON DELETE CASCADE ); INSERT INTO "networks_acls_config_new" SELECT * FROM "networks_acls_config"; DROP TABLE "networks_acls_config"; ALTER TABLE "networks_acls_config_new" RENAME TO "networks_acls_config"; CREATE TABLE "networks_config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_id, node_id, key), FOREIGN KEY (network_id) REFERENCES "networks" (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE ); INSERT INTO "networks_config_new" SELECT * FROM "networks_config"; DROP TABLE "networks_config"; ALTER TABLE "networks_config_new" RENAME TO "networks_config"; CREATE UNIQUE INDEX networks_unique_network_id_node_id_key ON "networks_config" (network_id, IFNULL(node_id, -1), key); CREATE TABLE "networks_forwards_config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_forward_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_forward_id, key), FOREIGN KEY (network_forward_id) REFERENCES "networks_forwards" (id) ON DELETE CASCADE ); INSERT INTO "networks_forwards_config_new" SELECT * FROM "networks_forwards_config"; DROP TABLE "networks_forwards_config"; ALTER TABLE "networks_forwards_config_new" RENAME TO "networks_forwards_config"; CREATE TABLE "networks_peers_config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_peer_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_peer_id, key), FOREIGN KEY (network_peer_id) REFERENCES "networks_peers" (id) ON DELETE CASCADE ); INSERT INTO "networks_peers_config_new" SELECT * FROM "networks_peers_config"; DROP TABLE "networks_peers_config"; ALTER TABLE "networks_peers_config_new" RENAME TO "networks_peers_config"; CREATE TABLE "networks_zones_config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_zone_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_zone_id, key), FOREIGN KEY (network_zone_id) REFERENCES "networks_zones" (id) ON DELETE CASCADE ); INSERT INTO "networks_zones_config_new" SELECT * FROM "networks_zones_config"; DROP TABLE "networks_zones_config"; ALTER TABLE "networks_zones_config_new" RENAME TO "networks_zones_config"; CREATE TABLE networks_zones_records_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_zone_record_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_zone_record_id, key), FOREIGN KEY (network_zone_record_id) REFERENCES networks_zones_records (id) ON DELETE CASCADE ); INSERT INTO "networks_zones_records_config_new" SELECT * FROM "networks_zones_records_config"; DROP TABLE "networks_zones_records_config"; ALTER TABLE "networks_zones_records_config_new" RENAME TO "networks_zones_records_config"; CREATE TABLE "nodes_config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, node_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE, UNIQUE (node_id, key) ); INSERT INTO "nodes_config_new" SELECT * FROM "nodes_config"; DROP TABLE "nodes_config"; ALTER TABLE "nodes_config_new" RENAME TO "nodes_config"; CREATE TABLE "profiles_config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (profile_id, key), FOREIGN KEY (profile_id) REFERENCES "profiles"(id) ON DELETE CASCADE ); INSERT INTO "profiles_config_new" SELECT * FROM "profiles_config"; DROP TABLE "profiles_config"; ALTER TABLE "profiles_config_new" RENAME TO "profiles_config"; CREATE TABLE "profiles_devices_config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (profile_device_id, key), FOREIGN KEY (profile_device_id) REFERENCES "profiles_devices" (id) ON DELETE CASCADE ); INSERT INTO "profiles_devices_config_new" SELECT * FROM "profiles_devices_config"; DROP TABLE "profiles_devices_config"; ALTER TABLE "profiles_devices_config_new" RENAME TO "profiles_devices_config"; CREATE TABLE "projects_config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE, UNIQUE (project_id, key) ); INSERT INTO "projects_config_new" SELECT * FROM "projects_config"; DROP TABLE "projects_config"; ALTER TABLE "projects_config_new" RENAME TO "projects_config"; CREATE TABLE "storage_pools_config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (storage_pool_id, node_id, key), FOREIGN KEY (storage_pool_id) REFERENCES "storage_pools" (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE ); INSERT INTO "storage_pools_config_new" SELECT * FROM "storage_pools_config"; DROP TABLE "storage_pools_config"; ALTER TABLE "storage_pools_config_new" RENAME TO "storage_pools_config"; CREATE UNIQUE INDEX storage_pools_unique_storage_pool_id_node_id_key ON storage_pools_config (storage_pool_id, IFNULL(node_id, -1), key); CREATE TABLE "storage_volumes_config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (storage_volume_id, key), FOREIGN KEY (storage_volume_id) REFERENCES "storage_volumes" (id) ON DELETE CASCADE ); INSERT INTO "storage_volumes_config_new" SELECT * FROM "storage_volumes_config"; DROP TABLE "storage_volumes_config"; ALTER TABLE "storage_volumes_config_new" RENAME TO "storage_volumes_config"; CREATE TABLE "storage_volumes_snapshots_config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_snapshot_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, FOREIGN KEY (storage_volume_snapshot_id) REFERENCES "storage_volumes_snapshots" (id) ON DELETE CASCADE, UNIQUE (storage_volume_snapshot_id, key) ); INSERT INTO "storage_volumes_snapshots_config_new" SELECT * FROM "storage_volumes_snapshots_config"; DROP TABLE "storage_volumes_snapshots_config"; ALTER TABLE "storage_volumes_snapshots_config_new" RENAME TO "storage_volumes_snapshots_config"; `) if err != nil { return fmt.Errorf("Failed altering config tables schema: %w", err) } return nil } // updateFromV60 creates the networks_load_balancers and networks_load_balancers_config tables. func updateFromV60(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE "networks_load_balancers" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER, listen_address TEXT NOT NULL, description TEXT NOT NULL, backends TEXT NOT NULL, ports TEXT NOT NULL, UNIQUE (network_id, node_id, listen_address), FOREIGN KEY (network_id) REFERENCES "networks" (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE ); CREATE TABLE "networks_load_balancers_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_load_balancer_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (network_load_balancer_id, key), FOREIGN KEY (network_load_balancer_id) REFERENCES "networks_load_balancers" (id) ON DELETE CASCADE ); `) if err != nil { return fmt.Errorf("Failed creating network load balancers tables: %w", err) } return nil } func updateFromV59(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE networks_zones_records ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_zone_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, entries TEXT NOT NULL, UNIQUE (name), FOREIGN KEY (network_zone_id) REFERENCES networks_zones (id) ON DELETE CASCADE ); CREATE TABLE networks_zones_records_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_zone_record_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (network_zone_record_id, key), FOREIGN KEY (network_zone_record_id) REFERENCES networks_zones_records (id) ON DELETE CASCADE ); `) if err != nil { return fmt.Errorf("Failed creating network zone records tables: %w", err) } return nil } func updateFromV58(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` UPDATE sqlite_sequence SET seq = ( SELECT max( (SELECT coalesce(max(storage_volumes.id), 0) FROM storage_volumes), (SELECT coalesce(max(storage_volumes_snapshots.id), 0) FROM storage_volumes_snapshots))) WHERE name='storage_volumes'; `) return err } func updateFromV57(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` UPDATE sqlite_sequence SET seq = ( SELECT coalesce(max(max(coalesce(storage_volumes.id, 0)), max(coalesce(storage_volumes_snapshots.id, 0))), 0) FROM storage_volumes, storage_volumes_snapshots) WHERE name='storage_volumes'; `) return err } func updateFromV56(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` UPDATE sqlite_sequence SET seq = ( SELECT max(max(coalesce(storage_volumes.id, 0)), max(coalesce(storage_volumes_snapshots.id, 0))) FROM storage_volumes, storage_volumes_snapshots) WHERE name='storage_volumes'; `) return err } func updateFromV55(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` DROP VIEW storage_volumes_all; CREATE TABLE projects_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, UNIQUE (name) ); INSERT INTO projects_new (id, name, description) SELECT id, name, IFNULL(description, '') FROM projects; CREATE TABLE certificates_projects_new ( certificate_id INTEGER NOT NULL, project_id INTEGER NOT NULL, FOREIGN KEY (certificate_id) REFERENCES certificates (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES projects_new (id) ON DELETE CASCADE, UNIQUE (certificate_id, project_id) ); INSERT INTO certificates_projects_new (certificate_id, project_id) SELECT certificate_id, project_id FROM certificates_projects; CREATE TABLE images_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, fingerprint TEXT NOT NULL, filename TEXT NOT NULL, size INTEGER NOT NULL, public INTEGER NOT NULL DEFAULT 0, architecture INTEGER NOT NULL, creation_date DATETIME, expiry_date DATETIME, upload_date DATETIME NOT NULL, cached INTEGER NOT NULL DEFAULT 0, last_use_date DATETIME, auto_update INTEGER NOT NULL DEFAULT 0, project_id INTEGER NOT NULL, type INTEGER NOT NULL DEFAULT 0, UNIQUE (project_id, fingerprint), FOREIGN KEY (project_id) REFERENCES projects_new (id) ON DELETE CASCADE ); INSERT INTO images_new (id, fingerprint, filename, size, public, architecture, creation_date, expiry_date, upload_date, cached, last_use_date, auto_update, project_id, type) SELECT id, fingerprint, filename, size, public, architecture, creation_date, expiry_date, upload_date, cached, last_use_date, auto_update, project_id, type FROM images; CREATE TABLE images_aliases_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, image_id INTEGER NOT NULL, description TEXT NOT NULL, project_id INTEGER NOT NULL, UNIQUE (project_id, name), FOREIGN KEY (image_id) REFERENCES images_new (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES projects_new (id) ON DELETE CASCADE ); INSERT INTO images_aliases_new (id, name, image_id, description, project_id) SELECT id, name, image_id, IFNULL(description, ''), project_id FROM images_aliases; CREATE TABLE nodes_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, address TEXT NOT NULL, schema INTEGER NOT NULL, api_extensions INTEGER NOT NULL, heartbeat DATETIME DEFAULT CURRENT_TIMESTAMP, state INTEGER NOT NULL DEFAULT 0, arch INTEGER NOT NULL DEFAULT 0 CHECK (arch > 0), failure_domain_id INTEGER DEFAULT NULL REFERENCES nodes_failure_domains (id) ON DELETE SET NULL, UNIQUE (name), UNIQUE (address) ); INSERT INTO nodes_new (id, name, description, address, schema, api_extensions, heartbeat, state, arch, failure_domain_id) SELECT id, name, IFNULL(description, ''), address, schema, api_extensions, heartbeat, state, arch, failure_domain_id FROM nodes; CREATE TABLE images_nodes_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, node_id INTEGER NOT NULL, UNIQUE (image_id, node_id), FOREIGN KEY (image_id) REFERENCES images_new (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes_new (id) ON DELETE CASCADE ); INSERT INTO images_nodes_new (id, image_id, node_id) SELECT id, image_id, node_id FROM images_nodes; CREATE TABLE profiles_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, project_id INTEGER NOT NULL, UNIQUE (project_id, name), FOREIGN KEY (project_id) REFERENCES projects_new (id) ON DELETE CASCADE ); INSERT INTO profiles_new (id, name, description, project_id) SELECT id, name, IFNULL(description, ''), project_id FROM profiles; CREATE TABLE images_profiles_new ( image_id INTEGER NOT NULL, profile_id INTEGER NOT NULL, FOREIGN KEY (image_id) REFERENCES images_new (id) ON DELETE CASCADE, FOREIGN KEY (profile_id) REFERENCES profiles_new (id) ON DELETE CASCADE, UNIQUE (image_id, profile_id) ); INSERT INTO images_profiles_new (image_id, profile_id) SELECT image_id, profile_id FROM images_profiles; CREATE TABLE images_properties_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, type INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (image_id) REFERENCES images_new (id) ON DELETE CASCADE ); INSERT INTO images_properties_new (id, image_id, type, key, value) SELECT id, image_id, type, key, value FROM images_properties; CREATE TABLE images_source_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, server TEXT NOT NULL, protocol INTEGER NOT NULL, certificate TEXT NOT NULL, alias TEXT NOT NULL, FOREIGN KEY (image_id) REFERENCES images_new (id) ON DELETE CASCADE ); INSERT INTO images_source_new (id, image_id, server, protocol, certificate, alias) SELECT id, image_id, server, protocol, certificate, alias FROM images_source; CREATE TABLE instances_new ( id INTEGER primary key AUTOINCREMENT NOT NULL, node_id INTEGER NOT NULL, name TEXT NOT NULL, architecture INTEGER NOT NULL, type INTEGER NOT NULL, ephemeral INTEGER NOT NULL DEFAULT 0, creation_date DATETIME NOT NULL DEFAULT 0, stateful INTEGER NOT NULL DEFAULT 0, last_use_date DATETIME, description TEXT NOT NULL, project_id INTEGER NOT NULL, expiry_date DATETIME, UNIQUE (project_id, name), FOREIGN KEY (node_id) REFERENCES nodes_new (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES projects_new (id) ON DELETE CASCADE ); INSERT INTO instances_new (id, node_id, name, architecture, type, ephemeral, creation_date, stateful, last_use_date, description, project_id, expiry_date) SELECT id, node_id, name, architecture, type, ephemeral, creation_date, stateful, last_use_date, IFNULL(description, ''), project_id, expiry_date FROM instances; CREATE TABLE instances_backups_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, instance_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, creation_date DATETIME, expiry_date DATETIME, container_only INTEGER NOT NULL default 0, optimized_storage INTEGER NOT NULL default 0, FOREIGN KEY (instance_id) REFERENCES instances_new (id) ON DELETE CASCADE, UNIQUE (instance_id, name) ); INSERT INTO instances_backups_new (id, instance_id, name, creation_date, expiry_date, container_only, optimized_storage) SELECT id, instance_id, name, creation_date, expiry_date, container_only, optimized_storage FROM instances_backups; CREATE TABLE instances_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, instance_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (instance_id) REFERENCES instances_new (id) ON DELETE CASCADE, UNIQUE (instance_id, key) ); INSERT INTO instances_config_new (id, instance_id, key, value) SELECT id, instance_id, key, value FROM instances_config; CREATE TABLE instances_devices_new ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_id INTEGER NOT NULL, name TEXT NOT NULL, type INTEGER NOT NULL default 0, FOREIGN KEY (instance_id) REFERENCES instances_new (id) ON DELETE CASCADE, UNIQUE (instance_id, name) ); INSERT INTO instances_devices_new (id, instance_id, name, type) SELECT id, instance_id, name, type FROM instances_devices; CREATE TABLE instances_devices_config_new ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (instance_device_id) REFERENCES instances_devices_new (id) ON DELETE CASCADE, UNIQUE (instance_device_id, key) ); INSERT INTO instances_devices_config_new (id, instance_device_id, key, value) SELECT id, instance_device_id, key, value FROM instances_devices_config; CREATE TABLE instances_profiles_new ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_id INTEGER NOT NULL, profile_id INTEGER NOT NULL, apply_order INTEGER NOT NULL default 0, UNIQUE (instance_id, profile_id), FOREIGN KEY (instance_id) REFERENCES instances_new (id) ON DELETE CASCADE, FOREIGN KEY (profile_id) REFERENCES profiles_new(id) ON DELETE CASCADE ); INSERT INTO instances_profiles_new (id, instance_id, profile_id, apply_order) SELECT id, instance_id, profile_id, apply_order FROM instances_profiles; CREATE TABLE instances_snapshots_new ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_id INTEGER NOT NULL, name TEXT NOT NULL, creation_date DATETIME NOT NULL DEFAULT 0, stateful INTEGER NOT NULL DEFAULT 0, description TEXT NOT NULL, expiry_date DATETIME, UNIQUE (instance_id, name), FOREIGN KEY (instance_id) REFERENCES instances_new (id) ON DELETE CASCADE ); INSERT INTO instances_snapshots_new (id, instance_id, name, creation_date, stateful, description, expiry_date) SELECT id, instance_id, name, creation_date, stateful, IFNULL(description, ''), expiry_date FROM instances_snapshots; CREATE TABLE instances_snapshots_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, instance_snapshot_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (instance_snapshot_id) REFERENCES instances_snapshots_new (id) ON DELETE CASCADE, UNIQUE (instance_snapshot_id, key) ); INSERT INTO instances_snapshots_config_new (id, instance_snapshot_id, key, value) SELECT id, instance_snapshot_id, key, value FROM instances_snapshots_config; CREATE TABLE instances_snapshots_devices_new ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_snapshot_id INTEGER NOT NULL, name TEXT NOT NULL, type INTEGER NOT NULL default 0, FOREIGN KEY (instance_snapshot_id) REFERENCES instances_snapshots_new (id) ON DELETE CASCADE, UNIQUE (instance_snapshot_id, name) ); INSERT INTO instances_snapshots_devices_new (id, instance_snapshot_id, name, type) SELECT id, instance_snapshot_id, name, type FROM instances_snapshots_devices; CREATE TABLE instances_snapshots_devices_config_new ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_snapshot_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (instance_snapshot_device_id) REFERENCES instances_snapshots_devices_new (id) ON DELETE CASCADE, UNIQUE (instance_snapshot_device_id, key) ); INSERT INTO instances_snapshots_devices_config_new (id, instance_snapshot_device_id, key, value) SELECT id, instance_snapshot_device_id, key, value FROM instances_snapshots_devices_config; CREATE TABLE networks_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, state INTEGER NOT NULL DEFAULT 0, type INTEGER NOT NULL DEFAULT 0, UNIQUE (project_id, name), FOREIGN KEY (project_id) REFERENCES projects_new (id) ON DELETE CASCADE ); INSERT INTO networks_new (id, project_id, name, description, state, type) SELECT id, project_id, name, IFNULL(description, ''), state, type FROM networks; CREATE TABLE networks_acls_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, ingress TEXT NOT NULL, egress TEXT NOT NULL, UNIQUE (project_id, name), FOREIGN KEY (project_id) REFERENCES projects_new (id) ON DELETE CASCADE ); INSERT INTO networks_acls_new (id, project_id, name, description, ingress, egress) SELECT id, project_id, name, IFNULL(description, ''), ingress, egress FROM networks_acls; CREATE TABLE networks_acls_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_acl_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (network_acl_id, key), FOREIGN KEY (network_acl_id) REFERENCES networks_acls_new (id) ON DELETE CASCADE ); CREATE TABLE networks_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER, key TEXT NOT NULL, value TEXT, UNIQUE (network_id, node_id, key), FOREIGN KEY (network_id) REFERENCES networks_new (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes_new (id) ON DELETE CASCADE ); INSERT INTO networks_config_new (id, network_id, node_id, key, value) SELECT id, network_id, node_id, key, value FROM networks_config; CREATE TABLE networks_forwards_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER, listen_address TEXT NOT NULL, description TEXT NOT NULL, ports TEXT NOT NULL, UNIQUE (network_id, node_id, listen_address), FOREIGN KEY (network_id) REFERENCES networks_new (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes_new (id) ON DELETE CASCADE ); INSERT INTO networks_forwards_new (id, network_id, node_id, listen_address, description, ports) SELECT id, network_id, node_id, listen_address, IFNULL(description, ''), ports FROM networks_forwards; CREATE TABLE networks_forwards_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_forward_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (network_forward_id, key), FOREIGN KEY (network_forward_id) REFERENCES networks_forwards_new (id) ON DELETE CASCADE ); INSERT INTO networks_forwards_config_new (id, network_forward_id, key, value) SELECT id, network_forward_id, key, value FROM networks_forwards_config; CREATE TABLE networks_nodes_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER NOT NULL, state INTEGER NOT NULL DEFAULT 0, UNIQUE (network_id, node_id), FOREIGN KEY (network_id) REFERENCES networks_new (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes_new (id) ON DELETE CASCADE ); INSERT INTO networks_nodes_new (id, network_id, node_id, state) SELECT id, network_id, node_id, state FROM networks_nodes; CREATE TABLE networks_peers_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, target_network_project TEXT NULL, target_network_name TEXT NULL, target_network_id INTEGER NULL, UNIQUE (network_id, name), UNIQUE (network_id, target_network_project, target_network_name), UNIQUE (network_id, target_network_id), FOREIGN KEY (network_id) REFERENCES networks_new (id) ON DELETE CASCADE ); INSERT INTO networks_peers_new (id, network_id, name, description, target_network_project, target_network_name, target_network_id) SELECT id, network_id, name, IFNULL(description, ''), target_network_project, target_network_name, target_network_id FROM networks_peers; CREATE TABLE networks_peers_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_peer_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (network_peer_id, key), FOREIGN KEY (network_peer_id) REFERENCES networks_peers_new (id) ON DELETE CASCADE ); INSERT INTO networks_peers_config_new (id, network_peer_id, key, value) SELECT id, network_peer_id, key, value FROM networks_peers_config; CREATE TABLE networks_zones_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, UNIQUE (name), FOREIGN KEY (project_id) REFERENCES projects_new (id) ON DELETE CASCADE ); INSERT INTO networks_zones_new (id, project_id, name, description) SELECT id, project_id, name, IFNULL(description, '') FROM networks_zones; CREATE TABLE networks_zones_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_zone_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (network_zone_id, key), FOREIGN KEY (network_zone_id) REFERENCES networks_zones_new (id) ON DELETE CASCADE ); INSERT INTO networks_zones_config_new (id, network_zone_id, key, value) SELECT id, network_zone_id, key, value FROM networks_zones_config; CREATE TABLE nodes_cluster_groups_new ( node_id INTEGER NOT NULL, group_id INTEGER NOT NULL, FOREIGN KEY (node_id) REFERENCES nodes_new (id) ON DELETE CASCADE, FOREIGN KEY (group_id) REFERENCES cluster_groups (id) ON DELETE CASCADE, UNIQUE (node_id, group_id) ); INSERT INTO nodes_cluster_groups_new (node_id, group_id) SELECT node_id, group_id FROM nodes_cluster_groups; CREATE TABLE nodes_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, node_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (node_id) REFERENCES nodes_new (id) ON DELETE CASCADE, UNIQUE (node_id, key) ); INSERT INTO nodes_config_new (id, node_id, key, value) SELECT id, node_id, key, value FROM nodes_config; CREATE TABLE nodes_roles_new ( node_id INTEGER NOT NULL, role INTEGER NOT NULL, FOREIGN KEY (node_id) REFERENCES nodes_new (id) ON DELETE CASCADE, UNIQUE (node_id, role) ); INSERT INTO nodes_roles_new (node_id, role) SELECT node_id, role FROM nodes_roles; CREATE TABLE operations_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, uuid TEXT NOT NULL, node_id TEXT NOT NULL, type INTEGER NOT NULL DEFAULT 0, project_id INTEGER, UNIQUE (uuid), FOREIGN KEY (node_id) REFERENCES nodes_new (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES projects_new (id) ON DELETE CASCADE ); INSERT INTO operations_new (id, uuid, node_id, type, project_id) SELECT id, uuid, node_id, type, project_id FROM operations; CREATE TABLE profiles_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (profile_id, key), FOREIGN KEY (profile_id) REFERENCES profiles_new(id) ON DELETE CASCADE ); INSERT INTO profiles_config_new (id, profile_id, key, value) SELECT id, profile_id, key, value FROM profiles_config; CREATE TABLE profiles_devices_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_id INTEGER NOT NULL, name TEXT NOT NULL, type INTEGER NOT NULL default 0, UNIQUE (profile_id, name), FOREIGN KEY (profile_id) REFERENCES profiles_new (id) ON DELETE CASCADE ); INSERT INTO profiles_devices_new (id, profile_id, name, type) SELECT id, profile_id, name, type FROM profiles_devices; CREATE TABLE profiles_devices_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (profile_device_id, key), FOREIGN KEY (profile_device_id) REFERENCES profiles_devices_new (id) ON DELETE CASCADE ); INSERT INTO profiles_devices_config_new (id, profile_device_id, key, value) SELECT id, profile_device_id, key, value FROM profiles_devices_config; CREATE TABLE projects_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (project_id) REFERENCES projects_new (id) ON DELETE CASCADE, UNIQUE (project_id, key) ); INSERT INTO projects_config_new (id, project_id, key, value) SELECT id, project_id, key, value FROM projects_config; CREATE TABLE storage_pools_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, driver TEXT NOT NULL, description TEXT NOT NULL, state INTEGER NOT NULL DEFAULT 0, UNIQUE (name) ); INSERT INTO storage_pools_new (id, name, driver, description, state) SELECT id, name, driver, IFNULL(description, ''), state FROM storage_pools; CREATE TABLE storage_pools_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER, key TEXT NOT NULL, value TEXT, UNIQUE (storage_pool_id, node_id, key), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools_new (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes_new (id) ON DELETE CASCADE ); INSERT INTO storage_pools_config_new (id, storage_pool_id, node_id, key, value) SELECT id, storage_pool_id, node_id, key, value FROM storage_pools_config; CREATE TABLE storage_pools_nodes_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER NOT NULL, state INTEGER NOT NULL DEFAULT 0, UNIQUE (storage_pool_id, node_id), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools_new (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes_new (id) ON DELETE CASCADE ); INSERT INTO storage_pools_nodes_new (id, storage_pool_id, node_id, state) SELECT id, storage_pool_id, node_id, state FROM storage_pools_nodes; CREATE TABLE storage_volumes_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER, type INTEGER NOT NULL, description TEXT NOT NULL, project_id INTEGER NOT NULL, content_type INTEGER NOT NULL DEFAULT 0, UNIQUE (storage_pool_id, node_id, project_id, name, type), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools_new (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes_new (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES projects_new (id) ON DELETE CASCADE ); INSERT INTO storage_volumes_new (id, name, storage_pool_id, node_id, type, description, project_id, content_type) SELECT id, name, storage_pool_id, node_id, type, IFNULL(description, ''), project_id, content_type FROM storage_volumes; CREATE TABLE storage_volumes_backups_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, creation_date DATETIME, expiry_date DATETIME, volume_only INTEGER NOT NULL default 0, optimized_storage INTEGER NOT NULL default 0, FOREIGN KEY (storage_volume_id) REFERENCES storage_volumes_new (id) ON DELETE CASCADE, UNIQUE (storage_volume_id, name) ); INSERT INTO storage_volumes_backups_new (id, storage_volume_id, name, creation_date, expiry_date, volume_only, optimized_storage) SELECT id, storage_volume_id, name, creation_date, expiry_date, volume_only, optimized_storage FROM storage_volumes_backups; CREATE TABLE storage_volumes_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (storage_volume_id, key), FOREIGN KEY (storage_volume_id) REFERENCES storage_volumes_new (id) ON DELETE CASCADE ); INSERT INTO storage_volumes_config_new (id, storage_volume_id, key, value) SELECT id, storage_volume_id, key, value FROM storage_volumes_config; CREATE TABLE storage_volumes_snapshots_new ( id INTEGER NOT NULL, storage_volume_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, expiry_date DATETIME, UNIQUE (id), UNIQUE (storage_volume_id, name), FOREIGN KEY (storage_volume_id) REFERENCES storage_volumes_new (id) ON DELETE CASCADE ); INSERT INTO storage_volumes_snapshots_new (id, storage_volume_id, name, description, expiry_date) SELECT id, storage_volume_id, name, IFNULL(description, ''), expiry_date FROM storage_volumes_snapshots; CREATE TABLE storage_volumes_snapshots_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_snapshot_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (storage_volume_snapshot_id) REFERENCES storage_volumes_snapshots_new (id) ON DELETE CASCADE, UNIQUE (storage_volume_snapshot_id, key) ); INSERT INTO storage_volumes_snapshots_config_new (id, storage_volume_snapshot_id, key, value) SELECT id, storage_volume_snapshot_id, key, value FROM storage_volumes_snapshots_config; CREATE TABLE warnings_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, node_id INTEGER, project_id INTEGER, entity_type_code INTEGER, entity_id INTEGER, uuid TEXT NOT NULL, type_code INTEGER NOT NULL, status INTEGER NOT NULL, first_seen_date DATETIME NOT NULL, last_seen_date DATETIME NOT NULL, updated_date DATETIME, last_message TEXT NOT NULL, count INTEGER NOT NULL, UNIQUE (uuid), FOREIGN KEY (node_id) REFERENCES nodes_new(id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES projects_new (id) ON DELETE CASCADE ); INSERT INTO warnings_new (id, node_id, project_id, entity_type_code, entity_id, uuid, type_code, status, first_seen_date, last_seen_date, updated_date, last_message, count) SELECT id, node_id, project_id, entity_type_code, entity_id, uuid, type_code, status, first_seen_date, last_seen_date, updated_date, last_message, count FROM warnings; DROP TABLE warnings; DROP TABLE storage_volumes_snapshots_config; DROP TABLE storage_volumes_snapshots; DROP TABLE storage_volumes_config; DROP TABLE storage_volumes_backups; DROP TABLE storage_volumes; DROP TABLE storage_pools_nodes; DROP TABLE storage_pools_config; DROP TABLE storage_pools; DROP TABLE projects_config; DROP TABLE profiles_devices_config; DROP TABLE profiles_devices; DROP TABLE profiles_config; DROP TABLE operations; DROP TABLE nodes_roles; DROP TABLE nodes_config; DROP TABLE nodes_cluster_groups; DROP TABLE networks_zones_config; DROP TABLE networks_zones; DROP TABLE networks_peers_config; DROP TABLE networks_peers; DROP TABLE networks_nodes; DROP TABLE networks_forwards_config; DROP TABLE networks_forwards; DROP TABLE networks_config; DROP TABLE networks_acls_config; DROP TABLE networks_acls; DROP TABLE networks; DROP TABLE instances_snapshots_devices_config; DROP TABLE instances_snapshots_devices; DROP TABLE instances_snapshots_config; DROP TABLE instances_snapshots; DROP TABLE instances_profiles; DROP TABLE instances_devices_config; DROP TABLE instances_devices; DROP TABLE instances_config; DROP TABLE instances_backups; DROP TABLE instances; DROP TABLE images_source; DROP TABLE images_properties; DROP TABLE images_profiles; DROP TABLE profiles; DROP TABLE images_nodes; DROP TABLE images_aliases; DROP TABLE nodes; DROP TABLE certificates_projects; DROP TABLE images; DROP TABLE projects; ALTER TABLE projects_new RENAME TO projects; ALTER TABLE certificates_projects_new RENAME TO certificates_projects; ALTER TABLE images_new RENAME TO images; ALTER TABLE images_aliases_new RENAME TO images_aliases; ALTER TABLE nodes_new RENAME TO nodes; ALTER TABLE images_nodes_new RENAME TO images_nodes; ALTER TABLE profiles_new RENAME TO profiles; ALTER TABLE images_profiles_new RENAME TO images_profiles; ALTER TABLE images_properties_new RENAME TO images_properties; ALTER TABLE images_source_new RENAME TO images_source; ALTER TABLE instances_new RENAME TO instances; ALTER TABLE instances_backups_new RENAME TO instances_backups; ALTER TABLE instances_config_new RENAME TO instances_config; ALTER TABLE instances_devices_new RENAME TO instances_devices; ALTER TABLE instances_devices_config_new RENAME TO instances_devices_config; ALTER TABLE instances_profiles_new RENAME TO instances_profiles; ALTER TABLE instances_snapshots_new RENAME TO instances_snapshots; ALTER TABLE instances_snapshots_config_new RENAME TO instances_snapshots_config; ALTER TABLE instances_snapshots_devices_new RENAME TO instances_snapshots_devices; ALTER TABLE instances_snapshots_devices_config_new RENAME TO instances_snapshots_devices_config; ALTER TABLE networks_new RENAME TO networks; ALTER TABLE networks_acls_new RENAME TO networks_acls; ALTER TABLE networks_acls_config_new RENAME TO networks_acls_config; ALTER TABLE networks_config_new RENAME TO networks_config; ALTER TABLE networks_forwards_new RENAME TO networks_forwards; ALTER TABLE networks_forwards_config_new RENAME TO networks_forwards_config; ALTER TABLE networks_nodes_new RENAME TO networks_nodes; ALTER TABLE networks_peers_new RENAME TO networks_peers; ALTER TABLE networks_peers_config_new RENAME TO networks_peers_config; ALTER TABLE networks_zones_new RENAME TO networks_zones; ALTER TABLE networks_zones_config_new RENAME TO networks_zones_config; ALTER TABLE nodes_cluster_groups_new RENAME TO nodes_cluster_groups; ALTER TABLE nodes_config_new RENAME TO nodes_config; ALTER TABLE nodes_roles_new RENAME TO nodes_roles; ALTER TABLE operations_new RENAME TO operations; ALTER TABLE profiles_config_new RENAME TO profiles_config; ALTER TABLE profiles_devices_new RENAME TO profiles_devices; ALTER TABLE profiles_devices_config_new RENAME TO profiles_devices_config; ALTER TABLE projects_config_new RENAME TO projects_config; ALTER TABLE storage_pools_new RENAME TO storage_pools; ALTER TABLE storage_pools_config_new RENAME TO storage_pools_config; ALTER TABLE storage_pools_nodes_new RENAME TO storage_pools_nodes; ALTER TABLE storage_volumes_new RENAME TO storage_volumes; ALTER TABLE storage_volumes_backups_new RENAME TO storage_volumes_backups; ALTER TABLE storage_volumes_config_new RENAME TO storage_volumes_config; ALTER TABLE storage_volumes_snapshots_new RENAME TO storage_volumes_snapshots; ALTER TABLE storage_volumes_snapshots_config_new RENAME TO storage_volumes_snapshots_config; ALTER TABLE warnings_new RENAME TO warnings; CREATE INDEX images_aliases_project_id_idx ON images_aliases (project_id); CREATE INDEX images_project_id_idx ON images (project_id); CREATE INDEX instances_project_id_and_name_idx ON instances (project_id, name); CREATE INDEX instances_project_id_and_node_id_and_name_idx ON instances (project_id, node_id, name); CREATE INDEX instances_project_id_and_node_id_idx ON instances (project_id, node_id); CREATE INDEX instances_project_id_idx ON instances (project_id); CREATE UNIQUE INDEX storage_pools_unique_storage_pool_id_node_id_key ON storage_pools_config (storage_pool_id, IFNULL(node_id, -1), key); CREATE INDEX instances_node_id_idx ON instances (node_id); CREATE UNIQUE INDEX networks_unique_network_id_node_id_key ON "networks_config" (network_id, IFNULL(node_id, -1), key); CREATE INDEX profiles_project_id_idx ON profiles (project_id); CREATE UNIQUE INDEX warnings_unique_node_id_project_id_entity_type_code_entity_id_type_code ON warnings(IFNULL(node_id, -1), IFNULL(project_id, -1), entity_type_code, entity_id, type_code); CREATE TRIGGER storage_volumes_check_id BEFORE INSERT ON storage_volumes WHEN NEW.id IN (SELECT id FROM storage_volumes_snapshots) BEGIN SELECT RAISE(FAIL, "invalid ID"); END; CREATE TRIGGER storage_volumes_snapshots_check_id BEFORE INSERT ON storage_volumes_snapshots WHEN NEW.id IN (SELECT id FROM storage_volumes) BEGIN SELECT RAISE(FAIL, "invalid ID"); END; CREATE VIEW storage_volumes_all ( id, name, storage_pool_id, node_id, type, description, project_id, content_type) AS SELECT id, name, storage_pool_id, node_id, type, description, project_id, content_type FROM storage_volumes UNION SELECT storage_volumes_snapshots.id, printf('%s/%s', storage_volumes.name, storage_volumes_snapshots.name), storage_volumes.storage_pool_id, storage_volumes.node_id, storage_volumes.type, storage_volumes_snapshots.description, storage_volumes.project_id, storage_volumes.content_type FROM storage_volumes JOIN storage_volumes_snapshots ON storage_volumes.id = storage_volumes_snapshots.storage_volume_id; `) if err != nil { return fmt.Errorf("Could not add not null constraint to description field: %w", err) } return nil } func updateFromV54(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` DROP VIEW certificates_projects_ref; DROP VIEW instances_config_ref; DROP VIEW instances_devices_ref; DROP VIEW instances_profiles_ref; DROP VIEW instances_snapshots_config_ref; DROP VIEW instances_snapshots_devices_ref; DROP VIEW profiles_config_ref; DROP VIEW profiles_devices_ref; DROP VIEW profiles_used_by_ref; DROP VIEW projects_config_ref; DROP VIEW projects_used_by_ref; `) if err != nil { return fmt.Errorf("Failed to drop database views: %w", err) } return nil } // updateFromV53 creates the cluster_groups and nodes_cluster_groups tables. func updateFromV53(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE "cluster_groups" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, UNIQUE (name) ); CREATE TABLE "nodes_cluster_groups" ( node_id INTEGER NOT NULL, group_id INTEGER NOT NULL, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE, FOREIGN KEY (group_id) REFERENCES cluster_groups (id) ON DELETE CASCADE, UNIQUE (node_id, group_id) ); INSERT INTO cluster_groups (id, name, description) VALUES (1, 'default', 'Default cluster group'); INSERT INTO nodes_cluster_groups (node_id, group_id) SELECT id, 1 FROM nodes; `) if err != nil { return fmt.Errorf("Failed creating cluster group tables: %w", err) } return nil } // updateFromV52 creates the networks_zones and networks_zones_config tables. func updateFromV52(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE "networks_zones" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, UNIQUE (name), FOREIGN KEY (project_id) REFERENCES "projects" (id) ON DELETE CASCADE ); CREATE TABLE "networks_zones_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_zone_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (network_zone_id, key), FOREIGN KEY (network_zone_id) REFERENCES "networks_zones" (id) ON DELETE CASCADE ); `) if err != nil { return fmt.Errorf("Failed creating network zones tables: %w", err) } return nil } // updateFromV51 creates the networks_peers and networks_peers_config tables. func updateFromV51(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE "networks_peers" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, target_network_project TEXT NULL, target_network_name TEXT NULL, target_network_id INTEGER NULL, UNIQUE (network_id, name), UNIQUE (network_id, target_network_project, target_network_name), UNIQUE (network_id, target_network_id), FOREIGN KEY (network_id) REFERENCES "networks" (id) ON DELETE CASCADE ); CREATE TABLE "networks_peers_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_peer_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (network_peer_id, key), FOREIGN KEY (network_peer_id) REFERENCES "networks_peers" (id) ON DELETE CASCADE ); `) if err != nil { return fmt.Errorf("Failed creating network peers tables: %w", err) } return nil } // updateFromV50 creates the nodes_config table. func updateFromV50(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE "nodes_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, node_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE, UNIQUE (node_id, key) ); `) if err != nil { return fmt.Errorf("Failed creating nodes_config table: %w", err) } return nil } // updateFromV49 creates the networks_forwards and networks_forwards_config tables. func updateFromV49(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE "networks_forwards" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER, listen_address TEXT NOT NULL, description TEXT NOT NULL, ports TEXT NOT NULL, UNIQUE (network_id, node_id, listen_address), FOREIGN KEY (network_id) REFERENCES "networks" (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES "nodes" (id) ON DELETE CASCADE ); CREATE TABLE "networks_forwards_config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_forward_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (network_forward_id, key), FOREIGN KEY (network_forward_id) REFERENCES "networks_forwards" (id) ON DELETE CASCADE ); `) if err != nil { return fmt.Errorf("Failed creating network forwards tables: %w", err) } return nil } // updateFromV48 renames the "pending" column to "state" in the "nodes" table. func updateFromV48(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` ALTER TABLE nodes RENAME COLUMN pending TO state; `) if err != nil { return fmt.Errorf(`Failed to rename column "pending" to "state" in table "nodes": %w`, err) } return nil } // updateFromV47 adds warnings. func updateFromV47(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE warnings ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, node_id INTEGER, project_id INTEGER, entity_type_code INTEGER, entity_id INTEGER, uuid TEXT NOT NULL, type_code INTEGER NOT NULL, status INTEGER NOT NULL, first_seen_date DATETIME NOT NULL, last_seen_date DATETIME NOT NULL, updated_date DATETIME, last_message TEXT NOT NULL, count INTEGER NOT NULL, UNIQUE (uuid), FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE ); CREATE UNIQUE INDEX warnings_unique_node_id_project_id_entity_type_code_entity_id_type_code ON warnings(IFNULL(node_id, -1), IFNULL(project_id, -1), entity_type_code, entity_id, type_code); `) if err != nil { return fmt.Errorf("Failed to create warnings table and warnings_unique_node_id_project_id_entity_type_code_entity_id_type_code index: %w", err) } return err } // updateFromV46 adds support for restricting certificates to projects. func updateFromV46(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` ALTER TABLE certificates ADD COLUMN restricted INTEGER NOT NULL DEFAULT 0; CREATE TABLE certificates_projects ( certificate_id INTEGER NOT NULL, project_id INTEGER NOT NULL, FOREIGN KEY (certificate_id) REFERENCES certificates (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE, UNIQUE (certificate_id, project_id) ); CREATE VIEW certificates_projects_ref (fingerprint, value) AS SELECT certificates.fingerprint, projects.name FROM certificates_projects JOIN certificates ON certificates.id=certificates_projects.certificate_id JOIN projects ON projects.id=certificates_projects.project_id ORDER BY projects.name; `) if err != nil { return fmt.Errorf("Failed extending certificates to support project restrictions: %w", err) } return nil } // updateFromV45 updates projects_used_by_ref to include ceph volumes. func updateFromV45(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` DROP VIEW projects_used_by_ref; CREATE VIEW projects_used_by_ref (name, value) AS SELECT projects.name, printf('/1.0/instances/%s?project=%s', "instances".name, projects.name) FROM "instances" JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/images/%s?project=%s', images.fingerprint, projects.name) FROM images JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/storage-pools/%s/volumes/custom/%s?project=%s&target=%s', storage_pools.name, storage_volumes.name, projects.name, nodes.name) FROM storage_volumes JOIN storage_pools ON storage_pool_id=storage_pools.id JOIN nodes ON node_id=nodes.id JOIN projects ON project_id=projects.id WHERE storage_volumes.type=2 UNION SELECT projects.name, printf('/1.0/storage-pools/%s/volumes/custom/%s?project=%s', storage_pools.name, storage_volumes.name, projects.name) FROM storage_volumes JOIN storage_pools ON storage_pool_id=storage_pools.id JOIN projects ON project_id=projects.id WHERE storage_volumes.type=2 AND storage_volumes.node_id IS NULL UNION SELECT projects.name, printf('/1.0/profiles/%s?project=%s', profiles.name, projects.name) FROM profiles JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/networks/%s?project=%s', networks.name, projects.name) FROM networks JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/network-acls/%s?project=%s', networks_acls.name, projects.name) FROM networks_acls JOIN projects ON project_id=projects.id; `) if err != nil { return fmt.Errorf("Failed to update projects_used_by_ref: %w", err) } return nil } // updateFromV44 adds networks_acls table, and adds a foreign key relationship between networks and projects. // API extension: network_acl. func updateFromV44(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` DROP VIEW projects_used_by_ref; CREATE TABLE networks_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT, state INTEGER NOT NULL DEFAULT 0, type INTEGER NOT NULL DEFAULT 0, UNIQUE (project_id, name), FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE ); INSERT INTO networks_new (id, project_id, name, description, state, type) SELECT id, project_id, name, description, state, type FROM networks; CREATE TABLE networks_nodes_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER NOT NULL, state INTEGER NOT NULL DEFAULT 0, UNIQUE (network_id, node_id), FOREIGN KEY (network_id) REFERENCES networks_new (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); INSERT INTO networks_nodes_new (id, network_id, node_id, state) SELECT id, network_id, node_id, state FROM networks_nodes; CREATE TABLE networks_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER, key TEXT NOT NULL, value TEXT, UNIQUE (network_id, node_id, key), FOREIGN KEY (network_id) REFERENCES networks_new (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); INSERT INTO networks_config_new (id, network_id, node_id, key, value) SELECT id, network_id, node_id, key, value FROM networks_config; DROP TABLE networks; DROP TABLE networks_nodes; DROP TABLE networks_config; CREATE UNIQUE INDEX networks_unique_network_id_node_id_key ON networks_config_new (network_id, IFNULL(node_id, -1), key); ALTER TABLE networks_new RENAME TO networks; ALTER TABLE networks_nodes_new RENAME TO networks_nodes; ALTER TABLE networks_config_new RENAME TO networks_config; CREATE TABLE networks_acls ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT NOT NULL, ingress TEXT NOT NULL, egress TEXT NOT NULL, UNIQUE (project_id, name), FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE ); CREATE TABLE networks_acls_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_acl_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (network_acl_id, key), FOREIGN KEY (network_acl_id) REFERENCES networks_acls (id) ON DELETE CASCADE ); CREATE VIEW projects_used_by_ref (name, value) AS SELECT projects.name, printf('/1.0/instances/%s?project=%s', "instances".name, projects.name) FROM "instances" JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/images/%s?project=%s', images.fingerprint, projects.name) FROM images JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/storage-pools/%s/volumes/custom/%s?project=%s&target=%s', storage_pools.name, storage_volumes.name, projects.name, nodes.name) FROM storage_volumes JOIN storage_pools ON storage_pool_id=storage_pools.id JOIN nodes ON node_id=nodes.id JOIN projects ON project_id=projects.id WHERE storage_volumes.type=2 UNION SELECT projects.name, printf('/1.0/profiles/%s?project=%s', profiles.name, projects.name) FROM profiles JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/networks/%s?project=%s', networks.name, projects.name) FROM networks JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/network-acls/%s?project=%s', networks_acls.name, projects.name) FROM networks_acls JOIN projects ON project_id=projects.id; `) if err != nil { return fmt.Errorf("Failed to add networks_acls and networks_acls_config tables, and update projects_used_by_ref view: %w", err) } return nil } // updateFromV43 adds a unique index to the storage_pools_config and networks_config tables. func updateFromV43(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` CREATE UNIQUE INDEX storage_pools_unique_storage_pool_id_node_id_key ON storage_pools_config (storage_pool_id, IFNULL(node_id, -1), key); CREATE UNIQUE INDEX networks_unique_network_id_node_id_key ON networks_config (network_id, IFNULL(node_id, -1), key); `) if err != nil { return fmt.Errorf("Failed adding unique index to storage_pools_config and networks_config tables: %w", err) } return nil } // updateFromV42 removes any duplicated storage pool config rows that have the same value. // This can occur when multiple create requests have been issued when setting up a clustered storage pool. func updateFromV42(ctx context.Context, tx *sql.Tx) error { // Find all duplicated config rows and return comma delimited list of affected row IDs for each dupe set. stmt := `SELECT storage_pool_id, IFNULL(node_id, -1), key, value, COUNT(*) AS rowCount, GROUP_CONCAT(id, ",") AS dupeRowIDs FROM storage_pools_config GROUP BY storage_pool_id, node_id, key, value HAVING rowCount > 1 ` rows, err := tx.QueryContext(ctx, stmt) if err != nil { return fmt.Errorf("Failed running query: %w", err) } defer func() { _ = rows.Close() }() type dupeRow struct { storagePoolID int64 nodeID int64 key string value string rowCount int64 dupeRowIDs string } var dupeRows []dupeRow for rows.Next() { r := dupeRow{} err = rows.Scan(&r.storagePoolID, &r.nodeID, &r.key, &r.value, &r.rowCount, &r.dupeRowIDs) if err != nil { return fmt.Errorf("Failed scanning rows: %w", err) } dupeRows = append(dupeRows, r) } err = rows.Err() if err != nil { return fmt.Errorf("Got a row error: %w", err) } for _, r := range dupeRows { logger.Warn("Found duplicated storage pool config rows", logger.Ctx{"storagePoolID": r.storagePoolID, "nodeID": r.nodeID, "key": r.key, "value": r.value, "rowCount": r.rowCount, "dupeRowIDs": r.dupeRowIDs}) rowIDs := strings.Split(r.dupeRowIDs, ",") // Iterate and delete all but 1 of the rowIDs so we leave just one left. for i := range len(rowIDs) - 1 { rowID, err := strconv.Atoi(rowIDs[i]) if err != nil { return fmt.Errorf("Failed converting row ID: %w", err) } _, err = tx.Exec("DELETE FROM storage_pools_config WHERE id = ?", rowID) if err != nil { return fmt.Errorf("Failed deleting storage pool config row with ID %d: %w", rowID, err) } logger.Warn("Deleted duplicated storage pool config row", logger.Ctx{"storagePoolID": r.storagePoolID, "nodeID": r.nodeID, "key": r.key, "value": r.value, "rowCount": r.rowCount, "rowID": rowID}) } } return nil } // updateFromV41 removes any duplicated network config rows that have the same value. // This can occur when multiple create requests have been issued when setting up a clustered network. func updateFromV41(ctx context.Context, tx *sql.Tx) error { // Find all duplicated config rows and return comma delimited list of affected row IDs for each dupe set. stmt := `SELECT network_id, IFNULL(node_id, -1), key, value, COUNT(*) AS rowCount, GROUP_CONCAT(id, ",") AS dupeRowIDs FROM networks_config GROUP BY network_id, node_id, key, value HAVING rowCount > 1 ` rows, err := tx.QueryContext(ctx, stmt) if err != nil { return fmt.Errorf("Failed running query: %w", err) } defer func() { _ = rows.Close() }() type dupeRow struct { networkID int64 nodeID int64 key string value string rowCount int64 dupeRowIDs string } var dupeRows []dupeRow for rows.Next() { r := dupeRow{} err = rows.Scan(&r.networkID, &r.nodeID, &r.key, &r.value, &r.rowCount, &r.dupeRowIDs) if err != nil { return fmt.Errorf("Failed scanning rows: %w", err) } dupeRows = append(dupeRows, r) } err = rows.Err() if err != nil { return fmt.Errorf("Got a row error: %w", err) } for _, r := range dupeRows { logger.Warn("Found duplicated network config rows", logger.Ctx{"networkID": r.networkID, "nodeID": r.nodeID, "key": r.key, "value": r.value, "rowCount": r.rowCount, "dupeRowIDs": r.dupeRowIDs}) rowIDs := strings.Split(r.dupeRowIDs, ",") // Iterate and delete all but 1 of the rowIDs so we leave just one left. for i := range len(rowIDs) - 1 { rowID, err := strconv.Atoi(rowIDs[i]) if err != nil { return fmt.Errorf("Failed converting row ID: %w", err) } _, err = tx.Exec("DELETE FROM networks_config WHERE id = ?", rowID) if err != nil { return fmt.Errorf("Failed deleting network config row with ID %d: %w", rowID, err) } logger.Warn("Deleted duplicated network config row", logger.Ctx{"networkID": r.networkID, "nodeID": r.nodeID, "key": r.key, "value": r.value, "rowCount": r.rowCount, "rowID": rowID}) } } return nil } // Add state column to storage_pools_nodes tables. Set existing row's state to 1 ("created"). func updateFromV40(ctx context.Context, tx *sql.Tx) error { stmt := ` ALTER TABLE storage_pools_nodes ADD COLUMN state INTEGER NOT NULL DEFAULT 0; UPDATE storage_pools_nodes SET state = 1; ` _, err := tx.Exec(stmt) return err } // Add state column to networks_nodes tables. Set existing row's state to 1 ("created"). func updateFromV39(ctx context.Context, tx *sql.Tx) error { stmt := ` ALTER TABLE networks_nodes ADD COLUMN state INTEGER NOT NULL DEFAULT 0; UPDATE networks_nodes SET state = 1; ` _, err := tx.Exec(stmt) return err } // Add storage_volumes_backups table. func updateFromV38(ctx context.Context, tx *sql.Tx) error { stmt := ` CREATE TABLE storage_volumes_backups ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, creation_date DATETIME, expiry_date DATETIME, volume_only INTEGER NOT NULL default 0, optimized_storage INTEGER NOT NULL default 0, FOREIGN KEY (storage_volume_id) REFERENCES "storage_volumes" (id) ON DELETE CASCADE, UNIQUE (storage_volume_id, name) ); ` _, err := tx.Exec(stmt) if err != nil { return err } return nil } // Attempt to add missing project features.networks feature to default project. func updateFromV37(ctx context.Context, tx *sql.Tx) error { ids, err := query.SelectIntegers(ctx, tx, `SELECT id FROM projects WHERE name = "default" LIMIT 1`) if err != nil { return err } if len(ids) == 1 { _, _ = tx.Exec("INSERT INTO projects_config (project_id, key, value) VALUES (?, 'features.networks', 'true');", ids[0]) } return nil } // Add networks to projects references. func updateFromV36(ctx context.Context, tx *sql.Tx) error { stmts := ` DROP VIEW projects_used_by_ref; CREATE VIEW projects_used_by_ref (name, value) AS SELECT projects.name, printf('/1.0/instances/%s?project=%s', "instances".name, projects.name) FROM "instances" JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/images/%s?project=%s', images.fingerprint, projects.name) FROM images JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/storage-pools/%s/volumes/custom/%s?project=%s&target=%s', storage_pools.name, storage_volumes.name, projects.name, nodes.name) FROM storage_volumes JOIN storage_pools ON storage_pool_id=storage_pools.id JOIN nodes ON node_id=nodes.id JOIN projects ON project_id=projects.id WHERE storage_volumes.type=2 UNION SELECT projects.name, printf('/1.0/profiles/%s?project=%s', profiles.name, projects.name) FROM profiles JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/networks/%s?project=%s', networks.name, projects.name) FROM networks JOIN projects ON project_id=projects.id; ` _, err := tx.Exec(stmts) return err } // This fixes node IDs of storage volumes on non-remote pools which were // wrongly set to NULL. func updateFromV35(ctx context.Context, tx *sql.Tx) error { stmts := ` WITH storage_volumes_tmp (id, node_id) AS ( SELECT storage_volumes.id, storage_pools_nodes.node_id FROM storage_volumes JOIN storage_pools_nodes ON storage_pools_nodes.storage_pool_id=storage_volumes.storage_pool_id JOIN storage_pools ON storage_pools.id=storage_volumes.storage_pool_id WHERE storage_pools.driver NOT IN ("ceph", "cephfs")) UPDATE storage_volumes SET node_id=( SELECT storage_volumes_tmp.node_id FROM storage_volumes_tmp WHERE storage_volumes.id=storage_volumes_tmp.id) WHERE id IN (SELECT id FROM storage_volumes_tmp) AND node_id IS NULL ` _, err := tx.Exec(stmts) if err != nil { return err } return nil } // Remove multiple entries of the same volume when using remote storage. // Also, allow node ID to be null for the instances and storage_volumes tables, and set it to null // for instances and storage volumes using remote storage. func updateFromV34(ctx context.Context, tx *sql.Tx) error { stmts := ` SELECT storage_volumes.id, storage_volumes.name FROM storage_volumes JOIN storage_pools ON storage_pools.id=storage_volumes.storage_pool_id WHERE storage_pools.driver IN ("ceph", "cephfs") ORDER BY storage_volumes.name ` // Get the total number of storage volume rows. count, err := query.Count(ctx, tx, "storage_volumes JOIN storage_pools ON storage_pools.id=storage_volumes.storage_pool_id", `storage_pools.driver IN ("ceph", "cephfs")`) if err != nil { return fmt.Errorf("Failed to get storage volumes count: %w", err) } type volume struct { ID int Name string StoragePoolID int NodeID string Type int Description string ProjectID int ContentType int } volumes := make([]volume, 0, count) err = query.Scan(ctx, tx, stmts, func(scan func(dest ...any) error) error { vol := volume{} err := scan(&vol.ID, &vol.Name) if err != nil { return err } volumes = append(volumes, vol) return nil }) if err != nil { return fmt.Errorf("Failed to fetch storage volumes with remote storage: %w", err) } // Remove multiple entries of the same volume when using remote storage for i := 1; i < count; i++ { if volumes[i-1].Name == volumes[i].Name { _, err = tx.Exec(`DELETE FROM storage_volumes WHERE id=?`, volumes[i-1].ID) if err != nil { return fmt.Errorf("Failed to delete row from storage_volumes: %w", err) } } } stmts = ` CREATE TABLE storage_volumes_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER, type INTEGER NOT NULL, description TEXT, project_id INTEGER NOT NULL, content_type INTEGER NOT NULL DEFAULT 0, UNIQUE (storage_pool_id, node_id, project_id, name, type), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE );` // Create new tables where node ID can be null. _, err = tx.Exec(stmts) if err != nil { return err } // Copy rows from storage_volumes to storage_volumes_new count, err = query.Count(ctx, tx, "storage_volumes", "") if err != nil { return fmt.Errorf("Failed to get storage_volumes count: %w", err) } storageVolumes := make([]volume, 0, count) sqlStr := ` SELECT id, name, storage_pool_id, node_id, type, coalesce(description, ''), project_id, content_type FROM storage_volumes` err = query.Scan(ctx, tx, sqlStr, func(scan func(dest ...any) error) error { vol := volume{} err := scan(&vol.ID, &vol.Name, &vol.StoragePoolID, &vol.NodeID, &vol.Type, &vol.Description, &vol.ProjectID, &vol.ContentType) if err != nil { return err } storageVolumes = append(storageVolumes, vol) return nil }) if err != nil { return fmt.Errorf("Failed to fetch storage volumes: %w", err) } for _, storageVolume := range storageVolumes { _, err = tx.Exec(` INSERT INTO storage_volumes_new (id, name, storage_pool_id, node_id, type, description, project_id, content_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?);`, storageVolume.ID, storageVolume.Name, storageVolume.StoragePoolID, storageVolume.NodeID, storageVolume.Type, storageVolume.Description, storageVolume.ProjectID, storageVolume.ContentType) if err != nil { return err } } // Store rows of storage_volumes_config as we need to re-add them at the end. count, err = query.Count(ctx, tx, "storage_volumes_config", "") if err != nil { return fmt.Errorf("Failed to get storage_volumes_config count: %w", err) } type volumeConfig struct { ID int StorageVolumeID int Key string Value string } storageVolumeConfigs := make([]volumeConfig, 0, count) sqlStr = `SELECT * FROM storage_volumes_config;` err = query.Scan(ctx, tx, sqlStr, func(scan func(dest ...any) error) error { config := volumeConfig{} err := scan(&config.ID, &config.StorageVolumeID, &config.Key, &config.Value) if err != nil { return err } storageVolumeConfigs = append(storageVolumeConfigs, config) return nil }) if err != nil { return fmt.Errorf("Failed to fetch storage volume configs: %w", err) } // Store rows of storage_volumes_snapshots as we need to re-add them at the end. count, err = query.Count(ctx, tx, "storage_volumes_snapshots", "") if err != nil { return fmt.Errorf("Failed to get storage_volumes_snapshots count: %w", err) } type volumeSnapshot struct { ID int StorageVolumeID int Name string Description string ExpiryDate sql.NullTime } sqlStr = `SELECT * FROM storage_volumes_snapshots;` storageVolumeSnapshots := make([]volumeSnapshot, 0, count) err = query.Scan(ctx, tx, sqlStr, func(scan func(dest ...any) error) error { vol := volumeSnapshot{} err := scan(&vol.ID, &vol.StorageVolumeID, &vol.Name, &vol.Description, &vol.ExpiryDate) if err != nil { return err } storageVolumeSnapshots = append(storageVolumeSnapshots, vol) return nil }) if err != nil { return fmt.Errorf("Failed to fetch storage volume snapshots: %w", err) } // Store rows of storage_volumes_snapshots_config as we need to re-add them at the end. count, err = query.Count(ctx, tx, "storage_volumes_snapshots_config", "") if err != nil { return fmt.Errorf("Failed to get storage_volumes_snapshots_config count: %w", err) } type volumeSnapshotConfig struct { ID int StorageVolumeSnapshotID int Key string Value string } storageVolumeSnapshotConfigs := make([]volumeSnapshotConfig, 0, count) sqlStr = `SELECT * FROM storage_volumes_snapshots_config;` err = query.Scan(ctx, tx, sqlStr, func(scan func(dest ...any) error) error { config := volumeSnapshotConfig{} err := scan(&config.ID, &config.StorageVolumeSnapshotID, &config.Key, &config.Value) if err != nil { return err } storageVolumeSnapshotConfigs = append(storageVolumeSnapshotConfigs, config) return nil }) if err != nil { return fmt.Errorf("Failed to fetch storage volume snapshot configs: %w", err) } _, err = tx.Exec(` PRAGMA foreign_keys = OFF; PRAGMA legacy_alter_table = ON; DROP TABLE storage_volumes; ALTER TABLE storage_volumes_new RENAME TO storage_volumes; UPDATE storage_volumes SET node_id=null WHERE storage_volumes.id IN ( SELECT storage_volumes.id FROM storage_volumes JOIN storage_pools ON storage_volumes.storage_pool_id=storage_pools.id WHERE storage_pools.driver IN ("ceph", "cephfs") ); PRAGMA foreign_keys = ON; PRAGMA legacy_alter_table = OFF; CREATE TRIGGER storage_volumes_check_id BEFORE INSERT ON storage_volumes WHEN NEW.id IN (SELECT id FROM storage_volumes_snapshots) BEGIN SELECT RAISE(FAIL, "invalid ID"); END; `) if err != nil { return err } // When we dropped the storage_volumes table earlier, all config entries // were removed as well. Let's re-add them. for _, storageVolumeConfig := range storageVolumeConfigs { _, err = tx.Exec(`INSERT INTO storage_volumes_config (id, storage_volume_id, key, value) VALUES (?, ?, ?, ?);`, storageVolumeConfig.ID, storageVolumeConfig.StorageVolumeID, storageVolumeConfig.Key, storageVolumeConfig.Value) if err != nil { return err } } // When we dropped the storage_volumes table earlier, all snapshot entries // were removed as well. Let's re-add them. for _, storageVolumeSnapshot := range storageVolumeSnapshots { _, err = tx.Exec(`INSERT INTO storage_volumes_snapshots (id, storage_volume_id, name, description, expiry_date) VALUES (?, ?, ?, ?, ?);`, storageVolumeSnapshot.ID, storageVolumeSnapshot.StorageVolumeID, storageVolumeSnapshot.Name, storageVolumeSnapshot.Description, storageVolumeSnapshot.ExpiryDate) if err != nil { return err } } // When we dropped the storage_volumes table earlier, all snapshot config entries // were removed as well. Let's re-add them. for _, storageVolumeSnapshotConfig := range storageVolumeSnapshotConfigs { _, err = tx.Exec(`INSERT INTO storage_volumes_snapshots_config (id, storage_volume_snapshot_id, key, value) VALUES (?, ?, ?, ?);`, storageVolumeSnapshotConfig.ID, storageVolumeSnapshotConfig.StorageVolumeSnapshotID, storageVolumeSnapshotConfig.Key, storageVolumeSnapshotConfig.Value) if err != nil { return err } } count, err = query.Count(ctx, tx, "storage_volumes_all", "") if err != nil { return fmt.Errorf("Failed to get storage_volumes count: %w", err) } if count > 0 { var maxID int64 row := tx.QueryRowContext(ctx, "SELECT MAX(id) FROM storage_volumes_all LIMIT 1") err = row.Scan(&maxID) if err != nil { return err } // Set sqlite_sequence to max(id) _, err = tx.Exec("UPDATE sqlite_sequence SET seq = ? WHERE name = 'storage_volumes'", maxID) if err != nil { return fmt.Errorf("Increment storage volumes sequence: %w", err) } } return nil } // Add project_id field to networks, add unique index across project_id and name, // and set existing networks to project_id 1. // This is made a lot more complex because it requires re-creating the referenced tables as there is no way to // disable foreign keys temporarily within a transaction. func updateFromV33(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec(` CREATE TABLE networks_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT, state INTEGER NOT NULL DEFAULT 0, type INTEGER NOT NULL DEFAULT 0, UNIQUE (project_id, name) ); INSERT INTO networks_new (id, project_id, name, description, state, type) SELECT id, 1, name, description, state, type FROM networks; CREATE TABLE networks_nodes_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER NOT NULL, UNIQUE (network_id, node_id), FOREIGN KEY (network_id) REFERENCES networks_new (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); INSERT INTO networks_nodes_new (id, network_id, node_id) SELECT id, network_id, node_id FROM networks_nodes; CREATE TABLE networks_config_new ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER, key TEXT NOT NULL, value TEXT, UNIQUE (network_id, node_id, key), FOREIGN KEY (network_id) REFERENCES networks_new (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); INSERT INTO networks_config_new (id, network_id, node_id, key, value) SELECT id, network_id, node_id, key, value FROM networks_config; DROP TABLE networks; DROP TABLE networks_nodes; DROP TABLE networks_config; ALTER TABLE networks_new RENAME TO networks; ALTER TABLE networks_nodes_new RENAME TO networks_nodes; ALTER TABLE networks_config_new RENAME TO networks_config; `) if err != nil { return fmt.Errorf("Failed to add project_id column to networks table: %w", err) } return nil } // Add type field to networks. func updateFromV32(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec("ALTER TABLE networks ADD COLUMN type INTEGER NOT NULL DEFAULT 0;") if err != nil { return fmt.Errorf("Failed to add type column to networks table: %w", err) } return nil } // Add failure_domain column to nodes table. func updateFromV31(ctx context.Context, tx *sql.Tx) error { stmts := ` CREATE TABLE nodes_failure_domains ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, UNIQUE (name) ); ALTER TABLE nodes ADD COLUMN failure_domain_id INTEGER DEFAULT NULL REFERENCES nodes_failure_domains (id) ON DELETE SET NULL; ` _, err := tx.Exec(stmts) if err != nil { return err } return nil } // Add content type field to storage volumes. func updateFromV30(ctx context.Context, tx *sql.Tx) error { stmts := `ALTER TABLE storage_volumes ADD COLUMN content_type INTEGER NOT NULL DEFAULT 0; UPDATE storage_volumes SET content_type = 1 WHERE type = 3; UPDATE storage_volumes SET content_type = 1 WHERE storage_volumes.id IN ( SELECT storage_volumes.id FROM storage_volumes JOIN images ON storage_volumes.name = images.fingerprint WHERE images.type = 1 ); DROP VIEW storage_volumes_all; CREATE VIEW storage_volumes_all ( id, name, storage_pool_id, node_id, type, description, project_id, content_type) AS SELECT id, name, storage_pool_id, node_id, type, description, project_id, content_type FROM storage_volumes UNION SELECT storage_volumes_snapshots.id, printf('%s/%s', storage_volumes.name, storage_volumes_snapshots.name), storage_volumes.storage_pool_id, storage_volumes.node_id, storage_volumes.type, storage_volumes_snapshots.description, storage_volumes.project_id, storage_volumes.content_type FROM storage_volumes JOIN storage_volumes_snapshots ON storage_volumes.id = storage_volumes_snapshots.storage_volume_id; ` _, err := tx.Exec(stmts) if err != nil { return fmt.Errorf("Failed to add storage volume content type: %w", err) } return nil } // Add storage volumes to projects references and fix images. func updateFromV29(ctx context.Context, tx *sql.Tx) error { stmts := ` DROP VIEW projects_used_by_ref; CREATE VIEW projects_used_by_ref (name, value) AS SELECT projects.name, printf('/1.0/instances/%s?project=%s', "instances".name, projects.name) FROM "instances" JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/images/%s?project=%s', images.fingerprint, projects.name) FROM images JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/storage-pools/%s/volumes/custom/%s?project=%s&target=%s', storage_pools.name, storage_volumes.name, projects.name, nodes.name) FROM storage_volumes JOIN storage_pools ON storage_pool_id=storage_pools.id JOIN nodes ON node_id=nodes.id JOIN projects ON project_id=projects.id WHERE storage_volumes.type=2 UNION SELECT projects.name, printf('/1.0/profiles/%s?project=%s', profiles.name, projects.name) FROM profiles JOIN projects ON project_id=projects.id; ` _, err := tx.Exec(stmts) return err } // Attempt to add missing project feature. func updateFromV28(ctx context.Context, tx *sql.Tx) error { _, _ = tx.Exec("INSERT INTO projects_config (project_id, key, value) VALUES (1, 'features.storage.volumes', 'true');") return nil } // Add expiry date to storage volume snapshots. func updateFromV27(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec("ALTER TABLE storage_volumes_snapshots ADD COLUMN expiry_date DATETIME;") return err } // Bump the sqlite_sequence value for storage volumes, to avoid unique // constraint violations when inserting new snapshots. func updateFromV26(ctx context.Context, tx *sql.Tx) error { ids, err := query.SelectIntegers(ctx, tx, "SELECT coalesce(max(id), 0) FROM storage_volumes_all") if err != nil { return err } _, err = tx.Exec("UPDATE sqlite_sequence SET seq = ? WHERE name = 'storage_volumes'", ids[0]) return err } // Create new storage snapshot tables and migrate data to them. func updateFromV25(ctx context.Context, tx *sql.Tx) error { // Get the total number of snapshot rows in the storage_volumes table. count, err := query.Count(ctx, tx, "storage_volumes", "snapshot=1") if err != nil { return fmt.Errorf("Failed to volume snapshot count: %w", err) } type snapshot struct { ID int Name string StoragePoolID int NodeID int Type int Description string ProjectID int Config map[string]string } sql := ` SELECT id, name, storage_pool_id, node_id, type, coalesce(description, ''), project_id FROM storage_volumes WHERE snapshot=1 ` if err != nil { return fmt.Errorf("Failed to prepare volume snapshot query: %w", err) } // Fetch all snapshot rows in the storage_volumes table. snapshots := make([]snapshot, 0, count) err = query.Scan(ctx, tx, sql, func(scan func(dest ...any) error) error { s := snapshot{} err := scan(&s.ID, &s.Name, &s.StoragePoolID, &s.NodeID, &s.Type, &s.Description, &s.ProjectID) if err != nil { return err } snapshots = append(snapshots, s) return nil }) if err != nil { return fmt.Errorf("Failed to fetch instances: %w", err) } for i, snapshot := range snapshots { config, err := query.SelectConfig(ctx, tx, "storage_volumes_config", "storage_volume_id=?", snapshot.ID) if err != nil { return fmt.Errorf("Failed to fetch volume snapshot config: %w", err) } snapshots[i].Config = config } stmts := ` ALTER TABLE storage_volumes RENAME TO old_storage_volumes; CREATE TABLE "storage_volumes" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER NOT NULL, type INTEGER NOT NULL, description TEXT, project_id INTEGER NOT NULL, UNIQUE (storage_pool_id, node_id, project_id, name, type), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE ); ALTER TABLE storage_volumes_config RENAME TO old_storage_volumes_config; CREATE TABLE storage_volumes_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (storage_volume_id, key), FOREIGN KEY (storage_volume_id) REFERENCES storage_volumes (id) ON DELETE CASCADE ); INSERT INTO storage_volumes(id, name, storage_pool_id, node_id, type, description, project_id) SELECT id, name, storage_pool_id, node_id, type, description, project_id FROM old_storage_volumes WHERE snapshot=0; INSERT INTO storage_volumes_config SELECT * FROM old_storage_volumes_config WHERE storage_volume_id IN (SELECT id FROM storage_volumes); DROP TABLE old_storage_volumes; DROP TABLE old_storage_volumes_config; CREATE TABLE storage_volumes_snapshots ( id INTEGER NOT NULL, storage_volume_id INTEGER NOT NULL, name TEXT NOT NULL, description TEXT, UNIQUE (id), UNIQUE (storage_volume_id, name), FOREIGN KEY (storage_volume_id) REFERENCES storage_volumes (id) ON DELETE CASCADE ); CREATE TRIGGER storage_volumes_check_id BEFORE INSERT ON storage_volumes WHEN NEW.id IN (SELECT id FROM storage_volumes_snapshots) BEGIN SELECT RAISE(FAIL, "invalid ID"); END; CREATE TRIGGER storage_volumes_snapshots_check_id BEFORE INSERT ON storage_volumes_snapshots WHEN NEW.id IN (SELECT id FROM storage_volumes) BEGIN SELECT RAISE(FAIL, "invalid ID"); END; CREATE TABLE storage_volumes_snapshots_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_snapshot_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (storage_volume_snapshot_id) REFERENCES storage_volumes_snapshots (id) ON DELETE CASCADE, UNIQUE (storage_volume_snapshot_id, key) ); CREATE VIEW storage_volumes_all ( id, name, storage_pool_id, node_id, type, description, project_id) AS SELECT id, name, storage_pool_id, node_id, type, description, project_id FROM storage_volumes UNION SELECT storage_volumes_snapshots.id, printf('%s/%s', storage_volumes.name, storage_volumes_snapshots.name), storage_volumes.storage_pool_id, storage_volumes.node_id, storage_volumes.type, storage_volumes_snapshots.description, storage_volumes.project_id FROM storage_volumes JOIN storage_volumes_snapshots ON storage_volumes.id = storage_volumes_snapshots.storage_volume_id; ` _, err = tx.Exec(stmts) if err != nil { return fmt.Errorf("Failed to create storage snapshots tables: %w", err) } // Migrate snapshots to the new tables. for _, snapshot := range snapshots { parts := strings.Split(snapshot.Name, internalInstance.SnapshotDelimiter) if len(parts) != 2 { logger.Errorf("Invalid volume snapshot name: %s", snapshot.Name) continue } volume := parts[0] name := parts[1] ids, err := query.SelectIntegers(ctx, tx, "SELECT id FROM storage_volumes WHERE name=?", volume) if err != nil { return err } if len(ids) != 1 { logger.Errorf("Volume snapshot %s has no parent", snapshot.Name) continue } volumeID := ids[0] _, err = tx.Exec(` INSERT INTO storage_volumes_snapshots(id, storage_volume_id, name, description) VALUES(?, ?, ?, ?) `, snapshot.ID, volumeID, name, snapshot.Description) if err != nil { return err } for key, value := range snapshot.Config { _, err = tx.Exec(` INSERT INTO storage_volumes_snapshots_config(storage_volume_snapshot_id, key, value) VALUES(?, ?, ?) `, snapshot.ID, key, value) if err != nil { return err } } } return nil } // The ceph.user.name config key is required for Ceph to function. func updateFromV24(ctx context.Context, tx *sql.Tx) error { // Fetch the IDs of all existing Ceph pools. poolIDs, err := query.SelectIntegers(ctx, tx, `SELECT id FROM storage_pools WHERE driver='ceph'`) if err != nil { return fmt.Errorf("Failed to get IDs of current ceph pools: %w", err) } for _, poolID := range poolIDs { // Fetch the config for this Ceph pool. config, err := query.SelectConfig(ctx, tx, "storage_pools_config", "storage_pool_id=?", poolID) if err != nil { return fmt.Errorf("Failed to fetch of ceph pool config: %w", err) } // Check if already set. _, ok := config["ceph.user.name"] if ok { continue } // Add ceph.user.name config entry. _, err = tx.Exec("INSERT INTO storage_pools_config (storage_pool_id, key, value) VALUES (?, 'ceph.user.name', 'admin')", poolID) if err != nil { return fmt.Errorf("Failed to create ceph.user.name config: %w", err) } } return nil } // The lvm.vg_name config key is required for LVM to function. func updateFromV23(ctx context.Context, tx *sql.Tx) error { // Fetch the IDs of all existing nodes. nodeIDs, err := query.SelectIntegers(ctx, tx, "SELECT id FROM nodes") if err != nil { return fmt.Errorf("Failed to get IDs of current nodes: %w", err) } // Fetch the IDs of all existing lvm pools. poolIDs, err := query.SelectIntegers(ctx, tx, `SELECT id FROM storage_pools WHERE driver='lvm'`) if err != nil { return fmt.Errorf("Failed to get IDs of current lvm pools: %w", err) } for _, poolID := range poolIDs { for _, nodeID := range nodeIDs { // Fetch the config for this lvm pool. config, err := query.SelectConfig(ctx, tx, "storage_pools_config", "storage_pool_id=? AND node_id=?", poolID, nodeID) if err != nil { return fmt.Errorf("Failed to fetch of lvm pool config: %w", err) } // Check if already set. _, ok := config["lvm.vg_name"] if ok { continue } // Add lvm.vg_name config entry. _, err = tx.Exec(` INSERT INTO storage_pools_config(storage_pool_id, node_id, key, value) SELECT ?, ?, 'lvm.vg_name', name FROM storage_pools WHERE id=? `, poolID, nodeID, poolID) if err != nil { return fmt.Errorf("Failed to create lvm.vg_name node config: %w", err) } } } return nil } // The zfs.pool_name config key is required for ZFS to function. func updateFromV22(ctx context.Context, tx *sql.Tx) error { // Fetch the IDs of all existing nodes. nodeIDs, err := query.SelectIntegers(ctx, tx, "SELECT id FROM nodes") if err != nil { return fmt.Errorf("Failed to get IDs of current nodes: %w", err) } // Fetch the IDs of all existing zfs pools. poolIDs, err := query.SelectIntegers(ctx, tx, `SELECT id FROM storage_pools WHERE driver='zfs'`) if err != nil { return fmt.Errorf("Failed to get IDs of current zfs pools: %w", err) } for _, poolID := range poolIDs { for _, nodeID := range nodeIDs { // Fetch the config for this zfs pool. config, err := query.SelectConfig(ctx, tx, "storage_pools_config", "storage_pool_id=? AND node_id=?", poolID, nodeID) if err != nil { return fmt.Errorf("Failed to fetch of zfs pool config: %w", err) } // Check if already set. _, ok := config["zfs.pool_name"] if ok { continue } // Add zfs.pool_name config entry _, err = tx.Exec(` INSERT INTO storage_pools_config(storage_pool_id, node_id, key, value) SELECT ?, ?, 'zfs.pool_name', name FROM storage_pools WHERE id=? `, poolID, nodeID, poolID) if err != nil { return fmt.Errorf("Failed to create zfs.pool_name node config: %w", err) } } } return nil } // Fix "images_profiles" table (missing UNIQUE). func updateFromV21(ctx context.Context, tx *sql.Tx) error { stmts := ` ALTER TABLE images_profiles RENAME TO old_images_profiles; CREATE TABLE images_profiles ( image_id INTEGER NOT NULL, profile_id INTEGER NOT NULL, FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE, FOREIGN KEY (profile_id) REFERENCES profiles (id) ON DELETE CASCADE, UNIQUE (image_id, profile_id) ); INSERT INTO images_profiles SELECT * FROM old_images_profiles; DROP TABLE old_images_profiles; ` _, err := tx.Exec(stmts) return err } // Add "images_profiles" table. func updateFromV20(ctx context.Context, tx *sql.Tx) error { stmts := ` CREATE TABLE images_profiles ( image_id INTEGER NOT NULL, profile_id INTEGER NOT NULL, FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE, FOREIGN KEY (profile_id) REFERENCES profiles (id) ON DELETE CASCADE, UNIQUE (image_id, profile_id) ); INSERT INTO images_profiles (image_id, profile_id) SELECT images.id, profiles.id FROM images JOIN profiles ON images.project_id = profiles.project_id WHERE profiles.name = 'default'; INSERT INTO images_profiles (image_id, profile_id) SELECT images.id, profiles.id FROM projects_config AS R JOIN projects_config AS S ON R.project_id = S.project_id JOIN images ON images.project_id = R.project_id JOIN profiles ON profiles.project_id = 1 AND profiles.name = "default" WHERE R.key = "features.images" AND S.key = "features.profiles" AND R.value = "true" AND S.value != "true"; INSERT INTO images_profiles (image_id, profile_id) SELECT images.id, profiles.id FROM projects_config AS R JOIN projects_config AS S ON R.project_id = S.project_id JOIN profiles ON profiles.project_id = R.project_id JOIN images ON images.project_id = 1 WHERE R.key = "features.images" AND S.key = "features.profiles" AND R.value != "true" AND S.value = "true" AND profiles.name = "default"; ` _, err := tx.Exec(stmts) return err } // Add a new "arch" column to the "nodes" table. func updateFromV19(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec("PRAGMA ignore_check_constraints=on") if err != nil { return err } defer func() { _, _ = tx.Exec("PRAGMA ignore_check_constraints=off") }() // The column has a not-null constraint and a default value of // 0. However, leaving the 0 default won't effectively be accepted when // creating a new, due to the check constraint, so we are sure to end // up with a valid value. _, err = tx.Exec("ALTER TABLE nodes ADD COLUMN arch INTEGER NOT NULL DEFAULT 0 CHECK (arch > 0)") if err != nil { return err } arch, err := osarch.ArchitectureGetLocalID() if err != nil { return err } _, err = tx.Exec("UPDATE nodes SET arch = ?", arch) if err != nil { return err } return nil } // Rename 'containers' to 'instances' in *_used_by_ref views. func updateFromV18(ctx context.Context, tx *sql.Tx) error { stmts := ` DROP VIEW profiles_used_by_ref; CREATE VIEW profiles_used_by_ref (project, name, value) AS SELECT projects.name, profiles.name, printf('/1.0/instances/%s?project=%s', "instances".name, instances_projects.name) FROM profiles JOIN projects ON projects.id=profiles.project_id JOIN "instances_profiles" ON "instances_profiles".profile_id=profiles.id JOIN "instances" ON "instances".id="instances_profiles".instance_id JOIN projects AS instances_projects ON instances_projects.id="instances".project_id; DROP VIEW projects_used_by_ref; CREATE VIEW projects_used_by_ref (name, value) AS SELECT projects.name, printf('/1.0/instances/%s?project=%s', "instances".name, projects.name) FROM "instances" JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/images/%s', images.fingerprint) FROM images JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('/1.0/profiles/%s?project=%s', profiles.name, projects.name) FROM profiles JOIN projects ON project_id=projects.id; ` _, err := tx.Exec(stmts) return err } // Add nodes_roles table. func updateFromV17(ctx context.Context, tx *sql.Tx) error { stmts := ` CREATE TABLE nodes_roles ( node_id INTEGER NOT NULL, role INTEGER NOT NULL, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE, UNIQUE (node_id, role) ); ` _, err := tx.Exec(stmts) return err } // Add image type column. func updateFromV16(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec("ALTER TABLE images ADD COLUMN type INTEGER NOT NULL DEFAULT 0;") return err } // Create new snapshot tables and migrate data to them. func updateFromV15(ctx context.Context, tx *sql.Tx) error { stmts := ` CREATE TABLE instances_snapshots ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_id INTEGER NOT NULL, name TEXT NOT NULL, creation_date DATETIME NOT NULL DEFAULT 0, stateful INTEGER NOT NULL DEFAULT 0, description TEXT, expiry_date DATETIME, UNIQUE (instance_id, name), FOREIGN KEY (instance_id) REFERENCES instances (id) ON DELETE CASCADE ); CREATE TABLE instances_snapshots_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, instance_snapshot_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (instance_snapshot_id) REFERENCES instances_snapshots (id) ON DELETE CASCADE, UNIQUE (instance_snapshot_id, key) ); CREATE TABLE instances_snapshots_devices ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_snapshot_id INTEGER NOT NULL, name TEXT NOT NULL, type INTEGER NOT NULL default 0, FOREIGN KEY (instance_snapshot_id) REFERENCES instances_snapshots (id) ON DELETE CASCADE, UNIQUE (instance_snapshot_id, name) ); CREATE TABLE instances_snapshots_devices_config ( id INTEGER primary key AUTOINCREMENT NOT NULL, instance_snapshot_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (instance_snapshot_device_id) REFERENCES instances_snapshots_devices (id) ON DELETE CASCADE, UNIQUE (instance_snapshot_device_id, key) ); CREATE VIEW instances_snapshots_config_ref ( project, instance, name, key, value) AS SELECT projects.name, instances.name, instances_snapshots.name, instances_snapshots_config.key, instances_snapshots_config.value FROM instances_snapshots_config JOIN instances_snapshots ON instances_snapshots.id=instances_snapshots_config.instance_snapshot_id JOIN instances ON instances.id=instances_snapshots.instance_id JOIN projects ON projects.id=instances.project_id; CREATE VIEW instances_snapshots_devices_ref ( project, instance, name, device, type, key, value) AS SELECT projects.name, instances.name, instances_snapshots.name, instances_snapshots_devices.name, instances_snapshots_devices.type, coalesce(instances_snapshots_devices_config.key, ''), coalesce(instances_snapshots_devices_config.value, '') FROM instances_snapshots_devices LEFT OUTER JOIN instances_snapshots_devices_config ON instances_snapshots_devices_config.instance_snapshot_device_id=instances_snapshots_devices.id JOIN instances ON instances.id=instances_snapshots.instance_id JOIN projects ON projects.id=instances.project_id JOIN instances_snapshots ON instances_snapshots.id=instances_snapshots_devices.instance_snapshot_id ` _, err := tx.Exec(stmts) if err != nil { return fmt.Errorf("Failed to create snapshots tables: %w", err) } // Get the total number of rows in the instances table. count, err := query.Count(ctx, tx, "instances", "") if err != nil { return fmt.Errorf("Failed to count rows in instances table: %w", err) } // Fetch all rows in the instances table. type instance struct { ID int Name string Type int CreationDate time.Time Stateful bool Description string ExpiryDate sql.NullTime } sql := `SELECT id, name, type, creation_date, stateful, coalesce(description, ''), expiry_date FROM instances` instances := make([]instance, 0, count) err = query.Scan(ctx, tx, sql, func(scan func(dest ...any) error) error { inst := instance{} err := scan(&inst.ID, &inst.Name, &inst.Type, &inst.CreationDate, &inst.Stateful, &inst.Description, &inst.ExpiryDate) if err != nil { return err } instances = append(instances, inst) return nil }) if err != nil { return fmt.Errorf("Failed to fetch instances: %w", err) } // Create an index mapping instance names to their IDs. instanceIDsByName := make(map[string]int) for _, instance := range instances { if instance.Type == 1 { continue } instanceIDsByName[instance.Name] = instance.ID } // Fetch all rows in the instances_config table that references // snapshots and index them by instance ID. count, err = query.Count( ctx, tx, "instances_config JOIN instances ON instances_config.instance_id = instances.id", "instances.type = 1") if err != nil { return fmt.Errorf("Failed to count rows in instances_config table: %w", err) } type instanceConfig struct { ID int InstanceID int Key string Value string } configs := make([]instanceConfig, 0, count) sql = ` SELECT instances_config.id, instance_id, key, value FROM instances_config JOIN instances ON instances_config.instance_id = instances.id WHERE instances.type = 1 ` err = query.Scan(ctx, tx, sql, func(scan func(dest ...any) error) error { config := instanceConfig{} err := scan(&config.ID, &config.InstanceID, &config.Key, &config.Value) if err != nil { return err } configs = append(configs, config) return nil }) if err != nil { return fmt.Errorf("Failed to fetch snapshots config: %w", err) } configBySnapshotID := make(map[int]map[string]string) for _, config := range configs { c, ok := configBySnapshotID[config.InstanceID] if !ok { c = make(map[string]string) configBySnapshotID[config.InstanceID] = c } c[config.Key] = config.Value } // Fetch all rows in the instances_devices table that references // snapshots and index them by instance ID. count, err = query.Count( ctx, tx, "instances_devices JOIN instances ON instances_devices.instance_id = instances.id", "instances.type = 1") if err != nil { return fmt.Errorf("Failed to count rows in instances_devices table: %w", err) } type device struct { ID int InstanceID int Name string Type int } devices := make([]device, 0, count) sql = ` SELECT instances_devices.id, instance_id, instances_devices.name, instances_devices.type FROM instances_devices JOIN instances ON instances_devices.instance_id = instances.id WHERE instances.type = 1 ` err = query.Scan(ctx, tx, sql, func(scan func(dest ...any) error) error { d := device{} err := scan(&d.ID, &d.InstanceID, &d.Name, &d.Type) if err != nil { return err } devices = append(devices, d) return nil }) if err != nil { return fmt.Errorf("Failed to fetch snapshots devices: %w", err) } devicesBySnapshotID := make(map[int]map[string]struct { Type int Config map[string]string }) for _, device := range devices { d, ok := devicesBySnapshotID[device.InstanceID] if !ok { d = make(map[string]struct { Type int Config map[string]string }) devicesBySnapshotID[device.InstanceID] = d } // Fetch the config for this device. config, err := query.SelectConfig(ctx, tx, "instances_devices_config", "instance_device_id = ?", device.ID) if err != nil { return fmt.Errorf("Failed to fetch snapshots devices config: %w", err) } d[device.Name] = struct { Type int Config map[string]string }{ Type: device.Type, Config: config, } } // Migrate all snapshots to the new tables. for _, instance := range instances { if instance.Type == 0 { continue } // Figure out the instance and snapshot names. parts := strings.SplitN(instance.Name, internalInstance.SnapshotDelimiter, 2) if len(parts) != 2 { return fmt.Errorf("Snapshot %s has an invalid name", instance.Name) } instanceName := parts[0] instanceID, ok := instanceIDsByName[instanceName] if !ok { return fmt.Errorf("Found snapshot %s with no associated instance", instance.Name) } snapshotName := parts[1] // Insert a new row in instances_snapshots columns := []string{ "instance_id", "name", "creation_date", "stateful", "description", "expiry_date", } id, err := query.UpsertObject( tx, "instances_snapshots", columns, []any{ instanceID, snapshotName, instance.CreationDate, instance.Stateful, instance.Description, instance.ExpiryDate, }, ) if err != nil { return fmt.Errorf("Failed migrate snapshot %s: %w", instance.Name, err) } // Migrate the snapshot config for key, value := range configBySnapshotID[instance.ID] { columns := []string{ "instance_snapshot_id", "key", "value", } _, err := query.UpsertObject( tx, "instances_snapshots_config", columns, []any{ id, key, value, }, ) if err != nil { return fmt.Errorf("Failed migrate config %s/%s for snapshot %s: %w", key, value, instance.Name, err) } } // Migrate the snapshot devices for name, device := range devicesBySnapshotID[instance.ID] { columns := []string{ "instance_snapshot_id", "name", "type", } deviceID, err := query.UpsertObject( tx, "instances_snapshots_devices", columns, []any{ id, name, device.Type, }, ) if err != nil { return fmt.Errorf("Failed migrate device %s for snapshot %s: %w", name, instance.Name, err) } for key, value := range device.Config { columns := []string{ "instance_snapshot_device_id", "key", "value", } _, err := query.UpsertObject( tx, "instances_snapshots_devices_config", columns, []any{ deviceID, key, value, }, ) if err != nil { return fmt.Errorf("Failed migrate config %s/%s for device %s of snapshot %s: %w", key, value, name, instance.Name, err) } } } deleted, err := query.DeleteObject(tx, "instances", int64(instance.ID)) if err != nil { return fmt.Errorf("Failed to delete snapshot %s: %w", instance.Name, err) } if !deleted { return fmt.Errorf("Expected to delete snapshot %s", instance.Name) } } // Make sure that no snapshot is left in the instances table. count, err = query.Count(ctx, tx, "instances", "type = 1") if err != nil { return fmt.Errorf("Failed to count leftover snapshot rows: %w", err) } if count != 0 { return fmt.Errorf("Found %d unexpected snapshots left in instances table", count) } return nil } // Rename all containers* tables to instances*/. func updateFromV14(ctx context.Context, tx *sql.Tx) error { stmts := ` ALTER TABLE containers RENAME TO instances; ALTER TABLE containers_backups RENAME COLUMN container_id TO instance_id; ALTER TABLE containers_backups RENAME TO instances_backups; ALTER TABLE containers_config RENAME COLUMN container_id TO instance_id; ALTER TABLE containers_config RENAME TO instances_config; DROP VIEW containers_config_ref; CREATE VIEW instances_config_ref (project, node, name, key, value) AS SELECT projects.name, nodes.name, instances.name, instances_config.key, instances_config.value FROM instances_config JOIN instances ON instances.id=instances_config.instance_id JOIN projects ON projects.id=instances.project_id JOIN nodes ON nodes.id=instances.node_id; ALTER TABLE containers_devices RENAME COLUMN container_id TO instance_id; ALTER TABLE containers_devices RENAME TO instances_devices; ALTER TABLE containers_devices_config RENAME COLUMN container_device_id TO instance_device_id; ALTER TABLE containers_devices_config RENAME TO instances_devices_config; DROP VIEW containers_devices_ref; CREATE VIEW instances_devices_ref (project, node, name, device, type, key, value) AS SELECT projects.name, nodes.name, instances.name, instances_devices.name, instances_devices.type, coalesce(instances_devices_config.key, ''), coalesce(instances_devices_config.value, '') FROM instances_devices LEFT OUTER JOIN instances_devices_config ON instances_devices_config.instance_device_id=instances_devices.id JOIN instances ON instances.id=instances_devices.instance_id JOIN projects ON projects.id=instances.project_id JOIN nodes ON nodes.id=instances.node_id; DROP INDEX containers_node_id_idx; CREATE INDEX instances_node_id_idx ON instances (node_id); ALTER TABLE containers_profiles RENAME COLUMN container_id TO instance_id; ALTER TABLE containers_profiles RENAME TO instances_profiles; DROP VIEW containers_profiles_ref; CREATE VIEW instances_profiles_ref (project, node, name, value) AS SELECT projects.name, nodes.name, instances.name, profiles.name FROM instances_profiles JOIN instances ON instances.id=instances_profiles.instance_id JOIN profiles ON profiles.id=instances_profiles.profile_id JOIN projects ON projects.id=instances.project_id JOIN nodes ON nodes.id=instances.node_id ORDER BY instances_profiles.apply_order; DROP INDEX containers_project_id_and_name_idx; DROP INDEX containers_project_id_and_node_id_and_name_idx; DROP INDEX containers_project_id_and_node_id_idx; DROP INDEX containers_project_id_idx; CREATE INDEX instances_project_id_and_name_idx ON instances (project_id, name); CREATE INDEX instances_project_id_and_node_id_and_name_idx ON instances (project_id, node_id, name); CREATE INDEX instances_project_id_and_node_id_idx ON instances (project_id, node_id); CREATE INDEX instances_project_id_idx ON instances (project_id); DROP VIEW profiles_used_by_ref; CREATE VIEW profiles_used_by_ref (project, name, value) AS SELECT projects.name, profiles.name, printf('/1.0/containers/%s?project=%s', "instances".name, instances_projects.name) FROM profiles JOIN projects ON projects.id=profiles.project_id JOIN "instances_profiles" ON "instances_profiles".profile_id=profiles.id JOIN "instances" ON "instances".id="instances_profiles".instance_id JOIN projects AS instances_projects ON instances_projects.id="instances".project_id; ` _, err := tx.Exec(stmts) return err } func updateFromV13(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec("ALTER TABLE containers ADD COLUMN expiry_date DATETIME;") return err } func updateFromV12(ctx context.Context, tx *sql.Tx) error { stmts := ` DROP VIEW profiles_used_by_ref; CREATE VIEW profiles_used_by_ref (project, name, value) AS SELECT projects.name, profiles.name, printf('/1.0/containers/%s?project=%s', containers.name, containers_projects.name) FROM profiles JOIN projects ON projects.id=profiles.project_id JOIN containers_profiles ON containers_profiles.profile_id=profiles.id JOIN containers ON containers.id=containers_profiles.container_id JOIN projects AS containers_projects ON containers_projects.id=containers.project_id; ` _, err := tx.Exec(stmts) return err } func updateFromV11(ctx context.Context, tx *sql.Tx) error { // There was at least a case of dangling references to rows in the // containers table that don't exist anymore. So sanitize them before // we move forward. See #5176. stmts := ` DELETE FROM containers_config WHERE container_id NOT IN (SELECT id FROM containers); DELETE FROM containers_backups WHERE container_id NOT IN (SELECT id FROM containers); DELETE FROM containers_devices WHERE container_id NOT IN (SELECT id FROM containers); DELETE FROM containers_devices_config WHERE container_device_id NOT IN (SELECT id FROM containers_devices); DELETE FROM containers_profiles WHERE container_id NOT IN (SELECT id FROM containers); DELETE FROM containers_profiles WHERE profile_id NOT IN (SELECT id FROM profiles); DELETE FROM images_aliases WHERE image_id NOT IN (SELECT id FROM images); DELETE FROM images_properties WHERE image_id NOT IN (SELECT id FROM images); DELETE FROM images_source WHERE image_id NOT IN (SELECT id FROM images); DELETE FROM networks_config WHERE network_id NOT IN (SELECT id FROM networks); DELETE FROM profiles_config WHERE profile_id NOT IN (SELECT id FROM profiles); DELETE FROM profiles_devices WHERE profile_id NOT IN (SELECT id FROM profiles); DELETE FROM profiles_devices_config WHERE profile_device_id NOT IN (SELECT id FROM profiles_devices); DELETE FROM storage_pools_config WHERE storage_pool_id NOT IN (SELECT id FROM storage_pools); DELETE FROM storage_volumes WHERE storage_pool_id NOT IN (SELECT id FROM storage_pools); DELETE FROM storage_volumes_config WHERE storage_volume_id NOT IN (SELECT id FROM storage_volumes); ` _, err := tx.Exec(stmts) if err != nil { return fmt.Errorf("Remove dangling references to containers: %w", err) } // Before doing anything save the counts of all tables, so we can later // check that we don't accidentally delete or add anything. counts1, err := query.CountAll(ctx, tx) if err != nil { return fmt.Errorf("Failed to count rows in current tables: %w", err) } // Temporarily increase the cache size and disable page spilling, to // avoid unnecessary writes to the WAL. _, err = tx.Exec("PRAGMA cache_size=100000") if err != nil { return fmt.Errorf("Increase cache size: %w", err) } _, err = tx.Exec("PRAGMA cache_spill=0") if err != nil { return fmt.Errorf("Disable spilling cache pages to disk: %w", err) } // Use a large timeout since the update might take a while, due to the // new indexes being created. ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() stmts = ` CREATE TABLE projects ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT, UNIQUE (name) ); CREATE TABLE projects_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, project_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE, UNIQUE (project_id, key) ); CREATE VIEW projects_config_ref (name, key, value) AS SELECT projects.name, projects_config.key, projects_config.value FROM projects_config JOIN projects ON projects.id=projects_config.project_id; -- Insert the default project, with ID 1 INSERT INTO projects (name, description) VALUES ('default', 'Default Incus project'); INSERT INTO projects_config (project_id, key, value) VALUES (1, 'features.images', 'true'); INSERT INTO projects_config (project_id, key, value) VALUES (1, 'features.profiles', 'true'); -- Add a project_id column to all tables that need to be project-scoped. -- The column is added without the FOREIGN KEY constraint ALTER TABLE containers ADD COLUMN project_id INTEGER NOT NULL DEFAULT 1; ALTER TABLE images ADD COLUMN project_id INTEGER NOT NULL DEFAULT 1; ALTER TABLE images_aliases ADD COLUMN project_id INTEGER NOT NULL DEFAULT 1; ALTER TABLE profiles ADD COLUMN project_id INTEGER NOT NULL DEFAULT 1; ALTER TABLE storage_volumes ADD COLUMN project_id INTEGER NOT NULL DEFAULT 1; ALTER TABLE operations ADD COLUMN project_id INTEGER; -- Create new versions of the above tables, this time with the FOREIGN key constraint CREATE TABLE new_containers ( id INTEGER primary key AUTOINCREMENT NOT NULL, node_id INTEGER NOT NULL, name TEXT NOT NULL, architecture INTEGER NOT NULL, type INTEGER NOT NULL, ephemeral INTEGER NOT NULL DEFAULT 0, creation_date DATETIME NOT NULL DEFAULT 0, stateful INTEGER NOT NULL DEFAULT 0, last_use_date DATETIME, description TEXT, project_id INTEGER NOT NULL, UNIQUE (project_id, name), FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE ); CREATE TABLE new_images ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, fingerprint TEXT NOT NULL, filename TEXT NOT NULL, size INTEGER NOT NULL, public INTEGER NOT NULL DEFAULT 0, architecture INTEGER NOT NULL, creation_date DATETIME, expiry_date DATETIME, upload_date DATETIME NOT NULL, cached INTEGER NOT NULL DEFAULT 0, last_use_date DATETIME, auto_update INTEGER NOT NULL DEFAULT 0, project_id INTEGER NOT NULL, UNIQUE (project_id, fingerprint), FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE ); CREATE TABLE new_images_aliases ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, image_id INTEGER NOT NULL, description TEXT, project_id INTEGER NOT NULL, UNIQUE (project_id, name), FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE ); CREATE TABLE new_profiles ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT, project_id INTEGER NOT NULL, UNIQUE (project_id, name), FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE ); CREATE TABLE new_storage_volumes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER NOT NULL, type INTEGER NOT NULL, description TEXT, snapshot INTEGER NOT NULL DEFAULT 0, project_id INTEGER NOT NULL, UNIQUE (storage_pool_id, node_id, project_id, name, type), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE ); CREATE TABLE new_operations ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, uuid TEXT NOT NULL, node_id TEXT NOT NULL, type INTEGER NOT NULL DEFAULT 0, project_id INTEGER, UNIQUE (uuid), FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE, FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE ); -- Create copy version of all the tables that have direct or indirect references -- to the tables above, which we are going to drop. The copy just have the data, -- without FOREIGN KEY references. CREATE TABLE containers_backups_copy ( id INTEGER NOT NULL, container_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, creation_date DATETIME, expiry_date DATETIME, container_only INTEGER NOT NULL default 0, optimized_storage INTEGER NOT NULL default 0, UNIQUE (container_id, name) ); INSERT INTO containers_backups_copy SELECT * FROM containers_backups; CREATE TABLE containers_config_copy ( id INTEGER NOT NULL, container_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (container_id, key) ); INSERT INTO containers_config_copy SELECT * FROM containers_config; CREATE TABLE containers_devices_copy ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, name TEXT NOT NULL, type INTEGER NOT NULL default 0, UNIQUE (container_id, name) ); INSERT INTO containers_devices_copy SELECT * FROM containers_devices; CREATE TABLE containers_devices_config_copy ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (container_device_id, key) ); INSERT INTO containers_devices_config_copy SELECT * FROM containers_devices_config; CREATE TABLE containers_profiles_copy ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, profile_id INTEGER NOT NULL, apply_order INTEGER NOT NULL default 0, UNIQUE (container_id, profile_id) ); INSERT INTO containers_profiles_copy SELECT * FROM containers_profiles; CREATE TABLE images_aliases_copy ( id INTEGER NOT NULL, name TEXT NOT NULL, image_id INTEGER NOT NULL, description TEXT, project_id INTEGER NOT NULL, UNIQUE (name) ); INSERT INTO images_aliases_copy SELECT * FROM images_aliases; CREATE TABLE images_nodes_copy ( id INTEGER NOT NULL, image_id INTEGER NOT NULL, node_id INTEGER NOT NULL, UNIQUE (image_id, node_id) FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); INSERT INTO images_nodes_copy SELECT * FROM images_nodes; CREATE TABLE images_properties_copy ( id INTEGER NOT NULL, image_id INTEGER NOT NULL, type INTEGER NOT NULL, key TEXT NOT NULL, value TEXT ); INSERT INTO images_properties_copy SELECT * FROM images_properties; CREATE TABLE images_source_copy ( id INTEGER NOT NULL, image_id INTEGER NOT NULL, server TEXT NOT NULL, protocol INTEGER NOT NULL, certificate TEXT NOT NULL, alias TEXT NOT NULL ); INSERT INTO images_source_copy SELECT * FROM images_source; CREATE TABLE profiles_config_copy ( id INTEGER NOT NULL, profile_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (profile_id, key) ); INSERT INTO profiles_config_copy SELECT * FROM profiles_config; CREATE TABLE profiles_devices_copy ( id INTEGER NOT NULL, profile_id INTEGER NOT NULL, name TEXT NOT NULL, type INTEGER NOT NULL default 0, UNIQUE (profile_id, name) ); INSERT INTO profiles_devices_copy SELECT * FROM profiles_devices; CREATE TABLE profiles_devices_config_copy ( id INTEGER NOT NULL, profile_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (profile_device_id, key) ); INSERT INTO profiles_devices_config_copy SELECT * FROM profiles_devices_config; CREATE TABLE storage_volumes_config_copy ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (storage_volume_id, key) ); INSERT INTO storage_volumes_config_copy SELECT * FROM storage_volumes_config; -- Copy existing data into the new tables with the project_id reference INSERT INTO new_containers SELECT * FROM containers; INSERT INTO new_images SELECT * FROM images; INSERT INTO new_profiles SELECT * FROM profiles; INSERT INTO new_storage_volumes SELECT * FROM storage_volumes; INSERT INTO new_operations SELECT * FROM operations; -- Drop the old table and rename the new ones. This will trigger cascading -- deletes on all tables that have direct or indirect references to the old -- table, but we have a copy of them that we will use for restoring. DROP TABLE containers; ALTER TABLE new_containers RENAME TO containers; DROP TABLE images; ALTER TABLE new_images RENAME TO images; DROP TABLE profiles; ALTER TABLE new_profiles RENAME TO profiles; DROP TABLE storage_volumes; ALTER TABLE new_storage_volumes RENAME TO storage_volumes; INSERT INTO new_images_aliases SELECT * FROM images_aliases_copy; DROP TABLE images_aliases; DROP TABLE images_aliases_copy; ALTER TABLE new_images_aliases RENAME TO images_aliases; DROP TABLE operations; ALTER TABLE new_operations RENAME TO operations; -- Restore the content of the tables with direct or indirect references. INSERT INTO containers_backups SELECT * FROM containers_backups_copy; INSERT INTO containers_config SELECT * FROM containers_config_copy; INSERT INTO containers_devices SELECT * FROM containers_devices_copy; INSERT INTO containers_devices_config SELECT * FROM containers_devices_config_copy; INSERT INTO containers_profiles SELECT * FROM containers_profiles_copy; INSERT INTO images_nodes SELECT * FROM images_nodes_copy; INSERT INTO images_properties SELECT * FROM images_properties_copy; INSERT INTO images_source SELECT * FROM images_source_copy; INSERT INTO profiles_config SELECT * FROM profiles_config_copy; INSERT INTO profiles_devices SELECT * FROM profiles_devices_copy; INSERT INTO profiles_devices_config SELECT * FROM profiles_devices_config_copy; INSERT INTO storage_volumes_config SELECT * FROM storage_volumes_config_copy; -- Drop the copies. DROP TABLE containers_backups_copy; DROP TABLE containers_config_copy; DROP TABLE containers_devices_copy; DROP TABLE containers_devices_config_copy; DROP TABLE containers_profiles_copy; DROP TABLE images_nodes_copy; DROP TABLE images_properties_copy; DROP TABLE images_source_copy; DROP TABLE profiles_config_copy; DROP TABLE profiles_devices_copy; DROP TABLE profiles_devices_config_copy; DROP TABLE storage_volumes_config_copy; -- Create some indexes to speed up queries filtered by project ID and node ID CREATE INDEX containers_node_id_idx ON containers (node_id); CREATE INDEX containers_project_id_idx ON containers (project_id); CREATE INDEX containers_project_id_and_name_idx ON containers (project_id, name); CREATE INDEX containers_project_id_and_node_id_idx ON containers (project_id, node_id); CREATE INDEX containers_project_id_and_node_id_and_name_idx ON containers (project_id, node_id, name); CREATE INDEX images_project_id_idx ON images (project_id); CREATE INDEX images_aliases_project_id_idx ON images_aliases (project_id); CREATE INDEX profiles_project_id_idx ON profiles (project_id); ` _, err = tx.ExecContext(ctx, stmts) if err != nil { return fmt.Errorf("Failed to add project_id column: %w", err) } // Create a view to easily query all resources using a certain project stmt := fmt.Sprintf(` CREATE VIEW projects_used_by_ref (name, value) AS SELECT projects.name, printf('%s', containers.name, projects.name) FROM containers JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('%s', images.fingerprint) FROM images JOIN projects ON project_id=projects.id UNION SELECT projects.name, printf('%s', profiles.name, projects.name) FROM profiles JOIN projects ON project_id=projects.id `, EntityURIs[TypeContainer], EntityURIs[TypeImage], EntityURIs[TypeProfile]) _, err = tx.Exec(stmt) if err != nil { return fmt.Errorf("Failed to create projects_used_by_ref view: %w", err) } // Create a view to easily query all profiles used by a certain container stmt = ` CREATE VIEW containers_profiles_ref (project, node, name, value) AS SELECT projects.name, nodes.name, containers.name, profiles.name FROM containers_profiles JOIN containers ON containers.id=containers_profiles.container_id JOIN profiles ON profiles.id=containers_profiles.profile_id JOIN projects ON projects.id=containers.project_id JOIN nodes ON nodes.id=containers.node_id ORDER BY containers_profiles.apply_order ` _, err = tx.Exec(stmt) if err != nil { return fmt.Errorf("Failed to containers_profiles_ref view: %w", err) } // Create a view to easily query the config of a certain container. stmt = ` CREATE VIEW containers_config_ref (project, node, name, key, value) AS SELECT projects.name, nodes.name, containers.name, containers_config.key, containers_config.value FROM containers_config JOIN containers ON containers.id=containers_config.container_id JOIN projects ON projects.id=containers.project_id JOIN nodes ON nodes.id=containers.node_id ` _, err = tx.Exec(stmt) if err != nil { return fmt.Errorf("Failed to containers_config_ref view: %w", err) } // Create a view to easily query the devices of a certain container. stmt = ` CREATE VIEW containers_devices_ref (project, node, name, device, type, key, value) AS SELECT projects.name, nodes.name, containers.name, containers_devices.name, containers_devices.type, coalesce(containers_devices_config.key, ''), coalesce(containers_devices_config.value, '') FROM containers_devices LEFT OUTER JOIN containers_devices_config ON containers_devices_config.container_device_id=containers_devices.id JOIN containers ON containers.id=containers_devices.container_id JOIN projects ON projects.id=containers.project_id JOIN nodes ON nodes.id=containers.node_id ` _, err = tx.Exec(stmt) if err != nil { return fmt.Errorf("Failed to containers_devices_ref view: %w", err) } // Create a view to easily query the config of a certain profile. stmt = ` CREATE VIEW profiles_config_ref (project, name, key, value) AS SELECT projects.name, profiles.name, profiles_config.key, profiles_config.value FROM profiles_config JOIN profiles ON profiles.id=profiles_config.profile_id JOIN projects ON projects.id=profiles.project_id ` _, err = tx.Exec(stmt) if err != nil { return fmt.Errorf("Failed to profiles_config_ref view: %w", err) } // Create a view to easily query the devices of a certain profile. stmt = ` CREATE VIEW profiles_devices_ref (project, name, device, type, key, value) AS SELECT projects.name, profiles.name, profiles_devices.name, profiles_devices.type, coalesce(profiles_devices_config.key, ''), coalesce(profiles_devices_config.value, '') FROM profiles_devices LEFT OUTER JOIN profiles_devices_config ON profiles_devices_config.profile_device_id=profiles_devices.id JOIN profiles ON profiles.id=profiles_devices.profile_id JOIN projects ON projects.id=profiles.project_id ` _, err = tx.Exec(stmt) if err != nil { return fmt.Errorf("Failed to profiles_devices_ref view: %w", err) } // Create a view to easily query all resources using a certain profile stmt = fmt.Sprintf(` CREATE VIEW profiles_used_by_ref (project, name, value) AS SELECT projects.name, profiles.name, printf('%s', containers.name, projects.name) FROM profiles JOIN projects ON projects.id=profiles.project_id JOIN containers_profiles ON containers_profiles.profile_id=profiles.id JOIN containers ON containers.id=containers_profiles.container_id `, EntityURIs[TypeContainer]) _, err = tx.Exec(stmt) if err != nil { return fmt.Errorf("Failed to create profiles_used_by_ref view: %w", err) } // Check that the count of all rows in the database is unchanged // (i.e. we didn't accidentally delete or add anything). counts2, err := query.CountAll(ctx, tx) if err != nil { return fmt.Errorf("Failed to count rows in updated tables: %w", err) } delete(counts2, "projects") for table, count1 := range counts1 { if table == "sqlite_sequence" { continue } count2 := counts2[table] if count1 != count2 { return fmt.Errorf("Row count mismatch in table '%s': %d vs %d", table, count1, count2) } } // Restore default cache values. _, err = tx.Exec("PRAGMA cache_size=2000") if err != nil { return fmt.Errorf("Increase cache size: %w", err) } _, err = tx.Exec("PRAGMA cache_spill=1") if err != nil { return fmt.Errorf("Disable spilling cache pages to disk: %w", err) } return err } func updateFromV10(ctx context.Context, tx *sql.Tx) error { stmt := ` ALTER TABLE storage_volumes ADD COLUMN snapshot INTEGER NOT NULL DEFAULT 0; UPDATE storage_volumes SET snapshot = 0; ` _, err := tx.Exec(stmt) return err } // Add a new 'type' column to the operations table. func updateFromV9(ctx context.Context, tx *sql.Tx) error { stmts := ` ALTER TABLE operations ADD COLUMN type INTEGER NOT NULL DEFAULT 0; UPDATE operations SET type = 0; ` _, err := tx.Exec(stmts) return err } // The lvm.thinpool_name and lvm.vg_name config keys are node-specific and need // to be linked to nodes. func updateFromV8(ctx context.Context, tx *sql.Tx) error { // Moved to patchLvmNodeSpecificConfigKeys, since there's no schema // change. That makes it easier to backport. return nil } func updateFromV7(ctx context.Context, tx *sql.Tx) error { stmts := ` CREATE TABLE containers_backups ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, creation_date DATETIME, expiry_date DATETIME, container_only INTEGER NOT NULL default 0, optimized_storage INTEGER NOT NULL default 0, FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE, UNIQUE (container_id, name) ); ` _, err := tx.Exec(stmts) return err } // The zfs.pool_name config key is node-specific, and needs to be linked to // nodes. func updateFromV6(ctx context.Context, tx *sql.Tx) error { // Fetch the IDs of all existing nodes. nodeIDs, err := query.SelectIntegers(ctx, tx, "SELECT id FROM nodes") if err != nil { return fmt.Errorf("failed to get IDs of current nodes: %w", err) } // Fetch the IDs of all existing zfs pools. poolIDs, err := query.SelectIntegers(ctx, tx, `SELECT id FROM storage_pools WHERE driver='zfs'`) if err != nil { return fmt.Errorf("failed to get IDs of current zfs pools: %w", err) } for _, poolID := range poolIDs { // Fetch the config for this zfs pool and check if it has the zfs.pool_name key config, err := query.SelectConfig(ctx, tx, "storage_pools_config", "storage_pool_id=? AND node_id IS NULL", poolID) if err != nil { return fmt.Errorf("failed to fetch of zfs pool config: %w", err) } poolName, ok := config["zfs.pool_name"] if !ok { continue // This zfs storage pool does not have a zfs.pool_name config } // Delete the current zfs.pool_name key _, err = tx.Exec(` DELETE FROM storage_pools_config WHERE key='zfs.pool_name' AND storage_pool_id=? AND node_id IS NULL `, poolID) if err != nil { return fmt.Errorf("failed to delete zfs.pool_name config: %w", err) } // Add zfs.pool_name config entry for each node for _, nodeID := range nodeIDs { _, err := tx.Exec(` INSERT INTO storage_pools_config(storage_pool_id, node_id, key, value) VALUES(?, ?, 'zfs.pool_name', ?) `, poolID, nodeID, poolName) if err != nil { return fmt.Errorf("failed to create zfs.pool_name node config: %w", err) } } } return nil } // For ceph volumes, add node-specific rows for all existing nodes, since any // node is able to access those volumes. func updateFromV5(ctx context.Context, tx *sql.Tx) error { // Fetch the IDs of all existing nodes. nodeIDs, err := query.SelectIntegers(ctx, tx, "SELECT id FROM nodes") if err != nil { return fmt.Errorf("failed to get IDs of current nodes: %w", err) } // Fetch the IDs of all existing ceph volumes. volumeIDs, err := query.SelectIntegers(ctx, tx, ` SELECT storage_volumes.id FROM storage_volumes JOIN storage_pools ON storage_volumes.storage_pool_id=storage_pools.id WHERE storage_pools.driver='ceph' `) if err != nil { return fmt.Errorf("failed to get IDs of current ceph volumes: %w", err) } // Fetch all existing ceph volumes. type volume struct { ID int Name string StoragePoolID int NodeID int Type int Description string } volumes := make([]volume, 0, len(volumeIDs)) sql := ` SELECT storage_volumes.id, storage_volumes.name, storage_volumes.storage_pool_id, storage_volumes.node_id, storage_volumes.type, storage_volumes.description FROM storage_volumes JOIN storage_pools ON storage_volumes.storage_pool_id=storage_pools.id WHERE storage_pools.driver='ceph' ` err = query.Scan(ctx, tx, sql, func(scan func(dest ...any) error) error { vol := volume{} err := scan(&vol.ID, &vol.Name, &vol.StoragePoolID, &vol.NodeID, &vol.Type, &vol.Description) if err != nil { return err } volumes = append(volumes, vol) return nil }) if err != nil { return fmt.Errorf("failed to fetch current volumes: %w", err) } // Duplicate each volume row across all nodes, and keep track of the // new volume IDs that we've inserted. created := make(map[int][]int64) // Existing volume ID to new volumes IDs. columns := []string{"name", "storage_pool_id", "node_id", "type", "description"} for _, volume := range volumes { for _, nodeID := range nodeIDs { if volume.NodeID == nodeID { // This node already has the volume row continue } values := []any{ volume.Name, volume.StoragePoolID, nodeID, volume.Type, volume.Description, } id, err := query.UpsertObject(tx, "storage_volumes", columns, values) if err != nil { return fmt.Errorf("failed to insert new volume: %w", err) } _, ok := created[volume.ID] if !ok { created[volume.ID] = make([]int64, 0) } created[volume.ID] = append(created[volume.ID], id) } } // Duplicate each volume config row across all nodes. for id, newIDs := range created { config, err := query.SelectConfig(ctx, tx, "storage_volumes_config", "storage_volume_id=?", id) if err != nil { return fmt.Errorf("failed to fetch volume config: %w", err) } for _, newID := range newIDs { for key, value := range config { _, err := tx.Exec(` INSERT INTO storage_volumes_config(storage_volume_id, key, value) VALUES(?, ?, ?) `, newID, key, value) if err != nil { return fmt.Errorf("failed to insert new volume config: %w", err) } } } } return nil } func updateFromV4(ctx context.Context, tx *sql.Tx) error { stmt := "UPDATE networks SET state = 1" _, err := tx.Exec(stmt) return err } func updateFromV3(ctx context.Context, tx *sql.Tx) error { stmt := ` CREATE TABLE storage_pools_nodes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER NOT NULL, UNIQUE (storage_pool_id, node_id), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); ALTER TABLE storage_pools ADD COLUMN state INTEGER NOT NULL DEFAULT 0; UPDATE storage_pools SET state = 1; ` _, err := tx.Exec(stmt) return err } func updateFromV2(ctx context.Context, tx *sql.Tx) error { stmt := ` CREATE TABLE operations ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, uuid TEXT NOT NULL, node_id TEXT NOT NULL, UNIQUE (uuid), FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); ` _, err := tx.Exec(stmt) return err } func updateFromV1(ctx context.Context, tx *sql.Tx) error { stmt := ` CREATE TABLE certificates ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, fingerprint TEXT NOT NULL, type INTEGER NOT NULL, name TEXT NOT NULL, certificate TEXT NOT NULL, UNIQUE (fingerprint) ); CREATE TABLE config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (key) ); CREATE TABLE containers ( id INTEGER primary key AUTOINCREMENT NOT NULL, node_id INTEGER NOT NULL, name TEXT NOT NULL, architecture INTEGER NOT NULL, type INTEGER NOT NULL, ephemeral INTEGER NOT NULL DEFAULT 0, creation_date DATETIME NOT NULL DEFAULT 0, stateful INTEGER NOT NULL DEFAULT 0, last_use_date DATETIME, description TEXT, UNIQUE (name), FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); CREATE TABLE containers_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE, UNIQUE (container_id, key) ); CREATE TABLE containers_devices ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, name TEXT NOT NULL, type INTEGER NOT NULL default 0, FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE, UNIQUE (container_id, name) ); CREATE TABLE containers_devices_config ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (container_device_id) REFERENCES containers_devices (id) ON DELETE CASCADE, UNIQUE (container_device_id, key) ); CREATE TABLE containers_profiles ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, profile_id INTEGER NOT NULL, apply_order INTEGER NOT NULL default 0, UNIQUE (container_id, profile_id), FOREIGN KEY (container_id) REFERENCES containers(id) ON DELETE CASCADE, FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE ); CREATE TABLE images ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, fingerprint TEXT NOT NULL, filename TEXT NOT NULL, size INTEGER NOT NULL, public INTEGER NOT NULL DEFAULT 0, architecture INTEGER NOT NULL, creation_date DATETIME, expiry_date DATETIME, upload_date DATETIME NOT NULL, cached INTEGER NOT NULL DEFAULT 0, last_use_date DATETIME, auto_update INTEGER NOT NULL DEFAULT 0, UNIQUE (fingerprint) ); CREATE TABLE images_aliases ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, image_id INTEGER NOT NULL, description TEXT, FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE, UNIQUE (name) ); CREATE TABLE images_properties ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, type INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE ); CREATE TABLE images_source ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, server TEXT NOT NULL, protocol INTEGER NOT NULL, certificate TEXT NOT NULL, alias TEXT NOT NULL, FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE ); CREATE TABLE images_nodes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, node_id INTEGER NOT NULL, UNIQUE (image_id, node_id), FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); CREATE TABLE networks ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT, state INTEGER NOT NULL DEFAULT 0, UNIQUE (name) ); CREATE TABLE networks_nodes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER NOT NULL, UNIQUE (network_id, node_id), FOREIGN KEY (network_id) REFERENCES networks (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); CREATE TABLE networks_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER, key TEXT NOT NULL, value TEXT, UNIQUE (network_id, node_id, key), FOREIGN KEY (network_id) REFERENCES networks (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); CREATE TABLE profiles ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT, UNIQUE (name) ); CREATE TABLE profiles_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (profile_id, key), FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE ); CREATE TABLE profiles_devices ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_id INTEGER NOT NULL, name TEXT NOT NULL, type INTEGER NOT NULL default 0, UNIQUE (profile_id, name), FOREIGN KEY (profile_id) REFERENCES profiles (id) ON DELETE CASCADE ); CREATE TABLE profiles_devices_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (profile_device_id, key), FOREIGN KEY (profile_device_id) REFERENCES profiles_devices (id) ON DELETE CASCADE ); CREATE TABLE storage_pools ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, driver TEXT NOT NULL, description TEXT, UNIQUE (name) ); CREATE TABLE storage_pools_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER, key TEXT NOT NULL, value TEXT, UNIQUE (storage_pool_id, node_id, key), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); CREATE TABLE storage_volumes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER NOT NULL, type INTEGER NOT NULL, description TEXT, UNIQUE (storage_pool_id, node_id, name, type), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); CREATE TABLE storage_volumes_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (storage_volume_id, key), FOREIGN KEY (storage_volume_id) REFERENCES storage_volumes (id) ON DELETE CASCADE ); ` _, err := tx.Exec(stmt) return err } func updateFromV0(ctx context.Context, tx *sql.Tx) error { // v0..v1 the dawn of clustering stmt := ` CREATE TABLE nodes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT DEFAULT '', address TEXT NOT NULL, schema INTEGER NOT NULL, api_extensions INTEGER NOT NULL, heartbeat DATETIME DEFAULT CURRENT_TIMESTAMP, pending INTEGER NOT NULL DEFAULT 0, UNIQUE (name), UNIQUE (address) ); ` _, err := tx.Exec(stmt) return err } incus-7.0.0/internal/server/db/cluster/update_test.go000066400000000000000000000611721517523235500227160ustar00rootroot00000000000000package cluster_test import ( "context" "database/sql" "errors" "fmt" "testing" "time" "github.com/mattn/go-sqlite3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/shared/osarch" ) func TestUpdateFromV0(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(1, nil) require.NoError(t, err) stmt := "INSERT INTO nodes VALUES (1, 'foo', 'blah', '1.2.3.4:666', 1, 32, ?, 0)" _, err = db.Exec(stmt, time.Now()) require.NoError(t, err) // Unique constraint on name stmt = "INSERT INTO nodes VALUES (2, 'foo', 'gosh', '5.6.7.8:666', 5, 20, ?, 0)" _, err = db.Exec(stmt, time.Now()) require.Error(t, err) // Unique constraint on address stmt = "INSERT INTO nodes VALUES (3, 'bar', 'gasp', '1.2.3.4:666', 9, 11), ?, 0)" _, err = db.Exec(stmt, time.Now()) require.Error(t, err) } func TestUpdateFromV1_Certificates(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(2, nil) require.NoError(t, err) _, err = db.Exec("INSERT INTO certificates VALUES (1, 'abcd:efgh', 1, 'foo', 'FOO')") require.NoError(t, err) // Unique constraint on fingerprint. _, err = db.Exec("INSERT INTO certificates VALUES (2, 'abcd:efgh', 2, 'bar', 'BAR')") require.Error(t, err) } func TestUpdateFromV1_Config(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(2, nil) require.NoError(t, err) _, err = db.Exec("INSERT INTO config VALUES (1, 'foo', 'blah')") require.NoError(t, err) // Unique constraint on key. _, err = db.Exec("INSERT INTO config VALUES (2, 'foo', 'gosh')") require.Error(t, err) } func TestUpdateFromV1_Containers(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(2, nil) require.NoError(t, err) _, err = db.Exec("INSERT INTO nodes VALUES (1, 'one', '', '1.1.1.1', 666, 999, ?, 0)", time.Now()) require.NoError(t, err) _, err = db.Exec("INSERT INTO nodes VALUES (2, 'two', '', '2.2.2.2', 666, 999, ?, 0)", time.Now()) require.NoError(t, err) _, err = db.Exec(` INSERT INTO containers VALUES (1, 1, 'jammy', 1, 1, 0, ?, 0, ?, 'Jammy Jellyfish') `, time.Now(), time.Now()) require.NoError(t, err) // Unique constraint on name _, err = db.Exec(` INSERT INTO containers VALUES (2, 2, 'jammy', 2, 2, 1, ?, 1, ?, 'Ubuntu LTS') `, time.Now(), time.Now()) require.Error(t, err) // Cascading delete _, err = db.Exec("INSERT INTO containers_config VALUES (1, 1, 'thekey', 'thevalue')") require.NoError(t, err) _, err = db.Exec("DELETE FROM containers") require.NoError(t, err) result, err := db.Exec("DELETE FROM containers_config") require.NoError(t, err) n, err := result.RowsAffected() require.NoError(t, err) assert.Equal(t, int64(0), n) // The row was already deleted by the previous query } func TestUpdateFromV1_Network(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(2, nil) require.NoError(t, err) _, err = db.Exec("INSERT INTO networks VALUES (1, 'foo', 'blah', 1)") require.NoError(t, err) // Unique constraint on name. _, err = db.Exec("INSERT INTO networks VALUES (2, 'foo', 'gosh', 1)") require.Error(t, err) } func TestUpdateFromV1_ConfigTables(t *testing.T) { testConfigTable(t, "networks", func(db *sql.DB) { _, err := db.Exec("INSERT INTO networks VALUES (1, 'foo', 'blah', 1)") require.NoError(t, err) }) testConfigTable(t, "storage_pools", func(db *sql.DB) { _, err := db.Exec("INSERT INTO storage_pools VALUES (1, 'default', 'dir', '')") require.NoError(t, err) }) } func testConfigTable(t *testing.T, table string, setup func(db *sql.DB)) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(2, nil) require.NoError(t, err) _, err = db.Exec("INSERT INTO nodes VALUES (1, 'one', '', '1.1.1.1', 666, 999, ?, 0)", time.Now()) require.NoError(t, err) _, err = db.Exec("INSERT INTO nodes VALUES (2, 'two', '', '2.2.2.2', 666, 999, ?, 0)", time.Now()) require.NoError(t, err) stmt := func(format string) string { return fmt.Sprintf(format, table) } setup(db) _, err = db.Exec(stmt("INSERT INTO %s_config VALUES (1, 1, 1, 'bar', 'baz')")) require.NoError(t, err) // Unique constraint on _id/node_id/key. _, err = db.Exec(stmt("INSERT INTO %s_config VALUES (2, 1, 1, 'bar', 'egg')")) require.Error(t, err) _, err = db.Exec(stmt("INSERT INTO %s_config VALUES (3, 1, 2, 'bar', 'egg')")) require.NoError(t, err) // Reference constraint on _id. _, err = db.Exec(stmt("INSERT INTO %s_config VALUES (4, 2, 1, 'fuz', 'buz')")) require.Error(t, err) // Reference constraint on node_id. _, err = db.Exec(stmt("INSERT INTO %s_config VALUES (5, 1, 3, 'fuz', 'buz')")) require.Error(t, err) // Cascade deletes on node_id result, err := db.Exec("DELETE FROM nodes WHERE id=2") require.NoError(t, err) n, err := result.RowsAffected() require.NoError(t, err) assert.Equal(t, int64(1), n) result, err = db.Exec(stmt("UPDATE %s_config SET value='yuk'")) require.NoError(t, err) n, err = result.RowsAffected() require.NoError(t, err) assert.Equal(t, int64(1), n) // Only one row was affected, since the other got deleted // Cascade deletes on _id result, err = db.Exec(stmt("DELETE FROM %s")) require.NoError(t, err) n, err = result.RowsAffected() require.NoError(t, err) assert.Equal(t, int64(1), n) result, err = db.Exec(stmt("DELETE FROM %s_config")) require.NoError(t, err) n, err = result.RowsAffected() require.NoError(t, err) assert.Equal(t, int64(0), n) // The row was already deleted by the previous query } func TestUpdateFromV2(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(3, nil) require.NoError(t, err) _, err = db.Exec("INSERT INTO nodes VALUES (1, 'one', '', '1.1.1.1', 666, 999, ?, 0)", time.Now()) require.NoError(t, err) _, err = db.Exec("INSERT INTO operations VALUES (1, 'abcd', 1)") require.NoError(t, err) // Unique constraint on uuid _, err = db.Exec("INSERT INTO operations VALUES (2, 'abcd', 1)") require.Error(t, err) // Cascade delete on node_id _, err = db.Exec("DELETE FROM nodes") require.NoError(t, err) result, err := db.Exec("DELETE FROM operations") require.NoError(t, err) n, err := result.RowsAffected() require.NoError(t, err) assert.Equal(t, int64(0), n) } func TestUpdateFromV3(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(4, nil) require.NoError(t, err) _, err = db.Exec("INSERT INTO nodes VALUES (1, 'c1', '', '1.1.1.1', 666, 999, ?, 0)", time.Now()) require.NoError(t, err) _, err = db.Exec("INSERT INTO storage_pools VALUES (1, 'p1', 'zfs', '', 0)") require.NoError(t, err) _, err = db.Exec("INSERT INTO storage_pools_nodes VALUES (1, 1, 1)") require.NoError(t, err) // Unique constraint on storage_pool_id/node_id _, err = db.Exec("INSERT INTO storage_pools_nodes VALUES (1, 1, 1)") require.Error(t, err) } func TestUpdateFromV5(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(6, func(db *sql.DB) { // Create two nodes. _, err := db.Exec( "INSERT INTO nodes VALUES (1, 'n1', '', '1.2.3.4:666', 1, 32, ?, 0)", time.Now()) require.NoError(t, err) _, err = db.Exec( "INSERT INTO nodes VALUES (2, 'n2', '', '5.6.7.8:666', 1, 32, ?, 0)", time.Now()) require.NoError(t, err) // Create a pool p1 of type zfs. _, err = db.Exec("INSERT INTO storage_pools VALUES (1, 'p1', 'zfs', '', 0)") require.NoError(t, err) // Create a pool p2 of type ceph. _, err = db.Exec("INSERT INTO storage_pools VALUES (2, 'p2', 'ceph', '', 0)") // Create a volume v1 on pool p1, associated with n1 and a config. require.NoError(t, err) _, err = db.Exec("INSERT INTO storage_volumes VALUES (1, 'v1', 1, 1, 1, '')") require.NoError(t, err) _, err = db.Exec("INSERT INTO storage_volumes_config VALUES (1, 1, 'k', 'v')") require.NoError(t, err) // Create a volume v1 on pool p2, associated with n1 and a config. require.NoError(t, err) _, err = db.Exec("INSERT INTO storage_volumes VALUES (2, 'v1', 2, 1, 1, '')") require.NoError(t, err) _, err = db.Exec("INSERT INTO storage_volumes_config VALUES (2, 2, 'k', 'v')") require.NoError(t, err) // Create a volume v2 on pool p2, associated with n2 and no config. require.NoError(t, err) _, err = db.Exec("INSERT INTO storage_volumes VALUES (3, 'v2', 2, 2, 1, '')") require.NoError(t, err) }) require.NoError(t, err) // Check that a volume row for n2 was added for v1 on p2. tx, err := db.Begin() require.NoError(t, err) defer func() { _ = tx.Rollback() }() nodeIDs, err := query.SelectIntegers(context.Background(), tx, ` SELECT node_id FROM storage_volumes WHERE storage_pool_id=2 AND name='v1' ORDER BY node_id `) require.NoError(t, err) require.Equal(t, []int{1, 2}, nodeIDs) // Check that a volume row for n1 was added for v2 on p2. nodeIDs, err = query.SelectIntegers(context.Background(), tx, ` SELECT node_id FROM storage_volumes WHERE storage_pool_id=2 AND name='v2' ORDER BY node_id `) require.NoError(t, err) require.Equal(t, []int{1, 2}, nodeIDs) // Check that the config for volume v1 on p2 was duplicated. volumeIDs, err := query.SelectIntegers(context.Background(), tx, ` SELECT id FROM storage_volumes WHERE storage_pool_id=2 AND name='v1' ORDER BY id `) require.NoError(t, err) require.Equal(t, []int{2, 4}, volumeIDs) config1, err := query.SelectConfig(context.Background(), tx, "storage_volumes_config", "storage_volume_id=?", volumeIDs[0]) require.NoError(t, err) config2, err := query.SelectConfig(context.Background(), tx, "storage_volumes_config", "storage_volume_id=?", volumeIDs[1]) require.NoError(t, err) require.Equal(t, config1, config2) } func TestUpdateFromV6(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(7, func(db *sql.DB) { // Create two nodes. _, err := db.Exec( "INSERT INTO nodes VALUES (1, 'n1', '', '1.2.3.4:666', 1, 32, ?, 0)", time.Now()) require.NoError(t, err) _, err = db.Exec( "INSERT INTO nodes VALUES (2, 'n2', '', '5.6.7.8:666', 1, 32, ?, 0)", time.Now()) require.NoError(t, err) // Create a pool p1 of type zfs. _, err = db.Exec("INSERT INTO storage_pools VALUES (1, 'p1', 'zfs', '', 0)") require.NoError(t, err) // Create a pool p2 of type zfs. _, err = db.Exec("INSERT INTO storage_pools VALUES (2, 'p2', 'zfs', '', 0)") require.NoError(t, err) // Create a zfs.pool_name config for p1. _, err = db.Exec(` INSERT INTO storage_pools_config(storage_pool_id, node_id, key, value) VALUES(1, NULL, 'zfs.pool_name', 'my-pool') `) require.NoError(t, err) // Create a zfs.clone_copy config for p2. _, err = db.Exec(` INSERT INTO storage_pools_config(storage_pool_id, node_id, key, value) VALUES(2, NULL, 'zfs.clone_copy', 'true') `) require.NoError(t, err) }) require.NoError(t, err) tx, err := db.Begin() require.NoError(t, err) defer func() { _ = tx.Rollback() }() // Check the zfs.pool_name config is now node-specific. for _, nodeID := range []int{1, 2} { config, err := query.SelectConfig(context.Background(), tx, "storage_pools_config", "storage_pool_id=1 AND node_id=?", nodeID) require.NoError(t, err) assert.Equal(t, map[string]string{"zfs.pool_name": "my-pool"}, config) } // Check the zfs.clone_copy is still global config, err := query.SelectConfig(context.Background(), tx, "storage_pools_config", "storage_pool_id=2 AND node_id IS NULL") require.NoError(t, err) assert.Equal(t, map[string]string{"zfs.clone_copy": "true"}, config) } func TestUpdateFromV9(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(10, func(db *sql.DB) { // Create a node. _, err := db.Exec( "INSERT INTO nodes VALUES (1, 'n1', '', '1.2.3.4:666', 1, 32, ?, 0)", time.Now()) require.NoError(t, err) // Create an operation. _, err = db.Exec("INSERT INTO operations VALUES (1, 'op1', 1)") require.NoError(t, err) }) require.NoError(t, err) // Check that a type column has been added and that existing rows get type 0. tx, err := db.Begin() require.NoError(t, err) defer func() { _ = tx.Rollback() }() types, err := query.SelectIntegers(context.Background(), tx, `SELECT type FROM operations`) require.NoError(t, err) require.Equal(t, []int{0}, types) } func TestUpdateFromV11(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(12, func(db *sql.DB) { // Insert a node. _, err := db.Exec( "INSERT INTO nodes VALUES (1, 'n1', '', '1.2.3.4:666', 1, 32, ?, 0)", time.Now()) require.NoError(t, err) // Insert a container. _, err = db.Exec(` INSERT INTO containers VALUES (1, 1, 'bionic', 1, 1, 0, ?, 0, ?, 'Bionic Beaver') `, time.Now(), time.Now()) require.NoError(t, err) // Insert an image. _, err = db.Exec(` INSERT INTO images VALUES (1, 'abcd', 'img.tgz', 123, 0, 0, NULL, NULL, ?, 0, NULL, 0) `, time.Now()) require.NoError(t, err) // Insert an image alias. _, err = db.Exec(` INSERT INTO images_aliases VALUES (1, 'my-img', 1, NULL) `, time.Now()) require.NoError(t, err) // Insert some profiles. _, err = db.Exec(` INSERT INTO profiles VALUES (1, 'default', NULL); INSERT INTO profiles VALUES(2, 'users', ''); INSERT INTO profiles_config VALUES(2, 2, 'boot.autostart', 'false'); INSERT INTO profiles_config VALUES(3, 2, 'limits.cpu.allowance', '50%'); INSERT INTO profiles_devices VALUES(1, 1, 'eth0', 1); INSERT INTO profiles_devices VALUES(2, 1, 'root', 1); INSERT INTO profiles_devices_config VALUES(1, 1, 'nictype', 'bridged'); INSERT INTO profiles_devices_config VALUES(2, 1, 'parent', 'incusbr0'); INSERT INTO profiles_devices_config VALUES(3, 2, 'path', '/'); INSERT INTO profiles_devices_config VALUES(4, 2, 'pool', 'default'); `, time.Now()) require.NoError(t, err) }) require.NoError(t, err) tx, err := db.Begin() require.NoError(t, err) defer func() { _ = tx.Rollback() }() // Check that a project_id column has been added to the various talbles // and that existing rows default to 1 (the ID of the default project). for _, table := range []string{"containers", "images", "images_aliases"} { count, err := query.Count(context.Background(), tx, table, "") require.NoError(t, err) assert.Equal(t, 1, count) stmt := fmt.Sprintf("SELECT project_id FROM %s", table) ids, err := query.SelectIntegers(context.Background(), tx, stmt) require.NoError(t, err) assert.Equal(t, []int{1}, ids) } // Create a new project. _, err = tx.Exec(` INSERT INTO projects VALUES (2, 'staging', 'Staging environment')`) require.NoError(t, err) // Check that it's possible to have two containers with the same name // as long as they are in different projects. _, err = tx.Exec(` INSERT INTO containers VALUES (2, 1, 'xenial', 1, 1, 0, ?, 0, ?, 'Xenial Xerus', 1) `, time.Now(), time.Now()) require.NoError(t, err) _, err = tx.Exec(` INSERT INTO containers VALUES (3, 1, 'xenial', 1, 1, 0, ?, 0, ?, 'Xenial Xerus', 2) `, time.Now(), time.Now()) require.NoError(t, err) // Check that it's not possible to have two containers with the same name // in the same project. _, err = tx.Exec(` INSERT INTO containers VALUES (4, 1, 'xenial', 1, 1, 0, ?, 0, ?, 'Xenial Xerus', 1) `, time.Now(), time.Now()) assert.EqualError(t, err, "UNIQUE constraint failed: containers.project_id, containers.name") } func TestUpdateFromV14(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(15, func(db *sql.DB) { // Insert a node. _, err := db.Exec( "INSERT INTO nodes VALUES (1, 'n1', '', '1.2.3.4:666', 1, 32, ?, 0)", time.Now()) require.NoError(t, err) // Insert a container. _, err = db.Exec(` INSERT INTO containers VALUES (1, 1, 'eoan', 1, 1, 0, ?, 0, ?, 'Eoan Ermine', 1, NULL) `, time.Now(), time.Now()) require.NoError(t, err) }) require.NoError(t, err) tx, err := db.Begin() require.NoError(t, err) defer func() { _ = tx.Rollback() }() // Check that the new instances table can be queried. count, err := query.Count(context.Background(), tx, "instances", "") require.NoError(t, err) assert.Equal(t, 1, count) } func TestUpdateFromV15(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(16, func(db *sql.DB) { // Insert a node. _, err := db.Exec( "INSERT INTO nodes VALUES (1, 'n1', '', '1.2.3.4:666', 1, 32, ?, 0)", time.Now()) require.NoError(t, err) // Insert an instance. _, err = db.Exec(` INSERT INTO instances VALUES (1, 1, 'eoan', 2, 0, 0, ?, 0, ?, NULL, 1, ?) `, time.Now(), time.Now(), time.Now()) require.NoError(t, err) _, err = db.Exec("INSERT INTO instances_config VALUES (1, 1, 'key', 'value2')") require.NoError(t, err) _, err = db.Exec("INSERT INTO instances_devices VALUES (1, 1, 'dev', 0)") require.NoError(t, err) _, err = db.Exec("INSERT INTO instances_devices_config VALUES (1, 1, 'k', 'v')") require.NoError(t, err) // Insert an instance snapshot. expiryDate := time.Date(2019, 8, 14, 11, 9, 0, 0, time.UTC) _, err = db.Exec(` INSERT INTO instances VALUES (2, 1, 'eoan/snap', 2, 1, 0, ?, 0, ?, 'Eoan Ermine Snapshot', 1, ?) `, time.Now(), time.Now(), expiryDate) require.NoError(t, err) _, err = db.Exec("INSERT INTO instances_config VALUES (2, 2, 'key', 'value1')") require.NoError(t, err) _, err = db.Exec("INSERT INTO instances_devices VALUES (2, 2, 'dev', 0)") require.NoError(t, err) _, err = db.Exec("INSERT INTO instances_devices_config VALUES (2, 2, 'k', 'v')") require.NoError(t, err) }) require.NoError(t, err) tx, err := db.Begin() require.NoError(t, err) defer func() { _ = tx.Rollback() }() // Check that snapshots were migrated to the new tables. count, err := query.Count(context.Background(), tx, "instances", "") require.NoError(t, err) assert.Equal(t, 1, count) count, err = query.Count(context.Background(), tx, "instances_config", "") require.NoError(t, err) assert.Equal(t, 1, count) count, err = query.Count(context.Background(), tx, "instances_devices", "") require.NoError(t, err) assert.Equal(t, 1, count) count, err = query.Count(context.Background(), tx, "instances_devices_config", "") require.NoError(t, err) assert.Equal(t, 1, count) count, err = query.Count(context.Background(), tx, "instances_snapshots", "") require.NoError(t, err) assert.Equal(t, 1, count) count, err = query.Count(context.Background(), tx, "instances_snapshots_config", "") require.NoError(t, err) assert.Equal(t, 1, count) count, err = query.Count(context.Background(), tx, "instances_snapshots_devices", "") require.NoError(t, err) assert.Equal(t, 1, count) count, err = query.Count(context.Background(), tx, "instances_snapshots_devices_config", "") require.NoError(t, err) assert.Equal(t, 1, count) config, err := query.SelectConfig(context.Background(), tx, "instances_config", "id = 1") require.NoError(t, err) assert.Equal(t, config, map[string]string{"key": "value2"}) config, err = query.SelectConfig(context.Background(), tx, "instances_snapshots_config", "id = 1") require.NoError(t, err) assert.Equal(t, config, map[string]string{"key": "value1"}) config, err = query.SelectConfig(context.Background(), tx, "instances_devices_config", "id = 1") require.NoError(t, err) assert.Equal(t, config, map[string]string{"k": "v"}) config, err = query.SelectConfig(context.Background(), tx, "instances_snapshots_devices_config", "id = 1") require.NoError(t, err) assert.Equal(t, config, map[string]string{"k": "v"}) } func TestUpdateFromV19(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(20, func(db *sql.DB) { // Insert a node. _, err := db.Exec( "INSERT INTO nodes VALUES (1, 'n1', '', '1.2.3.4:666', 1, 32, ?, 0)", time.Now()) require.NoError(t, err) }) require.NoError(t, err) defer func() { _ = db.Close() }() expectedArch, err := osarch.ArchitectureGetLocalID() require.NoError(t, err) row := db.QueryRow("SELECT arch FROM nodes") arch := 0 err = row.Scan(&arch) require.NoError(t, err) assert.Equal(t, expectedArch, arch) // Trying to create a row without specifying the architecture results // in an error. _, err = db.Exec(` INSERT INTO nodes(id, name, description, address, schema, api_extensions, heartbeat, pending) VALUES (2, 'n2', '', '2.2.3.4:666', 1, 32, ?, 0)`, time.Now()) if err == nil { t.Fatal("expected insertion to fail") } var sqliteErr sqlite3.Error ok := errors.As(err, &sqliteErr) require.True(t, ok) assert.Equal(t, sqliteErr.Code, sqlite3.ErrConstraint) } func TestUpdateFromV25(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(26, func(db *sql.DB) { // Insert a node. _, err := db.Exec( "INSERT INTO nodes VALUES (1, 'n1', '', '1.2.3.4:666', 1, 32, ?, 0, 1)", time.Now()) require.NoError(t, err) // Insert a pool _, err = db.Exec("INSERT INTO storage_pools VALUES (1, 'p1', 'zfs', '', 0)") require.NoError(t, err) // Create a volume v1 on pool p1, associated with n1 and a config. _, err = db.Exec("INSERT INTO storage_volumes VALUES (1, 'v1', 1, 1, 1, '', 0, 1)") require.NoError(t, err) _, err = db.Exec("INSERT INTO storage_volumes_config VALUES (1, 1, 'k', 'v')") require.NoError(t, err) // Create a snapshot v1/snap0 with a config. _, err = db.Exec("INSERT INTO storage_volumes VALUES (2, 'v1/snap0', 1, 1, 1, '', 1, 1)") require.NoError(t, err) _, err = db.Exec("INSERT INTO storage_volumes_config VALUES (2, 2, 'k', 'v-old')") require.NoError(t, err) }) require.NoError(t, err) defer func() { _ = db.Close() }() tx, err := db.Begin() require.NoError(t, err) defer func() { _ = tx.Rollback() }() // Check that regular volumes were kept. count, err := query.Count(context.Background(), tx, "storage_volumes", "") require.NoError(t, err) assert.Equal(t, 1, count) count, err = query.Count(context.Background(), tx, "storage_volumes_config", "") require.NoError(t, err) assert.Equal(t, 1, count) // Check that volume snapshots were migrated. count, err = query.Count(context.Background(), tx, "storage_volumes_snapshots", "") require.NoError(t, err) assert.Equal(t, 1, count) config, err := query.SelectConfig(context.Background(), tx, "storage_volumes_snapshots_config", "") require.NoError(t, err) assert.Len(t, config, 1) assert.Equal(t, config["k"], "v-old") } func TestUpdateFromV26_WithoutVolumes(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(27, func(db *sql.DB) {}) require.NoError(t, err) defer func() { _ = db.Close() }() } func TestUpdateFromV26_WithVolumes(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(27, func(db *sql.DB) { // Insert a node. _, err := db.Exec( "INSERT INTO nodes VALUES (1, 'n1', '', '1.2.3.4:666', 1, 32, ?, 0, 1)", time.Now()) require.NoError(t, err) // Insert a pool _, err = db.Exec("INSERT INTO storage_pools VALUES (1, 'p1', 'zfs', '', 0)") require.NoError(t, err) // Create a volume v1 on pool p1 _, err = db.Exec("INSERT INTO storage_volumes VALUES (1, 'v1', 1, 1, 1, '', 1)") require.NoError(t, err) // Create a snapshot snap0. _, err = db.Exec("INSERT INTO storage_volumes_snapshots VALUES (2, 1, 'snap0', '')") require.NoError(t, err) // Mess up the sqlite_sequence value. _, err = db.Exec("UPDATE sqlite_sequence SET seq = 1 WHERE name = 'storage_volumes'") require.NoError(t, err) }) require.NoError(t, err) defer func() { _ = db.Close() }() tx, err := db.Begin() require.NoError(t, err) defer func() { _ = tx.Rollback() }() ids, err := query.SelectIntegers(context.Background(), tx, "SELECT seq FROM sqlite_sequence WHERE name = 'storage_volumes'") require.NoError(t, err) assert.Equal(t, ids[0], 2) } func TestUpdateFromV34(t *testing.T) { schema := cluster.Schema() db, err := schema.ExerciseUpdate(35, func(db *sql.DB) { // Insert two nodes. _, err := db.Exec( "INSERT INTO nodes VALUES (1, 'n1', '', '1.2.3.4:666', 1, 32, ?, 0, 1, NULL)", time.Now()) require.NoError(t, err) _, err = db.Exec( "INSERT INTO nodes VALUES (2, 'n2', '', '5.6.7.8:666', 1, 32, ?, 0, 1, NULL)", time.Now()) require.NoError(t, err) // Insert a storage pool. _, err = db.Exec("INSERT INTO storage_pools VALUES (1, 'p1', 'ceph', NULL, 0)") require.NoError(t, err) // Create two rows for the same volume on different nodes. _, err = db.Exec("INSERT INTO storage_volumes VALUES (1, 'v1', 1, 1, 1, NULL, 1, 0)") require.NoError(t, err) _, err = db.Exec("INSERT INTO storage_volumes VALUES (2, 'v1', 1, 2, 1, NULL, 1, 0)") require.NoError(t, err) }) require.NoError(t, err) defer func() { _ = db.Close() }() tx, err := db.Begin() require.NoError(t, err) defer func() { _ = tx.Rollback() }() // Only one volume is left and it's node ID is set to NULL. count, err := query.Count(context.Background(), tx, "storage_volumes", "") require.NoError(t, err) assert.Equal(t, count, 1) row := tx.QueryRow("SELECT id, node_id FROM storage_volumes") var id int var nodeID any require.NoError(t, row.Scan(&id, &nodeID)) assert.Equal(t, id, 2) assert.Equal(t, nodeID, nil) } incus-7.0.0/internal/server/db/cluster/warnings.go000066400000000000000000000053771517523235500222320ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import ( "time" "github.com/lxc/incus/v7/internal/server/db/warningtype" "github.com/lxc/incus/v7/shared/api" ) // Code generation directives. // //generate-database:mapper target warnings.mapper.go //generate-database:mapper reset -i -b "//go:build linux && cgo && !agent" // //generate-database:mapper stmt -e warning objects //generate-database:mapper stmt -e warning objects-by-UUID //generate-database:mapper stmt -e warning objects-by-Project //generate-database:mapper stmt -e warning objects-by-Status //generate-database:mapper stmt -e warning objects-by-Node-and-TypeCode //generate-database:mapper stmt -e warning objects-by-Node-and-TypeCode-and-Project //generate-database:mapper stmt -e warning objects-by-Node-and-TypeCode-and-Project-and-EntityTypeCode-and-EntityID //generate-database:mapper stmt -e warning delete-by-UUID //generate-database:mapper stmt -e warning delete-by-EntityTypeCode-and-EntityID //generate-database:mapper stmt -e warning id // //generate-database:mapper method -i -e warning GetMany //generate-database:mapper method -i -e warning GetOne-by-UUID //generate-database:mapper method -i -e warning DeleteOne-by-UUID //generate-database:mapper method -i -e warning DeleteMany-by-EntityTypeCode-and-EntityID //generate-database:mapper method -i -e warning ID //generate-database:mapper method -i -e warning Exists struct=Warning // Warning is a value object holding db-related details about a warning. type Warning struct { ID int Node string `db:"coalesce=''&leftjoin=nodes.name"` Project string `db:"coalesce=''&leftjoin=projects.name"` EntityTypeCode int `db:"coalesce=-1"` EntityID int `db:"coalesce=-1"` UUID string `db:"primary=yes"` TypeCode warningtype.Type Status warningtype.Status FirstSeenDate time.Time LastSeenDate time.Time UpdatedDate time.Time LastMessage string Count int } // WarningFilter specifies potential query parameter fields. type WarningFilter struct { ID *int UUID *string Project *string Node *string TypeCode *warningtype.Type EntityTypeCode *int EntityID *int Status *warningtype.Status } // ToAPI returns an API entry. func (w Warning) ToAPI() api.Warning { typeCode := warningtype.Type(w.TypeCode) return api.Warning{ WarningPut: api.WarningPut{ Status: warningtype.Statuses[warningtype.Status(w.Status)], }, UUID: w.UUID, Location: w.Node, Project: w.Project, Type: warningtype.TypeNames[typeCode], Count: w.Count, FirstSeenAt: w.FirstSeenDate, LastSeenAt: w.LastSeenDate, LastMessage: w.LastMessage, Severity: warningtype.Severities[typeCode.Severity()], } } incus-7.0.0/internal/server/db/cluster/warnings.interface.mapper.go000066400000000000000000000023361517523235500254440ustar00rootroot00000000000000//go:build linux && cgo && !agent package cluster import "context" // WarningGenerated is an interface of generated methods for Warning. type WarningGenerated interface { // GetWarnings returns all available warnings. // generator: warning GetMany GetWarnings(ctx context.Context, db dbtx, filters ...WarningFilter) ([]Warning, error) // GetWarning returns the warning with the given key. // generator: warning GetOne-by-UUID GetWarning(ctx context.Context, db dbtx, uuid string) (*Warning, error) // DeleteWarning deletes the warning matching the given key parameters. // generator: warning DeleteOne-by-UUID DeleteWarning(ctx context.Context, db dbtx, uuid string) error // DeleteWarnings deletes the warning matching the given key parameters. // generator: warning DeleteMany-by-EntityTypeCode-and-EntityID DeleteWarnings(ctx context.Context, db dbtx, entityTypeCode int, entityID int) error // GetWarningID return the ID of the warning with the given key. // generator: warning ID GetWarningID(ctx context.Context, db tx, uuid string) (int64, error) // WarningExists checks if a warning with the given key exists. // generator: warning Exists WarningExists(ctx context.Context, db dbtx, uuid string) (bool, error) } incus-7.0.0/internal/server/db/cluster/warnings.mapper.go000066400000000000000000000406101517523235500235020ustar00rootroot00000000000000//go:build linux && cgo && !agent // Code generated by generate-database from the incus project - DO NOT EDIT. package cluster import ( "context" "database/sql" "errors" "fmt" "strings" ) var warningObjects = RegisterStmt(` SELECT warnings.id, coalesce(nodes.name, '') AS node, coalesce(projects.name, '') AS project, coalesce(warnings.entity_type_code, -1), coalesce(warnings.entity_id, -1), warnings.uuid, warnings.type_code, warnings.status, warnings.first_seen_date, warnings.last_seen_date, warnings.updated_date, warnings.last_message, warnings.count FROM warnings LEFT JOIN nodes ON warnings.node_id = nodes.id LEFT JOIN projects ON warnings.project_id = projects.id ORDER BY warnings.uuid `) var warningObjectsByUUID = RegisterStmt(` SELECT warnings.id, coalesce(nodes.name, '') AS node, coalesce(projects.name, '') AS project, coalesce(warnings.entity_type_code, -1), coalesce(warnings.entity_id, -1), warnings.uuid, warnings.type_code, warnings.status, warnings.first_seen_date, warnings.last_seen_date, warnings.updated_date, warnings.last_message, warnings.count FROM warnings LEFT JOIN nodes ON warnings.node_id = nodes.id LEFT JOIN projects ON warnings.project_id = projects.id WHERE ( warnings.uuid = ? ) ORDER BY warnings.uuid `) var warningObjectsByProject = RegisterStmt(` SELECT warnings.id, coalesce(nodes.name, '') AS node, coalesce(projects.name, '') AS project, coalesce(warnings.entity_type_code, -1), coalesce(warnings.entity_id, -1), warnings.uuid, warnings.type_code, warnings.status, warnings.first_seen_date, warnings.last_seen_date, warnings.updated_date, warnings.last_message, warnings.count FROM warnings LEFT JOIN nodes ON warnings.node_id = nodes.id LEFT JOIN projects ON warnings.project_id = projects.id WHERE ( coalesce(project, '') = ? ) ORDER BY warnings.uuid `) var warningObjectsByStatus = RegisterStmt(` SELECT warnings.id, coalesce(nodes.name, '') AS node, coalesce(projects.name, '') AS project, coalesce(warnings.entity_type_code, -1), coalesce(warnings.entity_id, -1), warnings.uuid, warnings.type_code, warnings.status, warnings.first_seen_date, warnings.last_seen_date, warnings.updated_date, warnings.last_message, warnings.count FROM warnings LEFT JOIN nodes ON warnings.node_id = nodes.id LEFT JOIN projects ON warnings.project_id = projects.id WHERE ( warnings.status = ? ) ORDER BY warnings.uuid `) var warningObjectsByNodeAndTypeCode = RegisterStmt(` SELECT warnings.id, coalesce(nodes.name, '') AS node, coalesce(projects.name, '') AS project, coalesce(warnings.entity_type_code, -1), coalesce(warnings.entity_id, -1), warnings.uuid, warnings.type_code, warnings.status, warnings.first_seen_date, warnings.last_seen_date, warnings.updated_date, warnings.last_message, warnings.count FROM warnings LEFT JOIN nodes ON warnings.node_id = nodes.id LEFT JOIN projects ON warnings.project_id = projects.id WHERE ( coalesce(node, '') = ? AND warnings.type_code = ? ) ORDER BY warnings.uuid `) var warningObjectsByNodeAndTypeCodeAndProject = RegisterStmt(` SELECT warnings.id, coalesce(nodes.name, '') AS node, coalesce(projects.name, '') AS project, coalesce(warnings.entity_type_code, -1), coalesce(warnings.entity_id, -1), warnings.uuid, warnings.type_code, warnings.status, warnings.first_seen_date, warnings.last_seen_date, warnings.updated_date, warnings.last_message, warnings.count FROM warnings LEFT JOIN nodes ON warnings.node_id = nodes.id LEFT JOIN projects ON warnings.project_id = projects.id WHERE ( coalesce(node, '') = ? AND warnings.type_code = ? AND coalesce(project, '') = ? ) ORDER BY warnings.uuid `) var warningObjectsByNodeAndTypeCodeAndProjectAndEntityTypeCodeAndEntityID = RegisterStmt(` SELECT warnings.id, coalesce(nodes.name, '') AS node, coalesce(projects.name, '') AS project, coalesce(warnings.entity_type_code, -1), coalesce(warnings.entity_id, -1), warnings.uuid, warnings.type_code, warnings.status, warnings.first_seen_date, warnings.last_seen_date, warnings.updated_date, warnings.last_message, warnings.count FROM warnings LEFT JOIN nodes ON warnings.node_id = nodes.id LEFT JOIN projects ON warnings.project_id = projects.id WHERE ( coalesce(node, '') = ? AND warnings.type_code = ? AND coalesce(project, '') = ? AND coalesce(warnings.entity_type_code, -1) = ? AND coalesce(warnings.entity_id, -1) = ? ) ORDER BY warnings.uuid `) var warningDeleteByUUID = RegisterStmt(` DELETE FROM warnings WHERE uuid = ? `) var warningDeleteByEntityTypeCodeAndEntityID = RegisterStmt(` DELETE FROM warnings WHERE entity_type_code = ? AND entity_id = ? `) var warningID = RegisterStmt(` SELECT warnings.id FROM warnings WHERE warnings.uuid = ? `) // warningColumns returns a string of column names to be used with a SELECT statement for the entity. // Use this function when building statements to retrieve database entries matching the Warning entity. func warningColumns() string { return "warnings.id, coalesce(nodes.name, '') AS node, coalesce(projects.name, '') AS project, coalesce(warnings.entity_type_code, -1), coalesce(warnings.entity_id, -1), warnings.uuid, warnings.type_code, warnings.status, warnings.first_seen_date, warnings.last_seen_date, warnings.updated_date, warnings.last_message, warnings.count" } // getWarnings can be used to run handwritten sql.Stmts to return a slice of objects. func getWarnings(ctx context.Context, stmt *sql.Stmt, args ...any) ([]Warning, error) { objects := make([]Warning, 0) dest := func(scan func(dest ...any) error) error { w := Warning{} err := scan(&w.ID, &w.Node, &w.Project, &w.EntityTypeCode, &w.EntityID, &w.UUID, &w.TypeCode, &w.Status, &w.FirstSeenDate, &w.LastSeenDate, &w.UpdatedDate, &w.LastMessage, &w.Count) if err != nil { return err } objects = append(objects, w) return nil } err := selectObjects(ctx, stmt, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"warnings\" table: %w", err) } return objects, nil } // getWarningsRaw can be used to run handwritten query strings to return a slice of objects. func getWarningsRaw(ctx context.Context, db dbtx, sql string, args ...any) ([]Warning, error) { objects := make([]Warning, 0) dest := func(scan func(dest ...any) error) error { w := Warning{} err := scan(&w.ID, &w.Node, &w.Project, &w.EntityTypeCode, &w.EntityID, &w.UUID, &w.TypeCode, &w.Status, &w.FirstSeenDate, &w.LastSeenDate, &w.UpdatedDate, &w.LastMessage, &w.Count) if err != nil { return err } objects = append(objects, w) return nil } err := scan(ctx, db, sql, dest, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"warnings\" table: %w", err) } return objects, nil } // GetWarnings returns all available warnings. // generator: warning GetMany func GetWarnings(ctx context.Context, db dbtx, filters ...WarningFilter) (_ []Warning, _err error) { defer func() { _err = mapErr(_err, "Warning") }() var err error // Result slice. objects := make([]Warning, 0) // Pick the prepared statement and arguments to use based on active criteria. var sqlStmt *sql.Stmt args := []any{} queryParts := [2]string{} if len(filters) == 0 { sqlStmt, err = Stmt(db, warningObjects) if err != nil { return nil, fmt.Errorf("Failed to get \"warningObjects\" prepared statement: %w", err) } } for i, filter := range filters { if filter.Node != nil && filter.TypeCode != nil && filter.Project != nil && filter.EntityTypeCode != nil && filter.EntityID != nil && filter.ID == nil && filter.UUID == nil && filter.Status == nil { args = append(args, []any{filter.Node, filter.TypeCode, filter.Project, filter.EntityTypeCode, filter.EntityID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, warningObjectsByNodeAndTypeCodeAndProjectAndEntityTypeCodeAndEntityID) if err != nil { return nil, fmt.Errorf("Failed to get \"warningObjectsByNodeAndTypeCodeAndProjectAndEntityTypeCodeAndEntityID\" prepared statement: %w", err) } break } query, err := StmtString(warningObjectsByNodeAndTypeCodeAndProjectAndEntityTypeCodeAndEntityID) if err != nil { return nil, fmt.Errorf("Failed to get \"warningObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Node != nil && filter.TypeCode != nil && filter.Project != nil && filter.ID == nil && filter.UUID == nil && filter.EntityTypeCode == nil && filter.EntityID == nil && filter.Status == nil { args = append(args, []any{filter.Node, filter.TypeCode, filter.Project}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, warningObjectsByNodeAndTypeCodeAndProject) if err != nil { return nil, fmt.Errorf("Failed to get \"warningObjectsByNodeAndTypeCodeAndProject\" prepared statement: %w", err) } break } query, err := StmtString(warningObjectsByNodeAndTypeCodeAndProject) if err != nil { return nil, fmt.Errorf("Failed to get \"warningObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Node != nil && filter.TypeCode != nil && filter.ID == nil && filter.UUID == nil && filter.Project == nil && filter.EntityTypeCode == nil && filter.EntityID == nil && filter.Status == nil { args = append(args, []any{filter.Node, filter.TypeCode}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, warningObjectsByNodeAndTypeCode) if err != nil { return nil, fmt.Errorf("Failed to get \"warningObjectsByNodeAndTypeCode\" prepared statement: %w", err) } break } query, err := StmtString(warningObjectsByNodeAndTypeCode) if err != nil { return nil, fmt.Errorf("Failed to get \"warningObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.UUID != nil && filter.ID == nil && filter.Project == nil && filter.Node == nil && filter.TypeCode == nil && filter.EntityTypeCode == nil && filter.EntityID == nil && filter.Status == nil { args = append(args, []any{filter.UUID}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, warningObjectsByUUID) if err != nil { return nil, fmt.Errorf("Failed to get \"warningObjectsByUUID\" prepared statement: %w", err) } break } query, err := StmtString(warningObjectsByUUID) if err != nil { return nil, fmt.Errorf("Failed to get \"warningObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Status != nil && filter.ID == nil && filter.UUID == nil && filter.Project == nil && filter.Node == nil && filter.TypeCode == nil && filter.EntityTypeCode == nil && filter.EntityID == nil { args = append(args, []any{filter.Status}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, warningObjectsByStatus) if err != nil { return nil, fmt.Errorf("Failed to get \"warningObjectsByStatus\" prepared statement: %w", err) } break } query, err := StmtString(warningObjectsByStatus) if err != nil { return nil, fmt.Errorf("Failed to get \"warningObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.Project != nil && filter.ID == nil && filter.UUID == nil && filter.Node == nil && filter.TypeCode == nil && filter.EntityTypeCode == nil && filter.EntityID == nil && filter.Status == nil { args = append(args, []any{filter.Project}...) if len(filters) == 1 { sqlStmt, err = Stmt(db, warningObjectsByProject) if err != nil { return nil, fmt.Errorf("Failed to get \"warningObjectsByProject\" prepared statement: %w", err) } break } query, err := StmtString(warningObjectsByProject) if err != nil { return nil, fmt.Errorf("Failed to get \"warningObjects\" prepared statement: %w", err) } parts := strings.SplitN(query, "ORDER BY", 2) if i == 0 { copy(queryParts[:], parts) continue } _, where, _ := strings.Cut(parts[0], "WHERE") queryParts[0] += "OR" + where } else if filter.ID == nil && filter.UUID == nil && filter.Project == nil && filter.Node == nil && filter.TypeCode == nil && filter.EntityTypeCode == nil && filter.EntityID == nil && filter.Status == nil { return nil, fmt.Errorf("Cannot filter on empty WarningFilter") } else { return nil, errors.New("No statement exists for the given Filter") } } // Select. if sqlStmt != nil { objects, err = getWarnings(ctx, sqlStmt, args...) } else { queryStr := strings.Join(queryParts[:], "ORDER BY") objects, err = getWarningsRaw(ctx, db, queryStr, args...) } if err != nil { return nil, fmt.Errorf("Failed to fetch from \"warnings\" table: %w", err) } return objects, nil } // GetWarning returns the warning with the given key. // generator: warning GetOne-by-UUID func GetWarning(ctx context.Context, db dbtx, uuid string) (_ *Warning, _err error) { defer func() { _err = mapErr(_err, "Warning") }() filter := WarningFilter{} filter.UUID = &uuid objects, err := GetWarnings(ctx, db, filter) if err != nil { return nil, fmt.Errorf("Failed to fetch from \"warnings\" table: %w", err) } switch len(objects) { case 0: return nil, ErrNotFound case 1: return &objects[0], nil default: return nil, fmt.Errorf("More than one \"warnings\" entry matches") } } // DeleteWarning deletes the warning matching the given key parameters. // generator: warning DeleteOne-by-UUID func DeleteWarning(ctx context.Context, db dbtx, uuid string) (_err error) { defer func() { _err = mapErr(_err, "Warning") }() stmt, err := Stmt(db, warningDeleteByUUID) if err != nil { return fmt.Errorf("Failed to get \"warningDeleteByUUID\" prepared statement: %w", err) } result, err := stmt.Exec(uuid) if err != nil { return fmt.Errorf("Delete \"warnings\": %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } if n == 0 { return ErrNotFound } else if n > 1 { return fmt.Errorf("Query deleted %d Warning rows instead of 1", n) } return nil } // DeleteWarnings deletes the warning matching the given key parameters. // generator: warning DeleteMany-by-EntityTypeCode-and-EntityID func DeleteWarnings(ctx context.Context, db dbtx, entityTypeCode int, entityID int) (_err error) { defer func() { _err = mapErr(_err, "Warning") }() stmt, err := Stmt(db, warningDeleteByEntityTypeCodeAndEntityID) if err != nil { return fmt.Errorf("Failed to get \"warningDeleteByEntityTypeCodeAndEntityID\" prepared statement: %w", err) } result, err := stmt.Exec(entityTypeCode, entityID) if err != nil { return fmt.Errorf("Delete \"warnings\": %w", err) } _, err = result.RowsAffected() if err != nil { return fmt.Errorf("Fetch affected rows: %w", err) } return nil } // GetWarningID return the ID of the warning with the given key. // generator: warning ID func GetWarningID(ctx context.Context, db tx, uuid string) (_ int64, _err error) { defer func() { _err = mapErr(_err, "Warning") }() stmt, err := Stmt(db, warningID) if err != nil { return -1, fmt.Errorf("Failed to get \"warningID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, uuid) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return -1, ErrNotFound } if err != nil { return -1, fmt.Errorf("Failed to get \"warnings\" ID: %w", err) } return id, nil } // WarningExists checks if a warning with the given key exists. // generator: warning Exists func WarningExists(ctx context.Context, db dbtx, uuid string) (_ bool, _err error) { defer func() { _err = mapErr(_err, "Warning") }() stmt, err := Stmt(db, warningID) if err != nil { return false, fmt.Errorf("Failed to get \"warningID\" prepared statement: %w", err) } row := stmt.QueryRowContext(ctx, uuid) var id int64 err = row.Scan(&id) if errors.Is(err, sql.ErrNoRows) { return false, nil } if err != nil { return false, fmt.Errorf("Failed to get \"warnings\" ID: %w", err) } return true, nil } incus-7.0.0/internal/server/db/config.go000066400000000000000000000017501517523235500201550ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "github.com/lxc/incus/v7/internal/server/db/query" ) // Config fetches all server-level config keys. func (n *NodeTx) Config(ctx context.Context) (map[string]string, error) { return query.SelectConfig(ctx, n.tx, "config", "") } // UpdateConfig updates the given server-level configuration keys in the // config table. Config keys set to empty values will be deleted. func (n *NodeTx) UpdateConfig(values map[string]string) error { return query.UpdateConfig(n.tx, "config", values) } // Config fetches all cluster config keys. func (c *ClusterTx) Config(ctx context.Context) (map[string]string, error) { return query.SelectConfig(ctx, c.tx, "config", "") } // UpdateClusterConfig updates the given cluster configuration keys in the // config table. Config keys set to empty values will be deleted. func (c *ClusterTx) UpdateClusterConfig(values map[string]string) error { return query.UpdateConfig(c.tx, "config", values) } incus-7.0.0/internal/server/db/config_test.go000066400000000000000000000025441517523235500212160ustar00rootroot00000000000000//go:build linux && cgo && !agent package db_test import ( "context" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" ) // Node-local configuration values are initially empty. func TestTx_Config(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() values, err := tx.Config(context.Background()) require.NoError(t, err) assert.Equal(t, map[string]string{}, values) assert.NoError(t, err) } // Node-local configuration values can be updated with UpdateConfig. func TestTx_UpdateConfig(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() err := tx.UpdateConfig(map[string]string{"foo": "x", "bar": "y"}) require.NoError(t, err) values, err := tx.Config(context.Background()) require.NoError(t, err) assert.Equal(t, map[string]string{"foo": "x", "bar": "y"}, values) } // Keys that are associated with empty strings are deleted. func TestTx_UpdateConfigUnsetKeys(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() err := tx.UpdateConfig(map[string]string{"foo": "x", "bar": "y"}) require.NoError(t, err) err = tx.UpdateConfig(map[string]string{"foo": "x", "bar": ""}) require.NoError(t, err) values, err := tx.Config(context.Background()) require.NoError(t, err) assert.Equal(t, map[string]string{"foo": "x"}, values) } incus-7.0.0/internal/server/db/db.go000066400000000000000000000350261517523235500173000ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "database/sql" "errors" "fmt" "os" "regexp" "sort" "strings" "sync" "time" "github.com/cowsql/go-cowsql/driver" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/node" "github.com/lxc/incus/v7/internal/server/db/query" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" ) // DB represents access to global and local databases. type DB struct { Node *Node Cluster *Cluster } // Node mediates access to data stored in the node-local SQLite database. type Node struct { db *sql.DB // Handle to the node-local SQLite database file. dir string // Reference to the directory where the database file lives. } // OpenNode creates a new Node object. // // The fresh hook parameter is used by the daemon to mark all known patch names // as applied when a brand new database is created. // // Return the newly created Node object. func OpenNode(dir string, fresh func(*Node) error) (*Node, error) { db, err := node.Open(dir) if err != nil { return nil, err } db.SetMaxOpenConns(1) db.SetMaxIdleConns(1) initial, err := node.EnsureSchema(db, dir) if err != nil { return nil, err } node := &Node{ db: db, dir: dir, } if initial == 0 { if fresh != nil { err := fresh(node) if err != nil { return nil, err } } } return node, nil } // DirectAccess is a bit of a hack which allows getting a database Node struct from any standard Go sql.DB. // This is primarily used to access the "db.bin" read-only copy of the database during startup. func DirectAccess(db *sql.DB) *Node { return &Node{db: db} } // DB returns the low level database handle to the node-local SQLite // database. // // FIXME: this is used for compatibility with some legacy code, and should be // dropped once there are no call sites left. func (n *Node) DB() *sql.DB { return n.db } // Dir returns the directory of the underlying SQLite database file. func (n *Node) Dir() string { return n.dir } // Transaction creates a new NodeTx object and transactionally executes the // node-level database interactions invoked by the given function. If the // function returns no error, all database changes are committed to the // node-level database, otherwise they are rolled back. func (n *Node) Transaction(ctx context.Context, f func(context.Context, *NodeTx) error) error { nodeTx := &NodeTx{} return query.Transaction(ctx, n.db, func(ctx context.Context, tx *sql.Tx) error { nodeTx.tx = tx return f(ctx, nodeTx) }) } // Close the database facade. func (n *Node) Close() error { return n.db.Close() } // Cluster mediates access to data stored in the cluster dqlite database. type Cluster struct { db *sql.DB // Handle to the cluster dqlite database, gated behind gRPC SQL. nodeID int64 // Node ID of this server. mu sync.RWMutex closingCtx context.Context } // OpenCluster creates a new Cluster object for interacting with the dqlite // database. // // - name: Basename of the database file holding the data. Typically "db.bin". // - dialer: Function used to connect to the dqlite backend via gRPC SQL. // - address: Network address of this node (or empty string). // - dir: Base database directory (e.g. /var/lib/incus/database) // - timeout: Give up trying to open the database after this amount of time. // // The address and api parameters will be used to determine if the cluster // database matches our version, and possibly trigger a schema update. If the // schema update can't be performed right now, because some nodes are still // behind, an Upgrading error is returned. // Accepts a closingCtx context argument used to indicate when the daemon is shutting down. func OpenCluster(closingCtx context.Context, name string, store driver.NodeStore, address, dir string, timeout time.Duration, options ...driver.Option) (*Cluster, error) { db, err := cluster.Open(name, store, options...) if err != nil { return nil, fmt.Errorf("Failed to open database: %w", err) } db.SetMaxOpenConns(1) db.SetMaxIdleConns(1) // Test that the cluster database is operational. We wait up to the // given timeout , in case there's no quorum of nodes online yet. connectCtx, connectCancel := context.WithTimeout(closingCtx, timeout) defer connectCancel() for i := 0; ; i++ { // Log initial attempts at debug level, but use warn // level after the 5'th attempt (about 10 seconds). // After the 15'th attempt (about 30 seconds), log // only one attempt every 5. logPriority := 1 // 0 is discard, 1 is Debug, 2 is Error if i > 5 { logPriority = 2 if i > 15 && !((i % 5) == 0) { logPriority = 0 } } logger.Info("Connecting to global database") pingCtx, pingCancel := context.WithTimeout(connectCtx, time.Second*5) err = db.PingContext(pingCtx) pingCancel() logCtx := logger.Ctx{"err": err, "attempt": i} if err != nil && !errors.Is(err, driver.ErrNoAvailableLeader) { return nil, err } else if err == nil { logger.Info("Connected to global database") break } switch logPriority { case 1: logger.Debug("Failed connecting to global database", logCtx) case 2: logger.Error("Failed connecting to global database", logCtx) } select { case <-connectCtx.Done(): return nil, connectCtx.Err() default: time.Sleep(2 * time.Second) } } // FIXME: https://github.com/canonical/dqlite/issues/163 _, err = db.Exec("PRAGMA cache_size=-50000") if err != nil { return nil, fmt.Errorf("Failed to set page cache size: %w", err) } nodesVersionsMatch, err := cluster.EnsureSchema(db, address, dir) if err != nil { return nil, fmt.Errorf("failed to ensure schema: %w", err) } if !nodesVersionsMatch { cluster := &Cluster{ db: db, closingCtx: closingCtx, } return cluster, ErrSomeNodesAreBehind } stmts, err := cluster.PrepareStmts(db, false) if err != nil { return nil, fmt.Errorf("Failed to prepare statements: %w", err) } cluster.PreparedStmts = stmts clusterDB := &Cluster{ db: db, closingCtx: closingCtx, } err = clusterDB.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error { // Figure out the ID of this node. members, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members: %w", err) } memberID := int64(-1) if len(members) == 1 && members[0].Address == "0.0.0.0" { // We're not clustered memberID = 1 } else { for _, member := range members { if member.Address == address { memberID = member.ID break } } } if memberID < 0 { return fmt.Errorf("No node registered with address %s", address) } // Set the local member ID clusterDB.NodeID(memberID) // Delete any operation tied to this member err = cluster.DeleteOperations(ctx, tx.tx, memberID) if err != nil { return err } return nil }) if err != nil { return nil, err } return clusterDB, err } // ErrSomeNodesAreBehind is returned by OpenCluster if some of the nodes in the // cluster have a schema or API version that is less recent than this node. var ErrSomeNodesAreBehind = errors.New("some nodes are behind this node's version") // ForLocalInspection is a aid for the hack in initializeDbObject, which // sets the db-related Daemon attributes upfront, to be backward compatible // with the legacy patches that need to interact with the database. func ForLocalInspection(db *sql.DB) *Cluster { return &Cluster{ db: db, closingCtx: context.Background(), } } // ForLocalInspectionWithPreparedStmts is the same as ForLocalInspection but it // also prepares the statements used in auto-generated database code. func ForLocalInspectionWithPreparedStmts(db *sql.DB) (*Cluster, error) { c := ForLocalInspection(db) stmts, err := cluster.PrepareStmts(c.db, true) if err != nil { return nil, fmt.Errorf("Prepare database statements: %w", err) } cluster.PreparedStmts = stmts return c, nil } // GetNodeID returns the current nodeID (0 if not set). func (c *Cluster) GetNodeID() int64 { return c.nodeID } // Transaction creates a new ClusterTx object and transactionally executes the // cluster database interactions invoked by the given function. If the function // returns no error, all database changes are committed to the cluster database // database, otherwise they are rolled back. // // If EnterExclusive has been called before, calling Transaction will block // until ExitExclusive has been called as well to release the lock. func (c *Cluster) Transaction(ctx context.Context, f func(context.Context, *ClusterTx) error) error { c.mu.RLock() defer c.mu.RUnlock() return c.transaction(ctx, f) } // EnterExclusive acquires a lock on the cluster db, so any successive call to // Transaction will block until ExitExclusive has been called. func (c *Cluster) EnterExclusive() error { logger.Debug("Acquiring exclusive lock on cluster db") ch := make(chan struct{}) go func() { c.mu.Lock() ch <- struct{}{} }() timeout := 20 * time.Second select { case <-ch: return nil case <-time.After(timeout): return fmt.Errorf("timeout (%s)", timeout) } } // ExitExclusive runs the given transaction and then releases the lock acquired // with EnterExclusive. func (c *Cluster) ExitExclusive(ctx context.Context, f func(context.Context, *ClusterTx) error) error { logger.Debug("Releasing exclusive lock on cluster db") defer c.mu.Unlock() return c.transaction(ctx, f) } func (c *Cluster) transaction(ctx context.Context, f func(context.Context, *ClusterTx) error) error { clusterTx := &ClusterTx{ nodeID: c.nodeID, } return query.Retry(ctx, func(ctx context.Context) error { txFunc := func(ctx context.Context, tx *sql.Tx) error { clusterTx.tx = tx return f(ctx, clusterTx) } err := query.Transaction(ctx, c.db, txFunc) if errors.Is(err, context.DeadlineExceeded) { // If the query timed out it likely means that the leader has abruptly become unreachable. // Now that this query has been cancelled, a leader election should have taken place by now. // So let's retry the transaction once more in case the global database is now available again. logger.Debug("Transaction timed out, will be retried", logger.Ctx{"member": c.nodeID, "err": err}) return query.Transaction(ctx, c.db, txFunc) } return err }) } // NodeID sets the node NodeID associated with this cluster instance. It's used for // backward-compatibility of all db-related APIs that were written before // clustering and don't accept a node NodeID, so in those cases we automatically // use this value as implicit node NodeID. func (c *Cluster) NodeID(id int64) { c.nodeID = id } // Close the database facade. func (c *Cluster) Close() error { for _, stmt := range cluster.PreparedStmts { _ = stmt.Close() } return c.db.Close() } // DB returns the low level database handle to the cluster database. // // FIXME: this is used for compatibility with some legacy code, and should be // dropped once there are no call sites left. func (c *Cluster) DB() *sql.DB { return c.db } // Begin a new transaction against the cluster database. // // FIXME: legacy method. func (c *Cluster) Begin() (*sql.Tx, error) { return begin(c.db) } func begin(db *sql.DB) (*sql.Tx, error) { for range 1000 { tx, err := db.Begin() if err == nil { return tx, nil } if !query.IsRetriableError(err) { logger.Debugf("DbBegin: error %q", err) return nil, err } time.Sleep(30 * time.Millisecond) } logger.Debugf("DbBegin: DB still locked") logger.Debug(logger.GetStack()) return nil, errors.New("DB is locked") } // TxCommit commits the given transaction. func TxCommit(tx *sql.Tx) error { err := tx.Commit() if err == nil || errors.Is(err, sql.ErrTxDone) { // Ignore duplicate commits/rollbacks return nil } return err } // DqliteLatestSegment returns the latest segment ID in the global database. func DqliteLatestSegment() (string, error) { dir := internalUtil.VarPath("database", "global") file, err := os.Open(dir) if err != nil { return "", fmt.Errorf("Unable to open directory %s with error %v", dir, err) } defer func() { _ = file.Close() }() fileNames, err := file.Readdirnames(0) if err != nil { return "", fmt.Errorf("Unable to read file names in directory %s with error %v", dir, err) } if len(fileNames) == 0 { return "none", nil } sort.Strings(fileNames) r, err := regexp.Compile(`^[0-9]+-[0-9]+$`) if err != nil { return "none", err } for i := range fileNames { fileName := fileNames[len(fileNames)-1-i] if r.MatchString(fileName) { segment := strings.Split(fileName, "-")[1] // Trim leading o's. index := 0 for i, c := range segment { index = i if c != '0' { break } } return segment[index:], nil } } return "none", nil } func dbQueryRowScan(ctx context.Context, c *ClusterTx, q string, args []any, outargs []any) error { return c.tx.QueryRowContext(ctx, q, args...).Scan(outargs...) } /* * . db a reference to a sql.DB instance * . q is the database query * . inargs is an array of interfaces containing the query arguments * . outfmt is an array of interfaces containing the right types of output * arguments, i.e. * var arg1 string * var arg2 int * outfmt := {}any{arg1, arg2} * * The result will be an array (one per output row) of arrays (one per output argument) * of interfaces, containing pointers to the actual output arguments. */ func queryScan(ctx context.Context, c *ClusterTx, q string, inargs []any, outfmt []any) ([][]any, error) { result := [][]any{} rows, err := c.tx.QueryContext(ctx, q, inargs...) if err != nil { return [][]any{}, err } defer func() { _ = rows.Close() }() for rows.Next() { ptrargs := make([]any, len(outfmt)) for i := range outfmt { switch t := outfmt[i].(type) { case string: str := "" ptrargs[i] = &str case int: integer := 0 ptrargs[i] = &integer case int64: integer := int64(0) ptrargs[i] = &integer case bool: boolean := bool(false) ptrargs[i] = &boolean default: return [][]any{}, fmt.Errorf("Bad interface type: %s", t) } } err = rows.Scan(ptrargs...) if err != nil { return [][]any{}, err } newargs := make([]any, len(outfmt)) for i := range ptrargs { switch t := outfmt[i].(type) { case string: newargs[i] = *ptrargs[i].(*string) case int: newargs[i] = *ptrargs[i].(*int) case int64: newargs[i] = *ptrargs[i].(*int64) case bool: newargs[i] = *ptrargs[i].(*bool) default: return [][]any{}, fmt.Errorf("Bad interface type: %s", t) } } result = append(result, newargs) } err = rows.Err() if err != nil { return [][]any{}, err } return result, nil } incus-7.0.0/internal/server/db/db_internal_test.go000066400000000000000000000232361517523235500222330ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "database/sql" "net/http" "testing" "time" "github.com/stretchr/testify/suite" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) const fixtures string = ` INSERT INTO instances (node_id, name, architecture, type, project_id, description) VALUES (1, 'thename', 1, 1, 1, ''); INSERT INTO profiles (name, project_id, description) VALUES ('theprofile', 1, ''); INSERT INTO instances_profiles (instance_id, profile_id) VALUES (1, 2); INSERT INTO instances_config (instance_id, key, value) VALUES (1, 'thekey', 'thevalue'); INSERT INTO instances_devices (instance_id, name, type) VALUES (1, 'somename', 1); INSERT INTO instances_devices_config (key, value, instance_device_id) VALUES ('configkey', 'configvalue', 1); INSERT INTO images (fingerprint, filename, size, architecture, creation_date, expiry_date, upload_date, auto_update, project_id) VALUES ('fingerprint', 'filename', 1024, 0, 1431547174, 1431547175, 1431547176, 1, 1); INSERT INTO images_aliases (name, image_id, description, project_id, description) VALUES ('somealias', 1, 'some description', 1, ''); INSERT INTO images_properties (image_id, type, key, value) VALUES (1, 0, 'thekey', 'some value'); INSERT INTO profiles_config (profile_id, key, value) VALUES (2, 'thekey', 'thevalue'); INSERT INTO profiles_devices (profile_id, name, type) VALUES (2, 'devicename', 1); INSERT INTO profiles_devices_config (profile_device_id, key, value) VALUES (1, 'devicekey', 'devicevalue'); ` type dbTestSuite struct { suite.Suite db *Cluster cleanup func() } func (s *dbTestSuite) SetupTest() { s.db, s.cleanup = s.CreateTestDb() tx, commit := s.CreateTestTx() defer commit() _, err := tx.Exec(fixtures) s.Nil(err) } func (s *dbTestSuite) TearDownTest() { s.cleanup() } // Initialize a test in-memory DB. func (s *dbTestSuite) CreateTestDb() (*Cluster, func()) { var err error // Setup logging if main() hasn't been called/when testing if logger.Log == nil { err = logger.InitLogger("", "", true, true, nil) s.Nil(err) } db, cleanup := NewTestCluster(s.T()) return db, cleanup } // Enter a transaction on the test in-memory DB. func (s *dbTestSuite) CreateTestTx() (*sql.Tx, func()) { tx, err := s.db.DB().Begin() s.Nil(err) commit := func() { s.Nil(tx.Commit()) } return tx, commit } func TestDBTestSuite(t *testing.T) { suite.Run(t, &dbTestSuite{}) } func (s *dbTestSuite) Test_deleting_a_container_cascades_on_related_tables() { var err error var count int var statements string // Drop the container we just created. statements = `DELETE FROM instances WHERE name = 'thename';` tx, commit := s.CreateTestTx() defer commit() _, err = tx.Exec(statements) s.Nil(err, "Error deleting container!") // Make sure there are 0 container_profiles entries left. statements = `SELECT count(*) FROM instances_profiles;` err = tx.QueryRow(statements).Scan(&count) s.Nil(err) s.Equal(count, 0, "Deleting a container didn't delete the profile association!") // Make sure there are 0 containers_config entries left. statements = `SELECT count(*) FROM instances_config;` err = tx.QueryRow(statements).Scan(&count) s.Nil(err) s.Equal(count, 0, "Deleting a container didn't delete the associated container_config!") // Make sure there are 0 containers_devices entries left. statements = `SELECT count(*) FROM instances_devices;` err = tx.QueryRow(statements).Scan(&count) s.Nil(err) s.Equal(count, 0, "Deleting a container didn't delete the associated container_devices!") // Make sure there are 0 containers_devices_config entries left. statements = `SELECT count(*) FROM instances_devices_config;` err = tx.QueryRow(statements).Scan(&count) s.Nil(err) s.Equal(count, 0, "Deleting a container didn't delete the associated container_devices_config!") } func (s *dbTestSuite) Test_deleting_a_profile_cascades_on_related_tables() { var err error var count int var statements string // Drop the profile we just created. statements = `DELETE FROM profiles WHERE name = 'theprofile';` tx, commit := s.CreateTestTx() defer commit() _, err = tx.Exec(statements) s.Nil(err) // Make sure there are 0 container_profiles entries left. statements = `SELECT count(*) FROM instances_profiles WHERE profile_id = 2;` err = tx.QueryRow(statements).Scan(&count) s.Nil(err) s.Equal(count, 0, "Deleting a profile didn't delete the container association!") // Make sure there are 0 profiles_devices entries left. statements = `SELECT count(*) FROM profiles_devices WHERE profile_id == 2;` err = tx.QueryRow(statements).Scan(&count) s.Nil(err) s.Equal(count, 0, "Deleting a profile didn't delete the related profiles_devices!") // Make sure there are 0 profiles_config entries left. statements = `SELECT count(*) FROM profiles_config WHERE profile_id == 2;` err = tx.QueryRow(statements).Scan(&count) s.Nil(err) s.Equal(count, 0, "Deleting a profile didn't delete the related profiles_config! There are %d left") // Make sure there are 0 profiles_devices_config entries left. statements = `SELECT count(*) FROM profiles_devices_config WHERE profile_device_id == 3;` err = tx.QueryRow(statements).Scan(&count) s.Nil(err) s.Equal(count, 0, "Deleting a profile didn't delete the related profiles_devices_config!") } func (s *dbTestSuite) Test_deleting_an_image_cascades_on_related_tables() { var err error var count int var statements string // Drop the image we just created. statements = `DELETE FROM images;` tx, commit := s.CreateTestTx() defer commit() _, err = tx.Exec(statements) s.Nil(err) // Make sure there are 0 images_aliases entries left. statements = `SELECT count(*) FROM images_aliases;` err = tx.QueryRow(statements).Scan(&count) s.Nil(err) s.Equal(count, 0, "Deleting an image didn't delete the image alias association!") // Make sure there are 0 images_properties entries left. statements = `SELECT count(*) FROM images_properties;` err = tx.QueryRow(statements).Scan(&count) s.Nil(err) s.Equal(count, 0, "Deleting an image didn't delete the related images_properties!") } func (s *dbTestSuite) Test_ImageGet_finds_image_for_fingerprint() { var result *api.Image project := "default" err := s.db.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error { var err error _, result, err = tx.GetImage(ctx, "fingerprint", cluster.ImageFilter{Project: &project}) return err }) s.Nil(err) s.NotNil(result) s.Equal(result.Filename, "filename") s.Equal(result.CreatedAt.UTC(), time.Unix(1431547174, 0).UTC()) s.Equal(result.ExpiresAt.UTC(), time.Unix(1431547175, 0).UTC()) s.Equal(result.UploadedAt.UTC(), time.Unix(1431547176, 0).UTC()) } func (s *dbTestSuite) Test_ImageGet_for_missing_fingerprint() { project := "default" err := s.db.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error { _, _, err := tx.GetImage(ctx, "unknown", cluster.ImageFilter{Project: &project}) return err }) s.True(api.StatusErrorCheck(err, http.StatusNotFound)) } func (s *dbTestSuite) Test_ImageExists_true() { var exists bool err := s.db.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error { var err error exists, err = tx.ImageExists(ctx, "default", "fingerprint") return err }) s.Nil(err) s.True(exists) } func (s *dbTestSuite) Test_ImageExists_false() { var exists bool err := s.db.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error { var err error exists, err = tx.ImageExists(ctx, "default", "foobar") return err }) s.Nil(err) s.False(exists) } func (s *dbTestSuite) Test_GetImageAlias_alias_exists() { _ = s.db.Transaction(context.Background(), func(ctx context.Context, tx *ClusterTx) error { _, alias, err := tx.GetImageAlias(ctx, "default", "somealias", true) s.Nil(err) s.Equal(alias.Target, "fingerprint") return nil }) } func (s *dbTestSuite) Test_GetImageAlias_alias_does_not_exists() { _ = s.db.Transaction(context.Background(), func(ctx context.Context, tx *ClusterTx) error { _, _, err := tx.GetImageAlias(ctx, "default", "whatever", true) s.True(api.StatusErrorCheck(err, http.StatusNotFound)) return nil }) } func (s *dbTestSuite) Test_CreateImageAlias() { _ = s.db.Transaction(context.Background(), func(ctx context.Context, tx *ClusterTx) error { err := tx.CreateImageAlias(ctx, "default", "Chaosphere", 1, "Someone will like the name") s.Nil(err) _, alias, err := tx.GetImageAlias(ctx, "default", "Chaosphere", true) s.Nil(err) s.Equal(alias.Target, "fingerprint") return nil }) } func (s *dbTestSuite) Test_GetCachedImageSourceFingerprint() { project := "default" _ = s.db.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error { imageID, _, err := tx.GetImage(ctx, "fingerprint", cluster.ImageFilter{Project: &project}) s.Nil(err) err = tx.CreateImageSource(ctx, imageID, "server.remote", "simplestreams", "", "test") s.Nil(err) fingerprint, err := tx.GetCachedImageSourceFingerprint(ctx, "server.remote", "simplestreams", "test", "container", 0) s.Nil(err) s.Equal(fingerprint, "fingerprint") return nil }) } func (s *dbTestSuite) Test_GetCachedImageSourceFingerprint_no_match() { project := "default" _ = s.db.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error { imageID, _, err := tx.GetImage(ctx, "fingerprint", cluster.ImageFilter{Project: &project}) s.Nil(err) err = tx.CreateImageSource(ctx, imageID, "server.remote", "simplestreams", "", "test") s.Nil(err) _, err = tx.GetCachedImageSourceFingerprint(ctx, "server.remote", "incus", "test", "container", 0) s.True(api.StatusErrorCheck(err, http.StatusNotFound)) return nil }) } incus-7.0.0/internal/server/db/db_test.go000066400000000000000000000023771517523235500203420ustar00rootroot00000000000000//go:build linux && cgo && !agent package db_test import ( "context" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/query" ) // Node database objects automatically initialize their schema as needed. func TestNode_Schema(t *testing.T) { node, cleanup := db.NewTestNode(t) defer cleanup() // The underlying node-level database has exactly one row in the schema // table. db := node.DB() tx, err := db.Begin() require.NoError(t, err) n, err := query.Count(context.Background(), tx, "schema", "") require.NoError(t, err) assert.Equal(t, 1, n) assert.NoError(t, tx.Commit()) assert.NoError(t, db.Close()) } // A gRPC SQL connection is established when starting to interact with the // cluster database. func TestCluster_Setup(t *testing.T) { cluster, cleanup := db.NewTestCluster(t) defer cleanup() // The underlying node-level database has exactly one row in the schema // table. db := cluster.DB() tx, err := db.Begin() require.NoError(t, err) n, err := query.Count(context.Background(), tx, "schema", "") require.NoError(t, err) assert.Equal(t, 1, n) assert.NoError(t, tx.Commit()) assert.NoError(t, db.Close()) } incus-7.0.0/internal/server/db/entity.go000066400000000000000000000143221517523235500202230ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "errors" "fmt" "strings" "github.com/lxc/incus/v7/internal/server/db/cluster" ) // ErrUnknownEntityID describes the unknown entity ID error. var ErrUnknownEntityID = errors.New("Unknown entity ID") // GetURIFromEntity returns the URI for the given entity type and entity ID. func (c *ClusterTx) GetURIFromEntity(ctx context.Context, entityType int, entityID int) (string, error) { if entityID == -1 || entityType == -1 { return "", nil } _, ok := cluster.EntityNames[entityType] if !ok { return "", errors.New("Unknown entity type") } var uri string switch entityType { case cluster.TypeImage: images, err := cluster.GetImages(ctx, c.tx) if err != nil { return "", fmt.Errorf("Failed to get images: %w", err) } for _, image := range images { if image.ID != entityID { continue } uri = fmt.Sprintf(cluster.EntityURIs[entityType], image.Fingerprint, image.Project) break } if uri == "" { return "", ErrUnknownEntityID } case cluster.TypeProfile: profiles, err := cluster.GetProfiles(ctx, c.Tx()) if err != nil { return "", fmt.Errorf("Failed to get profiles: %w", err) } for _, profile := range profiles { if profile.ID != entityID { continue } uri = fmt.Sprintf(cluster.EntityURIs[entityType], profile.Name, profile.Project) break } if uri == "" { return "", ErrUnknownEntityID } case cluster.TypeProject: projects, err := cluster.GetProjectIDsToNames(ctx, c.tx) if err != nil { return "", fmt.Errorf("Failed to get project names and IDs: %w", err) } name, ok := projects[int64(entityID)] if !ok { return "", ErrUnknownEntityID } uri = fmt.Sprintf(cluster.EntityURIs[entityType], name) case cluster.TypeCertificate: certificates, err := cluster.GetCertificates(ctx, c.tx) if err != nil { return "", fmt.Errorf("Failed to get certificates: %w", err) } for _, cert := range certificates { if cert.ID != entityID { continue } uri = fmt.Sprintf(cluster.EntityURIs[entityType], cert.Name) break } if uri == "" { return "", ErrUnknownEntityID } case cluster.TypeContainer: fallthrough case cluster.TypeInstance: instances, err := cluster.GetInstances(ctx, c.tx) if err != nil { return "", fmt.Errorf("Failed to get instances: %w", err) } for _, instance := range instances { if instance.ID != entityID { continue } uri = fmt.Sprintf(cluster.EntityURIs[entityType], instance.Name, instance.Project) break } if uri == "" { return "", ErrUnknownEntityID } case cluster.TypeInstanceBackup: instanceBackup, err := c.GetInstanceBackupWithID(ctx, entityID) if err != nil { return "", fmt.Errorf("Failed to get instance backup: %w", err) } instances, err := cluster.GetInstances(ctx, c.tx) if err != nil { return "", fmt.Errorf("Failed to get instances: %w", err) } for _, instance := range instances { if instance.ID != instanceBackup.InstanceID { continue } uri = fmt.Sprintf(cluster.EntityURIs[entityType], instance.Name, instanceBackup.Name, instance.Project) break } if uri == "" { return "", ErrUnknownEntityID } case cluster.TypeInstanceSnapshot: snapshots, err := cluster.GetInstanceSnapshots(ctx, c.Tx()) if err != nil { return "", fmt.Errorf("Failed to get instance snapshots: %w", err) } for _, snapshot := range snapshots { if snapshot.ID != entityID { continue } uri = fmt.Sprintf(cluster.EntityURIs[entityType], snapshot.Name, snapshot.Project) break } if uri == "" { return "", ErrUnknownEntityID } case cluster.TypeNetwork: networkName, projectName, err := c.GetNetworkNameAndProjectWithID(ctx, entityID) if err != nil { return "", fmt.Errorf("Failed to get network name and project name: %w", err) } uri = fmt.Sprintf(cluster.EntityURIs[entityType], networkName, projectName) case cluster.TypeNetworkACL: acls, err := cluster.GetNetworkACLs(ctx, c.tx, cluster.NetworkACLFilter{ID: &entityID}) if err != nil { return "", err } if len(acls) == 0 { return "", ErrUnknownEntityID } uri = fmt.Sprintf(cluster.EntityURIs[entityType], acls[0].Name, acls[0].Project) case cluster.TypeNode: nodeInfo, err := c.GetNodeWithID(ctx, entityID) if err != nil { return "", fmt.Errorf("Failed to get node information: %w", err) } uri = fmt.Sprintf(cluster.EntityURIs[entityType], nodeInfo.Name) case cluster.TypeOperation: id := int64(entityID) filter := cluster.OperationFilter{ID: &id} ops, err := cluster.GetOperations(ctx, c.tx, filter) if err != nil { return "", fmt.Errorf("Failed to get operation: %w", err) } if len(ops) > 1 { return "", errors.New("Failed to get operation: More than one operation matches") } op := ops[0] uri = fmt.Sprintf(cluster.EntityURIs[entityType], op.UUID) case cluster.TypeStoragePool: _, pool, _, err := c.GetStoragePoolWithID(ctx, entityID) if err != nil { return "", fmt.Errorf("Failed to get storage pool: %w", err) } uri = fmt.Sprintf(cluster.EntityURIs[entityType], pool.Name) case cluster.TypeStorageVolume: args, err := c.GetStoragePoolVolumeWithID(ctx, entityID) if err != nil { return "", fmt.Errorf("Failed to get storage volume: %w", err) } uri = fmt.Sprintf(cluster.EntityURIs[entityType], args.PoolName, args.TypeName, args.Name, args.ProjectName) case cluster.TypeStorageVolumeBackup: backup, err := c.GetStoragePoolVolumeBackupWithID(ctx, entityID) if err != nil { return "", fmt.Errorf("Failed to get volume backup: %w", err) } volume, err := c.GetStoragePoolVolumeWithID(ctx, int(backup.VolumeID)) if err != nil { return "", fmt.Errorf("Failed to get storage volume: %w", err) } uri = fmt.Sprintf(cluster.EntityURIs[entityType], volume.PoolName, volume.TypeName, volume.Name, backup.Name, volume.ProjectName) case cluster.TypeStorageVolumeSnapshot: snapshot, err := c.GetStorageVolumeSnapshotWithID(ctx, entityID) if err != nil { return "", fmt.Errorf("Failed to get volume snapshot: %w", err) } fields := strings.Split(snapshot.Name, "/") uri = fmt.Sprintf(cluster.EntityURIs[entityType], snapshot.PoolName, snapshot, snapshot.TypeName, fields[0], fields[1], snapshot.ProjectName) } return uri, nil } incus-7.0.0/internal/server/db/errors.go000066400000000000000000000005401517523235500202200ustar00rootroot00000000000000package db import ( "errors" ) var ( // ErrAlreadyDefined happens when the given entry already exists, // for example a container. ErrAlreadyDefined = errors.New("The record already exists") // ErrNoClusterMember is used to indicate no cluster member has been found for a resource. ErrNoClusterMember = errors.New("No cluster member found") ) incus-7.0.0/internal/server/db/images.go000066400000000000000000001031551517523235500201570ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "database/sql" "errors" "fmt" "net/http" "slices" "strings" "time" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/util" ) // ImageSourceProtocol maps image source protocol codes to human-readable names. var ImageSourceProtocol = map[int]string{ 0: "incus", 1: "direct", 2: "simplestreams", 3: "oci", } // GetLocalImagesFingerprints returns the fingerprints of all local images. func (c *ClusterTx) GetLocalImagesFingerprints(ctx context.Context) ([]string, error) { q := ` SELECT images.fingerprint FROM images_nodes JOIN images ON images.id = images_nodes.image_id WHERE node_id = ? ` return query.SelectStrings(ctx, c.tx, q, c.nodeID) } // GetImageSource returns the image source with the given ID. func (c *ClusterTx) GetImageSource(ctx context.Context, imageID int) (int, api.ImageSource, error) { q := `SELECT id, server, protocol, certificate, alias FROM images_source WHERE image_id=?` type imagesSource struct { ID int Server string Protocol int Certificate string Alias string } sources := []imagesSource{} err := query.Scan(ctx, c.tx, q, func(scan func(dest ...any) error) error { s := imagesSource{} err := scan(&s.ID, &s.Server, &s.Protocol, &s.Certificate, &s.Alias) if err != nil { return err } sources = append(sources, s) return nil }, imageID) if err != nil { return -1, api.ImageSource{}, err } if len(sources) == 0 { return -1, api.ImageSource{}, api.StatusErrorf(http.StatusNotFound, "Image source not found") } source := sources[0] protocol, found := ImageSourceProtocol[source.Protocol] if !found { return -1, api.ImageSource{}, fmt.Errorf("Invalid protocol: %d", source.Protocol) } result := api.ImageSource{ Server: source.Server, Protocol: protocol, Certificate: source.Certificate, Alias: source.Alias, } return source.ID, result, nil } // Fill extra image fields such as properties and alias. This is called after // fetching a single row from the images table. func (c *ClusterTx) imageFill(ctx context.Context, id int, image *api.Image, create, expire, used, upload *time.Time, arch int, imageType int) error { // Some of the dates can be nil in the DB, let's process them. if create != nil { image.CreatedAt = *create } else { image.CreatedAt = time.Time{} } if expire != nil { image.ExpiresAt = *expire } else { image.ExpiresAt = time.Time{} } if used != nil { image.LastUsedAt = *used } else { image.LastUsedAt = time.Time{} } image.Architecture, _ = osarch.ArchitectureName(arch) image.Type = instancetype.Type(imageType).String() // The upload date is enforced by NOT NULL in the schema, so it can never be nil. image.UploadedAt = *upload // Get the properties properties, err := query.SelectConfig(ctx, c.tx, "images_properties", "image_id=?", id) if err != nil { return err } image.Properties = properties q := "SELECT name, description FROM images_aliases WHERE image_id=?" // Get the aliases aliases := []api.ImageAlias{} err = query.Scan(ctx, c.tx, q, func(scan func(dest ...any) error) error { alias := api.ImageAlias{} err := scan(&alias.Name, &alias.Description) if err != nil { return err } aliases = append(aliases, alias) return nil }, id) if err != nil { return err } image.Aliases = aliases _, source, err := c.GetImageSource(ctx, id) if err == nil { image.UpdateSource = &source } return nil } func (c *ClusterTx) imageFillProfiles(ctx context.Context, id int, image *api.Image, project string) error { // Check which project name to use enabled, err := cluster.ProjectHasProfiles(context.Background(), c.tx, project) if err != nil { return fmt.Errorf("Check if project has profiles: %w", err) } if !enabled { project = "default" } // Get the profiles q := ` SELECT profiles.name FROM profiles JOIN images_profiles ON images_profiles.profile_id = profiles.id JOIN projects ON profiles.project_id = projects.id WHERE images_profiles.image_id = ? AND projects.name = ? ` profiles, err := query.SelectStrings(ctx, c.tx, q, id, project) if err != nil { return err } image.Profiles = profiles return nil } // GetImagesFingerprints returns the names of all images (optionally only the public ones). func (c *ClusterTx) GetImagesFingerprints(ctx context.Context, projectName string, publicOnly bool) ([]string, error) { q := ` SELECT fingerprint FROM images JOIN projects ON projects.id = images.project_id WHERE projects.name = ? ` if publicOnly { q += " AND public=1" } var fingerprints []string enabled, err := cluster.ProjectHasImages(ctx, c.tx, projectName) if err != nil { return nil, fmt.Errorf("Check if project has images: %w", err) } if !enabled { projectName = "default" } fingerprints, err = query.SelectStrings(ctx, c.tx, q, projectName) if err != nil { return nil, err } return fingerprints, nil } // CreateImageSource inserts a new image source. func (c *ClusterTx) CreateImageSource(ctx context.Context, id int, server string, protocol string, certificate string, alias string) error { protocolInt := -1 for protoInt, protoString := range ImageSourceProtocol { if protoString == protocol { protocolInt = protoInt } } if protocolInt == -1 { return fmt.Errorf("Invalid protocol: %s", protocol) } _, err := query.UpsertObject(c.tx, "images_source", []string{ "image_id", "server", "protocol", "certificate", "alias", }, []any{ id, server, protocolInt, certificate, alias, }) return err } // GetCachedImageSourceFingerprint tries to find a source entry of a locally // cached image that matches the given remote details (server, protocol and // alias). Return the fingerprint linked to the matching entry, if any. func (c *ClusterTx) GetCachedImageSourceFingerprint(ctx context.Context, server string, protocol string, alias string, typeName string, architecture int) (string, error) { imageType := instancetype.Any if typeName != "" { var err error imageType, err = instancetype.New(typeName) if err != nil { return "", err } } protocolInt := -1 for protoInt, protoString := range ImageSourceProtocol { if protoString == protocol { protocolInt = protoInt } } if protocolInt == -1 { return "", fmt.Errorf("Invalid protocol: %s", protocol) } q := `SELECT images.fingerprint FROM images_source INNER JOIN images ON images_source.image_id=images.id WHERE server=? AND protocol=? AND alias=? AND auto_update=1 AND images.architecture=? ` args := []any{server, protocolInt, alias, architecture} if imageType != instancetype.Any { q += "AND images.type=?\n" args = append(args, imageType) } q += "ORDER BY creation_date DESC" fingerprints, err := query.SelectStrings(ctx, c.tx, q, args...) if err != nil { return "", err } if len(fingerprints) == 0 { return "", api.StatusErrorf(http.StatusNotFound, "Image source not found") } return fingerprints[0], nil } // ImageExists returns whether an image with the given fingerprint exists. func (c *ClusterTx) ImageExists(ctx context.Context, project string, fingerprint string) (bool, error) { table := "images JOIN projects ON projects.id = images.project_id" where := "projects.name = ? AND fingerprint=?" enabled, err := cluster.ProjectHasImages(ctx, c.tx, project) if err != nil { return false, fmt.Errorf("Check if project has images: %w", err) } if !enabled { project = "default" } count, err := query.Count(ctx, c.tx, table, where, project, fingerprint) if err != nil { return false, err } return count > 0, nil } // ImageIsReferencedByOtherProjects returns true if the image with the given // fingerprint is referenced by projects other than the given one. func (c *ClusterTx) ImageIsReferencedByOtherProjects(ctx context.Context, project string, fingerprint string) (bool, error) { table := "images JOIN projects ON projects.id = images.project_id" where := "projects.name != ? AND fingerprint=?" enabled, err := cluster.ProjectHasImages(ctx, c.tx, project) if err != nil { return false, fmt.Errorf("Check if project has images: %w", err) } if !enabled { project = "default" } count, err := query.Count(ctx, c.tx, table, where, project, fingerprint) if err != nil { return false, err } return count > 0, nil } // GetImage gets an Image object from the database. // // The fingerprint argument will be queried with a LIKE query, means you can // pass a shortform and will get the full fingerprint. However in case the // shortform matches more than one image, an error will be returned. // publicOnly, when true, will return the image only if it is public; // a false value will return any image matching the fingerprint prefix. func (c *ClusterTx) GetImage(ctx context.Context, fingerprintPrefix string, filter cluster.ImageFilter) (int, *api.Image, error) { id, image, err := c.GetImageByFingerprintPrefix(ctx, fingerprintPrefix, filter) if err != nil { return -1, nil, err } return id, image, nil } // GetImageByFingerprintPrefix gets an Image object from the database. // // The fingerprint argument will be queried with a LIKE query, means you can // pass a shortform and will get the full fingerprint. However in case the // shortform matches more than one image, an error will be returned. // publicOnly, when true, will return the image only if it is public; // a false value will return any image matching the fingerprint prefix. func (c *ClusterTx) GetImageByFingerprintPrefix(ctx context.Context, fingerprintPrefix string, filter cluster.ImageFilter) (int, *api.Image, error) { var image api.Image var object cluster.Image if fingerprintPrefix == "" { return -1, nil, errors.New("No fingerprint prefix specified for the image") } if filter.Project == nil { return -1, nil, errors.New("No project specified for the image") } profileProject := *filter.Project enabled, err := cluster.ProjectHasImages(ctx, c.tx, *filter.Project) if err != nil { return -1, nil, fmt.Errorf("Check if project has images: %w", err) } if !enabled { project := "default" filter.Project = &project } images, err := c.getImagesByFingerprintPrefix(ctx, fingerprintPrefix, filter) if err != nil { return -1, nil, fmt.Errorf("Failed to fetch images: %w", err) } switch len(images) { case 0: return -1, nil, api.StatusErrorf(http.StatusNotFound, "Image not found") case 1: object = images[0] default: return -1, nil, errors.New("More than one image matches") } image.Fingerprint = object.Fingerprint image.Filename = object.Filename image.Size = object.Size image.Cached = object.Cached image.Public = object.Public image.AutoUpdate = object.AutoUpdate image.Project = object.Project err = c.imageFill( ctx, object.ID, &image, &object.CreationDate.Time, &object.ExpiryDate.Time, &object.LastUseDate.Time, &object.UploadDate, object.Architecture, object.Type) if err != nil { return -1, nil, fmt.Errorf("Fill image details: %w", err) } err = c.imageFillProfiles(ctx, object.ID, &image, profileProject) if err != nil { return -1, nil, fmt.Errorf("Fill image profiles: %w", err) } return object.ID, &image, nil } // GetImageFromAnyProject returns an image matching the given fingerprint, if // it exists in any project. func (c *ClusterTx) GetImageFromAnyProject(ctx context.Context, fingerprint string) (int, *api.Image, error) { // The object we'll actually return var image api.Image var object cluster.Image images, err := c.getImagesByFingerprintPrefix(ctx, fingerprint, cluster.ImageFilter{}) if err != nil { return -1, nil, fmt.Errorf("Get image %q: Failed to fetch images: %w", fingerprint, err) } if len(images) == 0 { return -1, nil, fmt.Errorf("Get image %q: %w", fingerprint, api.StatusErrorf(http.StatusNotFound, "Image not found")) } object = images[0] image.Fingerprint = object.Fingerprint image.Filename = object.Filename image.Size = object.Size image.Cached = object.Cached image.Public = object.Public image.AutoUpdate = object.AutoUpdate err = c.imageFill( ctx, object.ID, &image, &object.CreationDate.Time, &object.ExpiryDate.Time, &object.LastUseDate.Time, &object.UploadDate, object.Architecture, object.Type) if err != nil { return -1, nil, fmt.Errorf("Get image %q: Fill image details: %w", fingerprint, err) } return object.ID, &image, nil } // getImagesByFingerprintPrefix returns the images with fingerprints matching the prefix. // Optional filters 'project' and 'public' will be included if not nil. func (c *ClusterTx) getImagesByFingerprintPrefix(ctx context.Context, fingerprintPrefix string, filter cluster.ImageFilter) ([]cluster.Image, error) { sql := ` SELECT images.id, projects.name AS project, images.fingerprint, images.type, images.filename, images.size, images.public, images.architecture, images.creation_date, images.expiry_date, images.upload_date, images.cached, images.last_use_date, images.auto_update FROM images JOIN projects ON images.project_id = projects.id WHERE images.fingerprint LIKE ? ` args := []any{fingerprintPrefix + "%"} if filter.Project != nil { sql += `AND project = ? ` args = append(args, *filter.Project) } if filter.Public != nil { sql += `AND images.public = ? ` args = append(args, *filter.Public) } sql += `ORDER BY projects.id, images.fingerprint ` images := make([]cluster.Image, 0) err := query.Scan(ctx, c.Tx(), sql, func(scan func(dest ...any) error) error { var img cluster.Image err := scan( &img.ID, &img.Project, &img.Fingerprint, &img.Type, &img.Filename, &img.Size, &img.Public, &img.Architecture, &img.CreationDate, &img.ExpiryDate, &img.UploadDate, &img.Cached, &img.LastUseDate, &img.AutoUpdate, ) if err != nil { return err } images = append(images, img) return nil }, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch images: %w", err) } return images, nil } // LocateImage returns the address of an online node that has a local copy of // the given image, or an empty string if the image is already available on this // node. // // If the image is not available on any online node, an error is returned. func (c *ClusterTx) LocateImage(ctx context.Context, fingerprint string) (string, error) { stmt := ` SELECT nodes.address FROM nodes LEFT JOIN images_nodes ON images_nodes.node_id = nodes.id LEFT JOIN images ON images_nodes.image_id = images.id WHERE images.fingerprint = ? ` var localAddress string // Address of this node var addresses []string // Addresses of online nodes with the image offlineThreshold, err := c.GetNodeOfflineThreshold(ctx) if err != nil { return "", err } localAddress, err = c.GetLocalNodeAddress(ctx) if err != nil { return "", err } allAddresses, err := query.SelectStrings(ctx, c.tx, stmt, fingerprint) if err != nil { return "", err } for _, address := range allAddresses { node, err := c.GetNodeByAddress(ctx, address) if err != nil { return "", err } if address != localAddress && node.IsOffline(offlineThreshold) { continue } addresses = append(addresses, address) } if len(addresses) == 0 { return "", errors.New("Image not available on any online member") } if slices.Contains(addresses, localAddress) { return "", nil } return addresses[0], nil } // AddImageToLocalNode creates a new entry in the images_nodes table for // tracking that the local member has the given image. func (c *ClusterTx) AddImageToLocalNode(ctx context.Context, project, fingerprint string) error { imageID, _, err := c.GetImage(ctx, fingerprint, cluster.ImageFilter{Project: &project}) if err != nil { return err } _, err = c.tx.ExecContext(ctx, "INSERT INTO images_nodes(image_id, node_id) VALUES(?, ?)", imageID, c.nodeID) return err } // DeleteImage deletes the image with the given ID. func (c *ClusterTx) DeleteImage(ctx context.Context, id int) error { deleted, err := query.DeleteObject(c.tx, "images", int64(id)) if err != nil { return err } if !deleted { return fmt.Errorf("No image with ID %d", id) } return nil } // GetImageAliases returns the names of the aliases of all images. func (c *ClusterTx) GetImageAliases(ctx context.Context, projectName string) ([]string, error) { var names []string q := ` SELECT images_aliases.name FROM images_aliases JOIN projects ON projects.id=images_aliases.project_id WHERE projects.name=? ` enabled, err := cluster.ProjectHasImages(ctx, c.tx, projectName) if err != nil { return nil, fmt.Errorf("Check if project has images: %w", err) } if !enabled { projectName = "default" } names, err = query.SelectStrings(ctx, c.tx, q, projectName) if err != nil { return nil, err } return names, nil } // GetImageAlias returns the alias with the given name in the given project. func (c *ClusterTx) GetImageAlias(ctx context.Context, projectName string, imageName string, isTrustedClient bool) (int, api.ImageAliasesEntry, error) { id := -1 entry := api.ImageAliasesEntry{} q := `SELECT images_aliases.id, images.fingerprint, images.type, images_aliases.description FROM images_aliases INNER JOIN images ON images_aliases.image_id=images.id INNER JOIN projects ON images_aliases.project_id=projects.id WHERE projects.name=? AND images_aliases.name=?` if !isTrustedClient { q = q + ` AND images.public=1` } enabled, err := cluster.ProjectHasImages(ctx, c.tx, projectName) if err != nil { return -1, api.ImageAliasesEntry{}, fmt.Errorf("Check if project has images: %w", err) } if !enabled { projectName = "default" } var fingerprint, description string var imageType int arg1 := []any{projectName, imageName} arg2 := []any{&id, &fingerprint, &imageType, &description} err = c.tx.QueryRowContext(ctx, q, arg1...).Scan(arg2...) if err != nil { if errors.Is(err, sql.ErrNoRows) { return -1, api.ImageAliasesEntry{}, api.StatusErrorf(http.StatusNotFound, "Image alias not found") } return 0, entry, err } entry.Name = imageName entry.Target = fingerprint entry.Description = description entry.Type = instancetype.Type(imageType).String() return id, entry, nil } // RenameImageAlias renames the alias with the given ID. func (c *ClusterTx) RenameImageAlias(ctx context.Context, id int, name string) error { q := "UPDATE images_aliases SET name=? WHERE id=?" _, err := c.tx.ExecContext(ctx, q, name, id) return err } // DeleteImageAlias deletes the alias with the given name. func (c *ClusterTx) DeleteImageAlias(ctx context.Context, projectName string, name string) error { q := ` DELETE FROM images_aliases WHERE project_id = (SELECT id FROM projects WHERE name = ?) AND name = ? ` enabled, err := cluster.ProjectHasImages(ctx, c.tx, projectName) if err != nil { return fmt.Errorf("Check if project has images: %w", err) } if !enabled { projectName = "default" } _, err = c.tx.ExecContext(ctx, q, projectName, name) if err != nil { return err } return nil } // MoveImageAlias changes the image ID associated with an alias. func (c *ClusterTx) MoveImageAlias(ctx context.Context, source int, destination int) error { q := "UPDATE images_aliases SET image_id=? WHERE image_id=?" _, err := c.tx.ExecContext(ctx, q, destination, source) return err } // CreateImageAlias inserts an alias ento the database. func (c *ClusterTx) CreateImageAlias(ctx context.Context, projectName, aliasName string, imageID int, desc string) error { stmt := `INSERT INTO images_aliases (name, image_id, description, project_id) VALUES (?, ?, ?, (SELECT id FROM projects WHERE name = ?)) ` enabled, err := cluster.ProjectHasImages(ctx, c.tx, projectName) if err != nil { return fmt.Errorf("Check if project has images: %w", err) } if !enabled { projectName = "default" } _, err = c.tx.Exec(stmt, aliasName, imageID, desc, projectName) if err != nil { return err } return nil } // UpdateImageAlias updates the alias with the given ID. func (c *ClusterTx) UpdateImageAlias(ctx context.Context, aliasID int, imageID int, desc string) error { stmt := `UPDATE images_aliases SET image_id=?, description=? WHERE id=?` _, err := c.tx.ExecContext(ctx, stmt, imageID, desc, aliasID) return err } // CopyDefaultImageProfiles copies default profiles from id to new_id. func (c *ClusterTx) CopyDefaultImageProfiles(ctx context.Context, id int, newID int) error { // Delete all current associations. _, err := c.tx.ExecContext(ctx, "DELETE FROM images_profiles WHERE image_id=?", newID) if err != nil { return err } // Copy the entries over. _, err = c.tx.ExecContext(ctx, "INSERT INTO images_profiles (image_id, profile_id) SELECT ?, profile_id FROM images_profiles WHERE image_id=?", newID, id) if err != nil { return err } return nil } // UpdateImageLastUseDate updates the last_use_date field of the image with the // given fingerprint. func (c *ClusterTx) UpdateImageLastUseDate(ctx context.Context, projectName string, fingerprint string, lastUsed time.Time) error { stmt := `UPDATE images SET last_use_date=? WHERE fingerprint=? AND project_id = (SELECT id FROM projects WHERE name = ? LIMIT 1)` _, err := c.tx.ExecContext(ctx, stmt, lastUsed, fingerprint, projectName) return err } // SetImageCachedAndLastUseDate sets the cached and last_use_date field of the image with the given fingerprint. func (c *ClusterTx) SetImageCachedAndLastUseDate(ctx context.Context, projectName string, fingerprint string, lastUsed time.Time) error { enabled, err := cluster.ProjectHasImages(ctx, c.tx, projectName) if err != nil { return fmt.Errorf("Check if project has images: %w", err) } if !enabled { projectName = api.ProjectDefaultName } stmt := `UPDATE images SET cached=1, last_use_date=? WHERE fingerprint=? AND project_id = (SELECT id FROM projects WHERE name = ? LIMIT 1)` _, err = c.tx.ExecContext(ctx, stmt, lastUsed, fingerprint, projectName) return err } // UpdateImage updates the image with the given ID. func (c *ClusterTx) UpdateImage(ctx context.Context, id int, fname string, sz int64, public bool, autoUpdate bool, architecture string, createdAt time.Time, expiresAt time.Time, properties map[string]string, project string, profileIds []int64) error { arch, err := osarch.ArchitectureID(architecture) if err != nil { arch = 0 } publicInt := 0 if public { publicInt = 1 } autoUpdateInt := 0 if autoUpdate { autoUpdateInt = 1 } sql := `UPDATE images SET filename=?, size=?, public=?, auto_update=?, architecture=?, creation_date=?, expiry_date=? WHERE id=?` _, err = c.tx.ExecContext(ctx, sql, fname, sz, publicInt, autoUpdateInt, arch, createdAt, expiresAt, id) if err != nil { return err } _, err = c.tx.ExecContext(ctx, `DELETE FROM images_properties WHERE image_id=?`, id) if err != nil { return err } sql = `INSERT INTO images_properties (image_id, type, key, value) VALUES (?, ?, ?, ?)` for key, value := range properties { if value == "" { continue } _, err = c.tx.ExecContext(ctx, sql, id, 0, key, value) if err != nil { return err } } if project != "" && profileIds != nil { enabled, err := cluster.ProjectHasProfiles(ctx, c.tx, project) if err != nil { return err } if !enabled { project = "default" } q := `DELETE FROM images_profiles WHERE image_id = ? AND profile_id IN ( SELECT profiles.id FROM profiles JOIN projects ON profiles.project_id = projects.id WHERE projects.name = ? )` _, err = c.tx.ExecContext(ctx, q, id, project) if err != nil { return err } sql = `INSERT INTO images_profiles (image_id, profile_id) VALUES (?, ?)` for _, profileID := range profileIds { _, err = c.tx.ExecContext(ctx, sql, id, profileID) if err != nil { return err } } } return nil } // CreateImage creates a new image. func (c *ClusterTx) CreateImage(ctx context.Context, project string, fp string, fname string, sz int64, public bool, autoUpdate bool, architecture string, createdAt time.Time, expiresAt time.Time, properties map[string]string, typeName string, profileIds []int64) error { arch, err := osarch.ArchitectureID(architecture) if err != nil { arch = 0 } imageType := instancetype.Any if typeName != "" { var err error imageType, err = instancetype.New(typeName) if err != nil { return err } } if imageType == -1 { return fmt.Errorf("Invalid image type: %v", typeName) } imageProject := project enabled, err := cluster.ProjectHasImages(ctx, c.tx, imageProject) if err != nil { return fmt.Errorf("Check if project has images: %w", err) } if !enabled { imageProject = "default" } publicInt := 0 if public { publicInt = 1 } autoUpdateInt := 0 if autoUpdate { autoUpdateInt = 1 } sql := `INSERT INTO images (project_id, fingerprint, filename, size, public, auto_update, architecture, creation_date, expiry_date, upload_date, type) VALUES ((SELECT id FROM projects WHERE name = ?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` result, err := c.tx.ExecContext(ctx, sql, imageProject, fp, fname, sz, publicInt, autoUpdateInt, arch, createdAt, expiresAt, time.Now().UTC(), imageType) if err != nil { return fmt.Errorf("Failed saving main image record: %w", err) } var id int { id64, err := result.LastInsertId() if err != nil { return fmt.Errorf("Failed getting image ID: %w", err) } id = int(id64) } if len(properties) > 0 { sql = `INSERT INTO images_properties (image_id, type, key, value) VALUES (?, 0, ?, ?)` for k, v := range properties { // we can assume, that there is just one // value per key _, err = c.tx.ExecContext(ctx, sql, id, k, v) if err != nil { return fmt.Errorf("Failed saving image properties %d: %w", id, err) } } } if profileIds != nil { sql = `INSERT INTO images_profiles (image_id, profile_id) VALUES (?, ?)` for _, profileID := range profileIds { _, err = c.tx.ExecContext(ctx, sql, id, profileID) if err != nil { return fmt.Errorf("Failed saving image profiles: %w", err) } } } else { dbProfiles, err := cluster.GetProfilesIfEnabled(ctx, c.tx, project, []string{"default"}) if err != nil { return err } if len(dbProfiles) != 1 { return fmt.Errorf("Failed to find default profile in project %q", project) } _, err = c.tx.ExecContext(ctx, "INSERT INTO images_profiles(image_id, profile_id) VALUES(?, ?)", id, dbProfiles[0].ID) if err != nil { return fmt.Errorf("Failed saving image prfofiles: %w", err) } } // All projects with features.images=false can use all images added to the "default" project. // If these projects also have features.profiles=true, their default profiles should be associated // with all created images. if imageProject == "default" { allProjects, err := cluster.GetProjects(ctx, c.tx) if err != nil { return err } projects := []*api.Project{} for _, p := range allProjects { project, err := p.ToAPI(ctx, c.tx) if err != nil { return err } // Select the default project and projects with 'features.images' disabled and 'features.profiles' enabled. if (util.IsFalse(project.Config["features.images"]) && util.IsTrue(project.Config["features.profiles"])) || project.Name == api.ProjectDefaultName { projects = append(projects, project) } } pIDs := []int{} for _, p := range projects { dbProfiles, err := cluster.GetProfilesIfEnabled(ctx, c.tx, p.Name, []string{"default"}) if err != nil { return err } if len(dbProfiles) != 1 { return fmt.Errorf("Failed to find default profile in project %q", project) } pIDs = append(pIDs, dbProfiles[0].ID) } sql = `INSERT OR IGNORE INTO images_profiles (image_id, profile_id) VALUES (?, ?)` for _, profileID := range pIDs { _, err = c.tx.ExecContext(ctx, sql, id, profileID) if err != nil { return err } } } _, err = c.tx.ExecContext(ctx, "INSERT INTO images_nodes(image_id, node_id) VALUES(?, ?)", id, c.nodeID) if err != nil { return fmt.Errorf("Failed saving image member info: %w", err) } return nil } // GetPoolsWithImage get the IDs of all storage pools on which a given image exists. func (c *ClusterTx) GetPoolsWithImage(ctx context.Context, imageFingerprint string) ([]int64, error) { q := "SELECT storage_pool_id FROM storage_volumes WHERE (node_id=? OR node_id IS NULL) AND name=? AND type=?" ids, err := query.SelectIntegers(ctx, c.tx, q, c.nodeID, imageFingerprint, StoragePoolVolumeTypeImage) if err != nil { return nil, err } poolIDs := make([]int64, len(ids)) for i, id := range ids { poolIDs[i] = int64(id) } return poolIDs, nil } // GetPoolNamesFromIDs get the names of the storage pools with the given IDs. func (c *ClusterTx) GetPoolNamesFromIDs(ctx context.Context, poolIDs []int64) ([]string, error) { params := make([]string, len(poolIDs)) args := make([]any, len(poolIDs)) for i, id := range poolIDs { params[i] = "?" args[i] = id } q := fmt.Sprintf("SELECT name FROM storage_pools WHERE id IN (%s)", strings.Join(params, ",")) poolNames, err := query.SelectStrings(ctx, c.tx, q, args...) if err != nil { return nil, err } if len(poolNames) != len(poolIDs) { return nil, fmt.Errorf("Found only %d matches, expected %d", len(poolNames), len(poolIDs)) } return poolNames, nil } // GetImages returns all images. func (c *ClusterTx) GetImages(ctx context.Context) (map[string][]string, error) { images := make(map[string][]string) // key is fingerprint, value is list of projects stmt := ` SELECT images.fingerprint, projects.name FROM images LEFT JOIN projects ON images.project_id = projects.id ` rows, err := c.tx.QueryContext(ctx, stmt) if err != nil { return nil, err } var fingerprint string var projectName string for rows.Next() { err := rows.Scan(&fingerprint, &projectName) if err != nil { return nil, err } images[fingerprint] = append(images[fingerprint], projectName) } return images, rows.Err() } // GetImagesOnLocalNode returns all images that the local server holds. func (c *ClusterTx) GetImagesOnLocalNode(ctx context.Context) (map[string][]string, error) { return c.GetImagesOnNode(ctx, c.nodeID) } // GetImagesOnNode returns all images that the node with the given id has. func (c *ClusterTx) GetImagesOnNode(ctx context.Context, id int64) (map[string][]string, error) { images := make(map[string][]string) // key is fingerprint, value is list of projects stmt := ` SELECT images.fingerprint, projects.name FROM images LEFT JOIN images_nodes ON images.id = images_nodes.image_id LEFT JOIN nodes ON images_nodes.node_id = nodes.id LEFT JOIN projects ON images.project_id = projects.id WHERE nodes.id = ? ` rows, err := c.tx.QueryContext(ctx, stmt, id) if err != nil { return nil, err } var fingerprint string var projectName string for rows.Next() { err := rows.Scan(&fingerprint, &projectName) if err != nil { return nil, err } images[fingerprint] = append(images[fingerprint], projectName) } return images, rows.Err() } // GetNodesWithImage returns the addresses of online nodes which already have the image. func (c *ClusterTx) GetNodesWithImage(ctx context.Context, fingerprint string) ([]string, error) { q := ` SELECT DISTINCT nodes.address FROM nodes LEFT JOIN images_nodes ON images_nodes.node_id = nodes.id LEFT JOIN images ON images_nodes.image_id = images.id WHERE images.fingerprint = ? ` return c.getNodesByImageFingerprint(ctx, q, fingerprint, nil) } // GetNodesWithImageAndAutoUpdate returns the addresses of online nodes which already have the image. func (c *ClusterTx) GetNodesWithImageAndAutoUpdate(ctx context.Context, fingerprint string, autoUpdate bool) ([]string, error) { q := ` SELECT DISTINCT nodes.address FROM nodes JOIN images_nodes ON images_nodes.node_id = nodes.id JOIN images ON images_nodes.image_id = images.id WHERE images.fingerprint = ? AND images.auto_update = ? ` return c.getNodesByImageFingerprint(ctx, q, fingerprint, &autoUpdate) } // GetNodesWithoutImage returns the addresses of online nodes which don't have the image. func (c *ClusterTx) GetNodesWithoutImage(ctx context.Context, fingerprint string) ([]string, error) { q := ` SELECT DISTINCT nodes.address FROM nodes WHERE nodes.address NOT IN ( SELECT DISTINCT nodes.address FROM nodes LEFT JOIN images_nodes ON images_nodes.node_id = nodes.id LEFT JOIN images ON images_nodes.image_id = images.id WHERE images.fingerprint = ?) ` return c.getNodesByImageFingerprint(ctx, q, fingerprint, nil) } func (c *ClusterTx) getNodesByImageFingerprint(ctx context.Context, stmt string, fingerprint string, autoUpdate *bool) ([]string, error) { var addresses []string // Addresses of online nodes with the image offlineThreshold, err := c.GetNodeOfflineThreshold(ctx) if err != nil { return nil, err } var allAddresses []string if autoUpdate == nil { allAddresses, err = query.SelectStrings(ctx, c.tx, stmt, fingerprint) } else { allAddresses, err = query.SelectStrings(ctx, c.tx, stmt, fingerprint, autoUpdate) } if err != nil { return nil, err } for _, address := range allAddresses { node, err := c.GetNodeByAddress(ctx, address) if err != nil { return nil, err } if node.IsOffline(offlineThreshold) { continue } addresses = append(addresses, address) } return addresses, nil } // GetProjectsUsingImage get the project names using an image by fingerprint. func (c *ClusterTx) GetProjectsUsingImage(ctx context.Context, fingerprint string) ([]string, error) { var err error var imgProjectNames []string q := ` SELECT projects.name FROM images JOIN projects ON projects.id=images.project_id WHERE fingerprint = ? ` err = query.Scan(ctx, c.Tx(), q, func(scan func(dest ...any) error) error { var imgProjectName string err = scan(&imgProjectName) if err != nil { return err } imgProjectNames = append(imgProjectNames, imgProjectName) return nil }, fingerprint) if err != nil { return nil, err } return imgProjectNames, nil } incus-7.0.0/internal/server/db/images_test.go000066400000000000000000000061521517523235500212150ustar00rootroot00000000000000//go:build linux && cgo && !agent package db_test import ( "context" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" ) func TestLocateImage(t *testing.T) { cluster, cleanup := db.NewTestCluster(t) defer cleanup() _ = cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { err := tx.CreateImage(ctx, "default", "abc", "x.gz", 16, false, false, "amd64", time.Now(), time.Now(), map[string]string{}, "container", nil) require.NoError(t, err) address, err := tx.LocateImage(ctx, "abc") require.NoError(t, err) assert.Equal(t, "", address) // Pretend that the function is being run on another node. tx.NodeID(2) address, err = tx.LocateImage(ctx, "abc") require.NoError(t, err) assert.Equal(t, "0.0.0.0", address) // Pretend that the target node is down err = tx.SetNodeHeartbeat("0.0.0.0", time.Now().Add(-time.Minute)) require.NoError(t, err) address, err = tx.LocateImage(ctx, "abc") require.Equal(t, "", address) require.EqualError(t, err, "Image not available on any online member") return nil }) } func TestImageExists(t *testing.T) { cluster, cleanup := db.NewTestCluster(t) defer cleanup() _ = cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { exists, err := tx.ImageExists(ctx, "default", "abc") require.NoError(t, err) assert.False(t, exists) err = tx.CreateImage(ctx, "default", "abc", "x.gz", 16, false, false, "amd64", time.Now(), time.Now(), map[string]string{}, "container", nil) require.NoError(t, err) exists, err = tx.ImageExists(ctx, "default", "abc") require.NoError(t, err) assert.True(t, exists) return nil }) } func TestGetImage(t *testing.T) { dbCluster, cleanup := db.NewTestCluster(t) defer cleanup() project := "default" _ = dbCluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // public image with 'default' project err := tx.CreateImage(ctx, project, "abcd1", "x.gz", 16, true, false, "amd64", time.Now(), time.Now(), map[string]string{}, "container", nil) require.NoError(t, err) // 'public' is ignored if 'false' id, img, err := tx.GetImage(ctx, "a", cluster.ImageFilter{Project: &project}) require.NoError(t, err) assert.Equal(t, img.Public, true) assert.NotEqual(t, id, -1) // non-public image with 'default' project err = tx.CreateImage(ctx, project, "abcd2", "x.gz", 16, false, false, "amd64", time.Now(), time.Now(), map[string]string{}, "container", nil) require.NoError(t, err) // empty project fails _, _, err = tx.GetImage(ctx, "a", cluster.ImageFilter{}) require.Error(t, err) // 'public' is ignored if 'false', returning both entries _, _, err = tx.GetImage(ctx, "a", cluster.ImageFilter{Project: &project}) require.Error(t, err) public := true id, img, err = tx.GetImage(ctx, "a", cluster.ImageFilter{Project: &project, Public: &public}) require.NoError(t, err) assert.Equal(t, img.Public, true) assert.NotEqual(t, id, -1) return nil }) } incus-7.0.0/internal/server/db/instances.go000066400000000000000000001105251517523235500207000ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "database/sql" "errors" "fmt" "net/http" "sort" "strings" "time" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/db/query" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/api" ) // InstanceArgs is a value object holding all db-related details about an instance. type InstanceArgs struct { // Don't set manually ID int Node string Type instancetype.Type Snapshot bool // Creation only Project string BaseImage string CreationDate time.Time Architecture int Config map[string]string Description string Devices deviceConfig.Devices Ephemeral bool LastUsedDate time.Time Name string Profiles []api.Profile Stateful bool ExpiryDate time.Time } // GetInstanceNames returns the names of all containers the given project. func (c *ClusterTx) GetInstanceNames(ctx context.Context, project string) ([]string, error) { stmt := ` SELECT instances.name FROM instances JOIN projects ON projects.id = instances.project_id WHERE projects.name = ? ` return query.SelectStrings(ctx, c.tx, stmt, project) } // GetNodeAddressOfInstance returns the address of the node hosting the // instance with the given name in the given project. // // It returns the empty string if the container is hosted on this node. func (c *ClusterTx) GetNodeAddressOfInstance(ctx context.Context, project string, name string) (string, error) { var stmt string args := make([]any, 0, 4) // Expect up to 4 filters. var filters strings.Builder // Project filter. filters.WriteString("projects.name = ?") args = append(args, project) // Instance type filter. if strings.Contains(name, internalInstance.SnapshotDelimiter) { parts := strings.SplitN(name, internalInstance.SnapshotDelimiter, 2) // Instance name filter. filters.WriteString(" AND instances.name = ?") args = append(args, parts[0]) // Snapshot name filter. filters.WriteString(" AND instances_snapshots.name = ?") args = append(args, parts[1]) stmt = fmt.Sprintf(` SELECT nodes.id, nodes.address FROM nodes JOIN instances ON instances.node_id = nodes.id JOIN projects ON projects.id = instances.project_id JOIN instances_snapshots ON instances_snapshots.instance_id = instances.id WHERE %s `, filters.String()) } else { // Instance name filter. filters.WriteString(" AND instances.name = ?") args = append(args, name) stmt = fmt.Sprintf(` SELECT nodes.id, nodes.address FROM nodes JOIN instances ON instances.node_id = nodes.id JOIN projects ON projects.id = instances.project_id WHERE %s `, filters.String()) } var address string var id int64 rows, err := c.tx.QueryContext(ctx, stmt, args...) if err != nil { return "", err } defer func() { _ = rows.Close() }() if !rows.Next() { return "", api.StatusErrorf(http.StatusNotFound, "Instance not found") } err = rows.Scan(&id, &address) if err != nil { return "", err } if rows.Next() { return "", errors.New("More than one cluster member associated with instance") } err = rows.Err() if err != nil { return "", err } if id == c.nodeID { return "", nil } return address, nil } // Instance represents basic instance info. type Instance struct { ID int64 Name string Project string Location string Type instancetype.Type } // GetInstancesByMemberAddress returns the instances associated to each cluster member address. // The member address of instances running on the local member is set to the empty string, to distinguish it from // remote nodes. Instances whose member is down are added to the special address "0.0.0.0". func (c *ClusterTx) GetInstancesByMemberAddress(ctx context.Context, offlineThreshold time.Duration, projects []string) (map[string][]Instance, error) { args := make([]any, 0, 2) // Expect up to 2 filters. var q strings.Builder q.WriteString(`SELECT instances.id, instances.name, instances.type, nodes.id, nodes.name, nodes.address, nodes.heartbeat, projects.name FROM instances JOIN nodes ON nodes.id = instances.node_id JOIN projects ON projects.id = instances.project_id `) // Project filter. q.WriteString(fmt.Sprintf("WHERE projects.name IN %s", query.Params(len(projects)))) for _, project := range projects { args = append(args, project) } q.WriteString(" ORDER BY instances.id") rows, err := c.tx.QueryContext(ctx, q.String(), args...) if err != nil { return nil, err } defer func() { _ = rows.Close() }() memberAddressInstances := make(map[string][]Instance) for rows.Next() { var inst Instance var memberAddress string var memberID int64 var memberHeartbeat time.Time err := rows.Scan(&inst.ID, &inst.Name, &inst.Type, &memberID, &inst.Location, &memberAddress, &memberHeartbeat, &inst.Project) if err != nil { return nil, err } if memberID == c.nodeID { memberAddress = "" } else if nodeIsOffline(offlineThreshold, memberHeartbeat) { memberAddress = "0.0.0.0" } memberAddressInstances[memberAddress] = append(memberAddressInstances[memberAddress], inst) } err = rows.Err() if err != nil { return nil, err } return memberAddressInstances, nil } // ErrInstanceListStop used as return value from InstanceList's instanceFunc when prematurely stopping the search. var ErrInstanceListStop = errors.New("search stopped") // InstanceList loads all instances across all projects and for each instance runs the instanceFunc passing in the // instance and it's project and profiles. Accepts optional filter arguments to specify a subset of instances. func (c *ClusterTx) InstanceList(ctx context.Context, instanceFunc func(inst InstanceArgs, project api.Project) error, filters ...cluster.InstanceFilter) error { projectsByName := make(map[string]*api.Project) var instances map[int]InstanceArgs emptyFilter := cluster.InstanceFilter{} validFilters := []cluster.InstanceFilter{} for _, filter := range filters { if filter.Type != nil && *filter.Type == instancetype.Any { filter.Type = nil } if filter != emptyFilter { validFilters = append(validFilters, filter) } } // Retrieve required info from the database in single transaction for performance. // Get all projects. projects, err := cluster.GetProjects(ctx, c.tx) if err != nil { return fmt.Errorf("Failed loading projects: %w", err) } // Get all instances using supplied filter. dbInstances, err := cluster.GetInstances(ctx, c.tx, validFilters...) if err != nil { return fmt.Errorf("Failed loading instances: %w", err) } // Fill instances with config, devices and profiles. instances, err = c.InstancesToInstanceArgs(ctx, true, dbInstances...) if err != nil { return err } // Record which projects are referenced by at least one instance in the list. for _, instance := range instances { _, ok := projectsByName[instance.Project] if !ok { projectsByName[instance.Project] = nil } } // Populate projectsByName map entry for referenced projects. // This way we only call ToAPI() on the projects actually referenced by the instances in // the list, which can reduce the number of queries run. for _, project := range projects { _, ok := projectsByName[project.Name] if !ok { continue } projectsByName[project.Name], err = project.ToAPI(ctx, c.tx) if err != nil { return err } } // Call the instanceFunc provided for each instance after the transaction has ended, as we don't know if // the instanceFunc will be slow or may need to make additional DB queries. for _, instance := range instances { project := projectsByName[instance.Project] if project == nil { return fmt.Errorf("Instance references %d project %q that isn't loaded", instance.ID, instance.Project) } err = instanceFunc(instance, *project) if err != nil { return err } } return nil } // instanceConfigFill function loads config for all specified instances in a single query and then updates // the entries in the instances map. func (c *ClusterTx) instanceConfigFill(ctx context.Context, snapshotsMode bool, instanceArgs *map[int]InstanceArgs) error { instances := *instanceArgs // Don't use query parameters for the IN statement to workaround an issue in Dqlite (apparently) // that means that >255 query parameters causes partial result sets. See #10705 // This is safe as the inputs are ints. var q strings.Builder if snapshotsMode { q.WriteString(`SELECT instance_snapshot_id, key, value FROM instances_snapshots_config WHERE instance_snapshot_id IN (`) } else { q.WriteString(`SELECT instance_id, key, value FROM instances_config WHERE instance_id IN (`) } q.Grow(len(instances) * 2) // We know the minimum length of the separators and integers. first := true for instanceID := range instances { if !first { q.WriteString(",") } first = false q.WriteString(fmt.Sprintf("%d", instanceID)) } q.WriteString(`)`) return query.Scan(ctx, c.Tx(), q.String(), func(scan func(dest ...any) error) error { var instanceID int var key, value string err := scan(&instanceID, &key, &value) if err != nil { return err } _, found := instances[instanceID] if !found { return fmt.Errorf("Failed loading instance config, referenced instance %d not loaded", instanceID) } if instances[instanceID].Config == nil { inst := instances[instanceID] inst.Config = make(map[string]string) instances[instanceID] = inst } _, found = instances[instanceID].Config[key] if found { return fmt.Errorf("Duplicate config row found for key %q for instance ID %d", key, instanceID) } instances[instanceID].Config[key] = value return nil }) } // instanceDevicesFill loads the device config for all instances specified in a single query and then updates // the entries in the instances map. func (c *ClusterTx) instanceDevicesFill(ctx context.Context, snapshotsMode bool, instanceArgs *map[int]InstanceArgs) error { instances := *instanceArgs // Don't use query parameters for the IN statement to workaround an issue in Dqlite (apparently) // that means that >255 query parameters causes partial result sets. See #10705 // This is safe as the inputs are ints. var q strings.Builder if snapshotsMode { q.WriteString(` SELECT instances_snapshots_devices.instance_snapshot_id AS instance_snapshot_id, instances_snapshots_devices.name AS device_name, instances_snapshots_devices.type AS device_type, instances_snapshots_devices_config.key, instances_snapshots_devices_config.value FROM instances_snapshots_devices_config JOIN instances_snapshots_devices ON instances_snapshots_devices.id = instances_snapshots_devices_config.instance_snapshot_device_id WHERE instances_snapshots_devices.instance_snapshot_id IN (`) } else { q.WriteString(` SELECT instances_devices.instance_id AS instance_id, instances_devices.name AS device_name, instances_devices.type AS device_type, instances_devices_config.key, instances_devices_config.value FROM instances_devices_config JOIN instances_devices ON instances_devices.id = instances_devices_config.instance_device_id WHERE instances_devices.instance_id IN (`) } q.Grow(len(instances) * 2) // We know the minimum length of the separators and integers. first := true for instanceID := range instances { if !first { q.WriteString(",") } first = false q.WriteString(fmt.Sprintf("%d", instanceID)) } q.WriteString(`)`) return query.Scan(ctx, c.Tx(), q.String(), func(scan func(dest ...any) error) error { var instanceID int var deviceType cluster.DeviceType var deviceName, key, value string err := scan(&instanceID, &deviceName, &deviceType, &key, &value) if err != nil { return err } _, found := instances[instanceID] if !found { return fmt.Errorf("Failed loading instance device, referenced instance %d not loaded", instanceID) } if instances[instanceID].Devices == nil { inst := instances[instanceID] inst.Devices = make(deviceConfig.Devices) instances[instanceID] = inst } _, found = instances[instanceID].Devices[deviceName] if !found { instances[instanceID].Devices[deviceName] = deviceConfig.Device{ "type": deviceType.String(), // Map instances_devices type to config field. } } _, found = instances[instanceID].Devices[deviceName][key] if found && key != "type" { // For legacy reasons the type value is in both the instances_devices and // instances_devices_config tables. We use the one from the instances_devices. return fmt.Errorf("Duplicate device row found for device %q key %q for instance ID %d", deviceName, key, instanceID) } instances[instanceID].Devices[deviceName][key] = value return nil }) } // instanceProfiles loads the profile IDs to apply to an instance (in the application order) for all // instanceIDs in a single query and then updates the instanceApplyProfileIDs and profilesByID maps. func (c *ClusterTx) instanceProfilesFill(ctx context.Context, snapshotsMode bool, instanceArgs *map[int]InstanceArgs) error { instances := *instanceArgs // Get profiles referenced by instances. // Don't use query parameters for the IN statement to workaround an issue in Dqlite (apparently) // that means that >255 query parameters causes partial result sets. See #10705 // This is safe as the inputs are ints. var q strings.Builder if snapshotsMode { q.WriteString(` SELECT instances_snapshots.id AS snapshot_id, instances_profiles.profile_id AS profile_id FROM instances_profiles JOIN instances_snapshots ON instances_snapshots.instance_id = instances_profiles.instance_id WHERE instances_snapshots.id IN (`) } else { q.WriteString(` SELECT instances_profiles.instance_id AS instance_id, instances_profiles.profile_id AS profile_id FROM instances_profiles WHERE instances_profiles.instance_id IN (`) } q.Grow(len(instances) * 2) // We know the minimum length of the separators and integers. first := true for instanceID := range instances { if !first { q.WriteString(",") } first = false q.WriteString(fmt.Sprintf("%d", instanceID)) } q.WriteString(`) ORDER BY instances_profiles.instance_id, instances_profiles.apply_order`) profilesByID := make(map[int]*api.Profile) instanceApplyProfileIDs := make(map[int64][]int, len(instances)) err := query.Scan(ctx, c.Tx(), q.String(), func(scan func(dest ...any) error) error { var instanceID int64 var profileID int err := scan(&instanceID, &profileID) if err != nil { return err } instanceApplyProfileIDs[instanceID] = append(instanceApplyProfileIDs[instanceID], profileID) // Record that this profile is referenced by at least one instance in the list. _, ok := profilesByID[profileID] if !ok { profilesByID[profileID] = nil } return nil }) if err != nil { return err } // Get all profiles. profiles, err := cluster.GetProfiles(context.TODO(), c.Tx()) if err != nil { return fmt.Errorf("Failed loading profiles: %w", err) } // Get all the profile configs. profileConfigs, err := cluster.GetAllProfileConfigs(context.TODO(), c.Tx()) if err != nil { return fmt.Errorf("Failed loading profile configs: %w", err) } // Get all the profile devices. profileDevices, err := cluster.GetAllProfileDevices(context.TODO(), c.Tx()) if err != nil { return fmt.Errorf("Failed loading profile devices: %w", err) } // Populate profilesByID map entry for referenced profiles. // This way we only call ToAPI() on the profiles actually referenced by the instances in // the list, which can reduce the number of queries run. for _, profile := range profiles { _, ok := profilesByID[profile.ID] if !ok { continue } profilesByID[profile.ID], err = profile.ToAPI(context.TODO(), c.tx, profileConfigs, profileDevices) if err != nil { return err } } // Populate instance profiles list in apply order. for instanceID := range instances { inst := instances[instanceID] inst.Profiles = make([]api.Profile, 0, len(inst.Profiles)) for _, applyProfileID := range instanceApplyProfileIDs[int64(inst.ID)] { profile := profilesByID[applyProfileID] if profile == nil { return fmt.Errorf("Instance %d references profile %d that isn't loaded", inst.ID, applyProfileID) } inst.Profiles = append(inst.Profiles, *profile) } instances[instanceID] = inst } return nil } // InstancesToInstanceArgs converts many cluster.Instance to a map of InstanceArgs in as few queries as possible. // Accepts fillProfiles argument that controls whether or not the returned InstanceArgs have their Profiles field // populated. This avoids the need to load profile info from the database if it is already available in the // caller's context and can be populated afterwards. func (c *ClusterTx) InstancesToInstanceArgs(ctx context.Context, fillProfiles bool, instances ...cluster.Instance) (map[int]InstanceArgs, error) { var instanceCount, snapshotCount uint // Convert instances to partial InstanceArgs slice (Config, Devices and Profiles not populated yet). instanceArgs := make(map[int]InstanceArgs, len(instances)) for _, instance := range instances { if instance.Snapshot { snapshotCount++ } else { instanceCount++ } args := InstanceArgs{ ID: instance.ID, Project: instance.Project, Name: instance.Name, Node: instance.Node, Type: instance.Type, Snapshot: instance.Snapshot, Architecture: instance.Architecture, Ephemeral: instance.Ephemeral, CreationDate: instance.CreationDate, Stateful: instance.Stateful, LastUsedDate: instance.LastUseDate.Time, Description: instance.Description, ExpiryDate: instance.ExpiryDate.Time, } instanceArgs[instance.ID] = args } if instanceCount > 0 && snapshotCount > 0 { return nil, errors.New("Cannot use InstancesToInstanceArgs with mixed instance and instance snapshots") } // Populate instance config. err := c.instanceConfigFill(ctx, snapshotCount > 0, &instanceArgs) if err != nil { return nil, fmt.Errorf("Failed loading instance config: %w", err) } // Populate instance devices. err = c.instanceDevicesFill(ctx, snapshotCount > 0, &instanceArgs) if err != nil { return nil, fmt.Errorf("Failed loading instance devices: %w", err) } // Populate instance profiles if requested. if fillProfiles { err = c.instanceProfilesFill(ctx, snapshotCount > 0, &instanceArgs) if err != nil { return nil, fmt.Errorf("Failed loading instance profiles: %w", err) } } return instanceArgs, nil } // UpdateInstanceNode changes the name of an instance and the cluster member hosting it. // It's meant to be used when moving a non-running instance backed by remote storage from one cluster node to another. func (c *ClusterTx) UpdateInstanceNode(ctx context.Context, project string, oldName string, newName string, newMemberName string, poolID int64, volumeType int) error { // Update the name of the instance and its snapshots, and the member ID they are associated with. instanceID, err := cluster.GetInstanceID(ctx, c.tx, project, oldName) if err != nil { return fmt.Errorf("Failed to get instance's ID: %w", err) } member, err := c.GetNodeByName(ctx, newMemberName) if err != nil { return fmt.Errorf("Failed to get new member %q info: %w", newMemberName, err) } stmt := "UPDATE instances SET node_id=?, name=? WHERE id=?" result, err := c.tx.Exec(stmt, member.ID, newName, instanceID) if err != nil { return fmt.Errorf("Failed to update instance's name and member ID: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Failed to get rows affected by instance update: %w", err) } if n != 1 { return fmt.Errorf("Unexpected number of updated rows in instances table: %d", n) } // No need to update storage_volumes if the name is identical if newName == oldName { return nil } stmt = "UPDATE storage_volumes SET name=? WHERE name=? AND storage_pool_id=? AND type=?" result, err = c.tx.Exec(stmt, newName, oldName, poolID, volumeType) if err != nil { return fmt.Errorf("Failed to update instance's volume name: %w", err) } n, err = result.RowsAffected() if err != nil { return fmt.Errorf("Failed to get rows affected by instance volume update: %w", err) } if n != 1 { return fmt.Errorf("Unexpected number of updated rows in volumes table: %d", n) } return nil } // GetLocalInstancesInProject retuurns all instances of the given type on the local member in the given project. // If projectName is empty then all instances in all projects are returned. func (c *ClusterTx) GetLocalInstancesInProject(ctx context.Context, filter cluster.InstanceFilter) ([]cluster.Instance, error) { node, err := c.GetLocalNodeName(ctx) if err != nil { return nil, fmt.Errorf("Local node name: %w", err) } if node != "" { filter.Node = &node } return cluster.GetInstances(ctx, c.tx, filter) } // CreateInstanceConfig inserts a new config for the container with the given ID. func (c *ClusterTx) CreateInstanceConfig(ctx context.Context, id int, config map[string]string) error { return CreateInstanceConfig(ctx, c.tx, id, config) } // UpdateInstanceConfig inserts/updates/deletes the provided keys. func (c *ClusterTx) UpdateInstanceConfig(id int, values map[string]string) error { insertSQL := "INSERT OR REPLACE INTO instances_config (instance_id, key, value) VALUES" deleteSQL := "DELETE FROM instances_config WHERE key IN %s AND instance_id=?" return c.configUpdate(id, values, insertSQL, deleteSQL) } func (c *ClusterTx) configUpdate(id int, values map[string]string, insertSQL, deleteSQL string) error { changes := map[string]string{} deletes := []string{} // Figure out which key to set/unset for key, value := range values { if value == "" { deletes = append(deletes, key) continue } changes[key] = value } // Insert/update keys if len(changes) > 0 { query := insertSQL exprs := []string{} params := []any{} for key, value := range changes { exprs = append(exprs, "(?, ?, ?)") params = append(params, []any{id, key, value}...) } query += strings.Join(exprs, ",") _, err := c.tx.Exec(query, params...) if err != nil { return err } } // Delete keys if len(deletes) > 0 { query := fmt.Sprintf(deleteSQL, query.Params(len(deletes))) params := []any{} for _, key := range deletes { params = append(params, key) } params = append(params, id) _, err := c.tx.Exec(query, params...) if err != nil { return err } } return nil } // DeleteInstanceConfigKey removes the given key from the config of the instance // with the given ID. func (c *ClusterTx) DeleteInstanceConfigKey(ctx context.Context, id int64, key string) error { q := "DELETE FROM instances_config WHERE key=? AND instance_id=?" _, err := c.tx.ExecContext(ctx, q, key, id) return err } // UpdateInstancePowerState sets the power state of the container with the given ID. func (c *ClusterTx) UpdateInstancePowerState(id int, state string) error { // Set the new value str := "INSERT OR REPLACE INTO instances_config (instance_id, key, value) VALUES (?, 'volatile.last_state.power', ?)" _, err := c.tx.Exec(str, id, state) if err != nil { return err } return nil } // UpdateInstanceLastUsedDate updates the last_use_date field of the instance // with the given ID. func (c *ClusterTx) UpdateInstanceLastUsedDate(id int, date time.Time) error { str := `UPDATE instances SET last_use_date=? WHERE id=?` _, err := c.tx.Exec(str, date, id) if err != nil { return err } return nil } // GetInstanceSnapshotsWithName returns all snapshots of a given instance in date created order, oldest first. func (c *ClusterTx) GetInstanceSnapshotsWithName(ctx context.Context, project string, name string) ([]cluster.Instance, error) { instance, err := cluster.GetInstance(ctx, c.tx, project, name) if err != nil { return nil, err } filter := cluster.InstanceSnapshotFilter{ Project: &project, Instance: &name, } snapshots, err := cluster.GetInstanceSnapshots(ctx, c.tx, filter) if err != nil { return nil, err } sort.SliceStable(snapshots, func(i, j int) bool { return snapshots[i].CreationDate.Before(snapshots[j].CreationDate) }) instances := make([]cluster.Instance, len(snapshots)) for i, snapshot := range snapshots { instances[i] = snapshot.ToInstance(instance.Name, instance.Node, instance.Type, instance.Architecture) } return instances, nil } // GetLocalInstanceWithVsockID returns all available instances with the given config key and value. func (c *ClusterTx) GetLocalInstanceWithVsockID(ctx context.Context, vsockID int) (*cluster.Instance, error) { q := ` SELECT instances.id, projects.name AS project, instances.name, nodes.name AS node, instances.type, instances.architecture, instances.ephemeral, instances.creation_date, instances.stateful, instances.last_use_date, coalesce(instances.description, ''), instances.expiry_date FROM instances JOIN projects ON instances.project_id = projects.id JOIN nodes ON instances.node_id = nodes.id JOIN instances_config ON instances.id = instances_config.instance_id WHERE instances.node_id = ? AND instances.type = ? AND instances_config.key = "volatile.vsock_id" AND instances_config.value = ? LIMIT 1 ` inargs := []any{c.nodeID, instancetype.VM, vsockID} inst := cluster.Instance{} err := c.tx.QueryRowContext(ctx, q, inargs...).Scan(&inst.ID, &inst.Project, &inst.Name, &inst.Node, &inst.Type, &inst.Architecture, &inst.Ephemeral, &inst.CreationDate, &inst.Stateful, &inst.LastUseDate, &inst.Description, &inst.ExpiryDate) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, api.StatusErrorf(http.StatusNotFound, "Instance not found") } return nil, err } return &inst, nil } // GetInstancePool returns the storage pool of a given instance (or snapshot). func (c *ClusterTx) GetInstancePool(ctx context.Context, projectName string, instanceName string) (string, error) { // Strip snapshot name if supplied in instanceName, and lookup the storage pool of the parent instance // as that must always be the same as the snapshot's storage pool. instanceName, _, _ = api.GetParentAndSnapshotName(instanceName) remoteDrivers := StorageRemoteDriverNames() // Get container storage volume. Since container names are globally // unique, and their storage volumes carry the same name, their storage // volumes are unique too. poolName := "" query := fmt.Sprintf(` SELECT storage_pools.name FROM storage_pools JOIN storage_volumes_all ON storage_pools.id=storage_volumes_all.storage_pool_id JOIN instances ON instances.name=storage_volumes_all.name JOIN projects ON projects.id=instances.project_id WHERE projects.name=? AND storage_volumes_all.name=? AND storage_volumes_all.type IN (?,?) AND storage_volumes_all.project_id = instances.project_id AND (storage_volumes_all.node_id=? OR storage_volumes_all.node_id IS NULL AND storage_pools.driver IN %s)`, query.Params(len(remoteDrivers))) inargs := []any{projectName, instanceName, StoragePoolVolumeTypeContainer, StoragePoolVolumeTypeVM, c.nodeID} outargs := []any{&poolName} for _, driver := range remoteDrivers { inargs = append(inargs, driver) } err := c.tx.QueryRowContext(ctx, query, inargs...).Scan(outargs...) if err != nil { if errors.Is(err, sql.ErrNoRows) { return "", api.StatusErrorf(http.StatusNotFound, "Instance storage pool not found") } return "", err } return poolName, nil } // DeleteInstance removes the instance with the given name from the database. func (c *ClusterTx) DeleteInstance(ctx context.Context, project, name string) error { if strings.Contains(name, internalInstance.SnapshotDelimiter) { parts := strings.SplitN(name, internalInstance.SnapshotDelimiter, 2) return cluster.DeleteInstanceSnapshot(ctx, c.tx, project, parts[0], parts[1]) } return cluster.DeleteInstance(ctx, c.tx, project, name) } // GetInstanceProjectAndName returns the project and the name of the instance // with the given ID. func (c *ClusterTx) GetInstanceProjectAndName(ctx context.Context, id int) (string, string, error) { var project string var name string q := ` SELECT projects.name, instances.name FROM instances JOIN projects ON projects.id = instances.project_id WHERE instances.id=? ` err := c.tx.QueryRowContext(ctx, q, id).Scan(&project, &name) if errors.Is(err, sql.ErrNoRows) { return "", "", api.StatusErrorf(http.StatusNotFound, "Instance not found") } return project, name, err } // GetInstanceID returns the ID of the instance with the given name. func (c *ClusterTx) GetInstanceID(ctx context.Context, project, name string) (int, error) { id, err := cluster.GetInstanceID(ctx, c.tx, project, name) return int(id), err } // GetInstanceConfig returns the value of the given key in the configuration // of the instance with the given ID. func (c *ClusterTx) GetInstanceConfig(ctx context.Context, id int, key string) (string, error) { q := "SELECT value FROM instances_config WHERE instance_id=? AND key=?" value := "" err := c.tx.QueryRowContext(ctx, q, id, key).Scan(&value) if errors.Is(err, sql.ErrNoRows) { return "", api.StatusErrorf(http.StatusNotFound, "Instance config not found") } return value, err } // UpdateInstanceStatefulFlag toggles the stateful flag of the instance with // the given ID. func (c *ClusterTx) UpdateInstanceStatefulFlag(ctx context.Context, id int, stateful bool) error { statefulInt := 0 if stateful { statefulInt = 1 } _, err := c.tx.ExecContext(ctx, "UPDATE instances SET stateful=? WHERE id=?", statefulInt, id) if err != nil { return fmt.Errorf("Failed updating instance stateful flag: %w", err) } return nil } // UpdateInstanceSnapshotCreationDate updates the creation_date field of the instance snapshot with ID. func (c *ClusterTx) UpdateInstanceSnapshotCreationDate(ctx context.Context, instanceID int, date time.Time) error { stmt := `UPDATE instances_snapshots SET creation_date=? WHERE id=?` _, err := c.tx.ExecContext(ctx, stmt, date, instanceID) if err != nil { return fmt.Errorf("Failed updating instance snapshot creation date: %w", err) } return nil } // GetInstanceSnapshotsNames returns the names of all snapshots of the instance // in the given project with the given name. // Returns snapshots slice ordered by when they were created, oldest first. func (c *ClusterTx) GetInstanceSnapshotsNames(ctx context.Context, project, name string) ([]string, error) { result := []string{} q := ` SELECT instances_snapshots.name FROM instances_snapshots JOIN instances ON instances.id = instances_snapshots.instance_id JOIN projects ON projects.id = instances.project_id WHERE projects.name=? AND instances.name=? ORDER BY instances_snapshots.creation_date, instances_snapshots.id ` inargs := []any{project, name} outfmt := []any{name} dbResults, err := queryScan(ctx, c, q, inargs, outfmt) if err != nil { return result, err } for _, r := range dbResults { result = append(result, name+internalInstance.SnapshotDelimiter+r[0].(string)) } return result, nil } // GetNextInstanceSnapshotIndex returns the index that the next snapshot of the // instance with the given name and pattern should have. func (c *ClusterTx) GetNextInstanceSnapshotIndex(ctx context.Context, project string, name string, pattern string) int { q := ` SELECT instances_snapshots.name FROM instances_snapshots JOIN instances ON instances.id = instances_snapshots.instance_id JOIN projects ON projects.id = instances.project_id WHERE projects.name=? AND instances.name=? ORDER BY instances_snapshots.creation_date, instances_snapshots.id ` var numstr string inargs := []any{project, name} outfmt := []any{numstr} results, err := queryScan(ctx, c, q, inargs, outfmt) if err != nil { return 0 } max := 0 for _, r := range results { snapOnlyName := r[0].(string) fields := strings.SplitN(pattern, "%d", 2) var num int count, err := fmt.Sscanf(snapOnlyName, fmt.Sprintf("%s%%d%s", fields[0], fields[1]), &num) if err != nil || count != 1 { continue } if num >= max { max = num + 1 } } return max } // DeleteReadyStateFromLocalInstances deletes the volatile.last_state.ready config key // from all local instances. func (c *ClusterTx) DeleteReadyStateFromLocalInstances(ctx context.Context) error { nodeID := c.GetNodeID() _, err := c.tx.ExecContext(ctx, ` DELETE FROM instances_config WHERE instances_config.id IN ( SELECT instances_config.id FROM instances_config JOIN instances ON instances_config.instance_id=instances.id JOIN nodes ON instances.node_id=nodes.id WHERE key="volatile.last_state.ready" AND nodes.id=? )`, nodeID) if err != nil { return fmt.Errorf("Failed deleting ready state from local instances: %w", err) } return nil } // CreateInstanceConfig inserts a new config for the instance with the given ID. func CreateInstanceConfig(ctx context.Context, tx *sql.Tx, id int, config map[string]string) error { sql := "INSERT INTO instances_config (instance_id, key, value) values (?, ?, ?)" for k, v := range config { if v == "" { continue } _, err := tx.ExecContext(ctx, sql, id, k, v) if err != nil { return fmt.Errorf("Error adding configuration item %q = %q to instance %d: %w", k, v, id, err) } } return nil } // UpdateInstance updates the description, architecture and ephemeral flag of // the instance with the given ID. func UpdateInstance(tx *sql.Tx, id int, description string, architecture int, ephemeral bool, expiryDate time.Time, ) error { str := "UPDATE instances SET description=?, architecture=?, ephemeral=?, expiry_date=? WHERE id=?" ephemeralInt := 0 if ephemeral { ephemeralInt = 1 } var err error if expiryDate.IsZero() { _, err = tx.Exec(str, description, architecture, ephemeralInt, "", id) } else { _, err = tx.Exec(str, description, architecture, ephemeralInt, expiryDate, id) } if err != nil { return err } return nil } // GetInstancesCount returns the number of instances with possible filtering for project or location. // It also supports looking for instances currently being created. func (c *ClusterTx) GetInstancesCount(ctx context.Context, projectName string, locationName string, includePending bool) (int, error) { var err error // Load the project ID if needed. projectID := int64(-1) if projectName != "" { projectID, err = cluster.GetProjectID(ctx, c.Tx(), projectName) if err != nil { return -1, err } } // Load the cluster member ID if needed. nodeID := int64(-1) if locationName != "" { nodeID, err = cluster.GetNodeID(ctx, c.Tx(), locationName) if err != nil { return -1, err } } // Count the instances. var count int if projectID != -1 && nodeID != -1 { // Count for specified project and cluster member. created, err := query.Count(ctx, c.tx, "instances", "project_id=? AND node_id=?", projectID, nodeID) if err != nil { return -1, fmt.Errorf("Failed to get instances count: %w", err) } count += created if includePending { pending, err := query.Count(ctx, c.tx, "operations", "project_id=? AND node_id=? AND type=?", projectID, nodeID, operationtype.InstanceCreate) if err != nil { return -1, fmt.Errorf("Failed to get pending instances count: %w", err) } count += pending } } else if projectID != -1 { // Count for specified project. created, err := query.Count(ctx, c.tx, "instances", "project_id=?", projectID) if err != nil { return -1, fmt.Errorf("Failed to get instances count: %w", err) } count += created if includePending { pending, err := query.Count(ctx, c.tx, "operations", "project_id=? AND type=?", projectID, operationtype.InstanceCreate) if err != nil { return -1, fmt.Errorf("Failed to get pending instances count: %w", err) } count += pending } } else if nodeID != -1 { // Count for specified cluster member. created, err := query.Count(ctx, c.tx, "instances", "node_id=?", nodeID) if err != nil { return -1, fmt.Errorf("Failed to get instances count: %w", err) } count += created if includePending { pending, err := query.Count(ctx, c.tx, "operations", "node_id=? AND type=?", nodeID, operationtype.InstanceCreate) if err != nil { return -1, fmt.Errorf("Failed to get pending instances count: %w", err) } count += pending } } else { // Count everything. created, err := query.Count(ctx, c.tx, "instances", "") if err != nil { return -1, fmt.Errorf("Failed to get instances count: %w", err) } count += created if includePending { pending, err := query.Count(ctx, c.tx, "operations", "type=?", operationtype.InstanceCreate) if err != nil { return -1, fmt.Errorf("Failed to get pending instances count: %w", err) } count += pending } } return count, nil } incus-7.0.0/internal/server/db/instances_test.go000066400000000000000000000443671517523235500217510ustar00rootroot00000000000000//go:build linux && cgo && !agent package db_test import ( "context" "database/sql" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/api" ) func TestContainerList(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() nodeID1 := int64(1) // This is the default local member nodeID2, err := tx.CreateNode("node2", "1.2.3.4:666") require.NoError(t, err) addContainer(t, tx, nodeID2, "c1") addContainer(t, tx, nodeID1, "c2") addContainer(t, tx, nodeID2, "c3") addContainerConfig(t, tx, "c2", "x", "y") addContainerConfig(t, tx, "c3", "z", "w") addContainerConfig(t, tx, "c3", "a", "b") addContainerDevice(t, tx, "c2", "eth0", "nic", nil) addContainerDevice(t, tx, "c3", "root", "disk", map[string]string{"x": "y"}) instType := instancetype.Container containers, err := cluster.GetInstances(context.TODO(), tx.Tx(), cluster.InstanceFilter{Type: &instType}) require.NoError(t, err) assert.Len(t, containers, 3) c1 := containers[0] c1Devices, err := cluster.GetInstanceDevices(context.TODO(), tx.Tx(), c1.ID) require.NoError(t, err) c1Config, err := cluster.GetInstanceConfig(context.TODO(), tx.Tx(), c1.ID) require.NoError(t, err) assert.Equal(t, "c1", c1.Name) assert.Equal(t, "node2", c1.Node) assert.Equal(t, map[string]string{}, c1Config) assert.Len(t, c1Devices, 0) c2 := containers[1] c2Devices, err := cluster.GetInstanceDevices(context.TODO(), tx.Tx(), c2.ID) require.NoError(t, err) c2Config, err := cluster.GetInstanceConfig(context.TODO(), tx.Tx(), c2.ID) require.NoError(t, err) assert.Equal(t, "c2", c2.Name) assert.Equal(t, map[string]string{"x": "y"}, c2Config) assert.Equal(t, "none", c2.Node) assert.Len(t, c2Devices, 1) assert.Equal(t, "eth0", c2Devices["eth0"].Name) assert.Equal(t, "nic", c2Devices["eth0"].Type.String()) c3 := containers[2] c3Devices, err := cluster.GetInstanceDevices(context.TODO(), tx.Tx(), c3.ID) require.NoError(t, err) c3Config, err := cluster.GetInstanceConfig(context.TODO(), tx.Tx(), c3.ID) require.NoError(t, err) assert.Equal(t, "c3", c3.Name) assert.Equal(t, map[string]string{"z": "w", "a": "b"}, c3Config) assert.Equal(t, "node2", c3.Node) assert.Len(t, c3Devices, 1) assert.Equal(t, "root", c3Devices["root"].Name) assert.Equal(t, "disk", c3Devices["root"].Type.String()) assert.Equal(t, map[string]string{"x": "y"}, c3Devices["root"].Config) } func TestContainerList_FilterByNode(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() nodeID1 := int64(1) // This is the default local member nodeID2, err := tx.CreateNode("node2", "1.2.3.4:666") require.NoError(t, err) addContainer(t, tx, nodeID2, "c1") addContainer(t, tx, nodeID1, "c2") addContainer(t, tx, nodeID2, "c3") project := "default" node := "node2" instType := instancetype.Container filter := cluster.InstanceFilter{Project: &project, Node: &node, Type: &instType} containers, err := cluster.GetInstances(context.TODO(), tx.Tx(), filter) require.NoError(t, err) assert.Len(t, containers, 2) assert.Equal(t, 1, containers[0].ID) assert.Equal(t, "c1", containers[0].Name) assert.Equal(t, "node2", containers[0].Node) assert.Equal(t, 3, containers[1].ID) assert.Equal(t, "c3", containers[1].Name) assert.Equal(t, "node2", containers[1].Node) } func TestInstanceList_ContainerWithSameNameInDifferentProjects(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() ctx := context.Background() // Create a project with no features project1 := cluster.Project{} project1.Name = "blah" _, err := cluster.CreateProject(ctx, tx.Tx(), project1) require.NoError(t, err) // Create a project with the profiles feature and a custom profile. project2 := cluster.Project{} project2.Name = "test" project2Config := map[string]string{"features.profiles": "true"} id, err := cluster.CreateProject(ctx, tx.Tx(), project2) require.NoError(t, err) err = cluster.CreateProjectConfig(ctx, tx.Tx(), id, project2Config) require.NoError(t, err) profile := cluster.Profile{ Project: "test", Name: "intranet", } _, err = cluster.CreateProfile(ctx, tx.Tx(), profile) require.NoError(t, err) // Create a container in project1 using the default profile from the // default project. c1p1 := cluster.Instance{ Project: "blah", Name: "c1", Node: "none", Type: instancetype.Container, Architecture: 1, Ephemeral: false, Stateful: true, } id, err = cluster.CreateInstance(context.TODO(), tx.Tx(), c1p1) require.NoError(t, err) err = cluster.UpdateInstanceProfiles(context.TODO(), tx.Tx(), int(id), c1p1.Project, []string{"default"}) require.NoError(t, err) // Create a container in project2 using the custom profile from the // project. c1p2 := cluster.Instance{ Project: "test", Name: "c1", Node: "none", Type: instancetype.Container, Architecture: 1, Ephemeral: false, Stateful: true, } id, err = cluster.CreateInstance(context.TODO(), tx.Tx(), c1p2) require.NoError(t, err) err = cluster.UpdateInstanceProfiles(context.TODO(), tx.Tx(), int(id), c1p2.Project, []string{"intranet"}) require.NoError(t, err) containers, err := cluster.GetInstances(context.TODO(), tx.Tx()) require.NoError(t, err) c1Profiles, err := cluster.GetInstanceProfiles(context.TODO(), tx.Tx(), containers[0].ID) require.NoError(t, err) c2Profiles, err := cluster.GetInstanceProfiles(context.TODO(), tx.Tx(), containers[1].ID) require.NoError(t, err) assert.Len(t, containers, 2) assert.Equal(t, "blah", containers[0].Project) assert.Len(t, c1Profiles, 1) assert.Equal(t, "default", c1Profiles[0].Name) assert.Equal(t, "test", containers[1].Project) assert.Len(t, c2Profiles, 1) assert.Equal(t, "intranet", c2Profiles[0].Name) } func TestInstanceList(t *testing.T) { c, clusterCleanup := db.NewTestCluster(t) defer clusterCleanup() err := c.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { profile := cluster.Profile{ Project: "default", Name: "profile1", } profileConfig := map[string]string{"a": "1"} profileDevices := map[string]cluster.Device{ "root": { Name: "root", Type: cluster.TypeDisk, Config: map[string]string{"b": "2"}, }, } id, err := cluster.CreateProfile(ctx, tx.Tx(), profile) if err != nil { return err } err = cluster.CreateProfileConfig(ctx, tx.Tx(), id, profileConfig) if err != nil { return err } err = cluster.CreateProfileDevices(ctx, tx.Tx(), id, profileDevices) if err != nil { return err } container := cluster.Instance{ Project: "default", Name: "c1", Node: "none", Type: instancetype.Container, Architecture: 1, Ephemeral: false, Stateful: true, } id, err = cluster.CreateInstance(context.TODO(), tx.Tx(), container) if err != nil { return err } err = cluster.CreateInstanceConfig(context.TODO(), tx.Tx(), id, map[string]string{"c": "3"}) if err != nil { return err } err = cluster.CreateInstanceDevices(context.TODO(), tx.Tx(), id, map[string]cluster.Device{"eth0": {Name: "eth0", Type: cluster.TypeNIC, Config: map[string]string{"d": "4"}}}) if err != nil { return err } err = cluster.UpdateInstanceProfiles(context.TODO(), tx.Tx(), int(id), container.Project, []string{"default", "profile1"}) if err != nil { return err } return nil }) require.NoError(t, err) var instances []db.InstanceArgs err = c.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.InstanceList(ctx, func(dbInst db.InstanceArgs, p api.Project) error { dbInst.Config = db.ExpandInstanceConfig(dbInst.Config, dbInst.Profiles) dbInst.Devices = db.ExpandInstanceDevices(dbInst.Devices, dbInst.Profiles) instances = append(instances, dbInst) return nil }) }) require.NoError(t, err) assert.Len(t, instances, 1) assert.Equal(t, map[string]string{"a": "1", "c": "3"}, instances[0].Config) assert.Equal(t, map[string]map[string]string{ "root": {"type": "disk", "b": "2"}, "eth0": {"type": "nic", "d": "4"}, }, instances[0].Devices.CloneNative()) } func TestCreateInstance(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() object := cluster.Instance{ Project: "default", Name: "c1", Type: 0, Node: "none", Architecture: 1, Ephemeral: true, Stateful: true, LastUseDate: sql.NullTime{Time: time.Now(), Valid: true}, Description: "container 1", } id, err := cluster.CreateInstance(context.TODO(), tx.Tx(), object) require.NoError(t, err) err = cluster.CreateInstanceConfig(context.TODO(), tx.Tx(), id, map[string]string{"x": "y"}) require.NoError(t, err) err = cluster.CreateInstanceDevices(context.TODO(), tx.Tx(), id, map[string]cluster.Device{"root": {Name: "root", Config: map[string]string{"type": "disk", "x": "y"}}}) require.NoError(t, err) err = cluster.UpdateInstanceProfiles(context.TODO(), tx.Tx(), int(id), object.Project, []string{"default"}) require.NoError(t, err) assert.Equal(t, int64(1), id) c1, err := cluster.GetInstance(context.TODO(), tx.Tx(), "default", "c1") require.NoError(t, err) c1Devices, err := cluster.GetInstanceDevices(context.TODO(), tx.Tx(), c1.ID) require.NoError(t, err) c1Config, err := cluster.GetInstanceConfig(context.TODO(), tx.Tx(), c1.ID) require.NoError(t, err) c1Profiles, err := cluster.GetInstanceProfiles(context.TODO(), tx.Tx(), c1.ID) require.NoError(t, err) assert.Equal(t, "c1", c1.Name) assert.Equal(t, map[string]string{"x": "y"}, c1Config) assert.Len(t, c1Devices, 1) assert.Equal(t, "root", c1Devices["root"].Name) assert.Equal(t, map[string]string{"type": "disk", "x": "y"}, c1Devices["root"].Config) assert.Len(t, c1Profiles, 1) assert.Equal(t, "default", c1Profiles[0].Name) } func TestCreateInstance_Snapshot(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() instance := cluster.Instance{ Project: "default", Name: "foo", Type: 0, Node: "none", Architecture: 2, Ephemeral: false, Stateful: false, LastUseDate: sql.NullTime{Time: time.Now(), Valid: true}, Description: "container 1", } id, err := cluster.CreateInstance(context.TODO(), tx.Tx(), instance) require.NoError(t, err) err = cluster.CreateInstanceConfig(context.TODO(), tx.Tx(), id, map[string]string{ "image.architecture": "x86_64", "image.description": "BusyBox x86_64", "image.name": "busybox-x86_64", "image.os": "BusyBox", "volatile.base_image": "1f7f054e6ccb", }) require.NoError(t, err) err = cluster.UpdateInstanceProfiles(context.TODO(), tx.Tx(), int(id), instance.Project, []string{"default"}) require.NoError(t, err) assert.Equal(t, int64(1), id) snapshot := cluster.Instance{ Project: "default", Name: "foo/snap0", Type: 1, Node: "none", Architecture: 2, Ephemeral: false, Stateful: false, LastUseDate: sql.NullTime{Time: time.Now(), Valid: true}, Description: "container 1", } id, err = cluster.CreateInstance(context.TODO(), tx.Tx(), snapshot) require.NoError(t, err) err = cluster.CreateInstanceConfig(context.TODO(), tx.Tx(), id, map[string]string{ "image.architecture": "x86_64", "image.description": "BusyBox x86_64", "image.name": "busybox-x86_64", "image.os": "BusyBox", "volatile.apply_template": "create", "volatile.base_image": "1f7f054e6ccb", "volatile.eth0.hwaddr": "10:66:6a:2a:3f:e2", "volatile.idmap.base": "0", }) require.NoError(t, err) err = cluster.UpdateInstanceProfiles(context.TODO(), tx.Tx(), int(id), instance.Project, []string{"default"}) require.NoError(t, err) assert.Equal(t, int64(2), id) _, err = cluster.GetInstance(context.TODO(), tx.Tx(), "default", "foo/snap0") require.NoError(t, err) } // Containers are grouped by node address. func TestGetInstancesByMemberAddress(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() nodeID1 := int64(1) // This is the default local member nodeID2, err := tx.CreateNode("node2", "1.2.3.4:666") require.NoError(t, err) nodeID3, err := tx.CreateNode("node3", "5.6.7.8:666") require.NoError(t, err) require.NoError(t, tx.SetNodeHeartbeat("5.6.7.8:666", time.Now().Add(-time.Minute))) addContainer(t, tx, nodeID2, "c1") addContainer(t, tx, nodeID1, "c2") addContainer(t, tx, nodeID3, "c3") addContainer(t, tx, nodeID2, "c4") result, err := tx.GetInstancesByMemberAddress(context.Background(), time.Duration(db.DefaultOfflineThreshold)*time.Second, []string{"default"}) require.NoError(t, err) assert.Equal( t, map[string][]db.Instance{ "": {{ID: 2, Project: api.ProjectDefaultName, Name: "c2", Location: "none"}}, "1.2.3.4:666": {{ID: 1, Project: api.ProjectDefaultName, Name: "c1", Location: "node2"}, {ID: 4, Project: api.ProjectDefaultName, Name: "c4", Location: "node2"}}, "0.0.0.0": {{ID: 3, Project: api.ProjectDefaultName, Name: "c3", Location: "node3"}}, }, result) } func TestGetInstancePool(t *testing.T) { dbCluster, cleanup := db.NewTestCluster(t) defer cleanup() err := dbCluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { poolID, err := tx.CreateStoragePool(ctx, "default", "", "dir", nil) if err != nil { return err } _, err = tx.CreateStoragePoolVolume(ctx, "default", "c1", "", db.StoragePoolVolumeTypeContainer, poolID, nil, db.StoragePoolVolumeContentTypeFS, time.Now()) if err != nil { return err } container := cluster.Instance{ Project: "default", Name: "c1", Node: "none", } id, err := cluster.CreateInstance(context.TODO(), tx.Tx(), container) if err != nil { return err } err = cluster.CreateInstanceDevices(context.TODO(), tx.Tx(), id, map[string]cluster.Device{ "root": { Name: "root", Config: map[string]string{ "path": "/", "pool": "default", "type": "disk", }, }, }) if err != nil { return err } poolName, err := tx.GetInstancePool(ctx, "default", "c1") if err != nil { return err } assert.Equal(t, "default", poolName) return nil }) require.NoError(t, err) } // All containers on a node are loaded in bulk. func TestGetLocalInstancesInProject(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() nodeID1 := int64(1) // This is the default local member nodeID2, err := tx.CreateNode("node2", "1.2.3.4:666") require.NoError(t, err) addContainer(t, tx, nodeID2, "c1") addContainer(t, tx, nodeID1, "c2") addContainer(t, tx, nodeID1, "c3") addContainer(t, tx, nodeID1, "c4") addContainerConfig(t, tx, "c2", "x", "y") addContainerConfig(t, tx, "c3", "z", "w") addContainerConfig(t, tx, "c3", "a", "b") addContainerDevice(t, tx, "c2", "eth0", "nic", nil) addContainerDevice(t, tx, "c4", "root", "disk", map[string]string{"x": "y"}) instType := instancetype.Container containers, err := tx.GetLocalInstancesInProject(context.TODO(), cluster.InstanceFilter{Type: &instType}) require.NoError(t, err) assert.Len(t, containers, 3) assert.Equal(t, "c2", containers[0].Name) assert.Equal(t, "c3", containers[1].Name) assert.Equal(t, "c4", containers[2].Name) assert.Equal(t, "none", containers[0].Node) assert.Equal(t, "none", containers[1].Node) assert.Equal(t, "none", containers[2].Node) c1Config, err := cluster.GetInstanceConfig(context.TODO(), tx.Tx(), containers[0].ID) require.NoError(t, err) c1Devices, err := cluster.GetInstanceDevices(context.TODO(), tx.Tx(), containers[0].ID) require.NoError(t, err) assert.Equal(t, map[string]string{"x": "y"}, c1Config) assert.Equal(t, map[string]map[string]string{"eth0": {"type": "nic"}}, cluster.DevicesToAPI(c1Devices)) c2Config, err := cluster.GetInstanceConfig(context.TODO(), tx.Tx(), containers[1].ID) require.NoError(t, err) c2Devices, err := cluster.GetInstanceDevices(context.TODO(), tx.Tx(), containers[1].ID) require.NoError(t, err) assert.Equal(t, map[string]string{"z": "w", "a": "b"}, c2Config) assert.Len(t, c2Devices, 0) c3Config, err := cluster.GetInstanceConfig(context.TODO(), tx.Tx(), containers[2].ID) require.NoError(t, err) c3Devices, err := cluster.GetInstanceDevices(context.TODO(), tx.Tx(), containers[2].ID) require.NoError(t, err) assert.Len(t, c3Config, 0) assert.Equal(t, map[string]map[string]string{"root": {"type": "disk", "x": "y"}}, cluster.DevicesToAPI(c3Devices)) } func addContainer(t *testing.T, tx *db.ClusterTx, nodeID int64, name string) { stmt := ` INSERT INTO instances(node_id, name, architecture, type, project_id, description) VALUES (?, ?, 1, ?, 1, '') ` _, err := tx.Tx().Exec(stmt, nodeID, name, instancetype.Container) require.NoError(t, err) } func addContainerConfig(t *testing.T, tx *db.ClusterTx, container, key, value string) { id := getContainerID(t, tx, container) stmt := ` INSERT INTO instances_config(instance_id, key, value) VALUES (?, ?, ?) ` _, err := tx.Tx().Exec(stmt, id, key, value) require.NoError(t, err) } func addContainerDevice(t *testing.T, tx *db.ClusterTx, container, name, typ string, config map[string]string) { id := getContainerID(t, tx, container) code, err := cluster.NewDeviceType(typ) require.NoError(t, err) stmt := ` INSERT INTO instances_devices(instance_id, name, type) VALUES (?, ?, ?) ` _, err = tx.Tx().Exec(stmt, id, name, code) require.NoError(t, err) deviceID := getDeviceID(t, tx, id, name) for key, value := range config { stmt := ` INSERT INTO instances_devices_config(instance_device_id, key, value) VALUES (?, ?, ?) ` _, err = tx.Tx().Exec(stmt, deviceID, key, value) require.NoError(t, err) } } // Return the container ID given its name. func getContainerID(t *testing.T, tx *db.ClusterTx, name string) int64 { var id int64 stmt := "SELECT id FROM instances WHERE name=?" row := tx.Tx().QueryRow(stmt, name) err := row.Scan(&id) require.NoError(t, err) return id } // Return the device ID given its container ID and name. func getDeviceID(t *testing.T, tx *db.ClusterTx, containerID int64, name string) int64 { var id int64 stmt := "SELECT id FROM instances_devices WHERE instance_id=? AND name=?" row := tx.Tx().QueryRow(stmt, containerID, name) err := row.Scan(&id) require.NoError(t, err) return id } incus-7.0.0/internal/server/db/networks.go000066400000000000000000000644011517523235500205660ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "database/sql" "errors" "fmt" "net/http" "regexp" "slices" "strings" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // GetNetworksLocalConfig returns a map associating each network name to its // node-specific config values on the local member (i.e. the ones where node_id // equals the ID of the local member). func (c *ClusterTx) GetNetworksLocalConfig(ctx context.Context) (map[string]map[string]string, error) { names, err := query.SelectStrings(ctx, c.tx, "SELECT name FROM networks") if err != nil { return nil, err } networks := make(map[string]map[string]string, len(names)) for _, name := range names { table := "networks_config JOIN networks ON networks.id=networks_config.network_id" config, err := query.SelectConfig(ctx, c.tx, table, "networks.name=? AND networks_config.node_id=?", name, c.nodeID) if err != nil { return nil, err } networks[name] = config } return networks, nil } // GetNonPendingNetworkIDs returns a map associating each network name to its ID. // // Pending networks are skipped. func (c *ClusterTx) GetNonPendingNetworkIDs(ctx context.Context) (map[string]map[string]int64, error) { type network struct { id int64 name string projectName string } networks := []network{} sql := "SELECT networks.id, networks.name, projects.name FROM networks JOIN projects on projects.id = networks.project_id WHERE NOT networks.state=?" err := query.Scan(ctx, c.tx, sql, func(scan func(dest ...any) error) error { n := network{} err := scan(&n.id, &n.name, &n.projectName) if err != nil { return err } networks = append(networks, n) return nil }, networkPending) if err != nil { return nil, err } ids := map[string]map[string]int64{} for _, network := range networks { if ids[network.projectName] == nil { ids[network.projectName] = map[string]int64{} } ids[network.projectName][network.name] = network.id } return ids, nil } // GetCreatedNetworks returns a map of api.Network associated to project and network ID. // Only networks that have are in state networkCreated are returned. func (c *ClusterTx) GetCreatedNetworks(ctx context.Context) (map[string]map[int64]api.Network, error) { return c.getCreatedNetworks(ctx, "") } // GetCreatedNetworkNamesByProject returns the names of all networks that are in state networkCreated. func (c *ClusterTx) GetCreatedNetworkNamesByProject(ctx context.Context, project string) ([]string, error) { return c.networks(ctx, project, "state=?", networkCreated) } // GetCreatedNetworksByProject returns a map of api.Network in a project associated to network ID. // Only networks that have are in state networkCreated are returned. func (c *ClusterTx) GetCreatedNetworksByProject(ctx context.Context, projectName string) (map[int64]api.Network, error) { nets, err := c.getCreatedNetworks(ctx, projectName) if err != nil { return nil, err } return nets[projectName], nil } // getCreatedNetworks returns a map of api.Network associated to project and network ID. // Supports an optional projectName filter. If projectName is empty, all networks in created state are returned. func (c *ClusterTx) getCreatedNetworks(ctx context.Context, projectName string) (map[string]map[int64]api.Network, error) { var sb strings.Builder sb.WriteString(`SELECT projects.name, networks.id, networks.name, coalesce(networks.description, ''), networks.type, networks.state FROM networks JOIN projects on projects.id = networks.project_id WHERE networks.state = ? `) args := []any{networkCreated} if projectName != "" { sb.WriteString(" AND projects.name = ?") args = append(args, projectName) } rows, err := c.tx.QueryContext(ctx, sb.String(), args...) if err != nil { return nil, err } defer func() { _ = rows.Close() }() projectNetworks := make(map[string]map[int64]api.Network) for i := 0; rows.Next(); i++ { var projectName string var networkID int64 var networkType NetworkType var networkState NetworkState var network api.Network err := rows.Scan(&projectName, &networkID, &network.Name, &network.Description, &networkType, &networkState) if err != nil { return nil, err } // Populate Status and Type fields by converting from DB values. network.Status = NetworkStateToAPIStatus(networkState) networkFillType(&network, networkType) if projectNetworks[projectName] != nil { projectNetworks[projectName][networkID] = network } else { projectNetworks[projectName] = map[int64]api.Network{ networkID: network, } } } err = rows.Err() if err != nil { return nil, err } // Populate config. for projectName, networks := range projectNetworks { for networkID, network := range networks { networkConfig, err := query.SelectConfig(ctx, c.tx, "networks_config", "network_id=? AND (node_id=? OR node_id IS NULL)", networkID, c.nodeID) if err != nil { return nil, err } network.Config = networkConfig nodes, err := c.NetworkNodes(ctx, networkID) if err != nil { return nil, err } for _, node := range nodes { network.Locations = append(network.Locations, node.Name) } projectNetworks[projectName][networkID] = network } } return projectNetworks, nil } // GetNetworkID returns the ID of the network with the given name. func (c *ClusterTx) GetNetworkID(ctx context.Context, projectName string, name string) (int64, error) { stmt := "SELECT id FROM networks WHERE project_id = (SELECT id FROM projects WHERE name = ?) AND name=?" ids, err := query.SelectIntegers(ctx, c.tx, stmt, projectName, name) if err != nil { return -1, err } switch len(ids) { case 0: return -1, api.StatusErrorf(http.StatusNotFound, "Network not found") case 1: return int64(ids[0]), nil default: return -1, errors.New("More than one network has the given name") } } // GetNetworkNameAndProjectWithID returns the network name and project name for the given ID. func (c *ClusterTx) GetNetworkNameAndProjectWithID(ctx context.Context, networkID int) (string, string, error) { var networkName string var projectName string q := `SELECT networks.name, projects.name FROM networks JOIN projects ON projects.id=networks.project_id WHERE networks.id=?` inargs := []any{networkID} outargs := []any{&networkName, &projectName} err := dbQueryRowScan(ctx, c, q, inargs, outargs) if err != nil { if errors.Is(err, sql.ErrNoRows) { return "", "", api.StatusErrorf(http.StatusNotFound, "Network not found") } return "", "", err } return networkName, projectName, nil } // CreateNetworkConfig adds a new entry in the networks_config table. func (c *ClusterTx) CreateNetworkConfig(networkID, nodeID int64, config map[string]string) error { return networkConfigAdd(c.tx, networkID, nodeID, config) } // NetworkNodeJoin adds a new entry in the networks_nodes table. // // It should only be used when a new node joins the cluster, when it's safe to // assume that the relevant network has already been created on the joining node, // and we just need to track it. func (c *ClusterTx) NetworkNodeJoin(networkID, nodeID int64) error { columns := []string{"network_id", "node_id", "state"} // Create network node with networkCreated state as we expect the network to already be setup. values := []any{networkID, nodeID, networkCreated} _, err := query.UpsertObject(c.tx, "networks_nodes", columns, values) return err } // NetworkNodeConfigs returns the node-specific configuration of all // nodes grouped by node name, for the given networkID. // // If the network is not defined on all nodes, an error is returned. func (c *ClusterTx) NetworkNodeConfigs(ctx context.Context, networkID int64) (map[string]map[string]string, error) { // Fetch all nodes. nodes, err := c.GetNodes(ctx) if err != nil { return nil, err } // Fetch the names of the nodes where the storage network is defined. stmt := ` SELECT nodes.name FROM nodes LEFT JOIN networks_nodes ON networks_nodes.node_id = nodes.id LEFT JOIN networks ON networks_nodes.network_id = networks.id WHERE networks.id = ? AND networks.state = ? ` defined, err := query.SelectStrings(ctx, c.tx, stmt, networkID, networkPending) if err != nil { return nil, err } // Figure which nodes are missing missing := []string{} for _, node := range nodes { if !slices.Contains(defined, node.Name) { missing = append(missing, node.Name) } } if len(missing) > 0 { return nil, fmt.Errorf("Network not defined on nodes: %s", strings.Join(missing, ", ")) } configs := map[string]map[string]string{} for _, node := range nodes { config, err := query.SelectConfig(ctx, c.tx, "networks_config", "node_id=?", node.ID) if err != nil { return nil, err } configs[node.Name] = config } return configs, nil } // CreatePendingNetwork creates a new pending network on the node with the given name. func (c *ClusterTx) CreatePendingNetwork(ctx context.Context, node string, projectName string, name string, description string, netType NetworkType, conf map[string]string) error { // First check if a network with the given name exists, and, if so, that it's in the pending state. network := struct { id int64 state NetworkState netType NetworkType }{} sql := "SELECT id, state, type FROM networks WHERE project_id = (SELECT id FROM projects WHERE name = ?) AND name=?" count := 0 err := query.Scan(ctx, c.tx, sql, func(scan func(dest ...any) error) error { // Ensure that there is at most one network with the given name. if count != 0 { return errors.New("More than one network exists with the given name") } count++ return scan(&network.id, &network.state, &network.netType) }, projectName, name) if err != nil { return err } networkID := network.id if networkID == 0 { projectID, err := cluster.GetProjectID(context.Background(), c.tx, projectName) if err != nil { return fmt.Errorf("Fetch project ID: %w", err) } // No existing network with the given name was found, let's create one. columns := []string{"project_id", "name", "type", "description"} values := []any{projectID, name, netType, description} networkID, err = query.UpsertObject(c.tx, "networks", columns, values) if err != nil { return err } } else { // Check that the existing network is in the networkPending state. if network.state != networkPending { return errors.New("Network is not in pending state") } // Check that the existing network type matches the requested type. if network.netType != netType { return errors.New("Requested network type doesn't match type in existing database record") } } // Get the ID of the node with the given name. nodeInfo, err := c.GetNodeByName(ctx, node) if err != nil { return err } // Check that no network entry for this node and network exists yet. count, err = query.Count(ctx, c.tx, "networks_nodes", "network_id=? AND node_id=?", networkID, nodeInfo.ID) if err != nil { return err } if count != 0 { return ErrAlreadyDefined } // Insert the node-specific configuration with state networkPending. columns := []string{"network_id", "node_id", "state"} values := []any{networkID, nodeInfo.ID, networkPending} _, err = query.UpsertObject(c.tx, "networks_nodes", columns, values) if err != nil { return err } err = c.CreateNetworkConfig(networkID, nodeInfo.ID, conf) if err != nil { return err } return nil } // NetworkCreated sets the state of the given network to networkCreated. func (c *ClusterTx) NetworkCreated(project string, name string) error { return c.networkState(project, name, networkCreated) } // NetworkErrored sets the state of the given network to networkErrored. func (c *ClusterTx) NetworkErrored(project string, name string) error { return c.networkState(project, name, networkErrored) } func (c *ClusterTx) networkState(project string, name string, state NetworkState) error { stmt := "UPDATE networks SET state=? WHERE project_id = (SELECT id FROM projects WHERE name = ?) AND name=?" result, err := c.tx.Exec(stmt, state, project, name) if err != nil { return err } n, err := result.RowsAffected() if err != nil { return err } if n != 1 { return api.StatusErrorf(http.StatusNotFound, "Network not found") } return nil } // NetworkNodeCreated sets the state of the given network for the local member to networkCreated. func (c *ClusterTx) NetworkNodeCreated(networkID int64) error { return c.networkNodeState(networkID, networkCreated) } // networkNodeState updates the network member state for the local member and specified network ID. func (c *ClusterTx) networkNodeState(networkID int64, state NetworkState) error { stmt := "UPDATE networks_nodes SET state=? WHERE network_id = ? and node_id = ?" result, err := c.tx.Exec(stmt, state, networkID, c.nodeID) if err != nil { return err } n, err := result.RowsAffected() if err != nil { return err } if n != 1 { return api.StatusErrorf(http.StatusNotFound, "Network not found") } return nil } // NetworkNodes returns the nodes keyed by node ID that the given network is defined on. func (c *ClusterTx) NetworkNodes(ctx context.Context, networkID int64) (map[int64]NetworkNode, error) { nodes := []NetworkNode{} sql := ` SELECT nodes.id, nodes.name, networks_nodes.state FROM nodes JOIN networks_nodes ON networks_nodes.node_id = nodes.id WHERE networks_nodes.network_id = ? ` err := query.Scan(ctx, c.tx, sql, func(scan func(dest ...any) error) error { node := NetworkNode{} err := scan(&node.ID, &node.Name, &node.State) if err != nil { return err } nodes = append(nodes, node) return nil }, networkID) if err != nil { return nil, err } netNodes := map[int64]NetworkNode{} for _, node := range nodes { netNodes[node.ID] = node } return netNodes, nil } // GetNetworkURIs returns the URIs for the networks with the given project. func (c *ClusterTx) GetNetworkURIs(ctx context.Context, projectID int, project string) ([]string, error) { sql := `SELECT networks.name from networks WHERE networks.project_id = ?` names, err := query.SelectStrings(ctx, c.tx, sql, projectID) if err != nil { return nil, fmt.Errorf("Unable to get URIs for network: %w", err) } uris := make([]string, len(names)) for i := range names { uris[i] = api.NewURL().Path(version.APIVersion, "networks", names[i]).Project(project).String() } return uris, nil } // GetNetworks returns the names of existing networks. func (c *ClusterTx) GetNetworks(ctx context.Context, project string) ([]string, error) { return c.networks(ctx, project, "") } // GetNetworksAllProjects returns the names of all networks across all projects. func (c *ClusterTx) GetNetworksAllProjects(ctx context.Context) (map[string][]string, error) { q := "SELECT projects.name, networks.name FROM networks JOIN projects ON networks.project_id=projects.id" var projectName string var networkName string outfmt := []any{projectName, networkName} result, err := queryScan(ctx, c, q, nil, outfmt) if err != nil { return nil, err } response := map[string][]string{} for _, r := range result { projectName, ok := r[0].(string) if !ok { continue } networkName, ok := r[1].(string) if !ok { continue } _, ok = response[projectName] if !ok { response[projectName] = []string{} } response[projectName] = append(response[projectName], networkName) } return response, nil } // Get all networks matching the given WHERE filter (if given). func (c *ClusterTx) networks(ctx context.Context, project string, where string, args ...any) ([]string, error) { q := "SELECT name FROM networks WHERE project_id = (SELECT id FROM projects WHERE name = ?)" inargs := []any{project} if where != "" { q += fmt.Sprintf(" AND %s", where) inargs = append(inargs, args...) } var name string outfmt := []any{name} result, err := queryScan(ctx, c, q, inargs, outfmt) if err != nil { return []string{}, err } response := []string{} for _, r := range result { response = append(response, r[0].(string)) } return response, nil } // NetworkState indicates the state of the network or network node. type NetworkState int // Network state. const ( networkPending NetworkState = iota // Network defined but not yet created globally or on specific node. networkCreated // Network created globally or on specific node. networkErrored // Deprecated (should no longer occur). ) // NetworkType indicates type of network. type NetworkType int // Network types. const ( NetworkTypeBridge NetworkType = iota // Network type bridge. NetworkTypeMacvlan // Network type macvlan. NetworkTypeSriov // Network type sriov. NetworkTypeOVN // Network type ovn. NetworkTypePhysical // Network type physical. ) // NetworkNode represents a network node. type NetworkNode struct { ID int64 Name string State NetworkState } // GetNetworkInAnyState returns the network with the given name. The network can be in any state. // Returns network ID, network info, and network cluster member info. func (c *ClusterTx) GetNetworkInAnyState(ctx context.Context, projectName string, networkName string) (int64, *api.Network, map[int64]NetworkNode, error) { return c.getNetworkByProjectAndName(ctx, projectName, networkName, -1) } // getNetworkByProjectAndName returns the network with the given project, name and state. // If stateFilter is -1, then a network can be in any state. // Returns network ID, network info, and network cluster member info. func (c *ClusterTx) getNetworkByProjectAndName(ctx context.Context, projectName string, networkName string, stateFilter NetworkState) (int64, *api.Network, map[int64]NetworkNode, error) { networkID, networkState, networkType, network, err := c.getPartialNetworkByProjectAndName(ctx, c, projectName, networkName, stateFilter) if err != nil { return -1, nil, nil, err } nodes, err := c.networkPopulatePeerInfo(ctx, c, networkID, network, networkState, networkType) if err != nil { return -1, nil, nil, err } return networkID, network, nodes, nil } // getPartialNetworkByProjectAndName gets the network with the given project, name and state. // If stateFilter is -1, then a network can be in any state. // Returns network ID, network state, network type, and partially populated network info. func (c *ClusterTx) getPartialNetworkByProjectAndName(ctx context.Context, tx *ClusterTx, projectName string, networkName string, stateFilter NetworkState) (int64, NetworkState, NetworkType, *api.Network, error) { var err error var networkID int64 = int64(-1) var network api.Network var networkState NetworkState var networkType NetworkType // Managed networks exist in the database. network.Managed = true network.Project = projectName var q strings.Builder q.WriteString(`SELECT n.id, n.name, IFNULL(n.description, "") as description, n.state, n.type FROM networks AS n WHERE n.project_id = (SELECT id FROM projects WHERE name = ? LIMIT 1) AND n.name=? `) args := []any{projectName, networkName} if stateFilter > -1 { q.WriteString(" AND n.state=?") args = append(args, networkCreated) } q.WriteString(" LIMIT 1") err = c.tx.QueryRowContext(ctx, q.String(), args...).Scan(&networkID, &network.Name, &network.Description, &networkState, &networkType) if err != nil { if errors.Is(err, sql.ErrNoRows) { return -1, -1, -1, nil, api.StatusErrorf(http.StatusNotFound, "Network not found") } return -1, -1, -1, nil, err } return networkID, networkState, networkType, &network, err } // networkPopulatePeerInfo takes a pointer to partially populated network info struct and enriches it. // Returns the network cluster member info. func (c *ClusterTx) networkPopulatePeerInfo(ctx context.Context, tx *ClusterTx, networkID int64, network *api.Network, networkState NetworkState, networkType NetworkType) (map[int64]NetworkNode, error) { var err error // Populate Status and Type fields by converting from DB values. network.Status = NetworkStateToAPIStatus(networkState) networkFillType(network, networkType) err = c.getNetworkConfig(ctx, tx, networkID, network) if err != nil { return nil, err } // Populate Location field. nodes, err := tx.NetworkNodes(ctx, networkID) if err != nil { return nil, err } network.Locations = make([]string, 0, len(nodes)) for _, node := range nodes { network.Locations = append(network.Locations, node.Name) } return nodes, nil } // NetworkStateToAPIStatus converts DB NetworkState to API status string. func NetworkStateToAPIStatus(state NetworkState) string { switch state { case networkPending: return api.NetworkStatusPending case networkCreated: return api.NetworkStatusCreated case networkErrored: return api.NetworkStatusErrored default: return api.NetworkStatusUnknown } } func networkFillType(network *api.Network, netType NetworkType) { switch netType { case NetworkTypeBridge: network.Type = "bridge" case NetworkTypeMacvlan: network.Type = "macvlan" case NetworkTypeSriov: network.Type = "sriov" case NetworkTypeOVN: network.Type = "ovn" case NetworkTypePhysical: network.Type = "physical" default: network.Type = "" // Unknown } } // getNetworkConfig populates the config map of the Network with the given ID. func (c *ClusterTx) getNetworkConfig(ctx context.Context, tx *ClusterTx, networkID int64, network *api.Network) error { q := ` SELECT key, value FROM networks_config WHERE network_id=? AND (node_id=? OR node_id IS NULL) ` network.Config = map[string]string{} return query.Scan(ctx, c.tx, q, func(scan func(dest ...any) error) error { var key, value string err := scan(&key, &value) if err != nil { return err } _, found := network.Config[key] if found { return fmt.Errorf("Duplicate config row found for key %q for network ID %d", key, networkID) } network.Config[key] = value return nil }, networkID, c.nodeID) } // CreateNetwork creates a new network. func (c *ClusterTx) CreateNetwork(ctx context.Context, projectName string, name string, description string, netType NetworkType, config map[string]string) (int64, error) { // Insert a new network record with state networkCreated. result, err := c.tx.ExecContext(ctx, "INSERT INTO networks (project_id, name, description, state, type) VALUES ((SELECT id FROM projects WHERE name = ?), ?, ?, ?, ?)", projectName, name, description, networkCreated, netType) if err != nil { return -1, err } id, err := result.LastInsertId() if err != nil { return -1, err } // Insert a node-specific entry pointing to ourselves with state networkPending. columns := []string{"network_id", "node_id", "state"} values := []any{id, c.nodeID, networkPending} _, err = query.UpsertObject(c.tx, "networks_nodes", columns, values) if err != nil { return -1, err } err = networkConfigAdd(c.tx, id, c.nodeID, config) if err != nil { return -1, err } return id, nil } // UpdateNetwork updates the network with the given name. func (c *ClusterTx) UpdateNetwork(ctx context.Context, project string, name, description string, config map[string]string) error { id, _, _, err := c.GetNetworkInAnyState(ctx, project, name) if err != nil { return err } err = c.UpdateNetworkDescription(id, description) if err != nil { return err } err = clearNetworkConfig(c.tx, id, c.nodeID) if err != nil { return err } err = networkConfigAdd(c.tx, id, c.nodeID, config) if err != nil { return err } return nil } // UpdateNetworkDescription updates the description of the network with the given ID. func (c *ClusterTx) UpdateNetworkDescription(id int64, description string) error { _, err := c.tx.Exec("UPDATE networks SET description=? WHERE id=?", description, id) return err } func networkConfigAdd(tx *sql.Tx, networkID, nodeID int64, config map[string]string) error { str := "INSERT INTO networks_config (network_id, node_id, key, value) VALUES(?, ?, ?, ?)" stmt, err := tx.Prepare(str) if err != nil { return err } defer func() { _ = stmt.Close() }() for k, v := range config { if v == "" { continue } var nodeIDValue any if !IsNodeSpecificNetworkConfig(k) { nodeIDValue = nil } else { nodeIDValue = nodeID } _, err = stmt.Exec(networkID, nodeIDValue, k, v) if err != nil { return err } } return nil } // Remove any the config of the network with the given ID // associated with the node with the given ID. func clearNetworkConfig(tx *sql.Tx, networkID, nodeID int64) error { _, err := tx.Exec( "DELETE FROM networks_config WHERE network_id=? AND (node_id=? OR node_id IS NULL)", networkID, nodeID) if err != nil { return err } return nil } // DeleteNetwork deletes the network with the given name. func (c *ClusterTx) DeleteNetwork(ctx context.Context, project string, name string) error { id, _, _, err := c.GetNetworkInAnyState(ctx, project, name) if err != nil { return err } _, err = c.tx.ExecContext(ctx, "DELETE FROM networks WHERE id=?", id) return err } // RenameNetwork renames a network. func (c *ClusterTx) RenameNetwork(ctx context.Context, project string, oldName string, newName string) error { id, _, _, err := c.GetNetworkInAnyState(ctx, project, oldName) if err != nil { return err } _, err = c.tx.ExecContext(ctx, "UPDATE networks SET name=? WHERE id=?", newName, id) return err } // IsNodeSpecificNetworkConfig returns true for a given network config key, if // the key is node-specific. Otherwise false is returned. func IsNodeSpecificNetworkConfig(key string) bool { if slices.Contains(nodeSpecificNetworkConfig, key) { return true } if nodeSpecificNetworkConfigRe.MatchString(key) { return true } return false } // StripNodeSpecificNetworkConfig returns a new network config map with all the // node-specific keys removed. The source map is left unchanged. func StripNodeSpecificNetworkConfig(config map[string]string) map[string]string { strippedConfig := make(map[string]string, len(config)) for key, value := range config { if IsNodeSpecificNetworkConfig(key) { continue } strippedConfig[key] = value } return strippedConfig } // nodeSpecificNetworkConfig lists all static network config keys which are node-specific. var nodeSpecificNetworkConfig = []string{ "bgp.ipv4.nexthop", "bgp.ipv6.nexthop", "bridge.external_interfaces", "parent", } // nodeSpecificNetworkConfigRe lists dynamic network config keys which are node-specific. var nodeSpecificNetworkConfigRe = regexp.MustCompile(`^tunnel\.[^.]+\.(interface|local)$`) incus-7.0.0/internal/server/db/networks_test.go000066400000000000000000000075571517523235500216360ustar00rootroot00000000000000//go:build linux && cgo && !agent package db_test import ( "context" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/shared/api" ) // The GetNetworksLocalConfigs method returns only node-specific config values. func TestGetNetworksLocalConfigs(t *testing.T) { cluster, cleanup := db.NewTestCluster(t) defer cleanup() err := cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, err := tx.CreateNetwork(ctx, api.ProjectDefaultName, "incusbr0", "", db.NetworkTypeBridge, map[string]string{ "dns.mode": "none", "bridge.external_interfaces": "vlan0", }) return err }) require.NoError(t, err) var config map[string]map[string]string err = cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error config, err = tx.GetNetworksLocalConfig(ctx) return err }) require.NoError(t, err) assert.Equal(t, config, map[string]map[string]string{ "incusbr0": {"bridge.external_interfaces": "vlan0"}, }) } func TestCreatePendingNetwork(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() _, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) _, err = tx.CreateNode("rusp", "5.6.7.8:666") require.NoError(t, err) config := map[string]string{"bridge.external_interfaces": "foo"} err = tx.CreatePendingNetwork(context.Background(), "buzz", api.ProjectDefaultName, "network1", "", db.NetworkTypeBridge, config) require.NoError(t, err) networkID, err := tx.GetNetworkID(context.Background(), api.ProjectDefaultName, "network1") require.NoError(t, err) assert.True(t, networkID > 0) config = map[string]string{"bridge.external_interfaces": "bar"} err = tx.CreatePendingNetwork(context.Background(), "rusp", api.ProjectDefaultName, "network1", "", db.NetworkTypeBridge, config) require.NoError(t, err) // The initial node (whose name is 'none' by default) is missing. _, err = tx.NetworkNodeConfigs(context.Background(), networkID) require.EqualError(t, err, "Network not defined on nodes: none") config = map[string]string{"bridge.external_interfaces": "egg,if1/eth0/1001"} err = tx.CreatePendingNetwork(context.Background(), "none", api.ProjectDefaultName, "network1", "", db.NetworkTypeBridge, config) require.NoError(t, err) // Now the storage is defined on all nodes. configs, err := tx.NetworkNodeConfigs(context.Background(), networkID) require.NoError(t, err) assert.Len(t, configs, 3) assert.Equal(t, map[string]string{"bridge.external_interfaces": "foo"}, configs["buzz"]) assert.Equal(t, map[string]string{"bridge.external_interfaces": "bar"}, configs["rusp"]) assert.Equal(t, map[string]string{"bridge.external_interfaces": "egg,if1/eth0/1001"}, configs["none"]) } // If an entry for the given network and node already exists, an error is // returned. func TestNetworksCreatePending_AlreadyDefined(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() _, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) err = tx.CreatePendingNetwork(context.Background(), "buzz", api.ProjectDefaultName, "network1", "", db.NetworkTypeBridge, map[string]string{}) require.NoError(t, err) err = tx.CreatePendingNetwork(context.Background(), "buzz", api.ProjectDefaultName, "network1", "", db.NetworkTypeBridge, map[string]string{}) require.Equal(t, db.ErrAlreadyDefined, err) } // If no node with the given name is found, an error is returned. func TestNetworksCreatePending_NonExistingNode(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() err := tx.CreatePendingNetwork(context.Background(), "buzz", api.ProjectDefaultName, "network1", "", db.NetworkTypeBridge, map[string]string{}) require.True(t, response.IsNotFoundError(err)) } incus-7.0.0/internal/server/db/node.go000066400000000000000000001032611517523235500176350ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "database/sql" "errors" "fmt" "net/http" "slices" "sort" "strconv" "strings" "time" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/query" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/osarch" ) // ClusterRole represents the role of a member in a cluster. type ClusterRole string // ClusterRoleDatabase represents the database role in a cluster. const ClusterRoleDatabase = ClusterRole("database") // ClusterRoleDatabaseStandBy represents the database stand-by role in a cluster. const ClusterRoleDatabaseStandBy = ClusterRole("database-standby") // ClusterRoleDatabaseLeader represents the database leader role in a cluster. const ClusterRoleDatabaseLeader = ClusterRole("database-leader") // ClusterRoleDatabaseClient represents a cluster member that prevents the affected member from being elected as a voter or stand-by in the cowsql cluster. const ClusterRoleDatabaseClient = ClusterRole("database-client") // ClusterRoleEventHub represents a cluster member who operates as an event hub. const ClusterRoleEventHub = ClusterRole("event-hub") // ClusterRoleOVNChassis represents a cluster member who operates as an OVN chassis. const ClusterRoleOVNChassis = ClusterRole("ovn-chassis") // ClusterRoles maps role ids into human-readable names. // // Note: the database role is currently stored directly in the raft // configuration which acts as single source of truth for it. This map should // only contain Incus-specific cluster roles. var ClusterRoles = map[int]ClusterRole{ 1: ClusterRoleEventHub, 2: ClusterRoleOVNChassis, 3: ClusterRoleDatabaseClient, } // Numeric type codes identifying different cluster member states. const ( ClusterMemberStateCreated = 0 ClusterMemberStatePending = 1 ClusterMemberStateEvacuated = 2 ClusterMemberStateEvacuating = 3 ClusterMemberStateRestoring = 4 ) // NodeInfo holds information about a single member in a cluster. type NodeInfo struct { ID int64 // Stable node identifier Name string // User-assigned name of the node Address string // Network address of the node Description string // Node description (optional) Schema int // Schema version of the daemon running the member APIExtensions int // Number of API extensions of the daemon running the member Heartbeat time.Time // Timestamp of the last heartbeat Roles []ClusterRole // List of cluster roles Architecture int // Node architecture State int // Node state Config map[string]string // Configuration for the node Groups []string // Cluster groups } // IsOffline returns true if the last successful heartbeat time of the node is // older than the given threshold. func (n NodeInfo) IsOffline(threshold time.Duration) bool { return nodeIsOffline(threshold, n.Heartbeat) } // NodeInfoArgs provides information about the cluster environment for use with NodeInfo.ToAPI(). type NodeInfoArgs struct { LeaderAddress string FailureDomains map[uint64]string MemberFailureDomains map[string]uint64 OfflineThreshold time.Duration MaxMemberVersion [2]int RaftNodes []RaftNode } // ToAPI returns an API entry. func (n NodeInfo) ToAPI(ctx context.Context, tx *ClusterTx, args NodeInfoArgs) (*api.ClusterMember, error) { var err error var failureDomain string domainID := args.MemberFailureDomains[n.Address] failureDomain = args.FailureDomains[domainID] // From local database. var raftNode *RaftNode for _, node := range args.RaftNodes { if node.Address == n.Address { raftNode = &node break } } // Fill in the struct. result := api.ClusterMember{} result.Description = n.Description result.ServerName = n.Name result.URL = fmt.Sprintf("https://%s", n.Address) result.Database = false result.Config = n.Config result.Roles = make([]string, 0, len(n.Roles)) for _, r := range n.Roles { result.Roles = append(result.Roles, string(r)) } result.Groups = n.Groups // Check if member is the leader. if args.LeaderAddress == n.Address { result.Roles = append(result.Roles, string(ClusterRoleDatabaseLeader)) result.Database = true } if raftNode != nil && raftNode.Role == RaftVoter { result.Roles = append(result.Roles, string(ClusterRoleDatabase)) result.Database = true } if raftNode != nil && raftNode.Role == RaftStandBy { result.Roles = append(result.Roles, string(ClusterRoleDatabaseStandBy)) result.Database = true } result.Architecture, err = osarch.ArchitectureName(n.Architecture) if err != nil { return nil, err } result.FailureDomain = failureDomain // Set state and message. result.Status = "Online" result.Message = "Fully operational" if slices.Contains([]int{ClusterMemberStateEvacuated, ClusterMemberStateEvacuating, ClusterMemberStateRestoring}, n.State) { result.Message = "Unavailable due to maintenance" switch n.State { case ClusterMemberStateEvacuated: result.Status = "Evacuated" case ClusterMemberStateEvacuating: result.Status = "Evacuating" case ClusterMemberStateRestoring: result.Status = "Restoring" default: } } else if n.IsOffline(args.OfflineThreshold) { result.Status = "Offline" result.Message = fmt.Sprintf("No heartbeat for %s (%s)", time.Since(n.Heartbeat), n.Heartbeat) } else { // Check for max DB schema and API extensions. maxVersion, err := tx.GetNodeMaxVersion(ctx) if err != nil { return nil, err } // Check if up to date. ret, err := localUtil.CompareVersions(maxVersion, n.Version(), true) if err != nil { return nil, err } if ret == 1 { result.Status = "Blocked" result.Message = "Needs updating to newer version" } } return &result, nil } // Version returns the node's version, composed by its schema level and // number of extensions. func (n NodeInfo) Version() [2]int { return [2]int{n.Schema, n.APIExtensions} } // GetNodeByAddress returns the node with the given network address. func (c *ClusterTx) GetNodeByAddress(ctx context.Context, address string) (NodeInfo, error) { null := NodeInfo{} nodes, err := c.nodes(ctx, false /* not pending */, "address=?", address) if err != nil { return null, err } switch len(nodes) { case 0: return null, api.StatusErrorf(http.StatusNotFound, "Cluster member not found") case 1: return nodes[0], nil default: return null, errors.New("more than one node matches") } } // GetNodeMaxVersion returns the highest schema and API versions possible on the cluster. func (c *ClusterTx) GetNodeMaxVersion(ctx context.Context) ([2]int, error) { version := [2]int{} // Get the maximum DB schema. var maxSchema int row := c.tx.QueryRowContext(ctx, "SELECT MAX(schema) FROM nodes") err := row.Scan(&maxSchema) if err != nil { return version, err } // Get the maximum API extension. var maxAPI int row = c.tx.QueryRowContext(ctx, "SELECT MAX(api_extensions) FROM nodes") err = row.Scan(&maxAPI) if err != nil { return version, err } // Compute the combined version. version = [2]int{maxSchema, maxAPI} return version, nil } // GetNodeWithID returns the node with the given ID. func (c *ClusterTx) GetNodeWithID(ctx context.Context, nodeID int) (NodeInfo, error) { null := NodeInfo{} nodes, err := c.nodes(ctx, false /* not pending */, "id=?", nodeID) if err != nil { return null, err } switch len(nodes) { case 0: return null, api.StatusErrorf(http.StatusNotFound, "Cluster member not found") case 1: return nodes[0], nil default: return null, errors.New("More than one cluster member matches") } } // GetPendingNodeByAddress returns the pending node with the given network address. func (c *ClusterTx) GetPendingNodeByAddress(ctx context.Context, address string) (NodeInfo, error) { null := NodeInfo{} nodes, err := c.nodes(ctx, true /*pending */, "address=?", address) if err != nil { return null, err } switch len(nodes) { case 0: return null, api.StatusErrorf(http.StatusNotFound, "Cluster member not found") case 1: return nodes[0], nil default: return null, errors.New("More than one cluster member matches") } } // GetPendingNodeByName returns the pending node with the given name. func (c *ClusterTx) GetPendingNodeByName(ctx context.Context, name string) (NodeInfo, error) { null := NodeInfo{} nodes, err := c.nodes(ctx, true /* pending */, "name=?", name) if err != nil { return null, err } switch len(nodes) { case 0: return null, api.StatusErrorf(http.StatusNotFound, "Cluster member not found") case 1: return nodes[0], nil default: return null, errors.New("More than one cluster member matches") } } // GetNodeByName returns the node with the given name. func (c *ClusterTx) GetNodeByName(ctx context.Context, name string) (NodeInfo, error) { null := NodeInfo{} nodes, err := c.nodes(ctx, false /* not pending */, "name=?", name) if err != nil { return null, err } switch len(nodes) { case 0: return null, api.StatusErrorf(http.StatusNotFound, "Cluster member not found") case 1: return nodes[0], nil default: return null, errors.New("More than one cluster member matches") } } // GetLocalNodeName returns the name of the node this method is invoked on. // Usually you should not use this function directly but instead use the cached State.ServerName value. func (c *ClusterTx) GetLocalNodeName(ctx context.Context) (string, error) { stmt := "SELECT name FROM nodes WHERE id=?" names, err := query.SelectStrings(ctx, c.tx, stmt, c.nodeID) if err != nil { return "", err } switch len(names) { case 0: return "", nil case 1: return names[0], nil default: return "", errors.New("inconsistency: non-unique node ID") } } // GetLocalNodeAddress returns the address of the node this method is invoked on. func (c *ClusterTx) GetLocalNodeAddress(ctx context.Context) (string, error) { stmt := "SELECT address FROM nodes WHERE id=?" addresses, err := query.SelectStrings(ctx, c.tx, stmt, c.nodeID) if err != nil { return "", err } switch len(addresses) { case 0: return "", nil case 1: return addresses[0], nil default: return "", errors.New("inconsistency: non-unique node ID") } } // NodeIsOutdated returns true if there's some cluster node having an API or // schema version greater than the node this method is invoked on. func (c *ClusterTx) NodeIsOutdated(ctx context.Context) (bool, error) { nodes, err := c.nodes(ctx, false /* not pending */, "") if err != nil { return false, fmt.Errorf("Failed to fetch nodes: %w", err) } // Figure our own version. version := [2]int{} for _, node := range nodes { if node.ID == c.nodeID { version = node.Version() } } if version[0] == 0 || version[1] == 0 { return false, errors.New("Inconsistency: local member not found") } // Check if any of the other nodes is greater than us. for _, node := range nodes { if node.ID == c.nodeID { continue } n, err := localUtil.CompareVersions(node.Version(), version, true) if err != nil { return false, fmt.Errorf("Failed to compare with version of member %s: %w", node.Name, err) } if n == 1 { // The other node's version is greater than ours. return true, nil } } return false, nil } // GetNodes returns all cluster members that are part of the cluster. // // If this server is not clustered, a list with a single member whose address is 0.0.0.0 is returned. func (c *ClusterTx) GetNodes(ctx context.Context) ([]NodeInfo, error) { return c.nodes(ctx, false /* not pending */, "") } // GetNodesCount returns the number of members in the cluster. // // Since there's always at least one node row, even when not-clustered, the // return value is greater than zero. func (c *ClusterTx) GetNodesCount(ctx context.Context) (int, error) { count, err := query.Count(ctx, c.tx, "nodes", "") if err != nil { return 0, fmt.Errorf("failed to count existing nodes: %w", err) } return count, nil } // RenameNode changes the name of an existing node. // // Return an error if a node with the same name already exists. func (c *ClusterTx) RenameNode(ctx context.Context, old string, new string) error { count, err := query.Count(ctx, c.tx, "nodes", "name=?", new) if err != nil { return fmt.Errorf("failed to check existing nodes: %w", err) } if count != 0 { return ErrAlreadyDefined } stmt := `UPDATE nodes SET name=? WHERE name=?` result, err := c.tx.Exec(stmt, new, old) if err != nil { return fmt.Errorf("failed to update node name: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("failed to get rows count: %w", err) } if n != 1 { return fmt.Errorf("expected to update one row, not %d", n) } return nil } // SetDescription changes the description of the given node. func (c *ClusterTx) SetDescription(id int64, description string) error { stmt := `UPDATE nodes SET description=? WHERE id=?` result, err := c.tx.Exec(stmt, description, id) if err != nil { return fmt.Errorf("Failed to update node name: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Failed to get rows count: %w", err) } if n != 1 { return fmt.Errorf("Expected to update one row, not %d", n) } return nil } // Nodes returns all members part of the cluster. func (c *ClusterTx) nodes(ctx context.Context, pending bool, where string, args ...any) ([]NodeInfo, error) { // Get node roles sql := "SELECT node_id, role FROM nodes_roles" nodeRoles := map[int64][]ClusterRole{} err := query.Scan(ctx, c.Tx(), sql, func(scan func(dest ...any) error) error { var nodeID int64 var role int err := scan(&nodeID, &role) if err != nil { return err } if nodeRoles[nodeID] == nil { nodeRoles[nodeID] = []ClusterRole{} } roleName := string(ClusterRoles[role]) nodeRoles[nodeID] = append(nodeRoles[nodeID], ClusterRole(roleName)) return nil }) if err != nil && err.Error() != "no such table: nodes_roles" { // Don't fail on a missing table, we need to handle updates return nil, err } // Get node groups sql = `SELECT node_id, cluster_groups.name FROM nodes_cluster_groups JOIN cluster_groups ON cluster_groups.id = nodes_cluster_groups.group_id` nodeGroups := map[int64][]string{} err = query.Scan(ctx, c.Tx(), sql, func(scan func(dest ...any) error) error { var nodeID int64 var group string err := scan(&nodeID, &group) if err != nil { return err } if nodeGroups[nodeID] == nil { nodeGroups[nodeID] = []string{} } nodeGroups[nodeID] = append(nodeGroups[nodeID], group) return nil }) if err != nil && err.Error() != "no such table: nodes_cluster_groups" { // Don't fail on a missing table, we need to handle updates return nil, err } // Get the node entries sql = "SELECT id, name, address, description, schema, api_extensions, heartbeat, arch, state FROM nodes " if pending { // Include only pending nodes sql += "WHERE state=? " } else { // Include created and evacuated nodes sql += "WHERE state!=? " } args = append([]any{ClusterMemberStatePending}, args...) if where != "" { sql += fmt.Sprintf("AND %s ", where) } sql += "ORDER BY id" // Process node entries nodes := []NodeInfo{} err = query.Scan(ctx, c.tx, sql, func(scan func(dest ...any) error) error { node := NodeInfo{} err := scan(&node.ID, &node.Name, &node.Address, &node.Description, &node.Schema, &node.APIExtensions, &node.Heartbeat, &node.Architecture, &node.State) if err != nil { return err } nodes = append(nodes, node) return nil }, args...) if err != nil { return nil, fmt.Errorf("Failed to fetch nodes: %w", err) } // Add the roles for i, node := range nodes { roles, ok := nodeRoles[node.ID] if ok { nodes[i].Roles = roles } } // Add the groups for i, node := range nodes { groups, ok := nodeGroups[node.ID] if ok { nodes[i].Groups = groups } } config, err := cluster.GetConfig(context.TODO(), c.Tx(), "nodes", "node") if err != nil { return nil, fmt.Errorf("Failed to fetch nodes config: %w", err) } for i := range nodes { data, ok := config[int(nodes[i].ID)] if !ok { nodes[i].Config = map[string]string{} } else { nodes[i].Config = data } } return nodes, nil } // CreateNode adds a node to the current list of members that are part of the // cluster. The node's architecture will be the architecture of the machine the // method is being run on. It returns the ID of the newly inserted row. func (c *ClusterTx) CreateNode(name string, address string) (int64, error) { arch, err := osarch.ArchitectureGetLocalID() if err != nil { return -1, err } return c.CreateNodeWithArch(name, address, arch) } // CreateNodeWithArch is the same as NodeAdd, but lets setting the node // architecture explicitly. func (c *ClusterTx) CreateNodeWithArch(name string, address string, arch int) (int64, error) { columns := []string{"name", "address", "schema", "api_extensions", "arch", "description"} values := []any{name, address, cluster.SchemaVersion, version.APIExtensionsCount(), arch, ""} return query.UpsertObject(c.tx, "nodes", columns, values) } // SetNodePendingFlag toggles the pending flag for the node. A node is pending when // it's been accepted in the cluster, but has not yet actually joined it. func (c *ClusterTx) SetNodePendingFlag(id int64, pending bool) error { value := 0 if pending { value = 1 } result, err := c.tx.Exec("UPDATE nodes SET state=? WHERE id=?", value, id) if err != nil { return err } n, err := result.RowsAffected() if err != nil { return err } if n != 1 { return fmt.Errorf("query updated %d rows instead of 1", n) } return nil } // BootstrapNode sets the name and address of the first cluster member, with id: 1. func (c *ClusterTx) BootstrapNode(name string, address string) error { result, err := c.tx.Exec("UPDATE nodes SET name=?, address=? WHERE id=1", name, address) if err != nil { return err } n, err := result.RowsAffected() if err != nil { return err } if n != 1 { return fmt.Errorf("query updated %d rows instead of 1", n) } return nil } // UpdateNodeConfig updates the replaces the node's config with the specified config. func (c *ClusterTx) UpdateNodeConfig(ctx context.Context, id int64, config map[string]string) error { err := cluster.UpdateConfig(ctx, c.Tx(), "nodes", "node", int(id), config) if err != nil { return fmt.Errorf("Unable to update node config: %w", err) } return nil } // UpdateNodeRoles changes the list of roles on a member. func (c *ClusterTx) UpdateNodeRoles(id int64, roles []ClusterRole) error { getRoleID := func(role ClusterRole) (int, error) { for k, v := range ClusterRoles { if v == role { return k, nil } } return -1, fmt.Errorf("Invalid cluster role %q", role) } // Translate role names to ids roleIDs := []int{} for _, role := range roles { // Skip internal-only roles. if role == ClusterRoleDatabase || role == ClusterRoleDatabaseStandBy || role == ClusterRoleDatabaseLeader { continue } roleID, err := getRoleID(role) if err != nil { return err } roleIDs = append(roleIDs, roleID) } // Update the database record _, err := c.tx.Exec("DELETE FROM nodes_roles WHERE node_id=?", id) if err != nil { return err } for _, roleID := range roleIDs { _, err := c.tx.Exec("INSERT INTO nodes_roles (node_id, role) VALUES (?, ?)", id, roleID) if err != nil { return err } } return nil } // UpdateNodeClusterGroups changes the list of cluster groups the member belongs to. func (c *ClusterTx) UpdateNodeClusterGroups(ctx context.Context, id int64, groups []string) error { nodeInfo, err := c.GetNodeWithID(ctx, int(id)) if err != nil { return err } oldGroups, err := c.GetClusterGroupsWithNode(ctx, nodeInfo.Name) if err != nil { return err } skipGroups := []string{} // Check if node already belongs to the given groups. for _, newGroup := range groups { if slices.Contains(oldGroups, newGroup) { // Node already belongs to this group. skipGroups = append(skipGroups, newGroup) continue } // Add node to new group. err = c.AddNodeToClusterGroup(ctx, newGroup, nodeInfo.Name) if err != nil { return fmt.Errorf("Failed to add member to cluster group: %w", err) } } for _, oldGroup := range oldGroups { if slices.Contains(skipGroups, oldGroup) { continue } // Get the cluster group. clusterGroup, err := cluster.GetClusterGroup(ctx, c.Tx(), oldGroup) if err != nil { return err } memberInstances, err := c.GetClusterGroupMemberInstances(ctx, clusterGroup, nodeInfo.Name) if err != nil { return err } if len(memberInstances) > 0 { return fmt.Errorf("Cluster group member is currently in use") } // Remove node from group. err = c.RemoveNodeFromClusterGroup(ctx, oldGroup, nodeInfo.Name) if err != nil { return fmt.Errorf("Failed to remove member from cluster group: %w", err) } } return nil } // UpdateNodeFailureDomain changes the failure domain of a node. func (c *ClusterTx) UpdateNodeFailureDomain(ctx context.Context, id int64, domain string) error { var domainID any if domain == "" { return errors.New("Failure domain name can't be empty") } if domain == "default" { domainID = nil } else { row := c.tx.QueryRowContext(ctx, "SELECT id FROM nodes_failure_domains WHERE name=?", domain) err := row.Scan(&domainID) if err != nil { if !errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("Load failure domain name: %w", err) } result, err := c.tx.Exec("INSERT INTO nodes_failure_domains (name) VALUES (?)", domain) if err != nil { return fmt.Errorf("Create new failure domain: %w", err) } domainID, err = result.LastInsertId() if err != nil { return fmt.Errorf("Get last inserted ID: %w", err) } } } result, err := c.tx.Exec("UPDATE nodes SET failure_domain_id=? WHERE id=?", domainID, id) if err != nil { return err } n, err := result.RowsAffected() if err != nil { return err } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } return nil } // UpdateNodeStatus changes the state of a node. func (c *ClusterTx) UpdateNodeStatus(id int64, state int) error { result, err := c.tx.Exec("UPDATE nodes SET state=? WHERE id=?", state, id) if err != nil { return err } n, err := result.RowsAffected() if err != nil { return err } if n != 1 { return fmt.Errorf("Query updated %d rows instead of 1", n) } return nil } // GetNodeFailureDomain returns the failure domain associated with the node with the given ID. func (c *ClusterTx) GetNodeFailureDomain(ctx context.Context, id int64) (string, error) { stmt := ` SELECT coalesce(nodes_failure_domains.name,'default') FROM nodes LEFT JOIN nodes_failure_domains ON nodes.failure_domain_id = nodes_failure_domains.id WHERE nodes.id=? ` var domain string err := c.tx.QueryRowContext(ctx, stmt, id).Scan(&domain) if err != nil { return "", err } return domain, nil } // GetNodesFailureDomains returns a map associating each node address with its // failure domain code. func (c *ClusterTx) GetNodesFailureDomains(ctx context.Context) (map[string]uint64, error) { sql := "SELECT address, coalesce(failure_domain_id, 0) FROM nodes" type failureDomain struct { Address string FailureDomainID int64 } rows := []failureDomain{} err := query.Scan(ctx, c.tx, sql, func(scan func(dest ...any) error) error { fd := failureDomain{} err := scan(&fd.Address, &fd.FailureDomainID) if err != nil { return err } rows = append(rows, fd) return nil }) if err != nil { return nil, err } domains := map[string]uint64{} for _, row := range rows { domains[row.Address] = uint64(row.FailureDomainID) } return domains, nil } // GetFailureDomainsNames return a map associating failure domain IDs to their // names. func (c *ClusterTx) GetFailureDomainsNames(ctx context.Context) (map[uint64]string, error) { sql := "SELECT id, name FROM nodes_failure_domains" type failureDomain struct { ID int64 Name string } rows := []failureDomain{} err := query.Scan(ctx, c.tx, sql, func(scan func(dest ...any) error) error { fd := failureDomain{} err := scan(&fd.ID, &fd.Name) if err != nil { return err } rows = append(rows, fd) return nil }) if err != nil { return nil, err } domains := map[uint64]string{ 0: "default", // Default failure domain, when not set } for _, row := range rows { domains[uint64(row.ID)] = row.Name } return domains, nil } // RemoveNode removes the node with the given id. func (c *ClusterTx) RemoveNode(id int64) error { result, err := c.tx.Exec("DELETE FROM nodes WHERE id=?", id) if err != nil { return err } n, err := result.RowsAffected() if err != nil { return err } if n != 1 { return fmt.Errorf("query deleted %d rows instead of 1", n) } return nil } // SetNodeHeartbeat updates the heartbeat column of the node with the given address. func (c *ClusterTx) SetNodeHeartbeat(address string, heartbeat time.Time) error { stmt := "UPDATE nodes SET heartbeat=? WHERE address=?" result, err := c.tx.Exec(stmt, heartbeat, address) if err != nil { return err } n, err := result.RowsAffected() if err != nil { return err } if n < 1 { return api.StatusErrorf(http.StatusNotFound, "Cluster member not found") } else if n > 1 { return fmt.Errorf("Expected to update one row and not %d", n) } return nil } // NodeIsEmpty returns an empty string if the node with the given ID has no // instances or images associated with it. Otherwise, it returns a message // say what's left. func (c *ClusterTx) NodeIsEmpty(ctx context.Context, id int64) (string, error) { // Check if the node has any instances. instances, err := query.SelectStrings(ctx, c.tx, "SELECT name FROM instances WHERE node_id=?", id) if err != nil { return "", fmt.Errorf("Failed to get instances for node %d: %w", id, err) } if len(instances) > 0 { message := fmt.Sprintf( "Node still has the following instances: %s", strings.Join(instances, ", ")) return message, nil } // Check if the node has any images available only in it. type image struct { fingerprint string nodeID int64 } images := []image{} sql := `SELECT fingerprint, node_id FROM images JOIN images_nodes ON images.id=images_nodes.image_id` err = query.Scan(ctx, c.tx, sql, func(scan func(dest ...any) error) error { img := image{} err := scan(&img.fingerprint, &img.nodeID) if err != nil { return err } images = append(images, img) return nil }) if err != nil { return "", fmt.Errorf("Failed to get image list for node %d: %w", id, err) } index := map[string][]int64{} // Map fingerprints to IDs of nodes for _, image := range images { index[image.fingerprint] = append(index[image.fingerprint], image.nodeID) } fingerprints := []string{} for fingerprint, ids := range index { if len(ids) > 1 { continue } if ids[0] == id { fingerprints = append(fingerprints, fingerprint) } } if len(fingerprints) > 0 { message := fmt.Sprintf( "Node still has the following images: %s", strings.Join(fingerprints, ", ")) return message, nil } // Check if the node has any custom volumes. drivers := make([]string, len(StorageRemoteDriverNames())) for i, entry := range StorageRemoteDriverNames() { drivers[i] = fmt.Sprintf("'%s'", entry) } sql = ` SELECT storage_volumes.name FROM storage_volumes JOIN storage_pools ON storage_volumes.storage_pool_id=storage_pools.id WHERE storage_volumes.node_id=? AND storage_volumes.type=? AND storage_pools.driver NOT IN (%s) ` volumes, err := query.SelectStrings(ctx, c.tx, fmt.Sprintf(sql, strings.Join(drivers, ", ")), id, StoragePoolVolumeTypeCustom) if err != nil { return "", fmt.Errorf("Failed to get custom volumes for node %d: %w", id, err) } if len(volumes) > 0 { message := fmt.Sprintf( "Node still has the following custom volumes: %s", strings.Join(volumes, ", ")) return message, nil } return "", nil } // ClearNode removes any instance or image associated with this node. func (c *ClusterTx) ClearNode(ctx context.Context, id int64) error { _, err := c.tx.Exec("DELETE FROM instances WHERE node_id=?", id) if err != nil { return err } // Get the IDs of the images this node is hosting. ids, err := query.SelectIntegers(ctx, c.tx, "SELECT image_id FROM images_nodes WHERE node_id=?", id) if err != nil { return err } // Delete the association _, err = c.tx.Exec("DELETE FROM images_nodes WHERE node_id=?", id) if err != nil { return err } // Delete the image as well if this was the only node with it. for _, id := range ids { count, err := query.Count(ctx, c.tx, "images_nodes", "image_id=?", id) if err != nil { return err } if count > 0 { continue } _, err = c.tx.Exec("DELETE FROM images WHERE id=?", id) if err != nil { return err } } return nil } // GetNodeOfflineThreshold returns the amount of time that needs to elapse after // which a series of unsuccessful heartbeat will make the node be considered // offline. func (c *ClusterTx) GetNodeOfflineThreshold(ctx context.Context) (time.Duration, error) { threshold := time.Duration(DefaultOfflineThreshold) * time.Second values, err := query.SelectStrings(ctx, c.tx, "SELECT value FROM config WHERE key='cluster.offline_threshold'") if err != nil { return -1, err } if len(values) > 0 { seconds, err := strconv.Atoi(values[0]) if err != nil { return -1, err } threshold = time.Duration(seconds) * time.Second } return threshold, nil } // GetCandidateMembers returns cluster members that are online, in created state and don't need manual targeting. // It excludes members that do not support any of the targetArchitectures (if non-nil) or not in targetClusterGroup // (if non-empty). It also takes into account any restrictions on allowedClusterGroups (if non-nil). func (c *ClusterTx) GetCandidateMembers(ctx context.Context, allMembers []NodeInfo, targetArchitectures []int, targetClusterGroup string, allowedClusterGroups []string, offlineThreshold time.Duration) ([]NodeInfo, error) { var candidateMembers []NodeInfo for _, member := range allMembers { // Skip pending, evacuated or offline members. if member.State != ClusterMemberStateCreated || member.IsOffline(offlineThreshold) { continue } // Skip manually targeted members. if member.Config["scheduler.instance"] == "manual" { continue } // Skip group-only members if targeted cluster group doesn't match. if member.Config["scheduler.instance"] == "group" && !slices.Contains(member.Groups, targetClusterGroup) { continue } // Skip if a group is requested and member isn't part of it. if targetClusterGroup != "" && !slices.Contains(member.Groups, targetClusterGroup) { continue } // Skip if working with a restricted set of cluster groups and member isn't part of any. if allowedClusterGroups != nil { // Load the list of cluster groups. groupNames := []string{} clusterGroups, err := cluster.GetClusterGroups(ctx, c.Tx()) if err != nil { return nil, err } for _, group := range clusterGroups { groupNames = append(groupNames, group.Name) } // Filter based on groups. found := false for _, allowedClusterGroup := range allowedClusterGroups { if !slices.Contains(groupNames, allowedClusterGroup) { return nil, fmt.Errorf("Cluster group %q doesn't exist", allowedClusterGroup) } if slices.Contains(member.Groups, allowedClusterGroup) { found = true break } } if !found { continue } } // Consider target architectures if specified. if targetArchitectures != nil { // Get member personalities too. personalities, err := osarch.ArchitecturePersonalities(member.Architecture) if err != nil { return nil, err } supportedArchitectures := append([]int{member.Architecture}, personalities...) for _, supportedArchitecture := range supportedArchitectures { if slices.Contains(targetArchitectures, supportedArchitecture) { candidateMembers = append(candidateMembers, member) break } } } else { // Otherwise consider member a candidate irrespective of architecture. candidateMembers = append(candidateMembers, member) } } sort.Slice(candidateMembers, func(i int, j int) bool { iCount, _ := c.GetInstancesCount(ctx, "", candidateMembers[i].Name, true) jCount, _ := c.GetInstancesCount(ctx, "", candidateMembers[j].Name, true) return iCount < jCount }) return candidateMembers, nil } // SetNodeVersion updates the schema and API version of the node with the // given id. This is used only in tests. func (c *ClusterTx) SetNodeVersion(id int64, version [2]int) error { stmt := "UPDATE nodes SET schema=?, api_extensions=? WHERE id=?" result, err := c.tx.Exec(stmt, version[0], version[1], id) if err != nil { return fmt.Errorf("Failed to update nodes table: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Failed to get affected rows: %w", err) } if n != 1 { return errors.New("Expected exactly one row to be updated") } return nil } func nodeIsOffline(threshold time.Duration, heartbeat time.Time) bool { offlineTime := time.Now().UTC().Add(-threshold) return heartbeat.Before(offlineTime) || heartbeat.Equal(offlineTime) } // LocalNodeIsEvacuated returns whether the local member is in the evacuated state. func (c *Cluster) LocalNodeIsEvacuated() bool { isEvacuated := false err := c.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error { name, err := tx.GetLocalNodeName(ctx) if err != nil { return err } node, err := tx.GetNodeByName(ctx, name) if err != nil { return nil } isEvacuated = slices.Contains([]int{ClusterMemberStateEvacuated, ClusterMemberStateEvacuating, ClusterMemberStateRestoring}, node.State) return nil }) if err != nil { return false } return isEvacuated } // DefaultOfflineThreshold is the default value for the // cluster.offline_threshold configuration key, expressed in seconds. const DefaultOfflineThreshold = 20 incus-7.0.0/internal/server/db/node/000077500000000000000000000000001517523235500173035ustar00rootroot00000000000000incus-7.0.0/internal/server/db/node/open.go000066400000000000000000000025161517523235500205770ustar00rootroot00000000000000package node import ( "context" "database/sql" "fmt" "path/filepath" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" ) // Open the node-local database object. func Open(dir string) (*sql.DB, error) { path := filepath.Join(dir, "local.db") db, err := sqliteOpen(path) if err != nil { return nil, fmt.Errorf("cannot open node database: %w", err) } return db, nil } // EnsureSchema applies all relevant schema updates to the node-local // database. // // Return the initial schema version found before starting the update, along // with any error occurred. func EnsureSchema(db *sql.DB, dir string) (int, error) { backupDone := false schema := Schema() schema.File(filepath.Join(dir, "patch.local.sql")) // Optional custom queries schema.Hook(func(ctx context.Context, version int, tx *sql.Tx) error { if !backupDone { logger.Infof("Updating the database schema. Backup made as \"local.db.bak\"") path := filepath.Join(dir, "local.db") err := internalUtil.FileCopy(path, path+".bak") if err != nil { return err } backupDone = true } if version == -1 { logger.Debugf("Running pre-update queries from file for local DB schema") } else { logger.Debugf("Updating DB schema from %d to %d", version, version+1) } return nil }) return schema.Ensure(db) } incus-7.0.0/internal/server/db/node/open_test.go000066400000000000000000000017501517523235500216350ustar00rootroot00000000000000package node_test import ( "os" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db/node" ) func TestOpen(t *testing.T) { dir, cleanup := newDir(t) defer cleanup() db, err := node.Open(dir) defer func() { _ = db.Close() }() require.NoError(t, err) } // When the node-local database is created from scratch, the value for the // initial patch is 0. func TestEnsureSchema(t *testing.T) { dir, cleanup := newDir(t) defer cleanup() db, err := node.Open(dir) require.NoError(t, err) defer func() { _ = db.Close() }() initial, err := node.EnsureSchema(db, dir) require.NoError(t, err) assert.Equal(t, 0, initial) } // Create a new temporary directory, along with a function to clean it up. func newDir(t *testing.T) (string, func()) { dir, err := os.MkdirTemp("", "incus-db-node-test-") require.NoError(t, err) cleanup := func() { require.NoError(t, os.RemoveAll(dir)) } return dir, cleanup } incus-7.0.0/internal/server/db/node/schema.go000066400000000000000000000020001517523235500210620ustar00rootroot00000000000000package node // DO NOT EDIT BY HAND // // This code was generated by the schema.DotGo function. If you need to // modify the database schema, please add a new schema update to update.go // and the run 'make update-schema'. const freshSchema = ` CREATE TABLE certificates ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, fingerprint TEXT NOT NULL, type INTEGER NOT NULL, name TEXT NOT NULL, certificate TEXT NOT NULL, UNIQUE (fingerprint) ); CREATE TABLE "config" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (key) ); CREATE TABLE patches ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, applied_at DATETIME NOT NULL, UNIQUE (name) ); CREATE TABLE raft_nodes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, address TEXT NOT NULL, role INTEGER NOT NULL DEFAULT 0, name TEXT NOT NULL default "", UNIQUE (address) ); INSERT INTO schema (version, updated_at) VALUES (43, strftime("%s")) ` incus-7.0.0/internal/server/db/node/sqlite.go000066400000000000000000000015241517523235500211350ustar00rootroot00000000000000package node import ( "database/sql" "fmt" "github.com/mattn/go-sqlite3" ) func init() { sql.Register("sqlite3_with_fk", &sqlite3.SQLiteDriver{ConnectHook: sqliteEnableForeignKeys}) } // Opens the node-level database with the correct parameters. func sqliteOpen(path string) (*sql.DB, error) { timeout := 5 // TODO - make this command-line configurable? // These are used to tune the transaction BEGIN behavior instead of using the // similar "locking_mode" pragma (locking for the whole database connection). openPath := fmt.Sprintf("%s?_busy_timeout=%d&_txlock=exclusive", path, timeout*1000) // Open the database. If the file doesn't exist it is created. return sql.Open("sqlite3_with_fk", openPath) } func sqliteEnableForeignKeys(conn *sqlite3.SQLiteConn) error { _, err := conn.Exec("PRAGMA foreign_keys=ON;", nil) return err } incus-7.0.0/internal/server/db/node/update.go000066400000000000000000000776041517523235500211320ustar00rootroot00000000000000package node import ( "context" "database/sql" "encoding/hex" "fmt" "os" "strconv" "strings" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/internal/server/db/schema" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // Schema for the local database. func Schema() *schema.Schema { schema := schema.NewFromMap(updates) schema.Fresh(freshSchema) return schema } // FreshSchema returns the fresh schema definition of the local database. func FreshSchema() string { return freshSchema } // SchemaDotGo refreshes the schema.go file in this package, using the updates // defined here. func SchemaDotGo() error { return schema.DotGo(updates, "schema") } /* Database updates are one-time actions that are needed to move an existing database from one version of the schema to the next. Those updates are applied at startup time before anything else is initialized. This means that they should be entirely self-contained and not touch anything but the database. Calling API functions isn't allowed as such functions may themselves depend on a newer DB schema and so would fail when upgrading a very old version. DO NOT USE this mechanism for one-time actions which do not involve changes to the database schema. Use patches instead (see patches.go). REMEMBER to run "make update-schema" after you add a new update function to this slice. That will refresh the schema declaration in db/schema.go and include the effect of applying your patch as well. Only append to the updates list, never remove entries and never re-order them. */ var updates = map[int]schema.Update{ 1: updateFromV0, 2: updateFromV1, 3: updateFromV2, 4: updateFromV3, 5: updateFromV4, 6: updateFromV5, 7: updateFromV6, 8: updateFromV7, 9: updateFromV8, 10: updateFromV9, 11: updateFromV10, 12: updateFromV11, 13: updateFromV12, 14: updateFromV13, 15: updateFromV14, 16: updateFromV15, 17: updateFromV16, 18: updateFromV17, 19: updateFromV18, 20: updateFromV19, 21: updateFromV20, 22: updateFromV21, 23: updateFromV22, 24: updateFromV23, 25: updateFromV24, 26: updateFromV25, 27: updateFromV26, 28: updateFromV27, 29: updateFromV28, 30: updateFromV29, 31: updateFromV30, 32: updateFromV31, 33: updateFromV32, 34: updateFromV33, 35: updateFromV34, 36: updateFromV35, 37: updateFromV36, 38: updateFromV37, 39: updateFromV38, 40: updateFromV39, 41: updateFromV40, 42: updateFromV41, 43: updateFromV42, } // UpdateFromPreClustering is the last schema version where clustering support // was not available, and hence no cluster dqlite database is used. const UpdateFromPreClustering = 36 // Schema updates begin here // updateFromV42 ensures key and value fields in config table are TEXT NOT NULL. func updateFromV42(ctx context.Context, tx *sql.Tx) error { stmt := ` CREATE TABLE "config_new" ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, UNIQUE (key) ); INSERT INTO "config_new" SELECT * FROM "config"; DROP TABLE "config"; ALTER TABLE "config_new" RENAME TO "config"; ` _, err := tx.Exec(stmt) return err } func updateFromV41(ctx context.Context, tx *sql.Tx) error { stmt := ` ALTER TABLE raft_nodes ADD COLUMN name TEXT NOT NULL default ""; ` _, err := tx.Exec(stmt) return err } func updateFromV40(ctx context.Context, tx *sql.Tx) error { stmt := ` CREATE TABLE certificates ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, fingerprint TEXT NOT NULL, type INTEGER NOT NULL, name TEXT NOT NULL, certificate TEXT NOT NULL, UNIQUE (fingerprint) ); ` _, err := tx.Exec(stmt) return err } // Fix the address of the bootstrap node being set to "0" in the raft_nodes // table. func updateFromV39(ctx context.Context, tx *sql.Tx) error { type node struct { ID uint64 Address string } sql := "SELECT id, address FROM raft_nodes" nodes := []node{} err := query.Scan(ctx, tx, sql, func(scan func(dest ...any) error) error { n := node{} err := scan(&n.ID, &n.Address) if err != nil { return err } nodes = append(nodes, n) return nil }) if err != nil { return fmt.Errorf("Failed to fetch raft nodes: %w", err) } if len(nodes) != 1 { return nil } info := nodes[0] if info.ID != 1 || info.Address != "0" { return nil } config, err := query.SelectConfig(ctx, tx, "config", "") if err != nil { return err } address := config["cluster.https_address"] if address != "" { _, err := tx.Exec("UPDATE raft_nodes SET address=? WHERE id=1", address) if err != nil { return err } } return nil } // Add role column to raft_nodes table. All existing entries will have role "0" // which means voter. func updateFromV38(ctx context.Context, tx *sql.Tx) error { stmt := ` ALTER TABLE raft_nodes ADD COLUMN role INTEGER NOT NULL DEFAULT 0; ` _, err := tx.Exec(stmt) return err } // Copy core.https_address to cluster.https_address in case this node is // clustered. func updateFromV37(ctx context.Context, tx *sql.Tx) error { count, err := query.Count(ctx, tx, "raft_nodes", "") if err != nil { return fmt.Errorf("Fetch count of Raft nodes: %w", err) } if count == 0 { // This node is not clustered, nothing to do. return nil } // Copy the core.https_address config. _, err = tx.Exec(` INSERT INTO config (key, value) SELECT 'cluster.https_address', value FROM config WHERE key = 'core.https_address' `) if err != nil { return fmt.Errorf("Insert cluster.https_address config: %w", err) } return nil } // Add a raft_nodes table to be used when running in clustered mode. It lists // the current nodes in the cluster that are participating in the dqlite // database Raft cluster. // // The 'id' column contains the raft server ID of the database node, and the // 'address' column its network address. Both are used internally by the raft // Go package to manage the cluster. // // Typical setups will have 3 cluster members that participate to the dqlite // database Raft cluster, and an arbitrary number of additional cluster // members that don't. Non-database nodes are not tracked in this table, but rather // in the nodes table of the cluster database itself. // // The data in this table must be replicated on all nodes of the // cluster, regardless of whether they are part of the raft cluster or not, and // all nodes will consult this table when they need to find out a leader to // send SQL queries to. func updateFromV36(ctx context.Context, tx *sql.Tx) error { stmts := ` CREATE TABLE raft_nodes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, address TEXT NOT NULL, UNIQUE (address) ); DELETE FROM config WHERE NOT key='core.https_address'; DROP TABLE certificates; DROP TABLE containers_devices_config; DROP TABLE containers_devices; DROP TABLE containers_config; DROP TABLE containers_profiles; DROP TABLE containers; DROP TABLE images_aliases; DROP TABLE images_properties; DROP TABLE images_source; DROP TABLE images; DROP TABLE networks_config; DROP TABLE networks; DROP TABLE profiles_devices_config; DROP TABLE profiles_devices; DROP TABLE profiles_config; DROP TABLE profiles; DROP TABLE storage_volumes_config; DROP TABLE storage_volumes; DROP TABLE storage_pools_config; DROP TABLE storage_pools; ` _, err := tx.Exec(stmts) return err } func updateFromV35(ctx context.Context, tx *sql.Tx) error { stmts := ` CREATE TABLE tmp ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, image_id INTEGER NOT NULL, description TEXT, FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE, UNIQUE (name) ); INSERT INTO tmp (id, name, image_id, description) SELECT id, name, image_id, description FROM images_aliases; DROP TABLE images_aliases; ALTER TABLE tmp RENAME TO images_aliases; ALTER TABLE networks ADD COLUMN description TEXT; ALTER TABLE storage_pools ADD COLUMN description TEXT; ALTER TABLE storage_volumes ADD COLUMN description TEXT; ALTER TABLE containers ADD COLUMN description TEXT; ` _, err := tx.Exec(stmts) return err } func updateFromV34(ctx context.Context, tx *sql.Tx) error { stmt := ` CREATE TABLE IF NOT EXISTS storage_pools ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, driver VARCHAR(255) NOT NULL, UNIQUE (name) ); CREATE TABLE IF NOT EXISTS storage_pools_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_pool_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (storage_pool_id, key), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools (id) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS storage_volumes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, storage_pool_id INTEGER NOT NULL, type INTEGER NOT NULL, UNIQUE (storage_pool_id, name, type), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools (id) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS storage_volumes_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (storage_volume_id, key), FOREIGN KEY (storage_volume_id) REFERENCES storage_volumes (id) ON DELETE CASCADE );` _, err := tx.Exec(stmt) return err } func updateFromV33(ctx context.Context, tx *sql.Tx) error { stmt := ` CREATE TABLE IF NOT EXISTS networks ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, UNIQUE (name) ); CREATE TABLE IF NOT EXISTS networks_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (network_id, key), FOREIGN KEY (network_id) REFERENCES networks (id) ON DELETE CASCADE );` _, err := tx.Exec(stmt) return err } func updateFromV32(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec("ALTER TABLE containers ADD COLUMN last_use_date DATETIME;") return err } func updateFromV31(ctx context.Context, tx *sql.Tx) error { stmt := ` CREATE TABLE IF NOT EXISTS patches ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, applied_at DATETIME NOT NULL, UNIQUE (name) );` _, err := tx.Exec(stmt) return err } func updateFromV30(ctx context.Context, tx *sql.Tx) error { // NOTE: this database update contained daemon-level logic which // was been moved to patchUpdateFromV15 in patches.go. return nil } func updateFromV29(ctx context.Context, tx *sql.Tx) error { if util.PathExists(internalUtil.VarPath("zfs.img")) { err := os.Chmod(internalUtil.VarPath("zfs.img"), 0o600) if err != nil { return err } } return nil } func updateFromV28(ctx context.Context, tx *sql.Tx) error { stmt := ` INSERT INTO profiles_devices (profile_id, name, type) SELECT id, "aadisable", 2 FROM profiles WHERE name="docker"; INSERT INTO profiles_devices_config (profile_device_id, key, value) SELECT profiles_devices.id, "source", "/dev/null" FROM profiles_devices LEFT JOIN profiles WHERE profiles_devices.profile_id = profiles.id AND profiles.name = "docker" AND profiles_devices.name = "aadisable"; INSERT INTO profiles_devices_config (profile_device_id, key, value) SELECT profiles_devices.id, "path", "/sys/module/apparmor/parameters/enabled" FROM profiles_devices LEFT JOIN profiles WHERE profiles_devices.profile_id = profiles.id AND profiles.name = "docker" AND profiles_devices.name = "aadisable";` _, _ = tx.Exec(stmt) return nil } func updateFromV27(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec("UPDATE profiles_devices SET type=3 WHERE type='unix-char';") return err } func updateFromV26(ctx context.Context, tx *sql.Tx) error { stmt := ` ALTER TABLE images ADD COLUMN auto_update INTEGER NOT NULL DEFAULT 0; CREATE TABLE IF NOT EXISTS images_source ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, server TEXT NOT NULL, protocol INTEGER NOT NULL, certificate TEXT NOT NULL, alias VARCHAR(255) NOT NULL, FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE );` _, err := tx.Exec(stmt) return err } func updateFromV25(ctx context.Context, tx *sql.Tx) error { stmt := ` INSERT INTO profiles (name, description) VALUES ("docker", "Profile supporting docker in containers"); INSERT INTO profiles_config (profile_id, key, value) SELECT id, "security.nesting", "true" FROM profiles WHERE name="docker"; INSERT INTO profiles_config (profile_id, key, value) SELECT id, "linux.kernel_modules", "overlay, nf_nat" FROM profiles WHERE name="docker"; INSERT INTO profiles_devices (profile_id, name, type) SELECT id, "fuse", "unix-char" FROM profiles WHERE name="docker"; INSERT INTO profiles_devices_config (profile_device_id, key, value) SELECT profiles_devices.id, "path", "/dev/fuse" FROM profiles_devices LEFT JOIN profiles WHERE profiles_devices.profile_id = profiles.id AND profiles.name = "docker";` _, _ = tx.Exec(stmt) return nil } func updateFromV24(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec("ALTER TABLE containers ADD COLUMN stateful INTEGER NOT NULL DEFAULT 0;") return err } func updateFromV23(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec("ALTER TABLE profiles ADD COLUMN description TEXT;") return err } func updateFromV22(ctx context.Context, tx *sql.Tx) error { stmt := ` DELETE FROM containers_devices_config WHERE key='type'; DELETE FROM profiles_devices_config WHERE key='type';` _, err := tx.Exec(stmt) return err } func updateFromV21(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec("ALTER TABLE containers ADD COLUMN creation_date DATETIME NOT NULL DEFAULT 0;") return err } func updateFromV20(ctx context.Context, tx *sql.Tx) error { stmt := ` UPDATE containers_devices SET name='__upgrade_root' WHERE name='root'; UPDATE profiles_devices SET name='__upgrade_root' WHERE name='root'; INSERT INTO containers_devices (container_id, name, type) SELECT id, "root", 2 FROM containers; INSERT INTO containers_devices_config (container_device_id, key, value) SELECT id, "path", "/" FROM containers_devices WHERE name='root';` _, err := tx.Exec(stmt) return err } func updateFromV19(ctx context.Context, tx *sql.Tx) error { stmt := ` DELETE FROM containers_config WHERE container_id NOT IN (SELECT id FROM containers); DELETE FROM containers_devices_config WHERE container_device_id NOT IN (SELECT id FROM containers_devices WHERE container_id IN (SELECT id FROM containers)); DELETE FROM containers_devices WHERE container_id NOT IN (SELECT id FROM containers); DELETE FROM containers_profiles WHERE container_id NOT IN (SELECT id FROM containers); DELETE FROM images_aliases WHERE image_id NOT IN (SELECT id FROM images); DELETE FROM images_properties WHERE image_id NOT IN (SELECT id FROM images);` _, err := tx.Exec(stmt) return err } func updateFromV18(ctx context.Context, tx *sql.Tx) error { var id int var value string // Update container config rows, err := tx.QueryContext(ctx, "SELECT id, value FROM containers_config WHERE key='limits.memory'") if err != nil { return err } defer func() { _ = rows.Close() }() for rows.Next() { err := rows.Scan(&id, &value) if err != nil { return err } // If already an integer, don't touch _, err = strconv.Atoi(value) if err == nil { continue } // Generate the new value value = strings.ToUpper(value) value += "B" // Deal with completely broken values _, err = units.ParseByteSizeString(value) if err != nil { logger.Debugf("Invalid container memory limit, id=%d value=%s, removing", id, value) _, err = tx.Exec("DELETE FROM containers_config WHERE id=?;", id) if err != nil { return err } } // Set the new value _, err = tx.Exec("UPDATE containers_config SET value=? WHERE id=?", value, id) if err != nil { return err } } err = rows.Err() if err != nil { return err } // Update profiles config rows, err = tx.QueryContext(ctx, "SELECT id, value FROM profiles_config WHERE key='limits.memory'") if err != nil { return err } defer func() { _ = rows.Close() }() for rows.Next() { err := rows.Scan(&id, &value) if err != nil { return err } // If already an integer, don't touch _, err = strconv.Atoi(value) if err == nil { continue } // Generate the new value value = strings.ToUpper(value) value += "B" // Deal with completely broken values _, err = units.ParseByteSizeString(value) if err != nil { logger.Debugf("Invalid profile memory limit, id=%d value=%s, removing", id, value) _, err = tx.Exec("DELETE FROM profiles_config WHERE id=?;", id) if err != nil { return err } } // Set the new value _, err = tx.Exec("UPDATE profiles_config SET value=? WHERE id=?", value, id) if err != nil { return err } } err = rows.Err() if err != nil { return err } return nil } func updateFromV17(ctx context.Context, tx *sql.Tx) error { stmt := ` DELETE FROM profiles_config WHERE key LIKE 'volatile.%'; UPDATE containers_config SET key='limits.cpu' WHERE key='limits.cpus'; UPDATE profiles_config SET key='limits.cpu' WHERE key='limits.cpus';` _, err := tx.Exec(stmt) return err } func updateFromV16(ctx context.Context, tx *sql.Tx) error { stmt := ` UPDATE config SET key='storage.lvm_vg_name' WHERE key = 'core.lvm_vg_name'; UPDATE config SET key='storage.lvm_thinpool_name' WHERE key = 'core.lvm_thinpool_name';` _, err := tx.Exec(stmt) return err } func updateFromV15(ctx context.Context, tx *sql.Tx) error { // NOTE: this database update contained daemon-level logic which // was been moved to patchUpdateFromV15 in patches.go. return nil } func updateFromV14(ctx context.Context, tx *sql.Tx) error { stmt := ` PRAGMA foreign_keys=OFF; -- So that integrity doesn't get in the way for now DELETE FROM containers_config WHERE key="volatile.last_state.power"; INSERT INTO containers_config (container_id, key, value) SELECT id, "volatile.last_state.power", "RUNNING" FROM containers WHERE power_state=1; INSERT INTO containers_config (container_id, key, value) SELECT id, "volatile.last_state.power", "STOPPED" FROM containers WHERE power_state != 1; CREATE TABLE tmp ( id INTEGER primary key AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, architecture INTEGER NOT NULL, type INTEGER NOT NULL, ephemeral INTEGER NOT NULL DEFAULT 0, UNIQUE (name) ); INSERT INTO tmp SELECT id, name, architecture, type, ephemeral FROM containers; DROP TABLE containers; ALTER TABLE tmp RENAME TO containers; PRAGMA foreign_keys=ON; -- Make sure we turn integrity checks back on.` _, err := tx.Exec(stmt) return err } func updateFromV13(ctx context.Context, tx *sql.Tx) error { stmt := ` UPDATE containers_config SET key='volatile.base_image' WHERE key = 'volatile.baseImage';` _, err := tx.Exec(stmt) return err } func updateFromV12(ctx context.Context, tx *sql.Tx) error { stmt := ` ALTER TABLE images ADD COLUMN cached INTEGER NOT NULL DEFAULT 0; ALTER TABLE images ADD COLUMN last_use_date DATETIME;` _, err := tx.Exec(stmt) return err } func updateFromV11(ctx context.Context, tx *sql.Tx) error { // NOTE: this database update contained daemon-level logic which // was been moved to patchUpdateFromV15 in patches.go. return nil } func updateFromV10(ctx context.Context, tx *sql.Tx) error { // NOTE: this database update contained daemon-level logic which // was been moved to patchUpdateFromV10 in patches.go. return nil } func updateFromV9(ctx context.Context, tx *sql.Tx) error { stmt := ` CREATE TABLE tmp ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, type VARCHAR(255) NOT NULL default "none", FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE, UNIQUE (container_id, name) ); INSERT INTO tmp SELECT * FROM containers_devices; UPDATE containers_devices SET type=0 WHERE id IN (SELECT id FROM tmp WHERE type="none"); UPDATE containers_devices SET type=1 WHERE id IN (SELECT id FROM tmp WHERE type="nic"); UPDATE containers_devices SET type=2 WHERE id IN (SELECT id FROM tmp WHERE type="disk"); UPDATE containers_devices SET type=3 WHERE id IN (SELECT id FROM tmp WHERE type="unix-char"); UPDATE containers_devices SET type=4 WHERE id IN (SELECT id FROM tmp WHERE type="unix-block"); DROP TABLE tmp; CREATE TABLE tmp ( id INTEGER primary key AUTOINCREMENT NOT NULL, profile_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, type VARCHAR(255) NOT NULL default "none", FOREIGN KEY (profile_id) REFERENCES profiles (id) ON DELETE CASCADE, UNIQUE (profile_id, name) ); INSERT INTO tmp SELECT * FROM profiles_devices; UPDATE profiles_devices SET type=0 WHERE id IN (SELECT id FROM tmp WHERE type="none"); UPDATE profiles_devices SET type=1 WHERE id IN (SELECT id FROM tmp WHERE type="nic"); UPDATE profiles_devices SET type=2 WHERE id IN (SELECT id FROM tmp WHERE type="disk"); UPDATE profiles_devices SET type=3 WHERE id IN (SELECT id FROM tmp WHERE type="unix-char"); UPDATE profiles_devices SET type=4 WHERE id IN (SELECT id FROM tmp WHERE type="unix-block"); DROP TABLE tmp;` _, err := tx.Exec(stmt) return err } func updateFromV8(ctx context.Context, tx *sql.Tx) error { stmt := ` UPDATE certificates SET fingerprint = replace(fingerprint, " ", "");` _, err := tx.Exec(stmt) return err } func updateFromV7(ctx context.Context, tx *sql.Tx) error { stmt := ` UPDATE config SET key='core.trust_password' WHERE key IN ('password', 'trust_password', 'trust-password', 'core.trust-password'); DELETE FROM config WHERE key != 'core.trust_password';` _, err := tx.Exec(stmt) return err } func updateFromV6(ctx context.Context, tx *sql.Tx) error { // This update recreates the schemas that need an ON DELETE CASCADE foreign // key. stmt := ` PRAGMA foreign_keys=OFF; -- So that integrity doesn't get in the way for now CREATE TEMP TABLE tmp AS SELECT * FROM containers_config; DROP TABLE containers_config; CREATE TABLE IF NOT EXISTS containers_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE, UNIQUE (container_id, key) ); INSERT INTO containers_config SELECT * FROM tmp; DROP TABLE tmp; CREATE TEMP TABLE tmp AS SELECT * FROM containers_devices; DROP TABLE containers_devices; CREATE TABLE IF NOT EXISTS containers_devices ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, type INTEGER NOT NULL default 0, FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE, UNIQUE (container_id, name) ); INSERT INTO containers_devices SELECT * FROM tmp; DROP TABLE tmp; CREATE TEMP TABLE tmp AS SELECT * FROM containers_devices_config; DROP TABLE containers_devices_config; CREATE TABLE IF NOT EXISTS containers_devices_config ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_device_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, FOREIGN KEY (container_device_id) REFERENCES containers_devices (id) ON DELETE CASCADE, UNIQUE (container_device_id, key) ); INSERT INTO containers_devices_config SELECT * FROM tmp; DROP TABLE tmp; CREATE TEMP TABLE tmp AS SELECT * FROM containers_profiles; DROP TABLE containers_profiles; CREATE TABLE IF NOT EXISTS containers_profiles ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, profile_id INTEGER NOT NULL, apply_order INTEGER NOT NULL default 0, UNIQUE (container_id, profile_id), FOREIGN KEY (container_id) REFERENCES containers(id) ON DELETE CASCADE, FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE ); INSERT INTO containers_profiles SELECT * FROM tmp; DROP TABLE tmp; CREATE TEMP TABLE tmp AS SELECT * FROM images_aliases; DROP TABLE images_aliases; CREATE TABLE IF NOT EXISTS images_aliases ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, image_id INTEGER NOT NULL, description VARCHAR(255), FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE, UNIQUE (name) ); INSERT INTO images_aliases SELECT * FROM tmp; DROP TABLE tmp; CREATE TEMP TABLE tmp AS SELECT * FROM images_properties; DROP TABLE images_properties; CREATE TABLE IF NOT EXISTS images_properties ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, type INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE ); INSERT INTO images_properties SELECT * FROM tmp; DROP TABLE tmp; CREATE TEMP TABLE tmp AS SELECT * FROM profiles_config; DROP TABLE profiles_config; CREATE TABLE IF NOT EXISTS profiles_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value VARCHAR(255), UNIQUE (profile_id, key), FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE ); INSERT INTO profiles_config SELECT * FROM tmp; DROP TABLE tmp; CREATE TEMP TABLE tmp AS SELECT * FROM profiles_devices; DROP TABLE profiles_devices; CREATE TABLE IF NOT EXISTS profiles_devices ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, type INTEGER NOT NULL default 0, UNIQUE (profile_id, name), FOREIGN KEY (profile_id) REFERENCES profiles (id) ON DELETE CASCADE ); INSERT INTO profiles_devices SELECT * FROM tmp; DROP TABLE tmp; CREATE TEMP TABLE tmp AS SELECT * FROM profiles_devices_config; DROP TABLE profiles_devices_config; CREATE TABLE IF NOT EXISTS profiles_devices_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_device_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (profile_device_id, key), FOREIGN KEY (profile_device_id) REFERENCES profiles_devices (id) ON DELETE CASCADE ); INSERT INTO profiles_devices_config SELECT * FROM tmp; DROP TABLE tmp; PRAGMA foreign_keys=ON; -- Make sure we turn integrity checks back on.` _, err := tx.Exec(stmt) if err != nil { return err } // Get the rows with broken foreign keys an nuke them rows, err := tx.QueryContext(ctx, "PRAGMA foreign_key_check;") if err != nil { return err } defer func() { _ = rows.Close() }() var tablestodelete []string var rowidtodelete []int for rows.Next() { var tablename string var rowid int var targetname string var keynumber int err := rows.Scan(&tablename, &rowid, &targetname, &keynumber) if err != nil { return err } tablestodelete = append(tablestodelete, tablename) rowidtodelete = append(rowidtodelete, rowid) } for i := range tablestodelete { _, err = tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE rowid = %d;", tablestodelete[i], rowidtodelete[i])) if err != nil { return err } } return nil } func updateFromV5(ctx context.Context, tx *sql.Tx) error { stmt := ` ALTER TABLE containers ADD COLUMN power_state INTEGER NOT NULL DEFAULT 0; ALTER TABLE containers ADD COLUMN ephemeral INTEGER NOT NULL DEFAULT 0;` _, err := tx.Exec(stmt) return err } func updateFromV4(ctx context.Context, tx *sql.Tx) error { stmt := ` CREATE TABLE IF NOT EXISTS config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (key) );` _, err := tx.Exec(stmt) if err != nil { return err } passfname := internalUtil.VarPath("adminpwd") passOut, err := os.Open(passfname) oldPassword := "" if err == nil { defer func() { _ = passOut.Close() }() buff := make([]byte, 96) _, err = passOut.Read(buff) if err != nil { return err } oldPassword = hex.EncodeToString(buff) stmt := `INSERT INTO config (key, value) VALUES ("core.trust_password", ?);` _, err := tx.Exec(stmt, oldPassword) if err != nil { return err } return os.Remove(passfname) } return nil } func updateFromV3(ctx context.Context, tx *sql.Tx) error { // Attempt to create a default profile (but don't fail if already there) _, _ = tx.Exec("INSERT INTO profiles (name) VALUES (\"default\");") return nil } func updateFromV2(ctx context.Context, tx *sql.Tx) error { stmt := ` CREATE TABLE IF NOT EXISTS containers_devices ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, type INTEGER NOT NULL default 0, FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE, UNIQUE (container_id, name) ); CREATE TABLE IF NOT EXISTS containers_devices_config ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_device_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, FOREIGN KEY (container_device_id) REFERENCES containers_devices (id), UNIQUE (container_device_id, key) ); CREATE TABLE IF NOT EXISTS containers_profiles ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, profile_id INTEGER NOT NULL, apply_order INTEGER NOT NULL default 0, UNIQUE (container_id, profile_id), FOREIGN KEY (container_id) REFERENCES containers(id) ON DELETE CASCADE, FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS profiles ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, UNIQUE (name) ); CREATE TABLE IF NOT EXISTS profiles_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value VARCHAR(255), UNIQUE (profile_id, key), FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS profiles_devices ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_id INTEGER NOT NULL, name VARCHAR(255) NOT NULL, type INTEGER NOT NULL default 0, UNIQUE (profile_id, name), FOREIGN KEY (profile_id) REFERENCES profiles (id) ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS profiles_devices_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_device_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (profile_device_id, key), FOREIGN KEY (profile_device_id) REFERENCES profiles_devices (id) );` _, err := tx.Exec(stmt) return err } func updateFromV1(ctx context.Context, tx *sql.Tx) error { // v1..v2 adds images aliases stmt := ` CREATE TABLE IF NOT EXISTS images_aliases ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, image_id INTEGER NOT NULL, description VARCHAR(255), FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE, UNIQUE (name) );` _, err := tx.Exec(stmt) return err } func updateFromV0(ctx context.Context, tx *sql.Tx) error { // v0..v1 the dawn of containers stmt := ` CREATE TABLE certificates ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, fingerprint VARCHAR(255) NOT NULL, type INTEGER NOT NULL, name VARCHAR(255) NOT NULL, certificate TEXT NOT NULL, UNIQUE (fingerprint) ); CREATE TABLE containers ( id INTEGER primary key AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, architecture INTEGER NOT NULL, type INTEGER NOT NULL, UNIQUE (name) ); CREATE TABLE containers_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, FOREIGN KEY (container_id) REFERENCES containers (id), UNIQUE (container_id, key) ); CREATE TABLE images ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, fingerprint VARCHAR(255) NOT NULL, filename VARCHAR(255) NOT NULL, size INTEGER NOT NULL, public INTEGER NOT NULL DEFAULT 0, architecture INTEGER NOT NULL, creation_date DATETIME, expiry_date DATETIME, upload_date DATETIME NOT NULL, UNIQUE (fingerprint) ); CREATE TABLE images_properties ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, type INTEGER NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, FOREIGN KEY (image_id) REFERENCES images (id) );` _, err := tx.Exec(stmt) return err } // UpdateFromV16 is used by a legacy test in the parent package. var UpdateFromV16 = updateFromV16 incus-7.0.0/internal/server/db/node/update_test.go000066400000000000000000000063061517523235500221600ustar00rootroot00000000000000package node_test import ( "context" "database/sql" "slices" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db/node" "github.com/lxc/incus/v7/internal/server/db/query" ) func TestUpdateFromV38_RaftNodes(t *testing.T) { schema := node.Schema() db, err := schema.ExerciseUpdate(39, func(db *sql.DB) { _, err := db.Exec("INSERT INTO raft_nodes VALUES (1, '1.2.3.4:666')") require.NoError(t, err) }) require.NoError(t, err) err = query.Transaction(context.TODO(), db, func(ctx context.Context, tx *sql.Tx) error { roles, err := query.SelectIntegers(ctx, tx, "SELECT role FROM raft_nodes") require.NoError(t, err) assert.Equal(t, roles, []int{0}) return nil }) require.NoError(t, err) } func TestUpdateFromV36_RaftNodes(t *testing.T) { schema := node.Schema() db, err := schema.ExerciseUpdate(37, nil) require.NoError(t, err) _, err = db.Exec("INSERT INTO raft_nodes VALUES (1, '1.2.3.4:666')") require.NoError(t, err) } // All model tables previously in the node database have been migrated to the // cluster database, and dropped from the node database. func TestUpdateFromV36_DropTables(t *testing.T) { schema := node.Schema() db, err := schema.ExerciseUpdate(37, nil) require.NoError(t, err) var current []string err = query.Transaction(context.TODO(), db, func(ctx context.Context, tx *sql.Tx) error { var err error stmt := "SELECT name FROM sqlite_master WHERE type='table'" current, err = query.SelectStrings(ctx, tx, stmt) return err }) require.NoError(t, err) deleted := []string{ "networks", "networks_config", } for _, name := range deleted { assert.False(t, slices.Contains(current, name)) } } // If clustering is enabled, the core.https_address config gets copied to // cluster.https_config. func TestUpdateFromV37_CopyCoreHTTPSAddress(t *testing.T) { schema := node.Schema() db, err := schema.ExerciseUpdate(38, func(db *sql.DB) { _, err := db.Exec("INSERT INTO raft_nodes VALUES (1, '1.2.3.4:666')") require.NoError(t, err) _, err = db.Exec("INSERT INTO config VALUES (1, 'core.https_address', '1.2.3.4:666')") require.NoError(t, err) }) require.NoError(t, err) var clusterAddress string err = query.Transaction(context.TODO(), db, func(ctx context.Context, tx *sql.Tx) error { stmt := "SELECT value FROM config WHERE key='cluster.https_address'" row := tx.QueryRow(stmt) err := row.Scan(&clusterAddress) return err }) require.NoError(t, err) assert.Equal(t, clusterAddress, "1.2.3.4:666") } // If clustering is not enabled, the core.https_address config does not get copied. func TestUpdateFromV37_NotClustered(t *testing.T) { schema := node.Schema() db, err := schema.ExerciseUpdate(38, func(db *sql.DB) { _, err := db.Exec("INSERT INTO config VALUES (1, 'core.https_address', '1.2.3.4:666')") require.NoError(t, err) }) require.NoError(t, err) var clusterAddress string err = query.Transaction(context.TODO(), db, func(ctx context.Context, tx *sql.Tx) error { stmt := "SELECT value FROM config WHERE key='cluster.https_address'" row := tx.QueryRow(stmt) err := row.Scan(&clusterAddress) return err }) require.EqualError(t, err, "sql: no rows in result set") } incus-7.0.0/internal/server/db/node_test.go000066400000000000000000000340231517523235500206730ustar00rootroot00000000000000//go:build linux && cgo && !agent package db_test import ( "context" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/osarch" ) // Add a new raft node. func TestNodeAdd(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() id, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) assert.Equal(t, int64(2), id) nodes, err := tx.GetNodes(context.Background()) require.NoError(t, err) require.Len(t, nodes, 2) node, err := tx.GetNodeByAddress(context.Background(), "1.2.3.4:666") require.NoError(t, err) assert.Equal(t, "buzz", node.Name) assert.Equal(t, "1.2.3.4:666", node.Address) assert.Equal(t, cluster.SchemaVersion, node.Schema) assert.Equal(t, len(version.APIExtensions), node.APIExtensions) assert.Equal(t, [2]int{cluster.SchemaVersion, len(version.APIExtensions)}, node.Version()) assert.False(t, node.IsOffline(20*time.Second)) node, err = tx.GetNodeByName(context.Background(), "buzz") require.NoError(t, err) assert.Equal(t, "buzz", node.Name) } func TestGetNodesCount(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() count, err := tx.GetNodesCount(context.Background()) require.NoError(t, err) assert.Equal(t, 1, count) // There's always at least one node. _, err = tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) count, err = tx.GetNodesCount(context.Background()) require.NoError(t, err) assert.Equal(t, 2, count) } func TestNodeIsOutdated_SingleNode(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() outdated, err := tx.NodeIsOutdated(context.Background()) require.NoError(t, err) assert.False(t, outdated) } func TestNodeIsOutdated_AllNodesAtSameVersion(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() _, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) outdated, err := tx.NodeIsOutdated(context.Background()) require.NoError(t, err) assert.False(t, outdated) } func TestNodeIsOutdated_OneNodeWithHigherVersion(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() id, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) version := [2]int{cluster.SchemaVersion + 1, len(version.APIExtensions)} err = tx.SetNodeVersion(id, version) require.NoError(t, err) outdated, err := tx.NodeIsOutdated(context.Background()) require.NoError(t, err) assert.True(t, outdated) } func TestNodeIsOutdated_OneNodeWithLowerVersion(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() id, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) version := [2]int{cluster.SchemaVersion, len(version.APIExtensions) - 1} err = tx.SetNodeVersion(id, version) require.NoError(t, err) outdated, err := tx.NodeIsOutdated(context.Background()) require.NoError(t, err) assert.False(t, outdated) } func TestGetLocalNodeName(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() name, err := tx.GetLocalNodeName(context.Background()) require.NoError(t, err) // The default node 1 has a conventional name 'none'. assert.Equal(t, "none", name) } // Rename a node. func TestRenameNode(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() _, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) err = tx.RenameNode(context.Background(), "buzz", "rusp") require.NoError(t, err) node, err := tx.GetNodeByName(context.Background(), "rusp") require.NoError(t, err) assert.Equal(t, "rusp", node.Name) _, err = tx.CreateNode("buzz", "5.6.7.8:666") require.NoError(t, err) err = tx.RenameNode(context.Background(), "rusp", "buzz") assert.Equal(t, db.ErrAlreadyDefined, err) } // Remove a new raft node. func TestRemoveNode(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() _, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) id, err := tx.CreateNode("rusp", "5.6.7.8:666") require.NoError(t, err) err = tx.RemoveNode(id) require.NoError(t, err) _, err = tx.GetNodeByName(context.Background(), "buzz") assert.NoError(t, err) _, err = tx.GetNodeByName(context.Background(), "rusp") assert.True(t, response.IsNotFoundError(err)) } // Mark a node has pending. func TestSetNodePendingFlag(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() id, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) // Add the pending flag err = tx.SetNodePendingFlag(id, true) require.NoError(t, err) // Pending nodes are skipped from regular listing _, err = tx.GetNodeByName(context.Background(), "buzz") assert.True(t, response.IsNotFoundError(err)) nodes, err := tx.GetNodes(context.Background()) require.NoError(t, err) assert.Len(t, nodes, 1) // But the key be retrieved with GetPendingNodeByAddress node, err := tx.GetPendingNodeByAddress(context.Background(), "1.2.3.4:666") require.NoError(t, err) assert.Equal(t, id, node.ID) // Remove the pending flag err = tx.SetNodePendingFlag(id, false) require.NoError(t, err) node, err = tx.GetNodeByName(context.Background(), "buzz") require.NoError(t, err) assert.Equal(t, id, node.ID) } // Update the heartbeat of a node. func TestSetNodeHeartbeat(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() _, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) err = tx.SetNodeHeartbeat("1.2.3.4:666", time.Now().Add(-time.Minute)) require.NoError(t, err) nodes, err := tx.GetNodes(context.Background()) require.NoError(t, err) require.Len(t, nodes, 2) node := nodes[1] assert.True(t, node.IsOffline(20*time.Second)) } // A node is considered empty only if it has no instances. func TestNodeIsEmpty_Instances(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() id, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) message, err := tx.NodeIsEmpty(context.Background(), id) require.NoError(t, err) assert.Equal(t, "", message) _, err = tx.Tx().Exec(` INSERT INTO instances (id, node_id, name, architecture, type, project_id, description) VALUES (1, ?, 'foo', 1, 1, 1, '') `, id) require.NoError(t, err) message, err = tx.NodeIsEmpty(context.Background(), id) require.NoError(t, err) assert.Equal(t, "Node still has the following instances: foo", message) err = tx.ClearNode(context.Background(), id) require.NoError(t, err) message, err = tx.NodeIsEmpty(context.Background(), id) require.NoError(t, err) assert.Equal(t, "", message) } // A node is considered empty only if it has no images that are available only // on that node. func TestNodeIsEmpty_Images(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() id, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) _, err = tx.Tx().Exec(` INSERT INTO images (id, fingerprint, filename, size, architecture, upload_date, project_id) VALUES (1, 'abc', 'foo', 123, 1, ?, 1)`, time.Now()) require.NoError(t, err) _, err = tx.Tx().Exec(` INSERT INTO images_nodes(image_id, node_id) VALUES(1, ?)`, id) require.NoError(t, err) message, err := tx.NodeIsEmpty(context.Background(), id) require.NoError(t, err) assert.Equal(t, "Node still has the following images: abc", message) // Insert a new image entry for node 1 (the default node). _, err = tx.Tx().Exec(` INSERT INTO images_nodes(image_id, node_id) VALUES(1, 1)`) require.NoError(t, err) message, err = tx.NodeIsEmpty(context.Background(), id) require.NoError(t, err) assert.Equal(t, "", message) } // A node is considered empty only if it has no custom volumes on it. func TestNodeIsEmpty_CustomVolumes(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() id, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) _, err = tx.Tx().Exec(` INSERT INTO storage_pools (id, name, driver, description) VALUES (1, 'local', 'zfs', '')`) require.NoError(t, err) _, err = tx.Tx().Exec(` INSERT INTO storage_volumes(name, storage_pool_id, node_id, type, project_id, description) VALUES ('data', 1, ?, ?, 1, '')`, id, db.StoragePoolVolumeTypeCustom) require.NoError(t, err) message, err := tx.NodeIsEmpty(context.Background(), id) require.NoError(t, err) assert.Equal(t, "Node still has the following custom volumes: data", message) } // If there are 2 online nodes, return the address of the one with the least // number of instances. func TestGetCandidateMembers(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() _, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) // Add an instance to the default node (ID 1) _, err = tx.Tx().Exec(` INSERT INTO instances (id, node_id, name, architecture, type, project_id, description) VALUES (1, 1, 'foo', 1, 1, 1, '') `) require.NoError(t, err) allMembers, err := tx.GetNodes(context.Background()) require.NoError(t, err) members, err := tx.GetCandidateMembers(context.Background(), allMembers, nil, "", nil, time.Duration(db.DefaultOfflineThreshold)*time.Second) require.NoError(t, err) require.Len(t, members, 2) assert.Equal(t, "buzz", members[0].Name) } // If there are nodes, and one of them is offline, return the name of the // online node, even if the offline one has more instances. func TestGetCandidateMembers_OfflineNode(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() id, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) // Add an instance to the newly created node. _, err = tx.Tx().Exec(` INSERT INTO instances (id, node_id, name, architecture, type, project_id, description) VALUES (1, ?, 'foo', 1, 1, 1, '') `, id) require.NoError(t, err) // Mark the default node has offline. err = tx.SetNodeHeartbeat("0.0.0.0", time.Now().Add(-time.Minute)) require.NoError(t, err) allMembers, err := tx.GetNodes(context.Background()) require.NoError(t, err) members, err := tx.GetCandidateMembers(context.Background(), allMembers, nil, "", nil, time.Duration(db.DefaultOfflineThreshold)*time.Second) require.NoError(t, err) require.Len(t, members, 1) assert.Equal(t, "buzz", members[0].Name) } // If there are 2 online nodes, and an instance is pending on one of them, // return the address of the other one number of instances. func TestGetCandidateMembers_Pending(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() _, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) // Add a pending instance to the default node (ID 1) _, err = tx.Tx().Exec(` INSERT INTO operations (id, uuid, node_id, type, project_id) VALUES (1, 'abc', 1, ?, 1) `, operationtype.InstanceCreate) require.NoError(t, err) allMembers, err := tx.GetNodes(context.Background()) require.NoError(t, err) members, err := tx.GetCandidateMembers(context.Background(), allMembers, nil, "", nil, time.Duration(db.DefaultOfflineThreshold)*time.Second) require.NoError(t, err) require.Len(t, members, 2) assert.Equal(t, "buzz", members[0].Name) } // If specific architectures were selected, return only nodes with those // architectures. func TestGetCandidateMembers_Architecture(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() localArch, err := osarch.ArchitectureGetLocalID() require.NoError(t, err) testArch := osarch.ARCH_64BIT_S390_BIG_ENDIAN if localArch == testArch { testArch = osarch.ARCH_64BIT_INTEL_X86 } _, err = tx.CreateNodeWithArch("buzz", "1.2.3.4:666", testArch) require.NoError(t, err) // Add an instance to the default node (ID 1) _, err = tx.Tx().Exec(` INSERT INTO instances (id, node_id, name, architecture, type, project_id, description) VALUES (1, 1, 'foo', 1, 1, 1, '') `) require.NoError(t, err) allMembers, err := tx.GetNodes(context.Background()) require.NoError(t, err) members, err := tx.GetCandidateMembers(context.Background(), allMembers, []int{localArch}, "", nil, time.Duration(db.DefaultOfflineThreshold)*time.Second) require.NoError(t, err) require.Len(t, members, 1) // The local member is returned despite it has more instances. assert.Equal(t, "none", members[0].Name) } func TestUpdateNodeFailureDomain(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() id, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) domain, err := tx.GetNodeFailureDomain(context.Background(), id) require.NoError(t, err) assert.Equal(t, "default", domain) assert.NoError(t, tx.UpdateNodeFailureDomain(context.Background(), id, "foo")) domain, err = tx.GetNodeFailureDomain(context.Background(), id) require.NoError(t, err) assert.Equal(t, "foo", domain) domains, err := tx.GetNodesFailureDomains(context.Background()) require.NoError(t, err) assert.Equal(t, map[string]uint64{"0.0.0.0": 0, "1.2.3.4:666": 1}, domains) assert.NoError(t, tx.UpdateNodeFailureDomain(context.Background(), id, "default")) domains, err = tx.GetNodesFailureDomains(context.Background()) require.NoError(t, err) assert.Equal(t, map[string]uint64{"0.0.0.0": 0, "1.2.3.4:666": 0}, domains) } func TestGetCandidateMembers_DefaultArch(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() localArch, err := osarch.ArchitectureGetLocalID() require.NoError(t, err) testArch := osarch.ARCH_64BIT_ARMV8_LITTLE_ENDIAN if localArch == testArch { testArch = osarch.ARCH_64BIT_INTEL_X86 } id, err := tx.CreateNodeWithArch("buzz", "1.2.3.4:666", testArch) require.NoError(t, err) // Add an instance to the newly created node. _, err = tx.Tx().Exec(` INSERT INTO instances (id, node_id, name, architecture, type, project_id, description) VALUES (1, ?, 'foo', 1, 1, 1, '') `, id) require.NoError(t, err) allMembers, err := tx.GetNodes(context.Background()) require.NoError(t, err) members, err := tx.GetCandidateMembers(context.Background(), allMembers, []int{testArch}, "", nil, time.Duration(db.DefaultOfflineThreshold)*time.Second) require.NoError(t, err) require.Len(t, members, 1) assert.Equal(t, "buzz", members[0].Name) } incus-7.0.0/internal/server/db/operations.go000066400000000000000000000037041517523235500210740ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/db/query" ) // GetAllNodesWithOperations returns a list of nodes that have operations in any project. func (c *ClusterTx) GetAllNodesWithOperations(ctx context.Context) ([]string, error) { stmt := ` SELECT DISTINCT nodes.address FROM operations JOIN nodes ON nodes.id = operations.node_id ` return query.SelectStrings(ctx, c.tx, stmt) } // GetNodesWithOperations returns a list of nodes that have operations. func (c *ClusterTx) GetNodesWithOperations(ctx context.Context, project string) ([]string, error) { stmt := ` SELECT DISTINCT nodes.address FROM operations LEFT OUTER JOIN projects ON projects.id = operations.project_id JOIN nodes ON nodes.id = operations.node_id WHERE projects.name = ? OR operations.project_id IS NULL ` return query.SelectStrings(ctx, c.tx, stmt, project) } // GetOperationsOfType returns a list operations that belong to the specified project and have the desired type. func (c *ClusterTx) GetOperationsOfType(ctx context.Context, projectName string, opType operationtype.Type) ([]cluster.Operation, error) { var ops []cluster.Operation stmt := ` SELECT operations.id, operations.uuid, operations.type, nodes.address FROM operations LEFT JOIN projects on projects.id = operations.project_id JOIN nodes on nodes.id = operations.node_id WHERE (projects.name = ? OR operations.project_id IS NULL) and operations.type = ? ` rows, err := c.tx.QueryContext(ctx, stmt, projectName, opType) if err != nil { return nil, err } defer func() { _ = rows.Close() }() for rows.Next() { var op cluster.Operation err := rows.Scan(&op.ID, &op.UUID, &op.Type, &op.NodeAddress) if err != nil { return nil, err } ops = append(ops, op) } if rows.Err() != nil { return nil, err } return ops, nil } incus-7.0.0/internal/server/db/operations_test.go000066400000000000000000000063251517523235500221350ustar00rootroot00000000000000//go:build linux && cgo && !agent package db_test import ( "context" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" ) // Add, get and remove an operation. func TestOperation(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() projectID, err := cluster.GetProjectID(context.Background(), tx.Tx(), "default") require.NoError(t, err) nodeID := tx.GetNodeID() uuid := "abcd" opInfo := cluster.Operation{ NodeID: nodeID, Type: operationtype.InstanceCreate, UUID: uuid, ProjectID: &projectID, } id, err := cluster.CreateOrReplaceOperation(context.TODO(), tx.Tx(), opInfo) require.NoError(t, err) assert.Equal(t, int64(1), id) filter := cluster.OperationFilter{NodeID: &nodeID} operations, err := cluster.GetOperations(context.TODO(), tx.Tx(), filter) require.NoError(t, err) assert.Len(t, operations, 1) assert.Equal(t, operations[0].UUID, "abcd") filter = cluster.OperationFilter{UUID: &uuid} ops, err := cluster.GetOperations(context.TODO(), tx.Tx(), filter) require.NoError(t, err) assert.Equal(t, len(ops), 1) operation := ops[0] assert.Equal(t, id, operation.ID) assert.Equal(t, operationtype.InstanceCreate, operation.Type) filter = cluster.OperationFilter{NodeID: &nodeID} ops, err = cluster.GetOperations(context.TODO(), tx.Tx(), filter) require.NoError(t, err) assert.Equal(t, "abcd", ops[0].UUID) err = cluster.DeleteOperation(context.TODO(), tx.Tx(), "abcd") require.NoError(t, err) filter = cluster.OperationFilter{UUID: &uuid} ops, err = cluster.GetOperations(context.TODO(), tx.Tx(), filter) require.NoError(t, err) assert.Equal(t, len(ops), 0) } // Add, get and remove an operation not associated with any project. func TestOperationNoProject(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() nodeID := tx.GetNodeID() uuid := "abcd" opInfo := cluster.Operation{ NodeID: nodeID, Type: operationtype.InstanceCreate, UUID: uuid, } id, err := cluster.CreateOrReplaceOperation(context.TODO(), tx.Tx(), opInfo) require.NoError(t, err) assert.Equal(t, int64(1), id) filter := cluster.OperationFilter{NodeID: &nodeID} operations, err := cluster.GetOperations(context.TODO(), tx.Tx(), filter) require.NoError(t, err) assert.Len(t, operations, 1) assert.Equal(t, operations[0].UUID, "abcd") filter = cluster.OperationFilter{UUID: &uuid} ops, err := cluster.GetOperations(context.TODO(), tx.Tx(), filter) require.NoError(t, err) assert.Equal(t, len(ops), 1) operation := ops[0] require.NoError(t, err) assert.Equal(t, id, operation.ID) assert.Equal(t, operationtype.InstanceCreate, operation.Type) filter = cluster.OperationFilter{NodeID: &nodeID} ops, err = cluster.GetOperations(context.TODO(), tx.Tx(), filter) require.NoError(t, err) assert.Equal(t, "abcd", ops[0].UUID) err = cluster.DeleteOperation(context.TODO(), tx.Tx(), "abcd") require.NoError(t, err) filter = cluster.OperationFilter{UUID: &uuid} ops, err = cluster.GetOperations(context.TODO(), tx.Tx(), filter) require.NoError(t, err) assert.Equal(t, len(ops), 0) } incus-7.0.0/internal/server/db/operationtype/000077500000000000000000000000001517523235500212605ustar00rootroot00000000000000incus-7.0.0/internal/server/db/operationtype/operation_type.go000066400000000000000000000214211517523235500246500ustar00rootroot00000000000000package operationtype import ( "github.com/lxc/incus/v7/internal/server/auth" ) // Type is a numeric code identifying the type of an Operation. type Type int64 // Possible values for Type // // WARNING: The type codes are stored in the database, so this list of // definitions should be normally append-only. Any other change // requires a database update. const ( Unknown Type = iota ClusterBootstrap ClusterJoin BackupCreate BackupRename BackupRestore BackupRemove ConsoleShow InstanceCreate InstanceUpdate InstanceRename InstanceMigrate InstanceLiveMigrate InstanceFreeze InstanceUnfreeze InstanceDelete InstanceStart InstanceStop InstanceRestart InstanceRebuild CommandExec SnapshotCreate SnapshotRename SnapshotRestore SnapshotTransfer SnapshotUpdate SnapshotDelete ImageDownload ImageDelete ImageToken ImageRefresh VolumeCopy VolumeCreate VolumeMigrate VolumeMove VolumeSnapshotCreate VolumeSnapshotDelete VolumeSnapshotUpdate ProjectRename ImagesExpire ImagesPruneLeftover ImagesUpdate ImagesSynchronize LogsExpire InstanceTypesUpdate BackupsExpire SnapshotsExpire CustomVolumeSnapshotsExpire CustomVolumeBackupCreate CustomVolumeBackupRemove CustomVolumeBackupRename CustomVolumeBackupRestore WarningsPruneResolved ClusterJoinToken VolumeSnapshotRename ClusterMemberEvacuate ClusterMemberRestore CertificateAddToken RemoveOrphanedOperations RenewServerCertificate RemoveExpiredTokens ClusterHeal BucketBackupCreate BucketBackupRemove BucketBackupRename BucketBackupRestore ) // Description return a human-readable description of the operation type. func (t Type) Description() string { switch t { case ClusterBootstrap: return "Creating bootstrap node" case ClusterJoin: return "Joining cluster" case BackupCreate: return "Backing up instance" case BackupRename: return "Renaming instance backup" case BackupRestore: return "Restoring backup" case BackupRemove: return "Removing instance backup" case ConsoleShow: return "Showing console" case InstanceCreate: return "Creating instance" case InstanceUpdate: return "Updating instance" case InstanceRename: return "Renaming instance" case InstanceMigrate: return "Migrating instance" case InstanceLiveMigrate: return "Live-migrating instance" case InstanceFreeze: return "Freezing instance" case InstanceUnfreeze: return "Unfreezing instance" case InstanceDelete: return "Deleting instance" case InstanceStart: return "Starting instance" case InstanceStop: return "Stopping instance" case InstanceRestart: return "Restarting instance" case InstanceRebuild: return "Rebuilding instance" case CommandExec: return "Executing command" case SnapshotCreate: return "Snapshotting instance" case SnapshotRename: return "Renaming snapshot" case SnapshotRestore: return "Restoring snapshot" case SnapshotTransfer: return "Transferring snapshot" case SnapshotUpdate: return "Updating snapshot" case SnapshotDelete: return "Deleting snapshot" case ImageDownload: return "Downloading image" case ImageDelete: return "Deleting image" case ImageToken: return "Image download token" case ImageRefresh: return "Refreshing image" case VolumeCopy: return "Copying storage volume" case VolumeCreate: return "Creating storage volume" case VolumeMigrate: return "Migrating storage volume" case VolumeMove: return "Moving storage volume" case VolumeSnapshotCreate: return "Creating storage volume snapshot" case VolumeSnapshotDelete: return "Deleting storage volume snapshot" case VolumeSnapshotUpdate: return "Updating storage volume snapshot" case VolumeSnapshotRename: return "Renaming storage volume snapshot" case ProjectRename: return "Renaming project" case ImagesExpire: return "Cleaning up expired images" case ImagesPruneLeftover: return "Pruning leftover image files" case ImagesUpdate: return "Updating images" case ImagesSynchronize: return "Synchronizing images" case LogsExpire: return "Expiring log files" case InstanceTypesUpdate: return "Updating instance types" case BackupsExpire: return "Cleaning up expired backups" case SnapshotsExpire: return "Cleaning up expired instance snapshots" case CustomVolumeSnapshotsExpire: return "Cleaning up expired volume snapshots" case CustomVolumeBackupCreate: return "Creating custom volume backup" case CustomVolumeBackupRemove: return "Deleting custom volume backup" case CustomVolumeBackupRename: return "Renaming custom volume backup" case CustomVolumeBackupRestore: return "Restoring custom volume backup" case WarningsPruneResolved: return "Pruning resolved warnings" case ClusterMemberEvacuate: return "Evacuating cluster member" case ClusterMemberRestore: return "Restoring cluster member" case RemoveOrphanedOperations: return "Remove orphaned operations" case RenewServerCertificate: return "Renewing server certificate" case RemoveExpiredTokens: return "Remove expired tokens" case ClusterHeal: return "Healing cluster" case BucketBackupCreate: return "Creating bucket backup" case BucketBackupRemove: return "Deleting bucket backup" case BucketBackupRename: return "Renaming bucket backup" case BucketBackupRestore: return "Restoring bucket backup" default: return "Executing operation" } } // Permission returns the auth.ObjectType and auth.Entitlement required to cancel the operation. func (t Type) Permission() (auth.ObjectType, auth.Entitlement) { switch t { case BackupCreate: return auth.ObjectTypeInstance, auth.EntitlementCanManageBackups case BackupRename: return auth.ObjectTypeInstance, auth.EntitlementCanManageBackups case BackupRestore: return auth.ObjectTypeInstance, auth.EntitlementCanManageBackups case BackupRemove: return auth.ObjectTypeInstance, auth.EntitlementCanManageBackups case ConsoleShow: return auth.ObjectTypeInstance, auth.EntitlementCanAccessConsole case InstanceFreeze: return auth.ObjectTypeInstance, auth.EntitlementCanUpdateState case InstanceUnfreeze: return auth.ObjectTypeInstance, auth.EntitlementCanUpdateState case InstanceStart: return auth.ObjectTypeInstance, auth.EntitlementCanUpdateState case InstanceStop: return auth.ObjectTypeInstance, auth.EntitlementCanUpdateState case InstanceRestart: return auth.ObjectTypeInstance, auth.EntitlementCanUpdateState case CommandExec: return auth.ObjectTypeInstance, auth.EntitlementCanExec case SnapshotCreate: return auth.ObjectTypeInstance, auth.EntitlementCanManageSnapshots case SnapshotRename: return auth.ObjectTypeInstance, auth.EntitlementCanManageSnapshots case SnapshotTransfer: return auth.ObjectTypeInstance, auth.EntitlementCanManageSnapshots case SnapshotUpdate: return auth.ObjectTypeInstance, auth.EntitlementCanManageSnapshots case SnapshotDelete: return auth.ObjectTypeInstance, auth.EntitlementCanManageSnapshots case InstanceCreate: return auth.ObjectTypeProject, auth.EntitlementCanCreateInstances case InstanceUpdate: return auth.ObjectTypeInstance, auth.EntitlementCanEdit case InstanceRename: return auth.ObjectTypeInstance, auth.EntitlementCanEdit case InstanceMigrate: return auth.ObjectTypeInstance, auth.EntitlementCanEdit case InstanceLiveMigrate: return auth.ObjectTypeInstance, auth.EntitlementCanEdit case InstanceDelete: return auth.ObjectTypeInstance, auth.EntitlementCanEdit case InstanceRebuild: return auth.ObjectTypeInstance, auth.EntitlementCanEdit case SnapshotRestore: return auth.ObjectTypeInstance, auth.EntitlementCanEdit case ImageDownload: return auth.ObjectTypeImage, auth.EntitlementCanEdit case ImageDelete: return auth.ObjectTypeImage, auth.EntitlementCanEdit case ImageToken: return auth.ObjectTypeImage, auth.EntitlementCanEdit case ImageRefresh: return auth.ObjectTypeImage, auth.EntitlementCanEdit case ImagesUpdate: return auth.ObjectTypeImage, auth.EntitlementCanEdit case ImagesSynchronize: return auth.ObjectTypeImage, auth.EntitlementCanEdit case CustomVolumeSnapshotsExpire: return auth.ObjectTypeStorageVolume, auth.EntitlementCanEdit case CustomVolumeBackupCreate: return auth.ObjectTypeStorageVolume, auth.EntitlementCanManageBackups case CustomVolumeBackupRemove: return auth.ObjectTypeStorageVolume, auth.EntitlementCanManageBackups case CustomVolumeBackupRename: return auth.ObjectTypeStorageVolume, auth.EntitlementCanManageBackups case CustomVolumeBackupRestore: return auth.ObjectTypeStorageVolume, auth.EntitlementCanEdit case BucketBackupCreate: return auth.ObjectTypeStorageVolume, auth.EntitlementCanManageBackups case BucketBackupRemove: return auth.ObjectTypeStorageVolume, auth.EntitlementCanManageBackups case BucketBackupRename: return auth.ObjectTypeStorageVolume, auth.EntitlementCanManageBackups case BucketBackupRestore: return auth.ObjectTypeStorageVolume, auth.EntitlementCanEdit default: return "", "" } } incus-7.0.0/internal/server/db/patches.go000066400000000000000000000015101517523235500203310ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "database/sql" "github.com/lxc/incus/v7/internal/server/db/query" ) // GetAppliedPatches returns the names of all patches currently applied on this node. func (n *Node) GetAppliedPatches() ([]string, error) { var response []string err := query.Transaction(context.TODO(), n.db, func(ctx context.Context, tx *sql.Tx) error { var err error response, err = query.SelectStrings(ctx, tx, "SELECT name FROM patches") return err }) if err != nil { return []string{}, err } return response, nil } // MarkPatchAsApplied marks the patch with the given name as applied on this node. func (n *Node) MarkPatchAsApplied(patch string) error { stmt := `INSERT INTO patches (name, applied_at) VALUES (?, strftime("%s"))` _, err := n.db.Exec(stmt, patch) return err } incus-7.0.0/internal/server/db/profiles.go000066400000000000000000000125471517523235500205410ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "fmt" "maps" "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/shared/api" ) // GetProfileNames returns the names of all profiles in the given project. func (c *ClusterTx) GetProfileNames(ctx context.Context, project string) ([]string, error) { q := ` SELECT profiles.name FROM profiles JOIN projects ON projects.id = profiles.project_id WHERE projects.name = ? ` var result [][]any enabled, err := cluster.ProjectHasProfiles(context.Background(), c.tx, project) if err != nil { return nil, fmt.Errorf("Check if project has profiles: %w", err) } if !enabled { project = "default" } inargs := []any{project} var name string outfmt := []any{name} result, err = queryScan(ctx, c, q, inargs, outfmt) if err != nil { return nil, err } response := []string{} for _, r := range result { response = append(response, r[0].(string)) } return response, nil } // GetProfile returns the profile with the given name. func (c *ClusterTx) GetProfile(ctx context.Context, project, name string) (int64, *api.Profile, error) { profiles, err := cluster.GetProfilesIfEnabled(ctx, c.tx, project, []string{name}) if err != nil { return -1, nil, err } if len(profiles) != 1 { return -1, nil, fmt.Errorf("Expected one profile with name %q, got %d profiles", name, len(profiles)) } profile := profiles[0] id := int64(profile.ID) result, err := profile.ToAPI(ctx, c.tx, nil, nil) if err != nil { return -1, nil, err } return id, result, nil } // GetProfiles returns the profiles with the given names in the given project. func (c *ClusterTx) GetProfiles(ctx context.Context, projectName string, profileNames []string) ([]api.Profile, error) { profiles := make([]api.Profile, len(profileNames)) dbProfiles, err := cluster.GetProfilesIfEnabled(ctx, c.tx, projectName, profileNames) if err != nil { return nil, err } // Get all the profile configs. profileConfigs, err := cluster.GetAllProfileConfigs(ctx, c.Tx()) if err != nil { return nil, err } // Get all the profile devices. profileDevices, err := cluster.GetAllProfileDevices(ctx, c.Tx()) if err != nil { return nil, err } for i, profile := range dbProfiles { apiProfile, err := profile.ToAPI(ctx, c.tx, profileConfigs, profileDevices) if err != nil { return nil, err } profiles[i] = *apiProfile } return profiles, nil } // GetInstancesWithProfile gets the names of the instance associated with the // profile with the given name in the given project. func (c *ClusterTx) GetInstancesWithProfile(ctx context.Context, project, profile string) (map[string][]string, error) { q := `SELECT instances.name, projects.name FROM instances JOIN instances_profiles ON instances.id == instances_profiles.instance_id JOIN projects ON projects.id == instances.project_id WHERE instances_profiles.profile_id == (SELECT profiles.id FROM profiles JOIN projects ON projects.id == profiles.project_id WHERE profiles.name=? AND projects.name=?)` results := map[string][]string{} var output [][]any enabled, err := cluster.ProjectHasProfiles(context.Background(), c.tx, project) if err != nil { return nil, fmt.Errorf("Check if project has profiles: %w", err) } if !enabled { project = "default" } inargs := []any{profile, project} var name string outfmt := []any{name, name} output, err = queryScan(ctx, c, q, inargs, outfmt) if err != nil { return nil, err } for _, r := range output { if results[r[1].(string)] == nil { results[r[1].(string)] = []string{} } results[r[1].(string)] = append(results[r[1].(string)], r[0].(string)) } return results, nil } // RemoveUnreferencedProfiles removes unreferenced profiles. func (c *ClusterTx) RemoveUnreferencedProfiles(ctx context.Context) error { stmt := ` DELETE FROM profiles_config WHERE profile_id NOT IN (SELECT id FROM profiles); DELETE FROM profiles_devices WHERE profile_id NOT IN (SELECT id FROM profiles); DELETE FROM profiles_devices_config WHERE profile_device_id NOT IN (SELECT id FROM profiles_devices); ` _, err := c.tx.ExecContext(ctx, stmt) return err } // ExpandInstanceConfig expands the given instance config with the config // values of the given profiles. func ExpandInstanceConfig(config map[string]string, profiles []api.Profile) map[string]string { expandedConfig := map[string]string{} // Apply all the profiles profileConfigs := make([]map[string]string, len(profiles)) for i, profile := range profiles { profileConfigs[i] = profile.Config } for i := range profileConfigs { maps.Copy(expandedConfig, profileConfigs[i]) } // Stick the given config on top maps.Copy(expandedConfig, config) return expandedConfig } // ExpandInstanceDevices expands the given instance devices with the devices // defined in the given profiles. func ExpandInstanceDevices(devices deviceConfig.Devices, profiles []api.Profile) deviceConfig.Devices { expandedDevices := deviceConfig.Devices{} // Apply all the profiles profileDevices := make([]deviceConfig.Devices, len(profiles)) for i, profile := range profiles { profileDevices[i] = deviceConfig.NewDevices(profile.Devices) } for i := range profileDevices { maps.Copy(expandedDevices, profileDevices[i]) } // Stick the given devices on top maps.Copy(expandedDevices, devices) return expandedDevices } incus-7.0.0/internal/server/db/projects.go000066400000000000000000000010761517523235500205420ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "github.com/lxc/incus/v7/internal/server/db/cluster" ) // GetProject returns the project with the given key. func (db *DB) GetProject(ctx context.Context, projectName string) (*cluster.Project, error) { var err error var p *cluster.Project err = db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error { p, err = cluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return err } return nil }) if err != nil { return nil, err } return p, nil } incus-7.0.0/internal/server/db/query/000077500000000000000000000000001517523235500175235ustar00rootroot00000000000000incus-7.0.0/internal/server/db/query/config.go000066400000000000000000000047601517523235500213260ustar00rootroot00000000000000package query import ( "context" "database/sql" "fmt" "strings" ) // SelectConfig executes a query statement against a "config" table, which must // have 'key' and 'value' columns. By default this query returns all keys, but // additional WHERE filters can be specified. // // Returns a map of key names to their associated values. func SelectConfig(ctx context.Context, tx *sql.Tx, table string, where string, args ...any) (map[string]string, error) { query := fmt.Sprintf("SELECT key, value FROM %s", table) if where != "" { query += fmt.Sprintf(" WHERE %s", where) } rows, err := tx.QueryContext(ctx, query, args...) if err != nil { return nil, err } defer func() { _ = rows.Close() }() values := map[string]string{} for rows.Next() { var key string var value string err := rows.Scan(&key, &value) if err != nil { return nil, err } values[key] = value } err = rows.Err() if err != nil { return nil, err } return values, nil } // UpdateConfig updates the given keys in the given table. Config keys set to // empty values will be deleted. func UpdateConfig(tx *sql.Tx, table string, values map[string]string) error { changes := map[string]string{} deletes := []string{} for key, value := range values { if value == "" { deletes = append(deletes, key) continue } changes[key] = value } err := upsertConfig(tx, table, changes) if err != nil { return fmt.Errorf("updating values failed: %w", err) } err = deleteConfig(tx, table, deletes) if err != nil { return fmt.Errorf("deleting values failed: %w", err) } return nil } // Insert or updates the key/value rows of the given config table. func upsertConfig(tx *sql.Tx, table string, values map[string]string) error { if len(values) == 0 { return nil // Nothing to update } query := fmt.Sprintf("INSERT OR REPLACE INTO %s (key, value) VALUES", table) exprs := []string{} params := []any{} for key, value := range values { exprs = append(exprs, "(?, ?)") params = append(params, key) params = append(params, value) } query += strings.Join(exprs, ",") _, err := tx.Exec(query, params...) return err } // Delete the given key rows from the given config table. func deleteConfig(tx *sql.Tx, table string, keys []string) error { n := len(keys) if n == 0 { return nil // Nothing to delete. } query := fmt.Sprintf("DELETE FROM %s WHERE key IN %s", table, Params(n)) values := make([]any, n) for i, key := range keys { values[i] = key } _, err := tx.Exec(query, values...) return err } incus-7.0.0/internal/server/db/query/config_test.go000066400000000000000000000036251517523235500223640ustar00rootroot00000000000000package query_test import ( "context" "database/sql" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db/query" ) func TestSelectConfig(t *testing.T) { tx := newTxForConfig(t) values, err := query.SelectConfig(context.Background(), tx, "test", "") require.NoError(t, err) assert.Equal(t, map[string]string{"foo": "x", "bar": "zz"}, values) } func TestSelectConfig_WithFilters(t *testing.T) { tx := newTxForConfig(t) values, err := query.SelectConfig(context.Background(), tx, "test", "key=?", "bar") require.NoError(t, err) assert.Equal(t, map[string]string{"bar": "zz"}, values) } // New keys are added to the table. func TestUpdateConfig_NewKeys(t *testing.T) { tx := newTxForConfig(t) values := map[string]string{"foo": "y"} err := query.UpdateConfig(tx, "test", values) require.NoError(t, err) values, err = query.SelectConfig(context.Background(), tx, "test", "") require.NoError(t, err) assert.Equal(t, map[string]string{"foo": "y", "bar": "zz"}, values) } // Unset keys are deleted from the table. func TestDeleteConfig_Delete(t *testing.T) { tx := newTxForConfig(t) values := map[string]string{"foo": ""} err := query.UpdateConfig(tx, "test", values) require.NoError(t, err) values, err = query.SelectConfig(context.Background(), tx, "test", "") require.NoError(t, err) assert.Equal(t, map[string]string{"bar": "zz"}, values) } // Return a new transaction against an in-memory SQLite database with a single // test table populated with a few rows. func newTxForConfig(t *testing.T) *sql.Tx { db, err := sql.Open("sqlite3", ":memory:") assert.NoError(t, err) _, err = db.Exec("CREATE TABLE test (key TEXT NOT NULL, value TEXT)") assert.NoError(t, err) _, err = db.Exec("INSERT INTO test VALUES ('foo', 'x'), ('bar', 'zz')") assert.NoError(t, err) tx, err := db.Begin() assert.NoError(t, err) return tx } incus-7.0.0/internal/server/db/query/count.go000066400000000000000000000026611517523235500212070ustar00rootroot00000000000000package query import ( "context" "database/sql" "errors" "fmt" ) // Count returns the number of rows in the given table. func Count(ctx context.Context, tx *sql.Tx, table string, where string, args ...any) (int, error) { stmt := fmt.Sprintf("SELECT COUNT(*) FROM %s", table) if where != "" { stmt += fmt.Sprintf(" WHERE %s", where) } rows, err := tx.QueryContext(ctx, stmt, args...) if err != nil { return -1, err } defer func() { _ = rows.Close() }() // Ensure we read one and only one row. if !rows.Next() { return -1, errors.New("no rows returned") } var count int err = rows.Scan(&count) if err != nil { return -1, errors.New("failed to scan count column") } if rows.Next() { return -1, errors.New("more than one row returned") } err = rows.Err() if err != nil { return -1, err } return count, nil } // CountAll returns a map associating each table name in the database // with the total count of its rows. func CountAll(ctx context.Context, tx *sql.Tx) (map[string]int, error) { tables, err := SelectStrings(ctx, tx, "SELECT name FROM sqlite_master WHERE type = 'table'") if err != nil { return nil, fmt.Errorf("Failed to fetch table names: %w", err) } counts := map[string]int{} for _, table := range tables { count, err := Count(ctx, tx, table, "") if err != nil { return nil, fmt.Errorf("Failed to count rows of %s: %w", table, err) } counts[table] = count } return counts, nil } incus-7.0.0/internal/server/db/query/count_test.go000066400000000000000000000030541517523235500222430ustar00rootroot00000000000000package query_test import ( "context" "database/sql" "strconv" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db/query" ) // Count returns the current number of rows. func TestCount(t *testing.T) { cases := []struct { where string args []any count int }{ { "id=?", []any{999}, 0, }, { "id=?", []any{1}, 1, }, { "", []any{}, 2, }, } for _, c := range cases { t.Run(strconv.Itoa(c.count), func(t *testing.T) { tx := newTxForCount(t) count, err := query.Count(context.Background(), tx, "test", c.where, c.args...) require.NoError(t, err) assert.Equal(t, c.count, count) }) } } func TestCountAll(t *testing.T) { tx := newTxForCount(t) defer func() { _ = tx.Rollback() }() counts, err := query.CountAll(context.Background(), tx) require.NoError(t, err) assert.Equal(t, map[string]int{ "test": 2, "test2": 1, }, counts) } // Return a new transaction against an in-memory SQLite database with a single // test table and a few rows. func newTxForCount(t *testing.T) *sql.Tx { db, err := sql.Open("sqlite3", ":memory:") assert.NoError(t, err) _, err = db.Exec("CREATE TABLE test (id INTEGER)") assert.NoError(t, err) _, err = db.Exec("INSERT INTO test VALUES (1), (2)") assert.NoError(t, err) _, err = db.Exec("CREATE TABLE test2 (id INTEGER)") assert.NoError(t, err) _, err = db.Exec("INSERT INTO test2 VALUES (1)") assert.NoError(t, err) tx, err := db.Begin() assert.NoError(t, err) return tx } incus-7.0.0/internal/server/db/query/doc.go000066400000000000000000000001761517523235500206230ustar00rootroot00000000000000// Package query implements helpers around database/sql to execute various // kinds of very common SQL queries. package query incus-7.0.0/internal/server/db/query/dump.go000066400000000000000000000134331517523235500210230ustar00rootroot00000000000000package query import ( "context" "database/sql" "fmt" "strconv" "strings" "time" ) // DumpOptions represents different types of dump. type DumpOptions int // Dump response options. const ( DumpDefault DumpOptions = iota Schema Tables ) // Dump returns specific database information depending on the dump type. func Dump(ctx context.Context, tx *sql.Tx, dumpOption DumpOptions) (string, error) { switch dumpOption { case DumpDefault: return dumpSchema(ctx, tx, false) case Schema: return dumpSchema(ctx, tx, true) case Tables: return dumpTables(ctx, tx) } return "", fmt.Errorf("Failed to perform dump due to missing dump option") } // dumpTables returns a SQL text dump of all table's name, similar to // sqlite3's dump feature. func dumpTables(ctx context.Context, tx *sql.Tx) (string, error) { _, entityNames, err := getEntitiesSchemas(ctx, tx) if err != nil { return "", err } var builder strings.Builder for _, tableName := range entityNames { builder.WriteString(tableName + "\n") } return builder.String(), nil } // dumpSchema returns a SQL text dump of all rows across all tables, similar to // sqlite3's dump feature. func dumpSchema(ctx context.Context, tx *sql.Tx, schemaOnly bool) (string, error) { entitiesSchemas, entityNames, err := getEntitiesSchemas(ctx, tx) if err != nil { return "", err } // Begin dump string. var builder strings.Builder builder.WriteString("PRAGMA foreign_keys=OFF;\n") builder.WriteString("BEGIN TRANSACTION;\n") // For each table, write the schema and optionally write the data. for _, tableName := range entityNames { builder.WriteString(entitiesSchemas[tableName][1] + "\n") if !schemaOnly && entitiesSchemas[tableName][0] == "table" { tableData, err := getTableData(ctx, tx, tableName) if err != nil { return "", err } for _, stmt := range tableData { builder.WriteString(stmt + "\n") } } } // Sequences (unless the schemaOnly flag is true). if !schemaOnly { builder.WriteString("DELETE FROM sqlite_sequence;\n") tableData, err := getTableData(ctx, tx, "sqlite_sequence") if err != nil { return "", fmt.Errorf("Failed to dump table sqlite_sequence: %w", err) } for _, stmt := range tableData { builder.WriteString(stmt + "\n") } } // Commit. builder.WriteString("COMMIT;\n") return builder.String(), nil } // getEntitiesSchemas gets all the tables, their kind, and their schema, as well as a list of entity names in their default order from // the sqlite_master table. The returned map values are arrays of length 2 whose first element contains the entity type and the second // contains it's schema. func getEntitiesSchemas(ctx context.Context, tx *sql.Tx) (map[string][2]string, []string, error) { rows, err := tx.QueryContext(ctx, `SELECT name, type, sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY rowid`) if err != nil { return nil, nil, fmt.Errorf("Could not get table names and their schema: %w", err) } defer func() { _ = rows.Close() }() tablesSchemas := make(map[string][2]string) var names []string for rows.Next() { var name string var kind string var schema string err := rows.Scan(&name, &kind, &schema) if err != nil { return nil, nil, fmt.Errorf("Could not scan table name and schema: %w", err) } // This is based on logic from dump_callback in sqlite source for sqlite3_db_dump function. if strings.HasPrefix(schema, `CREATE TABLE "`) { schema = strings.Replace(schema, "CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1) } names = append(names, name) tablesSchemas[name] = [2]string{kind, schema + ";"} } return tablesSchemas, names, nil } // getTableData gets all the data for a single table, returning a string slice where each element is an insert statement // for the data. func getTableData(ctx context.Context, tx *sql.Tx, table string) ([]string, error) { var statements []string // Query all rows. rows, err := tx.QueryContext(ctx, fmt.Sprintf("SELECT * FROM %s ORDER BY rowid", table)) if err != nil { return nil, fmt.Errorf("Failed to fetch rows for table %q: %w", table, err) } defer func() { _ = rows.Close() }() // Get the column names. columns, err := rows.Columns() if err != nil { return nil, fmt.Errorf("Failed to get columns for table %q: %w", table, err) } // Generate an INSERT statement for each row. for i := 0; rows.Next(); i++ { raw := make([]any, len(columns)) // Raw column values row := make([]any, len(columns)) for i := range raw { row[i] = &raw[i] } err := rows.Scan(row...) if err != nil { return nil, fmt.Errorf("Failed to scan row %d in table %q: %w", i, table, err) } values := make([]string, len(columns)) for j, v := range raw { switch v := v.(type) { case int64: values[j] = strconv.FormatInt(v, 10) case string: // This is based on logic from dump_callback in sqlite source for sqlite3_db_dump function. v = fmt.Sprintf("'%s'", strings.ReplaceAll(v, "'", "''")) if strings.Contains(v, "\r") { v = "replace(" + strings.ReplaceAll(v, "\r", "\\r") + ",'\\r',char(13))" } if strings.Contains(v, "\n") { v = "replace(" + strings.ReplaceAll(v, "\n", "\\n") + ",'\\n',char(10))" } values[j] = v case []byte: values[j] = fmt.Sprintf("'%s'", string(v)) case time.Time: // Try and match the sqlite3 .dump output format. format := "2006-01-02 15:04:05" if v.Nanosecond() > 0 { format = format + ".000000000" } format = format + "-07:00" values[j] = "'" + v.Format(format) + "'" default: if v != nil { return nil, fmt.Errorf("Bad type in column %q of row %d in table %q", columns[j], i, table) } values[j] = "NULL" } } statement := fmt.Sprintf("INSERT INTO %s VALUES(%s);", table, strings.Join(values, ",")) statements = append(statements, statement) } return statements, nil } incus-7.0.0/internal/server/db/query/dump_export_test.go000066400000000000000000000001431517523235500234550ustar00rootroot00000000000000package query var ( GetTableData = getTableData GetEntitiesSchemas = getEntitiesSchemas ) incus-7.0.0/internal/server/db/query/dump_test.go000066400000000000000000000327611517523235500220670ustar00rootroot00000000000000package query_test import ( "context" "database/sql" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db/query" ) func TestDumpTables(t *testing.T) { tx := newTxForDump(t, "local") dumpOption := query.DumpOptions(2) dump, err := query.Dump(context.Background(), tx, dumpOption) require.NoError(t, err) assert.Equal(t, `schema config patches raft_nodes config_key_idx `, dump) } func TestDumpSchema(t *testing.T) { tx := newTxForDump(t, "local") dumpOption := query.DumpOptions(1) dump, err := query.Dump(context.Background(), tx, dumpOption) require.NoError(t, err) assert.Equal(t, `PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE schema ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, version INTEGER NOT NULL, updated_at DATETIME NOT NULL, UNIQUE (version) ); CREATE TABLE config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (key) ); CREATE TABLE patches ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, applied_at DATETIME NOT NULL, UNIQUE (name) ); CREATE TABLE raft_nodes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, address TEXT NOT NULL, UNIQUE (address) ); CREATE INDEX config_key_idx ON config (key); COMMIT; `, dump) } func TestDump(t *testing.T) { tx := newTxForDump(t, "local") dumpOption := query.DumpOptions(0) dump, err := query.Dump(context.Background(), tx, dumpOption) require.NoError(t, err) assert.Equal(t, `PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE schema ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, version INTEGER NOT NULL, updated_at DATETIME NOT NULL, UNIQUE (version) ); INSERT INTO schema VALUES(1,37,'2018-04-17 06:26:06+00:00'); CREATE TABLE config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (key) ); CREATE TABLE patches ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, applied_at DATETIME NOT NULL, UNIQUE (name) ); INSERT INTO patches VALUES(1,'invalid_profile_names','2018-04-17 06:26:06+00:00'); INSERT INTO patches VALUES(2,'leftover_profile_config','2018-04-17 06:26:06+00:00'); CREATE TABLE raft_nodes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, address TEXT NOT NULL, UNIQUE (address) ); CREATE INDEX config_key_idx ON config (key); DELETE FROM sqlite_sequence; INSERT INTO sqlite_sequence VALUES('schema',1); INSERT INTO sqlite_sequence VALUES('patches',2); COMMIT; `, dump) } func TestDumpTablePatches(t *testing.T) { tx := newTxForDump(t, "local") dump, _, err := query.GetEntitiesSchemas(context.Background(), tx) require.NoError(t, err) assert.Equal(t, `CREATE TABLE patches ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, applied_at DATETIME NOT NULL, UNIQUE (name) );`, dump["patches"][1]) data, err := query.GetTableData(context.Background(), tx, "patches") require.NoError(t, err) assert.ElementsMatch(t, data, []string{ "INSERT INTO patches VALUES(1,'invalid_profile_names','2018-04-17 06:26:06+00:00');", "INSERT INTO patches VALUES(2,'leftover_profile_config','2018-04-17 06:26:06+00:00');", }) } func TestDumpTableConfig(t *testing.T) { tx := newTxForDump(t, "local") dump, _, err := query.GetEntitiesSchemas(context.Background(), tx) require.NoError(t, err) assert.Equal(t, `CREATE TABLE config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (key) );`, dump["config"][1]) } func TestDumpTableStoragePoolsConfig(t *testing.T) { tx := newTxForDump(t, "global") dump, _, err := query.GetEntitiesSchemas(context.Background(), tx) require.NoError(t, err) assert.Equal(t, `CREATE TABLE storage_pools_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER, key TEXT NOT NULL, value TEXT, UNIQUE (storage_pool_id, node_id, key), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE );`, dump["storage_pools_config"][1]) data, err := query.GetTableData(context.Background(), tx, "storage_pools_config") require.NoError(t, err) assert.Equal(t, data, []string{"INSERT INTO storage_pools_config VALUES(1,1,NULL,'k','v');"}) } // Return a new transaction against an in-memory SQLite database populated with // a few tables and data, according to the given schema. func newTxForDump(t *testing.T, schema string) *sql.Tx { db, err := sql.Open("sqlite3", ":memory:") require.NoError(t, err) _, err = db.Exec(`CREATE TABLE schema ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, version INTEGER NOT NULL, updated_at DATETIME NOT NULL, UNIQUE (version) );`) require.NoError(t, err) _, err = db.Exec(schemas[schema]) require.NoError(t, err) for _, stmt := range data[schema] { _, err = db.Exec(stmt) require.NoError(t, err) } tx, err := db.Begin() require.NoError(t, err) return tx } var schemas = map[string]string{ "local": ` CREATE TABLE config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, key VARCHAR(255) NOT NULL, value TEXT, UNIQUE (key) ); CREATE TABLE patches ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name VARCHAR(255) NOT NULL, applied_at DATETIME NOT NULL, UNIQUE (name) ); CREATE TABLE raft_nodes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, address TEXT NOT NULL, UNIQUE (address) ); CREATE INDEX config_key_idx ON config (key); `, "global": ` CREATE TABLE certificates ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, fingerprint TEXT NOT NULL, type INTEGER NOT NULL, name TEXT NOT NULL, certificate TEXT NOT NULL, UNIQUE (fingerprint) ); CREATE TABLE config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (key) ); CREATE TABLE containers ( id INTEGER primary key AUTOINCREMENT NOT NULL, node_id INTEGER NOT NULL, name TEXT NOT NULL, architecture INTEGER NOT NULL, type INTEGER NOT NULL, ephemeral INTEGER NOT NULL DEFAULT 0, creation_date DATETIME NOT NULL DEFAULT 0, stateful INTEGER NOT NULL DEFAULT 0, last_use_date DATETIME, description TEXT, UNIQUE (name), FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); CREATE TABLE containers_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE, UNIQUE (container_id, key) ); CREATE TABLE containers_devices ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, name TEXT NOT NULL, type INTEGER NOT NULL default 0, FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE, UNIQUE (container_id, name) ); CREATE TABLE containers_devices_config ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (container_device_id) REFERENCES containers_devices (id) ON DELETE CASCADE, UNIQUE (container_device_id, key) ); CREATE TABLE containers_profiles ( id INTEGER primary key AUTOINCREMENT NOT NULL, container_id INTEGER NOT NULL, profile_id INTEGER NOT NULL, apply_order INTEGER NOT NULL default 0, UNIQUE (container_id, profile_id), FOREIGN KEY (container_id) REFERENCES containers(id) ON DELETE CASCADE, FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE ); CREATE TABLE images ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, fingerprint TEXT NOT NULL, filename TEXT NOT NULL, size INTEGER NOT NULL, public INTEGER NOT NULL DEFAULT 0, architecture INTEGER NOT NULL, creation_date DATETIME, expiry_date DATETIME, upload_date DATETIME NOT NULL, cached INTEGER NOT NULL DEFAULT 0, last_use_date DATETIME, auto_update INTEGER NOT NULL DEFAULT 0, UNIQUE (fingerprint) ); CREATE TABLE images_aliases ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, image_id INTEGER NOT NULL, description TEXT, FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE, UNIQUE (name) ); CREATE TABLE images_nodes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, node_id INTEGER NOT NULL, UNIQUE (image_id, node_id), FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); CREATE TABLE images_properties ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, type INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE ); CREATE TABLE images_source ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, image_id INTEGER NOT NULL, server TEXT NOT NULL, protocol INTEGER NOT NULL, certificate TEXT NOT NULL, alias TEXT NOT NULL, FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE ); CREATE TABLE networks ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT, state INTEGER NOT NULL DEFAULT 0, UNIQUE (name) ); CREATE TABLE networks_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER, key TEXT NOT NULL, value TEXT, UNIQUE (network_id, node_id, key), FOREIGN KEY (network_id) REFERENCES networks (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); CREATE TABLE networks_nodes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, network_id INTEGER NOT NULL, node_id INTEGER NOT NULL, UNIQUE (network_id, node_id), FOREIGN KEY (network_id) REFERENCES networks (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); CREATE TABLE nodes ( id INTEGER PRIMARY KEY, name TEXT NOT NULL, description TEXT DEFAULT '', address TEXT NOT NULL, schema INTEGER NOT NULL, api_extensions INTEGER NOT NULL, heartbeat DATETIME DEFAULT CURRENT_TIMESTAMP, pending INTEGER NOT NULL DEFAULT 0, UNIQUE (name), UNIQUE (address) ); CREATE TABLE operations ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, uuid TEXT NOT NULL, node_id TEXT NOT NULL, UNIQUE (uuid), FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); CREATE TABLE profiles ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, description TEXT, UNIQUE (name) ); CREATE TABLE profiles_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (profile_id, key), FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE ); CREATE TABLE profiles_devices ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_id INTEGER NOT NULL, name TEXT NOT NULL, type INTEGER NOT NULL default 0, UNIQUE (profile_id, name), FOREIGN KEY (profile_id) REFERENCES profiles (id) ON DELETE CASCADE ); CREATE TABLE profiles_devices_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, profile_device_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (profile_device_id, key), FOREIGN KEY (profile_device_id) REFERENCES profiles_devices (id) ON DELETE CASCADE ); CREATE TABLE storage_pools ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, driver TEXT NOT NULL, description TEXT, state INTEGER NOT NULL DEFAULT 0, UNIQUE (name) ); CREATE TABLE storage_pools_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER, key TEXT NOT NULL, value TEXT, UNIQUE (storage_pool_id, node_id, key), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); CREATE TABLE storage_pools_nodes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER NOT NULL, UNIQUE (storage_pool_id, node_id), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); CREATE TABLE storage_volumes ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, storage_pool_id INTEGER NOT NULL, node_id INTEGER NOT NULL, type INTEGER NOT NULL, description TEXT, UNIQUE (storage_pool_id, node_id, name, type), FOREIGN KEY (storage_pool_id) REFERENCES storage_pools (id) ON DELETE CASCADE, FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE ); CREATE TABLE storage_volumes_config ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, storage_volume_id INTEGER NOT NULL, key TEXT NOT NULL, value TEXT, UNIQUE (storage_volume_id, key), FOREIGN KEY (storage_volume_id) REFERENCES storage_volumes (id) ON DELETE CASCADE ); `, } var data = map[string][]string{ "local": { "INSERT INTO schema VALUES(1,37,1523946366)", "INSERT INTO patches VALUES(1,'invalid_profile_names',1523946366)", "INSERT INTO patches VALUES(2,'leftover_profile_config',1523946366)", }, "global": { "INSERT INTO storage_pools VALUES(1,'p1','dir','',0)", "INSERT INTO storage_pools_config VALUES(1,1,NULL,'k','v')", }, } incus-7.0.0/internal/server/db/query/expr.go000066400000000000000000000006531517523235500210340ustar00rootroot00000000000000// Various utilities to generate/parse/manipulate SQL expressions. package query import ( "fmt" "strings" ) // Params returns a parameters expression with the given number of '?' // placeholders. E.g. Params(2) -> "(?, ?)". Useful for IN and VALUES // expressions. func Params(n int) string { tokens := make([]string, n) for i := range n { tokens[i] = "?" } return fmt.Sprintf("(%s)", strings.Join(tokens, ", ")) } incus-7.0.0/internal/server/db/query/marshal.go000066400000000000000000000012141517523235500214770ustar00rootroot00000000000000package query import ( "errors" ) type Marshaler interface { MarshalDB() (string, error) } type Unmarshaler interface { UnmarshalDB(string) error } func Marshal(v any) (string, error) { marshaller, ok := v.(Marshaler) if !ok { return "", errors.New("Cannot marshal data, type does not implement DBMarshaler") } return marshaller.MarshalDB() } func Unmarshal(data string, v any) error { if v == nil { return errors.New("Cannot unmarshal data into nil value") } unmarshaler, ok := v.(Unmarshaler) if !ok { return errors.New("Cannot marshal data, type does not implement DBUnmarshaler") } return unmarshaler.UnmarshalDB(data) } incus-7.0.0/internal/server/db/query/objects.go000066400000000000000000000054151517523235500215100ustar00rootroot00000000000000package query import ( "context" "database/sql" "errors" "fmt" "strings" ) // SelectObjects executes a statement which must yield rows with a specific // columns schema. It invokes the given Dest hook for each yielded row. func SelectObjects(ctx context.Context, stmt *sql.Stmt, rowFunc Dest, args ...any) error { rows, err := stmt.QueryContext(ctx, args...) if err != nil { return err } defer func() { _ = rows.Close() }() for rows.Next() { err = rowFunc(rows.Scan) if err != nil { return err } } return rows.Err() } // Scan runs a query with inArgs and provides the rowFunc with the scan function for each row. // It handles closing the rows and errors from the result set. func Scan(ctx context.Context, tx *sql.Tx, sql string, rowFunc Dest, inArgs ...any) error { rows, err := tx.QueryContext(ctx, sql, inArgs...) if err != nil { return err } defer func() { _ = rows.Close() }() for rows.Next() { err = rowFunc(rows.Scan) if err != nil { return err } } return rows.Err() } // Dest is a function that is expected to return the objects to pass to the // 'dest' argument of sql.Rows.Scan(). It is invoked by SelectObjects once per // yielded row, and it will be passed the index of the row being scanned. type Dest func(scan func(dest ...any) error) error // UpsertObject inserts or replaces a new row with the given column values, to // the given table using columns order. For example: // // UpsertObject(tx, "cars", []string{"id", "brand"}, []any{1, "ferrari"}) // // The number of elements in 'columns' must match the one in 'values'. func UpsertObject(tx *sql.Tx, table string, columns []string, values []any) (int64, error) { n := len(columns) if n == 0 { return -1, errors.New("columns length is zero") } if n != len(values) { return -1, errors.New("columns length does not match values length") } stmt := fmt.Sprintf( "INSERT OR REPLACE INTO %s (%s) VALUES %s", table, strings.Join(columns, ", "), Params(n)) result, err := tx.Exec(stmt, values...) if err != nil { return -1, fmt.Errorf("insert or replaced row: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("get last inserted ID: %w", err) } return id, nil } // DeleteObject removes the row identified by the given ID. The given table // must have a primary key column called 'id'. // // It returns a flag indicating if a matching row was actually found and // deleted or not. func DeleteObject(tx *sql.Tx, table string, id int64) (bool, error) { stmt := fmt.Sprintf("DELETE FROM %s WHERE id=?", table) result, err := tx.Exec(stmt, id) if err != nil { return false, err } n, err := result.RowsAffected() if err != nil { return false, err } if n > 1 { return true, errors.New("more than one row was deleted") } return n == 1, nil } incus-7.0.0/internal/server/db/query/objects_test.go000066400000000000000000000111201517523235500225350ustar00rootroot00000000000000package query_test import ( "context" "database/sql" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db/query" ) // Exercise possible failure modes. func TestSelectObjects_Error(t *testing.T) { cases := []struct { dest query.Dest query string error string }{ { func(scan func(dest ...any) error) error { var row any return scan(row) }, "SELECT id, name FROM test", "sql: expected 2 destination arguments in Scan, not 1", }, } for _, c := range cases { t.Run(c.query, func(t *testing.T) { tx := newTxForObjects(t) stmt, err := tx.Prepare(c.query) require.NoError(t, err) err = query.SelectObjects(context.TODO(), stmt, c.dest) assert.EqualError(t, err, c.error) }) } } // Scan rows yielded by the query. func TestSelectObjects(t *testing.T) { tx := newTxForObjects(t) objects := make([]struct { ID int Name string }, 1) object := objects[0] count := 0 dest := func(scan func(dest ...any) error) error { require.Equal(t, 0, count, "expected at most one row to be yielded") count++ return scan(&object.ID, &object.Name) } stmt, err := tx.Prepare("SELECT id, name FROM test WHERE name=?") require.NoError(t, err) err = query.SelectObjects(context.TODO(), stmt, dest, "bar") require.NoError(t, err) assert.Equal(t, 1, object.ID) assert.Equal(t, "bar", object.Name) } // Exercise possible failure modes. func TestUpsertObject_Error(t *testing.T) { cases := []struct { columns []string values []any error string }{ { []string{}, []any{}, "columns length is zero", }, { []string{"id"}, []any{2, "egg"}, "columns length does not match values length", }, } for _, c := range cases { t.Run(c.error, func(t *testing.T) { tx := newTxForObjects(t) id, err := query.UpsertObject(tx, "foo", c.columns, c.values) assert.Equal(t, int64(-1), id) assert.EqualError(t, err, c.error) }) } } // Insert a new row. func TestUpsertObject_Insert(t *testing.T) { tx := newTxForObjects(t) id, err := query.UpsertObject(tx, "test", []string{"name"}, []any{"egg"}) require.NoError(t, err) assert.Equal(t, int64(2), id) objects := make([]struct { ID int Name string }, 1) object := objects[0] count := 0 dest := func(scan func(dest ...any) error) error { require.Equal(t, 0, count, "expected at most one row to be yielded") count++ return scan(&object.ID, &object.Name) } sql := "SELECT id, name FROM test WHERE name=?" err = query.Scan(context.TODO(), tx, sql, dest, "egg") require.NoError(t, err) assert.Equal(t, 2, object.ID) assert.Equal(t, "egg", object.Name) } // Update an existing row. func TestUpsertObject_Update(t *testing.T) { tx := newTxForObjects(t) id, err := query.UpsertObject(tx, "test", []string{"id", "name"}, []any{1, "egg"}) require.NoError(t, err) assert.Equal(t, int64(1), id) objects := make([]struct { ID int Name string }, 1) object := objects[0] count := 0 dest := func(scan func(dest ...any) error) error { require.Equal(t, 0, count, "expected at most one row to be yielded") count++ return scan(&object.ID, &object.Name) } sql := "SELECT id, name FROM test WHERE name=?" require.NoError(t, err) err = query.Scan(context.TODO(), tx, sql, dest, "egg") require.NoError(t, err) assert.Equal(t, 1, object.ID) assert.Equal(t, "egg", object.Name) } // Exercise possible failure modes. func TestDeleteObject_Error(t *testing.T) { tx := newTxForObjects(t) deleted, err := query.DeleteObject(tx, "foo", 1) assert.False(t, deleted) assert.EqualError(t, err, "no such table: foo") } // If an row was actually deleted, the returned flag is true. func TestDeleteObject_Deleted(t *testing.T) { tx := newTxForObjects(t) deleted, err := query.DeleteObject(tx, "test", 1) assert.True(t, deleted) assert.NoError(t, err) } // If no row was actually deleted, the returned flag is false. func TestDeleteObject_NotDeleted(t *testing.T) { tx := newTxForObjects(t) deleted, err := query.DeleteObject(tx, "test", 1000) assert.False(t, deleted) assert.NoError(t, err) } // Return a new transaction against an in-memory SQLite database with a single // test table populated with a few rows for testing object-related queries. func newTxForObjects(t *testing.T) *sql.Tx { db, err := sql.Open("sqlite3", ":memory:") assert.NoError(t, err) _, err = db.Exec("CREATE TABLE test (id INTEGER PRIMARY KEY, name TEXT)") assert.NoError(t, err) _, err = db.Exec("INSERT INTO test VALUES (0, 'foo'), (1, 'bar')") assert.NoError(t, err) tx, err := db.Begin() assert.NoError(t, err) return tx } incus-7.0.0/internal/server/db/query/retry.go000066400000000000000000000045421517523235500212240ustar00rootroot00000000000000package query import ( "context" "database/sql" "errors" "math" "math/rand/v2" "net/http" "strings" "time" "github.com/cowsql/go-cowsql/driver" "github.com/mattn/go-sqlite3" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) const maxRetries = 250 // Retry wraps a function that interacts with the database, and retries it in // case a transient error is hit. // // This should by typically used to wrap transactions. func Retry(ctx context.Context, f func(ctx context.Context) error) error { var err error for i := range maxRetries { err = f(ctx) if err == nil { // The function succeeded, we're done here. break } if errors.Is(err, context.Canceled) { // The function was canceled, don't retry. break } // No point in re-trying or logging a no-row or not found error. if errors.Is(err, sql.ErrNoRows) || api.StatusErrorCheck(err, http.StatusNotFound) { break } // Process actual errors. if !IsRetriableError(err) { logger.Debug("Database error", logger.Ctx{"err": err}) break } if i == maxRetries { logger.Warn("Database error, giving up", logger.Ctx{"attempt": i, "err": err}) break } logger.Debug("Database error, retrying", logger.Ctx{"attempt": i, "err": err}) time.Sleep(jitterDeviation(0.8, 100*time.Millisecond)) } return err } func jitterDeviation(factor float64, duration time.Duration) time.Duration { floor := int64(math.Floor(float64(duration) * (1 - factor))) ceil := int64(math.Ceil(float64(duration) * (1 + factor))) return time.Duration(rand.Int64N(ceil-floor) + floor) } // IsRetriableError returns true if the given error might be transient and the // interaction can be safely retried. func IsRetriableError(err error) bool { var dErr *driver.Error if errors.As(err, &dErr) && dErr.Code == driver.ErrBusy { return true } if errors.Is(err, sqlite3.ErrLocked) || errors.Is(err, sqlite3.ErrBusy) { return true } // Unwrap errors one at a time. for ; err != nil; err = errors.Unwrap(err) { if strings.Contains(err.Error(), "database is locked") { return true } if strings.Contains(err.Error(), "cannot start a transaction within a transaction") { return true } if strings.Contains(err.Error(), "bad connection") { return true } if strings.Contains(err.Error(), "checkpoint in progress") { return true } } return false } incus-7.0.0/internal/server/db/query/slices.go000066400000000000000000000045741517523235500213460ustar00rootroot00000000000000package query import ( "context" "database/sql" "fmt" "strings" ) // SelectStrings executes a statement which must yield rows with a single string // column. It returns the list of column values. func SelectStrings(ctx context.Context, tx *sql.Tx, query string, args ...any) ([]string, error) { values := []string{} scan := func(rows *sql.Rows) error { var value string err := rows.Scan(&value) if err != nil { return err } values = append(values, value) return nil } err := scanSingleColumn(ctx, tx, query, args, "TEXT", scan) if err != nil { return nil, err } return values, nil } // SelectIntegers executes a statement which must yield rows with a single integer // column. It returns the list of column values. func SelectIntegers(ctx context.Context, tx *sql.Tx, query string, args ...any) ([]int, error) { values := []int{} scan := func(rows *sql.Rows) error { var value int err := rows.Scan(&value) if err != nil { return err } values = append(values, value) return nil } err := scanSingleColumn(ctx, tx, query, args, "INTEGER", scan) if err != nil { return nil, err } return values, nil } // InsertStrings inserts a new row for each of the given strings, using the // given insert statement template, which must define exactly one insertion // column and one substitution placeholder for the values. For example: // InsertStrings(tx, "INSERT INTO foo(name) VALUES %s", []string{"bar"}). func InsertStrings(tx *sql.Tx, stmt string, values []string) error { n := len(values) if n == 0 { return nil } params := make([]string, n) args := make([]any, n) for i, value := range values { params[i] = "(?)" args[i] = value } stmt = fmt.Sprintf(stmt, strings.Join(params, ", ")) _, err := tx.Exec(stmt, args...) return err } // Execute the given query and ensure that it yields rows with a single column // of the given database type. For every row yielded, execute the given // scanner. func scanSingleColumn(ctx context.Context, tx *sql.Tx, query string, args []any, typeName string, scan scanFunc) error { rows, err := tx.QueryContext(ctx, query, args...) if err != nil { return err } defer func() { _ = rows.Close() }() for rows.Next() { err := scan(rows) if err != nil { return err } } err = rows.Err() if err != nil { return err } return nil } // Function to scan a single row. type scanFunc func(*sql.Rows) error incus-7.0.0/internal/server/db/query/slices_test.go000066400000000000000000000052131517523235500223740ustar00rootroot00000000000000package query_test import ( "context" "database/sql" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db/query" ) // Exercise possible failure modes. func TestStrings_Error(t *testing.T) { for _, c := range testStringsErrorCases { t.Run(c.query, func(t *testing.T) { tx := newTxForSlices(t) values, err := query.SelectStrings(context.Background(), tx, c.query) assert.EqualError(t, err, c.error) assert.Nil(t, values) }) } } var testStringsErrorCases = []struct { query string error string }{ {"garbage", "near \"garbage\": syntax error"}, {"SELECT id, name FROM test", "sql: expected 2 destination arguments in Scan, not 1"}, } // All values yield by the query are returned. func TestStrings(t *testing.T) { tx := newTxForSlices(t) values, err := query.SelectStrings(context.Background(), tx, "SELECT name FROM test ORDER BY name") assert.Nil(t, err) assert.Equal(t, []string{"bar", "foo"}, values) } // Exercise possible failure modes. func TestIntegers_Error(t *testing.T) { for _, c := range testIntegersErrorCases { t.Run(c.query, func(t *testing.T) { tx := newTxForSlices(t) values, err := query.SelectIntegers(context.Background(), tx, c.query) assert.EqualError(t, err, c.error) assert.Nil(t, values) }) } } var testIntegersErrorCases = []struct { query string error string }{ {"garbage", "near \"garbage\": syntax error"}, {"SELECT id, name FROM test", "sql: expected 2 destination arguments in Scan, not 1"}, } // All values yield by the query are returned. func TestIntegers(t *testing.T) { tx := newTxForSlices(t) values, err := query.SelectIntegers(context.Background(), tx, "SELECT id FROM test ORDER BY id") assert.Nil(t, err) assert.Equal(t, []int{0, 1}, values) } // Insert new rows in bulk. func TestInsertStrings(t *testing.T) { tx := newTxForSlices(t) err := query.InsertStrings(tx, "INSERT INTO test(name) VALUES %s", []string{"xx", "yy"}) require.NoError(t, err) values, err := query.SelectStrings(context.Background(), tx, "SELECT name FROM test ORDER BY name DESC LIMIT 2") require.NoError(t, err) assert.Equal(t, values, []string{"yy", "xx"}) } // Return a new transaction against an in-memory SQLite database with a single // test table populated with a few rows. func newTxForSlices(t *testing.T) *sql.Tx { db, err := sql.Open("sqlite3", ":memory:") assert.NoError(t, err) _, err = db.Exec("CREATE TABLE test (id INTEGER, name TEXT)") assert.NoError(t, err) _, err = db.Exec("INSERT INTO test VALUES (0, 'foo'), (1, 'bar')") assert.NoError(t, err) tx, err := db.Begin() assert.NoError(t, err) return tx } incus-7.0.0/internal/server/db/query/transaction.go000066400000000000000000000025221517523235500224000ustar00rootroot00000000000000package query import ( "context" "database/sql" "errors" "fmt" "strings" "time" "github.com/lxc/incus/v7/shared/logger" ) // Transaction executes the given function within a database transaction with a 30s context timeout. func Transaction(ctx context.Context, db *sql.DB, f func(context.Context, *sql.Tx) error) error { ctx, cancel := context.WithTimeout(ctx, time.Second*30) defer cancel() tx, err := db.BeginTx(ctx, nil) if err != nil { // If there is a leftover transaction let's try to rollback, // we'll then retry again. if strings.Contains(err.Error(), "cannot start a transaction within a transaction") { _, _ = db.Exec("ROLLBACK") } return fmt.Errorf("Failed to begin transaction: %w", err) } err = f(ctx, tx) if err != nil { return rollback(tx, err) } err = tx.Commit() if errors.Is(err, sql.ErrTxDone) { err = nil // Ignore duplicate commits/rollbacks } return err } // Rollback a transaction after the given error occurred. If the rollback // succeeds the given error is returned, otherwise a new error that wraps it // gets generated and returned. func rollback(tx *sql.Tx, reason error) error { err := Retry(context.TODO(), func(_ context.Context) error { return tx.Rollback() }) if err != nil { logger.Warnf("Failed to rollback transaction after error (%v): %v", reason, err) } return reason } incus-7.0.0/internal/server/db/query/transaction_test.go000066400000000000000000000025251517523235500234420ustar00rootroot00000000000000package query_test import ( "context" "database/sql" "errors" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db/query" ) // Any error happening when beginning the transaction will be propagated. func TestTransaction_BeginError(t *testing.T) { db := newDB(t) err := db.Close() require.NoError(t, err) err = query.Transaction(context.TODO(), db, func(ctx context.Context, tx *sql.Tx) error { return nil }) assert.NotNil(t, err) assert.Contains(t, err.Error(), "Failed to begin transaction") } // Any error happening when in the transaction function will cause a rollback. func TestTransaction_FunctionError(t *testing.T) { db := newDB(t) err := query.Transaction(context.TODO(), db, func(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec("CREATE TABLE test (id INTEGER)") assert.NoError(t, err) return errors.New("boom") }) assert.EqualError(t, err, "boom") tx, err := db.Begin() assert.NoError(t, err) tables, err := query.SelectStrings(context.Background(), tx, "SELECT name FROM sqlite_master WHERE type = 'table'") assert.NoError(t, err) assert.NotContains(t, tables, "test") } // Return a new in-memory SQLite database. func newDB(t *testing.T) *sql.DB { db, err := sql.Open("sqlite3", ":memory:") assert.NoError(t, err) return db } incus-7.0.0/internal/server/db/raft.go000066400000000000000000000103041517523235500176370ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "errors" "fmt" "net/http" "github.com/cowsql/go-cowsql/client" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/shared/api" ) // RaftNode holds information about a single node in the dqlite raft cluster. // // This is just a convenience alias for the equivalent data structure in the // dqlite client package. type RaftNode struct { client.NodeInfo Name string } // RaftRole captures the role of dqlite/raft node. type RaftRole = client.NodeRole // RaftNode roles. const ( RaftVoter = client.Voter RaftStandBy = client.StandBy RaftSpare = client.Spare ) // GetRaftNodes returns information about all cluster members that are members of the // dqlite Raft cluster (possibly including the local member). If this server // is not running in clustered mode, an empty list is returned. func (n *NodeTx) GetRaftNodes(ctx context.Context) ([]RaftNode, error) { nodes := []RaftNode{} sql := "SELECT id, address, role, name FROM raft_nodes ORDER BY id" err := query.Scan(ctx, n.tx, sql, func(scan func(dest ...any) error) error { node := RaftNode{} err := scan(&node.ID, &node.Address, &node.Role, &node.Name) if err != nil { return err } nodes = append(nodes, node) return nil }) if err != nil { return nil, fmt.Errorf("Failed to fetch raft nodes: %w", err) } return nodes, nil } // GetRaftNodeAddresses returns the addresses of all servers that are members of // the dqlite Raft cluster (possibly including the local member). If this server // is not running in clustered mode, an empty list is returned. func (n *NodeTx) GetRaftNodeAddresses(ctx context.Context) ([]string, error) { return query.SelectStrings(ctx, n.tx, "SELECT address FROM raft_nodes") } // GetRaftNodeAddress returns the address of the raft node with the given ID, // if any matching row exists. func (n *NodeTx) GetRaftNodeAddress(ctx context.Context, id int64) (string, error) { stmt := "SELECT address FROM raft_nodes WHERE id=?" addresses, err := query.SelectStrings(ctx, n.tx, stmt, id) if err != nil { return "", err } switch len(addresses) { case 0: return "", api.StatusErrorf(http.StatusNotFound, "Raft member not found") case 1: return addresses[0], nil default: // This should never happen since we have a UNIQUE constraint // on the raft_nodes.id column. return "", errors.New("more than one match found") } } // CreateFirstRaftNode adds a the first node of the cluster. It ensures that the // database ID is 1, to match the server ID of the first raft log entry. // // This method is supposed to be called when there are no rows in raft_nodes, // and it will replace whatever existing row has ID 1. func (n *NodeTx) CreateFirstRaftNode(address string, name string) error { columns := []string{"id", "address", "name"} values := []any{int64(1), address, name} id, err := query.UpsertObject(n.tx, "raft_nodes", columns, values) if err != nil { return err } if id != 1 { return errors.New("could not set raft node ID to 1") } return nil } // CreateRaftNode adds a node to the current list of nodes that are part of the // dqlite Raft cluster. It returns the ID of the newly inserted row. func (n *NodeTx) CreateRaftNode(address string, name string) (int64, error) { columns := []string{"address", "name"} values := []any{address, name} return query.UpsertObject(n.tx, "raft_nodes", columns, values) } // RemoveRaftNode removes a node from the current list of nodes that are // part of the dqlite Raft cluster. func (n *NodeTx) RemoveRaftNode(id int64) error { deleted, err := query.DeleteObject(n.tx, "raft_nodes", id) if err != nil { return err } if !deleted { return api.StatusErrorf(http.StatusNotFound, "Raft member not found") } return nil } // ReplaceRaftNodes replaces the current list of raft nodes. func (n *NodeTx) ReplaceRaftNodes(nodes []RaftNode) error { _, err := n.tx.Exec("DELETE FROM raft_nodes") if err != nil { return err } columns := []string{"id", "address", "role", "name"} for _, node := range nodes { values := []any{node.ID, node.Address, node.Role, node.Name} _, err := query.UpsertObject(n.tx, "raft_nodes", columns, values) if err != nil { return err } } return nil } incus-7.0.0/internal/server/db/raft_test.go000066400000000000000000000066371517523235500207140ustar00rootroot00000000000000//go:build linux && cgo && !agent package db_test import ( "context" "testing" "github.com/cowsql/go-cowsql/client" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/response" ) // Fetch all raft nodes. func TestRaftNodes(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() id1, err := tx.CreateRaftNode("1.2.3.4:666", "test") require.NoError(t, err) id2, err := tx.CreateRaftNode("5.6.7.8:666", "test") require.NoError(t, err) nodes, err := tx.GetRaftNodes(context.Background()) require.NoError(t, err) assert.Equal(t, uint64(id1), nodes[0].ID) assert.Equal(t, uint64(id2), nodes[1].ID) assert.Equal(t, "1.2.3.4:666", nodes[0].Address) assert.Equal(t, "5.6.7.8:666", nodes[1].Address) } // Fetch the addresses of all raft nodes. func TestGetRaftNodeAddresses(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() _, err := tx.CreateRaftNode("1.2.3.4:666", "test") require.NoError(t, err) _, err = tx.CreateRaftNode("5.6.7.8:666", "test") require.NoError(t, err) addresses, err := tx.GetRaftNodeAddresses(context.Background()) require.NoError(t, err) assert.Equal(t, []string{"1.2.3.4:666", "5.6.7.8:666"}, addresses) } // Fetch the address of the raft node with the given ID. func TestGetRaftNodeAddress(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() _, err := tx.CreateRaftNode("1.2.3.4:666", "test") require.NoError(t, err) id, err := tx.CreateRaftNode("5.6.7.8:666", "test") require.NoError(t, err) address, err := tx.GetRaftNodeAddress(context.Background(), id) require.NoError(t, err) assert.Equal(t, "5.6.7.8:666", address) } // Add the first raft node. func TestCreateFirstRaftNode(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() err := tx.CreateFirstRaftNode("1.2.3.4:666", "test") assert.NoError(t, err) err = tx.RemoveRaftNode(1) assert.NoError(t, err) err = tx.CreateFirstRaftNode("5.6.7.8:666", "test") assert.NoError(t, err) address, err := tx.GetRaftNodeAddress(context.Background(), 1) require.NoError(t, err) assert.Equal(t, "5.6.7.8:666", address) } // Add a new raft node. func TestCreateRaftNode(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() id, err := tx.CreateRaftNode("1.2.3.4:666", "test") assert.Equal(t, int64(1), id) assert.NoError(t, err) } // Delete an existing raft node. func TestRemoveRaftNode(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() id, err := tx.CreateRaftNode("1.2.3.4:666", "test") require.NoError(t, err) err = tx.RemoveRaftNode(id) assert.NoError(t, err) } // Delete a non-existing raft node returns an error. func TestRemoveRaftNode_NonExisting(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() err := tx.RemoveRaftNode(1) assert.True(t, response.IsNotFoundError(err)) } // Replace all existing raft nodes. func TestReplaceRaftNodes(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() _, err := tx.CreateRaftNode("1.2.3.4:666", "test") require.NoError(t, err) nodes := []db.RaftNode{ {NodeInfo: client.NodeInfo{ID: 2, Address: "2.2.2.2:666"}}, {NodeInfo: client.NodeInfo{ID: 3, Address: "3.3.3.3:666"}}, } err = tx.ReplaceRaftNodes(nodes) assert.NoError(t, err) newNodes, err := tx.GetRaftNodes(context.Background()) require.NoError(t, err) assert.Equal(t, nodes, newNodes) } incus-7.0.0/internal/server/db/schema.go000066400000000000000000000002461517523235500201470ustar00rootroot00000000000000//go:build linux && cgo && !agent package db // Directive for regenerating both the cluster and node database schemas. // //go:generate generate-database db schema incus-7.0.0/internal/server/db/schema/000077500000000000000000000000001517523235500176165ustar00rootroot00000000000000incus-7.0.0/internal/server/db/schema/doc.go000066400000000000000000000001341517523235500207100ustar00rootroot00000000000000// Package schema offers utilities to create and maintain a database schema. package schema incus-7.0.0/internal/server/db/schema/errors.go000066400000000000000000000005311517523235500214600ustar00rootroot00000000000000package schema import ( "errors" ) // ErrGracefulAbort is a special error that can be returned by a Check function // to force Schema.Ensure to abort gracefully. // // Every change performed so by the Check will be committed, although // ErrGracefulAbort will be returned. var ErrGracefulAbort = errors.New("schema check gracefully aborted") incus-7.0.0/internal/server/db/schema/query.go000066400000000000000000000050101517523235500213060ustar00rootroot00000000000000package schema import ( "context" "database/sql" "errors" "fmt" "os" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/shared/util" ) // DoesSchemaTableExist return whether the schema table is present in the // database. func DoesSchemaTableExist(ctx context.Context, tx *sql.Tx) (bool, error) { statement := ` SELECT COUNT(name) FROM sqlite_master WHERE type = 'table' AND name = 'schema' ` rows, err := tx.QueryContext(ctx, statement) if err != nil { return false, err } defer func() { _ = rows.Close() }() if !rows.Next() { return false, errors.New("schema table query returned no rows") } var count int err = rows.Scan(&count) if err != nil { return false, err } return count == 1, nil } // Return all versions in the schema table, in increasing order. func selectSchemaVersions(ctx context.Context, tx *sql.Tx) ([]int, error) { statement := ` SELECT version FROM schema ORDER BY version ` return query.SelectIntegers(ctx, tx, statement) } // Return a list of SQL statements that can be used to create all tables in the // database. func selectTablesSQL(ctx context.Context, tx *sql.Tx) ([]string, error) { statement := ` SELECT sql FROM sqlite_master WHERE type IN ('table', 'index', 'view', 'trigger') AND name != 'schema' AND name NOT LIKE 'sqlite_%' ORDER BY name ` return query.SelectStrings(ctx, tx, statement) } // Create the schema table. func createSchemaTable(tx *sql.Tx) error { statement := ` CREATE TABLE schema ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, version INTEGER NOT NULL, updated_at DATETIME NOT NULL, UNIQUE (version) ) ` _, err := tx.Exec(statement) return err } // Insert a new version into the schema table. func insertSchemaVersion(tx *sql.Tx, newVersion int) error { statement := ` INSERT INTO schema (version, updated_at) VALUES (?, strftime("%s")) ` _, err := tx.Exec(statement, newVersion) return err } // Read the given file (if it exists) and executes all queries it contains. func execFromFile(ctx context.Context, tx *sql.Tx, path string, hook Hook) error { if !util.PathExists(path) { return nil } bytes, err := os.ReadFile(path) if err != nil { return fmt.Errorf("failed to read file: %w", err) } if hook != nil { err := hook(ctx, -1, tx) if err != nil { return fmt.Errorf("failed to execute hook: %w", err) } } _, err = tx.Exec(string(bytes)) if err != nil { return err } err = os.Remove(path) if err != nil { return fmt.Errorf("failed to remove file: %w", err) } return nil } incus-7.0.0/internal/server/db/schema/schema.go000066400000000000000000000313151517523235500214100ustar00rootroot00000000000000package schema import ( "context" "database/sql" "errors" "fmt" "slices" "sort" "strings" "github.com/lxc/incus/v7/internal/server/db/query" ) // Schema captures the schema of a database in terms of a series of ordered // updates. type Schema struct { updates []Update // Ordered series of updates making up the schema hook Hook // Optional hook to execute whenever a update gets applied fresh string // Optional SQL statement used to create schema from scratch check Check // Optional callback invoked before doing any update path string // Optional path to a file containing extra queries to run } // Update applies a specific schema change to a database, and returns an error // if anything goes wrong. type Update func(context.Context, *sql.Tx) error // Hook is a callback that gets fired when a update gets applied. type Hook func(context.Context, int, *sql.Tx) error // Check is a callback that gets fired all the times Schema.Ensure is invoked, // before applying any update. It gets passed the version that the schema is // currently at and a handle to the transaction. If it returns nil, the update // proceeds normally, otherwise it's aborted. If ErrGracefulAbort is returned, // the transaction will still be committed, giving chance to this function to // perform state changes. type Check func(context.Context, int, *sql.Tx) error // New creates a new schema Schema with the given updates. func New(updates []Update) *Schema { return &Schema{ updates: updates, } } // NewFromMap creates a new schema Schema with the updates specified in the // given map. The keys of the map are schema versions that when upgraded will // trigger the associated Update value. It's required that the minimum key in // the map is 1, and if key N is present then N-1 is present too, with N>1 // (i.e. there are no missing versions). // // NOTE: the regular New() constructor would be formally enough, but for extra // clarity we also support a map that indicates the version explicitly, // see also PR #3704. func NewFromMap(versionsToUpdates map[int]Update) *Schema { // Collect all version keys. versions := []int{} for version := range versionsToUpdates { versions = append(versions, version) } // Sort the versions, sort.Ints(versions) // Build the updates slice. updates := []Update{} for i, version := range versions { // Assert that we start from 1 and there are no gaps. if version != i+1 { panic(fmt.Sprintf("updates map misses version %d", i+1)) } updates = append(updates, versionsToUpdates[version]) } return &Schema{ updates: updates, } } // Empty creates a new schema with no updates. func Empty() *Schema { return New([]Update{}) } // Add a new update to the schema. It will be appended at the end of the // existing series. func (s *Schema) Add(update Update) { s.updates = append(s.updates, update) } // Hook instructs the schema to invoke the given function whenever a update is // about to be applied. The function gets passed the update version number and // the running transaction, and if it returns an error it will cause the schema // transaction to be rolled back. Any previously installed hook will be // replaced. func (s *Schema) Hook(hook Hook) { s.hook = hook } // Check instructs the schema to invoke the given function whenever Ensure is // invoked, before applying any due update. It can be used for aborting the // operation. func (s *Schema) Check(check Check) { s.check = check } // Fresh sets a statement that will be used to create the schema from scratch // when bootstrapping an empty database. It should be a "flattening" of the // available updates, generated using the Dump() method. If not given, all // patches will be applied in order. func (s *Schema) Fresh(statement string) { s.fresh = statement } // File extra queries from a file. If the file is exists, all SQL queries in it // will be executed transactionally at the very start of Ensure(), before // anything else is done. // // If a schema hook was set with Hook(), it will be run before running the // queries in the file and it will be passed a patch version equals to -1. func (s *Schema) File(path string) { s.path = path } // Ensure makes sure that the actual schema in the given database matches the // one defined by our updates. // // All updates are applied transactionally. In case any error occurs the // transaction will be rolled back and the database will remain unchanged. // // A update will be applied only if it hasn't been before (currently applied // updates are tracked in the a 'shema' table, which gets automatically // created). // // If no error occurs, the integer returned by this method is the // initial version that the schema has been upgraded from. func (s *Schema) Ensure(db *sql.DB) (int, error) { var current int aborted := false err := query.Transaction(context.TODO(), db, func(ctx context.Context, tx *sql.Tx) error { err := execFromFile(ctx, tx, s.path, s.hook) if err != nil { return fmt.Errorf("failed to execute queries from %s: %w", s.path, err) } err = ensureSchemaTableExists(ctx, tx) if err != nil { return err } current, err = queryCurrentVersion(ctx, tx) if err != nil { return err } if s.check != nil { err := s.check(ctx, current, tx) if errors.Is(err, ErrGracefulAbort) { // Abort the update gracefully, committing what // we've done so far. aborted = true return nil } if err != nil { return err } } // When creating the schema from scratch, use the fresh dump if // available. Otherwise just apply all relevant updates. if current == 0 && s.fresh != "" { _, err = tx.Exec(s.fresh) if err != nil { return fmt.Errorf("cannot apply fresh schema: %w", err) } } else { err = ensureUpdatesAreApplied(ctx, tx, current, s.updates, s.hook) if err != nil { return err } } return nil }) if err != nil { return -1, err } if aborted { return current, ErrGracefulAbort } return current, nil } // Dump returns a text of SQL commands that can be used to create this schema // from scratch in one go, without going through individual patches // (essentially flattening them). // // It requires that all patches in this schema have been applied, otherwise an // error will be returned. func (s *Schema) Dump(db *sql.DB) (string, error) { var statements []string err := query.Transaction(context.TODO(), db, func(ctx context.Context, tx *sql.Tx) error { err := checkAllUpdatesAreApplied(ctx, tx, s.updates) if err != nil { return err } statements, err = selectTablesSQL(ctx, tx) return err }) if err != nil { return "", err } for i, statement := range statements { statements[i] = formatSQL(statement) } // Add a statement for inserting the current schema version row. statements = append( statements, fmt.Sprintf(` INSERT INTO schema (version, updated_at) VALUES (%d, strftime("%%s")) `, len(s.updates))) return strings.Join(statements, ";\n"), nil } // Trim the schema updates to the given version (included). Updates with higher // versions will be discarded. Any fresh schema dump previously set will be // unset, since it's assumed to no longer be applicable. Return all updates // that have been trimmed. func (s *Schema) Trim(version int) []Update { trimmed := s.updates[version:] s.updates = s.updates[:version] s.fresh = "" return trimmed } // ExerciseUpdate is a convenience for exercising a particular update of a // schema. // // It first creates an in-memory SQLite database, then it applies all updates // up to the one with given version (excluded) and optionally executes the // given hook for populating the database with test data. Finally it applies // the update with the given version, returning the database handle for further // inspection of the resulting state. func (s *Schema) ExerciseUpdate(version int, hook func(*sql.DB)) (*sql.DB, error) { // Create an in-memory database. db, err := sql.Open("sqlite3", ":memory:?_foreign_keys=1") if err != nil { return nil, fmt.Errorf("failed to open memory database: %w", err) } // Apply all updates to the given version, excluded. trimmed := s.Trim(version - 1) _, err = s.Ensure(db) if err != nil { return nil, fmt.Errorf("failed to apply previous updates: %w", err) } // Execute the optional hook. if hook != nil { hook(db) } // Apply the update with the given version s.Add(trimmed[0]) _, err = s.Ensure(db) if err != nil { return nil, fmt.Errorf("failed to apply given update: %w", err) } return db, nil } // Ensure that the schema exists. func ensureSchemaTableExists(ctx context.Context, tx *sql.Tx) error { exists, err := DoesSchemaTableExist(ctx, tx) if err != nil { return fmt.Errorf("failed to check if schema table is there: %w", err) } if !exists { err := createSchemaTable(tx) if err != nil { return fmt.Errorf("failed to create schema table: %w", err) } } return nil } // Return the highest update version currently applied. Zero means that no // updates have been applied yet. func queryCurrentVersion(ctx context.Context, tx *sql.Tx) (int, error) { versions, err := selectSchemaVersions(ctx, tx) if err != nil { return -1, fmt.Errorf("failed to fetch update versions: %w", err) } // Fix bad upgrade code between 30 and 32 hasVersion := func(v int) bool { return slices.Contains(versions, v) } if hasVersion(30) && hasVersion(32) && !hasVersion(31) { err = insertSchemaVersion(tx, 31) if err != nil { return -1, errors.New("failed to insert missing schema version 31") } versions, err = selectSchemaVersions(ctx, tx) if err != nil { return -1, fmt.Errorf("failed to fetch update versions: %w", err) } } // Fix broken schema version between 37 and 38 if hasVersion(37) && !hasVersion(38) { count, err := query.Count(ctx, tx, "config", "key = 'cluster.https_address'") if err != nil { return -1, fmt.Errorf("Failed to check if cluster.https_address is set: %w", err) } if count == 1 { // Insert the missing version. err := insertSchemaVersion(tx, 38) if err != nil { return -1, errors.New("Failed to insert missing schema version 38") } versions = append(versions, 38) } } current := 0 if len(versions) > 0 { err = checkSchemaVersionsHaveNoHoles(versions) if err != nil { return -1, err } current = versions[len(versions)-1] // Highest recorded version } return current, nil } // Apply any pending update that was not yet applied. func ensureUpdatesAreApplied(ctx context.Context, tx *sql.Tx, current int, updates []Update, hook Hook) error { if current > len(updates) { return fmt.Errorf( "schema version '%d' is more recent than expected '%d'", current, len(updates)) } // If there are no updates, there's nothing to do. if len(updates) == 0 { return nil } // Apply missing updates. for _, update := range updates[current:] { if hook != nil { err := hook(ctx, current, tx) if err != nil { return fmt.Errorf( "failed to execute hook (version %d): %v", current, err) } } err := update(ctx, tx) if err != nil { return fmt.Errorf("failed to apply update %d: %w", current, err) } current++ err = insertSchemaVersion(tx, current) if err != nil { return fmt.Errorf("failed to insert version %d", current) } } return nil } // Check that the given list of update version numbers doesn't have "holes", // that is each version equal the preceding version plus 1. func checkSchemaVersionsHaveNoHoles(versions []int) error { // Ensure that there are no "holes" in the recorded versions. for i := range versions[:len(versions)-1] { if versions[i+1] != versions[i]+1 { return fmt.Errorf("Missing updates: %d to %d", versions[i], versions[i+1]) } } return nil } // Check that all the given updates are applied. func checkAllUpdatesAreApplied(ctx context.Context, tx *sql.Tx, updates []Update) error { versions, err := selectSchemaVersions(ctx, tx) if err != nil { return fmt.Errorf("failed to fetch update versions: %w", err) } if len(versions) == 0 { return errors.New("expected schema table to contain at least one row") } err = checkSchemaVersionsHaveNoHoles(versions) if err != nil { return err } current := versions[len(versions)-1] if current != len(updates) { return fmt.Errorf("update level is %d, expected %d", current, len(updates)) } return nil } // Format the given SQL statement in a human-readable way. // // In particular make sure that each column definition in a CREATE TABLE clause // is in its own row, since SQLite dumps occasionally stuff more than one // column in the same line. func formatSQL(statement string) string { lines := strings.Split(statement, "\n") for i, line := range lines { if strings.Contains(line, "UNIQUE") { // Let UNIQUE(x, y) constraints alone. continue } lines[i] = strings.ReplaceAll(line, ", ", ",\n ") } return strings.Join(lines, "\n") } incus-7.0.0/internal/server/db/schema/schema_test.go000066400000000000000000000340521517523235500224500ustar00rootroot00000000000000package schema_test import ( "context" "database/sql" "errors" "fmt" "os" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/internal/server/db/schema" "github.com/lxc/incus/v7/shared/util" ) // WriteTempFile creates a temp file with the specified content. func WriteTempFile(dir string, prefix string, content string) (string, error) { f, err := os.CreateTemp(dir, prefix) if err != nil { return "", err } defer func() { _ = f.Close() }() _, err = f.WriteString(content) if err != nil { return "", err } return f.Name(), f.Close() } // Create a new Schema by specifying an explicit map from versions to Update // functions. func TestNewFromMap(t *testing.T) { db := newDB(t) schema := schema.NewFromMap(map[int]schema.Update{ 1: updateCreateTable, 2: updateInsertValue, }) initial, err := schema.Ensure(db) assert.NoError(t, err) assert.Equal(t, 0, initial) } // Panic if there are missing versions in the map. func TestNewFromMap_MissingVersions(t *testing.T) { assert.Panics(t, func() { schema.NewFromMap(map[int]schema.Update{ 1: updateCreateTable, 3: updateInsertValue, }) }, "updates map misses version 2") } // If the database schema version is more recent than our update series, an // error is returned. func TestSchemaEnsure_VersionMoreRecentThanExpected(t *testing.T) { schema, db := newSchemaAndDB(t) schema.Add(updateNoop) _, err := schema.Ensure(db) assert.NoError(t, err) schema, _ = newSchemaAndDB(t) _, err = schema.Ensure(db) assert.NotNil(t, err) assert.EqualError(t, err, "schema version '1' is more recent than expected '0'") } // If a "fresh" SQL statement for creating the schema from scratch is provided, // but it fails to run, an error is returned. func TestSchemaEnsure_FreshStatementError(t *testing.T) { schema, db := newSchemaAndDB(t) schema.Add(updateNoop) schema.Fresh("garbage") _, err := schema.Ensure(db) assert.NotNil(t, err) assert.Contains(t, err.Error(), "cannot apply fresh schema") } // If the database schema contains "holes" in the applied versions, an error is // returned. func TestSchemaEnsure_MissingVersion(t *testing.T) { schema, db := newSchemaAndDB(t) schema.Add(updateNoop) _, err := schema.Ensure(db) assert.NoError(t, err) _, err = db.Exec(`INSERT INTO schema (version, updated_at) VALUES (3, strftime("%s"))`) assert.NoError(t, err) schema.Add(updateNoop) schema.Add(updateNoop) _, err = schema.Ensure(db) assert.NotNil(t, err) assert.EqualError(t, err, "Missing updates: 1 to 3") } // If the schema has no update, the schema table gets created and has no version. func TestSchemaEnsure_ZeroUpdates(t *testing.T) { schema, db := newSchemaAndDB(t) _, err := schema.Ensure(db) assert.NoError(t, err) tx, err := db.Begin() assert.NoError(t, err) versions, err := query.SelectIntegers(context.Background(), tx, "SELECT version FROM SCHEMA") assert.NoError(t, err) assert.Equal(t, []int{}, versions) } // If the schema has updates and no one was applied yet, all of them get // applied. func TestSchemaEnsure_ApplyAllUpdates(t *testing.T) { schema, db := newSchemaAndDB(t) schema.Add(updateCreateTable) schema.Add(updateInsertValue) initial, err := schema.Ensure(db) assert.NoError(t, err) assert.Equal(t, 0, initial) tx, err := db.Begin() assert.NoError(t, err) // THe update version is recorded. versions, err := query.SelectIntegers(context.Background(), tx, "SELECT version FROM SCHEMA") assert.NoError(t, err) assert.Equal(t, []int{1, 2}, versions) // The two updates have been applied in order. ids, err := query.SelectIntegers(context.Background(), tx, "SELECT id FROM test") assert.NoError(t, err) assert.Equal(t, []int{1}, ids) } // If the schema schema has been created using a dump, the schema table will // contain just one row with the update level associated with the dump. It's // possible to apply further updates from there, and only these new ones will // be inserted in the schema table. func TestSchemaEnsure_ApplyAfterInitialDumpCreation(t *testing.T) { schema, db := newSchemaAndDB(t) schema.Add(updateCreateTable) schema.Add(updateAddColumn) _, err := schema.Ensure(db) assert.NoError(t, err) dump, err := schema.Dump(db) assert.NoError(t, err) _, db = newSchemaAndDB(t) schema.Fresh(dump) _, err = schema.Ensure(db) assert.NoError(t, err) schema.Add(updateNoop) _, err = schema.Ensure(db) assert.NoError(t, err) tx, err := db.Begin() assert.NoError(t, err) // Only updates starting from the initial dump are recorded. versions, err := query.SelectIntegers(context.Background(), tx, "SELECT version FROM SCHEMA") assert.NoError(t, err) assert.Equal(t, []int{2, 3}, versions) } // If the schema has updates and part of them were already applied, only the // missing ones are applied. func TestSchemaEnsure_OnlyApplyMissing(t *testing.T) { schema, db := newSchemaAndDB(t) schema.Add(updateCreateTable) _, err := schema.Ensure(db) assert.NoError(t, err) schema.Add(updateInsertValue) initial, err := schema.Ensure(db) assert.NoError(t, err) assert.Equal(t, 1, initial) tx, err := db.Begin() assert.NoError(t, err) // All update versions are recorded. versions, err := query.SelectIntegers(context.Background(), tx, "SELECT version FROM SCHEMA") assert.NoError(t, err) assert.Equal(t, []int{1, 2}, versions) // The two updates have been applied in order. ids, err := query.SelectIntegers(context.Background(), tx, "SELECT id FROM test") assert.NoError(t, err) assert.Equal(t, []int{1}, ids) } // If a update fails, an error is returned, and all previous changes are rolled // back. func TestSchemaEnsure_FailingUpdate(t *testing.T) { schema, db := newSchemaAndDB(t) schema.Add(updateCreateTable) schema.Add(updateBoom) _, err := schema.Ensure(db) assert.EqualError(t, err, "failed to apply update 1: boom") tx, err := db.Begin() assert.NoError(t, err) // Not update was applied. tables, err := query.SelectStrings(context.Background(), tx, "SELECT name FROM sqlite_master WHERE type = 'table'") assert.NoError(t, err) assert.NotContains(t, tables, "schema") assert.NotContains(t, tables, "test") } // If a hook fails, an error is returned, and all previous changes are rolled // back. func TestSchemaEnsure_FailingHook(t *testing.T) { schema, db := newSchemaAndDB(t) schema.Add(updateCreateTable) schema.Hook(func(context.Context, int, *sql.Tx) error { return errors.New("boom") }) _, err := schema.Ensure(db) assert.EqualError(t, err, "failed to execute hook (version 0): boom") tx, err := db.Begin() assert.NoError(t, err) // Not update was applied. tables, err := query.SelectStrings(context.Background(), tx, "SELECT name FROM sqlite_master WHERE type = 'table'") assert.NoError(t, err) assert.NotContains(t, tables, "schema") assert.NotContains(t, tables, "test") } // If the schema check callback returns ErrGracefulAbort, the process is // aborted, although every change performed so far gets still committed. func TestSchemaEnsure_CheckGracefulAbort(t *testing.T) { check := func(ctx context.Context, current int, tx *sql.Tx) error { _, err := tx.Exec("CREATE TABLE test (n INTEGER)") require.NoError(t, err) return schema.ErrGracefulAbort } schema, db := newSchemaAndDB(t) schema.Add(updateCreateTable) schema.Check(check) _, err := schema.Ensure(db) require.EqualError(t, err, "schema check gracefully aborted") tx, err := db.Begin() assert.NoError(t, err) // The table created by the check function still got committed. // to insert the row was not. ids, err := query.SelectIntegers(context.Background(), tx, "SELECT n FROM test") assert.NoError(t, err) assert.Equal(t, []int{}, ids) } // The SQL text returns by Dump() can be used to create the schema from // scratch, without applying each individual update. func TestSchemaDump(t *testing.T) { schema, db := newSchemaAndDB(t) schema.Add(updateCreateTable) schema.Add(updateAddColumn) _, err := schema.Ensure(db) assert.NoError(t, err) dump, err := schema.Dump(db) assert.NoError(t, err) _, db = newSchemaAndDB(t) schema.Fresh(dump) _, err = schema.Ensure(db) assert.NoError(t, err) tx, err := db.Begin() assert.NoError(t, err) // All update versions are in place. versions, err := query.SelectIntegers(context.Background(), tx, "SELECT version FROM schema") assert.NoError(t, err) assert.Equal(t, []int{2}, versions) // Both the table added by the first update and the extra column added // by the second update are there. _, err = tx.Exec("SELECT id, name FROM test") assert.NoError(t, err) } // If not all updates are applied, Dump() returns an error. func TestSchemaDump_MissingUpdatees(t *testing.T) { schema, db := newSchemaAndDB(t) schema.Add(updateCreateTable) _, err := schema.Ensure(db) assert.NoError(t, err) schema.Add(updateAddColumn) _, err = schema.Dump(db) assert.EqualError(t, err, "update level is 1, expected 2") } // After trimming a schema, only the updates up to the trim point are applied. func TestSchema_Trim(t *testing.T) { updates := map[int]schema.Update{ 1: updateCreateTable, 2: updateInsertValue, 3: updateAddColumn, } schema := schema.NewFromMap(updates) trimmed := schema.Trim(2) assert.Len(t, trimmed, 1) db := newDB(t) _, err := schema.Ensure(db) require.NoError(t, err) tx, err := db.Begin() require.NoError(t, err) versions, err := query.SelectIntegers(context.Background(), tx, "SELECT version FROM schema") require.NoError(t, err) assert.Equal(t, []int{1, 2}, versions) } // Exercise a given update in a schema. func TestSchema_ExeciseUpdate(t *testing.T) { updates := map[int]schema.Update{ 1: updateCreateTable, 2: updateInsertValue, 3: updateAddColumn, } schema := schema.NewFromMap(updates) db, err := schema.ExerciseUpdate(2, nil) require.NoError(t, err) tx, err := db.Begin() require.NoError(t, err) // Update 2 has been applied. ids, err := query.SelectIntegers(context.Background(), tx, "SELECT id FROM test") require.NoError(t, err) assert.Equal(t, []int{1}, ids) // Update 3 has not been applied. _, err = query.SelectStrings(context.Background(), tx, "SELECT name FROM test") require.EqualError(t, err, "no such column: name") } // A custom schema file path is given, but it does not exists. This is a no-op. func TestSchema_File_NotExists(t *testing.T) { schema, db := newSchemaAndDB(t) schema.Add(updateCreateTable) schema.File("/non/existing/file/path") _, err := schema.Ensure(db) require.NoError(t, err) } // A custom schema file path is given, but it contains non valid SQL. An error // is returned an no change to the database is performed at all. func TestSchema_File_Garbage(t *testing.T) { schema, db := newSchemaAndDB(t) schema.Add(updateCreateTable) path, err := WriteTempFile("", "incus-db-schema-", "SELECT FROM baz") require.NoError(t, err) defer func() { _ = os.Remove(path) }() schema.File(path) _, err = schema.Ensure(db) message := fmt.Sprintf("failed to execute queries from %s: near \"FROM\": syntax error", path) require.EqualError(t, err, message) } // A custom schema file path is given, it runs some queries that repair an // otherwise broken update, before the update is run. func TestSchema_File(t *testing.T) { schema, db := newSchemaAndDB(t) // Add an update that would insert a value into a non-existing table. schema.Add(updateInsertValue) path, err := WriteTempFile("", "incus-db-schema-", `CREATE TABLE test (id INTEGER); INSERT INTO test VALUES (2); `) require.NoError(t, err) defer func() { _ = os.Remove(path) }() schema.File(path) _, err = schema.Ensure(db) require.NoError(t, err) // The file does not exist anymore. assert.False(t, util.PathExists(path)) // The table was created, and the extra row inserted as well. tx, err := db.Begin() require.NoError(t, err) ids, err := query.SelectIntegers(context.Background(), tx, "SELECT id FROM test ORDER BY id") require.NoError(t, err) assert.Equal(t, []int{1, 2}, ids) } // A both a custom schema file path and a hook are set, the hook runs before // the queries in the file are executed. func TestSchema_File_Hook(t *testing.T) { schema, db := newSchemaAndDB(t) // Add an update that would insert a value into a non-existing table. schema.Add(updateInsertValue) // Add a custom schema update query file that inserts a value into a // non-existing table. path, err := WriteTempFile("", "incus-db-schema-", "INSERT INTO test VALUES (2)") require.NoError(t, err) defer func() { _ = os.Remove(path) }() schema.File(path) // Add a hook that takes care of creating the test table, this shows // that it's run before anything else. schema.Hook(func(ctx context.Context, version int, tx *sql.Tx) error { if version == -1 { _, err := tx.Exec("CREATE TABLE test (id INTEGER)") return err } return nil }) _, err = schema.Ensure(db) require.NoError(t, err) // The table was created, and the both rows inserted as well. tx, err := db.Begin() require.NoError(t, err) ids, err := query.SelectIntegers(context.Background(), tx, "SELECT id FROM test ORDER BY id") require.NoError(t, err) assert.Equal(t, []int{1, 2}, ids) } // Return a new in-memory SQLite database. func newDB(t *testing.T) *sql.DB { db, err := sql.Open("sqlite3", ":memory:") assert.NoError(t, err) return db } // Return both an empty schema and a test database. func newSchemaAndDB(t *testing.T) (*schema.Schema, *sql.DB) { return schema.Empty(), newDB(t) } // An update that does nothing. func updateNoop(context.Context, *sql.Tx) error { return nil } // An update that creates a test table. func updateCreateTable(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec("CREATE TABLE test (id INTEGER)") return err } // An update that inserts a value into the test table. func updateInsertValue(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec("INSERT INTO test VALUES (1)") return err } // An update that adds a column to the test tabble. func updateAddColumn(ctx context.Context, tx *sql.Tx) error { _, err := tx.Exec("ALTER TABLE test ADD COLUMN name TEXT") return err } // An update that unconditionally fails with an error. func updateBoom(ctx context.Context, tx *sql.Tx) error { return errors.New("boom") } incus-7.0.0/internal/server/db/schema/update.go000066400000000000000000000034301517523235500214270ustar00rootroot00000000000000package schema import ( "database/sql" "fmt" "os" "path" "runtime" _ "github.com/mattn/go-sqlite3" // For opening the in-memory database ) // DotGo writes '.go' source file in the package of the calling function, containing // SQL statements that match the given schema updates. // // The .go file contains a "flattened" render of all given updates and // can be used to initialize brand new databases using Schema.Fresh(). func DotGo(updates map[int]Update, name string) error { // Apply all the updates that we have on a pristine database and dump // the resulting schema. db, err := sql.Open("sqlite3", ":memory:") if err != nil { return fmt.Errorf("failed to open schema.go for writing: %w", err) } defer db.Close() schema := NewFromMap(updates) _, err = schema.Ensure(db) if err != nil { return err } dump, err := schema.Dump(db) if err != nil { return err } // Passing 1 to runtime.Caller identifies our caller. _, filename, _, _ := runtime.Caller(1) file, err := os.Create(path.Join(path.Dir(filename), name+".go")) if err != nil { return fmt.Errorf("failed to open Go file for writing: %w", err) } defer file.Close() pkg := path.Base(path.Dir(filename)) _, err = file.Write(fmt.Appendf(nil, dotGoTemplate, pkg, dump)) if err != nil { return fmt.Errorf("failed to write to Go file: %w", err) } return nil } // Template for schema files (can't use backticks since we need to use backticks // inside the template itself). const dotGoTemplate = "package %s\n\n" + "// DO NOT EDIT BY HAND\n" + "//\n" + "// This code was generated by the schema.DotGo function. If you need to\n" + "// modify the database schema, please add a new schema update to update.go\n" + "// and the run 'make update-schema'.\n" + "const freshSchema = `\n" + "%s`\n" incus-7.0.0/internal/server/db/schema/update_test.go000066400000000000000000000010211517523235500224600ustar00rootroot00000000000000package schema_test import ( "os" "testing" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db/schema" "github.com/lxc/incus/v7/shared/util" ) // A Go source file matching the given prefix is created in the calling // package. func TestDotGo(t *testing.T) { updates := map[int]schema.Update{ 1: updateCreateTable, 2: updateInsertValue, } require.NoError(t, schema.DotGo(updates, "xyz")) require.Equal(t, true, util.PathExists("xyz.go")) require.NoError(t, os.Remove("xyz.go")) } incus-7.0.0/internal/server/db/snapshots.go000066400000000000000000000052341517523235500207330ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "database/sql" "time" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/query" ) // UpdateInstanceSnapshotConfig inserts/updates/deletes the provided config keys. func (c *ClusterTx) UpdateInstanceSnapshotConfig(id int, values map[string]string) error { insertSQL := "INSERT OR REPLACE INTO instances_snapshots_config (instance_snapshot_id, key, value) VALUES" deleteSQL := "DELETE FROM instances_snapshots_config WHERE key IN %s AND instance_snapshot_id=?" return c.configUpdate(id, values, insertSQL, deleteSQL) } // UpdateInstanceSnapshot updates the description and expiry date of the // instance snapshot with the given ID. func (c *ClusterTx) UpdateInstanceSnapshot(id int, description string, expiryDate time.Time) error { str := "UPDATE instances_snapshots SET description=?, expiry_date=? WHERE id=?" var err error if expiryDate.IsZero() { _, err = c.tx.Exec(str, description, "", id) } else { _, err = c.tx.Exec(str, description, expiryDate, id) } if err != nil { return err } return nil } // GetLocalExpiredInstanceSnapshots returns a list of expired snapshots. func (c *ClusterTx) GetLocalExpiredInstanceSnapshots(ctx context.Context) ([]cluster.InstanceSnapshot, error) { q := ` SELECT instances_snapshots.id, instances_snapshots.expiry_date FROM instances_snapshots JOIN instances ON instances.id=instances_snapshots.instance_id WHERE instances.node_id=? AND instances_snapshots.expiry_date != '0001-01-01T00:00:00Z' ` snapshotIDs := []int{} err := query.Scan(ctx, c.Tx(), q, func(scan func(dest ...any) error) error { var id int var expiry sql.NullTime // Read the row. err := scan(&id, &expiry) if err != nil { return err } // Skip if not expired. if !expiry.Valid || expiry.Time.Unix() <= 0 { return nil } if time.Now().Before(expiry.Time) { return nil } // Add the snapshot. snapshotIDs = append(snapshotIDs, id) return nil }, c.nodeID) if err != nil { return nil, err } // Fetch all the expired snapshot details. snapshots := make([]cluster.InstanceSnapshot, len(snapshotIDs)) for i, id := range snapshotIDs { snap, err := cluster.GetInstanceSnapshots(ctx, c.tx, cluster.InstanceSnapshotFilter{ID: &id}) if err != nil { return nil, err } snapshots[i] = snap[0] } return snapshots, nil } // GetInstanceSnapshotID returns the ID of the snapshot with the given name. func (c *ClusterTx) GetInstanceSnapshotID(ctx context.Context, project, instance, name string) (int, error) { id, err := cluster.GetInstanceSnapshotID(ctx, c.tx, project, instance, name) return int(id), err } incus-7.0.0/internal/server/db/snapshots_test.go000066400000000000000000000171531517523235500217750ustar00rootroot00000000000000//go:build linux && cgo && !agent package db_test import ( "context" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/instance/instancetype" ) func TestGetInstanceSnapshots(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() nodeID1 := int64(1) // This is the default local member addContainer(t, tx, nodeID1, "c1") addContainer(t, tx, nodeID1, "c2") addInstanceSnapshot(t, tx, 1, "snap1") addInstanceSnapshot(t, tx, 2, "snap2") addInstanceSnapshot(t, tx, 2, "snap3") addInstanceSnapshotConfig(t, tx, "c2", "snap2", "x", "y") addInstanceSnapshotDevice(t, tx, "c2", "snap2", "eth0", "nic", nil) addInstanceSnapshotDevice(t, tx, "c2", "snap3", "root", "disk", map[string]string{"x": "y"}) snapshots, err := cluster.GetInstanceSnapshots(context.TODO(), tx.Tx()) require.NoError(t, err) assert.Len(t, snapshots, 3) s1Config, err := cluster.GetInstanceSnapshotConfig(context.TODO(), tx.Tx(), snapshots[0].ID) require.NoError(t, err) s1Devices, err := cluster.GetInstanceSnapshotDevices(context.TODO(), tx.Tx(), snapshots[0].ID) require.NoError(t, err) s1 := snapshots[0] assert.Equal(t, "snap1", s1.Name) assert.Equal(t, "c1", s1.Instance) assert.Equal(t, map[string]string{}, s1Config) assert.Len(t, s1Devices, 0) s2Config, err := cluster.GetInstanceSnapshotConfig(context.TODO(), tx.Tx(), snapshots[1].ID) require.NoError(t, err) s2Devices, err := cluster.GetInstanceSnapshotDevices(context.TODO(), tx.Tx(), snapshots[1].ID) require.NoError(t, err) s2 := snapshots[1] assert.Equal(t, "snap2", s2.Name) assert.Equal(t, "c2", s2.Instance) assert.Equal(t, map[string]string{"x": "y"}, s2Config) assert.Len(t, s2Devices, 1) assert.Equal(t, "eth0", s2Devices["eth0"].Name) assert.Equal(t, "nic", s2Devices["eth0"].Type.String()) assert.Equal(t, map[string]string{}, s2Devices["eth0"].Config) s3Config, err := cluster.GetInstanceSnapshotConfig(context.TODO(), tx.Tx(), snapshots[2].ID) require.NoError(t, err) s3Devices, err := cluster.GetInstanceSnapshotDevices(context.TODO(), tx.Tx(), snapshots[2].ID) require.NoError(t, err) s3 := snapshots[2] assert.Equal(t, "snap3", s3.Name) assert.Equal(t, "c2", s3.Instance) assert.Equal(t, map[string]string{}, s3Config) assert.Len(t, s3Devices, 1) assert.Equal(t, "root", s3Devices["root"].Name) assert.Equal(t, "disk", s3Devices["root"].Type.String()) assert.Equal(t, map[string]string{"x": "y"}, s3Devices["root"].Config) } func TestGetInstanceSnapshots_FilterByInstance(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() nodeID1 := int64(1) // This is the default local member addContainer(t, tx, nodeID1, "c1") addContainer(t, tx, nodeID1, "c2") addInstanceSnapshot(t, tx, 1, "snap1") addInstanceSnapshot(t, tx, 2, "snap1") addInstanceSnapshot(t, tx, 2, "snap2") project := "default" instance := "c2" filter := cluster.InstanceSnapshotFilter{Project: &project, Instance: &instance} snapshots, err := cluster.GetInstanceSnapshots(context.TODO(), tx.Tx(), filter) require.NoError(t, err) assert.Len(t, snapshots, 2) s1 := snapshots[0] assert.Equal(t, "snap1", s1.Name) assert.Equal(t, "c2", s1.Instance) s2 := snapshots[1] assert.Equal(t, "snap2", s2.Name) assert.Equal(t, "c2", s2.Instance) } func TestGetInstanceSnapshots_SameNameInDifferentProjects(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() // Create an additional project project1 := cluster.Project{} project1.Name = "p1" _, err := cluster.CreateProject(context.Background(), tx.Tx(), project1) require.NoError(t, err) // Create an instance in the default project. i1default := cluster.Instance{ Project: "default", Name: "i1", Node: "none", Type: instancetype.Container, Architecture: 1, Ephemeral: false, Stateful: true, } _, err = cluster.CreateInstance(context.TODO(), tx.Tx(), i1default) require.NoError(t, err) // Create an instance in project p1 using the same name. i1p1 := cluster.Instance{ Project: "p1", Name: "i1", Node: "none", Type: instancetype.Container, Architecture: 1, Ephemeral: false, Stateful: true, } _, err = cluster.CreateInstance(context.TODO(), tx.Tx(), i1p1) require.NoError(t, err) // Create two snapshots with the same names. s1default := cluster.InstanceSnapshot{ Project: "default", Instance: "i1", Name: "s1", } _, err = cluster.CreateInstanceSnapshot(context.TODO(), tx.Tx(), s1default) require.NoError(t, err) s1p1 := cluster.InstanceSnapshot{ Project: "p1", Instance: "i1", Name: "s1", } _, err = cluster.CreateInstanceSnapshot(context.TODO(), tx.Tx(), s1p1) require.NoError(t, err) instance := "i1" project := "p1" filter := cluster.InstanceSnapshotFilter{Project: &project, Instance: &instance} snapshots, err := cluster.GetInstanceSnapshots(context.TODO(), tx.Tx(), filter) require.NoError(t, err) assert.Len(t, snapshots, 1) assert.Equal(t, "p1", snapshots[0].Project) assert.Equal(t, "i1", snapshots[0].Instance) assert.Equal(t, "s1", snapshots[0].Name) snapshot, err := cluster.GetInstanceSnapshot(context.TODO(), tx.Tx(), "default", "i1", "s1") require.NoError(t, err) assert.Equal(t, "default", snapshot.Project) assert.Equal(t, "i1", snapshot.Instance) assert.Equal(t, "s1", snapshot.Name) } func addInstanceSnapshot(t *testing.T, tx *db.ClusterTx, instanceID int64, name string) { stmt := ` INSERT INTO instances_snapshots(instance_id, name, creation_date, description) VALUES (?, ?, ?, '') ` _, err := tx.Tx().Exec(stmt, instanceID, name, time.Now()) require.NoError(t, err) } // Return the instance snapshot ID given its name and instance name. func getInstanceSnapshotID(t *testing.T, tx *db.ClusterTx, instance, name string) int64 { var id int64 stmt := ` SELECT instances_snapshots.id FROM instances_snapshots JOIN instances ON instances.id=instances_snapshots.instance_id WHERE instances.name=? AND instances_snapshots.name=? ` row := tx.Tx().QueryRow(stmt, instance, name) err := row.Scan(&id) require.NoError(t, err) return id } func addInstanceSnapshotConfig(t *testing.T, tx *db.ClusterTx, instance, name, key, value string) { id := getInstanceSnapshotID(t, tx, instance, name) stmt := ` INSERT INTO instances_snapshots_config(instance_snapshot_id, key, value) VALUES (?, ?, ?) ` _, err := tx.Tx().Exec(stmt, id, key, value) require.NoError(t, err) } // Return the instance snapshot device ID given its instance snapshot ID and name. func getInstanceSnapshotDeviceID(t *testing.T, tx *db.ClusterTx, instanceSnapshotID int64, name string) int64 { var id int64 stmt := "SELECT id FROM instances_snapshots_devices WHERE instance_snapshot_id=? AND name=?" row := tx.Tx().QueryRow(stmt, instanceSnapshotID, name) err := row.Scan(&id) require.NoError(t, err) return id } func addInstanceSnapshotDevice(t *testing.T, tx *db.ClusterTx, instance, snapshot, name, typ string, config map[string]string) { id := getInstanceSnapshotID(t, tx, instance, snapshot) code, err := cluster.NewDeviceType(typ) require.NoError(t, err) stmt := ` INSERT INTO instances_snapshots_devices(instance_snapshot_id, name, type) VALUES (?, ?, ?) ` _, err = tx.Tx().Exec(stmt, id, name, code) require.NoError(t, err) deviceID := getInstanceSnapshotDeviceID(t, tx, id, name) for key, value := range config { stmt := ` INSERT INTO instances_snapshots_devices_config(instance_snapshot_device_id, key, value) VALUES (?, ?, ?) ` _, err = tx.Tx().Exec(stmt, deviceID, key, value) require.NoError(t, err) } } incus-7.0.0/internal/server/db/storage_buckets.go000066400000000000000000000447641517523235500221100ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "database/sql" "errors" "fmt" "net/http" "strings" dqliteDriver "github.com/cowsql/go-cowsql/driver" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // StorageBucketFilter used for filtering storage buckets with GetStorageBuckets(). type StorageBucketFilter struct { PoolID *int64 PoolName *string Project *string Name *string } // StorageBucket represents a database storage bucket record. type StorageBucket struct { api.StorageBucket ID int64 PoolID int64 PoolName string } // GetStoragePoolBuckets returns all storage buckets. // If there are no buckets, it returns an empty list and no error. // Accepts filters for narrowing down the results returned. If memberSpecific is true, then the search is // restricted to buckets that belong to this member or belong to all members. func (c *ClusterTx) GetStoragePoolBuckets(ctx context.Context, memberSpecific bool, filters ...StorageBucketFilter) ([]*StorageBucket, error) { var q *strings.Builder = &strings.Builder{} var args []any q.WriteString(` SELECT projects.name as project, storage_pools.name, storage_buckets.id, storage_buckets.storage_pool_id, storage_buckets.name, storage_buckets.description, IFNULL(nodes.name, "") as location FROM storage_buckets JOIN projects ON projects.id = storage_buckets.project_id JOIN storage_pools ON storage_pools.id = storage_buckets.storage_pool_id LEFT JOIN nodes ON nodes.id = storage_buckets.node_id `) if memberSpecific { if len(args) == 0 { q.WriteString("WHERE ") } else { q.WriteString("AND ") } q.WriteString("(storage_buckets.node_id = ? OR storage_buckets.node_id IS NULL) ") args = append(args, c.nodeID) } if len(filters) > 0 { if len(args) == 0 { q.WriteString("WHERE (") } else { q.WriteString("AND (") } for i, filter := range filters { // Validate filter. if !memberSpecific && filter.Name != nil && ((filter.PoolID == nil && filter.PoolName == nil) || filter.Project == nil) { return nil, errors.New("Cannot filter by bucket name without specifying pool and project when doing member inspecific search") } var qFilters []string if filter.PoolID != nil { qFilters = append(qFilters, "storage_buckets.storage_pool_id= ?") args = append(args, *filter.PoolID) } if filter.PoolName != nil { qFilters = append(qFilters, "storage_pools.name= ?") args = append(args, *filter.PoolID) } if filter.Project != nil { qFilters = append(qFilters, "projects.name = ?") args = append(args, *filter.Project) } if filter.Name != nil { qFilters = append(qFilters, "storage_buckets.name = ?") args = append(args, *filter.Name) } if qFilters == nil { return nil, errors.New("Invalid storage bucket filter") } if i > 0 { q.WriteString(" OR ") } q.WriteString(fmt.Sprintf("(%s)", strings.Join(qFilters, " AND "))) } q.WriteString(")") } var err error var buckets []*StorageBucket err = query.Scan(ctx, c.Tx(), q.String(), func(scan func(dest ...any) error) error { var bucket StorageBucket err := scan(&bucket.Project, &bucket.PoolName, &bucket.ID, &bucket.PoolID, &bucket.Name, &bucket.Description, &bucket.Location) if err != nil { return err } buckets = append(buckets, &bucket) return nil }, args...) if err != nil { return nil, err } // Populate config. for i := range buckets { err = storagePoolBucketConfig(ctx, c, buckets[i].ID, &buckets[i].StorageBucket) if err != nil { return nil, err } } return buckets, nil } // storagePoolBucketConfig populates the config map of the Storage Bucket with the given ID. func storagePoolBucketConfig(ctx context.Context, tx *ClusterTx, bucketID int64, bucket *api.StorageBucket) error { q := ` SELECT key, value FROM storage_buckets_config WHERE storage_bucket_id=? ` bucket.Config = make(map[string]string) return query.Scan(ctx, tx.Tx(), q, func(scan func(dest ...any) error) error { var key, value string err := scan(&key, &value) if err != nil { return err } _, found := bucket.Config[key] if found { return fmt.Errorf("Duplicate config row found for key %q for storage bucket ID %d", key, bucketID) } bucket.Config[key] = value return nil }, bucketID) } // GetStoragePoolBucket returns the Storage Bucket for the given Storage Pool ID, Project Name and Bucket Name. // If memberSpecific is true, then the search is restricted to buckets that belong to this member or belong // to all members. func (c *ClusterTx) GetStoragePoolBucket(ctx context.Context, poolID int64, projectName string, memberSpecific bool, bucketName string) (*StorageBucket, error) { filters := []StorageBucketFilter{{ PoolID: &poolID, Project: &projectName, Name: &bucketName, }} buckets, err := c.GetStoragePoolBuckets(ctx, memberSpecific, filters...) bucketsLen := len(buckets) if (err == nil && bucketsLen <= 0) || errors.Is(err, sql.ErrNoRows) { return nil, api.StatusErrorf(http.StatusNotFound, "Storage bucket not found") } else if err == nil && bucketsLen > 1 { return nil, api.StatusErrorf(http.StatusConflict, "Storage bucket found on more than one cluster member. Please target a specific member") } else if err != nil { return nil, err } return buckets[0], nil } // GetStoragePoolLocalBucket returns the local Storage Bucket for the given bucket name. // The search is restricted to buckets that belong to this member. func (c *ClusterTx) GetStoragePoolLocalBucket(ctx context.Context, bucketName string) (*StorageBucket, error) { filters := []StorageBucketFilter{{ Name: &bucketName, }} buckets, err := c.GetStoragePoolBuckets(ctx, true, filters...) if err != nil && !errors.Is(err, sql.ErrNoRows) { return nil, err } for _, bucket := range buckets { if bucket.Location == "" { continue // Ignore buckets on remote storage pools. } return bucket, nil } return nil, api.StatusErrorf(http.StatusNotFound, "Storage bucket not found") } // GetStoragePoolLocalBucketByAccessKey returns the local Storage Bucket for the given bucket access key. // The search is restricted to buckets that belong to this member. func (c *ClusterTx) GetStoragePoolLocalBucketByAccessKey(ctx context.Context, accessKey string) (*StorageBucket, error) { var q *strings.Builder = &strings.Builder{} q.WriteString(` SELECT projects.name as project, storage_pools.name, storage_buckets.id, storage_buckets.storage_pool_id, storage_buckets.name, storage_buckets.description, IFNULL(nodes.name, "") as location FROM storage_buckets JOIN projects ON projects.id = storage_buckets.project_id JOIN storage_pools ON storage_pools.id = storage_buckets.storage_pool_id JOIN storage_buckets_keys ON storage_buckets_keys.storage_bucket_id = storage_buckets.id JOIN nodes ON nodes.id = storage_buckets.node_id WHERE storage_buckets.node_id = ? AND storage_buckets_keys.access_key = ? `) var err error var buckets []*StorageBucket args := []any{c.nodeID, accessKey} err = query.Scan(ctx, c.Tx(), q.String(), func(scan func(dest ...any) error) error { var bucket StorageBucket err := scan(&bucket.Project, &bucket.PoolName, &bucket.ID, &bucket.PoolID, &bucket.Name, &bucket.Description, &bucket.Location) if err != nil { return err } buckets = append(buckets, &bucket) return nil }, args...) if err != nil { return nil, err } bucketsLen := len(buckets) if bucketsLen == 1 { // Populate config. err = storagePoolBucketConfig(ctx, c, buckets[0].ID, &buckets[0].StorageBucket) if err != nil { return nil, err } return buckets[0], nil } else if bucketsLen > 1 { return nil, api.StatusErrorf(http.StatusConflict, "Multiple storage buckets found for access key") } return nil, api.StatusErrorf(http.StatusNotFound, "Storage bucket access key not found") } // CreateStoragePoolBucket creates a new Storage Bucket. // If memberSpecific is true, then the storage bucket is associated to the current member, rather than being // associated to all members. func (c *ClusterTx) CreateStoragePoolBucket(ctx context.Context, poolID int64, projectName string, memberSpecific bool, info api.StorageBucketsPost) (int64, error) { var err error var bucketID int64 var nodeID any if memberSpecific { nodeID = c.nodeID } // Insert a new Storage Bucket record. result, err := c.tx.ExecContext(ctx, ` INSERT INTO storage_buckets (storage_pool_id, node_id, name, description, project_id) VALUES (?, ?, ?, ?, (SELECT id FROM projects WHERE name = ?)) `, poolID, nodeID, info.Name, info.Description, projectName) if err != nil { var dqliteErr dqliteDriver.Error // Detect SQLITE_CONSTRAINT_UNIQUE (2067) errors. if errors.As(err, &dqliteErr) && dqliteErr.Code == 2067 { return -1, api.StatusErrorf(http.StatusConflict, "A bucket for that name already exists") } return -1, err } bucketID, err = result.LastInsertId() if err != nil { return -1, err } // Save config. err = storageBucketPoolConfigAdd(c.tx, bucketID, info.Config) if err != nil { return -1, err } return bucketID, err } // storageBucketPoolConfigAdd inserts Storage Bucket config keys. func storageBucketPoolConfigAdd(tx *sql.Tx, bucketID int64, config map[string]string) error { stmt, err := tx.Prepare(` INSERT INTO storage_buckets_config (storage_bucket_id, key, value) VALUES(?, ?, ?) `) if err != nil { return err } defer func() { _ = stmt.Close() }() for k, v := range config { if v == "" { continue } _, err = stmt.Exec(bucketID, k, v) if err != nil { return fmt.Errorf("Failed inserting config: %w", err) } } return nil } // UpdateStoragePoolBucket updates an existing Storage Bucket. func (c *ClusterTx) UpdateStoragePoolBucket(ctx context.Context, poolID int64, bucketID int64, info *api.StorageBucketPut) error { // Update existing Storage Bucket record. res, err := c.tx.ExecContext(ctx, ` UPDATE storage_buckets SET description = ? WHERE storage_pool_id = ? and id = ? `, info.Description, poolID, bucketID) if err != nil { return err } rowsAffected, err := res.RowsAffected() if err != nil { return err } if rowsAffected <= 0 { return api.StatusErrorf(http.StatusNotFound, "Storage bucket not found") } // Save config. _, err = c.tx.ExecContext(ctx, "DELETE FROM storage_buckets_config WHERE storage_bucket_id=?", bucketID) if err != nil { return err } err = storageBucketPoolConfigAdd(c.tx, bucketID, info.Config) if err != nil { return err } return nil } // DeleteStoragePoolBucket deletes an existing Storage Bucket. func (c *ClusterTx) DeleteStoragePoolBucket(ctx context.Context, poolID int64, bucketID int64) error { // Delete existing Storage Bucket record. res, err := c.tx.ExecContext(ctx, ` DELETE FROM storage_buckets WHERE storage_pool_id = ? and id = ? `, poolID, bucketID) if err != nil { return err } rowsAffected, err := res.RowsAffected() if err != nil { return err } if rowsAffected <= 0 { return api.StatusErrorf(http.StatusNotFound, "Storage bucket not found") } return nil } // StorageBucketKeyFilter used for filtering storage bucket keys with GetStoragePoolBucketKeys(). type StorageBucketKeyFilter struct { Name *string } // StorageBucketKey represents a database storage bucket key record. type StorageBucketKey struct { api.StorageBucketKey ID int64 } // GetStoragePoolBucketKeys returns all storage buckets keys attached to a given storage bucket. // If there are no bucket keys, it returns an empty list and no error. // Accepts filters for narrowing down the results returned. func (c *ClusterTx) GetStoragePoolBucketKeys(ctx context.Context, bucketID int64, filters ...StorageBucketKeyFilter) ([]*StorageBucketKey, error) { var q *strings.Builder = &strings.Builder{} args := []any{bucketID} q.WriteString(` SELECT storage_buckets_keys.id, storage_buckets_keys.name, storage_buckets_keys.description, storage_buckets_keys.role, storage_buckets_keys.access_key, storage_buckets_keys.secret_key FROM storage_buckets_keys WHERE storage_buckets_keys.storage_bucket_id = ? `) if len(filters) > 0 { q.WriteString("AND (") for i, filter := range filters { var qFilters []string if filter.Name != nil { qFilters = append(qFilters, "storage_buckets_keys.name = ?") args = append(args, *filter.Name) } if qFilters == nil { return nil, errors.New("Invalid storage bucket key filter") } if i > 0 { q.WriteString(" OR ") } q.WriteString(fmt.Sprintf("(%s)", strings.Join(qFilters, " AND "))) } q.WriteString(")") } var err error var bucketKeys []*StorageBucketKey err = query.Scan(ctx, c.Tx(), q.String(), func(scan func(dest ...any) error) error { var bucketKey StorageBucketKey err := scan(&bucketKey.ID, &bucketKey.Name, &bucketKey.Description, &bucketKey.Role, &bucketKey.AccessKey, &bucketKey.SecretKey) if err != nil { return err } bucketKeys = append(bucketKeys, &bucketKey) return nil }, args...) if err != nil { return nil, err } return bucketKeys, nil } // GetStoragePoolBucketKey returns the Storage Bucket Key for the given Bucket ID and Key Name. func (c *ClusterTx) GetStoragePoolBucketKey(ctx context.Context, bucketID int64, keyName string) (*StorageBucketKey, error) { filters := []StorageBucketKeyFilter{{ Name: &keyName, }} bucketKeys, err := c.GetStoragePoolBucketKeys(ctx, bucketID, filters...) bucketKeysLen := len(bucketKeys) if (err == nil && bucketKeysLen <= 0) || errors.Is(err, sql.ErrNoRows) { return nil, api.StatusErrorf(http.StatusNotFound, "Storage bucket key not found") } else if err == nil && bucketKeysLen > 1 { return nil, api.StatusErrorf(http.StatusConflict, "More than one storage bucket key found") } else if err != nil { return nil, err } return bucketKeys[0], nil } // CreateStoragePoolBucketKey creates a new Storage Bucket Key. func (c *ClusterTx) CreateStoragePoolBucketKey(ctx context.Context, bucketID int64, info api.StorageBucketKeysPost) (int64, error) { var err error var bucketKeyID int64 // Check there isn't another bucket with the same access key on the local server. bucket, err := c.GetStoragePoolLocalBucketByAccessKey(ctx, info.AccessKey) if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { return -1, err } else if bucket != nil { return -1, api.StatusErrorf(http.StatusConflict, "A bucket key using that access key already exists on this server") } // Insert a new Storage Bucket Key record. result, err := c.tx.ExecContext(ctx, ` INSERT INTO storage_buckets_keys (storage_bucket_id, name, description, role, access_key, secret_key) VALUES (?, ?, ?, ?, ?, ?) `, bucketID, info.Name, info.Description, info.Role, info.AccessKey, info.SecretKey) if err != nil { var dqliteErr dqliteDriver.Error // Detect SQLITE_CONSTRAINT_UNIQUE (2067) errors. if errors.As(err, &dqliteErr) && dqliteErr.Code == 2067 { return -1, api.StatusErrorf(http.StatusConflict, "A bucket key for that name already exists") } return -1, err } bucketKeyID, err = result.LastInsertId() if err != nil { return -1, err } return bucketKeyID, err } // UpdateStoragePoolBucketKey updates an existing Storage Bucket Key. func (c *ClusterTx) UpdateStoragePoolBucketKey(ctx context.Context, bucketID int64, bucketKeyID int64, info *api.StorageBucketKeyPut) error { // Check there isn't another bucket with the same access key on the local server. bucket, err := c.GetStoragePoolLocalBucketByAccessKey(ctx, info.AccessKey) if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { return err } else if bucket != nil && bucket.ID != bucketID { return api.StatusErrorf(http.StatusConflict, "A bucket key using that access key already exists on this server") } // Update existing Storage Bucket Key record. res, err := c.tx.ExecContext(ctx, ` UPDATE storage_buckets_keys SET description = ?, role = ?, access_key = ?, secret_key = ? WHERE storage_bucket_id = ? and id = ? `, info.Description, info.Role, info.AccessKey, info.SecretKey, bucketID, bucketKeyID) if err != nil { return err } rowsAffected, err := res.RowsAffected() if err != nil { return err } if rowsAffected <= 0 { return api.StatusErrorf(http.StatusNotFound, "Storage bucket key not found") } return nil } // DeleteStoragePoolBucketKey deletes an existing Storage Bucket Key. func (c *ClusterTx) DeleteStoragePoolBucketKey(ctx context.Context, bucketID int64, keyID int64) error { // Delete existing Storage Bucket record. res, err := c.tx.ExecContext(ctx, ` DELETE FROM storage_buckets_keys WHERE storage_bucket_id = ? and id = ? `, bucketID, keyID) if err != nil { return err } rowsAffected, err := res.RowsAffected() if err != nil { return err } if rowsAffected <= 0 { return api.StatusErrorf(http.StatusNotFound, "Storage bucket key not found") } return nil } // GetStoragePoolBucketWithID returns the volume with the given ID. func (c *ClusterTx) GetStoragePoolBucketWithID(ctx context.Context, bucketID int) (StorageBucket, error) { var response StorageBucket stmt := ` SELECT projects.name as project, storage_pools.name, storage_buckets.id, storage_buckets.storage_pool_id, storage_buckets.name, storage_buckets.description, IFNULL(nodes.name, "") as location FROM storage_buckets JOIN projects ON projects.id = storage_buckets.project_id JOIN storage_pools ON storage_pools.id = storage_buckets.storage_pool_id LEFT JOIN nodes ON nodes.id = storage_buckets.node_id WHERE storage_buckets.id = ? ` err := c.tx.QueryRowContext(ctx, stmt, bucketID).Scan(&response.Project, &response.PoolName, &response.ID, &response.PoolID, &response.Name, &response.Description, &response.Location) if err != nil { if errors.Is(err, sql.ErrNoRows) { return StorageBucket{}, api.StatusErrorf(http.StatusNotFound, "Storage pool bucket not found") } return StorageBucket{}, err } response.Config, err = c.storageVolumeConfigGet(ctx, response.ID, false) if err != nil { return StorageBucket{}, err } return response, nil } // GetStorageBucketURIs returns the URIs of the storage buckets, specifying // target node if applicable. func (c *ClusterTx) GetStorageBucketURIs(ctx context.Context, project string) ([]string, error) { filter := StorageBucketFilter{Project: &project} bucketInfo, err := c.GetStoragePoolBuckets(ctx, false, filter) if err != nil { return nil, err } uris := []string{} for _, info := range bucketInfo { uri := api.NewURL().Path(version.APIVersion, "storage-pools", info.PoolName, "buckets", info.Name).Project(project) if info.Location != "" { uri.Target(info.Location) } uris = append(uris, uri.String()) } return uris, nil } incus-7.0.0/internal/server/db/storage_pools.go000066400000000000000000000653241517523235500215770ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "database/sql" "errors" "fmt" "net/http" "slices" "strings" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/shared/api" ) // StorageRemoteDriverNames returns a list of remote storage driver names. var StorageRemoteDriverNames func() []string // GetStoragePoolsLocalConfig returns a map associating each storage pool name to // its node-specific config values (i.e. the ones where node_id is not NULL). func (c *ClusterTx) GetStoragePoolsLocalConfig(ctx context.Context) (map[string]map[string]string, error) { names, err := query.SelectStrings(ctx, c.tx, "SELECT name FROM storage_pools") if err != nil { return nil, err } pools := make(map[string]map[string]string, len(names)) for _, name := range names { table := ` storage_pools_config JOIN storage_pools ON storage_pools.id=storage_pools_config.storage_pool_id ` config, err := query.SelectConfig(ctx, c.tx, table, "storage_pools.name=? AND storage_pools_config.node_id=?", name, c.nodeID) if err != nil { return nil, err } pools[name] = config } return pools, nil } // GetStoragePoolID returns the ID of the pool with the given name. func (c *ClusterTx) GetStoragePoolID(ctx context.Context, name string) (int64, error) { stmt := "SELECT id FROM storage_pools WHERE name=?" ids, err := query.SelectIntegers(ctx, c.tx, stmt, name) if err != nil { return -1, err } switch len(ids) { case 0: return -1, api.StatusErrorf(http.StatusNotFound, "Storage pool not found") case 1: return int64(ids[0]), nil default: return -1, errors.New("More than one pool has the given name") } } // GetStoragePoolDriver returns the driver of the pool with the given ID. func (c *ClusterTx) GetStoragePoolDriver(ctx context.Context, id int64) (string, error) { stmt := "SELECT driver FROM storage_pools WHERE id=?" drivers, err := query.SelectStrings(ctx, c.tx, stmt, id) if err != nil { return "", err } switch len(drivers) { case 0: return "", api.StatusErrorf(http.StatusNotFound, "Storage pool not found") case 1: return drivers[0], nil default: return "", errors.New("More than one pool has the given id") } } // GetNonPendingStoragePoolsNamesToIDs returns a map associating each storage pool name to its ID. // // Pending storage pools are skipped. func (c *ClusterTx) GetNonPendingStoragePoolsNamesToIDs(ctx context.Context) (map[string]int64, error) { type pool struct { id int64 name string } sql := "SELECT id, name FROM storage_pools WHERE NOT state=?" pools := []pool{} err := query.Scan(ctx, c.tx, sql, func(scan func(dest ...any) error) error { var p pool err := scan(&p.id, &p.name) if err != nil { return err } pools = append(pools, p) return nil }, StoragePoolPending) if err != nil { return nil, err } ids := map[string]int64{} for _, pool := range pools { ids[pool.name] = pool.id } return ids, nil } // UpdateStoragePoolAfterNodeJoin adds a new entry in the storage_pools_nodes table. // // It should only be used when a new node joins the cluster, when it's safe to // assume that the relevant pool has already been created on the joining node, // and we just need to track it. func (c *ClusterTx) UpdateStoragePoolAfterNodeJoin(poolID, nodeID int64) error { columns := []string{"storage_pool_id", "node_id", "state"} // Create storage pool node with storagePoolCreated state as we expect the pool to already be setup. values := []any{poolID, nodeID, StoragePoolCreated} _, err := query.UpsertObject(c.tx, "storage_pools_nodes", columns, values) if err != nil { return fmt.Errorf("failed to add storage pools node entry: %w", err) } return nil } // UpdateRemoteStoragePoolAfterNodeJoin updates internal state to reflect that nodeID is // joining a cluster where poolID is a remote pool. func (c *ClusterTx) UpdateRemoteStoragePoolAfterNodeJoin(ctx context.Context, poolID int64, nodeID int64) error { // Get the IDs of the other servers (they should be all linked to the pool). stmt := "SELECT node_id FROM storage_pools_nodes WHERE storage_pool_id=?" nodeIDs, err := query.SelectIntegers(ctx, c.tx, stmt, poolID) if err != nil { return fmt.Errorf("Failed to fetch IDs of servers with remote pool: %w", err) } if len(nodeIDs) == 0 { return errors.New("Remote pool is not linked to any server") } otherNodeID := nodeIDs[0] // Create entries of all the volumes for the new server. _, err = c.tx.Exec(` INSERT INTO storage_volumes(name, storage_pool_id, node_id, type, description, project_id) SELECT name, storage_pool_id, ?, type, description, 1 FROM storage_volumes WHERE storage_pool_id=? AND node_id=? `, nodeID, poolID, otherNodeID) if err != nil { return fmt.Errorf("Failed to create volumes: %w", err) } // Create entries of all the volumes configs for the new server. stmt = ` SELECT id FROM storage_volumes WHERE storage_pool_id=? AND node_id=? ORDER BY name, type ` volumeIDs, err := query.SelectIntegers(ctx, c.tx, stmt, poolID, nodeID) if err != nil { return fmt.Errorf("Failed to get joining server's volume IDs: %w", err) } otherVolumeIDs, err := query.SelectIntegers(ctx, c.tx, stmt, poolID, otherNodeID) if err != nil { return fmt.Errorf("Failed to get other server's volume IDs: %w", err) } if len(volumeIDs) != len(otherVolumeIDs) { // Quick check. return errors.New("Not all remote volumes were copied") } for i, otherVolumeID := range otherVolumeIDs { volumeID := volumeIDs[i] config, err := query.SelectConfig(ctx, c.tx, "storage_volumes_config", "storage_volume_id=?", otherVolumeID) if err != nil { return fmt.Errorf("Failed to get storage volume config: %w", err) } for key, value := range config { _, err := c.tx.Exec(` INSERT INTO storage_volumes_config(storage_volume_id, key, value) VALUES(?, ?, ?) `, volumeID, key, value) if err != nil { return fmt.Errorf("Failed to copy volume config: %w", err) } } // Copy volume snapshots as well. otherSnapshotIDs, err := query.SelectIntegers(ctx, c.tx, "SELECT id FROM storage_volumes_snapshots WHERE storage_volume_id = ?", otherVolumeID) if err != nil { return err } for _, otherSnapshotID := range otherSnapshotIDs { var snapshotID int64 _, err := c.tx.Exec("UPDATE sqlite_sequence SET seq = seq + 1 WHERE name = 'storage_volumes'") if err != nil { return fmt.Errorf("Increment storage volumes sequence: %w", err) } row := c.tx.QueryRowContext(ctx, "SELECT seq FROM sqlite_sequence WHERE name = 'storage_volumes' LIMIT 1") err = row.Scan(&snapshotID) if err != nil { return fmt.Errorf("Fetch next storage volume ID: %w", err) } _, err = c.tx.Exec(` INSERT INTO storage_volumes_snapshots (id, storage_volume_id, name, description) SELECT ?, ?, name, description FROM storage_volumes_snapshots WHERE id=? `, snapshotID, volumeID, otherSnapshotID) if err != nil { return fmt.Errorf("Copy volume snapshot: %w", err) } _, err = c.tx.Exec(` INSERT INTO storage_volumes_snapshots_config (storage_volume_snapshot_id, key, value) SELECT ?, key, value FROM storage_volumes_snapshots_config WHERE storage_volume_snapshot_id=? `, snapshotID, otherSnapshotID) if err != nil { return fmt.Errorf("Copy volume snapshot config: %w", err) } } } return nil } // CreateStoragePoolConfig adds a new entry in the storage_pools_config table. func (c *ClusterTx) CreateStoragePoolConfig(poolID, nodeID int64, config map[string]string) error { return c.storagePoolConfigAdd(poolID, nodeID, config) } // StoragePoolState indicates the state of the storage pool or storage pool node. type StoragePoolState int // Storage pools state. const ( StoragePoolPending StoragePoolState = iota // Storage pool defined but not yet created globally or on specific node. StoragePoolCreated // Storage pool created globally or on specific node. storagePoolErrored // Deprecated (should no longer occur). ) // StoragePoolNode represents a storage pool node. type StoragePoolNode struct { ID int64 Name string State StoragePoolState } // CreatePendingStoragePool creates a new pending storage pool on the node with // the given name. func (c *ClusterTx) CreatePendingStoragePool(ctx context.Context, node string, name string, driver string, conf map[string]string) error { // First check if a storage pool with the given name exists, and, if // so, that it has a matching driver and it's in the pending state. pool := struct { id int64 driver string state StoragePoolState }{} sql := "SELECT id, driver, state FROM storage_pools WHERE name=?" count := 0 err := query.Scan(ctx, c.tx, sql, func(scan func(dest ...any) error) error { // Ensure that there is at most one pool with the given name. if count != 0 { return errors.New("more than one pool exists with the given name") } count++ return scan(&pool.id, &pool.driver, &pool.state) }, name) if err != nil { return err } poolID := pool.id if poolID == 0 { // No existing pool with the given name was found, let's create // one. columns := []string{"name", "driver", "description"} values := []any{name, driver, ""} poolID, err = query.UpsertObject(c.tx, "storage_pools", columns, values) if err != nil { return err } } else { // Check that the existing pools matches the given driver and // is in the pending state. if pool.driver != driver { return errors.New("Storage pool already exists with a different driver") } if pool.state != StoragePoolPending { return errors.New("Storage pool is not in pending state") } } // Get the ID of the node with the given name. nodeInfo, err := c.GetNodeByName(ctx, node) if err != nil { return err } // Check that no storage_pool entry of this node and pool exists yet. count, err = query.Count(ctx, c.tx, "storage_pools_nodes", "storage_pool_id=? AND node_id=?", poolID, nodeInfo.ID) if err != nil { return err } if count != 0 { return ErrAlreadyDefined } // Insert a node-specific entry pointing to ourselves with state storagePoolPending. columns := []string{"storage_pool_id", "node_id", "state"} values := []any{poolID, nodeInfo.ID, StoragePoolPending} _, err = query.UpsertObject(c.tx, "storage_pools_nodes", columns, values) if err != nil { return err } err = c.CreateStoragePoolConfig(poolID, nodeInfo.ID, conf) if err != nil { return err } return nil } // StoragePoolCreated sets the state of the given pool to storagePoolCreated. func (c *ClusterTx) StoragePoolCreated(name string) error { return c.storagePoolState(name, StoragePoolCreated) } // StoragePoolErrored sets the state of the given pool to storagePoolErrored. func (c *ClusterTx) StoragePoolErrored(name string) error { return c.storagePoolState(name, storagePoolErrored) } func (c *ClusterTx) storagePoolState(name string, state StoragePoolState) error { stmt := "UPDATE storage_pools SET state=? WHERE name=?" result, err := c.tx.Exec(stmt, state, name) if err != nil { return err } n, err := result.RowsAffected() if err != nil { return err } if n != 1 { return api.StatusErrorf(http.StatusNotFound, "Storage pool not found") } return nil } // storagePoolNodes returns the nodes keyed by node ID that the given storage pool is defined on. func (c *ClusterTx) storagePoolNodes(ctx context.Context, poolID int64) (map[int64]StoragePoolNode, error) { nodes := []StoragePoolNode{} sql := ` SELECT nodes.id, nodes.name, storage_pools_nodes.state FROM nodes JOIN storage_pools_nodes ON storage_pools_nodes.node_id = nodes.id WHERE storage_pools_nodes.storage_pool_id = ? ` err := query.Scan(ctx, c.tx, sql, func(scan func(dest ...any) error) error { node := StoragePoolNode{} err := scan(&node.ID, &node.Name, &node.State) if err != nil { return err } nodes = append(nodes, node) return nil }, poolID) if err != nil { return nil, err } poolNodes := map[int64]StoragePoolNode{} for _, node := range nodes { poolNodes[node.ID] = node } return poolNodes, nil } // StoragePoolNodeCreated sets the state of the given storage pool for the local member to storagePoolCreated. func (c *ClusterTx) StoragePoolNodeCreated(poolID int64) error { return c.storagePoolNodeState(poolID, StoragePoolCreated) } // storagePoolNodeState updates the storage pool member state for the local member and specified network ID. func (c *ClusterTx) storagePoolNodeState(poolID int64, state StoragePoolState) error { stmt := "UPDATE storage_pools_nodes SET state=? WHERE storage_pool_id = ? and node_id = ?" result, err := c.tx.Exec(stmt, state, poolID, c.nodeID) if err != nil { return err } n, err := result.RowsAffected() if err != nil { return err } if n != 1 { return api.StatusErrorf(http.StatusNotFound, "Storage pool not found") } return nil } // GetStoragePools returns map of Storage Pools keyed on ID and Storage Pool member info keyed on ID and Member ID. // Can optionally accept a state filter, if nil, then pools in any state are returned. // Can optionally accept one or more poolNames to further filter the returned pools. func (c *ClusterTx) GetStoragePools(ctx context.Context, state *StoragePoolState, poolNames ...string) (map[int64]api.StoragePool, map[int64]map[int64]StoragePoolNode, error) { var q *strings.Builder = &strings.Builder{} var args []any q.WriteString("SELECT id, name, driver, description, state FROM storage_pools ") if state != nil { q.WriteString("WHERE storage_pools.state = ? ") args = append(args, *state) } if len(poolNames) > 0 { verb := "WHERE" if len(args) > 0 { verb = "AND" } q.WriteString(fmt.Sprintf("%s storage_pools.name IN %s", verb, query.Params(len(poolNames)))) for _, poolName := range poolNames { args = append(args, poolName) } } var err error pools := make(map[int64]api.StoragePool) memberInfo := make(map[int64]map[int64]StoragePoolNode) err = query.Scan(ctx, c.tx, q.String(), func(scan func(dest ...any) error) error { var poolID int64 = int64(-1) var poolState StoragePoolState var pool api.StoragePool err := scan(&poolID, &pool.Name, &pool.Driver, &pool.Description, &poolState) if err != nil { return err } pool.Status = StoragePoolStateToAPIStatus(poolState) pools[poolID] = pool return nil }, args...) if err != nil { return nil, nil, err } for poolID := range pools { pool := pools[poolID] err = c.getStoragePoolConfig(ctx, c, poolID, &pool) if err != nil { return nil, nil, err } memberInfo[poolID], err = c.storagePoolNodes(ctx, poolID) if err != nil { return nil, nil, err } pool.Locations = make([]string, 0, len(memberInfo[poolID])) for _, node := range memberInfo[poolID] { pool.Locations = append(pool.Locations, node.Name) } pools[poolID] = pool } return pools, memberInfo, nil } // GetStoragePoolNodeConfigs returns the node-specific configuration of all // nodes grouped by node name, for the given poolID. // // If the storage pool is not defined on all nodes, an error is returned. func (c *ClusterTx) GetStoragePoolNodeConfigs(ctx context.Context, poolID int64) (map[string]map[string]string, error) { // Fetch all nodes. nodes, err := c.GetNodes(ctx) if err != nil { return nil, err } // Fetch the names of the nodes where the storage pool is defined. stmt := ` SELECT nodes.name FROM nodes LEFT JOIN storage_pools_nodes ON storage_pools_nodes.node_id = nodes.id LEFT JOIN storage_pools ON storage_pools_nodes.storage_pool_id = storage_pools.id WHERE storage_pools.id = ? AND storage_pools.state = ? ` defined, err := query.SelectStrings(ctx, c.tx, stmt, poolID, StoragePoolPending) if err != nil { return nil, err } // Figure which nodes are missing missing := []string{} for _, node := range nodes { if !slices.Contains(defined, node.Name) { missing = append(missing, node.Name) } } if len(missing) > 0 { return nil, fmt.Errorf("Pool not defined on nodes: %s", strings.Join(missing, ", ")) } configs := map[string]map[string]string{} for _, node := range nodes { config, err := query.SelectConfig(ctx, c.tx, "storage_pools_config", "storage_pool_id=? AND node_id=?", poolID, node.ID) if err != nil { return nil, err } configs[node.Name] = config } return configs, nil } // GetStoragePoolNames returns the names of all storage pools. func (c *ClusterTx) GetStoragePoolNames(ctx context.Context) ([]string, error) { return c.storagePools(ctx, "") } // GetCreatedStoragePoolNames returns the names of all storage pools that are created. func (c *ClusterTx) GetCreatedStoragePoolNames(ctx context.Context) ([]string, error) { return c.storagePools(ctx, "state=?", StoragePoolCreated) } // Get all storage pools matching the given WHERE filter (if given). func (c *ClusterTx) storagePools(ctx context.Context, where string, args ...any) ([]string, error) { var name string stmt := "SELECT name FROM storage_pools" inargs := []any{} outargs := []any{name} if where != "" { stmt += fmt.Sprintf(" WHERE %s", where) inargs = append(inargs, args...) } result, err := queryScan(ctx, c, stmt, inargs, outargs) if err != nil { return []string{}, err } if len(result) == 0 { return []string{}, api.StatusErrorf(http.StatusNotFound, "Storage pool(s) not found") } pools := []string{} for _, r := range result { pools = append(pools, r[0].(string)) } return pools, nil } // GetStoragePoolDrivers returns the names of all storage drivers currently // being used by at least one storage pool. func (c *ClusterTx) GetStoragePoolDrivers(ctx context.Context) ([]string, error) { var poolDriver string query := "SELECT DISTINCT driver FROM storage_pools" inargs := []any{} outargs := []any{poolDriver} result, err := queryScan(ctx, c, query, inargs, outargs) if err != nil { return []string{}, err } if len(result) == 0 { return []string{}, api.StatusErrorf(http.StatusNotFound, "Storage pool(s) not found") } drivers := []string{} for _, driver := range result { drivers = append(drivers, driver[0].(string)) } return drivers, nil } // GetStoragePool returns a single storage pool. // // The pool must be in the created stated, not pending. func (c *ClusterTx) GetStoragePool(ctx context.Context, poolName string) (int64, *api.StoragePool, map[int64]StoragePoolNode, error) { stateCreated := StoragePoolCreated pools, poolMembers, err := c.GetStoragePools(ctx, &stateCreated, poolName) if (err == nil && len(pools) <= 0) || errors.Is(err, sql.ErrNoRows) { return -1, nil, nil, api.StatusErrorf(http.StatusNotFound, "Storage pool not found") } else if err == nil && len(pools) > 1 { return -1, nil, nil, api.StatusErrorf(http.StatusConflict, "More than 1 storage pool found for that name") } else if err != nil { return -1, nil, nil, err } for poolID, pool := range pools { return poolID, &pool, poolMembers[poolID], err // Only single pool in map. } return -1, nil, nil, errors.New("Unexpected pool list size") } // GetStoragePoolInAnyState returns the storage pool with the given name. // // The pool can be in any state. func (c *ClusterTx) GetStoragePoolInAnyState(ctx context.Context, poolName string) (int64, *api.StoragePool, map[int64]StoragePoolNode, error) { pools, poolMembers, err := c.GetStoragePools(ctx, nil, poolName) if (err == nil && len(pools) <= 0) || errors.Is(err, sql.ErrNoRows) { return -1, nil, nil, api.StatusErrorf(http.StatusNotFound, "Storage pool not found") } else if err == nil && len(pools) > 1 { return -1, nil, nil, api.StatusErrorf(http.StatusConflict, "More than 1 storage pool found for that name") } else if err != nil { return -1, nil, nil, err } for poolID, pool := range pools { return poolID, &pool, poolMembers[poolID], err // Only single pool in map. } return -1, nil, nil, errors.New("Unexpected pool list size") } // GetStoragePoolWithID returns the storage pool with the given ID. func (c *ClusterTx) GetStoragePoolWithID(ctx context.Context, poolID int) (int64, *api.StoragePool, map[int64]StoragePoolNode, error) { return c.getStoragePool(ctx, true, "id=?", poolID) } // GetStoragePool returns a single storage pool. func (c *ClusterTx) getStoragePool(ctx context.Context, onlyCreated bool, where string, args ...any) (int64, *api.StoragePool, map[int64]StoragePoolNode, error) { var err error var q *strings.Builder = &strings.Builder{} q.WriteString("SELECT id, name, driver, description, state FROM storage_pools WHERE ") q.WriteString(where) if onlyCreated { q.WriteString(" AND state=?") args = append(args, StoragePoolCreated) } poolID := int64(-1) var pool api.StoragePool var nodes map[int64]StoragePoolNode var state StoragePoolState err = c.tx.QueryRowContext(ctx, q.String(), args...).Scan(&poolID, &pool.Name, &pool.Driver, &pool.Description, &state) if err != nil { if errors.Is(err, sql.ErrNoRows) { return -1, nil, nil, api.StatusErrorf(http.StatusNotFound, "Storage pool not found") } return -1, nil, nil, err } pool.Status = StoragePoolStateToAPIStatus(state) err = c.getStoragePoolConfig(ctx, c, poolID, &pool) if err != nil { if errors.Is(err, sql.ErrNoRows) { return -1, nil, nil, api.StatusErrorf(http.StatusNotFound, "Storage pool not found") } return -1, nil, nil, err } nodes, err = c.storagePoolNodes(ctx, poolID) if err != nil { if errors.Is(err, sql.ErrNoRows) { return -1, nil, nil, api.StatusErrorf(http.StatusNotFound, "Storage pool not found") } return -1, nil, nil, err } pool.Locations = make([]string, 0, len(nodes)) for _, node := range nodes { pool.Locations = append(pool.Locations, node.Name) } if err != nil { if errors.Is(err, sql.ErrNoRows) { return -1, nil, nil, api.StatusErrorf(http.StatusNotFound, "Storage pool not found") } return -1, nil, nil, err } return poolID, &pool, nodes, nil } // StoragePoolStateToAPIStatus converts DB StoragePoolState to API status string. func StoragePoolStateToAPIStatus(state StoragePoolState) string { switch state { case StoragePoolPending: return api.StoragePoolStatusPending case StoragePoolCreated: return api.StoragePoolStatusCreated case storagePoolErrored: return api.StoragePoolStatusErrored default: return api.StoragePoolStatusUnknown } } // getStoragePoolConfig populates the config map of the Storage pool with the given ID. func (c *ClusterTx) getStoragePoolConfig(ctx context.Context, tx *ClusterTx, poolID int64, pool *api.StoragePool) error { q := "SELECT key, value FROM storage_pools_config WHERE storage_pool_id=? AND (node_id=? OR node_id IS NULL)" pool.Config = map[string]string{} return query.Scan(ctx, c.tx, q, func(scan func(dest ...any) error) error { var key, value string err := scan(&key, &value) if err != nil { return err } _, found := pool.Config[key] if found { return fmt.Errorf("Duplicate config row found for key %q for storage pool ID %d", key, poolID) } pool.Config[key] = value return nil }, poolID, c.nodeID) } // CreateStoragePool creates new storage pool. Also creates a local member entry with state storagePoolPending. func (c *ClusterTx) CreateStoragePool(ctx context.Context, poolName string, poolDescription string, poolDriver string, poolConfig map[string]string) (int64, error) { var id int64 result, err := c.tx.ExecContext(ctx, "INSERT INTO storage_pools (name, description, driver, state) VALUES (?, ?, ?, ?)", poolName, poolDescription, poolDriver, StoragePoolCreated) if err != nil { return -1, err } id, err = result.LastInsertId() if err != nil { return -1, err } // Insert a node-specific entry pointing to ourselves with state storagePoolPending. columns := []string{"storage_pool_id", "node_id", "state"} values := []any{id, c.nodeID, StoragePoolPending} _, err = query.UpsertObject(c.tx, "storage_pools_nodes", columns, values) if err != nil { return -1, err } err = c.storagePoolConfigAdd(id, c.nodeID, poolConfig) if err != nil { return -1, err } return id, nil } // Add new storage pool config. func (c *ClusterTx) storagePoolConfigAdd(poolID, nodeID int64, poolConfig map[string]string) error { str := "INSERT INTO storage_pools_config (storage_pool_id, node_id, key, value) VALUES(?, ?, ?, ?)" stmt, err := c.tx.Prepare(str) if err != nil { return err } defer func() { _ = stmt.Close() }() driver, err := c.GetStoragePoolDriver(context.Background(), poolID) if err != nil { return err } nodeSpecificConfig := NodeSpecificStorageConfig(driver) for k, v := range poolConfig { if v == "" { continue } var nodeIDValue any if !slices.Contains(nodeSpecificConfig, k) { nodeIDValue = nil } else { nodeIDValue = nodeID } _, err = stmt.Exec(poolID, nodeIDValue, k, v) if err != nil { return err } } return nil } // UpdateStoragePool updates a storage pool. func (c *ClusterTx) UpdateStoragePool(ctx context.Context, poolName, description string, poolConfig map[string]string) error { poolID, _, _, err := c.GetStoragePoolInAnyState(ctx, poolName) if err != nil { return err } err = updateStoragePoolDescription(c.tx, poolID, description) if err != nil { return err } return c.UpdateStoragePoolConfig(poolID, c.nodeID, poolConfig) } // UpdateStoragePoolConfig updates a storage pool config. func (c *ClusterTx) UpdateStoragePoolConfig(poolID, nodeID int64, config map[string]string) error { err := clearStoragePoolConfig(c.tx, poolID, nodeID) if err != nil { return err } return c.storagePoolConfigAdd(poolID, nodeID, config) } // Uupdate the storage pool description. func updateStoragePoolDescription(tx *sql.Tx, id int64, description string) error { _, err := tx.Exec("UPDATE storage_pools SET description=? WHERE id=?", description, id) return err } // Delete the storage pool config. func clearStoragePoolConfig(tx *sql.Tx, poolID, nodeID int64) error { _, err := tx.Exec("DELETE FROM storage_pools_config WHERE storage_pool_id=? AND (node_id=? OR node_id IS NULL)", poolID, nodeID) if err != nil { return err } return nil } // RemoveStoragePool deletes storage pool. func (c *ClusterTx) RemoveStoragePool(ctx context.Context, poolName string) (*api.StoragePool, error) { poolID, pool, _, err := c.GetStoragePoolInAnyState(ctx, poolName) if err != nil { return nil, err } _, err = c.tx.ExecContext(ctx, "DELETE FROM storage_pools WHERE id=?", poolID) if err != nil { return nil, err } return pool, nil } // NodeSpecificStorageConfig lists all storage pool config keys which are node-specific. func NodeSpecificStorageConfig(driverName string) []string { configKeys := []string{ "source", "source.wipe", "volatile.initial_source", "zfs.pool_name", "lvm.thinpool_name", "lvm.vg_name", "lvm.vg.force_reuse", } if driverName != "lvmcluster" { configKeys = append(configKeys, "size") } return configKeys } // IsRemoteStorage return whether a given pool is backed by remote storage. func (c *ClusterTx) IsRemoteStorage(ctx context.Context, poolID int64) (bool, error) { driver, err := c.GetStoragePoolDriver(ctx, poolID) if err != nil { return false, err } isRemoteStorage := slices.Contains(StorageRemoteDriverNames(), driver) return isRemoteStorage, nil } incus-7.0.0/internal/server/db/storage_pools_test.go000066400000000000000000000227171517523235500226350ustar00rootroot00000000000000//go:build linux && cgo && !agent package db_test import ( "context" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/shared/api" ) func init() { db.StorageRemoteDriverNames = func() []string { // Tests only use ceph. return []string{"ceph"} } } // The GetStoragePoolsLocalConfigs method returns only node-specific config values. func TestGetStoragePoolsLocalConfigs(t *testing.T) { cluster, cleanup := db.NewTestCluster(t) defer cleanup() _ = cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Create a storage pool named "local" (like the default clustering one), then delete it and create another one. _, err := tx.CreateStoragePool(ctx, "local", "", "dir", map[string]string{ "rsync.bwlimit": "1", "source": "/foo/bar", }) require.NoError(t, err) _, err = tx.RemoveStoragePool(ctx, "local") require.NoError(t, err) _, err = tx.CreateStoragePool(ctx, "BTRFS", "", "dir", map[string]string{ "rsync.bwlimit": "1", "source": "/egg/baz", }) require.NoError(t, err) return nil }) // Check that the config map returned by StoragePoolsConfigs actually // contains the value of the "BTRFS" storage pool. var config map[string]map[string]string err := cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error config, err = tx.GetStoragePoolsLocalConfig(ctx) return err }) require.NoError(t, err) assert.Equal(t, config, map[string]map[string]string{ "BTRFS": {"source": "/egg/baz"}, }) } func TestStoragePoolsCreatePending(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() _, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) _, err = tx.CreateNode("rusp", "5.6.7.8:666") require.NoError(t, err) config := map[string]string{"source": "/foo"} err = tx.CreatePendingStoragePool(context.Background(), "buzz", "pool1", "dir", config) require.NoError(t, err) poolID, err := tx.GetStoragePoolID(context.Background(), "pool1") require.NoError(t, err) assert.True(t, poolID > 0) config = map[string]string{"source": "/bar"} err = tx.CreatePendingStoragePool(context.Background(), "rusp", "pool1", "dir", config) require.NoError(t, err) // The initial node (whose name is 'none' by default) is missing. _, err = tx.GetStoragePoolNodeConfigs(context.Background(), poolID) require.EqualError(t, err, "Pool not defined on nodes: none") config = map[string]string{"source": "/egg"} err = tx.CreatePendingStoragePool(context.Background(), "none", "pool1", "dir", config) require.NoError(t, err) // Now the storage is defined on all nodes. configs, err := tx.GetStoragePoolNodeConfigs(context.Background(), poolID) require.NoError(t, err) assert.Len(t, configs, 3) assert.Equal(t, map[string]string{"source": "/foo"}, configs["buzz"]) assert.Equal(t, map[string]string{"source": "/bar"}, configs["rusp"]) assert.Equal(t, map[string]string{"source": "/egg"}, configs["none"]) } func TestStoragePoolsCreatePending_OtherPool(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() // Create a pending pool named 'pool1' on two nodes (the default 'none' // and 'buzz') _, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) config := map[string]string{"source": "/foo"} err = tx.CreatePendingStoragePool(context.Background(), "none", "pool1", "dir", config) require.NoError(t, err) config = map[string]string{"source": "/bar"} err = tx.CreatePendingStoragePool(context.Background(), "buzz", "pool1", "dir", config) require.NoError(t, err) // Create a second pending pool named pool2 on the same two nodes. config = map[string]string{} err = tx.CreatePendingStoragePool(context.Background(), "none", "pool2", "dir", config) require.NoError(t, err) poolID, err := tx.GetStoragePoolID(context.Background(), "pool2") require.NoError(t, err) config = map[string]string{} err = tx.CreatePendingStoragePool(context.Background(), "buzz", "pool2", "dir", config) require.NoError(t, err) // The node-level configs of the second pool do not contain any key // from the first pool. configs, err := tx.GetStoragePoolNodeConfigs(context.Background(), poolID) require.NoError(t, err) assert.Len(t, configs, 2) assert.Equal(t, map[string]string{}, configs["none"]) assert.Equal(t, map[string]string{}, configs["buzz"]) } // If an entry for the given pool and node already exists, an error is // returned. func TestStoragePoolsCreatePending_AlreadyDefined(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() _, err := tx.CreateNode("buzz", "1.2.3.4:666") require.NoError(t, err) err = tx.CreatePendingStoragePool(context.Background(), "buzz", "pool1", "dir", map[string]string{}) require.NoError(t, err) err = tx.CreatePendingStoragePool(context.Background(), "buzz", "pool1", "dir", map[string]string{}) require.Equal(t, db.ErrAlreadyDefined, err) } // If no node with the given name is found, an error is returned. func TestStoragePoolsCreatePending_NonExistingNode(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() err := tx.CreatePendingStoragePool(context.Background(), "buzz", "pool1", "dir", map[string]string{}) require.True(t, response.IsNotFoundError(err)) } // If a pool with the given name already exists but has different driver, an // error is returned. Likewise, if volume is updated or deleted, it's updated // or deleted on all nodes. func TestStoragePoolVolume_Ceph(t *testing.T) { cluster, cleanup := db.NewTestCluster(t) defer cleanup() // Create a second node (beyond the default one). err := cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, err := tx.CreateNode("n1", "1.2.3.4:666") return err }) require.NoError(t, err) var poolID int64 _ = cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { poolID, err = tx.CreateStoragePool(ctx, "p1", "", "ceph", nil) require.NoError(t, err) return nil }) config := api.ConfigMap{"k": "v"} var volumeID int64 _ = cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { volumeID, err = tx.CreateStoragePoolVolume(ctx, "default", "v1", "", 1, poolID, config, db.StoragePoolVolumeContentTypeFS, time.Now()) return err }) require.NoError(t, err) getStoragePoolVolume := func(volumeProjectName string, volumeName string, volumeType int, poolID int64) (*db.StorageVolume, error) { var dbVolume *db.StorageVolume err = cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(context.Background(), poolID, volumeProjectName, volumeType, volumeName, true) return err }) if err != nil { return nil, err } return dbVolume, nil } // The returned volume ID is the one of the volume created on the local // node (node 1). thisVolume, err := getStoragePoolVolume("default", "v1", 1, poolID) require.NoError(t, err) assert.NotNil(t, thisVolume) assert.Equal(t, volumeID, thisVolume.ID) assert.Equal(t, thisVolume.Location, "") // Update the volume config["k"] = "v2" err = cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateStoragePoolVolume(ctx, "default", "v1", 1, poolID, "volume 1", config) }) require.NoError(t, err) volume, err := getStoragePoolVolume("default", "v1", 1, poolID) require.NoError(t, err) assert.Equal(t, "volume 1", volume.Description) assert.Equal(t, config, volume.Config) err = cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RenameStoragePoolVolume(ctx, "default", "v1", "v1-new", 1, poolID) }) require.NoError(t, err) volume, err = getStoragePoolVolume("default", "v1-new", 1, poolID) require.NoError(t, err) assert.NotNil(t, volume) // Delete the volume err = cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RemoveStoragePoolVolume(ctx, "default", "v1-new", 1, poolID) }) require.NoError(t, err) volume, err = getStoragePoolVolume("default", "v1-new", 1, poolID) assert.True(t, response.IsNotFoundError(err)) assert.Nil(t, volume) } // Test creating a volume snapshot. func TestCreateStoragePoolVolume_Snapshot(t *testing.T) { cluster, cleanup := db.NewTestCluster(t) defer cleanup() var poolID int64 var poolID1 int64 _ = cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error poolID, err = tx.CreateStoragePool(ctx, "p1", "", "dir", nil) require.NoError(t, err) poolID1, err = tx.CreateStoragePool(ctx, "p2", "", "dir", nil) require.NoError(t, err) config := map[string]string{"k": "v"} _, err = tx.CreateStoragePoolVolume(ctx, "default", "v1", "", 1, poolID, config, db.StoragePoolVolumeContentTypeFS, time.Now()) require.NoError(t, err) _, err = tx.CreateStoragePoolVolume(ctx, "default", "v1", "", 1, poolID1, config, db.StoragePoolVolumeContentTypeFS, time.Now()) require.NoError(t, err) config = map[string]string{"k": "v"} _, err = tx.CreateStorageVolumeSnapshot(ctx, "default", "v1/snap0", "", 1, poolID, config, time.Now(), time.Time{}) require.NoError(t, err) n := tx.GetNextStorageVolumeSnapshotIndex(ctx, "p1", "v1", 1, "snap%d") assert.Equal(t, n, 1) n = tx.GetNextStorageVolumeSnapshotIndex(ctx, "p2", "v1", 1, "snap%d") assert.Equal(t, n, 0) return nil }) } incus-7.0.0/internal/server/db/storage_volume_snapshots.go000066400000000000000000000160311517523235500240430ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "database/sql" "errors" "fmt" "net/http" "strings" "time" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/shared/api" ) // CreateStorageVolumeSnapshot creates a new storage volume snapshot attached to a given // storage pool. func (c *ClusterTx) CreateStorageVolumeSnapshot(ctx context.Context, projectName string, volumeName string, volumeDescription string, volumeType int, poolID int64, volumeConfig map[string]string, creationDate time.Time, expiryDate time.Time) (int64, error) { var volumeID int64 var snapshotName string parts := strings.Split(volumeName, internalInstance.SnapshotDelimiter) volumeName = parts[0] snapshotName = parts[1] // Figure out the volume ID of the parent. parentID, err := c.storagePoolVolumeGetTypeID(ctx, projectName, volumeName, volumeType, poolID, c.nodeID) if err != nil { return -1, fmt.Errorf("Failed finding parent volume record for snapshot: %w", err) } _, err = c.tx.ExecContext(ctx, "UPDATE sqlite_sequence SET seq = seq + 1 WHERE name = 'storage_volumes'") if err != nil { return -1, fmt.Errorf("Failed incrementing storage volumes sequence: %w", err) } row := c.tx.QueryRowContext(ctx, "SELECT seq FROM sqlite_sequence WHERE name = 'storage_volumes' LIMIT 1") err = row.Scan(&volumeID) if err != nil { return -1, fmt.Errorf("Failed getting storage volumes sequence: %w", err) } _, err = c.tx.ExecContext(ctx, "INSERT INTO storage_volumes_snapshots (id, storage_volume_id, name, description, creation_date, expiry_date) VALUES (?, ?, ?, ?, ?, ?)", volumeID, parentID, snapshotName, volumeDescription, creationDate, expiryDate) if err != nil { return -1, fmt.Errorf("Failed creating volume snapshot record: %w", err) } err = storageVolumeConfigAdd(c.tx, volumeID, volumeConfig, true) if err != nil { return -1, fmt.Errorf("Failed inserting storage volume snapshot record configuration: %w", err) } return volumeID, nil } // UpdateStorageVolumeSnapshot updates the storage volume snapshot attached to a given storage pool. func (c *ClusterTx) UpdateStorageVolumeSnapshot(ctx context.Context, projectName string, volumeName string, volumeType int, poolID int64, volumeDescription string, volumeConfig map[string]string, expiryDate time.Time) error { var err error if !strings.Contains(volumeName, internalInstance.SnapshotDelimiter) { return errors.New("Volume is not a snapshot") } volume, err := c.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, volumeName, true) if err != nil { return err } err = storageVolumeConfigClear(c.tx, volume.ID, true) if err != nil { return err } err = storageVolumeConfigAdd(c.tx, volume.ID, volumeConfig, true) if err != nil { return err } err = storageVolumeDescriptionUpdate(c.tx, volume.ID, volumeDescription, true) if err != nil { return err } err = storageVolumeSnapshotExpiryDateUpdate(c.tx, volume.ID, expiryDate) if err != nil { return err } return nil } // GetStorageVolumeSnapshotWithID returns the volume snapshot with the given ID. func (c *ClusterTx) GetStorageVolumeSnapshotWithID(ctx context.Context, snapshotID int) (StorageVolumeArgs, error) { args := StorageVolumeArgs{} q := ` SELECT volumes.id, volumes.name, volumes.creation_date, storage_pools.name, volumes.type, projects.name FROM storage_volumes_all AS volumes JOIN projects ON projects.id=volumes.project_id JOIN storage_pools ON storage_pools.id=volumes.storage_pool_id WHERE volumes.id=? ` arg1 := []any{snapshotID} outfmt := []any{&args.ID, &args.Name, &args.CreationDate, &args.PoolName, &args.Type, &args.ProjectName} err := dbQueryRowScan(ctx, c, q, arg1, outfmt) if err != nil { if errors.Is(err, sql.ErrNoRows) { return args, api.StatusErrorf(http.StatusNotFound, "Storage pool volume snapshot not found") } return args, err } if !strings.Contains(args.Name, internalInstance.SnapshotDelimiter) { return args, errors.New("Volume is not a snapshot") } args.TypeName = StoragePoolVolumeTypeNames[args.Type] return args, nil } // GetStorageVolumeSnapshotExpiry gets the expiry date of a storage volume snapshot. func (c *ClusterTx) GetStorageVolumeSnapshotExpiry(ctx context.Context, volumeID int64) (time.Time, error) { var expiry time.Time query := "SELECT expiry_date FROM storage_volumes_snapshots WHERE id=?" inargs := []any{volumeID} outargs := []any{&expiry} err := dbQueryRowScan(ctx, c, query, inargs, outargs) if err != nil { if errors.Is(err, sql.ErrNoRows) { return expiry, api.StatusErrorf(http.StatusNotFound, "Storage pool volume snapshot not found") } return expiry, err } return expiry, nil } // GetExpiredStorageVolumeSnapshots returns a list of expired volume snapshots. // If memberSpecific is true, then the search is restricted to volumes that belong to this member or belong to // all members. func (c *ClusterTx) GetExpiredStorageVolumeSnapshots(ctx context.Context, memberSpecific bool) ([]StorageVolumeArgs, error) { var q strings.Builder q.WriteString(` SELECT storage_volumes_snapshots.id, storage_volumes.name, storage_volumes_snapshots.name, storage_volumes_snapshots.creation_date, storage_volumes_snapshots.expiry_date, storage_pools.name, projects.name, IFNULL(storage_volumes.node_id, -1) FROM storage_volumes_snapshots JOIN storage_volumes ON storage_volumes_snapshots.storage_volume_id = storage_volumes.id JOIN storage_pools ON storage_volumes.storage_pool_id = storage_pools.id JOIN projects ON storage_volumes.project_id = projects.id WHERE storage_volumes.type = ? AND storage_volumes_snapshots.expiry_date != '0001-01-01T00:00:00Z' `) args := []any{StoragePoolVolumeTypeCustom} if memberSpecific { q.WriteString("AND (storage_volumes.node_id = ? OR storage_volumes.node_id IS NULL) ") args = append(args, c.nodeID) } var snapshots []StorageVolumeArgs err := query.Scan(ctx, c.Tx(), q.String(), func(scan func(dest ...any) error) error { var snap StorageVolumeArgs var snapName string var volName string var expiryTime sql.NullTime err := scan(&snap.ID, &volName, &snapName, &snap.CreationDate, &expiryTime, &snap.PoolName, &snap.ProjectName, &snap.NodeID) if err != nil { return err } snap.Name = volName + internalInstance.SnapshotDelimiter + snapName snap.ExpiryDate = expiryTime.Time // Convert nulls to zero. // Since zero time causes some issues due to timezones, we check the // unix timestamp instead of IsZero(). if snap.ExpiryDate.Unix() <= 0 { return nil // Backup doesn't expire. } // Check if snapshot has expired. if time.Now().Unix()-snap.ExpiryDate.Unix() >= 0 { snapshots = append(snapshots, snap) } return nil }, args...) if err != nil { return nil, err } return snapshots, nil } // Updates the expiry date of a storage volume snapshot. func storageVolumeSnapshotExpiryDateUpdate(tx *sql.Tx, volumeID int64, expiryDate time.Time) error { stmt := "UPDATE storage_volumes_snapshots SET expiry_date=? WHERE id=?" _, err := tx.Exec(stmt, expiryDate, volumeID) return err } incus-7.0.0/internal/server/db/storage_volumes.go000066400000000000000000000710721517523235500221320ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "database/sql" "errors" "fmt" "net/http" "slices" "strings" "time" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/db/query" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // GetStoragePoolVolumesWithType return a list of all volumes of the given type. // If memberSpecific is true, then the search is restricted to volumes that belong to this member or belong to // all members. func (c *ClusterTx) GetStoragePoolVolumesWithType(ctx context.Context, volumeType int, memberSpecific bool) ([]StorageVolumeArgs, error) { var q strings.Builder q.WriteString(` SELECT storage_volumes.id, storage_volumes.name, storage_volumes.description, storage_volumes.creation_date, storage_pools.name, projects.name, IFNULL(storage_volumes.node_id, -1) FROM storage_volumes JOIN storage_pools ON storage_pools.id = storage_volumes.storage_pool_id JOIN projects ON projects.id = storage_volumes.project_id WHERE storage_volumes.type = ? `) args := []any{volumeType} if memberSpecific { q.WriteString("AND (storage_volumes.node_id = ? OR storage_volumes.node_id IS NULL) ") args = append(args, c.nodeID) } result := []StorageVolumeArgs{} err := query.Scan(ctx, c.Tx(), q.String(), func(scan func(dest ...any) error) error { entry := StorageVolumeArgs{} err := scan(&entry.ID, &entry.Name, &entry.Description, &entry.CreationDate, &entry.PoolName, &entry.ProjectName, &entry.NodeID) if err != nil { return err } result = append(result, entry) return nil }, args...) if err != nil { return nil, err } for i := range result { result[i].Config, err = c.storageVolumeConfigGet(ctx, result[i].ID, false) if err != nil { return nil, err } } return result, nil } // GetStoragePoolVolumeWithID returns the volume with the given ID. func (c *ClusterTx) GetStoragePoolVolumeWithID(ctx context.Context, volumeID int) (StorageVolumeArgs, error) { var response StorageVolumeArgs stmt := ` SELECT storage_volumes.id, storage_volumes.name, storage_volumes.description, storage_volumes.creation_date, storage_volumes.type, IFNULL(storage_volumes.node_id, -1), storage_pools.name, projects.name FROM storage_volumes JOIN storage_pools ON storage_pools.id = storage_volumes.storage_pool_id JOIN projects ON projects.id = storage_volumes.project_id LEFT JOIN nodes ON nodes.id = storage_volumes.node_id WHERE storage_volumes.id = ? ` err := c.tx.QueryRowContext(ctx, stmt, volumeID).Scan(&response.ID, &response.Name, &response.Description, &response.CreationDate, &response.Type, &response.NodeID, &response.PoolName, &response.ProjectName) if err != nil { if errors.Is(err, sql.ErrNoRows) { return StorageVolumeArgs{}, api.StatusErrorf(http.StatusNotFound, "Storage pool volume not found") } return StorageVolumeArgs{}, err } response.Config, err = c.storageVolumeConfigGet(ctx, response.ID, false) if err != nil { return StorageVolumeArgs{}, err } response.TypeName = StoragePoolVolumeTypeNames[response.Type] return response, nil } // StorageVolumeFilter used for filtering storage volumes with GetStoragePoolVolumes(). type StorageVolumeFilter struct { Type *int Project *string Name *string } // StorageVolume represents a database storage volume record. type StorageVolume struct { api.StorageVolume ID int64 } // GetStoragePoolVolumes returns all storage volumes attached to a given storage pool. // If there are no volumes, it returns an empty list and no error. // Accepts filters for narrowing down the results returned. If memberSpecific is true, then the search is // restricted to volumes that belong to this member or belong to all members. func (c *ClusterTx) GetStoragePoolVolumes(ctx context.Context, poolID int64, memberSpecific bool, filters ...StorageVolumeFilter) ([]*StorageVolume, error) { var q *strings.Builder = &strings.Builder{} args := []any{poolID} q.WriteString(` SELECT projects.name as project, storage_volumes_all.id, storage_volumes_all.name, IFNULL(nodes.name, "") as location, storage_volumes_all.type, storage_volumes_all.content_type, storage_volumes_all.description, storage_volumes_all.creation_date FROM storage_volumes_all JOIN projects ON projects.id = storage_volumes_all.project_id LEFT JOIN nodes ON nodes.id = storage_volumes_all.node_id WHERE storage_volumes_all.storage_pool_id = ? `) if memberSpecific { q.WriteString("AND (storage_volumes_all.node_id = ? OR storage_volumes_all.node_id IS NULL) ") args = append(args, c.nodeID) } if len(filters) > 0 { q.WriteString("AND (") for i, filter := range filters { // Validate filter. if filter.Name != nil && filter.Type == nil { return nil, errors.New("Cannot filter by volume name if volume type not specified") } if filter.Name != nil && filter.Project == nil { return nil, errors.New("Cannot filter by volume name if volume project not specified") } var qFilters []string if filter.Type != nil { qFilters = append(qFilters, "storage_volumes_all.type = ?") args = append(args, *filter.Type) } if filter.Project != nil { qFilters = append(qFilters, "projects.name = ?") args = append(args, *filter.Project) } if filter.Name != nil { qFilters = append(qFilters, "storage_volumes_all.name = ?") args = append(args, *filter.Name) } if qFilters == nil { return nil, errors.New("Invalid storage volume filter") } if i > 0 { q.WriteString(" OR ") } q.WriteString(fmt.Sprintf("(%s)", strings.Join(qFilters, " AND "))) } q.WriteString(")") } var err error var volumes []*StorageVolume err = query.Scan(ctx, c.Tx(), q.String(), func(scan func(dest ...any) error) error { var volumeType int = int(-1) var contentType int = int(-1) var vol StorageVolume err := scan(&vol.Project, &vol.ID, &vol.Name, &vol.Location, &volumeType, &contentType, &vol.Description, &vol.CreatedAt) if err != nil { return err } vol.Type, err = StoragePoolVolumeTypeToName(volumeType) if err != nil { return err } vol.ContentType, err = storagePoolVolumeContentTypeToName(contentType) if err != nil { return err } volumes = append(volumes, &vol) return nil }, args...) if err != nil { return nil, err } // Populate config. for _, volume := range volumes { volume.Config, err = c.storageVolumeConfigGet(ctx, volume.ID, internalInstance.IsSnapshot(volume.Name)) if err != nil { return nil, fmt.Errorf("Failed loading volume config for %q: %w", volume.Name, err) } } return volumes, nil } // GetStoragePoolVolume returns the storage volume attached to a given storage pool. func (c *ClusterTx) GetStoragePoolVolume(ctx context.Context, poolID int64, projectName string, volumeType int, volumeName string, memberSpecific bool) (*StorageVolume, error) { filters := []StorageVolumeFilter{{ Project: &projectName, Type: &volumeType, Name: &volumeName, }} volumes, err := c.GetStoragePoolVolumes(ctx, poolID, memberSpecific, filters...) volumesLen := len(volumes) if (err == nil && volumesLen <= 0) || errors.Is(err, sql.ErrNoRows) { return nil, api.StatusErrorf(http.StatusNotFound, "Storage volume not found") } else if err == nil && volumesLen > 1 { return nil, api.StatusErrorf(http.StatusConflict, "Storage volume found on more than one cluster member. Please target a specific member") } else if err != nil { return nil, err } return volumes[0], nil } // GetLocalStoragePoolVolumeSnapshotsWithType get all snapshots of a storage volume // attached to a given storage pool of a given volume type, on the local member. // Returns snapshots slice ordered by when they were created, oldest first. func (c *ClusterTx) GetLocalStoragePoolVolumeSnapshotsWithType(ctx context.Context, projectName string, volumeName string, volumeType int, poolID int64) ([]StorageVolumeArgs, error) { remoteDrivers := StorageRemoteDriverNames() // ORDER BY creation_date and then id is important here as the users of this function can expect that the // results will be returned in the order that the snapshots were created. This is specifically used // during migration to ensure that the storage engines can re-create snapshots using the // correct deltas. queryStr := fmt.Sprintf(` SELECT storage_volumes_snapshots.id, storage_volumes_snapshots.name, storage_volumes_snapshots.description, storage_volumes_snapshots.creation_date, storage_volumes_snapshots.expiry_date, storage_volumes.content_type FROM storage_volumes_snapshots JOIN storage_volumes ON storage_volumes_snapshots.storage_volume_id = storage_volumes.id JOIN projects ON projects.id=storage_volumes.project_id JOIN storage_pools ON storage_pools.id=storage_volumes.storage_pool_id WHERE storage_volumes.storage_pool_id=? AND storage_volumes.type=? AND storage_volumes.name=? AND projects.name=? AND (storage_volumes.node_id=? OR storage_volumes.node_id IS NULL AND storage_pools.driver IN %s) ORDER BY storage_volumes_snapshots.creation_date, storage_volumes_snapshots.id`, query.Params(len(remoteDrivers))) args := []any{poolID, volumeType, volumeName, projectName, c.nodeID} for _, driver := range remoteDrivers { args = append(args, driver) } var snapshots []StorageVolumeArgs err := query.Scan(ctx, c.Tx(), queryStr, func(scan func(dest ...any) error) error { var s StorageVolumeArgs var snapName string var expiryDate sql.NullTime var contentType int err := scan(&s.ID, &snapName, &s.Description, &s.CreationDate, &expiryDate, &contentType) if err != nil { return err } s.Name = volumeName + internalInstance.SnapshotDelimiter + snapName s.PoolID = poolID s.ProjectName = projectName s.Snapshot = true s.ExpiryDate = expiryDate.Time // Convert null to zero. s.ContentType, err = storagePoolVolumeContentTypeToName(contentType) if err != nil { return err } snapshots = append(snapshots, s) return nil }, args...) if err != nil { return nil, err } // Populate config. for i := range snapshots { err := storageVolumeSnapshotConfig(ctx, c, snapshots[i].ID, &snapshots[i]) if err != nil { return nil, err } } return snapshots, nil } // storageVolumeSnapshotConfig populates the config map of the Storage Volume Snapshot with the given ID. func storageVolumeSnapshotConfig(ctx context.Context, tx *ClusterTx, volumeSnapshotID int64, volume *StorageVolumeArgs) error { q := "SELECT key, value FROM storage_volumes_snapshots_config WHERE storage_volume_snapshot_id = ?" volume.Config = make(map[string]string) return query.Scan(ctx, tx.Tx(), q, func(scan func(dest ...any) error) error { var key, value string err := scan(&key, &value) if err != nil { return err } _, found := volume.Config[key] if found { return fmt.Errorf("Duplicate config row found for key %q for storage volume snapshot ID %d", key, volumeSnapshotID) } volume.Config[key] = value return nil }, volumeSnapshotID) } // UpdateStoragePoolVolume updates the storage volume attached to a given storage pool. func (c *ClusterTx) UpdateStoragePoolVolume(ctx context.Context, projectName string, volumeName string, volumeType int, poolID int64, volumeDescription string, volumeConfig map[string]string) error { isSnapshot := strings.Contains(volumeName, internalInstance.SnapshotDelimiter) volume, err := c.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, volumeName, true) if err != nil { return err } err = storageVolumeConfigClear(c.tx, volume.ID, isSnapshot) if err != nil { return err } err = storageVolumeConfigAdd(c.tx, volume.ID, volumeConfig, isSnapshot) if err != nil { return err } err = storageVolumeDescriptionUpdate(c.tx, volume.ID, volumeDescription, isSnapshot) if err != nil { return err } return nil } // RemoveStoragePoolVolume deletes the storage volume attached to a given storage // pool. func (c *ClusterTx) RemoveStoragePoolVolume(ctx context.Context, projectName string, volumeName string, volumeType int, poolID int64) error { isSnapshot := strings.Contains(volumeName, internalInstance.SnapshotDelimiter) var stmt string if isSnapshot { stmt = "DELETE FROM storage_volumes_snapshots WHERE id=?" } else { stmt = "DELETE FROM storage_volumes WHERE id=?" } volume, err := c.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, volumeName, true) if err != nil { return err } _, err = c.tx.ExecContext(ctx, stmt, volume.ID) if err != nil { return err } return nil } // RenameStoragePoolVolume renames the storage volume attached to a given storage pool. func (c *ClusterTx) RenameStoragePoolVolume(ctx context.Context, projectName string, oldVolumeName string, newVolumeName string, volumeType int, poolID int64) error { isSnapshot := strings.Contains(oldVolumeName, internalInstance.SnapshotDelimiter) var stmt string if isSnapshot { parts := strings.Split(newVolumeName, internalInstance.SnapshotDelimiter) newVolumeName = parts[1] stmt = "UPDATE storage_volumes_snapshots SET name=? WHERE id=?" } else { stmt = "UPDATE storage_volumes SET name=? WHERE id=?" } volume, err := c.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, oldVolumeName, true) if err != nil { return err } _, err = c.tx.ExecContext(ctx, stmt, newVolumeName, volume.ID) if err != nil { return err } return nil } // CreateStoragePoolVolume creates a new storage volume attached to a given storage pool. func (c *ClusterTx) CreateStoragePoolVolume(ctx context.Context, projectName string, volumeName string, volumeDescription string, volumeType int, poolID int64, volumeConfig map[string]string, contentType int, creationDate time.Time) (int64, error) { var volumeID int64 if internalInstance.IsSnapshot(volumeName) { return -1, errors.New("Volume name may not be a snapshot") } remoteDrivers := StorageRemoteDriverNames() driver, err := c.GetStoragePoolDriver(ctx, poolID) if err != nil { return -1, err } var result sql.Result if slices.Contains(remoteDrivers, driver) { result, err = c.tx.ExecContext(ctx, ` INSERT INTO storage_volumes (storage_pool_id, type, name, description, project_id, content_type, creation_date) VALUES (?, ?, ?, ?, (SELECT id FROM projects WHERE name = ?), ?, ?) `, poolID, volumeType, volumeName, volumeDescription, projectName, contentType, creationDate) } else { result, err = c.tx.ExecContext(ctx, ` INSERT INTO storage_volumes (storage_pool_id, node_id, type, name, description, project_id, content_type, creation_date) VALUES (?, ?, ?, ?, ?, (SELECT id FROM projects WHERE name = ?), ?, ?) `, poolID, c.nodeID, volumeType, volumeName, volumeDescription, projectName, contentType, creationDate) } if err != nil { return -1, err } volumeID, err = result.LastInsertId() if err != nil { return -1, err } err = storageVolumeConfigAdd(c.tx, volumeID, volumeConfig, false) if err != nil { return -1, fmt.Errorf("Failed inserting storage volume record configuration: %w", err) } return volumeID, err } // Return the ID of a storage volume on a given storage pool of a given storage // volume type, on the given node. func (c *ClusterTx) storagePoolVolumeGetTypeID(ctx context.Context, project string, volumeName string, volumeType int, poolID, nodeID int64) (int64, error) { remoteDrivers := StorageRemoteDriverNames() s := fmt.Sprintf(` SELECT storage_volumes_all.id FROM storage_volumes_all JOIN storage_pools ON storage_volumes_all.storage_pool_id = storage_pools.id JOIN projects ON storage_volumes_all.project_id = projects.id WHERE projects.name=? AND storage_volumes_all.storage_pool_id=? AND storage_volumes_all.name=? AND storage_volumes_all.type=? AND (storage_volumes_all.node_id=? OR storage_volumes_all.node_id IS NULL AND storage_pools.driver IN %s)`, query.Params(len(remoteDrivers))) args := []any{project, poolID, volumeName, volumeType, nodeID} for _, driver := range remoteDrivers { args = append(args, driver) } result, err := query.SelectIntegers(ctx, c.tx, s, args...) if err != nil { return -1, err } if len(result) == 0 { return -1, api.StatusErrorf(http.StatusNotFound, "Storage pool volume not found") } return int64(result[0]), nil } // GetStoragePoolNodeVolumeID gets the ID of a storage volume on a given storage pool // of a given storage volume type and project, on the current node. func (c *ClusterTx) GetStoragePoolNodeVolumeID(ctx context.Context, projectName string, volumeName string, volumeType int, poolID int64) (int64, error) { return c.storagePoolVolumeGetTypeID(ctx, projectName, volumeName, volumeType, poolID, c.nodeID) } // XXX: this was extracted from storage_volume_utils.go, we find a way to // factor it independently from both the db and main packages. const ( StoragePoolVolumeTypeContainer = iota StoragePoolVolumeTypeImage StoragePoolVolumeTypeCustom StoragePoolVolumeTypeVM ) // Leave the string type in here! This guarantees that go treats this is as a // typed string constant. Removing it causes go to treat these as untyped string // constants which is not what we want. const ( StoragePoolVolumeTypeNameContainer string = "container" StoragePoolVolumeTypeNameVM string = "virtual-machine" StoragePoolVolumeTypeNameImage string = "image" StoragePoolVolumeTypeNameCustom string = "custom" ) // StoragePoolVolumeTypeNames represents a map of storage volume types and their names. var StoragePoolVolumeTypeNames = map[int]string{ StoragePoolVolumeTypeContainer: "container", StoragePoolVolumeTypeImage: "image", StoragePoolVolumeTypeCustom: "custom", StoragePoolVolumeTypeVM: "virtual-machine", } // Content types. const ( StoragePoolVolumeContentTypeFS = iota StoragePoolVolumeContentTypeBlock StoragePoolVolumeContentTypeISO ) // Content type names. const ( StoragePoolVolumeContentTypeNameFS string = "filesystem" StoragePoolVolumeContentTypeNameBlock string = "block" StoragePoolVolumeContentTypeNameISO string = "iso" ) // StorageVolumeArgs is a value object holding all db-related details about a // storage volume. type StorageVolumeArgs struct { ID int64 Name string // At least one of Type or TypeName must be set. Type int TypeName string // At least one of PoolID or PoolName must be set. PoolID int64 PoolName string Snapshot bool Config map[string]string Description string CreationDate time.Time ExpiryDate time.Time // At least on of ProjectID or ProjectName must be set. ProjectID int64 ProjectName string ContentType string NodeID int64 } // GetStorageVolumeNodes returns the node info of all nodes on which the volume with the given name is defined. // The volume name can be either a regular name or a volume snapshot name. // If the volume is defined, but without a specific node, then the ErrNoClusterMember error is returned. // If the volume is not found then an api.StatusError with code set to http.StatusNotFound is returned. func (c *ClusterTx) GetStorageVolumeNodes(ctx context.Context, poolID int64, projectName string, volumeName string, volumeType int) ([]NodeInfo, error) { nodes := []NodeInfo{} sql := ` SELECT coalesce(nodes.id,0) AS nodeID, coalesce(nodes.address,"") AS nodeAddress, coalesce(nodes.name,"") AS nodeName FROM storage_volumes_all JOIN projects ON projects.id = storage_volumes_all.project_id LEFT JOIN nodes ON storage_volumes_all.node_id=nodes.id WHERE storage_volumes_all.storage_pool_id=? AND projects.name=? AND storage_volumes_all.name=? AND storage_volumes_all.type=? ` err := query.Scan(ctx, c.tx, sql, func(scan func(dest ...any) error) error { node := NodeInfo{} err := scan(&node.ID, &node.Address, &node.Name) if err != nil { return err } nodes = append(nodes, node) return nil }, poolID, projectName, volumeName, volumeType) if err != nil { return nil, err } for _, node := range nodes { // Volume is defined without a cluster member. if node.ID == 0 { return nil, ErrNoClusterMember } } nodeCount := len(nodes) if nodeCount == 0 { return nil, api.StatusErrorf(http.StatusNotFound, "Storage pool volume not found") } else if nodeCount > 1 { driver, err := c.GetStoragePoolDriver(ctx, poolID) if err != nil { return nil, err } // Earlier schema versions created a volume DB record for each cluster member for remote storage // pools, so if the storage driver is one of those remote pools and the addressCount is >1 then we // take this to mean that the volume doesn't have an explicit cluster member and is therefore // equivalent to db.ErrNoClusterMember that is used in newer schemas where a single remote volume // DB record is created that is not associated to any single member. if StorageRemoteDriverNames == nil { return nil, errors.New("No remote storage drivers function defined") } remoteDrivers := StorageRemoteDriverNames() if slices.Contains(remoteDrivers, driver) { return nil, ErrNoClusterMember } } return nodes, nil } // Get the config of a storage volume. func (c *ClusterTx) storageVolumeConfigGet(ctx context.Context, volumeID int64, isSnapshot bool) (map[string]string, error) { var queryStr string if isSnapshot { queryStr = "SELECT key, value FROM storage_volumes_snapshots_config WHERE storage_volume_snapshot_id=?" } else { queryStr = "SELECT key, value FROM storage_volumes_config WHERE storage_volume_id=?" } config := map[string]string{} err := query.Scan(ctx, c.Tx(), queryStr, func(scan func(dest ...any) error) error { var key string var value string err := scan(&key, &value) if err != nil { return err } config[key] = value return nil }, volumeID) if err != nil { return nil, err } return config, nil } // GetNextStorageVolumeSnapshotIndex returns the index of the next snapshot of the storage // volume with the given name should have. // // Note, the code below doesn't deal with snapshots of snapshots. // To do that, we'll need to weed out based on # slashes in names. func (c *ClusterTx) GetNextStorageVolumeSnapshotIndex(ctx context.Context, pool, name string, typ int, pattern string) int { remoteDrivers := StorageRemoteDriverNames() q := fmt.Sprintf(` SELECT storage_volumes_snapshots.name FROM storage_volumes_snapshots JOIN storage_volumes ON storage_volumes_snapshots.storage_volume_id=storage_volumes.id JOIN storage_pools ON storage_volumes.storage_pool_id=storage_pools.id WHERE storage_volumes.type=? AND storage_volumes.name=? AND storage_pools.name=? AND (storage_volumes.node_id=? OR storage_volumes.node_id IS NULL AND storage_pools.driver IN %s) `, query.Params(len(remoteDrivers))) var numstr string inargs := []any{typ, name, pool, c.nodeID} outfmt := []any{numstr} for _, driver := range remoteDrivers { inargs = append(inargs, driver) } results, err := queryScan(ctx, c, q, inargs, outfmt) if err != nil { return 0 } max := 0 for _, r := range results { substr := r[0].(string) fields := strings.SplitN(pattern, "%d", 2) var num int count, err := fmt.Sscanf(substr, fmt.Sprintf("%s%%d%s", fields[0], fields[1]), &num) if err != nil || count != 1 { continue } if num >= max { max = num + 1 } } return max } // Updates the description of a storage volume. func storageVolumeDescriptionUpdate(tx *sql.Tx, volumeID int64, description string, isSnapshot bool) error { var table string if isSnapshot { table = "storage_volumes_snapshots" } else { table = "storage_volumes" } stmt := fmt.Sprintf("UPDATE %s SET description=? WHERE id=?", table) _, err := tx.Exec(stmt, description, volumeID) return err } // Add a new storage volume config into database. func storageVolumeConfigAdd(tx *sql.Tx, volumeID int64, volumeConfig map[string]string, isSnapshot bool) error { var str string if isSnapshot { str = "INSERT INTO storage_volumes_snapshots_config (storage_volume_snapshot_id, key, value) VALUES(?, ?, ?)" } else { str = "INSERT INTO storage_volumes_config (storage_volume_id, key, value) VALUES(?, ?, ?)" } stmt, err := tx.Prepare(str) if err != nil { return err } defer func() { _ = stmt.Close() }() for k, v := range volumeConfig { if v == "" { continue } _, err = stmt.Exec(volumeID, k, v) if err != nil { return err } } return nil } // Delete storage volume config. func storageVolumeConfigClear(tx *sql.Tx, volumeID int64, isSnapshot bool) error { var stmt string if isSnapshot { stmt = "DELETE FROM storage_volumes_snapshots_config WHERE storage_volume_snapshot_id=?" } else { stmt = "DELETE FROM storage_volumes_config WHERE storage_volume_id=?" } _, err := tx.Exec(stmt, volumeID) if err != nil { return err } return nil } // StoragePoolVolumeTypeToName converts a volume integer type code to its human-readable name. func StoragePoolVolumeTypeToName(volumeType int) (string, error) { switch volumeType { case StoragePoolVolumeTypeContainer: return StoragePoolVolumeTypeNameContainer, nil case StoragePoolVolumeTypeVM: return StoragePoolVolumeTypeNameVM, nil case StoragePoolVolumeTypeImage: return StoragePoolVolumeTypeNameImage, nil case StoragePoolVolumeTypeCustom: return StoragePoolVolumeTypeNameCustom, nil } return "", errors.New("Invalid storage volume type") } // Convert a volume integer content type code to its human-readable name. func storagePoolVolumeContentTypeToName(contentType int) (string, error) { switch contentType { case StoragePoolVolumeContentTypeFS: return StoragePoolVolumeContentTypeNameFS, nil case StoragePoolVolumeContentTypeBlock: return StoragePoolVolumeContentTypeNameBlock, nil case StoragePoolVolumeContentTypeISO: return StoragePoolVolumeContentTypeNameISO, nil } return "", errors.New("Invalid storage volume content type") } // GetCustomVolumesInProject returns all custom volumes in the given project. func (c *ClusterTx) GetCustomVolumesInProject(ctx context.Context, project string) ([]StorageVolumeArgs, error) { sql := ` SELECT storage_volumes.id, storage_volumes.name, storage_volumes.creation_date, storage_pools.name, IFNULL(storage_volumes.node_id, -1) FROM storage_volumes JOIN storage_pools ON storage_pools.id = storage_volumes.storage_pool_id JOIN projects ON projects.id = storage_volumes.project_id WHERE storage_volumes.type = ? AND projects.name = ? ` volumes := []StorageVolumeArgs{} err := query.Scan(ctx, c.tx, sql, func(scan func(dest ...any) error) error { volume := StorageVolumeArgs{} err := scan(&volume.ID, &volume.Name, &volume.CreationDate, &volume.PoolName, &volume.NodeID) if err != nil { return err } volumes = append(volumes, volume) return nil }, StoragePoolVolumeTypeCustom, project) if err != nil { return nil, fmt.Errorf("Fetch custom volumes: %w", err) } for i, volume := range volumes { config, err := query.SelectConfig(ctx, c.tx, "storage_volumes_config", "storage_volume_id=?", volume.ID) if err != nil { return nil, fmt.Errorf("Fetch custom volume config: %w", err) } volumes[i].Config = config } return volumes, nil } // GetStorageVolumeURIs returns the URIs of the storage volumes, specifying // target node if applicable. func (c *ClusterTx) GetStorageVolumeURIs(ctx context.Context, project string) ([]string, error) { volInfo, err := c.GetCustomVolumesInProject(ctx, project) if err != nil { return nil, err } uris := []string{} for _, info := range volInfo { uri := api.NewURL().Path(version.APIVersion, "storage-pools", info.PoolName, "volumes", "custom", info.Name).Project(project) // Skip checking nodes if node_id is NULL. if info.NodeID != -1 { nodeInfo, err := c.GetNodes(ctx) if err != nil { return nil, err } for _, node := range nodeInfo { if node.ID == info.NodeID { uri.Target(node.Name) break } } } uris = append(uris, uri.String()) } return uris, nil } // UpdateStorageVolumeNode changes the name of a storage volume and the cluster member hosting it. // It's meant to be used when moving a storage volume backed by a remote storage pool from one cluster node to another. func (c *ClusterTx) UpdateStorageVolumeNode(ctx context.Context, projectName string, oldName string, newName string, newMemberName string, poolID int64, volumeType int) error { volume, err := c.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, oldName, false) if err != nil { return err } member, err := c.GetNodeByName(ctx, newMemberName) if err != nil { return fmt.Errorf("Failed to get new member %q info: %w", newMemberName, err) } stmt := "UPDATE storage_volumes SET node_id=?, name=? WHERE id=?" result, err := c.tx.Exec(stmt, member.ID, newName, volume.ID) if err != nil { return fmt.Errorf("Failed to update volumes's name and member ID: %w", err) } n, err := result.RowsAffected() if err != nil { return fmt.Errorf("Failed to get rows affected by volume update: %w", err) } if n != 1 { return fmt.Errorf("Unexpected number of updated rows in storage_volumes table: %d", n) } return nil } incus-7.0.0/internal/server/db/storage_volumes_test.go000066400000000000000000000033001517523235500231560ustar00rootroot00000000000000//go:build linux && cgo && !agent package db_test import ( "context" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" ) // Addresses of all nodes with matching volume name are returned. func TestGetStorageVolumeNodes(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() nodeID1 := int64(1) // This is the default local member nodeID2, err := tx.CreateNode("node2", "1.2.3.4:666") require.NoError(t, err) nodeID3, err := tx.CreateNode("node3", "5.6.7.8:666") require.NoError(t, err) poolID := addPool(t, tx, "pool1") addVolume(t, tx, poolID, nodeID1, "volume1") addVolume(t, tx, poolID, nodeID2, "volume1") addVolume(t, tx, poolID, nodeID3, "volume2") addVolume(t, tx, poolID, nodeID2, "volume2") nodes, err := tx.GetStorageVolumeNodes(context.Background(), poolID, "default", "volume1", 1) require.NoError(t, err) assert.Equal(t, []db.NodeInfo{ { ID: nodeID1, Name: "none", Address: "0.0.0.0", }, { ID: nodeID2, Name: "node2", Address: "1.2.3.4:666", }, }, nodes) } func addPool(t *testing.T, tx *db.ClusterTx, name string) int64 { stmt := ` INSERT INTO storage_pools(name, driver, description) VALUES (?, 'dir', '') ` result, err := tx.Tx().Exec(stmt, name) require.NoError(t, err) id, err := result.LastInsertId() require.NoError(t, err) return id } func addVolume(t *testing.T, tx *db.ClusterTx, poolID, nodeID int64, name string) { stmt := ` INSERT INTO storage_volumes(storage_pool_id, node_id, name, type, project_id, description) VALUES (?, ?, ?, 1, 1, '') ` _, err := tx.Tx().Exec(stmt, poolID, nodeID, name) require.NoError(t, err) } incus-7.0.0/internal/server/db/testing.go000066400000000000000000000076011517523235500203660ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "errors" "fmt" "io/fs" "net" "os" "path/filepath" "testing" "time" dqlite "github.com/cowsql/go-cowsql" "github.com/cowsql/go-cowsql/client" "github.com/cowsql/go-cowsql/driver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // NewTestNode creates a new Node for testing purposes, along with a function // that can be used to clean it up when done. func NewTestNode(t *testing.T) (*Node, func()) { dir, err := os.MkdirTemp("", "incus-db-test-node-") require.NoError(t, err) db, err := OpenNode(dir, nil) require.NoError(t, err) cleanup := func() { require.NoError(t, db.Close()) require.NoError(t, os.RemoveAll(dir)) } return db, cleanup } // NewTestNodeTx returns a fresh NodeTx object, along with a function that can // be called to cleanup state when done with it. func NewTestNodeTx(t *testing.T) (*NodeTx, func()) { node, nodeCleanup := NewTestNode(t) var err error nodeTx := &NodeTx{} nodeTx.tx, err = node.db.Begin() require.NoError(t, err) cleanup := func() { require.NoError(t, nodeTx.tx.Commit()) nodeCleanup() } return nodeTx, cleanup } // NewTestCluster creates a new Cluster for testing purposes, along with a function // that can be used to clean it up when done. func NewTestCluster(t *testing.T) (*Cluster, func()) { // Create an in-memory dqlite SQL server and associated store. dir, store, serverCleanup := NewTestDqliteServer(t) log := newLogFunc(t) dial := func(ctx context.Context, address string) (net.Conn, error) { return net.Dial("unix", address) } cluster, err := OpenCluster(context.Background(), "test.db", store, "1", dir, 5*time.Second, driver.WithLogFunc(log), driver.WithDialFunc(dial)) require.NoError(t, err) cleanup := func() { require.NoError(t, cluster.Close()) serverCleanup() } return cluster, cleanup } // NewTestClusterTx returns a fresh ClusterTx object, along with a function that can // be called to cleanup state when done with it. func NewTestClusterTx(t *testing.T) (*ClusterTx, func()) { cluster, clusterCleanup := NewTestCluster(t) var err error clusterTx := &ClusterTx{nodeID: cluster.nodeID} clusterTx.tx, err = cluster.db.Begin() require.NoError(t, err) cleanup := func() { err := clusterTx.tx.Commit() require.NoError(t, err) clusterCleanup() } return clusterTx, cleanup } // NewTestDqliteServer creates a new test dqlite server. // // Return the directory backing the test server and a newly created server // store that can be used to connect to it. func NewTestDqliteServer(t *testing.T) (string, driver.NodeStore, func()) { t.Helper() listener, err := net.Listen("unix", "") require.NoError(t, err) address := listener.Addr().String() require.NoError(t, listener.Close()) dir, dirCleanup := newDir(t) err = os.Mkdir(filepath.Join(dir, "global"), 0o755) require.NoError(t, err) server, err := dqlite.New( uint64(1), address, filepath.Join(dir, "global"), dqlite.WithBindAddress(address)) require.NoError(t, err) err = server.Start() require.NoError(t, err) cleanup := func() { require.NoError(t, server.Close()) dirCleanup() } store, err := driver.DefaultNodeStore(":memory:") require.NoError(t, err) ctx := context.Background() require.NoError(t, store.Set(ctx, []driver.NodeInfo{{Address: address}})) return dir, store, cleanup } // Return a new temporary directory. func newDir(t *testing.T) (string, func()) { t.Helper() dir, err := os.MkdirTemp("", "dqlite-replication-test-") assert.NoError(t, err) cleanup := func() { _, err := os.Stat(dir) if err != nil { assert.True(t, errors.Is(err, fs.ErrNotExist)) } else { assert.NoError(t, os.RemoveAll(dir)) } } return dir, cleanup } func newLogFunc(t *testing.T) client.LogFunc { return func(l client.LogLevel, format string, a ...any) { format = fmt.Sprintf("%s: %s", l.String(), format) t.Logf(format, a...) } } incus-7.0.0/internal/server/db/transaction.go000066400000000000000000000020111517523235500212240ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "database/sql" ) // NodeTx models a single interaction with a server-local database. // // It wraps low-level sql.Tx objects and offers a high-level API to fetch and // update data. type NodeTx struct { tx *sql.Tx // Handle to a transaction in the node-level SQLite database. } // ClusterTx models a single interaction with a cluster database. // // It wraps low-level sql.Tx objects and offers a high-level API to fetch and // update data. type ClusterTx struct { tx *sql.Tx // Handle to a transaction in the cluster dqlite database. nodeID int64 // Node ID of this server. } // Tx retrieves the underlying transaction on the cluster database. func (c *ClusterTx) Tx() *sql.Tx { return c.tx } // NodeID sets the node NodeID associated with this cluster transaction. func (c *ClusterTx) NodeID(id int64) { c.nodeID = id } // GetNodeID gets the ID of the node associated with this cluster transaction. func (c *ClusterTx) GetNodeID() int64 { return c.nodeID } incus-7.0.0/internal/server/db/warnings.go000066400000000000000000000150051517523235500205360ustar00rootroot00000000000000//go:build linux && cgo && !agent package db import ( "context" "errors" "fmt" "net/http" "slices" "time" "github.com/google/uuid" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/warningtype" "github.com/lxc/incus/v7/shared/api" ) var warningCreate = cluster.RegisterStmt(` INSERT INTO warnings (node_id, project_id, entity_type_code, entity_id, uuid, type_code, status, first_seen_date, last_seen_date, updated_date, last_message, count) VALUES ((SELECT nodes.id FROM nodes WHERE nodes.name = ?), (SELECT projects.id FROM projects WHERE projects.name = ?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `) // UpsertWarningLocalNode creates or updates a warning for the local member. Returns error if no local member name. func (c *ClusterTx) UpsertWarningLocalNode(ctx context.Context, projectName string, entityTypeCode int, entityID int, typeCode warningtype.Type, message string) error { localName, err := c.GetLocalNodeName(ctx) if err != nil { return fmt.Errorf("Failed getting local member name: %w", err) } if localName == "" { return errors.New("Local member name not available") } return c.UpsertWarning(ctx, localName, projectName, entityTypeCode, entityID, typeCode, message) } // UpsertWarning creates or updates a warning. func (c *ClusterTx) UpsertWarning(ctx context.Context, nodeName string, projectName string, entityTypeCode int, entityID int, typeCode warningtype.Type, message string) error { // Validate _, err := c.GetURIFromEntity(ctx, entityTypeCode, entityID) if err != nil { return fmt.Errorf("Failed to get URI for entity ID %d with entity type code %d: %w", entityID, entityTypeCode, err) } _, ok := warningtype.TypeNames[typeCode] if !ok { return fmt.Errorf("Unknown warning type code %d", typeCode) } now := time.Now().UTC() filter := cluster.WarningFilter{ TypeCode: &typeCode, Node: &nodeName, Project: &projectName, EntityTypeCode: &entityTypeCode, EntityID: &entityID, } warnings, err := cluster.GetWarnings(ctx, c.tx, filter) if err != nil { return fmt.Errorf("Failed to retrieve warnings: %w", err) } if len(warnings) > 1 { // This shouldn't happen return fmt.Errorf("More than one warnings (%d) match the criteria: typeCode: %d, nodeName: %q, projectName: %q, entityTypeCode: %d, entityID: %d", len(warnings), typeCode, nodeName, projectName, entityTypeCode, entityID) } else if len(warnings) == 1 { // If there is a historical warning that was previously automatically resolved and the same // warning has now reoccurred then set the status back to warningtype.StatusNew so it shows as // a current active warning. newStatus := warnings[0].Status if newStatus == warningtype.StatusResolved { newStatus = warningtype.StatusNew } err = c.UpdateWarningState(warnings[0].UUID, message, newStatus) } else { warning := cluster.Warning{ Node: nodeName, Project: projectName, EntityTypeCode: entityTypeCode, EntityID: entityID, UUID: uuid.New().String(), TypeCode: typeCode, Status: warningtype.StatusNew, FirstSeenDate: now, LastSeenDate: now, UpdatedDate: time.Time{}.UTC(), LastMessage: message, Count: 1, } _, err = c.createWarning(ctx, warning) } if err != nil { return err } return nil } // UpdateWarningStatus updates the status of the warning with the given UUID. func (c *ClusterTx) UpdateWarningStatus(UUID string, status warningtype.Status) error { str := "UPDATE warnings SET status=?, updated_date=? WHERE uuid=?" res, err := c.tx.Exec(str, status, time.Now(), UUID) if err != nil { return fmt.Errorf("Failed to update warning status for warning %q: %w", UUID, err) } rowsAffected, err := res.RowsAffected() if err != nil { return fmt.Errorf("Failed to get affected rows to update warning status %q: %w", UUID, err) } if rowsAffected == 0 { return api.StatusErrorf(http.StatusNotFound, "Warning not found") } return nil } // UpdateWarningState updates the warning message and status with the given ID. func (c *ClusterTx) UpdateWarningState(UUID string, message string, status warningtype.Status) error { str := "UPDATE warnings SET last_message=?, last_seen_date=?, updated_date=?, status = ?, count=count+1 WHERE uuid=?" now := time.Now() res, err := c.tx.Exec(str, message, now, now, status, UUID) if err != nil { return fmt.Errorf("Failed to update warning %q: %w", UUID, err) } rowsAffected, err := res.RowsAffected() if err != nil { return fmt.Errorf("Failed to get affected rows to update warning state %q: %w", UUID, err) } if rowsAffected == 0 { return api.StatusErrorf(http.StatusNotFound, "Warning not found") } return nil } // createWarning adds a new warning to the database. func (c *ClusterTx) createWarning(ctx context.Context, object cluster.Warning) (int64, error) { // Check if a warning with the same key exists. exists, err := cluster.WarningExists(ctx, c.tx, object.UUID) if err != nil { return -1, fmt.Errorf("Failed to check for duplicates: %w", err) } if exists { return -1, errors.New("This warning already exists") } args := make([]any, 12) // Populate the statement arguments. if object.Node != "" { // Ensure node exists _, err = c.GetNodeByName(ctx, object.Node) if err != nil { return -1, fmt.Errorf("Failed to get node: %w", err) } args[0] = object.Node } if object.Project != "" { // Ensure project exists projects, err := cluster.GetProjectNames(context.Background(), c.tx) if err != nil { return -1, fmt.Errorf("Failed to get project names: %w", err) } if !slices.Contains(projects, object.Project) { return -1, fmt.Errorf("Unknown project %q", object.Project) } args[1] = object.Project } if object.EntityTypeCode != -1 { args[2] = object.EntityTypeCode } if object.EntityID != -1 { args[3] = object.EntityID } args[4] = object.UUID args[5] = object.TypeCode args[6] = object.Status args[7] = object.FirstSeenDate args[8] = object.LastSeenDate args[9] = object.UpdatedDate args[10] = object.LastMessage args[11] = object.Count // Prepared statement to use. stmt, err := cluster.Stmt(c.tx, warningCreate) if err != nil { return -1, fmt.Errorf("Failed to get \"warningCreate\" prepared statement: %w", err) } // Execute the statement. result, err := stmt.Exec(args...) if err != nil { return -1, fmt.Errorf("Failed to create warning: %w", err) } id, err := result.LastInsertId() if err != nil { return -1, fmt.Errorf("Failed to fetch warning ID: %w", err) } return id, nil } incus-7.0.0/internal/server/db/warningtype/000077500000000000000000000000001517523235500207255ustar00rootroot00000000000000incus-7.0.0/internal/server/db/warningtype/warning_severity.go000066400000000000000000000013371517523235500246570ustar00rootroot00000000000000//go:build linux && cgo && !agent package warningtype // Severity represents the warning severity. type Severity int const ( // SeverityLow represents the low Severity. SeverityLow Severity = 1 // SeverityModerate represents the moderate Severity. SeverityModerate Severity = 2 // SeverityHigh represents the high Severity. SeverityHigh Severity = 3 ) // Severities associates a severity code to its name. var Severities = map[Severity]string{ SeverityLow: "low", SeverityModerate: "moderate", SeverityHigh: "high", } // SeverityTypes associates a warning severity to its type code. var SeverityTypes = map[string]Severity{} func init() { for code, name := range Severities { SeverityTypes[name] = code } } incus-7.0.0/internal/server/db/warningtype/warning_status.go000066400000000000000000000013511517523235500243240ustar00rootroot00000000000000//go:build linux && cgo && !agent package warningtype // Status represents the warning status. type Status int const ( // StatusNew represents the New WarningStatus. StatusNew Status = 1 // StatusAcknowledged represents the Acknowledged WarningStatus. StatusAcknowledged Status = 2 // StatusResolved represents the Resolved WarningStatus. StatusResolved Status = 3 ) // Statuses associates a warning code to its name. var Statuses = map[Status]string{ StatusNew: "new", StatusAcknowledged: "acknowledged", StatusResolved: "resolved", } // StatusTypes associates a warning status to its type code. var StatusTypes = map[string]Status{} func init() { for code, name := range Statuses { StatusTypes[name] = code } } incus-7.0.0/internal/server/db/warningtype/warning_type.go000066400000000000000000000145431517523235500237710ustar00rootroot00000000000000//go:build linux && cgo && !agent package warningtype // Type is a numeric code identifying the type of warning. type Type int const ( // Undefined represents an undefined warning. Undefined Type = iota // MissingCGroupBlkio represents the missing CGroup blkio warning. MissingCGroupBlkio // MissingCGroupBlkioWeight represents the missing CGroup blkio.weight warning. MissingCGroupBlkioWeight // MissingCGroupCPUController represents the missing CGroup CPU controller warning. MissingCGroupCPUController // MissingCGroupCPUsetController represents the missing GCgroup CPUset controller warning. MissingCGroupCPUsetController // MissingCGroupCPUacctController represents the missing GCgroup CPUacct controller warning. MissingCGroupCPUacctController // MissingCGroupDevicesController represents the missing GCgroup devices controller warning. MissingCGroupDevicesController // MissingCGroupFreezerController represents the missing GCgroup freezer controller warning. MissingCGroupFreezerController // MissingCGroupHugetlbController represents the missing GCgroup hugetlb controller warning. MissingCGroupHugetlbController // MissingCGroupMemoryController represents the missing GCgroup memory controller warning. MissingCGroupMemoryController // Spacer where MissingCGroupNetworkPriorityController used to be. _ // MissingCGroupPidsController represents the missing GCgroup pids controller warning. MissingCGroupPidsController // MissingCGroupMemorySwapAccounting represents the missing GCgroup memory swap accounting warning. MissingCGroupMemorySwapAccounting // ClusterTimeSkew represents the cluster time skew warning. ClusterTimeSkew // AppArmorNotAvailable represents the AppArmor not available warning. AppArmorNotAvailable // MissingVirtiofsd represents the missing virtiofsd warning. MissingVirtiofsd // AppArmorDisabledDueToRawDnsmasq represents the disabled AppArmor due to raw.dnsmasq warning. AppArmorDisabledDueToRawDnsmasq // LargerIPv6PrefixThanSupported represents the larger IPv6 prefix than supported warning. LargerIPv6PrefixThanSupported // ProxyBridgeNetfilterNotEnabled represents the proxy bridge netfilter not enable warning. ProxyBridgeNetfilterNotEnabled // NetworkUnvailable represents a network that cannot be initialized on the local server. NetworkUnvailable // OfflineClusterMember represents the offline cluster members warning. OfflineClusterMember // InstanceAutostartFailure represents the failure of instance autostart process after three retries. InstanceAutostartFailure // InstanceTypeNotOperational represents the lack of support for an instance driver. InstanceTypeNotOperational // StoragePoolUnvailable represents a storage pool that cannot be initialized on the local server. StoragePoolUnvailable // UnableToUpdateClusterCertificate represents the unable to update cluster certificate warning. UnableToUpdateClusterCertificate // SELinuxNotAvailable represents the SELinux not available warning. SELinuxNotAvailable ) // TypeNames associates a warning code to its name. var TypeNames = map[Type]string{ Undefined: "Undefined warning", MissingCGroupBlkio: "Couldn't find the CGroup blkio", MissingCGroupBlkioWeight: "Couldn't find the CGroup blkio.weight", MissingCGroupCPUController: "Couldn't find the CGroup CPU controller", MissingCGroupCPUsetController: "Couldn't find the CGroup CPUset controller", MissingCGroupCPUacctController: "Couldn't find the CGroup CPUacct controller", MissingCGroupDevicesController: "Couldn't find the CGroup devices controller", MissingCGroupFreezerController: "Couldn't find the CGroup freezer controller", MissingCGroupHugetlbController: "Couldn't find the CGroup hugetlb controller", MissingCGroupMemoryController: "Couldn't find the CGroup memory controller", MissingCGroupPidsController: "Couldn't find the CGroup pids controller", MissingCGroupMemorySwapAccounting: "Couldn't find the CGroup memory swap accounting", ClusterTimeSkew: "Time skew detected between leader and local", AppArmorNotAvailable: "AppArmor support has been disabled", MissingVirtiofsd: "Missing virtiofsd", AppArmorDisabledDueToRawDnsmasq: "Skipping AppArmor for dnsmasq due to raw.dnsmasq being set", LargerIPv6PrefixThanSupported: "IPv6 networks with a prefix larger than 64 aren't properly supported by dnsmasq", ProxyBridgeNetfilterNotEnabled: "Proxy bridge netfilter not enabled", NetworkUnvailable: "Network unavailable", OfflineClusterMember: "Offline cluster member", InstanceAutostartFailure: "Failed to autostart instance", InstanceTypeNotOperational: "Instance type not operational", StoragePoolUnvailable: "Storage pool unavailable", UnableToUpdateClusterCertificate: "Unable to update cluster certificate", SELinuxNotAvailable: "SELinux support has been disabled", } // Severity returns the severity of the warning type. func (t Type) Severity() Severity { switch t { case Undefined: return SeverityLow case MissingCGroupBlkio: return SeverityLow case MissingCGroupBlkioWeight: return SeverityLow case MissingCGroupCPUController: return SeverityLow case MissingCGroupCPUsetController: return SeverityLow case MissingCGroupCPUacctController: return SeverityLow case MissingCGroupDevicesController: return SeverityLow case MissingCGroupFreezerController: return SeverityLow case MissingCGroupHugetlbController: return SeverityLow case MissingCGroupMemoryController: return SeverityLow case MissingCGroupPidsController: return SeverityLow case MissingCGroupMemorySwapAccounting: return SeverityLow case ClusterTimeSkew: return SeverityLow case AppArmorNotAvailable: return SeverityLow case MissingVirtiofsd: return SeverityLow case AppArmorDisabledDueToRawDnsmasq: return SeverityLow case LargerIPv6PrefixThanSupported: return SeverityLow case ProxyBridgeNetfilterNotEnabled: return SeverityLow case NetworkUnvailable: return SeverityHigh case OfflineClusterMember: return SeverityLow case InstanceAutostartFailure: return SeverityLow case InstanceTypeNotOperational: return SeverityLow case StoragePoolUnvailable: return SeverityHigh case UnableToUpdateClusterCertificate: return SeverityLow case SELinuxNotAvailable: return SeverityLow } return SeverityLow } incus-7.0.0/internal/server/device/000077500000000000000000000000001517523235500172305ustar00rootroot00000000000000incus-7.0.0/internal/server/device/config/000077500000000000000000000000001517523235500204755ustar00rootroot00000000000000incus-7.0.0/internal/server/device/config/consts.go000066400000000000000000000002531517523235500223350ustar00rootroot00000000000000package config // DefaultVMBlockFilesystemSize is the size of a VM root device block volume's associated filesystem volume. const DefaultVMBlockFilesystemSize = "500MiB" incus-7.0.0/internal/server/device/config/device_proxyaddress.go000066400000000000000000000002531517523235500250720ustar00rootroot00000000000000package config // ProxyAddress represents a proxy address configuration. type ProxyAddress struct { ConnType string Abstract bool Address string Ports []uint64 } incus-7.0.0/internal/server/device/config/device_runconfig.go000066400000000000000000000066461517523235500243510ustar00rootroot00000000000000package config import ( "github.com/lxc/incus/v7/shared/revert" ) // MountOwnerShiftNone do not use owner shifting. const MountOwnerShiftNone = "" // MountOwnerShiftDynamic use shifted mounts for dynamic owner shifting. const MountOwnerShiftDynamic = "dynamic" // MountOwnerShiftStatic statically modify ownership. const MountOwnerShiftStatic = "static" // RunConfigItem represents a single config item. type RunConfigItem struct { Key string Value string } // MountEntryItem represents a single mount entry item. type MountEntryItem struct { DevName string // The internal name for the device. DevPath string // Describes the block special device or remote filesystem to be mounted. BackingPath []string // Describes the block special device to be mounted as backing drive for qcow2. TargetPath string // Describes the mount point (target) for the filesystem. FSType string // Describes the type of the filesystem. Opts []string // Describes the mount options associated with the filesystem. Freq int // Used by dump(8) to determine which filesystems need to be dumped. Defaults to zero (don't dump) if not present. PassNo int // Used by fsck(8) to determine the order in which filesystem checks are done at boot time. Defaults to zero (don't fsck) if not present. OwnerShift string // Ownership shifting mode, use constants MountOwnerShiftNone, MountOwnerShiftStatic or MountOwnerShiftDynamic. Limits *DiskLimits // Disk limits. Size int64 // Expected disk size in bytes. } // RootFSEntryItem represents the root filesystem options for an Instance. type RootFSEntryItem struct { Path string // Describes the root file system source. Opts []string // Describes the mount options associated with the filesystem. } // USBDeviceItem represents a single USB device matched from a USB device specification. type USBDeviceItem struct { DeviceName string HostDevicePath string } // DiskLimits represents a set of I/O disk limits. type DiskLimits struct { ReadBytes int64 ReadIOps int64 WriteBytes int64 WriteIOps int64 } // RunConfig represents run-time config used for device setup/cleanup. type RunConfig struct { RootFS RootFSEntryItem // RootFS to setup. NetworkInterface []RunConfigItem // Network interface configuration settings. CGroups []RunConfigItem // Cgroup rules to setup. Mounts []MountEntryItem // Mounts to setup/remove. Uevents [][]string // Uevents to inject. PostHooks []func() error // Functions to be run after device attach/detach. GPUDevice []RunConfigItem // GPU device configuration settings. USBDevice []USBDeviceItem // USB device configuration settings. TPMDevice []RunConfigItem // TPM device configuration settings. PCIDevice []RunConfigItem // PCI device configuration settings. Revert revert.Hook // Revert setup of device on post-setup error. UseUSBBus bool // Whether to use a USB bus for the device. } // NICConfigDir shared constant used to indicate where NIC config is stored. const NICConfigDir = "nics" // NICConfig contains network interface configuration to be passed into a VM and applied by the agent. type NICConfig struct { DeviceName string `json:"device_name"` NICName string `json:"nic_name"` MACAddress string `json:"mac_address"` MTU uint32 `json:"mtu"` } incus-7.0.0/internal/server/device/config/devices.go000066400000000000000000000141561517523235500224550ustar00rootroot00000000000000package config import ( "fmt" "sort" "strings" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) // Device represents an instance device. type Device map[string]string // Clone returns a copy of the Device. func (device Device) Clone() Device { return util.CloneMap(device) } // Validate accepts a map of field/validation functions to run against the device's config. func (device Device) Validate(rules map[string]func(value string) error) error { checkedFields := map[string]struct{}{} for k, validator := range rules { checkedFields[k] = struct{}{} // Mark field as checked. err := validator(device[k]) if err != nil { return fmt.Errorf("Invalid value for device option %q: %w", k, err) } } // Look for any unchecked fields, as these are unknown fields and validation should fail. for k := range device { _, checked := checkedFields[k] if checked { continue } // Skip type fields are these are validated by the presence of an implementation. if k == "type" { continue } // Allow user.* configuration. if strings.HasPrefix(k, "user.") { continue } // Allow initial.* configuration. if strings.HasPrefix(k, "initial.") { continue } if k == "nictype" && (device["type"] == "nic" || device["type"] == "infiniband") { continue } if k == "gputype" && device["type"] == "gpu" { continue } return fmt.Errorf("Invalid device option %q", k) } return nil } // Devices represents a set of instance devices. type Devices map[string]Device // NewDevices creates a new Devices set from a native map[string]map[string]string set. func NewDevices(nativeSet map[string]map[string]string) Devices { newDevices := Devices{} for devName, devConfig := range nativeSet { newDev := Device{} for k, v := range devConfig { if v == "" { continue } newDev[k] = v } newDevices[devName] = newDev } return newDevices } // ApplyDeviceInitialValues applies a profile initial values to root disk devices. func ApplyDeviceInitialValues(devices Devices, profiles []api.Profile) Devices { for _, p := range profiles { for devName, devConfig := range p.Devices { // Apply only root disk device from profile devices to instance devices. if devConfig["type"] != "disk" || devConfig["path"] != "/" || devConfig["source"] != "" { continue } // Skip profile devices that are already present in the map of devices // because those devices should be already populated. _, ok := devices[devName] if ok { continue } // If profile device contains an initial.* key, add it to the map of devices. for k := range devConfig { if strings.HasPrefix(k, "initial.") { devices[devName] = devConfig break } } } } return devices } // Contains checks if a given device exists in the set and if it's identical to that provided. func (list Devices) Contains(k string, d Device) bool { // If it didn't exist, it's different if list[k] == nil { return false } old := list[k] return deviceEquals(old, d) } // Update returns the difference between two device sets (removed, added, updated devices) and a list of all // changed keys across all devices. Accepts a function to return which keys can be live updated, which prevents // them being removed and re-added if the device supports live updates of certain keys. func (list Devices) Update(newlist Devices, updateFields func(Device, Device) []string) (Devices, Devices, Devices, []string) { rmlist := map[string]Device{} addlist := map[string]Device{} updatelist := map[string]Device{} // Detect which devices have changed or been removed in in new list. for key, d := range list { if !newlist.Contains(key, d) { rmlist[key] = d } } // Detect which devices have changed or been added in in new list. for key, d := range newlist { if !list.Contains(key, d) { addlist[key] = d } } allChangedKeys := []string{} for key, d := range addlist { srcOldDevice := rmlist[key] oldDevice := srcOldDevice.Clone() srcNewDevice := newlist[key] newDevice := srcNewDevice.Clone() // Detect keys different between old and new device and append to the all changed keys list. allChangedKeys = append(allChangedKeys, deviceEqualsDiffKeys(oldDevice, newDevice)...) // Remove 'user.' fields that can be live-updated without adding/removing the device from instance. for k := range d { if strings.HasPrefix(k, "user.") { delete(oldDevice, k) delete(newDevice, k) } } // Remove any fields that can be live-updated without adding/removing the device from instance. if updateFields != nil { for _, k := range updateFields(oldDevice, newDevice) { delete(oldDevice, k) delete(newDevice, k) } } // If after removing the live-updatable keys the devices are equal, then we know the device has // been updated rather than added or removed, so add it to the update list, and remove it from // the added and removed lists. if deviceEquals(oldDevice, newDevice) { delete(rmlist, key) delete(addlist, key) updatelist[key] = d } } return rmlist, addlist, updatelist, allChangedKeys } // Clone returns a copy of the Devices set. func (list Devices) Clone() Devices { copy := make(Devices, len(list)) for deviceName, device := range list { copy[deviceName] = device.Clone() } return copy } // CloneNative returns a copy of the Devices set as a native map[string]map[string]string type. func (list Devices) CloneNative() map[string]map[string]string { copy := make(map[string]map[string]string, len(list)) for deviceName, device := range list { copy[deviceName] = device.Clone() } return copy } // Sorted returns the name of all devices in the set, sorted properly. func (list Devices) Sorted() DevicesSortable { sortable := DevicesSortable{} for k, d := range list { sortable = append(sortable, DeviceNamed{k, d}) } sort.Sort(sortable) return sortable } // Reversed returns the name of all devices in the set, sorted reversed. func (list Devices) Reversed() DevicesSortable { sortable := DevicesSortable{} for k, d := range list { sortable = append(sortable, DeviceNamed{k, d}) } sort.Sort(sort.Reverse(sortable)) return sortable } incus-7.0.0/internal/server/device/config/devices_sort.go000066400000000000000000000040651517523235500235220ustar00rootroot00000000000000package config // DeviceNamed contains the name of a device and its config. type DeviceNamed struct { Name string Config Device } // DevicesSortable is a sortable slice of device names and config. type DevicesSortable []DeviceNamed func (devices DevicesSortable) Len() int { return len(devices) } func (devices DevicesSortable) Less(i, j int) bool { a := devices[i] b := devices[j] // First sort by types. if a.Config["type"] != b.Config["type"] { // In VMs, network interface names are derived from PCI // location. As a result of that, we must ensure that nic devices will // always show up at the same spot regardless of what other devices may be // added. Easiest way to do this is to always have them show up first. if a.Config["type"] == "nic" { return true } if b.Config["type"] == "nic" { return false } // Start disks before other non-nic devices so that any unmounts triggered by deferred resizes // specified in volatile "apply_quota" key can occur first and the rest of the devices can rely on // the instance's root disk being mounted. if a.Config["type"] == "disk" { return true } if b.Config["type"] == "disk" { return false } // Otherwise start devices of same type together. return a.Config["type"] > b.Config["type"] } // Start non-nested NIC devices before nested NIC devices. if a.Config["type"] == "nic" && b.Config["type"] == "nic" && (a.Config["nested"] != "" || b.Config["nested"] != "") { if a.Config["nested"] == "" { return true } else if b.Config["nested"] == "" { return false } } // Start disk devices in path order. if a.Config["type"] == "disk" && b.Config["type"] == "disk" { if a.Config["path"] != b.Config["path"] { // The root device always goes first. if a.Config["path"] == "/" { return true } if b.Config["path"] == "/" { return false } return a.Config["path"] < b.Config["path"] } } // Fallback to sorting by names. return a.Name < b.Name } func (devices DevicesSortable) Swap(i, j int) { devices[i], devices[j] = devices[j], devices[i] } incus-7.0.0/internal/server/device/config/devices_test.go000066400000000000000000000041741517523235500235130ustar00rootroot00000000000000package config import ( "testing" "github.com/stretchr/testify/assert" ) func TestSortableDevices(t *testing.T) { devices := Devices{ "a-unix1": Device{"type": "unix"}, "a-unix2": Device{"type": "unix"}, "b-disk1": Device{"type": "disk", "path": "/foo/bar"}, "b-disk2": Device{"type": "disk", "path": "/foo"}, "b-disk3": Device{"type": "disk", "path": "/"}, "z-nic-nested1": Device{"type": "nic", "nested": "foo1"}, "z-nic-nested2": Device{"type": "nic", "nested": "foo2"}, "z-nic1": Device{"type": "nic"}, "z-nic2": Device{"type": "nic"}, } expectedSorted := DevicesSortable{ DeviceNamed{Name: "z-nic1", Config: Device{"type": "nic"}}, DeviceNamed{Name: "z-nic2", Config: Device{"type": "nic"}}, DeviceNamed{Name: "z-nic-nested1", Config: Device{"type": "nic", "nested": "foo1"}}, DeviceNamed{Name: "z-nic-nested2", Config: Device{"type": "nic", "nested": "foo2"}}, DeviceNamed{Name: "b-disk3", Config: Device{"type": "disk", "path": "/"}}, DeviceNamed{Name: "b-disk2", Config: Device{"type": "disk", "path": "/foo"}}, DeviceNamed{Name: "b-disk1", Config: Device{"type": "disk", "path": "/foo/bar"}}, DeviceNamed{Name: "a-unix1", Config: Device{"type": "unix"}}, DeviceNamed{Name: "a-unix2", Config: Device{"type": "unix"}}, } result := devices.Sorted() assert.Equal(t, expectedSorted, result) expectedReversed := DevicesSortable{ DeviceNamed{Name: "a-unix2", Config: Device{"type": "unix"}}, DeviceNamed{Name: "a-unix1", Config: Device{"type": "unix"}}, DeviceNamed{Name: "b-disk1", Config: Device{"type": "disk", "path": "/foo/bar"}}, DeviceNamed{Name: "b-disk2", Config: Device{"type": "disk", "path": "/foo"}}, DeviceNamed{Name: "b-disk3", Config: Device{"type": "disk", "path": "/"}}, DeviceNamed{Name: "z-nic-nested2", Config: Device{"type": "nic", "nested": "foo2"}}, DeviceNamed{Name: "z-nic-nested1", Config: Device{"type": "nic", "nested": "foo1"}}, DeviceNamed{Name: "z-nic2", Config: Device{"type": "nic"}}, DeviceNamed{Name: "z-nic1", Config: Device{"type": "nic"}}, } result = devices.Reversed() assert.Equal(t, expectedReversed, result) } incus-7.0.0/internal/server/device/config/devices_utils.go000066400000000000000000000012201517523235500236610ustar00rootroot00000000000000package config // deviceEquals checks for any difference and addition/removal of properties. func deviceEquals(old Device, d Device) bool { for k := range d { if d[k] != old[k] { return false } } for k := range old { if d[k] != old[k] { return false } } return true } // deviceEqualsDiffKeys checks for any difference and addition/removal of properties and returns a list of changes. func deviceEqualsDiffKeys(old Device, d Device) []string { keys := []string{} for k := range d { if d[k] != old[k] { keys = append(keys, k) } } for k := range old { if d[k] != old[k] { keys = append(keys, k) } } return keys } incus-7.0.0/internal/server/device/device_common.go000066400000000000000000000106631517523235500223740ustar00rootroot00000000000000package device import ( "fmt" "net" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/logger" ) // deviceCommon represents the common struct for all devices. type deviceCommon struct { logger logger.Logger inst instance.Instance name string config deviceConfig.Device state *state.State volatileGet func() map[string]string volatileSet func(map[string]string) error } // init stores the Instance, daemon state, device name and config into device. // It also needs to be provided with volatile get and set functions for the device to allow // persistent data to be accessed. This is implemented as part of deviceCommon so that the majority // of devices don't need to implement it and can just embed deviceCommon. func (d *deviceCommon) init(inst instance.Instance, s *state.State, name string, conf deviceConfig.Device, volatileGet VolatileGetter, volatileSet VolatileSetter) error { logCtx := logger.Ctx{"driver": conf["type"], "device": name} if inst != nil { logCtx["project"] = inst.Project().Name logCtx["instance"] = inst.Name() } d.logger = logger.AddContext(logCtx) d.inst = inst d.name = name d.config = conf d.state = s d.volatileGet = volatileGet d.volatileSet = volatileSet return nil } // Name returns the name of the device. func (d *deviceCommon) Name() string { return d.name } // Config returns the config for the device. func (d *deviceCommon) Config() deviceConfig.Device { return d.config } // Add returns nil error as majority of devices don't need to do any host-side setup. func (d *deviceCommon) Add() error { return nil } // Register returns nil error as majority of devices don't need to do any event registration. func (d *deviceCommon) Register() error { return nil } // CanHotPlug returns whether the device can be managed whilst the instance is running, // Returns true if instance type is container, as majority of devices can be started/stopped when // instance is running. If instance type is VM then returns false as this is not currently supported. func (d *deviceCommon) CanHotPlug() bool { return d.inst.Type() == instancetype.Container } // CanMigrate returns whether the device can be migrated to any other cluster member. func (d *deviceCommon) CanMigrate() bool { return false } // UpdatableFields returns an empty list of updatable fields as most devices do not support updates. func (d *deviceCommon) UpdatableFields(oldDevice Type) []string { return []string{} } // PreStartCheck indicates if the device is available for starting. func (d *deviceCommon) PreStartCheck() error { return nil } // Update returns an ErrCannotUpdate error as most devices do not support updates. func (d *deviceCommon) Update(oldDevices deviceConfig.Devices, isRunning bool) error { return ErrCannotUpdate } // Remove returns nil error as majority of devices don't need to do any host-side cleanup on delete. func (d *deviceCommon) Remove(cleanupDependencies bool) error { return nil } // generateHostName generates the name to use for the host side NIC interface based on the // instances.nic.host_name setting. // Accepts prefix argument to use with random interface generation. // Accepts optional hwaddr MAC address to use for generating the interface name in mac mode. // In mac mode the interface prefix is always "inc". func (d *deviceCommon) generateHostName(prefix string, hwaddr string) (string, error) { hostNameMode := d.state.GlobalConfig.InstancesNICHostname() // Handle instances.nic.host_name mac mode if a MAC address has been supplied. if hostNameMode == "mac" && hwaddr != "" { mac, err := net.ParseMAC(hwaddr) if err != nil { return "", fmt.Errorf("Failed parsing MAC address %q: %w", hwaddr, err) } return network.MACDevName(mac), nil } // Handle instances.nic.host_name random mode or where no MAC address supplied. return network.RandomDevName(prefix), nil } // setNICLink sets the link status (connected/disconnected) for the given NIC. func (d *deviceCommon) setNICLink() error { runConf := deviceConfig.RunConfig{} runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "connected", Value: d.config["connected"]}, } return d.inst.DeviceEventHandler(&runConf) } incus-7.0.0/internal/server/device/device_interface.go000066400000000000000000000077061517523235500230500ustar00rootroot00000000000000package device import ( deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" ) // VolatileSetter is a function that accepts one or more key/value strings to save into the // config for this instance. It should add the volatile device name prefix to each key when saving. type VolatileSetter func(map[string]string) error // VolatileGetter is a function that retrieves any key/value string that exists in the database // config for this instance. It should only return keys that match the volatile device name prefix, // and should remove the prefix before being returned. type VolatileGetter func() map[string]string // Type represents a device type. type Type interface { // CanHotPlug returns true if the device can be managed whilst instance is running. CanHotPlug() bool // CanMigrate returns true if the device should work properly on any cluster member. CanMigrate() bool // UpdatableFields returns a slice of config fields that can be updated. If only fields in this list have // changed then Update() is called rather triggering a device remove & add. UpdatableFields(oldDevice Type) []string } // Device represents a device that can be added to an instance. type Device interface { Type Config() deviceConfig.Device Name() string // Add performs any host-side setup when a device is added to an instance. // It is called irrespective of whether the instance is running or not. Add() error // PreStartCheck indicates if the device is available for starting. PreStartCheck() error // Start performs any host-side configuration required to start the device for the instance. // This can be when a device is plugged into a running instance or the instance is starting. // Returns run-time configuration needed for configuring the instance with the new device. Start() (*deviceConfig.RunConfig, error) // Register provides the ability for a device to subscribe to daemon generated events. // It is called after a device is started (after Start()) or on daemon startup. Register() error // Update performs host-side modifications for a device based on the difference between the // current config and previous devices config supplied as an argument. This called if the // only config fields that have changed are supplied in the list returned from UpdatableFields(). // The function also accepts a boolean indicating whether the instance is running or not. Update(oldDevices deviceConfig.Devices, running bool) error // Stop performs any host-side cleanup required when a device is removed from an instance, // either due to unplugging it from a running instance or instance is being shutdown. // Returns run-time configuration needed for detaching the device from the instance. Stop() (*deviceConfig.RunConfig, error) // Remove performs any host-side cleanup when a device is removed from an instance. Remove(cleanupDependencies bool) error } // device represents a sealed interface that implements Device, but also contains some internal // setup functions for a Device that should only be called by device.New() to avoid exposing devices // that are not in a known configured state. This is separate from the Device interface so that // Devices created outside of the device package can be used externally, but ensures that any devices // created by the device package will only be accessible after being configured properly by New(). type device interface { Device // init stores the Instance, daemon State and Config into device and performs any setup. init(instance.Instance, *state.State, string, deviceConfig.Device, VolatileGetter, VolatileSetter) error // validateConfig checks Config stored by init() is valid for the instance type. validateConfig(instance.ConfigReader, bool) error } // NICState provides the ability to access NIC state. type NICState interface { State() (*api.InstanceStateNetwork, error) } incus-7.0.0/internal/server/device/device_load.go000066400000000000000000000124721517523235500220230ustar00rootroot00000000000000package device import ( "errors" "fmt" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/device/nictype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/validate" ) // newByType returns a new uninitialized device based of the type indicated by the project and device config. func newByType(state *state.State, projectName string, conf deviceConfig.Device) (device, error) { if conf["type"] == "" { return nil, errors.New("Missing device type in config") } // NIC type is required to lookup network devices. nicType, err := nictype.NICType(state, projectName, conf) if err != nil { return nil, err } // Lookup device type implementation. var dev device switch conf["type"] { case "nic": switch nicType { case "physical": dev = &nicPhysical{} case "ipvlan": dev = &nicIPVLAN{} case "p2p": dev = &nicP2P{} case "bridged": dev = &nicBridged{} case "routed": dev = &nicRouted{} case "macvlan": dev = &nicMACVLAN{} case "sriov": dev = &nicSRIOV{} case "ovn": dev = &nicOVN{} } case "infiniband": // gendoc:generate(entity=devices, group=infiniband, key=nictype) // // --- // type: string // required: yes // shortdesc: The device type (one of `physical` or `sriov`) switch nicType { case "physical": dev = &infinibandPhysical{} case "sriov": dev = &infinibandSRIOV{} } case "gpu": switch conf["gputype"] { case "mig": dev = &gpuMIG{} case "mdev": dev = &gpuMdev{} case "sriov": dev = &gpuSRIOV{} default: dev = &gpuPhysical{} } case "proxy": dev = &proxy{} case "usb": dev = &usb{} case "unix-char", "unix-block": dev = &unixCommon{} case "unix-hotplug": dev = &unixHotplug{} case "disk": dev = &disk{} case "none": dev = &none{} case "tpm": dev = &tpm{} case "pci": dev = &pci{} } // Check a valid device type has been found. if dev == nil { return nil, ErrUnsupportedDevType } return dev, nil } // load instantiates a device and initializes its internal state. It does not validate the config supplied. func load(inst instance.Instance, state *state.State, projectName string, name string, conf deviceConfig.Device, volatileGet VolatileGetter, volatileSet VolatileSetter) (device, error) { // Warning: When validating a profile, inst is expected to be provided as nil. dev, err := newByType(state, projectName, conf) if err != nil { return nil, fmt.Errorf("Failed loading device %q: %w", name, err) } // Setup the device's internal variables. err = dev.init(inst, state, name, conf, volatileGet, volatileSet) if err != nil { return nil, fmt.Errorf("Failed loading device %q: %w", name, err) } return dev, nil } // New instantiates a new device struct, validates the supplied config and sets it into the device. // If the device type is valid, but the other config validation fails then an instantiated device // is still returned with the validation error. If an unknown device is requested or the device is // not compatible with the instance type then an ErrUnsupportedDevType error is returned. // Note: The supplied config may be modified during validation to enrich. If this is not desired, supply a copy. func New(inst instance.Instance, s *state.State, name string, conf deviceConfig.Device, partialValidation bool, volatileGet VolatileGetter, volatileSet VolatileSetter) (Device, error) { dev, err := load(inst, s, inst.Project().Name, name, conf, volatileGet, volatileSet) if err != nil { return nil, err } // We still return the instantiated device here, as in some scenarios the caller // may still want to use the device (such as when stopping or removing) even if // the config validation has failed. err = validate.IsDeviceName(name) if err != nil { return dev, err } err = dev.validateConfig(inst, partialValidation) if err != nil { return dev, err } return dev, nil } // Validate checks a device's config is valid. This only requires an instance.ConfigReader rather than an full // blown instance to allow profile devices to be validated too. // Note: The supplied config may be modified during validation to enrich. If this is not desired, supply a copy. func Validate(instConfig instance.ConfigReader, s *state.State, name string, conf deviceConfig.Device, partialValidation bool) error { err := validate.IsDeviceName(name) if err != nil { return err } dev, err := load(nil, s, instConfig.Project().Name, name, conf, nil, nil) if err != nil { return err } return dev.validateConfig(instConfig, partialValidation) } // Register performs a lightweight load of the device, bypassing most // validation to very quickly register the device on server startup. func Register(inst instance.Instance, s *state.State, name string, conf deviceConfig.Device) error { dev, err := load(inst, s, inst.Project().Name, name, conf, nil, nil) if err != nil { return err } return dev.Register() } // LoadByType loads a device by type based on its project and config. // It does not validate config beyond the type fields. func LoadByType(state *state.State, projectName string, conf deviceConfig.Device) (Type, error) { dev, err := newByType(state, projectName, conf) if err != nil { return nil, fmt.Errorf("Failed loading device type: %w", err) } return dev, nil } incus-7.0.0/internal/server/device/device_utils_disk.go000066400000000000000000000307241517523235500232560ustar00rootroot00000000000000package device import ( "context" "errors" "fmt" "net" "os" "os/exec" "path/filepath" "slices" "sort" "strings" "time" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/instance" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) // RBDFormatPrefix is the prefix used in disk paths to identify RBD. const RBDFormatPrefix = "rbd" // RBDFormatSeparator is the field separate used in disk paths for RBD devices. const RBDFormatSeparator = " " // DiskParseRBDFormat parses an rbd formatted string, and returns the pool name, volume name, and map of options. func DiskParseRBDFormat(rbd string) (string, string, map[string]string, error) { // Remove and check the prefix. prefix, rbd, _ := strings.Cut(rbd, RBDFormatSeparator) if prefix != RBDFormatPrefix { return "", "", nil, fmt.Errorf("Invalid rbd format, wrong prefix: %q", prefix) } // Split the path and options. path, rawOpts, _ := strings.Cut(rbd, RBDFormatSeparator) // Check for valid RBD path. pool, volume, validPath := strings.Cut(path, "/") if !validPath { return "", "", nil, fmt.Errorf("Invalid rbd format, missing pool and/or volume: %q", path) } // Parse options. opts := make(map[string]string) for _, o := range strings.Split(rawOpts, ":") { k, v, isValid := strings.Cut(o, "=") if !isValid { return "", "", nil, fmt.Errorf("Invalid rbd format, bad option: %q", o) } opts[k] = v } return pool, volume, opts, nil } // DiskGetRBDFormat returns a rbd formatted string with the given values. func DiskGetRBDFormat(clusterName string, userName string, poolName string, volumeName string) string { // Resolve any symlinks to config path. confPath := fmt.Sprintf("/etc/ceph/%s.conf", clusterName) target, err := filepath.EvalSymlinks(confPath) if err == nil { confPath = target } // Configuration values containing :, @, or = can be escaped with a leading \ character. // According to https://docs.ceph.com/docs/hammer/rbd/qemu-rbd/#usage optEscaper := strings.NewReplacer(":", `\:`, "@", `\@`, "=", `\=`) opts := []string{ fmt.Sprintf("id=%s", optEscaper.Replace(userName)), fmt.Sprintf("pool=%s", optEscaper.Replace(poolName)), fmt.Sprintf("cluster=%s", optEscaper.Replace(clusterName)), fmt.Sprintf("conf=%s", optEscaper.Replace(confPath)), } return fmt.Sprintf("%s%s%s/%s%s%s", RBDFormatPrefix, RBDFormatSeparator, optEscaper.Replace(poolName), optEscaper.Replace(volumeName), RBDFormatSeparator, strings.Join(opts, ":")) } // BlockFsDetect detects the type of block device. func BlockFsDetect(dev string) (string, error) { out, err := subprocess.RunCommand("blkid", "-s", "TYPE", "-o", "value", dev) if err != nil { return "", err } return strings.TrimSpace(out), nil } // IsBlockdev returns boolean indicating whether device is block type. func IsBlockdev(path string) bool { // Get a stat struct from the provided path. stat := unix.Stat_t{} err := unix.Stat(path, &stat) if err != nil { return false } // Check if it's a block device if stat.Mode&unix.S_IFMT == unix.S_IFBLK { return true } // Not a device return false } // DiskMount mounts a disk device. func DiskMount(srcPath string, dstPath string, recursive bool, propagation string, mountOptions []string, fsName string) error { var err error flags, mountOptionsStr := linux.ResolveMountOptions(mountOptions) var readonly bool if slices.Contains(mountOptions, "ro") { readonly = true } // Detect the filesystem if fsName == "none" { flags |= unix.MS_BIND } if propagation != "" { switch propagation { case "private": flags |= unix.MS_PRIVATE case "shared": flags |= unix.MS_SHARED case "slave": flags |= unix.MS_SLAVE case "unbindable": flags |= unix.MS_UNBINDABLE case "rprivate": flags |= unix.MS_PRIVATE | unix.MS_REC case "rshared": flags |= unix.MS_SHARED | unix.MS_REC case "rslave": flags |= unix.MS_SLAVE | unix.MS_REC case "runbindable": flags |= unix.MS_UNBINDABLE | unix.MS_REC default: return fmt.Errorf("Invalid propagation mode %q", propagation) } } if recursive { flags |= unix.MS_REC } // Mount the filesystem err = unix.Mount(srcPath, dstPath, fsName, uintptr(flags), mountOptionsStr) if err != nil { return fmt.Errorf("Unable to mount %q at %q with filesystem %q: %w", srcPath, dstPath, fsName, err) } // Remount bind mounts in readonly mode if requested if readonly && flags&unix.MS_BIND == unix.MS_BIND { flags = unix.MS_RDONLY | unix.MS_BIND | unix.MS_REMOUNT err = unix.Mount("", dstPath, fsName, uintptr(flags), "") if err != nil { return fmt.Errorf("Unable to mount %q in readonly mode: %w", dstPath, err) } } flags = unix.MS_REC | unix.MS_SLAVE err = unix.Mount("", dstPath, "", uintptr(flags), "") if err != nil { return fmt.Errorf("Unable to make mount %q private: %w", dstPath, err) } return nil } // DiskMountClear unmounts and removes the mount path used for disk shares. func DiskMountClear(mntPath string) error { if util.PathExists(mntPath) { if linux.IsMountPoint(mntPath) { err := storageDrivers.TryUnmount(mntPath, 0) if err != nil { return fmt.Errorf("Failed unmounting %q: %w", mntPath, err) } } err := os.Remove(mntPath) if err != nil { return fmt.Errorf("Failed removing %q: %w", mntPath, err) } } return nil } func diskCephRbdMap(clusterName string, userName string, poolName string, volumeName string) (string, error) { devPath, err := subprocess.RunCommand( "rbd", "--id", userName, "--cluster", clusterName, "--pool", poolName, "map", volumeName) if err != nil { return "", err } idx := strings.Index(devPath, "/dev/rbd") if idx < 0 { return "", errors.New("Failed to detect mapped device path") } devPath = devPath[idx:] return strings.TrimSpace(devPath), nil } func diskCephRbdUnmap(deviceName string) error { unmapImageName := deviceName busyCount := 0 again: _, err := subprocess.RunCommand( "rbd", "unmap", unmapImageName) if err != nil { var runError subprocess.RunError if errors.As(err, &runError) { var exitError *exec.ExitError if errors.As(runError.Unwrap(), &exitError) { if exitError.ExitCode() == 22 { // EINVAL (already unmapped) return nil } if exitError.ExitCode() == 16 { // EBUSY (currently in use) busyCount++ if busyCount == 10 { return err } // Wait a second an try again time.Sleep(time.Second) goto again } } } return err } goto again } // diskCephfsOptions returns the mntSrcPath and fsOptions to use for mounting a cephfs share. func diskCephfsOptions(clusterName string, userName string, fsName string, fsPath string) (string, []string, error) { // Get the FSID. fsid, err := storageDrivers.CephFsid(clusterName, userName) if err != nil { return "", nil, err } // Get the monitor list. monAddresses, err := storageDrivers.CephMonitors(clusterName, userName) if err != nil { return "", nil, err } // Get the keyring entry. secret, err := storageDrivers.CephKeyring(clusterName, userName) if err != nil { return "", nil, err } srcPath, fsOptions := storageDrivers.CephBuildMount( userName, secret, fsid, monAddresses, fsName, fsPath, ) return srcPath, fsOptions, nil } // DiskVMVirtiofsdStart starts a new virtiofsd process. // If the idmaps slice is supplied then the proxy process is run inside a user namespace using the supplied maps. // Returns UnsupportedError error if the host system or instance does not support virtiosfd, returns normal error // type if process cannot be started for other reasons. // Returns revert function and listener file handle on success. func DiskVMVirtiofsdStart(execPath string, inst instance.Instance, socketPath string, pidPath string, logPath string, sharePath string, idmaps []idmap.Entry, cacheOption string) (func(), net.Listener, error) { reverter := revert.New() defer reverter.Fail() if !filepath.IsAbs(sharePath) { return nil, nil, fmt.Errorf("Share path not absolute: %q", sharePath) } // Remove old socket if needed. _ = os.Remove(socketPath) // Locate virtiofsd. cmd, err := exec.LookPath("virtiofsd") if err != nil { if util.PathExists("/usr/lib/qemu/virtiofsd") { cmd = "/usr/lib/qemu/virtiofsd" } else if util.PathExists("/usr/libexec/virtiofsd") { cmd = "/usr/libexec/virtiofsd" } else if util.PathExists("/usr/lib/virtiofsd") { cmd = "/usr/lib/virtiofsd" } } if cmd == "" { return nil, nil, ErrMissingVirtiofsd } if util.IsTrue(inst.ExpandedConfig()["migration.stateful"]) { return nil, nil, UnsupportedError{"Stateful migration unsupported"} } if util.IsTrue(inst.ExpandedConfig()["security.sev"]) || util.IsTrue(inst.ExpandedConfig()["security.sev.policy.es"]) { return nil, nil, UnsupportedError{"SEV unsupported"} } // Trickery to handle paths > 107 chars. socketFileDir, err := os.Open(filepath.Dir(socketPath)) if err != nil { return nil, nil, err } defer func() { _ = socketFileDir.Close() }() socketFile := fmt.Sprintf("/proc/self/fd/%d/%s", socketFileDir.Fd(), filepath.Base(socketPath)) listener, err := net.Listen("unix", socketFile) if err != nil { return nil, nil, fmt.Errorf("Failed to create unix listener for virtiofsd: %w", err) } reverter.Add(func() { _ = listener.Close() _ = os.Remove(socketPath) }) unixListener, ok := listener.(*net.UnixListener) if !ok { return nil, nil, errors.New("Failed getting UnixListener for virtiofsd") } unixFile, err := unixListener.File() if err != nil { return nil, nil, fmt.Errorf("Failed to getting unix listener file for virtiofsd: %w", err) } defer func() { _ = unixFile.Close() }() switch cacheOption { case "metadata": cacheOption = "metadata" case "unsafe": cacheOption = "always" default: cacheOption = "never" } // Start the virtiofsd process in non-daemon mode. args := []string{"--fd=3", fmt.Sprintf("--cache=%s", cacheOption), fmt.Sprintf("--shared-dir=%s", sharePath)} if len(idmaps) > 0 { idmapSet := &idmap.Set{Entries: idmaps} sort.Sort(idmapSet) var lastUID int64 var lastGID int64 for _, entry := range idmapSet.Entries { if entry.IsUID { args = append(args, fmt.Sprintf("--translate-uid=map:%d:%d:%d", entry.NSID, entry.HostID, entry.MapRange)) args = append(args, fmt.Sprintf("--translate-uid=forbid-guest:%d:%d", lastUID, entry.NSID-lastUID)) lastUID = entry.NSID + entry.MapRange } if entry.IsGID { args = append(args, fmt.Sprintf("--translate-gid=map:%d:%d:%d", entry.NSID, entry.HostID, entry.MapRange)) args = append(args, fmt.Sprintf("--translate-gid=forbid-guest:%d:%d", lastGID, entry.NSID-lastGID)) lastGID = entry.NSID + entry.MapRange } } if lastUID < 4294967295 { args = append(args, fmt.Sprintf("--translate-uid=forbid-guest:%d:%d", lastUID, 4294967295-lastUID)) } if lastGID < 4294967295 { args = append(args, fmt.Sprintf("--translate-gid=forbid-guest:%d:%d", lastGID, 4294967295-lastGID)) } } else if inst.GuestOS() != "windows" { args = append(args, "--posix-acl") } proc, err := subprocess.NewProcess(cmd, args, logPath, logPath) if err != nil { return nil, nil, err } err = proc.StartWithFiles(context.Background(), []*os.File{unixFile}) if err != nil { return nil, nil, fmt.Errorf("Failed to start virtiofsd: %w", err) } reverter.Add(func() { _ = proc.Stop() }) err = proc.Save(pidPath) if err != nil { return nil, nil, fmt.Errorf("Failed to save virtiofsd state: %w", err) } cleanup := reverter.Clone().Fail reverter.Success() return cleanup, listener, err } // DiskVMVirtiofsdStop stops an existing virtiofsd process and cleans up. func DiskVMVirtiofsdStop(socketPath string, pidPath string) error { if util.PathExists(pidPath) { proc, err := subprocess.ImportProcess(pidPath) if err != nil { return err } err = proc.Stop() // The virtiofsd process will terminate automatically once the VM has stopped. // We therefore should only return an error if it's still running and fails to stop. if err != nil && !errors.Is(err, subprocess.ErrNotRunning) { return err } // Remove PID file if needed. err = os.Remove(pidPath) if err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("Failed to remove PID file: %w", err) } } // Remove socket file if needed. err := os.Remove(socketPath) if err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("Failed to remove socket file: %w", err) } return nil } incus-7.0.0/internal/server/device/device_utils_generic.go000066400000000000000000000065101517523235500237340ustar00rootroot00000000000000package device import ( "bufio" "fmt" "os" "path/filepath" "slices" "strconv" "strings" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/util" ) // deviceJoinPath joins together prefix and text delimited by a "." for device path generation. func deviceJoinPath(parts ...string) string { return strings.Join(parts, ".") } // validatePCIDevice returns whether a configured PCI device exists under the given address. // It also returns nil, if an empty address is supplied. func validatePCIDevice(address string) error { if address != "" && !util.PathExists(fmt.Sprintf("/sys/bus/pci/devices/%s", address)) { return fmt.Errorf("Invalid PCI address (no device found): %s", address) } return nil } // checkAttachedRunningProcess checks if a device is tied to running processes. func checkAttachedRunningProcesses(devicePath string) ([]string, error) { var processes []string procDir := "/proc" files, err := os.ReadDir(procDir) if err != nil { return nil, fmt.Errorf("failed to read /proc directory: %w", err) } for _, file := range files { // Check if the directory name is a number (i.e., a PID). _, err := strconv.Atoi(file.Name()) if err != nil { continue } mapsFile := filepath.Join(procDir, file.Name(), "maps") f, err := os.Open(mapsFile) if err != nil { continue // If we can't read a process's maps file, skip it. } defer f.Close() scanner := bufio.NewScanner(f) for scanner.Scan() { if strings.Contains(scanner.Text(), devicePath) { processes = append(processes, file.Name()) break } } } return processes, nil } // getNumaNodeSet returns two slices: // 1) the NUMA nodes parsed from the configuration, // 2) the fallback NUMA nodes. func getNumaNodeSet(config map[string]string) ([]int64, []int64, error) { // If NUMA restricted, build up a list of nodes. var numaNodeSet []int64 var numaNodeSetFallback []int64 numaNodes := config["limits.cpu.nodes"] if numaNodes != "" { if numaNodes == "balanced" { numaNodes = config["volatile.cpu.nodes"] } // Parse the NUMA restriction. numaNodeSet, err := resources.ParseNumaNodeSet(numaNodes) if err != nil { return nil, nil, err } // List all the CPUs. cpus, err := resources.GetCPU() if err != nil { return nil, nil, err } // Get list of socket IDs from the list of NUMA nodes. numaSockets := make([]uint64, 0, len(cpus.Sockets)) for _, cpuSocket := range cpus.Sockets { if slices.Contains(numaSockets, cpuSocket.Socket) { continue } for _, cpuCore := range cpuSocket.Cores { found := false for _, cpuThread := range cpuCore.Threads { if slices.Contains(numaNodeSet, int64(cpuThread.NUMANode)) { numaSockets = append(numaSockets, cpuSocket.Socket) found = true break } } if found { break } } } // Get the list of NUMA nodes from the socket list. numaNodeSetFallback = []int64{} for _, cpuSocket := range cpus.Sockets { if !slices.Contains(numaSockets, cpuSocket.Socket) { continue } for _, cpuCore := range cpuSocket.Cores { for _, cpuThread := range cpuCore.Threads { if !slices.Contains(numaNodeSetFallback, int64(cpuThread.NUMANode)) { numaNodeSetFallback = append(numaNodeSetFallback, int64(cpuThread.NUMANode)) } } } } } return numaNodeSet, numaNodeSetFallback, nil } incus-7.0.0/internal/server/device/device_utils_gpu.go000066400000000000000000000004771517523235500231210ustar00rootroot00000000000000package device import ( "strings" "github.com/lxc/incus/v7/shared/validate" ) // gpuValidMigUUID validates Nvidia MIG (Multi Instance GPU) UUID with or without "MIG-" prefix. func gpuValidMigUUID(value string) error { if value == "" { return nil } return validate.IsUUID(strings.TrimPrefix(value, "MIG-")) } incus-7.0.0/internal/server/device/device_utils_infiniband.go000066400000000000000000000107661517523235500244310ustar00rootroot00000000000000package device import ( "errors" "fmt" "net" "strings" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" ) // IBDevPrefix Infiniband devices prefix. const IBDevPrefix = "infiniband.unix" // infinibandDevices extracts the infiniband parent device from the supplied nic list and any free // associated virtual functions (VFs) that are on the same card and port as the specified parent. // This function expects that the supplied nic list does not include VFs that are already attached // to running instances. func infinibandDevices(nics *api.ResourcesNetwork, parent string) map[string]*api.ResourcesNetworkCardPort { ibDevs := make(map[string]*api.ResourcesNetworkCardPort) for _, card := range nics.Cards { for _, port := range card.Ports { // Skip non-infiniband ports. if port.Protocol != "infiniband" { continue } // Skip port if not parent. if port.ID != parent { continue } // Store infiniband port info. ibDevs[port.ID] = &port } // Skip virtual function (VF) extraction if SRIOV isn't supported on port. if card.SRIOV == nil { continue } // Record if parent has been found as a physical function (PF). parentDev, parentIsPF := ibDevs[parent] for _, VF := range card.SRIOV.VFs { for _, port := range VF.Ports { // Skip non-infiniband VFs. if port.Protocol != "infiniband" { continue } // Skip VF if parent is a PF and VF is not on same port as parent. if parentIsPF && parentDev.Port != port.Port { continue } // Skip VF if parent isn't a PF and VF doesn't match parent name. if !parentIsPF && port.ID != parent { continue } // Store infiniband VF port info. ibDevs[port.ID] = &port } } } return ibDevs } // infinibandAddDevices creates the UNIX devices for the provided IBF device and then configures the // supplied runConfig with the Cgroup rules and mount instructions to pass the device into instance. func infinibandAddDevices(s *state.State, devicesPath string, deviceName string, ibDev *api.ResourcesNetworkCardPort, runConf *deviceConfig.RunConfig) error { if ibDev.Infiniband == nil { return errors.New("No infiniband devices supplied") } // Add IsSM device if defined. if ibDev.Infiniband.IsSMName != "" { device := deviceConfig.Device{ "source": fmt.Sprintf("/dev/infiniband/%s", ibDev.Infiniband.IsSMName), } err := unixDeviceSetup(s, devicesPath, IBDevPrefix, deviceName, device, false, runConf) if err != nil { return err } } // Add MAD device if defined. if ibDev.Infiniband.MADName != "" { device := deviceConfig.Device{ "source": fmt.Sprintf("/dev/infiniband/%s", ibDev.Infiniband.MADName), } err := unixDeviceSetup(s, devicesPath, IBDevPrefix, deviceName, device, false, runConf) if err != nil { return err } } // Add Verb device if defined. if ibDev.Infiniband.VerbName != "" { device := deviceConfig.Device{ "source": fmt.Sprintf("/dev/infiniband/%s", ibDev.Infiniband.VerbName), } err := unixDeviceSetup(s, devicesPath, IBDevPrefix, deviceName, device, false, runConf) if err != nil { return err } } return nil } // infinibandValidMAC validates an infiniband MAC address. Supports both short and long variants, // e.g. "4a:c8:f9:1b:aa:57:ef:19" and "a0:00:0f:c0:fe:80:00:00:00:00:00:00:4a:c8:f9:1b:aa:57:ef:19". func infinibandValidMAC(value string) error { _, err := net.ParseMAC(value) // Check valid lengths and delimiter. if err != nil || (len(value) != 23 && len(value) != 59) || strings.ContainsAny(value, "-.") { return errors.New("Invalid value, must be either 8 or 20 bytes of hex separated by colons") } return nil } // infinibandSetDevMAC detects whether the supplied MAC is a short or long form variant. // If the short form variant is supplied then only the last 8 bytes of the ibDev device's hwaddr // are changed. If the long form variant is supplied then the full 20 bytes of the ibDev device's // hwaddr are changed. func infinibandSetDevMAC(ibDev string, hwaddr string) error { // Handle 20 byte variant, e.g. a0:00:14:c0:fe:80:00:00:00:00:00:00:4a:c8:f9:1b:aa:57:ef:19. if len(hwaddr) == 59 { return NetworkSetDevMAC(ibDev, hwaddr) } // Handle 8 byte variant, e.g. 4a:c8:f9:1b:aa:57:ef:19. if len(hwaddr) == 23 { curHwaddr, err := NetworkGetDevMAC(ibDev) if err != nil { return err } return NetworkSetDevMAC(ibDev, fmt.Sprintf("%s%s", curHwaddr[:36], hwaddr)) } return errors.New("Invalid length") } incus-7.0.0/internal/server/device/device_utils_instance.go000066400000000000000000000011051517523235500241170ustar00rootroot00000000000000package device import ( "slices" "github.com/lxc/incus/v7/internal/server/instance/instancetype" ) // instanceSupported is a helper function to check instance type is supported for validation. // Always returns true if supplied instance type is Any, to support profile validation. func instanceSupported(instType instancetype.Type, supportedTypes ...instancetype.Type) bool { // If instance type is Any, then profile validation is occurring and we need to support this. if instType == instancetype.Any { return true } return slices.Contains(supportedTypes, instType) } incus-7.0.0/internal/server/device/device_utils_jnfiniband_test.go000066400000000000000000000027411517523235500254630ustar00rootroot00000000000000package device import ( "fmt" ) func Example_infinibandValidMAC() { tests := []string{ "00:00:00:00:fe:80:00:00:00:00:00:00:02:00:5e:10:00:00:00:01", // valid long form "a0:00:0f:c0:fe:80:00:00:00:00:00:00:4a:c8:f9:1b:aa:57:ef:19", // valid long form "02:00:5e:10:00:00:00:01", // valid short form "4a:c8:f9:1b:aa:57:ef:19", // valid short form "00-00-00-00-fe-80-00-00-00-00-00-00-02-00-5e-10-00-00-00-01", // invalid delimiter long form "0000.0000.fe80.0000.0000.0000.0200.5e10.0000.0001", // invalid delimiter long form "02-00-5e-10-00-00-00-01", // invalid delimiter short form "0200.5e10.0000.0001", // invalid delimiter short form "00:00:5e:00:53:01", // invalid ethernet MAC "invalid", "", } for _, v := range tests { err := infinibandValidMAC(v) fmt.Printf("%s, %t\n", v, err == nil) } // Output: 00:00:00:00:fe:80:00:00:00:00:00:00:02:00:5e:10:00:00:00:01, true // a0:00:0f:c0:fe:80:00:00:00:00:00:00:4a:c8:f9:1b:aa:57:ef:19, true // 02:00:5e:10:00:00:00:01, true // 4a:c8:f9:1b:aa:57:ef:19, true // 00-00-00-00-fe-80-00-00-00-00-00-00-02-00-5e-10-00-00-00-01, false // 0000.0000.fe80.0000.0000.0000.0200.5e10.0000.0001, false // 02-00-5e-10-00-00-00-01, false // 0200.5e10.0000.0001, false // 00:00:5e:00:53:01, false // invalid, false // , false } incus-7.0.0/internal/server/device/device_utils_network.go000066400000000000000000001135071517523235500240160ustar00rootroot00000000000000package device import ( "context" "errors" "fmt" "net" "net/netip" "os" "slices" "strconv" "strings" "sync" "time" "github.com/mdlayher/arp" "github.com/mdlayher/ndp" "golang.org/x/sys/unix" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" pcidev "github.com/lxc/incus/v7/internal/server/device/pci" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // Instances can be started in parallel, so lock the creation of VLANs. var networkCreateSharedDeviceLock sync.Mutex // NetworkSetDevMTU sets the MTU setting for a named network device if different from current. func NetworkSetDevMTU(devName string, mtu uint32) error { curMTU, err := network.GetDevMTU(devName) if err != nil { return err } // Only try and change the MTU if the requested mac is different to current one. if curMTU != mtu { link := &ip.Link{Name: devName} err := link.SetMTU(mtu) if err != nil { return err } } return nil } // NetworkGetDevMAC retrieves the current MAC setting for a named network device. func NetworkGetDevMAC(devName string) (string, error) { content, err := os.ReadFile(fmt.Sprintf("/sys/class/net/%s/address", devName)) if err != nil { return "", err } return strings.TrimSpace(string(content)), nil } // NetworkSetDevMAC sets the MAC setting for a named network device if different from current. func NetworkSetDevMAC(devName string, mac string) error { curMac, err := NetworkGetDevMAC(devName) if err != nil { return err } // Only try and change the MAC if the requested mac is different to current one. if curMac != mac { hwaddr, err := net.ParseMAC(mac) if err != nil { return fmt.Errorf("Failed parsing MAC address %q: %w", mac, err) } link := &ip.Link{Name: devName} err = link.SetAddress(hwaddr) if err != nil { return err } } return nil } // networkRemoveInterfaceIfNeeded removes a network interface by name but only if no other instance is using it. func networkRemoveInterfaceIfNeeded(state *state.State, nic string, current instance.Instance, parent string, vlanID string) error { // Check if it's used by another instance. instances, err := instance.LoadNodeAll(state, instancetype.Any) if err != nil { return err } for _, inst := range instances { if inst.Name() == current.Name() && inst.Project().Name == current.Project().Name { continue } for devName, dev := range inst.ExpandedDevices() { if dev["type"] != "nic" || dev["vlan"] != vlanID || dev["parent"] != parent { continue } // Check if another running instance created the device, if so, don't touch it. if util.IsTrue(inst.ExpandedConfig()[fmt.Sprintf("volatile.%s.last_state.created", devName)]) { return nil } } } return network.InterfaceRemove(nic) } // networkCreateVlanDeviceIfNeeded creates a VLAN device if doesn't already exist. func networkCreateVlanDeviceIfNeeded(state *state.State, parent string, vlanDevice string, vlanID string, gvrp bool) (string, error) { if vlanID != "" { created, err := network.VLANInterfaceCreate(parent, vlanDevice, vlanID, gvrp) if err != nil { return "", err } if created { return "created", nil } // Check if it was created for another running instance. instances, err := instance.LoadNodeAll(state, instancetype.Any) if err != nil { return "", err } for _, inst := range instances { for devName, dev := range inst.ExpandedDevices() { if dev["type"] != "nic" || dev["vlan"] != vlanID || dev["parent"] != parent { continue } // Check if another running instance created the device, if so, mark it as created. if util.IsTrue(inst.ExpandedConfig()[fmt.Sprintf("volatile.%s.last_state.created", devName)]) { return "reused", nil } } } } return "existing", nil } // networkSnapshotPhysicalNIC records properties of the NIC to volatile so they can be restored later. func networkSnapshotPhysicalNIC(hostName string, volatile map[string]string) error { // Store current MTU for restoration on detach. mtu, err := network.GetDevMTU(hostName) if err != nil { return err } volatile["last_state.mtu"] = fmt.Sprintf("%d", mtu) // Store current MAC for restoration on detach mac, err := NetworkGetDevMAC(hostName) if err != nil { return err } volatile["last_state.hwaddr"] = mac return nil } // networkRestorePhysicalNIC restores NIC properties from volatile to what they were before it was attached. func networkRestorePhysicalNIC(hostName string, volatile map[string]string) error { // If we created the "physical" device and then it should be removed. if util.IsTrue(volatile["last_state.created"]) { return network.InterfaceRemove(hostName) } // Bring the interface down, as this is sometimes needed to change settings on the nic. link := &ip.Link{Name: hostName} err := link.SetDown() if err != nil { return fmt.Errorf("Failed to bring down \"%s\": %w", hostName, err) } // If MTU value is specified then there is an original MTU that needs restoring. if volatile["last_state.mtu"] != "" { mtuInt, err := strconv.ParseUint(volatile["last_state.mtu"], 10, 32) if err != nil { return fmt.Errorf("Failed to convert mtu for \"%s\" mtu \"%s\": %w", hostName, volatile["last_state.mtu"], err) } err = NetworkSetDevMTU(hostName, uint32(mtuInt)) if err != nil { return fmt.Errorf("Failed to restore physical dev \"%s\" mtu to \"%d\": %w", hostName, mtuInt, err) } } // If MAC value is specified then there is an original MAC that needs restoring. if volatile["last_state.hwaddr"] != "" { err := NetworkSetDevMAC(hostName, volatile["last_state.hwaddr"]) if err != nil { return fmt.Errorf("Failed to restore physical dev \"%s\" mac to \"%s\": %w", hostName, volatile["last_state.hwaddr"], err) } } return nil } // networkCreateVethPair creates and configures a veth pair. It will set the hwaddr and mtu settings // in the supplied config to the newly created peer interface. If mtu is not specified, but parent // is supplied in config, then the MTU of the new peer interface will inherit the parent MTU. // Accepts the name of the host side interface as a parameter and returns the peer interface name and MTU used. func networkCreateVethPair(hostName string, m deviceConfig.Device) (string, uint32, error) { var err error veth := &ip.Veth{ Link: ip.Link{ Name: hostName, Up: true, Master: m["vrf"], }, Peer: ip.Link{ Name: network.RandomDevName("veth"), }, } // Set the MTU on both ends. // The host side should always line up with the bridge to avoid accidentally lowering the bridge MTU. // The instance side should use the configured MTU (if any), if not, it should match the host side. var instanceMTU uint32 var parentMTU uint32 if m["parent"] != "" { mtu, err := network.GetDevMTU(m["parent"]) if err != nil { return "", 0, fmt.Errorf("Failed to get the parent MTU: %w", err) } parentMTU = uint32(mtu) } if m["mtu"] != "" { mtu, err := strconv.ParseUint(m["mtu"], 10, 32) if err != nil { return "", 0, fmt.Errorf("Invalid MTU specified: %w", err) } instanceMTU = uint32(mtu) } if instanceMTU == 0 && parentMTU > 0 { instanceMTU = parentMTU } if parentMTU == 0 && instanceMTU > 0 { parentMTU = instanceMTU } if instanceMTU > 0 { veth.Peer.MTU = instanceMTU } if parentMTU > 0 { veth.MTU = parentMTU } // Set the MAC address on peer. if m["hwaddr"] != "" { hwaddr, err := net.ParseMAC(m["hwaddr"]) if err != nil { return "", 0, fmt.Errorf("Failed parsing MAC address %q: %w", m["hwaddr"], err) } veth.Peer.Address = hwaddr } // Set TX queue length on both ends. if m["queue.tx.length"] != "" { nicTXqlen, err := strconv.ParseUint(m["queue.tx.length"], 10, 32) if err != nil { return "", 0, fmt.Errorf("Invalid txqueuelen specified: %w", err) } veth.TXQueueLength = uint32(nicTXqlen) } else if m["parent"] != "" { veth.TXQueueLength, err = network.GetTXQueueLength(m["parent"]) if err != nil { return "", 0, fmt.Errorf("Failed to get the parent txqueuelen: %w", err) } } veth.Peer.TXQueueLength = veth.TXQueueLength // Add and configure the interface in one operation to reduce the number of executions and to avoid // systemd-udevd from applying the default MACAddressPolicy=persistent policy. err = veth.Add() if err != nil { return "", 0, fmt.Errorf("Failed to create the veth interfaces %q and %q: %w", hostName, veth.Peer.Name, err) } return veth.Peer.Name, veth.Peer.MTU, nil } // networkCreateTap creates and configures a TAP device. // Returns the MTU used. func networkCreateTap(hostName string, m deviceConfig.Device) (uint32, error) { tuntap := &ip.Tuntap{ Name: hostName, Mode: "tap", MultiQueue: true, Master: m["vrf"], } err := tuntap.Add() if err != nil { return 0, fmt.Errorf("Failed to create the tap interfaces %q: %w", hostName, err) } reverter := revert.New() defer reverter.Fail() link := &ip.Link{Name: hostName} err = link.SetUp() if err != nil { return 0, fmt.Errorf("Failed to bring up the tap interface %q: %w", hostName, err) } reverter.Add(func() { _ = network.InterfaceRemove(hostName) }) // Set the MTU on both ends. // The host side should always line up with the bridge to avoid accidentally lowering the bridge MTU. // The instance side should use the configured MTU (if any), if not, it should match the host side. var mtu uint32 var instanceMTU uint32 var parentMTU uint32 if m["parent"] != "" { mtu, err := network.GetDevMTU(m["parent"]) if err != nil { return 0, fmt.Errorf("Failed to get the parent MTU: %w", err) } parentMTU = uint32(mtu) } if m["mtu"] != "" { mtu, err := strconv.ParseUint(m["mtu"], 10, 32) if err != nil { return 0, fmt.Errorf("Invalid MTU specified: %w", err) } instanceMTU = uint32(mtu) } mtu = max(instanceMTU, parentMTU) if mtu > 0 { err = NetworkSetDevMTU(hostName, mtu) if err != nil { return 0, fmt.Errorf("Failed to set the MTU %d: %w", mtu, err) } } // Set TX queue length on both ends. var txqueuelen uint32 if m["queue.tx.length"] != "" { nicTXqlen, err := strconv.ParseUint(m["queue.tx.length"], 10, 32) if err != nil { return 0, fmt.Errorf("Invalid txqueuelen specified: %w", err) } txqueuelen = uint32(nicTXqlen) } else if m["parent"] != "" { txqueuelen, err = network.GetTXQueueLength(m["parent"]) if err != nil { return 0, fmt.Errorf("Failed to get the parent txqueuelen: %w", err) } } if txqueuelen > 0 { err = link.SetTXQueueLength(txqueuelen) if err != nil { return 0, fmt.Errorf("Failed to set the TX queue length %d: %w", txqueuelen, err) } } reverter.Success() return mtu, nil } // networkVethFillFromVolatile fills veth host_name and hwaddr fields from volatile if not set in device config. func networkVethFillFromVolatile(device deviceConfig.Device, volatile map[string]string) { // If not configured, check if volatile data contains the most recently added host_name. if device["host_name"] == "" { device["host_name"] = volatile["host_name"] } // If not configured, check if volatile data contains the most recently added hwaddr. if device["hwaddr"] == "" { device["hwaddr"] = volatile["hwaddr"] } } // networkNICRouteAdd applies any static host-side routes configured for an instance NIC. // If viaIPv4 or viaIPv6 are non-empty, they are used as the next-hop gateway for routes of the matching family. func networkNICRouteAdd(routeDev string, viaIPv4 string, viaIPv6 string, routes ...string) error { if !network.InterfaceExists(routeDev) { return fmt.Errorf("Route interface missing %q", routeDev) } var parsedViaIPv4, parsedViaIPv6 net.IP if viaIPv4 != "" && viaIPv4 != "none" { parsedViaIPv4 = net.ParseIP(viaIPv4) if parsedViaIPv4 == nil || parsedViaIPv4.To4() == nil { return fmt.Errorf("Invalid IPv4 next-hop address %q", viaIPv4) } } if viaIPv6 != "" && viaIPv6 != "none" { parsedViaIPv6 = net.ParseIP(viaIPv6) if parsedViaIPv6 == nil || parsedViaIPv6.To4() != nil { return fmt.Errorf("Invalid IPv6 next-hop address %q", viaIPv6) } } reverter := revert.New() defer reverter.Fail() for _, r := range routes { route := r // Local var for revert. ipNet, err := ip.ParseIPNet(route) if err != nil { return fmt.Errorf("Invalid route %q: %w", route, err) } ipVersion := ip.FamilyV4 via := parsedViaIPv4 if ipNet.IP.To4() == nil { ipVersion = ip.FamilyV6 via = parsedViaIPv6 } // Add IP route (using boot proto to avoid conflicts with network defined static routes). r := &ip.Route{ DevName: routeDev, Route: ipNet, Proto: "boot", Family: ipVersion, Via: via, } err = r.Add() if err != nil { return err } reverter.Add(func() { r := &ip.Route{ DevName: routeDev, Route: ipNet, Proto: "boot", Family: ipVersion, Via: via, } _ = r.Flush() }) } reverter.Success() return nil } // networkNICRouteDelete deletes any static host-side routes configured for an instance NIC. // If viaIPv4 or viaIPv6 are non-empty, they are used as the next-hop gateway for routes of the matching family. // Logs any errors and continues to next route to remove. func networkNICRouteDelete(routeDev string, viaIPv4 string, viaIPv6 string, routes ...string) { if routeDev == "" { logger.Errorf("Failed removing static route, empty route device specified") return } if !network.InterfaceExists(routeDev) { return // Routes will already be gone if device doesn't exist. } var parsedViaIPv4, parsedViaIPv6 net.IP if viaIPv4 != "" && viaIPv4 != "none" { parsedViaIPv4 = net.ParseIP(viaIPv4) if parsedViaIPv4 == nil || parsedViaIPv4.To4() == nil { logger.Errorf("Failed to remove static routes from %q: Invalid IPv4 next-hop address %q", routeDev, viaIPv4) return } } if viaIPv6 != "" && viaIPv6 != "none" { parsedViaIPv6 = net.ParseIP(viaIPv6) if parsedViaIPv6 == nil || parsedViaIPv6.To4() != nil { logger.Errorf("Failed to remove static routes from %q: Invalid IPv6 next-hop address %q", routeDev, viaIPv6) return } } for _, r := range routes { route := r // Local var for revert. ipNet, err := ip.ParseIPNet(route) if err != nil { logger.Errorf("Failed to remove static route %q to %q: %v", route, routeDev, err) continue } ipVersion := ip.FamilyV4 via := parsedViaIPv4 if ipNet.IP.To4() == nil { ipVersion = ip.FamilyV6 via = parsedViaIPv6 } // Add IP route (using boot proto to avoid conflicts with network defined static routes). r := &ip.Route{ DevName: routeDev, Route: ipNet, Proto: "boot", Family: ipVersion, Via: via, } err = r.Flush() if err != nil { logger.Errorf("Failed to remove static route %q to %q: %v", route, routeDev, err) continue } } } // networkSetupHostVethLimits applies any network rate limits to the veth device specified in the config. func networkSetupHostVethLimits(d *deviceCommon, oldConfig deviceConfig.Device, bridged bool) error { var err error veth := d.config["host_name"] if veth == "" || !network.InterfaceExists(veth) { return fmt.Errorf("Unknown or missing host side veth device %q", veth) } // Apply max limit if d.config["limits.max"] != "" { d.config["limits.ingress"] = d.config["limits.max"] d.config["limits.egress"] = d.config["limits.max"] } // Parse the values var ingressInt int64 if d.config["limits.ingress"] != "" { ingressInt, err = units.ParseBitSizeString(d.config["limits.ingress"]) if err != nil { return err } } var egressInt int64 if d.config["limits.egress"] != "" { egressInt, err = units.ParseBitSizeString(d.config["limits.egress"]) if err != nil { return err } } // Clean any existing entry qdiscIngress := &ip.QdiscIngress{Qdisc: ip.Qdisc{Dev: veth, Handle: "ffff:0"}} err = qdiscIngress.Delete() if err != nil && !errors.Is(err, unix.ENOENT) { return err } qdiscHTB := &ip.QdiscHTB{Qdisc: ip.Qdisc{Dev: veth, Handle: "1:0", Parent: "root"}} err = qdiscHTB.Delete() if err != nil && !errors.Is(err, unix.ENOENT) { return err } // Apply new limits if d.config["limits.ingress"] != "" { qdiscHTB = &ip.QdiscHTB{Qdisc: ip.Qdisc{Dev: veth, Handle: "1:0", Parent: "root"}, Default: 0x10} err := qdiscHTB.Add() if err != nil { return fmt.Errorf("Failed to create root tc qdisc: %s", err) } classHTB := &ip.ClassHTB{Class: ip.Class{Dev: veth, Parent: "1:0", Classid: "1:10"}, Rate: fmt.Sprintf("%dbit", ingressInt)} err = classHTB.Add() if err != nil { return fmt.Errorf("Failed to create limit tc class: %s", err) } filter := &ip.U32Filter{Filter: ip.Filter{Dev: veth, Parent: "1:0", Protocol: "all", Flowid: "1:1"}, Value: 0, Mask: 0} err = filter.Add() if err != nil { return fmt.Errorf("Failed to create tc filter: %s", err) } } if d.config["limits.egress"] != "" { qdiscIngress = &ip.QdiscIngress{Qdisc: ip.Qdisc{Dev: veth, Handle: "ffff:0"}} err := qdiscIngress.Add() if err != nil { return fmt.Errorf("Failed to create ingress tc qdisc: %s", err) } police := &ip.ActionPolice{Rate: uint32(egressInt / 8), Burst: uint32(egressInt / 40), Mtu: 65535, Drop: true} filter := &ip.U32Filter{Filter: ip.Filter{Dev: veth, Parent: "ffff:0", Protocol: "all"}, Value: 0, Mask: 0, Actions: []ip.Action{police}} err = filter.Add() if err != nil { return fmt.Errorf("Failed to create ingress tc filter: %s", err) } } var networkPriority uint64 if d.config["limits.priority"] != "" { networkPriority, err = strconv.ParseUint(d.config["limits.priority"], 10, 32) if err != nil { return fmt.Errorf("Failed to parse limits.priority %q: %w", d.config["limits.priority"], err) } } if oldConfig != nil && oldConfig["limits.priority"] != d.config["limits.priority"] { err = d.state.Firewall.InstanceClearNetPrio(d.inst.Project().Name, d.inst.Name(), veth) if err != nil { return err } } if oldConfig == nil || oldConfig["limits.priority"] != d.config["limits.priority"] { if networkPriority != 0 { err = d.state.Firewall.InstanceSetupNetPrio(d.inst.Project().Name, d.inst.Name(), veth, uint32(networkPriority)) if err != nil { return fmt.Errorf("Failed to setup instance device network priority: %w", err) } } } return nil } // networkClearHostVethLimits clears any network rate limits to the veth device specified in the config. func networkClearHostVethLimits(d *deviceCommon) error { // Detached NICs cannot be cleaned up this way. if !util.IsTrueOrEmpty(d.config["attached"]) { return nil } return d.state.Firewall.InstanceClearNetPrio(d.inst.Project().Name, d.inst.Name(), d.config["host_name"]) } // networkValidGateway validates the gateway value. func networkValidGateway(value string) error { if slices.Contains([]string{"none", "auto"}, value) { return nil } return fmt.Errorf("Invalid gateway: %s", value) } // bgpAddPrefix adds external routes to the BGP server. func bgpAddPrefix(d *deviceCommon, n network.Network, config map[string]string) error { // BGP is only valid when tied to a managed network. if config["network"] == "" { return nil } // Parse nexthop configuration. nexthopV4 := net.ParseIP(n.Config()["bgp.ipv4.nexthop"]) if nexthopV4 == nil { nexthopV4 = net.ParseIP(n.Config()["volatile.network.ipv4.address"]) if nexthopV4 == nil { nexthopV4 = net.ParseIP("0.0.0.0") } } nexthopV6 := net.ParseIP(n.Config()["bgp.ipv6.nexthop"]) if nexthopV6 == nil { nexthopV6 = net.ParseIP(n.Config()["volatile.network.ipv6.address"]) if nexthopV6 == nil { nexthopV6 = net.ParseIP("::") } } // Add the prefixes. bgpOwner := fmt.Sprintf("instance_%d_%s", d.inst.ID(), d.name) if config["ipv4.routes.external"] != "" { for _, prefix := range util.SplitNTrimSpace(config["ipv4.routes.external"], ",", -1, true) { _, prefixNet, err := net.ParseCIDR(prefix) if err != nil { return err } err = d.state.BGP.AddPrefix(*prefixNet, nexthopV4, bgpOwner) if err != nil { return err } } } if config["ipv6.routes.external"] != "" { for _, prefix := range util.SplitNTrimSpace(config["ipv6.routes.external"], ",", -1, true) { _, prefixNet, err := net.ParseCIDR(prefix) if err != nil { return err } err = d.state.BGP.AddPrefix(*prefixNet, nexthopV6, bgpOwner) if err != nil { return err } } } return nil } func bgpRemovePrefix(d *deviceCommon, config map[string]string) error { // BGP is only valid when tied to a managed network. if config["network"] == "" { return nil } // Load the network configuration. err := d.state.BGP.RemovePrefixByOwner(fmt.Sprintf("instance_%d_%s", d.inst.ID(), d.name)) if err != nil { return err } return nil } // networkSRIOVParentVFInfo returns info about an SR-IOV virtual function from the parent NIC. func networkSRIOVParentVFInfo(vfParent string, vfID int) (ip.VirtFuncInfo, error) { link := &ip.Link{Name: vfParent} vfi, err := link.GetVFInfo(vfID) return vfi, err } // networkSRIOVSetupVF configures a SR-IOV virtual function (VF) on the parent (PF) and stores original properties // of the PF and VF devices into volatile for restoration on detach. // The useSpoofCheck argument controls whether to use the spoof check feature for the VF on the parent device. // If this is false then "security.mac_filtering" must not be enabled. // Returns VF PCI device info and IOMMU group number for VMs. func networkSRIOVSetupVF(d deviceCommon, vfParent string, vfDevice string, vfID int, volatile map[string]string) (pcidev.Device, uint64, error) { var vfPCIDev pcidev.Device // Retrieve VF settings from parent device. vfInfo, err := networkSRIOVParentVFInfo(vfParent, vfID) if err != nil { return vfPCIDev, 0, err } reverter := revert.New() defer reverter.Fail() // Record properties of VF settings on the parent device. volatile["last_state.vf.parent"] = vfParent volatile["last_state.vf.hwaddr"] = vfInfo.Address.String() volatile["last_state.vf.id"] = fmt.Sprintf("%d", vfID) volatile["last_state.vf.vlan"] = fmt.Sprintf("%d", vfInfo.VLAN) volatile["last_state.vf.spoofcheck"] = fmt.Sprintf("%t", vfInfo.SpoofCheck) // Only persist trust state if it's supported by the NIC. if vfInfo.Trusted != ^uint32(0) { volatile["last_state.vf.trusted"] = fmt.Sprintf("%d", vfInfo.Trusted) } // Record the host interface we represents the VF device which we will move into instance. volatile["host_name"] = vfDevice volatile["last_state.created"] = "false" // Indicates don't delete device at stop time. // Record properties of VF device. err = networkSnapshotPhysicalNIC(volatile["host_name"], volatile) if err != nil { return vfPCIDev, 0, fmt.Errorf("Failed recording NIC %q settings: %w", volatile["host_name"], err) } // Get VF device's PCI Slot Name so we can unbind and rebind it from the host. vfPCIDev, err = network.SRIOVGetVFDevicePCISlot(vfParent, volatile["last_state.vf.id"]) if err != nil { return vfPCIDev, 0, fmt.Errorf("Failed getting PCI slot for VF %q: %w", volatile["last_state.vf.id"], err) } // Unbind VF device from the host so that the settings will take effect when we rebind it. err = pcidev.DeviceUnbind(vfPCIDev) if err != nil { return vfPCIDev, 0, err } reverter.Add(func() { _ = pcidev.DeviceProbe(vfPCIDev) }) // Setup VF VLAN if specified. if d.config["vlan"] != "" { link := &ip.Link{Name: vfParent} err := link.SetVfVlan(volatile["last_state.vf.id"], d.config["vlan"]) if err != nil { return vfPCIDev, 0, fmt.Errorf("Failed setting VLAN for VF %q: %w", volatile["last_state.vf.id"], err) } } // Setup VF trust setting if specified if d.config["security.trusted"] != "" { if vfInfo.Trusted == ^uint32(0) { return vfPCIDev, 0, fmt.Errorf("Failed setting trusted for vf %q: The driver reported missing support for this feature", volatile["last_state.vf.id"]) } link := &ip.Link{Name: vfParent} toSet := util.IsTrue(d.config["security.trusted"]) err = link.SetVfTrusted(volatile["last_state.vf.id"], toSet) if err != nil { return vfPCIDev, 0, fmt.Errorf("Failed setting trusted to %t for VF %q: %w", toSet, volatile["last_state.vf.id"], err) } } // Setup VF MAC spoofing protection if specified. // The ordering of this section is very important, as Intel cards require a very specific // order of setup to allow setting custom MACs when using spoof check mode. if util.IsTrue(d.config["security.mac_filtering"]) { // If no MAC specified in config, use current VF interface MAC. mac := d.config["hwaddr"] if mac == "" { mac = volatile["last_state.hwaddr"] } // Set MAC on VF (this combined with spoof checking prevents any other MAC being used). link := &ip.Link{Name: vfParent} err = link.SetVfAddress(volatile["last_state.vf.id"], mac) if err != nil { return vfPCIDev, 0, fmt.Errorf("Failed setting MAC for VF %q: %w", volatile["last_state.vf.id"], err) } // Now that MAC is set on VF, we can enable spoof checking. err = link.SetVfSpoofchk(volatile["last_state.vf.id"], true) if err != nil { return vfPCIDev, 0, fmt.Errorf("Failed enabling spoof check for VF %q: %w", volatile["last_state.vf.id"], err) } } else { // Try to reset VF to ensure no previous MAC restriction exists, as some devices require this // before being able to set a new VF MAC or disable spoofchecking. However some devices don't // allow it so ignore failures. link := &ip.Link{Name: vfParent} _ = link.SetVfAddress(volatile["last_state.vf.id"], "00:00:00:00:00:00") // Ensure spoof checking is disabled if not enabled in instance (only for real VF). err = link.SetVfSpoofchk(volatile["last_state.vf.id"], false) if err != nil && d.config["security.mac_filtering"] != "" { return vfPCIDev, 0, fmt.Errorf("Failed disabling spoof check for VF %q: %w", volatile["last_state.vf.id"], err) } // Set MAC on VF if specified (this should be passed through into VM when it is bound to vfio-pci). if d.inst.Type() == instancetype.VM { // If no MAC specified in config, use current VF interface MAC. mac := d.config["hwaddr"] if mac == "" { mac = volatile["last_state.hwaddr"] } err = link.SetVfAddress(volatile["last_state.vf.id"], mac) if err != nil { return vfPCIDev, 0, fmt.Errorf("Failed setting MAC for VF %q: %w", volatile["last_state.vf.id"], err) } } } // pciIOMMUGroup, used for VM physical passthrough. var pciIOMMUGroup uint64 if d.inst.Type() == instancetype.Container { // Bind VF device onto the host so that the settings will take effect. err = networkPCIBindWaitInterface(vfPCIDev, volatile["host_name"]) if err != nil { return vfPCIDev, 0, err } } else if d.inst.Type() == instancetype.VM { pciIOMMUGroup, err = pcidev.DeviceIOMMUGroup(vfPCIDev.SlotName) if err != nil { return vfPCIDev, 0, fmt.Errorf("Failed getting IOMMU group for VF device %q: %w", vfPCIDev.SlotName, err) } if d.config["acceleration"] != "vdpa" { // Register VF device with vfio-pci driver so it can be passed to VM. err = pcidev.DeviceDriverOverride(vfPCIDev, "vfio-pci") if err != nil { return vfPCIDev, 0, fmt.Errorf("Failed overriding driver for VF device %q: %w", vfPCIDev.SlotName, err) } } else { // Bind VF device onto the host so that the settings will take effect. err = networkPCIBindWaitInterface(vfPCIDev, volatile["host_name"]) if err != nil { return vfPCIDev, 0, err } } // Record original driver used by VF device for restore. volatile["last_state.pci.driver"] = vfPCIDev.Driver } reverter.Success() return vfPCIDev, pciIOMMUGroup, nil } // networkSRIOVRestoreVF restores SR-IOV VF device settings on parent PF and on VF NIC. Used when removing a VF NIC // from an instance. Use volatile data that was stored when the device was first added with networkSRIOVSetupVF(). // The useSpoofCheck argument controls whether to use the spoof check feature for the VF on the parent device. func networkSRIOVRestoreVF(d deviceCommon, useSpoofCheck bool, volatile map[string]string) error { // Retrieve parent interface from config or volatile. parent := d.config["parent"] if parent == "" { parent = volatile["last_state.vf.parent"] } // Nothing to do if we don't know the original device name or the VF ID. if volatile["host_name"] == "" || volatile["last_state.vf.id"] == "" || parent == "" { return nil } reverter := revert.New() defer reverter.Fail() // Get VF device's PCI info so we can unbind and rebind it from the host. vfPCIDev, err := network.SRIOVGetVFDevicePCISlot(parent, volatile["last_state.vf.id"]) if err != nil { return err } // Unbind VF device from the host so that the restored settings will take effect when we rebind it. err = pcidev.DeviceUnbind(vfPCIDev) if err != nil { return err } if d.inst.Type() == instancetype.VM { // Before we bind the device back to the host, ensure we restore the original driver info as it // should be currently set to vfio-pci. err = pcidev.DeviceSetDriverOverride(vfPCIDev, volatile["last_state.pci.driver"]) if err != nil { return err } } // However we return from this function, we must try to rebind the VF so its not orphaned. // The OS won't let an already bound device be bound again so is safe to call twice. reverter.Add(func() { _ = pcidev.DeviceProbe(vfPCIDev) }) // Reset VF VLAN if specified if volatile["last_state.vf.vlan"] != "" { link := &ip.Link{Name: parent} err := link.SetVfVlan(volatile["last_state.vf.id"], volatile["last_state.vf.vlan"]) if err != nil { return err } } // Reset VF MAC spoofing protection if recorded. Do this first before resetting the MAC // to avoid any issues with zero MACs refusing to be set whilst spoof check is on. if volatile["last_state.vf.spoofcheck"] != "" { mode := util.IsTrue(volatile["last_state.vf.spoofcheck"]) link := &ip.Link{Name: parent} err := link.SetVfSpoofchk(volatile["last_state.vf.id"], mode) if err != nil && d.config["security.mac_filtering"] != "" { return err } } // Reset VF MAC specified if specified. if volatile["last_state.vf.hwaddr"] != "" { link := &ip.Link{Name: parent} err := link.SetVfAddress(volatile["last_state.vf.id"], volatile["last_state.vf.hwaddr"]) if err != nil { return err } } // Reset VF trusted if specified. if volatile["last_state.vf.trusted"] != "" { mode := util.IsTrue(volatile["last_state.vf.trusted"]) link := &ip.Link{Name: parent} err := link.SetVfTrusted(volatile["last_state.vf.id"], mode) if err != nil && d.config["security.trusted"] != "" { return err } } // Bind VF device onto the host so that the settings will take effect. err = networkPCIBindWaitInterface(vfPCIDev, volatile["host_name"]) if err != nil { return err } // Restore VF interface settings. err = networkRestorePhysicalNIC(volatile["host_name"], volatile) if err != nil { return err } reverter.Success() return nil } // networkPCIBindWaitInterface repeatedly requests the pciDev is probed to be bound to the override driver and // checks whether the expected network interface has appeared as the result of the device driver being bound. func networkPCIBindWaitInterface(pciDev pcidev.Device, ifName string) error { var err error waitDuration := time.Second * 10 waitUntil := time.Now().Add(waitDuration) // Keep requesting the device driver be probed in case it was not ready previously or the expected // interface has not appeared yet. The device can be probed multiple times safely. i := 0 for { err = pcidev.DeviceProbe(pciDev) if err == nil && network.InterfaceExists(ifName) { return nil } if time.Now().After(waitUntil) { if err != nil { return fmt.Errorf("Failed binding interface %q after %v: %w", ifName, waitDuration, err) } return fmt.Errorf("Failed binding interface %q after %v", ifName, waitDuration) } if i <= 5 { // Retry more quickly early on. time.Sleep(time.Millisecond * time.Duration(i) * 10) } else { time.Sleep(time.Second) } i++ } } // networkSRIOVSetupContainerVFNIC configures the VF NIC interface ready for moving into container. // It configures the MAC address and MTU, then brings the interface up. func networkSRIOVSetupContainerVFNIC(hostName string, macPattern string, config map[string]string) error { // Set the MAC address. if config["hwaddr"] != "" { hwaddr, err := net.ParseMAC(config["hwaddr"]) if err != nil { return fmt.Errorf("Failed parsing MAC address %q: %w", config["hwaddr"], err) } // Retry a few times as some vendors take a little while to initialize. link := &ip.Link{Name: hostName} for range 10 { err = link.SetAddress(hwaddr) if err == nil { break } time.Sleep(500 * time.Millisecond) } if err != nil { return fmt.Errorf("Failed setting MAC address %q on %q: %w", config["hwaddr"], hostName, err) } } // Set the MTU. if config["mtu"] != "" { mtu, err := strconv.ParseUint(config["mtu"], 10, 32) if err != nil { return fmt.Errorf("Invalid VF MTU specified %q: %w", config["mtu"], err) } link := &ip.Link{Name: hostName} err = link.SetMTU(uint32(mtu)) if err != nil { return fmt.Errorf("Failed setting MTU %q on %q: %w", config["mtu"], hostName, err) } } // Bring the interface up. link := &ip.Link{Name: hostName} err := link.SetUp() if err != nil { if config["hwaddr"] != "" { return fmt.Errorf("Failed to bring up VF interface %q: %w", hostName, err) } upErr := err // If interface fails to come up and MAC not previously set, some NICs require us to set // a specific MAC before being allowed to bring up the VF interface. So check if interface // has an empty MAC and set a random one if needed. vfIF, err := net.InterfaceByName(hostName) if err != nil { return fmt.Errorf("Failed getting interface info for VF %q: %w", hostName, err) } // If the VF interface has a MAC already, something else prevented bringing interface up. if vfIF.HardwareAddr.String() != "00:00:00:00:00:00" { return fmt.Errorf("Failed to bring up VF interface %q: %w", hostName, upErr) } // Try using a random MAC address and bringing interface up. randMAC, err := instance.DeviceNextInterfaceHWAddr(macPattern) if err != nil { return fmt.Errorf("Failed generating random MAC for VF %q: %w", hostName, err) } hwaddr, err := net.ParseMAC(randMAC) if err != nil { return fmt.Errorf("Failed parsing MAC address %q: %w", randMAC, err) } link := &ip.Link{Name: hostName} err = link.SetAddress(hwaddr) if err != nil { return fmt.Errorf("Failed to set random MAC address %q on %q: %w", randMAC, hostName, err) } err = link.SetUp() if err != nil { return fmt.Errorf("Failed to bring up VF interface %q: %w", hostName, err) } } return nil } // isIPAvailable checks if address responds to ARP/NDP neighbour probe on the parentInterface. // Returns true if IP is in use. func isIPAvailable(ctx context.Context, address net.IP, parentInterface string) (bool, error) { deadline, ok := ctx.Deadline() if !ok { // Set default timeout of 500ms if no deadline context provided. var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, time.Duration(500*time.Millisecond)) defer cancel() deadline, _ = ctx.Deadline() } // Handle IPv4 address. if address.To4() != nil { err := pingOverIfaceByName(deadline, address, parentInterface) if err != nil { var ne net.Error if errors.As(err, &ne) && ne.Timeout() { return false, nil } return false, err } return true, nil } // Handle IPv6 address. networkInterface, err := net.InterfaceByName(parentInterface) if err != nil { return false, err } conn, _, err := ndp.Listen(networkInterface, ndp.LinkLocal) if err != nil { return false, err } defer func() { _ = conn.Close() }() netipAddr, ok := netip.AddrFromSlice(address) if !ok { return false, errors.New("Couldn't convert address to netip") } solicitedNodeMulticast, err := ndp.SolicitedNodeMulticast(netipAddr) if err != nil { return false, err } neighbourSolicitationMessage := &ndp.NeighborSolicitation{ TargetAddress: netipAddr, } _ = conn.SetDeadline(deadline) err = conn.WriteTo(neighbourSolicitationMessage, nil, solicitedNodeMulticast) if err != nil { return false, err } _ = conn.SetDeadline(deadline) msg, _, _, err := conn.ReadFrom() if err != nil { var cause net.Error if errors.As(err, &cause) && cause.Timeout() { return false, nil } return false, err } neighbourAdvertisement, ok := msg.(*ndp.NeighborAdvertisement) if ok && neighbourAdvertisement.TargetAddress == netipAddr { return true, nil } return false, nil } // networkVLANListExpand takes in a list of raw VLAN values (string) that includes // different VLAN formats ("number" and "start-end") and convert them into a list of // expanded VLAN values in integer. func networkVLANListExpand(rawVLANValues []string) ([]int, error) { var networkVLANList []int for _, vlan := range rawVLANValues { start, count, err := validate.ParseNetworkVLANRange(vlan) if err != nil { return nil, err } for i := start; i < start+count; i++ { networkVLANList = append(networkVLANList, i) } } return networkVLANList, nil } // pingOverIfaceByName sends an ARP request to the given IPv4 address using the specified network interface. // It respects the provided deadline and returns an error if resolution fails (unless due to timeout). func pingOverIfaceByName(deadline time.Time, address net.IP, parentInterface string) error { // Obtain the network interface. ifi, err := net.InterfaceByName(parentInterface) if err != nil { return err } // Open an ARP client on that interface. c, err := arp.Dial(ifi) if err != nil { return err } defer func() { _ = c.Close() }() // Honour the caller’s deadline. _ = c.SetDeadline(deadline) // Convert to netip.Addr which arp.Client expects. netipAddr, ok := netip.AddrFromSlice(address.To4()) if !ok { return fmt.Errorf("Invalid IPv4 address: %v", address) } // Try to resolve the IP → MAC. If it answers, the IP is in use. _, err = c.Resolve(netipAddr) if err != nil { return err } return nil } incus-7.0.0/internal/server/device/device_utils_unix.go000066400000000000000000000446771517523235500233230ustar00rootroot00000000000000package device import ( "errors" "fmt" "io/fs" "maps" "os" "path/filepath" "slices" "strconv" "strings" "golang.org/x/sys/unix" internalIO "github.com/lxc/incus/v7/internal/io" "github.com/lxc/incus/v7/internal/linux" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) // unixDefaultMode default mode to create unix devices with if not specified in device config. const unixDefaultMode = 0o660 // unixDeviceAttributes returns the device type, major and minor numbers for a device. func unixDeviceAttributes(path string) (string, uint32, uint32, error) { // Get a stat struct from the provided path stat := unix.Stat_t{} err := unix.Stat(path, &stat) if err != nil { return "", 0, 0, err } // Check what kind of file it is dType := "" if stat.Mode&unix.S_IFMT == unix.S_IFBLK { dType = "b" } else if stat.Mode&unix.S_IFMT == unix.S_IFCHR { dType = "c" } else { return "", 0, 0, errors.New("Not a device") } // Return the device information major := unix.Major(uint64(stat.Rdev)) minor := unix.Minor(uint64(stat.Rdev)) return dType, major, minor, nil } // unixDeviceModeOct converts a string unix octal mode to an int. func unixDeviceModeOct(strmode string) (int, error) { i, err := strconv.ParseInt(strmode, 8, 32) if err != nil { return 0, fmt.Errorf("Bad device mode: %s", strmode) } return int(i), nil } // UnixDevice contains information about a created UNIX device. type UnixDevice struct { HostPath string // Absolute path to the device on the host. RelativePath string // Relative path where the device will be mounted inside instance. Type string // Type of device; c (for char) or b for (block). Major uint32 // Major number. Minor uint32 // Minor number. Mode os.FileMode // File mode. UID int // Owner UID. GID int // Owner GID. } // unixDeviceSourcePath returns the absolute path for a device on the host. // This is based on the "source" property of the device's config, or the "path" property if "source" // not define. func unixDeviceSourcePath(m deviceConfig.Device) string { srcPath := m["source"] if srcPath == "" { srcPath = m["path"] } return srcPath } // unixDeviceDestPath returns the absolute path for a device inside an instance. // This is based on the "path" property of the device's config, or the "source" property if "path" // not defined. func unixDeviceDestPath(m deviceConfig.Device) string { destPath := m["path"] if destPath == "" { destPath = m["source"] } return destPath } // UnixDeviceCreate creates a UNIX device (either block or char). If the supplied device config map // contains a major and minor number for the device, then a stat is avoided, otherwise this info // retrieved from the origin device. Similarly, if a mode is supplied in the device config map or // defaultMode is set as true, then the device is created with the supplied or default mode (0660) // respectively, otherwise the origin device's mode is used. If the device config doesn't contain a // type field then it defaults to created a unix-char device. The ownership of the created device // defaults to root (0) but can be specified with the uid and gid fields in the device config map. // It returns a UnixDevice containing information about the device created. func UnixDeviceCreate(s *state.State, idmapSet *idmap.Set, devicesPath string, prefix string, m deviceConfig.Device, defaultMode bool) (*UnixDevice, error) { var err error d := UnixDevice{} // Extra checks for nesting. if s.OS.RunningInUserNS { for key, value := range m { if slices.Contains([]string{"major", "minor", "mode", "uid", "gid"}, key) && value != "" { return nil, fmt.Errorf("The \"%s\" property may not be set when adding a device to a nested container", key) } } } srcPath := unixDeviceSourcePath(m) // Get the major/minor of the device we want to create. if m["major"] == "" && m["minor"] == "" { // If no major and minor are set, use those from the device on the host. _, d.Major, d.Minor, err = unixDeviceAttributes(srcPath) if err != nil { return nil, fmt.Errorf("Failed to get device attributes for %s: %w", srcPath, err) } } else if m["major"] == "" || m["minor"] == "" { return nil, fmt.Errorf("Both major and minor must be supplied for device: %s", srcPath) } else { tmp, err := strconv.ParseUint(m["major"], 10, 32) if err != nil { return nil, fmt.Errorf("Bad major %s in device %s", m["major"], srcPath) } d.Major = uint32(tmp) tmp, err = strconv.ParseUint(m["minor"], 10, 32) if err != nil { return nil, fmt.Errorf("Bad minor %s in device %s", m["minor"], srcPath) } d.Minor = uint32(tmp) } // Get the device mode (defaults to unixDefaultMode if not supplied). d.Mode = os.FileMode(unixDefaultMode) if m["mode"] != "" { tmp, err := unixDeviceModeOct(m["mode"]) if err != nil { return nil, fmt.Errorf("Bad mode %s in device %s", m["mode"], srcPath) } d.Mode = os.FileMode(tmp) } else if !defaultMode { // If not specified mode in device config, and default mode is false, then try and // read the source device's mode and use that inside the instance. d.Mode, err = internalIO.GetPathMode(srcPath) if err != nil { errno, isErrno := linux.GetErrno(err) if !isErrno || !errors.Is(errno, unix.ENOENT) { return nil, fmt.Errorf("Failed to retrieve mode of device %s: %w", srcPath, err) } d.Mode = os.FileMode(unixDefaultMode) } } if m["type"] == "unix-block" { d.Mode |= unix.S_IFBLK d.Type = "b" } else { d.Mode |= unix.S_IFCHR d.Type = "c" } // Get the device owner. if m["uid"] != "" { d.UID, err = strconv.Atoi(m["uid"]) if err != nil { return nil, fmt.Errorf("Invalid uid %s in device %s", m["uid"], srcPath) } } if m["gid"] != "" { d.GID, err = strconv.Atoi(m["gid"]) if err != nil { return nil, fmt.Errorf("Invalid gid %s in device %s", m["gid"], srcPath) } } // Create the devices directory if missing. if !util.PathExists(devicesPath) { err := os.Mkdir(devicesPath, 0o711) if err != nil { return nil, fmt.Errorf("Failed to create devices path: %s", err) } } destPath := unixDeviceDestPath(m) relativeDestPath := strings.TrimPrefix(destPath, "/") devName := linux.PathNameEncode(deviceJoinPath(prefix, relativeDestPath)) devPath := filepath.Join(devicesPath, devName) // Create the new entry. if !s.OS.RunningInUserNS { if s.OS.Nodev { return nil, errors.New("Can't create device as devices path is mounted nodev") } devNum := int(unix.Mkdev(d.Major, d.Minor)) err := unix.Mknod(devPath, uint32(d.Mode), devNum) if err != nil { return nil, fmt.Errorf("Failed to create device %s for %s: %w", devPath, srcPath, err) } err = os.Chown(devPath, d.UID, d.GID) if err != nil { return nil, fmt.Errorf("Failed to chown device %s: %w", devPath, err) } // Needed as mknod respects the umask. err = os.Chmod(devPath, d.Mode) if err != nil { return nil, fmt.Errorf("Failed to chmod device %s: %w", devPath, err) } if idmapSet != nil { err := idmapSet.ShiftPath(devPath, nil) if err != nil { // uidshift failing is weird, but not a big problem. Log and proceed. logger.Debugf("Failed to uidshift device %s: %s\n", srcPath, err) } } } else { f, err := os.Create(devPath) if err != nil { return nil, err } _ = f.Close() err = DiskMount(srcPath, devPath, false, "", nil, "none") if err != nil { return nil, err } } d.HostPath = devPath d.RelativePath = relativeDestPath return &d, nil } // unixDeviceSetup creates a UNIX device on host and then configures supplied RunConfig with the // mount and cgroup rule instructions to have it be attached to the instance. If defaultMode is true // or mode is supplied in the device config then the origin device does not need to be accessed for // its file mode. func unixDeviceSetup(s *state.State, devicesPath string, typePrefix string, deviceName string, m deviceConfig.Device, defaultMode bool, runConf *deviceConfig.RunConfig) error { // Before creating the device, check that another existing device isn't using the same mount // path inside the instance as our device. If we find an existing device with the same mount // path we will skip mounting our device inside the instance. This can happen when multiple // devices share the same parent device (such as Nvidia GPUs and Infiniband devices). // Convert the requested dest path inside the instance to an encoded relative one. ourDestPath := unixDeviceDestPath(m) ourEncRelDestFile := linux.PathNameEncode(strings.TrimPrefix(ourDestPath, "/")) // Load all existing host devices. dents, err := os.ReadDir(devicesPath) if err != nil { if !errors.Is(err, fs.ErrNotExist) { return err } } dupe := false for _, ent := range dents { devName := ent.Name() // Remove the device type and name prefix, leaving just the encoded dest path. idx := strings.LastIndex(devName, ".") if idx == -1 { continue } encRelDestFile := devName[idx+1:] // If the encoded relative path of the device file matches the encoded relative dest // path of our new device then return as we do not want to have // it mounted or cgroup rules created. if encRelDestFile == ourEncRelDestFile { dupe = true // There is an existing device using the same mount path. break } } // Create the device on the host. ourPrefix := deviceJoinPath(typePrefix, deviceName) d, err := UnixDeviceCreate(s, nil, devicesPath, ourPrefix, m, defaultMode) if err != nil { return err } // If there was an existing device using the same mount path detected then skip mounting. if dupe { return nil } // Ask for a mount to be performed. runConf.Mounts = append(runConf.Mounts, deviceConfig.MountEntryItem{ DevPath: d.HostPath, TargetPath: d.RelativePath, FSType: "none", Opts: []string{"bind", "create=file"}, OwnerShift: deviceConfig.MountOwnerShiftStatic, }) // Ask for cgroups to be configured. runConf.CGroups = append(runConf.CGroups, deviceConfig.RunConfigItem{ Key: "devices.allow", Value: fmt.Sprintf("%s %d:%d rwm", d.Type, d.Major, d.Minor), }) return nil } // unixDeviceSetupCharNum calls unixDeviceSetup and overrides the supplied device config with the // type as "unix-char" and the supplied major and minor numbers. This function can be used when you // already know the device's major and minor numbers to avoid unixDeviceSetup() having to stat the // device to ascertain these attributes. If defaultMode is true or mode is supplied in the device // config then the origin device does not need to be accessed for its file mode. func unixDeviceSetupCharNum(s *state.State, devicesPath string, typePrefix string, deviceName string, m deviceConfig.Device, major uint32, minor uint32, path string, defaultMode bool, runConf *deviceConfig.RunConfig) error { configCopy := deviceConfig.Device{} maps.Copy(configCopy, m) // Overridng these in the config copy should avoid the need for unixDeviceSetup to stat // the origin device to ascertain this information. configCopy["type"] = "unix-char" configCopy["major"] = fmt.Sprintf("%d", major) configCopy["minor"] = fmt.Sprintf("%d", minor) configCopy["path"] = path return unixDeviceSetup(s, devicesPath, typePrefix, deviceName, configCopy, defaultMode, runConf) } // unixDeviceSetupBlockNum calls unixDeviceSetup and overrides the supplied device config with the // type as "unix-block" and the supplied major and minor numbers. This function can be used when you // already know the device's major and minor numbers to avoid unixDeviceSetup() having to stat the // device to ascertain these attributes. If defaultMode is true or mode is supplied in the device // config then the origin device does not need to be accessed for its file mode. func unixDeviceSetupBlockNum(s *state.State, devicesPath string, typePrefix string, deviceName string, m deviceConfig.Device, major uint32, minor uint32, path string, defaultMode bool, runConf *deviceConfig.RunConfig) error { configCopy := deviceConfig.Device{} maps.Copy(configCopy, m) // Overridng these in the config copy should avoid the need for unixDeviceSetup to stat // the origin device to ascertain this information. configCopy["type"] = "unix-block" configCopy["major"] = fmt.Sprintf("%d", major) configCopy["minor"] = fmt.Sprintf("%d", minor) configCopy["path"] = path return unixDeviceSetup(s, devicesPath, typePrefix, deviceName, configCopy, defaultMode, runConf) } // UnixDeviceExists checks if the unix device already exists in devices path. func UnixDeviceExists(devicesPath string, prefix string, path string) bool { relativeDestPath := strings.TrimPrefix(path, "/") devName := fmt.Sprintf("%s.%s", linux.PathNameEncode(prefix), linux.PathNameEncode(relativeDestPath)) devPath := filepath.Join(devicesPath, devName) return util.PathExists(devPath) } // unixRemoveDevice identifies all files related to the supplied typePrefix and deviceName and then // populates the supplied runConf with the instructions to remove cgroup rules and unmount devices. // It detects if any other devices attached to the instance that share the same prefix have the same // relative mount path inside the instance encoded into the file name. If there is another device // that shares the same mount path then the unmount rule is not added to the runConf as the device // may still be in use with another device. // Accepts an optional file prefix that will be used to narrow the selection of files to remove. func unixDeviceRemove(devicesPath string, typePrefix string, deviceName string, optPrefix string, runConf *deviceConfig.RunConfig) error { // Load all devices. dents, err := os.ReadDir(devicesPath) if err != nil { if !errors.Is(err, fs.ErrNotExist) { return err } } var ourPrefix string // If a prefix override has been supplied, use that for filtering the devices to remove. if optPrefix != "" { ourPrefix = linux.PathNameEncode(deviceJoinPath(typePrefix, deviceName, optPrefix)) } else { ourPrefix = linux.PathNameEncode(deviceJoinPath(typePrefix, deviceName)) } ourDevs := []string{} otherDevs := []string{} for _, ent := range dents { devName := ent.Name() // This device file belongs to our device. if strings.HasPrefix(devName, ourPrefix) { ourDevs = append(ourDevs, devName) continue } // This device file belongs to another device. otherDevs = append(otherDevs, devName) } // It is possible for some devices to share the same device on the same mount point // inside the instance. We extract the relative path of the device that is encoded into its // name on the host so that we can compare the device files for our own device and check // none of them use the same mount point. encRelDevFiles := []string{} for _, otherDev := range otherDevs { // Remove the device type and name prefix, leaving just the encoded dest path. idx := strings.LastIndex(otherDev, ".") if idx == -1 { continue } encRelDestFile := otherDev[idx+1:] encRelDevFiles = append(encRelDevFiles, encRelDestFile) } // Check that none of our devices are in use by another device. for _, ourDev := range ourDevs { // Remove the device type and name prefix, leaving just the encoded dest path. idx := strings.LastIndex(ourDev, ".") if idx == -1 { return fmt.Errorf("Invalid device name \"%s\"", ourDev) } ourEncRelDestFile := ourDev[idx+1:] // Look for devices for other devices that match the same path. dupe := slices.Contains(encRelDevFiles, ourEncRelDestFile) // If a device has been found that points to the same device inside the instance // then we cannot request it be umounted inside the instance as it's still in use. if dupe { continue } // Append this device to the mount rules (these will be unmounted). runConf.Mounts = append(runConf.Mounts, deviceConfig.MountEntryItem{ TargetPath: linux.PathNameDecode(ourEncRelDestFile), }) absDevPath := filepath.Join(devicesPath, ourDev) dType, dMajor, dMinor, err := unixDeviceAttributes(absDevPath) if err != nil { return fmt.Errorf("Failed to get UNIX device attributes for '%s': %w", absDevPath, err) } // Append a deny cgroup rule for this device. runConf.CGroups = append(runConf.CGroups, deviceConfig.RunConfigItem{ Key: "devices.deny", Value: fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor), }) } return nil } // unixDeviceDeleteFiles removes all host side device files for a particular device. // Accepts an optional file prefix that will be used to narrow the selection of files to delete. // This should be run after the files have been detached from the instance as a post hook. func unixDeviceDeleteFiles(s *state.State, devicesPath string, typePrefix string, deviceName string, optPrefix string) error { var ourPrefix string // If a prefix override has been supplied, use that for filtering the devices to remove. if optPrefix != "" { ourPrefix = linux.PathNameEncode(deviceJoinPath(typePrefix, deviceName, optPrefix)) } else { ourPrefix = linux.PathNameEncode(deviceJoinPath(typePrefix, deviceName)) } // Load all devices. dents, err := os.ReadDir(devicesPath) if err != nil { if !errors.Is(err, fs.ErrNotExist) { return err } } // Remove our host side device files. for _, ent := range dents { devName := ent.Name() // This device file belongs to our device. if strings.HasPrefix(devName, ourPrefix) { devPath := filepath.Join(devicesPath, devName) // Remove the host side mount. if s.OS.RunningInUserNS { _ = unix.Unmount(devPath, unix.MNT_DETACH) } // Remove the host side device file. err := os.Remove(devPath) if err != nil { return err } } } return nil } // unixValidDeviceNum validates the major and minor numbers for a UNIX device. func unixValidDeviceNum(value string) error { if value == "" { return nil } _, err := strconv.ParseUint(value, 10, 32) if err != nil { return errors.New("Invalid value for a UNIX device number") } return nil } // unixValidUserID validates the UNIX UID and GID values for ownership. func unixValidUserID(value string) error { if value == "" { return nil } _, err := strconv.ParseUint(value, 10, 32) if err != nil { return errors.New("Invalid value for a UNIX ID") } return nil } // unixValidOctalFileMode validates the UNIX file mode. func unixValidOctalFileMode(value string) error { if value == "" { return nil } _, err := strconv.ParseUint(value, 8, 32) if err != nil { return errors.New("Invalid value for an octal file mode") } return nil } incus-7.0.0/internal/server/device/device_utils_unix_events.go000066400000000000000000000115661517523235500246760ustar00rootroot00000000000000package device import ( "errors" "fmt" "path/filepath" "strings" "sync" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) // UnixEvent represents the properties of a Unix device inotify event. type UnixEvent struct { Action string // The type of event, either add or remove. Path string // The absolute source path on the host. } // UnixSubscription used to subscribe to specific events. type UnixSubscription struct { Path string // The absolute source path on the host. Handler func(UnixEvent) (*deviceConfig.RunConfig, error) // The function to run when an event occurs. } // unixHandlers stores the event handler callbacks for Unix events. var unixHandlers = map[string]UnixSubscription{} // unixMutex controls access to the unixHandlers map. var unixMutex sync.Mutex // unixRegisterHandler registers a handler function to be called whenever a Unix device event occurs. func unixRegisterHandler(s *state.State, inst instance.Instance, deviceName, path string, handler func(UnixEvent) (*deviceConfig.RunConfig, error)) error { if path == "" || handler == nil { return errors.New("Invalid subscription") } unixMutex.Lock() // Null delimited string of project name, instance name and device name. key := fmt.Sprintf("%s\000%s\000%s", inst.Project().Name, inst.Name(), deviceName) unixHandlers[key] = UnixSubscription{ Path: path, Handler: handler, } unixMutex.Unlock() identifier := fmt.Sprintf("%d_%s", inst.ID(), deviceName) path = filepath.Clean(path) // Add inotify watcher to its nearest existing ancestor. err := s.DevMonitor.Watch(path, identifier, func(path, event string) bool { e := unixNewEvent(event, path) unixRunHandlers(s, &e) return true }) if err != nil { return fmt.Errorf("Failed to add %q to watch targets: %w", path, err) } logger.Debug("Added watch target", logger.Ctx{"path": path}) return nil } // unixUnregisterHandler removes a registered Unix handler function for a device. func unixUnregisterHandler(s *state.State, inst instance.Instance, deviceName string) error { unixMutex.Lock() // Null delimited string of project name, instance name and device name. key := fmt.Sprintf("%s\000%s\000%s", inst.Project().Name, inst.Name(), deviceName) sub, exists := unixHandlers[key] if !exists { unixMutex.Unlock() return nil } // Remove active subscription for this device. delete(unixHandlers, key) unixMutex.Unlock() identifier := fmt.Sprintf("%d_%s", inst.ID(), deviceName) err := s.DevMonitor.Unwatch(sub.Path, identifier) if err != nil { return fmt.Errorf("Failed to remove %q from inotify targets: %w", sub.Path, err) } return nil } // unixRunHandlers executes any handlers registered for Unix events. func unixRunHandlers(state *state.State, event *UnixEvent) { unixMutex.Lock() defer unixMutex.Unlock() for key, sub := range unixHandlers { keyParts := strings.SplitN(key, "\000", 3) projectName := keyParts[0] instanceName := keyParts[1] deviceName := keyParts[2] // Delete subscription if no handler function defined. if sub.Handler == nil { delete(unixHandlers, key) continue } // Don't execute handler if subscription path and event paths don't match. if sub.Path != event.Path { continue } // Run handler function. runConf, err := sub.Handler(*event) if err != nil { logger.Error("Unix event hook failed", logger.Ctx{"project": projectName, "instance": instanceName, "device": deviceName, "path": sub.Path, "action": event.Action, "err": err}) continue } // If runConf supplied, load instance and call its Unix event handler function so // any instance specific device actions can occur. if runConf != nil { instance, err := instance.LoadByProjectAndName(state, projectName, instanceName) if err != nil { logger.Error("Unix event loading instance failed", logger.Ctx{"err": err, "project": projectName, "instance": instanceName, "device": deviceName}) continue } err = instance.DeviceEventHandler(runConf) if err != nil { logger.Error("Unix event instance handler failed", logger.Ctx{"err": err, "project": projectName, "instance": instanceName, "device": deviceName}) continue } } } } // unixNewEvent returns a newly created Unix device event struct. // If an empty action is supplied then the action of the event is derived from whether the path // exists (add) or not (removed). This allows the peculiarities of the inotify API to be somewhat // masked by the consuming event handler functions. func unixNewEvent(action string, path string) UnixEvent { if action == "" { if util.PathExists(path) { action = "add" } else { action = "remove" } } return UnixEvent{ Action: action, Path: path, } } incus-7.0.0/internal/server/device/device_utils_unix_hotplug_events.go000066400000000000000000000074221517523235500264340ustar00rootroot00000000000000package device import ( "fmt" "strconv" "strings" "sync" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/logger" ) // UnixHotplugEvent represents the properties of a Unix hotplug device uevent. type UnixHotplugEvent struct { Action string Vendor string Product string PCI string Path string Major uint32 Minor uint32 Subsystem string UeventParts []string UeventLen int } // unixHotplugHandlers stores the event handler callbacks for Unix hotplug events. var unixHotplugHandlers = map[string]func(UnixHotplugEvent) (*deviceConfig.RunConfig, error){} // unixHotplugMutex controls access to the unixHotplugHandlers map. var unixHotplugMutex sync.Mutex // unixHotplugRegisterHandler registers a handler function to be called whenever a Unix hotplug device event occurs. func unixHotplugRegisterHandler(instance instance.Instance, deviceName string, handler func(UnixHotplugEvent) (*deviceConfig.RunConfig, error)) { unixHotplugMutex.Lock() defer unixHotplugMutex.Unlock() // Null delimited string of project name, instance name and device name. key := fmt.Sprintf("%s\000%s\000%s", instance.Project().Name, instance.Name(), deviceName) unixHotplugHandlers[key] = handler } // unixHotplugUnregisterHandler removes a registered Unix hotplug handler function for a device. func unixHotplugUnregisterHandler(instance instance.Instance, deviceName string) { unixHotplugMutex.Lock() defer unixHotplugMutex.Unlock() // Null delimited string of project name, instance name and device name. key := fmt.Sprintf("%s\000%s\000%s", instance.Project().Name, instance.Name(), deviceName) delete(unixHotplugHandlers, key) } // UnixHotplugRunHandlers executes any handlers registered for Unix hotplug events. func UnixHotplugRunHandlers(state *state.State, event *UnixHotplugEvent) { unixHotplugMutex.Lock() defer unixHotplugMutex.Unlock() for key, hook := range unixHotplugHandlers { keyParts := strings.SplitN(key, "\000", 3) projectName := keyParts[0] instanceName := keyParts[1] deviceName := keyParts[2] if hook == nil { delete(unixHotplugHandlers, key) continue } runConf, err := hook(*event) if err != nil { logger.Error("Unix hotplug event hook failed", logger.Ctx{"err": err, "project": projectName, "instance": instanceName, "device": deviceName}) continue } // If runConf supplied, load instance and call its Unix hotplug event handler function so // any instance specific device actions can occur. if runConf != nil { instance, err := instance.LoadByProjectAndName(state, projectName, instanceName) if err != nil { logger.Error("Unix hotplug event loading instance failed", logger.Ctx{"err": err, "project": projectName, "instance": instanceName, "device": deviceName}) continue } err = instance.DeviceEventHandler(runConf) if err != nil { logger.Error("Unix hotplug event instance handler failed", logger.Ctx{"err": err, "project": projectName, "instance": instanceName, "device": deviceName}) continue } } } } // UnixHotplugNewEvent instantiates a new UnixHotplugEvent struct. func UnixHotplugNewEvent(action string, vendor string, product string, pci string, major string, minor string, subsystem string, devname string, ueventParts []string, ueventLen int) (UnixHotplugEvent, error) { majorInt, err := strconv.ParseUint(major, 10, 32) if err != nil { return UnixHotplugEvent{}, err } minorInt, err := strconv.ParseUint(minor, 10, 32) if err != nil { return UnixHotplugEvent{}, err } return UnixHotplugEvent{ action, vendor, product, pci, devname, uint32(majorInt), uint32(minorInt), subsystem, ueventParts, ueventLen, }, nil } incus-7.0.0/internal/server/device/device_utils_usb_events.go000066400000000000000000000075051517523235500245020ustar00rootroot00000000000000package device import ( "fmt" "path/filepath" "strconv" "strings" "sync" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/logger" ) // USBEvent represents the properties of a USB device uevent. type USBEvent struct { Action string Vendor string Product string Serial string Path string Major uint32 Minor uint32 UeventParts []string UeventLen int BusNum int DevNum int } // usbHandlers stores the event handler callbacks for USB events. var usbHandlers = map[string]func(USBEvent) (*deviceConfig.RunConfig, error){} // usbMutex controls access to the usbHandlers map. var usbMutex sync.Mutex // usbRegisterHandler registers a handler function to be called whenever a USB device event occurs. func usbRegisterHandler(inst instance.Instance, deviceName string, handler func(USBEvent) (*deviceConfig.RunConfig, error)) { usbMutex.Lock() defer usbMutex.Unlock() // Null delimited string of project name, instance name and device name. key := fmt.Sprintf("%s\000%s\000%s", inst.Project().Name, inst.Name(), deviceName) usbHandlers[key] = handler } // usbUnregisterHandler removes a registered USB handler function for a device. func usbUnregisterHandler(inst instance.Instance, deviceName string) { usbMutex.Lock() defer usbMutex.Unlock() // Null delimited string of project name, instance name and device name. key := fmt.Sprintf("%s\000%s\000%s", inst.Project().Name, inst.Name(), deviceName) delete(usbHandlers, key) } // USBRunHandlers executes any handlers registered for USB events. func USBRunHandlers(state *state.State, event *USBEvent) { usbMutex.Lock() defer usbMutex.Unlock() for key, hook := range usbHandlers { keyParts := strings.SplitN(key, "\000", 3) projectName := keyParts[0] instanceName := keyParts[1] deviceName := keyParts[2] if hook == nil { delete(usbHandlers, key) continue } runConf, err := hook(*event) if err != nil { logger.Error("USB event hook failed", logger.Ctx{"err": err, "project": projectName, "instance": instanceName, "device": deviceName}) continue } // If runConf supplied, load instance and call its USB event handler function so // any instance specific device actions can occur. if runConf != nil { instance, err := instance.LoadByProjectAndName(state, projectName, instanceName) if err != nil { logger.Error("USB event loading instance failed", logger.Ctx{"err": err, "project": projectName, "instance": instanceName, "device": deviceName}) continue } err = instance.DeviceEventHandler(runConf) if err != nil { logger.Error("USB event instance handler failed", logger.Ctx{"err": err, "project": projectName, "instance": instanceName, "device": deviceName}) continue } } } } // USBNewEvent instantiates a new USBEvent struct. func USBNewEvent(action string, vendor string, product string, serial string, major string, minor string, busnum string, devnum string, devname string, ueventParts []string, ueventLen int) (USBEvent, error) { majorInt, err := strconv.ParseUint(major, 10, 32) if err != nil { return USBEvent{}, err } minorInt, err := strconv.ParseUint(minor, 10, 32) if err != nil { return USBEvent{}, err } busnumInt, err := strconv.Atoi(busnum) if err != nil { return USBEvent{}, err } devnumInt, err := strconv.Atoi(devnum) if err != nil { return USBEvent{}, err } path := devname if devname == "" { path = fmt.Sprintf("/dev/bus/usb/%03d/%03d", busnumInt, devnumInt) } else { if !filepath.IsAbs(devname) { path = fmt.Sprintf("/dev/%s", devname) } } return USBEvent{ action, vendor, product, serial, path, uint32(majorInt), uint32(minorInt), ueventParts, ueventLen, busnumInt, devnumInt, }, nil } incus-7.0.0/internal/server/device/disk.go000066400000000000000000003223261517523235500205210ustar00rootroot00000000000000package device import ( "bufio" "context" "errors" "fmt" "io/fs" "net/http" "os" "os/exec" "path/filepath" "slices" "strconv" "strings" "sync" "golang.org/x/sys/unix" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/rsync" "github.com/lxc/incus/v7/internal/server/cgroup" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/warningtype" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/response" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/internal/server/warnings" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var diskISOGenerateMu sync.Mutex // Special disk "source" value used for generating a VM cloud-init config ISO. const diskSourceCloudInit = "cloud-init:config" // Special disk "source" value used for generating a VM agent ISO. const diskSourceAgent = "agent:config" // Special disk "source" identifier used for tmpfs mounts. const diskSourceTmpfs = "tmpfs:" // Special disk "source" identifier for tmpfs in overlayfs mounts. const diskSourceTmpfsOverlay = "tmpfs-overlay:" // DiskVirtiofsdSockMountOpt indicates the mount option prefix used to provide the virtiofsd socket path to // the QEMU driver. const DiskVirtiofsdSockMountOpt = "virtiofsdSock" // DiskFileDescriptorMountPrefix indicates the mount dev path is using a file descriptor rather than a normal path. // The Mount.DevPath field will be expected to be in the format: "fd::". // It still includes the original dev path so that the instance driver can perform additional probing of the path // to ascertain additional information if needed. However it will not be used to actually pass the path into the // instance. const DiskFileDescriptorMountPrefix = "fd" // DiskDirectIO is used to indicate disk should use direct I/O. const DiskDirectIO = "directio" // DiskIOUring is used to indicate disk should use io_uring if the system supports it. const DiskIOUring = "io_uring" // DiskLoopBacked is used to indicate disk is backed onto a loop device. const DiskLoopBacked = "loop" // IsSpecialDisk checks whether the provided source is a special disk. func IsSpecialDisk(source string) bool { return slices.Contains([]string{diskSourceCloudInit, diskSourceAgent, diskSourceTmpfs, diskSourceTmpfsOverlay}, source) } type diskBlockLimit struct { readBps int64 readIops int64 writeBps int64 writeIops int64 } // diskSourceNotFoundError error used to indicate source not found. type diskSourceNotFoundError struct { msg string err error } func (e diskSourceNotFoundError) Error() string { return fmt.Sprintf("%s: %v", e.msg, e.err) } func (e diskSourceNotFoundError) Unwrap() error { return e.err } type disk struct { deviceCommon restrictedParentSourcePath string pool storagePools.Pool // io.bus can contain imprecise information about the actual bus being used. bus string } // CanMigrate returns whether the device can be migrated to any other cluster member. func (d *disk) CanMigrate() bool { // Root disk is always migratable. if d.config["path"] == "/" { return true } // Remote disks are migratable. if d.pool != nil && d.pool.Driver().Info().Remote { return true } // Virtual disks are migratable. if IsSpecialDisk(d.config["source"]) { return true } return false } // sourceIsCephFs returns true if the disks source config setting is a CephFS share. func (d *disk) sourceIsCephFs() bool { return strings.HasPrefix(d.config["source"], "cephfs:") } // sourceIsCeph returns true if the disks source config setting is a Ceph RBD. func (d *disk) sourceIsCeph() bool { return strings.HasPrefix(d.config["source"], "ceph:") } // CanHotPlug returns whether the device can be managed whilst the instance is running. func (d *disk) CanHotPlug() bool { if d.config["source"] == diskSourceTmpfs || d.config["source"] == diskSourceTmpfsOverlay { return false } // 9p mounts cannot be hotplugged. However, with io.bus=auto, we can't know at startup time // if we are dealing with a 9p mount. Still, it's better to fail early. At stop time, we can // extract the info from the volatile key. All other disks can be hotplugged. if d.bus == "" || d.bus == "auto" { return d.volatileGet()["io.bus"] != "9p" } return d.bus != "9p" } // isRequired indicates whether the supplied device config requires this device to start OK. func (d *disk) isRequired(devConfig deviceConfig.Device) bool { // Defaults to required. if util.IsTrueOrEmpty(devConfig["required"]) && util.IsFalseOrEmpty(devConfig["optional"]) { return true } return false } // isAvailable checks whether the source is currently available. func (d *disk) isAvailable() bool { if d.config["pool"] == "" { // We only check managed volumes. return true } if d.config["path"] == "/" { // Root disks are always considered available. return true } // Load the pool if missing. var err error d.pool, err = storagePools.LoadByName(d.state, d.config["pool"]) if err != nil { // If we can't load the pool, the volume isn't available. return false } // We need instance data for further checks. if d.inst == nil { return true } // Check if the volume exists. storageProjectName, err := project.StorageVolumeProject(d.state.DB.Cluster, d.inst.Project().Name, db.StoragePoolVolumeTypeCustom) if err != nil { return false } // Parse the volume name and path. volName, _ := internalInstance.SplitVolumeSource(d.config["source"]) // GetStoragePoolVolume returns a volume with an empty Location field for remote drivers. err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, err = tx.GetStoragePoolVolume(ctx, d.pool.ID(), storageProjectName, db.StoragePoolVolumeTypeCustom, volName, true) return err }) if err != nil { return false } // We managed to get the entry from the database, consider it available. return true } // sourceIsLocalPath returns true if the source supplied should be considered a local path on the host. // It returns false if the disk source is empty, a VM cloud-init config drive, or a remote ceph/cephfs path. func (d *disk) sourceIsLocalPath(source string) bool { if source == "" { return false } if source == diskSourceCloudInit { return false } if source == diskSourceAgent { return false } if source == diskSourceTmpfs || source == diskSourceTmpfsOverlay { return false } if d.sourceIsCeph() || d.sourceIsCephFs() { return false } return true } // validateConfig checks the supplied config for correctness. func (d *disk) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container, instancetype.VM) { return ErrUnsupportedDevType } // Supported propagation types. // If an empty value is supplied the default behavior is to assume "private" mode. // These come from https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt propagationTypes := []string{"", "private", "shared", "slave", "unbindable", "rshared", "rslave", "runbindable", "rprivate"} validatePropagation := func(input string) error { if !slices.Contains(propagationTypes, d.config["bind"]) { return fmt.Errorf("Invalid propagation value. Must be one of: %s", strings.Join(propagationTypes, ", ")) } return nil } rules := map[string]func(string) error{ // gendoc:generate(entity=devices, group=disk, key=required) // // --- // type: bool // default: `true` // required: no // shortdesc: Controls whether to fail if the source doesn't exist "required": validate.Optional(validate.IsBool), "optional": validate.Optional(validate.IsBool), // "optional" is deprecated, replaced by "required". // gendoc:generate(entity=devices, group=disk, key=readonly) // // --- // type: bool // default: `false` // required: no // shortdesc: Controls whether to make the mount read-only "readonly": validate.Optional(validate.IsBool), // gendoc:generate(entity=devices, group=disk, key=recursive) // // --- // type: bool // default: `false` // required: no // shortdesc: Controls whether to recursively mount the source path "recursive": validate.Optional(validate.IsBool), // gendoc:generate(entity=devices, group=disk, key=shift) // // --- // type: bool // default: `false` // required: no // shortdesc: Sets up a shifting overlay to translate the source UID/GID to match the instance (only for containers) "shift": validate.Optional(validate.IsBool), // gendoc:generate(entity=devices, group=disk, key=source) // // --- // type: string // required: yes // shortdesc: Source of a file system or block device (see {ref}`devices-disk-types` for details) "source": validate.IsAny, // gendoc:generate(entity=devices, group=disk, key=limits.read) // // --- // type: string // required: no // shortdesc: I/O limit in byte/s (various suffixes supported, see {ref}`instances-limit-units`) or in IOPS (must be suffixed with `iops`) - see also {ref}`storage-configure-IO` "limits.read": validate.IsAny, // gendoc:generate(entity=devices, group=disk, key=limits.write) // // --- // type: string // required: no // shortdesc: I/O limit in byte/s (various suffixes supported, see {ref}`instances-limit-units`) or in IOPS (must be suffixed with `iops`) - see also {ref}`storage-configure-IO` "limits.write": validate.IsAny, // gendoc:generate(entity=devices, group=disk, key=limits.max) // // --- // type: string // required: no // shortdesc: I/O limit in byte/s or IOPS for both read and write (same as setting both `limits.read` and `limits.write`) "limits.max": validate.IsAny, // gendoc:generate(entity=devices, group=disk, key=size) // // --- // type: string // required: no // shortdesc: Disk size in bytes (various suffixes supported, see {ref}`instances-limit-units`) - only supported for the `rootfs` (`/`) "size": validate.Optional(validate.IsSize), // gendoc:generate(entity=devices, group=disk, key=size.state) // // --- // type: string // required: no // shortdesc: Same as `size`, but applies to the file-system volume used for saving runtime state in VMs "size.state": validate.Optional(validate.IsSize), // gendoc:generate(entity=devices, group=disk, key=pool) // // --- // type: string // required: no // shortdesc: The storage pool to which the disk device belongs (only applicable for storage volumes managed by Incus) "pool": validate.IsAny, // gendoc:generate(entity=devices, group=disk, key=propagation) // // --- // type: string // required: no // shortdesc: Controls how a bind-mount is shared between the instance and the host (can be one of `private`, the default, or `shared`, `slave`, `unbindable`, `rshared`, `rslave`, `runbindable`, `rprivate`; see the Linux Kernel [shared subtree](https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt) documentation for a full explanation) "propagation": validatePropagation, // gendoc:generate(entity=devices, group=disk, key=raw.mount.options) // // --- // type: string // required: no // shortdesc: File system specific mount options "raw.mount.options": validate.IsAny, // gendoc:generate(entity=devices, group=disk, key=ceph.cluster_name) // // --- // type: string // default: `ceph` // required: no // shortdesc: The cluster name of the Ceph cluster (required for Ceph or CephFS sources) "ceph.cluster_name": validate.IsAny, // gendoc:generate(entity=devices, group=disk, key=ceph.user_name) // // --- // type: string // default: `admin` // required: no // shortdesc: The user name of the Ceph cluster (required for Ceph or CephFS sources) "ceph.user_name": validate.IsAny, // gendoc:generate(entity=devices, group=disk, key=boot.priority) // // --- // type: integer // required: no // shortdesc: Boot priority for VMs (higher value boots first) "boot.priority": validate.Optional(validate.IsUint32), // gendoc:generate(entity=devices, group=disk, key=path) // This controls which path inside the instance the disk should be mounted on. // // With containers, this option supports mounting file system disk devices, and paths and single files within them. // // With VMs, this option supports mounting file system disk devices and paths within them. Mounting single files is not supported. // --- // type: string // required: yes // shortdesc: Path inside the instance where the disk will be mounted (only for file system disk devices) "path": validate.IsAny, // gendoc:generate(entity=devices, group=disk, key=io.cache) // This controls what bus a disk device should be attached to. // // For block devices (disks), this is one of: // - `none` (default) // - `writeback` // - `unsafe` // // For file systems (shared directories or custom volumes), this is one of: // - `none` (default) // - `metadata` // - `unsafe` // --- // type: string // default: `none` // required: no // shortdesc: Only for VMs: Override the caching mode for the device "io.cache": validate.Optional(validate.IsOneOf("none", "metadata", "writeback", "unsafe")), // gendoc:generate(entity=devices, group=disk, key=io.bus) // This controls what bus a disk device should be attached to. // // For block devices (disks), this is one of: // - `nvme` // - `virtio-blk` // - `virtio-scsi` (default) // - `usb` // // For file systems (shared directories or custom volumes), this is one of: // - `9p` // - `auto` (default) (`virtiofs` if possible, else `9p`) // - `virtiofs` // // `9p` doesn't support hotplugging and `virtiofs` doesn't support live migration. `auto` tries // to use `virtiofs` if possible (`migration.stateful` not set to `true` and host support for // `virtiofsd`) and falls back to `9p` otherwise. // --- // type: string // default: `virtio-scsi` for block, `auto` for file system // required: no // shortdesc: Only for VMs: Override the bus for the device "io.bus": validate.Optional(validate.IsOneOf("nvme", "virtio-blk", "virtio-scsi", "auto", "9p", "virtiofs", "usb")), // gendoc:generate(entity=devices, group=disk, key=attached) // // --- // type: bool // default: `true` // required: no // shortdesc: Only for VMs: Whether the disk is attached or ejected "attached": validate.Optional(validate.IsBool), // gendoc:generate(entity=devices, group=disk, key=dependent) // // --- // type: bool // default: `false` // required: no // shortdesc: Specifies if the disk is instance dependent "dependent": validate.Optional(validate.IsBool), // gendoc:generate(entity=devices, group=disk, key=wwn) // // --- // type: bool // default: `` // required: no // shortdesc: Only for VMs: Set the disk World Wide Name (only supported on `virtio-scsi` bus) "wwn": validate.Optional(validate.IsWWN), } err := d.config.Validate(rules) if err != nil { return err } if instConf.Type() == instancetype.Container && d.config["io.bus"] != "" { return errors.New("IO bus configuration cannot be applied to containers") } if instConf.Type() == instancetype.Container && d.config["io.cache"] != "" { return errors.New("IO cache configuration cannot be applied to containers") } if instConf.Type() == instancetype.Container && d.config["wwn"] != "" { return errors.New("WWN cannot be applied to containers") } if d.config["wwn"] != "" && !slices.Contains([]string{"", "virtio-scsi"}, d.config["io.bus"]) { return errors.New("WWN can only be set on virtio-scsi disks") } if d.config["required"] != "" && d.config["optional"] != "" { return errors.New(`Cannot use both "required" and deprecated "optional" properties at the same time`) } if d.config["source"] == "" && d.config["path"] != "/" { return errors.New(`Disk entry is missing the required "source" or "path" property`) } if d.config["path"] == "/" && d.config["source"] != "" { return errors.New(`Root disk entry may not have a "source" property set`) } if d.config["path"] == "/" && d.config["pool"] == "" { return errors.New(`Root disk entry must have a "pool" property set`) } if d.config["size"] != "" && d.config["path"] != "/" && d.config["source"] != diskSourceTmpfs && d.config["source"] != diskSourceTmpfsOverlay { return errors.New("Only root or tmpfs disks can have a size quota") } if d.config["size.state"] != "" && d.config["path"] != "/" { return errors.New("Only the root disk may have a migration size quota") } if d.config["recursive"] != "" && (d.config["path"] == "/" || !internalUtil.IsDir(d.config["source"])) { return errors.New("The recursive option is only supported for additional bind-mounted paths") } if util.IsTrue(d.config["recursive"]) && util.IsTrue(d.config["readonly"]) { return errors.New("Recursive read-only bind-mounts aren't currently supported by the kernel") } // Check ceph options are only used when ceph or cephfs type source is specified. if !(d.sourceIsCeph() || d.sourceIsCephFs()) && (d.config["ceph.cluster_name"] != "" || d.config["ceph.user_name"] != "") { return fmt.Errorf("Invalid options ceph.cluster_name/ceph.user_name for source %q", d.config["source"]) } if (d.config["source"] == diskSourceTmpfs || d.config["source"] == diskSourceTmpfsOverlay) && d.config["path"] == "" { return errors.New(`Missing mount "path" setting`) } // Check no other devices also have the same path as us. Use LocalDevices for this check so // that we can check before the config is expanded or when a profile is being checked. // Don't take into account the device names, only count active devices that point to the // same path, so that if merged profiles share the same the path and then one is removed // this can still be cleanly removed. pathCount := 0 for _, devConfig := range instConf.LocalDevices() { if devConfig["type"] == "disk" && d.config["path"] != "" && devConfig["path"] == d.config["path"] { pathCount++ if pathCount > 1 { return fmt.Errorf("More than one disk device uses the same path %q", d.config["path"]) } } } srcPathIsLocal := d.config["pool"] == "" && d.sourceIsLocalPath(d.config["source"]) srcPathIsAbs := filepath.IsAbs(d.config["source"]) if srcPathIsLocal && !srcPathIsAbs { return errors.New("Source path must be absolute for local sources") } // Check that external disk source path exists. External disk sources have a non-empty "source" property // that contains the path of the external source, and do not have a "pool" property. We only check the // source path exists when the disk device is required, is not an external ceph/cephfs source and is not a // VM cloud-init drive. We only check this when an instance is loaded to avoid validating snapshot configs // that may contain older config that no longer exists which can prevent migrations. if d.inst != nil && srcPathIsLocal && d.isRequired(d.config) && !util.PathExists(d.config["source"]) { return fmt.Errorf("Missing source path %q for disk %q", d.config["source"], d.name) } if d.config["pool"] != "" { if d.config["shift"] != "" { return errors.New(`The "shift" property cannot be used with custom storage volumes (set "security.shifted=true" on the volume instead)`) } if srcPathIsAbs { return errors.New("Storage volumes cannot be specified as absolute paths") } var dbVolume *db.StorageVolume var storageProjectName string if !partialValidation && d.inst != nil && !d.inst.IsSnapshot() && d.config["source"] != "" && d.config["path"] != "/" && (d.isRequired(d.config) || d.isAvailable()) { d.pool, err = storagePools.LoadByName(d.state, d.config["pool"]) if err != nil { return fmt.Errorf("Failed to get storage pool %q: %w", d.config["pool"], err) } // Derive the effective storage project name from the instance config's project. storageProjectName, err = project.StorageVolumeProject(d.state.DB.Cluster, instConf.Project().Name, db.StoragePoolVolumeTypeCustom) if err != nil { return err } // Parse the volume name and path. volName, _ := internalInstance.SplitVolumeSource(d.config["source"]) // GetStoragePoolVolume returns a volume with an empty Location field for remote drivers. err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, d.pool.ID(), storageProjectName, db.StoragePoolVolumeTypeCustom, volName, true) return err }) if err != nil { return fmt.Errorf("Failed loading custom volume: %w", err) } // Check that block volumes are *only* attached to VM instances. contentType, err := storagePools.VolumeContentTypeNameToContentType(dbVolume.ContentType) if err != nil { return err } // Check that the dependent disk is attached to exactly one instance. if util.IsTrue(dbVolume.Config["dependent"]) { count, err := d.getAttachedInstanceCount(storageProjectName, dbVolume) if err != nil { return err } if count > 0 { return errors.New("Cannot add dependent custom storage block volume to more than one instance") } } // Check that only shared custom storage block volume are added to profiles, or multiple instances. if util.IsFalseOrEmpty(dbVolume.Config["security.shared"]) && contentType == db.StoragePoolVolumeContentTypeBlock { if instConf.Type() == instancetype.Any { return errors.New("Cannot add un-shared custom storage block volume to profile") } count, err := d.getAttachedInstanceCount(storageProjectName, dbVolume) if err != nil { return err } if count > 0 { return errors.New("Cannot add un-shared custom storage block volume to more than one instance") } } } // Only perform expensive instance pool volume checks when not validating a profile and after // device expansion has occurred (to avoid doing it twice during instance load). if d.inst != nil && !d.inst.IsSnapshot() && len(instConf.ExpandedDevices()) > 0 { if d.pool == nil { d.pool, err = storagePools.LoadByName(d.state, d.config["pool"]) if err != nil { return fmt.Errorf("Failed to get storage pool %q: %w", d.config["pool"], err) } } if d.pool.Status() == "Pending" { return fmt.Errorf("Pool %q is pending", d.config["pool"]) } // Custom volume validation. if !partialValidation && d.config["source"] != "" && d.config["path"] != "/" && (d.isRequired(d.config) || d.isAvailable()) { if storageProjectName == "" { // Derive the effective storage project name from the instance config's project. storageProjectName, err = project.StorageVolumeProject(d.state.DB.Cluster, instConf.Project().Name, db.StoragePoolVolumeTypeCustom) if err != nil { return err } } // Parse the volume name and path. volName, volPath := internalInstance.SplitVolumeSource(d.config["source"]) if dbVolume == nil { // GetStoragePoolVolume returns a volume with an empty Location field for remote drivers. err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, d.pool.ID(), storageProjectName, db.StoragePoolVolumeTypeCustom, volName, true) return err }) if err != nil { return fmt.Errorf("Failed loading custom volume: %w", err) } } // Check storage volume is available to mount on this cluster member. remoteInstance, err := storagePools.VolumeUsedByExclusiveRemoteInstancesWithProfiles(d.state, d.config["pool"], storageProjectName, &dbVolume.StorageVolume) if err != nil { return fmt.Errorf("Failed checking if custom volume is exclusively attached to another instance: %w", err) } if dbVolume.ContentType != db.StoragePoolVolumeContentTypeNameISO && remoteInstance != nil && remoteInstance.ID != instConf.ID() { return errors.New("Custom volume is already attached to an instance on a different node") } // Check that block volumes are *only* attached to VM instances. contentType, err := storagePools.VolumeContentTypeNameToContentType(dbVolume.ContentType) if err != nil { return err } if d.config["attached"] != "" { if instConf.Type() == instancetype.Container { return errors.New("Attached configuration cannot be applied to containers") } else if instConf.Type() == instancetype.Any { return errors.New("Attached configuration cannot be applied to profiles") } else if contentType != db.StoragePoolVolumeContentTypeISO { return errors.New("Attached configuration can only be applied to ISO volumes") } } if contentType == db.StoragePoolVolumeContentTypeBlock { if instConf.Type() == instancetype.Container { return errors.New("Custom block volumes cannot be used on containers") } if d.config["path"] != "" { return errors.New("Custom block volumes cannot have a path defined") } if volPath != "" { return errors.New("Custom block volume snapshots cannot be used directly") } } else if contentType == db.StoragePoolVolumeContentTypeISO { if instConf.Type() == instancetype.Container { return errors.New("Custom ISO volumes cannot be used on containers") } if d.config["path"] != "" { return errors.New("Custom ISO volumes cannot have a path defined") } } else if d.config["path"] == "" { return errors.New("Custom filesystem volumes require a path to be defined") } if util.IsTrue(d.config["dependent"]) { err = storageDrivers.ValidateDependentConfigKey(dbVolume.Config) if err != nil { return err } } } // Extract initial configuration from the profile and validate them against appropriate // storage driver. Initial configuration is applicable to root disk devices and to non-root // custom volume disks (where initial.uid/gid/mode are used when auto-creating sub-directories). initialConfig := make(map[string]string) for k, v := range d.config { // gendoc:generate(entity=devices, group=disk, key=initial.*) // // For root disk devices, this is used to override the storage pool's default volume // configuration when creating the instance's root volume. // // For custom volumes, only `initial.uid`, `initial.gid` and `initial.mode` are // accepted and they are used when auto-creating sub-directories inside the custom // volume (when the `source` includes a sub-path that doesn't exist). // // `initial.uid`, `initial.gid` and `initial.mode` are also used to set the ownership // and mode of the file system when the `source` is `tmpfs:` or `tmpfs-overlay:`. // --- // type: string // required: no // shortdesc: Initial volume configuration for instance root disk devices prefix, newKey, found := strings.Cut(k, "initial.") if found && prefix == "" { initialConfig[newKey] = v } } if len(initialConfig) > 0 { if !internalInstance.IsRootDiskDevice(d.config) { // For non-root disks, only allow initial.uid/gid/mode (used for auto-creating // missing sub-directories on custom volumes). for k := range initialConfig { if k != "uid" && k != "gid" && k != "mode" { return fmt.Errorf("Non-root disk device only supports initial.uid, initial.gid and initial.mode configuration, not %q", "initial."+k) } } } else { volumeType, err := storagePools.InstanceTypeToVolumeType(d.inst.Type()) if err != nil { return err } // Create temporary volume definition. vol := storageDrivers.NewVolume( d.pool.Driver(), d.pool.Name(), volumeType, storagePools.InstanceContentType(d.inst), d.name, initialConfig, d.pool.Driver().Config()) err = d.pool.Driver().ValidateVolume(vol, true) if err != nil { return fmt.Errorf("Invalid initial device configuration: %v", err) } } } } } // Restrict disks allowed when live-migratable. if instConf.Type() == instancetype.VM && util.IsTrue(instConf.ExpandedConfig()["migration.stateful"]) { if d.config["pool"] == "" && !slices.Contains([]string{diskSourceCloudInit, diskSourceAgent}, d.config["source"]) { return errors.New("Only Incus-managed disks are allowed with migration.stateful=true") } if d.config["io.bus"] == "nvme" { return errors.New("NVME disks aren't supported with migration.stateful=true") } else if d.config["io.bus"] == "virtiofs" { return errors.New("Virtiofs mounts aren't supported with migration.stateful=true") } if d.config["path"] != "/" && d.pool != nil && !d.pool.Driver().Info().Remote && util.IsFalseOrEmpty(d.config["dependent"]) { return errors.New("Only additional disks coming from a shared storage pool are supported with migration.stateful=true") } } d.bus = d.config["io.bus"] return nil } // getDevicePath returns the absolute path on the host for this instance and supplied device config. func (d *disk) getDevicePath(devName string, devConfig deviceConfig.Device) string { relativeDestPath := strings.TrimPrefix(devConfig["path"], "/") devPath := linux.PathNameEncode(deviceJoinPath("disk", devName, relativeDestPath)) return filepath.Join(d.inst.DevicesPath(), devPath) } // validateEnvironmentSourcePath checks the source path property is valid and allowed by project. func (d *disk) validateEnvironmentSourcePath() error { srcPathIsLocal := d.config["pool"] == "" && d.sourceIsLocalPath(d.config["source"]) if !srcPathIsLocal { return nil } sourceHostPath := d.config["source"] // Check local external disk source path exists, but don't follow symlinks here (as we let openat2 do that // safely later). _, err := os.Lstat(sourceHostPath) if err != nil { if errors.Is(err, fs.ErrNotExist) { return diskSourceNotFoundError{msg: fmt.Sprintf("Missing source path %q", d.config["source"])} } return fmt.Errorf("Failed accessing source path %q for disk %q: %w", sourceHostPath, d.name, err) } // If project not default then check if using restricted disk paths. // Default project cannot be restricted, so don't bother loading the project config in that case. instProject := d.inst.Project() if instProject.Name != api.ProjectDefaultName { // If restricted disk paths are in force, then check the disk's source is allowed, and record the // allowed parent path for later user during device start up sequence. if util.IsTrue(instProject.Config["restricted"]) && instProject.Config["restricted.devices.disk.paths"] != "" { allowed, restrictedParentSourcePath := project.CheckRestrictedDevicesDiskPaths(instProject.Config, d.config["source"]) if !allowed { return fmt.Errorf("Disk source path %q not allowed by project for disk %q", d.config["source"], d.name) } if util.IsTrue(d.config["shift"]) { return errors.New(`The "shift" property cannot be used with a restricted source path`) } d.restrictedParentSourcePath = restrictedParentSourcePath } } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *disk) validateEnvironment() error { if d.inst.Type() != instancetype.VM && slices.Contains([]string{diskSourceCloudInit, diskSourceAgent}, d.config["source"]) { return fmt.Errorf("disks with source=%s are only supported by virtual machines", d.config["source"]) } if d.inst.Type() != instancetype.Container && slices.Contains([]string{diskSourceTmpfs, diskSourceTmpfsOverlay}, d.config["source"]) { return fmt.Errorf("disks with source=%s are only supported by containers", d.config["source"]) } err := d.validateEnvironmentSourcePath() if err != nil { return err } return nil } // UpdatableFields returns a list of fields that can be updated without triggering a device remove & add. func (d *disk) UpdatableFields(oldDevice Type) []string { // Check old and new device types match. _, match := oldDevice.(*disk) if !match { return []string{} } return []string{"limits.max", "limits.read", "limits.write", "size", "size.state", "dependent"} } // Register calls mount for the disk volume (which should already be mounted) to reinitialize the reference counter // for volumes attached to running instances on daemon restart. func (d *disk) Register() error { d.logger.Debug("Initialising mounted disk ref counter") if d.config["path"] == "/" { // Load the pool. pool, err := storagePools.LoadByInstance(d.state, d.inst) if err != nil { return err } // Try to mount the volume that should already be mounted to reinitialize the ref counter. _, err = pool.MountInstance(d.inst, nil) if err != nil { return err } } else if d.config["path"] != "/" && d.config["source"] != "" && d.config["pool"] != "" { storageProjectName, err := project.StorageVolumeProject(d.state.DB.Cluster, d.inst.Project().Name, db.StoragePoolVolumeTypeCustom) if err != nil { return err } // Load the pool. pool, err := storagePools.LoadByName(d.state, d.config["pool"]) if err != nil { return fmt.Errorf("Failed to get storage pool %q: %w", d.config["pool"], err) } // Parse the volume name and path. volName, _ := internalInstance.SplitVolumeSource(d.config["source"]) // Try to mount the volume that should already be mounted to reinitialize the ref counter. _, err = pool.MountCustomVolume(storageProjectName, volName, nil) if err != nil { return err } } return nil } // Add performs any setup when a device is added to an instance. // It is called irrespective of whether the instance is running or not. func (d *disk) Add() error { if d.config["pool"] != "" && d.config["source"] != "" { // During migration, the storage volume may not be present at this moment, // so do not raise an error. _, err := d.updateDependentConfig() if err != nil && !response.IsNotFoundError(err) { return err } } return nil } // PreStartCheck checks the storage pool is available (if relevant). func (d *disk) PreStartCheck() error { // volatile..io.bus needs to be reset so that we aren't reading the previous one. err := d.volatileSet(map[string]string{"io.bus": ""}) if err != nil { return err } // Non-pool disks are not relevant for checking pool availability. if d.pool == nil { return nil } // Custom volume disks that are not required don't need to be checked as if the pool is // not available we should still start the instance. if d.config["path"] != "/" && !d.isRequired(d.config) { return nil } // If disk is required and storage pool is not available, don't try and start instance. if d.pool.LocalStatus() == api.StoragePoolStatusUnvailable { return api.StatusErrorf(http.StatusServiceUnavailable, "Storage pool %q unavailable on this server", d.pool.Name()) } return nil } // Start is run when the device is added to the instance. func (d *disk) Start() (*deviceConfig.RunConfig, error) { // Ignore detached disks. if !util.IsTrueOrEmpty(d.config["attached"]) { return nil, nil } // Ignore missing disks. if !d.isRequired(d.config) && !d.isAvailable() { return nil, nil } var runConfig *deviceConfig.RunConfig err := d.validateEnvironment() if err == nil { if d.inst.Type() == instancetype.VM { runConfig, err = d.startVM() } else { runConfig, err = d.startContainer() } } if err != nil { var sourceNotFound diskSourceNotFoundError if errors.As(err, &sourceNotFound) && !d.isRequired(d.config) { d.logger.Warn(sourceNotFound.msg) return nil, nil } return nil, err } return runConfig, nil } // startContainer starts the disk device for a container instance. func (d *disk) startContainer() (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{} isReadOnly := util.IsTrue(d.config["readonly"]) // Apply cgroups only after all the mounts have been processed. runConf.PostHooks = append(runConf.PostHooks, func() error { runConf := deviceConfig.RunConfig{} err := d.generateLimits(&runConf) if err != nil { return err } err = d.inst.DeviceEventHandler(&runConf) if err != nil { return err } return nil }) reverter := revert.New() defer reverter.Fail() // Deal with a rootfs. if internalInstance.IsRootDiskDevice(d.config) { // Set the rootfs path. rootfs := deviceConfig.RootFSEntryItem{ Path: d.inst.RootfsPath(), } // Read-only rootfs (unlikely to work very well). if isReadOnly { rootfs.Opts = append(rootfs.Opts, "ro") } // Handle previous requests for setting new quotas. err := d.applyDeferredQuota() if err != nil { return nil, err } runConf.RootFS = rootfs } else if d.config["source"] == diskSourceTmpfs || d.config["source"] == diskSourceTmpfsOverlay { srcPath := d.config["source"] destPath := d.config["path"] relativeDestPath := strings.TrimPrefix(destPath, "/") options := []string{} if d.config["size"] != "" { size, err := units.ParseByteSizeString(d.config["size"]) if err != nil { return nil, err } options = append(options, fmt.Sprintf("size=%d", size)) } if d.config["initial.mode"] != "" { options = append(options, fmt.Sprintf("mode=%s", d.config["initial.mode"])) } if d.config["initial.uid"] != "" { options = append(options, fmt.Sprintf("uid=%s", d.config["initial.uid"])) } if d.config["initial.gid"] != "" { options = append(options, fmt.Sprintf("gid=%s", d.config["initial.gid"])) } if srcPath == diskSourceTmpfsOverlay { procUpperPath := "proc/upper" procWorkPath := "proc/work" overlayPath := relativeDestPath rootFsPaths := []string{ "/opt/incus/lib/lxc/rootfs/", "/usr/lib/x86_64-linux-gnu/lxc/rootfs/", "/usr/lib/aarch64-linux-gnu/lxc/rootfs/", } rootFsPath := "" for _, path := range rootFsPaths { if util.PathExists(path) { rootFsPath = path break } } if rootFsPath == "" { return nil, fmt.Errorf("Cannot find rootfs path for container") } lowerDirOpt := fmt.Sprintf("lowerdir=%s", filepath.Join(rootFsPath, overlayPath)) upperDirOpt := fmt.Sprintf("upperdir=%s", filepath.Join(rootFsPath, procUpperPath)) workDirOpt := fmt.Sprintf("workdir=%s", filepath.Join(rootFsPath, procWorkPath)) mounts := []deviceConfig.MountEntryItem{ { DevName: d.name, DevPath: "none", TargetPath: "proc", FSType: "tmpfs", Opts: append(options, "defaults"), }, { DevName: d.name, DevPath: "none", TargetPath: procUpperPath, FSType: "invalid", Opts: []string{"defaults", "create=dir", "optional"}, }, { DevName: d.name, DevPath: "none", TargetPath: procWorkPath, FSType: "invalid", Opts: []string{"defaults", "create=dir", "optional"}, }, { DevName: d.name, DevPath: "none", TargetPath: "sys", FSType: "overlay", Opts: []string{"userxattr", lowerDirOpt, upperDirOpt, workDirOpt}, }, { DevName: d.name, DevPath: filepath.Join(rootFsPath, "proc"), TargetPath: overlayPath, FSType: "none", Opts: []string{"move"}, }, { DevName: d.name, DevPath: filepath.Join(rootFsPath, "sys"), TargetPath: overlayPath, FSType: "none", Opts: []string{"move"}, }, } runConf.Mounts = append(runConf.Mounts, mounts...) } else { options := append(options, "create=dir") runConf.Mounts = append(runConf.Mounts, deviceConfig.MountEntryItem{ DevName: d.name, DevPath: "tmpfs", TargetPath: relativeDestPath, FSType: "tmpfs", Opts: options, }) } } else { // Source path. srcPath := d.config["source"] // Destination path. destPath := d.config["path"] relativeDestPath := strings.TrimPrefix(destPath, "/") // Option checks. isRecursive := util.IsTrue(d.config["recursive"]) ownerShift := deviceConfig.MountOwnerShiftNone if util.IsTrue(d.config["shift"]) { ownerShift = deviceConfig.MountOwnerShiftDynamic } // If ownerShift is none and pool is specified then check whether the pool itself // has owner shifting enabled, and if so enable shifting on this device too. if ownerShift == deviceConfig.MountOwnerShiftNone && d.config["pool"] != "" { // Only custom volumes can be attached currently. storageProjectName, err := project.StorageVolumeProject(d.state.DB.Cluster, d.inst.Project().Name, db.StoragePoolVolumeTypeCustom) if err != nil { return nil, err } // Parse the volume name and path. volName, _ := internalInstance.SplitVolumeSource(d.config["source"]) var dbVolume *db.StorageVolume err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, d.pool.ID(), storageProjectName, db.StoragePoolVolumeTypeCustom, volName, true) return err }) if err != nil { return nil, err } if util.IsTrue(dbVolume.Config["security.shifted"]) { ownerShift = "dynamic" } } options := []string{} if isReadOnly { options = append(options, "ro") } if isRecursive { options = append(options, "rbind") } else { options = append(options, "bind") } if d.config["propagation"] != "" { options = append(options, d.config["propagation"]) } // Mount the pool volume and set poolVolSrcPath for createDevice below. if d.config["pool"] != "" { var err error var revertFunc func() var mountInfo *storagePools.MountInfo revertFunc, srcPath, mountInfo, err = d.mountPoolVolume() if err != nil { return nil, diskSourceNotFoundError{msg: "Failed mounting volume", err: err} } reverter.Add(revertFunc) // Handle post hooks. runConf.PostHooks = append(runConf.PostHooks, func() error { for _, hook := range mountInfo.PostHooks { err := hook(d.inst) if err != nil { return err } } return nil }) } // Mount the source in the instance devices directory. revertFunc, sourceDevPath, isFile, err := d.createDevice(srcPath) if err != nil { return nil, err } reverter.Add(revertFunc) if isFile { options = append(options, "create=file") } else { options = append(options, "create=dir") } // Ask for the mount to be performed. runConf.Mounts = append(runConf.Mounts, deviceConfig.MountEntryItem{ DevName: d.name, DevPath: sourceDevPath, TargetPath: relativeDestPath, FSType: "none", Opts: options, OwnerShift: ownerShift, }) // Unmount host-side mount once instance is started. runConf.PostHooks = append(runConf.PostHooks, d.postStart) } reverter.Success() return &runConf, nil } // vmVirtiofsdPaths returns the path for the socket and PID file to use with virtiofsd process. func (d *disk) vmVirtiofsdPaths() (string, string) { sockPath := filepath.Join(d.inst.DevicesPath(), fmt.Sprintf("virtio-fs.%s.sock", d.name)) pidPath := filepath.Join(d.inst.DevicesPath(), fmt.Sprintf("virtio-fs.%s.pid", d.name)) return sockPath, pidPath } func (d *disk) detectVMPoolMountOpts() []string { var opts []string driverConf := d.pool.Driver().Config() // If the pool's source is a normal file, rather than a block device or directory, then we consider it to // be a loop backed stored pool. fileInfo, _ := os.Stat(driverConf["source"]) if fileInfo != nil && !linux.IsBlockdev(fileInfo.Mode()) && !fileInfo.IsDir() { opts = append(opts, DiskLoopBacked) } if d.pool.Driver().Info().DirectIO { opts = append(opts, DiskDirectIO) } if d.pool.Driver().Info().IOUring { opts = append(opts, DiskIOUring) } return opts } // setBus adds bus overrides to mount options and sets the io.bus volatile key. func (d *disk) setBus(entry *deviceConfig.MountEntryItem) error { if d.bus == "" { return nil } entry.Opts = append(entry.Opts, "bus="+d.bus) return d.volatileSet(map[string]string{"io.bus": d.bus}) } // startVM starts the disk device for a virtual machine instance. func (d *disk) startVM() (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{} reverter := revert.New() defer reverter.Fail() // Handle user overrides. opts := []string{} // Allow the user to override the caching mode. if d.config["io.cache"] != "" { opts = append(opts, fmt.Sprintf("cache=%s", d.config["io.cache"])) } // Apply the WWN if provided. if d.config["wwn"] != "" { opts = append(opts, fmt.Sprintf("wwn=%s", d.config["wwn"])) } // Add I/O limits if set. var diskLimits *deviceConfig.DiskLimits if d.config["limits.read"] != "" || d.config["limits.write"] != "" || d.config["limits.max"] != "" { // Parse the limits into usable values. readBps, readIops, writeBps, writeIops, err := d.parseLimit(d.config) if err != nil { return nil, err } diskLimits = &deviceConfig.DiskLimits{ ReadBytes: readBps, ReadIOps: readIops, WriteBytes: writeBps, WriteIOps: writeIops, } } if internalInstance.IsRootDiskDevice(d.config) { // Handle previous requests for setting new quotas. err := d.applyDeferredQuota() if err != nil { return nil, err } opts = append(opts, d.detectVMPoolMountOpts()...) mount := deviceConfig.MountEntryItem{ TargetPath: d.config["path"], // Indicator used that this is the root device. DevName: d.name, Opts: opts, Limits: diskLimits, } err = d.setBus(&mount) if err != nil { return nil, err } runConf.Mounts = []deviceConfig.MountEntryItem{mount} return &runConf, nil } else if d.config["source"] == diskSourceAgent { // This is a special virtual disk source that can be attached to a VM to provide agent binary and config. isoPath, err := d.generateVMAgentDrive() if err != nil { return nil, err } // Open file handle to isoPath source. f, err := os.OpenFile(isoPath, unix.O_PATH|unix.O_CLOEXEC, 0) if err != nil { return nil, fmt.Errorf("Failed opening source path %q: %w", isoPath, err) } reverter.Add(func() { _ = f.Close() }) runConf.PostHooks = append(runConf.PostHooks, f.Close) runConf.Revert = func() { _ = f.Close() } // Close file on VM start failure. // Encode the file descriptor and original isoPath into the DevPath field. mount := deviceConfig.MountEntryItem{ DevPath: fmt.Sprintf("%s:%d:%s", DiskFileDescriptorMountPrefix, f.Fd(), isoPath), DevName: d.name, FSType: "iso9660", Opts: opts, } err = d.setBus(&mount) if err != nil { return nil, err } runConf.Mounts = []deviceConfig.MountEntryItem{mount} reverter.Success() return &runConf, nil } else if d.config["source"] == diskSourceCloudInit { // This is a special virtual disk source that can be attached to a VM to provide cloud-init config. isoPath, err := d.generateVMConfigDrive() if err != nil { return nil, err } // Open file handle to isoPath source. f, err := os.OpenFile(isoPath, unix.O_PATH|unix.O_CLOEXEC, 0) if err != nil { return nil, fmt.Errorf("Failed opening source path %q: %w", isoPath, err) } reverter.Add(func() { _ = f.Close() }) runConf.PostHooks = append(runConf.PostHooks, f.Close) runConf.Revert = func() { _ = f.Close() } // Close file on VM start failure. // Encode the file descriptor and original isoPath into the DevPath field. mount := deviceConfig.MountEntryItem{ DevPath: fmt.Sprintf("%s:%d:%s", DiskFileDescriptorMountPrefix, f.Fd(), isoPath), DevName: d.name, FSType: "iso9660", Opts: opts, } err = d.setBus(&mount) if err != nil { return nil, err } runConf.Mounts = []deviceConfig.MountEntryItem{mount} reverter.Success() return &runConf, nil } else if d.config["source"] != "" { if d.sourceIsCeph() { // Get the pool and volume names. fields := strings.SplitN(d.config["source"], ":", 2) fields = strings.SplitN(fields[1], "/", 2) clusterName, userName := d.cephCreds() mount := deviceConfig.MountEntryItem{ DevPath: DiskGetRBDFormat(clusterName, userName, fields[0], fields[1]), DevName: d.name, Opts: opts, Limits: diskLimits, } err := d.setBus(&mount) if err != nil { return nil, err } runConf.Mounts = []deviceConfig.MountEntryItem{mount} } else { // Default to block device or image file passthrough first. mount := deviceConfig.MountEntryItem{ DevPath: d.config["source"], DevName: d.name, Opts: opts, Limits: diskLimits, } err := d.setBus(&mount) if err != nil { return nil, err } // Mount the pool volume and update srcPath to mount path so it can be recognised as dir // if the volume is a filesystem volume type (if it is a block volume the srcPath will // be returned as the path to the block device). if d.config["pool"] != "" { var revertFunc func() var mountInfo *storagePools.MountInfo // Derive the effective storage project name from the instance config's project. storageProjectName, err := project.StorageVolumeProject(d.state.DB.Cluster, d.inst.Project().Name, db.StoragePoolVolumeTypeCustom) if err != nil { return nil, err } // Parse the volume name and path. volName, _ := internalInstance.SplitVolumeSource(d.config["source"]) // GetStoragePoolVolume returns a volume with an empty Location field for remote drivers. var dbVolume *db.StorageVolume err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, d.pool.ID(), storageProjectName, db.StoragePoolVolumeTypeCustom, volName, true) return err }) if err != nil { return nil, fmt.Errorf("Failed loading custom volume: %w", err) } contentType, err := storagePools.VolumeContentTypeNameToContentType(dbVolume.ContentType) if err != nil { return nil, err } if contentType == db.StoragePoolVolumeContentTypeISO { mount.FSType = "iso9660" } // If the pool is ceph backed and a block device, don't mount it, instead pass config to QEMU instance // to use the built in RBD support. if d.pool.Driver().Info().Name == "ceph" && (contentType == db.StoragePoolVolumeContentTypeBlock || contentType == db.StoragePoolVolumeContentTypeISO) { config := d.pool.ToAPI().Config poolName := config["ceph.osd.pool_name"] userName := config["ceph.user.name"] if userName == "" { userName = storageDrivers.CephDefaultUser } clusterName := config["ceph.cluster_name"] if clusterName == "" { clusterName = storageDrivers.CephDefaultUser } mount := deviceConfig.MountEntryItem{ DevPath: DiskGetRBDFormat(clusterName, userName, poolName, d.config["source"]), DevName: d.name, Opts: opts, Limits: diskLimits, } err = d.setBus(&mount) if err != nil { return nil, err } if contentType == db.StoragePoolVolumeContentTypeISO { mount.FSType = "iso9660" } runConf.Mounts = []deviceConfig.MountEntryItem{mount} return &runConf, nil } revertFunc, mount.DevPath, mountInfo, err = d.mountPoolVolume() if err != nil { return nil, diskSourceNotFoundError{msg: "Failed mounting volume", err: err} } reverter.Add(revertFunc) mount.Opts = append(mount.Opts, d.detectVMPoolMountOpts()...) mount.BackingPath = append(mount.BackingPath, mountInfo.BackingPath...) } if util.IsTrue(d.config["readonly"]) { mount.Opts = append(mount.Opts, "ro") } // If the source being added is a directory or cephfs share, then we will use the agent // directory sharing feature to mount the directory inside the VM, and as such we need to // indicate to the VM the target path to mount to. if internalUtil.IsDir(mount.DevPath) || d.sourceIsCephFs() { // Confirm we're using filesystem options. err := validate.Optional(validate.IsOneOf("auto", "9p", "virtiofs"))(d.bus) if err != nil { return nil, err } err = validate.Optional(validate.IsOneOf("none", "metadata", "unsafe"))(d.config["io.cache"]) if err != nil { return nil, err } if d.config["path"] == "" { return nil, errors.New(`Missing mount "path" setting`) } // Mount the source in the instance devices directory. // This will ensure that if the exported directory configured as readonly that this // takes effect event if using virtio-fs (which doesn't support read only mode) by // having the underlying mount setup as readonly. var revertFunc func() revertFunc, mount.DevPath, _, err = d.createDevice(mount.DevPath) if err != nil { return nil, err } reverter.Add(revertFunc) mount.TargetPath = d.config["path"] mount.FSType = "9p" rawIDMaps, err := idmap.NewSetFromIncusIDMap(d.inst.ExpandedConfig()["raw.idmap"]) if err != nil { return nil, fmt.Errorf(`Failed parsing instance "raw.idmap": %w`, err) } if d.bus == "" || d.bus == "auto" { if d.inst.CanLiveMigrate() { d.bus = "9p" } else { d.bus = "auto" } } // Start virtiofsd for virtio-fs share. If for some reason we can't, create a 9p share as // a fallback. err = func() error { // Check if we should start virtiofsd. if d.bus != "auto" && d.bus != "virtiofs" { return nil } sockPath, pidPath := d.vmVirtiofsdPaths() logPath := filepath.Join(d.inst.LogPath(), fmt.Sprintf("disk.%s.log", d.name)) _ = os.Remove(logPath) // Remove old log if needed. revertFunc, unixListener, err := DiskVMVirtiofsdStart(d.state.OS.ExecPath, d.inst, sockPath, pidPath, logPath, mount.DevPath, rawIDMaps.Entries, d.config["io.cache"]) if err != nil { var errUnsupported UnsupportedError if d.bus != "virtiofs" && errors.As(err, &errUnsupported) { d.logger.Warn("Unable to use virtio-fs for device, using 9p as a fallback", logger.Ctx{"err": errUnsupported}) // Fallback to 9p-only. d.bus = "9p" if errors.Is(errUnsupported, ErrMissingVirtiofsd) { _ = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpsertWarningLocalNode(ctx, d.inst.Project().Name, cluster.TypeInstance, d.inst.ID(), warningtype.MissingVirtiofsd, "Using 9p as a fallback") }) } else { // Resolve previous warning. _ = warnings.ResolveWarningsByLocalNodeAndProjectAndType(d.state.DB.Cluster, d.inst.Project().Name, warningtype.MissingVirtiofsd) } return nil } return err } reverter.Add(revertFunc) d.bus = "virtiofs" // Request the unix listener is closed after QEMU has connected on startup. runConf.PostHooks = append(runConf.PostHooks, unixListener.Close) // Resolve previous warning _ = warnings.ResolveWarningsByLocalNodeAndProjectAndType(d.state.DB.Cluster, d.inst.Project().Name, warningtype.MissingVirtiofsd) // Add the socket path to the mount options to indicate to the qemu driver // that this share is available. // Note: the sockPath is not passed to the QEMU via mount.DevPath like the // 9p share above. This is because we run the 9p share concurrently // and can only pass one DevPath at a time. Instead pass the sock path to // the QEMU driver via the mount opts field as virtiofsdSock to allow the // QEMU driver also setup the virtio-fs share. mount.Opts = append(mount.Opts, fmt.Sprintf("%s=%s", DiskVirtiofsdSockMountOpt, sockPath)) return nil }() if err != nil { return nil, fmt.Errorf("Failed to setup virtiofsd for device %q: %w", d.name, err) } // Once we're here, we know which bus to use. Because 9p mounts cannot be hotplugged, we // need to check if the instance is running. if d.bus == "9p" && d.inst.IsRunning() { if d.config["io.bus"] == "9p" { return nil, errors.New("9p doesn't support hotplugging") } return nil, errors.New("Virtiofsd cannot be used for this share, and 9p doesn't support hotplugging") } err = d.setBus(&mount) if err != nil { return nil, err } // 9p doesn't support idmap if d.bus == "9p" && len(rawIDMaps.Entries) > 0 { return nil, errors.New("9p shares do not support identity mapping") } } else { // Forbid mounting files to FS paths. if d.config["path"] != "" { return nil, errors.New(`The "path" setting is not supported on VMs for non-directory sources`) } // Confirm we're dealing with block options. err := validate.Optional(validate.IsOneOf("nvme", "virtio-blk", "virtio-scsi", "usb"))(d.bus) if err != nil { return nil, err } err = validate.Optional(validate.IsOneOf("none", "writeback", "unsafe"))(d.config["io.cache"]) if err != nil { return nil, err } f, err := d.localSourceOpen(mount.DevPath) if err != nil { return nil, err } reverter.Add(func() { _ = f.Close() }) runConf.PostHooks = append(runConf.PostHooks, f.Close) runConf.Revert = func() { _ = f.Close() } // Close file on VM start failure. // Detect ISO files to set correct FSType before DevPath is encoded below. // This is very important to support Windows ISO images (amongst other). if strings.HasSuffix(mount.DevPath, ".iso") { mount.FSType = "iso9660" } // Encode the file descriptor and original srcPath into the DevPath field. mount.DevPath = fmt.Sprintf("%s:%d:%s", DiskFileDescriptorMountPrefix, f.Fd(), mount.DevPath) } // Add successfully setup mount config to runConf. runConf.Mounts = []deviceConfig.MountEntryItem{mount} } reverter.Success() return &runConf, nil } return nil, errors.New("Disk type not supported for VMs") } // postStart is run after the instance is started. func (d *disk) postStart() error { devPath := d.getDevicePath(d.name, d.config) // Unmount the host side. err := unix.Unmount(devPath, unix.MNT_DETACH) if err != nil { return err } return nil } // Update applies configuration changes to a started device. func (d *disk) Update(oldDevices deviceConfig.Devices, isRunning bool) error { expandedDevices := d.inst.ExpandedDevices() reverter := revert.New() defer reverter.Fail() if internalInstance.IsRootDiskDevice(d.config) { // Make sure we have a valid root disk device (and only one). newRootDiskDeviceKey, _, err := internalInstance.GetRootDiskDevice(expandedDevices.CloneNative()) if err != nil { return fmt.Errorf("Detect root disk device: %w", err) } // Retrieve the first old root disk device key, even if there are duplicates. oldRootDiskDeviceKey := "" for k, v := range oldDevices { if internalInstance.IsRootDiskDevice(v) { oldRootDiskDeviceKey = k break } } // Check for pool change. oldRootDiskDevicePool := oldDevices[oldRootDiskDeviceKey]["pool"] newRootDiskDevicePool := expandedDevices[newRootDiskDeviceKey]["pool"] if oldRootDiskDevicePool != newRootDiskDevicePool { return errors.New("The storage pool of the root disk can only be changed through move") } // Deal with quota changes. oldRootDiskDeviceSize := oldDevices[oldRootDiskDeviceKey]["size"] newRootDiskDeviceSize := expandedDevices[newRootDiskDeviceKey]["size"] oldRootDiskDeviceMigrationSize := oldDevices[oldRootDiskDeviceKey]["size.state"] newRootDiskDeviceMigrationSize := expandedDevices[newRootDiskDeviceKey]["size.state"] // Apply disk quota changes. if newRootDiskDeviceSize != oldRootDiskDeviceSize || oldRootDiskDeviceMigrationSize != newRootDiskDeviceMigrationSize { // Remove any outstanding volatile apply_quota key if applying a new quota. v := d.volatileGet() if v["apply_quota"] != "" { err = d.volatileSet(map[string]string{"apply_quota": ""}) if err != nil { return err } } err := d.applyQuota(false) if errors.Is(err, storageDrivers.ErrInUse) { // Save volatile apply_quota key for next boot if cannot apply now. err = d.volatileSet(map[string]string{"apply_quota": "true"}) if err != nil { return err } d.logger.Warn("Could not apply quota because disk is in use, deferring until next start") } else if err != nil { return err } else if d.inst.Type() == instancetype.VM && d.inst.IsRunning() { // Get the disk size in bytes. size, err := units.ParseByteSizeString(newRootDiskDeviceSize) if err != nil { return err } // Notify to reload disk size. runConf := deviceConfig.RunConfig{} runConf.Mounts = []deviceConfig.MountEntryItem{ { DevName: d.name, Size: size, }, } err = d.inst.DeviceEventHandler(&runConf) if err != nil { return err } } } } else if d.config["pool"] != "" && d.config["source"] != "" { cleanup, err := d.updateDependentConfig() if err != nil { return err } reverter.Add(func() { _ = cleanup() }) } // Only apply IO limits and attach/detach logic if instance is running. if isRunning { runConf := deviceConfig.RunConfig{} if d.inst.Type() == instancetype.Container { err := d.generateLimits(&runConf) if err != nil { return err } } if d.inst.Type() == instancetype.VM { var diskLimits *deviceConfig.DiskLimits runConf.Mounts = []deviceConfig.MountEntryItem{} if d.config["limits.read"] != "" || d.config["limits.write"] != "" || d.config["limits.max"] != "" { // Parse the limits into usable values. readBps, readIops, writeBps, writeIops, err := d.parseLimit(d.config) if err != nil { return err } // Apply the limits to a minimal mount entry. diskLimits = &deviceConfig.DiskLimits{ ReadBytes: readBps, ReadIOps: readIops, WriteBytes: writeBps, WriteIOps: writeIops, } runConf.Mounts = append(runConf.Mounts, deviceConfig.MountEntryItem{ DevName: d.name, Limits: diskLimits, }) } } err := d.inst.DeviceEventHandler(&runConf) if err != nil { return err } } reverter.Success() return nil } // applyDeferredQuota attempts to apply the deferred quota specified in the volatile "apply_quota" key if set. // If successfully applies new quota then removes the volatile "apply_quota" key. func (d *disk) applyDeferredQuota() error { v := d.volatileGet() if v["apply_quota"] != "" { d.logger.Info("Applying deferred quota change") // Indicate that we want applyQuota to unmount the volume first, this is so we can perform resizes // that cannot be done when the volume is in use. err := d.applyQuota(true) if err != nil { return fmt.Errorf("Failed to apply deferred quota from %q: %w", fmt.Sprintf("volatile.%s.apply_quota", d.name), err) } // Remove volatile apply_quota key if successful. err = d.volatileSet(map[string]string{"apply_quota": ""}) if err != nil { return err } } return nil } // applyQuota attempts to resize the instance root disk to the specified size. // If remount is true, attempts to unmount first before resizing and then mounts again afterwards. func (d *disk) applyQuota(remount bool) error { rootDisk, _, err := internalInstance.GetRootDiskDevice(d.inst.ExpandedDevices().CloneNative()) if err != nil { return fmt.Errorf("Detect root disk device: %w", err) } newSize := d.inst.ExpandedDevices()[rootDisk]["size"] newMigrationSize := d.inst.ExpandedDevices()[rootDisk]["size.state"] pool, err := storagePools.LoadByInstance(d.state, d.inst) if err != nil { return err } if remount { err := pool.UnmountInstance(d.inst, nil) if err != nil { return err } } quotaErr := pool.SetInstanceQuota(d.inst, newSize, newMigrationSize, nil) if remount { _, err = pool.MountInstance(d.inst, nil) } // Return quota set error if failed. if quotaErr != nil { return quotaErr } // Return remount error if mount failed. if err != nil { return err } return nil } // generateLimits adds a set of cgroup rules to apply specified limits to the supplied RunConfig. func (d *disk) generateLimits(runConf *deviceConfig.RunConfig) error { // Disk throttle limits. hasDiskLimits := false for _, dev := range d.inst.ExpandedDevices() { if dev["type"] != "disk" { continue } if dev["limits.read"] != "" || dev["limits.write"] != "" || dev["limits.max"] != "" { hasDiskLimits = true } } if hasDiskLimits { if !cgroup.Supports(cgroup.IO) { return errors.New("Cannot apply disk limits as IO cgroup controller is missing") } diskLimits, err := d.getDiskLimits() if err != nil { return err } cg, err := cgroup.New(&cgroupWriter{runConf}) if err != nil { return err } for block, limit := range diskLimits { if limit.readBps > 0 { err = cg.SetBlkioLimit(block, "read", "bps", limit.readBps) if err != nil { return err } } if limit.readIops > 0 { err = cg.SetBlkioLimit(block, "read", "iops", limit.readIops) if err != nil { return err } } if limit.writeBps > 0 { err = cg.SetBlkioLimit(block, "write", "bps", limit.writeBps) if err != nil { return err } } if limit.writeIops > 0 { err = cg.SetBlkioLimit(block, "write", "iops", limit.writeIops) if err != nil { return err } } } } return nil } type cgroupWriter struct { runConf *deviceConfig.RunConfig } // Get is unimplemented for this cgroup handler. func (w *cgroupWriter) Get(controller string, key string) (string, error) { return "", errors.New("This cgroup handler does not support reading") } // Set queues a cgroup key/value to be applied as part of the run config. func (w *cgroupWriter) Set(controller string, key string, value string) error { w.runConf.CGroups = append(w.runConf.CGroups, deviceConfig.RunConfigItem{ Key: key, Value: value, }) return nil } // mountPoolVolume mounts the pool volume specified in d.config["source"] from pool specified in d.config["pool"] // and return the mount path and MountInfo struct. If the instance type is container volume will be shifted if needed. func (d *disk) mountPoolVolume() (func(), string, *storagePools.MountInfo, error) { reverter := revert.New() defer reverter.Fail() var mountInfo *storagePools.MountInfo // Deal with mounting storage volumes created via the storage api. Extract the name of the storage volume // that we are supposed to attach. if filepath.IsAbs(d.config["source"]) { return nil, "", nil, errors.New(`When the "pool" property is set "source" must specify the name of a volume, not a path`) } // Parse the volume name and path. volName, _ := internalInstance.SplitVolumeSource(d.config["source"]) // Only custom volumes can be attached currently. storageProjectName, err := project.StorageVolumeProject(d.state.DB.Cluster, d.inst.Project().Name, db.StoragePoolVolumeTypeCustom) if err != nil { return nil, "", nil, err } volStorageName := project.StorageVolume(storageProjectName, volName) srcPath := storageDrivers.GetVolumeMountPath(d.config["pool"], storageDrivers.VolumeTypeCustom, volStorageName) mountInfo, err = d.pool.MountCustomVolume(storageProjectName, volName, nil) if err != nil { return nil, "", nil, fmt.Errorf("Failed mounting custom storage volume %q on storage pool %q: %w", volName, d.pool.Name(), err) } reverter.Add(func() { _, _ = d.pool.UnmountCustomVolume(storageProjectName, volName, nil) }) var dbVolume *db.StorageVolume err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, d.pool.ID(), storageProjectName, db.StoragePoolVolumeTypeCustom, volName, true) return err }) if err != nil { return nil, "", nil, fmt.Errorf("Failed to fetch local storage volume record: %w", err) } if d.inst.Type() == instancetype.Container { if dbVolume.ContentType == db.StoragePoolVolumeContentTypeNameFS { err = d.storagePoolVolumeAttachShift(storageProjectName, d.pool.Name(), volName, db.StoragePoolVolumeTypeCustom, srcPath) if err != nil { return nil, "", nil, fmt.Errorf("Failed shifting custom storage volume %q on storage pool %q: %w", volName, d.pool.Name(), err) } } else { return nil, "", nil, errors.New("Only filesystem volumes are supported for containers") } } if dbVolume.ContentType == db.StoragePoolVolumeContentTypeNameBlock || dbVolume.ContentType == db.StoragePoolVolumeContentTypeNameISO { srcPath, err = d.pool.GetCustomVolumeDisk(storageProjectName, volName) if err != nil { return nil, "", nil, fmt.Errorf("Failed to get disk path: %w", err) } } cleanup := reverter.Clone().Fail // Clone before calling revert.Success() so we can return the Fail func. reverter.Success() return cleanup, srcPath, mountInfo, err } // createDevice creates a disk device mount on host. // The srcPath argument is the source of the disk device on the host. // Returns the created device path, and whether the path is a file or not. func (d *disk) createDevice(srcPath string) (func(), string, bool, error) { reverter := revert.New() defer reverter.Fail() // Paths. devPath := d.getDevicePath(d.name, d.config) isReadOnly := util.IsTrue(d.config["readonly"]) isRecursive := util.IsTrue(d.config["recursive"]) mntOptions := util.SplitNTrimSpace(d.config["raw.mount.options"], ",", -1, true) fsName := "none" var isFile bool if d.config["pool"] == "" { if d.sourceIsCephFs() { // Get fs name and path from d.config. fields := strings.SplitN(d.config["source"], ":", 2) fields = strings.SplitN(fields[1], "/", 2) mdsName := fields[0] mdsPath := fields[1] clusterName, userName := d.cephCreds() // Get the mount options. mntSrcPath, fsOptions, fsErr := diskCephfsOptions(clusterName, userName, mdsName, mdsPath) if fsErr != nil { return nil, "", false, fsErr } // Join the options with any provided by the user. mntOptions = append(mntOptions, fsOptions...) fsName = "ceph" srcPath = mntSrcPath isFile = false } else if d.sourceIsCeph() { // Get the pool and volume names. fields := strings.SplitN(d.config["source"], ":", 2) fields = strings.SplitN(fields[1], "/", 2) poolName := fields[0] volumeName := fields[1] clusterName, userName := d.cephCreds() // Map the RBD. rbdPath, err := diskCephRbdMap(clusterName, userName, poolName, volumeName) if err != nil { return nil, "", false, diskSourceNotFoundError{msg: "Failed mapping Ceph RBD volume", err: err} } fsName, err = BlockFsDetect(rbdPath) if err != nil { return nil, "", false, fmt.Errorf("Failed detecting source path %q block device filesystem: %w", rbdPath, err) } // Record the device path. err = d.volatileSet(map[string]string{"ceph_rbd": rbdPath}) if err != nil { return nil, "", false, err } srcPath = rbdPath isFile = false } else { fileInfo, err := os.Stat(srcPath) if err != nil { return nil, "", false, fmt.Errorf("Failed accessing source path %q: %w", srcPath, err) } fileMode := fileInfo.Mode() if linux.IsBlockdev(fileMode) { fsName, err = BlockFsDetect(srcPath) if err != nil { return nil, "", false, fmt.Errorf("Failed detecting source path %q block device filesystem: %w", srcPath, err) } } else if !fileMode.IsDir() { isFile = true } f, err := d.localSourceOpen(srcPath) if err != nil { return nil, "", false, err } defer func() { _ = f.Close() }() srcPath = fmt.Sprintf("/proc/self/fd/%d", f.Fd()) } } else if d.config["source"] != "" { // Handle mounting a sub-path. volName, volPath := internalInstance.SplitVolumeSource(d.config["source"]) if volPath != "" { // Get the parent volume. storageProjectName, err := project.StorageVolumeProject(d.state.DB.Cluster, d.inst.Project().Name, db.StoragePoolVolumeTypeCustom) if err != nil { return nil, "", false, err } var dbVolume *db.StorageVolume err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, d.pool.ID(), storageProjectName, db.StoragePoolVolumeTypeCustom, volName, true) return err }) if err != nil { return nil, "", false, err } // Open file handle to parent for use with openat2 later. // Has to use unix.O_PATH to support directories and sockets. srcVolPath, err := os.OpenFile(srcPath, unix.O_PATH, 0) if err != nil { return nil, "", false, fmt.Errorf("Failed opening volume path %q: %w", srcPath, err) } defer func() { _ = srcVolPath.Close() }() openHow := &unix.OpenHow{ Flags: unix.O_PATH | unix.O_CLOEXEC, Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_MAGICLINKS, } // Use openat2 to prevent resolving to a mount path outside of the volume. fd, err := unix.Openat2(int(srcVolPath.Fd()), volPath, openHow) if err != nil { if errors.Is(err, unix.EXDEV) { return nil, "", false, fmt.Errorf("Volume sub-path %q resolves outside of the volume", volPath) } if !errors.Is(err, unix.ENOENT) { return nil, "", false, fmt.Errorf("Failed opening volume sub-path %q: %w", volPath, err) } // Sub-path doesn't exist, attempt to create the missing directories // using the initial.* configuration (if provided). err = d.createVolumeSubPath(dbVolume.Config, srcPath, volPath) if err != nil { return nil, "", false, err } fd, err = unix.Openat2(int(srcVolPath.Fd()), volPath, openHow) if err != nil { return nil, "", false, fmt.Errorf("Failed opening volume sub-path %q: %w", volPath, err) } } srcPathFd := os.NewFile(uintptr(fd), volPath) defer func() { _ = srcPathFd.Close() }() // Check if the sub-path is a file or a directory. fullSubPath := filepath.Join(srcPath, volPath) volSubPathInfo, err := os.Stat(fullSubPath) if err != nil { return nil, "", false, fmt.Errorf("Failed accessing volume sub-path %q: %w", fullSubPath, err) } if !volSubPathInfo.IsDir() { isFile = true } srcPath = fmt.Sprintf("/proc/self/fd/%d", srcPathFd.Fd()) } } // Create the devices directory if missing. if !util.PathExists(d.inst.DevicesPath()) { err := os.Mkdir(d.inst.DevicesPath(), 0o711) if err != nil { return nil, "", false, err } } // Clean any existing entry. if util.PathExists(devPath) { err := os.Remove(devPath) if err != nil { return nil, "", false, err } } // Create the mount point. if isFile { f, err := os.Create(devPath) if err != nil { return nil, "", false, err } _ = f.Close() } else { err := os.Mkdir(devPath, 0o700) if err != nil { return nil, "", false, err } } if isReadOnly { mntOptions = append(mntOptions, "ro") } // Mount the fs. err := DiskMount(srcPath, devPath, isRecursive, d.config["propagation"], mntOptions, fsName) if err != nil { return nil, "", false, err } reverter.Add(func() { _ = DiskMountClear(devPath) }) cleanup := reverter.Clone().Fail // Clone before calling revert.Success() so we can return the Fail func. reverter.Success() return cleanup, devPath, isFile, err } // createVolumeSubPath creates any missing directory components of volPath, anchored at volRootPath. // It uses os.Root to ensure that path resolution stays within the volume root. Newly created // directories use the initial.uid, initial.gid and initial.mode configuration when provided. func (d *disk) createVolumeSubPath(volConfig map[string]string, volRootPath string, volPath string) error { // Get the instance idmap. var nextIdmap *idmap.Set if util.IsFalseOrEmpty(volConfig["security.shifted"]) { var err error c, ok := d.inst.(instance.Container) // Get the container's idmap. if ok { if c.IsRunning() { nextIdmap, err = c.CurrentIdmap() if err != nil { return err } } else { nextIdmap, err = c.NextIdmap() if err != nil { return err } } } } mode := os.FileMode(0o711) if d.config["initial.mode"] != "" { m, err := strconv.ParseInt(d.config["initial.mode"], 8, 0) if err != nil { return fmt.Errorf(`Invalid "initial.mode" value %q: %w`, d.config["initial.mode"], err) } mode = os.FileMode(m) } uid := int64(0) if d.config["initial.uid"] != "" { v, err := strconv.Atoi(d.config["initial.uid"]) if err != nil { return fmt.Errorf(`Invalid "initial.uid" value %q: %w`, d.config["initial.uid"], err) } uid = int64(v) } gid := int64(0) if d.config["initial.gid"] != "" { v, err := strconv.Atoi(d.config["initial.gid"]) if err != nil { return fmt.Errorf(`Invalid "initial.gid" value %q: %w`, d.config["initial.gid"], err) } gid = int64(v) } if nextIdmap != nil { uid, gid = nextIdmap.ShiftIntoNS(uid, gid) } volRoot, err := os.OpenRoot(volRootPath) if err != nil { return fmt.Errorf("Failed opening volume path %q: %w", volRootPath, err) } defer func() { _ = volRoot.Close() }() var current string for _, component := range strings.Split(volPath, "/") { if component == "" || component == "." { continue } if component == ".." { return fmt.Errorf("Volume sub-path %q must not contain %q components", volPath, "..") } if current == "" { current = component } else { current = current + "/" + component } _, err := volRoot.Lstat(current) if err == nil { continue } if !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed checking volume sub-path component %q: %w", current, err) } err = volRoot.Mkdir(current, mode.Perm()) if err != nil { return fmt.Errorf("Failed creating volume sub-path component %q: %w", current, err) } err = volRoot.Lchown(current, int(uid), int(gid)) if err != nil { return fmt.Errorf("Failed setting ownership on volume sub-path component %q: %w", current, err) } err = volRoot.Chmod(current, mode.Perm()) if err != nil { return fmt.Errorf("Failed setting permissions on volume sub-path component %q: %w", current, err) } } return nil } // localSourceOpen opens a local disk source path and returns a file handle to it. // If d.restrictedParentSourcePath has been set during validation, then the openat2 syscall is used to ensure that // the srcPath opened doesn't resolve above the allowed parent source path. func (d *disk) localSourceOpen(srcPath string) (*os.File, error) { var err error var f *os.File if d.restrictedParentSourcePath != "" { // Get relative srcPath in relation to allowed parent source path. relSrcPath, err := filepath.Rel(d.restrictedParentSourcePath, srcPath) if err != nil { return nil, fmt.Errorf("Failed resolving source path %q relative to restricted parent source path %q: %w", srcPath, d.restrictedParentSourcePath, err) } // Open file handle to parent for use with openat2 later. // Has to use unix.O_PATH to support directories and sockets. allowedParent, err := os.OpenFile(d.restrictedParentSourcePath, unix.O_PATH, 0) if err != nil { return nil, fmt.Errorf("Failed opening allowed parent source path %q: %w", d.restrictedParentSourcePath, err) } defer func() { _ = allowedParent.Close() }() // For restricted source paths we use openat2 to prevent resolving to a mount path above the // allowed parent source path. Requires Linux kernel >= 5.6. fd, err := unix.Openat2(int(allowedParent.Fd()), relSrcPath, &unix.OpenHow{ Flags: unix.O_PATH | unix.O_CLOEXEC, Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_MAGICLINKS, }) if err != nil { if errors.Is(err, unix.EXDEV) { return nil, fmt.Errorf("Source path %q resolves outside of restricted parent source path %q", srcPath, d.restrictedParentSourcePath) } return nil, fmt.Errorf("Failed opening restricted source path %q: %w", srcPath, err) } f = os.NewFile(uintptr(fd), srcPath) } else { // Open file handle to local source. Has to use unix.O_PATH to support directories and sockets. f, err = os.OpenFile(srcPath, unix.O_PATH|unix.O_CLOEXEC, 0) if err != nil { return nil, fmt.Errorf("Failed opening source path %q: %w", srcPath, err) } } return f, nil } func (d *disk) storagePoolVolumeAttachShift(projectName, poolName, volumeName string, volumeType int, remapPath string) error { var err error var dbVolume *db.StorageVolume err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, d.pool.ID(), projectName, volumeType, volumeName, true) return err }) if err != nil { return err } poolVolumePut := dbVolume.StorageVolume.Writable() // Check if unmapped. if util.IsTrue(poolVolumePut.Config["security.unmapped"]) { // No need to look at containers and maps for unmapped volumes. return nil } // Get the on-disk idmap for the volume. var lastIdmap *idmap.Set if poolVolumePut.Config["volatile.idmap.last"] != "" { lastIdmap, err = idmap.NewSetFromJSON(poolVolumePut.Config["volatile.idmap.last"]) if err != nil { d.logger.Error("Failed to unmarshal last idmapping", logger.Ctx{"idmap": poolVolumePut.Config["volatile.idmap.last"], "err": err}) return err } } var nextIdmap *idmap.Set nextJSONMap := "[]" if util.IsFalseOrEmpty(poolVolumePut.Config["security.shifted"]) { c := d.inst.(instance.Container) // Get the container's idmap. if c.IsRunning() { nextIdmap, err = c.CurrentIdmap() } else { nextIdmap, err = c.NextIdmap() } if err != nil { return err } if nextIdmap != nil { nextJSONMap, err = nextIdmap.ToJSON() if err != nil { return err } } } poolVolumePut.Config["volatile.idmap.next"] = nextJSONMap if !nextIdmap.Equals(lastIdmap) { d.logger.Debug("Shifting storage volume") if util.IsFalseOrEmpty(poolVolumePut.Config["security.shifted"]) { volumeUsedBy := []instance.Instance{} err = storagePools.VolumeUsedByInstanceDevices(d.state, poolName, projectName, &dbVolume.StorageVolume, true, func(dbInst db.InstanceArgs, project api.Project, usedByDevices []string) error { inst, err := instance.Load(d.state, dbInst, project) if err != nil { return err } volumeUsedBy = append(volumeUsedBy, inst) return nil }) if err != nil { return err } if len(volumeUsedBy) > 1 { for _, inst := range volumeUsedBy { if inst.Type() != instancetype.Container { continue } ct := inst.(instance.Container) var ctNextIdmap *idmap.Set if ct.IsRunning() { ctNextIdmap, err = ct.CurrentIdmap() } else { ctNextIdmap, err = ct.NextIdmap() } if err != nil { return errors.New("Failed to retrieve idmap of container") } if !nextIdmap.Equals(ctNextIdmap) { return fmt.Errorf("Idmaps of container %q and storage volume %q are not identical", ct.Name(), volumeName) } } } else if len(volumeUsedBy) == 1 { // If we're the only one who's attached that container // we can shift the storage volume. // I'm not sure if we want some locking here. if volumeUsedBy[0].Name() != d.inst.Name() { return errors.New("Idmaps of container and storage volume are not identical") } } } // Unshift rootfs. if lastIdmap != nil { var err error if d.pool.Driver().Info().Name == "zfs" { err = lastIdmap.UnshiftPath(remapPath, storageDrivers.ShiftZFSSkipper) } else { err = lastIdmap.UnshiftPath(remapPath, nil) } if err != nil { d.logger.Error("Failed to unshift", logger.Ctx{"path": remapPath, "err": err}) return err } d.logger.Debug("Unshifted", logger.Ctx{"path": remapPath}) } // Shift rootfs. if nextIdmap != nil { var err error if d.pool.Driver().Info().Name == "zfs" { err = nextIdmap.ShiftPath(remapPath, storageDrivers.ShiftZFSSkipper) } else { err = nextIdmap.ShiftPath(remapPath, nil) } if err != nil { d.logger.Error("Failed to shift", logger.Ctx{"path": remapPath, "err": err}) return err } d.logger.Debug("Shifted", logger.Ctx{"path": remapPath}) } d.logger.Debug("Shifted storage volume") } jsonIdmap, err := nextIdmap.ToJSON() if err != nil { d.logger.Error("Failed to marshal idmap", logger.Ctx{"idmap": nextIdmap, "err": err}) return err } // Update last idmap. poolVolumePut.Config["volatile.idmap.last"] = jsonIdmap err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateStoragePoolVolume(ctx, projectName, volumeName, volumeType, d.pool.ID(), poolVolumePut.Description, poolVolumePut.Config) }) if err != nil { return err } return nil } // Stop is run when the device is removed from the instance. func (d *disk) Stop() (*deviceConfig.RunConfig, error) { // Ignore missing disks. if !d.isRequired(d.config) && !d.isAvailable() { return nil, nil } if d.inst.Type() == instancetype.VM { return d.stopVM() } runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } // Figure out the paths relativeDestPath := strings.TrimPrefix(d.config["path"], "/") devPath := d.getDevicePath(d.name, d.config) // The disk device doesn't exist do nothing. if !util.PathExists(devPath) { return nil, nil } // Request an unmount of the device inside the instance. runConf.Mounts = append(runConf.Mounts, deviceConfig.MountEntryItem{ TargetPath: relativeDestPath, }) return &runConf, nil } func (d *disk) stopVM() (*deviceConfig.RunConfig, error) { // Stop the virtiofsd process and clean up. err := DiskVMVirtiofsdStop(d.vmVirtiofsdPaths()) if err != nil { return &deviceConfig.RunConfig{}, fmt.Errorf("Failed cleaning up virtiofsd: %w", err) } runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *disk) postStop() error { // Clean any existing device mount entry. Should occur first before custom volume unmounts. err := DiskMountClear(d.getDevicePath(d.name, d.config)) if err != nil { return err } // Check if pool-specific action should be taken to unmount custom volume disks. if d.config["pool"] != "" && d.config["path"] != "/" { // Only custom volumes can be attached currently. storageProjectName, err := project.StorageVolumeProject(d.state.DB.Cluster, d.inst.Project().Name, db.StoragePoolVolumeTypeCustom) if err != nil { return err } // Parse the volume name and path. volName, _ := internalInstance.SplitVolumeSource(d.config["source"]) _, err = d.pool.UnmountCustomVolume(storageProjectName, volName, nil) if err != nil && !errors.Is(err, storageDrivers.ErrInUse) { return err } } if d.sourceIsCeph() { v := d.volatileGet() err := diskCephRbdUnmap(v["ceph_rbd"]) if err != nil { d.logger.Error("Failed to unmap RBD volume", logger.Ctx{"rbd": v["ceph_rbd"], "err": err}) } } return nil } // getDiskLimits calculates Block I/O limits. func (d *disk) getDiskLimits() (map[string]diskBlockLimit, error) { result := map[string]diskBlockLimit{} // Build a list of all valid block devices validBlocks := []string{} parentBlocks := map[string]string{} dents, err := os.ReadDir("/sys/class/block/") if err != nil { return nil, err } for _, f := range dents { fPath := filepath.Join("/sys/class/block/", f.Name()) // Ignore partitions. if util.PathExists(fmt.Sprintf("%s/partition", fPath)) { continue } // Only select real block devices. if !util.PathExists(fmt.Sprintf("%s/dev", fPath)) { continue } block, err := os.ReadFile(fmt.Sprintf("%s/dev", fPath)) if err != nil { return nil, err } // Add the block to the list. blockIdentifier := strings.TrimSuffix(string(block), "\n") validBlocks = append(validBlocks, blockIdentifier) // Look for partitions. subDents, err := os.ReadDir(fPath) if err != nil { return nil, err } for _, sub := range subDents { // Skip files. if !sub.IsDir() { continue } // Select partitions. if !util.PathExists(filepath.Join(fPath, sub.Name(), "partition")) { continue } // Get the block identifier for the partition. partition, err := os.ReadFile(filepath.Join(fPath, sub.Name(), "dev")) if err != nil { return nil, err } // Add the partition to the map. partitionIdentifier := strings.TrimSuffix(string(partition), "\n") parentBlocks[partitionIdentifier] = blockIdentifier } } // Process all the limits blockLimits := map[string][]diskBlockLimit{} for devName, dev := range d.inst.ExpandedDevices() { if dev["type"] != "disk" { continue } // Parse the user input readBps, readIops, writeBps, writeIops, err := d.parseLimit(dev) if err != nil { return nil, err } // Set the source path source := d.getDevicePath(devName, dev) if dev["source"] == "" { source = d.inst.RootfsPath() } if !util.PathExists(source) { // Require that device is mounted before resolving block device if required. if d.isRequired(dev) { return nil, fmt.Errorf("Block device path doesn't exist %q", source) } continue // Do not resolve block device if device isn't mounted. } // Get the backing block devices (major:minor) blocks, err := d.getParentBlocks(source) if err != nil { if readBps == 0 && readIops == 0 && writeBps == 0 && writeIops == 0 { // If the device doesn't exist, there is no limit to clear so ignore the failure continue } else { return nil, err } } device := diskBlockLimit{readBps: readBps, readIops: readIops, writeBps: writeBps, writeIops: writeIops} for _, block := range blocks { blockStr := "" if slices.Contains(validBlocks, block) { // Straightforward entry (full block device) blockStr = block } else if parentBlocks[block] != "" { // Known partition. blockStr = parentBlocks[block] } else { // Attempt to deal with a partition (guess its parent) fields := strings.SplitN(block, ":", 2) fields[1] = "0" if slices.Contains(validBlocks, fmt.Sprintf("%s:%s", fields[0], fields[1])) { blockStr = fmt.Sprintf("%s:%s", fields[0], fields[1]) } } if blockStr == "" { return nil, fmt.Errorf("Block device doesn't support quotas %q", block) } if blockLimits[blockStr] == nil { blockLimits[blockStr] = []diskBlockLimit{} } blockLimits[blockStr] = append(blockLimits[blockStr], device) } } // Average duplicate limits for block, limits := range blockLimits { var readBpsCount, readBpsTotal, readIopsCount, readIopsTotal, writeBpsCount, writeBpsTotal, writeIopsCount, writeIopsTotal int64 for _, limit := range limits { if limit.readBps > 0 { readBpsCount++ readBpsTotal += limit.readBps } if limit.readIops > 0 { readIopsCount++ readIopsTotal += limit.readIops } if limit.writeBps > 0 { writeBpsCount++ writeBpsTotal += limit.writeBps } if limit.writeIops > 0 { writeIopsCount++ writeIopsTotal += limit.writeIops } } device := diskBlockLimit{} if readBpsCount > 0 { device.readBps = readBpsTotal / readBpsCount } if readIopsCount > 0 { device.readIops = readIopsTotal / readIopsCount } if writeBpsCount > 0 { device.writeBps = writeBpsTotal / writeBpsCount } if writeIopsCount > 0 { device.writeIops = writeIopsTotal / writeIopsCount } result[block] = device } return result, nil } // parseLimit parses the disk configuration for its I/O limits and returns the I/O bytes/iops limits. func (d *disk) parseLimit(dev deviceConfig.Device) (int64, int64, int64, int64, error) { readSpeed := dev["limits.read"] writeSpeed := dev["limits.write"] // Apply max limit. if dev["limits.max"] != "" { readSpeed = dev["limits.max"] writeSpeed = dev["limits.max"] } // parseValue parses a single value to either a B/s limit or iops limit. parseValue := func(value string) (int64, int64, error) { var err error bps := int64(0) iops := int64(0) if value == "" { return bps, iops, nil } if strings.HasSuffix(value, "iops") { iops, err = strconv.ParseInt(strings.TrimSuffix(value, "iops"), 10, 64) if err != nil { return -1, -1, err } } else { bps, err = units.ParseByteSizeString(value) if err != nil { return -1, -1, err } } return bps, iops, nil } // Process reads. readBps, readIops, err := parseValue(readSpeed) if err != nil { return -1, -1, -1, -1, err } // Process writes. writeBps, writeIops, err := parseValue(writeSpeed) if err != nil { return -1, -1, -1, -1, err } return readBps, readIops, writeBps, writeIops, nil } func (d *disk) getParentBlocks(path string) ([]string, error) { var devices []string var dev []string // Expand the mount path absPath, err := filepath.Abs(path) if err != nil { return nil, err } expPath, err := filepath.EvalSymlinks(absPath) if err != nil { expPath = absPath } // Find the source mount of the path file, err := os.Open("/proc/self/mountinfo") if err != nil { return nil, err } defer func() { _ = file.Close() }() scanner := bufio.NewScanner(file) match := "" for scanner.Scan() { line := scanner.Text() rows := strings.Fields(line) if len(rows[4]) <= len(match) { continue } if expPath != rows[4] && !strings.HasPrefix(expPath, rows[4]) { continue } match = rows[4] // Go backward to avoid problems with optional fields dev = []string{rows[2], rows[len(rows)-2]} } if dev == nil { return nil, errors.New("Couldn't find a match /proc/self/mountinfo entry") } // Handle the most simple case if !strings.HasPrefix(dev[0], "0:") { return []string{dev[0]}, nil } // Deal with per-filesystem oddities. We don't care about failures here // because any non-special filesystem => directory backend. fs, _ := linux.DetectFilesystem(expPath) if fs == "zfs" && util.PathExists("/dev/zfs") { // Accessible zfs filesystems poolName := strings.Split(dev[1], "/")[0] output, err := subprocess.RunCommand("zpool", "status", "-P", "-L", poolName) if err != nil { return nil, fmt.Errorf("Failed to query zfs filesystem information for %q: %w", dev[1], err) } header := true for _, line := range strings.Split(output, "\n") { fields := strings.Fields(line) if len(fields) < 5 { continue } if !slices.Contains([]string{"ONLINE", "DEGRADED"}, fields[1]) { continue } if header { header = false continue } var path string if util.PathExists(fields[0]) { if linux.IsBlockdevPath(fields[0]) { path = fields[0] } else { subDevices, err := d.getParentBlocks(fields[0]) if err != nil { return nil, err } devices = append(devices, subDevices...) } } else { continue } if path != "" { _, major, minor, err := unixDeviceAttributes(path) if err != nil { continue } devices = append(devices, fmt.Sprintf("%d:%d", major, minor)) } } if len(devices) == 0 { return nil, fmt.Errorf("Unable to find backing block for zfs pool %q", poolName) } } else if fs == "btrfs" && util.PathExists(dev[1]) { // Accessible btrfs filesystems output, err := subprocess.RunCommand("btrfs", "filesystem", "show", dev[1]) if err != nil { // Fallback to using device path to support BTRFS on block volumes (like LVM). _, major, minor, errFallback := unixDeviceAttributes(dev[1]) if errFallback != nil { return nil, fmt.Errorf("Failed to query btrfs filesystem information for %q: %w", dev[1], err) } devices = append(devices, fmt.Sprintf("%d:%d", major, minor)) } for _, line := range strings.Split(output, "\n") { fields := strings.Fields(line) if len(fields) == 0 || fields[0] != "devid" { continue } _, major, minor, err := unixDeviceAttributes(fields[len(fields)-1]) if err != nil { return nil, err } devices = append(devices, fmt.Sprintf("%d:%d", major, minor)) } } else if util.PathExists(dev[1]) { // Anything else with a valid path _, major, minor, err := unixDeviceAttributes(dev[1]) if err != nil { return nil, err } devices = append(devices, fmt.Sprintf("%d:%d", major, minor)) } else { return nil, fmt.Errorf("Invalid block device %q", dev[1]) } return devices, nil } // generateVMAgent generates an ISO containing the VM agent binary and config. // Returns the path to the ISO. func (d *disk) generateVMAgentDrive() (string, error) { // Take a lock to avoid concurrent start/migrate filling up the disk. diskISOGenerateMu.Lock() defer diskISOGenerateMu.Unlock() scratchDir := filepath.Join(d.inst.DevicesPath(), linux.PathNameEncode(d.name)) defer func() { _ = os.RemoveAll(scratchDir) }() // Check we have the mkisofs or genisoimage tool available. var mkisofsPath string var err error mkisofsPath, err = exec.LookPath("mkisofs") if err != nil { mkisofsPath, err = exec.LookPath("genisoimage") if err != nil { return "", errors.New("Neither mkisofs nor genisoimage could be found in $PATH") } } // Create agent drive dir. err = os.MkdirAll(scratchDir, 0o100) if err != nil { return "", err } // Copy the instance config data over. configPath := filepath.Join(d.inst.Path(), "config") _, err = rsync.LocalCopy(configPath, scratchDir, "", false) if err != nil { return "", err } // Include the most likely agent. if util.PathExists(os.Getenv("INCUS_AGENT_PATH")) { dstFilename := "incus-agent" guestOS := d.inst.GuestOS() switch guestOS { case "unknown": guestOS = "linux" case "windows": dstFilename = "incus-agent.exe" } archName, err := osarch.ArchitectureName(d.inst.Architecture()) if err != nil { return "", err } srcFilename := fmt.Sprintf("incus-agent.%s.%s", guestOS, archName) agentInstallPath := filepath.Join(scratchDir, dstFilename) os.Remove(agentInstallPath) err = internalUtil.FileCopy(filepath.Join(os.Getenv("INCUS_AGENT_PATH"), srcFilename), agentInstallPath) if err != nil { return "", err } err = os.Chmod(agentInstallPath, 0o500) if err != nil { return "", err } err = os.Chown(agentInstallPath, 0, 0) if err != nil { return "", err } } // Finally convert the agent drive dir into an ISO file. The incus-agent label is important // as this is what incus-agent-loader uses to detect the drive. isoPath := filepath.Join(d.inst.Path(), "agent.iso") _, err = subprocess.RunCommand(mkisofsPath, "-joliet", "-rock", "-input-charset", "utf8", "-output-charset", "utf8", "-volid", "incus-agent", "-o", isoPath, scratchDir) if err != nil { return "", err } return isoPath, nil } // generateVMConfigDrive generates an ISO containing the cloud init config for a VM. // Returns the path to the ISO. func (d *disk) generateVMConfigDrive() (string, error) { // Take a lock to avoid concurrent start/migrate filling up the disk. diskISOGenerateMu.Lock() defer diskISOGenerateMu.Unlock() scratchDir := filepath.Join(d.inst.DevicesPath(), linux.PathNameEncode(d.name)) defer func() { _ = os.RemoveAll(scratchDir) }() // Check we have the mkisofs tool available. mkisofsPath, err := exec.LookPath("mkisofs") if err != nil { return "", err } // Create config drive dir. err = os.MkdirAll(scratchDir, 0o100) if err != nil { return "", err } instanceConfig := d.inst.ExpandedConfig() // Use an empty vendor-data file if no custom vendor-data supplied. vendorData, ok := instanceConfig["cloud-init.vendor-data"] if !ok { vendorData = instanceConfig["user.vendor-data"] if vendorData == "" { vendorData = "#cloud-config\n{}" } } err = os.WriteFile(filepath.Join(scratchDir, "vendor-data"), []byte(vendorData), 0o400) if err != nil { return "", err } // Use an empty user-data file if no custom user-data supplied. userData, ok := instanceConfig["cloud-init.user-data"] if !ok { userData = instanceConfig["user.user-data"] if userData == "" { userData = "#cloud-config\n{}" } } err = os.WriteFile(filepath.Join(scratchDir, "user-data"), []byte(userData), 0o400) if err != nil { return "", err } // Include a network-config file if the user configured it. networkConfig, ok := instanceConfig["cloud-init.network-config"] if !ok { networkConfig = instanceConfig["user.network-config"] } if networkConfig != "" { err = os.WriteFile(filepath.Join(scratchDir, "network-config"), []byte(networkConfig), 0o400) if err != nil { return "", err } } // Append any custom meta-data to our predefined meta-data config. metaData := fmt.Sprintf(`instance-id: %s local-hostname: %s %s `, d.inst.Name(), d.inst.Name(), instanceConfig["user.meta-data"]) err = os.WriteFile(filepath.Join(scratchDir, "meta-data"), []byte(metaData), 0o400) if err != nil { return "", err } // Finally convert the config drive dir into an ISO file. The cidata label is important // as this is what cloud-init uses to detect, mount the drive and run the cloud-init // templates on first boot. The vendor-data template then modifies the system so that the // config drive is mounted and the agent is started on subsequent boots. isoPath := filepath.Join(d.inst.Path(), "config.iso") _, err = subprocess.RunCommand(mkisofsPath, "-joliet", "-rock", "-input-charset", "utf8", "-output-charset", "utf8", "-volid", "cidata", "-o", isoPath, scratchDir) if err != nil { return "", err } return isoPath, nil } // cephCreds returns cluster name and user name to use for ceph disks. func (d *disk) cephCreds() (string, string) { // Apply the ceph configuration. userName := d.config["ceph.user_name"] if userName == "" { userName = storageDrivers.CephDefaultUser } clusterName := d.config["ceph.cluster_name"] if clusterName == "" { clusterName = storageDrivers.CephDefaultCluster } return clusterName, userName } // Remove cleans up the device when it is removed from an instance. func (d *disk) Remove(cleanupDependencies bool) error { // Remove the config.iso file for cloud-init config drives. if d.config["source"] == diskSourceCloudInit { pool, err := storagePools.LoadByInstance(d.state, d.inst) if err != nil { return err } _, err = pool.MountInstance(d.inst, nil) if err != nil { return err } defer func() { _ = pool.UnmountInstance(d.inst, nil) }() isoPath := filepath.Join(d.inst.Path(), "config.iso") err = os.Remove(isoPath) if err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("Failed removing %s file: %w", diskSourceCloudInit, err) } } if d.config["pool"] != "" && d.config["source"] != "" && util.IsTrue(d.config["dependent"]) && cleanupDependencies { d.config["dependent"] = "" // If the volume doesn't exist, ignore this as we remove the device anyway. _, err := d.updateDependentConfig() if err != nil && !response.IsNotFoundError(err) { return err } } return nil } // getAttachedInstanceCount returns the number of instances // to which this disk is currently attached. func (d *disk) getAttachedInstanceCount(projectName string, volume *db.StorageVolume) (int, error) { count := 0 err := storagePools.VolumeUsedByInstanceDevices(d.state, d.pool.Name(), projectName, &volume.StorageVolume, true, func(inst db.InstanceArgs, project api.Project, usedByDevices []string) error { // Don't count the current instance. if d.inst != nil && d.inst.Project().Name == inst.Project && d.inst.Name() == inst.Name { return nil } count += 1 return nil }) if err != nil { return -1, err } return count, nil } // updateDependentConfig applies changes to dependent configuration settings. func (d *disk) updateDependentConfig() (func() error, error) { // Parse the volume name and path. volName, _ := internalInstance.SplitVolumeSource(d.config["source"]) storageProjectName, err := project.StorageVolumeProject(d.state.DB.Cluster, d.inst.Project().Name, db.StoragePoolVolumeTypeCustom) if err != nil { return nil, err } var dbVolume *db.StorageVolume err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, d.pool.ID(), storageProjectName, db.StoragePoolVolumeTypeCustom, volName, true) return err }) if err != nil { return nil, err } if dbVolume.Type == db.StoragePoolVolumeTypeNameCustom { if util.IsTrue(d.config["dependent"]) { var volSnapshots []db.StorageVolumeArgs err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { volSnapshots, err = tx.GetLocalStoragePoolVolumeSnapshotsWithType(ctx, storageProjectName, volName, db.StoragePoolVolumeTypeCustom, d.pool.ID()) return err }) if err != nil { return nil, fmt.Errorf("Failed loading custom volume: %w", err) } if len(volSnapshots) > 0 { return nil, fmt.Errorf("Cannot attach volume with snapshots as dependent") } var snapshots []cluster.Instance err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { snapshots, err = tx.GetInstanceSnapshotsWithName(ctx, d.inst.Project().Name, d.inst.Name()) return err }) if err != nil { return nil, err } for _, snap := range snapshots { _, snapName, _ := api.GetParentAndSnapshotName(snap.Name) err = d.pool.CreateCustomVolumeSnapshot(storageProjectName, volName, snapName, snap.ExpiryDate.Time, false, nil) if err != nil { return nil, err } } } poolVolumePut := dbVolume.Writable() poolVolumePut.Config["dependent"] = d.config["dependent"] err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateStoragePoolVolume(ctx, storageProjectName, volName, db.StoragePoolVolumeTypeCustom, d.pool.ID(), poolVolumePut.Description, poolVolumePut.Config) }) if err != nil { return nil, err } } cleanup := func() error { poolVolumePut := dbVolume.Writable() if util.IsTrue(d.config["dependent"]) { poolVolumePut.Config["dependent"] = "" } else { poolVolumePut.Config["dependent"] = "true" } err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateStoragePoolVolume(ctx, storageProjectName, volName, db.StoragePoolVolumeTypeCustom, d.pool.ID(), poolVolumePut.Description, poolVolumePut.Config) }) if err != nil { return err } return nil } return cleanup, nil } incus-7.0.0/internal/server/device/errors.go000066400000000000000000000012621517523235500210740ustar00rootroot00000000000000package device import ( "errors" ) // UnsupportedError used for indicating the error is caused due to a lack of support. type UnsupportedError struct { msg string } func (e UnsupportedError) Error() string { return e.msg } // ErrUnsupportedDevType is the error that occurs when an unsupported device type is created. var ErrUnsupportedDevType = UnsupportedError{msg: "Unsupported device type"} // ErrCannotUpdate is the error that occurs when a device cannot be updated. var ErrCannotUpdate = errors.New("Device does not support updates") // ErrMissingVirtiofsd is the error that occurs if virtiofsd is missing. var ErrMissingVirtiofsd = UnsupportedError{msg: "Virtiofsd missing"} incus-7.0.0/internal/server/device/gpu.go000066400000000000000000000045571517523235500203650ustar00rootroot00000000000000package device import ( "fmt" "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/validate" ) func gpuValidationRules(requiredFields []string, optionalFields []string) map[string]func(value string) error { // Define a set of default validators for each field name. defaultValidators := map[string]func(value string) error{ "vendorid": validate.Optional(validate.IsDeviceID), "productid": validate.Optional(validate.IsDeviceID), "id": validate.IsAny, "pci": validate.IsPCIAddress, "uid": unixValidUserID, "gid": unixValidUserID, "mode": unixValidOctalFileMode, "mig.gi": validate.IsUint8, "mig.ci": validate.IsUint8, "mig.uuid": gpuValidMigUUID, "mdev": validate.IsAny, } validators := map[string]func(value string) error{} for _, k := range optionalFields { defaultValidator := defaultValidators[k] // If field doesn't have a known validator, it is an unknown field, skip. if defaultValidator == nil { continue } // Wrap the default validator in an empty check as field is optional. validators[k] = func(value string) error { if value == "" { return nil } return defaultValidator(value) } } // Add required fields last, that way if they are specified in both required and optional // field sets, the required one will overwrite the optional validators. for _, k := range requiredFields { defaultValidator := defaultValidators[k] // If field doesn't have a known validator, it is an unknown field, skip. if defaultValidator == nil { continue } // Wrap the default validator in a not empty check as field is required. validators[k] = func(value string) error { err := validate.IsNotEmpty(value) if err != nil { return err } return defaultValidator(value) } } return validators } // Check if the device matches the given GPU card. // It matches based on vendorid, pci, productid or id setting of the device. func gpuSelected(device config.Device, gpu api.ResourcesGPUCard) bool { return !((device["vendorid"] != "" && gpu.VendorID != device["vendorid"]) || (device["pci"] != "" && gpu.PCIAddress != device["pci"]) || (device["productid"] != "" && gpu.ProductID != device["productid"]) || (device["id"] != "" && (gpu.DRM == nil || fmt.Sprintf("%d", gpu.DRM.ID) != device["id"]))) } incus-7.0.0/internal/server/device/gpu_mdev.go000066400000000000000000000165041517523235500213730ustar00rootroot00000000000000package device import ( "errors" "fmt" "io/fs" "os" "path/filepath" "sync" "github.com/google/uuid" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" pcidev "github.com/lxc/incus/v7/internal/server/device/pci" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" ) var gpuMdevMu sync.Mutex type gpuMdev struct { deviceCommon } // Start is run when the device is added to the container. func (d *gpuMdev) Start() (*deviceConfig.RunConfig, error) { err := d.validateEnvironment() if err != nil { return nil, err } return d.startVM() } // Stop is run when the device is removed from the instance. func (d *gpuMdev) Stop() (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } return &runConf, nil } // startVM detects the requested GPU devices and related virtual functions and rebinds them to the vfio-pci driver. func (d *gpuMdev) startVM() (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{} // Lock to prevent multiple concurrent mdev devices being setup. gpuMdevMu.Lock() defer gpuMdevMu.Unlock() // Get any existing UUID. v := d.volatileGet() mdevUUID := v["vgpu.uuid"] // Get the local GPUs. gpus, err := resources.GetGPU() if err != nil { return nil, err } reverter := revert.New() defer reverter.Fail() var pciAddress string for _, gpu := range gpus.Cards { // Skip any cards that are not selected. if !gpuSelected(d.Config(), gpu) { continue } if pciAddress != "" { return nil, errors.New("VMs cannot match multiple GPUs per device") } pciAddress = gpu.PCIAddress // Look for the requested mdev profile on the GPU itself. mdevFound := false mdevAvailable := false for k, v := range gpu.Mdev { if d.config["mdev"] == k { mdevFound = true if v.Available > 0 { mdevAvailable = true } break } } // If no mdev found on the GPU and SR-IOV is present, look on the VFs. if !mdevFound && gpu.SRIOV != nil { for _, vf := range gpu.SRIOV.VFs { for k, v := range vf.Mdev { if d.config["mdev"] == k { mdevFound = true if v.Available > 0 { mdevAvailable = true // Replace the PCI address with that of the VF. pciAddress = vf.PCIAddress } break } } if mdevAvailable { break } } } if !mdevFound { return nil, fmt.Errorf("Invalid mdev profile %q", d.config["mdev"]) } if !mdevAvailable { return nil, fmt.Errorf("No available mdev for profile %q", d.config["mdev"]) } // Create the vGPU. if mdevUUID == "" || !util.PathExists(fmt.Sprintf("/sys/bus/pci/devices/%s/%s", pciAddress, mdevUUID)) { mdevUUID = uuid.New().String() err = os.WriteFile(filepath.Join(fmt.Sprintf("/sys/bus/pci/devices/%s/mdev_supported_types/%s/create", pciAddress, d.config["mdev"])), []byte(mdevUUID), 0o200) if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil, fmt.Errorf("The requested profile %q does not exist", d.config["mdev"]) } return nil, fmt.Errorf("Failed to create virtual gpu %q: %w", mdevUUID, err) } reverter.Add(func() { path := fmt.Sprintf("/sys/bus/mdev/devices/%s", mdevUUID) if util.PathExists(path) { err := os.WriteFile(filepath.Join(path, "remove"), []byte("1\n"), 0o200) if err != nil { d.logger.Error("Failed to remove vgpu", logger.Ctx{"device": mdevUUID, "err": err}) } } }) } } if pciAddress == "" { return nil, errors.New("Failed to detect requested GPU device") } // Get PCI information about the GPU device. devicePath := filepath.Join("/sys/bus/pci/devices", pciAddress) pciDev, err := pcidev.ParseUeventFile(filepath.Join(devicePath, "uevent")) if err != nil { return nil, fmt.Errorf("Failed to get PCI device info for GPU %q: %w", pciAddress, err) } // Prepare the new volatile keys. saveData := make(map[string]string) saveData["last_state.pci.slot.name"] = pciDev.SlotName saveData["last_state.pci.driver"] = pciDev.Driver saveData["vgpu.uuid"] = mdevUUID runConf.GPUDevice = append(runConf.GPUDevice, []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "pciSlotName", Value: saveData["last_state.pci.slot.name"]}, {Key: "vgpu", Value: mdevUUID}, }...) err = d.volatileSet(saveData) if err != nil { return nil, err } reverter.Success() return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *gpuMdev) postStop() error { defer func() { _ = d.volatileSet(map[string]string{ "last_state.pci.slot.name": "", "last_state.pci.driver": "", "vgpu.uuid": "", }) }() v := d.volatileGet() if v["vgpu.uuid"] != "" { path := fmt.Sprintf("/sys/bus/mdev/devices/%s", v["vgpu.uuid"]) if util.PathExists(path) { err := os.WriteFile(filepath.Join(path, "remove"), []byte("1\n"), 0o200) if err != nil { d.logger.Error("Failed to remove vgpu", logger.Ctx{"device": v["vgpu.uuid"], "err": err}) } } } return nil } // validateConfig checks the supplied config for correctness. func (d *gpuMdev) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.VM) { return ErrUnsupportedDevType } requiredFields := []string{ // gendoc:generate(entity=devices, group=gpu_mdev, key=mdev) // // --- // type: string // required: yes // shortdesc: The mediated device profile to use (required - for example, `i915-GVTg_V5_4`) "mdev", } optionalFields := []string{ // gendoc:generate(entity=devices, group=gpu_mdev, key=vendorid) // // --- // type: string // required: no // shortdesc: The vendor ID of the GPU device "vendorid", // gendoc:generate(entity=devices, group=gpu_mdev, key=productid) // // --- // type: string // required: no // shortdesc: The product ID of the GPU device "productid", // gendoc:generate(entity=devices, group=gpu_mdev, key=id) // // --- // type: string // required: no // shortdesc: The DRM card ID of the GPU device "id", // gendoc:generate(entity=devices, group=gpu_mdev, key=pci // // --- // type: strong // required: no // shortdesc: The PCI address of the GPU device "pci", } err := d.config.Validate(gpuValidationRules(requiredFields, optionalFields)) if err != nil { return err } if d.config["pci"] != "" { for _, field := range []string{"id", "productid", "vendorid"} { if d.config[field] != "" { return fmt.Errorf(`Cannot use %q when "pci" is set`, field) } } d.config["pci"] = pcidev.NormaliseAddress(d.config["pci"]) } if d.config["id"] != "" { for _, field := range []string{"pci", "productid", "vendorid"} { if d.config[field] != "" { return fmt.Errorf(`Cannot use %q when "id" is set`, field) } } } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *gpuMdev) validateEnvironment() error { if d.inst.Type() == instancetype.VM && util.IsTrue(d.inst.ExpandedConfig()["migration.stateful"]) { return errors.New("GPU devices cannot be used when migration.stateful is enabled") } return validatePCIDevice(d.config["pci"]) } incus-7.0.0/internal/server/device/gpu_mig.go000066400000000000000000000131341517523235500212100ustar00rootroot00000000000000package device import ( "errors" "fmt" "strings" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" pcidev "github.com/lxc/incus/v7/internal/server/device/pci" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/util" ) type gpuMIG struct { deviceCommon } // GPUNvidiaDeviceKey is the key used for NVIDIA devices through libnvidia-container. const GPUNvidiaDeviceKey = "nvidia.device" // validateConfig checks the supplied config for correctness. func (d *gpuMIG) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container) { return ErrUnsupportedDevType } requiredFields := []string{} optionalFields := []string{ // gendoc:generate(entity=devices, group=gpu_mig, key=vendorid) // // --- // type: string // required: no // shortdesc: The vendor ID of the GPU device "vendorid", // gendoc:generate(entity=devices, group=gpu_mig, key=productid) // // --- // type: string // required: no // shortdesc: The product ID of the GPU device "productid", // gendoc:generate(entity=devices, group=gpu_mig, key=id) // // --- // type: string // required: no // shortdesc: The DRM card ID of the GPU device "id", // gendoc:generate(entity=devices, group=gpu_mig, key=pci) // // --- // type: string // required: no // shortdesc: The PCI address of the GPU device "pci", // gendoc:generate(entity=devices, group=gpu_mig, key=mig.gi) // // --- // type: int // required: no // shortdesc: Existing MIG GPU instance ID "mig.gi", // gendoc:generate(entity=devices, group=gpu_mig, key=mig.ci) // // --- // type: int // required: no // shortdesc: Existing MIG compute instance ID "mig.ci", // gendoc:generate(entity=devices, group=gpu_mig, key=mig.uuid) // // --- // type: string // required: no // shortdesc: Existing MIG device UUID (MIG- prefix can be omitted) "mig.uuid", } err := d.config.Validate(gpuValidationRules(requiredFields, optionalFields)) if err != nil { return err } if d.config["pci"] != "" { for _, field := range []string{"id", "productid", "vendorid"} { if d.config[field] != "" { return fmt.Errorf(`Cannot use %q when "pci" is set`, field) } } d.config["pci"] = pcidev.NormaliseAddress(d.config["pci"]) } if d.config["id"] != "" { for _, field := range []string{"pci", "productid", "vendorid"} { if d.config[field] != "" { return fmt.Errorf(`Cannot use %q when "id" is set`, field) } } } if d.config["mig.uuid"] != "" { for _, field := range []string{"mig.gi", "mig.ci"} { if d.config[field] != "" { return fmt.Errorf(`Cannot use %q when "mig.uuid" is set`, field) } } } else if d.config["mig.gi"] == "" || d.config["mig.ci"] == "" { return errors.New(`Either "mig.uuid" or both "mig.gi" and "mig.ci" must be set`) } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *gpuMIG) validateEnvironment() error { if util.IsFalseOrEmpty(d.inst.ExpandedConfig()["nvidia.runtime"]) { return errors.New("nvidia.runtime must be set to true for MIG GPUs to work") } return validatePCIDevice(d.config["pci"]) } // buildMIGDeviceName builds the name of the MIG device based on old/new format. func (d *gpuMIG) buildMIGDeviceName(gpu api.ResourcesGPUCard) string { if d.config["mig.uuid"] != "" { if strings.HasPrefix(d.config["mig.uuid"], "MIG-") { return d.config["mig.uuid"] } return fmt.Sprintf("MIG-%s", d.config["mig.uuid"]) } return fmt.Sprintf("MIG-%s/%s/%s", gpu.Nvidia.UUID, d.config["mig.gi"], d.config["mig.ci"]) } // CanHotPlug returns whether the device can be managed whilst the instance is running,. func (d *gpuMIG) CanHotPlug() bool { return false } // Start is run when the device is added to the container. func (d *gpuMIG) Start() (*deviceConfig.RunConfig, error) { // Check the basic config. err := d.validateEnvironment() if err != nil { return nil, err } runConf := deviceConfig.RunConfig{} // Get all the GPUs. gpus, err := resources.GetGPU() if err != nil { return nil, err } var pciAddress string for _, gpu := range gpus.Cards { // Skip any cards that are not selected. if !gpuSelected(d.Config(), gpu) { continue } // We found a match. if pciAddress != "" { return nil, errors.New("More than one GPU matched the MIG device") } pciAddress = gpu.PCIAddress // Validate the GPU. if gpu.Nvidia == nil { return nil, errors.New("Card isn't a NVIDIA GPU or driver isn't properly setup") } // Validate the MIG. fields := strings.SplitN(gpu.Nvidia.CardDevice, ":", 2) if len(fields) != 2 { return nil, errors.New("Bad NVIDIA GPU (couldn't find ID)") } gpuID := fields[1] if d.config["mig.uuid"] == "" { if !util.PathExists(fmt.Sprintf("/proc/driver/nvidia/capabilities/gpu%s/mig/gi%s/ci%s/access", gpuID, d.config["mig.gi"], d.config["mig.ci"])) { return nil, fmt.Errorf("MIG device gi=%s ci=%s doesn't exist on GPU %s", d.config["mig.gi"], d.config["mig.ci"], gpuID) } } runConf.GPUDevice = append(runConf.GPUDevice, []deviceConfig.RunConfigItem{ {Key: GPUNvidiaDeviceKey, Value: d.buildMIGDeviceName(gpu)}, }...) } if pciAddress == "" { return nil, errors.New("Failed to detect requested GPU device") } return &runConf, nil } // Stop is run when the device is removed from the instance. func (d *gpuMIG) Stop() (*deviceConfig.RunConfig, error) { return nil, nil } incus-7.0.0/internal/server/device/gpu_physical.go000066400000000000000000000344731517523235500222610ustar00rootroot00000000000000package device import ( "errors" "fmt" "io/fs" "os" "path/filepath" "regexp" "strconv" "strings" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" pcidev "github.com/lxc/incus/v7/internal/server/device/pci" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/util" ) const gpuDRIDevPath = "/dev/dri" // Non-card devices such as {/dev/nvidiactl, /dev/nvidia-uvm, ...}. type nvidiaNonCardDevice struct { path string major uint32 minor uint32 } type gpuPhysical struct { deviceCommon } // validateConfig checks the supplied config for correctness. func (d *gpuPhysical) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container, instancetype.VM) { return ErrUnsupportedDevType } optionalFields := []string{ // gendoc:generate(entity=devices, group=gpu_physical, key=vendorid) // // --- // type: string // required: no // shortdesc: The vendor ID of the GPU device "vendorid", // gendoc:generate(entity=devices, group=gpu_physical, key=productid) // // --- // type: string // required: no // shortdesc: The product ID of the GPU device "productid", // gendoc:generate(entity=devices, group=gpu_physical, key=id) // // --- // type: string // required: no // shortdesc: The DRM card ID of the GPU device "id", // gendoc:generate(entity=devices, group=gpu_physical, key=pci) // // --- // type: string // required: no // shortdesc: The PCI address of the GPU device "pci", } if instConf.Type() == instancetype.Container || instConf.Type() == instancetype.Any { // gendoc:generate(entity=devices, group=gpu_physical, key=uid) // // --- // type: int // default: 0 // required: no // shortdesc: UID of the device owner in the instance (container only) // gendoc:generate(entity=devices, group=gpu_physical, key=gid) // // --- // type: int // default: 0 // required: no // shortdesc: GID of the device owner in the instance (container only) // gendoc:generate(entity=devices, group=gpu_physical, key=mode) // // --- // type: int // default: 0660 // required: no // shortdesc: Mode of the device in the instance (container only) optionalFields = append(optionalFields, "uid", "gid", "mode") } err := d.config.Validate(gpuValidationRules(nil, optionalFields)) if err != nil { return err } if d.config["pci"] != "" { for _, field := range []string{"id", "productid", "vendorid"} { if d.config[field] != "" { return fmt.Errorf(`Cannot use %q when "pci" is set`, field) } } d.config["pci"] = pcidev.NormaliseAddress(d.config["pci"]) } if d.config["id"] != "" { for _, field := range []string{"pci", "productid", "vendorid"} { if d.config[field] != "" { return fmt.Errorf(`Cannot use %q when "id" is set`, field) } } } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *gpuPhysical) validateEnvironment() error { if d.inst.Type() == instancetype.VM && util.IsTrue(d.inst.ExpandedConfig()["migration.stateful"]) { return errors.New("GPU devices cannot be used when migration.stateful is enabled") } return validatePCIDevice(d.config["pci"]) } // Start is run when the device is added to the container. func (d *gpuPhysical) Start() (*deviceConfig.RunConfig, error) { err := d.validateEnvironment() if err != nil { return nil, err } if d.inst.Type() == instancetype.VM { return d.startVM() } return d.startContainer() } // startContainer detects the requested GPU devices and sets up unix-char devices. // Returns RunConfig populated with mount info required to pass the unix-char devices into the container. func (d *gpuPhysical) startContainer() (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{} gpus, err := resources.GetGPU() if err != nil { return nil, err } sawNvidia := false found := false for _, gpu := range gpus.Cards { // Skip any cards that are not selected. if !gpuSelected(d.Config(), gpu) { continue } // We found a match. found = true // Setup DRM unix-char devices if present. if gpu.DRM != nil { if gpu.DRM.CardName != "" && gpu.DRM.CardDevice != "" && util.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.CardName)) { path := filepath.Join(gpuDRIDevPath, gpu.DRM.CardName) major, minor, err := d.deviceNumStringToUint32(gpu.DRM.CardDevice) if err != nil { return nil, err } err = unixDeviceSetupCharNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, major, minor, path, false, &runConf) if err != nil { return nil, err } } if gpu.DRM.RenderName != "" && gpu.DRM.RenderDevice != "" && util.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.RenderName)) { path := filepath.Join(gpuDRIDevPath, gpu.DRM.RenderName) major, minor, err := d.deviceNumStringToUint32(gpu.DRM.RenderDevice) if err != nil { return nil, err } err = unixDeviceSetupCharNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, major, minor, path, false, &runConf) if err != nil { return nil, err } } if gpu.DRM.ControlName != "" && gpu.DRM.ControlDevice != "" && util.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.ControlName)) { path := filepath.Join(gpuDRIDevPath, gpu.DRM.ControlName) major, minor, err := d.deviceNumStringToUint32(gpu.DRM.ControlDevice) if err != nil { return nil, err } err = unixDeviceSetupCharNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, major, minor, path, false, &runConf) if err != nil { return nil, err } } } // Add Nvidia device if present. if gpu.Nvidia != nil && gpu.Nvidia.CardName != "" && gpu.Nvidia.CardDevice != "" && util.PathExists(filepath.Join("/dev", gpu.Nvidia.CardName)) { sawNvidia = true path := filepath.Join("/dev", gpu.Nvidia.CardName) major, minor, err := d.deviceNumStringToUint32(gpu.Nvidia.CardDevice) if err != nil { return nil, err } err = unixDeviceSetupCharNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, major, minor, path, false, &runConf) if err != nil { return nil, err } } } // Setup additional unix-char devices for nvidia cards. // No need to mount additional nvidia non-card devices as the nvidia.runtime setting will do this for us. if sawNvidia { instanceConfig := d.inst.ExpandedConfig() if util.IsFalseOrEmpty(instanceConfig["nvidia.runtime"]) { nvidiaDevices, err := d.getNvidiaNonCardDevices() if err != nil { return nil, err } for _, dev := range nvidiaDevices { prefix := deviceJoinPath("unix", d.name) if UnixDeviceExists(d.inst.DevicesPath(), prefix, dev.path) { continue } err = unixDeviceSetupCharNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, dev.major, dev.minor, dev.path, false, &runConf) if err != nil { return nil, err } } } } if !found { return nil, errors.New("Failed to detect requested GPU device") } return &runConf, nil } // startVM detects the requested GPU devices and related virtual functions and rebinds them to the vfio-pci driver. func (d *gpuPhysical) startVM() (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{} gpus, err := resources.GetGPU() if err != nil { return nil, err } saveData := make(map[string]string) var pciAddress string for _, gpu := range gpus.Cards { // Skip any cards that are not selected. if !gpuSelected(d.Config(), gpu) { continue } // Check for existing running processes tied to the GPU. // Failing early here in case of attached running processes to the card // avoids a blocking call to os.WriteFile() when unbinding the device. if gpu.Nvidia != nil && gpu.Nvidia.CardName != "" && util.PathExists(filepath.Join("/dev", gpu.Nvidia.CardName)) { devPath := filepath.Join("/dev", gpu.Nvidia.CardName) runningProcs, err := checkAttachedRunningProcesses(devPath) if err != nil { return nil, err } if len(runningProcs) > 0 { return nil, fmt.Errorf( "Cannot use device %q, %d processes are still attached to it:\n\t%s", devPath, len(runningProcs), strings.Join(runningProcs, "\n\t"), ) } } if pciAddress != "" { return nil, errors.New("VMs cannot match multiple GPUs per device") } pciAddress = gpu.PCIAddress } if pciAddress == "" { return nil, errors.New("Failed to detect requested GPU device") } // Make sure that vfio-pci is loaded. err = linux.LoadModule("vfio-pci") if err != nil { return nil, fmt.Errorf("Error loading %q module: %w", "vfio-pci", err) } // Get PCI information about the GPU device. devicePath := filepath.Join("/sys/bus/pci/devices", pciAddress) pciDev, err := pcidev.ParseUeventFile(filepath.Join(devicePath, "uevent")) if err != nil { return nil, fmt.Errorf("Failed to get PCI device info for GPU %q: %w", pciAddress, err) } saveData["last_state.pci.slot.name"] = pciDev.SlotName saveData["last_state.pci.driver"] = pciDev.Driver err = d.pciDeviceDriverOverrideIOMMU(pciDev, "vfio-pci", false) if err != nil { return nil, fmt.Errorf("Failed to override IOMMU group driver: %w", err) } runConf.GPUDevice = append(runConf.GPUDevice, []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "pciSlotName", Value: saveData["last_state.pci.slot.name"]}, }...) err = d.volatileSet(saveData) if err != nil { return nil, err } return &runConf, nil } // pciDeviceDriverOverrideIOMMU overrides all functions in the specified device's IOMMU group (if exists) that // are functions of the device. If IOMMU group doesn't exist, only the device itself is overridden. // If restore argument is true, then IOMMU VF devices related to the main device have their driver override cleared // rather than being set to the driverOverride specified. This allows for IOMMU VFs that were using a different // driver (or no driver) when being overridden are not restored back to the main device's driver. func (d *gpuPhysical) pciDeviceDriverOverrideIOMMU(pciDev pcidev.Device, driverOverride string, restore bool) error { iommuGroupPath := filepath.Join("/sys/bus/pci/devices", pciDev.SlotName, "iommu_group", "devices") if util.PathExists(iommuGroupPath) { // Extract parent slot name by removing any virtual function ID. parts := strings.SplitN(pciDev.SlotName, ".", 2) prefix := parts[0] // Iterate the members of the IOMMU group and override any that match the parent slot name prefix. err := filepath.Walk(iommuGroupPath, func(path string, _ os.FileInfo, err error) error { if err != nil { return err } iommuSlotName := filepath.Base(path) // Virtual function's address is dir name. if strings.HasPrefix(iommuSlotName, prefix) { iommuPciDev := pcidev.Device{ Driver: pciDev.Driver, SlotName: iommuSlotName, } if iommuSlotName != pciDev.SlotName && restore { // We don't know the original driver for VFs, so just remove override. err = pcidev.DeviceDriverOverride(iommuPciDev, "") } else { err = pcidev.DeviceDriverOverride(iommuPciDev, driverOverride) } if err != nil { return err } } return nil }) if err != nil { return err } } else { err := pcidev.DeviceDriverOverride(pciDev, driverOverride) if err != nil { return err } } return nil } // Stop is run when the device is removed from the instance. func (d *gpuPhysical) Stop() (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } if d.inst.Type() == instancetype.Container { err := unixDeviceRemove(d.inst.DevicesPath(), "unix", d.name, "", &runConf) if err != nil { return nil, err } } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *gpuPhysical) postStop() error { defer func() { _ = d.volatileSet(map[string]string{ "last_state.pci.slot.name": "", "last_state.pci.driver": "", "vgpu.uuid": "", }) }() v := d.volatileGet() if d.inst.Type() == instancetype.Container { // Remove host files for this device. err := unixDeviceDeleteFiles(d.state, d.inst.DevicesPath(), "unix", d.name, "") if err != nil { return fmt.Errorf("Failed to delete files for device '%s': %w", d.name, err) } } // If VM physical pass through, unbind from vfio-pci and bind back to host driver. if d.inst.Type() == instancetype.VM && v["last_state.pci.slot.name"] != "" { pciDev := pcidev.Device{ Driver: "vfio-pci", SlotName: v["last_state.pci.slot.name"], } err := d.pciDeviceDriverOverrideIOMMU(pciDev, v["last_state.pci.driver"], true) if err != nil { return err } } return nil } // deviceNumStringToUint32 converts a device number string (major:minor) into separare major and // minor uint32s. func (d *gpuPhysical) deviceNumStringToUint32(devNum string) (uint32, uint32, error) { devParts := strings.SplitN(devNum, ":", 2) tmp, err := strconv.ParseUint(devParts[0], 10, 32) if err != nil { return 0, 0, err } major := uint32(tmp) tmp, err = strconv.ParseUint(devParts[1], 10, 32) if err != nil { return 0, 0, err } minor := uint32(tmp) return major, minor, nil } // getNvidiaNonCardDevices returns device information about Nvidia non-card devices. func (d *gpuPhysical) getNvidiaNonCardDevices() ([]nvidiaNonCardDevice, error) { nvidiaEnts, err := os.ReadDir("/dev") if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil, err } } regexNvidiaCard, err := regexp.Compile(`^nvidia[0-9]+`) if err != nil { return nil, err } nvidiaDevices := []nvidiaNonCardDevice{} for _, nvidiaEnt := range nvidiaEnts { if !strings.HasPrefix(nvidiaEnt.Name(), "nvidia") { continue } // Skip the nvidia directories for now (require extra MIG support). if nvidiaEnt.IsDir() { continue } if regexNvidiaCard.MatchString(nvidiaEnt.Name()) { continue } nvidiaPath := filepath.Join("/dev", nvidiaEnt.Name()) stat := unix.Stat_t{} err = unix.Stat(nvidiaPath, &stat) if err != nil { continue } tmpNividiaGpu := nvidiaNonCardDevice{ path: nvidiaPath, major: unix.Major(uint64(stat.Rdev)), minor: unix.Minor(uint64(stat.Rdev)), } nvidiaDevices = append(nvidiaDevices, tmpNividiaGpu) } return nvidiaDevices, nil } incus-7.0.0/internal/server/device/gpu_sriov.go000066400000000000000000000244771517523235500216120ustar00rootroot00000000000000package device import ( "errors" "fmt" "slices" "sync" "github.com/lxc/incus/v7/internal/linux" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" pcidev "github.com/lxc/incus/v7/internal/server/device/pci" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" ) // sriovMu is used to lock concurrent GPU allocations. var sriovMu sync.Mutex type gpuSRIOV struct { deviceCommon } // validateConfig checks the supplied config for correctness. func (d *gpuSRIOV) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.VM) { return ErrUnsupportedDevType } requiredFields := []string{} optionalFields := []string{ // gendoc:generate(entity=devices, group=gpu_sriov, key=vendorid) // // --- // type: string // required: no // shortdesc: The vendor ID of the parent GPU device "vendorid", // gendoc:generate(entity=devices, group=gpu_sriov, key=productid) // // --- // type: string // required: no // shortdesc: The product ID of the parent GPU device "productid", // gendoc:generate(entity=devices, group=gpu_sriov, key=id) // // --- // type: string // required: no // shortdesc: The DRM card ID of the parent GPU device "id", // gendoc:generate(entity=devices, group=gpu_sriov, key=pci) // // --- // type: string // required: no // shortdesc: The PCI address of the parent GPU device "pci", } err := d.config.Validate(gpuValidationRules(requiredFields, optionalFields)) if err != nil { return err } if d.config["pci"] != "" { for _, field := range []string{"id", "productid", "vendorid"} { if d.config[field] != "" { return fmt.Errorf(`Cannot use %q when "pci" is set`, field) } } d.config["pci"] = pcidev.NormaliseAddress(d.config["pci"]) } if d.config["id"] != "" { for _, field := range []string{"pci", "productid", "vendorid"} { if d.config[field] != "" { return fmt.Errorf(`Cannot use %q when "id" is set`, field) } } } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *gpuSRIOV) validateEnvironment() error { if d.inst.Type() == instancetype.VM && util.IsTrue(d.inst.ExpandedConfig()["migration.stateful"]) { return errors.New("GPU devices cannot be used when migration.stateful is enabled") } return validatePCIDevice(d.config["pci"]) } // Start is run when the device is added to the instance. func (d *gpuSRIOV) Start() (*deviceConfig.RunConfig, error) { err := d.validateEnvironment() if err != nil { return nil, err } runConf := deviceConfig.RunConfig{} saveData := make(map[string]string) // Make sure that vfio-pci is loaded. err = linux.LoadModule("vfio-pci") if err != nil { return nil, fmt.Errorf("Error loading %q module: %w", "vfio-pci", err) } // Get global SR-IOV lock to prevent concurrent allocations of the VF. sriovMu.Lock() defer sriovMu.Unlock() // Get SRIOV VF. parentPCIAddress, vfID, err := d.getVF() if err != nil { return nil, err } vfPCIDev, err := d.setupSriovParent(parentPCIAddress, vfID, saveData) if err != nil { return nil, err } err = d.volatileSet(saveData) if err != nil { return nil, err } runConf.GPUDevice = append(runConf.GPUDevice, []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "pciSlotName", Value: vfPCIDev.SlotName}, }...) return &runConf, nil } // getVF returns the parent PCI address and VF id for a matching GPU. func (d *gpuSRIOV) getVF() (string, int, error) { // List all the GPUs. gpus, err := resources.GetGPU() if err != nil { return "", -1, err } // If NUMA restricted, build up a list of nodes. numaNodeSet, numaNodeSetFallback, err := getNumaNodeSet(d.inst.ExpandedConfig()) if err != nil { return "", -1, err } // Locate a suitable VF from the least loaded suitable card. var pciAddress string var vfID int var cardTotal int var cardAvailable int cardNUMA := -1 for _, gpu := range gpus.Cards { // Skip any cards that are not selected. if !gpuSelected(d.Config(), gpu) { continue } // Skip any card without SR-IOV. if gpu.SRIOV == nil { continue } // Find available VFs. vfs := []int{} for id, vf := range gpu.SRIOV.VFs { if vf.Driver == "" { vfs = append(vfs, id) } } // Skip if no available VFs. if len(vfs) == 0 { continue } // Handle NUMA. if numaNodeSet != nil { // Switch to current card if it matches our main NUMA node and existing card doesn't. if !slices.Contains(numaNodeSet, int64(cardNUMA)) && slices.Contains(numaNodeSet, int64(gpu.NUMANode)) { pciAddress = gpu.PCIAddress vfID = vfs[0] cardAvailable = len(vfs) cardTotal = int(gpu.SRIOV.CurrentVFs) cardNUMA = int(gpu.NUMANode) continue } // Skip current card if we already have a card matching our main NUMA node and this card doesn't. if slices.Contains(numaNodeSet, int64(cardNUMA)) && !slices.Contains(numaNodeSet, int64(gpu.NUMANode)) { continue } // Switch to current card if it matches a fallback NUMA node and existing card doesn't. if !slices.Contains(numaNodeSetFallback, int64(cardNUMA)) && slices.Contains(numaNodeSetFallback, int64(gpu.NUMANode)) { pciAddress = gpu.PCIAddress vfID = vfs[0] cardAvailable = len(vfs) cardTotal = int(gpu.SRIOV.CurrentVFs) cardNUMA = int(gpu.NUMANode) continue } // Skip current card if we already have a card matching a fallback NUMA node and this card isn't on the main or fallback node. if slices.Contains(numaNodeSetFallback, int64(cardNUMA)) && !slices.Contains(numaNodeSetFallback, int64(gpu.NUMANode)) && !slices.Contains(numaNodeSet, int64(gpu.NUMANode)) { continue } } // Prioritize less busy cards. if pciAddress == "" || (float64(len(vfs))/float64(gpu.SRIOV.CurrentVFs)) > (float64(cardAvailable)/float64(cardTotal)) { pciAddress = gpu.PCIAddress vfID = vfs[0] cardAvailable = len(vfs) cardTotal = int(gpu.SRIOV.CurrentVFs) cardNUMA = int(gpu.NUMANode) continue } } // Check if any physical GPU was found to match. if pciAddress == "" { return "", -1, errors.New("Couldn't find a matching GPU with available VFs") } return pciAddress, vfID, nil } // setupSriovParent configures a SR-IOV virtual function (VF) device on parent and stores original properties of // the physical device into voltatile for restoration on detach. Returns VF PCI device info. func (d *gpuSRIOV) setupSriovParent(parentPCIAddress string, vfID int, volatile map[string]string) (pcidev.Device, error) { reverter := revert.New() defer reverter.Fail() volatile["last_state.pci.parent"] = parentPCIAddress volatile["last_state.vf.id"] = fmt.Sprintf("%d", vfID) volatile["last_state.created"] = "false" // Indicates don't delete device at stop time. // Get VF device's PCI Slot Name so we can unbind and rebind it from the host. vfPCIDev, err := d.getVFDevicePCISlot(parentPCIAddress, volatile["last_state.vf.id"]) if err != nil { return vfPCIDev, err } // Unbind VF device from the host so that the settings will take effect when we rebind it. err = pcidev.DeviceUnbind(vfPCIDev) if err != nil { return vfPCIDev, err } reverter.Add(func() { _ = pcidev.DeviceProbe(vfPCIDev) }) // Register VF device with vfio-pci driver so it can be passed to VM. err = pcidev.DeviceDriverOverride(vfPCIDev, "vfio-pci") if err != nil { return vfPCIDev, err } // Record original driver used by VF device for restore. volatile["last_state.pci.driver"] = vfPCIDev.Driver reverter.Success() return vfPCIDev, nil } // getVFDevicePCISlot returns the PCI slot name for a PCI virtual function device. func (d *gpuSRIOV) getVFDevicePCISlot(parentPCIAddress string, vfID string) (pcidev.Device, error) { ueventFile := fmt.Sprintf("/sys/bus/pci/devices/%s/virtfn%s/uevent", parentPCIAddress, vfID) pciDev, err := pcidev.ParseUeventFile(ueventFile) if err != nil { return pciDev, err } return pciDev, nil } // Stop is run when the device is removed from the instance. func (d *gpuSRIOV) Stop() (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *gpuSRIOV) postStop() error { defer func() { _ = d.volatileSet(map[string]string{ "last_state.created": "", "last_state.vf.id": "", "last_state.pci.driver": "", "last_state.pci.parent": "", }) }() v := d.volatileGet() err := d.restoreSriovParent(v) if err != nil { return err } return nil } // restoreSriovParent restores SR-IOV parent device settings when removed from an instance using the // volatile data that was stored when the device was first added with setupSriovParent(). func (d *gpuSRIOV) restoreSriovParent(volatile map[string]string) error { // Nothing to do if we don't know the original device name or the VF ID. if volatile["last_state.pci.parent"] == "" || volatile["last_state.vf.id"] == "" || (d.config["pci"] == "" && d.config["id"] == "" && d.config["vendorid"] == "" && d.config["productid"] == "") { return nil } reverter := revert.New() defer reverter.Fail() // Get VF device's PCI info so we can unbind and rebind it from the host. vfPCIDev, err := d.getVFDevicePCISlot(volatile["last_state.pci.parent"], volatile["last_state.vf.id"]) if err != nil { return err } // Unbind VF device from the host so that the restored settings will take effect when we rebind it. err = pcidev.DeviceUnbind(vfPCIDev) if err != nil { return err } if d.inst.Type() == instancetype.VM { // Before we bind the device back to the host, ensure we restore the original driver info as it // should be currently set to vfio-pci. err = pcidev.DeviceSetDriverOverride(vfPCIDev, volatile["last_state.pci.driver"]) if err != nil { return err } } // However we return from this function, we must try to rebind the VF so its not orphaned. // The OS won't let an already bound device be bound again so is safe to call twice. reverter.Add(func() { _ = pcidev.DeviceProbe(vfPCIDev) }) // Bind VF device onto the host so that the settings will take effect. err = pcidev.DeviceProbe(vfPCIDev) if err != nil { return err } reverter.Success() return nil } incus-7.0.0/internal/server/device/infiniband_physical.go000066400000000000000000000174211517523235500235610ustar00rootroot00000000000000package device import ( "errors" "fmt" "strconv" "github.com/lxc/incus/v7/internal/linux" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" pcidev "github.com/lxc/incus/v7/internal/server/device/pci" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/util" ) type infinibandPhysical struct { deviceCommon } // validateConfig checks the supplied config for correctness. func (d *infinibandPhysical) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { requiredFields := []string{ // gendoc:generate(entity=devices, group=infiniband, key=parent) // // --- // type: string // required: no // defaultdesc: kernel assigned // shortdesc: The name of the interface inside the instance "parent", } optionalFields := []string{ // gendoc:generate(entity=devices, group=infiniband, key=name) // // --- // type: string // required: no // defaultdesc: kernel assigned // shortdesc: The name of the interface inside the instance "name", // gendoc:generate(entity=devices, group=infiniband, key=mtu) // // --- // type: integer // required: no // defaultdesc: parent MTU // shortdesc: The MTU of the new interface "mtu", // gendoc:generate(entity=devices, group=infiniband, key=hwaddr) // // --- // type: string // required: no // defaultdesc: randomly assigned // shortdesc: The MAC address of the new interface (can be either the full 20-byte variant or the short 8-byte variant, which will only modify the last 8 bytes of the parent device) "hwaddr", } rules := nicValidationRules(requiredFields, optionalFields, instConf) rules["hwaddr"] = func(value string) error { if value == "" { return nil } return infinibandValidMAC(value) } err := d.config.Validate(rules) if err != nil { return err } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *infinibandPhysical) validateEnvironment() error { if d.inst.Type() == instancetype.Container && d.config["name"] == "" { return errors.New("Requires name property to start") } if !util.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["parent"])) { return fmt.Errorf("Parent device '%s' doesn't exist", d.config["parent"]) } return nil } // Start is run when the device is added to a running instance or instance is starting up. func (d *infinibandPhysical) Start() (*deviceConfig.RunConfig, error) { err := d.validateEnvironment() if err != nil { return nil, err } saveData := make(map[string]string) // pciIOMMUGroup, used for VM physical passthrough. var pciIOMMUGroup uint64 // If VM, then try and load the vfio-pci module first. if d.inst.Type() == instancetype.VM { err = linux.LoadModule("vfio-pci") if err != nil { return nil, fmt.Errorf("Error loading %q module: %w", "vfio-pci", err) } } runConf := deviceConfig.RunConfig{} // Load network interface info. nics, err := resources.GetNetwork() if err != nil { return nil, err } // Filter the network interfaces to just infiniband devices related to parent. ibDevs := infinibandDevices(nics, d.config["parent"]) ibDev, found := ibDevs[d.config["parent"]] if !found { return nil, fmt.Errorf("Specified infiniband device \"%s\" not found", d.config["parent"]) } saveData["host_name"] = ibDev.ID if d.inst.Type() == instancetype.Container { // Record hwaddr and mtu before potentially modifying them. err = networkSnapshotPhysicalNIC(saveData["host_name"], saveData) if err != nil { return nil, err } // Set the MAC address. if d.config["hwaddr"] != "" { err := infinibandSetDevMAC(saveData["host_name"], d.config["hwaddr"]) if err != nil { return nil, fmt.Errorf("Failed to set the MAC address: %s", err) } } // Set the MTU. if d.config["mtu"] != "" { mtu, err := strconv.ParseUint(d.config["mtu"], 10, 32) if err != nil { return nil, fmt.Errorf("Invalid MTU specified %q: %w", d.config["mtu"], err) } link := &ip.Link{Name: saveData["host_name"]} err = link.SetMTU(uint32(mtu)) if err != nil { return nil, fmt.Errorf("Failed setting MTU %q on %q: %w", d.config["mtu"], saveData["host_name"], err) } } // Configure runConf with infiniband setup instructions. err = infinibandAddDevices(d.state, d.inst.DevicesPath(), d.name, ibDev, &runConf) if err != nil { return nil, err } } else if d.inst.Type() == instancetype.VM { // Get PCI information about the network interface. ueventPath := fmt.Sprintf("/sys/class/net/%s/device/uevent", saveData["host_name"]) pciDev, err := pcidev.ParseUeventFile(ueventPath) if err != nil { return nil, fmt.Errorf("Failed to get PCI device info for %q: %w", saveData["host_name"], err) } saveData["last_state.pci.slot.name"] = pciDev.SlotName saveData["last_state.pci.driver"] = pciDev.Driver err = pcidev.DeviceDriverOverride(pciDev, "vfio-pci") if err != nil { return nil, err } pciIOMMUGroup, err = pcidev.DeviceIOMMUGroup(saveData["last_state.pci.slot.name"]) if err != nil { return nil, err } // Record original driver used by device for restore. saveData["last_state.pci.driver"] = pciDev.Driver } err = d.volatileSet(saveData) if err != nil { return nil, err } runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "type", Value: "phys"}, {Key: "name", Value: d.config["name"]}, {Key: "flags", Value: "up"}, {Key: "link", Value: saveData["host_name"]}, } if d.inst.Type() == instancetype.VM { runConf.NetworkInterface = append(runConf.NetworkInterface, []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "pciSlotName", Value: saveData["last_state.pci.slot.name"]}, {Key: "pciIOMMUGroup", Value: fmt.Sprintf("%d", pciIOMMUGroup)}, }...) } return &runConf, nil } // Stop is run when the device is removed from the instance. func (d *infinibandPhysical) Stop() (*deviceConfig.RunConfig, error) { v := d.volatileGet() runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, NetworkInterface: []deviceConfig.RunConfigItem{ {Key: "link", Value: v["host_name"]}, }, } if d.inst.Type() == instancetype.Container { err := unixDeviceRemove(d.inst.DevicesPath(), IBDevPrefix, d.name, "", &runConf) if err != nil { return nil, err } } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *infinibandPhysical) postStop() error { defer func() { _ = d.volatileSet(map[string]string{ "host_name": "", "last_state.hwaddr": "", "last_state.mtu": "", "last_state.pci.slot.name": "", "last_state.pci.driver": "", }) }() v := d.volatileGet() // If VM physical pass through, unbind from vfio-pci and bind back to host driver. if d.inst.Type() == instancetype.VM && v["last_state.pci.slot.name"] != "" { vfioDev := pcidev.Device{ Driver: "vfio-pci", SlotName: v["last_state.pci.slot.name"], } // Unbind device from the host so that the restored settings will take effect when we rebind it. err := pcidev.DeviceUnbind(vfioDev) if err != nil { return err } err = pcidev.DeviceDriverOverride(vfioDev, v["last_state.pci.driver"]) if err != nil { return err } } else if d.inst.Type() == instancetype.Container { // Remove infiniband host files for this device. err := unixDeviceDeleteFiles(d.state, d.inst.DevicesPath(), IBDevPrefix, d.name, "") if err != nil { return fmt.Errorf("Failed to delete files for device '%s': %w", d.name, err) } } // Restore hwaddr and mtu. if v["host_name"] != "" { err := networkRestorePhysicalNIC(v["host_name"], v) if err != nil { return err } } return nil } incus-7.0.0/internal/server/device/infiniband_sriov.go000066400000000000000000000250041517523235500231030ustar00rootroot00000000000000package device import ( "errors" "fmt" "os" "path/filepath" "strconv" "strings" "github.com/lxc/incus/v7/internal/linux" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" pcidev "github.com/lxc/incus/v7/internal/server/device/pci" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" ) type infinibandSRIOV struct { deviceCommon } // validateConfig checks the supplied config for correctness. func (d *infinibandSRIOV) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { requiredFields := []string{"parent"} optionalFields := []string{ "name", "mtu", "hwaddr", } rules := nicValidationRules(requiredFields, optionalFields, instConf) rules["hwaddr"] = func(value string) error { if value == "" { return nil } return infinibandValidMAC(value) } err := d.config.Validate(rules) if err != nil { return err } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *infinibandSRIOV) validateEnvironment() error { if d.inst.Type() == instancetype.Container && d.config["name"] == "" { return errors.New("Requires name property to start") } if !util.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["parent"])) { return fmt.Errorf("Parent device '%s' doesn't exist", d.config["parent"]) } return nil } func (d *infinibandSRIOV) startContainer() (*deviceConfig.RunConfig, error) { saveData := make(map[string]string) // Load network interface info. nics, err := resources.GetNetwork() if err != nil { return nil, err } // Filter the network interfaces to just infiniband devices related to parent. ibDevs := infinibandDevices(nics, d.config["parent"]) // We don't count the parent as an available VF. delete(ibDevs, d.config["parent"]) // Load any interfaces already allocated to other devices. reservedDevices, err := network.SRIOVGetHostDevicesInUse(d.state) if err != nil { return nil, err } // Remove reserved devices from available list. for k := range reservedDevices { delete(ibDevs, k) } if len(ibDevs) < 1 { return nil, errors.New("All virtual functions on parent device are already in use") } // Get first VF device that is free. var vfDev *api.ResourcesNetworkCardPort for _, v := range ibDevs { vfDev = v break } saveData["host_name"] = vfDev.ID // Record hwaddr and mtu before potentially modifying them. err = networkSnapshotPhysicalNIC(saveData["host_name"], saveData) if err != nil { return nil, err } // Set the MAC address. if d.config["hwaddr"] != "" { err := infinibandSetDevMAC(saveData["host_name"], d.config["hwaddr"]) if err != nil { return nil, fmt.Errorf("Failed to set the MAC address: %s", err) } } // Set the MTU. if d.config["mtu"] != "" { mtu, err := strconv.ParseUint(d.config["mtu"], 10, 32) if err != nil { return nil, fmt.Errorf("Invalid MTU specified %q: %w", d.config["mtu"], err) } link := &ip.Link{Name: saveData["host_name"]} err = link.SetMTU(uint32(mtu)) if err != nil { return nil, fmt.Errorf("Failed setting MTU %q on %q: %w", d.config["mtu"], saveData["host_name"], err) } } runConf := deviceConfig.RunConfig{} // Configure runConf with infiniband setup instructions. err = infinibandAddDevices(d.state, d.inst.DevicesPath(), d.name, vfDev, &runConf) if err != nil { return nil, err } err = d.volatileSet(saveData) if err != nil { return nil, err } runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "type", Value: "phys"}, {Key: "name", Value: d.config["name"]}, {Key: "flags", Value: "up"}, {Key: "link", Value: saveData["host_name"]}, } return &runConf, nil } func (d *infinibandSRIOV) startVM() (*deviceConfig.RunConfig, error) { saveData := make(map[string]string) err := linux.LoadModule("vfio-pci") if err != nil { return nil, fmt.Errorf("Error loading %q module: %w", "vfio-pci", err) } // Load network interface info. nics, err := resources.GetNetwork() if err != nil { return nil, err } var parentPCIAddress string for _, card := range nics.Cards { found := false for _, port := range card.Ports { if port.ID == d.config["parent"] { found = true break } } if !found { continue } parentPCIAddress = card.PCIAddress break } // Get PCI information about the GPU device. devicePath := filepath.Join("/sys/bus/pci/devices", parentPCIAddress) pciParentDev, err := pcidev.ParseUeventFile(filepath.Join(devicePath, "uevent")) if err != nil { return nil, fmt.Errorf("Failed to get PCI device info for %q: %w", parentPCIAddress, err) } vfID, err := d.findFreeVirtualFunction(pciParentDev) if err != nil { return nil, fmt.Errorf("Failed to find free virtual function: %w", err) } if vfID == -1 { return nil, errors.New("All virtual functions on parent device are already in use") } vfPCIDev, err := d.setupSriovParent(parentPCIAddress, vfID, saveData) if err != nil { return nil, err } pciIOMMUGroup, err := pcidev.DeviceIOMMUGroup(vfPCIDev.SlotName) if err != nil { return nil, err } err = d.volatileSet(saveData) if err != nil { return nil, err } runConf := deviceConfig.RunConfig{} runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "type", Value: "phys"}, {Key: "name", Value: d.config["name"]}, {Key: "flags", Value: "up"}, } runConf.NetworkInterface = append(runConf.NetworkInterface, []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "pciSlotName", Value: vfPCIDev.SlotName}, {Key: "pciIOMMUGroup", Value: fmt.Sprintf("%d", pciIOMMUGroup)}, }...) return &runConf, nil } // Start is run when the device is added to a running instance or instance is starting up. func (d *infinibandSRIOV) Start() (*deviceConfig.RunConfig, error) { err := d.validateEnvironment() if err != nil { return nil, err } if d.inst.Type() == instancetype.VM { return d.startVM() } return d.startContainer() } // Stop is run when the device is removed from the instance. func (d *infinibandSRIOV) Stop() (*deviceConfig.RunConfig, error) { v := d.volatileGet() runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, NetworkInterface: []deviceConfig.RunConfigItem{{Key: "link", Value: v["host_name"]}}, } if d.inst.Type() == instancetype.Container { err := unixDeviceRemove(d.inst.DevicesPath(), IBDevPrefix, d.name, "", &runConf) if err != nil { return nil, err } } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *infinibandSRIOV) postStop() error { defer func() { _ = d.volatileSet(map[string]string{ "host_name": "", "last_state.hwaddr": "", "last_state.mtu": "", "last_state.pci.slot.name": "", "last_state.pci.driver": "", "last_state.pci.parent": "", }) }() if d.inst.Type() == instancetype.Container { // Remove infiniband host files for this device. err := unixDeviceDeleteFiles(d.state, d.inst.DevicesPath(), IBDevPrefix, d.name, "") if err != nil { return fmt.Errorf("Failed to delete files for device '%s': %w", d.name, err) } } // Restore hwaddr and mtu. v := d.volatileGet() if v["host_name"] != "" { err := networkRestorePhysicalNIC(v["host_name"], v) if err != nil { return err } } // Unbind from vfio-pci and bind back to host driver. if d.inst.Type() == instancetype.VM && v["last_state.pci.slot.name"] != "" { pciDev := pcidev.Device{ Driver: "vfio-pci", SlotName: v["last_state.pci.slot.name"], } // Unbind VF device from the host so that the restored settings will take effect when we rebind it. err := pcidev.DeviceUnbind(pciDev) if err != nil { return err } err = pcidev.DeviceDriverOverride(pciDev, v["last_state.pci.driver"]) if err != nil { return err } } return nil } // setupSriovParent configures a SR-IOV virtual function (VF) device on parent and stores original properties of // the physical device into voltatile for restoration on detach. Returns VF PCI device info. func (d *infinibandSRIOV) setupSriovParent(parentPCIAddress string, vfID int, volatile map[string]string) (pcidev.Device, error) { reverter := revert.New() defer reverter.Fail() volatile["last_state.pci.parent"] = parentPCIAddress volatile["last_state.vf.id"] = fmt.Sprintf("%d", vfID) volatile["last_state.created"] = "false" // Indicates don't delete device at stop time. // Get VF device's PCI Slot Name so we can unbind and rebind it from the host. vfPCIDev, err := d.getVFDevicePCISlot(parentPCIAddress, volatile["last_state.vf.id"]) if err != nil { return vfPCIDev, err } // Unbind VF device from the host so that the settings will take effect when we rebind it. err = pcidev.DeviceUnbind(vfPCIDev) if err != nil { return vfPCIDev, err } reverter.Add(func() { _ = pcidev.DeviceProbe(vfPCIDev) }) // Register VF device with vfio-pci driver so it can be passed to VM. err = pcidev.DeviceDriverOverride(vfPCIDev, "vfio-pci") if err != nil { return vfPCIDev, err } // Record original driver used by VF device for restore. volatile["last_state.pci.driver"] = vfPCIDev.Driver reverter.Success() return vfPCIDev, nil } // getVFDevicePCISlot returns the PCI slot name for a PCI virtual function device. func (d *infinibandSRIOV) getVFDevicePCISlot(parentPCIAddress string, vfID string) (pcidev.Device, error) { ueventFile := fmt.Sprintf("/sys/bus/pci/devices/%s/virtfn%s/uevent", parentPCIAddress, vfID) pciDev, err := pcidev.ParseUeventFile(ueventFile) if err != nil { return pciDev, err } return pciDev, nil } func (d *infinibandSRIOV) findFreeVirtualFunction(parentDev pcidev.Device) (int, error) { // Get number of currently enabled VFs. sriovNumVFs := fmt.Sprintf("/sys/bus/pci/devices/%s/sriov_numvfs", parentDev.SlotName) sriovNumVfsBuf, err := os.ReadFile(sriovNumVFs) if err != nil { return 0, err } sriovNumVfsStr := strings.TrimSpace(string(sriovNumVfsBuf)) sriovNum, err := strconv.Atoi(sriovNumVfsStr) if err != nil { return 0, err } vfID := -1 for i := range sriovNum { pciDev, err := pcidev.ParseUeventFile(fmt.Sprintf("/sys/bus/pci/devices/%s/virtfn%d/uevent", parentDev.SlotName, i)) if err != nil { return 0, err } // We assume the virtual function is free if there's no driver bound to it. if pciDev.Driver == "" { vfID = i break } } return vfID, nil } incus-7.0.0/internal/server/device/nic.go000066400000000000000000000156271517523235500203430ustar00rootroot00000000000000package device import ( "errors" "fmt" "slices" "strings" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/network/acl" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // nicValidationRules returns config validation rules for nic devices. func nicValidationRules(requiredFields []string, optionalFields []string, instConf instance.ConfigReader) map[string]func(value string) error { // Define a set of default validators for each field name. defaultValidators := map[string]func(value string) error{ "acceleration": validate.Optional(validate.IsOneOf("none", "sriov", "vdpa")), "name": validate.Optional(validate.IsInterfaceName, func(_ string) error { return nicCheckNamesUnique(instConf) }), "parent": validate.IsAny, "network": validate.IsAny, "mtu": validate.Optional(validate.IsNetworkMTU), "vlan": validate.IsNetworkVLAN, "vlan.tagged": validate.IsAny, "gvrp": validate.Optional(validate.IsBool), "hwaddr": validate.IsNetworkMAC, "host_name": validate.IsAny, "limits.ingress": validate.IsAny, "limits.egress": validate.IsAny, "limits.max": validate.IsAny, "limits.priority": validate.Optional(validate.IsUint32), "security.mac_filtering": validate.IsAny, "security.trusted": validate.Optional(validate.IsBool), "security.ipv4_filtering": validate.IsAny, "security.ipv6_filtering": validate.IsAny, "security.port_isolation": validate.Optional(validate.IsBool), "ipv4.address": validate.Optional(validate.IsNetworkAddressV4), "ipv6.address": validate.Optional(validate.IsNetworkAddressV6), "ipv4.routes": validate.Optional(validate.IsListOf(validate.IsNetworkV4)), "ipv6.routes": validate.Optional(validate.IsListOf(validate.IsNetworkV6)), "boot.priority": validate.Optional(validate.IsUint32), "ipv4.gateway": networkValidGateway, "ipv6.gateway": networkValidGateway, "ipv4.host_address": validate.Optional(validate.IsNetworkAddressV4), "ipv6.host_address": validate.Optional(validate.IsNetworkAddressV6), "ipv4.host_table": validate.Optional(validate.IsUint32), "ipv6.host_table": validate.Optional(validate.IsUint32), "queue.tx.length": validate.Optional(validate.IsUint32), "ipv4.routes.external": validate.Optional(validate.IsListOf(validate.IsNetworkV4)), "ipv6.routes.external": validate.Optional(validate.IsListOf(validate.IsNetworkV6)), "nested": validate.IsAny, "security.acls": validate.IsAny, "security.acls.default.ingress.action": validate.Optional(validate.IsOneOf(acl.ValidActions...)), "security.acls.default.egress.action": validate.Optional(validate.IsOneOf(acl.ValidActions...)), "security.acls.default.ingress.logged": validate.Optional(validate.IsBool), "security.acls.default.egress.logged": validate.Optional(validate.IsBool), "security.promiscuous": validate.Optional(validate.IsBool), "mode": validate.Optional(validate.IsOneOf("bridge", "vepa", "passthru", "private")), "io.bus": validate.Optional(func(value string) error { return nicCheckIOBus(instConf, value) }), "vendorid": validate.Optional(validate.IsDeviceID), "productid": validate.Optional(validate.IsDeviceID), "pci": validate.IsPCIAddress, "attached": validate.Optional(validate.IsBool), "connected": validate.Optional(validate.IsBool), } validators := map[string]func(value string) error{} for _, k := range optionalFields { defaultValidator := defaultValidators[k] // If field doesn't have a known validator, it is an unknown field, skip. if defaultValidator == nil { continue } // Wrap the default validator in an empty check as field is optional. validators[k] = func(value string) error { if value == "" { return nil } return defaultValidator(value) } } // Add required fields last, that way if they are specified in both required and optional // field sets, the required one will overwrite the optional validators. for _, k := range requiredFields { defaultValidator := defaultValidators[k] // If field doesn't have a known validator, it is an unknown field, skip. if defaultValidator == nil { continue } // Wrap the default validator in a not empty check as field is required. validators[k] = func(value string) error { err := validate.IsNotEmpty(value) if err != nil { return err } return defaultValidator(value) } } return validators } // nicHasAutoGateway takes the value of the "ipv4.gateway" or "ipv6.gateway" config keys and returns whether they // specify whether the gateway mode is automatic or not. func nicHasAutoGateway(value string) bool { if value == "" || value == "auto" { return true } return false } // nicCheckNamesUnique checks that all the NICs in the instConf's expanded devices have a unique (or unset) name. func nicCheckNamesUnique(instConf instance.ConfigReader) error { seenNICNames := []string{} for _, devConfig := range instConf.ExpandedDevices() { if devConfig["type"] != "nic" || devConfig["name"] == "" { continue } if slices.Contains(seenNICNames, devConfig["name"]) { return fmt.Errorf("Duplicate NIC name detected %q", devConfig["name"]) } seenNICNames = append(seenNICNames, devConfig["name"]) } return nil } // nicCheckDNSNameConflict returns if instNameA matches instNameB (case insensitive). func nicCheckDNSNameConflict(instNameA string, instNameB string) bool { return strings.EqualFold(instNameA, instNameB) } // nicCheckIOBus validates the io.bus value. func nicCheckIOBus(instConf instance.ConfigReader, value string) error { if instConf.Type() != instancetype.VM { return errors.New("This option is only supported on virtual machines") } err := validate.IsOneOf("virtio", "usb")(value) if err != nil { return err } if value == "usb" && util.IsTrue(instConf.ExpandedConfig()["migration.stateful"]) { return errors.New("USB devices cannot be used when migration.stateful is enabled") } return nil } incus-7.0.0/internal/server/device/nic_bridged.go000066400000000000000000002137661517523235500220270ustar00rootroot00000000000000package device import ( "bufio" "bytes" "context" "encoding/binary" "encoding/hex" "errors" "fmt" "io/fs" "math/rand" "net" "net/http" "os" "slices" "strconv" "strings" "github.com/google/gopacket" "github.com/google/gopacket/layers" "github.com/mdlayher/netx/eui64" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/dnsmasq" "github.com/lxc/incus/v7/internal/server/dnsmasq/dhcpalloc" firewallDrivers "github.com/lxc/incus/v7/internal/server/firewall/drivers" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/network/acl" addressSet "github.com/lxc/incus/v7/internal/server/network/address-set" "github.com/lxc/incus/v7/internal/server/project" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) type bridgeNetwork interface { UsesDNSMasq() bool } type nicBridged struct { deviceCommon network network.Network // Populated in validateConfig(). } // CanHotPlug returns whether the device can be managed whilst the instance is running. Returns true. func (d *nicBridged) CanHotPlug() bool { return true } // CanMigrate returns whether the device can be migrated to any other cluster member. func (d *nicBridged) CanMigrate() bool { return d.network != nil } // validateConfig checks the supplied config for correctness. func (d *nicBridged) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container, instancetype.VM) { return ErrUnsupportedDevType } var requiredFields []string optionalFields := []string{ // gendoc:generate(entity=devices, group=nic_bridged, key=name) // // --- // type: string // default: kernel assigned // managed: no // shortdesc: The name of the interface inside the instance "name", // gendoc:generate(entity=devices, group=nic_bridged, key=network) // // --- // type: string // managed: no // shortdesc: The managed network to link the device to (instead of specifying the `nictype` directly) "network", // gendoc:generate(entity=devices, group=nic_bridged, key=parent) // // --- // type: string // managed: yes // shortdesc: The name of the parent host device (required if specifying the `nictype` directly) "parent", // gendoc:generate(entity=devices, group=nic_bridged, key=mtu) // // --- // type: integer // default: MTU of the parent device // managed: yes // shortdesc: The Maximum Transmit Unit (MTU) of the new interface "mtu", // gendoc:generate(entity=devices, group=nic_bridged, key=queue.tx.length) // // --- // type: integer // managed: no // shortdesc: The transmit queue length for the NIC "queue.tx.length", // gendoc:generate(entity=devices, group=nic_bridged, key=hwaddr) // // --- // type: string // default: randomly assigned // managed: no // shortdesc: The MAC address of the new interface "hwaddr", // gendoc:generate(entity=devices, group=nic_bridged, key=host_name) // // --- // type: string // default: randomly assigned // managed: no // shortdesc: The name of the interface on the host "host_name", // gendoc:generate(entity=devices, group=nic_bridged, key=limits.ingress) // // --- // type: string // managed: no // shortdesc: I/O limit in bit/s for incoming traffic (various suffixes supported, see {ref}`instances-limit-units`) "limits.ingress", // gendoc:generate(entity=devices, group=nic_bridged, key=limits.egress) // // --- // type: string // managed: no // shortdesc: I/O limit in bit/s for outgoing traffic (various suffixes supported, see {ref}`instances-limit-units`) "limits.egress", // gendoc:generate(entity=devices, group=nic_bridged, key=limits.max) // // --- // type: string // managed: no // shortdesc: I/O limit in bit/s for both incoming and outgoing traffic (same as setting both limits.ingress and limits.egress) "limits.max", // gendoc:generate(entity=devices, group=nic_bridged, key=limits.priority) // // --- // type: integer // managed: no // shortdesc: The priority for outgoing traffic, to be used by the kernel queuing discipline to prioritize network packets "limits.priority", // gendoc:generate(entity=devices, group=nic_bridged, key=ipv4.address) // // --- // type: string // managed: no // shortdesc: An IPv4 address to assign to the instance through DHCP (can be `none` to restrict all IPv4 traffic when `security.ipv4_filtering` is set) "ipv4.address", // gendoc:generate(entity=devices, group=nic_bridged, key=ipv6.address) // // --- // type: string // managed: no // shortdesc: An IPv6 address to assign to the instance through DHCP (can be `none` to restrict all IPv6 traffic when `security.ipv6_filtering` is set) "ipv6.address", // gendoc:generate(entity=devices, group=nic_bridged, key=ipv4.routes) // // --- // type: string // managed: no // shortdesc: Comma-delimited list of IPv4 static routes to add on host to NIC "ipv4.routes", // gendoc:generate(entity=devices, group=nic_bridged, key=ipv6.routes) // // --- // type: string // managed: no // shortdesc: Comma-delimited list of IPv6 static routes to add on host to NIC "ipv6.routes", // gendoc:generate(entity=devices, group=nic_bridged, key=ipv4.routes.external) // // --- // type: string // managed: no // shortdesc: Comma-delimited list of IPv4 static routes to route to the NIC and publish on uplink network (BGP) "ipv4.routes.external", // gendoc:generate(entity=devices, group=nic_bridged, key=ipv6.routes.external) // // --- // type: string // managed: no // shortdesc: Comma-delimited list of IPv6 static routes to route to the NIC and publish on uplink network (BGP) "ipv6.routes.external", // gendoc:generate(entity=devices, group=nic_bridged, key=security.mac_filtering) // // --- // type: bool // default: false // managed: no // shortdesc: Prevent the instance from spoofing another instance's MAC address "security.mac_filtering", // gendoc:generate(entity=devices, group=nic_bridged, key=security.ipv4_filtering) // // --- // type: bool // default: false // managed: no // shortdesc: Prevent the instance from spoofing another instance's IPv4 address (enables `security.mac_filtering`) "security.ipv4_filtering", // gendoc:generate(entity=devices, group=nic_bridged, key=security.ipv6_filtering) // // --- // type: bool // default: false // managed: no // shortdesc: Prevent the instance from spoofing another instance's IPv6 address (enables `security.mac_filtering`) "security.ipv6_filtering", // gendoc:generate(entity=devices, group=nic_bridged, key=security.port_isolation) // // --- // type: bool // default: false // managed: no // shortdesc: Prevent the NIC from communicating with other NICs in the network that have port isolation enabled "security.port_isolation", // gendoc:generate(entity=devices, group=nic_bridged, key=security.acls) // // --- // type: string // managed: no // shortdesc: Comma-separated list of network ACLs to apply "security.acls", // gendoc:generate(entity=devices, group=nic_bridged, key=security.acls.default.ingress.action) // // --- // type: string // default: drop // managed: no // shortdesc: Action to use for ingress traffic that doesn't match any ACL rule "security.acls.default.ingress.action", // gendoc:generate(entity=devices, group=nic_bridged, key=security.acls.default.egress.action) // // --- // type: string // default: drop // managed: no // shortdesc: Action to use for egress traffic that doesn't match any ACL rule "security.acls.default.egress.action", // gendoc:generate(entity=devices, group=nic_bridged, key=security.acls.default.ingress.logged) // // --- // type: bool // default: false // managed: no // shortdesc: Whether to log ingress traffic that doesn't match any ACL rule "security.acls.default.ingress.logged", // gendoc:generate(entity=devices, group=nic_bridged, key=security.acls.default.egress.logged) // // --- // type: bool // default: false // managed: no // shortdesc: Whether to log egress traffic that doesn't match any ACL rule "security.acls.default.egress.logged", // gendoc:generate(entity=devices, group=nic_bridged, key=boot.priority) // // --- // type: integer // managed: no // shortdesc: Boot priority for VMs (higher value boots first) "boot.priority", // gendoc:generate(entity=devices, group=nic_bridged, key=vlan) // // --- // type: integer // managed: no // shortdesc: The VLAN ID to use for non-tagged traffic (can be none to remove port from default VLAN) "vlan", // gendoc:generate(entity=devices, group=nic_bridged, key=io.bus) // // --- // type: string // default: `virtio` // managed: no // shortdesc: Override the bus for the device (can be `virtio` or `usb`) (VM only) "io.bus", // gendoc:generate(entity=devices, group=nic_bridged, key=attached) // // --- // type: bool // default: `true` // required: no // shortdesc: Whether the NIC is plugged in or not "attached", // gendoc:generate(entity=devices, group=nic_bridged, key=connected) // // --- // type: bool // default: `true` // required: no // shortdesc: Whether the NIC is connected to the host network "connected", } // checkWithManagedNetwork validates the device's settings against the managed network. checkWithManagedNetwork := func(n network.Network) error { if n.Status() != api.NetworkStatusCreated { return errors.New("Specified network is not fully created") } if n.Type() != "bridge" && (n.Type() != "physical" || !util.PathExists(fmt.Sprintf("/sys/class/net/%s/bridge", d.config["parent"]))) { return errors.New("Specified network must be of type bridge") } netConfig := n.Config() if d.config["ipv4.address"] != "" { dhcpv4Subnet := n.DHCPv4Subnet() // Check that DHCPv4 is enabled on parent network (needed to use static assigned IPs) when // IP filtering isn't enabled (if it is we allow the use of static IPs for this purpose). if dhcpv4Subnet == nil && util.IsFalseOrEmpty(d.config["security.ipv4_filtering"]) { return fmt.Errorf(`Cannot specify "ipv4.address" when DHCP is disabled (unless using security.ipv4_filtering) on network %q`, n.Name()) } // Check the static IP supplied is valid for the linked network. It should be part of the // network's subnet, but not necessarily part of the dynamic allocation ranges. if dhcpv4Subnet != nil && d.config["ipv4.address"] != "none" && !dhcpalloc.DHCPValidIP(dhcpv4Subnet, nil, net.ParseIP(d.config["ipv4.address"])) { return fmt.Errorf("Device IP address %q not within network %q subnet", d.config["ipv4.address"], n.Name()) } parentAddress := netConfig["ipv4.address"] if slices.Contains([]string{"", "none"}, parentAddress) { return nil } ip, _, err := net.ParseCIDR(parentAddress) if err != nil { return fmt.Errorf("Invalid network ipv4.address: %w", err) } if d.config["ipv4.address"] == "none" && util.IsFalseOrEmpty(d.config["security.ipv4_filtering"]) { return errors.New("Cannot have ipv4.address as none unless using security.ipv4_filtering") } // IP should not be the same as the parent managed network address. if ip.Equal(net.ParseIP(d.config["ipv4.address"])) { return fmt.Errorf("IP address %q is assigned to parent managed network device %q", d.config["ipv4.address"], d.config["parent"]) } } if d.config["ipv6.address"] != "" { dhcpv6Subnet := n.DHCPv6Subnet() // Check that DHCPv6 is enabled on parent network (needed to use static assigned IPs) when // IP filtering isn't enabled (if it is we allow the use of static IPs for this purpose). if (dhcpv6Subnet == nil || util.IsFalseOrEmpty(netConfig["ipv6.dhcp.stateful"])) && util.IsFalseOrEmpty(d.config["security.ipv6_filtering"]) { return fmt.Errorf(`Cannot specify "ipv6.address" when DHCP or "ipv6.dhcp.stateful" are disabled (unless using security.ipv6_filtering) on network %q`, n.Name()) } // Check the static IP supplied is valid for the linked network. It should be part of the // network's subnet, but not necessarily part of the dynamic allocation ranges. if dhcpv6Subnet != nil && d.config["ipv6.address"] != "none" && !dhcpalloc.DHCPValidIP(dhcpv6Subnet, nil, net.ParseIP(d.config["ipv6.address"])) { return fmt.Errorf("Device IP address %q not within network %q subnet", d.config["ipv6.address"], n.Name()) } parentAddress := netConfig["ipv6.address"] if slices.Contains([]string{"", "none"}, parentAddress) { return nil } ip, _, err := net.ParseCIDR(parentAddress) if err != nil { return fmt.Errorf("Invalid network ipv6.address: %w", err) } if d.config["ipv6.address"] == "none" && util.IsFalseOrEmpty(d.config["security.ipv6_filtering"]) { return errors.New("Cannot have ipv6.address as none unless using security.ipv6_filtering") } // IP should not be the same as the parent managed network address. if ip.Equal(net.ParseIP(d.config["ipv6.address"])) { return fmt.Errorf("IP address %q is assigned to parent managed network device %q", d.config["ipv6.address"], d.config["parent"]) } } // When we know the parent network is managed, we can validate the NIC's VLAN settings based on // on the bridge driver type. if slices.Contains([]string{"", "native"}, netConfig["bridge.driver"]) { // Check VLAN 0 isn't set when using a native Linux managed bridge, as not supported. if d.config["vlan"] == "0" { return errors.New("VLAN ID 0 is not allowed for native Linux bridges") } // Check that none of the supplied VLAN IDs are VLAN 0 when using a native Linux managed // bridge, as not supported. networkVLANList, err := networkVLANListExpand(util.SplitNTrimSpace(d.config["vlan.tagged"], ",", -1, true)) if err != nil { return err } if slices.Contains(networkVLANList, 0) { return errors.New("VLAN tagged ID 0 is not allowed for native Linux bridges") } } return nil } // Check that if network proeperty is set that conflicting keys are not present. if d.config["network"] != "" { requiredFields = append(requiredFields, "network") bannedKeys := []string{"nictype", "parent", "mtu"} for _, bannedKey := range bannedKeys { if d.config[bannedKey] != "" { return fmt.Errorf("Cannot use %q property in conjunction with %q property", bannedKey, "network") } } // Load managed network. api.ProjectDefaultName is used here as bridge networks don't support projects. var err error d.network, err = network.LoadByName(d.state, api.ProjectDefaultName, d.config["network"]) if err != nil { return fmt.Errorf("Error loading network config for %q: %w", d.config["network"], err) } // Validate NIC settings with managed network. err = checkWithManagedNetwork(d.network) if err != nil { return err } // Apply network settings to NIC. netConfig := d.network.Config() // Link device to network bridge. d.config["parent"] = d.config["network"] // Apply network level config options to device config before validation. if netConfig["bridge.mtu"] != "" { d.config["mtu"] = netConfig["bridge.mtu"] } } else { // If no network property supplied, then parent property is required. requiredFields = append(requiredFields, "parent") // Check if parent is a managed network. // api.ProjectDefaultName is used here as bridge networks don't support projects. d.network, _ = network.LoadByName(d.state, api.ProjectDefaultName, d.config["parent"]) if d.network != nil { // Validate NIC settings with managed network. err := checkWithManagedNetwork(d.network) if err != nil { return err } } else { // Check that static IPs are only specified with IP filtering when using an unmanaged // parent bridge. if util.IsTrue(d.config["security.ipv4_filtering"]) { if d.config["ipv4.address"] == "" { return errors.New("IPv4 filtering requires a manually specified ipv4.address when using an unmanaged parent bridge") } } else if d.config["ipv4.address"] != "" { // Static IP cannot be used with unmanaged parent. return errors.New("Cannot use manually specified ipv4.address when using unmanaged parent bridge") } if util.IsTrue(d.config["security.ipv6_filtering"]) { if d.config["ipv6.address"] == "" { return errors.New("IPv6 filtering requires a manually specified ipv6.address when using an unmanaged parent bridge") } } else if d.config["ipv6.address"] != "" { // Static IP cannot be used with unmanaged parent. return errors.New("Cannot use manually specified ipv6.address when using unmanaged parent bridge") } } } // Check that IP filtering isn't being used with VLAN filtering. if util.IsTrue(d.config["security.ipv4_filtering"]) || util.IsTrue(d.config["security.ipv6_filtering"]) { if d.config["vlan"] != "" || d.config["vlan.tagged"] != "" { return errors.New("IP filtering cannot be used with VLAN filtering") } } // Check there isn't another NIC with any of the same addresses specified on the same cluster member. // Can only validate this when the instance is supplied (and not doing profile validation). if d.inst != nil { err := d.checkAddressConflict() if err != nil { return err } } // Check if security ACL(s) are configured. if d.config["security.acls"] != "" { if d.state.Firewall.String() != "nftables" { return errors.New("Security ACLs are only supported when using nftables firewall") } // The NIC's network may be a non-default project, so lookup project and get network's project name. networkProjectName, _, err := project.NetworkProject(d.state.DB.Cluster, instConf.Project().Name) if err != nil { return fmt.Errorf("Failed loading network project name: %w", err) } err = acl.Exists(d.state, networkProjectName, util.SplitNTrimSpace(d.config["security.acls"], ",", -1, true)...) if err != nil { return err } } rules := nicValidationRules(requiredFields, optionalFields, instConf) // Add bridge specific vlan validation. rules["vlan"] = func(value string) error { if value == "" || value == "none" { return nil } return validate.IsNetworkVLAN(value) } // Add bridge specific vlan.tagged validation. // gendoc:generate(entity=devices, group=nic_bridged, key=vlan.tagged) // // --- // type: integer // managed: no // shortdesc: Comma-delimited list of VLAN IDs or VLAN ranges to join for tagged traffic rules["vlan.tagged"] = func(value string) error { if value == "" { return nil } // Check that none of the supplied VLAN IDs are the same as the untagged VLAN ID. for _, vlanID := range util.SplitNTrimSpace(value, ",", -1, true) { if vlanID == d.config["vlan"] { return fmt.Errorf("Tagged VLAN ID %q cannot be the same as untagged VLAN ID", vlanID) } _, _, err := validate.ParseNetworkVLANRange(vlanID) if err != nil { return err } } return nil } // Add bridge specific ipv4/ipv6 validation rules rules["ipv4.address"] = func(value string) error { if value == "" || value == "none" { return nil } return validate.IsNetworkAddressV4(value) } rules["ipv6.address"] = func(value string) error { if value == "" || value == "none" { return nil } return validate.IsNetworkAddressV6(value) } // Now run normal validation. err := d.config.Validate(rules) if err != nil { return err } return nil } // checkAddressConflict checks for conflicting IP/MAC addresses on another NIC connected to same network on the // same cluster member. Can only validate this when the instance is supplied (and not doing profile validation). // Returns api.StatusError with status code set to http.StatusConflict if conflicting address found. func (d *nicBridged) checkAddressConflict() error { node := d.inst.Location() ourNICIPs := make(map[string]net.IP, 2) ourNICIPs["ipv4.address"] = net.ParseIP(d.config["ipv4.address"]) ourNICIPs["ipv6.address"] = net.ParseIP(d.config["ipv6.address"]) ourNICMAC, _ := net.ParseMAC(d.config["hwaddr"]) if ourNICMAC == nil { ourNICMAC, _ = net.ParseMAC(d.volatileGet()["hwaddr"]) } // Check if any instance devices use this network. // Managed bridge networks have a per-server DHCP daemon so perform a node level search. filter := cluster.InstanceFilter{Node: &node} // Set network name for comparison (needs to support connecting to unmanaged networks). networkName := d.config["parent"] if d.network != nil { networkName = d.network.Name() } // Bridge networks are always in the default project. return network.UsedByInstanceDevices(d.state, api.ProjectDefaultName, networkName, "bridge", func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error { // Skip our own device. This avoids triggering duplicate device errors during // updates or when making temporary copies of our instance during migrations. sameLogicalInstance := instance.IsSameLogicalInstance(d.inst, &inst) if sameLogicalInstance && d.Name() == nicName { return nil } // Skip NICs connected to other VLANs (not perfect though as one NIC could // explicitly specify the default untagged VLAN and these would be connected to // same L2 even though the values are different, and there is a different default // value for native and openvswith parent bridges). if d.config["vlan"] != nicConfig["vlan"] { return nil } // Check there isn't another instance with the same DNS name connected to a managed network // that has DNS enabled and is connected to the same untagged VLAN. if d.network != nil && d.network.Config()["dns.mode"] != "none" && nicCheckDNSNameConflict(d.inst.Name(), inst.Name) { if sameLogicalInstance { return api.StatusErrorf(http.StatusConflict, "Instance DNS name %q conflict between %q and %q because both are connected to same network", strings.ToLower(inst.Name), d.name, nicName) } return api.StatusErrorf(http.StatusConflict, "Instance DNS name %q already used on network", strings.ToLower(inst.Name)) } // Check NIC's MAC address doesn't match this NIC's MAC address. devNICMAC, _ := net.ParseMAC(nicConfig["hwaddr"]) if devNICMAC == nil { devNICMAC, _ = net.ParseMAC(inst.Config[fmt.Sprintf("volatile.%s.hwaddr", nicName)]) } if ourNICMAC != nil && devNICMAC != nil && bytes.Equal(ourNICMAC, devNICMAC) { return api.StatusErrorf(http.StatusConflict, "MAC address %q already defined on another NIC", devNICMAC.String()) } // Check NIC's static IPs don't match this NIC's static IPs. for _, key := range []string{"ipv4.address", "ipv6.address"} { if d.config[key] == "" { continue // No static IP specified on this NIC. } // Parse IPs to avoid being tripped up by presentation differences. devNICIP := net.ParseIP(nicConfig[key]) if ourNICIPs[key] != nil && devNICIP != nil && ourNICIPs[key].Equal(devNICIP) { return api.StatusErrorf(http.StatusConflict, "IP address %q already defined on another NIC", devNICIP.String()) } } return nil }, filter) } // validateEnvironment checks the runtime environment for correctness. func (d *nicBridged) validateEnvironment() error { if d.inst.Type() == instancetype.Container && d.config["name"] == "" { return errors.New("Requires name property to start") } if !util.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["parent"])) { return fmt.Errorf("Parent device %q doesn't exist", d.config["parent"]) } return nil } // UpdatableFields returns a list of fields that can be updated without triggering a device remove & add. func (d *nicBridged) UpdatableFields(oldDevice Type) []string { // Check old and new device types match. _, match := oldDevice.(*nicBridged) if !match { return []string{} } return []string{"limits.ingress", "limits.egress", "limits.max", "limits.priority", "ipv4.routes", "ipv6.routes", "ipv4.routes.external", "ipv6.routes.external", "ipv4.address", "ipv6.address", "security.mac_filtering", "security.ipv4_filtering", "security.ipv6_filtering", "security.acls", "security.acls.default.egress.action", "security.acls.default.egress.logged", "security.acls.default.ingress.action", "security.acls.default.ingress.logged", "connected"} } // Add is run when a device is added to a non-snapshot instance whether or not the instance is running. func (d *nicBridged) Add() error { networkVethFillFromVolatile(d.config, d.volatileGet()) // Rebuild dnsmasq entry if needed and reload. err := d.rebuildDnsmasqEntry() if err != nil { return err } return nil } // PreStartCheck checks the managed parent network is available (if relevant). func (d *nicBridged) PreStartCheck() error { // Non-managed network NICs are not relevant for checking managed network availability. if d.network == nil { return nil } // If managed network is not available, don't try and start instance. if d.network.LocalStatus() == api.NetworkStatusUnavailable { return api.StatusErrorf(http.StatusServiceUnavailable, "Network %q unavailable on this server", d.network.Name()) } return nil } // Start is run when the device is added to a running instance or instance is starting up. func (d *nicBridged) Start() (*deviceConfig.RunConfig, error) { // Ignore detached NICs. if !util.IsTrueOrEmpty(d.config["attached"]) { return nil, nil } err := d.validateEnvironment() if err != nil { return nil, err } reverter := revert.New() defer reverter.Fail() saveData := make(map[string]string) saveData["host_name"] = d.config["host_name"] var peerName string var mtu uint32 // Create veth pair and configure the peer end with custom hwaddr and mtu if supplied. if d.inst.Type() == instancetype.Container { if saveData["host_name"] == "" { saveData["host_name"], err = d.generateHostName("veth", d.config["hwaddr"]) if err != nil { return nil, err } } peerName, mtu, err = networkCreateVethPair(saveData["host_name"], d.config) } else if d.inst.Type() == instancetype.VM { if saveData["host_name"] == "" { saveData["host_name"], err = d.generateHostName("tap", d.config["hwaddr"]) if err != nil { return nil, err } } peerName = saveData["host_name"] // VMs use the host_name to link to the TAP FD. mtu, err = networkCreateTap(saveData["host_name"], d.config) } if err != nil { return nil, err } reverter.Add(func() { _ = network.InterfaceRemove(saveData["host_name"]) }) // Populate device config with volatile fields if needed. networkVethFillFromVolatile(d.config, saveData) // Rebuild dnsmasq config if parent is a managed bridge network using dnsmasq and static lease file is // missing. bridgeNet, ok := d.network.(bridgeNetwork) if ok && d.network.IsManaged() && bridgeNet.UsesDNSMasq() { deviceStaticFileName := dnsmasq.DHCPStaticAllocationPath(d.network.Name(), dnsmasq.StaticAllocationFileName(d.inst.Project().Name, d.inst.Name(), d.Name())) if !util.PathExists(deviceStaticFileName) { err = d.rebuildDnsmasqEntry() if err != nil { return nil, fmt.Errorf("Failed creating DHCP static allocation: %w", err) } } } // Apply host-side routes to bridge interface. routes := []string{} routes = append(routes, util.SplitNTrimSpace(d.config["ipv4.routes"], ",", -1, true)...) routes = append(routes, util.SplitNTrimSpace(d.config["ipv6.routes"], ",", -1, true)...) routes = append(routes, util.SplitNTrimSpace(d.config["ipv4.routes.external"], ",", -1, true)...) routes = append(routes, util.SplitNTrimSpace(d.config["ipv6.routes.external"], ",", -1, true)...) err = networkNICRouteAdd(d.config["parent"], d.config["ipv4.address"], d.config["ipv6.address"], routes...) if err != nil { return nil, err } // Apply host-side limits. err = networkSetupHostVethLimits(&d.deviceCommon, nil, true) if err != nil { return nil, err } // Disable IPv6 on host-side veth interface (prevents host-side interface getting link-local address) // which isn't needed because the host-side interface is connected to a bridge. err = localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", saveData["host_name"]), "1") if err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, err } // Apply and host-side network filters (uses enriched host_name from networkVethFillFromVolatile). r, err := d.setupHostFilters(nil) if err != nil { return nil, err } reverter.Add(r) // Attach host side veth interface to bridge. err = network.AttachInterface(d.state, d.config["parent"], saveData["host_name"]) if err != nil { return nil, err } reverter.Add(func() { _ = network.DetachInterface(d.state, d.config["parent"], saveData["host_name"]) }) // Attempt to disable router advertisement acceptance. err = localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", saveData["host_name"]), "0") if err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, err } // Attempt to enable port isolation. if util.IsTrue(d.config["security.port_isolation"]) { link := &ip.Link{Name: saveData["host_name"]} err = link.BridgeLinkSetIsolated(true) if err != nil { return nil, err } } // Detect bridge type. nativeBridge := network.IsNativeBridge(d.config["parent"]) // Setup VLAN settings on bridge port. if nativeBridge { err = d.setupNativeBridgePortVLANs(saveData["host_name"]) } else { err = d.setupOVSBridgePortVLANs(saveData["host_name"]) } if err != nil { return nil, err } // Check if hairpin mode needs to be enabled. if nativeBridge && d.network != nil { brNetfilterEnabled := false for _, ipVersion := range []uint{4, 6} { if network.BridgeNetfilterEnabled(ipVersion) == nil { brNetfilterEnabled = true break } } if brNetfilterEnabled { var listenAddresses map[int64]string err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := d.network.ID() dbRecords, err := cluster.GetNetworkForwards(ctx, tx.Tx(), cluster.NetworkForwardFilter{ NetworkID: &networkID, }) if err != nil { return err } listenAddresses = make(map[int64]string) for _, dbRecord := range dbRecords { if !dbRecord.NodeID.Valid || (dbRecord.NodeID.Int64 == tx.GetNodeID()) { listenAddresses[dbRecord.ID] = dbRecord.ListenAddress } } return nil }) if err != nil { return nil, fmt.Errorf("Failed loading network forwards: %w", err) } // If br_netfilter is enabled and bridge has forwards, we enable hairpin mode on NIC's // bridge port in case any of the forwards target this NIC and the instance attempts to // connect to the forward's listener. Without hairpin mode on the target of the forward // will not be able to connect to the listener. if len(listenAddresses) > 0 { link := &ip.Link{Name: saveData["host_name"]} err = link.BridgeLinkSetHairpin(true) if err != nil { return nil, fmt.Errorf("Error enabling hairpin mode on bridge port %q: %w", link.Name, err) } d.logger.Debug("Enabled hairpin mode on NIC bridge port", logger.Ctx{"dev": link.Name}) } } } err = d.volatileSet(saveData) if err != nil { return nil, err } runConf := deviceConfig.RunConfig{} runConf.PostHooks = []func() error{d.postStart} runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "type", Value: "phys"}, {Key: "name", Value: d.config["name"]}, {Key: "flags", Value: "up"}, {Key: "link", Value: peerName}, {Key: "hwaddr", Value: d.config["hwaddr"]}, {Key: "connected", Value: d.config["connected"]}, } if d.config["io.bus"] == "usb" { runConf.UseUSBBus = true } if d.inst.Type() == instancetype.VM { runConf.NetworkInterface = append(runConf.NetworkInterface, []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "mtu", Value: fmt.Sprintf("%d", mtu)}, }...) } reverter.Success() return &runConf, nil } // postStart is run after the device is added to the instance. func (d *nicBridged) postStart() error { err := bgpAddPrefix(&d.deviceCommon, d.network, d.config) if err != nil { return err } return nil } // Update applies configuration changes to a started device. func (d *nicBridged) Update(oldDevices deviceConfig.Devices, isRunning bool) error { oldConfig := oldDevices[d.name] v := d.volatileGet() // Populate device config with volatile fields if needed. networkVethFillFromVolatile(d.config, v) networkVethFillFromVolatile(oldConfig, v) // If an IPv6 address has changed, flush all existing IPv6 leases for instance so instance // isn't allocated old IP. This is important with IPv6 because DHCPv6 supports multiple IP // address allocation and would result in instance having leases for both old and new IPs. if d.config["hwaddr"] != "" && d.config["ipv6.address"] != oldConfig["ipv6.address"] { err := d.networkClearLease(d.inst.Name(), d.config["parent"], d.config["hwaddr"], clearLeaseIPv6Only) if err != nil { return err } } reverter := revert.New() defer reverter.Fail() // If instance is running, apply host side limits and filters first before rebuilding // dnsmasq config below so that existing config can be used as part of the filter removal. if isRunning { err := d.validateEnvironment() if err != nil { return err } // Validate old config so that it is enriched with network parent config needed for route removal. err = Validate(d.inst, d.state, d.name, oldConfig, false) if err != nil { return err } // Remove old host-side routes from bridge interface. oldRoutes := []string{} oldRoutes = append(oldRoutes, util.SplitNTrimSpace(oldConfig["ipv4.routes"], ",", -1, true)...) oldRoutes = append(oldRoutes, util.SplitNTrimSpace(oldConfig["ipv6.routes"], ",", -1, true)...) oldRoutes = append(oldRoutes, util.SplitNTrimSpace(oldConfig["ipv4.routes.external"], ",", -1, true)...) oldRoutes = append(oldRoutes, util.SplitNTrimSpace(oldConfig["ipv6.routes.external"], ",", -1, true)...) networkNICRouteDelete(oldConfig["parent"], oldConfig["ipv4.address"], oldConfig["ipv6.address"], oldRoutes...) // Apply host-side routes to bridge interface. routes := []string{} routes = append(routes, util.SplitNTrimSpace(d.config["ipv4.routes"], ",", -1, true)...) routes = append(routes, util.SplitNTrimSpace(d.config["ipv6.routes"], ",", -1, true)...) routes = append(routes, util.SplitNTrimSpace(d.config["ipv4.routes.external"], ",", -1, true)...) routes = append(routes, util.SplitNTrimSpace(d.config["ipv6.routes.external"], ",", -1, true)...) err = networkNICRouteAdd(d.config["parent"], d.config["ipv4.address"], d.config["ipv6.address"], routes...) if err != nil { return err } // Apply host-side limits. err = networkSetupHostVethLimits(&d.deviceCommon, oldConfig, true) if err != nil { return err } // Apply and host-side network filters (uses enriched host_name from networkVethFillFromVolatile). r, err := d.setupHostFilters(oldConfig) if err != nil { return err } reverter.Add(r) } // Rebuild dnsmasq entry if needed and reload. err := d.rebuildDnsmasqEntry() if err != nil { return err } // If an IPv6 address has changed, if the instance is running we should bounce the host-side // veth interface to give the instance a chance to detect the change and re-apply for an // updated lease with new IP address. if d.config["ipv6.address"] != oldConfig["ipv6.address"] && d.config["host_name"] != "" && util.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["host_name"])) { link := &ip.Link{Name: d.config["host_name"]} err := link.SetDown() if err != nil { return err } err = link.SetUp() if err != nil { return err } } // If an external address changed, update the BGP advertisements. err = bgpRemovePrefix(&d.deviceCommon, oldConfig) if err != nil { return err } err = bgpAddPrefix(&d.deviceCommon, d.network, d.config) if err != nil { return err } if isRunning { err = d.setNICLink() if err != nil { return err } } reverter.Success() return nil } // Stop is run when the device is removed from the instance. func (d *nicBridged) Stop() (*deviceConfig.RunConfig, error) { // Remove BGP announcements. err := bgpRemovePrefix(&d.deviceCommon, d.config) if err != nil { return nil, err } // Populate device config with volatile fields (hwaddr and host_name) if needed. networkVethFillFromVolatile(d.config, d.volatileGet()) err = networkClearHostVethLimits(&d.deviceCommon) if err != nil { return nil, err } // Setup post-stop actions. runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *nicBridged) postStop() error { // Handle the case where validation fails but the device still must be removed. bridgeName := d.config["parent"] if bridgeName == "" && d.config["network"] != "" { bridgeName = d.config["network"] } defer func() { _ = d.volatileSet(map[string]string{ "host_name": "", }) }() v := d.volatileGet() networkVethFillFromVolatile(d.config, v) if d.config["host_name"] != "" && network.InterfaceExists(d.config["host_name"]) { // Detach host-side end of veth pair from bridge (required for openvswitch particularly). err := network.DetachInterface(d.state, bridgeName, d.config["host_name"]) if err != nil { return fmt.Errorf("Failed to detach interface %q from %q: %w", d.config["host_name"], bridgeName, err) } // Removing host-side end of veth pair will delete the peer end too. err = network.InterfaceRemove(d.config["host_name"]) if err != nil { return fmt.Errorf("Failed to remove interface %q: %w", d.config["host_name"], err) } } // Remove host-side routes from bridge interface. routes := []string{} routes = append(routes, util.SplitNTrimSpace(d.config["ipv4.routes"], ",", -1, true)...) routes = append(routes, util.SplitNTrimSpace(d.config["ipv6.routes"], ",", -1, true)...) routes = append(routes, util.SplitNTrimSpace(d.config["ipv4.routes.external"], ",", -1, true)...) routes = append(routes, util.SplitNTrimSpace(d.config["ipv6.routes.external"], ",", -1, true)...) networkNICRouteDelete(bridgeName, d.config["ipv4.address"], d.config["ipv6.address"], routes...) if util.IsTrue(d.config["security.mac_filtering"]) || util.IsTrue(d.config["security.ipv4_filtering"]) || util.IsTrue(d.config["security.ipv6_filtering"]) || d.config["security.acls"] != "" { d.removeFilters(d.config) } return nil } // Remove is run when the device is removed from the instance or the instance is deleted. func (d *nicBridged) Remove(cleanupDependencies bool) error { // Handle the case where validation fails but the device still must be removed. bridgeName := d.config["parent"] if bridgeName == "" && d.config["network"] != "" { bridgeName = d.config["network"] } if bridgeName != "" { dnsmasq.ConfigMutex.Lock() defer dnsmasq.ConfigMutex.Unlock() if network.InterfaceExists(bridgeName) { err := d.networkClearLease(d.inst.Name(), bridgeName, d.config["hwaddr"], clearLeaseAll) if err != nil { return fmt.Errorf("Failed clearing leases: %w", err) } } // Remove dnsmasq config if it exists (doesn't return error if file is missing). err := dnsmasq.RemoveStaticEntry(bridgeName, d.inst.Project().Name, d.inst.Name(), d.Name()) if err != nil { return err } // Reload dnsmasq to apply new settings if dnsmasq is running. err = dnsmasq.Kill(bridgeName, true) if err != nil { return err } } return nil } // rebuildDnsmasqEntry rebuilds the dnsmasq host entry if connected to a managed network and reloads dnsmasq. func (d *nicBridged) rebuildDnsmasqEntry() error { // Rebuild dnsmasq config if parent is a managed bridge network using dnsmasq. bridgeNet, ok := d.network.(bridgeNetwork) if !ok || !d.network.IsManaged() || !bridgeNet.UsesDNSMasq() { return nil } dnsmasq.ConfigMutex.Lock() defer dnsmasq.ConfigMutex.Unlock() ipv4Address := d.config["ipv4.address"] ipv6Address := d.config["ipv6.address"] // If address is set to none treat it the same as not being specified if ipv4Address == "none" { ipv4Address = "" } if ipv6Address == "none" { ipv6Address = "" } // If IP filtering is enabled, and no static IP in config, check if there is already a // dynamically assigned static IP in dnsmasq config and write that back out in new config. if (util.IsTrue(d.config["security.ipv4_filtering"]) && ipv4Address == "") || (util.IsTrue(d.config["security.ipv6_filtering"]) && ipv6Address == "") { deviceStaticFileName := dnsmasq.StaticAllocationFileName(d.inst.Project().Name, d.inst.Name(), d.Name()) _, curIPv4, curIPv6, err := dnsmasq.DHCPStaticAllocation(d.config["parent"], deviceStaticFileName) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } if ipv4Address == "" && curIPv4.IP != nil { ipv4Address = curIPv4.IP.String() } if ipv6Address == "" && curIPv6.IP != nil { ipv6Address = curIPv6.IP.String() } } err := dnsmasq.UpdateStaticEntry(d.config["parent"], d.inst.Project().Name, d.inst.Name(), d.Name(), d.network.Config(), d.config["hwaddr"], ipv4Address, ipv6Address) if err != nil { return err } // Reload dnsmasq to apply new settings. err = dnsmasq.Kill(d.config["parent"], true) if err != nil { return err } return nil } // setupHostFilters applies any host side network filters. // Returns a revert fail function that can be used to undo this function if a subsequent step fails. func (d *nicBridged) setupHostFilters(oldConfig deviceConfig.Device) (revert.Hook, error) { reverter := revert.New() defer reverter.Fail() // Check br_netfilter kernel module is loaded and enabled for IPv6 before clearing existing rules. // We won't try to load it as its default mode can cause unwanted traffic blocking. if util.IsTrue(d.config["security.ipv6_filtering"]) { err := network.BridgeNetfilterEnabled(6) if err != nil { return nil, fmt.Errorf("security.ipv6_filtering requires bridge netfilter: %w", err) } } // Remove any old network filters if non-empty oldConfig supplied as part of update. if oldConfig != nil && (util.IsTrue(oldConfig["security.mac_filtering"]) || util.IsTrue(oldConfig["security.ipv4_filtering"]) || util.IsTrue(oldConfig["security.ipv6_filtering"]) || oldConfig["security.acls"] != "") { d.removeFilters(oldConfig) } // Setup network filters. if util.IsTrue(d.config["security.mac_filtering"]) || util.IsTrue(d.config["security.ipv4_filtering"]) || util.IsTrue(d.config["security.ipv6_filtering"]) || d.config["security.acls"] != "" { err := d.setFilters() if err != nil { return nil, err } reverter.Add(func() { d.removeFilters(d.config) }) } cleanup := reverter.Clone().Fail reverter.Success() return cleanup, nil } // removeFilters removes any network level filters defined for the instance. func (d *nicBridged) removeFilters(m deviceConfig.Device) { if m["hwaddr"] == "" { d.logger.Error("Failed to remove network filters: hwaddr not defined") return } if m["host_name"] == "" { d.logger.Error("Failed to remove network filters: host_name not defined") return } IPv4Nets, IPv6Nets, err := allowedIPNets(m) if err != nil { d.logger.Error("Failed to calculate static IP network filters", logger.Ctx{"err": err}) return } // Remove filters for static MAC and IPs (if specified above). // This covers the case when filtering is used with an unmanaged bridge. d.logger.Debug("Clearing instance firewall static filters", logger.Ctx{"parent": m["parent"], "host_name": m["host_name"], "hwaddr": m["hwaddr"], "IPv4Nets": IPv4Nets, "IPv6Nets": IPv6Nets}) err = d.state.Firewall.InstanceClearBridgeFilter(d.inst.Project().Name, d.inst.Name(), d.name, m["parent"], m["host_name"], m["hwaddr"], IPv4Nets, IPv6Nets) if err != nil { d.logger.Error("Failed to remove static IP network filters", logger.Ctx{"err": err}) } // If allowedIPNets returned nil for IPv4 or IPv6, it is possible that total protocol blocking was set up // because the device has a managed parent network with DHCP disabled. Pass in empty slices to catch this case. d.logger.Debug("Clearing instance total protocol filters", logger.Ctx{"parent": m["parent"], "host_name": m["host_name"], "hwaddr": m["hwaddr"], "IPv4Nets": IPv4Nets, "IPv6Nets": IPv6Nets}) err = d.state.Firewall.InstanceClearBridgeFilter(d.inst.Project().Name, d.inst.Name(), d.name, m["parent"], m["host_name"], m["hwaddr"], make([]*net.IPNet, 0), make([]*net.IPNet, 0)) if err != nil { d.logger.Error("Failed to remove total protocol network filters", logger.Ctx{"err": err}) } // Read current static DHCP IP allocation configured from dnsmasq host config (if exists). // This covers the case when IPs are not defined in config, but have been assigned in managed DHCP. deviceStaticFileName := dnsmasq.StaticAllocationFileName(d.inst.Project().Name, d.inst.Name(), d.Name()) _, IPv4Alloc, IPv6Alloc, err := dnsmasq.DHCPStaticAllocation(m["parent"], deviceStaticFileName) if err != nil { if errors.Is(err, fs.ErrNotExist) { return } d.logger.Error("Failed to get static IP allocations for filter removal", logger.Ctx{"err": err}) return } // We have already cleared any "ipv{n}.routes" etc. above, so we just need to clear the DHCP allocated IPs. var IPv4AllocNets []*net.IPNet if len(IPv4Alloc.IP) > 0 { _, IPv4AllocNet, err := net.ParseCIDR(fmt.Sprintf("%s/32", IPv4Alloc.IP.String())) if err != nil { d.logger.Error("Failed to generate subnet from dynamically generated IPv4 address", logger.Ctx{"err": err}) } else { IPv4AllocNets = append(IPv4AllocNets, IPv4AllocNet) } } var IPv6AllocNets []*net.IPNet if len(IPv6Alloc.IP) > 0 { _, IPv6AllocNet, err := net.ParseCIDR(fmt.Sprintf("%s/128", IPv6Alloc.IP.String())) if err != nil { d.logger.Error("Failed to generate subnet from dynamically generated IPv6Address", logger.Ctx{"err": err}) } else { IPv6AllocNets = append(IPv6AllocNets, IPv6AllocNet) } } d.logger.Debug("Clearing instance firewall dynamic filters", logger.Ctx{"parent": m["parent"], "host_name": m["host_name"], "hwaddr": m["hwaddr"], "ipv4": IPv4Alloc.IP, "ipv6": IPv6Alloc.IP}) err = d.state.Firewall.InstanceClearBridgeFilter(d.inst.Project().Name, d.inst.Name(), d.name, m["parent"], m["host_name"], m["hwaddr"], IPv4AllocNets, IPv6AllocNets) if err != nil { logger.Errorf("Failed to remove DHCP network assigned filters for %q: %v", d.name, err) } d.logger.Debug("Clearing instance firewall unused address sets") err = d.state.Firewall.NetworkDeleteAddressSetsIfUnused("bridge") if err != nil { logger.Errorf("Failed to remove network address set for %q: %v", d.name, err) } } // setFilters sets up any network level filters defined for the instance. // These are controlled by the security.mac_filtering, security.ipv4_Filtering, security.ipv6_filtering and security.acls config keys. func (d *nicBridged) setFilters() (err error) { if d.config["hwaddr"] == "" { return errors.New("Failed to set network filters: require hwaddr defined") } if d.config["host_name"] == "" { return errors.New("Failed to set network filters: require host_name defined") } if d.config["parent"] == "" { return errors.New("Failed to set network filters: require parent defined") } // Parse device config. mac, err := net.ParseMAC(d.config["hwaddr"]) if err != nil { return fmt.Errorf("Invalid hwaddr: %w", err) } // Parse static IPs, relies on invalid IPs being set to nil. IPv4 := net.ParseIP(d.config["ipv4.address"]) IPv6 := net.ParseIP(d.config["ipv6.address"]) // If parent bridge is unmanaged check that a manually specified IP is available if IP filtering enabled. if d.network == nil { if util.IsTrue(d.config["security.ipv4_filtering"]) && d.config["ipv4.address"] == "" { return errors.New("IPv4 filtering requires a manually specified ipv4.address when using an unmanaged parent bridge") } if util.IsTrue(d.config["security.ipv6_filtering"]) && d.config["ipv6.address"] == "" { return errors.New("IPv6 filtering requires a manually specified ipv6.address when using an unmanaged parent bridge") } } // Use a clone of the config. This can be amended with the allocated IPs so that the correct ones are added to the firewall. config := d.config.Clone() // If parent bridge is managed, allocate the static IPs (if needed). if d.network != nil && (IPv4 == nil || IPv6 == nil) { opts := &dhcpalloc.Options{ ProjectName: d.inst.Project().Name, HostName: d.inst.Name(), DeviceName: d.Name(), HostMAC: mac, Network: d.network, } err = dhcpalloc.AllocateTask(opts, func(t *dhcpalloc.Transaction) error { if util.IsTrue(config["security.ipv4_filtering"]) && IPv4 == nil && config["ipv4.address"] != "none" { IPv4, err = t.AllocateIPv4() config["ipv4.address"] = IPv4.String() // If DHCP not supported, skip error and set the address to "none", and will result in total protocol filter. if errors.Is(err, dhcpalloc.ErrDHCPNotSupported) { config["ipv4.address"] = "none" } else if err != nil { return err } } if util.IsTrue(config["security.ipv6_filtering"]) && IPv6 == nil && config["ipv6.address"] != "none" { IPv6, err = t.AllocateIPv6() config["ipv6.address"] = IPv6.String() // If DHCP not supported, skip error and set the address to "none", and will result in total protocol filter. if errors.Is(err, dhcpalloc.ErrDHCPNotSupported) { config["ipv6.address"] = "none" } else if err != nil { return err } } return nil }) if err != nil && !errors.Is(err, dhcpalloc.ErrDHCPNotSupported) { return err } } // If anything goes wrong, clean up so we don't leave orphaned rules. reverter := revert.New() defer reverter.Fail() reverter.Add(func() { d.removeFilters(config) }) ipv4Nets, ipv6Nets, err := allowedIPNets(config) if err != nil { return err } var ipv4DNS []string var ipv6DNS []string if d.network != nil { netConfig := d.network.Config() ipv4DNS = []string{} ipv6DNS = []string{} // Pull directly configured DNS name servers (if any). nsList := util.SplitNTrimSpace(netConfig["dns.nameservers"], ",", -1, false) for _, ns := range nsList { if ns == "" { continue } nsIP := net.ParseIP(ns) if nsIP == nil { return errors.New("Invalid DNS nameserver") } if nsIP.To4() == nil { ipv4DNS = append(ipv4DNS, ns) } else { ipv6DNS = append(ipv6DNS, ns) } } // Add IPv4 router. if netConfig["ipv4.address"] != "" && netConfig["ipv4.address"] != "none" { ipv4DNS = append(ipv4DNS, strings.Split(netConfig["ipv4.address"], "/")[0]) } // Add IPv6 router. if netConfig["ipv6.address"] != "" && netConfig["ipv6.address"] != "none" { ipv6DNS = append(ipv6DNS, strings.Split(netConfig["ipv6.address"], "/")[0]) } } var aclRules []firewallDrivers.ACLRule var aclNames []string if config["security.acls"] != "" { aclNames = util.SplitNTrimSpace(config["security.acls"], ",", -1, false) networkProjectName, _, err := project.NetworkProject(d.state.DB.Cluster, d.inst.Project().Name) if err != nil { return err } aclRules, err = acl.FirewallACLRules(d.state, d.name, networkProjectName, d.config) if err != nil { return err } // Ensure address sets for ACL, we state bridge because // this is the table firewall driver will use for this kind of NIC. err = addressSet.FirewallApplyAddressSetsForACLRules(d.state, "bridge", d.inst.Project().Name, aclNames) if err != nil { return err } } err = d.state.Firewall.InstanceSetupBridgeFilter(d.inst.Project().Name, d.inst.Name(), d.name, d.config["parent"], d.config["host_name"], d.config["hwaddr"], ipv4Nets, ipv6Nets, ipv4DNS, ipv6DNS, d.network != nil, util.IsTrue(config["security.mac_filtering"]), aclRules) if err != nil { return err } reverter.Success() return nil } // allowedIPNets accepts a device config. For each IP version it returns nil if all addresses should be allowed, // an empty slice if all addresses should be blocked, and a populated slice of subnets to allow traffic from specific ranges. func allowedIPNets(config deviceConfig.Device) (IPv4Nets []*net.IPNet, IPv6Nets []*net.IPNet, err error) { getAllowedNets := func(ipVersion int) ([]*net.IPNet, error) { if util.IsFalseOrEmpty(config[fmt.Sprintf("security.ipv%d_filtering", ipVersion)]) { // Return nil (allow all) return nil, nil } ipAddr := config[fmt.Sprintf("ipv%d.address", ipVersion)] if ipAddr == "none" { // Return an empty slice to block all traffic. return []*net.IPNet{}, nil } var routes []string // Get a CIDR string for the instance address if ipAddr != "" { if ipVersion == 4 { routes = append(routes, fmt.Sprintf("%s/32", ipAddr)) } else if ipVersion == 6 { routes = append(routes, fmt.Sprintf("%s/128", ipAddr)) } } // Get remaining allowed routes from config. routes = append(routes, util.SplitNTrimSpace(config[fmt.Sprintf("ipv%d.routes", ipVersion)], ",", -1, true)...) routes = append(routes, util.SplitNTrimSpace(config[fmt.Sprintf("ipv%d.routes.external", ipVersion)], ",", -1, true)...) var allowedNets []*net.IPNet for _, route := range routes { ipNet, err := network.ParseIPCIDRToNet(route) if err != nil { return nil, err } allowedNets = append(allowedNets, ipNet) } return allowedNets, nil } IPv4Nets, err = getAllowedNets(4) if err != nil { return nil, nil, err } IPv6Nets, err = getAllowedNets(6) if err != nil { return nil, nil, err } return IPv4Nets, IPv6Nets, nil } const ( clearLeaseAll = iota clearLeaseIPv4Only clearLeaseIPv6Only ) // networkClearLease clears leases from a running dnsmasq process. func (d *nicBridged) networkClearLease(name string, network string, hwaddr string, mode int) error { leaseFile := internalUtil.VarPath("networks", network, "dnsmasq.leases") // Check that we are in fact running a dnsmasq for the network if !util.PathExists(leaseFile) { return nil } // Convert MAC string to bytes to avoid any case comparison issues later. srcMAC, err := net.ParseMAC(hwaddr) if err != nil { return err } iface, err := net.InterfaceByName(network) if err != nil { return fmt.Errorf("Failed getting bridge interface state for %q: %w", network, err) } // Get IPv4 and IPv6 address of interface running dnsmasq on host. addrs, err := iface.Addrs() if err != nil { return fmt.Errorf("Failed getting bridge interface addresses for %q: %w", network, err) } var dstIPv4, dstIPv6 net.IP for _, addr := range addrs { ip, _, err := net.ParseCIDR(addr.String()) if err != nil { return err } if !ip.IsGlobalUnicast() { continue } if ip.To4() == nil { dstIPv6 = ip } else { dstIPv4 = ip } } // Iterate the dnsmasq leases file looking for matching leases for this instance to release. file, err := os.Open(leaseFile) if err != nil { return err } defer func() { _ = file.Close() }() var dstDUID string errs := []error{} scanner := bufio.NewScanner(file) for scanner.Scan() { fields := strings.Fields(scanner.Text()) fieldsLen := len(fields) // Handle lease lines if fieldsLen == 5 { if (mode == clearLeaseAll || mode == clearLeaseIPv4Only) && srcMAC.String() == fields[1] { // Handle IPv4 leases by matching MAC address to lease. srcIP := net.ParseIP(fields[2]) if dstIPv4 == nil { logger.Warnf("Failed to release DHCPv4 lease for instance %q, IP %q, MAC %q, %v", name, srcIP, srcMAC, "No server address found") continue // Can't send release packet if no dstIP found. } err = d.networkDHCPv4Release(srcMAC, srcIP, dstIPv4) if err != nil { errs = append(errs, fmt.Errorf("Failed to release DHCPv4 lease for instance %q, IP %q, MAC %q, %v", name, srcIP, srcMAC, err)) } } else if (mode == clearLeaseAll || mode == clearLeaseIPv6Only) && name == fields[3] { // Handle IPv6 addresses by matching hostname to lease. IAID := fields[1] srcIP := net.ParseIP(fields[2]) DUID := fields[4] // Skip IPv4 addresses. if srcIP.To4() != nil { continue } if dstIPv6 == nil { logger.Warnf("Failed to release DHCPv6 lease for instance %q, IP %q, DUID %q, IAID %q: %q", name, srcIP, DUID, IAID, "No server address found") continue // Can't send release packet if no dstIP found. } if dstDUID == "" { errs = append(errs, fmt.Errorf("Failed to release DHCPv6 lease for instance %q, IP %q, DUID %q, IAID %q: %s", name, srcIP, DUID, IAID, "No server DUID found")) continue // Can't send release packet if no dstDUID found. } err = d.networkDHCPv6Release(DUID, IAID, srcIP, dstIPv6, dstDUID) if err != nil { errs = append(errs, fmt.Errorf("Failed to release DHCPv6 lease for instance %q, IP %q, DUID %q, IAID %q: %w", name, srcIP, DUID, IAID, err)) } } } else if fieldsLen == 2 && fields[0] == "duid" { // Handle server DUID line needed for releasing IPv6 leases. // This should come before the IPv6 leases in the lease file. dstDUID = fields[1] } } if len(errs) > 0 { return fmt.Errorf("%v", errs) } err = scanner.Err() if err != nil { return err } return nil } // networkDHCPv4Release sends a DHCPv4 release packet to a DHCP server. func (d *nicBridged) networkDHCPv4Release(srcMAC net.HardwareAddr, srcIP net.IP, dstIP net.IP) error { dstAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:67", dstIP.String())) if err != nil { return err } conn, err := net.DialUDP("udp", nil, dstAddr) if err != nil { return err } defer func() { _ = conn.Close() }() // Random DHCP transaction ID xid := rand.Uint32() // Construct a DHCP packet pretending to be from the source IP and MAC supplied. dhcp := layers.DHCPv4{ Operation: layers.DHCPOpRequest, HardwareType: layers.LinkTypeEthernet, ClientHWAddr: srcMAC, ClientIP: srcIP, Xid: xid, } // Add options to DHCP release packet. dhcp.Options = append(dhcp.Options, layers.NewDHCPOption(layers.DHCPOptMessageType, []byte{byte(layers.DHCPMsgTypeRelease)}), layers.NewDHCPOption(layers.DHCPOptServerID, dstIP.To4()), ) buf := gopacket.NewSerializeBuffer() opts := gopacket.SerializeOptions{ ComputeChecksums: true, FixLengths: true, } err = gopacket.SerializeLayers(buf, opts, &dhcp) if err != nil { return err } _, err = conn.Write(buf.Bytes()) if err != nil { return err } return conn.Close() } // networkDHCPv6Release sends a DHCPv6 release packet to a DHCP server. func (d *nicBridged) networkDHCPv6Release(srcDUID string, srcIAID string, srcIP net.IP, dstIP net.IP, dstDUID string) error { dstAddr, err := net.ResolveUDPAddr("udp6", fmt.Sprintf("[%s]:547", dstIP.String())) if err != nil { return err } conn, err := net.DialUDP("udp6", nil, dstAddr) if err != nil { return err } defer func() { _ = conn.Close() }() // Construct a DHCPv6 packet pretending to be from the source IP and MAC supplied. dhcp := layers.DHCPv6{ MsgType: layers.DHCPv6MsgTypeRelease, } // Convert Server DUID from string to byte array dstDUIDRaw, err := hex.DecodeString(strings.ReplaceAll(dstDUID, ":", "")) if err != nil { return err } // Convert DUID from string to byte array srcDUIDRaw, err := hex.DecodeString(strings.ReplaceAll(srcDUID, ":", "")) if err != nil { return err } // Convert IAID string to int srcIAIDRaw, err := strconv.ParseUint(srcIAID, 10, 32) if err != nil { return err } srcIAIDRaw32 := uint32(srcIAIDRaw) // Build the Identity Association details option manually (as not provided by gopacket). iaAddr := d.networkDHCPv6CreateIAAddress(srcIP) ianaRaw := d.networkDHCPv6CreateIANA(srcIAIDRaw32, iaAddr) // Add options to DHCP release packet. dhcp.Options = append(dhcp.Options, layers.NewDHCPv6Option(layers.DHCPv6OptServerID, dstDUIDRaw), layers.NewDHCPv6Option(layers.DHCPv6OptClientID, srcDUIDRaw), layers.NewDHCPv6Option(layers.DHCPv6OptIANA, ianaRaw), ) buf := gopacket.NewSerializeBuffer() opts := gopacket.SerializeOptions{ ComputeChecksums: true, FixLengths: true, } err = gopacket.SerializeLayers(buf, opts, &dhcp) if err != nil { return err } _, err = conn.Write(buf.Bytes()) if err != nil { return err } return conn.Close() } // networkDHCPv6CreateIANA creates a DHCPv6 Identity Association for Non-temporary Address (rfc3315 IA_NA) option. func (d *nicBridged) networkDHCPv6CreateIANA(IAID uint32, IAAddr []byte) []byte { data := make([]byte, 12) binary.BigEndian.PutUint32(data[0:4], IAID) // Identity Association Identifier binary.BigEndian.PutUint32(data[4:8], uint32(0)) // T1 binary.BigEndian.PutUint32(data[8:12], uint32(0)) // T2 data = append(data, IAAddr...) // Append the IA Address details return data } // networkDHCPv6CreateIAAddress creates a DHCPv6 Identity Association Address (rfc3315) option. func (d *nicBridged) networkDHCPv6CreateIAAddress(IP net.IP) []byte { data := make([]byte, 28) binary.BigEndian.PutUint16(data[0:2], uint16(layers.DHCPv6OptIAAddr)) // Sub-Option type binary.BigEndian.PutUint16(data[2:4], uint16(24)) // Length (fixed at 24 bytes) copy(data[4:20], IP) // IPv6 address to be released binary.BigEndian.PutUint32(data[20:24], uint32(0)) // Preferred liftetime binary.BigEndian.PutUint32(data[24:28], uint32(0)) // Valid lifetime return data } // setupNativeBridgePortVLANs configures the bridge port with the specified VLAN settings on the native bridge. func (d *nicBridged) setupNativeBridgePortVLANs(hostName string) error { link := &ip.Link{Name: hostName} // Check vlan_filtering is enabled on bridge if needed. if d.config["vlan"] != "" || d.config["vlan.tagged"] != "" { vlanFilteringStatus, err := network.BridgeVLANFilteringStatus(d.config["parent"]) if err != nil { return err } if vlanFilteringStatus != "1" { return fmt.Errorf("VLAN filtering is not enabled in parent bridge %q", d.config["parent"]) } } // Set port on bridge to specified untagged PVID. if d.config["vlan"] != "" { // Reject VLAN ID 0 if specified (as validation allows VLAN ID 0 on unmanaged bridges for OVS). if d.config["vlan"] == "0" { return errors.New("VLAN ID 0 is not allowed for native Linux bridges") } // Get default PVID membership on port. defaultPVID, err := network.BridgeVLANDefaultPVID(d.config["parent"]) if err != nil { return err } // If the bridge has a default PVID and it is different to the specified untagged VLAN or if tagged // VLAN is set to "none" then remove the default untagged membership. if defaultPVID != "0" && (defaultPVID != d.config["vlan"] || d.config["vlan"] == "none") { err = link.BridgeVLANDelete(defaultPVID, false) if err != nil { return fmt.Errorf("Failed removing default PVID membership: %w", err) } } // Configure the untagged membership settings of the port if VLAN ID specified. if d.config["vlan"] != "none" { err = link.BridgeVLANAdd(d.config["vlan"], true, true, false) if err != nil { return err } } } // Add any tagged VLAN memberships. if d.config["vlan.tagged"] != "" { networkVLANList, err := networkVLANListExpand(util.SplitNTrimSpace(d.config["vlan.tagged"], ",", -1, true)) if err != nil { return err } for _, vlanID := range networkVLANList { // Reject VLAN ID 0 if specified (as validation allows VLAN ID 0 on unmanaged bridges for OVS). if vlanID == 0 { return errors.New("VLAN tagged ID 0 is not allowed for native Linux bridges") } err := link.BridgeVLANAdd(fmt.Sprintf("%d", vlanID), false, false, false) if err != nil { return err } } } return nil } // setupOVSBridgePortVLANs configures the bridge port with the specified VLAN settings on the openvswitch bridge. func (d *nicBridged) setupOVSBridgePortVLANs(hostName string) error { vswitch, err := d.state.OVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } // Set port on bridge to specified untagged PVID. if d.config["vlan"] != "" { if d.config["vlan"] == "none" && d.config["vlan.tagged"] == "" { return errors.New("vlan=none is not supported with openvswitch bridges when not using vlan.tagged") } // Configure the untagged 'native' membership settings of the port if VLAN ID specified. // Also set the vlan_mode=access, which will drop any tagged frames. // Order is important here, as vlan_mode is set to "access", assuming that vlan.tagged is not used. // If vlan.tagged is specified, then we expect it to also change the vlan_mode as needed. if d.config["vlan"] != "none" { vlanID, err := strconv.Atoi(d.config["vlan"]) if err != nil { return err } err = vswitch.UpdateBridgePortVLANs(context.TODO(), hostName, "access", vlanID, nil) if err != nil { return err } } } // Add any tagged VLAN memberships. if d.config["vlan.tagged"] != "" { intNetworkVLANs, err := networkVLANListExpand(util.SplitNTrimSpace(d.config["vlan.tagged"], ",", -1, true)) if err != nil { return err } vlanMode := "trunk" // Default to only allowing tagged frames (drop untagged frames). if d.config["vlan"] != "none" { // If untagged vlan mode isn't "none" then allow untagged frames for port's 'native' VLAN. vlanMode = "native-untagged" } // Configure the tagged membership settings of the port if VLAN ID specified. // Also set the vlan_mode as needed from above. // Must come after the PortSet command used for setting "vlan" mode above so that the correct // vlan_mode is retained. err = vswitch.UpdateBridgePortVLANs(context.TODO(), hostName, vlanMode, 0, intNetworkVLANs) if err != nil { return err } } return nil } // State gets the state of a bridged NIC by parsing the local DHCP server leases file. func (d *nicBridged) State() (*api.InstanceStateNetwork, error) { v := d.volatileGet() // Populate device config with volatile fields if needed. networkVethFillFromVolatile(d.config, v) ips := []net.IP{} var v4mask string var v6mask string // ipStore appends an IP to ips if not already stored. ipStore := func(newIP net.IP) { for _, ip := range ips { if ip.Equal(newIP) { return } } ips = append(ips, newIP) } hwAddr, _ := net.ParseMAC(d.config["hwaddr"]) if d.network != nil { // Extract subnet sizes from bridge addresses if available. netConfig := d.network.Config() _, v4subnet, _ := net.ParseCIDR(netConfig["ipv4.address"]) _, v6subnet, _ := net.ParseCIDR(netConfig["ipv6.address"]) if v4subnet != nil { mask, _ := v4subnet.Mask.Size() v4mask = fmt.Sprintf("%d", mask) } if v6subnet != nil { mask, _ := v6subnet.Mask.Size() v6mask = fmt.Sprintf("%d", mask) } if d.config["hwaddr"] != "" { // Parse the leases file if parent network is managed. leaseIPs, err := network.GetLeaseAddresses(d.network.Name(), d.config["hwaddr"]) if err == nil { for _, leaseIP := range leaseIPs { ipStore(leaseIP) } } if util.IsFalseOrEmpty(d.network.Config()["ipv6.dhcp.stateful"]) && v6subnet != nil { // If stateful DHCPv6 is disabled, and IPv6 is enabled on the bridge, the NIC // is likely to use its MAC and SLAAC to configure its address. if hwAddr != nil { ip, err := eui64.ParseMAC(v6subnet.IP, hwAddr) if err == nil { ipStore(ip) } } } } } // Get IP addresses from IP neighbour cache if present. neighIPs, err := network.GetNeighbourIPs(d.config["parent"], hwAddr) if err == nil { validStates := []ip.NeighbourIPState{ ip.NeighbourIPStatePermanent, ip.NeighbourIPStateNoARP, ip.NeighbourIPStateReachable, } // Add any valid-state neighbour IP entries first. for _, neighIP := range neighIPs { if slices.Contains(validStates, neighIP.State) { ipStore(neighIP.Addr) } } // Add any non-failed-state entries. for _, neighIP := range neighIPs { if neighIP.State != ip.NeighbourIPStateFailed && !slices.Contains(validStates, neighIP.State) { ipStore(neighIP.Addr) } } } // Convert IPs to InstanceStateNetworkAddresses. addresses := []api.InstanceStateNetworkAddress{} for _, ip := range ips { addr := api.InstanceStateNetworkAddress{} addr.Address = ip.String() addr.Family = "inet" addr.Netmask = v4mask if ip.To4() == nil { addr.Family = "inet6" addr.Netmask = v6mask } if ip.IsLinkLocalUnicast() { addr.Scope = "link" if addr.Family == "inet6" { addr.Netmask = "64" // Link-local IPv6 addresses are /64. } else { addr.Netmask = "16" // Link-local IPv4 addresses are /16. } } else { addr.Scope = "global" } addresses = append(addresses, addr) } mtu, err := d.getHostMTU() if err != nil { d.logger.Warn("Failed getting host interface state for MTU", logger.Ctx{"host_name": d.config["host_name"], "err": err}) } // Retrieve the host counters, as we report the values from the instance's point of view, // those counters need to be reversed below. hostCounters, err := resources.GetNetworkCounters(d.config["host_name"]) if err != nil { return nil, fmt.Errorf("Failed getting network interface counters: %w", err) } network := api.InstanceStateNetwork{ Addresses: addresses, Counters: api.InstanceStateNetworkCounters{ BytesReceived: hostCounters.BytesSent, BytesSent: hostCounters.BytesReceived, PacketsReceived: hostCounters.PacketsSent, PacketsSent: hostCounters.PacketsReceived, }, Hwaddr: d.config["hwaddr"], HostName: d.config["host_name"], Mtu: mtu, State: "up", Type: "broadcast", } return &network, nil } func (d *nicBridged) getHostMTU() (int, error) { // Get MTU of host interface if exists. iface, err := net.InterfaceByName(d.config["host_name"]) if err != nil { return 0, err } mtu := -1 if iface != nil { mtu = iface.MTU } return mtu, nil } // Register sets up anything needed on startup. func (d *nicBridged) Register() error { // Skip when not using a managed network. if d.config["network"] == "" { return nil } // Load managed network. api.ProjectDefaultName is used here as bridge networks don't support projects. n, err := network.LoadByName(d.state, api.ProjectDefaultName, d.config["network"]) if err != nil { return fmt.Errorf("Error loading network config for %q: %w", d.config["network"], err) } // Add BGP prefix. err = bgpAddPrefix(&d.deviceCommon, n, d.config) if err != nil { return err } return nil } incus-7.0.0/internal/server/device/nic_ipvlan.go000066400000000000000000000451151517523235500217070ustar00rootroot00000000000000package device import ( "errors" "fmt" "net" "slices" "strings" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/network" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) const ( ipvlanModeL3S = "l3s" ipvlanModeL2 = "l2" ) type nicIPVLAN struct { deviceCommon } // CanHotPlug returns whether the device can be managed whilst the instance is running,. func (d *nicIPVLAN) CanHotPlug() bool { return false } // validateConfig checks the supplied config for correctness. func (d *nicIPVLAN) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container) { return ErrUnsupportedDevType } requiredFields := []string{"parent"} optionalFields := []string{ // gendoc:generate(entity=devices, group=nic_ipvlan, key=name) // // --- // type: string // default: kernel assigned // shortdesc: The name of the interface inside the instance "name", // gendoc:generate(entity=devices, group=nic_ipvlan, key=mtu) // // --- // type: integer // default: MTU of the parent device // shortdesc: The Maximum Transmit Unit (MTU) of the new interface "mtu", // gendoc:generate(entity=devices, group=nic_ipvlan, key=hwaddr) // // --- // type: string // default: randomly assigned // shortdesc: The MAC address of the new interface "hwaddr", // gendoc:generate(entity=devices, group=nic_ipvlan, key=vlan) // // --- // type: integer // shortdesc: The VLAN ID to attach to "vlan", // gendoc:generate(entity=devices, group=nic_ipvlan, key=ipv4.gateway) // // --- // type: string // default: `auto` (in `l3s` mode), `-` (in `l2` mode) // shortdesc: In `l3s` mode, whether to add an automatic default IPv4 gateway (can be `auto` or `none`). In `l2` mode, the IPv4 address of the gateway "ipv4.gateway", // gendoc:generate(entity=devices, group=nic_ipvlan, key=ipv6.gateway) // // --- // type: string // default: `auto` (in `l3s` mode), `-` (in `l2` mode) // shortdesc: In `l3s` mode, whether to add an automatic default IPv6 gateway (can be `auto` or `none`). In `l2` mode, the IPv6 address of the gateway "ipv6.gateway", // gendoc:generate(entity=devices, group=nic_ipvlan, key=ipv4.host_table) // // --- // type: integer // shortdesc: The custom policy routing table ID to add IPv4 static routes to (in addition to the main routing table) "ipv4.host_table", // gendoc:generate(entity=devices, group=nic_ipvlan, key=ipv6.host_table) // // --- // type: integer // shortdesc: The custom policy routing table ID to add IPv6 static routes to (in addition to the main routing table) "ipv6.host_table", // gendoc:generate(entity=devices, group=nic_ipvlan, key=gvrp) // // --- // type: bool // default: false // shortdesc: Register VLAN using GARP VLAN Registration Protocol "gvrp", // gendoc:generate(entity=devices, group=nic_ipvlan, key=attached) // // --- // type: bool // default: `true` // required: no // shortdesc: Whether the NIC is plugged in or not "attached", } rules := nicValidationRules(requiredFields, optionalFields, instConf) rules["gvrp"] = validate.Optional(validate.IsBool) // gendoc:generate(entity=devices, group=nic_ipvlan, key=ipv4.address) // // --- // type: string // shortdesc: Comma-delimited list of IPv4 static addresses to add to the instance (in l2 mode, these can be specified as CIDR values or singular addresses using a subnet of /24) rules["ipv4.address"] = func(value string) error { if value == "" { return nil } if d.config["mode"] == ipvlanModeL2 { for _, v := range strings.Split(value, ",") { v = strings.TrimSpace(v) // If valid non-CIDR address specified, append a /24 subnet. if validate.IsNetworkAddressV4(v) == nil { v = fmt.Sprintf("%s/24", v) } ip, _, err := net.ParseCIDR(v) if err != nil { return err } if ip.To4() == nil { return fmt.Errorf("Not an IPv4 CIDR address: %s", v) } } return nil } return validate.IsListOf(validate.IsNetworkAddressV4)(value) } // gendoc:generate(entity=devices, group=nic_ipvlan, key=ipv6.address) // // --- // type: string // shortdesc: Comma-delimited list of IPv6 static addresses to add to the instance (in `l2` mode, these can be specified as CIDR values or singular addresses using a subnet of /64) rules["ipv6.address"] = func(value string) error { if value == "" { return nil } if d.config["mode"] == ipvlanModeL2 { for _, v := range strings.Split(value, ",") { v = strings.TrimSpace(v) // If valid non-CIDR address specified, append a /64 subnet. if validate.IsNetworkAddressV6(v) == nil { v = fmt.Sprintf("%s/64", v) } ip, _, err := net.ParseCIDR(v) if err != nil { return err } if ip == nil || ip.To4() != nil { return fmt.Errorf("Not an IPv6 CIDR address: %s", v) } } return nil } return validate.IsListOf(validate.IsNetworkAddressV6)(value) } // gendoc:generate(entity=devices, group=nic_ipvlan, key=mode) // // --- // type: string // default: `l3s` // shortdesc: The IPVLAN mode (either `l2` or `l3s`) rules["mode"] = func(value string) error { if value == "" { return nil } validModes := []string{ipvlanModeL3S, ipvlanModeL2} if !slices.Contains(validModes, value) { return fmt.Errorf("Must be one of: %v", strings.Join(validModes, ", ")) } return nil } if d.config["mode"] == ipvlanModeL2 { rules["ipv4.gateway"] = validate.Optional(validate.IsNetworkAddressV4) rules["ipv6.gateway"] = validate.Optional(validate.IsNetworkAddressV6) } err := d.config.Validate(rules) if err != nil { return err } if d.config["mode"] == ipvlanModeL2 && d.config["host_table"] != "" { return errors.New("host_table option cannot be used in l2 mode") } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *nicIPVLAN) validateEnvironment() error { if d.inst.Type() == instancetype.Container && d.config["name"] == "" { return errors.New("Requires name property to start") } // gendoc:generate(entity=devices, group=nic_ipvlan, key=parent) // // --- // type: string // shortdesc: The name of the host device (required) if !network.InterfaceExists(d.config["parent"]) { return fmt.Errorf("Parent device '%s' doesn't exist", d.config["parent"]) } if d.config["parent"] == "" && d.config["vlan"] != "" { return errors.New("The vlan setting can only be used when combined with a parent interface") } // Only check sysctls for l2proxy if mode is l3s. if d.mode() != ipvlanModeL3S { return nil } // Generate effective parent name, including the VLAN part if option used. effectiveParentName := network.GetHostDevice(d.config["parent"], d.config["vlan"]) // If the effective parent doesn't exist and the vlan option is specified, it means we are going to create // the VLAN parent at start, and we will configure the needed sysctls so don't need to check them yet. if d.config["vlan"] != "" && !network.InterfaceExists(effectiveParentName) { return nil } if d.config["ipv4.address"] != "" { // Check necessary sysctls are configured for use with l2proxy parent in IPVLAN l3s mode. ipv4FwdPath := fmt.Sprintf("net/ipv4/conf/%s/forwarding", effectiveParentName) sysctlVal, err := localUtil.SysctlGet(ipv4FwdPath) if err != nil { return fmt.Errorf("Error reading net sysctl %s: %w", ipv4FwdPath, err) } if sysctlVal != "1\n" { // Replace . in parent name with / for sysctl formatting. return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv4.conf.%s.forwarding=1", strings.ReplaceAll(effectiveParentName, ".", "/")) } } if d.config["ipv6.address"] != "" { // Check necessary sysctls are configured for use with l2proxy parent in IPVLAN l3s mode. ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", effectiveParentName) sysctlVal, err := localUtil.SysctlGet(ipv6FwdPath) if err != nil { return fmt.Errorf("Error reading net sysctl %s: %w", ipv6FwdPath, err) } if sysctlVal != "1\n" { // Replace . in parent name with / for sysctl formatting. return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv6.conf.%s.forwarding=1", strings.ReplaceAll(effectiveParentName, ".", "/")) } ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", effectiveParentName) sysctlVal, err = localUtil.SysctlGet(ipv6ProxyNdpPath) if err != nil { return fmt.Errorf("Error reading net sysctl %s: %w", ipv6ProxyNdpPath, err) } if sysctlVal != "1\n" { // Replace . in parent name with / for sysctl formatting. return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", strings.ReplaceAll(effectiveParentName, ".", "/")) } } return nil } // Start is run when the instance is starting up (IPVLAN doesn't support hot plugging). func (d *nicIPVLAN) Start() (*deviceConfig.RunConfig, error) { // Ignore detached NICs. if !util.IsTrueOrEmpty(d.config["attached"]) { return nil, nil } err := d.validateEnvironment() if err != nil { return nil, err } // Lock to avoid issues with containers starting in parallel. networkCreateSharedDeviceLock.Lock() defer networkCreateSharedDeviceLock.Unlock() reverter := revert.New() defer reverter.Fail() saveData := make(map[string]string) // Record a random host name to use to detach the ipvlan interface back onto the host at stop time so we // can remove it and not have to rely on the kernel to do it when the namespace is destroyed, as this is // not always reliable. saveData["host_name"], err = d.generateHostName("inc", d.config["hwaddr"]) if err != nil { return nil, err } // Decide which parent we should use based on VLAN setting. parentName := network.GetHostDevice(d.config["parent"], d.config["vlan"]) statusDev, err := networkCreateVlanDeviceIfNeeded(d.state, d.config["parent"], parentName, d.config["vlan"], util.IsTrue(d.config["gvrp"])) if err != nil { return nil, err } // Record whether we created this device or not so it can be removed on stop. saveData["last_state.created"] = fmt.Sprintf("%t", statusDev != "existing") mode := d.mode() // If we created a VLAN interface, we need to setup the sysctls on that interface for l3s mode l2proxy. if statusDev == "created" && mode == ipvlanModeL3S { err := d.setupParentSysctls(parentName) if err != nil { return nil, err } } err = d.volatileSet(saveData) if err != nil { return nil, err } runConf := deviceConfig.RunConfig{} nic := []deviceConfig.RunConfigItem{ {Key: "name", Value: d.config["name"]}, {Key: "type", Value: "ipvlan"}, {Key: "flags", Value: "up"}, {Key: "ipvlan.mode", Value: mode}, {Key: "ipvlan.isolation", Value: "bridge"}, {Key: "link", Value: parentName}, } if d.config["mtu"] != "" { nic = append(nic, deviceConfig.RunConfigItem{Key: "mtu", Value: d.config["mtu"]}) } // Perform network configuration. for _, keyPrefix := range []string{"ipv4", "ipv6"} { var ipFamily ip.Family switch keyPrefix { case "ipv4": ipFamily = ip.FamilyV4 case "ipv6": ipFamily = ip.FamilyV6 } addresses := util.SplitNTrimSpace(d.config[fmt.Sprintf("%s.address", keyPrefix)], ",", -1, true) // Setup address configuration. for _, addr := range addresses { addr, err := d.parseAddress(addr, keyPrefix, mode) if err != nil { return nil, err } nic = append(nic, deviceConfig.RunConfigItem{ Key: fmt.Sprintf("%s.address", keyPrefix), Value: addr.String(), }) // Perform host-side address configuration. if mode == ipvlanModeL3S { // Apply host-side static routes to main routing table to allow neighbour proxy. r := ip.Route{ DevName: "lo", Route: addr, Table: "main", Family: ipFamily, } err = r.Add() if err != nil { return nil, fmt.Errorf("Failed adding host route %q: %w", r.Route, err) } reverter.Add(func() { _ = r.Delete() }) // Add static routes to instance IPs from custom routing tables if specified. hostTableKey := fmt.Sprintf("%s.host_table", keyPrefix) if d.config[hostTableKey] != "" { r := &ip.Route{ DevName: "lo", Route: addr, Table: d.config[hostTableKey], Family: ipFamily, } err := r.Add() if err != nil { return nil, fmt.Errorf("Failed adding host route %q: %w", r.Route, err) } reverter.Add(func() { _ = r.Delete() }) } // Add neighbour proxy entries on the host for l3s mode. np := ip.NeighProxy{ DevName: parentName, Addr: addr.IP, } err = np.Add() if err != nil { return nil, fmt.Errorf("Failed adding neighbour proxy %q to %q: %w", np.Addr.String(), np.DevName, err) } reverter.Add(func() { _ = np.Delete() }) } } // Setup gateway configuration. if len(addresses) > 0 { gwKeyName := fmt.Sprintf("%s.gateway", keyPrefix) if mode == ipvlanModeL3S && nicHasAutoGateway(d.config[gwKeyName]) { nic = append(nic, deviceConfig.RunConfigItem{ Key: gwKeyName, Value: "dev", }) } if mode == ipvlanModeL2 && d.config[gwKeyName] != "" { nic = append(nic, deviceConfig.RunConfigItem{ Key: gwKeyName, Value: d.config[gwKeyName], }) } } } runConf.NetworkInterface = nic reverter.Success() return &runConf, nil } // setupParentSysctls configures the required sysctls on the parent to allow l2proxy to work. // Because of our policy not to modify sysctls on existing interfaces, this should only be called // if we created the parent interface. func (d *nicIPVLAN) setupParentSysctls(parentName string) error { if d.config["ipv4.address"] != "" { // Set necessary sysctls for use with l2proxy parent in IPVLAN l3s mode. ipv4FwdPath := fmt.Sprintf("net/ipv4/conf/%s/forwarding", parentName) err := localUtil.SysctlSet(ipv4FwdPath, "1") if err != nil { return fmt.Errorf("Error setting net sysctl %s: %w", ipv4FwdPath, err) } } if d.config["ipv6.address"] != "" { // Set necessary sysctls use with l2proxy parent in IPVLAN l3s mode. ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", parentName) err := localUtil.SysctlSet(ipv6FwdPath, "1") if err != nil { return fmt.Errorf("Error setting net sysctl %s: %w", ipv6FwdPath, err) } ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", parentName) err = localUtil.SysctlSet(ipv6ProxyNdpPath, "1") if err != nil { return fmt.Errorf("Error setting net sysctl %s: %w", ipv6ProxyNdpPath, err) } } return nil } // Stop is run when the device is removed from the instance. func (d *nicIPVLAN) Stop() (*deviceConfig.RunConfig, error) { v := d.volatileGet() runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } // Add instruction for removal of ipvlan interface back to host if set. if v["host_name"] != "" { runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "link", Value: v["host_name"]}, } } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *nicIPVLAN) postStop() error { defer func() { _ = d.volatileSet(map[string]string{ "last_state.created": "", "host_name": "", }) }() v := d.volatileGet() networkVethFillFromVolatile(d.config, v) errs := []error{} // Delete host-side detached interface if not removed by liblxc. if network.InterfaceExists(d.config["host_name"]) { err := network.InterfaceRemove(d.config["host_name"]) if err != nil { errs = append(errs, fmt.Errorf("Failed to remove interface %q: %w", d.config["host_name"], err)) } } mode := d.mode() parentName := network.GetHostDevice(d.config["parent"], d.config["vlan"]) // Clean up host-side network configuration. for _, keyPrefix := range []string{"ipv4", "ipv6"} { var ipFamily ip.Family switch keyPrefix { case "ipv4": ipFamily = ip.FamilyV4 case "ipv6": ipFamily = ip.FamilyV6 } addresses := util.SplitNTrimSpace(d.config[fmt.Sprintf("%s.address", keyPrefix)], ",", -1, true) // Remove host-side address configuration. for _, addr := range addresses { addr, err := d.parseAddress(addr, keyPrefix, mode) if err != nil { errs = append(errs, err) continue } // Remove static routes and neighbour proxy rules to instance IPs from main routing table. if mode == ipvlanModeL3S { r := ip.Route{ DevName: "lo", Route: addr, Table: "main", Family: ipFamily, } err := r.Delete() if err != nil { errs = append(errs, err) } np := ip.NeighProxy{ DevName: parentName, Addr: addr.IP, } err = np.Delete() if err != nil { errs = append(errs, err) } // Remove static routes to instance IPs from custom routing tables if specified. hostTableKey := fmt.Sprintf("%s.host_table", keyPrefix) if d.config[hostTableKey] != "" { r := &ip.Route{ DevName: "lo", Route: addr, Table: d.config[hostTableKey], Family: ipFamily, } err := r.Delete() if err != nil { errs = append(errs, err) } } } } } // This will delete the parent interface if we created it for VLAN parent. if util.IsTrue(v["last_state.created"]) { err := networkRemoveInterfaceIfNeeded(d.state, parentName, d.inst, d.config["parent"], d.config["vlan"]) if err != nil { errs = append(errs, err) } } if len(errs) > 0 { return fmt.Errorf("%v", errs) } return nil } // mode returns the ipvlan mode to use. func (d *nicIPVLAN) mode() string { if d.config["mode"] == ipvlanModeL2 { return ipvlanModeL2 } return ipvlanModeL3S } // parseAddress converts the specified address into a CIDR based on the IP family and mode. func (d *nicIPVLAN) parseAddress(addr string, ipFamily string, mode string) (*net.IPNet, error) { // If singular IP specified then convert to appropriate CIDR value for family and mode. if !strings.Contains(addr, "/") { var defaultSubnetSize int switch mode { case ipvlanModeL3S: switch ipFamily { case "ipv4": defaultSubnetSize = 32 case "ipv6": defaultSubnetSize = 128 } case ipvlanModeL2: switch ipFamily { case "ipv4": defaultSubnetSize = 24 case "ipv6": defaultSubnetSize = 64 } default: return nil, fmt.Errorf("Invalid mode %q", mode) } addr = fmt.Sprintf("%s/%d", addr, defaultSubnetSize) } cidr, err := network.ParseIPCIDRToNet(addr) if err != nil { return nil, fmt.Errorf("Invalid address %q", addr) } return cidr, nil } incus-7.0.0/internal/server/device/nic_macvlan.go000066400000000000000000000311301517523235500220270ustar00rootroot00000000000000package device import ( "errors" "fmt" "io/fs" "net" "net/http" "strconv" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/network" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" ) type nicMACVLAN struct { deviceCommon network network.Network // Populated in validateConfig(). } // CanHotPlug returns whether the device can be managed whilst the instance is running. Returns true. func (d *nicMACVLAN) CanHotPlug() bool { return true } // CanMigrate returns whether the device can be migrated to any other cluster member. func (d *nicMACVLAN) CanMigrate() bool { return d.config["network"] != "" } // validateConfig checks the supplied config for correctness. func (d *nicMACVLAN) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container, instancetype.VM) { return ErrUnsupportedDevType } var requiredFields []string optionalFields := []string{ // gendoc:generate(entity=devices, group=nic_macvlan, key=name) // // --- // type: string // default: kernel assigned // managed: no // shortdesc: The name of the interface inside the instance "name", // gendoc:generate(entity=devices, group=nic_macvlan, key=network) // // --- // type: string // managed: no // shortdesc: The managed network to link the device to (instead of specifying the `nictype` directly) "network", // gendoc:generate(entity=devices, group=nic_macvlan, key=parent) // // --- // type: string // managed: yes // shortdesc: The name of the parent host device (required if specifying the `nictype` directly) "parent", // gendoc:generate(entity=devices, group=nic_macvlan, key=mtu) // // --- // type: integer // default: MTU of the parent device // managed: yes // shortdesc: The Maximum Transmit Unit (MTU) of the new interface "mtu", // gendoc:generate(entity=devices, group=nic_macvlan, key=hwaddr) // // --- // type: string // default: randomly assigned // managed: no // shortdesc: The MAC address of the new interface "hwaddr", // gendoc:generate(entity=devices, group=nic_macvlan, key=vlan) // // --- // type: integer // managed: no // shortdesc: The VLAN ID to attach to "vlan", // gendoc:generate(entity=devices, group=nic_macvlan, key=boot.priority) // // --- // type: integer // managed: no // shortdesc: Boot priority for VMs (higher value boots first) "boot.priority", // gendoc:generate(entity=devices, group=nic_macvlan, key=gvrp) // // --- // type: bool // default: false // managed: no // shortdesc: Register VLAN using GARP VLAN Registration Protocol "gvrp", // gendoc:generate(entity=devices, group=nic_macvlan, key=mode) // // --- // type: string // default: bridge // managed: no // shortdesc: Macvlan mode (one of `bridge`, `vepa`, `passthru` or `private`) "mode", // gendoc:generate(entity=devices, group=nic_macvlan, key=io.bus) // // --- // type: string // default: `virtio` // managed: no // shortdesc: Override the bus for the device (can be `virtio` or `usb`) (VM only) "io.bus", // gendoc:generate(entity=devices, group=nic_macvlan, key=attached) // // --- // type: bool // default: `true` // required: no // shortdesc: Whether the NIC is plugged in or not "attached", // gendoc:generate(entity=devices, group=nic_macvlan, key=connected) // // --- // type: bool // default: `true` // required: no // shortdesc: Whether the NIC is connected to the host network (VM only) "connected", } // Check that if network proeperty is set that conflicting keys are not present. if d.config["network"] != "" { requiredFields = append(requiredFields, "network") bannedKeys := []string{"nictype", "parent", "mtu", "vlan", "gvrp", "mode"} for _, bannedKey := range bannedKeys { if d.config[bannedKey] != "" { return fmt.Errorf("Cannot use %q property in conjunction with %q property", bannedKey, "network") } } // If network property is specified, lookup network settings and apply them to the device's config. // api.ProjectDefaultName is used here as macvlan networks don't support projects. var err error d.network, err = network.LoadByName(d.state, api.ProjectDefaultName, d.config["network"]) if err != nil { return fmt.Errorf("Error loading network config for %q: %w", d.config["network"], err) } if d.network.Status() != api.NetworkStatusCreated { return errors.New("Specified network is not fully created") } if d.network.Type() != "macvlan" { return errors.New("Specified network must be of type macvlan") } netConfig := d.network.Config() // Get actual parent device from network's parent setting. d.config["parent"] = netConfig["parent"] // Copy certain keys verbatim from the network's settings. inheritKeys := []string{"mtu", "vlan", "gvrp"} for _, inheritKey := range inheritKeys { _, found := netConfig[inheritKey] if found { d.config[inheritKey] = netConfig[inheritKey] } } } else { // If no network property supplied, then parent property is required. requiredFields = append(requiredFields, "parent") } if instConf.Type() != instancetype.VM && d.config["connected"] != "" { return errors.New("The \"connected\" option is only supported on virtual machines for macvlan NICs") } err := d.config.Validate(nicValidationRules(requiredFields, optionalFields, instConf)) if err != nil { return err } return nil } // PreStartCheck checks the managed parent network is available (if relevant). func (d *nicMACVLAN) PreStartCheck() error { // Non-managed network NICs are not relevant for checking managed network availability. if d.network == nil { return nil } // If managed network is not available, don't try and start instance. if d.network.LocalStatus() == api.NetworkStatusUnavailable { return api.StatusErrorf(http.StatusServiceUnavailable, "Network %q unavailable on this server", d.network.Name()) } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *nicMACVLAN) validateEnvironment() error { if d.inst.Type() == instancetype.Container && d.config["name"] == "" { return errors.New("Requires name property to start") } if !util.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["parent"])) { return fmt.Errorf("Parent device '%s' doesn't exist", d.config["parent"]) } return nil } // Start is run when the device is added to a running instance or instance is starting up. func (d *nicMACVLAN) Start() (*deviceConfig.RunConfig, error) { // Ignore detached NICs. if !util.IsTrueOrEmpty(d.config["attached"]) { return nil, nil } err := d.validateEnvironment() if err != nil { return nil, err } // Lock to avoid issues with containers starting in parallel. networkCreateSharedDeviceLock.Lock() defer networkCreateSharedDeviceLock.Unlock() reverter := revert.New() defer reverter.Fail() saveData := make(map[string]string) // Decide which parent we should use based on VLAN setting. actualParentName := network.GetHostDevice(d.config["parent"], d.config["vlan"]) // Record the temporary device name used for deletion later. saveData["host_name"], err = d.generateHostName("mac", d.config["hwaddr"]) if err != nil { return nil, err } // Create VLAN parent device if needed. statusDev, err := networkCreateVlanDeviceIfNeeded(d.state, d.config["parent"], actualParentName, d.config["vlan"], util.IsTrue(d.config["gvrp"])) if err != nil { return nil, err } // Record whether we created the parent device or not so it can be removed on stop. saveData["last_state.created"] = fmt.Sprintf("%t", statusDev != "existing") if util.IsTrue(saveData["last_state.created"]) { reverter.Add(func() { _ = networkRemoveInterfaceIfNeeded(d.state, actualParentName, d.inst, d.config["parent"], d.config["vlan"]) }) } // Create MACVLAN interface. link := &ip.Macvlan{ Link: ip.Link{ Name: saveData["host_name"], Parent: actualParentName, }, } mode := d.config["mode"] if mode != "" { // Validate the provided mode. switch mode { case "bridge", "vepa", "passthru", "private": link.Mode = mode default: return nil, fmt.Errorf("Invalid MACVLAN mode specified: %q", mode) } } else { // Default to bridge mode if not specified. link.Mode = "bridge" } // Set the MAC address. if d.config["hwaddr"] != "" { hwaddr, err := net.ParseMAC(d.config["hwaddr"]) if err != nil { return nil, fmt.Errorf("Failed parsing MAC address %q: %w", d.config["hwaddr"], err) } link.Address = hwaddr } // Set the MTU. if d.config["mtu"] != "" { mtu, err := strconv.ParseUint(d.config["mtu"], 10, 32) if err != nil { return nil, fmt.Errorf("Invalid MTU specified %q: %w", d.config["mtu"], err) } link.MTU = uint32(mtu) } if d.inst.Type() == instancetype.VM { // Enable all multicast processing which is required for IPv6 NDP functionality. link.AllMulticast = true // Bring the interface up on host side. link.Up = true // Create macvtap interface using common macvlan settings. link := &ip.Macvtap{ Macvlan: *link, } err = link.Add() if err != nil { return nil, err } } else { // Create macvlan interface. err = link.Add() if err != nil { return nil, err } } reverter.Add(func() { _ = network.InterfaceRemove(saveData["host_name"]) }) if d.inst.Type() == instancetype.VM { // Disable IPv6 on host interface to avoid getting IPv6 link-local addresses unnecessarily. err = localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", link.Name), "1") if err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, fmt.Errorf("Failed to disable IPv6 on host interface %q: %w", link.Name, err) } } err = d.volatileSet(saveData) if err != nil { return nil, err } runConf := deviceConfig.RunConfig{} runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "type", Value: "phys"}, {Key: "name", Value: d.config["name"]}, {Key: "flags", Value: "up"}, {Key: "link", Value: saveData["host_name"]}, {Key: "hwaddr", Value: d.config["hwaddr"]}, {Key: "connected", Value: d.config["connected"]}, } if d.config["io.bus"] == "usb" { runConf.UseUSBBus = true } if d.inst.Type() == instancetype.VM { runConf.NetworkInterface = append(runConf.NetworkInterface, []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "mtu", Value: d.config["mtu"]}, }...) } reverter.Success() return &runConf, nil } // Stop is run when the device is removed from the instance. func (d *nicMACVLAN) Stop() (*deviceConfig.RunConfig, error) { v := d.volatileGet() runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } if util.IsTrueOrEmpty(d.config["attached"]) { runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "link", Value: v["host_name"]}, } } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *nicMACVLAN) postStop() error { defer func() { _ = d.volatileSet(map[string]string{ "host_name": "", "last_state.hwaddr": "", "last_state.mtu": "", "last_state.created": "", }) }() errs := []error{} v := d.volatileGet() // Delete the detached device. if v["host_name"] != "" && util.PathExists(fmt.Sprintf("/sys/class/net/%s", v["host_name"])) { err := network.InterfaceRemove(v["host_name"]) if err != nil { errs = append(errs, err) } } // This will delete the parent interface if we created it for VLAN parent. if util.IsTrue(v["last_state.created"]) { actualParentName := network.GetHostDevice(d.config["parent"], d.config["vlan"]) err := networkRemoveInterfaceIfNeeded(d.state, actualParentName, d.inst, d.config["parent"], d.config["vlan"]) if err != nil { errs = append(errs, err) } } if len(errs) > 0 { return fmt.Errorf("%v", errs) } return nil } // UpdatableFields returns a list of fields that can be updated without triggering a device remove & add. func (d *nicMACVLAN) UpdatableFields(oldDevice Type) []string { // Check old and new device types match. _, match := oldDevice.(*nicMACVLAN) if !match { return []string{} } return []string{"connected"} } // Update applies configuration changes to a started device. func (d *nicMACVLAN) Update(oldDevices deviceConfig.Devices, isRunning bool) error { if isRunning { return d.setNICLink() } return nil } incus-7.0.0/internal/server/device/nic_ovn.go000066400000000000000000001501621517523235500212170ustar00rootroot00000000000000package device import ( "bytes" "context" "errors" "fmt" "io/fs" "net" "net/http" "os" "slices" "strconv" "strings" "github.com/mdlayher/netx/eui64" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" pcidev "github.com/lxc/incus/v7/internal/server/device/pci" "github.com/lxc/incus/v7/internal/server/dnsmasq/dhcpalloc" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/network/acl" addressset "github.com/lxc/incus/v7/internal/server/network/address-set" "github.com/lxc/incus/v7/internal/server/network/ovn" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // ovnNet defines an interface for accessing instance specific functions on OVN network. type ovnNet interface { network.Network InstanceDevicePortValidateExternalRoutes(deviceInstance instance.Instance, deviceName string, externalRoutes []*net.IPNet) error InstanceDevicePortAdd(instanceUUID string, deviceName string, deviceConfig deviceConfig.Device) error InstanceDevicePortStart(opts *network.OVNInstanceNICSetupOpts, securityACLsRemove []string) (ovn.OVNSwitchPort, []net.IP, error) InstanceDevicePortStop(ovsExternalOVNPort ovn.OVNSwitchPort, opts *network.OVNInstanceNICStopOpts) error InstanceDevicePortRemove(instanceUUID string, devName string, devConfig deviceConfig.Device, hasDuplicate bool) error InstanceDevicePortIPs(instanceUUID string, deviceName string) ([]net.IP, error) } type nicOVN struct { deviceCommon network ovnNet // Populated in validateConfig(). ovnnb *ovn.NB ovnsb *ovn.SB } // CanHotPlug returns whether the device can be managed whilst the instance is running. func (d *nicOVN) CanHotPlug() bool { return true } // CanMigrate returns whether the device can be migrated to any other cluster member. func (d *nicOVN) CanMigrate() bool { return true } // UpdatableFields returns a list of fields that can be updated without triggering a device remove & add. func (d *nicOVN) UpdatableFields(oldDevice Type) []string { // Check old and new device types match. _, match := oldDevice.(*nicOVN) if !match { return []string{} } return []string{"security.acls", "limits.ingress", "limits.egress", "limits.max", "limits.priority", "connected"} } // validateConfig checks the supplied config for correctness. func (d *nicOVN) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container, instancetype.VM) { return ErrUnsupportedDevType } requiredFields := []string{ // gendoc:generate(entity=devices, group=nic_ovn, key=network) // // --- // type: string // managed: yes // shortdesc: The managed network to link the device to (required) "network", } optionalFields := []string{ // gendoc:generate(entity=devices, group=nic_ovn, key=name) // // --- // type: string // default: kernel assigned // managed: no // shortdesc: The name of the interface inside the instance "name", // gendoc:generate(entity=devices, group=nic_ovn, key=hwaddr) // // --- // type: string // default: randomly assigned // managed: no // shortdesc: The MAC address of the new interface "hwaddr", // gendoc:generate(entity=devices, group=nic_ovn, key=host_name) // // --- // type: string // default: randomly assigned // managed: no // shortdesc: The name of the interface inside the host "host_name", // gendoc:generate(entity=devices, group=nic_ovn, key=mtu) // // --- // type: integer // default: MTU of the parent network // managed: yes // shortdesc: The Maximum Transmit Unit (MTU) of the new interface "mtu", // gendoc:generate(entity=devices, group=nic_ovn, key=ipv4.address) // // --- // type: string // managed: no // shortdesc: An IPv4 address to assign to the instance through DHCP, `none` can be used to disable IP allocation "ipv4.address", // gendoc:generate(entity=devices, group=nic_ovn, key=ipv6.address) // // --- // type: string // managed: no // shortdesc: An IPv6 address to assign to the instance through DHCP, `none` can be used to disable IP allocation "ipv6.address", // gendoc:generate(entity=devices, group=nic_ovn, key=ipv4.address.external) // // --- // type: string // managed: no // shortdesc: Select a specific external address (typically from a network forward) "ipv4.address.external", // gendoc:generate(entity=devices, group=nic_ovn, key=ipv6.address.external) // // --- // type: string // managed: no // shortdesc: Select a specific external address (typically from a network forward) "ipv6.address.external", // gendoc:generate(entity=devices, group=nic_ovn, key=ipv4.routes) // // --- // type: string // managed: no // shortdesc: Comma-delimited list of IPv4 static routes to route to the NIC "ipv4.routes", // gendoc:generate(entity=devices, group=nic_ovn, key=ipv6.routes) // // --- // type: string // managed: no // shortdesc: Comma-delimited list of IPv6 static routes to route to the NIC "ipv6.routes", // gendoc:generate(entity=devices, group=nic_ovn, key=ipv4.routes.external) // // --- // type: string // managed: no // shortdesc: Comma-delimited list of IPv4 static routes to route to the NIC and publish on uplink network "ipv4.routes.external", // gendoc:generate(entity=devices, group=nic_ovn, key=ipv6.routes.external) // // --- // type: string // managed: no // shortdesc: Comma-delimited list of IPv6 static routes to route to the NIC and publish on uplink network "ipv6.routes.external", // gendoc:generate(entity=devices, group=nic_ovn, key=boot.priority) // // --- // type: integer // managed: no // shortdesc: Boot priority for VMs (higher value boots first) "boot.priority", // gendoc:generate(entity=devices, group=nic_ovn, key=security.acls) // // --- // type: string // managed: no // shortdesc: Comma-separated list of network ACLs to apply "security.acls", // gendoc:generate(entity=devices, group=nic_ovn, key=security.acls.default.ingress.action) // // --- // type: string // default: reject // managed: no // shortdesc: Action to use for ingress traffic that doesn't match any ACL rule "security.acls.default.ingress.action", // gendoc:generate(entity=devices, group=nic_ovn, key=security.acls.default.egress.action) // // --- // type: string // default: reject // managed: no // shortdesc: Action to use for egress traffic that doesn't match any ACL rule "security.acls.default.egress.action", // gendoc:generate(entity=devices, group=nic_ovn, key=security.acls.default.ingress.logged) // // --- // type: bool // default: false // managed: no // shortdesc: Whether to log ingress traffic that doesn't match any ACL rule "security.acls.default.ingress.logged", // gendoc:generate(entity=devices, group=nic_ovn, key=security.acls.default.egress.logged) // // --- // type: bool // default: false // managed: no // shortdesc: Whether to log egress traffic that doesn't match any ACL rule "security.acls.default.egress.logged", // gendoc:generate(entity=devices, group=nic_ovn, key=security.promiscuous) // // --- // type: bool // default: false // managed: no // shortdesc: Have OVN send unknown network traffic to this network interface (required for some nesting cases) "security.promiscuous", // gendoc:generate(entity=devices, group=nic_ovn, key=acceleration) // // --- // type: string // default: none // managed: no // shortdesc: Enable hardware offloading (either `none`, `sriov` or `vdpa`) "acceleration", // gendoc:generate(entity=devices, group=nic_ovn, key=nested) // // --- // type: string // managed: no // shortdesc: The parent NIC name to nest this NIC under (see also `vlan`) "nested", // gendoc:generate(entity=devices, group=nic_ovn, key=vlan) // // --- // type: integer // managed: no // shortdesc: The VLAN ID to use when nesting (see also `nested`) "vlan", // gendoc:generate(entity=devices, group=nic_ovn, key=limits.ingress) // // --- // type: string // managed: no // shortdesc: I/O limit in bit/s for incoming traffic (various suffixes supported, see {ref}`instances-limit-units`) "limits.ingress", // gendoc:generate(entity=devices, group=nic_ovn, key=limits.egress) // // --- // type: string // managed: no // shortdesc: I/O limit in bit/s for outgoing traffic (various suffixes supported, see {ref}`instances-limit-units`) "limits.egress", // gendoc:generate(entity=devices, group=nic_ovn, key=limits.max) // // --- // type: string // managed: no // shortdesc: I/O limit in bit/s for both incoming and outgoing traffic. (same as setting both limits.ingress and limits.egress / mutually exclusive with limits.ingress and limits.egress) "limits.max", // gendoc:generate(entity=devices, group=nic_ovn, key=limits.priority) // // --- // default: 100 // type: integer // managed: no // shortdesc: The priority for outgoing traffic, to be used by the kernel queuing discipline to prioritize network packets "limits.priority", // gendoc:generate(entity=devices, group=nic_ovn, key=attached) // // --- // type: bool // default: `true` // required: no // shortdesc: Whether the NIC is plugged in or not "attached", // gendoc:generate(entity=devices, group=nic_ovn, key=connected) // // --- // type: bool // default: `true` // required: no // shortdesc: Whether the NIC is connected to the host network (requires `acceleration` set to `none`) "connected", // gendoc:generate(entity=devices, group=nic_ovn, key=io.bus) // // --- // type: string // default: `virtio` // managed: no // shortdesc: Override the bus for the device (can be `virtio` or `usb`, requires `acceleration` set to `none`) (VM only) "io.bus", } // The NIC's network may be a non-default project, so lookup project and get network's project name. networkProjectName, _, err := project.NetworkProject(d.state.DB.Cluster, instConf.Project().Name) if err != nil { return fmt.Errorf("Failed loading network project name: %w", err) } // Lookup network settings and apply them to the device's config. n, err := network.LoadByName(d.state, networkProjectName, d.config["network"]) if err != nil { return fmt.Errorf("Error loading network config for %q: %w", d.config["network"], err) } if n.Status() != api.NetworkStatusCreated { return errors.New("Specified network is not fully created") } if n.Type() != "ovn" { return errors.New("Specified network must be of type ovn") } bannedKeys := []string{"mtu"} for _, bannedKey := range bannedKeys { if d.config[bannedKey] != "" { return fmt.Errorf("Cannot use %q property in conjunction with %q property", bannedKey, "network") } } ovnNet, ok := n.(ovnNet) if !ok { return errors.New("Network is not ovnNet interface type") } d.network = ovnNet // Stored loaded network for use by other functions. netConfig := d.network.Config() if d.config["ipv4.address"] != "" && d.config["ipv4.address"] != "none" { ip, subnet, err := net.ParseCIDR(netConfig["ipv4.address"]) if err != nil { return fmt.Errorf("Invalid network ipv4.address: %w", err) } // Check the static IP supplied is valid for the linked network. It should be part of the // network's subnet, but not necessarily part of the dynamic allocation ranges. if !dhcpalloc.DHCPValidIP(subnet, nil, net.ParseIP(d.config["ipv4.address"])) { return fmt.Errorf("Device IP address %q not within network %q subnet", d.config["ipv4.address"], d.config["network"]) } // IP should not be the same as the parent managed network address. if ip.Equal(net.ParseIP(d.config["ipv4.address"])) { return fmt.Errorf("IP address %q is assigned to parent managed network device %q", d.config["ipv4.address"], d.config["parent"]) } } if d.config["ipv6.address"] != "" && d.config["ipv6.address"] != "none" { // Static IPv6 is allowed only if static IPv4 is set as well. if d.config["ipv4.address"] == "" { return fmt.Errorf("Cannot specify %q when %q is not set", "ipv6.address", "ipv4.address") } ip, subnet, err := net.ParseCIDR(netConfig["ipv6.address"]) if err != nil { return fmt.Errorf("Invalid network ipv6.address: %w", err) } // Check the static IP supplied is valid for the linked network. It should be part of the // network's subnet, but not necessarily part of the dynamic allocation ranges. if !dhcpalloc.DHCPValidIP(subnet, nil, net.ParseIP(d.config["ipv6.address"])) { return fmt.Errorf("Device IP address %q not within network %q subnet", d.config["ipv6.address"], d.config["network"]) } // IP should not be the same as the parent managed network address. if ip.Equal(net.ParseIP(d.config["ipv6.address"])) { return fmt.Errorf("IP address %q is assigned to parent managed network device %q", d.config["ipv6.address"], d.config["parent"]) } } // Apply network level config options to device config before validation. d.config["mtu"] = netConfig["bridge.mtu"] // Check VLAN ID is valid. if d.config["vlan"] != "" { nestedVLAN, err := strconv.ParseUint(d.config["vlan"], 10, 16) if err != nil { return fmt.Errorf("Invalid VLAN ID %q: %w", d.config["vlan"], err) } if nestedVLAN < 1 || nestedVLAN > 4095 { return fmt.Errorf("Invalid VLAN ID %q: Must be between 1 and 4095 inclusive", d.config["vlan"]) } } // Perform checks that require instance (those not appropriate to do during profile validation). if d.inst != nil { // Check nested VLAN combination settings are valid. Requires instance for validation as settings // may come from a combination of profile and instance configs. if d.config["nested"] != "" { if d.config["vlan"] == "" { return errors.New("VLAN must be specified with a nested NIC") } // Check the NIC that this NIC is neted under exists on this instance and shares same // parent network. var nestedParentNIC string for devName, devConfig := range instConf.ExpandedDevices() { if devName != d.config["nested"] || devConfig["type"] != "nic" { continue } if devConfig["network"] != d.config["network"] { return errors.New("The nested parent NIC must be connected to same network as this NIC") } nestedParentNIC = devName break } if nestedParentNIC == "" { return fmt.Errorf("Instance does not have a NIC called %q for nesting under", d.config["nested"]) } } else if d.config["vlan"] != "" { return errors.New("Specifying a VLAN requires that this NIC be nested") } // Check there isn't another NIC with any of the same addresses specified on the same network. // Can only validate this when the instance is supplied (and not doing profile validation). err := d.checkAddressConflict() if err != nil { return err } } rules := nicValidationRules(requiredFields, optionalFields, instConf) // Override ipv4.address and ipv6.address to allow none value. rules["ipv4.address"] = validate.Optional(func(value string) error { if value == "none" { return nil } return validate.IsNetworkAddressV4(value) }) rules["ipv6.address"] = validate.Optional(func(value string) error { if value == "none" { return nil } return validate.IsNetworkAddressV6(value) }) // Validate the external address against the list of network forwards. isNetworkForward := func(value string) error { return d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { netID, _, _, err := tx.GetNetworkInAnyState(ctx, networkProjectName, d.config["network"]) if err != nil { return fmt.Errorf("Failed getting network ID: %w", err) } _, err = dbCluster.GetNetworkForward(ctx, tx.Tx(), netID, value) if err != nil { return fmt.Errorf("External address %q is not a network forward on network %q: %w", value, d.config["network"], err) } return nil }) } rules["ipv4.address.external"] = validate.Optional(validate.And(validate.IsNetworkAddressV4, isNetworkForward)) rules["ipv6.address.external"] = validate.Optional(validate.And(validate.IsNetworkAddressV6, isNetworkForward)) // Now run normal validation. err = d.config.Validate(rules) if err != nil { return err } // Check IP external routes are within the network's external routes. var externalRoutes []*net.IPNet for _, k := range []string{"ipv4.routes.external", "ipv6.routes.external"} { if d.config[k] == "" { continue } externalRoutes, err = network.SubnetParseAppend(externalRoutes, util.SplitNTrimSpace(d.config[k], ",", -1, false)...) if err != nil { return err } } if len(externalRoutes) > 0 { err = d.network.InstanceDevicePortValidateExternalRoutes(d.inst, d.name, externalRoutes) if err != nil { return err } } // Check Security ACLs exist. if d.config["security.acls"] != "" { err = acl.Exists(d.state, networkProjectName, util.SplitNTrimSpace(d.config["security.acls"], ",", -1, true)...) if err != nil { return err } } // Avoid setting both ingress/egress and max to avoid confusion or implicit behavior. if d.config["limits.max"] != "" && (d.config["limits.ingress"] != "" || d.config["limits.egress"] != "") { return errors.New("limits.max is mutually exclusive with limits.ingress and limits.egress") } if d.config["limits.priority"] != "" { priority, err := strconv.Atoi(d.config["limits.priority"]) if err != nil { return errors.New("limits.priority must be an integer") } if priority < 0 || priority > 32767 { return errors.New("limits.priority must be between 0 an 32767, inclusive") } } if d.config["limits.max"] != "" { limitsMax, err := units.ParseBitSizeString(d.config["limits.max"]) if err != nil { return errors.New("limits.max must be an integer") } limitsMax /= 1000 // Convert to kbps if limitsMax < 1 || limitsMax > 4294967295 { return errors.New("limits.max must be between 1 an 4294967295 bps, inclusive") } } if d.config["limits.ingress"] != "" { ingress, err := units.ParseBitSizeString(d.config["limits.ingress"]) if err != nil { return errors.New("limits.ingress must be an integer") } ingress /= 1000 // Convert to kbps if ingress < 1 || ingress > 4294967295 { return errors.New("limits.ingress must be between 1 an 4294967295 bps, inclusive") } } if d.config["limits.egress"] != "" { egress, err := units.ParseBitSizeString(d.config["limits.egress"]) if err != nil { return errors.New("limits.egress must be an integer") } egress /= 1000 // Convert to kbps if egress < 1 || egress > 4294967295 { return errors.New("limits.egress must be between 1 an 4294967295 bps, inclusive") } } if !d.isVirtualNIC() { // The connected option can only be handled properly if acceleration is set to none. if d.config["connected"] != "" { return errors.New("The \"connected\" option requires setting acceleration=none for OVN NICs") } // The bus can only be set if we actually have control over the guest device. if d.config["io.bus"] != "" { return errors.New("The \"io.bus\" option requires setting acceleration=none for OVN NICs") } } return nil } // checkAddressConflict checks for conflicting IP/MAC addresses on another NIC connected to same network. // Can only validate this when the instance is supplied (and not doing profile validation). // Returns api.StatusError with status code set to http.StatusConflict if conflicting address found. func (d *nicOVN) checkAddressConflict() error { ourNICIPs := make(map[string]net.IP, 2) ourNICIPs["ipv4.address"] = net.ParseIP(d.config["ipv4.address"]) ourNICIPs["ipv6.address"] = net.ParseIP(d.config["ipv6.address"]) // Shortcut when no IP needs to be assigned. if ourNICIPs["ipv4.address"] == nil && ourNICIPs["ipv6.address"] == nil { return nil } ourNICMAC, _ := net.ParseMAC(d.config["hwaddr"]) if ourNICMAC == nil { ourNICMAC, _ = net.ParseMAC(d.volatileGet()["hwaddr"]) } // Check if any instance devices use this network. return network.UsedByInstanceDevices(d.state, d.network.Project(), d.network.Name(), d.network.Type(), func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error { // Skip our own device. This avoids triggering duplicate device errors during // updates or when making temporary copies of our instance during migrations. sameLogicalInstance := instance.IsSameLogicalInstance(d.inst, &inst) if sameLogicalInstance && d.Name() == nicName { return nil } // Check there isn't another instance with the same DNS name connected to managed network. sameLogicalInstanceNestedNIC := sameLogicalInstance && (d.config["nested"] != "" || nicConfig["nested"] != "") if d.network != nil && !sameLogicalInstanceNestedNIC && nicCheckDNSNameConflict(d.inst.Name(), inst.Name) { if sameLogicalInstance { return api.StatusErrorf(http.StatusConflict, "Instance DNS name %q conflict between %q and %q because both are connected to same network", strings.ToLower(inst.Name), d.name, nicName) } return api.StatusErrorf(http.StatusConflict, "Instance DNS name %q already used on network", strings.ToLower(inst.Name)) } // Check NIC's MAC address doesn't match this NIC's MAC address. devNICMAC, _ := net.ParseMAC(nicConfig["hwaddr"]) if devNICMAC == nil { devNICMAC, _ = net.ParseMAC(inst.Config[fmt.Sprintf("volatile.%s.hwaddr", nicName)]) } if ourNICMAC != nil && devNICMAC != nil && bytes.Equal(ourNICMAC, devNICMAC) { return api.StatusErrorf(http.StatusConflict, "MAC address %q already defined on another NIC", devNICMAC.String()) } // Check NIC's static IPs don't match this NIC's static IPs. for _, key := range []string{"ipv4.address", "ipv6.address"} { if d.config[key] == "" { continue // No static IP specified on this NIC. } // Parse IPs to avoid being tripped up by presentation differences. devNICIP := net.ParseIP(nicConfig[key]) if ourNICIPs[key] != nil && devNICIP != nil && ourNICIPs[key].Equal(devNICIP) { return api.StatusErrorf(http.StatusConflict, "IP address %q already defined on another NIC", devNICIP.String()) } } return nil }) } // Add is run when a device is added to a non-snapshot instance whether or not the instance is running. func (d *nicOVN) Add() error { return d.network.InstanceDevicePortAdd(d.inst.LocalConfig()["volatile.uuid"], d.name, d.config) } // PreStartCheck checks the managed parent network is available (if relevant). func (d *nicOVN) PreStartCheck() error { // Non-managed network NICs are not relevant for checking managed network availability. if d.network == nil { return nil } // If managed network is not available, don't try and start instance. if d.network.LocalStatus() == api.NetworkStatusUnavailable { return api.StatusErrorf(http.StatusServiceUnavailable, "Network %q unavailable on this server", d.network.Name()) } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *nicOVN) validateEnvironment() error { if d.inst.Type() == instancetype.Container && d.config["name"] == "" { return errors.New("Requires name property to start") } integrationBridge := d.state.GlobalConfig.NetworkOVNIntegrationBridge() if !util.PathExists(fmt.Sprintf("/sys/class/net/%s", integrationBridge)) { return fmt.Errorf("OVS integration bridge device %q doesn't exist", integrationBridge) } return nil } func (d *nicOVN) init(inst instance.Instance, s *state.State, name string, conf deviceConfig.Device, volatileGet VolatileGetter, volatileSet VolatileSetter) error { // Check that OVN is available. ovnnb, ovnsb, err := s.OVN() if err != nil { return err } d.ovnnb = ovnnb d.ovnsb = ovnsb return d.deviceCommon.init(inst, s, name, conf, volatileGet, volatileSet) } // Start is run when the device is added to a running instance or instance is starting up. func (d *nicOVN) Start() (*deviceConfig.RunConfig, error) { // Ignore detached NICs. if !util.IsTrueOrEmpty(d.config["attached"]) { return nil, nil } err := d.validateEnvironment() if err != nil { return nil, err } reverter := revert.New() defer reverter.Fail() saveData := make(map[string]string) saveData["host_name"] = d.config["host_name"] // Load uplink network config. uplinkNetworkName := d.network.Config()["network"] var uplink *api.Network var uplinkConfig map[string]string if uplinkNetworkName != "none" { err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, uplink, _, err = tx.GetNetworkInAnyState(ctx, api.ProjectDefaultName, uplinkNetworkName) return err }) if err != nil { return nil, fmt.Errorf("Failed to load uplink network %q: %w", uplinkNetworkName, err) } uplinkConfig = uplink.Config } // Setup the host network interface (if not nested). var peerName, integrationBridgeNICName string var mtu uint32 var vfPCIDev pcidev.Device var vDPADevice *ip.VDPADev var pciIOMMUGroup uint64 if d.config["nested"] != "" { delete(saveData, "host_name") // Nested NICs don't have a host side interface. } else { if d.config["acceleration"] == "sriov" { vswitch, err := d.state.OVS() if err != nil { return nil, fmt.Errorf("Failed to connect to OVS: %w", err) } offload, err := vswitch.GetHardwareOffload(context.TODO()) if err != nil { return nil, err } if !offload { return nil, errors.New("SR-IOV acceleration requires hardware offloading be enabled in OVS") } // If VM, then try and load the vfio-pci module first. if d.inst.Type() == instancetype.VM { err := linux.LoadModule("vfio-pci") if err != nil { return nil, fmt.Errorf("Error loading %q module: %w", "vfio-pci", err) } } integrationBridge := d.state.GlobalConfig.NetworkOVNIntegrationBridge() // Find free VF exclusively. network.SRIOVVirtualFunctionMutex.Lock() vfParent, vfRepresentor, vfDev, vfID, err := network.SRIOVFindFreeVFAndRepresentor(d.state, integrationBridge) if err != nil { network.SRIOVVirtualFunctionMutex.Unlock() return nil, fmt.Errorf("Failed finding a suitable free virtual function on %q: %w", integrationBridge, err) } // Claim the SR-IOV virtual function (VF) on the parent (PF) and get the PCI information. vfPCIDev, pciIOMMUGroup, err = networkSRIOVSetupVF(d.deviceCommon, vfParent, vfDev, vfID, saveData) if err != nil { network.SRIOVVirtualFunctionMutex.Unlock() return nil, fmt.Errorf("Failed setting up VF: %w", err) } reverter.Add(func() { _ = networkSRIOVRestoreVF(d.deviceCommon, false, saveData) }) network.SRIOVVirtualFunctionMutex.Unlock() // Setup the guest network interface. if d.inst.Type() == instancetype.Container { err := networkSRIOVSetupContainerVFNIC(saveData["host_name"], d.inst.MACPattern(), d.config) if err != nil { return nil, fmt.Errorf("Failed setting up container VF NIC: %w", err) } } integrationBridgeNICName = vfRepresentor peerName = vfDev } else if d.config["acceleration"] == "vdpa" { vswitch, err := d.state.OVS() if err != nil { return nil, fmt.Errorf("Failed to connect to OVS: %w", err) } offload, err := vswitch.GetHardwareOffload(context.TODO()) if err != nil { return nil, err } if !offload { return nil, errors.New("SR-IOV acceleration requires hardware offloading be enabled in OVS") } err = linux.LoadModule("vdpa") if err != nil { return nil, fmt.Errorf("Error loading %q module: %w", "vdpa", err) } // If VM, then try and load the vhost_vdpa module first. if d.inst.Type() == instancetype.VM { err = linux.LoadModule("vhost_vdpa") if err != nil { return nil, fmt.Errorf("Error loading %q module: %w", "vhost_vdpa", err) } } integrationBridge := d.state.GlobalConfig.NetworkOVNIntegrationBridge() // Find free VF exclusively. network.SRIOVVirtualFunctionMutex.Lock() vfParent, vfRepresentor, vfDev, vfID, err := network.SRIOVFindFreeVFAndRepresentor(d.state, integrationBridge) if err != nil { network.SRIOVVirtualFunctionMutex.Unlock() return nil, fmt.Errorf("Failed finding a suitable free virtual function on %q: %w", integrationBridge, err) } // Claim the SR-IOV virtual function (VF) on the parent (PF) and get the PCI information. vfPCIDev, pciIOMMUGroup, err = networkSRIOVSetupVF(d.deviceCommon, vfParent, vfDev, vfID, saveData) if err != nil { network.SRIOVVirtualFunctionMutex.Unlock() return nil, err } reverter.Add(func() { _ = networkSRIOVRestoreVF(d.deviceCommon, false, saveData) }) // Create the vDPA management device vDPADevice, err = ip.AddVDPADevice(vfPCIDev.SlotName, saveData) if err != nil { network.SRIOVVirtualFunctionMutex.Unlock() return nil, err } network.SRIOVVirtualFunctionMutex.Unlock() // Setup the guest network interface. if d.inst.Type() == instancetype.Container { return nil, errors.New("VDPA acceleration is not supported for containers") } integrationBridgeNICName = vfRepresentor peerName = vfDev } else { // Create veth pair and configure the peer end with custom hwaddr and mtu if supplied. if d.inst.Type() == instancetype.Container { if saveData["host_name"] == "" { saveData["host_name"], err = d.generateHostName("veth", d.config["hwaddr"]) if err != nil { return nil, err } } integrationBridgeNICName = saveData["host_name"] peerName, mtu, err = networkCreateVethPair(saveData["host_name"], d.config) if err != nil { return nil, err } } else if d.inst.Type() == instancetype.VM { if saveData["host_name"] == "" { saveData["host_name"], err = d.generateHostName("tap", d.config["hwaddr"]) if err != nil { return nil, err } } integrationBridgeNICName = saveData["host_name"] peerName = saveData["host_name"] // VMs use the host_name to link to the TAP FD. mtu, err = networkCreateTap(saveData["host_name"], d.config) if err != nil { return nil, err } } reverter.Add(func() { _ = network.InterfaceRemove(saveData["host_name"]) }) } } // Populate device config with volatile fields if needed. networkVethFillFromVolatile(d.config, saveData) v := d.volatileGet() // Retrieve any last state IPs from volatile and pass them to OVN driver for potential use with sticky // DHCPv4 allocations. var lastStateIPs []net.IP for _, ipStr := range util.SplitNTrimSpace(v["last_state.ip_addresses"], ",", -1, true) { lastStateIP := net.ParseIP(ipStr) if lastStateIP != nil { lastStateIPs = append(lastStateIPs, lastStateIP) } } // Add new OVN logical switch port for instance. logicalPortName, dnsIPs, err := d.network.InstanceDevicePortStart(&network.OVNInstanceNICSetupOpts{ InstanceUUID: d.inst.LocalConfig()["volatile.uuid"], DNSName: d.inst.Name(), DeviceName: d.name, DeviceConfig: d.config, UplinkConfig: uplinkConfig, LastStateIPs: lastStateIPs, // Pass in volatile last state IPs for use with sticky DHCPv4 hint. }, nil) if err != nil { return nil, fmt.Errorf("Failed setting up OVN port: %w", err) } // Record switch port DNS IPs to volatile so they can be used as sticky DHCPv4 hint in the future in order // to allocate the same IPs on next start if they are still available/appropriate. // This volatile key will not be removed when instance stops. var dnsIPsStr strings.Builder for i, dnsIP := range dnsIPs { if i > 0 { dnsIPsStr.WriteString(",") } dnsIPsStr.WriteString(dnsIP.String()) } saveData["last_state.ip_addresses"] = dnsIPsStr.String() reverter.Add(func() { _ = d.network.InstanceDevicePortStop("", &network.OVNInstanceNICStopOpts{ InstanceUUID: d.inst.LocalConfig()["volatile.uuid"], DeviceName: d.name, DeviceConfig: d.config, }) }) // Associated host side interface to OVN logical switch port (if not nested). if integrationBridgeNICName != "" { cleanup, err := d.setupHostNIC(integrationBridgeNICName, logicalPortName) if err != nil { return nil, err } reverter.Add(cleanup) } runConf := deviceConfig.RunConfig{} // Get local chassis ID for chassis group. vswitch, err := d.state.OVS() if err != nil { return nil, fmt.Errorf("Failed to connect to OVS: %w", err) } chassisID, err := vswitch.GetChassisID(context.TODO()) if err != nil { return nil, fmt.Errorf("Failed getting OVS Chassis ID: %w", err) } // Add post start hook for setting logical switch port chassis once instance has been started. runConf.PostHooks = append(runConf.PostHooks, func() error { err := d.ovnnb.UpdateLogicalSwitchPortOptions(context.TODO(), logicalPortName, map[string]string{"requested-chassis": chassisID}) if err != nil { return fmt.Errorf("Failed setting logical switch port chassis ID: %w", err) } return nil }) runConf.PostHooks = append(runConf.PostHooks, d.postStart) err = d.volatileSet(saveData) if err != nil { return nil, err } // Return instance network interface configuration (if not nested). if saveData["host_name"] != "" { runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "type", Value: "phys"}, {Key: "name", Value: d.config["name"]}, {Key: "flags", Value: "up"}, {Key: "link", Value: peerName}, } if d.isVirtualNIC() { runConf.NetworkInterface = append(runConf.NetworkInterface, deviceConfig.RunConfigItem{Key: "connected", Value: d.config["connected"]}) } instType := d.inst.Type() if instType == instancetype.VM { runConf.NetworkInterface = append(runConf.NetworkInterface, []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "mtu", Value: fmt.Sprintf("%d", mtu)}, }...) if d.config["acceleration"] == "sriov" { runConf.NetworkInterface = append(runConf.NetworkInterface, []deviceConfig.RunConfigItem{ {Key: "pciSlotName", Value: vfPCIDev.SlotName}, {Key: "pciIOMMUGroup", Value: fmt.Sprintf("%d", pciIOMMUGroup)}, }...) } else if d.config["acceleration"] == "vdpa" { if vDPADevice == nil { return nil, errors.New("vDPA device is nil") } runConf.NetworkInterface = append(runConf.NetworkInterface, []deviceConfig.RunConfigItem{ {Key: "pciSlotName", Value: vfPCIDev.SlotName}, {Key: "pciIOMMUGroup", Value: fmt.Sprintf("%d", pciIOMMUGroup)}, {Key: "maxVQP", Value: fmt.Sprintf("%d", vDPADevice.MaxVQs/2)}, {Key: "vDPADevName", Value: vDPADevice.Name}, {Key: "vhostVDPAPath", Value: vDPADevice.VhostVDPA.Path}, }...) } else { runConf.NetworkInterface = append(runConf.NetworkInterface, []deviceConfig.RunConfigItem{ {Key: "hwaddr", Value: d.config["hwaddr"]}, }...) } } else if instType == instancetype.Container { runConf.NetworkInterface = append(runConf.NetworkInterface, deviceConfig.RunConfigItem{Key: "hwaddr", Value: d.config["hwaddr"]}, ) } if d.config["io.bus"] == "usb" { runConf.UseUSBBus = true } } reverter.Success() return &runConf, nil } // postStart is run after the device is added to the instance. func (d *nicOVN) postStart() error { err := bgpAddPrefix(&d.deviceCommon, d.network, d.config) if err != nil { return err } return nil } // Update applies configuration changes to a started device. func (d *nicOVN) Update(oldDevices deviceConfig.Devices, isRunning bool) error { oldConfig := oldDevices[d.name] // Populate device config with volatile fields if needed. networkVethFillFromVolatile(d.config, d.volatileGet()) // If an IPv6 address has changed, if the instance is running we should bounce the host-side // veth interface to give the instance a chance to detect the change and re-apply for an // updated lease with new IP address. if d.config["ipv6.address"] != oldConfig["ipv6.address"] && d.config["host_name"] != "" && network.InterfaceExists(d.config["host_name"]) { link := &ip.Link{Name: d.config["host_name"]} err := link.SetDown() if err != nil { return err } err = link.SetUp() if err != nil { return err } } // Apply any changes needed when assigned ACLs change. if d.config["security.acls"] != oldConfig["security.acls"] { // Work out which ACLs have been removed and remove logical port from those groups. oldACLs := util.SplitNTrimSpace(oldConfig["security.acls"], ",", -1, true) newACLs := util.SplitNTrimSpace(d.config["security.acls"], ",", -1, true) removedACLs := []string{} for _, oldACL := range oldACLs { if !slices.Contains(newACLs, oldACL) { removedACLs = append(removedACLs, oldACL) } } // Setup address sets for new ACLs _, err := addressset.OVNEnsureAddressSetsViaACLs(d.state, d.logger, d.ovnnb, d.network.Project(), newACLs) if err != nil { return fmt.Errorf("Failed removing unused OVN address sets: %w", err) } // Setup the logical port with new ACLs if running. if isRunning { // Load uplink network config. uplinkNetworkName := d.network.Config()["network"] var uplink *api.Network var uplinkConfig map[string]string if uplinkNetworkName != "none" { err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error _, uplink, _, err = tx.GetNetworkInAnyState(ctx, api.ProjectDefaultName, uplinkNetworkName) return err }) if err != nil { return fmt.Errorf("Failed to load uplink network %q: %w", uplinkNetworkName, err) } uplinkConfig = uplink.Config } // Update OVN logical switch port for instance. _, _, err := d.network.InstanceDevicePortStart(&network.OVNInstanceNICSetupOpts{ InstanceUUID: d.inst.LocalConfig()["volatile.uuid"], DNSName: d.inst.Name(), DeviceName: d.name, DeviceConfig: d.config, UplinkConfig: uplinkConfig, }, removedACLs) if err != nil { return fmt.Errorf("Failed updating OVN port: %w", err) } } if len(removedACLs) > 0 { err := addressset.OVNDeleteAddressSetsViaACLs(d.state, d.logger, d.ovnnb, d.network.Project(), removedACLs) if err != nil { return fmt.Errorf("Failed removing unused OVN address sets: %w", err) } err = acl.OVNPortGroupDeleteIfUnused(d.state, d.logger, d.ovnnb, d.network.Project(), d.inst, d.name, newACLs...) if err != nil { return fmt.Errorf("Failed removing unused OVN port groups: %w", err) } } } // If an external address changed, update the BGP advertisements. err := bgpRemovePrefix(&d.deviceCommon, oldConfig) if err != nil { return err } err = bgpAddPrefix(&d.deviceCommon, d.network, d.config) if err != nil { return err } if isRunning && d.isVirtualNIC() { return d.setNICLink() } return nil } func (d *nicOVN) findRepresentorPort(volatile map[string]string) (string, error) { physSwitchID, pfID, err := network.SRIOVGetSwitchAndPFID(volatile["last_state.vf.parent"]) if err != nil { return "", fmt.Errorf("Failed finding physical parent switch and PF ID to release representor port: %w", err) } sysClassNet := "/sys/class/net" nics, err := os.ReadDir(sysClassNet) if err != nil { return "", fmt.Errorf("Failed reading NICs directory %q: %w", sysClassNet, err) } vfID, err := strconv.Atoi(volatile["last_state.vf.id"]) if err != nil { return "", fmt.Errorf("Failed parsing last VF ID %q: %w", volatile["last_state.vf.id"], err) } // Track down the representor port to remove it from the integration bridge. representorPort := network.SRIOVFindRepresentorPort(nics, string(physSwitchID), pfID, vfID) if representorPort == "" { return "", errors.New("Failed finding representor") } return representorPort, nil } // Stop is run when the device is removed from the instance. func (d *nicOVN) Stop() (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } v := d.volatileGet() var err error // Try and retrieve the last associated OVN switch port for the instance interface in the local OVS DB. // If we cannot get this, don't fail, as InstanceDevicePortStop will then try and generate the likely // port name using the same regime it does for new ports. This part is only here in order to allow // instance ports generated under an older regime to be cleaned up properly. networkVethFillFromVolatile(d.config, v) vswitch, err := d.state.OVS() if err != nil { d.logger.Error("Failed to connect to OVS", logger.Ctx{"err": err}) } var ovsExternalOVNPort string if d.config["nested"] == "" { ovsExternalOVNPort, err = vswitch.GetInterfaceAssociatedOVNSwitchPort(context.TODO(), d.config["host_name"]) if err != nil { d.logger.Warn("Could not find OVN Switch port associated to OVS interface", logger.Ctx{"interface": d.config["host_name"]}) } } integrationBridgeNICName := d.config["host_name"] if d.config["acceleration"] == "sriov" || d.config["acceleration"] == "vdpa" { integrationBridgeNICName, err = d.findRepresentorPort(v) if err != nil { d.logger.Error("Failed finding representor port to detach from OVS integration bridge", logger.Ctx{"err": err}) } } // If there is integrationBridgeNICName specified, then try and remove it from the OVS integration bridge. // Do this early on during the stop process to prevent any future error from leaving the OVS port present // as if the instance is being migrated, this can cause port conflicts in OVN if the instance comes up on // another host later. if integrationBridgeNICName != "" { integrationBridge := d.state.GlobalConfig.NetworkOVNIntegrationBridge() // Detach host-side end of veth pair from OVS integration bridge. err = vswitch.DeleteBridgePort(context.TODO(), integrationBridge, integrationBridgeNICName) if err != nil { // Don't fail here as we want the postStop hook to run to clean up the local veth pair. d.logger.Error("Failed detaching interface from OVS integration bridge", logger.Ctx{"interface": integrationBridgeNICName, "bridge": integrationBridge, "err": err}) } } instanceUUID := d.inst.LocalConfig()["volatile.uuid"] err = d.network.InstanceDevicePortStop(ovn.OVNSwitchPort(ovsExternalOVNPort), &network.OVNInstanceNICStopOpts{ InstanceUUID: instanceUUID, DeviceName: d.name, DeviceConfig: d.config, }) if err != nil { // Don't fail here as we still want the postStop hook to run to clean up the local veth pair. d.logger.Error("Failed to remove OVN device port", logger.Ctx{"err": err}) } // Remove BGP announcements. err = bgpRemovePrefix(&d.deviceCommon, d.config) if err != nil { return nil, err } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *nicOVN) postStop() error { defer func() { _ = d.volatileSet(map[string]string{ "host_name": "", "last_state.hwaddr": "", "last_state.mtu": "", "last_state.created": "", "last_state.vdpa.name": "", "last_state.vf.parent": "", "last_state.vf.id": "", "last_state.vf.hwaddr": "", "last_state.vf.vlan": "", "last_state.vf.spoofcheck": "", "last_state.pci.driver": "", }) }() v := d.volatileGet() networkVethFillFromVolatile(d.config, v) if d.config["acceleration"] == "sriov" { // Restoring host-side interface. network.SRIOVVirtualFunctionMutex.Lock() err := networkSRIOVRestoreVF(d.deviceCommon, false, v) if err != nil { network.SRIOVVirtualFunctionMutex.Unlock() return err } network.SRIOVVirtualFunctionMutex.Unlock() link := &ip.Link{Name: d.config["host_name"]} err = link.SetDown() if err != nil { return fmt.Errorf("Failed to bring down the host interface %s: %w", d.config["host_name"], err) } } else if d.config["acceleration"] == "vdpa" { // Retrieve the last state vDPA device name. network.SRIOVVirtualFunctionMutex.Lock() vDPADevName, ok := v["last_state.vdpa.name"] if !ok { network.SRIOVVirtualFunctionMutex.Unlock() return errors.New("Failed to find PCI slot name for vDPA device") } // Delete the vDPA management device. err := ip.DeleteVDPADevice(vDPADevName) if err != nil { network.SRIOVVirtualFunctionMutex.Unlock() return err } // Restoring host-side interface. network.SRIOVVirtualFunctionMutex.Lock() err = networkSRIOVRestoreVF(d.deviceCommon, false, v) if err != nil { network.SRIOVVirtualFunctionMutex.Unlock() return err } network.SRIOVVirtualFunctionMutex.Unlock() link := &ip.Link{Name: d.config["host_name"]} err = link.SetDown() if err != nil { return fmt.Errorf("Failed to bring down the host interface %q: %w", d.config["host_name"], err) } } else if d.config["host_name"] != "" && util.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["host_name"])) { // Removing host-side end of veth pair will delete the peer end too. err := network.InterfaceRemove(d.config["host_name"]) if err != nil { return fmt.Errorf("Failed to remove interface %q: %w", d.config["host_name"], err) } } return nil } // Remove is run when the device is removed from the instance or the instance is deleted. func (d *nicOVN) Remove(cleanupDependencies bool) error { // Check for port groups that will become unused (and need deleting) as this NIC is deleted. securityACLs := util.SplitNTrimSpace(d.config["security.acls"], ",", -1, true) if len(securityACLs) > 0 { err := acl.OVNPortGroupDeleteIfUnused(d.state, d.logger, d.ovnnb, d.network.Project(), d.inst, d.name) if err != nil { return fmt.Errorf("Failed removing unused OVN port groups: %w", err) } } return d.network.InstanceDevicePortRemove(d.inst.LocalConfig()["volatile.uuid"], d.name, d.config, d.checkAddressConflict() != nil) } // State gets the state of an OVN NIC by querying the OVN Northbound logical switch port record. func (d *nicOVN) State() (*api.InstanceStateNetwork, error) { // Populate device config with volatile fields (hwaddr and host_name) if needed. networkVethFillFromVolatile(d.config, d.volatileGet()) addresses := []api.InstanceStateNetworkAddress{} netConfig := d.network.Config() // Extract subnet sizes from bridge addresses. _, v4subnet, _ := net.ParseCIDR(netConfig["ipv4.address"]) _, v6subnet, _ := net.ParseCIDR(netConfig["ipv6.address"]) var v4mask string if v4subnet != nil { mask, _ := v4subnet.Mask.Size() v4mask = fmt.Sprintf("%d", mask) } var v6mask string if v6subnet != nil { mask, _ := v6subnet.Mask.Size() v6mask = fmt.Sprintf("%d", mask) } // OVN only supports dynamic IP allocation if neither IPv4 or IPv6 are statically set. if d.config["ipv4.address"] == "" && d.config["ipv6.address"] == "" { instanceUUID := d.inst.LocalConfig()["volatile.uuid"] devIPs, err := d.network.InstanceDevicePortIPs(instanceUUID, d.name) if err == nil { for _, devIP := range devIPs { family := "inet" netmask := v4mask if devIP.To4() == nil { family = "inet6" netmask = v6mask } addresses = append(addresses, api.InstanceStateNetworkAddress{ Family: family, Address: devIP.String(), Netmask: netmask, Scope: "global", }) } } else { d.logger.Warn("Failed getting OVN port device IPs", logger.Ctx{"err": err}) } } else { if d.config["ipv4.address"] != "" && d.config["ipv4.address"] != "none" { // Static DHCPv4 allocation present, that is likely to be the NIC's IPv4. So assume that. addresses = append(addresses, api.InstanceStateNetworkAddress{ Family: "inet", Address: d.config["ipv4.address"], Netmask: v4mask, Scope: "global", }) } if d.config["ipv6.address"] != "" && d.config["ipv6.address"] != "none" { // Static DHCPv6 allocation present, that is likely to be the NIC's IPv6. So assume that. addresses = append(addresses, api.InstanceStateNetworkAddress{ Family: "inet6", Address: d.config["ipv6.address"], Netmask: v6mask, Scope: "global", }) } else if util.IsFalseOrEmpty(netConfig["ipv6.dhcp.stateful"]) && d.config["hwaddr"] != "" && v6subnet != nil { // If no static DHCPv6 allocation and stateful DHCPv6 is disabled, and IPv6 is enabled on // the bridge, the NIC is likely to use its MAC and SLAAC to configure its address. hwAddr, err := net.ParseMAC(d.config["hwaddr"]) if err == nil { ip, err := eui64.ParseMAC(v6subnet.IP, hwAddr) if err == nil { addresses = append(addresses, api.InstanceStateNetworkAddress{ Family: "inet6", Address: ip.String(), Netmask: v6mask, Scope: "global", }) } } } } n := api.InstanceStateNetwork{ Addresses: addresses, Hwaddr: d.config["hwaddr"], State: "up", Type: "broadcast", } // When not on a nested NIC, fetch some details from the host. if d.config["nested"] == "" { // Get MTU of host interface that connects to OVN integration bridge if exists. iface, err := net.InterfaceByName(d.config["host_name"]) if err != nil { d.logger.Warn("Failed getting host interface state for MTU", logger.Ctx{"host_name": d.config["host_name"], "err": err}) } mtu := -1 if iface != nil { mtu = iface.MTU } // Retrieve the host counters, as we report the values from the instance's point of view, // those counters need to be reversed below. hostCounters, err := resources.GetNetworkCounters(d.config["host_name"]) if err != nil { return nil, fmt.Errorf("Failed getting network interface counters: %w", err) } n.Counters = api.InstanceStateNetworkCounters{ BytesReceived: hostCounters.BytesSent, BytesSent: hostCounters.BytesReceived, PacketsReceived: hostCounters.PacketsSent, PacketsSent: hostCounters.PacketsReceived, } n.HostName = d.config["host_name"] n.Mtu = mtu } return &n, nil } // Register sets up anything needed on startup. func (d *nicOVN) Register() error { // Skip when not using a managed network. if d.config["network"] == "" { return nil } // The NIC's network may be a non-default project, so lookup project and get network's project name. networkProjectName, _, err := project.NetworkProject(d.state.DB.Cluster, d.inst.Project().Name) if err != nil { return fmt.Errorf("Failed loading network project name: %w", err) } // Lookup network settings and apply them to the device's config. n, err := network.LoadByName(d.state, networkProjectName, d.config["network"]) if err != nil { return fmt.Errorf("Error loading network config for %q: %w", d.config["network"], err) } err = bgpAddPrefix(&d.deviceCommon, n, d.config) if err != nil { return err } return nil } func (d *nicOVN) setupHostNIC(hostName string, ovnPortName ovn.OVNSwitchPort) (revert.Hook, error) { reverter := revert.New() defer reverter.Fail() // Disable IPv6 on host-side veth interface (prevents host-side interface getting link-local address and // accepting router advertisements) as not needed because the host-side interface is connected to a bridge. err := localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", hostName), "1") if err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, err } // Attempt to disable IPv4 forwarding. err = localUtil.SysctlSet(fmt.Sprintf("net/ipv4/conf/%s/forwarding", hostName), "0") if err != nil { return nil, err } // Attach host side veth interface to bridge. integrationBridge := d.state.GlobalConfig.NetworkOVNIntegrationBridge() vswitch, err := d.state.OVS() if err != nil { return nil, fmt.Errorf("Failed to connect to OVS: %w", err) } err = vswitch.CreateBridgePort(context.TODO(), integrationBridge, hostName, true) if err != nil { return nil, err } reverter.Add(func() { _ = vswitch.DeleteBridgePort(context.TODO(), integrationBridge, hostName) }) // Link OVS port to OVN logical port. err = vswitch.AssociateInterfaceOVNSwitchPort(context.TODO(), hostName, string(ovnPortName)) if err != nil { return nil, err } // Make sure the port is up. link := &ip.Link{Name: hostName} err = link.SetUp() if err != nil { return nil, fmt.Errorf("Failed to bring up the host interface %s: %w", hostName, err) } cleanup := reverter.Clone().Fail reverter.Success() return cleanup, err } // isVirtualNIC determines whether the device is non-accelerated. func (d *nicOVN) isVirtualNIC() bool { return slices.Contains([]string{"", "none"}, d.config["acceleration"]) } incus-7.0.0/internal/server/device/nic_p2p.go000066400000000000000000000233671517523235500211240ustar00rootroot00000000000000package device import ( "errors" "fmt" "io/fs" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/network" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" ) type nicP2P struct { deviceCommon } // CanHotPlug returns whether the device can be managed whilst the instance is running. Returns true. func (d *nicP2P) CanHotPlug() bool { return true } // validateConfig checks the supplied config for correctness. func (d *nicP2P) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container, instancetype.VM) { return ErrUnsupportedDevType } optionalFields := []string{ // gendoc:generate(entity=devices, group=nic_p2p, key=name) // // --- // type: string // default: kernel assigned // shortdesc: The name of the interface inside the instance "name", // gendoc:generate(entity=devices, group=nic_p2p, key=mtu) // // --- // type: integer // default: kernel assigned // shortdesc: The Maximum Transmit Unit (MTU) of the new interface "mtu", // gendoc:generate(entity=devices, group=nic_p2p, key=queue.tx.length) // // --- // type: integer // shortdesc: The transmit queue length for the NIC "queue.tx.length", // gendoc:generate(entity=devices, group=nic_p2p, key=hwaddr) // // --- // type: string // default: randomly assigned // shortdesc: The MAC address of the new interface "hwaddr", // gendoc:generate(entity=devices, group=nic_p2p, key=host_name) // // --- // type: string // default: randomly assigned // shortdesc: The name of the interface on the host "host_name", // gendoc:generate(entity=devices, group=nic_p2p, key=limits.ingress) // // --- // type: string // shortdesc: I/O limit in bit/s for incoming traffic (various suffixes supported, see {ref}`instances-limit-units`) "limits.ingress", // gendoc:generate(entity=devices, group=nic_p2p, key=limits.egress) // // --- // type: string // shortdesc: I/O limit in bit/s for outgoing traffic (various suffixes supported, see {ref}`instances-limit-units`) "limits.egress", // gendoc:generate(entity=devices, group=nic_p2p, key=limits.max) // // --- // type: string // shortdesc: I/O limit in bit/s for both incoming and outgoing traffic (same as setting both limits.ingress and limits.egress) "limits.max", // gendoc:generate(entity=devices, group=nic_p2p, key=limits.priority) // // --- // type: integer // shortdesc: The priority for outgoing traffic, to be used by the kernel queuing discipline to prioritize network packets "limits.priority", // gendoc:generate(entity=devices, group=nic_p2p, key=ipv4.routes) // // --- // type: string // shortdesc: Comma-delimited list of IPv4 static routes to add on host to NIC "ipv4.routes", // gendoc:generate(entity=devices, group=nic_p2p, key=ipv6.routes) // // --- // type: string // shortdesc: Comma-delimited list of IPv6 static routes to add on host to NIC "ipv6.routes", // gendoc:generate(entity=devices, group=nic_p2p, key=boot.priority) // // --- // type: integer // shortdesc: Boot priority for VMs (higher value boots first) "boot.priority", // gendoc:generate(entity=devices, group=nic_p2p, key=io.bus) // // --- // type: string // default: `virtio` // shortdesc: Override the bus for the device (can be `virtio` or `usb`) (VM only) "io.bus", // gendoc:generate(entity=devices, group=nic_p2p, key=attached) // // --- // type: bool // default: `true` // required: no // shortdesc: Whether the NIC is plugged in or not "attached", // gendoc:generate(entity=devices, group=nic_p2p, key=connected) // // --- // type: bool // default: `true` // required: no // shortdesc: Whether the NIC is connected to the host network "connected", } err := d.config.Validate(nicValidationRules([]string{}, optionalFields, instConf)) if err != nil { return err } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *nicP2P) validateEnvironment() error { if d.inst.Type() == instancetype.Container && d.config["name"] == "" { return errors.New("Requires name property to start") } return nil } // UpdatableFields returns a list of fields that can be updated without triggering a device remove & add. func (d *nicP2P) UpdatableFields(oldDevice Type) []string { // Check old and new device types match. _, match := oldDevice.(*nicP2P) if !match { return []string{} } return []string{"limits.ingress", "limits.egress", "limits.max", "limits.priority", "ipv4.routes", "ipv6.routes", "connected"} } // Start is run when the device is added to a running instance or instance is starting up. func (d *nicP2P) Start() (*deviceConfig.RunConfig, error) { // Ignore detached NICs. if !util.IsTrueOrEmpty(d.config["attached"]) { return nil, nil } err := d.validateEnvironment() if err != nil { return nil, err } reverter := revert.New() defer reverter.Fail() saveData := make(map[string]string) saveData["host_name"] = d.config["host_name"] var peerName string var mtu uint32 // Create veth pair and configure the peer end with custom hwaddr and mtu if supplied. if d.inst.Type() == instancetype.Container { if saveData["host_name"] == "" { saveData["host_name"], err = d.generateHostName("veth", d.config["hwaddr"]) if err != nil { return nil, err } } peerName, mtu, err = networkCreateVethPair(saveData["host_name"], d.config) } else if d.inst.Type() == instancetype.VM { if saveData["host_name"] == "" { saveData["host_name"], err = d.generateHostName("tap", d.config["hwaddr"]) if err != nil { return nil, err } } peerName = saveData["host_name"] // VMs use the host_name to link to the TAP FD. mtu, err = networkCreateTap(saveData["host_name"], d.config) } if err != nil { return nil, err } reverter.Add(func() { _ = network.InterfaceRemove(saveData["host_name"]) }) // Attempt to disable router advertisement acceptance. err = localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", saveData["host_name"]), "0") if err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, err } // Populate device config with volatile fields if needed. networkVethFillFromVolatile(d.config, saveData) // Apply host-side routes to veth interface. err = networkNICRouteAdd(d.config["host_name"], "", "", append(util.SplitNTrimSpace(d.config["ipv4.routes"], ",", -1, true), util.SplitNTrimSpace(d.config["ipv6.routes"], ",", -1, true)...)...) if err != nil { return nil, err } // Apply host-side limits. err = networkSetupHostVethLimits(&d.deviceCommon, nil, false) if err != nil { return nil, err } err = d.volatileSet(saveData) if err != nil { return nil, err } runConf := deviceConfig.RunConfig{} runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "type", Value: "phys"}, {Key: "name", Value: d.config["name"]}, {Key: "flags", Value: "up"}, {Key: "link", Value: peerName}, {Key: "hwaddr", Value: d.config["hwaddr"]}, {Key: "connected", Value: d.config["connected"]}, } if d.config["io.bus"] == "usb" { runConf.UseUSBBus = true } if d.inst.Type() == instancetype.VM { runConf.NetworkInterface = append(runConf.NetworkInterface, []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "mtu", Value: fmt.Sprintf("%d", mtu)}, }...) } reverter.Success() return &runConf, nil } // Update applies configuration changes to a started device. func (d *nicP2P) Update(oldDevices deviceConfig.Devices, isRunning bool) error { if !isRunning { return nil } err := d.validateEnvironment() if err != nil { return err } oldConfig := oldDevices[d.name] v := d.volatileGet() // Populate device config with volatile fields if needed. networkVethFillFromVolatile(d.config, v) networkVethFillFromVolatile(oldConfig, v) // Remove old host-side routes from veth interface. networkNICRouteDelete(oldConfig["host_name"], "", "", append(util.SplitNTrimSpace(oldConfig["ipv4.routes"], ",", -1, true), util.SplitNTrimSpace(oldConfig["ipv6.routes"], ",", -1, true)...)...) // Apply host-side routes to veth interface. err = networkNICRouteAdd(d.config["host_name"], "", "", append(util.SplitNTrimSpace(d.config["ipv4.routes"], ",", -1, true), util.SplitNTrimSpace(d.config["ipv6.routes"], ",", -1, true)...)...) if err != nil { return err } // Apply host-side limits. err = networkSetupHostVethLimits(&d.deviceCommon, oldConfig, false) if err != nil { return err } return d.setNICLink() } // Stop is run when the device is removed from the instance. func (d *nicP2P) Stop() (*deviceConfig.RunConfig, error) { // Populate device config with volatile fields (hwaddr and host_name) if needed. networkVethFillFromVolatile(d.config, d.volatileGet()) err := networkClearHostVethLimits(&d.deviceCommon) if err != nil { return nil, err } runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *nicP2P) postStop() error { defer func() { _ = d.volatileSet(map[string]string{ "host_name": "", }) }() v := d.volatileGet() networkVethFillFromVolatile(d.config, v) if d.config["host_name"] != "" && network.InterfaceExists(d.config["host_name"]) { // Removing host-side end of veth pair will delete the peer end too. err := network.InterfaceRemove(d.config["host_name"]) if err != nil { return fmt.Errorf("Failed to remove interface %s: %w", d.config["host_name"], err) } } return nil } incus-7.0.0/internal/server/device/nic_physical.go000066400000000000000000000426511517523235500222340ustar00rootroot00000000000000package device import ( "context" "errors" "fmt" "net" "strconv" "strings" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/db" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" pcidev "github.com/lxc/incus/v7/internal/server/device/pci" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" ) type nicPhysical struct { deviceCommon network network.Network // Populated in validateConfig(). } // CanHotPlug returns whether the device can be managed whilst the instance is running. Returns true. func (d *nicPhysical) CanHotPlug() bool { return true } // CanMigrate returns whether the device can be migrated to any other cluster member. func (d *nicPhysical) CanMigrate() bool { // If we're looking at a managed physical NIC, assume it's migratable. if d.config["network"] != "" { return true } return false } // validateConfig checks the supplied config for correctness. func (d *nicPhysical) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container, instancetype.VM) { return ErrUnsupportedDevType } requiredFields := []string{} optionalFields := []string{ // gendoc:generate(entity=devices, group=nic_physical, key=parent) // // --- // type: string // managed: yes // shortdesc: The name of the parent host device (required if specifying the `nictype` directly) "parent", // gendoc:generate(entity=devices, group=nic_physical, key=name) // // --- // type: string // default: kernel assigned // managed: no // shortdesc: The name of the interface inside the instance "name", // gendoc:generate(entity=devices, group=nic_physical, key=boot.priority) // // --- // type: integer // managed: no // shortdesc: Boot priority for VMs (higher value boots first) "boot.priority", // gendoc:generate(entity=devices, group=nic_physical, key=attached) // // --- // type: bool // default: `true` // required: no // shortdesc: Whether the NIC is plugged in or not "attached", } if instConf.Type() == instancetype.Container || instConf.Type() == instancetype.Any { // gendoc:generate(entity=devices, group=nic_physical, key=gvrp) // // --- // type: bool // default: false // managed: no // condition: container // shortdesc: Register VLAN using GARP VLAN Registration Protocol // gendoc:generate(entity=devices, group=nic_physical, key=hwaddr) // // --- // type: string // default: randomly assigned // managed: no // condition: container // shortdesc: The MAC address of the new interface // gendoc:generate(entity=devices, group=nic_physical, key=mtu) // // --- // type: integer // default: MTU of the parent device // managed: no // condition: container // shortdesc: The Maximum Transmit Unit (MTU) of the new interface // gendoc:generate(entity=devices, group=nic_physical, key=vlan) // // --- // type: integer // managed: no // condition: container // shortdesc: The VLAN ID to attach to // gendoc:generate(entity=devices, group=nic_physical, key=vlan.tagged) // // --- // type: integer // managed: no // condition: container // shortdesc: Comma-delimited list of VLAN IDs or VLAN ranges to join for tagged traffic optionalFields = append(optionalFields, "gvrp", "hwaddr", "mtu", "vlan", "vlan.tagged") } // gendoc:generate(entity=devices, group=nic_physical, key=network) // // --- // type: string // managed: no // shortdesc: The managed network to link the device to (instead of specifying the `nictype` directly) if d.config["network"] != "" { // List of properties we import from the parent network. networkFields := []string{"gvrp", "mtu", "vlan", "vlan.tagged"} requiredFields = append(requiredFields, "network") bannedKeys := append([]string{"nictype", "parent"}, networkFields...) for _, bannedKey := range bannedKeys { if d.config[bannedKey] != "" { return fmt.Errorf("Cannot use %q property in conjunction with %q property", bannedKey, "network") } } // If network property is specified, lookup network settings and apply them to the device's config. // api.ProjectDefaultName is used here as physical networks don't support projects. var err error d.network, err = network.LoadByName(d.state, api.ProjectDefaultName, d.config["network"]) if err != nil { return fmt.Errorf("Error loading network config for %q: %w", d.config["network"], err) } if d.network.Status() != api.NetworkStatusCreated { return errors.New("Specified network is not fully created") } if d.network.Type() != "physical" { return errors.New("Specified network must be of type physical") } netConfig := d.network.Config() // Get actual parent device from network's parent setting. d.config["parent"] = netConfig["parent"] // Copy certain keys verbatim from the parent network's settings. for _, field := range networkFields { _, found := netConfig[field] if found { d.config[field] = netConfig[field] } } // Check if the parent is a bridge. isParentBridge := d.config["parent"] != "" && util.PathExists(fmt.Sprintf("/sys/class/net/%s/bridge", d.config["parent"])) if isParentBridge { // Validate the NIC as if bridged. bridgedConfig := d.config.Clone() bridgedConfig["type"] = "nic" bridgedConfig["nictype"] = "bridged" bridgedConfig["network"] = "" // Instantiate the new device. bridged, err := load(nil, d.state, instConf.Project().Name, d.name, bridgedConfig, nil, nil) if err != nil { return fmt.Errorf("Failed to initialize bridged device: %w", err) } // Forward the validateConfig call. return bridged.validateConfig(instConf, partialValidation) } } else { // If no network property supplied, then parent property is required. requiredFields = append(requiredFields, "parent") } err := d.config.Validate(nicValidationRules(requiredFields, optionalFields, instConf)) if err != nil { return err } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *nicPhysical) validateEnvironment() error { isParentBridge := util.PathExists(fmt.Sprintf("/sys/class/net/%s/bridge", d.config["parent"])) if d.inst.Type() == instancetype.VM && util.IsTrue(d.inst.ExpandedConfig()["migration.stateful"]) && !isParentBridge { return errors.New("Network physical devices cannot be used when migration.stateful is enabled") } if d.inst.Type() == instancetype.Container && d.config["name"] == "" { return errors.New("Requires name property to start") } if !util.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["parent"])) { return fmt.Errorf("Parent device '%s' doesn't exist", d.config["parent"]) } return nil } // Start is run when the device is added to a running instance or instance is starting up. func (d *nicPhysical) Start() (*deviceConfig.RunConfig, error) { // Ignore detached NICs. if !util.IsTrueOrEmpty(d.config["attached"]) { return nil, nil } err := d.validateEnvironment() if err != nil { return nil, err } // Handle the case where the parent is a bridge. isParentBridge := util.PathExists(fmt.Sprintf("/sys/class/net/%s/bridge", d.config["parent"])) if isParentBridge { // Convert the device to a nictype=bridged internally. bridgedConfig := d.config.Clone() bridgedConfig["type"] = "nic" bridgedConfig["nictype"] = "bridged" bridgedConfig["network"] = "" // Instantiate the new device. bridged, err := load(d.inst, d.state, d.inst.Project().Name, d.name, bridgedConfig, d.volatileGet, d.volatileSet) if err != nil { return nil, fmt.Errorf("Failed to initialize bridged device: %w", err) } // Forward the start call. return bridged.Start() } // Lock to avoid issues with containers starting in parallel. networkCreateSharedDeviceLock.Lock() defer networkCreateSharedDeviceLock.Unlock() saveData := make(map[string]string) reverter := revert.New() defer reverter.Fail() // pciIOMMUGroup, used for VM physical passthrough. var pciIOMMUGroup uint64 // If VM, then try and load the vfio-pci module first. if d.inst.Type() == instancetype.VM { err = linux.LoadModule("vfio-pci") if err != nil { return nil, fmt.Errorf("Error loading %q module: %w", "vfio-pci", err) } } // Record the host_name device used for restoration later. saveData["host_name"] = network.GetHostDevice(d.config["parent"], d.config["vlan"]) if d.inst.Type() == instancetype.Container { statusDev, err := networkCreateVlanDeviceIfNeeded(d.state, d.config["parent"], saveData["host_name"], d.config["vlan"], util.IsTrue(d.config["gvrp"])) if err != nil { return nil, err } // Record whether we created this device or not so it can be removed on stop. saveData["last_state.created"] = fmt.Sprintf("%t", statusDev != "existing") if util.IsTrue(saveData["last_state.created"]) { reverter.Add(func() { _ = networkRemoveInterfaceIfNeeded(d.state, saveData["host_name"], d.inst, d.config["parent"], d.config["vlan"]) }) } // If we didn't create the device we should track various properties so we can restore them when the // instance is stopped or the device is detached. if util.IsFalse(saveData["last_state.created"]) { err = networkSnapshotPhysicalNIC(saveData["host_name"], saveData) if err != nil { return nil, err } } // Set the MAC address. if d.config["hwaddr"] != "" { hwaddr, err := net.ParseMAC(d.config["hwaddr"]) if err != nil { return nil, fmt.Errorf("Failed parsing MAC address %q: %w", d.config["hwaddr"], err) } link := &ip.Link{Name: saveData["host_name"]} err = link.SetAddress(hwaddr) if err != nil { return nil, fmt.Errorf("Failed to set the MAC address: %s", err) } } // Set the MTU. if d.config["mtu"] != "" { mtu, err := strconv.ParseUint(d.config["mtu"], 10, 32) if err != nil { return nil, fmt.Errorf("Invalid MTU specified %q: %w", d.config["mtu"], err) } link := &ip.Link{Name: saveData["host_name"]} err = link.SetMTU(uint32(mtu)) if err != nil { return nil, fmt.Errorf("Failed setting MTU %q on %q: %w", d.config["mtu"], saveData["host_name"], err) } } } else if d.inst.Type() == instancetype.VM { // Try to get PCI information about the network interface. ueventPath := fmt.Sprintf("/sys/class/net/%s/device/uevent", saveData["host_name"]) pciDev, err := pcidev.ParseUeventFile(ueventPath) if err != nil { if errors.Is(err, pcidev.ErrDeviceIsUSB) { // Device is USB rather than PCI. return d.startVMUSB(saveData["host_name"]) } return nil, fmt.Errorf("Failed to get PCI device info for %q: %w", saveData["host_name"], err) } saveData["last_state.pci.slot.name"] = pciDev.SlotName saveData["last_state.pci.driver"] = pciDev.Driver pciIOMMUGroup, err = pcidev.DeviceIOMMUGroup(saveData["last_state.pci.slot.name"]) if err != nil { return nil, err } err = pcidev.DeviceDriverOverride(pciDev, "vfio-pci") if err != nil { return nil, err } } err = d.volatileSet(saveData) if err != nil { return nil, err } runConf := deviceConfig.RunConfig{} runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "type", Value: "phys"}, {Key: "name", Value: d.config["name"]}, {Key: "flags", Value: "up"}, {Key: "link", Value: saveData["host_name"]}, } if d.inst.Type() == instancetype.VM { runConf.NetworkInterface = append(runConf.NetworkInterface, []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "pciSlotName", Value: saveData["last_state.pci.slot.name"]}, {Key: "pciIOMMUGroup", Value: fmt.Sprintf("%d", pciIOMMUGroup)}, }...) } reverter.Success() return &runConf, nil } func (d *nicPhysical) startVMUSB(name string) (*deviceConfig.RunConfig, error) { // Get the list of network interfaces. interfaces, err := resources.GetNetwork() if err != nil { return nil, err } // Look for our USB device. var addr string for _, card := range interfaces.Cards { for _, port := range card.Ports { if port.ID == name { addr = card.USBAddress break } } if addr != "" { break } } if addr == "" { return nil, fmt.Errorf("Failed to get USB device info for %q", name) } // Parse the USB address. fields := strings.Split(addr, ":") if len(fields) != 2 { return nil, fmt.Errorf("Bad USB device info for %q", name) } usbBus, err := strconv.Atoi(fields[0]) if err != nil { return nil, fmt.Errorf("Bad USB device info for %q: %w", name, err) } usbDev, err := strconv.Atoi(fields[1]) if err != nil { return nil, fmt.Errorf("Bad USB device info for %q: %w", name, err) } // Record the addresses. saveData := map[string]string{} saveData["last_state.usb.bus"] = fmt.Sprintf("%03d", usbBus) saveData["last_state.usb.device"] = fmt.Sprintf("%03d", usbDev) err = d.volatileSet(saveData) if err != nil { return nil, err } // Generate a config. runConf := deviceConfig.RunConfig{} runConf.USBDevice = append(runConf.USBDevice, deviceConfig.USBDeviceItem{ DeviceName: fmt.Sprintf("%s-%03d-%03d", d.name, usbBus, usbDev), HostDevicePath: fmt.Sprintf("/dev/bus/usb/%03d/%03d", usbBus, usbDev), }) return &runConf, nil } // Stop is run when the device is removed from the instance. func (d *nicPhysical) Stop() (*deviceConfig.RunConfig, error) { // Handle the case where the parent is a bridge. isParentBridge := util.PathExists(fmt.Sprintf("/sys/class/net/%s/bridge", d.config["parent"])) if isParentBridge { // Convert the device to a nictype=bridged internally. bridgedConfig := d.config.Clone() bridgedConfig["type"] = "nic" bridgedConfig["nictype"] = "bridged" bridgedConfig["network"] = "" // Instantiate the new device. bridged, err := load(d.inst, d.state, d.inst.Project().Name, d.name, bridgedConfig, d.volatileGet, d.volatileSet) if err != nil { return nil, fmt.Errorf("Failed to initialize bridged device: %w", err) } // Forward the stop call. return bridged.Stop() } v := d.volatileGet() runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } if v["last_state.usb.bus"] != "" && v["last_state.usb.device"] != "" { // Handle USB NICs. runConf.USBDevice = append(runConf.USBDevice, deviceConfig.USBDeviceItem{ DeviceName: fmt.Sprintf("%s-%s-%s", d.name, v["last_state.usb.bus"], v["last_state.usb.device"]), HostDevicePath: fmt.Sprintf("/dev/bus/usb/%s/%s", v["last_state.usb.bus"], v["last_state.usb.device"]), }) } else if util.IsTrueOrEmpty(d.config["attached"]) { // Handle all other NICs. runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "link", Value: v["host_name"]}, } } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *nicPhysical) postStop() error { defer func() { _ = d.volatileSet(map[string]string{ "host_name": "", "last_state.hwaddr": "", "last_state.mtu": "", "last_state.created": "", "last_state.pci.slot.name": "", "last_state.pci.driver": "", "last_state.usb.bus": "", "last_state.usb.device": "", }) }() v := d.volatileGet() // If VM physical pass through, unbind from vfio-pci and bind back to host driver. if d.inst.Type() == instancetype.VM && v["last_state.pci.slot.name"] != "" { vfioDev := pcidev.Device{ Driver: "vfio-pci", SlotName: v["last_state.pci.slot.name"], } err := pcidev.DeviceDriverOverride(vfioDev, v["last_state.pci.driver"]) if err != nil { return err } } else if d.inst.Type() == instancetype.Container { hostName := network.GetHostDevice(d.config["parent"], d.config["vlan"]) // This will delete the parent interface if we created it for VLAN parent. if util.IsTrue(v["last_state.created"]) { err := networkRemoveInterfaceIfNeeded(d.state, hostName, d.inst, d.config["parent"], d.config["vlan"]) if err != nil { return err } } else if v["last_state.pci.slot.name"] == "" { err := networkRestorePhysicalNIC(hostName, v) if err != nil { return err } } } return nil } // IsPhysicalNICWithBridge returns true if the given NIC is of type "physical" // and has a non-empty Parent field, indicating it's attached to a bridge. func IsPhysicalNICWithBridge(s *state.State, deviceProjectName string, d deviceConfig.Device) bool { if d["network"] != "" { // Translate device's project name into a network project name. networkProjectName, _, err := project.NetworkProject(s.DB.Cluster, deviceProjectName) if err != nil { return false } var netInfo *api.Network err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, netInfo, _, err = tx.GetNetworkInAnyState(ctx, networkProjectName, d["network"]) return err }) if err != nil { return false } if netInfo.Type != "physical" { return false } parent := netInfo.Config["parent"] if parent == "" { return false } return util.PathExists(fmt.Sprintf("/sys/class/net/%s/bridge", parent)) } return false } // Update applies configuration changes to a started device. func (d *nicPhysical) Update(oldDevices deviceConfig.Devices, isRunning bool) error { if isRunning { return d.setNICLink() } return nil } incus-7.0.0/internal/server/device/nic_routed.go000066400000000000000000000746331517523235500217270ustar00rootroot00000000000000package device import ( "context" "errors" "fmt" "io/fs" "net" "strings" "time" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/network" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var nicRoutedIPGateway = map[string]net.IP{ "ipv4": net.IPv4(169, 254, 0, 1), // 169.254.0.1 "ipv6": {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x01}, // fe80::1 } type nicRouted struct { deviceCommon effectiveParentName string } // CanHotPlug returns whether the device can be managed whilst the instance is running. func (d *nicRouted) CanHotPlug() bool { return true } // UpdatableFields returns a list of fields that can be updated without triggering a device remove & add. func (d *nicRouted) UpdatableFields(oldDevice Type) []string { // Check old and new device types match. _, match := oldDevice.(*nicRouted) if !match { return []string{} } return []string{"limits.ingress", "limits.egress", "limits.max", "limits.priority", "connected"} } // validateConfig checks the supplied config for correctness. func (d *nicRouted) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container, instancetype.VM) { return ErrUnsupportedDevType } err := d.isUniqueWithGatewayAutoMode(instConf) if err != nil { return err } requiredFields := []string{} optionalFields := []string{ // gendoc:generate(entity=devices, group=nic_routed, key=name) // // --- // type: string // default: kernel assigned // shortdesc: The name of the interface inside the instance "name", // gendoc:generate(entity=devices, group=nic_routed, key=parent) // // --- // type: string // shortdesc: The name of the parent host device to join the instance to "parent", // gendoc:generate(entity=devices, group=nic_routed, key=mtu) // // --- // type: integer // default: parent MTU // shortdesc: The Maximum Transmit Unit (MTU) of the new interface "mtu", // gendoc:generate(entity=devices, group=nic_routed, key=queue.tx.length) // // --- // type: integer // shortdesc: The transmit queue length for the NIC "queue.tx.length", // gendoc:generate(entity=devices, group=nic_routed, key=hwaddr) // // --- // type: string // default: randomly assigned // shortdesc: The MAC address of the new interface "hwaddr", // gendoc:generate(entity=devices, group=nic_routed, key=host_name) // // --- // type: string // default: randomly assigned // shortdesc: The name of the interface on the host "host_name", // gendoc:generate(entity=devices, group=nic_routed, key=vlan) // // --- // type: integer // shortdesc: The VLAN ID to attach to "vlan", // gendoc:generate(entity=devices, group=nic_routed, key=limits.ingress) // // --- // type: string // shortdesc: I/O limit in bit/s for incoming traffic (various suffixes supported, see {ref}`instances-limit-units`) "limits.ingress", // gendoc:generate(entity=devices, group=nic_routed, key=limits.egress) // // --- // type: string // shortdesc: I/O limit in bit/s for outgoing traffic (various suffixes supported, see {ref}`instances-limit-units`) "limits.egress", // gendoc:generate(entity=devices, group=nic_routed, key=limits.max) // // --- // type: string // shortdesc: I/O limit in bit/s for both incoming and outgoing traffic (same as setting both limits.ingress and limits.egress) "limits.max", // gendoc:generate(entity=devices, group=nic_routed, key=limits.priority) // // --- // type: integer // shortdesc: The priority for outgoing traffic, to be used by the kernel queuing discipline to prioritize network packets "limits.priority", // gendoc:generate(entity=devices, group=nic_routed, key=ipv4.gateway) // // --- // type: string // default: auto // shortdesc: Whether to add an automatic default IPv4 gateway (can be `auto` or `none`) "ipv4.gateway", // gendoc:generate(entity=devices, group=nic_routed, key=ipv6.gateway) // // --- // type: string // default: auto // shortdesc: Whether to add an automatic default IPv6 gateway (can be `auto` or `none`) "ipv6.gateway", // gendoc:generate(entity=devices, group=nic_routed, key=ipv4.routes) // // --- // type: string // shortdesc: Comma-delimited list of IPv4 static routes to add on host to NIC (without L2 ARP/NDP proxy) "ipv4.routes", // gendoc:generate(entity=devices, group=nic_routed, key=ipv6.routes) // // --- // type: string // shortdesc: Comma-delimited list of IPv6 static routes to add on host to NIC (without L2 ARP/NDP proxy) "ipv6.routes", // gendoc:generate(entity=devices, group=nic_routed, key=ipv4.host_address) // // --- // type: string // default: `169.254.0.1` // shortdesc: The IPv4 address to add to the host-side `veth` interface "ipv4.host_address", // gendoc:generate(entity=devices, group=nic_routed, key=ipv6.host_address) // // --- // type: string // default: `fe80::1` // shortdesc: The IPv6 address to add to the host-side `veth` interface "ipv6.host_address", // gendoc:generate(entity=devices, group=nic_routed, key=ipv4.host_table) // // The custom policy routing table ID to add IPv4 static routes to (in addition to the main routing table) // // --- // type: integer // shortdesc: Deprecated: Use `ipv4.host_tables` instead "ipv4.host_table", // gendoc:generate(entity=devices, group=nic_routed, key=ipv6.host_table) // // The custom policy routing table ID to add IPv6 static routes to (in addition to the main routing table) // // --- // type: integer // shortdesc: Deprecated: Use `ipv6.host_tables` instead "ipv6.host_table", // gendoc:generate(entity=devices, group=nic_routed, key=ipv4.host_tables) // // --- // type: string // default: 254 // shortdesc: Comma-delimited list of routing tables IDs to add IPv4 static routes to "ipv4.host_tables", // gendoc:generate(entity=devices, group=nic_routed, key=ipv6.host_tables) // // --- // type: string // default: 254 // shortdesc: Comma-delimited list of routing tables IDs to add IPv6 static routes to "ipv6.host_tables", // gendoc:generate(entity=devices, group=nic_routed, key=gvrp) // // --- // type: bool // default: false // shortdesc: Register VLAN using GARP VLAN Registration Protocol "gvrp", // gendoc:generate(entity=devices, group=nic_routed, key=vrf) // // --- // type: string // shortdesc: The VRF on the host in which the host-side interface and routes are created "vrf", // gendoc:generate(entity=devices, group=nic_routed, key=io.bus) // // --- // type: string // default: `virtio` // shortdesc: Override the bus for the device (can be `virtio` or `usb`) (VM only) "io.bus", // gendoc:generate(entity=devices, group=nic_routed, key=attached) // // --- // type: bool // default: `true` // required: no // shortdesc: Whether the NIC is plugged in or not "attached", // gendoc:generate(entity=devices, group=nic_routed, key=connected) // // --- // type: bool // default: `true` // required: no // shortdesc: Whether the NIC is connected to the host network "connected", } rules := nicValidationRules(requiredFields, optionalFields, instConf) // gendoc:generate(entity=devices, group=nic_routed, key=ipv4.address) // // --- // type: string // shortdesc: Comma-delimited list of IPv4 static addresses to add to the instance rules["ipv4.address"] = validate.Optional(validate.IsListOf(validate.IsNetworkAddressV4)) // gendoc:generate(entity=devices, group=nic_routed, key=ipv6.address) // // --- // type: string // shortdesc: Comma-delimited list of IPv6 static addresses to add to the instance rules["ipv6.address"] = validate.Optional(validate.IsListOf(validate.IsNetworkAddressV6)) // gendoc:generate(entity=devices, group=nic_routed, key=ipv4.neighbor_probe) // // --- // type: bool // default: true // shortdesc: Whether to probe the parent network for IP address availability rules["ipv4.neighbor_probe"] = validate.Optional(validate.IsBool) // gendoc:generate(entity=devices, group=nic_routed, key=ipv6.neighbor_probe) // // --- // type: bool // default: true // shortdesc: Whether to probe the parent network for IP address availability rules["ipv6.neighbor_probe"] = validate.Optional(validate.IsBool) rules["ipv4.host_tables"] = validate.Optional(validate.IsListOf(validate.IsInRange(0, 255))) rules["ipv6.host_tables"] = validate.Optional(validate.IsListOf(validate.IsInRange(0, 255))) rules["gvrp"] = validate.Optional(validate.IsBool) rules["vrf"] = validate.Optional(validate.IsAny) err = d.config.Validate(rules) if err != nil { return err } // Detect duplicate IPs in config. for _, key := range []string{"ipv4.address", "ipv6.address"} { ips := make(map[string]struct{}) if d.config[key] != "" { for _, addr := range strings.Split(d.config[key], ",") { addr = strings.TrimSpace(addr) _, dupe := ips[addr] if dupe { return fmt.Errorf("Duplicate address %q in %q", addr, key) } ips[addr] = struct{}{} } } } // Ensure that address is set if routes is set. for _, keyPrefix := range []string{"ipv4", "ipv6"} { if d.config[fmt.Sprintf("%s.routes", keyPrefix)] != "" && d.config[fmt.Sprintf("%s.address", keyPrefix)] == "" { return fmt.Errorf("%s.routes requires %s.address to be set", keyPrefix, keyPrefix) } } // Ensure that VLAN setting is only used with parent setting. if d.config["parent"] == "" && d.config["vlan"] != "" { return errors.New("The vlan setting can only be used when combined with a parent interface") } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *nicRouted) validateEnvironment() error { if d.inst.Type() == instancetype.Container && d.config["name"] == "" { return errors.New("Requires name property to start") } if d.config["parent"] != "" { // Check parent interface exists (don't use d.effectiveParentName here as we want to check the // parent of any VLAN interface exists too). The VLAN interface will be created later if needed. if !network.InterfaceExists(d.config["parent"]) { return fmt.Errorf("Parent device %q doesn't exist", d.config["parent"]) } // Detect the effective parent interface that we will be using (taking into account VLAN setting). d.effectiveParentName = network.GetHostDevice(d.config["parent"], d.config["vlan"]) // If the effective parent doesn't exist and the vlan option is specified, it means we are going to // create the VLAN parent at start, and we will configure the needed sysctls then, so skip checks // on the effective parent. if d.config["vlan"] != "" && !network.InterfaceExists(d.effectiveParentName) { return nil } // Check necessary "all" sysctls are configured for use with l2proxy parent for routed mode. if d.config["ipv6.address"] != "" { // net.ipv6.conf.all.forwarding=1 is required to enable general packet forwarding for IPv6. ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", "all") sysctlVal, err := localUtil.SysctlGet(ipv6FwdPath) if err != nil { return fmt.Errorf("Error reading net sysctl %s: %w", ipv6FwdPath, err) } if sysctlVal != "1\n" { return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.forwarding=1", "all") } // net.ipv6.conf.all.proxy_ndp=1 is needed otherwise unicast neighbour solicitations are . // rejected This causes periodic latency spikes every 15-20s as the neighbour has to resort // to using multicast NDP resolution and expires the previous neighbour entry. ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", "all") sysctlVal, err = localUtil.SysctlGet(ipv6ProxyNdpPath) if err != nil { return fmt.Errorf("Error reading net sysctl %s: %w", ipv6ProxyNdpPath, err) } if sysctlVal != "1\n" { return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", "all") } } // Check necessary sysctls are configured for use with l2proxy parent for routed mode. if d.config["ipv4.address"] != "" { ipv4FwdPath := fmt.Sprintf("net/ipv4/conf/%s/forwarding", d.effectiveParentName) sysctlVal, err := localUtil.SysctlGet(ipv4FwdPath) if err != nil { return fmt.Errorf("Error reading net sysctl %s: %w", ipv4FwdPath, err) } if sysctlVal != "1\n" { // Replace . in parent name with / for sysctl formatting. return fmt.Errorf("Routed mode requires sysctl net.ipv4.conf.%s.forwarding=1", strings.ReplaceAll(d.effectiveParentName, ".", "/")) } } // Check necessary device specific sysctls are configured for use with l2proxy parent for routed mode. if d.config["ipv6.address"] != "" { ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", d.effectiveParentName) sysctlVal, err := localUtil.SysctlGet(ipv6FwdPath) if err != nil { return fmt.Errorf("Error reading net sysctl %s: %w", ipv6FwdPath, err) } if sysctlVal != "1\n" { // Replace . in parent name with / for sysctl formatting. return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.forwarding=1", strings.ReplaceAll(d.effectiveParentName, ".", "/")) } ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", d.effectiveParentName) sysctlVal, err = localUtil.SysctlGet(ipv6ProxyNdpPath) if err != nil { return fmt.Errorf("Error reading net sysctl %s: %w", ipv6ProxyNdpPath, err) } if sysctlVal != "1\n" { // Replace . in parent name with / for sysctl formatting. return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", strings.ReplaceAll(d.effectiveParentName, ".", "/")) } } } if d.config["vrf"] != "" { // Check if the vrf interface exists. if !network.InterfaceExists(d.config["vrf"]) { return fmt.Errorf("VRF %q doesn't exist", d.config["vrf"]) } } return nil } // checkIPAvailability checks using ARP and NDP neighbour probes whether any of the NIC's IPs are already in use. func (d *nicRouted) checkIPAvailability(parent string) error { var addresses []net.IP if util.IsTrueOrEmpty(d.config["ipv4.neighbor_probe"]) { ipv4Addrs := util.SplitNTrimSpace(d.config["ipv4.address"], ",", -1, true) for _, addr := range ipv4Addrs { addresses = append(addresses, net.ParseIP(addr)) } } if util.IsTrueOrEmpty(d.config["ipv6.neighbor_probe"]) { ipv6Addrs := util.SplitNTrimSpace(d.config["ipv6.address"], ",", -1, true) for _, addr := range ipv6Addrs { addresses = append(addresses, net.ParseIP(addr)) } } errs := make(chan error, len(addresses)) for _, address := range addresses { go func(address net.IP) { ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() inUse, err := isIPAvailable(ctx, address, parent) if err != nil { d.logger.Warn("Failed checking IP address available on parent network", logger.Ctx{"IP": address, "parent": parent, "err": err}) } if inUse { errs <- fmt.Errorf("IP address %q in use on parent network %q", address, parent) } else { errs <- nil } }(address) } for range addresses { err := <-errs if err != nil { return err } } return nil } // Start is run when the instance is starting up (Routed mode doesn't support hot plugging). func (d *nicRouted) Start() (*deviceConfig.RunConfig, error) { // Ignore detached NICs. if !util.IsTrueOrEmpty(d.config["attached"]) { return nil, nil } err := d.validateEnvironment() if err != nil { return nil, err } // Lock to avoid issues with containers starting in parallel. networkCreateSharedDeviceLock.Lock() defer networkCreateSharedDeviceLock.Unlock() reverter := revert.New() defer reverter.Fail() saveData := make(map[string]string) // Decide which parent we should use based on VLAN setting. if d.config["vlan"] != "" { statusDev, err := networkCreateVlanDeviceIfNeeded(d.state, d.config["parent"], d.effectiveParentName, d.config["vlan"], util.IsTrue(d.config["gvrp"])) if err != nil { return nil, err } // Record whether we created this device or not so it can be removed on stop. saveData["last_state.created"] = fmt.Sprintf("%t", statusDev != "existing") // If we created a VLAN interface, we need to setup the sysctls on that interface. if util.IsTrue(saveData["last_state.created"]) { reverter.Add(func() { _ = networkRemoveInterfaceIfNeeded(d.state, d.effectiveParentName, d.inst, d.config["parent"], d.config["vlan"]) }) err := d.setupParentSysctls(d.effectiveParentName) if err != nil { return nil, err } } } if d.effectiveParentName != "" { err := d.checkIPAvailability(d.effectiveParentName) if err != nil { return nil, err } } saveData["host_name"] = d.config["host_name"] var peerName string var mtu uint32 // Create veth pair and configure the peer end with custom hwaddr and mtu if supplied. if d.inst.Type() == instancetype.Container { if saveData["host_name"] == "" { saveData["host_name"], err = d.generateHostName("veth", d.config["hwaddr"]) if err != nil { return nil, err } } peerName, mtu, err = networkCreateVethPair(saveData["host_name"], d.config) } else if d.inst.Type() == instancetype.VM { if saveData["host_name"] == "" { saveData["host_name"], err = d.generateHostName("tap", d.config["hwaddr"]) if err != nil { return nil, err } } peerName = saveData["host_name"] // VMs use the host_name to link to the TAP FD. mtu, err = networkCreateTap(saveData["host_name"], d.config) } if err != nil { return nil, err } reverter.Add(func() { _ = network.InterfaceRemove(saveData["host_name"]) }) // Populate device config with volatile fields if needed. networkVethFillFromVolatile(d.config, saveData) // Apply host-side limits. err = networkSetupHostVethLimits(&d.deviceCommon, nil, false) if err != nil { return nil, err } // Attempt to disable IPv6 router advertisement acceptance from instance. err = localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", saveData["host_name"]), "0") if err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, err } // Prevent source address spoofing by requiring a return path. err = localUtil.SysctlSet(fmt.Sprintf("net/ipv4/conf/%s/rp_filter", saveData["host_name"]), "1") if err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, err } // Apply firewall rules for reverse path filtering of IPv4 and IPv6. err = d.state.Firewall.InstanceSetupRPFilter(d.inst.Project().Name, d.inst.Name(), d.name, saveData["host_name"]) if err != nil { return nil, fmt.Errorf("Error setting up reverse path filter: %w", err) } // Perform host-side address configuration. for _, keyPrefix := range []string{"ipv4", "ipv6"} { subnetSize := 32 ipFamilyArg := ip.FamilyV4 if keyPrefix == "ipv6" { subnetSize = 128 ipFamilyArg = ip.FamilyV6 } addresses := util.SplitNTrimSpace(d.config[fmt.Sprintf("%s.address", keyPrefix)], ",", -1, true) // Add host-side gateway addresses. if len(addresses) > 0 { // Add gateway IPs to the host end of the veth pair. This ensures that liveness detection // of the gateways inside the instance work and ensure that traffic doesn't periodically // halt whilst ARP/NDP is re-detected (which is what happens with just neighbour proxies). addr := &ip.Addr{ DevName: saveData["host_name"], Address: &net.IPNet{ IP: d.ipHostAddress(keyPrefix), Mask: net.CIDRMask(subnetSize, subnetSize), }, Family: ipFamilyArg, } err = addr.Add() if err != nil { return nil, fmt.Errorf("Failed adding host gateway IP %q: %w", addr.Address, err) } // Enable IP forwarding on host_name. err = localUtil.SysctlSet(fmt.Sprintf("net/%s/conf/%s/forwarding", keyPrefix, saveData["host_name"]), "1") if err != nil { return nil, err } } getTables := func() []string { // New plural form – honour exactly what the user gives. v := d.config[fmt.Sprintf("%s.host_tables", keyPrefix)] if v != "" { return util.SplitNTrimSpace(v, ",", -1, true) } // Legacy – single key: include it plus 254. v = d.config[fmt.Sprintf("%s.host_table", keyPrefix)] if v != "" { if v == "254" { return []string{"254"} // user asked for main only } return []string{v, "254"} // custom + main } // Default – main only. return []string{"254"} } tables := getTables() // Perform per-address host-side configuration (static routes and neighbour proxy entries). for _, addrStr := range addresses { // Apply host-side static routes to main routing table or VRF. address := net.ParseIP(addrStr) if address == nil { return nil, fmt.Errorf("Invalid address %q", addrStr) } // If a VRF is set we still add a route into the VRF's own table (empty Table value). if d.config["vrf"] != "" { r := ip.Route{ DevName: saveData["host_name"], Route: &net.IPNet{ IP: address, Mask: net.CIDRMask(subnetSize, subnetSize), }, Table: "", Family: ipFamilyArg, VRF: d.config["vrf"], } err = r.Add() if err != nil { return nil, fmt.Errorf("Failed adding host route %q: %w", r.Route, err) } } // Add routes to all requested tables. for _, tbl := range tables { r := ip.Route{ DevName: saveData["host_name"], Route: &net.IPNet{ IP: address, Mask: net.CIDRMask(subnetSize, subnetSize), }, Table: tbl, Family: ipFamilyArg, } err = r.Add() if err != nil { return nil, fmt.Errorf("Failed adding host route %q to table %q: %w", r.Route, r.Table, err) } } // If there is a parent interface, add neighbour proxy entry. if d.effectiveParentName != "" { np := ip.NeighProxy{ DevName: d.effectiveParentName, Addr: net.ParseIP(addrStr), } err = np.Add() if err != nil { return nil, fmt.Errorf("Failed adding neighbour proxy %q to %q: %w", np.Addr.String(), np.DevName, err) } reverter.Add(func() { _ = np.Delete() }) } } if d.config[fmt.Sprintf("%s.routes", keyPrefix)] != "" { routes := util.SplitNTrimSpace(d.config[fmt.Sprintf("%s.routes", keyPrefix)], ",", -1, true) if len(addresses) == 0 { return nil, fmt.Errorf("%s.routes requires %s.address to be set", keyPrefix, keyPrefix) } viaAddress := net.ParseIP(addresses[0]) if viaAddress == nil { return nil, fmt.Errorf("Invalid address %q", addresses[0]) } // Add routes for _, routeStr := range routes { route, err := ip.ParseIPNet(routeStr) if err != nil { return nil, fmt.Errorf("Invalid route %q: %w", routeStr, err) } // If a VRF is set we still add a route into the VRF's own table (empty Table value). if d.config["vrf"] != "" { r := ip.Route{ DevName: saveData["host_name"], Route: route, Table: "", Family: ipFamilyArg, Via: viaAddress, VRF: d.config["vrf"], } err = r.Add() if err != nil { return nil, fmt.Errorf("Failed adding route %q: %w", r.Route, err) } } // Add routes to all requested tables. for _, tbl := range tables { r := ip.Route{ DevName: saveData["host_name"], Route: route, Table: tbl, Family: ipFamilyArg, Via: viaAddress, } err = r.Add() if err != nil { return nil, fmt.Errorf("Failed adding route %q to table %q: %w", r.Route, r.Table, err) } } } } } err = d.volatileSet(saveData) if err != nil { return nil, err } // Perform instance NIC configuration. runConf := deviceConfig.RunConfig{} runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "type", Value: "phys"}, {Key: "name", Value: d.config["name"]}, {Key: "flags", Value: "up"}, {Key: "link", Value: peerName}, {Key: "hwaddr", Value: d.config["hwaddr"]}, {Key: "connected", Value: d.config["connected"]}, } if d.config["io.bus"] == "usb" { runConf.UseUSBBus = true } if d.inst.Type() == instancetype.Container { for _, keyPrefix := range []string{"ipv4", "ipv6"} { ipAddresses := util.SplitNTrimSpace(d.config[fmt.Sprintf("%s.address", keyPrefix)], ",", -1, true) // Use a fixed address as the auto next-hop default gateway if using this IP family. if len(ipAddresses) > 0 && nicHasAutoGateway(d.config[fmt.Sprintf("%s.gateway", keyPrefix)]) { runConf.NetworkInterface = append(runConf.NetworkInterface, deviceConfig.RunConfigItem{Key: fmt.Sprintf("%s.gateway", keyPrefix), Value: d.ipHostAddress(keyPrefix).String()}, ) } for _, addrStr := range ipAddresses { // Add addresses to instance NIC. if keyPrefix == "ipv6" { runConf.NetworkInterface = append(runConf.NetworkInterface, deviceConfig.RunConfigItem{Key: "ipv6.address", Value: fmt.Sprintf("%s/128", addrStr)}, ) } else { // Specify the broadcast address as 0.0.0.0 as there is no broadcast address on // this link. This stops liblxc from trying to calculate a broadcast address // (and getting it wrong) which can prevent instances communicating with each other // using adjacent IP addresses. runConf.NetworkInterface = append(runConf.NetworkInterface, deviceConfig.RunConfigItem{Key: "ipv4.address", Value: fmt.Sprintf("%s/32 0.0.0.0", addrStr)}, ) } } } } else if d.inst.Type() == instancetype.VM { runConf.NetworkInterface = append(runConf.NetworkInterface, []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "mtu", Value: fmt.Sprintf("%d", mtu)}, }...) } reverter.Success() return &runConf, nil } // setupParentSysctls configures the required sysctls on the parent to allow l2proxy to work. // Because of our policy not to modify sysctls on existing interfaces, this should only be called // if we created the parent interface. func (d *nicRouted) setupParentSysctls(parentName string) error { if d.config["ipv4.address"] != "" { // Set necessary sysctls for use with l2proxy parent in routed mode. ipv4FwdPath := fmt.Sprintf("net/ipv4/conf/%s/forwarding", parentName) err := localUtil.SysctlSet(ipv4FwdPath, "1") if err != nil { return fmt.Errorf("Error setting net sysctl %s: %w", ipv4FwdPath, err) } } if d.config["ipv6.address"] != "" { // Set necessary sysctls use with l2proxy parent in routed mode. ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", parentName) err := localUtil.SysctlSet(ipv6FwdPath, "1") if err != nil { return fmt.Errorf("Error setting net sysctl %s: %w", ipv6FwdPath, err) } ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", parentName) err = localUtil.SysctlSet(ipv6ProxyNdpPath, "1") if err != nil { return fmt.Errorf("Error setting net sysctl %s: %w", ipv6ProxyNdpPath, err) } } return nil } // Update returns an error as most devices do not support live updates without being restarted. func (d *nicRouted) Update(oldDevices deviceConfig.Devices, isRunning bool) error { v := d.volatileGet() // If instance is running, apply host side limits. if isRunning { err := d.validateEnvironment() if err != nil { return err } // Populate device config with volatile fields if needed. networkVethFillFromVolatile(d.config, v) // Apply host-side limits. err = networkSetupHostVethLimits(&d.deviceCommon, oldDevices[d.name], false) if err != nil { return err } return d.setNICLink() } return nil } // Stop is run when the device is removed from the instance. func (d *nicRouted) Stop() (*deviceConfig.RunConfig, error) { // Populate device config with volatile fields (hwaddr and host_name) if needed. networkVethFillFromVolatile(d.config, d.volatileGet()) err := networkClearHostVethLimits(&d.deviceCommon) if err != nil { return nil, err } runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *nicRouted) postStop() error { defer func() { _ = d.volatileSet(map[string]string{ "last_state.created": "", "host_name": "", }) }() errs := []error{} v := d.volatileGet() networkVethFillFromVolatile(d.config, v) if d.config["parent"] != "" { d.effectiveParentName = network.GetHostDevice(d.config["parent"], d.config["vlan"]) } // Delete host-side interface. if network.InterfaceExists(d.config["host_name"]) { // Removing host-side end of veth pair will delete the peer end too. err := network.InterfaceRemove(d.config["host_name"]) if err != nil { errs = append(errs, fmt.Errorf("Failed to remove interface %q: %w", d.config["host_name"], err)) } } // Delete IP neighbour proxy entries on the parent. if d.effectiveParentName != "" { for _, key := range []string{"ipv4.address", "ipv6.address"} { for _, addr := range util.SplitNTrimSpace(d.config[key], ",", -1, true) { neighProxy := &ip.NeighProxy{ DevName: d.effectiveParentName, Addr: net.ParseIP(addr), } _ = neighProxy.Delete() } } } // This will delete the parent interface if we created it for VLAN parent. if util.IsTrue(v["last_state.created"]) { err := networkRemoveInterfaceIfNeeded(d.state, d.effectiveParentName, d.inst, d.config["parent"], d.config["vlan"]) if err != nil { errs = append(errs, err) } } // Remove reverse path filters. err := d.state.Firewall.InstanceClearRPFilter(d.inst.Project().Name, d.inst.Name(), d.name) if err != nil { errs = append(errs, err) } if len(errs) > 0 { return fmt.Errorf("%v", errs) } return nil } func (d *nicRouted) ipHostAddress(ipFamily string) net.IP { key := fmt.Sprintf("%s.host_address", ipFamily) if d.config[key] != "" { return net.ParseIP(d.config[key]) } return nicRoutedIPGateway[ipFamily] } func (d *nicRouted) isUniqueWithGatewayAutoMode(instConf instance.ConfigReader) error { instDevs := instConf.ExpandedDevices() for _, k := range []string{"ipv4.gateway", "ipv6.gateway"} { if d.config[k] != "auto" && d.config[k] != "" { continue // nothing to do as auto not being used. } // Check other routed NIC devices don't have auto set. for nicName, nicConfig := range instDevs { if nicName == d.name || nicConfig["nictype"] != "routed" { continue // Skip ourselves. } if nicConfig[k] == "auto" || nicConfig[k] == "" { return fmt.Errorf("Existing NIC %q already uses %q in auto mode", nicName, k) } } } return nil } incus-7.0.0/internal/server/device/nic_sriov.go000066400000000000000000000353121517523235500215560ustar00rootroot00000000000000package device import ( "errors" "fmt" "net/http" "slices" "github.com/lxc/incus/v7/internal/linux" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" pcidev "github.com/lxc/incus/v7/internal/server/device/pci" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/util" ) type nicSRIOV struct { deviceCommon network network.Network // Populated in validateConfig(). } // CanHotPlug returns whether the device can be managed whilst the instance is running. Returns true. func (d *nicSRIOV) CanHotPlug() bool { return true } // CanMigrate returns whether the device can be migrated to any other cluster member. func (d *nicSRIOV) CanMigrate() bool { return d.config["network"] != "" } // validateConfig checks the supplied config for correctness. func (d *nicSRIOV) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container, instancetype.VM) { return ErrUnsupportedDevType } var requiredFields []string optionalFields := []string{ // gendoc:generate(entity=devices, group=nic_sriov, key=name) // // --- // type: string // default: kernel assigned // managed: no // shortdesc: The name of the interface inside the instance "name", // gendoc:generate(entity=devices, group=nic_sriov, key=network) // // --- // type: string // managed: no // shortdesc: The managed network to link the device to (instead of specifying the `nictype` directly) "network", // gendoc:generate(entity=devices, group=nic_sriov, key=parent) // // --- // type: string // managed: yes // shortdesc: The name of the parent host device (required if specifying the `nictype` directly) "parent", // gendoc:generate(entity=devices, group=nic_sriov, key=hwaddr) // // --- // type: string // default: randomly assigned // managed: no // shortdesc: The MAC address of the new interface "hwaddr", // gendoc:generate(entity=devices, group=nic_sriov, key=mtu) // // --- // type: integer // default: kernel assigned // managed: yes // shortdesc: The Maximum Transmit Unit (MTU) of the new interface "mtu", // gendoc:generate(entity=devices, group=nic_sriov, key=vlan) // // --- // type: integer // managed: no // shortdesc: The VLAN ID to attach to "vlan", // gendoc:generate(entity=devices, group=nic_sriov, key=security.mac_filtering) // // --- // type: bool // default: false // managed: no // shortdesc: Prevent the instance from spoofing another instance's MAC address "security.mac_filtering", // gendoc:generate(entity=devices, group=nic_sriov, key=security.trusted) // // --- // type: bool // default: false, if supported by parent device // managed: no // shortdesc: Allows the instance to configure the NIC in ways that may negatively impact security. "security.trusted", // gendoc:generate(entity=devices, group=nic_sriov, key=boot.priority) // // --- // type: integer // managed: no // shortdesc: Boot priority for VMs (higher value boots first) "boot.priority", // gendoc:generate(entity=devices, group=nic_sriov, key=vendorid) // // --- // type: string // required: no // shortdesc: The vendor ID of the parent host device "vendorid", // gendoc:generate(entity=devices, group=nic_sriov, key=productid) // // --- // type: string // required: no // shortdesc: The product ID of the parent host device "productid", // gendoc:generate(entity=devices, group=nic_sriov, key=pci) // // --- // type: string // required: no // shortdesc: The PCI address of the parent host device "pci", // gendoc:generate(entity=devices, group=nic_sriov, key=attached) // // --- // type: bool // default: `true` // required: no // shortdesc: Whether the NIC is plugged in or not "attached", } // Check that if network property is set that conflicting keys are not present. if d.config["network"] != "" { requiredFields = append(requiredFields, "network") bannedKeys := []string{"nictype", "parent", "mtu", "vlan"} for _, bannedKey := range bannedKeys { if d.config[bannedKey] != "" { return fmt.Errorf("Cannot use %q property in conjunction with %q property", bannedKey, "network") } } // If network property is specified, lookup network settings and apply them to the device's config. // api.ProjectDefaultName is used here as macvlan networks don't support projects. var err error d.network, err = network.LoadByName(d.state, api.ProjectDefaultName, d.config["network"]) if err != nil { return fmt.Errorf("Error loading network config for %q: %w", d.config["network"], err) } if d.network.Status() != api.NetworkStatusCreated { return errors.New("Specified network is not fully created") } if d.network.Type() != "sriov" { return errors.New("Specified network must be of type macvlan") } netConfig := d.network.Config() // Get actual parent device from network's parent setting. d.config["parent"] = netConfig["parent"] // Copy certain keys verbatim from the network's settings. inheritKeys := []string{"mtu", "vlan"} for _, inheritKey := range inheritKeys { _, found := netConfig[inheritKey] if found { d.config[inheritKey] = netConfig[inheritKey] } } } else if d.isParentRequired() { // If no network property supplied, then parent property is required. requiredFields = append(requiredFields, "parent") } err := d.config.Validate(nicValidationRules(requiredFields, optionalFields, instConf)) if err != nil { return err } if d.config["parent"] != "" { for _, field := range []string{"pci", "productid", "vendorid"} { if d.config[field] != "" { return fmt.Errorf(`Cannot use %q when "parent" is set`, field) } } } if d.config["pci"] != "" { for _, field := range []string{"parent", "productid", "vendorid"} { if d.config[field] != "" { return fmt.Errorf(`Cannot use %q when "pci" is set`, field) } } d.config["pci"] = pcidev.NormaliseAddress(d.config["pci"]) } return nil } // PreStartCheck checks the managed parent network is available (if relevant). func (d *nicSRIOV) PreStartCheck() error { // Non-managed network NICs are not relevant for checking managed network availability. if d.network == nil { return nil } // If managed network is not available, don't try and start instance. if d.network.LocalStatus() == api.NetworkStatusUnavailable { return api.StatusErrorf(http.StatusServiceUnavailable, "Network %q unavailable on this server", d.network.Name()) } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *nicSRIOV) validateEnvironment() error { if d.inst.Type() == instancetype.VM && util.IsTrue(d.inst.ExpandedConfig()["migration.stateful"]) { return errors.New("Network SR-IOV devices cannot be used when migration.stateful is enabled") } if d.inst.Type() == instancetype.Container && d.config["name"] == "" { return errors.New("Requires name property to start") } if d.isParentRequired() && !network.InterfaceExists(d.config["parent"]) { return fmt.Errorf("Parent device %q doesn't exist", d.config["parent"]) } return nil } // Start is run when the device is added to a running instance or instance is starting up. func (d *nicSRIOV) Start() (*deviceConfig.RunConfig, error) { // Ignore detached NICs. if !util.IsTrueOrEmpty(d.config["attached"]) { return nil, nil } err := d.validateEnvironment() if err != nil { return nil, err } saveData := make(map[string]string) // If VM, then try and load the vfio-pci module first. if d.inst.Type() == instancetype.VM { err = linux.LoadModule("vfio-pci") if err != nil { return nil, fmt.Errorf("Error loading %q module: %w", "vfio-pci", err) } } parent := d.config["parent"] // Try to find parent if not set. if parent == "" { parent, err = d.findParent() if err != nil { return nil, err } } // Find free VF exclusively. network.SRIOVVirtualFunctionMutex.Lock() vfDev, vfID, err := network.SRIOVFindFreeVirtualFunction(d.state, parent) if err != nil { network.SRIOVVirtualFunctionMutex.Unlock() return nil, err } // Claim the SR-IOV virtual function (VF) on the parent (PF) and get the PCI information. vfPCIDev, pciIOMMUGroup, err := networkSRIOVSetupVF(d.deviceCommon, parent, vfDev, vfID, saveData) if err != nil { network.SRIOVVirtualFunctionMutex.Unlock() return nil, err } network.SRIOVVirtualFunctionMutex.Unlock() if d.inst.Type() == instancetype.Container { err := networkSRIOVSetupContainerVFNIC(saveData["host_name"], d.inst.MACPattern(), d.config) if err != nil { return nil, err } } // Save new volatile keys. err = d.volatileSet(saveData) if err != nil { return nil, err } // Get all volatile keys. volatile := d.volatileGet() // Apply stable MAC address. if d.config["hwaddr"] == "" { d.config["hwaddr"] = volatile["hwaddr"] } runConf := deviceConfig.RunConfig{} runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "type", Value: "phys"}, {Key: "name", Value: d.config["name"]}, {Key: "flags", Value: "up"}, {Key: "link", Value: saveData["host_name"]}, {Key: "hwaddr", Value: d.config["hwaddr"]}, } if d.inst.Type() == instancetype.VM { runConf.NetworkInterface = append(runConf.NetworkInterface, []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "pciSlotName", Value: vfPCIDev.SlotName}, {Key: "pciIOMMUGroup", Value: fmt.Sprintf("%d", pciIOMMUGroup)}, }...) } return &runConf, nil } // Stop is run when the device is removed from the instance. func (d *nicSRIOV) Stop() (*deviceConfig.RunConfig, error) { v := d.volatileGet() runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } if util.IsTrueOrEmpty(d.config["attached"]) { runConf.NetworkInterface = []deviceConfig.RunConfigItem{ {Key: "link", Value: v["host_name"]}, } } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *nicSRIOV) postStop() error { defer func() { _ = d.volatileSet(map[string]string{ "host_name": "", "last_state.hwaddr": "", "last_state.mtu": "", "last_state.created": "", "last_state.vf.parent": "", "last_state.vf.id": "", "last_state.vf.hwaddr": "", "last_state.vf.vlan": "", "last_state.vf.spoofcheck": "", "last_state.vf.trusted": "", "last_state.pci.driver": "", }) }() v := d.volatileGet() network.SRIOVVirtualFunctionMutex.Lock() err := networkSRIOVRestoreVF(d.deviceCommon, true, v) if err != nil { network.SRIOVVirtualFunctionMutex.Unlock() return err } network.SRIOVVirtualFunctionMutex.Unlock() return nil } // findParent selects the best NIC based on vendorid, productid or PCI address, // considering NUMA nodes. func (d *nicSRIOV) findParent() (string, error) { // List all the NICs. interfaces, err := resources.GetNetwork() if err != nil { return "", err } numaNodeSet, numaNodeSetFallback, err := getNumaNodeSet(d.inst.ExpandedConfig()) if err != nil { return "", err } parent := "" vfFreeRatio := 0.0 cardNUMA := -1 for _, nic := range interfaces.Cards { // Skip any cards that are not selected. if !nicSelected(d.Config(), nic) { continue } // Skip any card without SR-IOV. if nic.SRIOV == nil { d.logger.Debug("Skip card without SR-IOV", logger.Ctx{"pci": nic.PCIAddress}) continue } // Find available VFs. currentVfFreeRatio := 0.0 currentParent := "" network.SRIOVVirtualFunctionMutex.Lock() for _, port := range nic.Ports { freeVf, totalVf, err := network.SRIOVCountFreeVirtualFunctions(d.state, port.ID) if err != nil { network.SRIOVVirtualFunctionMutex.Unlock() return "", err } tmpRatio := float64(freeVf) / float64(totalVf) if tmpRatio > currentVfFreeRatio { currentVfFreeRatio = tmpRatio currentParent = port.ID } } network.SRIOVVirtualFunctionMutex.Unlock() // Skip if no available VFs. if currentVfFreeRatio == 0 { d.logger.Debug("No available VFs on card", logger.Ctx{"pci": nic.PCIAddress}) continue } // Handle NUMA. if numaNodeSet != nil { // Switch to current card if it matches our main NUMA node and existing card doesn't. if !slices.Contains(numaNodeSet, int64(cardNUMA)) && slices.Contains(numaNodeSet, int64(nic.NUMANode)) { parent = currentParent vfFreeRatio = currentVfFreeRatio cardNUMA = int(nic.NUMANode) continue } // Skip current card if we already have a card matching our main NUMA node and this card doesn't. if slices.Contains(numaNodeSet, int64(cardNUMA)) && !slices.Contains(numaNodeSet, int64(nic.NUMANode)) { continue } // Switch to current card if it matches a fallback NUMA node and existing card doesn't. if !slices.Contains(numaNodeSetFallback, int64(cardNUMA)) && slices.Contains(numaNodeSetFallback, int64(nic.NUMANode)) { parent = currentParent vfFreeRatio = currentVfFreeRatio cardNUMA = int(nic.NUMANode) continue } // Skip current card if we already have a card matching a fallback NUMA node and this card isn't on the main or fallback node. if slices.Contains(numaNodeSetFallback, int64(cardNUMA)) && !slices.Contains(numaNodeSetFallback, int64(nic.NUMANode)) && !slices.Contains(numaNodeSet, int64(nic.NUMANode)) { continue } } // Prioritize less busy cards. if parent == "" || currentVfFreeRatio > vfFreeRatio { parent = currentParent vfFreeRatio = currentVfFreeRatio cardNUMA = int(nic.NUMANode) d.logger.Debug("Selected NIC", logger.Ctx{"PCI": nic.PCIAddress, "parent": parent}) continue } } // Check if any NIC was found to match. if parent == "" { return "", errors.New("Couldn't find a matching NIC") } return parent, nil } // isParentRequired checks whether the parent config option is required. func (d *nicSRIOV) isParentRequired() bool { if d.config["pci"] == "" && d.config["vendorid"] == "" && d.config["productid"] == "" { return true } return false } // Check if the device matches the given NIC. // It matches based on vendorid, productid or pci setting of the device. func nicSelected(device deviceConfig.Device, nic api.ResourcesNetworkCard) bool { if device["pci"] != "" && nic.PCIAddress == device["pci"] { return true } if device["vendorid"] != "" && device["productid"] != "" { if nic.VendorID == device["vendorid"] && nic.ProductID == device["productid"] { return true } } else if device["vendorid"] != "" { if nic.VendorID == device["vendorid"] { return true } } return false } incus-7.0.0/internal/server/device/nictype/000077500000000000000000000000001517523235500207035ustar00rootroot00000000000000incus-7.0.0/internal/server/device/nictype/nictype.go000066400000000000000000000043611517523235500227110ustar00rootroot00000000000000// Package nictype is a small package to allow resolving NIC "network" key to "nictype" key. // It is it's own package to avoid circular dependency issues. package nictype import ( "context" "fmt" "github.com/lxc/incus/v7/internal/server/db" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" ) // NICType resolves the NIC Type for the supplied NIC device config. // If the device "type" is "nic" and the "network" property is specified in the device config, then NIC type is // resolved from the network's type. Otherwise the device's "nictype" property is returned (which may be empty if // used with non-NIC device configs). func NICType(s *state.State, deviceProjectName string, d deviceConfig.Device) (string, error) { // NIC devices support resolving their "nictype" from their "network" property. if d["type"] == "nic" { if d["network"] != "" { // Translate device's project name into a network project name. networkProjectName, _, err := project.NetworkProject(s.DB.Cluster, deviceProjectName) if err != nil { return "", fmt.Errorf("Failed to translate device project %q into network project: %w", deviceProjectName, err) } var netInfo *api.Network err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, netInfo, _, err = tx.GetNetworkInAnyState(ctx, networkProjectName, d["network"]) return err }) if err != nil { return "", fmt.Errorf("Failed to load network %q for project %q: %w", d["network"], networkProjectName, err) } var nicType string switch netInfo.Type { case "bridge": nicType = "bridged" case "macvlan": nicType = "macvlan" case "sriov": nicType = "sriov" case "ovn": nicType = "ovn" case "physical": nicType = "physical" default: return "", fmt.Errorf("Unrecognised NIC network type for network %q", d["network"]) } return nicType, nil } } // Infiniband devices use "nictype" without supporting "network" property, so just return it directly, // which is the same as accessing the property directly from the config. return d["nictype"], nil } incus-7.0.0/internal/server/device/none.go000066400000000000000000000021141517523235500205140ustar00rootroot00000000000000package device import ( deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" ) type none struct { deviceCommon } // CanMigrate returns whether the device can be migrated to any other cluster member. func (d *none) CanMigrate() bool { return true } // CanHotPlug returns whether the device can be managed whilst the instance is running. func (d *none) CanHotPlug() bool { return true } // validateConfig checks the supplied config for correctness. // validateConfig checks the supplied config for correctness. func (d *none) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { rules := map[string]func(string) error{} // No fields allowed. err := d.config.Validate(rules) if err != nil { return err } return nil } // Start is run when the device is added to the container. func (d *none) Start() (*deviceConfig.RunConfig, error) { return nil, nil } // Stop is run when the device is removed from the instance. func (d *none) Stop() (*deviceConfig.RunConfig, error) { return nil, nil } incus-7.0.0/internal/server/device/pci.go000066400000000000000000000103021517523235500203260ustar00rootroot00000000000000package device import ( "errors" "fmt" "path/filepath" "github.com/lxc/incus/v7/internal/linux" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" pcidev "github.com/lxc/incus/v7/internal/server/device/pci" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) type pci struct { deviceCommon } // validateConfig checks the supplied config for correctness. func (d *pci) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.VM) { return ErrUnsupportedDevType } rules := map[string]func(string) error{ // gendoc:generate(entity=devices, group=pci, key=address) // // --- // type: string // required: yes // shortdesc: PCI address of the device "address": validate.IsPCIAddress, // gendoc:generate(entity=devices, group=pci, key=firmware) // // --- // type: bool // required: no // default: true // shortdesc: Whether to expose the device's option ROM to the VM "firmware": validate.Optional(validate.IsBool), } err := d.config.Validate(rules) if err != nil { return fmt.Errorf("Failed to validate config: %w", err) } d.config["address"] = pcidev.NormaliseAddress(d.config["address"]) return nil } // validateEnvironment checks if the PCI device is available. func (d *pci) validateEnvironment() error { if d.inst.Type() == instancetype.VM && util.IsTrue(d.inst.ExpandedConfig()["migration.stateful"]) { return errors.New("PCI devices cannot be used when migration.stateful is enabled") } return validatePCIDevice(d.config["address"]) } // Start is run when the device is added to the instance. func (d *pci) Start() (*deviceConfig.RunConfig, error) { err := d.validateEnvironment() if err != nil { return nil, fmt.Errorf("Failed to validate environment: %w", err) } runConf := deviceConfig.RunConfig{} saveData := make(map[string]string) // Make sure that vfio-pci is loaded. err = linux.LoadModule("vfio-pci") if err != nil { return nil, fmt.Errorf("Error loading %q module: %w", "vfio-pci", err) } // Get PCI information about the device. pciAddress := d.config["address"] devicePath := filepath.Join("/sys/bus/pci/devices", pciAddress) pciDev, err := pcidev.ParseUeventFile(filepath.Join(devicePath, "uevent")) if err != nil { return nil, fmt.Errorf("Failed to get PCI device info for %q: %w", pciAddress, err) } saveData["last_state.pci.slot.name"] = pciDev.SlotName saveData["last_state.pci.driver"] = pciDev.Driver pciIOMMUGroup, err := pcidev.DeviceIOMMUGroup(saveData["last_state.pci.slot.name"]) if err != nil { return nil, err } err = pcidev.DeviceDriverOverride(pciDev, "vfio-pci") if err != nil { return nil, fmt.Errorf("Failed to override IOMMU group driver: %w", err) } runConf.PCIDevice = append(runConf.PCIDevice, []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "firmware", Value: d.config["firmware"]}, {Key: "pciSlotName", Value: saveData["last_state.pci.slot.name"]}, {Key: "pciIOMMUGroup", Value: fmt.Sprintf("%d", pciIOMMUGroup)}, }...) err = d.volatileSet(saveData) if err != nil { return nil, err } return &runConf, nil } // CanHotPlug returns whether the device can be managed whilst the instance is running. func (d *pci) CanHotPlug() bool { return true } // Stop is run when the device is removed from the instance. func (d *pci) Stop() (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *pci) postStop() error { defer func() { _ = d.volatileSet(map[string]string{ "last_state.pci.slot.name": "", "last_state.pci.driver": "", }) }() v := d.volatileGet() // Unbind from vfio-pci and bind back to host driver. if v["last_state.pci.slot.name"] != "" { pciDev := pcidev.Device{ Driver: "vfio-pci", SlotName: v["last_state.pci.slot.name"], } err := pcidev.DeviceDriverOverride(pciDev, v["last_state.pci.driver"]) if err != nil { return err } } return nil } incus-7.0.0/internal/server/device/pci/000077500000000000000000000000001517523235500200035ustar00rootroot00000000000000incus-7.0.0/internal/server/device/pci/pci.go000066400000000000000000000131361517523235500211110ustar00rootroot00000000000000package pci import ( "bufio" "errors" "fmt" "io/fs" "os" "path/filepath" "strconv" "strings" "time" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" ) // ErrDeviceIsUSB is returned when dealing with a USB device. var ErrDeviceIsUSB = errors.New("Device is USB instead of PCI") // Device represents info about a PCI uevent device. type Device struct { ID string SlotName string Driver string } // ParseUeventFile returns the PCI device info for a given uevent file. func ParseUeventFile(ueventFilePath string) (Device, error) { dev := Device{} file, err := os.Open(ueventFilePath) if err != nil { return dev, err } defer func() { _ = file.Close() }() scanner := bufio.NewScanner(file) for scanner.Scan() { // Looking for something like this "PCI_SLOT_NAME=0000:05:10.0" fields := strings.SplitN(scanner.Text(), "=", 2) if len(fields) == 2 { if fields[0] == "PCI_SLOT_NAME" { dev.SlotName = fields[1] } else if fields[0] == "PCI_ID" { dev.ID = fields[1] } else if fields[0] == "DEVTYPE" && fields[1] == "usb_interface" { return dev, ErrDeviceIsUSB } else if fields[0] == "DRIVER" { dev.Driver = fields[1] } } } err = scanner.Err() if err != nil { return dev, err } if dev.SlotName == "" { return dev, errors.New("Device uevent file could not be parsed") } return dev, nil } // DeviceUnbind unbinds a PCI device from the OS using its PCI Slot Name. func DeviceUnbind(pciDev Device) error { driverUnbindPath := fmt.Sprintf("/sys/bus/pci/devices/%s/driver/unbind", pciDev.SlotName) err := os.WriteFile(driverUnbindPath, []byte(pciDev.SlotName), 0o600) if err != nil { if !errors.Is(err, fs.ErrNotExist) || !util.PathExists(fmt.Sprintf("/sys/bus/pci/devices/%s/", pciDev.SlotName)) { return fmt.Errorf("Failed unbinding device %q via %q: %w", pciDev.SlotName, driverUnbindPath, err) } } return nil } // DeviceSetDriverOverride registers an override driver for a PCI device using its PCI Slot Name. func DeviceSetDriverOverride(pciDev Device, driverOverride string) error { overridePath := filepath.Join("/sys/bus/pci/devices", pciDev.SlotName, "driver_override") // The "\n" at end is important to allow the driver override to be cleared by passing "" in. err := os.WriteFile(overridePath, fmt.Appendf(nil, "%s\n", driverOverride), 0o600) if err != nil { return fmt.Errorf("Failed setting driver override %q for device %q via %q: %w", driverOverride, pciDev.SlotName, overridePath, err) } return nil } // DeviceProbe probes a PCI device using its PCI Slot Name. func DeviceProbe(pciDev Device) error { driveProbePath := "/sys/bus/pci/drivers_probe" err := os.WriteFile(driveProbePath, []byte(pciDev.SlotName), 0o600) if err != nil { return fmt.Errorf("Failed probing device %q via %q: %w", pciDev.SlotName, driveProbePath, err) } return nil } // DeviceDriverOverride unbinds the device, sets the driver override preference, then probes the device, and // waits for it to be activated with the specified driver. func DeviceDriverOverride(pciDev Device, driverOverride string) error { reverter := revert.New() defer reverter.Fail() // Check if already bound to the target driver. _, err := os.Stat(filepath.Join("/sys/bus/pci/drivers", driverOverride, pciDev.SlotName)) if err == nil { return nil } // Unbind the device from the host (ignore if not bound). err = DeviceUnbind(pciDev) if err != nil && errors.Is(err, fs.ErrNotExist) { return err } reverter.Add(func() { // Reset the driver override and rebind to original driver (if needed). _ = DeviceUnbind(pciDev) _ = DeviceSetDriverOverride(pciDev, pciDev.Driver) _ = DeviceProbe(pciDev) }) // Set driver override. err = DeviceSetDriverOverride(pciDev, driverOverride) if err != nil { return err } // Probe device to bind it to overridden driver. err = DeviceProbe(pciDev) if err != nil { return err } vfioDev := Device{ Driver: driverOverride, SlotName: pciDev.SlotName, } // Wait for the device to be bound to the overridden driver if specified. if vfioDev.Driver != "" { err = deviceProbeWait(vfioDev) if err != nil { return err } } reverter.Success() return nil } // deviceProbeWait waits for PCI device to be activated with the specified driver after being probed. func deviceProbeWait(pciDev Device) error { driverPath := fmt.Sprintf("/sys/bus/pci/drivers/%s/%s", pciDev.Driver, pciDev.SlotName) for range 10 { if util.PathExists(driverPath) { return nil } time.Sleep(50 * time.Millisecond) } return fmt.Errorf("Device took too long to activate at %q", driverPath) } // NormaliseAddress converts common PCI address notation to the kernel's notation. func NormaliseAddress(addr string) string { // PCI devices can be specified as "0000:XX:XX.X" or "XX:XX.X". // However, the devices in /sys/bus/pci/devices use the long format which // is why we need to make sure the prefix is present. if len(addr) == 7 { addr = fmt.Sprintf("0000:%s", addr) } // Ensure all addresses are lowercase. addr = strings.ToLower(addr) return addr } // DeviceIOMMUGroup returns the IOMMU group for a PCI device. func DeviceIOMMUGroup(slotName string) (uint64, error) { iommuGroupSymPath := fmt.Sprintf("/sys/bus/pci/devices/%s/iommu_group", slotName) _, err := os.Lstat(iommuGroupSymPath) if err != nil { return 0, err } iommuGroupPath, err := os.Readlink(iommuGroupSymPath) if err != nil { return 0, err } iommuGroupStr := filepath.Base(iommuGroupPath) iommuGroup, err := strconv.ParseUint(iommuGroupStr, 10, 64) if err != nil { return 0, fmt.Errorf("Failed to parse %q: %w", iommuGroupStr, err) } return iommuGroup, nil } incus-7.0.0/internal/server/device/pci/pci_test.go000066400000000000000000000010201517523235500221350ustar00rootroot00000000000000package pci_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/lxc/incus/v7/internal/server/device/pci" ) func TestNormaliseAddress(t *testing.T) { cases := map[string]string{ "": "", "0000:00:00.0": "0000:00:00.0", "1000:00:00.0": "1000:00:00.0", "00:00.0": "0000:00:00.0", "0000:AB:00.0": "0000:ab:00.0", "1000:AB:00.0": "1000:ab:00.0", "00:AB.0": "0000:00:ab.0", } for k, v := range cases { res := pci.NormaliseAddress(k) assert.Equal(t, res, v) } } incus-7.0.0/internal/server/device/proxy.go000066400000000000000000000530411517523235500207430ustar00rootroot00000000000000package device import ( "bufio" "context" "errors" "fmt" "net" "os" "path/filepath" "slices" "strconv" "strings" "time" liblxc "github.com/lxc/go-lxc" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/apparmor" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/warningtype" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/device/nictype" firewallDrivers "github.com/lxc/incus/v7/internal/server/firewall/drivers" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/warnings" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) type proxy struct { deviceCommon } type proxyProcInfo struct { listenPid string listenPidFd string connectPid string connectPidFd string connectAddr string listenAddr string listenAddrGID string listenAddrUID string listenAddrMode string securityUID string securityGID string proxyProtocol string inheritFds []*os.File } // CanHotPlug returns whether the device can be managed whilst the instance is running. func (d *proxy) CanHotPlug() bool { return true } // validateConfig checks the supplied config for correctness. func (d *proxy) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container, instancetype.VM) { return ErrUnsupportedDevType } validateAddr := func(input string) error { _, err := network.ProxyParseAddr(input) return err } // Supported bind types are: "host" or "instance" (or "guest" or "container", legacy options equivalent to "instance"). // If an empty value is supplied the default behavior is to assume "host" bind mode. validateBind := func(input string) error { if !slices.Contains([]string{"host", "instance", "guest", "container"}, d.config["bind"]) { return errors.New("Invalid binding side given. Must be \"host\" or \"instance\"") } return nil } rules := map[string]func(string) error{ // gendoc:generate(entity=devices, group=proxy, key=listen) // // --- // type: string // required: yes // shortdesc: The address and port to bind and listen (`::[-][,]`) "listen": validate.Required(validateAddr), // gendoc:generate(entity=devices, group=proxy, key=connect) // // --- // type: string // required: yes // shortdesc: The address and port to connect to (`::[-][,]`) "connect": validate.Required(validateAddr), // gendoc:generate(entity=devices, group=proxy, key=bind) // // --- // type: string // required: no // default: `host` // shortdesc: Which side to bind on (`host`/`instance`) "bind": validate.Optional(validateBind), // gendoc:generate(entity=devices, group=proxy, key=mode) // // --- // type: int // required: no // default: `0644` // shortdesc: Mode for the listening Unix socket "mode": validate.Optional(unixValidOctalFileMode), // gendoc:generate(entity=devices, group=proxy, key=nat) // // --- // type: bool // required: no // default: `false` // shortdesc: Whether to optimize proxying via NAT (requires that the instance NIC has a static IP address) "nat": validate.Optional(validate.IsBool), // gendoc:generate(entity=devices, group=proxy, key=gid) // // --- // type: int // required: no // default: `0` // shortdesc: GID of the owner of the listening Unix socket "gid": validate.Optional(unixValidUserID), // gendoc:generate(entity=devices, group=proxy, key=uid) // // --- // type: int // required: no // default: `0` // shortdesc: UID of the owner of the listening Unix socket "uid": validate.Optional(unixValidUserID), // gendoc:generate(entity=devices, group=proxy, key=security.uid) // // --- // type: int // required: no // default: `0` // shortdesc: What UID to drop privilege to "security.uid": validate.Optional(unixValidUserID), // gendoc:generate(entity=devices, group=proxy, key=security.gid) // // --- // type: int // required: no // default: `0` // shortdesc: What GID to drop privilege to "security.gid": validate.Optional(unixValidUserID), // gendoc:generate(entity=devices, group=proxy, key=proxy_protocol) // // --- // type: bool // required: no // default: `false` // shortdesc: Whether to use the HAProxy PROXY protocol to transmit sender information "proxy_protocol": validate.Optional(validate.IsBool), } err := d.config.Validate(rules) if err != nil { return err } if instConf.Type() == instancetype.VM && util.IsFalseOrEmpty(d.config["nat"]) { return errors.New("Only NAT mode is supported for proxies on VM instances") } listenAddr, err := network.ProxyParseAddr(d.config["listen"]) if err != nil { return err } connectAddr, err := network.ProxyParseAddr(d.config["connect"]) if err != nil { return err } err = d.validateListenAddressConflicts(net.ParseIP(listenAddr.Address)) if err != nil { return err } if (listenAddr.ConnType != "unix" && len(connectAddr.Ports) > len(listenAddr.Ports)) || (listenAddr.ConnType == "unix" && len(connectAddr.Ports) > 1) { // Cannot support single address (or port) -> multiple port. return errors.New("Mismatch between listen port(s) and connect port(s) count") } if util.IsTrue(d.config["proxy_protocol"]) && (!strings.HasPrefix(d.config["connect"], "tcp") || util.IsTrue(d.config["nat"])) { return errors.New("The PROXY header can only be sent to tcp servers in non-nat mode") } if (!strings.HasPrefix(d.config["listen"], "unix:") || strings.HasPrefix(d.config["listen"], "unix:@")) && (d.config["uid"] != "" || d.config["gid"] != "" || d.config["mode"] != "") { return errors.New("Only proxy devices for non-abstract unix sockets can carry uid, gid, or mode properties") } if util.IsTrue(d.config["nat"]) { if d.inst != nil { // Default project always has networks feature so don't bother loading the project config // in that case. instProject := d.inst.Project() if instProject.Name != api.ProjectDefaultName && util.IsTrue(instProject.Config["features.networks"]) { // Prevent use of NAT mode on non-default projects with networks feature. // This is because OVN networks don't allow the host to communicate directly with // instance NICs and so DNAT rules on the host won't work. return errors.New("NAT mode cannot be used in projects that have the networks feature") } } if d.config["bind"] != "" && d.config["bind"] != "host" { return errors.New("Only host-bound proxies can use NAT") } // Support TCP <-> TCP and UDP <-> UDP only. if listenAddr.ConnType == "unix" || connectAddr.ConnType == "unix" || listenAddr.ConnType != connectAddr.ConnType { return fmt.Errorf("Proxying %s <-> %s is not supported when using NAT", listenAddr.ConnType, connectAddr.ConnType) } listenAddress := net.ParseIP(listenAddr.Address) if listenAddress.Equal(net.IPv4zero) || listenAddress.Equal(net.IPv6zero) { return fmt.Errorf("Cannot listen on wildcard address %q when in nat mode", listenAddress.String()) } // Records which listen address IP version, as these cannot be mixed in NAT mode. listenIPVersion := uint(4) if listenAddress.To4() == nil { listenIPVersion = 6 } // Check connect address against the listen address IP version and check they match. connectAddress := net.ParseIP(connectAddr.Address) connectIPVersion := uint(4) if connectAddress.To4() == nil { connectIPVersion = 6 } if listenIPVersion != connectIPVersion { return errors.New("Cannot mix IP versions between listen and connect in nat mode") } } return nil } // validateEnvironment checks the runtime environment for correctness. func (d *proxy) validateEnvironment() error { if d.name == "" { return errors.New("Device name cannot be empty") } return nil } // validateListenAddressConflicts checks that the proxy device about to be created does not // overlap on existing network forward (both entities can't have the same listening address with // the same port number). func (d *proxy) validateListenAddressConflicts(proxyListenAddr net.IP) error { return d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var projectNetworksForwardsOnUplink map[string]map[int64][]string networksByProjects, err := tx.GetNetworksAllProjects(ctx) if err != nil { return fmt.Errorf("Failed loading network forward listen addresses: %w", err) } for projectName, networks := range networksByProjects { for _, networkName := range networks { networkID, err := tx.GetNetworkID(ctx, projectName, networkName) if err != nil { return fmt.Errorf("Failed loading network forward listen addresses: %w", err) } // Get all network forward listen addresses for all networks (of any type) connected to our uplink. networkForwards, err := cluster.GetNetworkForwards(ctx, tx.Tx(), cluster.NetworkForwardFilter{ NetworkID: &networkID, }) if err != nil { return fmt.Errorf("Failed loading network forward listen addresses: %w", err) } projectNetworksForwardsOnUplink = make(map[string]map[int64][]string) for _, forward := range networkForwards { // Filter network forwards that belong to this specific cluster member if forward.NodeID.Valid && (forward.NodeID.Int64 == tx.GetNodeID()) { if projectNetworksForwardsOnUplink[projectName] == nil { projectNetworksForwardsOnUplink[projectName] = make(map[int64][]string) } projectNetworksForwardsOnUplink[projectName][networkID] = append(projectNetworksForwardsOnUplink[projectName][networkID], forward.ListenAddress) } } } } for _, networks := range projectNetworksForwardsOnUplink { for _, listenAddresses := range networks { for _, netFwdAddr := range listenAddresses { if proxyListenAddr.Equal(net.ParseIP(netFwdAddr)) { return fmt.Errorf("Listen address %q conflicts with existing network forward", netFwdAddr) } } } } return nil }) } // Start is run when the device is added to the instance. func (d *proxy) Start() (*deviceConfig.RunConfig, error) { err := d.validateEnvironment() if err != nil { return nil, err } // Proxy devices have to be setup once the instance is running. runConf := deviceConfig.RunConfig{} runConf.PostHooks = []func() error{ func() error { if util.IsTrue(d.config["nat"]) { err = d.setupNAT() if err != nil { return fmt.Errorf("Failed to start device %q: %w", d.name, err) } return nil // Don't proceed with forkproxy setup. } proxyValues, err := d.setupProxyProcInfo() if err != nil { return err } devFileName := fmt.Sprintf("proxy.%s", d.name) pidPath := filepath.Join(d.inst.DevicesPath(), devFileName) logFileName := fmt.Sprintf("proxy.%s.log", d.name) logPath := filepath.Join(d.inst.LogPath(), logFileName) // Load the apparmor profile err = apparmor.ForkproxyLoad(d.state.OS, d.inst, d) if err != nil { return fmt.Errorf("Failed to start device %q: %w", d.name, err) } // Spawn the daemon using subprocess command := d.state.OS.ExecPath forkproxyargs := []string{ "forkproxy", "--", proxyValues.listenPid, proxyValues.listenPidFd, proxyValues.listenAddr, proxyValues.connectPid, proxyValues.connectPidFd, proxyValues.connectAddr, proxyValues.listenAddrGID, proxyValues.listenAddrUID, proxyValues.listenAddrMode, proxyValues.securityGID, proxyValues.securityUID, proxyValues.proxyProtocol, } p, err := subprocess.NewProcess(command, forkproxyargs, logPath, logPath) if err != nil { return fmt.Errorf("Failed to start device %q: Failed to creating subprocess: %w", d.name, err) } p.SetApparmor(apparmor.ForkproxyProfileName(d.inst, d)) err = p.StartWithFiles(context.Background(), proxyValues.inheritFds) if err != nil { return fmt.Errorf("Failed to start device %q: Failed running: %s %s: %w", d.name, command, strings.Join(forkproxyargs, " "), err) } for _, file := range proxyValues.inheritFds { _ = file.Close() } // Poll log file a few times until we see "Started" to indicate successful start. for range 10 { started, err := d.checkProcStarted(logPath) if err != nil { _ = p.Stop() return fmt.Errorf("Error occurred when starting proxy device: %s", err) } if started { err = p.Save(pidPath) if err != nil { // Kill Process if started, but could not save the file err2 := p.Stop() if err != nil { return fmt.Errorf("Could not kill subprocess while handling saving error: %s: %s", err, err2) } return fmt.Errorf("Failed to start device %q: Failed saving subprocess details: %w", d.name, err) } return nil } time.Sleep(time.Second) } _ = p.Stop() return fmt.Errorf("Failed to start device %q: Please look in %s", d.name, logPath) }, } return &runConf, nil } // checkProcStarted checks for the "Started" line in the log file. Returns true if found, false // if not, and error if any other error occurs. func (d *proxy) checkProcStarted(logPath string) (bool, error) { file, err := os.Open(logPath) if err != nil { return false, err } defer func() { _ = file.Close() }() scanner := bufio.NewScanner(file) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "Status: Started" { return true, nil } if strings.HasPrefix(line, "Error:") { return false, fmt.Errorf("%s", line) } } err = scanner.Err() if err != nil { return false, err } return false, nil } // Stop is run when the device is removed from the instance. func (d *proxy) Stop() (*deviceConfig.RunConfig, error) { // Remove possible firewall entries. err := d.state.Firewall.InstanceClearProxyNAT(d.inst.Project().Name, d.inst.Name(), d.name) if err != nil { logger.Errorf("Failed to remove proxy NAT filters: %v", err) } devFileName := fmt.Sprintf("proxy.%s", d.name) devPath := filepath.Join(d.inst.DevicesPath(), devFileName) if !util.PathExists(devPath) { // There's no proxy process if NAT is enabled return nil, nil } err = d.killProxyProc(devPath) if err != nil { return nil, err } // Unload apparmor profile. err = apparmor.ForkproxyUnload(d.state.OS, d.inst, d) if err != nil { return nil, err } return nil, nil } func (d *proxy) setupNAT() error { listenAddr, err := network.ProxyParseAddr(d.config["listen"]) if err != nil { return err } connectAddr, err := network.ProxyParseAddr(d.config["connect"]) if err != nil { return err } ipVersion := uint(4) if strings.Contains(listenAddr.Address, ":") { ipVersion = 6 } var connectIP net.IP var hostName string for devName, devConfig := range d.inst.ExpandedDevices() { if devConfig["type"] != "nic" { continue } nicType, err := nictype.NICType(d.state, d.inst.Project().Name, devConfig) if err != nil { return err } // Check if the instance has a NIC with a static IP that is reachable from the host. if !slices.Contains([]string{"bridged", "routed"}, nicType) { continue } // Ensure the connect IP matches one of the NIC's static IPs otherwise we could mess with other // instance's network traffic. If the wildcard address is supplied as the connect host then the // first bridged NIC which has a static IP address defined is selected as the connect host IP. if ipVersion == 4 && devConfig["ipv4.address"] != "" { if connectAddr.Address == devConfig["ipv4.address"] || connectAddr.Address == "0.0.0.0" { connectIP = net.ParseIP(devConfig["ipv4.address"]) } } else if ipVersion == 6 && devConfig["ipv6.address"] != "" { if connectAddr.Address == devConfig["ipv6.address"] || connectAddr.Address == "::" { connectIP = net.ParseIP(devConfig["ipv6.address"]) } } if connectIP != nil { // Get host_name of device so we can enable hairpin mode on bridge port. hostName = d.inst.ExpandedConfig()[fmt.Sprintf("volatile.%s.host_name", devName)] break // Found a match, stop searching. } } if connectIP == nil { if connectAddr.Address == "0.0.0.0" || connectAddr.Address == "::" { return fmt.Errorf("Instance has no static IPv%d address assigned to be used as the connect IP", ipVersion) } return fmt.Errorf("Connect IP %q must be one of the instance's static IPv%d addresses", connectAddr.Address, ipVersion) } // Override the host part of the connectAddr.Addr to the chosen connect IP. connectAddr.Address = connectIP.String() err = network.BridgeNetfilterEnabled(ipVersion) if err != nil { msg := fmt.Sprintf("IPv%d bridge netfilter not enabled. Instances using the bridge will not be able to connect to the proxy listen IP", ipVersion) d.logger.Warn(msg, logger.Ctx{"err": err}) err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpsertWarningLocalNode(ctx, d.inst.Project().Name, cluster.TypeInstance, d.inst.ID(), warningtype.ProxyBridgeNetfilterNotEnabled, fmt.Sprintf("%s: %v", msg, err)) }) if err != nil { logger.Warn("Failed to create warning", logger.Ctx{"err": err}) } } else { err = warnings.ResolveWarningsByLocalNodeAndProjectAndTypeAndEntity(d.state.DB.Cluster, d.inst.Project().Name, warningtype.ProxyBridgeNetfilterNotEnabled, cluster.TypeInstance, d.inst.ID()) if err != nil { logger.Warn("Failed to resolve warning", logger.Ctx{"err": err}) } if hostName == "" { return errors.New("Proxy cannot find bridge port host_name to enable hairpin mode") } // br_netfilter is enabled, so we need to enable hairpin mode on instance's bridge port otherwise // the instances on the bridge will not be able to connect to the proxy device's listen IP and the // NAT rule added by the firewall below to allow instance <-> instance traffic will also not work. link := &ip.Link{Name: hostName} err = link.BridgeLinkSetHairpin(true) if err != nil { return fmt.Errorf("Error enabling hairpin mode on bridge port %q: %w", hostName, err) } } // Convert proxy listen & connect addresses for firewall AddressForward. addressForward := firewallDrivers.AddressForward{ Protocol: listenAddr.ConnType, ListenAddress: net.ParseIP(listenAddr.Address), ListenPorts: listenAddr.Ports, TargetAddress: net.ParseIP(connectAddr.Address), TargetPorts: connectAddr.Ports, } err = d.state.Firewall.InstanceSetupProxyNAT(d.inst.Project().Name, d.inst.Name(), d.name, &addressForward) if err != nil { return err } return nil } func (d *proxy) setupProxyProcInfo() (*proxyProcInfo, error) { cname := project.Instance(d.inst.Project().Name, d.inst.Name()) cc, err := liblxc.NewContainer(cname, d.state.OS.LxcPath) if err != nil { return nil, err } defer func() { _ = cc.Release() }() containerPid := strconv.Itoa(cc.InitPid()) daemonPid := strconv.Itoa(os.Getpid()) cPidFd, err := cc.InitPidFd() if err != nil { return nil, err } dPidFd, err := linux.PidFdOpen(os.Getpid(), 0) if err != nil { _ = cPidFd.Close() return nil, err } inheritFd := []*os.File{cPidFd, dPidFd} containerPidFd := 3 daemonPidFd := 4 var listenPid, listenPidFd, connectPid, connectPidFd string connectAddr := d.config["connect"] listenAddr := d.config["listen"] switch d.config["bind"] { case "host", "": listenPid = daemonPid listenPidFd = fmt.Sprintf("%d", daemonPidFd) connectPid = containerPid connectPidFd = fmt.Sprintf("%d", containerPidFd) case "instance", "guest", "container": listenPid = containerPid listenPidFd = fmt.Sprintf("%d", containerPidFd) connectPid = daemonPid connectPidFd = fmt.Sprintf("%d", daemonPidFd) default: return nil, errors.New("Invalid binding side given. Must be \"host\" or \"instance\"") } listenAddrMode := "0644" if d.config["mode"] != "" { listenAddrMode = d.config["mode"] } p := &proxyProcInfo{ listenPid: listenPid, listenPidFd: listenPidFd, connectPid: connectPid, connectPidFd: connectPidFd, connectAddr: connectAddr, listenAddr: listenAddr, listenAddrGID: d.config["gid"], listenAddrUID: d.config["uid"], listenAddrMode: listenAddrMode, securityGID: d.config["security.gid"], securityUID: d.config["security.uid"], proxyProtocol: d.config["proxy_protocol"], inheritFds: inheritFd, } return p, nil } func (d *proxy) killProxyProc(pidPath string) error { // If the pid file doesn't exist, there is no process to kill. if !util.PathExists(pidPath) { return nil } p, err := subprocess.ImportProcess(pidPath) if err != nil { return fmt.Errorf("Could not read pid file: %s", err) } err = p.Stop() if err != nil && !errors.Is(err, subprocess.ErrNotRunning) { return fmt.Errorf("Unable to kill forkproxy: %s", err) } _ = os.Remove(pidPath) return nil } // Remove cleans up the device when it is removed from an instance. func (d *proxy) Remove(cleanupDependencies bool) error { err := warnings.DeleteWarningsByLocalNodeAndProjectAndTypeAndEntity(d.state.DB.Cluster, d.inst.Project().Name, warningtype.ProxyBridgeNetfilterNotEnabled, cluster.TypeInstance, d.inst.ID()) if err != nil { logger.Warn("Failed to delete warning", logger.Ctx{"err": err}) } // Delete apparmor profile. err = apparmor.ForkproxyDelete(d.state.OS, d.inst, d) if err != nil { return err } return nil } incus-7.0.0/internal/server/device/tpm.go000066400000000000000000000212031517523235500203550ustar00rootroot00000000000000package device import ( "context" "errors" "fmt" "os" "os/exec" "path/filepath" "strings" "time" "github.com/lxc/incus/v7/internal/linux" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) type tpm struct { deviceCommon } // CanMigrate returns whether the device can be migrated to any other cluster member. func (d *tpm) CanMigrate() bool { return true } // validateConfig checks the supplied config for correctness. func (d *tpm) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container, instancetype.VM) { return ErrUnsupportedDevType } rules := map[string]func(string) error{} if instConf.Type() == instancetype.Container { // gendoc:generate(entity=devices, group=tpm, key=path) // // --- // type: string // default: - // required: for containers // shortdesc: Only for containers: path inside the instance (for example, `/dev/tpm0`) rules["path"] = validate.IsNotEmpty // gendoc:generate(entity=devices, group=tpm, key=pathrm) // // --- // type: string // default: - // required: for containers // shortdesc: Only for containers: resource manager path inside the instance (for example, `/dev/tpmrm0`) rules["pathrm"] = validate.IsNotEmpty } else { rules["path"] = validate.Optional(validate.IsNotEmpty) rules["pathrm"] = validate.Optional(validate.IsNotEmpty) } err := d.config.Validate(rules) if err != nil { return fmt.Errorf("Failed to validate config: %w", err) } return nil } // validateEnvironment checks if the TPM emulator is available. func (d *tpm) validateEnvironment() error { // Validate the required binary. _, err := exec.LookPath("swtpm") if err != nil { return fmt.Errorf("Required tool '%s' is missing", "swtpm") } if d.inst.Type() == instancetype.Container { // Load module tpm_vtpm_proxy which creates the /dev/vtpmx device, required // by the TPM emulator. module := "tpm_vtpm_proxy" err := linux.LoadModule(module) if err != nil { return fmt.Errorf("Failed to load kernel module %q: %w", module, err) } } return nil } // Start is run when the device is added to the instance. func (d *tpm) Start() (*deviceConfig.RunConfig, error) { err := d.validateEnvironment() if err != nil { return nil, fmt.Errorf("Failed to validate environment: %w", err) } tpmDevPath := filepath.Join(d.inst.Path(), fmt.Sprintf("tpm.%s", d.name)) if !util.PathExists(tpmDevPath) { err := os.Mkdir(tpmDevPath, 0o700) if err != nil { return nil, fmt.Errorf("Failed to create device path %q: %w", tpmDevPath, err) } } if d.inst.Type() == instancetype.VM { return d.startVM() } return d.startContainer() } func (d *tpm) startContainer() (*deviceConfig.RunConfig, error) { tpmDevPath := filepath.Join(d.inst.Path(), fmt.Sprintf("tpm.%s", d.name)) logFileName := fmt.Sprintf("tpm.%s.log", d.name) logPath := filepath.Join(d.inst.LogPath(), logFileName) proc, err := subprocess.NewProcess("swtpm", []string{"chardev", "--tpm2", "--tpmstate", fmt.Sprintf("dir=%s", tpmDevPath), "--vtpm-proxy"}, logPath, "") if err != nil { return nil, fmt.Errorf("Failed to create new process: %w", err) } err = proc.Start(context.Background()) if err != nil { return nil, fmt.Errorf("Failed to start process %q: %w", "swtpm", err) } reverter := revert.New() defer reverter.Fail() // Stop the TPM emulator if anything goes wrong. reverter.Add(func() { _ = proc.Stop() }) pidPath := filepath.Join(d.inst.DevicesPath(), fmt.Sprintf("%s.pid", d.name)) err = proc.Save(pidPath) if err != nil { return nil, fmt.Errorf("Failed to save swtpm state for device %q: %w", d.name, err) } const TPM_MINOR = 244 const TPM_NUM_DEVICES = 65536 var major, minor, minorRM int // We need to capture the output of the TPM emulator since it contains the device path. To do // that, we wait until something has been written to the log file (stdout redirect), and then // read it. for range 20 { fi, err := os.Stat(logPath) if err != nil { return nil, fmt.Errorf("Failed to stat %q: %w", logPath, err) } if fi.Size() > 0 { break } time.Sleep(500 * time.Millisecond) } line, err := os.ReadFile(logPath) if err != nil { return nil, fmt.Errorf("Failed to read %q: %w", logPath, err) } // The output will be something like: // New TPM device: /dev/tpm1 (major/minor = 253/1) // We just need the major/minor numbers. fields := strings.Split(string(line), " ") if len(fields) < 7 { return nil, errors.New("Failed to get TPM device information") } _, err = fmt.Sscanf(fields[6], "%d/%d)", &major, &minor) if err != nil { return nil, fmt.Errorf("Failed to retrieve major/minor number: %w", err) } // Return error as we were unable to retrieve information regarding the TPM device. if major == 0 && minor == 0 { return nil, errors.New("Failed to get TPM device information") } if minor == TPM_MINOR { minorRM = TPM_NUM_DEVICES } else { minorRM = TPM_NUM_DEVICES + minor } runConf := deviceConfig.RunConfig{} err = unixDeviceSetupCharNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, uint32(major), uint32(minor), d.config["path"], false, &runConf) if err != nil { return nil, fmt.Errorf("Failed to setup unix device: %w", err) } err = unixDeviceSetupCharNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, uint32(major), uint32(minorRM), d.config["pathrm"], false, &runConf) if err != nil { return nil, fmt.Errorf("Failed to setup unix device: %w", err) } reverter.Success() return &runConf, nil } func (d *tpm) startVM() (*deviceConfig.RunConfig, error) { if d.inst.Type() == instancetype.VM && util.IsTrue(d.inst.ExpandedConfig()["migration.stateful"]) { return nil, errors.New("TPM devices cannot be used when migration.stateful is enabled") } tpmDevPath := filepath.Join(d.inst.Path(), fmt.Sprintf("tpm.%s", d.name)) socketPath := filepath.Join(tpmDevPath, fmt.Sprintf("swtpm-%s.sock", d.name)) runConf := deviceConfig.RunConfig{ TPMDevice: []deviceConfig.RunConfigItem{ {Key: "devName", Value: d.name}, {Key: "path", Value: socketPath}, }, } // Delete any leftover socket. _ = os.Remove(socketPath) proc, err := subprocess.NewProcess("swtpm", []string{"socket", "--tpm2", "--tpmstate", fmt.Sprintf("dir=%s", tpmDevPath), "--ctrl", fmt.Sprintf("type=unixio,path=swtpm-%s.sock", d.name)}, "", "") if err != nil { return nil, err } proc.Cwd = tpmDevPath // Start the TPM emulator. err = proc.Start(context.Background()) if err != nil { return nil, fmt.Errorf("Failed to start swtpm for device %q: %w", d.name, err) } reverter := revert.New() defer reverter.Fail() reverter.Add(func() { _ = proc.Stop() }) pidPath := filepath.Join(d.inst.DevicesPath(), fmt.Sprintf("%s.pid", d.name)) err = proc.Save(pidPath) if err != nil { return nil, fmt.Errorf("Failed to save swtpm state for device %q: %w", d.name, err) } // Wait for the socket to be available. exists := false for range 20 { if util.PathExists(socketPath) { exists = true break } time.Sleep(100 * time.Millisecond) } if !exists { return nil, errors.New("swtpm socket didn't appear within 2s") } reverter.Success() return &runConf, nil } // Stop terminates the TPM emulator. func (d *tpm) Stop() (*deviceConfig.RunConfig, error) { pidPath := filepath.Join(d.inst.DevicesPath(), fmt.Sprintf("%s.pid", d.name)) runConf := deviceConfig.RunConfig{} defer func() { _ = os.Remove(pidPath) }() if util.PathExists(pidPath) { proc, err := subprocess.ImportProcess(pidPath) if err != nil { return nil, fmt.Errorf("Failed to import process %q: %w", pidPath, err) } // The TPM emulator will usually exit automatically when the tpm device is no longer in use, // i.e. the instance is stopped. Therefore, we only fail if the running process couldn't // be stopped. err = proc.Stop() if err != nil && !errors.Is(err, subprocess.ErrNotRunning) { return nil, fmt.Errorf("Failed to stop imported process %q: %w", pidPath, err) } } if d.inst.Type() == instancetype.Container { err := unixDeviceRemove(d.inst.DevicesPath(), "unix", d.name, "", &runConf) if err != nil { return nil, fmt.Errorf("Failed to remove unix device: %w", err) } } return &runConf, nil } // Remove removes the TPM state file. func (d *tpm) Remove(cleanupDependencies bool) error { tpmDevPath := filepath.Join(d.inst.Path(), fmt.Sprintf("tpm.%s", d.name)) return os.RemoveAll(tpmDevPath) } incus-7.0.0/internal/server/device/unix_common.go000066400000000000000000000204501517523235500221130ustar00rootroot00000000000000package device import ( "errors" "fmt" "io/fs" "path/filepath" "strings" "github.com/lxc/incus/v7/internal/linux" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/fsmonitor/drivers" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // unixIsOurDeviceType checks that device file type matches what we are expecting in the config. func unixIsOurDeviceType(config deviceConfig.Device, dType string) bool { if config["type"] == "unix-char" && dType == "c" { return true } if config["type"] == "unix-block" && dType == "b" { return true } return false } type unixCommon struct { deviceCommon } // isRequired indicates whether the device config requires this device to start OK. func (d *unixCommon) isRequired() bool { // Defaults to required. return util.IsTrueOrEmpty(d.config["required"]) } // validateConfig checks the supplied config for correctness. func (d *unixCommon) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container) { return ErrUnsupportedDevType } rules := map[string]func(string) error{ // gendoc:generate(entity=devices, group=unix-char-block, key=source) // // --- // type: string // shortdesc: Path on the host (one of `source` and `path` must be set) "source": func(value string) error { if value == "" { return nil } if strings.HasPrefix(value, d.state.DevMonitor.PrefixPath()) { return nil } return &drivers.ErrInvalidPath{PrefixPath: d.state.DevMonitor.PrefixPath()} }, // gendoc:generate(entity=devices, group=unix-char-block, key=gid) // // --- // type: int // default: 0 // shortdesc: GID of the device owner in the instance "gid": unixValidUserID, // gendoc:generate(entity=devices, group=unix-char-block, key=major) // // --- // type: int // default: device on host // shortdesc: Device major number "major": unixValidDeviceNum, // gendoc:generate(entity=devices, group=unix-char-block, key=minor) // // --- // type: int // default: device on host // shortdesc: Device minor number "minor": unixValidDeviceNum, // gendoc:generate(entity=devices, group=unix-char-block, key=mode) // // --- // type: int // default: 0660 // shortdesc: Mode of the device in the instance "mode": unixValidOctalFileMode, // gendoc:generate(entity=devices, group=unix-char-block, key=path) // // --- // type: string // shortdesc: Path inside the instance (one of `source` and `path` must be set) "path": validate.IsAny, // gendoc:generate(entity=devices, group=unix-char-block, key=required) // // --- // type: bool // default: true // shortdesc: Whether this device is required to start the instance "required": validate.Optional(validate.IsBool), // gendoc:generate(entity=devices, group=unix-char-block, key=uid) // // --- // type: int // default: 0 // shortdesc: UID of the device owner in the instance "uid": unixValidUserID, } err := d.config.Validate(rules) if err != nil { return err } if d.config["source"] == "" && d.config["path"] == "" { return errors.New("Unix device entry is missing the required \"source\" or \"path\" property") } return nil } // Register is run after the device is started or on daemon startup. func (d *unixCommon) Register() error { // Don't register for hot plug events if the device is required. if d.isRequired() { return nil } // Extract variables needed to run the event hook so that the reference to this device // struct is not needed to be kept in memory. devicesPath := d.inst.DevicesPath() devConfig := d.config deviceName := d.name state := d.state // Handler for when a Unix event occurs. f := func(e UnixEvent) (*deviceConfig.RunConfig, error) { // Check if the event is for a device file that this device wants. if unixDeviceSourcePath(devConfig) != e.Path { return nil, nil } // Derive the host side path for the instance device file. ourPrefix := deviceJoinPath("unix", deviceName) relativeDestPath := strings.TrimPrefix(unixDeviceDestPath(devConfig), "/") devName := linux.PathNameEncode(deviceJoinPath(ourPrefix, relativeDestPath)) devPath := filepath.Join(devicesPath, devName) runConf := deviceConfig.RunConfig{} if e.Action == "add" { // Skip if host side instance device file already exists. if util.PathExists(devPath) { return nil, nil } // Get the file type and ensure it matches what the user was expecting. dType, _, _, err := unixDeviceAttributes(e.Path) if err != nil { if errors.Is(err, fs.ErrNotExist) { // Skip if host side source device doesn't exist. // This could be an event for the parent directory being added. return nil, nil } return nil, fmt.Errorf("Failed getting device attributes: %w", err) } if !unixIsOurDeviceType(d.config, dType) { return nil, fmt.Errorf("Path specified is not a %s device", d.config["type"]) } err = unixDeviceSetup(state, devicesPath, "unix", deviceName, devConfig, true, &runConf) if err != nil { return nil, err } } else if e.Action == "remove" { // Skip if host side instance device file doesn't exist. if !util.PathExists(devPath) { return nil, nil } err := unixDeviceRemove(devicesPath, "unix", deviceName, relativeDestPath, &runConf) if err != nil { return nil, err } // Add a post hook function to remove the specific USB device file after unmount. runConf.PostHooks = []func() error{func() error { err := unixDeviceDeleteFiles(state, devicesPath, "unix", deviceName, relativeDestPath) if err != nil { return fmt.Errorf("Failed to delete files for device '%s': %w", deviceName, err) } return nil }} } return &runConf, nil } // Register the handler function against the device's source path. subPath := unixDeviceSourcePath(devConfig) err := unixRegisterHandler(d.state, d.inst, d.name, subPath, f) if err != nil { return err } return nil } // Start is run when the device is added to the container. func (d *unixCommon) Start() (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{} runConf.PostHooks = []func() error{d.Register} srcPath := unixDeviceSourcePath(d.config) // If device file already exists on system, proceed to add it whether its required or not. dType, _, _, err := unixDeviceAttributes(srcPath) if err == nil { // Ensure device type matches what the device config is expecting. if !unixIsOurDeviceType(d.config, dType) { return nil, fmt.Errorf("Path specified is not a %s device", d.config["type"]) } err = unixDeviceSetup(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, true, &runConf) if err != nil { return nil, err } } else { // If the device file doesn't exist on the system, but major & minor numbers have // been provided in the config then we can go ahead and create the device anyway. if d.config["major"] != "" && d.config["minor"] != "" { err := unixDeviceSetup(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, true, &runConf) if err != nil { return nil, err } } else if d.isRequired() { // If the file is missing and the device is required then we cannot proceed. return nil, errors.New("The required device path doesn't exist and the major and minor settings are not specified") } } return &runConf, nil } // Stop is run when the device is removed from the instance. func (d *unixCommon) Stop() (*deviceConfig.RunConfig, error) { // Unregister any Unix event handlers for this device. err := unixUnregisterHandler(d.state, d.inst, d.name) if err != nil { return nil, err } runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } err = unixDeviceRemove(d.inst.DevicesPath(), "unix", d.name, "", &runConf) if err != nil { return nil, err } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *unixCommon) postStop() error { // Remove host files for this device. err := unixDeviceDeleteFiles(d.state, d.inst.DevicesPath(), "unix", d.name, "") if err != nil { return fmt.Errorf("Failed to delete files for device '%s': %w", d.name, err) } return nil } incus-7.0.0/internal/server/device/unix_hotplug.go000066400000000000000000000205321517523235500223060ustar00rootroot00000000000000//go:build linux && cgo package device import ( "errors" "fmt" "strings" "github.com/jochenvg/go-udev" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // unixHotplugIsOurDevice indicates whether the unixHotplug device event qualifies as part of our device. // This function is not defined against the unixHotplug struct type so that it can be used in event // callbacks without needing to keep a reference to the unixHotplug device struct. func unixHotplugIsOurDevice(config deviceConfig.Device, unixHotplug *UnixHotplugEvent) bool { // Check if event matches criteria for this device, if not return. if config["vendorid"] != "" && config["vendorid"] != unixHotplug.Vendor { return false } if config["productid"] != "" && config["productid"] != unixHotplug.Product { return false } if config["pci"] != "" && config["pci"] != unixHotplug.PCI { return false } return true } type unixHotplug struct { deviceCommon } // isRequired indicates whether the device config requires this device to start OK. func (d *unixHotplug) isRequired() bool { // Defaults to not required. return util.IsTrue(d.config["required"]) } // validateConfig checks the supplied config for correctness. func (d *unixHotplug) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container) { return ErrUnsupportedDevType } rules := map[string]func(string) error{ // gendoc:generate(entity=devices, group=unix-hotplug, key=vendorid) // // --- // type: string // shortdesc: The vendor ID of the USB device "vendorid": validate.Optional(validate.IsDeviceID), // gendoc:generate(entity=devices, group=unix-hotplug, key=productid) // // --- // type: string // shortdesc: The product ID of the USB device "productid": validate.Optional(validate.IsDeviceID), // gendoc:generate(entity=devices, group=unix-hotplug, key=pci) // // --- // type: string // shortdesc: The PCI address of a USB controller to monitor "pci": validate.Optional(validate.IsPCIAddress), // gendoc:generate(entity=devices, group=unix-hotplug, key=uid) // // --- // type: int // default: 0 // shortdesc: UID of the device owner in the instance "uid": unixValidUserID, // gendoc:generate(entity=devices, group=unix-hotplug, key=gid) // // --- // type: int // default: 0 // shortdesc: GID of the device owner in the instance "gid": unixValidUserID, // gendoc:generate(entity=devices, group=unix-hotplug, key=mode) // // --- // type: int // default: 0660 // shortdesc: Mode of the device in the instance "mode": unixValidOctalFileMode, // gendoc:generate(entity=devices, group=unix-hotplug, key=required) // // --- // type: bool // default: true // shortdesc: Whether this device is required to start the instance "required": validate.Optional(validate.IsBool), } err := d.config.Validate(rules) if err != nil { return err } if d.config["vendorid"] == "" && d.config["productid"] == "" && d.config["pci"] == "" { return errors.New("Unix hotplug devices require a vendorid, productid or PCI address") } return nil } // Register is run after the device is started or on daemon startup. func (d *unixHotplug) Register() error { // Extract variables needed to run the event hook so that the reference to this device // struct is not needed to be kept in memory. devicesPath := d.inst.DevicesPath() devConfig := d.config deviceName := d.name state := d.state // Handler for when a UnixHotplug event occurs. f := func(e UnixHotplugEvent) (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{} if e.Action == "add" { if !unixHotplugIsOurDevice(devConfig, &e) { return nil, nil } if e.Subsystem == "block" { err := unixDeviceSetupBlockNum(state, devicesPath, "unix", deviceName, devConfig, e.Major, e.Minor, e.Path, false, &runConf) if err != nil { return nil, err } } else { err := unixDeviceSetupCharNum(state, devicesPath, "unix", deviceName, devConfig, e.Major, e.Minor, e.Path, false, &runConf) if err != nil { return nil, err } } } else if e.Action == "remove" { relativeTargetPath := strings.TrimPrefix(e.Path, "/") err := unixDeviceRemove(devicesPath, "unix", deviceName, relativeTargetPath, &runConf) if err != nil { return nil, err } // Return early to prevent injecting duplicate uevents. if len(runConf.Mounts) == 0 { return nil, nil } // Add a post hook function to remove the specific unix hotplug device file after unmount. runConf.PostHooks = []func() error{func() error { err := unixDeviceDeleteFiles(state, devicesPath, "unix", deviceName, relativeTargetPath) if err != nil { return fmt.Errorf("Failed to delete files for device '%s': %w", deviceName, err) } return nil }} } runConf.Uevents = append(runConf.Uevents, e.UeventParts) return &runConf, nil } unixHotplugRegisterHandler(d.inst, d.name, f) return nil } // Start is run when the device is added to the instance. func (d *unixHotplug) Start() (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{} runConf.PostHooks = []func() error{d.Register} device := d.loadUnixDevice() if d.isRequired() && device == nil { return nil, errors.New("Required Unix Hotplug device not found") } if device == nil { return &runConf, nil } devnum := device.Devnum() major := uint32(devnum.Major()) minor := uint32(devnum.Minor()) // setup device var err error if device.Subsystem() == "block" { err = unixDeviceSetupBlockNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, major, minor, device.Devnode(), false, &runConf) } else { err = unixDeviceSetupCharNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, major, minor, device.Devnode(), false, &runConf) } if err != nil { return nil, err } return &runConf, nil } // Stop is run when the device is removed from the instance. func (d *unixHotplug) Stop() (*deviceConfig.RunConfig, error) { unixHotplugUnregisterHandler(d.inst, d.name) runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } err := unixDeviceRemove(d.inst.DevicesPath(), "unix", d.name, "", &runConf) if err != nil { return nil, err } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *unixHotplug) postStop() error { err := unixDeviceDeleteFiles(d.state, d.inst.DevicesPath(), "unix", d.name, "") if err != nil { return fmt.Errorf("Failed to delete files for device '%s': %w", d.name, err) } return nil } // loadUnixDevice scans the host machine for unix devices with matching product/vendor ids // and returns the first matching device with the subsystem type char or block. func (d *unixHotplug) loadUnixDevice() *udev.Device { // Find device if exists u := udev.Udev{} e := u.NewEnumerate() if d.config["vendorid"] != "" { err := e.AddMatchProperty("ID_VENDOR_ID", d.config["vendorid"]) if err != nil { logger.Warn("Failed to add property to device", logger.Ctx{"property_name": "ID_VENDOR_ID", "property_value": d.config["vendorid"], "err": err}) } } if d.config["productid"] != "" { err := e.AddMatchProperty("ID_MODEL_ID", d.config["productid"]) if err != nil { logger.Warn("Failed to add property to device", logger.Ctx{"property_name": "ID_MODEL_ID", "property_value": d.config["productid"], "err": err}) } } if d.config["pci"] != "" { err := e.AddMatchProperty("ID_PCI_ID", d.config["pci"]) if err != nil { logger.Warn("Failed to add property to device", logger.Ctx{"property_name": "ID_PCI_ID", "property_value": d.config["pci"], "err": err}) } } err := e.AddMatchIsInitialized() if err != nil { logger.Warn("Failed to add initialized property to device", logger.Ctx{"err": err}) } devices, _ := e.Devices() var device *udev.Device for i := range devices { device = devices[i] if device == nil { continue } devnum := device.Devnum() if devnum.Major() == 0 || devnum.Minor() == 0 { continue } if device.Devnode() == "" { continue } if !strings.HasPrefix(device.Subsystem(), "usb") { return device } } return nil } incus-7.0.0/internal/server/device/usb.go000066400000000000000000000256671517523235500203700ustar00rootroot00000000000000package device import ( "errors" "fmt" "io/fs" "os" "path" "strings" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // usbDevPath is the path where USB devices can be enumerated. const usbDevPath = "/sys/bus/usb/devices" // usbIsOurDevice indicates whether the USB device event qualifies as part of our device. // This function is not defined against the usb struct type so that it can be used in event // callbacks without needing to keep a reference to the usb device struct. func usbIsOurDevice(config deviceConfig.Device, usb *USBEvent) bool { // Check if event matches criteria for this device, if not return. if (config["vendorid"] != "" && config["vendorid"] != usb.Vendor) || (config["productid"] != "" && config["productid"] != usb.Product) || (config["serial"] != "" && config["serial"] != usb.Serial) || (config["busnum"] != "" && config["busnum"] != fmt.Sprintf("%d", usb.BusNum)) || (config["devnum"] != "" && config["devnum"] != fmt.Sprintf("%d", usb.DevNum)) { return false } return true } type usb struct { deviceCommon } // isRequired indicates whether the device config requires this device to start OK. func (d *usb) isRequired() bool { // Defaults to not required. return util.IsTrue(d.config["required"]) } // validateConfig checks the supplied config for correctness. func (d *usb) validateConfig(instConf instance.ConfigReader, partialValidation bool) error { if !instanceSupported(instConf.Type(), instancetype.Container, instancetype.VM) { return ErrUnsupportedDevType } if instConf.Architecture() == osarch.ARCH_64BIT_S390_BIG_ENDIAN { return errors.New("USB devices aren't supported on s390x") } rules := map[string]func(string) error{ // gendoc:generate(entity=devices, group=usb, key=vendorid) // // --- // type: string // shortdesc: The vendor ID of the USB device "vendorid": validate.Optional(validate.IsDeviceID), // gendoc:generate(entity=devices, group=usb, key=productid) // // --- // type: string // shortdesc: The product ID of the USB device "productid": validate.Optional(validate.IsDeviceID), // gendoc:generate(entity=devices, group=usb, key=serial) // // --- // type: string // shortdesc: The serial number of the USB device "serial": validate.Optional(validate.IsAny), // gendoc:generate(entity=devices, group=usb, key=uid) // // --- // type: int // defaultdesc: `0` // shortdesc: Only for containers: UID of the device owner in the instance "uid": unixValidUserID, // gendoc:generate(entity=devices, group=usb, key=gid) // // --- // type: int // defaultdesc: `0` // shortdesc: Only for containers: GID of the device owner in the instance "gid": unixValidUserID, // gendoc:generate(entity=devices, group=usb, key=mode) // // --- // type: int // defaultdesc: `0660` // shortdesc: Only for containers: Mode of the device in the instance "mode": unixValidOctalFileMode, // gendoc:generate(entity=devices, group=usb, key=required) // // --- // type: bool // defaultdesc: `false` // shortdesc: Whether this device is required to start the instance (the default is `false`, and all devices can be hotplugged) "required": validate.Optional(validate.IsBool), // gendoc:generate(entity=devices, group=usb, key=busnum) // // --- // type: int // shortdesc: The bus number of which the USB device is attached "busnum": validate.Optional(validate.IsUint32), // gendoc:generate(entity=devices, group=usb, key=devnum) // // --- // type: int // shortdesc: The device number of the USB device "devnum": validate.Optional(validate.IsUint32), // gendoc:generate(entity=devices, group=usb, key=attached) // // --- // type: bool // default: `true` // required: no // shortdesc: Whether the USB device is plugged in or not "attached": validate.Optional(validate.IsBool), } err := d.config.Validate(rules) if err != nil { return err } return nil } // Register is run after the device is started or on daemon startup. func (d *usb) Register() error { // Extract variables needed to run the event hook so that the reference to this device // struct is not needed to be kept in memory. devicesPath := d.inst.DevicesPath() devConfig := d.config deviceName := d.name state := d.state // Handler for when a USB event occurs. f := func(e USBEvent) (*deviceConfig.RunConfig, error) { if !usbIsOurDevice(devConfig, &e) || !util.IsTrueOrEmpty(devConfig["attached"]) { return nil, nil } runConf := deviceConfig.RunConfig{} if e.Action == "add" { err := unixDeviceSetupCharNum(state, devicesPath, "unix", deviceName, devConfig, e.Major, e.Minor, e.Path, false, &runConf) if err != nil { return nil, err } } else if e.Action == "remove" { relativeTargetPath := strings.TrimPrefix(e.Path, "/") err := unixDeviceRemove(devicesPath, "unix", deviceName, relativeTargetPath, &runConf) if err != nil { return nil, err } // Add a post hook function to remove the specific USB device file after unmount. runConf.PostHooks = []func() error{func() error { err := unixDeviceDeleteFiles(state, devicesPath, "unix", deviceName, relativeTargetPath) if err != nil { return fmt.Errorf("Failed to delete files for device '%s': %w", deviceName, err) } return nil }} } runConf.Uevents = append(runConf.Uevents, e.UeventParts) // Add the USB device to runConf so that the device handler can handle physical hotplugging. runConf.USBDevice = append(runConf.USBDevice, deviceConfig.USBDeviceItem{ DeviceName: d.getUniqueDeviceNameFromUSBEvent(e), HostDevicePath: e.Path, }) return &runConf, nil } usbRegisterHandler(d.inst, d.name, f) return nil } // Start is run when the device is added to the instance. func (d *usb) Start() (*deviceConfig.RunConfig, error) { attached := util.IsTrueOrEmpty(d.config["attached"]) if d.inst.Type() == instancetype.VM { return d.startVM(attached) } return d.startContainer(attached) } func (d *usb) startContainer(attached bool) (*deviceConfig.RunConfig, error) { usbs, err := d.loadUsb() if err != nil { return nil, err } runConf := deviceConfig.RunConfig{} runConf.PostHooks = []func() error{d.Register} found := 0 for _, usb := range usbs { if usbIsOurDevice(d.config, &usb) { found++ if attached { err := unixDeviceSetupCharNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, usb.Major, usb.Minor, usb.Path, false, &runConf) if err != nil { return nil, err } } } } if d.isRequired() && found <= 0 { return nil, errors.New("Required USB device not found") } return &runConf, nil } func (d *usb) startVM(attached bool) (*deviceConfig.RunConfig, error) { if d.inst.Type() == instancetype.VM && util.IsTrue(d.inst.ExpandedConfig()["migration.stateful"]) { return nil, errors.New("USB devices cannot be used when migration.stateful is enabled") } usbs, err := d.loadUsb() if err != nil { return nil, err } runConf := deviceConfig.RunConfig{} runConf.PostHooks = []func() error{d.Register} found := 0 for _, usb := range usbs { if usbIsOurDevice(d.config, &usb) { found++ if attached { runConf.USBDevice = append(runConf.USBDevice, deviceConfig.USBDeviceItem{ DeviceName: d.getUniqueDeviceNameFromUSBEvent(usb), HostDevicePath: fmt.Sprintf("/dev/bus/usb/%03d/%03d", usb.BusNum, usb.DevNum), }) } } } if d.isRequired() && found <= 0 { return nil, errors.New("Required USB device not found") } return &runConf, nil } // Stop is run when the device is removed from the instance. func (d *usb) Stop() (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{ PostHooks: []func() error{d.postStop}, } usbs, err := d.loadUsb() if err != nil { return nil, err } for _, usb := range usbs { if usbIsOurDevice(d.config, &usb) { runConf.USBDevice = append(runConf.USBDevice, deviceConfig.USBDeviceItem{ DeviceName: d.getUniqueDeviceNameFromUSBEvent(usb), HostDevicePath: fmt.Sprintf("/dev/bus/usb/%03d/%03d", usb.BusNum, usb.DevNum), }) } } if d.inst.Type() == instancetype.Container { // Unregister any USB event handlers for this device. usbUnregisterHandler(d.inst, d.name) err := unixDeviceRemove(d.inst.DevicesPath(), "unix", d.name, "", &runConf) if err != nil { return nil, err } } return &runConf, nil } // postStop is run after the device is removed from the instance. func (d *usb) postStop() error { // Remove host files for this device. err := unixDeviceDeleteFiles(d.state, d.inst.DevicesPath(), "unix", d.name, "") if err != nil { return fmt.Errorf("Failed to delete files for device '%s': %w", d.name, err) } return nil } // loadUsb scans the host machine for USB devices. func (d *usb) loadUsb() ([]USBEvent, error) { result := []USBEvent{} ents, err := os.ReadDir(usbDevPath) if err != nil { /* if there are no USB devices, let's render an empty list, * i.e. no usb devices */ if errors.Is(err, fs.ErrNotExist) { return result, nil } return nil, err } for _, ent := range ents { values, err := d.loadRawValues(path.Join(usbDevPath, ent.Name())) if err != nil { if errors.Is(err, fs.ErrNotExist) { continue } return []USBEvent{}, err } parts := strings.Split(values["dev"], ":") if len(parts) != 2 { return []USBEvent{}, fmt.Errorf("invalid device value %s", values["dev"]) } usb, err := USBNewEvent( "add", values["idVendor"], values["idProduct"], values["serial"], parts[0], parts[1], values["busnum"], values["devnum"], values["devname"], []string{}, 0, ) if err != nil { if errors.Is(err, fs.ErrNotExist) { continue } return nil, err } result = append(result, usb) } return result, nil } func (d *usb) loadRawValues(p string) (map[string]string, error) { values := map[string]string{ "idVendor": "", "idProduct": "", "serial": "", "dev": "", "busnum": "", "devnum": "", } for k := range values { v, err := os.ReadFile(path.Join(p, k)) if err != nil { if k == "serial" && errors.Is(err, fs.ErrNotExist) { continue } return nil, err } values[k] = strings.TrimSpace(string(v)) } return values, nil } // getUniqueDeviceNameFromUSBEvent returns a unique device name including the bus and device number. // Previously, the device name contained a simple incremental value as suffix. This would make the // device unidentifiable when using hotplugging. Including the bus and device number makes the // device identifiable. func (d *usb) getUniqueDeviceNameFromUSBEvent(e USBEvent) string { return fmt.Sprintf("%s-%03d-%03d", d.name, e.BusNum, e.DevNum) } // CanHotPlug returns whether the device can be managed whilst the instance is running. func (d *usb) CanHotPlug() bool { return true } incus-7.0.0/internal/server/dns/000077500000000000000000000000001517523235500165555ustar00rootroot00000000000000incus-7.0.0/internal/server/dns/debug.go000066400000000000000000000003631517523235500201740ustar00rootroot00000000000000package dns // Debug returns a dump of the current configuration. func (s *Server) Debug(zone string) string { // Locking. s.mu.Lock() defer s.mu.Unlock() return s.debug(zone) } func (s *Server) debug(zone string) string { return "" } incus-7.0.0/internal/server/dns/handler.go000066400000000000000000000101721517523235500205220ustar00rootroot00000000000000package dns import ( "fmt" "net" "strings" "time" "github.com/miekg/dns" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) type dnsHandler struct { server *Server } func (d dnsHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { // Don't allow concurrent queries. d.server.mu.Lock() defer d.server.mu.Unlock() // Check if we're ready to serve queries. if d.server.zoneRetriever == nil { m := &dns.Msg{} m.SetRcode(r, dns.RcodeServerFailure) err := w.WriteMsg(m) if err != nil { logger.Error("Unable to write message", logger.Ctx{"err": err}) } return } // Only allow a single request. if len(r.Question) != 1 { m := &dns.Msg{} m.SetRcode(r, dns.RcodeServerFailure) err := w.WriteMsg(m) if err != nil { logger.Error("Unable to write message", logger.Ctx{"err": err}) } return } // Check that it's a supported request type. if r.Question[0].Qtype != dns.TypeAXFR && r.Question[0].Qtype != dns.TypeIXFR && r.Question[0].Qtype != dns.TypeSOA { m := &dns.Msg{} m.SetRcode(r, dns.RcodeNotImplemented) err := w.WriteMsg(m) if err != nil { logger.Error("Unable to write message", logger.Ctx{"err": err}) } return } // Extract the request information. name := strings.TrimSuffix(r.Question[0].Name, ".") ip, _, err := net.SplitHostPort(w.RemoteAddr().String()) if err != nil { m := &dns.Msg{} m.SetRcode(r, dns.RcodeServerFailure) err := w.WriteMsg(m) if err != nil { logger.Error("Unable to write message", logger.Ctx{"err": err}) } return } // Prepare the response. m := &dns.Msg{} m.SetReply(r) m.Authoritative = true // Load the zone. zone, err := d.server.zoneRetriever(name, r.Question[0].Qtype != dns.TypeSOA) if err != nil { // On failure, return NXDOMAIN. m := &dns.Msg{} m.SetRcode(r, dns.RcodeNameError) err := w.WriteMsg(m) if err != nil { logger.Error("Unable to write message", logger.Ctx{"err": err}) } return } // Check access. if !isAllowed(zone.Info, ip, r.IsTsig(), w.TsigStatus() == nil) { // On auth failure, return NXDOMAIN to avoid information leaks. m := &dns.Msg{} m.SetRcode(r, dns.RcodeNameError) err := w.WriteMsg(m) if err != nil { logger.Error("Unable to write message", logger.Ctx{"err": err}) } return } zoneRR := dns.NewZoneParser(strings.NewReader(zone.Content), "", "") for { rr, ok := zoneRR.Next() if !ok { err := zoneRR.Err() if err != nil { logger.Errorf("Bad DNS record in zone %q: %v", name, err) m := &dns.Msg{} m.SetRcode(r, dns.RcodeFormatError) err := w.WriteMsg(m) if err != nil { logger.Error("Unable to write message", logger.Ctx{"err": err}) } return } break } m.Answer = append(m.Answer, rr) } tsig := r.IsTsig() if tsig != nil && w.TsigStatus() == nil { m.SetTsig(tsig.Hdr.Name, tsig.Algorithm, 300, time.Now().Unix()) } err = w.WriteMsg(m) if err != nil { logger.Error("Unable to write message", logger.Ctx{"err": err}) } } func isAllowed(zone api.NetworkZone, ip string, tsig *dns.TSIG, tsigStatus bool) bool { type peer struct { address string key string } // Build a list of peers. peers := map[string]*peer{} for k, v := range zone.Config { if !strings.HasPrefix(k, "peers.") { continue } // Extract the fields. fields := strings.SplitN(k, ".", 3) if len(fields) != 3 { continue } peerName := fields[1] if peers[peerName] == nil { peers[peerName] = &peer{} } // Add the correct validation rule for the dynamic field based on last part of key. switch fields[2] { case "address": peers[peerName].address = v case "key": peers[peerName].key = v } } // Validate access. for peerName, peer := range peers { peerKeyName := fmt.Sprintf("%s_%s.", zone.Name, peerName) if peer.address != "" && ip != peer.address { // Bad IP address. continue } if peer.key != "" && (tsig == nil || !tsigStatus) { // Missing or invalid TSIG. continue } if peer.key != "" && tsig.Hdr.Name != peerKeyName { // Bad key name (valid TSIG but potentially for another domain). continue } // We have a trusted peer. return true } return false } incus-7.0.0/internal/server/dns/server.go000066400000000000000000000136531517523235500204220ustar00rootroot00000000000000package dns import ( "context" "fmt" "strings" "sync" "time" "github.com/miekg/dns" "github.com/lxc/incus/v7/internal/ports" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" ) // ZoneRetriever is a function which fetches a DNS zone. type ZoneRetriever func(name string, full bool) (*Zone, error) // Server represents a DNS server instance. type Server struct { tcpDNS *dns.Server udpDNS *dns.Server // External dependencies. db *db.Cluster zoneRetriever ZoneRetriever // Internal state (to handle reconfiguration). address string cmd chan serverCmdInfo mu sync.Mutex } type serverCmd int const ( serverCmdStart serverCmd = iota serverCmdRestart serverCmdStop serverCmdReconfigure serverCmdHandleError ) type serverCmdInfo struct { cmd serverCmd address string err error } // NewServer returns a new server instance. func NewServer(db *db.Cluster, retriever ZoneRetriever) *Server { // Setup new struct. s := &Server{db: db, zoneRetriever: retriever} return s } func (s *Server) handleErr(err error) { s.cmd <- serverCmdInfo{ cmd: serverCmdHandleError, err: err, } } func (s *Server) runDNSServer() { shouldRun := false address := "" for cmd := range s.cmd { switch cmd.cmd { case serverCmdStart: if shouldRun { continue } shouldRun = true address = cmd.address s.mu.Lock() err := s.start(cmd.address) if err != nil { // Run in new goroutine to avoid deadlock. go s.handleErr(err) } s.mu.Unlock() case serverCmdRestart: s.mu.Lock() // don't start if the server shouldn't run or is already running (s.address is set when the server starts) if !shouldRun || s.address != "" { s.mu.Unlock() continue } err := s.start(address) if err != nil { // Run in new goroutine to avoid deadlock. go s.handleErr(err) } s.mu.Unlock() case serverCmdStop: shouldRun = false s.mu.Lock() s.stop() s.mu.Unlock() case serverCmdReconfigure: s.mu.Lock() s.stop() if cmd.address == "" { shouldRun = false } else { shouldRun = true address = cmd.address err := s.start(cmd.address) if err != nil { // Run in new goroutine to avoid deadlock. go s.handleErr(err) } } s.mu.Unlock() case serverCmdHandleError: if cmd.err == nil { continue } logger.Errorf("DNS server encountered an error, restarting in 10s: %v", cmd.err) s.mu.Lock() s.stop() s.mu.Unlock() go func() { <-time.NewTimer(time.Second * 10).C s.cmd <- serverCmdInfo{cmd: serverCmdRestart} }() } } } // Start sets up the DNS listener. func (s *Server) Start(address string) error { s.mu.Lock() start := s.cmd == nil if start { s.cmd = make(chan serverCmdInfo) go s.runDNSServer() } s.mu.Unlock() if start { s.cmd <- serverCmdInfo{ cmd: serverCmdStart, address: address, } } else { s.cmd <- serverCmdInfo{ cmd: serverCmdReconfigure, address: address, } } return nil } func (s *Server) start(address string) error { // Set default port if needed. address = internalUtil.CanonicalNetworkAddress(address, ports.DNSDefaultPort) // Setup the handler. handler := dnsHandler{} handler.server = s // Spawn the DNS server. s.tcpDNS = &dns.Server{Addr: address, Net: "tcp", Handler: handler} go func() { err := s.tcpDNS.ListenAndServe() if err != nil { s.handleErr(fmt.Errorf("Failed to listen on TCP DNS address %q: %v", address, err)) } }() s.udpDNS = &dns.Server{Addr: address, Net: "udp", Handler: handler} go func() { err := s.udpDNS.ListenAndServe() if err != nil { s.handleErr(fmt.Errorf("Failed to listen on UDP DNS address %q: %v", address, err)) } }() // TSIG handling. err := s.updateTSIG() if err != nil { return err } // Record the address. s.address = address return nil } // Stop tears down the DNS listener. func (s *Server) Stop() error { s.cmd <- serverCmdInfo{ cmd: serverCmdStop, } return nil } func (s *Server) stop() { // Skip if no instance. if s.tcpDNS == nil || s.udpDNS == nil { return } // Stop the listener. _ = s.tcpDNS.Shutdown() _ = s.udpDNS.Shutdown() // Unset the address. s.address = "" } // Reconfigure updates the listener with a new configuration. func (s *Server) Reconfigure(address string) error { return s.Start(address) } // UpdateTSIG fetches all TSIG keys and loads them into the DNS server. func (s *Server) UpdateTSIG() error { // Locking. s.mu.Lock() defer s.mu.Unlock() return s.updateTSIG() } func (s *Server) updateTSIG() error { // Skip if no instance. if s.tcpDNS == nil || s.udpDNS == nil || s.db == nil { return nil } secrets := make(map[string]string) err := s.db.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get all the network zones. zones, err := dbCluster.GetNetworkZones(ctx, tx.Tx()) if err != nil { return err } // For each zone, get its config. for _, zone := range zones { // Get all configs for this zone. config, err := dbCluster.GetNetworkZoneConfig(ctx, tx.Tx(), zone.ID) if err != nil { return err } // Process each config entry. for key, value := range config { // Check if the key matches the pattern 'peers.%.key'. if !strings.HasPrefix(key, "peers.") || !strings.HasSuffix(key, ".key") { continue } // Split the key to extract the peer name. fields := strings.SplitN(key, ".", 3) if len(fields) != 3 { // Skip invalid values. continue } // Format as a valid TSIG secret (encode domain name, key name and make valid FQDN). secretKey := fmt.Sprintf("%s_%s.", zone.Name, fields[1]) secrets[secretKey] = value } } return nil }) if err != nil { return err } // Apply to the DNS servers. s.tcpDNS.TsigSecret = secrets s.udpDNS.TsigSecret = secrets return nil } incus-7.0.0/internal/server/dns/zone.go000066400000000000000000000002721517523235500200600ustar00rootroot00000000000000package dns import ( "github.com/lxc/incus/v7/shared/api" ) // Zone represents a DNS zone configuration and its content. type Zone struct { Info api.NetworkZone Content string } incus-7.0.0/internal/server/dnsmasq/000077500000000000000000000000001517523235500174375ustar00rootroot00000000000000incus-7.0.0/internal/server/dnsmasq/dhcpalloc/000077500000000000000000000000001517523235500213705ustar00rootroot00000000000000incus-7.0.0/internal/server/dnsmasq/dhcpalloc/dhcpalloc.go000066400000000000000000000303421517523235500236520ustar00rootroot00000000000000package dhcpalloc import ( "bytes" "encoding/binary" "errors" "io/fs" "math" "math/big" "net" "github.com/mdlayher/netx/eui64" "github.com/lxc/incus/v7/internal/iprange" "github.com/lxc/incus/v7/internal/server/dnsmasq" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) // ErrDHCPNotSupported indicates network doesn't support DHCP for this IP protocol. var ErrDHCPNotSupported error = errors.New("Network doesn't support DHCP") // DHCPValidIP returns whether an IP fits inside one of the supplied DHCP ranges and subnet. func DHCPValidIP(subnet *net.IPNet, ranges []iprange.Range, IP net.IP) bool { inSubnet := subnet.Contains(IP) if !inSubnet { return false } if len(ranges) == 0 { return true } for _, IPRange := range ranges { if bytes.Compare(IP, IPRange.Start) >= 0 && bytes.Compare(IP, IPRange.End) <= 0 { return true } } return false } // GetIP returns a net.IP representing the IP belonging to the subnet for the host number supplied. func GetIP(subnet *net.IPNet, host int64) net.IP { // Convert IP to a big int. bigIP := big.NewInt(0) bigIP.SetBytes(subnet.IP.To16()) // Deal with negative offsets. bigHost := big.NewInt(host) bigCount := big.NewInt(host) if host < 0 { mask, size := subnet.Mask.Size() bigHosts := big.NewFloat(0) bigHosts.SetFloat64((math.Pow(2, float64(size-mask)))) bigHostsInt, _ := bigHosts.Int(nil) bigCount.Set(bigHostsInt) bigCount.Add(bigCount, bigHost) } // Get the new IP int. bigIP.Add(bigIP, bigCount) // Generate an IPv6. if subnet.IP.To4() == nil { newIP := bigIP.Bytes() return newIP } // Generate an IPv4. newIP := make(net.IP, 4) binary.BigEndian.PutUint32(newIP, uint32(bigIP.Int64())) return newIP } // Network represents an Incus network responsible for running dnsmasq. type Network interface { Name() string Type() string Config() map[string]string DHCPv4Subnet() *net.IPNet DHCPv6Subnet() *net.IPNet DHCPv4Ranges() []iprange.Range DHCPv6Ranges() []iprange.Range } // Options to initialize the allocator with. type Options struct { ProjectName string HostName string DeviceName string HostMAC net.HardwareAddr Network Network } // Transaction is a locked transaction of the dnsmasq config files that allows IP allocations for a host. type Transaction struct { opts *Options currentDHCPMAC net.HardwareAddr currentDHCPv4 dnsmasq.DHCPAllocation currentDHCPv6 dnsmasq.DHCPAllocation allocationsDHCPv4 map[[4]byte]dnsmasq.DHCPAllocation allocationsDHCPv6 map[[16]byte]dnsmasq.DHCPAllocation allocatedIPv4 net.IP allocatedIPv6 net.IP } // AllocateIPv4 allocate an IPv4 static DHCP allocation. func (t *Transaction) AllocateIPv4() (net.IP, error) { var err error // Should have a (at least empty) map if DHCP is supported. if t.allocationsDHCPv4 == nil { return nil, ErrDHCPNotSupported } dhcpSubnet := t.opts.Network.DHCPv4Subnet() if dhcpSubnet == nil { return nil, ErrDHCPNotSupported } // Check the existing allocated IP is still valid in the network's subnet & ranges, if not then // we'll need to generate a new one. if t.allocatedIPv4 != nil { ranges := t.opts.Network.DHCPv4Ranges() if !DHCPValidIP(dhcpSubnet, ranges, t.allocatedIPv4.To4()) { t.allocatedIPv4 = nil // We need a new IP allocated. } } // Allocate a new IPv4 address if needed. if t.allocatedIPv4 == nil { t.allocatedIPv4, err = t.getDHCPFreeIPv4(t.allocationsDHCPv4, t.opts.HostName, t.opts.HostMAC) if err != nil { return nil, err } } return t.allocatedIPv4, nil } // AllocateIPv6 allocate an IPv6 static DHCP allocation. func (t *Transaction) AllocateIPv6() (net.IP, error) { var err error // Should have a (at least empty) map if DHCP is supported. if t.allocationsDHCPv6 == nil { return nil, ErrDHCPNotSupported } dhcpSubnet := t.opts.Network.DHCPv6Subnet() if dhcpSubnet == nil { return nil, ErrDHCPNotSupported } // Check the existing allocated IP is still valid in the network's subnet & ranges, if not then // we'll need to generate a new one. if t.allocatedIPv6 != nil { ranges := t.opts.Network.DHCPv6Ranges() if !DHCPValidIP(dhcpSubnet, ranges, t.allocatedIPv6.To16()) { t.allocatedIPv6 = nil // We need a new IP allocated. } } // Allocate a new IPv6 address if needed. if t.allocatedIPv6 == nil { t.allocatedIPv6, err = t.getDHCPFreeIPv6(t.allocationsDHCPv6, t.opts.HostName, t.opts.HostMAC) if err != nil { return nil, err } } return t.allocatedIPv6, nil } // getDHCPFreeIPv4 attempts to find a free IPv4 address for the device. // It first checks whether there is an existing allocation for the instance. // If no previous allocation, then a free IP is picked from the ranges configured. func (t *Transaction) getDHCPFreeIPv4(usedIPs map[[4]byte]dnsmasq.DHCPAllocation, deviceStaticFileName string, mac net.HardwareAddr) (net.IP, error) { ip, subnet, err := net.ParseCIDR(t.opts.Network.Config()["ipv4.address"]) if err != nil { return nil, err } dhcpRanges := t.opts.Network.DHCPv4Ranges() // Lets see if there is already an allocation for our device and that it sits within subnet. // If there are custom DHCP ranges defined, check also that the IP falls within one of the ranges. for _, DHCP := range usedIPs { if (deviceStaticFileName == DHCP.StaticFileName || bytes.Equal(mac, DHCP.MAC)) && DHCPValidIP(subnet, dhcpRanges, DHCP.IP) { return DHCP.IP, nil } } // If no custom ranges defined, convert subnet pool to a range. if len(dhcpRanges) <= 0 { dhcpRanges = append(dhcpRanges, iprange.Range{ Start: GetIP(subnet, 1).To4(), End: GetIP(subnet, -2).To4(), }, ) } // If no valid existing allocation found, try and find a free one in the subnet pool/ranges. for _, IPRange := range dhcpRanges { inc := big.NewInt(1) startBig := big.NewInt(0) startBig.SetBytes(IPRange.Start) endBig := big.NewInt(0) endBig.SetBytes(IPRange.End) for { if startBig.Cmp(endBig) >= 0 { break } IP := net.IP(startBig.Bytes()) // Check IP generated is not Incus's IP. if IP.Equal(ip) { startBig.Add(startBig, inc) continue } // Check IP is not already allocated. var IPKey [4]byte copy(IPKey[:], IP.To4()) _, inUse := usedIPs[IPKey] if inUse { startBig.Add(startBig, inc) continue } return IP, nil } } return nil, errors.New("No available IP could not be found") } // getDHCPFreeIPv6 attempts to find a free IPv6 address for the device. // It first checks whether there is an existing allocation for the instance. Due to the limitations // of dnsmasq lease file format, we can only search for previous static allocations. // If no previous allocation, then if SLAAC (stateless) mode is enabled on the network, or if // DHCPv6 stateful mode is enabled without custom ranges, then an EUI64 IP is generated from the // device's MAC address. Finally if stateful custom ranges are enabled, then a free IP is picked // from the ranges configured. func (t *Transaction) getDHCPFreeIPv6(usedIPs map[[16]byte]dnsmasq.DHCPAllocation, deviceStaticFileName string, mac net.HardwareAddr) (net.IP, error) { ip, subnet, err := net.ParseCIDR(t.opts.Network.Config()["ipv6.address"]) if err != nil { return nil, err } dhcpRanges := t.opts.Network.DHCPv6Ranges() // Lets see if there is already an allocation for our device and that it sits within subnet. // Because of dnsmasq's lease file format we can only match safely against static // allocations using instance name. If there are custom DHCP ranges defined, check also // that the IP falls within one of the ranges. for _, DHCP := range usedIPs { if deviceStaticFileName == DHCP.StaticFileName && DHCPValidIP(subnet, dhcpRanges, DHCP.IP) { return DHCP.IP, nil } } netConfig := t.opts.Network.Config() // Try using an EUI64 IP when in either SLAAC or DHCPv6 stateful mode without custom ranges. if util.IsFalseOrEmpty(netConfig["ipv6.dhcp.stateful"]) || netConfig["ipv6.dhcp.ranges"] == "" { IP, err := eui64.ParseMAC(subnet.IP, mac) if err != nil { return nil, err } // Check IP is not already allocated and not the Incus IP. var IPKey [16]byte copy(IPKey[:], IP.To16()) _, inUse := usedIPs[IPKey] if !inUse && !IP.Equal(ip) { return IP, nil } } // If no custom ranges defined, convert subnet pool to a range. if len(dhcpRanges) <= 0 { dhcpRanges = append(dhcpRanges, iprange.Range{ Start: GetIP(subnet, 1).To16(), End: GetIP(subnet, -1).To16(), }, ) } // If we get here, then someone already has our SLAAC IP, or we are using custom ranges. // Try and find a free one in the subnet pool/ranges. for _, IPRange := range dhcpRanges { inc := big.NewInt(1) startBig := big.NewInt(0) startBig.SetBytes(IPRange.Start) endBig := big.NewInt(0) endBig.SetBytes(IPRange.End) for { if startBig.Cmp(endBig) >= 0 { break } IP := net.IP(startBig.Bytes()) // Check IP generated is not Incus's IP. if IP.Equal(ip) { startBig.Add(startBig, inc) continue } // Check IP is not already allocated. var IPKey [16]byte copy(IPKey[:], IP.To16()) _, inUse := usedIPs[IPKey] if inUse { startBig.Add(startBig, inc) continue } return IP, nil } } return nil, errors.New("No available IP could not be found") } // AllocateTask initializes a new locked Transaction for a specific host and executes the supplied function on it. // The lock on the dnsmasq config is released when the function returns. func AllocateTask(opts *Options, f func(*Transaction) error) error { l := logger.AddContext(logger.Ctx{"driver": opts.Network.Type(), "network": opts.Network.Name(), "project": opts.ProjectName, "host": opts.HostName}) dnsmasq.ConfigMutex.Lock() defer dnsmasq.ConfigMutex.Unlock() var err error t := &Transaction{opts: opts} // Read current static IP allocation configured from dnsmasq host config (if exists). deviceStaticFileName := dnsmasq.StaticAllocationFileName(opts.ProjectName, opts.HostName, opts.DeviceName) t.currentDHCPMAC, t.currentDHCPv4, t.currentDHCPv6, err = dnsmasq.DHCPStaticAllocation(opts.Network.Name(), deviceStaticFileName) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } // Set the current allocated IPs internally (these may be changed later), and if they are it will trigger // a dnsmasq static host config file rebuild. t.allocatedIPv4 = t.currentDHCPv4.IP t.allocatedIPv6 = t.currentDHCPv6.IP // Get all existing allocations in network if leases file exists. If not then we will detect this later // due to the existing allocations maps being nil. if util.PathExists(internalUtil.VarPath("networks", opts.Network.Name(), "dnsmasq.leases")) { t.allocationsDHCPv4, t.allocationsDHCPv6, err = dnsmasq.DHCPAllAllocations(opts.Network.Name()) if err != nil { return err } } // Run the supplied allocation function. err = f(t) if err != nil { return err } // If the user's function didn't fail despite DHCP allocation not being available for either protocol, // then presumably they ignored allocation errors. So fail here, rather than try to write the new config // out, which will fail with a less helpful error about missing paths. if t.allocationsDHCPv4 == nil && t.allocationsDHCPv6 == nil { return ErrDHCPNotSupported } // If MAC or either IPv4 or IPv6 assigned is different than what is in dnsmasq config, rebuild config. macChanged := !bytes.Equal(opts.HostMAC, t.currentDHCPMAC) ipv4Changed := t.allocatedIPv4 != nil && !t.currentDHCPv4.IP.Equal(t.allocatedIPv4.To4()) ipv6Changed := t.allocatedIPv6 != nil && !t.currentDHCPv6.IP.Equal(t.allocatedIPv6.To16()) if macChanged || ipv4Changed || ipv6Changed { var IPv4Str, IPv6Str string if t.allocatedIPv4 != nil { IPv4Str = t.allocatedIPv4.String() } if t.allocatedIPv6 != nil { IPv6Str = t.allocatedIPv6.String() } // Write out new dnsmasq static host allocation config file. err = dnsmasq.UpdateStaticEntry(opts.Network.Name(), opts.ProjectName, opts.HostName, opts.DeviceName, opts.Network.Config(), opts.HostMAC.String(), IPv4Str, IPv6Str) if err != nil { return err } l.Debug("Updated static DHCP entry", logger.Ctx{"mac": opts.HostMAC.String(), "IPv4": IPv4Str, "IPv6": IPv6Str}) // Reload dnsmasq. err = dnsmasq.Kill(opts.Network.Name(), true) if err != nil { return err } } return nil } incus-7.0.0/internal/server/dnsmasq/dhcpalloc/dhcpalloc_test.go000066400000000000000000000032071517523235500247110ustar00rootroot00000000000000package dhcpalloc import ( "net" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/lxc/incus/v7/internal/iprange" ) func Test_DHCPValidIP(t *testing.T) { tests := []struct { name string subnet string ranges []string ip string expected bool }{ {name: "in subnet, no ranges", subnet: "192.168.0.0/16", ranges: nil, ip: "192.168.0.2", expected: true}, {name: "not in subnet, no ranges", subnet: "192.168.0.0/16", ranges: nil, ip: "10.10.0.2", expected: false}, {name: "in subnet, in given range", subnet: "192.168.0.0/16", ranges: []string{"192.168.0.0-192.168.0.10"}, ip: "192.168.0.2", expected: true}, {name: "in subnet, not in given range", subnet: "192.168.0.0/16", ranges: []string{"192.168.0.0-192.168.0.10"}, ip: "192.168.0.12", expected: false}, {name: "not in subnet, in given range", subnet: "192.168.0.0/16", ranges: []string{"10.10.0.0-10.10.0.10"}, ip: "10.10.0.2", expected: false}, {name: "not in subnet, not in given range", subnet: "192.168.0.0/16", ranges: []string{"192.168.0.0-192.168.0.10"}, ip: "10.10.0.12", expected: false}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { // arrange _, subnet, _ := net.ParseCIDR(test.subnet) var ranges []iprange.Range for _, rangesString := range test.ranges { rangeIps := strings.Split(rangesString, "-") ranges = append(ranges, iprange.Range{ Start: net.ParseIP(rangeIps[0]), End: net.ParseIP(rangeIps[1]), }) } ip := net.ParseIP(test.ip) // act isValidIP := DHCPValidIP(subnet, ranges, ip) // assert assert.Equal(t, test.expected, isValidIP) }) } } incus-7.0.0/internal/server/dnsmasq/dnsmasq.go000066400000000000000000000201571517523235500214410ustar00rootroot00000000000000package dnsmasq import ( "bufio" "errors" "fmt" "io/fs" "net" "os" "strings" "sync" "time" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/project" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) const staticAllocationDeviceSeparator = "." // DHCPAllocation represents an IP allocation from dnsmasq. type DHCPAllocation struct { IP net.IP StaticFileName string MAC net.HardwareAddr } // ConfigMutex used to coordinate access to the dnsmasq config files. var ConfigMutex sync.Mutex // UpdateStaticEntry writes a single dhcp-host line for a network/instance combination. func UpdateStaticEntry(network string, projectName string, instanceName string, deviceName string, netConfig map[string]string, hwaddr string, ipv4Address string, ipv6Address string) error { hwaddr = strings.ToLower(hwaddr) line := hwaddr // Generate the dhcp-host line if ipv4Address != "" { line += fmt.Sprintf(",%s", ipv4Address) } if ipv6Address != "" { line += fmt.Sprintf(",[%s]", ipv6Address) } if netConfig["dns.mode"] == "" || netConfig["dns.mode"] == "managed" { line += fmt.Sprintf(",%s", instanceName) } if line == hwaddr { return nil } deviceStaticFileName := StaticAllocationFileName(projectName, instanceName, deviceName) err := os.WriteFile(internalUtil.VarPath("networks", network, "dnsmasq.hosts", deviceStaticFileName), []byte(line+"\n"), 0o644) if err != nil { return err } return nil } // RemoveStaticEntry removes a single dhcp-host line for a network/instance combination. func RemoveStaticEntry(network string, projectName string, instanceName string, deviceName string) error { deviceStaticFileName := StaticAllocationFileName(projectName, instanceName, deviceName) err := os.Remove(internalUtil.VarPath("networks", network, "dnsmasq.hosts", deviceStaticFileName)) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } return nil } // Kill kills dnsmasq for a particular network (or optionally reloads it). func Kill(name string, reload bool) error { pidPath := internalUtil.VarPath("networks", name, "dnsmasq.pid") // If the pid file doesn't exist, there is no process to kill. if !util.PathExists(pidPath) { return nil } // Import saved subprocess details p, err := subprocess.ImportProcess(pidPath) if err != nil { return fmt.Errorf("Could not read pid file: %s", err) } if reload { err = p.Reload() if err != nil && !errors.Is(err, subprocess.ErrNotRunning) { return fmt.Errorf("Could not reload dnsmasq: %s", err) } return nil } err = p.Stop() if err != nil && !errors.Is(err, subprocess.ErrNotRunning) { return fmt.Errorf("Unable to kill dnsmasq: %s", err) } time.Sleep(100 * time.Millisecond) // Give OS time to release sockets. return nil } // DHCPStaticAllocationPath returns the path to the DHCP static allocation file. func DHCPStaticAllocationPath(network string, deviceStaticFileName string) string { return internalUtil.VarPath("networks", network, "dnsmasq.hosts", deviceStaticFileName) } // DHCPStaticAllocation retrieves the dnsmasq statically allocated MAC and IPs for an instance device static file. // Returns MAC, IPv4 and IPv6 DHCPAllocation structs respectively. func DHCPStaticAllocation(network string, deviceStaticFileName string) (net.HardwareAddr, DHCPAllocation, DHCPAllocation, error) { var IPv4, IPv6 DHCPAllocation var mac net.HardwareAddr file, err := os.Open(DHCPStaticAllocationPath(network, deviceStaticFileName)) if err != nil { return nil, IPv4, IPv6, err } defer func() { _ = file.Close() }() scanner := bufio.NewScanner(file) for scanner.Scan() { fields := strings.SplitN(scanner.Text(), ",", -1) for _, field := range fields { // Check if field is IPv4 or IPv6 address. if strings.Count(field, ".") == 3 { IP := net.ParseIP(field) if IP.To4() == nil { return nil, IPv4, IPv6, fmt.Errorf("Error parsing IP address %q", field) } IPv4 = DHCPAllocation{StaticFileName: deviceStaticFileName, IP: IP.To4(), MAC: mac} } else if strings.HasPrefix(field, "[") && strings.HasSuffix(field, "]") { IP := net.ParseIP(field[1 : len(field)-1]) if IP == nil { return nil, IPv4, IPv6, fmt.Errorf("Error parsing IP address %q", field) } IPv6 = DHCPAllocation{StaticFileName: deviceStaticFileName, IP: IP, MAC: mac} } else if strings.Count(field, ":") == 5 { // This field is expected to come first, so that mac variable can be used with // populating the DHCPAllocation structs too. mac, err = net.ParseMAC(field) if err != nil { return nil, IPv4, IPv6, fmt.Errorf("Error parsing MAC address %q", field) } } } } err = scanner.Err() if err != nil { return nil, IPv4, IPv6, err } return mac, IPv4, IPv6, nil } // DHCPAllAllocations returns a map of IPs currently allocated (statically and dynamically) // in dnsmasq for a specific network. The returned map is keyed by a 16 byte array representing // the net.IP format. The value of each map item is a DHCPAllocation struct containing at least // whether the allocation was static or dynamic and optionally instance name or MAC address. // MAC addresses are only included for dynamic IPv4 allocations (where name is not reliable). // Static allocations are not overridden by dynamic allocations, allowing for instance name to be // included for static IPv6 allocations. IPv6 addresses that are dynamically assigned cannot be // reliably linked to instances using either name or MAC because dnsmasq does not record the MAC // address for these records, and the recorded host name can be set by the instance if the dns.mode // for the network is set to "dynamic" and so cannot be trusted, so in this case we do not return // any identifying info. func DHCPAllAllocations(network string) (map[[4]byte]DHCPAllocation, map[[16]byte]DHCPAllocation, error) { IPv4s := make(map[[4]byte]DHCPAllocation) IPv6s := make(map[[16]byte]DHCPAllocation) // First read all statically allocated IPs. files, err := os.ReadDir(internalUtil.VarPath("networks", network, "dnsmasq.hosts")) if err != nil && errors.Is(err, fs.ErrNotExist) { return nil, nil, err } for _, entry := range files { _, IPv4, IPv6, err := DHCPStaticAllocation(network, entry.Name()) if err != nil { return nil, nil, err } if IPv4.IP != nil { var IPKey [4]byte copy(IPKey[:], IPv4.IP.To4()) IPv4s[IPKey] = IPv4 } if IPv6.IP != nil { var IPKey [16]byte copy(IPKey[:], IPv6.IP.To16()) IPv6s[IPKey] = IPv6 } } // Next read all dynamic allocated IPs. file, err := os.Open(internalUtil.VarPath("networks", network, "dnsmasq.leases")) if err != nil { return nil, nil, err } defer func() { _ = file.Close() }() scanner := bufio.NewScanner(file) for scanner.Scan() { fields := strings.Fields(scanner.Text()) if len(fields) == 5 { IP := net.ParseIP(fields[2]) if IP == nil { return nil, nil, fmt.Errorf("Error parsing IP address: %v", fields[2]) } // Handle IPv6 addresses. if IP.To4() == nil { var IPKey [16]byte copy(IPKey[:], IP.To16()) // Don't replace IPs from static config as more reliable. if IPv6s[IPKey].StaticFileName != "" { continue } IPv6s[IPKey] = DHCPAllocation{ IP: IP.To16(), } } else { // MAC only available in IPv4 leases. MAC, err := net.ParseMAC(fields[1]) if err != nil { return nil, nil, err } var IPKey [4]byte copy(IPKey[:], IP.To4()) // Don't replace IPs from static config as more reliable. if IPv4s[IPKey].StaticFileName != "" { continue } IPv4s[IPKey] = DHCPAllocation{ MAC: MAC, IP: IP.To4(), } } } } err = scanner.Err() if err != nil { return nil, nil, err } return IPv4s, IPv6s, nil } // StaticAllocationFileName returns the file name to use for a dnsmasq instance device static allocation. func StaticAllocationFileName(projectName string, instanceName string, deviceName string) string { escapedDeviceName := linux.PathNameEncode(deviceName) return strings.Join([]string{project.Instance(projectName, instanceName), escapedDeviceName}, staticAllocationDeviceSeparator) } incus-7.0.0/internal/server/dnsmasq/dnsmasq_test.go000066400000000000000000000006021517523235500224710ustar00rootroot00000000000000package dnsmasq import ( "testing" "github.com/stretchr/testify/assert" ) func Test_staticAllocationFileName(t *testing.T) { projectName := "test.project" instanceName := "test-instance" deviceName := "test/.-_--.device" fileName := StaticAllocationFileName(projectName, instanceName, deviceName) assert.Equal(t, "test.project_test-instance.test-.--_----.device", fileName) } incus-7.0.0/internal/server/endpoints/000077500000000000000000000000001517523235500177745ustar00rootroot00000000000000incus-7.0.0/internal/server/endpoints/cluster.go000066400000000000000000000044211517523235500220050ustar00rootroot00000000000000package endpoints import ( "fmt" "net" "time" "github.com/lxc/incus/v7/internal/ports" "github.com/lxc/incus/v7/internal/server/endpoints/listeners" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" ) // ClusterAddress returns the cluster address of the cluster endpoint, or an // empty string if there's no cluster endpoint or the cluster endpoint is provided by the network listener. func (e *Endpoints) clusterAddress() string { e.mu.RLock() defer e.mu.RUnlock() listener := e.listeners[cluster] if listener == nil { return "" } return listener.Addr().String() } // ClusterUpdateAddress updates the address for the cluster endpoint, shutting // it down and restarting it. func (e *Endpoints) ClusterUpdateAddress(address string) error { networkAddress := e.NetworkAddress() if address != "" { address = internalUtil.CanonicalNetworkAddress(address, ports.HTTPSDefaultPort) } oldAddress := e.clusterAddress() if address == oldAddress { return nil } logger.Infof("Update cluster address") e.mu.Lock() defer e.mu.Unlock() // Close the previous socket _ = e.closeListener(cluster) // If turning off listening, we're done if address == "" { return nil } // If networkAddress is set and address is covered, we don't need a new listener. if networkAddress != "" && internalUtil.IsAddressCovered(address, networkAddress) { return nil } // Attempt to setup the new listening socket getListener := func(address string) (*net.Listener, error) { var err error var listener net.Listener for range 10 { // Ten retries over a second seems reasonable. listener, err = net.Listen("tcp", address) if err == nil { break } time.Sleep(100 * time.Millisecond) } if err != nil { return nil, fmt.Errorf("Cannot listen on cluster HTTPS socket %q: %w", address, err) } return &listener, nil } // set up the listener listener, err := getListener(address) if err != nil { // Attempt to revert to the previous address listener, err1 := getListener(oldAddress) if err1 == nil { e.listeners[cluster] = listeners.NewFancyTLSListener(*listener, e.cert) e.serve(cluster) } return err } e.listeners[cluster] = listeners.NewFancyTLSListener(*listener, e.cert) e.serve(cluster) return nil } incus-7.0.0/internal/server/endpoints/cluster_test.go000066400000000000000000000021511517523235500230420ustar00rootroot00000000000000package endpoints_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // If both a network and a cluster address are set, and they differ, a new // network TCP socket will be created. func TestEndpoints_ClusterCreateTCPSocket(t *testing.T) { endpoints, config, cleanup := newEndpoints(t) defer cleanup() config.NetworkAddress = "127.0.0.1:12345" config.ClusterAddress = "127.0.0.1:54321" require.NoError(t, endpoints.Up(config)) assert.NoError(t, httpGetOverTLSSocket(endpoints.NetworkAddressAndCert())) assert.NoError(t, httpGetOverTLSSocket(endpoints.ClusterAddressAndCert())) } // When the cluster address is actually covered by the network one, no new port // is opened. func TestEndpoints_ClusterUpdateAddressIsCovered(t *testing.T) { endpoints, config, cleanup := newEndpoints(t) defer cleanup() config.NetworkAddress = "[::]:12345" config.ClusterAddress = "" require.NoError(t, endpoints.Up(config)) require.NoError(t, endpoints.ClusterUpdateAddress("127.0.0.1:12345")) assert.NoError(t, httpGetOverTLSSocket(endpoints.NetworkAddressAndCert())) } incus-7.0.0/internal/server/endpoints/dev_incus.go000066400000000000000000000021551517523235500223050ustar00rootroot00000000000000//go:build linux && cgo package endpoints import ( "net" "path/filepath" ) // Create a new net.Listener bound to the unix socket of the devIncus endpoint. func createDevIncuslListener(dir string) (net.Listener, error) { path := filepath.Join(dir, "guestapi", "sock") // If this socket exists, that means a previous daemon died and // didn't clean up. We assume that such a daemon is actually dead // if we get this far, since localCreateListener() tries to connect to // the actual socket to make sure that it is actually dead. So, it // is safe to remove it here without any checks. // // Also, it would be nice to SO_REUSEADDR here so we don't have to // delete the socket, but we can't: // http://stackoverflow.com/questions/15716302/so-reuseaddr-and-af-unix // // Note that this will force clients to reconnect on restart. err := socketUnixRemoveStale(path) if err != nil { return nil, err } listener, err := socketUnixListen(path) if err != nil { return nil, err } err = socketUnixSetPermissions(path, 0o666) if err != nil { _ = listener.Close() return nil, err } return listener, nil } incus-7.0.0/internal/server/endpoints/dev_incus_test.go000066400000000000000000000011621517523235500233410ustar00rootroot00000000000000package endpoints_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/shared/util" ) // If no socket-based activation is detected, a new local unix socket will be // created. func TestEndpoints_DevIncusCreateUnixSocket(t *testing.T) { endpoints, config, cleanup := newEndpoints(t) defer cleanup() require.NoError(t, endpoints.Up(config)) path := endpoints.DevIncusSocketPath() assert.NoError(t, httpGetOverUnixSocket(path)) // The unix socket file gets removed after shutdown. cleanup() assert.Equal(t, false, util.PathExists(path)) } incus-7.0.0/internal/server/endpoints/endpoints.go000066400000000000000000000335551517523235500223410ustar00rootroot00000000000000package endpoints import ( "errors" "fmt" "net" "net/http" "sync" "time" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/endpoints/listeners" "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" localtls "github.com/lxc/incus/v7/shared/tls" ) // Config holds various configuration values that affect endpoints initialization. type Config struct { // The directory to create Unix sockets in. Dir string // UnixSocket is the path to the Unix socket to bind UnixSocket string // HTTP server handling requests for the REST API. RestServer *http.Server // HTTP server for the internal /dev/incus API exposed to containers. DevIncusServer *http.Server // The TLS keypair and optional CA to use for the network endpoint. It // must be always provided, since the pubblic key will be included in // the response of the /1.0 REST API as part of the server info. // // It can be updated after the endpoints are up using NetworkUpdateCert(). Cert *localtls.CertInfo // System group name to which the unix socket for the local endpoint should be // chgrp'ed when starting. The default is to use the process group. An empty // string means "use the default". LocalUnixSocketGroup string // SELinux label to apply to the soecket. LocalUnixSocketLabel string // NetworkSetAddress sets the address for the network endpoint. If not // set, the network endpoint won't be started (unless it's passed via // socket-based activation). // // It can be updated after the endpoints are up using NetworkUpdateAddress(). NetworkAddress string // Optional dedicated network address for clustering traffic. If not // set, NetworkAddress will be used. // // It can be updated after the endpoints are up using ClusterUpdateAddress(). ClusterAddress string // Address of the debug endpoint. // // It can be updated after the endpoints are up using PprofUpdateAddress(). DebugAddress string // HTTP server handling requests for the metrics API. MetricsServer *http.Server // HTTP server handling requests for the storage buckets API. StorageBucketsServer *http.Server // HTTP server handling requests from VMs via the vsock. VsockServer *http.Server // True if VMs are supported. VsockSupport bool } // Up brings up all applicable endpoints and starts accepting HTTP // requests. // // The endpoints will be activated in the following order and according to the // following rules: // // local endpoint (unix socket) // ---------------------------- // // If socket-based activation is detected, look for a unix socket among the // inherited file descriptors and use it for the local endpoint (or if no such // file descriptor exists, don't bring up the local endpoint at all). // // If no socket-based activation is detected, create a unix socket using the // default /unix.socket path. The file mode of this socket will be set // to 660, the file owner will be set to the process' UID, and the file group // will be set to the process GID, or to the GID of the system group name // specified via config.LocalUnixSocketGroup. // // devIncus endpoint (unix socket) // ---------------------------- // // Created using /dev_incus/sock, with file mode set to 666 (actual // authorization will be performed by the HTTP server using the socket ucred // struct). // // remote endpoint (TCP socket with TLS) // ------------------------------------- // // If socket-based activation is detected, look for a network socket among the // inherited file descriptors and use it for the network endpoint. // // If a network address was set via config.NetworkAddress, then close any listener // that was detected via socket-based activation and create a new network // socket bound to the given address. // // The network endpoint socket will use TLS encryption, using the certificate // keypair and CA passed via config.Cert. // // cluster endpoint (TCP socket with TLS) // ------------------------------------- // // If a network address was set via config.ClusterAddress, then attach // config.RestServer to it. func Up(config *Config) (*Endpoints, error) { if config.Dir == "" { return nil, errors.New("No directory configured") } if config.UnixSocket == "" { return nil, errors.New("No unix socket configured") } if config.RestServer == nil { return nil, errors.New("No REST server configured") } if config.DevIncusServer == nil { return nil, errors.New("No devIncus server configured") } if config.Cert == nil { return nil, errors.New("No TLS certificate configured") } endpoints := &Endpoints{ systemdListenFDsStart: linux.SystemdListenFDsStart, } err := endpoints.up(config) if err != nil { _ = endpoints.Down() return nil, err } return endpoints, nil } // Endpoints are in charge of bringing up and down the HTTP endpoints for // serving the REST API. type Endpoints struct { tomb *Tomb // Controls the HTTP servers shutdown. mu sync.RWMutex // Serialize access to internal state. listeners map[kind]net.Listener // Activer listeners by endpoint type. servers map[kind]*http.Server // HTTP servers by endpoint type. cert *localtls.CertInfo // Keypair and CA to use for TLS. inherited map[kind]bool // Store whether the listener came through socket activation systemdListenFDsStart int // First socket activation FD, for tests. } // Up brings up all configured endpoints and starts accepting HTTP requests. func (e *Endpoints) up(config *Config) error { e.mu.Lock() defer e.mu.Unlock() e.servers = map[kind]*http.Server{ devIncus: config.DevIncusServer, local: config.RestServer, network: config.RestServer, cluster: config.RestServer, pprof: pprofCreateServer(), metrics: config.MetricsServer, storageBuckets: config.StorageBucketsServer, vmvsock: config.VsockServer, } e.cert = config.Cert e.inherited = map[kind]bool{} var err error // Check for socket activation. systemdListeners := linux.GetSystemdListeners(e.systemdListenFDsStart) if len(systemdListeners) > 0 { e.listeners = activatedListeners(systemdListeners, e.cert) for kind := range e.listeners { e.inherited[kind] = true } } else { e.listeners = map[kind]net.Listener{} e.listeners[local], err = localCreateListener(config.UnixSocket, config.LocalUnixSocketGroup, config.LocalUnixSocketLabel) if err != nil { return fmt.Errorf("Local endpoint: %w", err) } } // Setup STARTTLS layer on local listener. if e.listeners[local] != nil { e.listeners[local] = listeners.NewSTARTTLSListener(e.listeners[local], e.cert) } // Start the devIncus listener e.listeners[devIncus], err = createDevIncuslListener(config.Dir) if err != nil { return err } // Start the VM sock listener. if config.VsockSupport { e.listeners[vmvsock], err = createVsockListener(e.cert) if err != nil { return err } } if config.NetworkAddress != "" { listener, ok := e.listeners[network] if ok { logger.Infof("Replacing inherited TCP socket with configured one") _ = listener.Close() e.inherited[network] = false } // Errors here are not fatal and are just logged (unless we're clustered, see below). var networkAddressErr error attempts := 0 againHttps: e.listeners[network], networkAddressErr = networkCreateListener(config.NetworkAddress, e.cert) isCovered := util.IsAddressCovered(config.ClusterAddress, config.NetworkAddress) if config.ClusterAddress != "" { if isCovered { // In case of clustering we fail if we can't bind the network address. if networkAddressErr != nil { if attempts == 0 { logger.Infof("Unable to bind https address %q, re-trying for a minute", config.NetworkAddress) } attempts++ if attempts < 60 { time.Sleep(1 * time.Second) goto againHttps } return networkAddressErr } e.serve(cluster) } } else if networkAddressErr != nil { logger.Error("Cannot currently listen on https socket, re-trying once in 30s...", logger.Ctx{"err": networkAddressErr}) go func() { time.Sleep(30 * time.Second) err := e.NetworkUpdateAddress(config.NetworkAddress) if err != nil { logger.Error("Still unable to listen on https socket", logger.Ctx{"err": err}) } }() } } isCovered := false if config.NetworkAddress != "" { isCovered = util.IsAddressCovered(config.ClusterAddress, config.NetworkAddress) } if config.ClusterAddress != "" && !isCovered { attempts := 0 againCluster: e.listeners[cluster], err = networkCreateListener(config.ClusterAddress, e.cert) if err != nil { if attempts == 0 { logger.Infof("Unable to bind cluster address %q, re-trying for a minute", config.ClusterAddress) } attempts++ if attempts < 60 { time.Sleep(1 * time.Second) goto againCluster } return err } e.serve(cluster) } if config.DebugAddress != "" { e.listeners[pprof], err = pprofCreateListener(config.DebugAddress) if err != nil { return err } e.serve(pprof) } for kind := range e.listeners { e.serve(kind) } return nil } // UpMetrics brings up metrics listener on specified address. func (e *Endpoints) UpMetrics(listenAddress string) error { var err error e.listeners[metrics], err = metricsCreateListener(listenAddress, e.cert) if err != nil { return fmt.Errorf("Failed starting metrics listener: %w", err) } e.serve(metrics) return nil } // UpStorageBuckets brings up storage buvkets listener on specified address. func (e *Endpoints) UpStorageBuckets(listenAddress string) error { var err error e.listeners[storageBuckets], err = storageBucketsCreateListener(listenAddress, e.cert) if err != nil { return fmt.Errorf("Failed starting storage buckets listener: %w", err) } e.serve(storageBuckets) return nil } // Down brings down all endpoints and stops serving HTTP requests. func (e *Endpoints) Down() error { e.mu.Lock() defer e.mu.Unlock() if e.listeners[network] != nil || e.listeners[local] != nil { err := e.closeListener(network) if err != nil { return err } err = e.closeListener(local) if err != nil { return err } } if e.listeners[cluster] != nil { err := e.closeListener(cluster) if err != nil { return err } } if e.listeners[devIncus] != nil { err := e.closeListener(devIncus) if err != nil { return err } } if e.listeners[pprof] != nil { err := e.closeListener(pprof) if err != nil { return err } } if e.listeners[metrics] != nil { err := e.closeListener(metrics) if err != nil { return err } } if e.listeners[storageBuckets] != nil { err := e.closeListener(storageBuckets) if err != nil { return err } } if e.listeners[vmvsock] != nil { err := e.closeListener(vmvsock) if err != nil { return err } } if e.tomb != nil { e.tomb.Kill(nil) _ = e.tomb.Wait() } return nil } // Start an HTTP server for the endpoint associated with the given code. func (e *Endpoints) serve(kind kind) { listener := e.listeners[kind] if listener == nil { return } ctx := logger.Ctx{"type": kind.String(), "socket": listener.Addr()} if e.inherited[kind] { ctx["inherited"] = true } logger.Info("Binding socket", ctx) server := e.servers[kind] // Defer the creation of the tomb, so Down() doesn't wait on it unless // we actually have spawned at least a server. if e.tomb == nil { e.tomb = &Tomb{} } e.tomb.Go(func() error { return server.Serve(listener) }) } // Stop the HTTP server of the endpoint associated with the given code. The // associated socket will be shutdown too. func (e *Endpoints) closeListener(kind kind) error { listener := e.listeners[kind] if listener == nil { return nil } delete(e.listeners, kind) logger.Info("Closing socket", logger.Ctx{"type": kind.String(), "socket": listener.Addr()}) return listener.Close() } // Use the listeners associated with the file descriptors passed via // socket-based activation. func activatedListeners(systemdListeners []net.Listener, cert *localtls.CertInfo) map[kind]net.Listener { activatedListeners := map[kind]net.Listener{} for _, listener := range systemdListeners { var kind kind switch listener.(type) { case *net.UnixListener: kind = local case *net.TCPListener: kind = network listener = listeners.NewFancyTLSListener(listener, cert) default: continue } activatedListeners[kind] = listener } return activatedListeners } // Numeric code identifying a specific API endpoint type. type kind int // String returns human readable name of endpoint kind. func (k kind) String() string { return descriptions[k] } // Numeric codes identifying the various endpoints. const ( local kind = iota devIncus network pprof cluster metrics vmvsock storageBuckets ) // Human-readable descriptions of the various kinds of endpoints. var descriptions = map[kind]string{ local: "REST API Unix socket", devIncus: "devIncus socket", network: "REST API TCP socket", pprof: "pprof socket", cluster: "cluster socket", metrics: "metrics socket", vmvsock: "VM socket", storageBuckets: "Storage buckets socket", } // Tomb tracks the lifecycle of one or more goroutines. type Tomb struct { wg sync.WaitGroup count int mutex sync.RWMutex errOnce sync.Once err error } func (g *Tomb) add(delta int) { g.mutex.Lock() defer g.mutex.Unlock() g.count += delta if g.count >= 0 { g.wg.Add(delta) } } // Go runs f in a new goroutine and tracks its termination. func (g *Tomb) Go(f func() error) { g.add(1) go func() { defer g.add(-1) err := f() if err != nil { g.errOnce.Do(func() { g.err = err }) } }() } // Kill marks all running goroutings done. func (g *Tomb) Kill(err error) { if err != nil { g.errOnce.Do(func() { g.err = err }) } g.mutex.RLock() count := g.count g.mutex.RUnlock() if count != 0 { g.add(-count) } } // Wait blocks until all goroutines have finished running. func (g *Tomb) Wait() error { g.wg.Wait() return g.err } incus-7.0.0/internal/server/endpoints/endpoints_exported_test.go000066400000000000000000000031101517523235500252720ustar00rootroot00000000000000package endpoints import ( "github.com/lxc/incus/v7/internal/linux" localtls "github.com/lxc/incus/v7/shared/tls" ) // New creates a new Endpoints instance without bringing it up. func Unstarted() *Endpoints { return &Endpoints{ systemdListenFDsStart: linux.SystemdListenFDsStart, } } func (e *Endpoints) Up(config *Config) error { return e.up(config) } // Return the path to the devIncus socket file. func (e *Endpoints) DevIncusSocketPath() string { e.mu.RLock() defer e.mu.RUnlock() listener := e.listeners[devIncus] return listener.Addr().String() } func (e *Endpoints) LocalSocketPath() string { e.mu.RLock() defer e.mu.RUnlock() listener := e.listeners[local] return listener.Addr().String() } // Return the network address and server certificate of the network // endpoint. This method is supposed to be used in conjunction with // the httpGetOverTLSSocket test helper. func (e *Endpoints) NetworkAddressAndCert() (string, *localtls.CertInfo) { return e.NetworkAddress(), e.cert } // Return the cluster address and server certificate of the network // endpoint. This method is supposed to be used in conjunction with // the httpGetOverTLSSocket test helper. func (e *Endpoints) ClusterAddressAndCert() (string, *localtls.CertInfo) { return e.clusterAddress(), e.cert } // Set the file descriptor number marker that will be used when detecting // socket activation. Needed because "go test" might open unrelated file // descriptor starting at number 3. func (e *Endpoints) SystemdListenFDsStart(start int) { e.mu.Lock() defer e.mu.Unlock() e.systemdListenFDsStart = start } incus-7.0.0/internal/server/endpoints/endpoints_test.go000066400000000000000000000066761517523235500234040ustar00rootroot00000000000000package endpoints_test import ( "context" "fmt" "io" "log" "net" "net/http" "os" "path/filepath" "runtime" "strconv" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/endpoints" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/api" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/tls/tlstest" "github.com/lxc/incus/v7/shared/util" ) // Return a new unstarted Endpoints instance, a Config with stub rest/devIncus // servers, and a cleanup function that can be used to clear all state // associated with the endpoints (e.g. the temporary var dir and any // goroutine that was spawned by the tomb). func newEndpoints(t *testing.T) (*endpoints.Endpoints, *endpoints.Config, func()) { dir, err := os.MkdirTemp("", "incus-endpoints-test-") require.NoError(t, err) require.NoError(t, os.Mkdir(filepath.Join(dir, "guestapi"), 0o755)) config := &endpoints.Config{ Dir: dir, UnixSocket: filepath.Join(dir, "unix.socket"), RestServer: newServer(), DevIncusServer: newServer(), Cert: tlstest.TestingKeyPair(t), VsockServer: newServer(), } endpoints := endpoints.Unstarted() cleanup := func() { assert.NoError(t, endpoints.Down()) // We need to kick the garbage collector because otherwise FDs // will be left open and confuse the http.GetListeners() code // that detects socket activation. runtime.GC() if util.PathExists(dir) { require.NoError(t, os.RemoveAll(dir)) } } return endpoints, config, cleanup } // Perform an HTTP GET "/" over the unix socket at the given path. func httpGetOverUnixSocket(path string) error { dial := func(_ context.Context, network, addr string) (net.Conn, error) { return net.Dial("unix", path) } client := &http.Client{Transport: &http.Transport{DialContext: dial}} _, err := client.Get("http://unix.socket/") return err } // Perform an HTTP GET "/" over TLS, using the given network address and server // certificate. func httpGetOverTLSSocket(addr string, cert *localtls.CertInfo) error { tlsConfig, _ := localtls.GetTLSConfigMem("", "", "", string(cert.PublicKey()), false) client := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsConfig}} _, err := client.Get(fmt.Sprintf("https://%s/", addr)) return err } // Returns a minimal stub for the REST API server, just realistic // enough to make incus.ConnectIncusUnix succeed. func newServer() *http.Server { mux := http.NewServeMux() mux.HandleFunc("/1.0/", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _ = localUtil.WriteJSON(w, api.ResponseRaw{}, nil) }) return &http.Server{Handler: mux, ErrorLog: log.New(io.Discard, "", 0)} } // Set the environment-variable for socket-based activation using the given // file. func setupSocketBasedActivation(endpoints *endpoints.Endpoints, file *os.File) { _ = os.Setenv("LISTEN_PID", strconv.Itoa(os.Getpid())) _ = os.Setenv("LISTEN_FDS", "1") endpoints.SystemdListenFDsStart(int(file.Fd())) } // Assert that there are no socket-based activation variables in the // environment. func assertNoSocketBasedActivation(t *testing.T) { // The environment variables are automatically cleaned, to avoid // confusing child processes or other logic. for _, name := range []string{"LISTEN_PID", "LISTEN_FDS"} { _, ok := os.LookupEnv(name) assert.Equal(t, false, ok) } } incus-7.0.0/internal/server/endpoints/listeners/000077500000000000000000000000001517523235500220045ustar00rootroot00000000000000incus-7.0.0/internal/server/endpoints/listeners/fancytls.go000066400000000000000000000035041517523235500241600ustar00rootroot00000000000000package listeners import ( "crypto/tls" "net" "slices" "sync" "github.com/pires/go-proxyproto" "github.com/lxc/incus/v7/internal/server/util" localtls "github.com/lxc/incus/v7/shared/tls" ) // FancyTLSListener is a variation of the standard tls.Listener that supports // atomically swapping the underlying TLS configuration and proxy protocol wrapping. // Requests served before the swap will continue using the old configuration. type FancyTLSListener struct { net.Listener mu sync.RWMutex config *tls.Config trustedProxy []net.IP } // NewFancyTLSListener creates a new FancyTLSListener. func NewFancyTLSListener(inner net.Listener, cert *localtls.CertInfo) *FancyTLSListener { listener := &FancyTLSListener{ Listener: inner, } listener.Config(cert) return listener } // Accept waits for and returns the next incoming TLS connection then use the // current TLS configuration to handle it. func (l *FancyTLSListener) Accept() (net.Conn, error) { c, err := l.Listener.Accept() if err != nil { return nil, err } l.mu.RLock() defer l.mu.RUnlock() config := l.config if isProxy(c.RemoteAddr().String(), l.trustedProxy) { c = proxyproto.NewConn(c) } return tls.Server(c, config), nil } // Config safely swaps the underlying TLS configuration. func (l *FancyTLSListener) Config(cert *localtls.CertInfo) { config := util.ServerTLSConfig(cert) l.mu.Lock() defer l.mu.Unlock() l.config = config } // TrustedProxy sets new the https trusted proxy configuration. func (l *FancyTLSListener) TrustedProxy(trustedProxy []net.IP) { l.mu.Lock() defer l.mu.Unlock() l.trustedProxy = trustedProxy } func isProxy(addr string, proxies []net.IP) bool { host, _, err := net.SplitHostPort(addr) if err != nil { return false } hostIP := net.ParseIP(host) return slices.ContainsFunc(proxies, hostIP.Equal) } incus-7.0.0/internal/server/endpoints/listeners/starttls.go000066400000000000000000000050111517523235500242100ustar00rootroot00000000000000package listeners import ( "bufio" "crypto/tls" "errors" "net" "sync" "github.com/lxc/incus/v7/internal/server/util" localtls "github.com/lxc/incus/v7/shared/tls" ) // StarttlsListener is a variation of the standard tls.Listener that supports // atomically swapping the underlying TLS configuration. Requests served // before the swap will continue using the old configuration. type StarttlsListener struct { net.Listener mu sync.RWMutex config *tls.Config } // NewSTARTTLSListener creates a new STARTTLS listener. func NewSTARTTLSListener(inner net.Listener, cert *localtls.CertInfo) *StarttlsListener { listener := &StarttlsListener{ Listener: inner, } listener.Config(cert) return listener } // Accept waits for and returns the next incoming TLS connection then use the // current TLS configuration to handle it. func (l *StarttlsListener) Accept() (net.Conn, error) { c, err := l.Listener.Accept() if err != nil { return nil, err } // Setup buffered connection. bufConn := BufferedUnixConn{bufio.NewReader(c), c.(*net.UnixConn)} // Peak to see if STARTTLS. header, err := bufConn.Peek(8) if err == nil && string(header) == "STARTTLS" { discarded, err := bufConn.Discard(9) if err != nil { return nil, err } if discarded < 9 { return nil, errors.New("Bad STARTTLS header on connection") } l.mu.RLock() defer l.mu.RUnlock() config := l.config return tls.Server(bufConn, config), nil } return bufConn, nil } // Config safely swaps the underlying TLS configuration. func (l *StarttlsListener) Config(cert *localtls.CertInfo) { config := util.ServerTLSConfig(cert) // Always use network certificate's DNS name rather than server cert, so that it matches. x509Cert, err := cert.PublicKeyX509() if err == nil && len(x509Cert.DNSNames) > 0 { config.ServerName = x509Cert.DNSNames[0] } l.mu.Lock() defer l.mu.Unlock() l.config = config } // BufferedUnixConn is a UnixConn wrapped in a Bufio Reader. type BufferedUnixConn struct { r *bufio.Reader *net.UnixConn } // Discard allows discarding some bytes from the buffer. func (b BufferedUnixConn) Discard(n int) (int, error) { return b.r.Discard(n) } // Peek allows reading some bytes without moving the read pointer. func (b BufferedUnixConn) Peek(n int) ([]byte, error) { return b.r.Peek(n) } // Read allows normal reads on the buffered connection. func (b BufferedUnixConn) Read(p []byte) (int, error) { return b.r.Read(p) } // Unix returns the inner UnixConn. func (b BufferedUnixConn) Unix() *net.UnixConn { return b.UnixConn } incus-7.0.0/internal/server/endpoints/local.go000066400000000000000000000021061517523235500214140ustar00rootroot00000000000000//go:build linux && cgo package endpoints import ( "net" ) // Create a new net.Listener bound to the unix socket of the local endpoint. func localCreateListener(path string, group string, label string) (net.Listener, error) { err := CheckAlreadyRunning(path) if err != nil { return nil, err } err = socketUnixRemoveStale(path) if err != nil { return nil, err } listener, err := socketUnixListen(path) if err != nil { return nil, err } err = localSetAccess(path, group, label) if err != nil { _ = listener.Close() return nil, err } return listener, nil } // Change the file mode and ownership of the local endpoint unix socket file, // so access is granted only to the process user and to the given group (or the // process group if group is empty). func localSetAccess(path string, group string, label string) error { err := socketUnixSetPermissions(path, 0o660) if err != nil { return err } err = socketUnixSetOwnership(path, group) if err != nil { return err } err = socketUnixSetLabel(path, label) if err != nil { return err } return nil } incus-7.0.0/internal/server/endpoints/local_test.go000066400000000000000000000057161517523235500224650ustar00rootroot00000000000000package endpoints_test import ( "net" "os" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/shared/util" ) // If no socket-based activation is detected, a new local unix socket will be // created. func TestEndpoints_LocalCreateUnixSocket(t *testing.T) { endpoints, config, cleanup := newEndpoints(t) defer cleanup() require.NoError(t, endpoints.Up(config)) path := endpoints.LocalSocketPath() assert.NoError(t, httpGetOverUnixSocket(path)) // The unix socket file gets removed after shutdown. cleanup() assert.Equal(t, false, util.PathExists(path)) } // If socket-based activation is detected, it will be used for binding the API // Endpoints' unix socket. func TestEndpoints_LocalSocketBasedActivation(t *testing.T) { listener := newUnixListener(t) defer func() { _ = listener.Close() }() // This will also remove the underlying file file, err := listener.File() require.NoError(t, err) defer func() { _ = file.Close() }() endpoints, config, cleanup := newEndpoints(t) defer cleanup() setupSocketBasedActivation(endpoints, file) require.NoError(t, endpoints.Up(config)) assertNoSocketBasedActivation(t) path := endpoints.LocalSocketPath() assert.NoError(t, httpGetOverUnixSocket(path)) // The unix socket file does not get removed after shutdown (thanks to // this change in Go 1.6: // // https://github.com/golang/go/commit/a4fd325c178ea29f554d69de4f2c3ffa09b53874 // // which prevents listeners created from file descriptors from removing // their socket files on close). cleanup() assert.Equal(t, true, util.PathExists(path)) } // If a custom group for the unix socket is specified, but no such one exists, // an error is returned. func TestEndpoints_LocalUnknownUnixGroup(t *testing.T) { endpoints, config, cleanup := newEndpoints(t) defer cleanup() config.LocalUnixSocketGroup = "xquibaz" err := endpoints.Up(config) assert.EqualError( t, err, "Local endpoint: cannot get group ID of 'xquibaz': group: unknown group xquibaz") } // If another endpoint is already listening on the unix socket, an error is returned. func TestEndpoints_LocalAlreadyRunning(t *testing.T) { endpoints1, config1, cleanup1 := newEndpoints(t) defer cleanup1() require.NoError(t, endpoints1.Up(config1)) endpoints2, config2, cleanup2 := newEndpoints(t) config2.Dir = config1.Dir config2.UnixSocket = config1.UnixSocket defer cleanup2() err := endpoints2.Up(config2) assert.EqualError(t, err, "Local endpoint: Incus is already running") } // Create a UnixListener using a random and unique file name. func newUnixListener(t *testing.T) *net.UnixListener { file, err := os.CreateTemp("", "incus-endpoints-test") require.NoError(t, err) path := file.Name() require.NoError(t, file.Close()) err = os.Remove(path) require.NoError(t, err) addr, err := net.ResolveUnixAddr("unix", path) require.NoError(t, err) listener, err := net.ListenUnix("unix", addr) require.NoError(t, err) return listener } incus-7.0.0/internal/server/endpoints/metrics.go000066400000000000000000000052461517523235500220000ustar00rootroot00000000000000package endpoints import ( "fmt" "net" "strings" "time" "github.com/lxc/incus/v7/internal/ports" "github.com/lxc/incus/v7/internal/server/endpoints/listeners" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" localtls "github.com/lxc/incus/v7/shared/tls" ) func metricsCreateListener(address string, cert *localtls.CertInfo) (net.Listener, error) { // Listening on `tcp` network with address 0.0.0.0 will end up with listening // on both IPv4 and IPv6 interfaces. Pass `tcp4` to make it // work only on 0.0.0.0. https://go-review.googlesource.com/c/go/+/45771/ listenAddress := internalUtil.CanonicalNetworkAddress(address, ports.HTTPSMetricsDefaultPort) protocol := "tcp" if strings.HasPrefix(listenAddress, "0.0.0.0") { protocol = "tcp4" } listener, err := net.Listen(protocol, listenAddress) if err != nil { return nil, fmt.Errorf("Bind network address: %w", err) } return listeners.NewFancyTLSListener(listener, cert), nil } // MetricsAddress returns the network address of the metrics endpoint, or an // empty string if there's no metrics endpoint. func (e *Endpoints) MetricsAddress() string { e.mu.RLock() defer e.mu.RUnlock() listener := e.listeners[metrics] if listener == nil { return "" } return listener.Addr().String() } // MetricsUpdateAddress updates the address for the metrics endpoint, shutting it down and restarting it. func (e *Endpoints) MetricsUpdateAddress(address string, cert *localtls.CertInfo) error { if address != "" { address = internalUtil.CanonicalNetworkAddress(address, ports.HTTPSMetricsDefaultPort) } oldAddress := e.MetricsAddress() if address == oldAddress { return nil } logger.Infof("Update metrics address") e.mu.Lock() defer e.mu.Unlock() // Close the previous socket _ = e.closeListener(metrics) // If turning off listening, we're done if address == "" { return nil } // Attempt to setup the new listening socket getListener := func(address string) (*net.Listener, error) { var err error var listener net.Listener for range 10 { // Ten retries over a second seems reasonable. listener, err = metricsCreateListener(address, cert) if err == nil { break } time.Sleep(100 * time.Millisecond) } if err != nil { return nil, fmt.Errorf("Cannot listen on http socket: %w", err) } return &listener, nil } // Set up the listener listener, err := getListener(address) if err != nil { // Attempt to revert to the previous address listener, err1 := getListener(oldAddress) if err1 == nil { e.listeners[metrics] = *listener e.serve(metrics) } return err } e.listeners[metrics] = *listener e.serve(metrics) return nil } incus-7.0.0/internal/server/endpoints/network.go000066400000000000000000000117171517523235500220230ustar00rootroot00000000000000package endpoints import ( "fmt" "log" "net" "strings" "time" "github.com/lxc/incus/v7/internal/ports" "github.com/lxc/incus/v7/internal/server/endpoints/listeners" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) // NetworkPublicKey returns the public key of the TLS certificate used by the // network endpoint. func (e *Endpoints) NetworkPublicKey() []byte { e.mu.RLock() defer e.mu.RUnlock() return e.cert.PublicKey() } // NetworkPrivateKey returns the private key of the TLS certificate used by the // network endpoint. func (e *Endpoints) NetworkPrivateKey() []byte { e.mu.RLock() defer e.mu.RUnlock() return e.cert.PrivateKey() } // NetworkCert returns the full TLS certificate information for this endpoint. func (e *Endpoints) NetworkCert() *localtls.CertInfo { e.mu.RLock() defer e.mu.RUnlock() return e.cert } // NetworkAddress returns the network address of the network endpoint, or an // empty string if there's no network endpoint. func (e *Endpoints) NetworkAddress() string { e.mu.RLock() defer e.mu.RUnlock() listener := e.listeners[network] if listener == nil { return "" } return listener.Addr().String() } // NetworkUpdateAddress updates the address for the network endpoint, shutting // it down and restarting it. func (e *Endpoints) NetworkUpdateAddress(address string) error { if address != "" { address = internalUtil.CanonicalNetworkAddress(address, ports.HTTPSDefaultPort) } oldAddress := e.NetworkAddress() if address == oldAddress { return nil } clusterAddress := e.clusterAddress() logger.Infof("Update network address") e.mu.Lock() defer e.mu.Unlock() // Close the previous socket _ = e.closeListener(network) // If turning off listening, we're done. if address == "" { return nil } // If the new address covers the cluster one, turn off the cluster // listener. if clusterAddress != "" && internalUtil.IsAddressCovered(clusterAddress, address) { _ = e.closeListener(cluster) } // Attempt to setup the new listening socket getListener := func(address string) (*net.Listener, error) { var err error var listener net.Listener for range 10 { // Ten retries over a second seems reasonable. listener, err = net.Listen("tcp", address) if err == nil { break } time.Sleep(100 * time.Millisecond) } if err != nil { return nil, fmt.Errorf("Cannot listen on network HTTPS socket %q: %w", address, err) } return &listener, nil } // Set up the listener listener, err := getListener(address) if err != nil { // Attempt to revert to the previous address listener, err1 := getListener(oldAddress) if err1 == nil { e.listeners[network] = listeners.NewFancyTLSListener(*listener, e.cert) e.serve(network) } return err } e.listeners[network] = listeners.NewFancyTLSListener(*listener, e.cert) e.serve(network) return nil } // NetworkUpdateCert updates the TLS keypair and CA used by the network // endpoint. // // If the network endpoint is active, in-flight requests will continue using // the old certificate, and only new requests will use the new one. func (e *Endpoints) NetworkUpdateCert(cert *localtls.CertInfo) { e.mu.Lock() defer e.mu.Unlock() e.cert = cert for _, listenerKey := range []kind{network, cluster, vmvsock, storageBuckets, metrics} { listener, found := e.listeners[listenerKey] if found { listener.(*listeners.FancyTLSListener).Config(cert) } } } // NetworkUpdateTrustedProxy updates the https trusted proxy used by the network endpoint. func (e *Endpoints) NetworkUpdateTrustedProxy(trustedProxy string) { var proxies []net.IP for _, p := range util.SplitNTrimSpace(trustedProxy, ",", -1, true) { proxyIP := net.ParseIP(p) if proxyIP == nil { continue } proxies = append(proxies, proxyIP) } e.mu.Lock() defer e.mu.Unlock() for _, kind := range []kind{network, cluster} { listener, ok := e.listeners[kind] if !ok || listener == nil { continue } listener.(*listeners.FancyTLSListener).TrustedProxy(proxies) } server, ok := e.servers[network] if ok && server != nil { server.ErrorLog = log.New(networkServerErrorLogWriter{proxies: proxies}, "", 0) } } // Create a new net.Listener bound to the tcp socket of the network endpoint. func networkCreateListener(address string, cert *localtls.CertInfo) (net.Listener, error) { // Listening on `tcp` network with address 0.0.0.0 will end up with listening // on both IPv4 and IPv6 interfaces. Pass `tcp4` to make it // work only on 0.0.0.0. https://go-review.googlesource.com/c/go/+/45771/ listenAddress := internalUtil.CanonicalNetworkAddress(address, ports.HTTPSDefaultPort) protocol := "tcp" if strings.HasPrefix(listenAddress, "0.0.0.0") { protocol = "tcp4" } listener, err := net.Listen(protocol, listenAddress) if err != nil { return nil, fmt.Errorf("Bind network address: %w", err) } return listeners.NewFancyTLSListener(listener, cert), nil } incus-7.0.0/internal/server/endpoints/network_test.go000066400000000000000000000063541517523235500230630ustar00rootroot00000000000000package endpoints_test import ( "fmt" "net" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/shared/tls/tlstest" ) // If no socket-based activation is detected, and a network address is set, a // new network TCP socket will be created. func TestEndpoints_NetworkCreateTCPSocket(t *testing.T) { endpoints, config, cleanup := newEndpoints(t) defer cleanup() config.NetworkAddress = "127.0.0.1:0" require.NoError(t, endpoints.Up(config)) assert.NoError(t, httpGetOverTLSSocket(endpoints.NetworkAddressAndCert())) } // It's possible to replace the TLS certificate used by the network endpoint. func TestEndpoints_NetworkUpdateCert(t *testing.T) { endpoints, config, cleanup := newEndpoints(t) defer cleanup() config.NetworkAddress = "127.0.0.1:0" require.NoError(t, endpoints.Up(config)) oldCert := config.Cert newCert := tlstest.TestingAltKeyPair(t) endpoints.NetworkUpdateCert(newCert) address := endpoints.NetworkAddress() assert.NoError(t, httpGetOverTLSSocket(address, newCert)) // The old cert does not work anymore assert.Error(t, httpGetOverTLSSocket(address, oldCert)) } // If socket-based activation is detected, it will be used for binding the API // Endpoints' unix socket. func TestEndpoints_NetworkSocketBasedActivation(t *testing.T) { endpoints, config, cleanup := newEndpoints(t) defer cleanup() listener := newTCPListener(t) defer func() { _ = listener.Close() }() file, err := listener.File() require.NoError(t, err) setupSocketBasedActivation(endpoints, file) require.NoError(t, endpoints.Up(config)) assertNoSocketBasedActivation(t) assert.NoError(t, httpGetOverTLSSocket(endpoints.NetworkAddressAndCert())) } // When the network address is updated, any previous network socket gets // closed. func TestEndpoints_NetworkUpdateAddress(t *testing.T) { endpoints, config, cleanup := newEndpoints(t) defer cleanup() config.NetworkAddress = "127.0.0.1:0" require.NoError(t, endpoints.Up(config)) // Use "localhost" instead of "127.0.0.1" just to make the address // different and actually trigger an endpoint change. require.NoError(t, endpoints.NetworkUpdateAddress("localhost:0")) assert.NoError(t, httpGetOverTLSSocket(endpoints.NetworkAddressAndCert())) } // Create a TCPListener using a random port. func newTCPListener(t *testing.T) *net.TCPListener { addr, err := net.ResolveTCPAddr("tcp", "localhost:0") require.NoError(t, err) listener, err := net.ListenTCP("tcp", addr) require.NoError(t, err) return listener } // Create IPv4 0.0.0.0 listener using random port // and ensure it is not accessible via IPv6 request. func TestEndpoints_NetworkCreateTCPSocketIPv4(t *testing.T) { endpoints, config, cleanup := newEndpoints(t) defer cleanup() config.NetworkAddress = "0.0.0.0:0" require.NoError(t, endpoints.Up(config)) address, certificate := endpoints.NetworkAddressAndCert() parts := strings.Split(address, ":") ipv6Address := fmt.Sprintf("[::1]:%s", parts[1]) ipv4Address := fmt.Sprintf("127.0.0.1:%s", parts[1]) // Check accessibility over IPv4 request assert.NoError(t, httpGetOverTLSSocket(ipv4Address, certificate)) // Check accessibility over IPv6 request assert.Error(t, httpGetOverTLSSocket(ipv6Address, certificate)) } incus-7.0.0/internal/server/endpoints/network_util.go000066400000000000000000000025451517523235500230570ustar00rootroot00000000000000package endpoints import ( "bytes" "net" "regexp" "github.com/lxc/incus/v7/shared/logger" ) type networkServerErrorLogWriter struct { proxies []net.IP } // Regex for the log we want to ignore. var unwantedLogRegex = regexp.MustCompile(`^http: TLS handshake error from ([^\[:]+?|\[([^\]]+?)\]):[0-9]+: .+: connection reset by peer$`) func (d networkServerErrorLogWriter) Write(p []byte) (int, error) { strippedLog := d.stripLog(p) if strippedLog == "" { return 0, nil } logger.Info(strippedLog) return len(p), nil } func (d networkServerErrorLogWriter) stripLog(p []byte) string { // Strip the beginning of the log until we reach "http:". for len(p) > 5 && string(p[0:5]) != "http:" { p = bytes.TrimLeftFunc(p, func(r rune) bool { return r != 'h' }) } // Strip the newline from the end. p = bytes.TrimRightFunc(p, func(r rune) bool { return r == '\n' }) // Get the source IP address. match := unwantedLogRegex.FindSubmatch(p) var sourceIP string if match != nil { if match[2] != nil { // Inner match omits parentheses of ipv6 address. sourceIP = string(match[2]) } else if match[1] != nil { sourceIP = string(match[1]) } } // Discard the log if the source is in our list of trusted proxies. if sourceIP != "" { for _, ip := range d.proxies { if ip.String() == sourceIP { return "" } } } return string(p) } incus-7.0.0/internal/server/endpoints/network_util_test.go000066400000000000000000000105151517523235500241120ustar00rootroot00000000000000package endpoints import ( "net" "testing" "github.com/stretchr/testify/assert" ) func Test_networkServerErrorLogWriter_shouldDiscard(t *testing.T) { tests := []struct { name string proxies []net.IP log []byte want string }{ { name: "ipv4 trusted proxy (write)", proxies: []net.IP{net.ParseIP("10.24.0.32")}, log: []byte("Sep 17 04:58:30 abydos incus.daemon[21884]: 2021/09/17 04:58:30 http: TLS handshake error from 10.24.0.32:55672: write tcp 10.24.0.22:8443->10.24.0.32:55672: write: connection reset by peer\n"), want: "", }, { name: "ipv4 non-trusted proxy (write)", proxies: []net.IP{net.ParseIP("10.24.0.33")}, log: []byte("Sep 17 04:58:30 abydos incus.daemon[21884]: 2021/09/17 04:58:30 http: TLS handshake error from 10.24.0.32:55672: write tcp 10.24.0.22:8443->10.24.0.32:55672: write: connection reset by peer\n"), want: "http: TLS handshake error from 10.24.0.32:55672: write tcp 10.24.0.22:8443->10.24.0.32:55672: write: connection reset by peer", }, { name: "ipv6 trusted proxy (write)", proxies: []net.IP{net.ParseIP("2602:fd23:8:1003:1266:6aff:fefa:7670")}, log: []byte("Sep 17 04:58:30 abydos incus.daemon[21884]: 2021/09/17 04:58:30 http: TLS handshake error from [2602:fd23:8:1003:1266:6aff:fefa:7670]:55672: write tcp [2602:fd23:8:101::100]:8443->[2602:fd23:8:1003:1266:6aff:fefa:7670]:55672: write: connection reset by peer\n"), want: "", }, { name: "ipv6 non-trusted proxy (write)", proxies: []net.IP{net.ParseIP("2602:fd23:8:1003:1266:6aff:fefa:7671")}, log: []byte("Sep 17 04:58:30 abydos incus.daemon[21884]: 2021/09/17 04:58:30 http: TLS handshake error from [2602:fd23:8:1003:1266:6aff:fefa:7670]:55672: write tcp [2602:fd23:8:101::100]:8443->[2602:fd23:8:1003:1266:6aff:fefa:7670]:55672: write: connection reset by peer\n"), want: "http: TLS handshake error from [2602:fd23:8:1003:1266:6aff:fefa:7670]:55672: write tcp [2602:fd23:8:101::100]:8443->[2602:fd23:8:1003:1266:6aff:fefa:7670]:55672: write: connection reset by peer", }, { name: "ipv4 trusted proxy (read)", proxies: []net.IP{net.ParseIP("10.24.0.32")}, log: []byte("Sep 17 04:58:30 abydos incus.daemon[21884]: 2021/09/17 04:58:30 http: TLS handshake error from 10.24.0.32:55672: read tcp 10.24.0.22:8443->10.24.0.32:55672: read: connection reset by peer\n"), want: "", }, { name: "ipv4 non-trusted proxy (read)", proxies: []net.IP{net.ParseIP("10.24.0.33")}, log: []byte("Sep 17 04:58:30 abydos incus.daemon[21884]: 2021/09/17 04:58:30 http: TLS handshake error from 10.24.0.32:55672: read tcp 10.24.0.22:8443->10.24.0.32:55672: read: connection reset by peer\n"), want: "http: TLS handshake error from 10.24.0.32:55672: read tcp 10.24.0.22:8443->10.24.0.32:55672: read: connection reset by peer", }, { name: "ipv6 trusted proxy (read)", proxies: []net.IP{net.ParseIP("2602:fd23:8:1003:1266:6aff:fefa:7670")}, log: []byte("Sep 17 04:58:30 abydos incus.daemon[21884]: 2021/09/17 04:58:30 http: TLS handshake error from [2602:fd23:8:1003:1266:6aff:fefa:7670]:55672: read tcp [2602:fd23:8:101::100]:8443->[2602:fd23:8:1003:1266:6aff:fefa:7670]:55672: read: connection reset by peer\n"), want: "", }, { name: "ipv6 non-trusted proxy (read)", proxies: []net.IP{net.ParseIP("2602:fd23:8:1003:1266:6aff:fefa:7671")}, log: []byte("Sep 17 04:58:30 abydos incus.daemon[21884]: 2021/09/17 04:58:30 http: TLS handshake error from [2602:fd23:8:1003:1266:6aff:fefa:7670]:55672: read tcp [2602:fd23:8:101::100]:8443->[2602:fd23:8:1003:1266:6aff:fefa:7670]:55672: read: connection reset by peer\n"), want: "http: TLS handshake error from [2602:fd23:8:1003:1266:6aff:fefa:7670]:55672: read tcp [2602:fd23:8:101::100]:8443->[2602:fd23:8:1003:1266:6aff:fefa:7670]:55672: read: connection reset by peer", }, { name: "unrelated", proxies: []net.IP{}, log: []byte("Sep 17 04:58:30 abydos incus.daemon[21884]: 2021/09/17 04:58:30 http: response.WriteHeader on hijacked connection from yourfunction (yourfile.go:80)\n"), want: "http: response.WriteHeader on hijacked connection from yourfunction (yourfile.go:80)", }, } for i, tt := range tests { t.Logf("Case %d: %s", i, tt.name) d := networkServerErrorLogWriter{ proxies: tt.proxies, } assert.Equal(t, tt.want, d.stripLog(tt.log)) } } incus-7.0.0/internal/server/endpoints/notlinux.go000066400000000000000000000005061517523235500222040ustar00rootroot00000000000000//go:build !linux || !cgo package endpoints import ( "errors" "net" ) func localCreateListener(path string, group string) (net.Listener, error) { return nil, errors.New("Platform isn't supported") } func createDevIncuslListener(path string) (net.Listener, error) { return nil, errors.New("Platform isn't supported") } incus-7.0.0/internal/server/endpoints/object.go000066400000000000000000000055561517523235500216040ustar00rootroot00000000000000package endpoints import ( "fmt" "net" "strings" "time" "github.com/lxc/incus/v7/internal/ports" "github.com/lxc/incus/v7/internal/server/endpoints/listeners" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" localtls "github.com/lxc/incus/v7/shared/tls" ) func storageBucketsCreateListener(address string, cert *localtls.CertInfo) (net.Listener, error) { // Listening on `tcp` network with address 0.0.0.0 will end up with listening // on both IPv4 and IPv6 interfaces. Pass `tcp4` to make it // work only on 0.0.0.0. https://go-review.googlesource.com/c/go/+/45771/ listenAddress := internalUtil.CanonicalNetworkAddress(address, ports.HTTPSStorageBucketsDefaultPort) protocol := "tcp" if strings.HasPrefix(listenAddress, "0.0.0.0") { protocol = "tcp4" } listener, err := net.Listen(protocol, listenAddress) if err != nil { return nil, fmt.Errorf("Bind network address: %w", err) } return listeners.NewFancyTLSListener(listener, cert), nil } // StorageBucketsAddress returns the network address of the storage buckets endpoint, or an // empty string if there's no storage buckets endpoint. func (e *Endpoints) StorageBucketsAddress() string { e.mu.RLock() defer e.mu.RUnlock() listener := e.listeners[storageBuckets] if listener == nil { return "" } return listener.Addr().String() } // StorageBucketsUpdateAddress updates the address for the storage buckets endpoint, shutting it down and // restarting it. func (e *Endpoints) StorageBucketsUpdateAddress(address string, cert *localtls.CertInfo) error { if address != "" { address = internalUtil.CanonicalNetworkAddress(address, ports.HTTPSStorageBucketsDefaultPort) } oldAddress := e.StorageBucketsAddress() if address == oldAddress { return nil } logger.Infof("Update storage buckets address") e.mu.Lock() defer e.mu.Unlock() // Close the previous socket _ = e.closeListener(storageBuckets) // If turning off listening, we're done if address == "" { return nil } // Attempt to setup the new listening socket getListener := func(address string) (*net.Listener, error) { var err error var listener net.Listener for range 10 { // Ten retries over a second seems reasonable. listener, err = storageBucketsCreateListener(address, cert) if err == nil { break } time.Sleep(100 * time.Millisecond) } if err != nil { return nil, fmt.Errorf("Cannot listen on http socket: %w", err) } return &listener, nil } // If setting a new address, setup the listener if address != "" { listener, err := getListener(address) if err != nil { // Attempt to revert to the previous address listener, err1 := getListener(oldAddress) if err1 == nil { e.listeners[storageBuckets] = *listener e.serve(storageBuckets) } return err } e.listeners[storageBuckets] = *listener e.serve(storageBuckets) } return nil } incus-7.0.0/internal/server/endpoints/pprof.go000066400000000000000000000044011517523235500214500ustar00rootroot00000000000000package endpoints import ( "fmt" "net" "net/http" _ "net/http/pprof" // pprof magic "time" "github.com/lxc/incus/v7/internal/ports" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" ) func pprofCreateServer() *http.Server { // Undo the magic that importing pprof does pprofMux := http.DefaultServeMux http.DefaultServeMux = http.NewServeMux() // Setup an http server srv := &http.Server{ Handler: pprofMux, } return srv } func pprofCreateListener(address string) (net.Listener, error) { return net.Listen("tcp", address) } // PprofAddress returns the network address of the pprof endpoint, or an empty string if there's no pprof endpoint. func (e *Endpoints) PprofAddress() string { e.mu.RLock() defer e.mu.RUnlock() listener := e.listeners[pprof] if listener == nil { return "" } return listener.Addr().String() } // PprofUpdateAddress updates the address for the pprof endpoint, shutting it down and restarting it. func (e *Endpoints) PprofUpdateAddress(address string) error { if address != "" { address = internalUtil.CanonicalNetworkAddress(address, ports.HTTPDebugDefaultPort) } oldAddress := e.NetworkAddress() if address == oldAddress { return nil } logger.Infof("Update pprof address") e.mu.Lock() defer e.mu.Unlock() // Close the previous socket _ = e.closeListener(pprof) // If turning off listening, we're done if address == "" { return nil } // Attempt to setup the new listening socket getListener := func(address string) (*net.Listener, error) { var err error var listener net.Listener for range 10 { // Ten retries over a second seems reasonable. listener, err = net.Listen("tcp", address) if err == nil { break } time.Sleep(100 * time.Millisecond) } if err != nil { return nil, fmt.Errorf("Cannot listen on http socket: %w", err) } return &listener, nil } // If setting a new address, setup the listener if address != "" { listener, err := getListener(address) if err != nil { // Attempt to revert to the previous address listener, err1 := getListener(oldAddress) if err1 == nil { e.listeners[pprof] = *listener e.serve(pprof) } return err } e.listeners[pprof] = *listener e.serve(pprof) } return nil } incus-7.0.0/internal/server/endpoints/socket.go000066400000000000000000000060741517523235500216220ustar00rootroot00000000000000//go:build linux && cgo package endpoints import ( "errors" "fmt" "net" "os" "os/exec" "os/user" "strconv" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) // Bind to the given unix socket path. func socketUnixListen(path string) (*net.UnixListener, error) { addr, err := net.ResolveUnixAddr("unix", path) if err != nil { return nil, fmt.Errorf("cannot resolve socket address: %w", err) } listener, err := net.ListenUnix("unix", addr) if err != nil { return nil, fmt.Errorf("cannot bind socket: %w", err) } return listener, err } // CheckAlreadyRunning checks if the socket at the given path is already // bound to a running process, and return an error if so. // // FIXME: We should probably rather just try a regular unix socket // connection without using the client. However this is the way // this logic has historically behaved, so let's keep it like it // was. func CheckAlreadyRunning(path string) error { // If socket activated, nothing to do pid, err := strconv.Atoi(os.Getenv("LISTEN_PID")) if err == nil { if pid == os.Getpid() { return nil } } // If there's no socket file at all, there's nothing to do. if !util.PathExists(path) { return nil } _, err = incus.ConnectIncusUnix(path, nil) // If the connection succeeded it means there's another daemon running. if err == nil { return errors.New("Incus is already running") } return nil } // Remove any stale socket file at the given path. func socketUnixRemoveStale(path string) error { // If there's no socket file at all, there's nothing to do. if !util.PathExists(path) { return nil } logger.Debugf("Detected stale unix socket, deleting") err := os.Remove(path) if err != nil { return fmt.Errorf("could not delete stale local socket: %w", err) } return nil } // Change the file mode of the given unix socket file,. func socketUnixSetPermissions(path string, mode os.FileMode) error { err := os.Chmod(path, mode) if err != nil { return fmt.Errorf("cannot set permissions on local socket: %w", err) } return nil } // Change the ownership of the given unix socket file,. func socketUnixSetOwnership(path string, groupName string) error { var gid int var err error if groupName != "" { g, err := user.LookupGroup(groupName) if err != nil { return fmt.Errorf("cannot get group ID of '%s': %w", groupName, err) } gid, err = strconv.Atoi(g.Gid) if err != nil { return err } } else { gid = os.Getgid() } err = os.Chown(path, os.Getuid(), gid) if err != nil { return fmt.Errorf("cannot change ownership on local socket: %w", err) } return nil } // Set the SELinux label on the socket. func socketUnixSetLabel(path string, label string) error { // Skip if no label requested. if label == "" { return nil } // Check if chcon is installed. _, err := exec.LookPath("chcon") if err != nil { return nil } // Attempt to apply (don't fail as kernel may not support it). _, _ = subprocess.RunCommand("chcon", label, path) return nil } incus-7.0.0/internal/server/endpoints/vsock.go000066400000000000000000000021771517523235500214570ustar00rootroot00000000000000package endpoints import ( "errors" "math" "math/rand" "net" "github.com/mdlayher/vsock" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/server/endpoints/listeners" localtls "github.com/lxc/incus/v7/shared/tls" ) func createVsockListener(cert *localtls.CertInfo) (net.Listener, error) { for range 10 { // Get random port between 1024 and 65535. port := 1024 + rand.Int31n(math.MaxUint16-1024) // Setup listener on host context ID for inbound connections from the agent running inside VMs. listener, err := vsock.ListenContextID(vsock.Host, uint32(port), nil) if err != nil { // Try a different port. if errors.Is(err, unix.EADDRINUSE) { continue } return nil, err } return listeners.NewFancyTLSListener(listener, cert), nil } return nil, errors.New("Failed finding free listen port for vsock listener") } // VsockAddress returns the network address of the vsock endpoint, or nil if there's no vsock endpoint. func (e *Endpoints) VsockAddress() net.Addr { e.mu.RLock() defer e.mu.RUnlock() listener := e.listeners[vmvsock] if listener == nil { return nil } return listener.Addr() } incus-7.0.0/internal/server/events/000077500000000000000000000000001517523235500172755ustar00rootroot00000000000000incus-7.0.0/internal/server/events/common.go000066400000000000000000000031501517523235500211130ustar00rootroot00000000000000package events import ( "context" "sync" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/cancel" "github.com/lxc/incus/v7/shared/logger" ) // EventHandler called when the connection receives an event from the client. type EventHandler func(event api.Event) // serverCommon represents an instance of a common event server. type serverCommon struct { debug bool verbose bool lock sync.Mutex } // listenerCommon describes a common event listener. type listenerCommon struct { EventListenerConnection messageTypes []string done *cancel.Canceller id string lock sync.Mutex recvFunc EventHandler } func (e *listenerCommon) start() { logger.Debug("Event listener server handler started", logger.Ctx{"id": e.id, "local": e.LocalAddr(), "remote": e.RemoteAddr()}) e.Reader(e.done.Context, e.recvFunc) e.Close() } // IsClosed returns true if the listener is closed. func (e *listenerCommon) IsClosed() bool { return e.done.Err() != nil } // ID returns the listener ID. func (e *listenerCommon) ID() string { return e.id } // Wait waits for a message on its active channel or the context is cancelled, then returns. func (e *listenerCommon) Wait(ctx context.Context) { select { case <-ctx.Done(): case <-e.done.Done(): } } // Close Disconnects the listener. func (e *listenerCommon) Close() { e.lock.Lock() defer e.lock.Unlock() if e.IsClosed() { return } logger.Debug("Event listener server handler stopped", logger.Ctx{"listener": e.ID(), "local": e.LocalAddr(), "remote": e.RemoteAddr()}) _ = e.EventListenerConnection.Close() e.done.Cancel() } incus-7.0.0/internal/server/events/connections.go000066400000000000000000000140631517523235500221520ustar00rootroot00000000000000package events import ( "context" "encoding/json" "errors" "fmt" "io" "net" "sync" "time" "github.com/gorilla/websocket" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) // EventListenerConnection represents an event listener connection. type EventListenerConnection interface { Reader(ctx context.Context, recvFunc EventHandler) WriteJSON(event any) error Close() error LocalAddr() net.Addr // Used for logging RemoteAddr() net.Addr // Used for logging } type websockListenerConnection struct { *websocket.Conn lock sync.Mutex pongsPending int } type streamListenerConnection struct { net.Conn lock sync.Mutex } type simpleListenerConnection struct { rwc io.ReadWriteCloser lock sync.Mutex } // NewWebsocketListenerConnection returns a new websocket listener connection. func NewWebsocketListenerConnection(connection *websocket.Conn) EventListenerConnection { return &websockListenerConnection{ Conn: connection, } } func (e *websockListenerConnection) Reader(ctx context.Context, recvFunc EventHandler) { ctx, cancelFunc := context.WithCancel(ctx) closeFunc := func() { e.lock.Lock() defer e.lock.Unlock() if ctx.Err() != nil { return } err := e.Close() if err != nil { logger.Warn("Failed closing connection", logger.Ctx{"err": err}) } cancelFunc() } defer closeFunc() pingInterval := time.Second * 10 e.pongsPending = 0 e.SetPongHandler(func(msg string) error { e.lock.Lock() e.pongsPending = 0 e.lock.Unlock() return nil }) // Start reader from client. go func() { defer closeFunc() if recvFunc != nil { for { var event api.Event err := e.Conn.ReadJSON(&event) if err != nil { return // This detects if client has disconnected or sent invalid data. } // Pass received event to the handler. recvFunc(event) } } else { // Run a blocking reader to detect if the client has disconnected. We don't expect to get // anything from the remote side, so this should remain blocked until disconnected. _, _, _ = e.Conn.NextReader() } }() t := time.NewTicker(pingInterval) defer t.Stop() for { if ctx.Err() != nil { return } e.lock.Lock() if e.pongsPending > 2 { e.lock.Unlock() return } err := e.WriteControl(websocket.PingMessage, []byte("keepalive"), time.Now().Add(5*time.Second)) if err != nil { e.lock.Unlock() return } e.pongsPending++ e.lock.Unlock() select { case <-t.C: case <-ctx.Done(): return } } } func (e *websockListenerConnection) WriteJSON(event any) error { e.lock.Lock() defer e.lock.Unlock() err := e.Conn.SetWriteDeadline(time.Now().Add(5 * time.Second)) if err != nil { return fmt.Errorf("Failed setting write deadline: %w", err) } return e.Conn.WriteJSON(event) } // NewStreamListenerConnection returns a new http stream listener connection. func NewStreamListenerConnection(connection net.Conn) (EventListenerConnection, error) { // Send HTTP response to let the client know what to expect. // This is only sent once, and is followed by events. // // The X-Content-Type-Options response HTTP header is a marker used by the server to indicate // that the MIME types advertised in the Content-Type headers should be followed and not be // changed. The header allows you to avoid MIME type sniffing by saying that the MIME types are // deliberately configured. _, err := io.WriteString(connection, `HTTP/1.1 200 OK Connection: keep-alive Content-Type: application/json X-Content-Type-Options: nosniff `) if err != nil { return nil, fmt.Errorf("Failed sending initial HTTP response: %w", err) } return &streamListenerConnection{ Conn: connection, }, nil } func (e *streamListenerConnection) Reader(ctx context.Context, recvFunc EventHandler) { ctx, cancelFunc := context.WithCancel(ctx) closeFunc := func() { e.lock.Lock() defer e.lock.Unlock() if ctx.Err() != nil { return } err := e.Close() if err != nil { logger.Warn("Failed closing connection", logger.Ctx{"err": err}) } cancelFunc() } defer closeFunc() // Start reader from client. go func() { defer closeFunc() buf := make([]byte, 1) // This is used to determine whether the client has terminated. _, err := e.Read(buf) if err != nil && errors.Is(err, io.EOF) { return } }() if ctx.Err() != nil { return } <-ctx.Done() } func (e *streamListenerConnection) WriteJSON(event any) error { e.lock.Lock() defer e.lock.Unlock() err := e.SetWriteDeadline(time.Now().Add(5 * (time.Second))) if err != nil { return fmt.Errorf("Failed setting write deadline: %w", err) } err = json.NewEncoder(e.Conn).Encode(event) if err != nil { return fmt.Errorf("Failed sending event: %w", err) } return nil } func (e *streamListenerConnection) Close() error { return e.Conn.Close() } // NewSimpleListenerConnection returns a new simple listener connection. func NewSimpleListenerConnection(rwc io.ReadWriteCloser) EventListenerConnection { return &simpleListenerConnection{ rwc: rwc, } } func (e *simpleListenerConnection) Reader(ctx context.Context, recvFunc EventHandler) { ctx, cancelFunc := context.WithCancel(ctx) closeFunc := func() { e.lock.Lock() defer e.lock.Unlock() if ctx.Err() != nil { return } err := e.Close() if err != nil { logger.Warn("Failed closing connection", logger.Ctx{"err": err}) } cancelFunc() } defer closeFunc() // Start reader from client. go func() { defer closeFunc() buf := make([]byte, 1) // This is used to determine whether the client has terminated. _, err := e.rwc.Read(buf) if err != nil && errors.Is(err, io.EOF) { return } }() if ctx.Err() != nil { return } <-ctx.Done() } func (e *simpleListenerConnection) WriteJSON(event any) error { err := json.NewEncoder(e.rwc).Encode(event) if err != nil { return err } return nil } func (e *simpleListenerConnection) Close() error { return e.rwc.Close() } func (e *simpleListenerConnection) LocalAddr() net.Addr { // Used for logging return nil } func (e *simpleListenerConnection) RemoteAddr() net.Addr { // Used for logging return nil } incus-7.0.0/internal/server/events/dev_incus_events.go000066400000000000000000000051001517523235500231630ustar00rootroot00000000000000package events import ( "context" "encoding/json" "fmt" "slices" "time" "github.com/google/uuid" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/cancel" ) // DevIncusServer represents an instance of an devIncus event server. type DevIncusServer struct { serverCommon listeners map[string]*DevIncusListener } // NewDevIncusServer returns a new devIncus event server. func NewDevIncusServer(debug bool, verbose bool) *DevIncusServer { server := &DevIncusServer{ serverCommon: serverCommon{ debug: debug, verbose: verbose, }, listeners: map[string]*DevIncusListener{}, } return server } // AddListener creates and returns a new event listener. func (s *DevIncusServer) AddListener(instanceID int, connection EventListenerConnection, messageTypes []string) (*DevIncusListener, error) { listener := &DevIncusListener{ listenerCommon: listenerCommon{ EventListenerConnection: connection, messageTypes: messageTypes, done: cancel.New(context.Background()), id: uuid.New().String(), }, instanceID: instanceID, } s.lock.Lock() defer s.lock.Unlock() if s.listeners[listener.id] != nil { return nil, fmt.Errorf("A listener with ID %q already exists", listener.id) } s.listeners[listener.id] = listener go listener.start() return listener, nil } // Send broadcasts a custom event. func (s *DevIncusServer) Send(instanceID int, eventType string, eventMessage any) error { encodedMessage, err := json.Marshal(eventMessage) if err != nil { return err } event := api.Event{ Type: eventType, Timestamp: time.Now(), Metadata: encodedMessage, } return s.broadcast(instanceID, event) } func (s *DevIncusServer) broadcast(instanceID int, event api.Event) error { s.lock.Lock() listeners := s.listeners for _, listener := range listeners { if !slices.Contains(listener.messageTypes, event.Type) { continue } if listener.instanceID != instanceID { continue } go func(listener *DevIncusListener, event api.Event) { // Check that the listener still exists if listener == nil { return } // Make sure we're not done already if listener.IsClosed() { return } err := listener.WriteJSON(event) if err != nil { // Remove the listener from the list s.lock.Lock() delete(s.listeners, listener.id) s.lock.Unlock() listener.Close() } }(listener, event) } s.lock.Unlock() return nil } // DevIncusListener describes a devIncus event listener. type DevIncusListener struct { listenerCommon instanceID int } incus-7.0.0/internal/server/events/events.go000066400000000000000000000150431517523235500211330ustar00rootroot00000000000000package events import ( "context" "encoding/json" "errors" "fmt" "slices" "time" "github.com/google/uuid" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/cancel" "github.com/lxc/incus/v7/shared/logger" ) // EventSource indicates the source of an event. type EventSource uint8 // EventSourceLocal indicates the event was generated locally. const EventSourceLocal = 0 // EventSourcePull indicates the event was received from an outbound event listener stream. const EventSourcePull = 1 // EventSourcePush indicates the event was received from an event listener client connected to us. const EventSourcePush = 2 // InjectFunc is used to inject an event received by a listener into the local events dispatcher. type InjectFunc func(event api.Event, eventSource EventSource) // NotifyFunc is called when an event is dispatched. type NotifyFunc func(event api.Event) // Server represents an instance of an event server. type Server struct { serverCommon listeners map[string]*Listener notify NotifyFunc location string } // NewServer returns a new event server. func NewServer(debug bool, verbose bool, notify NotifyFunc) *Server { server := &Server{ serverCommon: serverCommon{ debug: debug, verbose: verbose, }, listeners: map[string]*Listener{}, notify: notify, } return server } // SetLocalLocation sets the local location of this member. // This value will be added to the Location event field if not populated from another member. func (s *Server) SetLocalLocation(location string) { s.lock.Lock() defer s.lock.Unlock() s.location = location } // AddListener creates and returns a new event listener. func (s *Server) AddListener(projectName string, allProjects bool, projectPermissionFunc auth.PermissionChecker, connection EventListenerConnection, messageTypes []string, excludeSources []EventSource, recvFunc EventHandler, excludeLocations []string) (*Listener, error) { if allProjects && projectName != "" { return nil, errors.New("Cannot specify project name when listening for events on all projects") } if projectPermissionFunc == nil { projectPermissionFunc = func(auth.Object) bool { return true } } listener := &Listener{ listenerCommon: listenerCommon{ EventListenerConnection: connection, messageTypes: messageTypes, done: cancel.New(context.Background()), id: uuid.New().String(), recvFunc: recvFunc, }, allProjects: allProjects, projectName: projectName, projectPermissionFunc: projectPermissionFunc, excludeSources: excludeSources, excludeLocations: excludeLocations, } s.lock.Lock() defer s.lock.Unlock() if s.listeners[listener.id] != nil { return nil, fmt.Errorf("A listener with ID %q already exists", listener.id) } s.listeners[listener.id] = listener go listener.start() return listener, nil } // SendLifecycle broadcasts a lifecycle event. func (s *Server) SendLifecycle(projectName string, event api.EventLifecycle) { _ = s.Send(projectName, api.EventTypeLifecycle, event) } // Send broadcasts a custom event. func (s *Server) Send(projectName string, eventType string, eventMessage any) error { encodedMessage, err := json.Marshal(eventMessage) if err != nil { return err } event := api.Event{ Type: eventType, Timestamp: time.Now(), Metadata: encodedMessage, Project: projectName, } return s.broadcast(event, EventSourceLocal) } // Inject an event from another member into the local events dispatcher. // eventSource is used to indicate where this event was received from. func (s *Server) Inject(event api.Event, eventSource EventSource) { if event.Type == api.EventTypeLogging { // Parse the message logEntry := api.EventLogging{} err := json.Unmarshal(event.Metadata, &logEntry) if err != nil { return } if !s.debug && logEntry.Level == "debug" { return } if !s.debug && !s.verbose && logEntry.Level == "info" { return } } err := s.broadcast(event, eventSource) if err != nil { logger.Warn("Failed to forward event from member", logger.Ctx{"member": event.Location, "err": err}) } } func (s *Server) broadcast(event api.Event, eventSource EventSource) error { sourceInSlice := func(source EventSource, sources []EventSource) bool { return slices.Contains(sources, source) } s.lock.Lock() // Set the Location for local events to the local serverName if not already populated (do it here rather // than in Send as the lock to read s.location has been taken here already). if eventSource == EventSourceLocal && event.Location == "" { event.Location = s.location } // If a notification hook is present, then call it for locally produced events. // This can be used to send local events to another target (such as an event-hub member). if s.notify != nil && eventSource == EventSourceLocal { s.notify(event) } listeners := s.listeners for _, listener := range listeners { // If the event is project specific, check if the listener is requesting events from that project. if event.Project != "" && !listener.allProjects && event.Project != listener.projectName { continue } // If the event is project specific, ensure we have permission to view it. if event.Project != "" && !listener.projectPermissionFunc(auth.ObjectProject(event.Project)) { continue } if sourceInSlice(eventSource, listener.excludeSources) { continue } if !slices.Contains(listener.messageTypes, event.Type) { continue } // If the event doesn't come from this member and has been excluded by listener, don't deliver it. if eventSource != EventSourceLocal && slices.Contains(listener.excludeLocations, event.Location) { continue } go func(listener *Listener, event api.Event) { // Check that the listener still exists if listener == nil { return } // Make sure we're not done already if listener.IsClosed() { // Remove the listener from the list s.lock.Lock() delete(s.listeners, listener.id) s.lock.Unlock() return } err := listener.WriteJSON(event) if err != nil { // Remove the listener from the list s.lock.Lock() delete(s.listeners, listener.id) s.lock.Unlock() listener.Close() } }(listener, event) } s.lock.Unlock() return nil } // Listener describes an event listener. type Listener struct { listenerCommon allProjects bool projectName string projectPermissionFunc auth.PermissionChecker excludeSources []EventSource excludeLocations []string } incus-7.0.0/internal/server/events/internalListener.go000066400000000000000000000053621517523235500231540ustar00rootroot00000000000000package events import ( "context" "encoding/json" "sync" "github.com/lxc/incus/v7/internal/server/storage/memorypipe" "github.com/lxc/incus/v7/shared/api" ) // InternalListener represents a internal event listener. type InternalListener struct { handlers map[string]EventHandler listener *Listener server *Server ctx context.Context listenerCtx context.Context listenerCancel context.CancelFunc lock sync.Mutex wg sync.WaitGroup } // NewInternalListener returns an InternalListener. func NewInternalListener(ctx context.Context, server *Server) *InternalListener { return &InternalListener{ ctx: ctx, handlers: map[string]EventHandler{}, server: server, } } // startListener creates a new listener connection and listener. Also, it starts the gorountines // needed to notify any registered handlers about new events. func (l *InternalListener) startListener() { var err error l.listenerCtx, l.listenerCancel = context.WithCancel(l.ctx) aEnd, bEnd := memorypipe.NewPipePair(l.listenerCtx) listenerConnection := NewSimpleListenerConnection(aEnd) l.listener, err = l.server.AddListener("", true, nil, listenerConnection, []string{"lifecycle", "logging", "network-acl"}, []EventSource{EventSourcePull}, nil, nil) if err != nil { return } go func(ctx context.Context) { l.listener.Wait(ctx) l.listener.Close() l.listener = nil }(l.listenerCtx) l.wg.Add(1) go func(ctx context.Context, handlers map[string]EventHandler) { defer l.wg.Done() for { select { case <-ctx.Done(): return default: var event api.Event _ = json.NewDecoder(bEnd).Decode(&event) for _, handler := range handlers { if handler == nil { continue } go handler(event) } } } }(l.listenerCtx, l.handlers) } // stopListener cancels the context thus stopping the listener. func (l *InternalListener) stopListener() { if l.listenerCancel != nil { l.listenerCancel() l.wg.Wait() } } // AddHandler adds a new event handler. func (l *InternalListener) AddHandler(name string, handler EventHandler) { l.lock.Lock() defer l.lock.Unlock() if handler == nil { return } // Add handler to the list of handlers. l.handlers[name] = handler if l.listener == nil { // Create a listener if necessary. This avoids having a listener around if there are no handlers. l.startListener() } } // RemoveHandler removes the event handler with the given name. func (l *InternalListener) RemoveHandler(name string) { l.lock.Lock() defer l.lock.Unlock() for handlerName := range l.handlers { if handlerName == name { delete(l.handlers, name) break } } if len(l.handlers) == 0 { // Stop listener to avoid unnecessary goroutines. l.stopListener() } } incus-7.0.0/internal/server/events/logging.go000066400000000000000000000017421517523235500212560ustar00rootroot00000000000000package events import ( "fmt" "github.com/sirupsen/logrus" "github.com/lxc/incus/v7/shared/api" ) // LoggingServer controls what server to use for messages coming from the logger. var LoggingServer *Server // Handler describes an event handler. type Handler struct{} // NewEventHandler creates and returns a new event handler. func NewEventHandler() logrus.Hook { return &Handler{} } // Fire sends a new logging event. func (h Handler) Fire(entry *logrus.Entry) error { if LoggingServer == nil { return nil } return LoggingServer.Send("", api.EventTypeLogging, api.EventLogging{ Message: entry.Message, Level: entry.Level.String(), Context: logContextMap(entry.Data), }) } // Levels returns the list of supported log levels. func (h Handler) Levels() []logrus.Level { return logrus.AllLevels } func logContextMap(ctx logrus.Fields) map[string]string { ctxMap := map[string]string{} for k, v := range ctx { ctxMap[k] = fmt.Sprintf("%v", v) } return ctxMap } incus-7.0.0/internal/server/firewall/000077500000000000000000000000001517523235500175765ustar00rootroot00000000000000incus-7.0.0/internal/server/firewall/drivers/000077500000000000000000000000001517523235500212545ustar00rootroot00000000000000incus-7.0.0/internal/server/firewall/drivers/drivers_consts.go000066400000000000000000000045721517523235500246620ustar00rootroot00000000000000package drivers import ( "net" ) // FeatureOpts specify how firewall features are setup. type FeatureOpts struct { ICMPDHCPDNSAccess bool // Add rules to allow ICMP, DHCP and DNS access. ForwardingAllow bool // Add rules to allow IP forwarding. Blocked if false. } // SNATOpts specify how SNAT rules are setup. type SNATOpts struct { Append bool // Append rules (has no effect if driver doesn't support it). Subnet *net.IPNet // Subnet of source network used to identify candidate traffic. SNATAddress net.IP // SNAT IP address to use. If nil then MASQUERADE is used. } // Opts for setting up the firewall. type Opts struct { FeaturesV4 *FeatureOpts // Enable IPv4 firewall with specified options. Off if not provided. FeaturesV6 *FeatureOpts // Enable IPv6 firewall with specified options. Off if not provided. SNATV4 *SNATOpts // Enable IPv4 SNAT with specified options. Off if not provided. SNATV6 *SNATOpts // Enable IPv6 SNAT with specified options. Off if not provided. ACL bool // Enable ACL during setup. AddressSet bool // Enable address sets, only for netfilter. } // ACLRule represents an ACL rule that can be added to a firewall. type ACLRule struct { Direction string // Either "ingress" or "egress. Action string Log bool // Whether or not to log matched packets. LogName string // Log label name (requires Log be true). Source string Destination string Protocol string SourcePort string DestinationPort string ICMPType string ICMPCode string } // AddressForward represents a NAT address forward. type AddressForward struct { ListenAddress net.IP TargetAddress net.IP Protocol string ListenPorts []uint64 TargetPorts []uint64 SNAT bool } // AddressSet represent an address set. type AddressSet struct { Name string Addresses []string } // NftListSetsOutput structure to read JSON output of set listing. type NftListSetsOutput struct { Nftables []NftListSetsEntry `json:"nftables"` } // NftListSetsEntry structure to read JSON output of nft set listing. type NftListSetsEntry struct { Set *NftSet `json:"set,omitempty"` } // NftSet structure to parse the JSON of a set returned by nft -j list sets. type NftSet struct { Family string `json:"family"` Name string `json:"name"` Table string `json:"table"` } incus-7.0.0/internal/server/firewall/drivers/drivers_nftables.go000066400000000000000000001471461517523235500251540ustar00rootroot00000000000000package drivers import ( "context" "encoding/hex" "encoding/json" "errors" "fmt" "net" "os/exec" "slices" "strings" "text/template" "github.com/google/uuid" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) const ( nftablesNamespace = "incus" nftablesContentTemplate = "nftablesContent" ) // nftablesChainSeparator The "." character is specifically chosen here so as to prevent the ability for collisions // between project prefix (which is empty if project is default) and device name combinations that both are allowed // to contain underscores (where as instance name is not). const nftablesChainSeparator = "." // Nftables is an implementation of Incus firewall using nftables. type Nftables struct{} // String returns the driver name. func (d Nftables) String() string { return "nftables" } // Compat returns whether the driver backend is in use, and any host compatibility errors. func (d Nftables) Compat() (bool, error) { // Check if nftables nft command exists. _, err := exec.LookPath("nft") if err != nil { return false, fmt.Errorf("Backend command %q missing", "nft") } // Check that nftables works at all (some kernels let you list ruleset despite missing support). testTable := fmt.Sprintf("incus_test_%s", uuid.New().String()) _, err = subprocess.RunCommandCLocale("nft", "create", "table", testTable) if err != nil { return false, fmt.Errorf("Failed to create a test table: %w", err) } _, err = subprocess.RunCommandCLocale("nft", "delete", "table", testTable) if err != nil { return false, fmt.Errorf("Failed to delete a test table: %w", err) } // Check whether in use by parsing ruleset and looking for existing rules. ruleset, err := d.nftParseRuleset() if err != nil { return false, fmt.Errorf("Failed parsing nftables existing ruleset: %w", err) } for _, item := range ruleset { if item.ItemType == "rule" { return true, nil // At least one rule found indicates in use. } } return false, nil } // nftGenericItem represents some common fields amongst the different nftables types. type nftGenericItem struct { ItemType string `json:"-"` // Type of item (table, chain or rule). Populated by Incus. Family string `json:"family"` // Family of item (ip, ip6, bridge etc). Table string `json:"table"` // Table the item belongs to (for chains and rules). Chain string `json:"chain"` // Chain the item belongs to (for rules). Name string `json:"name"` // Name of item (for tables and chains). } // nftParseRuleset parses the ruleset and returns the generic parts as a slice of items. func (d Nftables) nftParseRuleset() ([]nftGenericItem, error) { // Dump ruleset as JSON. Use -nn flags to avoid doing DNS lookups of IPs mentioned in any rules. cmd := exec.Command("nft", "--json", "-nn", "list", "ruleset") stdout, err := cmd.StdoutPipe() if err != nil { return nil, err } err = cmd.Start() if err != nil { return nil, err } defer func() { _ = cmd.Wait() }() // This only extracts certain generic parts of the ruleset, see man libnftables-json for more info. v := &struct { Nftables []map[string]nftGenericItem `json:"nftables"` }{} err = json.NewDecoder(stdout).Decode(v) if err != nil { return nil, err } items := []nftGenericItem{} for _, item := range v.Nftables { rule, foundRule := item["rule"] chain, foundChain := item["chain"] table, foundTable := item["table"] if foundRule { rule.ItemType = "rule" items = append(items, rule) } else if foundChain { chain.ItemType = "chain" items = append(items, chain) } else if foundTable { table.ItemType = "table" items = append(items, table) } } err = cmd.Wait() if err != nil { return nil, err } return items, nil } // networkSetupForwardingPolicy allows forwarding dependent on boolean argument. func (d Nftables) networkSetupForwardingPolicy(networkName string, ip4Allow *bool, ip6Allow *bool) error { tplFields := map[string]any{ "namespace": nftablesNamespace, "chainSeparator": nftablesChainSeparator, "networkName": networkName, "family": "inet", } if ip4Allow != nil { ip4Action := "reject" if *ip4Allow { ip4Action = "accept" } tplFields["ip4Action"] = ip4Action } if ip6Allow != nil { ip6Action := "reject" if *ip6Allow { ip6Action = "accept" } tplFields["ip6Action"] = ip6Action } err := d.applyNftConfig(nftablesNetForwardingPolicy, tplFields) if err != nil { return fmt.Errorf("Failed adding forwarding policy rules for network %q (%s): %w", networkName, tplFields["family"], err) } return nil } // networkSetupOutboundNAT configures outbound NAT. // If srcIP is non-nil then SNAT is used with the specified address, otherwise MASQUERADE mode is used. // Append mode is always on and so the append argument is ignored. func (d Nftables) networkSetupOutboundNAT(networkName string, SNATV4 *SNATOpts, SNATV6 *SNATOpts) error { rules := make(map[string]*SNATOpts) tplFields := map[string]any{ "namespace": nftablesNamespace, "chainSeparator": nftablesChainSeparator, "networkName": networkName, "family": "inet", } // If SNAT IP not supplied then use the IP of the outbound interface (MASQUERADE). if SNATV4 != nil { rules["ip"] = SNATV4 } if SNATV6 != nil { rules["ip6"] = SNATV6 } tplFields["rules"] = rules err := d.applyNftConfig(nftablesNetOutboundNAT, tplFields) if err != nil { return fmt.Errorf("Failed adding outbound NAT rules for network %q (%s): %w", networkName, tplFields["family"], err) } return nil } // networkSetupICMPDHCPDNSAccess sets up basic nftables overrides for ICMP, DHCP and DNS. func (d Nftables) networkSetupICMPDHCPDNSAccess(networkName string, ipVersions []uint) error { ipFamilies := []string{} for _, ipVersion := range ipVersions { switch ipVersion { case 4: ipFamilies = append(ipFamilies, "ip") case 6: ipFamilies = append(ipFamilies, "ip6") } } tplFields := map[string]any{ "namespace": nftablesNamespace, "chainSeparator": nftablesChainSeparator, "networkName": networkName, "family": "inet", "ipFamilies": ipFamilies, } err := d.applyNftConfig(nftablesNetICMPDHCPDNS, tplFields) if err != nil { return fmt.Errorf("Failed adding ICMP, DHCP and DNS access rules for network %q (%s): %w", networkName, tplFields["family"], err) } return nil } func (d Nftables) networkSetupACLChainAndJumpRules(networkName string) error { tplFields := map[string]any{ "namespace": nftablesNamespace, "chainSeparator": nftablesChainSeparator, "networkName": networkName, "family": "inet", } config := &strings.Builder{} err := nftablesNetACLSetup.Execute(config, tplFields) if err != nil { return fmt.Errorf("Failed running %q template: %w", nftablesNetACLSetup.Name(), err) } err = subprocess.RunCommandWithFds(context.TODO(), strings.NewReader(config.String()), nil, "nft", "-f", "-") if err != nil { return err } return nil } // NetworkSetup configure network firewall. func (d Nftables) NetworkSetup(networkName string, opts Opts) error { // Do this first before adding other network rules, so jump to ACL rules come first. if opts.ACL { err := d.networkSetupACLChainAndJumpRules(networkName) if err != nil { return err } } if opts.SNATV4 != nil || opts.SNATV6 != nil { err := d.networkSetupOutboundNAT(networkName, opts.SNATV4, opts.SNATV6) if err != nil { return err } } dhcpDNSAccess := []uint{} var ip4ForwardingAllow, ip6ForwardingAllow *bool if opts.FeaturesV4 != nil || opts.FeaturesV6 != nil { if opts.FeaturesV4 != nil { if opts.FeaturesV4.ICMPDHCPDNSAccess { dhcpDNSAccess = append(dhcpDNSAccess, 4) } ip4ForwardingAllow = &opts.FeaturesV4.ForwardingAllow } if opts.FeaturesV6 != nil { if opts.FeaturesV6.ICMPDHCPDNSAccess { dhcpDNSAccess = append(dhcpDNSAccess, 6) } ip6ForwardingAllow = &opts.FeaturesV6.ForwardingAllow } err := d.networkSetupForwardingPolicy(networkName, ip4ForwardingAllow, ip6ForwardingAllow) if err != nil { return err } err = d.networkSetupICMPDHCPDNSAccess(networkName, dhcpDNSAccess) if err != nil { return err } } return nil } // NetworkClear removes the Incus network related chains and address sets. // The delete and ipeVersions arguments have no effect for nftables driver. func (d Nftables) NetworkClear(networkName string, _ bool, _ []uint) error { removeChains := []string{ "fwd", "pstrt", "in", "out", // Chains used for network operation rules. "aclin", "aclout", "aclfwd", "acl", // Chains used by ACL rules. "fwdprert", "fwdout", "fwdpstrt", // Chains used by Address Forward rules. "egress", // Chains added for limits.priority option } // Remove chains created by network rules. // Remove from ip and ip6 tables to ensure cleanup for instances started before we moved to inet table err := d.removeChains([]string{"inet", "ip", "ip6", "netdev"}, networkName, removeChains...) if err != nil { return fmt.Errorf("Failed clearing nftables rules for network %q: %w", networkName, err) } // Attempt to delete our address sets. // This will fail so long as there are still rules referencing them (other networks). _ = d.RemoveIncusAddressSets("bridge") return nil } // instanceDeviceLabel returns the unique label used for instance device chains. func (d Nftables) instanceDeviceLabel(projectName, instanceName, deviceName string) string { return fmt.Sprintf("%s%s%s", project.Instance(projectName, instanceName), nftablesChainSeparator, deviceName) } // InstanceSetupBridgeFilter sets up the filter rules to apply bridged device IP filtering. func (d Nftables) InstanceSetupBridgeFilter(projectName string, instanceName string, deviceName string, parentName string, hostName string, hwAddr string, IPv4Nets []*net.IPNet, IPv6Nets []*net.IPNet, IPv4DNS []string, IPv6DNS []string, parentManaged bool, macFiltering bool, aclRules []ACLRule) error { deviceLabel := d.instanceDeviceLabel(projectName, instanceName, deviceName) mac, err := net.ParseMAC(hwAddr) if err != nil { return err } tplFields := map[string]any{ "namespace": nftablesNamespace, "chainSeparator": nftablesChainSeparator, "family": "bridge", "deviceLabel": deviceLabel, "parentName": parentName, "hostName": hostName, "hwAddr": hwAddr, "hwAddrHex": fmt.Sprintf("0x%s", hex.EncodeToString(mac)), } if macFiltering { tplFields["macFiltering"] = true } // Filter unwanted ethernet frames when using IP filtering. if len(IPv4Nets)+len(IPv6Nets) > 0 { tplFields["filterUnwantedFrames"] = true tplFields["macFiltering"] = true } if IPv4Nets != nil && len(IPv4Nets) == 0 { tplFields["ipv4FilterAll"] = true tplFields["macFiltering"] = true } ipv4Nets := make([]string, 0, len(IPv4Nets)) for _, ipv4Net := range IPv4Nets { ipv4Nets = append(ipv4Nets, ipv4Net.String()) } if IPv6Nets != nil && len(IPv6Nets) == 0 { tplFields["ipv6FilterAll"] = true tplFields["macFiltering"] = true } ipv6NetsList := make([]string, 0, len(IPv6Nets)) ipv6NetsPrefixList := make([]string, 0, len(IPv6Nets)) for _, ipv6Net := range IPv6Nets { ones, _ := ipv6Net.Mask.Size() prefix, err := subnetPrefixHex(ipv6Net) if err != nil { return err } ipv6NetsList = append(ipv6NetsList, ipv6Net.String()) ipv6NetsPrefixList = append(ipv6NetsPrefixList, fmt.Sprintf("@nh,384,%d != 0x%s", ones, prefix)) } tplFields["ipv4NetsList"] = strings.Join(ipv4Nets, ", ") tplFields["ipv6NetsList"] = strings.Join(ipv6NetsList, ", ") tplFields["ipv6NetsPrefixList"] = strings.Join(ipv6NetsPrefixList, " ") // Process the assigned ACL rules and convert them to NFT rules nftRules, err := d.aclRulesToNftRules(hostName, aclRules) if err != nil { return fmt.Errorf("Failed generating bridge ACL rules for instance device %q (%s): %w", deviceLabel, tplFields["family"], err) } // Set the template fields for the ACL rules. tplFields["aclInDropRules"] = nftRules.inDropRules tplFields["aclInRejectRules"] = nftRules.inRejectRules tplFields["aclInRejectRulesConverted"] = nftRules.inRejectRulesConverted tplFields["aclInAcceptRules"] = append(nftRules.inAcceptRules4, nftRules.inAcceptRules6...) tplFields["aclInDefaultRule"] = nftRules.defaultInRule tplFields["aclInDefaultRuleConverted"] = nftRules.defaultInRuleConverted tplFields["aclOutDropRules"] = nftRules.outDropRules tplFields["aclOutAcceptRules"] = nftRules.outAcceptRules tplFields["aclOutDefaultRule"] = nftRules.defaultOutRule // Required for basic connectivity tplFields["dnsIPv4"] = IPv4DNS tplFields["dnsIPv6"] = IPv6DNS err = d.applyNftConfig(nftablesInstanceBridgeFilter, tplFields) if err != nil { return fmt.Errorf("Failed adding bridge filter rules for instance device %q (%s): %w", deviceLabel, tplFields["family"], err) } return nil } // InstanceClearBridgeFilter removes any filter rules that were added to apply bridged device IP filtering. func (d Nftables) InstanceClearBridgeFilter(projectName string, instanceName string, deviceName string, parentName string, hostName string, hwAddr string, _ []*net.IPNet, _ []*net.IPNet) error { deviceLabel := d.instanceDeviceLabel(projectName, instanceName, deviceName) // Remove chains created by bridge filter rules. err := d.removeChains([]string{"bridge"}, deviceLabel, "in", "fwd", "out") if err != nil { return fmt.Errorf("Failed clearing bridge filter rules for instance device %q: %w", deviceLabel, err) } return nil } // InstanceSetupProxyNAT creates DNAT rules for proxy devices. func (d Nftables) InstanceSetupProxyNAT(projectName string, instanceName string, deviceName string, forward *AddressForward) error { if forward.ListenAddress == nil { return errors.New("Listen address is required") } if forward.TargetAddress == nil { return errors.New("Target address is required") } listenPortsLen := len(forward.ListenPorts) if listenPortsLen <= 0 { return errors.New("At least 1 listen port must be supplied") } // If multiple target ports supplied, check they match the listen port(s) count. targetPortsLen := len(forward.TargetPorts) if targetPortsLen != 1 && targetPortsLen != listenPortsLen { return errors.New("Mismatch between listen port(s) and target port(s) count") } ipFamily := "ip" if forward.ListenAddress.To4() == nil { ipFamily = "ip6" } listenAddressStr := forward.ListenAddress.String() targetAddressStr := forward.TargetAddress.String() // Generate slices of rules to add. var dnatRules []map[string]any var snatRules []map[string]any targetPortRanges := portRangesFromSlice(forward.TargetPorts) for _, targetPortRange := range targetPortRanges { targetPortRangeStr := portRangeStr(targetPortRange, "-") snatRules = append(snatRules, map[string]any{ "ipFamily": ipFamily, "protocol": forward.Protocol, "targetHost": targetAddressStr, "targetPorts": targetPortRangeStr, }) } dnatRanges := getOptimisedDNATRanges(forward) for listenPortRange, targetPortRange := range dnatRanges { // Format the destination host/port as appropriate targetDest := targetAddressStr if targetPortRange[1] == 1 { targetPortStr := portRangeStr(targetPortRange, ":") targetDest = fmt.Sprintf("%s:%s", targetAddressStr, targetPortStr) if ipFamily == "ip6" { targetDest = fmt.Sprintf("[%s]:%s", targetAddressStr, targetPortStr) } } dnatRules = append(dnatRules, map[string]any{ "ipFamily": ipFamily, "protocol": forward.Protocol, "listenAddress": listenAddressStr, "listenPorts": portRangeStr(listenPortRange, "-"), "targetDest": targetDest, }) } deviceLabel := d.instanceDeviceLabel(projectName, instanceName, deviceName) tplFields := map[string]any{ "namespace": nftablesNamespace, "chainSeparator": nftablesChainSeparator, "chainPrefix": "", // Empty prefix for backwards compatibility with existing device chains. "family": "inet", "label": deviceLabel, "dnatRules": dnatRules, "snatRules": snatRules, } config := &strings.Builder{} err := nftablesNetProxyNAT.Execute(config, tplFields) if err != nil { return fmt.Errorf("Failed running %q template: %w", nftablesNetProxyNAT.Name(), err) } err = subprocess.RunCommandWithFds(context.TODO(), strings.NewReader(config.String()), nil, "nft", "-f", "-") if err != nil { return err } return nil } // InstanceClearProxyNAT remove DNAT rules for proxy devices. func (d Nftables) InstanceClearProxyNAT(projectName string, instanceName string, deviceName string) error { deviceLabel := d.instanceDeviceLabel(projectName, instanceName, deviceName) // Remove from ip and ip6 tables to ensure cleanup for instances started before we moved to inet table. err := d.removeChains([]string{"inet", "ip", "ip6"}, deviceLabel, "out", "prert", "pstrt") if err != nil { return fmt.Errorf("Failed clearing proxy rules for instance device %q: %w", deviceLabel, err) } return nil } // nftRulesCollection contains the ACL rules translated to NFT rules and split in groups. type nftRulesCollection struct { inDropRules []string inRejectRules []string inRejectRulesConverted []string inAcceptRules4 []string inAcceptRules6 []string outDropRules []string outAcceptRules []string defaultInRule string defaultInRuleConverted string defaultOutRule string } // aclRulesToNftRules converts ACL rules applied to the device to NFT rules. func (d Nftables) aclRulesToNftRules(hostName string, aclRules []ACLRule) (*nftRulesCollection, error) { nftRules := nftRulesCollection{ inDropRules: make([]string, 0), inRejectRules: make([]string, 0), inRejectRulesConverted: make([]string, 0), // To be used in the forward chain where reject is not supported inAcceptRules4: make([]string, 0), inAcceptRules6: make([]string, 0), outDropRules: make([]string, 0), outAcceptRules: make([]string, 0), defaultInRule: "", defaultInRuleConverted: "", // To be used in the forward chain where reject is not supported defaultOutRule: "", } hostNameQuoted := "\"" + hostName + "\"" rulesCount := len(aclRules) for i, rule := range aclRules { if i >= rulesCount-2 { // The last two rules are the default ACL rules and we should keep them separate. // As aclRuleCriteriaToRules return a set of rules instead of a rule to manage address sets in source / dests. // We use rules[0] for default rules because those rules do not use address sets and will be in one fragment. var partial bool var err error var defaultRules []string if rule.Direction == "egress" { defaultRules, partial, err = d.aclRuleCriteriaToRules(hostNameQuoted, 4, &rule) if len(defaultRules) > 1 { return nil, fmt.Errorf("Default rules slice has invalid len: %d", len(defaultRules)) } nftRules.defaultInRule = defaultRules[0] if err == nil && !partial && rule.Action == "reject" { // Convert egress reject rules to drop rules to address nftables limitation. rule.Action = "drop" defaultRules, partial, err = d.aclRuleCriteriaToRules(hostNameQuoted, 4, &rule) if len(defaultRules) > 1 { return nil, fmt.Errorf("Default rules slice has invalid len: %d", len(defaultRules)) } nftRules.defaultInRuleConverted = defaultRules[0] } else { nftRules.defaultInRuleConverted = nftRules.defaultInRule } } else { if rule.Action == "reject" { // Always convert ingress reject rules to drop rules to address nftables limitation. rule.Action = "drop" } defaultRules, partial, err = d.aclRuleCriteriaToRules(hostNameQuoted, 4, &rule) if len(defaultRules) > 1 { return nil, fmt.Errorf("Default rules slice has invalid len: %d", len(defaultRules)) } nftRules.defaultOutRule = defaultRules[0] } if err != nil { return nil, err } if partial { return nil, errors.New("Invalid default rule generated") } continue } if rule.Direction == "ingress" && rule.Action == "reject" { // Convert ingress reject rules to drop rules to address nftables limitation. rule.Action = "drop" } nft4Rules, nft6Rules, newNftRules, err := d.aclRuleToNftRules(hostNameQuoted, rule) if err != nil { return nil, err } switch rule.Direction { case "ingress": switch rule.Action { case "drop": nftRules.outDropRules = append(nftRules.outDropRules, newNftRules...) case "reject": nftRules.outDropRules = append(nftRules.outDropRules, newNftRules...) case "allow": nftRules.outAcceptRules = append(nftRules.outAcceptRules, newNftRules...) default: return nil, fmt.Errorf("Unrecognised action %q", rule.Action) } case "egress": switch rule.Action { case "drop": nftRules.inDropRules = append(nftRules.inDropRules, newNftRules...) case "reject": nftRules.inRejectRules = append(nftRules.inRejectRules, newNftRules...) // Generate reject rule converted to a drop rule. rule.Action = "drop" _, _, newNftRules, err = d.aclRuleToNftRules(hostNameQuoted, rule) if err != nil { return nil, err } nftRules.inRejectRulesConverted = append(nftRules.inRejectRulesConverted, newNftRules...) case "allow": if len(nft4Rules) != 0 { nftRules.inAcceptRules4 = append(nftRules.inAcceptRules4, nft4Rules...) } if len(nft6Rules) != 0 { nftRules.inAcceptRules6 = append(nftRules.inAcceptRules6, nft6Rules...) } default: return nil, fmt.Errorf("Unrecognised action %q", rule.Action) } default: return nil, fmt.Errorf("Unrecognised direction %q", rule.Direction) } } return &nftRules, nil } func (d Nftables) aclRuleToNftRules(hostNameQuoted string, rule ACLRule) ([]string, []string, []string, error) { nft6Rules := []string{} // First try generating rules with IPv4 or IP agnostic criteria. nft4Rules, partial, err := d.aclRuleCriteriaToRules(hostNameQuoted, 4, &rule) if err != nil { return nil, nil, nil, err } if partial { // If we couldn't fully generate the ruleset with only IPv4 or IP agnostic criteria, then // fill in the remaining parts using IPv6 criteria. nft6Rules, _, err = d.aclRuleCriteriaToRules(hostNameQuoted, 6, &rule) if err != nil { return nil, nil, nil, err } if len(nft6Rules) == 0 { return nil, nil, nil, errors.New("Invalid empty rule generated") } } else if len(nft4Rules) == 0 { return nil, nil, nil, errors.New("Invalid empty rule generated") } nftRules := []string{} if len(nft4Rules) != 0 { nftRules = append(nftRules, nft4Rules...) } if len(nft6Rules) != 0 { nftRules = append(nftRules, nft6Rules...) } return nft4Rules, nft6Rules, nftRules, nil } // applyNftConfig loads the specified config template and then applies it to the common template before sending to // the nft command to be atomically applied to the system. func (d Nftables) applyNftConfig(tpl *template.Template, tplFields map[string]any) error { // Load the specified template into the common template's parse tree under the nftableContentTemplate // name so that the nftableContentTemplate template can use it with the generic name. _, err := nftablesCommonTable.AddParseTree(nftablesContentTemplate, tpl.Tree) if err != nil { return fmt.Errorf("Failed loading %q template: %w", tpl.Name(), err) } config := &strings.Builder{} err = nftablesCommonTable.Execute(config, tplFields) if err != nil { return fmt.Errorf("Failed running %q template: %w", tpl.Name(), err) } err = subprocess.RunCommandWithFds(context.TODO(), strings.NewReader(config.String()), nil, "nft", "-f", "-") if err != nil { return fmt.Errorf("Failed apply nftables config: %w", err) } return nil } // removeChains removes the specified chains from the specified families. // If not empty, chain suffix is appended to each chain name, separated with "_". func (d Nftables) removeChains(families []string, chainSuffix string, chains ...string) error { ruleset, err := d.nftParseRuleset() if err != nil { return err } fullChains := chains if chainSuffix != "" { fullChains = make([]string, 0, len(chains)) for _, chain := range chains { fullChains = append(fullChains, fmt.Sprintf("%s%s%s", chain, nftablesChainSeparator, chainSuffix)) } } // Search ruleset for chains we are looking for. foundChains := make(map[string]nftGenericItem) for _, family := range families { for _, item := range ruleset { if item.ItemType == "chain" && item.Family == family && item.Table == nftablesNamespace && slices.Contains(fullChains, item.Name) { foundChains[item.Name] = item } } } // Delete the chains in the order specified in chains slice (to avoid dependency issues). for _, fullChain := range fullChains { item, found := foundChains[fullChain] if !found { continue } _, err = subprocess.RunCommand("nft", "flush", "chain", item.Family, nftablesNamespace, item.Name, ";", "delete", "chain", item.Family, nftablesNamespace, item.Name) if err != nil { return fmt.Errorf("Failed deleting nftables chain %q (%s): %w", item.Name, item.Family, err) } } return nil } // InstanceSetupRPFilter activates reverse path filtering for the specified instance device on the host interface. func (d Nftables) InstanceSetupRPFilter(projectName string, instanceName string, deviceName string, hostName string) error { deviceLabel := d.instanceDeviceLabel(projectName, instanceName, deviceName) tplFields := map[string]any{ "namespace": nftablesNamespace, "chainSeparator": nftablesChainSeparator, "deviceLabel": deviceLabel, "hostName": hostName, "family": "inet", } err := d.applyNftConfig(nftablesInstanceRPFilter, tplFields) if err != nil { return fmt.Errorf("Failed adding reverse path filter rules for instance device %q (%s): %w", deviceLabel, tplFields["family"], err) } return nil } // InstanceClearRPFilter removes reverse path filtering for the specified instance device on the host interface. func (d Nftables) InstanceClearRPFilter(projectName string, instanceName string, deviceName string) error { deviceLabel := d.instanceDeviceLabel(projectName, instanceName, deviceName) // Remove from ip and ip6 tables to ensure cleanup for instances started before we moved to inet table. err := d.removeChains([]string{"inet", "ip", "ip6"}, deviceLabel, "prert") if err != nil { return fmt.Errorf("Failed clearing reverse path filter rules for instance device %q: %w", deviceLabel, err) } return nil } // InstanceSetupNetPrio activates setting of skb->priority for the specified instance device on the host interface. func (d Nftables) InstanceSetupNetPrio(projectName string, instanceName string, deviceName string, netPrio uint32) error { deviceLabel := d.instanceDeviceLabel(projectName, instanceName, deviceName) tplFields := map[string]any{ "namespace": nftablesNamespace, "family": "netdev", "chainSeparator": nftablesChainSeparator, "deviceLabel": deviceLabel, "deviceName": deviceName, "netPrio": netPrio, } err := d.applyNftConfig(nftablesInstanceNetPrio, tplFields) if err != nil { return fmt.Errorf("Failed adding netprio rules for instance device %q: %w", deviceLabel, err) } return nil } // InstanceClearNetPrio removes setting of skb->priority for the specified instance device on the host interface. func (d Nftables) InstanceClearNetPrio(projectName string, instanceName string, deviceName string) error { if deviceName == "" { return fmt.Errorf("Failed clearing netprio rules for instance %q in project %q: device name is empty", projectName, instanceName) } deviceLabel := d.instanceDeviceLabel(projectName, instanceName, deviceName) chainLabel := fmt.Sprintf("netprio%s%s", nftablesChainSeparator, deviceLabel) err := d.removeChains([]string{"netdev"}, chainLabel, "egress") if err != nil { return fmt.Errorf("Failed clearing netprio rules for instance device %q: %w", deviceLabel, err) } return nil } // NetworkApplyACLRules applies ACL rules to the existing firewall chains. func (d Nftables) NetworkApplyACLRules(networkName string, rules []ACLRule) error { completeNftRules := make([]string, 0) for _, rule := range rules { // First try generating rules with IPv4 or IP agnostic criteria. // If protocol is icmpv6 skip nftRules, partial, err := d.aclRuleCriteriaToRules(networkName, 4, &rule) if err != nil { return err } if len(nftRules) != 0 { completeNftRules = append(completeNftRules, nftRules...) } if partial { // If we couldn't fully generate the ruleset with only IPv4 or IP agnostic criteria, then // fill in the remaining parts using IPv6 criteria. nftRules, _, err = d.aclRuleCriteriaToRules(networkName, 6, &rule) if err != nil { return err } if len(nftRules) == 0 { // When using address set we may generates empty rules without it being an error. continue } completeNftRules = append(completeNftRules, nftRules...) } } tplFields := map[string]any{ "namespace": nftablesNamespace, "chainSeparator": nftablesChainSeparator, "networkName": networkName, "family": "inet", "rules": completeNftRules, } config := &strings.Builder{} err := nftablesNetACLRules.Execute(config, tplFields) if err != nil { return fmt.Errorf("Failed running %q template: %w", nftablesNetACLRules.Name(), err) } err = subprocess.RunCommandWithFds(context.TODO(), strings.NewReader(config.String()), nil, "nft", "-f", "-") if err != nil { return err } return nil } // buildRemainingRuleParts is a helper that returns the protocol, port, logging, and action parts of a rule. func (d Nftables) buildRemainingRuleParts(rule *ACLRule, ipVersion uint) (string, error) { args := []string{} // Add protocol filters. if slices.Contains([]string{"tcp", "udp"}, rule.Protocol) { args = append(args, "meta", "l4proto", rule.Protocol) if rule.SourcePort != "" { args = append(args, d.aclRulePortToACLMatch("sport", util.SplitNTrimSpace(rule.SourcePort, ",", -1, false)...)...) } if rule.DestinationPort != "" { args = append(args, d.aclRulePortToACLMatch("dport", util.SplitNTrimSpace(rule.DestinationPort, ",", -1, false)...)...) } } else if slices.Contains([]string{"icmp4", "icmp6"}, rule.Protocol) { var protoName string switch rule.Protocol { case "icmp4": protoName = "icmp" args = append(args, "ip", "protocol", protoName) case "icmp6": protoName = "icmpv6" args = append(args, "ip6", "nexthdr", protoName) } if rule.ICMPType != "" { args = append(args, protoName, "type", rule.ICMPType) if rule.ICMPCode != "" { args = append(args, protoName, "code", rule.ICMPCode) } } } // Handle logging. if rule.Log { args = append(args, "log") if rule.LogName != "" { // Append a trailing space for readability in logs. args = append(args, "prefix", fmt.Sprintf(`"%s "`, rule.LogName)) } } // Handle action. action := rule.Action if action == "allow" { action = "accept" } args = append(args, action) return strings.Join(args, " "), nil } // aclRuleCriteriaToRules converts an ACL rule into one or more nftables rule strings. // It uses aclRuleSubjectToACLMatch to generate separate fragments for subject criteria. // The function returns a slice of complete rule strings, a partial flag, and an error. func (d Nftables) aclRuleCriteriaToRules(networkName string, ipVersion uint, rule *ACLRule) ([]string, bool, error) { // Build a base argument list with the interface name. baseArgs := []string{} var useAddressSets bool if rule.Direction == "ingress" { // For ingress, the rule applies to packets coming from the host into the network's interface. baseArgs = append(baseArgs, "oifname", networkName) } else { // For egress, packets leaving the network's interface toward the host. baseArgs = append(baseArgs, "iifname", networkName) } // We'll accumulate rule fragments in this slice. var ruleFragments [][]string var ruleStrings []string overallPartial := false // Process source criteria if present. if rule.Source != "" { var err error var matchFragments []string matchFragments, overallPartial, err = d.aclRuleSubjectToACLMatch("saddr", ipVersion, util.SplitNTrimSpace(rule.Source, ",", -1, false)...) if err != nil { return nil, overallPartial, err } if len(matchFragments) == 0 { overallPartial = true } else { // For each fragment generated from the source criteria, // start a new rule fragment beginning with the base arguments. for _, frag := range matchFragments { // if fragment contain IP address sets of different family than icmp drop fragment // This is ok for icmp only as we may apply both ipv4 and ipv6 restriction in match field for tcp/udp ruleFragments = append(ruleFragments, append(slices.Clone(baseArgs), frag)) } } } // Process destination criteria if present. if rule.Destination != "" { var err error var matchFragments []string matchFragments, overallPartial, err = d.aclRuleSubjectToACLMatch("daddr", ipVersion, util.SplitNTrimSpace(rule.Destination, ",", -1, false)...) if err != nil { return nil, overallPartial, err } if len(matchFragments) == 0 { overallPartial = true } else { if len(ruleFragments) > 0 { // Combine each existing fragment with each destination fragment. var combined [][]string contains := func(fragMap [][]string, item []string) bool { for _, s := range fragMap { if strings.Join(s, " ") == strings.Join(item, " ") { return true } } return false } for _, frag := range ruleFragments { for _, df := range matchFragments { newRule := append(slices.Clone(frag), df) if !contains(combined, newRule) { combined = append(combined, newRule) } } } ruleFragments = combined } else { // If no source criteria were provided, start with baseArgs and add destination fragments. for _, df := range matchFragments { ruleFragments = append(ruleFragments, append(slices.Clone(baseArgs), df)) } } } } // If source and destination are empty we want to build base rules at least if rule.Source == "" && rule.Destination == "" { ruleFragments = append(ruleFragments, slices.Clone(baseArgs)) } // Build the remaining parts (protocol, ports, logging, action). suffixParts, err := d.buildRemainingRuleParts(rule, ipVersion) if err != nil { return nil, overallPartial, err } // Append the common suffix parts to every fragment. for _, frag := range ruleFragments { fullFrag := append(frag, suffixParts) // Filter out for icmp address sets not in the correct ip version ruleString := strings.Join(fullFrag, " ") if slices.Contains([]string{"icmp4", "icmp6"}, rule.Protocol) { var icmpIPVersion uint switch rule.Protocol { case "icmp4": icmpIPVersion = 4 case "icmp6": icmpIPVersion = 6 } if strings.Contains(rule.Source, "$") || strings.Contains(rule.Destination, "$") { useAddressSets = true } if ipVersion != icmpIPVersion { if !useAddressSets { // If we got this far it means that source/destination are either empty or are filled // with at least some subjects in the same family as ipVersion. So if the icmpIPVersion // doesn't match the ipVersion then it means the rule contains mixed-version subjects // which is invalid when using an IP version specific ICMP protocol. if rule.Source != "" || rule.Destination != "" { return nil, overallPartial, fmt.Errorf("Invalid use of %q protocol with non-IPv%d source/destination criteria", rule.Protocol, ipVersion) } // Otherwise it means this is just a blanket ICMP rule and is only appropriate for use // with the corresponding ipVersion nft command. return nil, true, nil // Rule is not appropriate for ipVersion. } if strings.Contains(ruleString, fmt.Sprintf("_ipv%d", ipVersion)) { continue } } } ruleStrings = append(ruleStrings, ruleString) } return ruleStrings, overallPartial, nil } // aclRuleSubjectToACLMatch converts a list of subject criteria into one or more nft rule fragments. // It splits the criteria into address-set references and literal addresses. // For each address set reference (criteria starting with "$"), it creates a fragment using the set reference // (without braces). For literal addresses, it creates one fragment combining them in braces. // It returns a slice of fragments, a partial flag, and an error. func (d Nftables) aclRuleSubjectToACLMatch(direction string, ipVersion uint, subjectCriteria ...string) ([]string, bool, error) { var setRefs []string var literals []string partial := false // Process each criterion for _, subjectCriterion := range subjectCriteria { after, ok := strings.CutPrefix(subjectCriterion, "$") if ok { // This is an address set reference. setName := after // With an address we won't guess if it only contains ipv4 or ipv6 address so partial is set partial = true switch ipVersion { case 6: setRefs = append(setRefs, fmt.Sprintf(" @%s_ipv6", setName)) case 4: setRefs = append(setRefs, fmt.Sprintf(" @%s_ipv4", setName)) } } else { // Process literal address or range. if validate.IsNetworkRange(subjectCriterion) == nil { criterionParts := strings.SplitN(subjectCriterion, "-", 2) if len(criterionParts) < 2 { return nil, false, fmt.Errorf("Invalid IP range %q", subjectCriterion) } ip := net.ParseIP(criterionParts[0]) if ip != nil { var subjectIPVersion uint = 4 if ip.To4() == nil { subjectIPVersion = 6 } if ipVersion != subjectIPVersion { partial = true continue // Skip subjects that are not for the ipVersion we are looking for. } literals = append(literals, fmt.Sprintf("%s-%s", criterionParts[0], criterionParts[1])) } } else { ip := net.ParseIP(subjectCriterion) if ip == nil { ip, _, _ = net.ParseCIDR(subjectCriterion) } if ip == nil { return nil, false, fmt.Errorf("Unsupported nftables subject %q", subjectCriterion) } var subjectIPVersion uint = 4 if ip.To4() == nil { subjectIPVersion = 6 } if ipVersion != subjectIPVersion { partial = true continue // Skip subjects that are not for the ipVersion we are looking for. } literals = append(literals, subjectCriterion) } } } // Build the result fragments. var fragments []string ipFamily := "ip" if ipVersion == 6 { ipFamily = "ip6" } // For each set reference, create its own fragment. if len(setRefs) > 0 { for _, ref := range setRefs { fragments = append(fragments, fmt.Sprintf("%s %s %s", ipFamily, direction, ref)) } } // If there are literal addresses, create one fragment combining them. if len(literals) > 0 { fragments = append(fragments, fmt.Sprintf("%s %s {%s}", ipFamily, direction, strings.Join(literals, ","))) } if len(fragments) == 0 { return nil, partial, nil } return fragments, partial, nil } // aclRulePortToACLMatch converts protocol (tcp/udp), direction (sports/dports) and port criteria list into // nftables args. func (d Nftables) aclRulePortToACLMatch(direction string, portCriteria ...string) []string { fieldParts := make([]string, 0, len(portCriteria)) for _, portCriterion := range portCriteria { criterionParts := strings.SplitN(portCriterion, "-", 2) if len(criterionParts) > 1 { fieldParts = append(fieldParts, fmt.Sprintf("%s-%s", criterionParts[0], criterionParts[1])) } else { fieldParts = append(fieldParts, criterionParts[0]) } } return []string{"th", direction, fmt.Sprintf("{%s}", strings.Join(fieldParts, ","))} } // NetworkApplyForwards apply network address forward rules to firewall. func (d Nftables) NetworkApplyForwards(networkName string, rules []AddressForward) error { var dnatRules []map[string]any var snatRules []map[string]any // Build up rules, ordering by port specific listen rules first, followed by default target rules. // This is so the generated firewall rules will apply the port specific rules first. for _, listenPortsOnly := range []bool{true, false} { for ruleIndex, rule := range rules { // Process the rules in order of outer loop. listenPortsLen := len(rule.ListenPorts) if (listenPortsOnly && listenPortsLen < 1) || (!listenPortsOnly && listenPortsLen > 0) { continue } // Validate the rule. if rule.ListenAddress == nil { return fmt.Errorf("Invalid rule %d, listen address is required", ruleIndex) } if rule.TargetAddress == nil { return fmt.Errorf("Invalid rule %d, target address is required", ruleIndex) } if listenPortsLen == 0 && rule.Protocol != "" { return fmt.Errorf("Invalid rule %d, default target rule but non-empty protocol", ruleIndex) } switch len(rule.TargetPorts) { case 0: // No target ports specified, use listen ports (only valid when protocol is specified). rule.TargetPorts = rule.ListenPorts case 1: // Single target port specified, OK. case len(rule.ListenPorts): // One-to-one match with listen ports, OK. default: return fmt.Errorf("Invalid rule %d, mismatch between listen port(s) and target port(s) count", ruleIndex) } ipFamily := "ip" if rule.ListenAddress.To4() == nil { ipFamily = "ip6" } listenAddressStr := rule.ListenAddress.String() targetAddressStr := rule.TargetAddress.String() if rule.Protocol != "" { targetPortRanges := portRangesFromSlice(rule.TargetPorts) for _, targetPortRange := range targetPortRanges { targetPortRangeStr := portRangeStr(targetPortRange, "-") snatRules = append(snatRules, map[string]any{ "ipFamily": ipFamily, "protocol": rule.Protocol, "targetHost": targetAddressStr, "targetPorts": targetPortRangeStr, }) } dnatRanges := getOptimisedDNATRanges(&rule) for listenPortRange, targetPortRange := range dnatRanges { // Format the destination host/port as appropriate targetDest := targetAddressStr if targetPortRange[1] == 1 { targetPortStr := portRangeStr(targetPortRange, ":") targetDest = fmt.Sprintf("%s:%s", targetAddressStr, targetPortStr) if ipFamily == "ip6" { targetDest = fmt.Sprintf("[%s]:%s", targetAddressStr, targetPortStr) } } dnatRules = append(dnatRules, map[string]any{ "ipFamily": ipFamily, "protocol": rule.Protocol, "listenAddress": listenAddressStr, "listenPorts": portRangeStr(listenPortRange, "-"), "targetDest": targetDest, }) if rule.SNAT { snatRules = append(snatRules, map[string]any{ "ipFamily": ipFamily, "protocol": rule.Protocol, "listenAddress": listenAddressStr, "listenPorts": portRangeStr(listenPortRange, "-"), "targetAddress": targetAddressStr, "targetPorts": portRangeStr(targetPortRange, "-"), }) } } } else { // Format the destination host/port as appropriate. targetDest := targetAddressStr if ipFamily == "ip6" { targetDest = fmt.Sprintf("[%s]", targetAddressStr) } dnatRules = append(dnatRules, map[string]any{ "ipFamily": ipFamily, "listenAddress": listenAddressStr, "targetDest": targetDest, "targetHost": targetAddressStr, }) snatRules = append(snatRules, map[string]any{ "ipFamily": ipFamily, "targetHost": targetAddressStr, }) } } } tplFields := map[string]any{ "namespace": nftablesNamespace, "chainSeparator": nftablesChainSeparator, "chainPrefix": "fwd", // Differentiate from proxy device forwards. "family": "inet", "label": networkName, "dnatRules": dnatRules, "snatRules": snatRules, } // Apply rules or remove chains if no rules generated. if len(dnatRules) > 0 || len(snatRules) > 0 { config := &strings.Builder{} err := nftablesNetProxyNAT.Execute(config, tplFields) if err != nil { return fmt.Errorf("Failed running %q template: %w", nftablesNetProxyNAT.Name(), err) } err = subprocess.RunCommandWithFds(context.TODO(), strings.NewReader(config.String()), nil, "nft", "-f", "-") if err != nil { return err } } else { err := d.removeChains([]string{"inet", "ip", "ip6"}, networkName, "fwdprert", "fwdout", "fwdpstrt") if err != nil { return fmt.Errorf("Failed clearing nftables forward rules for network %q: %w", networkName, err) } } return nil } // NetworkApplyAddressSets creates or updates named nft sets for all address sets. func (d Nftables) NetworkApplyAddressSets(sets []AddressSet, nftTable string) error { _, err := subprocess.RunCommand("nft", "create", "table", nftTable, nftablesNamespace) if err != nil { if !strings.Contains(err.Error(), "Could not process rule: File exists") { return fmt.Errorf("Failed to create table %q: %w", nftTable, err) } } for _, set := range sets { var ipv4Addrs, ipv6Addrs, ethAddrs []string name := set.Name addresses := set.Addresses // Flush current addresses in set if set exists for _, suffix := range []string{"ipv4", "ipv6", "eth"} { flush := &strings.Builder{} setName := fmt.Sprintf("%s_%s", name, suffix) exists, err := d.NamedAddressSetExists(setName, nftTable) if err != nil { return fmt.Errorf("Failed to check existence of set %q: %w", setName, err) } if exists { // Append a flush command for this set. fmt.Fprintf(flush, " flush set %s %s %s\n", nftTable, nftablesNamespace, setName) err = subprocess.RunCommandWithFds(context.TODO(), strings.NewReader(flush.String()), nil, "nft", "-f", "-") if err != nil { return fmt.Errorf("Failed to flush nft set for address set %q: %w", setName, err) } } } for _, addr := range addresses { // Try IP first. ip := net.ParseIP(addr) if ip != nil { if ip.To4() != nil { ipv4Addrs = append(ipv4Addrs, addr) continue } else { ipv6Addrs = append(ipv6Addrs, addr) continue } } // Try to parse as CIDR. _, ipNet, err := net.ParseCIDR(addr) if err == nil { if ipNet.IP.To4() != nil { ipv4Addrs = append(ipv4Addrs, addr) } else { ipv6Addrs = append(ipv6Addrs, addr) } continue } // Try MAC perhaps future support _, err = net.ParseMAC(addr) if err == nil { ethAddrs = append(ethAddrs, addr) continue } return fmt.Errorf("unsupported address format: %q", addr) } // Build NFT config. configv4 := &strings.Builder{} configv6 := &strings.Builder{} configeth := &strings.Builder{} if len(ipv4Addrs) >= 0 { // Create v4 named set fmt.Fprintf(configv4, "add set %s %s ", nftTable, nftablesNamespace) setExtendedName := fmt.Sprintf("%s_ipv4", name) if len(ipv4Addrs) == 0 { // Create empty set to avoid errors fmt.Fprintf(configv4, " %s {\n type ipv4_addr;\n flags interval;\n}\n", setExtendedName) } else { fmt.Fprintf(configv4, " %s {\n type ipv4_addr;\n flags interval;\n elements = { %s }\n }\n", setExtendedName, strings.Join(ipv4Addrs, ", ")) } err := subprocess.RunCommandWithFds(context.TODO(), strings.NewReader(configv4.String()), nil, "nft", "-f", "-") if err != nil { return fmt.Errorf("Failed to apply nft sets for address set %q: %w", name, err) } } if len(ipv6Addrs) >= 0 { fmt.Fprintf(configv6, "add set %s %s ", nftTable, nftablesNamespace) setExtendedName := fmt.Sprintf("%s_ipv6", name) // Create v6 named set if len(ipv6Addrs) == 0 { // Create empty set to avoid errors fmt.Fprintf(configv6, " %s {\n type ipv6_addr;\n flags interval;\n}\n", setExtendedName) } else { fmt.Fprintf(configv6, " %s {\n type ipv6_addr;\n flags interval;\n elements = { %s }\n }\n", setExtendedName, strings.Join(ipv6Addrs, ", ")) } err := subprocess.RunCommandWithFds(context.TODO(), strings.NewReader(configv6.String()), nil, "nft", "-f", "-") if err != nil { return fmt.Errorf("Failed to apply nft sets for address set %q: %w", name, err) } } // Should be >= but since we do not support it for now leave it as a dead portion if len(ethAddrs) > 0 { fmt.Fprintf(configeth, "add set %s %s ", nftTable, nftablesNamespace) setExtendedName := fmt.Sprintf("%s_eth", name) // Create eth named set perhaps future support if len(ethAddrs) == 0 { fmt.Fprintf(configeth, " set %s {\n type ether_addr;\n}\n", setExtendedName) } else { fmt.Fprintf(configeth, " set %s {\n type ether_addr;\n elements = { %s }\n }\n", setExtendedName, strings.Join(ethAddrs, ", ")) } err := subprocess.RunCommandWithFds(context.TODO(), strings.NewReader(configv6.String()), nil, "nft", "-f", "-") if err != nil { return fmt.Errorf("Failed to apply nft sets for address set %q: %w", name, err) } } } return nil } // NamedAddressSetExists checks if a named set exists in nftables. // It returns true if the set exists in the nftables namespace. func (d Nftables) NamedAddressSetExists(setName string, family string) (bool, error) { // Execute the nft command with JSON output using subprocess. output, err := subprocess.RunCommand("nft", "-j", "list", "sets") if err != nil { return false, fmt.Errorf("Failed to execute nft command: %w", err) } var setsOutput NftListSetsOutput err = json.Unmarshal([]byte(output), &setsOutput) if err != nil { return false, fmt.Errorf("Failed to parse nft command output: %w", err) } // Iterate through the sets to find a match. for _, entry := range setsOutput.Nftables { if entry.Set != nil { if strings.EqualFold(entry.Set.Name, setName) && strings.EqualFold(entry.Set.Family, family) && strings.EqualFold(entry.Set.Table, nftablesNamespace) { return true, nil } } } // Set not found. return false, nil } // RemoveIncusAddressSets remove every address set in incus namespace. func (d Nftables) RemoveIncusAddressSets(nftTable string) error { // Execute the nft command with JSON output using subprocess. output, err := subprocess.RunCommand("nft", "-j", "list", "sets", nftTable) if err != nil { return fmt.Errorf("Failed to execute nft command: %w", err) } var setsOutput NftListSetsOutput err = json.Unmarshal([]byte(output), &setsOutput) if err != nil { return fmt.Errorf("Failed to parse nft command output: %w", err) } for _, setEntry := range setsOutput.Nftables { if setEntry.Set == nil { continue // Skip entries that do not contain a set. } if strings.EqualFold(setEntry.Set.Table, nftablesNamespace) { _, err := subprocess.RunCommand("nft", "delete", "set", nftTable, nftablesNamespace, setEntry.Set.Name) if err != nil { return fmt.Errorf("Failed to delete nft named set %s: %w", setEntry.Set.Name, err) } } } return nil } // NetworkDeleteAddressSetsIfUnused delete unused address set from table nftTable. func (d Nftables) NetworkDeleteAddressSetsIfUnused(nftTable string) error { // List all sets in the given table. outputSets, err := subprocess.RunCommand("nft", "-j", "list", "sets", nftTable) if err != nil { return fmt.Errorf("Failed to list nft sets in table %q: %w", nftTable, err) } var setsOutput NftListSetsOutput err = json.Unmarshal([]byte(outputSets), &setsOutput) if err != nil { return fmt.Errorf("Failed to parse nft sets output: %w", err) } // Collect set names. setNames := make(map[string]struct{}) for _, entry := range setsOutput.Nftables { if entry.Set != nil && entry.Set.Family == nftTable { setNames[entry.Set.Name] = struct{}{} } } // List rules to check for usage of sets aka @setName. outputRules, err := subprocess.RunCommand("nft", "list", "ruleset", nftTable) if err != nil { return fmt.Errorf("Failed to list nft ruleset: %w", err) } // Check which sets are actually used. usedSets := make(map[string]struct{}) for setName := range setNames { if strings.Contains(outputRules, fmt.Sprintf("@%s", setName)) { usedSets[setName] = struct{}{} } } // Delete sets that are unused. for setName := range setNames { _, used := usedSets[setName] if used { continue } _, err := subprocess.RunCommand("nft", "delete", "set", nftTable, nftablesNamespace, setName) if err != nil { return fmt.Errorf("Failed to delete unused set %q: %w", setName, err) } } return nil } incus-7.0.0/internal/server/firewall/drivers/drivers_nftables_templates.go000066400000000000000000000341611517523235500272220ustar00rootroot00000000000000package drivers import ( "text/template" ) var nftablesCommonTable = template.Must(template.New("nftablesCommonTable").Parse(` table {{.family}} {{.namespace}} { {{ template "nftablesContent" . }} } `)) var nftablesNetForwardingPolicy = template.Must(template.New("nftablesNetForwardingPolicy").Parse(` chain fwd{{.chainSeparator}}{{.networkName}} { type filter hook forward priority 0; policy accept; {{ if .ip4Action }} ip version 4 oifname "{{.networkName}}" {{.ip4Action}} ip version 4 iifname "{{.networkName}}" {{.ip4Action}} {{ end }} {{ if .ip6Action }} ip6 version 6 oifname "{{.networkName}}" {{.ip6Action}} ip6 version 6 iifname "{{.networkName}}" {{.ip6Action}} {{ end }} } `)) var nftablesNetOutboundNAT = template.Must(template.New("nftablesNetOutboundNAT").Parse(` chain pstrt{{.chainSeparator}}{{.networkName}} { type nat hook postrouting priority 100; policy accept; {{ range $ipFamily, $config := .rules }} {{ if $config.SNATAddress }} {{$ipFamily}} saddr {{$config.Subnet}} {{$ipFamily}} daddr != {{$config.Subnet}} snat {{$config.SNATAddress}} {{ else }} {{$ipFamily}} saddr {{$config.Subnet}} {{$ipFamily}} daddr != {{$config.Subnet}} masquerade {{ end }} {{ end }} } `)) var nftablesNetICMPDHCPDNS = template.Must(template.New("nftablesNetDHCPDNS").Parse(` chain in{{.chainSeparator}}{{.networkName}} { type filter hook input priority 0; policy accept; iifname "{{.networkName}}" tcp dport 53 accept iifname "{{.networkName}}" udp dport 53 accept {{ range .ipFamilies }} {{ if eq . "ip" }} iifname "{{$.networkName}}" icmp type {3, 11, 12} accept iifname "{{$.networkName}}" udp dport 67 accept iifname "{{$.networkName}}" ip protocol udp udp checksum set 0 {{ else }} iifname "{{$.networkName}}" icmpv6 type {1, 2, 3, 4, 133, 135, 136, 143} accept iifname "{{$.networkName}}" udp dport 547 accept {{ end }} {{ end }} } chain out{{.chainSeparator}}{{.networkName}} { type filter hook output priority 0; policy accept; oifname "{{.networkName}}" tcp sport 53 accept oifname "{{.networkName}}" udp sport 53 accept {{ range .ipFamilies }} {{ if eq . "ip" }} oifname "{{$.networkName}}" icmp type {3, 11, 12} accept oifname "{{$.networkName}}" udp sport 67 accept oifname "{{$.networkName}}" ip protocol udp udp checksum set 0 {{ else }} oifname "{{$.networkName}}" icmpv6 type {1, 2, 3, 4, 128, 134, 135, 136, 143} accept oifname "{{$.networkName}}" udp sport 547 accept {{ end }} {{ end }} } `)) var nftablesNetProxyNAT = template.Must(template.New("nftablesNetProxyNAT").Parse(` add table {{.family}} {{.namespace}} add chain {{.family}} {{.namespace}} {{.chainPrefix}}prert{{.chainSeparator}}{{.label}} {type nat hook prerouting priority -100; policy accept;} add chain {{.family}} {{.namespace}} {{.chainPrefix}}out{{.chainSeparator}}{{.label}} {type nat hook output priority -100; policy accept;} add chain {{.family}} {{.namespace}} {{.chainPrefix}}pstrt{{.chainSeparator}}{{.label}} {type nat hook postrouting priority 100; policy accept;} flush chain {{.family}} {{.namespace}} {{.chainPrefix}}prert{{.chainSeparator}}{{.label}} flush chain {{.family}} {{.namespace}} {{.chainPrefix}}out{{.chainSeparator}}{{.label}} flush chain {{.family}} {{.namespace}} {{.chainPrefix}}pstrt{{.chainSeparator}}{{.label}} table {{.family}} {{.namespace}} { chain {{.chainPrefix}}prert{{.chainSeparator}}{{.label}} { type nat hook prerouting priority -100; policy accept; {{ range .dnatRules }} {{.ipFamily}} daddr {{.listenAddress}} {{ if .protocol }}{{.protocol}} dport {{.listenPorts}}{{ end }} dnat to {{.targetDest}} {{ end }} } chain {{.chainPrefix}}out{{.chainSeparator}}{{.label}} { type nat hook output priority -100; policy accept; {{ range .dnatRules }} {{.ipFamily}} daddr {{.listenAddress}} {{ if .protocol }}{{.protocol}} dport {{.listenPorts}}{{ end }} dnat to {{.targetDest}} {{ end }} } chain {{.chainPrefix}}pstrt{{.chainSeparator}}{{.label}} { type nat hook postrouting priority 100; policy accept; {{ range .snatRules }} {{ if .targetHost }} {{.ipFamily}} saddr {{.targetHost}} {{.ipFamily}} daddr {{.targetHost}} {{ if .protocol }}{{.protocol}} dport {{.targetPorts}}{{ end }} masquerade {{ else }} {{.ipFamily}} saddr {{.targetAddress}} {{.protocol}} sport {{.targetPorts}} snat to {{.listenAddress}}:{{.listenPorts}} {{ end }} {{ end }} } } `)) var nftablesNetACLSetup = template.Must(template.New("nftablesNetACLSetup").Parse(` add table {{.family}} {{.namespace}} add chain {{.family}} {{.namespace}} acl{{.chainSeparator}}{{.networkName}} add chain {{.family}} {{.namespace}} aclin{{.chainSeparator}}{{.networkName}} {type filter hook input priority filter; policy accept;} add chain {{.family}} {{.namespace}} aclout{{.chainSeparator}}{{.networkName}} {type filter hook output priority filter; policy accept;} add chain {{.family}} {{.namespace}} aclfwd{{.chainSeparator}}{{.networkName}} {type filter hook forward priority filter; policy accept;} flush chain {{.family}} {{.namespace}} acl{{.chainSeparator}}{{.networkName}} flush chain {{.family}} {{.namespace}} aclin{{.chainSeparator}}{{.networkName}} flush chain {{.family}} {{.namespace}} aclout{{.chainSeparator}}{{.networkName}} flush chain {{.family}} {{.namespace}} aclfwd{{.chainSeparator}}{{.networkName}} table {{.family}} {{.namespace}} { chain aclin{{.chainSeparator}}{{.networkName}} { # Allow DNS to Incus host. iifname "{{.networkName}}" tcp dport 53 accept iifname "{{.networkName}}" udp dport 53 accept # Allow DHCPv6 to Incus host. iifname "{{$.networkName}}" udp dport 67 accept iifname "{{$.networkName}}" udp dport 547 accept # Allow core ICMPv4 to Incus host. iifname "{{$.networkName}}" icmp type {3, 11, 12} accept # Allow core ICMPv6 to Incus host. iifname "{{$.networkName}}" icmpv6 type {1, 2, 3, 4, 133, 135, 136, 143} accept iifname "{{.networkName}}" jump acl{{.chainSeparator}}{{.networkName}} } chain aclout{{.chainSeparator}}{{.networkName}} { # Allow DHCPv6 from Incus host. oifname "{{$.networkName}}" udp sport 67 accept oifname "{{$.networkName}}" udp sport 547 accept # Allow core ICMPv4 from Incus host. oifname "{{$.networkName}}" icmp type {3, 11, 12} accept # Allow ICMPv6 ping from host into network as dnsmasq uses this to probe IP allocations. oifname "{{$.networkName}}" icmpv6 type {1, 2, 3, 4, 128, 134, 135, 136, 143} accept oifname "{{.networkName}}" jump acl{{.chainSeparator}}{{.networkName}} } chain aclfwd{{.chainSeparator}}{{.networkName}} { iifname "{{.networkName}}" jump acl{{.chainSeparator}}{{.networkName}} oifname "{{.networkName}}" jump acl{{.chainSeparator}}{{.networkName}} } } `)) var nftablesNetACLRules = template.Must(template.New("nftablesNetACLRules").Parse(` flush chain {{.family}} {{.namespace}} acl{{.chainSeparator}}{{.networkName}} table {{.family}} {{.namespace}} { chain acl{{.chainSeparator}}{{.networkName}} { ct state established,related accept {{ range .rules }} {{.}} {{ end }} } } `)) // nftablesInstanceBridgeFilter defines the rules needed for MAC, IPv4 and IPv6 bridge security filtering. // To prevent instances from using IPs that are different from their assigned IPs we use ARP and NDP filtering // to prevent neighbour advertisements that are not allowed. However in order for DHCPv4 & DHCPv6 to work back to // the Incus host we need to allow DHCPv4 inbound and for IPv6 we need to allow IPv6 Router Solicitation and DHPCv6. // Nftables doesn't support the equivalent of "arp saddr" and "arp saddr ether" at this time so in order to filter // NDP advertisements that come from the genuine Ethernet MAC address but have a spoofed NDP source MAC/IP address // we need to use manual header offset extraction. This also drops IPv6 router advertisements from instance. // If IP filtering is enabled, this also drops unwanted ethernet frames. var nftablesInstanceBridgeFilter = template.Must(template.New("nftablesInstanceBridgeFilter").Parse(` chain in{{.chainSeparator}}{{.deviceLabel}} { type filter hook input priority -200; policy accept; # MAC filtering {{ if .macFiltering }} iifname "{{.hostName}}" ether saddr != {{.hwAddr}} drop iifname "{{.hostName}}" ether type arp arp saddr ether != {{.hwAddr}} drop iifname "{{.hostName}}" ether type ip6 icmpv6 type 136 @nh,528,48 != {{.hwAddrHex}} drop {{ end }} {{ if or .aclInDropRules .aclInRejectRules .aclInAcceptRules .aclOutDropRules .aclOutAcceptRules .aclInDefaultRule .ipv4NetsList }} iifname "{{.hostName}}" ether type ip ip saddr 0.0.0.0 ip daddr 255.255.255.255 udp dport 67 accept {{ end }} # IPv4 filtering {{ if .ipv4NetsList }} iifname "{{.hostName}}" ether type arp arp saddr ip != { {{.ipv4NetsList}} } drop iifname "{{.hostName}}" ether type ip ip saddr != { {{.ipv4NetsList}} } drop {{ end }} {{ if .ipv4FilterAll }} iifname "{{.hostName}}" ether type arp drop iifname "{{.hostName}}" ether type ip drop {{ end }} {{ if or .aclInDropRules .aclInRejectRules .aclInAcceptRules .aclOutDropRules .aclOutAcceptRules .aclInDefaultRule .ipv6NetsList }} iifname "{{.hostName}}" ether type ip6 ip6 saddr fe80::/10 ip6 daddr ff02::1:2 udp dport 547 accept iifname "{{.hostName}}" ether type ip6 ip6 saddr fe80::/10 ip6 daddr ff02::2 icmpv6 type 133 accept {{ end }} # IPv6 filtering {{ if .ipv6NetsList }} iifname "{{.hostName}}" ether type ip6 icmpv6 type 134 drop iifname "{{.hostName}}" ether type ip6 icmpv6 type 136 {{.ipv6NetsPrefixList}} drop iifname "{{.hostName}}" ether type ip6 ip6 saddr != { {{.ipv6NetsList}} } drop {{ end }} {{ if .ipv6FilterAll }} iifname "{{.hostName}}" ether type ip6 drop {{ end }} {{ if or .aclInDropRules .aclInRejectRules .aclInAcceptRules .aclOutDropRules .aclOutAcceptRules .aclInDefaultRule }} ct state established,related accept iifname "{{.hostName}}" ether type arp accept iifname "{{.hostName}}" ip6 nexthdr ipv6-icmp icmpv6 type { nd-neighbor-solicit, nd-neighbor-advert } accept {{ if .dnsIPv4 }} {{ range .dnsIPv4 }} iifname "{{$.hostName}}" ip daddr "{{.}}" tcp dport 53 accept iifname "{{$.hostName}}" ip daddr "{{.}}" udp dport 53 accept {{ end }} {{ end }} {{ if .dnsIPv6 }} {{ range .dnsIPv6 }} iifname "{{$.hostName}}" ip6 daddr "{{.}}" tcp dport 53 accept iifname "{{$.hostName}}" ip6 daddr "{{.}}" udp dport 53 accept {{ end }} {{ end }} {{ end }} # ACLs {{ range .aclInDropRules }} {{.}} {{ end }} {{ range .aclInRejectRules }} {{.}} {{ end }} {{ range .aclInAcceptRules }} {{.}} {{ end }} {{ if .filterUnwantedFrames }} iifname "{{.hostName}}" ether type != {arp, ip, ip6} drop {{ end }} {{.aclInDefaultRule}} } chain fwd{{.chainSeparator}}{{.deviceLabel}} { type filter hook forward priority -200; policy accept; # MAC filtering {{ if .macFiltering }} iifname "{{.hostName}}" ether saddr != {{.hwAddr}} drop iifname "{{.hostName}}" ether type arp arp saddr ether != {{.hwAddr}} drop iifname "{{.hostName}}" ether type ip6 icmpv6 type 136 @nh,528,48 != {{.hwAddrHex}} drop {{ end }} # IPv4 filtering {{ if .ipv4NetsList }} iifname "{{.hostName}}" ether type arp arp saddr ip != { {{.ipv4NetsList}} } drop iifname "{{.hostName}}" ether type ip ip saddr != { {{.ipv4NetsList}} } drop {{ end }} {{ if .ipv4FilterAll }} iifname "{{.hostName}}" ether type arp drop iifname "{{.hostName}}" ether type ip drop {{ end }} # IPv6 filtering {{ if .ipv6NetsList }} iifname "{{.hostName}}" ether type ip6 icmpv6 type 134 drop iifname "{{.hostName}}" ether type ip6 icmpv6 type 136 {{.ipv6NetsPrefixList}} drop iifname "{{.hostName}}" ether type ip6 ip6 saddr != { {{.ipv6NetsList}} } drop {{ end }} {{ if .ipv6FilterAll }} iifname "{{.hostName}}" ether type ip6 drop {{ end }} # Network ACLs {{ if or .aclInDropRules .aclInRejectRulesConverted .aclInAcceptRules .aclOutDropRules .aclOutAcceptRules .aclInDefaultRuleConverted .aclOutDefaultRule }} ct state established,related accept {{ end }} {{ range .aclInDropRules}} {{.}} {{ end }} {{ range .aclInRejectRulesConverted}} {{.}} {{ end }} {{ range .aclOutDropRules}} {{.}} {{ end }} {{ range .aclInAcceptRules }} {{.}} {{ end }} {{ range .aclOutAcceptRules }} {{.}} {{ end }} {{ if .filterUnwantedFrames }} iifname "{{.hostName}}" ether type != {arp, ip, ip6} drop {{ end }} {{ if or .aclInDropRules .aclInRejectRulesConverted .aclInAcceptRules .aclOutDropRules .aclOutAcceptRules .aclInDefaultRuleConverted .aclOutDefaultRule }} iifname "{{.hostName}}" ether type arp accept iifname "{{.hostName}}" ip6 nexthdr ipv6-icmp icmpv6 type { nd-neighbor-solicit, nd-neighbor-advert } accept oifname "{{.hostName}}" ether type arp accept oifname "{{.hostName}}" ip6 nexthdr ipv6-icmp icmpv6 type { nd-neighbor-solicit, nd-neighbor-advert } accept {{ end }} {{.aclInDefaultRuleConverted}} {{.aclOutDefaultRule}} } {{ if or .aclInDropRules .aclInRejectRulesConverted .aclInAcceptRules .aclOutDropRules .aclOutAcceptRules .aclInDefaultRule .aclOutDefaultRule }} chain out{{.chainSeparator}}{{.deviceLabel}} { type filter hook output priority filter; policy accept; # Basic connectivity ct state established,related accept oifname "{{.hostName}}" ether type arp accept oifname "{{.hostName}}" ip6 nexthdr ipv6-icmp icmpv6 type { nd-neighbor-solicit, nd-neighbor-advert } accept oifname "{{.hostName}}" udp sport 67 udp dport 68 accept oifname "{{.hostName}}" ip6 saddr fe80::/10 udp sport 547 accept oifname "{{.hostName}}" ip6 saddr fe80::/10 icmpv6 type {1, 2, 3, 4, 128, 134, 135, 136, 143} accept # Network ACLs {{ range .aclOutDropRules}} {{.}} {{ end }} {{ range .aclOutAcceptRules}} {{.}} {{ end }} {{.aclOutDefaultRule}} } {{ end }} `)) // nftablesInstanceRPFilter defines the rules to perform reverse path filtering. var nftablesInstanceRPFilter = template.Must(template.New("nftablesInstanceRPFilter").Parse(` chain prert{{.chainSeparator}}{{.deviceLabel}} { type filter hook prerouting priority -300; policy accept; iif "{{.hostName}}" fib saddr . iif oif missing drop # codespell:ignore iif } `)) // nftablesInstanceNetPrio defines the rules to perform setting of skb->priority. var nftablesInstanceNetPrio = template.Must(template.New("nftablesInstanceNetPrio").Parse(` chain egress{{.chainSeparator}}netprio{{.chainSeparator}}{{.deviceLabel}} { type filter hook egress device "{{.deviceName}}" priority 0 ; meta priority set "{{.netPrio}}" } `)) incus-7.0.0/internal/server/firewall/drivers/drivers_util.go000066400000000000000000000124661517523235500243270ustar00rootroot00000000000000package drivers import ( "encoding/hex" "errors" "fmt" "net" ) // portRangesFromSlice checks if adjacent indices in the given slice contain consecutive // numbers and returns a slice of port ranges ([startNumber, rangeSize]) accordingly. // // Note that this function cannot differentiate ranges from adjacent ports e.g. if the given // slice is "[80,81,82]" then the returned range will be "80-82", regardless of whether the // user input was parsed from "80-82" or "80,81,82". func portRangesFromSlice(ports []uint64) [][2]uint64 { if len(ports) == 0 { return nil } portRanges := make([][2]uint64, 0, len(ports)) startIdx := 0 size := uint64(0) for i := range ports { if i == len(ports)-1 || ports[i+1] != ports[i]+1 { size = ports[i] - ports[startIdx] + 1 portRanges = append(portRanges, [2]uint64{ports[startIdx], size}) startIdx = i + 1 } } return portRanges } func portRangeStr(portRange [2]uint64, delimiter string) string { if portRange[1] < 1 { return "" } else if portRange[1] == 1 { return fmt.Sprintf("%d", portRange[0]) } return fmt.Sprintf("%d%s%d", portRange[0], delimiter, portRange[0]+portRange[1]-1) } // getOptimisedDNATRanges returns a map of listen port ranges to target port ranges that can be // applied in any order. // // Nftables is able to apply rules for multiple listen ports at a time when a // listen port range exactly matches the corresponding target port range (e.g. "80-85" to "80-85") // or when there is a single target port (e.g. "80-85" to "80"). This function checks when these // conditions are met and returns a map of listen and target port ranges to be applied by the loaded // driver. func getOptimisedDNATRanges(forward *AddressForward) map[[2]uint64][2]uint64 { targetPortsLen := len(forward.TargetPorts) listenPortsLen := len(forward.ListenPorts) snatRules := make(map[[2]uint64][2]uint64, listenPortsLen) listenPortRanges := portRangesFromSlice(forward.ListenPorts) // If there is only one target port, DNAT rules can be optimised for all listen ranges. if targetPortsLen == 1 { targetPort := forward.TargetPorts[0] for _, listenPortRange := range listenPortRanges { snatRules[listenPortRange] = [2]uint64{targetPort, 1} } return snatRules } // For a given listen range, the corresponding target range may not simply be targetRange[i] (where "i" // is the index of the listen range). For example, "100-101,300" to "100-102" would be valid config // because the number of listen and target ports are equal, but there are two listen ranges and one // target range. Instead, to check if there is a target range we create a map of port range starting // values and check if the current target port is in the map. targetPortRanges := portRangesFromSlice(forward.TargetPorts) targetPortRangeMap := make(map[uint64]uint64, len(targetPortRanges)) for _, targetPortRange := range targetPortRanges { targetPortRangeMap[targetPortRange[0]] = targetPortRange[1] } nProcessedPorts := 0 for _, listenPortRange := range listenPortRanges { rangeEndIdx := nProcessedPorts + int(listenPortRange[1]) currentTargetPort := forward.TargetPorts[nProcessedPorts] targetPortRangeSize, ok := targetPortRangeMap[currentTargetPort] // Check that we have a target port range and that the listen and target port ranges start // at the same value. if ok && listenPortRange[0] == currentTargetPort { targetPortRange := [2]uint64{currentTargetPort, targetPortRangeSize} // Check if the listen and target ranges are the same size. if listenPortRange[1] == targetPortRangeSize { // Port ranges are identical. One to one mapping. snatRules[listenPortRange] = targetPortRange nProcessedPorts += int(listenPortRange[1]) } else { // Port ranges are identical until the end of the target range. // Rules can be optimised for a portion of ports in the listen range. snatRules[[2]uint64{listenPortRange[0], targetPortRangeSize}] = targetPortRange nProcessedPorts += int(targetPortRangeSize) } } // Remaining ports in the listen range cannot be optimised. for ; nProcessedPorts < rangeEndIdx; nProcessedPorts++ { snatRules[[2]uint64{forward.ListenPorts[nProcessedPorts], 1}] = [2]uint64{forward.TargetPorts[nProcessedPorts], 1} } } return snatRules } // subnetMask returns the subnet mask of the given network as a string. Both IPv4 and IPv6 are handled. func subnetMask(ipNet *net.IPNet) string { if ipNet.IP.To4() != nil { return fmt.Sprintf("%d.%d.%d.%d", ipNet.Mask[0], ipNet.Mask[1], ipNet.Mask[2], ipNet.Mask[3]) } var hexMask []rune for i, r := range ipNet.Mask.String() { if i%4 == 0 && i != 0 { hexMask = append(hexMask, ':') } hexMask = append(hexMask, r) } // Shorten into canonical form. return net.ParseIP(string(hexMask)).String() } // subnetPrefixHex returns the hex string which prefixes a subnet (e.g. the hex prefix of "fd25:c7e3:5dec:e4dd:ef14::1/64" // is "fd25c7e35dece4dd"). Only for use with IPv6 networks. func subnetPrefixHex(ipNet *net.IPNet) (string, error) { if ipNet == nil || ipNet.IP.To4() != nil { return "", errors.New("Cannot create a hex prefix for empty or IPv4 subnets") } hexStr := hex.EncodeToString(ipNet.IP) ones, _ := ipNet.Mask.Size() if ones%8 != 0 { return "", errors.New("Cannot create a hex prefix for an IPv6 subnet whose CIDR range is not divisible by 8") } return hexStr[:ones/4], nil } incus-7.0.0/internal/server/firewall/drivers/drivers_util_test.go000066400000000000000000000057751517523235500253730ustar00rootroot00000000000000package drivers import ( "log" "testing" "github.com/stretchr/testify/assert" ) func Test_portRangesFromSlice(t *testing.T) { tests := []struct { name string ports []uint64 expected [][2]uint64 }{ { name: "Single port", ports: []uint64{80}, expected: [][2]uint64{{80, 1}}, }, { name: "Single range", ports: []uint64{80, 81, 82, 83}, expected: [][2]uint64{{80, 4}}, }, { name: "Multiple (single) ports", ports: []uint64{80, 90, 100}, expected: [][2]uint64{ {80, 1}, {90, 1}, {100, 1}, }, }, { name: "Multiple ranges", ports: []uint64{80, 81, 82, 90, 91, 92, 100, 101, 102}, expected: [][2]uint64{ {80, 3}, {90, 3}, {100, 3}, }, }, { name: "Mixed ranges and single ports", ports: []uint64{80, 81, 82, 87, 90, 91, 92, 88, 100, 101, 102, 89}, expected: [][2]uint64{ {80, 3}, {87, 1}, {90, 3}, {88, 1}, {100, 3}, {89, 1}, }, }, } for i, tt := range tests { log.Printf("Running test #%d: %s", i, tt.name) ranges := portRangesFromSlice(tt.ports) assert.ElementsMatch(t, ranges, tt.expected) } } func Test_getOptimisedSNATRanges(t *testing.T) { tests := []struct { name string forward *AddressForward expected map[[2]uint64][2]uint64 }{ { name: "Equal ports (single)", forward: &AddressForward{ ListenPorts: []uint64{80}, TargetPorts: []uint64{80}, }, expected: map[[2]uint64][2]uint64{ {80, 1}: {80, 1}, }, }, { name: "Equal ports (range)", forward: &AddressForward{ ListenPorts: []uint64{80, 81, 82, 83}, TargetPorts: []uint64{80, 81, 82, 83}, }, expected: map[[2]uint64][2]uint64{ {80, 4}: {80, 4}, }, }, { name: "Unequal ports (single)", forward: &AddressForward{ ListenPorts: []uint64{80}, TargetPorts: []uint64{8080}, }, expected: map[[2]uint64][2]uint64{ {80, 1}: {8080, 1}, }, }, { name: "Unequal ports (range)", forward: &AddressForward{ ListenPorts: []uint64{80, 81, 82, 83}, TargetPorts: []uint64{90, 91, 92, 93}, }, expected: map[[2]uint64][2]uint64{ {80, 1}: {90, 1}, {81, 1}: {91, 1}, {82, 1}: {92, 1}, {83, 1}: {93, 1}, }, }, { name: "Unequal ports (range)", forward: &AddressForward{ ListenPorts: []uint64{80, 81, 82, 83}, TargetPorts: []uint64{90, 91, 92, 93}, }, expected: map[[2]uint64][2]uint64{ {80, 1}: {90, 1}, {81, 1}: {91, 1}, {82, 1}: {92, 1}, {83, 1}: {93, 1}, }, }, { name: "Mixed ranges and single ports", forward: &AddressForward{ ListenPorts: []uint64{80, 81, 82, 83, 200, 201, 202, 203, 100, 101}, TargetPorts: []uint64{80, 81, 110, 120, 200, 201, 202, 203, 100, 101}, }, expected: map[[2]uint64][2]uint64{ {80, 2}: {80, 2}, {82, 1}: {110, 1}, {83, 1}: {120, 1}, {200, 4}: {200, 4}, {100, 2}: {100, 2}, }, }, } for _, tt := range tests { actual := getOptimisedDNATRanges(tt.forward) assert.Equal(t, tt.expected, actual) } } incus-7.0.0/internal/server/firewall/firewall_interface.go000066400000000000000000000035761517523235500237650ustar00rootroot00000000000000package firewall import ( "net" "github.com/lxc/incus/v7/internal/server/firewall/drivers" ) // FirewallRules represents a set of firewall rules. type FirewallRules struct { Rules []drivers.ACLRule IngressAction string IngressLogged bool EgressAction string EgressLogged bool } // Firewall represents an Incus firewall. type Firewall interface { String() string Compat() (bool, error) NetworkSetup(networkName string, opts drivers.Opts) error NetworkClear(networkName string, removeChains bool, ipVersions []uint) error NetworkApplyACLRules(networkName string, rules []drivers.ACLRule) error NetworkApplyForwards(networkName string, rules []drivers.AddressForward) error NetworkApplyAddressSets(sets []drivers.AddressSet, nftTable string) error NetworkDeleteAddressSetsIfUnused(nftTable string) error InstanceSetupBridgeFilter(projectName string, instanceName string, deviceName string, parentName string, hostName string, hwAddr string, IPv4Nets []*net.IPNet, IPv6Nets []*net.IPNet, IPv4DNS []string, IPv6DNS []string, parentManaged bool, macFiltering bool, aclRules []drivers.ACLRule) error InstanceClearBridgeFilter(projectName string, instanceName string, deviceName string, parentName string, hostName string, hwAddr string, IPv4Nets []*net.IPNet, IPv6Nets []*net.IPNet) error InstanceSetupProxyNAT(projectName string, instanceName string, deviceName string, forward *drivers.AddressForward) error InstanceClearProxyNAT(projectName string, instanceName string, deviceName string) error InstanceSetupRPFilter(projectName string, instanceName string, deviceName string, hostName string) error InstanceClearRPFilter(projectName string, instanceName string, deviceName string) error InstanceSetupNetPrio(projectName string, instanceName string, deviceName string, netPrio uint32) error InstanceClearNetPrio(projectName string, instanceName string, deviceName string) error } incus-7.0.0/internal/server/firewall/firewall_load.go000066400000000000000000000006451517523235500227360ustar00rootroot00000000000000package firewall import ( "github.com/lxc/incus/v7/internal/server/firewall/drivers" "github.com/lxc/incus/v7/shared/logger" ) // New returns the nftables firewall implementation. func New() Firewall { nftables := drivers.Nftables{} _, err := nftables.Compat() if err != nil { logger.Warnf(`Firewall detected "nftables" incompatibility (some features may not work as expected): %v`, err) } return nftables } incus-7.0.0/internal/server/fsmonitor/000077500000000000000000000000001517523235500200115ustar00rootroot00000000000000incus-7.0.0/internal/server/fsmonitor/drivers/000077500000000000000000000000001517523235500214675ustar00rootroot00000000000000incus-7.0.0/internal/server/fsmonitor/drivers/common.go000066400000000000000000000032521517523235500233100ustar00rootroot00000000000000package drivers import ( "path/filepath" "strings" "sync" "github.com/lxc/incus/v7/shared/logger" ) type common struct { logger logger.Logger mu sync.Mutex watches map[string]map[string]func(string, string) bool prefixPath string } func (d *common) init(logger logger.Logger, path string) { d.logger = logger d.watches = make(map[string]map[string]func(string, string) bool) d.prefixPath = path } // PrefixPath returns the prefix path. func (d *common) PrefixPath() string { return d.prefixPath } // Watch creates a watch for a path which may or may not yet exist. If the provided path gets an // inotify event, f() is called. If there already is a watch on the provided path, the callback // function will simply be replaced without returning an error. // Note: If f() returns false, the watch is removed. func (d *common) Watch(path string, identifier string, f func(path string, event string) bool) error { if f == nil { return ErrInvalidFunction } path = filepath.Clean(path) if !strings.HasPrefix(path, d.prefixPath) { return &ErrInvalidPath{PrefixPath: d.prefixPath} } d.mu.Lock() defer d.mu.Unlock() _, ok := d.watches[path] if !ok { d.watches[path] = make(map[string]func(string, string) bool) } _, ok = d.watches[path][identifier] if ok { return ErrWatchExists } d.watches[path][identifier] = f return nil } // Unwatch removes a watch. func (d *common) Unwatch(path string, identifier string) error { d.mu.Lock() defer d.mu.Unlock() path = filepath.Clean(path) _, ok := d.watches[path] if !ok { return nil } delete(d.watches[path], identifier) if len(d.watches[path]) == 0 { delete(d.watches, path) } return nil } incus-7.0.0/internal/server/fsmonitor/drivers/driver_fanotify.go000066400000000000000000000114561517523235500252170ustar00rootroot00000000000000package drivers import ( "bytes" "context" "encoding/binary" "errors" "fmt" "os" "path/filepath" "strings" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/shared/logger" ) var fanotifyLoaded bool type fanotify struct { common fd int } type fanotifyEventInfoHeader struct { InfoType uint8 Pad uint8 Len uint16 } type fanotifyEventInfoFid struct { fanotifyEventInfoHeader FSID uint64 } func (d *fanotify) Name() string { return "fanotify" } func (d *fanotify) load(ctx context.Context) error { if fanotifyLoaded { return nil } var err error d.fd, err = unix.FanotifyInit(unix.FAN_CLOEXEC|unix.FAN_REPORT_DFID_NAME, unix.O_CLOEXEC) if err != nil { return fmt.Errorf("Failed to initialize fanotify: %w", err) } err = unix.FanotifyMark(d.fd, unix.FAN_MARK_ADD|unix.FAN_MARK_FILESYSTEM, unix.FAN_CREATE|unix.FAN_DELETE|unix.FAN_ONDIR, unix.AT_FDCWD, d.prefixPath) if err != nil { _ = unix.Close(d.fd) return fmt.Errorf("Failed to watch directory %q: %w", d.prefixPath, err) } fd, err := unix.Open(d.prefixPath, unix.O_DIRECTORY|unix.O_RDONLY|unix.O_CLOEXEC, 0) if err != nil { _ = unix.Close(d.fd) return fmt.Errorf("Failed to open directory %q: %w", d.prefixPath, err) } go func() { <-ctx.Done() _ = unix.Close(d.fd) fanotifyLoaded = false }() go d.getEvents(ctx, fd) fanotifyLoaded = true return nil } func (d *fanotify) getEvents(ctx context.Context, mountFd int) { for { buf := make([]byte, 4096) // Read enough bytes to handle multiple event records returned in one read call. n, err := unix.Read(d.fd, buf) if err != nil { // Stop listening for events as the fanotify fd has been closed due to cleanup. if ctx.Err() != nil || errors.Is(err, unix.EBADF) { _ = unix.Close(mountFd) return } d.logger.Error("Failed to read event", logger.Ctx{"err": err}) continue } processed := 0 for processed < n { rd := bytes.NewReader(buf[processed:]) event := unix.FanotifyEventMetadata{} err = binary.Read(rd, binary.LittleEndian, &event) if err != nil { d.logger.Error("Failed to read event metadata", logger.Ctx{"err": err}) continue } processed += int(event.Event_len) // Kernel queue overflow means events were dropped before userspace could read them. if event.Mask&unix.FAN_Q_OVERFLOW != 0 { d.logger.Warn("fanotify queue overflow detected, events may have been dropped") } // Read event info fid fid := fanotifyEventInfoFid{} err = binary.Read(rd, binary.LittleEndian, &fid) if err != nil { d.logger.Error("Failed to read event fid", logger.Ctx{"err": err}) continue } // Although unix.FileHandle exists, it cannot be used with binary.Read() as the // variables inside are not exported. type fileHandleInfo struct { Bytes uint32 Type int32 } // Read file handle information fhInfo := fileHandleInfo{} err = binary.Read(rd, binary.LittleEndian, &fhInfo) if err != nil { d.logger.Error("Failed to read file handle info", logger.Ctx{"err": err}) continue } // Read file handle fileHandle := make([]byte, fhInfo.Bytes) err = binary.Read(rd, binary.LittleEndian, fileHandle) if err != nil { d.logger.Error("Failed to read file handle", logger.Ctx{"err": err}) continue } fh := unix.NewFileHandle(fhInfo.Type, fileHandle) fd, err := unix.OpenByHandleAt(mountFd, fh, 0) if err != nil { if !errors.Is(err, unix.ESTALE) { d.logger.Error("Failed to open file", logger.Ctx{"err": err}) } continue } // Determine the directory of the created or deleted file. target, err := os.Readlink(fmt.Sprintf("/proc/self/fd/%d", fd)) _ = unix.Close(fd) if err != nil { d.logger.Error("Failed to read symlink", logger.Ctx{"err": err}) continue } // If the target file has been deleted, the returned value might contain a " (deleted)" suffix. // This needs to be removed. target = strings.TrimSuffix(target, " (deleted)") // The file handle is followed by a null terminated string that identifies the // created/deleted directory entry name. sb := strings.Builder{} sb.WriteString(target + "/") for { b, err := rd.ReadByte() if err != nil || b == 0 { break } err = sb.WriteByte(b) if err != nil { break } } eventPath := filepath.Clean(sb.String()) var action Event if event.Mask&unix.FAN_CREATE != 0 { action = Add } else if event.Mask&unix.FAN_DELETE != 0 || event.Mask&unix.FAN_DELETE_SELF != 0 { action = Remove } else { continue } d.mu.Lock() for identifier, f := range d.watches[eventPath] { ret := f(eventPath, action.String()) if !ret { delete(d.watches[eventPath], identifier) if len(d.watches[eventPath]) == 0 { delete(d.watches, eventPath) } } } d.mu.Unlock() } } } incus-7.0.0/internal/server/fsmonitor/drivers/driver_inotify.go000066400000000000000000000075311517523235500250600ustar00rootroot00000000000000package drivers import ( "context" "errors" "fmt" "os" "path/filepath" "strings" in "k8s.io/utils/inotify" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) var inotifyLoaded bool type inotify struct { common watcher *in.Watcher } func (d *inotify) Name() string { return "inotify" } func (d *inotify) load(ctx context.Context) error { if inotifyLoaded { return nil } var err error d.watcher, err = in.NewWatcher() if err != nil { return fmt.Errorf("Failed to initialize: %w", err) } err = d.watchFSTree(d.prefixPath) if err != nil { _ = d.watcher.Close() inotifyLoaded = false return fmt.Errorf("Failed to watch directory %q: %w", d.prefixPath, err) } go d.getEvents(ctx) inotifyLoaded = true return nil } func (d *inotify) getEvents(ctx context.Context) { for { select { // Clean up if context is done. case <-ctx.Done(): _ = d.watcher.Close() inotifyLoaded = false return case event := <-d.watcher.Event: event.Name = filepath.Clean(event.Name) isCreate := event.Mask&in.InCreate != 0 // codespell:ignore increate isDelete := event.Mask&in.InDelete != 0 // Only consider create and delete events. if !isCreate && !isDelete { continue } // New event for a directory. if event.Mask&in.InIsdir != 0 { // If it's a create event, then setup watches on any sub-directories. if isCreate { _ = d.watchFSTree(event.Name) } // Check whether there's a watch on the directory. d.mu.Lock() var action Event if isCreate { action = Add } else { action = Remove } for path := range d.watches { // Always call the handlers that have a prefix of the event path, // in case a watched file is inside the newly created or now deleted // directory, otherwise we'll miss the event. The handlers themselves are // expected to check the state of the specific path they are interested in. if !strings.HasPrefix(path, event.Name) { continue } for identifier, f := range d.watches[path] { ret := f(path, action.String()) if !ret { delete(d.watches[path], identifier) if len(d.watches[path]) == 0 { delete(d.watches, path) } } } } d.mu.Unlock() continue } // Check whether there's a watch on a specific file or directory. d.mu.Lock() var action Event if isCreate { action = Add } else { action = Remove } for path := range d.watches { if event.Name != path { continue } for identifier, f := range d.watches[path] { ret := f(path, action.String()) if !ret { delete(d.watches[path], identifier) if len(d.watches[path]) == 0 { delete(d.watches, path) } } } break } d.mu.Unlock() case err := <-d.watcher.Error: d.logger.Error("Received event error", logger.Ctx{"err": err}) } } } func (d *inotify) watchFSTree(path string) error { if !util.PathExists(path) { return errors.New("Path doesn't exist") } err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { // Check for errors here as we only care about directories. Files and symlinks aren't of interest for this. if err != nil { if os.IsPermission(err) { return nil } d.logger.Warn("Error visiting path", logger.Ctx{"path": path, "err": err}) return nil } // Ignore files and symlinks. if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { return nil } // Only watch on real paths for CREATE and DELETE events. err = d.watcher.AddWatch(path, in.InCreate|in.InDelete) // codespell:ignore increate if err != nil { d.logger.Warn("Failed to watch path", logger.Ctx{"path": path, "err": err}) return nil } return nil }) if err != nil { return fmt.Errorf("Failed to watch directory tree: %w", err) } return nil } incus-7.0.0/internal/server/fsmonitor/drivers/errors.go000066400000000000000000000011431517523235500233310ustar00rootroot00000000000000package drivers import ( "errors" "fmt" ) // ErrInvalidFunction is the "Invalid function" error. var ErrInvalidFunction = errors.New("Invalid function") // ErrUnknownDriver is the "Unknown driver" error. var ErrUnknownDriver = errors.New("Unknown driver") // ErrWatchExists is the "Watch already exists" error. var ErrWatchExists = errors.New("Watch already exists") // ErrInvalidPath is the "Invalid path" error. type ErrInvalidPath struct { PrefixPath string } // Error returns the error string. func (e *ErrInvalidPath) Error() string { return fmt.Sprintf("Path needs to be in %s", e.PrefixPath) } incus-7.0.0/internal/server/fsmonitor/drivers/events.go000066400000000000000000000004521517523235500233230ustar00rootroot00000000000000package drivers // Event is a numeric code identifying the event. type Event int const ( // Add represents the add event. Add Event = iota // Remove represents the remove event. Remove ) func (e Event) String() string { return map[Event]string{ Add: "add", Remove: "remove", }[e] } incus-7.0.0/internal/server/fsmonitor/drivers/interface.go000066400000000000000000000007511517523235500237610ustar00rootroot00000000000000package drivers import ( "context" "github.com/lxc/incus/v7/shared/logger" ) // driver is the extended internal interface. type driver interface { Driver init(logger logger.Logger, path string) load(ctx context.Context) error } // Driver represents a low-level fs notification driver. type Driver interface { Name() string PrefixPath() string Watch(path string, identifier string, f func(path string, event string) bool) error Unwatch(path string, identifier string) error } incus-7.0.0/internal/server/fsmonitor/drivers/load.go000066400000000000000000000011031517523235500227300ustar00rootroot00000000000000package drivers import ( "context" "github.com/lxc/incus/v7/shared/logger" ) var drivers = map[string]func() driver{ "inotify": func() driver { return &inotify{} }, "fanotify": func() driver { return &fanotify{} }, } // Load returns a Driver for an existing low-level FS monitor. func Load(ctx context.Context, logger logger.Logger, driverName string, path string) (Driver, error) { df, ok := drivers[driverName] if !ok { return nil, ErrUnknownDriver } d := df() d.init(logger, path) err := d.load(ctx) if err != nil { return nil, err } return d, nil } incus-7.0.0/internal/server/fsmonitor/fsmonitor.go000066400000000000000000000016661517523235500223710ustar00rootroot00000000000000package fsmonitor import ( "github.com/lxc/incus/v7/internal/server/fsmonitor/drivers" "github.com/lxc/incus/v7/shared/logger" ) type fsMonitor struct { driver drivers.Driver logger logger.Logger } // PrefixPath returns the prefix path. func (fs *fsMonitor) PrefixPath() string { return fs.driver.PrefixPath() } // Watch creates a watch for a path which may or may not yet exist. If the provided path gets an // inotify event, f() is called. // Note: If f() returns false, the watch is removed. func (fs *fsMonitor) Watch(path string, identifier string, f func(path string, event string) bool) error { fs.logger.Info("Watching path", logger.Ctx{"path": path}) return fs.driver.Watch(path, identifier, f) } // Unwatch removes the given path from the watchlist. func (fs *fsMonitor) Unwatch(path string, identifier string) error { fs.logger.Info("Unwatching path", logger.Ctx{"path": path}) return fs.driver.Unwatch(path, identifier) } incus-7.0.0/internal/server/fsmonitor/fsmonitor_interface.go000066400000000000000000000003701517523235500244000ustar00rootroot00000000000000package fsmonitor // FSMonitor represents a filesystem monitor. type FSMonitor interface { PrefixPath() string Watch(path string, identifier string, f func(path string, event string) bool) error Unwatch(path string, identifier string) error } incus-7.0.0/internal/server/fsmonitor/load.go000066400000000000000000000021561517523235500212630ustar00rootroot00000000000000package fsmonitor import ( "context" "errors" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/fsmonitor/drivers" "github.com/lxc/incus/v7/shared/logger" ) // New creates a new FSMonitor instance. func New(ctx context.Context, path string) (FSMonitor, error) { startMonitor := func(driverName string) (drivers.Driver, logger.Logger, error) { logger := logger.AddContext(logger.Ctx{"driver": driverName}) driver, err := drivers.Load(ctx, logger, driverName, path) if err != nil { return nil, nil, err } return driver, logger, nil } if !linux.IsMountPoint(path) { return nil, errors.New("Path needs to be a mountpoint") } driver, monLogger, err := startMonitor("fanotify") if err != nil { logger.Warn("Failed to initialize fanotify, falling back on inotify", logger.Ctx{"err": err}) driver, monLogger, err = startMonitor("inotify") if err != nil { return nil, err } } logger.Info("Initialized filesystem monitor", logger.Ctx{"path": path, "driver": driver.Name()}) monitor := fsMonitor{ driver: driver, logger: monLogger, } return &monitor, nil } incus-7.0.0/internal/server/instance/000077500000000000000000000000001517523235500175755ustar00rootroot00000000000000incus-7.0.0/internal/server/instance/drivers/000077500000000000000000000000001517523235500212535ustar00rootroot00000000000000incus-7.0.0/internal/server/instance/drivers/agent-loader/000077500000000000000000000000001517523235500236155ustar00rootroot00000000000000incus-7.0.0/internal/server/instance/drivers/agent-loader/incus-agent-freebsd000066400000000000000000000014311517523235500273640ustar00rootroot00000000000000#!/bin/sh set -eu PREFIX=/var/run/incus_agent mkdir -p "${PREFIX}/.mnt" # Functions. mount_9p() { kldload virtio_p9fs >/dev/null 2>&1 || true mount -t p9fs -o ro agent "${PREFIX}/.mnt" >/dev/null 2>&1 } fail() { echo "$1, failing" exit 1 } # Mount the agent share. mount_9p || fail "Couldn't mount 9p, failing." # Guess a Linux-style architecture. ARCH="$(uname -m)" if [ "$ARCH" = amd64 ]; then ARCH=x86_64 elif [ "$ARCH" = arm64 ]; then ARCH=aarch64 fi # Transfer the agent binary. rm -f "${PREFIX}/incus-agent" cp -a "${PREFIX}/.mnt/incus-agent.freebsd.$ARCH" "${PREFIX}/incus-agent" chown root:wheel "${PREFIX}/incus-agent" # Unmount the temporary mount. umount "${PREFIX}/.mnt" rmdir "${PREFIX}/.mnt" # Re-exec the agent. exec "${PREFIX}/incus-agent" "$@" incus-7.0.0/internal/server/instance/drivers/agent-loader/incus-agent-linux000066400000000000000000000013451517523235500271150ustar00rootroot00000000000000#!/bin/sh PREFIX="/run/incus_agent" # Legacy handling if [ ! -e "${PREFIX}" ] && [ -d "/run/lxd_agent" ]; then ln -s "/run/lxd_agent" "${PREFIX}" fi mkdir -p "${PREFIX}/.mnt" # Functions. mount_9p() { modprobe 9pnet_virtio >/dev/null 2>&1 || true mount -t 9p agent "${PREFIX}/.mnt" -o ro,access=0,trans=virtio,size=1048576 >/dev/null 2>&1 } # Mount the agent share. mount_9p || fail "Couldn't mount 9p, failing." # Transfer the agent binary. rm -f "${PREFIX}/incus-agent" cp -a "${PREFIX}/.mnt/incus-agent.linux.$(uname -m)" "${PREFIX}/incus-agent" chown root:root "${PREFIX}/incus-agent" # Unmount the temporary mount. umount "${PREFIX}/.mnt" rmdir "${PREFIX}/.mnt" # Re-exec the agent. exec "${PREFIX}/incus-agent" "$@" incus-7.0.0/internal/server/instance/drivers/agent-loader/incus-agent-macos000066400000000000000000000007361517523235500270630ustar00rootroot00000000000000#!/bin/sh set -eu SHARE=agent VOLUME="/Volumes/$SHARE" PREFIX=/var/run/incus_agent mkdir -p "$PREFIX" # Mount the agent share. if [ ! -e "$VOLUME" ]; then mount_9p "$SHARE" || fail "Couldn't mount 9p" fi # Transfer the agent binary. rm -f "$PREFIX/incus-agent" cp -a "$VOLUME/incus-agent.macos.$(uname -m)" "$PREFIX/incus-agent" chown root:wheel "$PREFIX/incus-agent" # Unmount the temporary mount. umount "$VOLUME" # Re-exec the agent. exec "$PREFIX/incus-agent" "$@" incus-7.0.0/internal/server/instance/drivers/agent-loader/incus-agent-setup-freebsd000066400000000000000000000017071517523235500305300ustar00rootroot00000000000000#!/bin/sh set -eu PREFIX="/var/run/incus_agent" # Functions. mount_9p() { kldload virtio_p9fs >/dev/null 2>&1 || true mount -t p9fs -o ro config "$PREFIX.mnt" >/dev/null 2>&1 } fail() { # Check if we already have an agent in place. if [ -x "$PREFIX/incus-agent" ]; then echo "$1, reusing existing agent" exit 0 fi # Cleanup and fail. umount "$PREFIX" >/dev/null 2>&1 || true rmdir "$PREFIX" >/dev/null 2>&1 || true echo "$1, failing" exit 1 } # Try getting an agent drive. mkdir -p "$PREFIX.mnt" mount_9p || fail "Couldn't mount 9p" # Setup the mount target. umount "$PREFIX" >/dev/null 2>&1 || true mkdir -p "$PREFIX" mount -t tmpfs -o mode=0700,size=50M tmpfs "$PREFIX" # Copy the data. cp -Ra "$PREFIX.mnt/"* "$PREFIX" # Unmount the temporary mount. umount "$PREFIX.mnt" rmdir "$PREFIX.mnt" # Fix up permissions. chown -R root:wheel "$PREFIX" # Load pty kldload pty >/dev/null 2>&1 || true exit 0 incus-7.0.0/internal/server/instance/drivers/agent-loader/incus-agent-setup-linux000066400000000000000000000030151517523235500302470ustar00rootroot00000000000000#!/bin/sh set -eu PREFIX="/run/incus_agent" CDROM="/dev/disk/by-label/incus-agent" # Functions. mount_cdrom() { mount "${CDROM}" "${PREFIX}.mnt" >/dev/null 2>&1 } mount_9p() { modprobe 9pnet_virtio >/dev/null 2>&1 || true mount -t 9p config "${PREFIX}.mnt" -o access=0,trans=virtio,size=1048576 >/dev/null 2>&1 } fail() { # Check if we already have an agent in place. # This will typically be true during restart in the case of a cdrom-based setup. if [ -x "${PREFIX}/incus-agent" ]; then echo "${1}, reusing existing agent" exit 0 fi # Cleanup and fail. umount -l "${PREFIX}" >/dev/null 2>&1 || true eject "${CDROM}" >/dev/null 2>&1 || true rmdir "${PREFIX}" >/dev/null 2>&1 || true echo "${1}, failing" exit 1 } # Try getting an agent drive. mkdir -p "${PREFIX}.mnt" mount_9p || mount_cdrom || fail "Couldn't mount 9p or cdrom" # Setup the mount target. umount -l "${PREFIX}" >/dev/null 2>&1 || true mkdir -p "${PREFIX}" mount -t tmpfs tmpfs "${PREFIX}" -o mode=0700,size=50M # Copy the data. cp -Ra "${PREFIX}.mnt/"* "${PREFIX}" # Unmount the temporary mount. umount "${PREFIX}.mnt" rmdir "${PREFIX}.mnt" # Eject the cdrom in case it's present. eject "${CDROM}" >/dev/null 2>&1 || true # Fix up permissions. chown -R root:root "${PREFIX}" # Legacy. if [ ! -e "${PREFIX}/incus-agent" ] && [ -e "${PREFIX}/lxd-agent" ]; then ln -s lxd-agent "${PREFIX}"/incus-agent fi # Attempt to restore SELinux labels. restorecon -R "${PREFIX}" >/dev/null 2>&1 || true exit 0 incus-7.0.0/internal/server/instance/drivers/agent-loader/incus-agent-setup-macos000066400000000000000000000016051517523235500302150ustar00rootroot00000000000000#!/bin/sh set -eu SHARE=config VOLUME="/Volumes/$SHARE" PREFIX=/var/run/incus_agent # Functions. fail() { # Check if we already have an agent in place. if [ -x "$PREFIX/incus-agent" ]; then echo "$1, reusing existing agent" cd "$PREFIX" ./incus-agent || true exit 0 fi # Cleanup and fail. umount "$PREFIX" >/dev/null 2>&1 || true rmdir "${PREFIX}" >/dev/null 2>&1 || true echo "${1}, failing" exit 1 } # Try getting an agent drive. if [ ! -e "$VOLUME" ]; then mount_9p "$SHARE" || fail "Couldn't mount 9p" fi # Setup the mount target. umount "$PREFIX" >/dev/null 2>&1 || true mkdir -p "$PREFIX" mount_tmpfs -s 50M "$PREFIX" chmod 0700 "$PREFIX" # Copy the data. cp -RaX "$VOLUME/"* "$PREFIX" # Unmount the temporary mount. umount "$VOLUME" # Fix up permissions. chown -R root:wheel "$PREFIX" cd "$PREFIX" exec ./incus-agent incus-7.0.0/internal/server/instance/drivers/agent-loader/incus-agent-setup.ps1000066400000000000000000000055701517523235500276240ustar00rootroot00000000000000# Variables setup # Installation folder in ProgramData $destFolder = "C:\ProgramData\Incus-Agent" $agentExecutable = "incus-agent.exe" $serviceName = "Incus-Agent" $serviceDisplayName = "Incus Agent Service" $serviceDescription = "Incus Agent Service" function ExitSetup { # Recursively delete the old agent as we only want the agent to run if the CDROM is present. # Failsafe in case it was not deleted on shutdown. # Close the firewall. Remove-NetFirewallRule -Name $serviceName -ErrorAction SilentlyContinue # Stop the service in case it was running. Stop-Service $serviceName -Force # Delete the service. sc.exe delete $serviceName # Delete all files Remove-Item -Path "$destFolder" -Recurse -Force } $targetDrive = Get-WmiObject -Class Win32_Volume | Where-Object { $_.Label -eq "incus-agent" } if (!$targetDrive) { Write-Host "Drive containing the agent was not found." ExitSetup } Write-Host "Drive containing the agent was found: $($targetDrive.DriveLetter)" if (!(Test-Path $destFolder)) { Write-Host "Creating $destFolder..." New-Item -ItemType Directory -Path $destFolder -Force | Out-Null if (!$?) { Write-Host "Could not create $destFolder..." ExitSetup } } Write-Host "Copying the content of the CD-ROM to $destFolder..." Copy-Item ` -Recurse ` -Path "$($targetDrive.Name)*" ` -Destination $destFolder ` -Exclude '*.ps1', '*.bat' ` -Force if (!$?) { Write-Host "Failed to copy the agent files." ExitSetup } Write-Host "Ejecting CD-ROM..." (New-Object -ComObject Shell.Application).Namespace(17).ParseName($targetDrive.DriveLetter).InvokeVerb("Eject") # Dumb search for firewall rule assuming the name of the rule is "$serviceName". if (!(Get-NetFirewallRule -Name "$serviceName" -ErrorAction SilentlyContinue)) { New-NetFirewallRule -Name "$serviceName" -DisplayName "Allow Port 8443 for Incus-Agent" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8443 } $serviceCommand = "`"$destFolder\$agentExecutable`" --service --secrets-location $destFolder" if (Get-Service -Name $serviceName -ErrorAction SilentlyContinue) { Write-Host "Service exists. Updating configuration..." # Do not Out-Null to see if there is any error doing the following command as it is important. sc.exe config $serviceName binPath= "$serviceCommand" DisplayName= "$serviceDisplayName" sc.exe description $serviceName "$serviceDescription" | Out-Null Set-Service -Name $serviceName -StartupType Manual Write-Host "Service '$serviceName' updated successfully." } else { Write-Host "Service does not exist. Creating new service..." New-Service -Name $serviceName -BinaryPathName $serviceCommand -DisplayName $serviceDisplayName -Description $serviceDescription -StartupType Manual Write-Host "Service '$serviceName' created successfully." } Restart-Service $serviceName -Force incus-7.0.0/internal/server/instance/drivers/agent-loader/install-freebsd.sh000066400000000000000000000007541517523235500272350ustar00rootroot00000000000000#!/bin/sh set -eu if [ ! -e "rc.d" ] || [ ! -e "incus-agent" ]; then echo "This script must be run from within the 9p mount" exit 1 fi # Install the service. mkdir -p /usr/local/etc/rc.d mkdir -p /usr/local/libexec cp rc.d/incus-agent /usr/local/etc/rc.d/ cp incus-agent-setup /usr/local/libexec/ sysrc incus_agent_enable=YES echo "" echo "Incus agent has been installed, reboot to confirm setup." echo "To start it now, unmount this filesystem and run: service incus-agent start" incus-7.0.0/internal/server/instance/drivers/agent-loader/install-linux.sh000066400000000000000000000026551517523235500267640ustar00rootroot00000000000000#!/bin/sh if [ ! -e "systemd" ] || [ ! -e "incus-agent" ]; then echo "This script must be run from within the 9p mount" exit 1 fi # Find target path. TARGET="" for alternative in /usr/lib /lib /etc; do [ -w "${alternative}/systemd" ] || continue [ -w "${alternative}/udev" ] || continue TARGET="${alternative}" break done if [ "${TARGET}" = "" ]; then echo "This script only works on systemd systems" exit 1 fi echo "Installing agent into ${TARGET}" # Install the units. cp udev/99-incus-agent.rules "${TARGET}/udev/rules.d/" cp systemd/incus-agent.service "${TARGET}/systemd/system/" cp systemd/incus-agent-setup "${TARGET}/systemd/" # Replacing the variables. sed -i "s#TARGET#${TARGET}#g" "${TARGET}/udev/rules.d/99-incus-agent.rules" sed -i "s#TARGET#${TARGET}#g" "${TARGET}/systemd/system/incus-agent.service" sed -i "s#TARGET#${TARGET}#g" "${TARGET}/systemd/incus-agent-setup" # Make sure systemd is aware of them. systemctl daemon-reload # SELinux handling. if getenforce >/dev/null 2>&1 && type semanage >/dev/null 2>&1; then # Run semanage for both /var/run and /run due to different distro policies for run_path in /var/run /run; do semanage fcontext -a -t bin_t "${run_path}/incus_agent/incus-agent" >/dev/null 2>&1 done fi echo "" echo "Incus agent has been installed, reboot to confirm setup." echo "To start it now, unmount this filesystem and run: systemctl start incus-agent" incus-7.0.0/internal/server/instance/drivers/agent-loader/install-macos.sh000066400000000000000000000016701517523235500267230ustar00rootroot00000000000000#!/bin/sh set -eu if [ ! -e "launchd" ] || [ ! -e "incus-agent" ]; then echo "This script must be run from within the 9p mount" exit 1 fi AGENT=org.linuxcontainers.incus.macos-agent echo "Installing agent" # Uninstall the previous daemon. if launchctl print "system/$AGENT" >/dev/null 2>&1; then launchctl bootout system "/Library/LaunchDaemons/$AGENT.plist" || true fi # Install the daemon. cp "launchd/$AGENT.plist" /Library/LaunchDaemons/ chown root:wheel "/Library/LaunchDaemons/$AGENT.plist" mkdir -p /usr/local/bin cp incus-agent-setup /usr/local/bin/ chown root:wheel /usr/local/bin/incus-agent-setup # Bootstrap it. launchctl bootstrap system "/Library/LaunchDaemons/$AGENT.plist" # Enable it. launchctl enable "system/$AGENT" echo "" echo "Incus agent has been installed, reboot to confirm setup." echo "To start it now, run: sudo launchctl kickstart -k system/$AGENT" echo "Don't forget to allow full disk access to sh." incus-7.0.0/internal/server/instance/drivers/agent-loader/install.ps1000066400000000000000000000035611517523235500257150ustar00rootroot00000000000000$IsAdmin = ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) if (!$IsAdmin) { Write-Host "This script requires local administrator privilege. Rerun the script in an administrator PowerShell." exit 1 } $destFolder = "C:\Program Files\Incus-Agent" $targetDrive = Get-WmiObject -Class Win32_Volume | Where-Object { $_.Label -eq "incus-agent" } if (!$targetDrive) { Write-Host "Drive containing the agent was not found." exit 1 } Write-Host "Drive containing the agent was found: $($targetDrive.DriveLetter)" if (!(Test-Path $destFolder)) { Write-Host "Creating $destFolder..." New-Item -ItemType Directory -Path $destFolder -Force | Out-Null if (!$?) { Write-Host "Could not create $destFolder..." exit 1 } } Write-Host "Copying the content of the CD-ROM to $destFolder..." Copy-Item ` -Path "$($targetDrive.DriveLetter)\incus-agent-setup.*" ` -Destination "$destFolder\" ` -Force if (!$?) { Write-Host "Failed to copy the agent files." exit 1 } # Override the scheduled task even if it exists $destFolder = "C:\Program Files\Incus-Agent" $taskFile = "$destFolder\incus-agent-setup.ps1" $taskAction = New-ScheduledTaskAction -Execute "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe" -Argument "-ExecutionPolicy Bypass -File `"$taskFile`"" $taskTrigger = New-ScheduledTaskTrigger -AtStartup $taskPrincipal = New-ScheduledTaskPrincipal -UserID "NT AUTHORITY\SYSTEM" -LogonType ServiceAccount -RunLevel Highest Register-ScheduledTask -Action $taskAction -Trigger $taskTrigger -Principal $taskPrincipal -TaskName "Incus Agent Setup" -Description "Every setup required for the Incus agent including copying the files, opening the firewall, etc." -Force # Start the PowerShell script to simulate start up & "$taskFile" incus-7.0.0/internal/server/instance/drivers/agent-loader/launchd/000077500000000000000000000000001517523235500252335ustar00rootroot00000000000000org.linuxcontainers.incus.macos-agent.plist000066400000000000000000000007221517523235500355620ustar00rootroot00000000000000incus-7.0.0/internal/server/instance/drivers/agent-loader/launchd Label org.linuxcontainers.incus.macos-agent Program /usr/local/bin/incus-agent-setup ProcessType Adaptive KeepAlive ThrottleInterval 5 incus-7.0.0/internal/server/instance/drivers/agent-loader/rc.d/000077500000000000000000000000001517523235500244435ustar00rootroot00000000000000incus-7.0.0/internal/server/instance/drivers/agent-loader/rc.d/incus-agent000066400000000000000000000007771517523235500266160ustar00rootroot00000000000000#!/bin/sh # # PROVIDE: incus_agent # REQUIRE: FILESYSTEMS # . /etc/rc.subr name=incus_agent rcvar=incus_agent_enable pidfile="/var/run/${name}.pid" procname=/usr/sbin/daemon start_precmd="${name}_prestart" start_cmd="${name}_start" load_rc_config "$name" : ${incus_agent_enable:="NO"} incus_agent_prestart() { /usr/local/libexec/incus-agent-setup } incus_agent_start() { cd /var/run/incus_agent || return 1 /usr/sbin/daemon -P "${pidfile}" -r -f /var/run/incus_agent/incus-agent } run_rc_command "$1" incus-7.0.0/internal/server/instance/drivers/agent-loader/systemd/000077500000000000000000000000001517523235500253055ustar00rootroot00000000000000incus-7.0.0/internal/server/instance/drivers/agent-loader/systemd/incus-agent.rules000066400000000000000000000001551517523235500305770ustar00rootroot00000000000000SYMLINK=="virtio-ports/org.linuxcontainers.incus", TAG+="systemd", ENV{SYSTEMD_WANTS}+="incus-agent.service" incus-7.0.0/internal/server/instance/drivers/agent-loader/systemd/incus-agent.service000066400000000000000000000006431517523235500311070ustar00rootroot00000000000000[Unit] Description=Incus - agent Documentation=https://linuxcontainers.org/incus/docs/main/ Before=multi-user.target cloud-init.target cloud-init.service cloud-init-local.service DefaultDependencies=no [Service] Type=notify WorkingDirectory=-/run/incus_agent ExecStartPre=TARGET/systemd/incus-agent-setup ExecStart=/run/incus_agent/incus-agent Restart=on-failure RestartSec=5s StartLimitInterval=60 StartLimitBurst=10 incus-7.0.0/internal/server/instance/drivers/cfg/000077500000000000000000000000001517523235500220125ustar00rootroot00000000000000incus-7.0.0/internal/server/instance/drivers/cfg/cfg.go000066400000000000000000000003241517523235500230770ustar00rootroot00000000000000package cfg // Section holds QEMU configuration sections. type Section struct { Name string `json:"name"` Comment string `json:"comment"` Entries map[string]string `json:"entries"` } incus-7.0.0/internal/server/instance/drivers/driver_common.go000066400000000000000000001504251517523235500244540ustar00rootroot00000000000000package drivers import ( "cmp" "context" "errors" "fmt" "math" "math/rand/v2" "net/http" "os" "path/filepath" "slices" "sort" "strconv" "strings" "sync" "syscall" "time" "github.com/google/uuid" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/device" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/device/nictype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/instance/operationlock" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/locking" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) // Track last autorestart of an instance. var ( instancesLastRestart = map[int][10]time.Time{} muInstancesLastRestart sync.Mutex ) // ErrExecCommandNotFound indicates the command is not found. var ErrExecCommandNotFound = api.StatusErrorf(http.StatusBadRequest, "Command not found") // ErrExecCommandNotExecutable indicates the command is not executable. var ErrExecCommandNotExecutable = api.StatusErrorf(http.StatusBadRequest, "Command not executable") // ErrInstanceIsStopped indicates that the instance is stopped. var ErrInstanceIsStopped error = api.StatusErrorf(http.StatusBadRequest, "The instance is already stopped") // muNUMA is used to serialize NUMA node selection. var muNUMA sync.Mutex // deviceManager is an interface that allows managing device lifecycle. type deviceManager interface { deviceAdd(dev device.Device, instanceRunning bool) error deviceRemove(dev device.Device, instanceRunning bool, cleanupDependencies bool) error deviceStart(dev device.Device, instanceRunning bool) (*deviceConfig.RunConfig, error) deviceStop(dev device.Device, instanceRunning bool, stopHookNetnsPath string) error } // common provides structure common to all instance types. type common struct { op *operations.Operation state *state.State architecture int creationDate time.Time dbType instancetype.Type description string ephemeral bool expandedConfig map[string]string expandedDevices deviceConfig.Devices expiryDate time.Time id int lastUsedDate time.Time localConfig map[string]string localDevices deviceConfig.Devices logger logger.Logger name string node string profiles []api.Profile project api.Project isSnapshot bool stateful bool // Cached handles. // Do not use these variables directly, instead use their associated get functions so they // will be initialized on demand. storagePool storagePools.Pool // volatileSetPersistDisable indicates whether the VolatileSet function should persist changes to the DB. volatileSetPersistDisable bool } // // SECTION: property getters // // Architecture returns the instance's architecture. func (d *common) Architecture() int { return d.architecture } // CreationDate returns the instance's creation date. func (d *common) CreationDate() time.Time { return d.creationDate } // UpdateDevices overrides the instance's devices without persisting changes to the database. func (d *common) UpdateDevices(devices deviceConfig.Devices) error { d.localDevices = devices for name, devConfig := range devices { d.expandedDevices[name] = devConfig } return nil } // Type returns the instance's type. func (d *common) Type() instancetype.Type { return d.dbType } // Description returns the instance's description. func (d *common) Description() string { return d.description } // IsEphemeral returns whether the instanc is ephemeral or not. func (d *common) IsEphemeral() bool { return d.ephemeral } // ExpandedConfig returns instance's expanded config. func (d *common) ExpandedConfig() map[string]string { return d.expandedConfig } // ExpandedDevices returns instance's expanded device config. func (d *common) ExpandedDevices() deviceConfig.Devices { return d.expandedDevices } // ExpiryDate returns when this snapshot expires. func (d *common) ExpiryDate() time.Time { if d.isSnapshot { return d.expiryDate } // Return zero time if the instance is not a snapshot. return time.Time{} } func (d *common) shouldAutoRestart() bool { if !util.IsTrue(d.expandedConfig["boot.autorestart"]) { return false } muInstancesLastRestart.Lock() defer muInstancesLastRestart.Unlock() // Check if the instance was ever auto-restarted. timestamps, ok := instancesLastRestart[d.id] if !ok || len(timestamps) == 0 { // If not, record it and allow the auto-restart. instancesLastRestart[d.id] = [10]time.Time{time.Now()} return true } // If it has been auto-restarted, look for the oldest non-zero timestamp. oldestIndex := 0 validTimestamps := 0 for i, timestamp := range timestamps { if timestamp.IsZero() { // We found an unused slot, lets use it. timestamps[i] = time.Now() instancesLastRestart[d.id] = timestamps return true } validTimestamps++ if timestamp.Before(timestamps[oldestIndex]) { oldestIndex = i } } // Check if the oldest restart was more than a minute ago. if timestamps[oldestIndex].Before(time.Now().Add(-1 * time.Minute)) { // Remove the old timestamp and replace it with ours. timestamps[oldestIndex] = time.Now() instancesLastRestart[d.id] = timestamps return true } // If not and all slots are used return false } // ID gets instances's ID. func (d *common) ID() int { return d.id } // LastUsedDate returns the instance's last used date. func (d *common) LastUsedDate() time.Time { return d.lastUsedDate } // LocalConfig returns the instance's local config. func (d *common) LocalConfig() map[string]string { return d.localConfig } // LocalDevices returns the instance's local device config. func (d *common) LocalDevices() deviceConfig.Devices { return d.localDevices } // Name returns the instance's name. func (d *common) Name() string { return d.name } // CloudInitID returns the cloud-init instance-id. func (d *common) CloudInitID() string { id := d.LocalConfig()["volatile.cloud-init.instance-id"] if id != "" { return id } return d.name } // Location returns instance's location. func (d *common) Location() string { return d.node } // Profiles returns the instance's profiles. func (d *common) Profiles() []api.Profile { return d.profiles } // Project returns instance's project. func (d *common) Project() api.Project { return d.project } // IsSnapshot returns whether instance is snapshot or not. func (d *common) IsSnapshot() bool { return d.isSnapshot } // IsStateful returns whether instance is stateful or not. func (d *common) IsStateful() bool { return d.stateful } // Operation returns the instance's current operation. func (d *common) Operation() *operations.Operation { return d.op } // // SECTION: general functions // // Backups returns a list of backups. func (d *common) Backups() ([]backup.InstanceBackup, error) { var backupNames []string // Get all the backups err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error backupNames, err = tx.GetInstanceBackups(ctx, d.project.Name, d.name) return err }) if err != nil { return nil, err } // Build the backup list backups := []backup.InstanceBackup{} for _, backupName := range backupNames { backup, err := instance.BackupLoadByName(d.state, d.project.Name, backupName) if err != nil { return nil, err } backups = append(backups, *backup) } return backups, nil } // DeferTemplateApply records a template trigger to apply on next instance start. func (d *common) DeferTemplateApply(trigger instance.TemplateTrigger) error { // Avoid over-writing triggers that have already been set. if d.localConfig["volatile.apply_template"] != "" { return nil } err := d.VolatileSet(map[string]string{"volatile.apply_template": string(trigger)}) if err != nil { return fmt.Errorf("Failed to set apply_template volatile key: %w", err) } return nil } // MACPattern computes the most specific MAC address pattern for this instance. func (d *common) MACPattern() string { macPattern, ok := d.project.Config["network.hwaddr_pattern"] if !ok { return d.state.GlobalConfig.NetworkHWAddrPattern() } return macPattern } // SetOperation sets the current operation. func (d *common) SetOperation(op *operations.Operation) { d.op = op } // Snapshots returns a list of snapshots. func (d *common) Snapshots() ([]instance.Instance, error) { if d.isSnapshot { return []instance.Instance{}, nil } var snapshotArgs map[int]db.InstanceArgs // Get all the snapshots for instance. err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { filter := dbCluster.InstanceSnapshotFilter{ Project: &d.project.Name, Instance: &d.name, } dbSnapshots, err := dbCluster.GetInstanceSnapshots(ctx, tx.Tx(), filter) if err != nil { return err } dbInstances := make([]dbCluster.Instance, len(dbSnapshots)) for i, s := range dbSnapshots { dbInstances[i] = s.ToInstance(d.name, d.node, d.dbType, d.architecture) } snapshotArgs, err = tx.InstancesToInstanceArgs(ctx, false, dbInstances...) if err != nil { return err } return nil }) if err != nil { return nil, err } // Stop if no snapshots. if len(snapshotArgs) == 0 { return []instance.Instance{}, nil } snapshots := make([]instance.Instance, 0, len(snapshotArgs)) for _, snapshotArg := range snapshotArgs { // Populate profile info that was already loaded. snapshotArg.Profiles = d.profiles snapInst, err := instance.Load(d.state, snapshotArg, d.project) if err != nil { return nil, err } // Pass through the current operation. snapInst.SetOperation(d.op) snapshots = append(snapshots, instance.Instance(snapInst)) } sort.SliceStable(snapshots, func(i, j int) bool { iCreation := snapshots[i].CreationDate() jCreation := snapshots[j].CreationDate() // Prefer sorting by creation date. if iCreation.Before(jCreation) { return true } // But if creation date is the same, then sort by ID. if iCreation.Equal(jCreation) && snapshots[i].ID() < snapshots[j].ID() { return true } return false }) return snapshots, nil } // VolatileSet sets one or more volatile config keys. func (d *common) VolatileSet(changes map[string]string) error { // Quick check. for key := range changes { if !strings.HasPrefix(key, internalInstance.ConfigVolatilePrefix) { return errors.New("Only volatile keys can be modified with VolatileSet") } } // Update the database if required. if !d.volatileSetPersistDisable { var err error if d.isSnapshot { err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateInstanceSnapshotConfig(d.id, changes) }) } else { err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateInstanceConfig(d.id, changes) }) } if err != nil { return fmt.Errorf("Failed to set volatile config: %w", err) } } // Apply the change locally. for key, value := range changes { if value == "" { delete(d.expandedConfig, key) delete(d.localConfig, key) continue } d.expandedConfig[key] = value d.localConfig[key] = value } return nil } // // SECTION: path getters // // ConsoleBufferLogPath returns the instance's console buffer log path. func (d *common) ConsoleBufferLogPath() string { return filepath.Join(d.LogPath(), "console.log") } // DevicesPath returns the instance's devices path. func (d *common) DevicesPath() string { name := project.Instance(d.project.Name, d.name) return internalUtil.VarPath("devices", name) } // LogPath returns the instance's log path. func (d *common) LogPath() string { name := project.Instance(d.project.Name, d.name) return internalUtil.LogPath(name) } // RunPath returns the instance's runtime path. func (d *common) RunPath() string { name := project.Instance(d.project.Name, d.name) return internalUtil.RunPath(name) } // Path returns the instance's path. func (d *common) Path() string { return storagePools.InstancePath(d.dbType, d.project.Name, d.name, d.isSnapshot) } // ExecOutputPath returns the instance's exec output path. func (d *common) ExecOutputPath() string { return filepath.Join(d.Path(), "exec-output") } // RootfsPath returns the instance's rootfs path. func (d *common) RootfsPath() string { return filepath.Join(d.Path(), "rootfs") } // ShmountsPath returns the instance's shared mounts path. func (d *common) ShmountsPath() string { name := project.Instance(d.project.Name, d.name) return internalUtil.VarPath("shmounts", name) } // StatePath returns the instance's state path. func (d *common) StatePath() string { return filepath.Join(d.Path(), "state") } // TemplatesPath returns the instance's templates path. func (d *common) TemplatesPath() string { return filepath.Join(d.Path(), "templates") } // StoragePool returns the storage pool name. func (d *common) StoragePool() (string, error) { pool, err := d.getStoragePool() if err != nil { return "", err } return pool.Name(), nil } // // SECTION: internal functions // // deviceVolatileReset resets a device's volatile data when its removed or updated in such a way // that it is removed then added immediately afterwards. func (d *common) deviceVolatileReset(devName string, oldConfig, newConfig deviceConfig.Device) error { volatileClear := make(map[string]string) devicePrefix := fmt.Sprintf("volatile.%s.", devName) newNICType, err := nictype.NICType(d.state, d.project.Name, newConfig) if err != nil { return err } oldNICType, err := nictype.NICType(d.state, d.project.Name, oldConfig) if err != nil { return err } // If the device type has changed, remove all old volatile keys. // This will occur if the newConfig is empty (i.e the device is actually being removed) or // if the device type is being changed but keeping the same name. if newConfig["type"] != oldConfig["type"] || newNICType != oldNICType { for k := range d.localConfig { if !strings.HasPrefix(k, devicePrefix) { continue } volatileClear[k] = "" } return d.VolatileSet(volatileClear) } // If the device type remains the same, then just remove any volatile keys that have // the same key name present in the new config (i.e the new config is replacing the // old volatile key). for k := range d.localConfig { if !strings.HasPrefix(k, devicePrefix) { continue } devKey := strings.TrimPrefix(k, devicePrefix) _, found := newConfig[devKey] if found { volatileClear[k] = "" } } return d.VolatileSet(volatileClear) } // deviceVolatileGetFunc returns a function that retrieves a named device's volatile config and // removes its device prefix from the keys. func (d *common) deviceVolatileGetFunc(devName string) func() map[string]string { return func() map[string]string { volatile := make(map[string]string) prefix := fmt.Sprintf("volatile.%s.", devName) for k, v := range d.localConfig { after, ok := strings.CutPrefix(k, prefix) if ok { volatile[after] = v } } return volatile } } // deviceVolatileSetFunc returns a function that can be called to save a named device's volatile // config using keys that do not have the device's name prefixed. func (d *common) deviceVolatileSetFunc(devName string) func(save map[string]string) error { return func(save map[string]string) error { volatileSave := make(map[string]string) for k, v := range save { volatileSave[fmt.Sprintf("volatile.%s.%s", devName, k)] = v } return d.VolatileSet(volatileSave) } } // expandConfig applies the config of each profile in order, followed by the local config. func (d *common) expandConfig() error { d.expandedConfig = db.ExpandInstanceConfig(d.localConfig, d.profiles) d.expandedDevices = db.ExpandInstanceDevices(d.localDevices, d.profiles) return nil } // restartCommon handles the common part of instance restarts. func (d *common) restartCommon(inst instance.Instance, timeout time.Duration) error { // Setup a new operation for the stop/shutdown phase. op, err := operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionRestart, true, true) if err != nil { return fmt.Errorf("Create restart operation: %w", err) } // Handle ephemeral instances. ephemeral := inst.IsEphemeral() ctxMap := logger.Ctx{ "action": "shutdown", "created": d.creationDate, "ephemeral": ephemeral, "used": d.lastUsedDate, "timeout": timeout, } d.logger.Info("Restarting instance", ctxMap) if ephemeral { // Unset ephemeral flag args := db.InstanceArgs{ Architecture: inst.Architecture(), Config: inst.LocalConfig(), Description: inst.Description(), Devices: inst.LocalDevices(), Ephemeral: false, Profiles: inst.Profiles(), Project: inst.Project().Name, Type: inst.Type(), Snapshot: inst.IsSnapshot(), } err := inst.Update(args, false) if err != nil { return err } // On function return, set the flag back on defer func() { args.Ephemeral = ephemeral _ = inst.Update(args, false) }() } if timeout == 0 { err := inst.Stop(false) if err != nil { op.Done(err) return err } } else { if inst.IsFrozen() { err = errors.New("Instance is not running") op.Done(err) return err } err := inst.Shutdown(timeout) if err != nil { op.Done(err) return err } } // Setup a new operation for the start phase. op, err = operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionRestart, nil, true, false) if err != nil { if errors.Is(err, operationlock.ErrNonReusuableSucceeded) { // An existing matching operation has now succeeded, return. return nil } return fmt.Errorf("Create restart (for start) operation: %w", err) } err = inst.Start(false) if err != nil { op.Done(err) return err } d.logger.Info("Restarted instance", ctxMap) d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceRestarted.Event(d, nil)) return nil } // rebuildCommon handles the common part of instance rebuilds. func (d *common) rebuildCommon(inst instance.Instance, img *api.Image, op *operations.Operation) error { instLocalConfig := d.localConfig // Reset the "image.*" keys. for k := range instLocalConfig { if strings.HasPrefix(k, "image.") { delete(instLocalConfig, k) } } delete(instLocalConfig, "volatile.base_image") if img != nil { for k, v := range img.Properties { instLocalConfig[fmt.Sprintf("image.%s", k)] = v } instLocalConfig["volatile.base_image"] = img.Fingerprint instLocalConfig["volatile.uuid.generation"] = instLocalConfig["volatile.uuid"] } // Reset relevant volatile keys. delete(instLocalConfig, "volatile.idmap.next") delete(instLocalConfig, "volatile.last_state.idmap") pool, err := d.getStoragePool() if err != nil { return err } err = pool.DeleteInstance(inst, op) if err != nil { return err } // Rebuild as empty if there is no image provided. if img == nil { err = pool.CreateInstance(inst, nil) if err != nil { return err } } else { err = pool.CreateInstanceFromImage(inst, img.Fingerprint, op) if err != nil { return err } } err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { err = dbCluster.UpdateInstanceConfig(ctx, tx.Tx(), int64(inst.ID()), instLocalConfig) if err != nil { return err } if img != nil { err = tx.UpdateImageLastUseDate(ctx, inst.Project().Name, img.Fingerprint, time.Now().UTC()) if err != nil { return err } } return nil }) if err != nil { return err } d.localConfig = instLocalConfig return nil } // runHooks executes the callback functions returned from a function. func (d *common) runHooks(hooks []func() error) error { // Run any post start hooks. for _, hook := range hooks { err := hook() if err != nil { return err } } return nil } // snapshot handles the common part of the snapshotting process. func (d *common) snapshotCommon(inst instance.Instance, name string, expiry time.Time, stateful bool) error { reverter := revert.New() defer reverter.Fail() // Setup the arguments. args := db.InstanceArgs{ Project: inst.Project().Name, Architecture: inst.Architecture(), Config: inst.LocalConfig(), Description: inst.Description(), Type: inst.Type(), Snapshot: true, Devices: inst.LocalDevices(), Ephemeral: inst.IsEphemeral(), Name: inst.Name() + internalInstance.SnapshotDelimiter + name, Profiles: inst.Profiles(), Stateful: stateful, ExpiryDate: expiry, } // Create the snapshot. snap, snapInstOp, cleanup, err := instance.CreateInternal(d.state, args, d.op, true, true, false) if err != nil { return fmt.Errorf("Failed creating instance snapshot record %q: %w", name, err) } reverter.Add(cleanup) defer snapInstOp.Done(err) pool, err := storagePools.LoadByInstance(d.state, snap) if err != nil { return err } err = pool.CreateInstanceSnapshot(snap, inst, d.op) if err != nil { return fmt.Errorf("Create instance snapshot: %w", err) } reverter.Add(func() { _ = snap.Delete(true, true) }) // Mount volume for backup.yaml writing. _, err = pool.MountInstance(inst, d.op) if err != nil { return fmt.Errorf("Create instance snapshot (mount source): %w", err) } defer func() { _ = pool.UnmountInstance(inst, d.op) }() // Attempt to update backup.yaml for instance. err = inst.UpdateBackupFile() if err != nil { return err } reverter.Success() return nil } // updateProgress updates the operation metadata with a new progress string. func (d *common) updateProgress(progress string) { if d.op == nil { return } meta := d.op.Metadata() if meta == nil { meta = make(map[string]any) } if meta["container_progress"] != progress { _ = d.op.ExtendMetadata(map[string]any{"container_progress": progress}) } } // insertConfigkey function attempts to insert the instance config key into the database. If the insert fails // then the database is queried to check whether another query inserted the same key. If the key is still // unpopulated then the insert querty is retried until it succeeds or a retry limit is reached. // If the insert succeeds or the key is found to have been populated then the value of the key is returned. func (d *common) insertConfigkey(key string, value string) (string, error) { err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { err := tx.CreateInstanceConfig(ctx, d.id, map[string]string{key: value}) if err == nil { return nil } // Check if something else filled it in behind our back. existingValue, errCheckExists := tx.GetInstanceConfig(ctx, d.id, key) if errCheckExists != nil { return err } value = existingValue return nil }) if err != nil { return "", err } return value, nil } // isRunningStatusCode returns if instance is running from status code. func (d *common) isRunningStatusCode(statusCode api.StatusCode) bool { return statusCode != api.Error && statusCode != api.Stopped } // isErrorStatusCode returns if instance is errored from status code. func (d *common) isErrorStatusCode(statusCode api.StatusCode) bool { return statusCode == api.Error } // isStartableStatusCode returns an error if the status code means the instance cannot be started currently. func (d *common) isStartableStatusCode(statusCode api.StatusCode) error { if d.isRunningStatusCode(statusCode) { return errors.New("The instance is already running") } // If the instance process exists but is crashed, don't allow starting until its been cleaned up, as it // would likely fail to start anyway or leave the old process untracked. if statusCode == api.Error { return fmt.Errorf("The instance cannot be started as in %s status", statusCode) } return nil } // getStartupSnapNameAndExpiry returns the name and expiry for a snapshot to be taken at startup. func (d *common) getStartupSnapNameAndExpiry(inst instance.Instance) (string, *time.Time, error) { schedule := strings.ToLower(d.expandedConfig["snapshots.schedule"]) if schedule == "" { return "", nil, nil } triggers := strings.Split(schedule, ", ") if !slices.Contains(triggers, "@startup") { return "", nil, nil } expiry, err := internalInstance.GetExpiry(time.Now(), d.expandedConfig["snapshots.expiry"]) if err != nil { return "", nil, err } name, err := instance.NextSnapshotName(d.state, inst, "snap%d") if err != nil { return "", nil, err } return name, &expiry, nil } // validateStartup checks any constraints that would prevent start up from succeeding under normal circumstances. func (d *common) validateStartup(stateful bool, statusCode api.StatusCode) error { // Because the root disk is special and is mounted before the root disk device is setup we duplicate the // pre-start check here before the isStartableStatusCode check below so that if there is a problem loading // the instance status because the storage pool isn't available we don't mask the StatusServiceUnavailable // error with an ERROR status code from the instance check instead. _, rootDiskConf, err := internalInstance.GetRootDiskDevice(d.expandedDevices.CloneNative()) if err != nil { return err } if !storagePools.IsAvailable(rootDiskConf["pool"]) { return api.StatusErrorf(http.StatusServiceUnavailable, "Storage pool %q unavailable on this server", rootDiskConf["pool"]) } // Validate architecture. if !slices.Contains(d.state.OS.Architectures, d.architecture) { return errors.New("Requested architecture isn't supported by this host") } // Must happen before creating operation Start lock to avoid the status check returning Stopped due to the // existence of a Start operation lock. err = d.isStartableStatusCode(statusCode) if err != nil { return err } return nil } // onStopOperationSetup creates or picks up the relevant operation. This is used in the stopns and stop hooks to // ensure that a lock on their activities is held before the instance process is stopped. This prevents a start // request run at the same time from overlapping with the stop process. // Returns the operation (with the instance initiated marker set if the operation was created). func (d *common) onStopOperationSetup(target string) (*operationlock.InstanceOperation, error) { var err error // Pick up the existing stop operation lock created in Start(), Restart(), Shutdown() or Stop() functions. // If there is another ongoing operation that isn't in our inheritable list, wait until that has finished // before proceeding to run the hook. op := operationlock.Get(d.Project().Name, d.Name()) if op != nil && !op.ActionMatch(operationlock.ActionStart, operationlock.ActionRestart, operationlock.ActionStop, operationlock.ActionRestore, operationlock.ActionMigrate) { d.logger.Debug("Waiting for existing operation lock to finish before running hook", logger.Ctx{"action": op.Action()}) _ = op.Wait(context.Background()) op = nil } if op == nil { d.logger.Debug("Instance initiated stop", logger.Ctx{"action": target}) action := operationlock.ActionStop if target == "reboot" { action = operationlock.ActionRestart } op, err = operationlock.Create(d.Project().Name, d.Name(), d.op, action, false, false) if err != nil { return nil, fmt.Errorf("Failed creating %q operation: %w", action, err) } op.SetInstanceInitiated(true) } else { d.logger.Debug("Instance operation lock inherited for stop", logger.Ctx{"action": op.Action()}) } return op, nil } // warningsDelete deletes any persistent warnings for the instance. func (d *common) warningsDelete() error { err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return dbCluster.DeleteWarnings(ctx, tx.Tx(), dbCluster.TypeInstance, d.ID()) }) if err != nil { return fmt.Errorf("Failed deleting persistent warnings: %w", err) } return nil } // canMigrate determines if the given instance can be migrated and what kind of migration to attempt. func (d *common) canMigrate(inst instance.Instance) string { // Check policy for the instance. config := d.ExpandedConfig() val, ok := config["cluster.evacuate"] if !ok { val = "auto" } // If not using auto, just return the migration type. if val != "auto" { return val } // Look at attached devices. for _, entry := range d.ExpandedDevices().Sorted() { dev, err := d.deviceLoad(inst, entry.Name, entry.Config, false) if err != nil { logger.Warn("Instance will not be migrated due to a device error", logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "device": dev.Name(), "err": err}) return "stop" } if !dev.CanMigrate() { logger.Warn("Instance will not be migrated because its device cannot be migrated", logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "device": dev.Name()}) return "stop" } } // Check if set up for live migration. // Limit automatic live-migration to virtual machines for now. if inst.Type() == instancetype.VM && util.IsTrue(config["migration.stateful"]) { return "live-migrate" } return "migrate" } // recordLastState records last power and used time into local config and database config. func (d *common) recordLastState() error { var err error // Record power state. d.localConfig["volatile.last_state.power"] = instance.PowerStateRunning d.expandedConfig["volatile.last_state.power"] = instance.PowerStateRunning // Database updates return d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Record power state. err = tx.UpdateInstancePowerState(d.id, instance.PowerStateRunning) if err != nil { err = fmt.Errorf("Error updating instance power state: %w", err) return err } // Update time instance last started time. err = tx.UpdateInstanceLastUsedDate(d.id, time.Now().UTC()) if err != nil { err = fmt.Errorf("Error updating instance last used: %w", err) return err } return nil }) } func (d *common) setCoreSched(pids []int) error { args := []string{ "forkcoresched", "0", } for _, pid := range pids { args = append(args, strconv.Itoa(pid)) } _, err := subprocess.RunCommand(d.state.OS.ExecPath, args...) return err } // getRootDiskDevice gets the name and configuration of the root disk device of an instance. func (d *common) getRootDiskDevice() (string, map[string]string, error) { devices := d.ExpandedDevices() if d.IsSnapshot() { parentName, _, _ := api.GetParentAndSnapshotName(d.name) // Load the parent. storageInstance, err := instance.LoadByProjectAndName(d.state, d.project.Name, parentName) if err != nil { return "", nil, err } devices = storageInstance.ExpandedDevices() } // Retrieve the instance's storage pool. name, configuration, err := internalInstance.GetRootDiskDevice(devices.CloneNative()) if err != nil { return "", nil, err } return name, configuration, nil } // resetInstanceID generates a new UUID and puts it in volatile. func (d *common) resetInstanceID() error { err := d.VolatileSet(map[string]string{"volatile.cloud-init.instance-id": uuid.New().String()}) if err != nil { return fmt.Errorf("Failed to set volatile.cloud-init.instance-id: %w", err) } return nil } // needsNewInstanceID checks the changed data in an Update call to determine if a new instance-id is necessary. func (d *common) needsNewInstanceID(changedConfig []string, oldExpandedDevices deviceConfig.Devices) bool { // Look for cloud-init related config changes. for _, key := range []string{ "cloud-init.vendor-data", "cloud-init.user-data", "cloud-init.network-config", "user.vendor-data", "user.user-data", "user.network-config", } { if slices.Contains(changedConfig, key) { return true } } // Look for changes in network interface names. getNICNames := func(devs deviceConfig.Devices) []string { names := make([]string, 0, len(devs)) for devName, dev := range devs { if dev["type"] != "nic" { continue } if dev["name"] != "" { names = append(names, dev["name"]) continue } configKey := fmt.Sprintf("volatile.%s.name", devName) volatileName := d.localConfig[configKey] if volatileName != "" { names = append(names, dev["name"]) continue } names = append(names, devName) } return names } oldNames := getNICNames(oldExpandedDevices) newNames := getNICNames(d.expandedDevices) for _, entry := range oldNames { if !slices.Contains(newNames, entry) { return true } } for _, entry := range newNames { if !slices.Contains(oldNames, entry) { return true } } return false } // getStoragePool returns the current storage pool handle. To avoid a DB lookup each time this // function is called, the handle is cached internally in the struct. func (d *common) getStoragePool() (storagePools.Pool, error) { if d.storagePool != nil { return d.storagePool, nil } var poolName string err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error poolName, err = tx.GetInstancePool(ctx, d.Project().Name, d.Name()) if err != nil { return fmt.Errorf("Failed getting instance pool: %w", err) } return nil }) if err != nil { return nil, err } pool, err := storagePools.LoadByName(d.state, poolName) if err != nil { return nil, err } d.storagePool = pool return d.storagePool, nil } // deviceLoad instantiates and validates a new device and returns it along with enriched config. func (d *common) deviceLoad(inst instance.Instance, deviceName string, rawConfig deviceConfig.Device, partialValidation bool) (device.Device, error) { var configCopy deviceConfig.Device var err error // Create copy of config and load some fields from volatile if device is nic or infiniband. if slices.Contains([]string{"nic", "infiniband"}, rawConfig["type"]) { configCopy, err = inst.FillNetworkDevice(deviceName, rawConfig) if err != nil { return nil, err } } else { // Otherwise copy the config so it cannot be modified by device. configCopy = rawConfig.Clone() } dev, err := device.New(inst, d.state, deviceName, configCopy, partialValidation, d.deviceVolatileGetFunc(deviceName), d.deviceVolatileSetFunc(deviceName)) // If validation fails with unsupported device type then don't return the device for use. if errors.Is(err, device.ErrUnsupportedDevType) { return nil, err } // Return device even if error occurs as caller may still attempt to use device for stop and remove. return dev, err } // deviceAdd loads a new device and calls its Add() function. func (d *common) deviceAdd(dev device.Device, instanceRunning bool) error { l := d.logger.AddContext(logger.Ctx{"device": dev.Name(), "type": dev.Config()["type"]}) l.Debug("Adding device") if instanceRunning && !dev.CanHotPlug() { return errors.New("Device cannot be added when instance is running") } return dev.Add() } // deviceRemove loads a new device and calls its Remove() function. func (d *common) deviceRemove(dev device.Device, instanceRunning bool, cleanupDependencies bool) error { l := d.logger.AddContext(logger.Ctx{"device": dev.Name(), "type": dev.Config()["type"]}) l.Debug("Removing device") if instanceRunning && !dev.CanHotPlug() { return errors.New("Device cannot be removed when instance is running") } return dev.Remove(cleanupDependencies) } // devicesAdd adds devices to instance. func (d *common) devicesAdd(inst instance.Instance, instanceRunning bool, partialValidation bool) (revert.Hook, error) { reverter := revert.New() defer reverter.Fail() for _, entry := range d.expandedDevices.Sorted() { dev, err := d.deviceLoad(inst, entry.Name, entry.Config, partialValidation) if err != nil { if errors.Is(err, device.ErrUnsupportedDevType) { continue // Skip unsupported device (allows for mixed instance type profiles). } // If device conflicts with another device then do not call the deviceAdd function below // as this could cause the original device to be disrupted (such as allowing conflicting // static NIC DHCP leases to be created). Instead just log an error. // This will allow instances to be created with conflicting devices (such as when copying // or restoring a backup) and allows the user to manually fix the conflicts in order to // allow the instance to start. if api.StatusErrorCheck(err, http.StatusConflict) { d.logger.Error("Failed add validation for device, skipping add action", logger.Ctx{"device": entry.Name, "err": err}) continue } // Clear any volatile key that could have been set during validation. _ = d.deviceVolatileReset(entry.Name, entry.Config, nil) return nil, fmt.Errorf("Failed add validation for device %q: %w", entry.Name, err) } err = d.deviceAdd(dev, instanceRunning) if err != nil { return nil, fmt.Errorf("Failed to add device %q: %w", dev.Name(), err) } reverter.Add(func() { _ = d.deviceRemove(dev, instanceRunning, true) }) } cleanup := reverter.Clone().Fail reverter.Success() return cleanup, nil } // devicesRegister calls the Register() function on all of the instance's devices. func (d *common) devicesRegister(inst instance.Instance) { for _, entry := range d.ExpandedDevices().Sorted() { err := device.Register(inst, d.state, entry.Name, entry.Config) if err != nil { if errors.Is(err, device.ErrUnsupportedDevType) { continue // Skip unsupported device (allows for mixed instance type profiles). } d.logger.Error("Failed to register device", logger.Ctx{"err": err, "device": entry.Name}) continue } } } // devicesUpdate applies device changes to an instance. func (d *common) devicesUpdate(inst instance.Instance, removeDevices deviceConfig.Devices, addDevices deviceConfig.Devices, updateDevices deviceConfig.Devices, oldExpandedDevices deviceConfig.Devices, instanceRunning bool, userRequested bool) error { reverter := revert.New() defer reverter.Fail() dm, ok := inst.(deviceManager) if !ok { return errors.New("Instance is not compatible with deviceManager interface") } // Remove devices in reverse order to how they were added. for _, entry := range removeDevices.Reversed() { l := d.logger.AddContext(logger.Ctx{"device": entry.Name, "userRequested": userRequested}) dev, err := d.deviceLoad(inst, entry.Name, entry.Config, false) if err != nil { if errors.Is(err, device.ErrUnsupportedDevType) { continue // Skip unsupported device (allows for mixed instance type profiles). } // Just log an error, but still allow the device to be removed if usable device returned. l.Error("Failed remove validation for device", logger.Ctx{"err": err}) } // If a device was returned from deviceLoad even if validation fails, then try to stop and remove. if dev != nil { if instanceRunning { err = dm.deviceStop(dev, instanceRunning, "") if err != nil { return fmt.Errorf("Failed to stop device %q: %w", dev.Name(), err) } } err = d.deviceRemove(dev, instanceRunning, true) if err != nil && !errors.Is(err, device.ErrUnsupportedDevType) { return fmt.Errorf("Failed to remove device %q: %w", dev.Name(), err) } } // Check whether we are about to add the same device back with updated config and // if not, or if the device type has changed, then remove all volatile keys for // this device (as its an actual removal or a device type change). err = d.deviceVolatileReset(entry.Name, entry.Config, addDevices[entry.Name]) if err != nil { return fmt.Errorf("Failed to reset volatile data for device %q: %w", entry.Name, err) } } // Add devices in sorted order, this ensures that device mounts are added in path order. for _, entry := range addDevices.Sorted() { l := d.logger.AddContext(logger.Ctx{"device": entry.Name, "userRequested": userRequested}) dev, err := d.deviceLoad(inst, entry.Name, entry.Config, false) if err != nil { if errors.Is(err, device.ErrUnsupportedDevType) { continue // Skip unsupported device (allows for mixed instance type profiles). } if userRequested { // Clear any volatile key that could have been set during validation. _ = d.deviceVolatileReset(entry.Name, entry.Config, nil) return fmt.Errorf("Failed add validation for device %q: %w", entry.Name, err) } // If update is non-user requested (i.e from a snapshot restore), there's nothing we can // do to fix the config and we don't want to prevent the snapshot restore so log and allow. l.Error("Failed add validation for device, skipping as non-user requested", logger.Ctx{"err": err}) continue } err = d.deviceAdd(dev, instanceRunning) if err != nil { if userRequested { return fmt.Errorf("Failed to add device %q: %w", dev.Name(), err) } // If update is non-user requested (i.e from a snapshot restore), there's nothing we can // do to fix the config and we don't want to prevent the snapshot restore so log and allow. l.Error("Failed to add device, skipping as non-user requested", logger.Ctx{"err": err}) } reverter.Add(func() { _ = d.deviceRemove(dev, instanceRunning, true) }) if instanceRunning { err = dev.PreStartCheck() if err != nil { return fmt.Errorf("Failed pre-start check for device %q: %w", dev.Name(), err) } _, err := dm.deviceStart(dev, instanceRunning) if err != nil && !errors.Is(err, device.ErrUnsupportedDevType) { return fmt.Errorf("Failed to start device %q: %w", dev.Name(), err) } reverter.Add(func() { _ = dm.deviceStop(dev, instanceRunning, "") }) } // For the root disk, call Update as its size may change. // Update will invoke applyQuota, which resizes the disk if necessary. if internalInstance.IsRootDiskDevice(dev.Config()) { err = dev.Update(oldExpandedDevices, instanceRunning) if err != nil { return fmt.Errorf("Failed to update device %q: %w", dev.Name(), err) } } } for _, entry := range updateDevices.Sorted() { l := d.logger.AddContext(logger.Ctx{"device": entry.Name, "userRequested": userRequested}) dev, err := d.deviceLoad(inst, entry.Name, entry.Config, false) if err != nil { if errors.Is(err, device.ErrUnsupportedDevType) { continue // Skip unsupported device (allows for mixed instance type profiles). } if userRequested { return fmt.Errorf("Failed update validation for device %q: %w", entry.Name, err) } // If update is non-user requested (i.e from a snapshot restore), there's nothing we can // do to fix the config and we don't want to prevent the snapshot restore so log and allow. // By not calling dev.Update on validation error we avoid potentially disrupting another // existing device if this device conflicts with it (such as allowing conflicting static // NIC DHCP leases to be created). l.Error("Failed update validation for device, removing device", logger.Ctx{"err": err}) // If a device was returned from deviceLoad when validation fails, then try to stop and // remove it. This is to prevent devices being left in a state that is different to the // invalid non-user requested config that has been applied to DB. The safest thing to do // is to cleanup the device and wait for the config to be corrected. if dev != nil { if instanceRunning { err = dm.deviceStop(dev, instanceRunning, "") if err != nil { l.Error("Failed to stop device after update validation failed", logger.Ctx{"err": err}) } } err = d.deviceRemove(dev, instanceRunning, true) if err != nil && !errors.Is(err, device.ErrUnsupportedDevType) { l.Error("Failed to remove device after update validation failed", logger.Ctx{"err": err}) } } continue } err = dev.Update(oldExpandedDevices, instanceRunning) if err != nil { return fmt.Errorf("Failed to update device %q: %w", dev.Name(), err) } } reverter.Success() return nil } // devicesRemove runs device removal function for each device. func (d *common) devicesRemove(inst instance.Instance, cleanupDependencies bool) { for _, entry := range d.expandedDevices.Reversed() { dev, err := d.deviceLoad(inst, entry.Name, entry.Config, true) if err != nil { if errors.Is(err, device.ErrUnsupportedDevType) { continue // Skip unsupported device (allows for mixed instance type profiles). } // Just log an error, but still allow the device to be removed if usable device returned. d.logger.Error("Failed remove validation for device", logger.Ctx{"device": entry.Name, "err": err}) } // If a usable device was returned from deviceLoad try to remove anyway, even if validation fails. // This allows for the scenario where a new version has additional validation restrictions // than older versions and we still need to allow previously valid devices to be stopped even if // they are no longer considered valid. if dev != nil { err = d.deviceRemove(dev, false, cleanupDependencies) if err != nil { d.logger.Error("Failed to remove device", logger.Ctx{"device": dev.Name(), "err": err}) } } } } // updateBackupFileLock acquires the update backup file lock that protects concurrent access to actions that will call UpdateBackupFile() as part of their operation. func (d *common) updateBackupFileLock(ctx context.Context) (locking.UnlockFunc, error) { parentName, _, _ := api.GetParentAndSnapshotName(d.Name()) return locking.Lock(ctx, fmt.Sprintf("instance_updatebackupfile_%s_%s", d.Project().Name, parentName)) } // deleteSnapshots calls the deleteFunc on each of the instance's snapshots in reverse order. func (d *common) deleteSnapshots(deleteFunc func(snapInst instance.Instance) error) error { snapInsts, err := d.Snapshots() if err != nil { return err } snapInstsCount := len(snapInsts) for k := range snapInsts { // Delete the snapshots in reverse order. k = snapInstsCount - 1 - k err = deleteFunc(snapInsts[k]) if err != nil { return fmt.Errorf("Failed deleting snapshot %q: %w", snapInsts[k].Name(), err) } } return nil } // balanceNUMANodes looks at all other instances and picks the least used NUMA node(s). func (d *common) balanceNUMANodes() error { muNUMA.Lock() defer muNUMA.Unlock() // Get the CPU information. cpu, err := resources.GetCPU() if err != nil { return err } // Get a list of NUMA nodes. nodes := []uint64{} for _, cpuSocket := range cpu.Sockets { for _, cpuCore := range cpuSocket.Cores { for _, cpuThread := range cpuCore.Threads { if !slices.Contains(nodes, cpuThread.NUMANode) { nodes = append(nodes, cpuThread.NUMANode) } } } } // Shortcut on single-node systems. if len(nodes) == 1 { return d.VolatileSet(map[string]string{"volatile.cpu.nodes": fmt.Sprintf("%d", nodes[0])}) } // Get all local instances. insts, err := instance.LoadNodeAll(d.state, instancetype.Any) if err != nil { return err } // Record current NUMA assignment (number of instance). numaUsage := map[int64]int{} for _, inst := range insts { conf := inst.ExpandedConfig() // Ignore ourselves. if inst.ID() == d.id { continue } // Ignore instances without any NUMA pinning. if conf["limits.cpu.nodes"] == "" { continue } // Parse the used NUMA nodes. nodes := conf["limits.cpu.nodes"] if nodes == "balanced" { nodes = conf["volatile.cpu.nodes"] } numaNodeSet, err := resources.ParseNumaNodeSet(nodes) if err != nil { continue } for _, numaNode := range numaNodeSet { numaUsage[numaNode]++ } } // Sort NUMA nodes by usage. slices.SortFunc(nodes, func(i, j uint64) int { return cmp.Compare(numaUsage[int64(i)], numaUsage[int64(j)]) }) // If `limits.cpu` is greater than the number of CPUs per NUMA node, // then figure out how many NUMA nodes to use. conf := d.ExpandedConfig() cpusPerNumaNode := int(cpu.Total) / len(nodes) limitsCPU, err := strconv.Atoi(conf["limits.cpu"]) if err == nil && limitsCPU > cpusPerNumaNode { numaNodesToUse := int(math.Ceil(float64(limitsCPU) / float64(cpusPerNumaNode))) selectedNumaNodes := make([]string, numaNodesToUse) for i, node := range nodes[:numaNodesToUse] { selectedNumaNodes[i] = strconv.FormatUint(node, 10) } joinedNumaNodes := strings.Join(selectedNumaNodes, ",") return d.VolatileSet(map[string]string{"volatile.cpu.nodes": joinedNumaNodes}) } return d.VolatileSet(map[string]string{"volatile.cpu.nodes": fmt.Sprintf("%d", nodes[0])}) } // Gets the process starting time. func (d *common) processStartedAt(pid int) (time.Time, error) { if pid < 1 { return time.Time{}, fmt.Errorf("Invalid PID %d", pid) } file, err := os.Stat(fmt.Sprintf("/proc/%d", pid)) if err != nil { return time.Time{}, err } linuxInfo, ok := file.Sys().(*syscall.Stat_t) if !ok { return time.Time{}, errors.New("Bad stat type") } return time.Unix(int64(linuxInfo.Ctim.Sec), int64(linuxInfo.Ctim.Nsec)), nil } // ETag returns the instance configuration ETag data for pre-condition validation. func (d *common) ETag() []any { if d.IsSnapshot() { return []any{d.expiryDate} } // Prepare the ETag etag := []any{d.architecture, d.ephemeral, d.profiles, d.localDevices.Sorted()} configKeys := make([]string, 0, len(d.localConfig)) for k := range d.localConfig { configKeys = append(configKeys, k) } sort.Strings(configKeys) for _, k := range configKeys { etag = append(etag, fmt.Sprintf("%s=%s", k, d.localConfig[k])) } return etag } // ClearLimitsCPUNodes clears the "volatile.cpu.nodes" configuration if necessary. func (d *common) ClearLimitsCPUNodes(changedConfig []string) { if !slices.Contains(changedConfig, "limits.cpu.nodes") { return } value := d.expandedConfig["limits.cpu.nodes"] if value == "balanced" { return } d.localConfig["volatile.cpu.nodes"] = "" } // setOOMPriority applies the OOM score adjustment to the instance. func (d *common) setOOMPriority(pid int) error { oomPriority := d.expandedConfig["limits.memory.oom_priority"] score := int64(0) var err error if oomPriority != "" { score, err = strconv.ParseInt(oomPriority, 10, 64) if err != nil { return fmt.Errorf("Invalid OOM priority value %q: %w", oomPriority, err) } } if pid <= 0 { return fmt.Errorf("Failed to set OOM priority: instance not running or PID not found") } err = os.WriteFile(fmt.Sprintf("/proc/%d/oom_score_adj", pid), []byte(fmt.Sprintf("%d", score)), 0o644) if err != nil { return fmt.Errorf("Failed to set OOM priority: %w", err) } return nil } // selinuxContext returns the SELinux context for the instance. func (d *common) selinuxContext(baseContext string) (string, error) { // Get all local instances. instances, err := instance.LoadNodeAll(d.state, instancetype.Any) if err != nil { return "", fmt.Errorf("Failed loading local instances: %w", err) } // Get all current values. seContexts := make([]string, 0, len(instances)) for _, inst := range instances { if !inst.IsRunning() { continue } val, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(inst.InitPID()), "attr", "current")) if err != nil { return "", err } seContexts = append(seContexts, strings.TrimSuffix(string(val), "\x00")) } // Generate a random set of categories. for { c1 := rand.IntN(1023) c2 := rand.IntN(1023) if c1 == c2 { continue } if c1 > c2 { c1, c2 = c2, c1 } seContext := baseContext + ":c" + strconv.Itoa(c1) + ",c" + strconv.Itoa(c2) if slices.Contains(seContexts, seContext) { continue } return seContext, nil } } // HasDependentDisk checks whether the instance has any dependent volumes. func (d *common) HasDependentDisk() bool { for _, dev := range d.ExpandedDevices().Sorted() { if dev.Config["type"] != "disk" || util.IsFalseOrEmpty(dev.Config["dependent"]) || dev.Config["path"] == "/" || dev.Config["pool"] == "" { continue } return true } return false } // ForEachDependentDiskType executes the given function for each dependent disk on the instance. func (d *common) ForEachDependentDiskType(diskAction func(dev deviceConfig.DeviceNamed) error) error { for _, dev := range d.ExpandedDevices().Sorted() { if dev.Config["type"] != "disk" || util.IsFalseOrEmpty(dev.Config["dependent"]) || dev.Config["path"] == "/" || dev.Config["pool"] == "" { continue } err := diskAction(dev) if err != nil { return err } } return nil } incus-7.0.0/internal/server/instance/drivers/driver_lxc.go000066400000000000000000007721721517523235500237630ustar00rootroot00000000000000package drivers import ( "bufio" "bytes" "context" "database/sql" "encoding/base64" "encoding/json" "errors" "fmt" "io" "io/fs" "maps" "net" "net/http" "os" "os/exec" "path" "path/filepath" "runtime" "slices" "sort" "strconv" "strings" "sync" "syscall" "time" "github.com/checkpoint-restore/go-criu/v8/crit" "github.com/flosch/pongo2/v6" "github.com/google/uuid" "github.com/gorilla/websocket" "github.com/kballard/go-shellquote" liblxc "github.com/lxc/go-lxc" ociSpecs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/sftp" yaml "go.yaml.in/yaml/v4" "golang.org/x/sync/errgroup" "golang.org/x/sys/unix" "google.golang.org/protobuf/proto" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/instancewriter" internalIO "github.com/lxc/incus/v7/internal/io" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/netutils" "github.com/lxc/incus/v7/internal/rsync" "github.com/lxc/incus/v7/internal/server/apparmor" "github.com/lxc/incus/v7/internal/server/cgroup" "github.com/lxc/incus/v7/internal/server/daemon" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/device" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/device/nictype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/instance/operationlock" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/locking" "github.com/lxc/incus/v7/internal/server/metrics" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/seccomp" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/termios" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/ws" ) // Helper functions. func lxcSetConfigItem(c *liblxc.Container, key string, value string) error { if c == nil { return errors.New("Uninitialized go-lxc struct") } if !liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 2, 1, 0) { switch key { case "lxc.uts.name": key = "lxc.utsname" case "lxc.pty.max": key = "lxc.pts" case "lxc.tty.dir": key = "lxc.devttydir" case "lxc.tty.max": key = "lxc.tty" case "lxc.apparmor.profile": key = "lxc.aa_profile" case "lxc.apparmor.allow_incomplete": key = "lxc.aa_allow_incomplete" case "lxc.selinux.context": key = "lxc.se_context" case "lxc.mount.fstab": key = "lxc.mount" case "lxc.console.path": key = "lxc.console" case "lxc.seccomp.profile": key = "lxc.seccomp" case "lxc.signal.halt": key = "lxc.haltsignal" case "lxc.signal.reboot": key = "lxc.rebootsignal" case "lxc.signal.stop": key = "lxc.stopsignal" case "lxc.log.syslog": key = "lxc.syslog" case "lxc.log.level": key = "lxc.loglevel" case "lxc.log.file": key = "lxc.logfile" case "lxc.init.cmd": key = "lxc.init_cmd" case "lxc.init.uid": key = "lxc.init_uid" case "lxc.init.gid": key = "lxc.init_gid" case "lxc.idmap": key = "lxc.id_map" } } if strings.HasPrefix(key, "lxc.prlimit.") { if !liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 2, 1, 0) { return errors.New(`Process limits require liblxc >= 2.1`) } } err := c.SetConfigItem(key, value) if err != nil { return fmt.Errorf("Failed to set LXC config: %s=%s", key, value) } return nil } func lxcStatusCode(state liblxc.State) api.StatusCode { return map[int]api.StatusCode{ 1: api.Stopped, 2: api.Starting, 3: api.Running, 4: api.Stopping, 5: api.Aborting, 6: api.Freezing, 7: api.Frozen, 8: api.Thawed, 9: api.Error, }[int(state)] } // lxcCreate creates the DB storage records and sets up instance devices. // Returns a revert fail function that can be used to undo this function if a subsequent step fails. func lxcCreate(s *state.State, args db.InstanceArgs, p api.Project, partialDeviceValidation bool, op *operations.Operation) (instance.Instance, revert.Hook, error) { reverter := revert.New() defer reverter.Fail() // Create the container struct d := &lxc{ common: common{ state: s, op: op, architecture: args.Architecture, creationDate: args.CreationDate, dbType: args.Type, description: args.Description, ephemeral: args.Ephemeral, expiryDate: args.ExpiryDate, id: args.ID, lastUsedDate: args.LastUsedDate, localConfig: args.Config, localDevices: args.Devices, logger: logger.AddContext(logger.Ctx{"instanceType": args.Type, "instance": args.Name, "project": args.Project}), name: args.Name, node: args.Node, profiles: args.Profiles, project: p, isSnapshot: args.Snapshot, stateful: args.Stateful, }, } // Cleanup the zero values if d.expiryDate.IsZero() { d.expiryDate = time.Time{} } if d.creationDate.IsZero() { d.creationDate = time.Time{} } if d.lastUsedDate.IsZero() { d.lastUsedDate = time.Time{} } if args.Snapshot { d.logger.Info("Creating instance snapshot", logger.Ctx{"ephemeral": d.ephemeral}) } else { d.logger.Info("Creating instance", logger.Ctx{"ephemeral": d.ephemeral}) } // Load the config. err := d.init() if err != nil { return nil, nil, fmt.Errorf("Failed to expand config: %w", err) } // When not a snapshot, perform full validation. if !args.Snapshot { // Validate expanded config (allows mixed instance types for profiles). err = instance.ValidConfig(s.OS, d.expandedConfig, true, instancetype.Any) if err != nil { return nil, nil, fmt.Errorf("Invalid config: %w", err) } err = instance.ValidDevices(s, d.project, d.Type(), d.localDevices, d.expandedDevices) if err != nil { return nil, nil, fmt.Errorf("Invalid devices: %w", err) } } _, rootDiskDevice, err := d.getRootDiskDevice() if err != nil { return nil, nil, fmt.Errorf("Failed getting root disk: %w", err) } if rootDiskDevice["pool"] == "" { return nil, nil, errors.New("The instance's root device is missing the pool property") } // Initialize the storage pool. d.storagePool, err = storagePools.LoadByName(d.state, rootDiskDevice["pool"]) if err != nil { return nil, nil, fmt.Errorf("Failed loading storage pool: %w", err) } volType, err := storagePools.InstanceTypeToVolumeType(d.Type()) if err != nil { return nil, nil, err } storagePoolSupported := slices.Contains(d.storagePool.Driver().Info().VolumeTypes, volType) if !storagePoolSupported { return nil, nil, errors.New("Storage pool does not support instance type") } // Setup the initial idmap config. var idmapSet *idmap.Set base := int64(0) if !d.IsPrivileged() { idmapSet, base, err = d.findIdmap() if err != nil { return nil, nil, err } } idmapSetJSON, err := idmapSet.ToJSON() if err != nil { return nil, nil, fmt.Errorf("Failed to encode ID map: %w", err) } v := map[string]string{ "volatile.idmap.next": idmapSetJSON, "volatile.idmap.base": fmt.Sprintf("%v", base), } // Invalidate the idmap cache. d.idmapset = nil // Set last_state if not currently set. if d.localConfig["volatile.last_state.idmap"] == "" { v["volatile.last_state.idmap"] = "[]" } err = d.VolatileSet(v) if err != nil { return nil, nil, err } // Re-run init to update the idmap. err = d.init() if err != nil { return nil, nil, err } if !d.IsSnapshot() { // Add devices to container. cleanup, err := d.devicesAdd(d, false, partialDeviceValidation) if err != nil { return nil, nil, err } reverter.Add(cleanup) } if d.isSnapshot { d.logger.Info("Created instance snapshot", logger.Ctx{"ephemeral": d.ephemeral}) } else { d.logger.Info("Created instance", logger.Ctx{"ephemeral": d.ephemeral}) } if d.isSnapshot { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceSnapshotCreated.Event(d, nil)) } else { // Add instance to authorizer. err = d.state.Authorizer.AddInstance(d.state.ShutdownCtx, d.project.Name, d.Name()) if err != nil { logger.Error("Failed to add instance to authorizer", logger.Ctx{"instanceName": d.Name(), "projectName": d.project.Name, "error": err}) } reverter.Add(func() { _ = d.state.Authorizer.DeleteInstance(d.state.ShutdownCtx, d.project.Name, d.Name()) }) d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceCreated.Event(d, map[string]any{ "type": api.InstanceTypeContainer, "storage-pool": d.storagePool.Name(), "location": d.Location(), })) } cleanup := reverter.Clone().Fail reverter.Success() return d, cleanup, err } func lxcLoad(s *state.State, args db.InstanceArgs, p api.Project) (instance.Instance, error) { // Create the container struct d := lxcInstantiate(s, args, nil, p) // Expand config and devices err := d.(*lxc).expandConfig() if err != nil { return nil, err } return d, nil } // Unload is called by the garbage collector. func lxcUnload(d *lxc) { d.release() } // release releases any internal reference to a liblxc container, invalidating the go-lxc cache. func (d *lxc) release() { d.cMu.Lock() defer d.cMu.Unlock() if d.c != nil { _ = d.c.Release() d.c = nil } } // Create a container struct without initializing it. func lxcInstantiate(s *state.State, args db.InstanceArgs, expandedDevices deviceConfig.Devices, p api.Project) instance.Instance { d := &lxc{ common: common{ state: s, architecture: args.Architecture, creationDate: args.CreationDate, dbType: args.Type, description: args.Description, ephemeral: args.Ephemeral, expiryDate: args.ExpiryDate, id: args.ID, lastUsedDate: args.LastUsedDate, localConfig: args.Config, localDevices: args.Devices, logger: logger.AddContext(logger.Ctx{"instanceType": args.Type, "instance": args.Name, "project": args.Project}), name: args.Name, node: args.Node, profiles: args.Profiles, project: p, isSnapshot: args.Snapshot, stateful: args.Stateful, }, } // Cleanup the zero values if d.expiryDate.IsZero() { d.expiryDate = time.Time{} } if d.creationDate.IsZero() { d.creationDate = time.Time{} } if d.lastUsedDate.IsZero() { d.lastUsedDate = time.Time{} } // This is passed during expanded config validation. if expandedDevices != nil { d.expandedDevices = expandedDevices } return d } // The LXC container driver. type lxc struct { common // Config handling. fromHook bool cMu sync.Mutex cFinalizer sync.Once // Cached handles. // Do not use these variables directly, instead use their associated get functions so they // will be initialized on demand. c *liblxc.Container // Use d.initLXC() instead of accessing this directly. cConfig bool idmapset *idmap.Set } var idmapLock sync.Mutex func (d *lxc) findIdmap() (*idmap.Set, int64, error) { if d.state.OS.IdmapSet == nil { return nil, 0, errors.New("System doesn't have a functional idmap setup") } idmapSize := func(size string) (int64, error) { var idMapSize int64 if size == "" || size == "auto" { if util.IsTrue(d.expandedConfig["security.idmap.isolated"]) { idMapSize = 65536 } else { if len(d.state.OS.IdmapSet.Entries) != 2 { return 0, fmt.Errorf("Bad initial idmap: %v", d.state.OS.IdmapSet) } idMapSize = d.state.OS.IdmapSet.Entries[0].MapRange } } else { size, err := strconv.ParseInt(size, 10, 64) if err != nil { return 0, err } idMapSize = size } return idMapSize, nil } rawMaps, err := idmap.NewSetFromIncusIDMap(d.expandedConfig["raw.idmap"]) if err != nil { return nil, 0, err } mkIdmap := func(offset int64, size int64) (*idmap.Set, error) { set := &idmap.Set{Entries: []idmap.Entry{ {IsUID: true, NSID: 0, HostID: offset, MapRange: size}, {IsGID: true, NSID: 0, HostID: offset, MapRange: size}, }} for _, ent := range rawMaps.Entries { err := set.AddSafe(ent) if err != nil && errors.Is(err, idmap.ErrHostIDIsSubID) { return nil, err } } return set, nil } if !util.IsTrue(d.expandedConfig["security.idmap.isolated"]) { // Create a new set based from the global one. newIdmapset := idmap.Set{Entries: make([]idmap.Entry, len(d.state.OS.IdmapSet.Entries))} copy(newIdmapset.Entries, d.state.OS.IdmapSet.Entries) // Restrict the range sizes if specified. if d.expandedConfig["security.idmap.size"] != "" { size, err := idmapSize(d.expandedConfig["security.idmap.size"]) if err != nil { return nil, 0, err } for k, ent := range newIdmapset.Entries { if ent.MapRange < size { continue } newIdmapset.Entries[k].MapRange = size } } // Apply the raw idmap entries. for _, ent := range rawMaps.Entries { err := newIdmapset.AddSafe(ent) if err != nil && errors.Is(err, idmap.ErrHostIDIsSubID) { return nil, 0, err } } return &newIdmapset, 0, nil } size, err := idmapSize(d.expandedConfig["security.idmap.size"]) if err != nil { return nil, 0, err } if d.expandedConfig["security.idmap.base"] != "" { offset, err := strconv.ParseInt(d.expandedConfig["security.idmap.base"], 10, 64) if err != nil { return nil, 0, err } set, err := mkIdmap(offset, size) if err != nil && errors.Is(err, idmap.ErrHostIDIsSubID) { return nil, 0, err } return set, offset, nil } idmapLock.Lock() defer idmapLock.Unlock() cts, err := instance.LoadNodeAll(d.state, instancetype.Container) if err != nil { return nil, 0, err } offset := d.state.OS.IdmapSet.Entries[0].HostID + 65536 mapentries := idmap.ByHostID{} for _, container := range cts { if container.Type() != instancetype.Container { continue } /* Don't change our map Just Because. */ if container.ID() == d.id { continue } if container.IsPrivileged() { continue } if util.IsFalseOrEmpty(container.ExpandedConfig()["security.idmap.isolated"]) { continue } if container.ExpandedConfig()["volatile.idmap.base"] == "" { continue } cBase, err := strconv.ParseInt(container.ExpandedConfig()["volatile.idmap.base"], 10, 64) if err != nil { return nil, 0, err } cSize, err := idmapSize(container.ExpandedConfig()["security.idmap.size"]) if err != nil { return nil, 0, err } mapentries.Entries = append(mapentries.Entries, idmap.Entry{HostID: int64(cBase), MapRange: cSize}) } sort.Sort(mapentries) for i := range mapentries.Entries { if i == 0 { if mapentries.Entries[0].HostID < offset+size { offset = mapentries.Entries[0].HostID + mapentries.Entries[0].MapRange continue } set, err := mkIdmap(offset, size) if err != nil && errors.Is(err, idmap.ErrHostIDIsSubID) { return nil, 0, err } return set, offset, nil } if mapentries.Entries[i-1].HostID+mapentries.Entries[i-1].MapRange > offset { offset = mapentries.Entries[i-1].HostID + mapentries.Entries[i-1].MapRange continue } offset = mapentries.Entries[i-1].HostID + mapentries.Entries[i-1].MapRange if offset+size < mapentries.Entries[i].HostID { set, err := mkIdmap(offset, size) if err != nil && errors.Is(err, idmap.ErrHostIDIsSubID) { return nil, 0, err } return set, offset, nil } offset = mapentries.Entries[i].HostID + mapentries.Entries[i].MapRange } if offset+size <= d.state.OS.IdmapSet.Entries[0].HostID+d.state.OS.IdmapSet.Entries[0].MapRange { set, err := mkIdmap(offset, size) if err != nil && errors.Is(err, idmap.ErrHostIDIsSubID) { return nil, 0, err } return set, offset, nil } return nil, 0, errors.New("Not enough uid/gid available for the container") } func (d *lxc) init() error { // Compute the expanded config and device list err := d.expandConfig() if err != nil { return err } return nil } func (d *lxc) initLXC(config bool) (*liblxc.Container, error) { d.cMu.Lock() defer d.cMu.Unlock() // No need to go through all that for snapshots if d.IsSnapshot() { return nil, nil } // Check if being called from a hook if d.fromHook { return nil, errors.New("You can't use go-lxc from inside a LXC hook") } // Check if already initialized if d.c != nil && (!config || d.cConfig) { return d.c, nil } // As we are now going to be initialising a liblxc.Container reference, set the finalizer so that it is // cleaned up (if needed) when the garbage collector destroys this instance struct. d.cFinalizer.Do(func() { runtime.SetFinalizer(d, lxcUnload) }) reverter := revert.New() defer reverter.Fail() // Load the go-lxc struct cname := project.Instance(d.Project().Name, d.Name()) cc, err := liblxc.NewContainer(cname, d.state.OS.LxcPath) if err != nil { return nil, err } reverter.Add(func() { _ = cc.Release() }) // Load cgroup abstraction cg, err := d.cgroup(cc, false) if err != nil { return nil, err } // Setup logging logfile := d.LogFilePath() err = lxcSetConfigItem(cc, "lxc.log.file", logfile) if err != nil { return nil, err } logLevel := "warn" if daemon.Debug { logLevel = "trace" } else if daemon.Verbose { logLevel = "info" } err = lxcSetConfigItem(cc, "lxc.log.level", logLevel) if err != nil { return nil, err } if liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 3, 0, 0) { // Default size log buffer err = lxcSetConfigItem(cc, "lxc.console.buffer.size", "auto") if err != nil { return nil, err } err = lxcSetConfigItem(cc, "lxc.console.size", "auto") if err != nil { return nil, err } // File to dump ringbuffer contents to when requested or // container shutdown. consoleBufferLogFile := d.ConsoleBufferLogPath() err = lxcSetConfigItem(cc, "lxc.console.logfile", consoleBufferLogFile) if err != nil { return nil, err } } err = lxcSetConfigItem(cc, "lxc.sched.core", "1") if err != nil { return nil, err } // Allow for lightweight init d.cConfig = config if !config { if d.c != nil { _ = d.c.Release() } d.c = cc reverter.Success() return cc, err } if d.IsPrivileged() { // Base config toDrop := "sys_time sys_module sys_rawio" if !d.state.OS.AppArmorStacking || d.state.OS.AppArmorStacked { toDrop = toDrop + " mac_admin mac_override" } err = lxcSetConfigItem(cc, "lxc.cap.drop", toDrop) if err != nil { return nil, err } } // Set an appropriate /proc, /sys/ and /sys/fs/cgroup mounts := []string{} if d.IsPrivileged() && !d.state.OS.RunningInUserNS { mounts = append(mounts, "proc:mixed") mounts = append(mounts, "sys:mixed") } else { mounts = append(mounts, "proc:rw") mounts = append(mounts, "sys:rw") } mounts = append(mounts, "cgroup:rw:force") err = lxcSetConfigItem(cc, "lxc.mount.auto", strings.Join(mounts, " ")) if err != nil { return nil, err } err = lxcSetConfigItem(cc, "lxc.autodev", "1") if err != nil { return nil, err } err = lxcSetConfigItem(cc, "lxc.pty.max", "1024") if err != nil { return nil, err } bindMounts := []string{ "/dev/fuse", "/dev/net/tun", "/sys/firmware/efi/efivars", "/sys/fs/fuse/connections", "/sys/fs/pstore", "/sys/kernel/config", "/sys/kernel/debug", "/sys/kernel/security", "/sys/kernel/tracing", } // Handle unprivileged binfmt_misc. if d.IsPrivileged() { bindMounts = append(bindMounts, "/proc/sys/fs/binfmt_misc") } // Pass in /dev/zfs to the container if delegation is supported on the system. // This is only done for unprivileged containers as delegation is tied to the user namespace. if !d.IsPrivileged() && storageDrivers.ZFSSupportsDelegation() && util.PathExists("/dev/zfs") { bindMounts = append(bindMounts, "/dev/zfs") } if d.IsPrivileged() && !d.state.OS.RunningInUserNS { err = lxcSetConfigItem(cc, "lxc.mount.entry", "mqueue dev/mqueue mqueue rw,relatime,create=dir,optional 0 0") if err != nil { return nil, err } } else { bindMounts = append(bindMounts, "/dev/mqueue") } for _, mnt := range bindMounts { if !util.PathExists(mnt) { continue } if internalUtil.IsDir(mnt) { err = lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s %s none rbind,create=dir,optional 0 0", mnt, strings.TrimPrefix(mnt, "/"))) if err != nil { return nil, err } } else { err = lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s %s none bind,create=file,optional 0 0", mnt, strings.TrimPrefix(mnt, "/"))) if err != nil { return nil, err } } } // For lxcfs templateConfDir := os.Getenv("INCUS_LXC_TEMPLATE_CONFIG") if templateConfDir == "" { templateConfDir = "/usr/share/lxc/config" } if util.PathExists(fmt.Sprintf("%s/common.conf.d/", templateConfDir)) { err = lxcSetConfigItem(cc, "lxc.include", fmt.Sprintf("%s/common.conf.d/", templateConfDir)) if err != nil { return nil, err } } // Configure devices cgroup if d.IsPrivileged() && !d.state.OS.RunningInUserNS { err = lxcSetConfigItem(cc, "lxc.cgroup2.devices.deny", "a") if err != nil { return nil, err } devices := []string{ "b *:* m", // Allow mknod of block devices "c *:* m", // Allow mknod of char devices "c 136:* rwm", // /dev/pts devices "c 1:3 rwm", // /dev/null "c 1:5 rwm", // /dev/zero "c 1:7 rwm", // /dev/full "c 1:8 rwm", // /dev/random "c 1:9 rwm", // /dev/urandom "c 5:0 rwm", // /dev/tty "c 5:1 rwm", // /dev/console "c 5:2 rwm", // /dev/ptmx "c 10:229 rwm", // /dev/fuse "c 10:200 rwm", // /dev/net/tun } if storageDrivers.ZFSSupportsDelegation() { devices = append(devices, "c 10:249 rwm") } for _, dev := range devices { err = lxcSetConfigItem(cc, "lxc.cgroup2.devices.allow", dev) if err != nil { return nil, err } } } if d.IsNesting() { /* * mount extra /proc and /sys to work around kernel * restrictions on remounting them when covered */ err = lxcSetConfigItem(cc, "lxc.mount.entry", "proc dev/.lxc/proc proc create=dir,optional 0 0") if err != nil { return nil, err } err = lxcSetConfigItem(cc, "lxc.mount.entry", "sys dev/.lxc/sys sysfs create=dir,optional 0 0") if err != nil { return nil, err } } // Setup architecture personality, err := osarch.ArchitecturePersonality(d.architecture) if err != nil { personality, err = osarch.ArchitecturePersonality(d.state.OS.Architectures[0]) if err != nil { return nil, err } } err = lxcSetConfigItem(cc, "lxc.arch", personality) if err != nil { return nil, err } // Setup the hooks err = lxcSetConfigItem(cc, "lxc.hook.version", "1") if err != nil { return nil, err } // Call the onstart hook on start. err = lxcSetConfigItem(cc, "lxc.hook.pre-start", fmt.Sprintf("/proc/%d/exe callhook %s %s %s start", os.Getpid(), internalUtil.VarPath(""), util.SingleQuote(d.Project().Name), util.SingleQuote(d.Name()))) if err != nil { return nil, err } // Call the onstopns hook on stop but before namespaces are unmounted. err = lxcSetConfigItem(cc, "lxc.hook.stop", fmt.Sprintf("%s callhook %s %s %s stopns", d.state.OS.ExecPath, internalUtil.VarPath(""), util.SingleQuote(d.Project().Name), util.SingleQuote(d.Name()))) if err != nil { return nil, err } // Call the onstop hook on stop. err = lxcSetConfigItem(cc, "lxc.hook.post-stop", fmt.Sprintf("%s callhook %s %s %s stop", d.state.OS.ExecPath, internalUtil.VarPath(""), util.SingleQuote(d.Project().Name), util.SingleQuote(d.Name()))) if err != nil { return nil, err } // Setup the console err = lxcSetConfigItem(cc, "lxc.tty.max", "0") if err != nil { return nil, err } // Setup the hostname err = lxcSetConfigItem(cc, "lxc.uts.name", d.Name()) if err != nil { return nil, err } // Setup devIncus if util.IsTrueOrEmpty(d.expandedConfig["security.guestapi"]) { err = lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s dev/incus none bind,create=dir 0 0", internalUtil.VarPath("guestapi"))) if err != nil { return nil, err } } // Setup AppArmor if d.state.OS.AppArmorAvailable { if d.state.OS.AppArmorConfined || !d.state.OS.AppArmorAdmin { // If confined but otherwise able to use AppArmor, use our own profile curProfile := localUtil.AppArmorProfile() curProfile = strings.TrimSuffix(curProfile, " (enforce)") err := lxcSetConfigItem(cc, "lxc.apparmor.profile", curProfile) if err != nil { return nil, err } } else { // If not currently confined, use the container's profile profile := apparmor.InstanceProfileName(d) /* In the nesting case, we want to enable the inside * daemon to load its profile. Unprivileged containers can * load profiles, but privileged containers cannot, so * let's not use a namespace so they can fall back to * the old way of nesting, i.e. using the parent's * profile. */ if d.state.OS.AppArmorStacking && !d.state.OS.AppArmorStacked { profile = fmt.Sprintf("%s//&:%s:", profile, apparmor.InstanceNamespaceName(d)) } err := lxcSetConfigItem(cc, "lxc.apparmor.profile", profile) if err != nil { return nil, err } } } else { // Make sure that LXC won't try to apply an apparmor profile. // This may fail on liblxc compiled without apparmor, so ignore errors. _ = lxcSetConfigItem(cc, "lxc.apparmor.profile", "unconfined") } // Setup SELinux. if d.state.OS.SELinuxAvailable && d.state.OS.SELinuxContextInstanceLXC != "" { seContext, err := d.selinuxContext(d.state.OS.SELinuxContextInstanceLXC) if err != nil { return nil, err } err = lxcSetConfigItem(cc, "lxc.selinux.context", seContext) if err != nil { return nil, err } } // Setup Seccomp if necessary if seccomp.InstanceNeedsPolicy(d) { err = lxcSetConfigItem(cc, "lxc.seccomp.profile", seccomp.ProfilePath(d)) if err != nil { return nil, err } // Setup notification socket // System requirement errors are handled during policy generation instead of here ok, err := seccomp.InstanceNeedsIntercept(d.state, d) if err == nil && ok { err = lxcSetConfigItem(cc, "lxc.seccomp.notify.proxy", fmt.Sprintf("unix:%s", internalUtil.RunPath("seccomp.socket"))) if err != nil { return nil, err } } } // Setup idmap idmapset, err := d.NextIdmap() if err != nil { return nil, err } if idmapset != nil { lines := idmapset.ToLXCString() for _, line := range lines { err := lxcSetConfigItem(cc, "lxc.idmap", line) if err != nil { return nil, err } } } // Setup environment for k, v := range d.expandedConfig { // gendoc:generate(entity=instance, group=miscellaneous, key=environment.*) // The specified key/value environment variables are exported to the instance and set for `incus exec`. // --- // type: string // liveupdate: yes (exec) // shortdesc: Environment variables to export after, ok := strings.CutPrefix(k, "environment.") if ok { err = lxcSetConfigItem(cc, "lxc.environment", fmt.Sprintf("%s=%s", after, v)) if err != nil { return nil, err } } } err = lxcSetConfigItem(cc, "lxc.environment", "CREDENTIALS_DIRECTORY=/dev/.incus-systemd-credentials") if err != nil { return nil, err } err = lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s dev/.incus-systemd-credentials none bind,ro,create=dir 0 0", filepath.Join(d.Path(), "credentials"))) if err != nil { return nil, err } // Setup NVIDIA runtime if util.IsTrue(d.expandedConfig["nvidia.runtime"]) { hookDir := os.Getenv("INCUS_LXC_HOOK") if hookDir == "" { hookDir = "/usr/share/lxc/hooks" } hookPath := filepath.Join(hookDir, "nvidia") if !util.PathExists(hookPath) { return nil, errors.New("The NVIDIA LXC hook couldn't be found") } _, err := exec.LookPath("nvidia-container-cli") if err != nil { return nil, errors.New("The NVIDIA container tools couldn't be found") } err = lxcSetConfigItem(cc, "lxc.environment", "NVIDIA_VISIBLE_DEVICES=none") if err != nil { return nil, err } nvidiaDriver := d.expandedConfig["nvidia.driver.capabilities"] if nvidiaDriver == "" { err = lxcSetConfigItem(cc, "lxc.environment", "NVIDIA_DRIVER_CAPABILITIES=compute,utility") if err != nil { return nil, err } } else { err = lxcSetConfigItem(cc, "lxc.environment", fmt.Sprintf("NVIDIA_DRIVER_CAPABILITIES=%s", nvidiaDriver)) if err != nil { return nil, err } } nvidiaRequireCuda := d.expandedConfig["nvidia.require.cuda"] if nvidiaRequireCuda == "" { err = lxcSetConfigItem(cc, "lxc.environment", fmt.Sprintf("NVIDIA_REQUIRE_CUDA=%s", nvidiaRequireCuda)) if err != nil { return nil, err } } nvidiaRequireDriver := d.expandedConfig["nvidia.require.driver"] if nvidiaRequireDriver == "" { err = lxcSetConfigItem(cc, "lxc.environment", fmt.Sprintf("NVIDIA_REQUIRE_DRIVER=%s", nvidiaRequireDriver)) if err != nil { return nil, err } } err = lxcSetConfigItem(cc, "lxc.hook.mount", hookPath) if err != nil { return nil, err } } // Memory limits if cgroup.Supports(cgroup.Memory) { memory := d.expandedConfig["limits.memory"] memoryEnforce := d.expandedConfig["limits.memory.enforce"] memorySwap := d.expandedConfig["limits.memory.swap"] memorySwapPriority := d.expandedConfig["limits.memory.swap.priority"] // Configure the memory limits if memory != "" { valueInt, err := ParseMemoryStr(memory) if err != nil { return nil, err } if memoryEnforce == "soft" { err = cg.SetMemorySoftLimit(valueInt) if err != nil { return nil, err } } else { err = cg.SetMemoryLimit(valueInt) if err != nil { return nil, err } if util.IsTrueOrEmpty(memorySwap) || util.IsFalse(memorySwap) { err = cg.SetMemorySwapLimit(0) if err != nil { return nil, err } } else { // Additional memory as swap. swapInt, err := units.ParseByteSizeString(memorySwap) if err != nil { return nil, err } err = cg.SetMemorySwapLimit(swapInt) if err != nil { return nil, err } } } } // Configure the swappiness if util.IsFalse(memorySwap) { err = cg.SetMemorySwappiness(0) if err != nil { return nil, err } } else if memorySwapPriority != "" { priority, err := strconv.Atoi(memorySwapPriority) if err != nil { return nil, err } // Maximum priority (10) should be default swappiness (60). err = cg.SetMemorySwappiness(int64(70 - priority)) if err != nil { return nil, err } } } // CPU limits cpuPriority := d.expandedConfig["limits.cpu.priority"] cpuAllowance := d.expandedConfig["limits.cpu.allowance"] if (cpuPriority != "" || cpuAllowance != "") && cgroup.Supports(cgroup.CPU) { cpuShares, cpuCfsQuota, cpuCfsPeriod, err := cgroup.ParseCPU(cpuAllowance, cpuPriority) if err != nil { return nil, err } if cpuShares != 1024 { err = cg.SetCPUShare(cpuShares) if err != nil { return nil, err } } if cpuCfsPeriod != -1 && cpuCfsQuota != -1 { err = cg.SetCPUCfsLimit(cpuCfsPeriod, cpuCfsQuota) if err != nil { return nil, err } } } // Disk priority limits. diskPriority := d.ExpandedConfig()["limits.disk.priority"] if diskPriority != "" { if !cgroup.Supports(cgroup.IO) { return nil, errors.New("Cannot apply limits.disk.priority as blkio.weight cgroup controller is missing") } priorityInt, err := strconv.Atoi(diskPriority) if err != nil { return nil, err } priority := priorityInt * 100 // Minimum valid value is 10 if priority == 0 { priority = 10 } err = cg.SetBlkioWeight(int64(priority)) if err != nil { return nil, err } } // Processes if cgroup.Supports(cgroup.Pids) { processes := d.expandedConfig["limits.processes"] if processes != "" { valueInt, err := strconv.ParseInt(processes, 10, 64) if err != nil { return nil, err } err = cg.SetMaxProcesses(valueInt) if err != nil { return nil, err } } } // Hugepages if cgroup.Supports(cgroup.Hugetlb) { for i, key := range internalInstance.HugePageSizeKeys { value := d.expandedConfig[key] if value != "" { value, err := units.ParseByteSizeString(value) if err != nil { return nil, err } err = cg.SetHugepagesLimit(internalInstance.HugePageSizeSuffix[i], value) if err != nil { return nil, err } } } } // Setup process limits for k, v := range d.expandedConfig { after, ok := strings.CutPrefix(k, "limits.kernel.") if ok { prlimitSuffix := after prlimitKey := fmt.Sprintf("lxc.prlimit.%s", prlimitSuffix) err = lxcSetConfigItem(cc, prlimitKey, v) if err != nil { return nil, err } } } // Setup sysctls for k, v := range d.expandedConfig { // gendoc:generate(entity=instance, group=miscellaneous, key=linux.sysctl.*) // // --- // type: string // liveupdate: no // condition: container // shortdesc: Override for the corresponding `sysctl` setting in the container after, ok := strings.CutPrefix(k, "linux.sysctl.") if ok { sysctlSuffix := after sysctlKey := fmt.Sprintf("lxc.sysctl.%s", sysctlSuffix) err = lxcSetConfigItem(cc, sysctlKey, v) if err != nil { return nil, err } } } // Setup shmounts err = lxcSetConfigItem(cc, "lxc.mount.auto", fmt.Sprintf("shmounts:%s:/dev/.incus-mounts", d.ShmountsPath())) if err != nil { return nil, err } if d.c != nil { _ = d.c.Release() } d.c = cc reverter.Success() return cc, err } var ( idmappedStorageMap map[unix.Fsid]idmap.StorageType = map[unix.Fsid]idmap.StorageType{} idmappedStorageMapString map[string]idmap.StorageType = map[string]idmap.StorageType{} idmappedStorageMapLock sync.Mutex ) // IdmappedStorage determines if the container can use idmapped mounts. func (d *lxc) IdmappedStorage(fspath string, fstype string) idmap.StorageType { var mode idmap.StorageType = idmap.StorageTypeNone var bindMount bool = fstype == "none" || fstype == "" buf := &unix.Statfs_t{} if bindMount { err := unix.Statfs(fspath, buf) if err != nil { d.logger.Error("Failed to statfs", logger.Ctx{"path": fspath, "err": err}) return mode } } idmappedStorageMapLock.Lock() defer idmappedStorageMapLock.Unlock() if bindMount { val, ok := idmappedStorageMap[buf.Fsid] if ok { // Return recorded idmapping type. return val } } else { val, ok := idmappedStorageMapString[fstype] if ok { // Return recorded idmapping type. return val } } if idmap.CanIdmapMount(fspath, fstype) { // Use idmapped mounts. mode = idmap.StorageTypeIdmapped } if bindMount { idmappedStorageMap[buf.Fsid] = mode } else { idmappedStorageMapString[fstype] = mode } return mode } func (d *lxc) devIncusEventSend(eventType string, eventMessage map[string]any) error { event := jmap.Map{} event["type"] = eventType event["timestamp"] = time.Now() event["metadata"] = eventMessage return d.state.DevIncusEvents.Send(d.ID(), eventType, eventMessage) } // RegisterDevices calls the Register() function on all of the instance's devices. func (d *lxc) RegisterDevices() { d.devicesRegister(d) } // deviceStart loads a new device and calls its Start() function. func (d *lxc) deviceStart(dev device.Device, instanceRunning bool) (*deviceConfig.RunConfig, error) { configCopy := dev.Config() l := d.logger.AddContext(logger.Ctx{"device": dev.Name(), "type": configCopy["type"]}) l.Debug("Starting device") reverter := revert.New() defer reverter.Fail() if instanceRunning && !dev.CanHotPlug() { return nil, errors.New("Device cannot be started when instance is running") } runConf, err := dev.Start() if err != nil { return nil, err } reverter.Add(func() { runConf, _ := dev.Stop() if runConf != nil { _ = d.runHooks(runConf.PostHooks) } }) // If runConf supplied, perform any container specific setup of device. if runConf != nil { // Shift device file ownership if needed before mounting into container. // This needs to be done whether or not container is running. if len(runConf.Mounts) > 0 { err := d.deviceStaticShiftMounts(runConf.Mounts) if err != nil { return nil, err } } // If container is running and then live attach device. if instanceRunning { // Attach mounts if requested. if len(runConf.Mounts) > 0 { err = d.deviceHandleMounts(runConf.Mounts) if err != nil { return nil, err } } // Add cgroup rules if requested. if len(runConf.CGroups) > 0 { err = d.deviceAddCgroupRules(runConf.CGroups) if err != nil { return nil, err } } // Attach network interface if requested. if len(runConf.NetworkInterface) > 0 { err = d.deviceAttachNIC(dev.Name(), configCopy, runConf.NetworkInterface) if err != nil { return nil, err } } // If running, run post start hooks now (if not running, they will be run // once the instance is started). err = d.runHooks(runConf.PostHooks) if err != nil { return nil, err } } } reverter.Success() return runConf, nil } // deviceStaticShiftMounts statically shift device mount files ownership to active idmap if needed. func (d *lxc) deviceStaticShiftMounts(mounts []deviceConfig.MountEntryItem) error { idmapSet, err := d.CurrentIdmap() if err != nil { return fmt.Errorf("Failed to get idmap for device: %s", err) } // If there is an idmap being applied and the daemon is not running in a user namespace then shift the // device files before they are mounted. if idmapSet != nil && !d.state.OS.RunningInUserNS { for _, mount := range mounts { // Skip UID/GID shifting if OwnerShift mode is not static, or the host-side // DevPath is empty (meaning an unmount request that doesn't need shifting). if mount.OwnerShift != deviceConfig.MountOwnerShiftStatic || mount.DevPath == "" { continue } err := idmapSet.ShiftPath(mount.DevPath, nil) if err != nil { // uidshift failing is weird, but not a big problem. Log and proceed. d.logger.Debug("Failed to uidshift device", logger.Ctx{"mountDevPath": mount.DevPath, "err": err}) } } } return nil } // deviceAddCgroupRules live adds cgroup rules to a container. func (d *lxc) deviceAddCgroupRules(cgroups []deviceConfig.RunConfigItem) error { _, err := d.initLXC(false) if err != nil { return err } for _, rule := range cgroups { // Only apply devices cgroup rules if container is running privileged and host has devices cgroup controller. if strings.HasPrefix(rule.Key, "devices.") && (!d.isCurrentlyPrivileged() || d.state.OS.RunningInUserNS) { continue } // Add the new device cgroup rule. err := d.CGroupSet(rule.Key, rule.Value) if err != nil { return fmt.Errorf("Failed to add cgroup rule for device: %w", err) } } return nil } // deviceAttachNIC live attaches a NIC device to a container. func (d *lxc) deviceAttachNIC(devName string, configCopy map[string]string, netIF []deviceConfig.RunConfigItem) error { ctDevName := "" connected := true for _, dev := range netIF { if dev.Key == "link" { ctDevName = dev.Value } else if dev.Key == "connected" { connected = util.IsTrueOrEmpty(dev.Value) } } if ctDevName == "" { return errors.New("Device didn't provide a link property to use") } // Load the go-lxc struct. cc, err := d.initLXC(false) if err != nil { return err } // Add the interface to the container. err = cc.AttachInterface(ctDevName, configCopy["name"]) if err != nil { return fmt.Errorf("Failed to attach interface: %s to %s: %w", ctDevName, configCopy["name"], err) } return d.setNICLink(devName, connected, true) } // deviceStop loads a new device and calls its Stop() function. // Accepts a stopHookNetnsPath argument which is required when run from the onStopNS hook before the // container's network namespace is unmounted (which is required for NIC device cleanup). func (d *lxc) deviceStop(dev device.Device, instanceRunning bool, stopHookNetnsPath string) error { configCopy := dev.Config() l := d.logger.AddContext(logger.Ctx{"device": dev.Name(), "type": configCopy["type"]}) l.Debug("Stopping device") if instanceRunning && !dev.CanHotPlug() { return errors.New("Device cannot be stopped when instance is running") } runConf, err := dev.Stop() if err != nil { return err } if runConf != nil { // If network interface settings returned, then detach NIC from container. if len(runConf.NetworkInterface) > 0 { err = d.deviceDetachNIC(configCopy, runConf.NetworkInterface, instanceRunning, stopHookNetnsPath) if err != nil { return err } } // Add cgroup rules if requested and container is running. if len(runConf.CGroups) > 0 && instanceRunning { err = d.deviceAddCgroupRules(runConf.CGroups) if err != nil { return err } } // Detach mounts if requested and container is running. if len(runConf.Mounts) > 0 && instanceRunning { err = d.deviceHandleMounts(runConf.Mounts) if err != nil { return err } } // Run post stop hooks irrespective of run state of instance. err = d.runHooks(runConf.PostHooks) if err != nil { return err } } return nil } // deviceDetachNIC detaches a NIC device from a container. // Accepts a stopHookNetnsPath argument which is required when run from the onStopNS hook before the // container's network namespace is unmounted (which is required for NIC device cleanup). func (d *lxc) deviceDetachNIC(configCopy map[string]string, netIF []deviceConfig.RunConfigItem, instanceRunning bool, stopHookNetnsPath string) error { // Get requested device name to detach interface back to on the host. devName := "" for _, dev := range netIF { if dev.Key == "link" { devName = dev.Value break } } if devName == "" { return errors.New("Device didn't provide a link property to use") } // If container is running, perform live detach of interface back to host. if instanceRunning { // For some reason, having network config confuses detach, so get our own go-lxc struct. cname := project.Instance(d.Project().Name, d.Name()) cc, err := liblxc.NewContainer(cname, d.state.OS.LxcPath) if err != nil { return err } defer func() { _ = cc.Release() }() // Get interfaces inside container. ifaces, err := cc.Interfaces() if err != nil { return fmt.Errorf("Failed to list network interfaces: %w", err) } // If interface doesn't exist inside container, cannot proceed. if !slices.Contains(ifaces, configCopy["name"]) { return nil } err = cc.DetachInterfaceRename(configCopy["name"], devName) if err != nil { return fmt.Errorf("Failed to detach interface: %q to %q: %w", configCopy["name"], devName, err) } } else { // Currently liblxc does not move devices back to the host on stop that were added // after the container was started. For this reason we utilise the lxc.hook.stop // hook so that we can capture the netns path, enter the namespace and move the nics // back to the host and rename them if liblxc hasn't already done it. // We can only move back devices that have an expected host_name record and where // that device doesn't already exist on the host as if a device exists on the host // we can't know whether that is because liblxc has moved it back already or whether // it is a conflicting device. if !util.PathExists(fmt.Sprintf("/sys/class/net/%s", devName)) { if stopHookNetnsPath == "" { return fmt.Errorf("Cannot detach NIC device %q without stopHookNetnsPath being provided", devName) } err := d.detachInterfaceRename(stopHookNetnsPath, configCopy["name"], devName) if err != nil { return fmt.Errorf("Failed to detach interface: %q to %q: %w", configCopy["name"], devName, err) } d.logger.Debug("Detached NIC device interface", logger.Ctx{"name": configCopy["name"], "device": devName}) } } return nil } // deviceHandleMounts live attaches or detaches mounts on a container. // If the mount DevPath is empty the mount action is treated as unmount. func (d *lxc) deviceHandleMounts(mounts []deviceConfig.MountEntryItem) error { for _, mount := range mounts { if mount.DevPath != "" { flags := 0 // Convert options into flags. for _, opt := range mount.Opts { if opt == "bind" { flags |= unix.MS_BIND } else if opt == "rbind" { flags |= unix.MS_BIND | unix.MS_REC } else if opt == "ro" { flags |= unix.MS_RDONLY } } var idmapType idmap.StorageType = idmap.StorageTypeNone if !d.IsPrivileged() && mount.OwnerShift == deviceConfig.MountOwnerShiftDynamic { idmapType = d.IdmappedStorage(mount.DevPath, mount.FSType) if idmapType == idmap.StorageTypeNone { return errors.New("Required idmapping abilities not available") } } // Mount it into the container. err := d.insertMount(mount.DevPath, mount.TargetPath, mount.FSType, flags, idmapType) if err != nil { return fmt.Errorf("Failed to add mount for device inside container: %s", err) } } else { relativeTargetPath := strings.TrimPrefix(mount.TargetPath, "/") // Connect to files API. files, err := d.FileSFTP() if err != nil { return err } defer func() { _ = files.Close() }() _, err = files.Lstat(relativeTargetPath) if err == nil { err := d.removeMount(mount.TargetPath) if err != nil { return fmt.Errorf("Error unmounting the device path inside container: %s", err) } // Only remove mountpoints created in /dev. if strings.HasPrefix(mount.TargetPath, "dev/") { err := files.Remove(relativeTargetPath) if err != nil { return err } } } } } return nil } // DeviceEventHandler actions the results of a RunConfig after an event has occurred on a device. func (d *lxc) DeviceEventHandler(runConf *deviceConfig.RunConfig) error { // Device events can only be processed when the container is running. // We use InitPID here rather than IsRunning because this task can be triggered during the // container startup process, which is during the time that the start lock is held, which causes // IsRunning to return false (because the container hasn't fully started yet). if d.InitPID() <= 0 { return nil } if runConf == nil { return nil } // Shift device file ownership if needed before mounting devices into container. if len(runConf.Mounts) > 0 { err := d.deviceStaticShiftMounts(runConf.Mounts) if err != nil { return err } err = d.deviceHandleMounts(runConf.Mounts) if err != nil { return err } } // Add cgroup rules if requested. if len(runConf.CGroups) > 0 { err := d.deviceAddCgroupRules(runConf.CGroups) if err != nil { return err } } // Handle NIC reconfiguration. var devName string var connected bool for _, dev := range runConf.NetworkInterface { switch dev.Key { case "devName": devName = dev.Value case "connected": connected = util.IsTrueOrEmpty(dev.Value) } } if devName != "" { err := d.setNICLink(devName, connected, false) if err != nil { return err } } // Run any post hooks requested. err := d.runHooks(runConf.PostHooks) if err != nil { return err } // Generate uevent inside container if requested. if len(runConf.Uevents) > 0 { pidFd, err := d.InitPidFd() if err != nil { return err } defer func() { _ = pidFd.Close() }() for _, eventParts := range runConf.Uevents { length := 0 for _, part := range eventParts { length = length + len(part) + 1 } args := []string{ "forkuevent", "inject", "--", fmt.Sprintf("%d", d.InitPID()), "3", fmt.Sprintf("%d", length), } args = append(args, eventParts...) _, _, err = subprocess.RunCommandSplit( context.TODO(), nil, []*os.File{pidFd}, d.state.OS.ExecPath, args...) if err != nil { return err } } } return nil } func (d *lxc) handleIdmappedStorage() (idmap.StorageType, *idmap.Set, error) { diskIdmap, err := d.DiskIdmap() if err != nil { return idmap.StorageTypeNone, nil, fmt.Errorf("Set last ID map: %w", err) } nextIdmap, err := d.NextIdmap() if err != nil { return idmap.StorageTypeNone, nil, fmt.Errorf("Set ID map: %w", err) } // Identical on-disk idmaps so no changes required. if nextIdmap.Equals(diskIdmap) { return idmap.StorageTypeNone, nextIdmap, nil } // There's no on-disk idmap applied and the container can use idmapped // storage. idmapType := d.IdmappedStorage(d.RootfsPath(), "none") if diskIdmap == nil && idmapType != idmap.StorageTypeNone { return idmapType, nextIdmap, nil } // We need to change the on-disk idmap but the container is protected // against idmap changes. if util.IsTrue(d.expandedConfig["security.protection.shift"]) { return idmap.StorageTypeNone, nil, errors.New("Container is protected against filesystem shifting") } d.logger.Debug("Container idmap changed, remapping") d.updateProgress("Remapping container filesystem") storageType, err := d.getStorageType() if err != nil { return idmap.StorageTypeNone, nil, fmt.Errorf("Storage type: %w", err) } // Revert the currently applied on-disk idmap. if diskIdmap != nil { if storageType == "zfs" { err = diskIdmap.UnshiftPath(d.RootfsPath(), storageDrivers.ShiftZFSSkipper) } else if storageType == "btrfs" { err = storageDrivers.UnshiftBtrfsRootfs(d.RootfsPath(), diskIdmap) } else { err = diskIdmap.UnshiftPath(d.RootfsPath(), nil) } if err != nil { return idmap.StorageTypeNone, nil, err } } jsonDiskIdmap := "[]" // If the container can't use idmapped storage apply the new on-disk // idmap of the container now. Otherwise we will later instruct LXC to // make use of idmapped storage. if nextIdmap != nil && idmapType == idmap.StorageTypeNone { if storageType == "zfs" { err = nextIdmap.ShiftPath(d.RootfsPath(), storageDrivers.ShiftZFSSkipper) } else if storageType == "btrfs" { err = storageDrivers.ShiftBtrfsRootfs(d.RootfsPath(), nextIdmap) } else { err = nextIdmap.ShiftPath(d.RootfsPath(), nil) } if err != nil { return idmap.StorageTypeNone, nil, err } idmapJSON, err := nextIdmap.ToJSON() if err != nil { return idmap.StorageTypeNone, nil, err } jsonDiskIdmap = idmapJSON } err = d.VolatileSet(map[string]string{"volatile.last_state.idmap": jsonDiskIdmap}) if err != nil { return idmap.StorageTypeNone, nextIdmap, fmt.Errorf("Set volatile.last_state.idmap config key on container %q (id %d): %w", d.name, d.id, err) } d.updateProgress("") return idmapType, nextIdmap, nil } // Start functions. func (d *lxc) startCommon() (string, []func() error, error) { postStartHooks := []func() error{} reverter := revert.New() defer reverter.Fail() // Assign NUMA node(s) if needed. if d.expandedConfig["limits.cpu.nodes"] == "balanced" { err := d.balanceNUMANodes() if err != nil { return "", nil, err } } // Check if idmap needs changing. if !d.IsPrivileged() { nextMap, err := d.NextIdmap() if err != nil { return "", nil, err } // Check if we need to change idmap. if nextMap != nil && d.state.OS.IdmapSet != nil && !d.state.OS.IdmapSet.Includes(nextMap) { // Update the idmap. idmapSet, base, err := d.findIdmap() if err != nil { return "", nil, fmt.Errorf("Failed to get ID map: %w", err) } idmapSetJSON, err := idmapSet.ToJSON() if err != nil { return "", nil, fmt.Errorf("Failed to encode ID map: %w", err) } err = d.VolatileSet(map[string]string{ "volatile.idmap.next": idmapSetJSON, "volatile.idmap.base": fmt.Sprintf("%v", base), }) if err != nil { return "", nil, fmt.Errorf("Failed to update volatile idmap: %w", err) } // Invalidate the idmap cache. d.idmapset = nil } } // Load the go-lxc struct cc, err := d.initLXC(true) if err != nil { return "", nil, fmt.Errorf("Load go-lxc struct: %w", err) } // gendoc:generate(entity=image, group=requirements, key=requirements.cgroup) // // --- // type: string // shortdesc: If set to `v1`, indicates that the image requires the host to run cgroup v1. // // Ensure cgroup v1 configuration is set appropriately with the image using systemd if d.localConfig["image.requirements.cgroup"] == "v1" && !util.PathExists("/sys/fs/cgroup/systemd") { return "", nil, errors.New("The image used by this instance requires a CGroupV1 host system") } // gendoc:generate(entity=image, group=requirements, key=requirements.privileged) // // --- // type: bool // shortdesc: If set to `false`, indicates that the image cannot work as a privileged container. // // Ensure privileged is turned off for images that cannot work privileged if util.IsFalse(d.localConfig["image.requirements.privileged"]) && util.IsTrue(d.expandedConfig["security.privileged"]) { return "", nil, errors.New("The image used by this instance is incompatible with privileged containers. Please unset security.privileged on the instance") } // Load any required kernel modules kernelModules := d.expandedConfig["linux.kernel_modules"] if kernelModules != "" { for _, module := range strings.Split(kernelModules, ",") { module = strings.TrimPrefix(module, " ") err := linux.LoadModule(module) if err != nil { return "", nil, fmt.Errorf("Failed to load kernel module '%s': %w", module, err) } } } // Rotate the log file. logfile := d.LogFilePath() if util.PathExists(logfile) { _ = os.Remove(logfile + ".old") err := os.Rename(logfile, logfile+".old") if err != nil && !errors.Is(err, fs.ErrNotExist) { return "", nil, err } } // Wait for any file operations to complete. // This is to avoid having an active mount by forkfile and so all file operations // from this point will use the container's namespace rather than a chroot. d.stopForkfile(false) // Mount instance root volume. mountInfo, err := d.mount() if err != nil { return "", nil, err } // Handle post hooks. postStartHooks = append(postStartHooks, func() error { for _, hook := range mountInfo.PostHooks { err := hook(d) if err != nil { return err } } return nil }) reverter.Add(func() { _ = d.unmount() }) idmapType, nextIdmap, err := d.handleIdmappedStorage() if err != nil { return "", nil, fmt.Errorf("Failed to handle idmapped storage: %w", err) } nextIdmapJSON, err := nextIdmap.ToJSON() if err != nil { return "", nil, fmt.Errorf("Failed to encode ID map: %w", err) } if d.localConfig["volatile.idmap.current"] != nextIdmapJSON { err = d.VolatileSet(map[string]string{"volatile.idmap.current": nextIdmapJSON}) if err != nil { return "", nil, fmt.Errorf("Set volatile.idmap.current config key on container %q (id %d): %w", d.name, d.id, err) } } // Generate the Seccomp profile err = seccomp.CreateProfile(d.state, d) if err != nil { return "", nil, err } // Cleanup any existing leftover devices _ = d.removeUnixDevices() _ = d.removeDiskDevices() // Create any missing directories. err = os.MkdirAll(d.LogPath(), 0o700) if err != nil { return "", nil, err } err = os.MkdirAll(d.RunPath(), 0o700) if err != nil { return "", nil, err } err = os.MkdirAll(d.DevicesPath(), 0o711) if err != nil { return "", nil, err } err = os.MkdirAll(d.ShmountsPath(), 0o711) if err != nil { return "", nil, err } volatileSet := make(map[string]string) // Generate UUID if not present (do this before UpdateBackupFile() call). instUUID := d.localConfig["volatile.uuid"] if instUUID == "" { instUUID = uuid.New().String() volatileSet["volatile.uuid"] = instUUID } // For a container instance, we must also set the generation UUID. genUUID := d.localConfig["volatile.uuid.generation"] if genUUID == "" { genUUID = instUUID volatileSet["volatile.uuid.generation"] = genUUID } // Create the devices nicID := -1 nvidiaDevices := []string{} sortedDevices := d.expandedDevices.Sorted() startDevices := make([]device.Device, 0, len(sortedDevices)) // Load devices in sorted order, this ensures that device mounts are added in path order. // Loading all devices first means that validation of all devices occurs before starting any of them. for _, entry := range sortedDevices { dev, err := d.deviceLoad(d, entry.Name, entry.Config, false) if err != nil { if errors.Is(err, device.ErrUnsupportedDevType) { continue // Skip unsupported device (allows for mixed instance type profiles). } return "", nil, fmt.Errorf("Failed start validation for device %q: %w", entry.Name, err) } // Run pre-start of check all devices before starting any device to avoid expensive revert. err = dev.PreStartCheck() if err != nil { return "", nil, fmt.Errorf("Failed pre-start check for device %q: %w", dev.Name(), err) } startDevices = append(startDevices, dev) } // Start devices in order. for i := range startDevices { dev := startDevices[i] // Local var for revert. // Start the device. runConf, err := d.deviceStart(dev, false) if err != nil { return "", nil, fmt.Errorf("Failed to start device %q: %w", dev.Name(), err) } // Stop device on failure to setup container. reverter.Add(func() { err := d.deviceStop(dev, false, "") if err != nil { d.logger.Error("Failed to cleanup device", logger.Ctx{"device": dev.Name(), "err": err}) } }) if runConf == nil { continue } if runConf.Revert != nil { reverter.Add(runConf.Revert) } // Process rootfs setup. if runConf.RootFS.Path != "" { if !liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 2, 1, 0) { // Set the rootfs backend type if supported (must happen before any other lxc.rootfs) err := lxcSetConfigItem(cc, "lxc.rootfs.backend", "dir") if err == nil { value := cc.ConfigItem("lxc.rootfs.backend") if len(value) == 0 || value[0] != "dir" { _ = lxcSetConfigItem(cc, "lxc.rootfs.backend", "") } } } // Get an absolute path for the rootfs (avoid constantly traversing the symlink). absoluteRootfs, err := filepath.EvalSymlinks(runConf.RootFS.Path) if err != nil { return "", nil, fmt.Errorf("Unable to resolve container rootfs: %w", err) } if liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 2, 1, 0) { rootfsPath := fmt.Sprintf("dir:%s", absoluteRootfs) err = lxcSetConfigItem(cc, "lxc.rootfs.path", rootfsPath) } else { err = lxcSetConfigItem(cc, "lxc.rootfs", absoluteRootfs) } if err != nil { return "", nil, fmt.Errorf("Failed to setup device rootfs %q: %w", dev.Name(), err) } if len(runConf.RootFS.Opts) > 0 { err = lxcSetConfigItem(cc, "lxc.rootfs.options", strings.Join(runConf.RootFS.Opts, ",")) if err != nil { return "", nil, fmt.Errorf("Failed to setup device rootfs %q: %w", dev.Name(), err) } } if !d.IsPrivileged() && idmapType == idmap.StorageTypeIdmapped { err = lxcSetConfigItem(cc, "lxc.rootfs.options", "idmap=container") if err != nil { return "", nil, fmt.Errorf("Failed to set \"idmap=container\" rootfs option: %w", err) } } } // Pass any cgroups rules into LXC. if len(runConf.CGroups) > 0 { for _, rule := range runConf.CGroups { if strings.HasPrefix(rule.Key, "devices.") && (!d.isCurrentlyPrivileged() || d.state.OS.RunningInUserNS) { continue } err = lxcSetConfigItem(cc, fmt.Sprintf("lxc.cgroup2.%s", rule.Key), rule.Value) if err != nil { return "", nil, fmt.Errorf("Failed to setup device cgroup %q: %w", dev.Name(), err) } } } // Pass any mounts into LXC. if len(runConf.Mounts) > 0 { escapePathFstab := func(path string) string { r := strings.NewReplacer( " ", "\\040", "\t", "\\011", "\n", "\\012", "\\", "\\\\") return r.Replace(path) } for _, mount := range runConf.Mounts { if slices.Contains(mount.Opts, "propagation") && !liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 3, 0, 0) { return "", nil, fmt.Errorf("Failed to setup device mount %q: %w", dev.Name(), errors.New("liblxc 3.0 is required for mount propagation configuration")) } mntOptions := strings.Join(mount.Opts, ",") if !d.IsPrivileged() && mount.OwnerShift == deviceConfig.MountOwnerShiftDynamic { switch d.IdmappedStorage(mount.DevPath, mount.FSType) { case idmap.StorageTypeIdmapped: mntOptions = strings.Join([]string{mntOptions, "idmap=container"}, ",") case idmap.StorageTypeNone: return "", nil, fmt.Errorf("Failed to setup device mount %q: %w", dev.Name(), errors.New("idmapping abilities are required but aren't supported on system")) } } mntVal := fmt.Sprintf("%s %s %s %s %d %d", escapePathFstab(mount.DevPath), escapePathFstab(mount.TargetPath), mount.FSType, mntOptions, mount.Freq, mount.PassNo) err = lxcSetConfigItem(cc, "lxc.mount.entry", mntVal) if err != nil { return "", nil, fmt.Errorf("Failed to setup device mount %q: %w", dev.Name(), err) } } } // Pass any network setup config into LXC. if len(runConf.NetworkInterface) > 0 { // Increment nicID so that LXC network index is unique per device. nicID++ networkKeyPrefix := "lxc.net" if !liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 2, 1, 0) { networkKeyPrefix = "lxc.network" } for _, nicItem := range runConf.NetworkInterface { // The connected state is not a LXC configuration key; we defer its handling to a post hook. if nicItem.Key == "connected" { runConf.PostHooks = append(runConf.PostHooks, func() error { return d.setNICLink(dev.Name(), util.IsTrueOrEmpty(nicItem.Value), true) }) continue } err = lxcSetConfigItem(cc, fmt.Sprintf("%s.%d.%s", networkKeyPrefix, nicID, nicItem.Key), nicItem.Value) if err != nil { return "", nil, fmt.Errorf("Failed to setup device network interface %q: %w", dev.Name(), err) } } } // Add any post start hooks. if len(runConf.PostHooks) > 0 { postStartHooks = append(postStartHooks, runConf.PostHooks...) } // Build list of NVIDIA GPUs (used for MIG). if len(runConf.GPUDevice) > 0 { for _, entry := range runConf.GPUDevice { if entry.Key == device.GPUNvidiaDeviceKey { nvidiaDevices = append(nvidiaDevices, entry.Value) } } } } // Initialize the credentials directory. err = d.setupCredentials(false) if err != nil { return "", nil, err } // Override NVIDIA_VISIBLE_DEVICES if we have devices that need it. if len(nvidiaDevices) > 0 { err = lxcSetConfigItem(cc, "lxc.environment", fmt.Sprintf("NVIDIA_VISIBLE_DEVICES=%s", strings.Join(nvidiaDevices, ","))) if err != nil { return "", nil, fmt.Errorf("Unable to set NVIDIA_VISIBLE_DEVICES in LXC environment: %w", err) } } // Handle application containers. if util.PathExists(filepath.Join(d.Path(), "config.json")) { // Parse the OCI config. data, err := os.ReadFile(filepath.Join(d.Path(), "config.json")) if err != nil { return "", nil, err } var config ociSpecs.Spec err = json.Unmarshal([]byte(data), &config) if err != nil { return "", nil, err } // Mark the container as an OCI container if not already set. if !util.IsTrue(d.expandedConfig["volatile.container.oci"]) { volatileSet["volatile.container.oci"] = "true" } // Allow unprivileged users to use ping. if !d.state.OS.RunningInUserNS { maxGid := int64(4294967294) if !d.IsPrivileged() { maxGid = 0 idMap, err := d.CurrentIdmap() if err != nil { return "", nil, err } for _, entry := range idMap.Entries { if entry.NSID+entry.MapRange-1 > maxGid { maxGid = entry.NSID + entry.MapRange - 1 } } } err = lxcSetConfigItem(cc, "lxc.sysctl.net.ipv4.ping_group_range", fmt.Sprintf("0 %d", maxGid)) if err != nil { return "", nil, err } } // Allow unprivileged users to use low ports. err = lxcSetConfigItem(cc, "lxc.sysctl.net.ipv4.ip_unprivileged_port_start", "0") if err != nil { return "", nil, err } // Configure the entry point. entrypoint := config.Process.Args if d.expandedConfig["oci.entrypoint"] != "" { entrypoint, err = shellquote.Split(d.expandedConfig["oci.entrypoint"]) if err != nil { return "", nil, err } } // Compute the entrypoint string. initCmd := shellquote.Join(entrypoint...) // As we feed this to execve and not to a real shell, un-escape some sequences. initCmd = strings.ReplaceAll(initCmd, "\\(", "(") initCmd = strings.ReplaceAll(initCmd, "\\)", ")") if len(entrypoint) > 0 && slices.Contains([]string{"/init", "/sbin/init", "/s6-init", "/usr/bin/init"}, entrypoint[0]) { // For regular init systems, call them directly as PID1. err = lxcSetConfigItem(cc, "lxc.init.cmd", initCmd) if err != nil { return "", nil, err } } else { // For anything else, run them under our own PID1. err = lxcSetConfigItem(cc, "lxc.execute.cmd", initCmd) if err != nil { return "", nil, err } } // Configure the cwd. if d.expandedConfig["oci.cwd"] != "" { err = lxcSetConfigItem(cc, "lxc.init.cwd", d.expandedConfig["oci.cwd"]) if err != nil { return "", nil, err } } else { err = lxcSetConfigItem(cc, "lxc.init.cwd", config.Process.Cwd) if err != nil { return "", nil, err } } // Configure the UID if d.expandedConfig["oci.uid"] != "" { err = lxcSetConfigItem(cc, "lxc.init.uid", d.expandedConfig["oci.uid"]) if err != nil { return "", nil, err } } else { err = lxcSetConfigItem(cc, "lxc.init.uid", fmt.Sprintf("%d", config.Process.User.UID)) if err != nil { return "", nil, err } } // Configure the GID if d.expandedConfig["oci.gid"] != "" { err = lxcSetConfigItem(cc, "lxc.init.gid", d.expandedConfig["oci.gid"]) if err != nil { return "", nil, err } } else { err = lxcSetConfigItem(cc, "lxc.init.gid", fmt.Sprintf("%d", config.Process.User.GID)) if err != nil { return "", nil, err } } // Get all mounts so far. lxcMounts := []string{"/dev", "/proc", "/sys", "/sys/fs/cgroup"} for _, mount := range cc.ConfigItem("lxc.mount.entry") { fields := strings.Split(mount, " ") if len(fields) < 2 || fields[1][0] == '/' { continue } lxcMounts = append(lxcMounts, filepath.Clean(fmt.Sprintf("/%s", fields[1]))) } // Configure mounts. for _, mount := range config.Mounts { // We only support simple tmpfs at this stage. if len(mount.UIDMappings) > 0 || len(mount.GIDMappings) > 0 || mount.Type != "tmpfs" { continue } // Skip all our own mounts. if slices.Contains(lxcMounts, filepath.Clean(mount.Destination)) { continue } err := lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s %s %s %s 0 0", mount.Source, strings.TrimLeft(mount.Destination, "/"), mount.Type, strings.Join(append(mount.Options, "create=dir"), ","))) if err != nil { return "", nil, err } lxcMounts = append(lxcMounts, mount.Destination) } // Mount /run as a tmpfs if it exists and isn't already mounted. if !slices.Contains(lxcMounts, "/run") { err := lxcSetConfigItem(cc, "lxc.mount.entry", "none run tmpfs none,mode=755,optional") if err != nil { return "", nil, err } } // Configure network handling. err = os.MkdirAll(filepath.Join(d.Path(), "network"), 0o711) if err != nil { return "", nil, err } err = os.MkdirAll(filepath.Join(d.RootfsPath(), "etc"), 0o755) if err != nil && !os.IsExist(err) { return "", nil, err } err = os.WriteFile(filepath.Join(d.Path(), "network", "hosts"), fmt.Appendf(nil, `127.0.0.1 localhost 127.0.1.1 %s ::1 localhost ip6-localhost ip6-loopback fe00::0 ip6-localnet ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters `, d.name), 0o644) if err != nil { return "", nil, err } err = lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s etc/hosts none bind,create=file", filepath.Join(d.Path(), "network", "hosts"))) if err != nil { return "", nil, err } err = os.WriteFile(filepath.Join(d.Path(), "network", "hostname"), fmt.Appendf(nil, "%s\n", d.name), 0o644) if err != nil { return "", nil, err } err = lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s etc/hostname none bind,create=file", filepath.Join(d.Path(), "network", "hostname"))) if err != nil { return "", nil, err } f, err := os.OpenFile(filepath.Join(d.Path(), "network", "resolv.conf"), os.O_RDWR|os.O_CREATE, 0o644) if err != nil { return "", nil, err } f.Close() err = lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s etc/resolv.conf none bind,create=file", filepath.Join(d.Path(), "network", "resolv.conf"))) if err != nil { return "", nil, err } forknetDhcpLogfilePath := filepath.Join(d.LogPath(), "forknet-dhcp.log") forknetDhcpLogfile, err := os.Create(forknetDhcpLogfilePath) if err != nil { return "", nil, err } err = forknetDhcpLogfile.Close() if err != nil { return "", nil, err } err = lxcSetConfigItem(cc, "lxc.hook.start-host", fmt.Sprintf( "/proc/%d/exe forknet dhcp %s %s", os.Getpid(), filepath.Join(d.Path(), "network"), forknetDhcpLogfilePath, )) if err != nil { return "", nil, err } } else { // Clear OCI config key if present. if d.expandedConfig["volatile.container.oci"] != "" { volatileSet["volatile.container.oci"] = "" } } // Check if we should start a dedicated LXCFS. if d.state.GlobalConfig.InstancesLXCFSPerInstance() { if !util.PathExists(filepath.Join(d.RunPath(), "lxcfs", "proc")) { // Make sure all the paths exist. err := os.Mkdir(filepath.Join(d.DevicesPath(), "lxcfs"), 0o711) if err != nil && !os.IsExist(err) { return "", nil, err } err = os.Mkdir(filepath.Join(d.RunPath(), "lxcfs"), 0o700) if err != nil && !os.IsExist(err) { return "", nil, err } // Prepare a new LXCFS instance. args := []string{ "-f", "-p", filepath.Join(d.RunPath(), "lxcfs.pid"), "--runtime-dir", filepath.Join(d.RunPath(), "lxcfs"), } if os.Getenv("LXCFS_OPTS") != "" { userArgs, err := shellquote.Split(os.Getenv("LXCFS_OPTS")) if err != nil { return "", nil, err } args = append(args, userArgs...) } args = append(args, filepath.Join(d.DevicesPath(), "lxcfs")) lxcfs, err := subprocess.NewProcess("lxcfs", args, "", "") if err != nil { return "", nil, err } // Start LXCFS. err = lxcfs.Start(context.TODO()) if err != nil { return "", nil, err } // Write down our process tracking. err = lxcfs.Save(filepath.Join(d.RunPath(), "lxcfs.yaml")) if err != nil { return "", nil, err } } // Over-mount the system LXCFS (if found). for _, entry := range []string{"/var/lib/lxcfs", "/var/lib/incus-lxcfs"} { if !util.PathExists(entry) { continue } err = lxcSetConfigItem(cc, "lxc.hook.pre-mount", fmt.Sprintf("mount -o bind %s %s/", filepath.Join(d.DevicesPath(), "lxcfs"), entry)) if err != nil { return "", nil, err } } } // Setup BPF token delegation if enabled bpfConfig := d.bpfTokenConfig() if bpfConfig.enable { err = lxcSetConfigItem(cc, "lxc.hook.start-host", shellquote.Join( fmt.Sprintf("/proc/%d/exe", os.Getpid()), "forkbpf", bpfConfig.mountPath, bpfConfig.cmdTypes, bpfConfig.mapTypes, bpfConfig.progTypes, bpfConfig.attachTypes, )) if err != nil { return "", nil, err } } // Load the LXC raw config. err = d.loadRawLXCConfig(cc) if err != nil { return "", nil, err } // Generate the LXC config configPath := filepath.Join(d.RunPath(), "lxc.conf") err = cc.SaveConfigFile(configPath) if err != nil { _ = os.Remove(configPath) return "", nil, err } // Set ownership to match container root currentIdmapset, err := d.CurrentIdmap() if err != nil { return "", nil, err } uid := int64(0) if currentIdmapset != nil { uid, _ = currentIdmapset.ShiftFromNS(0, 0) } err = os.Chown(d.Path(), int(uid), 0) if err != nil { return "", nil, err } // We only need traversal by root in the container err = os.Chmod(d.Path(), 0o100) if err != nil { return "", nil, err } // If starting stateless, wipe state if !d.IsStateful() && util.PathExists(d.StatePath()) { _ = os.RemoveAll(d.StatePath()) } // Snapshot if needed. snapName, expiry, err := d.getStartupSnapNameAndExpiry(d) if err != nil { return "", nil, fmt.Errorf("Failed getting startup snapshot info: %w", err) } if snapName != "" && expiry != nil { err := d.snapshot(snapName, *expiry, false) if err != nil { return "", nil, fmt.Errorf("Failed taking startup snapshot: %w", err) } } // Apply any volatile changes that need to be made. err = d.VolatileSet(volatileSet) if err != nil { return "", nil, fmt.Errorf("Failed setting volatile keys: %w", err) } // Update the backup.yaml file just before starting the instance process, but after all devices have been // setup, so that the backup file contains the volatile keys used for this instance start, so that they // can be used for instance cleanup. err = d.UpdateBackupFile() if err != nil { return "", nil, err } reverter.Success() return configPath, postStartHooks, nil } type bpfTokenConfig struct { enable bool mountPath string cmdTypes string mapTypes string progTypes string attachTypes string } func (d *lxc) bpfTokenConfig() bpfTokenConfig { if util.IsTrue(d.expandedConfig["security.privileged"]) { return bpfTokenConfig{} } cfg := bpfTokenConfig{ mountPath: d.expandedConfig["security.bpffs.path"], cmdTypes: d.expandedConfig["security.bpffs.delegate_cmds"], mapTypes: d.expandedConfig["security.bpffs.delegate_maps"], progTypes: d.expandedConfig["security.bpffs.delegate_progs"], attachTypes: d.expandedConfig["security.bpffs.delegate_attachs"], } if cfg.cmdTypes != "" || cfg.mapTypes != "" || cfg.progTypes != "" || cfg.attachTypes != "" { cfg.enable = true } if cfg.mountPath == "" { cfg.mountPath = "/sys/fs/bpf" } return cfg } // detachInterfaceRename enters the container's network namespace and moves the named interface // in ifName back to the network namespace of the running process as the name specified in hostName. func (d *lxc) detachInterfaceRename(netns string, ifName string, hostName string) error { daemonPID := os.Getpid() // Run forknet detach _, err := subprocess.RunCommand( d.state.OS.ExecPath, "forknet", "detach", "--", netns, fmt.Sprintf("%d", daemonPID), ifName, hostName, ) // Process forknet detach response if err != nil { return err } return nil } // Start starts the instance. func (d *lxc) Start(stateful bool) error { // Check that migration.stateful is set for stateful actions. if stateful && !d.CanLiveMigrate() { return errors.New("Stateful start requires that the instance migration.stateful be set to true") } d.logger.Debug("Start started", logger.Ctx{"stateful": stateful}) defer d.logger.Debug("Start finished", logger.Ctx{"stateful": stateful}) // Check that we are startable before creating an operation lock. // Must happen before creating operation Start lock to avoid the status check returning Stopped due to the // existence of a Start operation lock. err := d.validateStartup(stateful, d.statusCode()) if err != nil { return err } // Setup a new operation. op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStart, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore}, false, false) if err != nil { if errors.Is(err, operationlock.ErrNonReusuableSucceeded) { // An existing matching operation has now succeeded, return. return nil } return fmt.Errorf("Failed to create instance start operation: %w", err) } defer op.Done(nil) if !daemon.SharedMountsSetup { err = errors.New("Daemon failed to setup shared mounts base. Does security.nesting need to be turned on?") op.Done(err) return err } ctxMap := logger.Ctx{ "action": op.Action(), "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, "stateful": stateful, } if op.Action() == "start" { d.logger.Info("Starting instance", ctxMap) } // If stateful, restore now. if stateful && d.stateful { d.logger.Info("Restoring stateful checkpoint") criuMigrationArgs := instance.CriuMigrationArgs{ Cmd: liblxc.MIGRATE_RESTORE, StateDir: d.StatePath(), Function: "snapshot", Stop: false, ActionScript: false, DumpDir: "", PreDumpDir: "", } err = d.migrate(&criuMigrationArgs) if err != nil && !d.IsRunning() { op.Done(err) return fmt.Errorf("Failed restoring stateful checkpoint: %w", err) } _ = os.RemoveAll(d.StatePath()) d.stateful = false err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateInstanceStatefulFlag(ctx, d.id, false) }) if err != nil { op.Done(err) return fmt.Errorf("Failed clearing instance stateful flag: %w", err) } if op.Action() == "start" { d.logger.Info("Started instance", ctxMap) d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceStarted.Event(d, nil)) } return nil } else if d.stateful { /* stateless start required when we have state, let's delete it */ err := os.RemoveAll(d.StatePath()) if err != nil { op.Done(err) return err } d.stateful = false err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateInstanceStatefulFlag(ctx, d.id, false) }) if err != nil { op.Done(err) return fmt.Errorf("Failed clearing instance stateful flag: %w", err) } } // Run the shared start code. configPath, postStartHooks, err := d.startCommon() if err != nil { op.Done(err) return err } name := project.Instance(d.Project().Name, d.name) // Setup minimal environment for forkstart. envDict := map[string]string{ "container": "lxc", } for k, v := range d.expandedConfig { after, ok := strings.CutPrefix(k, "environment.") if ok { envDict[after] = v } } for _, keepEnv := range []string{"LD_LIBRARY_PATH", "INCUS_DIR", "INCUS_SOCKET"} { if os.Getenv(keepEnv) != "" { envDict[keepEnv] = os.Getenv(keepEnv) } } _, ok := envDict["PATH"] if !ok { envDict["PATH"] = os.Getenv("PATH") } env := make([]string, 0, len(envDict)) for k, v := range envDict { env = append(env, fmt.Sprintf("%s=%s", k, v)) } // Start the LXC container. _, _, err = subprocess.RunCommandSplit( context.TODO(), env, nil, d.state.OS.ExecPath, "forkstart", name, d.state.OS.LxcPath, configPath, d.LogPath()) if err != nil && !d.IsRunning() { // Attempt to extract the LXC errors lxcLog := "" logPath := filepath.Join(d.LogPath(), "lxc.log") if util.PathExists(logPath) { logContent, err := os.ReadFile(logPath) if err == nil { for _, line := range strings.Split(string(logContent), "\n") { fields := strings.Fields(line) if len(fields) < 4 { continue } // We only care about errors if fields[2] != "ERROR" { continue } // Prepend the line break if len(lxcLog) == 0 { lxcLog += "\n" } lxcLog += fmt.Sprintf(" %s\n", strings.Join(fields[0:], " ")) } } } d.logger.Error("Failed starting instance", ctxMap) // Return the actual error op.Done(err) return err } // Run any post start hooks. err = d.runHooks(postStartHooks) if err != nil { op.Done(err) // Must come before Stop() otherwise stop will not proceed. // Attempt to stop container. _ = d.Stop(false) return err } // Apply OOM priority after container is started and hooks completed. err = d.setOOMPriority(d.InitPID()) if err != nil { d.logger.Warn("Failed to set OOM priority", logger.Ctx{ "err": err, "instance": d.Name(), "project": d.Project().Name, }) } if op.Action() == "start" { d.logger.Info("Started instance", ctxMap) d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceStarted.Event(d, nil)) } return nil } // OnHook is the top-level hook handler. func (d *lxc) OnHook(hookName string, args map[string]string) error { switch hookName { case instance.HookStart: return d.onStart(args) case instance.HookStopNS: return d.onStopNS(args) case instance.HookStop: return d.onStop(args) default: return instance.ErrNotImplemented } } // onStart implements the start hook. func (d *lxc) onStart(_ map[string]string) error { // Make sure we can't call go-lxc functions by mistake d.fromHook = true // Load the container AppArmor profile err := apparmor.InstanceLoad(d.state.OS, d, nil) if err != nil { return err } // Template anything that needs templating key := "volatile.apply_template" if d.localConfig[key] != "" { // Run any template that needs running err = d.templateApplyNow(instance.TemplateTrigger(d.localConfig[key])) if err != nil { _ = apparmor.InstanceUnload(d.state.OS, d) return err } err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Remove the volatile key from the DB return tx.DeleteInstanceConfigKey(ctx, int64(d.id), key) }) if err != nil { _ = apparmor.InstanceUnload(d.state.OS, d) return err } } err = d.templateApplyNow("start") if err != nil { _ = apparmor.InstanceUnload(d.state.OS, d) return err } // Trigger a rebalance defer cgroup.TaskSchedulerTrigger("container", d.name, "started") // Record last start state. err = d.recordLastState() if err != nil { return err } return nil } // validateStartup checks any constraints that would prevent start up from succeeding under normal circumstances. func (d *lxc) validateStartup(stateful bool, statusCode api.StatusCode) error { err := d.common.validateStartup(stateful, statusCode) if err != nil { return err } // gendoc:generate(entity=image, group=requirements, key=requirements.nesting) // // --- // type: bool // shortdesc: If set to `true`, indicates that the image cannot work without nesting enabled. // // Ensure nesting is turned on for images that require nesting. if util.IsTrue(d.localConfig["image.requirements.nesting"]) && util.IsFalseOrEmpty(d.expandedConfig["security.nesting"]) { return errors.New("The image used by this instance requires nesting. Please set security.nesting=true on the instance") } return nil } // Stop functions. func (d *lxc) Stop(stateful bool) error { d.logger.Debug("Stop started", logger.Ctx{"stateful": stateful}) defer d.logger.Debug("Stop finished", logger.Ctx{"stateful": stateful}) // Check that migration.stateful is set for stateful actions. if stateful && !d.CanLiveMigrate() { return errors.New("Stateful stop requires the instance to have migration.stateful be set to true") } // Must be run prior to creating the operation lock. if !d.IsRunning() { return ErrInstanceIsStopped } // Setup a new operation op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStop, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore, operationlock.ActionMigrate}, false, true) if err != nil { if errors.Is(err, operationlock.ErrNonReusuableSucceeded) { // An existing matching operation has now succeeded, return. return nil } return err } ctxMap := logger.Ctx{ "action": op.Action(), "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, "stateful": stateful, } if op.Action() == "stop" { d.logger.Info("Stopping instance", ctxMap) } // Forcefully stop any forkfile process if running. d.stopForkfile(true) // Release liblxc container once done. defer func() { d.release() }() // Load the go-lxc struct var cc *liblxc.Container if d.expandedConfig["raw.lxc"] != "" { cc, err = d.initLXC(true) if err != nil { op.Done(err) return err } err = d.loadRawLXCConfig(cc) if err != nil { op.Done(err) return err } } else { cc, err = d.initLXC(false) if err != nil { op.Done(err) return err } } // Handle stateful stop if stateful { // Cleanup any existing state stateDir := d.StatePath() _ = os.RemoveAll(stateDir) err := os.MkdirAll(stateDir, 0o700) if err != nil { op.Done(err) return err } criuMigrationArgs := instance.CriuMigrationArgs{ Cmd: liblxc.MIGRATE_DUMP, StateDir: stateDir, Function: "snapshot", Stop: true, ActionScript: false, DumpDir: "", PreDumpDir: "", } // Checkpoint err = d.migrate(&criuMigrationArgs) if err != nil { op.Done(err) return err } err = op.Wait(context.Background()) if err != nil && d.IsRunning() { return err } d.stateful = true err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateInstanceStatefulFlag(ctx, d.id, true) }) if err != nil { return fmt.Errorf("Failed updating instance stateful flag: %w", err) } d.logger.Info("Stopped instance", ctxMap) d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceStopped.Event(d, nil)) return nil } else if util.PathExists(d.StatePath()) { _ = os.RemoveAll(d.StatePath()) } // Load cgroup abstraction cg, err := d.cgroup(cc, true) if err != nil { op.Done(err) return err } // Fork-bomb mitigation, prevent forking from this point on if cgroup.Supports(cgroup.Pids) { // Attempt to disable forking new processes _ = cg.SetMaxProcesses(0) } else { // Attempt to freeze the container freezer := make(chan bool, 1) go func() { _ = d.Freeze() freezer <- true }() select { case <-freezer: case <-time.After(time.Second * 5): _ = d.Unfreeze() } } err = cc.Stop() if err != nil { op.Done(err) return err } // Wait for operation lock to be Done. This is normally completed by onStop which picks up the same // operation lock and then marks it as Done after the instance stops and the devices have been cleaned up. // However if the operation has failed for another reason we will collect the error here. err = op.Wait(context.Background()) status := d.statusCode() if status != api.Stopped { errPrefix := fmt.Errorf("Failed stopping instance, status is %q", status) if err != nil { return fmt.Errorf("%s: %w", errPrefix.Error(), err) } return errPrefix } else if op.Action() == "stop" { // If instance stopped, send lifecycle event (even if there has been an error cleaning up). d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceStopped.Event(d, nil)) } // Now handle errors from stop sequence and return to caller if wasn't completed cleanly. if err != nil { return err } return nil } // Shutdown stops the instance. func (d *lxc) Shutdown(timeout time.Duration) error { d.logger.Debug("Shutdown started", logger.Ctx{"timeout": timeout}) defer d.logger.Debug("Shutdown finished", logger.Ctx{"timeout": timeout}) // Must be run prior to creating the operation lock. statusCode := d.statusCode() if !d.isRunningStatusCode(statusCode) { if statusCode == api.Error { return fmt.Errorf("The instance cannot be cleanly shutdown as in %s status", statusCode) } return ErrInstanceIsStopped } // Setup a new operation op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStop, []operationlock.Action{operationlock.ActionRestart}, true, true) if err != nil { if errors.Is(err, operationlock.ErrNonReusuableSucceeded) { // An existing matching operation has now succeeded, return. return nil } return err } // If frozen, resume so the signal can be handled. if d.IsFrozen() { err := d.Unfreeze() if err != nil { return err } // Wait 3s for init to be running enough to get the next signal handle. time.Sleep(3 * time.Second) } ctxMap := logger.Ctx{ "action": "shutdown", "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, "timeout": timeout, } if op.Action() == "stop" { d.logger.Info("Shutting down instance", ctxMap) } // Release liblxc container once done. defer func() { d.release() }() // Load the go-lxc struct var cc *liblxc.Container if d.expandedConfig["raw.lxc"] != "" { cc, err = d.initLXC(true) if err != nil { op.Done(err) return err } err = d.loadRawLXCConfig(cc) if err != nil { op.Done(err) return err } } else { cc, err = d.initLXC(false) if err != nil { op.Done(err) return err } } // Request shutdown, but don't wait for container to stop. If call fails then cancel operation with error, // otherwise expect the onStop() hook to cancel operation when done (when the container has stopped). err = cc.Shutdown(0) if err != nil { op.Done(err) } d.logger.Debug("Shutdown request sent to instance") ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() // Wait for operation lock to be Done or context to timeout. The operation lock is normally completed by // onStop which picks up the same lock and then marks it as Done after the instance stops and the devices // have been cleaned up. However if the operation has failed for another reason we collect the error here. err = op.Wait(ctx) status := d.statusCode() if status != api.Stopped { errPrefix := fmt.Errorf("Failed shutting down instance, status is %q", status) if err != nil { return fmt.Errorf("%s: %w", errPrefix.Error(), err) } return errPrefix } else if op.Action() == "stop" { // If instance stopped, send lifecycle event (even if there has been an error cleaning up). d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceShutdown.Event(d, nil)) } // Now handle errors from shutdown sequence and return to caller if wasn't completed cleanly. if err != nil { return err } return nil } // Restart restart the instance. func (d *lxc) Restart(timeout time.Duration) error { return d.restartCommon(d, timeout) } // Rebuild rebuilds the instance using the supplied image fingerprint as source. func (d *lxc) Rebuild(img *api.Image, op *operations.Operation) error { return d.rebuildCommon(d, img, op) } // onStopNS is triggered by LXC's stop hook once a container is shutdown but before the container's // namespaces have been closed. The netns path of the stopped container is provided. func (d *lxc) onStopNS(args map[string]string) error { target := args["target"] netns := args["netns"] // Validate target. if !slices.Contains([]string{"stop", "reboot"}, target) { d.logger.Error("Container sent invalid target to OnStopNS", logger.Ctx{"target": target}) return fmt.Errorf("Invalid stop target %q", target) } // Create/pick up operation, but don't complete it as we leave operation running for the onStop hook below. _, err := d.onStopOperationSetup(target) if err != nil { return err } // Clean up devices. d.cleanupDevices(false, netns) return nil } // onStop is triggered by LXC's post-stop hook once a container is shutdown and after the // container's namespaces have been closed. func (d *lxc) onStop(args map[string]string) error { target := args["target"] // Validate target if !slices.Contains([]string{"stop", "reboot"}, target) { d.logger.Error("Container sent invalid target to OnStop", logger.Ctx{"target": target}) return fmt.Errorf("Invalid stop target: %s", target) } // Create/pick up operation. op, err := d.onStopOperationSetup(target) if err != nil { return err } // Make sure we can't call go-lxc functions by mistake d.fromHook = true // Record power state. err = d.VolatileSet(map[string]string{ "volatile.last_state.power": instance.PowerStateStopped, "volatile.last_state.ready": "false", }) if err != nil { // Don't return an error here as we still want to cleanup the instance even if DB not available. d.logger.Error("Failed recording last power state", logger.Ctx{"err": err}) } go func(d *lxc, target string, op *operationlock.InstanceOperation) { d.fromHook = false err = nil // Set operation if missing. if d.op == nil { d.op = op.GetOperation() } // Unlock on return defer op.Done(nil) d.logger.Debug("Instance stopped, cleaning up") // Wait for any file operations to complete. // This is to required so we can actually unmount the container. d.stopForkfile(false) // Clean up devices. d.cleanupDevices(false, "") // Stop DHCP client if any. if util.PathExists(filepath.Join(d.Path(), "network", "dhcp.pid")) { dhcpPIDStr, err := os.ReadFile(filepath.Join(d.Path(), "network", "dhcp.pid")) if err == nil { dhcpPID, err := strconv.Atoi(strings.TrimSpace(string(dhcpPIDStr))) if err == nil { _ = unix.Kill(dhcpPID, unix.SIGTERM) } } } // Remove directory ownership (to avoid issue if uidmap is reused) err := os.Chown(d.Path(), 0, 0) if err != nil { op.Done(fmt.Errorf("Failed clearing ownership: %w", err)) return } err = os.Chmod(d.Path(), 0o100) if err != nil { op.Done(fmt.Errorf("Failed clearing permissions: %w", err)) return } // Stop the storage for this container err = d.unmount() if err != nil && !errors.Is(err, storageDrivers.ErrInUse) { err = fmt.Errorf("Failed unmounting instance: %w", err) op.Done(err) return } // Unload the apparmor profile err = apparmor.InstanceUnload(d.state.OS, d) if err != nil { op.Done(fmt.Errorf("Failed to destroy apparmor namespace: %w", err)) return } // Clean all the unix devices err = d.removeUnixDevices() if err != nil { op.Done(fmt.Errorf("Failed to remove unix devices: %w", err)) return } // Clean all the disk devices err = d.removeDiskDevices() if err != nil { op.Done(fmt.Errorf("Failed to remove disk devices: %w", err)) return } // Stop dedicated LXCFS. if util.PathExists(filepath.Join(d.DevicesPath(), "lxcfs", "proc")) && util.PathExists(filepath.Join(d.RunPath(), "lxcfs.yaml")) { // Import the running LXCFS. lxcfs, err := subprocess.ImportProcess(filepath.Join(d.RunPath(), "lxcfs.yaml")) if err != nil && !os.IsExist(err) { op.Done(fmt.Errorf("Failed to stop LXCFS: %w", err)) return } // Stop LXCFS. err = lxcfs.Stop() if err != nil && !errors.Is(err, subprocess.ErrNotRunning) { op.Done(fmt.Errorf("Failed to stop LXCFS: %w", err)) return } _ = unix.Unmount(filepath.Join(d.DevicesPath(), "lxcfs"), unix.MNT_DETACH) } // Determine if instance should be auto-restarted. var autoRestart bool if target != "reboot" && op.GetInstanceInitiated() && d.shouldAutoRestart() { autoRestart = true // Mark current shutdown as complete. op.Done(nil) // Create a new restart operation. op, err = operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionRestart, nil, true, false) if err == nil { defer op.Done(nil) } else { d.logger.Error("Failed to setup new restart operation", logger.Ctx{"err": err}) } } // Log and emit lifecycle if not user triggered if target != "reboot" && !autoRestart && op.GetInstanceInitiated() { ctxMap := logger.Ctx{ "action": target, "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, "stateful": false, } d.logger.Info("Shut down instance", ctxMap) d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceShutdown.Event(d, nil)) } // Reboot the container if target == "reboot" || autoRestart { // Start the container again err = d.Start(false) if err != nil { op.Done(fmt.Errorf("Failed restarting instance: %w", err)) return } d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceRestarted.Event(d, nil)) return } // Trigger a rebalance defer cgroup.TaskSchedulerTrigger("container", d.name, "stopped") // Destroy ephemeral containers if d.ephemeral { err = d.delete(true, true) if err != nil { op.Done(fmt.Errorf("Failed deleting ephemeral instance: %w", err)) return } } }(d, target, op) return nil } // cleanupDevices performs any needed device cleanup steps when container is stopped. // Accepts a stopHookNetnsPath argument which is required when run from the onStopNS hook before the // container's network namespace is unmounted (which is required for NIC device cleanup). func (d *lxc) cleanupDevices(instanceRunning bool, stopHookNetnsPath string) { for _, entry := range d.expandedDevices.Reversed() { // Only stop NIC devices when run from the onStopNS hook, and stop all other devices when run from // the onStop hook. This way disk devices are stopped after the instance has been fully stopped. if (stopHookNetnsPath != "" && entry.Config["type"] != "nic") || (stopHookNetnsPath == "" && entry.Config["type"] == "nic") { continue } dev, err := d.deviceLoad(d, entry.Name, entry.Config, false) if err != nil { if errors.Is(err, device.ErrUnsupportedDevType) { continue // Skip unsupported device (allows for mixed instance type profiles). } // Just log an error, but still allow the device to be stopped if usable device returned. d.logger.Error("Failed stop validation for device", logger.Ctx{"device": entry.Name, "err": err}) } // If a usable device was returned from deviceLoad try to stop anyway, even if validation fails. // This allows for the scenario where a new version has additional validation restrictions // than older versions and we still need to allow previously valid devices to be stopped even if // they are no longer considered valid. if dev != nil { err = d.deviceStop(dev, instanceRunning, stopHookNetnsPath) if err != nil { d.logger.Error("Failed to stop device", logger.Ctx{"device": dev.Name(), "err": err}) } } } } // Freeze functions. func (d *lxc) Freeze() error { ctxMap := logger.Ctx{ "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, } // Check that we're running if !d.IsRunning() { return errors.New("The instance isn't running") } // Load the go-lxc struct cc, err := d.initLXC(false) if err != nil { ctxMap["err"] = err d.logger.Error("Failed freezing container", ctxMap) return err } // Check that we're not already frozen if d.IsFrozen() { return errors.New("The container is already frozen") } d.logger.Info("Freezing container", ctxMap) err = cc.Freeze() if err != nil { ctxMap["err"] = err d.logger.Error("Failed freezing container", ctxMap) return err } d.logger.Info("Froze container", ctxMap) d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstancePaused.Event(d, nil)) return err } // Unfreeze unfreezes the instance. func (d *lxc) Unfreeze() error { ctxMap := logger.Ctx{ "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, } // Check that we're running if !d.IsRunning() { return errors.New("The container isn't running") } // Load the go-lxc struct cc, err := d.initLXC(false) if err != nil { d.logger.Error("Failed unfreezing container", ctxMap) return err } // Check that we're frozen if !d.IsFrozen() { return errors.New("The container is already running") } d.logger.Info("Unfreezing container", ctxMap) err = cc.Unfreeze() if err != nil { d.logger.Error("Failed unfreezing container", ctxMap) } d.logger.Info("Unfroze container", ctxMap) d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceResumed.Event(d, nil)) return err } // Get lxc container state, with 1 second timeout. // If we don't get a reply, assume the lxc monitor is unresponsive. func (d *lxc) getLxcState() (liblxc.State, error) { if d.IsSnapshot() { return liblxc.StateMap["STOPPED"], nil } // Load the go-lxc struct cc, err := d.initLXC(false) if err != nil { return liblxc.StateMap["STOPPED"], err } monitor := make(chan liblxc.State, 1) go func(c *liblxc.Container) { monitor <- c.State() }(cc) select { case state := <-monitor: return state, nil case <-time.After(5 * time.Second): return liblxc.StateMap["FROZEN"], errors.New("Monitor is unresponsive") } } // RenderWithUsage renders the API response including disk usage. func (d *lxc) RenderWithUsage() (any, any, error) { resp, etag, err := d.Render() if err != nil { return nil, nil, err } // Currently only snapshot data needs usage added. snapResp, ok := resp.(*api.InstanceSnapshot) if !ok { return resp, etag, nil } pool, err := d.getStoragePool() if err != nil { return nil, nil, err } // It is important that the snapshot not be mounted here as mounting a snapshot can trigger a very // expensive filesystem UUID regeneration, so we rely on the driver implementation to get the info // we are requesting as cheaply as possible. volumeState, err := pool.GetInstanceUsage(d) if err != nil { return resp, etag, nil } snapResp.Size = volumeState.Used return snapResp, etag, nil } // Render renders the state of the instance. func (d *lxc) Render() (any, any, error) { // Ignore err as the arch string on error is correct (unknown) architectureName, _ := osarch.ArchitectureName(d.architecture) profileNames := make([]string, 0, len(d.profiles)) for _, profile := range d.profiles { profileNames = append(profileNames, profile.Name) } if d.IsSnapshot() { // Prepare the response. snapState := api.InstanceSnapshot{ CreatedAt: d.creationDate, Description: d.description, ExpandedConfig: d.expandedConfig, ExpandedDevices: d.expandedDevices.CloneNative(), LastUsedAt: d.lastUsedDate, Name: strings.SplitN(d.name, "/", 2)[1], Stateful: d.stateful, Size: -1, // Default to uninitialized/error state (0 means no CoW usage). } snapState.Architecture = architectureName snapState.Config = d.localConfig snapState.Devices = d.localDevices.CloneNative() snapState.Ephemeral = d.ephemeral snapState.Profiles = profileNames snapState.ExpiresAt = d.expiryDate return &snapState, d.ETag(), nil } // Prepare the response. statusCode := d.statusCode() instState := api.Instance{ ExpandedConfig: d.expandedConfig, ExpandedDevices: d.expandedDevices.CloneNative(), Name: d.name, Status: statusCode.String(), StatusCode: statusCode, Location: d.node, Type: d.Type().String(), } instState.Description = d.description instState.Architecture = architectureName instState.Config = d.localConfig instState.CreatedAt = d.creationDate instState.Devices = d.localDevices.CloneNative() instState.Ephemeral = d.ephemeral instState.LastUsedAt = d.lastUsedDate instState.Profiles = profileNames instState.Stateful = d.stateful instState.Project = d.project.Name return &instState, d.ETag(), nil } // RenderFull renders the full state of the instance. func (d *lxc) RenderFull(hostInterfaces []net.Interface) (*api.InstanceFull, any, error) { if d.IsSnapshot() { return nil, nil, errors.New("RenderFull only works with containers") } // Get the Container struct base, etag, err := d.Render() if err != nil { return nil, nil, err } // Convert to ContainerFull ct := api.InstanceFull{Instance: *base.(*api.Instance)} // Add the ContainerState ct.State, err = d.renderState(ct.StatusCode, hostInterfaces) if err != nil { return nil, nil, err } // Add the ContainerSnapshots snaps, err := d.Snapshots() if err != nil { return nil, nil, err } for _, snap := range snaps { render, _, err := snap.Render() if err != nil { return nil, nil, err } if ct.Snapshots == nil { ct.Snapshots = []api.InstanceSnapshot{} } ct.Snapshots = append(ct.Snapshots, *render.(*api.InstanceSnapshot)) } // Add the ContainerBackups backups, err := d.Backups() if err != nil { return nil, nil, err } for _, backup := range backups { render := backup.Render() if ct.Backups == nil { ct.Backups = []api.InstanceBackup{} } ct.Backups = append(ct.Backups, *render) } return &ct, etag, nil } // renderState renders just the running state of the instance. func (d *lxc) renderState(statusCode api.StatusCode, hostInterfaces []net.Interface) (*api.InstanceState, error) { status := api.InstanceState{ Status: statusCode.String(), StatusCode: statusCode, } // If container is in error state, we're done here. if d.isErrorStatusCode(statusCode) { return &status, nil } pid := d.InitPID() processesState, _ := d.processesState(pid) if d.isRunningStatusCode(statusCode) { var err error status.CPU = d.cpuState() status.Memory = d.memoryState() status.Network = d.networkState(hostInterfaces) status.Pid = int64(pid) status.Processes = processesState status.StartedAt, err = d.processStartedAt(d.InitPID()) if err != nil { return nil, err } } status.Disk = d.diskState() d.release() return &status, nil } // RenderState renders just the running state of the instance. func (d *lxc) RenderState(hostInterfaces []net.Interface) (*api.InstanceState, error) { return d.renderState(d.statusCode(), hostInterfaces) } // snapshot creates a snapshot of the instance. func (d *lxc) snapshot(name string, expiry time.Time, stateful bool) error { // Check that migration.stateful is set for stateful actions. if stateful && !d.CanLiveMigrate() { return errors.New("Stateful snapshots require that the instance has migration.stateful be set to true") } // Deal with state. if stateful { // Quick checks. if !d.IsRunning() { return errors.New("Unable to create a stateful snapshot. The instance isn't running") } _, err := exec.LookPath("criu") if err != nil { return errors.New("Unable to create a stateful snapshot. CRIU isn't installed") } // Cleanup any existing state stateDir := d.StatePath() _ = os.RemoveAll(stateDir) // Create the state path and make sure we don't keep state around after the snapshot has been made. err = os.MkdirAll(stateDir, 0o700) if err != nil { return err } defer func() { _ = os.RemoveAll(stateDir) }() // Release liblxc container once done. defer func() { d.release() }() // Load the go-lxc struct if d.expandedConfig["raw.lxc"] != "" { cc, err := d.initLXC(true) if err != nil { return err } err = d.loadRawLXCConfig(cc) if err != nil { return err } } else { _, err = d.initLXC(false) if err != nil { return err } } /* TODO: ideally we would freeze here and unfreeze below after * we've copied the filesystem, to make sure there are no * changes by the container while snapshotting. Unfortunately * there is abug in CRIU where it doesn't leave the container * in the same state it found it w.r.t. freezing, i.e. CRIU * freezes too, and then /always/ thaws, even if the container * was frozen. Until that's fixed, all calls to Unfreeze() * after snapshotting will fail. */ criuMigrationArgs := instance.CriuMigrationArgs{ Cmd: liblxc.MIGRATE_DUMP, StateDir: stateDir, Function: "snapshot", Stop: false, ActionScript: false, DumpDir: "", PreDumpDir: "", } // Dump the state. err = d.migrate(&criuMigrationArgs) if err != nil { return fmt.Errorf("Failed taking stateful checkpoint: %w", err) } } // Wait for any file operations to complete to have a more consistent snapshot. d.stopForkfile(false) return d.snapshotCommon(d, name, expiry, stateful) } // Snapshot takes a new snapshot. func (d *lxc) Snapshot(name string, expiry time.Time, stateful bool) error { return d.snapshot(name, expiry, stateful) } // Restore restores a snapshot. func (d *lxc) Restore(sourceContainer instance.Instance, stateful bool, diskOnly bool) error { var ctxMap logger.Ctx op, err := operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionRestore, false, false) if err != nil { return fmt.Errorf("Failed to create instance restore operation: %w", err) } defer op.Done(nil) // Initialize storage interface for the container. pool, err := storagePools.LoadByInstance(d.state, d) if err != nil { op.Done(err) return err } err = pool.CanRestoreInstanceSnapshot(d, sourceContainer) if err != nil { op.Done(err) return err } // Stop the container. wasRunning := d.IsRunning() if wasRunning { ephemeral := d.IsEphemeral() if ephemeral { // Unset ephemeral flag. args := db.InstanceArgs{ Architecture: d.Architecture(), Config: d.LocalConfig(), Description: d.Description(), Devices: d.LocalDevices(), Ephemeral: false, Profiles: d.Profiles(), Project: d.Project().Name, Type: d.Type(), Snapshot: d.IsSnapshot(), } err := d.Update(args, false) if err != nil { op.Done(err) return err } // On function return, set the flag back on. defer func() { args.Ephemeral = ephemeral _ = d.Update(args, false) }() } // This will unmount the container storage. err := d.Stop(false) if err != nil { op.Done(err) return err } // Refresh the operation as that one is now complete. op, err = operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionRestore, false, false) if err != nil { return fmt.Errorf("Failed to create instance restore operation: %w", err) } defer op.Done(nil) } ctxMap = logger.Ctx{ "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, "source": sourceContainer.Name(), } d.logger.Info("Restoring instance", ctxMap) // Wait for any file operations to complete. // This is required so we can actually unmount the container and restore its rootfs. d.stopForkfile(false) d.logger.Debug("Mounting instance to check for CRIU state path existence") reverter := revert.New() defer reverter.Fail() // Ensure that storage is mounted for state path checks and for backup.yaml updates. _, err = d.mount() if err != nil { op.Done(err) return err } reverter.Add(func() { _ = d.unmount() }) // Check for CRIU if necessary, before doing a bunch of filesystem manipulations. // Requires container be mounted to check StatePath exists. if util.PathExists(d.StatePath()) { _, err := exec.LookPath("criu") if err != nil { err = errors.New("Failed to restore container state. CRIU isn't installed") op.Done(err) return err } } err = d.unmount() if err != nil { op.Done(err) return err } reverter.Success() // Restore the rootfs. err = pool.RestoreInstanceSnapshot(d, sourceContainer, nil) if err != nil { op.Done(err) return err } args := db.InstanceArgs{} if !diskOnly { // Restore the configuration. args = db.InstanceArgs{ Architecture: sourceContainer.Architecture(), Config: sourceContainer.LocalConfig(), Description: sourceContainer.Description(), Devices: sourceContainer.LocalDevices(), Ephemeral: sourceContainer.IsEphemeral(), Profiles: sourceContainer.Profiles(), Project: sourceContainer.Project().Name, Type: sourceContainer.Type(), Snapshot: sourceContainer.IsSnapshot(), } } else { args = db.InstanceArgs{ Architecture: d.Architecture(), Config: d.LocalConfig(), Description: d.Description(), Devices: d.LocalDevices(), Ephemeral: d.IsEphemeral(), Profiles: d.Profiles(), Project: d.Project().Name, Type: d.Type(), Snapshot: d.IsSnapshot(), } args.Config["volatile.uuid.generation"] = sourceContainer.LocalConfig()["volatile.uuid.generation"] } // Don't pass as user-requested as there's no way to fix a bad config. // This will call d.UpdateBackupFile() to ensure snapshot list is up to date. err = d.Update(args, false) if err != nil { op.Done(err) return err } // If the container wasn't running but was stateful, should we restore it as running? if stateful { if !util.PathExists(d.StatePath()) { err = errors.New("Stateful snapshot restore requested but snapshot is stateless") op.Done(err) return err } d.logger.Debug("Performing stateful restore", ctxMap) d.stateful = true criuMigrationArgs := instance.CriuMigrationArgs{ Cmd: liblxc.MIGRATE_RESTORE, StateDir: d.StatePath(), Function: "snapshot", Stop: false, ActionScript: false, DumpDir: "", PreDumpDir: "", } // Checkpoint. err = d.migrate(&criuMigrationArgs) if err != nil { op.Done(err) return fmt.Errorf("Failed taking stateful checkpoint: %w", err) } // Remove the state from the parent container; we only keep this in snapshots. err2 := os.RemoveAll(d.StatePath()) if err2 != nil && !errors.Is(err, fs.ErrNotExist) { op.Done(err) return err } if err != nil { op.Done(err) return err } d.logger.Debug("Performed stateful restore", ctxMap) d.logger.Info("Restored instance", ctxMap) return nil } // Restart the container. if wasRunning { d.logger.Debug("Starting instance after snapshot restore") err = d.Start(false) if err != nil { op.Done(err) return err } } d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceRestored.Event(d, map[string]any{"snapshot": sourceContainer.Name()})) d.logger.Info("Restored instance", ctxMap) return nil } func (d *lxc) cleanup() { // Unmount any leftovers _ = d.removeUnixDevices() _ = d.removeDiskDevices() // Remove the security profiles _ = apparmor.InstanceDelete(d.state.OS, d) seccomp.DeleteProfile(d) // Remove the devices path _ = os.Remove(d.DevicesPath()) // Remove the shmounts path _ = os.RemoveAll(d.ShmountsPath()) } // Delete deletes the instance. // cleanupDependencies controls whether dependent resources (e.g. volumes, // and related state) are removed along with the instance. // When false, dependencies are preserved (e.g. storage-only moves). func (d *lxc) Delete(force bool, cleanupDependencies bool) error { // Setup a new operation. op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionDelete, nil, false, false) if err != nil { return fmt.Errorf("Failed to create instance delete operation: %w", err) } defer op.Done(nil) if d.IsRunning() { return api.StatusErrorf(http.StatusBadRequest, "Instance is running") } err = d.delete(force, cleanupDependencies) if err != nil { return err } // If dealing with a snapshot, refresh the backup file on the parent. if d.IsSnapshot() { parentName, _, _ := api.GetParentAndSnapshotName(d.name) // Load the parent. parent, err := instance.LoadByProjectAndName(d.state, d.project.Name, parentName) if err != nil { return fmt.Errorf("Invalid parent: %w", err) } // Update the backup file. err = parent.UpdateBackupFile() if err != nil { return err } } return nil } // Delete deletes the instance without creating an operation lock. func (d *lxc) delete(force bool, cleanupDependencies bool) error { ctxMap := logger.Ctx{ "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, } if d.isSnapshot { d.logger.Info("Deleting instance snapshot", ctxMap) } else { d.logger.Info("Deleting instance", ctxMap) } if !force && util.IsTrue(d.expandedConfig["security.protection.delete"]) && !d.IsSnapshot() { err := errors.New("Instance is protected") d.logger.Warn("Failed to delete instance", logger.Ctx{"err": err}) return err } // Wait for any file operations to complete. // This is required so we can actually unmount the container and delete it. if !d.IsSnapshot() { d.stopForkfile(false) } // Delete any persistent warnings for instance. err := d.warningsDelete() if err != nil { return err } pool, err := storagePools.LoadByInstance(d.state, d) if err != nil && !response.IsNotFoundError(err) { return err } else if pool != nil { if d.IsSnapshot() { // Remove snapshot volume and database record. err = pool.DeleteInstanceSnapshot(d, nil) if err != nil { return err } } else { // Remove all snapshots. err := d.deleteSnapshots(func(snapInst instance.Instance) error { return snapInst.(*lxc).delete(true, cleanupDependencies) // Internal delete function that doesn't lock. }) if err != nil { return fmt.Errorf("Failed deleting instance snapshots: %w", err) } // Remove the storage volume and database records. err = pool.DeleteInstance(d, nil) if err != nil { return err } if cleanupDependencies { // Delete all dependent volumes associated with this instance. err = d.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error { // Load the pool for the disk. diskPool, err := storagePools.LoadByName(d.state, dev.Config["pool"]) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } err = diskPool.DeleteCustomVolume(d.Project().Name, dev.Config["source"], nil) if err != nil { return err } return nil }) if err != nil { return err } } } } // Perform other cleanup steps if not snapshot. if !d.IsSnapshot() { // Remove all backups. backups, err := d.Backups() if err != nil { return err } for _, backup := range backups { err = backup.Delete() if err != nil { return err } } // Run device removal function for each device. d.devicesRemove(d, cleanupDependencies) // Clean things up. d.cleanup() } err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Remove the database record of the instance or snapshot instance. return tx.DeleteInstance(ctx, d.project.Name, d.Name()) }) if err != nil { d.logger.Error("Failed deleting instance entry", logger.Ctx{"err": err}) return err } if d.isSnapshot { d.logger.Info("Deleted instance snapshot", ctxMap) } else { d.logger.Info("Deleted instance", ctxMap) } if d.isSnapshot { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceSnapshotDeleted.Event(d, nil)) } else { err = d.state.Authorizer.DeleteInstance(d.state.ShutdownCtx, d.project.Name, d.Name()) if err != nil { logger.Error("Failed to remove instance from authorizer", logger.Ctx{"name": d.Name(), "project": d.project.Name, "error": err}) } d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceDeleted.Event(d, nil)) } return nil } // Rename renames the instance. Accepts an argument to enable applying deferred TemplateTriggerRename. func (d *lxc) Rename(newName string, applyTemplateTrigger bool) error { oldName := d.Name() ctxMap := logger.Ctx{ "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, "newname": newName, } d.logger.Info("Renaming instance", ctxMap) // Quick checks. err := instance.ValidName(newName, d.IsSnapshot()) if err != nil { return err } if d.IsRunning() { return errors.New("Renaming of running instance not allowed") } // Clean things up. d.cleanup() pool, err := storagePools.LoadByInstance(d.state, d) if err != nil { return fmt.Errorf("Failed loading instance storage pool: %w", err) } if d.IsSnapshot() { _, newSnapName, _ := api.GetParentAndSnapshotName(newName) err = pool.RenameInstanceSnapshot(d, newSnapName, nil) if err != nil { return fmt.Errorf("Rename instance snapshot: %w", err) } } else { err = pool.RenameInstance(d, newName, nil) if err != nil { return fmt.Errorf("Rename instance: %w", err) } if applyTemplateTrigger { err = d.DeferTemplateApply(instance.TemplateTriggerRename) if err != nil { return err } } } if !d.IsSnapshot() { var results []string err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Rename all the instance snapshot database entries. results, err = tx.GetInstanceSnapshotsNames(ctx, d.project.Name, oldName) if err != nil { d.logger.Error("Failed to get instance snapshots", ctxMap) return fmt.Errorf("Failed to get instance snapshots: Failed getting instance snapshot names: %w", err) } for _, sname := range results { // Rename the snapshot. oldSnapName := strings.SplitN(sname, internalInstance.SnapshotDelimiter, 2)[1] baseSnapName := filepath.Base(sname) err := cluster.RenameInstanceSnapshot(ctx, tx.Tx(), d.project.Name, oldName, oldSnapName, baseSnapName) if err != nil { d.logger.Error("Failed renaming snapshot", ctxMap) return fmt.Errorf("Failed renaming snapshot: %w", err) } } return nil }) if err != nil { return err } } // Rename the instance database entry. err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { if d.IsSnapshot() { oldParts := strings.SplitN(oldName, internalInstance.SnapshotDelimiter, 2) newParts := strings.SplitN(newName, internalInstance.SnapshotDelimiter, 2) return cluster.RenameInstanceSnapshot(ctx, tx.Tx(), d.project.Name, oldParts[0], oldParts[1], newParts[1]) } return cluster.RenameInstance(ctx, tx.Tx(), d.project.Name, oldName, newName) }) if err != nil { d.logger.Error("Failed renaming instance", ctxMap) return fmt.Errorf("Failed renaming instance: %w", err) } // Rename the logging path. newFullName := project.Instance(d.Project().Name, d.Name()) _ = os.RemoveAll(internalUtil.LogPath(newFullName)) if util.PathExists(d.LogPath()) { err := os.Rename(d.LogPath(), internalUtil.LogPath(newFullName)) if err != nil { d.logger.Error("Failed renaming instance", ctxMap) return fmt.Errorf("Failed renaming instance: %w", err) } } // Rename the runtime path. newFullName = project.Instance(d.Project().Name, d.Name()) _ = os.RemoveAll(internalUtil.RunPath(newFullName)) if util.PathExists(d.RunPath()) { err := os.Rename(d.RunPath(), internalUtil.RunPath(newFullName)) if err != nil { d.logger.Error("Failed renaming instance", ctxMap) return fmt.Errorf("Failed renaming instance: %w", err) } } reverter := revert.New() defer reverter.Fail() // Set the new name in the struct. d.name = newName reverter.Add(func() { d.name = oldName }) // Rename the backups. backups, err := d.Backups() if err != nil { return err } for _, backup := range backups { b := backup oldName := b.Name() backupName := strings.Split(oldName, "/")[1] newName := fmt.Sprintf("%s/%s", newName, backupName) err = b.Rename(newName) if err != nil { return err } reverter.Add(func() { _ = b.Rename(oldName) }) } // Invalidate the go-lxc cache. d.release() d.cConfig = false // Update lease files. err = network.UpdateDNSMasqStatic(d.state, "") if err != nil { return err } // Reset cloud-init instance-id (causes a re-run on name changes). if !d.IsSnapshot() { err = d.resetInstanceID() if err != nil { return err } } // Update the backup file. err = d.UpdateBackupFile() if err != nil { return err } d.logger.Info("Renamed instance", ctxMap) if d.isSnapshot { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceSnapshotRenamed.Event(d, map[string]any{"old_name": oldName})) } else { err = d.state.Authorizer.RenameInstance(d.state.ShutdownCtx, d.project.Name, oldName, newName) if err != nil { logger.Error("Failed to rename instance in authorizer", logger.Ctx{"old_name": oldName, "new_name": newName, "project": d.project.Name, "error": err}) } d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceRenamed.Event(d, map[string]any{"old_name": oldName})) } reverter.Success() return nil } // CGroupSet sets a cgroup value for the instance. func (d *lxc) CGroupSet(key string, value string) error { // Load the go-lxc struct cc, err := d.initLXC(false) if err != nil { return err } // Make sure the container is running. // We use InitPID here rather than IsRunning because this task can be triggered during the container's // startup process, which is during the time that the start lock is held, which causes IsRunning to // return false (because the container hasn't fully started yet) but it is sufficiently started to // have its cgroup disk limits set. if d.InitPID() <= 0 { return errors.New("Can't set cgroups on a stopped container") } err = cc.SetCgroupItem(key, value) if err != nil { return fmt.Errorf("Failed to set cgroup %s=\"%s\": %w", key, value, err) } return nil } // Update applies updated config. func (d *lxc) Update(args db.InstanceArgs, userRequested bool) error { // Setup a new operation op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionUpdate, []operationlock.Action{operationlock.ActionCreate, operationlock.ActionRestart, operationlock.ActionRestore}, false, false) if err != nil { return fmt.Errorf("Failed to create instance update operation: %w", err) } defer op.Done(nil) // Set sane defaults for unset keys if args.Project == "" { args.Project = api.ProjectDefaultName } if args.Architecture == 0 { args.Architecture = d.architecture } if args.Config == nil { args.Config = map[string]string{} } if args.Devices == nil { args.Devices = deviceConfig.Devices{} } if args.Profiles == nil { args.Profiles = []api.Profile{} } if userRequested { // Validate the new config err := instance.ValidConfig(d.state.OS, args.Config, false, d.dbType) if err != nil { return fmt.Errorf("Invalid config: %w", err) } // Validate the new devices without using expanded devices validation (expensive checks disabled). err = instance.ValidDevices(d.state, d.project, d.Type(), args.Devices, nil) if err != nil { return fmt.Errorf("Invalid devices: %w", err) } } var profiles []string err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Validate the new profiles profiles, err = tx.GetProfileNames(ctx, args.Project) return err }) if err != nil { return fmt.Errorf("Failed to get profiles: %w", err) } checkedProfiles := []string{} for _, profile := range args.Profiles { if !slices.Contains(profiles, profile.Name) { return fmt.Errorf("Requested profile '%s' doesn't exist", profile.Name) } if slices.Contains(checkedProfiles, profile.Name) { return errors.New("Duplicate profile found in request") } checkedProfiles = append(checkedProfiles, profile.Name) } // Validate the new architecture if args.Architecture != 0 { _, err = osarch.ArchitectureName(args.Architecture) if err != nil { return fmt.Errorf("Invalid architecture id: %s", err) } } // Get a copy of the old configuration oldDescription := d.Description() oldArchitecture := 0 err = util.DeepCopy(&d.architecture, &oldArchitecture) if err != nil { return err } oldEphemeral := false err = util.DeepCopy(&d.ephemeral, &oldEphemeral) if err != nil { return err } oldExpandedDevices := deviceConfig.Devices{} err = util.DeepCopy(&d.expandedDevices, &oldExpandedDevices) if err != nil { return err } oldExpandedConfig := map[string]string{} err = util.DeepCopy(&d.expandedConfig, &oldExpandedConfig) if err != nil { return err } oldLocalDevices := deviceConfig.Devices{} err = util.DeepCopy(&d.localDevices, &oldLocalDevices) if err != nil { return err } oldLocalConfig := map[string]string{} err = util.DeepCopy(&d.localConfig, &oldLocalConfig) if err != nil { return err } oldProfiles := []api.Profile{} err = util.DeepCopy(&d.profiles, &oldProfiles) if err != nil { return err } oldExpiryDate := d.expiryDate // Define a function which reverts everything. Defer this function // so that it doesn't need to be explicitly called in every failing // return path. Track whether or not we want to undo the changes // using a closure. undoChanges := true defer func() { if undoChanges { d.description = oldDescription d.architecture = oldArchitecture d.ephemeral = oldEphemeral d.expandedConfig = oldExpandedConfig d.expandedDevices = oldExpandedDevices d.localConfig = oldLocalConfig d.localDevices = oldLocalDevices d.profiles = oldProfiles d.expiryDate = oldExpiryDate d.release() d.cConfig = false _, _ = d.initLXC(true) _ = d.setupCredentials(true) cgroup.TaskSchedulerTrigger("container", d.name, "changed") } }() // Apply the various changes d.description = args.Description d.architecture = args.Architecture d.ephemeral = args.Ephemeral d.localConfig = args.Config d.localDevices = args.Devices d.profiles = args.Profiles d.expiryDate = args.ExpiryDate // Expand the config and refresh the LXC config err = d.expandConfig() if err != nil { return err } // Diff the configurations changedConfig := []string{} for key := range oldExpandedConfig { if oldExpandedConfig[key] != d.expandedConfig[key] { if !slices.Contains(changedConfig, key) { changedConfig = append(changedConfig, key) } } } for key := range d.expandedConfig { if oldExpandedConfig[key] != d.expandedConfig[key] { if !slices.Contains(changedConfig, key) { changedConfig = append(changedConfig, key) } } } // Diff the devices removeDevices, addDevices, updateDevices, allUpdatedKeys := oldExpandedDevices.Update(d.expandedDevices, func(oldDevice deviceConfig.Device, newDevice deviceConfig.Device) []string { // This function needs to return a list of fields that are excluded from differences // between oldDevice and newDevice. The result of this is that as long as the // devices are otherwise identical except for the fields returned here, then the // device is considered to be being "updated" rather than "added & removed". oldDevType, err := device.LoadByType(d.state, d.Project().Name, oldDevice) if err != nil { return []string{} // Couldn't create Device, so this cannot be an update. } newDevType, err := device.LoadByType(d.state, d.Project().Name, newDevice) if err != nil { return []string{} // Couldn't create Device, so this cannot be an update. } // Detached devices need to be fully recreated on update so that the update logic doesn't // try to access non-existing LXC devices. if !util.IsTrueOrEmpty(oldDevice["attached"]) { return []string{} } return newDevType.UpdatableFields(oldDevType) }) // Prevent adding or updating device initial configuration. if util.StringPrefixInSlice("initial.", allUpdatedKeys) { for devName, newDev := range addDevices { for k, newVal := range newDev { if !strings.HasPrefix(k, "initial.") { continue } if strings.HasPrefix(newDev["source"], "tmpfs:") || strings.HasPrefix(newDev["source"], "tmpfs-overlay:") { continue } if newDev["pool"] != "" && newDev["path"] != "/" && strings.Contains(newDev["source"], "/") { continue } oldDev, ok := removeDevices[devName] if !ok { return errors.New("New device with initial configuration cannot be added once the instance is created") } oldVal, ok := oldDev[k] if !ok { return errors.New("Device initial configuration cannot be added once the instance is created") } // If newVal is an empty string it means the initial configuration // has been removed. if newVal != "" && newVal != oldVal { return errors.New("Device initial configuration cannot be modified once the instance is created") } } } } if userRequested { // Look for deleted idmap keys. protectedKeys := []string{ "volatile.idmap.base", "volatile.idmap.current", "volatile.idmap.next", "volatile.last_state.idmap", } for _, k := range changedConfig { if !slices.Contains(protectedKeys, k) { continue } _, ok := d.expandedConfig[k] if !ok { return errors.New("Volatile idmap keys can't be deleted by the user") } } // Do some validation of the config diff (allows mixed instance types for profiles). err = instance.ValidConfig(d.state.OS, d.expandedConfig, true, instancetype.Any) if err != nil { return fmt.Errorf("Invalid expanded config: %w", err) } // Do full expanded validation of the devices diff. err = instance.ValidDevices(d.state, d.project, d.Type(), d.localDevices, d.expandedDevices) if err != nil { return fmt.Errorf("Invalid expanded devices: %w", err) } // Validate root device _, oldRootDev, oldErr := internalInstance.GetRootDiskDevice(oldExpandedDevices.CloneNative()) _, newRootDev, newErr := internalInstance.GetRootDiskDevice(d.expandedDevices.CloneNative()) if oldErr == nil && newErr == nil && oldRootDev["pool"] != newRootDev["pool"] { return fmt.Errorf("Cannot update root disk device pool name to %q", newRootDev["pool"]) } // Ensure the instance has a root disk. if newErr != nil { return fmt.Errorf("Invalid root disk device: %w", newErr) } } // Run through initLXC to catch anything we missed if userRequested { d.release() d.cConfig = false _, err = d.initLXC(true) if err != nil { return fmt.Errorf("Initialize LXC: %w", err) } } // If raw.lxc changed, re-validate the config. if slices.Contains(changedConfig, "raw.lxc") && d.expandedConfig["raw.lxc"] != "" { // Get a new liblxc instance. cc, err := liblxc.NewContainer(d.name, d.state.OS.LxcPath) if err != nil { return err } err = d.loadRawLXCConfig(cc) if err != nil { // Release the liblxc instance. _ = cc.Release() return err } // Release the liblxc instance. _ = cc.Release() } // If apparmor changed, re-validate the apparmor profile (even if not running). if slices.Contains(changedConfig, "raw.apparmor") || slices.Contains(changedConfig, "security.nesting") { err = apparmor.InstanceValidate(d.state.OS, d, nil) if err != nil { return fmt.Errorf("Parse AppArmor profile: %w", err) } } if slices.Contains(changedConfig, "security.idmap.isolated") || slices.Contains(changedConfig, "security.idmap.base") || slices.Contains(changedConfig, "security.idmap.size") || slices.Contains(changedConfig, "raw.idmap") || slices.Contains(changedConfig, "security.privileged") { var idmapSet *idmap.Set base := int64(0) if !d.IsPrivileged() { // Update the idmap. idmapSet, base, err = d.findIdmap() if err != nil { return fmt.Errorf("Failed to get ID map: %w", err) } } jsonIdmap, err := idmapSet.ToJSON() if err != nil { return fmt.Errorf("Failed to encode ID map: %w", err) } d.localConfig["volatile.idmap.next"] = jsonIdmap d.localConfig["volatile.idmap.base"] = fmt.Sprintf("%v", base) // Invalidate the idmap cache. d.idmapset = nil } isRunning := d.IsRunning() // Use the device interface to apply update changes. err = d.devicesUpdate(d, removeDevices, addDevices, updateDevices, oldExpandedDevices, isRunning, userRequested) if err != nil { return err } // Apply the live changes if isRunning { cc, err := d.initLXC(false) if err != nil { return err } cg, err := d.cgroup(cc, true) if err != nil { return err } // Live update the container config for _, key := range changedConfig { value := d.expandedConfig[key] if key == "raw.apparmor" || key == "security.nesting" { // Update the AppArmor profile err = apparmor.InstanceLoad(d.state.OS, d, nil) if err != nil { return err } } else if key == "security.guestapi" { if util.IsTrueOrEmpty(value) { err = d.insertMount(internalUtil.VarPath("guestapi"), "/dev/incus", "none", unix.MS_BIND, idmap.StorageTypeNone) if err != nil { return err } } else { // Connect to files API. files, err := d.FileSFTP() if err != nil { return err } defer func() { _ = files.Close() }() _, err = files.Lstat("/dev/incus") if err == nil { err = d.removeMount("/dev/incus") if err != nil { return err } err = files.Remove("/dev/incus") if err != nil { return err } } } } else if key == "linux.kernel_modules" && value != "" { for _, module := range strings.Split(value, ",") { module = strings.TrimPrefix(module, " ") err := linux.LoadModule(module) if err != nil { return fmt.Errorf("Failed to load kernel module '%s': %w", module, err) } } } else if key == "limits.disk.priority" { if !cgroup.Supports(cgroup.IO) { continue } priorityInt := 5 diskPriority := d.expandedConfig["limits.disk.priority"] if diskPriority != "" { priorityInt, err = strconv.Atoi(diskPriority) if err != nil { return err } } // Minimum valid value is 10 priority := int64(priorityInt * 100) if priority == 0 { priority = 10 } err = cg.SetBlkioWeight(priority) if err != nil { return err } } else if key == "limits.memory.oom_priority" { // Configure the OOM priority. err = d.setOOMPriority(cc.InitPid()) if err != nil { d.logger.Warn("Failed to set OOM priority", logger.Ctx{ "err": err, "instance": d.Name(), "project": d.Project().Name, }) } } else if key == "limits.memory" || strings.HasPrefix(key, "limits.memory.") { // Skip if no memory CGroup if !cgroup.Supports(cgroup.Memory) { continue } // Set the new memory limit memory := d.expandedConfig["limits.memory"] memoryEnforce := d.expandedConfig["limits.memory.enforce"] memorySwap := d.expandedConfig["limits.memory.swap"] var memoryInt int64 // Parse memory if memory == "" { memoryInt = -1 } else { memoryInt, err = ParseMemoryStr(memory) if err != nil { return err } } // Store the old values for revert oldMemswLimit := int64(-1) if cgroup.Supports(cgroup.Memory) { oldMemswLimit, err = cg.GetMemorySwapLimit() if err != nil { oldMemswLimit = -1 } } oldLimit, err := cg.GetMemoryLimit() if err != nil { oldLimit = -1 } oldSoftLimit, err := cg.GetMemorySoftLimit() if err != nil { oldSoftLimit = -1 } revertMemory := func() { if oldSoftLimit != -1 { _ = cg.SetMemorySoftLimit(oldSoftLimit) } if oldLimit != -1 { _ = cg.SetMemoryLimit(oldLimit) } if oldMemswLimit != -1 { _ = cg.SetMemorySwapLimit(oldMemswLimit) } } // Reset everything if cgroup.Supports(cgroup.Memory) { err = cg.SetMemorySwapLimit(-1) if err != nil { revertMemory() return err } } err = cg.SetMemoryLimit(-1) if err != nil { revertMemory() return err } err = cg.SetMemorySoftLimit(-1) if err != nil { revertMemory() return err } // Set the new values if memoryEnforce == "soft" { // Set new limit. err = cg.SetMemorySoftLimit(memoryInt) if err != nil { revertMemory() return err } } else { err = cg.SetMemoryLimit(memoryInt) if err != nil { revertMemory() return err } if cgroup.Supports(cgroup.Memory) { if util.IsTrueOrEmpty(memorySwap) || util.IsFalse(memorySwap) { err = cg.SetMemorySwapLimit(0) if err != nil { revertMemory() return err } } else { // Additional memory as swap. swapInt, err := units.ParseByteSizeString(memorySwap) if err != nil { revertMemory() return err } err = cg.SetMemorySwapLimit(swapInt) if err != nil { revertMemory() return err } } } } // Configure the swappiness if key == "limits.memory.swap" || key == "limits.memory.swap.priority" { memorySwapPriority := d.expandedConfig["limits.memory.swap.priority"] if util.IsFalse(memorySwap) { err = cg.SetMemorySwappiness(0) if err != nil { return err } } else { priority := 10 if memorySwapPriority != "" { priority, err = strconv.Atoi(memorySwapPriority) if err != nil { return err } } // Maximum priority (10) should be default swappiness (60). err = cg.SetMemorySwappiness(int64(70 - priority)) if err != nil { return err } } } } else if key == "limits.cpu" || key == "limits.cpu.nodes" { // Clear the "volatile.cpu.nodes" if needed. d.ClearLimitsCPUNodes(changedConfig) // Trigger a scheduler re-run defer cgroup.TaskSchedulerTrigger("container", d.name, "changed") //nolint:revive } else if key == "limits.cpu.priority" || key == "limits.cpu.allowance" { // Skip if no cpu CGroup if !cgroup.Supports(cgroup.CPU) { continue } // Apply new CPU limits cpuShares, cpuCfsQuota, cpuCfsPeriod, err := cgroup.ParseCPU(d.expandedConfig["limits.cpu.allowance"], d.expandedConfig["limits.cpu.priority"]) if err != nil { return err } err = cg.SetCPUShare(cpuShares) if err != nil { return err } err = cg.SetCPUCfsLimit(cpuCfsPeriod, cpuCfsQuota) if err != nil { return err } } else if key == "limits.processes" { if !cgroup.Supports(cgroup.Pids) { continue } if value == "" { err = cg.SetMaxProcesses(-1) if err != nil { return err } } else { valueInt, err := strconv.ParseInt(value, 10, 64) if err != nil { return err } err = cg.SetMaxProcesses(valueInt) if err != nil { return err } } } else if strings.HasPrefix(key, "limits.hugepages.") { if !cgroup.Supports(cgroup.Hugetlb) { continue } pageType := "" switch key { case "limits.hugepages.64KB": pageType = "64KB" case "limits.hugepages.1MB": pageType = "1MB" case "limits.hugepages.2MB": pageType = "2MB" case "limits.hugepages.1GB": pageType = "1GB" } valueInt := int64(-1) if value != "" { valueInt, err = units.ParseByteSizeString(value) if err != nil { return err } } err = cg.SetHugepagesLimit(pageType, valueInt) if err != nil { return err } } } // Update the credentials directory. err = d.setupCredentials(true) if err != nil { return err } } // Re-generate the instance-id if needed. if !d.IsSnapshot() && d.needsNewInstanceID(changedConfig, oldExpandedDevices) { err = d.resetInstanceID() if err != nil { return err } } // Finally, apply the changes to the database err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Snapshots should update only their descriptions and expiry date. if d.IsSnapshot() { return tx.UpdateInstanceSnapshot(d.id, d.description, d.expiryDate) } object, err := cluster.GetInstance(ctx, tx.Tx(), d.project.Name, d.name) if err != nil { return err } object.Description = d.description object.Architecture = d.architecture object.Ephemeral = d.ephemeral object.ExpiryDate = sql.NullTime{Time: d.expiryDate, Valid: true} err = cluster.UpdateInstance(ctx, tx.Tx(), d.project.Name, d.name, *object) if err != nil { return err } err = cluster.UpdateInstanceConfig(ctx, tx.Tx(), int64(object.ID), d.localConfig) if err != nil { return err } devices, err := cluster.APIToDevices(d.localDevices.CloneNative()) if err != nil { return err } err = cluster.UpdateInstanceDevices(ctx, tx.Tx(), int64(object.ID), devices) if err != nil { return err } profileNames := make([]string, 0, len(d.profiles)) for _, profile := range d.profiles { profileNames = append(profileNames, profile.Name) } return cluster.UpdateInstanceProfiles(ctx, tx.Tx(), object.ID, object.Project, profileNames) }) if err != nil { return fmt.Errorf("Failed to update database: %w", err) } err = d.UpdateBackupFile() if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to write backup file: %w", err) } // Send devIncus notifications if isRunning { // Config changes (only for user.* keys for _, key := range changedConfig { if !strings.HasPrefix(key, "user.") { continue } msg := map[string]any{ "key": key, "old_value": oldExpandedConfig[key], "value": d.expandedConfig[key], } err = d.devIncusEventSend("config", msg) if err != nil { return err } } // Device changes for k, m := range removeDevices { msg := map[string]any{ "action": "removed", "name": k, "config": m, } err = d.devIncusEventSend("device", msg) if err != nil { return err } } for k, m := range updateDevices { msg := map[string]any{ "action": "updated", "name": k, "config": m, } err = d.devIncusEventSend("device", msg) if err != nil { return err } } for k, m := range addDevices { msg := map[string]any{ "action": "added", "name": k, "config": m, } err = d.devIncusEventSend("device", msg) if err != nil { return err } } } // Success, update the closure to mark that the changes should be kept. undoChanges = false if userRequested { if d.isSnapshot { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceSnapshotUpdated.Event(d, nil)) } else { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceUpdated.Event(d, nil)) } } return nil } // Export backs up the instance. func (d *lxc) Export(metaWriter io.Writer, rootfsWriter io.Writer, properties map[string]string, expiration time.Time, tracker *ioprogress.ProgressTracker) (*api.ImageMetadata, error) { ctxMap := logger.Ctx{ "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, } if d.IsRunning() { return nil, errors.New("Cannot export a running instance as an image") } d.logger.Info("Exporting instance", ctxMap) // Start the storage. _, err := d.mount() if err != nil { d.logger.Error("Failed exporting instance", ctxMap) return nil, err } defer func() { _ = d.unmount() }() // Get IDMap to unshift container as the tarball is created. idmap, err := d.DiskIdmap() if err != nil { d.logger.Error("Failed exporting instance", ctxMap) return nil, err } // Create the tarball. metaTarWriter := instancewriter.NewInstanceTarWriter(metaWriter, idmap) var rootfsTarWriter *instancewriter.InstanceTarWriter if rootfsWriter != nil { rootfsTarWriter = instancewriter.NewInstanceTarWriter(rootfsWriter, idmap) } // Keep track of the first path we saw for each path with nlink>1. cDir := d.Path() // Path inside the tar image is the pathname starting after cDir. // For the rootfs tarball in a split image, the path inside is the pathname starting after rootfs/ metaOffset := len(cDir) + 1 rootfsOffset := len(d.RootfsPath()) writeToMetaTar := func(fPath string, fi os.FileInfo, err error) error { if err != nil { return err } err = metaTarWriter.WriteFile(fPath[metaOffset:], fPath, fi, false) if err != nil { d.logger.Debug("Error tarring up", logger.Ctx{"path": fPath, "err": err}) return err } return nil } var writeToRootfsTar func(string, os.FileInfo, error) error if rootfsWriter != nil { writeToRootfsTar = func(path string, fi os.FileInfo, err error) error { if err != nil { return err } err = rootfsTarWriter.WriteFile(path[rootfsOffset:], path, fi, false) if err != nil { d.logger.Debug("Error tarring up", logger.Ctx{"path": path, "err": err}) return err } return nil } } // Get the instance's architecture. var arch string if d.IsSnapshot() { parentName, _, _ := api.GetParentAndSnapshotName(d.name) parent, err := instance.LoadByProjectAndName(d.state, d.project.Name, parentName) if err != nil { _ = metaTarWriter.Close() if rootfsTarWriter != nil { _ = rootfsTarWriter.Close() } d.logger.Error("Failed exporting instance", ctxMap) return nil, err } arch, _ = osarch.ArchitectureName(parent.Architecture()) } else { arch, _ = osarch.ArchitectureName(d.architecture) } if arch == "" { arch, err = osarch.ArchitectureName(d.state.OS.Architectures[0]) if err != nil { d.logger.Error("Failed exporting instance", ctxMap) return nil, err } } // Generate metadata.yaml. meta := api.ImageMetadata{} fnam := filepath.Join(cDir, "metadata.yaml") if util.PathExists(fnam) { // Parse the metadata. content, err := os.ReadFile(fnam) if err != nil { _ = metaTarWriter.Close() if rootfsTarWriter != nil { _ = rootfsTarWriter.Close() } d.logger.Error("Failed exporting instance", ctxMap) return nil, err } err = yaml.Load(content, &meta) if err != nil { _ = metaTarWriter.Close() if rootfsTarWriter != nil { _ = rootfsTarWriter.Close() } d.logger.Error("Failed exporting instance", ctxMap) return nil, err } } // Fill in the metadata. meta.Architecture = arch meta.CreationDate = time.Now().UTC().Unix() if meta.Properties == nil { meta.Properties = map[string]string{} } maps.Copy(meta.Properties, properties) if !expiration.IsZero() { meta.ExpiryDate = expiration.UTC().Unix() } // Write the new metadata.yaml. tempDir, err := os.MkdirTemp("", "incus_metadata_") if err != nil { _ = metaTarWriter.Close() if rootfsTarWriter != nil { _ = rootfsTarWriter.Close() } d.logger.Error("Failed exporting instance", ctxMap) return nil, err } defer func() { _ = os.RemoveAll(tempDir) }() data, err := yaml.Dump(&meta, yaml.V2) if err != nil { _ = metaTarWriter.Close() if rootfsTarWriter != nil { _ = rootfsTarWriter.Close() } d.logger.Error("Failed exporting instance", ctxMap) return nil, err } fnam = filepath.Join(tempDir, "metadata.yaml") err = os.WriteFile(fnam, data, 0o644) if err != nil { _ = metaTarWriter.Close() if rootfsTarWriter != nil { _ = rootfsTarWriter.Close() } d.logger.Error("Failed exporting instance", ctxMap) return nil, err } // Add metadata.yaml to the tarball. fi, err := os.Lstat(fnam) if err != nil { _ = metaTarWriter.Close() if rootfsTarWriter != nil { _ = rootfsTarWriter.Close() } d.logger.Error("Failed exporting instance", ctxMap) return nil, err } tmpOffset := len(filepath.Dir(fnam)) + 1 err = metaTarWriter.WriteFile(fnam[tmpOffset:], fnam, fi, false) if err != nil { _ = metaTarWriter.Close() if rootfsTarWriter != nil { _ = rootfsTarWriter.Close() } d.logger.Debug("Error writing to tarfile", logger.Ctx{"err": err}) d.logger.Error("Failed exporting instance", ctxMap) return nil, err } // If present, add config.json (OCI) to the tarball. fnam = filepath.Join(d.Path(), "config.json") if util.PathExists(fnam) { fi, err := os.Lstat(fnam) if err != nil { _ = metaTarWriter.Close() if rootfsTarWriter != nil { _ = rootfsTarWriter.Close() } d.logger.Error("Failed exporting instance", ctxMap) return nil, err } tmpOffset := len(filepath.Dir(fnam)) + 1 err = metaTarWriter.WriteFile(fnam[tmpOffset:], fnam, fi, false) if err != nil { _ = metaTarWriter.Close() if rootfsTarWriter != nil { _ = rootfsTarWriter.Close() } d.logger.Debug("Error writing to tarfile", logger.Ctx{"err": err}) d.logger.Error("Failed exporting instance", ctxMap) return nil, err } } // Include all the rootfs files. fnam = d.RootfsPath() if rootfsWriter == nil { err = filepath.Walk(fnam, writeToMetaTar) if err != nil { d.logger.Error("Failed exporting instance", ctxMap) return nil, err } } else { err = filepath.Walk(fnam, writeToRootfsTar) if err != nil { d.logger.Error("Failed exporting instance", ctxMap) return nil, err } } // Include all the templates. fnam = d.TemplatesPath() if util.PathExists(fnam) { err = filepath.Walk(fnam, writeToMetaTar) if err != nil { d.logger.Error("Failed exporting instance", ctxMap) return nil, err } } err = metaTarWriter.Close() if err != nil { d.logger.Error("Failed exporting instance", ctxMap) return nil, err } if rootfsTarWriter != nil { err = rootfsTarWriter.Close() if err != nil { d.logger.Error("Failed exporting instance", ctxMap) return nil, err } } d.logger.Info("Exported instance", ctxMap) return &meta, nil } func collectCRIULogFile(d instance.Instance, imagesDir string, function string, method string) error { t := time.Now().Format(time.RFC3339) newPath := filepath.Join(d.LogPath(), fmt.Sprintf("%s_%s_%s.log", function, method, t)) return internalUtil.FileCopy(filepath.Join(imagesDir, fmt.Sprintf("%s.log", method)), newPath) } func getCRIULogErrors(imagesDir string, method string) (string, error) { f, err := os.Open(path.Join(imagesDir, fmt.Sprintf("%s.log", method))) if err != nil { return "", err } defer func() { _ = f.Close() }() scanner := bufio.NewScanner(f) ret := []string{} for scanner.Scan() { line := scanner.Text() if strings.Contains(line, "Error") || strings.Contains(line, "Warn") { ret = append(ret, scanner.Text()) } } return strings.Join(ret, "\n"), nil } // Check if CRIU supports pre-dumping and number of pre-dump iterations. func (d *lxc) migrationSendCheckForPreDumpSupport() (bool, int) { // Check if this architecture/kernel/criu combination supports pre-copy dirty memory tracking feature. _, err := subprocess.RunCommand("criu", "check", "--feature", "mem_dirty_track") if err != nil { // CRIU says it does not know about dirty memory tracking. // This means the rest of this function is irrelevant. return false, 0 } // CRIU says it can actually do pre-dump. Let's set it to true // unless the user wants something else. usePreDumps := true // What does the configuration say about pre-copy tmp := d.ExpandedConfig()["migration.incremental.memory"] if tmp != "" { usePreDumps = util.IsTrue(tmp) } var maxIterations int // migration.incremental.memory.iterations is the value after which the // container will be definitely migrated, even if the remaining number // of memory pages is below the defined threshold. tmp = d.ExpandedConfig()["migration.incremental.memory.iterations"] if tmp != "" { maxIterations, _ = strconv.Atoi(tmp) } else { // default to 10 maxIterations = 10 } if maxIterations > 999 { // the pre-dump directory is hardcoded to a string // with maximal 3 digits. 999 pre-dumps makes no // sense at all, but let's make sure the number // is not higher than this. maxIterations = 999 } logger.Debugf("Using maximal %d iterations for pre-dumping", maxIterations) return usePreDumps, maxIterations } func (d *lxc) migrationSendWriteActionScript(directory string, operation string, secret string, execPath string) error { script := fmt.Sprintf(`#!/bin/sh -e if [ "$CRTOOLS_SCRIPT_ACTION" = "post-dump" ]; then %s migratedumpsuccess %s %s fi `, execPath, operation, secret) f, err := os.Create(filepath.Join(directory, "action.sh")) if err != nil { return err } err = f.Chmod(0o500) if err != nil { return err } _, err = f.WriteString(script) if err != nil { return err } return f.Close() } func (d *lxc) MigrateSend(args instance.MigrateSendArgs) error { d.logger.Debug("Migration send starting") defer d.logger.Debug("Migration send stopped") // Check for an existing operation. // Setup a new operation. op := operationlock.Get(d.Project().Name, d.Name()) if op != nil && op.ActionMatch(operationlock.ActionMigrate) { return errors.New("The instance is already being migrated") } op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionMigrate, nil, false, true) if err != nil { return err } // If not running, stop any forkfile instance. if !d.IsRunning() { d.stopForkfile(false) } // Wait for essential migration connections before negotiation. connectionsCtx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() filesystemConn, err := args.FilesystemConn(connectionsCtx) if err != nil { op.Done(err) return err } var stateConn io.ReadWriteCloser if args.Live { stateConn, err = args.StateConn(connectionsCtx) if err != nil { op.Done(err) return err } } pool, err := storagePools.LoadByInstance(d.state, d) if err != nil { err := fmt.Errorf("Failed loading instance: %w", err) op.Done(err) return err } clusterMove := args.ClusterMoveSourceName != "" storageMove := args.StoragePool != "" // The refresh argument passed to MigrationTypes() is always set to false here. // The migration source/sender doesn't need to care whether or not it's doing a refresh as the migration // sink/receiver will know this, and adjust the migration types accordingly. // The same applies for clusterMove and storageMove, which are set to the most optimized defaults. poolMigrationTypes := pool.MigrationTypes(storagePools.InstanceContentType(d), false, args.Snapshots, true, false) if len(poolMigrationTypes) == 0 { err := errors.New("No source migration types available") op.Done(err) return err } // Convert the pool's migration type options to an offer header to target. // Populate the Fs, ZfsFeatures and RsyncFeatures fields. offerHeader := localMigration.TypesToHeader(poolMigrationTypes...) // Offer to send index header. indexHeaderVersion := localMigration.IndexHeaderVersion offerHeader.IndexHeaderVersion = &indexHeaderVersion // Add CRIU and predump info to source header. maxDumpIterations := 0 if args.Live { var offerUsePreDumps bool offerUsePreDumps, maxDumpIterations = d.migrationSendCheckForPreDumpSupport() offerHeader.Predump = proto.Bool(offerUsePreDumps) offerHeader.Criu = migration.CRIUType_CRIU_RSYNC.Enum() } else { offerHeader.Predump = proto.Bool(false) if d.IsRunning() { // Indicate instance is running to target (can trigger MultiSync mode). offerHeader.Criu = migration.CRIUType_NONE.Enum() } } // Add idmap info to source header for containers. idmapset, err := d.DiskIdmap() if err != nil { err := fmt.Errorf("Failed getting container disk idmap: %w", err) op.Done(err) return err } else if idmapset != nil { offerHeader.Idmap = make([]*migration.IDMapType, 0, len(idmapset.Entries)) for _, ctnIdmap := range idmapset.Entries { idmap := migration.IDMapType{ Isuid: proto.Bool(ctnIdmap.IsUID), Isgid: proto.Bool(ctnIdmap.IsGID), Hostid: proto.Int32(int32(ctnIdmap.HostID)), Nsid: proto.Int32(int32(ctnIdmap.NSID)), Maprange: proto.Int32(int32(ctnIdmap.MapRange)), } offerHeader.Idmap = append(offerHeader.Idmap, &idmap) } } srcConfig, err := pool.GenerateInstanceBackupConfig(d, args.Snapshots, true, d.op) if err != nil { err := fmt.Errorf("Failed generating instance migration config: %w", err) op.Done(err) return err } dependentVolumesOffer, err := storagePools.GenerateDependentVolumesOffer(d.state, srcConfig, d.Project().Name, args.Snapshots, args.Devices, args.ClusterMoveSourceName != "") if err != nil { err := fmt.Errorf("Failed generating instance depending volumes offer: %w", err) op.Done(err) return err } offerHeader.DependentVolumes = dependentVolumesOffer // If we are copying snapshots, retrieve a list of snapshots from source volume. if args.Snapshots { offerHeader.SnapshotNames = make([]string, 0, len(srcConfig.Snapshots)) offerHeader.Snapshots = make([]*migration.Snapshot, 0, len(srcConfig.Snapshots)) for i := range srcConfig.Snapshots { offerHeader.SnapshotNames = append(offerHeader.SnapshotNames, srcConfig.Snapshots[i].Name) offerHeader.Snapshots = append(offerHeader.Snapshots, instance.SnapshotToProtobuf(srcConfig.Snapshots[i])) } } // Send offer to target. d.logger.Debug("Sending migration offer to target") err = args.ControlSend(offerHeader) if err != nil { err := fmt.Errorf("Failed sending migration offer: %w", err) op.Done(err) return err } // Receive response from target. d.logger.Debug("Waiting for migration offer response from target") respHeader := &migration.MigrationHeader{} err = args.ControlReceive(respHeader, true) if err != nil { err := fmt.Errorf("Failed receiving migration offer response: %w", err) op.Done(err) return err } d.logger.Debug("Got migration offer response from target") // Negotiated migration types. migrationTypes, err := localMigration.MatchTypes(respHeader, migration.MigrationFSType_RSYNC, poolMigrationTypes) if err != nil { err := fmt.Errorf("Failed to negotiate migration type: %w", err) op.Done(err) return err } volumesWithTypes, err := storagePools.DependentVolumesMatchMigrationType(d.state, respHeader.DependentVolumes, args.Snapshots, nil, true) if err != nil { err := fmt.Errorf("Failed to negotiate migration types for dependent volumes: %w", err) op.Done(err) return err } d.logger.Debug("Generate dependent volumes args") dependentVolumes := []localMigration.DependentVolumeArgs{} for _, volWithType := range volumesWithTypes { dependentVolumes = append(dependentVolumes, localMigration.ProtobufToDependentVolume(volWithType.Volume, volWithType.VolumeTypes[0], nil)) } volSourceArgs := &localMigration.VolumeSourceArgs{ IndexHeaderVersion: respHeader.GetIndexHeaderVersion(), // Enable index header frame if supported. Name: d.Name(), MigrationType: migrationTypes[0], Snapshots: offerHeader.SnapshotNames, TrackProgress: true, Refresh: respHeader.GetRefresh(), AllowInconsistent: args.AllowInconsistent, VolumeOnly: !args.Snapshots, Info: &localMigration.Info{Config: srcConfig}, ClusterMove: clusterMove, StorageMove: storageMove, DependentVolumes: dependentVolumes, } // Only send the snapshots that the target requests when refreshing. if respHeader.GetRefresh() { volSourceArgs.Snapshots = respHeader.GetSnapshotNames() allSnapshots := volSourceArgs.Info.Config.VolumeSnapshots // Ensure that only the requested snapshots are included in the migration index header. volSourceArgs.Info.Config.VolumeSnapshots = make([]*api.StorageVolumeSnapshot, 0, len(volSourceArgs.Snapshots)) for i := range allSnapshots { if slices.Contains(volSourceArgs.Snapshots, allSnapshots[i].Name) { volSourceArgs.Info.Config.VolumeSnapshots = append(volSourceArgs.Info.Config.VolumeSnapshots, allSnapshots[i]) } } } // If s.live is true or Criu is set to CRIUType_NONE rather than nil, it indicates that the source instance // is running, and if we are doing a non-optimized transfer (i.e using rsync or raw block transfer) then we // should do a two stage transfer to minimize downtime. instanceRunning := args.Live || (respHeader.Criu != nil && *respHeader.Criu == migration.CRIUType_NONE) nonOptimizedMigration := volSourceArgs.MigrationType.FSType == migration.MigrationFSType_RSYNC || volSourceArgs.MigrationType.FSType == migration.MigrationFSType_BLOCK_AND_RSYNC if instanceRunning && nonOptimizedMigration { // Indicate this info to the storage driver so that it can alter its behaviour if needed. volSourceArgs.MultiSync = true } g, ctx := errgroup.WithContext(context.Background()) // Start control connection monitor. g.Go(func() error { d.logger.Debug("Migrate send control monitor started") defer d.logger.Debug("Migrate send control monitor finished") controlResult := make(chan error, 1) // Buffered to allow go routine to end if no readers. // This will read the result message from the target side and detect disconnections. go func() { resp := migration.MigrationControl{} err := args.ControlReceive(&resp, false) if err != nil { err = fmt.Errorf("Error reading migration control target: %w", err) } else if !resp.GetSuccess() { err = fmt.Errorf("Error from migration control target: %s", resp.GetMessage()) } controlResult <- err }() // End as soon as we get control message/disconnection from the target side or a local error. select { case <-ctx.Done(): err = ctx.Err() case err = <-controlResult: } return err }) // Start error monitoring routine, this will detect when an error is returned from the other routines, // and if that happens it will disconnect the migration connections which will trigger the other routines // to finish. go func() { <-ctx.Done() args.Disconnect() }() restoreSuccess := make(chan bool, 1) defer close(restoreSuccess) // Don't defer close this one as its needed potentially after this function has ended. dumpSuccess := make(chan error, 1) g.Go(func() error { d.logger.Debug("Migrate send transfer started") defer d.logger.Debug("Migrate send transfer finished") var err error d.logger.Debug("Starting storage migration phase") err = pool.MigrateInstance(d, filesystemConn, volSourceArgs, d.op) if err != nil { return err } d.logger.Debug("Finished storage migration phase") if args.Live { d.logger.Debug("Starting live migration phase") // Setup rsync options (used for CRIU state transfers). rsyncBwlimit := pool.Driver().Config()["rsync.bwlimit"] rsyncFeatures := respHeader.GetRsyncFeaturesSlice() if !slices.Contains(rsyncFeatures, "bidirectional") { // If no bi-directional support, assume 3.7 level. // NOTE: Do NOT extend this list of arguments. rsyncFeatures = []string{"xattrs", "delete", "compress"} } if respHeader.Criu == nil { return errors.New("Got no CRIU socket type for live migration") } else if *respHeader.Criu != migration.CRIUType_CRIU_RSYNC { return fmt.Errorf("Formats other than criu rsync not understood (%q)", respHeader.Criu) } checkpointDir, err := os.MkdirTemp("", "incus_checkpoint_") if err != nil { return err } if liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 2, 0, 4) { // What happens below is slightly convoluted. Due to various complications // with networking, there's no easy way for criu to exit and leave the // container in a frozen state for us to somehow resume later. // Instead, we use what criu calls an "action-script", which is basically a // callback that lets us know when the dump is done. (Unfortunately, we // can't pass arguments, just an executable path, so we write a custom // action script with the real command we want to run.) // This script then blocks until the migration operation either finishes // successfully or fails, and exits 1 or 0, which causes criu to either // leave the container running or kill it as we asked. dumpDone := make(chan bool, 1) actionScriptOpSecret, err := internalUtil.RandomHexString(32) if err != nil { _ = os.RemoveAll(checkpointDir) return err } actionScriptOp, err := operations.OperationCreate( d.state, d.Project().Name, operations.OperationClassWebsocket, operationtype.InstanceLiveMigrate, nil, nil, func(op *operations.Operation) error { result := <-restoreSuccess if !result { return errors.New("restore failed, failing CRIU") } return nil }, nil, func(op *operations.Operation, r *http.Request, w http.ResponseWriter) error { secret := r.FormValue("secret") if secret == "" { return errors.New("Missing action script secret") } if secret != actionScriptOpSecret { return os.ErrPermission } c, err := ws.Upgrader.Upgrade(w, r, nil) if err != nil { return err } dumpDone <- true closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") return c.WriteMessage(websocket.CloseMessage, closeMsg) }, nil, ) if err != nil { _ = os.RemoveAll(checkpointDir) return err } err = d.migrationSendWriteActionScript(checkpointDir, actionScriptOp.URL(), actionScriptOpSecret, d.state.OS.ExecPath) if err != nil { _ = os.RemoveAll(checkpointDir) return err } preDumpCounter := 0 preDumpDir := "" // Check if the other side knows about pre-dumping and the associated // rsync protocol. if respHeader.GetPredump() { d.logger.Debug("The other side does support pre-copy") final := false for !final { preDumpCounter++ if preDumpCounter < maxDumpIterations { final = false } else { final = true } dumpDir := fmt.Sprintf("%03d", preDumpCounter) loopArgs := preDumpLoopArgs{ stateConn: stateConn, checkpointDir: checkpointDir, bwlimit: rsyncBwlimit, preDumpDir: preDumpDir, dumpDir: dumpDir, final: final, rsyncFeatures: rsyncFeatures, } final, err = d.migrateSendPreDumpLoop(&loopArgs) if err != nil { _ = os.RemoveAll(checkpointDir) return err } preDumpDir = fmt.Sprintf("%03d", preDumpCounter) preDumpCounter++ } } else { d.logger.Debug("The other side does not support pre-copy") } err = actionScriptOp.Start() if err != nil { _ = os.RemoveAll(checkpointDir) return err } go func() { d.logger.Debug("Final CRIU dump started") defer d.logger.Debug("Final CRIU dump stopped") criuMigrationArgs := instance.CriuMigrationArgs{ Cmd: liblxc.MIGRATE_DUMP, Stop: true, ActionScript: true, PreDumpDir: preDumpDir, DumpDir: "final", StateDir: checkpointDir, Function: "migration", } // Do the final CRIU dump. This is needs no special handling if // pre-dumps are used or not. dumpSuccess <- d.migrate(&criuMigrationArgs) _ = os.RemoveAll(checkpointDir) }() select { // The checkpoint failed, let's just abort. case err = <-dumpSuccess: return err // The dump finished, let's continue on to the restore. case <-dumpDone: d.logger.Debug("Dump finished, continuing with restore...") } } else { d.logger.Debug("The version of liblxc is older than 2.0.4 and the live migration will probably fail") defer func() { _ = os.RemoveAll(checkpointDir) }() criuMigrationArgs := instance.CriuMigrationArgs{ Cmd: liblxc.MIGRATE_DUMP, StateDir: checkpointDir, Function: "migration", Stop: true, ActionScript: false, DumpDir: "final", PreDumpDir: "", } err = d.migrate(&criuMigrationArgs) if err != nil { return err } } // We do the transfer serially right now, but there's really no reason for us to; // since we have separate websockets, we can do it in parallel if we wanted to. // However assuming we're network bound, there's really no reason to do these in. // parallel. In the future when we're using p.haul's protocol, it will make sense // to do these in parallel. ctName, _, _ := api.GetParentAndSnapshotName(d.Name()) err = rsync.Send(ctName, internalUtil.AddSlash(checkpointDir), stateConn, nil, rsyncFeatures, rsyncBwlimit, d.state.OS.ExecPath) if err != nil { return err } d.logger.Debug("Finished live migration phase") } // Perform final sync if in multi sync mode. if volSourceArgs.MultiSync { d.logger.Debug("Starting final storage migration phase") // Indicate to the storage driver we are doing final sync and because of this don't send // snapshots as they don't need to have a final sync as not being modified. volSourceArgs.FinalSync = true volSourceArgs.Snapshots = nil volSourceArgs.Info.Config.VolumeSnapshots = nil err = pool.MigrateInstance(d, filesystemConn, volSourceArgs, d.op) if err != nil { return err } d.logger.Debug("Finished final storage migration phase") } return nil }) { // Wait for routines to finish and collect first error. err := g.Wait() if args.Live { restoreSuccess <- err == nil if err == nil { err := <-dumpSuccess if err != nil { d.logger.Error("Dump failed after successful restore", logger.Ctx{"err": err}) } } } if err != nil { op.Done(err) return err } op.Done(nil) d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceMigrated.Event(d, nil)) return nil } } type preDumpLoopArgs struct { stateConn io.ReadWriteCloser checkpointDir string bwlimit string preDumpDir string dumpDir string final bool rsyncFeatures []string } // migrateSendPreDumpLoop is the main logic behind the pre-copy migration. // This function contains the actual pre-dump, the corresponding rsync transfer and it tells the outer loop to // abort if the threshold of memory pages transferred by pre-dumping has been reached. func (d *lxc) migrateSendPreDumpLoop(args *preDumpLoopArgs) (bool, error) { // Do a CRIU pre-dump criuMigrationArgs := instance.CriuMigrationArgs{ Cmd: liblxc.MIGRATE_PRE_DUMP, Stop: false, ActionScript: false, PreDumpDir: args.preDumpDir, DumpDir: args.dumpDir, StateDir: args.checkpointDir, Function: "migration", } d.logger.Debug("Doing another CRIU pre-dump", logger.Ctx{"preDumpDir": args.preDumpDir}) final := args.final if d.Type() != instancetype.Container { return false, errors.New("Instance is not container type") } err := d.migrate(&criuMigrationArgs) if err != nil { return final, fmt.Errorf("Failed sending instance: %w", err) } // Send the pre-dump. ctName, _, _ := api.GetParentAndSnapshotName(d.Name()) err = rsync.Send(ctName, internalUtil.AddSlash(args.checkpointDir), args.stateConn, nil, args.rsyncFeatures, args.bwlimit, d.state.OS.ExecPath) if err != nil { return final, err } // The function readCriuStatsDump() reads the CRIU 'stats-dump' file // in path and returns the pages_written, pages_skipped_parent, error. readCriuStatsDump := func(path string) (uint64, uint64, error) { // Get dump statistics with crit dumpStats, err := crit.GetDumpStats(path) if err != nil { return 0, 0, fmt.Errorf("Failed to parse CRIU's 'stats-dump' file: %w", err) } return dumpStats.GetPagesWritten(), dumpStats.GetPagesSkippedParent(), nil } // Read the CRIU's 'stats-dump' file dumpPath := internalUtil.AddSlash(args.checkpointDir) dumpPath += internalUtil.AddSlash(args.dumpDir) written, skippedParent, err := readCriuStatsDump(dumpPath) if err != nil { return final, err } totalPages := written + skippedParent var percentageSkipped int if totalPages > 0 { percentageSkipped = int(100 - ((100 * written) / totalPages)) } d.logger.Debug("CRIU pages", logger.Ctx{"pages": written, "skipped": skippedParent, "skippedPerc": percentageSkipped}) // threshold is the percentage of memory pages that needs // to be pre-copied for the pre-copy migration to stop. var threshold int tmp := d.ExpandedConfig()["migration.incremental.memory.goal"] if tmp != "" { threshold, _ = strconv.Atoi(tmp) } else { // defaults to 70% threshold = 70 } if percentageSkipped > threshold { d.logger.Debug("Memory pages skipped due to pre-copy is larger than threshold", logger.Ctx{"skippedPerc": percentageSkipped, "thresholdPerc": threshold}) d.logger.Debug("This was the last pre-dump; next dump is the final dump") final = true } // If in pre-dump mode, the receiving side expects a message to know if this was the last pre-dump. logger.Debug("Sending another CRIU pre-dump header") sync := migration.MigrationSync{ FinalPreDump: proto.Bool(final), } data, err := proto.Marshal(&sync) if err != nil { return false, err } _, err = args.stateConn.Write(data) if err != nil { return final, err } d.logger.Debug("Sending another CRIU pre-dump header done") return final, nil } func (d *lxc) resetContainerDiskIdmap(srcIdmap *idmap.Set) error { dstIdmap, err := d.DiskIdmap() if err != nil { return err } if dstIdmap == nil { dstIdmap = &idmap.Set{} } if !srcIdmap.Equals(dstIdmap) { jsonIdmap, err := srcIdmap.ToJSON() if err != nil { return fmt.Errorf("Failed to encode ID map: %w", err) } d.logger.Debug("Setting new volatile.last_state.idmap from source instance", logger.Ctx{"sourceIdmap": srcIdmap}) err = d.VolatileSet(map[string]string{"volatile.last_state.idmap": jsonIdmap}) if err != nil { return err } } return nil } func (d *lxc) MigrateReceive(args instance.MigrateReceiveArgs) error { d.logger.Debug("Migration receive starting") defer d.logger.Debug("Migration receive stopped") // Wait for essential migration connections before negotiation. connectionsCtx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() filesystemConn, err := args.FilesystemConn(connectionsCtx) if err != nil { return err } var stateConn io.ReadWriteCloser if args.Live { stateConn, err = args.StateConn(connectionsCtx) if err != nil { return err } } // Receive offer from source. d.logger.Debug("Waiting for migration offer from source") offerHeader := &migration.MigrationHeader{} err = args.ControlReceive(offerHeader, true) if err != nil { return fmt.Errorf("Failed receiving migration offer from source: %w", err) } criuType := migration.CRIUType_CRIU_RSYNC.Enum() if offerHeader.Criu != nil && *offerHeader.Criu == migration.CRIUType_NONE { criuType = migration.CRIUType_NONE.Enum() } else { if !args.Live { criuType = nil } } // When doing a cluster same-name move we cannot load the storage pool using the instance's volume DB // record because it may be associated to the wrong cluster member. Instead we ascertain the pool to load // using the instance's root disk device. if args.ClusterMoveSourceName == d.name { _, rootDiskDevice, err := d.getRootDiskDevice() if err != nil { return fmt.Errorf("Failed getting root disk: %w", err) } if rootDiskDevice["pool"] == "" { return errors.New("The instance's root device is missing the pool property") } // Initialize the storage pool cache. d.storagePool, err = storagePools.LoadByName(d.state, rootDiskDevice["pool"]) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } } pool, err := storagePools.LoadByInstance(d.state, d) if err != nil { return err } // The source will never set Refresh in the offer header. // However, to determine the correct migration type Refresh needs to be set. offerHeader.Refresh = &args.Refresh clusterMove := args.ClusterMoveSourceName != "" storageMove := args.StoragePool != "" // Extract the source's migration type and then match it against our pool's supported types and features. // If a match is found the combined features list will be sent back to requester. contentType := storagePools.InstanceContentType(d) respTypes, err := localMigration.MatchTypes(offerHeader, storagePools.FallbackMigrationType(contentType), pool.MigrationTypes(contentType, args.Refresh, args.Snapshots, clusterMove, storageMove)) if err != nil { return err } // The migration header to be sent back to source with our target options. // Convert response type to response header and copy snapshot info into it. respHeader := localMigration.TypesToHeader(respTypes...) // Respond with our maximum supported header version if the requested version is higher than ours. // Otherwise just return the requested header version to the source. indexHeaderVersion := min(offerHeader.GetIndexHeaderVersion(), localMigration.IndexHeaderVersion) respHeader.IndexHeaderVersion = &indexHeaderVersion respHeader.SnapshotNames = offerHeader.SnapshotNames respHeader.Snapshots = offerHeader.Snapshots respHeader.Refresh = &args.Refresh // Add CRIU info to response. respHeader.Criu = criuType localDevices := d.LocalDevices().CloneNative() volumesWithTypes, err := storagePools.DependentVolumesMatchMigrationType(d.state, offerHeader.DependentVolumes, args.Snapshots, localDevices, false) if err != nil { return fmt.Errorf("Failed to negotiate migration types for dependent volumes: %w", err) } dependentVolumes := []localMigration.DependentVolumeArgs{} for _, volWithType := range volumesWithTypes { respHeader.DependentVolumes = append(respHeader.DependentVolumes, volWithType.Volume) vol := localMigration.ProtobufToDependentVolume(volWithType.Volume, volWithType.VolumeTypes[0], localDevices[*volWithType.Volume.DeviceName]) dependentVolumes = append(dependentVolumes, vol) } if args.Refresh { // Get the remote snapshots on the source. sourceSnapshots := offerHeader.GetSnapshots() sourceSnapshotComparable := make([]storagePools.ComparableSnapshot, 0, len(sourceSnapshots)) for _, sourceSnap := range sourceSnapshots { sourceSnapshotComparable = append(sourceSnapshotComparable, storagePools.ComparableSnapshot{ Name: sourceSnap.GetName(), CreationDate: time.Unix(sourceSnap.GetCreationDate(), 0), }) } // Get existing snapshots on the local target. targetSnapshots, err := d.Snapshots() if err != nil { return err } targetSnapshotsComparable := make([]storagePools.ComparableSnapshot, 0, len(targetSnapshots)) for _, targetSnap := range targetSnapshots { _, targetSnapName, _ := api.GetParentAndSnapshotName(targetSnap.Name()) targetSnapshotsComparable = append(targetSnapshotsComparable, storagePools.ComparableSnapshot{ Name: targetSnapName, CreationDate: targetSnap.CreationDate(), }) } // Compare the two sets. syncSourceSnapshotIndexes, deleteTargetSnapshotIndexes := storagePools.CompareSnapshots(sourceSnapshotComparable, targetSnapshotsComparable, args.RefreshExcludeOlder) // Delete the extra local snapshots first. for _, deleteTargetSnapshotIndex := range deleteTargetSnapshotIndexes { err := targetSnapshots[deleteTargetSnapshotIndex].Delete(true, true) if err != nil { return err } } // Only request to send the snapshots that need updating. syncSnapshotNames := make([]string, 0, len(syncSourceSnapshotIndexes)) syncSnapshots := make([]*migration.Snapshot, 0, len(syncSourceSnapshotIndexes)) for _, syncSourceSnapshotIndex := range syncSourceSnapshotIndexes { syncSnapshotNames = append(syncSnapshotNames, sourceSnapshots[syncSourceSnapshotIndex].GetName()) syncSnapshots = append(syncSnapshots, sourceSnapshots[syncSourceSnapshotIndex]) } respHeader.Snapshots = syncSnapshots respHeader.SnapshotNames = syncSnapshotNames offerHeader.Snapshots = syncSnapshots offerHeader.SnapshotNames = syncSnapshotNames } if offerHeader.GetPredump() { // If the other side wants pre-dump and if this side supports it, let's use it. respHeader.Predump = proto.Bool(true) } else { respHeader.Predump = proto.Bool(false) } // Get rsync options from sender, these are passed into mySink function as part of // MigrationSinkArgs below. rsyncFeatures := respHeader.GetRsyncFeaturesSlice() // Send response to source. d.logger.Debug("Sending migration response to source") err = args.ControlSend(respHeader) if err != nil { return fmt.Errorf("Failed sending migration response to source: %w", err) } d.logger.Debug("Sent migration response to source") srcIdmap := &idmap.Set{} for _, idmapSet := range offerHeader.Idmap { e := idmap.Entry{ IsUID: *idmapSet.Isuid, IsGID: *idmapSet.Isgid, NSID: int64(*idmapSet.Nsid), HostID: int64(*idmapSet.Hostid), MapRange: int64(*idmapSet.Maprange), } srcIdmap.Entries = append(srcIdmap.Entries, e) } reverter := revert.New() defer reverter.Fail() g, ctx := errgroup.WithContext(context.Background()) // Start control connection monitor. g.Go(func() error { d.logger.Debug("Migrate receive control monitor started") defer d.logger.Debug("Migrate receive control monitor finished") controlResult := make(chan error, 1) // Buffered to allow go routine to end if no readers. // This will read the result message from the source side and detect disconnections. go func() { resp := migration.MigrationControl{} err := args.ControlReceive(&resp, false) if err != nil { err = fmt.Errorf("Error reading migration control source: %w", err) } else if !resp.GetSuccess() { err = fmt.Errorf("Error from migration control source: %s", resp.GetMessage()) } controlResult <- err }() // End as soon as we get control message/disconnection from the source side or a local error. select { case <-ctx.Done(): err = ctx.Err() case err = <-controlResult: } return err }) // Start error monitoring routine, this will detect when an error is returned from the other routines, // and if that happens it will disconnect the migration connections which will trigger the other routines // to finish. go func() { <-ctx.Done() args.Disconnect() }() // Start filesystem transfer routine and initialize a channel that is closed when the routine finishes. fsTransferDone := make(chan struct{}) g.Go(func() error { defer close(fsTransferDone) d.logger.Debug("Migrate receive filesystem transfer started") defer d.logger.Debug("Migrate receive filesystem transfer finished") var err error // We do the fs receive in parallel so we don't have to reason about when to receive // what. The sending side is smart enough to send the filesystem bits that it can // before it seizes the container to start checkpointing, so the total transfer time // will be minimized even if we're dumb here. snapshots := []*migration.Snapshot{} // Legacy: we only sent the snapshot names, so we just copy the container's // config over, same as we used to do. if len(offerHeader.SnapshotNames) != len(offerHeader.Snapshots) { // Convert the instance to an api.InstanceSnapshot. profileNames := make([]string, 0, len(d.Profiles())) for _, p := range d.Profiles() { profileNames = append(profileNames, p.Name) } architectureName, _ := osarch.ArchitectureName(d.Architecture()) apiInstSnap := &api.InstanceSnapshot{ InstanceSnapshotPut: api.InstanceSnapshotPut{ ExpiresAt: time.Time{}, }, Architecture: architectureName, CreatedAt: d.CreationDate(), LastUsedAt: d.LastUsedDate(), Config: d.LocalConfig(), Description: d.Description(), Devices: d.LocalDevices().CloneNative(), Ephemeral: d.IsEphemeral(), Stateful: d.IsStateful(), Profiles: profileNames, } for _, name := range offerHeader.SnapshotNames { base := instance.SnapshotToProtobuf(apiInstSnap) base.Name = &name snapshots = append(snapshots, base) } } else { snapshots = offerHeader.Snapshots } // Default to not expecting to receive the final rootfs sync. sendFinalFsDelta := false // If we are doing a stateful live transfer or the CRIU type indicates we // are doing a stateless transfer with a running instance then we should // expect the source to send us a final rootfs sync. if args.Live { sendFinalFsDelta = true } else if criuType != nil && *criuType == migration.CRIUType_NONE { sendFinalFsDelta = true } volTargetArgs := localMigration.VolumeTargetArgs{ IndexHeaderVersion: respHeader.GetIndexHeaderVersion(), Name: d.Name(), MigrationType: respTypes[0], Refresh: args.Refresh, // Indicate to receiver volume should exist. TrackProgress: true, // Use a progress tracker on receiver to get in-cluster progress information. Live: sendFinalFsDelta, // Indicates we will get a final rootfs sync. VolumeSize: offerHeader.GetVolumeSize(), // Block size setting override. VolumeOnly: !args.Snapshots, ClusterMoveSourceName: args.ClusterMoveSourceName, StoragePool: args.StoragePool, DependentVolumes: dependentVolumes, } // At this point we have already figured out the parent container's root // disk device so we can simply retrieve it from the expanded devices. parentStoragePool := "" parentExpandedDevices := d.ExpandedDevices() parentLocalRootDiskDeviceKey, parentLocalRootDiskDevice, _ := internalInstance.GetRootDiskDevice(parentExpandedDevices.CloneNative()) if parentLocalRootDiskDeviceKey != "" { parentStoragePool = parentLocalRootDiskDevice["pool"] } if parentStoragePool == "" { return errors.New("Instance's root device is missing the pool property") } // A zero length Snapshots slice indicates volume only migration in // VolumeTargetArgs. So if VolumeOnly was requested, do not populate them. if args.Snapshots { volTargetArgs.Snapshots = make([]*migration.Snapshot, 0, len(snapshots)) for _, snap := range snapshots { volTargetArgs.Snapshots = append(volTargetArgs.Snapshots, &migration.Snapshot{Name: snap.Name}) // Only create snapshot instance DB records if not doing a cluster same-name move. // As otherwise the DB records will already exist. if args.ClusterMoveSourceName != d.name { snapArgs, err := instance.SnapshotProtobufToInstanceArgs(d.state, d, snap) if err != nil { return err } // Ensure that snapshot and parent container have the same // storage pool in their local root disk device. If the root // disk device for the snapshot comes from a profile on the // new instance as well we don't need to do anything. if snapArgs.Devices != nil { snapLocalRootDiskDeviceKey, _, _ := internalInstance.GetRootDiskDevice(snapArgs.Devices.CloneNative()) if snapLocalRootDiskDeviceKey != "" { snapArgs.Devices[snapLocalRootDiskDeviceKey]["pool"] = parentStoragePool } } // Create the snapshot instance. _, snapInstOp, cleanup, err := instance.CreateInternal(d.state, *snapArgs, d.op, true, false, false) if err != nil { return fmt.Errorf("Failed creating instance snapshot record %q: %w", snapArgs.Name, err) } reverter.Add(cleanup) defer snapInstOp.Done(err) } } } err = pool.CreateInstanceFromMigration(d, filesystemConn, volTargetArgs, d.op) if err != nil { return fmt.Errorf("Failed creating instance on target: %w", err) } isRemoteClusterMove := clusterMove && pool.Driver().Info().Remote // Only delete all instance volumes on error if the pool volume creation has succeeded to // avoid deleting an existing conflicting volume. if !volTargetArgs.Refresh && !isRemoteClusterMove { reverter.Add(func() { snapshots, _ := d.Snapshots() snapshotCount := len(snapshots) for k := range snapshots { // Delete the snapshots in reverse order. k = snapshotCount - 1 - k _ = pool.DeleteInstanceSnapshot(snapshots[k], nil) } _ = pool.DeleteInstance(d, nil) }) } // For containers, the fs map of the source is sent as part of the migration // stream, then at the end we need to record that map as last_state so that // shifting can happen on startup if needed. err = d.resetContainerDiskIdmap(srcIdmap) if err != nil { return err } if args.ClusterMoveSourceName != d.name { err = d.DeferTemplateApply(instance.TemplateTriggerCopy) if err != nil { return err } } return nil }) // Start live state transfer routine (if required) and initialize a channel that is closed when the // routine finishes. It is never closed if the routine is not started. stateTransferDone := make(chan struct{}) if args.Live { g.Go(func() error { d.logger.Debug("Migrate receive state transfer started") defer d.logger.Debug("Migrate receive state transfer finished") defer close(stateTransferDone) imagesDir, err := os.MkdirTemp("", "incus_restore_") if err != nil { return err } defer func() { _ = os.RemoveAll(imagesDir) }() sync := &migration.MigrationSync{ FinalPreDump: proto.Bool(false), } if respHeader.GetPredump() { for !sync.GetFinalPreDump() { d.logger.Debug("Waiting to receive pre-dump rsync") // Transfer a CRIU pre-dump. err = rsync.Recv(internalUtil.AddSlash(imagesDir), stateConn, nil, rsyncFeatures) if err != nil { return fmt.Errorf("Failed receiving pre-dump rsync: %w", err) } d.logger.Debug("Done receiving pre-dump rsync") d.logger.Debug("Waiting to receive pre-dump header") // We can't use io.ReadAll here because sender doesn't call Close() to // send the frame end indicator after writing the pre-dump header. // So define a small buffer sufficient to fit migration.MigrationSync and // then read what we have into it. buf := make([]byte, 128) n, err := stateConn.Read(buf) if err != nil { return fmt.Errorf("Failed receiving pre-dump header: %w", err) } err = proto.Unmarshal(buf[:n], sync) if err != nil { return fmt.Errorf("Failed unmarshalling pre-dump header: %w (%v)", err, string(buf)) } d.logger.Debug("Done receiving pre-dump header") } } // Final CRIU dump. d.logger.Debug("About to receive final dump rsync") err = rsync.Recv(internalUtil.AddSlash(imagesDir), stateConn, nil, rsyncFeatures) if err != nil { return fmt.Errorf("Failed receiving final dump rsync: %w", err) } d.logger.Debug("Done receiving final dump rsync") // Wait until filesystem transfer is done before starting final state sync and restore. <-fsTransferDone // But only proceed if no errors have occurred thus far. err = ctx.Err() if err != nil { return err } criuMigrationArgs := instance.CriuMigrationArgs{ Cmd: liblxc.MIGRATE_RESTORE, StateDir: imagesDir, Function: "migration", Stop: false, ActionScript: false, DumpDir: "final", PreDumpDir: "", } // Currently we only do a single CRIU pre-dump so we can hardcode "final" // here since we know that "final" is the folder for CRIU's final dump. err = d.migrate(&criuMigrationArgs) if err != nil { return err } return nil }) } { // Wait until the filesystem transfer and state transfer routines have finished. <-fsTransferDone if args.Live { <-stateTransferDone } // If context is cancelled by this stage, then an error has occurred. // Wait for all routines to finish and collect the first error that occurred. if ctx.Err() != nil { err := g.Wait() // Send failure response to source. msg := migration.MigrationControl{ Success: proto.Bool(err == nil), } if err != nil { msg.Message = proto.String(err.Error()) } d.logger.Debug("Sending migration failure response to source", logger.Ctx{"err": err}) sendErr := args.ControlSend(&msg) if sendErr != nil { d.logger.Warn("Failed sending migration failure to source", logger.Ctx{"err": sendErr}) } return err } // Send success response to source to control as nothing has gone wrong so far. msg := migration.MigrationControl{ Success: proto.Bool(true), } d.logger.Debug("Sending migration success response to source", logger.Ctx{"success": msg.GetSuccess()}) err := args.ControlSend(&msg) if err != nil { d.logger.Warn("Failed sending migration success to source", logger.Ctx{"err": err}) return fmt.Errorf("Failed sending migration success to source: %w", err) } // Wait for all routines to finish (in this case it will be the control monitor) but do // not collect the error, as it will just be a disconnect error from the source. _ = g.Wait() reverter.Success() return nil } } // Migrate migrates the instance to another node. func (d *lxc) migrate(args *instance.CriuMigrationArgs) error { ctxMap := logger.Ctx{ "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, "statedir": args.StateDir, "actionscript": args.ActionScript, "predumpdir": args.PreDumpDir, "features": args.Features, "stop": args.Stop, } _, err := exec.LookPath("criu") if err != nil { return localMigration.ErrNoLiveMigration } d.logger.Info("Migrating container", ctxMap) prettyCmd := "" switch args.Cmd { case liblxc.MIGRATE_PRE_DUMP: prettyCmd = "pre-dump" case liblxc.MIGRATE_DUMP: prettyCmd = "dump" case liblxc.MIGRATE_RESTORE: prettyCmd = "restore" default: prettyCmd = "unknown" d.logger.Warn("Unknown migrate call", logger.Ctx{"cmd": args.Cmd}) } pool, err := d.getStoragePool() if err != nil { return err } preservesInodes := pool.Driver().Info().PreservesInodes /* This feature was only added in 2.0.1, let's not ask for it * before then or migrations will fail. */ if !liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 2, 0, 1) { preservesInodes = false } finalStateDir := args.StateDir var migrateErr error /* For restore, we need an extra fork so that we daemonize the monitor * instead of having it be a child. So let's hijack the command * here and do the extra fork. */ if args.Cmd == liblxc.MIGRATE_RESTORE { // Check that we're not already running. if d.IsRunning() { return errors.New("The container is already running") } // Run the shared start code. configPath, postStartHooks, err := d.startCommon() if err != nil { if args.Op != nil { args.Op.Done(err) } return err } /* * For unprivileged containers we need to shift the * perms on the images images so that they can be * opened by the process after it is in its user * namespace. */ idmapset, err := d.CurrentIdmap() if err != nil { return err } if idmapset != nil { storageType, err := d.getStorageType() if err != nil { return fmt.Errorf("Storage type: %w", err) } if storageType == "zfs" { err = idmapset.ShiftPath(args.StateDir, storageDrivers.ShiftZFSSkipper) } else if storageType == "btrfs" { err = storageDrivers.ShiftBtrfsRootfs(args.StateDir, idmapset) } else { err = idmapset.ShiftPath(args.StateDir, nil) } if err != nil { return err } } if args.DumpDir != "" { finalStateDir = fmt.Sprintf("%s/%s", args.StateDir, args.DumpDir) } _, migrateErr = subprocess.RunCommand( d.state.OS.ExecPath, "forkmigrate", d.name, d.state.OS.LxcPath, configPath, finalStateDir, fmt.Sprintf("%v", preservesInodes), ) if migrateErr == nil { // Run any post start hooks. err = d.runHooks(postStartHooks) if err != nil { if args.Op != nil { args.Op.Done(err) // Must come before Stop() otherwise stop will not proceed. } // Attempt to stop container. _ = d.Stop(false) return err } } } else { // Load the go-lxc struct var cc *liblxc.Container if d.expandedConfig["raw.lxc"] != "" { cc, err = d.initLXC(true) if err != nil { return err } err = d.loadRawLXCConfig(cc) if err != nil { return err } } else { cc, err = d.initLXC(false) if err != nil { return err } } script := "" if args.ActionScript { script = filepath.Join(args.StateDir, "action.sh") } if args.DumpDir != "" { finalStateDir = fmt.Sprintf("%s/%s", args.StateDir, args.DumpDir) } // TODO: make this configurable? Ultimately I think we don't // want to do that; what we really want to do is have "modes" // of criu operation where one is "make this succeed" and the // other is "make this fast". Anyway, for now, let's choose a // really big size so it almost always succeeds, even if it is // slow. ghostLimit := uint64(256 * 1024 * 1024) opts := liblxc.MigrateOptions{ Stop: args.Stop, Directory: finalStateDir, Verbose: true, PreservesInodes: preservesInodes, ActionScript: script, GhostLimit: ghostLimit, } if args.PreDumpDir != "" { opts.PredumpDir = fmt.Sprintf("../%s", args.PreDumpDir) } if !d.IsRunning() { // otherwise the migration will needlessly fail args.Stop = false } migrateErr = cc.Migrate(args.Cmd, opts) } collectErr := collectCRIULogFile(d, finalStateDir, args.Function, prettyCmd) if collectErr != nil { d.logger.Error("Error collecting checkpoint log file", logger.Ctx{"err": collectErr}) } if migrateErr != nil { log, err2 := getCRIULogErrors(finalStateDir, prettyCmd) if err2 == nil { d.logger.Warn("Failed migrating container", ctxMap) migrateErr = fmt.Errorf("%s %s failed\n%s", args.Function, prettyCmd, log) } return migrateErr } d.logger.Info("Migrated container", ctxMap) return nil } func (d *lxc) templateApplyNow(trigger instance.TemplateTrigger) error { // If there's no metadata, just return fname := filepath.Join(d.Path(), "metadata.yaml") if !util.PathExists(fname) { return nil } // Parse the metadata content, err := os.ReadFile(fname) if err != nil { return fmt.Errorf("Failed to read metadata: %w", err) } metadata := &api.ImageMetadata{} err = yaml.Load(content, &metadata) if err != nil { return fmt.Errorf("Could not parse %s: %w", fname, err) } // Find rootUID and rootGID idmapset, err := d.DiskIdmap() if err != nil { return fmt.Errorf("Failed to set ID map: %w", err) } rootUID := int64(0) rootGID := int64(0) // Get the right uid and gid for the container if idmapset != nil { rootUID, rootGID = idmapset.ShiftIntoNS(0, 0) } // Figure out the container architecture arch, err := osarch.ArchitectureName(d.architecture) if err != nil { arch, err = osarch.ArchitectureName(d.state.OS.Architectures[0]) if err != nil { return fmt.Errorf("Failed to detect system architecture: %w", err) } } // Generate the container metadata containerMeta := make(map[string]string) containerMeta["name"] = d.name containerMeta["type"] = "container" containerMeta["architecture"] = arch if d.ephemeral { containerMeta["ephemeral"] = "true" } else { containerMeta["ephemeral"] = "false" } if d.IsPrivileged() { containerMeta["privileged"] = "true" } else { containerMeta["privileged"] = "false" } // Setup security check. rootfsPath, err := os.OpenFile(d.RootfsPath(), unix.O_PATH, 0) if err != nil { return fmt.Errorf("Failed to open instance rootfs path: %w", err) } defer func() { _ = rootfsPath.Close() }() checkBeneath := func(targetPath string) error { fd, err := unix.Openat2(int(rootfsPath.Fd()), targetPath, &unix.OpenHow{ Flags: unix.O_PATH | unix.O_CLOEXEC, Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_MAGICLINKS, }) if err != nil { if errors.Is(err, unix.EXDEV) { return errors.New("Template is attempting access to path outside of container") } return nil } _ = unix.Close(fd) return nil } // Go through the templates for tplPath, tpl := range metadata.Templates { err = func(tplPath string, tpl *api.ImageMetadataTemplate) error { var w *os.File // Check if the template should be applied now found := slices.Contains(tpl.When, string(trigger)) if !found { return nil } // Perform some security checks. relPath := strings.TrimLeft(tplPath, "/") err = checkBeneath(relPath) if err != nil { return err } if filepath.Base(tpl.Template) != tpl.Template { return errors.New("Template path is attempting to read outside of template directory") } tplDirStat, err := os.Lstat(d.TemplatesPath()) if err != nil { return fmt.Errorf("Couldn't access template directory: %w", err) } if !tplDirStat.IsDir() { return errors.New("Template directory isn't a regular directory") } tplFileStat, err := os.Lstat(filepath.Join(d.TemplatesPath(), tpl.Template)) if err != nil { return fmt.Errorf("Couldn't access template file: %w", err) } if tplFileStat.Mode()&os.ModeSymlink == os.ModeSymlink { return errors.New("Template file is a symlink") } // Open the file to template, create if needed fullpath := filepath.Join(d.RootfsPath(), relPath) if util.PathExists(fullpath) { if tpl.CreateOnly { return nil } // Open the existing file w, err = os.Create(fullpath) if err != nil { return fmt.Errorf("Failed to create template file: %w", err) } } else { // UID and GID fileUID := int64(0) fileGID := int64(0) if tpl.UID != "" { id, err := strconv.ParseInt(tpl.UID, 10, 64) if err != nil { return fmt.Errorf("Bad file UID %q for %q: %w", tpl.UID, tplPath, err) } fileUID = id } if tpl.GID != "" { id, err := strconv.ParseInt(tpl.GID, 10, 64) if err != nil { return fmt.Errorf("Bad file GID %q for %q: %w", tpl.GID, tplPath, err) } fileGID = id } if idmapset != nil { fileUID, fileGID = idmapset.ShiftIntoNS(fileUID, fileGID) } // Mode fileMode := fs.FileMode(0o644) if tpl.Mode != "" { if len(tpl.Mode) == 3 { tpl.Mode = fmt.Sprintf("0%s", tpl.Mode) } mode, err := strconv.ParseInt(tpl.Mode, 0, 0) if err != nil { return fmt.Errorf("Bad mode %q for %q: %w", tpl.Mode, tplPath, err) } fileMode = os.FileMode(mode) & os.ModePerm } // Create the directories leading to the file err = internalUtil.MkdirAllOwner(path.Dir(fullpath), 0o755, int(rootUID), int(rootGID)) if err != nil { return err } // Create the file itself w, err = os.Create(fullpath) if err != nil { return err } // Fix ownership and mode err = w.Chown(int(fileUID), int(fileGID)) if err != nil { return err } err = w.Chmod(fileMode) if err != nil { return err } } defer func() { _ = w.Close() }() // Read the template tplString, err := os.ReadFile(filepath.Join(d.TemplatesPath(), tpl.Template)) if err != nil { return fmt.Errorf("Failed to read template file: %w", err) } configGet := func(confKey, confDefault *pongo2.Value) *pongo2.Value { val, ok := d.expandedConfig[confKey.String()] if !ok { return confDefault } return pongo2.AsValue(strings.TrimRight(val, "\r\n")) } err = internalUtil.RenderTemplateFile(w, string(tplString), pongo2.Context{ "trigger": trigger, "path": tplPath, "container": containerMeta, "instance": containerMeta, "config": d.expandedConfig, "devices": d.expandedDevices, "properties": tpl.Properties, "config_get": configGet, }) if err != nil { return fmt.Errorf("Failed to render template: %w", err) } return w.Close() }(tplPath, tpl) if err != nil { return err } } return nil } // FileSFTPConn returns a connection to the forkfile handler. func (d *lxc) FileSFTPConn() (net.Conn, error) { // Lock to avoid concurrent spawning. spawnUnlock, err := locking.Lock(context.TODO(), fmt.Sprintf("forkfile_%d", d.id)) if err != nil { return nil, err } defer spawnUnlock() // Create any missing directories in case the instance has never been started before. err = os.MkdirAll(d.RunPath(), 0o700) if err != nil { return nil, err } // Trickery to handle paths > 108 chars. dirFile, err := os.Open(d.RunPath()) if err != nil { return nil, err } defer func() { _ = dirFile.Close() }() forkfileAddr, err := net.ResolveUnixAddr("unix", fmt.Sprintf("/proc/self/fd/%d/forkfile.sock", dirFile.Fd())) if err != nil { return nil, err } // Attempt to connect on existing socket. forkfilePath := filepath.Join(d.RunPath(), "forkfile.sock") forkfileConn, err := net.DialUnix("unix", nil, forkfileAddr) if err == nil { // Found an existing server. return forkfileConn, nil } // Check for ongoing operations (that may involve shifting or replacing the root volume) so as to avoid // allowing SFTP access while the container's filesystem setup is in flux. // If there is an update operation ongoing and the instance is running then do not wait for the operation // to complete before continuing as it is possible that a disk device is being removed that requires SFTP // to clean up the path inside the container. Also it is not possible to be shifting/replacing the root // volume when the instance is running, so there should be no reason to wait for the operation to finish. op := operationlock.Get(d.Project().Name, d.Name()) if op.Action() != operationlock.ActionUpdate || !d.IsRunning() { _ = op.Wait(context.Background()) } // Setup reverter. reverter := revert.New() defer reverter.Fail() // Create the listener. _ = os.Remove(forkfilePath) forkfileListener, err := net.ListenUnix("unix", forkfileAddr) if err != nil { return nil, err } reverter.Add(func() { _ = forkfileListener.Close() _ = os.Remove(forkfilePath) }) // Spawn forkfile in a Go routine. chReady := make(chan error) go func() { // Lock to avoid concurrent running forkfile. runUnlock, err := locking.Lock(context.TODO(), d.forkfileRunningLockName()) if err != nil { chReady <- err return } defer runUnlock() // Mount the filesystem if needed. if !d.IsRunning() { // Mount the root filesystem if required. _, err := d.mount() if err != nil { chReady <- err return } defer func() { _ = d.unmount() }() } // Start building the command. args := []string{ d.state.OS.ExecPath, "forkfile", "--", } extraFiles := []*os.File{} // Get the listener file. forkfileFile, err := forkfileListener.File() if err != nil { chReady <- err return } defer func() { _ = forkfileFile.Close() }() args = append(args, "3") extraFiles = append(extraFiles, forkfileFile) // Get the rootfs. rootfsFile, err := os.Open(d.RootfsPath()) if err != nil { chReady <- err return } defer func() { _ = rootfsFile.Close() }() args = append(args, "4") extraFiles = append(extraFiles, rootfsFile) // Get the pidfd if the container is running. if d.IsRunning() { pidFd, err := d.InitPidFd() if err != nil { chReady <- err return } defer func() { _ = pidFd.Close() }() args = append(args, "5") extraFiles = append(extraFiles, pidFd) } else { args = append(args, "-1") } // Finalize the args. args = append(args, fmt.Sprintf("%d", d.InitPID())) // Prepare sftp server. forkfile := exec.Cmd{ Path: d.state.OS.ExecPath, Args: args, ExtraFiles: extraFiles, } var stderr bytes.Buffer forkfile.Stderr = &stderr if !d.IsRunning() { // Get the disk idmap. idmapset, err := d.DiskIdmap() if err != nil { chReady <- err return } if idmapset != nil { forkfile.SysProcAttr = &syscall.SysProcAttr{ Cloneflags: syscall.CLONE_NEWUSER, Credential: &syscall.Credential{ Uid: uint32(0), Gid: uint32(0), }, UidMappings: idmapset.ToUIDMappings(), GidMappings: idmapset.ToGIDMappings(), } } } // Start the server. err = forkfile.Start() if err != nil { chReady <- fmt.Errorf("Failed to run forkfile: %w: %s", err, strings.TrimSpace(stderr.String())) return } // Write PID file. pidFile := filepath.Join(d.RunPath(), "forkfile.pid") err = os.WriteFile(pidFile, fmt.Appendf(nil, "%d\n", forkfile.Process.Pid), 0o600) if err != nil { chReady <- fmt.Errorf("Failed to write forkfile PID: %w", err) return } // Close the listener and delete the socket immediately after forkfile exits to avoid clients // thinking a listener is available while other deferred calls are being processed. defer func() { _ = forkfileListener.Close() _ = os.Remove(forkfilePath) _ = os.Remove(pidFile) }() // Indicate the process was spawned without error. close(chReady) // Wait for completion. err = forkfile.Wait() if err != nil { d.logger.Error("SFTP server stopped with error", logger.Ctx{"err": err, "stderr": strings.TrimSpace(stderr.String())}) return } }() // Wait for forkfile to have been spawned. err = <-chReady if err != nil { return nil, err } // Connect to the new server. forkfileConn, err = net.DialUnix("unix", nil, forkfileAddr) if err != nil { return nil, err } // All done. reverter.Success() return forkfileConn, nil } // FileSFTP returns an SFTP connection to the forkfile handler. func (d *lxc) FileSFTP() (*sftp.Client, error) { // Connect to the forkfile daemon. conn, err := d.FileSFTPConn() if err != nil { return nil, err } // Get a SFTP client. client, err := sftp.NewClientPipe(conn, conn) if err != nil { _ = conn.Close() return nil, err } go func() { // Wait for the client to be done before closing the connection. _ = client.Wait() _ = conn.Close() }() return client, nil } // stopForkFile attempts to send SIGTERM (if force is true) or SIGINT to forkfile then waits for it to exit. func (d *lxc) stopForkfile(force bool) { // Make sure that when the function exits, no forkfile is running by acquiring the lock (which indicates // that forkfile isn't running and holding the lock) and then releasing it. defer func() { unlock, err := locking.Lock(context.TODO(), d.forkfileRunningLockName()) if err != nil { return } unlock() }() content, err := os.ReadFile(filepath.Join(d.RunPath(), "forkfile.pid")) if err != nil { return } pid, err := strconv.ParseInt(strings.TrimSpace(string(content)), 10, 64) if err != nil { return } d.logger.Debug("Stopping forkfile", logger.Ctx{"pid": pid, "force": force}) if force { // Forcefully kill the running process. _ = unix.Kill(int(pid), unix.SIGTERM) } else { // Try to send SIGINT to forkfile to indicate it should not accept any new connection. _ = unix.Kill(int(pid), unix.SIGINT) } } // Console attaches to the instance console. func (d *lxc) Console(protocol string) (*os.File, chan error, error) { if protocol != instance.ConsoleTypeConsole { return nil, nil, fmt.Errorf("Container instances don't support %q output", protocol) } chDisconnect := make(chan error, 1) args := []string{ d.state.OS.ExecPath, "forkconsole", project.Instance(d.Project().Name, d.Name()), d.state.OS.LxcPath, filepath.Join(d.RunPath(), "lxc.conf"), "tty=0", "escape=-1", } idmapset, err := d.CurrentIdmap() if err != nil { return nil, nil, err } var rootUID, rootGID int64 if idmapset != nil { rootUID, rootGID = idmapset.ShiftIntoNS(0, 0) } // Create a PTS pair. ptx, pty, err := linux.OpenPty(rootUID, rootGID) if err != nil { return nil, nil, err } // Switch the console file descriptor into raw mode. _, err = termios.MakeRaw(int(ptx.Fd())) if err != nil { return nil, nil, err } cmd := exec.Cmd{} cmd.Path = d.state.OS.ExecPath cmd.Args = args cmd.Stdin = pty cmd.Stdout = pty cmd.Stderr = pty err = cmd.Start() if err != nil { return nil, nil, err } go func() { err = cmd.Wait() _ = ptx.Close() _ = pty.Close() }() go func() { <-chDisconnect _ = cmd.Process.Kill() }() d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceConsole.Event(d, logger.Ctx{"type": instance.ConsoleTypeConsole})) return ptx, chDisconnect, nil } // ConsoleLog returns console log. func (d *lxc) ConsoleLog(opts liblxc.ConsoleLogOptions) (string, error) { cc, err := d.initLXC(false) if err != nil { return "", err } msg, err := cc.ConsoleLog(opts) if err != nil { return "", err } if opts.ClearLog { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceConsoleReset.Event(d, nil)) } else if opts.ReadLog && opts.WriteToLogFile { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceConsoleRetrieved.Event(d, nil)) } return string(msg), nil } // Exec executes a command inside the instance. func (d *lxc) Exec(req api.InstanceExecPost, stdin *os.File, stdout *os.File, stderr *os.File) (instance.Cmd, error) { // Generate the LXC config if missing. configPath := filepath.Join(d.RunPath(), "lxc.conf") if !util.PathExists(configPath) { cc, err := d.initLXC(true) if err != nil { return nil, fmt.Errorf("Load go-lxc struct: %w", err) } err = cc.SaveConfigFile(configPath) if err != nil { _ = os.Remove(configPath) return nil, err } } // Prepare the environment envSlice := []string{} for k, v := range req.Environment { envSlice = append(envSlice, fmt.Sprintf("%s=%s", k, v)) } // Setup logfile logPath := filepath.Join(d.LogPath(), "forkexec.log") logFile, err := os.OpenFile(logPath, os.O_WRONLY|os.O_CREATE|os.O_SYNC, 0o644) if err != nil { return nil, err } defer func() { _ = logFile.Close() }() // Prepare the subcommand cname := project.Instance(d.Project().Name, d.Name()) args := []string{ d.state.OS.ExecPath, "forkexec", cname, d.state.OS.LxcPath, filepath.Join(d.RunPath(), "lxc.conf"), req.Cwd, fmt.Sprintf("%d", req.User), fmt.Sprintf("%d", req.Group), } args = append(args, "0") args = append(args, "--") args = append(args, "env") args = append(args, envSlice...) args = append(args, "--") args = append(args, "cmd") args = append(args, req.Command...) cmd := exec.Cmd{} cmd.Path = d.state.OS.ExecPath cmd.Args = args cmd.Stdin = nil cmd.Stdout = logFile cmd.Stderr = logFile // Mitigation for CVE-2019-5736 useRexec := false if d.expandedConfig["raw.idmap"] != "" { err := instance.AllowedUnprivilegedOnlyMap(d.expandedConfig["raw.idmap"]) if err != nil { useRexec = true } } if util.IsTrue(d.expandedConfig["security.privileged"]) { useRexec = true } if useRexec { cmd.Env = append(os.Environ(), "LXC_MEMFD_REXEC=1") } // Setup communication PIPE rStatus, wStatus, err := os.Pipe() defer func() { _ = rStatus.Close() }() if err != nil { return nil, err } cmd.ExtraFiles = []*os.File{stdin, stdout, stderr, wStatus} err = cmd.Start() _ = wStatus.Close() if err != nil { return nil, err } attachedPid := linux.ReadPid(rStatus) if attachedPid <= 0 { _ = cmd.Wait() d.logger.Error("Failed to retrieve PID of executing child process") return nil, errors.New("Failed to retrieve PID of executing child process") } d.logger.Debug("Retrieved PID of executing child process", logger.Ctx{"attachedPid": attachedPid}) d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceExec.Event(d, logger.Ctx{"command": req.Command})) instCmd := &lxcCmd{ cmd: &cmd, attachedChildPid: int(attachedPid), } return instCmd, nil } func (d *lxc) cpuStateUsage(cg *cgroup.CGroup) (int64, bool) { if !cgroup.Supports(cgroup.CPU) { return -1, false } value, err := cg.GetCPUAcctUsage() if err != nil { return -1, true } return value, true } func (d *lxc) cpuState() api.InstanceStateCPU { cpu := api.InstanceStateCPU{} cc, err := d.initLXC(false) if err != nil { return cpu } cg, err := d.cgroup(cc, true) if err != nil { return cpu } cpuUsage, ok := d.cpuStateUsage(cg) if ok { cpu.Usage = cpuUsage } cpuCount, err := cg.GetEffectiveCPUs() if err != nil { return cpu } limitPeriod, limitQuota, err := cg.GetCPUCfsLimit() if err != nil { return cpu } if limitQuota == -1 { cpu.AllocatedTime = int64(cpuCount) * 1_000_000_000 } else { cpu.AllocatedTime = 1_000_000_000 * limitQuota / limitPeriod } return cpu } func (d *lxc) diskState() map[string]api.InstanceStateDisk { disk := map[string]api.InstanceStateDisk{} for _, dev := range d.expandedDevices.Sorted() { if dev.Config["type"] != "disk" { continue } var usage *storagePools.VolumeUsage if dev.Config["path"] == "/" { pool, err := d.getStoragePool() if err != nil { d.logger.Error("Error loading storage pool", logger.Ctx{"err": err}) continue } usage, err = pool.GetInstanceUsage(d) if err != nil { if !errors.Is(err, storageDrivers.ErrNotSupported) { d.logger.Error("Error getting disk usage", logger.Ctx{"err": err}) } continue } } else if dev.Config["pool"] != "" { pool, err := storagePools.LoadByName(d.state, dev.Config["pool"]) if err != nil { d.logger.Error("Error loading storage pool", logger.Ctx{"poolName": dev.Config["pool"], "err": err}) continue } volName, _ := internalInstance.SplitVolumeSource(dev.Config["source"]) usage, err = pool.GetCustomVolumeUsage(d.Project().Name, volName) if err != nil { if !errors.Is(err, storageDrivers.ErrNotSupported) { d.logger.Error("Error getting volume usage", logger.Ctx{"volume": dev.Config["source"], "err": err}) } continue } } else { continue } state := api.InstanceStateDisk{} if usage != nil { state.Usage = usage.Used state.Total = usage.Total } disk[dev.Name] = state } return disk } func (d *lxc) memoryState() api.InstanceStateMemory { memory := api.InstanceStateMemory{} cc, err := d.initLXC(false) if err != nil { return memory } cg, err := d.cgroup(cc, true) if err != nil { return memory } if !cgroup.Supports(cgroup.Memory) { return memory } // Memory in bytes value, err := cg.GetMemoryUsage() if err == nil { memory.Usage = value } // Memory peak in bytes value, err = cg.GetMemoryMaxUsage() if err == nil { memory.UsagePeak = value } // Memory total in bytes value, err = cg.GetEffectiveMemoryLimit() if err == nil { memory.Total = value } // Swap in bytes if memory.Usage > 0 { value, err := cg.GetMemorySwapUsage() if err == nil { memory.SwapUsage = value } } // Swap peak in bytes if memory.UsagePeak > 0 { value, err := cg.GetMemorySwapMaxUsage() if err == nil { memory.SwapUsagePeak = value } } return memory } func (d *lxc) networkState(hostInterfaces []net.Interface) map[string]api.InstanceStateNetwork { result := map[string]api.InstanceStateNetwork{} pid := d.InitPID() if pid < 1 { return result } nw, err := netutils.NetnsGetifaddrs(int32(pid), hostInterfaces) if err != nil { d.logger.Error("Failed to retrieve network information via netlink", logger.Ctx{"err": err, "pid": pid}) return result } result = nw // Get host_name from volatile data if not set already. for name, dev := range result { if dev.HostName == "" { dev.HostName = d.localConfig[fmt.Sprintf("volatile.%s.host_name", name)] result[name] = dev } } return result } func (d *lxc) processesState(pid int) (int64, error) { // Return 0 if not running if pid == -1 { return 0, errors.New("PID of LXC instance could not be initialized") } cc, err := d.initLXC(false) if err != nil { return -1, err } cg, err := d.cgroup(cc, true) if err != nil { return 0, err } if cgroup.Supports(cgroup.Pids) { value, err := cg.GetProcessesUsage() if err != nil { return -1, err } return value, nil } pids := []int64{int64(pid)} // Go through the pid list, adding new pids at the end so we go through them all for i := range pids { fname := fmt.Sprintf("/proc/%d/task/%d/children", pids[i], pids[i]) fcont, err := os.ReadFile(fname) if err != nil { // the process terminated during execution of this loop continue } content := strings.Split(string(fcont), " ") for j := range content { pid, err := strconv.ParseInt(content[j], 10, 64) if err == nil { pids = append(pids, pid) } } } return int64(len(pids)), nil } // getStorageType returns the storage type of the instance's storage pool. func (d *lxc) getStorageType() (string, error) { pool, err := d.getStoragePool() if err != nil { return "", err } return pool.Driver().Info().Name, nil } // mount the instance's rootfs volume if needed. func (d *lxc) mount() (*storagePools.MountInfo, error) { pool, err := d.getStoragePool() if err != nil { return nil, err } if d.IsSnapshot() { mountInfo, err := pool.MountInstanceSnapshot(d, nil) if err != nil { return nil, err } return mountInfo, nil } mountInfo, err := pool.MountInstance(d, nil) if err != nil { return nil, err } return mountInfo, nil } // unmount the instance's rootfs volume if needed. func (d *lxc) unmount() error { pool, err := d.getStoragePool() if err != nil { return err } if d.IsSnapshot() { err = pool.UnmountInstanceSnapshot(d, nil) if err != nil { return err } return nil } err = pool.UnmountInstance(d, nil) if err != nil { return err } return nil } // insertMountGo inserts a mount into a container. // This function is used for the seccomp notifier and so cannot call any // functions that would cause LXC to talk to the container's monitor. Otherwise // we'll have a deadlock (with a timeout but still). The InitPID() call here is // the exception since the seccomp notifier will make sure to always pass a // valid PID. func (d *lxc) insertMountGo(source, target, fstype string, flags int, mntnsPID int, idmapType idmap.StorageType) error { pid := mntnsPID if pid <= 0 { // Get the init PID pid = d.InitPID() if pid == -1 { // Container isn't running return errors.New("Can't insert mount into stopped container") } } // Create the temporary mount target var tmpMount string var err error if internalUtil.IsDir(source) { tmpMount, err = os.MkdirTemp(d.ShmountsPath(), "incus_mount_") if err != nil { return fmt.Errorf("Failed to create shmounts path: %s", err) } } else { f, err := os.CreateTemp(d.ShmountsPath(), "incus_mount_") if err != nil { return fmt.Errorf("Failed to create shmounts path: %s", err) } tmpMount = f.Name() _ = f.Close() } defer func() { _ = os.Remove(tmpMount) }() // Mount the filesystem err = unix.Mount(source, tmpMount, fstype, uintptr(flags), "") if err != nil { return fmt.Errorf("Failed to setup temporary mount: %s", err) } defer func() { _ = unix.Unmount(tmpMount, unix.MNT_DETACH) }() // Ensure that only flags modifying mount _properties_ make it through. // Strip things such as MS_BIND which would cause the creation of a // shifted mount to be skipped. // (Fyi, this is just one of the reasons why multiplexers are bad; // specifically when they do heinous things such as confusing flags // with commands.) // This is why multiplexers are bad shiftfsFlags := (flags & (unix.MS_RDONLY | unix.MS_NOSUID | unix.MS_NODEV | unix.MS_NOEXEC | unix.MS_DIRSYNC | unix.MS_NOATIME | unix.MS_NODIRATIME)) // Move the mount inside the container mntsrc := filepath.Join("/dev/.incus-mounts", filepath.Base(tmpMount)) pidStr := fmt.Sprintf("%d", pid) pidFdNr, pidFd, err := seccomp.MakePidFd(pid) if err != nil { return err } defer func() { _ = pidFd.Close() }() if !strings.HasPrefix(target, "/") { target = "/" + target } _, err = subprocess.RunCommandInheritFds( context.Background(), []*os.File{pidFd}, d.state.OS.ExecPath, "forkmount", "go-mount", "--", pidStr, fmt.Sprintf("%d", pidFdNr), mntsrc, target, string(idmapType), fmt.Sprintf("%d", shiftfsFlags)) if err != nil { return err } return nil } func (d *lxc) insertMountLXC(source, target, fstype string, flags int) error { cname := project.Instance(d.Project().Name, d.Name()) configPath := filepath.Join(d.RunPath(), "lxc.conf") if fstype == "" { fstype = "none" } if !strings.HasPrefix(target, "/") { target = "/" + target } _, err := subprocess.RunCommand( d.state.OS.ExecPath, "forkmount", "lxc-mount", "--", cname, d.state.OS.LxcPath, configPath, source, target, fstype, fmt.Sprintf("%d", flags)) if err != nil { return err } return nil } func (d *lxc) moveMount(source, target, fstype string, flags int, idmapType idmap.StorageType) error { // Get the init PID pid := d.InitPID() if pid == -1 { // Container isn't running return errors.New("Can't insert mount into stopped container") } switch idmapType { case idmap.StorageTypeIdmapped: case idmap.StorageTypeNone: default: return errors.New("Invalid idmap value specified") } pidFdNr, pidFd, err := seccomp.MakePidFd(pid) if err != nil { return err } defer func() { _ = pidFd.Close() }() pidStr := fmt.Sprintf("%d", pid) if !strings.HasPrefix(target, "/") { target = "/" + target } _, err = subprocess.RunCommandInheritFds( context.Background(), []*os.File{pidFd}, d.state.OS.ExecPath, "forkmount", "move-mount", "--", pidStr, fmt.Sprintf("%d", pidFdNr), fstype, source, target, string(idmapType), fmt.Sprintf("%d", flags)) if err != nil { return err } return nil } func (d *lxc) insertMount(source, target, fstype string, flags int, idmapType idmap.StorageType) error { if idmapType == idmap.StorageTypeIdmapped { return d.moveMount(source, target, fstype, flags, idmapType) } if idmapType == idmap.StorageTypeNone { return d.insertMountLXC(source, target, fstype, flags) } return d.insertMountGo(source, target, fstype, flags, -1, idmapType) } func (d *lxc) removeMount(mount string) error { // Get the init PID if d.InitPID() == -1 { // Container isn't running return errors.New("Can't remove mount from stopped container") } configPath := filepath.Join(d.RunPath(), "lxc.conf") cname := project.Instance(d.Project().Name, d.Name()) if !strings.HasPrefix(mount, "/") { mount = "/" + mount } _, err := subprocess.RunCommand( d.state.OS.ExecPath, "forkmount", "lxc-umount", "--", cname, d.state.OS.LxcPath, configPath, mount) if err != nil { return err } return nil } // InsertSeccompUnixDevice inserts a seccomp device. func (d *lxc) InsertSeccompUnixDevice(prefix string, m deviceConfig.Device, pid int) error { if pid < 0 { return errors.New("Invalid request PID specified") } rootLink := fmt.Sprintf("/proc/%d/root", pid) rootPath, err := os.Readlink(rootLink) if err != nil { return err } uid, gid, _, _, err := seccomp.TaskIDs(pid) if err != nil { return err } idmapset, err := d.CurrentIdmap() if err != nil { return err } nsuid, nsgid := idmapset.ShiftFromNS(uid, gid) m["uid"] = fmt.Sprintf("%d", nsuid) m["gid"] = fmt.Sprintf("%d", nsgid) if !path.IsAbs(m["path"]) { cwdLink := fmt.Sprintf("/proc/%d/cwd", pid) prefixPath, err := os.Readlink(cwdLink) if err != nil { return err } prefixPath = strings.TrimPrefix(prefixPath, rootPath) m["path"] = filepath.Join(rootPath, prefixPath, m["path"]) } else { m["path"] = filepath.Join(rootPath, m["path"]) } idmapSet, err := d.CurrentIdmap() if err != nil { return err } dev, err := device.UnixDeviceCreate(d.state, idmapSet, d.DevicesPath(), prefix, m, true) if err != nil { return fmt.Errorf("Failed to setup device: %s", err) } devPath := dev.HostPath tgtPath := dev.RelativePath // Bind-mount it into the container defer func() { _ = os.Remove(devPath) }() return d.insertMountGo(devPath, tgtPath, "none", unix.MS_BIND, pid, idmap.StorageTypeNone) } func (d *lxc) removeUnixDevices() error { // Check that we indeed have devices to remove if !util.PathExists(d.DevicesPath()) { return nil } // Load the directory listing dents, err := os.ReadDir(d.DevicesPath()) if err != nil { return err } // Go through all the unix devices for _, f := range dents { // Skip non-Unix devices if !strings.HasPrefix(f.Name(), "forkmknod.unix.") && !strings.HasPrefix(f.Name(), "unix.") && !strings.HasPrefix(f.Name(), "infiniband.unix.") { continue } // Remove the entry devicePath := filepath.Join(d.DevicesPath(), f.Name()) err := os.Remove(devicePath) if err != nil { d.logger.Error("Failed removing unix device", logger.Ctx{"err": err, "path": devicePath}) } } return nil } // FillNetworkDevice takes a nic or infiniband device type and enriches it with automatically // generated name and hwaddr properties if these are missing from the device. func (d *lxc) FillNetworkDevice(name string, m deviceConfig.Device) (deviceConfig.Device, error) { var err error newDevice := m.Clone() // Function to try and guess an available name nextInterfaceName := func() (string, error) { devNames := []string{} // Include all static interface names for _, dev := range d.expandedDevices.Sorted() { if dev.Config["name"] != "" && !slices.Contains(devNames, dev.Config["name"]) { devNames = append(devNames, dev.Config["name"]) } } // Include all currently allocated interface names for k, v := range d.expandedConfig { if !strings.HasPrefix(k, internalInstance.ConfigVolatilePrefix) { continue } fields := strings.SplitN(k, ".", 3) if len(fields) != 3 { continue } if fields[2] != "name" || slices.Contains(devNames, v) { continue } devNames = append(devNames, v) } // Attempt to include all existing interfaces cname := project.Instance(d.Project().Name, d.Name()) cc, err := liblxc.NewContainer(cname, d.state.OS.LxcPath) if err == nil { defer func() { _ = cc.Release() }() interfaces, err := cc.Interfaces() if err == nil { for _, name := range interfaces { if slices.Contains(devNames, name) { continue } devNames = append(devNames, name) } } } i := 0 name := "" for { if m["type"] == "infiniband" { name = fmt.Sprintf("ib%d", i) } else { name = fmt.Sprintf("eth%d", i) } // Find a free device name if !slices.Contains(devNames, name) { return name, nil } i++ } } nicType, err := nictype.NICType(d.state, d.Project().Name, m) if err != nil { return nil, err } isPhysicalWithBridge := device.IsPhysicalNICWithBridge(d.state, d.Project().Name, m) // Fill in the MAC address. if (!slices.Contains([]string{"physical", "ipvlan"}, nicType) || isPhysicalWithBridge) && m["hwaddr"] == "" { configKey := fmt.Sprintf("volatile.%s.hwaddr", name) volatileHwaddr := d.localConfig[configKey] if volatileHwaddr == "" { // Generate a new MAC address. volatileHwaddr, err = instance.DeviceNextInterfaceHWAddr(d.MACPattern()) if err != nil || volatileHwaddr == "" { return nil, fmt.Errorf("Failed generating %q: %w", configKey, err) } // Update the database and update volatileHwaddr with stored value. volatileHwaddr, err = d.insertConfigkey(configKey, volatileHwaddr) if err != nil { return nil, fmt.Errorf("Failed storing generated config key %q: %w", configKey, err) } // Set stored value into current instance config. d.localConfig[configKey] = volatileHwaddr d.expandedConfig[configKey] = volatileHwaddr } if volatileHwaddr == "" { return nil, fmt.Errorf("Failed getting %q", configKey) } newDevice["hwaddr"] = volatileHwaddr } // Fill in the interface name. if m["name"] == "" { configKey := fmt.Sprintf("volatile.%s.name", name) volatileName := d.localConfig[configKey] if volatileName == "" { // Generate a new interface name. volatileName, err = nextInterfaceName() if err != nil || volatileName == "" { return nil, fmt.Errorf("Failed generating %q: %w", configKey, err) } // Update the database and update volatileName with stored value. volatileName, err = d.insertConfigkey(configKey, volatileName) if err != nil { return nil, fmt.Errorf("Failed storing generated config key %q: %w", configKey, err) } // Set stored value into current instance config. d.localConfig[configKey] = volatileName d.expandedConfig[configKey] = volatileName } if volatileName == "" { return nil, fmt.Errorf("Failed getting %q", configKey) } newDevice["name"] = volatileName } return newDevice, nil } func (d *lxc) removeDiskDevices() error { // Check that we indeed have devices to remove if !util.PathExists(d.DevicesPath()) { return nil } // Load the directory listing dents, err := os.ReadDir(d.DevicesPath()) if err != nil { return err } // Go through all the unix devices for _, f := range dents { // Skip non-disk devices if !strings.HasPrefix(f.Name(), "disk.") { continue } // Always try to unmount the host side _ = unix.Unmount(filepath.Join(d.DevicesPath(), f.Name()), unix.MNT_DETACH) // Remove the entry diskPath := filepath.Join(d.DevicesPath(), f.Name()) err := os.Remove(diskPath) if err != nil { d.logger.Error("Failed to remove disk device path", logger.Ctx{"err": err, "path": diskPath}) } } return nil } // IsFrozen returns if instance is frozen. func (d *lxc) IsFrozen() bool { return d.statusCode() == api.Frozen } // IsNesting returns if instance is nested. func (d *lxc) IsNesting() bool { return util.IsTrue(d.expandedConfig["security.nesting"]) } func (d *lxc) isCurrentlyPrivileged() bool { if !d.IsRunning() { return d.IsPrivileged() } idmap, err := d.CurrentIdmap() if err != nil { return d.IsPrivileged() } return idmap == nil } // IsPrivileged returns if instance is privileged. func (d *lxc) IsPrivileged() bool { return util.IsTrue(d.expandedConfig["security.privileged"]) } // IsRunning returns if instance is running. func (d *lxc) IsRunning() bool { return d.isRunningStatusCode(d.statusCode()) } // CanMigrate returns whether the instance can be migrated. func (d *lxc) CanMigrate() string { return d.canMigrate(d) } // LockExclusive attempts to get exclusive access to the instance's root volume. func (d *lxc) LockExclusive() (*operationlock.InstanceOperation, error) { if d.IsRunning() { return nil, errors.New("Instance is running") } // Prevent concurrent operations the instance. op, err := operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionCreate, false, false) if err != nil { return nil, err } // Stop forkfile as otherwise it will hold the root volume open preventing unmount. d.stopForkfile(false) return op, err } // InitPID returns PID of init process. func (d *lxc) InitPID() int { // Load the go-lxc struct cc, err := d.initLXC(false) if err != nil { return -1 } return cc.InitPid() } // InitPidFd returns pidfd of init process. func (d *lxc) InitPidFd() (*os.File, error) { // Load the go-lxc struct cc, err := d.initLXC(false) if err != nil { return nil, err } return cc.InitPidFd() } // DevptsFd returns dirfd of devpts mount. func (d *lxc) DevptsFd() (*os.File, error) { // Load the go-lxc struct cc, err := d.initLXC(false) if err != nil { return nil, err } defer d.release() return cc.DevptsFd() } // CurrentIdmap returns current IDMAP. func (d *lxc) CurrentIdmap() (*idmap.Set, error) { jsonIdmap, ok := d.LocalConfig()["volatile.idmap.current"] if !ok { return d.DiskIdmap() } return idmap.NewSetFromJSON(jsonIdmap) } // DiskIdmap returns DISK IDMAP. func (d *lxc) DiskIdmap() (*idmap.Set, error) { jsonIdmap, ok := d.LocalConfig()["volatile.last_state.idmap"] if !ok { return nil, nil } return idmap.NewSetFromJSON(jsonIdmap) } // NextIdmap returns next IDMAP. func (d *lxc) NextIdmap() (*idmap.Set, error) { jsonIdmap, ok := d.LocalConfig()["volatile.idmap.next"] if !ok { return d.CurrentIdmap() } return idmap.NewSetFromJSON(jsonIdmap) } // statusCode returns instance status code. func (d *lxc) statusCode() api.StatusCode { // Shortcut to avoid spamming liblxc during ongoing operations. op := operationlock.Get(d.Project().Name, d.Name()) if op != nil { if op.Action() == operationlock.ActionStart { return api.Stopped } if op.Action() == operationlock.ActionStop { if util.IsTrue(d.LocalConfig()["volatile.last_state.ready"]) { return api.Ready } return api.Running } } state, err := d.getLxcState() if err != nil { return api.Error } statusCode := lxcStatusCode(state) if statusCode == api.Running && util.IsTrue(d.LocalConfig()["volatile.last_state.ready"]) { return api.Ready } return statusCode } // State returns instance state. func (d *lxc) State() string { return strings.ToUpper(d.statusCode().String()) } // LogFilePath log file path. func (d *lxc) LogFilePath() string { return filepath.Join(d.LogPath(), "lxc.log") } func (d *lxc) CGroup() (*cgroup.CGroup, error) { // Load the go-lxc struct cc, err := d.initLXC(false) if err != nil { return nil, err } return d.cgroup(cc, true) } func (d *lxc) cgroup(cc *liblxc.Container, running bool) (*cgroup.CGroup, error) { if cc == nil { return nil, errors.New("Container not initialized for cgroup") } rw := lxcCgroupReadWriter{} rw.cc = cc rw.running = running return cgroup.New(&rw) } type lxcCgroupReadWriter struct { cc *liblxc.Container running bool } // Get reads the value of a cgroup key. func (rw *lxcCgroupReadWriter) Get(controller string, key string) (string, error) { if !rw.running { return strings.Join(rw.cc.ConfigItem(fmt.Sprintf("lxc.cgroup2.%s", key)), "\n"), nil } return strings.Join(rw.cc.CgroupItem(key), "\n"), nil } // Set writes a value to a cgroup key. func (rw *lxcCgroupReadWriter) Set(controller string, key string, value string) error { if !rw.running { return lxcSetConfigItem(rw.cc, fmt.Sprintf("lxc.cgroup2.%s", key), value) } return rw.cc.SetCgroupItem(key, value) } // UpdateBackupFile writes the instance's backup.yaml file to storage. func (d *lxc) UpdateBackupFile() error { // Prevent concurrent updates to the backup file. unlock, err := d.updateBackupFileLock(context.Background()) if err != nil { return err } defer unlock() // Write the current instance state to backup file. pool, err := d.getStoragePool() if err != nil { return err } return pool.UpdateInstanceBackupFile(d, true, nil) } // Info returns "lxc" and the currently loaded version of LXC. func (d *lxc) Info() instance.Info { return instance.Info{ Name: "lxc", Version: liblxc.Version(), Type: instancetype.Container, Error: nil, } } func (d *lxc) Metrics(hostInterfaces []net.Interface) (*metrics.MetricSet, error) { out := metrics.NewMetricSet(map[string]string{"project": d.project.Name, "name": d.name, "type": instancetype.Container.String()}) if !d.IsRunning() { return nil, ErrInstanceIsStopped } cc, err := d.initLXC(false) if err != nil { return nil, err } // Load cgroup abstraction cg, err := d.cgroup(cc, true) if err != nil { return nil, err } // Get Memory limit. memoryLimit, err := cg.GetEffectiveMemoryLimit() if err != nil { d.logger.Warn("Failed getting effective memory limit", logger.Ctx{"err": err}) } memoryCached := int64(0) // Get memory stats. memStats, err := cg.GetMemoryStats() if err != nil { d.logger.Warn("Failed to get memory stats", logger.Ctx{"err": err}) } else { for k, v := range memStats { var metricType metrics.MetricType switch k { case "active_anon": metricType = metrics.MemoryActiveAnonBytes case "active_file": metricType = metrics.MemoryActiveFileBytes case "active": metricType = metrics.MemoryActiveBytes case "inactive_anon": metricType = metrics.MemoryInactiveAnonBytes case "inactive_file": metricType = metrics.MemoryInactiveFileBytes case "inactive": metricType = metrics.MemoryInactiveBytes case "unevictable": metricType = metrics.MemoryUnevictableBytes case "writeback": metricType = metrics.MemoryWritebackBytes case "dirty": metricType = metrics.MemoryDirtyBytes case "mapped": metricType = metrics.MemoryMappedBytes case "rss": metricType = metrics.MemoryRSSBytes case "shmem": metricType = metrics.MemoryShmemBytes case "cache": metricType = metrics.MemoryCachedBytes memoryCached = int64(v) } out.AddSamples(metricType, metrics.Sample{Value: float64(v)}) } } // Get memory usage. memoryUsage, err := cg.GetMemoryUsage() if err != nil { d.logger.Warn("Failed to get memory usage", logger.Ctx{"err": err}) } if memoryLimit > 0 { out.AddSamples(metrics.MemoryMemTotalBytes, metrics.Sample{Value: float64(memoryLimit)}) out.AddSamples(metrics.MemoryMemAvailableBytes, metrics.Sample{Value: float64(memoryLimit - memoryUsage + memoryCached)}) out.AddSamples(metrics.MemoryMemFreeBytes, metrics.Sample{Value: float64(memoryLimit - memoryUsage)}) } // Get oom kills. oomKills, err := cg.GetOOMKills() if err != nil { d.logger.Warn("Failed to get oom kills", logger.Ctx{"err": err}) } out.AddSamples(metrics.MemoryOOMKillsTotal, metrics.Sample{Value: float64(oomKills)}) // Handle swap. if cgroup.Supports(cgroup.Memory) { swapUsage, err := cg.GetMemorySwapUsage() if err != nil { d.logger.Warn("Failed to get swap usage", logger.Ctx{"err": err}) } else { out.AddSamples(metrics.MemorySwapBytes, metrics.Sample{Value: float64(swapUsage)}) } } // Get CPU stats usage, err := cg.GetCPUAcctUsageAll() if err != nil { d.logger.Warn("Failed to get CPU usage", logger.Ctx{"err": err}) } else { for cpu, stats := range usage { cpuID := strconv.Itoa(int(cpu)) out.AddSamples(metrics.CPUSecondsTotal, metrics.Sample{Value: float64(stats.System) / 1000000000, Labels: map[string]string{"mode": "system", "cpu": cpuID}}) out.AddSamples(metrics.CPUSecondsTotal, metrics.Sample{Value: float64(stats.User) / 1000000000, Labels: map[string]string{"mode": "user", "cpu": cpuID}}) } } // Get CPUs. CPUs, err := cg.GetEffectiveCPUs() if err != nil { d.logger.Warn("Failed to get CPUs", logger.Ctx{"err": err}) } else { out.AddSamples(metrics.CPUs, metrics.Sample{Value: float64(CPUs)}) } // Get disk stats diskStats, err := cg.GetIOStats() if err != nil { d.logger.Warn("Failed to get disk stats", logger.Ctx{"err": err}) } else { for disk, stats := range diskStats { labels := map[string]string{"device": disk} out.AddSamples(metrics.DiskReadBytesTotal, metrics.Sample{Value: float64(stats.ReadBytes), Labels: labels}) out.AddSamples(metrics.DiskReadsCompletedTotal, metrics.Sample{Value: float64(stats.ReadsCompleted), Labels: labels}) out.AddSamples(metrics.DiskWrittenBytesTotal, metrics.Sample{Value: float64(stats.WrittenBytes), Labels: labels}) out.AddSamples(metrics.DiskWritesCompletedTotal, metrics.Sample{Value: float64(stats.WritesCompleted), Labels: labels}) } } // Get filesystem stats fsStats, err := d.getFSStats() if err != nil { d.logger.Warn("Failed to get fs stats", logger.Ctx{"err": err}) } else { out.Merge(fsStats) } // Get network stats networkState := d.networkState(hostInterfaces) for name, state := range networkState { labels := map[string]string{"device": name} out.AddSamples(metrics.NetworkReceiveBytesTotal, metrics.Sample{Value: float64(state.Counters.BytesReceived), Labels: labels}) out.AddSamples(metrics.NetworkReceivePacketsTotal, metrics.Sample{Value: float64(state.Counters.PacketsReceived), Labels: labels}) out.AddSamples(metrics.NetworkTransmitBytesTotal, metrics.Sample{Value: float64(state.Counters.BytesSent), Labels: labels}) out.AddSamples(metrics.NetworkTransmitPacketsTotal, metrics.Sample{Value: float64(state.Counters.PacketsSent), Labels: labels}) out.AddSamples(metrics.NetworkReceiveErrsTotal, metrics.Sample{Value: float64(state.Counters.ErrorsReceived), Labels: labels}) out.AddSamples(metrics.NetworkTransmitErrsTotal, metrics.Sample{Value: float64(state.Counters.ErrorsSent), Labels: labels}) out.AddSamples(metrics.NetworkReceiveDropTotal, metrics.Sample{Value: float64(state.Counters.PacketsDroppedInbound), Labels: labels}) out.AddSamples(metrics.NetworkTransmitDropTotal, metrics.Sample{Value: float64(state.Counters.PacketsDroppedOutbound), Labels: labels}) } // Get number of processes pids, err := d.processesState(d.InitPID()) if err != nil { d.logger.Warn("Failed to get total number of processes", logger.Ctx{"err": err}) } else { out.AddSamples(metrics.ProcsTotal, metrics.Sample{Value: float64(pids)}) } // Set the timestamps startedAt, err := d.processStartedAt(d.InitPID()) if err != nil { d.logger.Warn("Failed to get instance startup time", logger.Ctx{"err": err}) } else { out.AddSamples(metrics.BootTimeSeconds, metrics.Sample{Value: float64(startedAt.Unix())}) } out.AddSamples(metrics.TimeSeconds, metrics.Sample{Value: float64(time.Now().Unix())}) return out, nil } func (d *lxc) getFSStats() (*metrics.MetricSet, error) { type mountInfo struct { Mountpoint string FSType string } out := metrics.NewMetricSet(nil) mounts, err := os.ReadFile("/proc/mounts") if err != nil { return nil, fmt.Errorf("Failed to read /proc/mounts: %w", err) } mountMap := make(map[string]mountInfo) scanner := bufio.NewScanner(bytes.NewReader(mounts)) for scanner.Scan() { fields := strings.Fields(scanner.Text()) mountMap[fields[0]] = mountInfo{Mountpoint: fields[1], FSType: fields[2]} } // Get disk devices for _, dev := range d.expandedDevices { if dev["type"] != "disk" || dev["path"] == "" { continue } var statfs *unix.Statfs_t labels := make(map[string]string) realDev := "" if dev["pool"] != "" { // Expected volume name. var volName string var volType storageDrivers.VolumeType if dev["source"] != "" { volName = project.StorageVolume(d.project.Name, dev["source"]) volType = storageDrivers.VolumeTypeCustom } else { volName = project.Instance(d.project.Name, d.name) volType = storageDrivers.VolumeTypeContainer } // Check that we have a mountpoint. mountpoint := storageDrivers.GetVolumeMountPath(dev["pool"], volType, volName) if mountpoint == "" || !util.PathExists(mountpoint) { continue } // Grab the filesystem information. statfs, err = linux.StatVFS(mountpoint) if err != nil { return nil, fmt.Errorf("Failed to stat %s: %w", mountpoint, err) } // Grab the pool information to compare. poolStatfs, err := linux.StatVFS(internalUtil.VarPath("storage-pools", dev["pool"])) if err != nil { return nil, fmt.Errorf("Failed to stat %s: %w", mountpoint, err) } // Check if we have actual mount-specific information. if statfs.Type == poolStatfs.Type && statfs.Blocks == poolStatfs.Blocks && statfs.Bfree == poolStatfs.Bfree && statfs.Bavail == poolStatfs.Bavail { continue } // Check if mountPath is in mountMap isMounted := false for mountDev, mountInfo := range mountMap { if mountInfo.Mountpoint != mountpoint { continue } isMounted = true realDev = mountDev break } if !isMounted { realDev = dev["source"] } } else { // Skip special disks. if device.IsSpecialDisk(dev["source"]) { continue } source := dev["source"] statfs, err = linux.StatVFS(source) if err != nil { return nil, fmt.Errorf("Failed to stat %s: %w", dev["source"], err) } isMounted := false // Check if mountPath is in mountMap for mountDev, mountInfo := range mountMap { if mountInfo.Mountpoint != source { continue } isMounted = true stat := unix.Stat_t{} // Check if dev has a backing file err = unix.Stat(source, &stat) if err != nil { return nil, fmt.Errorf("Failed to stat %s: %w", dev["source"], err) } backingFilePath := fmt.Sprintf("/sys/dev/block/%d:%d/loop/backing_file", unix.Major(uint64(stat.Dev)), unix.Minor(uint64(stat.Dev))) if util.PathExists(backingFilePath) { // Read backing file backingFile, err := os.ReadFile(backingFilePath) if err != nil { return nil, fmt.Errorf("Failed to read %s: %w", backingFilePath, err) } realDev = string(backingFile) } else { // Use dev as device realDev = mountDev } break } if !isMounted { realDev = dev["source"] } } // Add labels labels["device"] = realDev labels["mountpoint"] = dev["path"] fsType, err := linux.FSTypeToName(int32(statfs.Type)) if err == nil { labels["fstype"] = fsType } // Add sample statfsBsize := uint64(statfs.Bsize) out.AddSamples(metrics.FilesystemSizeBytes, metrics.Sample{Value: float64(statfs.Blocks * statfsBsize), Labels: labels}) out.AddSamples(metrics.FilesystemAvailBytes, metrics.Sample{Value: float64(statfs.Bavail * statfsBsize), Labels: labels}) out.AddSamples(metrics.FilesystemFreeBytes, metrics.Sample{Value: float64(statfs.Bfree * statfsBsize), Labels: labels}) } return out, nil } func (d *lxc) loadRawLXCConfig(cc *liblxc.Container) error { // Load the LXC raw config. lxcConfig, ok := d.expandedConfig["raw.lxc"] if !ok { return nil } // Write to temp config file. f, err := os.CreateTemp("", "incus_config_") if err != nil { return err } err = internalIO.WriteAll(f, []byte(lxcConfig)) if err != nil { return err } err = f.Close() if err != nil { return err } // Load the config. err = cc.LoadConfigFile(f.Name()) if err != nil { return fmt.Errorf("Failed to load config file %q: %w", f.Name(), err) } _ = os.Remove(f.Name()) return nil } // forfileRunningLockName returns the forkfile-running_ID lock name. func (d *common) forkfileRunningLockName() string { return fmt.Sprintf("forkfile-running_%d", d.id) } // ReloadDevice triggers an empty Update call to the underlying device. func (d *lxc) ReloadDevice(devName string) error { dev, err := d.deviceLoad(d, devName, d.expandedDevices[devName], false) if err != nil { return err } return dev.Update(d.expandedDevices, true) } // CanLiveMigrate returns whether the container is live-migratable. func (d *lxc) CanLiveMigrate() bool { return util.IsTrue(d.expandedConfig["migration.stateful"]) } // setupCredentials sets up the systemd credentials directory. func (d *lxc) setupCredentials(update bool) error { // Skip updating if the container isn't running. if update && !d.IsRunning() { return nil } credentialsDir := filepath.Join(d.Path(), "credentials") credentials := map[string][]byte{} oldCredentials := map[string]bool{} var rootUID, rootGID int64 idmapset, err := d.NextIdmap() if err != nil { return err } if idmapset != nil { rootUID, rootGID = idmapset.ShiftIntoNS(0, 0) } for k, v := range d.expandedConfig { after, ok := strings.CutPrefix(k, "systemd.credential.") if ok { credentials[after] = []byte(v) continue } after, ok = strings.CutPrefix(k, "systemd.credential-binary.") if ok { data, err := base64.RawStdEncoding.DecodeString(strings.TrimRight(v, "=")) if err != nil { return fmt.Errorf("Invalid base64 value for %q: %q", k, v) } credentials[after] = data } } // Cleanup the credentials directory. if update && util.PathExists(credentialsDir) { credEntries, err := os.ReadDir(credentialsDir) if err != nil { return fmt.Errorf("Failed to list credentials directory: %w", err) } for _, entry := range credEntries { oldCredentials[entry.Name()] = true } } else { _ = os.RemoveAll(credentialsDir) err = internalUtil.MkdirAllOwner(credentialsDir, 0o100, int(rootUID), int(rootGID)) if err != nil { return fmt.Errorf("Failed to create credentials directory: %w", err) } } credsRoot, err := os.OpenRoot(credentialsDir) if err != nil { return fmt.Errorf("Failed to open the credentials directory: %w", err) } defer func() { _ = credsRoot.Close() }() for k, v := range credentials { err := credsRoot.WriteFile(k, v, 0o400) if err != nil { return fmt.Errorf("Failed to write credential %q: %w", k, err) } err = credsRoot.Chown(k, int(rootUID), int(rootGID)) if err != nil { return fmt.Errorf("Failed setting permissions for file %q: %w", k, err) } delete(oldCredentials, k) } for oldCredential := range oldCredentials { err = os.Remove(filepath.Join(credentialsDir, oldCredential)) if err != nil { return fmt.Errorf("Failed to remove credential %q: %w", oldCredential, err) } } return nil } // GuestOS returns the guest OS. For containers, we can safely assume Linux. func (d *lxc) GuestOS() string { return "linux" } // CreateQcow2Snapshot creates a qcow2 snapshot for a running instance. Not supported by containers. func (d *lxc) CreateQcow2Snapshot(devPath string, devName string, snapName string, backingFilename string, stateful bool) error { return instance.ErrNotImplemented } // DeleteQcow2Snapshot deletes a qcow2 snapshot for a running instance. Not supported by containers. func (d *lxc) DeleteQcow2Snapshot(devName string, snapshotIndex int, backingFilename string) error { return instance.ErrNotImplemented } // ExportQcow2Block exports a qcow2 block device. Not supported by containers. func (d *lxc) ExportQcow2Block(diskName string, diskIndex int) (func(), string, error) { return nil, "", instance.ErrNotImplemented } // ConnectNBD exports a disk over NBD. Not supported by containers. func (d *lxc) ConnectNBD(diskName string, volSize int64, writable bool) (net.Conn, func(), error) { return nil, nil, instance.ErrNotImplemented } // setNICLink sets the link status of the given device. func (d *lxc) setNICLink(devName string, connected bool, assumeUp bool) error { // This check is added so that devices that cannot handle link states do not fail to initialize. if connected && assumeUp { return nil } link, err := ip.LinkByName(d.localConfig["volatile."+devName+".host_name"]) if err != nil { return fmt.Errorf("Failed to find interface %s: %w", devName, err) } if connected { err = link.SetUp() if err != nil { return fmt.Errorf("Failed to bring %s up: %w", devName, err) } } else { err = link.SetDown() if err != nil { return fmt.Errorf("Failed to bring %s down: %w", devName, err) } } return nil } // CreateBitmap creates a dirty bitmap. Not supported by containers. func (d *lxc) CreateBitmap(deviceNames []string, data api.StorageVolumeBitmapsPost) error { return instance.ErrNotImplemented } // DeleteBitmap deletes a dirty bitmap. Not supported by containers. func (d *lxc) DeleteBitmap(deviceName string, bitmapName string) error { return instance.ErrNotImplemented } // GetBitmaps fetches dirty bitmaps. Not supported by containers. func (d *lxc) GetBitmaps(deviceName string) ([]api.StorageVolumeBitmap, error) { return nil, instance.ErrNotImplemented } incus-7.0.0/internal/server/instance/drivers/driver_lxc_cmd.go000066400000000000000000000024321517523235500245670ustar00rootroot00000000000000package drivers import ( "os/exec" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/shared/logger" ) // lxcCmd represents a running command for an LXC container. type lxcCmd struct { attachedChildPid int cmd *exec.Cmd } // PID returns the attached child's process ID. func (c *lxcCmd) PID() int { return c.attachedChildPid } // Signal sends a signal to the command. func (c *lxcCmd) Signal(sig unix.Signal) error { err := unix.Kill(c.attachedChildPid, sig) if err != nil { return err } logger.Debugf(`Forwarded signal "%d" to PID "%d"`, sig, c.PID()) return nil } // Wait for the command to end and returns its exit code and any error. func (c *lxcCmd) Wait() (int, error) { exitStatus, err := linux.ExitStatus(c.cmd.Wait()) // Convert special exit statuses into errors. switch exitStatus { case 127: err = ErrExecCommandNotFound case 126: err = ErrExecCommandNotExecutable } return exitStatus, err } // WindowResize resizes the running command's window. func (c *lxcCmd) WindowResize(fd, winchWidth, winchHeight int) error { err := linux.SetPtySize(fd, winchWidth, winchHeight) if err != nil { return err } logger.Debugf(`Set window size "%dx%d" of PID "%d"`, winchWidth, winchHeight, c.PID()) return nil } incus-7.0.0/internal/server/instance/drivers/driver_qemu.go000066400000000000000000011747751517523235500241520ustar00rootroot00000000000000package drivers import ( "bufio" "bytes" "compress/gzip" "context" "crypto/tls" "crypto/x509" "database/sql" "embed" "encoding/base64" "encoding/json" "errors" "fmt" "io" "io/fs" "maps" "net" "net/http" "net/url" "os" "os/exec" "path/filepath" "regexp" "slices" "sort" "strconv" "strings" "time" "unsafe" "github.com/flosch/pongo2/v6" "github.com/google/uuid" "github.com/gorilla/websocket" "github.com/kballard/go-shellquote" "github.com/mdlayher/vsock" "github.com/pkg/sftp" "go.yaml.in/yaml/v4" "golang.org/x/sync/errgroup" "golang.org/x/sys/unix" "google.golang.org/protobuf/proto" incus "github.com/lxc/incus/v7/client" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/jmap" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/ports" "github.com/lxc/incus/v7/internal/server/apparmor" "github.com/lxc/incus/v7/internal/server/cgroup" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/device" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/device/nictype" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/drivers/cfg" "github.com/lxc/incus/v7/internal/server/instance/drivers/edk2" "github.com/lxc/incus/v7/internal/server/instance/drivers/qemudefault" "github.com/lxc/incus/v7/internal/server/instance/drivers/qmp" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/instance/operationlock" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/metrics" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/scriptlet" scriptletLoad "github.com/lxc/incus/v7/internal/server/scriptlet/load" "github.com/lxc/incus/v7/internal/server/state" storagePools "github.com/lxc/incus/v7/internal/server/storage" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" localUtil "github.com/lxc/incus/v7/internal/server/util" localvsock "github.com/lxc/incus/v7/internal/server/vsock" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" agentAPI "github.com/lxc/incus/v7/shared/api/agent" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // incus-agent files // //go:embed agent-loader/* var incusAgentLoader embed.FS // qemuSerialChardevName is used to communicate state with QEMU via QMP. const qemuSerialChardevName = "qemu_serial-chardev" // qemuPCIDeviceIDStart is the first PCI slot used for user configurable devices. const qemuPCIDeviceIDStart = 4 // qemuDeviceIDPrefix used as part of the name given QEMU devices generated from user added devices. const qemuDeviceIDPrefix = "dev-incus_" // qemuNetDevIDPrefix used as part of the name given QEMU netdevs generated from user added devices. const qemuNetDevIDPrefix = "incus_" // qemuBlockDevIDPrefix used as part of the name given QEMU blockdevs generated from user added devices. const qemuBlockDevIDPrefix = "incus_" // qemuMountTagMaxLength defines the maximum number of characters allowed for the mount tag added to the QEMU configuration. const qemuMountTagMaxLength = 30 // qemuMountTag9pMaxLength defines the maximum number of characters allowed for the mount tag added to the QEMU configuration when using 9p. const qemuMountTag9pMaxLength = 25 // qemuMountTagPrefix is the prefix used for QEMU mount tags for directory shares. const qemuMountTagPrefix = "incus_" // qemuSparseUSBPorts is the amount of sparse USB ports for VMs. // 4 are reserved, and the other 4 can be used for any USB device. const qemuSparseUSBPorts = 8 var errQemuAgentOffline = errors.New("VM agent isn't currently running") type monitorHook func(m *qmp.Monitor) error // qemuLoad creates a Qemu instance from the supplied InstanceArgs. func qemuLoad(s *state.State, args db.InstanceArgs, p api.Project) (instance.Instance, error) { // Create the instance struct. d := qemuInstantiate(s, args, nil, p) // Expand config and devices. err := d.expandConfig() if err != nil { return nil, err } return d, nil } // qemuInstantiate creates a Qemu struct without expanding config. The expandedDevices argument is // used during device config validation when the devices have already been expanded and we do not // have access to the profiles used to do it. This can be safely passed as nil if not required. func qemuInstantiate(s *state.State, args db.InstanceArgs, expandedDevices deviceConfig.Devices, p api.Project) *qemu { d := &qemu{ common: common{ state: s, architecture: args.Architecture, creationDate: args.CreationDate, dbType: args.Type, description: args.Description, ephemeral: args.Ephemeral, expiryDate: args.ExpiryDate, id: args.ID, lastUsedDate: args.LastUsedDate, localConfig: args.Config, localDevices: args.Devices, logger: logger.AddContext(logger.Ctx{"instanceType": args.Type, "instance": args.Name, "project": args.Project}), name: args.Name, node: args.Node, profiles: args.Profiles, project: p, isSnapshot: args.Snapshot, stateful: args.Stateful, }, } // Get the architecture name. archName, err := osarch.ArchitectureName(d.architecture) if err == nil { d.architectureName = archName } // Cleanup the zero values. if d.expiryDate.IsZero() { d.expiryDate = time.Time{} } if d.creationDate.IsZero() { d.creationDate = time.Time{} } if d.lastUsedDate.IsZero() { d.lastUsedDate = time.Time{} } // This is passed during expanded config validation. if expandedDevices != nil { d.expandedDevices = expandedDevices } return d } // qemuCreate creates a new storage volume record and returns an initialized Instance. // Returns a revert fail function that can be used to undo this function if a subsequent step fails. func qemuCreate(s *state.State, args db.InstanceArgs, p api.Project, partialDeviceValidation bool, op *operations.Operation) (instance.Instance, revert.Hook, error) { reverter := revert.New() defer reverter.Fail() // Create the instance struct. d := &qemu{ common: common{ state: s, op: op, architecture: args.Architecture, creationDate: args.CreationDate, dbType: args.Type, description: args.Description, ephemeral: args.Ephemeral, expiryDate: args.ExpiryDate, id: args.ID, lastUsedDate: args.LastUsedDate, localConfig: args.Config, localDevices: args.Devices, logger: logger.AddContext(logger.Ctx{"instanceType": args.Type, "instance": args.Name, "project": args.Project}), name: args.Name, node: args.Node, profiles: args.Profiles, project: p, isSnapshot: args.Snapshot, stateful: args.Stateful, }, } // Get the architecture name. archName, err := osarch.ArchitectureName(d.architecture) if err == nil { d.architectureName = archName } // Cleanup the zero values. if d.expiryDate.IsZero() { d.expiryDate = time.Time{} } if d.creationDate.IsZero() { d.creationDate = time.Time{} } if d.lastUsedDate.IsZero() { d.lastUsedDate = time.Time{} } if args.Snapshot { d.logger.Info("Creating instance snapshot", logger.Ctx{"ephemeral": d.ephemeral}) } else { d.logger.Info("Creating instance", logger.Ctx{"ephemeral": d.ephemeral}) } // Load the config. err = d.init() if err != nil { return nil, nil, fmt.Errorf("Failed to expand config: %w", err) } // When not a snapshot, perform full validation. if !args.Snapshot { // Validate expanded config (allows mixed instance types for profiles). err = instance.ValidConfig(s.OS, d.expandedConfig, true, instancetype.Any) if err != nil { return nil, nil, fmt.Errorf("Invalid config: %w", err) } err = instance.ValidDevices(s, d.project, d.Type(), d.localDevices, d.expandedDevices) if err != nil { return nil, nil, fmt.Errorf("Invalid devices: %w", err) } } // Retrieve the instance's storage pool. _, rootDiskDevice, err := d.getRootDiskDevice() if err != nil { return nil, nil, fmt.Errorf("Failed getting root disk: %w", err) } if rootDiskDevice["pool"] == "" { return nil, nil, errors.New("The instance's root device is missing the pool property") } // Initialize the storage pool. d.storagePool, err = storagePools.LoadByName(d.state, rootDiskDevice["pool"]) if err != nil { return nil, nil, fmt.Errorf("Failed loading storage pool: %w", err) } volType, err := storagePools.InstanceTypeToVolumeType(d.Type()) if err != nil { return nil, nil, err } storagePoolSupported := slices.Contains(d.storagePool.Driver().Info().VolumeTypes, volType) if !storagePoolSupported { return nil, nil, errors.New("Storage pool does not support instance type") } if !d.IsSnapshot() { // Add devices to instance. cleanup, err := d.devicesAdd(d, false, partialDeviceValidation) if err != nil { return nil, nil, err } reverter.Add(cleanup) } if d.isSnapshot { d.logger.Info("Created instance snapshot", logger.Ctx{"ephemeral": d.ephemeral}) } else { d.logger.Info("Created instance", logger.Ctx{"ephemeral": d.ephemeral}) } if d.isSnapshot { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceSnapshotCreated.Event(d, nil)) } else { err = d.state.Authorizer.AddInstance(d.state.ShutdownCtx, d.project.Name, d.Name()) if err != nil { logger.Error("Failed to add instance to authorizer", logger.Ctx{"name": d.Name(), "project": d.project.Name, "error": err}) } reverter.Add(func() { _ = d.state.Authorizer.DeleteInstance(d.state.ShutdownCtx, d.project.Name, d.Name()) }) d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceCreated.Event(d, map[string]any{ "type": api.InstanceTypeVM, "storage-pool": d.storagePool.Name(), "location": d.Location(), })) } cleanup := reverter.Clone().Fail reverter.Success() return d, cleanup, err } // qemu is the QEMU virtual machine driver. type qemu struct { common // Cached handles. // Do not use these variables directly, instead use their associated get functions so they // will be initialized on demand. architectureName string // Stateful migration streams. migrationReceiveStateful map[string]io.ReadWriteCloser // Indicate whether the root disk will be live-migrated. migrationRootDisk bool disksToMigrate []localMigration.DependentVolumeArgs // Indicates whether this is an inner-cluster or cross-cluster move. migrationClusterMove bool // Keep a reference to the console socket when switching backends, so we can properly cleanup when switching back to a ring buffer. consoleSocket *net.UnixListener consoleSocketFile *os.File // Keep a record of QEMU configuration. cmdArgs []string conf []cfg.Section } // qmpConnect connects to the QMP monitor. func (d *qemu) qmpConnect() (*qmp.Monitor, error) { return qmp.Connect(d.monitorPath(), qemuSerialChardevName, d.getMonitorEventHandler(), d.QMPLogFilePath(), qemuDetachDisk(d.state, d.id)) } // getAgentClient returns the current agent client handle. // Callers should check that the instance is running (and therefore mounted) before calling this function, // otherwise the qmp.Connect call will fail to use the monitor socket file. func (d *qemu) getAgentClient() (*http.Client, error) { // Check that the VM is in a state where the agent may be reachable. status := d.statusCode() if !d.isRunningStatusCode(status) || status == api.Frozen { return nil, errQemuAgentOffline } // Only Linux and Windows support VirtIO vsock. if slices.Contains([]string{"darwin", "freebsd"}, d.GuestOS()) { // Get known network details. networks, err := d.getNetworkState() if err != nil { return nil, errQemuAgentOffline } // The connection uses mutual authentication, so use the server's key & cert for client. agentCert, _, clientCert, clientKey, err := d.generateAgentCert() if err != nil { return nil, err } // Get the TLS configuration. tlsConfig, err := localtls.GetTLSConfigMem(clientCert, clientKey, "", agentCert, false) if err != nil { return nil, err } // Setup an HTTPS client. client := &http.Client{} client.CheckRedirect = func(req *http.Request, via []*http.Request) error { // Replicate the headers. req.Header = via[len(via)-1].Header return nil } for _, netInterface := range networks { for _, address := range netInterface.Addresses { if address.Scope != "global" { continue } networkAddress := net.JoinHostPort(address.Address, strconv.Itoa(ports.HTTPSDefaultPort)) client.Transport = &http.Transport{ TLSClientConfig: tlsConfig, DialContext: func(_ context.Context, network, addr string) (net.Conn, error) { return net.DialTimeout("tcp", networkAddress, 100*time.Millisecond) }, DisableKeepAlives: true, ExpectContinueTimeout: time.Second * 3, ResponseHeaderTimeout: time.Second * 3600, TLSHandshakeTimeout: time.Second * 3, } _, err := client.Get("https://agent/") if err == nil { return client, nil } } } return nil, errQemuAgentOffline } // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return nil, err } if !monitor.AgenStarted() { return nil, errQemuAgentOffline } // The connection uses mutual authentication, so use the server's key & cert for client. agentCert, _, clientCert, clientKey, err := d.generateAgentCert() if err != nil { return nil, err } // Existing vsock ID from volatile. vsockID, err := d.getVsockID() if err != nil { return nil, err } agent, err := localvsock.HTTPClient(vsockID, ports.HTTPSDefaultPort, clientCert, clientKey, agentCert) if err != nil { return nil, err } return agent, nil } func (d *qemu) getMonitorEventHandler() func(event string, data map[string]any) { // Create local variables from instance properties we need so as not to keep references to instance around // after we have returned the callback function. instProject := d.Project() instanceName := d.Name() state := d.state return func(event string, data map[string]any) { if !slices.Contains([]string{qmp.EventVMShutdown, qmp.EventVMReset, qmp.EventAgentStarted, qmp.EventAgentStopped, qmp.EventRTCChange, qmp.EventBlockJobCompleted, qmp.EventBlockJobError}, event) { return // Don't bother loading the instance from DB if we aren't going to handle the event. } var err error var d *qemu // Redefine d as local variable inside callback to avoid keeping references around. inst := instanceRefGet(instProject.Name, instanceName) if inst == nil { inst, err = instance.LoadByProjectAndName(state, instProject.Name, instanceName) if err != nil { l := logger.AddContext(logger.Ctx{"project": instProject.Name, "instance": instanceName}) // If DB not available, try loading from backup file. l.Warn("Failed loading instance from database to handle monitor event, trying backup file", logger.Ctx{"err": err}) instancePath := filepath.Join(internalUtil.VarPath("virtual-machines"), project.Instance(instProject.Name, instanceName)) inst, err = instance.LoadFromBackup(state, instProject.Name, instancePath, false) if err != nil { l.Error("Failed loading instance to handle monitor event", logger.Ctx{"err": err}) return } } } d = inst.(*qemu) switch event { case qmp.EventAgentStarted: d.logger.Debug("Instance agent started") err := d.advertiseVsockAddress() if err != nil { d.logger.Warn("Failed to advertise vsock address to instance agent", logger.Ctx{"err": err}) return } state.Events.SendLifecycle(instProject.Name, lifecycle.InstanceAgentStarted.Event(d, nil)) case qmp.EventAgentStopped: d.logger.Debug("Instance agent stopped") state.Events.SendLifecycle(instProject.Name, lifecycle.InstanceAgentStopped.Event(d, nil)) case qmp.EventVMReset: monitor, err := d.qmpConnect() if err == nil { if monitor.HandleReset() { // This RESET corresponds to a deliberate system_reset we triggered // (e.g. the boot-config rebuild during startup), so let QEMU handle // it internally rather than tearing the VM down. break } if !d.needsFullRestart() { // If a quick restart is possible, let QEMU handle it. d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceRestarted.Event(d, nil)) break } } fallthrough case qmp.EventVMShutdown: var reason string target := "stop" entry, ok := data["reason"] if ok { entryStr, ok := entry.(string) if ok { reason = entryStr } } if reason == "guest-reset" { target = "reboot" } if reason == qmp.EventVMShutdownReasonDisconnect { d.logger.Warn("Instance stopped", logger.Ctx{"target": target, "reason": data["reason"]}) } else { d.logger.Debug("Instance stopped", logger.Ctx{"target": target, "reason": data["reason"]}) } err = d.onStop(target, reason) if err != nil { d.logger.Error("Failed to cleanly stop instance", logger.Ctx{"err": err}) return } case qmp.EventRTCChange: val, ok := data["offset"].(float64) if !ok { d.logger.Debug("No offset in data", logger.Ctx{"data": data}) return } err = d.onRTCChange(int(val)) if err != nil { d.logger.Error("Failed to apply rtc change", logger.Ctx{"offset": val, "err": err}) } case qmp.EventBlockJobCompleted, qmp.EventBlockJobError: monitor, _ := d.qmpConnect() monitor.PushEvent(event, data) monitor.CleanupEventChannel(data["device"].(string)) } } } // mount the instance's config volume if needed. func (d *qemu) mount() (*storagePools.MountInfo, error) { var pool storagePools.Pool pool, err := d.getStoragePool() if err != nil { return nil, err } if d.IsSnapshot() { mountInfo, err := pool.MountInstanceSnapshot(d, nil) if err != nil { return nil, err } return mountInfo, nil } mountInfo, err := pool.MountInstance(d, nil) if err != nil { return nil, err } return mountInfo, nil } // unmount the instance's config volume if needed. func (d *qemu) unmount() error { pool, err := d.getStoragePool() if err != nil { return err } err = pool.UnmountInstance(d, nil) if err != nil { return err } return nil } // generateAgentCert creates the necessary server key and certificate if needed. func (d *qemu) generateAgentCert() (string, string, string, string, error) { agentCertFile := filepath.Join(d.Path(), "agent.crt") agentKeyFile := filepath.Join(d.Path(), "agent.key") clientCertFile := filepath.Join(d.Path(), "agent-client.crt") clientKeyFile := filepath.Join(d.Path(), "agent-client.key") // Create server certificate. err := localtls.FindOrGenCert(agentCertFile, agentKeyFile, false, false) if err != nil { return "", "", "", "", err } // Create client certificate. err = localtls.FindOrGenCert(clientCertFile, clientKeyFile, true, false) if err != nil { return "", "", "", "", err } // Read all the files agentCert, err := os.ReadFile(agentCertFile) if err != nil { return "", "", "", "", err } agentKey, err := os.ReadFile(agentKeyFile) if err != nil { return "", "", "", "", err } clientCert, err := os.ReadFile(clientCertFile) if err != nil { return "", "", "", "", err } clientKey, err := os.ReadFile(clientKeyFile) if err != nil { return "", "", "", "", err } return string(agentCert), string(agentKey), string(clientCert), string(clientKey), nil } // Freeze freezes the instance. func (d *qemu) Freeze() error { // Connect to the monitor. monitor, err := d.qmpConnect() if err != nil { return err } // Send the stop command. err = monitor.Pause() if err != nil { return err } d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstancePaused.Event(d, nil)) return nil } // configDriveMountPath returns the path for the config drive bind mount. func (d *qemu) configDriveMountPath() string { return filepath.Join(d.DevicesPath(), "config.mount") } // configDriveMountPathClear attempts to unmount the config drive bind mount and remove the directory. func (d *qemu) configDriveMountPathClear() error { return device.DiskMountClear(d.configDriveMountPath()) } // pidWait waits for the QEMU process to exit. Does this in a way that doesn't require the process to be a // parent of the QEMU process (in order to allow for the daemon to be restarted after the VM was started). // Returns true if process stopped, false if timeout was exceeded. func (d *qemu) pidWait(timeout time.Duration) bool { waitUntil := time.Now().Add(timeout) for { pid, _ := d.pid() if pid <= 0 { break } if time.Now().After(waitUntil) { return false } time.Sleep(time.Millisecond * time.Duration(250)) } return true } // onStop is run when the instance stops. func (d *qemu) onStop(target string, reason string) error { d.logger.Debug("onStop hook started", logger.Ctx{"target": target, "reason": reason}) defer d.logger.Debug("onStop hook finished", logger.Ctx{"target": target, "reason": reason}) // Create/pick up operation. op, err := d.onStopOperationSetup(target) if err != nil { return err } // Unlock on return defer op.Done(nil) // Set operation if missing. if d.op == nil { d.op = op.GetOperation() } // If QEMU is still running, stop it (handles reboot). monitor, err := qmp.Connect(d.monitorPath(), qemuSerialChardevName, nil, d.QMPLogFilePath(), qemuDetachDisk(d.state, d.id)) if err == nil { _ = monitor.Quit() } // Wait for QEMU process to end (to avoiding racing start when restarting). // Wait up to 5 minutes to allow for flushing any pending data to disk. d.logger.Debug("Waiting for VM process to finish") waitTimeout := time.Minute * 5 if d.pidWait(waitTimeout) { d.logger.Debug("VM process finished") } else { // Log a warning, but continue clean up as best we can. d.logger.Error("VM process failed to stop", logger.Ctx{"timeout": waitTimeout}) } // Record power state. err = d.VolatileSet(map[string]string{ "volatile.last_state.power": instance.PowerStateStopped, "volatile.last_state.ready": "false", }) if err != nil { // Don't return an error here as we still want to cleanup the instance even if DB not available. d.logger.Error("Failed recording last power state", logger.Ctx{"err": err}) } // Cleanup. d.cleanupDevices() // Must be called before unmount. _ = os.Remove(d.pidFilePath()) _ = os.Remove(d.monitorPath()) _ = os.Remove(d.spicePath()) // Stop the storage for the instance. err = d.unmount() if err != nil && !errors.Is(err, storageDrivers.ErrInUse) { err = fmt.Errorf("Failed unmounting instance: %w", err) op.Done(err) return err } // Unload the apparmor profile err = apparmor.InstanceUnload(d.state.OS, d) if err != nil { op.Done(err) return err } // Determine if instance should be auto-restarted. cleanShutdown := reason == qmp.EventVMShutdownReasonGuestShutdown || reason == qmp.EventVMShutdownReasonQuit var autoRestart bool if target != "reboot" && !cleanShutdown && d.shouldAutoRestart() { autoRestart = true // Mark current shutdown as complete. op.Done(nil) // Create a new restart operation. op, err = operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionRestart, nil, true, false) if err == nil { defer op.Done(nil) } else { d.logger.Error("Failed to setup new restart operation", logger.Ctx{"err": err}) } } // Log and emit lifecycle if not user triggered. if target != "reboot" && !autoRestart && op.Action() != operationlock.ActionMigrate && op.Action() != operationlock.ActionRestart { if op.GetInstanceInitiated() { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceShutdown.Event(d, nil)) } else { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceStopped.Event(d, nil)) } // agent stopped when shutdown d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceAgentStopped.Event(d, nil)) } // Reboot the instance. if target == "reboot" || autoRestart { err = d.Start(false) if err != nil { op.Done(err) return err } d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceRestarted.Event(d, nil)) } else if d.ephemeral { // Destroy ephemeral virtual machines. err = d.delete(true, true) if err != nil { op.Done(err) return err } } return nil } // Shutdown shuts the instance down. func (d *qemu) Shutdown(timeout time.Duration) error { d.logger.Debug("Shutdown started", logger.Ctx{"timeout": timeout}) defer d.logger.Debug("Shutdown finished", logger.Ctx{"timeout": timeout}) // Must be run prior to creating the operation lock. statusCode := d.statusCode() if !d.isRunningStatusCode(statusCode) { if statusCode == api.Error { return fmt.Errorf("The instance cannot be cleanly shutdown as in %s status", statusCode) } return ErrInstanceIsStopped } // Save the console log from ring buffer before the instance is shutdown. Must be run prior to creating the operation lock. _, err := d.ConsoleLog() if err != nil { return err } // Setup a new operation. // Allow inheriting of ongoing restart operation (we are called from restartCommon). // Allow reuse when creating a new stop operation. This allows the Stop() function to inherit operation. // Allow reuse of a reusable ongoing stop operation as Shutdown() may be called earlier, which allows reuse // of its operations. This allow for multiple Shutdown() attempts. op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStop, []operationlock.Action{operationlock.ActionRestart}, true, true) if err != nil { if errors.Is(err, operationlock.ErrNonReusuableSucceeded) { // An existing matching operation has now succeeded, return. return nil } return err } // If frozen, resume so the signal can be handled. if d.IsFrozen() { err := d.Unfreeze() if err != nil { return err } } // Connect to the monitor. monitor, err := d.qmpConnect() if err != nil { op.Done(err) return err } // Indicate to the onStop hook that if the VM stops it was due to a clean shutdown because the VM responded // to the powerdown request. op.SetInstanceInitiated(true) // Send the system_powerdown command. err = monitor.Powerdown() if err != nil { if errors.Is(err, qmp.ErrMonitorDisconnect) { op.Done(nil) return nil } op.Done(err) return err } // Wait 500ms for the first event to be received by the guest. time.Sleep(500 * time.Millisecond) // Attempt to send a second system_powerdown command (required to get Windows to shutdown). _ = monitor.Powerdown() d.logger.Debug("Shutdown request sent to instance") ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() // Wait for operation lock to be Done or context to timeout. The operation lock is normally completed by // onStop which picks up the same lock and then marks it as Done after the instance stops and the devices // have been cleaned up. However if the operation has failed for another reason we collect the error here. err = op.Wait(ctx) status := d.statusCode() if status != api.Stopped { errPrefix := fmt.Errorf("Failed shutting down instance, status is %q", status) if err != nil { return fmt.Errorf("%s: %w", errPrefix.Error(), err) } return errPrefix } // Now handle errors from shutdown sequence and return to caller if wasn't completed cleanly. if err != nil { return err } return nil } // Restart restart the instance. func (d *qemu) Restart(timeout time.Duration) error { return d.restartCommon(d, timeout) } // Rebuild rebuilds the instance using the supplied image fingerprint as source. func (d *qemu) Rebuild(img *api.Image, op *operations.Operation) error { return d.rebuildCommon(d, img, op) } // killQemuProcess kills specified process. Optimistically attempts to wait for the process to fully exit, but does // not return an error if the Wait call fails. This is because this function is used in scenarios where the daemon has // been restarted after the VM has been started and is no longer the parent of the QEMU process. // The caller should use another method to ensure that the QEMU process has fully exited instead. // Returns an error if the Kill signal couldn't be sent to the process (for any other reason apart from the process // not existing). func (d *qemu) killQemuProcess(pid int) error { proc, err := os.FindProcess(pid) if err != nil { if errors.Is(err, os.ErrProcessDone) { return nil } return err } err = proc.Kill() if err != nil { if errors.Is(err, os.ErrProcessDone) { return nil } return err } // Wait for process to exit, but don't return an error if this fails as it may be called when the daemon isn't // the parent of the process, and we have still sent the kill signal as per the function's description. _, err = proc.Wait() if err != nil { if strings.Contains(err.Error(), "no child processes") { return nil } d.logger.Warn("Failed to collect VM process exit status", logger.Ctx{"pid": pid, "err": err}) } return nil } // restoreState restores the VM state from a file handle. func (d *qemu) restoreStateHandle(ctx context.Context, monitor *qmp.Monitor, f *os.File) error { err := monitor.SendFile("migration", f) if err != nil { return err } err = monitor.MigrateIncoming(ctx, "migration") if err != nil { if errors.Is(err, qmp.ErrMonitorDisconnect) && util.PathExists(d.LogFilePath()) { qemuError, err := os.ReadFile(d.LogFilePath()) if err != nil { return err } return fmt.Errorf("QEMU crashed on VM restore: %s", string(qemuError)) } return err } return nil } // receiveMigrationSnapshot handles an incoming disk snapshot during migration. func (d *qemu) receiveMigrationSnapshot(monitor *qmp.Monitor, blockExport string, filesystemConn io.ReadWriteCloser) error { nbdConn, err := monitor.NBDServerStart() if err != nil { return fmt.Errorf("Failed starting NBD server: %w", err) } d.logger.Debug("Migration NBD server started") defer func() { _ = nbdConn.Close() _ = monitor.NBDServerStop() }() err = monitor.NBDBlockExportAdd(blockExport, true, nil) if err != nil { return fmt.Errorf("Failed adding root disk to NBD server: %w", err) } d.logger.Debug("Migration storage NBD export starting") go func() { _, _ = util.SafeCopy(filesystemConn, nbdConn) }() _, _ = util.SafeCopy(nbdConn, filesystemConn) filesystemConn.Close() d.logger.Debug("Migration storage NBD export finished") return nil } // restoreState restores VM state from state file or from migration source if d.migrationReceiveStateful set. func (d *qemu) restoreState(monitor *qmp.Monitor) error { if d.migrationReceiveStateful != nil { stateConn := d.migrationReceiveStateful[api.SecretNameState] if stateConn == nil { return errors.New("Migration state connection is not initialized") } // Perform non-shared storage transfer if requested. filesystemConn := d.migrationReceiveStateful[api.SecretNameFilesystem] if filesystemConn != nil { go func() { if d.migrationRootDisk { rootDiskName, _, err := internalInstance.GetRootDiskDevice(d.expandedDevices.CloneNative()) if err != nil { d.logger.Error("Failed getting instance root disk", logger.Ctx{"err": err}) return } escapedDeviceName := linux.PathNameEncode(rootDiskName) rootDiskName = d.blockNodeName(escapedDeviceName) err = d.receiveMigrationSnapshot(monitor, rootDiskName, filesystemConn) if err != nil { d.logger.Error("Failed receiving migration snapshot", logger.Ctx{"err": err}) return } } pool, err := d.getStoragePool() if err != nil { d.logger.Error("Failed fetching instance storage pool", logger.Ctx{"err": err}) return } config, err := pool.GenerateInstanceBackupConfig(d, false, true, nil) if err != nil { d.logger.Error("Failed generating instance backup config", logger.Ctx{"err": err}) return } devicesMap := storagePools.DevicesMapFromBackupConfig(config) for _, vol := range d.disksToMigrate { d.logger.Debug("Receiving dependent volume", logger.Ctx{"name": vol.Name, "pool": vol.Pool}) deviceName := storagePools.DeviceByPoolAndVolume(devicesMap, vol.Pool, vol.Name) if deviceName == "" { d.logger.Error("Failed to find requested device", logger.Ctx{"pool": vol.Pool, "volName": vol.Name}) return } diskName := d.blockNodeName(linux.PathNameEncode(deviceName)) err = d.receiveMigrationSnapshot(monitor, diskName, filesystemConn) if err != nil { d.logger.Error("Failed receiving migration snapshot", logger.Ctx{"err": err}) } } }() } // Receive checkpoint from QEMU process on source. d.logger.Debug("Stateful migration checkpoint receive starting") pipeRead, pipeWrite, err := os.Pipe() if err != nil { return err } go func() { _, _ = util.SafeCopy(pipeWrite, stateConn) _ = pipeRead.Close() _ = pipeWrite.Close() }() err = d.restoreStateHandle(context.Background(), monitor, pipeRead) if err != nil { return fmt.Errorf("Failed restoring checkpoint from source: %w", err) } d.logger.Debug("Stateful migration checkpoint receive finished") } else { statePath := d.StatePath() d.logger.Debug("Stateful checkpoint restore starting", logger.Ctx{"source": statePath}) defer d.logger.Debug("Stateful checkpoint restore finished", logger.Ctx{"source": statePath}) stateFile, err := os.Open(statePath) if err != nil { return fmt.Errorf("Failed opening state file %q: %w", statePath, err) } defer func() { _ = stateFile.Close() }() uncompressedState, err := gzip.NewReader(stateFile) if err != nil { return fmt.Errorf("Failed opening state gzip reader: %w", err) } defer func() { _ = uncompressedState.Close() }() pipeRead, pipeWrite, err := os.Pipe() if err != nil { return err } go func() { _, err := util.SafeCopy(pipeWrite, uncompressedState) if err != nil { d.logger.Warn("Failed reading from state file", logger.Ctx{"path": statePath, "err": err}) } _ = pipeRead.Close() _ = pipeWrite.Close() }() err = d.restoreStateHandle(context.Background(), monitor, pipeRead) if err != nil { return fmt.Errorf("Failed restoring state from %q: %w", stateFile.Name(), err) } } return nil } // saveStateHandle dumps the current VM state to a file handle. // Once started, the VM is in a paused state and it's up to the caller to wait for the transfer to complete and // resume or kill the VM guest. func (d *qemu) saveStateHandle(monitor *qmp.Monitor, f *os.File) error { // Send the target file to qemu. err := monitor.SendFile("migration", f) if err != nil { return err } // Issue the migration command. err = monitor.Migrate("migration") if err != nil { return err } return nil } // saveState dumps the current VM state to the state file. // Once dumped, the VM is in a paused state and it's up to the caller to resume or kill it. func (d *qemu) saveState(monitor *qmp.Monitor) error { statePath := d.StatePath() d.logger.Debug("Stateful checkpoint starting", logger.Ctx{"target": statePath}) defer d.logger.Debug("Stateful checkpoint finished", logger.Ctx{"target": statePath}) // Save the checkpoint to state file. _ = os.Remove(statePath) // Prepare the state file. stateFile, err := os.Create(statePath) if err != nil { return err } defer func() { _ = stateFile.Close() }() compressedState, err := gzip.NewWriterLevel(stateFile, gzip.BestSpeed) if err != nil { return err } defer func() { _ = compressedState.Close() }() pipeRead, pipeWrite, err := os.Pipe() if err != nil { return err } defer func() { _ = pipeRead.Close() _ = pipeWrite.Close() }() go func() { _, _ = util.SafeCopy(compressedState, pipeRead) }() err = d.saveStateHandle(monitor, pipeWrite) if err != nil { return fmt.Errorf("Failed initializing state save to %q: %w", stateFile.Name(), err) } err = monitor.MigrateWait(context.Background(), "completed") if err != nil { return fmt.Errorf("Failed saving state to %q: %w", stateFile.Name(), err) } return nil } // validateStartup checks any constraints that would prevent start up from succeeding under normal circumstances. func (d *qemu) validateStartup(stateful bool, statusCode api.StatusCode) error { err := d.common.validateStartup(stateful, statusCode) if err != nil { return err } // Cannot perform stateful start unless config is appropriately set. // NOTE: We can't use CanLiveMigrate during instance startup as the boot state hasn't yet been recorded. if stateful && util.IsFalseOrEmpty(d.expandedConfig["migration.stateful"]) { return errors.New("Stateful start requires migration.stateful to be set to true") } // gendoc:generate(entity=image, group=requirements, key=requirements.secureboot) // // --- // type: bool // shortdesc: If set to `false`, indicates that the image cannot boot under secure boot. // // Ensure secureboot is turned off for images that are not secureboot enabled. if util.IsFalse(d.localConfig["image.requirements.secureboot"]) && util.IsTrueOrEmpty(d.expandedConfig["security.secureboot"]) { return errors.New("The image used by this instance is incompatible with secureboot. Please set security.secureboot=false on the instance") } // Ensure secureboot is turned off when CSM is on. if util.IsTrue(d.expandedConfig["security.csm"]) && util.IsTrueOrEmpty(d.expandedConfig["security.secureboot"]) { return errors.New("Secure boot can't be enabled while CSM is turned on. Please set security.secureboot=false on the instance") } // gendoc:generate(entity=image, group=requirements, key=requirements.cdrom_agent) // // --- // type: bool // shortdesc: If set to `true`, indicates that the VM requires an `agent:config` disk be added. // // Ensure an agent drive is present if the image requires it. if util.IsTrue(d.localConfig["image.requirements.cdrom_agent"]) { found := false for _, dev := range d.expandedDevices { if dev["type"] == "disk" && dev["source"] == "agent:config" { found = true break } } if !found { return errors.New("This virtual machine image requires an agent:config disk be added") } } // gendoc:generate(entity=image, group=requirements, key=requirements.cdrom_cloud_init) // // --- // type: bool // shortdesc: If set to `true`, indicates that the VM requires a `cloud-init:config` disk be present every time `cloud-init` should be run. // // Ensure a `cloud-init` drive is present if the image and the VM state require it. if util.IsTrue(d.localConfig["image.requirements.cdrom_cloud_init"]) && d.localConfig["volatile.apply_template"] != "" { found := false for _, dev := range d.expandedDevices { if dev["type"] == "disk" && dev["source"] == "cloud-init:config" { found = true break } } if !found { return errors.New("This virtual machine image requires a cloud-init:config disk be added") } } return nil } func (d *qemu) checkStateStorage() error { // For some operations, the "size.state" of the instance root disk device must be larger than the instance memory. // Otherwise, there will not be enough disk space to write the instance state to disk during any subsequent stops. // (Only check when migration.stateful is true, otherwise the memory won't be dumped when this instance stops). _, rootDiskDevice, err := d.getRootDiskDevice() if err != nil { return err } // Don't access d.storagePool directly since it isn't populated at this stage. pool, err := d.getStoragePool() if err != nil { return err } stateDiskSizeStr := pool.Driver().Info().DefaultVMBlockFilesystemSize if rootDiskDevice["size.state"] != "" { stateDiskSizeStr = rootDiskDevice["size.state"] } stateDiskSize, err := units.ParseByteSizeString(stateDiskSizeStr) if err != nil { return err } memoryLimitStr := qemudefault.MemSize if d.expandedConfig["limits.memory"] != "" { memoryLimitStr = d.expandedConfig["limits.memory"] } memoryLimit, err := ParseMemoryStr(memoryLimitStr) if err != nil { return err } if stateDiskSize < memoryLimit { return errors.New("Stateful stop and snapshots require the instance limits.memory be less than or equal to the root disk size.state property") } return nil } // Start starts the instance. func (d *qemu) Start(stateful bool) error { return d.start(stateful, nil) } // runStartupScriptlet runs startup scriptlets at config, early, pre-start and post-start stages. func (d *qemu) runStartupScriptlet(monitor *qmp.Monitor, stage string) error { _, ok := d.expandedConfig["raw.qemu.scriptlet"] if ok { // Render cannot return errors here. render, _, _ := d.Render() instanceData, ok := render.(*api.Instance) if !ok { return errors.New("Unexpected instance type") } err := scriptlet.QEMURun(logger.Log, instanceData, &d.cmdArgs, &d.conf, monitor, stage) if err != nil { err = fmt.Errorf("Failed running QEMU scriptlet at %s stage: %w", stage, err) return err } } return nil } // startupHook executes QMP commands and runs startup scriptlets at early, pre-start and post-start // stages. func (d *qemu) startupHook(monitor *qmp.Monitor, stage string) error { commands, ok := d.expandedConfig["raw.qemu.qmp."+stage] if ok { var commandList []map[string]any err := json.Unmarshal([]byte(commands), &commandList) if err != nil { err = fmt.Errorf("Failed to parse QMP commands at %s stage (expected JSON list of objects): %w", stage, err) return err } for _, command := range commandList { id := monitor.IncreaseID() command["id"] = id var jsonCommand []byte jsonCommand, err = json.Marshal(command) if err != nil { err = fmt.Errorf("Failed to marshal command at %s stage: %w", stage, err) return err } err = monitor.RunJSON(jsonCommand, nil, true, id) if err != nil { err = fmt.Errorf("Failed to run QMP command %s at %s stage: %w", jsonCommand, stage, err) return err } } } return d.runStartupScriptlet(monitor, stage) } // start starts the instance and can use an existing InstanceOperation lock. func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error { d.logger.Debug("Start started", logger.Ctx{"stateful": stateful}) defer d.logger.Debug("Start finished", logger.Ctx{"stateful": stateful}) // Check that we are startable before creating an operation lock. // Must happen before creating operation Start lock to avoid the status check returning Stopped due to the // existence of a Start operation lock. err := d.validateStartup(stateful, d.statusCode()) if err != nil { return err } // Setup a new operation if needed. if op == nil { op, err = operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStart, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore}, false, false) if err != nil { if errors.Is(err, operationlock.ErrNonReusuableSucceeded) { // An existing matching operation has now succeeded, return. return nil } return fmt.Errorf("Failed to create instance start operation: %w", err) } } defer op.Done(err) // Record (or load) boot state. bs := &qemuBootState{ Version: qemuBootStateVersion, } if stateful { bs, err = d.getBootState() if err != nil { return err } } // Assign NUMA node(s) if needed. if d.expandedConfig["limits.cpu.nodes"] == "balanced" { err := d.balanceNUMANodes() if err != nil { op.Done(err) return err } } // Ensure the correct vhost_vsock kernel module is loaded before establishing the vsock. err = linux.LoadModule("vhost_vsock") if err != nil { op.Done(err) return err } reverter := revert.New() defer reverter.Fail() // Rotate the log files. for _, logfile := range []string{d.LogFilePath(), d.ConsoleBufferLogPath(), d.QMPLogFilePath()} { if util.PathExists(logfile) { _ = os.Remove(logfile + ".old") err := os.Rename(logfile, logfile+".old") if err != nil && !errors.Is(err, fs.ErrNotExist) { op.Done(err) return err } } } // Remove old pid file if needed. if util.PathExists(d.pidFilePath()) { err = os.Remove(d.pidFilePath()) if err != nil { op.Done(err) return fmt.Errorf("Failed removing old PID file %q: %w", d.pidFilePath(), err) } } // Cleanup old sockets. for _, socketPath := range []string{d.consolePath(), d.spicePath(), d.monitorPath()} { _ = os.Remove(socketPath) } // Mount the instance's config volume. mountInfo, err := d.mount() if err != nil { op.Done(err) return err } reverter.Add(func() { _ = d.unmount() }) // Define a set of files to open and pass their file descriptors to QEMU command. fdFiles := make([]*os.File, 0) // Ensure passed files are closed after start has returned (either because QEMU has started or on error). defer func() { for _, file := range fdFiles { _ = file.Close() } }() // New or existing vsock ID from volatile. vsockID, vsockF, err := d.nextVsockID() if err != nil { return err } // Add allocated QEMU vhost file descriptor. vsockFD := d.addFileDescriptor(&fdFiles, vsockF) volatileSet := make(map[string]string) if !stateful { volatileSet["volatile.vm.needs_reset"] = "" } // Update vsock ID in volatile if needed for recovery (do this before UpdateBackupFile() call). oldVsockID := d.localConfig["volatile.vsock_id"] newVsockID := strconv.FormatUint(uint64(vsockID), 10) if oldVsockID != newVsockID { volatileSet["volatile.vsock_id"] = newVsockID } // Generate UUID if not present (do this before UpdateBackupFile() call). instUUID := d.localConfig["volatile.uuid"] if instUUID == "" { instUUID = uuid.New().String() volatileSet["volatile.uuid"] = instUUID } // For a VM instance, we must also set the VM generation ID. vmGenUUID := d.localConfig["volatile.uuid.generation"] if vmGenUUID == "" { vmGenUUID = instUUID volatileSet["volatile.uuid.generation"] = vmGenUUID } // Generate the config drive. err = d.generateConfigShare(volatileSet) if err != nil { op.Done(err) return err } // Create all needed paths. err = os.MkdirAll(d.LogPath(), 0o700) if err != nil { op.Done(err) return err } err = os.MkdirAll(d.RunPath(), 0o700) if err != nil { op.Done(err) return err } err = os.MkdirAll(d.DevicesPath(), 0o711) if err != nil { op.Done(err) return err } err = os.MkdirAll(d.ShmountsPath(), 0o711) if err != nil { op.Done(err) return err } // Copy EDK2 settings firmware to nvram file if needed. // Set up EDK2 NVRAM when on EFI. if d.architectureSupportsUEFI(d.architecture) { fi, err := os.Lstat(d.nvramPath()) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } // Generate new NVRAM if missing, or if requested by the user or if the NVRAM file is of an invalid format (needs to be a valid symlink). if util.IsTrue(d.localConfig["volatile.apply_nvram"]) || fi == nil || fi.Mode()&os.ModeSymlink != os.ModeSymlink { err = d.setupNvram() if err != nil { op.Done(err) return err } } } // Clear volatile.apply_nvram if set. if d.localConfig["volatile.apply_nvram"] != "" { volatileSet["volatile.apply_nvram"] = "" } // Apply any volatile changes that need to be made. err = d.VolatileSet(volatileSet) if err != nil { err = fmt.Errorf("Failed setting volatile keys: %w", err) op.Done(err) return err } devConfs := make([]*deviceConfig.RunConfig, 0, len(d.expandedDevices)) postStartHooks := []func() error{} sortedDevices := d.expandedDevices.Sorted() startDevices := make([]device.Device, 0, len(sortedDevices)) // Load devices in sorted order, this ensures that device mounts are added in path order. // Loading all devices first means that validation of all devices occurs before starting any of them. for _, entry := range sortedDevices { dev, err := d.deviceLoad(d, entry.Name, entry.Config, false) if err != nil { if errors.Is(err, device.ErrUnsupportedDevType) { continue // Skip unsupported device (allows for mixed instance type profiles). } err = fmt.Errorf("Failed start validation for device %q: %w", entry.Name, err) op.Done(err) return err } // Run pre-start of check all devices before starting any device to avoid expensive revert. err = dev.PreStartCheck() if err != nil { op.Done(err) return fmt.Errorf("Failed pre-start check for device %q: %w", dev.Name(), err) } startDevices = append(startDevices, dev) } // Start devices in order. for i := range startDevices { dev := startDevices[i] // Local var for revert. // Start the device. runConf, err := d.deviceStart(dev, false) if err != nil { err = fmt.Errorf("Failed to start device %q: %w", dev.Name(), err) op.Done(err) return err } reverter.Add(func() { err := d.deviceStop(dev, false, "") if err != nil { d.logger.Error("Failed to cleanup device", logger.Ctx{"device": dev.Name(), "err": err}) } }) if runConf == nil { continue } if runConf.Revert != nil { reverter.Add(runConf.Revert) } // Add post-start hooks if len(runConf.PostHooks) > 0 { postStartHooks = append(postStartHooks, runConf.PostHooks...) } devConfs = append(devConfs, runConf) } // Setup the config drive readonly bind mount. Important that this come after the root disk device start. // in order to allow unmounts triggered by deferred resizes of the root volume. configMntPath := d.configDriveMountPath() err = d.configDriveMountPathClear() if err != nil { err = fmt.Errorf("Failed cleaning config drive mount path %q: %w", configMntPath, err) op.Done(err) return err } err = os.Mkdir(configMntPath, 0o700) if err != nil { err = fmt.Errorf("Failed creating device mount path %q for config drive: %w", configMntPath, err) op.Done(err) return err } reverter.Add(func() { _ = d.configDriveMountPathClear() }) // Mount the config drive device as readonly. This way it will be readonly irrespective of whether its // exported via 9p for virtio-fs. configSrcPath := filepath.Join(d.Path(), "config") err = device.DiskMount(configSrcPath, configMntPath, false, "", []string{"ro"}, "none") if err != nil { err = fmt.Errorf("Failed mounting device mount path %q for config drive: %w", configMntPath, err) op.Done(err) return err } // Get qemu configuration and check qemu is installed. qemuPath, qemuBus, err := d.qemuArchConfig(d.architecture) if err != nil { op.Done(err) return err } // Snapshot if needed. snapName, expiry, err := d.getStartupSnapNameAndExpiry(d) if err != nil { err = fmt.Errorf("Failed getting startup snapshot info: %w", err) op.Done(err) return err } if snapName != "" && expiry != nil { err := d.snapshot(snapName, *expiry, false) if err != nil { err = fmt.Errorf("Failed taking startup snapshot: %w", err) op.Done(err) return err } } // Setup the CPU. if bs.CPUTopology == nil { // Get the CPU topology. cpuTopology, err := d.cpuTopology() if err != nil { return err } bs.CPUTopology = cpuTopology } if bs.CPUType == "" { cpuType, err := d.cpuType(bs) if err != nil { return err } bs.CPUType = cpuType } // Setup the memory. if bs.MemoryTopology == nil { // Get the memory topology. memoryTopology, err := d.memoryTopology(bs) if err != nil { return err } bs.MemoryTopology = memoryTopology } // Generate the QEMU configuration. monHooks, err := d.generateQemuConfig(bs, mountInfo, qemuBus, vsockFD, devConfs, &fdFiles) if err != nil { op.Done(err) return err } confFile := filepath.Join(d.RunPath(), "qemu.conf") // Start QEMU. qemuArgs := []string{ "-S", "-name", d.Name(), "-uuid", instUUID, "-daemonize", "-cpu", bs.CPUType, "-nographic", "-serial", "chardev:console", "-nodefaults", "-no-user-config", "-sandbox", "on,obsolete=deny,elevateprivileges=allow,spawn=allow,resourcecontrol=deny", "-readconfig", confFile, "-pidfile", d.pidFilePath(), "-D", d.LogFilePath(), } // Get the feature flags. info := DriverStatuses()[instancetype.VM].Info _, spiceSupported := info.Features["spice"] if spiceSupported { qemuArgs = append(qemuArgs, "-spice", d.spiceCmdlineConfig()) } // If stateful, restore now. if stateful { if d.stateful { qemuArgs = append(qemuArgs, "-incoming", "defer") } else { // No state to restore, just start as normal. stateful = false } } else if d.stateful { // Stateless start requested but state is present, delete it. err := os.Remove(d.StatePath()) if err != nil && !errors.Is(err, fs.ErrNotExist) { op.Done(err) return err } d.stateful = false err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateInstanceStatefulFlag(ctx, d.id, false) }) if err != nil { op.Done(err) return fmt.Errorf("Error updating instance stateful flag: %w", err) } } // SMBIOS only on x86_64 and aarch64. if d.architectureSupportsUEFI(d.architecture) { qemuArgs = append(qemuArgs, "-smbios", "type=2,manufacturer=LinuxContainers,product=Incus") for k, v := range d.expandedConfig { var configPrefix, smbiosPrefix string if strings.HasPrefix(k, "smbios11.") { configPrefix = "smbios11." smbiosPrefix = "" } else if strings.HasPrefix(k, "systemd.credential.") { configPrefix = "systemd.credential." smbiosPrefix = "io.systemd.credential:" } else if strings.HasPrefix(k, "systemd.credential-binary.") { configPrefix = "systemd.credential-binary." smbiosPrefix = "io.systemd.credential.binary:" data, err := base64.RawStdEncoding.DecodeString(strings.TrimRight(v, "=")) if err != nil { return fmt.Errorf("Invalid base64 value for %q: %q", k, v) } v = base64.StdEncoding.EncodeToString(data) } else { continue } qemuArgs = append(qemuArgs, "-smbios", fmt.Sprintf("type=11,value=%s%s=%s", smbiosPrefix, strings.TrimPrefix(k, configPrefix), qemuEscapeCmdline(v))) } } // Attempt to drop privileges (doesn't work when restoring state). if !stateful && d.state.OS.UnprivUser != "" { qemuVer, _ := d.version() qemuVer91, _ := version.NewDottedVersion("9.1.0") // Since QEMU 9.1 the parameter `runas` has been marked as deprecated. if qemuVer != nil && qemuVer.Compare(qemuVer91) >= 0 { qemuArgs = append(qemuArgs, "-run-with", fmt.Sprintf("user=%s", d.state.OS.UnprivUser)) } else { qemuArgs = append(qemuArgs, "-runas", d.state.OS.UnprivUser) } nvRAMPath := d.nvramPath() if d.architectureSupportsUEFI(d.architecture) && util.PathExists(nvRAMPath) { // Ensure UEFI nvram file is writable by the QEMU process. // This is needed when doing stateful snapshots because the QEMU process will reopen the // file for writing. err = os.Chown(nvRAMPath, int(d.state.OS.UnprivUID), -1) if err != nil { op.Done(err) return err } err = os.Chmod(nvRAMPath, 0o600) if err != nil { op.Done(err) return err } } // Change ownership of main instance directory. err = os.Chown(d.Path(), int(d.state.OS.UnprivUID), -1) if err != nil { op.Done(err) return fmt.Errorf("Failed to chown instance path: %w", err) } // Change ownership of config directory files so they are accessible to the // unprivileged qemu process so that the 9p share can work. // // Security note: The 9P share will present the UID owner of these files on the host // to the VM. In order to ensure that non-root users in the VM cannot access these // files be sure to mount the 9P share in the VM with the "access=0" option to allow // only root user in VM to access the mounted share. err := filepath.Walk(filepath.Join(d.Path(), "config"), func(path string, info os.FileInfo, err error) error { if err != nil { return err } err = os.Chown(path, int(d.state.OS.UnprivUID), -1) if err != nil { return err } return nil }) if err != nil { op.Done(err) return err } } // Handle hugepages on architectures where we don't set NUMA nodes. if d.architecture != osarch.ARCH_64BIT_INTEL_X86 && util.IsTrue(d.expandedConfig["limits.memory.hugepages"]) { hugetlb, err := localUtil.HugepagesPath() if err != nil { op.Done(err) return err } qemuArgs = append(qemuArgs, "-mem-path", hugetlb, "-mem-prealloc") } if d.expandedConfig["raw.qemu"] != "" { fields, err := shellquote.Split(d.expandedConfig["raw.qemu"]) if err != nil { op.Done(err) return err } qemuArgs = append(qemuArgs, fields...) } // Apply the RTC configuration. // This needs to happen close to creating the full qemu cmd or the time might drift in between. adjustment := d.getStartupRTCAdjustment() if d.GuestOS() == "windows" || adjustment != 0 { base := time.Now().Add(adjustment) if d.GuestOS() == "windows" { // set base to localtime on windows. base = base.Local() } else { // set base to UTC on !windows. base = base.UTC() } datetime := base.Format("2006-01-02T15:04:05") qemuArgs = append(qemuArgs, "-rtc", fmt.Sprintf("base=%s", datetime)) } d.cmdArgs = qemuArgs // Precompile the QEMU scriptlet src, ok := d.expandedConfig["raw.qemu.scriptlet"] if ok { instanceName := d.Name() err := scriptletLoad.QEMUSet(src, instanceName) if err != nil { err = fmt.Errorf("Failed loading QEMU scriptlet: %w", err) return err } } // Config startup hook. err = d.runStartupScriptlet(nil, "config") if err != nil { op.Done(err) return err } // Write the config file. err = d.writeQemuConfigFile(confFile) if err != nil { op.Done(err) return err } // Run the qemu command via forklimits so we can selectively increase ulimits. forkLimitsCmd := []string{ "forklimits", } if !d.state.OS.RunningInUserNS { // Required for PCI passthrough. forkLimitsCmd = append(forkLimitsCmd, "limit=memlock:unlimited:unlimited") } for i := range fdFiles { // Pass through any file descriptors as 3+i (as first 3 file descriptors are taken as standard). forkLimitsCmd = append(forkLimitsCmd, fmt.Sprintf("fd=%d", 3+i)) } // Log the QEMU command line. fullCmd := append(forkLimitsCmd, "--", qemuPath) fullCmd = append(fullCmd, d.cmdArgs...) d.logger.Debug("Starting QEMU", logger.Ctx{"command": fullCmd}) // Setup background process. p, err := subprocess.NewProcess(d.state.OS.ExecPath, fullCmd, d.EarlyLogFilePath(), d.EarlyLogFilePath()) if err != nil { op.Done(err) return err } // Load the AppArmor profile err = apparmor.InstanceLoad(d.state.OS, d, []string{qemuPath}) if err != nil { op.Done(err) return err } p.SetApparmor(apparmor.InstanceProfileName(d)) // Update the backup.yaml file just before starting the instance process, but after all devices have been // setup, so that the backup file contains the volatile keys used for this instance start, so that they can // be used for instance cleanup. err = d.UpdateBackupFile() if err != nil { err = fmt.Errorf("Failed updating backup file: %w", err) op.Done(err) return err } err = p.StartWithFiles(context.Background(), fdFiles) if err != nil { op.Done(err) return err } _, err = p.Wait(context.Background()) if err != nil { stderr, _ := os.ReadFile(d.EarlyLogFilePath()) err = fmt.Errorf("Failed to run: %s: %s: %w", strings.Join(p.Args, " "), string(stderr), err) op.Done(err) return err } pid, err := d.pid() if err != nil || pid <= 0 { d.logger.Error("Failed to get VM process ID", logger.Ctx{"err": err, "pid": pid}) op.Done(err) return err } reverter.Add(func() { _ = d.killQemuProcess(pid) }) // Start QMP monitoring. monitor, err := d.qmpConnect() if err != nil { op.Done(err) return err } // Record the QEMU machine definition. // NOTE: We can't use CanLiveMigrate during instance startup as the boot state hasn't yet been recorded. if !stateful && util.IsTrue(d.expandedConfig["migration.stateful"]) { definition, err := monitor.MachineDefinition() if err != nil { op.Done(err) return err } bs.MachineType = definition } // Don't allow the monitor to trigger a disconnection shutdown event until cleanly started so that the // onStop hook isn't triggered prematurely (as this function's reverter will clean up on failure to start). monitor.SetInitialized(false) // Early startup hook err = d.startupHook(monitor, "early") if err != nil { op.Done(err) return err } // Apply CPU pinning. if bs.CPUTopology.vCPUs == nil { if d.architectureSupportsCPUHotplug() && bs.CPUTopology.Cores > 1 { // Hotplug the CPUs. err := d.setCPUs(monitor, bs.CPUTopology.Cores) if err != nil { err = fmt.Errorf("Failed to add CPUs: %w", err) op.Done(err) return err } } } else { // Get the list of PIDs from the VM. pids, err := monitor.GetCPUs() if err != nil { op.Done(err) return err } // Confirm nothing weird is going on. if len(bs.CPUTopology.vCPUs) != len(pids) { err = errors.New("QEMU has less vCPUs than configured") op.Done(err) return err } // Apply the CPU pins. for i, pid := range pids { set := unix.CPUSet{} set.Set(int(bs.CPUTopology.vCPUs[uint64(i)])) // Apply the pin. err := unix.SchedSetaffinity(pid, &set) if err != nil { op.Done(err) return err } } // Create a core scheduling group. err = d.setCoreSched(pids) if err != nil { err = fmt.Errorf("Failed to allocate new core scheduling domain for vCPU threads: %w", err) op.Done(err) return err } } // Run monitor hooks from devices. for _, monHook := range monHooks { err = monHook(monitor) if err != nil { op.Done(err) return fmt.Errorf("Failed setting up device via monitor: %w", err) } } // Pre-start startup hook err = d.startupHook(monitor, "pre-start") if err != nil { op.Done(err) return err } // Due to a bug in QEMU, devices added using QMP's device_add command do not have their bootindex option // respected (even if added before emuation is started). To workaround this we must reset the VM in order // for it to rebuild its boot config and to take into account the devices bootindex settings. // This also means we cannot start the QEMU process with the -no-reboot flag, so we set the same reboot // action below after this call. err = monitor.Reset() if err != nil { op.Done(err) return fmt.Errorf("Failed resetting VM: %w", err) } // Set our default actions. Those can still be overridden in the event handler when Incus is running. actions := map[string]string{ "shutdown": "poweroff", "reboot": "reset", "panic": "exit-failure", } err = monitor.SetAction(actions) if err != nil { op.Done(err) return fmt.Errorf("Failed setting reboot action: %w", err) } // Restore the state. if stateful { // Add back any memory hotplug slot. if bs.MemoryTopology != nil { for _, memSize := range bs.MemoryTopology.Extra { err := d.hotplugMemory(monitor, memSize) if err != nil { return err } } } // Receive the state. err = d.restoreState(monitor) if err != nil { op.Done(err) return err } } // Start the VM. err = monitor.Start() if err != nil { err = fmt.Errorf("Failed starting VM: %w", err) op.Done(err) return err } // Finish handling stateful start. if stateful { // Cleanup state. _ = os.Remove(d.StatePath()) d.stateful = false err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateInstanceStatefulFlag(ctx, d.id, false) }) if err != nil { op.Done(err) return fmt.Errorf("Error updating instance stateful flag: %w", err) } } // Record last start state. err = d.recordLastState() if err != nil { op.Done(err) return err } reverter.Success() // Post-start startup hook err = d.startupHook(monitor, "post-start") if err != nil { op.Done(err) // Shut down the VM if the post-start commands fail. _ = d.Stop(false) return err } // Run any post-start hooks. err = d.runHooks(postStartHooks) if err != nil { op.Done(err) // Must come before Stop() otherwise stop will not proceed. // Shut down the VM if hooks fail. _ = d.Stop(false) return err } // Apply OOM priority after container is started and hooks completed. err = d.setOOMPriority(d.InitPID()) if err != nil { d.logger.Warn("Failed to set OOM priority", logger.Ctx{ "err": err, "instance": d.Name(), "project": d.Project().Name, }) } if op.Action() == "start" { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceStarted.Event(d, nil)) } // The VM started cleanly so now enable the unexpected disconnection event to ensure the onStop hook is // run if QMP unexpectedly disconnects. monitor.SetInitialized(true) op.Done(nil) // Record the final boot state data. err = d.saveBootState(*bs) if err != nil { return err } return nil } func (d *qemu) setupSEV(fdFiles *[]*os.File) (*qemuSevOpts, error) { if d.architecture != osarch.ARCH_64BIT_INTEL_X86 { return nil, errors.New("AMD SEV support is only available on x86_64 systems") } // Get the QEMU features to check if AMD SEV is supported. info := DriverStatuses()[instancetype.VM].Info _, smeFound := info.Features["sme"] // codespell:ignore sme sev, sevFound := info.Features["sev"] if !smeFound || !sevFound { return nil, errors.New("AMD SEV is not supported by the host") } // Get the SEV guest `cbitpos` and `reducedPhysBits`. sevCapabilities, ok := sev.(qmp.AMDSEVCapabilities) if !ok { return nil, errors.New(`Failed to get the guest "sev" capabilities`) } cbitpos := sevCapabilities.CBitPos reducedPhysBits := sevCapabilities.ReducedPhysBits // Write user's dh-cert and session-data to file descriptors. var dhCertFD, sessionDataFD int if d.expandedConfig["security.sev.session.dh"] != "" { dhCert, err := os.CreateTemp("", "incus_sev_dh_cert_") if err != nil { return nil, err } err = os.Remove(dhCert.Name()) if err != nil { return nil, err } _, err = dhCert.WriteString(d.expandedConfig["security.sev.session.dh"]) if err != nil { return nil, err } dhCertFD = d.addFileDescriptor(fdFiles, dhCert) } if d.expandedConfig["security.sev.session.data"] != "" { sessionData, err := os.CreateTemp("", "incus_sev_session_data_") if err != nil { return nil, err } err = os.Remove(sessionData.Name()) if err != nil { return nil, err } _, err = sessionData.WriteString(d.expandedConfig["security.sev.session.data"]) if err != nil { return nil, err } sessionDataFD = d.addFileDescriptor(fdFiles, sessionData) } sevOpts := &qemuSevOpts{} sevOpts.cbitpos = cbitpos sevOpts.reducedPhysBits = reducedPhysBits if dhCertFD > 0 && sessionDataFD > 0 { sevOpts.dhCertFD = fmt.Sprintf("/proc/self/fd/%d", dhCertFD) sevOpts.sessionDataFD = fmt.Sprintf("/proc/self/fd/%d", sessionDataFD) } if util.IsTrue(d.expandedConfig["security.sev.policy.es"]) { _, sevES := info.Features["sev-es"] if sevES { // This bit mask is used to specify a guest policy. '0x5' is for SEV-ES. The details of the available policies can be found in the link below (see chapter 3) // https://www.amd.com/system/files/TechDocs/55766_SEV-KM_API_Specification.pdf sevOpts.policy = "0x5" } else { return nil, errors.New("AMD SEV-ES is not supported by the host") } } else { // '0x1' is for a regular SEV policy. sevOpts.policy = "0x1" } return sevOpts, nil } // getAgentConnectionInfo returns the connection info the agent needs to connect to the server. func (d *qemu) getAgentConnectionInfo() (*agentAPI.API10Put, error) { addr := d.state.Endpoints.VsockAddress() if addr == nil { return nil, nil } vsockaddr, ok := addr.(*vsock.Addr) if !ok { return nil, errors.New("Listen address is not vsock.Addr") } req := agentAPI.API10Put{ Certificate: string(d.state.Endpoints.NetworkCert().PublicKey()), DevIncus: util.IsTrueOrEmpty(d.expandedConfig["security.guestapi"]), CID: vsock.Host, // Always tell the agent to connect to the server using the Host Context ID to support nesting. Port: vsockaddr.Port, } return &req, nil } // advertiseVsockAddress advertises the CID and port to the VM. func (d *qemu) advertiseVsockAddress() error { client, err := d.getAgentClient() if err != nil { return fmt.Errorf("Failed getting agent client handle: %w", err) } agentArgs := &incus.ConnectionArgs{ SkipGetEvents: true, SkipGetServer: true, } ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() agent, err := incus.ConnectIncusHTTPWithContext(ctx, agentArgs, client) if err != nil { return fmt.Errorf("Failed connecting to the agent: %w", err) } defer agent.Disconnect() connInfo, err := d.getAgentConnectionInfo() if err != nil { return err } if connInfo == nil { return nil } _, _, err = agent.RawQuery("PUT", "/1.0", connInfo, "") if err != nil { return fmt.Errorf("Failed sending host vsock information to the agent: %w", err) } return nil } // AgentCertificate returns the server certificate of the agent. func (d *qemu) AgentCertificate() *x509.Certificate { agentCert := filepath.Join(d.Path(), "config", "agent.crt") if !util.PathExists(agentCert) { return nil } cert, err := localtls.ReadCert(agentCert) if err != nil { return nil } return cert } func (d *qemu) architectureSupportsUEFI(arch int) bool { return slices.Contains([]int{osarch.ARCH_64BIT_INTEL_X86, osarch.ARCH_64BIT_ARMV8_LITTLE_ENDIAN}, arch) } func (d *qemu) setupNvram() error { var err error d.logger.Debug("Generating NVRAM") // Cleanup existing variables. firmwares, err := edk2.GetArchitectureFirmwarePairs(d.architecture) if err != nil { return err } for _, firmwarePair := range firmwares { err := os.Remove(filepath.Join(d.Path(), filepath.Base(firmwarePair.Vars))) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } } // Determine expected firmware. if util.IsTrue(d.expandedConfig["security.csm"]) { firmwares, err = edk2.GetArchitectureFirmwarePairsForUsage(d.architecture, edk2.CSM) if err != nil { return err } } else if util.IsTrueOrEmpty(d.expandedConfig["security.secureboot"]) { firmwares, err = edk2.GetArchitectureFirmwarePairsForUsage(d.architecture, edk2.SECUREBOOT) if err != nil { return err } } else { firmwares, err = edk2.GetArchitectureFirmwarePairsForUsage(d.architecture, edk2.GENERIC) if err != nil { return err } } // Find the template file. var efiVarsPath string var efiVarsName string for _, firmware := range firmwares { varsPath, err := filepath.EvalSymlinks(firmware.Vars) if err != nil { continue } if util.PathExists(varsPath) { efiVarsPath = varsPath efiVarsName = filepath.Base(firmware.Vars) break } } if efiVarsPath == "" { return fmt.Errorf("Couldn't find one of the required UEFI firmware files: %+v", firmwares) } // Copy the template. err = internalUtil.FileCopy(efiVarsPath, filepath.Join(d.Path(), efiVarsName)) if err != nil { return err } nvramPath := d.nvramPath() // Handle the case where the firmware vars filename matches our internal one. if efiVarsName == filepath.Base(nvramPath) { return nil } // Generate a symlink. // This is so qemu.nvram can always be assumed to be the EDK2 vars file. // The real file name is then used to determine what firmware must be selected. _ = os.Remove(nvramPath) err = os.Symlink(efiVarsName, nvramPath) if err != nil { return err } return nil } func (d *qemu) qemuArchConfig(arch int) (string, string, error) { var bus string var qemuCmd string switch arch { case osarch.ARCH_64BIT_INTEL_X86: qemuCmd = "qemu-system-x86_64" bus = "pcie" case osarch.ARCH_64BIT_ARMV8_LITTLE_ENDIAN: qemuCmd = "qemu-system-aarch64" bus = "pcie" case osarch.ARCH_64BIT_POWERPC_LITTLE_ENDIAN: qemuCmd = "qemu-system-ppc64" bus = "pci" case osarch.ARCH_64BIT_S390_BIG_ENDIAN: qemuCmd = "qemu-system-s390x" bus = "ccw" default: return "", "", errors.New("Architecture isn't supported for virtual machines") } qemuPath, err := exec.LookPath(qemuCmd) if err != nil { if d.state != nil && d.state.OS != nil && len(d.state.OS.Architectures) > 0 && arch == d.state.OS.Architectures[0] && util.PathExists("/usr/libexec/qemu-kvm") { return "/usr/libexec/qemu-kvm", bus, nil } return "", "", err } return qemuPath, bus, nil } // RegisterDevices calls the Register() function on all of the instance's devices. func (d *qemu) RegisterDevices() { d.devicesRegister(d) } func (d *qemu) saveConnectionInfo(connInfo *agentAPI.API10Put) error { configDrivePath := filepath.Join(d.Path(), "config") f, err := os.Create(filepath.Join(configDrivePath, "agent.conf")) if err != nil { return err } defer func() { _ = f.Close() }() err = json.NewEncoder(f).Encode(connInfo) if err != nil { return err } return nil } // OnHook is the top-level hook handler. func (d *qemu) OnHook(hookName string, args map[string]string) error { return instance.ErrNotImplemented } // deviceStart loads a new device and calls its Start() function. func (d *qemu) deviceStart(dev device.Device, instanceRunning bool) (*deviceConfig.RunConfig, error) { configCopy := dev.Config() l := d.logger.AddContext(logger.Ctx{"device": dev.Name(), "type": configCopy["type"]}) l.Debug("Starting device") reverter := revert.New() defer reverter.Fail() if instanceRunning && !dev.CanHotPlug() { return nil, errors.New("Device cannot be started when instance is running") } runConf, err := dev.Start() if err != nil { return nil, err } reverter.Add(func() { runConf, _ := dev.Stop() if runConf != nil { _ = d.runHooks(runConf.PostHooks) } }) // If runConf supplied, perform any instance specific setup of device. if runConf != nil { // If instance is running and then live attach device. if instanceRunning { // Attach NIC to running instance. if len(runConf.NetworkInterface) > 0 { err = d.deviceAttachNIC(dev.Name(), configCopy, runConf) if err != nil { return nil, err } } // Attach disk to running instance. for _, mount := range runConf.Mounts { if mount.FSType == "9p" { err = d.deviceAttachPath(dev.Name(), configCopy, mount) if err != nil { return nil, err } } else if mount.TargetPath != "/" { err = d.deviceAttachBlockDevice(dev.Name(), configCopy, mount) if err != nil { return nil, err } } } // Attach USB to running instance. for _, usbDev := range runConf.USBDevice { err = d.deviceAttachUSB(usbDev) if err != nil { return nil, err } } // Attach PCI to running instance. if len(runConf.PCIDevice) > 0 { err = d.deviceAttachPCI(dev.Name(), configCopy, runConf.PCIDevice) if err != nil { return nil, err } } // If running, run post start hooks now (if not, they will be run // once the instance is started). err = d.runHooks(runConf.PostHooks) if err != nil { return nil, err } } } reverter.Success() return runConf, nil } func (d *qemu) deviceAttachPath(deviceName string, configCopy map[string]string, mount deviceConfig.MountEntryItem) error { // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return fmt.Errorf("Failed to connect to QMP monitor: %w", err) } monHook, err := d.addDriveDirConfigVirtiofs(nil, nil, mount) if err != nil { return fmt.Errorf("Failed to add drive config: %w", err) } err = monHook(monitor) if err != nil { return fmt.Errorf("Failed to call monitor hook for block device: %w", err) } return nil } func (d *qemu) deviceAttachBlockDevice(deviceName string, configCopy map[string]string, mount deviceConfig.MountEntryItem) error { // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return fmt.Errorf("Failed to connect to QMP monitor: %w", err) } monHook, err := d.addDriveConfig(nil, nil, mount) if err != nil { return fmt.Errorf("Failed to add drive config: %w", err) } err = monHook(monitor) if err != nil { return fmt.Errorf("Failed to call monitor hook for block device: %w", err) } return nil } func (d *qemu) deviceDetachPath(deviceName string, rawConfig deviceConfig.Device) error { escapedDeviceName := linux.PathNameEncode(deviceName) deviceID := fmt.Sprintf("%s%s", qemuDeviceIDPrefix, escapedDeviceName) mountTag := d.mountTagName(deviceName, qemuMountTagMaxLength) // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return err } err = monitor.RemoveDevice(deviceID) if err != nil { return err } waitDuration := time.Duration(time.Second * time.Duration(10)) waitUntil := time.Now().Add(waitDuration) for { err = monitor.RemoveCharDevice(mountTag) if err == nil { break } if api.StatusErrorCheck(err, http.StatusLocked) { time.Sleep(time.Second * time.Duration(2)) continue } if time.Now().After(waitUntil) { return fmt.Errorf("Failed to detach path device after %v", waitDuration) } } return nil } func (d *qemu) deviceDetachBlockDevice(deviceName string, rawConfig deviceConfig.Device) error { // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return err } escapedDeviceName := linux.PathNameEncode(deviceName) deviceID := fmt.Sprintf("%s%s", qemuDeviceIDPrefix, escapedDeviceName) blockDevName := d.blockNodeName(escapedDeviceName) blockDevs, err := d.fetchBlockDeviceChain(monitor, blockDevName) if err != nil { return err } for i := len(blockDevs); i > 0; i-- { blockDev := blockDevs[i-1] err = monitor.RemoveFDFromFDSet(blockDev) if err != nil { return err } } err = monitor.RemoveDevice(deviceID) if err != nil { return err } if rawConfig["io.bus"] == "usb" { // When dealing with USB, remove the intermediate USB device too. err = monitor.RemoveDevice("usb_" + deviceID) if err != nil { return err } } for i := len(blockDevs); i > 0; i-- { blockDev := blockDevs[i-1] err = d.detachBlockDeviceAndWait(monitor, blockDev) if err != nil { return err } } return nil } func (d *qemu) detachBlockDeviceAndWait(m *qmp.Monitor, blockDevName string) error { waitDuration := time.Duration(time.Second * time.Duration(10)) waitUntil := time.Now().Add(waitDuration) for { err := m.RemoveBlockDevice(blockDevName) if err == nil { break } if api.StatusErrorCheck(err, http.StatusLocked) { time.Sleep(time.Second * time.Duration(2)) continue } if time.Now().After(waitUntil) { return fmt.Errorf("Failed to detach block device after %v", waitDuration) } } return nil } // deviceAttachNIC live attaches a NIC device to the instance. func (d *qemu) deviceAttachNIC(deviceName string, configCopy map[string]string, runConf *deviceConfig.RunConfig) error { devName := "" for _, dev := range runConf.NetworkInterface { if dev.Key == "link" { devName = dev.Value break } } if devName == "" { return errors.New("Device didn't provide a link property to use") } _, qemuBus, err := d.qemuArchConfig(d.architecture) if err != nil { return err } // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return err } qemuDev := make(map[string]any) if runConf.UseUSBBus { qemuBus = "usb" qemuDev["bus"] = "qemu_usb.0" } else if slices.Contains([]string{"pcie", "pci"}, qemuBus) { // Try to get a PCI address for hotplugging. pciDeviceName, err := d.getPCIHotplug() if err != nil { return err } d.logger.Debug("Using PCI bus device to hotplug NIC into", logger.Ctx{"device": deviceName, "port": pciDeviceName}) qemuDev["bus"] = pciDeviceName qemuDev["addr"] = "00.0" } monHook, err := d.addNetDevConfig(qemuBus, qemuDev, nil, runConf.NetworkInterface) if err != nil { return err } err = monHook(monitor) if err != nil { return err } return nil } func (d *qemu) getPCIHotplug() (string, error) { // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return "", err } // Get the current PCI devices. devices, err := monitor.QueryPCI() if err != nil { return "", err } for _, dev := range devices { // Skip built-in devices. if dev.DevID == "" || dev.DevID == "qemu_iommu" { continue } // Skip used bridges. if len(dev.Bridge.Devices) > 0 { continue } // Found an empty slot. return dev.DevID, nil } return "", errors.New("No available PCI hotplug slots could be found") } // deviceAttachPCI live attaches a generic PCI device to the instance. func (d *qemu) deviceAttachPCI(deviceName string, configCopy map[string]string, pciConfig []deviceConfig.RunConfigItem) error { reverter := revert.New() defer reverter.Fail() // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return err } // Get the device config. var devName, pciSlotName, pciIOMMUGroup string for _, pciItem := range pciConfig { if pciItem.Key == "devName" { devName = pciItem.Value } else if pciItem.Key == "pciSlotName" { pciSlotName = pciItem.Value } else if pciItem.Key == "pciIOMMUGroup" { pciIOMMUGroup = pciItem.Value } } // PCIe and PCI require a port device name to hotplug the NIC into. _, qemuBus, err := d.qemuArchConfig(d.architecture) if err != nil { return err } if !slices.Contains([]string{"pcie", "pci"}, qemuBus) { return errors.New("Attempting PCI passthrough on a non-PCI system") } // Try to get a PCI address for hotplugging. pciDeviceName, err := d.getPCIHotplug() if err != nil { return err } qemuDev := make(map[string]any) escapedDeviceName := linux.PathNameEncode(devName) d.logger.Debug("Using PCI bus device to hotplug NIC into", logger.Ctx{"device": deviceName, "port": pciDeviceName}) qemuDev["bus"] = pciDeviceName qemuDev["addr"] = "00.0" qemuDev["driver"] = "vfio-pci" qemuDev["id"] = fmt.Sprintf("%s%s", qemuDeviceIDPrefix, escapedDeviceName) qemuDev["host"] = pciSlotName if d.state.OS.UnprivUser != "" { if pciIOMMUGroup == "" { return errors.New("No PCI IOMMU group supplied") } vfioGroupFile := fmt.Sprintf("/dev/vfio/%s", pciIOMMUGroup) err := os.Chown(vfioGroupFile, int(d.state.OS.UnprivUID), -1) if err != nil { return fmt.Errorf("Failed to chown vfio group device %q: %w", vfioGroupFile, err) } } err = monitor.AddDevice(qemuDev) if err != nil { return fmt.Errorf("Failed setting up device %q: %w", devName, err) } return nil } // deviceStop loads a new device and calls its Stop() function. func (d *qemu) deviceStop(dev device.Device, instanceRunning bool, _ string) error { configCopy := dev.Config() l := d.logger.AddContext(logger.Ctx{"device": dev.Name(), "type": configCopy["type"]}) l.Debug("Stopping device") if instanceRunning && !dev.CanHotPlug() { return errors.New("Device cannot be stopped when instance is running") } runConf, err := dev.Stop() if err != nil { return err } if instanceRunning { // Detach NIC from running instance. if configCopy["type"] == "nic" { for _, usbDev := range runConf.USBDevice { err = d.deviceDetachUSB(usbDev) if err != nil { return err } } err = d.deviceDetachNIC(dev.Name()) if err != nil { return err } } // Detach USB from running instance. if configCopy["type"] == "usb" && runConf != nil { for _, usbDev := range runConf.USBDevice { err = d.deviceDetachUSB(usbDev) if err != nil { return err } } } // Detach disk from running instance. if configCopy["type"] == "disk" { if configCopy["path"] != "" { err = d.deviceDetachPath(dev.Name(), configCopy) if err != nil { return err } } else { err = d.deviceDetachBlockDevice(dev.Name(), configCopy) if err != nil { return err } } } // Detach generic PCI device from running instance. if configCopy["type"] == "pci" { err = d.deviceDetachPCI(dev.Name()) if err != nil { return err } } } if runConf != nil { // Run post stop hooks irrespective of run state of instance. err = d.runHooks(runConf.PostHooks) if err != nil { return err } } return nil } // deviceDetachNIC detaches a NIC device from a running instance. func (d *qemu) deviceDetachNIC(deviceName string) error { // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return err } escapedDeviceName := linux.PathNameEncode(deviceName) deviceID := fmt.Sprintf("%s%s", qemuDeviceIDPrefix, escapedDeviceName) netDevID := fmt.Sprintf("%s%s", qemuNetDevIDPrefix, escapedDeviceName) // Request removal of device. err = monitor.RemoveDevice(deviceID) if err != nil { return fmt.Errorf("Failed removing NIC device: %w", err) } err = monitor.RemoveNIC(netDevID) if err != nil { return err } _, qemuBus, err := d.qemuArchConfig(d.architecture) if err != nil { return err } if slices.Contains([]string{"pcie", "pci"}, qemuBus) { // Wait until the device is actually removed (or we timeout waiting). waitDuration := time.Duration(time.Second * time.Duration(10)) waitUntil := time.Now().Add(waitDuration) for { devExists, err := monitor.CheckPCIDevice(deviceID) if err != nil { return fmt.Errorf("Failed getting PCI devices to check for NIC detach: %w", err) } if !devExists { break } if time.Now().After(waitUntil) { return fmt.Errorf("Failed to detach NIC after %v", waitDuration) } d.logger.Debug("Waiting for NIC device to be detached", logger.Ctx{"device": deviceName}) time.Sleep(time.Second * time.Duration(2)) } } return nil } // deviceDetachPCI detaches a generic PCI device from a running instance. func (d *qemu) deviceDetachPCI(deviceName string) error { // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return err } escapedDeviceName := linux.PathNameEncode(deviceName) deviceID := fmt.Sprintf("%s%s", qemuDeviceIDPrefix, escapedDeviceName) // Request removal of device. err = monitor.RemoveDevice(deviceID) if err != nil { return fmt.Errorf("Failed removing PCI device: %w", err) } _, qemuBus, err := d.qemuArchConfig(d.architecture) if err != nil { return err } if slices.Contains([]string{"pcie", "pci"}, qemuBus) { // Wait until the device is actually removed (or we timeout waiting). waitDuration := time.Duration(time.Second * time.Duration(10)) waitUntil := time.Now().Add(waitDuration) for { devExists, err := monitor.CheckPCIDevice(deviceID) if err != nil { return fmt.Errorf("Failed getting PCI devices to check for detach: %w", err) } if !devExists { break } if time.Now().After(waitUntil) { return fmt.Errorf("Failed to detach PCI device after %v", waitDuration) } d.logger.Debug("Waiting for PCI device to be detached", logger.Ctx{"device": deviceName}) time.Sleep(time.Second * time.Duration(2)) } } return nil } func (d *qemu) monitorPath() string { return filepath.Join(d.RunPath(), "qemu.monitor") } func (d *qemu) nvramPath() string { return filepath.Join(d.Path(), "qemu.nvram") } func (d *qemu) consolePath() string { return filepath.Join(d.RunPath(), "qemu.console") } func (d *qemu) spicePath() string { return filepath.Join(d.RunPath(), "qemu.spice") } func (d *qemu) migrateSockPath() string { return filepath.Join(d.RunPath(), "migrate.sock") } func (d *qemu) spiceCmdlineConfig() string { return fmt.Sprintf("unix=on,disable-ticketing=on,addr=%s", d.spicePath()) } // generateConfigShare generates the config share directory that will be exported to the VM via // a 9P share. Due to the unknown size of templates inside the images this directory is created // inside the VM's config volume so that it can be restricted by quota. // Requires the instance be mounted before calling this function. func (d *qemu) generateConfigShare(volatileSet map[string]string) error { configDrivePath := filepath.Join(d.Path(), "config") // Create config drive dir if doesn't exist, if it does exist, leave it around so we don't regenerate all // files causing unnecessary config drive snapshot usage. err := os.MkdirAll(configDrivePath, 0o500) if err != nil { return err } guestOS := d.GuestOS() if guestOS == "unknown" { guestOS = "linux" } // Windows doesn't handle shares. if guestOS != "windows" { // Add the VM agent loader. agentSrcPath, _ := exec.LookPath("incus-agent") if util.PathExists(os.Getenv("INCUS_AGENT_PATH")) { // Install incus-agent script (loads from agent share). agentFile, err := incusAgentLoader.ReadFile("agent-loader/incus-agent-" + guestOS) if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "incus-agent"), agentFile, 0o700) if err != nil { return err } // Legacy support. _ = os.Remove(filepath.Join(configDrivePath, "lxd-agent")) err = os.Symlink("incus-agent", filepath.Join(configDrivePath, "lxd-agent")) if err != nil { return err } } else if agentSrcPath != "" { // Install agent into config drive dir if found. agentSrcPath, err = filepath.EvalSymlinks(agentSrcPath) if err != nil { return err } agentSrcInfo, err := os.Stat(agentSrcPath) if err != nil { return fmt.Errorf("Failed getting info for incus-agent source %q: %w", agentSrcPath, err) } agentInstallPath := filepath.Join(configDrivePath, "incus-agent") agentNeedsInstall := true if util.PathExists(agentInstallPath) { agentInstallInfo, err := os.Stat(agentInstallPath) if err != nil { return fmt.Errorf("Failed getting info for existing incus-agent install %q: %w", agentInstallPath, err) } if agentInstallInfo.ModTime().Equal(agentSrcInfo.ModTime()) && agentInstallInfo.Size() == agentSrcInfo.Size() { agentNeedsInstall = false } } // Only install the agent into config drive if the existing one is different to the source one. // Otherwise we would end up copying it again and this can cause unnecessary snapshot usage. if agentNeedsInstall { d.logger.Debug("Installing incus-agent", logger.Ctx{"srcPath": agentSrcPath, "installPath": agentInstallPath}) err = internalUtil.FileCopy(agentSrcPath, agentInstallPath) if err != nil { return err } err = os.Chmod(agentInstallPath, 0o500) if err != nil { return err } err = os.Chown(agentInstallPath, 0, 0) if err != nil { return err } // Ensure we copy the source file's timestamps so they can be used for comparison later. err = os.Chtimes(agentInstallPath, agentSrcInfo.ModTime(), agentSrcInfo.ModTime()) if err != nil { return fmt.Errorf("Failed setting incus-agent timestamps: %w", err) } } else { d.logger.Debug("Skipping incus-agent install as unchanged", logger.Ctx{"srcPath": agentSrcPath, "installPath": agentInstallPath}) } // Legacy support. _ = os.Remove(filepath.Join(configDrivePath, "lxd-agent")) err = os.Symlink("incus-agent", filepath.Join(configDrivePath, "lxd-agent")) if err != nil { return err } } else { d.logger.Warn("incus-agent not found, skipping its inclusion in the VM config drive", logger.Ctx{"err": err}) } } agentCert, agentKey, clientCert, _, err := d.generateAgentCert() if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "server.crt"), []byte(clientCert), 0o400) if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "agent.crt"), []byte(agentCert), 0o400) if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "agent.key"), []byte(agentKey), 0o400) if err != nil { return err } // OS-specific configuration. switch guestOS { case "freebsd": // rc.d service. err = os.MkdirAll(filepath.Join(configDrivePath, "rc.d"), 0o500) if err != nil { return err } // rc.d service for incus-agent. agentFile, err := incusAgentLoader.ReadFile("agent-loader/rc.d/incus-agent") if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "rc.d", "incus-agent"), agentFile, 0o500) if err != nil { return err } // Setup script for incus-agent that is executed by the incus-agent service before starting. // The script sets up a temporary mount point, copies data from the mount (including incus-agent binary), // and then unmounts it. It also ensures appropriate permissions for the Incus agent's runtime directory. agentFile, err = incusAgentLoader.ReadFile("agent-loader/incus-agent-setup-freebsd") if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "incus-agent-setup"), agentFile, 0o500) if err != nil { return err } // Install script for manual installs. agentFile, err = incusAgentLoader.ReadFile("agent-loader/install-freebsd.sh") if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "install.sh"), agentFile, 0o500) if err != nil { return err } case "linux": // Systemd units. err = os.MkdirAll(filepath.Join(configDrivePath, "systemd"), 0o500) if err != nil { return err } // Systemd unit for incus-agent. It ensures the incus-agent is copied from the shared filesystem before it is // started. The service is triggered dynamically via udev rules when certain virtio-ports are detected, // rather than being enabled at boot. agentFile, err := incusAgentLoader.ReadFile("agent-loader/systemd/incus-agent.service") if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "systemd", "incus-agent.service"), agentFile, 0o400) if err != nil { return err } // Setup script for incus-agent that is executed by the incus-agent systemd unit before incus-agent is started. // The script sets up a temporary mount point, copies data from the mount (including incus-agent binary), // and then unmounts it. It also ensures appropriate permissions for the Incus agent's runtime directory. agentFile, err = incusAgentLoader.ReadFile("agent-loader/incus-agent-setup-linux") if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "systemd", "incus-agent-setup"), agentFile, 0o500) if err != nil { return err } err = os.MkdirAll(filepath.Join(configDrivePath, "udev"), 0o500) if err != nil { return err } // Udev rules to start the incus-agent.service when QEMU serial devices (symlinks in virtio-ports) appear. agentFile, err = incusAgentLoader.ReadFile("agent-loader/systemd/incus-agent.rules") if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "udev", "99-incus-agent.rules"), agentFile, 0o400) if err != nil { return err } // Install script for manual installs. agentFile, err = incusAgentLoader.ReadFile("agent-loader/install-linux.sh") if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "install.sh"), agentFile, 0o700) if err != nil { return err } case "macos": // Launchd daemons. err = os.MkdirAll(filepath.Join(configDrivePath, "launchd"), 0o500) if err != nil { return err } // Launchd daemon for incus-agent. agentFile, err := incusAgentLoader.ReadFile("agent-loader/launchd/org.linuxcontainers.incus.macos-agent.plist") if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "launchd", "org.linuxcontainers.incus.macos-agent.plist"), agentFile, 0o644) if err != nil { return err } // Setup script for incus-agent that is executed by launchd. For convenience, this script also // launches the agent. Because of Apple TCC, sh must be given full disk access in the relevant // system settings page. Other than that, this agent behaves roughly the same as the Linux one. agentFile, err = incusAgentLoader.ReadFile("agent-loader/incus-agent-setup-macos") if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "incus-agent-setup"), agentFile, 0o500) if err != nil { return err } // Install script for manual installs. agentFile, err = incusAgentLoader.ReadFile("agent-loader/install-macos.sh") if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "install.sh"), agentFile, 0o700) if err != nil { return err } case "windows": // Setup script for incus-agent that is executed by Service Control Manager (SCM). Since by // default Windows cannot run a PowerShell script as a service without the help of a third // party, a bat file is used to then execute the PowerShell script doing the job. agentFile, err := incusAgentLoader.ReadFile("agent-loader/incus-agent-setup.ps1") if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "incus-agent-setup.ps1"), agentFile, 0o500) if err != nil { return err } // Install script for manual installs. agentFile, err = incusAgentLoader.ReadFile("agent-loader/install.ps1") if err != nil { return err } err = os.WriteFile(filepath.Join(configDrivePath, "install.ps1"), agentFile, 0o700) if err != nil { return err } } // Templated files. templateFilesPath := filepath.Join(configDrivePath, "files") // Clear path and recreate. _ = os.RemoveAll(templateFilesPath) err = os.MkdirAll(templateFilesPath, 0o500) if err != nil { return err } // Template anything that needs templating. key := "volatile.apply_template" if d.localConfig[key] != "" { // Run any template that needs running. err = d.templateApplyNow(instance.TemplateTrigger(d.localConfig[key]), templateFilesPath) if err != nil { return err } // Record that the instance devices got modified and a full reset will be needed to get a consistent state. volatileSet[key] = "" volatileSet["volatile.vm.needs_reset"] = "true" } err = d.templateApplyNow("start", templateFilesPath) if err != nil { return err } // Copy the template metadata itself too. metaPath := filepath.Join(d.Path(), "metadata.yaml") if util.PathExists(metaPath) { err = internalUtil.FileCopy(metaPath, filepath.Join(templateFilesPath, "metadata.yaml")) if err != nil { return err } } // Only Linux guests support dynamic NIC configuration. if guestOS == "linux" { // Clear NICConfigDir to ensure that no leftover configuration is erroneously applied by the agent. nicConfigPath := filepath.Join(configDrivePath, deviceConfig.NICConfigDir) _ = os.RemoveAll(nicConfigPath) err = os.MkdirAll(nicConfigPath, 0o500) if err != nil { return err } // Add the NIC config. if util.IsTrue(d.expandedConfig["agent.nic_config"]) { sortedDevices := d.expandedDevices.Sorted() for _, entry := range sortedDevices { if entry.Config["type"] != "nic" { continue // Only keep NIC devices. } dev, err := d.FillNetworkDevice(entry.Name, entry.Config) if err != nil { return err } err = d.writeNICDevConfig(dev["mtu"], entry.Name, dev["name"], dev["hwaddr"]) if err != nil { return fmt.Errorf("Failed writing NIC config for device %q: %w", entry.Name, err) } } } // Writing the connection info the config drive allows the agent to start /dev/incus very // early. This is important for systemd services which want or require /dev/incus/sock. connInfo, err := d.getAgentConnectionInfo() if err != nil { return err } if connInfo != nil { err = d.saveConnectionInfo(connInfo) if err != nil { return err } } } return nil } func (d *qemu) templateApplyNow(trigger instance.TemplateTrigger, path string) error { // If there's no metadata, just return. fname := filepath.Join(d.Path(), "metadata.yaml") if !util.PathExists(fname) { return nil } // Parse the metadata. content, err := os.ReadFile(fname) if err != nil { return fmt.Errorf("Failed to read metadata: %w", err) } metadata := &api.ImageMetadata{} err = yaml.Load(content, metadata) if err != nil { return fmt.Errorf("Could not parse %s: %w", fname, err) } // Figure out the instance architecture. arch, err := osarch.ArchitectureName(d.architecture) if err != nil { arch, err = osarch.ArchitectureName(d.state.OS.Architectures[0]) if err != nil { return fmt.Errorf("Failed to detect system architecture: %w", err) } } // Generate the instance metadata. instanceMeta := make(map[string]string) instanceMeta["name"] = d.name instanceMeta["type"] = "virtual-machine" instanceMeta["architecture"] = arch if d.ephemeral { instanceMeta["ephemeral"] = "true" } else { instanceMeta["ephemeral"] = "false" } // Go through the templates. for tplPath, tpl := range metadata.Templates { err = func(tplPath string, tpl *api.ImageMetadataTemplate) error { var w *os.File // Check if the template should be applied now. found := slices.Contains(tpl.When, string(trigger)) if !found { return nil } // Create the file itself. w, err = os.Create(filepath.Join(path, fmt.Sprintf("%s.out", tpl.Template))) if err != nil { return err } // Fix ownership and mode. err = w.Chmod(0o644) if err != nil { return err } defer func() { _ = w.Close() }() // Read the template. tplString, err := os.ReadFile(filepath.Join(d.TemplatesPath(), tpl.Template)) if err != nil { return fmt.Errorf("Failed to read template file: %w", err) } configGet := func(confKey, confDefault *pongo2.Value) *pongo2.Value { val, ok := d.expandedConfig[confKey.String()] if !ok { return confDefault } return pongo2.AsValue(strings.TrimRight(val, "\r\n")) } // Render the template. err = internalUtil.RenderTemplateFile(w, string(tplString), pongo2.Context{ "trigger": trigger, "path": tplPath, "container": instanceMeta, // FIXME: remove once most images have moved away. "instance": instanceMeta, "config": d.expandedConfig, "devices": d.expandedDevices, "properties": tpl.Properties, "config_get": configGet, }) if err != nil { return fmt.Errorf("Failed to render template: %w", err) } return w.Close() }(tplPath, tpl) if err != nil { return err } } return nil } // deviceBootPriorities returns a map keyed on device name containing the boot index to use. // Qemu tries to boot devices in order of boot index (lowest first). func (d *qemu) deviceBootPriorities(base int) (map[string]int, error) { type devicePrios struct { Name string BootPrio uint32 } devices := []devicePrios{} for _, dev := range d.expandedDevices.Sorted() { if dev.Config["type"] != "disk" && dev.Config["type"] != "nic" { continue } bootPrio := uint32(0) // Default to lowest priority. if dev.Config["boot.priority"] != "" { prio, err := strconv.ParseInt(dev.Config["boot.priority"], 10, 32) if err != nil { return nil, fmt.Errorf("Invalid boot.priority for device %q: %w", dev.Name, err) } bootPrio = uint32(prio) } else if dev.Config["path"] == "/" { bootPrio = 1 // Set boot priority of root disk higher than any device without a boot prio. } devices = append(devices, devicePrios{Name: dev.Name, BootPrio: bootPrio}) } // Sort devices by priority (use SliceStable so that devices with the same boot priority stay in the same // order each boot based on the device order provided by the d.expandedDevices.Sorted() function). // This is important because as well as providing a predictable boot index order, the boot index number can // also be used for other properties (such as disk SCSI ID) which can result in it being given different // device names inside the guest based on the device order. sort.SliceStable(devices, func(i, j int) bool { return devices[i].BootPrio > devices[j].BootPrio }) sortedDevs := make(map[string]int, len(devices)) for bootIndex, dev := range devices { sortedDevs[dev.Name] = bootIndex + base } return sortedDevs, nil } func (d *qemu) getStartupRTCAdjustment() time.Duration { // Get the current values. adjustment := d.parseRTC("volatile.vm.rtc_adjustment") offset := d.parseRTC("volatile.vm.rtc_offset") // Reset to handle new VM-generated updates. adjustment += offset offset = 0 changes := map[string]string{ "volatile.vm.rtc_adjustment": strconv.Itoa(adjustment), "volatile.vm.rtc_offset": strconv.Itoa(offset), } err := d.VolatileSet(changes) if err != nil { d.logger.Error("Failed to set RTC change offset", logger.Ctx{"changes": changes, "err": err}) } return time.Duration(adjustment) * time.Second } func (d *qemu) parseRTC(key string) int { offset := 0 val, ok := d.localConfig[key] if ok { var err error offset, err = strconv.Atoi(val) if err != nil { offset = 0 d.logger.Error("Failed to parse RTC volatile key") } } return offset } // onRTCChange saves rtc change. func (d *qemu) onRTCChange(change int) error { offset := d.parseRTC("volatile.vm.rtc_offset") if offset != change { changes := map[string]string{"volatile.vm.rtc_offset": strconv.Itoa(change)} err := d.VolatileSet(changes) if err != nil { d.logger.Error("Failed to set rtc change offset ", logger.Ctx{"changes": changes, "err": err}) } return err } return nil } // generateQemuConfig generates the QEMU configuration. func (d *qemu) generateQemuConfig(bs *qemuBootState, mountInfo *storagePools.MountInfo, busName string, vsockFD int, devConfs []*deviceConfig.RunConfig, fdFiles *[]*os.File) ([]monitorHook, error) { var monHooks []monitorHook isWindows := d.GuestOS() == "windows" conf := qemuBase(&qemuBaseOpts{d.Architecture(), util.IsTrue(d.expandedConfig["security.iommu"]), bs.MachineType}) err := d.addCPUMemoryConfig(&conf, bs) if err != nil { return nil, err } // Parse raw.qemu. rawOptions := []string{} if d.expandedConfig["raw.qemu"] != "" { rawOptions, err = shellquote.Split(d.expandedConfig["raw.qemu"]) if err != nil { return nil, err } } // Allow disabling the UEFI firmware. if slices.Contains(rawOptions, "-bios") || slices.Contains(rawOptions, "-kernel") { d.logger.Warn("Starting VM without default firmware (-bios or -kernel in raw.qemu)") } else if d.architectureSupportsUEFI(d.architecture) { // Open the UEFI NVRAM file and pass it via file descriptor to QEMU. // This is so the QEMU process can still read/write the file after it has dropped its user privs. nvRAMFile, err := os.Open(d.nvramPath()) if err != nil { return nil, fmt.Errorf("Failed opening NVRAM file: %w", err) } // Determine expected firmware. var firmwares []edk2.FirmwarePair if util.IsTrue(d.expandedConfig["security.csm"]) { firmwares, err = edk2.GetArchitectureFirmwarePairsForUsage(d.architecture, edk2.CSM) if err != nil { return nil, err } } else if util.IsTrueOrEmpty(d.expandedConfig["security.secureboot"]) { firmwares, err = edk2.GetArchitectureFirmwarePairsForUsage(d.architecture, edk2.SECUREBOOT) if err != nil { return nil, err } } else { firmwares, err = edk2.GetArchitectureFirmwarePairsForUsage(d.architecture, edk2.GENERIC) if err != nil { return nil, err } } var efiCode string for _, firmware := range firmwares { if util.PathExists(filepath.Join(d.Path(), filepath.Base(firmware.Vars))) { efiCode = firmware.Code break } } if efiCode == "" { return nil, fmt.Errorf("Unable to locate matching firmware: %+v", firmwares) } driveFirmwareOpts := qemuDriveFirmwareOpts{ roPath: efiCode, nvramPath: fmt.Sprintf("/dev/fd/%d", d.addFileDescriptor(fdFiles, nvRAMFile)), } conf = append(conf, qemuDriveFirmware(&driveFirmwareOpts)...) } // QMP socket. conf = append(conf, qemuControlSocket(&qemuControlSocketOpts{d.monitorPath()})...) // Console output. conf = append(conf, qemuConsole()...) // VM core info (memory dump). if !slices.Contains([]int{osarch.ARCH_64BIT_POWERPC_LITTLE_ENDIAN, osarch.ARCH_64BIT_S390_BIG_ENDIAN}, d.architecture) { conf = append(conf, qemuCoreInfo()...) } // Setup the bus allocator. bus := qemuNewBus(busName, &conf) // Add IOMMU. if util.IsTrue(d.expandedConfig["security.iommu"]) && d.architectureSupportsUEFI(d.architecture) { devBus, devAddr, multi := bus.allocateDirect() iommuOpts := qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, } conf = append(conf, qemuIOMMU(&iommuOpts, isWindows)...) } // Now add the fixed set of devices. The multi-function groups used for these fixed internal devices are // specifically chosen to ensure that we consume exactly 4 PCI bus ports (on PCIe bus). This ensures that // the first user device NIC added will use the 5th PCI bus port and will be consistently named enp5s0 // on PCIe (which we need to maintain compatibility with network configuration in our existing VM images). // It's also meant to group all low-bandwidth internal devices onto a single address. PCIe bus allows a // total of 256 devices, but this assumes 32 chassis * 8 function. By using VFs for the internal fixed // devices we avoid consuming a chassis for each one. See also the qemuPCIDeviceIDStart constant. devBus, devAddr, multi := bus.allocate(busFunctionGroupGeneric) balloonOpts := qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, } conf = append(conf, qemuBalloon(&balloonOpts)...) devBus, devAddr, multi = bus.allocate(busFunctionGroupGeneric) rngOpts := qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, } conf = append(conf, qemuRNG(&rngOpts)...) devBus, devAddr, multi = bus.allocate(busFunctionGroupGeneric) keyboardOpts := qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, } conf = append(conf, qemuKeyboard(&keyboardOpts)...) devBus, devAddr, multi = bus.allocate(busFunctionGroupGeneric) tabletOpts := qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, } conf = append(conf, qemuTablet(&tabletOpts)...) // Existing vsock ID from volatile. vsockID, err := d.getVsockID() if err != nil { return nil, err } devBus, devAddr, multi = bus.allocate(busFunctionGroupGeneric) vsockOpts := qemuVsockOpts{ dev: qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, }, vsockFD: vsockFD, vsockID: vsockID, } conf = append(conf, qemuVsock(&vsockOpts)...) info := DriverStatuses()[instancetype.VM].Info _, spice := info.Features["spice"] _, plan9 := info.Features["plan9"] _, virtioSound := info.Features["virtio-sound"] devBus, devAddr, multi = bus.allocate(busFunctionGroupGeneric) serialOpts := qemuSerialOpts{ dev: qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, }, charDevName: qemuSerialChardevName, ringbufSizeBytes: qmp.RingbufSize, spice: spice, } conf = append(conf, qemuSerial(&serialOpts)...) // s390x doesn't really have USB. if d.architecture != osarch.ARCH_64BIT_S390_BIG_ENDIAN { devBus, devAddr, multi = bus.allocate(busFunctionGroupGeneric) usbOpts := qemuUSBOpts{ devBus: devBus, devAddr: devAddr, multifunction: multi, ports: qemuSparseUSBPorts, spice: spice, } conf = append(conf, qemuUSB(&usbOpts)...) } // virtio-sound-pci devices can't be migrated and don't have a CCW equivalent. // NOTE: We can't use CanLiveMigrate during instance startup as the boot state hasn't yet been recorded. if virtioSound && !isWindows && util.IsFalseOrEmpty(d.expandedConfig["migration.stateful"]) && d.architecture != osarch.ARCH_64BIT_S390_BIG_ENDIAN { devBus, devAddr, multi = bus.allocate(busFunctionGroupGeneric) audioOpts := qemuAudioOpts{ dev: qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, }, spice: spice, } conf = append(conf, qemuAudio(&audioOpts)...) } if util.IsTrue(d.expandedConfig["security.csm"]) { // Allocate a regular entry to keep things aligned normally (avoid NICs getting a different name). _, _, _ = bus.allocate(busFunctionGroupNone) // Allocate a direct entry so the SCSI controller can be seen by seabios. devBus, devAddr, multi = bus.allocateDirect() } else { devBus, devAddr, multi = bus.allocate(busFunctionGroupNone) } scsiOpts := qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, } conf = append(conf, qemuSCSI(&scsiOpts, bs.getSCSIQueues())...) // Export the config directory and agent as 9p drives when supported. if !isWindows && plan9 { devBus, devAddr, multi = bus.allocate(busFunctionGroup9p) driveConfig9pOpts := qemuDriveConfigOpts{ dev: qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, }, name: "config", protocol: "9p", path: d.configDriveMountPath(), } conf = append(conf, qemuDriveConfig(&driveConfig9pOpts)...) // Pass in the agents if INCUS_AGENT_PATH is set. if util.PathExists(os.Getenv("INCUS_AGENT_PATH")) { devBus, devAddr, multi = bus.allocate(busFunctionGroup9p) driveConfig9pOpts := qemuDriveConfigOpts{ dev: qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, }, name: "agent", protocol: "9p", path: os.Getenv("INCUS_AGENT_PATH"), } conf = append(conf, qemuDriveConfig(&driveConfig9pOpts)...) } } // If user has requested AMD SEV, check if supported and add to QEMU config. if util.IsTrue(d.expandedConfig["security.sev"]) { sevOpts, err := d.setupSEV(fdFiles) if err != nil { return nil, err } if sevOpts != nil { for i := range conf { if conf[i].Name == "machine" { conf[i].Entries["memory-encryption"] = "sev0" break } } conf = append(conf, qemuSEV(sevOpts)...) } } if util.IsTrue(d.expandedConfig["security.csm"]) { // Allocate a regular entry to keep things aligned normally (avoid NICs getting a different name). _, _, _ = bus.allocate(busFunctionGroupNone) // Allocate a direct entry so the GPU can be seen by seabios. devBus, devAddr, multi = bus.allocateDirect() } else { devBus, devAddr, multi = bus.allocate(busFunctionGroupNone) } gpuOpts := qemuGpuOpts{ dev: qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, }, architecture: d.Architecture(), } conf = append(conf, qemuGPU(&gpuOpts)...) // Dynamic devices. base := 0 if slices.Contains(rawOptions, "-kernel") { base = 1 } bootIndexes, err := d.deviceBootPriorities(base) if err != nil { return nil, fmt.Errorf("Error calculating boot indexes: %w", err) } // Record the mounts we are going to do inside the VM using the agent. agentMounts := []instancetype.VMAgentMount{} // These devices are sorted so that NICs are added first to ensure that the first NIC can use the 5th // PCIe bus port and will be consistently named enp5s0 for compatibility with network configuration in our // existing VM images. Even on non-PCIe buses having NICs first means that their names won't change when // other devices are added. for _, runConf := range devConfs { // Add drive devices. if len(runConf.Mounts) > 0 { for _, drive := range runConf.Mounts { var monHook monitorHook // Check if the user has overridden the bus. busName := "virtio-scsi" for _, opt := range drive.Opts { if !strings.HasPrefix(opt, "bus=") { continue } busName = strings.TrimPrefix(opt, "bus=") break } qemuDev := make(map[string]any) if slices.Contains([]string{"9p", "nvme", "virtio-blk", "virtiofs"}, busName) { // Allocate a PCI(e) port and write it to the config file so QMP can "hotplug" the // drive into it later. functionGroup := busFunctionGroupNone if busName == "9p" { functionGroup = busFunctionGroup9p } devBus, devAddr, multi := bus.allocate(functionGroup) // Populate the qemu device with port info. qemuDev["bus"] = devBus qemuDev["addr"] = devAddr qemuDev["multifunction"] = multi } if drive.TargetPath == "/" { monHook, err = d.addRootDriveConfig(qemuDev, mountInfo, bootIndexes, drive) } else if drive.FSType == "9p" { if busName == "9p" { conf = append(conf, d.driveDirConfig9p(qemuDev, bus.name, &agentMounts, drive)...) } else { monHook, err = d.addDriveDirConfigVirtiofs(qemuDev, &agentMounts, drive) } } else { monHook, err = d.addDriveConfig(qemuDev, bootIndexes, drive) } if err != nil { return nil, fmt.Errorf("Failed setting up disk device %q: %w", drive.DevName, err) } if monHook != nil { monHooks = append(monHooks, monHook) } } } // Add network device. if len(runConf.NetworkInterface) > 0 { qemuDev := make(map[string]any) busName := bus.name if runConf.UseUSBBus { busName = "usb" qemuDev["bus"] = "qemu_usb.0" } else if slices.Contains([]string{"pcie", "pci"}, busName) { // Allocate a PCI(e) port and write it to the config file so QMP can "hotplug" the // NIC into it later. devBus, devAddr, multi := bus.allocate(busFunctionGroupNone) // Populate the qemu device with port info. qemuDev["bus"] = devBus qemuDev["addr"] = devAddr if multi { qemuDev["multifunction"] = true } } monHook, err := d.addNetDevConfig(busName, qemuDev, bootIndexes, runConf.NetworkInterface) if err != nil { return nil, err } monHooks = append(monHooks, monHook) } // Add GPU device. if len(runConf.GPUDevice) > 0 { err = d.addGPUDevConfig(&conf, bus, runConf.GPUDevice) if err != nil { return nil, err } } // Add PCI device. if len(runConf.PCIDevice) > 0 { err = d.addPCIDevConfig(&conf, bus, runConf.PCIDevice) if err != nil { return nil, err } } // Add USB devices. for _, usbDev := range runConf.USBDevice { monHook, err := d.addUSBDeviceConfig(usbDev) if err != nil { return nil, err } monHooks = append(monHooks, monHook) } // Add TPM device. if len(runConf.TPMDevice) > 0 { err = d.addTPMDeviceConfig(&conf, runConf.TPMDevice, fdFiles) if err != nil { return nil, err } } } // VM generation ID is only available on x86. if d.architecture == osarch.ARCH_64BIT_INTEL_X86 { err = d.addVmgenDeviceConfig(&conf, d.localConfig["volatile.uuid.generation"]) if err != nil { return nil, err } } // Allocate 8 PCI slots for hotplug devices. for range 8 { bus.allocate(busFunctionGroupNone) } if !isWindows { // Write the agent mount config. agentMountJSON, err := json.Marshal(agentMounts) if err != nil { return nil, fmt.Errorf("Failed marshalling agent mounts to JSON: %w", err) } agentMountFile := filepath.Join(d.Path(), "config", "agent-mounts.json") err = os.WriteFile(agentMountFile, agentMountJSON, 0o400) if err != nil { return nil, fmt.Errorf("Failed writing agent mounts file: %w", err) } } // process any user-specified overrides confOverride, ok := d.expandedConfig["raw.qemu.conf"] if ok { d.conf, err = qemuRawCfgOverride(conf, confOverride) if err != nil { return nil, err } } else { d.conf = conf } return monHooks, nil } // writeQemuConfigFile writes the QEMU config file. // It writes the config file inside the VM's log path. func (d *qemu) writeQemuConfigFile(configPath string) error { // Write the config file to disk. sb := qemuStringifyCfg(d.conf...) return os.WriteFile(configPath, []byte(sb.String()), 0o640) } // getCPUOpts retrieves configuration options for virtualized CPUs and memory. func (d *qemu) getCPUOpts(cpuInfo *qemuCPUTopology, memSizeBytes int64) (*qemuCPUOpts, error) { cpuOpts := qemuCPUOpts{ architecture: d.architectureName, } hostNodes := []uint64{} if cpuInfo.vCPUs == nil { // If not pinning, default to exposing cores. // Only one CPU will be added here, as the others will be hotplugged during start. if d.architectureSupportsCPUHotplug() { cpuOpts.cpuCount = 1 cpuOpts.cpuCores = 1 // Expose the total requested by the user already so the hotplug limit can be set higher if needed. cpuOpts.cpuRequested = cpuInfo.Cores } else { cpuOpts.cpuCount = cpuInfo.Cores cpuOpts.cpuCores = cpuInfo.Cores } cpuOpts.cpuSockets = 1 cpuOpts.cpuThreads = 1 hostNodes = []uint64{0} // Handle NUMA restrictions. numaNodes := d.expandedConfig["limits.cpu.nodes"] if numaNodes != "" { if numaNodes == "balanced" { numaNodes = d.expandedConfig["volatile.cpu.nodes"] } // Parse the NUMA restriction. numaNodeSet, err := resources.ParseNumaNodeSet(numaNodes) if err != nil { return nil, err } cpuOpts.memoryHostNodes = numaNodeSet } } else { // Figure out socket-id/core-id/thread-id for all vcpus. vcpuSocket := map[uint64]uint64{} vcpuCore := map[uint64]uint64{} vcpuThread := map[uint64]uint64{} vcpu := uint64(0) for i := range cpuInfo.Sockets { for j := range cpuInfo.Cores { for k := range cpuInfo.Threads { vcpuSocket[vcpu] = uint64(i) vcpuCore[vcpu] = uint64(j) vcpuThread[vcpu] = uint64(k) vcpu++ } } } // Prepare the NUMA map. numa := []qemuNumaEntry{} numaIDs := []uint64{} numaNode := uint64(0) for hostNode, entry := range cpuInfo.nodes { hostNodes = append(hostNodes, hostNode) numaIDs = append(numaIDs, numaNode) for _, vcpu := range entry { numa = append(numa, qemuNumaEntry{ node: numaNode, socket: vcpuSocket[vcpu], core: vcpuCore[vcpu], thread: vcpuThread[vcpu], }) } numaNode++ } // Prepare context. cpuOpts.cpuCount = len(cpuInfo.vCPUs) cpuOpts.cpuSockets = cpuInfo.Sockets cpuOpts.cpuCores = cpuInfo.Cores cpuOpts.cpuThreads = cpuInfo.Threads cpuOpts.cpuNumaNodes = numaIDs cpuOpts.cpuNumaMapping = numa cpuOpts.cpuNumaHostNodes = hostNodes } cpuOpts.hugepages = "" if util.IsTrue(d.expandedConfig["limits.memory.hugepages"]) { hugetlb, err := localUtil.HugepagesPath() if err != nil { return nil, err } cpuOpts.hugepages = hugetlb } // Determine per-node memory limit. memSizeMB := memSizeBytes / 1024 / 1024 nodeMemory := int64(memSizeMB / int64(len(hostNodes))) cpuOpts.memory = nodeMemory return &cpuOpts, nil } // addCPUMemoryConfig adds the qemu config required for setting the number of virtualised CPUs and memory. // If sb is nil then no config is written. func (d *qemu) addCPUMemoryConfig(conf *[]cfg.Section, bs *qemuBootState) error { cpuOpts, err := d.getCPUOpts(bs.CPUTopology, bs.MemoryTopology.Base) if err != nil { return err } cpuPinning := bs.CPUTopology.vCPUs != nil *conf = append(*conf, qemuMemory(&qemuMemoryOpts{bs.MemoryTopology.Base / 1024 / 1024, bs.MemoryTopology.Max / 1024 / 1024})...) *conf = append(*conf, qemuCPU(cpuOpts, cpuPinning)...) return nil } // addFileDescriptor adds a file path to the list of files to open and pass file descriptor to qemu. // Returns the file descriptor number that qemu will receive. func (d *qemu) addFileDescriptor(fdFiles *[]*os.File, file *os.File) int { // Append the tap device file path to the list of files to be opened and passed to qemu. *fdFiles = append(*fdFiles, file) return 2 + len(*fdFiles) // Use 2+fdFiles count, as first user file descriptor is 3. } // addRootDriveConfig adds the qemu config required for adding the root drive. func (d *qemu) addRootDriveConfig(qemuDev map[string]any, mountInfo *storagePools.MountInfo, bootIndexes map[string]int, rootDriveConf deviceConfig.MountEntryItem) (monitorHook, error) { if rootDriveConf.TargetPath != "/" { return nil, errors.New("Non-root drive config supplied") } if !d.storagePool.Driver().Info().Remote && mountInfo.DiskPath == "" { return nil, errors.New("No root disk path available from mount") } // Generate a new device config with the root device path expanded. driveConf := deviceConfig.MountEntryItem{ DevName: rootDriveConf.DevName, DevPath: mountInfo.DiskPath, BackingPath: mountInfo.BackingPath, Opts: rootDriveConf.Opts, TargetPath: rootDriveConf.TargetPath, Limits: rootDriveConf.Limits, } if d.storagePool.Driver().Info().Remote { vol := d.storagePool.GetVolume(storageDrivers.VolumeTypeVM, storageDrivers.ContentTypeBlock, project.Instance(d.project.Name, d.name), nil) if slices.Contains([]string{"ceph", "cephfs"}, d.storagePool.Driver().Info().Name) { config := d.storagePool.ToAPI().Config userName := config["ceph.user.name"] if userName == "" { userName = storageDrivers.CephDefaultUser } clusterName := config["ceph.cluster_name"] if clusterName == "" { clusterName = storageDrivers.CephDefaultUser } driveConf.DevPath = device.DiskGetRBDFormat(clusterName, userName, config["ceph.osd.pool_name"], vol.Name()) } } return d.addDriveConfig(qemuDev, bootIndexes, driveConf) } // driveDirConfig9p generates the qemu config required for adding a supplementary drive directory share using 9p. func (d *qemu) driveDirConfig9p(qemuDev map[string]any, busName string, agentMounts *[]instancetype.VMAgentMount, driveConf deviceConfig.MountEntryItem) []cfg.Section { mountTag := d.mountTagName(driveConf.DevName, qemuMountTag9pMaxLength) agentMount := instancetype.VMAgentMount{ Source: mountTag, Target: driveConf.TargetPath, FSType: "9p", // We need to specify to use the virtio transport to support more VM guest OSes. // Also set the msize to 32MB to allow for reasonably fast 9p access. Options: []string{"trans=virtio,msize=33554432"}, } readonly := slices.Contains(driveConf.Opts, "ro") // Indicate to agent to mount this readonly. Note: This is purely to indicate to VM guest that this is // readonly, it should *not* be used as a security measure, as the VM guest could remount it R/W. if readonly { agentMount.Options = append(agentMount.Options, "ro") } // Record the mount for the agent. *agentMounts = append(*agentMounts, agentMount) // Add 9p share config. driveDir9pOpts := qemuDriveDirOpts{ dev: qemuDevOpts{ busName: busName, devBus: qemuDev["bus"].(string), devAddr: qemuDev["addr"].(string), multifunction: qemuDev["multifunction"].(bool), }, devName: driveConf.DevName, mountTag: mountTag, readonly: readonly, path: driveConf.DevPath, protocol: "9p", } return qemuDriveDir(&driveDir9pOpts) } // addDriveDirConfigVirtiofs adds the qemu config required for adding a supplementary drive directory share using virtiofs. func (d *qemu) addDriveDirConfigVirtiofs(qemuDev map[string]any, agentMounts *[]instancetype.VMAgentMount, driveConf deviceConfig.MountEntryItem) (monitorHook, error) { escapedDeviceName := linux.PathNameEncode(driveConf.DevName) deviceID := qemuDeviceIDPrefix + escapedDeviceName mountTag := d.mountTagName(driveConf.DevName, qemuMountTagMaxLength) if agentMounts != nil { agentMount := instancetype.VMAgentMount{ Source: mountTag, Target: driveConf.TargetPath, FSType: "virtiofs", } // Indicate to agent to mount this readonly. Note: This is purely to indicate to VM guest that this is // readonly, it should *not* be used as a security measure, as the VM guest could remount it R/W. if slices.Contains(driveConf.Opts, "ro") { agentMount.Options = append(agentMount.Options, "ro") } // Record the mount for the agent. *agentMounts = append(*agentMounts, agentMount) } if qemuDev == nil { qemuDev = map[string]any{} } qemuDev["driver"] = "vhost-user-fs-pci" qemuDev["tag"] = mountTag qemuDev["chardev"] = mountTag qemuDev["id"] = deviceID monHook := func(m *qmp.Monitor) error { reverter := revert.New() defer reverter.Fail() // Detect virtiofsd path. virtiofsdSockPath := filepath.Join(d.DevicesPath(), fmt.Sprintf("virtio-fs.%s.sock", driveConf.DevName)) if !util.PathExists(virtiofsdSockPath) { return errors.New("Virtiofsd isn't running") } addr, err := net.ResolveUnixAddr("unix", virtiofsdSockPath) if err != nil { return err } virtiofsSock, err := net.DialUnix("unix", nil, addr) if err != nil { return fmt.Errorf("Error connecting to virtiofs socket %q: %w", virtiofsdSockPath, err) } defer func() { _ = virtiofsSock.Close() }() // Close file after device has been added. virtiofsFile, err := virtiofsSock.File() if err != nil { return fmt.Errorf("Error opening virtiofs socket %q: %w", virtiofsdSockPath, err) } err = m.SendFile(virtiofsdSockPath, virtiofsFile) if err != nil { return fmt.Errorf("Failed to send virtiofs file descriptor: %w", err) } reverter.Add(func() { _ = m.CloseFile(virtiofsdSockPath) }) err = m.AddCharDevice(map[string]any{ "id": mountTag, "backend": map[string]any{ "type": "socket", "data": map[string]any{ "addr": map[string]any{ "type": "fd", "data": map[string]any{ "str": virtiofsdSockPath, }, }, "server": false, }, }, }) if err != nil { return fmt.Errorf("Failed to add the character device: %w", err) } reverter.Add(func() { _ = m.RemoveCharDevice(mountTag) }) _, ok := qemuDev["bus"] if !ok { // Try to get a PCI address for hotplugging. pciDeviceName, err := d.getPCIHotplug() if err != nil { return err } d.logger.Debug("Using PCI bus device to hotplug virtiofs into", logger.Ctx{"device": driveConf.DevName, "port": pciDeviceName, "was": qemuDev["bus"]}) qemuDev["bus"] = pciDeviceName qemuDev["addr"] = "00.0" } err = m.AddDevice(qemuDev) if err != nil { return fmt.Errorf("Failed to add the virtiofs device: %w", err) } reverter.Success() return nil } return monHook, nil } // addDriveConfig adds the qemu config required for adding a supplementary drive. func (d *qemu) addDriveConfig(qemuDev map[string]any, bootIndexes map[string]int, driveConf deviceConfig.MountEntryItem) (monitorHook, error) { aioMode := "native" // Use native kernel async IO and O_DIRECT by default. cacheMode := "none" // Bypass host cache, use O_DIRECT semantics by default. media := "disk" isRBDImage := strings.HasPrefix(driveConf.DevPath, device.RBDFormatPrefix) // Use io_uring over native for added performance when supported by QEMU. info := DriverStatuses()[instancetype.VM].Info _, ioUring := info.Features["io_uring"] if slices.Contains(driveConf.Opts, device.DiskIOUring) && ioUring { aioMode = "io_uring" } var isBlockDev bool var srcDevPath string // Detect device caches and I/O modes. if isRBDImage { // For RBD, we want writeback to allow for the system-configured "rbd cache" to take effect if present. cacheMode = "writeback" } else { srcDevPath = driveConf.DevPath // This should not be used for passing to QEMU, only for probing. // Detect if existing file descriptor format is being supplied. if strings.HasPrefix(driveConf.DevPath, fmt.Sprintf("%s:", device.DiskFileDescriptorMountPrefix)) { // Expect devPath in format "fd::". devPathParts := strings.SplitN(driveConf.DevPath, ":", 3) if len(devPathParts) != 3 || !strings.HasPrefix(driveConf.DevPath, fmt.Sprintf("%s:", device.DiskFileDescriptorMountPrefix)) { return nil, fmt.Errorf("Unexpected devPath file descriptor format %q", driveConf.DevPath) } // Map the file descriptor to the file descriptor path it will be in the QEMU process. fd, err := strconv.Atoi(devPathParts[1]) if err != nil { return nil, fmt.Errorf("Invalid file descriptor %q: %w", devPathParts[1], err) } // Extract original dev path for additional probing below. srcDevPath = devPathParts[2] if srcDevPath == "" { return nil, errors.New("Device source path is empty") } driveConf.DevPath = fmt.Sprintf("/proc/self/fd/%d", fd) } else if driveConf.TargetPath != "/" { // Only the root disk device is allowed to pass local devices to us without using an FD. return nil, fmt.Errorf("Invalid device path format %q", driveConf.DevPath) } srcDevPathInfo, err := os.Stat(srcDevPath) if err != nil { return nil, fmt.Errorf("Invalid source path %q: %w", srcDevPath, err) } isBlockDev = linux.IsBlockdev(srcDevPathInfo.Mode()) // Handle I/O mode configuration. if !isBlockDev { // Disk dev path is a file, check what the backing filesystem is. fsType, err := linux.DetectFilesystem(srcDevPath) if err != nil { return nil, fmt.Errorf("Failed detecting filesystem type of %q: %w", srcDevPath, err) } // If backing FS is ZFS or BTRFS, avoid using direct I/O and use host page cache only. // We've seen ZFS lock up and BTRFS checksum issues when using direct I/O on image files. if fsType == "zfs" || fsType == "btrfs" { aioMode = "threads" cacheMode = "writeback" // Use host cache, with neither O_DSYNC nor O_DIRECT semantics. } else { // Use host cache, with neither O_DSYNC nor O_DIRECT semantics if filesystem // doesn't support Direct I/O. f, err := os.OpenFile(srcDevPath, unix.O_DIRECT|unix.O_RDONLY, 0) if err != nil { cacheMode = "writeback" } else { _ = f.Close() // Don't leak FD. } } if cacheMode == "writeback" && driveConf.FSType != "iso9660" { // Only warn about using writeback cache if the drive image is writable. d.logger.Warn("Using writeback cache I/O", logger.Ctx{"device": driveConf.DevName, "devPath": srcDevPath, "fsType": fsType}) } } else if !slices.Contains(driveConf.Opts, device.DiskDirectIO) { // If drive config indicates we need to use unsafe I/O then use it. d.logger.Warn("Using unsafe cache I/O", logger.Ctx{"device": driveConf.DevName, "devPath": srcDevPath}) aioMode = "threads" cacheMode = "unsafe" // Use host cache, but ignore all sync requests from guest. } } // Special case ISO images as cdroms. if driveConf.FSType == "iso9660" { media = "cdrom" } // Check if the user has overridden the bus. bus := "virtio-scsi" for _, opt := range driveConf.Opts { if !strings.HasPrefix(opt, "bus=") { continue } bus = strings.TrimPrefix(opt, "bus=") break } // Check if the user has overridden the cache mode. for _, opt := range driveConf.Opts { if !strings.HasPrefix(opt, "cache=") { continue } cacheMode = strings.TrimPrefix(opt, "cache=") break } // Check if the user has overridden the WWN. var wwn string for _, opt := range driveConf.Opts { if !strings.HasPrefix(opt, "wwn=") { continue } wwn = strings.TrimPrefix(opt, "wwn=") break } // QMP uses two separate values for the cache. directCache := true // Bypass host cache, use O_DIRECT semantics by default. noFlushCache := false // Don't ignore any flush requests for the device. if cacheMode == "unsafe" { aioMode = "threads" directCache = false noFlushCache = true } else if cacheMode == "writeback" { aioMode = "threads" directCache = false } escapedDeviceName := linux.PathNameEncode(driveConf.DevName) blockDev := map[string]any{ "aio": aioMode, "cache": map[string]any{ "direct": directCache, "no-flush": noFlushCache, }, "discard": "unmap", // Forward as an unmap request. This is the same as `discard=on` in the qemu config file. "driver": "file", "node-name": d.blockNodeName(escapedDeviceName), "read-only": false, } var rbdSecret string // If driver is "file", QEMU requires the file to be a regular file. // However, if the file is a character or block device, driver needs to be set to "host_device". if isBlockDev { blockDev["driver"] = "host_device" } else if isRBDImage { blockDev["driver"] = "rbd" poolName, volName, opts, err := device.DiskParseRBDFormat(driveConf.DevPath) if err != nil { return nil, fmt.Errorf("Failed parsing rbd string: %w", err) } // Driver and pool name arguments can be ignored as CephGetRBDImageName doesn't need them. volumeType := storageDrivers.VolumeTypeCustom volumeName := project.StorageVolume(d.project.Name, volName) // Handle different name for instance volumes. if driveConf.TargetPath == "/" { volumeType = storageDrivers.VolumeTypeVM volumeName = volName } // Identify the right content type. rbdContentType := storageDrivers.ContentTypeBlock if driveConf.FSType == "iso9660" { rbdContentType = storageDrivers.ContentTypeISO } // Get the RBD image name. vol := storageDrivers.NewVolume(nil, "", volumeType, rbdContentType, volumeName, nil, nil) rbdImageName := storageDrivers.CephGetRBDImageName(vol, "", false) // Scan & pass through options. clusterName := storageDrivers.CephDefaultCluster userName := storageDrivers.CephDefaultUser blockDev["pool"] = poolName blockDev["image"] = rbdImageName for key, val := range opts { // We use 'id' where qemu uses 'user'. if key == "id" { blockDev["user"] = val userName = val } else if key == "cluster" { clusterName = val } else { blockDev[key] = val } } // Parse the secret (QEMU runs unprivileged and can't read the keyring directly). rbdSecret, err = storageDrivers.CephKeyring(clusterName, userName) if err != nil { return nil, err } // The aio option isn't available when using the rbd driver. delete(blockDev, "aio") } readonly := slices.Contains(driveConf.Opts, "ro") || media == "cdrom" if readonly { blockDev["read-only"] = true } if !isRBDImage { blockDev["locking"] = "off" } if qemuDev == nil { qemuDev = map[string]any{} } qemuDev["id"] = fmt.Sprintf("%s%s", qemuDeviceIDPrefix, escapedDeviceName) qemuDev["drive"] = blockDev["node-name"].(string) // Max serial length is 36 characters: prefix + 30 chars. // For nvme and virtio-blk, the maximum serial length is 20 characters: prefix + 14 chars. serialMaxLength := 30 if slices.Contains([]string{"nvme", "virtio-blk"}, bus) { serialMaxLength = 14 } qemuDev["serial"] = fmt.Sprintf("%s%s", qemuBlockDevIDPrefix, hashValue(escapedDeviceName, serialMaxLength)) if wwn != "" { wwnID, err := strconv.ParseUint(strings.TrimPrefix(wwn, "0x"), 16, 64) if err != nil { return nil, err } qemuDev["wwn"] = wwnID } if bus == "virtio-scsi" { qemuDev["device_id"] = d.blockNodeName(escapedDeviceName) qemuDev["channel"] = 0 qemuDev["lun"] = 1 qemuDev["bus"] = "qemu_scsi.0" if media == "disk" { qemuDev["driver"] = "scsi-hd" } else if media == "cdrom" { qemuDev["driver"] = "scsi-cd" } } else if slices.Contains([]string{"nvme", "virtio-blk"}, bus) { if qemuDev["bus"] == nil { // Try to get a PCI address for hotplugging. pciDeviceName, err := d.getPCIHotplug() if err != nil { return nil, err } d.logger.Debug("Using PCI bus device to hotplug drive into", logger.Ctx{"device": driveConf.DevName, "port": pciDeviceName}) qemuDev["bus"] = pciDeviceName qemuDev["addr"] = "00.0" } qemuDev["driver"] = bus } else if bus == "usb" { qemuDev["lun"] = 0 switch media { case "disk": qemuDev["driver"] = "scsi-hd" case "cdrom": qemuDev["driver"] = "scsi-cd" } } if bootIndexes != nil { qemuDev["bootindex"] = bootIndexes[driveConf.DevName] } monHook := func(m *qmp.Monitor) error { reverter := revert.New() defer reverter.Fail() nodeName := d.blockNodeName(escapedDeviceName) if isRBDImage { secretID := fmt.Sprintf("pool_%s_%s", blockDev["pool"], blockDev["user"]) err := m.AddSecret(secretID, rbdSecret) if err != nil { return err } blockDev["key-secret"] = secretID } else { permissions := unix.O_RDWR if readonly { permissions = unix.O_RDONLY } if directCache { permissions |= unix.O_DIRECT } f, err := os.OpenFile(driveConf.DevPath, permissions, 0) if err != nil { return fmt.Errorf("Failed opening file descriptor for disk device %q: %w", driveConf.DevName, err) } defer func() { _ = f.Close() }() info, err := m.SendFileWithFDSet(nodeName, f, readonly) if err != nil { return fmt.Errorf("Failed sending file descriptor of %q for disk device %q: %w", f.Name(), driveConf.DevName, err) } reverter.Add(func() { _ = m.RemoveFDFromFDSet(nodeName) }) isQcow2, err := d.isQCOW2(srcDevPath) if err != nil { return fmt.Errorf("Failed checking disk format: %w", err) } if isQcow2 { blockDev = map[string]any{ "driver": "qcow2", "discard": "unmap", // Forward as an unmap request. This is the same as `discard=on` in the qemu config file. "node-name": d.blockNodeName(escapedDeviceName), "read-only": false, "file": map[string]any{ "driver": "host_device", "filename": fmt.Sprintf("/dev/fdset/%d", info.ID), "aio": aioMode, "cache": map[string]any{ "direct": directCache, "no-flush": noFlushCache, }, }, } // If there are any children, load block information about them. if len(driveConf.BackingPath) > 0 { backingBlockDev, err := d.qcow2BlockDev(m, nodeName, aioMode, directCache, noFlushCache, permissions, readonly, driveConf.BackingPath, 0) if err != nil { return err } blockDev["backing"] = backingBlockDev } } else { blockDev["filename"] = fmt.Sprintf("/dev/fdset/%d", info.ID) } } err := m.AddBlockDevice(blockDev, qemuDev, bus == "usb") if err != nil { return fmt.Errorf("Failed adding block device for disk device %q: %w", driveConf.DevName, err) } if driveConf.Limits != nil { err = m.SetBlockThrottle(qemuDev["id"].(string), int(driveConf.Limits.ReadBytes), int(driveConf.Limits.WriteBytes), int(driveConf.Limits.ReadIOps), int(driveConf.Limits.WriteIOps)) if err != nil { return fmt.Errorf("Failed applying limits for disk device %q: %w", driveConf.DevName, err) } } reverter.Success() return nil } return monHook, nil } // addNetDevConfig adds the qemu config required for adding a network device. // The qemuDev map is expected to be preconfigured with the settings for an existing port to use for the device. func (d *qemu) addNetDevConfig(busName string, qemuDev map[string]any, bootIndexes map[string]int, nicConfig []deviceConfig.RunConfigItem) (monitorHook, error) { reverter := revert.New() defer reverter.Fail() var devName, nicName, devHwaddr, pciSlotName, pciIOMMUGroup, vDPADevName, vhostVDPAPath, maxVQP string connected := true for _, nicItem := range nicConfig { if nicItem.Key == "devName" { devName = nicItem.Value } else if nicItem.Key == "link" { nicName = nicItem.Value } else if nicItem.Key == "hwaddr" { devHwaddr = nicItem.Value } else if nicItem.Key == "pciSlotName" { pciSlotName = nicItem.Value } else if nicItem.Key == "pciIOMMUGroup" { pciIOMMUGroup = nicItem.Value } else if nicItem.Key == "vDPADevName" { vDPADevName = nicItem.Value } else if nicItem.Key == "vhostVDPAPath" { vhostVDPAPath = nicItem.Value } else if nicItem.Key == "maxVQP" { maxVQP = nicItem.Value } else if nicItem.Key == "connected" { connected = util.IsTrueOrEmpty(nicItem.Value) } } escapedDeviceName := linux.PathNameEncode(devName) qemuDev["id"] = fmt.Sprintf("%s%s", qemuDeviceIDPrefix, escapedDeviceName) if len(bootIndexes) > 0 { bootIndex, found := bootIndexes[devName] if found { qemuDev["bootindex"] = bootIndex } } var monHook func(m *qmp.Monitor) error // configureQueues modifies qemuDev with the queue configuration based on vCPUs. // Returns the number of queues to use with NIC. configureQueues := func(cpuCount int) int { // Number of queues is the same as number of vCPUs. Run with a minimum of two queues. queueCount := max(cpuCount, 2) // Number of vectors is number of vCPUs * 2 (RX/TX) + 2 (config/control MSI-X). vectors := 2*queueCount + 2 if busName != "usb" { qemuDev["mq"] = true if slices.Contains([]string{"pcie", "pci"}, busName) { qemuDev["vectors"] = vectors } } return queueCount } // tapMonHook is a helper function used as the monitor hook for macvtap and tap interfaces to open // multi-queue file handles to both the interface device and the vhost-net device and pass them to QEMU. tapMonHook := func(deviceFile func() (*os.File, error)) func(m *qmp.Monitor) error { return func(m *qmp.Monitor) error { reverter := revert.New() defer reverter.Fail() cpus, err := m.QueryCPUs() if err != nil { return errors.New("Failed getting CPU list for NIC queues") } queueCount := configureQueues(len(cpus)) // Enable vhost_net offloading if available. info := DriverStatuses()[instancetype.VM].Info _, vhostNetEnabled := info.Features["vhost_net"] // Open the device once for each queue and pass to QEMU. fds := make([]string, 0, queueCount) vhostfds := make([]string, 0, queueCount) for i := range queueCount { devFile, err := deviceFile() if err != nil { return fmt.Errorf("Error opening netdev file for queue %d: %w", i, err) } defer func() { _ = devFile.Close() }() // Close file after device has been added. devFDName := fmt.Sprintf("%s.%d", devFile.Name(), i) err = m.SendFile(devFDName, devFile) if err != nil { return fmt.Errorf("Failed to send %q file descriptor for queue %d: %w", devFDName, i, err) } reverter.Add(func() { _ = m.CloseFile(devFDName) }) fds = append(fds, devFDName) if vhostNetEnabled { // Open a vhost-net file handle for each device file handle. vhostFile, err := os.OpenFile("/dev/vhost-net", os.O_RDWR, 0) if err != nil { return fmt.Errorf("Error opening /dev/vhost-net for queue %d: %w", i, err) } defer func() { _ = vhostFile.Close() }() // Close file after device has been added. vhostFDName := fmt.Sprintf("%s.%d", vhostFile.Name(), i) err = m.SendFile(vhostFDName, vhostFile) if err != nil { return fmt.Errorf("Failed to send %q file descriptor for queue %d: %w", vhostFDName, i, err) } reverter.Add(func() { _ = m.CloseFile(vhostFDName) }) vhostfds = append(vhostfds, vhostFDName) } } qemuNetDev := map[string]any{ "id": fmt.Sprintf("%s%s", qemuNetDevIDPrefix, escapedDeviceName), "type": "tap", "vhost": vhostNetEnabled, } if slices.Contains([]string{"pcie", "pci"}, busName) { qemuDev["driver"] = "virtio-net-pci" } else if busName == "ccw" { qemuDev["driver"] = "virtio-net-ccw" } else if busName == "usb" { qemuDev["driver"] = "usb-net" } qemuNetDev["fds"] = strings.Join(fds, ":") if len(vhostfds) > 0 { qemuNetDev["vhostfds"] = strings.Join(vhostfds, ":") } qemuDev["netdev"] = qemuNetDev["id"].(string) qemuDev["mac"] = devHwaddr err = m.AddNIC(qemuNetDev, qemuDev, connected) if err != nil { return fmt.Errorf("Failed setting up device %q: %w", devName, err) } reverter.Success() return nil } } // Detect MACVTAP interface types and figure out which tap device is being used. // This is so we can open a file handle to the tap device and pass it to the qemu process. if util.PathExists(fmt.Sprintf("/sys/class/net/%s/macvtap", nicName)) { content, err := os.ReadFile(fmt.Sprintf("/sys/class/net/%s/ifindex", nicName)) if err != nil { return nil, fmt.Errorf("Error getting tap device ifindex: %w", err) } ifindex, err := strconv.Atoi(strings.TrimSpace(string(content))) if err != nil { return nil, fmt.Errorf("Error parsing tap device ifindex: %w", err) } devFile := func() (*os.File, error) { return os.OpenFile(fmt.Sprintf("/dev/tap%d", ifindex), os.O_RDWR, 0) } monHook = tapMonHook(devFile) } else if util.PathExists(fmt.Sprintf("/sys/class/net/%s/tun_flags", nicName)) { // Detect TAP interface and use IOCTL TUNSETIFF on /dev/net/tun to get the file handle to it. // This is so we can open a file handle to the tap device and pass it to the qemu process. devFile := func() (*os.File, error) { reverter := revert.New() defer reverter.Fail() f, err := os.OpenFile("/dev/net/tun", os.O_RDWR, 0) if err != nil { return nil, err } reverter.Add(func() { _ = f.Close() }) ifr, err := unix.NewIfreq(nicName) if err != nil { return nil, fmt.Errorf("Error creating new ifreq for %q: %w", nicName, err) } // These settings need to be compatible with what the device created the interface with // and what QEMU is expecting. ifr.SetUint16(unix.IFF_TAP | unix.IFF_NO_PI | unix.IFF_ONE_QUEUE | unix.IFF_MULTI_QUEUE | unix.IFF_VNET_HDR) // Sets the file handle to point to the requested NIC interface. err = unix.IoctlIfreq(int(f.Fd()), unix.TUNSETIFF, ifr) if err != nil { return nil, fmt.Errorf("Error getting TAP file handle for %q: %w", nicName, err) } reverter.Success() return f, nil } monHook = tapMonHook(devFile) } else if util.PathExists(vhostVDPAPath) { monHook = func(m *qmp.Monitor) error { reverter := revert.New() defer reverter.Fail() vdpaDevFile, err := os.OpenFile(vhostVDPAPath, os.O_RDWR, 0) if err != nil { return fmt.Errorf("Error opening vDPA device file %q: %w", vdpaDevFile.Name(), err) } defer func() { _ = vdpaDevFile.Close() }() // Close file after device has been added. vDPADevFDName := fmt.Sprintf("%s.0", vdpaDevFile.Name()) err = m.SendFile(vDPADevFDName, vdpaDevFile) if err != nil { return fmt.Errorf("Failed to send %q file descriptor: %w", vDPADevFDName, err) } reverter.Add(func() { _ = m.CloseFile(vDPADevFDName) }) queues, err := strconv.Atoi(maxVQP) if err != nil { return fmt.Errorf("Failed to convert maxVQP (%q) to int: %w", maxVQP, err) } qemuNetDev := map[string]any{ "id": fmt.Sprintf("vhost-%s", vDPADevName), "type": "vhost-vdpa", "vhostfd": vDPADevFDName, "queues": queues, } if slices.Contains([]string{"pcie", "pci"}, busName) { qemuDev["driver"] = "virtio-net-pci" } else if busName == "ccw" { qemuDev["driver"] = "virtio-net-ccw" } else if busName == "usb" { qemuDev["driver"] = "usb-net" } qemuDev["netdev"] = qemuNetDev["id"].(string) qemuDev["page-per-vq"] = true qemuDev["iommu_platform"] = true qemuDev["disable-legacy"] = true err = m.AddNIC(qemuNetDev, qemuDev, connected) if err != nil { return fmt.Errorf("Failed setting up device %q: %w", devName, err) } reverter.Success() return nil } } else if pciSlotName != "" { // Detect physical passthrough device. if slices.Contains([]string{"pcie", "pci"}, busName) { qemuDev["driver"] = "vfio-pci" } else if busName == "ccw" { qemuDev["driver"] = "vfio-ccw" } qemuDev["host"] = pciSlotName if d.state.OS.UnprivUser != "" { if pciIOMMUGroup == "" { return nil, errors.New("No PCI IOMMU group supplied") } vfioGroupFile := fmt.Sprintf("/dev/vfio/%s", pciIOMMUGroup) err := os.Chown(vfioGroupFile, int(d.state.OS.UnprivUID), -1) if err != nil { return nil, fmt.Errorf("Failed to chown vfio group device %q: %w", vfioGroupFile, err) } reverter.Add(func() { _ = os.Chown(vfioGroupFile, 0, -1) }) } monHook = func(m *qmp.Monitor) error { err := m.AddNIC(nil, qemuDev, connected) if err != nil { return fmt.Errorf("Failed setting up device %q: %w", devName, err) } return nil } } if monHook == nil { return nil, errors.New("Unrecognised device type") } reverter.Success() return monHook, nil } // writeNICDevConfig writes the NIC config for the specified device into the NICConfigDir. // This will be used by the agent to rename the NIC interfaces inside the VM guest. func (d *qemu) writeNICDevConfig(mtuStr string, devName string, nicName string, devHwaddr string) error { // Parse MAC address to ensure it is in a canonical form (avoiding casing/presentation differences). hw, err := net.ParseMAC(devHwaddr) if err != nil { return fmt.Errorf("Failed parsing MAC %q: %w", devHwaddr, err) } nicConfig := deviceConfig.NICConfig{ DeviceName: devName, NICName: nicName, MACAddress: hw.String(), } if mtuStr != "" { mtuInt, err := strconv.ParseUint(mtuStr, 10, 32) if err != nil { return fmt.Errorf("Failed parsing MTU: %w", err) } nicConfig.MTU = uint32(mtuInt) } nicConfigBytes, err := json.Marshal(nicConfig) if err != nil { return fmt.Errorf("Failed encoding NIC config: %w", err) } nicFile := filepath.Join(d.Path(), "config", deviceConfig.NICConfigDir, fmt.Sprintf("%s.json", linux.PathNameEncode(nicConfig.DeviceName))) err = os.WriteFile(nicFile, nicConfigBytes, 0o700) if err != nil { return fmt.Errorf("Failed writing NIC config: %w", err) } return nil } // addPCIDevConfig adds the qemu config required for adding a raw PCI device. func (d *qemu) addPCIDevConfig(conf *[]cfg.Section, bus *qemuBus, pciConfig []deviceConfig.RunConfigItem) error { var devName, pciSlotName string firmware := true for _, pciItem := range pciConfig { if pciItem.Key == "devName" { devName = pciItem.Value } else if pciItem.Key == "pciSlotName" { pciSlotName = pciItem.Value } else if pciItem.Key == "firmware" { firmware = util.IsTrueOrEmpty(pciItem.Value) } } devBus, devAddr, multi := bus.allocate(fmt.Sprintf("incus_%s", devName)) pciPhysicalOpts := qemuPCIPhysicalOpts{ dev: qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, }, devName: devName, pciSlotName: pciSlotName, firmware: firmware, } *conf = append(*conf, qemuPCIPhysical(&pciPhysicalOpts)...) return nil } // addGPUDevConfig adds the qemu config required for adding a GPU device. func (d *qemu) addGPUDevConfig(conf *[]cfg.Section, bus *qemuBus, gpuConfig []deviceConfig.RunConfigItem) error { var devName, pciSlotName, vgpu string for _, gpuItem := range gpuConfig { if gpuItem.Key == "devName" { devName = gpuItem.Value } else if gpuItem.Key == "pciSlotName" { pciSlotName = gpuItem.Value } else if gpuItem.Key == "vgpu" { vgpu = gpuItem.Value } } vgaMode := func() bool { // No VGA mode on mdev. if vgpu != "" { return false } // No VGA mode on non-x86. if d.architecture != osarch.ARCH_64BIT_INTEL_X86 { return false } // Only enable if present on the card. if !util.PathExists(filepath.Join("/sys/bus/pci/devices", pciSlotName, "boot_vga")) { return false } // Skip SRIOV VFs as those are shared with the host card. if util.PathExists(filepath.Join("/sys/bus/pci/devices", pciSlotName, "physfn")) { return false } return true }() devBus, devAddr, multi := bus.allocate(fmt.Sprintf("incus_%s", devName)) gpuDevPhysicalOpts := qemuGPUDevPhysicalOpts{ dev: qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, }, devName: devName, pciSlotName: pciSlotName, vga: vgaMode, vgpu: vgpu, } // Add main GPU device in VGA mode to qemu config. *conf = append(*conf, qemuGPUDevPhysical(&gpuDevPhysicalOpts)...) var iommuGroupPath string if vgpu != "" { iommuGroupPath = filepath.Join("/sys/bus/mdev/devices", vgpu, "iommu_group", "devices") } else { // Add any other related IOMMU VFs as generic PCI devices. iommuGroupPath = filepath.Join("/sys/bus/pci/devices", pciSlotName, "iommu_group", "devices") } if util.PathExists(iommuGroupPath) { // Extract parent slot name by removing any virtual function ID. parts := strings.SplitN(pciSlotName, ".", 2) prefix := parts[0] // Iterate the members of the IOMMU group and override any that match the parent slot name prefix. err := filepath.Walk(iommuGroupPath, func(path string, _ os.FileInfo, err error) error { if err != nil { return err } iommuSlotName := filepath.Base(path) // Virtual function's address is dir name. // Match any VFs that are related to the GPU device (but not the GPU device itself). if strings.HasPrefix(iommuSlotName, prefix) && iommuSlotName != pciSlotName { // Add VF device without VGA mode to qemu config. devBus, devAddr, multi := bus.allocate(fmt.Sprintf("incus_%s", devName)) gpuDevPhysicalOpts := qemuGPUDevPhysicalOpts{ dev: qemuDevOpts{ busName: bus.name, devBus: devBus, devAddr: devAddr, multifunction: multi, }, // Generate associated device name by combining main device name and VF ID. devName: fmt.Sprintf("%s_%s", devName, devAddr), pciSlotName: iommuSlotName, vga: false, vgpu: "", } *conf = append(*conf, qemuGPUDevPhysical(&gpuDevPhysicalOpts)...) } return nil }) if err != nil { return err } } return nil } func (d *qemu) addUSBDeviceConfig(usbDev deviceConfig.USBDeviceItem) (monitorHook, error) { qemuDev := map[string]any{ "id": fmt.Sprintf("%s%s", qemuDeviceIDPrefix, usbDev.DeviceName), "driver": "usb-host", "bus": "qemu_usb.0", } monHook := func(m *qmp.Monitor) error { reverter := revert.New() defer reverter.Fail() f, err := os.OpenFile(usbDev.HostDevicePath, unix.O_RDWR, 0) if err != nil { return fmt.Errorf("Failed to open host device: %w", err) } defer func() { _ = f.Close() }() info, err := m.SendFileWithFDSet(qemuDev["id"].(string), f, false) if err != nil { return fmt.Errorf("Failed to send file descriptor: %w", err) } reverter.Add(func() { _ = m.RemoveFDFromFDSet(qemuDev["id"].(string)) }) qemuDev["hostdevice"] = fmt.Sprintf("/dev/fdset/%d", info.ID) err = m.AddDevice(qemuDev) if err != nil { return fmt.Errorf("Failed to add device: %w", err) } reverter.Success() return nil } return monHook, nil } func (d *qemu) addTPMDeviceConfig(conf *[]cfg.Section, tpmConfig []deviceConfig.RunConfigItem, fdFiles *[]*os.File) error { var devName, socketPath string for _, tpmItem := range tpmConfig { if tpmItem.Key == "path" { socketPath = tpmItem.Value } else if tpmItem.Key == "devName" { devName = tpmItem.Value } } fd, err := unix.Open(socketPath, unix.O_PATH|unix.O_CLOEXEC, 0) if err != nil { return err } tpmFD := d.addFileDescriptor(fdFiles, os.NewFile(uintptr(fd), socketPath)) tpmDriver := "tpm-tis-device" if d.architecture == osarch.ARCH_64BIT_INTEL_X86 { tpmDriver = "tpm-crb" } tpmOpts := qemuTPMOpts{ devName: devName, path: fmt.Sprintf("/proc/self/fd/%d", tpmFD), driver: tpmDriver, } *conf = append(*conf, qemuTPM(&tpmOpts)...) return nil } func (d *qemu) addVmgenDeviceConfig(conf *[]cfg.Section, guid string) error { vmgenIDOpts := qemuVmgenIDOpts{ guid: guid, } *conf = append(*conf, qemuVmgen(&vmgenIDOpts)...) return nil } // pidFilePath returns the path where the qemu process should write its PID. func (d *qemu) pidFilePath() string { return filepath.Join(d.RunPath(), "qemu.pid") } // pid gets the PID of the running qemu process. Returns 0 if PID file or process not found, and -1 if err non-nil. func (d *qemu) pid() (int, error) { pidStr, err := os.ReadFile(d.pidFilePath()) if errors.Is(err, fs.ErrNotExist) { return 0, nil // PID file has gone. } if err != nil { return -1, err } pid, err := strconv.Atoi(strings.TrimSpace(string(pidStr))) if err != nil { return -1, err } cmdLineProcFilePath := fmt.Sprintf("/proc/%d/cmdline", pid) cmdLine, err := os.ReadFile(cmdLineProcFilePath) if err != nil { return 0, nil // Process has gone. } qemuSearchString := []byte("qemu-system") instUUID := []byte(d.localConfig["volatile.uuid"]) if !bytes.Contains(cmdLine, qemuSearchString) || !bytes.Contains(cmdLine, instUUID) { return -1, errors.New("PID doesn't match the running process") } return pid, nil } // forceStop kills the QEMU prorcess if running. func (d *qemu) forceStop() error { pid, _ := d.pid() if pid > 0 { err := d.killQemuProcess(pid) if err != nil { return fmt.Errorf("Failed to stop VM process %d: %w", pid, err) } } return nil } // Stop the VM. func (d *qemu) Stop(stateful bool) error { d.logger.Debug("Stop started", logger.Ctx{"stateful": stateful}) defer d.logger.Debug("Stop finished", logger.Ctx{"stateful": stateful}) // Must be run prior to creating the operation lock. // Allow to proceed if statusCode is Error or Frozen as we may need to forcefully kill the QEMU process. // Also Stop() is called from migrateSendLive in some cases, and instance status will be Frozen then. statusCode := d.statusCode() if !d.isRunningStatusCode(statusCode) && statusCode != api.Error && statusCode != api.Frozen { return ErrInstanceIsStopped } // Check for stateful. if stateful { // Confirm the instance has stateful migration enabled. if !d.CanLiveMigrate() { return errors.New("Stateful stop requires migration.stateful to be set to true") } // Confirm the instance has sufficient reserved state space. err := d.checkStateStorage() if err != nil { return err } } // Attempt to save the console log from ring buffer before the instance is stopped. Must be run prior to creating the operation lock. _, _ = d.ConsoleLog() // Setup a new operation. // Allow inheriting of ongoing restart or restore operation (we are called from restartCommon and Restore). // Don't allow reuse when creating a new stop operation. This prevents other operations from interfering. // Allow reuse of a reusable ongoing stop operation as Shutdown() may be called first, which allows reuse // of its operations. This allow for Stop() to inherit from Shutdown() where instance is stuck. op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStop, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore, operationlock.ActionMigrate}, false, true) if err != nil { if errors.Is(err, operationlock.ErrNonReusuableSucceeded) { // An existing matching operation has now succeeded, return. return nil } return err } // Connect to the monitor. monitor, err := d.qmpConnect() if err != nil { d.logger.Warn("Failed connecting to monitor, forcing stop", logger.Ctx{"err": err}) // If we fail to connect, it's most likely because the VM is already off, but it could also be // because the qemu process is not responding, check if process still exists and kill it if needed. err = d.forceStop() if err != nil { op.Done(err) return err } // Wait for QEMU process to exit and perform device cleanup. // Treat as host-qmp-quit so autoRestart isn't triggered for a user-requested force stop. err = d.onStop("stop", qmp.EventVMShutdownReasonQuit) if err != nil { op.Done(err) return err } op.Done(nil) return nil } // Handle stateful stop. if stateful { // Dump the state. err = d.saveState(monitor) if err != nil { op.Done(err) return err } d.stateful = true err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateInstanceStatefulFlag(ctx, d.id, true) }) if err != nil { op.Done(err) return err } } // Get the wait channel. chDisconnect, err := monitor.Wait() if err != nil { d.logger.Warn("Failed getting monitor disconnection channel, forcing stop", logger.Ctx{"err": err}) err = d.forceStop() if err != nil { op.Done(err) return err } } else { // Request the VM stop immediately. err = monitor.Quit() if err != nil { d.logger.Warn("Failed sending monitor quit command, forcing stop", logger.Ctx{"err": err}) err = d.forceStop() if err != nil { op.Done(err) return err } } // Wait for QEMU to exit (can take a while if pending I/O). // As this is a forceful stop of the VM we don't wait as long as during a clean shutdown because // the QEMU process may be not responding correctly. ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() select { case <-chDisconnect: case <-ctx.Done(): d.logger.Warn("Timed out waiting for monitor to disconnect, forcing stop") err = d.forceStop() if err != nil { op.Done(err) return err } } } // Wait for operation lock to be Done. This is normally completed by onStop which picks up the same // operation lock and then marks it as Done after the instance stops and the devices have been cleaned up. // However if the operation has failed for another reason we will collect the error here. err = op.Wait(context.Background()) status := d.statusCode() if status != api.Stopped { errPrefix := fmt.Errorf("Failed stopping instance, status is %q", status) if err != nil { return fmt.Errorf("%s: %w", errPrefix.Error(), err) } return errPrefix } // Now handle errors from stop sequence and return to caller if wasn't completed cleanly. if err != nil { return err } return nil } // Unfreeze restores the instance to running. func (d *qemu) Unfreeze() error { // Connect to the monitor. monitor, err := d.qmpConnect() if err != nil { return err } // Send the cont command. err = monitor.Start() if err != nil { return err } d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceResumed.Event(d, nil)) return nil } // IsPrivileged does not apply to virtual machines. Always returns false. func (d *qemu) IsPrivileged() bool { return false } // snapshot creates a snapshot of the instance. func (d *qemu) snapshot(name string, expiry time.Time, stateful bool) error { var err error var monitor *qmp.Monitor // Deal with state. if stateful { // Confirm the instance has stateful migration enabled. if !d.CanLiveMigrate() { return errors.New("Stateful snapshot requires migration.stateful to be set to true") } // Confirm the instance has sufficient reserved state space. err = d.checkStateStorage() if err != nil { return err } // Quick checks. if !d.IsRunning() { return errors.New("Unable to create a stateful snapshot. The instance isn't running") } // Connect to the monitor. monitor, err = d.qmpConnect() if err != nil { return err } // Dump the state. err = d.saveState(monitor) if err != nil { return err } } // Create the snapshot. err = d.snapshotCommon(d, name, expiry, stateful) if err != nil { return err } // Resume the VM once the disk state has been saved. if stateful { // Remove the state from the main volume. err = os.Remove(d.StatePath()) if err != nil { return err } err = monitor.Start() if err != nil { return err } } return nil } // Snapshot takes a new snapshot. func (d *qemu) Snapshot(name string, expiry time.Time, stateful bool) error { return d.snapshot(name, expiry, stateful) } // Restore restores an instance snapshot. func (d *qemu) Restore(source instance.Instance, stateful bool, diskOnly bool) error { op, err := operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionRestore, false, false) if err != nil { return fmt.Errorf("Failed to create instance restore operation: %w", err) } defer op.Done(nil) var ctxMap logger.Ctx // Load the storage driver. pool, err := storagePools.LoadByInstance(d.state, d) if err != nil { op.Done(err) return err } err = pool.CanRestoreInstanceSnapshot(d, source) if err != nil { op.Done(err) return err } // Stop the instance. wasRunning := false if d.IsRunning() { wasRunning = true ephemeral := d.IsEphemeral() if ephemeral { // Unset ephemeral flag. args := db.InstanceArgs{ Architecture: d.Architecture(), Config: d.LocalConfig(), Description: d.Description(), Devices: d.LocalDevices(), Ephemeral: false, Profiles: d.Profiles(), Project: d.Project().Name, Type: d.Type(), Snapshot: d.IsSnapshot(), } err := d.Update(args, false) if err != nil { op.Done(err) return err } // On function return, set the flag back on. defer func() { args.Ephemeral = ephemeral _ = d.Update(args, false) }() } // This will unmount the instance storage. err := d.Stop(false) if err != nil { op.Done(err) return err } // Refresh the operation as that one is now complete. op, err = operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionRestore, false, false) if err != nil { return fmt.Errorf("Failed to create instance restore operation: %w", err) } defer op.Done(nil) } ctxMap = logger.Ctx{ "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, "source": source.Name(), } d.logger.Info("Restoring instance", ctxMap) // Restore the rootfs. err = pool.RestoreInstanceSnapshot(d, source, nil) if err != nil { op.Done(err) return err } args := db.InstanceArgs{} if !diskOnly { // Restore the configuration. args = db.InstanceArgs{ Architecture: source.Architecture(), Config: source.LocalConfig(), Description: source.Description(), Devices: source.LocalDevices(), Ephemeral: source.IsEphemeral(), Profiles: source.Profiles(), Project: source.Project().Name, Type: source.Type(), Snapshot: source.IsSnapshot(), } } else { args = db.InstanceArgs{ Architecture: d.Architecture(), Config: d.LocalConfig(), Description: d.Description(), Devices: d.LocalDevices(), Ephemeral: d.IsEphemeral(), Profiles: d.Profiles(), Project: d.Project().Name, Type: d.Type(), Snapshot: d.IsSnapshot(), } args.Config["volatile.uuid.generation"] = source.LocalConfig()["volatile.uuid.generation"] } // Don't pass as user-requested as there's no way to fix a bad config. // This will call d.UpdateBackupFile() to ensure snapshot list is up to date. err = d.Update(args, false) if err != nil { op.Done(err) return err } d.stateful = stateful // Restart the instance. if wasRunning || stateful { d.logger.Debug("Starting instance after snapshot restore") err := d.Start(stateful) if err != nil { op.Done(err) return err } } d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceRestored.Event(d, map[string]any{"snapshot": source.Name()})) d.logger.Info("Restored instance", ctxMap) return nil } // Rename the instance. Accepts an argument to enable applying deferred TemplateTriggerRename. func (d *qemu) Rename(newName string, applyTemplateTrigger bool) error { oldName := d.Name() ctxMap := logger.Ctx{ "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, "newname": newName, } d.logger.Info("Renaming instance", ctxMap) // Quick checks. err := instance.ValidName(newName, d.IsSnapshot()) if err != nil { return err } if d.IsRunning() { return errors.New("Renaming of running instance not allowed") } // Clean things up. d.cleanup() pool, err := storagePools.LoadByInstance(d.state, d) if err != nil { return fmt.Errorf("Failed loading instance storage pool: %w", err) } if d.IsSnapshot() { _, newSnapName, _ := api.GetParentAndSnapshotName(newName) err = pool.RenameInstanceSnapshot(d, newSnapName, nil) if err != nil { return fmt.Errorf("Rename instance snapshot: %w", err) } } else { err = pool.RenameInstance(d, newName, nil) if err != nil { return fmt.Errorf("Rename instance: %w", err) } if applyTemplateTrigger { err = d.DeferTemplateApply(instance.TemplateTriggerRename) if err != nil { return err } } } if !d.IsSnapshot() { var results []string err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Rename all the instance snapshot database entries. results, err = tx.GetInstanceSnapshotsNames(ctx, d.project.Name, oldName) if err != nil { d.logger.Error("Failed to get instance snapshots", ctxMap) return fmt.Errorf("Failed to get instance snapshots: Failed getting instance snapshot names: %w", err) } for _, sname := range results { // Rename the snapshot. oldSnapName := strings.SplitN(sname, internalInstance.SnapshotDelimiter, 2)[1] baseSnapName := filepath.Base(sname) err := dbCluster.RenameInstanceSnapshot(ctx, tx.Tx(), d.project.Name, oldName, oldSnapName, baseSnapName) if err != nil { d.logger.Error("Failed renaming snapshot", ctxMap) return err } } return nil }) if err != nil { return err } } // Rename the instance database entry. err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { if d.IsSnapshot() { oldParts := strings.SplitN(oldName, internalInstance.SnapshotDelimiter, 2) newParts := strings.SplitN(newName, internalInstance.SnapshotDelimiter, 2) return dbCluster.RenameInstanceSnapshot(ctx, tx.Tx(), d.project.Name, oldParts[0], oldParts[1], newParts[1]) } return dbCluster.RenameInstance(ctx, tx.Tx(), d.project.Name, oldName, newName) }) if err != nil { d.logger.Error("Failed renaming instance", ctxMap) return err } // Rename the logging path. newFullName := project.Instance(d.Project().Name, d.Name()) _ = os.RemoveAll(internalUtil.LogPath(newFullName)) if util.PathExists(d.LogPath()) { err := os.Rename(d.LogPath(), internalUtil.LogPath(newFullName)) if err != nil { d.logger.Error("Failed renaming instance", ctxMap) return err } } // Rename the runtime path. newFullName = project.Instance(d.Project().Name, d.Name()) _ = os.RemoveAll(internalUtil.RunPath(newFullName)) if util.PathExists(d.RunPath()) { err := os.Rename(d.RunPath(), internalUtil.RunPath(newFullName)) if err != nil { d.logger.Error("Failed renaming instance", ctxMap) return err } } reverter := revert.New() defer reverter.Fail() // Set the new name in the struct. d.name = newName reverter.Add(func() { d.name = oldName }) // Rename the backups. backups, err := d.Backups() if err != nil { return err } for _, backup := range backups { b := backup oldName := b.Name() backupName := strings.Split(oldName, "/")[1] newName := fmt.Sprintf("%s/%s", newName, backupName) err = b.Rename(newName) if err != nil { return err } reverter.Add(func() { _ = b.Rename(oldName) }) } // Update lease files. err = network.UpdateDNSMasqStatic(d.state, "") if err != nil { return err } // Reset cloud-init instance-id (causes a re-run on name changes). if !d.IsSnapshot() { err = d.resetInstanceID() if err != nil { return err } } // Update the backup file. err = d.UpdateBackupFile() if err != nil { return err } d.logger.Info("Renamed instance", ctxMap) if d.isSnapshot { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceSnapshotRenamed.Event(d, map[string]any{"old_name": oldName})) } else { err = d.state.Authorizer.RenameInstance(d.state.ShutdownCtx, d.project.Name, oldName, newName) if err != nil { logger.Error("Failed to rename instance in authorizer", logger.Ctx{"old_name": oldName, "new_name": newName, "project": d.project.Name, "error": err}) } d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceRenamed.Event(d, map[string]any{"old_name": oldName})) } reverter.Success() return nil } // Indirection to detachDisk. func qemuDetachDisk(s *state.State, id int) func(string) error { return func(name string) error { inst, err := instance.LoadByID(s, id) if err != nil { return err } qemuInst, ok := inst.(*qemu) if !ok { return fmt.Errorf("Couldn't assert QEMU object from interface") } return qemuInst.detachDisk(name) } } // Detach a disk from the instance. func (d *qemu) detachDisk(name string) error { diskName := linux.PathNameDecode(strings.TrimPrefix(name, qemuDeviceIDPrefix)) // Load and detach the disk. config, ok := d.expandedDevices[diskName] if !ok { return fmt.Errorf("Couldn't find device %s", diskName) } dev, err := d.deviceLoad(d, diskName, config, false) if err != nil { return err } err = d.deviceStop(dev, true, "") if err != nil { return err } // Check if it's a special device or an inherited device for which we can't save state. _, ok = d.localDevices[diskName] if !ok || slices.Contains([]string{"agent:config", "cloud-init:config"}, config["source"]) { // Record that the instance devices got modified and a full reset will be needed to get a consistent state. err = d.VolatileSet(map[string]string{ "volatile.vm.needs_reset": "true", }) if err != nil { return err } return nil } // Record the device as detached. d.localDevices[diskName]["attached"] = "false" return d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { devices, err := dbCluster.APIToDevices(d.localDevices.CloneNative()) if err != nil { return err } return dbCluster.UpdateInstanceDevices(ctx, tx.Tx(), int64(d.id), devices) }) } // Update the instance config. func (d *qemu) Update(args db.InstanceArgs, userRequested bool) error { // Setup a new operation. op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionUpdate, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore}, false, false) if err != nil { return fmt.Errorf("Failed to create instance update operation: %w", err) } defer op.Done(nil) // Setup the reverter. reverter := revert.New() defer reverter.Fail() // Set sane defaults for unset keys. if args.Project == "" { args.Project = api.ProjectDefaultName } if args.Architecture == 0 { args.Architecture = d.architecture } if args.Config == nil { args.Config = map[string]string{} } if args.Devices == nil { args.Devices = deviceConfig.Devices{} } if args.Profiles == nil { args.Profiles = []api.Profile{} } if userRequested { // Validate the new config. err := instance.ValidConfig(d.state.OS, args.Config, false, d.dbType) if err != nil { return fmt.Errorf("Invalid config: %w", err) } // Validate the new devices without using expanded devices validation (expensive checks disabled). err = instance.ValidDevices(d.state, d.project, d.Type(), args.Devices, nil) if err != nil { return fmt.Errorf("Invalid devices: %w", err) } } var profiles []string err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Validate the new profiles. profiles, err = tx.GetProfileNames(ctx, args.Project) return err }) if err != nil { return fmt.Errorf("Failed to get profiles: %w", err) } checkedProfiles := []string{} for _, profile := range args.Profiles { if !slices.Contains(profiles, profile.Name) { return fmt.Errorf("Requested profile '%s' doesn't exist", profile.Name) } if slices.Contains(checkedProfiles, profile.Name) { return errors.New("Duplicate profile found in request") } checkedProfiles = append(checkedProfiles, profile.Name) } // Validate the new architecture. if args.Architecture != 0 { _, err = osarch.ArchitectureName(args.Architecture) if err != nil { return fmt.Errorf("Invalid architecture ID: %s", err) } } // Get a copy of the old configuration. oldDescription := d.Description() oldArchitecture := 0 err = util.DeepCopy(&d.architecture, &oldArchitecture) if err != nil { return err } oldEphemeral := false err = util.DeepCopy(&d.ephemeral, &oldEphemeral) if err != nil { return err } oldExpandedDevices := deviceConfig.Devices{} err = util.DeepCopy(&d.expandedDevices, &oldExpandedDevices) if err != nil { return err } oldExpandedConfig := map[string]string{} err = util.DeepCopy(&d.expandedConfig, &oldExpandedConfig) if err != nil { return err } oldLocalDevices := deviceConfig.Devices{} err = util.DeepCopy(&d.localDevices, &oldLocalDevices) if err != nil { return err } oldLocalConfig := map[string]string{} err = util.DeepCopy(&d.localConfig, &oldLocalConfig) if err != nil { return err } oldProfiles := []api.Profile{} err = util.DeepCopy(&d.profiles, &oldProfiles) if err != nil { return err } oldExpiryDate := d.expiryDate // Revert local changes if update fails. reverter.Add(func() { d.description = oldDescription d.architecture = oldArchitecture d.ephemeral = oldEphemeral d.expandedConfig = oldExpandedConfig d.expandedDevices = oldExpandedDevices d.localConfig = oldLocalConfig d.localDevices = oldLocalDevices d.profiles = oldProfiles d.expiryDate = oldExpiryDate }) // Apply the various changes to local vars. d.description = args.Description d.architecture = args.Architecture d.ephemeral = args.Ephemeral d.localConfig = args.Config d.localDevices = args.Devices d.profiles = args.Profiles d.expiryDate = args.ExpiryDate // Expand the config. err = d.expandConfig() if err != nil { return err } // Diff the configurations. changedConfig := []string{} for key := range oldExpandedConfig { if oldExpandedConfig[key] != d.expandedConfig[key] { if !slices.Contains(changedConfig, key) { changedConfig = append(changedConfig, key) } } } for key := range d.expandedConfig { if oldExpandedConfig[key] != d.expandedConfig[key] { if !slices.Contains(changedConfig, key) { changedConfig = append(changedConfig, key) } } } // Diff the devices. removeDevices, addDevices, updateDevices, allUpdatedKeys := oldExpandedDevices.Update(d.expandedDevices, func(oldDevice deviceConfig.Device, newDevice deviceConfig.Device) []string { // This function needs to return a list of fields that are excluded from differences // between oldDevice and newDevice. The result of this is that as long as the // devices are otherwise identical except for the fields returned here, then the // device is considered to be being "updated" rather than "added & removed". oldDevType, err := device.LoadByType(d.state, d.Project().Name, oldDevice) if err != nil { return []string{} // Couldn't create Device, so this cannot be an update. } newDevType, err := device.LoadByType(d.state, d.Project().Name, newDevice) if err != nil { return []string{} // Couldn't create Device, so this cannot be an update. } // Detached devices need to be fully recreated on update so that the update logic doesn't // try to access non-existing QEMU devices. if !util.IsTrueOrEmpty(oldDevice["attached"]) { return []string{} } return newDevType.UpdatableFields(oldDevType) }) // Prevent adding or updating device initial configuration. if util.StringPrefixInSlice("initial.", allUpdatedKeys) { for devName, newDev := range addDevices { for k, newVal := range newDev { if !strings.HasPrefix(k, "initial.") { continue } if newDev["pool"] != "" && newDev["path"] != "/" && strings.Contains(newDev["source"], "/") { continue } oldDev, ok := removeDevices[devName] if !ok { return errors.New("New device with initial configuration cannot be added once the instance is created") } oldVal, ok := oldDev[k] if !ok { return errors.New("Device initial configuration cannot be added once the instance is created") } // If newVal is an empty string it means the initial configuration // has been removed. if newVal != "" && newVal != oldVal { return errors.New("Device initial configuration cannot be modified once the instance is created") } } } } if userRequested { // Do some validation of the config diff (allows mixed instance types for profiles). err = instance.ValidConfig(d.state.OS, d.expandedConfig, true, instancetype.Any) if err != nil { return fmt.Errorf("Invalid expanded config: %w", err) } // Do full expanded validation of the devices diff. err = instance.ValidDevices(d.state, d.project, d.Type(), d.localDevices, d.expandedDevices) if err != nil { return fmt.Errorf("Invalid expanded devices: %w", err) } // Validate root device _, oldRootDev, oldErr := internalInstance.GetRootDiskDevice(oldExpandedDevices.CloneNative()) _, newRootDev, newErr := internalInstance.GetRootDiskDevice(d.expandedDevices.CloneNative()) if oldErr == nil && newErr == nil && oldRootDev["pool"] != newRootDev["pool"] { return fmt.Errorf("Cannot update root disk device pool name to %q", newRootDev["pool"]) } // Ensure the instance has a root disk. if newErr != nil { return fmt.Errorf("Invalid root disk device: %w", newErr) } } // If apparmor changed, re-validate the apparmor profile (even if not running). if slices.Contains(changedConfig, "raw.apparmor") { qemuPath, _, err := d.qemuArchConfig(d.architecture) if err != nil { return err } err = apparmor.InstanceValidate(d.state.OS, d, []string{qemuPath}) if err != nil { return fmt.Errorf("Parse AppArmor profile: %w", err) } } isRunning := d.IsRunning() // Use the device interface to apply update changes. err = d.devicesUpdate(d, removeDevices, addDevices, updateDevices, oldExpandedDevices, isRunning, userRequested) if err != nil { return err } if isRunning { // Only certain keys can be changed on a running VM. liveUpdateKeys := []string{ "cluster.evacuate", "limits.memory", "security.agent.metrics", "security.csm", "security.protection.delete", "security.guestapi", "security.secureboot", } liveUpdateKeyPrefixes := []string{ "boot.", "cloud-init.", "environment.", "image.", "snapshots.", "user.", "volatile.", } isLiveUpdatable := func(key string) bool { // Skip container config keys for VMs _, ok := internalInstance.InstanceConfigKeysContainer[key] if ok { return true } if key == "limits.cpu" { return d.architectureSupportsCPUHotplug() } if slices.Contains(liveUpdateKeys, key) { return true } if util.StringHasPrefix(key, liveUpdateKeyPrefixes...) { return true } if key == "limits.memory.oom_priority" { return true } return false } // Check only keys that support live update have changed. for _, key := range changedConfig { if !isLiveUpdatable(key) { return fmt.Errorf("Key %q cannot be updated when VM is running", key) } } // Mark the VM as needing a full reset on next reboot. err = d.VolatileSet(map[string]string{ "volatile.vm.needs_reset": "true", }) if err != nil { return err } // Apply live update for each key. for _, key := range changedConfig { value := d.expandedConfig[key] if key == "limits.cpu" { oldValue := oldExpandedConfig["limits.cpu"] if oldValue != "" { _, err := strconv.Atoi(oldValue) if err != nil { return fmt.Errorf("Cannot update key %q when using CPU pinning and the VM is running", key) } } // If the key is being unset, set it to default value. if value == "" { value = "1" } limit, err := strconv.Atoi(value) if err != nil { return errors.New("Cannot change CPU pinning when VM is running") } // Hotplug the CPUs. err = d.setCPUs(nil, limit) if err != nil { return fmt.Errorf("Failed updating cpu limit: %w", err) } } else if key == "limits.memory" { err = d.updateMemoryLimit(value) if err != nil { if err != nil { return fmt.Errorf("Failed updating memory limit: %w", err) } } } else if key == "security.csm" { // Defer rebuilding nvram until next start. d.localConfig["volatile.apply_nvram"] = "true" } else if key == "security.secureboot" { // Defer rebuilding nvram until next start. d.localConfig["volatile.apply_nvram"] = "true" } else if key == "security.guestapi" { err = d.advertiseVsockAddress() if err != nil { return err } } else if key == "limits.memory.oom_priority" { // Configure the OOM priority. err = d.setOOMPriority(d.InitPID()) if err != nil { d.logger.Warn("Failed to set OOM priority", logger.Ctx{ "err": err, "instance": d.Name(), "project": d.Project().Name, }) } } } } // Clear the "volatile.cpu.nodes" if needed. d.ClearLimitsCPUNodes(changedConfig) if d.architectureSupportsUEFI(d.architecture) && (slices.Contains(changedConfig, "security.secureboot") || slices.Contains(changedConfig, "security.csm")) { // setupNvram() requires instance's config volume to be mounted. // The easiest way to detect that is to check if instance is running. // TODO: extend storage API to be able to check if volume is already mounted? if !isRunning { // Mount the instance's config volume. _, err := d.mount() if err != nil { return err } defer func() { _ = d.unmount() }() } // Re-generate the NVRAM. err = d.setupNvram() if err != nil { return err } } // Re-generate the instance-id if needed. if !d.IsSnapshot() && d.needsNewInstanceID(changedConfig, oldExpandedDevices) { err = d.resetInstanceID() if err != nil { return err } } // Finally, apply the changes to the database. err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Snapshots should update only their descriptions and expiry date. if d.IsSnapshot() { return tx.UpdateInstanceSnapshot(d.id, d.description, d.expiryDate) } object, err := dbCluster.GetInstance(ctx, tx.Tx(), d.project.Name, d.name) if err != nil { return err } object.Description = d.description object.Architecture = d.architecture object.Ephemeral = d.ephemeral object.ExpiryDate = sql.NullTime{Time: d.expiryDate, Valid: true} err = dbCluster.UpdateInstance(ctx, tx.Tx(), d.project.Name, d.name, *object) if err != nil { return err } err = dbCluster.UpdateInstanceConfig(ctx, tx.Tx(), int64(object.ID), d.localConfig) if err != nil { return err } devices, err := dbCluster.APIToDevices(d.localDevices.CloneNative()) if err != nil { return err } err = dbCluster.UpdateInstanceDevices(ctx, tx.Tx(), int64(object.ID), devices) if err != nil { return err } profileNames := make([]string, 0, len(d.profiles)) for _, profile := range d.profiles { profileNames = append(profileNames, profile.Name) } return dbCluster.UpdateInstanceProfiles(ctx, tx.Tx(), object.ID, object.Project, profileNames) }) if err != nil { return fmt.Errorf("Failed to update database: %w", err) } err = d.UpdateBackupFile() if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to write backup file: %w", err) } // Changes have been applied and recorded, do not revert if an error occurs from here. reverter.Success() if isRunning { // Send devIncus notifications only for user.* key changes for _, key := range changedConfig { if !strings.HasPrefix(key, "user.") { continue } msg := map[string]any{ "key": key, "old_value": oldExpandedConfig[key], "value": d.expandedConfig[key], } err = d.devIncusEventSend("config", msg) if err != nil { return err } } // Device changes for k, m := range removeDevices { msg := map[string]any{ "action": "removed", "name": k, "config": m, } err = d.devIncusEventSend("device", msg) if err != nil { return err } } for k, m := range updateDevices { msg := map[string]any{ "action": "updated", "name": k, "config": m, } err = d.devIncusEventSend("device", msg) if err != nil { return err } } for k, m := range addDevices { msg := map[string]any{ "action": "added", "name": k, "config": m, } err = d.devIncusEventSend("device", msg) if err != nil { return err } } } if userRequested { if d.isSnapshot { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceSnapshotUpdated.Event(d, nil)) } else { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceUpdated.Event(d, nil)) } } return nil } // updateMemoryLimit live updates the VM's memory limit by reszing the balloon device. func (d *qemu) updateMemoryLimit(newLimit string) error { if newLimit == "" { return nil } if util.IsTrue(d.expandedConfig["limits.memory.hugepages"]) { return errors.New("Cannot live update memory limit when using huge pages") } // Check new size string is valid and convert to bytes. newSizeBytes, err := ParseMemoryStr(newLimit) if err != nil { return fmt.Errorf("Invalid memory size: %w", err) } newSizeMB := newSizeBytes / 1024 / 1024 // Connect to the monitor. monitor, err := d.qmpConnect() if err != nil { return err // The VM isn't running as no monitor socket available. } baseSizeBytes, err := monitor.GetMemorySizeBytes() if err != nil { return err } baseSizeMB := baseSizeBytes / 1024 / 1024 curSizeBytes, err := monitor.GetMemoryBalloonSizeBytes() if err != nil { return err } curSizeMB := curSizeBytes / 1024 / 1024 if curSizeMB == newSizeMB { return nil } else if baseSizeMB < newSizeMB { if util.IsFalse(d.expandedConfig["limits.memory.hotplug"]) || d.GuestOS() == "freebsd" { return fmt.Errorf("Memory hotplug feature is disabled") } // Grab the current memory configuration. baseMem, maxMem, _, err := monitor.MemoryConfiguration() if err != nil { return err } // Make sure that we're not exceeding the VM's configured maximum. if newSizeBytes > maxMem { return fmt.Errorf("Requested memory total of %s exceeds instance current maximum of %s, restart required", units.GetByteSizeStringIEC(newSizeBytes, 2), units.GetByteSizeStringIEC(maxMem, 2)) } // Add the memory. err = d.hotplugMemory(monitor, newSizeBytes-curSizeBytes) if err != nil { return err } // If migratable, update state information following hotplug. if d.CanLiveMigrate() { // Prepare an updated memory topology struct. memTopology := qemuMemoryTopology{ Base: baseMem, Max: maxMem, Extra: []int64{}, } memDevs, err := monitor.GetMemdev() if err != nil { return err } memSlots := map[string]int64{} memSlotsKeys := []string{} for _, memDev := range memDevs { // Skip base memory node. if memDev.ID == "mem0" { continue } memSlots[memDev.ID] = int64(memDev.Size) memSlotsKeys = append(memSlotsKeys, memDev.ID) } // The list out of QEMU is in random order... sort.Strings(memSlotsKeys) for _, k := range memSlotsKeys { memTopology.Extra = append(memTopology.Extra, memSlots[k]) } // Update the boot state record. bs, err := d.getBootState() if err != nil { return err } bs.MemoryTopology = &memTopology err = d.saveBootState(*bs) if err != nil { return err } } } // Set effective memory size. err = monitor.SetMemoryBalloonSizeBytes(newSizeBytes) if err != nil { return err } // Changing the memory balloon can take time, so poll the effective size to check it has shrunk within 1% // of the target size, which we then take as success (it may still continue to shrink closer to target). for range 10 { curSizeBytes, err = monitor.GetMemoryBalloonSizeBytes() if err != nil { return err } curSizeMB = curSizeBytes / 1024 / 1024 var diff int64 if curSizeMB < newSizeMB { diff = newSizeMB - curSizeMB } else { diff = curSizeMB - newSizeMB } if diff <= (newSizeMB / 100) { return nil // We reached to within 1% of our target size. } time.Sleep(500 * time.Millisecond) } return fmt.Errorf("Failed setting memory to %dMiB (currently %dMiB) as it was taking too long", newSizeMB, curSizeMB) } // hotplugMemory attaches a memory device to a running VM, // respecting NUMA node placement and hugepages. func (d *qemu) hotplugMemory(monitor *qmp.Monitor, sizeBytes int64) error { // Get CPU information. cpuInfo, err := d.cpuTopology() if err != nil { return err } // Fetch memory configuration cpuOpts, err := d.getCPUOpts(cpuInfo, sizeBytes) if err != nil { return err } cpuPinning := cpuInfo.vCPUs != nil // Get CPUs and memory configuration conf := qemuCPU(cpuOpts, cpuPinning) memoryObjects := map[int]cfg.Section{} for _, section := range conf { // Name is in the form 'object "mem0"', so the last quote needs to be removed. // This allows proper parsing of the memory object index. sectionName := section.Name[:len(section.Name)-1] index, err := extractTrailingNumber(sectionName, "object \"mem") if err != nil { continue } memoryObjects[index] = section } // Find first available memory object index. nextMemIndex, err := findNextMemoryIndex(monitor) if err != nil { return err } // Find first available pc-dimm device index. nextDimmIndex, err := findNextDimmIndex(monitor) if err != nil { return err } for index, memory := range memoryObjects { memIndex := nextMemIndex + index dimmIndex := nextDimmIndex + index memObj := memoryConfigSectionToMap(&memory) memObj["id"] = fmt.Sprintf("mem%d", memIndex) err = monitor.AddObject(memObj) if err != nil { return err } memDev := map[string]any{ "driver": "pc-dimm", "id": fmt.Sprintf("dimm%d", dimmIndex), "memdev": fmt.Sprintf("mem%d", memIndex), "node": index, } err = monitor.AddDevice(memDev) if err != nil { return err } } return nil } func (d *qemu) removeUnixDevices() error { // Check that we indeed have devices to remove. if !util.PathExists(d.DevicesPath()) { return nil } // Load the directory listing. dents, err := os.ReadDir(d.DevicesPath()) if err != nil { return err } for _, f := range dents { // Skip non-Unix devices. if !strings.HasPrefix(f.Name(), "forkmknod.unix.") && !strings.HasPrefix(f.Name(), "unix.") && !strings.HasPrefix(f.Name(), "infiniband.unix.") { continue } // Remove the entry devicePath := filepath.Join(d.DevicesPath(), f.Name()) err := os.Remove(devicePath) if err != nil { d.logger.Error("Failed removing unix device", logger.Ctx{"err": err, "path": devicePath}) } } return nil } func (d *qemu) removeDiskDevices() error { // Check that we indeed have devices to remove. if !util.PathExists(d.DevicesPath()) { return nil } // Load the directory listing. dents, err := os.ReadDir(d.DevicesPath()) if err != nil { return err } for _, f := range dents { // Skip non-disk devices if !strings.HasPrefix(f.Name(), "disk.") { continue } // Always try to unmount the host side. _ = unix.Unmount(filepath.Join(d.DevicesPath(), f.Name()), unix.MNT_DETACH) // Remove the entry. diskPath := filepath.Join(d.DevicesPath(), f.Name()) err := os.Remove(diskPath) if err != nil { d.logger.Error("Failed to remove disk device path", logger.Ctx{"err": err, "path": diskPath}) } } return nil } func (d *qemu) cleanup() { // Unmount any leftovers _ = d.removeUnixDevices() _ = d.removeDiskDevices() // Remove the security profiles _ = apparmor.InstanceDelete(d.state.OS, d) // Remove the devices path _ = os.Remove(d.DevicesPath()) // Remove the shmounts path _ = os.RemoveAll(d.ShmountsPath()) } // cleanupDevices performs any needed device cleanup steps when instance is stopped. // Must be called before root volume is unmounted. func (d *qemu) cleanupDevices() { // Clear up the config drive mount. err := d.configDriveMountPathClear() if err != nil { d.logger.Warn("Failed cleaning up config drive mount", logger.Ctx{"err": err}) } for _, entry := range d.expandedDevices.Reversed() { dev, err := d.deviceLoad(d, entry.Name, entry.Config, false) if err != nil { if errors.Is(err, device.ErrUnsupportedDevType) { continue // Skip unsupported device (allows for mixed instance type profiles). } // Just log an error, but still allow the device to be stopped if usable device returned. d.logger.Error("Failed stop validation for device", logger.Ctx{"device": entry.Name, "err": err}) } // If a usable device was returned from deviceLoad try to stop anyway, even if validation fails. // This allows for the scenario where a new version has additional validation restrictions // than older versions and we still need to allow previously valid devices to be stopped even if // they are no longer considered valid. if dev != nil { err = d.deviceStop(dev, false, "") if err != nil { d.logger.Error("Failed to stop device", logger.Ctx{"device": dev.Name(), "err": err}) } } } } func (d *qemu) init() error { // Compute the expanded config and device list. err := d.expandConfig() if err != nil { return err } return nil } // Delete the instance. // cleanupDependencies controls whether dependent resources (e.g. volumes, // and related state) are removed along with the instance. // When false, dependencies are preserved (e.g. storage-only moves). func (d *qemu) Delete(force bool, cleanupDependencies bool) error { // Setup a new operation. op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionDelete, nil, false, false) if err != nil { return fmt.Errorf("Failed to create instance delete operation: %w", err) } defer op.Done(nil) if d.IsRunning() { return api.StatusErrorf(http.StatusBadRequest, "Instance is running") } err = d.delete(force, cleanupDependencies) if err != nil { return err } // If dealing with a snapshot, refresh the backup file on the parent. if d.IsSnapshot() { parentName, _, _ := api.GetParentAndSnapshotName(d.name) // Load the parent. parent, err := instance.LoadByProjectAndName(d.state, d.project.Name, parentName) if err != nil { return fmt.Errorf("Invalid parent: %w", err) } // Update the backup file. err = parent.UpdateBackupFile() if err != nil { return err } } return nil } // Delete the instance without creating an operation lock. func (d *qemu) delete(force bool, cleanupDependencies bool) error { ctxMap := logger.Ctx{ "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, } if d.isSnapshot { d.logger.Info("Deleting instance snapshot", ctxMap) } else { d.logger.Info("Deleting instance", ctxMap) } // Check if instance is delete protected. if !force && util.IsTrue(d.expandedConfig["security.protection.delete"]) && !d.IsSnapshot() { return errors.New("Instance is protected") } // Delete any persistent warnings for instance. err := d.warningsDelete() if err != nil { return err } // Attempt to initialize storage interface for the instance. pool, err := d.getStoragePool() if err != nil && !response.IsNotFoundError(err) { return err } else if pool != nil { if d.IsSnapshot() { // Remove snapshot volume and database record. err = pool.DeleteInstanceSnapshot(d, nil) if err != nil { return err } } else { // Remove all snapshots. err := d.deleteSnapshots(func(snapInst instance.Instance) error { return snapInst.(*qemu).delete(true, cleanupDependencies) // Internal delete function that doesn't lock. }) if err != nil { return fmt.Errorf("Failed deleting instance snapshots: %w", err) } // Remove the storage volume and database records. err = pool.DeleteInstance(d, nil) if err != nil { return err } if cleanupDependencies { // Delete all dependent volumes associated with this instance. err = d.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error { // Load the pool for the disk. diskPool, err := storagePools.LoadByName(d.state, dev.Config["pool"]) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } err = diskPool.DeleteCustomVolume(d.Project().Name, dev.Config["source"], nil) if err != nil { return err } return nil }) if err != nil { return err } } } } // Perform other cleanup steps if not snapshot. if !d.IsSnapshot() { // Remove all backups. backups, err := d.Backups() if err != nil { return err } for _, backup := range backups { err = backup.Delete() if err != nil { return err } } // Run device removal function for each device. d.devicesRemove(d, cleanupDependencies) // Clean things up. d.cleanup() } err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Remove the database record of the instance or snapshot instance. return tx.DeleteInstance(ctx, d.Project().Name, d.Name()) }) if err != nil { d.logger.Error("Failed deleting instance entry", logger.Ctx{"project": d.Project().Name}) return err } if d.isSnapshot { d.logger.Info("Deleted instance snapshot", ctxMap) } else { d.logger.Info("Deleted instance", ctxMap) } if d.isSnapshot { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceSnapshotDeleted.Event(d, nil)) } else { err = d.state.Authorizer.DeleteInstance(d.state.ShutdownCtx, d.project.Name, d.Name()) if err != nil { logger.Error("Failed to remove instance from authorizer", logger.Ctx{"name": d.Name(), "project": d.project.Name, "error": err}) } d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceDeleted.Event(d, nil)) } return nil } // Export publishes the instance. func (d *qemu) Export(metaWriter io.Writer, rootfsWriter io.Writer, properties map[string]string, expiration time.Time, tracker *ioprogress.ProgressTracker) (*api.ImageMetadata, error) { ctxMap := logger.Ctx{ "created": d.creationDate, "ephemeral": d.ephemeral, "used": d.lastUsedDate, } if d.IsRunning() { return nil, errors.New("Cannot export a running instance as an image") } d.logger.Info("Exporting instance", ctxMap) // Start the storage. mountInfo, err := d.mount() if err != nil { d.logger.Error("Failed exporting instance", ctxMap) return nil, err } defer func() { _ = d.unmount() }() // Create the tarball. metaTarWriter := instancewriter.NewInstanceTarWriter(metaWriter, nil) // Path inside the tar image is the pathname starting after cDir. cDir := d.Path() offset := len(cDir) + 1 writeToMetaTar := func(path string, fi os.FileInfo, err error) error { if err != nil { return err } err = metaTarWriter.WriteFile(path[offset:], path, fi, false) if err != nil { d.logger.Debug("Error tarring up", logger.Ctx{"path": path, "err": err}) return err } return nil } // Get the instance's architecture. var arch string if d.IsSnapshot() { parentName, _, _ := api.GetParentAndSnapshotName(d.name) parent, err := instance.LoadByProjectAndName(d.state, d.project.Name, parentName) if err != nil { _ = metaTarWriter.Close() d.logger.Error("Failed exporting instance", ctxMap) return nil, err } arch, _ = osarch.ArchitectureName(parent.Architecture()) } else { arch, _ = osarch.ArchitectureName(d.architecture) } if arch == "" { arch, err = osarch.ArchitectureName(d.state.OS.Architectures[0]) if err != nil { d.logger.Error("Failed exporting instance", ctxMap) return nil, err } } // Generate metadata.yaml. meta := api.ImageMetadata{} fnam := filepath.Join(cDir, "metadata.yaml") if util.PathExists(fnam) { // Parse the metadata. content, err := os.ReadFile(fnam) if err != nil { _ = metaTarWriter.Close() d.logger.Error("Failed exporting instance", ctxMap) return nil, err } err = yaml.Load(content, &meta) if err != nil { _ = metaTarWriter.Close() d.logger.Error("Failed exporting instance", ctxMap) return nil, err } } // Fill in the metadata. meta.Architecture = arch meta.CreationDate = time.Now().UTC().Unix() if meta.Properties == nil { meta.Properties = map[string]string{} } maps.Copy(meta.Properties, properties) if !expiration.IsZero() { meta.ExpiryDate = expiration.UTC().Unix() } // Write the new metadata.yaml. tempDir, err := os.MkdirTemp("", "incus_metadata_") if err != nil { _ = metaTarWriter.Close() d.logger.Error("Failed exporting instance", ctxMap) return nil, err } defer func() { _ = os.RemoveAll(tempDir) }() data, err := yaml.Dump(&meta, yaml.V2) if err != nil { _ = metaTarWriter.Close() d.logger.Error("Failed exporting instance", ctxMap) return nil, err } fnam = filepath.Join(tempDir, "metadata.yaml") err = os.WriteFile(fnam, data, 0o644) if err != nil { _ = metaTarWriter.Close() d.logger.Error("Failed exporting instance", ctxMap) return nil, err } // Add metadata.yaml to the tarball. fi, err := os.Lstat(fnam) if err != nil { _ = metaTarWriter.Close() d.logger.Error("Failed exporting instance", ctxMap) return nil, err } tmpOffset := len(filepath.Dir(fnam)) + 1 err = metaTarWriter.WriteFile(fnam[tmpOffset:], fnam, fi, false) if err != nil { _ = metaTarWriter.Close() d.logger.Debug("Error writing to tarfile", logger.Ctx{"err": err}) d.logger.Error("Failed exporting instance", ctxMap) return nil, err } // Convert from raw to qcow2 and add to tarball. tmpPath, err := os.MkdirTemp(internalUtil.VarPath("images"), "incus_export_") if err != nil { return nil, err } defer func() { _ = os.RemoveAll(tmpPath) }() if mountInfo.DiskPath == "" { return nil, errors.New("No disk path available from mount") } fPath := fmt.Sprintf("%s/rootfs.img", tmpPath) // Convert to qcow2 image. cmd := []string{ "nice", "-n19", // Run with low priority to reduce CPU impact on other processes. "qemu-img", "convert", "-p", "-f", "raw", "-O", "qcow2", } if rootfsWriter != nil { // Compress the qcow2 image if publishing a split image. cmd = append(cmd, "-c") } reverter := revert.New() defer reverter.Fail() // Check for Direct I/O support. from, err := os.OpenFile(mountInfo.DiskPath, unix.O_DIRECT|unix.O_RDONLY, 0) if err == nil { cmd = append(cmd, "-T", "none") _ = from.Close() } to, err := os.OpenFile(fPath, unix.O_DIRECT|unix.O_CREAT, 0o600) if err == nil { cmd = append(cmd, "-t", "none") _ = to.Close() } reverter.Add(func() { _ = os.Remove(fPath) }) cmd = append(cmd, mountInfo.DiskPath, fPath) _, err = apparmor.QemuImg(d.state.OS, cmd, mountInfo.DiskPath, fPath, tracker) if err != nil { return nil, fmt.Errorf("Failed converting instance to qcow2: %w", err) } // Read converted file info and write file to tarball in the case of unified image // For split images, just write as a qcow2 file if rootfsWriter == nil { imgOffset := len(tmpPath) + 1 fi, err = os.Lstat(fPath) if err != nil { return nil, err } err = metaTarWriter.WriteFile(fPath[imgOffset:], fPath, fi, false) if err != nil { return nil, err } } else { f, err := os.Open(fPath) if err != nil { return nil, err } r := io.Reader(f) _, err = util.SafeCopy(rootfsWriter, r) if err != nil { return nil, err } } // Include all the templates. fnam = d.TemplatesPath() if util.PathExists(fnam) { err = filepath.Walk(fnam, writeToMetaTar) if err != nil { d.logger.Error("Failed exporting instance", ctxMap) return nil, err } } err = metaTarWriter.Close() if err != nil { d.logger.Error("Failed exporting instance", ctxMap) return nil, err } reverter.Success() d.logger.Info("Exported instance", ctxMap) return &meta, nil } // MigrateSend is not currently supported. func (d *qemu) MigrateSend(args instance.MigrateSendArgs) error { d.logger.Debug("Migration send starting") defer d.logger.Debug("Migration send stopped") // Check for stateful support. if args.Live && !d.CanLiveMigrate() { return errors.New("Live migration requires migration.stateful to be set to true") } // Setup a new operation. op := operationlock.Get(d.Project().Name, d.Name()) if op != nil && op.ActionMatch(operationlock.ActionMigrate) { return errors.New("The instance is already being migrated") } op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionMigrate, nil, false, true) if err != nil { return err } // Wait for essential migration connections before negotiation. connectionsCtx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() filesystemConn, err := args.FilesystemConn(connectionsCtx) if err != nil { op.Done(err) return err } pool, err := storagePools.LoadByInstance(d.state, d) if err != nil { err := fmt.Errorf("Failed loading instance: %w", err) op.Done(err) return err } clusterMove := args.ClusterMoveSourceName != "" remoteClusterMove := clusterMove && pool.Driver().Info().Remote storageMove := args.StoragePool != "" // The refresh argument passed to MigrationTypes() is always set // to false here. The migration source/sender doesn't need to care whether // or not it's doing a refresh as the migration sink/receiver will know // this, and adjust the migration types accordingly. // The same applies for clusterMove and storageMove, which are set to the most optimized defaults. poolMigrationTypes := pool.MigrationTypes(storagePools.InstanceContentType(d), false, args.Snapshots, true, false) if len(poolMigrationTypes) == 0 { err := errors.New("No source migration types available") op.Done(err) return err } // Convert the pool's migration type options to an offer header to target. // Populate the Fs, ZfsFeatures and RsyncFeatures fields. offerHeader := localMigration.TypesToHeader(poolMigrationTypes...) // Offer to send index header. indexHeaderVersion := localMigration.IndexHeaderVersion offerHeader.IndexHeaderVersion = &indexHeaderVersion // For VMs, send block device size hint in offer header so that target can create the volume the same size. blockSize, err := storagePools.InstanceDiskBlockSize(pool, d, d.op) if err != nil { err := fmt.Errorf("Failed getting source disk size: %w", err) op.Done(err) return err } d.logger.Debug("Set migration offer volume size", logger.Ctx{"blockSize": blockSize}) offerHeader.VolumeSize = &blockSize srcConfig, err := pool.GenerateInstanceBackupConfig(d, args.Snapshots, true, d.op) if err != nil { err := fmt.Errorf("Failed generating instance migration config: %w", err) op.Done(err) return err } dependentVolumesOffer, err := storagePools.GenerateDependentVolumesOffer(d.state, srcConfig, d.Project().Name, args.Snapshots, args.Devices, args.ClusterMoveSourceName != "") if err != nil { err := fmt.Errorf("Failed generating instance depending volumes offer: %w", err) op.Done(err) return err } offerHeader.DependentVolumes = dependentVolumesOffer contentType := storagePools.InstanceContentType(d) // If we are copying snapshots, retrieve a list of snapshots from source volume. if args.Snapshots { offerHeader.SnapshotNames = make([]string, 0, len(srcConfig.Snapshots)) offerHeader.Snapshots = make([]*migration.Snapshot, 0, len(srcConfig.Snapshots)) for i := range srcConfig.Snapshots { offerHeader.SnapshotNames = append(offerHeader.SnapshotNames, srcConfig.Snapshots[i].Name) // Calculating snapshot size can be very slow, skip unless absolutely needed. if !remoteClusterMove || storageMove { snapSize, err := storagePools.CalculateVolumeSnapshotSize(d.Project().Name, pool, contentType, storageDrivers.VolumeTypeVM, d.Name(), srcConfig.Snapshots[i].Name) if err != nil { return err } srcConfig.Snapshots[i].Config["size"] = fmt.Sprintf("%d", snapSize) } offerHeader.Snapshots = append(offerHeader.Snapshots, instance.SnapshotToProtobuf(srcConfig.Snapshots[i])) } } // Offer QEMU to QEMU live state transfer state transfer feature. // If the request is for live migration, then offer that live QEMU to QEMU state transfer can proceed. // Otherwise we'll fallback to doing stateful stop, migrate, and then stateful start, which will still // fulfil the "live" part of the request, albeit with longer pause of the instance during the process. if args.Live { offerHeader.Criu = migration.CRIUType_VM_QEMU.Enum() } // Send offer to target. d.logger.Debug("Sending migration offer to target") err = args.ControlSend(offerHeader) if err != nil { err := fmt.Errorf("Failed sending migration offer header: %w", err) op.Done(err) return err } // Receive response from target. d.logger.Debug("Waiting for migration offer response from target") respHeader := &migration.MigrationHeader{} err = args.ControlReceive(respHeader, true) if err != nil { err := fmt.Errorf("Failed receiving migration offer response: %w", err) op.Done(err) return err } d.logger.Debug("Got migration offer response from target") // Negotiated migration types. migrationTypes, err := localMigration.MatchTypes(respHeader, migration.MigrationFSType_RSYNC, poolMigrationTypes) if err != nil { err := fmt.Errorf("Failed to negotiate migration type: %w", err) op.Done(err) return err } volumesWithTypes, err := storagePools.DependentVolumesMatchMigrationType(d.state, respHeader.DependentVolumes, args.Snapshots, nil, true) if err != nil { err := fmt.Errorf("Failed to negotiate migration types for dependent volumes: %w", err) op.Done(err) return err } dependentVolumes := []localMigration.DependentVolumeArgs{} for _, volWithType := range volumesWithTypes { dependentVolumes = append(dependentVolumes, localMigration.ProtobufToDependentVolume(volWithType.Volume, volWithType.VolumeTypes[0], nil)) } volSourceArgs := &localMigration.VolumeSourceArgs{ IndexHeaderVersion: respHeader.GetIndexHeaderVersion(), // Enable index header frame if supported. Name: d.Name(), MigrationType: migrationTypes[0], Snapshots: offerHeader.SnapshotNames, TrackProgress: true, Refresh: respHeader.GetRefresh(), AllowInconsistent: args.AllowInconsistent, VolumeOnly: !args.Snapshots, Info: &localMigration.Info{Config: srcConfig}, ClusterMove: clusterMove, StorageMove: storageMove, DependentVolumes: dependentVolumes, } // Only send the snapshots that the target requests when refreshing. if respHeader.GetRefresh() { volSourceArgs.Snapshots = respHeader.GetSnapshotNames() allSnapshots := volSourceArgs.Info.Config.VolumeSnapshots // Ensure that only the requested snapshots are included in the migration index header. volSourceArgs.Info.Config.VolumeSnapshots = make([]*api.StorageVolumeSnapshot, 0, len(volSourceArgs.Snapshots)) for i := range allSnapshots { if slices.Contains(volSourceArgs.Snapshots, allSnapshots[i].Name) { volSourceArgs.Info.Config.VolumeSnapshots = append(volSourceArgs.Info.Config.VolumeSnapshots, allSnapshots[i]) } } } // Detect whether the far side has chosen to use QEMU to QEMU live state transfer mode, and if so then // wait for the connection to be established. var stateConn io.ReadWriteCloser if args.Live && respHeader.Criu != nil && *respHeader.Criu == migration.CRIUType_VM_QEMU { stateConn, err = args.StateConn(connectionsCtx) if err != nil { op.Done(err) return err } } g, ctx := errgroup.WithContext(context.Background()) // Start control connection monitor. g.Go(func() error { d.logger.Debug("Migrate send control monitor started") defer d.logger.Debug("Migrate send control monitor finished") controlResult := make(chan error, 1) // Buffered to allow go routine to end if no readers. // This will read the result message from the target side and detect disconnections. go func() { resp := migration.MigrationControl{} err := args.ControlReceive(&resp, false) if err != nil { err = fmt.Errorf("Error reading migration control target: %w", err) } else if !resp.GetSuccess() { err = fmt.Errorf("Error from migration control target: %s", resp.GetMessage()) } controlResult <- err }() // End as soon as we get control message/disconnection from the target side or a local error. select { case <-ctx.Done(): err = ctx.Err() case err = <-controlResult: } return err }) // Start error monitoring routine, this will detect when an error is returned from the other routines, // and if that happens it will disconnect the migration connections which will trigger the other routines // to finish. go func() { <-ctx.Done() args.Disconnect() }() g.Go(func() error { d.logger.Debug("Migrate send transfer started") defer d.logger.Debug("Migrate send transfer finished") var err error // Start live state transfer using state connection if supported. if stateConn != nil { // When performing intra-cluster same-name move, take steps to prevent corruption // of volatile device config keys during start & stop of instance on source/target. if args.ClusterMoveSourceName == d.name { // Disable VolatileSet from persisting changes to the database. // This is so the volatile changes written by the running receiving member // are not lost when the source instance is stopped. d.volatileSetPersistDisable = true // Store a reference to this instance (which has the old volatile settings) // to allow the onStop hook to pick it up, which allows the devices being // stopped to access their volatile settings stored when the instance // originally started on this cluster member. instanceRefSet(d) defer instanceRefClear(d) } err = d.migrateSendLive(ctx, pool, args.ClusterMoveSourceName, args.StoragePool, blockSize, filesystemConn, stateConn, volSourceArgs) if err != nil { return err } } else { // Perform stateful stop if live state transfer is not supported by target. if args.Live { err = d.Stop(true) if err != nil { return fmt.Errorf("Failed statefully stopping instance: %w", err) } } err = pool.MigrateInstance(d, filesystemConn, volSourceArgs, d.op) if err != nil { return err } } return nil }) // Wait for routines to finish and collect first error. { err := g.Wait() if err != nil { op.Done(err) return err } op.Done(nil) d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceMigrated.Event(d, nil)) return nil } } // createEphemeralSnapshot creates a temporary snapshot of the disk that is // intended for short-lived operations. func (d *qemu) createEphemeralSnapshot(diskName string, diskSize int64) (func(), error) { monitor, err := d.qmpConnect() if err != nil { return nil, err } snapshotDiskName := ephemeralSnapshotName(diskName) // Create snapshot of the disk. // We use the VM's config volume for this so that the maximum size of the snapshot can be limited // by setting the root disk's `size.state` property. snapshotFile := filepath.Join(d.Path(), fmt.Sprintf("%s.qcow2", snapshotDiskName)) // Ensure there are no existing migration snapshot files. err = os.Remove(snapshotFile) if err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, err } // Create qcow2 disk image with the maximum size set to the instance's root disk size for use as // a CoW target for the migration snapshot. This will be used during migration to store writes in // the guest whilst the storage driver is transferring the root disk and snapshots to the target. _, err = subprocess.RunCommand("qemu-img", "create", "-f", "qcow2", snapshotFile, fmt.Sprintf("%d", diskSize)) if err != nil { return nil, fmt.Errorf("Failed opening file image for migration storage snapshot %q: %w", snapshotFile, err) } defer func() { _ = os.Remove(snapshotFile) }() // Pass the snapshot file to the running QEMU process. snapFile, err := os.OpenFile(snapshotFile, unix.O_RDWR, 0) if err != nil { return nil, fmt.Errorf("Failed opening file descriptor for migration storage snapshot %q: %w", snapshotFile, err) } defer func() { _ = snapFile.Close() }() // Remove the snapshot file as we don't want to sync this to the target. err = os.Remove(snapshotFile) if err != nil { return nil, err } info, err := monitor.SendFileWithFDSet(snapshotDiskName, snapFile, false) if err != nil { return nil, fmt.Errorf("Failed sending file descriptor of %q for migration storage snapshot: %w", snapFile.Name(), err) } defer func() { _ = monitor.RemoveFDFromFDSet(snapshotDiskName) }() _ = snapFile.Close() // Don't prevent clean unmount when instance is stopped. // Add the snapshot file as a block device (not visible to the guest OS). err = monitor.AddBlockDevice(map[string]any{ "driver": "qcow2", "node-name": snapshotDiskName, "read-only": false, "file": map[string]any{ "driver": "file", "filename": fmt.Sprintf("/dev/fdset/%d", info.ID), }, }, nil, false) if err != nil { return nil, fmt.Errorf("Failed adding migration storage snapshot block device: %w", err) } // Take a snapshot of the disk and redirect writes to the snapshot disk. blockDevs, err := d.fetchBlockDeviceChain(monitor, diskName) if err != nil { return nil, fmt.Errorf("Failed fetching block device chain: %w", err) } blockDevName := blockDevs[len(blockDevs)-1] err = monitor.BlockDevSnapshot(blockDevName, snapshotDiskName) if err != nil { return nil, fmt.Errorf("Failed taking temporary migration storage snapshot: %w", err) } cleanup := func() { // Resume guest (this is needed as it will prevent merging the snapshot if paused). err = monitor.Start() if err != nil { d.logger.Warn("Failed resuming instance", logger.Ctx{"err": err}) } // Try and merge snapshot back to the source disk on failure so we don't lose writes. err = monitor.BlockCommit(snapshotDiskName, "", "") if err != nil { d.logger.Error("Failed merging temporary storage snapshot", logger.Ctx{"err": err}) } err = monitor.RemoveBlockDevice(snapshotDiskName) if err != nil { d.logger.Error("Failed removing temporary snapshot disk device", logger.Ctx{"err": err}) } } return cleanup, nil } // sendMigrationSnapshot transfers the snapshot to the target. // If finalize is true, it performs cleanup after migration. // Otherwise, it returns a finalize function that can be called later. func (d *qemu) sendMigrationSnapshot(diskName string, filesystemConn io.ReadWriteCloser, finalize bool) (func() error, error) { monitor, err := d.qmpConnect() if err != nil { return nil, err } reverter := revert.New() defer reverter.Fail() targetDiskName := migrationNBDTarget(diskName) snapshotDiskName := ephemeralSnapshotName(diskName) listener, err := net.Listen("unix", "") if err != nil { return nil, fmt.Errorf("Failed creating NBD unix listener: %w", err) } defer func() { _ = listener.Close() }() g, _ := errgroup.WithContext(context.Background()) g.Go(func() error { d.logger.Debug("NBD listener waiting for accept") nbdConn, err := listener.Accept() if err != nil { return fmt.Errorf("Failed accepting connection to NBD client unix listener: %w", err) } defer func() { _ = nbdConn.Close() }() d.logger.Debug("NBD connection on source started") go func() { _, _ = util.SafeCopy(filesystemConn, nbdConn) }() _, _ = util.SafeCopy(nbdConn, filesystemConn) d.logger.Debug("NBD connection on source finished") return nil }) // Connect to NBD migration target and add it the source instance as a disk device. d.logger.Debug("Connecting to migration NBD storage target", logger.Ctx{"targetDiskName": targetDiskName, "diskName": diskName}) err = monitor.AddBlockDevice(map[string]any{ "node-name": targetDiskName, "driver": "raw", "file": map[string]any{ "driver": "nbd", "export": diskName, "server": map[string]any{ "type": "unix", "abstract": true, "path": strings.TrimPrefix(listener.Addr().String(), "@"), }, }, }, nil, false) if err != nil { return nil, fmt.Errorf("Failed adding NBD device: %w", err) } reverter.Add(func() { time.Sleep(time.Second) // Wait for it to be released. err := monitor.RemoveBlockDevice(targetDiskName) if err != nil { d.logger.Warn("Failed removing NBD storage target device", logger.Ctx{"err": err}) } }) d.logger.Debug("Connected to migration NBD storage target") // Begin transferring any writes that occurred during the storage migration by transferring the // contents of the (top) migration snapshot to the target disk to bring them into sync. // Once this has completed the guest OS will be paused. d.logger.Debug("Migration storage snapshot transfer started") err = monitor.BlockDevMirror(snapshotDiskName, targetDiskName) if err != nil { return nil, fmt.Errorf("Failed transferring migration storage snapshot: %w", err) } reverter.Add(func() { err = monitor.BlockJobCancel(snapshotDiskName) if err != nil { d.logger.Error("Failed cancelling block job", logger.Ctx{"err": err}) } }) d.logger.Debug("Migration storage snapshot transfer finished") finalizeFunc := func() error { filesystemConn.Close() _ = g.Wait() // Complete the migration snapshot sync process (the guest OS will remain paused). d.logger.Debug("Migration storage snapshot transfer commit started") err = monitor.BlockJobCancel(snapshotDiskName) if err != nil { return fmt.Errorf("Failed cancelling block job: %w", err) } d.logger.Debug("Migration storage snapshot transfer commit finished") time.Sleep(time.Second) // Wait for it to be released. // Remove the NBD client disk. err = monitor.RemoveBlockDevice(targetDiskName) if err != nil { d.logger.Warn("Failed removing NBD storage target device", logger.Ctx{"err": err}) } d.logger.Debug("Removed NBD storage target device") // Merge snapshot back to the source disk so we don't lose the writes. err = monitor.BlockCommit(snapshotDiskName, "", "") if err != nil { return fmt.Errorf("Failed merging migration storage snapshot: %w", err) } err = monitor.RemoveBlockDevice(snapshotDiskName) if err != nil { return fmt.Errorf("Failed removing temporary snapshot disk device: %w", err) } return nil } if finalize { err = finalizeFunc() if err != nil { return nil, err } } reverter.Success() return finalizeFunc, nil } // migrateSendLive performs live migration send process. func (d *qemu) migrateSendLive(ctx context.Context, pool storagePools.Pool, clusterMoveSourceName string, storagePool string, rootDiskSize int64, filesystemConn io.ReadWriteCloser, stateConn io.ReadWriteCloser, volSourceArgs *localMigration.VolumeSourceArgs) error { monitor, err := d.qmpConnect() if err != nil { return err } // Get the root disk device config. rootDiskName, _, err := d.getRootDiskDevice() if err != nil { return err } rootDiskName = d.blockNodeName(linux.PathNameEncode(rootDiskName)) // If we are performing an intra-cluster member move on a Ceph storage pool without storage change // then we can treat this as shared storage and avoid needing to sync the root disk. sameSharedStorage := clusterMoveSourceName != "" && pool.Driver().Info().Remote && storagePool == "" disksToMigrate := len(volSourceArgs.DependentVolumes) > 0 dependentVolumeMove := clusterMoveSourceName != "" && disksToMigrate reverter := revert.New() // Non-shared storage snapshot setup. if !sameSharedStorage || dependentVolumeMove { // Setup migration capabilities. capabilities := map[string]bool{ // Automatically throttle down the guest to speed up convergence of RAM migration. "auto-converge": true, // Allow the migration to be paused after the source qemu releases the block devices but // before the serialisation of the device state, to avoid a race condition between // migration and blockdev-mirror. This requires that the migration be continued after it // has reached the "pre-switchover" status. "pause-before-switchover": true, // During storage migration encode blocks of zeroes efficiently. "zero-blocks": true, } err = monitor.MigrateSetCapabilities(capabilities) if err != nil { return fmt.Errorf("Failed setting migration capabilities: %w", err) } parameters := map[string]any{ "cpu-throttle-initial": 50, "throttle-trigger-threshold": 20, } err = monitor.MigrateSetParameters(parameters) if err != nil { return fmt.Errorf("Failed setting migration parameters: %w", err) } if !sameSharedStorage { cleanup, err := d.createEphemeralSnapshot(rootDiskName, rootDiskSize) if err != nil { return fmt.Errorf("Failed creating migration snapshot: %w", err) } reverter.Add(cleanup) } for _, vol := range volSourceArgs.DependentVolumes { diskName := d.blockNodeName(linux.PathNameEncode(vol.DeviceName)) d.logger.Debug("Create snapshot for dependent volume", logger.Ctx{"name": vol.Name, "size": vol.VolumeSize, "diskName": diskName}) cleanup, err := d.createEphemeralSnapshot(diskName, vol.VolumeSize) if err != nil { return fmt.Errorf("Failed creating migration snapshot: %w", err) } reverter.Add(cleanup) } d.logger.Debug("Setup temporary migration storage snapshot") } else { // Still set some options for shared storage. capabilities := map[string]bool{ // Automatically throttle down the guest to speed up convergence of RAM migration. "auto-converge": true, } err = monitor.MigrateSetCapabilities(capabilities) if err != nil { return fmt.Errorf("Failed setting migration capabilities: %w", err) } parameters := map[string]any{ "cpu-throttle-initial": 50, "throttle-trigger-threshold": 20, } err = monitor.MigrateSetParameters(parameters) if err != nil { return fmt.Errorf("Failed setting migration parameters: %w", err) } } // Perform storage transfer while instance is still running. // For shared storage the storage driver will likely not do much here, but we still call it anyway for the // sense checks it performs. // We enable AllowInconsistent mode as this allows for transferring the VM storage whilst it is running // and the snapshot we took earlier is designed to provide consistency anyway. volSourceArgs.AllowInconsistent = true err = pool.MigrateInstance(d, filesystemConn, volSourceArgs, d.op) if err != nil { return err } // Derive the effective storage project name from the instance config's project. storageProjectName, err := project.StorageVolumeProject(d.state.DB.Cluster, d.project.Name, db.StoragePoolVolumeTypeCustom) if err != nil { return err } // Notify the shared disks that they're going to be accessed from another system, // but only when performing a move within the same storage pool. if storagePool == "" && clusterMoveSourceName != "" { for _, dev := range d.expandedDevices.Sorted() { if dev.Config["type"] != "disk" || dev.Config["path"] == "/" || dev.Config["pool"] == "" { continue } // Load the pool for the disk. diskPool, err := storagePools.LoadByName(d.state, dev.Config["pool"]) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } // Check that we're on shared storage. if !diskPool.Driver().Info().Remote { continue } // Setup the volume entry. extraSourceArgs := &localMigration.VolumeSourceArgs{ ClusterMove: true, } vol := diskPool.GetVolume(storageDrivers.VolumeTypeCustom, storageDrivers.ContentTypeBlock, project.StorageVolume(storageProjectName, dev.Config["source"]), nil) // Call MigrateVolume on the source. err = diskPool.Driver().MigrateVolume(vol, nil, extraSourceArgs, nil) if err != nil { return fmt.Errorf("Failed to prepare device %q for migration: %w", dev.Name, err) } } } var finalizeRootTransfer func() error if !sameSharedStorage { finalizeRootTransfer, err = d.sendMigrationSnapshot(rootDiskName, filesystemConn, false) if err != nil { return fmt.Errorf("Failed transferring snapshot disk: %w", err) } } d.logger.Debug("Stateful migration checkpoint send starting") // Send checkpoint to QEMU process on target. This will pause the guest OS (if not already paused). pipeRead, pipeWrite, err := os.Pipe() if err != nil { return err } defer func() { _ = pipeRead.Close() _ = pipeWrite.Close() }() go func() { _, _ = util.SafeCopy(stateConn, pipeRead) }() err = d.saveStateHandle(monitor, pipeWrite) if err != nil { return fmt.Errorf("Failed starting state transfer to target: %w", err) } // Start monitoring the migration progress. chMonitor := make(chan bool, 1) if d.op != nil { go func() { for { // Wait for next update. select { case <-chMonitor: return case <-time.After(time.Second): } // Get current migration progress. progress, err := monitor.QueryMigrate() if err != nil { // Stop monitoring on error. return } // Post update. percent := int64(float64(progress.RAM.Transferred) / float64(progress.RAM.Total) * float64(100)) speed := int64(progress.RAM.MBps * 1024 * 1024 / 8) metadata := map[string]any{} metadata["progress"] = map[string]string{ "stage": "live_migrate_instance", "processed": strconv.FormatInt(progress.RAM.Transferred, 10), "percent": strconv.FormatInt(percent, 10), "speed": strconv.FormatInt(speed, 10), } metadata["live_migrate_instance_progress"] = fmt.Sprintf("Live migration: %s remaining (%s/s) (%d%% CPU throttle)", units.GetByteSizeString(progress.RAM.Remaining, 2), units.GetByteSizeString(speed, 2), progress.CPUThrottlePercentage) _ = d.op.UpdateMetadata(metadata) } }() } // Non-shared storage snapshot transfer finalization. if !sameSharedStorage || dependentVolumeMove { // Wait until state transfer has reached pre-switchover state (the guest OS will remain paused). err = monitor.MigrateWait(ctx, "pre-switchover") if err != nil { return fmt.Errorf("Failed waiting for state transfer to reach pre-switchover stage: %w", err) } d.logger.Debug("Stateful migration checkpoint reached pre-switchover phase") if finalizeRootTransfer != nil { err = finalizeRootTransfer() if err != nil { return fmt.Errorf("Failed transferring root snapshot disk: %w", err) } } for _, vol := range volSourceArgs.DependentVolumes { diskName := d.blockNodeName(linux.PathNameEncode(vol.DeviceName)) _, err = d.sendMigrationSnapshot(diskName, filesystemConn, true) if err != nil { return fmt.Errorf("Failed transferring snapshot disk: %w", err) } } // Finalise the migration state transfer (the guest OS will remain paused). err = monitor.MigrateContinue("pre-switchover") if err != nil { return fmt.Errorf("Failed continuing state transfer: %w", err) } d.logger.Debug("Stateful migration checkpoint send continuing") } // Wait until the migration state transfer has completed (the guest OS will remain paused). err = monitor.MigrateWait(ctx, "completed") if err != nil { return fmt.Errorf("Failed waiting for state transfer to reach completed stage: %w", err) } close(chMonitor) d.logger.Debug("Stateful migration checkpoint send finished") if clusterMoveSourceName != "" { // If doing an intra-cluster member move then we will be deleting the instance on the source, // so lets just stop it after migration is completed. err = d.Stop(false) if err != nil { return fmt.Errorf("Failed stopping instance: %w", err) } } else { // Resume guest. err = monitor.Start() if err != nil { return fmt.Errorf("Failed resuming instance: %w", err) } d.logger.Debug("Resumed instance") } reverter.Success() return nil } func (d *qemu) MigrateReceive(args instance.MigrateReceiveArgs) error { d.logger.Debug("Migration receive starting") defer d.logger.Debug("Migration receive stopped") // Wait for essential migration connections before negotiation. connectionsCtx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() filesystemConn, err := args.FilesystemConn(connectionsCtx) if err != nil { return err } // Receive offer from source. d.logger.Debug("Waiting for migration offer from source") offerHeader := &migration.MigrationHeader{} err = args.ControlReceive(offerHeader, true) if err != nil { return fmt.Errorf("Failed receiving migration offer from source: %w", err) } // When doing a cluster same-name move we cannot load the storage pool using the instance's volume DB // record because it may be associated to the wrong cluster member. Instead we ascertain the pool to load // using the instance's root disk device. if args.ClusterMoveSourceName == d.name { if args.StoragePool != "" { d.storagePool, err = storagePools.LoadByName(d.state, args.StoragePool) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } } else { _, rootDiskDevice, err := d.getRootDiskDevice() if err != nil { return fmt.Errorf("Failed getting root disk: %w", err) } if rootDiskDevice["pool"] == "" { return errors.New("The instance's root device is missing the pool property") } // Initialize the storage pool cache. d.storagePool, err = storagePools.LoadByName(d.state, rootDiskDevice["pool"]) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } } } pool, err := storagePools.LoadByInstance(d.state, d) if err != nil { return err } // The source will never set Refresh in the offer header. // However, to determine the correct migration type Refresh needs to be set. offerHeader.Refresh = &args.Refresh clusterMove := args.ClusterMoveSourceName != "" storageMove := args.StoragePool != "" // Extract the source's migration type and then match it against our pool's supported types and features. // If a match is found the combined features list will be sent back to requester. contentType := storagePools.InstanceContentType(d) respTypes, err := localMigration.MatchTypes(offerHeader, storagePools.FallbackMigrationType(contentType), pool.MigrationTypes(contentType, args.Refresh, args.Snapshots, clusterMove, storageMove)) if err != nil { return err } // The migration header to be sent back to source with our target options. // Convert response type to response header and copy snapshot info into it. respHeader := localMigration.TypesToHeader(respTypes...) // Respond with our maximum supported header version if the requested version is higher than ours. // Otherwise just return the requested header version to the source. indexHeaderVersion := min(offerHeader.GetIndexHeaderVersion(), localMigration.IndexHeaderVersion) respHeader.IndexHeaderVersion = &indexHeaderVersion respHeader.SnapshotNames = offerHeader.SnapshotNames respHeader.Snapshots = offerHeader.Snapshots respHeader.Refresh = &args.Refresh localDevices := d.localDevices.CloneNative() volumesWithTypes, err := storagePools.DependentVolumesMatchMigrationType(d.state, offerHeader.DependentVolumes, args.Snapshots, localDevices, false) if err != nil { return fmt.Errorf("Failed to negotiate migration types for dependent volumes: %w", err) } dependentVolumes := []localMigration.DependentVolumeArgs{} for _, volWithType := range volumesWithTypes { respHeader.DependentVolumes = append(respHeader.DependentVolumes, volWithType.Volume) vol := localMigration.ProtobufToDependentVolume(volWithType.Volume, volWithType.VolumeTypes[0], localDevices[*volWithType.Volume.DeviceName]) dependentVolumes = append(dependentVolumes, vol) } if args.Refresh { // Get the remote snapshots on the source. sourceSnapshots := offerHeader.GetSnapshots() sourceSnapshotComparable := make([]storagePools.ComparableSnapshot, 0, len(sourceSnapshots)) for _, sourceSnap := range sourceSnapshots { sourceSnapshotComparable = append(sourceSnapshotComparable, storagePools.ComparableSnapshot{ Name: sourceSnap.GetName(), CreationDate: time.Unix(sourceSnap.GetCreationDate(), 0), }) } // Get existing snapshots on the local target. targetSnapshots, err := d.Snapshots() if err != nil { return err } targetSnapshotsComparable := make([]storagePools.ComparableSnapshot, 0, len(targetSnapshots)) for _, targetSnap := range targetSnapshots { _, targetSnapName, _ := api.GetParentAndSnapshotName(targetSnap.Name()) targetSnapshotsComparable = append(targetSnapshotsComparable, storagePools.ComparableSnapshot{ Name: targetSnapName, CreationDate: targetSnap.CreationDate(), }) } // Compare the two sets. syncSourceSnapshotIndexes, deleteTargetSnapshotIndexes := storagePools.CompareSnapshots(sourceSnapshotComparable, targetSnapshotsComparable, args.RefreshExcludeOlder) // Delete the extra local snapshots first. for _, deleteTargetSnapshotIndex := range deleteTargetSnapshotIndexes { err := targetSnapshots[deleteTargetSnapshotIndex].Delete(true, true) if err != nil { return err } } // Only request to send the snapshots that need updating. syncSnapshotNames := make([]string, 0, len(syncSourceSnapshotIndexes)) syncSnapshots := make([]*migration.Snapshot, 0, len(syncSourceSnapshotIndexes)) for _, syncSourceSnapshotIndex := range syncSourceSnapshotIndexes { syncSnapshotNames = append(syncSnapshotNames, sourceSnapshots[syncSourceSnapshotIndex].GetName()) syncSnapshots = append(syncSnapshots, sourceSnapshots[syncSourceSnapshotIndex]) } respHeader.Snapshots = syncSnapshots respHeader.SnapshotNames = syncSnapshotNames offerHeader.Snapshots = syncSnapshots offerHeader.SnapshotNames = syncSnapshotNames } // Negotiate support for QEMU to QEMU live state transfer. // If the request is for live migration, then respond that live QEMU to QEMU state transfer can proceed. // Otherwise we'll fallback to doing stateful stop, migrate, and then stateful start, which will still // fulfil the "live" part of the request, albeit with longer pause of the instance during the process. poolInfo := pool.Driver().Info() var useStateConn bool if args.Live && offerHeader.Criu != nil && *offerHeader.Criu == migration.CRIUType_VM_QEMU { respHeader.Criu = migration.CRIUType_VM_QEMU.Enum() useStateConn = true } // Send response to source. d.logger.Debug("Sending migration response to source") err = args.ControlSend(respHeader) if err != nil { return fmt.Errorf("Failed sending migration response to source: %w", err) } d.logger.Debug("Sent migration response to source") // Establish state transfer connection if needed. var stateConn io.ReadWriteCloser if args.Live && useStateConn { stateConn, err = args.StateConn(connectionsCtx) if err != nil { return err } } reverter := revert.New() defer reverter.Fail() g, ctx := errgroup.WithContext(context.Background()) // Start control connection monitor. g.Go(func() error { d.logger.Debug("Migrate receive control monitor started") defer d.logger.Debug("Migrate receive control monitor finished") controlResult := make(chan error, 1) // Buffered to allow go routine to end if no readers. // This will read the result message from the source side and detect disconnections. go func() { resp := migration.MigrationControl{} err := args.ControlReceive(&resp, false) if err != nil { err = fmt.Errorf("Error reading migration control source: %w", err) } else if !resp.GetSuccess() { err = fmt.Errorf("Error from migration control source: %s", resp.GetMessage()) } controlResult <- err }() // End as soon as we get control message/disconnection from the source side or a local error. select { case <-ctx.Done(): err = ctx.Err() case err = <-controlResult: } return err }) // Start error monitoring routine, this will detect when an error is returned from the other routines, // and if that happens it will disconnect the migration connections which will trigger the other routines // to finish. go func() { <-ctx.Done() args.Disconnect() }() // Start filesystem transfer routine and initialize a channel that is closed when the routine finishes. fsTransferDone := make(chan struct{}) g.Go(func() error { defer close(fsTransferDone) d.logger.Debug("Migrate receive transfer started") defer d.logger.Debug("Migrate receive transfer finished") var err error snapshots := make([]*migration.Snapshot, 0) // Legacy: we only sent the snapshot names, so we just copy the instances's config over, // same as we used to do. if len(offerHeader.SnapshotNames) != len(offerHeader.Snapshots) { // Convert the instance to an api.InstanceSnapshot. profileNames := make([]string, 0, len(d.Profiles())) for _, p := range d.Profiles() { profileNames = append(profileNames, p.Name) } architectureName, _ := osarch.ArchitectureName(d.Architecture()) apiInstSnap := &api.InstanceSnapshot{ InstanceSnapshotPut: api.InstanceSnapshotPut{ ExpiresAt: time.Time{}, }, Architecture: architectureName, CreatedAt: d.CreationDate(), LastUsedAt: d.LastUsedDate(), Config: d.LocalConfig(), Description: d.Description(), Devices: d.LocalDevices().CloneNative(), Ephemeral: d.IsEphemeral(), Stateful: d.IsStateful(), Profiles: profileNames, } for _, name := range offerHeader.SnapshotNames { base := instance.SnapshotToProtobuf(apiInstSnap) base.Name = &name snapshots = append(snapshots, base) } } else { snapshots = offerHeader.Snapshots } volTargetArgs := localMigration.VolumeTargetArgs{ IndexHeaderVersion: respHeader.GetIndexHeaderVersion(), Name: d.Name(), MigrationType: respTypes[0], Refresh: args.Refresh, // Indicate to receiver volume should exist. TrackProgress: true, // Use a progress tracker on receiver to get in-cluster progress information. Live: args.Live, VolumeSize: offerHeader.GetVolumeSize(), // Block size setting override. VolumeOnly: !args.Snapshots, ClusterMoveSourceName: args.ClusterMoveSourceName, StoragePool: args.StoragePool, DependentVolumes: dependentVolumes, } // At this point we have already figured out the parent instances's root // disk device so we can simply retrieve it from the expanded devices. parentStoragePool := "" parentExpandedDevices := d.ExpandedDevices() parentLocalRootDiskDeviceKey, parentLocalRootDiskDevice, _ := internalInstance.GetRootDiskDevice(parentExpandedDevices.CloneNative()) if parentLocalRootDiskDeviceKey != "" { parentStoragePool = parentLocalRootDiskDevice["pool"] } if parentStoragePool == "" { return errors.New("Instance's root device is missing the pool property") } // A zero length Snapshots slice indicates volume only migration in // VolumeTargetArgs. So if VolumeOnly was requested, do not populate them. if args.Snapshots { volTargetArgs.Snapshots = make([]*migration.Snapshot, 0, len(snapshots)) for _, snap := range snapshots { migrationSnapshot := &migration.Snapshot{Name: snap.Name} migration.SetSnapshotConfigValue(migrationSnapshot, "size", migration.GetSnapshotConfigValue(snap, "size")) volTargetArgs.Snapshots = append(volTargetArgs.Snapshots, migrationSnapshot) // Only create snapshot instance DB records if not doing a cluster same-name move. // As otherwise the DB records will already exist. if args.ClusterMoveSourceName != d.name { snapArgs, err := instance.SnapshotProtobufToInstanceArgs(d.state, d, snap) if err != nil { return err } // The offerHeader, depending on the case, stores information about either an InstanceSnapshot // or a StorageVolumeSnapshot. In the Config, we pass information about the volume size, // but an InstanceSnapshot config cannot have a 'size' key. This key should be removed // before passing the data to the CreateInternal method. delete(snapArgs.Config, "size") // Ensure that snapshot and parent instance have the same storage pool in // their local root disk device. If the root disk device for the snapshot // comes from a profile on the new instance as well we don't need to do // anything. if snapArgs.Devices != nil { snapLocalRootDiskDeviceKey, _, _ := internalInstance.GetRootDiskDevice(snapArgs.Devices.CloneNative()) if snapLocalRootDiskDeviceKey != "" { snapArgs.Devices[snapLocalRootDiskDeviceKey]["pool"] = parentStoragePool } } // Create the snapshot instance. _, snapInstOp, cleanup, err := instance.CreateInternal(d.state, *snapArgs, d.op, true, false, false) if err != nil { return fmt.Errorf("Failed creating instance snapshot record %q: %w", snapArgs.Name, err) } reverter.Add(cleanup) defer snapInstOp.Done(err) } } } err = pool.CreateInstanceFromMigration(d, filesystemConn, volTargetArgs, d.op) if err != nil { return fmt.Errorf("Failed creating instance on target: %w", err) } isRemoteClusterMove := clusterMove && poolInfo.Remote reverter.Add(func() { // Delete the instance unless it is moved within the same cluster on a shared pool. if (!isRemoteClusterMove && !storageMove) || storageMove { _ = pool.DeleteInstance(d, d.op) } }) // Derive the effective storage project name from the instance config's project. storageProjectName, err := project.StorageVolumeProject(d.state.DB.Cluster, d.project.Name, db.StoragePoolVolumeTypeCustom) if err != nil { return err } // Notify the shared disks that they're going to be accessed from another system, // but only when performing a move within the same storage pool. if !storageMove && args.ClusterMoveSourceName != "" { for _, dev := range d.expandedDevices.Sorted() { if dev.Config["type"] != "disk" || dev.Config["path"] == "/" || dev.Config["pool"] == "" { continue } // Load the pool for the disk. diskPool, err := storagePools.LoadByName(d.state, dev.Config["pool"]) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } // Check that we're on shared storage. if !diskPool.Driver().Info().Remote { continue } // Setup the volume entry. extraTargetArgs := localMigration.VolumeTargetArgs{ ClusterMoveSourceName: args.ClusterMoveSourceName, StoragePool: args.StoragePool, } vol := diskPool.GetVolume(storageDrivers.VolumeTypeCustom, storageDrivers.ContentTypeBlock, project.StorageVolume(storageProjectName, dev.Config["source"]), nil) // Call CreateVolumeFromMigration on the target. err = diskPool.Driver().CreateVolumeFromMigration(vol, nil, extraTargetArgs, nil, nil) if err != nil { return fmt.Errorf("Failed to prepare device %q for migration: %w", dev.Name, err) } } } // Only delete all instance volumes on error if the pool volume creation has succeeded to // avoid deleting an existing conflicting volume. if !volTargetArgs.Refresh && !isRemoteClusterMove { reverter.Add(func() { snapshots, _ := d.Snapshots() snapshotCount := len(snapshots) for k := range snapshots { // Delete the snapshots in reverse order. k = snapshotCount - 1 - k _ = pool.DeleteInstanceSnapshot(snapshots[k], nil) } _ = pool.DeleteInstance(d, nil) }) } if args.ClusterMoveSourceName != d.name { err = d.DeferTemplateApply(instance.TemplateTriggerCopy) if err != nil { return err } } if args.Live { // Start live state transfer using state connection if supported. if stateConn != nil { d.migrationReceiveStateful = map[string]io.ReadWriteCloser{ api.SecretNameState: stateConn, } for _, vol := range dependentVolumes { d.disksToMigrate = append(d.disksToMigrate, vol) } dependentVolumeMove := args.ClusterMoveSourceName != "" && len(d.disksToMigrate) > 0 // Populate the filesystem connection handle if doing non-shared storage migration. sameSharedStorage := args.ClusterMoveSourceName != "" && poolInfo.Remote && args.StoragePool == "" if !sameSharedStorage || dependentVolumeMove { d.migrationReceiveStateful[api.SecretNameFilesystem] = filesystemConn } d.migrationRootDisk = !sameSharedStorage d.migrationClusterMove = args.ClusterMoveSourceName != "" } // Although the instance technically isn't considered stateful, we set this to allow // starting from the migrated state file or migration state connection. d.stateful = true err = d.start(true, args.InstanceOperation) if err != nil { return err } } return nil }) { // Wait until the filesystem transfer routine has finished. <-fsTransferDone // If context is cancelled by this stage, then an error has occurred. // Wait for all routines to finish and collect the first error that occurred. if ctx.Err() != nil { err := g.Wait() // Send failure response to source. msg := migration.MigrationControl{ Success: proto.Bool(err == nil), } if err != nil { msg.Message = proto.String(err.Error()) } d.logger.Debug("Sending migration failure response to source", logger.Ctx{"err": err}) sendErr := args.ControlSend(&msg) if sendErr != nil { d.logger.Warn("Failed sending migration failure to source", logger.Ctx{"err": sendErr}) } return err } // Send success response to source to control as nothing has gone wrong so far. msg := migration.MigrationControl{ Success: proto.Bool(true), } d.logger.Debug("Sending migration success response to source", logger.Ctx{"success": msg.GetSuccess()}) err := args.ControlSend(&msg) if err != nil { d.logger.Warn("Failed sending migration success to source", logger.Ctx{"err": err}) return fmt.Errorf("Failed sending migration success to source: %w", err) } // Wait for all routines to finish (in this case it will be the control monitor) but do // not collect the error, as it will just be a disconnect error from the source. _ = g.Wait() reverter.Success() return nil } } // CGroupSet is not implemented for VMs. func (d *qemu) CGroup() (*cgroup.CGroup, error) { return nil, instance.ErrNotImplemented } // FileSFTPConn returns a connection to the agent SFTP endpoint. func (d *qemu) FileSFTPConn() (net.Conn, error) { // VMs, unlike containers, cannot perform file operations if not running and using the agent. if !d.IsRunning() { return nil, errors.New("Instance is not running") } // Connect to the agent. client, err := d.getAgentClient() if err != nil { return nil, err } // Get the HTTP transport. httpTransport := client.Transport.(*http.Transport) // Send the upgrade request. u, err := url.Parse("https://custom.socket/1.0/sftp") if err != nil { return nil, err } req := &http.Request{ Method: http.MethodGet, URL: u, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1, Header: make(http.Header), Host: u.Host, } req.Header["Upgrade"] = []string{"sftp"} req.Header["Connection"] = []string{"Upgrade"} conn, err := httpTransport.DialContext(context.Background(), "tcp", "8443") if err != nil { return nil, err } tlsConn := tls.Client(conn, httpTransport.TLSClientConfig) err = tlsConn.Handshake() if err != nil { return nil, err } err = req.Write(tlsConn) if err != nil { return nil, err } resp, err := http.ReadResponse(bufio.NewReader(tlsConn), req) if err != nil { return nil, err } if resp.StatusCode != http.StatusSwitchingProtocols { return nil, fmt.Errorf("Dialing failed: expected status code 101 got %d", resp.StatusCode) } if resp.Header.Get("Upgrade") != "sftp" { return nil, errors.New("Missing or unexpected Upgrade header in response") } return tlsConn, nil } // FileSFTP returns an SFTP connection to the agent endpoint. func (d *qemu) FileSFTP() (*sftp.Client, error) { // Connect to the forkfile daemon. conn, err := d.FileSFTPConn() if err != nil { return nil, err } // Get a SFTP client. client, err := sftp.NewClientPipe(conn, conn) if err != nil { _ = conn.Close() return nil, err } go func() { // Wait for the client to be done before closing the connection. _ = client.Wait() _ = conn.Close() }() return client, nil } // Console gets access to the instance's console. func (d *qemu) Console(protocol string) (*os.File, chan error, error) { var path string switch protocol { case instance.ConsoleTypeConsole: path = d.consolePath() case instance.ConsoleTypeVGA: info := DriverStatuses()[instancetype.VM].Info _, spiceSupported := info.Features["spice"] if !spiceSupported { return nil, nil, fmt.Errorf("SPICE is not supported by the host") } path = d.spicePath() default: return nil, nil, fmt.Errorf("Unknown protocol %q", protocol) } // When activating the text-based console, swap the backend to be a socket for an interactive connection. if protocol == instance.ConsoleTypeConsole { // Look for existing connections and reset. conn, err := net.Dial("unix", path) if err == nil { _ = d.consoleSwapSocketWithRB() _ = conn.Close() // Allow for cleanup to complete on the existing connection. time.Sleep(time.Second) } err = d.consoleSwapRBWithSocket() if err != nil { _ = d.consoleSwapSocketWithRB() return nil, nil, fmt.Errorf("Failed to swap console ring buffer with socket: %w", err) } } // Disconnection notification. chDisconnect := make(chan error, 1) // Open the console socket. conn, err := net.Dial("unix", path) if err != nil { if protocol == instance.ConsoleTypeConsole { _ = d.consoleSwapSocketWithRB() } return nil, nil, fmt.Errorf("Connect to console socket %q: %w", path, err) } file, err := (conn.(*net.UnixConn)).File() if err != nil { if protocol == instance.ConsoleTypeConsole { _ = d.consoleSwapSocketWithRB() } return nil, nil, fmt.Errorf("Get socket file: %w", err) } _ = conn.Close() // Handle disconnections. go func() { <-chDisconnect _ = d.consoleSwapSocketWithRB() }() // Only emit a lifecycle event for the text console here. SPICE clients open one socket per channel // (display, cursor, inputs, ...) and would otherwise produce a flurry of instance-console events // for a single user session; the VGA emit is handled once per session by the console request handler. if protocol == instance.ConsoleTypeConsole { d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceConsole.Event(d, logger.Ctx{"type": protocol})) } return file, chDisconnect, nil } // Exec a command inside the instance. func (d *qemu) Exec(req api.InstanceExecPost, stdin *os.File, stdout *os.File, stderr *os.File) (instance.Cmd, error) { reverter := revert.New() defer reverter.Fail() client, err := d.getAgentClient() if err != nil { return nil, err } agent, err := incus.ConnectIncusHTTP(nil, client) if err != nil { d.logger.Error("Failed to connect to the agent", logger.Ctx{"err": err}) return nil, errors.New("Failed to connect to the agent") } reverter.Add(agent.Disconnect) dataDone := make(chan bool) controlSendCh := make(chan api.InstanceExecControl) controlResCh := make(chan error) // This is the signal control handler, it receives signals from lxc CLI and forwards them to the VM agent. controlHandler := func(control *websocket.Conn) { closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") defer func() { _ = control.WriteMessage(websocket.CloseMessage, closeMsg) }() for { select { case cmd := <-controlSendCh: controlResCh <- control.WriteJSON(cmd) case <-dataDone: return } } } args := incus.InstanceExecArgs{ Stdin: stdin, Stdout: stdout, Stderr: stderr, DataDone: dataDone, Control: controlHandler, } // Always needed for VM exec, as even for non-websocket requests from the client we need to connect the // websockets for control and for capturing output to a file on the server. req.WaitForWS = true // Similarly, output recording is performed on the host rather than in the guest, so clear that bit from the request. req.RecordOutput = false op, err := agent.ExecInstance("", req, &args) if err != nil { return nil, err } instCmd := &qemuCmd{ cmd: op, attachedChildPid: 0, // Process is not running on the host. dataDone: args.DataDone, cleanupFunc: reverter.Clone().Fail, // Pass revert function clone as clean up function. controlSendCh: controlSendCh, controlResCh: controlResCh, } d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceExec.Event(d, logger.Ctx{"command": req.Command})) reverter.Success() return instCmd, nil } // RenderWithUsage renders the API response including disk usage. func (d *qemu) RenderWithUsage() (any, any, error) { resp, etag, err := d.Render() if err != nil { return nil, nil, err } // Currently only snapshot data needs usage added. snapResp, ok := resp.(*api.InstanceSnapshot) if !ok { return resp, etag, nil } pool, err := d.getStoragePool() if err != nil { return nil, nil, err } // It is important that the snapshot not be mounted here as mounting a snapshot can trigger a very // expensive filesystem UUID regeneration, so we rely on the driver implementation to get the info // we are requesting as cheaply as possible. volumeState, err := pool.GetInstanceUsage(d) if err != nil { return resp, etag, nil } snapResp.Size = volumeState.Used return snapResp, etag, nil } // Render returns info about the instance. func (d *qemu) Render() (any, any, error) { profileNames := make([]string, 0, len(d.profiles)) for _, profile := range d.profiles { profileNames = append(profileNames, profile.Name) } if d.IsSnapshot() { // Prepare the response. snapState := api.InstanceSnapshot{ CreatedAt: d.creationDate, Description: d.description, ExpandedConfig: d.expandedConfig, ExpandedDevices: d.expandedDevices.CloneNative(), LastUsedAt: d.lastUsedDate, Name: strings.SplitN(d.name, "/", 2)[1], Stateful: d.stateful, Size: -1, // Default to uninitialized/error state (0 means no CoW usage). } snapState.Architecture = d.architectureName snapState.Config = d.localConfig snapState.Devices = d.localDevices.CloneNative() snapState.Ephemeral = d.ephemeral snapState.Profiles = profileNames snapState.ExpiresAt = d.expiryDate return &snapState, d.ETag(), nil } // Prepare the response. statusCode := d.statusCode() instState := api.Instance{ ExpandedConfig: d.expandedConfig, ExpandedDevices: d.expandedDevices.CloneNative(), Name: d.name, Status: statusCode.String(), StatusCode: statusCode, Location: d.node, Type: d.Type().String(), } instState.Description = d.description instState.Architecture = d.architectureName instState.Config = d.localConfig instState.CreatedAt = d.creationDate instState.Devices = d.localDevices.CloneNative() instState.Ephemeral = d.ephemeral instState.LastUsedAt = d.lastUsedDate instState.Profiles = profileNames instState.Stateful = d.stateful instState.Project = d.project.Name return &instState, d.ETag(), nil } // RenderFull returns all info about the instance. func (d *qemu) RenderFull(hostInterfaces []net.Interface) (*api.InstanceFull, any, error) { if d.IsSnapshot() { return nil, nil, errors.New("RenderFull doesn't work with snapshots") } // Get the Instance struct. base, etag, err := d.Render() if err != nil { return nil, nil, err } // Convert to InstanceFull. vmState := api.InstanceFull{Instance: *base.(*api.Instance)} // Add the InstanceState. vmState.State, err = d.renderState(vmState.StatusCode) if err != nil { return nil, nil, err } // Add the InstanceSnapshots. snaps, err := d.Snapshots() if err != nil { return nil, nil, err } for _, snap := range snaps { render, _, err := snap.Render() if err != nil { return nil, nil, err } if vmState.Snapshots == nil { vmState.Snapshots = []api.InstanceSnapshot{} } vmState.Snapshots = append(vmState.Snapshots, *render.(*api.InstanceSnapshot)) } // Add the InstanceBackups. backups, err := d.Backups() if err != nil { return nil, nil, err } for _, backup := range backups { render := backup.Render() if vmState.Backups == nil { vmState.Backups = []api.InstanceBackup{} } vmState.Backups = append(vmState.Backups, *render) } return &vmState, etag, nil } // renderState returns just state info about the instance. func (d *qemu) renderState(statusCode api.StatusCode) (*api.InstanceState, error) { // Initialize the return struct. status := &api.InstanceState{ Processes: -1, Status: statusCode.String(), StatusCode: statusCode, } // If VM is stopped or errored, we're done here. if d.isErrorStatusCode(statusCode) || !d.isRunningStatusCode(statusCode) { return status, nil } // If possible, get the metrics from the agent. if d.agentMetricsEnabled() { agentStatus, err := d.agentGetState() if err != nil { if !errors.Is(err, errQemuAgentOffline) { d.logger.Warn("Could not get VM state from agent", logger.Ctx{"err": err}) } } else { status = agentStatus } } // Override VM state back to QEMU state. status.Status = statusCode.String() status.StatusCode = statusCode // Add the network details if missing. if len(status.Network) == 0 { networkState, err := d.getNetworkState() if err != nil { return nil, err } status.Network = networkState } // Add the memory details if missing. if status.Memory.Usage <= 0 { monitor, err := d.qmpConnect() if err != nil { d.logger.Warn("Error getting QEMU monitor", logger.Ctx{"err": err}) } memoryMetrics, err := d.getQemuMemoryMetrics(monitor) if err != nil { d.logger.Warn("Error getting memory metrics", logger.Ctx{"err": err}) } status.Memory.Total = int64(memoryMetrics.MemTotalBytes) status.Memory.Usage = int64(memoryMetrics.MemTotalBytes - memoryMetrics.MemAvailableBytes) } // Populate the disk information. diskState, err := d.diskState() if err != nil && !errors.Is(err, storageDrivers.ErrNotSupported) { d.logger.Warn("Error getting disk usage", logger.Ctx{"err": err}) } status.Disk = diskState // Populate the CPU time allocation. limitsCPU, ok := d.expandedConfig["limits.cpu"] if ok { cpuCount, err := strconv.ParseInt(limitsCPU, 10, 64) if err != nil { status.CPU.AllocatedTime = cpuCount * 1_000_000_000 } } else { status.CPU.AllocatedTime = qemudefault.CPUCores * 1_000_000_000 } // Populate host_name for network devices. for k, m := range d.ExpandedDevices() { // We only care about nics. if m["type"] != "nic" { continue } // Get hwaddr from static or volatile config. hwaddr := m["hwaddr"] if hwaddr == "" { hwaddr = d.localConfig[fmt.Sprintf("volatile.%s.hwaddr", k)] } // We have to match on hwaddr as device name can be different from the configured device // name when reported from the agent inside the VM (due to the guest OS choosing name). for netName, netStatus := range status.Network { if netStatus.Hwaddr == hwaddr { if netStatus.HostName == "" { netStatus.HostName = d.localConfig[fmt.Sprintf("volatile.%s.host_name", k)] status.Network[netName] = netStatus } } } } // Populate the process information. pid, _ := d.pid() status.Pid = int64(pid) status.StartedAt, err = d.processStartedAt(d.InitPID()) if err != nil { return status, err } return status, nil } // RenderState returns just state info about the instance. func (d *qemu) RenderState(hostInterfaces []net.Interface) (*api.InstanceState, error) { return d.renderState(d.statusCode()) } // diskState gets disk usage info. func (d *qemu) diskState() (map[string]api.InstanceStateDisk, error) { pool, err := d.getStoragePool() if err != nil { return nil, err } // Get the root disk device config. rootDiskName, _, err := d.getRootDiskDevice() if err != nil { return nil, err } usage, err := pool.GetInstanceUsage(d) if err != nil { return nil, err } disk := map[string]api.InstanceStateDisk{} disk[rootDiskName] = api.InstanceStateDisk{ Usage: usage.Used, Total: usage.Total, } return disk, nil } // agentGetState connects to the agent inside of the VM and does // an API call to get the current state. func (d *qemu) agentGetState() (*api.InstanceState, error) { client, err := d.getAgentClient() if err != nil { return nil, err } ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() agent, err := incus.ConnectIncusHTTPWithContext(ctx, nil, client) if err != nil { return nil, fmt.Errorf("Failed connecting to agent: %w", err) } defer agent.Disconnect() status, _, err := agent.GetInstanceState("") if err != nil { return nil, err } return status, nil } // IsRunning returns whether or not the instance is running. func (d *qemu) IsRunning() bool { return d.isRunningStatusCode(d.statusCode()) } // IsFrozen returns whether the instance frozen or not. func (d *qemu) IsFrozen() bool { return d.statusCode() == api.Frozen } // CanMigrate returns whether the instance can be migrated. func (d *qemu) CanMigrate() string { return d.canMigrate(d) } // LockExclusive attempts to get exclusive access to the instance's root volume. func (d *qemu) LockExclusive() (*operationlock.InstanceOperation, error) { if d.IsRunning() { return nil, errors.New("Instance is running") } // Prevent concurrent operations the instance. op, err := operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionCreate, false, false) if err != nil { return nil, err } return op, err } // DeviceEventHandler handles events occurring on the instance's devices. func (d *qemu) DeviceEventHandler(runConf *deviceConfig.RunConfig) error { if !d.IsRunning() || runConf == nil { return nil } // Handle uevents. for _, uevent := range runConf.Uevents { for _, event := range uevent { fields := strings.SplitN(event, "=", 2) if fields[0] != "ACTION" { continue } switch fields[1] { case "add": for _, usbDev := range runConf.USBDevice { // This ensures that the device is actually removed from QEMU before adding it again. // In most cases the device will already be removed, but it is possible that the // device still exists in QEMU before trying to add it again. // If a USB device is physically detached from a running VM while the server // itself is stopped, QEMU in theory will not delete the device. err := d.deviceDetachUSB(usbDev) if err != nil { return err } err = d.deviceAttachUSB(usbDev) if err != nil { return err } } case "remove": for _, usbDev := range runConf.USBDevice { err := d.deviceDetachUSB(usbDev) if err != nil { return err } } } } } // Handle disk reconfiguration. for _, mount := range runConf.Mounts { if mount.Limits == nil && mount.Size == 0 { continue } // Get the QMP monitor. m, err := d.qmpConnect() if err != nil { return err } // Figure out the QEMU device ID. devID := fmt.Sprintf("%s%s", qemuDeviceIDPrefix, linux.PathNameEncode(mount.DevName)) if mount.Limits != nil { // Apply the limits. err = m.SetBlockThrottle(devID, int(mount.Limits.ReadBytes), int(mount.Limits.WriteBytes), int(mount.Limits.ReadIOps), int(mount.Limits.WriteIOps)) if err != nil { return fmt.Errorf("Failed applying limits for disk device %q: %w", mount.DevName, err) } } if mount.Size > 0 { // Update the size. err = m.UpdateBlockSize(strings.SplitN(devID, "-", 2)[1], mount.Size) if err != nil { return fmt.Errorf("Failed updating disk size %q: %w", mount.DevName, err) } } } // Handle NIC reconfiguration. var devName string var connected bool for _, dev := range runConf.NetworkInterface { switch dev.Key { case "devName": devName = dev.Value case "connected": connected = util.IsTrueOrEmpty(dev.Value) } } if devName != "" { // Get the QMP monitor. m, err := d.qmpConnect() if err != nil { return err } // Figure out the QEMU device ID. devID := fmt.Sprintf("%s%s", qemuDeviceIDPrefix, linux.PathNameEncode(devName)) err = m.SetNICLink(devID, connected) if err != nil { return fmt.Errorf("Failed setting NIC device link status: %w", err) } } return nil } // reservedVsockID returns true if the given vsockID equals 0, 1 or 2. // Those are reserved and we cannot use them. func (d *qemu) reservedVsockID(vsockID uint32) bool { return vsockID <= 2 } // getVsockID returns the vsock Context ID for the VM. func (d *qemu) getVsockID() (uint32, error) { existingVsockID, ok := d.localConfig["volatile.vsock_id"] if !ok { return 0, errors.New("Context ID not set in volatile.vsock_id") } vsockID, err := strconv.ParseUint(existingVsockID, 10, 32) if err != nil { return 0, fmt.Errorf("Failed to parse volatile.vsock_id: %q: %w", existingVsockID, err) } if d.reservedVsockID(uint32(vsockID)) { return 0, fmt.Errorf("Failed to use reserved vsock Context ID: %d", vsockID) } return uint32(vsockID), nil } // acquireVsockID tries to occupy the given vsock Context ID. // If the ID is free it returns the corresponding file handle. func (d *qemu) acquireVsockID(vsockID uint32) (*os.File, error) { reverter := revert.New() defer reverter.Fail() vsockF, err := os.OpenFile("/dev/vhost-vsock", os.O_RDWR, 0) if err != nil { return nil, fmt.Errorf("Failed to open vhost socket: %w", err) } reverter.Add(func() { _ = vsockF.Close() }) // The vsock Context ID cannot be supplied as type uint32. vsockIDInt := uint64(vsockID) // Call the ioctl to set the context ID. _, _, errno := unix.Syscall(unix.SYS_IOCTL, vsockF.Fd(), linux.IoctlVhostVsockSetGuestCid, uintptr(unsafe.Pointer(&vsockIDInt))) if errno != 0 { if !errors.Is(errno, unix.EADDRINUSE) { return nil, fmt.Errorf("Failed ioctl syscall to vhost socket: %q", errno.Error()) } // vsock Context ID is already in use. return nil, nil } reverter.Success() return vsockF, nil } // acquireExistingVsockID tries to acquire an already existing vsock Context ID from volatile. // It returns both the acquired ID and opened vsock file handle for QEMU. func (d *qemu) acquireExistingVsockID() (uint32, *os.File, error) { vsockID, err := d.getVsockID() if err != nil { return 0, nil, err } // Check if the vsockID from last VM start is still not acquired in case the VM was stopped. f, err := d.acquireVsockID(vsockID) if err != nil { return 0, nil, err } return vsockID, f, nil } // nextVsockID tries to acquire the next free vsock Context ID for the VM. // It returns both the acquired ID and opened vsock file handle for QEMU. func (d *qemu) nextVsockID() (uint32, *os.File, error) { // Check if vsock ID from last VM start is present in volatile, then use that. // This allows a running VM to be recovered after DB record deletion and that an agent connection still works // after the VM's instance ID has changed. // Continue in case of error since the caller requires a valid vsockID in any case. vsockID, vsockF, _ := d.acquireExistingVsockID() if vsockID != 0 && vsockF != nil { return vsockID, vsockF, nil } // Ignore the error from before and start to acquire a new Context ID. instanceUUID, err := uuid.Parse(d.localConfig["volatile.uuid"]) if err != nil { return 0, nil, fmt.Errorf("Failed to parse instance UUID from volatile.uuid: %w", err) } r, err := localUtil.GetStableRandomGenerator(instanceUUID.String()) if err != nil { return 0, nil, fmt.Errorf("Failed generating stable random seed from instance UUID %q: %w", instanceUUID, err) } timeout := time.Now().Add(5 * time.Second) // Try to find a new Context ID. for { if time.Now().After(timeout) { return 0, nil, errors.New("Timeout exceeded whilst trying to acquire the next vsock Context ID") } candidateVsockID := r.Uint32() if d.reservedVsockID(candidateVsockID) { continue } vsockF, err := d.acquireVsockID(candidateVsockID) if err != nil { return 0, nil, err } if vsockF != nil { return candidateVsockID, vsockF, nil } } } // InitPID returns the instance's current process ID. func (d *qemu) InitPID() int { pid, _ := d.pid() return pid } func (d *qemu) statusCode() api.StatusCode { // Shortcut to avoid spamming QMP during ongoing operations. op := operationlock.Get(d.Project().Name, d.Name()) if op != nil { if op.Action() == operationlock.ActionStart { return api.Stopped } if op.Action() == operationlock.ActionStop { if util.IsTrue(d.LocalConfig()["volatile.last_state.ready"]) { return api.Ready } return api.Running } } // Connect to the monitor. monitor, err := d.qmpConnect() if err != nil { // If cannot connect to monitor, but qemu process in pid file still exists, then likely qemu // is unresponsive and this instance is in an error state. pid, _ := d.pid() if pid > 0 { return api.Error } // If we fail to connect, chances are the VM isn't running. return api.Stopped } status, err := monitor.Status() if err != nil { if errors.Is(err, qmp.ErrMonitorDisconnect) { // If cannot connect to monitor, but qemu process in pid file still exists, then likely // qemu is unresponsive and this instance is in an error state. pid, _ := d.pid() if pid > 0 { return api.Error } return api.Stopped } return api.Error } switch status { case "prelaunch", "running": if status == "running" && util.IsTrue(d.LocalConfig()["volatile.last_state.ready"]) { return api.Ready } return api.Running case "inmigrate", "postmigrate", "finish-migrate", "save-vm", "suspended", "paused": return api.Frozen default: return api.Error } } // State returns the instance's state code. func (d *qemu) State() string { return strings.ToUpper(d.statusCode().String()) } // EarlyLogFilePath returns the instance's early log path. func (d *qemu) EarlyLogFilePath() string { return filepath.Join(d.LogPath(), "qemu.early.log") } // LogFilePath returns the instance's log path. func (d *qemu) LogFilePath() string { return filepath.Join(d.LogPath(), "qemu.log") } // QMPLogFilePath returns the instance's QMP log path. func (d *qemu) QMPLogFilePath() string { return filepath.Join(d.LogPath(), "qemu.qmp.log") } // FillNetworkDevice takes a nic or infiniband device type and enriches it with automatically // generated name and hwaddr properties if these are missing from the device. func (d *qemu) FillNetworkDevice(name string, m deviceConfig.Device) (deviceConfig.Device, error) { var err error newDevice := m.Clone() nicType, err := nictype.NICType(d.state, d.Project().Name, m) if err != nil { return nil, err } isPhysicalWithBridge := device.IsPhysicalNICWithBridge(d.state, d.Project().Name, m) // Fill in the MAC address. if (!slices.Contains([]string{"physical", "ipvlan"}, nicType) || isPhysicalWithBridge) && m["hwaddr"] == "" { configKey := fmt.Sprintf("volatile.%s.hwaddr", name) volatileHwaddr := d.localConfig[configKey] if volatileHwaddr == "" { // Generate a new MAC address. volatileHwaddr, err = instance.DeviceNextInterfaceHWAddr(d.MACPattern()) if err != nil || volatileHwaddr == "" { return nil, fmt.Errorf("Failed generating %q: %w", configKey, err) } // Update the database and update volatileHwaddr with stored value. volatileHwaddr, err = d.insertConfigkey(configKey, volatileHwaddr) if err != nil { return nil, fmt.Errorf("Failed storing generated config key %q: %w", configKey, err) } // Set stored value into current instance config. d.localConfig[configKey] = volatileHwaddr d.expandedConfig[configKey] = volatileHwaddr } if volatileHwaddr == "" { return nil, fmt.Errorf("Failed getting %q", configKey) } newDevice["hwaddr"] = volatileHwaddr } return newDevice, nil } // UpdateBackupFile writes the instance's backup.yaml file to storage. func (d *qemu) UpdateBackupFile() error { // Prevent concurrent updates to the backup file. unlock, err := d.updateBackupFileLock(context.Background()) if err != nil { return err } defer unlock() // Write the current instance state to backup file. pool, err := d.getStoragePool() if err != nil { return err } return pool.UpdateInstanceBackupFile(d, true, nil) } func (d *qemu) devIncusEventSend(eventType string, eventMessage map[string]any) error { event := jmap.Map{} event["type"] = eventType event["timestamp"] = time.Now() event["metadata"] = eventMessage client, err := d.getAgentClient() if err != nil { // Don't fail if the VM simply doesn't have an agent. if errors.Is(err, errQemuAgentOffline) { return nil } return err } agentArgs := &incus.ConnectionArgs{ SkipGetEvents: true, SkipGetServer: true, } ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() agent, err := incus.ConnectIncusHTTPWithContext(ctx, agentArgs, client) if err != nil { d.logger.Error("Failed to connect to the agent", logger.Ctx{"err": err}) return errors.New("Failed to connect to the agent") } defer agent.Disconnect() _, _, err = agent.RawQuery("POST", "/1.0/events", &event, "") if err != nil { return err } return nil } // Info returns "qemu" and the currently loaded qemu version. func (d *qemu) Info() instance.Info { data := instance.Info{ Name: "qemu", Features: make(map[string]any), Type: instancetype.VM, Error: errors.New("Unknown error"), } if !util.PathExists("/dev/kvm") { data.Error = errors.New("KVM support is missing (no /dev/kvm)") return data } err := linux.LoadModule("vhost_vsock") if err != nil { data.Error = errors.New("vhost_vsock kernel module not loaded") return data } if !util.PathExists("/dev/vsock") { data.Error = errors.New("Vsock support is missing (no /dev/vsock)") return data } hostArch, err := osarch.ArchitectureGetLocalID() if err != nil { logger.Errorf("Failed getting CPU architecture during QEMU initialization: %v", err) data.Error = errors.New("Failed getting CPU architecture") return data } qemuPath, _, err := d.qemuArchConfig(hostArch) if err != nil { data.Error = fmt.Errorf("QEMU command not available: %v", err) return data } out, err := exec.Command(qemuPath, "--version").Output() if err != nil { logger.Errorf("Failed getting version during QEMU initialization: %v", err) data.Error = errors.New("Failed getting QEMU version") return data } qemuOutput := strings.Fields(string(out)) if len(qemuOutput) >= 4 { qemuVersion := strings.Fields(string(out))[3] data.Version = qemuVersion } else { data.Version = "unknown" // Not necessarily an error that should prevent us using driver. } data.Features, err = d.checkFeatures(hostArch, qemuPath) if err != nil { logger.Errorf("Unable to run feature checks during QEMU initialization: %v", err) data.Error = errors.New("QEMU failed to run feature checks") return data } data.Error = nil return data } func (d *qemu) checkFeatures(hostArch int, qemuPath string) (map[string]any, error) { monitorPath, err := os.CreateTemp("", "") if err != nil { return nil, err } defer func() { _ = os.Remove(monitorPath.Name()) }() qemuArgs := []string{ qemuPath, "-S", // Do not start virtualisation. "-nographic", "-nodefaults", "-no-user-config", "-chardev", fmt.Sprintf("socket,id=monitor,path=%s,server=on,wait=off", qemuEscapeCmdline(monitorPath.Name())), "-mon", "chardev=monitor,mode=control", "-machine", qemuMachineType(hostArch), } if hostArch == osarch.ARCH_64BIT_INTEL_X86 { // On Intel, use KVM acceleration as it's needed for SEV detection. // This also happens to be less resource intensive but can't // trivially be performed on all architectures without extra care about the // machine type. qemuArgs = append(qemuArgs, "-accel", "kvm") } if d.architectureSupportsUEFI(hostArch) { // Try to locate a UEFI firmware. var efiPath string firmwares, err := edk2.GetArchitectureFirmwarePairsForUsage(hostArch, edk2.GENERIC) if err != nil { return nil, err } for _, firmwarePair := range firmwares { if util.PathExists(firmwarePair.Code) { efiPath = firmwarePair.Code break } } if efiPath == "" { return nil, errors.New("Unable to locate a UEFI firmware") } qemuArgs = append(qemuArgs, "-drive", fmt.Sprintf("if=pflash,format=raw,readonly=on,file=%s", qemuEscapeCmdline(efiPath))) } var stderr bytes.Buffer checkFeature := exec.Cmd{ Path: qemuPath, Args: qemuArgs, Stderr: &stderr, } err = checkFeature.Start() if err != nil { // QEMU not operational. VM support missing. return nil, fmt.Errorf("Failed starting QEMU: %w", err) } defer func() { _ = checkFeature.Process.Kill() }() // Start go routine that waits for QEMU to exit and captures the exit error (if any). errWaitCh := make(chan error, 1) go func() { errWaitCh <- checkFeature.Wait() }() // Start go routine that tries to connect to QEMU's QMP socket in a loop (giving QEMU a chance to open it). ctx, cancelMonitorConnect := context.WithTimeout(context.Background(), 5*time.Second) defer cancelMonitorConnect() errMonitorCh := make(chan error, 1) var monitor *qmp.Monitor go func() { var err error // Try and connect to QMP socket until cancelled. for { monitor, err = qmp.Connect(monitorPath.Name(), qemuSerialChardevName, nil, "", d.detachDisk) // QMP successfully connected or we have been cancelled. if err == nil || ctx.Err() != nil { break } time.Sleep(50 * time.Millisecond) } // Return last QMP connection error. errMonitorCh <- err }() // Wait for premature QEMU exit or QMP to connect or timeout. select { case errMonitor := <-errMonitorCh: // A non-nil error here means that QMP failed to connect before timing out. // The last connection error is returned. // A nil error means QMP successfully connected and we can continue. if errMonitor != nil { return nil, fmt.Errorf("QEMU monitor connect error: %w", errMonitor) } case errWait := <-errWaitCh: // Any sort of premature exit, even a non-error one is problematic here, and should not occur. return nil, fmt.Errorf("QEMU premature exit: %w (%v)", errWait, strings.TrimSpace(stderr.String())) } defer monitor.Disconnect() features := make(map[string]any) blockDevPath, err := os.CreateTemp("", "") if err != nil { return nil, err } defer func() { _ = os.Remove(blockDevPath.Name()) }() // Check io_uring feature. blockDev := map[string]any{ "node-name": d.blockNodeName("feature-check"), "driver": "file", "filename": blockDevPath.Name(), "aio": "io_uring", } err = monitor.AddBlockDevice(blockDev, nil, false) if err != nil { logger.Debug("Failed adding block device during VM feature check", logger.Ctx{"err": err}) } else { features["io_uring"] = struct{}{} } // Check CPU hotplug feature. _, err = monitor.QueryHotpluggableCPUs() if err != nil { logger.Debug("Failed querying hotpluggable CPUs during VM feature check", logger.Ctx{"err": err}) } else { features["cpu_hotplug"] = struct{}{} } // Check AMD SEV features (only for x86 architecture) if hostArch == osarch.ARCH_64BIT_INTEL_X86 { cmdline, err := os.ReadFile("/proc/cmdline") if err != nil { return nil, err } parts := strings.Split(string(cmdline), " ") // Check if SME is enabled in the kernel command line. // codespell:ignore sme if slices.Contains(parts, "mem_encrypt=on") || util.PathExists("/dev/sev") { features["sme"] = struct{}{} // codespell:ignore sme } // Check if SEV/SEV-ES are enabled sev, err := os.ReadFile("/sys/module/kvm_amd/parameters/sev") if err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, err } else if strings.TrimSpace(string(sev)) == "Y" { // Host supports SEV, check if QEMU supports it as well. capabilities, err := monitor.SEVCapabilities() if err != nil { logger.Debug("Failed querying SEV capability during VM feature check", logger.Ctx{"err": err}) } else { features["sev"] = capabilities // If SEV is enabled on host and supported by QEMU, // check if the SEV-ES extension is enabled. sevES, err := os.ReadFile("/sys/module/kvm_amd/parameters/sev_es") if err != nil { logger.Debug("Failed querying SEV-ES capability during VM feature check", logger.Ctx{"err": err}) } else if strings.TrimSpace(string(sevES)) == "Y" { features["sev-es"] = struct{}{} } } } } // Check if vhost-net accelerator (for NIC CPU offloading) is available. if util.PathExists("/dev/vhost-net") { features["vhost_net"] = struct{}{} } // Check if SPICE is compiled into QEMU. err = monitor.QuerySpice() if err != nil { logger.Debug("Failed querying SPICE during VM feature check", logger.Ctx{"err": err}) } else { features["spice"] = struct{}{} } // Check if virtio-9p-pci is compiled into QEMU. err = monitor.Query9pDevice() if err != nil { logger.Debug("Failed querying virtio-9p-pci during VM feature check", logger.Ctx{"err": err}) } else { features["plan9"] = struct{}{} } // Check if virtio-sound-pci is compiled into QEMU. err = monitor.QueryVirtioSoundDevice() if err != nil { logger.Debug("Failed querying virtio-sound-pci during VM feature check", logger.Ctx{"err": err}) } else { features["virtio-sound"] = struct{}{} } // Check if running nested. cpus, err := resources.GetCPU() if err != nil { return nil, err } nested := false for _, socket := range cpus.Sockets { for _, core := range socket.Cores { if slices.Contains(core.Flags, "hypervisor") { nested = true } } } if nested { features["nested"] = struct{}{} } // Get the host CPU model (x86_64 only for now). if hostArch == osarch.ARCH_64BIT_INTEL_X86 { model, err := monitor.QueryCPUModel("kvm64") if err != nil { return nil, err } cpuFlags := map[string]bool{} for k, v := range model.Flags { value, ok := v.(bool) if !ok { continue } cpuFlags[k] = value } features["flags"] = cpuFlags } return features, nil } // version returns the QEMU version. func (d *qemu) version() (*version.DottedVersion, error) { info := DriverStatuses()[instancetype.VM].Info qemuVer, err := version.NewDottedVersion(info.Version) if err != nil { return nil, fmt.Errorf("Failed parsing QEMU version: %w", err) } return qemuVer, nil } func (d *qemu) Metrics(hostInterfaces []net.Interface) (*metrics.MetricSet, error) { if !d.IsRunning() { return nil, ErrInstanceIsStopped } if d.agentMetricsEnabled() { metrics, err := d.getAgentMetrics() if err != nil { if !errors.Is(err, errQemuAgentOffline) { d.logger.Warn("Could not get VM metrics from agent", logger.Ctx{"err": err}) } // Fallback data if agent is not reachable. return d.getQemuMetrics() } return metrics, nil } return d.getQemuMetrics() } func (d *qemu) getAgentMetrics() (*metrics.MetricSet, error) { client, err := d.getAgentClient() if err != nil { return nil, err } agentArgs := &incus.ConnectionArgs{ SkipGetEvents: true, SkipGetServer: true, } ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() agent, err := incus.ConnectIncusHTTPWithContext(ctx, agentArgs, client) if err != nil { d.logger.Error("Failed to connect to the agent", logger.Ctx{"project": d.Project().Name, "instance": d.Name(), "err": err}) return nil, errors.New("Failed to connect to the agent") } defer agent.Disconnect() resp, _, err := agent.RawQuery("GET", "/1.0/metrics", nil, "") if err != nil { return nil, err } var m metrics.Metrics err = json.Unmarshal(resp.Metadata, &m) if err != nil { return nil, err } metricSet, err := metrics.MetricSetFromAPI(&m, map[string]string{"project": d.project.Name, "name": d.name, "type": instancetype.VM.String()}) if err != nil { return nil, err } return metricSet, nil } func (d *qemu) getNetworkState() (map[string]api.InstanceStateNetwork, error) { networks := map[string]api.InstanceStateNetwork{} for k, m := range d.ExpandedDevices() { if m["type"] != "nic" { continue } dev, err := d.deviceLoad(d, k, m, false) if err != nil { if errors.Is(err, device.ErrUnsupportedDevType) { continue // Skip unsupported device (allows for mixed instance type profiles). } d.logger.Warn("Failed state validation for device", logger.Ctx{"device": k, "err": err}) continue } // Only some NIC types support fallback state mechanisms when there is no agent. nic, ok := dev.(device.NICState) if !ok { continue } network, err := nic.State() if err != nil { return nil, fmt.Errorf("Failed getting NIC state for %q: %w", k, err) } if network != nil { networks[k] = *network } } return networks, nil } func (d *qemu) agentMetricsEnabled() bool { return util.IsTrueOrEmpty(d.expandedConfig["security.agent.metrics"]) } func (d *qemu) deviceAttachUSB(usbConf deviceConfig.USBDeviceItem) error { // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return err } monHook, err := d.addUSBDeviceConfig(usbConf) if err != nil { return err } err = monHook(monitor) if err != nil { return err } return nil } func (d *qemu) deviceDetachUSB(usbDev deviceConfig.USBDeviceItem) error { // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return err } deviceID := fmt.Sprintf("%s%s", qemuDeviceIDPrefix, usbDev.DeviceName) err = monitor.RemoveDevice(deviceID) if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { return fmt.Errorf("Failed removing device: %w", err) } err = monitor.RemoveFDFromFDSet(deviceID) if err != nil { return fmt.Errorf("Failed removing FD set: %w", err) } return nil } // Block node names may only be up to 31 characters long, so use a hash if longer. func (d *qemu) blockNodeName(name string) string { // Apply the prefix. return fmt.Sprintf("%s%s", qemuBlockDevIDPrefix, hashValue(name, 25)) } // Mount tag names may only be up to 31 or 36 characters long, so use a hash if longer. func (d *qemu) mountTagName(name string, maxLength int) string { // Apply the prefix. return fmt.Sprintf("%s%s", qemuMountTagPrefix, hashValue(name, maxLength)) } func (d *qemu) setCPUs(monitor *qmp.Monitor, count int) error { if count == 0 { return nil } // Check if the agent is running. if monitor == nil { var err error monitor, err = d.qmpConnect() if err != nil { return err } } cpus, err := monitor.QueryHotpluggableCPUs() if err != nil { return fmt.Errorf("Failed to query hotpluggable CPUs: %w", err) } var availableCPUs []qmp.HotpluggableCPU var hotpluggedCPUs []qmp.HotpluggableCPU // Count the available and hotplugged CPUs. for _, cpu := range cpus { // If qom-path is unset, the CPU is available. if cpu.QOMPath == "" { availableCPUs = append(availableCPUs, cpu) } else if strings.HasPrefix(cpu.QOMPath, "/machine/peripheral") { hotpluggedCPUs = append(hotpluggedCPUs, cpu) } } // The reserved CPUs includes both the hotplugged CPUs as well as the fixed one. totalReservedCPUs := len(hotpluggedCPUs) + 1 // Nothing to do as the count matches the already reserved CPUs. if count == totalReservedCPUs { return nil } reverter := revert.New() defer reverter.Fail() // More CPUs requested. if count > totalReservedCPUs { // Cannot allocate more CPUs than the system provides. if count > len(cpus) { return fmt.Errorf("Requested CPU count of %d exceeds instance current maximum of %d, restart required", count, len(cpus)) } // This shouldn't trigger, but if it does, don't panic. if count-totalReservedCPUs > len(availableCPUs) { return errors.New("Unable to allocate more CPUs, not enough hotpluggable CPUs available") } // Only allocate the difference in CPUs. for i := range count - totalReservedCPUs { cpu := availableCPUs[i] devID := fmt.Sprintf("cpu%d%d%d", cpu.Props.SocketID, cpu.Props.CoreID, cpu.Props.ThreadID) qemuDev := map[string]any{ "id": devID, "driver": cpu.Type, "core-id": cpu.Props.CoreID, } // No such thing as sockets and threads on s390x. if d.architecture != osarch.ARCH_64BIT_S390_BIG_ENDIAN { qemuDev["socket-id"] = cpu.Props.SocketID qemuDev["thread-id"] = cpu.Props.ThreadID } err := monitor.AddDevice(qemuDev) if err != nil { return fmt.Errorf("Failed to add device: %w", err) } reverter.Add(func() { err := monitor.RemoveDevice(devID) d.logger.Warn("Failed to remove CPU device", logger.Ctx{"err": err}) }) } } else { if totalReservedCPUs-count > len(hotpluggedCPUs) { // This shouldn't trigger, but if it does, don't panic. return errors.New("Unable to remove CPUs, not enough hotpluggable CPUs available") } // Less CPUs requested. for i := range totalReservedCPUs - count { cpu := hotpluggedCPUs[i] fields := strings.Split(cpu.QOMPath, "/") devID := fields[len(fields)-1] err := monitor.RemoveDevice(devID) if err != nil { return fmt.Errorf("Failed to remove CPU: %w", err) } reverter.Add(func() { err := monitor.AddDevice(map[string]any{ "id": devID, "driver": cpu.Type, "socket-id": cpu.Props.SocketID, "core-id": cpu.Props.CoreID, "thread-id": cpu.Props.ThreadID, }) d.logger.Warn("Failed to add CPU device", logger.Ctx{"err": err}) }) } // QEMU doesn't immediately remove the thread from the vCPU list. // Wait a second to allow the thread to fully exit and disappear from the vCPU list. time.Sleep(time.Second) } reverter.Success() // Run post-hotplug tasks. err = d.postCPUHotplug(monitor) if err != nil { return err } return nil } func (d *qemu) architectureSupportsCPUHotplug() bool { // Check supported features. info := DriverStatuses()[instancetype.VM].Info _, found := info.Features["cpu_hotplug"] return found } func (d *qemu) postCPUHotplug(monitor *qmp.Monitor) error { // Get the vCPU PID list. pids, err := monitor.GetCPUs() if err != nil { return err } // Handle NUMA node restrictions. numaNodes := d.expandedConfig["limits.cpu.nodes"] if numaNodes != "" { if numaNodes == "balanced" { numaNodes = d.expandedConfig["volatile.cpu.nodes"] } // Parse the NUMA restriction. numaNodeSet, err := resources.ParseNumaNodeSet(numaNodes) if err != nil { return err } // Get the CPU topology. cpusTopology, err := resources.GetCPU() if err != nil { return err } // Get the isolated CPU ids. isolatedCpusInt := resources.GetCPUIsolated() // Build a map of NUMA node to CPU threads. numaNodeToCPU := make(map[int64][]int64) for _, cpu := range cpusTopology.Sockets { for _, core := range cpu.Cores { for _, thread := range core.Threads { // Skip any isolated CPU thread. if slices.Contains(isolatedCpusInt, thread.ID) { continue } numaNodeToCPU[int64(thread.NUMANode)] = append(numaNodeToCPU[int64(thread.NUMANode)], thread.ID) } } } // Figure out the list of CPU threads for the NUMA node(s). set := unix.CPUSet{} for _, numaNode := range numaNodeSet { for _, id := range numaNodeToCPU[numaNode] { set.Set(int(id)) } } // Apply the restriction. for _, pid := range pids { // Apply the pin. err := unix.SchedSetaffinity(pid, &set) if err != nil { return err } } } // Create a core scheduling group. err = d.setCoreSched(pids) if err != nil { return fmt.Errorf("Failed to allocate new core scheduling domain for vCPU threads: %w", err) } return nil } // ConsoleLog returns all output sent to the instance's console's ring buffer since startup. func (d *qemu) ConsoleLog() (string, error) { // Setup a new operation. op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionConsoleRetrieve, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore, operationlock.ActionMigrate}, false, true) if err != nil { return "", err } // Only mark the operation as done if only processing the console retrieval. if op.Action() == operationlock.ActionConsoleRetrieve { defer op.Done(nil) } // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return "", err } logString, err := monitor.RingbufRead("console") if err != nil { // If a VM was started by an older version of Incus which was then upgraded, its // console device won't be a ring buffer. We don't want to cause an error in this // case, so just return an empty string. if errors.Is(err, qmp.ErrNotARingbuf) { return "", nil } return "", err } // If we got data back, append it to the log file for this instance. if logString != "" { logFile, err := os.OpenFile(d.ConsoleBufferLogPath(), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o600) if err != nil { return "", err } defer logFile.Close() _, err = logFile.WriteString(logString) if err != nil { return "", err } } // Read and return the complete log for this instance. fullLog, err := os.ReadFile(d.ConsoleBufferLogPath()) if err != nil { if errors.Is(err, fs.ErrNotExist) { // If there's no log file yet, such as right at VM creation, return an empty string. return "", nil } return "", err } return string(fullLog), nil } // consoleSwapRBWithSocket swaps the qemu backend for the instance's console to a unix socket. func (d *qemu) consoleSwapRBWithSocket() error { // This will wipe out anything in the existing ring buffer; save any buffered data to log file first. _, err := d.ConsoleLog() if err != nil { return err } // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return err } // Create the unix socket here, which will be passed via file descriptor to qemu. d.consoleSocket, err = net.ListenUnix("unix", &net.UnixAddr{Name: d.consolePath(), Net: "unix"}) if err != nil { return err } d.consoleSocketFile, err = d.consoleSocket.File() if err != nil { _ = d.consoleSocket.Close() _ = os.Remove(d.consolePath()) return err } return monitor.ChardevChange("console", qmp.ChardevChangeInfo{Type: "socket", FDName: "consoleSocket", File: d.consoleSocketFile}) } // consoleSwapSocketWithRB swaps the qemu backend for the instance's console to a ring buffer. func (d *qemu) consoleSwapSocketWithRB() error { // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return err } defer func() { // Clean up the old socket. _ = d.consoleSocketFile.Close() _ = d.consoleSocket.Close() _ = os.Remove(d.consolePath()) }() return monitor.ChardevChange("console", qmp.ChardevChangeInfo{Type: "ringbuf"}) } // ConsoleScreenshot returns a screenshot of the current VGA console in PNG format. func (d *qemu) ConsoleScreenshot(screenshotFile *os.File) error { if !d.IsRunning() { return errors.New("Instance is not running") } // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return err } err = screenshotFile.Chown(int(d.state.OS.UnprivUID), -1) if err != nil { return fmt.Errorf("Failed to chown screenshot path: %w", err) } // Take the screenshot. err = monitor.Screendump(screenshotFile.Name()) if err != nil { return fmt.Errorf("Failed taking screenshot: %w", err) } return nil } // ReloadDevice triggers an empty Update call to the underlying device. func (d *qemu) ReloadDevice(devName string) error { dev, err := d.deviceLoad(d, devName, d.expandedDevices[devName], false) if err != nil { return err } return dev.Update(d.expandedDevices, true) } // DumpGuestMemory dumps the guest memory to a file in the specified format. func (d *qemu) DumpGuestMemory(w *os.File, format string) error { if !d.IsRunning() { return errors.New("Instance is not running") } // Check if the agent is running. monitor, err := d.qmpConnect() if err != nil { return err } defer monitor.Disconnect() // Dump the guest memory. err = monitor.SendFile("memory-dump", w) if err != nil { return err } err = monitor.DumpGuestMemory("memory-dump", format) if err != nil { return err } // Close the writer. err = w.Close() if err != nil { return err } return nil } // CanLiveMigrate returns whether the VM is live-migratable. func (d *qemu) CanLiveMigrate() bool { // Refuse migration if not enabled. if !util.IsTrue(d.expandedConfig["migration.stateful"]) { return false } // Additional checks when the VM is running. if d.IsRunning() { // Check if it may have been enabled through a later profile (and so still not supported). bs, err := d.getBootState() if err != nil { return false } // Ideally we'd just check for Version > 0 but that would prevent // all migrations from older Incus versions that predate the introduction // of the boot state recording. So instead rely on the machine type // recording which is going to be present on those older versions too. if bs.MachineType == "" { return false } } return true } // GuestOS returns the guest OS. In this driver, we consider anything unknown to be Linux. func (d *qemu) GuestOS() string { imageOS := strings.ToLower(d.expandedConfig["image.os"]) matches := func(names ...string) bool { for _, name := range names { if strings.Contains(imageOS, name) { return true } } return false } if matches("windows") { return "windows" } if matches("darwin", "macos", "mac os") { return "macos" } if matches("freebsd", "opnsense", "pfsense") { return "freebsd" } return "unknown" } // CreateQcow2Snapshot creates a qcow2 snapshot for a running instance. func (d *qemu) CreateQcow2Snapshot(devPath string, devName string, snapshotName string, backingFilename string, stateful bool) error { reverter := revert.New() defer reverter.Fail() monitor, err := d.qmpConnect() if err != nil { return err } f, err := os.OpenFile(devPath, unix.O_RDWR, 0) if err != nil { return fmt.Errorf("Failed opening file descriptor for disk device %s: %w", devPath, err) } defer func() { _ = f.Close() }() devName = d.blockNodeName(linux.PathNameEncode(devName)) // Select all block devices related to a qcow2 backing chain. blockDevs, err := d.fetchBlockDeviceChain(monitor, devName) if err != nil { return err } // Fetch the current maximum overlay index. overlayNodeIndex := currentQcow2OverlayIndex(blockDevs, devName) nextOverlayName := fmt.Sprintf("%s_overlay%d", devName, overlayNodeIndex+1) currentRootNode := blockDevs[len(blockDevs)-1] info, err := monitor.SendFileWithFDSet(nextOverlayName, f, false) if err != nil { return fmt.Errorf("Failed sending file descriptor of %q for disk device: %w", f.Name(), err) } reverter.Add(func() { _ = monitor.RemoveFDFromFDSet(nextOverlayName) }) blockDev := map[string]any{ "driver": "qcow2", "discard": "unmap", // Forward as an unmap request. This is the same as `discard=on` in the qemu config file. "node-name": nextOverlayName, "read-only": false, "file": map[string]any{ "driver": "host_device", "filename": fmt.Sprintf("/dev/fdset/%d", info.ID), }, } // Add overlay block dev. err = monitor.AddBlockDevice(blockDev, nil, false) if err != nil { return fmt.Errorf("Fail to add block device: %w", err) } reverter.Add(func() { _ = monitor.RemoveBlockDevice(nextOverlayName) }) // Take a snapshot of the root disk and redirect writes to the snapshot disk. err = monitor.BlockDevSnapshot(currentRootNode, nextOverlayName) if err != nil { return fmt.Errorf("Failed taking storage snapshot: %w", err) } reverter.Add(func() { _ = monitor.BlockCommit(nextOverlayName, "", "") }) // Update metadata of the backing file. // Use the Qcow2Rebase method when performing stateful snapshots. // Using QMP to modify a volume that was added while the VM is paused can cause QEMU to crash. if stateful { err = storageDrivers.Qcow2Rebase(devPath, backingFilename) if err != nil { return err } } else { err = monitor.ChangeBackingFile(nextOverlayName, nextOverlayName, backingFilename) if err != nil { return fmt.Errorf("Failed changing backing file: %w", err) } } reverter.Success() return nil } // fetchBlockDeviceChain returns the ordered list of block device names // required to load a volume, including any dependent layers. func (d *qemu) fetchBlockDeviceChain(m *qmp.Monitor, blockDevName string) ([]string, error) { // Fetch information about block devices. blockdevNames, err := m.QueryNamedBlockNodes() if err != nil { return nil, fmt.Errorf("Failed fetching block nodes names: %w", err) } return filterAndSortQcow2Blockdevs(blockdevNames, blockDevName), nil } // fetchRootBlockDeviceChain returns the ordered list of block device names // required to load the root disk device, including any dependent layers. func (d *qemu) fetchRootBlockDeviceChain(m *qmp.Monitor) ([]string, error) { rootDevName, _, err := internalInstance.GetRootDiskDevice(d.expandedDevices.CloneNative()) if err != nil { return nil, fmt.Errorf("Failed getting instance root disk: %w", err) } escapedDeviceName := linux.PathNameEncode(rootDevName) rootNodeName := d.blockNodeName(escapedDeviceName) return d.fetchBlockDeviceChain(m, rootNodeName) } // DeleteQcow2Snapshot deletes a qcow2 snapshot for a running instance. func (d *qemu) DeleteQcow2Snapshot(devName string, snapshotIndex int, backingFilename string) error { monitor, err := d.qmpConnect() if err != nil { return err } devName = d.blockNodeName(linux.PathNameEncode(devName)) // Select all block devices related to a qcow2 backing chain. blockDevs, err := d.fetchBlockDeviceChain(monitor, devName) if err != nil { return err } d.logger.Debug("QCOW2 blockdev chain:", logger.Ctx{"blockdev": blockDevs, "snapshotIndex": snapshotIndex, "devName": devName}) if snapshotIndex < 0 || (snapshotIndex+1) >= len(blockDevs) { return fmt.Errorf("Incorrect snapshot index: %d", snapshotIndex) } rootDevName := blockDevs[len(blockDevs)-1] snapChildDevName := blockDevs[snapshotIndex+1] snapDevName := blockDevs[snapshotIndex] err = monitor.BlockCommit(rootDevName, snapChildDevName, snapDevName) if err != nil { return err } err = monitor.RemoveBlockDevice(snapChildDevName) if err != nil { d.logger.Error("Remove block device error", logger.Ctx{"err": err}) return err } err = monitor.RemoveFDFromFDSet(snapChildDevName) if err != nil { d.logger.Error("Remove fd from fd set", logger.Ctx{"err": err}) return err } if backingFilename == "" { return nil } if snapshotIndex+2 >= len(blockDevs) { return fmt.Errorf("Incorrect snapshot index for backing file update: %d", snapshotIndex) } // Update metadata of the backing file. err = monitor.ChangeBackingFile(rootDevName, blockDevs[snapshotIndex+2], backingFilename) if err != nil { return fmt.Errorf("Failed changing backing file: %w", err) } return nil } // ExportQcow2Block exports a qcow2 block device by exposing it through a QEMU NBD server. func (d *qemu) ExportQcow2Block(diskName string, blockIndex int) (func(), string, error) { monitor, err := d.qmpConnect() if err != nil { return nil, "", err } socketPath := d.migrateSockPath() addr, err := net.ResolveUnixAddr("unix", socketPath) if err != nil { return nil, "", err } migrationSock, err := net.ListenUnix("unix", addr) if err != nil { return nil, "", fmt.Errorf("Error connecting to migration socket %q: %w", socketPath, err) } migrationFile, err := migrationSock.File() if err != nil { return nil, "", fmt.Errorf("Error opening migration socket %q: %w", socketPath, err) } err = monitor.SendFile(socketPath, migrationFile) if err != nil { return nil, "", fmt.Errorf("Failed to send migration file descriptor: %w", err) } err = monitor.NBDUnixServerStart(socketPath) if err != nil { return nil, "", fmt.Errorf("Failed starting NBD server: %w", err) } escapedDeviceName := linux.PathNameEncode(diskName) nodeName := d.blockNodeName(escapedDeviceName) // Selects all block devices related to this instance (backing, root disk, overlays). blockDevs, err := d.fetchBlockDeviceChain(monitor, nodeName) if err != nil { return nil, "", err } d.logger.Debug("Instance block devices:", logger.Ctx{"blockdev": blockDevs, "blockIndex": blockIndex}) if blockIndex < 0 || blockIndex >= len(blockDevs) { return nil, "", fmt.Errorf("Incorrect block device index: %d", blockIndex) } exportBlockName := blockDevs[blockIndex] exportDiskPath := fmt.Sprintf("nbd+unix:///%s?socket=%s", exportBlockName, socketPath) err = monitor.NBDBlockExportAdd(exportBlockName, false, nil) if err != nil { return nil, "", fmt.Errorf("Failed adding disk to NBD server: %w", err) } return func() { _ = monitor.NBDServerStop() _ = migrationSock.Close() }, exportDiskPath, nil } func (d *qemu) isQCOW2(devPath string) (bool, error) { imgInfo, err := storageDrivers.Qcow2Info(devPath) if err != nil { return false, err } return imgInfo.Format == storageDrivers.BlockVolumeTypeQcow2, nil } func (d *qemu) qcow2BlockDev(m *qmp.Monitor, nodeName string, aioMode string, directCache bool, noFlushCache bool, permissions int, readonly bool, backingPaths []string, iter int) (string, error) { devName := backingPaths[0] backingNodeName := fmt.Sprintf("%s_backing%d", nodeName, iter) f, err := os.OpenFile(devName, permissions, 0) if err != nil { return "", fmt.Errorf("Failed opening file descriptor for disk device %q: %w", devName, err) } defer func() { _ = f.Close() }() info, err := m.SendFileWithFDSet(backingNodeName, f, readonly) if err != nil { return "", fmt.Errorf("Failed sending file descriptor of %q for disk device %q: %w", f.Name(), devName, err) } blockDev := map[string]any{ "driver": "qcow2", "discard": "unmap", // Forward as an unmap request. This is the same as `discard=on` in the qemu config file. "node-name": backingNodeName, "read-only": false, "file": map[string]any{ "driver": "host_device", "filename": fmt.Sprintf("/dev/fdset/%d", info.ID), "aio": aioMode, "cache": map[string]any{ "direct": directCache, "no-flush": noFlushCache, }, }, } // If there are any children, load block information about them. if len(backingPaths) > 1 { parentNodeName, err := d.qcow2BlockDev(m, nodeName, aioMode, directCache, noFlushCache, permissions, readonly, backingPaths[1:], iter+1) if err != nil { return "", err } blockDev["backing"] = parentNodeName } err = m.AddBlockDevice(blockDev, nil, false) if err != nil { return "", err } return backingNodeName, nil } // currentQcow2OverlayIndex returns the current maximum overlay index. func currentQcow2OverlayIndex(names []string, prefix string) int { re := regexp.MustCompile(fmt.Sprintf(`^%s_overlay(\d+)$`, prefix)) maxIndex := -1 for _, name := range names { m := re.FindStringSubmatch(name) if len(m) == 2 { n, err := strconv.Atoi(m[1]) if err == nil && n > maxIndex { maxIndex = n } } } return maxIndex } func (d *qemu) needsFullRestart() bool { // Check if we have a pending change. if d.localConfig["volatile.vm.needs_reset"] != "" { return true } // Check if the QEMU binary has changed. pid, _ := d.pid() if pid <= 0 { return true } exePath, _, err := d.qemuArchConfig(d.architecture) if err != nil { return true } var curExe unix.Stat_t err = unix.Stat(fmt.Sprintf("/proc/%d/exe", pid), &curExe) if err != nil { return true } var nextExe unix.Stat_t err = unix.Stat(exePath, &nextExe) if err != nil { return true } if curExe.Size != nextExe.Size || curExe.Ino != nextExe.Ino || curExe.Dev != nextExe.Dev { return true } // Full restart isn't required. return false } // ConnectNBD exports a disk over NBD. Not supported by containers. func (d *qemu) ConnectNBD(diskName string, volSize int64, writable bool) (net.Conn, func(), error) { monitor, err := d.qmpConnect() if err != nil { return nil, nil, err } // Check for existing NBD block exports to detect if another operation is in progress. blocks, err := monitor.QueryNBDBlockExports() if err == nil && len(blocks) > 0 { return nil, nil, fmt.Errorf("Another NBD operation is already in progress for: %s", blocks[0].NodeName) } nbdConn, err := monitor.NBDServerStart() if err != nil { return nil, nil, fmt.Errorf("Failed starting NBD server: %w", err) } d.logger.Debug("User requested NBD server started") reverter := revert.New() defer reverter.Fail() disconnect := func() { d.logger.Debug("User requested NBD server stopped") _ = nbdConn.Close() _ = monitor.NBDServerStop() } reverter.Add(disconnect) bitmaps, err := d.GetBitmaps(diskName) if err != nil { return nil, nil, fmt.Errorf("Failed fetching bitmaps for %q: %w", diskName, err) } bitmapNames := []string{} for _, b := range bitmaps { if b.Inconsistent { continue } bitmapNames = append(bitmapNames, b.Name) } escapedDeviceName := linux.PathNameEncode(diskName) nodeName := d.blockNodeName(escapedDeviceName) blockDevs, err := d.fetchBlockDeviceChain(monitor, nodeName) if err != nil { return nil, nil, fmt.Errorf("Failed fetching disk chain: %w", err) } blockExport := blockDevs[len(blockDevs)-1] if !writable { cleanupSnapshot, err := d.createEphemeralSnapshot(blockExport, volSize) if err != nil { return nil, nil, fmt.Errorf("Failed creating temporary snapshot: %w", err) } reverter.Add(cleanupSnapshot) } err = monitor.NBDBlockExportAdd(blockExport, writable, bitmapNames) if err != nil { return nil, nil, fmt.Errorf("Failed adding disk to NBD server: %w", err) } cleanup := reverter.Clone().Fail reverter.Success() return nbdConn, cleanup, nil } // CreateBitmap creates a dirty bitmap. func (d *qemu) CreateBitmap(deviceNames []string, data api.StorageVolumeBitmapsPost) error { monitor, err := d.qmpConnect() if err != nil { return err } blockNames := []string{} for _, devName := range deviceNames { escapedDeviceName := linux.PathNameEncode(devName) nodeName := d.blockNodeName(escapedDeviceName) blockDevs, err := d.fetchBlockDeviceChain(monitor, nodeName) if err != nil { return fmt.Errorf("Failed fetching disk chain: %w", err) } blockNames = append(blockNames, blockDevs[len(blockDevs)-1]) } err = monitor.AddDirtyBitmap(blockNames, data.Name, data.Granularity, data.Persistent, data.Disabled) if err != nil { return err } return nil } // DeleteBitmap deletes a dirty bitmap. func (d *qemu) DeleteBitmap(deviceName string, bitmapName string) error { monitor, err := d.qmpConnect() if err != nil { return err } escapedDeviceName := linux.PathNameEncode(deviceName) nodeName := d.blockNodeName(escapedDeviceName) blockDevs, err := d.fetchBlockDeviceChain(monitor, nodeName) if err != nil { return fmt.Errorf("Failed fetching disk chain: %w", err) } blockName := blockDevs[len(blockDevs)-1] err = monitor.RemoveDirtyBitmap(blockName, bitmapName) if err != nil { return err } return nil } // GetBitmaps fetches dirty bitmaps. func (d *qemu) GetBitmaps(deviceName string) ([]api.StorageVolumeBitmap, error) { monitor, err := d.qmpConnect() if err != nil { return nil, err } escapedDeviceName := linux.PathNameEncode(deviceName) nodeName := d.blockNodeName(escapedDeviceName) blockDevs, err := d.fetchBlockDeviceChain(monitor, nodeName) if err != nil { return nil, fmt.Errorf("Failed fetching disk chain: %w", err) } blockName := blockDevs[len(blockDevs)-1] blocks, err := monitor.QueryBlock() if err != nil { return nil, err } result := []api.StorageVolumeBitmap{} for _, block := range blocks { if block.Inserted.NodeName == blockName { for _, bitmap := range block.Inserted.DirtyBitmaps { result = append(result, api.StorageVolumeBitmap{ Name: bitmap.Name, Count: bitmap.Count, Granularity: bitmap.Granularity, Recording: bitmap.Recording, Busy: bitmap.Busy, Persistent: bitmap.Persistent, Inconsistent: bitmap.Inconsistent, }) } return result, nil } } return nil, fmt.Errorf("Requested device not found") } incus-7.0.0/internal/server/instance/drivers/driver_qemu_bus.go000066400000000000000000000111731517523235500250000ustar00rootroot00000000000000package drivers import ( "fmt" "github.com/lxc/incus/v7/internal/server/instance/drivers/cfg" ) const ( busFunctionGroupNone = "" // Add a non multi-function port. busFunctionGroupGeneric = "generic" // Add multi-function port to generic group (used for internal devices). busFunctionGroup9p = "9p" // Add multi-function port to 9p group (used for 9p shares). busDevicePortPrefix = "qemu_pcie" // Prefix used for name of PCIe ports. ) type qemuBusEntry struct { bridgeDev int // Device number on the root bridge. bridgeFn int // Function number on the root bridge. dev string // Existing device name. fn int // Function number on the existing device. } type qemuBus struct { name string // Bus type. cfg *[]cfg.Section // pointer to Section slice. portNum int // Next available port/chassis on the bridge. devNum int // Next available device number on the bridge. rootPort *qemuBusEntry // Current root port. entries map[string]*qemuBusEntry // Map of qemuBusEntry for a particular shared device. } func (a *qemuBus) allocateRoot() *qemuBusEntry { if a.rootPort == nil { a.rootPort = &qemuBusEntry{ bridgeDev: a.devNum, } a.devNum++ } else { if a.rootPort.bridgeFn == 7 { a.rootPort.bridgeFn = 0 a.rootPort.bridgeDev = a.devNum a.devNum++ } else { a.rootPort.bridgeFn++ } } return a.rootPort } // allocate() does any needed port allocation and returns the bus name, // address and whether the device needs to be configured as multi-function. // // The multiFunctionGroup parameter allows for grouping devices together as one or more multi-function devices. // It automatically keeps track of the number of functions already used and will allocate new ports as needed. func (a *qemuBus) allocate(multiFunctionGroup string) (string, string, bool) { return a.allocateInternal(multiFunctionGroup, true) } // allocateDirect() works like allocate() but will directly attach the device to the root PCI bridge. // This prevents hotplug or hotremove of the device but is sometimes required for compatibility reasons. func (a *qemuBus) allocateDirect() (string, string, bool) { return a.allocateInternal(busFunctionGroupNone, false) } func (a *qemuBus) allocateInternal(multiFunctionGroup string, hotplug bool) (string, string, bool) { if a.name == "ccw" { return "", "", false } // Find a device multi-function group if specified. var p *qemuBusEntry if multiFunctionGroup != "" { var ok bool p, ok = a.entries[multiFunctionGroup] if ok { // Check if existing multi-function group is full. if p.fn == 7 { p.fn = 0 if a.name == "pci" { p.bridgeDev = a.devNum a.devNum++ } else if a.name == "pcie" { r := a.allocateRoot() p.bridgeDev = r.bridgeDev p.bridgeFn = r.bridgeFn } } else { p.fn++ } } else { // Create a new multi-function group. p = &qemuBusEntry{} if a.name == "pci" { p.bridgeDev = a.devNum a.devNum++ } else if a.name == "pcie" { r := a.allocateRoot() p.bridgeDev = r.bridgeDev p.bridgeFn = r.bridgeFn } a.entries[multiFunctionGroup] = p } } else { // Create a temporary single function group. p = &qemuBusEntry{} if a.name == "pci" || !hotplug { p.bridgeDev = a.devNum a.devNum++ } else if a.name == "pcie" { r := a.allocateRoot() p.bridgeDev = r.bridgeDev p.bridgeFn = r.bridgeFn } } // The first device added to a multi-function port needs to specify the multi-function feature. multi := p.fn == 0 && multiFunctionGroup != "" if a.name == "pci" || !hotplug { return fmt.Sprintf("%s.0", a.name), fmt.Sprintf("%x.%d", p.bridgeDev, p.fn), multi } if a.name == "pcie" { if p.fn == 0 { portName := fmt.Sprintf("%s%d", busDevicePortPrefix, a.portNum) pcieOpts := qemuPCIeOpts{ portName: portName, index: a.portNum, devAddr: fmt.Sprintf("%x.%d", p.bridgeDev, p.bridgeFn), // First root port added on a bridge bus address needs multi-function enabled. multifunction: p.bridgeFn == 0, } *a.cfg = append(*a.cfg, qemuPCIe(&pcieOpts)...) p.dev = portName a.portNum++ } return p.dev, fmt.Sprintf("00.%d", p.fn), multi } return "", "", false } // qemuNewBus instantiates a new qemu bus allocator. Accepts the type name of the bus and the qemu config builder // which it will use to write root port config entries too as ports are allocated. func qemuNewBus(name string, conf *[]cfg.Section) *qemuBus { a := &qemuBus{ name: name, cfg: conf, portNum: 0, // No PCIe ports are used in the default config. devNum: 1, // Address 0 is used by the DRAM controller. entries: map[string]*qemuBusEntry{}, } return a } incus-7.0.0/internal/server/instance/drivers/driver_qemu_cmd.go000066400000000000000000000053101517523235500247460ustar00rootroot00000000000000package drivers import ( "errors" "io" "strconv" "golang.org/x/sys/unix" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) // ErrExecDisconnected is returned when the guest disconnects the exec session. var ErrExecDisconnected = errors.New("Disconnected") // Cmd represents a running command for an Qemu VM. type qemuCmd struct { attachedChildPid int cmd incus.Operation dataDone chan bool controlSendCh chan api.InstanceExecControl controlResCh chan error cleanupFunc func() } // PID returns the attached child's process ID. func (c *qemuCmd) PID() int { return c.attachedChildPid } // Signal sends a signal to the command. func (c *qemuCmd) Signal(sig unix.Signal) error { command := api.InstanceExecControl{ Command: "signal", Signal: int(sig), } // Check handler hasn't finished. select { case <-c.dataDone: return errors.New("no such process") // Aligns with error returned from unix.Kill in lxc's Signal(). default: } c.controlSendCh <- command err := <-c.controlResCh if err != nil { return err } logger.Debugf(`Forwarded signal "%d" to the agent`, sig) return nil } // Wait for the command to end and returns its exit code and any error. func (c *qemuCmd) Wait() (int, error) { err := c.cmd.Wait() exitStatus := -1 opAPI := c.cmd.Get() if opAPI.Metadata != nil { exitStatusRaw, ok := opAPI.Metadata["return"].(float64) if ok { exitStatus = int(exitStatusRaw) // Convert special exit statuses into errors. switch exitStatus { case 127: err = ErrExecCommandNotFound case 126: err = ErrExecCommandNotExecutable } } } if err != nil { // Error of type EOF indicates the session ended unexpectedly, // so we inform the client of the disconnection with a more // descriptive message. if errors.Is(err, io.EOF) { return exitStatus, ErrExecDisconnected } return exitStatus, err } <-c.dataDone if c.cleanupFunc != nil { defer c.cleanupFunc() } return exitStatus, nil } // WindowResize resizes the running command's window. func (c *qemuCmd) WindowResize(fd, winchWidth, winchHeight int) error { command := api.InstanceExecControl{ Command: "window-resize", Args: map[string]string{ "width": strconv.Itoa(winchWidth), "height": strconv.Itoa(winchHeight), }, } // Check handler hasn't finished. select { case <-c.dataDone: return errors.New("no such process") // Aligns with error returned from unix.Kill in lxc's Signal(). default: } c.controlSendCh <- command err := <-c.controlResCh if err != nil { return err } logger.Debugf(`Forwarded window resize "%dx%d" to the agent`, winchWidth, winchHeight) return nil } incus-7.0.0/internal/server/instance/drivers/driver_qemu_config_override.go000066400000000000000000000116251517523235500273550ustar00rootroot00000000000000package drivers import ( "bufio" "fmt" "maps" "strconv" "strings" "github.com/lxc/incus/v7/internal/server/instance/drivers/cfg" ) // sectionContent represents the content of a section, without its name. type sectionContent struct { comment string entries map[string]string } // section represents a section pointing to its content. type section struct { name string content *sectionContent } // qemuRawCfgOverride generates a new QEMU configuration from an original one and an override entry. func qemuRawCfgOverride(conf []cfg.Section, confOverride string) ([]cfg.Section, error) { // We define a data structure optimized for lookup and insertion… indexedSections := map[string]map[int]*sectionContent{} // … and another one keeping insertion order. This saves us a few cycles at the expense of a few // bytes of RAM. orderedSections := []section{} // We first iterate over the original config to populate both our data structures. for _, sec := range conf { indexedSection, ok := indexedSections[sec.Name] if !ok { indexedSection = map[int]*sectionContent{} indexedSections[sec.Name] = indexedSection } // We perform a copy of the map to avoid modifying the original one entries := make(map[string]string, len(sec.Entries)) maps.Copy(entries, sec.Entries) content := §ionContent{ comment: sec.Comment, entries: entries, } indexedSection[len(indexedSection)] = content orderedSections = append(orderedSections, section{ name: sec.Name, content: content, }) } var currentIndexedSection map[int]*sectionContent var currentIndex int var currentSectionName string // We set the changed flag to true for our first iteration. changed := true scanner := bufio.NewScanner(strings.NewReader(confOverride)) // Then, we parse the override string. for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) length := len(line) if length == 0 { continue } if strings.HasPrefix(line, "[") { // Find closing `]` for section name. end := strings.IndexByte(line, ']') if end <= 1 { return nil, fmt.Errorf("Invalid section header (must be a section name enclosed in square brackets): %q", line) } // If a section is defined in the override but has no key, remove its entries. if !changed { (*currentIndexedSection[currentIndex]).entries = make(map[string]string) } changed = false currentSectionName = strings.TrimSpace(line[1:end]) currentIndex = 0 if length > end+1 { // Optional section index rest := line[end+1:] e := fmt.Errorf("Invalid section index (must be an integer enclosed in square brackets): %q", rest) if !strings.HasPrefix(rest, "[") || !strings.HasSuffix(rest, "]") { return nil, e } var err error currentIndex, err = strconv.Atoi(rest[1 : len(rest)-1]) if err != nil { return nil, e } } var ok bool currentIndexedSection, ok = indexedSections[currentSectionName] if !ok { // If there is no section with this name, we are creating a new section. currentIndexedSection = map[int]*sectionContent{} indexedSections[currentSectionName] = currentIndexedSection } _, ok = currentIndexedSection[currentIndex] if !ok { // If there is no section with this index, we are creating a new section. emptyContent := §ionContent{entries: make(map[string]string)} currentIndexedSection[currentIndex] = emptyContent indexedSections[currentSectionName] = currentIndexedSection orderedSections = append(orderedSections, section{ name: currentSectionName, content: emptyContent, }) } } else { if currentSectionName == "" { return nil, fmt.Errorf("Expected section header, got: %q", line) } eqLoc := strings.IndexByte(line, '=') if eqLoc < 1 { return nil, fmt.Errorf("Invalid property override line (must be `key=value`): %q", line) } key := strings.TrimSpace(line[:eqLoc]) value := strings.TrimSpace(line[eqLoc+1:]) if strings.HasPrefix(value, "\"") { // We are dealing with a quoted value. var err error value, err = strconv.Unquote(value) if err != nil { return nil, fmt.Errorf("Invalid quoted value: %q", value) } } changed = true if value == "" { // If the value associated to this key is empty, delete the key. delete((*currentIndexedSection[currentIndex]).entries, key) } else { (*currentIndexedSection[currentIndex]).entries[key] = value } } } // Same as above, if a section is defined in the override but has no key, remove its entries. if !changed { (*currentIndexedSection[currentIndex]).entries = make(map[string]string) } res := []cfg.Section{} for _, orderedSection := range orderedSections { if len((*orderedSection.content).entries) > 0 { res = append(res, cfg.Section{ Name: orderedSection.name, Comment: (*orderedSection.content).comment, Entries: (*orderedSection.content).entries, }) } } return res, nil } incus-7.0.0/internal/server/instance/drivers/driver_qemu_config_test.go000066400000000000000000001003151517523235500265100ustar00rootroot00000000000000package drivers import ( "regexp" "strings" "testing" "github.com/lxc/incus/v7/internal/server/instance/drivers/cfg" "github.com/lxc/incus/v7/shared/osarch" ) func TestQemuConfigTemplates(t *testing.T) { indent := regexp.MustCompile(`(?m)^[ \t]+`) normalize := func(s string) string { return strings.TrimSpace(indent.ReplaceAllString(s, "$1")) } runTest := func(expected string, sections []cfg.Section) { t.Run(expected, func(t *testing.T) { actual := normalize(qemuStringifyCfgPredictably(sections...).String()) expected = normalize(expected) if actual != expected { t.Errorf("Expected: %s. Got: %s", expected, actual) } }) } t.Run("qemu_base", func(t *testing.T) { testCases := []struct { opts qemuBaseOpts expected string }{{ qemuBaseOpts{architecture: osarch.ARCH_64BIT_INTEL_X86}, `# Machine [machine] accel = "kvm" graphics = "off" type = "q35" usb = "off" [global] driver = "ICH9-LPC" property = "disable_s3" value = "1" [global] driver = "ICH9-LPC" property = "disable_s4" value = "0" [boot-opts] strict = "on"`, }, { qemuBaseOpts{architecture: osarch.ARCH_64BIT_ARMV8_LITTLE_ENDIAN}, `# Machine [machine] accel = "kvm" gic-version = "max" graphics = "off" type = "virt" usb = "off" [boot-opts] strict = "on"`, }, { qemuBaseOpts{architecture: osarch.ARCH_64BIT_POWERPC_LITTLE_ENDIAN}, `# Machine [machine] accel = "kvm" cap-large-decr = "off" graphics = "off" type = "pseries" usb = "off" [boot-opts] strict = "on"`, }, { qemuBaseOpts{architecture: osarch.ARCH_64BIT_S390_BIG_ENDIAN}, `# Machine [machine] accel = "kvm" graphics = "off" type = "s390-ccw-virtio" usb = "off" [boot-opts] strict = "on"`, }} for _, tc := range testCases { runTest(tc.expected, qemuBase(&tc.opts)) } }) t.Run("qemu_memory", func(t *testing.T) { testCases := []struct { opts qemuMemoryOpts expected string }{{ qemuMemoryOpts{4096, 16384}, `# Memory [memory] maxmem = "16384M" size = "4096M" slots = "8"`, }, { qemuMemoryOpts{8192, 16384}, `# Memory [memory] maxmem = "16384M" size = "8192M" slots = "8"`, }} for _, tc := range testCases { runTest(tc.expected, qemuMemory(&tc.opts)) } }) t.Run("qemu_serial", func(t *testing.T) { testCases := []struct { opts qemuSerialOpts expected string }{{ qemuSerialOpts{qemuDevOpts{"pci", "qemu_pcie0", "00.5", false}, "qemu_serial-chardev", 32, true}, `# Virtual serial bus [device "dev-qemu_serial"] addr = "00.5" bus = "qemu_pcie0" driver = "virtio-serial-pci" # Serial identifier [chardev "qemu_serial-chardev"] backend = "ringbuf" size = "32B" [device "qemu_serial"] bus = "dev-qemu_serial.0" chardev = "qemu_serial-chardev" driver = "virtserialport" name = "org.linuxcontainers.incus" [device "qemu_serial_legacy"] bus = "dev-qemu_serial.0" driver = "virtserialport" name = "org.linuxcontainers.lxd" # Spice agent [chardev "qemu_spice-chardev"] backend = "spicevmc" name = "vdagent" [device "qemu_spice"] bus = "dev-qemu_serial.0" chardev = "qemu_spice-chardev" driver = "virtserialport" name = "com.redhat.spice.0" # Spice folder [chardev "qemu_spicedir-chardev"] backend = "spiceport" name = "org.spice-space.webdav.0" [device "qemu_spicedir"] bus = "dev-qemu_serial.0" chardev = "qemu_spicedir-chardev" driver = "virtserialport" name = "org.spice-space.webdav.0" `, }, { qemuSerialOpts{qemuDevOpts{"pci", "qemu_pcie0", "00.5", false}, "qemu_serial-chardev", 32, false}, `# Virtual serial bus [device "dev-qemu_serial"] addr = "00.5" bus = "qemu_pcie0" driver = "virtio-serial-pci" # Serial identifier [chardev "qemu_serial-chardev"] backend = "ringbuf" size = "32B" [device "qemu_serial"] bus = "dev-qemu_serial.0" chardev = "qemu_serial-chardev" driver = "virtserialport" name = "org.linuxcontainers.incus" [device "qemu_serial_legacy"] bus = "dev-qemu_serial.0" driver = "virtserialport" name = "org.linuxcontainers.lxd" `, }} for _, tc := range testCases { runTest(tc.expected, qemuSerial(&tc.opts)) } }) t.Run("qemu_pcie", func(t *testing.T) { testCases := []struct { opts qemuPCIeOpts expected string }{{ qemuPCIeOpts{"qemu_pcie0", 0, "1.0", true}, `[device "qemu_pcie0"] addr = "1.0" bus = "pcie.0" chassis = "0" driver = "pcie-root-port" multifunction = "on" `, }, { qemuPCIeOpts{"qemu_pcie2", 3, "2.0", false}, `[device "qemu_pcie2"] addr = "2.0" bus = "pcie.0" chassis = "3" driver = "pcie-root-port" `, }} for _, tc := range testCases { runTest(tc.expected, qemuPCIe(&tc.opts)) } }) t.Run("qemu_scsi", func(t *testing.T) { testCases := []struct { opts qemuDevOpts expected string }{{ qemuDevOpts{"pci", "qemu_pcie1", "00.0", false}, `# SCSI controller [device "qemu_scsi"] addr = "00.0" bus = "qemu_pcie1" driver = "virtio-scsi-pci" num_queues = "4" `, }, { qemuDevOpts{"ccw", "qemu_pcie2", "00.2", true}, `# SCSI controller [device "qemu_scsi"] driver = "virtio-scsi-ccw" multifunction = "on" num_queues = "4" `, }} for _, tc := range testCases { runTest(tc.expected, qemuSCSI(&tc.opts, 4)) } }) t.Run("qemu_balloon", func(t *testing.T) { testCases := []struct { opts qemuDevOpts expected string }{{ qemuDevOpts{"pcie", "qemu_pcie0", "00.0", true}, `# Balloon driver [device "qemu_balloon"] addr = "00.0" bus = "qemu_pcie0" driver = "virtio-balloon-pci" multifunction = "on" `, }, { qemuDevOpts{"ccw", "qemu_pcie0", "00.0", false}, `# Balloon driver [device "qemu_balloon"] driver = "virtio-balloon-ccw" `, }} for _, tc := range testCases { runTest(tc.expected, qemuBalloon(&tc.opts)) } }) t.Run("qemu_rng", func(t *testing.T) { testCases := []struct { opts qemuDevOpts expected string }{{ qemuDevOpts{"pci", "qemu_pcie0", "00.1", false}, `# Random number generator [object "qemu_rng"] filename = "/dev/urandom" qom-type = "rng-random" [device "dev-qemu_rng"] addr = "00.1" bus = "qemu_pcie0" driver = "virtio-rng-pci" rng = "qemu_rng" `, }, { qemuDevOpts{"ccw", "qemu_pcie0", "00.1", true}, `# Random number generator [object "qemu_rng"] filename = "/dev/urandom" qom-type = "rng-random" [device "dev-qemu_rng"] driver = "virtio-rng-ccw" multifunction = "on" rng = "qemu_rng" `, }} for _, tc := range testCases { runTest(tc.expected, qemuRNG(&tc.opts)) } }) t.Run("qemu_vsock", func(t *testing.T) { testCases := []struct { opts qemuVsockOpts expected string }{{ qemuVsockOpts{qemuDevOpts{"pcie", "qemu_pcie0", "00.4", true}, 4, 14}, `# Vsock [device "qemu_vsock"] addr = "00.4" bus = "qemu_pcie0" driver = "vhost-vsock-pci" guest-cid = "14" multifunction = "on" vhostfd = "4" `, }, { qemuVsockOpts{qemuDevOpts{"ccw", "qemu_pcie0", "00.4", false}, 4, 3}, `# Vsock [device "qemu_vsock"] driver = "vhost-vsock-ccw" guest-cid = "3" vhostfd = "4" `, }} for _, tc := range testCases { runTest(tc.expected, qemuVsock(&tc.opts)) } }) t.Run("qemu_gpu", func(t *testing.T) { testCases := []struct { opts qemuGpuOpts expected string }{{ qemuGpuOpts{dev: qemuDevOpts{"pci", "qemu_pcie3", "00.0", true}, architecture: osarch.ARCH_64BIT_INTEL_X86}, `# GPU [device "qemu_gpu"] addr = "00.0" bus = "qemu_pcie3" driver = "virtio-vga" multifunction = "on"`, }, { qemuGpuOpts{dev: qemuDevOpts{"pci", "qemu_pci3", "00.1", false}, architecture: osarch.ARCH_UNKNOWN}, `# GPU [device "qemu_gpu"] addr = "00.1" bus = "qemu_pci3" driver = "virtio-gpu-pci"`, }, { qemuGpuOpts{dev: qemuDevOpts{"ccw", "devBus", "busAddr", true}, architecture: osarch.ARCH_UNKNOWN}, `# GPU [device "qemu_gpu"] driver = "virtio-gpu-ccw" multifunction = "on"`, }, { qemuGpuOpts{dev: qemuDevOpts{"ccw", "devBus", "busAddr", false}, architecture: osarch.ARCH_64BIT_INTEL_X86}, `# GPU [device "qemu_gpu"] driver = "virtio-gpu-ccw"`, }} for _, tc := range testCases { runTest(tc.expected, qemuGPU(&tc.opts)) } }) t.Run("qemu_keyboard", func(t *testing.T) { testCases := []struct { opts qemuDevOpts expected string }{{ qemuDevOpts{"pci", "qemu_pcie3", "00.0", false}, `# Input [device "qemu_keyboard"] addr = "00.0" bus = "qemu_pcie3" driver = "virtio-keyboard-pci"`, }, { qemuDevOpts{"pcie", "qemu_pcie3", "00.0", true}, `# Input [device "qemu_keyboard"] addr = "00.0" bus = "qemu_pcie3" driver = "virtio-keyboard-pci" multifunction = "on"`, }, { qemuDevOpts{"ccw", "qemu_pcie3", "00.0", false}, `# Input [device "qemu_keyboard"] driver = "virtio-keyboard-ccw"`, }, { qemuDevOpts{"ccw", "qemu_pcie3", "00.0", true}, `# Input [device "qemu_keyboard"] driver = "virtio-keyboard-ccw" multifunction = "on"`, }} for _, tc := range testCases { runTest(tc.expected, qemuKeyboard(&tc.opts)) } }) t.Run("qemu_tablet", func(t *testing.T) { testCases := []struct { opts qemuDevOpts expected string }{{ qemuDevOpts{"pci", "qemu_pcie0", "00.3", true}, `# Input [device "qemu_tablet"] addr = "00.3" bus = "qemu_pcie0" driver = "virtio-tablet-pci" multifunction = "on" `, }, { qemuDevOpts{"ccw", "qemu_pcie0", "00.3", true}, `# Input [device "qemu_tablet"] driver = "virtio-tablet-ccw" multifunction = "on" `, }} for _, tc := range testCases { runTest(tc.expected, qemuTablet(&tc.opts)) } }) t.Run("qemu_cpu", func(t *testing.T) { testCases := []struct { opts qemuCPUOpts expected string }{{ qemuCPUOpts{ architecture: "x86_64", cpuCount: 8, cpuSockets: 1, cpuCores: 4, cpuThreads: 2, cpuNumaNodes: []uint64{}, cpuNumaMapping: []qemuNumaEntry{}, cpuNumaHostNodes: []uint64{}, hugepages: "", memory: 7629, }, `# CPU [smp-opts] cores = "4" cpus = "8" sockets = "1" threads = "2" [object "mem0"] qom-type = "memory-backend-memfd" share = "on" size = "7629M" [numa] memdev = "mem0" nodeid = "0" type = "node"`, }, { qemuCPUOpts{ architecture: "x86_64", cpuCount: 2, cpuSockets: 1, cpuCores: 2, cpuThreads: 1, cpuNumaNodes: []uint64{4, 5}, cpuNumaMapping: []qemuNumaEntry{ {node: 20, socket: 21, core: 22, thread: 23}, }, cpuNumaHostNodes: []uint64{8, 9, 10}, hugepages: "/hugepages/path", memory: 12000, }, `# CPU [smp-opts] cores = "2" cpus = "2" sockets = "1" threads = "1" [object "mem0"] discard-data = "on" host-nodes.0 = "8" mem-path = "/hugepages/path" policy = "bind" prealloc = "on" qom-type = "memory-backend-file" share = "on" size = "12000M" [numa] memdev = "mem0" nodeid = "0" type = "node" [object "mem1"] discard-data = "on" host-nodes.0 = "9" mem-path = "/hugepages/path" policy = "bind" prealloc = "on" qom-type = "memory-backend-file" share = "on" size = "12000M" [numa] memdev = "mem1" nodeid = "1" type = "node" [object "mem2"] discard-data = "on" host-nodes.0 = "10" mem-path = "/hugepages/path" policy = "bind" prealloc = "on" qom-type = "memory-backend-file" share = "on" size = "12000M" [numa] memdev = "mem2" nodeid = "2" type = "node" [numa] core-id = "22" node-id = "20" socket-id = "21" thread-id = "23" type = "cpu"`, }, { qemuCPUOpts{ architecture: "x86_64", cpuCount: 2, cpuSockets: 1, cpuCores: 2, cpuThreads: 1, cpuNumaNodes: []uint64{4, 5}, cpuNumaMapping: []qemuNumaEntry{ {node: 20, socket: 21, core: 22, thread: 23}, }, cpuNumaHostNodes: []uint64{8, 9, 10}, hugepages: "", memory: 12000, }, `# CPU [smp-opts] cores = "2" cpus = "2" sockets = "1" threads = "1" [object "mem0"] host-nodes.0 = "8" policy = "bind" qom-type = "memory-backend-memfd" size = "12000M" [numa] memdev = "mem0" nodeid = "0" type = "node" [object "mem1"] host-nodes.0 = "9" policy = "bind" qom-type = "memory-backend-memfd" size = "12000M" [numa] memdev = "mem1" nodeid = "1" type = "node" [object "mem2"] host-nodes.0 = "10" policy = "bind" qom-type = "memory-backend-memfd" size = "12000M" [numa] memdev = "mem2" nodeid = "2" type = "node" [numa] core-id = "22" node-id = "20" socket-id = "21" thread-id = "23" type = "cpu"`, }, { qemuCPUOpts{ architecture: "x86_64", cpuCount: 4, cpuSockets: 1, cpuCores: 4, cpuThreads: 1, cpuNumaNodes: []uint64{4, 5, 6}, cpuNumaMapping: []qemuNumaEntry{ {node: 11, socket: 12, core: 13, thread: 14}, {node: 20, socket: 21, core: 22, thread: 23}, }, cpuNumaHostNodes: []uint64{8, 9, 10}, hugepages: "", memory: 12000, }, `# CPU [smp-opts] cores = "4" cpus = "4" sockets = "1" threads = "1" [object "mem0"] host-nodes.0 = "8" policy = "bind" qom-type = "memory-backend-memfd" size = "12000M" [numa] memdev = "mem0" nodeid = "0" type = "node" [object "mem1"] host-nodes.0 = "9" policy = "bind" qom-type = "memory-backend-memfd" size = "12000M" [numa] memdev = "mem1" nodeid = "1" type = "node" [object "mem2"] host-nodes.0 = "10" policy = "bind" qom-type = "memory-backend-memfd" size = "12000M" [numa] memdev = "mem2" nodeid = "2" type = "node" [numa] core-id = "13" node-id = "11" socket-id = "12" thread-id = "14" type = "cpu" [numa] core-id = "22" node-id = "20" socket-id = "21" thread-id = "23" type = "cpu"`, }, { qemuCPUOpts{ architecture: "arm64", cpuCount: 4, cpuSockets: 1, cpuCores: 4, cpuThreads: 1, cpuNumaNodes: []uint64{4, 5, 6}, cpuNumaMapping: []qemuNumaEntry{ {node: 11, socket: 12, core: 13, thread: 14}, {node: 20, socket: 21, core: 22, thread: 23}, }, cpuNumaHostNodes: []uint64{8, 9, 10}, hugepages: "/hugepages", memory: 12000, }, `# CPU [smp-opts] cores = "4" cpus = "4" sockets = "1" threads = "1"`, }} for _, tc := range testCases { runTest(tc.expected, qemuCPU(&tc.opts, true)) } }) t.Run("qemu_control_socket", func(t *testing.T) { testCases := []struct { opts qemuControlSocketOpts expected string }{{ qemuControlSocketOpts{"/dev/shm/control-socket"}, `# Qemu control [chardev "monitor"] backend = "socket" path = "/dev/shm/control-socket" server = "on" wait = "off" [mon] chardev = "monitor" mode = "control"`, }} for _, tc := range testCases { runTest(tc.expected, qemuControlSocket(&tc.opts)) } }) t.Run("qemu_console", func(t *testing.T) { testCases := []struct { expected string }{{ `# Console [chardev "console"] backend = "ringbuf" size = "1048576"`, }} for _, tc := range testCases { runTest(tc.expected, qemuConsole()) } }) t.Run("qemu_drive_firmware", func(t *testing.T) { testCases := []struct { opts qemuDriveFirmwareOpts expected string }{{ qemuDriveFirmwareOpts{"/tmp/ovmf.fd", "/tmp/settings.fd"}, `# Firmware (read only) [drive] file = "/tmp/ovmf.fd" format = "raw" if = "pflash" readonly = "on" unit = "0" # Firmware settings (writable) [drive] file = "/tmp/settings.fd" format = "raw" if = "pflash" unit = "1"`, }} for _, tc := range testCases { runTest(tc.expected, qemuDriveFirmware(&tc.opts)) } }) t.Run("qemu_drive_config", func(t *testing.T) { testCases := []struct { opts qemuDriveConfigOpts expected string }{{ qemuDriveConfigOpts{ name: "config", dev: qemuDevOpts{"pci", "qemu_pcie0", "00.5", false}, path: "/var/9p", protocol: "9p", }, `# Shared config drive (9p) [fsdev "qemu_config"] fsdriver = "local" path = "/var/9p" readonly = "on" security_model = "none" [device "dev-qemu_config-drive-9p"] addr = "00.5" bus = "qemu_pcie0" driver = "virtio-9p-pci" fsdev = "qemu_config" mount_tag = "config"`, }, { qemuDriveConfigOpts{ name: "config", dev: qemuDevOpts{"pcie", "qemu_pcie1", "10.2", true}, path: "/dev/virtio-fs", protocol: "virtio-fs", }, `# Shared config drive (virtio-fs) [chardev "qemu_config"] backend = "socket" path = "/dev/virtio-fs" [device "dev-qemu_config-drive-virtio-fs"] addr = "10.2" bus = "qemu_pcie1" chardev = "qemu_config" driver = "vhost-user-fs-pci" multifunction = "on" tag = "config"`, }, { qemuDriveConfigOpts{ name: "config", dev: qemuDevOpts{"ccw", "qemu_pcie0", "00.0", false}, path: "/var/virtio-fs", protocol: "virtio-fs", }, `# Shared config drive (virtio-fs) [chardev "qemu_config"] backend = "socket" path = "/var/virtio-fs" [device "dev-qemu_config-drive-virtio-fs"] chardev = "qemu_config" driver = "vhost-user-fs-ccw" tag = "config"`, }, { qemuDriveConfigOpts{ name: "config", dev: qemuDevOpts{"ccw", "qemu_pcie0", "00.0", true}, path: "/dev/9p", protocol: "9p", }, `# Shared config drive (9p) [fsdev "qemu_config"] fsdriver = "local" path = "/dev/9p" readonly = "on" security_model = "none" [device "dev-qemu_config-drive-9p"] driver = "virtio-9p-ccw" fsdev = "qemu_config" mount_tag = "config" multifunction = "on"`, }, { qemuDriveConfigOpts{ dev: qemuDevOpts{"ccw", "qemu_pcie0", "00.0", true}, path: "/dev/9p", protocol: "invalid", }, ``, }} for _, tc := range testCases { runTest(tc.expected, qemuDriveConfig(&tc.opts)) } }) t.Run("qemu_drive_dir", func(t *testing.T) { testCases := []struct { opts qemuDriveDirOpts expected string }{{ qemuDriveDirOpts{ dev: qemuDevOpts{"pci", "qemu_pcie0", "00.5", true}, devName: "stub", mountTag: "mtag", path: "/var/9p", protocol: "9p", readonly: false, }, `# stub drive (9p) [fsdev "incus_stub"] fsdriver = "local" path = "/var/9p" readonly = "off" security_model = "passthrough" [device "dev-incus_stub-9p"] addr = "00.5" bus = "qemu_pcie0" driver = "virtio-9p-pci" fsdev = "incus_stub" mount_tag = "mtag" multifunction = "on"`, }, { qemuDriveDirOpts{ dev: qemuDevOpts{"pcie", "qemu_pcie1", "10.2", false}, path: "/dev/virtio", devName: "vfs", mountTag: "vtag", protocol: "virtio-fs", }, `# vfs drive (virtio-fs) [chardev "incus_vfs"] backend = "socket" path = "/dev/virtio" [device "dev-incus_vfs-virtio-fs"] addr = "10.2" bus = "qemu_pcie1" chardev = "incus_vfs" driver = "vhost-user-fs-pci" tag = "vtag"`, }, { qemuDriveDirOpts{ dev: qemuDevOpts{"ccw", "qemu_pcie0", "00.0", true}, path: "/dev/vio", devName: "vfs", mountTag: "vtag", protocol: "virtio-fs", }, `# vfs drive (virtio-fs) [chardev "incus_vfs"] backend = "socket" path = "/dev/vio" [device "dev-incus_vfs-virtio-fs"] chardev = "incus_vfs" driver = "vhost-user-fs-ccw" multifunction = "on" tag = "vtag"`, }, { qemuDriveDirOpts{ dev: qemuDevOpts{"ccw", "qemu_pcie0", "00.0", false}, devName: "stub2", mountTag: "mtag2", path: "/var/9p", protocol: "9p", readonly: true, }, `# stub2 drive (9p) [fsdev "incus_stub2"] fsdriver = "local" path = "/var/9p" readonly = "on" security_model = "passthrough" [device "dev-incus_stub2-9p"] driver = "virtio-9p-ccw" fsdev = "incus_stub2" mount_tag = "mtag2"`, }, { qemuDriveDirOpts{ dev: qemuDevOpts{"ccw", "qemu_pcie0", "00.0", true}, path: "/dev/9p", protocol: "invalid", }, ``, }} for _, tc := range testCases { runTest(tc.expected, qemuDriveDir(&tc.opts)) } }) t.Run("qemu_pci_physical", func(t *testing.T) { testCases := []struct { opts qemuPCIPhysicalOpts expected string }{{ qemuPCIPhysicalOpts{ dev: qemuDevOpts{"pci", "qemu_pcie1", "00.0", false}, devName: "physical-pci-name", pciSlotName: "host-slot", firmware: true, }, `# PCI card ("physical-pci-name" device) [device "dev-incus_physical-pci-name"] addr = "00.0" bus = "qemu_pcie1" driver = "vfio-pci" host = "host-slot"`, }, { qemuPCIPhysicalOpts{ dev: qemuDevOpts{"ccw", "qemu_pcie2", "00.2", true}, devName: "physical-ccw-name", pciSlotName: "host-slot-ccw", firmware: true, }, `# PCI card ("physical-ccw-name" device) [device "dev-incus_physical-ccw-name"] driver = "vfio-ccw" host = "host-slot-ccw" multifunction = "on"`, }} for _, tc := range testCases { runTest(tc.expected, qemuPCIPhysical(&tc.opts)) } }) t.Run("qemu_gpu_dev_physical", func(t *testing.T) { testCases := []struct { opts qemuGPUDevPhysicalOpts expected string }{{ qemuGPUDevPhysicalOpts{ dev: qemuDevOpts{"pci", "qemu_pcie1", "00.0", false}, devName: "gpu-name", pciSlotName: "gpu-slot", }, `# GPU card ("gpu-name" device) [device "dev-incus_gpu-name"] addr = "00.0" bus = "qemu_pcie1" driver = "vfio-pci" host = "gpu-slot"`, }, { qemuGPUDevPhysicalOpts{ dev: qemuDevOpts{"ccw", "qemu_pcie1", "00.0", true}, devName: "gpu-name", pciSlotName: "gpu-slot", vga: true, }, `# GPU card ("gpu-name" device) [device "dev-incus_gpu-name"] driver = "vfio-ccw" host = "gpu-slot" multifunction = "on" x-vga = "on"`, }, { qemuGPUDevPhysicalOpts{ dev: qemuDevOpts{"pci", "qemu_pcie1", "00.0", true}, devName: "vgpu-name", vgpu: "vgpu-dev", }, `# GPU card ("vgpu-name" device) [device "dev-incus_vgpu-name"] addr = "00.0" bus = "qemu_pcie1" driver = "vfio-pci" multifunction = "on" sysfsdev = "/sys/bus/mdev/devices/vgpu-dev"`, }} for _, tc := range testCases { runTest(tc.expected, qemuGPUDevPhysical(&tc.opts)) } }) t.Run("qemu_usb", func(t *testing.T) { testCases := []struct { opts qemuUSBOpts expected string }{{ qemuUSBOpts{ devBus: "qemu_pcie1", devAddr: "00.0", multifunction: true, ports: 3, spice: true, }, `# USB controller [device "qemu_usb"] addr = "00.0" bus = "qemu_pcie1" driver = "qemu-xhci" multifunction = "on" p2 = "3" p3 = "3" [chardev "qemu_spice-usb-chardev1"] backend = "spicevmc" name = "usbredir" [device "qemu_spice-usb1"] chardev = "qemu_spice-usb-chardev1" driver = "usb-redir" [chardev "qemu_spice-usb-chardev2"] backend = "spicevmc" name = "usbredir" [device "qemu_spice-usb2"] chardev = "qemu_spice-usb-chardev2" driver = "usb-redir" [chardev "qemu_spice-usb-chardev3"] backend = "spicevmc" name = "usbredir" [device "qemu_spice-usb3"] chardev = "qemu_spice-usb-chardev3" driver = "usb-redir"`, }, { qemuUSBOpts{ devBus: "qemu_pcie1", devAddr: "00.0", multifunction: true, ports: 3, spice: false, }, `# USB controller [device "qemu_usb"] addr = "00.0" bus = "qemu_pcie1" driver = "qemu-xhci" multifunction = "on" p2 = "3" p3 = "3"`, }} for _, tc := range testCases { runTest(tc.expected, qemuUSB(&tc.opts)) } }) t.Run("qemu_tpm", func(t *testing.T) { testCases := []struct { opts qemuTPMOpts expected string }{{ qemuTPMOpts{ devName: "myTpm", path: "/dev/my/tpm", driver: "tpm-crb", }, `[chardev "qemu_tpm-chardev_myTpm"] backend = "socket" path = "/dev/my/tpm" [tpmdev "qemu_tpm-tpmdev_myTpm"] chardev = "qemu_tpm-chardev_myTpm" type = "emulator" [device "dev-incus_myTpm"] driver = "tpm-crb" tpmdev = "qemu_tpm-tpmdev_myTpm"`, }} for _, tc := range testCases { runTest(tc.expected, qemuTPM(&tc.opts)) } }) t.Run("qemu_raw_cfg_override", func(t *testing.T) { conf := []cfg.Section{{ Name: "global", Entries: map[string]string{ "driver": "ICH9-LPC", "property": "disable_s3", "value": "1", }, }, { Name: "global", Entries: map[string]string{ "driver": "ICH9-LPC", "property": "disable_s4", "value": "0", }, }, { Name: "memory", Entries: map[string]string{"size": "1024M"}, }, { Name: `device "qemu_gpu"`, Entries: map[string]string{ "driver": "virtio-gpu-pci", "bus": "qemu_pci3", "addr": "00.0", }, }, { Name: `device "qemu_keyboard"`, Entries: map[string]string{ "driver": "virtio-keyboard-pci", "bus": "qemu_pci2", "addr": "00.1", }, }} testCases := []struct { cfg []cfg.Section overrides string expected string }{{ // override some keys conf, `[memory] size = "4096M" [device "qemu_gpu"] driver = "qxl-vga"`, `[global] driver = "ICH9-LPC" property = "disable_s3" value = "1" [global] driver = "ICH9-LPC" property = "disable_s4" value = "0" [memory] size = "4096M" [device "qemu_gpu"] addr = "00.0" bus = "qemu_pci3" driver = "qxl-vga" [device "qemu_keyboard"] addr = "00.1" bus = "qemu_pci2" driver = "virtio-keyboard-pci"`, }, { // delete some keys conf, `[device "qemu_keyboard"] driver = "" [device "qemu_gpu"] addr = ""`, `[global] driver = "ICH9-LPC" property = "disable_s3" value = "1" [global] driver = "ICH9-LPC" property = "disable_s4" value = "0" [memory] size = "1024M" [device "qemu_gpu"] bus = "qemu_pci3" driver = "virtio-gpu-pci" [device "qemu_keyboard"] addr = "00.1" bus = "qemu_pci2"`, }, { // add some keys to existing sections conf, `[memory] somekey = "somevalue" somekey2 = "somevalue2" somekey3 = "somevalue3" somekey4="somevalue4" [device "qemu_keyboard"] multifunction="off" [device "qemu_gpu"] multifunction= "on"`, `[global] driver = "ICH9-LPC" property = "disable_s3" value = "1" [global] driver = "ICH9-LPC" property = "disable_s4" value = "0" [memory] size = "1024M" somekey = "somevalue" somekey2 = "somevalue2" somekey3 = "somevalue3" somekey4 = "somevalue4" [device "qemu_gpu"] addr = "00.0" bus = "qemu_pci3" driver = "virtio-gpu-pci" multifunction = "on" [device "qemu_keyboard"] addr = "00.1" bus = "qemu_pci2" driver = "virtio-keyboard-pci" multifunction = "off"`, }, { // edit/add/remove conf, `[memory] size = "2048M" [device "qemu_gpu"] multifunction = "on" [device "qemu_keyboard"] addr = "" bus = ""`, `[global] driver = "ICH9-LPC" property = "disable_s3" value = "1" [global] driver = "ICH9-LPC" property = "disable_s4" value = "0" [memory] size = "2048M" [device "qemu_gpu"] addr = "00.0" bus = "qemu_pci3" driver = "virtio-gpu-pci" multifunction = "on" [device "qemu_keyboard"] driver = "virtio-keyboard-pci"`, }, { // delete sections conf, `[memory] [device "qemu_keyboard"] [global][1]`, `[global] driver = "ICH9-LPC" property = "disable_s3" value = "1" [device "qemu_gpu"] addr = "00.0" bus = "qemu_pci3" driver = "virtio-gpu-pci"`, }, { // add sections conf, `[object1] key1 = "value1" key2 = "value2" [object "2"] key3 = "value3" [object "3"] key4 = "value4" [object "2"] key5 = "value5" [object1] key6 = "value6"`, `[global] driver = "ICH9-LPC" property = "disable_s3" value = "1" [global] driver = "ICH9-LPC" property = "disable_s4" value = "0" [memory] size = "1024M" [device "qemu_gpu"] addr = "00.0" bus = "qemu_pci3" driver = "virtio-gpu-pci" [device "qemu_keyboard"] addr = "00.1" bus = "qemu_pci2" driver = "virtio-keyboard-pci" [object1] key1 = "value1" key2 = "value2" key6 = "value6" [object "2"] key3 = "value3" key5 = "value5" [object "3"] key4 = "value4"`, }, { // add/remove sections conf, `[device "qemu_gpu"] [object "2"] key3 = "value3" [object "3"] key4 = "value4" [object "2"] key5 = "value5"`, `[global] driver = "ICH9-LPC" property = "disable_s3" value = "1" [global] driver = "ICH9-LPC" property = "disable_s4" value = "0" [memory] size = "1024M" [device "qemu_keyboard"] addr = "00.1" bus = "qemu_pci2" driver = "virtio-keyboard-pci" [object "2"] key3 = "value3" key5 = "value5" [object "3"] key4 = "value4"`, }, { // edit keys of repeated sections conf, `[global][1] property ="disable_s1" [global] property ="disable_s5" [global][1] value = "" [global][0] somekey ="somevalue" [global][1] anotherkey = "anothervalue"`, `[global] driver = "ICH9-LPC" property = "disable_s5" somekey = "somevalue" value = "1" [global] anotherkey = "anothervalue" driver = "ICH9-LPC" property = "disable_s1" [memory] size = "1024M" [device "qemu_gpu"] addr = "00.0" bus = "qemu_pci3" driver = "virtio-gpu-pci" [device "qemu_keyboard"] addr = "00.1" bus = "qemu_pci2" driver = "virtio-keyboard-pci"`, }, { // create multiple sections with same name conf, // note that for appending new sections, all that matters is that // the index is higher than the existing indexes `[global][2] property = "new section" [global][2] value = "new value" [object][3] k1 = "v1" [object][3] k2 = "v2" [object][4] k3 = "v1" [object][4] k2 = "v2" [object][11] k11 = "v11"`, `[global] driver = "ICH9-LPC" property = "disable_s3" value = "1" [global] driver = "ICH9-LPC" property = "disable_s4" value = "0" [memory] size = "1024M" [device "qemu_gpu"] addr = "00.0" bus = "qemu_pci3" driver = "virtio-gpu-pci" [device "qemu_keyboard"] addr = "00.1" bus = "qemu_pci2" driver = "virtio-keyboard-pci" [global] property = "new section" value = "new value" [object] k1 = "v1" k2 = "v2" [object] k2 = "v2" k3 = "v1" [object] k11 = "v11"`, }, { // create multiple sections with same name, with decreasing indices conf, `[object][3] k1 = "v1" [object][3] k2 = "v2" [object][2] k3 = "v1" [object][2] k2 = "v2"`, `[global] driver = "ICH9-LPC" property = "disable_s3" value = "1" [global] driver = "ICH9-LPC" property = "disable_s4" value = "0" [memory] size = "1024M" [device "qemu_gpu"] addr = "00.0" bus = "qemu_pci3" driver = "virtio-gpu-pci" [device "qemu_keyboard"] addr = "00.1" bus = "qemu_pci2" driver = "virtio-keyboard-pci" [object] k1 = "v1" k2 = "v2" [object] k2 = "v2" k3 = "v1"`, }, { // mix all operations conf, `[memory] size = "8192M" [device "qemu_keyboard"] multifunction=on bus = [device "qemu_gpu"] [object "3"] key4 = " value4 " [object "2"] key3 = value3 [object "3"] key5 = "value5"`, `[global] driver = "ICH9-LPC" property = "disable_s3" value = "1" [global] driver = "ICH9-LPC" property = "disable_s4" value = "0" [memory] size = "8192M" [device "qemu_keyboard"] addr = "00.1" driver = "virtio-keyboard-pci" multifunction = "on" [object "3"] key4 = " value4 " key5 = "value5" [object "2"] key3 = "value3"`, }} for _, tc := range testCases { overridden, err := qemuRawCfgOverride(tc.cfg, tc.overrides) if err != nil { t.Error(err) } runTest(tc.expected, overridden) } }) } incus-7.0.0/internal/server/instance/drivers/driver_qemu_machine.go000066400000000000000000000237541517523235500256230ustar00rootroot00000000000000package drivers import ( "context" "database/sql" "errors" "fmt" "math" "slices" "strconv" "strings" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/instance/drivers/qemudefault" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) type qemuCPUTopology struct { Sockets int `json:"sockets"` Cores int `json:"cores"` Threads int `json:"threads"` vCPUs map[uint64]uint64 nodes map[uint64][]uint64 } // cpuTopology sets up the qemuCPUTopology struct based on configured CPU limits, host system and guest OS. func (d *qemu) cpuTopology() (*qemuCPUTopology, error) { topology := &qemuCPUTopology{} // Set default to 1 vCPU. limit := d.expandedConfig["limits.cpu"] if limit == "" { limit = "1" } // Check if pinned or floating. nrLimit, err := strconv.Atoi(limit) if err == nil { // We're not dealing with a pinned setup. topology.Sockets = 1 topology.Cores = nrLimit topology.Threads = 1 return topology, nil } // Get CPU topology. cpus, err := resources.GetCPU() if err != nil { return nil, err } // Expand the pins. pins, err := resources.ParseCpuset(limit) if err != nil { return nil, err } // Match tracking. vcpus := map[uint64]uint64{} sockets := map[uint64][]uint64{} cores := map[uint64][]uint64{} numaNodes := map[uint64][]uint64{} // Go through the physical CPUs looking for matches. i := uint64(0) for _, cpu := range cpus.Sockets { for _, core := range cpu.Cores { for _, thread := range core.Threads { for _, pin := range pins { if thread.ID == int64(pin) { // Found a matching CPU. vcpus[i] = uint64(pin) // Track cores per socket. _, ok := sockets[cpu.Socket] if !ok { sockets[cpu.Socket] = []uint64{} } if !slices.Contains(sockets[cpu.Socket], core.Core) { sockets[cpu.Socket] = append(sockets[cpu.Socket], core.Core) } // Track threads per core. _, ok = cores[core.Core] if !ok { cores[core.Core] = []uint64{} } if !slices.Contains(cores[core.Core], thread.Thread) { cores[core.Core] = append(cores[core.Core], thread.Thread) } // Record NUMA node for thread. _, ok = cores[core.Core] if !ok { numaNodes[thread.NUMANode] = []uint64{} } numaNodes[thread.NUMANode] = append(numaNodes[thread.NUMANode], i) i++ } } } } } // Confirm we're getting the expected number of CPUs. if len(pins) != len(vcpus) { return nil, fmt.Errorf("Unavailable CPUs requested: %s", limit) } // Validate the topology. valid := true nrSockets := 0 nrCores := 0 nrThreads := 0 // Confirm that there is no balancing inconsistencies. countCores := -1 for _, cores := range sockets { if countCores != -1 && len(cores) != countCores { valid = false break } countCores = len(cores) } countThreads := -1 for _, threads := range cores { if countThreads != -1 && len(threads) != countThreads { valid = false break } countThreads = len(threads) } // Check against double listing of CPU. if len(sockets)*countCores*countThreads != len(vcpus) { valid = false } // Build up the topology. if valid { // Valid topology. nrSockets = len(sockets) nrCores = countCores nrThreads = countThreads } else { d.logger.Warn("Instance uses a CPU pinning profile which doesn't match hardware layout") // Fallback on pretending everything are cores. nrSockets = 1 nrCores = len(vcpus) nrThreads = 1 } // Prepare struct. topology.Sockets = nrSockets topology.Cores = nrCores topology.Threads = nrThreads topology.vCPUs = vcpus topology.nodes = numaNodes return topology, nil } // cpuType generates the QEMU cpu flag based on the CPU topology, guest OS and host system. func (d *qemu) cpuType(bs *qemuBootState) (string, error) { // Determine additional CPU flags. cpuExtensions := []string{} if d.architecture == osarch.ARCH_64BIT_INTEL_X86 { // Use HyperV optimizations on x86_64 when not live-migrating. if !d.CanLiveMigrate() { // x86_64 can use hv_time to improve Windows guest performance. cpuExtensions = append(cpuExtensions, "hv_passthrough") } // x86_64 requires the use of topoext when SMT is used. if bs.CPUTopology.Threads > 1 { cpuExtensions = append(cpuExtensions, "topoext") } } cpuType := "host" // Handle CPU flags. if d.state.ServerClustered && d.CanLiveMigrate() { // Get the cluster group config. var groupConfig map[string]string err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get the group name. clusterGroupName := d.localConfig["volatile.cluster.group"] if clusterGroupName == "" { clusterGroupName = "default" } // Try to get the cluster group. group, err := dbCluster.GetClusterGroup(ctx, tx.Tx(), clusterGroupName) if err != nil && !errors.Is(err, sql.ErrNoRows) { return err } // Fallback to default group. if errors.Is(err, sql.ErrNoRows) && clusterGroupName != "default" { group, err = dbCluster.GetClusterGroup(ctx, tx.Tx(), "default") if err != nil { return err } } // Get the config. groupConfig, err = dbCluster.GetClusterGroupConfig(ctx, tx.Tx(), group.ID) if err != nil { return err } return nil }) if err != nil { return "", err } // Get the local architecture name. archName, err := osarch.ArchitectureName(d.architecture) if err != nil { return "", err } // Set the cpu type and extensions. groupConfigBaseline := fmt.Sprintf("instances.vm.cpu.%s.baseline", archName) groupConfigFlags := fmt.Sprintf("instances.vm.cpu.%s.flags", archName) if groupConfig[groupConfigBaseline] != "" { // Apply group config if present. cpuType = groupConfig[groupConfigBaseline] cpuExtensions = append(cpuExtensions, util.SplitNTrimSpace(groupConfig[groupConfigFlags], ",", -1, true)...) } else if d.architecture == osarch.ARCH_64BIT_INTEL_X86 { // Apply automatic handling if on x86_64. cpuFlags, err := GetClusterCPUFlags(context.TODO(), d.state, nil, archName) if err != nil { return "", err } cpuType = "kvm64" cpuExtensions = append(cpuExtensions, cpuFlags...) } } // Get the feature flags. info := DriverStatuses()[instancetype.VM].Info _, nested := info.Features["nested"] // Add +invtsc for fast TSC on x86 when not expected to be migratable and not nested. if !nested && d.architecture == osarch.ARCH_64BIT_INTEL_X86 && !d.CanLiveMigrate() { cpuExtensions = append(cpuExtensions, "migratable=no", "+invtsc") } if len(cpuExtensions) > 0 { cpuType += "," + strings.Join(cpuExtensions, ",") } return cpuType, nil } type qemuMemoryTopology struct { Base int64 `json:"base"` Max int64 `json:"max"` Extra []int64 `json:"extra"` } func (d *qemu) memoryTopology(bs *qemuBootState) (*qemuMemoryTopology, error) { // Configure memory limit. memSize := d.expandedConfig["limits.memory"] if memSize == "" { memSize = qemudefault.MemSize // Default if no memory limit specified. } memSizeBytes, err := ParseMemoryStr(memSize) if err != nil { return nil, fmt.Errorf("Invalid limits.memory value %q: %w", memSize, err) } // Set hotplug limits. // kvm64 has a limit of 39 bits for aarch64 and 40 bits on x86_64, so just limit everyone to 39 bits (512GB). // Other types we don't know so just don't allow hotplug. var maxMemoryBytes int64 cpuPhysBits := uint64(39) limitsMemoryHotplug := d.expandedConfig["limits.memory.hotplug"] memoryHotplugEnabled := !util.IsFalse(limitsMemoryHotplug) if d.GuestOS() == "freebsd" { memoryHotplugEnabled = false // We handle the empty value a bit differently here, as FreeBSD doesn’t have memory hotplug. if !util.IsFalseOrEmpty(limitsMemoryHotplug) { return nil, errors.New("FreeBSD doesn't support setting 'limits.memory.hotplug'") } } cpuType := strings.Split(bs.CPUType, ",")[0] if (cpuType == "host" || cpuType == "kvm64") && memoryHotplugEnabled { if !util.IsTrueOrEmpty(limitsMemoryHotplug) { maxMemoryBytes, err = units.ParseByteSizeString(limitsMemoryHotplug) if err != nil { return nil, err } if maxMemoryBytes < memSizeBytes { return nil, errors.New("'limits.memory.hotplug' value should be greater than or equal to 'limits.memory'") } } if maxMemoryBytes == 0 { // Attempt to get the CPU physical address space limits. cpu, err := resources.GetCPU() if err != nil { return nil, err } var lowestPhysBits uint64 for _, socket := range cpu.Sockets { if socket.AddressSizes != nil && (socket.AddressSizes.PhysicalBits < lowestPhysBits || lowestPhysBits == 0) { lowestPhysBits = socket.AddressSizes.PhysicalBits } } // If a physical address size was detected, either align it with the VM (CPU passthrough) or use it as an upper bound. if lowestPhysBits > 0 && (cpuType == "host" || lowestPhysBits < cpuPhysBits) { cpuPhysBits = lowestPhysBits } // Reduce the maximum by one bit to allow QEMU some headroom. cpuPhysBits-- // Calculate the max memory limit. maxMemoryBytes = int64(math.Pow(2, float64(cpuPhysBits))) // Cap to 1TB. if maxMemoryBytes > 1024*1024*1024*1024 { maxMemoryBytes = 1024 * 1024 * 1024 * 1024 } // On standalone systems, further cap to the system's total memory. if !d.state.ServerClustered { totalMemory, err := linux.DeviceTotalMemory() if err != nil { return nil, err } maxMemoryBytes = totalMemory } } // Allow the user to go past any expected limit. if maxMemoryBytes < memSizeBytes { maxMemoryBytes = memSizeBytes } } else { // Prevent memory hotplug. maxMemoryBytes = memSizeBytes } // Create the struct. memInfo := &qemuMemoryTopology{} memInfo.Base = memSizeBytes memInfo.Max = maxMemoryBytes return memInfo, nil } incus-7.0.0/internal/server/instance/drivers/driver_qemu_metrics.go000066400000000000000000000133331517523235500256550ustar00rootroot00000000000000package drivers import ( "bufio" "errors" "fmt" "os" "path/filepath" "strconv" "strings" "github.com/lxc/incus/v7/internal/server/instance/drivers/qemudefault" "github.com/lxc/incus/v7/internal/server/instance/drivers/qmp" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/metrics" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) func (d *qemu) getQemuMetrics() (*metrics.MetricSet, error) { // Connect to the monitor. monitor, err := d.qmpConnect() if err != nil { return nil, err } out := metrics.Metrics{} cpuStats, err := d.getQemuCPUMetrics(monitor) if err != nil { d.logger.Warn("Failed to get CPU metrics", logger.Ctx{"err": err}) } else { out.CPU = cpuStats } memoryStats, err := d.getQemuMemoryMetrics(monitor) if err != nil { d.logger.Warn("Failed to get memory metrics", logger.Ctx{"err": err}) } else { out.Memory = memoryStats } diskStats, err := d.getQemuDiskMetrics(monitor) if err != nil { d.logger.Warn("Failed to get disk metrics", logger.Ctx{"err": err}) } else { out.Disk = diskStats } networkState, err := d.getNetworkState() if err != nil { d.logger.Warn("Failed to get network metrics", logger.Ctx{"err": err}) } else { out.Network = make([]metrics.NetworkMetrics, 0, len(networkState)) for name, state := range networkState { out.Network = append(out.Network, metrics.NetworkMetrics{ Device: name, ReceiveBytes: uint64(state.Counters.BytesReceived), ReceiveDrop: uint64(state.Counters.PacketsDroppedInbound), ReceiveErrors: uint64(state.Counters.ErrorsReceived), ReceivePackets: uint64(state.Counters.PacketsReceived), TransmitBytes: uint64(state.Counters.BytesSent), TransmitDrop: uint64(state.Counters.PacketsDroppedOutbound), TransmitErrors: uint64(state.Counters.ErrorsSent), TransmitPackets: uint64(state.Counters.PacketsSent), }) } } metricSet, err := metrics.MetricSetFromAPI(&out, map[string]string{"project": d.project.Name, "name": d.name, "type": instancetype.VM.String()}) if err != nil { return nil, err } return metricSet, nil } func (d *qemu) getQemuDiskMetrics(monitor *qmp.Monitor) ([]metrics.DiskMetrics, error) { stats, err := monitor.GetBlockStats() if err != nil { return nil, err } out := make([]metrics.DiskMetrics, 0, len(stats)) for dev, stat := range stats { out = append(out, metrics.DiskMetrics{ Device: dev, ReadBytes: uint64(stat.BytesRead), ReadsCompleted: uint64(stat.ReadsCompleted), WrittenBytes: uint64(stat.BytesWritten), WritesCompleted: uint64(stat.WritesCompleted), }) } return out, nil } func (d *qemu) getQemuMemoryMetrics(monitor *qmp.Monitor) (metrics.MemoryMetrics, error) { out := metrics.MemoryMetrics{} // Get the QEMU PID. pid, err := d.pid() if err != nil { return out, err } // Extract current QEMU RSS. f, err := os.Open(fmt.Sprintf("/proc/%d/status", pid)) if err != nil { return out, err } defer func() { _ = f.Close() }() // Read it line by line. memRSS := int64(-1) scan := bufio.NewScanner(f) for scan.Scan() { line := scan.Text() // We only care about VmRSS. if !strings.HasPrefix(line, "VmRSS:") { continue } // Extract the before last (value) and last (unit) fields fields := strings.Split(line, "\t") value := strings.ReplaceAll(fields[len(fields)-1], " ", "") // Feed the result to units.ParseByteSizeString to get an int value valueBytes, err := units.ParseByteSizeString(value) if err != nil { return out, err } memRSS = valueBytes break } if memRSS == -1 { return out, errors.New("Couldn't find VM memory usage") } // Get max memory usage. memTotal := d.expandedConfig["limits.memory"] if memTotal == "" { memTotal = qemudefault.MemSize // Default if no memory limit specified. } memTotalBytes, err := units.ParseByteSizeString(memTotal) if err != nil { return out, err } // Handle host usage being larger than limit. if memRSS > memTotalBytes { memRSS = memTotalBytes } // Prepare struct. out = metrics.MemoryMetrics{ MemAvailableBytes: uint64(memTotalBytes - memRSS), MemFreeBytes: uint64(memTotalBytes - memRSS), MemTotalBytes: uint64(memTotalBytes), } return out, nil } func (d *qemu) getQemuCPUMetrics(monitor *qmp.Monitor) ([]metrics.CPUMetrics, error) { // Get CPU metrics threadIDs, err := monitor.GetCPUs() if err != nil { return nil, err } cpuMetrics := make([]metrics.CPUMetrics, 0, len(threadIDs)) for i, threadID := range threadIDs { pid, err := os.ReadFile(d.pidFilePath()) if err != nil { return nil, err } statFile := filepath.Join("/proc", strings.TrimSpace(string(pid)), "task", strconv.Itoa(threadID), "stat") if !util.PathExists(statFile) { continue } content, err := os.ReadFile(statFile) if err != nil { return nil, err } fields := strings.Fields(string(content)) stats := metrics.CPUMetrics{} stats.SecondsUser, err = strconv.ParseFloat(fields[13], 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[13], err) } guestTime, err := strconv.ParseFloat(fields[42], 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[42], err) } // According to proc(5), utime includes guest_time which therefore needs to be subtracted to get the correct time. stats.SecondsUser -= guestTime stats.SecondsUser /= 100 stats.SecondsSystem, err = strconv.ParseFloat(fields[14], 64) if err != nil { return nil, fmt.Errorf("Failed to parse %q: %w", fields[14], err) } stats.SecondsSystem /= 100 stats.CPU = fmt.Sprintf("cpu%d", i) cpuMetrics = append(cpuMetrics, stats) } return cpuMetrics, nil } incus-7.0.0/internal/server/instance/drivers/driver_qemu_stateful.go000066400000000000000000000042741517523235500260420ustar00rootroot00000000000000package drivers import ( "encoding/json" "errors" "github.com/lxc/incus/v7/shared/util" ) type qemuBootState struct { Version int `json:"version"` CPUType string `json:"cpu_type"` CPUTopology *qemuCPUTopology `json:"cpu_topology"` MachineType string `json:"machine_type"` MemoryTopology *qemuMemoryTopology `json:"memory_topology"` } var qemuBootStateVersion = 1 func (bs *qemuBootState) getSCSIQueues() int { if bs.Version < 1 { return 1 } if bs.CPUTopology != nil { return bs.CPUTopology.Sockets * bs.CPUTopology.Cores } return 1 } func (d *qemu) getBootState() (*qemuBootState, error) { // Prepare a new state struct. bs := qemuBootState{ Version: -1, } // Check if modern tracking is available. if d.localConfig["volatile.vm.boot_state"] != "" { err := json.Unmarshal([]byte(d.localConfig["volatile.vm.boot_state"]), &bs) if err != nil { return nil, err } } else { // Import legacy values if available. if d.localConfig["volatile.vm.definition"] != "" { bs.MachineType = d.localConfig["volatile.vm.definition"] } if d.localConfig["volatile.vm.hotplug.memory"] != "" { err := json.Unmarshal([]byte(d.localConfig["volatile.vm.hotplug.memory"]), bs.MemoryTopology) if err != nil { return nil, err } } } // Check if dealing with newer state than current. if bs.Version > qemuBootStateVersion { return nil, errors.New("Received VM state is newer than what this server supports") } return &bs, nil } func (d *qemu) saveBootState(bs qemuBootState) error { // Build a list of volatile changes. volatileSet := make(map[string]string) // Clear all keys. volatileSet["volatile.vm.boot_state"] = "" volatileSet["volatile.vm.definition"] = "" volatileSet["volatile.vm.hotplug.memory"] = "" // If stateful isn't enabled, we're done. // NOTE: Can't use CanLiveMigrate here as it itself checks the boot state. if !util.IsTrue(d.expandedConfig["migration.stateful"]) { return d.VolatileSet(volatileSet) } // Serialize and save the boot state struct. encoded, err := json.Marshal(bs) if err != nil { return err } volatileSet["volatile.vm.boot_state"] = string(encoded) // Save the changes. return d.VolatileSet(volatileSet) } incus-7.0.0/internal/server/instance/drivers/driver_qemu_templates.go000066400000000000000000000553061517523235500262130ustar00rootroot00000000000000package drivers import ( "fmt" "sort" "strings" "github.com/lxc/incus/v7/internal/server/instance/drivers/cfg" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/resources" ) func writeHeader(sb *strings.Builder, comment string, name string) { if comment != "" { fmt.Fprintf(sb, "# %s\n", comment) } fmt.Fprintf(sb, "[%s]\n", name) } func writeEntry(sb *strings.Builder, key string, value string) { if value != "" { fmt.Fprintf(sb, "%s = \"%s\"\n", key, value) } } func qemuStringifyCfg(conf ...cfg.Section) *strings.Builder { sb := &strings.Builder{} for _, section := range conf { writeHeader(sb, section.Comment, section.Name) for key, value := range section.Entries { writeEntry(sb, key, value) } sb.WriteString("\n") } return sb } // qemuStringifyCfgPredictably is only there to ensure tests reproducibility. func qemuStringifyCfgPredictably(conf ...cfg.Section) *strings.Builder { sb := &strings.Builder{} for _, section := range conf { writeHeader(sb, section.Comment, section.Name) keys := make([]string, 0, len(section.Entries)) for key := range section.Entries { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { writeEntry(sb, key, section.Entries[key]) } sb.WriteString("\n") } return sb } func qemuMachineType(architecture int) string { var machineType string switch architecture { case osarch.ARCH_64BIT_INTEL_X86: machineType = "q35" case osarch.ARCH_64BIT_ARMV8_LITTLE_ENDIAN: machineType = "virt" case osarch.ARCH_64BIT_POWERPC_LITTLE_ENDIAN: machineType = "pseries" case osarch.ARCH_64BIT_S390_BIG_ENDIAN: machineType = "s390-ccw-virtio" } return machineType } type qemuBaseOpts struct { architecture int iommu bool definition string } func qemuBase(opts *qemuBaseOpts) []cfg.Section { machineType := qemuMachineType(opts.architecture) gicVersion := "" capLargeDecr := "" switch opts.architecture { case osarch.ARCH_64BIT_ARMV8_LITTLE_ENDIAN: gicVersion = "max" case osarch.ARCH_64BIT_POWERPC_LITTLE_ENDIAN: capLargeDecr = "off" } if opts.definition != "" { machineType = opts.definition } sections := []cfg.Section{{ Name: "machine", Comment: "Machine", Entries: map[string]string{ "graphics": "off", "type": machineType, "gic-version": gicVersion, "cap-large-decr": capLargeDecr, "accel": "kvm", "usb": "off", }, }} if opts.iommu { sections[0].Entries["kernel-irqchip"] = "split" } if opts.architecture == osarch.ARCH_64BIT_INTEL_X86 { sections = append(sections, []cfg.Section{{ Name: "global", Entries: map[string]string{ "driver": "ICH9-LPC", "property": "disable_s3", "value": "1", }, }, { Name: "global", Entries: map[string]string{ "driver": "ICH9-LPC", "property": "disable_s4", "value": "0", }, }}...) } return append( sections, cfg.Section{ Name: "boot-opts", Entries: map[string]string{"strict": "on"}, }) } type qemuMemoryOpts struct { memSizeMB int64 maxSizeMB int64 } func qemuMemory(opts *qemuMemoryOpts) []cfg.Section { // Sets fixed values for slots and maxmem to support memory hotplug. section := cfg.Section{ Name: "memory", Comment: "Memory", Entries: map[string]string{ "size": fmt.Sprintf("%dM", opts.memSizeMB), "maxmem": fmt.Sprintf("%dM", opts.maxSizeMB), // Some systems hit odd errors when using more than 8 hotplug slots. // That's even with maxmem capped at the total system memory. "slots": "8", }, } // Disable hotplug when already at maximum. if section.Entries["size"] == section.Entries["maxmem"] { delete(section.Entries, "slots") } return []cfg.Section{section} } type qemuDevOpts struct { busName string devBus string devAddr string multifunction bool } type qemuDevEntriesOpts struct { dev qemuDevOpts pciName string ccwName string } func qemuDeviceEntries(opts *qemuDevEntriesOpts) map[string]string { entries := make(map[string]string) if opts.dev.busName == "pci" || opts.dev.busName == "pcie" { entries["driver"] = opts.pciName entries["bus"] = opts.dev.devBus entries["addr"] = opts.dev.devAddr } else if opts.dev.busName == "ccw" { entries["driver"] = opts.ccwName } if opts.dev.multifunction { entries["multifunction"] = "on" } return entries } type qemuSerialOpts struct { dev qemuDevOpts charDevName string ringbufSizeBytes int spice bool } func qemuSerial(opts *qemuSerialOpts) []cfg.Section { entriesOpts := qemuDevEntriesOpts{ dev: opts.dev, pciName: "virtio-serial-pci", ccwName: "virtio-serial-ccw", } sections := []cfg.Section{{ Name: `device "dev-qemu_serial"`, Comment: "Virtual serial bus", Entries: qemuDeviceEntries(&entriesOpts), }, { // Ring buffer used by the incus agent to report (write) its status to. Incus server will read // its content via QMP using "ringbuf-read" command. Name: fmt.Sprintf(`chardev "%s"`, opts.charDevName), Comment: "Serial identifier", Entries: map[string]string{ "backend": "ringbuf", "size": fmt.Sprintf("%dB", opts.ringbufSizeBytes), }, }, { // QEMU serial device connected to the above ring buffer. Name: `device "qemu_serial"`, Entries: map[string]string{ "driver": "virtserialport", "name": "org.linuxcontainers.incus", "chardev": opts.charDevName, "bus": "dev-qemu_serial.0", }, }, { // Legacy QEMU serial device, not connected to any ring buffer. Its purpose is to // create a symlink in /dev/virtio-ports/, triggering a udev rule to start incus-agent. // This is necessary for backward compatibility with virtual machines lacking the // updated incus-agent-loader package, which includes updated udev rules and a systemd unit. Name: `device "qemu_serial_legacy"`, Entries: map[string]string{ "driver": "virtserialport", "name": "org.linuxcontainers.lxd", "bus": "dev-qemu_serial.0", }, }} if opts.spice { sections = append(sections, []cfg.Section{{ Name: `chardev "qemu_spice-chardev"`, Comment: "Spice agent", Entries: map[string]string{ "backend": "spicevmc", "name": "vdagent", }, }, { Name: `device "qemu_spice"`, Entries: map[string]string{ "driver": "virtserialport", "name": "com.redhat.spice.0", "chardev": "qemu_spice-chardev", "bus": "dev-qemu_serial.0", }, }, { Name: `chardev "qemu_spicedir-chardev"`, Comment: "Spice folder", Entries: map[string]string{ "backend": "spiceport", "name": "org.spice-space.webdav.0", }, }, { Name: `device "qemu_spicedir"`, Entries: map[string]string{ "driver": "virtserialport", "name": "org.spice-space.webdav.0", "chardev": "qemu_spicedir-chardev", "bus": "dev-qemu_serial.0", }, }}...) } return sections } type qemuPCIeOpts struct { portName string index int devAddr string multifunction bool } func qemuPCIe(opts *qemuPCIeOpts) []cfg.Section { entries := map[string]string{ "driver": "pcie-root-port", "bus": "pcie.0", "addr": opts.devAddr, "chassis": fmt.Sprintf("%d", opts.index), } if opts.multifunction { entries["multifunction"] = "on" } return []cfg.Section{{ Name: fmt.Sprintf(`device "%s"`, opts.portName), Entries: entries, }} } func qemuSCSI(opts *qemuDevOpts, numQueues int) []cfg.Section { entries := qemuDeviceEntries(&qemuDevEntriesOpts{ dev: *opts, pciName: "virtio-scsi-pci", ccwName: "virtio-scsi-ccw", }) entries["num_queues"] = fmt.Sprintf("%d", numQueues) return []cfg.Section{{ Name: `device "qemu_scsi"`, Comment: "SCSI controller", Entries: entries, }} } func qemuBalloon(opts *qemuDevOpts) []cfg.Section { entriesOpts := qemuDevEntriesOpts{ dev: *opts, pciName: "virtio-balloon-pci", ccwName: "virtio-balloon-ccw", } return []cfg.Section{{ Name: `device "qemu_balloon"`, Comment: "Balloon driver", Entries: qemuDeviceEntries(&entriesOpts), }} } func qemuRNG(opts *qemuDevOpts) []cfg.Section { entries := qemuDeviceEntries(&qemuDevEntriesOpts{ dev: *opts, pciName: "virtio-rng-pci", ccwName: "virtio-rng-ccw", }) entries["rng"] = "qemu_rng" return []cfg.Section{{ Name: `object "qemu_rng"`, Comment: "Random number generator", Entries: map[string]string{ "qom-type": "rng-random", "filename": "/dev/urandom", }, }, { Name: `device "dev-qemu_rng"`, Entries: entries, }} } func qemuCoreInfo() []cfg.Section { return []cfg.Section{{ Name: `device "qemu_vmcoreinfo"`, Comment: "VM core info driver", Entries: map[string]string{"driver": "vmcoreinfo"}, }} } func qemuIOMMU(opts *qemuDevOpts, isWindows bool) []cfg.Section { if isWindows { return []cfg.Section{{ Name: `device "intel-iommu"`, Comment: "IOMMU driver", Entries: map[string]string{ "driver": "intel-iommu", "intremap": "on", "caching-mode": "on", }, }} } entriesOpts := qemuDevEntriesOpts{ dev: *opts, pciName: "virtio-iommu-pci", } return []cfg.Section{{ Name: `device "qemu_iommu"`, Comment: "IOMMU driver", Entries: qemuDeviceEntries(&entriesOpts), }} } type qemuSevOpts struct { cbitpos int reducedPhysBits int policy string dhCertFD string sessionDataFD string } func qemuSEV(opts *qemuSevOpts) []cfg.Section { entries := map[string]string{ "qom-type": "sev-guest", "cbitpos": fmt.Sprintf("%d", opts.cbitpos), "reduced-phys-bits": fmt.Sprintf("%d", opts.reducedPhysBits), "policy": opts.policy, } if opts.dhCertFD != "" && opts.sessionDataFD != "" { entries["dh-cert-file"] = opts.dhCertFD entries["session-file"] = opts.sessionDataFD } return []cfg.Section{{ Name: `object "sev0"`, Comment: "Secure Encrypted Virtualization", Entries: entries, }} } type qemuVsockOpts struct { dev qemuDevOpts vsockFD int vsockID uint32 } func qemuVsock(opts *qemuVsockOpts) []cfg.Section { entries := qemuDeviceEntries(&qemuDevEntriesOpts{ dev: opts.dev, pciName: "vhost-vsock-pci", ccwName: "vhost-vsock-ccw", }) entries["guest-cid"] = fmt.Sprintf("%d", opts.vsockID) entries["vhostfd"] = fmt.Sprintf("%d", opts.vsockFD) return []cfg.Section{{ Name: `device "qemu_vsock"`, Comment: "Vsock", Entries: entries, }} } type qemuGpuOpts struct { dev qemuDevOpts architecture int } func qemuGPU(opts *qemuGpuOpts) []cfg.Section { var pciName string if opts.architecture == osarch.ARCH_64BIT_INTEL_X86 { pciName = "virtio-vga" } else { pciName = "virtio-gpu-pci" } entriesOpts := qemuDevEntriesOpts{ dev: opts.dev, pciName: pciName, ccwName: "virtio-gpu-ccw", } return []cfg.Section{{ Name: `device "qemu_gpu"`, Comment: "GPU", Entries: qemuDeviceEntries(&entriesOpts), }} } func qemuKeyboard(opts *qemuDevOpts) []cfg.Section { entriesOpts := qemuDevEntriesOpts{ dev: *opts, pciName: "virtio-keyboard-pci", ccwName: "virtio-keyboard-ccw", } return []cfg.Section{{ Name: `device "qemu_keyboard"`, Comment: "Input", Entries: qemuDeviceEntries(&entriesOpts), }} } func qemuTablet(opts *qemuDevOpts) []cfg.Section { entriesOpts := qemuDevEntriesOpts{ dev: *opts, pciName: "virtio-tablet-pci", ccwName: "virtio-tablet-ccw", } return []cfg.Section{{ Name: `device "qemu_tablet"`, Comment: "Input", Entries: qemuDeviceEntries(&entriesOpts), }} } type qemuNumaEntry struct { node uint64 socket uint64 core uint64 thread uint64 } type qemuCPUOpts struct { architecture string cpuCount int cpuRequested int cpuSockets int cpuCores int cpuThreads int cpuNumaNodes []uint64 cpuNumaMapping []qemuNumaEntry cpuNumaHostNodes []uint64 hugepages string memory int64 memoryHostNodes []int64 } func qemuCPUNumaHostNode(opts *qemuCPUOpts, index int) []cfg.Section { entries := make(map[string]string) if opts.hugepages != "" { entries["qom-type"] = "memory-backend-file" entries["mem-path"] = opts.hugepages entries["prealloc"] = "on" entries["discard-data"] = "on" } else { entries["qom-type"] = "memory-backend-memfd" } entries["size"] = fmt.Sprintf("%dM", opts.memory) return []cfg.Section{{ Name: fmt.Sprintf("object \"mem%d\"", index), Entries: entries, }, { Name: "numa", Entries: map[string]string{ "type": "node", "nodeid": fmt.Sprintf("%d", index), "memdev": fmt.Sprintf("mem%d", index), }, }} } func qemuCPU(opts *qemuCPUOpts, pinning bool) []cfg.Section { entries := map[string]string{"cpus": fmt.Sprintf("%d", opts.cpuCount)} if pinning { entries["sockets"] = fmt.Sprintf("%d", opts.cpuSockets) entries["cores"] = fmt.Sprintf("%d", opts.cpuCores) entries["threads"] = fmt.Sprintf("%d", opts.cpuThreads) } else { cpu, err := resources.GetCPU() if err != nil { return nil } // Cap the max number of CPUs to 64 unless directly assigned more. maxCpus := 64 if int(cpu.Total) < maxCpus { maxCpus = int(cpu.Total) } else if opts.cpuRequested > maxCpus { maxCpus = opts.cpuRequested } else if opts.cpuCount > maxCpus { maxCpus = opts.cpuCount } entries["maxcpus"] = fmt.Sprintf("%d", maxCpus) } sections := []cfg.Section{{ Name: "smp-opts", Comment: "CPU", Entries: entries, }} if opts.architecture != "x86_64" { return sections } if len(opts.cpuNumaHostNodes) == 0 { // Add one mem and one numa sections with index 0. numaHostNode := qemuCPUNumaHostNode(opts, 0) // Unconditionally append "share = "on" to the [object "mem0"] section numaHostNode[0].Entries["share"] = "on" // If NUMA memory restrictions are set, apply them. if len(opts.memoryHostNodes) > 0 { numaHostNode[0].Entries["policy"] = "bind" for index, element := range opts.memoryHostNodes { hostNodesKey := fmt.Sprintf("host-nodes.%d", index) numaHostNode[0].Entries[hostNodesKey] = fmt.Sprintf("%d", element) } } return append(sections, numaHostNode...) } for index, element := range opts.cpuNumaHostNodes { numaHostNode := qemuCPUNumaHostNode(opts, index) numaHostNode[0].Entries["policy"] = "bind" if opts.hugepages != "" { // append share = "on" only if hugepages is set numaHostNode[0].Entries["share"] = "on" } numaHostNode[0].Entries["host-nodes.0"] = fmt.Sprintf("%d", element) sections = append(sections, numaHostNode...) } for _, numa := range opts.cpuNumaMapping { sections = append(sections, cfg.Section{ Name: "numa", Entries: map[string]string{ "type": "cpu", "node-id": fmt.Sprintf("%d", numa.node), "socket-id": fmt.Sprintf("%d", numa.socket), "core-id": fmt.Sprintf("%d", numa.core), "thread-id": fmt.Sprintf("%d", numa.thread), }, }) } return sections } type qemuControlSocketOpts struct { path string } func qemuControlSocket(opts *qemuControlSocketOpts) []cfg.Section { return []cfg.Section{{ Name: `chardev "monitor"`, Comment: "Qemu control", Entries: map[string]string{ "backend": "socket", "path": opts.path, "server": "on", "wait": "off", }, }, { Name: "mon", Entries: map[string]string{ "chardev": "monitor", "mode": "control", }, }} } func qemuConsole() []cfg.Section { return []cfg.Section{{ Name: `chardev "console"`, Comment: "Console", Entries: map[string]string{ "backend": "ringbuf", "size": "1048576", }, }} } type qemuDriveFirmwareOpts struct { roPath string nvramPath string } func qemuDriveFirmware(opts *qemuDriveFirmwareOpts) []cfg.Section { return []cfg.Section{{ Name: "drive", Comment: "Firmware (read only)", Entries: map[string]string{ "file": opts.roPath, "if": "pflash", "format": "raw", "unit": "0", "readonly": "on", }, }, { Name: "drive", Comment: "Firmware settings (writable)", Entries: map[string]string{ "file": opts.nvramPath, "if": "pflash", "format": "raw", "unit": "1", }, }} } type qemuHostDriveOpts struct { dev qemuDevOpts name string nameSuffix string comment string fsdriver string mountTag string securityModel string path string sockFd string readonly bool protocol string } func qemuHostDrive(opts *qemuHostDriveOpts) []cfg.Section { var driveSection cfg.Section var entries map[string]string deviceOpts := qemuDevEntriesOpts{dev: opts.dev} if opts.protocol == "9p" { var readonly string if opts.readonly { readonly = "on" } else { readonly = "off" } driveSection = cfg.Section{ Name: fmt.Sprintf(`fsdev "%s"`, opts.name), Comment: opts.comment, Entries: map[string]string{ "fsdriver": opts.fsdriver, "sock_fd": opts.sockFd, "security_model": opts.securityModel, "readonly": readonly, "path": opts.path, }, } deviceOpts.pciName = "virtio-9p-pci" deviceOpts.ccwName = "virtio-9p-ccw" entries = qemuDeviceEntries(&deviceOpts) entries["mount_tag"] = opts.mountTag entries["fsdev"] = opts.name } else if opts.protocol == "virtio-fs" { driveSection = cfg.Section{ Name: fmt.Sprintf(`chardev "%s"`, opts.name), Comment: opts.comment, Entries: map[string]string{ "backend": "socket", "path": opts.path, }, } deviceOpts.pciName = "vhost-user-fs-pci" deviceOpts.ccwName = "vhost-user-fs-ccw" entries = qemuDeviceEntries(&deviceOpts) entries["tag"] = opts.mountTag entries["chardev"] = opts.name } else { return []cfg.Section{} } return []cfg.Section{ driveSection, { Name: fmt.Sprintf(`device "dev-%s%s-%s"`, opts.name, opts.nameSuffix, opts.protocol), Entries: entries, }, } } type qemuDriveConfigOpts struct { name string dev qemuDevOpts protocol string path string } func qemuDriveConfig(opts *qemuDriveConfigOpts) []cfg.Section { return qemuHostDrive(&qemuHostDriveOpts{ dev: opts.dev, // Devices use "qemu_" prefix indicating that this is a internally named device. name: fmt.Sprintf("qemu_%s", opts.name), nameSuffix: "-drive", comment: fmt.Sprintf("Shared %s drive (%s)", opts.name, opts.protocol), mountTag: opts.name, protocol: opts.protocol, fsdriver: "local", readonly: true, securityModel: "none", path: opts.path, }) } type qemuDriveDirOpts struct { dev qemuDevOpts devName string mountTag string path string protocol string readonly bool } func qemuDriveDir(opts *qemuDriveDirOpts) []cfg.Section { return qemuHostDrive(&qemuHostDriveOpts{ dev: opts.dev, name: fmt.Sprintf("incus_%s", opts.devName), comment: fmt.Sprintf("%s drive (%s)", opts.devName, opts.protocol), mountTag: opts.mountTag, protocol: opts.protocol, fsdriver: "local", readonly: opts.readonly, path: opts.path, securityModel: "passthrough", }) } type qemuPCIPhysicalOpts struct { dev qemuDevOpts devName string pciSlotName string firmware bool } func qemuPCIPhysical(opts *qemuPCIPhysicalOpts) []cfg.Section { deviceOpts := qemuDevEntriesOpts{ dev: opts.dev, pciName: "vfio-pci", ccwName: "vfio-ccw", } entries := qemuDeviceEntries(&deviceOpts) entries["host"] = opts.pciSlotName if !opts.firmware { entries["rombar"] = "0" } return []cfg.Section{{ Name: fmt.Sprintf(`device "%s%s"`, qemuDeviceIDPrefix, opts.devName), Comment: fmt.Sprintf(`PCI card ("%s" device)`, opts.devName), Entries: entries, }} } type qemuGPUDevPhysicalOpts struct { dev qemuDevOpts devName string pciSlotName string vgpu string vga bool } func qemuGPUDevPhysical(opts *qemuGPUDevPhysicalOpts) []cfg.Section { deviceOpts := qemuDevEntriesOpts{ dev: opts.dev, pciName: "vfio-pci", ccwName: "vfio-ccw", } entries := qemuDeviceEntries(&deviceOpts) if opts.vgpu != "" { sysfsdev := fmt.Sprintf("/sys/bus/mdev/devices/%s", opts.vgpu) entries["sysfsdev"] = sysfsdev } else { entries["host"] = opts.pciSlotName } if opts.vga { entries["x-vga"] = "on" } return []cfg.Section{{ Name: fmt.Sprintf(`device "%s%s"`, qemuDeviceIDPrefix, opts.devName), Comment: fmt.Sprintf(`GPU card ("%s" device)`, opts.devName), Entries: entries, }} } type qemuUSBOpts struct { devBus string devAddr string multifunction bool ports int spice bool } func qemuUSB(opts *qemuUSBOpts) []cfg.Section { deviceOpts := qemuDevEntriesOpts{ dev: qemuDevOpts{ busName: "pci", devAddr: opts.devAddr, devBus: opts.devBus, multifunction: opts.multifunction, }, pciName: "qemu-xhci", } entries := qemuDeviceEntries(&deviceOpts) entries["p2"] = fmt.Sprintf("%d", opts.ports) entries["p3"] = fmt.Sprintf("%d", opts.ports) sections := []cfg.Section{{ Name: `device "qemu_usb"`, Comment: "USB controller", Entries: entries, }} if opts.spice { for i := 1; i <= 3; i++ { chardev := fmt.Sprintf("qemu_spice-usb-chardev%d", i) sections = append(sections, []cfg.Section{{ Name: fmt.Sprintf(`chardev "%s"`, chardev), Entries: map[string]string{ "backend": "spicevmc", "name": "usbredir", }, }, { Name: fmt.Sprintf(`device "qemu_spice-usb%d"`, i), Entries: map[string]string{ "driver": "usb-redir", "chardev": chardev, }, }}...) } } return sections } type qemuAudioOpts struct { dev qemuDevOpts spice bool } func qemuAudio(opts *qemuAudioOpts) []cfg.Section { entries := qemuDeviceEntries(&qemuDevEntriesOpts{ dev: opts.dev, pciName: "virtio-sound-pci", }) audioDriver := "none" if opts.spice { audioDriver = "spice" } entries["audiodev"] = "qemu_sound-audiodev" return []cfg.Section{{ Name: `audiodev "qemu_sound-audiodev"`, Comment: "Sound device", Entries: map[string]string{ "driver": audioDriver, }, }, { Name: `device "qemu_sound"`, Entries: entries, }} } type qemuTPMOpts struct { devName string path string driver string } func qemuTPM(opts *qemuTPMOpts) []cfg.Section { chardev := fmt.Sprintf("qemu_tpm-chardev_%s", opts.devName) tpmdev := fmt.Sprintf("qemu_tpm-tpmdev_%s", opts.devName) return []cfg.Section{{ Name: fmt.Sprintf(`chardev "%s"`, chardev), Entries: map[string]string{ "backend": "socket", "path": opts.path, }, }, { Name: fmt.Sprintf(`tpmdev "%s"`, tpmdev), Entries: map[string]string{ "type": "emulator", "chardev": chardev, }, }, { Name: fmt.Sprintf(`device "%s%s"`, qemuDeviceIDPrefix, opts.devName), Entries: map[string]string{ "driver": opts.driver, "tpmdev": tpmdev, }, }} } type qemuVmgenIDOpts struct { guid string } func qemuVmgen(opts *qemuVmgenIDOpts) []cfg.Section { return []cfg.Section{{ Name: `device "vmgenid0"`, Comment: "VM Generation ID", Entries: map[string]string{ "driver": "vmgenid", "guid": opts.guid, }, }} } incus-7.0.0/internal/server/instance/drivers/edk2/000077500000000000000000000000001517523235500221005ustar00rootroot00000000000000incus-7.0.0/internal/server/instance/drivers/edk2/driver_edk2.go000066400000000000000000000167761517523235500246500ustar00rootroot00000000000000package edk2 import ( "fmt" "os" "path/filepath" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/util" ) // FirmwarePair represents a combination of firmware code (Code) and storage (Vars). type FirmwarePair struct { Code string Vars string } // Installation represents a set of available firmware at a given location on the system. type Installation struct { Path string Usage map[FirmwareUsage][]FirmwarePair } // FirmwareUsage represents the situation in which a given firmware file will be used. type FirmwareUsage int const ( // GENERIC is a generic EDK2 firmware. GENERIC FirmwareUsage = iota // SECUREBOOT is a UEFI Secure Boot enabled firmware. SECUREBOOT // CSM is a firmware with the UEFI Compatibility Support Module enabled to boot BIOS-only operating systems. CSM ) var architectureInstallations = map[int][]Installation{ osarch.ARCH_64BIT_INTEL_X86: {{ Path: "/usr/share/OVMF", Usage: map[FirmwareUsage][]FirmwarePair{ GENERIC: { {Code: "OVMF_CODE_4M.secboot.fd", Vars: "OVMF_VARS_4M.fd"}, {Code: "OVMF_CODE.4MB.fd", Vars: "OVMF_VARS.4MB.fd"}, {Code: "OVMF_CODE_4M.fd", Vars: "OVMF_VARS_4M.fd"}, {Code: "OVMF_CODE.4m.fd", Vars: "OVMF_VARS.4m.fd"}, {Code: "OVMF_CODE.2MB.fd", Vars: "OVMF_VARS.2MB.fd"}, {Code: "OVMF_CODE.fd", Vars: "OVMF_VARS.fd"}, {Code: "OVMF_CODE.fd", Vars: "qemu.nvram"}, }, SECUREBOOT: { {Code: "OVMF_CODE.4MB.fd", Vars: "OVMF_VARS.4MB.ms.fd"}, {Code: "OVMF_CODE_4M.ms.fd", Vars: "OVMF_VARS_4M.ms.fd"}, {Code: "OVMF_CODE_4M.secboot.fd", Vars: "OVMF_VARS_4M.secboot.fd"}, {Code: "OVMF_CODE.secboot.4m.fd", Vars: "OVMF_VARS.4m.fd"}, {Code: "OVMF_CODE.secboot.fd", Vars: "OVMF_VARS.secboot.fd"}, {Code: "OVMF_CODE.secboot.fd", Vars: "OVMF_VARS.fd"}, {Code: "OVMF_CODE.2MB.fd", Vars: "OVMF_VARS.2MB.ms.fd"}, {Code: "OVMF_CODE.fd", Vars: "OVMF_VARS.ms.fd"}, {Code: "OVMF_CODE.fd", Vars: "qemu.nvram"}, }, CSM: { {Code: "seabios.bin", Vars: "seabios.bin"}, {Code: "OVMF_CODE.4MB.CSM.fd", Vars: "OVMF_VARS.4MB.CSM.fd"}, {Code: "OVMF_CODE.csm.4m.fd", Vars: "OVMF_VARS.4m.fd"}, {Code: "OVMF_CODE.2MB.CSM.fd", Vars: "OVMF_VARS.2MB.CSM.fd"}, {Code: "OVMF_CODE.CSM.fd", Vars: "OVMF_VARS.CSM.fd"}, {Code: "OVMF_CODE.csm.fd", Vars: "OVMF_VARS.fd"}, }, }, }, { Path: "/usr/share/qemu", Usage: map[FirmwareUsage][]FirmwarePair{ GENERIC: { {Code: "ovmf-x86_64-4m-code.bin", Vars: "ovmf-x86_64-4m-vars.bin"}, {Code: "ovmf-x86_64.bin", Vars: "ovmf-x86_64-code.bin"}, {Code: "edk2-x86_64-code.fd", Vars: "edk2-i386-vars.fd"}, }, SECUREBOOT: { {Code: "ovmf-x86_64-ms-4m-code.bin", Vars: "ovmf-x86_64-ms-4m-vars.bin"}, {Code: "ovmf-x86_64-ms-code.bin", Vars: "ovmf-x86_64-ms-vars.bin"}, {Code: "edk2-x86_64-secure-code.fd", Vars: "edk2-i386-vars.fd"}, }, CSM: { {Code: "seabios.bin", Vars: "seabios.bin"}, {Code: "bios.bin", Vars: "bios.bin"}, }, }, }, { Path: "/usr/share/seabios", Usage: map[FirmwareUsage][]FirmwarePair{ CSM: { {Code: "bios.bin", Vars: "bios.bin"}, {Code: "bios-256k.bin", Vars: "bios-256k.bin"}, }, }, }, { Path: "/usr/share/edk2/x64", Usage: map[FirmwareUsage][]FirmwarePair{ GENERIC: { {Code: "OVMF_CODE.4m.fd", Vars: "OVMF_VARS.4m.fd"}, {Code: "OVMF_CODE.fd", Vars: "OVMF_VARS.fd"}, }, SECUREBOOT: { {Code: "OVMF_CODE.secure.4m.fd", Vars: "OVMF_VARS.4m.fd"}, {Code: "OVMF_CODE.secure.fd", Vars: "OVMF_VARS.fd"}, }, }, }, { Path: "/usr/share/OVMF/x64", Usage: map[FirmwareUsage][]FirmwarePair{ GENERIC: { {Code: "OVMF_CODE.4m.fd", Vars: "OVMF_VARS.4m.fd"}, {Code: "OVMF_CODE.fd", Vars: "OVMF_VARS.fd"}, }, CSM: { {Code: "OVMF_CODE.csm.4m.fd", Vars: "OVMF_VARS.4m.fd"}, {Code: "OVMF_CODE.csm.fd", Vars: "OVMF_VARS.fd"}, }, SECUREBOOT: { {Code: "OVMF_CODE.secboot.4m.fd", Vars: "OVMF_VARS.4m.fd"}, {Code: "OVMF_CODE.secboot.fd", Vars: "OVMF_VARS.fd"}, }, }, }}, osarch.ARCH_64BIT_ARMV8_LITTLE_ENDIAN: {{ Path: "/usr/share/AAVMF", Usage: map[FirmwareUsage][]FirmwarePair{ GENERIC: { {Code: "AAVMF_CODE.fd", Vars: "AAVMF_VARS.fd"}, {Code: "OVMF_CODE.4MB.fd", Vars: "OVMF_VARS.4MB.fd"}, {Code: "OVMF_CODE_4M.fd", Vars: "OVMF_VARS_4M.fd"}, {Code: "OVMF_CODE.4m.fd", Vars: "OVMF_VARS.4m.fd"}, {Code: "OVMF_CODE.2MB.fd", Vars: "OVMF_VARS.2MB.fd"}, {Code: "OVMF_CODE.fd", Vars: "OVMF_VARS.fd"}, {Code: "OVMF_CODE.fd", Vars: "qemu.nvram"}, }, SECUREBOOT: { {Code: "AAVMF_CODE.ms.fd", Vars: "AAVMF_VARS.ms.fd"}, {Code: "OVMF_CODE.4MB.fd", Vars: "OVMF_VARS.4MB.ms.fd"}, {Code: "OVMF_CODE_4M.ms.fd", Vars: "OVMF_VARS_4M.ms.fd"}, {Code: "OVMF_CODE_4M.secboot.fd", Vars: "OVMF_VARS_4M.secboot.fd"}, {Code: "OVMF_CODE.secboot.4m.fd", Vars: "OVMF_VARS.4m.fd"}, {Code: "OVMF_CODE.secboot.fd", Vars: "OVMF_VARS.secboot.fd"}, {Code: "OVMF_CODE.secboot.fd", Vars: "OVMF_VARS.fd"}, {Code: "OVMF_CODE.2MB.fd", Vars: "OVMF_VARS.2MB.ms.fd"}, {Code: "OVMF_CODE.fd", Vars: "OVMF_VARS.ms.fd"}, {Code: "OVMF_CODE.fd", Vars: "qemu.nvram"}, }, }, }}, } // GetArchitectureInstallations returns an array of installations for a specific host architecture. func GetArchitectureInstallations(hostArch int) []Installation { installations, found := architectureInstallations[hostArch] if found { return installations } return []Installation{} } // GetArchitectureFirmwarePairs creates an array of FirmwarePair for a // specific host architecture. If the environment variable INCUS_EDK2_PATH // has been set it will override the default installation path when // constructing Code & Vars paths. func GetArchitectureFirmwarePairs(hostArch int) ([]FirmwarePair, error) { firmwares := make([]FirmwarePair, 0) for _, usage := range []FirmwareUsage{GENERIC, SECUREBOOT, CSM} { firmware, err := GetArchitectureFirmwarePairsForUsage(hostArch, usage) if err != nil { return nil, err } firmwares = append(firmwares, firmware...) } return firmwares, nil } // GetArchitectureFirmwarePairsForUsage creates an array of FirmwarePair // for a specific host architecture and usage combination. If the // environment variable INCUS_EDK2_PATH has been set it will override the // default installation path when constructing Code & Vars paths. func GetArchitectureFirmwarePairsForUsage(hostArch int, usage FirmwareUsage) ([]FirmwarePair, error) { firmwares := make([]FirmwarePair, 0) incusEdk2Path, err := GetenvEdk2Path() if err != nil { return nil, err } for _, installation := range GetArchitectureInstallations(hostArch) { usage, found := installation.Usage[usage] if found { // Prefer the EDK2 override path if provided. for _, searchPath := range []string{incusEdk2Path, installation.Path} { if searchPath == "" || !util.PathExists(searchPath) { continue } for _, firmwarePair := range usage { codePath := filepath.Join(searchPath, firmwarePair.Code) varsPath := filepath.Join(searchPath, firmwarePair.Vars) if !util.PathExists(codePath) || !util.PathExists(varsPath) { continue } firmwares = append(firmwares, FirmwarePair{ Code: codePath, Vars: varsPath, }) } } } } return firmwares, nil } // GetenvEdk2Path returns the environment variable for overriding the path to use for EDK2 installations. func GetenvEdk2Path() (string, error) { value := os.Getenv("INCUS_EDK2_PATH") if value == "" { return "", nil } if !util.PathExists(value) { return "", fmt.Errorf("INCUS_EDK2_PATH set to %q but path doesn't exist", value) } return value, nil } incus-7.0.0/internal/server/instance/drivers/edk2/driver_edk2_test.go000066400000000000000000000032241517523235500256670ustar00rootroot00000000000000//go:build amd64 || arm64 package edk2 import ( "os" "path/filepath" "strings" "testing" "github.com/lxc/incus/v7/internal/server/util" ) func tSetup(t *testing.T) func() { t.Helper() tmpdir, err := os.MkdirTemp("", "edk2*") if err != nil { t.Fatal(err) } incusEdk2Path := filepath.Join(tmpdir, "ovmf") err = os.MkdirAll(incusEdk2Path, 0o700) if err != nil { t.Fatal(err) } for _, fn := range []string{"OVMF_CODE.fd", "OVMF.fd", "OVMF_VARS.fd"} { f, err := os.Create(filepath.Join(incusEdk2Path, fn)) if err != nil { t.Fatal(err) } err = f.Close() if err != nil { t.Fatal(err) } } err = os.Setenv("INCUS_EDK2_PATH", incusEdk2Path) if err != nil { t.Fatal(err) } return func() { err = os.Unsetenv("INCUS_EDK2_PATH") if err != nil { t.Fatal(err) } err = os.RemoveAll(tmpdir) if err != nil { t.Fatal(err) } } } func TestGetEdk2Path(t *testing.T) { incusEDK2Path, err := GetenvEdk2Path() if err != nil || incusEDK2Path != "" { t.Fatal(err, incusEDK2Path) } rb := tSetup(t) defer rb() incusEDK2Path, err = GetenvEdk2Path() if err != nil || incusEDK2Path == "" { t.Fatal(incusEDK2Path) } } func TestArchitectureFirmwarePairs(t *testing.T) { rb := tSetup(t) defer rb() architectures, err := util.GetArchitectures() if err != nil || len(architectures) == 0 { t.Fatal(err) } pairs, err := GetArchitectureFirmwarePairs(architectures[0]) if err != nil { t.Fatal(err) } if len(pairs) == 0 { t.Fatal(pairs) } pair1 := pairs[0] code, vars := pair1.Code, pair1.Vars if !strings.HasSuffix(code, "ovmf/OVMF_CODE.fd") || !strings.HasSuffix(vars, "ovmf/OVMF_VARS.fd") { t.Fatal(pair1) } } incus-7.0.0/internal/server/instance/drivers/load.go000066400000000000000000000156361517523235500225340ustar00rootroot00000000000000package drivers import ( "errors" "fmt" "slices" "sync" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/warningtype" "github.com/lxc/incus/v7/internal/server/device" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" ) // Instance driver definitions. var instanceDrivers = map[string]func() instance.Instance{ "lxc": func() instance.Instance { return &lxc{} }, "qemu": func() instance.Instance { return &qemu{} }, } // DriverStatus definition. type DriverStatus struct { Info instance.Info Warning *cluster.Warning Supported bool } // Supported instance drivers cache variables. var ( driverStatusesMu sync.Mutex driverStatuses map[instancetype.Type]*DriverStatus ) // Temporary instance reference storage (for hooks). var ( instanceRefsMu sync.Mutex instanceRefs map[string]instance.Instance ) func init() { // Expose load to the instance package, to avoid circular imports. instance.Load = load // Expose validDevices to the instance package, to avoid circular imports. instance.ValidDevices = validDevices // Expose create to the instance package, to avoid circular imports. instance.Create = create } // load creates the underlying instance type struct and returns it as an Instance. func load(s *state.State, args db.InstanceArgs, p api.Project) (instance.Instance, error) { var inst instance.Instance var err error if args.Type == instancetype.Container { inst, err = lxcLoad(s, args, p) } else if args.Type == instancetype.VM { inst, err = qemuLoad(s, args, p) } else { return nil, fmt.Errorf("Invalid instance type for instance %s", args.Name) } if err != nil { return nil, err } return inst, nil } // validDevices validate instance device configs. func validDevices(state *state.State, p api.Project, instanceType instancetype.Type, localDevices deviceConfig.Devices, expandedDevices deviceConfig.Devices) error { instConf := &common{ dbType: instanceType, localDevices: localDevices.Clone(), expandedDevices: expandedDevices.Clone(), project: p, } var checkedDevices []string checkDevices := func(devices deviceConfig.Devices, expanded bool) error { // Check each device individually using the device package. for deviceName, deviceConfig := range devices { if expanded && slices.Contains(checkedDevices, deviceName) { continue // Don't check the device twice if present in both local and expanded. } // Enforce a maximum name length of 64 characters. // This is a safe maximum allowing use for sockets and other filesystem use. if len(deviceName) > 64 { return errors.New("The maximum device name length is 64 characters") } err := device.Validate(instConf, state, deviceName, deviceConfig, false) if err != nil { if expanded && errors.Is(err, device.ErrUnsupportedDevType) { // Skip unsupported devices in expanded config. // This allows mixed instance type profiles to be used where some devices // are only supported with specific instance types. continue } return fmt.Errorf("Device validation failed for %q: %w", deviceName, err) } checkedDevices = append(checkedDevices, deviceName) } return nil } // Check each local device individually using the device package. // Use the cloned config from instConf.localDevices so that the driver cannot modify it. err := checkDevices(instConf.localDevices, false) if err != nil { return err } if len(expandedDevices) > 0 { // Check we have a root disk if in expanded validation mode. _, _, err := internalInstance.GetRootDiskDevice(expandedDevices.CloneNative()) if err != nil { return fmt.Errorf("Failed detecting root disk device: %w", err) } // Check each expanded device individually using the device package. // Use the cloned config from instConf.expandedDevices so that the driver cannot modify it. err = checkDevices(instConf.expandedDevices, true) if err != nil { return err } } return nil } func create(s *state.State, args db.InstanceArgs, p api.Project, partialDeviceValidation bool, op *operations.Operation) (instance.Instance, revert.Hook, error) { if args.Type == instancetype.Container { return lxcCreate(s, args, p, partialDeviceValidation, op) } else if args.Type == instancetype.VM { return qemuCreate(s, args, p, partialDeviceValidation, op) } return nil, nil, errors.New("Instance type invalid") } // DriverStatuses returns a map of DriverStatus structs for all instance type drivers. // The first time this function is called each of the instance drivers will be probed for support and the result // will be cached internally to make subsequent calls faster. func DriverStatuses() map[instancetype.Type]*DriverStatus { driverStatusesMu.Lock() defer driverStatusesMu.Unlock() if driverStatuses != nil { return driverStatuses } driverStatuses = make(map[instancetype.Type]*DriverStatus, len(instanceDrivers)) for _, instanceDriver := range instanceDrivers { driverStatus := &DriverStatus{} driverInfo := instanceDriver().Info() driverStatus.Info = driverInfo driverStatus.Supported = true if driverInfo.Error != nil || driverInfo.Version == "" { logger.Warn("Instance type not operational", logger.Ctx{"type": driverInfo.Type, "driver": driverInfo.Name, "err": driverInfo.Error}) driverStatus.Supported = false driverStatus.Warning = &cluster.Warning{ TypeCode: warningtype.InstanceTypeNotOperational, LastMessage: fmt.Sprintf("%v", driverInfo.Error), } } else { logger.Info("Instance type operational", logger.Ctx{"type": driverInfo.Type, "driver": driverInfo.Name}) } driverStatuses[driverInfo.Type] = driverStatus } return driverStatuses } // instanceRefGet retrieves an instance reference. func instanceRefGet(projectName string, instName string) instance.Instance { instanceRefsMu.Lock() defer instanceRefsMu.Unlock() return instanceRefs[project.Instance(projectName, instName)] } // instanceRefSet stores a reference to an instance. func instanceRefSet(inst instance.Instance) { instanceRefsMu.Lock() defer instanceRefsMu.Unlock() if instanceRefs == nil { instanceRefs = make(map[string]instance.Instance) } instanceRefs[project.Instance(inst.Project().Name, inst.Name())] = inst } // instanceRefClear removes an instance reference. func instanceRefClear(inst instance.Instance) { instanceRefsMu.Lock() defer instanceRefsMu.Unlock() delete(instanceRefs, project.Instance(inst.Project().Name, inst.Name())) } incus-7.0.0/internal/server/instance/drivers/qemudefault/000077500000000000000000000000001517523235500235675ustar00rootroot00000000000000incus-7.0.0/internal/server/instance/drivers/qemudefault/default.go000066400000000000000000000003321517523235500255400ustar00rootroot00000000000000package qemudefault // CPUCores defines the default number of cores a VM will get if no limit specified. const CPUCores = 1 // MemSize is the default memory size for VMs if no limit specified. const MemSize = "1GiB" incus-7.0.0/internal/server/instance/drivers/qmp/000077500000000000000000000000001517523235500220505ustar00rootroot00000000000000incus-7.0.0/internal/server/instance/drivers/qmp/commands.go000066400000000000000000001240631517523235500242060ustar00rootroot00000000000000package qmp import ( "context" "encoding/json" "errors" "fmt" "net" "net/http" "os" "strings" "time" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/revert" ) // ChardevChangeInfo contains information required to change the backend of a chardev. type ChardevChangeInfo struct { Type string `json:"type"` File *os.File `json:"file,omitempty"` FDName string `json:"fdname,omitempty"` } // FdsetFdInfo contains information about a file descriptor that belongs to an FD set. type FdsetFdInfo struct { FD int `json:"fd"` Opaque string `json:"opaque"` } // FdsetInfo contains information about an FD set. type FdsetInfo struct { ID int `json:"fdset-id"` FDs []FdsetFdInfo `json:"fds"` } // AddFdInfo contains information about a file descriptor that was added to an fd set. type AddFdInfo struct { ID int `json:"fdset-id"` FD int `json:"fd"` } // CPUInstanceProperties contains CPU instance properties. type CPUInstanceProperties struct { NodeID int `json:"node-id,omitempty"` SocketID int `json:"socket-id,omitempty"` DieID int `json:"die-id,omitempty"` ClusterID int `json:"cluster-id,omitempty"` CoreID int `json:"core-id,omitempty"` ThreadID int `json:"thread-id,omitempty"` } // CPU contains information about a CPU. type CPU struct { Index int `json:"cpu-index,omitempty"` QOMPath string `json:"qom-path,omitempty"` ThreadID int `json:"thread-id,omitempty"` Target string `json:"target,omitempty"` Props CPUInstanceProperties `json:"props"` } // HotpluggableCPU contains information about a hotpluggable CPU. type HotpluggableCPU struct { Type string `json:"type"` VCPUsCount int `json:"vcpus-count"` QOMPath string `json:"qom-path,omitempty"` Props CPUInstanceProperties `json:"props"` } // CPUModel contains information about a CPU model. type CPUModel struct { Name string `json:"name"` Flags map[string]any `json:"props"` } // MemoryDevice contains information about a memory device. type MemoryDevice struct { Type string `json:"type"` Data json.RawMessage `json:"data"` } // PCDimmDevice contains information about a memory device of type pc-dimm. type PCDimmDevice struct { ID string `json:"id"` Addr uint64 `json:"addr"` Slot int `json:"slot"` Node int `json:"node"` Memdev string `json:"memdev"` Hotpluggable bool `json:"hotpluggable"` Hotplugged bool `json:"hotplugged"` } // MemDev contains information about a memory device. type MemDev struct { ID string `json:"id"` Size int `json:"size"` Merge bool `json:"merge"` Dump bool `json:"dump"` Prealloc bool `json:"prealloc"` Share bool `json:"share"` Reserve bool `json:"reserve"` HostNodes []int `json:"host-nodes"` } // MigrationStatus contains information about the ongoing migration. type MigrationStatus struct { Status string `json:"status"` RAM struct { Transferred int64 `json:"transferred"` Remaining int64 `json:"remaining"` Total int64 `json:"total"` Duplicate int64 `json:"duplicate"` Normal int64 `json:"normal"` NormalBytes int64 `json:"normal-bytes"` DirtyPagesRate int64 `json:"diry-pages-rate"` MBps float64 `json:"mbps"` DirtySyncCount int64 `json:"diry-sync-count"` PostcopyRequests int64 `json:"postcopy-requests"` PageSize int64 `json:"page-size"` MultiFDBytes int64 `json:"multifd-bytes"` PagesPerSecond int64 `json:"pages-per-second"` PrecopyBytes int64 `json:"precopy-bytes"` DowntimeBytes int64 `json:"downtime-bytes"` PostcopyBytes int64 `json:"postcopy-bytes"` DirtySyncMissedZeroCopy int64 `json:"diry-sync-missed-zero-copy"` } `json:"ram"` TotalTime int64 `json:"total-time"` DownTime int64 `json:"down-time"` ExpectedDowntime int64 `json:"expected-downtime"` SetupTime int64 `json:"setup-time"` CPUThrottlePercentage int64 `json:"cpu-throttle-percentage"` PostcopyBlocktime int64 `json:"postcopy-blocktime"` PostcopyVCPUBlocktime []int64 `json:"postcopy-vcpu-blocktime"` DirtyLimitThrottleTimePerRound int64 `json:"dirty-limit-throttle-time-per-round"` DirtyLimitRingFullTime int64 `json:"dirty-limit-ring-full-time"` } // BlockExport contains information about the exported block. type BlockExport struct { NodeName string `json:"node-name"` Type string `json:"type"` } // BlockDirtyInfo contains dirty bitmap information. type BlockDirtyInfo struct { Name string `json:"name"` Count int `json:"count"` Granularity int `json:"granularity"` Recording bool `json:"recording"` Busy bool `json:"busy"` Persistent bool `json:"persistent"` Inconsistent bool `json:"inconsistent"` } // BlockDeviceInfo contains information about the backing device for a block device. type BlockDeviceInfo struct { NodeName string `json:"node-name"` DirtyBitmaps []BlockDirtyInfo `json:"dirty-bitmaps"` } // BlockInfo contains information about a virtual block device. type BlockInfo struct { Device string `json:"device"` Inserted BlockDeviceInfo `json:"inserted"` } // QueryCPUs returns a list of CPUs. func (m *Monitor) QueryCPUs() ([]CPU, error) { // Prepare the response. var resp struct { Return []CPU `json:"return"` } err := m.Run("query-cpus-fast", nil, &resp) if err != nil { return nil, fmt.Errorf("Failed to query CPUs: %w", err) } return resp.Return, nil } // QueryHotpluggableCPUs returns a list of hotpluggable CPUs. func (m *Monitor) QueryHotpluggableCPUs() ([]HotpluggableCPU, error) { // Prepare the response. var resp struct { Return []HotpluggableCPU `json:"return"` } err := m.Run("query-hotpluggable-cpus", nil, &resp) if err != nil { return nil, fmt.Errorf("Failed to query hotpluggable CPUs: %w", err) } return resp.Return, nil } // QueryCPUModel returns a CPUModel for the specified model name. func (m *Monitor) QueryCPUModel(model string) (*CPUModel, error) { // Prepare the response. var resp struct { Return struct { Model CPUModel `json:"model"` } `json:"return"` } args := map[string]any{ "model": map[string]string{"name": model}, "type": "full", } err := m.Run("query-cpu-model-expansion", args, &resp) if err != nil { return nil, fmt.Errorf("Failed to query CPU model: %w", err) } return &resp.Return.Model, nil } // Status returns the current VM status. func (m *Monitor) Status() (string, error) { // Prepare the response. var resp struct { Return struct { Status string `json:"status"` } `json:"return"` } // Query the status. err := m.Run("query-status", nil, &resp) if err != nil { return "", err } return resp.Return.Status, nil } // MachineDefinition returns the current QEMU machine definition name. func (m *Monitor) MachineDefinition() (string, error) { // Prepare the request. var req struct { Path string `json:"path"` Property string `json:"property"` } req.Path = "/machine" req.Property = "type" // Prepare the response. var resp struct { Return string `json:"return"` } // Query the machine. err := m.Run("qom-get", req, &resp) if err != nil { return "", err } return strings.TrimSuffix(resp.Return, "-machine"), nil } // MemoryConfiguration returns the current QEMU machine memory configuration (current, max, slots). func (m *Monitor) MemoryConfiguration() (int64, int64, int64, error) { // Prepare the request. var req struct { Path string `json:"path"` Property string `json:"property"` } req.Path = "/machine" req.Property = "memory" // Prepare the response. var resp struct { Return struct { Size int64 `json:"size"` Slots int64 `json:"slots"` MaxSize int64 `json:"max-size"` } `json:"return"` } // Query the machine. err := m.Run("qom-get", req, &resp) if err != nil { return -1, -1, -1, err } return resp.Return.Size, resp.Return.MaxSize, resp.Return.Slots, nil } // SendFile adds a new file descriptor to the QMP fd table associated to name. func (m *Monitor) SendFile(name string, file *os.File) error { // Check if disconnected. if m.disconnected { return ErrMonitorDisconnect } id := m.qmp.qmpIncreaseID() req := &qmpCommand{ ID: id, Execute: "getfd", Arguments: map[string]any{"fdname": name}, } reqJSON, err := json.Marshal(req) if err != nil { return err } // Query the status. _, err = m.qmp.runWithFile(reqJSON, file, id) if err != nil { // Confirm the daemon didn't die. errPing := m.ping() if errPing != nil { return errPing } return err } return nil } // CloseFile closes an existing file descriptor in the QMP fd table associated to name. func (m *Monitor) CloseFile(name string) error { var req struct { FDName string `json:"fdname"` } req.FDName = name err := m.Run("closefd", req, nil) if err != nil { return err } return nil } // SendFileWithFDSet adds a new file descriptor to an FD set. func (m *Monitor) SendFileWithFDSet(name string, file *os.File, readonly bool) (*AddFdInfo, error) { // Check if disconnected. if m.disconnected { return nil, ErrMonitorDisconnect } permissions := "rdwr" if readonly { permissions = "rdonly" } id := m.qmp.qmpIncreaseID() req := &qmpCommand{ ID: id, Execute: "add-fd", Arguments: map[string]any{ "opaque": fmt.Sprintf("%s:%s", permissions, name), }, } reqJSON, err := json.Marshal(req) if err != nil { return nil, err } ret, err := m.qmp.runWithFile(reqJSON, file, id) if err != nil { // Confirm the daemon didn't die. errPing := m.ping() if errPing != nil { return nil, errPing } return nil, err } // Prepare the response. var resp struct { Return AddFdInfo `json:"return"` } err = json.Unmarshal(ret, &resp) if err != nil { return nil, err } return &resp.Return, nil } // RemoveFDFromFDSet removes an FD with the given name from an FD set. func (m *Monitor) RemoveFDFromFDSet(name string) error { // Prepare the response. var resp struct { Return []FdsetInfo `json:"return"` } err := m.Run("query-fdsets", nil, &resp) if err != nil { return fmt.Errorf("Failed to query fd sets: %w", err) } for _, fdSet := range resp.Return { for _, fd := range fdSet.FDs { fields := strings.SplitN(fd.Opaque, ":", 2) opaque := "" if len(fields) == 2 { opaque = fields[1] } else { opaque = fields[0] } if opaque == name { args := map[string]any{ "fdset-id": fdSet.ID, } err = m.Run("remove-fd", args, nil) if err != nil { return fmt.Errorf("Failed to remove fd from fd set: %w", err) } } } } return nil } // MigrateSetCapabilities sets the capabilities used during migration. func (m *Monitor) MigrateSetCapabilities(caps map[string]bool) error { var args struct { Capabilities []struct { Capability string `json:"capability"` State bool `json:"state"` } `json:"capabilities"` } for capName, state := range caps { args.Capabilities = append(args.Capabilities, struct { Capability string `json:"capability"` State bool `json:"state"` }{ Capability: capName, State: state, }) } err := m.Run("migrate-set-capabilities", args, nil) if err != nil { return err } return nil } // MigrateSetParameters sets the parameters used during migration. func (m *Monitor) MigrateSetParameters(parameters map[string]any) error { err := m.Run("migrate-set-parameters", parameters, nil) if err != nil { return err } return nil } // Migrate starts a migration stream. func (m *Monitor) Migrate(name string) error { // Query the status. type migrateArgsChannel struct { ChannelType string `json:"channel-type"` Address map[string]string `json:"addr"` } type migrateArgs struct { Channels []migrateArgsChannel `json:"channels"` } args := migrateArgs{} args.Channels = []migrateArgsChannel{{ ChannelType: "main", Address: map[string]string{ "transport": "socket", "type": "fd", "str": name, }, }} err := m.Run("migrate", args, nil) if err != nil { return err } return nil } // QueryMigrate gets the current migration status. func (m *Monitor) QueryMigrate() (*MigrationStatus, error) { var resp struct { Return MigrationStatus `json:"return"` } err := m.Run("query-migrate", nil, &resp) if err != nil { return nil, err } return &resp.Return, nil } // MigrateWait waits until migration job reaches the specified status. // Returns nil if the migraton job reaches the specified status or an error if the migration job is in the failed // status. func (m *Monitor) MigrateWait(ctx context.Context, state string) error { // Wait until it completes or fails. for { select { case <-ctx.Done(): return ctx.Err() default: // Prepare the response. var resp struct { Return struct { Status string `json:"status"` } `json:"return"` } err := m.Run("query-migrate", nil, &resp) if err != nil { return err } if resp.Return.Status == "failed" { return errors.New("Migrate call failed") } if resp.Return.Status == state { return nil } time.Sleep(1 * time.Second) } } } // MigrateContinue continues a migration stream. func (m *Monitor) MigrateContinue(fromState string) error { var args struct { State string `json:"state"` } args.State = fromState err := m.Run("migrate-continue", args, nil) if err != nil { return err } return nil } // MigrateIncoming starts the receiver of a migration stream. func (m *Monitor) MigrateIncoming(ctx context.Context, name string) error { type migrateArgsChannel struct { ChannelType string `json:"channel-type"` Address map[string]string `json:"addr"` } type migrateArgs struct { Channels []migrateArgsChannel `json:"channels"` } args := migrateArgs{} args.Channels = []migrateArgsChannel{{ ChannelType: "main", Address: map[string]string{ "transport": "socket", "type": "fd", "str": name, }, }} // Query the status. err := m.Run("migrate-incoming", args, nil) if err != nil { return err } // Wait until it completes or fails. for { // Prepare the response. var resp struct { Return struct { Status string `json:"status"` } `json:"return"` } err := m.Run("query-migrate", nil, &resp) if err != nil { return err } if resp.Return.Status == "failed" { return errors.New("Migrate incoming call failed") } if resp.Return.Status == "completed" { return nil } // Check context is cancelled last after checking job status. // This way if the context is cancelled when the migration stream is ended this gives a chance to // check for job success/failure before checking if the context has been cancelled. err = ctx.Err() if err != nil { return err } time.Sleep(1 * time.Second) } } // Powerdown tells the VM to gracefully shutdown. func (m *Monitor) Powerdown() error { return m.Run("system_powerdown", nil, nil) } // Start tells QEMU to start the emulation. func (m *Monitor) Start() error { return m.Run("cont", nil, nil) } // Pause tells QEMU to temporarily stop the emulation. func (m *Monitor) Pause() error { return m.Run("stop", nil, nil) } // Quit tells QEMU to exit immediately. func (m *Monitor) Quit() error { return m.Run("quit", nil, nil) } // GetCPUs fetches the vCPU information for pinning. func (m *Monitor) GetCPUs() ([]int, error) { // Prepare the response. var resp struct { Return []struct { CPU int `json:"cpu-index"` PID int `json:"thread-id"` } `json:"return"` } // Query the consoles. err := m.Run("query-cpus-fast", nil, &resp) if err != nil { return nil, err } // Make a slice of PIDs. pids := []int{} for _, cpu := range resp.Return { pids = append(pids, cpu.PID) } return pids, nil } // GetMemorySizeBytes returns the current size of the base memory in bytes. func (m *Monitor) GetMemorySizeBytes() (int64, error) { // Prepare the response. var resp struct { Return struct { BaseMemory int64 `json:"base-memory"` } `json:"return"` } err := m.Run("query-memory-size-summary", nil, &resp) if err != nil { return -1, err } return resp.Return.BaseMemory, nil } // GetMemoryBalloonSizeBytes returns effective size of the memory in bytes (considering the current balloon size). func (m *Monitor) GetMemoryBalloonSizeBytes() (int64, error) { // Prepare the response. var resp struct { Return struct { Actual int64 `json:"actual"` } `json:"return"` } err := m.Run("query-balloon", nil, &resp) if err != nil { return -1, err } return resp.Return.Actual, nil } // SetMemoryBalloonSizeBytes sets the size of the memory in bytes (which will resize the balloon as needed). func (m *Monitor) SetMemoryBalloonSizeBytes(sizeBytes int64) error { args := map[string]int64{"value": sizeBytes} return m.Run("balloon", args, nil) } // GetMemdev retrieves memory devices by executing the query-memdev QMP command. func (m *Monitor) GetMemdev() ([]MemDev, error) { // Prepare the response. var resp struct { Return []MemDev `json:"return"` } err := m.Run("query-memdev", nil, &resp) if err != nil { return nil, err } return resp.Return, nil } // GetMemoryDevices retrieves memory devices by executing the query-memory-devices QMP command. func (m *Monitor) GetMemoryDevices() ([]MemoryDevice, error) { // Prepare the response. var resp struct { Return []MemoryDevice `json:"return"` } err := m.Run("query-memory-devices", nil, &resp) if err != nil { return nil, err } return resp.Return, nil } // GetDimmDevices returns a list of memory devices of type pc-dimm. func (m *Monitor) GetDimmDevices() ([]PCDimmDevice, error) { devices, err := m.GetMemoryDevices() if err != nil { return nil, err } result := []PCDimmDevice{} for _, dev := range devices { if dev.Type != "dimm" { continue } var dimmData PCDimmDevice err := json.Unmarshal(dev.Data, &dimmData) if err != nil { continue } result = append(result, dimmData) } return result, nil } // AddObject adds a new object. func (m *Monitor) AddObject(args map[string]any) error { err := m.Run("object-add", &args, nil) if err != nil { return fmt.Errorf("Failed adding object: %w", err) } return nil } // AddBlockDevice adds a block device. func (m *Monitor) AddBlockDevice(blockDev map[string]any, device map[string]any, usb bool) error { reverter := revert.New() defer reverter.Fail() // If USB, start with a USB block only controller. var usbID string if usb { id, ok := device["id"].(string) if !ok { return errors.New("Parent device must have an id") } usbID = "usb_" + id usbDev := map[string]any{ "driver": "usb-bot", "bus": "qemu_usb.0", "id": usbID, } err := m.AddDevice(usbDev) if err != nil { return fmt.Errorf("Failed adding USB device: %w", err) } reverter.Add(func() { _ = m.RemoveDevice(usbID) }) device["bus"] = usbID + ".0" } nodeName, ok := blockDev["node-name"].(string) if !ok { return errors.New("Device node name must be a string") } if blockDev != nil { err := m.Run("blockdev-add", blockDev, nil) if err != nil { return fmt.Errorf("Failed adding block device: %w", err) } reverter.Add(func() { _ = m.RemoveBlockDevice(nodeName) }) } err := m.AddDevice(device) if err != nil { return fmt.Errorf("Failed adding device: %w", err) } // If USB, bring the device up now. if usb { err = m.Run("qom-set", map[string]any{"path": usbID, "property": "attached", "value": true}, nil) if err != nil { return fmt.Errorf("Failed to bring up USB device: %w", err) } } reverter.Success() return nil } // RemoveBlockDevice removes a block device. func (m *Monitor) RemoveBlockDevice(blockDevName string) error { if blockDevName != "" { blockDevName := map[string]string{ "node-name": blockDevName, } err := m.Run("blockdev-del", blockDevName, nil) if err != nil { if strings.Contains(err.Error(), "is in use") { return api.StatusErrorf(http.StatusLocked, "%s", err.Error()) } if strings.Contains(err.Error(), "Failed to find") { return nil } return fmt.Errorf("Failed removing block device: %w", err) } } return nil } // AddCharDevice adds a new character device. func (m *Monitor) AddCharDevice(device map[string]any) error { if device != nil { err := m.Run("chardev-add", device, nil) if err != nil { return err } } return nil } // RemoveCharDevice removes a character device. func (m *Monitor) RemoveCharDevice(deviceID string) error { if deviceID != "" { deviceID := map[string]string{ "id": deviceID, } err := m.Run("chardev-remove", deviceID, nil) if err != nil { if strings.Contains(err.Error(), "not found") { return nil } return err } } return nil } // AddDevice adds a new device. func (m *Monitor) AddDevice(device map[string]any) error { if device != nil { err := m.Run("device_add", device, nil) if err != nil { return err } } return nil } // RemoveDevice removes a device. func (m *Monitor) RemoveDevice(deviceID string) error { if deviceID != "" { deviceID := map[string]string{ "id": deviceID, } err := m.Run("device_del", deviceID, nil) if err != nil { if strings.Contains(err.Error(), "not found") { return nil } return err } } return nil } // AddNIC adds a NIC device. func (m *Monitor) AddNIC(netDev map[string]any, device map[string]any, connected bool) error { reverter := revert.New() defer reverter.Fail() if netDev != nil { err := m.Run("netdev_add", netDev, nil) if err != nil { return fmt.Errorf("Failed adding NIC netdev: %w", err) } reverter.Add(func() { netDevDel := map[string]any{ "id": netDev["id"], } err = m.Run("netdev_del", netDevDel, nil) if err != nil { return } }) } err := m.AddDevice(device) if err != nil { return fmt.Errorf("Failed adding NIC device: %w", err) } id, ok := device["id"].(string) if !ok { return errors.New("NIC device must have an id") } // Set link down if asked to. if !connected { err = m.SetNICLink(id, connected) if err != nil { return fmt.Errorf("Failed setting NIC device link status: %w", err) } } reverter.Success() return nil } // RemoveNIC removes a NIC device. func (m *Monitor) RemoveNIC(netDevID string) error { if netDevID != "" { netDevID := map[string]string{ "id": netDevID, } err := m.Run("netdev_del", netDevID, nil) // Not all NICs need a netdev, so if its missing, its not a problem. if err != nil && !strings.Contains(err.Error(), "not found") { return fmt.Errorf("Failed removing NIC netdev: %w", err) } } return nil } // SetAction sets the actions the VM will take for certain scenarios. func (m *Monitor) SetAction(actions map[string]string) error { err := m.Run("set-action", actions, nil) if err != nil { return fmt.Errorf("Failed setting actions: %w", err) } return nil } // Reset VM. func (m *Monitor) Reset() error { // Announce that we're triggering a reset so the event handler can distinguish // our deliberate system_reset from a guest-initiated reboot. This must be set // before sending the command, since the RESET event is processed asynchronously // and may otherwise arrive after the startup goroutine has already flipped // the initialized flag. m.ExpectReset() err := m.Run("system_reset", nil, nil) if err != nil { m.HandleReset() return fmt.Errorf("Failed resetting: %w", err) } return nil } // PCIClassInfo info about a device's class. type PCIClassInfo struct { Class int `json:"class"` Description string `json:"desc"` } // PCIDevice represents a PCI device. type PCIDevice struct { DevID string `json:"qdev_id"` Bus int `json:"bus"` Slot int `json:"slot"` Function int `json:"function"` Devices []PCIDevice `json:"devices"` Class PCIClassInfo `json:"class_info"` Bridge PCIBridge `json:"pci_bridge"` } // PCIBridge represents a PCI bridge. type PCIBridge struct { Devices []PCIDevice `json:"devices"` } // QueryPCI returns info about PCI devices. func (m *Monitor) QueryPCI() ([]PCIDevice, error) { // Prepare the response. var resp struct { Return []struct { Devices []PCIDevice `json:"devices"` } `json:"return"` } err := m.Run("query-pci", nil, &resp) if err != nil { return nil, fmt.Errorf("Failed querying PCI devices: %w", err) } if len(resp.Return) > 0 { return resp.Return[0].Devices, nil } return nil, nil } // BlockStats represents block device stats. type BlockStats struct { BytesWritten int `json:"wr_bytes"` WritesCompleted int `json:"wr_operations"` BytesRead int `json:"rd_bytes"` ReadsCompleted int `json:"rd_operations"` } // GetBlockStats return block device stats. func (m *Monitor) GetBlockStats() (map[string]BlockStats, error) { // Prepare the response var resp struct { Return []struct { Stats BlockStats `json:"stats"` QDev string `json:"qdev"` } `json:"return"` } err := m.Run("query-blockstats", nil, &resp) if err != nil { return nil, fmt.Errorf("Failed querying block stats: %w", err) } out := make(map[string]BlockStats) for _, res := range resp.Return { out[res.QDev] = res.Stats } return out, nil } // AddSecret adds a secret object with the given ID and secret. This function won't return an error // if the secret object already exists. func (m *Monitor) AddSecret(id string, secret string) error { args := map[string]any{ "qom-type": "secret", "id": id, "data": secret, "format": "base64", } err := m.Run("object-add", &args, nil) if err != nil && !strings.Contains(err.Error(), "attempt to add duplicate property") { return fmt.Errorf("Failed adding object: %w", err) } return nil } // AMDSEVCapabilities represents the SEV capabilities of QEMU. type AMDSEVCapabilities struct { PDH string `json:"pdh"` // Platform Diffie-Hellman key (base64-encoded) CertChain string `json:"cert-chain"` // PDH certificate chain (base64-encoded) CPU0Id string `json:"cpu0-id"` // Unique ID of CPU0 (base64-encoded) CBitPos int `json:"cbitpos"` // C-bit location in page table entry ReducedPhysBits int `json:"reduced-phys-bits"` // Number of physical address bit reduction when SEV is enabled } // SEVCapabilities is used to get the SEV capabilities, and is supported on AMD X86 platforms only. func (m *Monitor) SEVCapabilities() (AMDSEVCapabilities, error) { // Prepare the response var resp struct { Return AMDSEVCapabilities `json:"return"` } err := m.Run("query-sev-capabilities", nil, &resp) if err != nil { return AMDSEVCapabilities{}, fmt.Errorf("Failed querying SEV capability for QEMU: %w", err) } return resp.Return, nil } // NBDServerStart starts internal NBD server and returns a connection to it. func (m *Monitor) NBDServerStart() (net.Conn, error) { var args struct { Addr struct { Data struct { Path string `json:"path"` Abstract bool `json:"abstract"` } `json:"data"` Type string `json:"type"` } `json:"addr"` MaxConnections int `json:"max-connections"` } // Create abstract unix listener. listener, err := net.Listen("unix", "") if err != nil { return nil, fmt.Errorf("Failed creating unix listener: %w", err) } // Get the random address, and then close the listener, and pass the address for use with nbd-server-start. listenAddress := listener.Addr().String() _ = listener.Close() args.Addr.Type = "unix" args.Addr.Data.Path = strings.TrimPrefix(listenAddress, "@") args.Addr.Data.Abstract = true args.MaxConnections = 1 err = m.Run("nbd-server-start", args, nil) if err != nil { return nil, err } // Connect to the NBD server and return the connection. conn, err := net.Dial("unix", listenAddress) if err != nil { return nil, fmt.Errorf("Failed connecting to NBD server: %w", err) } return conn, nil } // NBDUnixServerStart starts an internal NBD server listening on the specified Unix socket. func (m *Monitor) NBDUnixServerStart(path string) error { var args struct { Addr struct { Data struct { Str string `json:"str"` } `json:"data"` Type string `json:"type"` } `json:"addr"` MaxConnections int `json:"max-connections"` } args.Addr.Type = "fd" args.Addr.Data.Str = path args.MaxConnections = 1 err := m.Run("nbd-server-start", args, nil) if err != nil { return err } return nil } // NBDServerStop stops the internal NBD server. func (m *Monitor) NBDServerStop() error { err := m.Run("nbd-server-stop", nil, nil) if err != nil { return err } return nil } // NBDBlockExportAdd exports a writable device via the NBD server. func (m *Monitor) NBDBlockExportAdd(deviceNodeName string, writable bool, bitmapNames []string) error { var args struct { ID string `json:"id"` Type string `json:"type"` NodeName string `json:"node-name"` Writable bool `json:"writable"` Bitmaps []struct { Node string `json:"node"` Name string `json:"name"` } `json:"bitmaps,omitempty"` } args.ID = deviceNodeName args.Type = "nbd" args.NodeName = deviceNodeName args.Writable = writable for _, b := range bitmapNames { args.Bitmaps = append(args.Bitmaps, struct { Node string `json:"node"` Name string `json:"name"` }{ Node: deviceNodeName, Name: b, }) } err := m.Run("block-export-add", args, nil) if err != nil { return err } return nil } // QueryBlock returns a list of all virtual block devices. func (m *Monitor) QueryBlock() ([]BlockInfo, error) { var resp struct { Return []BlockInfo `json:"return"` } err := m.Run("query-block", nil, &resp) if err != nil { return nil, err } return resp.Return, nil } // QueryBlockExports returns exported blocks. func (m *Monitor) QueryBlockExports() ([]BlockExport, error) { var resp struct { Return []BlockExport `json:"return"` } err := m.Run("query-block-exports", nil, &resp) if err != nil { return nil, err } return resp.Return, nil } // QueryNBDBlockExports returns exported blocks of type 'nbd'. func (m *Monitor) QueryNBDBlockExports() ([]BlockExport, error) { blocks, err := m.QueryBlockExports() if err != nil { return nil, err } result := []BlockExport{} for _, b := range blocks { if b.Type != "nbd" { continue } result = append(result, b) } return result, nil } // QueryNamedBlockNodes returns block nodes names. func (m *Monitor) QueryNamedBlockNodes() ([]string, error) { var resp struct { Return []struct { NodeName string `json:"node-name"` } `json:"return"` } err := m.Run("query-named-block-nodes", nil, &resp) if err != nil { return nil, err } result := []string{} for _, r := range resp.Return { result = append(result, r.NodeName) } return result, nil } // ChangeBackingFile changes backing file name for node. func (m *Monitor) ChangeBackingFile(deviceNodeName string, imageNodeName string, backingFilename string) error { var args struct { Device string `json:"device"` NodeName string `json:"image-node-name"` BackingFile string `json:"backing-file"` } args.Device = deviceNodeName args.NodeName = imageNodeName args.BackingFile = backingFilename err := m.Run("change-backing-file", args, nil) if err != nil { return err } return nil } // BlockDevSnapshot creates a snapshot of a device using the specified snapshot device. func (m *Monitor) BlockDevSnapshot(deviceNodeName string, snapshotNodeName string) error { var args struct { Node string `json:"node"` Overlay string `json:"overlay"` } args.Node = deviceNodeName args.Overlay = snapshotNodeName err := m.Run("blockdev-snapshot", args, nil) if err != nil { return err } return nil } // blockJobWaitReady waits until the specified jobID is ready, errored or missing. // Returns nil if the job is ready, otherwise an error. func (m *Monitor) blockJobWaitReady(jobID string, exitOnNotFound bool) error { for { var resp struct { Return []struct { Device string `json:"device"` Ready bool `json:"ready"` Error string `json:"error"` } `json:"return"` } err := m.Run("query-block-jobs", nil, &resp) if err != nil { return err } found := false for _, job := range resp.Return { if job.Device != jobID { continue } if job.Error != "" { return fmt.Errorf("Failed block job: %s", job.Error) } if job.Ready { return nil } found = true } if !found && !exitOnNotFound { return errors.New("Specified block job not found") } else if !found && exitOnNotFound { return nil } time.Sleep(1 * time.Second) } } // BlockCommit merges a snapshot device back into its parent device. func (m *Monitor) BlockCommit(deviceNodeName string, top string, base string) error { var args struct { Device string `json:"device"` JobID string `json:"job-id"` Top string `json:"top-node,omitempty"` Base string `json:"base-node,omitempty"` } args.Device = deviceNodeName args.JobID = args.Device args.Top = top args.Base = base err := m.Run("block-commit", args, nil) if err != nil { return err } // From QEMU doc: If top has no overlays on top of it, or if it is in use by a writer, // the job will not be completed by itself. hasTop := top != deviceNodeName && top != "" err = m.blockJobWaitReady(args.JobID, hasTop) if err != nil { return err } if !hasTop { err = m.BlockJobComplete(args.JobID) if err != nil { return err } } return nil } // BlockDevMirror mirrors the top device to the target device. func (m *Monitor) BlockDevMirror(deviceNodeName string, targetNodeName string) error { var args struct { Device string `json:"device"` Target string `json:"target"` Sync string `json:"sync"` JobID string `json:"job-id"` CopyMode string `json:"copy-mode"` } args.Device = deviceNodeName args.Target = targetNodeName args.JobID = deviceNodeName // Only synchronise the top level device (usually a snapshot). args.Sync = "top" // When data is written to the source, write it (synchronously) to the target as well. // In addition, data is copied in background just like in background mode. // This ensures that the source and target converge at the cost of I/O performance during sync. args.CopyMode = "write-blocking" err := m.Run("blockdev-mirror", args, nil) if err != nil { return err } err = m.blockJobWaitReady(args.JobID, false) if err != nil { return err } return nil } // BlockJobCancel cancels an ongoing block job. func (m *Monitor) BlockJobCancel(deviceNodeName string) error { var args struct { Device string `json:"device"` } args.Device = deviceNodeName err := m.Run("block-job-cancel", args, nil) if err != nil { return err } return nil } // BlockJobComplete completes a block job that is in reader state. func (m *Monitor) BlockJobComplete(deviceNodeName string) error { var args struct { Device string `json:"device"` } args.Device = deviceNodeName ch, err := m.CreateEventChannel(args.Device) if err != nil { return err } err = m.Run("block-job-complete", args, nil) if err != nil { return err } event := <-ch switch event.Name { case EventBlockJobCompleted: return nil case EventBlockJobError: return fmt.Errorf("Error during block-job-complete") default: return fmt.Errorf("Not supported event: %q", event.Name) } } // UpdateBlockSize updates the size of a disk. func (m *Monitor) UpdateBlockSize(id string, size int64) error { var args struct { NodeName string `json:"node-name"` Size int64 `json:"size"` } args.NodeName = id args.Size = size err := m.Run("block_resize", args, nil) if err != nil { return err } return nil } // SetBlockThrottle applies an I/O limit on a disk. func (m *Monitor) SetBlockThrottle(id string, bytesRead int, bytesWrite int, iopsRead int, iopsWrite int) error { var args struct { ID string `json:"id"` Bytes int `json:"bps"` BytesRead int `json:"bps_rd"` BytesWrite int `json:"bps_wr"` IOPs int `json:"iops"` IOPsRead int `json:"iops_rd"` IOPsWrite int `json:"iops_wr"` } args.ID = id args.BytesRead = bytesRead args.BytesWrite = bytesWrite args.IOPsRead = iopsRead args.IOPsWrite = iopsWrite err := m.Run("block_set_io_throttle", args, nil) if err != nil { return err } return nil } // CheckPCIDevice checks if the deviceID exists as a bridged PCI device. func (m *Monitor) CheckPCIDevice(deviceID string) (bool, error) { pciDevs, err := m.QueryPCI() if err != nil { return false, err } for _, pciDev := range pciDevs { for _, bridgeDev := range pciDev.Bridge.Devices { if bridgeDev.DevID == deviceID { return true, nil } } } return false, nil } // RingbufRead returns the complete contents of the specified ring buffer. func (m *Monitor) RingbufRead(device string) (string, error) { // Begin by ensuring the device specified is actually a ring buffer. var queryResp struct { Return []struct { Label string `json:"label"` Filename string `json:"filename"` FrontendOpen bool `json:"frontend_open"` } `json:"return"` } err := m.Run("query-chardev", nil, &queryResp) if err != nil { return "", err } deviceFound := false for _, qemuDevice := range queryResp.Return { if qemuDevice.Label == device { deviceFound = true if qemuDevice.Filename != "ringbuf" { // Can't call `ringbuf-read` on a non-ringbuf device. return "", ErrNotARingbuf } break } } if !deviceFound { return "", fmt.Errorf("Specified qemu device %q doesn't exist", device) } // Now actually read from the ring buffer. var args struct { Device string `json:"device"` Size int `json:"size"` } args.Device = device args.Size = 10000 var readResp struct { Return string `json:"return"` } var sb strings.Builder for { err := m.Run("ringbuf-read", args, &readResp) if err != nil { return "", err } if len(readResp.Return) == 0 { break } sb.WriteString(readResp.Return) } return sb.String(), nil } // ChardevChange changes the backend of a specified chardev. Currently supports the socket and ringbuf backends. func (m *Monitor) ChardevChange(device string, info ChardevChangeInfo) error { if info.Type == "socket" { // Share the existing file descriptor with qemu. err := m.SendFile(info.FDName, info.File) if err != nil { return err } var args struct { ID string `json:"id"` Backend struct { Type string `json:"type"` Data struct { Addr struct { Type string `json:"type"` Data struct { Str string `json:"str"` } `json:"data"` } `json:"addr"` Server bool `json:"server"` Wait bool `json:"wait"` } `json:"data"` } `json:"backend"` } args.ID = device args.Backend.Type = info.Type args.Backend.Data.Addr.Type = "fd" args.Backend.Data.Addr.Data.Str = info.FDName args.Backend.Data.Server = true args.Backend.Data.Wait = false err = m.Run("chardev-change", args, nil) if err != nil { // If the chardev-change command failed for some reason, ensure qemu cleans up its file descriptor. _ = m.CloseFile(info.FDName) return err } return nil } else if info.Type == "ringbuf" { var args struct { ID string `json:"id"` Backend struct { Type string `json:"type"` Data struct { Size int `json:"size"` } `json:"data"` } `json:"backend"` } args.ID = device args.Backend.Type = info.Type args.Backend.Data.Size = 1048576 return m.Run("chardev-change", args, nil) } return fmt.Errorf("Unsupported chardev type %q", info.Type) } // Screendump takes a screenshot of the current VGA console. // The screendump is stored to the filename provided as argument. func (m *Monitor) Screendump(filename string) error { var args struct { Filename string `json:"filename"` Device string `json:"device,omitempty"` Head int `json:"head,omitempty"` Format string `json:"format,omitempty"` } args.Filename = filename args.Format = "png" var queryResp struct { Return struct{} `json:"return"` } return m.Run("screendump", args, &queryResp) } // DumpGuestMemory dumps guest memory to a file. func (m *Monitor) DumpGuestMemory(path string, format string) error { var args struct { Paging bool `json:"paging"` Protocol string `json:"protocol"` Format string `json:"format,omitempty"` Detach bool `json:"detach"` } args.Protocol = "fd:" + path args.Format = format var queryResp struct { Return struct{} `json:"return"` } return m.Run("dump-guest-memory", args, &queryResp) } // SetNICLink sets the link status of the given device. func (m *Monitor) SetNICLink(id string, connected bool) error { var args struct { Name string `json:"name"` Up bool `json:"up"` } args.Name = id args.Up = connected return m.Run("set_link", args, nil) } // QuerySpice checks whether SPICE support is available in QEMU. func (m *Monitor) QuerySpice() error { return m.Run("query-spice", nil, nil) } // Query9pDevice checks whether virtio-9p-pci support is available in QEMU. func (m *Monitor) Query9pDevice() error { return m.Run("device-list-properties", map[string]string{"typename": "virtio-9p-pci"}, nil) } // QueryVirtioSoundDevice checks whether virtio-sound-pci support is available in QEMU. func (m *Monitor) QueryVirtioSoundDevice() error { return m.Run("device-list-properties", map[string]string{"typename": "virtio-sound-pci"}, nil) } // AddDirtyBitmap creates a dirty bitmap for a block device. func (m *Monitor) AddDirtyBitmap(deviceNames []string, bitmapName string, granularity int, persistent bool, disabled bool) error { actions := []TransactionAction{} for _, d := range deviceNames { data := map[string]any{ "node": d, "name": bitmapName, "persistent": persistent, "disabled": disabled, } if granularity > 0 { data["granularity"] = granularity } actions = append(actions, TransactionAction{Type: "block-dirty-bitmap-add", Data: data}) } err := m.RunTransaction(actions) if err != nil { return err } return nil } // RemoveDirtyBitmap removes a dirty bitmap for a block device. func (m *Monitor) RemoveDirtyBitmap(deviceName string, bitmapName string) error { var args struct { Node string `json:"node"` Name string `json:"name"` } args.Node = deviceName args.Name = bitmapName err := m.Run("block-dirty-bitmap-remove", args, nil) if err != nil { return err } return nil } incus-7.0.0/internal/server/instance/drivers/qmp/errors.go000066400000000000000000000007501517523235500237150ustar00rootroot00000000000000package qmp import ( "errors" ) // ErrMonitorDisconnect is returned when interacting with a disconnected Monitor. var ErrMonitorDisconnect = errors.New("Monitor is disconnected") // ErrMonitorBadConsole is returned when the requested console doesn't exist. var ErrMonitorBadConsole = errors.New("Requested console couldn't be found") // ErrNotARingbuf is returned when the requested device isn't a ring buffer. var ErrNotARingbuf = errors.New("Requested device isn't a ring buffer") incus-7.0.0/internal/server/instance/drivers/qmp/log.go000066400000000000000000000021531517523235500231610ustar00rootroot00000000000000package qmp import ( "fmt" "os" "sync" ) type qmpLog struct { logFile string log *os.File mu sync.Mutex } func newQmpLog(logFile string) (*qmpLog, error) { if logFile == "" { return nil, fmt.Errorf("Log file path is empty") } ql := &qmpLog{ logFile: logFile, } err := ql.open() if err != nil { return nil, err } return ql, nil } func (ql *qmpLog) open() error { if ql.log == nil { ql.mu.Lock() defer ql.mu.Unlock() log, err := os.OpenFile(ql.logFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o600) if err != nil { return err } ql.log = log } return nil } // Write writes len(b) bytes from b to the channel. func (ql *qmpLog) Write(p []byte) (n int, err error) { if ql == nil || ql.log == nil { return 0, nil } ql.mu.Lock() defer ql.mu.Unlock() // Ignore writes after close. if ql.log == nil { return 0, nil } return ql.log.Write(p) } // Close closes the log and wait the channel clean. func (ql *qmpLog) Close() error { if ql == nil || ql.log == nil { return nil } ql.mu.Lock() defer ql.mu.Unlock() err := ql.log.Close() ql.log = nil return err } incus-7.0.0/internal/server/instance/drivers/qmp/log_test.go000066400000000000000000000042321517523235500242200ustar00rootroot00000000000000package qmp import ( "fmt" "os" "path/filepath" "strings" "testing" "time" "golang.org/x/sync/errgroup" ) func tQmpLogSetup(t *testing.T) *qmpLog { t.Helper() logFile := filepath.Join(t.TempDir(), t.Name()+"_qmp.log") qlog, err := newQmpLog(logFile) if err != nil { t.Fatal(err) } t.Cleanup(func() { err := os.RemoveAll(logFile) if err != nil { t.Fatal(err) } }) return qlog } func TestNewQmpLog(t *testing.T) { qlog := tQmpLogSetup(t) err := qlog.Close() if err != nil { t.Fatal(err) } } func TestQmpLogWrite(t *testing.T) { qlog := tQmpLogSetup(t) command := `{"execute":"cont","id":26}` reply := `{"return": {}, "id": 26}` _, err := fmt.Fprintf(qlog, "[%s] QUERY: %s\n", time.Now().Format(time.RFC3339), command) if err != nil { t.Fatal(err) } _, err = fmt.Fprintf(qlog, "[%s] REPLY: %s\n\n", time.Now().Format(time.RFC3339), reply) if err != nil { t.Fatal(err) } b, err := os.ReadFile(qlog.logFile) if err != nil { t.Fatal(err) } s := string(b) if !strings.Contains(s, command) || !strings.Contains(s, reply) { t.Fatal(s) } err = qlog.Close() if err != nil { t.Fatal(err) } } func TestQmpLogClose(t *testing.T) { qlog := tQmpLogSetup(t) eg := errgroup.Group{} command := `{"execute":"cont","id":26}` reply := `{"return": {}, "id": 26}` event := `{"event":"STOP"}` // simulate run command logging eg.Go(func() error { _, err := fmt.Fprintf(qlog, "[%s] QUERY: %s\n", time.Now().Format(time.RFC3339), command) if err != nil { return err } _, err = fmt.Fprintf(qlog, "[%s] REPLY: %s\n\n", time.Now().Format(time.RFC3339), reply) if err != nil { return err } return nil }) eg.Go(func() error { for range 10 { _, err := fmt.Fprintf(qlog, "[%s] EVENT: %s\n\n", time.Now().Format(time.RFC3339), event) if err != nil { return err } } return nil }) err := eg.Wait() if err != nil { t.Fatal(err) } b, err := os.ReadFile(qlog.logFile) if err != nil { t.Fatal(err) } s := string(b) if !strings.Contains(s, command) || !strings.Contains(s, reply) || !strings.Contains(s, event) { t.Fatal(s) } err = qlog.Close() if err != nil { t.Fatal(err) } } incus-7.0.0/internal/server/instance/drivers/qmp/monitor.go000066400000000000000000000310431517523235500240670ustar00rootroot00000000000000package qmp import ( "context" "encoding/json" "errors" "fmt" "net" "path/filepath" "slices" "strings" "sync" "time" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) var ( monitors = map[string]*Monitor{} monitorsLock sync.Mutex ) // RingbufSize is the size of the agent serial ringbuffer in bytes. var RingbufSize = 16 // EventAgentStarted is the event sent once the agent has started. var EventAgentStarted = "AGENT-STARTED" // EventAgentStopped is the event sent once the agent has stopped. var EventAgentStopped = "AGENT-STOPPED" // EventVMReset is the event sent when VM guest reboots. var EventVMReset = "RESET" // EventVMShutdown is the event sent when VM guest shuts down. var EventVMShutdown = "SHUTDOWN" // EventVMShutdownReasonDisconnect is used as the reason when the shutdown event is triggered by a QMP disconnect. var EventVMShutdownReasonDisconnect = "disconnect" // EventVMShutdownReasonGuestShutdown is set when the guest cleanly shut down (e.g. ACPI powerdown honored). var EventVMShutdownReasonGuestShutdown = "guest-shutdown" // EventVMShutdownReasonQuit is set when QEMU exits as a result of the host issuing a QMP quit command. var EventVMShutdownReasonQuit = "quit" // EventDiskEjected is used to indicate that a disk device was ejected by the guest. var EventDiskEjected = "DEVICE_TRAY_MOVED" // EventRTCChange is used to get RTC adjustment. var EventRTCChange = "RTC_CHANGE" // EventBlockJobCompleted is emitted when a block job has completed. var EventBlockJobCompleted = "BLOCK_JOB_COMPLETED" // EventBlockJobError is emitted when a block job has errored. var EventBlockJobError = "BLOCK_JOB_ERROR" // ExcludedCommands is used to filter verbose commands from the QMP logs. var ExcludedCommands = []string{"ringbuf-read"} // Event represents a QMP event. type Event struct { Name string Data map[string]any } // Monitor represents a QMP monitor. type Monitor struct { path string qmp *qemuMachineProtocol agentStarted bool agentStartedMu sync.Mutex disconnected bool chDisconnect chan struct{} eventHandler func(name string, data map[string]any) serialCharDev string initialized bool expectingReset bool stateMu sync.Mutex detachDisk func(name string) error eventMap map[string]chan Event eventMapLock sync.Mutex } // TransactionAction represents a single action within a QMP transaction. type TransactionAction struct { Type string `json:"type"` Data map[string]any `json:"data"` } // start handles the background goroutines for event handling and monitoring the ringbuffer. func (m *Monitor) start() error { // Ringbuffer monitoring function. checkBuffer := func() { // Prepare the response. var resp struct { Return string `json:"return"` } // Read the ringbuffer. args := map[string]any{ "device": m.serialCharDev, "size": RingbufSize, "format": "utf8", } err := m.Run("ringbuf-read", args, &resp) if err != nil { return } // Extract the last entry. entries := strings.Split(resp.Return, "\n") if len(entries) > 1 { m.processAgentStatus(entries[len(entries)-2]) } } // Start event monitoring go routine. chEvents, err := m.qmp.getEvents(context.Background()) if err != nil { return err } go func() { logger.Debug("QMP monitor started", logger.Ctx{"path": m.path}) defer logger.Debug("QMP monitor stopped", logger.Ctx{"path": m.path}) // Initial read from the ringbuffer. go checkBuffer() for { // Wait for an event, disconnection or timeout. select { case <-m.chDisconnect: return case e, more := <-chEvents: // Handle media ejection. if e.Event == EventDiskEjected { id, ok := e.Data["id"].(string) if ok { // Only handle events that result in the tray being open. trayOpen, ok := e.Data["tray-open"].(bool) if ok && trayOpen { go func() { err = m.detachDisk(id) if err != nil { logger.Warnf("Unable to eject media %q: %v", id, err) } }() } } } // Deliver non-empty events to the event handler. if m.eventHandler != nil && e.Event != "" { if e.Event == EventVMShutdown { // Prefer shutdown event generated by QEMU (contains useful info) // and prevent duplicate shutdown event when monitor disconnects. m.SetInitialized(false) } go m.eventHandler(e.Event, e.Data) } // Event channel is closed, lets disconnect. if !more { // If disconnection happened unexpectedly then send shutdown event if // enabled so that VM devices can be cleaned up on the host. if m.IsInitialized() && m.eventHandler != nil { go m.eventHandler(EventVMShutdown, map[string]any{"reason": EventVMShutdownReasonDisconnect}) } m.Disconnect() return } if e.Event == "" { logger.Warnf("Unexpected empty event received from qmp event channel") time.Sleep(time.Second) // Don't spin if we receive a lot of these. continue } // Check if the ringbuffer was updated (non-blocking). go checkBuffer() case <-time.After(10 * time.Second): // Check if the ringbuffer was updated (non-blocking). go checkBuffer() continue } } }() return nil } // processAgentStatus handles a status string read from the agent vserial ringbuffer. func (m *Monitor) processAgentStatus(status string) { m.agentStartedMu.Lock() defer m.agentStartedMu.Unlock() switch status { case "STARTED": if !m.agentStarted && m.eventHandler != nil { go m.eventHandler(EventAgentStarted, nil) } m.agentStarted = true case "STOPPED": if m.agentStarted && m.eventHandler != nil { go m.eventHandler(EventAgentStopped, nil) } m.agentStarted = false } } // ping is used to validate if the QMP socket is still active. func (m *Monitor) ping() error { // Check if disconnected if m.disconnected { return ErrMonitorDisconnect } id := m.qmp.qmpIncreaseID() // Query the capabilities to validate the monitor. _, err := m.qmp.run(fmt.Appendf([]byte{}, `{"execute": "query-version", "id": %d}`, id), id) if err != nil { m.Disconnect() return ErrMonitorDisconnect } return nil } // RunJSON executes a JSON-formatted command. func (m *Monitor) RunJSON(request []byte, resp any, logCommand bool, id uint32) error { // Check if disconnected if m.disconnected || m.qmp == nil { return ErrMonitorDisconnect } var err error if logCommand && m.qmp.log != nil { _, err = fmt.Fprintf(m.qmp.log, "[%s] QUERY: %s\n", time.Now().Format(time.RFC3339), request) if err != nil { return err } } out, err := m.qmp.run(request, id) if err != nil { // Confirm the daemon didn't die. errPing := m.ping() if errPing != nil { return errPing } return err } if logCommand && m.qmp.log != nil { _, err = fmt.Fprintf(m.qmp.log, "[%s] REPLY: %s\n\n", time.Now().Format(time.RFC3339), out) if err != nil { return err } } // Decode the response if needed. if resp != nil { err = json.Unmarshal(out, &resp) if err != nil { // Confirm the daemon didn't die. errPing := m.ping() if errPing != nil { return errPing } return fmt.Errorf("Unexpected monitor response: %w (%q)", err, string(out)) } } return nil } // IncreaseID returns on auto increment uint32 id. func (m *Monitor) IncreaseID() uint32 { return m.qmp.qmpIncreaseID() } // run executes a command. func (m *Monitor) Run(cmd string, args any, resp any) error { id := m.IncreaseID() // Construct the command. requestArgs := qmpCommand{ ID: id, Execute: cmd, Arguments: args, } request, err := json.Marshal(requestArgs) if err != nil { return err } logCommand := !slices.Contains(ExcludedCommands, cmd) return m.RunJSON(request, resp, logCommand, id) } // RunTransaction executes a series of commands as a single transaction. func (m *Monitor) RunTransaction(actions []TransactionAction) error { var args struct { Actions []TransactionAction `json:"actions"` } args.Actions = actions err := m.Run("transaction", args, nil) if err != nil { return err } return nil } // Connect creates or retrieves an existing QMP monitor for the path. func Connect(path string, serialCharDev string, eventHandler func(name string, data map[string]any), logFile string, detachDisk func(name string) error) (*Monitor, error) { monitorsLock.Lock() defer monitorsLock.Unlock() // Look for an existing monitor. monitor, ok := monitors[path] if ok { monitor.eventHandler = eventHandler return monitor, nil } // Setup the connection. unixaddr, err := net.ResolveUnixAddr("unix", path) if err != nil { return nil, err } uc, err := net.DialUnix("unix", nil, unixaddr) if err != nil { return nil, err } qmpConn := &qemuMachineProtocol{uc: uc} if logFile != "" && util.PathExists(filepath.Dir(logFile)) { qlog, err := newQmpLog(logFile) if err != nil { return nil, err } qmpConn.log = qlog } chError := make(chan error, 1) go func() { err = qmpConn.connect() chError <- err }() select { case err := <-chError: if err != nil { return nil, err } case <-time.After(5 * time.Second): _ = qmpConn.disconnect() return nil, errors.New("QMP connection timed out") } // Setup the monitor struct. monitor = &Monitor{} monitor.path = path monitor.qmp = qmpConn monitor.chDisconnect = make(chan struct{}, 1) monitor.eventHandler = eventHandler monitor.serialCharDev = serialCharDev monitor.detachDisk = detachDisk monitor.eventMap = make(map[string]chan Event) // Default to generating a shutdown event when the monitor disconnects so that devices can be // cleaned up. This will be disabled after a shutdown event is received from QEMU itself to avoid // causing multiple events. monitor.initialized = true // Spawn goroutines. err = monitor.start() if err != nil { return nil, err } // Register in global map. monitors[path] = monitor return monitor, nil } // AgenStarted indicates whether an agent has been detected. func (m *Monitor) AgenStarted() bool { m.agentStartedMu.Lock() defer m.agentStartedMu.Unlock() return m.agentStarted } // Disconnect forces a disconnection from QEMU. func (m *Monitor) Disconnect() { monitorsLock.Lock() defer monitorsLock.Unlock() // Stop all go routines and disconnect from socket. if !m.disconnected { close(m.chDisconnect) m.disconnected = true _ = m.qmp.disconnect() } // Remove from the map. delete(monitors, m.path) } // Wait returns a channel that will be closed on disconnection. func (m *Monitor) Wait() (chan struct{}, error) { // Check if disconnected if m.disconnected { return nil, ErrMonitorDisconnect } return m.chDisconnect, nil } // SetInitialized enables or disables the on disconnect event. func (m *Monitor) SetInitialized(enable bool) { m.stateMu.Lock() defer m.stateMu.Unlock() m.initialized = enable } // IsInitialized reports whether the monitor believes the VM has been fully initialized. func (m *Monitor) IsInitialized() bool { m.stateMu.Lock() defer m.stateMu.Unlock() return m.initialized } // ExpectReset marks the monitor as expecting a single RESET event triggered by us // (for example the deliberate system_reset issued during VM startup). The next RESET // event observed by the handler can be consumed via HandleReset. func (m *Monitor) ExpectReset() { m.stateMu.Lock() defer m.stateMu.Unlock() m.expectingReset = true } // HandleReset returns true and clears the flag if a deliberate reset // was previously announced via ExpectReset. It returns false otherwise. func (m *Monitor) HandleReset() bool { m.stateMu.Lock() defer m.stateMu.Unlock() if !m.expectingReset { return false } m.expectingReset = false return true } // CreateEventChannel creates and registers a new event channel for the given device. func (m *Monitor) CreateEventChannel(deviceName string) (chan Event, error) { m.eventMapLock.Lock() defer m.eventMapLock.Unlock() _, ok := m.eventMap[deviceName] if ok { return nil, fmt.Errorf("Event channel already exists for %s", deviceName) } ch := make(chan Event) m.eventMap[deviceName] = ch return ch, nil } // CleanupEventChannel removes the event channel associated with the given device. func (m *Monitor) CleanupEventChannel(deviceName string) { m.eventMapLock.Lock() defer m.eventMapLock.Unlock() ch, ok := m.eventMap[deviceName] if ok { delete(m.eventMap, deviceName) } if ok { close(ch) } } // PushEvent publishes an event to the registered channel for the given device. func (m *Monitor) PushEvent(event string, data map[string]any) { m.eventMapLock.Lock() defer m.eventMapLock.Unlock() ch, ok := m.eventMap[data["device"].(string)] if ok { ch <- Event{Name: event, Data: data} } } incus-7.0.0/internal/server/instance/drivers/qmp/qmp.go000066400000000000000000000170731517523235500232040ustar00rootroot00000000000000package qmp import ( "bufio" "context" "encoding/json" "errors" "fmt" "io" "net" "os" "slices" "sync" "sync/atomic" "time" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/shared/logger" ) type qemuMachineProtocol struct { oobSupported bool // Out of band support or not uc *net.UnixConn // Underlying unix socket connection mu sync.Mutex // Serialize running command replies sync.Map // Replies channels events <-chan qmpEvent // Events channel listeners atomic.Uint32 // Listeners number cid atomic.Uint32 // Auto increase command id log *qmpLog // qmp log } // qmpEvent represents a QEMU QMP event. type qmpEvent struct { // Event name, e.g., BLOCK_JOB_COMPLETE Event string `json:"event"` // Arbitrary event data Data map[string]any `json:"data"` // Event timestamp, provided by QEMU. Timestamp *struct { Seconds int64 `json:"seconds"` Microseconds int64 `json:"microseconds"` } `json:"timestamp"` } // qmpCommand represents a QMP command. type qmpCommand struct { // Name of the command to run Execute string `json:"execute,omitempty"` // Name of the Out-off-band execution to run ExecuteOutOfBand string `json:"exec-oob,omitempty"` // Optional arguments for the above command. Arguments any `json:"arguments,omitempty"` // Optional id for transaction identification associated with the command // execution // // According QMP spec it should be any json value type. For incus `uint32` // (skip zero) is good enough to identify transaction. ID uint32 `json:"id,omitempty"` } // qmpResponse represents a QMP response with id and return. type qmpResponse struct { // Optional id for transaction identification associated with the response ID uint32 `json:"id,omitempty"` // Return response return Return any `json:"return,omitempty"` } // qmpError represents a QMP response error. type qmpError struct { Class string `json:"class,omitempty"` Desc string `json:"desc,omitempty"` } func (e *qmpError) Error() string { if e == nil { return "" } return fmt.Sprintf("%s: %s", e.Class, e.Desc) } // rawResponse represents QMP raw response with id, error and raw bytes. type rawResponse struct { // Optional id for transaction identification associated with the response ID uint32 `json:"id"` // Error response error Error *qmpError `json:"error,omitempty"` raw []byte // raw data, json field ignored err error // runtime error, json field ignored } // disconnect closes the QEMU monitor socket connection. func (qmp *qemuMachineProtocol) disconnect() error { qmp.listeners.Store(0) if qmp.log != nil { err := qmp.log.Close() if err != nil { return err } qmp.log = nil } return qmp.uc.Close() } // qmpIncreaseID increase ID and skip zero. func (qmp *qemuMachineProtocol) qmpIncreaseID() uint32 { id := qmp.cid.Add(1) if id == 0 { id = qmp.cid.Add(1) } return id } // connect sets up a QMP connection. func (qmp *qemuMachineProtocol) connect() error { enc := json.NewEncoder(qmp.uc) dec := json.NewDecoder(qmp.uc) // Check for banner on startup ban := struct { QMP struct { Capabilities []string `json:"capabilities"` } `json:"QMP"` }{} err := dec.Decode(&ban) if err != nil { return err } qmp.oobSupported = slices.Contains(ban.QMP.Capabilities, "oob") // Issue capabilities handshake id := qmp.qmpIncreaseID() cmd := qmpCommand{Execute: "qmp_capabilities", ID: id} err = enc.Encode(cmd) if err != nil { return err } // Check for no error on return r := &rawResponse{} err = dec.Decode(r) if err != nil { return err } if r.Error != nil { return r.Error } if r.ID != id { return fmt.Errorf("reply id %d and command id %d mismatch", r.ID, id) } // Initialize listener for command responses and asynchronous events. events := make(chan qmpEvent, 128) go qmp.listen(qmp.uc, events, &qmp.replies) qmp.events = events return nil } // getEvents streams QEMU QMP Events. func (qmp *qemuMachineProtocol) getEvents(context.Context) (<-chan qmpEvent, error) { qmp.listeners.Add(1) return qmp.events, nil } func (qmp *qemuMachineProtocol) listen(r io.Reader, events chan<- qmpEvent, replies *sync.Map) { defer close(events) scanner := bufio.NewScanner(r) for scanner.Scan() { var e qmpEvent b := scanner.Bytes() err := json.Unmarshal(b, &e) if err != nil { continue } // If data does not have an event type, it must be in response to a command. if e.Event == "" { r := rawResponse{} err = json.Unmarshal(b, &r) if err != nil { continue } key := r.ID if key == 0 { // Discard response without a request ID. continue } val, ok := replies.LoadAndDelete(key) if !ok { // Discard unexpected response. continue } reply, ok := val.(chan rawResponse) if !ok { // Skip bad messages. logger.Error("Failed to cast QMP reply to chan rawResponse") continue } r.raw = make([]byte, len(b)) copy(r.raw, b) reply <- r continue } if qmp.log != nil && !slices.Contains([]string{"VSERPORT_CHANGE"}, e.Event) { _, err := fmt.Fprintf(qmp.log, "[%s] Event: %s\n", time.Now().Format(time.RFC3339), b) if err != nil { logger.Debugf("Failed to log event: %v", err) } } // If nobody is listening for events, do not bother sending them. if qmp.listeners.Load() == 0 { continue } events <- e } err := scanner.Err() if err == nil { err = errors.New("Monitor has exited") } // Return the error to all existing requests. replies.Range(func(k any, v any) bool { reply, ok := v.(chan rawResponse) if !ok { // Skip bad messages. logger.Error("Failed to cast QMP reply to chan rawResponse") return true } reply <- rawResponse{err: err} return true }) // Clear the map. replies.Clear() } // run executes the given QAPI command against a domain's QEMU instance. func (qmp *qemuMachineProtocol) run(command []byte, id uint32) ([]byte, error) { // Just call RunWithFile with no file return qmp.runWithFile(command, nil, id) } func (qmp *qemuMachineProtocol) qmpWriteMsg(b []byte, file *os.File) error { if file == nil { // Just send a normal command through. _, err := qmp.uc.Write(b) return err } if !qmp.oobSupported { return errors.New("The QEMU server doesn't support oob (needed for RunWithFile)") } // Send the command along with the file descriptor. oob := unix.UnixRights(int(file.Fd())) _, _, err := qmp.uc.WriteMsgUnix(b, oob, nil) if err != nil { return err } return nil } // runWithFile executes for passing a file through out-of-band data. func (qmp *qemuMachineProtocol) runWithFile(command []byte, file *os.File, id uint32) ([]byte, error) { // Only allow a single command to be run at a time to ensure that responses // to a command cannot be mixed with responses from another command qmp.mu.Lock() defer qmp.mu.Unlock() if id == 0 { id = qmp.qmpIncreaseID() b, err := qmp.qmpInjectID(command, id) if err != nil { return nil, err } command = b } repCh := make(chan rawResponse, 1) qmp.replies.Store(id, repCh) err := qmp.qmpWriteMsg(command, file) if err != nil { qmp.replies.Delete(id) return nil, err } // Wait for a response or error to our command res := <-repCh if res.err != nil { return nil, res.err } if res.Error != nil { return nil, res.Error } return res.raw, nil } func (qmp *qemuMachineProtocol) qmpInjectID(command []byte, id uint32) ([]byte, error) { req := &qmpCommand{} err := json.Unmarshal(command, req) if err != nil { return nil, err } req.ID = id b, err := json.Marshal(req) if err != nil { return nil, err } return b, nil } incus-7.0.0/internal/server/instance/drivers/qmp/qmp_test.go000066400000000000000000000125731517523235500242430ustar00rootroot00000000000000package qmp import ( "context" "encoding/json" "errors" "fmt" "net" "path/filepath" "reflect" "strings" "testing" "time" "golang.org/x/sync/errgroup" ) var testingGreeting = map[string]any{ "QMP": map[string]any{ "version": map[string]any{ "qemu": map[string]any{ "micro": 2, "minor": 2, "major": 9, }, "package": "v9.2.2", }, "capabilities": []string{"oob"}, }, } type testingErrReader struct { err error } func (r *testingErrReader) Read(b []byte) (int, error) { return 0, r.err } func TestConnectDisconnect(t *testing.T) { eg := &errgroup.Group{} m := &qemuMachineProtocol{} mockMonitorServer(t, eg, m) err := m.connect() if err != nil { t.Fatal(err) } err = m.disconnect() if err != nil { t.Fatal(err) } err = eg.Wait() if err != nil { t.Fatal(err) } } func TestEvents(t *testing.T) { eg := &errgroup.Group{} es := []qmpEvent{ {Event: "STOP"}, {Event: "SHUTDOWN"}, {Event: "RESET"}, } m := &qemuMachineProtocol{} mockMonitorServer(t, eg, m, func(nc net.Conn) error { enc := json.NewEncoder(nc) for i, e := range es { err := enc.Encode(e) if err != nil { t.Log(i, e, err) return err } } return nil }) err := m.connect() if err != nil { t.Fatal(err) } events, err := m.getEvents(context.Background()) if err != nil { t.Fatalf("unexpected error: %v", err) } for i, want := range es { got := <-events if !reflect.DeepEqual(want, got) { t.Fatal(i, want, got) } } err = eg.Wait() if err != nil { t.Fatal(err) } } func TestListenEmptyStream(t *testing.T) { mon := &qemuMachineProtocol{} r := strings.NewReader("") events := make(chan qmpEvent) replies := &mon.replies mon.listen(r, events, replies) _, ok := <-events if ok { t.Fatal("events channel should be closed") } replies.Range(func(key, value any) bool { t.Fatal("replies should be empty") return false }) } func TestListenScannerErr(t *testing.T) { mon := &qemuMachineProtocol{} errFoo := errors.New("foo") r := &testingErrReader{err: errFoo} events := make(chan qmpEvent) replies := &mon.replies repCh := make(chan rawResponse, 1) replies.Store(0, repCh) mon.listen(r, events, replies) res := <-repCh if errFoo != res.err { t.Fatalf("unexpected error:\n- want: %v\n- got: %v", errFoo, res.err) } } func TestListenInvalidJson(t *testing.T) { mon := &qemuMachineProtocol{} r := strings.NewReader("") events := make(chan qmpEvent) replies := &mon.replies mon.listen(r, events, replies) replies.Range(func(key, value any) bool { t.Fatal("replies should be empty") return false }) } func TestListenStreamResponse(t *testing.T) { mon := &qemuMachineProtocol{} id := uint32(1) want := `{"foo": "bar", "id": 1}` r := strings.NewReader(want) events := make(chan qmpEvent) replies := &mon.replies repCh := make(chan rawResponse, 1) replies.Store(id, repCh) go mon.listen(r, events, replies) res := <-repCh if res.err != nil { t.Fatalf("unexpected error: %v", res.err) } got := string(res.raw) if want != got { t.Fatalf("unexpected response:\n- want: %q\n- got: %q", want, got) } } func TestListenEventNoListeners(t *testing.T) { mon := &qemuMachineProtocol{} r := strings.NewReader(`{"event":"STOP"}`) events := make(chan qmpEvent) replies := &mon.replies go mon.listen(r, events, replies) _, ok := <-events if ok { t.Fatal("events channel should be closed") } } func TestListenEventOneListener(t *testing.T) { mon := &qemuMachineProtocol{} mon.listeners.Store(1) eventStop := "STOP" r := strings.NewReader(fmt.Sprintf(`{"event":%q}`, eventStop)) events := make(chan qmpEvent) replies := &mon.replies go mon.listen(r, events, replies) e := <-events want, got := eventStop, e.Event if want != got { t.Fatalf("unexpected event:\n- want: %q\n- got: %q", want, got) } } func mockMonitorServer(t *testing.T, eg *errgroup.Group, qmp *qemuMachineProtocol, hands ...func(net.Conn) error) { t.Helper() unixsock := filepath.Join(t.TempDir(), "mockmonitor.sock") unixaddr, err := net.ResolveUnixAddr("unix", unixsock) if err != nil { t.Fatal(err) } l, err := net.ListenUnix("unix", unixaddr) if err != nil { t.Fatal(err) } eg.Go(func() error { tc, err := l.Accept() if err != nil { t.Log(err) return err } enc := json.NewEncoder(tc) dec := json.NewDecoder(tc) err = enc.Encode(testingGreeting) if err != nil { t.Logf("unexpected error: %v", err) return err } var cmd qmpCommand err = dec.Decode(&cmd) if err != nil { err = fmt.Errorf("unexpected error: %w", err) t.Log(err) return err } if cmd.Execute != "qmp_capabilities" { err = fmt.Errorf("unexpected capabilities handshake:\n- want: %q\n- got: %q", "qmp_capabilities", cmd.Execute) t.Log(err) return err } err = enc.Encode(qmpResponse{ID: cmd.ID}) if err != nil { err = fmt.Errorf("unexpected error: %w", err) t.Log(err) return err } // wait client listen ready for qmp.events == nil { time.Sleep(time.Millisecond * 10) } for i, hand := range hands { err = hand(tc) if err != nil { t.Log(i, err) return err } } return err }) for qmp.uc == nil { uc, err := net.DialUnix("unix", nil, unixaddr) if err != nil { // Wait unix socket being created t.Log(err) time.Sleep(time.Millisecond * 10) continue } qmp.uc = uc } if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = l.Close() }) } incus-7.0.0/internal/server/instance/drivers/test_util.go000066400000000000000000000010631517523235500236160ustar00rootroot00000000000000package drivers import ( "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" ) // PrepareEqualTest modifies any unexported variables required for reflect.DeepEqual to complete safely. // This is used for tests to avoid infinite recursion loops. func PrepareEqualTest(insts ...instance.Instance) { for _, inst := range insts { if inst.Type() == instancetype.Container { // When loading from DB, we won't have a full LXC config. inst.(*lxc).c = nil inst.(*lxc).cConfig = false } } } incus-7.0.0/internal/server/instance/drivers/util.go000066400000000000000000000233611517523235500225640ustar00rootroot00000000000000package drivers import ( "context" "crypto/sha256" "encoding/base64" "errors" "fmt" "io/fs" "os" "regexp" "slices" "sort" "strconv" "strings" "time" yaml "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/instance/drivers/cfg" "github.com/lxc/incus/v7/internal/server/instance/drivers/qmp" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/state" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/units" ) // GetClusterCPUFlags returns the list of shared CPU flags across. func GetClusterCPUFlags(ctx context.Context, s *state.State, servers []string, archName string) ([]string, error) { // Get the list of cluster members. var nodes []db.RaftNode err := s.DB.Node.Transaction(ctx, func(ctx context.Context, tx *db.NodeTx) error { var err error nodes, err = tx.GetRaftNodes(ctx) return err }) if err != nil { return nil, err } // Get all the CPU flags for the architecture. flagMembers := map[string]int{} coreCount := 0 for _, node := range nodes { // Skip if not in the list of servers we're interested in. if servers != nil && !slices.Contains(servers, node.Name) { continue } // Get node resources. res, err := getNodeResources(s, node.Name, node.Address) if err != nil { logger.Errorf("Failed to get resources for CPU baseline on %q: %v", node.Name, err) continue } // Skip if not the correct architecture. if res.CPU.Architecture != archName { continue } // Add the CPU flags to the map. for _, socket := range res.CPU.Sockets { for _, core := range socket.Cores { coreCount += 1 for _, flag := range core.Flags { flagMembers[flag] += 1 } } } } // Get the host flags. info := DriverStatuses()[instancetype.VM].Info hostFlags, ok := info.Features["flags"].(map[string]bool) if !ok { // No CPU flags found. return nil, nil } // Build a set of flags common to all cores. flags := []string{} for k, v := range flagMembers { if v != coreCount { continue } hostVal, ok := hostFlags[k] if !ok || hostVal { continue } flags = append(flags, k) } return flags, nil } // ParseMemoryStr parses a human representation of memory value as int64 type. func ParseMemoryStr(memory string) (valueInt int64, err error) { if strings.HasSuffix(memory, "%") { var percent, memoryTotal int64 percent, err = strconv.ParseInt(strings.TrimSuffix(memory, "%"), 10, 64) if err != nil { return 0, err } memoryTotal, err = linux.DeviceTotalMemory() if err != nil { return 0, err } valueInt = (memoryTotal / 100) * percent } else { valueInt, err = units.ParseByteSizeString(memory) } return valueInt, err } func qemuEscapeCmdline(value string) string { return strings.ReplaceAll(value, ",", ",,") } // roundDownToBlockSize returns the largest multiple of blockSize less than or equal to the input value. func roundDownToBlockSize(value int64, blockSize int64) int64 { if value%blockSize == 0 { return value } return ((value / blockSize) - 1) * blockSize } // memoryConfigSectionToMap converts a memory object of type cfg.Section to type map[string]any. func memoryConfigSectionToMap(section *cfg.Section) map[string]any { const blockSize = 128 * 1024 * 1024 // 128MiB obj := map[string]any{} hostNodes := []int{} for key, value := range section.Entries { if strings.HasPrefix(key, "host-nodes") { hostNode, err := strconv.Atoi(value) if err != nil { continue } hostNodes = append(hostNodes, hostNode) } else if key == "size" { // Size in the config is specified in the format: 1024M, so the last character needs to be removed before parsing. memSizeMB, err := strconv.Atoi(value[:len(value)-1]) if err != nil { continue } obj["size"] = roundDownToBlockSize(int64(memSizeMB)*1024*1024, blockSize) } else if key == "merge" || key == "dump" || key == "prealloc" || key == "share" || key == "reserve" { val := false if value == "on" { val = true } obj[key] = val } else { obj[key] = value } } if len(hostNodes) > 0 { obj["host-nodes"] = hostNodes } return obj } // extractTrailingNumber extracts the trailing number from a string. // For example, given "dimm1", it returns 1. func extractTrailingNumber(s string, prefix string) (int, error) { if !strings.HasPrefix(s, prefix) { return -1, fmt.Errorf("Prefix %s not found in %s", prefix, s) } trimmed := strings.TrimPrefix(s, prefix) num, err := strconv.Atoi(trimmed) if err != nil { return -1, err } return num, nil } // findNextDimmIndex finds the next available index for a pc-dimm device // whose ID starts with the prefix "dimm". func findNextDimmIndex(monitor *qmp.Monitor) (int, error) { devices, err := monitor.GetDimmDevices() if err != nil { return -1, err } index := -1 for _, dev := range devices { i, err := extractTrailingNumber(dev.ID, "dimm") if err != nil { continue } if i > index { index = i } } return index + 1, nil } // findNextMemoryIndex finds the next available index for a memory object // whose ID starts with the prefix "mem". func findNextMemoryIndex(monitor *qmp.Monitor) (int, error) { memDevs, err := monitor.GetMemdev() if err != nil { return -1, err } memIndex := -1 for _, mem := range memDevs { var index int index, err := extractTrailingNumber(mem.ID, "mem") if err != nil { continue } if index > memIndex { memIndex = index } } return memIndex + 1, nil } // getNodeResources updates the cluster resource cache.. func getNodeResources(s *state.State, name string, address string) (*api.Resources, error) { resourcesPath := internalUtil.CachePath("resources", fmt.Sprintf("%s.yaml", name)) // Check if cache is recent (less than 24 hours). fi, err := os.Stat(resourcesPath) if err == nil && time.Since(fi.ModTime()) < 24*time.Hour { data, err := os.ReadFile(resourcesPath) if err == nil { var res api.Resources if yaml.Load(data, &res) == nil { return &res, nil } } } else if err != nil && !errors.Is(err, fs.ErrNotExist) { return nil, err } var res *api.Resources if name == s.ServerName { // Handle the local node. // We still cache the data as it's not particularly cheap to get. res, err = resources.GetResources() if err != nil { return nil, err } } else { // Handle remote nodes. client, err := cluster.Connect(address, s.Endpoints.NetworkCert(), s.ServerCert(), nil, true) if err != nil { return nil, err } res, err = client.GetServerResources() if err != nil { return nil, err } } // Cache the data. data, err := yaml.Dump(res, yaml.V2) if err == nil { _ = os.WriteFile(resourcesPath, data, 0o600) } return res, nil } type qcow2BlockdevKind int const ( backingBlockdevKind qcow2BlockdevKind = iota rootBlockdevKind overlayBlockdevKind ) type qcow2BlockdevInfo struct { name string kind qcow2BlockdevKind index int } // classifyQcow2Blockdev classifies a block device as a qcow2 backing, root, or overlay device. func classifyQcow2Blockdev(name string, rootDevName string) (*qcow2BlockdevInfo, bool) { reBacking := regexp.MustCompile(fmt.Sprintf(`^%s_backing(\d+)$`, rootDevName)) reOverlay := regexp.MustCompile(fmt.Sprintf(`^%s_overlay(\d+)$`, rootDevName)) if name == rootDevName { return &qcow2BlockdevInfo{name: name, kind: rootBlockdevKind, index: 0}, true } m := reBacking.FindStringSubmatch(name) if m != nil { i, _ := strconv.Atoi(m[1]) return &qcow2BlockdevInfo{name: name, kind: backingBlockdevKind, index: i}, true } m = reOverlay.FindStringSubmatch(name) if m != nil { i, _ := strconv.Atoi(m[1]) return &qcow2BlockdevInfo{name: name, kind: overlayBlockdevKind, index: i}, true } return nil, false } // filterAndSortQcow2Blockdevs selects qcow2 related block devices and sorts them in the correct order. // Backing blockdevs with higher indices represent older layers, while higher-index overlays represent newer layers. func filterAndSortQcow2Blockdevs(names []string, rootDevName string) []string { items := make([]qcow2BlockdevInfo, 0, len(names)) for _, n := range names { info, ok := classifyQcow2Blockdev(n, rootDevName) if ok { items = append(items, *info) } } sort.Slice(items, func(i, j int) bool { if items[i].kind != items[j].kind { return items[i].kind < items[j].kind } if items[i].kind == backingBlockdevKind { return items[i].index > items[j].index } return items[i].index < items[j].index }) result := make([]string, len(items)) for i, it := range items { result[i] = it.name } return result } // hashValue returns a hash of the name if it exceeds the given length limit. // Otherwise, it returns the original name unchanged. func hashValue(value string, maxLength int) string { if len(value) > maxLength { // If the name is too long, hash it as SHA-256 (32 bytes). // Then encode the SHA-256 binary hash as Base64 Raw URL format and trim down to 'maxLength' chars. // Raw URL avoids the use of "+" character and the padding "=" character which QEMU doesn't allow. hash256 := sha256.New() hash256.Write([]byte(value)) binaryHash := hash256.Sum(nil) value = base64.RawURLEncoding.EncodeToString(binaryHash) value = value[0:maxLength] } return value } // ephemeralSnapshotName returns a snapshot name derived from the disk name. func ephemeralSnapshotName(diskName string) string { return fmt.Sprintf("%s_snap", diskName) } // migrationNBDTarget returns a name for a disk exposed via the NBD server. func migrationNBDTarget(diskName string) string { return fmt.Sprintf("%s_nbd", diskName) } incus-7.0.0/internal/server/instance/drivers/util_test.go000066400000000000000000000040741517523235500236230ustar00rootroot00000000000000package drivers import ( "testing" "github.com/stretchr/testify/assert" "github.com/lxc/incus/v7/internal/server/instance/drivers/cfg" ) // Test roundDownToBlockSize. func TestRoundDownToBlockSize(t *testing.T) { const blockSize = 128 * 1024 * 1024 value := roundDownToBlockSize(1073741824, blockSize) assert.Equal(t, int64(1073741824), value) value = roundDownToBlockSize(1000000000, blockSize) assert.Equal(t, int64(805306368), value) } // Test memoryConfigSectionToMap. func TestMemoryConfigSectionToMap(t *testing.T) { result := memoryConfigSectionToMap( &cfg.Section{ Name: "object \"mem0\"", Entries: map[string]string{ "size": "1024M", "host-nodes.0": "0", "host-nodes.1": "1", "policy": "bind", "share": "on", }, }, ) expected := map[string]any{ "size": int64(1073741824), "host-nodes": []int{0, 1}, "policy": "bind", "share": true, } assert.Equal(t, expected["size"], result["size"]) assert.ElementsMatch(t, expected["host-nodes"], result["host-nodes"]) assert.Equal(t, expected["policy"], result["policy"]) assert.Equal(t, expected["share"], result["share"]) } // Test extractTraiingNumber. func TestExtractTrailingNumber(t *testing.T) { value, _ := extractTrailingNumber("mem0", "mem") assert.Equal(t, 0, value) value, _ = extractTrailingNumber("mem34", "mem") assert.Equal(t, 34, value) value, _ = extractTrailingNumber("dimm1", "dimm") assert.Equal(t, 1, value) expectedErr := "Prefix mem not found in dimm1" _, err := extractTrailingNumber("dimm1", "mem") if err.Error() != expectedErr { t.Errorf("unexpected error message: got %q, want %q", err.Error(), expectedErr) } } // Test hashValue. func TestHashValue(t *testing.T) { value := hashValue("test", 5) assert.Equal(t, "test", value) value = hashValue("test1", 5) assert.Equal(t, "test1", value) value = hashValue("test12", 5) assert.Equal(t, "qY7Fx", value) value = hashValue("test12345", 11) assert.Equal(t, "test12345", value) value = hashValue("test12345678", 11) assert.Equal(t, "9fvG_oTDZTF", value) } incus-7.0.0/internal/server/instance/filter.go000066400000000000000000000010641517523235500214120ustar00rootroot00000000000000package instance import ( "github.com/lxc/incus/v7/internal/filter" "github.com/lxc/incus/v7/shared/api" ) // FilterFull returns a filtered list of full instances that match the given clauses. func FilterFull(instances []*api.InstanceFull, clauses filter.ClauseSet) ([]*api.InstanceFull, error) { filtered := []*api.InstanceFull{} for _, instance := range instances { match, err := filter.Match(*instance, clauses) if err != nil { return nil, err } if !match { continue } filtered = append(filtered, instance) } return filtered, nil } incus-7.0.0/internal/server/instance/instance_errors.go000066400000000000000000000002231517523235500233210ustar00rootroot00000000000000package instance import ( "errors" ) // ErrNotImplemented is the "Not implemented" error. var ErrNotImplemented = errors.New("Not implemented") incus-7.0.0/internal/server/instance/instance_exec_cmd.go000066400000000000000000000003671517523235500235650ustar00rootroot00000000000000package instance import ( "golang.org/x/sys/unix" ) // Cmd represents a local or remote command being run. type Cmd interface { Wait() (int, error) PID() int Signal(s unix.Signal) error WindowResize(fd, winchWidth, winchHeight int) error } incus-7.0.0/internal/server/instance/instance_interface.go000066400000000000000000000201101517523235500237420ustar00rootroot00000000000000package instance import ( "context" "crypto/x509" "io" "net" "os" "time" liblxc "github.com/lxc/go-lxc" "github.com/pkg/sftp" "google.golang.org/protobuf/proto" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/cgroup" "github.com/lxc/incus/v7/internal/server/db" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/instance/operationlock" "github.com/lxc/incus/v7/internal/server/metrics" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/ioprogress" ) // HookStart hook used when instance has started. const HookStart = "onstart" // HookStopNS hook used when instance has stopped but before namespaces have been destroyed. const HookStopNS = "onstopns" // HookStop hook used when instance has stopped. const HookStop = "onstop" // Possible values for the protocol argument of the Instance.Console() method. const ( ConsoleTypeConsole = "console" ConsoleTypeVGA = "vga" ) // TemplateTrigger trigger name. type TemplateTrigger string // TemplateTriggerCreate for when an instance is created. const TemplateTriggerCreate TemplateTrigger = "create" // TemplateTriggerCopy for when an instance is copied. const TemplateTriggerCopy TemplateTrigger = "copy" // TemplateTriggerRename for when an instance is renamed. const TemplateTriggerRename TemplateTrigger = "rename" // PowerStateRunning represents the power state stored when an instance is running. const PowerStateRunning = "RUNNING" // PowerStateStopped represents the power state stored when an instance is stopped. const PowerStateStopped = "STOPPED" // ConfigReader is used to read instance config. type ConfigReader interface { Project() api.Project Type() instancetype.Type Architecture() int ID() int Name() string ExpandedConfig() map[string]string ExpandedDevices() deviceConfig.Devices LocalConfig() map[string]string LocalDevices() deviceConfig.Devices } // Instance interface. type Instance interface { ConfigReader // Instance actions. Freeze() error Shutdown(timeout time.Duration) error Start(stateful bool) error Stop(stateful bool) error Restart(timeout time.Duration) error Rebuild(img *api.Image, op *operations.Operation) error Unfreeze() error ReloadDevice(devName string) error RegisterDevices() Info() Info IsPrivileged() bool // Dependent volumes HasDependentDisk() bool ForEachDependentDiskType(diskAction func(dev deviceConfig.DeviceNamed) error) error // Snapshots & migration & backups. Restore(source Instance, stateful bool, diskOnly bool) error Snapshot(name string, expiry time.Time, stateful bool) error Snapshots() ([]Instance, error) Backups() ([]backup.InstanceBackup, error) UpdateBackupFile() error CanLiveMigrate() bool CreateQcow2Snapshot(diskPath string, devName string, snapshotName string, backingFilename string, stateful bool) error DeleteQcow2Snapshot(devName string, snapshotIndex int, backingFilename string) error ExportQcow2Block(diskName string, blockIndex int) (func(), string, error) ConnectNBD(diskName string, diskSize int64, writable bool) (net.Conn, func(), error) // Config handling. Rename(newName string, applyTemplateTrigger bool) error Update(newConfig db.InstanceArgs, userRequested bool) error UpdateDevices(devices deviceConfig.Devices) error Delete(force bool, cleanupDependencies bool) error Export(meta io.Writer, roofs io.Writer, properties map[string]string, expiration time.Time, tracker *ioprogress.ProgressTracker) (*api.ImageMetadata, error) // Live configuration. CGroup() (*cgroup.CGroup, error) VolatileSet(changes map[string]string) error // File handling. FileSFTPConn() (net.Conn, error) FileSFTP() (*sftp.Client, error) // Console - Allocate and run a console tty or a spice Unix socket. Console(protocol string) (*os.File, chan error, error) Exec(req api.InstanceExecPost, stdin *os.File, stdout *os.File, stderr *os.File) (Cmd, error) // Status Render() (any, any, error) RenderWithUsage() (any, any, error) RenderFull(hostInterfaces []net.Interface) (*api.InstanceFull, any, error) RenderState(hostInterfaces []net.Interface) (*api.InstanceState, error) IsRunning() bool IsFrozen() bool IsEphemeral() bool IsSnapshot() bool IsStateful() bool LockExclusive() (*operationlock.InstanceOperation, error) // Hooks. DeviceEventHandler(*deviceConfig.RunConfig) error OnHook(hookName string, args map[string]string) error // Properties. Location() string CloudInitID() string Description() string CreationDate() time.Time LastUsedDate() time.Time GuestOS() string Profiles() []api.Profile InitPID() int State() string ExpiryDate() time.Time FillNetworkDevice(name string, m deviceConfig.Device) (deviceConfig.Device, error) ETag() []any MACPattern() string // Paths. Path() string ExecOutputPath() string RootfsPath() string TemplatesPath() string StatePath() string LogFilePath() string ConsoleBufferLogPath() string LogPath() string RunPath() string DevicesPath() string // Storage. StoragePool() (string, error) // Migration. CanMigrate() string MigrateSend(args MigrateSendArgs) error MigrateReceive(args MigrateReceiveArgs) error // Progress reporting. SetOperation(op *operations.Operation) Operation() *operations.Operation DeferTemplateApply(trigger TemplateTrigger) error Metrics(hostInterfaces []net.Interface) (*metrics.MetricSet, error) // Bitmaps. CreateBitmap(deviceNames []string, data api.StorageVolumeBitmapsPost) error DeleteBitmap(deviceName string, bitmapName string) error GetBitmaps(deviceName string) ([]api.StorageVolumeBitmap, error) } // Container interface is for container specific functions. type Container interface { Instance CurrentIdmap() (*idmap.Set, error) DiskIdmap() (*idmap.Set, error) NextIdmap() (*idmap.Set, error) ConsoleLog(opts liblxc.ConsoleLogOptions) (string, error) InsertSeccompUnixDevice(prefix string, m deviceConfig.Device, pid int) error DevptsFd() (*os.File, error) IdmappedStorage(path string, fstype string) idmap.StorageType } // VM interface is for VM specific functions. type VM interface { Instance AgentCertificate() *x509.Certificate ConsoleLog() (string, error) ConsoleScreenshot(screenshotFile *os.File) error DumpGuestMemory(w *os.File, format string) error } // CriuMigrationArgs arguments for CRIU migration. type CriuMigrationArgs struct { Cmd uint StateDir string Function string Stop bool ActionScript bool DumpDir string PreDumpDir string Features liblxc.CriuFeatures Op *operationlock.InstanceOperation } // Info represents information about an instance driver. type Info struct { Name string // Name of an instance driver, e.g. "lxc" Version string // Version number of a loaded instance driver Error error // Whether there is an operational impediment. Type instancetype.Type // Instance type that the driver provides support for. Features map[string]any // Map of supported features. } // MigrateArgs represent arguments for instance migration send and receive. type MigrateArgs struct { ControlSend func(m proto.Message) error ControlReceive func(m proto.Message, handshake bool) error StateConn func(ctx context.Context) (io.ReadWriteCloser, error) FilesystemConn func(ctx context.Context) (io.ReadWriteCloser, error) Snapshots bool Live bool Disconnect func() ClusterMoveSourceName string // Will be empty if not a cluster move, othwise indicates the source instance. StoragePool string } // MigrateSendArgs represent arguments for instance migration send. type MigrateSendArgs struct { MigrateArgs AllowInconsistent bool Devices api.DevicesMap } // MigrateReceiveArgs represent arguments for instance migration receive. type MigrateReceiveArgs struct { MigrateArgs InstanceOperation *operationlock.InstanceOperation Refresh bool RefreshExcludeOlder bool } incus-7.0.0/internal/server/instance/instance_utils.go000066400000000000000000001143211517523235500231520ustar00rootroot00000000000000package instance import ( "bytes" "context" "crypto/rand" "database/sql" "errors" "fmt" "math/big" "net/http" "os" "path/filepath" "slices" "strconv" "strings" "time" "github.com/flosch/pongo2/v6" "github.com/google/uuid" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance/drivers/qemudefault" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/instance/operationlock" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/seccomp" "github.com/lxc/incus/v7/internal/server/state" storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/internal/server/sys" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // ValidDevices is linked from instance/drivers.validDevices to validate device config. var ValidDevices func(state *state.State, p api.Project, instanceType instancetype.Type, localDevices deviceConfig.Devices, expandedDevices deviceConfig.Devices) error // Load is linked from instance/drivers.load to allow different instance types to be loaded. var Load func(s *state.State, args db.InstanceArgs, p api.Project) (Instance, error) // Create is linked from instance/drivers.create to allow difference instance types to be created. // Returns a revert fail function that can be used to undo this function if a subsequent step fails. var Create func(s *state.State, args db.InstanceArgs, p api.Project, partialDeviceValidation bool, op *operations.Operation) (Instance, revert.Hook, error) func exclusiveConfigKeys(key1 string, key2 string, config map[string]string) (val string, ok bool, err error) { if config[key1] != "" && config[key2] != "" { return "", false, fmt.Errorf("Mutually exclusive keys %s and %s are set", key1, key2) } val, ok = config[key1] if ok { return } val, ok = config[key2] if ok { return } return "", false, nil } // ValidConfig validates an instance's config. func ValidConfig(sysOS *sys.OS, config map[string]string, expanded bool, instanceType instancetype.Type) error { if config == nil { return nil } for k, v := range config { if instanceType == instancetype.Any && !expanded && strings.HasPrefix(k, instance.ConfigVolatilePrefix) { return errors.New("Volatile keys can only be set on instances") } if instanceType == instancetype.Any && !expanded && strings.HasPrefix(k, "image.") { return errors.New("Image keys can only be set on instances") } err := validConfigKey(sysOS, k, v, instanceType) if err != nil { return err } after, ok := strings.CutPrefix(k, "systemd.credential.") if ok && config["systemd.credential-binary."+after] != "" { return fmt.Errorf("Mutually exclusive keys %s and systemd.credential-binary.%s are set", k, after) } } _, rawSeccomp := config["raw.seccomp"] _, isAllow, err := exclusiveConfigKeys("security.syscalls.allow", "security.syscalls.whitelist", config) if err != nil { return err } _, isDeny, err := exclusiveConfigKeys("security.syscalls.deny", "security.syscalls.blacklist", config) if err != nil { return err } val, _, err := exclusiveConfigKeys("security.syscalls.deny_default", "security.syscalls.blacklist_default", config) if err != nil { return err } isDenyDefault := util.IsTrue(val) val, _, err = exclusiveConfigKeys("security.syscalls.deny_compat", "security.syscalls.blacklist_compat", config) if err != nil { return err } isDenyCompat := util.IsTrue(val) if rawSeccomp && (isAllow || isDeny || isDenyDefault || isDenyCompat) { return errors.New("raw.seccomp is mutually exclusive with security.syscalls*") } if isAllow && (isDeny || isDenyDefault || isDenyCompat) { return errors.New("security.syscalls.allow is mutually exclusive with security.syscalls.deny*") } _, err = seccomp.SyscallInterceptMountFilter(config) if err != nil { return err } if instanceType == instancetype.Container && expanded && util.IsFalseOrEmpty(config["security.privileged"]) && sysOS.IdmapSet == nil { return errors.New("No uid/gid allocation configured. In this mode, only privileged containers are supported") } if util.IsTrue(config["security.privileged"]) && util.IsTrue(config["nvidia.runtime"]) { return errors.New("nvidia.runtime is incompatible with privileged containers") } return nil } func validConfigKey(os *sys.OS, key string, value string, instanceType instancetype.Type) error { f, err := instance.ConfigKeyChecker(key, instanceType.ToAPI()) if err != nil { return err } if err = f(value); err != nil { return err } if strings.HasPrefix(key, "limits.kernel.") && instanceType == instancetype.VM { return fmt.Errorf("%s isn't supported for VMs", key) } if key == "raw.lxc" { return lxcValidConfig(value) } if key == "security.syscalls.deny_compat" || key == "security.syscalls.blacklist_compat" { for _, arch := range os.Architectures { if arch == osarch.ARCH_64BIT_INTEL_X86 || arch == osarch.ARCH_64BIT_ARMV8_LITTLE_ENDIAN || arch == osarch.ARCH_64BIT_POWERPC_BIG_ENDIAN { return nil } } return fmt.Errorf("%s isn't supported on this architecture", key) } return nil } func lxcParseRawLXC(line string) (string, string, error) { // Ignore empty lines if len(line) == 0 { return "", "", nil } // Skip space {"\t", " "} line = strings.TrimLeft(line, "\t ") // Ignore comments if strings.HasPrefix(line, "#") { return "", "", nil } // Ensure the format is valid membs := strings.SplitN(line, "=", 2) if len(membs) != 2 { return "", "", fmt.Errorf("Invalid raw.lxc line: %s", line) } key := strings.ToLower(strings.Trim(membs[0], " \t")) val := strings.Trim(membs[1], " \t") return key, val, nil } func lxcValidConfig(rawLxc string) error { for _, line := range strings.Split(rawLxc, "\n") { key, _, err := lxcParseRawLXC(line) if err != nil { return err } if key == "" { continue } // block some keys if key == "lxc.logfile" || key == "lxc.log.file" { return errors.New("Setting lxc.logfile is not allowed") } if key == "lxc.syslog" || key == "lxc.log.syslog" { return errors.New("Setting lxc.log.syslog is not allowed") } if key == "lxc.ephemeral" { return errors.New("Setting lxc.ephemeral is not allowed") } if strings.HasPrefix(key, "lxc.prlimit.") { return fmt.Errorf(`Process limits should be set via ` + `"limits.kernel.[limit name]" and not ` + `directly via "lxc.prlimit.[limit name]"`) } } return nil } // AllowedUnprivilegedOnlyMap checks that root user is not mapped into instance. func AllowedUnprivilegedOnlyMap(rawIdmap string) error { rawMaps, err := idmap.NewSetFromIncusIDMap(rawIdmap) if err != nil { return err } for _, ent := range rawMaps.Entries { if ent.HostID == 0 { return errors.New("Cannot map root user into container as the server was configured to only allow unprivileged containers") } } return nil } // LoadByID loads an instance by ID. func LoadByID(s *state.State, id int) (Instance, error) { var project string var name string err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get the DB record project, name, err = tx.GetInstanceProjectAndName(ctx, id) if err != nil { return fmt.Errorf("Failed getting instance project and name: %w", err) } return nil }) if err != nil { return nil, err } return LoadByProjectAndName(s, project, name) } // LoadInstanceDatabaseObject loads a db.Instance object, accounting for snapshots. func LoadInstanceDatabaseObject(ctx context.Context, tx *db.ClusterTx, project, name string) (*cluster.Instance, error) { var container *cluster.Instance var err error if strings.Contains(name, instance.SnapshotDelimiter) { parts := strings.SplitN(name, instance.SnapshotDelimiter, 2) instanceName := parts[0] snapshotName := parts[1] instance, err := cluster.GetInstance(ctx, tx.Tx(), project, instanceName) if err != nil { return nil, fmt.Errorf("Failed to fetch instance %q in project %q: %w", name, project, err) } snapshot, err := cluster.GetInstanceSnapshot(ctx, tx.Tx(), project, instanceName, snapshotName) if err != nil { return nil, fmt.Errorf("Failed to fetch snapshot %q of instance %q in project %q: %w", snapshotName, instanceName, project, err) } c := snapshot.ToInstance(instance.Name, instance.Node, instance.Type, instance.Architecture) container = &c } else { container, err = cluster.GetInstance(ctx, tx.Tx(), project, name) if err != nil { return nil, fmt.Errorf("Failed to fetch instance %q in project %q: %w", name, project, err) } } return container, nil } // LoadByProjectAndName loads an instance by project and name. func LoadByProjectAndName(s *state.State, projectName string, instanceName string) (Instance, error) { // Get the DB record var args db.InstanceArgs var p *api.Project err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { proj, err := cluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return err } p, err = proj.ToAPI(ctx, tx.Tx()) if err != nil { return err } inst, err := LoadInstanceDatabaseObject(ctx, tx, projectName, instanceName) if err != nil { return err } instArgs, err := tx.InstancesToInstanceArgs(ctx, true, *inst) if err != nil { return err } args = instArgs[inst.ID] return nil }) if err != nil { return nil, err } inst, err := Load(s, args, *p) if err != nil { return nil, fmt.Errorf("Failed to load instance: %w", err) } return inst, nil } // LoadNodeAll loads all instances on this server. func LoadNodeAll(s *state.State, instanceType instancetype.Type) ([]Instance, error) { var err error var instances []Instance filter := cluster.InstanceFilter{Type: instanceType.Filter()} if s.ServerName != "" { filter.Node = &s.ServerName } err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.InstanceList(ctx, func(dbInst db.InstanceArgs, p api.Project) error { inst, err := Load(s, dbInst, p) if err != nil { return fmt.Errorf("Failed loading instance %q in project %q: %w", dbInst.Name, dbInst.Project, err) } instances = append(instances, inst) return nil }, filter) }) if err != nil { return nil, err } return instances, nil } // LoadFromBackup loads from a mounted instance's backup file. // If applyProfiles is false, then the profiles property will be cleared to prevent profile enrichment from DB. // Then the expanded config and expanded devices from the backup file will be applied to the local config and // local devices respectively. This is done to allow an expanded instance to be returned without needing the DB. func LoadFromBackup(s *state.State, projectName string, instancePath string, applyProfiles bool) (Instance, error) { var inst Instance backupYamlPath := filepath.Join(instancePath, "backup.yaml") backupConf, err := backup.ParseConfigYamlFile(backupYamlPath) if err != nil { return nil, fmt.Errorf("Failed parsing instance backup file from %q: %w", backupYamlPath, err) } instDBArgs, err := backup.ConfigToInstanceDBArgs(s, backupConf, projectName, applyProfiles) if err != nil { return nil, err } if !applyProfiles { // Stop instance.Load() from expanding profile config from DB, and apply expanded config from // backup file to local config. This way we can still see the devices even if DB not available. instDBArgs.Config = backupConf.Container.ExpandedConfig instDBArgs.Devices = deviceConfig.NewDevices(backupConf.Container.ExpandedDevices) } var p *api.Project err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { proj, err := cluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return err } p, err = proj.ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return nil, err } inst, err = Load(s, *instDBArgs, *p) if err != nil { return nil, fmt.Errorf("Failed loading instance from backup file %q: %w", backupYamlPath, err) } return inst, nil } // DeviceNextInterfaceHWAddr generates a random MAC address. func DeviceNextInterfaceHWAddr(pattern string) (string, error) { // Generate a new random MAC address using the given pattern. ret := bytes.Buffer{} for _, c := range pattern { if c == 'x' || c == 'X' { c, err := rand.Int(rand.Reader, big.NewInt(16)) if err != nil { return "", err } ret.WriteString(fmt.Sprintf("%x", c.Int64())) } else { ret.WriteString(string(c)) } } return ret.String(), nil } // BackupLoadByName load an instance backup from the database. func BackupLoadByName(s *state.State, project, name string) (*backup.InstanceBackup, error) { var args db.InstanceBackup // Get the backup database record err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error args, err = tx.GetInstanceBackup(ctx, project, name) return err }) if err != nil { return nil, err } // Load the instance it belongs to instance, err := LoadByID(s, args.InstanceID) if err != nil { return nil, err } return backup.NewInstanceBackup(s, instance, args.ID, name, args.CreationDate, args.ExpiryDate, args.InstanceOnly, args.RootOnly, args.OptimizedStorage), nil } // ResolveImage takes an instance source and returns a hash suitable for instance creation or download. func ResolveImage(ctx context.Context, tx *db.ClusterTx, projectName string, source api.InstanceSource) (string, error) { if source.Fingerprint != "" { return source.Fingerprint, nil } if source.Alias != "" { if source.Server != "" { return source.Alias, nil } _, alias, err := tx.GetImageAlias(ctx, projectName, source.Alias, true) if err != nil { return "", err } return alias.Target, nil } if source.Properties != nil { if source.Server != "" { return "", errors.New("Property match is only supported for local images") } hashes, err := tx.GetImagesFingerprints(ctx, projectName, false) if err != nil { return "", err } var image *api.Image for _, imageHash := range hashes { _, img, err := tx.GetImageByFingerprintPrefix(ctx, imageHash, cluster.ImageFilter{Project: &projectName}) if err != nil { continue } if image != nil && img.CreatedAt.Before(image.CreatedAt) { continue } match := true for key, value := range source.Properties { if img.Properties[key] != value { match = false break } } if !match { continue } image = img } if image != nil { return image.Fingerprint, nil } return "", errors.New("No matching image could be found") } return "", errors.New("Must specify one of alias, fingerprint or properties for init from image") } // SuitableArchitectures returns a slice of architecture ids based on an instance create request. // // An empty list indicates that the request may be handled by any architecture. // A nil list indicates that we can't tell at this stage, typically for private images. func SuitableArchitectures(ctx context.Context, s *state.State, tx *db.ClusterTx, projectName string, sourceInst *cluster.Instance, sourceImageRef string, req api.InstancesPost) ([]int, error) { // Handle cases where the architecture is already provided. if slices.Contains([]string{"migration", "none"}, req.Source.Type) && req.Architecture != "" { id, err := osarch.ArchitectureID(req.Architecture) if err != nil { return nil, err } return []int{id}, nil } // For migration, an architecture must be specified in the req. if req.Source.Type == "migration" && req.Architecture == "" { return nil, api.StatusErrorf(http.StatusBadRequest, "An architecture must be specified in migration requests") } // For none, allow any architecture. if req.Source.Type == "none" { return []int{}, nil } // For copy, always use the source architecture. if req.Source.Type == "copy" { return []int{sourceInst.Architecture}, nil } // For image, things get a bit more complicated. if req.Source.Type == "image" { // Handle local images. if req.Source.Server == "" { _, img, err := tx.GetImageByFingerprintPrefix(ctx, sourceImageRef, cluster.ImageFilter{Project: &projectName}) if err != nil { return nil, err } id, err := osarch.ArchitectureID(img.Architecture) if err != nil { return nil, err } return []int{id}, nil } // Handle remote images. if req.Source.Server != "" { if req.Source.Secret != "" { // We can't retrieve a private image, defer to later processing. return nil, nil } var err error var remote incus.ImageServer if slices.Contains([]string{"", "incus", "lxd"}, req.Source.Protocol) { // Remote image server. remote, err = incus.ConnectPublicIncus(req.Source.Server, &incus.ConnectionArgs{ TLSServerCert: req.Source.Certificate, UserAgent: version.UserAgent, Proxy: s.Proxy, CachePath: s.OS.CacheDir, CacheExpiry: time.Hour, SkipGetEvents: true, SkipGetServer: true, }) if err != nil { return nil, err } } else if req.Source.Protocol == "simplestreams" { // Remote simplestreams image server. remote, err = incus.ConnectSimpleStreams(req.Source.Server, &incus.ConnectionArgs{ TLSServerCert: req.Source.Certificate, UserAgent: version.UserAgent, Proxy: s.Proxy, CachePath: s.OS.CacheDir, CacheExpiry: time.Hour, }) if err != nil { return nil, err } } else if req.Source.Protocol == "oci" { // Remote OCI registry. remote, err = incus.ConnectOCI(req.Source.Server, &incus.ConnectionArgs{ TLSServerCert: req.Source.Certificate, UserAgent: version.UserAgent, Proxy: s.Proxy, CachePath: s.OS.CacheDir, CacheExpiry: time.Hour, }) if err != nil { return nil, err } } else { return nil, api.StatusErrorf(http.StatusBadRequest, "Unsupported remote image server protocol %q", req.Source.Protocol) } // Look for a matching alias. entries, err := remote.GetImageAliasArchitectures(string(req.Type), sourceImageRef) if err != nil { // Look for a matching image by fingerprint. img, _, err := remote.GetImage(sourceImageRef) if err != nil { return nil, err } id, err := osarch.ArchitectureID(img.Architecture) if err != nil { return nil, err } return []int{id}, nil } architectures := []int{} for arch := range entries { id, err := osarch.ArchitectureID(arch) if err != nil { return nil, err } architectures = append(architectures, id) } return architectures, nil } } // No other known types return nil, api.StatusErrorf(http.StatusBadRequest, "Unknown instance source type %q", req.Source.Type) } // ValidName validates an instance name. There are different validation rules for instance snapshot names // so it takes an argument indicating whether the name is to be used for a snapshot or not. func ValidName(instanceName string, isSnapshot bool) error { if isSnapshot { parentName, snapshotName, _ := api.GetParentAndSnapshotName(instanceName) err := validate.IsHostname(parentName) if err != nil { return fmt.Errorf("Invalid instance name: %w", err) } // Snapshot part is more flexible, but doesn't allow space or / character. if strings.ContainsAny(snapshotName, " /") { return errors.New("Invalid instance snapshot name: Cannot contain space or / characters") } } else { if strings.Contains(instanceName, instance.SnapshotDelimiter) { return fmt.Errorf("The character %q is reserved for snapshots", instance.SnapshotDelimiter) } err := validate.IsHostname(instanceName) if err != nil { return fmt.Errorf("Invalid instance name: %w", err) } } return nil } // CreateInternal creates an instance record and storage volume record in the database and sets up devices. // Accepts a reverter that revert steps this function does will be added to. It is up to the caller to // call the revert's Fail() or Success() function as needed. // Returns the created instance, along with a "create" operation lock that needs to be marked as Done once the // instance is fully completed, and a revert fail function that can be used to undo this function if a subsequent // step fails. func CreateInternal(s *state.State, args db.InstanceArgs, op *operations.Operation, clearLogDir bool, checkArchitecture bool, partialDeviceValidation bool) (Instance, *operationlock.InstanceOperation, revert.Hook, error) { reverter := revert.New() defer reverter.Fail() // Check instance type requested is supported by this machine. err := s.InstanceTypes[args.Type] if err != nil { return nil, nil, nil, fmt.Errorf("Instance type %q is not supported on this server: %w", args.Type, err) } // Set default values. if args.Project == "" { args.Project = api.ProjectDefaultName } if args.Profiles == nil { err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { args.Profiles, err = tx.GetProfiles(ctx, args.Project, []string{"default"}) return err }) if err != nil { return nil, nil, nil, errors.New("Failed to get default profile for new instance") } } if args.Config == nil { args.Config = map[string]string{} } if args.BaseImage != "" { args.Config["volatile.base_image"] = args.BaseImage } if args.Config["volatile.uuid"] == "" { args.Config["volatile.uuid"] = uuid.New().String() } args.Config["volatile.uuid.generation"] = args.Config["volatile.uuid"] if args.Devices == nil { args.Devices = deviceConfig.Devices{} } if args.Architecture == 0 { args.Architecture = s.OS.Architectures[0] } err = ValidName(args.Name, args.Snapshot) if err != nil { return nil, nil, nil, err } if !args.Snapshot { // Unset expiry date since instances don't expire. args.ExpiryDate = time.Time{} // Generate a cloud-init instance-id if not provided. // // This is generated here rather than in startCommon as only new // instances or those which get modified should receive an instance-id. // Existing instances will keep using their instance name as instance-id to // avoid triggering cloud-init on upgrade. if args.Config["volatile.cloud-init.instance-id"] == "" { args.Config["volatile.cloud-init.instance-id"] = uuid.New().String() } } // Validate instance config. err = ValidConfig(s.OS, args.Config, false, args.Type) if err != nil { return nil, nil, nil, err } // Leave validating devices to Create function call below. // Validate architecture. _, err = osarch.ArchitectureName(args.Architecture) if err != nil { return nil, nil, nil, err } if checkArchitecture && !slices.Contains(s.OS.Architectures, args.Architecture) { return nil, nil, nil, errors.New("Requested architecture isn't supported by this host") } var profiles []string err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Validate profiles. profiles, err = tx.GetProfileNames(ctx, args.Project) return err }) if err != nil { return nil, nil, nil, err } checkedProfiles := map[string]bool{} for _, profile := range args.Profiles { if !slices.Contains(profiles, profile.Name) { return nil, nil, nil, fmt.Errorf("Requested profile %q doesn't exist", profile.Name) } if checkedProfiles[profile.Name] { return nil, nil, nil, errors.New("Duplicate profile found in request") } checkedProfiles[profile.Name] = true } if args.CreationDate.IsZero() { args.CreationDate = time.Now().UTC() } if args.LastUsedDate.IsZero() { args.LastUsedDate = time.Unix(0, 0).UTC() } // Prevent concurrent create requests for same instance. opl, err := operationlock.Create(args.Project, args.Name, op, operationlock.ActionCreate, false, false) if err != nil { return nil, nil, nil, err } reverter.Add(func() { opl.Done(err) }) var dbInst cluster.Instance var p *api.Project err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { proj, err := cluster.GetProject(ctx, tx.Tx(), args.Project) if err != nil { return err } p, err = proj.ToAPI(ctx, tx.Tx()) if err != nil { return err } devices, err := cluster.APIToDevices(args.Devices.CloneNative()) if err != nil { return err } if args.Snapshot { parts := strings.SplitN(args.Name, instance.SnapshotDelimiter, 2) instanceName := parts[0] snapshotName := parts[1] instance, err := cluster.GetInstance(ctx, tx.Tx(), args.Project, instanceName) if err != nil { return fmt.Errorf("Get instance %q in project %q", instanceName, args.Project) } snapshot := cluster.InstanceSnapshot{ Project: args.Project, Instance: instanceName, Name: snapshotName, CreationDate: args.CreationDate, Stateful: args.Stateful, Description: args.Description, ExpiryDate: sql.NullTime{Time: args.ExpiryDate, Valid: true}, } id, err := cluster.CreateInstanceSnapshot(ctx, tx.Tx(), snapshot) if err != nil { return fmt.Errorf("Add snapshot info to the database: %w", err) } err = cluster.CreateInstanceSnapshotConfig(ctx, tx.Tx(), id, args.Config) if err != nil { return err } err = cluster.CreateInstanceSnapshotDevices(ctx, tx.Tx(), id, devices) if err != nil { return err } // Read back the snapshot, to get ID and creation time. s, err := cluster.GetInstanceSnapshot(ctx, tx.Tx(), args.Project, instanceName, snapshotName) if err != nil { return fmt.Errorf("Fetch created snapshot from the database: %w", err) } dbInst = s.ToInstance(instance.Name, instance.Node, instance.Type, instance.Architecture) newArgs, err := tx.InstancesToInstanceArgs(ctx, false, dbInst) if err != nil { return err } // Populate profile info that was already loaded. newInstArgs := newArgs[dbInst.ID] newInstArgs.Profiles = args.Profiles args = newInstArgs return nil } // Create the instance entry. dbInst = cluster.Instance{ Project: args.Project, Name: args.Name, Node: s.ServerName, Type: args.Type, Snapshot: args.Snapshot, Architecture: args.Architecture, Ephemeral: args.Ephemeral, CreationDate: args.CreationDate, Stateful: args.Stateful, LastUseDate: sql.NullTime{Time: args.LastUsedDate, Valid: true}, Description: args.Description, ExpiryDate: sql.NullTime{Time: args.ExpiryDate, Valid: true}, } instanceID, err := cluster.CreateInstance(ctx, tx.Tx(), dbInst) if err != nil { return fmt.Errorf("Add instance info to the database: %w", err) } err = cluster.CreateInstanceDevices(ctx, tx.Tx(), instanceID, devices) if err != nil { return err } err = cluster.CreateInstanceConfig(ctx, tx.Tx(), instanceID, args.Config) if err != nil { return err } profileNames := make([]string, 0, len(args.Profiles)) for _, profile := range args.Profiles { profileNames = append(profileNames, profile.Name) } err = cluster.UpdateInstanceProfiles(ctx, tx.Tx(), int(instanceID), dbInst.Project, profileNames) if err != nil { return err } // Read back the instance, to get ID and creation time. dbRow, err := cluster.GetInstance(ctx, tx.Tx(), args.Project, args.Name) if err != nil { return fmt.Errorf("Fetch created instance from the database: %w", err) } dbInst = *dbRow if dbInst.ID < 1 { return fmt.Errorf("Unexpected instance database ID %d: %w", dbInst.ID, err) } newArgs, err := tx.InstancesToInstanceArgs(ctx, false, dbInst) if err != nil { return err } // Populate profile info that was already loaded. newInstArgs := newArgs[dbInst.ID] newInstArgs.Profiles = args.Profiles args = newInstArgs return nil }) if err != nil { if errors.Is(err, db.ErrAlreadyDefined) { thing := "Instance" if instance.IsSnapshot(args.Name) { thing = "Snapshot" } return nil, nil, nil, fmt.Errorf("%s %q already exists", thing, args.Name) } return nil, nil, nil, err } reverter.Add(func() { _ = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.DeleteInstance(ctx, dbInst.Project, dbInst.Name) }) }) inst, cleanup, err := Create(s, args, *p, partialDeviceValidation, op) if err != nil { logger.Error("Failed initializing instance", logger.Ctx{"project": args.Project, "instance": args.Name, "type": args.Type, "err": err}) return nil, nil, nil, fmt.Errorf("Failed initializing instance: %w", err) } reverter.Add(cleanup) // Wipe any existing log for this instance name. if clearLogDir { _ = os.RemoveAll(inst.LogPath()) _ = os.RemoveAll(inst.RunPath()) } cleanup = reverter.Clone().Fail reverter.Success() return inst, opl, cleanup, err } // NextSnapshotName finds the next snapshot for an instance. func NextSnapshotName(s *state.State, inst Instance, defaultPattern string) (string, error) { var err error pattern := inst.ExpandedConfig()["snapshots.pattern"] if pattern == "" { pattern = defaultPattern } pattern, err = internalUtil.RenderTemplate(pattern, pongo2.Context{ "creation_date": time.Now(), }) if err != nil { return "", err } count := strings.Count(pattern, "%d") if count > 1 { return "", fmt.Errorf("Snapshot pattern may contain '%%d' only once") } else if count == 1 { var i int _ = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { i = tx.GetNextInstanceSnapshotIndex(ctx, inst.Project().Name, inst.Name(), pattern) return nil }) return strings.Replace(pattern, "%d", strconv.Itoa(i), 1), nil } snapshotExists := false snapshots, err := inst.Snapshots() if err != nil { return "", err } for _, snap := range snapshots { _, snapOnlyName, _ := api.GetParentAndSnapshotName(snap.Name()) if snapOnlyName == pattern { snapshotExists = true break } } // Append '-0', '-1', etc. if the actual pattern/snapshot name already exists if snapshotExists { pattern = fmt.Sprintf("%s-%%d", pattern) var i int _ = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { i = tx.GetNextInstanceSnapshotIndex(ctx, inst.Project().Name, inst.Name(), pattern) return nil }) return strings.Replace(pattern, "%d", strconv.Itoa(i), 1), nil } return pattern, nil } // temporaryName returns the temporary instance name using a stable random generator. // The returned string is a valid DNS name. func temporaryName(instUUID string) (string, error) { r, err := localUtil.GetStableRandomGenerator(instUUID) if err != nil { return "", err } // The longest temporary name is move-of-18446744073709551615 which has a length // of 30 characters since 18446744073709551615 is the biggest value for an uint64. // The prefix is attached to have a valid DNS name that doesn't start with numbers. return fmt.Sprintf("move-of-%d", r.Uint64()), nil } // MoveTemporaryName returns a name derived from the instance's volatile.uuid, to use when moving an instance // across pools or cluster members which can be used for the naming the temporary copy before deleting the original // instance and renaming the copy to the original name. // If volatile.uuid is not set, a new UUID is generated and stored in the instance's config. func MoveTemporaryName(inst Instance) (string, error) { instUUID := inst.LocalConfig()["volatile.uuid"] if instUUID == "" { instUUID = uuid.New().String() err := inst.VolatileSet(map[string]string{"volatile.uuid": instUUID}) if err != nil { return "", fmt.Errorf("Failed setting volatile.uuid to %s: %w", instUUID, err) } } return temporaryName(instUUID) } // IsSameLogicalInstance returns true if the supplied Instance and db.Instance have the same project and name or // if they have the same volatile.uuid values. func IsSameLogicalInstance(inst Instance, dbInst *db.InstanceArgs) bool { // Instance name is unique within a project. if dbInst.Project == inst.Project().Name && dbInst.Name == inst.Name() { return true } // Don't trigger duplicate resource errors for temporary copies. if dbInst.Config["volatile.uuid"] == inst.LocalConfig()["volatile.uuid"] { // Accommodate moving instances between storage pools. // Check temporary copy against source. tempName, err := temporaryName(inst.LocalConfig()["volatile.uuid"]) if err != nil { return false } if dbInst.Name == tempName { return true } // Check source against temporary copy. tempName, err = temporaryName(dbInst.Config["volatile.uuid"]) if err != nil { return false } if inst.Name() == tempName { return true } // Accommodate moving instances between projects. if dbInst.Project != inst.Project().Name { return true } } return false } // SnapshotToProtobuf converts a snapshot record to a migration snapshot record. func SnapshotToProtobuf(snap *api.InstanceSnapshot) *migration.Snapshot { config := make([]*migration.Config, 0, len(snap.Config)) for k, v := range snap.Config { kCopy := string(k) vCopy := string(v) config = append(config, &migration.Config{Key: &kCopy, Value: &vCopy}) } devices := make([]*migration.Device, 0, len(snap.Devices)) for name, d := range snap.Devices { props := make([]*migration.Config, 0, len(snap.Devices)) for k, v := range d { // Local loop vars. kCopy := string(k) vCopy := string(v) props = append(props, &migration.Config{Key: &kCopy, Value: &vCopy}) } nameCopy := name // Local loop var. devices = append(devices, &migration.Device{Name: &nameCopy, Config: props}) } isEphemeral := snap.Ephemeral archID, _ := osarch.ArchitectureID(snap.Architecture) arch := int32(archID) stateful := snap.Stateful creationDate := snap.CreatedAt.UTC().Unix() lastUsedDate := snap.LastUsedAt.UTC().Unix() expiryDate := snap.ExpiresAt.UTC().Unix() return &migration.Snapshot{ Name: &snap.Name, LocalConfig: config, Profiles: snap.Profiles, Ephemeral: &isEphemeral, LocalDevices: devices, Architecture: &arch, Stateful: &stateful, CreationDate: &creationDate, LastUsedDate: &lastUsedDate, ExpiryDate: &expiryDate, } } // SnapshotProtobufToInstanceArgs converts a migration snapshot record to DB instance record format. func SnapshotProtobufToInstanceArgs(s *state.State, inst Instance, snap *migration.Snapshot) (*db.InstanceArgs, error) { snapConfig := snap.GetLocalConfig() config := make(map[string]string, len(snapConfig)) for _, ent := range snapConfig { config[ent.GetKey()] = ent.GetValue() } snapDevices := snap.GetLocalDevices() devices := make(deviceConfig.Devices, len(snapDevices)) for _, ent := range snap.GetLocalDevices() { entConfig := ent.GetConfig() props := make(map[string]string, len(entConfig)) for _, prop := range entConfig { props[prop.GetKey()] = prop.GetValue() } devices[ent.GetName()] = props } var profiles []api.Profile err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error profiles, err = tx.GetProfiles(ctx, inst.Project().Name, snap.Profiles) return err }) if err != nil { return nil, err } args := db.InstanceArgs{ Architecture: int(snap.GetArchitecture()), Config: config, Description: inst.Description(), Type: inst.Type(), Snapshot: true, Devices: devices, Ephemeral: snap.GetEphemeral(), Name: inst.Name() + instance.SnapshotDelimiter + snap.GetName(), Profiles: profiles, Stateful: snap.GetStateful(), Project: inst.Project().Name, } if snap.GetCreationDate() != 0 { args.CreationDate = time.Unix(snap.GetCreationDate(), 0) } if snap.GetLastUsedDate() != 0 { args.LastUsedDate = time.Unix(snap.GetLastUsedDate(), 0) } if snap.GetExpiryDate() != 0 { args.ExpiryDate = time.Unix(snap.GetExpiryDate(), 0) } return &args, nil } // ResourceUsage returns an instance's expected CPU, memory and disk usage. func ResourceUsage(instConfig map[string]string, instDevices map[string]map[string]string, instType api.InstanceType) (int64, int64, int64, error) { var err error limitsCPU := instConfig["limits.cpu"] limitsMemory := instConfig["limits.memory"] cpuUsage := int64(0) memoryUsage := int64(0) diskUsage := int64(0) // Parse limits.cpu. if limitsCPU != "" { // Check if using shared CPU limits. cpuUsage, err = strconv.ParseInt(limitsCPU, 10, 64) if err != nil { // Or get count of pinned CPUs. pinnedCPUs, err := resources.ParseCpuset(limitsCPU) if err != nil { return -1, -1, -1, fmt.Errorf("Failed parsing instance resources limits.cpu: %w", err) } cpuUsage = int64(len(pinnedCPUs)) } } else if instType == api.InstanceTypeVM { // Apply VM CPU cores defaults if not specified. cpuUsage = qemudefault.CPUCores } // Parse limits.memory. memoryLimitStr := limitsMemory // Apply VM memory limit defaults if not specified. if instType == api.InstanceTypeVM && memoryLimitStr == "" { memoryLimitStr = qemudefault.MemSize } if memoryLimitStr != "" { memoryLimit, err := units.ParseByteSizeString(memoryLimitStr) if err != nil { return -1, -1, -1, fmt.Errorf("Failed parsing instance resources limits.memory: %w", err) } memoryUsage = int64(memoryLimit) } // Parse root disk size. _, rootDiskConfig, err := instance.GetRootDiskDevice(instDevices) if err == nil { rootDiskSizeStr := rootDiskConfig["size"] // Apply VM root disk size defaults if not specified. if instType == api.InstanceTypeVM && rootDiskSizeStr == "" { rootDiskSizeStr = storageDrivers.DefaultBlockSize } if rootDiskSizeStr != "" { rootDiskSize, err := units.ParseByteSizeString(rootDiskSizeStr) if err != nil { return -1, -1, -1, fmt.Errorf("Failed parsing instance resources root disk size: %w", err) } diskUsage = int64(rootDiskSize) } } return cpuUsage, memoryUsage, diskUsage, nil } incus-7.0.0/internal/server/instance/instancetype/000077500000000000000000000000001517523235500223035ustar00rootroot00000000000000incus-7.0.0/internal/server/instance/instancetype/instance_type.go000066400000000000000000000036601517523235500255040ustar00rootroot00000000000000package instancetype import ( "errors" "github.com/lxc/incus/v7/shared/api" ) // Type indicates the type of instance. type Type int const ( // Any represents any type of instance. Any = Type(-1) // Container represents a container instance type. Container = Type(0) // VM represents a virtual-machine instance type. VM = Type(1) ) // New validates the supplied string against the allowed types of instance and returns the internal // representation of that type. If empty string is supplied then the type returned is TypeContainer. // If an invalid name is supplied an error will be returned. func New(name string) (Type, error) { // If "container" or "" is supplied, return type as Container. if api.InstanceType(name) == api.InstanceTypeContainer || name == "" { return Container, nil } // If "virtual-machine" is supplied, return type as VM. if api.InstanceType(name) == api.InstanceTypeVM { return VM, nil } return -1, errors.New("Invalid instance type") } // String converts the internal representation of instance type to a string used in API requests. // Returns empty string if value is not a valid instance type. func (instanceType Type) String() string { if instanceType == Container { return string(api.InstanceTypeContainer) } if instanceType == VM { return string(api.InstanceTypeVM) } return "" } // ToAPI converts the internal representation of instance type to an api.InstanceType. func (instanceType Type) ToAPI() api.InstanceType { if instanceType == Container { return api.InstanceTypeContainer } if instanceType == VM { return api.InstanceTypeVM } if instanceType == Any { return api.InstanceTypeAny } return api.InstanceType("") } // Filter returns a valid filter field compatible with cluster.InstanceFilter. // 'Any' represents any possible instance type, and so it is omitted. func (instanceType Type) Filter() *Type { if instanceType == Any { return nil } return &instanceType } incus-7.0.0/internal/server/instance/instancetype/instance_vmagent.go000066400000000000000000000013671517523235500261660ustar00rootroot00000000000000package instancetype import ( deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" ) // VMAgentMount defines mounts to perform inside VM via agent. type VMAgentMount struct { Source string `json:"source"` Target string `json:"target"` FSType string `json:"fstype"` Options []string `json:"options"` } // VMAgentData represents the instance data exposed to the VM agent. type VMAgentData struct { Name string `json:"name"` CloudInitID string `json:"cloud_init_id"` Location string `json:"location"` Config map[string]string `json:"config,omitempty"` Devices map[string]deviceConfig.Device `json:"devices,omitempty"` } incus-7.0.0/internal/server/instance/operationlock/000077500000000000000000000000001517523235500224465ustar00rootroot00000000000000incus-7.0.0/internal/server/instance/operationlock/operationlock.go000066400000000000000000000174631517523235500256610ustar00rootroot00000000000000package operationlock import ( "context" "errors" "fmt" "slices" "sync" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/shared/logger" ) // Action indicates the operation action type. type Action string // ActionCreate for creating an instance. const ActionCreate Action = "create" // ActionStart for starting an instance. const ActionStart Action = "start" // ActionStop for stopping an instance. const ActionStop Action = "stop" // ActionRestart for restarting an instance. const ActionRestart Action = "restart" // ActionRestore for restoring an instance. const ActionRestore Action = "restore" // ActionUpdate for updating an instance. const ActionUpdate Action = "update" // ActionDelete for deleting an instance. const ActionDelete Action = "delete" // ActionMigrate for migrating an instance. const ActionMigrate Action = "migrate" // ActionConsoleRetrieve for retrieving and saving a VM's console history. const ActionConsoleRetrieve Action = "console_retrieve" // ErrNonReusuableSucceeded is returned when no operation is created due to having to wait for a matching // non-reusuable operation that has now completed successfully. var ErrNonReusuableSucceeded error = errors.New("A matching non-reusable operation has now succeeded") var ( instanceOperationsLock sync.Mutex instanceOperations = make(map[string]*InstanceOperation) ) // InstanceOperation operation locking. type InstanceOperation struct { action Action chanDone chan error err error projectName string instanceName string reusable bool instanceInitiated bool op *operations.Operation } // Create creates a new operation lock for an Instance if one does not already exist and returns it. // The lock will be released after TimeoutDefault or when Done() is called, which ever occurs first. // If createReusuable is set as true then future lock attempts can specify the reuseExisting argument as true // which will then trigger a reset of the timeout to TimeoutDefault on the existing lock and return it. func Create(projectName string, instanceName string, apiOp *operations.Operation, action Action, createReusuable bool, reuseExisting bool) (*InstanceOperation, error) { if projectName == "" || instanceName == "" { return nil, errors.New("Invalid project or instance name") } instanceOperationsLock.Lock() defer instanceOperationsLock.Unlock() opKey := project.Instance(projectName, instanceName) op := instanceOperations[opKey] if op != nil { if op.reusable && reuseExisting { logger.Debug("Instance operation lock reused", logger.Ctx{"project": op.projectName, "instance": op.instanceName, "action": op.action, "reusable": op.reusable}) return op, nil } return nil, fmt.Errorf("Instance is busy running a %q operation", op.action) } op = &InstanceOperation{} op.projectName = projectName op.instanceName = instanceName op.action = action op.reusable = createReusuable op.chanDone = make(chan error) op.op = apiOp instanceOperations[opKey] = op logger.Debug("Instance operation lock created", logger.Ctx{"project": op.projectName, "instance": op.instanceName, "action": op.action, "reusable": op.reusable}) return op, nil } // CreateWaitGet is a weird function which does what we happen to want most of the time. // // If the instance has an operation of the same type and it's not reusable // or the caller doesn't want to reuse it, the function will wait and // indicate that it did so. // // If the instance has an existing operation of one of the inheritableActions types, then the operation is returned // to the user. This allows an operation started in one function/routine to be inherited by another. // // If the instance doesn't have an ongoing operation, has an operation of a different type that is not in the // inheritableActions list or has the right type and is being reused, then this behaves as a Create call. // // Returns ErrWaitedForMatching if it waited for a matching operation to finish and it's finished successfully and // so didn't return create a new operation. func CreateWaitGet(projectName string, instanceName string, apiOp *operations.Operation, action Action, inheritableActions []Action, createReusuable bool, reuseExisting bool) (*InstanceOperation, error) { op := Get(projectName, instanceName) // No existing operation, call create. if op == nil { op, err := Create(projectName, instanceName, apiOp, action, createReusuable, reuseExisting) return op, err } // Operation action matches but is not reusable or we have been asked not to reuse, // so wait and return result. if op.action == action && (!reuseExisting || !op.reusable) { err := op.Wait(context.Background()) if err != nil { return nil, err } // The matching operation ended without error, but this means we've not created a new // operation for this request, so return a special error indicating this scenario. return nil, ErrNonReusuableSucceeded } // Operation action matches one the inheritable actions, return the operation. if op.ActionMatch(inheritableActions...) { logger.Debug("Instance operation lock inherited", logger.Ctx{"project": op.projectName, "instance": op.instanceName, "action": op.action, "reusable": op.reusable, "inheritedByAction": action}) return op, nil } // Send the rest to Create to try and create a new operation. op, err := Create(projectName, instanceName, apiOp, action, createReusuable, reuseExisting) return op, err } // Get retrieves an existing lock or returns nil if no lock exists. func Get(projectName string, instanceName string) *InstanceOperation { instanceOperationsLock.Lock() defer instanceOperationsLock.Unlock() opKey := project.Instance(projectName, instanceName) return instanceOperations[opKey] } // Action returns operation's action. func (op *InstanceOperation) Action() Action { // This function can be called on a nil struct. if op == nil { return "" } return op.action } // ActionMatch returns true if operation's action matches one of the matchActions. func (op *InstanceOperation) ActionMatch(matchActions ...Action) bool { return slices.Contains(matchActions, op.action) } // Wait waits for an operation to finish. func (op *InstanceOperation) Wait(ctx context.Context) error { // This function can be called on a nil struct. if op == nil { return nil } select { case <-op.chanDone: return op.err case <-ctx.Done(): return ctx.Err() } } // Done indicates the operation has finished. func (op *InstanceOperation) Done(err error) { // This function can be called on a nil struct. if op == nil { return } instanceOperationsLock.Lock() defer instanceOperationsLock.Unlock() opKey := project.Instance(op.projectName, op.instanceName) // Check if already done. runningOp, ok := instanceOperations[opKey] if !ok || runningOp != op { return } op.err = err delete(instanceOperations, opKey) // Delete before closing chanDone. close(op.chanDone) logger.Debug("Instance operation lock finished", logger.Ctx{"project": op.projectName, "instance": op.instanceName, "action": op.action, "reusable": op.reusable, "err": err}) } // SetInstanceInitiated sets the instance initiated marker. func (op *InstanceOperation) SetInstanceInitiated(instanceInitiated bool) { // This function can be called on a nil struct. if op == nil { return } op.instanceInitiated = instanceInitiated } // GetInstanceInitiated gets the instance initiated marker. func (op *InstanceOperation) GetInstanceInitiated() bool { // This function can be called on a nil struct. if op == nil { return false } return op.instanceInitiated } // GetOperation gets the API background operation. func (op *InstanceOperation) GetOperation() *operations.Operation { // This function can be called on a nil struct. if op == nil { return nil } return op.op } incus-7.0.0/internal/server/ip/000077500000000000000000000000001517523235500164015ustar00rootroot00000000000000incus-7.0.0/internal/server/ip/addr.go000066400000000000000000000033451517523235500176470ustar00rootroot00000000000000package ip import ( "fmt" "net" "github.com/vishvananda/netlink" ) // Addr represents arguments for address protocol manipulation. type Addr struct { DevName string Address *net.IPNet Scope string Family Family } // Add adds new protocol address. func (a *Addr) Add() error { scope, err := a.scopeNum() if err != nil { return err } err = netlink.AddrAdd(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: a.DevName, }, }, &netlink.Addr{ IPNet: a.Address, Scope: scope, }) if err != nil { return fmt.Errorf("Failed to add address %q: %w", a.Address.String(), err) } return nil } func (a *Addr) scopeNum() (int, error) { var scope netlink.Scope switch a.Scope { case "global", "universe", "": scope = netlink.SCOPE_UNIVERSE case "site": scope = netlink.SCOPE_SITE case "link": scope = netlink.SCOPE_LINK case "host": scope = netlink.SCOPE_HOST case "nowhere": scope = netlink.SCOPE_NOWHERE default: return 0, fmt.Errorf("Unknown address scope %q", a.Scope) } return int(scope), nil } // Flush flushes protocol addresses. func (a *Addr) Flush() error { link, err := linkByName(a.DevName) if err != nil { return err } addrs, err := netlink.AddrList(link, int(a.Family)) if err != nil { return fmt.Errorf("Failed to get addresses for device %s: %w", a.DevName, err) } scope, err := a.scopeNum() if err != nil { return err } // NOTE: If this becomes a bottleneck, there appears to be support for batching those kind of changes within netlink. for _, addr := range addrs { if a.Scope != "" && scope != addr.Scope { continue } err := netlink.AddrDel(link, &addr) if err != nil { return fmt.Errorf("Failed to delete address %v: %w", addr, err) } } return nil } incus-7.0.0/internal/server/ip/class.go000066400000000000000000000022361517523235500200400ustar00rootroot00000000000000package ip import ( "fmt" "github.com/vishvananda/netlink" "github.com/lxc/incus/v7/shared/units" ) // Class represents qdisc class object. type Class struct { Dev string Parent string Classid string } // ClassHTB represents htb qdisc class object. type ClassHTB struct { Class Rate string } // Add adds class to a node. func (class *ClassHTB) Add() error { link, err := linkByName(class.Dev) if err != nil { return err } parent, err := parseHandle(class.Parent) if err != nil { return err } classAttrs := netlink.ClassAttrs{ LinkIndex: link.Attrs().Index, Parent: parent, Statistics: nil, } htbClassAttrs := netlink.HtbClassAttrs{} if class.Classid != "" { handle, err := parseHandle(class.Classid) if err != nil { return err } classAttrs.Handle = handle } if class.Rate != "" { rate, err := units.ParseBitSizeString(class.Rate) if err != nil { return fmt.Errorf("Invalid rate %q: %w", class.Rate, err) } htbClassAttrs.Rate = uint64(rate) } err = netlink.ClassAdd(netlink.NewHtbClass(classAttrs, htbClassAttrs)) if err != nil { return fmt.Errorf("Failed to add htb class: %w", err) } return nil } incus-7.0.0/internal/server/ip/filter.go000066400000000000000000000043371517523235500202240ustar00rootroot00000000000000package ip import ( "fmt" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" ) // Action represents an action in filter. type Action interface { toNetlink() (netlink.Action, error) } // ActionPolice represents an action of 'police' type. type ActionPolice struct { Rate uint32 // in byte/s Burst uint32 // in byte Mtu uint32 // in byte Drop bool } func (a *ActionPolice) toNetlink() (netlink.Action, error) { action := netlink.NewPoliceAction() action.Rate = a.Rate action.Burst = a.Burst action.Mtu = a.Mtu if a.Drop { action.ExceedAction = netlink.TC_POLICE_SHOT } else { action.ExceedAction = netlink.TC_POLICE_RECLASSIFY } return action, nil } // Filter represents filter object. type Filter struct { Dev string Parent string Protocol string Flowid string } // U32Filter represents universal 32bit traffic control filter. type U32Filter struct { Filter Value uint32 Mask uint32 Actions []Action } func parseProtocol(proto string) (uint16, error) { switch proto { case "all": return unix.ETH_P_ALL, nil default: return 0, fmt.Errorf("Unknown protocol %q", proto) } } // Add adds universal 32bit traffic control filter to a node. func (u32 *U32Filter) Add() error { link, err := linkByName(u32.Dev) if err != nil { return err } proto, err := parseProtocol(u32.Protocol) if err != nil { return err } filter := &netlink.U32{ FilterAttrs: netlink.FilterAttrs{ LinkIndex: link.Attrs().Index, Protocol: proto, Chain: nil, }, Sel: &netlink.TcU32Sel{ Flags: netlink.TC_U32_TERMINAL, Nkeys: 1, Keys: []netlink.TcU32Key{ { Mask: u32.Mask, Val: u32.Value, }, }, }, } for _, action := range u32.Actions { netlinkAction, err := action.toNetlink() if err != nil { return err } filter.Actions = append(filter.Actions, netlinkAction) } if u32.Parent != "" { parent, err := parseHandle(u32.Parent) if err != nil { return err } filter.Parent = parent } if u32.Flowid != "" { flowid, err := parseHandle(u32.Flowid) if err != nil { return err } filter.ClassId = flowid } err = netlink.FilterAdd(filter) if err != nil { return fmt.Errorf("Failed to add filter %v: %w", filter, err) } return nil } incus-7.0.0/internal/server/ip/init.go000066400000000000000000000001641517523235500176740ustar00rootroot00000000000000package ip import ( "github.com/vishvananda/netlink/nl" ) func init() { nl.EnableErrorMessageReporting = true } incus-7.0.0/internal/server/ip/link.go000066400000000000000000000213611517523235500176700ustar00rootroot00000000000000package ip import ( "fmt" "net" "strconv" "github.com/vishvananda/netlink" ) // Link represents base arguments for link device. type Link struct { Name string Kind string MTU uint32 Parent string Address net.HardwareAddr TXQueueLength uint32 AllMulticast bool Master string Up bool } // LinkInfo has additional information about a link. type LinkInfo struct { Link OperationalState string SlaveKind string VlanID int } func (l *Link) netlinkAttrs() (netlink.LinkAttrs, error) { linkAttrs := netlink.NewLinkAttrs() linkAttrs.Name = l.Name if l.MTU != 0 { linkAttrs.MTU = int(l.MTU) } if l.Address != nil { linkAttrs.HardwareAddr = l.Address } if l.TXQueueLength != 0 { linkAttrs.TxQLen = int(l.TXQueueLength) } if l.Parent != "" { parentLink, err := linkByName(l.Parent) if err != nil { return netlink.LinkAttrs{}, err } linkAttrs.ParentIndex = parentLink.Attrs().Index } if l.Master != "" { masterLink, err := linkByName(l.Master) if err != nil { return netlink.LinkAttrs{}, err } linkAttrs.MasterIndex = masterLink.Attrs().Index } if l.Up { linkAttrs.Flags |= net.FlagUp } return linkAttrs, nil } func (l *Link) addLink(link netlink.Link) error { err := netlink.LinkAdd(link) if err != nil { return err } // ALLMULTI can't be set on create err = l.SetAllMulticast(l.AllMulticast) if err != nil { return err } return nil } // LinkByName returns a Link from a device name. func LinkByName(name string) (LinkInfo, error) { link, err := linkByName(name) if err != nil { return LinkInfo{}, err } var parent, master string if link.Attrs().ParentIndex != 0 { parentLink, err := netlink.LinkByIndex(link.Attrs().ParentIndex) if err == nil { parent = parentLink.Attrs().Name } } if link.Attrs().MasterIndex != 0 { masterLink, err := netlink.LinkByIndex(link.Attrs().MasterIndex) if err != nil { return LinkInfo{}, err } master = masterLink.Attrs().Name } var vlanID int vlan, ok := link.(*netlink.Vlan) if ok { vlanID = vlan.VlanId } return LinkInfo{ Link: Link{ Name: link.Attrs().Name, Kind: link.Type(), MTU: uint32(link.Attrs().MTU), Parent: parent, Address: link.Attrs().HardwareAddr, TXQueueLength: uint32(link.Attrs().TxQLen), AllMulticast: link.Attrs().Allmulti == 1, Master: master, Up: (link.Attrs().Flags & net.FlagUp) != 0, }, OperationalState: link.Attrs().OperState.String(), VlanID: vlanID, }, nil } // SetUp enables the link device. func (l *Link) SetUp() error { return netlink.LinkSetUp(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }) } // SetDown disables the link device. func (l *Link) SetDown() error { return netlink.LinkSetDown(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }) } // SetMTU sets the MTU of the link device. func (l *Link) SetMTU(mtu uint32) error { return netlink.LinkSetMTU(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }, int(mtu)) } // SetTXQueueLength sets the txqueuelen of the link device. func (l *Link) SetTXQueueLength(queueLength uint32) error { return netlink.LinkSetTxQLen(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }, int(queueLength)) } // SetAddress sets the address of the link device. func (l *Link) SetAddress(address net.HardwareAddr) error { return netlink.LinkSetHardwareAddr(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }, address) } // SetAllMulticast when enabled instructs network driver to retrieve all multicast packets from the network to the // kernel for further processing. func (l *Link) SetAllMulticast(enabled bool) error { link := &netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, } if enabled { return netlink.LinkSetAllmulticastOn(link) } return netlink.LinkSetAllmulticastOff(link) } // SetMaster sets the master of the link device. func (l *Link) SetMaster(master string) error { return netlink.LinkSetMaster( &netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }, &netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: master, }, }, ) } // SetNoMaster removes the master of the link device. func (l *Link) SetNoMaster() error { return netlink.LinkSetNoMaster(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }) } // SetName sets the name of the link device. func (l *Link) SetName(newName string) error { return netlink.LinkSetName(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }, newName) } // SetNetns moves the link to the selected network namespace. func (l *Link) SetNetns(netnsPid string) error { pid, err := strconv.Atoi(netnsPid) if err != nil { return err } return netlink.LinkSetNsPid(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }, pid) } // SetVfAddress changes the address for the specified vf. func (l *Link) SetVfAddress(vf string, address string) error { vfInt, err := strconv.Atoi(vf) if err != nil { return err } hwAddress, err := net.ParseMAC(address) if err != nil { return err } return netlink.LinkSetVfHardwareAddr(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }, vfInt, hwAddress) } // SetVfVlan changes the assigned VLAN for the specified vf. func (l *Link) SetVfVlan(vf string, vlan string) error { vfInt, err := strconv.Atoi(vf) if err != nil { return err } vlanInt, err := strconv.Atoi(vlan) if err != nil { return err } return netlink.LinkSetVfVlan(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }, vfInt, vlanInt) } // SetVfSpoofchk turns packet spoof checking on or off for the specified VF. func (l *Link) SetVfSpoofchk(vf string, on bool) error { vfInt, err := strconv.Atoi(vf) if err != nil { return err } return netlink.LinkSetVfSpoofchk(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }, vfInt, on) } // SetVfTrusted turns trusted on or off for the specified VF. func (l *Link) SetVfTrusted(vf string, on bool) error { vfInt, err := strconv.Atoi(vf) if err != nil { return err } return netlink.LinkSetVfTrust(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }, vfInt, on) } // VirtFuncInfo holds information about vf. type VirtFuncInfo struct { VF int Address net.HardwareAddr VLAN int SpoofCheck bool // The value is an uint32 because drivers may not support this property. // In that case, the kernel assigns -1 to this value, which we can check // for by comparing with ^uint32(0), as these are the same bits as the // two's complement representation of -1. // https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/net/core/rtnetlink.c?h=linux-6.18.y#n1513 Trusted uint32 } // GetVFInfo returns info about virtual function. func (l *Link) GetVFInfo(vfID int) (VirtFuncInfo, error) { link, err := linkByName(l.Name) if err != nil { return VirtFuncInfo{}, err } for _, vf := range link.Attrs().Vfs { if vf.ID == vfID { return VirtFuncInfo{ VF: vfID, Address: vf.Mac, VLAN: vf.Vlan, SpoofCheck: vf.Spoofchk, Trusted: vf.Trust, }, nil } } return VirtFuncInfo{}, fmt.Errorf("No matching virtual function found") } // Delete deletes the link device. func (l *Link) Delete() error { return netlink.LinkDel(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }) } // BridgeVLANAdd adds a new vlan filter entry. func (l *Link) BridgeVLANAdd(vid string, pvid bool, untagged bool, self bool) error { vidInt, err := strconv.Atoi(vid) if err != nil { return err } return netlink.BridgeVlanAdd(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }, uint16(vidInt), pvid, untagged, self, !self) } // BridgeVLANDelete removes an existing vlan filter entry. func (l *Link) BridgeVLANDelete(vid string, self bool) error { vidInt, err := strconv.Atoi(vid) if err != nil { return err } return netlink.BridgeVlanDel(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }, uint16(vidInt), false, false, self, !self) } // BridgeLinkSetIsolated sets bridge 'isolated' attribute on a port. func (l *Link) BridgeLinkSetIsolated(isolated bool) error { return netlink.LinkSetIsolated(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }, isolated) } // BridgeLinkSetHairpin sets bridge 'hairpin' attribute on a port. func (l *Link) BridgeLinkSetHairpin(hairpin bool) error { return netlink.LinkSetHairpin(&netlink.GenericLink{ LinkAttrs: netlink.LinkAttrs{ Name: l.Name, }, }, hairpin) } incus-7.0.0/internal/server/ip/link_bridge.go000066400000000000000000000005231517523235500212010ustar00rootroot00000000000000package ip import ( "github.com/vishvananda/netlink" ) // Bridge represents arguments for link device of type bridge. type Bridge struct { Link } // Add adds new virtual link. func (b *Bridge) Add() error { attrs, err := b.netlinkAttrs() if err != nil { return err } return b.addLink(&netlink.Bridge{ LinkAttrs: attrs, }) } incus-7.0.0/internal/server/ip/link_dummy.go000066400000000000000000000005161517523235500211020ustar00rootroot00000000000000package ip import ( "github.com/vishvananda/netlink" ) // Dummy represents arguments for link device of type dummy. type Dummy struct { Link } // Add adds new virtual link. func (d *Dummy) Add() error { attrs, err := d.netlinkAttrs() if err != nil { return err } return d.addLink(&netlink.Dummy{ LinkAttrs: attrs, }) } incus-7.0.0/internal/server/ip/link_gretap.go000066400000000000000000000006371517523235500212350ustar00rootroot00000000000000package ip import ( "net" "github.com/vishvananda/netlink" ) // Gretap represents arguments for link of type gretap. type Gretap struct { Link Local net.IP Remote net.IP } // Add adds new virtual link. func (g *Gretap) Add() error { attrs, err := g.netlinkAttrs() if err != nil { return err } return g.addLink(&netlink.Gretap{ LinkAttrs: attrs, Local: g.Local, Remote: g.Remote, }) } incus-7.0.0/internal/server/ip/link_macvlan.go000066400000000000000000000015371517523235500213740ustar00rootroot00000000000000package ip import ( "github.com/vishvananda/netlink" ) // Macvlan represents arguments for link of type macvlan. type Macvlan struct { Link Mode string } func (macvlan *Macvlan) netlinkMode() netlink.MacvlanMode { switch macvlan.Mode { case "default": return netlink.MACVLAN_MODE_DEFAULT case "private": return netlink.MACVLAN_MODE_PRIVATE case "vepa": return netlink.MACVLAN_MODE_VEPA case "bridge": return netlink.MACVLAN_MODE_BRIDGE case "passthru": return netlink.MACVLAN_MODE_PASSTHRU case "source": return netlink.MACVLAN_MODE_SOURCE default: return netlink.MACVLAN_MODE_DEFAULT } } // Add adds new virtual link. func (macvlan *Macvlan) Add() error { attrs, err := macvlan.netlinkAttrs() if err != nil { return err } return macvlan.addLink(&netlink.Macvlan{ LinkAttrs: attrs, Mode: macvlan.netlinkMode(), }) } incus-7.0.0/internal/server/ip/link_macvtap.go000066400000000000000000000006551517523235500214060ustar00rootroot00000000000000package ip import ( "github.com/vishvananda/netlink" ) // Macvtap represents arguments for link of type macvtap. type Macvtap struct { Macvlan } // Add adds new virtual link. func (macvtap *Macvtap) Add() error { attrs, err := macvtap.netlinkAttrs() if err != nil { return err } return macvtap.addLink(&netlink.Macvtap{ Macvlan: netlink.Macvlan{ LinkAttrs: attrs, Mode: macvtap.netlinkMode(), }, }) } incus-7.0.0/internal/server/ip/link_veth.go000066400000000000000000000013251517523235500207140ustar00rootroot00000000000000package ip import ( "github.com/vishvananda/netlink" ) // Veth represents arguments for link of type veth. type Veth struct { Link Peer Link } // Add adds new virtual link. func (veth *Veth) Add() error { attrs, err := veth.netlinkAttrs() if err != nil { return err } link := netlink.NewVeth(attrs) peerAttrs, err := veth.Peer.netlinkAttrs() if err != nil { return err } link.PeerMTU = uint32(peerAttrs.MTU) link.PeerName = peerAttrs.Name link.PeerNamespace = peerAttrs.Namespace link.PeerNumTxQueues = uint32(peerAttrs.NumTxQueues) link.PeerNumRxQueues = uint32(peerAttrs.NumRxQueues) link.PeerTxQLen = peerAttrs.TxQLen link.PeerHardwareAddr = peerAttrs.HardwareAddr return veth.addLink(link) } incus-7.0.0/internal/server/ip/link_vlan.go000066400000000000000000000010171517523235500207040ustar00rootroot00000000000000package ip import ( "fmt" "strconv" "github.com/vishvananda/netlink" ) // Vlan represents arguments for link of type vlan. type Vlan struct { Link VlanID string Gvrp bool } // Add adds new virtual link. func (vlan *Vlan) Add() error { attrs, err := vlan.netlinkAttrs() if err != nil { return err } id, err := strconv.Atoi(vlan.VlanID) if err != nil { return fmt.Errorf("Invalid VLAN ID: %w", err) } return vlan.addLink(&netlink.Vlan{ LinkAttrs: attrs, VlanId: id, Gvrp: &vlan.Gvrp, }) } incus-7.0.0/internal/server/ip/link_vxlan.go000066400000000000000000000023271517523235500211010ustar00rootroot00000000000000package ip import ( "fmt" "net" "github.com/vishvananda/netlink" ) // Vxlan represents arguments for link of type vxlan. type Vxlan struct { Link VxlanID int DevName string Local net.IP Remote net.IP Group net.IP DstPort int TTL int } // Add adds new virtual link. func (vxlan *Vxlan) Add() error { attrs, err := vxlan.netlinkAttrs() if err != nil { return err } var devIndex int if vxlan.DevName != "" { dev, err := linkByName(vxlan.DevName) if err != nil { return err } devIndex = dev.Attrs().Index } var group net.IP if vxlan.Group != nil { if !vxlan.Group.IsMulticast() { return fmt.Errorf("Group address must be multicast, got %q", vxlan.Group) } group = vxlan.Group } if vxlan.Remote != nil { if group != nil { return fmt.Errorf("Group and remote can not be specified together") } if vxlan.Remote.IsMulticast() { return fmt.Errorf("Remote address must not be multicast, got %q", vxlan.Remote) } group = vxlan.Remote } return vxlan.addLink(&netlink.Vxlan{ LinkAttrs: attrs, VxlanId: vxlan.VxlanID, VtepDevIndex: devIndex, SrcAddr: vxlan.Local, Group: group, TTL: vxlan.TTL, Port: vxlan.DstPort, }) } incus-7.0.0/internal/server/ip/neigh.go000066400000000000000000000053401517523235500200240ustar00rootroot00000000000000package ip import ( "fmt" "net" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" ) // NeighbourIPState can be { NeighbourIPStatePermanent | NeighbourIPStateNoARP | NeighbourIPStateReachable | NeighbourIPStateStale | NeighbourIPStateNone | NeighbourIPStateIncomplete | NeighbourIPStateDelay | NeighbourIPStateProbe | NeighbourIPStateFailed }. type NeighbourIPState int const ( // NeighbourIPStatePermanent the neighbour entry is valid forever and can be only be removed administratively. NeighbourIPStatePermanent NeighbourIPState = unix.NUD_PERMANENT // NeighbourIPStateNoARP the neighbour entry is valid. No attempts to validate this entry will be made but it can // be removed when its lifetime expires. NeighbourIPStateNoARP NeighbourIPState = unix.NUD_NOARP // NeighbourIPStateReachable the neighbour entry is valid until the reachability timeout expires. NeighbourIPStateReachable NeighbourIPState = unix.NUD_REACHABLE // NeighbourIPStateStale the neighbour entry is valid but suspicious. NeighbourIPStateStale NeighbourIPState = unix.NUD_STALE // NeighbourIPStateNone this is a pseudo state used when initially creating a neighbour entry or after trying to // remove it before it becomes free to do so. NeighbourIPStateNone NeighbourIPState = unix.NUD_NONE // NeighbourIPStateIncomplete the neighbour entry has not (yet) been validated/resolved. NeighbourIPStateIncomplete NeighbourIPState = unix.NUD_INCOMPLETE // NeighbourIPStateDelay neighbor entry validation is currently delayed. NeighbourIPStateDelay NeighbourIPState = unix.NUD_DELAY // NeighbourIPStateProbe neighbor is being probed. NeighbourIPStateProbe NeighbourIPState = unix.NUD_PROBE // NeighbourIPStateFailed max number of probes exceeded without success, neighbor validation has ultimately failed. NeighbourIPStateFailed NeighbourIPState = unix.NUD_FAILED ) // Neigh represents arguments for neighbour manipulation. type Neigh struct { DevName string Addr net.IP MAC net.HardwareAddr State NeighbourIPState } // Show list neighbour entries filtered by DevName and optionally MAC address. func (n *Neigh) Show() ([]Neigh, error) { link, err := linkByName(n.DevName) if err != nil { return nil, err } netlinkNeighbours, err := netlink.NeighList(link.Attrs().Index, netlink.FAMILY_ALL) if err != nil { return nil, fmt.Errorf("Failed to get neighbours for link %q: %w", n.DevName, err) } neighbours := make([]Neigh, 0, len(netlinkNeighbours)) for _, neighbour := range netlinkNeighbours { if neighbour.HardwareAddr.String() != n.MAC.String() { continue } neighbours = append(neighbours, Neigh{ Addr: neighbour.IP, MAC: neighbour.HardwareAddr, State: NeighbourIPState(neighbour.State), }) } return neighbours, nil } incus-7.0.0/internal/server/ip/neigh_proxy.go000066400000000000000000000030571517523235500212700ustar00rootroot00000000000000package ip import ( "fmt" "net" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" ) // NeighProxy represents arguments for neighbour proxy manipulation. type NeighProxy struct { DevName string Addr net.IP } // Show list neighbour proxy entries. func (n *NeighProxy) Show() ([]NeighProxy, error) { link, err := linkByName(n.DevName) if err != nil { return nil, err } list, err := netlink.NeighProxyList(link.Attrs().Index, netlink.FAMILY_ALL) if err != nil { return nil, fmt.Errorf("Failed to get neighbour proxies for link %q: %w", n.DevName, err) } entries := make([]NeighProxy, 0, len(list)) for _, neigh := range list { entries = append(entries, NeighProxy{ DevName: n.DevName, Addr: neigh.IP, }) } return entries, nil } func (n *NeighProxy) netlinkNeigh() (*netlink.Neigh, error) { link, err := linkByName(n.DevName) if err != nil { return nil, err } return &netlink.Neigh{ LinkIndex: link.Attrs().Index, Flags: unix.NTF_PROXY, IP: n.Addr, }, nil } // Add a neighbour proxy entry. func (n *NeighProxy) Add() error { neigh, err := n.netlinkNeigh() if err != nil { return err } err = netlink.NeighAdd(neigh) if err != nil { return fmt.Errorf("Failed to add neighbour proxy %v: %w", neigh, err) } return nil } // Delete a neighbour proxy entry. func (n *NeighProxy) Delete() error { neigh, err := n.netlinkNeigh() if err != nil { return err } err = netlink.NeighDel(neigh) if err != nil { return fmt.Errorf("Failed to delete neighbour proxy %v: %w", neigh, err) } return nil } incus-7.0.0/internal/server/ip/qdisc.go000066400000000000000000000016361517523235500200410ustar00rootroot00000000000000package ip import ( "errors" "strings" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" ) // Qdisc represents 'queueing discipline' object. type Qdisc struct { Dev string Handle string Parent string } func (q *Qdisc) netlinkAttrs() (netlink.QdiscAttrs, error) { link, err := linkByName(q.Dev) if err != nil { return netlink.QdiscAttrs{}, err } var handle uint32 if q.Handle != "" { handle, err = parseHandle(q.Handle) if err != nil { return netlink.QdiscAttrs{}, err } } var parent uint32 if q.Parent != "" { parent, err = parseHandle(q.Parent) if err != nil { return netlink.QdiscAttrs{}, err } } return netlink.QdiscAttrs{ LinkIndex: link.Attrs().Index, Handle: handle, Parent: parent, }, nil } func mapQdiscErr(err error) error { if errors.Is(err, unix.EINVAL) && strings.Contains(err.Error(), "Invalid handle") { return unix.ENOENT } return err } incus-7.0.0/internal/server/ip/qdisc_htb.go000066400000000000000000000015031517523235500206670ustar00rootroot00000000000000package ip import ( "fmt" "github.com/vishvananda/netlink" ) // QdiscHTB represents the hierarchy token bucket qdisc object. type QdiscHTB struct { Qdisc Default uint32 } // Add adds a htb qdisc to a device. func (q *QdiscHTB) Add() error { attrs, err := q.netlinkAttrs() if err != nil { return err } htb := netlink.NewHtb(attrs) htb.Defcls = q.Default err = netlink.QdiscAdd(htb) if err != nil { return fmt.Errorf("Failed to add qdisc htb %v: %w", htb, mapQdiscErr(err)) } return nil } // Delete deletes a htb qdisc from a device. func (q *QdiscHTB) Delete() error { attrs, err := q.netlinkAttrs() if err != nil { return err } htb := netlink.NewHtb(attrs) err = netlink.QdiscDel(htb) if err != nil { return fmt.Errorf("Failed to delete qdisc htb %v: %w", htb, mapQdiscErr(err)) } return nil } incus-7.0.0/internal/server/ip/qdisc_ingress.go000066400000000000000000000020271517523235500215660ustar00rootroot00000000000000package ip import ( "errors" "fmt" "github.com/vishvananda/netlink" ) // QdiscIngress represents an ingress qdisc object. type QdiscIngress struct { Qdisc } // Add adds an ingress qdisc to a device. func (q *QdiscIngress) Add() error { attrs, err := q.netlinkAttrs() if err != nil { return err } if q.Parent != "" { return errors.New("Ingress qdisc cannot have parent") } attrs.Parent = netlink.HANDLE_INGRESS ingress := &netlink.Ingress{ QdiscAttrs: attrs, } err = netlink.QdiscAdd(ingress) if err != nil { return fmt.Errorf("Failed to add ingress qdisc %v: %w", ingress, mapQdiscErr(err)) } return nil } // Delete deletes an ingress qdisc from a device. func (q *QdiscIngress) Delete() error { attrs, err := q.netlinkAttrs() if err != nil { return err } attrs.Parent = netlink.HANDLE_INGRESS ingress := &netlink.Ingress{ QdiscAttrs: attrs, } err = netlink.QdiscDel(ingress) if err != nil { return fmt.Errorf("Failed to delete ingress qdisc %v: %w", ingress, mapQdiscErr(err)) } return nil } incus-7.0.0/internal/server/ip/route.go000066400000000000000000000162361517523235500200760ustar00rootroot00000000000000package ip import ( "errors" "fmt" "net" "strconv" "syscall" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" ) // Route represents arguments for route manipulation. type Route struct { DevName string Route *net.IPNet Table string Src net.IP Proto string Family Family Via net.IP VRF string Scope string } type routeBuildMode int const ( routeApply routeBuildMode = iota routeQuery ) func (r *Route) netlinkRoute(mode routeBuildMode) (*netlink.Route, error) { route := &netlink.Route{ Family: int(r.Family), Dst: r.Route, Src: r.Src, Gw: r.Via, } // Device handling. if r.DevName != "" { link, err := linkByName(r.DevName) if err != nil { return nil, err } route.LinkIndex = link.Attrs().Index } else if mode == routeApply { // Mutating routes require a device (for our usage and consistency). return nil, fmt.Errorf("device name must be set") } if r.Table != "" { tableID, err := r.tableID() if err != nil { return nil, fmt.Errorf("Invalid table %q: %w", r.Table, err) } route.Table = tableID } else if r.VRF != "" { vrfDev, err := linkByName(r.VRF) if err != nil { return nil, err } vrf, ok := vrfDev.(*netlink.Vrf) if !ok { return nil, fmt.Errorf("%q is not a vrf", r.VRF) } route.Table = int(vrf.Table) } var err error route.Scope, err = r.netlinkScope() if err != nil { return nil, err } if r.Via == nil { route.Scope = netlink.SCOPE_LINK } if r.Proto != "" { proto, err := r.netlinkProto() if err != nil { return nil, err } route.Protocol = proto } return route, nil } func (r *Route) tableID() (int, error) { switch r.Table { case "default": return unix.RT_TABLE_DEFAULT, nil case "main": return unix.RT_TABLE_MAIN, nil case "local": return unix.RT_TABLE_LOCAL, nil default: return strconv.Atoi(r.Table) } } func (r *Route) netlinkScope() (netlink.Scope, error) { switch r.Scope { case "nowhere": return netlink.SCOPE_NOWHERE, nil case "host": return netlink.SCOPE_HOST, nil case "link": return netlink.SCOPE_LINK, nil case "universe": return netlink.SCOPE_UNIVERSE, nil case "": if r.Via == nil { return netlink.SCOPE_LINK, nil } return netlink.SCOPE_UNIVERSE, nil default: return 0, fmt.Errorf("Invalid scope %q", r.Scope) } } func (r *Route) netlinkProto() (netlink.RouteProtocol, error) { switch r.Proto { case "babel": return unix.RTPROT_BABEL, nil case "bgp": return unix.RTPROT_BGP, nil case "bird": return unix.RTPROT_BIRD, nil case "boot": return unix.RTPROT_BOOT, nil case "dhcp": return unix.RTPROT_DHCP, nil case "dnrouted": return unix.RTPROT_DNROUTED, nil case "eigrp": return unix.RTPROT_EIGRP, nil case "gated": return unix.RTPROT_GATED, nil case "isis": return unix.RTPROT_ISIS, nil case "keepalived": return unix.RTPROT_KEEPALIVED, nil case "kernel": return unix.RTPROT_KERNEL, nil case "mrouted": return unix.RTPROT_MROUTED, nil case "mrt": return unix.RTPROT_MRT, nil case "ntk": return unix.RTPROT_NTK, nil case "ospf": return unix.RTPROT_OSPF, nil case "ra": return unix.RTPROT_RA, nil case "redirect": return unix.RTPROT_REDIRECT, nil case "rip": return unix.RTPROT_RIP, nil case "static": return unix.RTPROT_STATIC, nil case "unspec": return unix.RTPROT_UNSPEC, nil case "xorp": return unix.RTPROT_XORP, nil case "zebra": return unix.RTPROT_ZEBRA, nil default: proto, err := strconv.Atoi(r.Proto) if err != nil { return 0, err } return netlink.RouteProtocol(proto), nil } } // Add adds new route. func (r *Route) Add() error { route, err := r.netlinkRoute(routeApply) if err != nil { return err } err = netlink.RouteAdd(route) if err != nil { return fmt.Errorf("Failed to add route %+v: %w", route, err) } return nil } // Delete deletes routing table. func (r *Route) Delete() error { route, err := r.netlinkRoute(routeApply) if err != nil { return err } err = netlink.RouteDel(route) if err != nil { return fmt.Errorf("Failed to delete route %+v: %w", route, err) } return nil } func routeFilterMask(route *netlink.Route) uint64 { var filterMask uint64 // By default include OIF to filter by interface; // callers (e.g. List) may clear this bit when no device filter is desired. filterMask |= netlink.RT_FILTER_OIF if route.Dst != nil { filterMask |= netlink.RT_FILTER_DST } if route.Gw != nil { filterMask |= netlink.RT_FILTER_GW } if route.Protocol != 0 { filterMask |= netlink.RT_FILTER_PROTOCOL } if route.Table != 0 { filterMask |= netlink.RT_FILTER_TABLE } return filterMask } // Flush flushes routing tables. func (r *Route) Flush() error { route, err := r.netlinkRoute(routeApply) if err != nil { return err } var iterErr error err = netlink.RouteListFilteredIter(route.Family, route, routeFilterMask(route), func(route netlink.Route) (cont bool) { iterErr = netlink.RouteDel(&route) // Ignore missing routes. if errors.Is(iterErr, syscall.ESRCH) { iterErr = nil return true } return iterErr == nil }) if err != nil { return fmt.Errorf("Failed to flush routes matching %+v: %w", route, err) } if iterErr != nil { return fmt.Errorf("Failed to flush routes matching %+v: %w", route, iterErr) } return nil } // Replace changes or adds new route. // If there is already a route with the same destination, metric, tos and table then that route is updated, // otherwise a new route is added. func (r *Route) Replace() error { route, err := r.netlinkRoute(routeApply) if err != nil { return err } err = netlink.RouteReplace(route) if err != nil { return fmt.Errorf("Failed to replace route %s: %w", route, err) } return nil } // List lists matching routes. func (r *Route) List() ([]Route, error) { route, err := r.netlinkRoute(routeQuery) if err != nil { return nil, err } filterMask := routeFilterMask(route) // If no device filter is desired, clear the OIF bit. if r.DevName == "" || route.LinkIndex == 0 { filterMask &^= netlink.RT_FILTER_OIF } netlinkRoutes, err := netlink.RouteListFiltered(route.Family, route, filterMask) if err != nil { return nil, fmt.Errorf("Failed to list routes matching %+v: %w", route, err) } routes := make([]Route, 0, len(netlinkRoutes)) for _, netlinkRoute := range netlinkRoutes { var table string switch netlinkRoute.Table { case unix.RT_TABLE_MAIN: table = "main" case unix.RT_TABLE_LOCAL: table = "local" case unix.RT_TABLE_DEFAULT: table = "default" default: table = strconv.Itoa(netlinkRoute.Table) } devName := r.DevName if devName == "" && netlinkRoute.LinkIndex != 0 { lnk, err := netlink.LinkByIndex(netlinkRoute.LinkIndex) if err == nil { devName = lnk.Attrs().Name } } routes = append(routes, Route{ DevName: devName, Route: netlinkRoute.Dst, Src: netlinkRoute.Src, Via: netlinkRoute.Gw, Scope: netlinkRoute.Scope.String(), Table: table, VRF: "", // adding a route to a VRF just adds it to the table associated with the VRF, so when retrieving routes that information is not available anymore and we just set the table Proto: netlinkRoute.Protocol.String(), Family: Family(netlinkRoute.Family), }) } return routes, nil } incus-7.0.0/internal/server/ip/tuntap.go000066400000000000000000000020171517523235500202430ustar00rootroot00000000000000package ip import ( "fmt" "github.com/vishvananda/netlink" ) // Tuntap represents arguments for tuntap manipulation. type Tuntap struct { Name string Mode string MultiQueue bool Master string } // Add adds new tuntap interface. func (t *Tuntap) Add() error { var mode netlink.TuntapMode switch t.Mode { case "tun": mode = netlink.TUNTAP_MODE_TUN case "tap": mode = netlink.TUNTAP_MODE_TAP default: return fmt.Errorf("Invalid tuntap mode %q", t.Mode) } var flags netlink.TuntapFlag if t.MultiQueue { flags = netlink.TUNTAP_MULTI_QUEUE_DEFAULTS } else { flags = netlink.TUNTAP_DEFAULTS } tuntap := &netlink.Tuntap{ LinkAttrs: netlink.LinkAttrs{ Name: t.Name, }, Mode: mode, Flags: flags, } if t.Master != "" { masterLink, err := linkByName(t.Master) if err != nil { return err } tuntap.MasterIndex = masterLink.Attrs().Index } err := netlink.LinkAdd(tuntap) if err != nil { return fmt.Errorf("Failed to create tuntap %q: %w", t.Name, err) } return nil } incus-7.0.0/internal/server/ip/utils.go000066400000000000000000000027131517523235500200730ustar00rootroot00000000000000package ip import ( "fmt" "net" "strconv" "strings" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" ) // Family can be { FamilyAll, FamilyV4, FamilyV6 }. type Family int const ( // FamilyAll specifies any/all family. FamilyAll Family = unix.AF_UNSPEC // FamilyV4 specifies the IPv4 family. FamilyV4 Family = unix.AF_INET // FamilyV6 specifies the IPv6 family. FamilyV6 Family = unix.AF_INET6 ) func linkByName(name string) (netlink.Link, error) { link, err := netlink.LinkByName(name) if err != nil { return nil, fmt.Errorf("Failed to get link %q: %w", name, err) } return link, nil } func parseHandle(id string) (uint32, error) { if id == "root" { return netlink.HANDLE_ROOT, nil } majorStr, minorStr, found := strings.Cut(id, ":") if !found { return 0, fmt.Errorf("Invalid handle %q", id) } major, err := strconv.ParseUint(majorStr, 16, 16) if err != nil { return 0, fmt.Errorf("Invalid handle %q: %w", id, err) } minor, err := strconv.ParseUint(minorStr, 16, 16) if err != nil { return 0, fmt.Errorf("Invalid handle %q: %w", id, err) } return netlink.MakeHandle(uint16(major), uint16(minor)), nil } // ParseIPNet parses a CIDR string and returns a *net.IPNet containing both the full address and the netmask, // Unlike net.ParseCIDR which zeroes out the host part in the returned *net.IPNet and returns the full address separately. func ParseIPNet(addr string) (*net.IPNet, error) { return netlink.ParseIPNet(addr) } incus-7.0.0/internal/server/ip/vdpa.go000066400000000000000000000065111517523235500176650ustar00rootroot00000000000000package ip import ( "fmt" "os" "path/filepath" "strings" "github.com/vishvananda/netlink" ) // MAX_VQP is the maximum number of VQPs supported by the vDPA device and is always the same as of now. const ( vDPAMaxVQP = uint16(16) ) // vDPA device classes. const ( vdpaBusDevDir = "/sys/bus/vdpa/devices" vdpaVhostDevDir = "/dev" ) // VhostVDPA is the vhost-vdpa device information. type VhostVDPA struct { Name string Path string } // VDPADev represents the vDPA device information. type VDPADev struct { // Name of the vDPA created device. e.g. "vdpa0" (note: the iproute2 associated command would look like `vdpa dev add mgmtdev pci/ name vdpa0 max_vqp `). Name string // Max VQs supported by the vDPA device. MaxVQs uint32 // Associated vhost-vdpa device. VhostVDPA *VhostVDPA } // getVhostVDPADevInPath returns the VhostVDPA found in the provided parent device's path. func getVhostVDPADevInPath(parentPath string) (*VhostVDPA, error) { fd, err := os.Open(parentPath) if err != nil { return nil, fmt.Errorf("Can not open %s: %v", parentPath, err) } defer func() { _ = fd.Close() }() entries, err := fd.ReadDir(-1) if err != nil { return nil, fmt.Errorf("Can not get DirEntries: %v", err) } for _, file := range entries { if strings.Contains(file.Name(), "vhost-vdpa") && file.IsDir() { devicePath := filepath.Join(vdpaVhostDevDir, file.Name()) info, err := os.Stat(devicePath) if err != nil { return nil, fmt.Errorf("Vhost device %s is not a valid device", devicePath) } if info.Mode()&os.ModeDevice == 0 { return nil, fmt.Errorf("Vhost device %s is not a valid device", devicePath) } return &VhostVDPA{ Name: file.Name(), Path: devicePath, }, nil } } return nil, fmt.Errorf("No vhost-vdpa device found in %s", parentPath) } // AddVDPADevice adds a new vDPA device. func AddVDPADevice(pciDevSlotName string, volatile map[string]string) (*VDPADev, error) { existingDevices, err := netlink.VDPAGetDevList() if err != nil { return nil, fmt.Errorf("Failed to get vdpa dev list: %w", err) } existingVDPADevNames := make(map[string]struct{}) for _, device := range existingDevices { existingVDPADevNames[device.Name] = struct{}{} } // Generate a unique attribute name for the vDPA device (i.e, vdpa0, vdpa1, etc.) baseVDPAName, idx, generatedVDPADevName := "vdpa", 0, "" for { generatedVDPADevName = fmt.Sprintf("%s%d", baseVDPAName, idx) _, ok := existingVDPADevNames[generatedVDPADevName] if !ok { break } idx++ } err = netlink.VDPANewDev(generatedVDPADevName, "pci", pciDevSlotName, netlink.VDPANewDevParams{ MaxVQP: vDPAMaxVQP, }) if err != nil { return nil, err } dev, err := netlink.VDPAGetDevByName(generatedVDPADevName) if err != nil { return nil, fmt.Errorf("Failed to get vdpa device %q: %w", generatedVDPADevName, err) } vhostVDPA, err := getVhostVDPADevInPath(filepath.Join(vdpaBusDevDir, dev.Name)) if err != nil { return nil, fmt.Errorf("Failed to get vdpa vhost %q: %w", dev.Name, err) } // Update the volatile map volatile["last_state.vdpa.name"] = generatedVDPADevName return &VDPADev{ Name: dev.Name, MaxVQs: dev.MaxVQS, VhostVDPA: vhostVDPA, }, nil } // DeleteVDPADevice deletes a vDPA management device. func DeleteVDPADevice(vDPADevName string) error { return netlink.VDPADelDev(vDPADevName) } incus-7.0.0/internal/server/lifecycle/000077500000000000000000000000001517523235500177305ustar00rootroot00000000000000incus-7.0.0/internal/server/lifecycle/certificate.go000066400000000000000000000016251517523235500225450ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // CertificateAction represents a lifecycle event action for Certificates. type CertificateAction string // All supported lifecycle events for Certificates. const ( CertificateCreated = CertificateAction(api.EventLifecycleCertificateCreated) CertificateDeleted = CertificateAction(api.EventLifecycleCertificateDeleted) CertificateUpdated = CertificateAction(api.EventLifecycleCertificateUpdated) ) // Event creates the lifecycle event for an action on a Certificate. func (a CertificateAction) Event(fingerprint string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "certificates", fingerprint) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/cluster.go000066400000000000000000000017051517523235500217430ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // ClusterAction represents a lifecycle event action for clusters. type ClusterAction string // All supported lifecycle events for clusters. const ( ClusterEnabled = ClusterAction(api.EventLifecycleClusterEnabled) ClusterDisabled = ClusterAction(api.EventLifecycleClusterDisabled) ClusterCertificateUpdated = ClusterAction(api.EventLifecycleClusterCertificateUpdated) ClusterTokenCreated = ClusterAction(api.EventLifecycleClusterTokenCreated) ) // Event creates the lifecycle event for an action on a cluster. func (a ClusterAction) Event(name string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "cluster", name) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/cluster_groups.go000066400000000000000000000017571517523235500233510ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // ClusterGroupAction represents a lifecycle event action for cluster groups. type ClusterGroupAction string // All supported lifecycle events for cluster groups. const ( ClusterGroupCreated = ClusterGroupAction(api.EventLifecycleClusterGroupCreated) ClusterGroupDeleted = ClusterGroupAction(api.EventLifecycleClusterGroupDeleted) ClusterGroupUpdated = ClusterGroupAction(api.EventLifecycleClusterGroupUpdated) ClusterGroupRenamed = ClusterGroupAction(api.EventLifecycleClusterGroupRenamed) ) // Event creates the lifecycle event for an action on a cluster group. func (a ClusterGroupAction) Event(name string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "cluster", "groups", name) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/cluster_member.go000066400000000000000000000024141517523235500232700ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // ClusterMemberAction represents a lifecycle event action for cluster members. type ClusterMemberAction string // All supported lifecycle events for cluster members. const ( ClusterMemberAdded = ClusterMemberAction(api.EventLifecycleClusterMemberAdded) ClusterMemberEvacuated = ClusterMemberAction(api.EventLifecycleClusterMemberEvacuated) ClusterMemberHealed = ClusterMemberAction(api.EventLifecycleClusterMemberHealed) ClusterMemberRemoved = ClusterMemberAction(api.EventLifecycleClusterMemberRemoved) ClusterMemberRenamed = ClusterMemberAction(api.EventLifecycleClusterMemberRenamed) ClusterMemberRestored = ClusterMemberAction(api.EventLifecycleClusterMemberRestored) ClusterMemberUpdated = ClusterMemberAction(api.EventLifecycleClusterMemberUpdated) ) // Event creates the lifecycle event for an action on a cluster member. func (a ClusterMemberAction) Event(name string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "cluster", "members", name) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/config.go000066400000000000000000000013151517523235500215240ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // ConfigAction represents a lifecycle event action for the server configuration. type ConfigAction string // All supported lifecycle events for the server configuration. const ( ConfigUpdated = ConfigAction(api.EventLifecycleConfigUpdated) ) // Event creates the lifecycle event for an action on the server configuration. func (a ConfigAction) Event(requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/image.go000066400000000000000000000020651517523235500213440ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // ImageAction represents a lifecycle event action for images. type ImageAction string // All supported lifecycle events for images. const ( ImageCreated = ImageAction(api.EventLifecycleImageCreated) ImageDeleted = ImageAction(api.EventLifecycleImageDeleted) ImageUpdated = ImageAction(api.EventLifecycleImageUpdated) ImageRetrieved = ImageAction(api.EventLifecycleImageRetrieved) ImageRefreshed = ImageAction(api.EventLifecycleImageRefreshed) ImageSecretCreated = ImageAction(api.EventLifecycleImageSecretCreated) ) // Event creates the lifecycle event for an action on an image. func (a ImageAction) Event(image string, projectName string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "images", image).Project(projectName) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/image_alias.go000066400000000000000000000017711517523235500225200ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // ImageAliasAction represents a lifecycle event action for image aliases. type ImageAliasAction string // All supported lifecycle events for image aliases. const ( ImageAliasCreated = ImageAliasAction(api.EventLifecycleImageAliasCreated) ImageAliasDeleted = ImageAliasAction(api.EventLifecycleImageAliasDeleted) ImageAliasUpdated = ImageAliasAction(api.EventLifecycleImageAliasUpdated) ImageAliasRenamed = ImageAliasAction(api.EventLifecycleImageAliasRenamed) ) // Event creates the lifecycle event for an action on an image alias. func (a ImageAliasAction) Event(image string, projectName string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "images", "aliases", image).Project(projectName) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/instance.go000066400000000000000000000053401517523235500220650ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // Internal copy of the instance interface. type instance interface { Name() string Project() api.Project Operation() *operations.Operation } // InstanceAction represents a lifecycle event action for instances. type InstanceAction string // All supported lifecycle events for instances. const ( InstanceAgentStarted = InstanceAction(api.EventLifecycleInstanceAgentStarted) InstanceAgentStopped = InstanceAction(api.EventLifecycleInstanceAgentStopped) InstanceConsole = InstanceAction(api.EventLifecycleInstanceConsole) InstanceConsoleReset = InstanceAction(api.EventLifecycleInstanceConsoleReset) InstanceConsoleRetrieved = InstanceAction(api.EventLifecycleInstanceConsoleRetrieved) InstanceCreated = InstanceAction(api.EventLifecycleInstanceCreated) InstanceDeleted = InstanceAction(api.EventLifecycleInstanceDeleted) InstanceExec = InstanceAction(api.EventLifecycleInstanceExec) InstanceFileDeleted = InstanceAction(api.EventLifecycleInstanceFileDeleted) InstanceFilePushed = InstanceAction(api.EventLifecycleInstanceFilePushed) InstanceFileRetrieved = InstanceAction(api.EventLifecycleInstanceFileRetrieved) InstanceMigrated = InstanceAction(api.EventLifecycleInstanceMigrated) InstancePaused = InstanceAction(api.EventLifecycleInstancePaused) InstanceReady = InstanceAction(api.EventLifecycleInstanceReady) InstanceRenamed = InstanceAction(api.EventLifecycleInstanceRenamed) InstanceRestarted = InstanceAction(api.EventLifecycleInstanceRestarted) InstanceRestored = InstanceAction(api.EventLifecycleInstanceRestored) InstanceResumed = InstanceAction(api.EventLifecycleInstanceResumed) InstanceShutdown = InstanceAction(api.EventLifecycleInstanceShutdown) InstanceStarted = InstanceAction(api.EventLifecycleInstanceStarted) InstanceStopped = InstanceAction(api.EventLifecycleInstanceStopped) InstanceUpdated = InstanceAction(api.EventLifecycleInstanceUpdated) ) // Event creates the lifecycle event for an action on an instance. func (a InstanceAction) Event(inst instance, ctx map[string]any) api.EventLifecycle { url := api.NewURL().Path(version.APIVersion, "instances", inst.Name()).Project(inst.Project().Name) var requestor *api.EventLifecycleRequestor if inst.Operation() != nil { requestor = inst.Operation().Requestor() } return api.EventLifecycle{ Action: string(a), Source: url.String(), Context: ctx, Requestor: requestor, Name: inst.Name(), Project: inst.Project().Name, } } incus-7.0.0/internal/server/lifecycle/instance_backup.go000066400000000000000000000023761517523235500234200ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // InstanceBackupAction represents a lifecycle event action for instance backups. type InstanceBackupAction string // All supported lifecycle events for instance backups. const ( InstanceBackupCreated = InstanceBackupAction(api.EventLifecycleInstanceBackupCreated) InstanceBackupDeleted = InstanceBackupAction(api.EventLifecycleInstanceBackupDeleted) InstanceBackupRenamed = InstanceBackupAction(api.EventLifecycleInstanceBackupRenamed) InstanceBackupRetrieved = InstanceBackupAction(api.EventLifecycleInstanceBackupRetrieved) ) // Event creates the lifecycle event for an action on an instance backup. func (a InstanceBackupAction) Event(fullBackupName string, inst instance, ctx map[string]any) api.EventLifecycle { _, backupName, _ := api.GetParentAndSnapshotName(fullBackupName) u := api.NewURL().Path(version.APIVersion, "instances", inst.Name(), "backups", backupName).Project(inst.Project().Name) var requestor *api.EventLifecycleRequestor if inst.Operation() != nil { requestor = inst.Operation().Requestor() } return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/instance_log.go000066400000000000000000000016041517523235500227250ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // InstanceLogAction represents a lifecycle event action for instance logs. type InstanceLogAction string // All supported lifecycle events for instance logs. const ( InstanceLogRetrieved = InstanceLogAction(api.EventLifecycleInstanceLogRetrieved) InstanceLogDeleted = InstanceLogAction(api.EventLifecycleInstanceLogDeleted) ) // Event creates the lifecycle event for an action on an instance log. func (a InstanceLogAction) Event(file string, inst instance, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "instances", inst.Name(), "backups", file).Project(inst.Project().Name) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/instance_metadata.go000066400000000000000000000016511517523235500237260ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // InstanceMetadataAction represents a lifecycle event action for instance metadata. type InstanceMetadataAction string // All supported lifecycle events for instance metadata. const ( InstanceMetadataUpdated = InstanceMetadataAction(api.EventLifecycleInstanceMetadataUpdated) InstanceMetadataRetrieved = InstanceMetadataAction(api.EventLifecycleInstanceMetadataRetrieved) ) // Event creates the lifecycle event for an action on instance metadata. func (a InstanceMetadataAction) Event(inst instance, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "instances", inst.Name(), "metadata").Project(inst.Project().Name) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/instance_metadata_template.go000066400000000000000000000022231517523235500256150ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // InstanceMetadataTemplateAction represents a lifecycle event action for instance metadata templates. type InstanceMetadataTemplateAction string // All supported lifecycle events for instance metadata templates. const ( InstanceMetadataTemplateDeleted = InstanceMetadataTemplateAction(api.EventLifecycleInstanceMetadataTemplateDeleted) InstanceMetadataTemplateCreated = InstanceMetadataTemplateAction(api.EventLifecycleInstanceMetadataTemplateCreated) InstanceMetadataTemplateRetrieved = InstanceMetadataTemplateAction(api.EventLifecycleInstanceMetadataTemplateRetrieved) ) // Event creates the lifecycle event for an action on instance metadata templates. func (a InstanceMetadataTemplateAction) Event(inst instance, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "instances", inst.Name(), "metadata", "templates").Project(inst.Project().Name) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/instance_snapshot.go000066400000000000000000000024041517523235500240020ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // InstanceSnapshotAction represents a lifecycle event action for instance snapshots. type InstanceSnapshotAction string // All supported lifecycle events for instance snapshots. const ( InstanceSnapshotCreated = InstanceSnapshotAction(api.EventLifecycleInstanceSnapshotCreated) InstanceSnapshotDeleted = InstanceSnapshotAction(api.EventLifecycleInstanceSnapshotDeleted) InstanceSnapshotRenamed = InstanceSnapshotAction(api.EventLifecycleInstanceSnapshotRenamed) InstanceSnapshotUpdated = InstanceSnapshotAction(api.EventLifecycleInstanceSnapshotUpdated) ) // Event creates the lifecycle event for an action on an instance snapshot. func (a InstanceSnapshotAction) Event(inst instance, ctx map[string]any) api.EventLifecycle { parentName, snapName, _ := api.GetParentAndSnapshotName(inst.Name()) u := api.NewURL().Path(version.APIVersion, "instances", parentName, "snapshots", snapName).Project(inst.Project().Name) var requestor *api.EventLifecycleRequestor if inst.Operation() != nil { requestor = inst.Operation().Requestor() } return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/network.go000066400000000000000000000020351517523235500217500ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // Internal copy of the network interface. type network interface { Name() string Project() string } // NetworkAction represents a lifecycle event action for network devices. type NetworkAction string // All supported lifecycle events for network devices. const ( NetworkCreated = NetworkAction(api.EventLifecycleNetworkCreated) NetworkDeleted = NetworkAction(api.EventLifecycleNetworkDeleted) NetworkUpdated = NetworkAction(api.EventLifecycleNetworkUpdated) NetworkRenamed = NetworkAction(api.EventLifecycleNetworkRenamed) ) // Event creates the lifecycle event for an action on a network device. func (a NetworkAction) Event(n network, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "networks", n.Name()).Project(n.Project()) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/network_acl.go000066400000000000000000000021351517523235500225700ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // Internal copy of the network acl interface. type networkACL interface { Info() *api.NetworkACL Project() string } // NetworkACLAction represents a lifecycle event action for network acls. type NetworkACLAction string // All supported lifecycle events for network acls. const ( NetworkACLCreated = NetworkACLAction(api.EventLifecycleNetworkACLCreated) NetworkACLDeleted = NetworkACLAction(api.EventLifecycleNetworkACLDeleted) NetworkACLUpdated = NetworkACLAction(api.EventLifecycleNetworkACLUpdated) NetworkACLRenamed = NetworkACLAction(api.EventLifecycleNetworkACLRenamed) ) // Event creates the lifecycle event for an action on a network acl. func (a NetworkACLAction) Event(n networkACL, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "network-acls", n.Info().Name).Project(n.Project()) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/network_address_set.go000066400000000000000000000024331517523235500243320ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // NetworkAddressSet ia an internal copy of the network address set interface. type NetworkAddressSet interface { Info() *api.NetworkAddressSet Project() string } // NetworkAddressSetAction represents a lifecycle event action for network address sets. type NetworkAddressSetAction string // All supported lifecycle events for network address sets. const ( NetworkAddressSetCreated = NetworkAddressSetAction(api.EventLifecycleNetworkAddressSetCreated) NetworkAddressSetDeleted = NetworkAddressSetAction(api.EventLifecycleNetworkAddressSetDeleted) NetworkAddressSetUpdated = NetworkAddressSetAction(api.EventLifecycleNetworkAddressSetUpdated) NetworkAddressSetRenamed = NetworkAddressSetAction(api.EventLifecycleNetworkAddressSetRenamed) ) // Event creates the lifecycle event for an action on a network address set. func (a NetworkAddressSetAction) Event(n NetworkAddressSet, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "network-address-sets", n.Info().Name).Project(n.Project()) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/network_forward.go000066400000000000000000000017731517523235500235040ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // NetworkForwardAction represents a lifecycle event action for network forwards. type NetworkForwardAction string // All supported lifecycle events for network forwards. const ( NetworkForwardCreated = NetworkForwardAction(api.EventLifecycleNetworkForwardCreated) NetworkForwardDeleted = NetworkForwardAction(api.EventLifecycleNetworkForwardDeleted) NetworkForwardUpdated = NetworkForwardAction(api.EventLifecycleNetworkForwardUpdated) ) // Event creates the lifecycle event for an action on a network forward. func (a NetworkForwardAction) Event(n network, listenAddress string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "networks", n.Name(), "forwards", listenAddress).Project(n.Project()) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/network_integration.go000066400000000000000000000021361517523235500243550ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // NetworkIntegrationAction represents a lifecycle event action for network integrations. type NetworkIntegrationAction string // All supported lifecycle events for network integrations. const ( NetworkIntegrationCreated = NetworkIntegrationAction(api.EventLifecycleNetworkIntegrationCreated) NetworkIntegrationDeleted = NetworkIntegrationAction(api.EventLifecycleNetworkIntegrationDeleted) NetworkIntegrationUpdated = NetworkIntegrationAction(api.EventLifecycleNetworkIntegrationUpdated) NetworkIntegrationRenamed = NetworkIntegrationAction(api.EventLifecycleNetworkIntegrationRenamed) ) // Event creates the lifecycle event for an action on a network integration. func (a NetworkIntegrationAction) Event(name string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "network-integrations", name) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/network_load_balancer.go000066400000000000000000000021171517523235500245770ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // NetworkLoadBalancerAction represents a lifecycle event action for network load balancers. type NetworkLoadBalancerAction string // All supported lifecycle events for network load balancers. const ( NetworkLoadBalancerCreated = NetworkLoadBalancerAction(api.EventLifecycleNetworkLoadBalancerCreated) NetworkLoadBalancerDeleted = NetworkLoadBalancerAction(api.EventLifecycleNetworkLoadBalancerDeleted) NetworkLoadBalancerUpdated = NetworkLoadBalancerAction(api.EventLifecycleNetworkLoadBalancerUpdated) ) // Event creates the lifecycle event for an action on a network load balancer. func (a NetworkLoadBalancerAction) Event(n network, listenAddress string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "networks", n.Name(), "load-balancers", listenAddress).Project(n.Project()) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/network_peer.go000066400000000000000000000017151517523235500227670ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // NetworkPeerAction represents a lifecycle event action for network peers. type NetworkPeerAction string // All supported lifecycle events for network peers. const ( NetworkPeerCreated = NetworkForwardAction(api.EventLifecycleNetworkPeerCreated) NetworkPeerDeleted = NetworkForwardAction(api.EventLifecycleNetworkPeerDeleted) NetworkPeerUpdated = NetworkForwardAction(api.EventLifecycleNetworkPeerUpdated) ) // Event creates the lifecycle event for an action on a network forward. func (a NetworkPeerAction) Event(n network, peerName string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "networks", n.Name(), "peers", peerName).Project(n.Project()) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/network_zone.go000066400000000000000000000036261517523235500230120ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // Internal copy of the network zone interface. type networkZone interface { Info() *api.NetworkZone Project() string } // NetworkZoneAction represents a lifecycle event action for network zones. type NetworkZoneAction string // NetworkZoneRecordAction represents a lifecycle event action for network zone records. type NetworkZoneRecordAction string // All supported lifecycle events for network zones. const ( NetworkZoneCreated = NetworkZoneAction(api.EventLifecycleNetworkZoneCreated) NetworkZoneDeleted = NetworkZoneAction(api.EventLifecycleNetworkZoneDeleted) NetworkZoneUpdated = NetworkZoneAction(api.EventLifecycleNetworkZoneUpdated) NetworkZoneRecordCreated = NetworkZoneRecordAction(api.EventLifecycleNetworkZoneRecordCreated) NetworkZoneRecordDeleted = NetworkZoneRecordAction(api.EventLifecycleNetworkZoneRecordDeleted) NetworkZoneRecordUpdated = NetworkZoneRecordAction(api.EventLifecycleNetworkZoneRecordUpdated) ) // Event creates the lifecycle event for an action on a network zone. func (a NetworkZoneAction) Event(n networkZone, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "network-zones", n.Info().Name).Project(n.Project()) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } // Event creates the lifecycle event for an action on a network zone record. func (a NetworkZoneRecordAction) Event(n networkZone, name string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "network-zones", n.Info().Name, "records", name).Project(n.Project()) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/operation.go000066400000000000000000000014701517523235500222610ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // Internal copy of the operation interface. type operation interface { ID() string } // OperationAction represents a lifecycle event action for operations. type OperationAction string // All supported lifecycle events for operations. const ( OperationCancelled = OperationAction(api.EventLifecycleOperationCancelled) ) // Event creates the lifecycle event for an action on an operation. func (a OperationAction) Event(op operation, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "operations", op.ID()) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/profile.go000066400000000000000000000016621517523235500217240ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // ProfileAction represents a lifecycle event action for profiles. type ProfileAction string // All supported lifecycle events for profiles. const ( ProfileCreated = ProfileAction(api.EventLifecycleProfileCreated) ProfileDeleted = ProfileAction(api.EventLifecycleProfileDeleted) ProfileUpdated = ProfileAction(api.EventLifecycleProfileUpdated) ProfileRenamed = ProfileAction(api.EventLifecycleProfileRenamed) ) // Event creates the lifecycle event for an action on a profile. func (a ProfileAction) Event(name string, projectName string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "profiles", name).Project(projectName) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/project.go000066400000000000000000000016111517523235500217240ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // ProjectAction represents a lifecycle event action for projects. type ProjectAction string // All supported lifecycle events for projects. const ( ProjectCreated = ProjectAction(api.EventLifecycleProjectCreated) ProjectDeleted = ProjectAction(api.EventLifecycleProjectDeleted) ProjectUpdated = ProjectAction(api.EventLifecycleProjectUpdated) ProjectRenamed = ProjectAction(api.EventLifecycleProjectRenamed) ) // Event creates the lifecycle event for an action on a project. func (a ProjectAction) Event(name string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "projects", name) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/storage_bucket.go000066400000000000000000000037731517523235500232720ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // Internal copy of the pool interface. type pool interface { Name() string } // StorageBucketAction represents a lifecycle event action for storage buckets. type StorageBucketAction string // StorageBucketKeyAction represents a lifecycle event action for storage bucket keys. type StorageBucketKeyAction string // All supported lifecycle events for storage buckets and keys. const ( StorageBucketCreated = StorageBucketAction(api.EventLifecycleStorageBucketCreated) StorageBucketDeleted = StorageBucketAction(api.EventLifecycleStorageBucketDeleted) StorageBucketUpdated = StorageBucketAction(api.EventLifecycleStorageBucketUpdated) StorageBucketKeyCreated = StorageBucketKeyAction(api.EventLifecycleStorageBucketKeyCreated) StorageBucketKeyDeleted = StorageBucketKeyAction(api.EventLifecycleStorageBucketKeyDeleted) StorageBucketKeyUpdated = StorageBucketKeyAction(api.EventLifecycleStorageBucketKeyUpdated) ) // Event creates the lifecycle event for an action on a storage bucket. func (a StorageBucketAction) Event(pool pool, projectName string, bucketName string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "storage-pools", pool.Name(), "buckets", bucketName).Project(projectName) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } // Event creates the lifecycle event for an action on a storage bucket. func (a StorageBucketKeyAction) Event(pool pool, projectName string, bucketName string, keyName string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "storage-pools", pool.Name(), "buckets", bucketName, "keys", keyName).Project(projectName) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/storage_bucket_backup.go000066400000000000000000000024701517523235500246100ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // StorageBucketBackupAction represents a lifecycle event action for storage bucket backups. type StorageBucketBackupAction string // All supported lifecycle events for storage volume backups. const ( StorageBucketBackupCreated = StorageBucketBackupAction(api.EventLifecycleStorageBucketBackupCreated) StorageBucketBackupDeleted = StorageBucketBackupAction(api.EventLifecycleStorageBucketBackupDeleted) StorageBucketBackupRetrieved = StorageBucketBackupAction(api.EventLifecycleStorageBucketBackupRetrieved) StorageBucketBackupRenamed = StorageBucketBackupAction(api.EventLifecycleStorageBucketBackupRenamed) ) // Event creates the lifecycle event for an action on a storage volume backup. func (a StorageBucketBackupAction) Event(poolName string, fullBackupName string, projectName string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { bucketName, backupName, _ := api.GetParentAndSnapshotName(fullBackupName) u := api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "buckets", bucketName, "backups", backupName).Project(projectName) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/storage_pool.go000066400000000000000000000016141517523235500227560ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // StoragePoolAction represents a lifecycle event action for storage pools. type StoragePoolAction string // All supported lifecycle events for storage pools. const ( StoragePoolCreated = StoragePoolAction(api.EventLifecycleStoragePoolCreated) StoragePoolDeleted = StoragePoolAction(api.EventLifecycleStoragePoolDeleted) StoragePoolUpdated = StoragePoolAction(api.EventLifecycleStoragePoolUpdated) ) // Event creates the lifecycle event for an action on an storage pool. func (a StoragePoolAction) Event(name string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "storage-pools", name) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/storage_volume.go000066400000000000000000000035251517523235500233170ustar00rootroot00000000000000package lifecycle import ( "strings" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // Internal copy of the volume interface. type volume interface { Name() string Pool() string } // StorageVolumeAction represents a lifecycle event action for storage volumes. type StorageVolumeAction string // All supported lifecycle events for storage volumes. const ( StorageVolumeCreated = StorageVolumeAction(api.EventLifecycleStorageVolumeCreated) StorageVolumeDeleted = StorageVolumeAction(api.EventLifecycleStorageVolumeDeleted) StorageVolumeFileDeleted = StorageVolumeAction(api.EventLifecycleStorageVolumeFileDeleted) StorageVolumeFilePushed = StorageVolumeAction(api.EventLifecycleStorageVolumeFilePushed) StorageVolumeFileRetrieved = StorageVolumeAction(api.EventLifecycleStorageVolumeFileRetrieved) StorageVolumeUpdated = StorageVolumeAction(api.EventLifecycleStorageVolumeUpdated) StorageVolumeRenamed = StorageVolumeAction(api.EventLifecycleStorageVolumeRenamed) StorageVolumeRestored = StorageVolumeAction(api.EventLifecycleStorageVolumeRestored) ) // Event creates the lifecycle event for an action on a storage volume. func (a StorageVolumeAction) Event(v volume, volumeType string, projectName string, op *operations.Operation, ctx map[string]any) api.EventLifecycle { volName := v.Name() if volumeType == "custom" { fields := strings.SplitN(volName, "_", 2) volName = fields[1] } u := api.NewURL().Path(version.APIVersion, "storage-pools", v.Pool(), "volumes", volumeType, volName).Project(projectName) var requestor *api.EventLifecycleRequestor if op != nil { requestor = op.Requestor() } return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/storage_volume_backup.go000066400000000000000000000025271517523235500246450ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // StorageVolumeBackupAction represents a lifecycle event action for storage volume backups. type StorageVolumeBackupAction string // All supported lifecycle events for storage volume backups. const ( StorageVolumeBackupCreated = StorageVolumeBackupAction(api.EventLifecycleStorageVolumeBackupCreated) StorageVolumeBackupDeleted = StorageVolumeBackupAction(api.EventLifecycleStorageVolumeBackupDeleted) StorageVolumeBackupRetrieved = StorageVolumeBackupAction(api.EventLifecycleStorageVolumeBackupRetrieved) StorageVolumeBackupRenamed = StorageVolumeBackupAction(api.EventLifecycleStorageVolumeBackupRenamed) ) // Event creates the lifecycle event for an action on a storage volume backup. func (a StorageVolumeBackupAction) Event(poolName string, volumeType string, fullBackupName string, projectName string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { volumeName, backupName, _ := api.GetParentAndSnapshotName(fullBackupName) u := api.NewURL().Path(version.APIVersion, "storage-pools", poolName, "volumes", volumeType, volumeName, "backups", backupName).Project(projectName) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/storage_volume_snapshot.go000066400000000000000000000027301517523235500252330ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // StorageVolumeSnapshotAction represents a lifecycle event action for storage volume snapshots. type StorageVolumeSnapshotAction string // All supported lifecycle events for storage volume snapshots. const ( StorageVolumeSnapshotCreated = StorageVolumeSnapshotAction(api.EventLifecycleStorageVolumeSnapshotCreated) StorageVolumeSnapshotDeleted = StorageVolumeSnapshotAction(api.EventLifecycleStorageVolumeSnapshotDeleted) StorageVolumeSnapshotUpdated = StorageVolumeSnapshotAction(api.EventLifecycleStorageVolumeSnapshotUpdated) StorageVolumeSnapshotRenamed = StorageVolumeSnapshotAction(api.EventLifecycleStorageVolumeSnapshotRenamed) ) // Event creates the lifecycle event for an action on a storage volume snapshot. func (a StorageVolumeSnapshotAction) Event(v volume, volumeType string, projectName string, op *operations.Operation, ctx map[string]any) api.EventLifecycle { parentName, snapshotName, _ := api.GetParentAndSnapshotName(v.Name()) u := api.NewURL().Path(version.APIVersion, "storage-pools", v.Pool(), "volumes", volumeType, parentName, "snapshots", snapshotName).Project(projectName) var requestor *api.EventLifecycleRequestor if op != nil { requestor = op.Requestor() } return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/lifecycle/warning.go000066400000000000000000000015251517523235500217270ustar00rootroot00000000000000package lifecycle import ( "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" ) // WarningAction represents a lifecycle event action for warnings. type WarningAction string // All supported lifecycle events for warnings. const ( WarningAcknowledged = WarningAction(api.EventLifecycleWarningAcknowledged) WarningReset = WarningAction(api.EventLifecycleWarningReset) WarningDeleted = WarningAction(api.EventLifecycleWarningDeleted) ) // Event creates the lifecycle event for an action on a warning. func (a WarningAction) Event(id string, requestor *api.EventLifecycleRequestor, ctx map[string]any) api.EventLifecycle { u := api.NewURL().Path(version.APIVersion, "warnings", id) return api.EventLifecycle{ Action: string(a), Source: u.String(), Context: ctx, Requestor: requestor, } } incus-7.0.0/internal/server/locking/000077500000000000000000000000001517523235500174175ustar00rootroot00000000000000incus-7.0.0/internal/server/locking/lock.go000066400000000000000000000044371517523235500207060ustar00rootroot00000000000000package locking import ( "context" "fmt" "sync" ) // locks is a hashmap that allows functions to check whether the operation they are about to perform // is already in progress. If it is the channel can be used to wait for the operation to finish. If it is not, the // function that wants to perform the operation should store its code in the hashmap. // Note that any access to this map must be done while holding a lock. var locks = map[string]chan struct{}{} // locksMutex is used to access locks safely. var locksMutex sync.Mutex // UnlockFunc unlocks the lock. type UnlockFunc func() // Lock creates a named lock to allow activities that require exclusive access to occur. // Will block until the lock is established or the context is cancelled. // On successfully acquiring the lock, it returns an unlock function which needs to be called to unlock the lock. // If the context is canceled then nil will be returned. func Lock(ctx context.Context, lockName string) (UnlockFunc, error) { for { // Get exclusive access to the map and see if there is already an operation ongoing. locksMutex.Lock() waitCh, ok := locks[lockName] if !ok { // No ongoing operation, create a new channel to indicate our new operation. waitCh = make(chan struct{}) locks[lockName] = waitCh locksMutex.Unlock() // Return a function that will complete the operation. return func() { // Get exclusive access to the map. locksMutex.Lock() doneCh, ok := locks[lockName] // Load our existing operation. if ok { // Close the channel to indicate to other waiting users // they can now try again to create a new operation. close(doneCh) // Remove our existing operation entry from the map. delete(locks, lockName) } // Release the lock now that the done channel is closed and the // map entry has been deleted, this will allow any waiting users // to try and get access to the map to create a new operation. locksMutex.Unlock() }, nil } // An existing operation is ongoing, lets wait for that to finish and then try // to get exclusive access to create a new operation again. locksMutex.Unlock() select { case <-waitCh: continue case <-ctx.Done(): return nil, fmt.Errorf("Failed to obtain lock %q: %w", lockName, ctx.Err()) } } } incus-7.0.0/internal/server/logging/000077500000000000000000000000001517523235500174175ustar00rootroot00000000000000incus-7.0.0/internal/server/logging/common.go000066400000000000000000000045171517523235500212450ustar00rootroot00000000000000package logging import ( "encoding/json" "github.com/sirupsen/logrus" clusterConfig "github.com/lxc/incus/v7/internal/server/cluster/config" "github.com/lxc/incus/v7/shared/api" ) // Logger is an interface that must be implemented by all loggers. type Logger interface { HandleEvent(event api.Event) Start() error Stop() Validate() error } // common embeds shared configuration fields for all logger types. type common struct { lifecycleProjects []string lifecycleTypes []string loggingLevel string name string types []string } // newCommonLogger instantiates a new common logger. func newCommonLogger(name string, cfg *clusterConfig.Config) common { lifecycleProjects, lifecycleTypes, loggingLevel, types := cfg.LoggingCommonConfig(name) return common{ loggingLevel: loggingLevel, lifecycleProjects: sliceFromString(lifecycleProjects), lifecycleTypes: sliceFromString(lifecycleTypes), name: name, types: sliceFromString(types), } } // processEvent verifies whether the event should be processed for the specific logger. func (c *common) processEvent(event api.Event) bool { switch event.Type { case api.EventTypeLifecycle: if !contains(c.types, "lifecycle") { return false } lifecycleEvent := api.EventLifecycle{} err := json.Unmarshal(event.Metadata, &lifecycleEvent) if err != nil { return false } if lifecycleEvent.Project != "" && len(c.lifecycleProjects) > 0 { if !contains(c.lifecycleProjects, lifecycleEvent.Project) { return false } } if len(c.lifecycleTypes) > 0 && !hasAnyPrefix(c.lifecycleTypes, lifecycleEvent.Action) { return false } return true case api.EventTypeLogging, api.EventTypeNetworkACL: if !contains(c.types, "logging") && event.Type == api.EventTypeLogging { return false } if !contains(c.types, "network-acl") && event.Type == api.EventTypeNetworkACL { return false } logEvent := api.EventLogging{} err := json.Unmarshal(event.Metadata, &logEvent) if err != nil { return false } // The errors can be ignored as the values are validated elsewhere. l1, _ := logrus.ParseLevel(logEvent.Level) l2, _ := logrus.ParseLevel(c.loggingLevel) // Only consider log messages with a certain log level. if l2 < l1 { return false } return true default: return false } } incus-7.0.0/internal/server/logging/controller.go000066400000000000000000000056761517523235500221470ustar00rootroot00000000000000package logging import ( "fmt" "github.com/lxc/incus/v7/internal/server/events" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/logger" ) // Controller is responsible for managing a set of loggers. type Controller struct { listener *events.InternalListener loggers map[string]Logger } // NewLoggingController instantiates a new LoggerController object. func NewLoggingController(listener *events.InternalListener) *Controller { return &Controller{ listener: listener, loggers: map[string]Logger{}, } } // AddLogger adds a new logger to the controller. func (c *Controller) AddLogger(s *state.State, name string, loggerType string) error { loggerClient, err := LoggerFromType(s, name, loggerType) if err != nil { return err } c.loggers[name] = loggerClient c.listener.AddHandler(name, loggerClient.HandleEvent) return nil } // RemoveLogger removes a logger from the controller. func (c *Controller) RemoveLogger(name string) { loggerClient, ok := c.loggers[name] if ok { loggerClient.Stop() delete(c.loggers, name) c.listener.RemoveHandler(name) } } // Setup is responsible for preparing a new set of loggers. func (c *Controller) Setup(s *state.State) error { loggingConfig, err := s.GlobalConfig.Loggers() if err != nil { return err } for loggerName, loggerType := range loggingConfig { err = c.AddLogger(s, loggerName, loggerType) if err != nil { logger.Error("Error creating a logger", logger.Ctx{"err": err}) } } return nil } // Reconfigure handles the reinitialization of loggers after configuration changes. func (c *Controller) Reconfigure(s *state.State, config map[string]struct{}) error { loggingConfig, err := s.GlobalConfig.Loggers() if err != nil { return err } for loggerName := range config { c.RemoveLogger(loggerName) loggerType, ok := loggingConfig[loggerName] if !ok { continue } err = c.AddLogger(s, loggerName, loggerType) if err != nil { logger.Error("Error creating logger", logger.Ctx{"err": err}) } } return nil } // Shutdown cleans up loggers. func (c *Controller) Shutdown() { for _, loggerClient := range c.loggers { loggerClient.Stop() } } // LoggerFromType returns a new logger based on its type. func LoggerFromType(s *state.State, loggerName string, loggerType string) (Logger, error) { if loggerType == "" { return nil, fmt.Errorf("No type definition for logger %s", loggerName) } var loggerClient Logger var err error switch loggerType { case "syslog": loggerClient, err = NewSyslogLogger(s, loggerName) case "loki": loggerClient, err = NewLokiLogger(s, loggerName) case "webhook": loggerClient, err = NewWebhookLogger(s, loggerName) default: return nil, fmt.Errorf("%s is not supported logger type", loggerType) } if err != nil { return nil, err } err = loggerClient.Validate() if err != nil { return nil, err } err = loggerClient.Start() if err != nil { return nil, err } return loggerClient, nil } incus-7.0.0/internal/server/logging/driver_loki.go000066400000000000000000000231551517523235500222650ustar00rootroot00000000000000package logging import ( "bufio" "bytes" "context" "encoding/json" "fmt" "io" "maps" "net/http" "net/url" "os" "reflect" "slices" "sort" "strconv" "strings" "sync" "time" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" localtls "github.com/lxc/incus/v7/shared/tls" ) // This is a modified version of https://github.com/grafana/loki/blob/v1.6.1/pkg/promtail/client/. const ( contentType = "application/json" maxErrMsgLen = 1024 ) type config struct { batchSize int batchWait time.Duration caCert string username string password string labels []string instance string location string retry int timeout time.Duration url *url.URL } type entry struct { labels LabelSet Entry } // LokiLogger represents a Loki client. type LokiLogger struct { common cfg config client *http.Client ctx context.Context quit chan struct{} once sync.Once entries chan entry wg sync.WaitGroup } // NewLokiLogger returns a logger of loki type. func NewLokiLogger(s *state.State, name string) (*LokiLogger, error) { urlStr, username, password, caCert, instance, labels, retry := s.GlobalConfig.LoggingConfigForLoki(name) // Set defaults. if retry == 0 { retry = 3 } // Validate the URL. u, err := url.Parse(urlStr) if err != nil { return nil, err } // Handle standalone systems. var location string if !s.ServerClustered { hostname, err := os.Hostname() if err != nil { return nil, err } location = hostname if instance == "" { instance = hostname } } else if instance == "" { instance = s.ServerName } loggerClient := LokiLogger{ common: newCommonLogger(name, s.GlobalConfig), cfg: config{ batchSize: 10 * 1024, batchWait: 1 * time.Second, caCert: caCert, username: username, password: password, instance: instance, location: location, labels: sliceFromString(labels), retry: retry, timeout: 10 * time.Second, url: u, }, client: &http.Client{}, ctx: s.ShutdownCtx, entries: make(chan entry), quit: make(chan struct{}), } if caCert != "" { tlsConfig, err := localtls.GetTLSConfigMem("", "", caCert, "", false) if err != nil { return nil, err } loggerClient.client.Transport = &http.Transport{ TLSClientConfig: tlsConfig, } } else { loggerClient.client = http.DefaultClient } return &loggerClient, nil } func (l *LokiLogger) run() { batch := newBatch() minWaitCheckFrequency := 10 * time.Millisecond maxWaitCheckFrequency := max(l.cfg.batchWait/10, minWaitCheckFrequency) maxWaitCheck := time.NewTicker(maxWaitCheckFrequency) defer func() { // Send all pending batches l.sendBatch(batch) l.wg.Done() }() for { select { case <-l.ctx.Done(): return case <-l.quit: return case e := <-l.entries: // If adding the entry to the batch will increase the size over the max // size allowed, we do send the current batch and then create a new one if batch.sizeBytesAfter(e) > l.cfg.batchSize { l.sendBatch(batch) batch = newBatch(e) break } // The max size of the batch isn't reached, so we can add the entry batch.add(e) case <-maxWaitCheck.C: // Send batch if max wait time has been reached if batch.age() < l.cfg.batchWait { break } l.sendBatch(batch) batch = newBatch() } } } func (l *LokiLogger) sendBatch(batch *batch) { if batch.empty() { return } buf, _, err := batch.encode() if err != nil { return } var status int for range l.cfg.retry { select { case <-l.quit: return default: // Try to send the message. status, err = l.send(l.ctx, buf) if err == nil { return } // Only retry 429s, 500s and connection-level errors. if status > 0 && status != 429 && status/100 != 5 { return } // Retry every 10s. time.Sleep(10 * time.Second) } } } func (l *LokiLogger) send(ctx context.Context, buf []byte) (int, error) { ctx, cancel := context.WithTimeout(ctx, l.cfg.timeout) defer cancel() req, err := http.NewRequest("POST", fmt.Sprintf("%s/loki/api/v1/push", l.cfg.url.String()), bytes.NewReader(buf)) if err != nil { return -1, err } req = req.WithContext(ctx) req.Header.Set("Content-Type", contentType) if l.cfg.username != "" && l.cfg.password != "" { req.SetBasicAuth(l.cfg.username, l.cfg.password) } resp, err := l.client.Do(req) if err != nil { return -1, err } if resp.StatusCode/100 != 2 { scanner := bufio.NewScanner(io.LimitReader(resp.Body, maxErrMsgLen)) line := "" if scanner.Scan() { line = scanner.Text() } err = fmt.Errorf("server returned HTTP status %s (%d): %s", resp.Status, resp.StatusCode, line) } return resp.StatusCode, err } // Start starts the loki logger. func (l *LokiLogger) Start() error { l.wg.Add(1) go l.run() return nil } // Stop stops the client. func (l *LokiLogger) Stop() { l.once.Do(func() { close(l.quit) }) l.wg.Wait() } // Validate checks whether the logger configuration is correct. func (l *LokiLogger) Validate() error { if l.cfg.url.String() == "" { return fmt.Errorf("%s: URL cannot be empty", l.name) } return nil } // HandleEvent handles the event received from the internal event listener. func (l *LokiLogger) HandleEvent(event api.Event) { if !l.processEvent(event) { return } // Support overriding the location field (used on standalone systems). location := event.Location if l.cfg.location != "" { location = l.cfg.location } entry := entry{ labels: LabelSet{ "app": "incus", "type": event.Type, "location": location, "instance": l.cfg.instance, }, Entry: Entry{ Timestamp: event.Timestamp, }, } ctx := make(map[string]string) switch event.Type { case api.EventTypeLifecycle: lifecycleEvent := api.EventLifecycle{} err := json.Unmarshal(event.Metadata, &lifecycleEvent) if err != nil { return } if lifecycleEvent.Name != "" { entry.labels["name"] = lifecycleEvent.Name } if lifecycleEvent.Project != "" { entry.labels["project"] = lifecycleEvent.Project } // Build map. These key-value pairs will either be added as labels, or be part of the // log message itself. ctx["action"] = lifecycleEvent.Action ctx["source"] = lifecycleEvent.Source maps.Copy(ctx, buildNestedContext("context", lifecycleEvent.Context)) if lifecycleEvent.Requestor != nil { ctx["requester-address"] = lifecycleEvent.Requestor.Address ctx["requester-protocol"] = lifecycleEvent.Requestor.Protocol ctx["requester-username"] = lifecycleEvent.Requestor.Username } // Get a sorted list of context keys. keys := make([]string, 0, len(ctx)) for k := range ctx { keys = append(keys, k) } sort.Strings(keys) // Add key-value pairs as labels but don't override any labels. for _, k := range keys { v := ctx[k] if slices.Contains(l.cfg.labels, k) { _, ok := entry.labels[k] if !ok { // Label names may not contain any hyphens. entry.labels[strings.ReplaceAll(k, "-", "_")] = v delete(ctx, k) } } } messagePrefix := "" // Add the remaining context as the message prefix. for k, v := range ctx { messagePrefix += fmt.Sprintf("%s=\"%s\" ", k, v) } entry.Line = fmt.Sprintf("%s%s", messagePrefix, lifecycleEvent.Action) case api.EventTypeLogging, api.EventTypeNetworkACL: logEvent := api.EventLogging{} err := json.Unmarshal(event.Metadata, &logEvent) if err != nil { return } tmpContext := map[string]any{} // Convert map[string]string to map[string]any as buildNestedContext takes the latter type. for k, v := range logEvent.Context { tmpContext[k] = v } // Build map. These key-value pairs will either be added as labels, or be part of the // log message itself. ctx["level"] = logEvent.Level maps.Copy(ctx, buildNestedContext("context", tmpContext)) // Add key-value pairs as labels but don't override any labels. for k, v := range ctx { if slices.Contains(l.cfg.labels, k) { _, ok := entry.labels[k] if !ok { entry.labels[k] = v delete(ctx, k) } } } keys := make([]string, 0, len(ctx)) for k := range ctx { keys = append(keys, k) } sort.Strings(keys) var message strings.Builder // Add the remaining context as the message prefix. The keys are sorted alphabetically. for _, k := range keys { message.WriteString(fmt.Sprintf("%s=%q ", k, ctx[k])) } message.WriteString(logEvent.Message) entry.Line = message.String() } l.entries <- entry } func buildNestedContext(prefix string, m map[string]any) map[string]string { labels := map[string]string{} for k, v := range m { t := reflect.TypeOf(v) if t != nil && t.Kind() == reflect.Map { for k, v := range buildNestedContext(k, v.(map[string]any)) { if prefix == "" { labels[k] = v } else { labels[fmt.Sprintf("%s-%s", prefix, k)] = v } } } else { if prefix == "" { labels[k] = fmt.Sprintf("%v", v) } else { labels[fmt.Sprintf("%s-%s", prefix, k)] = fmt.Sprintf("%v", v) } } } return labels } // MarshalJSON returns the JSON encoding of Entry. func (e Entry) MarshalJSON() ([]byte, error) { return fmt.Appendf(nil, "[\"%d\", %s]", e.Timestamp.UnixNano(), strconv.Quote(e.Line)), nil } // String implements the Stringer interface. It returns a formatted/sorted set of label key/value pairs. func (l LabelSet) String() string { var b strings.Builder keys := make([]string, 0, len(l)) for k := range l { keys = append(keys, k) } sort.Strings(keys) b.WriteByte('{') for i, k := range keys { if i > 0 { b.WriteByte(',') b.WriteByte(' ') } b.WriteString(k) b.WriteByte('=') b.WriteString(strconv.Quote(l[k])) } b.WriteByte('}') return b.String() } incus-7.0.0/internal/server/logging/driver_loki_batch.go000066400000000000000000000040531517523235500234220ustar00rootroot00000000000000package logging import ( "encoding/json" "time" ) // batch holds pending log streams waiting to be sent to Loki, and it's used // to reduce the number of push requests to Loki aggregating multiple log streams // and entries in a single batch request. type batch struct { streams map[string]*Stream bytes int createdAt time.Time } func newBatch(entries ...entry) *batch { b := &batch{ streams: map[string]*Stream{}, bytes: 0, createdAt: time.Now(), } // Add entries to the batch for _, entry := range entries { b.add(entry) } return b } // add an entry to the batch. func (b *batch) add(entry entry) { b.bytes += len(entry.Line) // Append the entry to an already existing stream (if any) labels := entry.labels.String() stream, ok := b.streams[labels] if ok { stream.Entries = append(stream.Entries, entry.Entry) return } // Add the entry as a new stream b.streams[labels] = &Stream{ Labels: entry.labels, Entries: []Entry{entry.Entry}, } } // sizeBytesAfter returns the size of the batch after the input entry // will be added to the batch itself. func (b *batch) sizeBytesAfter(entry entry) int { return b.bytes + len(entry.Line) } // age of the batch since its creation. func (b *batch) age() time.Duration { return time.Since(b.createdAt) } // encode the batch as push request, and returns the encoded bytes and the number of encoded // entries. func (b *batch) encode() ([]byte, int, error) { req, entriesCount := b.createPushRequest() buf, err := json.Marshal(req) if err != nil { return nil, 0, err } return buf, entriesCount, nil } // creates push request and returns it, together with number of entries. func (b *batch) createPushRequest() (*PushRequest, int) { req := PushRequest{ Streams: make([]*Stream, 0, len(b.streams)), } entriesCount := 0 for _, stream := range b.streams { req.Streams = append(req.Streams, stream) entriesCount += len(stream.Entries) } return &req, entriesCount } // empty returns true if streams is empty. func (b *batch) empty() bool { return len(b.streams) == 0 } incus-7.0.0/internal/server/logging/driver_loki_types.go000066400000000000000000000010521517523235500235010ustar00rootroot00000000000000package logging import ( "time" ) // PushRequest models a log stream push. type PushRequest struct { Streams []*Stream `json:"streams"` } // LabelSet is a key/value pair mapping of labels. type LabelSet map[string]string // Stream represents a log stream. It includes a set of log entries and their labels. type Stream struct { Labels LabelSet `json:"stream"` Entries []Entry `json:"values"` } // Entry represents a log entry. It includes a log message and the time it occurred at. type Entry struct { Timestamp time.Time Line string } incus-7.0.0/internal/server/logging/driver_syslog.go000066400000000000000000000063421517523235500226460ustar00rootroot00000000000000package logging import ( "encoding/json" "fmt" "log/syslog" "strings" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" ) var facilityMap = map[string]syslog.Priority{ "kern": syslog.LOG_KERN, "user": syslog.LOG_USER, "mail": syslog.LOG_MAIL, "daemon": syslog.LOG_DAEMON, "auth": syslog.LOG_AUTH, "syslog": syslog.LOG_SYSLOG, "lpr": syslog.LOG_LPR, "news": syslog.LOG_NEWS, "uucp": syslog.LOG_UUCP, "cron": syslog.LOG_CRON, "authpriv": syslog.LOG_AUTHPRIV, "ftp": syslog.LOG_FTP, } // SyslogLogger represents a syslog logger. type SyslogLogger struct { common address string facility syslog.Priority network string tag string writer *syslog.Writer } // NewSyslogLogger instantiates a new syslog logger. func NewSyslogLogger(s *state.State, name string) (*SyslogLogger, error) { addr, facility := s.GlobalConfig.LoggingConfigForSyslog(name) network, address := parseAddress(addr) return &SyslogLogger{ common: newCommonLogger(name, s.GlobalConfig), address: address, facility: parseFacility(facility), network: network, tag: "incus", }, nil } func (c *SyslogLogger) write(event api.Event) error { msg := fmt.Sprintf("type: %s log: %s", event.Type, string(event.Metadata)) lvl := "info" if event.Type == api.EventTypeLogging { logEvent := api.EventLogging{} err := json.Unmarshal(event.Metadata, &logEvent) if err != nil { return err } lvl = logEvent.Level } switch strings.ToLower(lvl) { case "panic": return c.writer.Err(msg) case "fatal": return c.writer.Err(msg) case "error": return c.writer.Err(msg) case "warn", "warning": return c.writer.Warning(msg) case "trace": return c.writer.Warning(msg) case "info": return c.writer.Info(msg) case "debug": return c.writer.Debug(msg) } return nil } // HandleEvent handles the event received from the internal event listener. func (c *SyslogLogger) HandleEvent(event api.Event) { if !c.processEvent(event) { return } _ = c.write(event) } // Start starts the syslog logger. func (c *SyslogLogger) Start() error { writer, err := syslog.Dial(c.network, c.address, c.facility, c.tag) if err != nil { return err } c.writer = writer return nil } // Stop cleans up the syslog logger. func (c *SyslogLogger) Stop() { if c.writer != nil { _ = c.writer.Close() } } // Validate checks whether the logger configuration is correct. func (c *SyslogLogger) Validate() error { if c.address == "" { return fmt.Errorf("%s: Address cannot be empty", c.name) } return nil } // parseAddress parses a syslog address into two parts: protocol and the address itself. func parseAddress(address string) (string, string) { protocol := "udp" port := "514" if strings.Contains(address, "://") { parts := strings.SplitN(address, "://", 2) protocol = parts[0] address = parts[1] } if !strings.Contains(address, ":") { address = fmt.Sprintf("%s:%s", address, port) } return protocol, address } // parseFacility parses a string into a syslog.Priority. func parseFacility(facility string) syslog.Priority { facility = strings.ToLower(strings.TrimSpace(facility)) val, ok := facilityMap[facility] if ok { return val } return syslog.LOG_DAEMON } incus-7.0.0/internal/server/logging/driver_webhook.go000066400000000000000000000051441517523235500227630ustar00rootroot00000000000000package logging import ( "bytes" "context" "crypto/tls" "crypto/x509" "encoding/json" "encoding/pem" "errors" "fmt" "net/http" "time" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" incustls "github.com/lxc/incus/v7/shared/tls" ) // WebhookLogger represents a webhook logger. type WebhookLogger struct { common client *http.Client address string username string password string retry int } // NewWebhookLogger instantiates a new webhook logger. func NewWebhookLogger(s *state.State, name string) (*WebhookLogger, error) { address, username, password, caCertificate, retry := s.GlobalConfig.LoggingConfigForWebhook(name) client := &http.Client{} // Set defaults. if retry == 0 { retry = 3 } // Setup the server for self-signed certirficates. if caCertificate != "" { // Prepare the TLS config. tlsConfig := &tls.Config{ MinVersion: tls.VersionTLS13, } // Parse the provided certificate. certBlock, _ := pem.Decode([]byte(caCertificate)) if certBlock == nil { return nil, errors.New("Invalid remote certificate") } serverCert, err := x509.ParseCertificate(certBlock.Bytes) if err != nil { return nil, fmt.Errorf("Invalid remote certificate: %w", err) } // Add the certificate to the TLS config. incustls.TLSConfigWithTrustedCert(tlsConfig, serverCert) // Configure the HTTP client with our TLS config. client.Transport = &http.Transport{TLSClientConfig: tlsConfig} } return &WebhookLogger{ common: newCommonLogger(name, s.GlobalConfig), client: client, address: address, username: username, password: password, retry: retry, }, nil } // HandleEvent handles the event received from the internal event listener. func (c *WebhookLogger) HandleEvent(event api.Event) { // JSON data. data, err := json.Marshal(event) if err != nil { return } // Prepare the request. req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, c.address, bytes.NewReader(data)) if err != nil { return } req.Header.Set("Content-Type", "application/json") for range c.retry { resp, err := c.client.Do(req) if err != nil { // Wait 10s and try again. time.Sleep(10 * time.Second) continue } _ = resp.Body.Close() } } // Start starts the webhook logger. func (c *WebhookLogger) Start() error { return nil } // Stop cleans up the webhook logger. func (c *WebhookLogger) Stop() { } // Validate checks whether the logger configuration is correct. func (c *WebhookLogger) Validate() error { if c.address == "" { return fmt.Errorf("%s: Address cannot be empty", c.name) } return nil } incus-7.0.0/internal/server/logging/util.go000066400000000000000000000014731517523235500207300ustar00rootroot00000000000000package logging import ( "slices" "strings" ) // sliceFromString converts a comma-separated string into a slice of strings. func sliceFromString(input string) []string { parts := strings.Split(input, ",") result := []string{} for _, v := range parts { part := strings.TrimSpace(v) if part != "" { result = append(result, part) } } return result } // contains checks if the target string is part of the slice. func contains(slice []string, target string) bool { return slices.Contains(slice, target) } // hasAnyPrefix checks if any string in the prefixes slice is a prefix of s. // It returns true as soon as a match is found, otherwise false. func hasAnyPrefix(prefixes []string, s string) bool { for _, prefix := range prefixes { if strings.HasPrefix(s, prefix) { return true } } return false } incus-7.0.0/internal/server/metadata/000077500000000000000000000000001517523235500175515ustar00rootroot00000000000000incus-7.0.0/internal/server/metadata/configuration.json000066400000000000000000007104531517523235500233250ustar00rootroot00000000000000{ "configs": { "cluster": { "cluster": { "keys": [ { "scheduler.instance": { "defaultdesc": "`all`", "longdesc": "Possible values are `all`, `manual`, and `group`. See\n{ref}`clustering-instance-placement` for more information.", "shortdesc": "Controls how instances are scheduled to run on this member", "type": "string" } }, { "user.*": { "longdesc": "User keys can be used in search.", "shortdesc": "Free form user key/value storage", "type": "string" } } ] } }, "cluster_group": { "common": { "keys": [ { "instances.vm.cpu.ARCHITECTURE.baseline": { "longdesc": "The CPU base architecture name as can be found through `qemu -cpu ?`.\n\nThis can be a generic definition like `qemu64` or `kvm64`, or it can be a specific hardware architecture like `EPYC-v2`.\nIt's important to ensure that all servers in the group match that baseline.", "shortdesc": "CPU base architecture name", "type": "string" } }, { "instances.vm.cpu.ARCHITECTURE.flags": { "longdesc": "A comma separated list of CPU flags to add on top of CPU baseline or a list of flags to remove from it.\n\nTo remove a flag, use `-flag`.", "shortdesc": "CPU flags to add/remove to/from the baseline", "type": "string" } }, { "user.*": { "longdesc": "User keys can be used in search.", "shortdesc": "Free form user key/value storage", "type": "string" } } ] } }, "devices": { "disk": { "keys": [ { "attached": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Only for VMs: Whether the disk is attached or ejected", "type": "bool" } }, { "boot.priority": { "longdesc": "", "required": "no", "shortdesc": "Boot priority for VMs (higher value boots first)", "type": "integer" } }, { "ceph.cluster_name": { "default": "`ceph`", "longdesc": "", "required": "no", "shortdesc": "The cluster name of the Ceph cluster (required for Ceph or CephFS sources)", "type": "string" } }, { "ceph.user_name": { "default": "`admin`", "longdesc": "", "required": "no", "shortdesc": "The user name of the Ceph cluster (required for Ceph or CephFS sources)", "type": "string" } }, { "dependent": { "default": "`false`", "longdesc": "", "required": "no", "shortdesc": "Specifies if the disk is instance dependent", "type": "bool" } }, { "initial.*": { "longdesc": "For root disk devices, this is used to override the storage pool's default volume\nconfiguration when creating the instance's root volume.\n\nFor custom volumes, only `initial.uid`, `initial.gid` and `initial.mode` are\naccepted and they are used when auto-creating sub-directories inside the custom\nvolume (when the `source` includes a sub-path that doesn't exist).\n\n`initial.uid`, `initial.gid` and `initial.mode` are also used to set the ownership\nand mode of the file system when the `source` is `tmpfs:` or `tmpfs-overlay:`.", "required": "no", "shortdesc": "Initial volume configuration for instance root disk devices", "type": "string" } }, { "io.bus": { "default": "`virtio-scsi` for block, `auto` for file system", "longdesc": "This controls what bus a disk device should be attached to.\n\nFor block devices (disks), this is one of:\n- `nvme`\n- `virtio-blk`\n- `virtio-scsi` (default)\n- `usb`\n\nFor file systems (shared directories or custom volumes), this is one of:\n- `9p`\n- `auto` (default) (`virtiofs` if possible, else `9p`)\n- `virtiofs`\n\n`9p` doesn't support hotplugging and `virtiofs` doesn't support live migration. `auto` tries\nto use `virtiofs` if possible (`migration.stateful` not set to `true` and host support for\n`virtiofsd`) and falls back to `9p` otherwise.", "required": "no", "shortdesc": "Only for VMs: Override the bus for the device", "type": "string" } }, { "io.cache": { "default": "`none`", "longdesc": "This controls what bus a disk device should be attached to.\n\nFor block devices (disks), this is one of:\n- `none` (default)\n- `writeback`\n- `unsafe`\n\nFor file systems (shared directories or custom volumes), this is one of:\n- `none` (default)\n- `metadata`\n- `unsafe`", "required": "no", "shortdesc": "Only for VMs: Override the caching mode for the device", "type": "string" } }, { "limits.max": { "longdesc": "", "required": "no", "shortdesc": "I/O limit in byte/s or IOPS for both read and write (same as setting both `limits.read` and `limits.write`)", "type": "string" } }, { "limits.read": { "longdesc": "", "required": "no", "shortdesc": "I/O limit in byte/s (various suffixes supported, see {ref}`instances-limit-units`) or in IOPS (must be suffixed with `iops`) - see also {ref}`storage-configure-IO`", "type": "string" } }, { "limits.write": { "longdesc": "", "required": "no", "shortdesc": "I/O limit in byte/s (various suffixes supported, see {ref}`instances-limit-units`) or in IOPS (must be suffixed with `iops`) - see also {ref}`storage-configure-IO`", "type": "string" } }, { "path": { "longdesc": "This controls which path inside the instance the disk should be mounted on.\n\nWith containers, this option supports mounting file system disk devices, and paths and single files within them.\n\nWith VMs, this option supports mounting file system disk devices and paths within them. Mounting single files is not supported.", "required": "yes", "shortdesc": "Path inside the instance where the disk will be mounted (only for file system disk devices)", "type": "string" } }, { "pool": { "longdesc": "", "required": "no", "shortdesc": "The storage pool to which the disk device belongs (only applicable for storage volumes managed by Incus)", "type": "string" } }, { "propagation": { "longdesc": "", "required": "no", "shortdesc": "Controls how a bind-mount is shared between the instance and the host (can be one of `private`, the default, or `shared`, `slave`, `unbindable`, `rshared`, `rslave`, `runbindable`, `rprivate`; see the Linux Kernel [shared subtree](https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt) documentation for a full explanation)", "type": "string" } }, { "raw.mount.options": { "longdesc": "", "required": "no", "shortdesc": "File system specific mount options", "type": "string" } }, { "readonly": { "default": "`false`", "longdesc": "", "required": "no", "shortdesc": "Controls whether to make the mount read-only", "type": "bool" } }, { "recursive": { "default": "`false`", "longdesc": "", "required": "no", "shortdesc": "Controls whether to recursively mount the source path", "type": "bool" } }, { "required": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Controls whether to fail if the source doesn't exist", "type": "bool" } }, { "shift": { "default": "`false`", "longdesc": "", "required": "no", "shortdesc": "Sets up a shifting overlay to translate the source UID/GID to match the instance (only for containers)", "type": "bool" } }, { "size": { "longdesc": "", "required": "no", "shortdesc": "Disk size in bytes (various suffixes supported, see {ref}`instances-limit-units`) - only supported for the `rootfs` (`/`)", "type": "string" } }, { "size.state": { "longdesc": "", "required": "no", "shortdesc": "Same as `size`, but applies to the file-system volume used for saving runtime state in VMs", "type": "string" } }, { "source": { "longdesc": "", "required": "yes", "shortdesc": "Source of a file system or block device (see {ref}`devices-disk-types` for details)", "type": "string" } }, { "wwn": { "default": "``", "longdesc": "", "required": "no", "shortdesc": "Only for VMs: Set the disk World Wide Name (only supported on `virtio-scsi` bus)", "type": "bool" } } ] }, "gpu_mdev": { "keys": [ { "id": { "longdesc": "", "required": "no", "shortdesc": "The DRM card ID of the GPU device", "type": "string" } }, { "mdev": { "longdesc": "", "required": "yes", "shortdesc": "The mediated device profile to use (required - for example, `i915-GVTg_V5_4`)", "type": "string" } }, { "productid": { "longdesc": "", "required": "no", "shortdesc": "The product ID of the GPU device", "type": "string" } }, { "vendorid": { "longdesc": "", "required": "no", "shortdesc": "The vendor ID of the GPU device", "type": "string" } } ] }, "gpu_mig": { "keys": [ { "id": { "longdesc": "", "required": "no", "shortdesc": "The DRM card ID of the GPU device", "type": "string" } }, { "mig.ci": { "longdesc": "", "required": "no", "shortdesc": "Existing MIG compute instance ID", "type": "int" } }, { "mig.gi": { "longdesc": "", "required": "no", "shortdesc": "Existing MIG GPU instance ID", "type": "int" } }, { "mig.uuid": { "longdesc": "", "required": "no", "shortdesc": "Existing MIG device UUID (MIG- prefix can be omitted)", "type": "string" } }, { "pci": { "longdesc": "", "required": "no", "shortdesc": "The PCI address of the GPU device", "type": "string" } }, { "productid": { "longdesc": "", "required": "no", "shortdesc": "The product ID of the GPU device", "type": "string" } }, { "vendorid": { "longdesc": "", "required": "no", "shortdesc": "The vendor ID of the GPU device", "type": "string" } } ] }, "gpu_physical": { "keys": [ { "gid": { "default": "0", "longdesc": "", "required": "no", "shortdesc": "GID of the device owner in the instance (container only)", "type": "int" } }, { "id": { "longdesc": "", "required": "no", "shortdesc": "The DRM card ID of the GPU device", "type": "string" } }, { "mode": { "default": "0660", "longdesc": "", "required": "no", "shortdesc": "Mode of the device in the instance (container only)", "type": "int" } }, { "pci": { "longdesc": "", "required": "no", "shortdesc": "The PCI address of the GPU device", "type": "string" } }, { "productid": { "longdesc": "", "required": "no", "shortdesc": "The product ID of the GPU device", "type": "string" } }, { "uid": { "default": "0", "longdesc": "", "required": "no", "shortdesc": "UID of the device owner in the instance (container only)", "type": "int" } }, { "vendorid": { "longdesc": "", "required": "no", "shortdesc": "The vendor ID of the GPU device", "type": "string" } } ] }, "gpu_sriov": { "keys": [ { "id": { "longdesc": "", "required": "no", "shortdesc": "The DRM card ID of the parent GPU device", "type": "string" } }, { "pci": { "longdesc": "", "required": "no", "shortdesc": "The PCI address of the parent GPU device", "type": "string" } }, { "productid": { "longdesc": "", "required": "no", "shortdesc": "The product ID of the parent GPU device", "type": "string" } }, { "vendorid": { "longdesc": "", "required": "no", "shortdesc": "The vendor ID of the parent GPU device", "type": "string" } } ] }, "infiniband": { "keys": [ { "hwaddr": { "defaultdesc": "randomly assigned", "longdesc": "", "required": "no", "shortdesc": "The MAC address of the new interface (can be either the full 20-byte variant or the short 8-byte variant, which will only modify the last 8 bytes of the parent device)", "type": "string" } }, { "mtu": { "defaultdesc": "parent MTU", "longdesc": "", "required": "no", "shortdesc": "The MTU of the new interface", "type": "integer" } }, { "name": { "defaultdesc": "kernel assigned", "longdesc": "", "required": "no", "shortdesc": "The name of the interface inside the instance", "type": "string" } }, { "nictype": { "longdesc": "", "required": "yes", "shortdesc": "The device type (one of `physical` or `sriov`)", "type": "string" } }, { "parent": { "defaultdesc": "kernel assigned", "longdesc": "", "required": "no", "shortdesc": "The name of the interface inside the instance", "type": "string" } } ] }, "nic_bridged": { "keys": [ { "attached": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether the NIC is plugged in or not", "type": "bool" } }, { "boot.priority": { "longdesc": "", "managed": "no", "shortdesc": "Boot priority for VMs (higher value boots first)", "type": "integer" } }, { "connected": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether the NIC is connected to the host network", "type": "bool" } }, { "host_name": { "default": "randomly assigned", "longdesc": "", "managed": "no", "shortdesc": "The name of the interface on the host", "type": "string" } }, { "hwaddr": { "default": "randomly assigned", "longdesc": "", "managed": "no", "shortdesc": "The MAC address of the new interface", "type": "string" } }, { "io.bus": { "default": "`virtio`", "longdesc": "", "managed": "no", "shortdesc": "Override the bus for the device (can be `virtio` or `usb`) (VM only)", "type": "string" } }, { "ipv4.address": { "longdesc": "", "managed": "no", "shortdesc": "An IPv4 address to assign to the instance through DHCP (can be `none` to restrict all IPv4 traffic when `security.ipv4_filtering` is set)", "type": "string" } }, { "ipv4.routes": { "longdesc": "", "managed": "no", "shortdesc": "Comma-delimited list of IPv4 static routes to add on host to NIC", "type": "string" } }, { "ipv4.routes.external": { "longdesc": "", "managed": "no", "shortdesc": "Comma-delimited list of IPv4 static routes to route to the NIC and publish on uplink network (BGP)", "type": "string" } }, { "ipv6.address": { "longdesc": "", "managed": "no", "shortdesc": "An IPv6 address to assign to the instance through DHCP (can be `none` to restrict all IPv6 traffic when `security.ipv6_filtering` is set)", "type": "string" } }, { "ipv6.routes": { "longdesc": "", "managed": "no", "shortdesc": "Comma-delimited list of IPv6 static routes to add on host to NIC", "type": "string" } }, { "ipv6.routes.external": { "longdesc": "", "managed": "no", "shortdesc": "Comma-delimited list of IPv6 static routes to route to the NIC and publish on uplink network (BGP)", "type": "string" } }, { "limits.egress": { "longdesc": "", "managed": "no", "shortdesc": "I/O limit in bit/s for outgoing traffic (various suffixes supported, see {ref}`instances-limit-units`)", "type": "string" } }, { "limits.ingress": { "longdesc": "", "managed": "no", "shortdesc": "I/O limit in bit/s for incoming traffic (various suffixes supported, see {ref}`instances-limit-units`)", "type": "string" } }, { "limits.max": { "longdesc": "", "managed": "no", "shortdesc": "I/O limit in bit/s for both incoming and outgoing traffic (same as setting both limits.ingress and limits.egress)", "type": "string" } }, { "limits.priority": { "longdesc": "", "managed": "no", "shortdesc": "The priority for outgoing traffic, to be used by the kernel queuing discipline to prioritize network packets", "type": "integer" } }, { "mtu": { "default": "MTU of the parent device", "longdesc": "", "managed": "yes", "shortdesc": "The Maximum Transmit Unit (MTU) of the new interface", "type": "integer" } }, { "name": { "default": "kernel assigned", "longdesc": "", "managed": "no", "shortdesc": "The name of the interface inside the instance", "type": "string" } }, { "network": { "longdesc": "", "managed": "no", "shortdesc": "The managed network to link the device to (instead of specifying the `nictype` directly)", "type": "string" } }, { "parent": { "longdesc": "", "managed": "yes", "shortdesc": "The name of the parent host device (required if specifying the `nictype` directly)", "type": "string" } }, { "queue.tx.length": { "longdesc": "", "managed": "no", "shortdesc": "The transmit queue length for the NIC", "type": "integer" } }, { "security.acls": { "longdesc": "", "managed": "no", "shortdesc": "Comma-separated list of network ACLs to apply", "type": "string" } }, { "security.acls.default.egress.action": { "default": "drop", "longdesc": "", "managed": "no", "shortdesc": "Action to use for egress traffic that doesn't match any ACL rule", "type": "string" } }, { "security.acls.default.egress.logged": { "default": "false", "longdesc": "", "managed": "no", "shortdesc": "Whether to log egress traffic that doesn't match any ACL rule", "type": "bool" } }, { "security.acls.default.ingress.action": { "default": "drop", "longdesc": "", "managed": "no", "shortdesc": "Action to use for ingress traffic that doesn't match any ACL rule", "type": "string" } }, { "security.acls.default.ingress.logged": { "default": "false", "longdesc": "", "managed": "no", "shortdesc": "Whether to log ingress traffic that doesn't match any ACL rule", "type": "bool" } }, { "security.ipv4_filtering": { "default": "false", "longdesc": "", "managed": "no", "shortdesc": "Prevent the instance from spoofing another instance's IPv4 address (enables `security.mac_filtering`)", "type": "bool" } }, { "security.ipv6_filtering": { "default": "false", "longdesc": "", "managed": "no", "shortdesc": "Prevent the instance from spoofing another instance's IPv6 address (enables `security.mac_filtering`)", "type": "bool" } }, { "security.mac_filtering": { "default": "false", "longdesc": "", "managed": "no", "shortdesc": "Prevent the instance from spoofing another instance's MAC address", "type": "bool" } }, { "security.port_isolation": { "default": "false", "longdesc": "", "managed": "no", "shortdesc": "Prevent the NIC from communicating with other NICs in the network that have port isolation enabled", "type": "bool" } }, { "vlan": { "longdesc": "", "managed": "no", "shortdesc": "The VLAN ID to use for non-tagged traffic (can be none to remove port from default VLAN)", "type": "integer" } }, { "vlan.tagged": { "longdesc": "", "managed": "no", "shortdesc": "Comma-delimited list of VLAN IDs or VLAN ranges to join for tagged traffic", "type": "integer" } } ] }, "nic_ipvlan": { "keys": [ { "attached": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether the NIC is plugged in or not", "type": "bool" } }, { "gvrp": { "default": "false", "longdesc": "", "shortdesc": "Register VLAN using GARP VLAN Registration Protocol", "type": "bool" } }, { "hwaddr": { "default": "randomly assigned", "longdesc": "", "shortdesc": "The MAC address of the new interface", "type": "string" } }, { "ipv4.address": { "longdesc": "", "shortdesc": "Comma-delimited list of IPv4 static addresses to add to the instance (in l2 mode, these can be specified as CIDR values or singular addresses using a subnet of /24)", "type": "string" } }, { "ipv4.gateway": { "default": "`auto` (in `l3s` mode), `-` (in `l2` mode)", "longdesc": "", "shortdesc": "In `l3s` mode, whether to add an automatic default IPv4 gateway (can be `auto` or `none`). In `l2` mode, the IPv4 address of the gateway", "type": "string" } }, { "ipv4.host_table": { "longdesc": "", "shortdesc": "The custom policy routing table ID to add IPv4 static routes to (in addition to the main routing table)", "type": "integer" } }, { "ipv6.address": { "longdesc": "", "shortdesc": "Comma-delimited list of IPv6 static addresses to add to the instance (in `l2` mode, these can be specified as CIDR values or singular addresses using a subnet of /64)", "type": "string" } }, { "ipv6.gateway": { "default": "`auto` (in `l3s` mode), `-` (in `l2` mode)", "longdesc": "", "shortdesc": "In `l3s` mode, whether to add an automatic default IPv6 gateway (can be `auto` or `none`). In `l2` mode, the IPv6 address of the gateway", "type": "string" } }, { "ipv6.host_table": { "longdesc": "", "shortdesc": "The custom policy routing table ID to add IPv6 static routes to (in addition to the main routing table)", "type": "integer" } }, { "mode": { "default": "`l3s`", "longdesc": "", "shortdesc": "The IPVLAN mode (either `l2` or `l3s`)", "type": "string" } }, { "mtu": { "default": "MTU of the parent device", "longdesc": "", "shortdesc": "The Maximum Transmit Unit (MTU) of the new interface", "type": "integer" } }, { "name": { "default": "kernel assigned", "longdesc": "", "shortdesc": "The name of the interface inside the instance", "type": "string" } }, { "parent": { "longdesc": "", "shortdesc": "The name of the host device (required)", "type": "string" } }, { "vlan": { "longdesc": "", "shortdesc": "The VLAN ID to attach to", "type": "integer" } } ] }, "nic_macvlan": { "keys": [ { "attached": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether the NIC is plugged in or not", "type": "bool" } }, { "boot.priority": { "longdesc": "", "managed": "no", "shortdesc": "Boot priority for VMs (higher value boots first)", "type": "integer" } }, { "connected": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether the NIC is connected to the host network (VM only)", "type": "bool" } }, { "gvrp": { "default": "false", "longdesc": "", "managed": "no", "shortdesc": "Register VLAN using GARP VLAN Registration Protocol", "type": "bool" } }, { "hwaddr": { "default": "randomly assigned", "longdesc": "", "managed": "no", "shortdesc": "The MAC address of the new interface", "type": "string" } }, { "io.bus": { "default": "`virtio`", "longdesc": "", "managed": "no", "shortdesc": "Override the bus for the device (can be `virtio` or `usb`) (VM only)", "type": "string" } }, { "mode": { "default": "bridge", "longdesc": "", "managed": "no", "shortdesc": "Macvlan mode (one of `bridge`, `vepa`, `passthru` or `private`)", "type": "string" } }, { "mtu": { "default": "MTU of the parent device", "longdesc": "", "managed": "yes", "shortdesc": "The Maximum Transmit Unit (MTU) of the new interface", "type": "integer" } }, { "name": { "default": "kernel assigned", "longdesc": "", "managed": "no", "shortdesc": "The name of the interface inside the instance", "type": "string" } }, { "network": { "longdesc": "", "managed": "no", "shortdesc": "The managed network to link the device to (instead of specifying the `nictype` directly)", "type": "string" } }, { "parent": { "longdesc": "", "managed": "yes", "shortdesc": "The name of the parent host device (required if specifying the `nictype` directly)", "type": "string" } }, { "vlan": { "longdesc": "", "managed": "no", "shortdesc": "The VLAN ID to attach to", "type": "integer" } } ] }, "nic_ovn": { "keys": [ { "acceleration": { "default": "none", "longdesc": "", "managed": "no", "shortdesc": "Enable hardware offloading (either `none`, `sriov` or `vdpa`)", "type": "string" } }, { "attached": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether the NIC is plugged in or not", "type": "bool" } }, { "boot.priority": { "longdesc": "", "managed": "no", "shortdesc": "Boot priority for VMs (higher value boots first)", "type": "integer" } }, { "connected": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether the NIC is connected to the host network (requires `acceleration` set to `none`)", "type": "bool" } }, { "host_name": { "default": "randomly assigned", "longdesc": "", "managed": "no", "shortdesc": "The name of the interface inside the host", "type": "string" } }, { "hwaddr": { "default": "randomly assigned", "longdesc": "", "managed": "no", "shortdesc": "The MAC address of the new interface", "type": "string" } }, { "io.bus": { "default": "`virtio`", "longdesc": "", "managed": "no", "shortdesc": "Override the bus for the device (can be `virtio` or `usb`, requires `acceleration` set to `none`) (VM only)", "type": "string" } }, { "ipv4.address": { "longdesc": "", "managed": "no", "shortdesc": "An IPv4 address to assign to the instance through DHCP, `none` can be used to disable IP allocation", "type": "string" } }, { "ipv4.address.external": { "longdesc": "", "managed": "no", "shortdesc": "Select a specific external address (typically from a network forward)", "type": "string" } }, { "ipv4.routes": { "longdesc": "", "managed": "no", "shortdesc": "Comma-delimited list of IPv4 static routes to route to the NIC", "type": "string" } }, { "ipv4.routes.external": { "longdesc": "", "managed": "no", "shortdesc": "Comma-delimited list of IPv4 static routes to route to the NIC and publish on uplink network", "type": "string" } }, { "ipv6.address": { "longdesc": "", "managed": "no", "shortdesc": "An IPv6 address to assign to the instance through DHCP, `none` can be used to disable IP allocation", "type": "string" } }, { "ipv6.address.external": { "longdesc": "", "managed": "no", "shortdesc": "Select a specific external address (typically from a network forward)", "type": "string" } }, { "ipv6.routes": { "longdesc": "", "managed": "no", "shortdesc": "Comma-delimited list of IPv6 static routes to route to the NIC", "type": "string" } }, { "ipv6.routes.external": { "longdesc": "", "managed": "no", "shortdesc": "Comma-delimited list of IPv6 static routes to route to the NIC and publish on uplink network", "type": "string" } }, { "limits.egress": { "longdesc": "", "managed": "no", "shortdesc": "I/O limit in bit/s for outgoing traffic (various suffixes supported, see {ref}`instances-limit-units`)", "type": "string" } }, { "limits.ingress": { "longdesc": "", "managed": "no", "shortdesc": "I/O limit in bit/s for incoming traffic (various suffixes supported, see {ref}`instances-limit-units`)", "type": "string" } }, { "limits.max": { "longdesc": "", "managed": "no", "shortdesc": "I/O limit in bit/s for both incoming and outgoing traffic. (same as setting both limits.ingress and limits.egress / mutually exclusive with limits.ingress and limits.egress)", "type": "string" } }, { "limits.priority": { "default": "100", "longdesc": "", "managed": "no", "shortdesc": "The priority for outgoing traffic, to be used by the kernel queuing discipline to prioritize network packets", "type": "integer" } }, { "mtu": { "default": "MTU of the parent network", "longdesc": "", "managed": "yes", "shortdesc": "The Maximum Transmit Unit (MTU) of the new interface", "type": "integer" } }, { "name": { "default": "kernel assigned", "longdesc": "", "managed": "no", "shortdesc": "The name of the interface inside the instance", "type": "string" } }, { "nested": { "longdesc": "", "managed": "no", "shortdesc": "The parent NIC name to nest this NIC under (see also `vlan`)", "type": "string" } }, { "network": { "longdesc": "", "managed": "yes", "shortdesc": "The managed network to link the device to (required)", "type": "string" } }, { "security.acls": { "longdesc": "", "managed": "no", "shortdesc": "Comma-separated list of network ACLs to apply", "type": "string" } }, { "security.acls.default.egress.action": { "default": "reject", "longdesc": "", "managed": "no", "shortdesc": "Action to use for egress traffic that doesn't match any ACL rule", "type": "string" } }, { "security.acls.default.egress.logged": { "default": "false", "longdesc": "", "managed": "no", "shortdesc": "Whether to log egress traffic that doesn't match any ACL rule", "type": "bool" } }, { "security.acls.default.ingress.action": { "default": "reject", "longdesc": "", "managed": "no", "shortdesc": "Action to use for ingress traffic that doesn't match any ACL rule", "type": "string" } }, { "security.acls.default.ingress.logged": { "default": "false", "longdesc": "", "managed": "no", "shortdesc": "Whether to log ingress traffic that doesn't match any ACL rule", "type": "bool" } }, { "security.promiscuous": { "default": "false", "longdesc": "", "managed": "no", "shortdesc": "Have OVN send unknown network traffic to this network interface (required for some nesting cases)", "type": "bool" } }, { "vlan": { "longdesc": "", "managed": "no", "shortdesc": "The VLAN ID to use when nesting (see also `nested`)", "type": "integer" } } ] }, "nic_p2p": { "keys": [ { "attached": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether the NIC is plugged in or not", "type": "bool" } }, { "boot.priority": { "longdesc": "", "shortdesc": "Boot priority for VMs (higher value boots first)", "type": "integer" } }, { "connected": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether the NIC is connected to the host network", "type": "bool" } }, { "host_name": { "default": "randomly assigned", "longdesc": "", "shortdesc": "The name of the interface on the host", "type": "string" } }, { "hwaddr": { "default": "randomly assigned", "longdesc": "", "shortdesc": "The MAC address of the new interface", "type": "string" } }, { "io.bus": { "default": "`virtio`", "longdesc": "", "shortdesc": "Override the bus for the device (can be `virtio` or `usb`) (VM only)", "type": "string" } }, { "ipv4.routes": { "longdesc": "", "shortdesc": "Comma-delimited list of IPv4 static routes to add on host to NIC", "type": "string" } }, { "ipv6.routes": { "longdesc": "", "shortdesc": "Comma-delimited list of IPv6 static routes to add on host to NIC", "type": "string" } }, { "limits.egress": { "longdesc": "", "shortdesc": "I/O limit in bit/s for outgoing traffic (various suffixes supported, see {ref}`instances-limit-units`)", "type": "string" } }, { "limits.ingress": { "longdesc": "", "shortdesc": "I/O limit in bit/s for incoming traffic (various suffixes supported, see {ref}`instances-limit-units`)", "type": "string" } }, { "limits.max": { "longdesc": "", "shortdesc": "I/O limit in bit/s for both incoming and outgoing traffic (same as setting both limits.ingress and limits.egress)", "type": "string" } }, { "limits.priority": { "longdesc": "", "shortdesc": "The priority for outgoing traffic, to be used by the kernel queuing discipline to prioritize network packets", "type": "integer" } }, { "mtu": { "default": "kernel assigned", "longdesc": "", "shortdesc": "The Maximum Transmit Unit (MTU) of the new interface", "type": "integer" } }, { "name": { "default": "kernel assigned", "longdesc": "", "shortdesc": "The name of the interface inside the instance", "type": "string" } }, { "queue.tx.length": { "longdesc": "", "shortdesc": "The transmit queue length for the NIC", "type": "integer" } } ] }, "nic_physical": { "keys": [ { "attached": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether the NIC is plugged in or not", "type": "bool" } }, { "boot.priority": { "longdesc": "", "managed": "no", "shortdesc": "Boot priority for VMs (higher value boots first)", "type": "integer" } }, { "gvrp": { "condition": "container", "default": "false", "longdesc": "", "managed": "no", "shortdesc": "Register VLAN using GARP VLAN Registration Protocol", "type": "bool" } }, { "hwaddr": { "condition": "container", "default": "randomly assigned", "longdesc": "", "managed": "no", "shortdesc": "The MAC address of the new interface", "type": "string" } }, { "mtu": { "condition": "container", "default": "MTU of the parent device", "longdesc": "", "managed": "no", "shortdesc": "The Maximum Transmit Unit (MTU) of the new interface", "type": "integer" } }, { "name": { "default": "kernel assigned", "longdesc": "", "managed": "no", "shortdesc": "The name of the interface inside the instance", "type": "string" } }, { "network": { "longdesc": "", "managed": "no", "shortdesc": "The managed network to link the device to (instead of specifying the `nictype` directly)", "type": "string" } }, { "parent": { "longdesc": "", "managed": "yes", "shortdesc": "The name of the parent host device (required if specifying the `nictype` directly)", "type": "string" } }, { "vlan": { "condition": "container", "longdesc": "", "managed": "no", "shortdesc": "The VLAN ID to attach to", "type": "integer" } }, { "vlan.tagged": { "condition": "container", "longdesc": "", "managed": "no", "shortdesc": "Comma-delimited list of VLAN IDs or VLAN ranges to join for tagged traffic", "type": "integer" } } ] }, "nic_routed": { "keys": [ { "attached": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether the NIC is plugged in or not", "type": "bool" } }, { "connected": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether the NIC is connected to the host network", "type": "bool" } }, { "gvrp": { "default": "false", "longdesc": "", "shortdesc": "Register VLAN using GARP VLAN Registration Protocol", "type": "bool" } }, { "host_name": { "default": "randomly assigned", "longdesc": "", "shortdesc": "The name of the interface on the host", "type": "string" } }, { "hwaddr": { "default": "randomly assigned", "longdesc": "", "shortdesc": "The MAC address of the new interface", "type": "string" } }, { "io.bus": { "default": "`virtio`", "longdesc": "", "shortdesc": "Override the bus for the device (can be `virtio` or `usb`) (VM only)", "type": "string" } }, { "ipv4.address": { "longdesc": "", "shortdesc": "Comma-delimited list of IPv4 static addresses to add to the instance", "type": "string" } }, { "ipv4.gateway": { "default": "auto", "longdesc": "", "shortdesc": "Whether to add an automatic default IPv4 gateway (can be `auto` or `none`)", "type": "string" } }, { "ipv4.host_address": { "default": "`169.254.0.1`", "longdesc": "", "shortdesc": "The IPv4 address to add to the host-side `veth` interface", "type": "string" } }, { "ipv4.host_table": { "longdesc": "The custom policy routing table ID to add IPv4 static routes to (in addition to the main routing table)\n", "shortdesc": "Deprecated: Use `ipv4.host_tables` instead", "type": "integer" } }, { "ipv4.host_tables": { "default": "254", "longdesc": "", "shortdesc": "Comma-delimited list of routing tables IDs to add IPv4 static routes to", "type": "string" } }, { "ipv4.neighbor_probe": { "default": "true", "longdesc": "", "shortdesc": "Whether to probe the parent network for IP address availability", "type": "bool" } }, { "ipv4.routes": { "longdesc": "", "shortdesc": "Comma-delimited list of IPv4 static routes to add on host to NIC (without L2 ARP/NDP proxy)", "type": "string" } }, { "ipv6.address": { "longdesc": "", "shortdesc": "Comma-delimited list of IPv6 static addresses to add to the instance", "type": "string" } }, { "ipv6.gateway": { "default": "auto", "longdesc": "", "shortdesc": "Whether to add an automatic default IPv6 gateway (can be `auto` or `none`)", "type": "string" } }, { "ipv6.host_address": { "default": "`fe80::1`", "longdesc": "", "shortdesc": "The IPv6 address to add to the host-side `veth` interface", "type": "string" } }, { "ipv6.host_table": { "longdesc": "The custom policy routing table ID to add IPv6 static routes to (in addition to the main routing table)\n", "shortdesc": "Deprecated: Use `ipv6.host_tables` instead", "type": "integer" } }, { "ipv6.host_tables": { "default": "254", "longdesc": "", "shortdesc": "Comma-delimited list of routing tables IDs to add IPv6 static routes to", "type": "string" } }, { "ipv6.neighbor_probe": { "default": "true", "longdesc": "", "shortdesc": "Whether to probe the parent network for IP address availability", "type": "bool" } }, { "ipv6.routes": { "longdesc": "", "shortdesc": "Comma-delimited list of IPv6 static routes to add on host to NIC (without L2 ARP/NDP proxy)", "type": "string" } }, { "limits.egress": { "longdesc": "", "shortdesc": "I/O limit in bit/s for outgoing traffic (various suffixes supported, see {ref}`instances-limit-units`)", "type": "string" } }, { "limits.ingress": { "longdesc": "", "shortdesc": "I/O limit in bit/s for incoming traffic (various suffixes supported, see {ref}`instances-limit-units`)", "type": "string" } }, { "limits.max": { "longdesc": "", "shortdesc": "I/O limit in bit/s for both incoming and outgoing traffic (same as setting both limits.ingress and limits.egress)", "type": "string" } }, { "limits.priority": { "longdesc": "", "shortdesc": "The priority for outgoing traffic, to be used by the kernel queuing discipline to prioritize network packets", "type": "integer" } }, { "mtu": { "default": "parent MTU", "longdesc": "", "shortdesc": "The Maximum Transmit Unit (MTU) of the new interface", "type": "integer" } }, { "name": { "default": "kernel assigned", "longdesc": "", "shortdesc": "The name of the interface inside the instance", "type": "string" } }, { "parent": { "longdesc": "", "shortdesc": "The name of the parent host device to join the instance to", "type": "string" } }, { "queue.tx.length": { "longdesc": "", "shortdesc": "The transmit queue length for the NIC", "type": "integer" } }, { "vlan": { "longdesc": "", "shortdesc": "The VLAN ID to attach to", "type": "integer" } }, { "vrf": { "longdesc": "", "shortdesc": "The VRF on the host in which the host-side interface and routes are created", "type": "string" } } ] }, "nic_sriov": { "keys": [ { "attached": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether the NIC is plugged in or not", "type": "bool" } }, { "boot.priority": { "longdesc": "", "managed": "no", "shortdesc": "Boot priority for VMs (higher value boots first)", "type": "integer" } }, { "hwaddr": { "default": "randomly assigned", "longdesc": "", "managed": "no", "shortdesc": "The MAC address of the new interface", "type": "string" } }, { "mtu": { "default": "kernel assigned", "longdesc": "", "managed": "yes", "shortdesc": "The Maximum Transmit Unit (MTU) of the new interface", "type": "integer" } }, { "name": { "default": "kernel assigned", "longdesc": "", "managed": "no", "shortdesc": "The name of the interface inside the instance", "type": "string" } }, { "network": { "longdesc": "", "managed": "no", "shortdesc": "The managed network to link the device to (instead of specifying the `nictype` directly)", "type": "string" } }, { "parent": { "longdesc": "", "managed": "yes", "shortdesc": "The name of the parent host device (required if specifying the `nictype` directly)", "type": "string" } }, { "pci": { "longdesc": "", "required": "no", "shortdesc": "The PCI address of the parent host device", "type": "string" } }, { "productid": { "longdesc": "", "required": "no", "shortdesc": "The product ID of the parent host device", "type": "string" } }, { "security.mac_filtering": { "default": "false", "longdesc": "", "managed": "no", "shortdesc": "Prevent the instance from spoofing another instance's MAC address", "type": "bool" } }, { "security.trusted": { "default": "false, if supported by parent device", "longdesc": "", "managed": "no", "shortdesc": "Allows the instance to configure the NIC in ways that may negatively impact security.", "type": "bool" } }, { "vendorid": { "longdesc": "", "required": "no", "shortdesc": "The vendor ID of the parent host device", "type": "string" } }, { "vlan": { "longdesc": "", "managed": "no", "shortdesc": "The VLAN ID to attach to", "type": "integer" } } ] }, "pci": { "keys": [ { "address": { "longdesc": "", "required": "yes", "shortdesc": "PCI address of the device", "type": "string" } }, { "firmware": { "default": "true", "longdesc": "", "required": "no", "shortdesc": "Whether to expose the device's option ROM to the VM", "type": "bool" } } ] }, "proxy": { "keys": [ { "bind": { "default": "`host`", "longdesc": "", "required": "no", "shortdesc": "Which side to bind on (`host`/`instance`)", "type": "string" } }, { "connect": { "longdesc": "", "required": "yes", "shortdesc": "The address and port to connect to (`\u003ctype\u003e:\u003caddr\u003e:\u003cport\u003e[-\u003cport\u003e][,\u003cport\u003e]`)", "type": "string" } }, { "gid": { "default": "`0`", "longdesc": "", "required": "no", "shortdesc": "GID of the owner of the listening Unix socket", "type": "int" } }, { "listen": { "longdesc": "", "required": "yes", "shortdesc": "The address and port to bind and listen (`\u003ctype\u003e:\u003caddr\u003e:\u003cport\u003e[-\u003cport\u003e][,\u003cport\u003e]`)", "type": "string" } }, { "mode": { "default": "`0644`", "longdesc": "", "required": "no", "shortdesc": "Mode for the listening Unix socket", "type": "int" } }, { "nat": { "default": "`false`", "longdesc": "", "required": "no", "shortdesc": "Whether to optimize proxying via NAT (requires that the instance NIC has a static IP address)", "type": "bool" } }, { "proxy_protocol": { "default": "`false`", "longdesc": "", "required": "no", "shortdesc": "Whether to use the HAProxy PROXY protocol to transmit sender information", "type": "bool" } }, { "security.gid": { "default": "`0`", "longdesc": "", "required": "no", "shortdesc": "What GID to drop privilege to", "type": "int" } }, { "security.uid": { "default": "`0`", "longdesc": "", "required": "no", "shortdesc": "What UID to drop privilege to", "type": "int" } }, { "uid": { "default": "`0`", "longdesc": "", "required": "no", "shortdesc": "UID of the owner of the listening Unix socket", "type": "int" } } ] }, "tpm": { "keys": [ { "path": { "default": "-", "longdesc": "", "required": "for containers", "shortdesc": "Only for containers: path inside the instance (for example, `/dev/tpm0`)", "type": "string" } }, { "pathrm": { "default": "-", "longdesc": "", "required": "for containers", "shortdesc": "Only for containers: resource manager path inside the instance (for example, `/dev/tpmrm0`)", "type": "string" } } ] }, "unix-char-block": { "keys": [ { "gid": { "default": "0", "longdesc": "", "shortdesc": "GID of the device owner in the instance", "type": "int" } }, { "major": { "default": "device on host", "longdesc": "", "shortdesc": "Device major number", "type": "int" } }, { "minor": { "default": "device on host", "longdesc": "", "shortdesc": "Device minor number", "type": "int" } }, { "mode": { "default": "0660", "longdesc": "", "shortdesc": "Mode of the device in the instance", "type": "int" } }, { "path": { "longdesc": "", "shortdesc": "Path inside the instance (one of `source` and `path` must be set)", "type": "string" } }, { "required": { "default": "true", "longdesc": "", "shortdesc": "Whether this device is required to start the instance", "type": "bool" } }, { "source": { "longdesc": "", "shortdesc": "Path on the host (one of `source` and `path` must be set)", "type": "string" } }, { "uid": { "default": "0", "longdesc": "", "shortdesc": "UID of the device owner in the instance", "type": "int" } } ] }, "unix-hotplug": { "keys": [ { "gid": { "default": "0", "longdesc": "", "shortdesc": "GID of the device owner in the instance", "type": "int" } }, { "mode": { "default": "0660", "longdesc": "", "shortdesc": "Mode of the device in the instance", "type": "int" } }, { "pci": { "longdesc": "", "shortdesc": "The PCI address of a USB controller to monitor", "type": "string" } }, { "productid": { "longdesc": "", "shortdesc": "The product ID of the USB device", "type": "string" } }, { "required": { "default": "true", "longdesc": "", "shortdesc": "Whether this device is required to start the instance", "type": "bool" } }, { "uid": { "default": "0", "longdesc": "", "shortdesc": "UID of the device owner in the instance", "type": "int" } }, { "vendorid": { "longdesc": "", "shortdesc": "The vendor ID of the USB device", "type": "string" } } ] }, "usb": { "keys": [ { "attached": { "default": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether the USB device is plugged in or not", "type": "bool" } }, { "busnum": { "longdesc": "", "shortdesc": "The bus number of which the USB device is attached", "type": "int" } }, { "devnum": { "longdesc": "", "shortdesc": "The device number of the USB device", "type": "int" } }, { "gid": { "defaultdesc": "`0`", "longdesc": "", "shortdesc": "Only for containers: GID of the device owner in the instance", "type": "int" } }, { "mode": { "defaultdesc": "`0660`", "longdesc": "", "shortdesc": "Only for containers: Mode of the device in the instance", "type": "int" } }, { "productid": { "longdesc": "", "shortdesc": "The product ID of the USB device", "type": "string" } }, { "required": { "defaultdesc": "`false`", "longdesc": "", "shortdesc": "Whether this device is required to start the instance (the default is `false`, and all devices can be hotplugged)", "type": "bool" } }, { "serial": { "longdesc": "", "shortdesc": "The serial number of the USB device", "type": "string" } }, { "uid": { "defaultdesc": "`0`", "longdesc": "", "shortdesc": "Only for containers: UID of the device owner in the instance", "type": "int" } }, { "vendorid": { "longdesc": "", "shortdesc": "The vendor ID of the USB device", "type": "string" } } ] } }, "image": { "requirements": { "keys": [ { "requirements.cdrom_agent": { "longdesc": "", "shortdesc": "If set to `true`, indicates that the VM requires an `agent:config` disk be added.", "type": "bool" } }, { "requirements.cdrom_cloud_init": { "longdesc": "", "shortdesc": "If set to `true`, indicates that the VM requires a `cloud-init:config` disk be present every time `cloud-init` should be run.", "type": "bool" } }, { "requirements.cgroup": { "longdesc": "", "shortdesc": "If set to `v1`, indicates that the image requires the host to run cgroup v1.", "type": "string" } }, { "requirements.nesting": { "longdesc": "", "shortdesc": "If set to `true`, indicates that the image cannot work without nesting enabled.", "type": "bool" } }, { "requirements.privileged": { "longdesc": "", "shortdesc": "If set to `false`, indicates that the image cannot work as a privileged container.", "type": "bool" } }, { "requirements.secureboot": { "longdesc": "", "shortdesc": "If set to `false`, indicates that the image cannot boot under secure boot.", "type": "bool" } } ] } }, "instance": { "boot": { "keys": [ { "boot.autorestart": { "liveupdate": "no", "longdesc": "If set to `true` will attempt up to 10 restarts over a 1 minute period upon unexpected instance exit.", "shortdesc": "Whether to automatically restart an instance on unexpected exit", "type": "bool" } }, { "boot.autostart": { "liveupdate": "no", "longdesc": "If unset or set to `last-state`, restores the last state.", "shortdesc": "Whether to always start the instance when the daemon starts", "type": "bool" } }, { "boot.autostart.delay": { "defaultdesc": "0", "liveupdate": "no", "longdesc": "The number of seconds to wait after the instance started before starting the next one.", "shortdesc": "Delay after starting the instance", "type": "integer" } }, { "boot.autostart.priority": { "liveupdate": "no", "longdesc": "The instance with the highest value is started first.\nInstances without a priority set will be started (with some parallelism) ahead of\ninstances with a priority set.", "shortdesc": "What order to start the instances in", "type": "integer" } }, { "boot.host_shutdown_action": { "defaultdesc": "stop", "liveupdate": "yes", "longdesc": "Action to take on host shut down\n\nValid values are: `stop`, `force-stop` or `stateful-stop`", "shortdesc": "What action to take on the instance when the host is shut down", "type": "string" } }, { "boot.host_shutdown_timeout": { "defaultdesc": "30", "liveupdate": "yes", "longdesc": "Number of seconds to wait for the instance to shut down before it is force-stopped.", "shortdesc": "How long to wait for the instance to shut down", "type": "integer" } }, { "boot.stop.priority": { "defaultdesc": "0", "liveupdate": "no", "longdesc": "The instance with the highest value is shut down first.", "shortdesc": "What order to shut down the instances in", "type": "integer" } } ] }, "cloud-init": { "keys": [ { "cloud-init.network-config": { "condition": "If supported by image", "defaultdesc": "`DHCP on eth0`", "liveupdate": "no", "longdesc": "The content is used as seed value for `cloud-init`.", "shortdesc": "Network configuration for `cloud-init`", "type": "string" } }, { "cloud-init.user-data": { "condition": "If supported by image", "defaultdesc": "`#cloud-config`", "liveupdate": "no", "longdesc": "The content is used as seed value for `cloud-init`.", "shortdesc": "User data for `cloud-init`", "type": "string" } }, { "cloud-init.vendor-data": { "condition": "If supported by image", "defaultdesc": "`#cloud-config`", "liveupdate": "no", "longdesc": "The content is used as seed value for `cloud-init`.", "shortdesc": "Vendor data for `cloud-init`", "type": "string" } }, { "user.network-config": { "condition": "If supported by image", "defaultdesc": "`DHCP on eth0`", "liveupdate": "no", "longdesc": "", "shortdesc": "Legacy version of `cloud-init.network-config`", "type": "string" } }, { "user.user-data": { "condition": "If supported by image", "defaultdesc": "`#cloud-config`", "liveupdate": "no", "longdesc": "", "shortdesc": "Legacy version of `cloud-init.user-data`", "type": "string" } }, { "user.vendor-data": { "condition": "If supported by image", "defaultdesc": "`#cloud-config`", "liveupdate": "no", "longdesc": "", "shortdesc": "Legacy version of `cloud-init.vendor-data`", "type": "string" } } ] }, "migration": { "keys": [ { "migration.incremental.memory": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "yes", "longdesc": "Using incremental memory transfer of the instance's memory can reduce downtime.", "shortdesc": "Whether to use incremental memory transfer", "type": "bool" } }, { "migration.incremental.memory.goal": { "condition": "container", "defaultdesc": "`70`", "liveupdate": "yes", "longdesc": "", "shortdesc": "Percentage of memory to have in sync before stopping the instance", "type": "integer" } }, { "migration.incremental.memory.iterations": { "condition": "container", "defaultdesc": "`10`", "liveupdate": "yes", "longdesc": "", "shortdesc": "Maximum number of transfer operations to go through before stopping the instance", "type": "integer" } }, { "migration.stateful": { "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "Enabling this option prevents the use of some features that are incompatible with it.", "shortdesc": "Whether to allow for stateful stop/start and snapshots", "type": "bool" } } ] }, "miscellaneous": { "keys": [ { "agent.nic_config": { "condition": "virtual machine", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "For containers, the name and MTU of the default network interfaces is used for the instance devices.\nFor virtual machines, set this option to `true` to set the name and MTU of the default network interfaces to be the same as the instance devices.", "shortdesc": "Whether to use the name and MTU of the default network interfaces", "type": "bool" } }, { "cluster.evacuate": { "defaultdesc": "`auto`", "liveupdate": "no", "longdesc": "The `cluster.evacuate` provides control over how instances are handled when a cluster member is being\nevacuated.\n\nAvailable Modes:\n - `auto` *(default)*: The system will automatically decide the best evacuation method based on the\n instance's type and configured devices:\n + If any device is not suitable for migration, the instance will not be migrated (only stopped).\n + Live migration will be used only for virtual machines with the `migration.stateful` setting\n enabled and for which all its devices can be migrated as well.\n - `live-migrate`: Instances are live-migrated to another server. This means the instance remains running\n and operational during the migration process, ensuring minimal disruption.\n - `migrate`: In this mode, instances are migrated to another server in the cluster. The migration\n process will not be live, meaning there will be a brief downtime for the instance during the\n migration.\n - `stop`: Instances are not migrated. Instead, they are stopped on the current server.\n - `stateful-stop`: Instances are not migrated. Instead, they are stopped on the current server\n but with their runtime state (memory) stored on disk for resuming on restore.\n - `force-stop`: Instances are not migrated. Instead, they are forcefully stopped.\n\nSee {ref}`cluster-evacuate` for more information.", "shortdesc": "What to do when evacuating the instance", "type": "string" } }, { "environment.*": { "liveupdate": "yes", "longdesc": "Extra environment variables to set on boot and during exec.", "shortdesc": "Free-form environment key/value", "type": "string" } }, { "linux.kernel_modules": { "condition": "container", "liveupdate": "yes", "longdesc": "Specify the kernel modules as a comma-separated list.", "shortdesc": "Kernel modules to load before starting the instance", "type": "string" } }, { "linux.sysctl.*": { "condition": "container", "liveupdate": "no", "longdesc": "", "shortdesc": "Override for the corresponding `sysctl` setting in the container", "type": "string" } }, { "smbios11.*": { "liveupdate": "yes", "longdesc": "`SMBIOS Type 11` configuration keys.", "shortdesc": "Free-form `SMBIOS Type 11` key/value", "type": "string" } }, { "systemd.credential-binary.*": { "liveupdate": "yes", "longdesc": "Systemd credential key/value pair passed as a read-only bind mount in containers and as `SMBIOS Type 11` data in virtual machines. The value is Base64 encoded.", "shortdesc": "Systemd credential key/value, where value is Base64 encoded", "type": "string" } }, { "systemd.credential.*": { "liveupdate": "yes", "longdesc": "Systemd credential key/value pair passed as a read-only bind mount in containers and as `SMBIOS Type 11` data in virtual machines.", "shortdesc": "Systemd credential key/value", "type": "string" } }, { "user.*": { "liveupdate": "yes", "longdesc": "User keys can be used in search.", "shortdesc": "Free-form user key/value storage", "type": "string" } } ] }, "nvidia": { "keys": [ { "nvidia.driver.capabilities": { "condition": "container", "defaultdesc": "`compute,utility`", "liveupdate": "no", "longdesc": "The specified driver capabilities are used to set `libnvidia-container NVIDIA_DRIVER_CAPABILITIES`.", "shortdesc": "What driver capabilities the instance needs", "type": "string" } }, { "nvidia.require.cuda": { "condition": "container", "liveupdate": "no", "longdesc": "The specified version expression is used to set `libnvidia-container NVIDIA_REQUIRE_CUDA`.", "shortdesc": "Required CUDA version", "type": "string" } }, { "nvidia.require.driver": { "condition": "container", "liveupdate": "no", "longdesc": "The specified version expression is used to set `libnvidia-container NVIDIA_REQUIRE_DRIVER`.", "shortdesc": "Required driver version", "type": "string" } }, { "nvidia.runtime": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "", "shortdesc": "Whether to pass the host NVIDIA and CUDA runtime libraries into the instance", "type": "bool" } } ] }, "oci": { "keys": [ { "oci.cwd": { "condition": "OCI container", "liveupdate": "no", "longdesc": "Override the working directory of an OCI container.", "shortdesc": "OCI container working directory", "type": "string" } }, { "oci.entrypoint": { "condition": "OCI container", "liveupdate": "no", "longdesc": "Override the entry point of an OCI container.", "shortdesc": "OCI container entry point", "type": "string" } }, { "oci.gid": { "condition": "OCI container", "liveupdate": "no", "longdesc": "Override the GID of the process run in an OCI container.", "shortdesc": "OCI container GID", "type": "string" } }, { "oci.uid": { "condition": "OCI container", "liveupdate": "no", "longdesc": "Override the UID of the process run in an OCI container.", "shortdesc": "OCI container UID", "type": "string" } } ] }, "raw": { "keys": [ { "raw.apparmor": { "liveupdate": "yes", "longdesc": "The specified entries are appended to the generated profile.", "shortdesc": "AppArmor profile entries", "type": "blob" } }, { "raw.idmap": { "condition": "unprivileged container", "liveupdate": "no", "longdesc": "For example: `both 1000 1000`", "shortdesc": "Raw idmap configuration", "type": "blob" } }, { "raw.lxc": { "condition": "container", "liveupdate": "no", "longdesc": "", "shortdesc": "Raw LXC configuration to be appended to the generated one", "type": "blob" } }, { "raw.qemu": { "condition": "virtual machine", "liveupdate": "no", "longdesc": "", "shortdesc": "Raw QEMU configuration to be appended to the generated command line", "type": "blob" } }, { "raw.qemu.conf": { "condition": "virtual machine", "liveupdate": "no", "longdesc": "See {ref}`instance-options-qemu` for more information.", "shortdesc": "Addition/override to the generated `qemu.conf` file", "type": "blob" } }, { "raw.qemu.qmp.early": { "condition": "virtual machine", "liveupdate": "no", "longdesc": "", "shortdesc": "QMP commands to run before Incus QEMU initialization", "type": "blob" } }, { "raw.qemu.qmp.post-start": { "condition": "virtual machine", "liveupdate": "no", "longdesc": "", "shortdesc": "QMP commands to run after the VM has started", "type": "blob" } }, { "raw.qemu.qmp.pre-start": { "condition": "virtual machine", "liveupdate": "no", "longdesc": "", "shortdesc": "QMP commands to run after Incus QEMU initialization and before the VM has started", "type": "blob" } }, { "raw.qemu.scriptlet": { "condition": "virtual machine", "liveupdate": "no", "longdesc": "", "shortdesc": "QEMU scriptlet to run at early, pre-start and post-start stages", "type": "string" } }, { "raw.seccomp": { "condition": "container", "liveupdate": "no", "longdesc": "", "shortdesc": "Raw Seccomp configuration", "type": "blob" } } ] }, "resource-limits": { "keys": [ { "limits.cpu": { "defaultdesc": "1 (VMs)", "liveupdate": "yes", "longdesc": "A number or a specific range of CPUs to expose to the instance.\n\nSee {ref}`instance-options-limits-cpu` for more information.", "shortdesc": "Which CPUs to expose to the instance", "type": "string" } }, { "limits.cpu.allowance": { "condition": "container", "defaultdesc": "100%", "liveupdate": "yes", "longdesc": "To control how much of the CPU can be used, specify either a percentage (`50%`) for a soft limit\nor a chunk of time (`25ms/100ms`) for a hard limit.\n\nSee {ref}`instance-options-limits-cpu-container` for more information.", "shortdesc": "How much of the CPU can be used", "type": "string" } }, { "limits.cpu.nodes": { "liveupdate": "yes", "longdesc": "A comma-separated list of NUMA node IDs or ranges to place the instance CPUs on.\nAlternatively, the value `balanced` may be used to have Incus pick the least busy NUMA node on startup.\n\nSee {ref}`instance-options-limits-cpu-container` for more information.", "shortdesc": "Which NUMA nodes to place the instance CPUs on", "type": "string" } }, { "limits.cpu.priority": { "condition": "container", "defaultdesc": "`10` (maximum)", "liveupdate": "yes", "longdesc": "When overcommitting resources, specify the CPU scheduling priority compared to other instances that share the same CPUs.\nSpecify an integer between 0 and 10.\n\nSee {ref}`instance-options-limits-cpu-container` for more information.", "shortdesc": "CPU scheduling priority compared to other instances", "type": "integer" } }, { "limits.disk.priority": { "defaultdesc": "`5` (medium)", "liveupdate": "yes", "longdesc": "Controls how much priority to give to the instance's I/O requests when under load.\n\nSpecify an integer between 0 and 10.", "shortdesc": "Priority of the instance's I/O requests", "type": "integer" } }, { "limits.hugepages.1GB": { "condition": "container", "liveupdate": "yes", "longdesc": "Fixed value (in bytes) to limit the number of 1 GB huge pages.\nVarious suffixes are supported (see {ref}`instances-limit-units`).\n\nSee {ref}`instance-options-limits-hugepages` for more information.", "shortdesc": "Limit for the number of 1 GB huge pages", "type": "string" } }, { "limits.hugepages.1MB": { "condition": "container", "liveupdate": "yes", "longdesc": "Fixed value (in bytes) to limit the number of 1 MB huge pages.\nVarious suffixes are supported (see {ref}`instances-limit-units`).\n\nSee {ref}`instance-options-limits-hugepages` for more information.", "shortdesc": "Limit for the number of 1 MB huge pages", "type": "string" } }, { "limits.hugepages.2MB": { "condition": "container", "liveupdate": "yes", "longdesc": "Fixed value (in bytes) to limit the number of 2 MB huge pages.\nVarious suffixes are supported (see {ref}`instances-limit-units`).\n\nSee {ref}`instance-options-limits-hugepages` for more information.", "shortdesc": "Limit for the number of 2 MB huge pages", "type": "string" } }, { "limits.hugepages.64KB": { "condition": "container", "liveupdate": "yes", "longdesc": "Fixed value (in bytes) to limit the number of 64 KB huge pages.\nVarious suffixes are supported (see {ref}`instances-limit-units`).\n\nSee {ref}`instance-options-limits-hugepages` for more information.", "shortdesc": "Limit for the number of 64 KB huge pages", "type": "string" } }, { "limits.memory": { "defaultdesc": "`1GiB` (VMs)", "liveupdate": "yes", "longdesc": "Percentage of the host's memory or a fixed value in bytes.\nVarious suffixes are supported.\n\nSee {ref}`instances-limit-units` for details.", "shortdesc": "Usage limit for the host's memory", "type": "string" } }, { "limits.memory.enforce": { "condition": "container", "defaultdesc": "`hard`", "liveupdate": "yes", "longdesc": "If the instance's memory limit is `hard`, the instance cannot exceed its limit.\nIf it is `soft`, the instance can exceed its memory limit when extra host memory is available.", "shortdesc": "Whether the memory limit is `hard` or `soft`", "type": "string" } }, { "limits.memory.hotplug": { "condition": "virtual machine", "defaultdesc": "`true`", "liveupdate": "yes", "longdesc": "If this option is set to `false`, disable memory hotplug entirely.\nAlternatively, it can be set to a bytes value which will define an upper limit for hotplugged memory.\nThe value must be greater than or equal to limits.memory.", "shortdesc": "Control upper limit for hotplugged memory or disable memory hotplug.", "type": "string" } }, { "limits.memory.hugepages": { "condition": "virtual machine", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "If this option is set to `false`, regular system memory is used.", "shortdesc": "Whether to back the instance using huge pages", "type": "bool" } }, { "limits.memory.oom_priority": { "defaultdesc": "`0`", "liveupdate": "yes", "longdesc": "Specify an integer between -1000 and 1000.\nA negative value makes the instance less likely to be killed by the Out Of Memory killer,\nwhile a positive value makes it more likely to be killed.\nThe default value of 0 means no adjustment to the Out Of Memory score.", "shortdesc": "Out Of Memory killer priority adjustment for the instance", "type": "integer" } }, { "limits.memory.swap": { "condition": "container", "defaultdesc": "`true`", "liveupdate": "yes", "longdesc": "When set to `true` or `false`, it controls whether the container is likely to get some of\nits memory swapped by the kernel. Alternatively, it can be set to a bytes value which will\nthen allow the container to make use of additional memory through swap.", "shortdesc": "Control swap usage by the instance", "type": "string" } }, { "limits.memory.swap.priority": { "condition": "container", "defaultdesc": "`10` (maximum)", "liveupdate": "yes", "longdesc": "Specify an integer between 0 and 10.\nThe higher the value, the less likely the instance is to be swapped to disk.", "shortdesc": "Prevents the instance from being swapped to disk", "type": "integer" } }, { "limits.processes": { "condition": "container", "defaultdesc": "empty", "liveupdate": "yes", "longdesc": "If left empty, no limit is set.", "shortdesc": "Maximum number of processes that can run in the instance", "type": "integer" } } ] }, "security": { "keys": [ { "security.agent.metrics": { "condition": "virtual machine", "defaultdesc": "`true`", "liveupdate": "no", "longdesc": "", "shortdesc": "Whether the `incus-agent` is queried for state information and metrics", "type": "bool" } }, { "security.bpffs.delegate_attachs": { "condition": "unprivileged container", "liveupdate": "no", "longdesc": "See {ref}`bpf-tokens` for more information.\n", "shortdesc": "What BPF attach types to delegate", "type": "string" } }, { "security.bpffs.delegate_cmds": { "condition": "unprivileged container", "liveupdate": "no", "longdesc": "See {ref}`bpf-tokens` for more information.\n", "shortdesc": "What BPF command types to delegate", "type": "string" } }, { "security.bpffs.delegate_maps": { "condition": "unprivileged container", "liveupdate": "no", "longdesc": "See {ref}`bpf-tokens` for more information.\n", "shortdesc": "What BPF map types to delegate", "type": "string" } }, { "security.bpffs.delegate_progs": { "condition": "unprivileged container", "liveupdate": "no", "longdesc": "See {ref}`bpf-tokens` for more information.\n", "shortdesc": "What BPF program types to delegate", "type": "string" } }, { "security.bpffs.path": { "condition": "unprivileged container", "defaultdesc": "`/sys/fs/bpf`", "liveupdate": "no", "longdesc": "The specified path must exist in the container.\nThe BPF file system is only mounted if any of the `security.bpffs.delegate_*` options are set.\nSee {ref}`bpf-tokens` for more information.\n", "shortdesc": "The path to mount the BPF file system at", "type": "string" } }, { "security.csm": { "condition": "virtual machine", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "When enabling this option, set {config:option}`instance-security:security.secureboot` to `false`.", "shortdesc": "Whether to use a firmware that supports UEFI-incompatible operating systems", "type": "bool" } }, { "security.guestapi": { "defaultdesc": "`true`", "liveupdate": "no", "longdesc": "See {ref}`dev-incus` for more information.", "shortdesc": "Whether `/dev/incus` is present in the instance", "type": "bool" } }, { "security.guestapi.images": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "", "shortdesc": "Controls the availability of the `/1.0/images` API over `guestapi`", "type": "bool" } }, { "security.idmap.base": { "condition": "unprivileged container", "liveupdate": "no", "longdesc": "Setting this option overrides auto-detection.", "shortdesc": "The base host ID to use for the allocation", "type": "integer" } }, { "security.idmap.isolated": { "condition": "unprivileged container", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "If specified, the idmap used for this instance is unique among instances that have this option set.", "shortdesc": "Whether to use a unique idmap for this instance", "type": "bool" } }, { "security.idmap.size": { "condition": "unprivileged container", "liveupdate": "no", "longdesc": "", "shortdesc": "The size of the idmap to use", "type": "integer" } }, { "security.iommu": { "condition": "virtual machine", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "", "shortdesc": "Whether to enable virtual IOMMU, useful for device passthrough and nesting", "type": "bool" } }, { "security.nesting": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "yes", "longdesc": "", "shortdesc": "Whether to support running Incus (nested) inside the instance", "type": "bool" } }, { "security.privileged": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "", "shortdesc": "Whether to run the instance in privileged mode", "type": "bool" } }, { "security.protection.delete": { "defaultdesc": "`false`", "liveupdate": "yes", "longdesc": "", "shortdesc": "Prevents the instance from being deleted", "type": "bool" } }, { "security.protection.shift": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "yes", "longdesc": "Set this option to `true` to prevent the instance's file system from being UID/GID shifted on startup.", "shortdesc": "Whether to protect the file system from being UID/GID shifted", "type": "bool" } }, { "security.secureboot": { "condition": "virtual machine", "defaultdesc": "`true`", "liveupdate": "no", "longdesc": "When disabling this option, consider enabling {config:option}`instance-security:security.csm`.", "shortdesc": "Whether UEFI secure boot is enforced with the default Microsoft keys", "type": "bool" } }, { "security.sev": { "condition": "virtual machine", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "", "shortdesc": "Whether AMD SEV (Secure Encrypted Virtualization) is enabled for this VM", "type": "bool" } }, { "security.sev.policy.es": { "condition": "virtual machine", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "", "shortdesc": "Whether AMD SEV-ES (SEV Encrypted State) is enabled for this VM", "type": "bool" } }, { "security.sev.session.data": { "condition": "virtual machine", "defaultdesc": "`true`", "liveupdate": "no", "longdesc": "", "shortdesc": "The guest owner's `base64`-encoded session blob", "type": "string" } }, { "security.sev.session.dh": { "condition": "virtual machine", "defaultdesc": "`true`", "liveupdate": "no", "longdesc": "", "shortdesc": "The guest owner's `base64`-encoded Diffie-Hellman key", "type": "string" } }, { "security.syscalls.allow": { "condition": "container", "liveupdate": "no", "longdesc": "A `\\n`-separated list of syscalls to allow.\nThis list must be mutually exclusive with `security.syscalls.deny*`.", "shortdesc": "List of syscalls to allow", "type": "string" } }, { "security.syscalls.deny": { "condition": "container", "liveupdate": "no", "longdesc": "A `\\n`-separated list of syscalls to deny.\nThis list must be mutually exclusive with `security.syscalls.allow`.", "shortdesc": "List of syscalls to deny", "type": "string" } }, { "security.syscalls.deny_compat": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "On `x86_64`, this option controls whether to block `compat_*` syscalls.\nOn other architectures, the option is ignored.", "shortdesc": "Whether to block `compat_*` syscalls (`x86_64` only)", "type": "bool" } }, { "security.syscalls.deny_default": { "condition": "container", "defaultdesc": "`true`", "liveupdate": "no", "longdesc": "", "shortdesc": "Whether to enable the default syscall deny", "type": "bool" } }, { "security.syscalls.intercept.bpf": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "", "shortdesc": "Whether to handle the `bpf()` system call", "type": "bool" } }, { "security.syscalls.intercept.bpf.devices": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "This option controls whether to allow BPF programs for the devices cgroup in the unified hierarchy to be loaded.", "shortdesc": "Whether to allow BPF programs", "type": "bool" } }, { "security.syscalls.intercept.mknod": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "These system calls allow creation of a limited subset of char/block devices.", "shortdesc": "Whether to handle the `mknod` and `mknodat` system calls", "type": "bool" } }, { "security.syscalls.intercept.mount": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "", "shortdesc": "Whether to handle the `mount` system call", "type": "bool" } }, { "security.syscalls.intercept.mount.allowed": { "condition": "container", "liveupdate": "yes", "longdesc": "Specify a comma-separated list of file systems that are safe to mount for processes inside the instance.", "shortdesc": "File systems that can be mounted", "type": "string" } }, { "security.syscalls.intercept.mount.fuse": { "condition": "container", "liveupdate": "yes", "longdesc": "Specify the mounts of a given file system that should be redirected to their FUSE implementation (for example, `ext4=fuse2fs`).", "shortdesc": "File system that should be redirected to FUSE implementation", "type": "string" } }, { "security.syscalls.intercept.mount.shift": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "yes", "longdesc": "", "shortdesc": "Whether to use idmapped mounts for syscall interception", "type": "bool" } }, { "security.syscalls.intercept.sched_setscheduler": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "This system call allows increasing process priority.", "shortdesc": "Whether to handle the `sched_setscheduler` system call", "type": "bool" } }, { "security.syscalls.intercept.setxattr": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "This system call allows setting a limited subset of restricted extended attributes.", "shortdesc": "Whether to handle the `setxattr` system call", "type": "bool" } }, { "security.syscalls.intercept.sysinfo": { "condition": "container", "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "This system call can be used to get cgroup-based resource usage information.", "shortdesc": "Whether to handle the `sysinfo` system call", "type": "bool" } } ] }, "snapshots": { "keys": [ { "snapshots.expiry": { "liveupdate": "no", "longdesc": "Specify an expression like `1M 2H 3d 4w 5m 6y`.", "shortdesc": "When snapshots are to be deleted", "type": "string" } }, { "snapshots.expiry.manual": { "liveupdate": "no", "longdesc": "Specify an expression like `1M 2H 3d 4w 5m 6y`.", "shortdesc": "When snapshots are to be deleted (for those not created through scheduling)", "type": "string" } }, { "snapshots.pattern": { "defaultdesc": "`snap%d`", "liveupdate": "no", "longdesc": "Specify a Pongo2 template string that represents the snapshot name.\nThis template is used for scheduled snapshots and for unnamed snapshots.\n\nSee {ref}`instance-options-snapshots-names` for more information.", "shortdesc": "Template for the snapshot name", "type": "string" } }, { "snapshots.schedule": { "defaultdesc": "empty", "liveupdate": "no", "longdesc": "Specify either a cron expression (`\u003cminute\u003e \u003chour\u003e \u003cdom\u003e \u003cmonth\u003e \u003cdow\u003e`), a comma-and-space-separated list of schedule aliases (`@startup`, `@hourly`, `@daily`, `@midnight`, `@weekly`, `@monthly`, `@annually`, `@yearly`), or leave empty to disable automatic snapshots.\n\nNote that unlike most other configuration keys, this one must be comma-and-space-separated and not just comma-separated as cron expression can themselves contain commas.\n", "shortdesc": "Schedule for automatic instance snapshots", "type": "string" } }, { "snapshots.schedule.stopped": { "defaultdesc": "`false`", "liveupdate": "no", "longdesc": "", "shortdesc": "Whether to automatically snapshot stopped instances", "type": "bool" } } ] }, "volatile": { "keys": [ { "volatile.\u003cname\u003e.apply_quota": { "longdesc": "The disk quota is applied the next time the instance starts.", "shortdesc": "Disk quota", "type": "string" } }, { "volatile.\u003cname\u003e.ceph_rbd": { "longdesc": "", "shortdesc": "RBD device path for Ceph disk devices", "type": "string" } }, { "volatile.\u003cname\u003e.host_name": { "longdesc": "", "shortdesc": "Network device name on the host", "type": "string" } }, { "volatile.\u003cname\u003e.hwaddr": { "longdesc": "The network device MAC address is used when no `hwaddr` property is set on the device itself.", "shortdesc": "Network device MAC address", "type": "string" } }, { "volatile.\u003cname\u003e.io.bus": { "longdesc": "The IO bus stores the actual IO bus being used, checked in case `io.bus=auto`.", "shortdesc": "IO bus in use", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.created": { "longdesc": "Possible values are `true` or `false`.", "shortdesc": "Whether the network device physical device was created", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.hwaddr": { "longdesc": "The original MAC that was used when moving a physical device into an instance.", "shortdesc": "Network device original MAC", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.ip_addresses": { "longdesc": "Comma-separated list of the last used IP addresses of the network device.", "shortdesc": "Last used IP addresses", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.mtu": { "longdesc": "The original MTU that was used when moving a physical device into an instance.", "shortdesc": "Network device original MTU", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.pci.driver": { "longdesc": "The original host driver for the PCI device.", "shortdesc": "PCI original host driver", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.pci.parent": { "longdesc": "The parent host device used when allocating a PCI device to an instance.", "shortdesc": "PCI parent host device", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.pci.slot.name": { "longdesc": "The parent host device PCI slot name.", "shortdesc": "PCI parent slot name", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.usb.bus": { "longdesc": "The original USB bus address.", "shortdesc": "USB bus address", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.usb.device": { "longdesc": "The original USB device identifier.", "shortdesc": "USB device identifier", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.vdpa.name": { "longdesc": "The VDPA device name used when moving a VDPA device file descriptor into an instance.", "shortdesc": "VDPA device name", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.vf.hwaddr": { "longdesc": "The original MAC used when moving a VF into an instance.", "shortdesc": "SR-IOV virtual function original MAC", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.vf.id": { "longdesc": "The ID used when moving a VF into an instance.", "shortdesc": "SR-IOV virtual function ID", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.vf.parent": { "longdesc": "The parent host device used when allocating a VF into an instance.", "shortdesc": "SR-IOV parent host device", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.vf.spoofcheck": { "longdesc": "The original spoof check setting used when moving a VF into an instance.", "shortdesc": "SR-IOV virtual function original spoof check setting", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.vf.trusted": { "longdesc": "The original trusted setting used when moving a VF into an instance.", "shortdesc": "SR-IOV virtual function original trusted setting", "type": "string" } }, { "volatile.\u003cname\u003e.last_state.vf.vlan": { "longdesc": "The original VLAN used when moving a VF into an instance.", "shortdesc": "SR-IOV virtual function original VLAN", "type": "string" } }, { "volatile.\u003cname\u003e.mig.uuid": { "longdesc": "The NVIDIA MIG instance UUID.", "shortdesc": "MIG instance UUID", "type": "string" } }, { "volatile.\u003cname\u003e.name": { "longdesc": "The network interface name inside of the instance when no `name` property is set on the device itself.", "shortdesc": "Network interface name inside of the instance", "type": "string" } }, { "volatile.\u003cname\u003e.vgpu.uuid": { "longdesc": "The NVIDIA virtual GPU instance UUID.", "shortdesc": "virtual GPU instance UUID", "type": "string" } }, { "volatile.apply_nvram": { "longdesc": "", "shortdesc": "Whether to regenerate VM NVRAM the next time the instance starts", "type": "bool" } }, { "volatile.apply_template": { "longdesc": "The template with the given name is triggered upon next startup.", "shortdesc": "Template hook", "type": "string" } }, { "volatile.base_image": { "longdesc": "The hash of the image that the instance was created from (empty if the instance was not created from an image).", "shortdesc": "Hash of the base image", "type": "string" } }, { "volatile.cloud_init.instance-id": { "longdesc": "", "shortdesc": "`instance-id` (UUID) exposed to `cloud-init`", "type": "string" } }, { "volatile.cluster.group": { "longdesc": "The cluster group(s) that the instance was restricted to at creation time.\nThis is used during re-scheduling events like an evacuation to keep the instance within the requested set.", "shortdesc": "The original cluster group for the instance", "type": "string" } }, { "volatile.container.oci": { "defaultdesc": "`false`", "longdesc": "", "shortdesc": "Whether the container is an OCI application container", "type": "bool" } }, { "volatile.cpu.nodes": { "longdesc": "The NUMA node that was selected for the instance.", "shortdesc": "Instance NUMA node", "type": "string" } }, { "volatile.evacuate.origin": { "longdesc": "The cluster member that the instance lived on before evacuation.", "shortdesc": "The origin of the evacuated instance", "type": "string" } }, { "volatile.idmap.base": { "longdesc": "", "shortdesc": "The first ID in the instance's primary idmap range", "type": "integer" } }, { "volatile.idmap.current": { "longdesc": "", "shortdesc": "The idmap currently in use by the instance", "type": "string" } }, { "volatile.idmap.next": { "longdesc": "", "shortdesc": "The idmap to use the next time the instance starts", "type": "string" } }, { "volatile.last_state.idmap": { "longdesc": "", "shortdesc": "Serialized instance UID/GID map", "type": "string" } }, { "volatile.last_state.power": { "longdesc": "", "shortdesc": "Instance state as of last host shutdown", "type": "string" } }, { "volatile.last_state.ready": { "longdesc": "", "shortdesc": "Instance marked itself as ready", "type": "string" } }, { "volatile.rebalance.last_move": { "longdesc": "", "shortdesc": "Timestamp of last move by automatic live-migration", "type": "integer" } }, { "volatile.uuid": { "longdesc": "The instance UUID is globally unique across all servers and projects.", "shortdesc": "Instance UUID", "type": "string" } }, { "volatile.uuid.generation": { "longdesc": "The instance generation UUID changes whenever the instance's place in time moves backwards.\nIt is globally unique across all servers and projects.", "shortdesc": "Instance generation UUID", "type": "string" } }, { "volatile.vm.boot_state": { "longdesc": "", "shortdesc": "JSON encoded VM properties used during live migration and other state restoration.", "type": "string" } }, { "volatile.vm.needs_reset": { "longdesc": "", "shortdesc": "Indicates that the VM needs a full reset on next reboot", "type": "bool" } }, { "volatile.vm.rtc_adjustment": { "longdesc": "Real Time Clock adjustment time to allow virtual machines to run on a different base than the host.", "shortdesc": "Real Time Clock change adjustment", "type": "int64" } }, { "volatile.vm.rtc_offset": { "longdesc": "Real Time Clock offset to allow virtual machines to run on a different base than the host.", "shortdesc": "Real Time Clock change offset", "type": "int64" } }, { "volatile.vsock_id": { "longdesc": "", "shortdesc": "Instance `vsock ID` used as of last start", "type": "string" } } ] } }, "kernel": { "limits": { "keys": [ { "limits.kernel.as": { "longdesc": "", "resource": "`RLIMIT_AS`", "shortdesc": "Maximum size of the process's virtual memory", "type": "string" } }, { "limits.kernel.core": { "longdesc": "", "resource": "`RLIMIT_CORE`", "shortdesc": "Maximum size of the process's core dump file", "type": "string" } }, { "limits.kernel.cpu": { "longdesc": "", "resource": "`RLIMIT_CPU`", "shortdesc": "Limit in seconds on the amount of CPU time the process can consume", "type": "string" } }, { "limits.kernel.data": { "longdesc": "", "resource": "`RLIMIT_DATA`", "shortdesc": "Maximum size of the process's data segment", "type": "string" } }, { "limits.kernel.fsize": { "longdesc": "", "resource": "`RLIMIT_FSIZE`", "shortdesc": "Maximum size of files the process may create", "type": "string" } }, { "limits.kernel.locks": { "longdesc": "", "resource": "`RLIMIT_LOCKS`", "shortdesc": "Limit on the number of file locks that this process may establish", "type": "string" } }, { "limits.kernel.memlock": { "longdesc": "", "resource": "`RLIMIT_MEMLOCK`", "shortdesc": "Limit on the number of bytes of memory that the process may lock in RAM", "type": "string" } }, { "limits.kernel.nice": { "longdesc": "", "resource": "`RLIMIT_NICE`", "shortdesc": "Maximum value to which the process's nice value can be raised", "type": "string" } }, { "limits.kernel.nofile": { "longdesc": "", "resource": "`RLIMIT_NOFILE`", "shortdesc": "Maximum number of open files for the process", "type": "string" } }, { "limits.kernel.nproc": { "longdesc": "", "resource": "`RLIMIT_NPROC`", "shortdesc": "Maximum number of processes that can be created for the user of the calling process", "type": "string" } }, { "limits.kernel.rtprio": { "longdesc": "", "resource": "`RLIMIT_RTPRIO`", "shortdesc": "Maximum value on the real-time-priority that may be set for this process", "type": "string" } }, { "limits.kernel.sigpending": { "longdesc": "", "resource": "`RLIMIT_SIGPENDING`", "shortdesc": "Limit on the number of bytes of memory that the process may lock in RAM", "type": "string" } } ] } }, "network_address_set": { "common": { "keys": [ { "user.*": { "longdesc": "User keys can be used in search.", "shortdesc": "Free form user key/value storage", "type": "string" } } ] } }, "network_bridge": { "bgp": { "keys": [ { "bgp.peers.NAME.address": { "condition": "BGP server", "defaultdesc": "-", "longdesc": "", "shortdesc": "Peer address (IPv4 or IPv6) for use by `ovn` downstream networks", "type": "string" } }, { "bgp.peers.NAME.asn": { "condition": "BGP server", "defaultdesc": "-", "longdesc": "", "shortdesc": "Peer AS number for use by `ovn` downstream networks", "type": "integer" } }, { "bgp.peers.NAME.holdtime": { "condition": "BGP server", "defaultdesc": "`180`", "longdesc": "", "shortdesc": "Peer session hold time (in seconds; optional)", "type": "integer" } }, { "bgp.peers.NAME.password": { "condition": "BGP server", "defaultdesc": "- (no password)", "longdesc": "", "shortdesc": "Peer session password (optional) for use by `ovn` downstream networks", "type": "string" } } ] }, "common": { "keys": [ { "bgp.ipv4.nexthop": { "condition": "BGP server", "default": "local address", "longdesc": "", "shortdesc": "Override the next-hop for advertised prefixes", "type": "string" } }, { "bgp.ipv6.nexthop": { "condition": "BGP server", "default": "local address", "longdesc": "", "shortdesc": "Override the next-hop for advertised prefixes", "type": "string" } }, { "bridge.driver": { "condition": "-", "default": "`native`", "longdesc": "", "shortdesc": "Bridge driver: `native` or `openvswitch`", "type": "string" } }, { "bridge.external_interfaces": { "condition": "-", "default": "-", "longdesc": "", "shortdesc": "Comma-separated list of unconfigured network interfaces to include in the bridge", "type": "string" } }, { "bridge.hwaddr": { "condition": "-", "default": "-", "longdesc": "", "shortdesc": "MAC address for the bridge", "type": "string" } }, { "bridge.mtu": { "condition": "-", "default": "`1500`", "longdesc": "", "shortdesc": "Bridge MTU (default varies if tunnel in use)", "type": "integer" } }, { "dns.domain": { "condition": "-", "default": "`incus`", "longdesc": "", "shortdesc": "Domain to advertise to DHCP clients and use for DNS resolution", "type": "string" } }, { "dns.mode": { "condition": "-", "default": "`managed`", "longdesc": "", "shortdesc": "DNS registration mode: none for no DNS record, managed for Incus-generated static records or dynamic for client-generated records", "type": "string" } }, { "dns.nameservers": { "condition": "-", "default": "IPv4 and IPv6 address", "longdesc": "", "shortdesc": "DNS server IPs to advertise to DHCP clients and via Router Advertisements. Both IPv4 and IPv6 addresses get pushed via DHCP, and IPv6 addresses are also advertised as RDNSS via RA.", "type": "string" } }, { "dns.search": { "condition": "-", "default": "-", "longdesc": "", "shortdesc": "Full comma-separated domain search list, defaulting to `dns.domain` value", "type": "string" } }, { "dns.zone.forward": { "condition": "-", "default": "`managed`", "longdesc": "", "shortdesc": "Comma-separated list of DNS zone names for forward DNS records", "type": "string" } }, { "dns.zone.reverse.ipv4": { "condition": "-", "default": "`managed`", "longdesc": "", "shortdesc": "DNS zone name for IPv4 reverse DNS records", "type": "string" } }, { "dns.zone.reverse.ipv6": { "condition": "-", "default": "`managed`", "longdesc": "", "shortdesc": "DNS zone name for IPv6 reverse DNS records", "type": "string" } }, { "ipv4.address": { "condition": "standard mode", "default": "- (initial value on creation: `auto`)", "longdesc": "", "shortdesc": "IPv4 address for the bridge (use `none` to turn off IPv4 or `auto` to generate a new random unused subnet) (CIDR)", "type": "string" } }, { "ipv4.dhcp": { "condition": "IPv4 address", "default": "`true`", "longdesc": "", "shortdesc": "Whether to allocate addresses using DHCP", "type": "bool" } }, { "ipv4.dhcp.expiry": { "condition": "IPv4 DHCP", "default": "`1h`", "longdesc": "", "shortdesc": "When to expire DHCP leases", "type": "string" } }, { "ipv4.dhcp.gateway": { "condition": "IPv4 DHCP", "default": "IPv4 address", "longdesc": "", "shortdesc": "Address of the gateway for the subnet (use `none` to turn off gateway announcement)", "type": "string" } }, { "ipv4.dhcp.ranges": { "condition": "IPv4 DHCP", "default": "all addresses", "longdesc": "", "shortdesc": "Comma-separated list of IP ranges to use for DHCP (FIRST-LAST format)", "type": "string" } }, { "ipv4.dhcp.routes": { "condition": "IPv4 DHCP", "default": "-", "longdesc": "", "shortdesc": "Static routes to provide via DHCP option 121, as a comma-separated list of alternating subnets (CIDR) and gateway addresses (same syntax as dnsmasq)", "type": "string" } }, { "ipv4.firewall": { "condition": "IPv4 address", "default": "`true`", "longdesc": "", "shortdesc": "Whether to generate filtering firewall rules for this network", "type": "bool" } }, { "ipv4.nat": { "condition": "IPv4 address", "default": "`false`(initial value on creation if `ipv4.address` is set to `auto`: `true`)", "longdesc": "", "shortdesc": "Whether to NAT", "type": "bool" } }, { "ipv4.nat.address": { "condition": "IPv4 address", "default": "-", "longdesc": "", "shortdesc": "The source address used for outbound traffic from the bridge", "type": "string" } }, { "ipv4.nat.order": { "condition": "IPv4 address", "default": "`before`", "longdesc": "", "shortdesc": "Whether to add the required NAT rules before or after any pre-existing rules", "type": "string" } }, { "ipv4.ovn.ranges": { "condition": "-", "default": "-", "longdesc": "", "shortdesc": "Comma-separated list of IPv4 ranges to use for child OVN network routers (FIRST-LAST format)", "type": "string" } }, { "ipv4.routes": { "condition": "IPv4 address", "default": "-", "longdesc": "", "shortdesc": "Comma-separated list of additional IPv4 CIDR subnets to route to the bridge", "type": "string" } }, { "ipv4.routing": { "condition": "IPv4 DHCP", "default": "`true`", "longdesc": "", "shortdesc": "Whether to route traffic in and out of the bridge", "type": "bool" } }, { "ipv6.address": { "condition": "standard mode", "default": "- (initial value on creation: `auto`)", "longdesc": "", "shortdesc": "IPv6 address for the bridge (use `none` to turn off IPv6 or `auto` to generate a new random unused subnet) (CIDR)", "type": "string" } }, { "ipv6.dhcp": { "condition": "IPv6 DHCP", "default": "`true`", "longdesc": "", "shortdesc": "Whether to provide additional network configuration over DHCP", "type": "bool" } }, { "ipv6.dhcp.expiry": { "condition": "IPv6 DHCP", "default": "`1h`", "longdesc": "", "shortdesc": "When to expire DHCP leases", "type": "string" } }, { "ipv6.dhcp.ranges": { "condition": "IPv6 stateful DHCP", "default": "all addresses", "longdesc": "", "shortdesc": "Comma-separated list of IPv6 ranges to use for DHCP (FIRST-LAST format)", "type": "string" } }, { "ipv6.dhcp.stateful": { "condition": "IPv6 DHCP", "default": "`false`", "longdesc": "", "shortdesc": "Whether to allocate addresses using DHCP", "type": "bool" } }, { "ipv6.firewall": { "condition": "IPv6 address", "default": "`true`", "longdesc": "", "shortdesc": "Whether to generate filtering firewall rules for this network", "type": "bool" } }, { "ipv6.nat": { "condition": "IPv6 address", "default": "`false` (initial value on creation if `ipv6.address` is set to `auto`: `true`)", "longdesc": "", "shortdesc": "Whether to NAT", "type": "bool" } }, { "ipv6.nat.address": { "condition": "IPv6 address", "default": "-", "longdesc": "", "shortdesc": "The source address used for outbound traffic from the bridge", "type": "string" } }, { "ipv6.nat.order": { "condition": "IPv6 address", "default": "`before`", "longdesc": "", "shortdesc": "Whether to add the required NAT rules before or after any pre-existing rules", "type": "string" } }, { "ipv6.ovn.ranges": { "condition": "-", "default": "-", "longdesc": "", "shortdesc": "Comma-separated list of IPv6 ranges to use for child OVN network routers (FIRST-LAST format)", "type": "string" } }, { "ipv6.routes": { "condition": "IPv6 address", "default": "-", "longdesc": "", "shortdesc": "Comma-separated list of additional IPv6 CIDR subnets to route to the bridge", "type": "string" } }, { "ipv6.routing": { "condition": "IPv6 address", "default": "`true`", "longdesc": "", "shortdesc": "Whether to route traffic in and out of the bridge", "type": "bool" } }, { "raw.dnsmasq": { "condition": "-", "default": "-", "longdesc": "", "shortdesc": "Additional dnsmasq configuration to append to the configuration file", "type": "string" } }, { "security.acls": { "condition": "-", "default": "-", "longdesc": "", "shortdesc": "Comma-separated list of Network ACLs to apply to NICs connected to this network (see {ref}`network-acls-bridge-limitations`)", "type": "string" } }, { "security.acls.default.egress.action": { "condition": "`security.acls`", "default": "`reject`", "longdesc": "", "shortdesc": "Action to use for egress traffic that doesn't match any ACL rule", "type": "string" } }, { "security.acls.default.egress.logged": { "condition": "`security.acls`", "default": "`false`", "longdesc": "", "shortdesc": "Whether to log egress traffic that doesn't match any ACL rule", "type": "bool" } }, { "security.acls.default.ingress.action": { "condition": "`security.acls`", "default": "`reject`", "longdesc": "", "shortdesc": "Action to use for ingress traffic that doesn't match any ACL rule", "type": "string" } }, { "security.acls.default.ingress.logged": { "condition": "`security.acls`", "default": "`false`", "longdesc": "", "shortdesc": "Whether to log ingress traffic that doesn't match any ACL rule", "type": "bool" } }, { "tunnel.NAME.group": { "condition": "`vxlan`", "default": "`239.0.0.1`", "longdesc": "", "shortdesc": "Multicast address for `vxlan` (used if local and remote aren't set)", "type": "string" } }, { "tunnel.NAME.id": { "condition": "`vxlan`", "default": "`0`", "longdesc": "", "shortdesc": "Specific tunnel ID to use for the `vxlan` tunnel", "type": "integer" } }, { "tunnel.NAME.interface": { "condition": "`vxlan`", "default": "-", "longdesc": "", "shortdesc": "Specific host interface to use for the tunnel", "type": "string" } }, { "tunnel.NAME.local": { "condition": "`gre` or `vxlan`", "default": "-", "longdesc": "", "shortdesc": "Local address for the tunnel (not necessary for multicast `vxlan`)", "type": "string" } }, { "tunnel.NAME.port": { "condition": "`vxlan`", "default": "`0`", "longdesc": "", "shortdesc": "Specific port to use for the `vxlan` tunnel", "type": "integer" } }, { "tunnel.NAME.protocol": { "condition": "standard mode", "default": "-", "longdesc": "", "shortdesc": "Tunneling protocol: `vxlan` or `gre`", "type": "string" } }, { "tunnel.NAME.remote": { "condition": "`gre` or `vxlan`", "default": "-", "longdesc": "", "shortdesc": "Remote address for the tunnel (not necessary for multicast `vxlan`)", "type": "string" } }, { "tunnel.NAME.ttl": { "condition": "`vxlan`", "default": "`1`", "longdesc": "", "shortdesc": "Specific TTL to use for multicast routing topologies", "type": "integer" } }, { "user.*": { "condition": "-", "default": "-", "longdesc": "", "shortdesc": "User-provided free-form key/value pairs", "type": "string" } } ] } }, "network_forward": { "common": { "keys": [ { "target_address": { "longdesc": "", "shortdesc": "Default target address for anything not covered through a port definition", "type": "string" } }, { "user.*": { "longdesc": "", "shortdesc": "User defined key/value configuration", "type": "string" } } ] } }, "network_integration": { "common": { "keys": [ { "user.*": { "longdesc": "User keys can be used in search.", "shortdesc": "Free form user key/value storage", "type": "string" } } ] }, "ovn": { "keys": [ { "ovn.ca_cert": { "longdesc": "", "scope": "global", "shortdesc": "OVN SSL certificate authority for the inter-connection database", "type": "string" } }, { "ovn.client_cert": { "longdesc": "", "scope": "global", "shortdesc": "OVN SSL client certificate", "type": "string" } }, { "ovn.client_key": { "longdesc": "", "scope": "global", "shortdesc": "OVN SSL client key", "type": "string" } }, { "ovn.northbound_connection": { "longdesc": "", "scope": "global", "shortdesc": "OVN northbound inter-connection connection string", "type": "string" } }, { "ovn.southbound_connection": { "longdesc": "", "scope": "global", "shortdesc": "OVN southbound inter-connection connection string", "type": "string" } }, { "ovn.transit.pattern": { "defaultdesc": "`ts-incus-{{ integrationName }}-{{ projectName }}-{{ networkName }}`", "longdesc": "Specify a Pongo2 template string that represents the transit switch name.\nThis template gets access to the project name (`projectName`),\nintegration name (`integrationName`), network name (`networkName`)\nand peer name (`peerName`).\n", "shortdesc": "Template for the transit switch name", "type": "string" } } ] } }, "network_load_balancer": { "common": { "keys": [ { "healthcheck": { "defaultdesc": "`false`", "longdesc": "", "shortdesc": "Whether to perform checks on the backends", "type": "bool" } }, { "healthcheck.failure_count": { "defaultdesc": "`3`", "longdesc": "", "shortdesc": "Number of failed tests to consider the backend offline", "type": "integer" } }, { "healthcheck.interval": { "defaultdesc": "`10`", "longdesc": "", "shortdesc": "Interval in seconds between health checks", "type": "integer" } }, { "healthcheck.success_count": { "defaultdesc": "`3`", "longdesc": "", "shortdesc": "Number of successful tests to consider the backend online", "type": "integer" } }, { "healthcheck.timeout": { "defaultdesc": "`30`", "longdesc": "", "shortdesc": "Test timeout", "type": "integer" } }, { "user.*": { "longdesc": "User keys can be used in search.", "shortdesc": "Free form user key/value storage", "type": "string" } } ] } }, "network_macvlan": { "common": { "keys": [ { "gvrp": { "condition": "-", "default": "`false`", "longdesc": "", "shortdesc": "Register VLAN using GARP VLAN Registration Protocol", "type": "bool" } }, { "mtu": { "condition": "-", "longdesc": "", "shortdesc": "The MTU of the new interface", "type": "int" } }, { "parent": { "condition": "-", "longdesc": "", "shortdesc": "Parent interface to create macvlan NICs on", "type": "string" } }, { "user.*": { "longdesc": "", "shortdesc": "User-provided free-form key/value pairs", "type": "string" } }, { "vlan": { "condition": "-", "longdesc": "", "shortdesc": "The VLAN ID to attach to", "type": "int" } } ] } }, "network_ovn": { "common": { "keys": [ { "bridge.external_interfaces": { "longdesc": "", "shortdesc": "Comma-separated list of unconfigured network interfaces to include in the bridge", "type": "string" } }, { "bridge.hwaddr": { "longdesc": "", "shortdesc": "MAC address for the virtual bridge interface", "type": "string" } }, { "bridge.mtu": { "default": "`1442`", "longdesc": "", "shortdesc": "Bridge MTU (default allows host to host Geneve tunnels)", "type": "integer" } }, { "dns.domain": { "default": "`incus`", "longdesc": "", "shortdesc": "Domain to advertise to DHCP clients and use for DNS resolution", "type": "string" } }, { "dns.mode": { "condition": "-", "default": "`managed`", "longdesc": "", "shortdesc": "DNS registration mode: none for no DNS record, managed for OVN managed records", "type": "string" } }, { "dns.nameservers": { "default": "Uplink DNS servers (IPv4 and IPv6 address if no uplink is configured)", "longdesc": "", "shortdesc": "DNS server IPs to advertise to DHCP clients and via Router Advertisements. Both IPv4 and IPv6 addresses get pushed via DHCP, and the first IPv6 address is also advertised as RDNSS via RA.", "type": "string" } }, { "dns.search": { "longdesc": "", "shortdesc": "Full comma-separated domain search list, defaulting to `dns.domain` value", "type": "string" } }, { "dns.zone.forward": { "longdesc": "", "shortdesc": "Comma-separated list of DNS zone names for forward DNS records", "type": "string" } }, { "dns.zone.reverse.ipv4": { "longdesc": "", "shortdesc": "DNS zone name for IPv4 reverse DNS records", "type": "string" } }, { "dns.zone.reverse.ipv6": { "longdesc": "", "shortdesc": "DNS zone name for IPv6 reverse DNS records", "type": "string" } }, { "ipv4.address": { "condition": "standard mode", "default": "(initial value on creation: `auto`)", "longdesc": "", "shortdesc": "IPv4 address for the bridge (use `none` to turn off IPv4 or `auto` to generate a new random unused subnet) (CIDR)", "type": "string" } }, { "ipv4.dhcp": { "condition": "IPv4 address", "default": "`true`", "longdesc": "", "shortdesc": "Whether to allocate addresses using DHCP", "type": "bool" } }, { "ipv4.dhcp.expiry": { "condition": "IPv4 DHCP", "default": "`1h`", "longdesc": "", "shortdesc": "When to expire DHCP leases", "type": "string" } }, { "ipv4.dhcp.gateway": { "condition": "IPv4 DHCP", "default": "IPv4 address", "longdesc": "", "shortdesc": "Address of the gateway for the subnet (use `none` to turn off gateway announcement)", "type": "string" } }, { "ipv4.dhcp.ranges": { "condition": "IPv4 DHCP", "default": "all addresses", "longdesc": "", "shortdesc": "Comma-separated list of IP ranges to use for DHCP (FIRST-LAST format)", "type": "string" } }, { "ipv4.dhcp.routes": { "condition": "IPv4 DHCP", "longdesc": "", "shortdesc": "Static routes to provide via DHCP option 121, as a comma-separated list of alternating subnets (CIDR) and gateway addresses (same syntax as dnsmasq and OVN)", "type": "string" } }, { "ipv4.l3only": { "condition": "IPv4 address", "default": "`false`", "longdesc": "", "shortdesc": "Whether to enable layer 3 only mode.", "type": "bool" } }, { "ipv4.nat": { "condition": "IPv4 address", "default": "`false` initial value on creation if `ipv4.address` is set to `auto: true`)", "longdesc": "", "shortdesc": "Whether to NAT", "type": "bool" } }, { "ipv4.nat.address": { "condition": "IPv4 address", "longdesc": "", "shortdesc": "The source address used for outbound traffic from the network (requires uplink `ovn.ingress_mode=routed`)", "type": "string" } }, { "ipv6.address": { "condition": "standard mode", "default": "(initial value on creation: `auto`)", "longdesc": "", "shortdesc": "IPv6 address for the bridge (use `none` to turn off IPv6 or `auto` to generate a new random unused subnet) (CIDR)", "type": "string" } }, { "ipv6.dhcp": { "condition": "IPv6 address", "default": "`true`", "longdesc": "", "shortdesc": "Whether to provide additional network configuration over DHCP", "type": "bool" } }, { "ipv6.dhcp.stateful": { "condition": "IPv6 DHCP", "default": "`false`", "longdesc": "", "shortdesc": "Whether to allocate addresses using DHCP", "type": "bool" } }, { "ipv6.l3only": { "condition": "IPv6 DHCP stateful", "default": "`false`", "longdesc": "", "shortdesc": "Whether to enable layer 3 only mode.", "type": "bool" } }, { "ipv6.nat": { "condition": "IPv6 address", "default": "`false` (initial value on creation if `ipv6.address` is set to `auto: true`)", "longdesc": "", "shortdesc": "Whether to NAT", "type": "bool" } }, { "ipv6.nat.address": { "condition": "IPv6 address", "longdesc": "", "shortdesc": "The source address used for outbound traffic from the network (requires uplink `ovn.ingress_mode=routed`)", "type": "string" } }, { "network": { "longdesc": "", "shortdesc": "Uplink network to use for external network access or `none` to keep isolated", "type": "string" } }, { "security.acls": { "longdesc": "", "shortdesc": "Comma-separated list of Network ACLs to apply to NICs connected to this network", "type": "string" } }, { "security.acls.default.egress.action": { "condition": "`security.acls`", "default": "`reject`", "longdesc": "", "shortdesc": "Action to use for egress traffic that doesn't match any ACL rule", "type": "string" } }, { "security.acls.default.egress.logged": { "condition": "`security.acls`", "default": "`false`", "longdesc": "", "shortdesc": "Whether to log egress traffic that doesn't match any ACL rule", "type": "bool" } }, { "security.acls.default.ingress.action": { "condition": "`security.acls`", "default": "`reject`", "longdesc": "", "shortdesc": "Action to use for ingress traffic that doesn't match any ACL rule", "type": "string" } }, { "security.acls.default.ingress.logged": { "condition": "`security.acls`", "default": "`false`", "longdesc": "", "shortdesc": "Whether to log ingress traffic that doesn't match any ACL rule", "type": "bool" } }, { "tunnel.NAME.group": { "condition": "`vxlan`", "default": "`239.0.0.1`", "longdesc": "", "shortdesc": "Multicast address for `vxlan`", "type": "string" } }, { "tunnel.NAME.id": { "condition": "`vxlan`", "default": "`0`", "longdesc": "", "shortdesc": "Specific tunnel ID to use for the `vxlan` tunnel", "type": "integer" } }, { "tunnel.NAME.interface": { "condition": "`vxlan`", "default": "-", "longdesc": "", "shortdesc": "Specific host interface to use for the tunnel", "type": "string" } }, { "tunnel.NAME.local": { "condition": "`gre`", "default": "-", "longdesc": "", "shortdesc": "Local address for the tunnel", "type": "string" } }, { "tunnel.NAME.port": { "condition": "`vxlan`", "default": "`0`", "longdesc": "", "shortdesc": "Specific port to use for the `vxlan` tunnel", "type": "integer" } }, { "tunnel.NAME.protocol": { "condition": "standard mode", "default": "-", "longdesc": "", "shortdesc": "Tunneling protocol: `vxlan` or `gre`", "type": "string" } }, { "tunnel.NAME.remote": { "condition": "`gre`", "default": "-", "longdesc": "", "shortdesc": "Remote address for the tunnel", "type": "string" } }, { "tunnel.NAME.ttl": { "condition": "`vxlan`", "default": "`1`", "longdesc": "", "shortdesc": "Specific TTL to use for multicast routing topologies", "type": "integer" } }, { "user.*": { "longdesc": "", "shortdesc": "User-provided free-form key/value pairs", "type": "string" } } ] } }, "network_physical": { "bgp": { "keys": [ { "bgp.peers.NAME.address": { "condition": "BGP server", "defaultdesc": "-", "longdesc": "", "shortdesc": "Peer address (IPv4 or IPv6) for use by `ovn` downstream networks", "type": "string" } }, { "bgp.peers.NAME.asn": { "condition": "BGP server", "defaultdesc": "-", "longdesc": "", "shortdesc": "Peer AS number for use by `ovn` downstream networks", "type": "integer" } }, { "bgp.peers.NAME.holdtime": { "condition": "BGP server", "defaultdesc": "`180`", "longdesc": "", "shortdesc": "Peer session hold time (in seconds; optional)", "type": "integer" } }, { "bgp.peers.NAME.password": { "condition": "BGP server", "defaultdesc": "- (no password)", "longdesc": "", "shortdesc": "Peer session password (optional) for use by `ovn` downstream networks", "type": "string" } } ] }, "common": { "keys": [ { "gvrp": { "condition": "-", "defaultdesc": "'false'", "longdesc": "", "shortdesc": "Register VLAN using GARP VLAN Registration Protocol", "type": "bool" } }, { "mtu": { "condition": "-", "longdesc": "", "shortdesc": "The MTU of the new interface", "type": "integer" } }, { "parent": { "condition": "-", "longdesc": "", "shortdesc": "Existing interface to use for network", "type": "string" } }, { "vlan": { "condition": "-", "longdesc": "", "shortdesc": "The VLAN ID to attach to", "type": "integer" } }, { "vlan.tagged": { "condition": "Parent must be an existing bridge", "longdesc": "", "shortdesc": "Comma-delimited list of VLAN IDs or VLAN ranges to join for tagged traffic", "type": "integer" } } ] }, "dns": { "keys": [ { "dns.nameservers": { "condition": "standard mode", "longdesc": "", "shortdesc": "List of DNS server IPs on `physical` network", "type": "string" } } ] }, "ipv4": { "keys": [ { "ipv4.gateway": { "condition": "standard mode", "longdesc": "", "shortdesc": "IPv4 address for the gateway and network (CIDR)", "type": "string" } }, { "ipv4.gateway.hwaddr": { "longdesc": "", "shortdesc": "MAC address of the gateway (to avoid discovery)", "type": "string" } }, { "ipv4.ovn.ranges": { "condition": "-", "longdesc": "", "shortdesc": "Comma-separated list of IPv4 ranges to use for child OVN network routers (FIRST-LAST format)", "type": "string" } }, { "ipv4.routes": { "condition": "IPv4 address", "longdesc": "", "shortdesc": "Comma-separated list of additional IPv4 CIDR subnets that can be used with child OVN networks `ipv4.routes.external` setting", "type": "string" } }, { "ipv4.routes.anycast": { "condition": "IPv4 address", "defaultdesc": "'false'", "longdesc": "", "shortdesc": "Allow the overlapping routes to be used on multiple networks/NIC at the same time", "type": "bool" } } ] }, "ipv6": { "keys": [ { "ipv6.gateway": { "condition": "standard mode", "longdesc": "", "shortdesc": "IPv6 address for the gateway and network (CIDR)", "type": "string" } }, { "ipv6.gateway.hwaddr": { "longdesc": "", "shortdesc": "MAC address of the gateway (to avoid discovery)", "type": "string" } }, { "ipv6.ovn.ranges": { "condition": "-", "longdesc": "", "shortdesc": "Comma-separated list of IPv6 ranges to use for child OVN network routers (FIRST-LAST format)", "type": "string" } }, { "ipv6.routes": { "condition": "IPv6 address", "longdesc": "", "shortdesc": "Comma-separated list of additional IPv6 CIDR subnets that can be used with child OVN networks `ipv6.routes.external` setting", "type": "string" } }, { "ipv6.routes.anycast": { "condition": "IPv6 address", "defaultdesc": "'false'", "longdesc": "", "shortdesc": "Allow the overlapping routes to be used on multiple networks/NIC at the same time", "type": "bool" } } ] }, "ovn": { "keys": [ { "ovn.ingress_mode": { "condition": "standard mode", "defaultdesc": "`l2proxy`", "longdesc": "", "shortdesc": "Sets the method how OVN NIC external IPs will be advertised on uplink network: `l2proxy` (proxy ARP/NDP) or `routed`", "type": "string" } } ] } }, "network_sriov": { "common": { "keys": [ { "mtu": { "condition": "-", "longdesc": "", "shortdesc": "The MTU of the new interface", "type": "integer" } }, { "parent": { "condition": "-", "longdesc": "", "shortdesc": "Parent interface to create `sriov` NICs on", "type": "string" } }, { "user.*": { "condition": "-", "longdesc": "", "shortdesc": "User-provided free-form key/value pairs", "type": "string" } }, { "vlan": { "condition": "-", "longdesc": "", "shortdesc": "The VLAN ID to attach to", "type": "integer" } } ] } }, "network_zone": { "common": { "keys": [ { "dns.contact": { "longdesc": "", "required": "no", "shortdesc": "Admin contact email for DNS server", "type": "string" } }, { "dns.nameservers": { "longdesc": "", "required": "no", "shortdesc": "Comma-separated list of DNS server FQDNs (for NS records)", "type": "string set" } }, { "network.nat": { "defaultdesc": "`true`", "longdesc": "", "required": "no", "shortdesc": "Whether to generate records for NAT-ed subnets", "type": "bool" } }, { "peers.NAME.address": { "longdesc": "", "required": "no", "shortdesc": "IP address of a DNS server", "type": "string" } }, { "peers.NAME.key": { "longdesc": "", "required": "no", "shortdesc": "TSIG key for the server", "type": "string" } }, { "user.*": { "longdesc": "", "required": "no", "shortdesc": "User-provided free-form key/value pairs", "type": "string" } } ] } }, "project": { "features": { "keys": [ { "features.images": { "defaultdesc": "`false`", "initialvaluedesc": "`true`", "longdesc": "This setting applies to both images and image aliases.", "shortdesc": "Whether to use a separate set of images for the project", "type": "bool" } }, { "features.networks": { "defaultdesc": "`false`", "initialvaluedesc": "`false`", "longdesc": "", "shortdesc": "Whether to use a separate set of networks for the project", "type": "bool" } }, { "features.networks.zones": { "defaultdesc": "`false`", "initialvaluedesc": "`false`", "longdesc": "", "shortdesc": "Whether to use a separate set of network zones for the project", "type": "bool" } }, { "features.profiles": { "defaultdesc": "`false`", "initialvaluedesc": "`true`", "longdesc": "", "shortdesc": "Whether to use a separate set of profiles for the project", "type": "bool" } }, { "features.storage.buckets": { "defaultdesc": "`false`", "initialvaluedesc": "`true`", "longdesc": "", "shortdesc": "Whether to use a separate set of storage buckets for the project", "type": "bool" } }, { "features.storage.volumes": { "defaultdesc": "`false`", "initialvaluedesc": "`true`", "longdesc": "", "shortdesc": "Whether to use a separate set of storage volumes for the project", "type": "bool" } } ] }, "limits": { "keys": [ { "limits.containers": { "longdesc": "", "shortdesc": "Maximum number of containers that can be created in the project", "type": "integer" } }, { "limits.cpu": { "longdesc": "This value is the maximum value for the sum of the individual {config:option}`instance-resource-limits:limits.cpu` configurations set on the instances of the project.", "shortdesc": "Maximum number of CPUs to use in the project", "type": "integer" } }, { "limits.disk": { "longdesc": "This value is the maximum value of the aggregate disk space used by all instance volumes, custom volumes, and images of the project.", "shortdesc": "Maximum disk space used by the project", "type": "string" } }, { "limits.disk.pool.POOL_NAME": { "longdesc": "This value is the maximum value of the aggregate disk\nspace used by all instance volumes, custom volumes, and images of the\nproject on this specific storage pool.", "shortdesc": "Maximum disk space used by the project on this pool", "type": "string" } }, { "limits.instances": { "longdesc": "", "shortdesc": "Maximum number of instances that can be created in the project", "type": "integer" } }, { "limits.memory": { "longdesc": "The value is the maximum value for the sum of the individual {config:option}`instance-resource-limits:limits.memory` configurations set on the instances of the project.", "shortdesc": "Usage limit for the host's memory for the project", "type": "string" } }, { "limits.networks": { "longdesc": "", "shortdesc": "Maximum number of networks that the project can have", "type": "integer" } }, { "limits.processes": { "longdesc": "This value is the maximum value for the sum of the individual {config:option}`instance-resource-limits:limits.processes` configurations set on the instances of the project.", "shortdesc": "Maximum number of processes within the project", "type": "integer" } }, { "limits.virtual-machines": { "longdesc": "", "shortdesc": "Maximum number of VMs that can be created in the project", "type": "integer" } } ] }, "restricted": { "keys": [ { "restricted": { "defaultdesc": "`false`", "longdesc": "This option must be enabled to allow the `restricted.*` keys to take effect.\nTo temporarily remove the restrictions, you can disable this option instead of clearing the related keys.", "shortdesc": "Whether to block access to security-sensitive features", "type": "bool" } }, { "restricted.backups": { "defaultdesc": "`block`", "longdesc": "Possible values are `allow` or `block`.", "shortdesc": "Whether to prevent creating instance or volume backups", "type": "string" } }, { "restricted.cluster.groups": { "longdesc": "If specified, this option prevents targeting cluster groups other than the provided ones.", "shortdesc": "Cluster groups that can be targeted", "type": "string" } }, { "restricted.cluster.target": { "defaultdesc": "`block`", "longdesc": "Possible values are `allow` or `block`.\nWhen set to `allow`, this option allows targeting of cluster members (either directly or via a group) when creating or moving instances.", "shortdesc": "Whether to prevent targeting of cluster members", "type": "string" } }, { "restricted.containers.interception": { "defaultdesc": "`block`", "longdesc": "Possible values are `allow`, `block`, or `full`.\nWhen set to `allow`, interception options that are usually safe are allowed.\nFile system mounting remains blocked.", "shortdesc": "Whether to prevent using system call interception options", "type": "string" } }, { "restricted.containers.lowlevel": { "defaultdesc": "`block`", "longdesc": "Possible values are `allow` or `block`.\nWhen set to `allow`, low-level container options like {config:option}`instance-raw:raw.lxc`, {config:option}`instance-raw:raw.idmap`, `volatile.*`, etc. can be used.", "shortdesc": "Whether to prevent using low-level container options", "type": "string" } }, { "restricted.containers.nesting": { "defaultdesc": "`block`", "longdesc": "Possible values are `allow` or `block`.\nWhen set to `allow`, {config:option}`instance-security:security.nesting` can be set to `true` for an instance.", "shortdesc": "Whether to prevent running nested Incus", "type": "string" } }, { "restricted.containers.privilege": { "defaultdesc": "`unprivileged`", "longdesc": "Possible values are `unprivileged`, `isolated`, and `allow`.\n\n- When set to `unprivileged`, this option prevents setting {config:option}`instance-security:security.privileged` to `true`.\n- When set to `isolated`, this option prevents setting {config:option}`instance-security:security.privileged` and {config:option}`instance-security:security.idmap.isolated` to `true`.\n- When set to `allow`, there is no restriction.", "shortdesc": "Which settings for privileged containers to prevent", "type": "string" } }, { "restricted.devices.disk": { "defaultdesc": "`managed`", "longdesc": "Possible values are `allow`, `block`, or `managed`.\n\n- When set to `block`, this option prevents using all disk devices except the root one.\n- When set to `managed`, this option allows using disk devices only if `pool=` is set.\n- When set to `allow`, there is no restriction on which disk devices can be used.", "shortdesc": "Which disk devices can be used", "type": "string" } }, { "restricted.devices.disk.paths": { "longdesc": "If {config:option}`project-restricted:restricted.devices.disk` is set to `allow`, this option controls which `source` can be used for `disk` devices.\nSpecify a comma-separated list of path prefixes that restrict the `source` setting.\nIf this option is left empty, all paths are allowed.", "shortdesc": "Which `source` can be used for `disk` devices", "type": "string" } }, { "restricted.devices.gpu": { "defaultdesc": "`block`", "longdesc": "Possible values are `allow` or `block`.", "shortdesc": "Whether to prevent using devices of type `gpu`", "type": "string" } }, { "restricted.devices.infiniband": { "defaultdesc": "`block`", "longdesc": "Possible values are `allow` or `block`.", "shortdesc": "Whether to prevent using devices of type `infiniband`", "type": "string" } }, { "restricted.devices.nic": { "defaultdesc": "`managed`", "longdesc": "Possible values are `allow`, `block`, or `managed`.\n\n- When set to `block`, this option prevents using all network devices.\n- When set to `managed`, this option allows using network devices only if `network=` is set.\n- When set to `allow`, there is no restriction on which network devices can be used.", "shortdesc": "Which network devices can be used", "type": "string" } }, { "restricted.devices.pci": { "defaultdesc": "`block`", "longdesc": "Possible values are `allow` or `block`.", "shortdesc": "Whether to prevent using devices of type `pci`", "type": "string" } }, { "restricted.devices.proxy": { "defaultdesc": "`block`", "longdesc": "Possible values are `allow` or `block`.", "shortdesc": "Whether to prevent using devices of type `proxy`", "type": "string" } }, { "restricted.devices.unix-block": { "defaultdesc": "`block`", "longdesc": "Possible values are `allow` or `block`.", "shortdesc": "Whether to prevent using devices of type `unix-block`", "type": "string" } }, { "restricted.devices.unix-char": { "defaultdesc": "`block`", "longdesc": "Possible values are `allow` or `block`.", "shortdesc": "Whether to prevent using devices of type `unix-char`", "type": "string" } }, { "restricted.devices.unix-hotplug": { "defaultdesc": "`block`", "longdesc": "Possible values are `allow` or `block`.", "shortdesc": "Whether to prevent using devices of type `unix-hotplug`", "type": "string" } }, { "restricted.devices.usb": { "defaultdesc": "`block`", "longdesc": "Possible values are `allow` or `block`.", "shortdesc": "Whether to prevent using devices of type `usb`", "type": "string" } }, { "restricted.idmap.gid": { "longdesc": "This option specifies the host GID ranges that are allowed in the instance's {config:option}`instance-raw:raw.idmap` setting.", "shortdesc": "Which host GID ranges are allowed in `raw.idmap`", "type": "string" } }, { "restricted.idmap.uid": { "longdesc": "This option specifies the host UID ranges that are allowed in the instance's {config:option}`instance-raw:raw.idmap` setting.", "shortdesc": "Which host UID ranges are allowed in `raw.idmap`", "type": "string" } }, { "restricted.images.servers": { "longdesc": "Specify a comma-delimited list of image servers domains that are allowed for use in this project.\nIf this option is not set, all image servers are accessible.", "shortdesc": "Which image servers (HTTP host) are allowed for us in this project", "type": "string" } }, { "restricted.networks.access": { "longdesc": "Specify a comma-delimited list of network names that are allowed for use in this project.\nIf this option is not set, all networks are accessible.\n\nNote that this setting depends on the {config:option}`project-restricted:restricted.devices.nic` setting.", "shortdesc": "Which network names are allowed for use in this project", "type": "string" } }, { "restricted.networks.integrations": { "longdesc": "Specify a comma-delimited list of network integrations that can be used by networks in this project.", "shortdesc": "Which network integrations can be used in this project", "type": "string" } }, { "restricted.networks.subnets": { "defaultdesc": "`block`", "longdesc": "Specify a comma-delimited list of network subnets from the uplink networks that are allocated for use in this project.\nUse the form `\u003cuplink\u003e:\u003csubnet\u003e`.", "shortdesc": "Which network subnets are allocated for use in this project", "type": "string" } }, { "restricted.networks.uplinks": { "longdesc": "Specify a comma-delimited list of network names that can be used as uplink for networks in this project.", "shortdesc": "Which network names can be used as uplink in this project", "type": "string" } }, { "restricted.networks.zones": { "defaultdesc": "`block`", "longdesc": "Specify a comma-delimited list of network zones that can be used (or something under them) in this project.", "shortdesc": "Which network zones can be used in this project", "type": "string" } }, { "restricted.snapshots": { "defaultdesc": "`block`", "longdesc": "", "shortdesc": "Whether to prevent creating instance or volume snapshots", "type": "string" } }, { "restricted.storage-pools.access": { "longdesc": "Specify a comma-delimited list of storage pool names that are allowed for use in this project.\nIf this option is not set, all storage pools are accessible.", "shortdesc": "Which storage pool names are allowed for use in this project", "type": "string" } }, { "restricted.virtual-machines.lowlevel": { "defaultdesc": "`block`", "longdesc": "Possible values are `allow` or `block`.\nWhen set to `allow`, low-level VM options like {config:option}`instance-raw:raw.qemu`, `volatile.*`, etc. can be used.", "shortdesc": "Whether to prevent using low-level VM options", "type": "string" } } ] }, "specific": { "keys": [ { "backups.compression_algorithm": { "longdesc": "Specify which compression algorithm to use for backups in this project.\nPossible values are `bzip2`, `gzip`, `lz4`, `lzma`, `xz`, `zstd` or `none`.", "shortdesc": "Compression algorithm to use for backups", "type": "string" } }, { "images.auto_update_cached": { "longdesc": "", "shortdesc": "Whether to automatically update cached images in the project", "type": "bool" } }, { "images.auto_update_interval": { "longdesc": "Specify the interval in hours.\nTo disable looking for updates to cached images, set this option to `0`.", "shortdesc": "Interval at which to look for updates to cached images", "type": "integer" } }, { "images.compression_algorithm": { "longdesc": "Possible values are `bzip2`, `gzip`, `lz4`, `lzma`, `xz`, `zstd` or `none`.", "shortdesc": "Compression algorithm to use for new images in the project", "type": "string" } }, { "images.default_architecture": { "longdesc": "", "shortdesc": "Default architecture to use in a mixed-architecture cluster", "type": "string" } }, { "images.remote_cache_expiry": { "longdesc": "Specify the number of days after which the unused cached image expires.", "shortdesc": "When an unused cached remote image is flushed in the project", "type": "integer" } }, { "network.hwaddr_pattern": { "longdesc": "Specify a MAC address template, e.g. `10:66:6a:xx:xx:xx`, to use within the cluster.\nEvery `x` in the template will be replaced by a random character in `0`–`f`.\nBeware of the birthday paradox! A single `xx` block leads to a 10% collision probability with only 8 addresses; for a double `xx:xx` block, 118 addresses; for a triple `xx:xx:xx` block, 1881; for a quadruple `xx:xx:xx:xx` block, 30084. We provide absolutely no guardrail against that.", "scope": "global", "shortdesc": "MAC address template", "type": "string" } }, { "user.*": { "longdesc": "", "shortdesc": "User-provided free-form key/value pairs", "type": "string" } } ] } }, "server": { "acme": { "keys": [ { "acme.agree_tos": { "defaultdesc": "`false`", "longdesc": "", "scope": "global", "shortdesc": "Agree to ACME terms of service", "type": "bool" } }, { "acme.ca_url": { "defaultdesc": "`https://acme-v02.api.letsencrypt.org/directory`", "longdesc": "", "scope": "global", "shortdesc": "URL to the directory resource of the ACME service", "type": "string" } }, { "acme.challenge": { "defaultdesc": "`HTTP-01`", "longdesc": "Possible values are `DNS-01` and `HTTP-01`.", "scope": "global", "shortdesc": "ACME challenge type to use", "type": "string" } }, { "acme.domain": { "longdesc": "", "scope": "global", "shortdesc": "Domain for which the certificate is issued", "type": "string" } }, { "acme.email": { "longdesc": "", "scope": "global", "shortdesc": "Email address used for the account registration", "type": "string" } }, { "acme.http.port": { "defaultdesc": "`:80`", "longdesc": "Set the port and interface to use for HTTP-01 based challenges to listen on", "scope": "global", "shortdesc": "Port and interface for HTTP server (used by HTTP-01)", "type": "string" } }, { "acme.provider": { "defaultdesc": "``", "longdesc": "", "scope": "global", "shortdesc": "Backend provider for the challenge (used by DNS-01)", "type": "string" } }, { "acme.provider.environment": { "defaultdesc": "``", "longdesc": "", "scope": "global", "shortdesc": "Environment variables to set during the challenge (used by DNS-01)", "type": "string" } }, { "acme.provider.resolvers": { "defaultdesc": "``", "longdesc": "DNS resolvers to use for performing (recursive) `CNAME` resolving and apex domain determination during DNS-01 challenge.", "scope": "global", "shortdesc": "Comma-separated list of DNS resolvers (used by DNS-01)", "type": "string" } } ] }, "cluster": { "keys": [ { "cluster.healing_threshold": { "defaultdesc": "`0`", "longdesc": "Specify the number of seconds after which an offline cluster member is to be evacuated.\nTo disable evacuating offline members, set this option to `0`.", "scope": "global", "shortdesc": "Threshold when to evacuate an offline cluster member", "type": "integer" } }, { "cluster.https_address": { "longdesc": "See {ref}`cluster-https-address`.", "scope": "local", "shortdesc": "Address to use for clustering traffic", "type": "string" } }, { "cluster.images_minimal_replica": { "defaultdesc": "`3`", "longdesc": "Specify the minimal number of cluster members that keep a copy of a particular image.\nSet this option to `1` for no replication, or to `-1` to replicate images on all members.", "scope": "global", "shortdesc": "Number of cluster members that replicate an image", "type": "integer" } }, { "cluster.join_token_expiry": { "defaultdesc": "`3H`", "longdesc": "", "scope": "global", "shortdesc": "Time after which a cluster join token expires", "type": "string" } }, { "cluster.max_standby": { "defaultdesc": "`2`", "longdesc": "Specify the maximum number of cluster members that are assigned the database stand-by role.\nThis must be a number between `0` and `5`.", "scope": "global", "shortdesc": "Number of database stand-by members", "type": "integer" } }, { "cluster.max_voters": { "defaultdesc": "`3`", "longdesc": "Specify the maximum number of cluster members that are assigned the database voter role.\nThis must be an odd number \u003e= `3`.", "scope": "global", "shortdesc": "Number of database voter members", "type": "integer" } }, { "cluster.offline_threshold": { "defaultdesc": "`20`", "longdesc": "Specify the number of seconds after which an unresponsive member is considered offline.", "scope": "global", "shortdesc": "Threshold when an unresponsive member is considered offline", "type": "integer" } }, { "cluster.rebalance.batch": { "defaultdesc": "`1`", "longdesc": "", "scope": "global", "shortdesc": "Maximum number of instances to move during one re-balancing run", "type": "integer" } }, { "cluster.rebalance.cooldown": { "defaultdesc": "`6H`", "longdesc": "", "scope": "global", "shortdesc": "Amount of time during which an instance will not be moved again", "type": "string" } }, { "cluster.rebalance.interval": { "defaultdesc": "`0`", "longdesc": "", "scope": "global", "shortdesc": "How often (in minutes) to consider re-balancing things. 0 to disable (default)", "type": "integer" } }, { "cluster.rebalance.threshold": { "defaultdesc": "`20`", "longdesc": "", "scope": "global", "shortdesc": "Percentage load difference between most and least busy server needed to trigger a migration", "type": "integer" } } ] }, "core": { "keys": [ { "core.bgp_address": { "longdesc": "See {ref}`network-bgp`.", "scope": "local", "shortdesc": "Address to bind the BGP server to", "type": "string" } }, { "core.bgp_asn": { "longdesc": "", "scope": "global", "shortdesc": "BGP Autonomous System Number for the local server", "type": "string" } }, { "core.bgp_routerid": { "longdesc": "The identifier must be formatted as an IPv4 address.", "scope": "local", "shortdesc": "A unique identifier for the BGP server", "type": "string" } }, { "core.debug_address": { "longdesc": "", "scope": "local", "shortdesc": "Address to bind the `pprof` debug server to (HTTP)", "type": "string" } }, { "core.dns_address": { "longdesc": "See {ref}`network-dns-server`.", "scope": "local", "shortdesc": "Address to bind the authoritative DNS server to", "type": "string" } }, { "core.https_address": { "longdesc": "See {ref}`server-expose`.", "scope": "local", "shortdesc": "Address to bind for the remote API (HTTPS)", "type": "string" } }, { "core.https_allowed_credentials": { "defaultdesc": "`false`", "longdesc": "If enabled, the `Access-Control-Allow-Credentials` HTTP header value is set to `true`.", "scope": "global", "shortdesc": "Whether to set `Access-Control-Allow-Credentials`", "type": "bool" } }, { "core.https_allowed_headers": { "longdesc": "", "scope": "global", "shortdesc": "`Access-Control-Allow-Headers` HTTP header value", "type": "string" } }, { "core.https_allowed_methods": { "longdesc": "", "scope": "global", "shortdesc": "`Access-Control-Allow-Methods` HTTP header value", "type": "string" } }, { "core.https_allowed_origin": { "longdesc": "", "scope": "global", "shortdesc": "`Access-Control-Allow-Origin` HTTP header value", "type": "string" } }, { "core.https_trusted_proxy": { "longdesc": "Specify a comma-separated list of IP addresses of trusted servers that provide the client's address through the proxy connection header.", "scope": "global", "shortdesc": "Trusted servers to provide the client's address", "type": "string" } }, { "core.metrics_address": { "longdesc": "See {ref}`metrics`.", "scope": "local", "shortdesc": "Address to bind the metrics server to (HTTPS)", "type": "string" } }, { "core.metrics_authentication": { "defaultdesc": "`true`", "longdesc": "", "scope": "global", "shortdesc": "Whether to enforce authentication on the metrics endpoint", "type": "bool" } }, { "core.proxy_http": { "longdesc": "If this option is not specified, the daemon falls back to the `HTTP_PROXY` environment variable (if set).", "scope": "global", "shortdesc": "HTTP proxy to use", "type": "string" } }, { "core.proxy_https": { "longdesc": "If this option is not specified, the daemon falls back to the `HTTPS_PROXY` environment variable (if set).", "scope": "global", "shortdesc": "HTTPS proxy to use", "type": "string" } }, { "core.proxy_ignore_hosts": { "longdesc": "Specify this option in a similar format to `NO_PROXY` (for example, `1.2.3.4,1.2.3.5`)\n\nIf this option is not specified, the daemon falls back to the `NO_PROXY` environment variable (if set).", "scope": "global", "shortdesc": "Hosts that don't need the proxy", "type": "string" } }, { "core.remote_token_expiry": { "defaultdesc": "no expiry", "longdesc": "", "scope": "global", "shortdesc": "Time after which a remote add token expires", "type": "string" } }, { "core.shutdown_action": { "defaultdesc": "`shutdown`", "longdesc": "Specify the action to take when the daemon is being shut down.\nSupported values are `shutdown` (stop all instances) and `evacuate` (attempt to evacuate the clustered server).", "scope": "global", "shortdesc": "Action to perform on server shutdown", "type": "string" } }, { "core.shutdown_timeout": { "defaultdesc": "`5`", "longdesc": "Specify the number of minutes to wait for running operations to complete before the daemon shuts down.", "scope": "global", "shortdesc": "How long to wait before shutdown", "type": "integer" } }, { "core.storage_buckets_address": { "longdesc": "See {ref}`howto-storage-buckets`.", "scope": "local", "shortdesc": "Address to bind the storage object server to (HTTPS)", "type": "string" } }, { "core.syslog_socket": { "defaultdesc": "`false`", "longdesc": "Set this option to `true` to enable the syslog unixgram socket to receive log messages from external processes.", "scope": "local", "shortdesc": "Whether to enable the syslog unixgram socket listener", "type": "bool" } }, { "core.trust_ca_certificates": { "defaultdesc": "`false`", "longdesc": "", "scope": "global", "shortdesc": "Whether to automatically trust clients signed by the CA", "type": "bool" } } ] }, "images": { "keys": [ { "images.auto_update_cached": { "defaultdesc": "`true`", "longdesc": "", "scope": "global", "shortdesc": "Whether to automatically update cached images", "type": "bool" } }, { "images.auto_update_interval": { "defaultdesc": "`6`", "longdesc": "Specify the interval in hours.\nTo disable looking for updates to cached images, set this option to `0`.", "scope": "global", "shortdesc": "Interval at which to look for updates to cached images", "type": "integer" } }, { "images.compression_algorithm": { "defaultdesc": "`gzip`", "longdesc": "Possible values are `bzip2`, `gzip`, `lz4`, `lzma`, `xz`, `zstd` or `none`.", "scope": "global", "shortdesc": "Compression algorithm to use for new images", "type": "string" } }, { "images.default_architecture": { "longdesc": "", "shortdesc": "Default architecture to use in a mixed-architecture cluster", "type": "string" } }, { "images.remote_cache_expiry": { "defaultdesc": "`10`", "longdesc": "Specify the number of days after which the unused cached image expires.", "scope": "global", "shortdesc": "When an unused cached remote image is flushed", "type": "integer" } } ] }, "logging": { "keys": [ { "logging.NAME.lifecycle.projects": { "longdesc": "", "scope": "global", "shortdesc": "Comma separate list of projects, empty means all", "type": "string" } }, { "logging.NAME.lifecycle.types": { "longdesc": "", "scope": "global", "shortdesc": "E.g., `instance`, comma separate, empty means all", "type": "string" } }, { "logging.NAME.logging.level": { "defaultdesc": "`info`", "longdesc": "", "scope": "global", "shortdesc": "Minimum log level to send to the logger", "type": "string" } }, { "logging.NAME.target.address": { "longdesc": "Specify the protocol, name or IP and port. For example `tcp://syslog01.int.example.net:514`.", "scope": "global", "shortdesc": "Address of the logger", "type": "string" } }, { "logging.NAME.target.ca_cert": { "longdesc": "", "scope": "global", "shortdesc": "CA certificate for the server", "type": "string" } }, { "logging.NAME.target.facility": { "longdesc": "", "scope": "global", "shortdesc": "The syslog facility defines the category of the log message", "type": "string" } }, { "logging.NAME.target.instance": { "defaultdesc": "Local server host name or cluster member name", "longdesc": "This allows replacing the default instance value (server host name) by a more relevant value like a cluster identifier.", "scope": "global", "shortdesc": "Name to use as the instance field in Loki events.", "type": "string" } }, { "logging.NAME.target.labels": { "longdesc": "Specify a comma-separated list of values that should be used as labels for a Loki log entry.", "scope": "global", "shortdesc": "Labels for a Loki log entry", "type": "string" } }, { "logging.NAME.target.password": { "longdesc": "", "scope": "global", "shortdesc": "Password used for authentication", "type": "string" } }, { "logging.NAME.target.retry": { "longdesc": "", "scope": "global", "shortdesc": "number of delivery retries, default 3", "type": "integer" } }, { "logging.NAME.target.type": { "longdesc": "", "scope": "global", "shortdesc": "The type of the logger. One of `loki`, `syslog` or `webhook`.", "type": "string" } }, { "logging.NAME.target.username": { "longdesc": "", "scope": "global", "shortdesc": "User name used for authentication", "type": "string" } }, { "logging.NAME.types": { "defaultdesc": "`lifecycle,logging`", "longdesc": "Specify a comma-separated list of events to send to the logger.\nThe events can be any combination of `lifecycle`, `logging`, and `network-acl`.", "scope": "global", "shortdesc": "Events to send to the logger", "type": "string" } } ] }, "loki": { "keys": [ { "loki.api.ca_cert": { "longdesc": "", "scope": "global", "shortdesc": "CA certificate for the Loki server", "type": "string" } }, { "loki.api.url": { "longdesc": "Specify the protocol, name or IP and port. For example `https://loki.example.com:3100`. Incus will automatically add the `/loki/api/v1/push` suffix so there's no need to add it here.", "scope": "global", "shortdesc": "URL to the Loki server", "type": "string" } }, { "loki.auth.password": { "longdesc": "", "scope": "global", "shortdesc": "Password used for Loki authentication", "type": "string" } }, { "loki.auth.username": { "longdesc": "", "scope": "global", "shortdesc": "User name used for Loki authentication", "type": "string" } }, { "loki.instance": { "defaultdesc": "Local server host name or cluster member name", "longdesc": "This allows replacing the default instance value (server host name) by a more relevant value like a cluster identifier.", "scope": "global", "shortdesc": "Name to use as the instance field in Loki events.", "type": "string" } }, { "loki.labels": { "longdesc": "Specify a comma-separated list of values that should be used as labels for a Loki log entry.", "scope": "global", "shortdesc": "Labels for a Loki log entry", "type": "string" } }, { "loki.loglevel": { "defaultdesc": "`info`", "longdesc": "", "scope": "global", "shortdesc": "Minimum log level to send to the Loki server", "type": "string" } }, { "loki.types": { "defaultdesc": "`lifecycle,logging`", "longdesc": "Specify a comma-separated list of events to send to the Loki server.\nThe events can be any combination of `lifecycle`, `logging`, and `network-acl`.", "scope": "global", "shortdesc": "Events to send to the Loki server", "type": "string" } } ] }, "miscellaneous": { "keys": [ { "authorization.scriptlet": { "longdesc": "When using scriptlet-based authorization, this option stores the scriptlet.", "scope": "global", "shortdesc": "Authorization scriptlet", "type": "string" } }, { "backups.compression_algorithm": { "defaultdesc": "`gzip`", "longdesc": "Possible values are `bzip2`, `gzip`, `lz4`, `lzma`, `xz`, `zstd` or `none`.", "scope": "global", "shortdesc": "Compression algorithm to use for backups", "type": "string" } }, { "instances.lxcfs.per_instance": { "defaultdesc": "`false`", "longdesc": "LXCFS is used to provide overlays for common `/proc` and `/sys`\nfiles which reflect the resource limits applied to the container.\n\nIt normally operates through a single file system mount on the host which is then shared by all containers.\nThis is very efficient but comes with the downside that a crash of LXCFS will break all containers.\n\nWith this option, it's now possible to run a LXCFS instance per\ncontainer instead, using more system resources but reducing the impact\nof a crash.", "scope": "global", "shortdesc": "Whether to run LXCFS on a per-instance basis", "type": "bool" } }, { "instances.nic.host_name": { "defaultdesc": "`random`", "longdesc": "Possible values are `random` and `mac`.\n\nIf set to `random`, use the random host interface name as the host name.\nIf set to `mac`, generate a host name in the form `inc\u003cmac_address\u003e` (MAC without leading two digits).", "scope": "global", "shortdesc": "How to set the host name for a NIC", "type": "string" } }, { "instances.placement.scriptlet": { "longdesc": "When using custom automatic instance placement logic, this option stores the scriptlet.\nSee {ref}`clustering-instance-placement-scriptlet` for more information.", "scope": "global", "shortdesc": "Instance placement scriptlet for automatic instance placement", "type": "string" } }, { "network.ovn.ca_cert": { "defaultdesc": "Content of `/etc/ovn/ovn-central.crt` if present", "longdesc": "", "scope": "global", "shortdesc": "OVN SSL certificate authority", "type": "string" } }, { "network.ovn.client_cert": { "defaultdesc": "Content of `/etc/ovn/cert_host` if present", "longdesc": "", "scope": "global", "shortdesc": "OVN SSL client certificate", "type": "string" } }, { "network.ovn.client_key": { "defaultdesc": "Content of `/etc/ovn/key_host` if present", "longdesc": "", "scope": "global", "shortdesc": "OVN SSL client key", "type": "string" } }, { "network.ovn.integration_bridge": { "defaultdesc": "`br-int`", "longdesc": "", "scope": "global", "shortdesc": "OVS integration bridge to use for OVN networks", "type": "string" } }, { "network.ovn.northbound_connection": { "defaultdesc": "`unix:/run/ovn/ovnnb_db.sock`", "longdesc": "", "scope": "global", "shortdesc": "OVN northbound database connection string", "type": "string" } }, { "network.ovs.connection": { "defaultdesc": "`unix:/run/openvswitch/db.sock`", "longdesc": "", "scope": "global", "shortdesc": "OVS socket path", "type": "string" } }, { "storage.backups_volume": { "longdesc": "Specify the volume using the syntax `POOL/VOLUME`.", "scope": "local", "shortdesc": "Volume to use to store backup tarballs", "type": "string" } }, { "storage.images_volume": { "longdesc": "Specify the volume using the syntax `POOL/VOLUME`.", "scope": "local", "shortdesc": "Volume to use to store the image tarballs", "type": "string" } }, { "storage.linstor.ca_cert": { "longdesc": "", "scope": "global", "shortdesc": "LINSTOR SSL certificate authority", "type": "string" } }, { "storage.linstor.client_cert": { "longdesc": "", "scope": "global", "shortdesc": "LINSTOR SSL client certificate", "type": "string" } }, { "storage.linstor.client_key": { "longdesc": "", "scope": "global", "shortdesc": "LINSTOR SSL client key", "type": "string" } }, { "storage.linstor.controller_connection": { "longdesc": "", "scope": "global", "shortdesc": "LINSTOR controller connection string", "type": "string" } }, { "storage.linstor.satellite.name": { "longdesc": "Set this option to the name of the local LINSTOR satellite node, should it be different from the Incus server name.", "scope": "global", "shortdesc": "LINSTOR satellite node name override", "type": "string" } }, { "storage.logs_volume": { "longdesc": "Specify the volume using the syntax `POOL/VOLUME`.", "scope": "local", "shortdesc": "Volume to use to store instance log directories", "type": "string" } } ] }, "network": { "keys": [ { "network.hwaddr_pattern": { "defaultdesc": "`10:66:6a:xx:xx:xx`", "longdesc": "Specify a MAC address template, e.g. `10:66:6a:xx:xx:xx`, to use within the cluster.\nEvery `x` in the template will be replaced by a random character in `0`–`f`.\nBeware of the birthday paradox! A single `xx` block leads to a 10% collision probability with only 8 addresses; for a double `xx:xx` block, 118 addresses; for a triple `xx:xx:xx` block, 1881; for a quadruple `xx:xx:xx:xx` block, 30084. We provide absolutely no guardrail against that.", "scope": "global", "shortdesc": "MAC address template", "type": "string" } } ] }, "oidc": { "keys": [ { "oidc.audience": { "longdesc": "This value is required by some providers.", "scope": "global", "shortdesc": "Expected audience value for the application", "type": "string" } }, { "oidc.claim": { "longdesc": "Note that the claim must be contained in the access token.", "scope": "global", "shortdesc": "OpenID Connect claim to use as the username", "type": "string" } }, { "oidc.client.id": { "longdesc": "", "scope": "global", "shortdesc": "OpenID Connect client ID", "type": "string" } }, { "oidc.issuer": { "longdesc": "", "scope": "global", "shortdesc": "OpenID Connect Discovery URL for the provider", "type": "string" } }, { "oidc.scopes": { "longdesc": "", "scope": "global", "shortdesc": "Comma separated list of OpenID Connect scopes", "type": "string" } } ] }, "openfga": { "keys": [ { "openfga.api.token": { "longdesc": "", "scope": "global", "shortdesc": "API token of the OpenFGA server", "type": "string" } }, { "openfga.api.url": { "longdesc": "", "scope": "global", "shortdesc": "URL of the OpenFGA server", "type": "string" } }, { "openfga.store.id": { "longdesc": "", "scope": "global", "shortdesc": "ID of the OpenFGA permission store", "type": "string" } } ] } }, "storage_btrfs": { "common": { "keys": [ { "btrfs.mount_options": { "default": "`user_subvol_rm_allowed`", "longdesc": "", "scope": "global", "shortdesc": "Mount options for block devices", "type": "string" } }, { "size": { "default": "auto (20% of free disk space, \u003e= 5 GiB and \u003c= 30 GiB)", "longdesc": "", "scope": "local", "shortdesc": "Size of the storage pool when creating loop-based pools (in bytes, suffixes supported, can be increased to grow storage pool)", "type": "string" } }, { "source": { "default": "-", "longdesc": "", "scope": "local", "shortdesc": "Path to an existing block device, loop file or Btrfs subvolume", "type": "string" } }, { "source.wipe": { "default": "`false`", "longdesc": "", "scope": "local", "shortdesc": "Wipe the block device specified in `source` prior to creating the storage pool", "type": "bool" } } ] } }, "storage_bucket_btrfs": { "common": { "keys": [ { "size": { "condition": "appropriate driver", "default": "same as `volume.size`", "longdesc": "", "shortdesc": "Size/quota of the storage bucket", "type": "string" } } ] } }, "storage_bucket_cephobject": { "common": { "keys": [ { "size": { "default": "-", "longdesc": "", "shortdesc": "Quota of the storage bucket", "type": "string" } } ] } }, "storage_bucket_lvm": { "common": { "keys": [ { "size": { "condition": "appropriate driver", "default": "same as `volume.size`", "longdesc": "", "shortdesc": "Size/quota of the storage bucket", "type": "string" } } ] } }, "storage_bucket_zfs": { "common": { "keys": [ { "size": { "condition": "appropriate driver", "default": "same as `volume.size`", "longdesc": "", "shortdesc": "Size/quota of the storage bucket", "type": "string" } } ] } }, "storage_ceph": { "common": { "keys": [ { "ceph.cluster_name": { "default": "`ceph`", "longdesc": "", "scope": "global", "shortdesc": "Name of the Ceph cluster in which to create new storage pools", "type": "string" } }, { "ceph.osd.data_pool_name": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "Name of the OSD data pool", "type": "string" } }, { "ceph.osd.force_reuse": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "Deprecated, should not be used.", "type": "bool" } }, { "ceph.osd.pg_name": { "default": "`32`", "longdesc": "", "scope": "global", "shortdesc": "Number of placement groups for the OSD storage pool", "type": "string" } }, { "ceph.osd.pool_name": { "default": "name of the pool", "longdesc": "", "scope": "global", "shortdesc": "Name of the OSD storage pool", "type": "string" } }, { "ceph.rbd.clone_copy": { "default": "`true`", "longdesc": "", "scope": "global", "shortdesc": "Whether to use RBD lightweight clones rather than full dataset copies", "type": "bool" } }, { "ceph.rbd.du": { "default": "`true`", "longdesc": "", "scope": "global", "shortdesc": "Whether to use RBD `du` to obtain disk usage data for stopped instances", "type": "bool" } }, { "ceph.rbd.features": { "default": "`layering`", "longdesc": "", "scope": "global", "shortdesc": "Comma-separated list of RBD features to enable on the volumes", "type": "string" } }, { "ceph.user.name": { "default": "`admin`", "longdesc": "", "scope": "global", "shortdesc": "The Ceph user to use when creating storage pools and volumes", "type": "string" } }, { "source": { "default": "-", "longdesc": "", "scope": "local", "shortdesc": "Existing OSD storage pool to use", "type": "string" } }, { "volatile.pool.pristine": { "default": "`true`", "longdesc": "", "scope": "global", "shortdesc": "Whether the pool was empty on creation time", "type": "string" } } ] } }, "storage_cephfs": { "common": { "keys": [ { "cephfs.cluster_name": { "default": "`ceph`", "longdesc": "", "scope": "global", "shortdesc": "Name of the Ceph cluster that contains the CephFS file system", "type": "string" } }, { "cephfs.create_missing": { "default": "`false`", "longdesc": "", "scope": "global", "shortdesc": "Create the file system and the missing data and metadata OSD pools", "type": "bool" } }, { "cephfs.data_pool": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "Data OSD pool name to create for the file system", "type": "string" } }, { "cephfs.fscache": { "default": "`false`", "longdesc": "", "scope": "global", "shortdesc": "Enable use of kernel `fscache` and `cachefilesd`", "type": "bool" } }, { "cephfs.meta_pool": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "Metadata OSD pool name to create for the file system", "type": "string" } }, { "cephfs.osd_pg_num": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "OSD pool `pg_num` to use when creating missing OSD pools", "type": "string" } }, { "cephfs.path": { "default": "`/`", "longdesc": "", "scope": "global", "shortdesc": "The base path for the CephFS mount", "type": "string" } }, { "cephfs.user.name": { "default": "`admin`", "longdesc": "", "scope": "global", "shortdesc": "The Ceph user to use", "type": "string" } }, { "source": { "default": "-", "longdesc": "", "scope": "local", "shortdesc": "Existing CephFS file system or file system path to use", "type": "string" } }, { "volatile.pool.pristine": { "default": "`true`", "longdesc": "", "scope": "global", "shortdesc": "Whether the CephFS file system was empty on creation time", "type": "string" } } ] } }, "storage_cephobject": { "common": { "keys": [ { "cephobject.bucket_name_prefix": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "Prefix to add to bucket names in Ceph", "type": "string" } }, { "cephobject.cluster_name": { "default": "`ceph`", "longdesc": "", "scope": "global", "shortdesc": "The Ceph cluster to use", "type": "string" } }, { "cephobject.radosgw.endpoint": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "URL of the `radosgw` gateway process", "type": "string" } }, { "cephobject.radosgw.endpoint_cert_file": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "Path to the file containing the TLS client certificate to use for endpoint communication", "type": "string" } }, { "cephobject.user.name": { "default": "`admin`", "longdesc": "", "scope": "global", "shortdesc": "The Ceph user to use", "type": "string" } }, { "volatile.pool.pristine": { "default": "`true`", "longdesc": "", "scope": "global", "shortdesc": "Whether the `radosgw` `incus-admin` user existed at creation time", "type": "string" } } ] } }, "storage_dir": { "common": { "keys": [ { "rsync.bwlimit": { "default": "`0` (no limit)", "longdesc": "", "scope": "global", "shortdesc": "The upper limit to be placed on the socket I/O when `rsync` must be used to transfer storage entities", "type": "string" } }, { "rsync.compression": { "default": "`true`", "longdesc": "", "scope": "global", "shortdesc": "Whether to use compression while migrating storage pools", "type": "bool" } }, { "source": { "default": "-", "longdesc": "", "scope": "local", "shortdesc": "Path to an existing directory", "type": "string" } } ] } }, "storage_linstor": { "common": { "keys": [ { "drbd.auto_add_quorum_tiebreaker": { "default": "`true`", "longdesc": "", "scope": "global", "shortdesc": "Whether to allow LINSTOR to automatically create diskless resources to act as quorum tiebreakers if needed (applied to the resource group)", "type": "bool" } }, { "drbd.auto_diskful": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "A duration string describing the time after which a primary diskless resource can be converted to diskful if storage is available on the node (applied to the resource group)", "type": "string" } }, { "drbd.on_no_quorum": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "The DRBD policy to use on resources when quorum is lost (applied to the resource group)", "type": "string" } }, { "linstor.resource_group.name": { "default": "`incus`", "longdesc": "", "scope": "global", "shortdesc": "Name of the LINSTOR resource group that will be used for the storage pool", "type": "string" } }, { "linstor.resource_group.place_count": { "default": "`2`", "longdesc": "", "scope": "global", "shortdesc": "Number of diskful replicas that should be created for resources in the resource group. Increasing the value of this option on a pool that already has volumes will result in LINSTOR creating new diskful replicas for all existing resources to match the new value", "type": "int" } }, { "linstor.resource_group.storage_pool": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "The storage pool name in which resources should be placed on satellite nodes", "type": "string" } }, { "linstor.volume.prefix": { "default": "`incus-volume-`", "longdesc": "", "scope": "global", "shortdesc": "The prefix to use for the internal names of LINSTOR-managed volumes. Cannot be updated after the storage pool is created", "type": "string" } }, { "source": { "default": "`incus`", "longdesc": "", "scope": "global", "shortdesc": "LINSTOR storage pool name. Alias for `linstor.resource_group.name`. Use either either one or the other or make sure they have the same value.", "type": "string" } }, { "volatile.pool.pristine": { "default": "`true`", "longdesc": "", "scope": "global", "shortdesc": "Whether the pool was empty on creation time", "type": "string" } } ] } }, "storage_lvm": { "common": { "keys": [ { "block.type": { "condition": "block-based volume", "default": "same as `volume.block.type`", "longdesc": "", "shortdesc": "Type of the block volume" } }, { "lvm.metadata_size": { "default": "`0` (auto)", "longdesc": "", "scope": "global", "shortdesc": "The size of the metadata space for the physical volume.", "type": "string" } }, { "lvm.thinpool_metadata_size": { "default": "`0` (auto)", "longdesc": "", "scope": "global", "shortdesc": "The size of the thin pool metadata volume (the default is to let LVM calculate an appropriate size). Not usable with `lvmcluster`.", "type": "string" } }, { "lvm.thinpool_name": { "default": "`IncusThinPool`", "longdesc": "", "scope": "local", "shortdesc": "Thin pool where volumes are created. Not usable with `lvmcluster`.", "type": "string" } }, { "lvm.use_thinpool": { "default": "`true`", "longdesc": "", "scope": "global", "shortdesc": "Whether the storage pool uses a thin pool for logical volumes. Not usable with `lvmcluster`.", "type": "bool" } }, { "lvm.vg.force_reuse": { "default": "`false`", "longdesc": "", "scope": "local", "shortdesc": "Force using an existing non-empty volume group. Not usable with `lvmcluster`.", "type": "bool" } }, { "lvm.vg_name": { "default": "name of the pool", "longdesc": "", "scope": "local", "shortdesc": "Name of the volume group to create.", "type": "string" } }, { "size": { "default": "auto (20% of free disk space, \u003e= 5 GiB and \u003c= 30 GiB) for `lvm`.", "longdesc": "", "scope": "local", "shortdesc": "Storage pool size (in bytes, suffixes supported, can be increased to grow storage pool). `lvmcluster` pools: cannot set at creation, but can be updated.", "type": "string" } }, { "source": { "default": "-", "longdesc": "", "scope": "local", "shortdesc": "Path to an existing block device, loop file or LVM volume group.", "type": "string" } }, { "source.wipe": { "default": "`false`", "longdesc": "", "scope": "local", "shortdesc": "Wipe the block device specified in `source` prior to creating the storage pool.", "type": "bool" } } ] } }, "storage_truenas": { "common": { "keys": [ { "source": { "default": "-", "longdesc": "", "scope": "local", "shortdesc": "ZFS dataset to use on the remote TrueNAS host. Format: `[\u003chost\u003e:]\u003cpool\u003e[/\u003cdataset\u003e][/]`. If `host` is omitted here, it must be set via `truenas.host`.", "type": "string" } }, { "truenas.allow_insecure": { "default": "`false`", "longdesc": "", "scope": "global", "shortdesc": "If set to `true`, allows insecure (non-TLS) connections to the TrueNAS API.", "type": "bool" } }, { "truenas.api_key": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "API key used to authenticate with the TrueNAS host.", "type": "string" } }, { "truenas.clone_copy": { "default": "`true`", "longdesc": "", "scope": "global", "shortdesc": "Whether to use lightweight clones rather than full {spellexception}`dataset` copies.", "type": "bool" } }, { "truenas.config": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "Path to a configuration file for the TrueNAS client tool.", "type": "string" } }, { "truenas.dataset": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "Remote dataset name. Typically inferred from `source`, but can be overridden.", "type": "string" } }, { "truenas.force_reuse": { "default": "`false`", "longdesc": "", "scope": "global", "shortdesc": "Allow to use an existing non-empty pool.", "type": "bool" } }, { "truenas.host": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "Hostname or IP address of the remote TrueNAS system. Optional if included in the `source`, or a configuration is used.", "type": "string" } }, { "truenas.initiator": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "iSCSI initiator name used during block volume attachment.", "type": "string" } }, { "truenas.portal": { "default": "-", "longdesc": "", "scope": "global", "shortdesc": "iSCSI portal address to use for block volume connections.", "type": "string" } } ] } }, "storage_volume_btrfs": { "common": { "keys": [ { "initial.gid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.gid` or `0`", "longdesc": "", "shortdesc": "GID of the volume owner in the instance", "type": "int" } }, { "initial.mode": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.mode` or `711`", "longdesc": "", "shortdesc": "Mode of the volume in the instance", "type": "int" } }, { "initial.uid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.uid` or `0`", "longdesc": "", "shortdesc": "UID of the volume owner in the instance", "type": "int" } }, { "security.shared": { "condition": "custom block volume", "default": "same as `volume.security.shared` or `false`", "longdesc": "", "shortdesc": "Enable sharing the volume across multiple instances", "type": "bool" } }, { "security.shifted": { "condition": "custom volume", "default": "same as `volume.security.shifted` or `false`", "longdesc": "", "shortdesc": "{{enable_ID_shifting}}", "type": "bool" } }, { "security.unmapped": { "condition": "custom volume", "default": "same as `volume.security.unmapped` or `false`", "longdesc": "", "shortdesc": "Disable ID mapping for the volume", "type": "bool" } }, { "size": { "condition": "appropriate driver", "default": "same as `volume.size`", "longdesc": "", "shortdesc": "Size/quota of the storage volume", "type": "string" } }, { "snapshots.expiry": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.expiry.manual": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry.manual`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.pattern": { "condition": "custom volume", "default": "same as `volume.snapshot.pattern` or `snap%d`", "longdesc": "", "shortdesc": "{{snapshot_pattern_format}} [^*]", "type": "string" } }, { "snapshots.schedule": { "condition": "custom volume", "default": "same as `volume.snapshot.schedule`", "longdesc": "", "shortdesc": "{{snapshot_schedule_format}}", "type": "string" } } ] } }, "storage_volume_ceph": { "common": { "keys": [ { "block.filesystem": { "condition": "block-based volume with content type `filesystem`", "default": "same as `volume.block.filesystem`", "longdesc": "", "shortdesc": "{{block_filesystem}}", "type": "string" } }, { "block.mount_options": { "condition": "block-based volume with content type `filesystem`", "default": "same as `volume.block.mount_options`", "longdesc": "", "shortdesc": "Mount options for block-backed file system volumes", "type": "string" } }, { "initial.gid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.gid` or `0`", "longdesc": "", "shortdesc": "GID of the volume owner in the instance", "type": "int" } }, { "initial.mode": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.mode` or `711`", "longdesc": "", "shortdesc": "Mode of the volume in the instance", "type": "int" } }, { "initial.uid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.uid` or `0`", "longdesc": "", "shortdesc": "UID of the volume owner in the instance", "type": "int" } }, { "security.shared": { "condition": "custom block volume", "default": "same as `volume.security.shared` or `false`", "longdesc": "", "shortdesc": "Enable sharing the volume across multiple instances", "type": "bool" } }, { "security.shifted": { "condition": "custom volume", "default": "same as `volume.security.shifted` or `false`", "longdesc": "", "shortdesc": "{{enable_ID_shifting}}", "type": "bool" } }, { "security.unmapped": { "condition": "custom volume", "default": "same as `volume.security.unmapped` or `false`", "longdesc": "", "shortdesc": "Disable ID mapping for the volume", "type": "bool" } }, { "size": { "condition": "-", "default": "same as `volume.size`", "longdesc": "", "shortdesc": "Size/quota of the storage volume", "type": "string" } }, { "snapshots.expiry": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.expiry.manual": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry.manual`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.pattern": { "condition": "custom volume", "default": "same as `volume.snapshot.pattern` or `snap%d`", "longdesc": "", "shortdesc": "{{snapshot_pattern_format}} [^*]", "type": "string" } }, { "snapshots.schedule": { "condition": "custom volume", "default": "same as `volume.snapshot.schedule`", "longdesc": "", "shortdesc": "{{snapshot_schedule_format}}", "type": "string" } } ] } }, "storage_volume_cephfs": { "common": { "keys": [ { "initial.gid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.gid` or `0`", "longdesc": "", "shortdesc": "GID of the volume owner in the instance", "type": "int" } }, { "initial.mode": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.mode` or `711`", "longdesc": "", "shortdesc": "Mode of the volume in the instance", "type": "int" } }, { "initial.uid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.uid` or `0`", "longdesc": "", "shortdesc": "UID of the volume owner in the instance", "type": "int" } }, { "security.shared": { "condition": "custom block volume", "default": "same as `volume.security.shared` or `false`", "longdesc": "", "shortdesc": "Enable sharing the volume across multiple instances", "type": "bool" } }, { "security.shifted": { "condition": "custom volume", "default": "same as `volume.security.shifted` or `false`", "longdesc": "", "shortdesc": "{{enable_ID_shifting}}", "type": "bool" } }, { "security.unmapped": { "condition": "custom volume", "default": "same as `volume.security.unmapped` or `false`", "longdesc": "", "shortdesc": "Disable ID mapping for the volume", "type": "bool" } }, { "size": { "condition": "appropriate driver", "default": "same as `volume.size`", "longdesc": "", "shortdesc": "Size/quota of the storage volume", "type": "string" } }, { "snapshots.expiry": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.expiry.manual": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry.manual`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.pattern": { "condition": "custom volume", "default": "same as `volume.snapshot.pattern` or `snap%d`", "longdesc": "", "shortdesc": "{{snapshot_pattern_format}} [^*]", "type": "string" } }, { "snapshots.schedule": { "condition": "custom volume", "default": "same as `volume.snapshot.schedule`", "longdesc": "", "shortdesc": "{{snapshot_schedule_format}}", "type": "string" } } ] } }, "storage_volume_dir": { "common": { "keys": [ { "initial.gid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.gid` or `0`", "longdesc": "", "shortdesc": "GID of the volume owner in the instance", "type": "int" } }, { "initial.mode": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.mode` or `711`", "longdesc": "", "shortdesc": "Mode of the volume in the instance", "type": "int" } }, { "initial.uid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.uid` or `0`", "longdesc": "", "shortdesc": "UID of the volume owner in the instance", "type": "int" } }, { "security.shared": { "condition": "custom block volume", "default": "same as `volume.security.shared` or `false`", "longdesc": "", "shortdesc": "Enable sharing the volume across multiple instances", "type": "bool" } }, { "security.shifted": { "condition": "custom volume", "default": "same as `volume.security.shifted` or `false`", "longdesc": "", "shortdesc": "{{enable_ID_shifting}}", "type": "bool" } }, { "security.size": { "condition": "appropriate driver", "default": "same as `volume.size`", "longdesc": "", "shortdesc": "Size/quota of the storage volume", "type": "string" } }, { "security.unmapped": { "condition": "custom volume", "default": "same as `volume.security.unmapped` or `false`", "longdesc": "", "shortdesc": "Disable ID mapping for the volume", "type": "bool" } }, { "snapshots.expiry": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.expiry.manual": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry.manual`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.pattern": { "condition": "custom volume", "default": "same as `volume.snapshot.pattern` or `snap%d`", "longdesc": "", "shortdesc": "{{snapshot_pattern_format}} [^*]", "type": "string" } }, { "snapshots.schedule": { "condition": "custom volume", "default": "same as `volume.snapshot.schedule`", "longdesc": "", "shortdesc": "{{snapshot_schedule_format}}", "type": "string" } } ] } }, "storage_volume_linstor": { "common": { "keys": [ { "block.filesystem": { "condition": "block-based volume with content type `filesystem`", "default": "same as `volume.block.filesystem`", "longdesc": "", "shortdesc": "{{block_filesystem}}", "type": "string" } }, { "block.mount_options": { "condition": "block-based volume with content type `filesystem`", "default": "same as `volume.block.mount_options`", "longdesc": "", "shortdesc": "Mount options for block-backed file system volumes", "type": "string" } }, { "drbd.auto_add_quorum_tiebreaker": { "condition": "-", "default": "`true`", "longdesc": "", "shortdesc": "Whether to allow LINSTOR to automatically create diskless resources to act as quorum tiebreakers if needed (applied to the resource definition)", "type": "bool" } }, { "drbd.auto_diskful": { "condition": "-", "default": "-", "longdesc": "", "shortdesc": "A duration string describing the time after which a primary diskless resource can be converted to diskful if storage is available on the node (applied to the resource definition)", "type": "string" } }, { "drbd.on_no_quorum": { "condition": "-", "default": "-", "longdesc": "", "shortdesc": "The DRBD policy to use on resources when quorum is lost (applied to the resource definition)", "type": "string" } }, { "initial.gid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.gid` or `0`", "longdesc": "", "shortdesc": "GID of the volume owner in the instance", "type": "int" } }, { "initial.mode": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.mode` or `711`", "longdesc": "", "shortdesc": "Mode of the volume in the instance", "type": "int" } }, { "initial.uid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.uid` or `0`", "longdesc": "", "shortdesc": "UID of the volume owner in the instance", "type": "int" } }, { "linstor.remove_snapshots": { "condition": "-", "default": "same as `volume.linstor.remove_snapshots` or `false`", "longdesc": "", "shortdesc": "Remove snapshots as needed", "type": "bool" } }, { "security.shared": { "condition": "custom block volume", "default": "same as `volume.security.shared` or `false`", "longdesc": "", "shortdesc": "Enable sharing the volume across multiple instances", "type": "bool" } }, { "security.shifted": { "condition": "custom volume", "default": "same as `volume.security.shifted` or `false`", "longdesc": "", "shortdesc": "{{enable_ID_shifting}}", "type": "bool" } }, { "security.unmapped": { "condition": "custom volume", "default": "same as `volume.security.unmapped` or `false`", "longdesc": "", "shortdesc": "Disable ID mapping for the volume", "type": "bool" } }, { "size": { "condition": "-", "default": "same as `volume.size`", "longdesc": "", "shortdesc": "Size/quota of the storage volume", "type": "string" } }, { "snapshots.expiry": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.expiry.manual": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry.manual`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.pattern": { "condition": "custom volume", "default": "same as `volume.snapshot.pattern` or `snap%d`", "longdesc": "", "shortdesc": "{{snapshot_pattern_format}} [^*]", "type": "string" } }, { "snapshots.schedule": { "condition": "custom volume", "default": "same as `volume.snapshot.schedule`", "longdesc": "", "shortdesc": "{{snapshot_schedule_format}}", "type": "string" } } ] } }, "storage_volume_lvm": { "common": { "keys": [ { "block.filesystem": { "condition": "block-based volume with content type `filesystem`", "default": "same as `volume.block.filesystem`", "longdesc": "", "shortdesc": "{{block_filesystem}}", "type": "string" } }, { "block.mount_options": { "condition": "block-based volume with content type `filesystem`", "default": "same as `volume.block.mount_options`", "longdesc": "", "shortdesc": "Mount options for block-backed file system volumes", "type": "string" } }, { "initial.gid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.gid` or `0`", "longdesc": "", "shortdesc": "GID of the volume owner in the instance", "type": "int" } }, { "initial.mode": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.mode` or `711`", "longdesc": "", "shortdesc": "Mode of the volume in the instance", "type": "int" } }, { "initial.uid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.uid` or `0`", "longdesc": "", "shortdesc": "UID of the volume owner in the instance", "type": "int" } }, { "lvm.stripes": { "condition": "-", "default": "same as `volume.lvm.stripes`", "longdesc": "", "shortdesc": "Number of stripes to use for new volumes (or thin pool volume)", "type": "string" } }, { "lvm.stripes.size": { "condition": "-", "default": "same as `volume.lvm.stripes.size`", "longdesc": "", "shortdesc": "Size of stripes to use (at least 4096 bytes and multiple of 512 bytes)", "type": "string" } }, { "lvmcluster.remove_snapshots": { "condition": "-", "default": "same as `volume.lvmcluster.remove_snapshots` or `false`", "longdesc": "", "shortdesc": "Remove snapshots as needed", "type": "bool" } }, { "security.shared": { "condition": "custom block volume", "default": "same as `volume.security.shared` or `false`", "longdesc": "", "shortdesc": "Enable sharing the volume across multiple instances", "type": "bool" } }, { "security.shifted": { "condition": "custom volume", "default": "same as `volume.security.shifted` or `false`", "longdesc": "", "shortdesc": "{{enable_ID_shifting}}", "type": "bool" } }, { "security.unmapped": { "condition": "custom volume", "default": "same as `volume.security.unmapped` or `false`", "longdesc": "", "shortdesc": "Disable ID mapping for the volume", "type": "bool" } }, { "size": { "condition": "default: same as `volume.size`", "longdesc": "", "shortdesc": "Size/quota of the storage volume", "type": "string" } }, { "snapshots.expiry": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.expiry.manual": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry.manual`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.pattern": { "condition": "custom volume", "default": "same as `volume.snapshot.pattern` or `snap%d`", "longdesc": "", "shortdesc": "{{snapshot_pattern_format}} [^*]", "type": "string" } }, { "snapshots.schedule": { "condition": "custom volume", "default": "same as `volume.snapshot.schedule`", "longdesc": "", "shortdesc": "{{snapshot_schedule_format}}", "type": "string" } } ] } }, "storage_volume_truenas": { "common": { "keys": [ { "block.filesystem": { "condition": "-", "default": "same as `volume.block.filesystem`", "longdesc": "", "shortdesc": "{{block_filesystem}}", "type": "string" } }, { "block.mount_options": { "condition": "-", "default": "same as `volume.block.mount_options`", "longdesc": "", "shortdesc": "Mount options for block-backed file system volumes", "type": "string" } }, { "initial.gid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.gid` or `0`", "longdesc": "", "shortdesc": "GID of the volume owner in the instance", "type": "int" } }, { "initial.mode": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.mode` or `711`", "longdesc": "", "shortdesc": "Mode of the volume in the instance", "type": "int" } }, { "initial.uid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.uid` or `0`", "longdesc": "", "shortdesc": "UID of the volume owner in the instance", "type": "int" } }, { "security.shared": { "condition": "custom block volume", "default": "same as `volume.security.shared` or `false`", "longdesc": "", "shortdesc": "Enable sharing the volume across multiple instances", "type": "bool" } }, { "security.shifted": { "condition": "custom volume", "default": "same as `volume.security.shifted` or `false`", "longdesc": "", "shortdesc": "{{enable_ID_shifting}}", "type": "bool" } }, { "security.unmapped": { "condition": "custom volume", "default": "same as `volume.security.unmapped` or `false`", "longdesc": "", "shortdesc": "Disable ID mapping for the volume", "type": "bool" } }, { "size": { "condition": "appropriate driver", "default": "same as `volume.size`", "longdesc": "", "shortdesc": "Size/quota of the storage volume", "type": "string" } }, { "snapshots.expiry": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.expiry.manual": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry.manual`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.pattern": { "condition": "custom volume", "default": "same as `volume.snapshot.pattern` or `snap%d`", "longdesc": "", "shortdesc": "{{snapshot_pattern_format}}", "type": "string" } }, { "snapshots.schedule": { "condition": "custom volume", "default": "same as `volume.snapshot.schedule`", "longdesc": "", "shortdesc": "{{snapshot_schedule_format}}", "type": "string" } }, { "truenas.blocksize": { "condition": "-", "default": "same as `volume.truenas.blocksize`", "longdesc": "", "shortdesc": "Size of the ZFS block in range from 512 bytes to 16 MiB (must be power of 2) - for block volume, a maximum value of 128 KiB will be used even if a higher value is set", "type": "string" } }, { "truenas.remove_snapshots": { "condition": "-", "default": "same as `volume.truenas.remove_snapshots` or `false`", "longdesc": "", "shortdesc": "Remove snapshots as needed", "type": "bool" } }, { "truenas.use_refquota": { "condition": "-", "default": "same as `volume.truenas.use_refquota` or `false`", "longdesc": "", "shortdesc": "Use `refquota` instead of `quota` for space", "type": "bool" } } ] } }, "storage_volume_zfs": { "common": { "keys": [ { "block.filesystem": { "condition": "block-based volume with content type `filesystem` (`zfs.block_mode` enabled)", "default": "same as `volume.block.filesystem`", "longdesc": "", "shortdesc": "{{block_filesystem}}", "type": "string" } }, { "block.mount_options": { "condition": "block-based volume with content type `filesystem` (`zfs.block_mode` enabled)", "default": "same as `volume.block.mount_options`", "longdesc": "", "shortdesc": "Mount options for block-backed file system volumes", "type": "string" } }, { "initial.gid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.gid` or `0`", "longdesc": "", "shortdesc": "GID of the volume owner in the instance", "type": "int" } }, { "initial.mode": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.mode` or `711`", "longdesc": "", "shortdesc": "Mode of the volume in the instance", "type": "int" } }, { "initial.uid": { "condition": "custom volume with content type `filesystem`", "default": "same as `volume.initial.uid` or `0`", "longdesc": "", "shortdesc": "UID of the volume owner in the instance", "type": "int" } }, { "security.shared": { "condition": "custom block volume", "default": "same as `volume.security.shared` or `false`", "longdesc": "", "shortdesc": "Enable sharing the volume across multiple instances", "type": "bool" } }, { "security.shifted": { "condition": "custom volume", "default": "same as `volume.security.shifted` or `false`", "longdesc": "", "shortdesc": "{{enable_ID_shifting}}", "type": "bool" } }, { "security.unmapped": { "condition": "custom volume", "default": "same as `volume.security.unmapped` or `false`", "longdesc": "", "shortdesc": "Disable ID mapping for the volume", "type": "bool" } }, { "size": { "condition": "-", "default": "same as `volume.size`", "longdesc": "", "shortdesc": "Size/quota of the storage volume", "type": "string" } }, { "snapshots.expiry": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.expiry.manual": { "condition": "custom volume", "default": "same as `volume.snapshot.expiry.manual`", "longdesc": "", "shortdesc": "{{snapshot_expiry_format}}", "type": "string" } }, { "snapshots.pattern": { "condition": "custom volume", "default": "same as `volume.snapshot.pattern` or `snap%d`", "longdesc": "", "shortdesc": "{{snapshot_pattern_format}} [^*]", "type": "string" } }, { "snapshots.schedule": { "condition": "custom volume", "default": "same as `volume.snapshot.schedule`", "longdesc": "", "shortdesc": "{{snapshot_schedule_format}}", "type": "string" } }, { "zfs.block_mode": { "condition": "-", "default": "same as `volume.zfs.block_mode`", "longdesc": "", "shortdesc": "Whether to use a formatted `zvol` rather than a {spellexception}`dataset` (`zfs.block_mode` can be set only for custom storage volumes; use `volume.zfs.block_mode` to enable ZFS block mode for all storage volumes in the pool, including instance volumes)", "type": "bool" } }, { "zfs.blocksize": { "condition": "-", "default": "same as `volume.zfs.blocksize`", "longdesc": "", "shortdesc": "Size of the ZFS block in range from 512 bytes to 16 MiB (must be power of 2) - for block volume, a maximum value of 128 KiB will be used even if a higher value is set", "type": "string" } }, { "zfs.delegate": { "condition": "ZFS 2.2 or higher", "default": "same as `volume.zfs.delegate`", "longdesc": "", "shortdesc": "Controls whether to delegate the ZFS dataset and anything underneath it to the container(s) using it. Allows the use of the `zfs` command in the container", "type": "bool" } }, { "zfs.remove_snapshots": { "condition": "-", "default": "same as `volume.zfs.remove_snapshots` or `false`", "longdesc": "", "shortdesc": "Remove snapshots as needed", "type": "bool" } }, { "zfs.reserve_space": { "condition": "-", "default": "same as `volume.zfs.reserve_space` or `false`", "longdesc": "", "shortdesc": "Use `reservation`/`refreservation` along with `quota`/`refquota`", "type": "bool" } }, { "zfs.use_refquota": { "condition": "-", "default": "same as `volume.zfsuse_refquota` or `false`", "longdesc": "", "shortdesc": "Use `refquota` instead of `quota` for space", "type": "bool" } } ] } }, "storage_zfs": { "common": { "keys": [ { "size": { "default": "auto (20% of free disk space, \u003e= 5 GiB and \u003c= 30 GiB)", "longdesc": "", "scope": "local", "shortdesc": "Size of the storage pool when creating loop-based pools (in bytes, suffixes supported, can be increased to grow storage pool)", "type": "string" } }, { "source": { "default": "-", "longdesc": "", "scope": "local", "shortdesc": "Path to existing block device(s), loop file or ZFS dataset/pool. Multiple block devices should be separated by `,`. When listing block devices, you can also prefix them with `vdev` type. To specify a `vdev` type, use an `=` sign between the `vdev` type and the block devices (e.g., `mirror=/dev/sda,/dev/sdb`). Only `stripe`, `mirror`, `raidz1` and `raidz2` `vdev` types are supported.", "type": "string" } }, { "source.wipe": { "default": "`false`", "longdesc": "", "scope": "local", "shortdesc": "Wipe the block device specified in `source` prior to creating the storage pool", "type": "bool" } }, { "zfs.clone_copy": { "default": "`true`", "longdesc": "", "scope": "global", "shortdesc": "Whether to use ZFS lightweight clones rather than full {spellexception}`dataset` copies (Boolean), or `rebase` to copy based on the initial image", "type": "string" } }, { "zfs.export": { "default": "`true`", "longdesc": "", "scope": "global", "shortdesc": "Disable zpool export while unmount performed", "type": "bool" } }, { "zfs.pool_name": { "default": "name of the pool", "longdesc": "", "scope": "local", "shortdesc": "Name of the zpool", "type": "string" } } ] } } } }incus-7.0.0/internal/server/metadata/metadata.go000066400000000000000000000005411517523235500216600ustar00rootroot00000000000000package metadata import ( "embed" "encoding/json" ) var Data map[string]any //go:embed configuration.json var generatedDoc embed.FS func init() { file, err := generatedDoc.ReadFile("configuration.json") if err != nil { panic(err) } var data map[string]any err = json.Unmarshal(file, &data) if err != nil { panic(err) } Data = data } incus-7.0.0/internal/server/metrics/000077500000000000000000000000001517523235500174375ustar00rootroot00000000000000incus-7.0.0/internal/server/metrics/api.go000066400000000000000000000120611517523235500205370ustar00rootroot00000000000000package metrics // Metrics represents instance metrics. type Metrics struct { CPU []CPUMetrics `json:"cpu_seconds_total" yaml:"cpu_seconds_total"` CPUs int `json:"cpus" yaml:"cpus"` Disk []DiskMetrics `json:"disk" yaml:"disk"` Filesystem []FilesystemMetrics `json:"filesystem" yaml:"filesystem"` Memory MemoryMetrics `json:"memory" yaml:"memory"` Network []NetworkMetrics `json:"network" yaml:"network"` ProcessesTotal uint64 `json:"procs_total" yaml:"procs_total"` BootTimeSeconds uint64 `json:"boot_time_seconds" yaml:"boot_time_seconds"` TimeSeconds uint64 `json:"time_seconds" yaml:"time_seconds"` } // CPUMetrics represents CPU metrics for an instance. type CPUMetrics struct { CPU string `json:"cpu" yaml:"cpu"` SecondsUser float64 `json:"cpu_seconds_user" yaml:"cpu_seconds_user"` SecondsNice float64 `json:"cpu_seconds_nice" yaml:"cpu_seconds_nice"` SecondsSystem float64 `json:"cpu_seconds_system" yaml:"cpu_seconds_system"` SecondsIdle float64 `json:"cpu_seconds_idle" yaml:"cpu_seconds_idle"` SecondsIOWait float64 `json:"cpu_seconds_iowait" yaml:"cpu_seconds_iowait"` SecondsIRQ float64 `json:"cpu_seconds_irq" yaml:"cpu_seconds_irq"` SecondsSoftIRQ float64 `json:"cpu_seconds_softirq" yaml:"cpu_seconds_softirq"` SecondsSteal float64 `json:"cpu_seconds_steal" yaml:"cpu_seconds_steal"` } // DiskMetrics represents disk metrics for an instance. type DiskMetrics struct { Device string `json:"device" yaml:"device"` ReadBytes uint64 `json:"disk_read_bytes" yaml:"disk_read_bytes"` ReadsCompleted uint64 `json:"disk_reads_completed" yaml:"disk_reads_completes"` WrittenBytes uint64 `json:"disk_written_bytes" yaml:"disk_written_bytes"` WritesCompleted uint64 `json:"disk_writes_completed" yaml:"disk_writes_completed"` } // FilesystemMetrics represents filesystem metrics for an instance. type FilesystemMetrics struct { Device string `json:"device" yaml:"device"` Mountpoint string `json:"mountpoint" yaml:"mountpoint"` FSType string `json:"fstype" yaml:"fstype"` AvailableBytes uint64 `json:"filesystem_avail_bytes" yaml:"filesystem_avail_bytes"` FreeBytes uint64 `json:"filesystem_free_bytes" yaml:"filesystem_free_bytes"` SizeBytes uint64 `json:"filesystem_size_bytes" yaml:"filesystem_size_bytes"` } // MemoryMetrics represents memory metrics for an instance. type MemoryMetrics struct { ActiveAnonBytes uint64 `json:"memory_active_anon_bytes" yaml:"memory_active_anon_bytes"` ActiveFileBytes uint64 `json:"memory_active_file_bytes" yaml:"memory_active_file_bytes"` ActiveBytes uint64 `json:"memory_active_bytes" yaml:"memory_active_bytes"` CachedBytes uint64 `json:"memory_cached_bytes" yaml:"memory_cached_bytes"` DirtyBytes uint64 `json:"memory_dirty_bytes" yaml:"memory_dirty_bytes"` HugepagesFreeBytes uint64 `json:"memory_hugepages_free_bytes" yaml:"memory_hugepages_Free_bytes"` HugepagesTotalBytes uint64 `json:"memory_hugepages_total_bytes" yaml:"memory_hugepages_total_bytes"` InactiveAnonBytes uint64 `json:"memory_inactive_anon_bytes" yaml:"memory_inactive_anon_bytes"` InactiveFileBytes uint64 `json:"memory_inactive_file_bytes" yaml:"memory_inactive_file_bytes"` InactiveBytes uint64 `json:"memory_inactive_bytes" yaml:"memory_inactive_bytes"` MappedBytes uint64 `json:"memory_mapped_bytes" yaml:"memory_mapped_bytes"` MemAvailableBytes uint64 `json:"memory_mem_available_bytes" yaml:"memory_mem_available_bytes"` MemFreeBytes uint64 `json:"memory_mem_free_bytes" yaml:"memory_mem_Free_bytes"` MemTotalBytes uint64 `json:"memory_mem_total_bytes" yaml:"memory_mem_total_bytes"` RSSBytes uint64 `json:"memory_rss_bytes" yaml:"memory_rss_bytes"` ShmemBytes uint64 `json:"memory_shmem_bytes" yaml:"memory_shmem_bytes"` SwapBytes uint64 `json:"memory_swap_bytes" yaml:"memory_swap_bytes"` UnevictableBytes uint64 `json:"memory_unevictable_bytes" yaml:"memory_unevictable_bytes"` WritebackBytes uint64 `json:"memory_writeback_bytes" yaml:"memory_writeback_bytes"` OOMKills uint64 `json:"memory_oom_kills" yaml:"memory_oom_kills"` } // NetworkMetrics represents network metrics for an instance. type NetworkMetrics struct { Device string `json:"device" yaml:"device"` ReceiveBytes uint64 `json:"network_receive_bytes" yaml:"network_receive_bytes"` ReceiveDrop uint64 `json:"network_receive_drop" yaml:"network_receive_drop"` ReceiveErrors uint64 `json:"network_receive_errs" yaml:"network_receive_errs"` ReceivePackets uint64 `json:"network_receive_packets" yaml:"network_receive_packets"` TransmitBytes uint64 `json:"network_transmit_bytes" yaml:"network_transmit_bytes"` TransmitDrop uint64 `json:"network_transmit_drop" yaml:"network_transmit_drop"` TransmitErrors uint64 `json:"network_transmit_errs" yaml:"network_transmit_errs"` TransmitPackets uint64 `json:"network_transmit_packets" yaml:"network_transmit_packets"` } incus-7.0.0/internal/server/metrics/metrics.go000066400000000000000000000234751517523235500214470ustar00rootroot00000000000000package metrics import ( "fmt" "slices" "sort" "strconv" "strings" "github.com/lxc/incus/v7/internal/server/auth" ) // ProcsTotal is a gauge according to the OpenMetrics spec as its value can decrease. var metricTypeGauges = []MetricType{ CPUs, GoGoroutines, GoHeapObjects, ProcsTotal, ProjectLimit, ProjectResourcesTotal, ProjectUsage, } // NewMetricSet returns a new MetricSet. func NewMetricSet(labels map[string]string) *MetricSet { out := MetricSet{set: make(map[MetricType][]Sample)} if labels != nil { out.labels = labels } else { out.labels = make(map[string]string) } return &out } // FilterSamples filters the existing MetricSet using the given permission checker. Samples not containing "project" and // "name" labels are skipped. func (m *MetricSet) FilterSamples(permissionChecker func(object auth.Object) bool) { for metricType, samples := range m.set { allowedSamples := make([]Sample, 0, len(samples)) for _, s := range samples { projectName := s.Labels["project"] instanceName := s.Labels["name"] if projectName == "" || instanceName == "" { if permissionChecker(auth.ObjectServer()) { allowedSamples = append(allowedSamples, s) } continue } if permissionChecker(auth.ObjectInstance(projectName, instanceName)) { allowedSamples = append(allowedSamples, s) } } m.set[metricType] = allowedSamples } } // AddSamples adds samples of the type metricType to the MetricSet. func (m *MetricSet) AddSamples(metricType MetricType, samples ...Sample) { for i := range samples { // Add global labels to samples for labelName, labelValue := range m.labels { // Ensure we always have a valid Labels map if samples[i].Labels == nil { samples[i].Labels = make(map[string]string) } samples[i].Labels[labelName] = labelValue } } m.set[metricType] = append(m.set[metricType], samples...) } // AddRaw allows for adding extra metrics directly to the output without having to parse them first. func (m *MetricSet) AddRaw(rawData []byte) { m.suffix = append(m.suffix, rawData...) } // Merge merges two MetricSets. Missing labels from m's samples are added to all samples in n. func (m *MetricSet) Merge(metricSet *MetricSet) { if metricSet == nil { return } for metricType := range metricSet.set { for _, sample := range metricSet.set[metricType] { // Add missing labels from m. for k, v := range m.labels { _, ok := sample.Labels[k] if !ok { sample.Labels[k] = v } } m.set[metricType] = append(m.set[metricType], sample) } } if metricSet.suffix != nil { m.suffix = append(m.suffix, metricSet.suffix...) } } func (m *MetricSet) String() string { var out strings.Builder metricTypes := []MetricType{} // Sort output by metric type name for metricType := range m.set { metricTypes = append(metricTypes, metricType) } sort.SliceStable(metricTypes, func(i, j int) bool { return int(metricTypes[i]) < int(metricTypes[j]) }) for _, metricType := range metricTypes { // Add HELP message as specified by OpenMetrics _, err := out.WriteString(MetricHeaders[metricType] + "\n") if err != nil { return "" } metricTypeName := "" if slices.Contains(metricTypeGauges, metricType) { metricTypeName = "gauge" } else if strings.HasSuffix(MetricNames[metricType], "_total") || strings.HasSuffix(MetricNames[metricType], "_seconds") { metricTypeName = "counter" } else if strings.HasSuffix(MetricNames[metricType], "_bytes") { metricTypeName = "gauge" } // Add TYPE message as specified by OpenMetrics _, err = out.WriteString(fmt.Sprintf("# TYPE %s %s\n", MetricNames[metricType], metricTypeName)) if err != nil { return "" } for _, sample := range m.set[metricType] { firstLabel := true labels := "" labelNames := []string{} // Add and sort labels if there are any for labelName := range sample.Labels { labelNames = append(labelNames, labelName) } sort.Strings(labelNames) for _, labelName := range labelNames { if !firstLabel { labels += "," } labels += fmt.Sprintf(`%s="%s"`, labelName, sample.Labels[labelName]) firstLabel = false } valueStr := strconv.FormatFloat(sample.Value, 'g', -1, 64) if labels != "" { _, err = out.WriteString(fmt.Sprintf("%s{%s} %s\n", MetricNames[metricType], labels, valueStr)) } else { _, err = out.WriteString(fmt.Sprintf("%s %s\n", MetricNames[metricType], valueStr)) } if err != nil { return "" } } } _, err := out.Write(m.suffix) if err != nil { return "" } _, err = out.WriteString("# EOF\n") if err != nil { return "" } return out.String() } // MetricSetFromAPI converts api.Metrics to a MetricSet, and returns it. func MetricSetFromAPI(metrics *Metrics, labels map[string]string) (*MetricSet, error) { set := NewMetricSet(labels) // CPU stats for _, stats := range metrics.CPU { getLabels := func(mode string) map[string]string { labels := map[string]string{"mode": mode} cpu := "" if stats.CPU != "cpu" { _, _ = fmt.Sscanf(stats.CPU, "cpu%s", &cpu) } if cpu != "" { labels["cpu"] = cpu } return labels } set.AddSamples(CPUSecondsTotal, Sample{ Value: stats.SecondsIOWait, Labels: getLabels("iowait"), }, Sample{ Value: stats.SecondsIRQ, Labels: getLabels("irq"), }, Sample{ Value: stats.SecondsIdle, Labels: getLabels("idle"), }, Sample{ Value: stats.SecondsNice, Labels: getLabels("nice"), }, Sample{ Value: stats.SecondsSoftIRQ, Labels: getLabels("softirq"), }, Sample{ Value: stats.SecondsSteal, Labels: getLabels("steal"), }, Sample{ Value: stats.SecondsSystem, Labels: getLabels("system"), }, Sample{ Value: stats.SecondsUser, Labels: getLabels("user"), }, ) } // CPUs set.AddSamples(CPUs, Sample{Value: float64(metrics.CPUs)}) // Disk stats for _, stats := range metrics.Disk { labels := map[string]string{"device": stats.Device} set.AddSamples(DiskReadBytesTotal, Sample{Value: float64(stats.ReadBytes), Labels: labels}) set.AddSamples(DiskReadsCompletedTotal, Sample{Value: float64(stats.ReadsCompleted), Labels: labels}) set.AddSamples(DiskWritesCompletedTotal, Sample{Value: float64(stats.WritesCompleted), Labels: labels}) set.AddSamples(DiskWrittenBytesTotal, Sample{Value: float64(stats.WrittenBytes), Labels: labels}) } // Filesystem stats for _, stats := range metrics.Filesystem { labels := map[string]string{"device": stats.Device, "fstype": stats.FSType, "mountpoint": stats.Mountpoint} set.AddSamples(FilesystemAvailBytes, Sample{Value: float64(stats.AvailableBytes), Labels: labels}) set.AddSamples(FilesystemFreeBytes, Sample{Value: float64(stats.FreeBytes), Labels: labels}) set.AddSamples(FilesystemSizeBytes, Sample{Value: float64(stats.SizeBytes), Labels: labels}) } // Memory stats set.AddSamples(MemoryActiveAnonBytes, Sample{Value: float64(metrics.Memory.ActiveAnonBytes)}) set.AddSamples(MemoryActiveBytes, Sample{Value: float64(metrics.Memory.ActiveBytes)}) set.AddSamples(MemoryActiveFileBytes, Sample{Value: float64(metrics.Memory.ActiveFileBytes)}) set.AddSamples(MemoryCachedBytes, Sample{Value: float64(metrics.Memory.CachedBytes)}) set.AddSamples(MemoryDirtyBytes, Sample{Value: float64(metrics.Memory.DirtyBytes)}) set.AddSamples(MemoryHugePagesFreeBytes, Sample{Value: float64(metrics.Memory.HugepagesFreeBytes)}) set.AddSamples(MemoryHugePagesTotalBytes, Sample{Value: float64(metrics.Memory.HugepagesTotalBytes)}) set.AddSamples(MemoryInactiveAnonBytes, Sample{Value: float64(metrics.Memory.InactiveAnonBytes)}) set.AddSamples(MemoryInactiveBytes, Sample{Value: float64(metrics.Memory.InactiveBytes)}) set.AddSamples(MemoryInactiveFileBytes, Sample{Value: float64(metrics.Memory.InactiveFileBytes)}) set.AddSamples(MemoryMappedBytes, Sample{Value: float64(metrics.Memory.MappedBytes)}) set.AddSamples(MemoryMemAvailableBytes, Sample{Value: float64(metrics.Memory.MemAvailableBytes)}) set.AddSamples(MemoryMemFreeBytes, Sample{Value: float64(metrics.Memory.MemFreeBytes)}) set.AddSamples(MemoryMemTotalBytes, Sample{Value: float64(metrics.Memory.MemTotalBytes)}) set.AddSamples(MemoryRSSBytes, Sample{Value: float64(metrics.Memory.RSSBytes)}) set.AddSamples(MemoryShmemBytes, Sample{Value: float64(metrics.Memory.ShmemBytes)}) set.AddSamples(MemorySwapBytes, Sample{Value: float64(metrics.Memory.SwapBytes)}) set.AddSamples(MemoryUnevictableBytes, Sample{Value: float64(metrics.Memory.UnevictableBytes)}) set.AddSamples(MemoryWritebackBytes, Sample{Value: float64(metrics.Memory.WritebackBytes)}) set.AddSamples(MemoryOOMKillsTotal, Sample{Value: float64(metrics.Memory.OOMKills)}) // Network stats for _, stats := range metrics.Network { labels := map[string]string{"device": stats.Device} set.AddSamples(NetworkReceiveBytesTotal, Sample{Value: float64(stats.ReceiveBytes), Labels: labels}) set.AddSamples(NetworkReceiveDropTotal, Sample{Value: float64(stats.ReceiveDrop), Labels: labels}) set.AddSamples(NetworkReceiveErrsTotal, Sample{Value: float64(stats.ReceiveErrors), Labels: labels}) set.AddSamples(NetworkReceivePacketsTotal, Sample{Value: float64(stats.ReceivePackets), Labels: labels}) set.AddSamples(NetworkTransmitBytesTotal, Sample{Value: float64(stats.TransmitBytes), Labels: labels}) set.AddSamples(NetworkTransmitDropTotal, Sample{Value: float64(stats.TransmitDrop), Labels: labels}) set.AddSamples(NetworkTransmitErrsTotal, Sample{Value: float64(stats.TransmitErrors), Labels: labels}) set.AddSamples(NetworkTransmitPacketsTotal, Sample{Value: float64(stats.TransmitPackets), Labels: labels}) } // Procs stats set.AddSamples(ProcsTotal, Sample{Value: float64(metrics.ProcessesTotal)}) // BootTimeSeconds & TimeSeconds set.AddSamples(BootTimeSeconds, Sample{Value: float64(metrics.BootTimeSeconds)}) set.AddSamples(TimeSeconds, Sample{Value: float64(metrics.TimeSeconds)}) return set, nil } incus-7.0.0/internal/server/metrics/metrics_test.go000066400000000000000000000027161517523235500225010ustar00rootroot00000000000000package metrics import ( "testing" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/auth" ) func TestMetricSet_FilterSamples(t *testing.T) { labels := map[string]string{"project": "default", "name": "jammy"} newMetricSet := func() *MetricSet { m := NewMetricSet(labels) require.Equal(t, labels, m.labels) m.AddSamples(CPUSecondsTotal, Sample{Value: 10}) require.Equal(t, []Sample{{Value: 10, Labels: labels}}, m.set[CPUSecondsTotal]) return m } m := newMetricSet() permissionChecker := func(object auth.Object) bool { return object == auth.ObjectInstance("default", "jammy") } m.FilterSamples(permissionChecker) // Should still contain the sample require.Equal(t, []Sample{{Value: 10, Labels: labels}}, m.set[CPUSecondsTotal]) m = newMetricSet() permissionChecker = func(object auth.Object) bool { return object == auth.ObjectInstance("not-default", "not-jammy") } m.FilterSamples(permissionChecker) // Should no longer contain the sample. require.Equal(t, []Sample{}, m.set[CPUSecondsTotal]) m = NewMetricSet(map[string]string{"project": "default"}) m.AddSamples(CPUSecondsTotal, Sample{Value: 10}) n := NewMetricSet(map[string]string{"name": "jammy"}) n.AddSamples(CPUSecondsTotal, Sample{Value: 20}) m.Merge(n) for _, sample := range m.set[CPUSecondsTotal] { hasKeys := []string{} for k := range sample.Labels { hasKeys = append(hasKeys, k) } require.Contains(t, hasKeys, "project") } } incus-7.0.0/internal/server/metrics/types.go000066400000000000000000000451641517523235500211440ustar00rootroot00000000000000package metrics // A Sample represents an OpenMetrics sample containing labels and the value. type Sample struct { Labels map[string]string Value float64 } // MetricSet represents a set of metrics. type MetricSet struct { set map[MetricType][]Sample labels map[string]string suffix []byte } // MetricType is a numeric code identifying the metric. type MetricType int const ( // BootTimeSeconds represents Unix time when the instance booted. BootTimeSeconds MetricType = iota // CPUSecondsTotal represents the total CPU seconds used. CPUSecondsTotal // CPUs represents the total number of effective CPUs. CPUs // DiskReadBytesTotal represents the read bytes for a disk. DiskReadBytesTotal // DiskReadsCompletedTotal represents the completed for a disk. DiskReadsCompletedTotal // DiskWrittenBytesTotal represents the written bytes for a disk. DiskWrittenBytesTotal // DiskWritesCompletedTotal represents the completed writes for a disk. DiskWritesCompletedTotal // FilesystemAvailBytes represents the available bytes on a filesystem. FilesystemAvailBytes // FilesystemFreeBytes represents the free bytes on a filesystem. FilesystemFreeBytes // FilesystemSizeBytes represents the size in bytes of a filesystem. FilesystemSizeBytes // MemoryActiveAnonBytes represents the amount of anonymous memory on active LRU list. MemoryActiveAnonBytes // MemoryActiveFileBytes represents the amount of file-backed memory on active LRU list. MemoryActiveFileBytes // MemoryActiveBytes represents the amount of memory on active LRU list. MemoryActiveBytes // MemoryCachedBytes represents the amount of cached memory. MemoryCachedBytes // MemoryDirtyBytes represents the amount of memory waiting to get written back to the disk. MemoryDirtyBytes // MemoryHugePagesFreeBytes represents the amount of free memory for hugetlb. MemoryHugePagesFreeBytes // MemoryHugePagesTotalBytes represents the amount of used memory for hugetlb. MemoryHugePagesTotalBytes // MemoryInactiveAnonBytes represents the amount of anonymous memory on inactive LRU list. MemoryInactiveAnonBytes // MemoryInactiveFileBytes represents the amount of file-backed memory on inactive LRU list. MemoryInactiveFileBytes // MemoryInactiveBytes represents the amount of memory on inactive LRU list. MemoryInactiveBytes // MemoryMappedBytes represents the amount of mapped memory. MemoryMappedBytes // MemoryMemAvailableBytes represents the amount of available memory. MemoryMemAvailableBytes // MemoryMemFreeBytes represents the amount of free memory. MemoryMemFreeBytes // MemoryMemTotalBytes represents the amount of used memory. MemoryMemTotalBytes // MemoryRSSBytes represents the amount of anonymous and swap cache memory. MemoryRSSBytes // MemoryShmemBytes represents the amount of cached filesystem data that is swap-backed. MemoryShmemBytes // MemorySwapBytes represents the amount of swap memory. MemorySwapBytes // MemoryUnevictableBytes represents the amount of unevictable memory. MemoryUnevictableBytes // MemoryWritebackBytes represents the amount of memory queued for syncing to disk. MemoryWritebackBytes // MemoryOOMKillsTotal represents the amount of oom kills. MemoryOOMKillsTotal // NetworkReceiveBytesTotal represents the amount of received bytes on a given interface. NetworkReceiveBytesTotal // NetworkReceiveDropTotal represents the amount of received dropped bytes on a given interface. NetworkReceiveDropTotal // NetworkReceiveErrsTotal represents the amount of received errors on a given interface. NetworkReceiveErrsTotal // NetworkReceivePacketsTotal represents the amount of received packets on a given interface. NetworkReceivePacketsTotal // NetworkTransmitBytesTotal represents the amount of transmitted bytes on a given interface. NetworkTransmitBytesTotal // NetworkTransmitDropTotal represents the amount of transmitted dropped bytes on a given interface. NetworkTransmitDropTotal // NetworkTransmitErrsTotal represents the amount of transmitted errors on a given interface. NetworkTransmitErrsTotal // NetworkTransmitPacketsTotal represents the amount of transmitted packets on a given interface. NetworkTransmitPacketsTotal // ProcsTotal represents the number of running processes. ProcsTotal // TimeSeconds represents current Unix time on the instance. TimeSeconds // OperationsTotal represents the number of running operations. OperationsTotal // WarningsTotal represents the number of active warnings. WarningsTotal // UptimeSeconds represents the daemon uptime in seconds. UptimeSeconds // ProjectResourcesTotal represents the current resource count in a project. ProjectResourcesTotal // ProjectLimit represents the current project resource limit. ProjectLimit // ProjectUsage represents the current project resource usage. ProjectUsage // GoGoroutines represents the number of goroutines that currently exist.. GoGoroutines // GoAllocBytes represents the number of bytes allocated and still in use. GoAllocBytes // GoAllocBytesTotal represents the total number of bytes allocated, even if freed. GoAllocBytesTotal // GoSysBytes represents the number of bytes obtained from system. GoSysBytes // GoLookupsTotal represents the total number of pointer lookups. GoLookupsTotal // GoMallocsTotal represents the total number of mallocs. GoMallocsTotal // GoFreesTotal represents the total number of frees. GoFreesTotal // GoHeapAllocBytes represents the number of heap bytes allocated and still in use. GoHeapAllocBytes // GoHeapSysBytes represents the number of heap bytes obtained from system. GoHeapSysBytes // GoHeapIdleBytes represents the number of heap bytes waiting to be used. GoHeapIdleBytes // GoHeapInuseBytes represents the number of heap bytes that are in use. GoHeapInuseBytes // GoHeapReleasedBytes represents the number of heap bytes released to OS. GoHeapReleasedBytes // GoHeapObjects represents the number of allocated objects. GoHeapObjects // GoStackInuseBytes represents the number of bytes in use by the stack allocator. GoStackInuseBytes // GoStackSysBytes represents the number of bytes obtained from system for stack allocator. GoStackSysBytes // GoMSpanInuseBytes represents the number of bytes in use by mspan structures. GoMSpanInuseBytes // GoMSpanSysBytes represents the number of bytes used for mspan structures obtained from system. GoMSpanSysBytes // GoMCacheInuseBytes represents the number of bytes in use by mcache structures. GoMCacheInuseBytes // GoMCacheSysBytes represents the number of bytes used for mcache structures obtained from system. GoMCacheSysBytes // GoBuckHashSysBytes represents the number of bytes used by the profiling bucket hash table. GoBuckHashSysBytes // GoGCSysBytes represents the number of bytes used for garbage collection system metadata. GoGCSysBytes // GoOtherSysBytes represents the number of bytes used for other system allocations. GoOtherSysBytes // GoNextGCBytes represents the number of heap bytes when next garbage collection will take place. GoNextGCBytes ) // MetricNames associates a metric type to its name. var MetricNames = map[MetricType]string{ BootTimeSeconds: "incus_boot_time_seconds", CPUSecondsTotal: "incus_cpu_seconds_total", CPUs: "incus_cpu_effective_total", DiskReadBytesTotal: "incus_disk_read_bytes_total", DiskReadsCompletedTotal: "incus_disk_reads_completed_total", DiskWrittenBytesTotal: "incus_disk_written_bytes_total", DiskWritesCompletedTotal: "incus_disk_writes_completed_total", FilesystemAvailBytes: "incus_filesystem_avail_bytes", FilesystemFreeBytes: "incus_filesystem_free_bytes", FilesystemSizeBytes: "incus_filesystem_size_bytes", GoAllocBytes: "incus_go_alloc_bytes", GoAllocBytesTotal: "incus_go_alloc_bytes_total", GoBuckHashSysBytes: "incus_go_buck_hash_sys_bytes", GoFreesTotal: "incus_go_frees_total", GoGCSysBytes: "incus_go_gc_sys_bytes", GoGoroutines: "incus_go_goroutines", GoHeapAllocBytes: "incus_go_heap_alloc_bytes", GoHeapIdleBytes: "incus_go_heap_idle_bytes", GoHeapInuseBytes: "incus_go_heap_inuse_bytes", GoHeapObjects: "incus_go_heap_objects", GoHeapReleasedBytes: "incus_go_heap_released_bytes", GoHeapSysBytes: "incus_go_heap_sys_bytes", GoLookupsTotal: "incus_go_lookups_total", GoMallocsTotal: "incus_go_mallocs_total", GoMCacheInuseBytes: "incus_go_mcache_inuse_bytes", GoMCacheSysBytes: "incus_go_mcache_sys_bytes", GoMSpanInuseBytes: "incus_go_mspan_inuse_bytes", GoMSpanSysBytes: "incus_go_mspan_sys_bytes", GoNextGCBytes: "incus_go_next_gc_bytes", GoOtherSysBytes: "incus_go_other_sys_bytes", GoStackInuseBytes: "incus_go_stack_inuse_bytes", GoStackSysBytes: "incus_go_stack_sys_bytes", GoSysBytes: "incus_go_sys_bytes", MemoryActiveAnonBytes: "incus_memory_Active_anon_bytes", MemoryActiveFileBytes: "incus_memory_Active_file_bytes", MemoryActiveBytes: "incus_memory_Active_bytes", MemoryCachedBytes: "incus_memory_Cached_bytes", MemoryDirtyBytes: "incus_memory_Dirty_bytes", MemoryHugePagesFreeBytes: "incus_memory_HugepagesFree_bytes", MemoryHugePagesTotalBytes: "incus_memory_HugepagesTotal_bytes", MemoryInactiveAnonBytes: "incus_memory_Inactive_anon_bytes", MemoryInactiveFileBytes: "incus_memory_Inactive_file_bytes", MemoryInactiveBytes: "incus_memory_Inactive_bytes", MemoryMappedBytes: "incus_memory_Mapped_bytes", MemoryMemAvailableBytes: "incus_memory_MemAvailable_bytes", MemoryMemFreeBytes: "incus_memory_MemFree_bytes", MemoryMemTotalBytes: "incus_memory_MemTotal_bytes", MemoryRSSBytes: "incus_memory_RSS_bytes", MemoryShmemBytes: "incus_memory_Shmem_bytes", MemorySwapBytes: "incus_memory_Swap_bytes", MemoryUnevictableBytes: "incus_memory_Unevictable_bytes", MemoryWritebackBytes: "incus_memory_Writeback_bytes", MemoryOOMKillsTotal: "incus_memory_OOM_kills_total", NetworkReceiveBytesTotal: "incus_network_receive_bytes_total", NetworkReceiveDropTotal: "incus_network_receive_drop_total", NetworkReceiveErrsTotal: "incus_network_receive_errs_total", NetworkReceivePacketsTotal: "incus_network_receive_packets_total", NetworkTransmitBytesTotal: "incus_network_transmit_bytes_total", NetworkTransmitDropTotal: "incus_network_transmit_drop_total", NetworkTransmitErrsTotal: "incus_network_transmit_errs_total", NetworkTransmitPacketsTotal: "incus_network_transmit_packets_total", OperationsTotal: "incus_operations_total", ProcsTotal: "incus_procs_total", ProjectLimit: "incus_project_limit", ProjectResourcesTotal: "incus_project_resources_total", ProjectUsage: "incus_project_usage", TimeSeconds: "incus_time_seconds", UptimeSeconds: "incus_uptime_seconds", WarningsTotal: "incus_warnings_total", } // MetricHeaders represents the metric headers which contain help messages as specified by OpenMetrics. var MetricHeaders = map[MetricType]string{ BootTimeSeconds: "# HELP incus_boot_time_seconds The unix epoch at the time of the instance start.", CPUSecondsTotal: "# HELP incus_cpu_seconds_total The total number of CPU time used in seconds.", CPUs: "# HELP incus_cpu_effective_total The total number of effective CPUs.", DiskReadBytesTotal: "# HELP incus_disk_read_bytes_total The total number of bytes read.", DiskReadsCompletedTotal: "# HELP incus_disk_reads_completed_total The total number of completed reads.", DiskWrittenBytesTotal: "# HELP incus_disk_written_bytes_total The total number of bytes written.", DiskWritesCompletedTotal: "# HELP incus_disk_writes_completed_total The total number of completed writes.", FilesystemAvailBytes: "# HELP incus_filesystem_avail_bytes The number of available space in bytes.", FilesystemFreeBytes: "# HELP incus_filesystem_free_bytes The number of free space in bytes.", FilesystemSizeBytes: "# HELP incus_filesystem_size_bytes The size of the filesystem in bytes.", GoAllocBytes: "# HELP incus_go_alloc_bytes Number of bytes allocated and still in use.", GoAllocBytesTotal: "# HELP incus_go_alloc_bytes_total Total number of bytes allocated, even if freed.", GoBuckHashSysBytes: "# HELP incus_go_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.", GoFreesTotal: "# HELP incus_go_frees_total Total number of frees.", GoGCSysBytes: "# HELP incus_go_gc_sys_bytes Number of bytes used for garbage collection system metadata.", GoGoroutines: "# HELP incus_go_goroutines Number of goroutines that currently exist.", GoHeapAllocBytes: "# HELP incus_go_heap_alloc_bytes Number of heap bytes allocated and still in use.", GoHeapIdleBytes: "# HELP incus_go_heap_idle_bytes Number of heap bytes waiting to be used.", GoHeapInuseBytes: "# HELP incus_go_heap_inuse_bytes Number of heap bytes that are in use.", GoHeapObjects: "# HELP incus_go_heap_objects Number of allocated objects.", GoHeapReleasedBytes: "# HELP incus_go_heap_released_bytes Number of heap bytes released to OS.", GoHeapSysBytes: "# HELP incus_go_heap_sys_bytes Number of heap bytes obtained from system.", GoLookupsTotal: "# HELP incus_go_lookups_total Total number of pointer lookups.", GoMallocsTotal: "# HELP incus_go_mallocs_total Total number of mallocs.", GoMCacheInuseBytes: "# HELP incus_go_mcache_inuse_bytes Number of bytes in use by mcache structures.", GoMCacheSysBytes: "# HELP incus_go_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.", GoMSpanInuseBytes: "# HELP incus_go_mspan_inuse_bytes Number of bytes in use by mspan structures.", GoMSpanSysBytes: "# HELP incus_go_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.", GoNextGCBytes: "# HELP incus_go_next_gc_bytes Number of heap bytes when next garbage collection will take place.", GoOtherSysBytes: "# HELP incus_go_other_sys_bytes Number of bytes used for other system allocations.", GoStackInuseBytes: "# HELP incus_go_stack_inuse_bytes Number of bytes in use by the stack allocator.", GoStackSysBytes: "# HELP incus_go_stack_sys_bytes Number of bytes obtained from system for stack allocator.", GoSysBytes: "# HELP incus_go_sys_bytes Number of bytes obtained from system.", MemoryActiveAnonBytes: "# HELP incus_memory_Active_anon_bytes The amount of anonymous memory on active LRU list.", MemoryActiveFileBytes: "# HELP incus_memory_Active_file_bytes The amount of file-backed memory on active LRU list.", MemoryActiveBytes: "# HELP incus_memory_Active_bytes The amount of memory on active LRU list.", MemoryCachedBytes: "# HELP incus_memory_Cached_bytes The amount of cached memory.", MemoryDirtyBytes: "# HELP incus_memory_Dirty_bytes The amount of memory waiting to get written back to the disk.", MemoryHugePagesFreeBytes: "# HELP incus_memory_HugepagesFree_bytes The amount of free memory for hugetlb.", MemoryHugePagesTotalBytes: "# HELP incus_memory_HugepagesTotal_bytes The amount of used memory for hugetlb.", MemoryInactiveAnonBytes: "# HELP incus_memory_Inactive_anon_bytes The amount of anonymous memory on inactive LRU list.", MemoryInactiveFileBytes: "# HELP incus_memory_Inactive_file_bytes The amount of file-backed memory on inactive LRU list.", MemoryInactiveBytes: "# HELP incus_memory_Inactive_bytes The amount of memory on inactive LRU list.", MemoryMappedBytes: "# HELP incus_memory_Mapped_bytes The amount of mapped memory.", MemoryMemAvailableBytes: "# HELP incus_memory_MemAvailable_bytes The amount of available memory.", MemoryMemFreeBytes: "# HELP incus_memory_MemFree_bytes The amount of free memory.", MemoryMemTotalBytes: "# HELP incus_memory_MemTotal_bytes The amount of used memory.", MemoryRSSBytes: "# HELP incus_memory_RSS_bytes The amount of anonymous and swap cache memory.", MemoryShmemBytes: "# HELP incus_memory_Shmem_bytes The amount of cached filesystem data that is swap-backed.", MemorySwapBytes: "# HELP incus_memory_Swap_bytes The amount of used swap memory.", MemoryUnevictableBytes: "# HELP incus_memory_Unevictable_bytes The amount of unevictable memory.", MemoryWritebackBytes: "# HELP incus_memory_Writeback_bytes The amount of memory queued for syncing to disk.", MemoryOOMKillsTotal: "# HELP incus_memory_OOM_kills_total The number of out of memory kills.", NetworkReceiveBytesTotal: "# HELP incus_network_receive_bytes_total The amount of received bytes on a given interface.", NetworkReceiveDropTotal: "# HELP incus_network_receive_drop_total The amount of received dropped bytes on a given interface.", NetworkReceiveErrsTotal: "# HELP incus_network_receive_errs_total The amount of received errors on a given interface.", NetworkReceivePacketsTotal: "# HELP incus_network_receive_packets_total The amount of received packets on a given interface.", NetworkTransmitBytesTotal: "# HELP incus_network_transmit_bytes_total The amount of transmitted bytes on a given interface.", NetworkTransmitDropTotal: "# HELP incus_network_transmit_drop_total The amount of transmitted dropped bytes on a given interface.", NetworkTransmitErrsTotal: "# HELP incus_network_transmit_errs_total The amount of transmitted errors on a given interface.", NetworkTransmitPacketsTotal: "# HELP incus_network_transmit_packets_total The amount of transmitted packets on a given interface.", OperationsTotal: "# HELP incus_operations_total The number of running operations", ProcsTotal: "# HELP incus_procs_total The number of running processes.", ProjectLimit: "# HELP incus_project_limit Current project resource limit.", ProjectResourcesTotal: "# HELP incus_project_resources_total Current resource count in a project.", ProjectUsage: "# HELP incus_project_usage Current project resource usage.", TimeSeconds: "# HELP incus_time_seconds The current unix epoch.", UptimeSeconds: "# HELP incus_uptime_seconds The daemon uptime in seconds.", WarningsTotal: "# HELP incus_warnings_total The number of active warnings.", } incus-7.0.0/internal/server/migration/000077500000000000000000000000001517523235500177625ustar00rootroot00000000000000incus-7.0.0/internal/server/migration/migration_volumes.go000066400000000000000000000333661517523235500240670ustar00rootroot00000000000000package migration import ( "fmt" "io" "net/http" "slices" "google.golang.org/protobuf/proto" "github.com/lxc/incus/v7/internal/migration" backupConfig "github.com/lxc/incus/v7/internal/server/backup/config" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/units" ) // Info represents the index frame sent if supported. type Info struct { Config *backupConfig.Config `json:"config,omitempty" yaml:"config,omitempty"` // Equivalent of backup.yaml but embedded in index. } // InfoResponse represents the response to the index frame sent if supported. // Right now this doesn't contain anything useful, its just used to indicate receipt of the index header. // But in the future the intention is to use it allow the target to send back additional information to the source // about which frames (such as snapshots) it needs for the migration after having inspected the Info index header. type InfoResponse struct { StatusCode int Error string Refresh *bool // This is used to let the source know whether to actually refresh a volume. } // Err returns the error of the response. func (r *InfoResponse) Err() error { if r.StatusCode != http.StatusOK { return api.StatusErrorf(r.StatusCode, "%s", r.Error) } return nil } // Type represents the migration transport type. It indicates the method by which the migration can // take place and what optional features are available. type Type struct { FSType migration.MigrationFSType // Transport mode selected. Features []string // Feature hints for selected FSType transport mode. } // DependentVolumeArgs represents the arguments needed to set up a dependent volume migration. type DependentVolumeArgs struct { Name string Pool string DeviceName string ContentType string MigrationType Type Snapshots []*migration.Snapshot VolumeSize int64 } // VolumeSourceArgs represents the arguments needed to setup a volume migration source. type VolumeSourceArgs struct { IndexHeaderVersion uint32 Name string Snapshots []string MigrationType Type TrackProgress bool MultiSync bool FinalSync bool Data any // Optional store to persist storage driver state between MultiSync phases. ContentType string AllowInconsistent bool Refresh bool Info *Info VolumeOnly bool ClusterMove bool StorageMove bool DependentVolumes []DependentVolumeArgs } // VolumeTargetArgs represents the arguments needed to setup a volume migration sink. type VolumeTargetArgs struct { IndexHeaderVersion uint32 Name string Description string Config map[string]string // Only used for custom volume migration. Snapshots []*migration.Snapshot MigrationType Type TrackProgress bool Refresh bool RefreshExcludeOlder bool Live bool VolumeSize int64 ContentType string VolumeOnly bool ClusterMoveSourceName string StoragePool string DependentVolumes []DependentVolumeArgs } // TypesToHeader converts one or more Types to a MigrationHeader. It uses the first type argument // supplied to indicate the preferred migration method and sets the MigrationHeader's Fs type // to that. If the preferred type is ZFS then it will also set the header's optional ZfsFeatures. // If the fallback Rsync type is present in any of the types even if it is not preferred, then its // optional features are added to the header's RsyncFeatures, allowing for fallback negotiation to // take place on the farside. func TypesToHeader(types ...Type) *migration.MigrationHeader { missingFeature := false hasFeature := true var preferredType Type if len(types) > 0 { preferredType = types[0] } header := migration.MigrationHeader{Fs: &preferredType.FSType} // Add ZFS features if preferred type is ZFS. if preferredType.FSType == migration.MigrationFSType_ZFS { features := migration.ZfsFeatures{ Compress: &missingFeature, } for _, feature := range preferredType.Features { switch feature { case "compress": features.Compress = &hasFeature case migration.ZFSFeatureMigrationHeader: features.MigrationHeader = &hasFeature case migration.ZFSFeatureZvolFilesystems: features.HeaderZvols = &hasFeature } } header.ZfsFeatures = &features } // Add BTRFS features if preferred type is BTRFS. if preferredType.FSType == migration.MigrationFSType_BTRFS { features := migration.BtrfsFeatures{ MigrationHeader: &missingFeature, HeaderSubvolumes: &missingFeature, } for _, feature := range preferredType.Features { switch feature { case migration.BTRFSFeatureMigrationHeader: features.MigrationHeader = &hasFeature case migration.BTRFSFeatureSubvolumes: features.HeaderSubvolumes = &hasFeature case migration.BTRFSFeatureSubvolumeUUIDs: features.HeaderSubvolumeUuids = &hasFeature } } header.BtrfsFeatures = &features } // Check all the types for an Rsync method, if found add its features to the header's RsyncFeatures list. for _, t := range types { if t.FSType != migration.MigrationFSType_RSYNC && t.FSType != migration.MigrationFSType_BLOCK_AND_RSYNC { continue } features := migration.RsyncFeatures{ Xattrs: &missingFeature, Delete: &missingFeature, Compress: &missingFeature, Bidirectional: &missingFeature, } for _, feature := range t.Features { switch feature { case "xattrs": features.Xattrs = &hasFeature case "delete": features.Delete = &hasFeature case "compress": features.Compress = &hasFeature case "bidirectional": features.Bidirectional = &hasFeature } } header.RsyncFeatures = &features break // Only use the first rsync transport type found to generate rsync features list. } return &header } // MatchTypes attempts to find matching migration transport types between an offered type sent from a remote // source and the types supported by a local storage pool. If matches are found then one or more Types are // returned containing the method and the matching optional features present in both. The function also takes a // fallback type which is used as an additional offer type preference in case the preferred remote type is not // compatible with the local type available. It is expected that both sides of the migration will support the // fallback type for the volume's content type that is being migrated. func MatchTypes(offer *migration.MigrationHeader, fallbackType migration.MigrationFSType, ourTypes []Type) ([]Type, error) { // Generate an offer types slice from the preferred type supplied from remote and the // fallback type supplied based on the content type of the transfer. offeredFSTypes := []migration.MigrationFSType{offer.GetFs(), fallbackType} matchedTypes := []Type{} // Find first matching type. for _, ourType := range ourTypes { for _, offerFSType := range offeredFSTypes { if offerFSType != ourType.FSType { continue // Not a match, try the next one. } // We got a match, now extract the relevant offered features. var offeredFeatures []string switch offerFSType { case migration.MigrationFSType_ZFS: offeredFeatures = offer.GetZfsFeaturesSlice() case migration.MigrationFSType_BTRFS: offeredFeatures = offer.GetBtrfsFeaturesSlice() case migration.MigrationFSType_RSYNC: offeredFeatures = offer.GetRsyncFeaturesSlice() } // Find common features in both our type and offered type. commonFeatures := []string{} for _, ourFeature := range ourType.Features { if slices.Contains(offeredFeatures, ourFeature) { commonFeatures = append(commonFeatures, ourFeature) } } if offer.GetRefresh() { // Optimized refresh with zfs only works if ZfsFeatureMigrationHeader is available. if ourType.FSType == migration.MigrationFSType_ZFS && !slices.Contains(commonFeatures, migration.ZFSFeatureMigrationHeader) { continue } // Optimized refresh with btrfs only works if BtrfsFeatureSubvolumeUUIDs is available. if ourType.FSType == migration.MigrationFSType_BTRFS && !slices.Contains(commonFeatures, migration.BTRFSFeatureSubvolumeUUIDs) { continue } } // Append type with combined features. matchedTypes = append(matchedTypes, Type{ FSType: ourType.FSType, Features: commonFeatures, }) } } if len(matchedTypes) < 1 { // No matching transport type found, generate an error with offered types and our types. offeredTypeStrings := make([]string, 0, len(offeredFSTypes)) for _, offerFSType := range offeredFSTypes { offeredTypeStrings = append(offeredTypeStrings, offerFSType.String()) } ourTypeStrings := make([]string, 0, len(ourTypes)) for _, ourType := range ourTypes { ourTypeStrings = append(ourTypeStrings, ourType.FSType.String()) } return matchedTypes, fmt.Errorf("No matching migration types found. Offered types: %v, our types: %v", offeredTypeStrings, ourTypeStrings) } return matchedTypes, nil } // DependentVolumeFromHeader creates a DependentVolume from a MigrationHeader. func DependentVolumeFromHeader(header *migration.MigrationHeader, volName string, poolName string, contentType string, volSize int64, deviceName string) *migration.DependentVolume { fs := header.GetFs() return &migration.DependentVolume{ Fs: &fs, BtrfsFeatures: header.GetBtrfsFeatures(), RsyncFeatures: header.GetRsyncFeatures(), ZfsFeatures: header.GetZfsFeatures(), Name: &volName, Pool: &poolName, ContentType: &contentType, VolumeSize: &volSize, DeviceName: &deviceName, } } // DependentVolumeUpdateHeader updates the migration header for a dependent volume. func DependentVolumeUpdateHeader(header *migration.MigrationHeader, volume *migration.DependentVolume) { fs := header.GetFs() volume.Fs = &fs volume.BtrfsFeatures = header.BtrfsFeatures volume.RsyncFeatures = header.RsyncFeatures volume.ZfsFeatures = header.ZfsFeatures } // HeaderFromDependentVolume creates a MigrationHeader from a DependentVolume. func HeaderFromDependentVolume(volume *migration.DependentVolume) *migration.MigrationHeader { return &migration.MigrationHeader{ Fs: volume.Fs, BtrfsFeatures: volume.GetBtrfsFeatures(), RsyncFeatures: volume.GetRsyncFeatures(), ZfsFeatures: volume.GetZfsFeatures(), } } // ProtobufToDependentVolume converts a migration.DependentVolume into DependentVolumeArgs. func ProtobufToDependentVolume(volume *migration.DependentVolume, migrationType Type, overrides map[string]string) DependentVolumeArgs { volName := *volume.Name poolName := *volume.Pool if overrides != nil { if overrides["pool"] != "" { poolName = overrides["pool"] } if overrides["source"] != "" { volName = overrides["source"] } } return DependentVolumeArgs{ Name: volName, Pool: poolName, DeviceName: *volume.DeviceName, ContentType: *volume.ContentType, MigrationType: migrationType, Snapshots: volume.Snapshots, VolumeSize: *volume.VolumeSize, } } // VolumeSnapshotToProtobuf converts an api.StorageVolumeSnapshot into a migration.Snapshot. func VolumeSnapshotToProtobuf(vol *api.StorageVolumeSnapshot) *migration.Snapshot { config := []*migration.Config{} for k, v := range vol.Config { kCopy := string(k) vCopy := string(v) config = append(config, &migration.Config{Key: &kCopy, Value: &vCopy}) } return &migration.Snapshot{ Name: &vol.Name, LocalConfig: config, Profiles: []string{}, Ephemeral: proto.Bool(false), LocalDevices: []*migration.Device{}, Architecture: proto.Int32(0), Stateful: proto.Bool(false), CreationDate: proto.Int64(vol.CreatedAt.UnixNano()), ExpiryDate: proto.Int64(vol.CreatedAt.UnixNano()), } } func progressWrapperRender(op *operations.Operation, key string, description string, progressInt int64, speedInt int64) { meta := map[string]any{} progress := fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(progressInt, 2), units.GetByteSizeString(speedInt, 2)) if description != "" { progress = fmt.Sprintf("%s: %s (%s/s)", description, units.GetByteSizeString(progressInt, 2), units.GetByteSizeString(speedInt, 2)) } if meta[key] != progress { meta[key] = progress _ = op.ExtendMetadata(meta) } } // ProgressReader reports the read progress. func ProgressReader(op *operations.Operation, key string, description string) func(io.ReadCloser) io.ReadCloser { return func(reader io.ReadCloser) io.ReadCloser { if op == nil { return reader } progress := func(progressInt int64, speedInt int64) { progressWrapperRender(op, key, description, progressInt, speedInt) } readPipe := &ioprogress.ProgressReader{ ReadCloser: reader, Tracker: &ioprogress.ProgressTracker{ Handler: progress, }, } return readPipe } } // ProgressWriter reports the write progress. func ProgressWriter(op *operations.Operation, key string, description string) func(io.WriteCloser) io.WriteCloser { return func(writer io.WriteCloser) io.WriteCloser { if op == nil { return writer } progress := func(progressInt int64, speedInt int64) { progressWrapperRender(op, key, description, progressInt, speedInt) } writePipe := &ioprogress.ProgressWriter{ WriteCloser: writer, Tracker: &ioprogress.ProgressTracker{ Handler: progress, }, } return writePipe } } // ProgressTracker returns a migration I/O tracker. func ProgressTracker(op *operations.Operation, key string, description string) *ioprogress.ProgressTracker { progress := func(progressInt int64, speedInt int64) { progressWrapperRender(op, key, description, progressInt, speedInt) } tracker := &ioprogress.ProgressTracker{ Handler: progress, } return tracker } incus-7.0.0/internal/server/migration/utils.go000066400000000000000000000016561517523235500214610ustar00rootroot00000000000000package migration import ( "fmt" "github.com/lxc/incus/v7/internal/migration" ) // IndexHeaderVersion version of the index header to be sent/recv. const IndexHeaderVersion uint32 = 1 // ControlResponse encapsulates MigrationControl with a receive error. type ControlResponse struct { migration.MigrationControl Err error } const ( unableToLiveMigrate = "Unable to perform live container migration." toMigrateLive = "To migrate the container, stop the container before migration or install CRIU" ) // nolint:revive var ( ErrNoLiveMigrationSource = fmt.Errorf("%s CRIU isn't installed on the source server. %s on the source server", unableToLiveMigrate, toMigrateLive) ErrNoLiveMigrationTarget = fmt.Errorf("%s CRIU isn't installed on the target server. %s on the target server", unableToLiveMigrate, toMigrateLive) ErrNoLiveMigration = fmt.Errorf("%s CRIU isn't installed. %s", unableToLiveMigrate, toMigrateLive) ) incus-7.0.0/internal/server/network/000077500000000000000000000000001517523235500174625ustar00rootroot00000000000000incus-7.0.0/internal/server/network/acl/000077500000000000000000000000001517523235500202215ustar00rootroot00000000000000incus-7.0.0/internal/server/network/acl/acl_bridge.go000066400000000000000000000021061517523235500226220ustar00rootroot00000000000000package acl import ( "fmt" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/logger" ) // BridgeUpdateACLs forces the update of all NIC devices who have the changed ACL applied. func BridgeUpdateACLs(s *state.State, l logger.Logger, aclProjectName string, aclNetDevices map[string]NetworkACLUsage) error { // Update of the bridge NICs affected by the ACL change for _, aclNetDevice := range aclNetDevices { inst, err := instance.LoadByProjectAndName(s, aclProjectName, aclNetDevice.InstanceName) if err != nil { return err } // Skip remote instances. if inst.Location() != "" && inst.Location() != s.ServerName { continue } // Skip stopped instances if !inst.IsRunning() { continue } // Trigger the device update. err = inst.ReloadDevice(aclNetDevice.DeviceName) if err != nil { return fmt.Errorf("Failed to trigger device update for device %q of instance %q in project %q: %w", aclNetDevice.DeviceName, inst.Name(), inst.Project().Name, err) } } return nil } incus-7.0.0/internal/server/network/acl/acl_firewall.go000066400000000000000000000116361517523235500232030ustar00rootroot00000000000000package acl import ( "context" "fmt" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" firewallDrivers "github.com/lxc/incus/v7/internal/server/firewall/drivers" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) // FirewallApplyACLRules applies ACL rules to network firewall. func FirewallApplyACLRules(s *state.State, logger logger.Logger, aclProjectName string, aclNet NetworkACLUsage) error { rules, err := FirewallACLRules(s, aclNet.Name, aclProjectName, aclNet.Config) if err != nil { return err } return s.Firewall.NetworkApplyACLRules(aclNet.Name, rules) } // FirewallACLRules returns ACL rules for network firewall. func FirewallACLRules(s *state.State, aclDeviceName string, aclProjectName string, config map[string]string) ([]firewallDrivers.ACLRule, error) { var dropRules []firewallDrivers.ACLRule var rejectRules []firewallDrivers.ACLRule var allowRules []firewallDrivers.ACLRule var allowStatelessRules []firewallDrivers.ACLRule // convertACLRules converts the ACL rules to Firewall ACL rules. convertACLRules := func(direction string, logPrefix string, rules ...api.NetworkACLRule) error { for ruleIndex, rule := range rules { if rule.State == "disabled" { continue } firewallACLRule := firewallDrivers.ACLRule{ Direction: direction, Action: rule.Action, Source: rule.Source, Destination: rule.Destination, Protocol: rule.Protocol, SourcePort: rule.SourcePort, DestinationPort: rule.DestinationPort, ICMPType: rule.ICMPType, ICMPCode: rule.ICMPCode, } if rule.State == "logged" { firewallACLRule.Log = true // Max 29 chars. firewallACLRule.LogName = fmt.Sprintf("%s-%s-%d", logPrefix, direction, ruleIndex) } switch { case rule.Action == "drop": dropRules = append(dropRules, firewallACLRule) case rule.Action == "reject": rejectRules = append(rejectRules, firewallACLRule) case rule.Action == "allow": allowRules = append(allowRules, firewallACLRule) case rule.Action == "allow-stateless": // TODO: add NOTRACK support allowStatelessRules = append(allowStatelessRules, firewallACLRule) default: return fmt.Errorf("Unrecognised action %q", rule.Action) } } return nil } logPrefix := aclDeviceName // Load ACLs specified by network. for _, aclName := range util.SplitNTrimSpace(config["security.acls"], ",", -1, true) { var aclInfo *api.NetworkACL err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error _, aclInfo, err = dbCluster.GetNetworkACLAPI(ctx, tx.Tx(), aclProjectName, aclName) return err }) if err != nil { return nil, fmt.Errorf("Failed loading ACL %q for network %q: %w", aclName, aclDeviceName, err) } err = convertACLRules("ingress", logPrefix, aclInfo.Ingress...) if err != nil { return nil, fmt.Errorf("Failed converting ACL %q ingress rules for network %q: %w", aclInfo.Name, aclDeviceName, err) } err = convertACLRules("egress", logPrefix, aclInfo.Egress...) if err != nil { return nil, fmt.Errorf("Failed converting ACL %q egress rules for network %q: %w", aclInfo.Name, aclDeviceName, err) } } var rules []firewallDrivers.ACLRule rules = append(rules, dropRules...) rules = append(rules, rejectRules...) rules = append(rules, allowRules...) rules = append(rules, allowStatelessRules...) // Add the automatic default ACL rule for the network. egressAction, egressLogged := firewallACLDefaults(config, "egress") ingressAction, ingressLogged := firewallACLDefaults(config, "ingress") rules = append(rules, firewallDrivers.ACLRule{ Direction: "egress", Action: egressAction, Log: egressLogged, LogName: fmt.Sprintf("%s-egress", logPrefix), }) rules = append(rules, firewallDrivers.ACLRule{ Direction: "ingress", Action: ingressAction, Log: ingressLogged, LogName: fmt.Sprintf("%s-ingress", logPrefix), }) return rules, nil } // firewallACLDefaults returns the action and logging mode to use for the specified direction's default rule. // If the security.acls.default.{in,e}gress.action or security.acls.default.{in,e}gress.logged settings are not // specified in the network config, then it returns "reject" and false respectively. func firewallACLDefaults(netConfig map[string]string, direction string) (string, bool) { defaults := map[string]string{ fmt.Sprintf("security.acls.default.%s.action", direction): "reject", fmt.Sprintf("security.acls.default.%s.logged", direction): "false", } for k := range defaults { if netConfig[k] != "" { defaults[k] = netConfig[k] } } return defaults[fmt.Sprintf("security.acls.default.%s.action", direction)], util.IsTrue(defaults[fmt.Sprintf("security.acls.default.%s.logged", direction)]) } incus-7.0.0/internal/server/network/acl/acl_interface.go000066400000000000000000000013671517523235500233360ustar00rootroot00000000000000package acl import ( "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" ) // NetworkACL represents a Network ACL. type NetworkACL interface { // Initialize. init(state *state.State, id int64, projectName string, aclInfo *api.NetworkACL) // Info. ID() int64 Project() string Info() *api.NetworkACL Etag() []any UsedBy() ([]string, error) // GetLog. GetLog(clientType request.ClientType) (string, error) // Internal validation. validateName(name string) error validateConfig(config *api.NetworkACLPut) error // Modifications. Update(config *api.NetworkACLPut, clientType request.ClientType) error Rename(newName string) error Delete() error } incus-7.0.0/internal/server/network/acl/acl_load.go000066400000000000000000000315361517523235500223160ustar00rootroot00000000000000package acl import ( "context" "fmt" "slices" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) // LoadByName loads and initializes a Network ACL from the database by project and name. func LoadByName(s *state.State, projectName string, name string) (NetworkACL, error) { var id int var aclInfo *api.NetworkACL err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error id, aclInfo, err = cluster.GetNetworkACLAPI(ctx, tx.Tx(), projectName, name) return err }) if err != nil { return nil, err } var acl NetworkACL = &common{} // Only a single driver currently. acl.init(s, int64(id), projectName, aclInfo) return acl, nil } // Create validates supplied record and creates new Network ACL record in the database. func Create(s *state.State, projectName string, aclInfo *api.NetworkACLsPost) error { var acl NetworkACL = &common{} // Only a single driver currently. acl.init(s, -1, projectName, nil) err := acl.validateName(aclInfo.Name) if err != nil { return err } err = acl.validateConfig(&aclInfo.NetworkACLPut) if err != nil { return err } err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Insert DB record. acl := cluster.NetworkACL{ Project: projectName, Name: aclInfo.Name, Description: aclInfo.Description, Ingress: aclInfo.Ingress, Egress: aclInfo.Egress, } id, err := cluster.CreateNetworkACL(ctx, tx.Tx(), acl) if err != nil { return err } if aclInfo.Config != nil { err := cluster.CreateNetworkACLConfig(ctx, tx.Tx(), id, aclInfo.Config) if err != nil { return err } } return nil }) if err != nil { return err } return nil } // Exists checks the ACL name(s) provided exists in the project. // If multiple names are provided, also checks that duplicate names aren't specified in the list. func Exists(s *state.State, projectName string, name ...string) error { var existingACLNames []string err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { acls, err := cluster.GetNetworkACLs(ctx, tx.Tx(), cluster.NetworkACLFilter{Project: &projectName}) if err != nil { return err } existingACLNames = make([]string, len(acls)) for i, acl := range acls { existingACLNames[i] = acl.Name } return nil }) if err != nil { return err } checkedACLNames := make(map[string]struct{}, len(name)) for _, aclName := range name { if !slices.Contains(existingACLNames, aclName) { return fmt.Errorf("Network ACL %q does not exist", aclName) } _, found := checkedACLNames[aclName] if found { return fmt.Errorf("Network ACL %q specified multiple times", aclName) } checkedACLNames[aclName] = struct{}{} } return nil } // UsedBy finds all networks, profiles and instance NICs that use any of the specified ACLs and executes usageFunc // once for each resource using one or more of the ACLs with info about the resource and matched ACLs being used. func UsedBy(s *state.State, aclProjectName string, usageFunc func(ctx context.Context, tx *db.ClusterTx, matchedACLNames []string, usageType any, nicName string, nicConfig map[string]string) error, matchACLNames ...string) error { if len(matchACLNames) <= 0 { return nil } var profiles []cluster.Profile profileDevices := map[string]map[string]cluster.Device{} err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Find networks using the ACLs. Cheapest to do. networkNames, err := tx.GetCreatedNetworkNamesByProject(ctx, aclProjectName) if err != nil && !response.IsNotFoundError(err) { return fmt.Errorf("Failed loading networks for project %q: %w", aclProjectName, err) } for _, networkName := range networkNames { _, network, _, err := tx.GetNetworkInAnyState(ctx, aclProjectName, networkName) if err != nil { return fmt.Errorf("Failed to get network config for %q: %w", networkName, err) } netACLNames := util.SplitNTrimSpace(network.Config["security.acls"], ",", -1, true) matchedACLNames := []string{} for _, netACLName := range netACLNames { if slices.Contains(matchACLNames, netACLName) { matchedACLNames = append(matchedACLNames, netACLName) } } if len(matchedACLNames) > 0 { // Call usageFunc with a list of matched ACLs and info about the network. err := usageFunc(ctx, tx, matchedACLNames, network, "", nil) if err != nil { return err } } } // Look for profiles. Next cheapest to do. profiles, err = cluster.GetProfiles(ctx, tx.Tx()) if err != nil { return err } // Get all the profile devices. profileDevicesByID, err := cluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return err } for _, profile := range profiles { devices := map[string]cluster.Device{} for _, dev := range profileDevicesByID[profile.ID] { devices[dev.Name] = dev } profileDevices[profile.Name] = devices } return nil }) if err != nil { return err } for _, profile := range profiles { // Get the profiles's effective network project name. profileNetworkProjectName, _, err := project.NetworkProject(s.DB.Cluster, profile.Project) if err != nil { return err } // Skip profiles who's effective network project doesn't match this Network ACL's project. if profileNetworkProjectName != aclProjectName { continue } // Iterate through each of the instance's devices, looking for NICs that are using any of the ACLs. for devName, devConfig := range deviceConfig.NewDevices(cluster.DevicesToAPI(profileDevices[profile.Name])) { matchedACLNames := isInUseByDevice(devConfig, matchACLNames...) if len(matchedACLNames) > 0 { // Call usageFunc with a list of matched ACLs and info about the instance NIC. err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return usageFunc(ctx, tx, matchedACLNames, profile, devName, devConfig) }) if err != nil { return err } } } } var aclNames []string err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { acls, err := cluster.GetNetworkACLs(ctx, tx.Tx(), cluster.NetworkACLFilter{Project: &aclProjectName}) if err != nil { return err } aclNames = make([]string, len(acls)) for i, acl := range acls { aclNames[i] = acl.Name } return nil }) if err != nil { return err } err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { for _, aclName := range aclNames { _, aclInfo, err := cluster.GetNetworkACLAPI(ctx, tx.Tx(), aclProjectName, aclName) if err != nil { return err } matchedACLNames := []string{} // Ingress rules can specify ACL names in their Source subjects. for _, rule := range aclInfo.Ingress { for _, subject := range util.SplitNTrimSpace(rule.Source, ",", -1, true) { // Look for new matching ACLs, but ignore our own ACL reference in our own rules. if slices.Contains(matchACLNames, subject) && !slices.Contains(matchedACLNames, subject) && subject != aclInfo.Name { matchedACLNames = append(matchedACLNames, subject) } } } // Egress rules can specify ACL names in their Destination subjects. for _, rule := range aclInfo.Egress { for _, subject := range util.SplitNTrimSpace(rule.Destination, ",", -1, true) { // Look for new matching ACLs, but ignore our own ACL reference in our own rules. if slices.Contains(matchACLNames, subject) && !slices.Contains(matchedACLNames, subject) && subject != aclInfo.Name { matchedACLNames = append(matchedACLNames, subject) } } } if len(matchedACLNames) > 0 { // Call usageFunc with a list of matched ACLs and info about the ACL. err = usageFunc(ctx, tx, matchedACLNames, aclInfo, "", nil) if err != nil { return err } } } // Find instances using the ACLs. Most expensive to do. err = tx.InstanceList(ctx, func(inst db.InstanceArgs, p api.Project) error { // Get the instance's effective network project name. instNetworkProject := project.NetworkProjectFromRecord(&p) // Skip instances who's effective network project doesn't match this Network ACL's project. if instNetworkProject != aclProjectName { return nil } devices := db.ExpandInstanceDevices(inst.Devices.Clone(), inst.Profiles) // Iterate through each of the instance's devices, looking for NICs that are using any of the ACLs. for devName, devConfig := range devices { matchedACLNames := isInUseByDevice(devConfig, matchACLNames...) if len(matchedACLNames) > 0 { // Call usageFunc with a list of matched ACLs and info about the instance NIC. err := usageFunc(ctx, tx, matchedACLNames, inst, devName, devConfig) if err != nil { return err } } } return nil }) if err != nil { return err } return nil }) if err != nil { return err } return nil } // isInUseByDevice returns any of the supplied matching ACL names found referenced by the NIC device. func isInUseByDevice(d deviceConfig.Device, matchACLNames ...string) []string { matchedACLNames := []string{} // Only NICs linked to managed networks can use network ACLs. if d["type"] != "nic" || d["network"] == "" { return matchedACLNames } for _, nicACLName := range util.SplitNTrimSpace(d["security.acls"], ",", -1, true) { if slices.Contains(matchACLNames, nicACLName) { matchedACLNames = append(matchedACLNames, nicACLName) } } return matchedACLNames } // NetworkACLUsage info about a network and what ACL it uses. type NetworkACLUsage struct { ID int64 Name string Type string Config map[string]string InstanceName string DeviceName string } // NetworkUsage populates the provided aclNets map with networks that are using any of the specified ACLs. func NetworkUsage(s *state.State, aclProjectName string, aclNames []string, aclNets map[string]NetworkACLUsage) error { supportedNetTypes := []string{"bridge", "ovn"} // Find all networks and instance/profile NICs that use any of the specified Network ACLs. err := UsedBy(s, aclProjectName, func(ctx context.Context, tx *db.ClusterTx, matchedACLNames []string, usageType any, devName string, nicConfig map[string]string) error { switch u := usageType.(type) { case cluster.Profile: networkID, network, _, err := tx.GetNetworkInAnyState(ctx, aclProjectName, nicConfig["network"]) if err != nil { return fmt.Errorf("Failed to load network %q: %w", nicConfig["network"], err) } if slices.Contains(supportedNetTypes, network.Type) { _, found := aclNets[network.Name] if !found { aclNets[network.Name] = NetworkACLUsage{ ID: networkID, Name: network.Name, Type: network.Type, Config: network.Config, } } } case db.InstanceArgs: networkID, network, _, err := tx.GetNetworkInAnyState(ctx, aclProjectName, nicConfig["network"]) if err != nil { return fmt.Errorf("Failed to load network %q: %w", nicConfig["network"], err) } if slices.Contains(supportedNetTypes, network.Type) { if network.Type == "bridge" && devName != "" { // Use different key for the usage by bridge NICs to avoid overwriting the usage by the bridge network itself. key := fmt.Sprintf("%s/%s/%s", network.Name, u.Name, devName) _, found := aclNets[key] if !found { aclNets[key] = NetworkACLUsage{ ID: networkID, Name: network.Name, Type: network.Type, Config: network.Config, InstanceName: u.Name, DeviceName: devName, } } } else { _, found := aclNets[network.Name] if !found { aclNets[network.Name] = NetworkACLUsage{ ID: networkID, Name: network.Name, Type: network.Type, Config: network.Config, } } } } case *api.Network: if slices.Contains(supportedNetTypes, u.Type) { _, found := aclNets[u.Name] if !found { networkID, network, _, err := tx.GetNetworkInAnyState(ctx, aclProjectName, u.Name) if err != nil { return fmt.Errorf("Failed to load network %q: %w", u.Name, err) } aclNets[u.Name] = NetworkACLUsage{ ID: networkID, Name: network.Name, Type: network.Type, Config: network.Config, } } } case *api.NetworkACL: return nil // Nothing to do for ACL rules referencing us. default: return fmt.Errorf("Unrecognised usage type %T", u) } return nil }, aclNames...) if err != nil { return err } return nil } incus-7.0.0/internal/server/network/acl/acl_ovn.go000066400000000000000000001551151517523235500222010ustar00rootroot00000000000000package acl import ( "context" "encoding/json" "errors" "fmt" "net" "slices" "strings" "time" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/network/ovn" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // OVN ACL rule priorities. const ( ovnACLPriorityPortGroupDefaultAction = 0 ovnACLPriorityNICDefaultActionIngress = 100 ) // ovnACLPriorityNICDefaultActionEgress needs to be >10 higher than ovnACLPriorityNICDefaultActionIngress so that // ingress reject rules (that OVN adds 10 to their priorities) don't prevent egress rules being tested first. const ( ovnACLPriorityNICDefaultActionEgress = 111 ovnACLPrioritySwitchAllow = 200 ovnACLPriorityPortGroupAllow = 300 ovnACLPriorityReversedPortGroupDrop = 300 ovnACLPriorityPortGroupReject = 400 ovnACLPriorityReversedPortGroupReject = 400 ovnACLPriorityPortGroupDrop = 500 ovnACLPriorityReversedPortGroupAllow = 500 ) // ovnACLPortGroupPrefix prefix used when naming ACL related port groups in OVN. const ovnACLPortGroupPrefix = "incus_acl" // DirectionalPortGroups defines the OVN port group names for traffic // matching in each direction, including both normal and reversed flows. type DirectionalPortGroups struct { Prefix string All ovn.OVNPortGroup Ingress ovn.OVNPortGroup Egress ovn.OVNPortGroup IngressReversed ovn.OVNPortGroup EgressReversed ovn.OVNPortGroup } // PortGroups returns all port group names as a slice. func (p *DirectionalPortGroups) PortGroups() []ovn.OVNPortGroup { return []ovn.OVNPortGroup{ p.All, p.Ingress, p.IngressReversed, p.Egress, p.EgressReversed, } } // CreatePortGroups creates directional port groups for ingress and egress rules. func (p *DirectionalPortGroups) CreatePortGroups(l logger.Logger, client *ovn.NB, reverter *revert.Reverter, projectID int64, aclName string) error { for _, portGroupName := range p.PortGroups() { // Check if port group exists. portGroupUUID, _, err := client.GetPortGroupInfo(context.TODO(), portGroupName) if err != nil { return fmt.Errorf("Failed getting port group UUID for security ACL %q setup: %w", aclName, err) } if portGroupUUID == "" { l.Debug("Creating empty referenced ACL OVN port group", logger.Ctx{"networkACL": aclName, "portGroup": portGroupName}) err := client.CreatePortGroup(context.TODO(), projectID, portGroupName, []ovn.OVNPortGroup{}, "") if err != nil { return fmt.Errorf("Failed creating port group %q for referenced security ACL %q setup: %w", portGroupName, aclName, err) } reverter.Add(func() { _ = client.DeletePortGroup(context.TODO(), portGroupName) }) } } return nil } // AddToChangeSet adds all ports from the specified port groups to the given changeSet. func (p *DirectionalPortGroups) AddToChangeSet(portUUID ovn.OVNSwitchPortUUID, changeSet map[ovn.OVNPortGroup][]ovn.OVNSwitchPortUUID) { for _, portGroup := range p.PortGroups() { OVNPortGroupInstanceNICSchedule(portUUID, changeSet, portGroup) } } // Remove deletes the specified port groups from the given map of port groups. func (p *DirectionalPortGroups) Remove(removeACLPortGroups map[ovn.OVNPortGroup]struct{}) { for _, portGroup := range p.PortGroups() { delete(removeACLPortGroups, portGroup) } } // Exist checks whether all port groups in the set exist. // It returns two values: // - exists: false if any port group does not exist. // - hasACLs: false if any existing port group has no ACLs. func (p *DirectionalPortGroups) Exist(client *ovn.NB) (bool, bool, error) { hasACLs := true for _, portGroup := range p.PortGroups() { portGroupUUID, portGroupHasACLs, err := client.GetPortGroupInfo(context.TODO(), portGroup) if err != nil { return false, false, fmt.Errorf("Failed getting port group %q UUID setup: %w", portGroup, err) } if portGroupUUID == "" { return false, false, nil } if !portGroupHasACLs { hasACLs = false } } return true, hasACLs, nil } // OVNACLPortGroupNamePrefix returns the port groups name prefix for a Network ACL ID. func OVNACLPortGroupNamePrefix(networkACLID int64) string { // OVN doesn't match port groups that have a "-" in them. So use an "_" for the separator. // This is because OVN port group names must match: [a-zA-Z_.][a-zA-Z_.0-9]*. return fmt.Sprintf("%s%d", ovnACLPortGroupPrefix, networkACLID) } // OVNACLDirectionalPortGroups returns the port group names of all kinds for a Network ACL ID. func OVNACLDirectionalPortGroups(networkACLID int64) *DirectionalPortGroups { prefix := OVNACLPortGroupNamePrefix(networkACLID) return &DirectionalPortGroups{ Prefix: prefix, All: ovn.OVNPortGroup(fmt.Sprintf("%s_all", prefix)), Ingress: ovn.OVNPortGroup(fmt.Sprintf("%s_ingress", prefix)), IngressReversed: ovn.OVNPortGroup(fmt.Sprintf("%s_ingress_reversed", prefix)), Egress: ovn.OVNPortGroup(fmt.Sprintf("%s_egress", prefix)), EgressReversed: ovn.OVNPortGroup(fmt.Sprintf("%s_egress_reversed", prefix)), } } // OVNACLNetworkPortGroupName returns the port group name for a Network ACL ID and Network ID. func OVNACLNetworkPortGroupName(networkACLID int64, networkID int64) ovn.OVNPortGroup { // OVN doesn't match port groups that have a "-" in them. So use an "_" for the separator. // This is because OVN port group names must match: [a-zA-Z_.][a-zA-Z_.0-9]*. return ovn.OVNPortGroup(fmt.Sprintf("%s%d_net%d", ovnACLPortGroupPrefix, networkACLID, networkID)) } // OVNIntSwitchPortGroupName returns the port group name for a Network ID. func OVNIntSwitchPortGroupName(networkID int64) ovn.OVNPortGroup { return ovn.OVNPortGroup(fmt.Sprintf("incus_net%d", networkID)) } // OVNIntSwitchPortGroupAddressSetPrefix returns the internal switch routes address set prefix for a Network ID. func OVNIntSwitchPortGroupAddressSetPrefix(networkID int64) ovn.OVNAddressSet { return ovn.OVNAddressSet(fmt.Sprintf("%s_routes", OVNIntSwitchPortGroupName(networkID))) } // OVNNetworkPrefix returns the prefix used for OVN entities related to a Network ID. func OVNNetworkPrefix(networkID int64) string { return fmt.Sprintf("incus-net%d", networkID) } // OVNIntSwitchName returns the internal logical switch name for a Network ID. func OVNIntSwitchName(networkID int64) ovn.OVNSwitch { return ovn.OVNSwitch(fmt.Sprintf("%s-ls-int", OVNNetworkPrefix(networkID))) } // OVNIntSwitchRouterPortName returns OVN logical internal switch router port name. func OVNIntSwitchRouterPortName(networkID int64) ovn.OVNSwitchPort { return ovn.OVNSwitchPort(fmt.Sprintf("%s-lsp-router", OVNIntSwitchName(networkID))) } // PortGroupActionPriority returns the priority for the specific action. func PortGroupActionPriority(action string, reversed bool) int { if reversed { switch action { case "allow": return ovnACLPriorityReversedPortGroupAllow case "reject": return ovnACLPriorityReversedPortGroupReject case "drop": return ovnACLPriorityReversedPortGroupDrop } } else { switch action { case "allow": return ovnACLPriorityPortGroupAllow case "reject": return ovnACLPriorityPortGroupReject case "drop": return ovnACLPriorityPortGroupDrop } } return 0 } // OVNEnsureACLs ensures that the requested aclNames exist as OVN port groups (creates & applies ACL rules if not), // If reapplyRules is true then the current ACL rules in the database are applied to the existing port groups // rather than just new ones. Any ACLs referenced in the requested ACLs rules are also created as empty OVN port // groups if needed. If a requested ACL exists, but has no ACL rules applied, then the current rules are loaded out // of the database and applied. For each network provided in aclNets, the network specific port group for each ACL // is checked for existence (it is created & applies network specific ACL rules if not). // Returns a revert fail function that can be used to undo this function if a subsequent step fails. func OVNEnsureACLs(s *state.State, l logger.Logger, client *ovn.NB, aclProjectName string, aclNameIDs map[string]int64, aclNets map[string]NetworkACLUsage, aclNames []string, reapplyRules bool) (revert.Hook, error) { reverter := revert.New() defer reverter.Fail() var err error var projectID int64 err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { projectID, err = cluster.GetProjectID(ctx, tx.Tx(), aclProjectName) if err != nil { return fmt.Errorf("Failed getting project ID for project %q: %w", aclProjectName, err) } return err }) if err != nil { return nil, err } peerTargetNetIDs := make(map[cluster.NetworkPeerConnection]int64) err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get created networks for the project. networks, err := tx.GetCreatedNetworksByProject(ctx, aclProjectName) if err != nil { return fmt.Errorf("Failed getting created networks for project %q: %w", aclProjectName, err) } for netID, network := range networks { // Filter for OVN networks in Go. if network.Type != "ovn" { continue } // Get peers for the current OVN network. peerFilter := cluster.NetworkPeerFilter{NetworkID: &netID} dbPeers, err := cluster.GetNetworkPeers(ctx, tx.Tx(), peerFilter) if err != nil { return fmt.Errorf("Failed loading network peers for network ID %d: %w", netID, err) } for _, dbPeer := range dbPeers { // Only include peers with a valid target network ID. if dbPeer.TargetNetworkID.Valid { peerKey := cluster.NetworkPeerConnection{ NetworkName: network.Name, PeerName: dbPeer.Name, } peerTargetNetIDs[peerKey] = dbPeer.TargetNetworkID.Int64 } } } return nil }) if err != nil { return nil, fmt.Errorf("Failed getting peer connection mappings: %w", err) } // First check all ACL Names map to IDs in supplied aclNameIDs. for _, aclName := range aclNames { _, found := aclNameIDs[aclName] if !found { return nil, fmt.Errorf("Cannot find security ACL ID for %q", aclName) } } // Next check which OVN port groups need creating and which exist already. type aclStatus struct { name string aclInfo *api.NetworkACL addACLNets map[string]NetworkACLUsage } existingACLPortGroups := []aclStatus{} createACLPortGroups := []aclStatus{} for _, aclName := range aclNames { dPortGroups := OVNACLDirectionalPortGroups(aclNameIDs[aclName]) // Check if port group exists and has ACLs. dPortGroupExists, portGroupHasACLs, err := dPortGroups.Exist(client) if err != nil { return nil, fmt.Errorf("Failed getting port group UUID for security ACL %q setup: %w", aclName, err) } if !dPortGroupExists { var aclInfo *api.NetworkACL err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Load the config we'll need to create the port group with ACL rules. _, aclInfo, err = cluster.GetNetworkACLAPI(ctx, tx.Tx(), aclProjectName, aclName) return err }) if err != nil { return nil, fmt.Errorf("Failed loading Network ACL %q: %w", aclName, err) } createACLPortGroups = append(createACLPortGroups, aclStatus{name: aclName, aclInfo: aclInfo}) } else { var aclInfo *api.NetworkACL addACLNets := make(map[string]NetworkACLUsage) // Check each per-ACL-per-network port group exists. for _, aclNet := range aclNets { netPortGroupName := OVNACLNetworkPortGroupName(aclNameIDs[aclName], aclNet.ID) netPortGroupUUID, _, err := client.GetPortGroupInfo(context.TODO(), netPortGroupName) if err != nil { return nil, fmt.Errorf("Failed getting port group UUID for security ACL %q setup: %w", aclName, err) } if netPortGroupUUID == "" { addACLNets[aclNet.Name] = aclNet } } // If we are being asked to forcefully reapply the rules, or if the port group exists but // doesn't have any rules, then we load the current rule set from the database to apply. // Note: An empty ACL list on a port group means it has only been partially setup, as // even Network ACLs with no rules should have at least 1 OVN ACL applied because of // the default rule we add. We also need to reapply the rules if we are adding any // new per-ACL-per-network port groups. if reapplyRules || !portGroupHasACLs || len(addACLNets) > 0 { err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, aclInfo, err = cluster.GetNetworkACLAPI(ctx, tx.Tx(), aclProjectName, aclName) return err }) if err != nil { return nil, fmt.Errorf("Failed loading Network ACL %q: %w", aclName, err) } } // Storing non-nil aclInfo in the aclStatus struct will trigger rule applying. existingACLPortGroups = append(existingACLPortGroups, aclStatus{name: aclName, aclInfo: aclInfo, addACLNets: addACLNets}) } } // Build a list of referenced ACLs in the rules of ACLs we need to create. // We will create port groups (without ACL rules) for any missing referenced ACL OVN port groups so that // when we add the rules for the new ACL port groups this doesn't trigger an OVN log error about missing // port groups. referencedACLs := make(map[string]struct{}) for _, aclStatus := range createACLPortGroups { ovnAddReferencedACLs(aclStatus.aclInfo, referencedACLs) } if reapplyRules { // Also add referenced ACLs in existing ACL rulesets if reapplying rules, as they may have changed. for _, aclStatus := range existingACLPortGroups { ovnAddReferencedACLs(aclStatus.aclInfo, referencedACLs) } } // Remove any references for our creation ACLs as we don't want to try and create them twice. for _, aclStatus := range createACLPortGroups { delete(referencedACLs, aclStatus.name) } // Create any missing port groups for the referenced ACLs before creating the requested ACL port groups. // This way the referenced port groups will exist for any rules that referenced them in the creation ACLs. // Note: We only create the empty port group, we do not add the ACL rules, so it is expected that any // future direct assignment of these referenced ACLs will trigger the ACL rules being added if needed. for aclName := range referencedACLs { dPortGroups := OVNACLDirectionalPortGroups(aclNameIDs[aclName]) err = dPortGroups.CreatePortGroups(l, client, reverter, projectID, aclName) if err != nil { return nil, err } } // Create the needed port groups and then apply ACL rules to new port groups. for _, aclStatus := range createACLPortGroups { dPortGroups := OVNACLDirectionalPortGroups(aclNameIDs[aclStatus.name]) err = dPortGroups.CreatePortGroups(l, client, reverter, projectID, aclStatus.name) if err != nil { return nil, err } // Create any per-ACL-per-network port groups needed. for _, aclNet := range aclNets { netPortGroupName := OVNACLNetworkPortGroupName(aclNameIDs[aclStatus.name], aclNet.ID) l.Debug("Creating ACL OVN network port group", logger.Ctx{"networkACL": aclStatus.name, "portGroup": netPortGroupName}) // Create OVN network specific port group and link it to switch by adding the router port. err = client.CreatePortGroup(context.TODO(), projectID, netPortGroupName, dPortGroups.PortGroups(), OVNIntSwitchName(aclNet.ID), OVNIntSwitchRouterPortName(aclNet.ID)) if err != nil { return nil, fmt.Errorf("Failed creating port group %q for security ACL %q and network %q setup: %w", netPortGroupName, aclStatus.name, aclNet.Name, err) } reverter.Add(func() { _ = client.DeletePortGroup(context.TODO(), netPortGroupName) }) } // Now apply our ACL rules to port group (and any per-ACL-per-network port groups needed). aclStatus.aclInfo.Project = aclProjectName err = ovnApplyToPortGroup(s, l, client, aclStatus.aclInfo, aclStatus.name, aclNameIDs, aclNets, peerTargetNetIDs) if err != nil { return nil, fmt.Errorf("Failed applying ACL rules to directional port groups %s for security ACL %q setup: %w", dPortGroups.Prefix, aclStatus.name, err) } } // Create any missing per-ACL-per-network port groups for existing ACL port groups, and apply the ACL rules // to them and the main ACL port group (if needed). for _, aclStatus := range existingACLPortGroups { dPortGroups := OVNACLDirectionalPortGroups(aclNameIDs[aclStatus.name]) // Create any missing per-ACL-per-network port groups. for _, aclNet := range aclStatus.addACLNets { netPortGroupName := OVNACLNetworkPortGroupName(aclNameIDs[aclStatus.name], aclNet.ID) l.Debug("Creating ACL OVN network port group", logger.Ctx{"networkACL": aclStatus.name, "portGroup": netPortGroupName}) // Create OVN network specific port group and link it to switch by adding the router port. err := client.CreatePortGroup(context.TODO(), projectID, netPortGroupName, dPortGroups.PortGroups(), OVNIntSwitchName(aclNet.ID), OVNIntSwitchRouterPortName(aclNet.ID)) if err != nil { return nil, fmt.Errorf("Failed creating port group %q for security ACL %q and network %q setup: %w", netPortGroupName, aclStatus.name, aclNet.Name, err) } reverter.Add(func() { _ = client.DeletePortGroup(context.TODO(), netPortGroupName) }) } // If aclInfo has been loaded, then we should use it to apply ACL rules to the existing port group // (and any per-ACL-per-network port groups needed). if aclStatus.aclInfo != nil { l.Debug("Applying ACL rules to OVN port group", logger.Ctx{"networkACL": aclStatus.name, "directionalPortGroup": dPortGroups.Prefix}) aclStatus.aclInfo.Project = aclProjectName err := ovnApplyToPortGroup(s, l, client, aclStatus.aclInfo, aclStatus.name, aclNameIDs, aclNets, peerTargetNetIDs) if err != nil { return nil, fmt.Errorf("Failed applying ACL rules to directional port groups %s for security ACL %q setup: %w", dPortGroups.Prefix, aclStatus.name, err) } } } cleanup := reverter.Clone().Fail reverter.Success() return cleanup, nil } // ovnAddReferencedACLs adds to the referencedACLNames any ACLs referenced by the rules in the supplied ACL. func ovnAddReferencedACLs(info *api.NetworkACL, referencedACLNames map[string]struct{}) { addACLNamesFrom := func(ruleSubjects []string) { for _, subject := range ruleSubjects { _, found := referencedACLNames[subject] if found { continue // Skip subjects already seen. } if slices.Contains(append(ruleSubjectInternalAliases, ruleSubjectExternalAliases...), subject) { continue // Skip special reserved subjects that are not ACL names. } if validate.IsNetworkAddressCIDR(subject) == nil || validate.IsNetworkRange(subject) == nil { continue // Skip if the subject is an IP CIDR or IP range. } // Anything else must be a referenced ACL name. // Record newly seen referenced ACL into authoritative list. referencedACLNames[subject] = struct{}{} } } for _, rule := range info.Ingress { addACLNamesFrom(util.SplitNTrimSpace(rule.Source, ",", -1, true)) } for _, rule := range info.Egress { addACLNamesFrom(util.SplitNTrimSpace(rule.Destination, ",", -1, true)) } } // replaceAddressSetNames performs replacements of address set names with OVN identifiers. func replaceAddressSetNames(subject string, addressSetIDs map[string]int) string { subjects := util.SplitNTrimSpace(subject, ",", -1, true) for i, subj := range subjects { after, ok := strings.CutPrefix(subj, "$") if ok { setID, found := addressSetIDs[after] if found { subjects[i] = fmt.Sprintf("$incus_set%d", setID) } } } return strings.Join(subjects, ",") } // ovnApplyToPortGroup applies the rules in the specified ACL to the specified port group. func ovnApplyToPortGroup(s *state.State, l logger.Logger, client *ovn.NB, aclInfo *api.NetworkACL, aclName string, aclNameIDs map[string]int64, aclNets map[string]NetworkACLUsage, peerTargetNetIDs map[cluster.NetworkPeerConnection]int64) error { directionalPortGroups := OVNACLDirectionalPortGroups(aclNameIDs[aclName]) // Create slice for port group rules that has the capacity for ingress and egress rules, plus default rule. ingressPGRules := make([]ovn.OVNACLRule, 0) egressPGRules := make([]ovn.OVNACLRule, 0) revIngressPGRules := make([]ovn.OVNACLRule, 0) revEgressPGRules := make([]ovn.OVNACLRule, 0) allPGRules := make([]ovn.OVNACLRule, 0) networkRules := make([]ovn.OVNACLRule, 0) networkPeersNeeded := make([]cluster.NetworkPeerConnection, 0) // First gather used address sets addressSetNamesSet := make(map[string]struct{}) extractAddressSets := func(rules []api.NetworkACLRule) { for _, rule := range rules { for _, subj := range util.SplitNTrimSpace(rule.Source, ",", -1, true) { if strings.HasPrefix(subj, "$") { addressSetNamesSet[subj] = struct{}{} } } for _, subj := range util.SplitNTrimSpace(rule.Destination, ",", -1, true) { if strings.HasPrefix(subj, "$") { addressSetNamesSet[subj] = struct{}{} } } } } extractAddressSets(aclInfo.Ingress) extractAddressSets(aclInfo.Egress) addressSetNames := make([]string, 0, len(addressSetNamesSet)) for setName := range addressSetNamesSet { addressSetNames = append(addressSetNames, setName) } // Map address set names to ID addressSetIDs := make(map[string]int, len(addressSetNames)) if len(addressSetNames) > 0 { for _, setName := range addressSetNames { err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { set, err := cluster.GetNetworkAddressSet(ctx, tx.Tx(), aclInfo.Project, strings.TrimPrefix(setName, "$")) if err != nil { return err } addressSetIDs[set.Name] = set.ID return nil }) if err != nil { return fmt.Errorf("Failed fetching address set %s IDs in project %s err: %w", strings.TrimPrefix(setName, "$"), aclInfo.Project, err) } } } // convertACLRules converts the ACL rules to OVN ACL rules. convertACLRules := func(portGroupName ovn.OVNPortGroup, direction string, reversed bool, rules ...api.NetworkACLRule) error { for ruleIndex, rule := range rules { if rule.State == "disabled" { continue } // Replace address set subjects rule.Source = replaceAddressSetNames(rule.Source, addressSetIDs) rule.Destination = replaceAddressSetNames(rule.Destination, addressSetIDs) ovnACLRule, isAllRule, networkSpecific, networkPeers, err := ovnRuleCriteriaToOVNACLRule(s, direction, &rule, portGroupName, aclNameIDs, peerTargetNetIDs, reversed) if err != nil { return err } if rule.State == "logged" { ovnACLRule.Log = true ovnACLRule.LogName = fmt.Sprintf("%s-%s-%d", portGroupName, direction, ruleIndex) } if networkSpecific { networkRules = append(networkRules, ovnACLRule) } else if isAllRule { allPGRules = append(allPGRules, ovnACLRule) } else if direction == "ingress" && !reversed { ingressPGRules = append(ingressPGRules, ovnACLRule) } else if direction == "ingress" && reversed { revIngressPGRules = append(revIngressPGRules, ovnACLRule) } else if direction == "egress" && !reversed { egressPGRules = append(egressPGRules, ovnACLRule) } else { revEgressPGRules = append(revEgressPGRules, ovnACLRule) } networkPeersNeeded = append(networkPeersNeeded, networkPeers...) } return nil } err := convertACLRules(directionalPortGroups.Ingress, "ingress", false, aclInfo.Ingress...) if err != nil { return fmt.Errorf("Failed converting ACL %q ingress rules for port group %q: %w", aclInfo.Name, directionalPortGroups.Ingress, err) } err = convertACLRules(directionalPortGroups.Egress, "egress", false, aclInfo.Egress...) if err != nil { return fmt.Errorf("Failed converting ACL %q egress rules for port group %q: %w", aclInfo.Name, directionalPortGroups.Egress, err) } err = convertACLRules(directionalPortGroups.IngressReversed, "ingress", true, aclInfo.Ingress...) if err != nil { return fmt.Errorf("Failed converting ACL %q reverted ingress rules for port group %q: %w", aclInfo.Name, directionalPortGroups.IngressReversed, err) } err = convertACLRules(directionalPortGroups.EgressReversed, "egress", true, aclInfo.Egress...) if err != nil { return fmt.Errorf("Failed converting ACL %q reverted egress rules for port group %q: %w", aclInfo.Name, directionalPortGroups.EgressReversed, err) } allPGRules = addPortGroupDefaultAction(directionalPortGroups.All, allPGRules) ingressPGRules = addPortGroupDefaultAction(directionalPortGroups.Ingress, ingressPGRules) egressPGRules = addPortGroupDefaultAction(directionalPortGroups.Egress, egressPGRules) revIngressPGRules = addPortGroupDefaultAction(directionalPortGroups.IngressReversed, revIngressPGRules) revEgressPGRules = addPortGroupDefaultAction(directionalPortGroups.EgressReversed, revEgressPGRules) // Check ACL is only being applied to networks that have the required peers. for _, aclNet := range aclNets { for _, peer := range networkPeersNeeded { if peer.NetworkName != aclNet.Name { return fmt.Errorf(`ACL requiring peer "%s/%s" cannot be applied to network %q`, peer.NetworkName, peer.PeerName, aclNet.Name) } } } // Clear all existing ACL rules from port group then add the new rules to the port group. err = client.UpdatePortGroupACLRules(context.TODO(), directionalPortGroups.All, nil, allPGRules...) if err != nil { return fmt.Errorf("Failed applying ACL %q rules to port group %q: %w", aclInfo.Name, directionalPortGroups.All, err) } err = client.UpdatePortGroupACLRules(context.TODO(), directionalPortGroups.Ingress, nil, ingressPGRules...) if err != nil { return fmt.Errorf("Failed applying ACL %q rules to port group %q: %w", aclInfo.Name, directionalPortGroups.Ingress, err) } err = client.UpdatePortGroupACLRules(context.TODO(), directionalPortGroups.Egress, nil, egressPGRules...) if err != nil { return fmt.Errorf("Failed applying ACL %q rules to port group %q: %w", aclInfo.Name, directionalPortGroups.Egress, err) } err = client.UpdatePortGroupACLRules(context.TODO(), directionalPortGroups.IngressReversed, nil, revIngressPGRules...) if err != nil { return fmt.Errorf("Failed applying ACL %q rules to port group %q: %w", aclInfo.Name, directionalPortGroups.IngressReversed, err) } err = client.UpdatePortGroupACLRules(context.TODO(), directionalPortGroups.EgressReversed, nil, revEgressPGRules...) if err != nil { return fmt.Errorf("Failed applying ACL %q rules to port group %q: %w", aclInfo.Name, directionalPortGroups.EgressReversed, err) } // Now apply the network specific rules to all networks requested (even if networkRules is empty). for _, aclNet := range aclNets { netPortGroupName := OVNACLNetworkPortGroupName(aclNameIDs[aclInfo.Name], aclNet.ID) l.Debug("Applying network specific ACL rules to network OVN port group", logger.Ctx{"networkACL": aclInfo.Name, "network": aclNet.Name, "portGroup": netPortGroupName}) // Setup per-network dynamic replacements for @internal/@external subject port selectors. matchReplace := map[string]string{ fmt.Sprintf("@%s", ruleSubjectInternal): fmt.Sprintf("@%s", OVNIntSwitchPortGroupName(aclNet.ID)), fmt.Sprintf("@%s", ruleSubjectExternal): fmt.Sprintf(`"%s"`, OVNIntSwitchRouterPortName(aclNet.ID)), } err = client.UpdatePortGroupACLRules(context.TODO(), netPortGroupName, matchReplace, networkRules...) if err != nil { return fmt.Errorf("Failed applying ACL %q rules to port group %q for network %q: %w", aclInfo.Name, netPortGroupName, aclNet.Name, err) } } return nil } // ovnRuleCriteriaToOVNACLRule converts an ACL rule into an OVNACLRule for an OVN port group or network. // Returns a bool indicating if any of the rule subjects are network specific. func ovnRuleCriteriaToOVNACLRule(s *state.State, direction string, rule *api.NetworkACLRule, portGroupName ovn.OVNPortGroup, aclNameIDs map[string]int64, peerTargetNetIDs map[cluster.NetworkPeerConnection]int64, reversed bool) (ovn.OVNACLRule, bool, bool, []cluster.NetworkPeerConnection, error) { networkSpecific := false isAllRule := false networkPeersNeeded := make([]cluster.NetworkPeerConnection, 0) portGroupRule := ovn.OVNACLRule{ Direction: "to-lport", // Always use this so that outport is available to Match. } // Populate Action and Priority based on rule's Action. switch rule.Action { case "allow": portGroupRule.Action = "allow-related" portGroupRule.Priority = PortGroupActionPriority("allow", reversed) case "allow-stateless": portGroupRule.Action = "allow-stateless" portGroupRule.Priority = PortGroupActionPriority("allow", reversed) case "reject": portGroupRule.Action = "reject" portGroupRule.Priority = PortGroupActionPriority("reject", reversed) case "drop": portGroupRule.Action = "drop" portGroupRule.Priority = PortGroupActionPriority("drop", reversed) } var matchParts []string // Add directional port filter so we only apply this rule to the ports in the port group. switch direction { case "ingress": matchParts = []string{fmt.Sprintf("outport == @%s", portGroupName)} // Traffic going to Instance. case "egress": matchParts = []string{fmt.Sprintf("inport == @%s", portGroupName)} // Traffic leaving Instance. default: matchParts = []string{fmt.Sprintf("inport == @%s || outport == @%s", portGroupName, portGroupName)} } // Add subject filters. if rule.Source != "" { match, allRule, netSpecificMatch, networkPeers, err := ovnRuleSubjectToOVNACLMatch(s, "src", aclNameIDs, peerTargetNetIDs, util.SplitNTrimSpace(rule.Source, ",", -1, false)...) if err != nil { return ovn.OVNACLRule{}, false, false, nil, err } if netSpecificMatch { networkSpecific = true } if allRule { isAllRule = true } matchParts = append(matchParts, match) networkPeersNeeded = append(networkPeersNeeded, networkPeers...) } if rule.Destination != "" { match, allRule, netSpecificMatch, networkPeers, err := ovnRuleSubjectToOVNACLMatch(s, "dst", aclNameIDs, peerTargetNetIDs, util.SplitNTrimSpace(rule.Destination, ",", -1, false)...) if err != nil { return ovn.OVNACLRule{}, false, false, nil, err } if netSpecificMatch { networkSpecific = true } if allRule { isAllRule = true } matchParts = append(matchParts, match) networkPeersNeeded = append(networkPeersNeeded, networkPeers...) } // Add protocol filters. if slices.Contains([]string{"tcp", "udp"}, rule.Protocol) { matchParts = append(matchParts, rule.Protocol) if rule.SourcePort != "" { matchParts = append(matchParts, ovnRulePortToOVNACLMatch(rule.Protocol, "src", util.SplitNTrimSpace(rule.SourcePort, ",", -1, false)...)) } if rule.DestinationPort != "" { matchParts = append(matchParts, ovnRulePortToOVNACLMatch(rule.Protocol, "dst", util.SplitNTrimSpace(rule.DestinationPort, ",", -1, false)...)) } } else if slices.Contains([]string{"icmp4", "icmp6"}, rule.Protocol) { matchParts = append(matchParts, rule.Protocol) if rule.ICMPType != "" { matchParts = append(matchParts, fmt.Sprintf("%s.type == %s", rule.Protocol, rule.ICMPType)) } if rule.ICMPCode != "" { matchParts = append(matchParts, fmt.Sprintf("%s.code == %s", rule.Protocol, rule.ICMPCode)) } } // Populate the Match field with the generated match parts. portGroupRule.Match = fmt.Sprintf("(%s)", strings.Join(matchParts, ") && (")) return portGroupRule, isAllRule, networkSpecific, networkPeersNeeded, nil } // ovnRulePortToOVNACLMatch converts protocol (tcp/udp), direction (src/dst) and port criteria list into an OVN // match statement. func ovnRulePortToOVNACLMatch(protocol string, direction string, portCriteria ...string) string { fieldParts := make([]string, 0, len(portCriteria)) for _, portCriterion := range portCriteria { criterionParts := strings.SplitN(portCriterion, "-", 2) if len(criterionParts) > 1 { fieldParts = append(fieldParts, fmt.Sprintf("(%s.%s >= %s && %s.%s <= %s)", protocol, direction, criterionParts[0], protocol, direction, criterionParts[1])) } else { fieldParts = append(fieldParts, fmt.Sprintf("%s.%s == %s", protocol, direction, criterionParts[0])) } } return strings.Join(fieldParts, " || ") } // ovnRuleSubjectToOVNACLMatch converts direction (src/dst) and subject criteria list into an OVN match statement. // Returns a bool indicating if any of the subjects are network specific. func ovnRuleSubjectToOVNACLMatch(s *state.State, direction string, aclNameIDs map[string]int64, peerTargetNetIDs map[cluster.NetworkPeerConnection]int64, subjectCriteria ...string) (string, bool, bool, []cluster.NetworkPeerConnection, error) { fieldParts := make([]string, 0, len(subjectCriteria)) networkSpecific := false allRule := false networkPeersNeeded := make([]cluster.NetworkPeerConnection, 0) // For each criterion check if value looks like an IP range or IP CIDR, and if not use it as an ACL name. for _, subjectCriterion := range subjectCriteria { if validate.IsNetworkRange(subjectCriterion) == nil { criterionParts := strings.SplitN(subjectCriterion, "-", 2) if len(criterionParts) > 1 { ip := net.ParseIP(criterionParts[0]) if ip != nil { protocol := "ip4" if ip.To4() == nil { protocol = "ip6" } fieldParts = append(fieldParts, fmt.Sprintf("(%s.%s >= %s && %s.%s <= %s)", protocol, direction, criterionParts[0], protocol, direction, criterionParts[1])) } } else { return "", false, false, nil, fmt.Errorf("Invalid IP range %q", subjectCriterion) } } else { // Try parsing subject as single IP or CIDR. ip := net.ParseIP(subjectCriterion) if ip == nil { ip, _, _ = net.ParseCIDR(subjectCriterion) } if ip != nil { protocol := "ip4" if ip.To4() == nil { protocol = "ip6" } fieldParts = append(fieldParts, fmt.Sprintf("%s.%s == %s", protocol, direction, subjectCriterion)) } else { // If not valid IP subnet, check if subject is ACL name or address set or network peer name. var subjectPortSelector ovn.OVNPortGroup if slices.Contains(ruleSubjectInternalAliases, subjectCriterion) { // Use pseudo port group name for special reserved port selector types. // These will be expanded later for each network specific rule. // Convert deprecated #internal to non-deprecated @internal if needed. subjectPortSelector = ovn.OVNPortGroup(ruleSubjectInternal) networkSpecific = true } else if slices.Contains(ruleSubjectExternalAliases, subjectCriterion) { // Use pseudo port group name for special reserved port selector types. // These will be expanded later for each network specific rule. // Convert deprecated #external to non-deprecated @external if needed. subjectPortSelector = ovn.OVNPortGroup(ruleSubjectExternal) networkSpecific = true } else if strings.HasPrefix(subjectCriterion, "$") { // Check if subject is an address set if so we use it as it is. fieldParts = append(fieldParts, fmt.Sprintf("ip6.%s == %s_ip6 || ip4.%s == %s_ip4", direction, subjectCriterion, direction, subjectCriterion)) continue } else { after, ok := strings.CutPrefix(subjectCriterion, "@") if ok { // Subject is a network peer name. Convert to address set criteria. peerParts := strings.SplitN(after, "/", 2) if len(peerParts) != 2 { return "", false, false, nil, fmt.Errorf("Cannot parse subject as peer %q", subjectCriterion) } peer := cluster.NetworkPeerConnection{ NetworkName: peerParts[0], PeerName: peerParts[1], } networkID, found := peerTargetNetIDs[peer] if !found { return "", false, false, nil, fmt.Errorf("Cannot find network ID for peer %q", subjectCriterion) } addrSetPrefix := OVNIntSwitchPortGroupAddressSetPrefix(networkID) fieldParts = append(fieldParts, fmt.Sprintf("ip6.%s == $%s_ip6 || ip4.%s == $%s_ip4", direction, addrSetPrefix, direction, addrSetPrefix)) networkPeersNeeded = append(networkPeersNeeded, peer) continue // Not a port based selector. } else { // Assume the bare name is an ACL name and convert to port group. aclID, found := aclNameIDs[subjectCriterion] if !found { return "", false, false, nil, fmt.Errorf("Cannot find security ACL ID for %q", subjectCriterion) } subjectPortSelector = OVNACLDirectionalPortGroups(aclID).All allRule = true } } portType := "inport" if direction == "dst" { portType = "outport" } fieldParts = append(fieldParts, fmt.Sprintf("%s == @%s", portType, subjectPortSelector)) } } } return strings.Join(fieldParts, " || "), allRule, networkSpecific, networkPeersNeeded, nil } // OVNApplyNetworkBaselineRules applies preset baseline logical switch rules to a allow access to network services. func OVNApplyNetworkBaselineRules(client *ovn.NB, switchName ovn.OVNSwitch, routerPortName ovn.OVNSwitchPort, intRouterIPs []*net.IPNet, dnsIPs []net.IP) error { rules := []ovn.OVNACLRule{ { Direction: "to-lport", Action: "allow", Priority: ovnACLPrioritySwitchAllow, Match: "(arp || nd)", // Neighbour discovery. // codespell:ignore nd }, { Direction: "to-lport", Action: "allow", Priority: ovnACLPrioritySwitchAllow, Match: fmt.Sprintf(`inport == "%s" && nd_ra`, routerPortName), // IPv6 router adverts from router. }, { Direction: "to-lport", Action: "allow", Priority: ovnACLPrioritySwitchAllow, Match: fmt.Sprintf(`outport == "%s" && nd_rs`, routerPortName), // IPv6 router solicitation to router. }, { Direction: "to-lport", Action: "allow", Priority: ovnACLPrioritySwitchAllow, Match: "icmp6 && icmp6.type == 143 && ip.ttl == 1 && ip6.dst == ff02::16", // IPv6 ICMP Multicast Listener Discovery reports. }, { Direction: "to-lport", Action: "allow", Priority: ovnACLPrioritySwitchAllow, Match: "igmp && ip.ttl == 1 && ip4.mcast", // IPv4 IGMP. }, { Direction: "to-lport", Action: "allow", Priority: ovnACLPrioritySwitchAllow, Match: fmt.Sprintf(`outport == "%s" && ((ip4 && udp.dst == 67) || (ip6 && udp.dst == 547))`, routerPortName), // DHCP to router. }, // These 3 rules allow packets sent by the ACL when matching a reject rule. It is very important // that they are allowed when no stateful rules are in use, otherwise a bug in OVN causes it to // enter an infinite loop rejecting its own generated reject packets, causing more to be generated, // and OVN will use 100% CPU. { Direction: "to-lport", Action: "allow", Priority: ovnACLPrioritySwitchAllow, Match: "icmp6 && icmp6.type == {1,2,3,4} && ip.ttl == 255", // IPv6 ICMP error messages for ACL reject. }, { Direction: "to-lport", Action: "allow", Priority: ovnACLPrioritySwitchAllow, Match: "icmp4 && icmp4.type == {3,11,12} && ip.ttl == 255", // IPv4 ICMP error messages for ACL reject. }, { Direction: "to-lport", Action: "allow", Priority: ovnACLPrioritySwitchAllow, Match: fmt.Sprintf("tcp && tcp.flags == %#.03x", ovn.TCPRST|ovn.TCPACK), // TCP RST|ACK messages for ACL reject. }, } // Add rules to allow ping to/from internal router IPs. for _, intRouterIP := range intRouterIPs { ipVersion := 4 icmpPingType := 8 icmpPingReplyType := 0 if intRouterIP.IP.To4() == nil { ipVersion = 6 icmpPingType = 128 icmpPingReplyType = 129 } rules = append(rules, ovn.OVNACLRule{ Direction: "to-lport", Action: "allow", Priority: ovnACLPrioritySwitchAllow, Match: fmt.Sprintf(`outport == "%s" && icmp%d.type == %d && ip%d.dst == %s`, routerPortName, ipVersion, icmpPingType, ipVersion, intRouterIP.IP), }, ovn.OVNACLRule{ Direction: "to-lport", Action: "allow", Priority: ovnACLPrioritySwitchAllow, Match: fmt.Sprintf(`inport == "%s" && icmp%d.type == %d && ip%d.src == %s`, routerPortName, ipVersion, icmpPingReplyType, ipVersion, intRouterIP.IP), }, ) } // Add rules to allow DNS to DNS IPs. for _, dnsIP := range dnsIPs { ipVersion := 4 if dnsIP.To4() == nil { ipVersion = 6 } rules = append(rules, ovn.OVNACLRule{ Direction: "to-lport", Action: "allow", Priority: ovnACLPrioritySwitchAllow, Match: fmt.Sprintf(`outport == "%s" && ip%d.dst == %s && (udp.dst == 53 || tcp.dst == 53)`, routerPortName, ipVersion, dnsIP), }, ) } err := client.UpdateLogicalSwitchACLRules(context.TODO(), switchName, rules...) if err != nil { return fmt.Errorf("Failed applying baseline ACL rules to logical switch %q: %w", switchName, err) } return nil } // OVNPortGroupDeleteIfUnused deletes unused port groups. Accepts optional ignoreUsageType and ignoreUsageNicName // arguments, allowing the used by logic to ignore an instance/profile NIC or network (useful if config not // applied to database yet). Also accepts optional list of ACLs to explicitly consider in use by OVN. // The combination of ignoring the specified usage type and explicit keep ACLs allows the caller to ensure that // the desired ACLs are considered unused by the usage type even if the referring config has not yet been removed // from the database. func OVNPortGroupDeleteIfUnused(s *state.State, l logger.Logger, client *ovn.NB, aclProjectName string, ignoreUsageType any, ignoreUsageNicName string, keepACLs ...string) error { var aclNameIDs map[string]int64 var aclNames []string var projectID int64 err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get all the ACLs. acls, err := cluster.GetNetworkACLs(ctx, tx.Tx(), cluster.NetworkACLFilter{Project: &aclProjectName}) if err != nil { return err } // Convert acls to aclNames slice for use with UsedBy. aclNames = make([]string, 0, len(acls)) aclNameIDs = make(map[string]int64) for _, acl := range acls { aclNames = append(aclNames, acl.Name) aclNameIDs[acl.Name] = int64(acl.ID) } // Get project ID. projectID, err = cluster.GetProjectID(ctx, tx.Tx(), aclProjectName) if err != nil { return fmt.Errorf("Failed getting project ID for project %q: %w", aclProjectName, err) } return nil }) if err != nil { return err } // Get list of OVN port groups associated to this project. portGroups, err := client.GetPortGroupsByProject(context.TODO(), projectID) if err != nil { return fmt.Errorf("Failed getting port groups for project %q: %w", aclProjectName, err) } // hasKeeperPrefix indicates if the port group provided matches the prefix of one of the keepACLs. // This will include ACL network port groups too. hasKeeperPrefix := func(portGroup ovn.OVNPortGroup) bool { for _, keepACLName := range keepACLs { keepACLPortGroup := OVNACLPortGroupNamePrefix(aclNameIDs[keepACLName]) if strings.HasPrefix(string(portGroup), keepACLPortGroup) { return true } } return false } // Filter project port group list by ACL related ones, and store them in a map keyed by port group name. // This contains the initial candidates for removal. But any found to be in use will be removed from list. removeACLPortGroups := make(map[ovn.OVNPortGroup]struct{}) for _, portGroup := range portGroups { // If port group is related to an ACL and is not related to one of keepACLs, then add it as a // candidate for removal. if strings.HasPrefix(string(portGroup), ovnACLPortGroupPrefix) && !hasKeeperPrefix(portGroup) { removeACLPortGroups[portGroup] = struct{}{} } } // Add keepACLs to ovnUsedACLs to indicate they are explicitly in use by OVN. This is important because it // also ensures that indirectly referred ACLs in the rulesets of these ACLs will also be kept even if not // found to be in use in the database yet. ovnUsedACLs := make(map[string]struct{}, len(keepACLs)) for _, keepACLName := range keepACLs { ovnUsedACLs[keepACLName] = struct{}{} } // Map to record ACLs being referenced by other ACLs. Need to check later if they are in use with OVN ACLs. aclUsedACLS := make(map[string][]string) // Find all ACLs that are either directly referred to by OVN entities (networks, instance/profile NICs) // or indirectly by being referred to by a ruleset of another ACL that is itself in use by OVN entities. // For the indirectly referred to ACLs, store a list of the ACLs that are referring to it. err = UsedBy(s, aclProjectName, func(ctx context.Context, tx *db.ClusterTx, matchedACLNames []string, usageType any, nicName string, nicConfig map[string]string) error { switch u := usageType.(type) { case db.InstanceArgs: ignoreInst, isIgnoreInst := ignoreUsageType.(instance.Instance) if isIgnoreInst && ignoreUsageNicName == "" { return errors.New("ignoreUsageNicName should be specified when providing an instance in ignoreUsageType") } // If an ignore instance was provided, then skip the device that the ACLs were just removed // from. In case DB record is not updated until the update process has completed otherwise // we would still consider it using the ACL. if isIgnoreInst && ignoreInst.Name() == u.Name && ignoreInst.Project().Name == u.Project && ignoreUsageNicName == nicName { return nil } netID, network, _, err := tx.GetNetworkInAnyState(ctx, aclProjectName, nicConfig["network"]) if err != nil { return fmt.Errorf("Failed to load network %q: %w", nicConfig["network"], err) } if network.Type == "ovn" { for _, matchedACLName := range matchedACLNames { ovnUsedACLs[matchedACLName] = struct{}{} // Record as in use by OVN entity. // Delete entries (if exist) for ACL and per-ACL-per-network port groups. dPortGroups := OVNACLDirectionalPortGroups(aclNameIDs[matchedACLName]) dPortGroups.Remove(removeACLPortGroups) delete(removeACLPortGroups, OVNACLNetworkPortGroupName(aclNameIDs[matchedACLName], netID)) } } case *api.Network: ignoreNet, isIgnoreNet := ignoreUsageType.(*api.Network) if isIgnoreNet && ignoreUsageNicName != "" { return errors.New("ignoreUsageNicName should be empty when providing a network in ignoreUsageType") } // If an ignore network was provided, then skip the network that the ACLs were just removed // from. In case DB record is not updated until the update process has completed otherwise // we would still consider it using the ACL. if isIgnoreNet && ignoreNet.Name == u.Name { return nil } if u.Type == "ovn" { netID, _, _, err := tx.GetNetworkInAnyState(ctx, aclProjectName, u.Name) if err != nil { return fmt.Errorf("Failed to load network %q: %w", nicConfig["network"], err) } for _, matchedACLName := range matchedACLNames { ovnUsedACLs[matchedACLName] = struct{}{} // Record as in use by OVN entity. // Delete entries (if exist) for ACL and per-ACL-per-network port groups. dPortGroups := OVNACLDirectionalPortGroups(aclNameIDs[matchedACLName]) dPortGroups.Remove(removeACLPortGroups) delete(removeACLPortGroups, OVNACLNetworkPortGroupName(aclNameIDs[matchedACLName], netID)) } } case cluster.Profile: ignoreProfile, isIgnoreProfile := ignoreUsageType.(cluster.Profile) if isIgnoreProfile && ignoreUsageNicName == "" { return errors.New("ignoreUsageNicName should be specified when providing a profile in ignoreUsageType") } // If an ignore profile was provided, then skip the device that the ACLs were just removed // from. In case DB record is not updated until the update process has completed otherwise // we would still consider it using the ACL. if isIgnoreProfile && ignoreProfile.Name == u.Name && ignoreProfile.Project == u.Project && ignoreUsageNicName == nicName { return nil } netID, network, _, err := tx.GetNetworkInAnyState(ctx, aclProjectName, nicConfig["network"]) if err != nil { return fmt.Errorf("Failed to load network %q: %w", nicConfig["network"], err) } if network.Type == "ovn" { for _, matchedACLName := range matchedACLNames { ovnUsedACLs[matchedACLName] = struct{}{} // Record as in use by OVN entity. // Delete entries (if exist) for ACL and per-ACL-per-network port groups. dPortGroups := OVNACLDirectionalPortGroups(aclNameIDs[matchedACLName]) dPortGroups.Remove(removeACLPortGroups) delete(removeACLPortGroups, OVNACLNetworkPortGroupName(aclNameIDs[matchedACLName], netID)) } } case *api.NetworkACL: // Record which ACLs this ACL's ruleset refers to. for _, matchedACLName := range matchedACLNames { if aclUsedACLS[matchedACLName] == nil { aclUsedACLS[matchedACLName] = make([]string, 0, 1) } if !slices.Contains(aclUsedACLS[matchedACLName], u.Name) { // Record as in use by another ACL entity. aclUsedACLS[matchedACLName] = append(aclUsedACLS[matchedACLName], u.Name) } } default: return fmt.Errorf("Unrecognised usage type %T", u) } return nil }, aclNames...) if err != nil && !errors.Is(err, db.ErrInstanceListStop) { return fmt.Errorf("Failed getting ACL usage: %w", err) } // usedByOvn checks if any of the aclNames are in use by an OVN entity (network or instance/profile NIC). usedByOvn := func(aclNames ...string) bool { for _, aclName := range aclNames { _, found := ovnUsedACLs[aclName] if found { return true } } return false } // Check each ACL referenced in the rulesets of other ACLs whether any of the ACLs they were referenced // from are in use by ACLs that are also being used by OVN. If not then we don't need to keep the // referenced port group in OVN. for aclName, refACLs := range aclUsedACLS { if usedByOvn(refACLs...) { // Delete entry (if exists) for ACL port group. dPortGroups := OVNACLDirectionalPortGroups(aclNameIDs[aclName]) dPortGroups.Remove(removeACLPortGroups) } } // Now remove any remaining port groups left in removeACLPortGroups. removePortGroups := make([]ovn.OVNPortGroup, 0, len(removeACLPortGroups)) for removeACLPortGroup := range removeACLPortGroups { removePortGroups = append(removePortGroups, removeACLPortGroup) l.Debug("Scheduled deletion of unused ACL OVN port group", logger.Ctx{"portGroup": removeACLPortGroup}) } if len(removePortGroups) > 0 { err = client.DeletePortGroup(context.TODO(), removePortGroups...) if err != nil { return fmt.Errorf("Failed to delete unused OVN port groups: %w", err) } } return nil } // OVNPortGroupInstanceNICSchedule adds the specified NIC port to the specified port groups in the changeSet. func OVNPortGroupInstanceNICSchedule(portUUID ovn.OVNSwitchPortUUID, changeSet map[ovn.OVNPortGroup][]ovn.OVNSwitchPortUUID, portGroups ...ovn.OVNPortGroup) { for _, portGroupName := range portGroups { _, found := changeSet[portGroupName] if !found { changeSet[portGroupName] = []ovn.OVNSwitchPortUUID{} } changeSet[portGroupName] = append(changeSet[portGroupName], portUUID) } } // OVNApplyInstanceNICDefaultRules applies instance NIC default rules to per-network port group. func OVNApplyInstanceNICDefaultRules(client *ovn.NB, switchPortGroup ovn.OVNPortGroup, logPrefix string, nicPortName ovn.OVNSwitchPort, ingressAction string, ingressLogged bool, egressAction string, egressLogged bool) error { if !slices.Contains(ValidActions, ingressAction) { return fmt.Errorf("Invalid ingress action %q", ingressAction) } if !slices.Contains(ValidActions, egressAction) { return fmt.Errorf("Invalid egress action %q", egressAction) } if egressAction == "allow" { egressAction = "allow-related" } rules := []ovn.OVNACLRule{ { Direction: "to-lport", Action: egressAction, Log: egressLogged, LogName: fmt.Sprintf("%s-egress", logPrefix), // Max 63 chars. Priority: ovnACLPriorityNICDefaultActionEgress, Match: fmt.Sprintf(`inport == "%s"`, nicPortName), // From NIC. }, { Direction: "to-lport", Action: ingressAction, Log: ingressLogged, LogName: fmt.Sprintf("%s-ingress", logPrefix), // Max 63 chars. Priority: ovnACLPriorityNICDefaultActionIngress, Match: fmt.Sprintf(`outport == "%s"`, nicPortName), // To NIC. }, } err := client.UpdatePortGroupPortACLRules(context.TODO(), switchPortGroup, nicPortName, rules...) if err != nil { return fmt.Errorf("Failed applying instance NIC default ACL rules for port %q: %w", nicPortName, err) } return nil } // ovnLogEntry is the type used for the JSON encoded entries on the log endpoint (when coming from OVN). type ovnLogEntry struct { Time string `json:"time"` Proto string `json:"proto"` Src string `json:"src"` Dst string `json:"dst"` SrcPort string `json:"src_port,omitempty"` DstPort string `json:"dst_port,omitempty"` ICMPType string `json:"icmp_type,omitempty"` ICMPCode string `json:"icmp_code,omitempty"` Action string `json:"action"` } // ovnParseLogEntry takes a log line and expected ACL prefix and returns a re-formated log entry if matching. func ovnParseLogEntry(input string, prefix string) string { fields := strings.Split(input, "|") // Skip unknown formatting. if len(fields) != 5 { return "" } // We only care about ACLs. if !strings.HasPrefix(fields[2], "acl_log") { return "" } // Parse the ACL log entry. aclEntry := map[string]string{} for _, entry := range util.SplitNTrimSpace(fields[4], ",", -1, true) { pair := strings.Split(entry, "=") if len(pair) != 2 { continue } aclEntry[strings.Trim(pair[0], "\"")] = strings.Trim(pair[1], "\"") } // Filter for our ACL. if !strings.HasPrefix(aclEntry["name"], prefix) { return "" } // Parse the timestamp. logTime, err := time.Parse(time.RFC3339, fields[0]) if err != nil { return "" } // Get the protocol. directionFields := strings.Split(aclEntry["direction"], " ") if len(directionFields) != 2 { return "" } protocol := directionFields[1] // Get the source and destination addresses. srcAddr, ok := aclEntry["nw_src"] if !ok { srcAddr, ok = aclEntry["ipv6_src"] if !ok { return "" } } dstAddr, ok := aclEntry["nw_dst"] if !ok { dstAddr, ok = aclEntry["ipv6_dst"] if !ok { return "" } } // Prepare the core log entry. newEntry := ovnLogEntry{ Time: logTime.UTC().Format(time.RFC3339), Proto: protocol, Src: srcAddr, Dst: dstAddr, ICMPType: aclEntry["icmp_type"], ICMPCode: aclEntry["icmp_code"], Action: aclEntry["verdict"], } // Add the source and destination ports. srcPort, ok := aclEntry["tp_src"] if ok { newEntry.SrcPort = srcPort } dstPort, ok := aclEntry["tp_dst"] if ok { newEntry.DstPort = dstPort } out, err := json.Marshal(&newEntry) if err != nil { return "" } return string(out) } func addPortGroupDefaultAction(portGroupName ovn.OVNPortGroup, portGroupRules []ovn.OVNACLRule) []ovn.OVNACLRule { // Add default rule to port group ACL. // This is a failsafe to drop unmatched traffic if the per-NIC default rule has unexpectedly not kicked in. defaultAction := "drop" defaultLogged := false return append(portGroupRules, ovn.OVNACLRule{ Direction: "to-lport", // Always use this so that outport is available to Match. Action: defaultAction, Priority: ovnACLPriorityPortGroupDefaultAction, // Lowest priority to catch only unmatched traffic. Match: fmt.Sprintf("(inport == @%s || outport == @%s)", portGroupName, portGroupName), Log: defaultLogged, LogName: string(portGroupName), }) } incus-7.0.0/internal/server/network/acl/acl_validation.go000066400000000000000000000013531517523235500235230ustar00rootroot00000000000000package acl import ( "fmt" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // ValidName checks the ACL name is valid. func ValidName(name string) error { err := validate.IsAPIName(name, false) if err != nil { return err } // Don't allow ACL names to start with special port selector characters to allow Incus to define special port // selectors without risking conflict with user defined ACL names. if util.StringHasPrefix(name, "@", "%", "#") { return fmt.Errorf("Name cannot start with reserved character %q", name[0]) } // Ensures we can differentiate an ACL name from an IP in rules that reference this ACL. err = validate.IsHostname(name) if err != nil { return err } return nil } incus-7.0.0/internal/server/network/acl/driver_common.go000066400000000000000000000646061517523235500234270ustar00rootroot00000000000000package acl import ( "bufio" "context" "errors" "fmt" "net" "net/http" "os" "slices" "sort" "strings" "sync" incus "github.com/lxc/incus/v7/client" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" addressset "github.com/lxc/incus/v7/internal/server/network/address-set" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // Define type for rule directions. type ruleDirection string const ( ruleDirectionIngress ruleDirection = "ingress" ruleDirectionEgress ruleDirection = "egress" ) // ReservedNetworkSubects contains a list of reserved network peer names (those starting with @ character) that // cannot be used when to name peering connections. Otherwise peer connections wouldn't be able to be referenced // in ACL rules using the "@" format without the potential of conflicts. var ReservedNetworkSubects = []string{"internal", "external"} // Define reserved ACL subjects. const ( ruleSubjectInternal = "@internal" ruleSubjectExternal = "@external" ) // Define aliases for reserved ACL subjects. This is to allow earlier deprecated names that used the "#" prefix. // They were deprecated to avoid confusion with YAML comments. So "#internal" and "#external" should not be used. var ( ruleSubjectInternalAliases = []string{ruleSubjectInternal, "#internal"} ruleSubjectExternalAliases = []string{ruleSubjectExternal, "#external"} ) // ValidActions defines valid actions for rules. var ValidActions = []string{"allow", "allow-stateless", "drop", "reject"} // common represents a Network ACL. type common struct { logger logger.Logger state *state.State id int64 projectName string info *api.NetworkACL } // init initialize internal variables. func (d *common) init(state *state.State, id int64, projectName string, info *api.NetworkACL) { if info == nil { d.info = &api.NetworkACL{} } else { d.info = info } d.logger = logger.AddContext(logger.Ctx{"project": projectName, "networkACL": d.info.Name}) d.id = id d.projectName = projectName d.state = state if d.info.Ingress == nil { d.info.Ingress = []api.NetworkACLRule{} } for i := range d.info.Ingress { d.info.Ingress[i].Normalise() } if d.info.Egress == nil { d.info.Egress = []api.NetworkACLRule{} } for i := range d.info.Egress { d.info.Egress[i].Normalise() } if d.info.Config == nil { d.info.Config = make(map[string]string) } } // ID returns the Network ACL ID. func (d *common) ID() int64 { return d.id } // Project returns the project name. func (d *common) Project() string { return d.projectName } // Info returns copy of internal info for the Network ACL. func (d *common) Info() *api.NetworkACL { // Copy internal info to prevent modification externally. info := api.NetworkACL{} info.Name = d.info.Name info.Description = d.info.Description info.Ingress = append(make([]api.NetworkACLRule, 0, len(d.info.Ingress)), d.info.Ingress...) info.Egress = append(make([]api.NetworkACLRule, 0, len(d.info.Egress)), d.info.Egress...) info.Config = localUtil.CopyConfig(d.info.Config) info.UsedBy = nil // To indicate its not populated (use Usedby() function to populate). info.Project = d.projectName return &info } // usedBy returns a list of API endpoints referencing this ACL. // If firstOnly is true then search stops at first result. func (d *common) usedBy(firstOnly bool) ([]string, error) { usedBy := []string{} // Find all networks, profiles and instance NICs that use this Network ACL. err := UsedBy(d.state, d.projectName, func(ctx context.Context, tx *db.ClusterTx, _ []string, usageType any, _ string, _ map[string]string) error { switch u := usageType.(type) { case db.InstanceArgs: uri := fmt.Sprintf("/%s/instances/%s", version.APIVersion, u.Name) if u.Project != api.ProjectDefaultName { uri += fmt.Sprintf("?project=%s", u.Project) } usedBy = append(usedBy, uri) case *api.Network: uri := fmt.Sprintf("/%s/networks/%s", version.APIVersion, u.Name) if d.projectName != api.ProjectDefaultName { uri += fmt.Sprintf("?project=%s", d.projectName) } usedBy = append(usedBy, uri) case dbCluster.Profile: uri := fmt.Sprintf("/%s/profiles/%s", version.APIVersion, u.Name) if u.Project != api.ProjectDefaultName { uri += fmt.Sprintf("?project=%s", u.Project) } usedBy = append(usedBy, uri) case *api.NetworkACL: uri := fmt.Sprintf("/%s/network-acls/%s", version.APIVersion, u.Name) if d.projectName != api.ProjectDefaultName { uri += fmt.Sprintf("?project=%s", d.projectName) } usedBy = append(usedBy, uri) default: return fmt.Errorf("Unrecognised usage type %T", u) } if firstOnly { return db.ErrInstanceListStop } return nil }, d.Info().Name) if err != nil { if errors.Is(err, db.ErrInstanceListStop) { return usedBy, nil } return nil, fmt.Errorf("Failed getting ACL usage: %w", err) } return usedBy, nil } // UsedBy returns a list of API endpoints referencing this ACL. func (d *common) UsedBy() ([]string, error) { return d.usedBy(false) } // isUsed returns whether or not the ACL is in use. func (d *common) isUsed() (bool, error) { usedBy, err := d.usedBy(true) if err != nil { return false, err } return len(usedBy) > 0, nil } // Etag returns the values used for etag generation. func (d *common) Etag() []any { return []any{d.info.Name, d.info.Description, d.info.Ingress, d.info.Egress, d.info.Config} } // validateName checks name is valid. func (d *common) validateName(name string) error { return ValidName(name) } // validateConfig checks the config and rules are valid. func (d *common) validateConfig(info *api.NetworkACLPut) error { err := d.validateConfigMap(info.Config, nil) if err != nil { return err } // Normalise rules before validation for duplicate detection. for i := range info.Ingress { info.Ingress[i].Normalise() } for i := range info.Egress { info.Egress[i].Normalise() } // Validate each ingress rule. for i, ingressRule := range info.Ingress { err := d.validateRule(ruleDirectionIngress, ingressRule) if err != nil { return fmt.Errorf("Invalid ingress rule %d: %w", i, err) } // Check for duplicates. for ri, r := range info.Ingress { if ri == i { continue // Skip ourselves. } if r == ingressRule { return fmt.Errorf("Duplicate of ingress rule %d", i) } } } // Validate each egress rule. for i, egressRule := range info.Egress { err := d.validateRule(ruleDirectionEgress, egressRule) if err != nil { return fmt.Errorf("Invalid egress rule %d: %w", i, err) } // Check for duplicates. for ri, r := range info.Egress { if ri == i { continue // Skip ourselves. } if r == egressRule { return fmt.Errorf("Duplicate of egress rule %d", i) } } } return nil } // validateConfigMap checks ACL config map against rules. func (d *common) validateConfigMap(config map[string]string, rules map[string]func(value string) error) error { checkedFields := map[string]struct{}{} // Run the validator against each field. for k, validator := range rules { checkedFields[k] = struct{}{} // Mark field as checked. err := validator(config[k]) if err != nil { return fmt.Errorf("Invalid value for config option %q: %w", k, err) } } // Look for any unchecked fields, as these are unknown fields and validation should fail. for k := range config { _, checked := checkedFields[k] if checked { continue } // User keys are not validated. if internalInstance.IsUserConfig(k) { continue } return fmt.Errorf("Invalid config option %q", k) } return nil } // validateRule validates the rule supplied. func (d *common) validateRule(direction ruleDirection, rule api.NetworkACLRule) error { // Validate Action field (required). if !slices.Contains(ValidActions, rule.Action) { return fmt.Errorf("Action must be one of: %s", strings.Join(ValidActions, ", ")) } // Validate State field (required). validStates := []string{"enabled", "disabled", "logged"} if !slices.Contains(validStates, rule.State) { return fmt.Errorf("State must be one of: %s", strings.Join(validStates, ", ")) } var acls map[string]int64 err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get map of ACL names to DB IDs (used for generating OVN port group names). dbAcls, err := dbCluster.GetNetworkACLs(ctx, tx.Tx(), dbCluster.NetworkACLFilter{Project: &d.projectName}) if err != nil { return err } acls = make(map[string]int64, len(dbAcls)) for _, acl := range dbAcls { acls[acl.Name] = int64(acl.ID) } return nil }) if err != nil { return fmt.Errorf("Failed getting network ACLs for security ACL subject validation: %w", err) } validSubjectNames := make([]string, 0, len(acls)+len(ruleSubjectInternalAliases)+len(ruleSubjectExternalAliases)) validSubjectNames = append(validSubjectNames, ruleSubjectInternalAliases...) validSubjectNames = append(validSubjectNames, ruleSubjectExternalAliases...) for aclName := range acls { validSubjectNames = append(validSubjectNames, aclName) } var srcHasName, srcHasIPv4, srcHasIPv6 bool var dstHasName, dstHasIPv4, dstHasIPv6 bool // Validate Source field. if rule.Source != "" { srcHasName, srcHasIPv4, srcHasIPv6, err = d.validateRuleSubjects("Source", direction, util.SplitNTrimSpace(rule.Source, ",", -1, false), validSubjectNames) if err != nil { return fmt.Errorf("Invalid Source: %w", err) } } // Validate Destination field. if rule.Destination != "" { dstHasName, dstHasIPv4, dstHasIPv6, err = d.validateRuleSubjects("Destination", direction, util.SplitNTrimSpace(rule.Destination, ",", -1, false), validSubjectNames) if err != nil { return fmt.Errorf("Invalid Destination: %w", err) } } // Check combination of subject types is valid for source/destination. if rule.Source != "" && rule.Destination != "" { if (srcHasIPv4 && !dstHasIPv4 && !dstHasName) || (dstHasIPv4 && !srcHasIPv4 && !srcHasName) || (srcHasIPv6 && !dstHasIPv6 && !dstHasName) || (dstHasIPv6 && !srcHasIPv6 && !srcHasName) { return errors.New("Conflicting IP family types used for Source and Destination") } } // Validate Protocol field. if rule.Protocol != "" { validProtocols := []string{"icmp4", "icmp6", "tcp", "udp"} if !slices.Contains(validProtocols, rule.Protocol) { return fmt.Errorf("Protocol must be one of: %s", strings.Join(validProtocols, ", ")) } } // Validate protocol dependent fields. if slices.Contains([]string{"tcp", "udp"}, rule.Protocol) { if rule.ICMPType != "" { return errors.New("ICMP type cannot be used with non-ICMP protocol") } if rule.ICMPCode != "" { return errors.New("ICMP code cannot be used with non-ICMP protocol") } // Validate SourcePort field. if rule.SourcePort != "" { err := d.validatePorts(util.SplitNTrimSpace(rule.SourcePort, ",", -1, false)) if err != nil { return fmt.Errorf("Invalid Source port: %w", err) } } // Validate DestinationPort field. if rule.DestinationPort != "" { err := d.validatePorts(util.SplitNTrimSpace(rule.DestinationPort, ",", -1, false)) if err != nil { return fmt.Errorf("Invalid Destination port: %w", err) } } } else if slices.Contains([]string{"icmp4", "icmp6"}, rule.Protocol) { if rule.SourcePort != "" { return fmt.Errorf("Source port cannot be used with %q protocol", rule.Protocol) } if rule.DestinationPort != "" { return fmt.Errorf("Destination port cannot be used with %q protocol", rule.Protocol) } if rule.Protocol == "icmp4" { if srcHasIPv6 { return fmt.Errorf("Cannot use IPv6 source addresses with %q protocol", rule.Protocol) } if dstHasIPv6 { return fmt.Errorf("Cannot use IPv6 destination addresses with %q protocol", rule.Protocol) } } else if rule.Protocol == "icmp6" { if srcHasIPv4 { return fmt.Errorf("Cannot use IPv4 source addresses with %q protocol", rule.Protocol) } if dstHasIPv4 { return fmt.Errorf("Cannot use IPv4 destination addresses with %q protocol", rule.Protocol) } } // Validate ICMPType field. if rule.ICMPType != "" { err := validate.IsUint8(rule.ICMPType) if err != nil { return fmt.Errorf("Invalid ICMP type: %w", err) } } // Validate ICMPCode field. if rule.ICMPCode != "" { err := validate.IsUint8(rule.ICMPCode) if err != nil { return fmt.Errorf("Invalid ICMP code: %w", err) } } } else { if rule.ICMPType != "" { return errors.New("ICMP type cannot be used without specifying protocol") } if rule.ICMPCode != "" { return errors.New("ICMP code cannot be used without specifying protocol") } if rule.SourcePort != "" { return errors.New("Source port cannot be used without specifying protocol") } if rule.DestinationPort != "" { return errors.New("Destination port cannot be used without specifying protocol") } } return nil } // validateRuleSubjects checks that the source or destination subjects for a rule are valid. // Accepts a validSubjectNames list of valid ACL or special classifier names. // Returns whether the subjects include names, IPv4 and IPv6 addresses respectively. func (d *common) validateRuleSubjects(fieldName string, direction ruleDirection, subjects []string, validSubjectNames []string) (bool, bool, bool, error) { // Check if named subjects are allowed in field/direction combination. allowSubjectNames := false if (fieldName == "Source" && direction == ruleDirectionIngress) || (fieldName == "Destination" && direction == ruleDirectionEgress) { allowSubjectNames = true } isNetworkAddress := func(value string) (uint, error) { ip := net.ParseIP(value) if ip == nil { return 0, fmt.Errorf("Not an IP address %q", value) } var ipVersion uint = 4 if ip.To4() == nil { ipVersion = 6 } return ipVersion, nil } isNetworkAddressCIDR := func(value string) (uint, error) { ip, _, err := net.ParseCIDR(value) if err != nil { return 0, err } var ipVersion uint = 4 if ip.To4() == nil { ipVersion = 6 } return ipVersion, nil } isNetworkRange := func(value string) (uint, error) { err := validate.IsNetworkRange(value) if err != nil { return 0, err } ips := strings.SplitN(value, "-", 2) if len(ips) != 2 { return 0, errors.New("IP range must contain start and end IP addresses") } ip := net.ParseIP(ips[0]) var ipVersion uint = 4 if ip.To4() == nil { ipVersion = 6 } return ipVersion, nil } checks := []func(s string) (uint, error){ isNetworkAddress, isNetworkAddressCIDR, isNetworkRange, } validSubject := func(subject string) (uint, error) { // Check if it is one of the network IP types. for _, c := range checks { ipVersion, err := c(subject) if err == nil { return ipVersion, nil // Found valid subject. } } // Check if it is one of the valid subject names. for _, n := range validSubjectNames { if subject == n { if allowSubjectNames { return 0, nil // Found valid subject. } return 0, fmt.Errorf("Named subjects not allowed in %q for %q rules", fieldName, direction) } } // Check if it looks like a network peer connection name. if strings.HasPrefix(subject, "@") { if allowSubjectNames { return 0, nil // Found valid subject. } return 0, fmt.Errorf("Named subjects not allowed in %q for %q rules", fieldName, direction) } if strings.HasPrefix(subject, "$") { addrSetName := strings.Trim(subject, "$") // Check that the address set exists. err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error _, err = dbCluster.GetNetworkAddressSet(ctx, tx.Tx(), d.Project(), addrSetName) return err }) if err != nil { return 0, fmt.Errorf("Failed getting network address set %q for subject validation: %w", addrSetName, err) } return 0, nil // Found valid subject. } return 0, fmt.Errorf("Invalid subject %q", subject) } hasIPv4 := false hasIPv6 := false hasName := false for _, s := range subjects { ipVersion, err := validSubject(s) if err != nil { return false, false, false, err } switch ipVersion { case 0: hasName = true case 4: hasIPv4 = true case 6: hasIPv6 = true } } return hasName, hasIPv4, hasIPv6, nil } // validatePorts checks that the source or destination ports for a rule are valid. func (d *common) validatePorts(ports []string) error { for _, port := range ports { err := validate.IsNetworkPortRange(port) if err != nil { return err } } return nil } // Update applies the supplied config to the ACL. func (d *common) Update(config *api.NetworkACLPut, clientType request.ClientType) error { // Validate the configuration. err := d.validateConfig(config) if err != nil { return err } reverter := revert.New() defer reverter.Fail() if clientType == request.ClientTypeNormal { oldConfig := d.info.NetworkACLPut err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Update database. Its important this occurs before we attempt to apply to networks using the ACL // as usage functions will inspect the database. return dbCluster.UpdateNetworkACLAPI(ctx, tx.Tx(), d.id, config) }) if err != nil { return err } // Apply changes internally and reinitialize. d.info.NetworkACLPut = *config d.init(d.state, d.id, d.projectName, d.info) reverter.Add(func() { _ = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return dbCluster.UpdateNetworkACLAPI(ctx, tx.Tx(), d.id, &oldConfig) }) d.info.NetworkACLPut = oldConfig d.init(d.state, d.id, d.projectName, d.info) }) } // Get a list of networks that are using this ACL (either directly or indirectly via a NIC). aclNets := map[string]NetworkACLUsage{} err = NetworkUsage(d.state, d.projectName, []string{d.info.Name}, aclNets) if err != nil { return fmt.Errorf("Failed getting ACL network usage: %w", err) } // Separate out OVN networks from non-OVN networks. This is because OVN networks share ACL config, and // so changes are not applied entirely on a per-network basis and need to be treated differently. // Separate the bridge networks used indirectly by NIC devices. This is because the ACL rules need to be // applied to the bridge interface, not the network. aclOVNNets := map[string]NetworkACLUsage{} aclBridgeNICs := map[string]NetworkACLUsage{} for k, v := range aclNets { if v.Type == "ovn" { delete(aclNets, k) aclOVNNets[k] = v } else if v.Type == "bridge" && v.DeviceName != "" { delete(aclNets, k) aclBridgeNICs[k] = v } else if v.Type != "bridge" { return fmt.Errorf("Unsupported network ACL type %q", v.Type) } } // Apply ACL changes to non-OVN networks on this member. for _, aclNet := range aclNets { err = addressset.FirewallApplyAddressSetsForACLRules(d.state, "inet", d.projectName, []string{d.info.Name}) if err != nil { return err } // Only trigger application on related bridged networks that directly use the ACL. networkACLs := util.SplitNTrimSpace(aclNet.Config["security.acls"], ",", -1, true) if slices.Contains(networkACLs, d.info.Name) { err = FirewallApplyACLRules(d.state, d.logger, d.projectName, aclNet) if err != nil { return err } } } // If there are affected bridge NICs, apply the ACL changes to the bridge interface filter. if len(aclBridgeNICs) > 0 { err = addressset.FirewallApplyAddressSetsForACLRules(d.state, "bridge", d.projectName, []string{d.info.Name}) if err != nil { return err } err := BridgeUpdateACLs(d.state, d.logger, d.projectName, aclBridgeNICs) if err != nil { return fmt.Errorf("Failed updating bridge NIC ACL: %w", err) } } // If there are affected OVN networks, then apply the changes, but only if the request type is normal. // This way we won't apply the same changes multiple times for each cluster member. if len(aclOVNNets) > 0 && clientType == request.ClientTypeNormal { // Check that OVN is available. ovnnb, _, err := d.state.OVN() if err != nil { return err } var aclNameIDs map[string]int64 err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get map of ACL names to DB IDs (used for generating OVN port group names). acls, err := dbCluster.GetNetworkACLs(ctx, tx.Tx(), dbCluster.NetworkACLFilter{Project: &d.projectName}) if err != nil { return err } aclNameIDs = make(map[string]int64, len(acls)) for _, acl := range acls { aclNameIDs[acl.Name] = int64(acl.ID) } return nil }) if err != nil { return fmt.Errorf("Failed getting network ACL IDs for security ACL update: %w", err) } // Request that the ACL and any referenced ACLs in the ruleset are created in OVN. // Pass aclOVNNets info, because although OVN networks share ACL port group definitions, when the // ACL rules themselves use network specific selectors such as @internal/@external, we then need to // apply those rules to each network affected by the ACL, so pass the full list of OVN networks // affected by this ACL (either because the ACL is assigned directly or because it is assigned to // an OVN NIC in an instance or profile). cleanup, err := OVNEnsureACLs(d.state, d.logger, ovnnb, d.projectName, aclNameIDs, aclOVNNets, []string{d.info.Name}, true) if err != nil { return fmt.Errorf("Failed ensuring ACL is configured in OVN: %w", err) } reverter.Add(cleanup) cleanup, err = addressset.OVNEnsureAddressSetsViaACLs(d.state, d.logger, ovnnb, d.projectName, []string{d.info.Name}) if err != nil { return fmt.Errorf("Failed ensuring Address sets is configured for ACL %s in OVN: %w", d.info.Name, err) } reverter.Add(cleanup) // Run unused port group cleanup in case any formerly referenced ACL in this ACL's rules means that // an ACL port group is now considered unused. err = OVNPortGroupDeleteIfUnused(d.state, d.logger, ovnnb, d.projectName, nil, "", d.info.Name) if err != nil { return fmt.Errorf("Failed removing unused OVN port groups: %w", err) } } // Apply ACL changes to non-OVN networks on cluster members. if clientType == request.ClientTypeNormal && len(aclNets) > 0 { // Notify all other nodes to update the network if no target specified. notifier, err := cluster.NewNotifier(d.state, d.state.Endpoints.NetworkCert(), d.state.ServerCert(), cluster.NotifyAll) if err != nil { return err } err = notifier(func(client incus.InstanceServer) error { return client.UseProject(d.projectName).UpdateNetworkACL(d.info.Name, d.info.NetworkACLPut, "") }) if err != nil { return err } } reverter.Success() return nil } // Rename renames the ACL if not in use. func (d *common) Rename(newName string) error { _, err := LoadByName(d.state, d.projectName, newName) if err == nil { return errors.New("An ACL by that name exists already") } isUsed, err := d.isUsed() if err != nil { return err } if isUsed { return errors.New("Cannot rename an ACL that is in use") } err = d.validateName(newName) if err != nil { return err } err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { idInt := int(d.id) acls, err := dbCluster.GetNetworkACLs(ctx, tx.Tx(), dbCluster.NetworkACLFilter{ID: &idInt}) if err != nil { return err } if len(acls) == 0 { return api.StatusErrorf(http.StatusNotFound, "Network ACL not found") } return dbCluster.RenameNetworkACL(ctx, tx.Tx(), acls[0].Project, acls[0].Name, newName) }) if err != nil { return err } // Apply changes internally. d.info.Name = newName return nil } // Delete deletes the ACL. func (d *common) Delete() error { isUsed, err := d.isUsed() if err != nil { return err } if isUsed { return errors.New("Cannot delete an ACL that is in use") } return d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return dbCluster.DeleteNetworkACL(ctx, tx.Tx(), int(d.id)) }) } // GetLog gets the ACL log. func (d *common) GetLog(clientType request.ClientType) (string, error) { // ACLs aren't specific to a particular network type but the log only works with OVN. logPath := "/var/log/ovn/ovn-controller.log" if !util.PathExists(logPath) { return "", errors.New("Only OVN log entries may be retrieved at this time") } // Open the log file. logFile, err := os.Open(logPath) if err != nil { return "", fmt.Errorf("Couldn't open OVN log file: %w", err) } defer func() { _ = logFile.Close() }() logEntries := []string{} scanner := bufio.NewScanner(logFile) for scanner.Scan() { logEntry := ovnParseLogEntry(scanner.Text(), fmt.Sprintf("incus_acl%d-", d.id)) if logEntry == "" { continue } logEntries = append(logEntries, logEntry) } err = scanner.Err() if err != nil { return "", fmt.Errorf("Failed to read OVN log file: %w", err) } // Aggregates the entries from the rest of the cluster. if clientType == request.ClientTypeNormal { // Setup notifier to reach the rest of the cluster. notifier, err := cluster.NewNotifier(d.state, d.state.Endpoints.NetworkCert(), d.state.ServerCert(), cluster.NotifyAll) if err != nil { return "", err } mu := sync.Mutex{} err = notifier(func(client incus.InstanceServer) error { // Get the entries. entries, err := client.UseProject(d.projectName).GetNetworkACLLogfile(d.info.Name) if err != nil { return err } defer func() { _ = entries.Close() }() // Prevent concurrent writes to the log entries slice. mu.Lock() defer mu.Unlock() // Parse the response and add to the slice. scanner := bufio.NewScanner(entries) for scanner.Scan() { entry := scanner.Text() if entry == "" { continue } logEntries = append(logEntries, entry) } err = scanner.Err() if err != nil { return fmt.Errorf("Failed to read OVN log file: %w", err) } return nil }) if err != nil { return "", err } } // Just return empty if no log entries (no need for trailing line break). if len(logEntries) == 0 { return "", nil } // Sort the entries (by timestamp). sort.Strings(logEntries) return strings.Join(logEntries, "\n") + "\n", nil } incus-7.0.0/internal/server/network/address-set/000077500000000000000000000000001517523235500217005ustar00rootroot00000000000000incus-7.0.0/internal/server/network/address-set/address_set_firewall.go000066400000000000000000000070551517523235500264230ustar00rootroot00000000000000package addressset import ( "context" "fmt" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" firewallDrivers "github.com/lxc/incus/v7/internal/server/firewall/drivers" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" ) // FirewallApplyAddressSets applies address set rules to the network firewall. func FirewallApplyAddressSets(s *state.State, projectName string, addressSet AddressSetUsage) error { sets, err := FirewallAddressSets(s, projectName) if err != nil { return err } // Here assume nftTable is inet because we are applying directly to firewall err = s.Firewall.NetworkApplyAddressSets(sets, "inet") if err != nil { return err } return nil } // FirewallApplyAddressSetsForACLRules apply address-sets from ACLNames to the correct nft Table. func FirewallApplyAddressSetsForACLRules(s *state.State, nftTable string, projectName string, ACLNames []string) error { // Build address set usage from network ACLs. var apiSets []*api.NetworkAddressSet var fwSets []firewallDrivers.AddressSet setsNames, err := GetAddressSetsForACLs(s, projectName, ACLNames) if err != nil { return err } // convertAddressSets convert the address set to a Firewall named set. convertAddressSets := func(apiSets []*api.NetworkAddressSet) error { for _, set := range apiSets { firewallAddressSet := firewallDrivers.AddressSet{ Name: set.Name, Addresses: set.Addresses, } fwSets = append(fwSets, firewallAddressSet) } return nil } for _, setName := range setsNames { err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error dbSet, err := dbCluster.GetNetworkAddressSet(ctx, tx.Tx(), projectName, setName) if err != nil { return err } set, err := dbSet.ToAPI(ctx, tx.Tx()) if err != nil { return err } apiSets = append(apiSets, set) return nil }) if err != nil { return fmt.Errorf("Failed loading address set %q: %w", setName, err) } } err = convertAddressSets(apiSets) if err != nil { return err } return s.Firewall.NetworkApplyAddressSets(fwSets, nftTable) } // FirewallAddressSets returns address sets for a network firewall. func FirewallAddressSets(s *state.State, addrSetProjectName string) ([]firewallDrivers.AddressSet, error) { var addressSets []firewallDrivers.AddressSet // convertAddressSets convert the address set to a Firewall named set. convertAddressSets := func(sets []*api.NetworkAddressSet) error { for _, set := range sets { firewallAddressSet := firewallDrivers.AddressSet{ Name: set.Name, Addresses: set.Addresses, } addressSets = append(addressSets, firewallAddressSet) } return nil } // Here we want to load every address set for a given project. var sets []*api.NetworkAddressSet err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error dbSets, err := dbCluster.GetNetworkAddressSets(ctx, tx.Tx(), dbCluster.NetworkAddressSetFilter{Project: &addrSetProjectName}) if err != nil { return err } for _, dbSet := range dbSets { set, err := dbSet.ToAPI(ctx, tx.Tx()) if err != nil { return err } sets = append(sets, set) } return nil }) if err != nil { return nil, fmt.Errorf("Failed loading address set names for network firewall: %w", err) } err = convertAddressSets(sets) if err != nil { return nil, fmt.Errorf("Failed converting address sets for network firewall: %w", err) } return addressSets, nil } incus-7.0.0/internal/server/network/address-set/address_set_interface.go000066400000000000000000000013401517523235500265450ustar00rootroot00000000000000package addressset import ( "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" ) // NetworkAddressSet represents a network address set. type NetworkAddressSet interface { // Initialize. init(s *state.State, id int, projectName string, info *api.NetworkAddressSet) // Info ID() int Project() string Info() *api.NetworkAddressSet Etag() []any UsedBy() ([]string, error) // Internal validation. validateName(name string) error validateConfig(config *api.NetworkAddressSetPut) error // Modifications. Update(config *api.NetworkAddressSetPut, clientType request.ClientType) error Rename(newName string) error Delete() error } incus-7.0.0/internal/server/network/address-set/address_set_load.go000066400000000000000000000432121517523235500255300ustar00rootroot00000000000000package addressset import ( "context" "errors" "fmt" "slices" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) // LoadByName loads and initializes a network address set from the database by project and name. func LoadByName(s *state.State, projectName string, name string) (NetworkAddressSet, error) { var id int var asInfo *api.NetworkAddressSet err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbSet, err := dbCluster.GetNetworkAddressSet(ctx, tx.Tx(), projectName, name) if err != nil { return err } asInfo, err = dbSet.ToAPI(ctx, tx.Tx()) if err != nil { return err } id = dbSet.ID return nil }) if err != nil { return nil, err } var as NetworkAddressSet = &common{} as.init(s, id, projectName, asInfo) return as, nil } // Create validates supplied record and creates a new network address set record in the database. func Create(s *state.State, projectName string, asInfo *api.NetworkAddressSetsPost) error { var addrSet NetworkAddressSet = &common{} addrSet.init(s, -1, projectName, nil) err := addrSet.validateName(asInfo.Name) if err != nil { return err } err = addrSet.validateConfig(&asInfo.NetworkAddressSetPut) if err != nil { return err } err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Insert DB record. id, err := dbCluster.CreateNetworkAddressSet(ctx, tx.Tx(), dbCluster.NetworkAddressSet{ Name: asInfo.Name, Description: asInfo.Description, Addresses: asInfo.Addresses, Project: projectName, }) if err != nil { return err } err = dbCluster.CreateNetworkAddressSetConfig(ctx, tx.Tx(), id, asInfo.Config) if err != nil { return err } return nil }) if err != nil { return err } return nil } // Exists checks the address set name(s) provided exist in the project. // If multiple names are provided, also checks that duplicate names aren't specified in the list. func Exists(s *state.State, projectName string, name ...string) error { var existingSetNames []string err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { sets, err := dbCluster.GetNetworkAddressSets(ctx, tx.Tx(), dbCluster.NetworkAddressSetFilter{Project: &projectName}) if err != nil { return err } for _, set := range sets { existingSetNames = append(existingSetNames, set.Name) } return nil }) if err != nil { return err } checkedSetNames := make(map[string]struct{}, len(name)) for _, setName := range name { if !slices.Contains(existingSetNames, setName) { return fmt.Errorf("Network address set %q does not exist", setName) } _, found := checkedSetNames[setName] if found { return fmt.Errorf("Network address set %q specified multiple times", setName) } checkedSetNames[setName] = struct{}{} } return nil } // AddressSetUsedBy calls usageFunc for each ACL that references the specified address set name. func AddressSetUsedBy(s *state.State, projectName string, usageFunc func(aclName string) error, addressSetName string) error { var aclNames []string var err error err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { acls, err := dbCluster.GetNetworkACLs(ctx, tx.Tx(), dbCluster.NetworkACLFilter{Project: &projectName}) if err != nil { return err } aclNames = make([]string, len(acls)) for i, acl := range acls { aclNames[i] = acl.Name } return nil }) if err != nil { return err } // Load each ACL and check rules. for _, aclName := range aclNames { var aclInfo *api.NetworkACL err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, aclInfo, err = dbCluster.GetNetworkACLAPI(ctx, tx.Tx(), projectName, aclName) return err }) if err != nil { return fmt.Errorf("Failed loading ACL %q: %w", aclName, err) } // Check all rules for reference to $addressSetName. refFound := false checkRule := func(rule api.NetworkACLRule) bool { // We assume address sets are referenced as "$addressSetName". if rule.Source != "" && subjectListReferences(rule.Source, addressSetName) { return true } if rule.Destination != "" && subjectListReferences(rule.Destination, addressSetName) { return true } return false } if slices.ContainsFunc(aclInfo.Ingress, checkRule) { refFound = true } if !refFound { if slices.ContainsFunc(aclInfo.Egress, checkRule) { refFound = true } } if refFound { err = usageFunc(aclName) if err != nil { if errors.Is(err, db.ErrInstanceListStop) { return err } return fmt.Errorf("Usage callback failed: %w", err) } } } return nil } // subjectListReferences checks if the subject list (comma separated) references $addressSetName. func subjectListReferences(subjects string, addressSetName string) bool { parts := util.SplitNTrimSpace(subjects, ",", -1, false) needle := "$" + addressSetName return slices.Contains(parts, needle) } // NetworkACLUsage info about a network and what ACL it uses. type NetworkACLUsage struct { ID int64 Name string Type string Config map[string]string InstanceName string DeviceName string } // AddressSetUsage holds info about a network using the address set. type AddressSetUsage struct { ID int Name string Type string DeviceName string Addresses []string Config map[string]string ACLNames []string } // AddressSetNetworkUsage retrieve the networks that use an address set by checking ACLs. func AddressSetNetworkUsage(s *state.State, projectName string, addressSetName string, addresses []string, asNets map[string]AddressSetUsage) error { // Get ACLs referencing this address set. aclNames := []string{} err := AddressSetUsedBy(s, projectName, func(aclName string) error { aclNames = append(aclNames, aclName) return nil }, addressSetName) if err != nil { return err } // Now get network usage from those ACLs. Reuse ACL's NetworkUsage function. aclNets := map[string]NetworkACLUsage{} err = ACLNetworkUsage(s, projectName, aclNames, aclNets) if err != nil { return err } // Convert ACLNetworkUsage entries into AddressSetUsage entries. for netName, netUsage := range aclNets { asNets[netName] = AddressSetUsage{ ID: int(netUsage.ID), Name: netUsage.Name, Type: netUsage.Type, DeviceName: netUsage.DeviceName, Addresses: addresses, Config: netUsage.Config, ACLNames: aclNames, } } return nil } // GetAddressSetsForACLs return the set of address sets used by given ACLs. func GetAddressSetsForACLs(s *state.State, projectName string, ACLNames []string) ([]string, error) { var projectSetsNames []string err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { sets, err := dbCluster.GetNetworkAddressSets(ctx, tx.Tx(), dbCluster.NetworkAddressSetFilter{Project: &projectName}) if err != nil { return err } for _, set := range sets { projectSetsNames = append(projectSetsNames, set.Name) } return nil }) if err != nil { return nil, fmt.Errorf("Failed loading address set names for project %q: %w", projectName, err) } // For every address set in project check if used by acls given via ACLNames. // If so store it in setsNames slice. var setsNames []string for _, setName := range projectSetsNames { err = AddressSetUsedBy(s, projectName, func(aclName string) error { if slices.Contains(ACLNames, aclName) { setsNames = append(setsNames, setName) } return nil }, setName) if err != nil { return nil, fmt.Errorf("Failed to fetch address set %s use", setName) } } return setsNames, nil } // Redefine ACL usage funcs because we run into circular import otherwise // ACLisInUseByDevice returns any of the supplied matching ACL names found referenced by the NIC device. func ACLisInUseByDevice(d deviceConfig.Device, matchACLNames ...string) []string { matchedACLNames := []string{} // Only NICs linked to managed networks can use network ACLs. if d["type"] != "nic" || d["network"] == "" { return matchedACLNames } for _, nicACLName := range util.SplitNTrimSpace(d["security.acls"], ",", -1, true) { if slices.Contains(matchACLNames, nicACLName) { matchedACLNames = append(matchedACLNames, nicACLName) } } return matchedACLNames } // ACLUsedBy finds all networks, profiles and instance NICs that use any of the specified ACLs and executes usageFunc // once for each resource using one or more of the ACLs with info about the resource and matched ACLs being used. func ACLUsedBy(s *state.State, aclProjectName string, usageFunc func(ctx context.Context, tx *db.ClusterTx, matchedACLNames []string, usageType any, nicName string, nicConfig map[string]string) error, matchACLNames ...string) error { if len(matchACLNames) <= 0 { return nil } var profiles []dbCluster.Profile profileDevices := map[string]map[string]dbCluster.Device{} err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Find networks using the ACLs. Cheapest to do. networkNames, err := tx.GetCreatedNetworkNamesByProject(ctx, aclProjectName) if err != nil && !response.IsNotFoundError(err) { return fmt.Errorf("Failed loading networks for project %q: %w", aclProjectName, err) } for _, networkName := range networkNames { _, network, _, err := tx.GetNetworkInAnyState(ctx, aclProjectName, networkName) if err != nil { return fmt.Errorf("Failed to get network config for %q: %w", networkName, err) } netACLNames := util.SplitNTrimSpace(network.Config["security.acls"], ",", -1, true) matchedACLNames := []string{} for _, netACLName := range netACLNames { if slices.Contains(matchACLNames, netACLName) { matchedACLNames = append(matchedACLNames, netACLName) } } if len(matchedACLNames) > 0 { // Call usageFunc with a list of matched ACLs and info about the network. err := usageFunc(ctx, tx, matchedACLNames, network, "", nil) if err != nil { return err } } } // Look for profiles. Next cheapest to do. profiles, err = dbCluster.GetProfiles(ctx, tx.Tx()) if err != nil { return err } // Get all the profile devices. profileDevicesByID, err := dbCluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return err } for _, profile := range profiles { devices := map[string]dbCluster.Device{} for _, dev := range profileDevicesByID[profile.ID] { devices[dev.Name] = dev } profileDevices[profile.Name] = devices } return nil }) if err != nil { return err } for _, profile := range profiles { // Get the profiles's effective network project name. profileNetworkProjectName, _, err := project.NetworkProject(s.DB.Cluster, profile.Project) if err != nil { return err } // Skip profiles who's effective network project doesn't match this Network ACL's project. if profileNetworkProjectName != aclProjectName { continue } // Iterate through each of the instance's devices, looking for NICs that are using any of the ACLs. for devName, devConfig := range deviceConfig.NewDevices(dbCluster.DevicesToAPI(profileDevices[profile.Name])) { matchedACLNames := ACLisInUseByDevice(devConfig, matchACLNames...) if len(matchedACLNames) > 0 { // Call usageFunc with a list of matched ACLs and info about the instance NIC. err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return usageFunc(ctx, tx, matchedACLNames, profile, devName, devConfig) }) if err != nil { return err } } } } var aclNames []string err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { acls, err := dbCluster.GetNetworkACLs(ctx, tx.Tx(), dbCluster.NetworkACLFilter{Project: &aclProjectName}) if err != nil { return err } aclNames = make([]string, len(acls)) for i, acl := range acls { aclNames[i] = acl.Name } return nil }) if err != nil { return err } err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { for _, aclName := range aclNames { _, aclInfo, err := dbCluster.GetNetworkACLAPI(ctx, tx.Tx(), aclProjectName, aclName) if err != nil { return err } matchedACLNames := []string{} // Ingress rules can specify ACL names in their Source subjects. for _, rule := range aclInfo.Ingress { for _, subject := range util.SplitNTrimSpace(rule.Source, ",", -1, true) { // Look for new matching ACLs, but ignore our own ACL reference in our own rules. if slices.Contains(matchACLNames, subject) && !slices.Contains(matchedACLNames, subject) && subject != aclInfo.Name { matchedACLNames = append(matchedACLNames, subject) } } } // Egress rules can specify ACL names in their Destination subjects. for _, rule := range aclInfo.Egress { for _, subject := range util.SplitNTrimSpace(rule.Destination, ",", -1, true) { // Look for new matching ACLs, but ignore our own ACL reference in our own rules. if slices.Contains(matchACLNames, subject) && !slices.Contains(matchedACLNames, subject) && subject != aclInfo.Name { matchedACLNames = append(matchedACLNames, subject) } } } if len(matchedACLNames) > 0 { // Call usageFunc with a list of matched ACLs and info about the ACL. err = usageFunc(ctx, tx, matchedACLNames, aclInfo, "", nil) if err != nil { return err } } } // Find instances using the ACLs. Most expensive to do. err = tx.InstanceList(ctx, func(inst db.InstanceArgs, p api.Project) error { // Get the instance's effective network project name. instNetworkProject := project.NetworkProjectFromRecord(&p) // Skip instances who's effective network project doesn't match this Network ACL's project. if instNetworkProject != aclProjectName { return nil } devices := db.ExpandInstanceDevices(inst.Devices.Clone(), inst.Profiles) // Iterate through each of the instance's devices, looking for NICs that are using any of the ACLs. for devName, devConfig := range devices { matchedACLNames := ACLisInUseByDevice(devConfig, matchACLNames...) if len(matchedACLNames) > 0 { // Call usageFunc with a list of matched ACLs and info about the instance NIC. err := usageFunc(ctx, tx, matchedACLNames, inst, devName, devConfig) if err != nil { return err } } } return nil }) if err != nil { return err } return nil }) if err != nil { return err } return nil } // ACLNetworkUsage populates the provided aclNets map with networks that are using any of the specified ACLs. func ACLNetworkUsage(s *state.State, aclProjectName string, aclNames []string, aclNets map[string]NetworkACLUsage) error { supportedNetTypes := []string{"bridge", "ovn"} // Find all networks and instance/profile NICs that use any of the specified Network ACLs. err := ACLUsedBy(s, aclProjectName, func(ctx context.Context, tx *db.ClusterTx, matchedACLNames []string, usageType any, devName string, nicConfig map[string]string) error { switch u := usageType.(type) { case dbCluster.Profile: networkID, network, _, err := tx.GetNetworkInAnyState(ctx, aclProjectName, nicConfig["network"]) if err != nil { return fmt.Errorf("Failed to load network %q: %w", nicConfig["network"], err) } if slices.Contains(supportedNetTypes, network.Type) { _, found := aclNets[network.Name] if !found { aclNets[network.Name] = NetworkACLUsage{ ID: networkID, Name: network.Name, Type: network.Type, Config: network.Config, } } } case db.InstanceArgs: networkID, network, _, err := tx.GetNetworkInAnyState(ctx, aclProjectName, nicConfig["network"]) if err != nil { return fmt.Errorf("Failed to load network %q: %w", nicConfig["network"], err) } if slices.Contains(supportedNetTypes, network.Type) { if network.Type == "bridge" && devName != "" { // Use different key for the usage by bridge NICs to avoid overwriting the usage by the bridge network itself. key := fmt.Sprintf("%s/%s/%s", network.Name, u.Name, devName) _, found := aclNets[key] if !found { aclNets[key] = NetworkACLUsage{ ID: networkID, Name: network.Name, Type: network.Type, Config: network.Config, InstanceName: u.Name, DeviceName: devName, } } } else { _, found := aclNets[network.Name] if !found { aclNets[network.Name] = NetworkACLUsage{ ID: networkID, Name: network.Name, Type: network.Type, Config: network.Config, } } } } case *api.Network: if slices.Contains(supportedNetTypes, u.Type) { _, found := aclNets[u.Name] if !found { networkID, network, _, err := tx.GetNetworkInAnyState(ctx, aclProjectName, u.Name) if err != nil { return fmt.Errorf("Failed to load network %q: %w", u.Name, err) } aclNets[u.Name] = NetworkACLUsage{ ID: networkID, Name: network.Name, Type: network.Type, Config: network.Config, } } } case *api.NetworkACL: return nil // Nothing to do for ACL rules referencing us. default: return fmt.Errorf("Unrecognised usage type %T", u) } return nil }, aclNames...) if err != nil { return err } return nil } incus-7.0.0/internal/server/network/address-set/address_set_ovn.go000066400000000000000000000226651517523235500254240ustar00rootroot00000000000000package addressset import ( "context" "errors" "fmt" "net" "strings" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/network/ovn" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" ) // OVNEnsureAddressSetsViaACLs ensure that every address set referenced by given acls are created in OVN NB DB. func OVNEnsureAddressSetsViaACLs(s *state.State, l logger.Logger, client *ovn.NB, projectName string, ACLNames []string) (revert.Hook, error) { // Build address set usage from network ACLs. setsNames, err := GetAddressSetsForACLs(s, projectName, ACLNames) if err != nil { return nil, err } // Then call OVNEnsureAddressSet. return OVNEnsureAddressSets(s, l, client, projectName, setsNames) } // OVNDeleteAddressSetsViaACLs remove address sets used by network ACLS. func OVNDeleteAddressSetsViaACLs(s *state.State, l logger.Logger, client *ovn.NB, projectName string, ACLNames []string) error { setsNames, err := GetAddressSetsForACLs(s, projectName, ACLNames) if err != nil { return err } if len(setsNames) != 0 { for _, setName := range setsNames { addrSet, err := LoadByName(s, projectName, setName) if err != nil { return fmt.Errorf("Failed loading address set %q: %w", setName, err) } err = client.DeleteAddressSet(context.TODO(), ovn.OVNAddressSet(fmt.Sprintf("incus_set%d", addrSet.ID()))) if err != nil { return fmt.Errorf("Failed removing address set %q from OVN: %w", setName, err) } l.Debug("Removed unused address set from OVN", logger.Ctx{"project": projectName, "addressSet": setName}) } } return nil } // OVNEnsureAddressSets ensures that the address sets and their addresses are created in OVN NB DB. // Returns a revert function to undo changes if needed. func OVNEnsureAddressSets(s *state.State, l logger.Logger, client *ovn.NB, projectName string, addressSetNames []string) (revert.Hook, error) { revertion := revert.New() defer revertion.Fail() if len(addressSetNames) == 0 { cleanup := revertion.Clone().Fail revertion.Success() return cleanup, nil } for _, addressSetName := range addressSetNames { addrSet, err := LoadByName(s, projectName, addressSetName) if err != nil { return nil, fmt.Errorf("Failed loading address set %q: %w", addressSetName, err) } asInfo := addrSet.Info() // Convert addresses into net.IPNet slices. var ipNets []net.IPNet for _, addr := range asInfo.Addresses { // Try to parse as IP or CIDR. if strings.Contains(addr, "/") { _, ipnet, err := net.ParseCIDR(addr) if err != nil { return nil, fmt.Errorf("Failed parsing CIDR address %q: %w", addr, err) } ipNets = append(ipNets, *ipnet) } else { // If single IP address, convert to /32 or /128. ip := net.ParseIP(addr) if ip == nil { // If MAC, skip IP sets. OVN address sets currently store MAC addresses but we don't use them. // If needed, you'd have separate sets for MAC. _, errMac := net.ParseMAC(addr) if errMac == nil { // We currently ignore MAC addresses. OVN address sets are primarily for IP addresses. // If future support for MAC sets needed, handle here. continue } return nil, fmt.Errorf("Unsupported address format: %q", addr) } bits := 32 if ip.To4() == nil { bits = 128 } mask := net.CIDRMask(bits, bits) ipNets = append(ipNets, net.IPNet{IP: ip, Mask: mask}) } } // Check if the address set exists in OVN. existingIPv4Set, existingIPv6Set, err := client.GetAddressSet(context.TODO(), ovn.OVNAddressSet(fmt.Sprintf("incus_set%d", addrSet.ID()))) // If address sets do not exist, create them. if errors.Is(err, ovn.ErrNotFound) { err = client.CreateAddressSet(context.TODO(), ovn.OVNAddressSet(fmt.Sprintf("incus_set%d", addrSet.ID())), ipNets...) ipNetStrings := make([]string, len(ipNets)) if err != nil { for i, ipNet := range ipNets { ipNetStrings[i] = ipNet.String() } return nil, fmt.Errorf("Failed creating address set %q with networks %s in OVN: %w", asInfo.Name, strings.Join(ipNetStrings, "-"), err) } revertion.Add(func() { _ = client.DeleteAddressSet(context.TODO(), ovn.OVNAddressSet(fmt.Sprintf("incus_set%d", addrSet.ID()))) }) } else { if err != nil && !errors.Is(err, ovn.ErrNotFound) { return nil, fmt.Errorf("Failed fetching address set %q (IPv4) from OVN: %w", asInfo.Name, err) } if err != nil && !errors.Is(err, ovn.ErrNotFound) { return nil, fmt.Errorf("Failed fetching address set %q (IPv6) from OVN: %w", asInfo.Name, err) } // Compute differences. existingIPv4Map := make(map[string]bool) existingIPv6Map := make(map[string]bool) for _, addr := range existingIPv4Set.Addresses { existingIPv4Map[addr] = true } for _, addr := range existingIPv6Set.Addresses { existingIPv6Map[addr] = true } var addIPv4, removeIPv4, addIPv6, removeIPv6 []net.IPNet for _, newIP := range ipNets { if newIP.IP.To4() != nil { if !existingIPv4Map[newIP.String()] { addIPv4 = append(addIPv4, newIP) } } else { if !existingIPv6Map[newIP.String()] { addIPv6 = append(addIPv6, newIP) } } } for existingIP := range existingIPv4Map { found := false for _, newIP := range ipNets { if newIP.String() == existingIP { found = true break } } if !found { // OVN always register CIDR in address set. _, network, err := net.ParseCIDR(existingIP) if err != nil { return nil, fmt.Errorf("Failed parsing existing IP in set %s err: %w", existingIP, err) } removeIPv4 = append(removeIPv4, *network) } } for existingIP := range existingIPv6Map { found := false for _, newIP := range ipNets { if newIP.String() == existingIP { found = true break } } if !found { _, network, err := net.ParseCIDR(existingIP) if err != nil { return nil, fmt.Errorf("Failed parsing existing IP in set %s err: %w", existingIP, err) } removeIPv6 = append(removeIPv6, *network) } } // Update OVN sets. if len(addIPv4) > 0 || len(addIPv6) > 0 { err = client.UpdateAddressSetAdd(context.TODO(), ovn.OVNAddressSet(fmt.Sprintf("incus_set%d", addrSet.ID())), append(addIPv4, addIPv6...)...) if err != nil { return nil, fmt.Errorf("Failed adding addresses to address set %q in OVN: %w", asInfo.Name, err) } } if len(removeIPv4) > 0 || len(removeIPv6) > 0 { err = client.UpdateAddressSetRemove(context.TODO(), ovn.OVNAddressSet(fmt.Sprintf("incus_set%d", addrSet.ID())), append(removeIPv4, removeIPv6...)...) if err != nil { return nil, fmt.Errorf("Failed removing addresses from address set %q in OVN: %w", asInfo.Name, err) } } } } cleanup := revertion.Clone().Fail revertion.Success() return cleanup, nil } // OVNAddressSetDeleteIfUnused checks if the specified address set is unused and if so, removes it from OVN. func OVNAddressSetDeleteIfUnused(s *state.State, l logger.Logger, client *ovn.NB, projectName string, setName string) error { addrSet, err := LoadByName(s, projectName, setName) if err != nil { // If not found, it's either already deleted or doesn't exist, so nothing to do. return nil } // Get a list of networks that indirectly reference this address set via ACLs. asNets := map[string]AddressSetUsage{} err = AddressSetNetworkUsage(s, projectName, setName, addrSet.Info().Addresses, asNets) if err != nil { return fmt.Errorf("Failed getting address set network usage: %w", err) } // Separate out OVN networks from non-OVN networks for different handling. asOVNNets := map[string]AddressSetUsage{} for k, v := range asNets { if v.Type == "ovn" { delete(asNets, k) asOVNNets[k] = v } } if len(asOVNNets) > 0 { l.Debug("Address set still in use, skipping removal", logger.Ctx{"project": projectName, "addressSet": setName, "usedByCount": len(asOVNNets)}) return nil } // Address set is unused by OVN, remove from OVN. err = client.DeleteAddressSet(context.TODO(), ovn.OVNAddressSet(fmt.Sprintf("incus_set%d", addrSet.ID()))) if err != nil { return fmt.Errorf("Failed removing address set %q from OVN: %w", setName, err) } l.Debug("Removed unused address set from OVN", logger.Ctx{"project": projectName, "addressSet": setName}) return nil } // OVNAddressSetsDeleteIfUnused remove all address sets in OVN that are not used. func OVNAddressSetsDeleteIfUnused(s *state.State, l logger.Logger, client *ovn.NB, projectName string) error { var Sets []dbCluster.NetworkAddressSet l.Debug("Removing remaining sets ...") err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error Sets, err = dbCluster.GetNetworkAddressSets(ctx, tx.Tx(), dbCluster.NetworkAddressSetFilter{Project: &projectName}) if err != nil { return err } return nil }) if err != nil { return fmt.Errorf("Failed loading address set names for project %q: %w", projectName, err) } for _, set := range Sets { _, _, err := client.GetAddressSet(context.TODO(), ovn.OVNAddressSet(fmt.Sprintf("incus_set%d", set.ID))) // If address sets do not exist, continue. if errors.Is(err, ovn.ErrNotFound) { continue } l.Debug("Trying to remove: ", logger.Ctx{"set": set}) err = OVNAddressSetDeleteIfUnused(s, l, client, projectName, set.Name) if err != nil { return err } } return nil } incus-7.0.0/internal/server/network/address-set/address_set_validation.go000066400000000000000000000010531517523235500267400ustar00rootroot00000000000000package addressset import ( "fmt" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // ValidName checks the address set name is valid. func ValidName(name string) error { err := validate.IsAPIName(name, false) if err != nil { return err } // Don't allow address set names to start with dollar or arobase. if util.StringHasPrefix(name, "@", "$") { return fmt.Errorf("Name cannot start with reserved character %q", name[0]) } err = validate.IsHostname(name) if err != nil { return err } return nil } incus-7.0.0/internal/server/network/address-set/driver_common.go000066400000000000000000000250611517523235500250760ustar00rootroot00000000000000package addressset import ( "context" "errors" "fmt" "net" "slices" "strings" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" ) // common represents a network address set. type common struct { logger logger.Logger state *state.State id int projectName string info *api.NetworkAddressSet } // init initialize internal variables. func (d *common) init(s *state.State, id int, projectName string, info *api.NetworkAddressSet) { if info == nil { d.info = &api.NetworkAddressSet{} } else { d.info = info } d.logger = logger.AddContext(logger.Ctx{"project": projectName, "networkAddressSet": d.info.Name}) d.id = id d.projectName = projectName d.state = s if d.info.Addresses == nil { d.info.Addresses = []string{} } if d.info.Config == nil { d.info.Config = make(map[string]string) } } // ID returns the network address set ID. func (d *common) ID() int { return d.id } // Project returns the project name. func (d *common) Project() string { return d.projectName } // Info returns copy of internal info for the Network AddressSet. func (d *common) Info() *api.NetworkAddressSet { // Copy internal info to prevent modification externally. info := api.NetworkAddressSet{} info.Name = d.info.Name info.Description = d.info.Description info.Addresses = slices.Clone(d.info.Addresses) info.Config = localUtil.CopyConfig(d.info.Config) info.UsedBy = nil // To indicate its not populated (use Usedby() function to populate). info.Project = d.projectName return &info } // usedBy returns a list of ACLs API endpoints referencing this address set. // If firstOnly is true then search stops at first result. func (d *common) usedBy(firstOnly bool) ([]string, error) { usedBy := []string{} // Find all ACLs that reference this address set. err := AddressSetUsedBy(d.state, d.projectName, func(aclName string) error { uri := fmt.Sprintf("/%s/network-acls/%s", version.APIVersion, aclName) if d.projectName != api.ProjectDefaultName { uri += fmt.Sprintf("?project=%s", d.projectName) } usedBy = append(usedBy, uri) if firstOnly { return db.ErrInstanceListStop } return nil }, d.info.Name) if err != nil { if errors.Is(err, db.ErrInstanceListStop) { return usedBy, nil } return nil, fmt.Errorf("Failed getting address set usage: %w", err) } return usedBy, nil } // UsedBy returns a list of ACL API endpoints referencing this address set. func (d *common) UsedBy() ([]string, error) { return d.usedBy(false) } // Etag returns the values used for etag generation. func (d *common) Etag() []any { return []any{d.info.Name, d.info.Description, d.info.Addresses, d.info.Config} } // validateName checks name is valid. func (d *common) validateName(name string) error { return ValidName(name) } // validateAddresses ensure set is valid. func (d *common) validateAddresses(addresses []string) error { seen := make(map[string]struct{}) for i, addr := range addresses { _, exists := seen[addr] if exists { return fmt.Errorf("Duplicate address %q found at index %d", addr, i) } seen[addr] = struct{}{} // Check if it's a valid plain IP address. if net.ParseIP(addr) != nil { continue } // Check if it's a valid CIDR. _, _, err := net.ParseCIDR(addr) if err == nil { continue } // Check if it's a valid MAC address. _, err = net.ParseMAC(addr) if err == nil { return fmt.Errorf("Unsupported MAC address format %q at index %d", addr, i) } return fmt.Errorf("Unsupported address format %q at index %d", addr, i) } return nil } // validateConfig checks the entire config including name and addresses. func (d *common) validateConfig(config *api.NetworkAddressSetPut) error { // Validate the address list. err := d.validateAddresses(config.Addresses) if err != nil { return fmt.Errorf("Invalid addresses: %w", err) } // Validate the configuration. configKeys := map[string]func(value string) error{} for k, v := range config.Config { // User keys are free for all. // gendoc:generate(entity=network_address_set, group=common, key=user.*) // User keys can be used in search. // --- // type: string // shortdesc: Free form user key/value storage if strings.HasPrefix(k, "user.") { continue } validator, ok := configKeys[k] if !ok { return fmt.Errorf("Invalid network integration configuration key %q", k) } err := validator(v) if err != nil { return fmt.Errorf("Invalid network integration configuration key %q value", k) } } return nil } // Update method is used to update an address set and apply to concerned networks. func (d *common) Update(config *api.NetworkAddressSetPut, clientType request.ClientType) error { reverter := revert.New() defer reverter.Fail() // Validate the new configuration. err := d.validateConfig(config) if err != nil { return err } if clientType == request.ClientTypeNormal { var dbRecord *dbCluster.NetworkAddressSet oldConfig := d.info.NetworkAddressSetPut err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get current record. dbRecord, err = dbCluster.GetNetworkAddressSet(ctx, tx.Tx(), d.projectName, d.info.Name) if err != nil { return err } // Update database. Its important this occurs before we attempt to apply to networks. dbRecord.Addresses = config.Addresses dbRecord.Description = config.Description err = dbCluster.UpdateNetworkAddressSet(ctx, tx.Tx(), d.projectName, d.info.Name, *dbRecord) if err != nil { return err } // Update the configuration. err = dbCluster.UpdateNetworkAddressSetConfig(ctx, tx.Tx(), int64(dbRecord.ID), config.Config) if err != nil { return err } return nil }) if err != nil { return err } // Apply changes internally and reinitialize. d.info.NetworkAddressSetPut = *config d.init(d.state, d.id, d.projectName, d.info) reverter.Add(func() { _ = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Update database. Its important this occurs before we attempt to apply to networks. dbRecord.Addresses = oldConfig.Addresses dbRecord.Description = oldConfig.Description err = dbCluster.UpdateNetworkAddressSet(ctx, tx.Tx(), d.projectName, d.info.Name, *dbRecord) if err != nil { return err } // Update the configuration. err = dbCluster.UpdateNetworkAddressSetConfig(ctx, tx.Tx(), int64(dbRecord.ID), oldConfig.Config) if err != nil { return err } return nil }) d.info.NetworkAddressSetPut = oldConfig d.init(d.state, d.id, d.projectName, d.info) }) } // Get a list of networks that indirectly reference this address set via ACLs. asNets := map[string]AddressSetUsage{} err = AddressSetNetworkUsage(d.state, d.projectName, d.info.Name, d.info.Addresses, asNets) if err != nil { return fmt.Errorf("Failed getting address set network usage: %w", err) } // Separate out OVN networks from non-OVN networks for different handling. asOVNNets := map[string]AddressSetUsage{} for k, v := range asNets { if v.Type == "ovn" { delete(asNets, k) asOVNNets[k] = v } else if v.Type != "bridge" { return fmt.Errorf("Unsupported network type %q using address set %q", v.Type, d.info.Name) } } // Apply address set changes to non-OVN networks on this member. if len(asNets) > 0 { for _, asNet := range asNets { if asNet.DeviceName != "" { err = FirewallApplyAddressSetsForACLRules(d.state, "bridge", d.projectName, asNet.ACLNames) if err != nil { return err } } else { err = FirewallApplyAddressSets(d.state, d.projectName, asNet) if err != nil { return err } } } } // If there are affected OVN networks, then apply changes if request type is normal. if len(asOVNNets) > 0 && clientType == request.ClientTypeNormal { // Check that OVN is available. ovnnb, _, err := d.state.OVN() if err != nil { return err } // Ensure address sets are created or updated in OVN. cleanup, err := OVNEnsureAddressSets(d.state, d.logger, ovnnb, d.projectName, []string{d.info.Name}) if err != nil { return fmt.Errorf("Failed ensuring address set %q is configured in OVN: %w", d.info.Name, err) } reverter.Add(cleanup) } // If normal request and asNets is not empty, notify other cluster members. if clientType == request.ClientTypeNormal && len(asNets) > 0 { notifier, err := cluster.NewNotifier(d.state, d.state.Endpoints.NetworkCert(), d.state.ServerCert(), cluster.NotifyAll) if err != nil { return err } err = notifier(func(client incus.InstanceServer) error { // Make sure we have a suitable endpoint for updating the address set on other members. return client.UseProject(d.projectName).UpdateNetworkAddressSet(d.info.Name, d.info.NetworkAddressSetPut, "") }) if err != nil { return err } } reverter.Success() return nil } // Rename is used to rename an address set. func (d *common) Rename(newName string) error { err := d.validateName(newName) if err != nil { return err } // Check if name already exists. _, err = LoadByName(d.state, d.projectName, newName) if err == nil { return fmt.Errorf("Address set by that name: %s exists already", newName) } usedBy, err := d.UsedBy() if err != nil { return err } if len(usedBy) > 0 { return errors.New("Cannot rename address set that is in use") } err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return dbCluster.RenameNetworkAddressSet(ctx, tx.Tx(), d.projectName, d.info.Name, newName) }) if err != nil { return err } d.info.Name = newName return nil } // Delete is used to delete an address set. func (d *common) Delete() error { usedBy, err := d.UsedBy() if err != nil { return err } if len(usedBy) > 0 { return errors.New("Cannot delete address set that is in use") } err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return dbCluster.DeleteNetworkAddressSet(ctx, tx.Tx(), d.projectName, d.info.Name) }) if err != nil { return errors.New("Error while deleting address set from db") } return nil } incus-7.0.0/internal/server/network/driver_bridge.go000066400000000000000000003166701517523235500226350ustar00rootroot00000000000000package network import ( "context" "database/sql" "errors" "fmt" "io/fs" "maps" "net" "net/http" "os" "os/exec" "slices" "strconv" "strings" "time" "github.com/mdlayher/netx/eui64" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/server/apparmor" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/daemon" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/warningtype" "github.com/lxc/incus/v7/internal/server/dnsmasq" "github.com/lxc/incus/v7/internal/server/dnsmasq/dhcpalloc" firewallDrivers "github.com/lxc/incus/v7/internal/server/firewall/drivers" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/network/acl" addressset "github.com/lxc/incus/v7/internal/server/network/address-set" "github.com/lxc/incus/v7/internal/server/project" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/server/warnings" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // Default MTU for bridge interface. const bridgeMTUDefault = 1500 // bridge represents a bridge network. type bridge struct { common } // DBType returns the network type DB ID. func (n *bridge) DBType() db.NetworkType { return db.NetworkTypeBridge } // Config returns the network driver info. func (n *bridge) Info() Info { info := n.common.Info() info.AddressForwards = true return info } // checkClusterWideMACSafe returns whether it is safe to use the same MAC address for the bridge interface on all // cluster nodes. It is not suitable to use a static MAC address when "bridge.external_interfaces" is non-empty and // the bridge interface has no IPv4 or IPv6 address set. This is because in a clustered environment the same bridge // config is applied to all nodes, and if the bridge is being used to connect multiple nodes to the same network // segment it would cause MAC conflicts to use the same MAC on all nodes. If an IP address is specified then // connecting multiple nodes to the same network segment would also cause IP conflicts, so if an IP is defined // then we assume this is not being done. However if IP addresses are explicitly set to "none" and // "bridge.external_interfaces" is set then it may not be safe to use a the same MAC address on all nodes. func (n *bridge) checkClusterWideMACSafe(config map[string]string) error { // We can't be sure that multiple clustered nodes aren't connected to the same network segment so don't // use a static MAC address for the bridge interface to avoid introducing a MAC conflict. if config["bridge.external_interfaces"] != "" && config["ipv4.address"] == "none" && config["ipv6.address"] == "none" { return errors.New(`Cannot use static "bridge.hwaddr" MAC address when bridge has no IP addresses and has external interfaces set`) } // We may have MAC conflicts if tunnels are in use. for k := range config { if strings.HasPrefix(k, "tunnel.") { return errors.New(`Cannot use static "bridge.hwaddr" MAC address when bridge has tunnels connected`) } } // If using a generated IPv6 address, we need a unique MAC. if config["ipv6.address"] != "none" && validate.IsNetworkV6(config["ipv6.address"]) == nil { return errors.New(`Cannot use static "bridge.hwaddr" MAC address when bridge uses a host-specific IPv6 address`) } return nil } // FillConfig fills requested config with any default values. func (n *bridge) FillConfig(config map[string]string) error { // Set some default values where needed. if config["ipv4.address"] == "" { config["ipv4.address"] = "auto" } if config["ipv4.address"] == "auto" && config["ipv4.nat"] == "" { config["ipv4.nat"] = "true" } if config["ipv6.address"] == "" { content, err := os.ReadFile("/proc/sys/net/ipv6/conf/default/disable_ipv6") if err == nil && string(content) == "0\n" { config["ipv6.address"] = "auto" } } if config["ipv6.address"] == "auto" && config["ipv6.nat"] == "" { config["ipv6.nat"] = "true" } // Now replace any "auto" keys with generated values. err := n.populateAutoConfig(config) if err != nil { return fmt.Errorf("Failed generating auto config: %w", err) } return nil } // populateAutoConfig replaces "auto" in config with generated values. func (n *bridge) populateAutoConfig(config map[string]string) error { changedConfig := false // Now populate "auto" values where needed. if config["ipv4.address"] == "auto" { subnet, err := randomSubnetV4() if err != nil { return err } config["ipv4.address"] = subnet changedConfig = true } if config["ipv6.address"] == "auto" { subnet, err := randomSubnetV6() if err != nil { return err } config["ipv6.address"] = subnet changedConfig = true } // Re-validate config if changed. if changedConfig && n.state != nil { return n.Validate(config, request.ClientTypeNormal) } return nil } // ValidateName validates network name. func (n *bridge) ValidateName(name string) error { err := validate.IsInterfaceName(name) if err != nil { return err } // Apply common name validation that applies to all network types. return n.common.ValidateName(name) } // Validate network config. func (n *bridge) Validate(config map[string]string, clientType request.ClientType) error { // Build driver specific rules dynamically. rules := map[string]func(value string) error{ // gendoc:generate(entity=network_bridge, group=common, key=bgp.ipv4.nexthop) // // --- // type: string // condition: BGP server // default: local address // shortdesc: Override the next-hop for advertised prefixes "bgp.ipv4.nexthop": validate.Optional(validate.IsNetworkAddressV4), // gendoc:generate(entity=network_bridge, group=common, key=bgp.ipv6.nexthop) // // --- // type: string // condition: BGP server // default: local address // shortdesc: Override the next-hop for advertised prefixes "bgp.ipv6.nexthop": validate.Optional(validate.IsNetworkAddressV6), // gendoc:generate(entity=network_bridge, group=common, key=bridge.driver) // // --- // type: string // condition: - // default: `native` // shortdesc: Bridge driver: `native` or `openvswitch` "bridge.driver": validate.Optional(validate.IsOneOf("native", "openvswitch")), // gendoc:generate(entity=network_bridge, group=common, key=bridge.external_interfaces) // // --- // type: string // condition: - // default: - // shortdesc: Comma-separated list of unconfigured network interfaces to include in the bridge "bridge.external_interfaces": validate.Optional(validateExternalInterfaces), // gendoc:generate(entity=network_bridge, group=common, key=bridge.hwaddr) // // --- // type: string // condition: - // default: - // shortdesc: MAC address for the bridge "bridge.hwaddr": validate.Optional(validate.IsNetworkMAC), // gendoc:generate(entity=network_bridge, group=common, key=bridge.mtu) // // --- // type: integer // condition: - // default: `1500` // shortdesc: Bridge MTU (default varies if tunnel in use) "bridge.mtu": validate.Optional(validate.IsNetworkMTU), // gendoc:generate(entity=network_bridge, group=common, key=ipv4.address) // // --- // type: string // condition: standard mode // default: - (initial value on creation: `auto`) // shortdesc: IPv4 address for the bridge (use `none` to turn off IPv4 or `auto` to generate a new random unused subnet) (CIDR) "ipv4.address": validate.Optional(func(value string) error { if validate.IsOneOf("none", "auto")(value) == nil { return nil } return validate.IsNetworkAddressCIDRV4(value) }), // gendoc:generate(entity=network_bridge, group=common, key=ipv4.firewall) // // --- // type: bool // condition: IPv4 address // default: `true` // shortdesc: Whether to generate filtering firewall rules for this network "ipv4.firewall": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_bridge, group=common, key=ipv4.nat) // // --- // type: bool // condition: IPv4 address // default: `false`(initial value on creation if `ipv4.address` is set to `auto`: `true`) // shortdesc: Whether to NAT "ipv4.nat": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_bridge, group=common, key=ipv4.nat.order) // // --- // type: string // condition: IPv4 address // default: `before` // shortdesc: Whether to add the required NAT rules before or after any pre-existing rules "ipv4.nat.order": validate.Optional(validate.IsOneOf("before", "after")), // gendoc:generate(entity=network_bridge, group=common, key=ipv4.nat.address) // // --- // type: string // condition: IPv4 address // default: - // shortdesc: The source address used for outbound traffic from the bridge "ipv4.nat.address": validate.Optional(validate.IsNetworkAddressV4), // gendoc:generate(entity=network_bridge, group=common, key=ipv4.dhcp) // // --- // type: bool // condition: IPv4 address // default: `true` // shortdesc: Whether to allocate addresses using DHCP "ipv4.dhcp": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_bridge, group=common, key=ipv4.dhcp.gateway) // // --- // type: string // condition: IPv4 DHCP // default: IPv4 address // shortdesc: Address of the gateway for the subnet (use `none` to turn off gateway announcement) "ipv4.dhcp.gateway": validate.Optional(func(value string) error { if value == "none" { return nil } return validate.IsNetworkAddressV4(value) }), // gendoc:generate(entity=network_bridge, group=common, key=ipv4.dhcp.expiry) // // --- // type: string // condition: IPv4 DHCP // default: `1h` // shortdesc: When to expire DHCP leases "ipv4.dhcp.expiry": validate.IsAny, // gendoc:generate(entity=network_bridge, group=common, key=ipv4.dhcp.ranges) // // --- // type: string // condition: IPv4 DHCP // default: all addresses // shortdesc: Comma-separated list of IP ranges to use for DHCP (FIRST-LAST format) "ipv4.dhcp.ranges": validate.Optional(validate.IsListOf(validate.IsNetworkRangeV4)), // gendoc:generate(entity=network_bridge, group=common, key=ipv4.dhcp.routes) // // --- // type: string // condition: IPv4 DHCP // default: - // shortdesc: Static routes to provide via DHCP option 121, as a comma-separated list of alternating subnets (CIDR) and gateway addresses (same syntax as dnsmasq) "ipv4.dhcp.routes": validate.Optional(validate.IsDHCPRouteList), // gendoc:generate(entity=network_bridge, group=common, key=ipv4.routes) // // --- // type: string // condition: IPv4 address // default: - // shortdesc: Comma-separated list of additional IPv4 CIDR subnets to route to the bridge "ipv4.routes": validate.Optional(validate.IsListOf(validate.IsNetworkV4)), // gendoc:generate(entity=network_bridge, group=common, key=ipv4.routing) // // --- // type: bool // condition: IPv4 DHCP // default: `true` // shortdesc: Whether to route traffic in and out of the bridge "ipv4.routing": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_bridge, group=common, key=ipv4.ovn.ranges) // // --- // type: string // condition: - // default: - // shortdesc: Comma-separated list of IPv4 ranges to use for child OVN network routers (FIRST-LAST format) "ipv4.ovn.ranges": validate.Optional(validate.IsListOf(validate.IsNetworkRangeV4)), // gendoc:generate(entity=network_bridge, group=common, key=ipv6.address) // // --- // type: string // condition: standard mode // default: - (initial value on creation: `auto`) // shortdesc: IPv6 address for the bridge (use `none` to turn off IPv6 or `auto` to generate a new random unused subnet) (CIDR) "ipv6.address": validate.Optional(func(value string) error { if validate.IsOneOf("none", "auto")(value) == nil { return nil } return validate.Or(validate.IsNetworkAddressCIDRV6, validate.IsNetworkV6)(value) }), // gendoc:generate(entity=network_bridge, group=common, key=ipv6.firewall) // // --- // type: bool // condition: IPv6 address // default: `true` // shortdesc: Whether to generate filtering firewall rules for this network "ipv6.firewall": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_bridge, group=common, key=ipv6.nat) // // --- // type: bool // condition: IPv6 address // default: `false` (initial value on creation if `ipv6.address` is set to `auto`: `true`) // shortdesc: Whether to NAT "ipv6.nat": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_bridge, group=common, key=ipv6.nat.order) // // --- // type: string // condition: IPv6 address // default: `before` // shortdesc: Whether to add the required NAT rules before or after any pre-existing rules "ipv6.nat.order": validate.Optional(validate.IsOneOf("before", "after")), // gendoc:generate(entity=network_bridge, group=common, key=ipv6.nat.address) // // --- // type: string // condition: IPv6 address // default: - // shortdesc: The source address used for outbound traffic from the bridge "ipv6.nat.address": validate.Optional(validate.IsNetworkAddressV6), // gendoc:generate(entity=network_bridge, group=common, key=ipv6.dhcp) // // --- // type: bool // condition: IPv6 DHCP // default: `true` // shortdesc: Whether to provide additional network configuration over DHCP "ipv6.dhcp": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_bridge, group=common, key=ipv6.dhcp.expiry) // // --- // type: string // condition: IPv6 DHCP // default: `1h` // shortdesc: When to expire DHCP leases "ipv6.dhcp.expiry": validate.IsAny, // gendoc:generate(entity=network_bridge, group=common, key=ipv6.dhcp.stateful) // // --- // type: bool // condition: IPv6 DHCP // default: `false` // shortdesc: Whether to allocate addresses using DHCP "ipv6.dhcp.stateful": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_bridge, group=common, key=ipv6.dhcp.ranges) // // --- // type: string // condition: IPv6 stateful DHCP // default: all addresses // shortdesc: Comma-separated list of IPv6 ranges to use for DHCP (FIRST-LAST format) "ipv6.dhcp.ranges": validate.Optional(validate.IsListOf(validate.IsNetworkRangeV6)), // gendoc:generate(entity=network_bridge, group=common, key=ipv6.routes) // // --- // type: string // condition: IPv6 address // default: - // shortdesc: Comma-separated list of additional IPv6 CIDR subnets to route to the bridge "ipv6.routes": validate.Optional(validate.IsListOf(validate.IsNetworkV6)), // gendoc:generate(entity=network_bridge, group=common, key=ipv6.routing) // // --- // type: bool // condition: IPv6 address // default: `true` // shortdesc: Whether to route traffic in and out of the bridge "ipv6.routing": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_bridge, group=common, key=ipv6.ovn.ranges) // // --- // type: string // condition: - // default: - // shortdesc: Comma-separated list of IPv6 ranges to use for child OVN network routers (FIRST-LAST format) "ipv6.ovn.ranges": validate.Optional(validate.IsListOf(validate.IsNetworkRangeV6)), // gendoc:generate(entity=network_bridge, group=common, key=dns.nameservers) // // --- // type: string // condition: - // default: IPv4 and IPv6 address // shortdesc: DNS server IPs to advertise to DHCP clients and via Router Advertisements. Both IPv4 and IPv6 addresses get pushed via DHCP, and IPv6 addresses are also advertised as RDNSS via RA. "dns.nameservers": validate.Optional(validate.IsListOf(validate.IsNetworkAddress)), // gendoc:generate(entity=network_bridge, group=common, key=dns.domain) // // --- // type: string // condition: - // default: `incus` // shortdesc: Domain to advertise to DHCP clients and use for DNS resolution "dns.domain": validate.IsAny, // gendoc:generate(entity=network_bridge, group=common, key=dns.mode) // // --- // type: string // condition: - // default: `managed` // shortdesc: DNS registration mode: none for no DNS record, managed for Incus-generated static records or dynamic for client-generated records "dns.mode": validate.Optional(validate.IsOneOf("dynamic", "managed", "none")), // gendoc:generate(entity=network_bridge, group=common, key=dns.search) // // --- // type: string // condition: - // default: - // shortdesc: Full comma-separated domain search list, defaulting to `dns.domain` value "dns.search": validate.IsAny, // gendoc:generate(entity=network_bridge, group=common, key=dns.zone.forward) // // --- // type: string // condition: - // default: `managed` // shortdesc: Comma-separated list of DNS zone names for forward DNS records "dns.zone.forward": validate.IsAny, // gendoc:generate(entity=network_bridge, group=common, key=dns.zone.reverse.ipv4) // // --- // type: string // condition: - // default: `managed` // shortdesc: DNS zone name for IPv4 reverse DNS records "dns.zone.reverse.ipv4": validate.IsAny, // gendoc:generate(entity=network_bridge, group=common, key=dns.zone.reverse.ipv6) // // --- // type: string // condition: - // default: `managed` // shortdesc: DNS zone name for IPv6 reverse DNS records "dns.zone.reverse.ipv6": validate.IsAny, // gendoc:generate(entity=network_bridge, group=common, key=raw.dnsmasq) // // --- // type: string // condition: - // default: - // shortdesc: Additional dnsmasq configuration to append to the configuration file "raw.dnsmasq": validate.IsAny, // gendoc:generate(entity=network_bridge, group=common, key=security.acls) // // --- // type: string // condition: - // default: - // shortdesc: Comma-separated list of Network ACLs to apply to NICs connected to this network (see {ref}`network-acls-bridge-limitations`) "security.acls": validate.IsAny, // gendoc:generate(entity=network_bridge, group=common, key=security.acls.default.ingress.action) // // --- // type: string // condition: `security.acls` // default: `reject` // shortdesc: Action to use for ingress traffic that doesn't match any ACL rule "security.acls.default.ingress.action": validate.Optional(validate.IsOneOf(acl.ValidActions...)), // gendoc:generate(entity=network_bridge, group=common, key=security.acls.default.egress.action) // // --- // type: string // condition: `security.acls` // default: `reject` // shortdesc: Action to use for egress traffic that doesn't match any ACL rule "security.acls.default.egress.action": validate.Optional(validate.IsOneOf(acl.ValidActions...)), // gendoc:generate(entity=network_bridge, group=common, key=security.acls.default.ingress.logged) // // --- // type: bool // condition: `security.acls` // default: `false` // shortdesc: Whether to log ingress traffic that doesn't match any ACL rule "security.acls.default.ingress.logged": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_bridge, group=common, key=security.acls.default.egress.logged) // // --- // type: bool // condition: `security.acls` // default: `false` // shortdesc: Whether to log egress traffic that doesn't match any ACL rule "security.acls.default.egress.logged": validate.Optional(validate.IsBool), } // Add dynamic validation rules. for k := range config { // Tunnel keys have the remote name in their name, extract the suffix. if strings.HasPrefix(k, "tunnel.") { // Validate remote name in key. fields := strings.Split(k, ".") if len(fields) != 3 { return fmt.Errorf("Invalid network configuration key: %s", k) } if len(n.name)+len(fields[1]) > 14 { return fmt.Errorf("Network name too long for tunnel interface: %s-%s", n.name, fields[1]) } tunnelKey := fields[2] // Add the correct validation rule for the dynamic field based on last part of key. switch tunnelKey { case "protocol": // gendoc:generate(entity=network_bridge, group=common, key=tunnel.NAME.protocol) // // --- // type: string // condition: standard mode // default: - // shortdesc: Tunneling protocol: `vxlan` or `gre` rules[k] = validate.Optional(validate.IsOneOf("gre", "vxlan")) case "local": // gendoc:generate(entity=network_bridge, group=common, key=tunnel.NAME.local) // // --- // type: string // condition: `gre` or `vxlan` // default: - // shortdesc: Local address for the tunnel (not necessary for multicast `vxlan`) rules[k] = validate.Optional(validate.IsNetworkAddress) case "remote": // gendoc:generate(entity=network_bridge, group=common, key=tunnel.NAME.remote) // // --- // type: string // condition: `gre` or `vxlan` // default: - // shortdesc: Remote address for the tunnel (not necessary for multicast `vxlan`) rules[k] = validate.Optional(validate.IsNetworkAddress) case "port": // gendoc:generate(entity=network_bridge, group=common, key=tunnel.NAME.port) // // --- // type: integer // condition: `vxlan` // default: `0` // shortdesc: Specific port to use for the `vxlan` tunnel rules[k] = networkValidPort case "group": // gendoc:generate(entity=network_bridge, group=common, key=tunnel.NAME.group) // // --- // type: string // condition: `vxlan` // default: `239.0.0.1` // shortdesc: Multicast address for `vxlan` (used if local and remote aren't set) rules[k] = validate.Optional(validate.IsNetworkAddress) case "id": // gendoc:generate(entity=network_bridge, group=common, key=tunnel.NAME.id) // // --- // type: integer // condition: `vxlan` // default: `0` // shortdesc: Specific tunnel ID to use for the `vxlan` tunnel rules[k] = validate.Optional(validate.IsInt64) case "interface": // gendoc:generate(entity=network_bridge, group=common, key=tunnel.NAME.interface) // // --- // type: string // condition: `vxlan` // default: - // shortdesc: Specific host interface to use for the tunnel rules[k] = validate.IsInterfaceName case "ttl": // gendoc:generate(entity=network_bridge, group=common, key=tunnel.NAME.ttl) // // --- // type: integer // condition: `vxlan` // default: `1` // shortdesc: Specific TTL to use for multicast routing topologies rules[k] = validate.Optional(validate.IsUint8) } } } // gendoc:generate(entity=network_bridge, group=bgp, key=bgp.peers.NAME.address) // // --- // type: string // condition: BGP server // defaultdesc: - // shortdesc: Peer address (IPv4 or IPv6) for use by `ovn` downstream networks // gendoc:generate(entity=network_bridge, group=bgp, key=bgp.peers.NAME.asn) // // --- // type: integer // condition: BGP server // defaultdesc: - // shortdesc: Peer AS number for use by `ovn` downstream networks // gendoc:generate(entity=network_bridge, group=bgp, key=bgp.peers.NAME.password) // // --- // type: string // condition: BGP server // defaultdesc: - (no password) // shortdesc: Peer session password (optional) for use by `ovn` downstream networks // gendoc:generate(entity=network_bridge, group=bgp, key=bgp.peers.NAME.holdtime) // // --- // type: integer // condition: BGP server // defaultdesc: `180` // shortdesc: Peer session hold time (in seconds; optional) // Add the BGP validation rules. bgpRules, err := n.bgpValidationRules(config) if err != nil { return err } maps.Copy(rules, bgpRules) // gendoc:generate(entity=network_bridge, group=common, key=user.*) // // --- // type: string // condition: - // default: - // shortdesc: User-provided free-form key/value pairs // Validate the configuration. err = n.validate(config, rules) if err != nil { return err } // Perform composite key checks after per-key validation. // Validate DNS zone names. err = n.validateZoneNames(config) if err != nil { return err } for k, v := range config { key := k // MTU checks if key == "bridge.mtu" && v != "" { mtu, err := strconv.ParseInt(v, 10, 64) if err != nil { return fmt.Errorf("Invalid value for an integer: %s", v) } ipv6 := config["ipv6.address"] if ipv6 != "" && ipv6 != "none" && mtu < 1280 { return errors.New("The minimum MTU for an IPv6 network is 1280") } ipv4 := config["ipv4.address"] if ipv4 != "" && ipv4 != "none" && mtu < 68 { return errors.New("The minimum MTU for an IPv4 network is 68") } } } // Check using same MAC address on every cluster node is safe. if config["bridge.hwaddr"] != "" { err = n.checkClusterWideMACSafe(config) if err != nil { return err } } // Check IPv4 OVN ranges. if config["ipv4.ovn.ranges"] != "" && util.IsTrueOrEmpty(config["ipv4.dhcp"]) { dhcpSubnet := n.DHCPv4Subnet() allowedNets := []*net.IPNet{} if dhcpSubnet != nil { if config["ipv4.dhcp.ranges"] == "" { return errors.New(`"ipv4.ovn.ranges" must be used in conjunction with non-overlapping "ipv4.dhcp.ranges" when DHCPv4 is enabled`) } allowedNets = append(allowedNets, dhcpSubnet) } ovnRanges, err := parseIPRanges(config["ipv4.ovn.ranges"], allowedNets...) if err != nil { return fmt.Errorf("Failed parsing ipv4.ovn.ranges: %w", err) } dhcpRanges, err := parseIPRanges(config["ipv4.dhcp.ranges"], allowedNets...) if err != nil { return fmt.Errorf("Failed parsing ipv4.dhcp.ranges: %w", err) } for _, ovnRange := range ovnRanges { for _, dhcpRange := range dhcpRanges { if IPRangesOverlap(ovnRange, dhcpRange) { return fmt.Errorf(`The range specified in "ipv4.ovn.ranges" (%q) cannot overlap with "ipv4.dhcp.ranges"`, ovnRange) } } } } // Check IPv6 OVN ranges. if config["ipv6.ovn.ranges"] != "" && util.IsTrueOrEmpty(config["ipv6.dhcp"]) { dhcpSubnet := n.DHCPv6Subnet() allowedNets := []*net.IPNet{} if dhcpSubnet != nil { if config["ipv6.dhcp.ranges"] == "" && util.IsTrue(config["ipv6.dhcp.stateful"]) { return errors.New(`"ipv6.ovn.ranges" must be used in conjunction with non-overlapping "ipv6.dhcp.ranges" when stateful DHCPv6 is enabled`) } allowedNets = append(allowedNets, dhcpSubnet) } ovnRanges, err := parseIPRanges(config["ipv6.ovn.ranges"], allowedNets...) if err != nil { return fmt.Errorf("Failed parsing ipv6.ovn.ranges: %w", err) } // If stateful DHCPv6 is enabled, check OVN ranges don't overlap with DHCPv6 stateful ranges. // Otherwise SLAAC will be being used to generate client IPs and predefined ranges aren't used. if dhcpSubnet != nil && util.IsTrue(config["ipv6.dhcp.stateful"]) { dhcpRanges, err := parseIPRanges(config["ipv6.dhcp.ranges"], allowedNets...) if err != nil { return fmt.Errorf("Failed parsing ipv6.dhcp.ranges: %w", err) } for _, ovnRange := range ovnRanges { for _, dhcpRange := range dhcpRanges { if IPRangesOverlap(ovnRange, dhcpRange) { return fmt.Errorf(`The range specified in "ipv6.ovn.ranges" (%q) cannot overlap with "ipv6.dhcp.ranges"`, ovnRange) } } } } } // Check Security ACLs are supported and exist. if config["security.acls"] != "" { err = acl.Exists(n.state, n.Project(), util.SplitNTrimSpace(config["security.acls"], ",", -1, true)...) if err != nil { return err } } return nil } // Create checks whether the bridge interface name is used already. func (n *bridge) Create(clientType request.ClientType) error { n.logger.Debug("Create", logger.Ctx{"clientType": clientType, "config": n.config}) if InterfaceExists(n.name) { return fmt.Errorf("Network interface %q already exists", n.name) } return nil } // isRunning returns whether the network is up. func (n *bridge) isRunning() bool { return InterfaceExists(n.name) } // Delete deletes a network. func (n *bridge) Delete(clientType request.ClientType) error { n.logger.Debug("Delete", logger.Ctx{"clientType": clientType}) if n.isRunning() { err := n.Stop() if err != nil { return err } } // Clean up extended external interfaces. if n.config["bridge.external_interfaces"] != "" { for _, entry := range strings.Split(n.config["bridge.external_interfaces"], ",") { entry = strings.TrimSpace(entry) entryParts := strings.Split(entry, "/") if len(entryParts) == 3 { ifName := strings.TrimSpace(entryParts[0]) _, err := net.InterfaceByName(ifName) if err == nil { err = InterfaceRemove(ifName) if err != nil { return err } } } } } // Delete apparmor profiles. err := apparmor.NetworkDelete(n.state.OS, n) if err != nil { return err } return n.common.delete(clientType) } // Rename renames a network. func (n *bridge) Rename(newName string) error { n.logger.Debug("Rename", logger.Ctx{"newName": newName}) if InterfaceExists(newName) { return fmt.Errorf("Network interface %q already exists", newName) } // Bring the network down. if n.isRunning() { err := n.Stop() if err != nil { return err } } // Rename common steps. err := n.common.rename(newName) if err != nil { return err } // Bring the network up. err = n.Start() if err != nil { return err } return nil } // Start starts the network. func (n *bridge) Start() error { n.logger.Debug("Start") reverter := revert.New() defer reverter.Fail() reverter.Add(func() { n.setUnavailable() }) err := n.setup(nil) if err != nil { return err } reverter.Success() // Ensure network is marked as available now its started. n.setAvailable() return nil } // setup restarts the network. func (n *bridge) setup(oldConfig map[string]string) error { // If we are in mock mode, just no-op. if n.state.OS.MockMode { return nil } n.logger.Debug("Setting up network") reverter := revert.New() defer reverter.Fail() // Create directory. if !util.PathExists(internalUtil.VarPath("networks", n.name)) { err := os.MkdirAll(internalUtil.VarPath("networks", n.name), 0o711) if err != nil { return err } } var err error // Build up the bridge interface's settings. bridge := ip.Bridge{ Link: ip.Link{ Name: n.name, MTU: bridgeMTUDefault, }, } // Get a list of tunnels. tunnels := n.getTunnels() // Decide the MTU for the bridge interface. if n.config["bridge.mtu"] != "" { mtuInt, err := strconv.ParseUint(n.config["bridge.mtu"], 10, 32) if err != nil { return fmt.Errorf("Invalid MTU %q: %w", n.config["bridge.mtu"], err) } bridge.MTU = uint32(mtuInt) } else if len(tunnels) > 0 { bridge.MTU = 1400 } // Decide the MAC address of bridge interface. if n.config["bridge.hwaddr"] != "" { bridge.Address, err = net.ParseMAC(n.config["bridge.hwaddr"]) if err != nil { return fmt.Errorf("Failed parsing MAC address %q: %w", n.config["bridge.hwaddr"], err) } } else { // If no cluster wide static MAC address set, then generate one. var seedNodeID int64 if n.checkClusterWideMACSafe(n.config) != nil { // If not safe to use a cluster wide MAC, then use cluster node's ID to // generate a stable per-node & network derived random MAC. seedNodeID = n.state.DB.Cluster.GetNodeID() } else { // If safe to use a cluster wide MAC, then use a static cluster node of 0 to generate a // stable per-network derived random MAC. seedNodeID = 0 } // Load server certificate. This is needs to be the same certificate for all nodes in a cluster. cert, err := internalUtil.LoadCert(n.state.OS.VarDir) if err != nil { return err } // Generate the random seed, this uses the server certificate fingerprint (to ensure that multiple // standalone nodes with the same network ID connected to the same external network don't generate // the same MAC for their networks). It relies on the certificate being the same for all nodes in a // cluster to allow the same MAC to be generated on each bridge interface in the network when // seedNodeID is 0 (when safe to do so). seed := fmt.Sprintf("%s.%d.%d", cert.Fingerprint(), seedNodeID, n.ID()) r, err := localUtil.GetStableRandomGenerator(seed) if err != nil { return fmt.Errorf("Failed generating stable random bridge MAC: %w", err) } randomHwaddr := n.randomHwaddr(r) bridge.Address, err = net.ParseMAC(randomHwaddr) if err != nil { return fmt.Errorf("Failed parsing MAC address %q: %w", randomHwaddr, err) } n.logger.Debug("Stable MAC generated", logger.Ctx{"seed": seed, "hwAddr": bridge.Address.String()}) } // Create the bridge interface if doesn't exist. if !n.isRunning() { if n.config["bridge.driver"] == "openvswitch" { // Handle IncusOS services. if n.state.OS.IncusOS != nil { ok, err := n.state.OS.IncusOS.IsServiceEnabled("ovn") if err != nil { return err } if !ok { return errors.New("IncusOS service \"ovn\" isn't currently enabled") } } // Try to connect to OVS. vswitch, err := n.state.OVS() if err != nil { return fmt.Errorf("Couldn't connect to OpenVSwitch: %v", err) } // Add and configure the interface in one operation to reduce the number of executions and // to avoid systemd-udevd from applying the default MACAddressPolicy=persistent policy. err = vswitch.CreateBridge(context.TODO(), n.name, false, bridge.Address, bridge.MTU) if err != nil { return err } reverter.Add(func() { _ = vswitch.DeleteBridge(context.Background(), n.name) }) } else { // Add and configure the interface in one operation to reduce the number of executions and // to avoid systemd-udevd from applying the default MACAddressPolicy=persistent policy. err := bridge.Add() if err != nil { return err } reverter.Add(func() { _ = bridge.Delete() }) } } else { // If bridge already exists then re-apply settings. If we just created a bridge then we don't // need to do this as the settings will have been applied as part of the add operation. // Set the MTU on the bridge interface. err := bridge.SetMTU(bridge.MTU) if err != nil { return err } // Set the MAC address on the bridge interface if specified. if bridge.Address != nil { err = bridge.SetAddress(bridge.Address) if err != nil { return err } } } // IPv6 bridge configuration. if !util.IsNoneOrEmpty(n.config["ipv6.address"]) { if !util.PathExists("/proc/sys/net/ipv6") { return errors.New("Network has ipv6.address but kernel IPv6 support is missing") } err := localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", n.name), "0") if err != nil { return err } err = localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/autoconf", n.name), "0") if err != nil { return err } err = localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/accept_dad", n.name), "0") if err != nil { return err } } else { // Disable IPv6 if no address is specified. This prevents the // host being reachable over a guessable link-local address as well as it // auto-configuring an address should an instance operate an IPv6 router. if util.PathExists("/proc/sys/net/ipv6") { err := localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", n.name), "1") if err != nil { return err } } } err = n.deleteChildren() if err != nil { return fmt.Errorf("Failed to delete bridge children interfaces: %w", err) } // Attempt to add a dummy device to the bridge to force the MTU. if bridge.MTU != bridgeMTUDefault && n.config["bridge.driver"] != "openvswitch" { dummy := &ip.Dummy{ Link: ip.Link{ Name: fmt.Sprintf("%s-mtu", n.name), MTU: bridge.MTU, }, } err = dummy.Add() if err == nil { reverter.Add(func() { _ = dummy.Delete() }) err = dummy.SetUp() if err == nil { _ = AttachInterface(n.state, n.name, fmt.Sprintf("%s-mtu", n.name)) } } } // Enable VLAN filtering for Linux bridges. if n.config["bridge.driver"] != "openvswitch" { // Enable filtering. err = BridgeVLANFilterSetStatus(n.name, "1") if err != nil { n.logger.Warn(fmt.Sprintf("Failed enabling VLAN filtering: %v", err)) } } // Bring it up. err = bridge.SetUp() if err != nil { return err } // Add any listed existing external interface. if n.config["bridge.external_interfaces"] != "" { for _, entry := range strings.Split(n.config["bridge.external_interfaces"], ",") { entry = strings.TrimSpace(entry) // Test for extended configuration of external interface. entryParts := strings.Split(entry, "/") ifParent := "" vlanID := 0 if len(entryParts) == 3 { vlanID, err = strconv.Atoi(entryParts[2]) if err != nil || vlanID < 1 || vlanID > 4094 { vlanID = 0 n.logger.Warn("Ignoring invalid VLAN ID", logger.Ctx{"interface": entry, "vlanID": entryParts[2]}) } else { entry = strings.TrimSpace(entryParts[0]) ifParent = strings.TrimSpace(entryParts[1]) } } iface, err := net.InterfaceByName(entry) if err != nil { if vlanID == 0 { n.logger.Warn("Skipping attaching missing external interface", logger.Ctx{"interface": entry}) continue } // If the interface doesn't exist and VLAN ID was provided, create the missing interface. ok, err := VLANInterfaceCreate(ifParent, entry, strconv.Itoa(vlanID), false) if ok { iface, err = net.InterfaceByName(entry) } if !ok || err != nil { return fmt.Errorf("Failed to create external interface %q", entry) } } else if vlanID > 0 { // If the interface exists and VLAN ID was provided, ensure it has the same parent and VLAN ID and is not attached to a different network. linkInfo, err := ip.LinkByName(entry) if err != nil { return fmt.Errorf("Failed to get link info for external interface %q", entry) } if n.config["bridge.driver"] != "openvswitch" { if linkInfo.Kind != "vlan" || linkInfo.Parent != ifParent || linkInfo.VlanID != vlanID || (linkInfo.Master != "" && linkInfo.Master != n.name) { return fmt.Errorf("External interface %q already in use", entry) } } else { vswitch, err := n.state.OVS() if err != nil { return err } ports, err := vswitch.GetBridgePorts(context.TODO(), n.name) if err != nil { return err } if linkInfo.Kind != "vlan" || linkInfo.Parent != ifParent || linkInfo.VlanID != vlanID || (linkInfo.Master != "" && !slices.Contains(ports, entry)) { return fmt.Errorf("External interface %q already in use", entry) } } } unused := true addrs, err := iface.Addrs() if err == nil { for _, addr := range addrs { ip, _, err := net.ParseCIDR(addr.String()) if ip != nil && err == nil && ip.IsGlobalUnicast() { unused = false break } } } if !unused { return errors.New("Only unconfigured network interfaces can be bridged") } err = AttachInterface(n.state, n.name, entry) if err != nil { return err } // Make sure the port is up. link := &ip.Link{Name: entry} err = link.SetUp() if err != nil { return fmt.Errorf("Failed to bring up the host interface %s: %w", entry, err) } } } // Remove any existing firewall rules. fwClearIPVersions := []uint{} if usesIPv4Firewall(n.config) || usesIPv4Firewall(oldConfig) { fwClearIPVersions = append(fwClearIPVersions, 4) } if usesIPv6Firewall(n.config) || usesIPv6Firewall(oldConfig) { fwClearIPVersions = append(fwClearIPVersions, 6) } if len(fwClearIPVersions) > 0 { n.logger.Debug("Clearing firewall") err = n.state.Firewall.NetworkClear(n.name, false, fwClearIPVersions) if err != nil { return fmt.Errorf("Failed clearing firewall: %w", err) } } // Initialize a new firewall option set. fwOpts := firewallDrivers.Opts{} if n.hasIPv4Firewall() { fwOpts.FeaturesV4 = &firewallDrivers.FeatureOpts{} } if n.hasIPv6Firewall() { fwOpts.FeaturesV6 = &firewallDrivers.FeatureOpts{} } if n.config["security.acls"] != "" { fwOpts.ACL = true } // Snapshot container specific IPv4 routes (added with boot proto) before removing IPv4 addresses. // This is because the kernel removes any static routes on an interface when all addresses removed. ctRoutes, err := n.bootRoutesV4() if err != nil { return err } // Flush all IPv4 addresses and routes. addr := &ip.Addr{ DevName: n.name, Scope: "global", Family: ip.FamilyV4, } err = addr.Flush() if err != nil { return err } r := &ip.Route{ DevName: n.name, Proto: "static", Family: ip.FamilyV4, } err = r.Flush() if err != nil { return err } // Configure IPv4 firewall. if !util.IsNoneOrEmpty(n.config["ipv4.address"]) { if n.hasDHCPv4() && n.hasIPv4Firewall() { fwOpts.FeaturesV4.ICMPDHCPDNSAccess = true } // Allow forwarding. if util.IsTrueOrEmpty(n.config["ipv4.routing"]) { err = localUtil.SysctlSet("net/ipv4/ip_forward", "1") if err != nil { return err } if n.hasIPv4Firewall() { fwOpts.FeaturesV4.ForwardingAllow = true } } } // Start building process using subprocess package. command := "dnsmasq" dnsmasqCmd := []string{ "--keep-in-foreground", "--strict-order", "--bind-interfaces", "--except-interface=lo", "--pid-file=", // Disable attempt at writing a PID file. "--no-ping", // --no-ping is very important to prevent delays to lease file updates. fmt.Sprintf("--interface=%s", n.name), } dnsmasqCmd = append(dnsmasqCmd, "--dhcp-rapid-commit", "--no-negcache") if !daemon.Debug { dnsmasqCmd = append(dnsmasqCmd, "--quiet-dhcp", "--quiet-dhcp6", "--quiet-ra") } var dnsIPv4 []string var dnsIPv6 []string for _, s := range util.SplitNTrimSpace(n.config["dns.nameservers"], ",", -1, false) { if net.ParseIP(s).To4() != nil { dnsIPv4 = append(dnsIPv4, s) } else { dnsIPv6 = append(dnsIPv6, s) } } // Configure IPv4. if !util.IsNoneOrEmpty(n.config["ipv4.address"]) { // Parse the subnet. ipAddress, subnet, err := net.ParseCIDR(n.config["ipv4.address"]) if err != nil { return fmt.Errorf("Failed parsing ipv4.address: %w", err) } // Update the dnsmasq config. dnsmasqCmd = append(dnsmasqCmd, fmt.Sprintf("--listen-address=%s", ipAddress.String())) if n.DHCPv4Subnet() != nil { if !slices.Contains(dnsmasqCmd, "--dhcp-no-override") { dnsmasqCmd = append(dnsmasqCmd, []string{"--dhcp-no-override", "--dhcp-authoritative", fmt.Sprintf("--dhcp-leasefile=%s", internalUtil.VarPath("networks", n.name, "dnsmasq.leases")), fmt.Sprintf("--dhcp-hostsfile=%s", internalUtil.VarPath("networks", n.name, "dnsmasq.hosts"))}...) } switch n.config["ipv4.dhcp.gateway"] { case "": case "none": dnsmasqCmd = append(dnsmasqCmd, "--dhcp-option=3") default: dnsmasqCmd = append(dnsmasqCmd, fmt.Sprintf("--dhcp-option-force=3,%s", n.config["ipv4.dhcp.gateway"])) } if n.config["dns.nameservers"] != "" { if len(dnsIPv4) == 0 { dnsmasqCmd = append(dnsmasqCmd, "--dhcp-option-force=6") } else { dnsmasqCmd = append(dnsmasqCmd, fmt.Sprintf("--dhcp-option-force=6,%s", strings.Join(dnsIPv4, ","))) } } if bridge.MTU != bridgeMTUDefault { dnsmasqCmd = append(dnsmasqCmd, fmt.Sprintf("--dhcp-option-force=26,%d", bridge.MTU)) } dnsSearch := n.config["dns.search"] if dnsSearch != "" { dnsmasqCmd = append(dnsmasqCmd, fmt.Sprintf("--dhcp-option-force=119,%s", strings.Trim(dnsSearch, " "))) } if n.config["ipv4.dhcp.routes"] != "" { dnsmasqCmd = append(dnsmasqCmd, fmt.Sprintf("--dhcp-option-force=121,%s", strings.ReplaceAll(n.config["ipv4.dhcp.routes"], " ", ""))) } expiry := "1h" if n.config["ipv4.dhcp.expiry"] != "" { expiry = n.config["ipv4.dhcp.expiry"] } if n.config["ipv4.dhcp.ranges"] != "" { for _, dhcpRange := range strings.Split(n.config["ipv4.dhcp.ranges"], ",") { dhcpRange = strings.TrimSpace(dhcpRange) dnsmasqCmd = append(dnsmasqCmd, []string{"--dhcp-range", fmt.Sprintf("%s,%s", strings.ReplaceAll(dhcpRange, "-", ","), expiry)}...) } } else { dnsmasqCmd = append(dnsmasqCmd, []string{"--dhcp-range", fmt.Sprintf("%s,%s,%s", dhcpalloc.GetIP(subnet, 2).String(), dhcpalloc.GetIP(subnet, -2).String(), expiry)}...) } } // Add the address. addr := &ip.Addr{ DevName: n.name, Address: &net.IPNet{ IP: ipAddress, Mask: subnet.Mask, }, Family: ip.FamilyV4, } err = addr.Add() if err != nil { return err } // Configure NAT. if util.IsTrue(n.config["ipv4.nat"]) { // If a SNAT source address is specified, use that, otherwise default to MASQUERADE mode. var srcIP net.IP if n.config["ipv4.nat.address"] != "" { srcIP = net.ParseIP(n.config["ipv4.nat.address"]) } fwOpts.SNATV4 = &firewallDrivers.SNATOpts{ SNATAddress: srcIP, Subnet: subnet, } if n.config["ipv4.nat.order"] == "after" { fwOpts.SNATV4.Append = true } } // Add additional routes. if n.config["ipv4.routes"] != "" { for _, route := range strings.Split(n.config["ipv4.routes"], ",") { route, err := ip.ParseIPNet(strings.TrimSpace(route)) if err != nil { return err } r := &ip.Route{ DevName: n.name, Route: route, Proto: "static", Family: ip.FamilyV4, } err = r.Add() if err != nil { return err } } } // Restore container specific IPv4 routes to interface. n.applyBootRoutesV4(ctRoutes) } // Snapshot container specific IPv6 routes (added with boot proto) before removing IPv6 addresses. // This is because the kernel removes any static routes on an interface when all addresses removed. ctRoutes, err = n.bootRoutesV6() if err != nil { return err } // Flush all IPv6 addresses and routes. addr = &ip.Addr{ DevName: n.name, Scope: "global", Family: ip.FamilyV6, } err = addr.Flush() if err != nil { return err } r = &ip.Route{ DevName: n.name, Proto: "static", Family: ip.FamilyV6, } err = r.Flush() if err != nil { return err } // Configure IPv6. if !util.IsNoneOrEmpty(n.config["ipv6.address"]) { // Enable IPv6 for the subnet. err := localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", n.name), "0") if err != nil { return err } // Parse the subnet. ipAddress, subnet, err := net.ParseCIDR(n.config["ipv6.address"]) if err != nil { return fmt.Errorf("Failed parsing ipv6.address: %w", err) } subnetSize, _ := subnet.Mask.Size() // Check if we need to generate a host-specific address. if ipAddress.String() == subnet.IP.String() { if subnetSize != 64 { return errors.New("Can't generate an EUI64 derived IPv6 address with a mask other than /64") } ipAddress, err = eui64.ParseMAC(subnet.IP, bridge.Address) if err != nil { return fmt.Errorf("Failed generating EUI64 value for ipv6.address: %w", err) } } if subnetSize > 64 { n.logger.Warn("IPv6 networks with a prefix larger than 64 aren't properly supported by dnsmasq") err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpsertWarningLocalNode(ctx, n.project, dbCluster.TypeNetwork, int(n.id), warningtype.LargerIPv6PrefixThanSupported, "") }) if err != nil { n.logger.Warn("Failed to create warning", logger.Ctx{"err": err}) } } else { err = warnings.ResolveWarningsByLocalNodeAndProjectAndTypeAndEntity(n.state.DB.Cluster, n.project, warningtype.LargerIPv6PrefixThanSupported, dbCluster.TypeNetwork, int(n.id)) if err != nil { n.logger.Warn("Failed to resolve warning", logger.Ctx{"err": err}) } } // Update the dnsmasq config. dnsmasqCmd = append(dnsmasqCmd, []string{fmt.Sprintf("--listen-address=%s", ipAddress.String()), "--enable-ra"}...) if n.DHCPv6Subnet() != nil { if n.hasIPv6Firewall() { fwOpts.FeaturesV6.ICMPDHCPDNSAccess = true } // Build DHCP configuration. if !slices.Contains(dnsmasqCmd, "--dhcp-no-override") { dnsmasqCmd = append(dnsmasqCmd, []string{"--dhcp-no-override", "--dhcp-authoritative", fmt.Sprintf("--dhcp-leasefile=%s", internalUtil.VarPath("networks", n.name, "dnsmasq.leases")), fmt.Sprintf("--dhcp-hostsfile=%s", internalUtil.VarPath("networks", n.name, "dnsmasq.hosts"))}...) } expiry := "1h" if n.config["ipv6.dhcp.expiry"] != "" { expiry = n.config["ipv6.dhcp.expiry"] } if util.IsTrue(n.config["ipv6.dhcp.stateful"]) { if n.config["ipv6.dhcp.ranges"] != "" { for _, dhcpRange := range strings.Split(n.config["ipv6.dhcp.ranges"], ",") { dhcpRange = strings.TrimSpace(dhcpRange) dnsmasqCmd = append(dnsmasqCmd, []string{"--dhcp-range", fmt.Sprintf("%s,%d,%s", strings.ReplaceAll(dhcpRange, "-", ","), subnetSize, expiry)}...) } } else { dnsmasqCmd = append(dnsmasqCmd, []string{"--dhcp-range", fmt.Sprintf("%s,%s,%d,%s", dhcpalloc.GetIP(subnet, 2), dhcpalloc.GetIP(subnet, -1), subnetSize, expiry)}...) } } else { dnsmasqCmd = append(dnsmasqCmd, []string{"--dhcp-range", fmt.Sprintf("::,constructor:%s,ra-stateless,ra-names", n.name)}...) } } else { dnsmasqCmd = append(dnsmasqCmd, []string{"--dhcp-range", fmt.Sprintf("::,constructor:%s,ra-only", n.name)}...) } if n.config["dns.nameservers"] != "" { if len(dnsIPv6) == 0 { dnsmasqCmd = append(dnsmasqCmd, "--dhcp-option-force=option6:dns-server") } else { dnsmasqCmd = append(dnsmasqCmd, fmt.Sprintf("--dhcp-option-force=option6:dns-server,[%s]", strings.Join(dnsIPv6, ","))) } } else { dnsmasqCmd = append(dnsmasqCmd, fmt.Sprintf("--dhcp-option-force=option6:dns-server,[%s]", ipAddress.String())) } // Disable receiving router advertisements from guests. err = localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", n.name), "0") if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } // Allow forwarding. if util.IsTrueOrEmpty(n.config["ipv6.routing"]) { // Get a list of proc entries. entries, err := os.ReadDir("/proc/sys/net/ipv6/conf/") if err != nil { return err } // First set accept_ra to 2 for all interfaces (if not disabled). // This ensures that the host can still receive IPv6 router advertisements even with // forwarding enabled (which enable below), as the default is to ignore router adverts // when forward is enabled, and this could render the host unreachable if it uses // SLAAC generated IPs. for _, entry := range entries { // Check that IPv6 router advertisement acceptance is enabled currently. // If its set to 0 then we don't want to enable, and if its already set to 2 then // we don't need to do anything. content, err := os.ReadFile(fmt.Sprintf("/proc/sys/net/ipv6/conf/%s/accept_ra", entry.Name())) if err == nil && string(content) != "1\n" { continue } // If IPv6 router acceptance is enabled (set to 1) then we now set it to 2. err = localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", entry.Name()), "2") if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } } // Then set forwarding for all of them. for _, entry := range entries { err = localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/forwarding", entry.Name()), "1") if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } } if n.hasIPv6Firewall() { fwOpts.FeaturesV6.ForwardingAllow = true } } // Add the address. addr := &ip.Addr{ DevName: n.name, Address: &net.IPNet{ IP: ipAddress, Mask: subnet.Mask, }, Family: ip.FamilyV6, } err = addr.Add() if err != nil { return err } // Configure NAT. if util.IsTrue(n.config["ipv6.nat"]) { // If a SNAT source address is specified, use that, otherwise default to MASQUERADE mode. var srcIP net.IP if n.config["ipv6.nat.address"] != "" { srcIP = net.ParseIP(n.config["ipv6.nat.address"]) } fwOpts.SNATV6 = &firewallDrivers.SNATOpts{ SNATAddress: srcIP, Subnet: subnet, } if n.config["ipv6.nat.order"] == "after" { fwOpts.SNATV6.Append = true } } // Add additional routes. if n.config["ipv6.routes"] != "" { for _, route := range strings.Split(n.config["ipv6.routes"], ",") { route, err := ip.ParseIPNet(route) if err != nil { return err } r := &ip.Route{ DevName: n.name, Route: route, Proto: "static", Family: ip.FamilyV6, } err = r.Add() if err != nil { return err } } } // Restore container specific IPv6 routes to interface. n.applyBootRoutesV6(ctRoutes) } // Configure tunnels. for _, tunnel := range tunnels { getConfig := func(key string) string { return n.config[fmt.Sprintf("tunnel.%s.%s", tunnel, key)] } tunProtocol := getConfig("protocol") tunLocal := net.ParseIP(getConfig("local")) tunRemote := net.ParseIP(getConfig("remote")) tunName := fmt.Sprintf("%s-%s", n.name, tunnel) // Configure the tunnel. if tunProtocol == "gre" { // Skip partial configs. if tunLocal == nil || tunRemote == nil { continue } gretap := &ip.Gretap{ Link: ip.Link{Name: tunName}, Local: tunLocal, Remote: tunRemote, } err := gretap.Add() if err != nil { return err } } else if tunProtocol == "vxlan" { tunGroup := net.ParseIP(getConfig("group")) tunInterface := getConfig("interface") vxlan := &ip.Vxlan{ Link: ip.Link{Name: tunName}, Local: tunLocal, } if tunRemote != nil { // Skip partial configs. if tunLocal == nil { continue } vxlan.Remote = tunRemote } else { if tunGroup == nil { tunGroup = net.IPv4(239, 0, 0, 1) // 239.0.0.1 } devName := tunInterface if devName == "" { _, devName, err = DefaultGatewaySubnetV4() if err != nil { return err } } vxlan.Group = tunGroup vxlan.DevName = devName } tunPort := getConfig("port") if tunPort != "" { vxlan.DstPort, err = strconv.Atoi(tunPort) if err != nil { return err } } tunID := getConfig("id") if tunID == "" { vxlan.VxlanID = 1 } else { vxlan.VxlanID, err = strconv.Atoi(tunID) if err != nil { return err } } tunTTL := getConfig("ttl") if tunTTL == "" { vxlan.TTL = 1 } else { vxlan.TTL, err = strconv.Atoi(tunTTL) if err != nil { return err } } err := vxlan.Add() if err != nil { return err } } // Bridge it and bring up. err = AttachInterface(n.state, n.name, tunName) if err != nil { return err } tunLink := &ip.Link{Name: tunName} err = tunLink.SetMTU(bridge.MTU) if err != nil { return err } // Bring up tunnel interface. err = tunLink.SetUp() if err != nil { return err } // Bring up network interface. err = bridge.SetUp() if err != nil { return err } } // Generate and load apparmor profiles. err = apparmor.NetworkLoad(n.state.OS, n) if err != nil { return err } // Kill any existing dnsmasq daemon for this network. err = dnsmasq.Kill(n.name, false) if err != nil { return err } // Configure dnsmasq. if n.UsesDNSMasq() { // Setup the dnsmasq domain. dnsDomain := n.config["dns.domain"] if dnsDomain == "" { dnsDomain = "incus" } if n.config["dns.mode"] != "none" { dnsmasqCmd = append(dnsmasqCmd, "-s", dnsDomain) dnsmasqCmd = append(dnsmasqCmd, "--interface-name", fmt.Sprintf("_gateway.%s,%s", dnsDomain, n.name)) dnsmasqCmd = append(dnsmasqCmd, "-S", fmt.Sprintf("/%s/", dnsDomain)) } // Create a config file to contain additional config (and to prevent dnsmasq from reading /etc/dnsmasq.conf) err = os.WriteFile(internalUtil.VarPath("networks", n.name, "dnsmasq.raw"), fmt.Appendf(nil, "%s\n", n.config["raw.dnsmasq"]), 0o644) if err != nil { return err } dnsmasqCmd = append(dnsmasqCmd, fmt.Sprintf("--conf-file=%s", internalUtil.VarPath("networks", n.name, "dnsmasq.raw"))) // Attempt to drop privileges. if n.state.OS.UnprivUser != "" { dnsmasqCmd = append(dnsmasqCmd, []string{"-u", n.state.OS.UnprivUser}...) } if n.state.OS.UnprivGroup != "" { dnsmasqCmd = append(dnsmasqCmd, []string{"-g", n.state.OS.UnprivGroup}...) } // Create DHCP hosts directory. if !util.PathExists(internalUtil.VarPath("networks", n.name, "dnsmasq.hosts")) { err = os.MkdirAll(internalUtil.VarPath("networks", n.name, "dnsmasq.hosts"), 0o755) if err != nil { return err } } // Check for dnsmasq. _, err := exec.LookPath("dnsmasq") if err != nil { return errors.New("dnsmasq is required for managed bridges") } // Update the static leases. err = UpdateDNSMasqStatic(n.state, n.name) if err != nil { return err } // Create subprocess object dnsmasq. dnsmasqLogPath := internalUtil.LogPath(fmt.Sprintf("dnsmasq.%s.log", n.name)) p, err := subprocess.NewProcess(command, dnsmasqCmd, "", dnsmasqLogPath) if err != nil { return fmt.Errorf("Failed to create subprocess: %s", err) } // Apply AppArmor confinement. if n.config["raw.dnsmasq"] == "" { p.SetApparmor(apparmor.DnsmasqProfileName(n)) err = warnings.ResolveWarningsByLocalNodeAndProjectAndTypeAndEntity(n.state.DB.Cluster, n.project, warningtype.AppArmorDisabledDueToRawDnsmasq, dbCluster.TypeNetwork, int(n.id)) if err != nil { n.logger.Warn("Failed to resolve warning", logger.Ctx{"err": err}) } } else { n.logger.Warn("Skipping AppArmor for dnsmasq due to raw.dnsmasq being set", logger.Ctx{"name": n.name}) err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpsertWarningLocalNode(ctx, n.project, dbCluster.TypeNetwork, int(n.id), warningtype.AppArmorDisabledDueToRawDnsmasq, "") }) if err != nil { n.logger.Warn("Failed to create warning", logger.Ctx{"err": err}) } } // Start dnsmasq. err = p.Start(context.Background()) if err != nil { return fmt.Errorf("Failed to run: %s %s: %w", command, strings.Join(dnsmasqCmd, " "), err) } // Check dnsmasq started OK. ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Millisecond*time.Duration(500))) _, err = p.Wait(ctx) if !errors.Is(err, context.DeadlineExceeded) { stderr, _ := os.ReadFile(dnsmasqLogPath) cancel() return fmt.Errorf("The DNS and DHCP service exited prematurely: %w (%q)", err, strings.TrimSpace(string(stderr))) } cancel() err = p.Save(internalUtil.VarPath("networks", n.name, "dnsmasq.pid")) if err != nil { // Kill Process if started, but could not save the file. err2 := p.Stop() if err2 != nil { return fmt.Errorf("Could not kill subprocess while handling saving error: %s: %s", err, err2) } return fmt.Errorf("Failed to save subprocess details: %s", err) } } else { // Clean up old dnsmasq config if exists and we are not starting dnsmasq. leasesPath := internalUtil.VarPath("networks", n.name, "dnsmasq.leases") if util.PathExists(leasesPath) { err := os.Remove(leasesPath) if err != nil { return fmt.Errorf("Failed to remove old dnsmasq leases file %q: %w", leasesPath, err) } } // Clean up old dnsmasq PID file. pidPath := internalUtil.VarPath("networks", n.name, "dnsmasq.pid") if util.PathExists(pidPath) { err := os.Remove(pidPath) if err != nil { return fmt.Errorf("Failed to remove old dnsmasq pid file %q: %w", pidPath, err) } } } // Setup firewall. n.logger.Debug("Setting up firewall") if n.state.Firewall.String() == "nftables" { n.logger.Debug("Address set feature enabled for nftables backend") fwOpts.AddressSet = true } err = n.state.Firewall.NetworkSetup(n.name, fwOpts) if err != nil { return fmt.Errorf("Failed to setup firewall: %w", err) } // Setup named sets for nft firewall. // We apply all available address sets to avoid missing some. if fwOpts.AddressSet { n.logger.Debug("Applying up firewall address sets") aclNames := util.SplitNTrimSpace(n.config["security.acls"], ",", -1, false) err = addressset.FirewallApplyAddressSetsForACLRules(n.state, "inet", n.Project(), aclNames) if err != nil { return err } } if fwOpts.ACL { aclNet := acl.NetworkACLUsage{ Name: n.Name(), Type: n.Type(), ID: n.ID(), Config: n.Config(), } n.logger.Debug("Applying up firewall ACLs") err = acl.FirewallApplyACLRules(n.state, n.logger, n.Project(), aclNet) if err != nil { return err } } // Setup network address forwards. err = n.forwardSetupFirewall() if err != nil { return err } // Setup BGP. err = n.bgpSetup(oldConfig) if err != nil { return err } reverter.Success() return nil } // Stop stops the network. func (n *bridge) Stop() error { n.logger.Debug("Stop") if !n.isRunning() { return nil } // Clear BGP. err := n.bgpClear(n.config) if err != nil { return err } err = n.deleteChildren() if err != nil { return fmt.Errorf("Failed to delete bridge children interfaces: %w", err) } // Destroy the bridge interface if n.config["bridge.driver"] == "openvswitch" { vswitch, err := n.state.OVS() if err != nil { return err } err = vswitch.DeleteBridge(context.TODO(), n.name) if err != nil { return err } } else { bridgeLink := &ip.Link{Name: n.name} err := bridgeLink.Delete() if err != nil { return err } } // Fully clear firewall setup. fwClearIPVersions := []uint{} if usesIPv4Firewall(n.config) { fwClearIPVersions = append(fwClearIPVersions, 4) } if usesIPv6Firewall(n.config) { fwClearIPVersions = append(fwClearIPVersions, 6) } if len(fwClearIPVersions) > 0 { n.logger.Debug("Deleting firewall") err := n.state.Firewall.NetworkClear(n.name, true, fwClearIPVersions) if err != nil { return fmt.Errorf("Failed deleting firewall: %w", err) } } // Kill any existing dnsmasq daemon for this network err = dnsmasq.Kill(n.name, false) if err != nil { return err } // Unload apparmor profiles. err = apparmor.NetworkUnload(n.state.OS, n) if err != nil { return err } return nil } // Update updates the network. Accepts notification boolean indicating if this update request is coming from a // cluster notification, in which case do not update the database, just apply local changes needed. func (n *bridge) Update(newNetwork api.NetworkPut, targetNode string, clientType request.ClientType) error { n.logger.Debug("Update", logger.Ctx{"clientType": clientType, "newNetwork": newNetwork}) err := n.populateAutoConfig(newNetwork.Config) if err != nil { return fmt.Errorf("Failed generating auto config: %w", err) } dbUpdateNeeded, changedKeys, oldNetwork, err := n.common.configChanged(newNetwork) if err != nil { return err } if !dbUpdateNeeded { return nil // Nothing changed. } // If the network as a whole has not had any previous creation attempts, or the node itself is still // pending, then don't apply the new settings to the node, just to the database record (ready for the // actual global create request to be initiated). if n.Status() == api.NetworkStatusPending || n.LocalStatus() == api.NetworkStatusPending { return n.common.update(newNetwork, targetNode, clientType) } reverter := revert.New() defer reverter.Fail() // Perform any pre-update cleanup needed if local member network was already created. if len(changedKeys) > 0 { // Define a function which reverts everything. reverter.Add(func() { // Reset changes to all nodes and database. _ = n.common.update(oldNetwork, targetNode, clientType) // Reset any change that was made to local bridge. _ = n.setup(newNetwork.Config) }) // Bring the bridge down entirely if the driver has changed. if slices.Contains(changedKeys, "bridge.driver") && n.isRunning() { err = n.Stop() if err != nil { return err } } // Detach any external interfaces should no longer be attached. if slices.Contains(changedKeys, "bridge.external_interfaces") && n.isRunning() { devices := []string{} for _, dev := range strings.Split(newNetwork.Config["bridge.external_interfaces"], ",") { dev = strings.TrimSpace(dev) devices = append(devices, dev) } for _, dev := range strings.Split(oldNetwork.Config["bridge.external_interfaces"], ",") { dev = strings.TrimSpace(dev) if dev == "" { continue } // Test for extended configuration of external interface. ifName := dev devParts := strings.Split(dev, "/") if len(devParts) == 3 { ifName = strings.TrimSpace(devParts[0]) } if !slices.Contains(devices, dev) && InterfaceExists(ifName) { err = DetachInterface(n.state, n.name, ifName) if err != nil { return err } // Remove the interface if it exists (and we created it). if len(devParts) == 3 { _, err := net.InterfaceByName(ifName) if err == nil { err = InterfaceRemove(ifName) if err != nil { return err } } } } } } } // Apply changes to all nodes and database. err = n.common.update(newNetwork, targetNode, clientType) if err != nil { return err } // Restart the network if needed. if len(changedKeys) > 0 { err = n.setup(oldNetwork.Config) if err != nil { return err } } reverter.Success() // Notify dependent networks (those using this network as their uplink) of the changes. // Do this after the network has been successfully updated so that a failure to notify a dependent network // doesn't prevent the network itself from being updated. if clientType == request.ClientTypeNormal && len(changedKeys) > 0 { n.notifyDependentNetworks(changedKeys) } return nil } func (n *bridge) getTunnels() []string { tunnels := []string{} for k := range n.config { if !strings.HasPrefix(k, "tunnel.") { continue } fields := strings.Split(k, ".") if !slices.Contains(tunnels, fields[1]) { tunnels = append(tunnels, fields[1]) } } return tunnels } // bootRoutesV4 returns a list of IPv4 boot routes on the network's device. func (n *bridge) bootRoutesV4() ([]ip.Route, error) { r := &ip.Route{ DevName: n.name, Proto: "boot", Family: ip.FamilyV4, } routes, err := r.List() if err != nil { return nil, err } return routes, nil } // bootRoutesV6 returns a list of IPv6 boot routes on the network's device. func (n *bridge) bootRoutesV6() ([]ip.Route, error) { r := &ip.Route{ DevName: n.name, Proto: "boot", Family: ip.FamilyV6, } routes, err := r.List() if err != nil { return nil, err } return routes, nil } // applyBootRoutesV4 applies a list of IPv4 boot routes to the network's device. func (n *bridge) applyBootRoutesV4(routes []ip.Route) { for _, route := range routes { err := route.Replace() if err != nil { // If it fails, then we can't stop as the route has already gone, so just log and continue. n.logger.Error("Failed to restore route", logger.Ctx{"err": err}) } } } // applyBootRoutesV6 applies a list of IPv6 boot routes to the network's device. func (n *bridge) applyBootRoutesV6(routes []ip.Route) { for _, route := range routes { err := route.Replace() if err != nil { // If it fails, then we can't stop as the route has already gone, so just log and continue. n.logger.Error("Failed to restore route", logger.Ctx{"err": err}) } } } // hasIPv4Firewall indicates whether the network has IPv4 firewall enabled. func (n *bridge) hasIPv4Firewall() bool { // IPv4 firewall is only enabled if there is a bridge ipv4.address and ipv4.firewall enabled. if !util.IsNoneOrEmpty(n.config["ipv4.address"]) && util.IsTrueOrEmpty(n.config["ipv4.firewall"]) { return true } return false } // hasIPv6Firewall indicates whether the network has IPv6 firewall enabled. func (n *bridge) hasIPv6Firewall() bool { // IPv6 firewall is only enabled if there is a bridge ipv6.address and ipv6.firewall enabled. if !util.IsNoneOrEmpty(n.config["ipv6.address"]) && util.IsTrueOrEmpty(n.config["ipv6.firewall"]) { return true } return false } // hasDHCPv4 indicates whether the network has DHCPv4 enabled. // An empty ipv4.dhcp setting indicates enabled by default. func (n *bridge) hasDHCPv4() bool { return util.IsTrueOrEmpty(n.config["ipv4.dhcp"]) } // hasDHCPv6 indicates whether the network has DHCPv6 enabled. // An empty ipv6.dhcp setting indicates enabled by default. func (n *bridge) hasDHCPv6() bool { return util.IsTrueOrEmpty(n.config["ipv6.dhcp"]) } // DHCPv4Subnet returns the DHCPv4 subnet (if DHCP is enabled on network). func (n *bridge) DHCPv4Subnet() *net.IPNet { // DHCP is disabled on this network. if !n.hasDHCPv4() { return nil } // Return configured bridge subnet directly. _, subnet, err := net.ParseCIDR(n.config["ipv4.address"]) if err != nil { return nil } return subnet } // DHCPv6Subnet returns the DHCPv6 subnet (if DHCP or SLAAC is enabled on network). func (n *bridge) DHCPv6Subnet() *net.IPNet { // DHCP is disabled on this network. if !n.hasDHCPv6() { return nil } _, subnet, err := net.ParseCIDR(n.config["ipv6.address"]) if err != nil { return nil } return subnet } // forwardConvertToFirewallForward converts forwards into format compatible with the firewall package. func (n *bridge) forwardConvertToFirewallForwards(listenAddress net.IP, defaultTargetAddress net.IP, portMaps []*forwardPortMap) []firewallDrivers.AddressForward { var vips []firewallDrivers.AddressForward if defaultTargetAddress != nil { vips = append(vips, firewallDrivers.AddressForward{ ListenAddress: listenAddress, TargetAddress: defaultTargetAddress, }) } for _, portMap := range portMaps { vips = append(vips, firewallDrivers.AddressForward{ ListenAddress: listenAddress, Protocol: portMap.protocol, TargetAddress: portMap.target.address, ListenPorts: portMap.listenPorts, TargetPorts: portMap.target.ports, SNAT: portMap.snat, }) } return vips } // bridgeProjectNetworks takes a map of all networks in all projects and returns a filtered map of bridge networks. func (n *bridge) bridgeProjectNetworks(projectNetworks map[string]map[int64]api.Network) map[string][]*api.Network { bridgeProjectNetworks := make(map[string][]*api.Network) for netProject, networks := range projectNetworks { for _, ni := range networks { network := ni // Local var creating pointer to rather than iterator. // Skip non-bridge networks. if network.Type != "bridge" { continue } if bridgeProjectNetworks[netProject] == nil { bridgeProjectNetworks[netProject] = []*api.Network{&network} } else { bridgeProjectNetworks[netProject] = append(bridgeProjectNetworks[netProject], &network) } } } return bridgeProjectNetworks } // bridgeNetworkExternalSubnets returns a list of external subnets used by bridge networks. Networks are considered // to be using external subnets for their ipv4.address and/or ipv6.address if they have NAT disabled, and/or if // they have external NAT addresses specified. func (n *bridge) bridgeNetworkExternalSubnets(bridgeProjectNetworks map[string][]*api.Network) ([]externalSubnetUsage, error) { externalSubnets := make([]externalSubnetUsage, 0) for netProject, networks := range bridgeProjectNetworks { for _, netInfo := range networks { for _, keyPrefix := range []string{"ipv4", "ipv6"} { // If NAT is disabled, then network subnet is an external subnet. if util.IsFalseOrEmpty(netInfo.Config[fmt.Sprintf("%s.nat", keyPrefix)]) { key := fmt.Sprintf("%s.address", keyPrefix) _, ipNet, err := net.ParseCIDR(netInfo.Config[key]) if err != nil { continue // Skip invalid/unspecified network addresses. } externalSubnets = append(externalSubnets, externalSubnetUsage{ subnet: *ipNet, networkProject: netProject, networkName: netInfo.Name, usageType: subnetUsageNetwork, }) } // Find any external subnets used for network SNAT. if netInfo.Config[fmt.Sprintf("%s.nat.address", keyPrefix)] != "" { key := fmt.Sprintf("%s.nat.address", keyPrefix) subnetSize := 128 if keyPrefix == "ipv4" { subnetSize = 32 } _, ipNet, err := net.ParseCIDR(fmt.Sprintf("%s/%d", netInfo.Config[key], subnetSize)) if err != nil { return nil, fmt.Errorf("Failed parsing %q of %q in project %q: %w", key, netInfo.Name, netProject, err) } externalSubnets = append(externalSubnets, externalSubnetUsage{ subnet: *ipNet, networkProject: netProject, networkName: netInfo.Name, usageType: subnetUsageNetworkSNAT, }) } // Find any routes being used by the network. for _, cidr := range util.SplitNTrimSpace(netInfo.Config[fmt.Sprintf("%s.routes", keyPrefix)], ",", -1, true) { _, ipNet, err := net.ParseCIDR(cidr) if err != nil { continue // Skip invalid/unspecified network addresses. } externalSubnets = append(externalSubnets, externalSubnetUsage{ subnet: *ipNet, networkProject: netProject, networkName: netInfo.Name, usageType: subnetUsageNetwork, }) } } } } return externalSubnets, nil } // bridgedNICExternalRoutes returns a list of external routes currently used by bridged NICs that are connected to // networks specified. func (n *bridge) bridgedNICExternalRoutes(bridgeProjectNetworks map[string][]*api.Network) ([]externalSubnetUsage, error) { externalRoutes := make([]externalSubnetUsage, 0) err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.InstanceList(ctx, func(inst db.InstanceArgs, p api.Project) error { // Get the instance's effective network project name. instNetworkProject := project.NetworkProjectFromRecord(&p) if instNetworkProject != api.ProjectDefaultName { return nil // Managed bridge networks can only exist in default project. } devices := db.ExpandInstanceDevices(inst.Devices, inst.Profiles) // Iterate through each of the instance's devices, looking for bridged NICs that are linked to // networks specified. for devName, devConfig := range devices { if devConfig["type"] != "nic" { continue } // Check whether the NIC device references one of the networks supplied. if !NICUsesNetwork(devConfig, bridgeProjectNetworks[instNetworkProject]...) { continue } // For bridged NICs that are connected to networks specified, check if they have any // routes or external routes configured, and if so add them to the list to return. for _, key := range []string{"ipv4.routes", "ipv6.routes", "ipv4.routes.external", "ipv6.routes.external"} { for _, cidr := range util.SplitNTrimSpace(devConfig[key], ",", -1, true) { _, ipNet, _ := net.ParseCIDR(cidr) if ipNet == nil { // Skip if NIC device doesn't have a valid route. continue } externalRoutes = append(externalRoutes, externalSubnetUsage{ subnet: *ipNet, networkProject: instNetworkProject, networkName: devConfig["network"], instanceProject: inst.Project, instanceName: inst.Name, instanceDevice: devName, usageType: subnetUsageInstance, }) } } } return nil }) }) if err != nil { return nil, err } return externalRoutes, nil } // getExternalSubnetInUse returns information about usage of external subnets by bridge networks (and NICs // connected to them) on this member. func (n *bridge) getExternalSubnetInUse() ([]externalSubnetUsage, error) { var err error var projectNetworks map[string]map[int64]api.Network var projectNetworksForwardsOnUplink map[string]map[int64][]string var externalSubnets []externalSubnetUsage err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get all managed networks across all projects. projectNetworks, err = tx.GetCreatedNetworks(ctx) if err != nil { return fmt.Errorf("Failed to load all networks: %w", err) } // Get all network forward listen addresses for forwards assigned to this specific cluster member. networksByProjects, err := tx.GetNetworksAllProjects(ctx) if err != nil { return fmt.Errorf("Failed loading network forward listen addresses: %w", err) } for projectName, networks := range networksByProjects { for _, networkName := range networks { networkID, err := tx.GetNetworkID(ctx, projectName, networkName) if err != nil { return fmt.Errorf("Failed loading network forward listen addresses: %w", err) } // Get all network forward listen addresses for all networks (of any type) connected to our uplink. networkForwards, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, }) if err != nil { return fmt.Errorf("Failed loading network forward listen addresses: %w", err) } projectNetworksForwardsOnUplink = make(map[string]map[int64][]string) for _, forward := range networkForwards { // Filter network forwards that belong to this specific cluster member if forward.NodeID.Valid && (forward.NodeID.Int64 == tx.GetNodeID()) { if projectNetworksForwardsOnUplink[projectName] == nil { projectNetworksForwardsOnUplink[projectName] = make(map[int64][]string) } projectNetworksForwardsOnUplink[projectName][networkID] = append(projectNetworksForwardsOnUplink[projectName][networkID], forward.ListenAddress) } } } } externalSubnets, err = n.common.getExternalSubnetInUse(ctx, tx, n.name, true) if err != nil { return fmt.Errorf("Failed getting external subnets in use: %w", err) } return nil }) if err != nil { return nil, err } // Get managed bridge networks. bridgeProjectNetworks := n.bridgeProjectNetworks(projectNetworks) // Get external subnets used by other managed bridge networks. bridgeNetworkExternalSubnets, err := n.bridgeNetworkExternalSubnets(bridgeProjectNetworks) if err != nil { return nil, err } // Get external routes configured on bridged NICs. bridgedNICExternalRoutes, err := n.bridgedNICExternalRoutes(bridgeProjectNetworks) if err != nil { return nil, err } externalSubnets = append(externalSubnets, bridgeNetworkExternalSubnets...) externalSubnets = append(externalSubnets, bridgedNICExternalRoutes...) err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Detect if there are any conflicting proxy devices on all instances with the to be created network forward return tx.InstanceList(ctx, func(inst db.InstanceArgs, p api.Project) error { devices := db.ExpandInstanceDevices(inst.Devices, inst.Profiles) for devName, devConfig := range devices { if devConfig["type"] != "proxy" { continue } proxyListenAddr, err := ProxyParseAddr(devConfig["listen"]) if err != nil { return err } proxySubnet, err := ParseIPToNet(proxyListenAddr.Address) if err != nil { continue // If proxy listen isn't a valid IP it can't conflict. } externalSubnets = append(externalSubnets, externalSubnetUsage{ usageType: subnetUsageProxy, subnet: *proxySubnet, instanceProject: inst.Project, instanceName: inst.Name, instanceDevice: devName, }) } return nil }) }) if err != nil { return nil, err } // Add forward listen addresses to this list. for projectName, networks := range projectNetworksForwardsOnUplink { for networkID, listenAddresses := range networks { for _, listenAddress := range listenAddresses { // Convert listen address to subnet. listenAddressNet, err := ParseIPToNet(listenAddress) if err != nil { return nil, fmt.Errorf("Invalid existing forward listen address %q", listenAddress) } // Create an externalSubnetUsage for the listen address by using the network ID // of the listen address to retrieve the already loaded network name from the // projectNetworks map. externalSubnets = append(externalSubnets, externalSubnetUsage{ subnet: *listenAddressNet, networkProject: projectName, networkName: projectNetworks[projectName][networkID].Name, usageType: subnetUsageNetworkForward, }) } } } return externalSubnets, nil } // ForwardCreate creates a network forward. func (n *bridge) ForwardCreate(forward api.NetworkForwardsPost, clientType request.ClientType) error { memberSpecific := true // bridge supports per-member forwards. err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Check if there is an existing forward using the same listen address. networkID := n.ID() dbRecords, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, ListenAddress: &forward.ListenAddress, }) if err != nil { return err } filteredRecords := make([]dbCluster.NetworkForward, 0, len(dbRecords)) for _, dbRecord := range dbRecords { // bridge supports per-member forwards so do memberSpecific filtering if !dbRecord.NodeID.Valid || (dbRecord.NodeID.Int64 == tx.GetNodeID()) { filteredRecords = append(filteredRecords, dbRecord) } } if len(filteredRecords) == 0 { return api.StatusErrorf(http.StatusNotFound, "Network forward not found") } if len(filteredRecords) > 1 { return api.StatusErrorf(http.StatusConflict, "Network forward found on more than one cluster member. Please target a specific member") } _, err = filteredRecords[0].ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err == nil { return api.StatusErrorf(http.StatusConflict, "A forward for that listen address already exists") } // Convert listen address to subnet so we can check its valid and can be used. listenAddressNet, err := ParseIPToNet(forward.ListenAddress) if err != nil { return fmt.Errorf("Failed parsing address forward listen address %q: %w", forward.ListenAddress, err) } _, err = n.forwardValidate(listenAddressNet.IP, &forward.NetworkForwardPut) if err != nil { return err } externalSubnetsInUse, err := n.getExternalSubnetInUse() if err != nil { return err } // Check the listen address subnet doesn't fall within any existing network external subnets. for _, externalSubnetUser := range externalSubnetsInUse { // Check if usage is from our own network. if externalSubnetUser.networkProject == n.project && externalSubnetUser.networkName == n.name { // Skip checking conflict with our own network's subnet or SNAT address. // But do not allow other conflict with other usage types within our own network. if externalSubnetUser.usageType == subnetUsageNetwork || externalSubnetUser.usageType == subnetUsageNetworkSNAT { continue } } if SubnetContains(&externalSubnetUser.subnet, listenAddressNet) || SubnetContains(listenAddressNet, &externalSubnetUser.subnet) { // This error is purposefully vague so that it doesn't reveal any names of // resources potentially outside of the network. return fmt.Errorf("Forward listen address %q overlaps with another network or NIC", listenAddressNet.String()) } } reverter := revert.New() defer reverter.Fail() var forwardID int64 err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Create forward DB record. nodeID := sql.NullInt64{ Valid: memberSpecific, Int64: tx.GetNodeID(), } dbRecord := dbCluster.NetworkForward{ NetworkID: n.ID(), NodeID: nodeID, ListenAddress: forward.ListenAddress, Description: forward.Description, Ports: forward.Ports, } if forward.Ports == nil { dbRecord.Ports = []api.NetworkForwardPort{} } forwardID, err = dbCluster.CreateNetworkForward(ctx, tx.Tx(), dbRecord) if err != nil { return err } err = dbCluster.CreateNetworkForwardConfig(ctx, tx.Tx(), forwardID, forward.Config) if err != nil { return err } return nil }) if err != nil { return err } reverter.Add(func() { _ = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return dbCluster.DeleteNetworkForward(ctx, tx.Tx(), n.ID(), forwardID) }) _ = n.forwardSetupFirewall() _ = n.forwardBGPSetupPrefixes() }) err = n.forwardSetupFirewall() if err != nil { return err } // Check if hairpin mode needs to be enabled on active NIC bridge ports. if n.config["bridge.driver"] != "openvswitch" { brNetfilterEnabled := false for _, ipVersion := range []uint{4, 6} { if BridgeNetfilterEnabled(ipVersion) == nil { brNetfilterEnabled = true break } } // If br_netfilter is enabled and bridge has forwards, we enable hairpin mode on each NIC's bridge // port in case any of the forwards target the NIC and the instance attempts to connect to the // forward's listener. Without hairpin mode on the target of the forward will not be able to // connect to the listener. if brNetfilterEnabled { var listenAddresses map[int64]string err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() dbRecords, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, }) if err != nil { return err } listenAddresses = make(map[int64]string) for _, dbRecord := range dbRecords { if !dbRecord.NodeID.Valid || (dbRecord.NodeID.Int64 == tx.GetNodeID()) { listenAddresses[dbRecord.ID] = dbRecord.ListenAddress } } return err }) if err != nil { return fmt.Errorf("Failed loading network forwards: %w", err) } // If we are the first forward on this bridge, enable hairpin mode on active NIC ports. if len(listenAddresses) <= 1 { filter := dbCluster.InstanceFilter{Node: &n.state.ServerName} err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.InstanceList(ctx, func(inst db.InstanceArgs, p api.Project) error { // Get the instance's effective network project name. instNetworkProject := project.NetworkProjectFromRecord(&p) if instNetworkProject != api.ProjectDefaultName { return nil // Managed bridge networks can only exist in default project. } devices := db.ExpandInstanceDevices(inst.Devices.Clone(), inst.Profiles) // Iterate through each of the instance's devices, looking for bridged NICs // that are linked to this network. for devName, devConfig := range devices { if devConfig["type"] != "nic" { continue } // Check whether the NIC device references our network.. if !NICUsesNetwork(devConfig, &api.Network{Name: n.Name()}) { continue } hostName := inst.Config[fmt.Sprintf("volatile.%s.host_name", devName)] if InterfaceExists(hostName) { link := &ip.Link{Name: hostName} err = link.BridgeLinkSetHairpin(true) if err != nil { return fmt.Errorf("Error enabling hairpin mode on bridge port %q: %w", link.Name, err) } n.logger.Debug("Enabled hairpin mode on NIC bridge port", logger.Ctx{"inst": inst.Name, "project": inst.Project, "device": devName, "dev": link.Name}) } } return nil }, filter) }) if err != nil { return err } } } } // Refresh exported BGP prefixes on local member. err = n.forwardBGPSetupPrefixes() if err != nil { return fmt.Errorf("Failed applying BGP prefixes for address forwards: %w", err) } reverter.Success() return nil } // ForwardUpdate updates a network forward. func (n *bridge) ForwardUpdate(listenAddress string, req api.NetworkForwardPut, clientType request.ClientType) error { var curForwardID int64 var curForward *api.NetworkForward var curNodeID sql.NullInt64 err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error networkID := n.ID() dbRecords, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, ListenAddress: &listenAddress, }) if err != nil { return err } filteredRecords := make([]dbCluster.NetworkForward, 0, len(dbRecords)) for _, dbRecord := range dbRecords { // bridge supports per-member forwards so do memberSpecific filtering if !dbRecord.NodeID.Valid || (dbRecord.NodeID.Int64 == tx.GetNodeID()) { filteredRecords = append(filteredRecords, dbRecord) } } if len(filteredRecords) == 0 { return api.StatusErrorf(http.StatusNotFound, "Network forward not found") } if len(filteredRecords) > 1 { return api.StatusErrorf(http.StatusConflict, "Network forward found on more than one cluster member. Please target a specific member") } curForwardID = filteredRecords[0].ID curNodeID = filteredRecords[0].NodeID curForward, err = filteredRecords[0].ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return err } _, err = n.forwardValidate(net.ParseIP(curForward.ListenAddress), &req) if err != nil { return err } curForwardEtagHash, err := localUtil.EtagHash(curForward.Etag()) if err != nil { return err } newForward := api.NetworkForward{ ListenAddress: curForward.ListenAddress, NetworkForwardPut: req, } newForwardEtagHash, err := localUtil.EtagHash(newForward.Etag()) if err != nil { return err } if curForwardEtagHash == newForwardEtagHash { return nil // Nothing has changed. } reverter := revert.New() defer reverter.Fail() err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { fwd := dbCluster.NetworkForward{ NetworkID: n.ID(), NodeID: curNodeID, ListenAddress: listenAddress, Description: newForward.Description, Ports: newForward.Ports, } err = dbCluster.UpdateNetworkForward(ctx, tx.Tx(), n.ID(), listenAddress, fwd) if err != nil { return err } err = dbCluster.UpdateNetworkForwardConfig(ctx, tx.Tx(), curForwardID, newForward.Config) if err != nil { return err } return nil }) if err != nil { return err } reverter.Add(func() { _ = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { fwd := dbCluster.NetworkForward{ NetworkID: n.ID(), NodeID: curNodeID, ListenAddress: listenAddress, Description: curForward.Description, Ports: curForward.Ports, } err = dbCluster.UpdateNetworkForward(ctx, tx.Tx(), n.ID(), listenAddress, fwd) if err != nil { return err } err = dbCluster.UpdateNetworkForwardConfig(ctx, tx.Tx(), curForwardID, curForward.Config) if err != nil { return err } return nil }) _ = n.forwardSetupFirewall() _ = n.forwardBGPSetupPrefixes() }) err = n.forwardSetupFirewall() if err != nil { return err } reverter.Success() return nil } // ForwardDelete deletes a network forward. func (n *bridge) ForwardDelete(listenAddress string, clientType request.ClientType) error { memberSpecific := true // bridge supports per-member forwards. var forwardID int64 var forward *api.NetworkForward err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error networkID := n.ID() dbRecords, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, ListenAddress: &listenAddress, }) if err != nil { return err } filteredRecords := make([]dbCluster.NetworkForward, 0, len(dbRecords)) for _, dbRecord := range dbRecords { // bridge supports per-member forwards so do memberSpecific filtering if !dbRecord.NodeID.Valid || (dbRecord.NodeID.Int64 == tx.GetNodeID()) { filteredRecords = append(filteredRecords, dbRecord) } } if len(filteredRecords) == 0 { return api.StatusErrorf(http.StatusNotFound, "Network forward not found") } if len(filteredRecords) > 1 { return api.StatusErrorf(http.StatusConflict, "Network forward found on more than one cluster member. Please target a specific member") } forwardID = filteredRecords[0].ID forward, err = filteredRecords[0].ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return err } reverter := revert.New() defer reverter.Fail() err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return dbCluster.DeleteNetworkForward(ctx, tx.Tx(), n.ID(), forwardID) }) if err != nil { return err } reverter.Add(func() { _ = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { nodeID := sql.NullInt64{ Valid: memberSpecific, Int64: tx.GetNodeID(), } dbRecord := dbCluster.NetworkForward{ NetworkID: n.ID(), NodeID: nodeID, ListenAddress: forward.ListenAddress, Description: forward.Description, Ports: forward.Ports, } if forward.Ports == nil { dbRecord.Ports = []api.NetworkForwardPort{} } forwardID, err = dbCluster.CreateNetworkForward(ctx, tx.Tx(), dbRecord) if err != nil { return err } err = dbCluster.CreateNetworkForwardConfig(ctx, tx.Tx(), forwardID, forward.Config) if err != nil { return err } return nil }) _ = n.forwardSetupFirewall() _ = n.forwardBGPSetupPrefixes() }) err = n.forwardSetupFirewall() if err != nil { return err } // Refresh exported BGP prefixes on local member. err = n.forwardBGPSetupPrefixes() if err != nil { return fmt.Errorf("Failed applying BGP prefixes for address forwards: %w", err) } reverter.Success() return nil } // forwardSetupFirewall applies all network address forwards defined for this network and this member. func (n *bridge) forwardSetupFirewall() error { var forwards map[int64]*api.NetworkForward err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error networkID := n.ID() dbRecords, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, }) if err != nil { return err } forwards = make(map[int64]*api.NetworkForward) for _, dbRecord := range dbRecords { if !dbRecord.NodeID.Valid || (dbRecord.NodeID.Int64 == tx.GetNodeID()) { forward, err := dbRecord.ToAPI(ctx, tx.Tx()) if err != nil { return err } forwards[dbRecord.ID] = forward } } return err }) if err != nil { return fmt.Errorf("Failed loading network forwards: %w", err) } var fwForwards []firewallDrivers.AddressForward ipVersions := make(map[uint]struct{}) for _, forward := range forwards { // Convert listen address to subnet so we can check its valid and can be used. listenAddressNet, err := ParseIPToNet(forward.ListenAddress) if err != nil { return fmt.Errorf("Failed parsing address forward listen address %q: %w", forward.ListenAddress, err) } // Track which IP versions we are using. if listenAddressNet.IP.To4() == nil { ipVersions[6] = struct{}{} } else { ipVersions[4] = struct{}{} } portMaps, err := n.forwardValidate(listenAddressNet.IP, &forward.NetworkForwardPut) if err != nil { return fmt.Errorf("Failed validating firewall address forward for listen address %q: %w", forward.ListenAddress, err) } fwForwards = append(fwForwards, n.forwardConvertToFirewallForwards(listenAddressNet.IP, net.ParseIP(forward.Config["target_address"]), portMaps)...) } if len(forwards) > 0 { // Check if br_netfilter is enabled to, and warn if not. brNetfilterWarning := false for ipVersion := range ipVersions { err = BridgeNetfilterEnabled(ipVersion) if err != nil { brNetfilterWarning = true msg := fmt.Sprintf("IPv%d bridge netfilter not enabled. Instances using the bridge will not be able to connect to the forward listen IPs", ipVersion) n.logger.Warn(msg, logger.Ctx{"err": err}) err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpsertWarningLocalNode(ctx, n.project, dbCluster.TypeNetwork, int(n.id), warningtype.ProxyBridgeNetfilterNotEnabled, fmt.Sprintf("%s: %v", msg, err)) }) if err != nil { n.logger.Warn("Failed to create warning", logger.Ctx{"err": err}) } } } if !brNetfilterWarning { err = warnings.ResolveWarningsByLocalNodeAndProjectAndTypeAndEntity(n.state.DB.Cluster, n.project, warningtype.ProxyBridgeNetfilterNotEnabled, dbCluster.TypeNetwork, int(n.id)) if err != nil { n.logger.Warn("Failed to resolve warning", logger.Ctx{"err": err}) } } } err = n.state.Firewall.NetworkApplyForwards(n.name, fwForwards) if err != nil { return fmt.Errorf("Failed applying firewall address forwards: %w", err) } return nil } // Leases returns a list of leases for the bridged network. It will reach out to other cluster members as needed. // The projectName passed here refers to the initial project from the API request which may differ from the network's project. func (n *bridge) Leases(projectName string, clientType request.ClientType) ([]api.NetworkLease, error) { var err error var projectMacs []string leases := []api.NetworkLease{} // Get all static leases. if clientType == request.ClientTypeNormal { // If requested project matches network's project then include gateway and downstream uplink IPs. if projectName == n.project { // Add our own gateway IPs. for _, addr := range []string{n.config["ipv4.address"], n.config["ipv6.address"]} { ip, _, _ := net.ParseCIDR(addr) if ip != nil { leases = append(leases, api.NetworkLease{ Hostname: fmt.Sprintf("%s.gw", n.Name()), Address: ip.String(), Type: "gateway", }) } } // Include downstream OVN routers using the network as an uplink. var projectNetworks map[string]map[int64]api.Network err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { projectNetworks, err = tx.GetCreatedNetworks(ctx) return err }) if err != nil { return nil, err } // Look for networks using the current network as an uplink. for projectName, networks := range projectNetworks { for _, network := range networks { if network.Config["network"] != n.name { continue } // Found a network, add leases. for _, k := range []string{"volatile.network.ipv4.address", "volatile.network.ipv6.address"} { v := network.Config[k] if v != "" { leases = append(leases, api.NetworkLease{ Hostname: fmt.Sprintf("%s-%s.uplink", projectName, network.Name), Address: v, Type: "uplink", }) } } } } } // Get all the instances in the requested project that are connected to this network. filter := dbCluster.InstanceFilter{Project: &projectName} err = UsedByInstanceDevices(n.state, n.Project(), n.Name(), n.Type(), func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error { // Fill in the hwaddr from volatile. if nicConfig["hwaddr"] == "" { nicConfig["hwaddr"] = inst.Config[fmt.Sprintf("volatile.%s.hwaddr", nicName)] } // Record the MAC. hwAddr, _ := net.ParseMAC(nicConfig["hwaddr"]) if hwAddr != nil { projectMacs = append(projectMacs, hwAddr.String()) } // Add the lease. nicIP4 := net.ParseIP(nicConfig["ipv4.address"]) if nicIP4 != nil { leases = append(leases, api.NetworkLease{ Hostname: inst.Name, Address: nicIP4.String(), Hwaddr: hwAddr.String(), Type: "static", Location: inst.Node, }) } nicIP6 := net.ParseIP(nicConfig["ipv6.address"]) if nicIP6 != nil { leases = append(leases, api.NetworkLease{ Hostname: inst.Name, Address: nicIP6.String(), Hwaddr: hwAddr.String(), Type: "static", Location: inst.Node, }) } // Add EUI64 records. _, netIP6, _ := net.ParseCIDR(n.config["ipv6.address"]) if netIP6 != nil && hwAddr != nil && util.IsFalseOrEmpty(n.config["ipv6.dhcp.stateful"]) { eui64IP6, err := eui64.ParseMAC(netIP6.IP, hwAddr) if err == nil { leases = append(leases, api.NetworkLease{ Hostname: inst.Name, Address: eui64IP6.String(), Hwaddr: hwAddr.String(), Type: "dynamic", Location: inst.Node, }) } } return nil }, filter) if err != nil { return nil, err } } // Get dynamic leases. leaseFile := internalUtil.VarPath("networks", n.name, "dnsmasq.leases") if !util.PathExists(leaseFile) { return leases, nil } content, err := os.ReadFile(leaseFile) if err != nil { return nil, err } for _, lease := range strings.Split(string(content), "\n") { fields := strings.Fields(lease) if len(fields) >= 5 { // Parse the MAC. mac := GetMACSlice(fields[1]) macStr := strings.Join(mac, ":") if len(macStr) < 17 && fields[4] != "" { macStr = fields[4][len(fields[4])-17:] } // Look for an existing static entry. found := false for _, entry := range leases { if entry.Hwaddr == macStr && entry.Address == fields[2] { found = true break } } if found { continue } // DHCPv6 leases can't be tracked down to a MAC so clear the field. // This means that instance project filtering will not work on IPv6 leases. if strings.Contains(fields[2], ":") { macStr = "" } // Skip leases that don't match any of the instance MACs from the project (only when we // have populated the projectMacs list in ClientTypeNormal mode). Otherwise get all local // leases and they will be filtered on the server handling the end user request. if clientType == request.ClientTypeNormal && macStr != "" && !slices.Contains(projectMacs, macStr) { continue } // Add the lease to the list. leases = append(leases, api.NetworkLease{ Hostname: fields[3], Address: fields[2], Hwaddr: macStr, Type: "dynamic", Location: n.state.ServerName, }) } } // Collect leases from other servers. if clientType == request.ClientTypeNormal { notifier, err := cluster.NewNotifier(n.state, n.state.Endpoints.NetworkCert(), n.state.ServerCert(), cluster.NotifyAll) if err != nil { return nil, err } err = notifier(func(client incus.InstanceServer) error { memberLeases, err := client.GetNetworkLeases(n.name) if err != nil { return err } // Add local leases from other members, filtering them for MACs that belong to the project. for _, lease := range memberLeases { if lease.Hwaddr != "" && slices.Contains(projectMacs, lease.Hwaddr) { leases = append(leases, lease) } } return nil }) if err != nil { return nil, err } } return leases, nil } // UsesDNSMasq indicates if network's config indicates if it needs to use dnsmasq. func (n *bridge) UsesDNSMasq() bool { // Skip dnsmasq when no connectivity is configured. if util.IsNoneOrEmpty(n.config["ipv4.address"]) && util.IsNoneOrEmpty(n.config["ipv6.address"]) { return false } // Start dnsmasq if providing instance DNS records. if n.config["dns.mode"] != "none" { return true } // Start dnsmassq if IPv6 is used (needed for SLAAC or DHCPv6). if !util.IsNoneOrEmpty(n.config["ipv6.address"]) && (util.IsTrueOrEmpty(n.config["ipv6.dhcp"]) || util.IsTrueOrEmpty(n.config["ipv6.routing"])) { ipAddress, _, err := net.ParseCIDR(n.config["ipv6.address"]) if err != nil { return true } // Only require dnsmasq if using a global address. if !ipAddress.IsLinkLocalUnicast() { return true } } // Start dnsmasq if IPv4 DHCP is used. if !util.IsNoneOrEmpty(n.config["ipv4.address"]) && n.hasDHCPv4() { return true } return false } func (n *bridge) deleteChildren() error { // Get a list of interfaces ifaces, err := net.Interfaces() if err != nil { return err } var externalInterfaces []string if n.config["bridge.external_interfaces"] != "" { for _, entry := range strings.Split(n.config["bridge.external_interfaces"], ",") { entry = strings.Split(strings.TrimSpace(entry), "/")[0] externalInterfaces = append(externalInterfaces, entry) } } kinds := []string{ "vxlan", "gretap", "dummy", } for _, iface := range ifaces { l, err := ip.LinkByName(iface.Name) if err != nil { // If we can't load the link, chances are the interface isn't one that we should be deleting. continue } if l.Master != n.name || slices.Contains(externalInterfaces, iface.Name) || !slices.Contains(kinds, l.Kind) { continue } err = l.Delete() if err != nil { return err } } return nil } incus-7.0.0/internal/server/network/driver_common.go000066400000000000000000001463061517523235500226660ustar00rootroot00000000000000package network import ( "bytes" "context" "errors" "fmt" "maps" "math/rand" "net" "os" "slices" "strconv" "strings" "unicode" incus "github.com/lxc/incus/v7/client" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/iprange" "github.com/lxc/incus/v7/internal/server/bgp" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/network/acl" "github.com/lxc/incus/v7/internal/server/state" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // Info represents information about a network driver. type Info struct { Projects bool // Indicates if driver can be used in network enabled projects. NodeSpecificConfig bool // Whether driver has cluster node specific config as a prerequisite for creation. AddressForwards bool // Indicates if driver supports address forwards. LoadBalancers bool // Indicates if driver supports load balancers. Peering bool // Indicates if the driver supports network peering. } // forwardTarget represents a single port forward target. type forwardTarget struct { address net.IP ports []uint64 } // forwardPortMap represents a mapping of listen port(s) to target port(s) for a protocol/target address pair. type forwardPortMap struct { listenPorts []uint64 protocol string target forwardTarget snat bool } type loadBalancerPortMap struct { listenPorts []uint64 protocol string targets []forwardTarget } // subnetUsageType indicates the type of use for a subnet. type subnetUsageType uint const ( subnetUsageNetwork subnetUsageType = iota subnetUsageNetworkSNAT subnetUsageNetworkForward subnetUsageNetworkLoadBalancer subnetUsageInstance subnetUsageProxy ) // externalSubnetUsage represents usage of a subnet by a network or NIC. type externalSubnetUsage struct { subnet net.IPNet usageType subnetUsageType networkProject string networkName string instanceProject string instanceName string instanceDevice string } // common represents a generic network. type common struct { logger logger.Logger state *state.State id int64 project string name string netType string description string config map[string]string status string managed bool nodes map[int64]db.NetworkNode } // init initialize internal variables. func (n *common) init(s *state.State, id int64, projectName string, netInfo *api.Network, netNodes map[int64]db.NetworkNode) error { n.logger = logger.AddContext(logger.Ctx{"project": projectName, "driver": netInfo.Type, "network": netInfo.Name}) n.id = id n.project = projectName n.name = netInfo.Name n.netType = netInfo.Type n.config = netInfo.Config n.state = s n.description = netInfo.Description n.status = netInfo.Status n.managed = netInfo.Managed n.nodes = netNodes return nil } // FillConfig fills requested config with any default values, by default this is a no-op. func (n *common) FillConfig(config map[string]string) error { return nil } // validationRules returns a map of config rules common to all drivers. func (n *common) validationRules() map[string]func(string) error { return map[string]func(string) error{} } // validate a network config against common rules and optional driver specific rules. func (n *common) validate(config map[string]string, driverRules map[string]func(value string) error) error { checkedFields := map[string]struct{}{} // Get rules common for all drivers. rules := n.validationRules() // Merge driver specific rules into common rules. maps.Copy(rules, driverRules) // Run the validator against each field. for k, validator := range rules { checkedFields[k] = struct{}{} // Mark field as checked. err := validator(config[k]) if err != nil { return fmt.Errorf("Invalid value for network %q option %q: %w", n.name, k, err) } } // Look for any unchecked fields, as these are unknown fields and validation should fail. for k := range config { _, checked := checkedFields[k] if checked { continue } // User keys are not validated. if internalInstance.IsUserConfig(k) { continue } return fmt.Errorf("Invalid option for network %q option %q", n.name, k) } return nil } // validateZoneNames checks the DNS zone names are valid in config. func (n *common) validateZoneNames(config map[string]string) error { // Check if DNS zones in use. if config["dns.zone.forward"] == "" && config["dns.zone.reverse.ipv4"] == "" && config["dns.zone.reverse.ipv6"] == "" { return nil } var err error var zones []dbCluster.NetworkZone zoneProjects := make(map[string]string) err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { zones, err = dbCluster.GetNetworkZones(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed to load all network zones: %w", err) } for _, zone := range zones { zoneProjects[zone.Name] = zone.Project } return nil }) if err != nil { return err } for _, keyName := range []string{"dns.zone.forward", "dns.zone.reverse.ipv4", "dns.zone.reverse.ipv6"} { keyZoneNames := util.SplitNTrimSpace(config[keyName], ",", -1, true) keyZoneNamesLen := len(keyZoneNames) if keyZoneNamesLen < 1 { continue } else if keyZoneNamesLen > 1 && (keyName == "dns.zone.reverse.ipv4" || keyName == "dns.zone.reverse.ipv6") { return fmt.Errorf("Invalid %q must contain only single DNS zone name", keyName) } zoneProjectsUsed := make(map[string]struct{}) for _, keyZoneName := range keyZoneNames { zoneProjectName, found := zoneProjects[keyZoneName] if !found { return fmt.Errorf("Invalid %q, network zone %q not found", keyName, keyZoneName) } _, zoneProjectUsed := zoneProjectsUsed[zoneProjectName] if zoneProjectUsed { return fmt.Errorf("Invalid %q, contains multiple zones from the same project", keyName) } zoneProjectsUsed[zoneProjectName] = struct{}{} } } return nil } // ValidateName validates network name. func (n *common) ValidateName(name string) error { if strings.Contains(name, ":") { return fmt.Errorf("Cannot contain %q", ":") } return nil } // ID returns the network ID. func (n *common) ID() int64 { return n.id } // Name returns the network name. func (n *common) Name() string { return n.name } // Type returns the network type. func (n *common) Type() string { return n.netType } // Project returns the network project. func (n *common) Project() string { return n.project } // Description returns the network description. func (n *common) Description() string { return n.description } // Status returns the network status. func (n *common) Status() string { return n.status } // LocalStatus returns network status of the local cluster member. func (n *common) LocalStatus() string { // Check if network is unavailable locally and replace status if so. if !IsAvailable(n.Project(), n.Name()) { return api.NetworkStatusUnavailable } node, exists := n.nodes[n.state.DB.Cluster.GetNodeID()] if !exists { return api.NetworkStatusUnknown } return db.NetworkStateToAPIStatus(node.State) } // Config returns the network config. func (n *common) Config() map[string]string { return n.config } func (n *common) IsManaged() bool { return n.managed } // Config returns the common network driver info. func (n *common) Info() Info { return Info{ Projects: false, NodeSpecificConfig: true, AddressForwards: false, LoadBalancers: false, } } // Locations returns the list of cluster members this network is configured on. func (n *common) Locations() []string { locations := make([]string, 0, len(n.nodes)) for _, netNode := range n.nodes { locations = append(locations, netNode.Name) } return locations } // IsUsed returns whether the network is in use by instances or by downstream networks. func (n *common) IsUsed(instanceOnly bool) (bool, error) { if instanceOnly { usedBy := 0 err := UsedByInstanceDevices(n.state, n.project, n.name, n.netType, func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error { usedBy++ return nil }) if err != nil { return false, err } return usedBy > 0, nil } usedBy, err := UsedBy(n.state, n.project, n.id, n.name, n.netType, true) if err != nil { return false, err } return len(usedBy) > 0, nil } // DHCPv4Subnet returns nil always. func (n *common) DHCPv4Subnet() *net.IPNet { return nil } // DHCPv6Subnet returns nil always. func (n *common) DHCPv6Subnet() *net.IPNet { return nil } // DHCPv4Ranges returns a parsed set of DHCPv4 ranges for this network. func (n *common) DHCPv4Ranges() []iprange.Range { dhcpRanges := make([]iprange.Range, 0) if n.config["ipv4.dhcp.ranges"] != "" { for _, r := range strings.Split(n.config["ipv4.dhcp.ranges"], ",") { parts := strings.SplitN(strings.TrimSpace(r), "-", 2) if len(parts) == 2 { startIP := net.ParseIP(parts[0]) endIP := net.ParseIP(parts[1]) dhcpRanges = append(dhcpRanges, iprange.Range{ Start: startIP.To4(), End: endIP.To4(), }) } } } return dhcpRanges } // DHCPv6Ranges returns a parsed set of DHCPv6 ranges for this network. func (n *common) DHCPv6Ranges() []iprange.Range { dhcpRanges := make([]iprange.Range, 0) if n.config["ipv6.dhcp.ranges"] != "" { for _, r := range strings.Split(n.config["ipv6.dhcp.ranges"], ",") { parts := strings.SplitN(strings.TrimSpace(r), "-", 2) if len(parts) == 2 { startIP := net.ParseIP(parts[0]) endIP := net.ParseIP(parts[1]) dhcpRanges = append(dhcpRanges, iprange.Range{ Start: startIP.To16(), End: endIP.To16(), }) } } } return dhcpRanges } // update the internal config variables, and if not cluster notification, notifies all nodes and updates database. func (n *common) update(applyNetwork api.NetworkPut, targetNode string, clientType request.ClientType) error { // Update internal config before database has been updated (so that if update is a notification we apply // the config being supplied and not that in the database). n.description = applyNetwork.Description n.config = applyNetwork.Config // If this update isn't coming via a cluster notification itself, then notify all nodes of change and then // update the database. if clientType != request.ClientTypeNotifier { if targetNode == "" { // Notify all other nodes to update the network if no target specified. notifier, err := cluster.NewNotifier(n.state, n.state.Endpoints.NetworkCert(), n.state.ServerCert(), cluster.NotifyAll) if err != nil { return err } sendNetwork := applyNetwork sendNetwork.Config = make(map[string]string) // Don't forward node specific keys (these will be merged in on recipient node). sendNetwork.Config = db.StripNodeSpecificNetworkConfig(applyNetwork.Config) err = notifier(func(client incus.InstanceServer) error { return client.UseProject(n.project).UpdateNetwork(n.name, sendNetwork, "") }) if err != nil { return err } } err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Update the database. return tx.UpdateNetwork(ctx, n.project, n.name, applyNetwork.Description, applyNetwork.Config) }) if err != nil { return err } } return nil } // configChanged compares supplied new config with existing config. Returns a boolean indicating if differences in // the config or description were found (and the database record needs updating), and a list of non-user config // keys that have changed, and a copy of the current internal network config that can be used to revert if needed. func (n *common) configChanged(newNetwork api.NetworkPut) (bool, []string, api.NetworkPut, error) { // Backup the current state. oldNetwork := api.NetworkPut{ Description: n.description, Config: map[string]string{}, } err := util.DeepCopy(&n.config, &oldNetwork.Config) if err != nil { return false, nil, oldNetwork, err } // Diff the configurations. changedKeys := []string{} dbUpdateNeeded := false if newNetwork.Description != n.description { dbUpdateNeeded = true } for k, v := range oldNetwork.Config { if v != newNetwork.Config[k] { dbUpdateNeeded = true // Add non-user changed key to list of changed keys. if !strings.HasPrefix(k, "user.") && !slices.Contains(changedKeys, k) { changedKeys = append(changedKeys, k) } } } for k, v := range newNetwork.Config { if v != oldNetwork.Config[k] { dbUpdateNeeded = true // Add non-user changed key to list of changed keys. if !strings.HasPrefix(k, "user.") && !slices.Contains(changedKeys, k) { changedKeys = append(changedKeys, k) } } } return dbUpdateNeeded, changedKeys, oldNetwork, nil } // rename the network directory, update database record and update internal variables. func (n *common) rename(newName string) error { // Clear new directory if exists. if util.PathExists(internalUtil.VarPath("networks", newName)) { _ = os.RemoveAll(internalUtil.VarPath("networks", newName)) } // Rename directory to new name. if util.PathExists(internalUtil.VarPath("networks", n.name)) { err := os.Rename(internalUtil.VarPath("networks", n.name), internalUtil.VarPath("networks", newName)) if err != nil { return err } } err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Rename the database entry. return tx.RenameNetwork(ctx, n.project, n.name, newName) }) if err != nil { return err } // Reinitialize internal name variable and logger context with new name. n.name = newName return nil } // warningsDelete deletes any persistent warnings for the network. func (n *common) warningsDelete() error { err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return dbCluster.DeleteWarnings(ctx, tx.Tx(), dbCluster.TypeNetwork, int(n.ID())) }) if err != nil { return fmt.Errorf("Failed deleting persistent warnings: %w", err) } return nil } // delete the network on local server. func (n *common) delete(clientType request.ClientType) error { // Delete any persistent warnings for network. err := n.warningsDelete() if err != nil { return err } // Cleanup storage. if util.PathExists(internalUtil.VarPath("networks", n.name)) { _ = os.RemoveAll(internalUtil.VarPath("networks", n.name)) } pn := ProjectNetwork{ ProjectName: n.Project(), NetworkName: n.Name(), } unavailableNetworksMu.Lock() delete(unavailableNetworks, pn) unavailableNetworksMu.Unlock() return nil } // Create is a no-op. func (n *common) Create(clientType request.ClientType) error { n.logger.Debug("Create", logger.Ctx{"clientType": clientType, "config": n.config}) return nil } // HandleHeartbeat is a no-op. func (n *common) HandleHeartbeat(heartbeatData *cluster.APIHeartbeat) error { return nil } // notifyDependentNetworks allows any dependent networks to apply changes to themselves when this network changes. func (n *common) notifyDependentNetworks(changedKeys []string) { if n.Project() != api.ProjectDefaultName { return // Only networks in the default project can be used as dependent networks. } // Get a list of projects. var err error var projectNames []string err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { projectNames, err = dbCluster.GetProjectNames(ctx, tx.Tx()) return err }) if err != nil { n.logger.Error("Failed to load projects", logger.Ctx{"err": err}) return } for _, projectName := range projectNames { var depNets []string err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get a list of managed networks in project. depNets, err = tx.GetCreatedNetworkNamesByProject(ctx, projectName) return err }) if err != nil { n.logger.Error("Failed to load networks in project", logger.Ctx{"project": projectName, "err": err}) continue // Continue to next project. } for _, depName := range depNets { depNet, err := LoadByName(n.state, projectName, depName) if err != nil { n.logger.Error("Failed to load dependent network", logger.Ctx{"project": projectName, "dependentNetwork": depName, "err": err}) continue // Continue to next network. } if depNet.Config()["network"] != n.Name() { continue // Skip network, as does not depend on our network. } err = depNet.handleDependencyChange(n.Name(), n.Config(), changedKeys) if err != nil { n.logger.Error("Failed notifying dependent network", logger.Ctx{"project": projectName, "dependentNetwork": depName, "err": err}) continue // Continue to next network. } } } } // handleDependencyChange is a placeholder for networks that don't need to handle changes from dependent networks. func (n *common) handleDependencyChange(netName string, netConfig map[string]string, changedKeys []string) error { return nil } // bgpValidate. func (n *common) bgpValidationRules(config map[string]string) (map[string]func(value string) error, error) { rules := map[string]func(value string) error{} for k := range config { // BGP keys have the peer name in their name, extract the suffix. if !strings.HasPrefix(k, "bgp.peers.") { continue } // Validate remote name in key. fields := strings.Split(k, ".") if len(fields) != 4 { return nil, fmt.Errorf("Invalid network configuration key: %q", k) } bgpKey := fields[3] // Add the correct validation rule for the dynamic field based on last part of key. switch bgpKey { case "address": rules[k] = validate.Optional(validate.IsNetworkAddress) case "asn": rules[k] = validate.Optional(validate.IsInRange(1, 4294967294)) case "password": rules[k] = validate.Optional(validate.IsAny) case "holdtime": rules[k] = validate.Optional(validate.IsInRange(9, 65535)) } } return rules, nil } // bgpSetup initializes BGP peers and prefixes. func (n *common) bgpSetup(oldConfig map[string]string) error { currentPeers := n.bgpGetPeers(n.config) oldPeers := n.bgpGetPeers(oldConfig) // Don't set up BGP on non-OVN networks when no peers are configured. if n.netType != "ovn" && len(currentPeers) == 0 { if len(oldPeers) > 0 { return n.bgpClear(oldConfig) } return nil } // Set up the peers. err := n.bgpSetupPeers(oldConfig) if err != nil { return fmt.Errorf("Failed setting up BGP peers: %w", err) } // Export the prefixes. err = n.bgpSetupPrefixes(oldConfig) if err != nil { return fmt.Errorf("Failed setting up BGP prefixes: %w", err) } err = n.forwardBGPSetupPrefixes() if err != nil { return fmt.Errorf("Failed applying BGP prefixes for address forwards: %w", err) } return nil } // bgpClear clears BGP peers and prefixes. func (n *common) bgpClear(config map[string]string) error { // Clear all peers. err := n.bgpClearPeers(config) if err != nil { return err } // Clear all prefixes. err = n.state.BGP.RemovePrefixByOwner(fmt.Sprintf("network_%d", n.id)) if err != nil { return err } // Clear existing address forward prefixes for network. err = n.state.BGP.RemovePrefixByOwner(fmt.Sprintf("network_%d_forward", n.id)) if err != nil { return err } return nil } // bgpClearPeers removes all BGP peers on the network. func (n *common) bgpClearPeers(config map[string]string) error { peers := n.bgpGetPeers(config) for _, peer := range peers { // Remove the peer. fields := strings.Split(peer, ",") err := n.state.BGP.RemovePeer(net.ParseIP(fields[0])) if err != nil && !errors.Is(err, bgp.ErrPeerNotFound) { return err } } return nil } // bgpSetupPeers updates the list of BGP peers. func (n *common) bgpSetupPeers(oldConfig map[string]string) error { // Setup BGP (and handled config changes). newPeers := n.bgpGetPeers(n.config) oldPeers := n.bgpGetPeers(oldConfig) // Remove old peers. for _, peer := range oldPeers { if slices.Contains(newPeers, peer) { continue } // Remove old peer. fields := strings.Split(peer, ",") err := n.state.BGP.RemovePeer(net.ParseIP(fields[0])) if err != nil { return err } } // Add new peers. for _, peer := range newPeers { if slices.Contains(oldPeers, peer) { continue } // Add new peer. fields := strings.Split(peer, ",") asn, err := strconv.ParseUint(fields[1], 10, 32) if err != nil { return err } var holdTime uint64 if fields[3] != "" { holdTime, err = strconv.ParseUint(fields[3], 10, 32) if err != nil { return err } } err = n.state.BGP.AddPeer(net.ParseIP(fields[0]), uint32(asn), fields[2], holdTime) if err != nil { return err } } return nil } // bgpNextHopAddress parses nexthop configuration and returns next hop address to use for BGP routes. // Uses first of bgp.ipv{ipVersion}.nexthop or volatile.network.ipv{ipVersion}.address or wildcard address. func (n *common) bgpNextHopAddress(ipVersion uint) net.IP { nextHopAddr := net.ParseIP(n.config[fmt.Sprintf("bgp.ipv%d.nexthop", ipVersion)]) if nextHopAddr == nil { nextHopAddr = net.ParseIP(n.config[fmt.Sprintf("volatile.network.ipv%d.address", ipVersion)]) if nextHopAddr == nil { if ipVersion == 4 { nextHopAddr = net.ParseIP("0.0.0.0") } else { nextHopAddr = net.ParseIP("::") } } } return nextHopAddr } // bgpSetupPrefixes refreshes the prefix list for the network. func (n *common) bgpSetupPrefixes(oldConfig map[string]string) error { // Clear existing prefixes. bgpOwner := fmt.Sprintf("network_%d", n.id) if oldConfig != nil { err := n.state.BGP.RemovePrefixByOwner(bgpOwner) if err != nil { return err } } // Add the new prefixes. for _, ipVersion := range []uint{4, 6} { nextHopAddr := n.bgpNextHopAddress(ipVersion) // If network has NAT enabled, then export network's NAT address if specified. if util.IsTrue(n.config[fmt.Sprintf("ipv%d.nat", ipVersion)]) { natAddressKey := fmt.Sprintf("ipv%d.nat.address", ipVersion) if n.config[natAddressKey] != "" { subnetSize := 128 if ipVersion == 4 { subnetSize = 32 } _, subnet, err := net.ParseCIDR(fmt.Sprintf("%s/%d", n.config[natAddressKey], subnetSize)) if err != nil { return err } err = n.state.BGP.AddPrefix(*subnet, nextHopAddr, bgpOwner) if err != nil { return err } } } else if !slices.Contains([]string{"", "none"}, n.config[fmt.Sprintf("ipv%d.address", ipVersion)]) { // If network has NAT disabled, then export the network's subnet if specified. netAddress := n.config[fmt.Sprintf("ipv%d.address", ipVersion)] _, subnet, err := net.ParseCIDR(netAddress) if err != nil { return fmt.Errorf("Failed parsing network address %q: %w", netAddress, err) } err = n.state.BGP.AddPrefix(*subnet, nextHopAddr, bgpOwner) if err != nil { return err } } } return nil } // bgpGetPeers returns a list of strings representing the BGP peers. func (n *common) bgpGetPeers(config map[string]string) []string { // Get a list of peer names. peerNames := []string{} for k := range config { if !strings.HasPrefix(k, "bgp.peers.") { continue } fields := strings.Split(k, ".") if !slices.Contains(peerNames, fields[2]) { peerNames = append(peerNames, fields[2]) } } // Build up a list of peer strings. peers := []string{} for _, peerName := range peerNames { peerAddress := config[fmt.Sprintf("bgp.peers.%s.address", peerName)] peerASN := config[fmt.Sprintf("bgp.peers.%s.asn", peerName)] peerPassword := config[fmt.Sprintf("bgp.peers.%s.password", peerName)] peerHoldTime := config[fmt.Sprintf("bgp.peers.%s.holdtime", peerName)] if peerAddress != "" && peerASN != "" { peers = append(peers, fmt.Sprintf("%s,%s,%s,%s", peerAddress, peerASN, peerPassword, peerHoldTime)) } } return peers } // forwardValidate validates the forward request. func (n *common) forwardValidate(listenAddress net.IP, forward *api.NetworkForwardPut) ([]*forwardPortMap, error) { if listenAddress == nil { return nil, errors.New("Invalid listen address") } if listenAddress.IsUnspecified() { return nil, fmt.Errorf("Cannot use unspecified address: %q", listenAddress.String()) } listenIsIP4 := listenAddress.To4() != nil // For checking target addresses are within network's subnet. netIPKey := "ipv4.address" if !listenIsIP4 { netIPKey = "ipv6.address" } netIPAddress := n.config[netIPKey] var err error var netSubnet *net.IPNet if netIPAddress != "" { _, netSubnet, err = net.ParseCIDR(n.config[netIPKey]) if err != nil { return nil, err } } // Look for any unknown config fields. for k := range forward.Config { if k == "target_address" { continue } // User keys are not validated. // gendoc:generate(entity=network_forward, group=common, key=user.*) // // --- // type: string // shortdesc: User defined key/value configuration if internalInstance.IsUserConfig(k) { continue } return nil, fmt.Errorf("Invalid option %q", k) } // Validate default target address. // gendoc:generate(entity=network_forward, group=common, key=target_address) // // --- // type: string // shortdesc: Default target address for anything not covered through a port definition defaultTargetAddress := net.ParseIP(forward.Config["target_address"]) if forward.Config["target_address"] != "" { if defaultTargetAddress == nil { return nil, errors.New("Invalid default target address") } defaultTargetIsIP4 := defaultTargetAddress.To4() != nil if listenIsIP4 != defaultTargetIsIP4 { return nil, errors.New("Cannot mix IP versions in listen address and default target address") } // Check default target address is within network's subnet. if netSubnet != nil && !SubnetContainsIP(netSubnet, defaultTargetAddress) { return nil, errors.New("Default target address is not within the network subnet") } if defaultTargetIsIP4 && IPisBroadcast(netSubnet, defaultTargetAddress) { return nil, errors.New("Default target address cannot be a broadcast address") } if IPisNetworkID(netSubnet, defaultTargetAddress) { return nil, errors.New("Default target address cannot be a network ID address") } } // Validate port rules. validPortProcols := []string{"tcp", "udp"} // Used to ensure that each listen port is only used once. listenPorts := map[string]map[int64]struct{}{ "tcp": make(map[int64]struct{}), "udp": make(map[int64]struct{}), } // Maps portSpecID to a portMap struct. portMaps := make([]*forwardPortMap, 0, len(forward.Ports)) for portSpecID, portSpec := range forward.Ports { if !slices.Contains(validPortProcols, portSpec.Protocol) { return nil, fmt.Errorf("Invalid port protocol in port specification %d, protocol must be one of: %s", portSpecID, strings.Join(validPortProcols, ", ")) } targetAddress := net.ParseIP(portSpec.TargetAddress) if targetAddress == nil { return nil, fmt.Errorf("Invalid target address in port specification %d", portSpecID) } if targetAddress.Equal(defaultTargetAddress) { return nil, fmt.Errorf("Target address is same as default target address in port specification %d", portSpecID) } targetIsIP4 := targetAddress.To4() != nil if listenIsIP4 != targetIsIP4 { return nil, fmt.Errorf("Cannot mix IP versions in listen address and port specification %d target address", portSpecID) } // Check target address is within network's subnet. if netSubnet != nil && !SubnetContainsIP(netSubnet, targetAddress) { return nil, fmt.Errorf("Target address is not within the network subnet in port specification %d", portSpecID) } if targetIsIP4 && IPisBroadcast(netSubnet, targetAddress) { return nil, errors.New("Default target address cannot be a broadcast address") } if IPisNetworkID(netSubnet, defaultTargetAddress) { return nil, errors.New("Default target address cannot be a network ID address") } // Check valid listen port(s) supplied. listenPortRanges := util.SplitNTrimSpace(portSpec.ListenPort, ",", -1, true) if len(listenPortRanges) <= 0 { return nil, fmt.Errorf("Missing listen port in port specification %d", portSpecID) } portMap := forwardPortMap{ listenPorts: make([]uint64, 0), target: forwardTarget{ address: targetAddress, }, protocol: portSpec.Protocol, snat: portSpec.SNAT, } for _, pr := range listenPortRanges { portFirst, portRange, err := ParsePortRange(pr) if err != nil { return nil, fmt.Errorf("Invalid listen port in port specification %d: %w", portSpecID, err) } for i := range portRange { port := portFirst + i _, found := listenPorts[portSpec.Protocol][port] if found { return nil, fmt.Errorf("Duplicate listen port %d for protocol %q in port specification %d", port, portSpec.Protocol, portSpecID) } listenPorts[portSpec.Protocol][port] = struct{}{} portMap.listenPorts = append(portMap.listenPorts, uint64(port)) } } // Check that SNAT is only used with bridges. if portSpec.SNAT && n.netType != "bridge" { return nil, errors.New("SNAT can only be used with bridge networks") } // Check valid target port(s) supplied. targetPortRanges := util.SplitNTrimSpace(portSpec.TargetPort, ",", -1, true) if len(targetPortRanges) > 0 { // Target ports can be at maximum the same length as listen ports. portMap.target.ports = make([]uint64, 0, len(portMap.listenPorts)) for _, pr := range targetPortRanges { portFirst, portRange, err := ParsePortRange(pr) if err != nil { return nil, fmt.Errorf("Invalid target port in port specification %d", portSpecID) } for i := range portRange { port := portFirst + i portMap.target.ports = append(portMap.target.ports, uint64(port)) } } // Only check if the target port count matches the listen port count if the target ports // don't equal 1, because we allow many-to-one type mapping. portSpectTargetPortsLen := len(portMap.target.ports) if portSpectTargetPortsLen != 1 && len(portMap.listenPorts) != portSpectTargetPortsLen { return nil, fmt.Errorf("Mismatch of listen port(s) and target port(s) count in port specification %d", portSpecID) } } portMaps = append(portMaps, &portMap) } return portMaps, err } // ForwardCreate returns ErrNotImplemented for drivers that do not support forwards. func (n *common) ForwardCreate(forward api.NetworkForwardsPost, clientType request.ClientType) error { return ErrNotImplemented } // ForwardUpdate returns ErrNotImplemented for drivers that do not support forwards. func (n *common) ForwardUpdate(listenAddress string, newForward api.NetworkForwardPut, clientType request.ClientType) error { return ErrNotImplemented } // ForwardDelete returns ErrNotImplemented for drivers that do not support forwards. func (n *common) ForwardDelete(listenAddress string, clientType request.ClientType) error { return ErrNotImplemented } // forwardBGPSetupPrefixes exports external forward addresses as prefixes. func (n *common) forwardBGPSetupPrefixes() error { var fwdListenAddresses map[int64]string err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Retrieve network forwards before clearing existing prefixes, and separate them by IP family. networkID := int64(n.ID()) dbRecords, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, }) if err != nil { return err } fwdListenAddresses = make(map[int64]string) for _, dbRecord := range dbRecords { // memberSpecific filtering if !dbRecord.NodeID.Valid || (dbRecord.NodeID.Int64 == tx.GetNodeID()) { // Get listen address forwardID := int64(dbRecord.ID) fwdListenAddresses[forwardID] = dbRecord.ListenAddress } } return nil }) if err != nil { return fmt.Errorf("Failed loading network forwards: %w", err) } fwdListenAddressesByFamily := map[uint][]string{ 4: make([]string, 0), 6: make([]string, 0), } for _, fwdListenAddress := range fwdListenAddresses { if strings.Contains(fwdListenAddress, ":") { fwdListenAddressesByFamily[6] = append(fwdListenAddressesByFamily[6], fwdListenAddress) } else { fwdListenAddressesByFamily[4] = append(fwdListenAddressesByFamily[4], fwdListenAddress) } } // Use forward specific owner string (different from the network prefixes) so that these can be reapplied // independently of the network's own prefixes. bgpOwner := fmt.Sprintf("network_%d_forward", n.id) // Clear existing address forward prefixes for network. err = n.state.BGP.RemovePrefixByOwner(bgpOwner) if err != nil { return err } // Add the new prefixes. for _, ipVersion := range []uint{4, 6} { nextHopAddr := n.bgpNextHopAddress(ipVersion) natEnabled := util.IsTrue(n.config[fmt.Sprintf("ipv%d.nat", ipVersion)]) _, netSubnet, _ := net.ParseCIDR(n.config[fmt.Sprintf("ipv%d.address", ipVersion)]) routeSubnetSize := 128 if ipVersion == 4 { routeSubnetSize = 32 } // Export external forward listen addresses. for _, fwdListenAddress := range fwdListenAddressesByFamily[ipVersion] { fwdListenAddr := net.ParseIP(fwdListenAddress) // Don't export internal address forwards (those inside the NAT enabled network's subnet). if natEnabled && netSubnet != nil && netSubnet.Contains(fwdListenAddr) { continue } _, ipRouteSubnet, err := net.ParseCIDR(fmt.Sprintf("%s/%d", fwdListenAddr.String(), routeSubnetSize)) if err != nil { return err } err = n.state.BGP.AddPrefix(*ipRouteSubnet, nextHopAddr, bgpOwner) if err != nil { return err } } } return nil } // getExternalSubnetInUse returns information about usage of external subnets by networks connected to, or used by, // the specified uplinkNetworkName. func (n *common) getExternalSubnetInUse(ctx context.Context, tx *db.ClusterTx, uplinkNetworkName string, memberSpecific bool) ([]externalSubnetUsage, error) { // Get a list of related networks. relatedNetworks := map[int64]*api.Network{} networksByProject, err := tx.GetNetworksAllProjects(ctx) if err != nil { return nil, fmt.Errorf("Failed loading network load balancer listen addresses: %w", err) } for projectName, networks := range networksByProject { for _, networkName := range networks { // Load the network. networkID, apiNetwork, _, err := tx.GetNetworkInAnyState(ctx, projectName, networkName) if err != nil { return nil, fmt.Errorf("Failed to load network: %w", err) } // Check if we're looking at our uplink. if projectName == api.ProjectDefaultName && uplinkNetworkName == networkName { relatedNetworks[networkID] = apiNetwork continue } // Check if the network shares the same uplink. if apiNetwork.Config["network"] == uplinkNetworkName { relatedNetworks[networkID] = apiNetwork continue } } } // Get all network load balancer and forward listen addresses for all networks (of any type) connected to our uplink. projectNetworksLoadBalancersOnUplink := map[string]map[string][]string{} projectNetworksForwardsOnUplink := map[string]map[string][]string{} for networkID, relatedNetwork := range relatedNetworks { // Get all load balancers associated with this network. loadBalancers, err := dbCluster.GetNetworkLoadBalancers(ctx, tx.Tx(), dbCluster.NetworkLoadBalancerFilter{ NetworkID: &networkID, }) if err != nil { return nil, fmt.Errorf("Failed getting list of network load balancer: %w", err) } for _, lb := range loadBalancers { if projectNetworksLoadBalancersOnUplink[relatedNetwork.Project] == nil { projectNetworksLoadBalancersOnUplink[relatedNetwork.Project] = map[string][]string{} } projectNetworksLoadBalancersOnUplink[relatedNetwork.Project][relatedNetwork.Name] = append(projectNetworksLoadBalancersOnUplink[relatedNetwork.Project][relatedNetwork.Name], lb.ListenAddress) } // Get all network forwards associated with this network. networkForwards, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, }) if err != nil { return nil, fmt.Errorf("Failed getting list of network forward: %w", err) } for _, fwd := range networkForwards { if !memberSpecific || (!fwd.NodeID.Valid || (fwd.NodeID.Int64 == tx.GetNodeID())) { if projectNetworksForwardsOnUplink[relatedNetwork.Project] == nil { projectNetworksForwardsOnUplink[relatedNetwork.Project] = map[string][]string{} } projectNetworksForwardsOnUplink[relatedNetwork.Project][relatedNetwork.Name] = append(projectNetworksForwardsOnUplink[relatedNetwork.Project][relatedNetwork.Name], fwd.ListenAddress) } } } externalSubnets := make([]externalSubnetUsage, 0, len(projectNetworksForwardsOnUplink)+len(projectNetworksLoadBalancersOnUplink)) // Add forward listen addresses to this list. for projectName, networks := range projectNetworksForwardsOnUplink { for networkName, listenAddresses := range networks { for _, listenAddress := range listenAddresses { // Convert listen address to subnet. listenAddressNet, err := ParseIPToNet(listenAddress) if err != nil { return nil, fmt.Errorf("Invalid existing forward listen address %q", listenAddress) } // Create an externalSubnetUsage for the listen address by using the network ID // of the listen address to retrieve the already loaded network name from the // projectNetworks map. externalSubnets = append(externalSubnets, externalSubnetUsage{ subnet: *listenAddressNet, networkProject: projectName, networkName: networkName, usageType: subnetUsageNetworkForward, }) } } } // Add load balancer listen addresses to this list. for projectName, networks := range projectNetworksLoadBalancersOnUplink { for networkName, listenAddresses := range networks { for _, listenAddress := range listenAddresses { // Convert listen address to subnet. listenAddressNet, err := ParseIPToNet(listenAddress) if err != nil { return nil, fmt.Errorf("Invalid existing load balancer listen address %q", listenAddress) } // Create an externalSubnetUsage for the listen address by using the network ID // of the listen address to retrieve the already loaded network name from the // projectNetworks map. externalSubnets = append(externalSubnets, externalSubnetUsage{ subnet: *listenAddressNet, networkProject: projectName, networkName: networkName, usageType: subnetUsageNetworkLoadBalancer, }) } } } return externalSubnets, nil } // loadBalancerValidate validates the load balancer request. func (n *common) loadBalancerValidate(listenAddress net.IP, forward *api.NetworkLoadBalancerPut) ([]*loadBalancerPortMap, error) { if listenAddress == nil { return nil, errors.New("Invalid listen address") } listenIsIP4 := listenAddress.To4() != nil // For checking target addresses are within network's subnet. netIPKey := "ipv4.address" if !listenIsIP4 { netIPKey = "ipv6.address" } netIPAddress := n.config[netIPKey] var err error var netSubnet *net.IPNet if netIPAddress != "" { _, netSubnet, err = net.ParseCIDR(n.config[netIPKey]) if err != nil { return nil, err } } // Check the configuration. lbOptions := map[string]func(value string) error{ // gendoc:generate(entity=network_load_balancer, group=common, key=healthcheck) // // --- // type: bool // defaultdesc: `false` // shortdesc: Whether to perform checks on the backends "healthcheck": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_load_balancer, group=common, key=healthcheck.interval) // // --- // type: integer // shortdesc: Interval in seconds between health checks // defaultdesc: `10` "healthcheck.interval": validate.IsUint32, // gendoc:generate(entity=network_load_balancer, group=common, key=healthcheck.success_count) // // --- // type: integer // shortdesc: Number of successful tests to consider the backend online // defaultdesc: `3` "healthcheck.success_count": validate.IsUint32, // gendoc:generate(entity=network_load_balancer, group=common, key=healthcheck.failure_count) // // --- // type: integer // shortdesc: Number of failed tests to consider the backend offline // defaultdesc: `3` "healthcheck.failure_count": validate.IsUint32, // gendoc:generate(entity=network_load_balancer, group=common, key=healthcheck.timeout) // // --- // type: integer // shortdesc: Test timeout // defaultdesc: `30` "healthcheck.timeout": validate.IsUint32, } for k, v := range forward.Config { // User keys are not validated. // gendoc:generate(entity=network_load_balancer, group=common, key=user.*) // User keys can be used in search. // --- // type: string // shortdesc: Free form user key/value storage if internalInstance.IsUserConfig(k) { continue } checker, ok := lbOptions[k] if ok { err := checker(v) if err != nil { return nil, err } continue } return nil, fmt.Errorf("Invalid option %q", k) } // Validate port rules. validPortProcols := []string{"tcp", "udp"} // Used to ensure that each listen port is only used once. listenPorts := map[string]map[int64]struct{}{ "tcp": make(map[int64]struct{}), "udp": make(map[int64]struct{}), } // Check backends config and store the parsed target by backend name. backendsByName := make(map[string]*forwardTarget, len(forward.Backends)) for backendSpecID, backendSpec := range forward.Backends { for _, r := range backendSpec.Name { if unicode.IsSpace(r) { return nil, fmt.Errorf("Name cannot contain white space in backend specification %d", backendSpecID) } } _, found := backendsByName[backendSpec.Name] if found { return nil, fmt.Errorf("Duplicate name %q in backend specification %d", backendSpec.Name, backendSpecID) } targetAddress := net.ParseIP(backendSpec.TargetAddress) if targetAddress == nil { return nil, fmt.Errorf("Invalid target address for backend %q", backendSpec.Name) } targetIsIP4 := targetAddress.To4() != nil if listenIsIP4 != targetIsIP4 { return nil, fmt.Errorf("Cannot mix IP versions in listen address and backend %q target address", backendSpec.Name) } // Check target address is within network's subnet. if netSubnet != nil && !SubnetContainsIP(netSubnet, targetAddress) { return nil, fmt.Errorf("Target address is not within the network subnet for backend %q", backendSpec.Name) } // Check valid target port(s) supplied. target := forwardTarget{ address: targetAddress, } for portSpecID, portSpec := range util.SplitNTrimSpace(backendSpec.TargetPort, ",", -1, true) { portFirst, portRange, err := ParsePortRange(portSpec) if err != nil { return nil, fmt.Errorf("Invalid backend port specification %d in backend specification %d: %w", portSpecID, backendSpecID, err) } for i := range portRange { port := portFirst + i target.ports = append(target.ports, uint64(port)) } } backendsByName[backendSpec.Name] = &target } // Check ports config. portMaps := make([]*loadBalancerPortMap, 0, len(forward.Ports)) for portSpecID, portSpec := range forward.Ports { if !slices.Contains(validPortProcols, portSpec.Protocol) { return nil, fmt.Errorf("Invalid port protocol in port specification %d, protocol must be one of: %s", portSpecID, strings.Join(validPortProcols, ", ")) } // Check valid listen port(s) supplied. listenPortRanges := util.SplitNTrimSpace(portSpec.ListenPort, ",", -1, true) if len(listenPortRanges) <= 0 { return nil, fmt.Errorf("Missing listen port in port specification %d", portSpecID) } portMap := loadBalancerPortMap{ listenPorts: make([]uint64, 0), protocol: portSpec.Protocol, targets: make([]forwardTarget, 0, len(portSpec.TargetBackend)), } for _, pr := range listenPortRanges { portFirst, portRange, err := ParsePortRange(pr) if err != nil { return nil, fmt.Errorf("Invalid listen port in port specification %d: %w", portSpecID, err) } for i := range portRange { port := portFirst + i _, found := listenPorts[portSpec.Protocol][port] if found { return nil, fmt.Errorf("Duplicate listen port %d for protocol %q in port specification %d", port, portSpec.Protocol, portSpecID) } listenPorts[portSpec.Protocol][port] = struct{}{} portMap.listenPorts = append(portMap.listenPorts, uint64(port)) } } // Check each of the backends specified are compatible with the listen ports. for _, backendName := range portSpec.TargetBackend { // Check backend exists. backend, found := backendsByName[backendName] if !found { return nil, fmt.Errorf("Invalid target backend name %q in port specification %d", backendName, portSpecID) } // Only check if the target port count matches the listen port count if the target ports // are greater than 1, because we allow many-to-one type mapping and one-to-one mapping if // no target ports specified. portSpectTargetPortsLen := len(backend.ports) if portSpectTargetPortsLen > 1 && len(portMap.listenPorts) != portSpectTargetPortsLen { return nil, fmt.Errorf("Mismatch of listen port(s) and target port(s) count for backend %q in port specification %d", backendName, portSpecID) } portMap.targets = append(portMap.targets, *backend) } portMaps = append(portMaps, &portMap) } return portMaps, err } // LoadBalancerCreate returns ErrNotImplemented for drivers that do not support load balancers. func (n *common) LoadBalancerCreate(loadBalancer api.NetworkLoadBalancersPost, clientType request.ClientType) error { return ErrNotImplemented } // LoadBalancerUpdate returns ErrNotImplemented for drivers that do not support load balancers.. func (n *common) LoadBalancerUpdate(listenAddress string, newLoadBalancer api.NetworkLoadBalancerPut, clientType request.ClientType) error { return ErrNotImplemented } // LoadBalancerState returns ErrNotImplemented for drivers that do not support load balancers.. func (n *common) LoadBalancerState(loadBalancer api.NetworkLoadBalancer) (*api.NetworkLoadBalancerState, error) { return nil, ErrNotImplemented } // LoadBalancerDelete returns ErrNotImplemented for drivers that do not support load balancers.. func (n *common) LoadBalancerDelete(listenAddress string, clientType request.ClientType) error { return ErrNotImplemented } // Leases returns ErrNotImplemented for drivers that don't support address leases. func (n *common) Leases(projectName string, clientType request.ClientType) ([]api.NetworkLease, error) { return nil, ErrNotImplemented } // PeerCrete returns ErrNotImplemented for drivers that do not support forwards. func (n *common) PeerCreate(forward api.NetworkPeersPost) error { return ErrNotImplemented } // PeerUpdate returns ErrNotImplemented for drivers that do not support forwards. func (n *common) PeerUpdate(peerName string, newPeer api.NetworkPeerPut) error { return ErrNotImplemented } // PeerDelete returns ErrNotImplemented for drivers that do not support forwards. func (n *common) PeerDelete(peerName string) error { return ErrNotImplemented } // peerValidate validates the peer request. func (n *common) peerValidate(peerName string, peer *api.NetworkPeerPut) error { err := acl.ValidName(peerName) if err != nil { return err } if slices.Contains(acl.ReservedNetworkSubects, peerName) { return fmt.Errorf("Name cannot be one of the reserved network subjects: %v", acl.ReservedNetworkSubects) } // Look for any unknown config fields. for k := range peer.Config { if k == "target_address" { continue } // User keys are not validated. if internalInstance.IsUserConfig(k) { continue } return fmt.Errorf("Invalid option %q", k) } return nil } // PeerUsedBy returns a list of API endpoints referencing this peer. func (n *common) PeerUsedBy(peerName string) ([]string, error) { return n.peerUsedBy(peerName, false) } // isUsed returns whether or not the peer is in use. func (n *common) peerIsUsed(peerName string) (bool, error) { usedBy, err := n.peerUsedBy(peerName, true) if err != nil { return false, err } return len(usedBy) > 0, nil } // peerUsedBy returns a list of API endpoints referencing this peer. func (n *common) peerUsedBy(peerName string, firstOnly bool) ([]string, error) { usedBy := []string{} rulesUsePeer := func(rules []api.NetworkACLRule) bool { for _, rule := range rules { for _, subject := range util.SplitNTrimSpace(rule.Source, ",", -1, true) { if !strings.HasPrefix(subject, "@") { continue } peerParts := strings.SplitN(strings.TrimPrefix(subject, "@"), "/", 2) if len(peerParts) != 2 { continue // Not a valid network/peer name combination. } peer := dbCluster.NetworkPeerConnection{ NetworkName: peerParts[0], PeerName: peerParts[1], } if peer.NetworkName == n.Name() && peer.PeerName == peerName { return true } } } return false } var aclNames []string err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { projectName := n.Project() acls, err := dbCluster.GetNetworkACLs(ctx, tx.Tx(), dbCluster.NetworkACLFilter{Project: &projectName}) if err != nil { return err } aclNames = make([]string, len(acls)) for i, acl := range acls { aclNames[i] = acl.Name } return nil }) if err != nil { return nil, err } for _, aclName := range aclNames { var aclInfo *api.NetworkACL err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, aclInfo, err = dbCluster.GetNetworkACLAPI(ctx, tx.Tx(), n.Project(), aclName) return err }) if err != nil { return nil, err } // Ingress rules can specify peer names in their Source subjects. for _, rules := range [][]api.NetworkACLRule{aclInfo.Ingress, aclInfo.Egress} { if rulesUsePeer(rules) { usedBy = append(usedBy, api.NewURL().Project(n.Project()).Path(version.APIVersion, "network-acls", aclName).String()) if firstOnly { return usedBy, err } break } } } return usedBy, nil } func (n *common) State() (*api.NetworkState, error) { if n.config["parent"] != "" { return resources.GetNetworkState(n.config["parent"]) } return resources.GetNetworkState(n.name) } func (n *common) setUnavailable() { pn := ProjectNetwork{ ProjectName: n.Project(), NetworkName: n.Name(), } unavailableNetworksMu.Lock() unavailableNetworks[pn] = struct{}{} unavailableNetworksMu.Unlock() } func (n *common) setAvailable() { pn := ProjectNetwork{ ProjectName: n.Project(), NetworkName: n.Name(), } unavailableNetworksMu.Lock() delete(unavailableNetworks, pn) unavailableNetworksMu.Unlock() } // RandomHwaddr generates a random MAC address from the provided random source. func (n *common) randomHwaddr(r *rand.Rand) string { var pattern string _ = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := dbCluster.GetProject(ctx, tx.Tx(), n.project) if err != nil { return err } projectConfig, err := dbCluster.GetProjectConfig(ctx, tx.Tx(), dbProject.ID) if err != nil { return err } pattern = projectConfig["network.hwaddr_pattern"] return nil }) if pattern == "" { pattern = n.state.GlobalConfig.NetworkHWAddrPattern() } // Generate a new random MAC address using the given pattern. ret := bytes.Buffer{} for _, c := range pattern { if c == 'x' || c == 'X' { fmt.Fprintf(&ret, "%x", r.Int31n(16)) } else { fmt.Fprint(&ret, string(c)) } } return ret.String() } incus-7.0.0/internal/server/network/driver_macvlan.go000066400000000000000000000111751517523235500230120ustar00rootroot00000000000000package network import ( "fmt" "slices" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/validate" ) // macvlan represents a macvlan network. type macvlan struct { common } // DBType returns the network type DB ID. func (n *macvlan) DBType() db.NetworkType { return db.NetworkTypeMacvlan } // Validate network config. func (n *macvlan) Validate(config map[string]string, clientType request.ClientType) error { rules := map[string]func(value string) error{ // gendoc:generate(entity=network_macvlan, group=common, key=parent) // // --- // type: string // condition: - // shortdesc: Parent interface to create macvlan NICs on "parent": validate.Required(validate.IsNotEmpty, validate.IsInterfaceName), // gendoc:generate(entity=network_macvlan, group=common, key=mtu) // // --- // type: int // condition: - // shortdesc: The MTU of the new interface "mtu": validate.Optional(validate.IsNetworkMTU), // gendoc:generate(entity=network_macvlan, group=common, key=vlan) // // --- // type: int // condition: - // shortdesc: The VLAN ID to attach to "vlan": validate.Optional(validate.IsNetworkVLAN), // gendoc:generate(entity=network_macvlan, group=common, key=gvrp) // // --- // type: bool // condition: - // default: `false` // shortdesc: Register VLAN using GARP VLAN Registration Protocol "gvrp": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_macvlan, group=common, key=user.*) // // --- // type: string // shortdesc: User-provided free-form key/value pairs } err := n.validate(config, rules) if err != nil { return err } return nil } // Delete deletes a network. func (n *macvlan) Delete(clientType request.ClientType) error { n.logger.Debug("Delete", logger.Ctx{"clientType": clientType}) return n.delete(clientType) } // Rename renames a network. func (n *macvlan) Rename(newName string) error { n.logger.Debug("Rename", logger.Ctx{"newName": newName}) // Rename common steps. err := n.rename(newName) if err != nil { return err } return nil } // Start starts the network. func (n *macvlan) Start() error { n.logger.Debug("Start") reverter := revert.New() defer reverter.Fail() reverter.Add(func() { n.setUnavailable() }) err := n.setup(n.config["parent"]) if err != nil { return err } reverter.Success() // Ensure network is marked as available now its started. n.setAvailable() return nil } // setup restarts the network. func (n *macvlan) setup(parent string) error { n.logger.Debug("Setting up network") if !InterfaceExists(parent) { return fmt.Errorf("Parent interface %q not found", parent) } // Make sure the port is up. link := &ip.Link{Name: parent} err := link.SetUp() if err != nil { return fmt.Errorf("Failed to bring up the host interface %s: %w", parent, err) } return nil } // Stop stops is a no-op. func (n *macvlan) Stop() error { n.logger.Debug("Stop") return nil } // Update updates the network. Accepts notification boolean indicating if this update request is coming from a // cluster notification, in which case do not update the database, just apply local changes needed. func (n *macvlan) Update(newNetwork api.NetworkPut, targetNode string, clientType request.ClientType) error { n.logger.Debug("Update", logger.Ctx{"clientType": clientType, "newNetwork": newNetwork}) dbUpdateNeeded, changedKeys, oldNetwork, err := n.configChanged(newNetwork) if err != nil { return err } if !dbUpdateNeeded { return nil // Nothing changed. } // If the network as a whole has not had any previous creation attempts, or the node itself is still // pending, then don't apply the new settings to the node, just to the database record (ready for the // actual global create request to be initiated). if n.Status() == api.NetworkStatusPending || n.LocalStatus() == api.NetworkStatusPending { return n.update(newNetwork, targetNode, clientType) } reverter := revert.New() defer reverter.Fail() // Define a function which reverts everything. reverter.Add(func() { // Reset changes to all nodes and database. _ = n.update(oldNetwork, targetNode, clientType) }) if slices.Contains(changedKeys, "parent") { err = n.setup(newNetwork.Config["parent"]) if err != nil { return err } } // Apply changes to all nodes and database. err = n.update(newNetwork, targetNode, clientType) if err != nil { return err } reverter.Success() return nil } incus-7.0.0/internal/server/network/driver_ovn.go000066400000000000000000010233721517523235500221760ustar00rootroot00000000000000package network import ( "bytes" "context" "database/sql" "errors" "fmt" "maps" "math/big" "net" "net/http" "net/netip" "os" "slices" "sort" "strconv" "strings" "time" "github.com/flosch/pongo2/v6" "github.com/mdlayher/netx/eui64" ovsClient "github.com/ovn-kubernetes/libovsdb/client" ovsdbModel "github.com/ovn-kubernetes/libovsdb/model" incus "github.com/lxc/incus/v7/client" "github.com/lxc/incus/v7/internal/iprange" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/dnsmasq/dhcpalloc" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/locking" "github.com/lxc/incus/v7/internal/server/network/acl" addressset "github.com/lxc/incus/v7/internal/server/network/address-set" networkOVN "github.com/lxc/incus/v7/internal/server/network/ovn" ovnNB "github.com/lxc/incus/v7/internal/server/network/ovn/schema/ovn-nb" ovnSB "github.com/lxc/incus/v7/internal/server/network/ovn/schema/ovn-sb" "github.com/lxc/incus/v7/internal/server/network/ovs" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) const ( ovnChassisPriorityMax = 32767 ovnVolatileUplinkIPv4 = "volatile.network.ipv4.address" ovnVolatileUplinkIPv6 = "volatile.network.ipv6.address" ) const ( ovnRouterPolicyPeerAllowPriority = 600 ovnRouterPolicyPeerDropPriority = 500 ) // ovnUplinkVars OVN object variables derived from uplink network. type ovnUplinkVars struct { // Router. routerExtPortIPv4Net string routerExtPortIPv6Net string routerExtGwIPv4 net.IP routerExtGwIPv6 net.IP // External Switch. extSwitchProviderName string // DNS. dnsIPv6 []net.IP dnsIPv4 []net.IP } // ovnUplinkPortBridgeVars uplink bridge port variables used for start/stop. type ovnUplinkPortBridgeVars struct { ovsBridge string uplinkEnd string ovsEnd string } // OVNInstanceNICSetupOpts options for starting an OVN Instance NIC. type OVNInstanceNICSetupOpts struct { InstanceUUID string DeviceName string DeviceConfig deviceConfig.Device UplinkConfig map[string]string DNSName string LastStateIPs []net.IP } // OVNInstanceNICStopOpts options for stopping an OVN Instance NIC. type OVNInstanceNICStopOpts struct { InstanceUUID string DeviceName string DeviceConfig deviceConfig.Device } // ovn represents an OVN network. type ovn struct { common ovnnb *networkOVN.NB ovnsb *networkOVN.SB } func (n *ovn) init(s *state.State, id int64, projectName string, netInfo *api.Network, netNodes map[int64]db.NetworkNode) error { if s != nil { // Handle IncusOS services. if s.OS.IncusOS != nil { ok, err := s.OS.IncusOS.IsServiceEnabled("ovn") if err != nil { return err } if !ok { return errors.New("IncusOS service \"ovn\" isn't currently enabled") } } // Check that OVN is available. ovnnb, ovnsb, err := s.OVN() if err != nil { return err } n.ovnnb = ovnnb n.ovnsb = ovnsb } return n.common.init(s, id, projectName, netInfo, netNodes) } // DBType returns the network type DB ID. func (n *ovn) DBType() db.NetworkType { return db.NetworkTypeOVN } // Config returns the network driver info. func (n *ovn) Info() Info { info := n.common.Info() info.Projects = true info.NodeSpecificConfig = false info.AddressForwards = true info.LoadBalancers = true info.Peering = true return info } func (n *ovn) State() (*api.NetworkState, error) { // Get the addresses. var addresses []api.NetworkStateAddress IPv4Net, err := ParseIPCIDRToNet(n.config["ipv4.address"]) if err == nil { ones, _ := IPv4Net.Mask.Size() addresses = append(addresses, api.NetworkStateAddress{ Family: "inet", Address: IPv4Net.IP.String(), Netmask: strconv.Itoa(ones), Scope: "link", }) } IPv6Net, err := ParseIPCIDRToNet(n.config["ipv6.address"]) if err == nil { ones, _ := IPv6Net.Mask.Size() addresses = append(addresses, api.NetworkStateAddress{ Family: "inet6", Address: IPv6Net.IP.String(), Netmask: strconv.Itoa(ones), Scope: "link", }) } var chassis string var hwaddr string var uplinkIPv4 string var uplinkIPv6 string logicalRouterName := n.getRouterName() logicalSwitchName := n.getIntSwitchName() // Check if an uplink network is present. if n.config["network"] != "none" { // Get the current active chassis. chassis, err = n.getActiveChassisName() if err != nil { return nil, err } // Get the IPv4 and IPv6 addresses on the uplink. if n.config[ovnVolatileUplinkIPv4] != "" { uplinkIPv4 = n.config[ovnVolatileUplinkIPv4] } if n.config[ovnVolatileUplinkIPv6] != "" { uplinkIPv6 = n.config[ovnVolatileUplinkIPv6] } } else if n.config["ipv4.address"] == "none" && n.config["ipv6.address"] == "none" { // Networks with no uplink and no IP addresses will not have a router. logicalRouterName = "" } // Add the gateway MAC address if one is present. if logicalRouterName != "" { var ok bool hwaddr, ok = n.config["bridge.hwaddr"] if !ok { hwaddr, err = n.ovnnb.GetLogicalRouterPortHardwareAddress(context.TODO(), n.getRouterIntPortName()) if err != nil { return nil, err } } } // Get the switch MTU. mtu := int(n.getBridgeMTU()) if mtu == 0 { mtu = 1500 } return &api.NetworkState{ Addresses: addresses, Hwaddr: hwaddr, Mtu: mtu, State: "up", Type: "broadcast", OVN: &api.NetworkStateOVN{ Chassis: chassis, LogicalRouter: string(logicalRouterName), LogicalSwitch: string(logicalSwitchName), UplinkIPv4: uplinkIPv4, UplinkIPv6: uplinkIPv6, }, }, nil } // uplinkRoutes parses ipv4.routes and ipv6.routes settings for an uplink network into a slice of *net.IPNet. func (n *ovn) uplinkRoutes(uplink *api.Network) ([]*net.IPNet, error) { var err error var uplinkRoutes []*net.IPNet for _, k := range []string{"ipv4.routes", "ipv6.routes"} { if uplink.Config[k] == "" { continue } uplinkRoutes, err = SubnetParseAppend(uplinkRoutes, util.SplitNTrimSpace(uplink.Config[k], ",", -1, false)...) if err != nil { return nil, err } } return uplinkRoutes, nil } // projectRestrictedSubnets parses the restrict.networks.subnets project setting and returns slice of *net.IPNet. // Returns nil slice if no project restrictions, or empty slice if no allowed subnets. func (n *ovn) projectRestrictedSubnets(p *api.Project, uplinkNetworkName string) ([]*net.IPNet, error) { // Parse project's restricted subnets. var projectRestrictedSubnets []*net.IPNet // Nil value indicates not restricted. if util.IsTrue(p.Config["restricted"]) && p.Config["restricted.networks.subnets"] != "" { projectRestrictedSubnets = []*net.IPNet{} // Empty slice indicates no allowed subnets. for _, subnetRaw := range util.SplitNTrimSpace(p.Config["restricted.networks.subnets"], ",", -1, false) { subnetParts := strings.SplitN(subnetRaw, ":", 2) if len(subnetParts) != 2 { return nil, fmt.Errorf(`Project subnet %q invalid, must be in the format of ":"`, subnetRaw) } subnetUplinkName := subnetParts[0] subnetStr := subnetParts[1] if subnetUplinkName != uplinkNetworkName { continue // Only include subnets for our uplink. } _, restrictedSubnet, err := net.ParseCIDR(subnetStr) if err != nil { return nil, err } projectRestrictedSubnets = append(projectRestrictedSubnets, restrictedSubnet) } } return projectRestrictedSubnets, nil } // validateExternalSubnet checks the supplied ipNet is allowed within the uplink routes and project // restricted subnets. If projectRestrictedSubnets is nil, then it is not checked as this indicates project has // no restrictions. Whereas if uplinkRoutes is nil/empty then this will always return an error. func (n *ovn) validateExternalSubnet(uplink *api.Network, projectRestrictedSubnets []*net.IPNet, ipNet *net.IPNet) error { // Check that the IP network is within the project's restricted subnets if restricted. if projectRestrictedSubnets != nil { foundMatch := false for _, projectRestrictedSubnet := range projectRestrictedSubnets { if SubnetContains(projectRestrictedSubnet, ipNet) { foundMatch = true break } } if !foundMatch { return fmt.Errorf("Project doesn't contain %q in its restricted uplink subnets", ipNet.String()) } } // Check if the IP network is within the uplink network's routes. uplinkRoutes, err := n.uplinkRoutes(uplink) if err != nil { return err } for _, uplinkRoute := range uplinkRoutes { if SubnetContains(uplinkRoute, ipNet) { return nil } } // Load uplink network details. uplinkIPv4CIDR := uplink.Config["ipv4.address"] if uplinkIPv4CIDR == "" { uplinkIPv4CIDR = uplink.Config["ipv4.gateway"] } uplinkIPv6CIDR := uplink.Config["ipv6.address"] if uplinkIPv6CIDR == "" { uplinkIPv6CIDR = uplink.Config["ipv6.gateway"] } uplinkIPv4, uplinkIPv4Net, _ := net.ParseCIDR(uplinkIPv4CIDR) uplinkIPv6, uplinkIPv6Net, _ := net.ParseCIDR(uplinkIPv6CIDR) // Check if the IP network is within the uplink network. if uplinkIPv4Net != nil && SubnetContains(uplinkIPv4Net, ipNet) { if ipNet.Contains(uplinkIPv4) { return api.StatusErrorf(http.StatusBadRequest, "Requested subnet %q would mask the uplink gateway", ipNet.String()) } return nil } if uplinkIPv6Net != nil && SubnetContains(uplinkIPv6Net, ipNet) { if ipNet.Contains(uplinkIPv6) { return api.StatusErrorf(http.StatusBadRequest, "Requested subnet %q would mask the uplink gateway", ipNet.String()) } return nil } return api.StatusErrorf(http.StatusBadRequest, "Uplink network doesn't contain %q in its routes", ipNet.String()) } // getExternalSubnetInUse returns information about usage of external subnets by networks and NICs connected to, // or used by, the specified uplinkNetworkName. func (n *ovn) getExternalSubnetInUse(uplinkNetworkName string) ([]externalSubnetUsage, error) { var err error var projectNetworks map[string]map[int64]api.Network var externalSubnets []externalSubnetUsage err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get all managed networks across all projects. projectNetworks, err = tx.GetCreatedNetworks(ctx) if err != nil { return fmt.Errorf("Failed to load all networks: %w", err) } externalSubnets, err = n.common.getExternalSubnetInUse(ctx, tx, uplinkNetworkName, false) if err != nil { return fmt.Errorf("Failed getting external subnets in use: %w", err) } return nil }) if err != nil { return nil, err } // Get OVN networks that use the same uplink as us. ovnProjectNetworksWithOurUplink := n.ovnProjectNetworksWithUplink(uplinkNetworkName, projectNetworks) // Get external subnets used by other OVN networks using our uplink. ovnNetworkExternalSubnets, err := n.ovnNetworkExternalSubnets(ovnProjectNetworksWithOurUplink) if err != nil { return nil, err } // Get external routes configured on OVN NICs using networks that use our uplink. ovnNICExternalRoutes, err := n.ovnNICExternalRoutes(ovnProjectNetworksWithOurUplink) if err != nil { return nil, err } externalSubnets = append(externalSubnets, ovnNetworkExternalSubnets...) externalSubnets = append(externalSubnets, ovnNICExternalRoutes...) return externalSubnets, nil } // validateUplinkAddressNotInUse checks for conflicting externAddresses in other networks/forwards/loadbalancers. func (n *ovn) validateUplinkAddressNotInUse(uplinkNetworkName string, ipAddress net.IP) error { if ipAddress == nil { return nil } return n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { projectNetworks, err := tx.GetCreatedNetworks(ctx) if err != nil { return fmt.Errorf("Failed to load all networks: %w", err) } for projectName, networks := range projectNetworks { for _, netInfo := range networks { if netInfo.Type != "ovn" || netInfo.Config["network"] != uplinkNetworkName { continue } // Skip our own network so updates that keep the same IP don't conflict with themselves. if projectName == n.project && netInfo.Name == n.name { continue } for _, k := range []string{ovnVolatileUplinkIPv4, ovnVolatileUplinkIPv6} { otherIP := net.ParseIP(netInfo.Config[k]) if otherIP != nil && otherIP.Equal(ipAddress) { // Vague error to avoid leaking resources from other projects. return fmt.Errorf("Uplink address %q is already in use by another OVN network", ipAddress.String()) } } } } // Check forward and load balancer listen addresses across networks connected to our uplink. externalSubnetsInUse, err := n.common.getExternalSubnetInUse(ctx, tx, uplinkNetworkName, false) if err != nil { return fmt.Errorf("Failed getting external subnets in use: %w", err) } for _, externalSubnetUser := range externalSubnetsInUse { if externalSubnetUser.usageType != subnetUsageNetworkForward && externalSubnetUser.usageType != subnetUsageNetworkLoadBalancer { continue } if SubnetContainsIP(&externalSubnetUser.subnet, ipAddress) { kind := "network forward" if externalSubnetUser.usageType == subnetUsageNetworkLoadBalancer { kind = "network load balancer" } return fmt.Errorf("Uplink address %q is already in use by a %s", ipAddress.String(), kind) } } return nil }) } // Validate network config. func (n *ovn) Validate(config map[string]string, clientType request.ClientType) error { rules := map[string]func(value string) error{ // gendoc:generate(entity=network_ovn, group=common, key=network) // // --- // type: string // shortdesc: Uplink network to use for external network access or `none` to keep isolated "network": validate.IsAny, // gendoc:generate(entity=network_ovn, group=common, key=bridge.hwaddr) // // --- // type: string // shortdesc: MAC address for the virtual bridge interface "bridge.hwaddr": validate.Optional(validate.IsNetworkMAC), // gendoc:generate(entity=network_ovn, group=common, key=bridge.mtu) // // --- // type: integer // shortdesc: Bridge MTU (default allows host to host Geneve tunnels) // default: `1442` "bridge.mtu": validate.Optional(validate.IsNetworkMTU), // gendoc:generate(entity=network_ovn, group=common, key=bridge.external_interfaces) // // --- // type: string // shortdesc: Comma-separated list of unconfigured network interfaces to include in the bridge "bridge.external_interfaces": validate.Optional(validateExternalInterfaces), // gendoc:generate(entity=network_ovn, group=common, key=ipv4.address) // // --- // type: string // shortdesc: IPv4 address for the bridge (use `none` to turn off IPv4 or `auto` to generate a new random unused subnet) (CIDR) // condition: standard mode // default: (initial value on creation: `auto`) "ipv4.address": validate.Optional(func(value string) error { if validate.IsOneOf("none", "auto")(value) == nil { return nil } return validate.IsNetworkAddressCIDRV4(value) }), // gendoc:generate(entity=network_ovn, group=common, key=ipv4.dhcp) // // --- // type: bool // shortdesc: Whether to allocate addresses using DHCP // condition: IPv4 address // default: `true` "ipv4.dhcp": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_ovn, group=common, key=ipv4.dhcp.gateway) // // --- // type: string // condition: IPv4 DHCP // default: IPv4 address // shortdesc: Address of the gateway for the subnet (use `none` to turn off gateway announcement) "ipv4.dhcp.gateway": validate.Optional(func(value string) error { if value == "none" { return nil } return validate.IsNetworkAddressV4(value) }), // gendoc:generate(entity=network_ovn, group=common, key=ipv4.dhcp.expiry) // // --- // type: string // shortdesc: When to expire DHCP leases // condition: IPv4 DHCP // default: `1h` "ipv4.dhcp.expiry": validate.Optional(func(value string) error { _, err := time.ParseDuration(value) return err }), // gendoc:generate(entity=network_ovn, group=common, key=ipv4.dhcp.ranges) // // --- // type: string // condition: IPv4 DHCP // default: all addresses // shortdesc: Comma-separated list of IP ranges to use for DHCP (FIRST-LAST format) "ipv4.dhcp.ranges": validate.Optional(validate.IsListOf(validate.IsNetworkRangeV4)), // gendoc:generate(entity=network_ovn, group=common, key=ipv4.dhcp.routes) // // --- // type: string // condition: IPv4 DHCP // shortdesc: Static routes to provide via DHCP option 121, as a comma-separated list of alternating subnets (CIDR) and gateway addresses (same syntax as dnsmasq and OVN) "ipv4.dhcp.routes": validate.Optional(validate.IsDHCPRouteList), // gendoc:generate(entity=network_ovn, group=common, key=ipv6.address) // // --- // type: string // shortdesc: IPv6 address for the bridge (use `none` to turn off IPv6 or `auto` to generate a new random unused subnet) (CIDR) // condition: standard mode // default: (initial value on creation: `auto`) "ipv6.address": validate.Optional(func(value string) error { if validate.IsOneOf("none", "auto")(value) == nil { return nil } return validate.IsNetworkAddressCIDRV6(value) }), // gendoc:generate(entity=network_ovn, group=common, key=ipv6.dhcp) // // --- // type: bool // condition: IPv6 address // shortdesc: Whether to provide additional network configuration over DHCP // default: `true` "ipv6.dhcp": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_ovn, group=common, key=ipv6.dhcp.stateful) // // --- // type: bool // condition: IPv6 DHCP // shortdesc: Whether to allocate addresses using DHCP // default: `false` "ipv6.dhcp.stateful": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_ovn, group=common, key=ipv4.nat) // // --- // type: bool // shortdesc: Whether to NAT // condition: IPv4 address // default: `false` initial value on creation if `ipv4.address` is set to `auto: true`) "ipv4.nat": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_ovn, group=common, key=ipv4.nat.address) // // --- // type: string // shortdesc: The source address used for outbound traffic from the network (requires uplink `ovn.ingress_mode=routed`) // condition: IPv4 address "ipv4.nat.address": validate.Optional(validate.IsNetworkAddressV4), // gendoc:generate(entity=network_ovn, group=common, key=ipv6.nat) // // --- // type: bool // condition: IPv6 address // shortdesc: Whether to NAT // default: `false` (initial value on creation if `ipv6.address` is set to `auto: true`) "ipv6.nat": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_ovn, group=common, key=ipv6.nat.address) // // --- // type: string // condition: IPv6 address // shortdesc: The source address used for outbound traffic from the network (requires uplink `ovn.ingress_mode=routed`) "ipv6.nat.address": validate.Optional(validate.IsNetworkAddressV6), // gendoc:generate(entity=network_ovn, group=common, key=ipv4.l3only) // // --- // type: bool // shortdesc: Whether to enable layer 3 only mode. // condition: IPv4 address // default: `false` "ipv4.l3only": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_ovn, group=common, key=ipv6.l3only) // // --- // type: bool // condition: IPv6 DHCP stateful // shortdesc: Whether to enable layer 3 only mode. // default: `false` "ipv6.l3only": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_ovn, group=common, key=dns.nameservers) // // --- // type: string // shortdesc: DNS server IPs to advertise to DHCP clients and via Router Advertisements. Both IPv4 and IPv6 addresses get pushed via DHCP, and the first IPv6 address is also advertised as RDNSS via RA. // default: Uplink DNS servers (IPv4 and IPv6 address if no uplink is configured) "dns.nameservers": validate.Optional(validate.IsListOf(validate.IsNetworkAddress)), // gendoc:generate(entity=network_ovn, group=common, key=dns.domain) // // --- // type: string // default: `incus` // shortdesc: Domain to advertise to DHCP clients and use for DNS resolution "dns.domain": validate.IsAny, // gendoc:generate(entity=network_ovn, group=common, key=dns.mode) // // --- // type: string // condition: - // default: `managed` // shortdesc: DNS registration mode: none for no DNS record, managed for OVN managed records "dns.mode": validate.Optional(validate.IsOneOf("managed", "none")), // gendoc:generate(entity=network_ovn, group=common, key=dns.search) // // --- // type: string // shortdesc: Full comma-separated domain search list, defaulting to `dns.domain` value "dns.search": validate.IsAny, // gendoc:generate(entity=network_ovn, group=common, key=dns.zone.forward) // // --- // type: string // shortdesc: Comma-separated list of DNS zone names for forward DNS records "dns.zone.forward": validate.IsAny, // gendoc:generate(entity=network_ovn, group=common, key=dns.zone.reverse.ipv4) // // --- // type: string // shortdesc: DNS zone name for IPv4 reverse DNS records "dns.zone.reverse.ipv4": validate.IsAny, // gendoc:generate(entity=network_ovn, group=common, key=dns.zone.reverse.ipv6) // // --- // type: string // shortdesc: DNS zone name for IPv6 reverse DNS records "dns.zone.reverse.ipv6": validate.IsAny, // gendoc:generate(entity=network_ovn, group=common, key=security.acls) // // --- // type: string // shortdesc: Comma-separated list of Network ACLs to apply to NICs connected to this network "security.acls": validate.IsAny, // gendoc:generate(entity=network_ovn, group=common, key=security.acls.default.ingress.action) // // --- // type: string // condition: `security.acls` // shortdesc: Action to use for ingress traffic that doesn't match any ACL rule // default: `reject` "security.acls.default.ingress.action": validate.Optional(validate.IsOneOf(acl.ValidActions...)), // gendoc:generate(entity=network_ovn, group=common, key=security.acls.default.egress.action) // // --- // type: string // shortdesc: Action to use for egress traffic that doesn't match any ACL rule // default: `reject` // condition: `security.acls` "security.acls.default.egress.action": validate.Optional(validate.IsOneOf(acl.ValidActions...)), // gendoc:generate(entity=network_ovn, group=common, key=security.acls.default.ingress.logged) // // --- // type: bool // condition: `security.acls` // shortdesc: Whether to log ingress traffic that doesn't match any ACL rule // default: `false` "security.acls.default.ingress.logged": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_ovn, group=common, key=security.acls.default.egress.logged) // // --- // type: bool // shortdesc: Whether to log egress traffic that doesn't match any ACL rule // default: `false` // condition: `security.acls` "security.acls.default.egress.logged": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_ovn, group=common, key=user.*) // // --- // type: string // shortdesc: User-provided free-form key/value pairs // Volatile keys populated automatically as needed. ovnVolatileUplinkIPv4: validate.Optional(validate.IsNetworkAddressV4), ovnVolatileUplinkIPv6: validate.Optional(validate.IsNetworkAddressV6), } // Add dynamic validation rules. for k := range config { // Tunnel keys have the remote name in their name, extract the suffix. if strings.HasPrefix(k, "tunnel.") { // Validate remote name in key. fields := strings.Split(k, ".") if len(fields) != 3 { return fmt.Errorf("Invalid network configuration key: %s", k) } // Full tunnel name: network_name + tunnel_name + interface_idx if len(n.name)+len(fields[1])+2 > 14 { return fmt.Errorf("Network name too long for tunnel interface: %s-%s-xx", n.name, fields[1]) } tunnelKey := fields[2] // Add the correct validation rule for the dynamic field based on last part of key. switch tunnelKey { case "protocol": // gendoc:generate(entity=network_ovn, group=common, key=tunnel.NAME.protocol) // // --- // type: string // condition: standard mode // default: - // shortdesc: Tunneling protocol: `vxlan` or `gre` rules[k] = validate.Optional(validate.IsOneOf("gre", "vxlan")) case "local": // gendoc:generate(entity=network_ovn, group=common, key=tunnel.NAME.local) // // --- // type: string // condition: `gre` // default: - // shortdesc: Local address for the tunnel rules[k] = validate.Optional(validate.IsNetworkAddress) case "remote": // gendoc:generate(entity=network_ovn, group=common, key=tunnel.NAME.remote) // // --- // type: string // condition: `gre` // default: - // shortdesc: Remote address for the tunnel rules[k] = validate.Optional(validate.IsListOf(validate.IsNetworkAddress)) case "port": // gendoc:generate(entity=network_ovn, group=common, key=tunnel.NAME.port) // // --- // type: integer // condition: `vxlan` // default: `0` // shortdesc: Specific port to use for the `vxlan` tunnel rules[k] = networkValidPort case "id": // gendoc:generate(entity=network_ovn, group=common, key=tunnel.NAME.id) // // --- // type: integer // condition: `vxlan` // default: `0` // shortdesc: Specific tunnel ID to use for the `vxlan` tunnel rules[k] = validate.Optional(validate.IsInt64) case "group": // gendoc:generate(entity=network_ovn, group=common, key=tunnel.NAME.group) // // --- // type: string // condition: `vxlan` // default: `239.0.0.1` // shortdesc: Multicast address for `vxlan` rules[k] = validate.Optional(validate.IsNetworkAddress) case "interface": // gendoc:generate(entity=network_ovn, group=common, key=tunnel.NAME.interface) // // --- // type: string // condition: `vxlan` // default: - // shortdesc: Specific host interface to use for the tunnel rules[k] = validate.IsInterfaceName case "ttl": // gendoc:generate(entity=network_ovn, group=common, key=tunnel.NAME.ttl) // // --- // type: integer // condition: `vxlan` // default: `1` // shortdesc: Specific TTL to use for multicast routing topologies rules[k] = validate.Optional(validate.IsUint8) } } } err := n.validate(config, rules) if err != nil { return err } // Perform composite key checks after per-key validation. // Validate DNS zone names. err = n.validateZoneNames(config) if err != nil { return err } if config["ipv4.address"] != "" { ipv4Addr, ipv4Net, _ := net.ParseCIDR(config["ipv4.address"]) if ipv4Net != nil { ovnRouter, err := netip.ParseAddr(dhcpalloc.GetIP(ipv4Net, -2).String()) if err != nil { return err } addr, err := netip.ParseAddr(ipv4Addr.String()) if err != nil { return err } if ovnRouter.Compare(addr) == 0 { return fmt.Errorf("'ipv4.address' cannot be set to %s because it is reserved for OVN load-balancer health checks", ovnRouter.String()) } } } // Check that if IPv6 enabled then the network size must be at least a /64 as both RA and DHCPv6 // in OVN (as it generates addresses using EUI64) require at least a /64 subnet to operate. _, ipv6Net, _ := net.ParseCIDR(config["ipv6.address"]) if ipv6Net != nil { ones, _ := ipv6Net.Mask.Size() if ones > 64 { return errors.New("IPv6 subnet must be at least a /64") } } // Load the project to get uplink network restrictions. var p *api.Project err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { project, err := dbCluster.GetProject(ctx, tx.Tx(), n.project) if err != nil { return err } p, err = project.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return fmt.Errorf("Failed to load network restrictions from project %q: %w", n.project, err) } // Check uplink network is valid and allowed in project. var uplink *api.Network projectRestrictedSubnets := []*net.IPNet{} if config["network"] != "none" && clientType != request.ClientTypeNotifier { uplinkNetworkName, err := n.validateUplinkNetwork(p, config["network"]) if err != nil { return err } err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get uplink routes. _, uplink, _, err = tx.GetNetworkInAnyState(ctx, api.ProjectDefaultName, uplinkNetworkName) return err }) if err != nil { return fmt.Errorf("Failed to load uplink network %q: %w", uplinkNetworkName, err) } // Get project restricted routes. projectRestrictedSubnets, err = n.projectRestrictedSubnets(p, uplinkNetworkName) if err != nil { return err } } // Parse the network's address subnets for further checks. netSubnets := make(map[string]*net.IPNet) for _, keyPrefix := range []string{"ipv4", "ipv6"} { addressKey := fmt.Sprintf("%s.address", keyPrefix) if validate.IsOneOf("", "none", "auto")(config[addressKey]) != nil { _, ipNet, err := net.ParseCIDR(config[addressKey]) if err != nil { return fmt.Errorf("Failed parsing %q: %w", addressKey, err) } netSubnets[addressKey] = ipNet } } // Check Security ACLs exist. if config["security.acls"] != "" { err = acl.Exists(n.state, n.project, util.SplitNTrimSpace(config["security.acls"], ",", -1, true)...) if err != nil { return err } } // Check that ipv6.l3only mode is used with ipvp.dhcp.stateful. // As otherwise the router advertisements will configure an address using the subnet's mask. if util.IsTrue(config["ipv6.l3only"]) && util.IsTrueOrEmpty(config["ipv6.dhcp"]) && util.IsFalseOrEmpty(config["ipv6.dhcp.stateful"]) { return errors.New("The ipv6.dhcp.stateful setting must be enabled when using ipv6.l3only mode with ipv6.dhcp enabled") } // All tests below are related to the uplink network, skip if we don't have one. if uplink == nil { return nil } // If a caller has explicitly pinned the OVN router's external uplink IP via the volatile keys, // make sure the chosen address isn't already used elsewhere on the same uplink. Without this // check two networks (or a network and a forward/load balancer) can end up sharing the same // external IP, which silently breaks routing. for _, key := range []string{ovnVolatileUplinkIPv4, ovnVolatileUplinkIPv6} { if config[key] == "" { continue } // Skip if unchanged and already created if config[key] == n.config[key] && n.status == api.NetworkStatusCreated { continue } ipAddress := net.ParseIP(config[key]) if ipAddress == nil { continue } err = n.validateUplinkAddressNotInUse(uplink.Name, ipAddress) if err != nil { return err } } // If NAT disabled, parse the external subnets that are being requested. var externalSubnets []*net.IPNet // Subnets to check for conflicts with other networks/NICs. for _, keyPrefix := range []string{"ipv4", "ipv6"} { addressKey := fmt.Sprintf("%s.address", keyPrefix) netSubnet := netSubnets[addressKey] if util.IsFalseOrEmpty(config[fmt.Sprintf("%s.nat", keyPrefix)]) && netSubnet != nil { // Add to list to check for conflicts. externalSubnets = append(externalSubnets, netSubnet) } } // Check SNAT addresses specified are allowed to be used based on uplink's ovn.ingress_mode setting. var externalSNATSubnets []*net.IPNet // Subnets to check for conflicts with other networks/NICs. for _, keyPrefix := range []string{"ipv4", "ipv6"} { snatAddressKey := fmt.Sprintf("%s.nat.address", keyPrefix) if config[snatAddressKey] != "" { if uplink.Config["ovn.ingress_mode"] != "routed" { return fmt.Errorf(`Cannot specify %q when uplink ovn.ingress_mode is not "routed"`, snatAddressKey) } subnetSize := 128 if keyPrefix == "ipv4" { subnetSize = 32 } _, snatIPNet, err := net.ParseCIDR(fmt.Sprintf("%s/%d", config[snatAddressKey], subnetSize)) if err != nil { return fmt.Errorf("Failed parsing %q: %w", snatAddressKey, err) } // Add to list to check for conflicts. externalSNATSubnets = append(externalSNATSubnets, snatIPNet) } } if len(externalSubnets) > 0 || len(externalSNATSubnets) > 0 { externalSubnetsInUse, err := n.getExternalSubnetInUse(config["network"]) if err != nil { return err } // Check if uplink has routed ingress anycast mode enabled, as this relaxes the overlap checks. ipv4UplinkAnycast := n.uplinkHasIngressRoutedAnycastIPv4(uplink) ipv6UplinkAnycast := n.uplinkHasIngressRoutedAnycastIPv6(uplink) for _, externalSubnet := range externalSubnets { // Check the external subnet is allowed within both the uplink's external routes and any // project restricted subnets. err = n.validateExternalSubnet(uplink, projectRestrictedSubnets, externalSubnet) if err != nil { return err } // Skip overlap checks if external subnet's protocol has anycast mode enabled on uplink. if externalSubnet.IP.To4() == nil { if ipv6UplinkAnycast { continue } } else if ipv4UplinkAnycast { continue } // Check the external subnet doesn't fall within any existing OVN network external subnets. for _, externalSubnetUser := range externalSubnetsInUse { // Skip our own network (but not NIC devices on our own network). if externalSubnetUser.usageType != subnetUsageInstance && externalSubnetUser.networkProject == n.project && externalSubnetUser.networkName == n.name { continue } if SubnetContains(&externalSubnetUser.subnet, externalSubnet) || SubnetContains(externalSubnet, &externalSubnetUser.subnet) { // This error is purposefully vague so that it doesn't reveal any names of // resources potentially outside of the network's project. return fmt.Errorf("External subnet %q overlaps with another network or NIC", externalSubnet.String()) } } } for _, externalSNATSubnet := range externalSNATSubnets { // Check the external subnet is allowed within both the uplink's external routes and any // project restricted subnets. err = n.validateExternalSubnet(uplink, projectRestrictedSubnets, externalSNATSubnet) if err != nil { return err } // Skip overlap checks if external subnet's protocol has anycast mode enabled on uplink. if externalSNATSubnet.IP.To4() == nil { if ipv6UplinkAnycast { continue } } else if ipv4UplinkAnycast { continue } // Check the external subnet doesn't fall within any existing OVN network external subnets. for _, externalSubnetUser := range externalSubnetsInUse { // Skip our own network (including NIC devices on our own network). // Because we may want to specify the SNAT address as the same address as one of // the NICs in our network. if externalSubnetUser.networkProject == n.project && externalSubnetUser.networkName == n.name { continue } if SubnetContains(&externalSubnetUser.subnet, externalSNATSubnet) || SubnetContains(externalSNATSubnet, &externalSubnetUser.subnet) { // This error is purposefully vague so that it doesn't reveal any names of // resources potentially outside of the network's project. return fmt.Errorf("NAT address %q overlaps with another OVN network or NIC", externalSNATSubnet.IP.String()) } } } } // Check any existing network forward target addresses are suitable for this network's subnet. var forwards map[int64]*api.NetworkForward err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() dbRecords, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, }) if err != nil { return err } forwards = make(map[int64]*api.NetworkForward) for _, dbRecord := range dbRecords { forward, err := dbRecord.ToAPI(ctx, tx.Tx()) if err != nil { return err } forwards[dbRecord.ID] = forward } return nil }) if err != nil { return fmt.Errorf("Failed loading network forwards: %w", err) } for _, forward := range forwards { if forward.Config["target_address"] != "" { defaultTargetIP := net.ParseIP(forward.Config["target_address"]) netSubnet := netSubnets["ipv4.address"] if defaultTargetIP.To4() == nil { netSubnet = netSubnets["ipv6.address"] } if !SubnetContainsIP(netSubnet, defaultTargetIP) { return api.StatusErrorf(http.StatusBadRequest, "Network forward for %q has a default target address %q that is not within the network subnet", forward.ListenAddress, defaultTargetIP.String()) } } for _, port := range forward.Ports { targetIP := net.ParseIP(port.TargetAddress) netSubnet := netSubnets["ipv4.address"] if targetIP.To4() == nil { netSubnet = netSubnets["ipv6.address"] } if !SubnetContainsIP(netSubnet, targetIP) { return api.StatusErrorf(http.StatusBadRequest, "Network forward for %q has a port target address %q that is not within the network subnet", forward.ListenAddress, targetIP.String()) } } } var dbLoadBalancers []dbCluster.NetworkLoadBalancer err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() // Get the load balancers. dbLoadBalancers, err = dbCluster.GetNetworkLoadBalancers(ctx, tx.Tx(), dbCluster.NetworkLoadBalancerFilter{ NetworkID: &networkID, }) if err != nil { return err } return nil }) if err != nil { return fmt.Errorf("Failed loading network load balancers: %w", err) } for _, dbLoadBalancer := range dbLoadBalancers { for _, port := range dbLoadBalancer.Backends { targetIP := net.ParseIP(port.TargetAddress) netSubnet := netSubnets["ipv4.address"] if targetIP.To4() == nil { netSubnet = netSubnets["ipv6.address"] } if !SubnetContainsIP(netSubnet, targetIP) { return api.StatusErrorf(http.StatusBadRequest, "Network load balancer for %q has a backend target address %q that is not within the network subnet", dbLoadBalancer.ListenAddress, targetIP.String()) } } } return nil } // getBridgeMTU returns MTU that should be used for the bridge and instance devices. // Will also be used to configure the OVN DHCP and IPv6 RA options. Returns 0 if the bridge.mtu is not set/invalid. func (n *ovn) getBridgeMTU() uint32 { if n.config["bridge.mtu"] != "" { mtu, err := strconv.ParseUint(n.config["bridge.mtu"], 10, 32) if err != nil { return 0 } return uint32(mtu) } return 0 } // getUnderlayInfo returns the MTU for the underlay network interface and the enscapsulation IP for OVN tunnels. func (n *ovn) getUnderlayInfo() (uint32, net.IP, error) { // findMTUFromIP searches all interfaces on the host looking for one that has specified IP. findMTUFromIP := func(findIP net.IP) (uint32, error) { // Look for interface that has the OVN enscapsulation IP assigned. ifaces, err := net.Interfaces() if err != nil { return 0, fmt.Errorf("Failed getting local network interfaces: %w", err) } for _, iface := range ifaces { addrs, err := iface.Addrs() if err != nil { continue } for _, addr := range addrs { ip, _, err := net.ParseCIDR(addr.String()) if err != nil { continue } if ip.Equal(findIP) { underlayMTU, err := GetDevMTU(iface.Name) if err != nil { return 0, fmt.Errorf("Failed getting MTU for %q: %w", iface.Name, err) } return underlayMTU, nil // Found what we were looking for. } } } return 0, fmt.Errorf("No matching interface found for OVN enscapsulation IP %q", findIP.String()) } vswitch, err := n.state.OVS() if err != nil { return 0, nil, fmt.Errorf("Failed to connect to OVS: %w", err) } encapIP, err := vswitch.GetOVNEncapIP(context.TODO()) if err != nil { return 0, nil, fmt.Errorf("Failed getting OVN enscapsulation IP from OVS: %w", err) } underlayMTU, err := findMTUFromIP(encapIP) if err != nil { return 0, nil, err } return underlayMTU, encapIP, nil } // getOptimalBridgeMTU returns the MTU that can be used for the bridge and instance devices based on the MTU value // of the OVN underlay network interface. This assumes that the OVN tunnel mechanism used is geneve and that the // same underlying network settings (MTU and encapsulation IP family) are used on all OVN nodes. func (n *ovn) getOptimalBridgeMTU() (uint32, error) { // Get underlay MTU and encapsulation IP. underlayMTU, encapIP, err := n.getUnderlayInfo() if err != nil { return 0, fmt.Errorf("Failed getting OVN underlay info: %w", err) } // Encapsulation family is IPv6. if encapIP.To4() == nil { // If the underlay's MTU is large enough to accommodate a 1500 overlay MTU and the geneve tunnel // overhead of 78 bytes (when used with IPv6 encapsulation) then indicate 1500 MTU can be used. if underlayMTU >= 1578 { return 1500, nil } // Default to 1422 which can work with an underlay MTU of 1500. return 1422, nil } // If the underlay's MTU is large enough to accommodate a 1500 overlay MTU and the geneve tunnel // overhead of 58 bytes (when used with IPv4 encapsulation) then indicate 1500 MTU can be used. if underlayMTU >= 1558 { return 1500, nil } // Default to 1442 which can work with underlay MTU of 1500. return 1442, nil } // getNetworkPrefix returns OVN network prefix to use for object names. func (n *ovn) getNetworkPrefix() string { return acl.OVNNetworkPrefix(n.id) } // getChassisGroup returns OVN chassis group name to use. func (n *ovn) getChassisGroupName() networkOVN.OVNChassisGroup { return networkOVN.OVNChassisGroup(n.getNetworkPrefix()) } // getRouterName returns OVN logical router name to use. func (n *ovn) getRouterName() networkOVN.OVNRouter { return networkOVN.OVNRouter(fmt.Sprintf("%s-lr", n.getNetworkPrefix())) } // getRouterExtPortName returns OVN logical router external port name to use. func (n *ovn) getRouterExtPortName() networkOVN.OVNRouterPort { return networkOVN.OVNRouterPort(fmt.Sprintf("%s-lrp-ext", n.getRouterName())) } // getRouterIntPortName returns OVN logical router internal port name to use. func (n *ovn) getRouterIntPortName() networkOVN.OVNRouterPort { return networkOVN.OVNRouterPort(fmt.Sprintf("%s-lrp-int", n.getRouterName())) } // getRouterMAC returns OVN router MAC address to use for ports. Uses a stable seed to return stable random MAC. func (n *ovn) getRouterMAC() (net.HardwareAddr, error) { hwAddr := n.config["bridge.hwaddr"] if hwAddr == "" { // Load server certificate. This is needs to be the same certificate for all nodes in a cluster. cert, err := internalUtil.LoadCert(n.state.OS.VarDir) if err != nil { return nil, err } // Generate the random seed, this uses the server certificate fingerprint (to ensure that multiple // standalone nodes on the same external network don't generate the same MAC for their networks). // It relies on the certificate being the same for all nodes in a cluster to allow the same MAC to // be generated on each bridge interface in the network. seed := fmt.Sprintf("%s.%d.%d", cert.Fingerprint(), 0, n.ID()) r, err := localUtil.GetStableRandomGenerator(seed) if err != nil { return nil, fmt.Errorf("Failed generating stable random router MAC: %w", err) } hwAddr = n.randomHwaddr(r) n.logger.Debug("Stable MAC generated", logger.Ctx{"seed": seed, "hwAddr": hwAddr}) } mac, err := net.ParseMAC(hwAddr) if err != nil { return nil, fmt.Errorf("Failed parsing router MAC address %q: %w", mac, err) } return mac, nil } // getRouterIntPortIPv4Net returns OVN logical router internal port IPv4 address and subnet. func (n *ovn) getRouterIntPortIPv4Net() string { return n.config["ipv4.address"] } // parseRouterIntPortIPv4Net returns OVN logical router internal port IPv4 address and subnet parsed (if set). func (n *ovn) parseRouterIntPortIPv4Net() (net.IP, *net.IPNet, error) { ipNet := n.getRouterIntPortIPv4Net() if validate.IsOneOf("none", "")(ipNet) != nil { routerIntPortIPv4, routerIntPortIPv4Net, err := net.ParseCIDR(ipNet) if err != nil { return nil, nil, fmt.Errorf("Failed parsing router's internal port IPv4 Net: %w", err) } return routerIntPortIPv4, routerIntPortIPv4Net, nil } return nil, nil, nil } // getRouterIntPortIPv4Net returns OVN logical router internal port IPv6 address and subnet. func (n *ovn) getRouterIntPortIPv6Net() string { return n.config["ipv6.address"] } // parseRouterIntPortIPv6Net returns OVN logical router internal port IPv6 address and subnet parsed (if set). func (n *ovn) parseRouterIntPortIPv6Net() (net.IP, *net.IPNet, error) { ipNet := n.getRouterIntPortIPv6Net() if validate.IsOneOf("none", "")(ipNet) != nil { routerIntPortIPv4, routerIntPortIPv4Net, err := net.ParseCIDR(ipNet) if err != nil { return nil, nil, fmt.Errorf("Failed parsing router's internal port IPv6 Net: %w", err) } return routerIntPortIPv4, routerIntPortIPv4Net, nil } return nil, nil, nil } // getDomainName returns OVN DHCP domain name. func (n *ovn) getDomainName() string { if n.config["dns.domain"] != "" { return n.config["dns.domain"] } return "incus" } // getGateway returns OVN DHCP domain name. func (n *ovn) getGatewayIpv4() (net.IP, error) { addr := n.config["ipv4.dhcp.gateway"] if addr == "none" { return nil, nil } if addr != "" { return net.ParseIP(addr), nil } routerIntPortIPv4, _, err := n.parseRouterIntPortIPv4Net() if err != nil { return nil, err } return routerIntPortIPv4, nil } // getDNSSearchList returns OVN DHCP DNS search list. If no search list set returns getDomainName() as list. func (n *ovn) getDNSSearchList() []string { if n.config["dns.search"] != "" { return util.SplitNTrimSpace(n.config["dns.search"], ",", -1, false) } return []string{n.getDomainName()} } // getExtSwitchName returns OVN logical external switch name. func (n *ovn) getExtSwitchName() networkOVN.OVNSwitch { return networkOVN.OVNSwitch(fmt.Sprintf("%s-ls-ext", n.getNetworkPrefix())) } // getExtSwitchRouterPortName returns OVN logical external switch router port name. func (n *ovn) getExtSwitchRouterPortName() networkOVN.OVNSwitchPort { return networkOVN.OVNSwitchPort(fmt.Sprintf("%s-lsp-router", n.getExtSwitchName())) } // getExtSwitchProviderPortName returns OVN logical external switch provider port name. func (n *ovn) getExtSwitchProviderPortName() networkOVN.OVNSwitchPort { return networkOVN.OVNSwitchPort(fmt.Sprintf("%s-lsp-provider", n.getExtSwitchName())) } // getIntSwitchName returns OVN logical internal switch name. func (n *ovn) getIntSwitchName() networkOVN.OVNSwitch { return acl.OVNIntSwitchName(n.id) } // getIntSwitchRouterPortName returns OVN logical internal switch router port name. func (n *ovn) getIntSwitchRouterPortName() networkOVN.OVNSwitchPort { return acl.OVNIntSwitchRouterPortName(n.id) } // getIntSwitchInstancePortPrefix returns OVN logical internal switch instance port name prefix. func (n *ovn) getIntSwitchInstancePortPrefix() string { return fmt.Sprintf("%s-instance", n.getNetworkPrefix()) } // getLoadBalancerName returns OVN load balancer name to use for a listen address. func (n *ovn) getLoadBalancerName(listenAddress string) networkOVN.OVNLoadBalancer { return networkOVN.OVNLoadBalancer(fmt.Sprintf("%s-lb-%s", n.getNetworkPrefix(), listenAddress)) } // getLogicalRouterPeerPortName returns OVN logical router port name to use for a peer connection. func (n *ovn) getLogicalRouterPeerPortName(peerNetworkID int64) networkOVN.OVNRouterPort { return networkOVN.OVNRouterPort(fmt.Sprintf("%s-lrp-peer-net%d", n.getRouterName(), peerNetworkID)) } // setupUplinkPort initializes the uplink connection. Returns the derived ovnUplinkVars settings used // during the initial creation of the logical network. func (n *ovn) setupUplinkPort(routerMAC net.HardwareAddr) (*ovnUplinkVars, error) { if n.config["network"] == "none" { return nil, nil } // Uplink network must be in default project. uplinkNet, err := LoadByName(n.state, api.ProjectDefaultName, n.config["network"]) if err != nil { return nil, fmt.Errorf("Failed loading uplink network %q: %w", n.config["network"], err) } switch uplinkNet.Type() { case "bridge": return n.setupUplinkPortBridge(uplinkNet, routerMAC) case "physical": return n.setupUplinkPortPhysical(uplinkNet, routerMAC) } return nil, fmt.Errorf("Failed setting up uplink port, network type %q unsupported as OVN uplink", uplinkNet.Type()) } // setupUplinkPortBridge allocates external IPs on the uplink bridge. // Returns the derived ovnUplinkVars settings. func (n *ovn) setupUplinkPortBridge(uplinkNet Network, routerMAC net.HardwareAddr) (*ovnUplinkVars, error) { bridgeNet, ok := uplinkNet.(*bridge) if !ok { return nil, errors.New("Network is not bridge type") } err := bridgeNet.checkClusterWideMACSafe(bridgeNet.config) if err != nil { return nil, fmt.Errorf("Network %q is not suitable for use as OVN uplink: %w", bridgeNet.name, err) } v, err := n.allocateUplinkPortIPs(uplinkNet, routerMAC) if err != nil { return nil, fmt.Errorf("Failed allocating uplink port IPs on network %q: %w", uplinkNet.Name(), err) } return v, nil } // setupUplinkPortPhysical allocates external IPs on the uplink network. // Returns the derived ovnUplinkVars settings. func (n *ovn) setupUplinkPortPhysical(uplinkNet Network, routerMAC net.HardwareAddr) (*ovnUplinkVars, error) { v, err := n.allocateUplinkPortIPs(uplinkNet, routerMAC) if err != nil { return nil, fmt.Errorf("Failed allocating uplink port IPs on network %q: %w", uplinkNet.Name(), err) } return v, nil } // allocateUplinkPortIPs attempts to find a free IP in the uplink network's OVN ranges and then stores it in // ovnVolatileUplinkIPv4 and ovnVolatileUplinkIPv6 config keys on this network. Returns ovnUplinkVars settings. func (n *ovn) allocateUplinkPortIPs(uplinkNet Network, routerMAC net.HardwareAddr) (*ovnUplinkVars, error) { v := &ovnUplinkVars{} uplinkNetConf := uplinkNet.Config() // Uplink derived settings. v.extSwitchProviderName = uplinkNet.Name() // Detect uplink gateway setting. uplinkIPv4CIDR := uplinkNetConf["ipv4.address"] if uplinkIPv4CIDR == "" { uplinkIPv4CIDR = uplinkNetConf["ipv4.gateway"] } uplinkIPv6CIDR := uplinkNetConf["ipv6.address"] if uplinkIPv6CIDR == "" { uplinkIPv6CIDR = uplinkNetConf["ipv6.gateway"] } // Optional uplink values. uplinkIPv4, uplinkIPv4Net, err := net.ParseCIDR(uplinkIPv4CIDR) if err == nil { v.dnsIPv4 = []net.IP{uplinkIPv4} v.routerExtGwIPv4 = uplinkIPv4 } uplinkIPv6, uplinkIPv6Net, err := net.ParseCIDR(uplinkIPv6CIDR) if err == nil { v.dnsIPv6 = []net.IP{uplinkIPv6} v.routerExtGwIPv6 = uplinkIPv6 } // Detect optional DNS server list. if uplinkNetConf["dns.nameservers"] != "" { // Reset nameservers. v.dnsIPv4 = nil v.dnsIPv6 = nil nsList := util.SplitNTrimSpace(uplinkNetConf["dns.nameservers"], ",", -1, false) for _, ns := range nsList { nsIP := net.ParseIP(ns) if nsIP == nil { return nil, errors.New("Invalid uplink nameserver") } if nsIP.To4() == nil { v.dnsIPv6 = append(v.dnsIPv6, nsIP) } else { v.dnsIPv4 = append(v.dnsIPv4, nsIP) } } } // Parse existing allocated IPs for this network on the uplink network (if not set yet, will be nil). routerExtPortIPv4 := net.ParseIP(n.config[ovnVolatileUplinkIPv4]) routerExtPortIPv6 := net.ParseIP(n.config[ovnVolatileUplinkIPv6]) // Check if uplink is viable at all. if uplinkIPv4Net == nil && uplinkIPv6Net == nil { return nil, errors.New("Uplink network doesn't have IPv4 or IPv6 configured") } // Decide whether we need to allocate new IP(s) and go to the expense of retrieving all allocated IPs. if (uplinkIPv4Net != nil && routerExtPortIPv4 == nil) || (uplinkIPv6Net != nil && routerExtPortIPv6 == nil) { err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { allAllocatedIPv4, allAllocatedIPv6, err := n.uplinkAllAllocatedIPs(ctx, tx, uplinkNet.Name()) if err != nil { return fmt.Errorf("Failed to get all allocated IPs for uplink: %w", err) } if uplinkIPv4Net != nil && routerExtPortIPv4 == nil { if uplinkNetConf["ipv4.ovn.ranges"] == "" { return errors.New(`Missing required "ipv4.ovn.ranges" config key on uplink network`) } dhcpSubnet := uplinkNet.DHCPv4Subnet() allowedNets := []*net.IPNet{} if dhcpSubnet != nil { allowedNets = append(allowedNets, dhcpSubnet) } else { allowedNets = append(allowedNets, uplinkIPv4Net) } ipRanges, err := parseIPRanges(uplinkNetConf["ipv4.ovn.ranges"], allowedNets...) if err != nil { return fmt.Errorf("Failed to parse uplink IPv4 OVN ranges: %w", err) } routerExtPortIPv4, err = n.uplinkAllocateIP(ipRanges, allAllocatedIPv4) if err != nil { return fmt.Errorf("Failed to allocate uplink IPv4 address: %w", err) } n.config[ovnVolatileUplinkIPv4] = routerExtPortIPv4.String() } if uplinkIPv6Net != nil && routerExtPortIPv6 == nil { // If IPv6 OVN ranges are specified by the uplink, allocate from them. if uplinkNetConf["ipv6.ovn.ranges"] != "" { dhcpSubnet := uplinkNet.DHCPv6Subnet() allowedNets := []*net.IPNet{} if dhcpSubnet != nil { allowedNets = append(allowedNets, dhcpSubnet) } else { allowedNets = append(allowedNets, uplinkIPv6Net) } ipRanges, err := parseIPRanges(uplinkNetConf["ipv6.ovn.ranges"], allowedNets...) if err != nil { return fmt.Errorf("Failed to parse uplink IPv6 OVN ranges: %w", err) } routerExtPortIPv6, err = n.uplinkAllocateIP(ipRanges, allAllocatedIPv6) if err != nil { return fmt.Errorf("Failed to allocate uplink IPv6 address: %w", err) } } else { // Otherwise use EUI64 derived from MAC address. routerExtPortIPv6, err = eui64.ParseMAC(uplinkIPv6Net.IP, routerMAC) if err != nil { return err } } n.config[ovnVolatileUplinkIPv6] = routerExtPortIPv6.String() } err = tx.UpdateNetwork(ctx, n.project, n.name, n.description, n.config) if err != nil { return fmt.Errorf("Failed saving allocated uplink network IPs: %w", err) } return nil }) if err != nil { return nil, err } } // Configure variables needed to configure OVN router. if uplinkIPv4Net != nil && routerExtPortIPv4 != nil { routerExtPortIPv4Net := &net.IPNet{ Mask: uplinkIPv4Net.Mask, IP: routerExtPortIPv4, } v.routerExtPortIPv4Net = routerExtPortIPv4Net.String() } if uplinkIPv6Net != nil { routerExtPortIPv6Net := &net.IPNet{ Mask: uplinkIPv6Net.Mask, IP: routerExtPortIPv6, } v.routerExtPortIPv6Net = routerExtPortIPv6Net.String() } return v, nil } // uplinkAllAllocatedIPs gets a list of all IPv4 and IPv6 addresses allocated to OVN networks connected to uplink. func (n *ovn) uplinkAllAllocatedIPs(ctx context.Context, tx *db.ClusterTx, uplinkNetName string) ([]net.IP, []net.IP, error) { // Get all managed networks across all projects. projectNetworks, err := tx.GetCreatedNetworks(ctx) if err != nil { return nil, nil, fmt.Errorf("Failed to load all networks: %w", err) } v4IPs := make([]net.IP, 0) v6IPs := make([]net.IP, 0) for _, networks := range projectNetworks { for _, netInfo := range networks { if netInfo.Type != "ovn" || netInfo.Config["network"] != uplinkNetName { continue } for _, k := range []string{ovnVolatileUplinkIPv4, ovnVolatileUplinkIPv6} { if netInfo.Config[k] != "" { ip := net.ParseIP(netInfo.Config[k]) if ip != nil { if ip.To4() != nil { v4IPs = append(v4IPs, ip) } else { v6IPs = append(v6IPs, ip) } } } } } } return v4IPs, v6IPs, nil } // uplinkAllocateIP allocates a free IP from one of the IP ranges. func (n *ovn) uplinkAllocateIP(ipRanges []*iprange.Range, allAllocated []net.IP) (net.IP, error) { for _, ipRange := range ipRanges { inc := big.NewInt(1) // Convert IPs in range to native representations to allow incrementing and comparison. startIP := ipRange.Start.To4() if startIP == nil { startIP = ipRange.Start.To16() } endIP := ipRange.End.To4() if endIP == nil { endIP = ipRange.End.To16() } startBig := big.NewInt(0) startBig.SetBytes(startIP) endBig := big.NewInt(0) endBig.SetBytes(endIP) // Iterate through IPs in range, return the first unallocated one found. for { if startBig.Cmp(endBig) > 0 { break } ip := net.IP(startBig.Bytes()) // Check IP is not already allocated. freeIP := true if slices.ContainsFunc(allAllocated, ip.Equal) { freeIP = false } if !freeIP { startBig.Add(startBig, inc) continue } return ip, nil } } return nil, errors.New("No free IPs available") } // startUplinkPort performs any network start up logic needed to connect the uplink connection to OVN. func (n *ovn) startUplinkPort() error { if n.config["network"] == "none" { return nil } // Uplink network must be in default project. uplinkNet, err := LoadByName(n.state, api.ProjectDefaultName, n.config["network"]) if err != nil { return fmt.Errorf("Failed loading uplink network %q: %w", n.config["network"], err) } // Skip if uplink is physical network with parent set to "none". if uplinkNet.Type() == "physical" && uplinkNet.Config()["parent"] == "none" { return nil } // Lock uplink network so that if multiple OVN networks are trying to connect to the same uplink we don't // race each other setting up the connection. unlock, err := locking.Lock(context.TODO(), n.uplinkOperationLockName(uplinkNet)) if err != nil { return err } defer unlock() switch uplinkNet.Type() { case "bridge": return n.startUplinkPortBridge(uplinkNet) case "physical": return n.startUplinkPortPhysical(uplinkNet) } return fmt.Errorf("Failed starting uplink port, network type %q unsupported as OVN uplink", uplinkNet.Type()) } // uplinkOperationLockName returns the lock name to use for operations on the uplink network. func (n *ovn) uplinkOperationLockName(uplinkNet Network) string { return fmt.Sprintf("network.ovn.%s", uplinkNet.Name()) } // uplinkPortBridgeVars returns the uplink port bridge variables needed for port start/stop. func (n *ovn) uplinkPortBridgeVars(uplinkNet Network) *ovnUplinkPortBridgeVars { ovsBridge := fmt.Sprintf("incusovn%d", uplinkNet.ID()) return &ovnUplinkPortBridgeVars{ ovsBridge: ovsBridge, uplinkEnd: fmt.Sprintf("%sa", ovsBridge), ovsEnd: fmt.Sprintf("%sb", ovsBridge), } } // startUplinkPortBridge creates veth pair (if doesn't exist), creates OVS bridge (if doesn't exist) and // connects veth pair to uplink bridge and OVS bridge. func (n *ovn) startUplinkPortBridge(uplinkNet Network) error { if uplinkNet.Config()["bridge.driver"] != "openvswitch" { return n.startUplinkPortBridgeNative(uplinkNet, uplinkNet.Name()) } return n.startUplinkPortBridgeOVS(uplinkNet, uplinkNet.Name()) } // startUplinkPortBridgeNative connects an OVN logical router to an uplink native bridge. func (n *ovn) startUplinkPortBridgeNative(uplinkNet Network, bridgeDevice string) error { // Do this after gaining lock so that on failure we revert before release locking. reverter := revert.New() defer reverter.Fail() // If uplink is a native bridge, then use a separate OVS bridge with veth pair connection to native bridge. vars := n.uplinkPortBridgeVars(uplinkNet) // Create veth pair if needed. if !InterfaceExists(vars.uplinkEnd) && !InterfaceExists(vars.ovsEnd) { veth := &ip.Veth{ Link: ip.Link{ Name: vars.uplinkEnd, }, Peer: ip.Link{ Name: vars.ovsEnd, }, } err := veth.Add() if err != nil { return fmt.Errorf("Failed to create the uplink veth interfaces %q and %q: %w", vars.uplinkEnd, vars.ovsEnd, err) } reverter.Add(func() { _ = veth.Delete() }) } // Ensure that the veth interfaces inherit the uplink bridge's MTU (which the OVS bridge also inherits). uplinkNetConfig := uplinkNet.Config() // Uplink may have type "bridge" or "physical" uplinkNetMTU, hasBridgeMTU := uplinkNetConfig["bridge.mtu"] if !hasBridgeMTU { uplinkNetMTU = uplinkNetConfig["mtu"] } if uplinkNetMTU != "" { mtu, err := strconv.ParseUint(uplinkNetMTU, 10, 32) if err != nil { return fmt.Errorf("Invalid uplink MTU %q: %w", uplinkNetMTU, err) } uplinkEndLink := &ip.Link{Name: vars.uplinkEnd} err = uplinkEndLink.SetMTU(uint32(mtu)) if err != nil { return fmt.Errorf("Failed setting MTU %q on %q: %w", uplinkNetMTU, uplinkEndLink.Name, err) } ovsEndLink := &ip.Link{Name: vars.ovsEnd} err = ovsEndLink.SetMTU(uint32(mtu)) if err != nil { return fmt.Errorf("Failed setting MTU %q on %q: %w", uplinkNetMTU, ovsEndLink.Name, err) } } // Ensure correct sysctls are set on uplink veth interfaces to avoid getting IPv6 link-local addresses. if util.PathExists("/proc/sys/net/ipv6") { err := localUtil.SysctlSet( fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", vars.uplinkEnd), "1", fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", vars.ovsEnd), "1", fmt.Sprintf("net/ipv6/conf/%s/forwarding", vars.uplinkEnd), "0", fmt.Sprintf("net/ipv6/conf/%s/forwarding", vars.ovsEnd), "0", ) if err != nil { return fmt.Errorf("Failed to configure uplink veth interfaces %q and %q: %w", vars.uplinkEnd, vars.ovsEnd, err) } } // Connect uplink end of veth pair to uplink bridge and bring up. link := &ip.Link{Name: vars.uplinkEnd} err := link.SetMaster(bridgeDevice) if err != nil { return fmt.Errorf("Failed to connect uplink veth interface %q to uplink bridge %q: %w", vars.uplinkEnd, bridgeDevice, err) } link = &ip.Link{Name: vars.uplinkEnd} err = link.SetUp() if err != nil { return fmt.Errorf("Failed to bring up uplink veth interface %q: %w", vars.uplinkEnd, err) } // Ensure uplink OVS end veth interface is up. link = &ip.Link{Name: vars.ovsEnd} err = link.SetUp() if err != nil { return fmt.Errorf("Failed to bring up uplink veth interface %q: %w", vars.ovsEnd, err) } // Create uplink OVS bridge if needed. vswitch, err := n.state.OVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } err = vswitch.CreateBridge(context.TODO(), vars.ovsBridge, true, nil, 0) if err != nil { return fmt.Errorf("Failed to create uplink OVS bridge %q: %w", vars.ovsBridge, err) } // Connect OVS end veth interface to OVS bridge. err = vswitch.CreateBridgePort(context.TODO(), vars.ovsBridge, vars.ovsEnd, true) if err != nil { return fmt.Errorf("Failed to connect uplink veth interface %q to uplink OVS bridge %q: %w", vars.ovsEnd, vars.ovsBridge, err) } // Associate OVS bridge to logical OVN provider. err = vswitch.AddOVNBridgeMapping(context.TODO(), vars.ovsBridge, uplinkNet.Name()) if err != nil { return fmt.Errorf("Failed to associate uplink OVS bridge %q to OVN provider %q: %w", vars.ovsBridge, uplinkNet.Name(), err) } // Attempt to learn uplink MAC. n.pingOVNRouter() reverter.Success() return nil } // startUplinkPortBridgeOVS connects an OVN logical router to an uplink OVS bridge. func (n *ovn) startUplinkPortBridgeOVS(uplinkNet Network, bridgeDevice string) error { // Do this after gaining lock so that on failure we revert before release locking. reverter := revert.New() defer reverter.Fail() // If uplink is an openvswitch bridge, have OVN logical provider connect directly to it. vswitch, err := n.state.OVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } err = vswitch.AddOVNBridgeMapping(context.TODO(), bridgeDevice, uplinkNet.Name()) if err != nil { return fmt.Errorf("Failed to associate uplink OVS bridge %q to OVN provider %q: %w", bridgeDevice, uplinkNet.Name(), err) } // Attempt to learn uplink MAC. n.pingOVNRouter() reverter.Success() return nil } // pingOVNRouter pings the OVN router's external IP addresses to attempt to trigger MAC learning on uplink. // This is to work around a bug in some versions of OVN. func (n *ovn) pingOVNRouter() { var ips []net.IP for _, key := range []string{ovnVolatileUplinkIPv4, ovnVolatileUplinkIPv6} { ip := net.ParseIP(n.config[key]) if ip != nil { ips = append(ips, ip) } } for i := range ips { ip := ips[i] // Local var // Now that the OVN router is connected to the uplink bridge, attempt to ping the OVN // router's external IPv6 from the host running the uplink bridge in an attempt to trigger the // OVN router to learn the uplink gateway's MAC address. This is to work around a bug in // older versions of OVN that meant that the OVN router would not attempt to learn the external // uplink IPv6 gateway MAC address when using SNAT, meaning that external IPv6 connectivity // wouldn't work until the next router advertisement was sent (which could be several minutes). // By pinging the OVN router's external IP this will trigger an NDP request from the uplink bridge // which will cause the OVN router to learn its MAC address. go func() { var err error // Try several attempts as it can take a few seconds for the network to come up. for range 5 { err = pingIP(context.TODO(), ip) if err == nil { n.logger.Debug("OVN router external IP address reachable", logger.Ctx{"ip": ip.String()}) return } time.Sleep(time.Second) } // We would expect this on a chassis node that isn't the active router gateway, it doesn't // always indicate a problem. n.logger.Debug("OVN router external IP address unreachable", logger.Ctx{"ip": ip.String(), "err": err}) }() } } // startUplinkPortPhysical creates OVS bridge (if doesn't exist) and connects uplink interface to the OVS bridge. func (n *ovn) startUplinkPortPhysical(uplinkNet Network) error { // Do this after gaining lock so that on failure we revert before release locking. reverter := revert.New() defer reverter.Fail() uplinkConfig := uplinkNet.Config() uplinkHostName := GetHostDevice(uplinkConfig["parent"], uplinkConfig["vlan"]) if !InterfaceExists(uplinkHostName) { return fmt.Errorf("Uplink network %q is not started (interface %q is missing)", uplinkNet.Name(), uplinkHostName) } // Detect if uplink interface is a native bridge. if IsNativeBridge(uplinkHostName) { return n.startUplinkPortBridgeNative(uplinkNet, uplinkHostName) } // Detect if uplink interface is a OVS bridge. vswitch, err := n.state.OVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } _, err = vswitch.GetBridge(context.TODO(), uplinkHostName) if err != nil && !errors.Is(err, ovs.ErrNotFound) { return err } else if err == nil { return n.startUplinkPortBridgeOVS(uplinkNet, uplinkHostName) } // If uplink is a normal physical interface, then use a separate OVS bridge and connect uplink to it. vars := n.uplinkPortBridgeVars(uplinkNet) // Check no global unicast IPs defined on uplink, as that may indicate it is in use by another application. addresses, _, err := InterfaceStatus(uplinkHostName) if err != nil { return fmt.Errorf("Failed getting interface status for %q: %w", uplinkHostName, err) } if len(addresses) > 0 { return fmt.Errorf("Cannot start network as uplink network interface %q has one or more IP addresses configured on it", uplinkHostName) } // Ensure correct sysctls are set on uplink interface to avoid getting IPv6 link-local addresses. err = localUtil.SysctlSet( fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", uplinkHostName), "1", fmt.Sprintf("net/ipv6/conf/%s/forwarding", uplinkHostName), "0", ) if err != nil { return fmt.Errorf("Failed to configure uplink interface %q: %w", uplinkHostName, err) } // Create uplink OVS bridge if needed. err = vswitch.CreateBridge(context.TODO(), vars.ovsBridge, true, nil, 0) if err != nil { return fmt.Errorf("Failed to create uplink OVS bridge %q: %w", vars.ovsBridge, err) } // Connect OVS end veth interface to OVS bridge. err = vswitch.CreateBridgePort(context.TODO(), vars.ovsBridge, uplinkHostName, true) if err != nil { return fmt.Errorf("Failed to connect uplink interface %q to uplink OVS bridge %q: %w", uplinkHostName, vars.ovsBridge, err) } // Associate OVS bridge to logical OVN provider. err = vswitch.AddOVNBridgeMapping(context.TODO(), vars.ovsBridge, uplinkNet.Name()) if err != nil { return fmt.Errorf("Failed to associate uplink OVS bridge %q to OVN provider %q: %w", vars.ovsBridge, uplinkNet.Name(), err) } // Bring uplink interface up. link := &ip.Link{Name: uplinkHostName} err = link.SetUp() if err != nil { return fmt.Errorf("Failed to bring up uplink interface %q: %w", uplinkHostName, err) } // Attempt to learn uplink MAC. n.pingOVNRouter() reverter.Success() return nil } // checkUplinkUse checks if uplink network is used by another OVN network. func (n *ovn) checkUplinkUse() (bool, error) { if n.config["network"] == "none" { return false, nil } // Get all managed networks across all projects. var err error var projectNetworks map[string]map[int64]api.Network err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { projectNetworks, err = tx.GetCreatedNetworks(ctx) return err }) if err != nil { return false, fmt.Errorf("Failed to load all networks: %w", err) } for projectName, networks := range projectNetworks { for _, network := range networks { if (projectName == n.project && network.Name == n.name) || network.Type != "ovn" { continue // Ignore our own DB record or non OVN networks. } // Check if another network is using our uplink. if network.Config["network"] == n.config["network"] { return true, nil } } } return false, nil } // deleteUplinkPort deletes the uplink connection. func (n *ovn) deleteUplinkPort() error { if n.config["network"] == "none" { return nil } // Uplink network must be in default project. if n.config["network"] != "" { uplinkNet, err := LoadByName(n.state, api.ProjectDefaultName, n.config["network"]) if err != nil { return fmt.Errorf("Failed loading uplink network %q: %w", n.config["network"], err) } // Lock uplink network so we don't race each other networks using the OVS uplink bridge. unlock, err := locking.Lock(context.TODO(), n.uplinkOperationLockName(uplinkNet)) if err != nil { return err } defer unlock() switch uplinkNet.Type() { case "bridge": return n.deleteUplinkPortBridge(uplinkNet) case "physical": return n.deleteUplinkPortPhysical(uplinkNet) } return fmt.Errorf("Failed deleting uplink port, network type %q unsupported as OVN uplink", uplinkNet.Type()) } return nil } // deleteUplinkPortBridge disconnects the uplink port from the bridge and performs any cleanup. func (n *ovn) deleteUplinkPortBridge(uplinkNet Network) error { if uplinkNet.Config()["bridge.driver"] != "openvswitch" { return n.deleteUplinkPortBridgeNative(uplinkNet) } return n.deleteUplinkPortBridgeOVS(uplinkNet, uplinkNet.Name()) } // deleteUplinkPortBridge deletes uplink OVS bridge, OVN bridge mappings and veth interfaces if not in use. func (n *ovn) deleteUplinkPortBridgeNative(uplinkNet Network) error { // Check OVS uplink bridge exists, if it does, check whether the uplink network is in use. removeVeths := false vars := n.uplinkPortBridgeVars(uplinkNet) if InterfaceExists(vars.ovsBridge) { uplinkUsed, err := n.checkUplinkUse() if err != nil { return err } // Remove OVS bridge if the uplink network isn't used by any other OVN networks. if !uplinkUsed { removeVeths = true vswitch, err := n.state.OVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } err = vswitch.RemoveOVNBridgeMapping(context.TODO(), vars.ovsBridge, uplinkNet.Name()) if err != nil { return err } err = vswitch.DeleteBridge(context.TODO(), vars.ovsBridge) if err != nil { return err } } } else { removeVeths = true // Remove the veths if OVS bridge already gone. } // Remove the veth interfaces if they exist. if removeVeths { if InterfaceExists(vars.uplinkEnd) { link := &ip.Link{Name: vars.uplinkEnd} err := link.Delete() if err != nil { return fmt.Errorf("Failed to delete the uplink veth interface %q: %w", vars.uplinkEnd, err) } } if InterfaceExists(vars.ovsEnd) { link := &ip.Link{Name: vars.ovsEnd} err := link.Delete() if err != nil { return fmt.Errorf("Failed to delete the uplink veth interface %q: %w", vars.ovsEnd, err) } } } return nil } // deleteUplinkPortBridge deletes OVN bridge mappings if not in use. func (n *ovn) deleteUplinkPortBridgeOVS(uplinkNet Network, ovsBridge string) error { uplinkUsed, err := n.checkUplinkUse() if err != nil { return err } // Remove uplink OVS bridge mapping if not in use by other OVN networks. if !uplinkUsed { vswitch, err := n.state.OVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } err = vswitch.RemoveOVNBridgeMapping(context.TODO(), ovsBridge, uplinkNet.Name()) if err != nil { return err } } return nil } // deleteUplinkPortPhysical deletes uplink OVS bridge and OVN bridge mappings if not in use. func (n *ovn) deleteUplinkPortPhysical(uplinkNet Network) error { uplinkConfig := uplinkNet.Config() uplinkHostName := GetHostDevice(uplinkConfig["parent"], uplinkConfig["vlan"]) // Detect if uplink interface is a native bridge. if IsNativeBridge(uplinkHostName) { return n.deleteUplinkPortBridgeNative(uplinkNet) } // Detect if uplink interface is a OVS bridge. vswitch, err := n.state.OVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } _, err = vswitch.GetBridge(context.TODO(), uplinkHostName) if err != nil && !errors.Is(err, ovs.ErrNotFound) { return err } else if err == nil { return n.deleteUplinkPortBridgeOVS(uplinkNet, uplinkHostName) } // Otherwise if uplink is normal physical interface, attempt cleanup of OVS bridge. // Check OVS uplink bridge exists, if it does, check whether the uplink network is in use. releaseIF := false vars := n.uplinkPortBridgeVars(uplinkNet) if InterfaceExists(vars.ovsBridge) { uplinkUsed, err := n.checkUplinkUse() if err != nil { return err } // Remove OVS bridge if the uplink network isn't used by any other OVN networks. if !uplinkUsed { releaseIF = true err = vswitch.RemoveOVNBridgeMapping(context.TODO(), vars.ovsBridge, uplinkNet.Name()) if err != nil { return err } err = vswitch.DeleteBridge(context.TODO(), vars.ovsBridge) if err != nil { return err } } } else { releaseIF = true // Bring uplink interface down if not needed. } // Bring down uplink interface if not used and exists. if releaseIF && InterfaceExists(uplinkHostName) { link := &ip.Link{Name: uplinkHostName} err := link.SetDown() if err != nil { return fmt.Errorf("Failed to bring down uplink interface %q: %w", uplinkHostName, err) } } return nil } // FillConfig fills requested config with any default values. func (n *ovn) FillConfig(config map[string]string) error { if config["ipv4.address"] == "" { config["ipv4.address"] = "auto" } if config["ipv6.address"] == "" { content, err := os.ReadFile("/proc/sys/net/ipv6/conf/default/disable_ipv6") if err == nil && string(content) == "0\n" { config["ipv6.address"] = "auto" } } // Now replace any "auto" keys with generated values. err := n.populateAutoConfig(config) if err != nil { return fmt.Errorf("Failed generating auto config: %w", err) } return nil } // populateAutoConfig replaces "auto" in config with generated values. func (n *ovn) populateAutoConfig(config map[string]string) error { changedConfig := false if config["ipv4.address"] == "auto" { subnet, err := randomSubnetV4() if err != nil { return err } config["ipv4.address"] = subnet if config["ipv4.nat"] == "" { config["ipv4.nat"] = "true" } changedConfig = true } if config["ipv6.address"] == "auto" { subnet, err := randomSubnetV6() if err != nil { return err } config["ipv6.address"] = subnet if config["ipv6.nat"] == "" { config["ipv6.nat"] = "true" } changedConfig = true } // Re-validate config if changed. if changedConfig && n.state != nil { return n.Validate(config, request.ClientTypeNormal) } return nil } // Create sets up network in OVN Northbound database. func (n *ovn) Create(clientType request.ClientType) error { n.logger.Debug("Create", logger.Ctx{"clientType": clientType, "config": n.config}) // We only need to setup the OVN Northbound database once, not on every clustered node. if clientType == request.ClientTypeNormal { err := n.setup(false) if err != nil { return err } } return nil } // allowedUplinkNetworks returns a list of allowed networks to use as uplinks based on project restrictions. func (n *ovn) allowedUplinkNetworks(p *api.Project) ([]string, error) { var uplinkNetworkNames []string err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Uplink networks are always from the default project. networks, err := tx.GetCreatedNetworksByProject(ctx, api.ProjectDefaultName) if err != nil { return fmt.Errorf("Failed getting uplink networks: %w", err) } // Add any compatible networks to the uplink network list. for _, network := range networks { if network.Type == "bridge" || network.Type == "physical" { uplinkNetworkNames = append(uplinkNetworkNames, network.Name) } } return nil }) if err != nil { return nil, err } // If project is not restricted, return full network list. if util.IsFalseOrEmpty(p.Config["restricted"]) { return uplinkNetworkNames, nil } allowedUplinkNetworkNames := []string{} // There are no allowed networks if restricted.networks.uplinks is not set. if p.Config["restricted.networks.uplinks"] == "" { return allowedUplinkNetworkNames, nil } // Parse the allowed uplinks and return any that are present in the actual defined networks. allowedRestrictedUplinks := util.SplitNTrimSpace(p.Config["restricted.networks.uplinks"], ",", -1, false) for _, allowedRestrictedUplink := range allowedRestrictedUplinks { if slices.Contains(uplinkNetworkNames, allowedRestrictedUplink) { allowedUplinkNetworkNames = append(allowedUplinkNetworkNames, allowedRestrictedUplink) } } return allowedUplinkNetworkNames, nil } // validateUplinkNetwork checks if uplink network is allowed, and if empty string is supplied then tries to select // an uplink network from the allowedUplinkNetworks() list if there is only one allowed network. // Returns chosen uplink network name to use. func (n *ovn) validateUplinkNetwork(p *api.Project, uplinkNetworkName string) (string, error) { allowedUplinkNetworks, err := n.allowedUplinkNetworks(p) if err != nil { return "", err } if uplinkNetworkName != "" { if !slices.Contains(allowedUplinkNetworks, uplinkNetworkName) { return "", fmt.Errorf(`Option "network" value %q is not one of the allowed uplink networks in project`, uplinkNetworkName) } return uplinkNetworkName, nil } allowedNetworkCount := len(allowedUplinkNetworks) if allowedNetworkCount == 0 { return "", errors.New(`No allowed uplink networks in project`) } else if allowedNetworkCount == 1 { // If there is only one allowed uplink network then use it if not specified by user. return allowedUplinkNetworks[0], nil } return "", errors.New(`Option "network" is required`) } // getDHCPv4Reservations returns list DHCP IPv4 reservations from NICs connected to this network. func (n *ovn) getDHCPv4Reservations() ([]iprange.Range, error) { routerIntPortIPv4, ipv4Net, err := n.parseRouterIntPortIPv4Net() if err != nil { return nil, fmt.Errorf("Failed parsing router's internal port IPv4 Net for DHCP reservation: %w", err) } var dhcpReserveIPv4s []iprange.Range if routerIntPortIPv4 != nil { if n.config["ipv4.dhcp.ranges"] == "" { dhcpReserveIPv4s = []iprange.Range{{Start: routerIntPortIPv4}, {Start: dhcpalloc.GetIP(ipv4Net, -2)}} } else { allowedNets := []*net.IPNet{n.DHCPv4Subnet()} dhcpRanges, err := parseIPRanges(n.config["ipv4.dhcp.ranges"], allowedNets...) if err != nil { return nil, err } sort.Slice(dhcpRanges, func(i, j int) bool { return bytes.Compare(dhcpRanges[i].Start, dhcpRanges[j].Start) < 0 }) reserverdIPs, err := complementRanges(dhcpRanges, ipv4Net) if err != nil { return nil, err } dhcpReserveIPv4s = append(dhcpReserveIPv4s, reserverdIPs...) if !ipInRanges(routerIntPortIPv4, dhcpReserveIPv4s) { dhcpReserveIPv4s = append(dhcpReserveIPv4s, iprange.Range{Start: routerIntPortIPv4}) } // Convert the 4-byte IPv4 address returned by 'dhcpalloc.GetIP' to a 16-byte form // using net.ParseIP, to ensure compatibility with other IPs stored in 16-byte format. // This is necessary because direct comparison with 4-byte IPs would fail. ovnRouter := net.ParseIP(dhcpalloc.GetIP(ipv4Net, -2).String()) if !ipInRanges(ovnRouter, dhcpReserveIPv4s) { dhcpReserveIPv4s = append(dhcpReserveIPv4s, iprange.Range{Start: ovnRouter}) } } } err = UsedByInstanceDevices(n.state, n.Project(), n.Name(), n.Type(), func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error { ip := net.ParseIP(nicConfig["ipv4.address"]) if ip != nil { if !ipInRanges(ip, dhcpReserveIPv4s) { dhcpReserveIPv4s = append(dhcpReserveIPv4s, iprange.Range{Start: ip}) } } return nil }) if err != nil { return nil, err } return dhcpReserveIPv4s, nil } func (n *ovn) setup(update bool) error { // If we are in mock mode, just no-op. if n.state.OS.MockMode { return nil } n.logger.Debug("Setting up network") reverter := revert.New() defer reverter.Fail() var routerExtPortIPv4, routerExtPortIPv6 net.IP var routerExtPortIPv4Net, routerExtPortIPv6Net *net.IPNet // Record updated config so we can store back into DB and n.config variable. updatedConfig := make(map[string]string) // Load the project to get uplink network restrictions. var p *api.Project var projectID int64 err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { project, err := dbCluster.GetProject(ctx, tx.Tx(), n.project) if err != nil { return err } projectID = int64(project.ID) p, err = project.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return fmt.Errorf("Failed to load network restrictions from project %q: %w", n.project, err) } // Check project restrictions and get uplink network to use. uplinkNetwork := "none" if n.config["network"] != "none" { uplinkNetwork, err = n.validateUplinkNetwork(p, n.config["network"]) if err != nil { return err } } // Ensure automatically selected uplink network is saved into "network" key. if uplinkNetwork != n.config["network"] { updatedConfig["network"] = uplinkNetwork } // Get bridge MTU to use. bridgeMTU := n.getBridgeMTU() if bridgeMTU == 0 { // If no manual bridge MTU specified, derive it from the underlay network. bridgeMTU, err = n.getOptimalBridgeMTU() if err != nil { return fmt.Errorf("Failed getting optimal bridge MTU: %w", err) } // Save to config so the value can be read by instances connecting to network. updatedConfig["bridge.mtu"] = fmt.Sprintf("%d", bridgeMTU) } // Get a list of all NICs connected to this network that have static DHCP IPv4 reservations. dhcpReserveIPv4s, err := n.getDHCPv4Reservations() if err != nil { return fmt.Errorf("Failed getting DHCPv4 IP reservations: %w", err) } // Apply any config dynamically generated to the current config and store back to DB in single transaction. if len(updatedConfig) > 0 { maps.Copy(n.config, updatedConfig) err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { err = tx.UpdateNetwork(ctx, n.project, n.name, n.description, n.config) if err != nil { return fmt.Errorf("Failed saving updated network config: %w", err) } return nil }) if err != nil { return err } } // Get router MAC address. routerMAC, err := n.getRouterMAC() if err != nil { return err } // Setup uplink port (do this first to check uplink is suitable). uplinkNet, err := n.setupUplinkPort(routerMAC) if err != nil { return err } // Parse router IP config. if uplinkNet != nil && uplinkNet.routerExtPortIPv4Net != "" { routerExtPortIPv4, routerExtPortIPv4Net, err = net.ParseCIDR(uplinkNet.routerExtPortIPv4Net) if err != nil { return fmt.Errorf("Failed parsing router's external uplink port IPv4 Net: %w", err) } } if uplinkNet != nil && uplinkNet.routerExtPortIPv6Net != "" { routerExtPortIPv6, routerExtPortIPv6Net, err = net.ParseCIDR(uplinkNet.routerExtPortIPv6Net) if err != nil { return fmt.Errorf("Failed parsing router's external uplink port IPv6 Net: %w", err) } } routerIntPortIPv4, routerIntPortIPv4Net, err := n.parseRouterIntPortIPv4Net() if err != nil { return fmt.Errorf("Failed parsing router's internal port IPv4 Net: %w", err) } routerIntPortIPv6, routerIntPortIPv6Net, err := n.parseRouterIntPortIPv6Net() if err != nil { return fmt.Errorf("Failed parsing router's internal port IPv6 Net: %w", err) } if n.config["network"] != "none" && routerIntPortIPv4 == nil && routerIntPortIPv6 == nil { return errors.New("IPv4 or IPv6 subnets must be specified on a non-isolated OVN network") } // Create chassis group. err = n.ovnnb.CreateChassisGroup(context.TODO(), n.getChassisGroupName(), update) if err != nil { return err } if !update { reverter.Add(func() { _ = n.ovnnb.DeleteChassisGroup(context.TODO(), n.getChassisGroupName()) }) } // Configure logical router. if routerIntPortIPv4 != nil || routerIntPortIPv6 != nil { // Create logical router. err = n.ovnnb.CreateLogicalRouter(context.TODO(), n.getRouterName(), update) if err != nil { return fmt.Errorf("Failed adding router: %w", err) } if !update { reverter.Add(func() { _ = n.ovnnb.DeleteLogicalRouter(context.TODO(), n.getRouterName()) }) } } else { err := n.ovnnb.DeleteLogicalRouter(context.TODO(), n.getRouterName()) if err != nil && !errors.Is(err, networkOVN.ErrNotFound) { return fmt.Errorf("Failed deleting router: %w", err) } } // Generate external router port IPs (in CIDR format). extRouterIPs := []*net.IPNet{} if routerExtPortIPv4Net != nil { extRouterIPs = append(extRouterIPs, &net.IPNet{ IP: routerExtPortIPv4, Mask: routerExtPortIPv4Net.Mask, }) } if routerExtPortIPv6Net != nil { extRouterIPs = append(extRouterIPs, &net.IPNet{ IP: routerExtPortIPv6, Mask: routerExtPortIPv6Net.Mask, }) } if len(extRouterIPs) > 0 { err = n.ovnnb.CreateLogicalSwitch(context.TODO(), n.getExtSwitchName(), update) if err != nil { return fmt.Errorf("Failed adding external switch: %w", err) } if !update { reverter.Add(func() { _ = n.ovnnb.DeleteLogicalSwitch(context.TODO(), n.getExtSwitchName()) }) } // Create external router port. err = n.ovnnb.CreateLogicalRouterPort(context.TODO(), n.getRouterName(), n.getRouterExtPortName(), routerMAC, bridgeMTU, extRouterIPs, n.getChassisGroupName(), update) if err != nil { return fmt.Errorf("Failed adding external router port: %w", err) } if !update { reverter.Add(func() { _ = n.ovnnb.DeleteLogicalRouterPort(context.TODO(), n.getRouterName(), n.getRouterExtPortName()) }) } // Create external switch port and link to router port. err = n.ovnnb.CreateLogicalSwitchPort(context.TODO(), n.getExtSwitchName(), n.getExtSwitchRouterPortName(), nil, update) if err != nil { return fmt.Errorf("Failed adding external switch router port: %w", err) } if !update { reverter.Add(func() { _ = n.ovnnb.DeleteLogicalSwitchPort(context.TODO(), n.getExtSwitchName(), n.getExtSwitchRouterPortName()) }) } err = n.ovnnb.UpdateLogicalSwitchPortLinkRouter(context.TODO(), n.getExtSwitchRouterPortName(), n.getRouterExtPortName()) if err != nil { return fmt.Errorf("Failed linking external router port to external switch port: %w", err) } // Create external switch port and link to external provider network. err = n.ovnnb.CreateLogicalSwitchPort(context.TODO(), n.getExtSwitchName(), n.getExtSwitchProviderPortName(), nil, update) if err != nil { return fmt.Errorf("Failed adding external switch provider port: %w", err) } if !update { reverter.Add(func() { _ = n.ovnnb.DeleteLogicalSwitchPort(context.TODO(), n.getExtSwitchName(), n.getExtSwitchProviderPortName()) }) } if uplinkNet != nil { err = n.ovnnb.UpdateLogicalSwitchPortLinkProviderNetwork(context.TODO(), n.getExtSwitchProviderPortName(), uplinkNet.extSwitchProviderName) if err != nil { return fmt.Errorf("Failed linking external switch provider port to external provider network: %w", err) } } // Remove any existing SNAT rules on update. As currently these are only defined from the network // config rather than from any instance NIC config, so we can re-create the active config below. if update { err = n.ovnnb.DeleteLogicalRouterNAT(context.TODO(), n.getRouterName(), "snat", true) if err != nil { return fmt.Errorf("Failed removing existing router SNAT rules: %w", err) } } // Add SNAT rules. if util.IsTrue(n.config["ipv4.nat"]) && routerIntPortIPv4Net != nil && routerExtPortIPv4 != nil { snatIP := routerExtPortIPv4 if n.config["ipv4.nat.address"] != "" { snatIP = net.ParseIP(n.config["ipv4.nat.address"]) if snatIP == nil { return fmt.Errorf("Failed parsing %q", "ipv4.nat.address") } } err = n.ovnnb.CreateLogicalRouterNAT(context.TODO(), n.getRouterName(), "snat", routerIntPortIPv4Net, snatIP, nil, false, update) if err != nil { return fmt.Errorf("Failed adding router IPv4 SNAT rule: %w", err) } } if util.IsTrue(n.config["ipv6.nat"]) && routerIntPortIPv6Net != nil && routerExtPortIPv6 != nil { snatIP := routerExtPortIPv6 if n.config["ipv6.nat.address"] != "" { snatIP = net.ParseIP(n.config["ipv6.nat.address"]) if snatIP == nil { return fmt.Errorf("Failed parsing %q", "ipv6.nat.address") } } err = n.ovnnb.CreateLogicalRouterNAT(context.TODO(), n.getRouterName(), "snat", routerIntPortIPv6Net, snatIP, nil, false, update) if err != nil { return fmt.Errorf("Failed adding router IPv6 SNAT rule: %w", err) } } // Check if uplink network states its gateway mac for static MAC binding. if uplinkNet != nil && n.config["network"] != "none" { // Load the uplink network. uplinkNetworkObj, err := LoadByName(n.state, api.ProjectDefaultName, n.config["network"]) if err != nil { return fmt.Errorf("Failed loading uplink network %q: %w", n.config["network"], err) } uplinkConfig := uplinkNetworkObj.Config() // Handle IPv4 MAC. if uplinkConfig["ipv4.gateway.hwaddr"] != "" { // Set a static MAc binding for the gateway's IP and MAC. uplinkGatewayIP, _, err := net.ParseCIDR(uplinkConfig["ipv4.gateway"]) if err != nil { return err } uplinkGatewayMAC, err := net.ParseMAC(uplinkConfig["ipv4.gateway.hwaddr"]) if err != nil { return err } err = n.ovnnb.CreateStaticMACBinding(context.TODO(), n.getRouterExtPortName(), uplinkGatewayIP, uplinkGatewayMAC, true) if err != nil { return err } } // Handle IPv6 MAC. if uplinkConfig["ipv6.gateway.hwaddr"] != "" { // Set a static MAc binding for the gateway's IP and MAC. uplinkGatewayIP, _, err := net.ParseCIDR(uplinkConfig["ipv6.gateway"]) if err != nil { return err } uplinkGatewayMAC, err := net.ParseMAC(uplinkConfig["ipv6.gateway.hwaddr"]) if err != nil { return err } err = n.ovnnb.CreateStaticMACBinding(context.TODO(), n.getRouterExtPortName(), uplinkGatewayIP, uplinkGatewayMAC, true) if err != nil { return err } } // Clear any leftover MAC binding. err = n.ovnnb.DeleteStaticMACBindings(context.TODO(), n.getRouterExtPortName(), uplinkConfig["ipv4.gateway.hwaddr"] == "", uplinkConfig["ipv6.gateway.hwaddr"] == "") if err != nil { return err } } // Clear default routes (if existing) and re-apply based on current config. defaultIPv4Route := net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)} defaultIPv6Route := net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)} deleteRoutes := []net.IPNet{defaultIPv4Route, defaultIPv6Route} defaultRoutes := make([]networkOVN.OVNRouterRoute, 0, 2) currentRoutes, err := n.ovnnb.GetLogicalRouterRoutes(context.TODO(), n.getRouterName()) if err != nil { return fmt.Errorf("Failed to retrieve currently set routes on network %s", n.name) } if routerIntPortIPv4Net != nil { // If l3only mode is enabled then each instance IPv4 will get its own /32 route added when // the instance NIC starts. However to stop packets toward unknown IPs within the internal // subnet escaping onto the uplink network we add a less specific discard route for the // whole internal subnet. if util.IsTrue(n.config["ipv4.l3only"]) { route := networkOVN.OVNRouterRoute{ Prefix: *routerIntPortIPv4Net, Discard: true, } if !ovnRouteExists(currentRoutes, route) { defaultRoutes = append(defaultRoutes, route) } } else { deleteRoutes = append(deleteRoutes, *routerIntPortIPv4Net) } } if routerIntPortIPv6Net != nil { // If l3only mode is enabled then each instance IPv6 will get its own /128 route added when // the instance NIC starts. However to stop packets toward unknown IPs within the internal // subnet escaping onto the uplink network we add a less specific discard route for the // whole internal subnet. if util.IsTrue(n.config["ipv6.l3only"]) { route := networkOVN.OVNRouterRoute{ Prefix: *routerIntPortIPv6Net, Discard: true, } if !ovnRouteExists(currentRoutes, route) { defaultRoutes = append(defaultRoutes, route) } } else { deleteRoutes = append(deleteRoutes, *routerIntPortIPv6Net) } } if uplinkNet.routerExtGwIPv4 != nil { defaultRoutes = append(defaultRoutes, networkOVN.OVNRouterRoute{ Prefix: defaultIPv4Route, NextHop: uplinkNet.routerExtGwIPv4, Port: n.getRouterExtPortName(), }) } if uplinkNet.routerExtGwIPv6 != nil { defaultRoutes = append(defaultRoutes, networkOVN.OVNRouterRoute{ Prefix: defaultIPv6Route, NextHop: uplinkNet.routerExtGwIPv6, Port: n.getRouterExtPortName(), }) } if len(deleteRoutes) > 0 { err = n.ovnnb.DeleteLogicalRouterRoute(context.TODO(), n.getRouterName(), deleteRoutes...) if err != nil { return fmt.Errorf("Failed removing default routes: %w", err) } } if len(defaultRoutes) > 0 { err = n.ovnnb.CreateLogicalRouterRoute(context.TODO(), n.getRouterName(), update, defaultRoutes...) if err != nil { return fmt.Errorf("Failed adding default routes: %w", err) } } } // Gather internal router port IPs (in CIDR format). intRouterIPs := []*net.IPNet{} intSubnets := []net.IPNet{} if routerIntPortIPv4Net != nil { intRouterIPNet := &net.IPNet{ IP: routerIntPortIPv4, Mask: routerIntPortIPv4Net.Mask, } // In l3only mode the router's internal IP has a /32 mask instead of the internal subnet's mask. if util.IsTrue(n.config["ipv4.l3only"]) { intRouterIPNet.Mask = net.CIDRMask(32, 32) } intRouterIPs = append(intRouterIPs, intRouterIPNet) intSubnets = append(intSubnets, *routerIntPortIPv4Net) } if routerIntPortIPv6Net != nil { intRouterIPNet := &net.IPNet{ IP: routerIntPortIPv6, Mask: routerIntPortIPv6Net.Mask, } // In l3only mode the router's internal IP has a /128 mask instead of the internal subnet's mask. if util.IsTrue(n.config["ipv6.l3only"]) { intRouterIPNet.Mask = net.CIDRMask(128, 128) } intRouterIPs = append(intRouterIPs, intRouterIPNet) intSubnets = append(intSubnets, *routerIntPortIPv6Net) } // Create internal logical switch if not updating. err = n.ovnnb.CreateLogicalSwitch(context.TODO(), n.getIntSwitchName(), update) if err != nil { return fmt.Errorf("Failed adding internal switch: %w", err) } if !update { reverter.Add(func() { _ = n.ovnnb.DeleteLogicalSwitch(context.TODO(), n.getIntSwitchName()) }) } // Add any listed existing external interface. if n.config["bridge.external_interfaces"] != "" { for _, entry := range strings.Split(n.config["bridge.external_interfaces"], ",") { entry = strings.TrimSpace(entry) // Test for extended configuration of external interface. entryParts := strings.Split(entry, "/") ifParent := "" vlanID := 0 if len(entryParts) == 3 { vlanID, err = strconv.Atoi(entryParts[2]) if err != nil || vlanID < 1 || vlanID > 4094 { vlanID = 0 n.logger.Warn("Ignoring invalid VLAN ID", logger.Ctx{"interface": entry, "vlanID": entryParts[2]}) } else { entry = strings.TrimSpace(entryParts[0]) ifParent = strings.TrimSpace(entryParts[1]) } } iface, err := net.InterfaceByName(entry) if err != nil { if vlanID == 0 { n.logger.Warn("Skipping attaching missing external interface", logger.Ctx{"interface": entry}) continue } // If the interface doesn't exist and VLAN ID was provided, create the missing interface. ok, err := VLANInterfaceCreate(ifParent, entry, strconv.Itoa(vlanID), false) if ok { iface, err = net.InterfaceByName(entry) } if !ok || err != nil { return fmt.Errorf("Failed to create external interface %q", entry) } } else if vlanID > 0 { // If the interface exists and VLAN ID was provided, ensure it has the same parent and VLAN ID and is not attached to a different network. linkInfo, err := ip.LinkByName(entry) if err != nil { return fmt.Errorf("Failed to get link info for external interface %q", entry) } if linkInfo.Kind != "vlan" || linkInfo.Parent != ifParent || linkInfo.VlanID != vlanID || (linkInfo.Master != "" && linkInfo.Master != n.name) { return fmt.Errorf("External interface %q already in use", entry) } } unused := true addrs, err := iface.Addrs() if err == nil { for _, addr := range addrs { ipAddr, _, err := net.ParseCIDR(addr.String()) if ipAddr != nil && err == nil && ipAddr.IsGlobalUnicast() { unused = false break } } } if !unused { return errors.New("Only unconfigured network interfaces can be bridged") } lspName := networkOVN.OVNSwitchPort(fmt.Sprintf("%s-external-n%d-%s", n.getNetworkPrefix(), n.state.DB.Cluster.GetNodeID(), entry)) err = n.ovnnb.CreateLogicalSwitchPort(context.TODO(), n.getIntSwitchName(), lspName, &networkOVN.OVNSwitchPortOpts{ IPV4: "none", IPV6: "none", Promiscuous: true, }, false) if err != nil { return fmt.Errorf("Failed to create logical switch port for %s: %w", entry, err) } reverter.Add(func() { _ = n.ovnnb.DeleteLogicalSwitchPort(context.TODO(), n.getIntSwitchName(), lspName) }) // Attach host side veth interface to bridge. integrationBridge := n.state.GlobalConfig.NetworkOVNIntegrationBridge() vswitch, err := n.state.OVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } err = vswitch.CreateBridgePort(context.TODO(), integrationBridge, entry, true) if err != nil { return err } reverter.Add(func() { _ = vswitch.DeleteBridgePort(context.TODO(), integrationBridge, entry) }) // Link OVS port to OVN logical port. err = vswitch.AssociateInterfaceOVNSwitchPort(context.TODO(), entry, string(lspName)) if err != nil { return err } // Make sure the port is up. link := &ip.Link{Name: entry} err = link.SetUp() if err != nil { return fmt.Errorf("Failed to bring up the host interface %s: %w", entry, err) } } } // Setup IP allocation config on logical switch. err = n.ovnnb.UpdateLogicalSwitchIPAllocation(context.TODO(), n.getIntSwitchName(), &networkOVN.OVNIPAllocationOpts{ PrefixIPv4: routerIntPortIPv4Net, PrefixIPv6: routerIntPortIPv6Net, ExcludeIPv4: dhcpReserveIPv4s, }) if err != nil { return fmt.Errorf("Failed setting IP allocation settings on internal switch: %w", err) } // Create internal switch address sets and add subnets to address set. if update { err = n.ovnnb.UpdateAddressSetAdd(context.TODO(), acl.OVNIntSwitchPortGroupAddressSetPrefix(n.ID()), intSubnets...) if err != nil { return fmt.Errorf("Failed adding internal subnet address set entries: %w", err) } } else { err = n.ovnnb.CreateAddressSet(context.TODO(), acl.OVNIntSwitchPortGroupAddressSetPrefix(n.ID()), intSubnets...) if err != nil { return fmt.Errorf("Failed creating internal subnet address set entries: %w", err) } reverter.Add(func() { _ = n.ovnnb.DeleteAddressSet(context.TODO(), acl.OVNIntSwitchPortGroupAddressSetPrefix(n.ID())) }) } if routerIntPortIPv4 != nil || routerIntPortIPv6 != nil { // Apply router security policy. err = n.logicalRouterPolicySetup(n.ovnnb) if err != nil { return fmt.Errorf("Failed applying router security policy: %w", err) } // Create internal router port. err = n.ovnnb.CreateLogicalRouterPort(context.TODO(), n.getRouterName(), n.getRouterIntPortName(), routerMAC, bridgeMTU, intRouterIPs, "", update) if err != nil { return fmt.Errorf("Failed adding internal router port: %w", err) } if !update { reverter.Add(func() { _ = n.ovnnb.DeleteLogicalRouterPort(context.TODO(), n.getRouterName(), n.getRouterIntPortName()) }) } } else { err := n.ovnnb.DeleteLogicalRouterPort(context.TODO(), n.getRouterName(), n.getRouterIntPortName()) if err != nil && !errors.Is(err, ovs.ErrNotFound) { return fmt.Errorf("Failed deleting logical router port: %w", err) } } // Configure DHCP option sets. var dhcpv4UUID, dhcpv6UUID networkOVN.OVNDHCPOptionsUUID dhcpV4Subnet := n.DHCPv4Subnet() dhcpV6Subnet := n.DHCPv6Subnet() if update { dhcpv4UUID, dhcpv6UUID, err = n.getDhcpOptionUUIDs() if err != nil { return err } var deleteDHCPRecords []networkOVN.OVNDHCPOptionsUUID if dhcpV4Subnet == nil && dhcpv4UUID != "" { deleteDHCPRecords = append(deleteDHCPRecords, dhcpv4UUID) } if dhcpV6Subnet == nil && dhcpv6UUID != "" { deleteDHCPRecords = append(deleteDHCPRecords, dhcpv6UUID) } if len(deleteDHCPRecords) > 0 { err = n.ovnnb.DeleteLogicalSwitchDHCPOption(context.TODO(), n.getIntSwitchName(), deleteDHCPRecords...) if err != nil { return fmt.Errorf("Failed deleting existing DHCP settings for internal switch: %w", err) } } } var dnsIPv4 []net.IP var dnsIPv6 []net.IP if n.config["dns.nameservers"] != "" { for _, s := range util.SplitNTrimSpace(n.config["dns.nameservers"], ",", -1, false) { nsIP := net.ParseIP(s) if nsIP.To4() != nil { dnsIPv4 = append(dnsIPv4, nsIP) } else { dnsIPv6 = append(dnsIPv6, nsIP) } } } else { if uplinkNet != nil { dnsIPv4 = uplinkNet.dnsIPv4 dnsIPv6 = uplinkNet.dnsIPv6 } if len(dnsIPv4) == 0 { dnsIPv4 = []net.IP{routerIntPortIPv4} } if len(dnsIPv6) == 0 { dnsIPv6 = []net.IP{routerIntPortIPv6} } } var dhcpv4Created, dhcpv6Created bool // Create DHCPv4 options for internal switch. if dhcpV4Subnet != nil { // In l3only mode we configure the DHCPv4 server to request the instances use a /32 subnet mask. var dhcpV4Netmask string if util.IsTrue(n.config["ipv4.l3only"]) { dhcpV4Netmask = "255.255.255.255" } leaseTime := time.Hour * 1 if n.config["ipv4.dhcp.expiry"] != "" { duration, err := time.ParseDuration(n.config["ipv4.dhcp.expiry"]) if err != nil { return fmt.Errorf("Failed to parse expiry: %w", err) } leaseTime = duration } dhcpV4Gateway, err := n.getGatewayIpv4() if err != nil { return fmt.Errorf("Failed parsing router's internal port IPv4 Net: %w", err) } opts := &networkOVN.OVNDHCPv4Opts{ ServerID: routerIntPortIPv4, ServerMAC: routerMAC, Router: dhcpV4Gateway, DomainName: n.getDomainName(), LeaseTime: leaseTime, MTU: bridgeMTU, Netmask: dhcpV4Netmask, DNSSearchList: n.getDNSSearchList(), StaticRoutes: n.config["ipv4.dhcp.routes"], RecursiveDNSServer: dnsIPv4, } err = n.ovnnb.UpdateLogicalSwitchDHCPv4Options(context.TODO(), n.getIntSwitchName(), dhcpv4UUID, dhcpV4Subnet, opts) if err != nil { return fmt.Errorf("Failed adding DHCPv4 settings for internal switch: %w", err) } if dhcpv4UUID == "" { dhcpv4Created = true } } // Create DHCPv6 options for internal switch. if dhcpV6Subnet != nil { opts := &networkOVN.OVNDHCPv6Opts{ ServerID: routerMAC, DNSSearchList: n.getDNSSearchList(), RecursiveDNSServer: dnsIPv6, DHCPv6Stateless: util.IsFalseOrEmpty(n.config["ipv6.dhcp.stateful"]), } err = n.ovnnb.UpdateLogicalSwitchDHCPv6Options(context.TODO(), n.getIntSwitchName(), dhcpv6UUID, dhcpV6Subnet, opts) if err != nil { return fmt.Errorf("Failed adding DHCPv6 settings for internal switch: %w", err) } if dhcpv6UUID == "" { dhcpv6Created = true } } if update && (dhcpv4Created || dhcpv6Created) { dhcpv4UUID, dhcpv6UUID, err = n.getDhcpOptionUUIDs() if err != nil { return err } ports, err := n.ovnnb.GetLogicalSwitchPorts(context.TODO(), n.getIntSwitchName()) if err != nil { return err } for portName := range ports { err := n.ovnnb.UpdateLogicalSwitchPortDHCP(context.TODO(), portName, dhcpv4UUID, dhcpv6UUID) if err != nil { return err } } } // Set IPv6 router advertisement settings. if routerIntPortIPv6Net != nil { adressMode := networkOVN.OVNIPv6AddressModeSLAAC if dhcpV6Subnet != nil { adressMode = networkOVN.OVNIPv6AddressModeDHCPStateless if util.IsTrue(n.config["ipv6.dhcp.stateful"]) { adressMode = networkOVN.OVNIPv6AddressModeDHCPStateful } } var recursiveDNSServer net.IP if len(dnsIPv6) > 0 { recursiveDNSServer = dnsIPv6[0] // OVN only supports 1 RA DNS server. } err = n.ovnnb.UpdateLogicalRouterPort(context.TODO(), n.getRouterIntPortName(), &networkOVN.OVNIPv6RAOpts{ AddressMode: adressMode, SendPeriodic: true, DNSSearchList: n.getDNSSearchList(), RecursiveDNSServer: recursiveDNSServer, MTU: bridgeMTU, // Keep these low until we support DNS search domains via DHCPv4, as otherwise RA DNSSL // won't take effect until advert after DHCPv4 has run on instance. MinInterval: time.Duration(time.Second * 30), MaxInterval: time.Duration(time.Minute * 1), }) if err != nil { return fmt.Errorf("Failed setting internal router port IPv6 advertisement settings: %w", err) } } else { err = n.ovnnb.UpdateLogicalRouterPort(context.TODO(), n.getRouterIntPortName(), &networkOVN.OVNIPv6RAOpts{}) if err != nil && !errors.Is(err, networkOVN.ErrNotFound) { return fmt.Errorf("Failed removing internal router port IPv6 advertisement settings: %w", err) } } // Create internal switch port and link to router port. if routerIntPortIPv4Net != nil || routerIntPortIPv6Net != nil { err = n.ovnnb.CreateLogicalSwitchPort(context.TODO(), n.getIntSwitchName(), n.getIntSwitchRouterPortName(), nil, update) if err != nil { return fmt.Errorf("Failed adding internal switch router port: %w", err) } if !update { reverter.Add(func() { _ = n.ovnnb.DeleteLogicalSwitchPort(context.TODO(), n.getIntSwitchName(), n.getIntSwitchRouterPortName()) }) } err = n.ovnnb.UpdateLogicalSwitchPortLinkRouter(context.TODO(), n.getIntSwitchRouterPortName(), n.getRouterIntPortName()) if err != nil { return fmt.Errorf("Failed linking internal router port to internal switch port: %w", err) } } else { err := n.ovnnb.DeleteLogicalSwitchPort(context.TODO(), n.getIntSwitchName(), n.getIntSwitchRouterPortName()) if err != nil && !errors.Is(err, networkOVN.ErrNotFound) { return fmt.Errorf("Failed removing logical switch port: %w", err) } } // Apply baseline ACL rules to internal logical switch. dnsServers := []net.IP{} if uplinkNet != nil { dnsServers = append(dnsServers, dnsIPv4...) dnsServers = append(dnsServers, dnsIPv6...) } err = acl.OVNApplyNetworkBaselineRules(n.ovnnb, n.getIntSwitchName(), n.getIntSwitchRouterPortName(), intRouterIPs, dnsServers) if err != nil { return fmt.Errorf("Failed applying baseline ACL rules to internal switch: %w", err) } // Create network port group if needed. err = n.ensureNetworkPortGroup(projectID) if err != nil { return fmt.Errorf("Failed to setup network port group: %w", err) } // Ensure any network assigned security ACL port groups are created ready for instance NICs to use. securityACLS := util.SplitNTrimSpace(n.config["security.acls"], ",", -1, true) if len(securityACLS) > 0 { var aclNameIDs map[string]int64 err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get map of ACL names to DB IDs (used for generating OVN port group names). acls, err := dbCluster.GetNetworkACLs(ctx, tx.Tx(), dbCluster.NetworkACLFilter{Project: &n.project}) if err != nil { return err } aclNameIDs = make(map[string]int64, len(acls)) for _, acl := range acls { aclNameIDs[acl.Name] = int64(acl.ID) } return nil }) if err != nil { return fmt.Errorf("Failed getting network ACL IDs for security ACL setup: %w", err) } // Request our network is setup with the specified ACLs. aclNets := map[string]acl.NetworkACLUsage{ n.Name(): {Name: n.Name(), Type: n.Type(), ID: n.ID(), Config: n.Config()}, } cleanup, err := addressset.OVNEnsureAddressSetsViaACLs(n.state, n.logger, n.ovnnb, n.Project(), securityACLS) if err != nil { return fmt.Errorf("Failed ensuring address sets for added ACLs are configured in OVN for network: %w", err) } reverter.Add(cleanup) cleanup, err = acl.OVNEnsureACLs(n.state, n.logger, n.ovnnb, n.Project(), aclNameIDs, aclNets, securityACLS, false) if err != nil { return fmt.Errorf("Failed ensuring security ACLs are configured in OVN for network: %w", err) } reverter.Add(cleanup) } reverter.Success() return nil } func (n *ovn) getDhcpOptionUUIDs() (v4Uuid networkOVN.OVNDHCPOptionsUUID, v6Uuid networkOVN.OVNDHCPOptionsUUID, err error) { // Find first existing DHCP options set for IPv4 and IPv6 and update them instead of adding sets. existingOpts, err := n.ovnnb.GetLogicalSwitchDHCPOptions(context.TODO(), n.getIntSwitchName()) if err != nil { return "", "", fmt.Errorf("Failed getting existing DHCP settings for internal switch: %w", err) } for _, existingOpt := range existingOpts { if existingOpt.CIDR.IP.To4() == nil { if v6Uuid != "" { return "", "", fmt.Errorf("Multiple matching DHCPv6 option sets found for switch %q", n.getIntSwitchName()) } v6Uuid = existingOpt.UUID } else { if v4Uuid != "" { return "", "", fmt.Errorf("Multiple matching DHCPv4 option sets found for switch %q", n.getIntSwitchName()) } v4Uuid = existingOpt.UUID } } return v4Uuid, v6Uuid, nil } // logicalRouterPolicySetup applies the security policy to the logical router (clearing any existing policies). // Optionally excludePeers takes a list of peer network IDs to exclude from the router policy. This is useful // when removing a peer connection as it allows the security policy to be removed from OVN for that peer before the // peer connection has been removed from the database. func (n *ovn) logicalRouterPolicySetup(ovnnb *networkOVN.NB, excludePeers ...int64) error { extRouterPort := n.getRouterExtPortName() intRouterPort := n.getRouterIntPortName() addrSetPrefix := acl.OVNIntSwitchPortGroupAddressSetPrefix(n.ID()) policies := []networkOVN.OVNRouterPolicy{ { // Allow IPv6 packets arriving from internal router port with valid source address. Priority: ovnRouterPolicyPeerAllowPriority, Match: fmt.Sprintf(`(inport == "%s" && ip6 && ip6.src == $%s_ip6)`, intRouterPort, addrSetPrefix), Action: "allow", }, { // Allow IPv4 packets arriving from internal router port with valid source address. Priority: ovnRouterPolicyPeerAllowPriority, Match: fmt.Sprintf(`(inport == "%s" && ip4 && ip4.src == $%s_ip4)`, intRouterPort, addrSetPrefix), Action: "allow", }, { // Drop all other traffic arriving from internal router port. // This prevents packets with a source address that is not valid to be dropped, and ensures // that we can use the internal address set in ACL rules and trust that this represents all // possible routed traffic from the internal network. Priority: ovnRouterPolicyPeerDropPriority, Match: fmt.Sprintf(`(inport == "%s")`, intRouterPort), Action: "drop", }, } // Add rules to drop inbound traffic arriving on external uplink port from peer connection addresses. // This prevents source address spoofing of peer connection routes from the external network, which in // turn allows us to use the peer connection's address set for referencing traffic from the peer in ACL. err := n.forPeers(func(targetOVNNet *ovn) error { if slices.Contains(excludePeers, targetOVNNet.ID()) { return nil // Don't setup rules for this peer network connection. } targetAddrSetPrefix := acl.OVNIntSwitchPortGroupAddressSetPrefix(targetOVNNet.ID()) // Associate the rules with the local peering port so we can identify them later if needed. comment := n.getLogicalRouterPeerPortName(targetOVNNet.ID()) policies = append(policies, networkOVN.OVNRouterPolicy{ Priority: ovnRouterPolicyPeerDropPriority, Match: fmt.Sprintf(`(inport == "%s" && ip6 && ip6.src == $%s_ip6) // %s`, extRouterPort, targetAddrSetPrefix, comment), Action: "drop", }, networkOVN.OVNRouterPolicy{ Priority: ovnRouterPolicyPeerDropPriority, Match: fmt.Sprintf(`(inport == "%s" && ip4 && ip4.src == $%s_ip4) // %s`, extRouterPort, targetAddrSetPrefix, comment), Action: "drop", }) return nil }) if err != nil { return err } return n.ovnnb.UpdateLogicalRouterPolicy(context.TODO(), n.getRouterName(), policies...) } // ensureNetworkPortGroup ensures that the network level port group (used for classifying NICs connected to this // network as internal) exists. func (n *ovn) ensureNetworkPortGroup(projectID int64) error { // Create port group (if needed) for NICs to classify as internal. intPortGroupName := acl.OVNIntSwitchPortGroupName(n.ID()) intPortGroupUUID, _, err := n.ovnnb.GetPortGroupInfo(context.TODO(), intPortGroupName) if err != nil { return fmt.Errorf("Failed getting port group UUID for network %q setup: %w", n.Name(), err) } if intPortGroupUUID == "" { // Create internal port group and associate it with the logical switch, so that it will be // removed when the logical switch is removed. err = n.ovnnb.CreatePortGroup(context.TODO(), projectID, intPortGroupName, []networkOVN.OVNPortGroup{}, n.getIntSwitchName()) if err != nil { return fmt.Errorf("Failed creating port group %q for network %q setup: %w", intPortGroupName, n.Name(), err) } } return nil } // addChassisGroupEntry adds an entry for the local OVS chassis to the OVN logical network's chassis group. // The chassis priority value is a stable-random value derived from chassis group name and node ID. This is so we // don't end up using the same chassis for the primary uplink chassis for all OVN networks in a cluster. func (n *ovn) addChassisGroupEntry() error { // Get local chassis ID for chassis group. vswitch, err := n.state.OVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } chassisID, err := vswitch.GetChassisID(context.TODO()) if err != nil { return fmt.Errorf("Failed getting OVS Chassis ID: %w", err) } // Seed the stable random number generator with the chassis group name. // This way each OVN network will have its own random seed, so that we don't end up using the same chassis // for the primary uplink chassis for all OVN networks in a cluster. chassisGroupName := n.getChassisGroupName() r, err := localUtil.GetStableRandomGenerator(string(chassisGroupName)) if err != nil { return fmt.Errorf("Failed generating stable random chassis group priority: %w", err) } // Get all members in cluster. ourMemberID := int(n.state.DB.Cluster.GetNodeID()) var memberIDs []int err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { members, err := tx.GetNodes(ctx) if err != nil { return fmt.Errorf("Failed getting cluster members for adding chassis group entry: %w", err) } for _, member := range members { memberIDs = append(memberIDs, int(member.ID)) } return nil }) if err != nil { return err } // Sort the nodes based on ID for stable priority generation. sort.Ints(memberIDs) // Generate a random priority from the seed for each node until we find a match for our node ID. // In this way the chassis priority for this node will be set to a per-node stable random value. var priority int for _, memberID := range memberIDs { priority = r.Intn(ovnChassisPriorityMax + 1) if memberID == ourMemberID { break } } err = n.ovnnb.SetChassisGroupPriority(context.TODO(), chassisGroupName, chassisID, priority) if err != nil { return fmt.Errorf("Failed adding OVS chassis %q with priority %d to chassis group %q: %w", chassisID, priority, chassisGroupName, err) } n.logger.Debug("Chassis group entry added", logger.Ctx{"chassisGroup": chassisGroupName, "memberID": ourMemberID, "priority": priority}) return nil } // deleteChassisGroupEntry deletes an entry for the local OVS chassis from the OVN logical network's chassis group. func (n *ovn) deleteChassisGroupEntry() error { // Remove local chassis from chassis group. vswitch, err := n.state.OVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } chassisID, err := vswitch.GetChassisID(context.TODO()) if err != nil { return fmt.Errorf("Failed getting OVS Chassis ID: %w", err) } err = n.ovnnb.SetChassisGroupPriority(context.TODO(), n.getChassisGroupName(), chassisID, -1) if err != nil && !errors.Is(err, ovs.ErrNotFound) { return fmt.Errorf("Failed deleting OVS chassis %q from chassis group %q: %w", chassisID, n.getChassisGroupName(), err) } return nil } // Delete deletes a network. func (n *ovn) Delete(clientType request.ClientType) error { n.logger.Debug("Delete", logger.Ctx{"clientType": clientType}) err := n.Stop() if err != nil { return err } if clientType == request.ClientTypeNormal { // Delete the router and anything tied to it (router ports, static routes, policies, nat, ...). err = n.ovnnb.DeleteLogicalRouter(context.TODO(), n.getRouterName()) if err != nil && !errors.Is(err, networkOVN.ErrNotFound) { return err } // Delete the external logical switch and anything tied to it (ports, ...). err = n.ovnnb.DeleteLogicalSwitch(context.TODO(), n.getExtSwitchName()) if err != nil && !errors.Is(err, networkOVN.ErrNotFound) { return err } // Delete the internal logical switch and anything tied to it (ports, ...). err = n.ovnnb.DeleteLogicalSwitch(context.TODO(), n.getIntSwitchName()) if err != nil && !errors.Is(err, networkOVN.ErrNotFound) { return err } // Delete any related address sets. err = n.ovnnb.DeleteAddressSet(context.TODO(), acl.OVNIntSwitchPortGroupAddressSetPrefix(n.ID())) if err != nil && !errors.Is(err, networkOVN.ErrNotFound) { return err } // Delete address sets used in ACLs. securityACLS := util.SplitNTrimSpace(n.config["security.acls"], ",", -1, true) // Load address sets referenced by ACLs. err = addressset.OVNDeleteAddressSetsViaACLs(n.state, n.logger, n.ovnnb, n.Project(), securityACLS) if err != nil { return fmt.Errorf("Failed deleting address sets for security ACLs in OVN for network: %w", err) } // Delete the chassis group for the network. err = n.ovnnb.DeleteChassisGroup(context.TODO(), n.getChassisGroupName()) if err != nil && !errors.Is(err, networkOVN.ErrNotFound) { return err } // Clean up any now unused port group. securityACLs := util.SplitNTrimSpace(n.config["security.acls"], ",", -1, true) if len(securityACLs) > 0 { err = acl.OVNPortGroupDeleteIfUnused(n.state, n.logger, n.ovnnb, n.project, &api.Network{Name: n.name}, "") if err != nil { return fmt.Errorf("Failed removing unused OVN port groups: %w", err) } } // Delete any network forwards and load balancers. forwardListenAddresses := map[int64]string{} loadBalancerListenAddresses := map[int64]string{} err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() // Get the forward addresses. dbForwards, err := dbCluster.GetNetworkForwards(ctx, tx.Tx(), dbCluster.NetworkForwardFilter{ NetworkID: &networkID, }) if err != nil { return fmt.Errorf("Failed loading network forwards: %w", err) } // Get the load balancers. dbLoadBalancers, err := dbCluster.GetNetworkLoadBalancers(ctx, tx.Tx(), dbCluster.NetworkLoadBalancerFilter{ NetworkID: &networkID, }) if err != nil { return fmt.Errorf("Failed loading network load balancers: %w", err) } for _, fwd := range dbForwards { forwardListenAddresses[fwd.ID] = fwd.ListenAddress } for _, lb := range dbLoadBalancers { loadBalancerListenAddresses[lb.ID] = lb.ListenAddress } return nil }) if err != nil { return err } loadBalancers := make([]networkOVN.OVNLoadBalancer, 0, len(forwardListenAddresses)+len(loadBalancerListenAddresses)) for _, listenAddress := range forwardListenAddresses { loadBalancers = append(loadBalancers, n.getLoadBalancerName(listenAddress)) } for _, listenAddress := range loadBalancerListenAddresses { loadBalancers = append(loadBalancers, n.getLoadBalancerName(listenAddress)) } err = n.ovnnb.DeleteLoadBalancer(context.TODO(), loadBalancers...) if err != nil { return fmt.Errorf("Failed deleting network forwards and load balancers: %w", err) } } return n.common.delete(clientType) } // Rename renames a network. func (n *ovn) Rename(newName string) error { n.logger.Debug("Rename", logger.Ctx{"newName": newName}) // Rename common steps. err := n.common.rename(newName) if err != nil { return err } return nil } // chassisEnabled checks the cluster config to see if this particular // member should act as an OVN chassis. func (n *ovn) chassisEnabled(ctx context.Context, tx *db.ClusterTx) (bool, error) { // Check that we have an uplink network, that it's physical, and that parent is not "none". if n.config["network"] == "none" { return false, nil } // Get uplink network to check its configuration. _, uplinkNet, _, err := tx.GetNetworkInAnyState(ctx, api.ProjectDefaultName, n.config["network"]) if err != nil { return false, fmt.Errorf("Failed to load uplink network %q: %w", n.config["network"], err) } if uplinkNet.Type == "physical" && uplinkNet.Config["parent"] == "none" { return false, nil } // Get the member info. memberID := tx.GetNodeID() members, err := tx.GetNodes(ctx) if err != nil { return false, fmt.Errorf("Failed getting cluster members: %w", err) } // Determine whether to add ourselves as a chassis. // If no server has the role, enable the chassis, otherwise only // enable if the local server has the role. enableChassis := -1 for _, member := range members { hasRole := slices.Contains(member.Roles, db.ClusterRoleOVNChassis) if hasRole { if member.ID == memberID { // Local node has the OVN chassis role, enable chassis. enableChassis = 1 break } // Some other node has the OVN chassis role, don't enable. enableChassis = 0 } } return enableChassis != 0, nil } // Start starts adds the local OVS chassis ID to the OVN chass group and starts the local OVS uplink port. func (n *ovn) Start() error { n.logger.Debug("Start") reverter := revert.New() defer reverter.Fail() var err error reverter.Add(func() { n.setUnavailable() }) // Check that uplink network is available. if n.config["network"] != "" && n.config["network"] != "none" && !IsAvailable(api.ProjectDefaultName, n.config["network"]) { return fmt.Errorf("Uplink network %q is unavailable", n.config["network"]) } var projectID int64 var chassisEnabled bool err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get the project ID. projectID, err = dbCluster.GetProjectID(context.Background(), tx.Tx(), n.project) if err != nil { return err } // Check if we should enable the chassis. chassisEnabled, err = n.chassisEnabled(ctx, tx) if err != nil { return err } return nil }) if err != nil { return fmt.Errorf("Failed getting project ID for project %q: %w", n.project, err) } // Ensure network level port group exists. err = n.ensureNetworkPortGroup(projectID) if err != nil { return err } // Handle chassis groups. if chassisEnabled { // Add local member's OVS chassis ID to logical chassis group. err = n.addChassisGroupEntry() if err != nil { return err } } else { // Make sure we don't have a group entry. err = n.deleteChassisGroupEntry() if err != nil { return err } } err = n.startUplinkPort() if err != nil { return err } // Setup BGP. err = n.bgpSetup(nil) if err != nil { return err } err = n.loadBalancerBGPSetupPrefixes() if err != nil { return fmt.Errorf("Failed applying BGP prefixes for load balancers: %w", err) } // Setup event handler for monitored services. handler := networkOVN.EventHandler{ Tables: []string{"Service_Monitor", "Port_Binding"}, Hook: func(action string, table string, oldObject ovsdbModel.Model, newObject ovsdbModel.Model) { // Skip invalid notifications. if oldObject == nil && newObject == nil { return } // Get the object. dbObject := newObject if dbObject == nil { dbObject = oldObject } switch ovnSBObject := dbObject.(type) { case *ovnSB.ServiceMonitor: // Check if this is our network. if !strings.HasPrefix(ovnSBObject.LogicalPort, fmt.Sprintf("incus-net%d-instance-", n.id)) { return } // Locate affected load-balancers. lbs, err := n.ovnnb.GetLoadBalancersByStatusUpdate(context.TODO(), *ovnSBObject) if err != nil { return } for _, lb := range lbs { // Check for status of all backends on this load-balancer. online, err := n.ovnsb.CheckLoadBalancerOnline(context.TODO(), lb) if err != nil { return } // Parse the name. fields := strings.Split(lb.Name, "-") listenAddr := net.ParseIP(fields[3]) if listenAddr == nil { return } // Check if we have a matching UDP load-balancer. fields[4] = "udp" lbUDP, _ := n.ovnnb.GetLoadBalancer(context.TODO(), networkOVN.OVNLoadBalancer(strings.Join(fields, "-"))) if lbUDP != nil { // UDP backends can't be checked, so have to assume online. online = true } // Prepare advertisement. ipVersion := uint(4) if listenAddr.To4() == nil { ipVersion = 6 } bgpOwner := fmt.Sprintf("network_%d_load_balancer", n.id) nextHopAddr := n.bgpNextHopAddress(ipVersion) natEnabled := util.IsTrue(n.config[fmt.Sprintf("ipv%d.nat", ipVersion)]) _, netSubnet, _ := net.ParseCIDR(n.config[fmt.Sprintf("ipv%d.address", ipVersion)]) routeSubnetSize := 128 if ipVersion == 4 { routeSubnetSize = 32 } // Don't export internal address forwards (those inside the NAT enabled network's subnet). if natEnabled && netSubnet != nil && netSubnet.Contains(listenAddr) { return } _, ipRouteSubnet, err := net.ParseCIDR(fmt.Sprintf("%s/%d", listenAddr.String(), routeSubnetSize)) if err != nil { return } // Update the BGP state. if online { err = n.state.BGP.AddPrefix(*ipRouteSubnet, nextHopAddr, bgpOwner) if err != nil { return } } else { err = n.state.BGP.RemovePrefix(*ipRouteSubnet, nextHopAddr) if err != nil { return } } } case *ovnSB.PortBinding: if ovnSBObject.Type != "chassisredirect" || ovnSBObject.LogicalPort != fmt.Sprintf("cr-incus-net%d-lr-lrp-ext", n.id) { return } err := n.updateTunnels(n.config, []string{}, true) if err != nil { return } } }, } err = networkOVN.AddOVNSBHandler(fmt.Sprintf("network_%d", n.id), handler) if err != nil { return err } reverter.Success() // Ensure network is marked as available now its started. n.setAvailable() return nil } // Stop deletes the local OVS uplink port (if unused) and deletes the local OVS chassis ID from the // OVN chassis group. func (n *ovn) Stop() error { n.logger.Debug("Stop") // Delete local OVS chassis ID from logical OVN HA chassis group. err := n.deleteChassisGroupEntry() if err != nil { return err } // Delete local uplink port if not used by other OVN networks. err = n.deleteUplinkPort() if err != nil { return err } // Clear BGP. err = n.bgpClear(n.config) if err != nil { return err } // Clear event handler for monitored services. err = networkOVN.RemoveOVNSBHandler(fmt.Sprintf("network_%d", n.id)) if err != nil { return err } return nil } // instanceNICGetRoutes returns list of routes defined in nicConfig. func (n *ovn) instanceNICGetRoutes(nicConfig map[string]string) []net.IPNet { var routes []net.IPNet routeKeys := []string{"ipv4.routes", "ipv4.routes.external", "ipv6.routes", "ipv6.routes.external"} for _, key := range routeKeys { for _, routeStr := range util.SplitNTrimSpace(nicConfig[key], ",", -1, true) { _, route, err := net.ParseCIDR(routeStr) if err != nil { continue // Skip invalid routes (should never happen). } routes = append(routes, *route) } } return routes } // Update updates the network. Accepts notification boolean indicating if this update request is coming from a // cluster notification, in which case do not update the database, just apply local changes needed. func (n *ovn) Update(newNetwork api.NetworkPut, targetNode string, clientType request.ClientType) error { n.logger.Debug("Update", logger.Ctx{"clientType": clientType, "newNetwork": newNetwork}) err := n.populateAutoConfig(newNetwork.Config) if err != nil { return fmt.Errorf("Failed generating auto config: %w", err) } dbUpdateNeeded, changedKeys, oldNetwork, err := n.configChanged(newNetwork) if err != nil { return err } if clientType == request.ClientTypeNotifier { // Reload BGP on notifications. err = n.bgpSetup(nil) if err != nil { return err } err = n.loadBalancerBGPSetupPrefixes() if err != nil { return fmt.Errorf("Failed applying BGP prefixes for load balancers: %w", err) } if len(n.getTunnelsFromChangedKeys(changedKeys)) > 0 { err = n.updateTunnels(newNetwork.Config, changedKeys, false) if err != nil { return err } } return nil } if !dbUpdateNeeded { return nil // Nothing changed. } // If the network as a whole has not had any previous creation attempts, or the node itself is still // pending, then don't apply the new settings to the node, just to the database record (ready for the // actual global create request to be initiated). if n.Status() == api.NetworkStatusPending || n.LocalStatus() == api.NetworkStatusPending { return n.common.update(newNetwork, targetNode, clientType) } reverter := revert.New() defer reverter.Fail() // Define a function which reverts everything. reverter.Add(func() { // Reset changes to all nodes and database. _ = n.common.update(oldNetwork, targetNode, clientType) // Reset any change that was made to logical network. if clientType == request.ClientTypeNormal { _ = n.setup(true) } _ = n.Start() }) // Stop network before new config applied if uplink network is changing. if slices.Contains(changedKeys, "network") { err = n.Stop() if err != nil { return err } // Remove volatile keys associated with old network in new config. delete(newNetwork.Config, ovnVolatileUplinkIPv4) delete(newNetwork.Config, ovnVolatileUplinkIPv6) } // Apply changes to all nodes and database. err = n.common.update(newNetwork, targetNode, clientType) if err != nil { return err } // Re-setup the logical network after config applied if needed. if len(changedKeys) > 0 && clientType == request.ClientTypeNormal { err = n.setup(true) if err != nil { return err } // Work out which ACLs have been added and removed. oldACLs := util.SplitNTrimSpace(oldNetwork.Config["security.acls"], ",", -1, true) newACLs := util.SplitNTrimSpace(newNetwork.Config["security.acls"], ",", -1, true) removedACLs := []string{} for _, oldACL := range oldACLs { if !slices.Contains(newACLs, oldACL) { removedACLs = append(removedACLs, oldACL) } } addedACLs := []string{} for _, newACL := range newACLs { if !slices.Contains(oldACLs, newACL) { addedACLs = append(addedACLs, newACL) } } // Detect if network default rule config has changed. defaultRuleKeys := []string{"security.acls.default.ingress.action", "security.acls.default.egress.action", "security.acls.default.ingress.logged", "security.acls.default.egress.logged"} changedDefaultRuleKeys := []string{} for _, k := range defaultRuleKeys { if slices.Contains(changedKeys, k) { changedDefaultRuleKeys = append(changedDefaultRuleKeys, k) } } var aclNameIDs map[string]int64 err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get map of ACL names to DB IDs (used for generating OVN port group names). acls, err := dbCluster.GetNetworkACLs(ctx, tx.Tx(), dbCluster.NetworkACLFilter{Project: &n.project}) if err != nil { return err } aclNameIDs = make(map[string]int64, len(acls)) for _, acl := range acls { aclNameIDs[acl.Name] = int64(acl.ID) } return nil }) if err != nil { return fmt.Errorf("Failed getting network ACL IDs for security ACL update: %w", err) } addChangeSet := map[networkOVN.OVNPortGroup][]networkOVN.OVNSwitchPortUUID{} removeChangeSet := map[networkOVN.OVNPortGroup][]networkOVN.OVNSwitchPortUUID{} // Get list of active switch ports (avoids repeated querying of OVN NB). activePorts, err := n.ovnnb.GetLogicalSwitchPorts(context.TODO(), n.getIntSwitchName()) if err != nil { return fmt.Errorf("Failed getting active ports: %w", err) } aclConfigChanged := len(addedACLs) > 0 || len(removedACLs) > 0 || len(changedDefaultRuleKeys) > 0 var localNICRoutes []net.IPNet // Apply ACL changes to running instance NICs that use this network. err = UsedByInstanceDevices(n.state, n.Project(), n.Name(), n.Type(), func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error { nicACLs := util.SplitNTrimSpace(nicConfig["security.acls"], ",", -1, true) // Get logical port UUID and name. instancePortName := n.getInstanceDevicePortName(inst.Config["volatile.uuid"], nicName) portUUID, found := activePorts[instancePortName] if !found { return nil // No need to update a port that isn't started yet. } // Apply security ACL and default rule changes. if aclConfigChanged { // Update relevant address sets and Remove from removedACL. if len(addedACLs) > 0 { cleanup, err := addressset.OVNEnsureAddressSetsViaACLs(n.state, n.logger, n.ovnnb, n.Project(), addedACLs) if err != nil { return fmt.Errorf("Failed ensuring address sets for added ACLs are configured in OVN for network: %w", err) } reverter.Add(cleanup) } if len(removedACLs) > 0 { err = addressset.OVNDeleteAddressSetsViaACLs(n.state, n.logger, n.ovnnb, n.Project(), removedACLs) if err != nil { return fmt.Errorf("Failed to delete address set for removed ACLs are configured in OVN for network: %w", err) } } ingressAction, ingressLogged := n.instanceDeviceACLDefaults(nicConfig, "ingress") egressAction, egressLogged := n.instanceDeviceACLDefaults(nicConfig, "egress") // Check whether we need to add any of the new ACLs to the NIC. for _, addedACL := range addedACLs { if slices.Contains(nicACLs, addedACL) { continue // NIC already has this ACL applied directly, so no need to add. } aclID, found := aclNameIDs[addedACL] if !found { return fmt.Errorf("Cannot find security ACL ID for %q", addedACL) } directionalPortGroups := acl.OVNACLDirectionalPortGroups(aclID) // Add NIC port to ACL port group. var ingressPortGroupName networkOVN.OVNPortGroup if ingressAction == "allow" { ingressPortGroupName = directionalPortGroups.IngressReversed } else { ingressPortGroupName = directionalPortGroups.Ingress } acl.OVNPortGroupInstanceNICSchedule(portUUID, addChangeSet, ingressPortGroupName) n.logger.Debug("Scheduled logical port for ACL port group addition", logger.Ctx{"networkACL": addedACL, "portGroup": ingressPortGroupName, "port": instancePortName}) var egressPortGroupName networkOVN.OVNPortGroup if egressAction == "allow" { egressPortGroupName = directionalPortGroups.EgressReversed } else { egressPortGroupName = directionalPortGroups.Egress } acl.OVNPortGroupInstanceNICSchedule(portUUID, addChangeSet, egressPortGroupName) n.logger.Debug("Scheduled logical port for ACL port group addition", logger.Ctx{"networkACL": addedACL, "portGroup": egressPortGroupName, "port": instancePortName}) acl.OVNPortGroupInstanceNICSchedule(portUUID, addChangeSet, directionalPortGroups.All) n.logger.Debug("Scheduled logical port for ACL port group addition", logger.Ctx{"networkACL": addedACL, "portGroup": directionalPortGroups.All, "port": instancePortName}) } // Check whether we need to remove any of the removed ACLs from the NIC. for _, removedACL := range removedACLs { if slices.Contains(nicACLs, removedACL) { continue // NIC still has this ACL applied directly, so don't remove. } aclID, found := aclNameIDs[removedACL] if !found { return fmt.Errorf("Cannot find security ACL ID for %q", removedACL) } // Remove NIC port from ACL port group. directionalPortGroups := acl.OVNACLDirectionalPortGroups(aclID) directionalPortGroups.AddToChangeSet(portUUID, removeChangeSet) n.logger.Debug("Scheduled logical port for ACL port group removal", logger.Ctx{"networkACL": removedACL, "portGroup": directionalPortGroups.Ingress, "port": instancePortName}) } // If there are no ACLs being applied to the NIC (either from network or NIC) then // we should remove the default rule from the NIC. if len(newACLs) <= 0 && len(nicACLs) <= 0 { err = n.ovnnb.ClearPortGroupPortACLRules(context.TODO(), acl.OVNIntSwitchPortGroupName(n.ID()), instancePortName) if err != nil { return fmt.Errorf("Failed clearing OVN default ACL rules for instance NIC: %w", err) } n.logger.Debug("Cleared NIC default rules", logger.Ctx{"port": instancePortName}) } else { defaultRuleChange := false // If there are ACLs being applied, then decide if the default rule config // has changed materially for the NIC and update it if needed. for _, k := range changedDefaultRuleKeys { _, found := nicConfig[k] if found { continue // Skip if changed key is overridden in NIC. } defaultRuleChange = true break } // If the default rule config has changed materially for this NIC or the // network previously didn't have any ACLs applied and now does, then add // the default rule to the NIC. if defaultRuleChange || len(oldACLs) <= 0 { // Set the automatic default ACL rule for the port. logPrefix := fmt.Sprintf("%s-%s", inst.Config["volatile.uuid"], nicName) err = acl.OVNApplyInstanceNICDefaultRules(n.ovnnb, acl.OVNIntSwitchPortGroupName(n.ID()), logPrefix, instancePortName, ingressAction, ingressLogged, egressAction, egressLogged) if err != nil { return fmt.Errorf("Failed applying OVN default ACL rules for instance NIC: %w", err) } n.logger.Debug("Set NIC default rule", logger.Ctx{"port": instancePortName, "ingressAction": ingressAction, "ingressLogged": ingressLogged, "egressAction": egressAction, "egressLogged": egressLogged}) } } } // Add NIC routes to list. localNICRoutes = append(localNICRoutes, n.instanceNICGetRoutes(nicConfig)...) return nil }) if err != nil { return err } // Apply add/remove changesets. if len(addChangeSet) > 0 || len(removeChangeSet) > 0 { n.logger.Debug("Applying ACL port group member change sets") err = n.ovnnb.UpdatePortGroupMembers(context.TODO(), addChangeSet, removeChangeSet) if err != nil { return fmt.Errorf("Failed applying OVN port group member change sets for instance NIC: %w", err) } } // Check if any of the removed ACLs should have any unused port groups deleted. if len(removedACLs) > 0 { err = acl.OVNPortGroupDeleteIfUnused(n.state, n.logger, n.ovnnb, n.project, &api.Network{Name: n.name}, "", newACLs...) if err != nil { return fmt.Errorf("Failed removing unused OVN port groups: %w", err) } } // Ensure all active NIC routes are present in internal switch's address set. err = n.ovnnb.UpdateAddressSetAdd(context.TODO(), acl.OVNIntSwitchPortGroupAddressSetPrefix(n.ID()), localNICRoutes...) if err != nil { return fmt.Errorf("Failed adding active NIC routes to switch address set: %w", err) } // Remove any old unused subnet addresses from the internal switch's address set. rebuildPeers := false for _, key := range []string{"ipv4.address", "ipv6.address"} { if slices.Contains(changedKeys, key) { rebuildPeers = true _, oldRouterIntPortIPNet, _ := net.ParseCIDR(oldNetwork.Config[key]) if oldRouterIntPortIPNet != nil { err = n.ovnnb.UpdateAddressSetRemove(context.TODO(), acl.OVNIntSwitchPortGroupAddressSetPrefix(n.ID()), *oldRouterIntPortIPNet) if err != nil { return fmt.Errorf("Failed removing old network subnet %q from switch address set: %w", oldRouterIntPortIPNet.String(), err) } } } } if rebuildPeers { // Rebuild peering config. opts, err := n.peerGetLocalOpts(localNICRoutes) if err != nil { return err } err = n.forPeers(func(targetOVNNet *ovn) error { err = n.peerSetup(n.ovnnb, targetOVNNet, *opts) if err != nil { return err } return nil }) if err != nil { return err } } } // If uplink network is changing, start network after config applied. if slices.Contains(changedKeys, "network") { err = n.Start() if err != nil { return err } // Notify all other members to refresh their BGP prefixes. notifier, err := cluster.NewNotifier(n.state, n.state.Endpoints.NetworkCert(), n.state.ServerCert(), cluster.NotifyAll) if err != nil { return err } err = notifier(func(client incus.InstanceServer) error { return client.UseProject(n.project).UpdateNetwork(n.name, newNetwork, "") }) if err != nil { return err } } else { // Setup BGP. err = n.bgpSetup(oldNetwork.Config) if err != nil { return err } } err = n.loadBalancerBGPSetupPrefixes() if err != nil { return fmt.Errorf("Failed applying BGP prefixes for load balancers: %w", err) } err = n.updateTunnels(newNetwork.Config, changedKeys, false) if err != nil { return err } if len(n.getTunnelsFromChangedKeys(changedKeys)) > 0 { // Notify all other members about tunnels configuration change. notifier, err := cluster.NewNotifier(n.state, n.state.Endpoints.NetworkCert(), n.state.ServerCert(), cluster.NotifyAll) if err != nil { return err } err = notifier(func(client incus.InstanceServer) error { return client.UseProject(n.project).UpdateNetwork(n.name, newNetwork, "") }) if err != nil { return err } } // Delete any address set that is unused err = addressset.OVNAddressSetsDeleteIfUnused(n.state, n.logger, n.ovnnb, n.Project()) if err != nil { return fmt.Errorf("Failed removing unused OVN address sets: %w", err) } reverter.Success() return nil } // getInstanceDevicePortName returns the switch port name to use for an instance device. func (n *ovn) getInstanceDevicePortName(instanceUUID string, deviceName string) networkOVN.OVNSwitchPort { return networkOVN.OVNSwitchPort(fmt.Sprintf("%s-%s-%s", n.getIntSwitchInstancePortPrefix(), instanceUUID, deviceName)) } // instanceDevicePortRoutesParse parses the instance NIC device config for internal routes and external routes. func (n *ovn) instanceDevicePortRoutesParse(deviceConfig map[string]string) ([]*net.IPNet, []*net.IPNet, error) { var err error internalRoutes := []*net.IPNet{} for _, key := range []string{"ipv4.routes", "ipv6.routes"} { if deviceConfig[key] == "" { continue } internalRoutes, err = SubnetParseAppend(internalRoutes, util.SplitNTrimSpace(deviceConfig[key], ",", -1, false)...) if err != nil { return nil, nil, fmt.Errorf("Invalid %q value: %w", key, err) } } externalRoutes := []*net.IPNet{} for _, key := range []string{"ipv4.routes.external", "ipv6.routes.external"} { if deviceConfig[key] == "" { continue } externalRoutes, err = SubnetParseAppend(externalRoutes, util.SplitNTrimSpace(deviceConfig[key], ",", -1, false)...) if err != nil { return nil, nil, fmt.Errorf("Invalid %q value: %w", key, err) } } return internalRoutes, externalRoutes, nil } // InstanceDevicePortValidateExternalRoutes validates the external routes for an OVN instance port. func (n *ovn) InstanceDevicePortValidateExternalRoutes(deviceInstance instance.Instance, deviceName string, portExternalRoutes []*net.IPNet) error { if n.config["network"] == "none" { return nil } var p *api.Project var uplink *api.Network err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get uplink routes. _, uplink, _, err = tx.GetNetworkInAnyState(ctx, api.ProjectDefaultName, n.config["network"]) return err }) if err != nil { return fmt.Errorf("Failed to load uplink network %q: %w", n.config["network"], err) } // Check port's external routes are sufficiently small when using l2proxy ingress mode on uplink. if slices.Contains([]string{"l2proxy", ""}, uplink.Config["ovn.ingress_mode"]) { for _, portExternalRoute := range portExternalRoutes { rOnes, rBits := portExternalRoute.Mask.Size() if rBits > 32 && rOnes < 122 { return fmt.Errorf("External route %q is too large. Maximum size for IPv6 external route is /122", portExternalRoute.String()) } else if rOnes < 26 { return fmt.Errorf("External route %q is too large. Maximum size for IPv4 external route is /26", portExternalRoute.String()) } } } // Load the project to get uplink network restrictions. err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { project, err := dbCluster.GetProject(ctx, tx.Tx(), n.project) if err != nil { return err } p, err = project.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return fmt.Errorf("Failed to load network restrictions from project %q: %w", n.project, err) } externalSubnetsInUse, err := n.getExternalSubnetInUse(n.config["network"]) if err != nil { return err } // Get project restricted routes. projectRestrictedSubnets, err := n.projectRestrictedSubnets(p, n.config["network"]) if err != nil { return err } // Check if uplink has routed ingress anycast mode enabled, as this relaxes the overlap checks. ipv4UplinkAnycast := n.uplinkHasIngressRoutedAnycastIPv4(uplink) ipv6UplinkAnycast := n.uplinkHasIngressRoutedAnycastIPv6(uplink) for _, portExternalRoute := range portExternalRoutes { // Check the external port route is allowed within both the uplink's external routes and any // project restricted subnets. err = n.validateExternalSubnet(uplink, projectRestrictedSubnets, portExternalRoute) if err != nil { return err } // Skip overlap checks if the external route's protocol has anycast mode enabled on the uplink. if portExternalRoute.IP.To4() == nil { if ipv6UplinkAnycast { continue } } else if ipv4UplinkAnycast { continue } // Check the external port route doesn't fall within any existing OVN network external subnets. for _, externalSubnetUser := range externalSubnetsInUse { // Skip our own network's SNAT address (as it can be used for NICs in the network). if externalSubnetUser.usageType == subnetUsageNetworkSNAT && externalSubnetUser.networkProject == n.project && externalSubnetUser.networkName == n.name { continue } if deviceInstance == nil { // Skip checking instance devices during profile validation, only do this when // an instance is supplied. if externalSubnetUser.instanceDevice != "" { continue } } else { // Skip our own NIC device. if externalSubnetUser.instanceProject == deviceInstance.Project().Name && externalSubnetUser.instanceName == deviceInstance.Name() && externalSubnetUser.instanceDevice == deviceName { continue } } if SubnetContains(&externalSubnetUser.subnet, portExternalRoute) || SubnetContains(portExternalRoute, &externalSubnetUser.subnet) { // This error is purposefully vague so that it doesn't reveal any names of // resources potentially outside of the network's project. return fmt.Errorf("External subnet %q overlaps with another network or NIC", portExternalRoute.String()) } } } return nil } // InstanceDevicePortAdd adds empty DNS record (to indicate port has been added) and any DHCP reservations for // instance device port. func (n *ovn) InstanceDevicePortAdd(instanceUUID string, deviceName string, deviceConfig deviceConfig.Device) error { instancePortName := n.getInstanceDevicePortName(instanceUUID, deviceName) reverter := revert.New() defer reverter.Fail() dnsUUID, err := n.ovnnb.UpdateLogicalSwitchPortDNS(context.TODO(), n.getIntSwitchName(), instancePortName, "", nil) if err != nil { return fmt.Errorf("Failed adding DNS record: %w", err) } reverter.Add(func() { _ = n.ovnnb.DeleteLogicalSwitchPortDNS(context.TODO(), n.getIntSwitchName(), dnsUUID, true) }) // If NIC has static IPv4 address then create a DHCPv4 reservation. if deviceConfig["ipv4.address"] != "" { ip := net.ParseIP(deviceConfig["ipv4.address"]) if ip != nil { dhcpReservations, err := n.ovnnb.GetLogicalSwitchDHCPv4Revervations(context.TODO(), n.getIntSwitchName()) if err != nil { return fmt.Errorf("Failed getting DHCPv4 reservations: %w", err) } if !n.hasDHCPv4Reservation(dhcpReservations, ip) { dhcpReservations = append(dhcpReservations, iprange.Range{Start: ip}) err = n.ovnnb.UpdateLogicalSwitchDHCPv4Revervations(context.TODO(), n.getIntSwitchName(), dhcpReservations) if err != nil { return fmt.Errorf("Failed adding DHCPv4 reservation for %q: %w", ip.String(), err) } } } } reverter.Success() return nil } // hasDHCPv4Reservation returns whether IP is in the supplied reservation list. func (n *ovn) hasDHCPv4Reservation(dhcpReservations []iprange.Range, ip net.IP) bool { for _, dhcpReservation := range dhcpReservations { if dhcpReservation.Start.Equal(ip) && dhcpReservation.End == nil { return true } } return false } // InstanceDevicePortStart sets up an instance device port to the internal logical switch. // Accepts a list of ACLs being removed from the NIC device (if called as part of a NIC update). // Returns the logical switch port name and a list of IPs that were allocated to the port for DNS. func (n *ovn) InstanceDevicePortStart(opts *OVNInstanceNICSetupOpts, securityACLsRemove []string) (networkOVN.OVNSwitchPort, []net.IP, error) { if opts.InstanceUUID == "" { return "", nil, errors.New("Instance UUID is required") } mac, err := net.ParseMAC(opts.DeviceConfig["hwaddr"]) if err != nil { return "", nil, err } ipv4 := opts.DeviceConfig["ipv4.address"] ipv6 := opts.DeviceConfig["ipv6.address"] internalRoutes, externalRoutes, err := n.instanceDevicePortRoutesParse(opts.DeviceConfig) if err != nil { return "", nil, fmt.Errorf("Failed parsing NIC device routes: %w", err) } reverter := revert.New() defer reverter.Fail() // Get existing DHCPv4 static reservations. // This is used for both checking sticky DHCPv4 allocation availability and for ensuring static DHCP // reservations exist. dhcpReservations, err := n.ovnnb.GetLogicalSwitchDHCPv4Revervations(context.TODO(), n.getIntSwitchName()) if err != nil { return "", nil, fmt.Errorf("Failed getting DHCPv4 reservations: %w", err) } dhcpv4Subnet := n.DHCPv4Subnet() dhcpv6Subnet := n.DHCPv6Subnet() var dhcpV4UUID, dhcpV6UUID networkOVN.OVNDHCPOptionsUUID if dhcpv4Subnet != nil || dhcpv6Subnet != nil { dhcpV4UUID, dhcpV6UUID, err = n.getDhcpOptionUUIDs() if err != nil { return "", nil, err } } if dhcpv4Subnet != nil { if dhcpV4UUID == "" { return "", nil, fmt.Errorf("Could not find DHCPv4 options for instance port for subnet %q", dhcpv4Subnet.String()) } // If using dynamic IPv4, look for previously used sticky IPs from the NIC's last state. var dhcpV4StickyIP net.IP if opts.DeviceConfig["ipv4.address"] == "" { for _, entry := range opts.LastStateIPs { if entry.To4() != nil && SubnetContainsIP(dhcpv4Subnet, entry) { dhcpV4StickyIP = entry break } } } // If a previously used IP has been found and its not one of the static IPs, then check if // the IP is available for use and if not then we can request this port use it statically. if dhcpV4StickyIP != nil && dhcpV4StickyIP.String() != ipv4 { // If the sticky IP isn't statically reserved, lets check its not used dynamically // on any active port. if !n.hasDHCPv4Reservation(dhcpReservations, dhcpV4StickyIP) { existingPortIPs, err := n.ovnnb.GetLogicalSwitchIPs(context.TODO(), n.getIntSwitchName()) if err != nil { return "", nil, fmt.Errorf("Failed getting existing switch port IPs: %w", err) } found := false for _, ips := range existingPortIPs { if IPInSlice(dhcpV4StickyIP, ips) { found = true break // IP is in use with another port, so cannot use it. } } // If IP is not in use then request OVN use previously used IP for port. if !found { ipv4 = dhcpV4StickyIP.String() } } } } if dhcpv6Subnet != nil { if dhcpV6UUID == "" { return "", nil, fmt.Errorf("Could not find DHCPv6 options for instance port for subnet %q", dhcpv6Subnet.String()) } // If port isn't going to have fully dynamic IPs allocated by OVN, and instead only static // IPv4 addresses have been added, then add an EUI64 static IPv6 address so that the switch // port has an IPv6 address that will be used to generate a DNS record. This works around a // limitation in OVN that prevents us requesting dynamic IPv6 address allocation when // static IPv4 allocation is used. if ipv4 != "" && ipv6 == "" { eui64IP, err := eui64.ParseMAC(dhcpv6Subnet.IP, mac) if err != nil { return "", nil, fmt.Errorf("Failed generating EUI64 for instance port %q: %w", mac.String(), err) } // Add EUI64 as the IPv6 address. ipv6 = eui64IP.String() } } instancePortName := n.getInstanceDevicePortName(opts.InstanceUUID, opts.DeviceName) var nestedPortParentName networkOVN.OVNSwitchPort var nestedPortVLAN uint16 if opts.DeviceConfig["nested"] != "" { nestedPortParentName = n.getInstanceDevicePortName(opts.InstanceUUID, opts.DeviceConfig["nested"]) nestedPortVLANInt64, err := strconv.ParseUint(opts.DeviceConfig["vlan"], 10, 16) if err != nil { return "", nil, fmt.Errorf("Invalid VLAN ID %q: %w", opts.DeviceConfig["vlan"], err) } nestedPortVLAN = uint16(nestedPortVLANInt64) } // Add port with mayExist set to true, so that if instance port exists, we don't fail and continue below // to configure the port as needed. This is required in case the OVN northbound database was unavailable // when the instance NIC was stopped and was unable to remove the port on last stop, which would otherwise // prevent future NIC starts. err = n.ovnnb.CreateLogicalSwitchPort(context.TODO(), n.getIntSwitchName(), instancePortName, &networkOVN.OVNSwitchPortOpts{ DHCPv4OptsID: dhcpV4UUID, DHCPv6OptsID: dhcpV6UUID, MAC: mac, IPV4: ipv4, IPV6: ipv6, Parent: nestedPortParentName, VLAN: nestedPortVLAN, Location: n.state.ServerName, Promiscuous: util.IsTrue(opts.DeviceConfig["security.promiscuous"]), }, true) if err != nil { return "", nil, err } reverter.Add(func() { _ = n.ovnnb.DeleteLogicalSwitchPort(context.TODO(), n.getIntSwitchName(), instancePortName) }) // Add DNS records for port's IPs, and retrieve the IP addresses used. var dnsIPv4, dnsIPv6 net.IP dnsIPs := make([]net.IP, 0, 2) // checkAndStoreIP checks if the supplied IP is valid and can be used for a missing DNS IP. // If the found IP is needed, stores into the relevant dnsIPv{X} variable and into dnsIPs slice. checkAndStoreIP := func(ip net.IP) { if ip != nil { isV4 := ip.To4() != nil if dnsIPv4 == nil && isV4 { dnsIPv4 = ip } else if dnsIPv6 == nil && !isV4 { dnsIPv6 = ip } dnsIPs = append(dnsIPs, ip) } } // Populate DNS IP variables with any static IPs first before checking if we need to extract dynamic IPs. for _, staticIP := range []string{ipv4, ipv6} { if staticIP == "" || staticIP == "none" { continue } checkAndStoreIP(net.ParseIP(staticIP)) } // Apply device specific external address if any. for _, keyPrefix := range []string{"ipv4", "ipv6"} { // Check if the address is present. value := opts.DeviceConfig[fmt.Sprintf("%s.address.external", keyPrefix)] if value == "" { continue } // Check if the family is configured. if keyPrefix == "ipv4" && ipv4 == "" { continue } if keyPrefix == "ipv6" && ipv6 == "" { continue } // Parse the internal address. var intNet *net.IPNet if keyPrefix == "ipv4" { _, intNet, err = net.ParseCIDR(fmt.Sprintf("%s/32", ipv4)) if err != nil { return "", nil, fmt.Errorf("Invalid internal address %q: %w", ipv4, err) } } else { _, intNet, err = net.ParseCIDR(fmt.Sprintf("%s/128", ipv6)) if err != nil { return "", nil, fmt.Errorf("Invalid internal address %q: %w", ipv6, err) } } // Parse the external address. extIP := net.ParseIP(value) if extIP == nil { return "", nil, fmt.Errorf("Invalid external address %q", value) } if err := n.ovnnb.CreateLogicalRouterNAT( context.TODO(), n.getRouterName(), "snat", intNet, extIP, nil, false, true, ); err != nil { return "", nil, fmt.Errorf("Failed to add SNAT %q: %w", value, err) } reverter.Add(func() { _ = n.ovnnb.DeleteLogicalRouterNAT( context.TODO(), n.getRouterName(), "snat", false, extIP, ) }) } // Get dynamic IPs for switch port if any IPs not assigned statically. if (ipv4 != "none" && dnsIPv4 == nil) || (ipv6 != "none" && dnsIPv6 == nil) { var dynamicIPs []net.IP // Retry a few times in case port has not yet allocated dynamic IPs. for range 40 { dynamicIPs, err = n.ovnnb.GetLogicalSwitchPortDynamicIPs(context.TODO(), instancePortName) if err == nil { if len(dynamicIPs) > 0 { break } } else if !errors.Is(err, ovsClient.ErrNotFound) { return "", nil, err } time.Sleep(250 * time.Millisecond) } for _, dynamicIP := range dynamicIPs { // Try and find the first IPv4 and IPv6 addresses from the dynamic address list. checkAndStoreIP(dynamicIP) } // Check, after considering all dynamic IPs, whether we have got the required ones. if (dnsIPv4 == nil && dhcpv4Subnet != nil) || (dnsIPv6 == nil && dhcpv6Subnet != nil) { return "", nil, errors.New("Insufficient dynamic addresses allocated") } } if n.config["dns.mode"] == "managed" || n.config["dns.mode"] == "" { dnsName := fmt.Sprintf("%s.%s", opts.DNSName, n.getDomainName()) dnsUUID, err := n.ovnnb.UpdateLogicalSwitchPortDNS(context.TODO(), n.getIntSwitchName(), instancePortName, dnsName, dnsIPs) if err != nil { return "", nil, fmt.Errorf("Failed setting DNS for %q: %w", dnsName, err) } reverter.Add(func() { _ = n.ovnnb.DeleteLogicalSwitchPortDNS(context.TODO(), n.getIntSwitchName(), dnsUUID, false) }) } // If NIC has static IPv4 address then ensure a DHCPv4 reservation exists. // Do this at start time as well as add time in case an instance was copied (causing a duplicate address // conflict at add time) which is later resolved by deleting the original instance, meaning a reservation needs to // be added when the copied instance next starts. if opts.DeviceConfig["ipv4.address"] != "" && dnsIPv4 != nil { if !n.hasDHCPv4Reservation(dhcpReservations, dnsIPv4) { dhcpReservations = append(dhcpReservations, iprange.Range{Start: dnsIPv4}) err = n.ovnnb.UpdateLogicalSwitchDHCPv4Revervations(context.TODO(), n.getIntSwitchName(), dhcpReservations) if err != nil { return "", nil, fmt.Errorf("Failed adding DHCPv4 reservation for %q: %w", dnsIPv4.String(), err) } } } // Publish NIC's IPs on uplink network if NAT is disabled and using l2proxy ingress mode on uplink. if slices.Contains([]string{"l2proxy", ""}, opts.UplinkConfig["ovn.ingress_mode"]) { for _, k := range []string{"ipv4.nat", "ipv6.nat"} { if util.IsTrue(n.config[k]) { continue } // Select the correct destination IP from the DNS records. var ip net.IP if k == "ipv4.nat" { ip = dnsIPv4 } else if k == "ipv6.nat" { ip = dnsIPv6 } if ip == nil { continue // No qualifying target IP from DNS records. } err = n.ovnnb.CreateLogicalRouterNAT(context.TODO(), n.getRouterName(), "dnat_and_snat", nil, ip, ip, true, true) if err != nil { return "", nil, err } reverter.Add(func() { _ = n.ovnnb.DeleteLogicalRouterNAT(context.TODO(), n.getRouterName(), "dnat_and_snat", false, ip) }) } } var routes []networkOVN.OVNRouterRoute // In l3only mode we add the instance port's IPs as static routes to the router. if util.IsTrue(n.config["ipv4.l3only"]) && dnsIPv4 != nil { ipNet := IPToNet(dnsIPv4) internalRoutes = append(internalRoutes, &ipNet) } if util.IsTrue(n.config["ipv6.l3only"]) && dnsIPv6 != nil { ipNet := IPToNet(dnsIPv6) internalRoutes = append(internalRoutes, &ipNet) } // Add each internal route (using the IPs set for DNS as target). for _, internalRoute := range internalRoutes { targetIP := dnsIPv4 if internalRoute.IP.To4() == nil { targetIP = dnsIPv6 } if targetIP == nil { return "", nil, fmt.Errorf("Cannot add static route for %q as target IP is not set", internalRoute.String()) } routes = append(routes, networkOVN.OVNRouterRoute{ Prefix: *internalRoute, NextHop: targetIP, Port: n.getRouterIntPortName(), }) } // Add each external route (using the IPs set for DNS as target). for _, externalRoute := range externalRoutes { targetIP := dnsIPv4 if externalRoute.IP.To4() == nil { targetIP = dnsIPv6 } if targetIP == nil { return "", nil, fmt.Errorf("Cannot add static route for %q as target IP is not set", externalRoute.String()) } routes = append(routes, networkOVN.OVNRouterRoute{ Prefix: *externalRoute, NextHop: targetIP, Port: n.getRouterIntPortName(), }) // When using l2proxy ingress mode on uplink, in order to advertise the external route to the // uplink network using proxy ARP/NDP we need to add a stateless dnat_and_snat rule (as to my // knowledge this is the only way to get the OVN router to respond to ARP/NDP requests for IPs that // it doesn't actually have). However we have to add each IP in the external route individually as // DNAT doesn't support whole subnets. if slices.Contains([]string{"l2proxy", ""}, opts.UplinkConfig["ovn.ingress_mode"]) { err = SubnetIterate(externalRoute, func(ip net.IP) error { err = n.ovnnb.CreateLogicalRouterNAT(context.TODO(), n.getRouterName(), "dnat_and_snat", nil, ip, ip, true, true) if err != nil { return err } reverter.Add(func() { _ = n.ovnnb.DeleteLogicalRouterNAT(context.TODO(), n.getRouterName(), "dnat_and_snat", false, ip) }) return nil }) if err != nil { return "", nil, err } } } if len(routes) > 0 { // Add routes to local router. err = n.ovnnb.CreateLogicalRouterRoute(context.TODO(), n.getRouterName(), true, routes...) if err != nil { return "", nil, err } routePrefixes := make([]net.IPNet, 0, len(routes)) for _, route := range routes { routePrefixes = append(routePrefixes, route.Prefix) } reverter.Add(func() { _ = n.ovnnb.DeleteLogicalRouterRoute(context.TODO(), n.getRouterName(), routePrefixes...) }) // Add routes to internal switch's address set for ACL usage. err = n.ovnnb.UpdateAddressSetAdd(context.TODO(), acl.OVNIntSwitchPortGroupAddressSetPrefix(n.ID()), routePrefixes...) if err != nil { return "", nil, fmt.Errorf("Failed adding switch address set entries: %w", err) } reverter.Add(func() { _ = n.ovnnb.UpdateAddressSetRemove(context.TODO(), acl.OVNIntSwitchPortGroupAddressSetPrefix(n.ID()), routePrefixes...) }) routerIntPortIPv4, _, err := n.parseRouterIntPortIPv4Net() if err != nil { return "", nil, fmt.Errorf("Failed parsing local router's peering port IPv4 Net: %w", err) } routerIntPortIPv6, _, err := n.parseRouterIntPortIPv6Net() if err != nil { return "", nil, fmt.Errorf("Failed parsing local router's peering port IPv6 Net: %w", err) } // Add routes to peer routers, and security policies for each peer port on local router. err = n.forPeers(func(targetOVNNet *ovn) error { targetRouterName := targetOVNNet.getRouterName() targetRouterPort := targetOVNNet.getLogicalRouterPeerPortName(n.ID()) targetRouterRoutes := make([]networkOVN.OVNRouterRoute, 0, len(routes)) for _, route := range routes { nexthop := routerIntPortIPv4 if route.Prefix.IP.To4() == nil { nexthop = routerIntPortIPv6 } if nexthop == nil { continue // Skip routes that cannot be supported by local router. } targetRouterRoutes = append(targetRouterRoutes, networkOVN.OVNRouterRoute{ Prefix: route.Prefix, NextHop: nexthop, Port: targetRouterPort, }) } err = n.ovnnb.CreateLogicalRouterRoute(context.TODO(), targetRouterName, true, targetRouterRoutes...) if err != nil { return fmt.Errorf("Failed adding static routes to peer network %q in project %q: %w", targetOVNNet.Name(), targetOVNNet.Project(), err) } reverter.Add(func() { _ = n.ovnnb.DeleteLogicalRouterRoute(context.TODO(), targetRouterName, routePrefixes...) }) return nil }) if err != nil { return "", nil, err } } // Merge network and NIC assigned security ACL lists. netACLNames := util.SplitNTrimSpace(n.config["security.acls"], ",", -1, true) nicACLNames := util.SplitNTrimSpace(opts.DeviceConfig["security.acls"], ",", -1, true) for _, aclName := range netACLNames { if !slices.Contains(nicACLNames, aclName) { nicACLNames = append(nicACLNames, aclName) } } // Apply Security ACL port group settings. addChangeSet := map[networkOVN.OVNPortGroup][]networkOVN.OVNSwitchPortUUID{} removeChangeSet := map[networkOVN.OVNPortGroup][]networkOVN.OVNSwitchPortUUID{} // Get logical port UUID. portUUID, err := n.ovnnb.GetLogicalSwitchPortUUID(context.TODO(), instancePortName) if err != nil || portUUID == "" { return "", nil, fmt.Errorf("Failed getting logical port UUID for security ACL removal: %w", err) } // Add NIC port to network port group (this includes the port in the @internal subject for ACL rules). acl.OVNPortGroupInstanceNICSchedule(portUUID, addChangeSet, acl.OVNIntSwitchPortGroupName(n.ID())) n.logger.Debug("Scheduled logical port for network port group addition", logger.Ctx{"portGroup": acl.OVNIntSwitchPortGroupName(n.ID()), "port": instancePortName}) ingressAction, ingressLogged := n.instanceDeviceACLDefaults(opts.DeviceConfig, "ingress") egressAction, egressLogged := n.instanceDeviceACLDefaults(opts.DeviceConfig, "egress") if len(nicACLNames) > 0 || len(securityACLsRemove) > 0 { var aclNameIDs map[string]int64 err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get map of ACL names to DB IDs (used for generating OVN port group names). acls, err := dbCluster.GetNetworkACLs(ctx, tx.Tx(), dbCluster.NetworkACLFilter{Project: &n.project}) if err != nil { return err } aclNameIDs = make(map[string]int64, len(acls)) for _, acl := range acls { aclNameIDs[acl.Name] = int64(acl.ID) } return nil }) if err != nil { return "", nil, fmt.Errorf("Failed getting network ACL IDs for security ACL setup: %w", err) } // Add port to ACLs requested. if len(nicACLNames) > 0 { // Request our network is setup with the specified ACLs. aclNets := map[string]acl.NetworkACLUsage{ n.Name(): {Name: n.Name(), Type: n.Type(), ID: n.ID(), Config: n.Config()}, } cleanup, err := addressset.OVNEnsureAddressSetsViaACLs(n.state, n.logger, n.ovnnb, n.Project(), nicACLNames) if err != nil { return "", nil, fmt.Errorf("Failed ensuring address sets for nic ACLs are configured in OVN for network: %w", err) } reverter.Add(cleanup) cleanup, err = acl.OVNEnsureACLs(n.state, n.logger, n.ovnnb, n.Project(), aclNameIDs, aclNets, nicACLNames, false) if err != nil { return "", nil, fmt.Errorf("Failed ensuring security ACLs are configured in OVN for instance: %w", err) } reverter.Add(cleanup) for _, aclName := range nicACLNames { aclID, found := aclNameIDs[aclName] if !found { return "", nil, fmt.Errorf("Cannot find security ACL ID for %q", aclName) } directionalPortGroups := acl.OVNACLDirectionalPortGroups(aclID) // Add NIC port to ACL port group. var ingressPortGroupName networkOVN.OVNPortGroup if ingressAction == "allow" { ingressPortGroupName = directionalPortGroups.IngressReversed } else { ingressPortGroupName = directionalPortGroups.Ingress } acl.OVNPortGroupInstanceNICSchedule(portUUID, addChangeSet, ingressPortGroupName) n.logger.Debug("Scheduled logical port for ACL port group addition", logger.Ctx{"networkACL": aclName, "portGroup": ingressPortGroupName, "port": instancePortName}) var egressPortGroupName networkOVN.OVNPortGroup if egressAction == "allow" { egressPortGroupName = directionalPortGroups.EgressReversed } else { egressPortGroupName = directionalPortGroups.Egress } acl.OVNPortGroupInstanceNICSchedule(portUUID, addChangeSet, egressPortGroupName) n.logger.Debug("Scheduled logical port for ACL port group addition", logger.Ctx{"networkACL": aclName, "portGroup": egressPortGroupName, "port": instancePortName}) acl.OVNPortGroupInstanceNICSchedule(portUUID, addChangeSet, directionalPortGroups.All) n.logger.Debug("Scheduled logical port for ACL port group addition", logger.Ctx{"networkACL": aclName, "portGroup": directionalPortGroups.All, "port": instancePortName}) } } // Remove port from ACLs requested. for _, aclName := range securityACLsRemove { // Don't remove ACLs that are in the add ACLs list (there are possibly added from // the network assigned ACLs). if slices.Contains(nicACLNames, aclName) { continue } aclID, found := aclNameIDs[aclName] if !found { return "", nil, fmt.Errorf("Cannot find security ACL ID for %q", aclName) } directionalPortGroups := acl.OVNACLDirectionalPortGroups(aclID) directionalPortGroups.AddToChangeSet(portUUID, removeChangeSet) } } // Add instance NIC switch port to port groups required. Always run this as the addChangeSet should always // be populated even if no ACLs being applied, because the NIC port needs to be added to the network level // port group. n.logger.Debug("Applying instance NIC port group member change sets") err = n.ovnnb.UpdatePortGroupMembers(context.TODO(), addChangeSet, removeChangeSet) if err != nil { return "", nil, fmt.Errorf("Failed applying OVN port group member change sets for instance NIC: %w", err) } // Set the automatic default ACL rule for the port. if len(nicACLNames) > 0 { logPrefix := fmt.Sprintf("%s-%s", opts.InstanceUUID, opts.DeviceName) err = acl.OVNApplyInstanceNICDefaultRules(n.ovnnb, acl.OVNIntSwitchPortGroupName(n.ID()), logPrefix, instancePortName, ingressAction, ingressLogged, egressAction, egressLogged) if err != nil { return "", nil, fmt.Errorf("Failed applying OVN default ACL rules for instance NIC: %w", err) } n.logger.Debug("Set NIC default rule", logger.Ctx{"port": instancePortName, "ingressAction": ingressAction, "ingressLogged": ingressLogged, "egressAction": egressAction, "egressLogged": egressLogged}) } else { err = n.ovnnb.ClearPortGroupPortACLRules(context.TODO(), acl.OVNIntSwitchPortGroupName(n.ID()), instancePortName) if err != nil { return "", nil, fmt.Errorf("Failed clearing OVN default ACL rules for instance NIC: %w", err) } err := addressset.OVNAddressSetsDeleteIfUnused(n.state, n.logger, n.ovnnb, n.Project()) if err != nil { return "", nil, fmt.Errorf("Failed removing unused OVN address sets: %w", err) } n.logger.Debug("Cleared NIC default rule", logger.Ctx{"port": instancePortName}) } var qosPriority uint64 if opts.DeviceConfig["limits.priority"] != "" { qosPriority, err = strconv.ParseUint(opts.DeviceConfig["limits.priority"], 10, 32) if err != nil { return "", nil, fmt.Errorf("Failed to parse limits.priority %q: %w", opts.DeviceConfig["limits.priority"], err) } } else { qosPriority = 100 } egressRate, err := units.ParseBitSizeString(opts.DeviceConfig["limits.egress"]) if err != nil { return "", nil, fmt.Errorf("Failed converting limits.egress to int: %w", err) } ingressRate, err := units.ParseBitSizeString(opts.DeviceConfig["limits.ingress"]) if err != nil { return "", nil, fmt.Errorf("Failed converting limits.ingress to int: %w", err) } if opts.DeviceConfig["limits.max"] != "" { maxRate, err := units.ParseBitSizeString(opts.DeviceConfig["limits.max"]) if err != nil { return "", nil, fmt.Errorf("Failed converting limits.max to int: %w", err) } // Overwrite the egress and ingress rate limits if the max rate limit is set. ingressRate = maxRate egressRate = maxRate } var rules []networkOVN.OVNQoSRule if opts.DeviceConfig["limits.egress"] != "" || opts.DeviceConfig["limits.max"] != "" { egressRate /= 1000 egressRule := networkOVN.OVNQoSRule{ Direction: ovnNB.QoSDirectionFromLport, Action: map[string]int{}, Bandwidth: map[string]int{ "rate": int(egressRate), }, Match: fmt.Sprintf("inport == \"%s\"", instancePortName), Priority: int(qosPriority), } rules = append(rules, egressRule) } if opts.DeviceConfig["limits.ingress"] != "" || opts.DeviceConfig["limits.max"] != "" { ingressRate /= 1000 ingressRule := networkOVN.OVNQoSRule{ Direction: ovnNB.QoSDirectionToLport, Action: map[string]int{}, Bandwidth: map[string]int{ "rate": int(ingressRate), }, Match: fmt.Sprintf("outport == \"%s\"", instancePortName), Priority: int(qosPriority), } rules = append(rules, ingressRule) } err = n.ovnnb.AddLogicalSwitchQoSRules(context.TODO(), n.getIntSwitchName(), instancePortName, rules...) if err != nil { return "", nil, err } reverter.Success() return instancePortName, dnsIPs, nil } // instanceDeviceACLDefaults returns the action and logging mode to use for the specified direction's default rule. // If the security.acls.default.{in,e}gress.action or security.acls.default.{in,e}gress.logged settings are not // specified in the NIC device config, then the settings on the network are used, and if not specified there then // it returns "reject" and false respectively. func (n *ovn) instanceDeviceACLDefaults(deviceConfig deviceConfig.Device, direction string) (string, bool) { defaults := map[string]string{ fmt.Sprintf("security.acls.default.%s.action", direction): "reject", fmt.Sprintf("security.acls.default.%s.logged", direction): "false", } for k := range defaults { if deviceConfig[k] != "" { defaults[k] = deviceConfig[k] } else if n.config[k] != "" { defaults[k] = n.config[k] } } return defaults[fmt.Sprintf("security.acls.default.%s.action", direction)], util.IsTrue(defaults[fmt.Sprintf("security.acls.default.%s.logged", direction)]) } // InstanceDevicePortIPs returns the allocated IPs for a device port. func (n *ovn) InstanceDevicePortIPs(instanceUUID string, deviceName string) ([]net.IP, error) { if instanceUUID == "" { return nil, errors.New("Instance UUID is required") } instancePortName := n.getInstanceDevicePortName(instanceUUID, deviceName) devIPs, err := n.ovnnb.GetLogicalSwitchPortIPs(context.TODO(), instancePortName) if err != nil { return nil, fmt.Errorf("Failed to get OVN switch port IPs: %w", err) } return devIPs, nil } // InstanceDevicePortStop deletes an instance device port from the internal logical switch. func (n *ovn) InstanceDevicePortStop(ovsExternalOVNPort networkOVN.OVNSwitchPort, opts *OVNInstanceNICStopOpts) error { // Decide whether to use OVS provided OVN port name or internally derived OVN port name. instancePortName := ovsExternalOVNPort source := "OVS" if ovsExternalOVNPort == "" { if opts.InstanceUUID == "" { return errors.New("Instance UUID is required") } instancePortName = n.getInstanceDevicePortName(opts.InstanceUUID, opts.DeviceName) source = "internal" } portLocation, err := n.ovnnb.GetLogicalSwitchPortLocation(context.TODO(), instancePortName) if err != nil { return fmt.Errorf("Failed getting instance switch port options: %w", err) } // Don't delete logical switch port if already active on another chassis (i.e during live cluster move). if portLocation != "" && portLocation != n.state.ServerName { return nil } n.logger.Debug("Deleting instance port", logger.Ctx{"port": instancePortName, "source": source}) internalRoutes, externalRoutes, err := n.instanceDevicePortRoutesParse(opts.DeviceConfig) if err != nil { return fmt.Errorf("Failed parsing NIC device routes: %w", err) } var uplink *api.Network if n.config["network"] != "none" { err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Load uplink network config. _, uplink, _, err = tx.GetNetworkInAnyState(ctx, api.ProjectDefaultName, n.config["network"]) return err }) if err != nil { return fmt.Errorf("Failed to load uplink network %q: %w", n.config["network"], err) } } // Get DNS records. dnsUUID, _, dnsIPs, err := n.ovnnb.GetLogicalSwitchPortDNS(context.TODO(), instancePortName) if err != nil { return err } portUUID, err := n.ovnnb.GetLogicalSwitchPortUUID(context.TODO(), instancePortName) if err != nil { return fmt.Errorf("Failed getting logical port UUID for port group removal: %w", err) } if portUUID != "" { portGroups, err := n.ovnnb.GetPortGroupsByPort(context.TODO(), portUUID) if err != nil { return fmt.Errorf("Failed getting port groups for instance NIC port: %w", err) } if len(portGroups) > 0 { removeChangeSet := map[networkOVN.OVNPortGroup][]networkOVN.OVNSwitchPortUUID{} for _, pg := range portGroups { acl.OVNPortGroupInstanceNICSchedule(portUUID, removeChangeSet, pg) } err = n.ovnnb.UpdatePortGroupMembers(context.TODO(), map[networkOVN.OVNPortGroup][]networkOVN.OVNSwitchPortUUID{}, removeChangeSet) if err != nil { return fmt.Errorf("Failed removing instance NIC port from port groups: %w", err) } } } // Cleanup logical switch port and associated config. err = n.ovnnb.CleanupLogicalSwitchPort(context.TODO(), instancePortName, n.getIntSwitchName(), acl.OVNIntSwitchPortGroupName(n.ID()), dnsUUID) if err != nil { return err } var removeRoutes []net.IPNet var removeNATIPs []net.IP if len(dnsIPs) > 0 { // When using l3only mode the instance port's IPs are added as static routes to the router. // So try and remove these in case l3only is (or was) being used. for _, dnsIP := range dnsIPs { removeRoutes = append(removeRoutes, IPToNet(dnsIP)) } // Delete any associated external IP DNAT rules for the DNS IPs. removeNATIPs = append(removeNATIPs, dnsIPs...) } // Delete internal routes. if len(internalRoutes) > 0 { for _, internalRoute := range internalRoutes { removeRoutes = append(removeRoutes, *internalRoute) } } // Delete external routes. for _, externalRoute := range externalRoutes { removeRoutes = append(removeRoutes, *externalRoute) // Remove the DNAT rules when using l2proxy ingress mode on uplink. if uplink != nil && slices.Contains([]string{"l2proxy", ""}, uplink.Config["ovn.ingress_mode"]) { err = SubnetIterate(externalRoute, func(ip net.IP) error { removeNATIPs = append(removeNATIPs, ip) return nil }) if err != nil { return err } } } if len(removeRoutes) > 0 { // Delete routes from local router. err = n.ovnnb.DeleteLogicalRouterRoute(context.TODO(), n.getRouterName(), removeRoutes...) if err != nil { return err } // Delete routes from switch address set. err = n.ovnnb.UpdateAddressSetRemove(context.TODO(), acl.OVNIntSwitchPortGroupAddressSetPrefix(n.ID()), removeRoutes...) if err != nil { return fmt.Errorf("Failed deleting switch address set entries: %w", err) } // Delete routes from peer routers. err = n.forPeers(func(targetOVNNet *ovn) error { targetRouterName := targetOVNNet.getRouterName() err = n.ovnnb.DeleteLogicalRouterRoute(context.TODO(), targetRouterName, removeRoutes...) if err != nil { return fmt.Errorf("Failed deleting static routes from peer network %q in project %q: %w", targetOVNNet.Name(), targetOVNNet.Project(), err) } return nil }) if err != nil { return err } } if len(removeNATIPs) > 0 { err = n.ovnnb.DeleteLogicalRouterNAT(context.TODO(), n.getRouterName(), "dnat_and_snat", false, removeNATIPs...) if err != nil { return err } } // Tear down per‑NIC egress SNAT rules (ipv4/ipv6.address.external) for _, keyPrefix := range []string{"ipv4", "ipv6"} { // Check if the address is present. value := opts.DeviceConfig[fmt.Sprintf("%s.address.external", keyPrefix)] if value == "" { continue } // Validate the address. extIP := net.ParseIP(value) if extIP == nil { return fmt.Errorf("Invalid external address %q", value) } // Remove the SNAT entry. err := n.ovnnb.DeleteLogicalRouterNAT(context.TODO(), n.getRouterName(), "snat", false, extIP) if err != nil && !errors.Is(err, networkOVN.ErrNotFound) { return err } } return nil } // InstanceDevicePortRemove unregisters the NIC device in the OVN database by removing the DNS entry that should // have been created during InstanceDevicePortAdd(). If the DNS record exists at remove time then this indicates // the NIC device was successfully added and this function also clears any DHCP reservations for the NIC's IPs. func (n *ovn) InstanceDevicePortRemove(instanceUUID string, devName string, devConfig deviceConfig.Device, hasDuplicate bool) error { instancePortName := n.getInstanceDevicePortName(instanceUUID, devName) reverter := revert.New() defer reverter.Fail() // If NIC has static IPv4 address then remove the DHCPv4 reservation. if devConfig["ipv4.address"] != "" && !hasDuplicate { ipv4 := net.ParseIP(devConfig["ipv4.address"]) if ipv4 != nil { dhcpReservations, err := n.ovnnb.GetLogicalSwitchDHCPv4Revervations(context.TODO(), n.getIntSwitchName()) if err != nil { return fmt.Errorf("Failed getting DHCPv4 reservations: %w", err) } dhcpReservations = append(dhcpReservations, iprange.Range{Start: ipv4}) dhcpReservationsNew := make([]iprange.Range, 0, len(dhcpReservations)) found := false for _, dhcpReservation := range dhcpReservations { if dhcpReservation.Start.Equal(ipv4) && dhcpReservation.End == nil { found = true continue } dhcpReservationsNew = append(dhcpReservationsNew, dhcpReservation) } if found { err = n.ovnnb.UpdateLogicalSwitchDHCPv4Revervations(context.TODO(), n.getIntSwitchName(), dhcpReservationsNew) if err != nil { return fmt.Errorf("Failed removing DHCPv4 reservation for %q: %w", ipv4.String(), err) } } } } // Remove DNS record if exists. dnsUUID, _, _, err := n.ovnnb.GetLogicalSwitchPortDNS(context.TODO(), instancePortName) if err != nil { return err } if dnsUUID != "" { err = n.ovnnb.DeleteLogicalSwitchPortDNS(context.TODO(), n.getIntSwitchName(), dnsUUID, true) if err != nil { return fmt.Errorf("Failed deleting DNS record: %w", err) } } reverter.Success() return nil } // DHCPv4Subnet returns the DHCPv4 subnet (if DHCP is enabled on network). func (n *ovn) DHCPv4Subnet() *net.IPNet { // DHCP is disabled on this network (an empty ipv4.dhcp setting indicates enabled by default). if util.IsFalse(n.config["ipv4.dhcp"]) { return nil } _, subnet, err := n.parseRouterIntPortIPv4Net() if err != nil { return nil } return subnet } // DHCPv6Subnet returns the DHCPv6 subnet (if DHCP or SLAAC is enabled on network). func (n *ovn) DHCPv6Subnet() *net.IPNet { // DHCP is disabled on this network (an empty ipv6.dhcp setting indicates enabled by default). if util.IsFalse(n.config["ipv6.dhcp"]) { return nil } _, subnet, err := n.parseRouterIntPortIPv6Net() if err != nil { return nil } if subnet != nil { ones, _ := subnet.Mask.Size() if ones < 64 { return nil // OVN only supports DHCPv6 allocated using EUI64 (which needs at least a /64). } } return subnet } // ovnNetworkExternalSubnets returns a list of external subnets used by OVN networks using the same uplink as this // OVN network. OVN networks are considered to be using external subnets for their ipv4.address and/or ipv6.address // if they have NAT disabled, and/or if they have external NAT addresses specified. func (n *ovn) ovnNetworkExternalSubnets(ovnProjectNetworksWithOurUplink map[string][]*api.Network) ([]externalSubnetUsage, error) { externalSubnets := make([]externalSubnetUsage, 0) for netProject, networks := range ovnProjectNetworksWithOurUplink { for _, netInfo := range networks { for _, keyPrefix := range []string{"ipv4", "ipv6"} { // If NAT is disabled, then network subnet is an external subnet. if util.IsFalseOrEmpty(netInfo.Config[fmt.Sprintf("%s.nat", keyPrefix)]) { key := fmt.Sprintf("%s.address", keyPrefix) _, ipNet, err := net.ParseCIDR(netInfo.Config[key]) if err != nil { continue // Skip invalid/unspecified network addresses. } externalSubnets = append(externalSubnets, externalSubnetUsage{ subnet: *ipNet, networkProject: netProject, networkName: netInfo.Name, usageType: subnetUsageNetwork, }) } // Find any external subnets used for network SNAT. if netInfo.Config[fmt.Sprintf("%s.nat.address", keyPrefix)] != "" { key := fmt.Sprintf("%s.nat.address", keyPrefix) subnetSize := 128 if keyPrefix == "ipv4" { subnetSize = 32 } _, ipNet, err := net.ParseCIDR(fmt.Sprintf("%s/%d", netInfo.Config[key], subnetSize)) if err != nil { return nil, fmt.Errorf("Failed parsing %q of %q in project %q: %w", key, netInfo.Name, netProject, err) } externalSubnets = append(externalSubnets, externalSubnetUsage{ subnet: *ipNet, networkProject: netProject, networkName: netInfo.Name, usageType: subnetUsageNetworkSNAT, }) } } } } return externalSubnets, nil } // ovnNICExternalRoutes returns a list of external routes currently used by OVN NICs that are connected to OVN // networks that share the same uplink as this network uses. func (n *ovn) ovnNICExternalRoutes(ovnProjectNetworksWithOurUplink map[string][]*api.Network) ([]externalSubnetUsage, error) { externalRoutes := make([]externalSubnetUsage, 0) err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.InstanceList(ctx, func(inst db.InstanceArgs, p api.Project) error { // Get the instance's effective network project name. instNetworkProject := project.NetworkProjectFromRecord(&p) devices := db.ExpandInstanceDevices(inst.Devices.Clone(), inst.Profiles) // Iterate through each of the instance's devices, looking for OVN NICs that are linked to networks // that use our uplink. for devName, devConfig := range devices { if devConfig["type"] != "nic" { continue } // Check whether the NIC device references one of the OVN networks supplied. if !NICUsesNetwork(devConfig, ovnProjectNetworksWithOurUplink[instNetworkProject]...) { continue } // For OVN NICs that are connected to networks that use the same uplink as we do, check // if they have any external routes configured, and if so add them to the list to return. for _, key := range []string{"ipv4.routes.external", "ipv6.routes.external"} { for _, cidr := range util.SplitNTrimSpace(devConfig[key], ",", -1, true) { _, ipNet, _ := net.ParseCIDR(cidr) if ipNet == nil { // Sip if NIC device doesn't have a valid route. continue } externalRoutes = append(externalRoutes, externalSubnetUsage{ subnet: *ipNet, networkProject: instNetworkProject, networkName: devConfig["network"], instanceProject: inst.Project, instanceName: inst.Name, instanceDevice: devName, usageType: subnetUsageInstance, }) } } } return nil }) }) if err != nil { return nil, err } return externalRoutes, nil } // ovnProjectNetworksWithUplink accepts a map of all networks in all projects and returns a filtered map of OVN // networks that use the uplink specified. func (n *ovn) ovnProjectNetworksWithUplink(uplink string, projectNetworks map[string]map[int64]api.Network) map[string][]*api.Network { ovnProjectNetworksWithOurUplink := make(map[string][]*api.Network) for netProject, networks := range projectNetworks { for _, ni := range networks { network := ni // Local var creating pointer to rather than iterator. // Skip non-OVN networks or those networks that don't use the uplink specified. if network.Type != "ovn" || network.Config["network"] != uplink { continue } if ovnProjectNetworksWithOurUplink[netProject] == nil { ovnProjectNetworksWithOurUplink[netProject] = []*api.Network{&network} } else { ovnProjectNetworksWithOurUplink[netProject] = append(ovnProjectNetworksWithOurUplink[netProject], &network) } } } return ovnProjectNetworksWithOurUplink } // uplinkHasIngressRoutedAnycastIPv4 returns true if the uplink network has IPv4 routed ingress anycast enabled. func (n *ovn) uplinkHasIngressRoutedAnycastIPv4(uplink *api.Network) bool { return util.IsTrue(uplink.Config["ipv4.routes.anycast"]) && uplink.Config["ovn.ingress_mode"] == "routed" } // uplinkHasIngressRoutedAnycastIPv6 returns true if the uplink network has routed IPv6 ingress anycast enabled. func (n *ovn) uplinkHasIngressRoutedAnycastIPv6(uplink *api.Network) bool { return util.IsTrue(uplink.Config["ipv6.routes.anycast"]) && uplink.Config["ovn.ingress_mode"] == "routed" } // handleDependencyChange applies changes from uplink network if specific watched keys have changed. func (n *ovn) handleDependencyChange(uplinkName string, uplinkConfig map[string]string, changedKeys []string) error { // Detect changes that need to be applied to the network. uplinkKeys := []string{"ipv4.ovn.ranges", "ipv6.ovn.ranges"} uplinkNetwork := n.config["network"] for _, k := range uplinkKeys { if !slices.Contains(changedKeys, k) { continue } // Clear any IP that's now invalid. ipv4Ranges, _ := parseIPRanges(uplinkConfig["ipv4.ovn.ranges"]) routerExtPortIPv4 := net.ParseIP(n.config[ovnVolatileUplinkIPv4]) if !ipInPointerRanges(routerExtPortIPv4, ipv4Ranges) { n.config[ovnVolatileUplinkIPv4] = "" } ipv6Ranges, _ := parseIPRanges(uplinkConfig["ipv6.ovn.ranges"]) routerExtPortIPv6 := net.ParseIP(n.config[ovnVolatileUplinkIPv6]) if !ipInPointerRanges(routerExtPortIPv6, ipv6Ranges) { n.config[ovnVolatileUplinkIPv6] = "" } // Save the configuration change. err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { err := tx.UpdateNetwork(ctx, n.project, n.name, n.description, n.config) if err != nil { return fmt.Errorf("Failed saving updated network config: %w", err) } return nil }) if err != nil { return err } // Disconnect from the existing uplink. n.config["network"] = "none" err = n.setup(true) if err != nil { return err } // And re-attach to it. n.config["network"] = uplinkNetwork err = n.setup(true) if err != nil { return err } break // Only run setup once per notification (all changes will be applied). } watchedKeys := []string{"dns.nameservers", "ipv4.gateway", "ipv6.gateway", "ipv4.gateway.hwaddr", "ipv6.gateway.hwaddr"} for _, k := range append(watchedKeys, uplinkKeys...) { if !slices.Contains(changedKeys, k) { continue } n.logger.Debug("Applying changes from uplink network", logger.Ctx{"uplink": uplinkName}) // Re-setup logical network in order to apply uplink changes. err := n.setup(true) if err != nil { return err } break // Only run setup once per notification (all changes will be applied). } // Add or remove the instance NIC l2proxy DNAT_AND_SNAT rules if uplink's ovn.ingress_mode has changed. if slices.Contains(changedKeys, "ovn.ingress_mode") { n.logger.Debug("Applying ingress mode changes from uplink network to instance NICs", logger.Ctx{"uplink": uplinkName}) if slices.Contains([]string{"l2proxy", ""}, uplinkConfig["ovn.ingress_mode"]) { // Get list of active switch ports (avoids repeated querying of OVN NB). activePorts, err := n.ovnnb.GetLogicalSwitchPorts(context.TODO(), n.getIntSwitchName()) if err != nil { return fmt.Errorf("Failed getting active ports: %w", err) } // Find all instance NICs that use this network, and re-add the logical OVN instance port. // This will restore the l2proxy DNAT_AND_SNAT rules. err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.InstanceList(ctx, func(inst db.InstanceArgs, p api.Project) error { // Get the instance's effective network project name. instNetworkProject := project.NetworkProjectFromRecord(&p) // Skip instances who's effective network project doesn't match this network's // project. if n.Project() != instNetworkProject { return nil } devices := db.ExpandInstanceDevices(inst.Devices.Clone(), inst.Profiles) // Iterate through each of the instance's devices, looking for NICs that are linked // this network. for devName, devConfig := range devices { if devConfig["type"] != "nic" || n.Name() != devConfig["network"] { continue } // Check if instance port exists, if not then we can skip. instanceUUID := inst.Config["volatile.uuid"] instancePortName := n.getInstanceDevicePortName(instanceUUID, devName) _, found := activePorts[instancePortName] if !found { continue // No need to update a port that isn't started yet. } if devConfig["hwaddr"] == "" { // Load volatile MAC if no static MAC specified. devConfig["hwaddr"] = inst.Config[fmt.Sprintf("volatile.%s.hwaddr", devName)] } // Re-add logical switch port to apply the l2proxy DNAT_AND_SNAT rules. n.logger.Debug("Re-adding instance OVN NIC port to apply ingress mode changes", logger.Ctx{"project": inst.Project, "instance": inst.Name, "device": devName}) _, _, err = n.InstanceDevicePortStart(&OVNInstanceNICSetupOpts{ InstanceUUID: instanceUUID, DNSName: inst.Name, DeviceName: devName, DeviceConfig: devConfig, UplinkConfig: uplinkConfig, }, nil) if err != nil { n.logger.Error("Failed re-adding instance OVN NIC port", logger.Ctx{"project": inst.Project, "instance": inst.Name, "err": err}) continue } } return nil }) }) if err != nil { return fmt.Errorf("Failed adding instance NIC ingress mode l2proxy rules: %w", err) } } else { // Remove all DNAT_AND_SNAT rules if not using l2proxy ingress mode, as currently we only // use DNAT_AND_SNAT rules for this feature so it is safe to do. err := n.ovnnb.DeleteLogicalRouterNAT(context.TODO(), n.getRouterName(), "dnat_and_snat", true) if err != nil { return fmt.Errorf("Failed deleting instance NIC ingress mode l2proxy rules: %w", err) } } } return nil } // forwardFlattenVIPs flattens forwards into format compatible with OVN load balancers. func (n *ovn) forwardFlattenVIPs(listenAddress net.IP, defaultTargetAddress net.IP, portMaps []*forwardPortMap) []networkOVN.OVNLoadBalancerVIP { var vips []networkOVN.OVNLoadBalancerVIP if defaultTargetAddress != nil { vips = append(vips, networkOVN.OVNLoadBalancerVIP{ ListenAddress: listenAddress, Targets: []networkOVN.OVNLoadBalancerTarget{{Address: defaultTargetAddress}}, }) } for _, portMap := range portMaps { targetPortsLen := len(portMap.target.ports) for i, lp := range portMap.listenPorts { targetPort := lp // Default to using same port as listen port for target port. if targetPortsLen == 1 { // If a single target port is specified, forward all listen ports to it. targetPort = portMap.target.ports[0] } else if targetPortsLen > 1 { // If more than 1 target port specified, use listen port index to get the // target port to use. targetPort = portMap.target.ports[i] } vips = append(vips, networkOVN.OVNLoadBalancerVIP{ ListenAddress: listenAddress, Protocol: portMap.protocol, ListenPort: lp, Targets: []networkOVN.OVNLoadBalancerTarget{ { Address: portMap.target.address, Port: targetPort, }, }, }) } } return vips } // ForwardCreate creates a network forward. func (n *ovn) ForwardCreate(forward api.NetworkForwardsPost, clientType request.ClientType) error { if n.config["network"] == "none" { return errors.New("Isolated OVN network cannot use network forwards") } reverter := revert.New() defer reverter.Fail() if clientType == request.ClientTypeNormal { memberSpecific := false // OVN doesn't support per-member forwards. err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Check if there is an existing forward using the same listen address. _, err := dbCluster.GetNetworkForward(ctx, tx.Tx(), n.ID(), forward.ListenAddress) return err }) if err == nil { return api.StatusErrorf(http.StatusConflict, "A forward for that listen address already exists") } // Convert listen address to subnet so we can check its valid and can be used. listenAddressNet, err := ParseIPToNet(forward.ListenAddress) if err != nil { return fmt.Errorf("Failed parsing %q: %w", forward.ListenAddress, err) } portMaps, err := n.forwardValidate(listenAddressNet.IP, &forward.NetworkForwardPut) if err != nil { return err } // Load the project to get uplink network restrictions. var p *api.Project var uplink *api.Network err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { project, err := dbCluster.GetProject(ctx, tx.Tx(), n.project) if err != nil { return fmt.Errorf("Failed to load network restrictions from project %q: %w", n.project, err) } p, err = project.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed to load network restrictions from project %q: %w", n.project, err) } // Get uplink routes. _, uplink, _, err = tx.GetNetworkInAnyState(ctx, api.ProjectDefaultName, n.config["network"]) if err != nil { return fmt.Errorf("Failed to load uplink network %q: %w", n.config["network"], err) } return nil }) if err != nil { return err } // Get project restricted routes. projectRestrictedSubnets, err := n.projectRestrictedSubnets(p, n.config["network"]) if err != nil { return err } externalSubnetsInUse, err := n.getExternalSubnetInUse(n.config["network"]) if err != nil { return err } // Check the listen address subnet is allowed within both the uplink's external routes and any // project restricted subnets. err = n.validateExternalSubnet(uplink, projectRestrictedSubnets, listenAddressNet) if err != nil { return err } // Check the listen address subnet doesn't fall within any existing OVN network external subnets. for _, externalSubnetUser := range externalSubnetsInUse { // Check if usage is from our own network. if externalSubnetUser.networkProject == n.project && externalSubnetUser.networkName == n.name { // Skip checking conflict with our own network's subnet or SNAT address. // But do not allow other conflict with other usage types within our own network. if externalSubnetUser.usageType == subnetUsageNetwork || externalSubnetUser.usageType == subnetUsageNetworkSNAT { continue } } if SubnetContains(&externalSubnetUser.subnet, listenAddressNet) || SubnetContains(listenAddressNet, &externalSubnetUser.subnet) { // This error is purposefully vague so that it doesn't reveal any names of // resources potentially outside of the network's project. return fmt.Errorf("Forward listen address %q overlaps with another network or NIC", listenAddressNet.String()) } } var forwardID int64 err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Create forward DB record. nodeID := sql.NullInt64{ Valid: memberSpecific, Int64: tx.GetNodeID(), } dbRecord := dbCluster.NetworkForward{ NetworkID: n.ID(), NodeID: nodeID, ListenAddress: forward.ListenAddress, Description: forward.Description, Ports: forward.Ports, } if forward.Ports == nil { dbRecord.Ports = []api.NetworkForwardPort{} } forwardID, err = dbCluster.CreateNetworkForward(ctx, tx.Tx(), dbRecord) if err != nil { return err } err = dbCluster.CreateNetworkForwardConfig(ctx, tx.Tx(), forwardID, forward.Config) if err != nil { return err } return nil }) if err != nil { return err } reverter.Add(func() { _ = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return dbCluster.DeleteNetworkForward(ctx, tx.Tx(), n.ID(), forwardID) }) _ = n.ovnnb.DeleteLoadBalancer(context.TODO(), n.getLoadBalancerName(forward.ListenAddress)) _ = n.forwardBGPSetupPrefixes() }) vips := n.forwardFlattenVIPs(net.ParseIP(forward.ListenAddress), net.ParseIP(forward.Config["target_address"]), portMaps) err = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(forward.ListenAddress), n.getRouterName(), n.getIntSwitchName(), vips...) if err != nil { return fmt.Errorf("Failed applying OVN load balancer: %w", err) } // Add internal static route to the network forward (helps with OVN IC). var nexthop net.IP if listenAddressNet.IP.To4() == nil { routerV6, _, err := n.parseRouterIntPortIPv6Net() if err == nil { nexthop = routerV6 } } else { routerV4, _, err := n.parseRouterIntPortIPv4Net() if err == nil { nexthop = routerV4 } } if nexthop != nil { err = n.ovnnb.CreateLogicalRouterRoute(context.TODO(), n.getRouterName(), true, networkOVN.OVNRouterRoute{NextHop: nexthop, Prefix: *listenAddressNet}) if err != nil { return err } reverter.Add(func() { _ = n.ovnnb.DeleteLogicalRouterRoute(context.TODO(), n.getRouterName(), *listenAddressNet) }) } // Notify all other members to refresh their BGP prefixes. notifier, err := cluster.NewNotifier(n.state, n.state.Endpoints.NetworkCert(), n.state.ServerCert(), cluster.NotifyAll) if err != nil { return err } err = notifier(func(client incus.InstanceServer) error { return client.UseProject(n.project).CreateNetworkForward(n.name, forward) }) if err != nil { return err } } // Refresh exported BGP prefixes on local member. err := n.forwardBGPSetupPrefixes() if err != nil { return fmt.Errorf("Failed applying BGP prefixes for address forwards: %w", err) } reverter.Success() return nil } // ForwardUpdate updates a network forward. func (n *ovn) ForwardUpdate(listenAddress string, req api.NetworkForwardPut, clientType request.ClientType) error { reverter := revert.New() defer reverter.Fail() if clientType == request.ClientTypeNormal { var curForwardID int64 var curForward *api.NetworkForward err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // No memberSpecific filtering needed because OVN doesn't support per-member-forwards dbRecord, err := dbCluster.GetNetworkForward(ctx, tx.Tx(), n.ID(), listenAddress) if err != nil { return err } curForwardID = dbRecord.ID curForward, err = dbRecord.ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return err } portMaps, err := n.forwardValidate(net.ParseIP(curForward.ListenAddress), &req) if err != nil { return err } curForwardEtagHash, err := localUtil.EtagHash(curForward.Etag()) if err != nil { return err } newForward := api.NetworkForward{ ListenAddress: curForward.ListenAddress, NetworkForwardPut: req, } newForwardEtagHash, err := localUtil.EtagHash(newForward.Etag()) if err != nil { return err } if curForwardEtagHash == newForwardEtagHash { return nil // Nothing has changed. } vips := n.forwardFlattenVIPs(net.ParseIP(newForward.ListenAddress), net.ParseIP(newForward.Config["target_address"]), portMaps) err = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(newForward.ListenAddress), n.getRouterName(), n.getIntSwitchName(), vips...) if err != nil { return fmt.Errorf("Failed applying OVN load balancer: %w", err) } reverter.Add(func() { // Apply old settings to OVN on failure. portMaps, err := n.forwardValidate(net.ParseIP(curForward.ListenAddress), &curForward.NetworkForwardPut) if err == nil { vips := n.forwardFlattenVIPs(net.ParseIP(curForward.ListenAddress), net.ParseIP(curForward.Config["target_address"]), portMaps) _ = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(curForward.ListenAddress), n.getRouterName(), n.getIntSwitchName(), vips...) _ = n.forwardBGPSetupPrefixes() } }) err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { fwd := dbCluster.NetworkForward{ NetworkID: n.ID(), ListenAddress: listenAddress, Description: newForward.Description, Ports: newForward.Ports, } err = dbCluster.UpdateNetworkForward(ctx, tx.Tx(), n.ID(), listenAddress, fwd) if err != nil { return err } err = dbCluster.UpdateNetworkForwardConfig(ctx, tx.Tx(), curForwardID, newForward.Config) if err != nil { return err } return nil }) if err != nil { return err } reverter.Add(func() { _ = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { fwd := dbCluster.NetworkForward{ NetworkID: n.ID(), ListenAddress: listenAddress, Description: curForward.Description, Ports: curForward.Ports, } err = dbCluster.UpdateNetworkForward(ctx, tx.Tx(), n.ID(), listenAddress, fwd) if err != nil { return err } err = dbCluster.UpdateNetworkForwardConfig(ctx, tx.Tx(), curForwardID, curForward.Config) if err != nil { return err } return nil }) }) // Notify all other members to refresh their BGP prefixes. notifier, err := cluster.NewNotifier(n.state, n.state.Endpoints.NetworkCert(), n.state.ServerCert(), cluster.NotifyAll) if err != nil { return err } err = notifier(func(client incus.InstanceServer) error { return client.UseProject(n.project).UpdateNetworkForward(n.name, curForward.ListenAddress, req, "") }) if err != nil { return err } } // Refresh exported BGP prefixes on local member. err := n.forwardBGPSetupPrefixes() if err != nil { return fmt.Errorf("Failed applying BGP prefixes for address forwards: %w", err) } reverter.Success() return nil } // ForwardDelete deletes a network forward. func (n *ovn) ForwardDelete(listenAddress string, clientType request.ClientType) error { if clientType == request.ClientTypeNormal { var forwardID int64 var forward *api.NetworkForward err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // No memberSpecific filtering needed because OVN doesn't support per-member-forwards dbRecord, err := dbCluster.GetNetworkForward(ctx, tx.Tx(), n.ID(), listenAddress) if err != nil { return err } forwardID = dbRecord.ID forward, err = dbRecord.ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return err } // Delete the network forward itself. err = n.ovnnb.DeleteLoadBalancer(context.TODO(), n.getLoadBalancerName(forward.ListenAddress)) if err != nil { return fmt.Errorf("Failed deleting OVN load balancer: %w", err) } // Delete static route to network forward if present. vip, err := ParseIPToNet(forward.ListenAddress) if err != nil { return err } _ = n.ovnnb.DeleteLogicalRouterRoute(context.TODO(), n.getRouterName(), *vip) // Delete the database records. err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return dbCluster.DeleteNetworkForward(ctx, tx.Tx(), n.ID(), forwardID) }) if err != nil { return err } // Notify all other members to refresh their BGP prefixes. notifier, err := cluster.NewNotifier(n.state, n.state.Endpoints.NetworkCert(), n.state.ServerCert(), cluster.NotifyAll) if err != nil { return err } err = notifier(func(client incus.InstanceServer) error { return client.UseProject(n.project).DeleteNetworkForward(n.name, forward.ListenAddress) }) if err != nil { return err } } // Refresh exported BGP prefixes on local member. err := n.forwardBGPSetupPrefixes() if err != nil { return fmt.Errorf("Failed applying BGP prefixes for address forwards: %w", err) } return nil } // loadBalancerFlattenVIPs flattens port maps into format compatible with OVN load balancers. func (n *ovn) loadBalancerFlattenVIPs(listenAddress net.IP, portMaps []*loadBalancerPortMap) []networkOVN.OVNLoadBalancerVIP { var vips []networkOVN.OVNLoadBalancerVIP for _, portMap := range portMaps { for i, lp := range portMap.listenPorts { vip := networkOVN.OVNLoadBalancerVIP{ ListenAddress: listenAddress, Protocol: portMap.protocol, ListenPort: lp, } for _, target := range portMap.targets { targetPort := lp // Default to using same port as listen port for target port. targetPortsLen := len(target.ports) if targetPortsLen == 1 { // If a single target port is specified, forward all listen ports to it. targetPort = target.ports[0] } else if targetPortsLen > 1 { // If more than 1 target port specified, use listen port index to get the // target port to use. targetPort = target.ports[i] } vip.Targets = append(vip.Targets, networkOVN.OVNLoadBalancerTarget{ Address: target.address, Port: targetPort, }) } vips = append(vips, vip) } } return vips } // LoadBalancerCreate creates a network load balancer. func (n *ovn) LoadBalancerCreate(loadBalancer api.NetworkLoadBalancersPost, clientType request.ClientType) error { if n.config["network"] == "none" { return errors.New("Isolated OVN network cannot use network load balancers") } reverter := revert.New() defer reverter.Fail() if clientType == request.ClientTypeNormal { err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Check if there is an existing load balancer using the same listen address. _, err := dbCluster.GetNetworkLoadBalancer(ctx, tx.Tx(), n.ID(), loadBalancer.ListenAddress) if err != nil { return err } return nil }) if err == nil { return api.StatusErrorf(http.StatusConflict, "A load balancer for that listen address already exists") } // Convert listen address to subnet so we can check its valid and can be used. listenAddressNet, err := ParseIPToNet(loadBalancer.ListenAddress) if err != nil { return fmt.Errorf("Failed parsing %q: %w", loadBalancer.ListenAddress, err) } portMaps, err := n.loadBalancerValidate(listenAddressNet.IP, &loadBalancer.NetworkLoadBalancerPut) if err != nil { return err } // Load the project to get uplink network restrictions. var p *api.Project var uplink *api.Network err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { project, err := dbCluster.GetProject(ctx, tx.Tx(), n.project) if err != nil { return fmt.Errorf("Failed to load network restrictions from project %q: %w", n.project, err) } p, err = project.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed to load network restrictions from project %q: %w", n.project, err) } // Get uplink routes. _, uplink, _, err = tx.GetNetworkInAnyState(ctx, api.ProjectDefaultName, n.config["network"]) if err != nil { return fmt.Errorf("Failed to load uplink network %q: %w", n.config["network"], err) } return nil }) if err != nil { return err } // Get project restricted routes. projectRestrictedSubnets, err := n.projectRestrictedSubnets(p, n.config["network"]) if err != nil { return err } externalSubnetsInUse, err := n.getExternalSubnetInUse(n.config["network"]) if err != nil { return err } // Check the listen address subnet is allowed within both the uplink's external routes and any // project restricted subnets. err = n.validateExternalSubnet(uplink, projectRestrictedSubnets, listenAddressNet) if err != nil { return err } // Check the listen address subnet doesn't fall within any existing OVN network external subnets. for _, externalSubnetUser := range externalSubnetsInUse { // Check if usage is from our own network. if externalSubnetUser.networkProject == n.project && externalSubnetUser.networkName == n.name { // Skip checking conflict with our own network's subnet or SNAT address. // But do not allow other conflict with other usage types within our own network. if externalSubnetUser.usageType == subnetUsageNetwork || externalSubnetUser.usageType == subnetUsageNetworkSNAT { continue } } if SubnetContains(&externalSubnetUser.subnet, listenAddressNet) || SubnetContains(listenAddressNet, &externalSubnetUser.subnet) { // This error is purposefully vague so that it doesn't reveal any names of // resources potentially outside of the network's project. return fmt.Errorf("Load balancer listen address %q overlaps with another network or NIC", listenAddressNet.String()) } } var loadBalancerID int64 err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Create load balancer DB record. lb := dbCluster.NetworkLoadBalancer{ NetworkID: n.ID(), ListenAddress: loadBalancer.ListenAddress, Description: loadBalancer.Description, Backends: loadBalancer.Backends, Ports: loadBalancer.Ports, } loadBalancerID, err = dbCluster.CreateNetworkLoadBalancer(ctx, tx.Tx(), lb) if err != nil { return err } // Save the load balancer configuration. err = dbCluster.CreateNetworkLoadBalancerConfig(ctx, tx.Tx(), loadBalancerID, loadBalancer.Config) if err != nil { return err } return nil }) if err != nil { return err } reverter.Add(func() { _ = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return dbCluster.DeleteNetworkLoadBalancer(ctx, tx.Tx(), n.ID(), loadBalancerID) }) _ = n.ovnnb.DeleteLoadBalancer(context.TODO(), n.getLoadBalancerName(loadBalancer.ListenAddress)) _ = n.loadBalancerBGPSetupPrefixes() }) vips := n.loadBalancerFlattenVIPs(net.ParseIP(loadBalancer.ListenAddress), portMaps) // Look at health checking configuration. healthCheck, err := n.getHealthCheck(loadBalancer.NetworkLoadBalancerPut) if err != nil { return err } if healthCheck != nil { for i := range vips { vips[i].HealthCheck = healthCheck } } err = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(loadBalancer.ListenAddress), n.getRouterName(), n.getIntSwitchName(), vips...) if err != nil { return fmt.Errorf("Failed applying OVN load balancer: %w", err) } // Add internal static route to the load-balancer (helps with OVN IC). var nexthop net.IP if listenAddressNet.IP.To4() == nil { routerV6, _, err := n.parseRouterIntPortIPv6Net() if err == nil { nexthop = routerV6 } } else { routerV4, _, err := n.parseRouterIntPortIPv4Net() if err == nil { nexthop = routerV4 } } if nexthop != nil { err = n.ovnnb.CreateLogicalRouterRoute(context.TODO(), n.getRouterName(), true, networkOVN.OVNRouterRoute{NextHop: nexthop, Prefix: *listenAddressNet}) if err != nil { return err } reverter.Add(func() { _ = n.ovnnb.DeleteLogicalRouterRoute(context.TODO(), n.getRouterName(), *listenAddressNet) }) } // Notify all other members to refresh their BGP prefixes. notifier, err := cluster.NewNotifier(n.state, n.state.Endpoints.NetworkCert(), n.state.ServerCert(), cluster.NotifyAll) if err != nil { return err } err = notifier(func(client incus.InstanceServer) error { return client.UseProject(n.project).CreateNetworkLoadBalancer(n.name, loadBalancer) }) if err != nil { return err } } // Refresh exported BGP prefixes on local member. err := n.loadBalancerBGPSetupPrefixes() if err != nil { return fmt.Errorf("Failed applying BGP prefixes for load balancers: %w", err) } reverter.Success() return nil } // LoadBalancerUpdate updates a network load balancer. func (n *ovn) LoadBalancerUpdate(listenAddress string, req api.NetworkLoadBalancerPut, clientType request.ClientType) error { reverter := revert.New() defer reverter.Fail() if clientType == request.ClientTypeNormal { var curLoadBalancer *api.NetworkLoadBalancer var curLoadBalancerID int64 err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() // Get the load balancer. dbLoadBalancers, err := dbCluster.GetNetworkLoadBalancers(ctx, tx.Tx(), dbCluster.NetworkLoadBalancerFilter{ NetworkID: &networkID, ListenAddress: &listenAddress, }) if err != nil { return err } if len(dbLoadBalancers) != 1 { return api.StatusErrorf(http.StatusNotFound, "Network load balancer not found") } // Get the API struct. curLoadBalancer, err = dbLoadBalancers[0].ToAPI(ctx, tx.Tx()) if err != nil { return err } curLoadBalancerID = dbLoadBalancers[0].ID return nil }) if err != nil { return err } portMaps, err := n.loadBalancerValidate(net.ParseIP(curLoadBalancer.ListenAddress), &req) if err != nil { return err } curEtagHash, err := localUtil.EtagHash(curLoadBalancer.Etag()) if err != nil { return err } newLoadBalancer := api.NetworkLoadBalancer{ ListenAddress: curLoadBalancer.ListenAddress, NetworkLoadBalancerPut: req, } newLoadBalancerEtagHash, err := localUtil.EtagHash(newLoadBalancer.Etag()) if err != nil { return err } if curEtagHash == newLoadBalancerEtagHash { return nil // Nothing has changed. } vips := n.loadBalancerFlattenVIPs(net.ParseIP(newLoadBalancer.ListenAddress), portMaps) // Look at health checking configuration. healthCheck, err := n.getHealthCheck(newLoadBalancer.NetworkLoadBalancerPut) if err != nil { return err } if healthCheck != nil { for i := range vips { vips[i].HealthCheck = healthCheck } } err = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(newLoadBalancer.ListenAddress), n.getRouterName(), n.getIntSwitchName(), vips...) if err != nil { return fmt.Errorf("Failed applying OVN load balancer: %w", err) } reverter.Add(func() { // Apply old settings to OVN on failure. portMaps, err := n.loadBalancerValidate(net.ParseIP(curLoadBalancer.ListenAddress), &curLoadBalancer.NetworkLoadBalancerPut) if err == nil { vips := n.loadBalancerFlattenVIPs(net.ParseIP(curLoadBalancer.ListenAddress), portMaps) _ = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(curLoadBalancer.ListenAddress), n.getRouterName(), n.getIntSwitchName(), vips...) _ = n.forwardBGPSetupPrefixes() } }) err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { lb := dbCluster.NetworkLoadBalancer{ NetworkID: n.ID(), ListenAddress: listenAddress, Description: newLoadBalancer.Description, Backends: newLoadBalancer.Backends, Ports: newLoadBalancer.Ports, } err = dbCluster.UpdateNetworkLoadBalancer(ctx, tx.Tx(), n.ID(), listenAddress, lb) if err != nil { return err } err = dbCluster.UpdateNetworkLoadBalancerConfig(ctx, tx.Tx(), curLoadBalancerID, newLoadBalancer.Config) if err != nil { return err } return nil }) if err != nil { return err } reverter.Add(func() { _ = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { lb := dbCluster.NetworkLoadBalancer{ NetworkID: n.ID(), ListenAddress: listenAddress, Description: curLoadBalancer.Description, Backends: curLoadBalancer.Backends, Ports: curLoadBalancer.Ports, } err = dbCluster.UpdateNetworkLoadBalancer(ctx, tx.Tx(), n.ID(), listenAddress, lb) if err != nil { return err } err = dbCluster.UpdateNetworkLoadBalancerConfig(ctx, tx.Tx(), curLoadBalancerID, curLoadBalancer.Config) if err != nil { return err } return nil }) }) // Notify all other members to refresh their BGP prefixes. notifier, err := cluster.NewNotifier(n.state, n.state.Endpoints.NetworkCert(), n.state.ServerCert(), cluster.NotifyAll) if err != nil { return err } err = notifier(func(client incus.InstanceServer) error { return client.UseProject(n.project).UpdateNetworkLoadBalancer(n.name, curLoadBalancer.ListenAddress, req, "") }) if err != nil { return err } } // Refresh exported BGP prefixes on local member. err := n.loadBalancerBGPSetupPrefixes() if err != nil { return fmt.Errorf("Failed applying BGP prefixes for load balancers: %w", err) } reverter.Success() return nil } // LoadBalancerState returns the current state of the load balancer. func (n *ovn) LoadBalancerState(lb api.NetworkLoadBalancer) (*api.NetworkLoadBalancerState, error) { lbState := &api.NetworkLoadBalancerState{} if util.IsTrue(lb.Config["healthcheck"]) { lbState.BackendHealth = map[string]api.NetworkLoadBalancerStateBackendHealth{} for _, backend := range lb.Backends { backendHealth := api.NetworkLoadBalancerStateBackendHealth{} backendHealth.Address = backend.TargetAddress backendHealth.Ports = []api.NetworkLoadBalancerStateBackendHealthPort{} for _, lbPort := range lb.Ports { if !slices.Contains(lbPort.TargetBackend, backend.Name) { continue } // Check valid listen port(s) supplied. listenPortRanges := util.SplitNTrimSpace(lbPort.ListenPort, ",", -1, true) if len(listenPortRanges) <= 0 { return nil, fmt.Errorf("Missing listen port in port specification %q", lbPort.ListenPort) } for _, pr := range listenPortRanges { portFirst, portRange, err := ParsePortRange(pr) if err != nil { return nil, fmt.Errorf("Invalid listen port in port specification %q: %w", lbPort.ListenPort, err) } for i := range portRange { port := portFirst + i status, err := n.ovnsb.GetServiceHealth(context.TODO(), backend.TargetAddress, lbPort.Protocol, int(port)) if err != nil { return nil, fmt.Errorf("Failed retrieving OVN load-balancer health: %w", err) } portHealth := api.NetworkLoadBalancerStateBackendHealthPort{ Protocol: lbPort.Protocol, Port: int(port), Status: status, } backendHealth.Ports = append(backendHealth.Ports, portHealth) } } } lbState.BackendHealth[backend.Name] = backendHealth } } return lbState, nil } // LoadBalancerDelete deletes a network load balancer. func (n *ovn) LoadBalancerDelete(listenAddress string, clientType request.ClientType) error { if clientType == request.ClientTypeNormal { var lb *dbCluster.NetworkLoadBalancer err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() dbLoadBalancers, err := dbCluster.GetNetworkLoadBalancers(ctx, tx.Tx(), dbCluster.NetworkLoadBalancerFilter{ NetworkID: &networkID, ListenAddress: &listenAddress, }) if err != nil { return err } if len(dbLoadBalancers) != 1 { return api.StatusErrorf(http.StatusNotFound, "Network load balancer not found") } lb = &dbLoadBalancers[0] return nil }) if err != nil { return err } // Delete the load balancer itself. err = n.ovnnb.DeleteLoadBalancer(context.TODO(), n.getLoadBalancerName(lb.ListenAddress)) if err != nil { return fmt.Errorf("Failed deleting OVN load balancer: %w", err) } // Delete static route to load-balancer if present. vip, err := ParseIPToNet(lb.ListenAddress) if err != nil { return err } _ = n.ovnnb.DeleteLogicalRouterRoute(context.TODO(), n.getRouterName(), *vip) // Delete the database records. err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return dbCluster.DeleteNetworkLoadBalancer(ctx, tx.Tx(), n.ID(), lb.ID) }) if err != nil { return err } // Notify all other members to refresh their BGP prefixes. notifier, err := cluster.NewNotifier(n.state, n.state.Endpoints.NetworkCert(), n.state.ServerCert(), cluster.NotifyAll) if err != nil { return err } err = notifier(func(client incus.InstanceServer) error { return client.UseProject(n.project).DeleteNetworkLoadBalancer(n.name, lb.ListenAddress) }) if err != nil { return err } } // Refresh exported BGP prefixes on local member. err := n.loadBalancerBGPSetupPrefixes() if err != nil { return fmt.Errorf("Failed applying BGP prefixes for address forwards: %w", err) } return nil } func (n *ovn) getHealthCheck(loadBalancer api.NetworkLoadBalancerPut) (*networkOVN.OVNLoadBalancerHealthCheck, error) { // Check if load-balancer is enabled. if !util.IsTrue(loadBalancer.Config["healthcheck"]) { return nil, nil } // Get IPv4 checker. var checkerIPV4 net.IP _, ipv4Net, err := n.parseRouterIntPortIPv4Net() if err == nil && ipv4Net != nil { checkerIPV4 = dhcpalloc.GetIP(ipv4Net, -2) } // Get IPv6 checker. var checkerIPV6 net.IP _, ipv6Net, err := n.parseRouterIntPortIPv6Net() if err == nil && ipv6Net != nil { checkerIPV6 = dhcpalloc.GetIP(ipv6Net, -2) } // Parse the healthcheck options. hcInterval, err := strconv.Atoi(loadBalancer.Config["healthcheck.interval"]) if err != nil && loadBalancer.Config["healthcheck.interval"] != "" { return nil, err } hcTimeout, err := strconv.Atoi(loadBalancer.Config["healthcheck.timeout"]) if err != nil && loadBalancer.Config["healthcheck.timeout"] != "" { return nil, err } hcFailureCount, err := strconv.Atoi(loadBalancer.Config["healthcheck.failure_count"]) if err != nil && loadBalancer.Config["healthcheck.failure_count"] != "" { return nil, err } hcSuccessCount, err := strconv.Atoi(loadBalancer.Config["healthcheck.success_count"]) if err != nil && loadBalancer.Config["healthcheck.success_count"] != "" { return nil, err } // Prepare the load-balancer health check. healthCheck := &networkOVN.OVNLoadBalancerHealthCheck{ CheckerIPV4: checkerIPV4, CheckerIPV6: checkerIPV6, Interval: hcInterval, Timeout: hcTimeout, FailureCount: hcFailureCount, SuccessCount: hcSuccessCount, } return healthCheck, nil } // Leases returns a list of leases for the OVN network. Those are directly extracted from the OVN database. func (n *ovn) Leases(projectName string, clientType request.ClientType) ([]api.NetworkLease, error) { var err error leases := []api.NetworkLease{} // If requested project matches network's project then include gateway IPs. if projectName == n.project { // Add our own gateway IPs. for _, addr := range []string{n.config["ipv4.address"], n.config["ipv6.address"]} { ip, _, _ := net.ParseCIDR(addr) if ip != nil { leases = append(leases, api.NetworkLease{ Hostname: fmt.Sprintf("%s.gw", n.Name()), Address: ip.String(), Type: "gateway", }) } } } // Get all the instances in the requested project that are connected to this network. filter := dbCluster.InstanceFilter{Project: &projectName} err = UsedByInstanceDevices(n.state, n.Project(), n.Name(), n.Type(), func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error { // Get the instance UUID needed for OVN port name generation. instanceUUID := inst.Config["volatile.uuid"] if instanceUUID == "" { return nil } devIPs, err := n.InstanceDevicePortIPs(instanceUUID, nicName) if err != nil { return nil // There is likely no active port and so no leases. } // Fill in the hwaddr from volatile. if nicConfig["hwaddr"] == "" { nicConfig["hwaddr"] = inst.Config[fmt.Sprintf("volatile.%s.hwaddr", nicName)] } // Parse the MAC. hwAddr, _ := net.ParseMAC(nicConfig["hwaddr"]) // Add the leases. for _, ip := range devIPs { leaseType := "dynamic" if nicConfig["ipv4.address"] == ip.String() || nicConfig["ipv6.address"] == ip.String() { leaseType = "static" } leases = append(leases, api.NetworkLease{ Hostname: inst.Name, Address: ip.String(), Hwaddr: hwAddr.String(), Type: leaseType, Location: inst.Node, }) } return nil }, filter) if err != nil { return nil, err } return leases, nil } // localPeerCreate creates a network peering with another local network. func (n *ovn) localPeerCreate(peer api.NetworkPeersPost) error { ctx := context.TODO() reverter := revert.New() defer reverter.Fail() // Get the peer DB record. var peerInfo *api.NetworkPeer err := n.state.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var err error // Load peering to get mutual peering info. dbPeer, err := dbCluster.GetNetworkPeer(ctx, tx.Tx(), n.id, peer.Name) if err != nil { return fmt.Errorf("Failed getting network peer DB object: %w", err) } var apiErr error peerInfo, apiErr = dbPeer.ToAPI(ctx, tx.Tx()) if apiErr != nil { return fmt.Errorf("Failed converting network peer DB object to API object: %w", apiErr) } return nil }) if err != nil { return err } // Validate the peer. if peerInfo.Status != api.NetworkStatusCreated { return fmt.Errorf("Only peerings in %q state can be setup", api.NetworkStatusCreated) } // Apply router security policies. // Should have been done during network setup, but ensure its done here anyway. err = n.logicalRouterPolicySetup(n.ovnnb) if err != nil { return fmt.Errorf("Failed applying local router security policy: %w", err) } activeLocalNICPorts, err := n.ovnnb.GetLogicalSwitchPorts(context.TODO(), n.getIntSwitchName()) if err != nil { return fmt.Errorf("Failed getting active NIC ports: %w", err) } var localNICRoutes []net.IPNet // Get routes on instance NICs connected to local network to be added as routes to target network. err = UsedByInstanceDevices(n.state, n.Project(), n.Name(), n.Type(), func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error { instancePortName := n.getInstanceDevicePortName(inst.Config["volatile.uuid"], nicName) _, found := activeLocalNICPorts[instancePortName] if !found { return nil // Don't add config for instance NICs that aren't started. } localNICRoutes = append(localNICRoutes, n.instanceNICGetRoutes(nicConfig)...) return nil }) if err != nil { return fmt.Errorf("Failed getting instance NIC routes on local network: %w", err) } targetNet, err := LoadByName(n.state, peer.TargetProject, peer.TargetNetwork) if err != nil { return fmt.Errorf("Failed loading target network: %w", err) } targetOVNNet, ok := targetNet.(*ovn) if !ok { return errors.New("Target network is not ovn interface type") } opts, err := n.peerGetLocalOpts(localNICRoutes) if err != nil { return err } // Ensure local subnets and all active NIC routes are present in internal switch's address set. err = n.ovnnb.UpdateAddressSetAdd(context.TODO(), acl.OVNIntSwitchPortGroupAddressSetPrefix(n.ID()), opts.TargetRouterRoutes...) if err != nil { return fmt.Errorf("Failed adding active NIC routes to switch address set: %w", err) } err = n.peerSetup(n.ovnnb, targetOVNNet, *opts) if err != nil { return err } reverter.Success() return nil } // remotePeerCreate creates a network peering with an OVN-IC. func (n *ovn) remotePeerCreate(peer api.NetworkPeersPost) error { ctx := context.TODO() reverter := revert.New() defer reverter.Fail() // Load the project. var p *api.Project err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { project, err := dbCluster.GetProject(ctx, tx.Tx(), n.project) if err != nil { return err } p, err = project.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return fmt.Errorf("Failed to load network restrictions from project %q: %w", n.project, err) } // Validate restrictions. if !project.NetworkIntegrationAllowed(p.Config, peer.TargetIntegration) { return api.StatusErrorf(http.StatusForbidden, "Project isn't allowed to use this network integration") } // Load the integration. var integration *api.NetworkIntegration err = n.state.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { entry, err := dbCluster.GetNetworkIntegration(ctx, tx.Tx(), peer.TargetIntegration) if err != nil { return err } integration, err = entry.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return fmt.Errorf("Failed to load network integration %q: %w", peer.TargetIntegration, err) } // Get ICNB. icnb, err := networkOVN.NewICNB(integration.Config["ovn.northbound_connection"], integration.Config["ovn.ca_cert"], integration.Config["ovn.client_cert"], integration.Config["ovn.client_key"]) if err != nil { return err } // Get ICSB. icsb, err := networkOVN.NewICSB(integration.Config["ovn.southbound_connection"], integration.Config["ovn.ca_cert"], integration.Config["ovn.client_cert"], integration.Config["ovn.client_key"]) if err != nil { return err } // Get the OVN AZ name. azName, err := n.ovnnb.GetName(ctx) if err != nil { return err } // Get the list of interconnect gateways. gateways, err := icsb.GetGateways(ctx, azName) if err != nil { return err } if len(gateways) == 0 { return errors.New("No chassis gateways available for interconnect") } // Determine the transit switch name. pattern := integration.Config["ovn.transit.pattern"] if pattern == "" { pattern = "ts-incus-{{ integrationName }}-{{ projectName }}-{{ networkName }}" } tsNameRendered, err := internalUtil.RenderTemplate(pattern, pongo2.Context{ "projectName": n.project, "networkName": n.name, "integrationName": integration.Name, "peerName": peer.Name, }) if err != nil { return err } tsName := networkOVN.OVNSwitch(tsNameRendered) // Determine the chassis group name. cgName := networkOVN.OVNChassisGroup(tsName) // Create the chassis group. err = n.ovnnb.CreateChassisGroup(ctx, cgName, false) if err != nil { return err } reverter.Add(func() { _ = n.ovnnb.DeleteChassisGroup(ctx, cgName) }) // Seed the stable random number generator with the transit switch name. // This should cause a reasonable spread of networks on the available IC gateway chassis. r, err := localUtil.GetStableRandomGenerator(tsNameRendered) if err != nil { return fmt.Errorf("Failed generating stable random chassis group priority: %w", err) } // Assign some priorities. for _, gateway := range gateways { err = n.ovnnb.SetChassisGroupPriority(ctx, cgName, gateway, r.Intn(ovnChassisPriorityMax+1)) if err != nil { return err } } // Create the transit switch if it doesn't exist already. err = icnb.CreateTransitSwitch(ctx, string(tsName), true) if err != nil { return err } // Check that the switch appeared on the local OVN. found := false for range 10 { // Try to get the switch. logicalSwitch, err := n.ovnnb.GetLogicalSwitch(ctx, tsName) if err != nil && !errors.Is(err, networkOVN.ErrNotFound) { return err } if logicalSwitch != nil { found = true break } time.Sleep(time.Second) } if !found { return errors.New("New transit switch didn't appear within 10s") } // Get router MAC address. routerMAC, err := n.getRouterMAC() if err != nil { return err } // Get bridge MTU. bridgeMTU := int(n.getBridgeMTU()) if bridgeMTU == 0 { bridgeMTU = 1500 } // Get peering addresses. ipv4Net, ipv6Net, err := icnb.CreateTransitSwitchAllocation(ctx, string(tsName), azName) if err != nil { return err } // Determine logical router port name. lrpName := networkOVN.OVNRouterPort(tsName) // Create the logical router port. err = n.ovnnb.CreateLogicalRouterPort(ctx, n.getRouterName(), lrpName, routerMAC, uint32(bridgeMTU), []*net.IPNet{ipv4Net, ipv6Net}, cgName, false) if err != nil { return err } reverter.Add(func() { _ = n.ovnnb.DeleteLogicalRouterPort(ctx, n.getRouterName(), lrpName) }) // Create the logical switch port. lspOpts := &networkOVN.OVNSwitchPortOpts{RouterPort: lrpName} err = n.ovnnb.CreateLogicalSwitchPort(ctx, tsName, networkOVN.OVNSwitchPort(fmt.Sprintf("%s-%s", tsName, azName)), lspOpts, false) if err != nil { return err } reverter.Add(func() { _ = n.ovnnb.DeleteLogicalSwitchPort(ctx, tsName, networkOVN.OVNSwitchPort(fmt.Sprintf("%s-%s", tsName, azName))) }) reverter.Success() return nil } // PeerCreate creates a network peering. func (n *ovn) PeerCreate(peer api.NetworkPeersPost) error { reverter := revert.New() defer reverter.Fail() // Default type is local. if peer.Type == "" { peer.Type = "local" } // Perform create-time validation. if peer.Type == "local" { // Default to network's project if target project not specified. if peer.TargetProject == "" { peer.TargetProject = n.Project() } // Target network name is required. if peer.TargetNetwork == "" { return api.StatusErrorf(http.StatusBadRequest, "Target network is required") } } else if peer.Type == "remote" { // Target integration name is required. if peer.TargetIntegration == "" { return api.StatusErrorf(http.StatusBadRequest, "Target integration is required") } } // Look for an existing entry. var peers map[int64]*api.NetworkPeer err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Use generated function to get peers. netID := n.ID() filter := dbCluster.NetworkPeerFilter{NetworkID: &netID} dbPeers, err := dbCluster.GetNetworkPeers(ctx, tx.Tx(), filter) if err != nil { return fmt.Errorf("Failed loading network peer DB objects: %w", err) } // Convert DB objects to API objects and build the map. peers = make(map[int64]*api.NetworkPeer, len(dbPeers)) for _, dbPeer := range dbPeers { peer, err := dbPeer.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed converting network peer DB object to API object: %w", err) } peers[dbPeer.ID] = peer } return nil }) if err != nil { return err } for _, existingPeer := range peers { if peer.Name == existingPeer.Name { return api.StatusErrorf(http.StatusConflict, "A peer for that name already exists") } if peer.Type == "local" && peer.TargetProject == existingPeer.TargetProject && peer.TargetNetwork == existingPeer.TargetNetwork { return api.StatusErrorf(http.StatusConflict, "A peer for that target network already exists") } } // Perform general (create and update) validation. err = n.peerValidate(peer.Name, &peer.NetworkPeerPut) if err != nil { return err } var peerID int64 var mutualExists bool err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Create peer DB record. record := dbCluster.NetworkPeer{ NetworkID: n.ID(), Name: peer.Name, Description: peer.Description, Type: dbCluster.NetworkPeerTypes[peer.Type], } switch peer.Type { case "remote": integrationID, err := dbCluster.GetNetworkIntegrationID(ctx, tx.Tx(), peer.TargetIntegration) if err != nil { return err } id := sql.NullInt64{} err = id.Scan(integrationID) if err != nil { return err } record.TargetNetworkIntegrationID = id case "local": // Check if target peer already exists. peers, err := dbCluster.GetNetworkPeers(ctx, tx.Tx(), dbCluster.NetworkPeerFilter{ Type: &record.Type, TargetNetworkProject: &n.project, TargetNetworkName: &n.name, }) if err != nil && !errors.Is(err, sql.ErrNoRows) { return err } if len(peers) == 1 { // Update the target peer. peer := peers[0] empty := sql.NullString{} peer.TargetNetworkProject = empty peer.TargetNetworkName = empty targetID := sql.NullInt64{} err = targetID.Scan(n.id) if err != nil { return err } peer.TargetNetworkID = targetID err = dbCluster.UpdateNetworkPeer(ctx, tx.Tx(), peer.NetworkID, peer.Name, peer) if err != nil { return err } // Set our target network ID to match. id := sql.NullInt64{} err = id.Scan(peer.NetworkID) if err != nil { return err } record.TargetNetworkID = id mutualExists = true } else if len(peers) == 0 { networkProjectName := sql.NullString{} err = networkProjectName.Scan(peer.TargetProject) if err != nil { return err } networkName := sql.NullString{} err = networkName.Scan(peer.TargetNetwork) if err != nil { return err } record.TargetNetworkProject = networkProjectName record.TargetNetworkName = networkName } else { return errors.New("More than one matching network peer was found") } } peerID, err = dbCluster.CreateNetworkPeer(ctx, tx.Tx(), record) if err != nil { return err } return nil }) if err != nil { return err } reverter.Add(func() { _ = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { err := dbCluster.DeleteNetworkPeer(ctx, tx.Tx(), n.ID(), peerID) if errors.Is(err, dbCluster.ErrNotFound) { return nil } return err }) }) // Apply the OVN configuration. if peer.Type == "local" && mutualExists { err := n.localPeerCreate(peer) if err != nil { return err } } else if peer.Type == "remote" { err := n.remotePeerCreate(peer) if err != nil { return err } } reverter.Success() return nil } // peerGetLocalOpts returns peering options prefilled with local router and local NIC routes config. // It can then be modified with the target peering network options. func (n *ovn) peerGetLocalOpts(localNICRoutes []net.IPNet) (*networkOVN.OVNRouterPeering, error) { localRouterPortMAC, err := n.getRouterMAC() if err != nil { return nil, fmt.Errorf("Failed getting router MAC address: %w", err) } opts := networkOVN.OVNRouterPeering{ LocalRouter: n.getRouterName(), LocalRouterPortMAC: localRouterPortMAC, TargetRouterRoutes: localNICRoutes, // Pre-fill with local NIC routes. } routerIntPortIPv4, routerIntPortIPv4Net, err := n.parseRouterIntPortIPv4Net() if err != nil { return nil, fmt.Errorf("Failed parsing local router's peering port IPv4 net: %w", err) } if routerIntPortIPv4 != nil && routerIntPortIPv4Net != nil { // Add a copy of the CIDR subnet to the target router's routes. opts.TargetRouterRoutes = append(opts.TargetRouterRoutes, *routerIntPortIPv4Net) // Convert the IPNet to include the specific router IP with a single host subnet. routerIntPortIPv4Net.IP = routerIntPortIPv4 routerIntPortIPv4Net.Mask = net.CIDRMask(32, 32) opts.LocalRouterPortIPs = append(opts.LocalRouterPortIPs, *routerIntPortIPv4Net) } routerIntPortIPv6, routerIntPortIPv6Net, err := n.parseRouterIntPortIPv6Net() if err != nil { return nil, fmt.Errorf("Failed parsing local router's peering port IPv6 net: %w", err) } if routerIntPortIPv6 != nil && routerIntPortIPv6Net != nil { // Add a copy of the CIDR subnet to the target router's routers. opts.TargetRouterRoutes = append(opts.TargetRouterRoutes, *routerIntPortIPv6Net) // Convert the IPNet to include the specific router IP with a single host subnet. routerIntPortIPv6Net.IP = routerIntPortIPv6 routerIntPortIPv6Net.Mask = net.CIDRMask(128, 128) opts.LocalRouterPortIPs = append(opts.LocalRouterPortIPs, *routerIntPortIPv6Net) } return &opts, err } // peerSetup applies the network peering configuration to both networks. // Accepts an OVN client, a target OVN network, and a set of OVNRouterPeering options pre-filled with local config. func (n *ovn) peerSetup(ovnnb *networkOVN.NB, targetOVNNet *ovn, opts networkOVN.OVNRouterPeering) error { targetRouterMAC, err := targetOVNNet.getRouterMAC() if err != nil { return fmt.Errorf("Failed getting target router MAC address: %w", err) } // Populate config based on target network. opts.LocalRouterPort = n.getLogicalRouterPeerPortName(targetOVNNet.ID()) opts.TargetRouter = targetOVNNet.getRouterName() opts.TargetRouterPort = targetOVNNet.getLogicalRouterPeerPortName(n.ID()) opts.TargetRouterPortMAC = targetRouterMAC routerIntPortIPv4, routerIntPortIPv4Net, err := targetOVNNet.parseRouterIntPortIPv4Net() if err != nil { return fmt.Errorf("Failed parsing target router's peering port IPv4 net: %w", err) } if routerIntPortIPv4 != nil && routerIntPortIPv4Net != nil { // Add a copy of the CIDR subnet to the local router's routers. opts.LocalRouterRoutes = append(opts.LocalRouterRoutes, *routerIntPortIPv4Net) // Convert the IPNet to include the specific router IP with a single host subnet. routerIntPortIPv4Net.IP = routerIntPortIPv4 routerIntPortIPv4Net.Mask = net.CIDRMask(32, 32) opts.TargetRouterPortIPs = append(opts.TargetRouterPortIPs, *routerIntPortIPv4Net) } routerIntPortIPv6, routerIntPortIPv6Net, err := targetOVNNet.parseRouterIntPortIPv6Net() if err != nil { return fmt.Errorf("Failed parsing target router's peering port IPv6 net: %w", err) } if routerIntPortIPv6 != nil && routerIntPortIPv6Net != nil { // Add a copy of the CIDR subnet to the local router's routers. opts.LocalRouterRoutes = append(opts.LocalRouterRoutes, *routerIntPortIPv6Net) // Convert the IPNet to include the specific router IP with a single host subnet. routerIntPortIPv6Net.IP = routerIntPortIPv6 routerIntPortIPv6Net.Mask = net.CIDRMask(128, 128) opts.TargetRouterPortIPs = append(opts.TargetRouterPortIPs, *routerIntPortIPv6Net) } // Get list of active switch ports (avoids repeated querying of OVN NB). activeTargetNICPorts, err := n.ovnnb.GetLogicalSwitchPorts(context.TODO(), targetOVNNet.getIntSwitchName()) if err != nil { return fmt.Errorf("Failed getting active NIC ports: %w", err) } // Get routes on instance NICs connected to target network to be added as routes to local network. err = UsedByInstanceDevices(n.state, targetOVNNet.Project(), targetOVNNet.Name(), targetOVNNet.Type(), func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error { instancePortName := targetOVNNet.getInstanceDevicePortName(inst.Config["volatile.uuid"], nicName) _, found := activeTargetNICPorts[instancePortName] if !found { return nil // Don't add config for instance NICs that aren't started. } opts.LocalRouterRoutes = append(opts.LocalRouterRoutes, n.instanceNICGetRoutes(nicConfig)...) return nil }) if err != nil { return fmt.Errorf("Failed getting instance NIC routes on target network: %w", err) } // Ensure routes are added to target switch address sets. err = n.ovnnb.UpdateAddressSetAdd(context.TODO(), acl.OVNIntSwitchPortGroupAddressSetPrefix(targetOVNNet.ID()), opts.LocalRouterRoutes...) if err != nil { return fmt.Errorf("Failed adding target switch subnet address set entries: %w", err) } err = targetOVNNet.logicalRouterPolicySetup(n.ovnnb) if err != nil { return fmt.Errorf("Failed applying target router security policy: %w", err) } err = n.ovnnb.CreateLogicalRouterPeering(context.TODO(), opts) if err != nil { return fmt.Errorf("Failed applying OVN network peering: %w", err) } return nil } // PeerUpdate updates a network peering. func (n *ovn) PeerUpdate(peerName string, req api.NetworkPeerPut) error { reverter := revert.New() defer reverter.Fail() var curPeer *api.NetworkPeer var dbCurPeer *dbCluster.NetworkPeer err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error dbCurPeer, err = dbCluster.GetNetworkPeer(ctx, tx.Tx(), n.id, peerName) if err != nil { return fmt.Errorf("Failed getting network peer DB object: %w", err) } curPeer, err = dbCurPeer.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed converting network peer DB object to API object: %w", err) } return nil }) if err != nil { return err } err = n.peerValidate(peerName, &req) if err != nil { return err } curPeerEtagHash, err := localUtil.EtagHash(curPeer.Etag()) if err != nil { return err } newPeer := api.NetworkPeer{ Name: curPeer.Name, NetworkPeerPut: req, } newPeerEtagHash, err := localUtil.EtagHash(newPeer.Etag()) if err != nil { return err } if curPeerEtagHash == newPeerEtagHash { return nil // Nothing has changed. } err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Update the description field from the input. dbCurPeer.Description = newPeer.Description // Update the main peer object. err = dbCluster.UpdateNetworkPeer(ctx, tx.Tx(), n.id, dbCurPeer.Name, *dbCurPeer) if err != nil { return fmt.Errorf("Failed to update network peer: %w", err) } // Update the peer configuration. err = dbCluster.UpdateNetworkPeerConfig(ctx, tx.Tx(), dbCurPeer.ID, newPeer.Config) if err != nil { return fmt.Errorf("Failed to update network peer config: %w", err) } return nil }) if err != nil { return err } reverter.Success() return nil } // localPeerDelete deletes a network peering with another local network. func (n *ovn) localPeerDelete(peer *api.NetworkPeer) error { targetNet, err := LoadByName(n.state, peer.TargetProject, peer.TargetNetwork) if err != nil { return fmt.Errorf("Failed loading target network: %w", err) } targetOVNNet, ok := targetNet.(*ovn) if !ok { return errors.New("Target network is not ovn interface type") } opts := networkOVN.OVNRouterPeering{ LocalRouter: n.getRouterName(), LocalRouterPort: n.getLogicalRouterPeerPortName(targetOVNNet.ID()), TargetRouter: targetOVNNet.getRouterName(), TargetRouterPort: targetOVNNet.getLogicalRouterPeerPortName(n.ID()), } err = n.ovnnb.DeleteLogicalRouterPeering(context.TODO(), opts) if err != nil { return fmt.Errorf("Failed deleting OVN network peering: %w", err) } err = n.logicalRouterPolicySetup(n.ovnnb, targetOVNNet.ID()) if err != nil { return fmt.Errorf("Failed applying local router security policy: %w", err) } err = targetOVNNet.logicalRouterPolicySetup(n.ovnnb, n.ID()) if err != nil { return fmt.Errorf("Failed applying target router security policy: %w", err) } return nil } // remotePeerDelete deletes a network peering with an OVN-IC. func (n *ovn) remotePeerDelete(peer *api.NetworkPeer) error { ctx := context.TODO() // Load the integration. var integration *api.NetworkIntegration err := n.state.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { entry, err := dbCluster.GetNetworkIntegration(ctx, tx.Tx(), peer.TargetIntegration) if err != nil { return err } integration, err = entry.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return fmt.Errorf("Failed to load network integration %q: %w", peer.TargetIntegration, err) } // Get ICNB. icnb, err := networkOVN.NewICNB(integration.Config["ovn.northbound_connection"], integration.Config["ovn.ca_cert"], integration.Config["ovn.client_cert"], integration.Config["ovn.client_key"]) if err != nil { return err } // Get the OVN AZ name. azName, err := n.ovnnb.GetName(ctx) if err != nil { return err } // Determine the transit switch name. pattern := integration.Config["ovn.transit.pattern"] if pattern == "" { pattern = "ts-incus-{{ integrationName }}-{{ projectName }}-{{ networkName }}" } tsNameRendered, err := internalUtil.RenderTemplate(pattern, pongo2.Context{ "projectName": n.project, "networkName": n.name, "integrationName": integration.Name, "peerName": peer.Name, }) if err != nil { return err } tsName := networkOVN.OVNSwitch(tsNameRendered) // Delete logical switch port err = n.ovnnb.DeleteLogicalSwitchPort(ctx, tsName, networkOVN.OVNSwitchPort(fmt.Sprintf("%s-%s", tsName, azName))) if err != nil { return err } // Determine logical router port name. lrpName := networkOVN.OVNRouterPort(tsName) // Delete logical router port err = n.ovnnb.DeleteLogicalRouterPort(ctx, n.getRouterName(), lrpName) if err != nil { return err } // Determine the chassis group name. cgName := networkOVN.OVNChassisGroup(tsName) // Delete chassis group. err = n.ovnnb.DeleteChassisGroup(ctx, cgName) if err != nil && !errors.Is(err, networkOVN.ErrNotManaged) { return err } // Delete transit switch if empty icSwitch, err := n.ovnnb.GetLogicalSwitch(ctx, tsName) if err != nil { return err } if len(icSwitch.Ports) == 0 { err = icnb.DeleteTransitSwitch(ctx, string(tsName), false) if err != nil && !errors.Is(err, networkOVN.ErrNotManaged) { return err } } else { // Get the OVN AZ name. azName, err := n.ovnnb.GetName(ctx) if err != nil { return err } // Release peering addresses. err = icnb.DeleteTransitSwitchAllocation(ctx, string(tsName), azName) if err != nil { return err } } return nil } // PeerDelete deletes a network peering. func (n *ovn) PeerDelete(peerName string) error { var peerID int64 var peer *api.NetworkPeer err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbPeer, err := dbCluster.GetNetworkPeer(ctx, tx.Tx(), n.id, peerName) if err != nil { return fmt.Errorf("Failed getting network peer DB object: %w", err) } peerID = dbPeer.ID peer, err = dbPeer.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed converting network peer DB object to API object: %w", err) } return nil }) if err != nil { return err } isUsed, err := n.peerIsUsed(peer.Name) if err != nil { return err } if isUsed { return errors.New("Cannot delete a peer that is in use") } if peer.Status == api.NetworkStatusCreated { if peer.Type == "local" { err := n.localPeerDelete(peer) if err != nil { return err } } else if peer.Type == "remote" { err := n.remotePeerDelete(peer) if err != nil { return err } } } err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Deactivate any existing peer. if peer.Type == "local" { peers, err := dbCluster.GetNetworkPeers(ctx, tx.Tx(), dbCluster.NetworkPeerFilter{TargetNetworkID: &n.id}) if err != nil && !errors.Is(err, sql.ErrNoRows) { return err } for _, peer := range peers { peer.TargetNetworkID = sql.NullInt64{} err = dbCluster.UpdateNetworkPeer(ctx, tx.Tx(), peer.NetworkID, peer.Name, peer) if err != nil { return err } } } // Delete the peer. err := dbCluster.DeleteNetworkPeer(ctx, tx.Tx(), n.id, peerID) if err != nil { return err } return nil }) if err != nil { return err } return nil } // forPeers runs f for each target peer network that this network is connected to. func (n *ovn) forPeers(f func(targetOVNNet *ovn) error) error { var peers map[int64]*api.NetworkPeer err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Use generated function to get peers. netID := n.ID() filter := dbCluster.NetworkPeerFilter{NetworkID: &netID} dbPeers, err := dbCluster.GetNetworkPeers(ctx, tx.Tx(), filter) if err != nil { return fmt.Errorf("Failed loading network peer DB objects: %w", err) } // Convert DB objects to API objects and build the map. peers = make(map[int64]*api.NetworkPeer, len(dbPeers)) for _, dbPeer := range dbPeers { peer, err := dbPeer.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed converting network peer DB object to API object: %w", err) } peers[dbPeer.ID] = peer } return nil }) if err != nil { return err } for _, peer := range peers { // Skip partially defined peers. if peer.Status != api.NetworkStatusCreated { continue } // Skip remote peers (no local networks to load). if peer.Type == "remote" { continue } targetNet, err := LoadByName(n.state, peer.TargetProject, peer.TargetNetwork) if err != nil { return fmt.Errorf("Failed loading target network: %w", err) } targetOVNNet, ok := targetNet.(*ovn) if !ok { return errors.New("Target network is not ovn interface type") } err = f(targetOVNNet) if err != nil { return err } } return nil } // loadBalancerBGPSetupPrefixes exports external load balancer addresses as prefixes. func (n *ovn) loadBalancerBGPSetupPrefixes() error { listenAddresses := []string{} err := n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { networkID := n.ID() dbLoadBalancers, err := dbCluster.GetNetworkLoadBalancers(ctx, tx.Tx(), dbCluster.NetworkLoadBalancerFilter{ NetworkID: &networkID, }) if err != nil { return err } for _, lb := range dbLoadBalancers { listenAddresses = append(listenAddresses, lb.ListenAddress) } return nil }) if err != nil { return fmt.Errorf("Failed loading network forwards: %w", err) } listenAddressesByFamily := map[uint][]string{ 4: make([]string, 0), 6: make([]string, 0), } for _, listenAddress := range listenAddresses { if strings.Contains(listenAddress, ":") { listenAddressesByFamily[6] = append(listenAddressesByFamily[6], listenAddress) } else { listenAddressesByFamily[4] = append(listenAddressesByFamily[4], listenAddress) } } // Use load balancer specific owner string (different from the network prefixes) so that these can be // reapplied independently of the network's own prefixes. bgpOwner := fmt.Sprintf("network_%d_load_balancer", n.id) // Clear existing address load balancer prefixes for network. err = n.state.BGP.RemovePrefixByOwner(bgpOwner) if err != nil { return err } // Add the new prefixes. for _, ipVersion := range []uint{4, 6} { nextHopAddr := n.bgpNextHopAddress(ipVersion) natEnabled := util.IsTrue(n.config[fmt.Sprintf("ipv%d.nat", ipVersion)]) _, netSubnet, _ := net.ParseCIDR(n.config[fmt.Sprintf("ipv%d.address", ipVersion)]) routeSubnetSize := 128 if ipVersion == 4 { routeSubnetSize = 32 } // Export external forward listen addresses. for _, listenAddress := range listenAddressesByFamily[ipVersion] { listenAddr := net.ParseIP(listenAddress) // Don't export internal address forwards (those inside the NAT enabled network's subnet). if natEnabled && netSubnet != nil && netSubnet.Contains(listenAddr) { continue } // Check health of load-balancer (if enabled). online := false for _, protocol := range []string{"tcp", "udp"} { lb, err := n.ovnnb.GetLoadBalancer(context.TODO(), networkOVN.OVNLoadBalancer(fmt.Sprintf("incus-net%d-lb-%s-%s", n.id, listenAddr.String(), protocol))) if err != nil { continue } lbOnline, err := n.ovnsb.CheckLoadBalancerOnline(context.TODO(), *lb) if err != nil { continue } if lbOnline { online = true break } } if !online { continue } _, ipRouteSubnet, err := net.ParseCIDR(fmt.Sprintf("%s/%d", listenAddr.String(), routeSubnetSize)) if err != nil { return err } err = n.state.BGP.AddPrefix(*ipRouteSubnet, nextHopAddr, bgpOwner) if err != nil { return err } } } return nil } // getTunnels fetches tunnels from a network configuration. func (n *ovn) getTunnels(config map[string]string) []string { tunnels := []string{} for k := range config { if !strings.HasPrefix(k, "tunnel.") { continue } fields := strings.Split(k, ".") if !slices.Contains(tunnels, fields[1]) { tunnels = append(tunnels, fields[1]) } } return tunnels } // getTunnelsFromChangedKeys fetches tunnels from a string slice. func (n *ovn) getTunnelsFromChangedKeys(keys []string) []string { tunnels := make(map[string]string) for _, v := range keys { tunnels[v] = "" } return n.getTunnels(tunnels) } // getTunnelFullName returns a full name of a tunnel. func (n *ovn) getTunnelFullName(tunnel string, idx int) string { return fmt.Sprintf("%s-%s-%d", n.name, tunnel, idx) } // getTunnelLspName returns a name of a lsp for a tunnel. func (n *ovn) getTunnelLspName(tunnel string) string { return fmt.Sprintf("tunnel-%s", tunnel) } // updateTunnels updates the tunnel configuration by removing and recreating all necessary objects. // When the 'reinitialize' flag is set, it removes and recreates all tunnels from the configuration, // not just those referenced in changedKeys. func (n *ovn) updateTunnels(newConfig map[string]string, changedKeys []string, reinitialize bool) error { err := n.deleteTunnels(changedKeys, false, reinitialize) if err != nil { return err } chassisName, err := n.getActiveChassisName() if err != nil { return err } if chassisName == n.state.ServerName || chassisName == n.state.OS.Hostname { err := n.deleteTunnels(changedKeys, true, reinitialize) if err != nil { return err } err = n.createTunnels(newConfig) if err != nil { return err } } return nil } // createTunnels creates tunnels specified in the config. func (n *ovn) createTunnels(newConfig map[string]string) error { var err error tunnels := n.getTunnels(newConfig) // Configure tunnels. for _, tunnel := range tunnels { getConfig := func(key string) string { return newConfig[fmt.Sprintf("tunnel.%s.%s", tunnel, key)] } tunProtocol := getConfig("protocol") // Configure the tunnel. switch tunProtocol { case "gre": remotes := getConfig("remote") tunLocal := net.ParseIP(getConfig("local")) for idx, remote := range strings.Split(remotes, ",") { tunRemote := net.ParseIP(strings.TrimSpace(remote)) tunName := n.getTunnelFullName(tunnel, idx) // Skip partial configs. if tunLocal == nil || tunRemote == nil { return nil } gretap := &ip.Gretap{ Link: ip.Link{Name: tunName}, Local: tunLocal, Remote: tunRemote, } err := gretap.Add() if err != nil { return err } err = gretap.SetUp() if err != nil { return err } err = n.createOVNTunnel(tunName) if err != nil { return err } } case "vxlan": tunName := n.getTunnelFullName(tunnel, 0) tunGroup := net.ParseIP(getConfig("group")) tunInterface := getConfig("interface") vxlan := &ip.Vxlan{ Link: ip.Link{Name: tunName}, } if tunGroup == nil { tunGroup = net.IPv4(239, 0, 0, 1) // 239.0.0.1 } devName := tunInterface if devName == "" { _, devName, err = DefaultGatewaySubnetV4() if err != nil { return err } } vxlan.Group = tunGroup vxlan.DevName = devName tunPort := getConfig("port") if tunPort != "" { vxlan.DstPort, err = strconv.Atoi(tunPort) if err != nil { return err } } tunID := getConfig("id") if tunID == "" { vxlan.VxlanID = 1 } else { vxlan.VxlanID, err = strconv.Atoi(tunID) if err != nil { return err } } tunTTL := getConfig("ttl") if tunTTL == "" { vxlan.TTL = 1 } else { vxlan.TTL, err = strconv.Atoi(tunTTL) if err != nil { return err } } err := vxlan.Add() if err != nil { return err } err = vxlan.SetUp() if err != nil { return err } err = n.createOVNTunnel(tunName) if err != nil { return err } } } return nil } // createOVNTunnel creates OVN objects needed for a tunnel. func (n *ovn) createOVNTunnel(tunName string) error { reverter := revert.New() defer reverter.Fail() lspName := networkOVN.OVNSwitchPort(n.getTunnelLspName(tunName)) err := n.ovnnb.CreateLogicalSwitchPort(context.TODO(), n.getIntSwitchName(), lspName, &networkOVN.OVNSwitchPortOpts{ IPV4: "none", IPV6: "none", Promiscuous: true, }, true) if err != nil { return fmt.Errorf("Failed to create logical switch port for %s: %w", tunName, err) } reverter.Add(func() { _ = n.ovnnb.DeleteLogicalSwitchPort(context.TODO(), n.getIntSwitchName(), lspName) }) integrationBridge := n.state.GlobalConfig.NetworkOVNIntegrationBridge() vswitch, err := n.state.OVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } err = vswitch.CreateBridgePort(context.TODO(), integrationBridge, tunName, true) if err != nil { return err } reverter.Add(func() { _ = vswitch.DeleteBridgePort(context.TODO(), integrationBridge, tunName) }) // Link OVS port to OVN logical port. err = vswitch.AssociateInterfaceOVNSwitchPort(context.TODO(), tunName, string(lspName)) if err != nil { return err } reverter.Success() return nil } // deleteTunnels removes tunnels from the system and OVN (if needed). // When the 'deleteAll' flag is set, it removes all tunnels specified in the // configuration, not just those listed in changedKeys. func (n *ovn) deleteTunnels(config []string, withOVN bool, deleteAll bool) error { var tunnels []string if deleteAll { tunnels = n.getTunnels(n.config) } else { tunnels = n.getTunnelsFromChangedKeys(config) } for _, tunnel := range tunnels { err := n.deleteLinkTunnel(tunnel) if err != nil { return err } if withOVN { err = n.deleteOVNTunnel(tunnel) if err != nil { return err } } } return nil } // deleteLinkTunnel removes tunnel-related interfaces from the system. func (n *ovn) deleteLinkTunnel(tunnel string) error { idx := 0 for { tunName := n.getTunnelFullName(tunnel, idx) l, err := ip.LinkByName(tunName) if err != nil { // If interface doesn't exist assume there is no more valid tunnels. return nil } err = l.Delete() if err != nil { return err } idx += 1 } } // deleteOVNTunnel removes tunnel-related objects from OVN. func (n *ovn) deleteOVNTunnel(tunnel string) error { idx := 0 for { tunName := n.getTunnelFullName(tunnel, idx) integrationBridge := n.state.GlobalConfig.NetworkOVNIntegrationBridge() vswitch, err := n.state.OVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } ports, err := vswitch.GetBridgePorts(context.TODO(), integrationBridge) if err != nil { return err } if !slices.Contains(ports, tunName) { // If logical port doesn't exist assume there is no more valid ports. return nil } err = vswitch.DeleteBridgePort(context.TODO(), integrationBridge, tunName) if err != nil { return err } lspName := networkOVN.OVNSwitchPort(n.getTunnelLspName(tunName)) err = n.ovnnb.DeleteLogicalSwitchPort(context.TODO(), n.getIntSwitchName(), lspName) if err != nil { return err } idx += 1 } } // getActiveChassisName retrieves the active chassis name. func (n *ovn) getActiveChassisName() (string, error) { if n.config["network"] == "none" { return "", nil } // Get the current active chassis. chassis, err := n.ovnsb.GetLogicalRouterPortActiveChassisHostname(context.TODO(), n.getRouterExtPortName()) if err != nil { return "", err } return chassis, nil } incus-7.0.0/internal/server/network/driver_physical.go000066400000000000000000000413611517523235500232050ustar00rootroot00000000000000package network import ( "context" "errors" "fmt" "maps" "net" "slices" "strconv" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/network/ovs" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // physical represents a physical network. type physical struct { common } // DBType returns the network type DB ID. func (n *physical) DBType() db.NetworkType { return db.NetworkTypePhysical } // Validate network config. func (n *physical) Validate(config map[string]string, clientType request.ClientType) error { rules := map[string]func(value string) error{ // gendoc:generate(entity=network_physical, group=common, key=parent) // // --- // type: string // condition: - // shortdesc: Existing interface to use for network "parent": validate.Required(validate.IsNotEmpty, validate.IsInterfaceName), // gendoc:generate(entity=network_physical, group=common, key=mtu) // // --- // type: integer // condition: - // shortdesc: The MTU of the new interface "mtu": validate.Optional(validate.IsNetworkMTU), // gendoc:generate(entity=network_physical, group=common, key=vlan) // // --- // type: integer // condition: - // shortdesc: The VLAN ID to attach to "vlan": validate.Optional(validate.IsNetworkVLAN), // gendoc:generate(entity=network_physical, group=common, key=vlan.tagged) // // --- // type: integer // condition: Parent must be an existing bridge // shortdesc: Comma-delimited list of VLAN IDs or VLAN ranges to join for tagged traffic "vlan.tagged": func(value string) error { if value == "" { return nil } // Check that none of the supplied VLAN IDs are the same as the untagged VLAN ID. for _, vlanID := range util.SplitNTrimSpace(value, ",", -1, true) { _, _, err := validate.ParseNetworkVLANRange(vlanID) if err != nil { return err } } return nil }, // gendoc:generate(entity=network_physical, group=common, key=gvrp) // // --- // type: bool // condition: - // defaultdesc: 'false' // shortdesc: Register VLAN using GARP VLAN Registration Protocol "gvrp": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_physical, group=ipv4, key=ipv4.gateway) // // --- // type: string // condition: standard mode // shortdesc: IPv4 address for the gateway and network (CIDR) "ipv4.gateway": validate.Optional(validate.IsNetworkAddressCIDRV4), // gendoc:generate(entity=network_physical, group=ipv6, key=ipv6.gateway) // // --- // type: string // condition: standard mode // shortdesc: IPv6 address for the gateway and network (CIDR) "ipv6.gateway": validate.Optional(validate.IsNetworkAddressCIDRV6), // gendoc:generate(entity=network_physical, group=ipv4, key=ipv4.gateway.hwaddr) // // --- // type: string // shortdesc: MAC address of the gateway (to avoid discovery) "ipv4.gateway.hwaddr": validate.Optional(validate.IsNetworkMAC), // gendoc:generate(entity=network_physical, group=ipv6, key=ipv6.gateway.hwaddr) // // --- // type: string // shortdesc: MAC address of the gateway (to avoid discovery) "ipv6.gateway.hwaddr": validate.Optional(validate.IsNetworkMAC), // gendoc:generate(entity=network_physical, group=ipv4, key=ipv4.ovn.ranges) // // --- // type: string // condition: - // shortdesc: Comma-separated list of IPv4 ranges to use for child OVN network routers (FIRST-LAST format) "ipv4.ovn.ranges": validate.Optional(validate.IsListOf(validate.IsNetworkRangeV4)), // gendoc:generate(entity=network_physical, group=ipv6, key=ipv6.ovn.ranges) // // --- // type: string // condition: - // shortdesc: Comma-separated list of IPv6 ranges to use for child OVN network routers (FIRST-LAST format) "ipv6.ovn.ranges": validate.Optional(validate.IsListOf(validate.IsNetworkRangeV6)), // gendoc:generate(entity=network_physical, group=ipv4, key=ipv4.routes) // // --- // type: string // condition: IPv4 address // shortdesc: Comma-separated list of additional IPv4 CIDR subnets that can be used with child OVN networks `ipv4.routes.external` setting "ipv4.routes": validate.Optional(validate.IsListOf(validate.IsNetworkV4)), // gendoc:generate(entity=network_physical, group=ipv4, key=ipv4.routes.anycast) // // --- // type: bool // condition: IPv4 address // defaultdesc: 'false' // shortdesc: Allow the overlapping routes to be used on multiple networks/NIC at the same time "ipv4.routes.anycast": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_physical, group=ipv6, key=ipv6.routes) // // --- // type: string // condition: IPv6 address // shortdesc: Comma-separated list of additional IPv6 CIDR subnets that can be used with child OVN networks `ipv6.routes.external` setting "ipv6.routes": validate.Optional(validate.IsListOf(validate.IsNetworkV6)), // gendoc:generate(entity=network_physical, group=ipv6, key=ipv6.routes.anycast) // // --- // type: bool // condition: IPv6 address // defaultdesc: 'false' // shortdesc: Allow the overlapping routes to be used on multiple networks/NIC at the same time "ipv6.routes.anycast": validate.Optional(validate.IsBool), // gendoc:generate(entity=network_physical, group=dns, key=dns.nameservers) // // --- // type: string // condition: standard mode // shortdesc: List of DNS server IPs on `physical` network "dns.nameservers": validate.Optional(validate.IsListOf(validate.IsNetworkAddress)), // gendoc:generate(entity=network_physical, group=ovn, key=ovn.ingress_mode) // // --- // type: string // condition: standard mode // defaultdesc: `l2proxy` // shortdesc: Sets the method how OVN NIC external IPs will be advertised on uplink network: `l2proxy` (proxy ARP/NDP) or `routed` "ovn.ingress_mode": validate.Optional(validate.IsOneOf("l2proxy", "routed")), "volatile.last_state.created": validate.Optional(validate.IsBool), } // gendoc:generate(entity=network_physical, group=bgp, key=bgp.peers.NAME.address) // // --- // type: string // condition: BGP server // defaultdesc: - // shortdesc: Peer address (IPv4 or IPv6) for use by `ovn` downstream networks // gendoc:generate(entity=network_physical, group=bgp, key=bgp.peers.NAME.asn) // // --- // type: integer // condition: BGP server // defaultdesc: - // shortdesc: Peer AS number for use by `ovn` downstream networks // gendoc:generate(entity=network_physical, group=bgp, key=bgp.peers.NAME.password) // // --- // type: string // condition: BGP server // defaultdesc: - (no password) // shortdesc: Peer session password (optional) for use by `ovn` downstream networks // gendoc:generate(entity=network_physical, group=bgp, key=bgp.peers.NAME.holdtime) // // --- // type: integer // condition: BGP server // defaultdesc: `180` // shortdesc: Peer session hold time (in seconds; optional) // Add the BGP validation rules. bgpRules, err := n.bgpValidationRules(config) if err != nil { return err } maps.Copy(rules, bgpRules) // Validate the configuration. err = n.validate(config, rules) if err != nil { return err } return nil } // checkParentUse checks if parent is already in use by another network or instance device. func (n *physical) checkParentUse(ourConfig map[string]string) (bool, error) { // Check if we're dealing with a physical network backed by a bridge. if ourConfig["parent"] != "" && util.PathExists(fmt.Sprintf("/sys/class/net/%s/bridge", ourConfig["parent"])) { return false, nil } // Get all managed networks across all projects. var err error var projectNetworks map[string]map[int64]api.Network err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { projectNetworks, err = tx.GetCreatedNetworks(ctx) return err }) if err != nil { return false, fmt.Errorf("Failed to load all networks: %w", err) } for projectName, networks := range projectNetworks { if projectName != api.ProjectDefaultName { continue // Only default project networks can possibly reference a physical interface. } for _, network := range networks { if network.Name == n.name { continue // Ignore our own DB record. } // Check if another network is using our parent. if network.Config["parent"] == ourConfig["parent"] { // If either network doesn't specify a vlan, or both specify same vlan, // then we can't use this parent. if (network.Config["vlan"] == "" || ourConfig["vlan"] == "") || network.Config["vlan"] == ourConfig["vlan"] { return true, nil } } } } return false, nil } // Create checks whether the referenced parent interface is used by other networks or instance devices, as we // need to have exclusive access to the interface. func (n *physical) Create(clientType request.ClientType) error { n.logger.Debug("Create", logger.Ctx{"clientType": clientType, "config": n.config}) // We only need to check in the database once, not on every clustered node. if clientType == request.ClientTypeNormal { inUse, err := n.checkParentUse(n.config) if err != nil { return err } if inUse { return fmt.Errorf("Parent interface %q in use by another network", n.config["parent"]) } } return nil } // Delete deletes a network. func (n *physical) Delete(clientType request.ClientType) error { n.logger.Debug("Delete", logger.Ctx{"clientType": clientType}) err := n.Stop() if err != nil { return err } return n.common.delete(clientType) } // Rename renames a network. func (n *physical) Rename(newName string) error { n.logger.Debug("Rename", logger.Ctx{"newName": newName}) // Rename common steps. err := n.common.rename(newName) if err != nil { return err } return nil } // Start sets up some global configuration. func (n *physical) Start() error { n.logger.Debug("Start") reverter := revert.New() defer reverter.Fail() reverter.Add(func() { n.setUnavailable() }) err := n.setup(nil) if err != nil { return err } reverter.Success() // Ensure network is marked as available now its started. n.setAvailable() return nil } func (n *physical) setup(oldConfig map[string]string) error { reverter := revert.New() defer reverter.Fail() if !InterfaceExists(n.config["parent"]) && n.config["parent"] != "none" { return fmt.Errorf("Parent interface %q not found", n.config["parent"]) } hostName := GetHostDevice(n.config["parent"], n.config["vlan"]) created, err := VLANInterfaceCreate(n.config["parent"], hostName, n.config["vlan"], util.IsTrue(n.config["gvrp"])) if err != nil { return err } if created { reverter.Add(func() { _ = InterfaceRemove(hostName) }) } // Set the MTU. if n.config["mtu"] != "" { mtu, err := strconv.ParseUint(n.config["mtu"], 10, 32) if err != nil { return fmt.Errorf("Invalid MTU %q: %w", n.config["mtu"], err) } phyLink := &ip.Link{Name: hostName} err = phyLink.SetMTU(uint32(mtu)) if err != nil { return fmt.Errorf("Failed setting MTU %q on %q: %w", n.config["mtu"], phyLink.Name, err) } } // Record if we created this device or not (if we have not already recorded that we created it previously), // so it can be removed on stop. This way we won't overwrite the setting on daemon restart. if util.IsFalseOrEmpty(n.config["volatile.last_state.created"]) { n.config["volatile.last_state.created"] = fmt.Sprintf("%t", created) err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateNetwork(ctx, n.project, n.name, n.description, n.config) }) if err != nil { return fmt.Errorf("Failed saving volatile config: %w", err) } } // Setup BGP. err = n.bgpSetup(oldConfig) if err != nil { return err } reverter.Success() return nil } // Stop stops is a no-op. func (n *physical) Stop() error { n.logger.Debug("Stop") // Clear BGP. err := n.bgpClear(n.config) if err != nil { return err } hostName := GetHostDevice(n.config["parent"], n.config["vlan"]) // Only try and remove created VLAN interfaces. if n.config["vlan"] != "" && util.IsTrue(n.config["volatile.last_state.created"]) && InterfaceExists(hostName) { err := InterfaceRemove(hostName) if err != nil { return err } } // Reset MTU back to 1500 if overridden in config. if n.config["mtu"] != "" && InterfaceExists(hostName) { var resetMTU uint32 = 1500 link := &ip.Link{Name: hostName} err := link.SetMTU(1500) if err != nil { return fmt.Errorf("Failed setting MTU %d on %q: %w", resetMTU, link.Name, err) } } // Remove last state config. delete(n.config, "volatile.last_state.created") err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateNetwork(ctx, n.project, n.name, n.description, n.config) }) if err != nil { return fmt.Errorf("Failed removing volatile config: %w", err) } return nil } // Update updates the network. Accepts notification boolean indicating if this update request is coming from a // cluster notification, in which case do not update the database, just apply local changes needed. func (n *physical) Update(newNetwork api.NetworkPut, targetNode string, clientType request.ClientType) error { n.logger.Debug("Update", logger.Ctx{"clientType": clientType, "newNetwork": newNetwork}) dbUpdateNeeded, changedKeys, oldNetwork, err := n.common.configChanged(newNetwork) if err != nil { return err } if !dbUpdateNeeded { return nil // Nothing changed. } // If the network as a whole has not had any previous creation attempts, or the node itself is still // pending, then don't apply the new settings to the node, just to the database record (ready for the // actual global create request to be initiated). if n.Status() == api.NetworkStatusPending || n.LocalStatus() == api.NetworkStatusPending { return n.common.update(newNetwork, targetNode, clientType) } reverter := revert.New() defer reverter.Fail() hostNameChanged := slices.Contains(changedKeys, "vlan") || slices.Contains(changedKeys, "parent") // We only need to check in the database once, not on every clustered node. if clientType == request.ClientTypeNormal { if hostNameChanged { isUsed, err := n.IsUsed(true) if isUsed || err != nil { return errors.New("Cannot update network parent interface when in use") } inUse, err := n.checkParentUse(newNetwork.Config) if err != nil { return err } if inUse { return fmt.Errorf("Parent interface %q in use by another network", newNetwork.Config["parent"]) } } } if hostNameChanged { err = n.Stop() if err != nil { return err } // Remove the volatile last state from submitted new config if present. delete(newNetwork.Config, "volatile.last_state.created") } // Define a function which reverts everything. reverter.Add(func() { // Reset changes to all nodes and database. _ = n.common.update(oldNetwork, targetNode, clientType) }) // Apply changes to all nodes and database. err = n.common.update(newNetwork, targetNode, clientType) if err != nil { return err } if !hostNameChanged { err = n.setup(oldNetwork.Config) if err != nil { return err } } else { err = n.setup(nil) if err != nil { return err } } // Update OVS bridge entries (for dependent OVN networks). if hostNameChanged { vswitch, err := n.state.OVS() if err == nil { ovsBridge := fmt.Sprintf("incusovn%d", n.id) oldHostName := GetHostDevice(oldNetwork.Config["parent"], oldNetwork.Config["vlan"]) newHostName := GetHostDevice(newNetwork.Config["parent"], newNetwork.Config["vlan"]) err := vswitch.DeleteBridgePort(context.TODO(), ovsBridge, oldHostName) if err != nil && !errors.Is(err, ovs.ErrNotFound) { return err } err = vswitch.CreateBridgePort(context.TODO(), ovsBridge, newHostName, true) if err != nil && !errors.Is(err, ovs.ErrNotFound) { return err } } } reverter.Success() // Notify dependent networks (those using this network as their uplink) of the changes. // Do this after the network has been successfully updated so that a failure to notify a dependent network // doesn't prevent the network itself from being updated. if clientType == request.ClientTypeNormal && len(changedKeys) > 0 { n.common.notifyDependentNetworks(changedKeys) } return nil } // DHCPv4Subnet returns the DHCPv4 subnet (if DHCP is enabled on network). func (n *physical) DHCPv4Subnet() *net.IPNet { _, subnet, err := net.ParseCIDR(n.config["ipv4.gateway"]) if err != nil { return nil } return subnet } // DHCPv6Subnet returns the DHCPv6 subnet (if DHCP or SLAAC is enabled on network). func (n *physical) DHCPv6Subnet() *net.IPNet { _, subnet, err := net.ParseCIDR(n.config["ipv6.gateway"]) if err != nil { return nil } return subnet } incus-7.0.0/internal/server/network/driver_sriov.go000066400000000000000000000075461517523235500225420ustar00rootroot00000000000000package network import ( "fmt" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/validate" ) // sriov represents an sriov network. type sriov struct { common } // DBType returns the network type DB ID. func (n *sriov) DBType() db.NetworkType { return db.NetworkTypeSriov } // Validate network config. func (n *sriov) Validate(config map[string]string, clientType request.ClientType) error { rules := map[string]func(value string) error{ // gendoc:generate(entity=network_sriov, group=common, key=parent) // // --- // type: string // condition: - // shortdesc: Parent interface to create `sriov` NICs on "parent": validate.Required(validate.IsNotEmpty, validate.IsInterfaceName), // gendoc:generate(entity=network_sriov, group=common, key=mtu) // // --- // type: integer // condition: - // shortdesc: The MTU of the new interface "mtu": validate.Optional(validate.IsNetworkMTU), // gendoc:generate(entity=network_sriov, group=common, key=vlan) // // --- // type: integer // condition: - // shortdesc: The VLAN ID to attach to "vlan": validate.Optional(validate.IsNetworkVLAN), // gendoc:generate(entity=network_sriov, group=common, key=user.*) // // --- // type: string // condition: - // shortdesc: User-provided free-form key/value pairs } err := n.validate(config, rules) if err != nil { return err } return nil } // Delete deletes a network. func (n *sriov) Delete(clientType request.ClientType) error { n.logger.Debug("Delete", logger.Ctx{"clientType": clientType}) return n.common.delete(clientType) } // Rename renames a network. func (n *sriov) Rename(newName string) error { n.logger.Debug("Rename", logger.Ctx{"newName": newName}) // Rename common steps. err := n.common.rename(newName) if err != nil { return err } return nil } // Start starts is a no-op. func (n *sriov) Start() error { n.logger.Debug("Start") reverter := revert.New() defer reverter.Fail() reverter.Add(func() { n.setUnavailable() }) if !InterfaceExists(n.config["parent"]) { return fmt.Errorf("Parent interface %q not found", n.config["parent"]) } reverter.Success() // Ensure network is marked as available now its started. n.setAvailable() return nil } // Stop stops is a no-op. func (n *sriov) Stop() error { n.logger.Debug("Stop") return nil } // Update updates the network. Accepts notification boolean indicating if this update request is coming from a // cluster notification, in which case do not update the database, just apply local changes needed. func (n *sriov) Update(newNetwork api.NetworkPut, targetNode string, clientType request.ClientType) error { n.logger.Debug("Update", logger.Ctx{"clientType": clientType, "newNetwork": newNetwork}) dbUpdateNeeded, _, oldNetwork, err := n.common.configChanged(newNetwork) if err != nil { return err } if !dbUpdateNeeded { return nil // Nothing changed. } // If the network as a whole has not had any previous creation attempts, or the node itself is still // pending, then don't apply the new settings to the node, just to the database record (ready for the // actual global create request to be initiated). if n.Status() == api.NetworkStatusPending || n.LocalStatus() == api.NetworkStatusPending { return n.common.update(newNetwork, targetNode, clientType) } reverter := revert.New() defer reverter.Fail() // Define a function which reverts everything. reverter.Add(func() { // Reset changes to all nodes and database. _ = n.common.update(oldNetwork, targetNode, clientType) }) // Apply changes to all nodes and database. err = n.common.update(newNetwork, targetNode, clientType) if err != nil { return err } reverter.Success() return nil } incus-7.0.0/internal/server/network/errors.go000066400000000000000000000003721517523235500213270ustar00rootroot00000000000000package network import ( "errors" ) // ErrUnknownDriver is the "Unknown driver" error. var ErrUnknownDriver = errors.New("Unknown driver") // ErrNotImplemented is the "Not implemented" error. var ErrNotImplemented = errors.New("Not implemented") incus-7.0.0/internal/server/network/network_interface.go000066400000000000000000000050571517523235500235310ustar00rootroot00000000000000package network import ( "net" "github.com/lxc/incus/v7/internal/iprange" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" ) // Type represents a network driver type. type Type interface { FillConfig(config map[string]string) error Info() Info ValidateName(name string) error Type() string DBType() db.NetworkType } // Network represents an instantiated network. type Network interface { Type // Load. init(s *state.State, id int64, projectName string, netInfo *api.Network, netNodes map[int64]db.NetworkNode) error // Config. Validate(config map[string]string, clientType request.ClientType) error ID() int64 Name() string Project() string Description() string Status() string LocalStatus() string Config() map[string]string Locations() []string IsUsed(instanceOnly bool) (bool, error) IsManaged() bool DHCPv4Subnet() *net.IPNet DHCPv6Subnet() *net.IPNet DHCPv4Ranges() []iprange.Range DHCPv6Ranges() []iprange.Range // Actions. Create(clientType request.ClientType) error Start() error Stop() error Rename(name string) error Update(newNetwork api.NetworkPut, targetNode string, clientType request.ClientType) error HandleHeartbeat(heartbeatData *cluster.APIHeartbeat) error Delete(clientType request.ClientType) error handleDependencyChange(netName string, netConfig map[string]string, changedKeys []string) error // Status. State() (*api.NetworkState, error) Leases(projectName string, clientType request.ClientType) ([]api.NetworkLease, error) // Address Forwards. ForwardCreate(forward api.NetworkForwardsPost, clientType request.ClientType) error ForwardUpdate(listenAddress string, newForward api.NetworkForwardPut, clientType request.ClientType) error ForwardDelete(listenAddress string, clientType request.ClientType) error // Load Balancers. LoadBalancerCreate(loadBalancer api.NetworkLoadBalancersPost, clientType request.ClientType) error LoadBalancerUpdate(listenAddress string, newLoadBalancer api.NetworkLoadBalancerPut, clientType request.ClientType) error LoadBalancerState(loadbalancer api.NetworkLoadBalancer) (*api.NetworkLoadBalancerState, error) LoadBalancerDelete(listenAddress string, clientType request.ClientType) error // Peerings. PeerCreate(forward api.NetworkPeersPost) error PeerUpdate(peerName string, newPeer api.NetworkPeerPut) error PeerDelete(peerName string) error PeerUsedBy(peerName string) ([]string, error) } incus-7.0.0/internal/server/network/network_load.go000066400000000000000000000052011517523235500224770ustar00rootroot00000000000000package network import ( "context" "fmt" "sync" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" ) var drivers = map[string]func() Network{ "bridge": func() Network { return &bridge{} }, "macvlan": func() Network { return &macvlan{} }, "sriov": func() Network { return &sriov{} }, "ovn": func() Network { return &ovn{} }, "physical": func() Network { return &physical{} }, } // ProjectNetwork is a composite type of project name and network name. type ProjectNetwork struct { ProjectName string NetworkName string } var ( unavailableNetworks = make(map[ProjectNetwork]struct{}) unavailableNetworksMu = sync.Mutex{} ) // LoadByType loads a network by driver type. func LoadByType(driverType string) (Type, error) { driverFunc, ok := drivers[driverType] if !ok { return nil, ErrUnknownDriver } n := driverFunc() err := n.init(nil, -1, "", &api.Network{Type: driverType}, nil) if err != nil { return nil, err } return n, nil } // LoadByName loads an instantiated network from the database by project and name. func LoadByName(s *state.State, projectName string, name string) (Network, error) { var id int64 var netInfo *api.Network var netNodes map[int64]db.NetworkNode err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error id, netInfo, netNodes, err = tx.GetNetworkInAnyState(ctx, projectName, name) return err }) if err != nil { return nil, err } driverFunc, ok := drivers[netInfo.Type] if !ok { return nil, ErrUnknownDriver } n := driverFunc() err = n.init(s, id, projectName, netInfo, netNodes) if err != nil { return nil, err } return n, nil } // PatchPreCheck checks if there are any unavailable networks. func PatchPreCheck() error { unavailableNetworksMu.Lock() if len(unavailableNetworks) > 0 { unavailableNetworkNames := make([]string, 0, len(unavailableNetworks)) for unavailablePoolName := range unavailableNetworks { unavailableNetworkNames = append(unavailableNetworkNames, fmt.Sprintf("%s/%s", unavailablePoolName.ProjectName, unavailablePoolName.NetworkName)) } unavailableNetworksMu.Unlock() return fmt.Errorf("Unavailable networks: %v", unavailableNetworkNames) } unavailableNetworksMu.Unlock() return nil } // IsAvailable checks if a network is available. func IsAvailable(projectName string, networkName string) bool { unavailableNetworksMu.Lock() defer unavailableNetworksMu.Unlock() pn := ProjectNetwork{ ProjectName: projectName, NetworkName: networkName, } _, found := unavailableNetworks[pn] return !found } incus-7.0.0/internal/server/network/network_utils.go000066400000000000000000001221041517523235500227220ustar00rootroot00000000000000package network import ( "bufio" "bytes" "context" cryptoRand "crypto/rand" "encoding/hex" "errors" "fmt" "io/fs" "math/big" "math/rand" "net" "net/netip" "os" "slices" "strconv" "strings" "sync" "time" "github.com/lxc/incus/v7/internal/iprange" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/device/nictype" "github.com/lxc/incus/v7/internal/server/dnsmasq" "github.com/lxc/incus/v7/internal/server/dnsmasq/dhcpalloc" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/ip" networkOVN "github.com/lxc/incus/v7/internal/server/network/ovn" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) func networkValidPort(value string) error { if value == "" { return nil } valueInt, err := strconv.ParseInt(value, 10, 64) if err != nil { return fmt.Errorf("Invalid value for an integer: %s", value) } if valueInt < 1 || valueInt > 65536 { return fmt.Errorf("Invalid port number: %s", value) } return nil } // RandomDevName returns a random device name with prefix. // If the random string combined with the prefix exceeds 13 characters then empty string is returned. // This is to ensure we support buggy dhclient applications: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=858580 func RandomDevName(prefix string) string { // Return a new random veth device name. randBytes := make([]byte, 4) _, _ = cryptoRand.Read(randBytes) iface := prefix + hex.EncodeToString(randBytes) if len(iface) > 13 { return "" } return iface } // MACDevName returns interface name with prefix 'inc' and MAC without leading 2 digits. func MACDevName(mac net.HardwareAddr) string { devName := strings.Join(strings.Split(mac.String(), ":"), "") return fmt.Sprintf("inc%s", devName[2:]) } // UsedByInstanceDevices looks for instance NIC devices using the network and runs the supplied usageFunc for each. // Accepts optional filter arguments to specify a subset of instances. func UsedByInstanceDevices(s *state.State, networkProjectName string, networkName string, networkType string, usageFunc func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error, filters ...cluster.InstanceFilter) error { // Get the instances. projects := map[string]api.Project{} instances := []db.InstanceArgs{} err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.InstanceList(ctx, func(inst db.InstanceArgs, p api.Project) error { projects[inst.Project] = p instances = append(instances, inst) return nil }, filters...) }) if err != nil { return err } // Go through the instances and run usageFunc. for _, inst := range instances { p := projects[inst.Project] // Get the instance's effective network project name. instNetworkProject := project.NetworkProjectFromRecord(&p) // Skip instances who's effective network project doesn't match this Network's project. if instNetworkProject != networkProjectName { continue } // Look for NIC devices using this network. devices := db.ExpandInstanceDevices(inst.Devices.Clone(), inst.Profiles) for devName, devConfig := range devices { if isInUseByDevice(networkName, networkType, devConfig) { err := usageFunc(inst, devName, devConfig) if err != nil { return err } } } } return nil } // UsedBy returns list of API resources using network. Accepts firstOnly argument to indicate that only the first // resource using network should be returned. This can help to quickly check if the network is in use. func UsedBy(s *state.State, networkProjectName string, networkID int64, networkName string, networkType string, firstOnly bool) ([]string, error) { var err error var usedBy []string // If managed network being passed in, check if it has any peerings in a created state. if networkID > 0 { var peers map[int64]*api.NetworkPeer err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Use generated function to get peers. filter := cluster.NetworkPeerFilter{NetworkID: &networkID} dbPeers, err := cluster.GetNetworkPeers(ctx, tx.Tx(), filter) if err != nil { return fmt.Errorf("Failed loading network peer DB objects: %w", err) } // Convert DB objects to API objects and build the map. peers = make(map[int64]*api.NetworkPeer, len(dbPeers)) for _, dbPeer := range dbPeers { peer, err := dbPeer.ToAPI(ctx, tx.Tx()) if err != nil { // Log the error but continue, as one peer failing shouldn't stop the whole check. logger.Error("Failed converting network peer DB object to API object", logger.Ctx{"peerID": dbPeer.ID, "err": err}) continue } peers[dbPeer.ID] = peer } return nil }) if err != nil { return nil, fmt.Errorf("Failed getting network peers: %w", err) } for _, peer := range peers { if peer.Status == api.NetworkStatusCreated { // Add the target project/network of the peering as using this network. usedBy = append(usedBy, api.NewURL().Path(version.APIVersion, "networks", peer.TargetNetwork).Project(peer.TargetProject).String()) if firstOnly { return usedBy, nil } } } } // Only networks defined in the default project can be used by other networks. Cheapest to do. if networkProjectName == api.ProjectDefaultName { // Get all managed networks across all projects. var projectNetworks map[string]map[int64]api.Network err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { projectNetworks, err = tx.GetCreatedNetworks(ctx) return err }) if err != nil { return nil, fmt.Errorf("Failed to load all networks: %w", err) } for projectName, networks := range projectNetworks { for _, network := range networks { if networkName == network.Name && networkProjectName == projectName { continue // Skip ourselves. } // The network's config references the network we are searching for. Either by // directly referencing our network or by referencing our interface as its parent. if network.Config["network"] == networkName || network.Config["parent"] == networkName { usedBy = append(usedBy, api.NewURL().Path(version.APIVersion, "networks", network.Name).Project(projectName).String()) if firstOnly { return usedBy, nil } } } } } // Look for profiles. Next cheapest to do. err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get all profiles profiles, err := cluster.GetProfiles(ctx, tx.Tx()) if err != nil { return err } // Get all the profile devices. profileDevices, err := cluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return err } for _, profile := range profiles { profileProject, err := cluster.GetProject(ctx, tx.Tx(), profile.Project) if err != nil { return err } apiProfileProject, err := profileProject.ToAPI(ctx, tx.Tx()) if err != nil { return err } devices := map[string]cluster.Device{} for _, dev := range profileDevices[profile.ID] { devices[dev.Name] = dev } inUse, err := usedByProfileDevices(s, devices, apiProfileProject, networkProjectName, networkName, networkType) if err != nil { return err } if inUse { usedBy = append(usedBy, api.NewURL().Path(version.APIVersion, "profiles", profile.Name).Project(profile.Project).String()) if firstOnly { return nil } } } return nil }) if err != nil { return nil, err } // Check if any instance devices use this network. err = UsedByInstanceDevices(s, networkProjectName, networkName, networkType, func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error { usedBy = append(usedBy, api.NewURL().Path(version.APIVersion, "instances", inst.Name).Project(inst.Project).String()) if firstOnly { // No need to consider other devices. return db.ErrInstanceListStop } return nil }) if err != nil { if errors.Is(err, db.ErrInstanceListStop) { return usedBy, nil } return nil, err } return usedBy, nil } // usedByProfileDevices indicates if network is referenced by a profile's NIC devices. // Checks if the device's parent or network properties match the network name. func usedByProfileDevices(s *state.State, profileDevices map[string]cluster.Device, profileProject *api.Project, networkProjectName string, networkName string, networkType string) (bool, error) { // Get the translated network project name from the profiles's project. // Skip profiles who's translated network project doesn't match the requested network's project. // Because its devices can't be using this network. profileNetworkProjectName := project.NetworkProjectFromRecord(profileProject) if networkProjectName != profileNetworkProjectName { return false, nil } for _, d := range deviceConfig.NewDevices(cluster.DevicesToAPI(profileDevices)) { if isInUseByDevice(networkName, networkType, d) { return true, nil } } return false, nil } // isInUseByDevices inspects a device's config to find references for a network being used. func isInUseByDevice(networkName string, networkType string, d deviceConfig.Device) bool { if d["type"] != "nic" { return false } if d["network"] != "" && d["network"] == networkName { return true } // OVN networks can only use managed networks. if networkType == "ovn" { return false } if d["parent"] != "" && GetHostDevice(d["parent"], d["vlan"]) == networkName { return true } return false } // GetDevMTU retrieves the current MTU setting for a named network device. func GetDevMTU(devName string) (uint32, error) { content, err := os.ReadFile(fmt.Sprintf("/sys/class/net/%s/mtu", devName)) if err != nil { return 0, err } // Parse value mtu, err := strconv.ParseUint(strings.TrimSpace(string(content)), 10, 32) if err != nil { return 0, err } return uint32(mtu), nil } // GetTXQueueLength retrieves the current txqlen setting for a named network device. func GetTXQueueLength(devName string) (uint32, error) { content, err := os.ReadFile(fmt.Sprintf("/sys/class/net/%s/tx_queue_len", devName)) if err != nil { return 0, err } // Parse value txqlen, err := strconv.ParseUint(strings.TrimSpace(string(content)), 10, 32) if err != nil { return 0, err } return uint32(txqlen), nil } // DefaultGatewaySubnetV4 returns subnet of default gateway interface. func DefaultGatewaySubnetV4() (*net.IPNet, string, error) { file, err := os.Open("/proc/net/route") if err != nil { return nil, "", err } defer func() { _ = file.Close() }() ifaceName := "" scanner := bufio.NewReader(file) for { line, _, err := scanner.ReadLine() if err != nil { break } fields := strings.Fields(string(line)) if fields[1] == "00000000" && fields[7] == "00000000" { ifaceName = fields[0] break } } if ifaceName == "" { return nil, "", errors.New("No default gateway for IPv4") } iface, err := net.InterfaceByName(ifaceName) if err != nil { return nil, "", err } addrs, err := iface.Addrs() if err != nil { return nil, "", err } var subnet *net.IPNet for _, addr := range addrs { addrIP, addrNet, err := net.ParseCIDR(addr.String()) if err != nil { return nil, "", err } if addrIP.To4() == nil { continue } if subnet != nil { return nil, "", errors.New("More than one IPv4 subnet on default interface") } subnet = addrNet } if subnet == nil { return nil, "", errors.New("No IPv4 subnet on default interface") } return subnet, ifaceName, nil } // UpdateDNSMasqStatic rebuilds the DNSMasq static allocations. func UpdateDNSMasqStatic(s *state.State, networkName string) error { // We don't want to race with ourselves here. dnsmasq.ConfigMutex.Lock() defer dnsmasq.ConfigMutex.Unlock() // Get all the networks. var networks []string if networkName == "" { var err error err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Pass api.ProjectDefaultName here, as currently dnsmasq (bridged) networks do not support projects. networks, err = tx.GetNetworks(ctx, api.ProjectDefaultName) return err }) if err != nil { return err } } else { networks = []string{networkName} } // Get all the instances. insts, err := instance.LoadNodeAll(s, instancetype.Any) if err != nil { return err } // Build a list of dhcp host entries. entries := map[string][][]string{} for _, inst := range insts { // Go through all its devices (including profiles). for deviceName, d := range inst.ExpandedDevices() { // Skip uninteresting entries. if d["type"] != "nic" { continue } nicType, err := nictype.NICType(s, inst.Project().Name, d) if err != nil || nicType != "bridged" { continue } // Temporarily populate parent from network setting if used. if d["network"] != "" { d["parent"] = d["network"] } // Skip devices not connected to managed networks. if !slices.Contains(networks, d["parent"]) { continue } // Fill in the hwaddr from volatile. d, err = inst.FillNetworkDevice(deviceName, d) if err != nil { continue } // Add the new host entries. _, ok := entries[d["parent"]] if !ok { entries[d["parent"]] = [][]string{} } if (util.IsTrue(d["security.ipv4_filtering"]) && d["ipv4.address"] == "") || (util.IsTrue(d["security.ipv6_filtering"]) && d["ipv6.address"] == "") { deviceStaticFileName := dnsmasq.StaticAllocationFileName(inst.Project().Name, inst.Name(), deviceName) _, curIPv4, curIPv6, err := dnsmasq.DHCPStaticAllocation(d["parent"], deviceStaticFileName) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } if d["ipv4.address"] == "" && curIPv4.IP != nil { d["ipv4.address"] = curIPv4.IP.String() } if d["ipv6.address"] == "" && curIPv6.IP != nil { d["ipv6.address"] = curIPv6.IP.String() } } entries[d["parent"]] = append(entries[d["parent"]], []string{d["hwaddr"], inst.Project().Name, inst.Name(), d["ipv4.address"], d["ipv6.address"], deviceName}) } } // Update the host files. for _, network := range networks { entries := entries[network] // Skip networks we don't manage (or don't have DHCP enabled). if !util.PathExists(internalUtil.VarPath("networks", network, "dnsmasq.pid")) { continue } // Pass api.ProjectDefaultName here, as currently dnsmasq (bridged) networks do not support projects. n, err := LoadByName(s, api.ProjectDefaultName, network) if err != nil { return fmt.Errorf("Failed to load network %q in project %q for dnsmasq update: %w", api.ProjectDefaultName, network, err) } config := n.Config() // Wipe everything clean. files, err := os.ReadDir(internalUtil.VarPath("networks", network, "dnsmasq.hosts")) if err != nil { return err } for _, entry := range files { err = os.Remove(internalUtil.VarPath("networks", network, "dnsmasq.hosts", entry.Name())) if err != nil { return err } } // Apply the changes. for entryIdx, entry := range entries { hwaddr := entry[0] projectName := entry[1] cName := entry[2] ipv4Address := entry[3] ipv6Address := entry[4] deviceName := entry[5] line := hwaddr // Look for duplicates. duplicate := false for iIdx, i := range entries { if project.Instance(entry[1], entry[2]) == project.Instance(i[1], i[2]) { // Skip ourselves. continue } if entry[0] == i[0] { // Find broken configurations logger.Errorf("Duplicate MAC detected: %s and %s", project.Instance(entry[1], entry[2]), project.Instance(i[1], i[2])) } if i[3] == "" && i[4] == "" { // Skip unconfigured. continue } if entry[3] == i[3] && entry[4] == i[4] { // Find identical containers (copies with static configuration). if entryIdx > iIdx { duplicate = true } else { line = fmt.Sprintf("%s,%s", line, i[0]) logger.Debugf("Found containers with duplicate IPv4/IPv6: %s and %s", project.Instance(entry[1], entry[2]), project.Instance(i[1], i[2])) } } } if duplicate { continue } // Generate the dhcp-host line. err := dnsmasq.UpdateStaticEntry(network, projectName, cName, deviceName, config, hwaddr, ipv4Address, ipv6Address) if err != nil { return err } } // Signal dnsmasq. err = dnsmasq.Kill(network, true) if err != nil { return err } } return nil } func randomSubnetV4() (string, error) { for range 100 { cidr := fmt.Sprintf("10.%d.%d.1/24", rand.Intn(255), rand.Intn(255)) _, subnet, err := net.ParseCIDR(cidr) if err != nil { continue } if inRoutingTable(subnet) { continue } if pingSubnet(subnet) { continue } return cidr, nil } return "", errors.New("Failed to automatically find an unused IPv4 subnet, manual configuration required") } func randomSubnetV6() (string, error) { for range 100 { cidr := fmt.Sprintf("fd42:%x:%x:%x::1/64", rand.Intn(65535), rand.Intn(65535), rand.Intn(65535)) _, subnet, err := net.ParseCIDR(cidr) if err != nil { continue } if inRoutingTable(subnet) { continue } if pingSubnet(subnet) { continue } return cidr, nil } return "", errors.New("Failed to automatically find an unused IPv6 subnet, manual configuration required") } func inRoutingTable(subnet *net.IPNet) bool { filename := "route" if subnet.IP.To4() == nil { filename = "ipv6_route" } file, err := os.Open(fmt.Sprintf("/proc/net/%s", filename)) if err != nil { return false } defer func() { _ = file.Close() }() scanner := bufio.NewReader(file) for { line, _, err := scanner.ReadLine() if err != nil { break } fields := strings.Fields(string(line)) // Get the IP var ip net.IP if filename == "ipv6_route" { ip, err = hex.DecodeString(fields[0]) if err != nil { continue } } else { bytes, err := hex.DecodeString(fields[1]) if err != nil { continue } ip = net.IPv4(bytes[3], bytes[2], bytes[1], bytes[0]) } // Get the mask var mask net.IPMask if filename == "ipv6_route" { size, err := strconv.ParseInt(fields[1], 16, 0) if err != nil { continue } mask = net.CIDRMask(int(size), 128) } else { bytes, err := hex.DecodeString(fields[7]) if err != nil { continue } mask = net.IPv4Mask(bytes[3], bytes[2], bytes[1], bytes[0]) } // Generate a new network lineNet := net.IPNet{IP: ip, Mask: mask} // Ignore default gateway if lineNet.IP.Equal(net.ParseIP("::")) { continue } if lineNet.IP.Equal(net.ParseIP("0.0.0.0")) { continue } // Check if we have a route to our new subnet if lineNet.Contains(subnet.IP) { return true } } return false } // pingIP sends a single ping packet to the specified IP, returns nil error if IP is reachable. // If ctx doesn't have a deadline then the default timeout used is 1s. func pingIP(ctx context.Context, ip net.IP) error { cmd := "ping" if ip.To4() == nil { cmd = "ping6" } timeout := time.Second * 1 deadline, ok := ctx.Deadline() if ok { timeout = time.Until(deadline) } _, err := subprocess.RunCommandContext(ctx, cmd, "-n", "-q", ip.String(), "-c", "1", "-w", fmt.Sprintf("%d", int(timeout.Seconds()))) return err } func pingSubnet(subnet *net.IPNet) bool { var fail bool var failLock sync.Mutex var wgChecks sync.WaitGroup ping := func(ip net.IP) { defer wgChecks.Done() if pingIP(context.TODO(), ip) != nil { return } // Remote answered failLock.Lock() fail = true failLock.Unlock() } poke := func(ip net.IP) { defer wgChecks.Done() addr := fmt.Sprintf("%s:22", ip.String()) if ip.To4() == nil { addr = fmt.Sprintf("[%s]:22", ip.String()) } _, err := net.DialTimeout("tcp", addr, time.Second) if err == nil { // Remote answered failLock.Lock() fail = true failLock.Unlock() return } } // Ping first IP wgChecks.Add(1) go ping(dhcpalloc.GetIP(subnet, 1)) // Poke port on first IP wgChecks.Add(1) go poke(dhcpalloc.GetIP(subnet, 1)) // Ping check if subnet.IP.To4() != nil { // Ping last IP wgChecks.Add(1) go ping(dhcpalloc.GetIP(subnet, -2)) // Poke port on last IP wgChecks.Add(1) go poke(dhcpalloc.GetIP(subnet, -2)) } wgChecks.Wait() return fail } // GetHostDevice returns the interface name to use for a combination of parent device name and VLAN ID. // If no vlan ID supplied, parent name is returned unmodified. If non-empty VLAN ID is supplied then it will look // for an existing VLAN device and return that, otherwise it will return the default "parent.vlan" format as name. func GetHostDevice(parent string, vlan string) string { // If no VLAN, just use the raw device if vlan == "" { return parent } // If no VLANs are configured, use the default pattern. defaultVlan := fmt.Sprintf("%s.%s", parent, vlan) // Handle long interface names. if len(defaultVlan) > 15 { defaultVlan = fmt.Sprintf("incus-vlan-%s", vlan) } if !util.PathExists("/proc/net/vlan/config") { return defaultVlan } // Look for an existing VLAN f, err := os.Open("/proc/net/vlan/config") if err != nil { return defaultVlan } defer func() { _ = f.Close() }() scanner := bufio.NewScanner(f) for scanner.Scan() { // Only grab the lines we're interested in s := strings.Split(scanner.Text(), "|") if len(s) != 3 { continue } vlanIface := strings.TrimSpace(s[0]) vlanID := strings.TrimSpace(s[1]) vlanParent := strings.TrimSpace(s[2]) if vlanParent == parent && vlanID == vlan { return vlanIface } } // Return the default pattern return defaultVlan } // GetNeighbourIPs returns the IP addresses in the neighbour cache for a particular interface and MAC. func GetNeighbourIPs(interfaceName string, hwaddr net.HardwareAddr) ([]ip.Neigh, error) { if hwaddr == nil { return nil, nil } neigh := &ip.Neigh{DevName: interfaceName, MAC: hwaddr} neighbours, err := neigh.Show() if err != nil { return nil, fmt.Errorf("Failed to get IP neighbours for interface %q: %w", interfaceName, err) } return neighbours, nil } // GetLeaseAddresses returns the lease addresses for a network and hwaddr. func GetLeaseAddresses(networkName string, hwaddr string) ([]net.IP, error) { leaseFile := internalUtil.VarPath("networks", networkName, "dnsmasq.leases") if !util.PathExists(leaseFile) { return nil, fmt.Errorf("Leases file not found for network %q", networkName) } content, err := os.ReadFile(leaseFile) if err != nil { return nil, err } addresses := []net.IP{} for _, lease := range strings.Split(string(content), "\n") { fields := strings.Fields(lease) if len(fields) < 5 { continue } // Parse the MAC. mac := GetMACSlice(fields[1]) macStr := strings.Join(mac, ":") if len(macStr) < 17 && fields[4] != "" { macStr = fields[4][len(fields[4])-17:] } if macStr != hwaddr { continue } // Parse the IP. ip := net.ParseIP(fields[2]) if ip != nil { addresses = append(addresses, ip) } } return addresses, nil } // GetMACSlice parses MAC address. func GetMACSlice(hwaddr string) []string { var buf []string if !strings.Contains(hwaddr, ":") { s, err := strconv.ParseUint(hwaddr, 10, 64) if err == nil { hwaddr = fmt.Sprintf("%x\n", s) var tuple string for i, r := range hwaddr { tuple = tuple + string(r) if i > 0 && (i+1)%2 == 0 { buf = append(buf, tuple) tuple = "" } } } } else { buf = strings.Split(strings.ToLower(hwaddr), ":") } return buf } // usesIPv4Firewall returns whether network config will need to use the IPv4 firewall. func usesIPv4Firewall(netConfig map[string]string) bool { if netConfig == nil { return false } if util.IsTrueOrEmpty(netConfig["ipv4.firewall"]) { return true } if util.IsTrue(netConfig["ipv4.nat"]) { return true } return false } // usesIPv6Firewall returns whether network config will need to use the IPv6 firewall. func usesIPv6Firewall(netConfig map[string]string) bool { if netConfig == nil { return false } if util.IsTrueOrEmpty(netConfig["ipv6.firewall"]) { return true } if util.IsTrue(netConfig["ipv6.nat"]) { return true } return false } // parseIPRange parses an IP range in the format "start-end" and converts it to a iprange.Range. // If allowedNets are supplied, then each IP in the range is checked that it belongs to at least one of them. // IPs in the range can be zero prefixed, e.g. "::1" or "0.0.0.1", however they should not overlap with any // supplied allowedNets prefixes. If they are within an allowed network, any zero prefixed addresses are // returned combined with the first allowed network they are within. // If no allowedNets supplied they are returned as-is. func parseIPRange(ipRange string, allowedNets ...*net.IPNet) (*iprange.Range, error) { inAllowedNet := func(ip net.IP, allowedNet *net.IPNet) net.IP { if ip == nil { return nil } ipv4 := ip.To4() // Only match IPv6 addresses against IPv6 networks. if ipv4 == nil && allowedNet.IP.To4() != nil { return nil } // Combine IP with network prefix if IP starts with a zero. // If IP is v4, then compare against 4-byte representation, otherwise use 16 byte representation. if (ipv4 != nil && ipv4[0] == 0) || (ipv4 == nil && ip[0] == 0) { allowedNet16 := allowedNet.IP.To16() ipCombined := make(net.IP, net.IPv6len) for i, b := range ip { ipCombined[i] = allowedNet16[i] | b } ip = ipCombined } // Check start IP is within one of the allowed networks. if !allowedNet.Contains(ip) { return nil } return ip } rangeParts := strings.SplitN(ipRange, "-", 2) if len(rangeParts) != 2 { return nil, fmt.Errorf("IP range %q must contain start and end IP addresses", ipRange) } startIP := net.ParseIP(rangeParts[0]) endIP := net.ParseIP(rangeParts[1]) if startIP == nil { return nil, fmt.Errorf("Start IP %q is invalid", rangeParts[0]) } if endIP == nil { return nil, fmt.Errorf("End IP %q is invalid", rangeParts[1]) } if bytes.Compare(startIP, endIP) > 0 { return nil, fmt.Errorf("Start IP %q must be less than End IP %q", startIP, endIP) } if len(allowedNets) > 0 { matchFound := false for _, allowedNet := range allowedNets { if allowedNet == nil { return nil, errors.New("Invalid allowed network") } combinedStartIP := inAllowedNet(startIP, allowedNet) if combinedStartIP == nil { continue } combinedEndIP := inAllowedNet(endIP, allowedNet) if combinedEndIP == nil { continue } // If both match then replace parsed IPs with combined IPs and stop searching. matchFound = true startIP = combinedStartIP endIP = combinedEndIP break } if !matchFound { return nil, fmt.Errorf("IP range %q does not fall within any of the allowed networks %v", ipRange, allowedNets) } } return &iprange.Range{ Start: startIP, End: endIP, }, nil } // parseIPRanges parses a comma separated list of IP ranges using parseIPRange. func parseIPRanges(ipRangesList string, allowedNets ...*net.IPNet) ([]*iprange.Range, error) { ipRanges := strings.Split(ipRangesList, ",") netIPRanges := make([]*iprange.Range, 0, len(ipRanges)) for _, ipRange := range ipRanges { netIPRange, err := parseIPRange(strings.TrimSpace(ipRange), allowedNets...) if err != nil { return nil, err } netIPRanges = append(netIPRanges, netIPRange) } return netIPRanges, nil } // VLANInterfaceCreate creates a VLAN interface on parent interface (if needed). // Returns boolean indicating if VLAN interface was created. func VLANInterfaceCreate(parent string, vlanDevice string, vlanID string, gvrp bool) (bool, error) { if vlanID == "" { return false, nil } if InterfaceExists(vlanDevice) { return false, nil } // Bring the parent interface up so we can add a vlan to it. link := &ip.Link{Name: parent} err := link.SetUp() if err != nil { return false, fmt.Errorf("Failed to bring up parent %q: %w", parent, err) } vlan := &ip.Vlan{ Link: ip.Link{ Name: vlanDevice, Parent: parent, }, VlanID: vlanID, Gvrp: gvrp, } err = vlan.Add() if err != nil { return false, fmt.Errorf("Failed to create VLAN interface %q on %q: %w", vlanDevice, parent, err) } err = vlan.SetUp() if err != nil { return false, fmt.Errorf("Failed to bring up interface %q: %w", vlanDevice, err) } // Attempt to disable IPv6 router advertisement acceptance. _ = localUtil.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", vlanDevice), "0") // We created a new vlan interface, return true. return true, nil } // InterfaceRemove removes a network interface by name. func InterfaceRemove(nic string) error { link := &ip.Link{Name: nic} err := link.Delete() return err } // InterfaceExists returns true if network interface exists. func InterfaceExists(nic string) bool { if nic != "" && util.PathExists(fmt.Sprintf("/sys/class/net/%s", nic)) { return true } return false } // IPInSlice returns true if slice has IP element. func IPInSlice(key net.IP, list []net.IP) bool { for _, entry := range list { if entry.Equal(key) { return true } } return false } // IPisBroadcast returns true if the IP address is a broadcast address. func IPisBroadcast(subnet *net.IPNet, ipAddress net.IP) bool { if ipAddress == nil || subnet == nil { return false } ipv4 := ipAddress.To4() if ipv4 == nil { return false } broadcast := make(net.IP, 4) for i := 0; i < 4; i++ { broadcast[i] = subnet.IP[i] | ^subnet.Mask[i] } return ipv4.Equal(broadcast) } // IPisNetworkID returns true if the IP address is a network ID. func IPisNetworkID(subnet *net.IPNet, ipAddress net.IP) bool { if ipAddress == nil || subnet == nil { return false } return ipAddress.Equal(subnet.IP) } // SubnetContains returns true if outerSubnet contains innerSubnet. func SubnetContains(outerSubnet *net.IPNet, innerSubnet *net.IPNet) bool { if outerSubnet == nil || innerSubnet == nil { return false } if !outerSubnet.Contains(innerSubnet.IP) { return false } outerOnes, outerBits := outerSubnet.Mask.Size() innerOnes, innerBits := innerSubnet.Mask.Size() // Check number of bits in mask match. if innerBits != outerBits { return false } // Check that the inner subnet isn't outside of the outer subnet. if innerOnes < outerOnes { return false } return true } // SubnetContainsIP returns true if outsetSubnet contains IP address. func SubnetContainsIP(outerSubnet *net.IPNet, ip net.IP) bool { // Convert ip to ipNet. ipIsIP4 := ip.To4() != nil prefix := 32 if !ipIsIP4 { prefix = 128 } _, ipSubnet, err := net.ParseCIDR(fmt.Sprintf("%s/%d", ip.String(), prefix)) if err != nil { return false } ipSubnet.IP = ip return SubnetContains(outerSubnet, ipSubnet) } // SubnetIterate iterates through each IP in a subnet calling a function for each IP. // If the ipFunc returns a non-nil error then the iteration stops and the error is returned. func SubnetIterate(subnet *net.IPNet, ipFunc func(ip net.IP) error) error { inc := big.NewInt(1) // Convert route start IP to native representations to allow incrementing. startIP := subnet.IP.To4() if startIP == nil { startIP = subnet.IP.To16() } startBig := big.NewInt(0) startBig.SetBytes(startIP) // Iterate through IPs in subnet, calling ipFunc for each one. for { ip := net.IP(startBig.Bytes()) if !subnet.Contains(ip) { break } err := ipFunc(ip) if err != nil { return err } startBig.Add(startBig, inc) } return nil } // SubnetParseAppend parses one or more string CIDR subnets. Appends to the supplied slice. Returns subnets slice. func SubnetParseAppend(subnets []*net.IPNet, parseSubnet ...string) ([]*net.IPNet, error) { for _, subnetStr := range parseSubnet { _, subnet, err := net.ParseCIDR(subnetStr) if err != nil { return nil, fmt.Errorf("Invalid subnet %q: %w", subnetStr, err) } subnets = append(subnets, subnet) } return subnets, nil } // IPRangesOverlap checks whether two ip ranges have ip addresses in common. func IPRangesOverlap(r1, r2 *iprange.Range) bool { if r1.End == nil { return r2.ContainsIP(r1.Start) } if r2.End == nil { return r1.ContainsIP(r2.Start) } return r1.ContainsIP(r2.Start) || r1.ContainsIP(r2.End) } // InterfaceStatus returns the global unicast IP addresses configured on an interface and whether it is up or not. func InterfaceStatus(nicName string) ([]net.IP, bool, error) { iface, err := net.InterfaceByName(nicName) if err != nil { return nil, false, fmt.Errorf("Failed loading interface %q: %w", nicName, err) } isUp := iface.Flags&net.FlagUp != 0 addresses, err := iface.Addrs() if err != nil { return nil, isUp, fmt.Errorf("Failed getting interface addresses for %q: %w", nicName, err) } var globalUnicastIPs []net.IP for _, address := range addresses { ip, _, _ := net.ParseCIDR(address.String()) if ip == nil { continue } if ip.IsGlobalUnicast() { globalUnicastIPs = append(globalUnicastIPs, ip) } } return globalUnicastIPs, isUp, nil } // ParsePortRange validates a port range in the form start-end. func ParsePortRange(r string) (int64, int64, error) { entries := strings.Split(r, "-") if len(entries) > 2 { return -1, -1, fmt.Errorf("Invalid port range %q", r) } base, err := strconv.ParseInt(entries[0], 10, 64) if err != nil { return -1, -1, err } size := int64(1) if len(entries) > 1 { size, err = strconv.ParseInt(entries[1], 10, 64) if err != nil { return -1, -1, err } if size <= base { return -1, -1, errors.New("End port should be higher than start port") } size -= base size++ } return base, size, nil } // ParseIPToNet parses a standalone IP address into a net.IPNet (with the IP field set to the IP supplied). // The address family is detected and the subnet size set to /32 for IPv4 or /128 for IPv6. func ParseIPToNet(ipAddress string) (*net.IPNet, error) { subnetSize := 32 if strings.Contains(ipAddress, ":") { subnetSize = 128 } listenAddress, listenAddressNet, err := net.ParseCIDR(fmt.Sprintf("%s/%d", ipAddress, subnetSize)) if err != nil { return nil, err } listenAddressNet.IP = listenAddress // Add IP back into parsed subnet. return listenAddressNet, err } // ParseIPCIDRToNet parses an IP in CIDR format into a net.IPNet (with the IP field set to the IP supplied). func ParseIPCIDRToNet(ipAddressCIDR string) (*net.IPNet, error) { listenAddress, listenAddressNet, err := net.ParseCIDR(ipAddressCIDR) if err != nil { return nil, err } listenAddressNet.IP = listenAddress // Add IP back into parsed subnet. return listenAddressNet, err } // IPToNet converts an IP to a single host IPNet. func IPToNet(ip net.IP) net.IPNet { bits := 32 if ip.To4() == nil { bits = 128 } return net.IPNet{ IP: ip, Mask: net.CIDRMask(bits, bits), } } // NICUsesNetwork returns true if the nicDev's "network" or "parent" property matches one of the networks names. func NICUsesNetwork(nicDev map[string]string, networks ...*api.Network) bool { for _, network := range networks { if network.Name == nicDev["network"] || network.Name == nicDev["parent"] { return true } } return false } // BridgeNetfilterEnabled checks whether the bridge netfilter feature is loaded and enabled. // If it is not an error is returned. This is needed in order for instances connected to a bridge to access DNAT // listeners on the host, as otherwise the packets from the bridge do have the SNAT netfilter rules applied. func BridgeNetfilterEnabled(ipVersion uint) error { sysctlName := "iptables" if ipVersion == 6 { sysctlName = "ip6tables" } sysctlPath := fmt.Sprintf("net/bridge/bridge-nf-call-%s", sysctlName) sysctlVal, err := localUtil.SysctlGet(sysctlPath) if err != nil { return errors.New("br_netfilter kernel module not loaded") } sysctlVal = strings.TrimSpace(sysctlVal) if sysctlVal != "1" { return fmt.Errorf("sysctl net.bridge.bridge-nf-call-%s not enabled", sysctlName) } return nil } // ProxyParseAddr validates a proxy address and parses it into its constituent parts. func ProxyParseAddr(data string) (*deviceConfig.ProxyAddress, error) { // Split into and
. fields := strings.SplitN(data, ":", 2) if !slices.Contains([]string{"tcp", "udp", "unix"}, fields[0]) { return nil, fmt.Errorf("Unknown protocol type %q", fields[0]) } if len(fields) < 2 || fields[1] == "" { return nil, errors.New("Missing address") } newProxyAddr := &deviceConfig.ProxyAddress{ ConnType: fields[0], Abstract: strings.HasPrefix(fields[1], "@"), } // unix addresses cannot have ports. if newProxyAddr.ConnType == "unix" { newProxyAddr.Address = fields[1] return newProxyAddr, nil } // Split
into
and . address, port, err := net.SplitHostPort(fields[1]) if err != nil { return nil, err } // Validate that it's a valid address. if slices.Contains([]string{"udp", "tcp"}, newProxyAddr.ConnType) { err := validate.Optional(validate.IsNetworkAddress)(address) if err != nil { return nil, err } } newProxyAddr.Address = address // Split into individual ports and port ranges. ports := strings.Split(port, ",") newProxyAddr.Ports = make([]uint64, 0, len(ports)) for _, p := range ports { portFirst, portRange, err := ParsePortRange(p) if err != nil { return nil, err } for i := range portRange { newProxyAddr.Ports = append(newProxyAddr.Ports, uint64(portFirst+i)) } } if len(newProxyAddr.Ports) <= 0 { return nil, errors.New("At least one port is required") } return newProxyAddr, nil } func validateExternalInterfaces(value string) error { for _, entry := range strings.Split(value, ",") { entry = strings.TrimSpace(entry) // Test for extended configuration of external interface. entryParts := strings.Split(entry, "/") if len(entryParts) == 3 { // The first part is the interface name. entry = strings.TrimSpace(entryParts[0]) } err := validate.IsInterfaceName(entry) if err != nil { return fmt.Errorf("Invalid interface name %q: %w", entry, err) } if len(entryParts) == 3 { // Check if the parent interface is valid. parent := strings.TrimSpace(entryParts[1]) err := validate.IsInterfaceName(parent) if err != nil { return fmt.Errorf("Invalid interface name %q: %w", parent, err) } // Check if the VLAN ID is valid. vlanID, err := strconv.Atoi(entryParts[2]) if err != nil { return fmt.Errorf("Invalid VLAN ID %q: %w", entryParts[2], err) } if vlanID < 1 || vlanID > 4094 { return fmt.Errorf("Invalid VLAN ID %q", entryParts[2]) } } } return nil } // complementRanges returns the complement of the provided IP network ranges. // It calculates the IP ranges that are *not* covered by the input slice. func complementRanges(ranges []*iprange.Range, netAddr *net.IPNet) ([]iprange.Range, error) { var complement []iprange.Range ipv4NetPrefix, err := netip.ParsePrefix(netAddr.String()) if err != nil { return nil, err } previousEnd := ipv4NetPrefix.Addr() for _, r := range ranges { startAddr, err := netip.ParseAddr(r.Start.String()) if err != nil { return nil, err } endAddr, err := netip.ParseAddr(r.End.String()) if err != nil { return nil, err } if startAddr.Compare(previousEnd.Next()) == 1 { newStart := previousEnd.Next() newEnd := startAddr.Prev() if newStart.Compare(newEnd) == 0 { complement = append(complement, iprange.Range{Start: net.ParseIP(newStart.String())}) } else { complement = append(complement, iprange.Range{Start: net.ParseIP(newStart.String()), End: net.ParseIP(newEnd.String())}) } } if endAddr.Compare(previousEnd) == 1 { previousEnd = endAddr } } endAddr, err := netip.ParseAddr(dhcpalloc.GetIP(netAddr, -2).String()) if err != nil { return nil, err } if previousEnd.Compare(endAddr) == -1 { complement = append(complement, iprange.Range{Start: net.ParseIP(previousEnd.Next().String()), End: net.ParseIP(endAddr.String())}) } return complement, nil } // ipInRanges checks whether the given IP address is contained within any of the // provided IP network ranges. func ipInRanges(ipAddr net.IP, ipRanges []iprange.Range) bool { for _, r := range ipRanges { containsIP := r.ContainsIP(ipAddr) if containsIP { return true } } return false } // ipInPointerRanges checks whether the given IP address is contained within any of the // provided pointer IP network ranges. func ipInPointerRanges(ipAddr net.IP, ipRanges []*iprange.Range) bool { for _, r := range ipRanges { if r == nil { // nil check since we're dealing with pointers here continue } return r.ContainsIP(ipAddr) } return false } // ovnRouteExists checks if the given route already exists in the provided list of current routes. func ovnRouteExists(currentRoutes []networkOVN.OVNRouterRoute, route networkOVN.OVNRouterRoute) bool { return slices.ContainsFunc(currentRoutes, func(existing networkOVN.OVNRouterRoute) bool { if existing.Prefix.String() != route.Prefix.String() { return false } if route.Discard != existing.Discard { return false } return existing.NextHop.Equal(route.NextHop) && existing.Port == route.Port }) } incus-7.0.0/internal/server/network/network_utils_bridge.go000066400000000000000000000057641517523235500242520ustar00rootroot00000000000000package network import ( "context" "fmt" "os" "strings" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/util" ) // BridgeVLANFilteringStatus returns whether VLAN filtering is enabled on a bridge interface. func BridgeVLANFilteringStatus(interfaceName string) (string, error) { content, err := os.ReadFile(fmt.Sprintf("/sys/class/net/%s/bridge/vlan_filtering", interfaceName)) if err != nil { return "", fmt.Errorf("Failed getting bridge VLAN status for %q: %w", interfaceName, err) } return strings.TrimSpace(string(content)), nil } // BridgeVLANFilterSetStatus sets the status of VLAN filtering on a bridge interface. func BridgeVLANFilterSetStatus(interfaceName string, status string) error { err := os.WriteFile(fmt.Sprintf("/sys/class/net/%s/bridge/vlan_filtering", interfaceName), []byte(status), 0) if err != nil { return fmt.Errorf("Failed enabling VLAN filtering on bridge %q: %w", interfaceName, err) } return nil } // BridgeVLANDefaultPVID returns the VLAN default port VLAN ID (PVID). func BridgeVLANDefaultPVID(interfaceName string) (string, error) { content, err := os.ReadFile(fmt.Sprintf("/sys/class/net/%s/bridge/default_pvid", interfaceName)) if err != nil { return "", fmt.Errorf("Failed getting bridge VLAN default PVID for %q: %w", interfaceName, err) } return strings.TrimSpace(string(content)), nil } // BridgeVLANSetDefaultPVID sets the VLAN default port VLAN ID (PVID). func BridgeVLANSetDefaultPVID(interfaceName string, vlanID string) error { err := os.WriteFile(fmt.Sprintf("/sys/class/net/%s/bridge/default_pvid", interfaceName), []byte(vlanID), 0) if err != nil { return fmt.Errorf("Failed setting bridge VLAN default PVID for %q: %w", interfaceName, err) } return nil } // IsNativeBridge returns whether the bridge name specified is a Linux native bridge. func IsNativeBridge(bridgeName string) bool { return util.PathExists(fmt.Sprintf("/sys/class/net/%s/bridge", bridgeName)) } // AttachInterface attaches an interface to a bridge. func AttachInterface(s *state.State, bridgeName string, devName string) error { if IsNativeBridge(bridgeName) { link := &ip.Link{Name: devName} err := link.SetMaster(bridgeName) if err != nil { return err } } else { vswitch, err := s.OVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } err = vswitch.CreateBridgePort(context.TODO(), bridgeName, devName, true) if err != nil { return err } } return nil } // DetachInterface detaches an interface from a bridge. func DetachInterface(s *state.State, bridgeName string, devName string) error { if IsNativeBridge(bridgeName) { link := &ip.Link{Name: devName} err := link.SetNoMaster() if err != nil { return err } } else { vswitch, err := s.OVS() if err != nil { return fmt.Errorf("Failed to connect to OVS: %w", err) } err = vswitch.DeleteBridgePort(context.TODO(), bridgeName, devName) if err != nil { return err } } return nil } incus-7.0.0/internal/server/network/network_utils_sriov.go000066400000000000000000000332401517523235500241460ustar00rootroot00000000000000package network import ( "bytes" "context" "encoding/json" "errors" "fmt" "io/fs" "os" "path/filepath" "strconv" "strings" "sync" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/device/pci" "github.com/lxc/incus/v7/internal/server/ip" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) // sriovReservedDevicesMutex used to coordinate access for checking reserved devices. var sriovReservedDevicesMutex sync.Mutex // SRIOVVirtualFunctionMutex used to coordinate access for finding and claiming free virtual functions. var SRIOVVirtualFunctionMutex sync.Mutex var sysClassNet = "/sys/class/net" // SRIOVGetHostDevicesInUse returns a map of host device names that have been used by devices in other instances // and networks on the local member. Used when selecting physical and SR-IOV VF devices to avoid conflicts. func SRIOVGetHostDevicesInUse(s *state.State) (map[string]struct{}, error) { sriovReservedDevicesMutex.Lock() defer sriovReservedDevicesMutex.Unlock() var err error var projectNetworks map[string]map[int64]api.Network err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Get all managed networks across all projects. projectNetworks, err = tx.GetCreatedNetworks(ctx) if err != nil { return fmt.Errorf("Failed to load all networks: %w", err) } return err }) if err != nil { return nil, err } filter := dbCluster.InstanceFilter{Node: &s.ServerName} reservedDevices := map[string]struct{}{} // Check if any instances are using the VF device. err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.InstanceList(ctx, func(dbInst db.InstanceArgs, p api.Project) error { // Expand configs so we take into account profile devices. dbInst.Config = db.ExpandInstanceConfig(dbInst.Config, dbInst.Profiles) dbInst.Devices = db.ExpandInstanceDevices(dbInst.Devices, dbInst.Profiles) for name, dev := range dbInst.Devices { // If device references a parent host interface name, mark that as reserved. parent := dev["parent"] if parent != "" { reservedDevices[parent] = struct{}{} } // If device references a volatile host interface name, mark that as reserved. hostName := dbInst.Config[fmt.Sprintf("volatile.%s.host_name", name)] if hostName != "" { reservedDevices[hostName] = struct{}{} } } return nil }, filter) }) if err != nil { return nil, err } // Check if any networks are using the VF device. for _, networks := range projectNetworks { for _, ni := range networks { // If network references a parent host interface name, mark that as reserved. parent := ni.Config["parent"] if parent != "" { reservedDevices[parent] = struct{}{} } } } return reservedDevices, nil } // SRIOVFindFreeVirtualFunction looks on the specified parent device for an unused virtual function. // Returns the name of the interface and virtual function index ID if found, error if not. func SRIOVFindFreeVirtualFunction(s *state.State, parentDev string) (string, int, error) { reservedDevices, err := SRIOVGetHostDevicesInUse(s) if err != nil { return "", -1, fmt.Errorf("Failed getting in use device list: %w", err) } sriovNumVFsFile := fmt.Sprintf("/sys/class/net/%s/device/sriov_numvfs", parentDev) // Verify that this is indeed a SR-IOV enabled device. if !util.PathExists(sriovNumVFsFile) { return "", -1, fmt.Errorf("Parent device %q doesn't support SR-IOV", parentDev) } // Get parent dev_port and dev_id values. pfDevPort, err := os.ReadFile(fmt.Sprintf("/sys/class/net/%s/dev_port", parentDev)) if err != nil { return "", -1, err } pfDevID, err := os.ReadFile(fmt.Sprintf("/sys/class/net/%s/dev_id", parentDev)) if err != nil { return "", -1, err } // Get number of currently enabled VFs. sriovNumVFsBuf, err := os.ReadFile(sriovNumVFsFile) if err != nil { return "", -1, err } sriovNumVFs, err := strconv.Atoi(strings.TrimSpace(string(sriovNumVFsBuf))) if err != nil { return "", -1, err } // Ensure parent is up (needed for Intel at least). link := &ip.Link{Name: parentDev} err = link.SetUp() if err != nil { return "", -1, err } // Check if any free VFs are already enabled. vfID, nicName, err := sriovGetFreeVFInterface(reservedDevices, parentDev, sriovNumVFs, 0, pfDevID, pfDevPort) if err != nil { return "", -1, err } if nicName == "" { return "", -1, fmt.Errorf("All virtual functions on parent device %q are already in use", parentDev) } // Found a free VF. return nicName, vfID, nil } // sriovGetFreeVFInterface checks the system for a free VF interface that belongs to the same device and port as // the parent device starting from the startVFID to the vfCount-1. Returns VF ID and VF interface name if found or // -1 and empty string if no free interface found. A free interface is one that is bound on the host, not in the // reservedDevices map, is down and has no global IPs defined on it. func sriovGetFreeVFInterface(reservedDevices map[string]struct{}, parentDev string, vfCount int, startVFID int, pfDevID []byte, pfDevPort []byte) (int, string, error) { for vfID := startVFID; vfID < vfCount; vfID++ { vfListPath := fmt.Sprintf("/sys/class/net/%s/device/virtfn%d/net", parentDev, vfID) if !util.PathExists(vfListPath) { continue // The vfListPath won't exist if the VF has been unbound and used with a VM. } ents, err := os.ReadDir(vfListPath) if err != nil { return -1, "", fmt.Errorf("Failed reading VF interface directory %q: %w", vfListPath, err) } for _, ent := range ents { // We expect the entry to be a directory for the VF's interface name. if !ent.IsDir() { continue } nicName := ent.Name() // We can't use this VF interface as it is reserved by another device. _, exists := reservedDevices[nicName] if exists { continue } // Get VF dev_port and dev_id values. vfDevPort, err := os.ReadFile(fmt.Sprintf("%s/%s/dev_port", vfListPath, nicName)) if err != nil { return -1, "", err } vfDevID, err := os.ReadFile(fmt.Sprintf("%s/%s/dev_id", vfListPath, nicName)) if err != nil { return -1, "", err } // Skip VFs if they do not relate to the same device and port as the parent PF. // Some card vendors change the device ID for each port. if !bytes.Equal(pfDevPort, vfDevPort) || !bytes.Equal(pfDevID, vfDevID) { continue } addresses, isUp, err := InterfaceStatus(nicName) if err != nil { return -1, "", err } // Ignore if interface is up or if interface has unicast IP addresses (may be in use by // another application already). if isUp || len(addresses) > 0 { continue } // Found a free VF. return vfID, nicName, err } } return -1, "", nil } // SRIOVCountFreeVirtualFunctions returns the number of available SR-IOV virtual functions. func SRIOVCountFreeVirtualFunctions(s *state.State, parentDev string) (int, int, error) { reservedDevices, err := SRIOVGetHostDevicesInUse(s) if err != nil { return -1, -1, fmt.Errorf("Failed getting in use device list: %w", err) } sriovNumVFsFile := fmt.Sprintf("/sys/class/net/%s/device/sriov_numvfs", parentDev) // Verify that this is indeed a SR-IOV enabled device. if !util.PathExists(sriovNumVFsFile) { return -1, -1, fmt.Errorf("Parent device %q doesn't support SR-IOV", parentDev) } // Get parent dev_port and dev_id values. pfDevPort, err := os.ReadFile(fmt.Sprintf("/sys/class/net/%s/dev_port", parentDev)) if err != nil { return -1, -1, err } pfDevID, err := os.ReadFile(fmt.Sprintf("/sys/class/net/%s/dev_id", parentDev)) if err != nil { return -1, -1, err } // Get number of currently enabled VFs. sriovNumVFsBuf, err := os.ReadFile(sriovNumVFsFile) if err != nil { return -1, -1, err } sriovNumVFs, err := strconv.Atoi(strings.TrimSpace(string(sriovNumVFsBuf))) if err != nil { return -1, -1, err } // Ensure parent is up (needed for Intel at least). link := &ip.Link{Name: parentDev} err = link.SetUp() if err != nil { return -1, -1, err } freeVfs := 0 nextVf := 0 for { // Check if any free VFs are already enabled. vfID, _, err := sriovGetFreeVFInterface(reservedDevices, parentDev, sriovNumVFs, nextVf, pfDevID, pfDevPort) if err != nil { return -1, -1, err } if vfID == -1 { break } freeVfs += 1 nextVf = vfID + 1 } return freeVfs, sriovNumVFs, nil } // SRIOVGetVFDevicePCISlot returns the PCI slot name for a network virtual function device. func SRIOVGetVFDevicePCISlot(parentDev string, vfID string) (pci.Device, error) { ueventFile := fmt.Sprintf("/sys/class/net/%s/device/virtfn%s/uevent", parentDev, vfID) pciDev, err := pci.ParseUeventFile(ueventFile) if err != nil { return pciDev, err } return pciDev, nil } // SRIOVSwitchdevEnabled returns true if switchdev mode is enabled on the given device. func SRIOVSwitchdevEnabled(deviceName string) bool { var buf bytes.Buffer ueventFile := fmt.Sprintf("%s/%s/device/uevent", sysClassNet, deviceName) pciDev, err := pci.ParseUeventFile(ueventFile) if err != nil { return false } slotName := fmt.Sprintf("pci/%s", pciDev.SlotName) err = subprocess.RunCommandWithFds(context.TODO(), nil, &buf, "devlink", "-j", "dev", "eswitch", "show", slotName) if err != nil { return false } dev := map[string]map[string]struct { Mode string }{} err = json.NewDecoder(&buf).Decode(&dev) if err != nil { return false } if dev["dev"][slotName].Mode == "switchdev" { return true } return false } // SRIOVFindRepresentorPort finds the associated representor port name for a switchdev VF ID. func SRIOVFindRepresentorPort(nicEntries []fs.DirEntry, pfSwitchID string, pfID int, vfID int) string { for _, nic := range nicEntries { nicSwitchID, err := os.ReadFile(filepath.Join(sysClassNet, nic.Name(), "phys_switch_id")) if err != nil { continue // Skip non-physical interfaces. } if string(nicSwitchID) != pfSwitchID { continue // Skip interfaces not connected to PF's switchdev. } // Check if this representor port matches the PF and VF by parsing phys_port_name. physPortName, err := os.ReadFile(filepath.Join(sysClassNet, nic.Name(), "phys_port_name")) if err != nil { continue // Skip interfaces with no physical port name. } var nicPFID, nicVFID int _, err = fmt.Sscanf(string(physPortName), "pf%dvf%d", &nicPFID, &nicVFID) if err != nil { continue // Skip non-VF interfaces. } if nicPFID == pfID && nicVFID == vfID { return nic.Name() // We have a match. } } return "" } func SRIOVGetSwitchAndPFID(parentDev string) (string, int, error) { physPortName, err := os.ReadFile(filepath.Join(sysClassNet, parentDev, "phys_port_name")) if err != nil { return "", -1, err // Skip non-physical ports. } // Check the port is a physical port and not an existing representor port connected to the bridge // but belonging to a physical device. This avoids trying to find a free VF repeatedly for the same // PF by mistakenly considering an existing representor ported connected to the bridge as a PF. if strings.HasPrefix(string(physPortName), "pf") || !strings.HasPrefix(string(physPortName), "p") { return "", -1, fmt.Errorf("Not a physical port: %s", string(physPortName)) } var pfID int _, err = fmt.Sscanf(string(physPortName), "p%d", &pfID) if err != nil { return "", -1, fmt.Errorf("Not a PF: %s.", string(physPortName)) // Skip non-PF interfaces. } // Check if switchdev is enabled on physical port. if !SRIOVSwitchdevEnabled(parentDev) { return "", -1, fmt.Errorf("Not a switchdev capable device: %s", parentDev) } physSwitchID, err := os.ReadFile(filepath.Join(sysClassNet, parentDev, "phys_switch_id")) if err != nil { return "", -1, fmt.Errorf("Unable to get phys_switch_id: %w", err) } return string(physSwitchID), pfID, nil } // SRIOVFindFreeVFAndRepresentor tries to find a free SR-IOV virtual function of a PF connected to an OVS bridge. // To do this it first looks at the ports on the OVS bridge specified and identifies which ones are PF ports in // switchdev mode. It then tries to find a free VF on that PF and the representor port associated to the VF ID. // It returns the PF name, representor port name, VF name, and VF ID. func SRIOVFindFreeVFAndRepresentor(state *state.State, ovsBridgeName string) (string, string, string, int, error) { nics, err := os.ReadDir(sysClassNet) if err != nil { return "", "", "", -1, fmt.Errorf("Failed to read directory %q: %w", sysClassNet, err) } vswitch, err := state.OVS() if err != nil { return "", "", "", -1, fmt.Errorf("Failed to connect to OVS: %w", err) } // Get all ports on the integration bridge. ports, err := vswitch.GetBridgePorts(context.TODO(), ovsBridgeName) if err != nil { return "", "", "", -1, fmt.Errorf("Failed to get port list: %w", err) } // Iterate through the list of ports and identify the PFs by trying to locate a VF (virtual function). for _, port := range ports { physSwitchID, pfID, err := SRIOVGetSwitchAndPFID(port) if err != nil { continue } vfName, vfID, err := SRIOVFindFreeVirtualFunction(state, port) if err != nil { continue } // Track down the representor port. The number of representor ports depends on the number of enabled VFs. // All representor ports have the same phys_switch_id as a PF connected to the same switch, and there may be // multiple PFs on one switch. representorPort := SRIOVFindRepresentorPort(nics, string(physSwitchID), pfID, vfID) if representorPort != "" { return port, representorPort, vfName, vfID, nil } } return "", "", "", -1, errors.New("No free virtual function and representor port found") } incus-7.0.0/internal/server/network/network_utils_test.go000066400000000000000000000151251517523235500237650ustar00rootroot00000000000000package network import ( "fmt" "net" "strings" "github.com/lxc/incus/v7/internal/iprange" ) func Example_parseIPRange() { _, allowedv4NetworkA, _ := net.ParseCIDR("192.168.1.0/24") _, allowedv4NetworkB, _ := net.ParseCIDR("192.168.0.0/16") _, allowedv6NetworkA, _ := net.ParseCIDR("fd22:c952:653e:3df6::/64") _, allowedv6NetworkB, _ := net.ParseCIDR("fd22:c952:653e::/48") ipRanges := []string{ // Ranges within allowedv4NetworkA. "192.168.1.1-192.168.1.255", "0.0.0.1-192.168.1.255", "0.0.0.1-0.0.0.255", // Ranges outsde of allowedv4NetworkA but within allowedv4NetworkB. "192.168.0.1-192.168.0.255", "192.168.0.0-192.168.0.0", "0.0.2.0-0.0.2.255", // Invalid IP ranges. "0.0.0.0.1-192.168.1.255", "192.0.0.1-192.0.0.255", "0.0.0.1-1.0.0.255", "0.0.2.1-0.0.0.255", // Ranges within allowedv6NetworkA. "fd22:c952:653e:3df6::1-fd22:c952:653e:3df6::FFFF", "::1-::FFFF", // Ranges outsde of allowedv6NetworkA but within allowedv6NetworkB. "fd22:c952:653e:FFFF::1-fd22:c952:653e:FFFF::FFFF", "::AAAA:FFFF:FFFF:FFFF:1-::AAAA:FFFF:FFFF:FFFF:FFFF", } fmt.Println("With allowed networks") for _, ipRange := range ipRanges { parsedRange, err := parseIPRange(ipRange, allowedv4NetworkA, allowedv4NetworkB, allowedv6NetworkA, allowedv6NetworkB) if err != nil { fmt.Printf("Err: %v\n", err) continue } fmt.Printf("Start: %s, End: %s\n", parsedRange.Start.String(), parsedRange.End.String()) } fmt.Println("Without allowed networks") for _, ipRange := range ipRanges { parsedRange, err := parseIPRange(ipRange) if err != nil { fmt.Printf("Err: %v\n", err) continue } fmt.Printf("Start: %s, End: %s\n", parsedRange.Start.String(), parsedRange.End.String()) } // Output: With allowed networks // Start: 192.168.1.1, End: 192.168.1.255 // Start: 192.168.1.1, End: 192.168.1.255 // Start: 192.168.1.1, End: 192.168.1.255 // Start: 192.168.0.1, End: 192.168.0.255 // Start: 192.168.0.0, End: 192.168.0.0 // Start: 192.168.2.0, End: 192.168.2.255 // Err: Start IP "0.0.0.0.1" is invalid // Err: IP range "192.0.0.1-192.0.0.255" does not fall within any of the allowed networks [192.168.1.0/24 192.168.0.0/16 fd22:c952:653e:3df6::/64 fd22:c952:653e::/48] // Err: IP range "0.0.0.1-1.0.0.255" does not fall within any of the allowed networks [192.168.1.0/24 192.168.0.0/16 fd22:c952:653e:3df6::/64 fd22:c952:653e::/48] // Err: Start IP "0.0.2.1" must be less than End IP "0.0.0.255" // Start: fd22:c952:653e:3df6::1, End: fd22:c952:653e:3df6::ffff // Start: fd22:c952:653e:3df6::1, End: fd22:c952:653e:3df6::ffff // Start: fd22:c952:653e:ffff::1, End: fd22:c952:653e:ffff::ffff // Start: fd22:c952:653e:aaaa:ffff:ffff:ffff:1, End: fd22:c952:653e:aaaa:ffff:ffff:ffff:ffff // Without allowed networks // Start: 192.168.1.1, End: 192.168.1.255 // Start: 0.0.0.1, End: 192.168.1.255 // Start: 0.0.0.1, End: 0.0.0.255 // Start: 192.168.0.1, End: 192.168.0.255 // Start: 192.168.0.0, End: 192.168.0.0 // Start: 0.0.2.0, End: 0.0.2.255 // Err: Start IP "0.0.0.0.1" is invalid // Start: 192.0.0.1, End: 192.0.0.255 // Start: 0.0.0.1, End: 1.0.0.255 // Err: Start IP "0.0.2.1" must be less than End IP "0.0.0.255" // Start: fd22:c952:653e:3df6::1, End: fd22:c952:653e:3df6::ffff // Start: ::1, End: ::ffff // Start: fd22:c952:653e:ffff::1, End: fd22:c952:653e:ffff::ffff // Start: ::aaaa:ffff:ffff:ffff:1, End: ::aaaa:ffff:ffff:ffff:ffff } func Example_ipRangesOverlap() { rangePairs := [][2]string{ {"10.1.1.1-10.1.1.2", "10.1.1.3-10.1.1.4"}, {"10.1.1.1-10.1.2.1", "10.1.1.254-10.1.1.255"}, {"10.1.1.1-10.1.1.6", "10.1.1.5-10.1.1.9"}, {"10.1.1.5-10.1.1.9", "10.1.1.1-10.1.1.6"}, {"::1-::2", "::3-::4"}, {"::1-::6", "::5-::9"}, {"::5-::9", "::1-::6"}, } for _, pair := range rangePairs { r0, _ := parseIPRange(pair[0]) r1, _ := parseIPRange(pair[1]) result := IPRangesOverlap(r0, r1) fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", r0, r1, result) } // also do a couple of tests with ranges that have no end singleIPRange := &iprange.Range{ Start: net.ParseIP("10.1.1.4"), } otherRange, _ := parseIPRange("10.1.1.1-10.1.1.6") fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", singleIPRange, otherRange, IPRangesOverlap(singleIPRange, otherRange)) fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", otherRange, singleIPRange, IPRangesOverlap(otherRange, singleIPRange)) fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", singleIPRange, singleIPRange, IPRangesOverlap(singleIPRange, singleIPRange)) otherRange, _ = parseIPRange("10.1.1.8-10.1.1.9") fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", singleIPRange, otherRange, IPRangesOverlap(singleIPRange, otherRange)) fmt.Printf("Range1: %v, Range2: %v, overlapped: %t\n", otherRange, singleIPRange, IPRangesOverlap(otherRange, singleIPRange)) // Output: // Range1: 10.1.1.1-10.1.1.2, Range2: 10.1.1.3-10.1.1.4, overlapped: false // Range1: 10.1.1.1-10.1.2.1, Range2: 10.1.1.254-10.1.1.255, overlapped: true // Range1: 10.1.1.1-10.1.1.6, Range2: 10.1.1.5-10.1.1.9, overlapped: true // Range1: 10.1.1.5-10.1.1.9, Range2: 10.1.1.1-10.1.1.6, overlapped: true // Range1: ::1-::2, Range2: ::3-::4, overlapped: false // Range1: ::1-::6, Range2: ::5-::9, overlapped: true // Range1: ::5-::9, Range2: ::1-::6, overlapped: true // Range1: 10.1.1.4, Range2: 10.1.1.1-10.1.1.6, overlapped: true // Range1: 10.1.1.1-10.1.1.6, Range2: 10.1.1.4, overlapped: true // Range1: 10.1.1.4, Range2: 10.1.1.4, overlapped: true // Range1: 10.1.1.4, Range2: 10.1.1.8-10.1.1.9, overlapped: false // Range1: 10.1.1.8-10.1.1.9, Range2: 10.1.1.4, overlapped: false } func Example_complementRanges() { _, ipnet, err := net.ParseCIDR("10.1.1.0/24") if err != nil { fmt.Printf("Err: %v\n", err) return } ranges := [][]*iprange.Range{ { {Start: net.ParseIP("10.1.1.1"), End: net.ParseIP("10.1.1.10")}, }, { {Start: net.ParseIP("10.1.1.10"), End: net.ParseIP("10.1.1.100")}, {Start: net.ParseIP("10.1.1.200"), End: net.ParseIP("10.1.1.230")}, }, { {Start: net.ParseIP("10.1.1.10"), End: net.ParseIP("10.1.1.20")}, {Start: net.ParseIP("10.1.1.15"), End: net.ParseIP("10.1.1.25")}, }, } for idx, r := range ranges { result, err := complementRanges(r, ipnet) if err != nil { fmt.Printf("Err: %v\n", err) return } parts := make([]string, len(result)) for i, r := range result { parts[i] = fmt.Sprintf("%s-%s", r.Start.String(), r.End.String()) } fmt.Printf("Range%d: %s\n", idx+1, strings.Join(parts, ", ")) } // Output: // Range1: 10.1.1.11-10.1.1.254 // Range2: 10.1.1.1-10.1.1.9, 10.1.1.101-10.1.1.199, 10.1.1.231-10.1.1.254 // Range3: 10.1.1.1-10.1.1.9, 10.1.1.26-10.1.1.254 } incus-7.0.0/internal/server/network/ovn/000077500000000000000000000000001517523235500202645ustar00rootroot00000000000000incus-7.0.0/internal/server/network/ovn/const.go000066400000000000000000000003201517523235500217340ustar00rootroot00000000000000package ovn // OVS TCP Flags from OVS lib/packets.h. const ( TCPFIN = 0x001 TCPSYN = 0x002 TCPRST = 0x004 TCPPSH = 0x008 TCPACK = 0x010 TCPURG = 0x020 TCPECE = 0x040 TCPCWR = 0x080 TCPNS = 0x100 ) incus-7.0.0/internal/server/network/ovn/errors.go000066400000000000000000000010621517523235500221260ustar00rootroot00000000000000package ovn import ( "errors" ovsdbClient "github.com/ovn-kubernetes/libovsdb/client" ) // ErrExists indicates that a DB record already exists. var ErrExists = errors.New("object already exists") // ErrNotFound indicates that a DB record doesn't exist. var ErrNotFound = ovsdbClient.ErrNotFound // ErrTooMany is returned when one match is expected but multiple are found. var ErrTooMany = errors.New("too many objects found") // ErrNotManaged indicates that a DB record wasn't created by Incus. var ErrNotManaged = errors.New("object not incus-managed") incus-7.0.0/internal/server/network/ovn/events.go000066400000000000000000000006351517523235500221230ustar00rootroot00000000000000package ovn import ( ovsdbModel "github.com/ovn-kubernetes/libovsdb/model" ) // EventHandler represents an OVN database event handler. type EventHandler struct { // Tables contains the list of OVN database tables to watch for events. Tables []string // Hook is the function being called on a matching event. Hook func(action string, table string, oldObject ovsdbModel.Model, newObject ovsdbModel.Model) } incus-7.0.0/internal/server/network/ovn/ovn_icnb.go000066400000000000000000000070661517523235500224210ustar00rootroot00000000000000package ovn import ( "context" "crypto/tls" "crypto/x509" "encoding/pem" "errors" "runtime" "strings" "time" "github.com/cenkalti/backoff/v5" "github.com/go-logr/logr" ovsdbClient "github.com/ovn-kubernetes/libovsdb/client" ovnICNB "github.com/lxc/incus/v7/internal/server/network/ovn/schema/ovn-ic-nb" ) // ICNB client. type ICNB struct { client ovsdbClient.Client cookie ovsdbClient.MonitorCookie } // NewICNB initializes new OVN client for Northbound IC operations. func NewICNB(dbAddr string, sslCACert string, sslClientCert string, sslClientKey string) (*ICNB, error) { // Create the NB struct. client := &ICNB{} // Prepare the OVSDB client. dbSchema, err := ovnICNB.FullDatabaseModel() if err != nil { return nil, err } discard := logr.Discard() options := []ovsdbClient.Option{ovsdbClient.WithLogger(&discard), ovsdbClient.WithReconnect(5*time.Second, &backoff.ZeroBackOff{})} for _, entry := range strings.Split(dbAddr, ",") { options = append(options, ovsdbClient.WithEndpoint(entry)) } // Handle SSL. if strings.Contains(dbAddr, "ssl:") { // Validation. if sslClientCert == "" { return nil, errors.New("OVN IC Northbound database is configured to use SSL but no client certificate was found") } if sslClientKey == "" { return nil, errors.New("OVN IC Northbound database is configured to use SSL but no client key was found") } // Prepare the client. clientCert, err := tls.X509KeyPair([]byte(sslClientCert), []byte(sslClientKey)) if err != nil { return nil, err } tlsConfig := &tls.Config{ Certificates: []tls.Certificate{clientCert}, InsecureSkipVerify: true, } // Add CA check if provided. if sslCACert != "" { tlsCAder, _ := pem.Decode([]byte(sslCACert)) if tlsCAder == nil { return nil, errors.New("Couldn't parse CA certificate") } tlsCAcert, err := x509.ParseCertificate(tlsCAder.Bytes) if err != nil { return nil, err } tlsCAcert.IsCA = true tlsCAcert.KeyUsage = x509.KeyUsageCertSign clientCAPool := x509.NewCertPool() clientCAPool.AddCert(tlsCAcert) tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, chains [][]*x509.Certificate) error { if len(rawCerts) < 1 { return errors.New("Missing server certificate") } // Load the chain. intermediates := x509.NewCertPool() if len(rawCerts) > 1 { for _, rawCert := range rawCerts[1:] { cert, _ := x509.ParseCertificate(rawCert) if cert != nil { intermediates.AddCert(cert) } } } // Load the main server certificate. cert, _ := x509.ParseCertificate(rawCerts[0]) if cert == nil { return errors.New("Bad server certificate") } // Validate. opts := x509.VerifyOptions{ Roots: clientCAPool, Intermediates: intermediates, } _, err := cert.Verify(opts) return err } } // Add the TLS config to the client. options = append(options, ovsdbClient.WithTLSConfig(tlsConfig)) } // Connect to OVSDB. ovn, err := ovsdbClient.NewOVSDBClient(dbSchema, options...) if err != nil { return nil, err } err = ovn.Connect(context.TODO()) if err != nil { return nil, err } err = ovn.Echo(context.TODO()) if err != nil { return nil, err } monitorCookie, err := ovn.MonitorAll(context.TODO()) if err != nil { return nil, err } // Add the client to the struct. client.client = ovn client.cookie = monitorCookie // Set finalizer to stop the monitor. runtime.SetFinalizer(client, func(o *ICNB) { _ = ovn.MonitorCancel(context.Background(), o.cookie) ovn.Close() }) return client, nil } incus-7.0.0/internal/server/network/ovn/ovn_icnb_actions.go000066400000000000000000000136721517523235500241410ustar00rootroot00000000000000package ovn import ( "context" "encoding/binary" "errors" "fmt" "math/rand" "net" "net/netip" "slices" "strings" ovsdbClient "github.com/ovn-kubernetes/libovsdb/client" "github.com/ovn-kubernetes/libovsdb/ovsdb" ovnICNB "github.com/lxc/incus/v7/internal/server/network/ovn/schema/ovn-ic-nb" ) // CreateTransitSwitch creates a new managed transit switch. func (o *ICNB) CreateTransitSwitch(ctx context.Context, name string, mayExist bool) error { // Look for an existing transit switch. transitSwitch := ovnICNB.TransitSwitch{ Name: name, } err := o.client.Get(ctx, &transitSwitch) if err != nil && !errors.Is(err, ovsdbClient.ErrNotFound) { return err } // Handle existing switches. if transitSwitch.UUID != "" { if !mayExist { return ErrExists } return nil } // Generate a random IPv4 subnet (/28). buf := make([]byte, 4) binary.LittleEndian.PutUint32(buf, rand.Uint32()) buf[0] = 169 buf[1] = 254 ipv4 := net.IP(buf) ipv4Net := net.IPNet{IP: ipv4.Mask(net.CIDRMask(28, 32)), Mask: net.CIDRMask(28, 32)} // Mark new switches as managed by Incus. transitSwitch.ExternalIDs = map[string]string{ "incus-managed": "true", "incus-subnet-ipv4": ipv4Net.String(), "incus-subnet-ipv6": fmt.Sprintf("fd42:%x:%x:%x::/64", rand.Intn(65535), rand.Intn(65535), rand.Intn(65535)), } // Create the switch. ops, err := o.client.Create(&transitSwitch) if err != nil { return err } resp, err := o.client.Transact(ctx, ops...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, ops) if err != nil { return err } return nil } // CreateTransitSwitchAllocation creates a new allocation on the switch. func (o *ICNB) CreateTransitSwitchAllocation(ctx context.Context, switchName string, azName string) (*net.IPNet, *net.IPNet, error) { // Get the switch. transitSwitch := ovnICNB.TransitSwitch{ Name: switchName, } err := o.client.Get(ctx, &transitSwitch) if err != nil { return nil, nil, err } // Check that it's managed by us. if transitSwitch.ExternalIDs == nil || transitSwitch.ExternalIDs["incus-managed"] != "true" { return nil, nil, errors.New("Transit switch isn't Incus managed") } // Check that prefixes are set. if transitSwitch.ExternalIDs["incus-subnet-ipv4"] == "" || transitSwitch.ExternalIDs["incus-subnet-ipv6"] == "" { return nil, nil, errors.New("No configured subnets on the transit switch") } // Get the allocated addresses. v4Addresses := []string{} v6Addresses := []string{} for k, v := range transitSwitch.ExternalIDs { if !strings.HasPrefix(k, "incus-allocation-") { continue } fields := strings.Split(v, ",") if len(fields) != 2 { continue } v4Addresses = append(v4Addresses, fields[0]) v6Addresses = append(v6Addresses, fields[1]) } // Get the prefixes. v4Prefix, err := netip.ParsePrefix(transitSwitch.ExternalIDs["incus-subnet-ipv4"]) if err != nil { return nil, nil, err } v6Prefix, err := netip.ParsePrefix(transitSwitch.ExternalIDs["incus-subnet-ipv6"]) if err != nil { return nil, nil, err } // Allocate new IPs in the subnet. v4Addr := v4Prefix.Addr() for { v4Addr = v4Addr.Next() if !v4Prefix.Contains(v4Addr) { return nil, nil, errors.New("Transit switch is out of IPv4 addresses") } if !slices.Contains(v4Addresses, v4Addr.String()) { break } } v6Addr := v6Prefix.Addr() for { v6Addr = v6Addr.Next() if !v6Prefix.Contains(v6Addr) { return nil, nil, errors.New("Transit switch is out of IPv6 addresses") } if !slices.Contains(v6Addresses, v6Addr.String()) { break } } // Update the record. transitSwitch.ExternalIDs[fmt.Sprintf("incus-allocation-%s", azName)] = fmt.Sprintf("%s,%s", v4Addr.String(), v6Addr.String()) ops, err := o.client.Where(&transitSwitch).Update(&transitSwitch) if err != nil { return nil, nil, err } resp, err := o.client.Transact(ctx, ops...) if err != nil { return nil, nil, err } _, err = ovsdb.CheckOperationResults(resp, ops) if err != nil { return nil, nil, err } return &net.IPNet{IP: net.IP(v4Addr.AsSlice()), Mask: net.CIDRMask(v4Prefix.Bits(), 32)}, &net.IPNet{IP: net.IP(v6Addr.AsSlice()), Mask: net.CIDRMask(v6Prefix.Bits(), 128)}, nil } // DeleteTransitSwitchAllocation removes a current allocation from the switch. func (o *ICNB) DeleteTransitSwitchAllocation(ctx context.Context, switchName string, azName string) error { // Get the switch. transitSwitch := ovnICNB.TransitSwitch{ Name: switchName, } err := o.client.Get(ctx, &transitSwitch) if err != nil { return err } // Check that it's managed by us. if transitSwitch.ExternalIDs == nil || transitSwitch.ExternalIDs["incus-managed"] != "true" { return errors.New("Transit switch isn't Incus managed") } // Update the record. delete(transitSwitch.ExternalIDs, fmt.Sprintf("incus-allocation-%s", azName)) ops, err := o.client.Where(&transitSwitch).Update(&transitSwitch) if err != nil { return err } resp, err := o.client.Transact(ctx, ops...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, ops) if err != nil { return err } return nil } // DeleteTransitSwitch deletes an existing transit switch. // The force parameter is required to delete a transit switch which wasn't created by Incus. func (o *ICNB) DeleteTransitSwitch(ctx context.Context, name string, force bool) error { // Get the current transit switch. transitSwitch := ovnICNB.TransitSwitch{ Name: name, } err := o.client.Get(ctx, &transitSwitch) if err != nil { // Already deleted. if errors.Is(err, ErrNotFound) { return nil } return err } // Check if the switch is incus-managed. if !force && transitSwitch.ExternalIDs["incus-managed"] != "true" { return ErrNotManaged } // Delete the switch. deleteOps, err := o.client.Where(&transitSwitch).Delete() if err != nil { return err } resp, err := o.client.Transact(ctx, deleteOps...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, deleteOps) if err != nil { return err } return nil } incus-7.0.0/internal/server/network/ovn/ovn_icsb.go000066400000000000000000000074331517523235500224240ustar00rootroot00000000000000package ovn import ( "context" "crypto/tls" "crypto/x509" "encoding/pem" "errors" "runtime" "strings" "time" "github.com/cenkalti/backoff/v5" "github.com/go-logr/logr" ovsdbClient "github.com/ovn-kubernetes/libovsdb/client" ovsdbModel "github.com/ovn-kubernetes/libovsdb/model" ovnICSB "github.com/lxc/incus/v7/internal/server/network/ovn/schema/ovn-ic-sb" ) // ICSB client. type ICSB struct { client ovsdbClient.Client cookie ovsdbClient.MonitorCookie } // NewICSB initializes new OVN client for Southbound IC operations. func NewICSB(dbAddr string, sslCACert string, sslClientCert string, sslClientKey string) (*ICSB, error) { // Create the SB struct. client := &ICSB{} // Prepare the OVSDB client. dbSchema, err := ovnICSB.FullDatabaseModel() if err != nil { return nil, err } // Add some missing indexes. dbSchema.SetIndexes(map[string][]ovsdbModel.ClientIndex{ "Gateway": {{Columns: []ovsdbModel.ColumnKey{{Column: "availability_zone"}}}}, }) discard := logr.Discard() options := []ovsdbClient.Option{ovsdbClient.WithLogger(&discard), ovsdbClient.WithReconnect(5*time.Second, &backoff.ZeroBackOff{})} for _, entry := range strings.Split(dbAddr, ",") { options = append(options, ovsdbClient.WithEndpoint(entry)) } // Handle SSL. if strings.Contains(dbAddr, "ssl:") { // Validation. if sslClientCert == "" { return nil, errors.New("OVN IC Southbound database is configured to use SSL but no client certificate was found") } if sslClientKey == "" { return nil, errors.New("OVN IC Southbound database is configured to use SSL but no client key was found") } // Prepare the client. clientCert, err := tls.X509KeyPair([]byte(sslClientCert), []byte(sslClientKey)) if err != nil { return nil, err } tlsConfig := &tls.Config{ Certificates: []tls.Certificate{clientCert}, InsecureSkipVerify: true, } // Add CA check if provided. if sslCACert != "" { tlsCAder, _ := pem.Decode([]byte(sslCACert)) if tlsCAder == nil { return nil, errors.New("Couldn't parse CA certificate") } tlsCAcert, err := x509.ParseCertificate(tlsCAder.Bytes) if err != nil { return nil, err } tlsCAcert.IsCA = true tlsCAcert.KeyUsage = x509.KeyUsageCertSign clientCAPool := x509.NewCertPool() clientCAPool.AddCert(tlsCAcert) tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, chains [][]*x509.Certificate) error { if len(rawCerts) < 1 { return errors.New("Missing server certificate") } // Load the chain. intermediates := x509.NewCertPool() if len(rawCerts) > 1 { for _, rawCert := range rawCerts[1:] { cert, _ := x509.ParseCertificate(rawCert) if cert != nil { intermediates.AddCert(cert) } } } // Load the main server certificate. cert, _ := x509.ParseCertificate(rawCerts[0]) if cert == nil { return errors.New("Bad server certificate") } // Validate. opts := x509.VerifyOptions{ Roots: clientCAPool, Intermediates: intermediates, } _, err := cert.Verify(opts) return err } } // Add the TLS config to the client. options = append(options, ovsdbClient.WithTLSConfig(tlsConfig)) } // Connect to OVSDB. ovn, err := ovsdbClient.NewOVSDBClient(dbSchema, options...) if err != nil { return nil, err } err = ovn.Connect(context.TODO()) if err != nil { return nil, err } err = ovn.Echo(context.TODO()) if err != nil { return nil, err } monitorCookie, err := ovn.MonitorAll(context.TODO()) if err != nil { return nil, err } // Add the client to the struct. client.client = ovn client.cookie = monitorCookie // Set finalizer to stop the monitor. runtime.SetFinalizer(client, func(o *ICSB) { _ = ovn.MonitorCancel(context.Background(), o.cookie) ovn.Close() }) return client, nil } incus-7.0.0/internal/server/network/ovn/ovn_icsb_actions.go000066400000000000000000000015431517523235500241400ustar00rootroot00000000000000package ovn import ( "context" ovnICSB "github.com/lxc/incus/v7/internal/server/network/ovn/schema/ovn-ic-sb" ) // GetGateways returns a slice of gateways for the specified availability zone. func (o *ICSB) GetGateways(ctx context.Context, name string) ([]string, error) { // Get the availability zone. availabilityZone := ovnICSB.AvailabilityZone{ Name: name, } err := o.client.Get(ctx, &availabilityZone) if err != nil { return nil, err } // Get the gateways in the availability zone. gateways := []ovnICSB.Gateway{} gateway := ovnICSB.Gateway{ AvailabilityZone: availabilityZone.UUID, } err = o.client.Where(&gateway).List(ctx, &gateways) if err != nil { return nil, err } // Extract the names. names := make([]string, 0, len(gateways)) for _, entry := range gateways { names = append(names, entry.Name) } return names, nil } incus-7.0.0/internal/server/network/ovn/ovn_nb.go000066400000000000000000000125351517523235500221020ustar00rootroot00000000000000package ovn import ( "context" "crypto/tls" "crypto/x509" "encoding/pem" "errors" "reflect" "runtime" "strings" "time" "github.com/cenkalti/backoff/v5" "github.com/go-logr/logr" ovsdbClient "github.com/ovn-kubernetes/libovsdb/client" ovsdbModel "github.com/ovn-kubernetes/libovsdb/model" ovnNB "github.com/lxc/incus/v7/internal/server/network/ovn/schema/ovn-nb" ) // NB client. type NB struct { client ovsdbClient.Client cookie ovsdbClient.MonitorCookie } var nb *NB // NewNB initializes new OVN client for Northbound operations. func NewNB(dbAddr string, sslCACert string, sslClientCert string, sslClientKey string) (*NB, error) { if nb != nil { return nb, nil } // Create the NB struct. client := &NB{} // Prepare the OVSDB client. dbSchema, err := ovnNB.FullDatabaseModel() if err != nil { return nil, err } // Add some missing indexes. dbSchema.SetIndexes(map[string][]ovsdbModel.ClientIndex{ "Load_Balancer": {{Columns: []ovsdbModel.ColumnKey{{Column: "name"}}}}, "Logical_Router": {{Columns: []ovsdbModel.ColumnKey{{Column: "name"}}}}, "Logical_Switch": {{Columns: []ovsdbModel.ColumnKey{{Column: "name"}}}}, "Logical_Switch_Port": {{Columns: []ovsdbModel.ColumnKey{{Column: "name"}}}}, }) discard := logr.Discard() options := []ovsdbClient.Option{ovsdbClient.WithLogger(&discard), ovsdbClient.WithReconnect(5*time.Second, &backoff.ZeroBackOff{})} for _, entry := range strings.Split(dbAddr, ",") { options = append(options, ovsdbClient.WithEndpoint(entry)) } // Handle SSL. if strings.Contains(dbAddr, "ssl:") { // Validation. if sslClientCert == "" { return nil, errors.New("OVN is configured to use SSL but no client certificate was found") } if sslClientKey == "" { return nil, errors.New("OVN is configured to use SSL but no client key was found") } // Prepare the client. clientCert, err := tls.X509KeyPair([]byte(sslClientCert), []byte(sslClientKey)) if err != nil { return nil, err } tlsConfig := &tls.Config{ Certificates: []tls.Certificate{clientCert}, InsecureSkipVerify: true, } // Add CA check if provided. if sslCACert != "" { tlsCAder, _ := pem.Decode([]byte(sslCACert)) if tlsCAder == nil { return nil, errors.New("Couldn't parse CA certificate") } tlsCAcert, err := x509.ParseCertificate(tlsCAder.Bytes) if err != nil { return nil, err } tlsCAcert.IsCA = true tlsCAcert.KeyUsage = x509.KeyUsageCertSign clientCAPool := x509.NewCertPool() clientCAPool.AddCert(tlsCAcert) tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, chains [][]*x509.Certificate) error { if len(rawCerts) < 1 { return errors.New("Missing server certificate") } // Load the chain. intermediates := x509.NewCertPool() if len(rawCerts) > 1 { for _, rawCert := range rawCerts[1:] { cert, _ := x509.ParseCertificate(rawCert) if cert != nil { intermediates.AddCert(cert) } } } // Load the main server certificate. cert, _ := x509.ParseCertificate(rawCerts[0]) if cert == nil { return errors.New("Bad server certificate") } // Validate. opts := x509.VerifyOptions{ Roots: clientCAPool, Intermediates: intermediates, } _, err := cert.Verify(opts) return err } } // Add the TLS config to the client. options = append(options, ovsdbClient.WithTLSConfig(tlsConfig)) } // Connect to OVSDB. ovn, err := ovsdbClient.NewOVSDBClient(dbSchema, options...) if err != nil { return nil, err } err = ovn.Connect(context.TODO()) if err != nil { return nil, err } err = ovn.Echo(context.TODO()) if err != nil { return nil, err } monitorCookie, err := ovn.MonitorAll(context.TODO()) if err != nil { return nil, err } // Add the client to the struct. client.client = ovn client.cookie = monitorCookie // Set finalizer to stop the monitor. runtime.SetFinalizer(client, func(o *NB) { _ = ovn.MonitorCancel(context.Background(), o.cookie) ovn.Close() }) nb = client return client, nil } // get is used to perform a libovsdb Get call while also makes use of the custom defined index. // For some reason the main Get() function only uses the built-in indices rather than considering the user provided ones. // This is apparently by design but makes it much more annoying to fetch records from some tables. func (o *NB) get(ctx context.Context, m ovsdbModel.Model) error { var collection any // Check if one of the broken types. switch m.(type) { case *ovnNB.LoadBalancer: s := []ovnNB.LoadBalancer{} collection = &s case *ovnNB.LogicalRouter: s := []ovnNB.LogicalRouter{} collection = &s case *ovnNB.LogicalSwitch: s := []ovnNB.LogicalSwitch{} collection = &s case *ovnNB.LogicalSwitchPort: s := []ovnNB.LogicalSwitchPort{} collection = &s default: // Fallback to normal Get. return o.client.Get(ctx, m) } // Check and assign the resulting value. err := o.client.Where(m).List(ctx, collection) if err != nil { return err } rVal := reflect.ValueOf(collection) if rVal.Kind() != reflect.Pointer { return errors.New("Bad collection type") } rVal = rVal.Elem() if rVal.Kind() != reflect.Slice { return errors.New("Bad collection type") } if rVal.Len() == 0 { return ovsdbClient.ErrNotFound } if rVal.Len() > 1 { return ErrTooMany } reflect.ValueOf(m).Elem().Set(rVal.Index(0)) return nil } incus-7.0.0/internal/server/network/ovn/ovn_nb_actions.go000066400000000000000000003350371517523235500236270ustar00rootroot00000000000000package ovn import ( "context" "errors" "fmt" "maps" "net" "slices" "strings" "time" ovsClient "github.com/ovn-kubernetes/libovsdb/client" ovsModel "github.com/ovn-kubernetes/libovsdb/model" "github.com/ovn-kubernetes/libovsdb/ovsdb" "github.com/lxc/incus/v7/internal/iprange" ovnNB "github.com/lxc/incus/v7/internal/server/network/ovn/schema/ovn-nb" ovnSB "github.com/lxc/incus/v7/internal/server/network/ovn/schema/ovn-sb" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/util" ) // OVNRouter OVN router name. type OVNRouter string // OVNRouterPort OVN router port name. type OVNRouterPort string // OVNSwitch OVN switch name. type OVNSwitch string // OVNSwitchPort OVN switch port name. type OVNSwitchPort string // OVNSwitchPortUUID OVN switch port UUID. type OVNSwitchPortUUID string // OVNChassisGroup OVN HA chassis group name. type OVNChassisGroup string // OVNDNSUUID OVN DNS record UUID. type OVNDNSUUID string // OVNDHCPOptionsUUID DHCP Options set UUID. type OVNDHCPOptionsUUID string // OVNPortGroup OVN port group name. type OVNPortGroup string // OVNPortGroupUUID OVN port group UUID. type OVNPortGroupUUID string // OVNLoadBalancer OVN load balancer name. type OVNLoadBalancer string // OVNAddressSet OVN address set for ACLs. type OVNAddressSet string // OVNIPAllocationOpts defines IP allocation settings that can be applied to a logical switch. type OVNIPAllocationOpts struct { PrefixIPv4 *net.IPNet PrefixIPv6 *net.IPNet ExcludeIPv4 []iprange.Range } // OVNIPv6AddressMode IPv6 router advertisement address mode. type OVNIPv6AddressMode string // OVNIPv6AddressModeSLAAC IPv6 SLAAC mode. const OVNIPv6AddressModeSLAAC OVNIPv6AddressMode = "slaac" // OVNIPv6AddressModeDHCPStateful IPv6 DHCPv6 stateful mode. const OVNIPv6AddressModeDHCPStateful OVNIPv6AddressMode = "dhcpv6_stateful" // OVNIPv6AddressModeDHCPStateless IPv6 DHCPv6 stateless mode. const OVNIPv6AddressModeDHCPStateless OVNIPv6AddressMode = "dhcpv6_stateless" // OVN External ID names used by Incus. const ( ovnExtIDIncusSwitch = "incus_switch" ovnExtIDIncusSwitchPort = "incus_switch_port" ovnExtIDIncusProjectID = "incus_project_id" ovnExtIDIncusPortGroup = "incus_port_group" ovnExtIDIncusLocation = "incus_location" ) // OVNIPv6RAOpts IPv6 router advertisements options that can be applied to a router. type OVNIPv6RAOpts struct { SendPeriodic bool AddressMode OVNIPv6AddressMode MinInterval time.Duration MaxInterval time.Duration RecursiveDNSServer net.IP DNSSearchList []string MTU uint32 } // OVNDHCPOptsSet is an existing DHCP options set in the northbound database. type OVNDHCPOptsSet struct { UUID OVNDHCPOptionsUUID CIDR *net.IPNet } // OVNDHCPv4Opts IPv4 DHCP options that can be applied to a switch port. type OVNDHCPv4Opts struct { ServerID net.IP ServerMAC net.HardwareAddr Router net.IP RecursiveDNSServer []net.IP DomainName string LeaseTime time.Duration MTU uint32 Netmask string DNSSearchList []string StaticRoutes string } // OVNDHCPv6Opts IPv6 DHCP option set that can be created (and then applied to a switch port by resulting ID). type OVNDHCPv6Opts struct { ServerID net.HardwareAddr RecursiveDNSServer []net.IP DNSSearchList []string DHCPv6Stateless bool } // OVNSwitchPortOpts options that can be applied to a switch port. type OVNSwitchPortOpts struct { MAC net.HardwareAddr // Optional, if nil will be set to dynamic. IPV4 string // Optional, if empty, allocate an address, if "none" then disable allocation. IPV6 string // Optional, if empty, allocate an address, if "none" then disable allocation. DHCPv4OptsID OVNDHCPOptionsUUID // Optional, if empty, no DHCPv4 enabled on port. DHCPv6OptsID OVNDHCPOptionsUUID // Optional, if empty, no DHCPv6 enabled on port. Parent OVNSwitchPort // Optional, if set a nested port is created. VLAN uint16 // Optional, use with Parent to request a specific VLAN for nested port. Location string // Optional, use to indicate the name of the server this port is bound to. RouterPort OVNRouterPort // Optional, the name of the associated logical router port. Promiscuous bool // Optional, controls whether to allow unknown traffic on the port. } // OVNACLRule represents an ACL rule that can be added to a logical switch or port group. type OVNACLRule struct { Direction string // Either "from-lport" or "to-lport". Action string // Either "allow-related", "allow", "drop", or "reject". Match string // Match criteria. See OVN Southbound database's Logical_Flow table match column usage. Priority int // Priority (between 0 and 32767, inclusive). Higher values take precedence. Log bool // Whether or not to log matched packets. LogName string // Log label name (requires Log be true). } // OVNQoSRule represents a QoS rule that can be added to a logical switch. type OVNQoSRule struct { Direction string Action map[string]int // Not settable, but ovn includes it in the db model Bandwidth map[string]int Match string Priority int } // OVNLoadBalancerTarget represents an OVN load balancer Virtual IP target. type OVNLoadBalancerTarget struct { Address net.IP Port uint64 } // OVNLoadBalancerHealthCheck represents an OVN load balancer health checker. type OVNLoadBalancerHealthCheck struct { Interval int Timeout int SuccessCount int FailureCount int CheckerIPV4 net.IP CheckerIPV6 net.IP } // OVNLoadBalancerVIP represents a OVN load balancer Virtual IP entry. type OVNLoadBalancerVIP struct { HealthCheck *OVNLoadBalancerHealthCheck Protocol string // Either "tcp" or "udp". But only applies to port based VIPs. ListenAddress net.IP ListenPort uint64 Targets []OVNLoadBalancerTarget } // OVNRouterRoute represents a static route added to a logical router. type OVNRouterRoute struct { Prefix net.IPNet NextHop net.IP Port OVNRouterPort Discard bool } // OVNRouterPolicy represents a router policy. type OVNRouterPolicy struct { Priority int Match string Action string NextHop net.IP } // OVNRouterPeering represents a the configuration of a peering connection between two OVN logical routers. type OVNRouterPeering struct { LocalRouter OVNRouter LocalRouterPort OVNRouterPort LocalRouterPortMAC net.HardwareAddr LocalRouterPortIPs []net.IPNet LocalRouterRoutes []net.IPNet TargetRouter OVNRouter TargetRouterPort OVNRouterPort TargetRouterPortMAC net.HardwareAddr TargetRouterPortIPs []net.IPNet TargetRouterRoutes []net.IPNet } // CreateLogicalRouter adds a named logical router. // If mayExist is true, then an existing resource of the same name is not treated as an error. func (o *NB) CreateLogicalRouter(ctx context.Context, routerName OVNRouter, mayExist bool) error { logicalRouter := ovnNB.LogicalRouter{ Name: string(routerName), } // Check if already exists. err := o.get(ctx, &logicalRouter) if err != nil && !errors.Is(err, ErrNotFound) { return err } if logicalRouter.UUID != "" { if !mayExist { return ErrExists } if logicalRouter.Options != nil { return nil } } // Set some options. logicalRouter.Options = map[string]string{ "always_learn_from_arp_request": "false", "dynamic_neigh_routers": "true", } // Create the record. var operations []ovsdb.Operation if logicalRouter.UUID == "" { operations, err = o.client.Create(&logicalRouter) if err != nil { return err } } else { operations, err = o.client.Where(&logicalRouter).Update(&logicalRouter) if err != nil { return err } } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // DeleteLogicalRouter deletes a named logical router. func (o *NB) DeleteLogicalRouter(ctx context.Context, routerName OVNRouter) error { logicalRouter := ovnNB.LogicalRouter{ Name: string(routerName), } err := o.get(ctx, &logicalRouter) if err != nil { // Logical router is already gone. if errors.Is(err, ErrNotFound) { return nil } return err } operations, err := o.client.Where(&logicalRouter).Delete() if err != nil { return err } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // GetLogicalRouter gets the OVN database record for the router. func (o *NB) GetLogicalRouter(ctx context.Context, routerName OVNRouter) (*ovnNB.LogicalRouter, error) { logicalRouter := &ovnNB.LogicalRouter{ Name: string(routerName), } err := o.get(ctx, logicalRouter) if err != nil { return nil, err } return logicalRouter, nil } // CreateLogicalRouterNAT adds an SNAT or DNAT rule to a logical router to translate packets from intNet to extIP. func (o *NB) CreateLogicalRouterNAT(ctx context.Context, routerName OVNRouter, natType string, intNet *net.IPNet, extIP net.IP, intIP net.IP, stateless bool, mayExist bool) error { // Prepare the addresses. var logicalIP string var externalIP string if natType == "snat" { logicalIP = intNet.String() externalIP = extIP.String() } else if natType == "dnat_and_snat" { logicalIP = intIP.String() externalIP = extIP.String() } else { return fmt.Errorf("Invalid NAT rule type %q", natType) } // Get the logical router. logicalRouter, err := o.GetLogicalRouter(ctx, routerName) if err != nil { return err } // Check if the rule already exists. for _, natUUID := range logicalRouter.Nat { natRule := ovnNB.NAT{ UUID: natUUID, } err = o.get(ctx, &natRule) if err != nil { return err } // Check if rule is of the requested type. if natRule.Type != natType { continue } // Check if matching our new rule. if natRule.LogicalIP == logicalIP && natRule.ExternalIP == externalIP { if mayExist { return nil } return ErrExists } } natRule := ovnNB.NAT{ UUID: "nat", Options: map[string]string{"stateless": fmt.Sprintf("%v", stateless)}, Type: natType, LogicalIP: logicalIP, ExternalIP: externalIP, } operations := []ovsdb.Operation{} createOps, err := o.client.Create(&natRule) if err != nil { return err } operations = append(operations, createOps...) // Add it to the router. updateOps, err := o.client.Where(logicalRouter).Mutate(logicalRouter, ovsModel.Mutation{ Field: &logicalRouter.Nat, Mutator: ovsdb.MutateOperationInsert, Value: []string{natRule.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // DeleteLogicalRouterNAT deletes all NAT rules of a particular type from a logical router. func (o *NB) DeleteLogicalRouterNAT(ctx context.Context, routerName OVNRouter, natType string, all bool, extIPs ...net.IP) error { // Quick checks. if all && len(extIPs) != 0 { return errors.New("Can't ask for all NAT rules to be deleted and specify specific addresses") } // Get the logical router. logicalRouter, err := o.GetLogicalRouter(ctx, routerName) if err != nil { return err } operations := []ovsdb.Operation{} // Go through all rules. for _, natUUID := range logicalRouter.Nat { natRule := ovnNB.NAT{ UUID: natUUID, } err = o.get(ctx, &natRule) if err != nil { return err } // Check if rule is of the requested type. if natRule.Type != natType { continue } // Check if the address matches. if !all { found := false for _, extIP := range extIPs { if natRule.ExternalIP == extIP.String() { found = true break } } if !found { continue } } // Delete the rule. deleteOps, err := o.client.Where(&natRule).Delete() if err != nil { return err } operations = append(operations, deleteOps...) // Delete the entry from the logical router. deleteOps, err = o.client.Where(logicalRouter).Mutate(logicalRouter, ovsModel.Mutation{ Field: &logicalRouter.Nat, Mutator: ovsdb.MutateOperationDelete, Value: []string{natRule.UUID}, }) if err != nil { return err } operations = append(operations, deleteOps...) } if len(operations) == 0 { return nil } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // CreateStaticMACBinding ensures a Static_Mac_Binding row exists in NB for this router port/IP. func (o *NB) CreateStaticMACBinding(ctx context.Context, portName OVNRouterPort, ip net.IP, mac net.HardwareAddr, mayExist bool) error { binding := ovnNB.StaticMACBinding{ LogicalPort: string(portName), IP: ip.String(), MAC: mac.String(), OverrideDynamicMAC: true, } // Check if the binding already exists. err := o.get(ctx, &binding) if err != nil && !errors.Is(err, ErrNotFound) { return err } if binding.UUID != "" { if !mayExist { return ErrExists } if binding.LogicalPort == string(portName) && binding.IP == ip.String() && binding.MAC == mac.String() && binding.OverrideDynamicMAC { return nil } } // Create the record. var operations []ovsdb.Operation if binding.UUID == "" { operations, err = o.client.Create(&binding) if err != nil { return err } } else { binding.LogicalPort = string(portName) binding.IP = ip.String() binding.MAC = mac.String() operations, err = o.client.Where(&binding).Update(&binding) if err != nil { return err } } // Apply the changes. reply, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(reply, operations) if err != nil { return err } return nil } // DeleteStaticMACBindings deletes all static MAC bindings from the specified router port. // It allows filtering what family to flush. func (o *NB) DeleteStaticMACBindings(ctx context.Context, portName OVNRouterPort, ipv4 bool, ipv6 bool) error { if !ipv4 && !ipv6 { return nil } // Get all the entries for this router port. staticMACBindings := []ovnNB.StaticMACBinding{} err := o.client.WhereCache(func(smb *ovnNB.StaticMACBinding) bool { return smb.LogicalPort == string(portName) }).List(ctx, &staticMACBindings) if err != nil { return err } // Check what needs to be deleted. var operations []ovsdb.Operation for _, binding := range staticMACBindings { ip := net.ParseIP(binding.IP) if ip != nil && ((ip.To4() != nil && ipv4) || (ip.To4() == nil && ipv6)) { op, err := o.client.Where(&ovnNB.StaticMACBinding{UUID: binding.UUID}).Delete() if err != nil { return err } operations = append(operations, op...) } } if len(operations) == 0 { return nil } // Apply the database changes. reply, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(reply, operations) if err != nil { return err } return nil } // CreateLogicalRouterRoute adds a static route to the logical router. func (o *NB) CreateLogicalRouterRoute(ctx context.Context, routerName OVNRouter, mayExist bool, routes ...OVNRouterRoute) error { // Get the logical router. logicalRouter, err := o.GetLogicalRouter(ctx, routerName) if err != nil { return err } // Get the existing routes. existingRoutes := make([]ovnNB.LogicalRouterStaticRoute, 0, len(logicalRouter.StaticRoutes)) for _, uuid := range logicalRouter.StaticRoutes { route := ovnNB.LogicalRouterStaticRoute{ UUID: uuid, } err = o.get(ctx, &route) if err != nil { return err } existingRoutes = append(existingRoutes, route) } // Add the new routes. operations := []ovsdb.Operation{} for i, route := range routes { // Check if already present. exists := false for _, existing := range existingRoutes { if existing.IPPrefix != route.Prefix.String() { continue } if existing.Nexthop == "discard" && !route.Discard { continue } if existing.Nexthop != route.NextHop.String() { continue } if existing.OutputPort == nil { if string(route.Port) != "" { continue } } else if *existing.OutputPort != string(route.Port) { continue } if mayExist { exists = true break } return ErrExists } // Don't add duplicate entries. if exists { continue } // Create the new record. staticRoute := ovnNB.LogicalRouterStaticRoute{ UUID: fmt.Sprintf("route_%d", i), IPPrefix: route.Prefix.String(), } if string(route.Port) != "" { value := string(route.Port) staticRoute.OutputPort = &value } if route.Discard { staticRoute.Nexthop = "discard" } else { staticRoute.Nexthop = route.NextHop.String() } createOps, err := o.client.Create(&staticRoute) if err != nil { return err } operations = append(operations, createOps...) // Add it to the router. updateOps, err := o.client.Where(logicalRouter).Mutate(logicalRouter, ovsModel.Mutation{ Field: &logicalRouter.StaticRoutes, Mutator: ovsdb.MutateOperationInsert, Value: []string{staticRoute.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) } if len(operations) == 0 { return nil } // Apply the database changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // DeleteLogicalRouterRoute deletes a static route from the logical router. func (o *NB) DeleteLogicalRouterRoute(ctx context.Context, routerName OVNRouter, prefixes ...net.IPNet) error { // Get the logical router. logicalRouter, err := o.GetLogicalRouter(ctx, routerName) if err != nil { return err } // Get the existing routes. existingRoutes := make([]ovnNB.LogicalRouterStaticRoute, 0, len(logicalRouter.StaticRoutes)) for _, uuid := range logicalRouter.StaticRoutes { route := ovnNB.LogicalRouterStaticRoute{ UUID: uuid, } err = o.client.Get(ctx, &route) if err != nil { return err } existingRoutes = append(existingRoutes, route) } // Delete the requested routes. operations := []ovsdb.Operation{} for _, prefix := range prefixes { var route ovnNB.LogicalRouterStaticRoute // Look for a matching entry. for _, existing := range existingRoutes { // Normal CIDR entry. if existing.IPPrefix == prefix.String() { route = existing break } // IP-only entry. ones, bits := prefix.Mask.Size() if ones == bits && existing.IPPrefix == prefix.IP.String() { route = existing break } } if route.UUID == "" { continue } // Delete the entry. deleteOps, err := o.client.Where(&route).Delete() if err != nil { return err } operations = append(operations, deleteOps...) // Remove from the router. updateOps, err := o.client.Where(logicalRouter).Mutate(logicalRouter, ovsModel.Mutation{ Field: &logicalRouter.StaticRoutes, Mutator: ovsdb.MutateOperationDelete, Value: []string{route.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) } if len(operations) == 0 { return nil } // Apply the database changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // GetLogicalRouterPort gets the OVN database record for the logical router port. func (o *NB) GetLogicalRouterPort(ctx context.Context, portName OVNRouterPort) (*ovnNB.LogicalRouterPort, error) { logicalRouterPort := &ovnNB.LogicalRouterPort{ Name: string(portName), } err := o.get(ctx, logicalRouterPort) if err != nil { return nil, err } return logicalRouterPort, nil } // CreateLogicalRouterPort adds a named logical router port to a logical router. func (o *NB) CreateLogicalRouterPort(ctx context.Context, routerName OVNRouter, portName OVNRouterPort, mac net.HardwareAddr, gatewayMTU uint32, ipAddr []*net.IPNet, haChassisGroupName OVNChassisGroup, mayExist bool) error { // Prepare the addresses. networks := make([]string, 0, len(ipAddr)) for _, addr := range ipAddr { networks = append(networks, addr.String()) } // Prepare the new router port entry. logicalRouterPort := ovnNB.LogicalRouterPort{ Name: string(portName), UUID: "lrp", } // Check if the entry already exists. err := o.get(ctx, &logicalRouterPort) if err != nil && !errors.Is(err, ErrNotFound) { return err } if logicalRouterPort.UUID != "lrp" && !mayExist { return ErrExists } // Apply the configuration. logicalRouterPort.MAC = mac.String() logicalRouterPort.Networks = networks if haChassisGroupName != "" { haChassisGroup := ovnNB.HAChassisGroup{ Name: string(haChassisGroupName), } err = o.get(ctx, &haChassisGroup) if err != nil { return err } logicalRouterPort.HaChassisGroup = &haChassisGroup.UUID } if gatewayMTU > 0 { if logicalRouterPort.Options == nil { logicalRouterPort.Options = map[string]string{} } logicalRouterPort.Options["gateway_mtu"] = fmt.Sprintf("%d", gatewayMTU) } operations := []ovsdb.Operation{} if logicalRouterPort.UUID != "lrp" { // If it already exists, update it. updateOps, err := o.client.Where(&logicalRouterPort).Update(&logicalRouterPort) if err != nil { return err } operations = append(operations, updateOps...) } else { // Else, create it. createOps, err := o.client.Create(&logicalRouterPort) if err != nil { return err } operations = append(operations, createOps...) // And connect it to the router. logicalRouter := ovnNB.LogicalRouter{ Name: string(routerName), } updateOps, err := o.client.Where(&logicalRouter).Mutate(&logicalRouter, ovsModel.Mutation{ Field: &logicalRouter.Ports, Mutator: ovsdb.MutateOperationInsert, Value: []string{logicalRouterPort.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) } // Apply the database changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // DeleteLogicalRouterPort deletes a named logical router port from a logical router. func (o *NB) DeleteLogicalRouterPort(ctx context.Context, routerName OVNRouter, portName OVNRouterPort) error { operations := []ovsdb.Operation{} // Get the logical router port. logicalRouterPort := ovnNB.LogicalRouterPort{ Name: string(portName), } err := o.get(ctx, &logicalRouterPort) if err != nil { // Logical router port is already gone. if errors.Is(err, ErrNotFound) { return nil } return err } // Remove the port from the router. logicalRouter := ovnNB.LogicalRouter{ Name: string(routerName), } updateOps, err := o.client.Where(&logicalRouter).Mutate(&logicalRouter, ovsModel.Mutation{ Field: &logicalRouter.Ports, Mutator: ovsdb.MutateOperationDelete, Value: []string{logicalRouterPort.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) // Delete the port itself. deleteOps, err := o.client.Where(&logicalRouterPort).Delete() if err != nil { return err } operations = append(operations, deleteOps...) // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // UpdateLogicalRouterPort updates properties of a logical router port. func (o *NB) UpdateLogicalRouterPort(ctx context.Context, portName OVNRouterPort, ipv6ra *OVNIPv6RAOpts) error { lrp, err := o.GetLogicalRouterPort(ctx, portName) if err != nil { return err } if ipv6ra != nil { ipv6conf := map[string]string{ "send_periodic": fmt.Sprintf("%t", ipv6ra.SendPeriodic), } if ipv6ra.AddressMode != "" { ipv6conf["address_mode"] = string(ipv6ra.AddressMode) } if ipv6ra.MaxInterval > 0 { ipv6conf["max_interval"] = fmt.Sprintf("%d", ipv6ra.MaxInterval/time.Second) } if ipv6ra.MinInterval > 0 { ipv6conf["min_interval"] = fmt.Sprintf("%d", ipv6ra.MinInterval/time.Second) } if ipv6ra.MTU > 0 { ipv6conf["mtu"] = fmt.Sprintf("%d", ipv6ra.MTU) } if len(ipv6ra.DNSSearchList) > 0 { ipv6conf["dnssl"] = strings.Join(ipv6ra.DNSSearchList, ",") } if ipv6ra.RecursiveDNSServer != nil { ipv6conf["rdnss"] = ipv6ra.RecursiveDNSServer.String() } lrp.Ipv6RaConfigs = ipv6conf } // Update the record. operations, err := o.client.Where(lrp).Update(lrp) if err != nil { return err } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // GetLogicalSwitch gets the OVN database record for the switch. func (o *NB) GetLogicalSwitch(ctx context.Context, switchName OVNSwitch) (*ovnNB.LogicalSwitch, error) { logicalSwitch := &ovnNB.LogicalSwitch{ Name: string(switchName), } err := o.get(ctx, logicalSwitch) if err != nil { return nil, err } return logicalSwitch, nil } // CreateLogicalSwitch adds a named logical switch. // If mayExist is true, then an existing resource of the same name is not treated as an error. func (o *NB) CreateLogicalSwitch(ctx context.Context, switchName OVNSwitch, mayExist bool) error { logicalSwitch := ovnNB.LogicalSwitch{ Name: string(switchName), } // Check if already exists. err := o.get(ctx, &logicalSwitch) if err != nil && !errors.Is(err, ErrNotFound) { return err } if logicalSwitch.UUID != "" { if mayExist { return nil } return ErrExists } // Create the record. operations, err := o.client.Create(&logicalSwitch) if err != nil { return err } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // DeleteLogicalSwitch deletes a named logical switch. func (o *NB) DeleteLogicalSwitch(ctx context.Context, switchName OVNSwitch) error { ls := ovnNB.LogicalSwitch{ Name: string(switchName), } // Check if the switch exists. err := o.get(ctx, &ls) if err != nil { return err } // Delete the switch itself. operations := []ovsdb.Operation{} deleteOps, err := o.client.Where(&ls).Delete() if err != nil { return err } operations = append(operations, deleteOps...) // Delete all associated port groups. portGroups := []ovnNB.PortGroup{} err = o.client.WhereCache(func(pg *ovnNB.PortGroup) bool { return pg.ExternalIDs != nil && pg.ExternalIDs[ovnExtIDIncusSwitch] == string(switchName) }).List(ctx, &portGroups) if err != nil { return err } for _, pg := range portGroups { deleteOps, err := o.client.Where(&pg).Delete() if err != nil { return err } operations = append(operations, deleteOps...) } // Delete all associated DHCP options. dhcpOptions := []ovnNB.DHCPOptions{} err = o.client.WhereCache(func(do *ovnNB.DHCPOptions) bool { return do.ExternalIDs != nil && do.ExternalIDs[ovnExtIDIncusSwitch] == string(switchName) }).List(ctx, &dhcpOptions) if err != nil { return err } for _, do := range dhcpOptions { deleteOps, err := o.client.Where(&do).Delete() if err != nil { return err } operations = append(operations, deleteOps...) } // Delete all associated DNS records. dnsRecords := []ovnNB.DNS{} err = o.client.WhereCache(func(dr *ovnNB.DNS) bool { return dr.ExternalIDs != nil && dr.ExternalIDs[ovnExtIDIncusSwitch] == string(switchName) }).List(ctx, &dnsRecords) if err != nil { return err } for _, dr := range dnsRecords { deleteOps, err := o.client.Where(&dr).Delete() if err != nil { return err } operations = append(operations, deleteOps...) } // Apply the database changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // logicalSwitchParseExcludeIPs parses the ips into OVN exclude_ips format. func (o *NB) logicalSwitchParseExcludeIPs(ips []iprange.Range) ([]string, error) { excludeIPs := make([]string, 0, len(ips)) for _, v := range ips { if v.Start == nil || v.Start.To4() == nil { return nil, errors.New("Invalid exclude IPv4 range start address") } else if v.End == nil { excludeIPs = append(excludeIPs, v.Start.String()) } else { if v.End != nil && v.End.To4() == nil { return nil, errors.New("Invalid exclude IPv4 range end address") } excludeIPs = append(excludeIPs, fmt.Sprintf("%s..%s", v.Start.String(), v.End.String())) } } return excludeIPs, nil } // UpdateLogicalSwitchIPAllocation sets the IP allocation config on the logical switch. func (o *NB) UpdateLogicalSwitchIPAllocation(ctx context.Context, switchName OVNSwitch, opts *OVNIPAllocationOpts) error { // Get the logical switch. logicalSwitch, err := o.GetLogicalSwitch(ctx, switchName) if err != nil { return err } // Update the configuration. if logicalSwitch.OtherConfig == nil { logicalSwitch.OtherConfig = map[string]string{} } if opts.PrefixIPv4 != nil { logicalSwitch.OtherConfig["subnet"] = opts.PrefixIPv4.String() } else { delete(logicalSwitch.OtherConfig, "subnet") } if opts.PrefixIPv6 != nil { logicalSwitch.OtherConfig["ipv6_prefix"] = opts.PrefixIPv6.String() } else { delete(logicalSwitch.OtherConfig, "ipv6_prefix") } if len(opts.ExcludeIPv4) > 0 { excludeIPs, err := o.logicalSwitchParseExcludeIPs(opts.ExcludeIPv4) if err != nil { return err } logicalSwitch.OtherConfig["exclude_ips"] = strings.Join(excludeIPs, " ") } else { delete(logicalSwitch.OtherConfig, "exclude_ips") } operations, err := o.client.Where(logicalSwitch).Update(logicalSwitch) if err != nil { return err } // Apply the database changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // UpdateLogicalSwitchDHCPv4Revervations sets the DHCPv4 IP reservations. func (o *NB) UpdateLogicalSwitchDHCPv4Revervations(ctx context.Context, switchName OVNSwitch, reservedIPs []iprange.Range) error { // Get the logical switch. logicalSwitch, err := o.GetLogicalSwitch(ctx, switchName) if err != nil { return err } // Update the configuration. if logicalSwitch.OtherConfig == nil { logicalSwitch.OtherConfig = map[string]string{} } if len(reservedIPs) > 0 { excludeIPs, err := o.logicalSwitchParseExcludeIPs(reservedIPs) if err != nil { return err } logicalSwitch.OtherConfig["exclude_ips"] = strings.Join(excludeIPs, " ") } else { delete(logicalSwitch.OtherConfig, "exclude_ips") } operations, err := o.client.Where(logicalSwitch).Update(logicalSwitch) if err != nil { return err } // Apply the database changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // GetLogicalSwitchDHCPv4Revervations gets the DHCPv4 IP reservations. func (o *NB) GetLogicalSwitchDHCPv4Revervations(ctx context.Context, switchName OVNSwitch) ([]iprange.Range, error) { // Get the logical switch. logicalSwitch, err := o.GetLogicalSwitch(ctx, switchName) if err != nil { return nil, err } // Get the list of excluded IPs. if logicalSwitch.OtherConfig == nil { return []iprange.Range{}, nil } // Check if no dynamic IPs set. excludeIPsRaw := strings.TrimSpace(logicalSwitch.OtherConfig["exclude_ips"]) if excludeIPsRaw == "" || excludeIPsRaw == "[]" { return []iprange.Range{}, nil } excludeIPsRaw, err = unquote(excludeIPsRaw) if err != nil { return nil, fmt.Errorf("Failed unquoting exclude_ips: %w", err) } excludeIPsParts := util.SplitNTrimSpace(strings.TrimSpace(excludeIPsRaw), " ", -1, true) excludeIPs := make([]iprange.Range, 0, len(excludeIPsParts)) for _, excludeIPsPart := range excludeIPsParts { ip := net.ParseIP(excludeIPsPart) // Check if single IP part. if ip == nil { // Check if IP range part. start, end, found := strings.Cut(excludeIPsPart, "..") if !found { return nil, fmt.Errorf("Unrecognised exclude_ips part: %q", excludeIPsPart) } startIP := net.ParseIP(start) endIP := net.ParseIP(end) if startIP == nil || endIP == nil { return nil, fmt.Errorf("Invalid exclude_ips range: %q", excludeIPsPart) } // Add range IP part to list. excludeIPs = append(excludeIPs, iprange.Range{Start: startIP, End: endIP}) } else { // Add single IP part to list. excludeIPs = append(excludeIPs, iprange.Range{Start: ip}) } } return excludeIPs, nil } // UpdateLogicalSwitchDHCPv4Options creates or updates a DHCPv4 option set associated with the specified switchName // and subnet. If uuid is non-empty then the record that exists with that ID is updated, otherwise a new record // is created. func (o *NB) UpdateLogicalSwitchDHCPv4Options(ctx context.Context, switchName OVNSwitch, uuid OVNDHCPOptionsUUID, subnet *net.IPNet, opts *OVNDHCPv4Opts) error { dhcpOption := ovnNB.DHCPOptions{} if uuid != "" { // Load the existing record. dhcpOption.UUID = string(uuid) err := o.get(ctx, &dhcpOption) if err != nil { return err } } if dhcpOption.ExternalIDs == nil { dhcpOption.ExternalIDs = map[string]string{} } if dhcpOption.Options == nil { dhcpOption.Options = map[string]string{} } dhcpOption.ExternalIDs[ovnExtIDIncusSwitch] = string(switchName) dhcpOption.Cidr = subnet.String() dhcpOption.Options["server_id"] = opts.ServerID.String() dhcpOption.Options["server_mac"] = opts.ServerMAC.String() dhcpOption.Options["lease_time"] = fmt.Sprintf("%d", opts.LeaseTime/time.Second) if opts.Router != nil { dhcpOption.Options["router"] = opts.Router.String() } else { delete(dhcpOption.Options, "router") } if len(opts.DNSSearchList) > 0 { // Special quoting to allow domain names. dhcpOption.Options["domain_search_list"] = fmt.Sprintf(`"%s"`, strings.Join(opts.DNSSearchList, ",")) } else { delete(dhcpOption.Options, "domain_search_list") } if len(opts.RecursiveDNSServer) > 0 { nsIPs := make([]string, 0, len(opts.RecursiveDNSServer)) for _, nsIP := range opts.RecursiveDNSServer { if nsIP.To4() == nil { continue // Only include IPv4 addresses. } nsIPs = append(nsIPs, nsIP.String()) } dhcpOption.Options["dns_server"] = fmt.Sprintf("{%s}", strings.Join(nsIPs, ",")) } else { delete(dhcpOption.Options, "dns_server") } if opts.DomainName != "" { // Special quoting to allow domain names. dhcpOption.Options["domain_name"] = fmt.Sprintf(`"%s"`, opts.DomainName) } else { delete(dhcpOption.Options, "domain_name") } if opts.MTU > 0 { dhcpOption.Options["mtu"] = fmt.Sprintf("%d", opts.MTU) } else { delete(dhcpOption.Options, "mtu") } if opts.Netmask != "" { dhcpOption.Options["netmask"] = opts.Netmask } else { delete(dhcpOption.Options, "netmask") } if opts.StaticRoutes != "" { dhcpOption.Options["classless_static_route"] = fmt.Sprintf("{%s}", opts.StaticRoutes) } else { delete(dhcpOption.Options, "classless_static_route") } // Prepare the changes. operations := []ovsdb.Operation{} if dhcpOption.UUID == "" { // Create a new record. createOps, err := o.client.Create(&dhcpOption) if err != nil { return err } operations = append(operations, createOps...) } else { // Update the record. updateOps, err := o.client.Where(&dhcpOption).Update(&dhcpOption) if err != nil { return err } operations = append(operations, updateOps...) } // Apply the database changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // UpdateLogicalSwitchDHCPv6Options creates or updates a DHCPv6 option set associated with the specified switchName // and subnet. If uuid is non-empty then the record that exists with that ID is updated, otherwise a new record // is created. func (o *NB) UpdateLogicalSwitchDHCPv6Options(ctx context.Context, switchName OVNSwitch, uuid OVNDHCPOptionsUUID, subnet *net.IPNet, opts *OVNDHCPv6Opts) error { dhcpOption := ovnNB.DHCPOptions{} if uuid != "" { // Load the existing record. dhcpOption.UUID = string(uuid) err := o.get(ctx, &dhcpOption) if err != nil { return err } } if dhcpOption.ExternalIDs == nil { dhcpOption.ExternalIDs = map[string]string{} } if dhcpOption.Options == nil { dhcpOption.Options = map[string]string{} } dhcpOption.ExternalIDs[ovnExtIDIncusSwitch] = string(switchName) dhcpOption.Cidr = subnet.String() dhcpOption.Options["server_id"] = opts.ServerID.String() if opts.DHCPv6Stateless { dhcpOption.Options["dhcpv6_stateless"] = "true" } else { dhcpOption.Options["dhcpv6_stateless"] = "false" } if len(opts.DNSSearchList) > 0 { // Special quoting to allow domain names. dhcpOption.Options["domain_search"] = fmt.Sprintf(`"%s"`, strings.Join(opts.DNSSearchList, ",")) } else { delete(dhcpOption.Options, "domain_search") } if opts.RecursiveDNSServer != nil { nsIPs := make([]string, 0, len(opts.RecursiveDNSServer)) for _, nsIP := range opts.RecursiveDNSServer { if nsIP.To4() != nil { continue // Only include IPv6 addresses. } nsIPs = append(nsIPs, nsIP.String()) } dhcpOption.Options["dns_server"] = fmt.Sprintf("{%s}", strings.Join(nsIPs, ",")) } else { delete(dhcpOption.Options, "dns_server") } // Prepare the changes. operations := []ovsdb.Operation{} if dhcpOption.UUID == "" { // Create a new record. createOps, err := o.client.Create(&dhcpOption) if err != nil { return err } operations = append(operations, createOps...) } else { // Update the record. updateOps, err := o.client.Where(&dhcpOption).Update(&dhcpOption) if err != nil { return err } operations = append(operations, updateOps...) } // Apply the database changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // GetLogicalSwitchDHCPOptions retrieves the existing DHCP options defined for a logical switch. func (o *NB) GetLogicalSwitchDHCPOptions(ctx context.Context, switchName OVNSwitch) ([]OVNDHCPOptsSet, error) { // Get the matching DHCP options. dhcpOptions := []ovnNB.DHCPOptions{} err := o.client.WhereCache(func(do *ovnNB.DHCPOptions) bool { return do.ExternalIDs != nil && do.ExternalIDs[ovnExtIDIncusSwitch] == string(switchName) }).List(ctx, &dhcpOptions) if err != nil { return nil, err } dhcpOpts := []OVNDHCPOptsSet{} for _, dhcpOption := range dhcpOptions { _, cidr, err := net.ParseCIDR(dhcpOption.Cidr) if err != nil { return nil, err } dhcpOpts = append(dhcpOpts, OVNDHCPOptsSet{ UUID: OVNDHCPOptionsUUID(dhcpOption.UUID), CIDR: cidr, }) } return dhcpOpts, nil } // DeleteLogicalSwitchDHCPOption deletes the specified DHCP options defined for a switch. func (o *NB) DeleteLogicalSwitchDHCPOption(ctx context.Context, switchName OVNSwitch, uuids ...OVNDHCPOptionsUUID) error { operations := []ovsdb.Operation{} // Prepare deletion requests. for _, uuid := range uuids { dhcpOption := ovnNB.DHCPOptions{ UUID: string(uuid), } deleteOps, err := o.client.Where(&dhcpOption).Delete() if err != nil { return err } operations = append(operations, deleteOps...) } // Check if there's anything to do. if len(operations) == 0 { return nil } // Apply the database changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // UpdateLogicalSwitchACLRules applies a set of rules to the specified logical switch. Any existing rules are removed. func (o *NB) UpdateLogicalSwitchACLRules(ctx context.Context, switchName OVNSwitch, aclRules ...OVNACLRule) error { operations := []ovsdb.Operation{} // Get the logical switch. ls, err := o.GetLogicalSwitch(ctx, switchName) if err != nil { return err } // Remove any existing rules assigned to the entity. for _, aclUUID := range ls.ACLs { updateOps, err := o.client.Where(ls).Mutate(ls, ovsModel.Mutation{ Field: &ls.ACLs, Mutator: ovsdb.MutateOperationDelete, Value: []string{aclUUID}, }) if err != nil { return err } operations = append(operations, updateOps...) } // Add new rules. externalIDs := map[string]string{ ovnExtIDIncusSwitch: string(switchName), } createOps, err := o.aclRuleAddOperations(ctx, "logical_switch", string(switchName), externalIDs, nil, aclRules...) if err != nil { return err } operations = append(operations, createOps...) // Apply the database changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // logicalSwitchPortQoSRules returns the QoS rule UUIDs belonging to a logical switch port. func (o *NB) logicalSwitchPortQoSRules(ctx context.Context, portName OVNSwitchPort) ([]string, error) { var qosRules []ovnNB.QoS err := o.client.WhereCache(func(qosRule *ovnNB.QoS) bool { return qosRule.ExternalIDs != nil && qosRule.ExternalIDs[ovnExtIDIncusSwitchPort] == string(portName) }).List(ctx, &qosRules) if err != nil { return nil, err } var ruleUUIDs []string for _, qosRule := range qosRules { ruleUUIDs = append(ruleUUIDs, qosRule.UUID) } return ruleUUIDs, nil } // logicalSwitchPortACLRules returns the ACL rule UUIDs belonging to a logical switch port. func (o *NB) logicalSwitchPortACLRules(ctx context.Context, portName OVNSwitchPort) ([]string, error) { acls := []ovnNB.ACL{} err := o.client.WhereCache(func(acl *ovnNB.ACL) bool { return acl.ExternalIDs != nil && acl.ExternalIDs[ovnExtIDIncusSwitchPort] == string(portName) }).List(ctx, &acls) if err != nil { return nil, err } ruleUUIDs := []string{} for _, acl := range acls { ruleUUIDs = append(ruleUUIDs, acl.UUID) } return ruleUUIDs, nil } // GetLogicalSwitchPorts returns a map of logical switch ports (name and UUID) for a switch. // Includes non-instance ports, such as the router port. func (o *NB) GetLogicalSwitchPorts(ctx context.Context, switchName OVNSwitch) (map[OVNSwitchPort]OVNSwitchPortUUID, error) { // Get the logical switch. logicalSwitch, err := o.GetLogicalSwitch(ctx, switchName) if err != nil { return nil, err } ports := make(map[OVNSwitchPort]OVNSwitchPortUUID, len(logicalSwitch.Ports)) for _, portUUID := range logicalSwitch.Ports { // Get the logical switch port. lsp := ovnNB.LogicalSwitchPort{ UUID: portUUID, } err := o.get(ctx, &lsp) if err != nil { return nil, err } ports[OVNSwitchPort(lsp.Name)] = OVNSwitchPortUUID(lsp.UUID) } return ports, nil } // GetLogicalSwitchIPs returns a list of IPs associated to each port connected to switch. func (o *NB) GetLogicalSwitchIPs(ctx context.Context, switchName OVNSwitch) (map[OVNSwitchPort][]net.IP, error) { lsps := []ovnNB.LogicalSwitchPort{} err := o.client.WhereCache(func(lsp *ovnNB.LogicalSwitchPort) bool { return lsp.ExternalIDs != nil && lsp.ExternalIDs[ovnExtIDIncusSwitch] == string(switchName) }).List(ctx, &lsps) if err != nil { return nil, err } portIPs := make(map[OVNSwitchPort][]net.IP, len(lsps)) for _, lsp := range lsps { var ips []net.IP // Extract all addresses from the Addresses field. for _, address := range lsp.Addresses { for _, entry := range util.SplitNTrimSpace(address, " ", -1, true) { ip := net.ParseIP(entry) if ip != nil { ips = append(ips, ip) } } } // Extract all addresses from the DynamicAddresses field. if lsp.DynamicAddresses != nil { for _, entry := range util.SplitNTrimSpace(*lsp.DynamicAddresses, " ", -1, true) { ip := net.ParseIP(entry) if ip != nil { ips = append(ips, ip) } } } portIPs[OVNSwitchPort(lsp.Name)] = ips } return portIPs, nil } // GetLogicalSwitchPortUUID returns the logical switch port UUID. func (o *NB) GetLogicalSwitchPortUUID(ctx context.Context, portName OVNSwitchPort) (OVNSwitchPortUUID, error) { // Get the logical switch port. lsp := ovnNB.LogicalSwitchPort{ Name: string(portName), } err := o.get(ctx, &lsp) if err != nil { return "", err } return OVNSwitchPortUUID(lsp.UUID), nil } // CreateLogicalSwitchPort adds a named logical switch port to a logical switch, and sets options if provided. // If mayExist is true, then an existing resource of the same name is not treated as an error. func (o *NB) CreateLogicalSwitchPort(ctx context.Context, switchName OVNSwitch, portName OVNSwitchPort, opts *OVNSwitchPortOpts, mayExist bool) error { // Prepare the new switch port entry. logicalSwitchPort := ovnNB.LogicalSwitchPort{ Name: string(portName), UUID: "lsp", } // Check if the entry already exists. err := o.get(ctx, &logicalSwitchPort) if err != nil && !errors.Is(err, ErrNotFound) { return err } if logicalSwitchPort.UUID != "lsp" && !mayExist { return ErrExists } if logicalSwitchPort.ExternalIDs == nil { logicalSwitchPort.ExternalIDs = map[string]string{} } // Set switch port options if supplied. if opts != nil { // Created nested VLAN port if requested. if opts.Parent != "" { parentName := string(opts.Parent) tag := int(opts.VLAN) logicalSwitchPort.ParentName = &parentName logicalSwitchPort.Tag = &tag } if opts.RouterPort != "" { logicalSwitchPort.Type = "router" logicalSwitchPort.Addresses = []string{"router"} logicalSwitchPort.Options = map[string]string{"router-port": string(opts.RouterPort)} } else { if (opts.IPV4 == "none" && opts.IPV6 != "none") || (opts.IPV6 == "none" && opts.IPV4 != "none") { return errors.New("OVN doesn't support disabling IP allocation on only one protocol") } addresses := []string{} if opts.IPV4 != "none" && opts.IPV6 != "none" { address := []string{} if opts.MAC != nil { address = append(address, opts.MAC.String()) } if opts.IPV4 == "" && opts.IPV6 == "" { address = append(address, "dynamic") } else { if opts.IPV4 != "" { address = append(address, opts.IPV4) } if opts.IPV6 != "" { address = append(address, opts.IPV6) } } addresses = append(addresses, strings.Join(address, " ")) } if opts.DHCPv4OptsID != "" { dhcp4opts := string(opts.DHCPv4OptsID) logicalSwitchPort.Dhcpv4Options = &dhcp4opts } if opts.DHCPv6OptsID != "" { dhcp6opts := string(opts.DHCPv6OptsID) logicalSwitchPort.Dhcpv6Options = &dhcp6opts } if opts.Promiscuous { addresses = append(addresses, "unknown") } logicalSwitchPort.Addresses = addresses } if opts.Location != "" { logicalSwitchPort.ExternalIDs[ovnExtIDIncusLocation] = opts.Location } } logicalSwitchPort.ExternalIDs[ovnExtIDIncusSwitch] = string(switchName) // Apply the changes. operations := []ovsdb.Operation{} if logicalSwitchPort.UUID != "lsp" { // If it already exists, update it. updateOps, err := o.client.Where(&logicalSwitchPort).Update(&logicalSwitchPort) if err != nil { return err } operations = append(operations, updateOps...) } else { // Else, create it. createOps, err := o.client.Create(&logicalSwitchPort) if err != nil { return err } operations = append(operations, createOps...) // And connect it to the switch. logicalSwitch := ovnNB.LogicalSwitch{ Name: string(switchName), } updateOps, err := o.client.Where(&logicalSwitch).Mutate(&logicalSwitch, ovsModel.Mutation{ Field: &logicalSwitch.Ports, Mutator: ovsdb.MutateOperationInsert, Value: []string{logicalSwitchPort.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) } // Apply the database changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // GetLogicalSwitchPortIPs returns a list of IPs for a switch port. func (o *NB) GetLogicalSwitchPortIPs(ctx context.Context, portName OVNSwitchPort) ([]net.IP, error) { lsp := ovnNB.LogicalSwitchPort{ Name: string(portName), } err := o.get(ctx, &lsp) if err != nil { return nil, err } addresses := []net.IP{} for _, address := range lsp.Addresses { for _, entry := range strings.Split(address, " ") { ip := net.ParseIP(entry) if ip != nil { addresses = append(addresses, ip) } } } if lsp.DynamicAddresses != nil { for _, entry := range strings.Split(*lsp.DynamicAddresses, " ") { ip := net.ParseIP(entry) if ip != nil { addresses = append(addresses, ip) } } } return addresses, nil } // GetLogicalSwitchPortDynamicIPs returns a list of dynamic IPs for a switch port. func (o *NB) GetLogicalSwitchPortDynamicIPs(ctx context.Context, portName OVNSwitchPort) ([]net.IP, error) { lsp := &ovnNB.LogicalSwitchPort{ Name: string(portName), } err := o.get(ctx, lsp) if err != nil { return []net.IP{}, err } // Check if no dynamic IPs set. if lsp.DynamicAddresses == nil { return []net.IP{}, nil } dynamicAddresses := strings.Split(*lsp.DynamicAddresses, " ") dynamicIPs := make([]net.IP, 0, len(dynamicAddresses)) for _, dynamicAddress := range dynamicAddresses { ip := net.ParseIP(dynamicAddress) if ip != nil { dynamicIPs = append(dynamicIPs, ip) } } return dynamicIPs, nil } // GetLogicalSwitchPortLocation returns the last set location of a logical switch port. func (o *NB) GetLogicalSwitchPortLocation(ctx context.Context, portName OVNSwitchPort) (string, error) { lsp := ovnNB.LogicalSwitchPort{ Name: string(portName), } err := o.get(ctx, &lsp) if err != nil { return "", err } if lsp.ExternalIDs == nil { return "", ErrNotFound } val, ok := lsp.ExternalIDs[ovnExtIDIncusLocation] if !ok { return "", ErrNotFound } return val, nil } // UpdateLogicalSwitchPortDHCP updates the DHCP options on the logical switch port. func (o *NB) UpdateLogicalSwitchPortDHCP(ctx context.Context, portName OVNSwitchPort, dhcpV4UUID OVNDHCPOptionsUUID, dhcpV6UUID OVNDHCPOptionsUUID) error { // Get the logical switch port. lsp := ovnNB.LogicalSwitchPort{ Name: string(portName), } err := o.get(ctx, &lsp) if err != nil { return err } if dhcpV4UUID != "" { dhcp4opts := string(dhcpV4UUID) lsp.Dhcpv4Options = &dhcp4opts } if dhcpV6UUID != "" { dhcp6opts := string(dhcpV6UUID) lsp.Dhcpv6Options = &dhcp6opts } // Update the record. operations, err := o.client.Where(&lsp).Update(&lsp) if err != nil { return err } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // UpdateLogicalSwitchPortOptions sets the options for a logical switch port. func (o *NB) UpdateLogicalSwitchPortOptions(ctx context.Context, portName OVNSwitchPort, options map[string]string) error { // Get the logical switch port. lsp := ovnNB.LogicalSwitchPort{ Name: string(portName), } err := o.get(ctx, &lsp) if err != nil { return err } // Apply the changes. if lsp.Options == nil { lsp.Options = map[string]string{} } maps.Copy(lsp.Options, options) // Update the record. operations, err := o.client.Where(&lsp).Update(&lsp) if err != nil { return err } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // UpdateLogicalSwitchPortDNS sets up the switch port DNS records for the DNS name. // Returns the DNS record UUID, IPv4 and IPv6 addresses used for DNS records. func (o *NB) UpdateLogicalSwitchPortDNS(ctx context.Context, switchName OVNSwitch, portName OVNSwitchPort, dnsName string, dnsIPs []net.IP) (OVNDNSUUID, error) { // Get the logical switch. ls, err := o.GetLogicalSwitch(ctx, switchName) if err != nil { return "", err } // Check if existing DNS record exists for switch port. dnsRecords := []ovnNB.DNS{} err = o.client.WhereCache(func(dnsRecord *ovnNB.DNS) bool { return dnsRecord.ExternalIDs != nil && dnsRecord.ExternalIDs[ovnExtIDIncusSwitchPort] == string(portName) }).List(ctx, &dnsRecords) if err != nil { return "", err } var dnsRecord ovnNB.DNS if len(dnsRecords) == 1 { dnsRecord = dnsRecords[0] } else if len(dnsRecords) == 0 { dnsRecord = ovnNB.DNS{} } else { return "", errors.New("Found more than one matching DNS record") } // Make sure the external IDs are set. if dnsRecord.ExternalIDs == nil { dnsRecord.ExternalIDs = map[string]string{} } dnsRecord.ExternalIDs[ovnExtIDIncusSwitch] = string(switchName) dnsRecord.ExternalIDs[ovnExtIDIncusSwitchPort] = string(portName) // Add the records. if dnsRecord.Records == nil { dnsRecord.Records = map[string]string{} } // Only include DNS name record if IPs supplied. if len(dnsIPs) > 0 { var dnsIPsStr strings.Builder for i, dnsIP := range dnsIPs { if i > 0 { dnsIPsStr.WriteString(" ") } dnsIPsStr.WriteString(dnsIP.String()) dnsRecord.Records[strings.TrimSuffix(localUtil.ReverseDNS(dnsIP), ".")] = strings.ToLower(dnsName) } dnsRecord.Records[strings.ToLower(dnsName)] = dnsIPsStr.String() } else { // Clear any existing DNS name if no IPs supplied. dnsRecord.Records = map[string]string{} } operations := []ovsdb.Operation{} if dnsRecord.UUID == "" { // Create a new record. dnsRecord.UUID = "record" createOps, err := o.client.Create(&dnsRecord) if err != nil { return "", err } operations = append(operations, createOps...) // Add it to the logical switch. updateOps, err := o.client.Where(ls).Mutate(ls, ovsModel.Mutation{ Field: &ls.DNSRecords, Mutator: ovsdb.MutateOperationInsert, Value: []string{dnsRecord.UUID}, }) if err != nil { return "", err } operations = append(operations, updateOps...) } else { // Update the record. updateOps, err := o.client.Where(&dnsRecord).Update(&dnsRecord) if err != nil { return "", err } operations = append(operations, updateOps...) } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return "", err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return "", err } if dnsRecord.UUID == "record" { dnsRecord.UUID = resp[0].UUID.GoUUID } return OVNDNSUUID(dnsRecord.UUID), nil } // GetLogicalSwitchPortDNS returns the logical switch port DNS info (UUID, name and IPs). func (o *NB) GetLogicalSwitchPortDNS(ctx context.Context, portName OVNSwitchPort) (OVNDNSUUID, string, []net.IP, error) { dnsRecords := []ovnNB.DNS{} err := o.client.WhereCache(func(dnsRecord *ovnNB.DNS) bool { return dnsRecord.ExternalIDs != nil && dnsRecord.ExternalIDs[ovnExtIDIncusSwitchPort] == string(portName) }).List(ctx, &dnsRecords) if err != nil { return "", "", nil, err } if len(dnsRecords) != 1 { return "", "", nil, nil } var ips []net.IP var dnsName string for key, value := range dnsRecords[0].Records { dnsName = key for _, ipPart := range strings.Split(value, " ") { ip := net.ParseIP(strings.TrimSpace(ipPart)) if ip != nil { ips = append(ips, ip) } } } return OVNDNSUUID(dnsRecords[0].UUID), dnsName, ips, nil } // logicalSwitchPortDeleteDNSOperations returns a list of ovsdb operations to remove DNS records from a switch port. // If destroyEntry the DNS entry record itself is also removed, otherwise it is just cleared but left in place. func (o *NB) logicalSwitchPortDeleteDNSOperations(ctx context.Context, switchName OVNSwitch, dnsUUID OVNDNSUUID, destroyEntry bool) ([]ovsdb.Operation, error) { operations := []ovsdb.Operation{} // Get the DNS entry. dnsEntry := ovnNB.DNS{ UUID: string(dnsUUID), } err := o.get(ctx, &dnsEntry) if err != nil { return nil, err } // Get the logical switch. ls, err := o.GetLogicalSwitch(ctx, switchName) if err != nil { return nil, err } // Remove from the logical switch. updateOps, err := o.client.Where(ls).Mutate(ls, ovsModel.Mutation{ Field: &ls.DNSRecords, Mutator: ovsdb.MutateOperationDelete, Value: []string{dnsEntry.UUID}, }) if err != nil { return nil, err } operations = append(operations, updateOps...) if destroyEntry { deleteOps, err := o.client.Where(&dnsEntry).Delete() if err != nil { return nil, err } operations = append(operations, deleteOps...) } else { dnsEntry.Records = nil updateOps, err := o.client.Where(&dnsEntry).Update(&dnsEntry) if err != nil { return nil, err } operations = append(operations, updateOps...) } return operations, nil } // DeleteLogicalSwitchPortDNS removes DNS records from a switch port. // If destroyEntry the DNS entry record itself is also removed, otherwise it is just cleared but left in place. func (o *NB) DeleteLogicalSwitchPortDNS(ctx context.Context, switchName OVNSwitch, dnsUUID OVNDNSUUID, destroyEntry bool) error { // Remove DNS record association from switch, and remove DNS record entry itself. operations, err := o.logicalSwitchPortDeleteDNSOperations(ctx, switchName, dnsUUID, destroyEntry) if err != nil { return err } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // logicalSwitchPortDeleteAppendArgs adds the commands to delete the specified logical switch port. func (o *NB) logicalSwitchPortDeleteOperations(ctx context.Context, switchName OVNSwitch, portName OVNSwitchPort) ([]ovsdb.Operation, error) { operations := []ovsdb.Operation{} // Get the logical switch port. logicalSwitchPort := ovnNB.LogicalSwitchPort{ Name: string(portName), } err := o.get(ctx, &logicalSwitchPort) if err != nil { // Logical switch port is already gone. if errors.Is(err, ErrNotFound) { return nil, nil } return nil, err } // Remove the port from the switch. logicalSwitch := ovnNB.LogicalSwitch{ Name: string(switchName), } updateOps, err := o.client.Where(&logicalSwitch).Mutate(&logicalSwitch, ovsModel.Mutation{ Field: &logicalSwitch.Ports, Mutator: ovsdb.MutateOperationDelete, Value: []string{logicalSwitchPort.UUID}, }) if err != nil { return nil, err } operations = append(operations, updateOps...) // Delete the port itself. deleteOps, err := o.client.Where(&logicalSwitchPort).Delete() if err != nil { return nil, err } operations = append(operations, deleteOps...) return operations, nil } // DeleteLogicalSwitchPort deletes a named logical switch port. func (o *NB) DeleteLogicalSwitchPort(ctx context.Context, switchName OVNSwitch, portName OVNSwitchPort) error { // Get the delete operations. operations, err := o.logicalSwitchPortDeleteOperations(ctx, switchName, portName) if err != nil { return err } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // CleanupLogicalSwitchPort deletes the named logical switch port and its associated config. func (o *NB) CleanupLogicalSwitchPort(ctx context.Context, portName OVNSwitchPort, switchName OVNSwitch, switchPortGroupName OVNPortGroup, dnsUUID OVNDNSUUID) error { operations := []ovsdb.Operation{} // Remove any existing rules assigned to the entity. removeACLRuleUUIDs, err := o.logicalSwitchPortACLRules(ctx, portName) if err != nil { return err } deleteOps, err := o.aclRuleDeleteOperations(ctx, "port_group", string(switchPortGroupName), removeACLRuleUUIDs) if err != nil { return err } operations = append(operations, deleteOps...) removeQoSRuleUUIDs, err := o.logicalSwitchPortQoSRules(ctx, portName) if err != nil { return err } deleteOps, err = o.qosRuleDeleteOperations(ctx, "logical_switch", string(switchName), removeQoSRuleUUIDs) if err != nil { return err } operations = append(operations, deleteOps...) // Remove logical switch port. deleteOps, err = o.logicalSwitchPortDeleteOperations(ctx, switchName, portName) if err != nil { return err } operations = append(operations, deleteOps...) // Remove DNS records. if dnsUUID != "" { deleteOps, err := o.logicalSwitchPortDeleteDNSOperations(ctx, switchName, dnsUUID, true) if err != nil { return err } operations = append(operations, deleteOps...) } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // UpdateLogicalSwitchPortLinkRouter links a logical switch port to a logical router port. func (o *NB) UpdateLogicalSwitchPortLinkRouter(ctx context.Context, switchPortName OVNSwitchPort, routerPortName OVNRouterPort) error { // Get the logical switch port. lsp := ovnNB.LogicalSwitchPort{ Name: string(switchPortName), } err := o.get(ctx, &lsp) if err != nil { return err } // Update the fields. lsp.Type = "router" lsp.Addresses = []string{"router"} if lsp.Options == nil { lsp.Options = map[string]string{} } lsp.Options["nat-addresses"] = "router" lsp.Options["router-port"] = string(routerPortName) // Update the record. operations, err := o.client.Where(&lsp).Update(&lsp) if err != nil { return err } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // UpdateLogicalSwitchPortLinkProviderNetwork links a logical switch port to a provider network. func (o *NB) UpdateLogicalSwitchPortLinkProviderNetwork(ctx context.Context, switchPortName OVNSwitchPort, extNetworkName string) error { // Get the logical switch port. lsp := ovnNB.LogicalSwitchPort{ Name: string(switchPortName), } err := o.get(ctx, &lsp) if err != nil { return err } // Update the fields. lsp.Type = "localnet" lsp.Addresses = []string{"unknown"} if lsp.Options == nil { lsp.Options = map[string]string{} } lsp.Options["network_name"] = extNetworkName // Update the record. operations, err := o.client.Where(&lsp).Update(&lsp) if err != nil { return err } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // CreateChassisGroup adds a new HA chassis group. // If mayExist is true, then an existing resource of the same name is not treated as an error. func (o *NB) CreateChassisGroup(ctx context.Context, haChassisGroupName OVNChassisGroup, mayExist bool) error { // Define the new group. haChassisGroup := ovnNB.HAChassisGroup{ Name: string(haChassisGroupName), } // Check if already exists. err := o.get(ctx, &haChassisGroup) if err != nil && !errors.Is(err, ErrNotFound) { return err } if haChassisGroup.UUID != "" { if mayExist { return nil } return ErrExists } // Create the record. operations, err := o.client.Create(&haChassisGroup) if err != nil { return err } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // DeleteChassisGroup deletes an HA chassis group. func (o *NB) DeleteChassisGroup(ctx context.Context, haChassisGroupName OVNChassisGroup) error { // Get the current chassis group. haChassisGroup := ovnNB.HAChassisGroup{ Name: string(haChassisGroupName), } err := o.get(ctx, &haChassisGroup) if err != nil { // Already gone. if errors.Is(err, ErrNotFound) { return nil } return err } // Delete the chassis group. deleteOps, err := o.client.Where(&haChassisGroup).Delete() if err != nil { return err } resp, err := o.client.Transact(ctx, deleteOps...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, deleteOps) if err != nil { return err } return nil } // SetChassisGroupPriority sets a given priority for the chassis ID in the chassis group.. func (o *NB) SetChassisGroupPriority(ctx context.Context, haChassisGroupName OVNChassisGroup, chassisID string, priority int) error { operations := []ovsdb.Operation{} // Get the chassis group. haGroup := ovnNB.HAChassisGroup{ Name: string(haChassisGroupName), } err := o.get(ctx, &haGroup) if err != nil { return err } // Look for the chassis in the group. var haChassis ovnNB.HAChassis for _, entry := range haGroup.HaChassis { chassis := ovnNB.HAChassis{UUID: entry} err = o.get(ctx, &chassis) if err != nil { return err } if chassis.ChassisName == chassisID { haChassis = chassis break } } if haChassis.UUID == "" { // If asked to remove, then we're done. if priority < 0 { return nil } // No entry found, add a new one. haChassis = ovnNB.HAChassis{ UUID: "chassis", ChassisName: chassisID, Priority: int(priority), } createOps, err := o.client.Create(&haChassis) if err != nil { return err } operations = append(operations, createOps...) // Add the HA Chassis to the group. updateOps, err := o.client.Where(&haGroup).Mutate(&haGroup, ovsModel.Mutation{ Field: &haGroup.HaChassis, Mutator: ovsdb.MutateOperationInsert, Value: []string{haChassis.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) } else if priority < 0 { // Proceed with removing the entry. deleteOps, err := o.client.Where(&haChassis).Delete() if err != nil { return err } operations = append(operations, deleteOps...) // And removing it from the group. updateOps, err := o.client.Where(&haGroup).Mutate(&haGroup, ovsModel.Mutation{ Field: &haGroup.HaChassis, Mutator: ovsdb.MutateOperationDelete, Value: []string{haChassis.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) } else if haChassis.Priority != priority { // Found but wrong priority, correct it. haChassis.Priority = int(priority) updateOps, err := o.client.Where(&haChassis).Update(&haChassis) if err != nil { return err } operations = append(operations, updateOps...) } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // GetPortGroupInfo returns the port group UUID or empty string if port doesn't exist, and whether the port group has // any ACL rules defined on it. func (o *NB) GetPortGroupInfo(ctx context.Context, portGroupName OVNPortGroup) (OVNPortGroupUUID, bool, error) { pg := &ovnNB.PortGroup{ Name: string(portGroupName), } err := o.get(ctx, pg) if err != nil { if errors.Is(err, ovsClient.ErrNotFound) { return "", false, nil } return "", false, err } return OVNPortGroupUUID(pg.UUID), len(pg.ACLs) > 0, nil } // CreatePortGroup creates a new port group and optionally adds logical switch ports to the group. func (o *NB) CreatePortGroup(ctx context.Context, projectID int64, portGroupName OVNPortGroup, associatedPortGroups []OVNPortGroup, associatedSwitch OVNSwitch, initialPortMembers ...OVNSwitchPort) error { // Resolve the initial members. members := []string{} for _, portName := range initialPortMembers { lsp := ovnNB.LogicalSwitchPort{ Name: string(portName), } err := o.get(ctx, &lsp) if err != nil { return err } members = append(members, lsp.UUID) } // Create the port group. pg := ovnNB.PortGroup{ Name: string(portGroupName), Ports: members, ExternalIDs: map[string]string{ ovnExtIDIncusProjectID: fmt.Sprintf("%d", projectID), }, } if len(associatedPortGroups) > 0 || associatedSwitch != "" { if len(associatedPortGroups) > 0 { var parts []string for _, v := range associatedPortGroups { parts = append(parts, string(v)) } pg.ExternalIDs[ovnExtIDIncusPortGroup] = string(strings.Join(parts, ",")) } if associatedSwitch != "" { pg.ExternalIDs[ovnExtIDIncusSwitch] = string(associatedSwitch) } } // Create the record. operations, err := o.client.Create(&pg) if err != nil { return err } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // DeletePortGroup deletes port groups along with their ACL rules. func (o *NB) DeletePortGroup(ctx context.Context, portGroupNames ...OVNPortGroup) error { operations := []ovsdb.Operation{} for _, portGroupName := range portGroupNames { pg := ovnNB.PortGroup{ Name: string(portGroupName), } err := o.get(ctx, &pg) if err != nil { if errors.Is(err, ErrNotFound) { // Already gone. continue } } deleteOps, err := o.client.Where(&pg).Delete() if err != nil { return err } operations = append(operations, deleteOps...) } // Check if we have anything to do. if len(operations) == 0 { return nil } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // GetPortGroupsByProject finds the port groups that are associated to the project ID. func (o *NB) GetPortGroupsByProject(ctx context.Context, projectID int64) ([]OVNPortGroup, error) { portGroups := []ovnNB.PortGroup{} err := o.client.WhereCache(func(pg *ovnNB.PortGroup) bool { return pg.ExternalIDs != nil && pg.ExternalIDs[ovnExtIDIncusProjectID] == fmt.Sprintf("%d", projectID) }).List(ctx, &portGroups) if err != nil { return nil, err } pgNames := make([]OVNPortGroup, 0, len(portGroups)) for _, portGroup := range portGroups { pgNames = append(pgNames, OVNPortGroup(portGroup.Name)) } return pgNames, nil } // GetPortGroupsByPort returns the names of all port groups that contain the given port UUID as a member. func (o *NB) GetPortGroupsByPort(ctx context.Context, portUUID OVNSwitchPortUUID) ([]OVNPortGroup, error) { portGroups := []ovnNB.PortGroup{} err := o.client.WhereCache(func(pg *ovnNB.PortGroup) bool { return slices.Contains(pg.Ports, string(portUUID)) }).List(ctx, &portGroups) if err != nil { return nil, err } pgNames := make([]OVNPortGroup, 0, len(portGroups)) for _, portGroup := range portGroups { pgNames = append(pgNames, OVNPortGroup(portGroup.Name)) } return pgNames, nil } // UpdatePortGroupMembers adds/removes logical switch ports (by UUID) to/from existing port groups. func (o *NB) UpdatePortGroupMembers(ctx context.Context, addMembers map[OVNPortGroup][]OVNSwitchPortUUID, removeMembers map[OVNPortGroup][]OVNSwitchPortUUID) error { operations := []ovsdb.Operation{} for portGroupName, portMemberUUIDs := range addMembers { pg := ovnNB.PortGroup{ Name: string(portGroupName), } for _, portUUID := range portMemberUUIDs { updateOps, err := o.client.Where(&pg).Mutate(&pg, ovsModel.Mutation{ Field: &pg.Ports, Mutator: ovsdb.MutateOperationInsert, Value: []string{string(portUUID)}, }) if err != nil { return err } operations = append(operations, updateOps...) } } for portGroupName, portMemberUUIDs := range removeMembers { pg := ovnNB.PortGroup{ Name: string(portGroupName), } for _, portUUID := range portMemberUUIDs { updateOps, err := o.client.Where(&pg).Mutate(&pg, ovsModel.Mutation{ Field: &pg.Ports, Mutator: ovsdb.MutateOperationDelete, Value: []string{string(portUUID)}, }) if err != nil { return err } operations = append(operations, updateOps...) } } // Check if anything changed. if len(operations) == 0 { return nil } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // UpdatePortGroupACLRules applies a set of rules to the specified port group. Any existing rules are removed. func (o *NB) UpdatePortGroupACLRules(ctx context.Context, portGroupName OVNPortGroup, matchReplace map[string]string, aclRules ...OVNACLRule) error { operations := []ovsdb.Operation{} // Get the port group. pg := ovnNB.PortGroup{ Name: string(portGroupName), } err := o.get(ctx, &pg) if err != nil { return err } // Remove any existing rules assigned to the port group. for _, aclUUID := range pg.ACLs { updateOps, err := o.client.Where(&pg).Mutate(&pg, ovsModel.Mutation{ Field: &pg.ACLs, Mutator: ovsdb.MutateOperationDelete, Value: []string{aclUUID}, }) if err != nil { return err } operations = append(operations, updateOps...) } // Add new rules. externalIDs := map[string]string{ ovnExtIDIncusPortGroup: string(portGroupName), } createOps, err := o.aclRuleAddOperations(ctx, "port_group", string(portGroupName), externalIDs, matchReplace, aclRules...) if err != nil { return err } operations = append(operations, createOps...) // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // AddLogicalSwitchQoSRules applies a set of rules to the specified logical switch port. func (o *NB) AddLogicalSwitchQoSRules(ctx context.Context, switchName OVNSwitch, switchPortName OVNSwitchPort, qosRules ...OVNQoSRule) error { var operations []ovsdb.Operation // Add new rules. externalIDs := map[string]string{ ovnExtIDIncusSwitch: string(switchName), ovnExtIDIncusSwitchPort: string(switchPortName), } createOps, err := o.qosRuleAddOperations(ctx, "logical_switch", string(switchName), externalIDs, nil, qosRules...) if err != nil { return err } operations = append(operations, createOps...) resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } func (o *NB) qosRuleAddOperations(ctx context.Context, entityTable string, entityName string, externalIDs map[string]string, matchReplace map[string]string, qosRules ...OVNQoSRule) ([]ovsdb.Operation, error) { operations := []ovsdb.Operation{} for i, rule := range qosRules { // Perform any replacements requested on the Match string. for find, replace := range matchReplace { rule.Match = strings.ReplaceAll(rule.Match, find, replace) } // Add new QoS. qos := ovnNB.QoS{ UUID: fmt.Sprintf("qos%d", i), Action: rule.Action, Direction: rule.Direction, Bandwidth: rule.Bandwidth, Priority: rule.Priority, Match: rule.Match, ExternalIDs: map[string]string{}, } maps.Copy(qos.ExternalIDs, externalIDs) createOps, err := o.client.Create(&qos) if err != nil { return nil, err } operations = append(operations, createOps...) if entityTable != "logical_switch" { return nil, fmt.Errorf("Unsupported entity table %q", entityTable) } ls := ovnNB.LogicalSwitch{ Name: entityName, } // Add QOS rule to entity. updateOps, err := o.client.Where(&ls).Mutate(&ls, ovsModel.Mutation{ Field: &ls.QOSRules, Mutator: ovsdb.MutateOperationInsert, Value: []string{qos.UUID}, }) if err != nil { return nil, err } operations = append(operations, updateOps...) } return operations, nil } func (o *NB) qosRuleDeleteOperations(ctx context.Context, entityTable string, entityName string, qosRuleUUIDs []string) ([]ovsdb.Operation, error) { var operations []ovsdb.Operation for _, qosRuleUUID := range qosRuleUUIDs { // Get the QOS. qos := ovnNB.QoS{ UUID: qosRuleUUID, } err := o.get(ctx, &qos) if err != nil { return nil, err } // Delete the QOS. deleteOps, err := o.client.Where(&qos).Delete() if err != nil { return nil, err } operations = append(operations, deleteOps...) if entityTable != "logical_switch" { return nil, fmt.Errorf("Unsupported entity table %q", entityTable) } ls := ovnNB.LogicalSwitch{ Name: entityName, } // Remove QOS rule from entity. updateOps, err := o.client.Where(&ls).Mutate(&ls, ovsModel.Mutation{ Field: &ls.QOSRules, Mutator: ovsdb.MutateOperationDelete, Value: []string{qos.UUID}, }) if err != nil { return nil, err } operations = append(operations, updateOps...) } return operations, nil } // aclRuleAddOperations returns the operations to add the provided ACL rules to the specified OVN entity. func (o *NB) aclRuleAddOperations(ctx context.Context, entityTable string, entityName string, externalIDs map[string]string, matchReplace map[string]string, aclRules ...OVNACLRule) ([]ovsdb.Operation, error) { operations := []ovsdb.Operation{} for i, rule := range aclRules { // Perform any replacements requested on the Match string. for find, replace := range matchReplace { rule.Match = strings.ReplaceAll(rule.Match, find, replace) } // Add new ACL. acl := ovnNB.ACL{ UUID: fmt.Sprintf("acl%d", i), Action: rule.Action, Direction: rule.Direction, Priority: rule.Priority, Match: rule.Match, ExternalIDs: map[string]string{}, } if rule.Log { acl.Log = true if rule.LogName != "" { logName := rule.LogName acl.Name = &logName } } maps.Copy(acl.ExternalIDs, externalIDs) createOps, err := o.client.Create(&acl) if err != nil { return nil, err } operations = append(operations, createOps...) // Add ACL rule to entity. if entityTable == "logical_switch" { ls := ovnNB.LogicalSwitch{ Name: entityName, } updateOps, err := o.client.Where(&ls).Mutate(&ls, ovsModel.Mutation{ Field: &ls.ACLs, Mutator: ovsdb.MutateOperationInsert, Value: []string{acl.UUID}, }) if err != nil { return nil, err } operations = append(operations, updateOps...) } else if entityTable == "port_group" { pg := ovnNB.PortGroup{ Name: entityName, } updateOps, err := o.client.Where(&pg).Mutate(&pg, ovsModel.Mutation{ Field: &pg.ACLs, Mutator: ovsdb.MutateOperationInsert, Value: []string{acl.UUID}, }) if err != nil { return nil, err } operations = append(operations, updateOps...) } else { return nil, fmt.Errorf("Unsupported entity table %q", entityTable) } } return operations, nil } // aclRuleDeleteOperations returns the operations that delete the provided ACL rules from the specified OVN entity. func (o *NB) aclRuleDeleteOperations(ctx context.Context, entityTable string, entityName string, aclRuleUUIDs []string) ([]ovsdb.Operation, error) { operations := []ovsdb.Operation{} for _, aclRuleUUID := range aclRuleUUIDs { // Get the ACL. acl := ovnNB.ACL{ UUID: aclRuleUUID, } err := o.get(ctx, &acl) if err != nil { return nil, err } // Delete the ACL. deleteOps, err := o.client.Where(&acl).Delete() if err != nil { return nil, err } operations = append(operations, deleteOps...) // Remove ACL rule from entity. if entityTable == "logical_switch" { ls := ovnNB.LogicalSwitch{ Name: entityName, } updateOps, err := o.client.Where(&ls).Mutate(&ls, ovsModel.Mutation{ Field: &ls.ACLs, Mutator: ovsdb.MutateOperationDelete, Value: []string{acl.UUID}, }) if err != nil { return nil, err } operations = append(operations, updateOps...) } else if entityTable == "port_group" { pg := ovnNB.PortGroup{ Name: entityName, } updateOps, err := o.client.Where(&pg).Mutate(&pg, ovsModel.Mutation{ Field: &pg.ACLs, Mutator: ovsdb.MutateOperationDelete, Value: []string{acl.UUID}, }) if err != nil { return nil, err } operations = append(operations, updateOps...) } else { return nil, fmt.Errorf("Unsupported entity table %q", entityTable) } } return operations, nil } // UpdatePortGroupPortACLRules applies a set of rules for the logical switch port in the specified port group. // Any existing rules for that logical switch port in the port group are removed. func (o *NB) UpdatePortGroupPortACLRules(ctx context.Context, portGroupName OVNPortGroup, portName OVNSwitchPort, aclRules ...OVNACLRule) error { operations := []ovsdb.Operation{} // Remove any existing rules assigned to the entity. removeACLRuleUUIDs, err := o.logicalSwitchPortACLRules(ctx, portName) if err != nil { return err } deleteOps, err := o.aclRuleDeleteOperations(ctx, "port_group", string(portGroupName), removeACLRuleUUIDs) if err != nil { return err } operations = append(operations, deleteOps...) // Add new rules. externalIDs := map[string]string{ ovnExtIDIncusPortGroup: string(portGroupName), ovnExtIDIncusSwitchPort: string(portName), } createOps, err := o.aclRuleAddOperations(ctx, "port_group", string(portGroupName), externalIDs, nil, aclRules...) if err != nil { return err } operations = append(operations, createOps...) // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // ClearPortGroupPortACLRules clears any rules assigned to the logical switch port in the specified port group. func (o *NB) ClearPortGroupPortACLRules(ctx context.Context, portGroupName OVNPortGroup, portName OVNSwitchPort) error { // Remove any existing rules assigned to the entity. removeACLRuleUUIDs, err := o.logicalSwitchPortACLRules(ctx, portName) if err != nil { return err } operations, err := o.aclRuleDeleteOperations(ctx, "port_group", string(portGroupName), removeACLRuleUUIDs) if err != nil { return err } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // CreateLoadBalancer creates a new load balancer (if doesn't exist) on the specified router and switch. // Providing an empty set of vips will delete the load balancer. func (o *NB) CreateLoadBalancer(ctx context.Context, loadBalancerName OVNLoadBalancer, routerName OVNRouter, switchName OVNSwitch, vips ...OVNLoadBalancerVIP) error { lbTCPName := fmt.Sprintf("%s-tcp", loadBalancerName) lbUDPName := fmt.Sprintf("%s-udp", loadBalancerName) operations := []ovsdb.Operation{} // ipToString wraps IPv6 addresses in square brackets. ipToString := func(ip net.IP) string { if ip.To4() == nil { return fmt.Sprintf("[%s]", ip.String()) } return ip.String() } // Remove existing load balancers if they exist. for _, name := range []string{lbTCPName, lbUDPName} { lb := ovnNB.LoadBalancer{ Name: name, } err := o.get(ctx, &lb) if err == nil || errors.Is(err, ErrTooMany) { // Delete the load balancer (by name in case there are duplicates). lb := ovnNB.LoadBalancer{ Name: name, } deleteOps, err := o.client.Where(&lb).Delete() if err != nil { return err } operations = append(operations, deleteOps...) } else if !errors.Is(err, ErrNotFound) { return err } } // Get the logical router. lr, err := o.GetLogicalRouter(ctx, routerName) if err != nil { return err } // Get the logical switch. ls, err := o.GetLogicalSwitch(ctx, switchName) if err != nil { return err } // Define the new load-balancers. lbtcp := &ovnNB.LoadBalancer{ UUID: "lbtcp", Name: lbTCPName, Protocol: &ovnNB.LoadBalancerProtocolTCP, } lbudp := &ovnNB.LoadBalancer{ UUID: "lbudp", Name: lbUDPName, Protocol: &ovnNB.LoadBalancerProtocolUDP, } // Keep track of health check settings. healthChecks := map[string]*OVNLoadBalancerHealthCheck{} // Build up the commands to add VIPs to the load balancer. for _, r := range vips { if r.ListenAddress == nil { return errors.New("Missing VIP listen address") } if len(r.Targets) == 0 { return errors.New("Missing VIP target(s)") } for _, lb := range []*ovnNB.LoadBalancer{lbtcp, lbudp} { if r.Protocol != "" && r.Protocol != *lb.Protocol { continue } if lb.Vips == nil { lb.Vips = map[string]string{} } targetAddresses := []string{} for _, target := range r.Targets { if (r.ListenPort > 0 && target.Port <= 0) || (target.Port > 0 && r.ListenPort <= 0) { return errors.New("The listen and target ports must be specified together") } // Determine the target address. var targetAddress string if r.ListenPort > 0 { targetAddress = fmt.Sprintf("%s:%d", ipToString(target.Address), target.Port) } else { targetAddress = ipToString(target.Address) } targetAddresses = append(targetAddresses, targetAddress) } // Determine the listen address. var listenAddress string if r.ListenPort > 0 { listenAddress = fmt.Sprintf("%s:%d", ipToString(r.ListenAddress), r.ListenPort) } else { listenAddress = ipToString(r.ListenAddress) } lb.Vips[listenAddress] = strings.Join(targetAddresses, ",") if r.HealthCheck != nil { healthChecks[listenAddress] = r.HealthCheck } } } // Create any used load-balancer. lbhcCount := 0 for _, lb := range []*ovnNB.LoadBalancer{lbtcp, lbudp} { if len(lb.Vips) == 0 { continue } // Create healthcheck records. lb.HealthCheck = []string{} lb.IPPortMappings = map[string]string{} for vip, targets := range lb.Vips { // Check if an health check exists for the VIP. healthCheck := healthChecks[vip] if healthCheck == nil { continue } // Create the record. lbhc := &ovnNB.LoadBalancerHealthCheck{ UUID: fmt.Sprintf("lbhc%d", lbhcCount), Options: map[string]string{}, Vip: vip, } // Set some defaults. if healthCheck.FailureCount == 0 { healthCheck.FailureCount = 3 } if healthCheck.SuccessCount == 0 { healthCheck.SuccessCount = 3 } if healthCheck.Interval == 0 { healthCheck.Interval = 10 } if healthCheck.Timeout == 0 { healthCheck.Timeout = 30 } // Apply the settings. lbhc.Options = map[string]string{ "failure_count": fmt.Sprintf("%d", healthCheck.FailureCount), "success_count": fmt.Sprintf("%d", healthCheck.SuccessCount), "interval": fmt.Sprintf("%d", healthCheck.Interval), "timeout": fmt.Sprintf("%d", healthCheck.Timeout), } // Create the load balancer health checker. createOps, err := o.client.Create(lbhc) if err != nil { return err } operations = append(operations, createOps...) // Link LB to LBHC. lb.HealthCheck = append(lb.HealthCheck, lbhc.UUID) // Set up the port bindings. for _, target := range strings.Split(targets, ",") { // Split host and port. host, _, err := net.SplitHostPort(target) if err == nil { target = host } // Skip existing entries. _, ok := lb.IPPortMappings[target] if ok { continue } // Lookup the logical switch port. var lspName string for _, port := range ls.Ports { lsp := ovnNB.LogicalSwitchPort{ UUID: port, } err = o.get(ctx, &lsp) if err != nil { return err } if lsp.ExternalIDs == nil { continue } if OVNSwitch(lsp.ExternalIDs[ovnExtIDIncusSwitch]) != switchName { continue } if lsp.DynamicAddresses != nil { fields := strings.Split(*lsp.DynamicAddresses, " ") if slices.Contains(fields, target) { lspName = lsp.Name break } } for _, address := range lsp.Addresses { fields := strings.Split(address, " ") if slices.Contains(fields, target) { lspName = lsp.Name break } } } if lspName == "" { return fmt.Errorf("Couldn't find a logical switch port for %q", target) } ip := net.ParseIP(target) if ip.To4() == nil { if healthCheck.CheckerIPV6 == nil { return errors.New("No IPv6 address for load balancer health check") } lb.IPPortMappings[fmt.Sprintf("[%s]", target)] = fmt.Sprintf("%s:[%s]", lspName, healthCheck.CheckerIPV6.String()) } else { if healthCheck.CheckerIPV4 == nil { return errors.New("No IPv4 address for load balancer health check") } lb.IPPortMappings[target] = fmt.Sprintf("%s:%s", lspName, healthCheck.CheckerIPV4.String()) } } lbhcCount++ } // Create the record. createOps, err := o.client.Create(lb) if err != nil { return err } operations = append(operations, createOps...) // Add to the router. updateOps, err := o.client.Where(lr).Mutate(lr, ovsModel.Mutation{ Field: &lr.LoadBalancer, Mutator: ovsdb.MutateOperationInsert, Value: []string{lb.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) // Add to the switch. updateOps, err = o.client.Where(ls).Mutate(ls, ovsModel.Mutation{ Field: &ls.LoadBalancer, Mutator: ovsdb.MutateOperationInsert, Value: []string{lb.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) } // Check if anything to delete. if len(operations) == 0 { return nil } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // DeleteLoadBalancer deletes the specified load balancer(s). func (o *NB) DeleteLoadBalancer(ctx context.Context, loadBalancerNames ...OVNLoadBalancer) error { operations := []ovsdb.Operation{} for _, loadBalancerName := range loadBalancerNames { // Check for a TCP load-balancer. lb := ovnNB.LoadBalancer{ Name: fmt.Sprintf("%s-tcp", loadBalancerName), } err := o.get(ctx, &lb) if err == nil { // Delete the load balancer. deleteOps, err := o.client.Where(&lb).Delete() if err != nil { return err } operations = append(operations, deleteOps...) } else if !errors.Is(err, ErrNotFound) { return err } // Check for a UDP load-balancer. lb = ovnNB.LoadBalancer{ Name: fmt.Sprintf("%s-udp", loadBalancerName), } err = o.get(ctx, &lb) if err == nil { // Delete the load balancer. deleteOps, err := o.client.Where(&lb).Delete() if err != nil { return err } operations = append(operations, deleteOps...) } else if !errors.Is(err, ErrNotFound) { return err } } // Check if anything to delete. if len(operations) == 0 { return nil } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // GetLoadBalancer gets the OVN database record for the load balancer. func (o *NB) GetLoadBalancer(ctx context.Context, lbName OVNLoadBalancer) (*ovnNB.LoadBalancer, error) { lb := &ovnNB.LoadBalancer{ Name: string(lbName), } err := o.get(ctx, lb) if err != nil { return nil, err } return lb, nil } // GetLoadBalancersByStatusUpdate locate load-balancer(s) which are affected by a particular service monitor update. func (o *NB) GetLoadBalancersByStatusUpdate(ctx context.Context, mon ovnSB.ServiceMonitor) ([]ovnNB.LoadBalancer, error) { lbs := []ovnNB.LoadBalancer{} err := o.client.WhereCache(func(lb *ovnNB.LoadBalancer) bool { if len(lb.HealthCheck) == 0 { return false } if lb.Protocol != nil && mon.Protocol != nil && *lb.Protocol != *mon.Protocol { return false } for k, v := range lb.IPPortMappings { if k == mon.IP && v == fmt.Sprintf("%s:%s", mon.LogicalPort, mon.SrcIP) { return true } } return false }).List(ctx, &lbs) if err != nil { return nil, err } return lbs, nil } // CreateAddressSet creates address sets for IP versions 4 and 6 in the format "_ip". // Populates them with the relevant addresses supplied. func (o *NB) CreateAddressSet(ctx context.Context, addressSetPrefix OVNAddressSet, addresses ...net.IPNet) error { // Define the new address sets. ipv4Set := ovnNB.AddressSet{ Name: fmt.Sprintf("%s_ip4", addressSetPrefix), Addresses: []string{}, } ipv6Set := ovnNB.AddressSet{ Name: fmt.Sprintf("%s_ip6", addressSetPrefix), Addresses: []string{}, } // Add addresses. for _, address := range addresses { if address.IP.To4() == nil { ipv6Set.Addresses = append(ipv6Set.Addresses, address.String()) } else { ipv4Set.Addresses = append(ipv4Set.Addresses, address.String()) } } // Create the records. operations := []ovsdb.Operation{} createOps, err := o.client.Create(&ipv4Set) if err != nil { return err } operations = append(operations, createOps...) createOps, err = o.client.Create(&ipv6Set) if err != nil { return err } operations = append(operations, createOps...) // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // UpdateAddressSetAdd adds the supplied addresses to the address sets. // If the set is missing, it will get automatically created. // The address set name used is "_ip", e.g. "foo_ip4". func (o *NB) UpdateAddressSetAdd(ctx context.Context, addressSetPrefix OVNAddressSet, addresses ...net.IPNet) error { // Get the address sets. ipv4Set := ovnNB.AddressSet{ Name: fmt.Sprintf("%s_ip4", addressSetPrefix), } err := o.get(ctx, &ipv4Set) if err != nil && !errors.Is(err, ErrNotFound) { return err } if ipv4Set.Addresses == nil { ipv4Set.Addresses = []string{} } ipv6Set := ovnNB.AddressSet{ Name: fmt.Sprintf("%s_ip6", addressSetPrefix), } err = o.get(ctx, &ipv6Set) if err != nil && !errors.Is(err, ErrNotFound) { return err } if ipv6Set.Addresses == nil { ipv6Set.Addresses = []string{} } // Add the addresses. for _, address := range addresses { if address.IP.To4() == nil { if !slices.Contains(ipv6Set.Addresses, address.String()) { ipv6Set.Addresses = append(ipv6Set.Addresses, address.String()) } } else { if !slices.Contains(ipv4Set.Addresses, address.String()) { ipv4Set.Addresses = append(ipv4Set.Addresses, address.String()) } } } // Prepare the records. operations := []ovsdb.Operation{} if ipv4Set.UUID == "" { createOps, err := o.client.Create(&ipv4Set) if err != nil { return err } operations = append(operations, createOps...) } else { updateOps, err := o.client.Where(&ipv4Set).Update(&ipv4Set) if err != nil { return err } operations = append(operations, updateOps...) } if ipv6Set.UUID == "" { createOps, err := o.client.Create(&ipv6Set) if err != nil { return err } operations = append(operations, createOps...) } else { updateOps, err := o.client.Where(&ipv6Set).Update(&ipv6Set) if err != nil { return err } operations = append(operations, updateOps...) } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // UpdateAddressSetRemove removes the supplied addresses from the address set. // The address set name used is "_ip", e.g. "foo_ip4". func (o *NB) UpdateAddressSetRemove(ctx context.Context, addressSetPrefix OVNAddressSet, addresses ...net.IPNet) error { // Get the address sets. ipv4Set := ovnNB.AddressSet{ Name: fmt.Sprintf("%s_ip4", addressSetPrefix), } err := o.get(ctx, &ipv4Set) if err != nil { return err } ipv6Set := ovnNB.AddressSet{ Name: fmt.Sprintf("%s_ip6", addressSetPrefix), } err = o.get(ctx, &ipv6Set) if err != nil { return err } // Filter entries. ipv4Addresses := []string{} for _, entry := range ipv4Set.Addresses { found := false for _, address := range addresses { if entry == address.String() { found = true break } } if !found { ipv4Addresses = append(ipv4Addresses, entry) } } ipv4Set.Addresses = ipv4Addresses ipv6Addresses := []string{} for _, entry := range ipv6Set.Addresses { found := false for _, address := range addresses { if entry == address.String() { found = true break } } if !found { ipv6Addresses = append(ipv6Addresses, entry) } } ipv6Set.Addresses = ipv6Addresses // Prepare the records. operations := []ovsdb.Operation{} updateOps, err := o.client.Where(&ipv4Set).Update(&ipv4Set) if err != nil { return err } operations = append(operations, updateOps...) updateOps, err = o.client.Where(&ipv6Set).Update(&ipv6Set) if err != nil { return err } operations = append(operations, updateOps...) // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // DeleteAddressSet deletes address sets for IP versions 4 and 6 in the format "_ip". func (o *NB) DeleteAddressSet(ctx context.Context, addressSetPrefix OVNAddressSet) error { // Get the address sets. ipv4Set := ovnNB.AddressSet{ Name: fmt.Sprintf("%s_ip4", addressSetPrefix), } err := o.get(ctx, &ipv4Set) if err != nil && !errors.Is(err, ErrNotFound) { return err } ipv6Set := ovnNB.AddressSet{ Name: fmt.Sprintf("%s_ip6", addressSetPrefix), } err = o.get(ctx, &ipv6Set) if err != nil && !errors.Is(err, ErrNotFound) { return err } // Delete the records. operations := []ovsdb.Operation{} if ipv4Set.UUID != "" { deleteOps, err := o.client.Where(&ipv4Set).Delete() if err != nil { return err } operations = append(operations, deleteOps...) } if ipv6Set.UUID != "" { deleteOps, err := o.client.Where(&ipv6Set).Delete() if err != nil { return err } operations = append(operations, deleteOps...) } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // GetAddressSet gets the two OVN database records (v4 and v6) for the address set. func (o *NB) GetAddressSet(ctx context.Context, asName OVNAddressSet) (*ovnNB.AddressSet, *ovnNB.AddressSet, error) { asV4 := &ovnNB.AddressSet{ Name: fmt.Sprintf("%s_ip4", asName), } asV6 := &ovnNB.AddressSet{ Name: fmt.Sprintf("%s_ip6", asName), } // Get the entries (we always expect both IPv4 and IPv6 sets to exist). err := o.get(ctx, asV4) if err != nil { return nil, nil, err } err = o.get(ctx, asV6) if err != nil { return nil, nil, err } return asV4, asV6, nil } // UpdateLogicalRouterPolicy removes any existing policies and applies the new policies to the specified router. func (o *NB) UpdateLogicalRouterPolicy(ctx context.Context, routerName OVNRouter, policies ...OVNRouterPolicy) error { operations := []ovsdb.Operation{} // Get the logical router. lr, err := o.GetLogicalRouter(ctx, routerName) if err != nil { return err } // Clear the existing policies. for _, policyUUID := range lr.Policies { // Delete the policy. policy := ovnNB.LogicalRouterPolicy{ UUID: policyUUID, } deleteOps, err := o.client.Where(&policy).Delete() if err != nil { return err } operations = append(operations, deleteOps...) // Remove from the logical router. updateOps, err := o.client.Where(lr).Mutate(lr, ovsModel.Mutation{ Field: &lr.Policies, Mutator: ovsdb.MutateOperationDelete, Value: []string{policy.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) } // Add the new policies. for i, routerPolicy := range policies { // Create the policy. policy := ovnNB.LogicalRouterPolicy{ UUID: fmt.Sprintf("policy%d", i), Priority: routerPolicy.Priority, Match: routerPolicy.Match, Action: routerPolicy.Action, } createOps, err := o.client.Create(&policy) if err != nil { return err } operations = append(operations, createOps...) // Add to the logical router. updateOps, err := o.client.Where(lr).Mutate(lr, ovsModel.Mutation{ Field: &lr.Policies, Mutator: ovsdb.MutateOperationInsert, Value: []string{policy.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) } // Check if anything changed. if len(operations) == 0 { return nil } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // GetLogicalRouterRoutes returns a list of static routes in the main route table of the logical router. func (o *NB) GetLogicalRouterRoutes(ctx context.Context, routerName OVNRouter) ([]OVNRouterRoute, error) { // Get the logical router. lr, err := o.GetLogicalRouter(ctx, routerName) if err != nil { return nil, err } // Get all the routes. routes := []OVNRouterRoute{} for _, routeUUID := range lr.StaticRoutes { // Get the static entry. route := ovnNB.LogicalRouterStaticRoute{ UUID: routeUUID, } err = o.get(ctx, &route) if err != nil { return nil, err } // Only use the main table. if route.RouteTable != "" { continue } // Create the route entry. var routerRoute OVNRouterRoute // Add CIDR if missing. ip := net.ParseIP(route.IPPrefix) if ip != nil { subnetSize := 32 if ip.To4() == nil { subnetSize = 128 } route.IPPrefix = fmt.Sprintf("%s/%d", ip.String(), subnetSize) } _, prefix, err := net.ParseCIDR(route.IPPrefix) if err != nil { return nil, fmt.Errorf("Invalid static route prefix %q", route.IPPrefix) } routerRoute.Prefix = *prefix routerRoute.NextHop = net.ParseIP(route.Nexthop) if route.Nexthop == "discard" { routerRoute.Discard = true } if route.OutputPort != nil { routerRoute.Port = OVNRouterPort(*route.OutputPort) } routes = append(routes, routerRoute) } return routes, nil } // CreateLogicalRouterPeering applies a peering relationship between two logical routers. func (o *NB) CreateLogicalRouterPeering(ctx context.Context, opts OVNRouterPeering) error { operations := []ovsdb.Operation{} if len(opts.LocalRouterPortIPs) <= 0 || len(opts.TargetRouterPortIPs) <= 0 { return errors.New("IPs not populated for both router ports") } // Remove peering router ports and static routes using ports from both routers. // Run the delete step as a separate command to workaround a bug in OVN. err := o.DeleteLogicalRouterPeering(ctx, opts) if err != nil { return err } // Will use the first IP from each family of the router port interfaces. localRouterGatewayIPs := make(map[uint]net.IP) targetRouterGatewayIPs := make(map[uint]net.IP) // Create the local router port. localPeerName := string(opts.TargetRouterPort) localLRP := ovnNB.LogicalRouterPort{ UUID: "locallrp", Name: string(opts.LocalRouterPort), MAC: opts.LocalRouterPortMAC.String(), Networks: []string{}, Peer: &localPeerName, } for _, ipNet := range opts.LocalRouterPortIPs { ipVersion := uint(4) if ipNet.IP.To4() == nil { ipVersion = 6 } if localRouterGatewayIPs[ipVersion] == nil { localRouterGatewayIPs[ipVersion] = ipNet.IP } localLRP.Networks = append(localLRP.Networks, ipNet.String()) } createOps, err := o.client.Create(&localLRP) if err != nil { return err } operations = append(operations, createOps...) // And connect it to the router. localRouter := ovnNB.LogicalRouter{ Name: string(opts.LocalRouter), } updateOps, err := o.client.Where(&localRouter).Mutate(&localRouter, ovsModel.Mutation{ Field: &localRouter.Ports, Mutator: ovsdb.MutateOperationInsert, Value: []string{localLRP.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) // Create the target router port. targetPeerName := string(opts.LocalRouterPort) targetLRP := ovnNB.LogicalRouterPort{ UUID: "targetlrp", Name: string(opts.TargetRouterPort), MAC: opts.TargetRouterPortMAC.String(), Networks: []string{}, Peer: &targetPeerName, } for _, ipNet := range opts.TargetRouterPortIPs { ipVersion := uint(4) if ipNet.IP.To4() == nil { ipVersion = 6 } if targetRouterGatewayIPs[ipVersion] == nil { targetRouterGatewayIPs[ipVersion] = ipNet.IP } targetLRP.Networks = append(targetLRP.Networks, ipNet.String()) } createOps, err = o.client.Create(&targetLRP) if err != nil { return err } operations = append(operations, createOps...) // And connect it to the router. targetRouter := ovnNB.LogicalRouter{ Name: string(opts.TargetRouter), } updateOps, err = o.client.Where(&targetRouter).Mutate(&targetRouter, ovsModel.Mutation{ Field: &targetRouter.Ports, Mutator: ovsdb.MutateOperationInsert, Value: []string{targetLRP.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) // Add routes using the first router gateway IP for each family for next hop address. localOutputPort := string(opts.LocalRouterPort) for i, route := range opts.LocalRouterRoutes { // Determine the nexthop. ipVersion := uint(4) if route.IP.To4() == nil { ipVersion = 6 } nextHopIP := targetRouterGatewayIPs[ipVersion] if nextHopIP == nil { return fmt.Errorf("Missing target router port IPv%d address for local route %q nexthop address", ipVersion, route.String()) } // Prepare the record. route := ovnNB.LogicalRouterStaticRoute{ UUID: fmt.Sprintf("local%d", i), IPPrefix: route.String(), Nexthop: nextHopIP.String(), OutputPort: &localOutputPort, } createOps, err := o.client.Create(&route) if err != nil { return err } operations = append(operations, createOps...) // Add it to the router. updateOps, err := o.client.Where(&localRouter).Mutate(&localRouter, ovsModel.Mutation{ Field: &localRouter.StaticRoutes, Mutator: ovsdb.MutateOperationInsert, Value: []string{route.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) } targetOutputPort := string(opts.TargetRouterPort) for i, route := range opts.TargetRouterRoutes { // Determine the nexthop. ipVersion := uint(4) if route.IP.To4() == nil { ipVersion = 6 } nextHopIP := localRouterGatewayIPs[ipVersion] if nextHopIP == nil { return fmt.Errorf("Missing local router port IPv%d address for target route %q nexthop address", ipVersion, route.String()) } // Prepare the record. route := ovnNB.LogicalRouterStaticRoute{ UUID: fmt.Sprintf("target%d", i), IPPrefix: route.String(), Nexthop: nextHopIP.String(), OutputPort: &targetOutputPort, } createOps, err := o.client.Create(&route) if err != nil { return err } operations = append(operations, createOps...) // Add it to the router. updateOps, err := o.client.Where(&targetRouter).Mutate(&targetRouter, ovsModel.Mutation{ Field: &targetRouter.StaticRoutes, Mutator: ovsdb.MutateOperationInsert, Value: []string{route.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // DeleteLogicalRouterPeering deletes a peering relationship between two logical routers. // Requires LocalRouter, LocalRouterPort, TargetRouter and TargetRouterPort opts fields to be populated. func (o *NB) DeleteLogicalRouterPeering(ctx context.Context, opts OVNRouterPeering) error { operations := []ovsdb.Operation{} // Remove peering router ports and static routes using ports from both routers. if opts.LocalRouter == "" || opts.TargetRouter == "" { return errors.New("Router names not populated for both routers") } deleteLogicalRouterPort := func(routerName OVNRouter, portName OVNRouterPort) error { // Get the logical router port. logicalRouterPort := ovnNB.LogicalRouterPort{ Name: string(portName), } err := o.get(ctx, &logicalRouterPort) if err != nil { if errors.Is(err, ErrNotFound) { // Logical router port is already gone. return nil } return err } // Get the logical router. logicalRouter := ovnNB.LogicalRouter{ Name: string(routerName), } err = o.get(ctx, &logicalRouter) if err != nil { return err } // Remove the port from the router. updateOps, err := o.client.Where(&logicalRouter).Mutate(&logicalRouter, ovsModel.Mutation{ Field: &logicalRouter.Ports, Mutator: ovsdb.MutateOperationDelete, Value: []string{logicalRouterPort.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) // Delete the port itself. deleteOps, err := o.client.Where(&logicalRouterPort).Delete() if err != nil { return err } operations = append(operations, deleteOps...) // Remove any associated route entries. for _, routeUUID := range logicalRouter.StaticRoutes { // Get the static entry. route := ovnNB.LogicalRouterStaticRoute{ UUID: routeUUID, } err = o.get(ctx, &route) if err != nil { return err } // Skip over anything that's not tied to the current port. if route.OutputPort == nil || *route.OutputPort != string(portName) { continue } // Remove the route from the router. updateOps, err := o.client.Where(&logicalRouter).Mutate(&logicalRouter, ovsModel.Mutation{ Field: &logicalRouter.StaticRoutes, Mutator: ovsdb.MutateOperationDelete, Value: []string{route.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) // Delete the route itself. deleteOps, err := o.client.Where(&route).Delete() if err != nil { return err } operations = append(operations, deleteOps...) } return nil } // Delete both source and target router ports. err := deleteLogicalRouterPort(opts.LocalRouter, opts.LocalRouterPort) if err != nil { return err } err = deleteLogicalRouterPort(opts.TargetRouter, opts.TargetRouterPort) if err != nil { return err } // Check if anything changed. if len(operations) == 0 { return nil } // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // GetLogicalRouterPortHardwareAddress gets the hardware address of the logical router port. func (o *NB) GetLogicalRouterPortHardwareAddress(ctx context.Context, ovnRouterPort OVNRouterPort) (string, error) { lrp, err := o.GetLogicalRouterPort(ctx, ovnRouterPort) if err != nil { return "", err } return lrp.MAC, nil } // GetName returns the OVN AZ name. func (o *NB) GetName(ctx context.Context) (string, error) { // Get the global configuration. nbGlobal := []ovnNB.NBGlobal{} err := o.client.List(ctx, &nbGlobal) if err != nil { return "", err } // Check that we got a result. if len(nbGlobal) != 1 { return "", ovsClient.ErrNotFound } return nbGlobal[0].Name, nil } incus-7.0.0/internal/server/network/ovn/ovn_sb.go000066400000000000000000000117171517523235500221100ustar00rootroot00000000000000package ovn import ( "context" "crypto/tls" "crypto/x509" "encoding/pem" "errors" "runtime" "slices" "strings" "time" "github.com/cenkalti/backoff/v5" "github.com/go-logr/logr" ovsdbCache "github.com/ovn-kubernetes/libovsdb/cache" ovsdbClient "github.com/ovn-kubernetes/libovsdb/client" ovsdbModel "github.com/ovn-kubernetes/libovsdb/model" ovnSB "github.com/lxc/incus/v7/internal/server/network/ovn/schema/ovn-sb" ) // SB client. type SB struct { client ovsdbClient.Client cookie ovsdbClient.MonitorCookie } // NewSB initializes new OVN client for Southbound operations. func NewSB(dbAddr string, sslCACert string, sslClientCert string, sslClientKey string) (*SB, error) { // Prepare the OVSDB client. dbSchema, err := ovnSB.FullDatabaseModel() if err != nil { return nil, err } discard := logr.Discard() options := []ovsdbClient.Option{ovsdbClient.WithLogger(&discard), ovsdbClient.WithReconnect(5*time.Second, &backoff.ZeroBackOff{})} for _, entry := range strings.Split(dbAddr, ",") { options = append(options, ovsdbClient.WithEndpoint(entry)) } // Handle SSL. if strings.Contains(dbAddr, "ssl:") { // Validation. if sslClientCert == "" { return nil, errors.New("OVN is configured to use SSL but no client certificate was found") } if sslClientKey == "" { return nil, errors.New("OVN is configured to use SSL but no client key was found") } // Prepare the client. clientCert, err := tls.X509KeyPair([]byte(sslClientCert), []byte(sslClientKey)) if err != nil { return nil, err } tlsConfig := &tls.Config{ Certificates: []tls.Certificate{clientCert}, InsecureSkipVerify: true, } // Add CA check if provided. if sslCACert != "" { tlsCAder, _ := pem.Decode([]byte(sslCACert)) if tlsCAder == nil { return nil, errors.New("Couldn't parse CA certificate") } tlsCAcert, err := x509.ParseCertificate(tlsCAder.Bytes) if err != nil { return nil, err } tlsCAcert.IsCA = true tlsCAcert.KeyUsage = x509.KeyUsageCertSign clientCAPool := x509.NewCertPool() clientCAPool.AddCert(tlsCAcert) tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, chains [][]*x509.Certificate) error { if len(rawCerts) < 1 { return errors.New("Missing server certificate") } // Load the chain. intermediates := x509.NewCertPool() if len(rawCerts) > 1 { for _, rawCert := range rawCerts[1:] { cert, _ := x509.ParseCertificate(rawCert) if cert != nil { intermediates.AddCert(cert) } } } // Load the main server certificate. cert, _ := x509.ParseCertificate(rawCerts[0]) if cert == nil { return errors.New("Bad server certificate") } // Validate. opts := x509.VerifyOptions{ Roots: clientCAPool, Intermediates: intermediates, } _, err := cert.Verify(opts) return err } } // Add the TLS config to the client. options = append(options, ovsdbClient.WithTLSConfig(tlsConfig)) } // Connect to OVSDB. ovn, err := ovsdbClient.NewOVSDBClient(dbSchema, options...) if err != nil { return nil, err } err = ovn.Connect(context.TODO()) if err != nil { return nil, err } err = ovn.Echo(context.TODO()) if err != nil { return nil, err } // Set up monitor for the tables we use. monitorCookie, err := ovn.Monitor(context.TODO(), ovn.NewMonitor( ovsdbClient.WithTable(&ovnSB.Chassis{}), ovsdbClient.WithTable(&ovnSB.PortBinding{}), ovsdbClient.WithTable(&ovnSB.ServiceMonitor{}))) if err != nil { return nil, err } // Set up event handlers. eventHandler := &ovsdbCache.EventHandlerFuncs{} eventHandler.AddFunc = func(table string, newModel ovsdbModel.Model) { sbEventHandlersMu.Lock() defer sbEventHandlersMu.Unlock() if sbEventHandlers == nil { return } for _, handler := range sbEventHandlers { if handler.Hook != nil && slices.Contains(handler.Tables, table) { go handler.Hook("add", table, nil, newModel) } } } eventHandler.UpdateFunc = func(table string, oldModel ovsdbModel.Model, newModel ovsdbModel.Model) { sbEventHandlersMu.Lock() defer sbEventHandlersMu.Unlock() if sbEventHandlers == nil { return } for _, handler := range sbEventHandlers { if handler.Hook != nil && slices.Contains(handler.Tables, table) { go handler.Hook("update", table, oldModel, newModel) } } } eventHandler.DeleteFunc = func(table string, oldModel ovsdbModel.Model) { sbEventHandlersMu.Lock() defer sbEventHandlersMu.Unlock() if sbEventHandlers == nil { return } for _, handler := range sbEventHandlers { if handler.Hook != nil && slices.Contains(handler.Tables, table) { go handler.Hook("remove", table, oldModel, nil) } } } ovn.Cache().AddEventHandler(eventHandler) // Create the SB struct. client := &SB{ client: ovn, cookie: monitorCookie, } // Set finalizer to stop the monitor. runtime.SetFinalizer(client, func(o *SB) { _ = ovn.MonitorCancel(context.Background(), o.cookie) ovn.Close() }) return client, nil } incus-7.0.0/internal/server/network/ovn/ovn_sb_actions.go000066400000000000000000000044511517523235500236250ustar00rootroot00000000000000package ovn import ( "context" "errors" "fmt" "net" "strconv" "strings" ovnNB "github.com/lxc/incus/v7/internal/server/network/ovn/schema/ovn-nb" ovnSB "github.com/lxc/incus/v7/internal/server/network/ovn/schema/ovn-sb" ) // GetLogicalRouterPortActiveChassisHostname gets the hostname of the chassis managing the logical router port. func (o *SB) GetLogicalRouterPortActiveChassisHostname(ctx context.Context, ovnRouterPort OVNRouterPort) (string, error) { // Look for the port binding. pb := &ovnSB.PortBinding{ LogicalPort: fmt.Sprintf("cr-%s", ovnRouterPort), } err := o.client.Get(ctx, pb) if err != nil { return "", err } if pb.Chassis == nil { return "", errors.New("No chassis found") } // Get the associated chassis. chassis := &ovnSB.Chassis{ UUID: *pb.Chassis, } err = o.client.Get(ctx, chassis) if err != nil { return "", err } return chassis.Hostname, nil } // GetServiceHealth returns the current health record for a particular server and port. func (o *SB) GetServiceHealth(ctx context.Context, address string, protocol string, port int) (string, error) { services := []ovnSB.ServiceMonitor{} err := o.client.WhereCache(func(srv *ovnSB.ServiceMonitor) bool { return srv.Protocol != nil && *srv.Protocol == protocol && srv.IP == address && srv.Port == port && srv.Status != nil }).List(ctx, &services) if err != nil { return "", err } if len(services) != 1 { return "unknown", nil } return *services[0].Status, nil } // CheckLoadBalancerOnline checks all backends for a particular load-balancer. func (o *SB) CheckLoadBalancerOnline(ctx context.Context, lb ovnNB.LoadBalancer) (bool, error) { // Invalid load balancers should be kept offline. if lb.Protocol == nil { return false, nil } // Load-balancers with no service checks should be kept online. if len(lb.HealthCheck) == 0 { return true, nil } for _, v := range lb.Vips { for _, backend := range strings.Split(v, ",") { host, port, err := net.SplitHostPort(backend) if err != nil { return false, err } portInt, err := strconv.Atoi(port) if err != nil { return false, err } status, err := o.GetServiceHealth(ctx, host, *lb.Protocol, portInt) if err != nil { return false, err } if status == "online" { return true, nil } } } return false, nil } incus-7.0.0/internal/server/network/ovn/ovn_sb_events.go000066400000000000000000000013151517523235500234650ustar00rootroot00000000000000package ovn import ( "sync" ) var ( sbEventHandlers map[string]EventHandler sbEventHandlersMu sync.Mutex ) // AddOVNSBHandler registers a new event handler with the OVN Southbound database. func AddOVNSBHandler(name string, handler EventHandler) error { sbEventHandlersMu.Lock() defer sbEventHandlersMu.Unlock() if sbEventHandlers == nil { sbEventHandlers = map[string]EventHandler{} } sbEventHandlers[name] = handler return nil } // RemoveOVNSBHandler removes a currently registered event handler. func RemoveOVNSBHandler(name string) error { sbEventHandlersMu.Lock() defer sbEventHandlersMu.Unlock() if sbEventHandlers == nil { return nil } delete(sbEventHandlers, name) return nil } incus-7.0.0/internal/server/network/ovn/schema/000077500000000000000000000000001517523235500215245ustar00rootroot00000000000000incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-nb/000077500000000000000000000000001517523235500233145ustar00rootroot00000000000000incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-nb/connection.go000066400000000000000000000012371517523235500260050ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ConnectionTable = "Connection" // Connection defines an object in Connection table type Connection struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` InactivityProbe *int `ovsdb:"inactivity_probe"` IsConnected bool `ovsdb:"is_connected"` MaxBackoff *int `ovsdb:"max_backoff" validate:"omitempty,min=1000"` OtherConfig map[string]string `ovsdb:"other_config"` Status map[string]string `ovsdb:"status"` Target string `ovsdb:"target"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-nb/ic_nb_global.go000066400000000000000000000006751517523235500262450ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ICNBGlobalTable = "IC_NB_Global" // ICNBGlobal defines an object in IC_NB_Global table type ICNBGlobal struct { UUID string `ovsdb:"_uuid"` Connections []string `ovsdb:"connections"` ExternalIDs map[string]string `ovsdb:"external_ids"` Options map[string]string `ovsdb:"options"` SSL *string `ovsdb:"ssl"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-nb/model.go000066400000000000000000000111011517523235500247350ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel import ( "encoding/json" "github.com/ovn-kubernetes/libovsdb/model" "github.com/ovn-kubernetes/libovsdb/ovsdb" ) // FullDatabaseModel returns the DatabaseModel object to be used in libovsdb func FullDatabaseModel() (model.ClientDBModel, error) { return model.NewClientDBModel("OVN_IC_Northbound", map[string]model.Model{ "Connection": &Connection{}, "IC_NB_Global": &ICNBGlobal{}, "SSL": &SSL{}, "Transit_Switch": &TransitSwitch{}, }) } var schema = `{ "name": "OVN_IC_Northbound", "version": "1.0.0", "tables": { "Connection": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "inactivity_probe": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "is_connected": { "type": "boolean", "ephemeral": true }, "max_backoff": { "type": { "key": { "type": "integer", "minInteger": 1000 }, "min": 0, "max": 1 } }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "status": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "target": { "type": "string" } }, "indexes": [ [ "target" ] ] }, "IC_NB_Global": { "columns": { "connections": { "type": { "key": { "type": "uuid", "refTable": "Connection" }, "min": 0, "max": "unlimited" } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ssl": { "type": { "key": { "type": "uuid", "refTable": "SSL" }, "min": 0, "max": 1 } } }, "isRoot": true }, "SSL": { "columns": { "bootstrap_ca_cert": { "type": "boolean" }, "ca_cert": { "type": "string" }, "certificate": { "type": "string" }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "private_key": { "type": "string" }, "ssl_ciphers": { "type": "string" }, "ssl_protocols": { "type": "string" } } }, "Transit_Switch": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } } }, "indexes": [ [ "name" ] ], "isRoot": true } } }` func Schema() ovsdb.DatabaseSchema { var s ovsdb.DatabaseSchema err := json.Unmarshal([]byte(schema), &s) if err != nil { panic(err) } return s } incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-nb/ssl.go000066400000000000000000000011451517523235500244450ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const SSLTable = "SSL" // SSL defines an object in SSL table type SSL struct { UUID string `ovsdb:"_uuid"` BootstrapCaCert bool `ovsdb:"bootstrap_ca_cert"` CaCert string `ovsdb:"ca_cert"` Certificate string `ovsdb:"certificate"` ExternalIDs map[string]string `ovsdb:"external_ids"` PrivateKey string `ovsdb:"private_key"` SSLCiphers string `ovsdb:"ssl_ciphers"` SSLProtocols string `ovsdb:"ssl_protocols"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-nb/transit_switch.go000066400000000000000000000006331517523235500267120ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const TransitSwitchTable = "Transit_Switch" // TransitSwitch defines an object in Transit_Switch table type TransitSwitch struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` Name string `ovsdb:"name"` OtherConfig map[string]string `ovsdb:"other_config"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-sb/000077500000000000000000000000001517523235500233215ustar00rootroot00000000000000incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-sb/availability_zone.go000066400000000000000000000004321517523235500273540ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const AvailabilityZoneTable = "Availability_Zone" // AvailabilityZone defines an object in Availability_Zone table type AvailabilityZone struct { UUID string `ovsdb:"_uuid"` Name string `ovsdb:"name"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-sb/connection.go000066400000000000000000000012371517523235500260120ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ConnectionTable = "Connection" // Connection defines an object in Connection table type Connection struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` InactivityProbe *int `ovsdb:"inactivity_probe"` IsConnected bool `ovsdb:"is_connected"` MaxBackoff *int `ovsdb:"max_backoff" validate:"omitempty,min=1000"` OtherConfig map[string]string `ovsdb:"other_config"` Status map[string]string `ovsdb:"status"` Target string `ovsdb:"target"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-sb/datapath_binding.go000066400000000000000000000007231517523235500271320ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const DatapathBindingTable = "Datapath_Binding" // DatapathBinding defines an object in Datapath_Binding table type DatapathBinding struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` TransitSwitch string `ovsdb:"transit_switch"` TunnelKey int `ovsdb:"tunnel_key" validate:"min=1,max=16777215"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-sb/encap.go000066400000000000000000000011251517523235500247350ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const EncapTable = "Encap" type ( EncapType = string ) var ( EncapTypeGeneve EncapType = "geneve" EncapTypeSTT EncapType = "stt" EncapTypeVxlan EncapType = "vxlan" ) // Encap defines an object in Encap table type Encap struct { UUID string `ovsdb:"_uuid"` GatewayName string `ovsdb:"gateway_name"` IP string `ovsdb:"ip"` Options map[string]string `ovsdb:"options"` Type EncapType `ovsdb:"type" validate:"oneof='geneve' 'stt' 'vxlan'"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-sb/gateway.go000066400000000000000000000010211517523235500253030ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const GatewayTable = "Gateway" // Gateway defines an object in Gateway table type Gateway struct { UUID string `ovsdb:"_uuid"` AvailabilityZone string `ovsdb:"availability_zone"` Encaps []string `ovsdb:"encaps" validate:"min=1"` ExternalIDs map[string]string `ovsdb:"external_ids"` Hostname string `ovsdb:"hostname"` Name string `ovsdb:"name"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-sb/ic_sb_global.go000066400000000000000000000006751517523235500262570ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ICSBGlobalTable = "IC_SB_Global" // ICSBGlobal defines an object in IC_SB_Global table type ICSBGlobal struct { UUID string `ovsdb:"_uuid"` Connections []string `ovsdb:"connections"` ExternalIDs map[string]string `ovsdb:"external_ids"` Options map[string]string `ovsdb:"options"` SSL *string `ovsdb:"ssl"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-sb/model.go000066400000000000000000000221621517523235500247530ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel import ( "encoding/json" "github.com/ovn-kubernetes/libovsdb/model" "github.com/ovn-kubernetes/libovsdb/ovsdb" ) // FullDatabaseModel returns the DatabaseModel object to be used in libovsdb func FullDatabaseModel() (model.ClientDBModel, error) { return model.NewClientDBModel("OVN_IC_Southbound", map[string]model.Model{ "Availability_Zone": &AvailabilityZone{}, "Connection": &Connection{}, "Datapath_Binding": &DatapathBinding{}, "Encap": &Encap{}, "Gateway": &Gateway{}, "IC_SB_Global": &ICSBGlobal{}, "Port_Binding": &PortBinding{}, "Route": &Route{}, "SSL": &SSL{}, }) } var schema = `{ "name": "OVN_IC_Southbound", "version": "1.1.1", "tables": { "Availability_Zone": { "columns": { "name": { "type": "string" } }, "indexes": [ [ "name" ] ], "isRoot": true }, "Connection": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "inactivity_probe": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "is_connected": { "type": "boolean", "ephemeral": true }, "max_backoff": { "type": { "key": { "type": "integer", "minInteger": 1000 }, "min": 0, "max": 1 } }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "status": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "target": { "type": "string" } }, "indexes": [ [ "target" ] ] }, "Datapath_Binding": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "transit_switch": { "type": "string" }, "tunnel_key": { "type": { "key": { "type": "integer", "minInteger": 1, "maxInteger": 16777215 } } } }, "indexes": [ [ "tunnel_key" ] ], "isRoot": true }, "Encap": { "columns": { "gateway_name": { "type": "string" }, "ip": { "type": "string" }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "type": { "type": { "key": { "type": "string", "enum": [ "set", [ "geneve", "stt", "vxlan" ] ] } } } }, "indexes": [ [ "type", "ip" ] ] }, "Gateway": { "columns": { "availability_zone": { "type": { "key": { "type": "uuid", "refTable": "Availability_Zone" } } }, "encaps": { "type": { "key": { "type": "uuid", "refTable": "Encap" }, "min": 1, "max": "unlimited" } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "hostname": { "type": "string" }, "name": { "type": "string" } }, "indexes": [ [ "name" ] ], "isRoot": true }, "IC_SB_Global": { "columns": { "connections": { "type": { "key": { "type": "uuid", "refTable": "Connection" }, "min": 0, "max": "unlimited" } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ssl": { "type": { "key": { "type": "uuid", "refTable": "SSL" }, "min": 0, "max": 1 } } }, "isRoot": true }, "Port_Binding": { "columns": { "address": { "type": "string" }, "availability_zone": { "type": { "key": { "type": "uuid", "refTable": "Availability_Zone" } } }, "encap": { "type": { "key": { "type": "uuid", "refTable": "Encap", "refType": "weak" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "gateway": { "type": "string" }, "logical_port": { "type": "string" }, "transit_switch": { "type": "string" }, "tunnel_key": { "type": { "key": { "type": "integer", "minInteger": 1, "maxInteger": 32767 } } } }, "indexes": [ [ "transit_switch", "tunnel_key" ], [ "logical_port" ] ], "isRoot": true }, "Route": { "columns": { "availability_zone": { "type": { "key": { "type": "uuid", "refTable": "Availability_Zone" } } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ip_prefix": { "type": "string" }, "nexthop": { "type": "string" }, "origin": { "type": { "key": { "type": "string", "enum": [ "set", [ "connected", "static" ] ] } } }, "route_table": { "type": "string" }, "transit_switch": { "type": "string" } }, "indexes": [ [ "transit_switch", "availability_zone", "route_table", "ip_prefix", "nexthop" ] ], "isRoot": true }, "SSL": { "columns": { "bootstrap_ca_cert": { "type": "boolean" }, "ca_cert": { "type": "string" }, "certificate": { "type": "string" }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "private_key": { "type": "string" }, "ssl_ciphers": { "type": "string" }, "ssl_protocols": { "type": "string" } } } } }` func Schema() ovsdb.DatabaseSchema { var s ovsdb.DatabaseSchema err := json.Unmarshal([]byte(schema), &s) if err != nil { panic(err) } return s } incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-sb/port_binding.go000066400000000000000000000013431517523235500263270ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const PortBindingTable = "Port_Binding" // PortBinding defines an object in Port_Binding table type PortBinding struct { UUID string `ovsdb:"_uuid"` Address string `ovsdb:"address"` AvailabilityZone string `ovsdb:"availability_zone"` Encap *string `ovsdb:"encap"` ExternalIDs map[string]string `ovsdb:"external_ids"` Gateway string `ovsdb:"gateway"` LogicalPort string `ovsdb:"logical_port"` TransitSwitch string `ovsdb:"transit_switch"` TunnelKey int `ovsdb:"tunnel_key" validate:"min=1,max=32767"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-sb/route.go000066400000000000000000000014351517523235500250110ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const RouteTable = "Route" type ( RouteOrigin = string ) var ( RouteOriginConnected RouteOrigin = "connected" RouteOriginStatic RouteOrigin = "static" ) // Route defines an object in Route table type Route struct { UUID string `ovsdb:"_uuid"` AvailabilityZone string `ovsdb:"availability_zone"` ExternalIDs map[string]string `ovsdb:"external_ids"` IPPrefix string `ovsdb:"ip_prefix"` Nexthop string `ovsdb:"nexthop"` Origin RouteOrigin `ovsdb:"origin" validate:"oneof='connected' 'static'"` RouteTable string `ovsdb:"route_table"` TransitSwitch string `ovsdb:"transit_switch"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-ic-sb/ssl.go000066400000000000000000000011451517523235500244520ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const SSLTable = "SSL" // SSL defines an object in SSL table type SSL struct { UUID string `ovsdb:"_uuid"` BootstrapCaCert bool `ovsdb:"bootstrap_ca_cert"` CaCert string `ovsdb:"ca_cert"` Certificate string `ovsdb:"certificate"` ExternalIDs map[string]string `ovsdb:"external_ids"` PrivateKey string `ovsdb:"private_key"` SSLCiphers string `ovsdb:"ssl_ciphers"` SSLProtocols string `ovsdb:"ssl_protocols"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/000077500000000000000000000000001517523235500227235ustar00rootroot00000000000000incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/acl.go000066400000000000000000000032471517523235500240170ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ACLTable = "ACL" type ( ACLAction = string ACLDirection = string ACLSeverity = string ) var ( ACLActionAllow ACLAction = "allow" ACLActionAllowRelated ACLAction = "allow-related" ACLActionAllowStateless ACLAction = "allow-stateless" ACLActionDrop ACLAction = "drop" ACLActionReject ACLAction = "reject" ACLDirectionFromLport ACLDirection = "from-lport" ACLDirectionToLport ACLDirection = "to-lport" ACLSeverityAlert ACLSeverity = "alert" ACLSeverityWarning ACLSeverity = "warning" ACLSeverityNotice ACLSeverity = "notice" ACLSeverityInfo ACLSeverity = "info" ACLSeverityDebug ACLSeverity = "debug" ) // ACL defines an object in ACL table type ACL struct { UUID string `ovsdb:"_uuid"` Action ACLAction `ovsdb:"action" validate:"oneof='allow' 'allow-related' 'allow-stateless' 'drop' 'reject'"` Direction ACLDirection `ovsdb:"direction" validate:"oneof='from-lport' 'to-lport'"` ExternalIDs map[string]string `ovsdb:"external_ids"` Label int `ovsdb:"label" validate:"min=0,max=4294967295"` Log bool `ovsdb:"log"` Match string `ovsdb:"match"` Meter *string `ovsdb:"meter"` Name *string `ovsdb:"name" validate:"omitempty,max=63"` Options map[string]string `ovsdb:"options"` Priority int `ovsdb:"priority" validate:"min=0,max=32767"` Severity *ACLSeverity `ovsdb:"severity" validate:"omitempty,oneof='alert' 'warning' 'notice' 'info' 'debug'"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/address_set.go000066400000000000000000000006111517523235500255500ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const AddressSetTable = "Address_Set" // AddressSet defines an object in Address_Set table type AddressSet struct { UUID string `ovsdb:"_uuid"` Addresses []string `ovsdb:"addresses"` ExternalIDs map[string]string `ovsdb:"external_ids"` Name string `ovsdb:"name"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/bfd.go000066400000000000000000000016271517523235500240130ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const BFDTable = "BFD" type ( BFDStatus = string ) var ( BFDStatusDown BFDStatus = "down" BFDStatusInit BFDStatus = "init" BFDStatusUp BFDStatus = "up" BFDStatusAdminDown BFDStatus = "admin_down" ) // BFD defines an object in BFD table type BFD struct { UUID string `ovsdb:"_uuid"` DetectMult *int `ovsdb:"detect_mult" validate:"omitempty,min=1"` DstIP string `ovsdb:"dst_ip"` ExternalIDs map[string]string `ovsdb:"external_ids"` LogicalPort string `ovsdb:"logical_port"` MinRx *int `ovsdb:"min_rx"` MinTx *int `ovsdb:"min_tx" validate:"omitempty,min=1"` Options map[string]string `ovsdb:"options"` Status *BFDStatus `ovsdb:"status" validate:"omitempty,oneof='down' 'init' 'up' 'admin_down'"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/chassis_template_var.go000066400000000000000000000006661517523235500274620ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ChassisTemplateVarTable = "Chassis_Template_Var" // ChassisTemplateVar defines an object in Chassis_Template_Var table type ChassisTemplateVar struct { UUID string `ovsdb:"_uuid"` Chassis string `ovsdb:"chassis"` ExternalIDs map[string]string `ovsdb:"external_ids"` Variables map[string]string `ovsdb:"variables"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/connection.go000066400000000000000000000012371517523235500254140ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ConnectionTable = "Connection" // Connection defines an object in Connection table type Connection struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` InactivityProbe *int `ovsdb:"inactivity_probe"` IsConnected bool `ovsdb:"is_connected"` MaxBackoff *int `ovsdb:"max_backoff" validate:"omitempty,min=1000"` OtherConfig map[string]string `ovsdb:"other_config"` Status map[string]string `ovsdb:"status"` Target string `ovsdb:"target"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/copp.go000066400000000000000000000005461517523235500242200ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const CoppTable = "Copp" // Copp defines an object in Copp table type Copp struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` Meters map[string]string `ovsdb:"meters"` Name string `ovsdb:"name"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/dhcp_options.go000066400000000000000000000006141517523235500257440ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const DHCPOptionsTable = "DHCP_Options" // DHCPOptions defines an object in DHCP_Options table type DHCPOptions struct { UUID string `ovsdb:"_uuid"` Cidr string `ovsdb:"cidr"` ExternalIDs map[string]string `ovsdb:"external_ids"` Options map[string]string `ovsdb:"options"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/dns.go000066400000000000000000000004641517523235500240420ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const DNSTable = "DNS" // DNS defines an object in DNS table type DNS struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` Records map[string]string `ovsdb:"records"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/forwarding_group.go000066400000000000000000000011011517523235500266210ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ForwardingGroupTable = "Forwarding_Group" // ForwardingGroup defines an object in Forwarding_Group table type ForwardingGroup struct { UUID string `ovsdb:"_uuid"` ChildPort []string `ovsdb:"child_port" validate:"min=1"` ExternalIDs map[string]string `ovsdb:"external_ids"` Liveness bool `ovsdb:"liveness"` Name string `ovsdb:"name"` Vip string `ovsdb:"vip"` Vmac string `ovsdb:"vmac"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/gateway_chassis.go000066400000000000000000000010361517523235500264300ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const GatewayChassisTable = "Gateway_Chassis" // GatewayChassis defines an object in Gateway_Chassis table type GatewayChassis struct { UUID string `ovsdb:"_uuid"` ChassisName string `ovsdb:"chassis_name"` ExternalIDs map[string]string `ovsdb:"external_ids"` Name string `ovsdb:"name"` Options map[string]string `ovsdb:"options"` Priority int `ovsdb:"priority" validate:"min=0,max=32767"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/ha_chassis.go000066400000000000000000000006461517523235500253650ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const HAChassisTable = "HA_Chassis" // HAChassis defines an object in HA_Chassis table type HAChassis struct { UUID string `ovsdb:"_uuid"` ChassisName string `ovsdb:"chassis_name"` ExternalIDs map[string]string `ovsdb:"external_ids"` Priority int `ovsdb:"priority" validate:"min=0,max=32767"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/ha_chassis_group.go000066400000000000000000000006401517523235500265730ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const HAChassisGroupTable = "HA_Chassis_Group" // HAChassisGroup defines an object in HA_Chassis_Group table type HAChassisGroup struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` HaChassis []string `ovsdb:"ha_chassis"` Name string `ovsdb:"name"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/load_balancer.go000066400000000000000000000032321517523235500260200ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const LoadBalancerTable = "Load_Balancer" type ( LoadBalancerProtocol = string LoadBalancerSelectionFields = string ) var ( LoadBalancerProtocolTCP LoadBalancerProtocol = "tcp" LoadBalancerProtocolUDP LoadBalancerProtocol = "udp" LoadBalancerProtocolSCTP LoadBalancerProtocol = "sctp" LoadBalancerSelectionFieldsEthSrc LoadBalancerSelectionFields = "eth_src" LoadBalancerSelectionFieldsEthDst LoadBalancerSelectionFields = "eth_dst" LoadBalancerSelectionFieldsIPSrc LoadBalancerSelectionFields = "ip_src" LoadBalancerSelectionFieldsIPDst LoadBalancerSelectionFields = "ip_dst" LoadBalancerSelectionFieldsTpSrc LoadBalancerSelectionFields = "tp_src" LoadBalancerSelectionFieldsTpDst LoadBalancerSelectionFields = "tp_dst" ) // LoadBalancer defines an object in Load_Balancer table type LoadBalancer struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` HealthCheck []string `ovsdb:"health_check"` IPPortMappings map[string]string `ovsdb:"ip_port_mappings"` Name string `ovsdb:"name"` Options map[string]string `ovsdb:"options"` Protocol *LoadBalancerProtocol `ovsdb:"protocol" validate:"omitempty,oneof='tcp' 'udp' 'sctp'"` SelectionFields []LoadBalancerSelectionFields `ovsdb:"selection_fields" validate:"dive,oneof='eth_src' 'eth_dst' 'ip_src' 'ip_dst' 'tp_src' 'tp_dst'"` Vips map[string]string `ovsdb:"vips"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/load_balancer_group.go000066400000000000000000000005441517523235500272370ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const LoadBalancerGroupTable = "Load_Balancer_Group" // LoadBalancerGroup defines an object in Load_Balancer_Group table type LoadBalancerGroup struct { UUID string `ovsdb:"_uuid"` LoadBalancer []string `ovsdb:"load_balancer"` Name string `ovsdb:"name"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/load_balancer_health_check.go000066400000000000000000000007131517523235500305030ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const LoadBalancerHealthCheckTable = "Load_Balancer_Health_Check" // LoadBalancerHealthCheck defines an object in Load_Balancer_Health_Check table type LoadBalancerHealthCheck struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` Options map[string]string `ovsdb:"options"` Vip string `ovsdb:"vip"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/logical_router.go000066400000000000000000000015661517523235500262740ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const LogicalRouterTable = "Logical_Router" // LogicalRouter defines an object in Logical_Router table type LogicalRouter struct { UUID string `ovsdb:"_uuid"` Copp *string `ovsdb:"copp"` Enabled *bool `ovsdb:"enabled"` ExternalIDs map[string]string `ovsdb:"external_ids"` LoadBalancer []string `ovsdb:"load_balancer"` LoadBalancerGroup []string `ovsdb:"load_balancer_group"` Name string `ovsdb:"name"` Nat []string `ovsdb:"nat"` Options map[string]string `ovsdb:"options"` Policies []string `ovsdb:"policies"` Ports []string `ovsdb:"ports"` StaticRoutes []string `ovsdb:"static_routes"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/logical_router_policy.go000066400000000000000000000020131517523235500276370ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const LogicalRouterPolicyTable = "Logical_Router_Policy" type ( LogicalRouterPolicyAction = string ) var ( LogicalRouterPolicyActionAllow LogicalRouterPolicyAction = "allow" LogicalRouterPolicyActionDrop LogicalRouterPolicyAction = "drop" LogicalRouterPolicyActionReroute LogicalRouterPolicyAction = "reroute" ) // LogicalRouterPolicy defines an object in Logical_Router_Policy table type LogicalRouterPolicy struct { UUID string `ovsdb:"_uuid"` Action LogicalRouterPolicyAction `ovsdb:"action" validate:"oneof='allow' 'drop' 'reroute'"` ExternalIDs map[string]string `ovsdb:"external_ids"` Match string `ovsdb:"match"` Nexthop *string `ovsdb:"nexthop"` Nexthops []string `ovsdb:"nexthops"` Options map[string]string `ovsdb:"options"` Priority int `ovsdb:"priority" validate:"min=0,max=32767"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/logical_router_port.go000066400000000000000000000016001517523235500273250ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const LogicalRouterPortTable = "Logical_Router_Port" // LogicalRouterPort defines an object in Logical_Router_Port table type LogicalRouterPort struct { UUID string `ovsdb:"_uuid"` Enabled *bool `ovsdb:"enabled"` ExternalIDs map[string]string `ovsdb:"external_ids"` GatewayChassis []string `ovsdb:"gateway_chassis"` HaChassisGroup *string `ovsdb:"ha_chassis_group"` Ipv6Prefix []string `ovsdb:"ipv6_prefix"` Ipv6RaConfigs map[string]string `ovsdb:"ipv6_ra_configs"` MAC string `ovsdb:"mac"` Name string `ovsdb:"name"` Networks []string `ovsdb:"networks" validate:"min=1"` Options map[string]string `ovsdb:"options"` Peer *string `ovsdb:"peer"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/logical_router_static_route.go000066400000000000000000000021231517523235500310470ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const LogicalRouterStaticRouteTable = "Logical_Router_Static_Route" type ( LogicalRouterStaticRoutePolicy = string ) var ( LogicalRouterStaticRoutePolicySrcIP LogicalRouterStaticRoutePolicy = "src-ip" LogicalRouterStaticRoutePolicyDstIP LogicalRouterStaticRoutePolicy = "dst-ip" ) // LogicalRouterStaticRoute defines an object in Logical_Router_Static_Route table type LogicalRouterStaticRoute struct { UUID string `ovsdb:"_uuid"` BFD *string `ovsdb:"bfd"` ExternalIDs map[string]string `ovsdb:"external_ids"` IPPrefix string `ovsdb:"ip_prefix"` Nexthop string `ovsdb:"nexthop"` Options map[string]string `ovsdb:"options"` OutputPort *string `ovsdb:"output_port"` Policy *LogicalRouterStaticRoutePolicy `ovsdb:"policy" validate:"omitempty,oneof='src-ip' 'dst-ip'"` RouteTable string `ovsdb:"route_table"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/logical_switch.go000066400000000000000000000016051517523235500262470ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const LogicalSwitchTable = "Logical_Switch" // LogicalSwitch defines an object in Logical_Switch table type LogicalSwitch struct { UUID string `ovsdb:"_uuid"` ACLs []string `ovsdb:"acls"` Copp *string `ovsdb:"copp"` DNSRecords []string `ovsdb:"dns_records"` ExternalIDs map[string]string `ovsdb:"external_ids"` ForwardingGroups []string `ovsdb:"forwarding_groups"` LoadBalancer []string `ovsdb:"load_balancer"` LoadBalancerGroup []string `ovsdb:"load_balancer_group"` Name string `ovsdb:"name"` OtherConfig map[string]string `ovsdb:"other_config"` Ports []string `ovsdb:"ports"` QOSRules []string `ovsdb:"qos_rules"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/logical_switch_port.go000066400000000000000000000023601517523235500273120ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const LogicalSwitchPortTable = "Logical_Switch_Port" // LogicalSwitchPort defines an object in Logical_Switch_Port table type LogicalSwitchPort struct { UUID string `ovsdb:"_uuid"` Addresses []string `ovsdb:"addresses"` Dhcpv4Options *string `ovsdb:"dhcpv4_options"` Dhcpv6Options *string `ovsdb:"dhcpv6_options"` DynamicAddresses *string `ovsdb:"dynamic_addresses"` Enabled *bool `ovsdb:"enabled"` ExternalIDs map[string]string `ovsdb:"external_ids"` HaChassisGroup *string `ovsdb:"ha_chassis_group"` MirrorRules []string `ovsdb:"mirror_rules"` Name string `ovsdb:"name"` Options map[string]string `ovsdb:"options"` ParentName *string `ovsdb:"parent_name"` PortSecurity []string `ovsdb:"port_security"` Tag *int `ovsdb:"tag" validate:"omitempty,min=1,max=4095"` TagRequest *int `ovsdb:"tag_request" validate:"omitempty,min=0,max=4095"` Type string `ovsdb:"type"` Up *bool `ovsdb:"up"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/meter.go000066400000000000000000000011451517523235500243670ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const MeterTable = "Meter" type ( MeterUnit = string ) var ( MeterUnitKbps MeterUnit = "kbps" MeterUnitPktps MeterUnit = "pktps" ) // Meter defines an object in Meter table type Meter struct { UUID string `ovsdb:"_uuid"` Bands []string `ovsdb:"bands" validate:"min=1"` ExternalIDs map[string]string `ovsdb:"external_ids"` Fair *bool `ovsdb:"fair"` Name string `ovsdb:"name"` Unit MeterUnit `ovsdb:"unit" validate:"oneof='kbps' 'pktps'"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/meter_band.go000066400000000000000000000011431517523235500253510ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const MeterBandTable = "Meter_Band" type ( MeterBandAction = string ) var MeterBandActionDrop MeterBandAction = "drop" // MeterBand defines an object in Meter_Band table type MeterBand struct { UUID string `ovsdb:"_uuid"` Action MeterBandAction `ovsdb:"action" validate:"oneof='drop'"` BurstSize int `ovsdb:"burst_size" validate:"min=0,max=4294967295"` ExternalIDs map[string]string `ovsdb:"external_ids"` Rate int `ovsdb:"rate" validate:"min=1,max=4294967295"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/mirror.go000066400000000000000000000015041517523235500245640ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const MirrorTable = "Mirror" type ( MirrorFilter = string MirrorType = string ) var ( MirrorFilterFromLport MirrorFilter = "from-lport" MirrorFilterToLport MirrorFilter = "to-lport" MirrorTypeGre MirrorType = "gre" MirrorTypeErspan MirrorType = "erspan" ) // Mirror defines an object in Mirror table type Mirror struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` Filter MirrorFilter `ovsdb:"filter" validate:"oneof='from-lport' 'to-lport'"` Index int `ovsdb:"index"` Name string `ovsdb:"name"` Sink string `ovsdb:"sink"` Type MirrorType `ovsdb:"type" validate:"oneof='gre' 'erspan'"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/model.go000066400000000000000000001220731517523235500243570ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel import ( "encoding/json" "github.com/ovn-kubernetes/libovsdb/model" "github.com/ovn-kubernetes/libovsdb/ovsdb" ) // FullDatabaseModel returns the DatabaseModel object to be used in libovsdb func FullDatabaseModel() (model.ClientDBModel, error) { return model.NewClientDBModel("OVN_Northbound", map[string]model.Model{ "ACL": &ACL{}, "Address_Set": &AddressSet{}, "BFD": &BFD{}, "Chassis_Template_Var": &ChassisTemplateVar{}, "Connection": &Connection{}, "Copp": &Copp{}, "DHCP_Options": &DHCPOptions{}, "DNS": &DNS{}, "Forwarding_Group": &ForwardingGroup{}, "Gateway_Chassis": &GatewayChassis{}, "HA_Chassis": &HAChassis{}, "HA_Chassis_Group": &HAChassisGroup{}, "Load_Balancer": &LoadBalancer{}, "Load_Balancer_Group": &LoadBalancerGroup{}, "Load_Balancer_Health_Check": &LoadBalancerHealthCheck{}, "Logical_Router": &LogicalRouter{}, "Logical_Router_Policy": &LogicalRouterPolicy{}, "Logical_Router_Port": &LogicalRouterPort{}, "Logical_Router_Static_Route": &LogicalRouterStaticRoute{}, "Logical_Switch": &LogicalSwitch{}, "Logical_Switch_Port": &LogicalSwitchPort{}, "Meter": &Meter{}, "Meter_Band": &MeterBand{}, "Mirror": &Mirror{}, "NAT": &NAT{}, "NB_Global": &NBGlobal{}, "Port_Group": &PortGroup{}, "QoS": &QoS{}, "SSL": &SSL{}, "Static_MAC_Binding": &StaticMACBinding{}, }) } var schema = `{ "name": "OVN_Northbound", "version": "7.0.0", "tables": { "ACL": { "columns": { "action": { "type": { "key": { "type": "string", "enum": [ "set", [ "allow", "allow-related", "allow-stateless", "drop", "reject" ] ] } } }, "direction": { "type": { "key": { "type": "string", "enum": [ "set", [ "from-lport", "to-lport" ] ] } } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "label": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4294967295 } } }, "log": { "type": "boolean" }, "match": { "type": "string" }, "meter": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "name": { "type": { "key": { "type": "string", "minLength": 63, "maxLength": 63 }, "min": 0, "max": 1 } }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "priority": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 32767 } } }, "severity": { "type": { "key": { "type": "string", "enum": [ "set", [ "alert", "warning", "notice", "info", "debug" ] ] }, "min": 0, "max": 1 } } } }, "Address_Set": { "columns": { "addresses": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" } }, "indexes": [ [ "name" ] ], "isRoot": true }, "BFD": { "columns": { "detect_mult": { "type": { "key": { "type": "integer", "minInteger": 1 }, "min": 0, "max": 1 } }, "dst_ip": { "type": "string" }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "logical_port": { "type": "string" }, "min_rx": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "min_tx": { "type": { "key": { "type": "integer", "minInteger": 1 }, "min": 0, "max": 1 } }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "status": { "type": { "key": { "type": "string", "enum": [ "set", [ "down", "init", "up", "admin_down" ] ] }, "min": 0, "max": 1 } } }, "indexes": [ [ "logical_port", "dst_ip" ] ], "isRoot": true }, "Chassis_Template_Var": { "columns": { "chassis": { "type": "string" }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "variables": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } } }, "indexes": [ [ "chassis" ] ], "isRoot": true }, "Connection": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "inactivity_probe": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "is_connected": { "type": "boolean", "ephemeral": true }, "max_backoff": { "type": { "key": { "type": "integer", "minInteger": 1000 }, "min": 0, "max": 1 } }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "status": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "target": { "type": "string" } }, "indexes": [ [ "target" ] ] }, "Copp": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "meters": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" } }, "indexes": [ [ "name" ] ], "isRoot": true }, "DHCP_Options": { "columns": { "cidr": { "type": "string" }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } } }, "isRoot": true }, "DNS": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "records": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } } }, "isRoot": true }, "Forwarding_Group": { "columns": { "child_port": { "type": { "key": { "type": "string" }, "min": 1, "max": "unlimited" } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "liveness": { "type": "boolean" }, "name": { "type": "string" }, "vip": { "type": "string" }, "vmac": { "type": "string" } } }, "Gateway_Chassis": { "columns": { "chassis_name": { "type": "string" }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "priority": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 32767 } } } }, "indexes": [ [ "name" ] ] }, "HA_Chassis": { "columns": { "chassis_name": { "type": "string" }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "priority": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 32767 } } } } }, "HA_Chassis_Group": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ha_chassis": { "type": { "key": { "type": "uuid", "refTable": "HA_Chassis", "refType": "strong" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" } }, "indexes": [ [ "name" ] ], "isRoot": true }, "Load_Balancer": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "health_check": { "type": { "key": { "type": "uuid", "refTable": "Load_Balancer_Health_Check", "refType": "strong" }, "min": 0, "max": "unlimited" } }, "ip_port_mappings": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "protocol": { "type": { "key": { "type": "string", "enum": [ "set", [ "tcp", "udp", "sctp" ] ] }, "min": 0, "max": 1 } }, "selection_fields": { "type": { "key": { "type": "string", "enum": [ "set", [ "eth_src", "eth_dst", "ip_src", "ip_dst", "tp_src", "tp_dst" ] ] }, "min": 0, "max": "unlimited" } }, "vips": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } } }, "isRoot": true }, "Load_Balancer_Group": { "columns": { "load_balancer": { "type": { "key": { "type": "uuid", "refTable": "Load_Balancer", "refType": "weak" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" } }, "indexes": [ [ "name" ] ], "isRoot": true }, "Load_Balancer_Health_Check": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "vip": { "type": "string" } } }, "Logical_Router": { "columns": { "copp": { "type": { "key": { "type": "uuid", "refTable": "Copp", "refType": "weak" }, "min": 0, "max": 1 } }, "enabled": { "type": { "key": { "type": "boolean" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "load_balancer": { "type": { "key": { "type": "uuid", "refTable": "Load_Balancer", "refType": "weak" }, "min": 0, "max": "unlimited" } }, "load_balancer_group": { "type": { "key": { "type": "uuid", "refTable": "Load_Balancer_Group" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" }, "nat": { "type": { "key": { "type": "uuid", "refTable": "NAT", "refType": "strong" }, "min": 0, "max": "unlimited" } }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "policies": { "type": { "key": { "type": "uuid", "refTable": "Logical_Router_Policy", "refType": "strong" }, "min": 0, "max": "unlimited" } }, "ports": { "type": { "key": { "type": "uuid", "refTable": "Logical_Router_Port", "refType": "strong" }, "min": 0, "max": "unlimited" } }, "static_routes": { "type": { "key": { "type": "uuid", "refTable": "Logical_Router_Static_Route", "refType": "strong" }, "min": 0, "max": "unlimited" } } }, "isRoot": true }, "Logical_Router_Policy": { "columns": { "action": { "type": { "key": { "type": "string", "enum": [ "set", [ "allow", "drop", "reroute" ] ] } } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "match": { "type": "string" }, "nexthop": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "nexthops": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "priority": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 32767 } } } } }, "Logical_Router_Port": { "columns": { "enabled": { "type": { "key": { "type": "boolean" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "gateway_chassis": { "type": { "key": { "type": "uuid", "refTable": "Gateway_Chassis", "refType": "strong" }, "min": 0, "max": "unlimited" } }, "ha_chassis_group": { "type": { "key": { "type": "uuid", "refTable": "HA_Chassis_Group", "refType": "strong" }, "min": 0, "max": 1 } }, "ipv6_prefix": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ipv6_ra_configs": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "mac": { "type": "string" }, "name": { "type": "string" }, "networks": { "type": { "key": { "type": "string" }, "min": 1, "max": "unlimited" } }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "peer": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } } }, "indexes": [ [ "name" ] ] }, "Logical_Router_Static_Route": { "columns": { "bfd": { "type": { "key": { "type": "uuid", "refTable": "BFD", "refType": "weak" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ip_prefix": { "type": "string" }, "nexthop": { "type": "string" }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "output_port": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "policy": { "type": { "key": { "type": "string", "enum": [ "set", [ "src-ip", "dst-ip" ] ] }, "min": 0, "max": 1 } }, "route_table": { "type": "string" } } }, "Logical_Switch": { "columns": { "acls": { "type": { "key": { "type": "uuid", "refTable": "ACL", "refType": "strong" }, "min": 0, "max": "unlimited" } }, "copp": { "type": { "key": { "type": "uuid", "refTable": "Copp", "refType": "weak" }, "min": 0, "max": 1 } }, "dns_records": { "type": { "key": { "type": "uuid", "refTable": "DNS", "refType": "weak" }, "min": 0, "max": "unlimited" } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "forwarding_groups": { "type": { "key": { "type": "uuid", "refTable": "Forwarding_Group", "refType": "strong" }, "min": 0, "max": "unlimited" } }, "load_balancer": { "type": { "key": { "type": "uuid", "refTable": "Load_Balancer", "refType": "weak" }, "min": 0, "max": "unlimited" } }, "load_balancer_group": { "type": { "key": { "type": "uuid", "refTable": "Load_Balancer_Group" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ports": { "type": { "key": { "type": "uuid", "refTable": "Logical_Switch_Port", "refType": "strong" }, "min": 0, "max": "unlimited" } }, "qos_rules": { "type": { "key": { "type": "uuid", "refTable": "QoS", "refType": "strong" }, "min": 0, "max": "unlimited" } } }, "isRoot": true }, "Logical_Switch_Port": { "columns": { "addresses": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } }, "dhcpv4_options": { "type": { "key": { "type": "uuid", "refTable": "DHCP_Options", "refType": "weak" }, "min": 0, "max": 1 } }, "dhcpv6_options": { "type": { "key": { "type": "uuid", "refTable": "DHCP_Options", "refType": "weak" }, "min": 0, "max": 1 } }, "dynamic_addresses": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "enabled": { "type": { "key": { "type": "boolean" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ha_chassis_group": { "type": { "key": { "type": "uuid", "refTable": "HA_Chassis_Group", "refType": "strong" }, "min": 0, "max": 1 } }, "mirror_rules": { "type": { "key": { "type": "uuid", "refTable": "Mirror", "refType": "weak" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "parent_name": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "port_security": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } }, "tag": { "type": { "key": { "type": "integer", "minInteger": 1, "maxInteger": 4095 }, "min": 0, "max": 1 } }, "tag_request": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4095 }, "min": 0, "max": 1 } }, "type": { "type": "string" }, "up": { "type": { "key": { "type": "boolean" }, "min": 0, "max": 1 } } }, "indexes": [ [ "name" ] ] }, "Meter": { "columns": { "bands": { "type": { "key": { "type": "uuid", "refTable": "Meter_Band", "refType": "strong" }, "min": 1, "max": "unlimited" } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "fair": { "type": { "key": { "type": "boolean" }, "min": 0, "max": 1 } }, "name": { "type": "string" }, "unit": { "type": { "key": { "type": "string", "enum": [ "set", [ "kbps", "pktps" ] ] } } } }, "indexes": [ [ "name" ] ], "isRoot": true }, "Meter_Band": { "columns": { "action": { "type": { "key": { "type": "string", "enum": "drop" } } }, "burst_size": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4294967295 } } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "rate": { "type": { "key": { "type": "integer", "minInteger": 1, "maxInteger": 4294967295 } } } } }, "Mirror": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "filter": { "type": { "key": { "type": "string", "enum": [ "set", [ "from-lport", "to-lport" ] ] } } }, "index": { "type": "integer" }, "name": { "type": "string" }, "sink": { "type": "string" }, "type": { "type": { "key": { "type": "string", "enum": [ "set", [ "gre", "erspan" ] ] } } } }, "indexes": [ [ "name" ] ], "isRoot": true }, "NAT": { "columns": { "allowed_ext_ips": { "type": { "key": { "type": "uuid", "refTable": "Address_Set", "refType": "strong" }, "min": 0, "max": 1 } }, "exempted_ext_ips": { "type": { "key": { "type": "uuid", "refTable": "Address_Set", "refType": "strong" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "external_ip": { "type": "string" }, "external_mac": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "external_port_range": { "type": "string" }, "gateway_port": { "type": { "key": { "type": "uuid", "refTable": "Logical_Router_Port", "refType": "weak" }, "min": 0, "max": 1 } }, "logical_ip": { "type": "string" }, "logical_port": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "type": { "type": { "key": { "type": "string", "enum": [ "set", [ "dnat", "snat", "dnat_and_snat" ] ] } } } } }, "NB_Global": { "columns": { "connections": { "type": { "key": { "type": "uuid", "refTable": "Connection" }, "min": 0, "max": "unlimited" } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "hv_cfg": { "type": "integer" }, "hv_cfg_timestamp": { "type": "integer" }, "ipsec": { "type": "boolean" }, "name": { "type": "string" }, "nb_cfg": { "type": "integer" }, "nb_cfg_timestamp": { "type": "integer" }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "sb_cfg": { "type": "integer" }, "sb_cfg_timestamp": { "type": "integer" }, "ssl": { "type": { "key": { "type": "uuid", "refTable": "SSL" }, "min": 0, "max": 1 } } }, "isRoot": true }, "Port_Group": { "columns": { "acls": { "type": { "key": { "type": "uuid", "refTable": "ACL", "refType": "strong" }, "min": 0, "max": "unlimited" } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" }, "ports": { "type": { "key": { "type": "uuid", "refTable": "Logical_Switch_Port", "refType": "weak" }, "min": 0, "max": "unlimited" } } }, "indexes": [ [ "name" ] ], "isRoot": true }, "QoS": { "columns": { "action": { "type": { "key": { "type": "string", "enum": "dscp" }, "value": { "type": "integer", "minInteger": 0, "maxInteger": 63 }, "min": 0, "max": "unlimited" } }, "bandwidth": { "type": { "key": { "type": "string", "enum": [ "set", [ "rate", "burst" ] ] }, "value": { "type": "integer", "minInteger": 1, "maxInteger": 4294967295 }, "min": 0, "max": "unlimited" } }, "direction": { "type": { "key": { "type": "string", "enum": [ "set", [ "from-lport", "to-lport" ] ] } } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "match": { "type": "string" }, "priority": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 32767 } } } } }, "SSL": { "columns": { "bootstrap_ca_cert": { "type": "boolean" }, "ca_cert": { "type": "string" }, "certificate": { "type": "string" }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "private_key": { "type": "string" }, "ssl_ciphers": { "type": "string" }, "ssl_protocols": { "type": "string" } } }, "Static_MAC_Binding": { "columns": { "ip": { "type": "string" }, "logical_port": { "type": "string" }, "mac": { "type": "string" }, "override_dynamic_mac": { "type": "boolean" } }, "indexes": [ [ "logical_port", "ip" ] ], "isRoot": true } } }` func Schema() ovsdb.DatabaseSchema { var s ovsdb.DatabaseSchema err := json.Unmarshal([]byte(schema), &s) if err != nil { panic(err) } return s } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/nat.go000066400000000000000000000020611517523235500240330ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const NATTable = "NAT" type ( NATType = string ) var ( NATTypeDNAT NATType = "dnat" NATTypeSNAT NATType = "snat" NATTypeDNATAndSNAT NATType = "dnat_and_snat" ) // NAT defines an object in NAT table type NAT struct { UUID string `ovsdb:"_uuid"` AllowedExtIPs *string `ovsdb:"allowed_ext_ips"` ExemptedExtIPs *string `ovsdb:"exempted_ext_ips"` ExternalIDs map[string]string `ovsdb:"external_ids"` ExternalIP string `ovsdb:"external_ip"` ExternalMAC *string `ovsdb:"external_mac"` ExternalPortRange string `ovsdb:"external_port_range"` GatewayPort *string `ovsdb:"gateway_port"` LogicalIP string `ovsdb:"logical_ip"` LogicalPort *string `ovsdb:"logical_port"` Options map[string]string `ovsdb:"options"` Type NATType `ovsdb:"type" validate:"oneof='dnat' 'snat' 'dnat_and_snat'"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/nb_global.go000066400000000000000000000015631517523235500251760ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const NBGlobalTable = "NB_Global" // NBGlobal defines an object in NB_Global table type NBGlobal struct { UUID string `ovsdb:"_uuid"` Connections []string `ovsdb:"connections"` ExternalIDs map[string]string `ovsdb:"external_ids"` HvCfg int `ovsdb:"hv_cfg"` HvCfgTimestamp int `ovsdb:"hv_cfg_timestamp"` Ipsec bool `ovsdb:"ipsec"` Name string `ovsdb:"name"` NbCfg int `ovsdb:"nb_cfg"` NbCfgTimestamp int `ovsdb:"nb_cfg_timestamp"` Options map[string]string `ovsdb:"options"` SbCfg int `ovsdb:"sb_cfg"` SbCfgTimestamp int `ovsdb:"sb_cfg_timestamp"` SSL *string `ovsdb:"ssl"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/port_group.go000066400000000000000000000006561517523235500254610ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const PortGroupTable = "Port_Group" // PortGroup defines an object in Port_Group table type PortGroup struct { UUID string `ovsdb:"_uuid"` ACLs []string `ovsdb:"acls"` ExternalIDs map[string]string `ovsdb:"external_ids"` Name string `ovsdb:"name"` Ports []string `ovsdb:"ports"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/qos.go000066400000000000000000000020001517523235500240440ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const QoSTable = "QoS" type ( QoSAction = string QoSBandwidth = string QoSDirection = string ) var ( QoSActionDSCP QoSAction = "dscp" QoSBandwidthRate QoSBandwidth = "rate" QoSBandwidthBurst QoSBandwidth = "burst" QoSDirectionFromLport QoSDirection = "from-lport" QoSDirectionToLport QoSDirection = "to-lport" ) // QoS defines an object in QoS table type QoS struct { UUID string `ovsdb:"_uuid"` Action map[string]int `ovsdb:"action" validate:"dive,keys,oneof='dscp',endkeys,min=0,max=63"` Bandwidth map[string]int `ovsdb:"bandwidth" validate:"dive,keys,oneof='rate' 'burst',endkeys,min=1,max=4294967295"` Direction QoSDirection `ovsdb:"direction" validate:"oneof='from-lport' 'to-lport'"` ExternalIDs map[string]string `ovsdb:"external_ids"` Match string `ovsdb:"match"` Priority int `ovsdb:"priority" validate:"min=0,max=32767"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/ssl.go000066400000000000000000000011451517523235500240540ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const SSLTable = "SSL" // SSL defines an object in SSL table type SSL struct { UUID string `ovsdb:"_uuid"` BootstrapCaCert bool `ovsdb:"bootstrap_ca_cert"` CaCert string `ovsdb:"ca_cert"` Certificate string `ovsdb:"certificate"` ExternalIDs map[string]string `ovsdb:"external_ids"` PrivateKey string `ovsdb:"private_key"` SSLCiphers string `ovsdb:"ssl_ciphers"` SSLProtocols string `ovsdb:"ssl_protocols"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-nb/static_mac_binding.go000066400000000000000000000007131517523235500270540ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const StaticMACBindingTable = "Static_MAC_Binding" // StaticMACBinding defines an object in Static_MAC_Binding table type StaticMACBinding struct { UUID string `ovsdb:"_uuid"` IP string `ovsdb:"ip"` LogicalPort string `ovsdb:"logical_port"` MAC string `ovsdb:"mac"` OverrideDynamicMAC bool `ovsdb:"override_dynamic_mac"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/000077500000000000000000000000001517523235500227305ustar00rootroot00000000000000incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/address_set.go000066400000000000000000000004621517523235500255610ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const AddressSetTable = "Address_Set" // AddressSet defines an object in Address_Set table type AddressSet struct { UUID string `ovsdb:"_uuid"` Addresses []string `ovsdb:"addresses"` Name string `ovsdb:"name"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/bfd.go000066400000000000000000000017261517523235500240200ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const BFDTable = "BFD" type ( BFDStatus = string ) var ( BFDStatusDown BFDStatus = "down" BFDStatusInit BFDStatus = "init" BFDStatusUp BFDStatus = "up" BFDStatusAdminDown BFDStatus = "admin_down" ) // BFD defines an object in BFD table type BFD struct { UUID string `ovsdb:"_uuid"` DetectMult int `ovsdb:"detect_mult"` Disc int `ovsdb:"disc"` DstIP string `ovsdb:"dst_ip"` ExternalIDs map[string]string `ovsdb:"external_ids"` LogicalPort string `ovsdb:"logical_port"` MinRx int `ovsdb:"min_rx"` MinTx int `ovsdb:"min_tx"` Options map[string]string `ovsdb:"options"` SrcPort int `ovsdb:"src_port" validate:"min=49152,max=65535"` Status BFDStatus `ovsdb:"status" validate:"oneof='down' 'init' 'up' 'admin_down'"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/chassis.go000066400000000000000000000013361517523235500247170ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ChassisTable = "Chassis" // Chassis defines an object in Chassis table type Chassis struct { UUID string `ovsdb:"_uuid"` Encaps []string `ovsdb:"encaps" validate:"min=1"` ExternalIDs map[string]string `ovsdb:"external_ids"` Hostname string `ovsdb:"hostname"` Name string `ovsdb:"name"` NbCfg int `ovsdb:"nb_cfg"` OtherConfig map[string]string `ovsdb:"other_config"` TransportZones []string `ovsdb:"transport_zones"` VtepLogicalSwitches []string `ovsdb:"vtep_logical_switches"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/chassis_private.go000066400000000000000000000010271517523235500264460ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ChassisPrivateTable = "Chassis_Private" // ChassisPrivate defines an object in Chassis_Private table type ChassisPrivate struct { UUID string `ovsdb:"_uuid"` Chassis *string `ovsdb:"chassis"` ExternalIDs map[string]string `ovsdb:"external_ids"` Name string `ovsdb:"name"` NbCfg int `ovsdb:"nb_cfg"` NbCfgTimestamp int `ovsdb:"nb_cfg_timestamp"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/chassis_template_var.go000066400000000000000000000005721517523235500274630ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ChassisTemplateVarTable = "Chassis_Template_Var" // ChassisTemplateVar defines an object in Chassis_Template_Var table type ChassisTemplateVar struct { UUID string `ovsdb:"_uuid"` Chassis string `ovsdb:"chassis"` Variables map[string]string `ovsdb:"variables"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/connection.go000066400000000000000000000014101517523235500254120ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ConnectionTable = "Connection" // Connection defines an object in Connection table type Connection struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` InactivityProbe *int `ovsdb:"inactivity_probe"` IsConnected bool `ovsdb:"is_connected"` MaxBackoff *int `ovsdb:"max_backoff" validate:"omitempty,min=1000"` OtherConfig map[string]string `ovsdb:"other_config"` ReadOnly bool `ovsdb:"read_only"` Role string `ovsdb:"role"` Status map[string]string `ovsdb:"status"` Target string `ovsdb:"target"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/controller_event.go000066400000000000000000000012341517523235500266430ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ControllerEventTable = "Controller_Event" type ( ControllerEventEventType = string ) var ControllerEventEventTypeEmptyLbBackends ControllerEventEventType = "empty_lb_backends" // ControllerEvent defines an object in Controller_Event table type ControllerEvent struct { UUID string `ovsdb:"_uuid"` Chassis *string `ovsdb:"chassis"` EventInfo map[string]string `ovsdb:"event_info"` EventType ControllerEventEventType `ovsdb:"event_type" validate:"oneof='empty_lb_backends'"` SeqNum int `ovsdb:"seq_num"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/datapath_binding.go000066400000000000000000000007231517523235500265410ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const DatapathBindingTable = "Datapath_Binding" // DatapathBinding defines an object in Datapath_Binding table type DatapathBinding struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` LoadBalancers []string `ovsdb:"load_balancers"` TunnelKey int `ovsdb:"tunnel_key" validate:"min=1,max=16777215"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/dhcp_options.go000066400000000000000000000017771517523235500257640ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const DHCPOptionsTable = "DHCP_Options" type ( DHCPOptionsType = string ) var ( DHCPOptionsTypeBool DHCPOptionsType = "bool" DHCPOptionsTypeUint8 DHCPOptionsType = "uint8" DHCPOptionsTypeUint16 DHCPOptionsType = "uint16" DHCPOptionsTypeUint32 DHCPOptionsType = "uint32" DHCPOptionsTypeIpv4 DHCPOptionsType = "ipv4" DHCPOptionsTypeStaticRoutes DHCPOptionsType = "static_routes" DHCPOptionsTypeStr DHCPOptionsType = "str" DHCPOptionsTypeHostID DHCPOptionsType = "host_id" DHCPOptionsTypeDomains DHCPOptionsType = "domains" ) // DHCPOptions defines an object in DHCP_Options table type DHCPOptions struct { UUID string `ovsdb:"_uuid"` Code int `ovsdb:"code" validate:"min=0,max=254"` Name string `ovsdb:"name"` Type DHCPOptionsType `ovsdb:"type" validate:"oneof='bool' 'uint8' 'uint16' 'uint32' 'ipv4' 'static_routes' 'str' 'host_id' 'domains'"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/dhcpv6_options.go000066400000000000000000000011571517523235500262300ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const DHCPv6OptionsTable = "DHCPv6_Options" type ( DHCPv6OptionsType = string ) var ( DHCPv6OptionsTypeIpv6 DHCPv6OptionsType = "ipv6" DHCPv6OptionsTypeStr DHCPv6OptionsType = "str" DHCPv6OptionsTypeMAC DHCPv6OptionsType = "mac" ) // DHCPv6Options defines an object in DHCPv6_Options table type DHCPv6Options struct { UUID string `ovsdb:"_uuid"` Code int `ovsdb:"code" validate:"min=0,max=254"` Name string `ovsdb:"name"` Type DHCPv6OptionsType `ovsdb:"type" validate:"oneof='ipv6' 'str' 'mac'"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/dns.go000066400000000000000000000005701517523235500240450ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const DNSTable = "DNS" // DNS defines an object in DNS table type DNS struct { UUID string `ovsdb:"_uuid"` Datapaths []string `ovsdb:"datapaths" validate:"min=1"` ExternalIDs map[string]string `ovsdb:"external_ids"` Records map[string]string `ovsdb:"records"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/encap.go000066400000000000000000000011251517523235500243440ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const EncapTable = "Encap" type ( EncapType = string ) var ( EncapTypeGeneve EncapType = "geneve" EncapTypeSTT EncapType = "stt" EncapTypeVxlan EncapType = "vxlan" ) // Encap defines an object in Encap table type Encap struct { UUID string `ovsdb:"_uuid"` ChassisName string `ovsdb:"chassis_name"` IP string `ovsdb:"ip"` Options map[string]string `ovsdb:"options"` Type EncapType `ovsdb:"type" validate:"oneof='geneve' 'stt' 'vxlan'"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/fdb.go000066400000000000000000000005341517523235500240140ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const FDBTable = "FDB" // FDB defines an object in FDB table type FDB struct { UUID string `ovsdb:"_uuid"` DpKey int `ovsdb:"dp_key" validate:"min=1,max=16777215"` MAC string `ovsdb:"mac"` PortKey int `ovsdb:"port_key" validate:"min=1,max=16777215"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/gateway_chassis.go000066400000000000000000000010311517523235500264300ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const GatewayChassisTable = "Gateway_Chassis" // GatewayChassis defines an object in Gateway_Chassis table type GatewayChassis struct { UUID string `ovsdb:"_uuid"` Chassis *string `ovsdb:"chassis"` ExternalIDs map[string]string `ovsdb:"external_ids"` Name string `ovsdb:"name"` Options map[string]string `ovsdb:"options"` Priority int `ovsdb:"priority" validate:"min=0,max=32767"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/ha_chassis.go000066400000000000000000000006411517523235500253650ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const HAChassisTable = "HA_Chassis" // HAChassis defines an object in HA_Chassis table type HAChassis struct { UUID string `ovsdb:"_uuid"` Chassis *string `ovsdb:"chassis"` ExternalIDs map[string]string `ovsdb:"external_ids"` Priority int `ovsdb:"priority" validate:"min=0,max=32767"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/ha_chassis_group.go000066400000000000000000000007251517523235500266040ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const HAChassisGroupTable = "HA_Chassis_Group" // HAChassisGroup defines an object in HA_Chassis_Group table type HAChassisGroup struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` HaChassis []string `ovsdb:"ha_chassis"` Name string `ovsdb:"name"` RefChassis []string `ovsdb:"ref_chassis"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/igmp_group.go000066400000000000000000000005641517523235500254340ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const IGMPGroupTable = "IGMP_Group" // IGMPGroup defines an object in IGMP_Group table type IGMPGroup struct { UUID string `ovsdb:"_uuid"` Address string `ovsdb:"address"` Chassis *string `ovsdb:"chassis"` Datapath *string `ovsdb:"datapath"` Ports []string `ovsdb:"ports"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/ip_multicast.go000066400000000000000000000012741517523235500257600ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const IPMulticastTable = "IP_Multicast" // IPMulticast defines an object in IP_Multicast table type IPMulticast struct { UUID string `ovsdb:"_uuid"` Datapath string `ovsdb:"datapath"` Enabled *bool `ovsdb:"enabled"` EthSrc string `ovsdb:"eth_src"` IdleTimeout *int `ovsdb:"idle_timeout"` Ip4Src string `ovsdb:"ip4_src"` Ip6Src string `ovsdb:"ip6_src"` Querier *bool `ovsdb:"querier"` QueryInterval *int `ovsdb:"query_interval"` QueryMaxResp *int `ovsdb:"query_max_resp"` SeqNo int `ovsdb:"seq_no"` TableSize *int `ovsdb:"table_size"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/load_balancer.go000066400000000000000000000016221517523235500260260ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const LoadBalancerTable = "Load_Balancer" type ( LoadBalancerProtocol = string ) var ( LoadBalancerProtocolTCP LoadBalancerProtocol = "tcp" LoadBalancerProtocolUDP LoadBalancerProtocol = "udp" LoadBalancerProtocolSCTP LoadBalancerProtocol = "sctp" ) // LoadBalancer defines an object in Load_Balancer table type LoadBalancer struct { UUID string `ovsdb:"_uuid"` DatapathGroup *string `ovsdb:"datapath_group"` Datapaths []string `ovsdb:"datapaths"` ExternalIDs map[string]string `ovsdb:"external_ids"` Name string `ovsdb:"name"` Options map[string]string `ovsdb:"options"` Protocol *LoadBalancerProtocol `ovsdb:"protocol" validate:"omitempty,oneof='tcp' 'udp' 'sctp'"` Vips map[string]string `ovsdb:"vips"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/logical_dp_group.go000066400000000000000000000004451517523235500265730ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const LogicalDPGroupTable = "Logical_DP_Group" // LogicalDPGroup defines an object in Logical_DP_Group table type LogicalDPGroup struct { UUID string `ovsdb:"_uuid"` Datapaths []string `ovsdb:"datapaths"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/logical_flow.go000066400000000000000000000021001517523235500257110ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const LogicalFlowTable = "Logical_Flow" type ( LogicalFlowPipeline = string ) var ( LogicalFlowPipelineIngress LogicalFlowPipeline = "ingress" LogicalFlowPipelineEgress LogicalFlowPipeline = "egress" ) // LogicalFlow defines an object in Logical_Flow table type LogicalFlow struct { UUID string `ovsdb:"_uuid"` Actions string `ovsdb:"actions"` ControllerMeter *string `ovsdb:"controller_meter"` ExternalIDs map[string]string `ovsdb:"external_ids"` LogicalDatapath *string `ovsdb:"logical_datapath"` LogicalDpGroup *string `ovsdb:"logical_dp_group"` Match string `ovsdb:"match"` Pipeline LogicalFlowPipeline `ovsdb:"pipeline" validate:"oneof='ingress' 'egress'"` Priority int `ovsdb:"priority" validate:"min=0,max=65535"` TableID int `ovsdb:"table_id" validate:"min=0,max=32"` Tags map[string]string `ovsdb:"tags"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/mac_binding.go000066400000000000000000000006441517523235500255150ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const MACBindingTable = "MAC_Binding" // MACBinding defines an object in MAC_Binding table type MACBinding struct { UUID string `ovsdb:"_uuid"` Datapath string `ovsdb:"datapath"` IP string `ovsdb:"ip"` LogicalPort string `ovsdb:"logical_port"` MAC string `ovsdb:"mac"` Timestamp int `ovsdb:"timestamp"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/meter.go000066400000000000000000000007111517523235500243720ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const MeterTable = "Meter" type ( MeterUnit = string ) var ( MeterUnitKbps MeterUnit = "kbps" MeterUnitPktps MeterUnit = "pktps" ) // Meter defines an object in Meter table type Meter struct { UUID string `ovsdb:"_uuid"` Bands []string `ovsdb:"bands" validate:"min=1"` Name string `ovsdb:"name"` Unit MeterUnit `ovsdb:"unit" validate:"oneof='kbps' 'pktps'"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/meter_band.go000066400000000000000000000010351517523235500253560ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const MeterBandTable = "Meter_Band" type ( MeterBandAction = string ) var MeterBandActionDrop MeterBandAction = "drop" // MeterBand defines an object in Meter_Band table type MeterBand struct { UUID string `ovsdb:"_uuid"` Action MeterBandAction `ovsdb:"action" validate:"oneof='drop'"` BurstSize int `ovsdb:"burst_size" validate:"min=0,max=4294967295"` Rate int `ovsdb:"rate" validate:"min=1,max=4294967295"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/mirror.go000066400000000000000000000015041517523235500245710ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const MirrorTable = "Mirror" type ( MirrorFilter = string MirrorType = string ) var ( MirrorFilterFromLport MirrorFilter = "from-lport" MirrorFilterToLport MirrorFilter = "to-lport" MirrorTypeGre MirrorType = "gre" MirrorTypeErspan MirrorType = "erspan" ) // Mirror defines an object in Mirror table type Mirror struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` Filter MirrorFilter `ovsdb:"filter" validate:"oneof='from-lport' 'to-lport'"` Index int `ovsdb:"index"` Name string `ovsdb:"name"` Sink string `ovsdb:"sink"` Type MirrorType `ovsdb:"type" validate:"oneof='gre' 'erspan'"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/model.go000066400000000000000000001114511517523235500243620ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel import ( "encoding/json" "github.com/ovn-kubernetes/libovsdb/model" "github.com/ovn-kubernetes/libovsdb/ovsdb" ) // FullDatabaseModel returns the DatabaseModel object to be used in libovsdb func FullDatabaseModel() (model.ClientDBModel, error) { return model.NewClientDBModel("OVN_Southbound", map[string]model.Model{ "Address_Set": &AddressSet{}, "BFD": &BFD{}, "Chassis": &Chassis{}, "Chassis_Private": &ChassisPrivate{}, "Chassis_Template_Var": &ChassisTemplateVar{}, "Connection": &Connection{}, "Controller_Event": &ControllerEvent{}, "DHCP_Options": &DHCPOptions{}, "DHCPv6_Options": &DHCPv6Options{}, "DNS": &DNS{}, "Datapath_Binding": &DatapathBinding{}, "Encap": &Encap{}, "FDB": &FDB{}, "Gateway_Chassis": &GatewayChassis{}, "HA_Chassis": &HAChassis{}, "HA_Chassis_Group": &HAChassisGroup{}, "IGMP_Group": &IGMPGroup{}, "IP_Multicast": &IPMulticast{}, "Load_Balancer": &LoadBalancer{}, "Logical_DP_Group": &LogicalDPGroup{}, "Logical_Flow": &LogicalFlow{}, "MAC_Binding": &MACBinding{}, "Meter": &Meter{}, "Meter_Band": &MeterBand{}, "Mirror": &Mirror{}, "Multicast_Group": &MulticastGroup{}, "Port_Binding": &PortBinding{}, "Port_Group": &PortGroup{}, "RBAC_Permission": &RBACPermission{}, "RBAC_Role": &RBACRole{}, "SB_Global": &SBGlobal{}, "SSL": &SSL{}, "Service_Monitor": &ServiceMonitor{}, "Static_MAC_Binding": &StaticMACBinding{}, }) } var schema = `{ "name": "OVN_Southbound", "version": "20.27.0", "tables": { "Address_Set": { "columns": { "addresses": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" } }, "indexes": [ [ "name" ] ], "isRoot": true }, "BFD": { "columns": { "detect_mult": { "type": "integer" }, "disc": { "type": "integer" }, "dst_ip": { "type": "string" }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "logical_port": { "type": "string" }, "min_rx": { "type": "integer" }, "min_tx": { "type": "integer" }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "src_port": { "type": { "key": { "type": "integer", "minInteger": 49152, "maxInteger": 65535 } } }, "status": { "type": { "key": { "type": "string", "enum": [ "set", [ "down", "init", "up", "admin_down" ] ] } } } }, "indexes": [ [ "logical_port", "dst_ip", "src_port", "disc" ] ], "isRoot": true }, "Chassis": { "columns": { "encaps": { "type": { "key": { "type": "uuid", "refTable": "Encap" }, "min": 1, "max": "unlimited" } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "hostname": { "type": "string" }, "name": { "type": "string" }, "nb_cfg": { "type": "integer" }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "transport_zones": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } }, "vtep_logical_switches": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } } }, "indexes": [ [ "name" ] ], "isRoot": true }, "Chassis_Private": { "columns": { "chassis": { "type": { "key": { "type": "uuid", "refTable": "Chassis", "refType": "weak" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" }, "nb_cfg": { "type": "integer" }, "nb_cfg_timestamp": { "type": "integer" } }, "indexes": [ [ "name" ] ], "isRoot": true }, "Chassis_Template_Var": { "columns": { "chassis": { "type": "string" }, "variables": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } } }, "indexes": [ [ "chassis" ] ], "isRoot": true }, "Connection": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "inactivity_probe": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "is_connected": { "type": "boolean", "ephemeral": true }, "max_backoff": { "type": { "key": { "type": "integer", "minInteger": 1000 }, "min": 0, "max": 1 } }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "read_only": { "type": "boolean" }, "role": { "type": "string" }, "status": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "target": { "type": "string" } }, "indexes": [ [ "target" ] ] }, "Controller_Event": { "columns": { "chassis": { "type": { "key": { "type": "uuid", "refTable": "Chassis", "refType": "weak" }, "min": 0, "max": 1 } }, "event_info": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "event_type": { "type": { "key": { "type": "string", "enum": "empty_lb_backends" } } }, "seq_num": { "type": "integer" } }, "isRoot": true }, "DHCP_Options": { "columns": { "code": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 254 } } }, "name": { "type": "string" }, "type": { "type": { "key": { "type": "string", "enum": [ "set", [ "bool", "uint8", "uint16", "uint32", "ipv4", "static_routes", "str", "host_id", "domains" ] ] } } } }, "isRoot": true }, "DHCPv6_Options": { "columns": { "code": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 254 } } }, "name": { "type": "string" }, "type": { "type": { "key": { "type": "string", "enum": [ "set", [ "ipv6", "str", "mac" ] ] } } } }, "isRoot": true }, "DNS": { "columns": { "datapaths": { "type": { "key": { "type": "uuid", "refTable": "Datapath_Binding" }, "min": 1, "max": "unlimited" } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "records": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } } }, "isRoot": true }, "Datapath_Binding": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "load_balancers": { "type": { "key": { "type": "uuid" }, "min": 0, "max": "unlimited" } }, "tunnel_key": { "type": { "key": { "type": "integer", "minInteger": 1, "maxInteger": 16777215 } } } }, "indexes": [ [ "tunnel_key" ] ], "isRoot": true }, "Encap": { "columns": { "chassis_name": { "type": "string" }, "ip": { "type": "string" }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "type": { "type": { "key": { "type": "string", "enum": [ "set", [ "geneve", "stt", "vxlan" ] ] } } } }, "indexes": [ [ "type", "ip" ] ] }, "FDB": { "columns": { "dp_key": { "type": { "key": { "type": "integer", "minInteger": 1, "maxInteger": 16777215 } } }, "mac": { "type": "string" }, "port_key": { "type": { "key": { "type": "integer", "minInteger": 1, "maxInteger": 16777215 } } } }, "indexes": [ [ "mac", "dp_key" ] ], "isRoot": true }, "Gateway_Chassis": { "columns": { "chassis": { "type": { "key": { "type": "uuid", "refTable": "Chassis", "refType": "weak" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "priority": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 32767 } } } }, "indexes": [ [ "name" ] ] }, "HA_Chassis": { "columns": { "chassis": { "type": { "key": { "type": "uuid", "refTable": "Chassis", "refType": "weak" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "priority": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 32767 } } } } }, "HA_Chassis_Group": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ha_chassis": { "type": { "key": { "type": "uuid", "refTable": "HA_Chassis", "refType": "strong" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" }, "ref_chassis": { "type": { "key": { "type": "uuid", "refTable": "Chassis", "refType": "weak" }, "min": 0, "max": "unlimited" } } }, "indexes": [ [ "name" ] ], "isRoot": true }, "IGMP_Group": { "columns": { "address": { "type": "string" }, "chassis": { "type": { "key": { "type": "uuid", "refTable": "Chassis", "refType": "weak" }, "min": 0, "max": 1 } }, "datapath": { "type": { "key": { "type": "uuid", "refTable": "Datapath_Binding", "refType": "weak" }, "min": 0, "max": 1 } }, "ports": { "type": { "key": { "type": "uuid", "refTable": "Port_Binding", "refType": "weak" }, "min": 0, "max": "unlimited" } } }, "indexes": [ [ "address", "datapath", "chassis" ] ], "isRoot": true }, "IP_Multicast": { "columns": { "datapath": { "type": { "key": { "type": "uuid", "refTable": "Datapath_Binding", "refType": "weak" } } }, "enabled": { "type": { "key": { "type": "boolean" }, "min": 0, "max": 1 } }, "eth_src": { "type": "string" }, "idle_timeout": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "ip4_src": { "type": "string" }, "ip6_src": { "type": "string" }, "querier": { "type": { "key": { "type": "boolean" }, "min": 0, "max": 1 } }, "query_interval": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "query_max_resp": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "seq_no": { "type": "integer" }, "table_size": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } } }, "indexes": [ [ "datapath" ] ], "isRoot": true }, "Load_Balancer": { "columns": { "datapath_group": { "type": { "key": { "type": "uuid", "refTable": "Logical_DP_Group" }, "min": 0, "max": 1 } }, "datapaths": { "type": { "key": { "type": "uuid", "refTable": "Datapath_Binding" }, "min": 0, "max": "unlimited" } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "protocol": { "type": { "key": { "type": "string", "enum": [ "set", [ "tcp", "udp", "sctp" ] ] }, "min": 0, "max": 1 } }, "vips": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } } }, "isRoot": true }, "Logical_DP_Group": { "columns": { "datapaths": { "type": { "key": { "type": "uuid", "refTable": "Datapath_Binding", "refType": "weak" }, "min": 0, "max": "unlimited" } } } }, "Logical_Flow": { "columns": { "actions": { "type": "string" }, "controller_meter": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "logical_datapath": { "type": { "key": { "type": "uuid", "refTable": "Datapath_Binding" }, "min": 0, "max": 1 } }, "logical_dp_group": { "type": { "key": { "type": "uuid", "refTable": "Logical_DP_Group" }, "min": 0, "max": 1 } }, "match": { "type": "string" }, "pipeline": { "type": { "key": { "type": "string", "enum": [ "set", [ "ingress", "egress" ] ] } } }, "priority": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 65535 } } }, "table_id": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 32 } } }, "tags": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } } }, "isRoot": true }, "MAC_Binding": { "columns": { "datapath": { "type": { "key": { "type": "uuid", "refTable": "Datapath_Binding" } } }, "ip": { "type": "string" }, "logical_port": { "type": "string" }, "mac": { "type": "string" }, "timestamp": { "type": "integer" } }, "indexes": [ [ "logical_port", "ip" ] ], "isRoot": true }, "Meter": { "columns": { "bands": { "type": { "key": { "type": "uuid", "refTable": "Meter_Band", "refType": "strong" }, "min": 1, "max": "unlimited" } }, "name": { "type": "string" }, "unit": { "type": { "key": { "type": "string", "enum": [ "set", [ "kbps", "pktps" ] ] } } } }, "indexes": [ [ "name" ] ], "isRoot": true }, "Meter_Band": { "columns": { "action": { "type": { "key": { "type": "string", "enum": "drop" } } }, "burst_size": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4294967295 } } }, "rate": { "type": { "key": { "type": "integer", "minInteger": 1, "maxInteger": 4294967295 } } } } }, "Mirror": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "filter": { "type": { "key": { "type": "string", "enum": [ "set", [ "from-lport", "to-lport" ] ] } } }, "index": { "type": "integer" }, "name": { "type": "string" }, "sink": { "type": "string" }, "type": { "type": { "key": { "type": "string", "enum": [ "set", [ "gre", "erspan" ] ] } } } }, "indexes": [ [ "name" ] ], "isRoot": true }, "Multicast_Group": { "columns": { "datapath": { "type": { "key": { "type": "uuid", "refTable": "Datapath_Binding" } } }, "name": { "type": "string" }, "ports": { "type": { "key": { "type": "uuid", "refTable": "Port_Binding", "refType": "weak" }, "min": 0, "max": "unlimited" } }, "tunnel_key": { "type": { "key": { "type": "integer", "minInteger": 32768, "maxInteger": 65535 } } } }, "indexes": [ [ "datapath", "tunnel_key" ], [ "datapath", "name" ] ], "isRoot": true }, "Port_Binding": { "columns": { "additional_chassis": { "type": { "key": { "type": "uuid", "refTable": "Chassis", "refType": "weak" }, "min": 0, "max": "unlimited" } }, "additional_encap": { "type": { "key": { "type": "uuid", "refTable": "Encap", "refType": "weak" }, "min": 0, "max": "unlimited" } }, "chassis": { "type": { "key": { "type": "uuid", "refTable": "Chassis", "refType": "weak" }, "min": 0, "max": 1 } }, "datapath": { "type": { "key": { "type": "uuid", "refTable": "Datapath_Binding" } } }, "encap": { "type": { "key": { "type": "uuid", "refTable": "Encap", "refType": "weak" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "gateway_chassis": { "type": { "key": { "type": "uuid", "refTable": "Gateway_Chassis", "refType": "strong" }, "min": 0, "max": "unlimited" } }, "ha_chassis_group": { "type": { "key": { "type": "uuid", "refTable": "HA_Chassis_Group", "refType": "strong" }, "min": 0, "max": 1 } }, "logical_port": { "type": "string" }, "mac": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } }, "mirror_rules": { "type": { "key": { "type": "uuid", "refTable": "Mirror", "refType": "weak" }, "min": 0, "max": "unlimited" } }, "nat_addresses": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "parent_port": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "port_security": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } }, "requested_additional_chassis": { "type": { "key": { "type": "uuid", "refTable": "Chassis", "refType": "weak" }, "min": 0, "max": "unlimited" } }, "requested_chassis": { "type": { "key": { "type": "uuid", "refTable": "Chassis", "refType": "weak" }, "min": 0, "max": 1 } }, "tag": { "type": { "key": { "type": "integer", "minInteger": 1, "maxInteger": 4095 }, "min": 0, "max": 1 } }, "tunnel_key": { "type": { "key": { "type": "integer", "minInteger": 1, "maxInteger": 32767 } } }, "type": { "type": "string" }, "up": { "type": { "key": { "type": "boolean" }, "min": 0, "max": 1 } }, "virtual_parent": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } } }, "indexes": [ [ "datapath", "tunnel_key" ], [ "logical_port" ] ], "isRoot": true }, "Port_Group": { "columns": { "name": { "type": "string" }, "ports": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } } }, "indexes": [ [ "name" ] ], "isRoot": true }, "RBAC_Permission": { "columns": { "authorization": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } }, "insert_delete": { "type": "boolean" }, "table": { "type": "string" }, "update": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } } }, "isRoot": true }, "RBAC_Role": { "columns": { "name": { "type": "string" }, "permissions": { "type": { "key": { "type": "string" }, "value": { "type": "uuid", "refTable": "RBAC_Permission", "refType": "weak" }, "min": 0, "max": "unlimited" } } }, "isRoot": true }, "SB_Global": { "columns": { "connections": { "type": { "key": { "type": "uuid", "refTable": "Connection" }, "min": 0, "max": "unlimited" } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ipsec": { "type": "boolean" }, "nb_cfg": { "type": "integer" }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ssl": { "type": { "key": { "type": "uuid", "refTable": "SSL" }, "min": 0, "max": 1 } } }, "isRoot": true }, "SSL": { "columns": { "bootstrap_ca_cert": { "type": "boolean" }, "ca_cert": { "type": "string" }, "certificate": { "type": "string" }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "private_key": { "type": "string" }, "ssl_ciphers": { "type": "string" }, "ssl_protocols": { "type": "string" } } }, "Service_Monitor": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ip": { "type": "string" }, "logical_port": { "type": "string" }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "port": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 65535 } } }, "protocol": { "type": { "key": { "type": "string", "enum": [ "set", [ "tcp", "udp" ] ] }, "min": 0, "max": 1 } }, "src_ip": { "type": "string" }, "src_mac": { "type": "string" }, "status": { "type": { "key": { "type": "string", "enum": [ "set", [ "online", "offline", "error" ] ] }, "min": 0, "max": 1 } } }, "indexes": [ [ "logical_port", "ip", "port", "protocol" ] ], "isRoot": true }, "Static_MAC_Binding": { "columns": { "datapath": { "type": { "key": { "type": "uuid", "refTable": "Datapath_Binding" } } }, "ip": { "type": "string" }, "logical_port": { "type": "string" }, "mac": { "type": "string" }, "override_dynamic_mac": { "type": "boolean" } }, "indexes": [ [ "logical_port", "ip" ] ], "isRoot": true } } }` func Schema() ovsdb.DatabaseSchema { var s ovsdb.DatabaseSchema err := json.Unmarshal([]byte(schema), &s) if err != nil { panic(err) } return s } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/multicast_group.go000066400000000000000000000006611517523235500265030ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const MulticastGroupTable = "Multicast_Group" // MulticastGroup defines an object in Multicast_Group table type MulticastGroup struct { UUID string `ovsdb:"_uuid"` Datapath string `ovsdb:"datapath"` Name string `ovsdb:"name"` Ports []string `ovsdb:"ports"` TunnelKey int `ovsdb:"tunnel_key" validate:"min=32768,max=65535"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/port_binding.go000066400000000000000000000034411517523235500257370ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const PortBindingTable = "Port_Binding" // PortBinding defines an object in Port_Binding table type PortBinding struct { UUID string `ovsdb:"_uuid"` AdditionalChassis []string `ovsdb:"additional_chassis"` AdditionalEncap []string `ovsdb:"additional_encap"` Chassis *string `ovsdb:"chassis"` Datapath string `ovsdb:"datapath"` Encap *string `ovsdb:"encap"` ExternalIDs map[string]string `ovsdb:"external_ids"` GatewayChassis []string `ovsdb:"gateway_chassis"` HaChassisGroup *string `ovsdb:"ha_chassis_group"` LogicalPort string `ovsdb:"logical_port"` MAC []string `ovsdb:"mac"` MirrorRules []string `ovsdb:"mirror_rules"` NatAddresses []string `ovsdb:"nat_addresses"` Options map[string]string `ovsdb:"options"` ParentPort *string `ovsdb:"parent_port"` PortSecurity []string `ovsdb:"port_security"` RequestedAdditionalChassis []string `ovsdb:"requested_additional_chassis"` RequestedChassis *string `ovsdb:"requested_chassis"` Tag *int `ovsdb:"tag" validate:"omitempty,min=1,max=4095"` TunnelKey int `ovsdb:"tunnel_key" validate:"min=1,max=32767"` Type string `ovsdb:"type"` Up *bool `ovsdb:"up"` VirtualParent *string `ovsdb:"virtual_parent"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/port_group.go000066400000000000000000000004351517523235500254610ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const PortGroupTable = "Port_Group" // PortGroup defines an object in Port_Group table type PortGroup struct { UUID string `ovsdb:"_uuid"` Name string `ovsdb:"name"` Ports []string `ovsdb:"ports"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/rbac_permission.go000066400000000000000000000006601517523235500264400ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const RBACPermissionTable = "RBAC_Permission" // RBACPermission defines an object in RBAC_Permission table type RBACPermission struct { UUID string `ovsdb:"_uuid"` Authorization []string `ovsdb:"authorization"` InsertDelete bool `ovsdb:"insert_delete"` Table string `ovsdb:"table"` Update []string `ovsdb:"update"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/rbac_role.go000066400000000000000000000005131517523235500252060ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const RBACRoleTable = "RBAC_Role" // RBACRole defines an object in RBAC_Role table type RBACRole struct { UUID string `ovsdb:"_uuid"` Name string `ovsdb:"name"` Permissions map[string]string `ovsdb:"permissions"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/sb_global.go000066400000000000000000000010201517523235500251740ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const SBGlobalTable = "SB_Global" // SBGlobal defines an object in SB_Global table type SBGlobal struct { UUID string `ovsdb:"_uuid"` Connections []string `ovsdb:"connections"` ExternalIDs map[string]string `ovsdb:"external_ids"` Ipsec bool `ovsdb:"ipsec"` NbCfg int `ovsdb:"nb_cfg"` Options map[string]string `ovsdb:"options"` SSL *string `ovsdb:"ssl"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/service_monitor.go000066400000000000000000000023761517523235500264760ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ServiceMonitorTable = "Service_Monitor" type ( ServiceMonitorProtocol = string ServiceMonitorStatus = string ) var ( ServiceMonitorProtocolTCP ServiceMonitorProtocol = "tcp" ServiceMonitorProtocolUDP ServiceMonitorProtocol = "udp" ServiceMonitorStatusOnline ServiceMonitorStatus = "online" ServiceMonitorStatusOffline ServiceMonitorStatus = "offline" ServiceMonitorStatusError ServiceMonitorStatus = "error" ) // ServiceMonitor defines an object in Service_Monitor table type ServiceMonitor struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` IP string `ovsdb:"ip"` LogicalPort string `ovsdb:"logical_port"` Options map[string]string `ovsdb:"options"` Port int `ovsdb:"port" validate:"min=0,max=65535"` Protocol *ServiceMonitorProtocol `ovsdb:"protocol" validate:"omitempty,oneof='tcp' 'udp'"` SrcIP string `ovsdb:"src_ip"` SrcMAC string `ovsdb:"src_mac"` Status *ServiceMonitorStatus `ovsdb:"status" validate:"omitempty,oneof='online' 'offline' 'error'"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/ssl.go000066400000000000000000000011451517523235500240610ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const SSLTable = "SSL" // SSL defines an object in SSL table type SSL struct { UUID string `ovsdb:"_uuid"` BootstrapCaCert bool `ovsdb:"bootstrap_ca_cert"` CaCert string `ovsdb:"ca_cert"` Certificate string `ovsdb:"certificate"` ExternalIDs map[string]string `ovsdb:"external_ids"` PrivateKey string `ovsdb:"private_key"` SSLCiphers string `ovsdb:"ssl_ciphers"` SSLProtocols string `ovsdb:"ssl_protocols"` } incus-7.0.0/internal/server/network/ovn/schema/ovn-sb/static_mac_binding.go000066400000000000000000000007711517523235500270650ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const StaticMACBindingTable = "Static_MAC_Binding" // StaticMACBinding defines an object in Static_MAC_Binding table type StaticMACBinding struct { UUID string `ovsdb:"_uuid"` Datapath string `ovsdb:"datapath"` IP string `ovsdb:"ip"` LogicalPort string `ovsdb:"logical_port"` MAC string `ovsdb:"mac"` OverrideDynamicMAC bool `ovsdb:"override_dynamic_mac"` } incus-7.0.0/internal/server/network/ovn/utils.go000066400000000000000000000006041517523235500217530ustar00rootroot00000000000000package ovn import ( "strconv" "strings" ) // unquote passes s through strconv.Unquote if the first character is a ", otherwise returns s unmodified. // This is useful as openvswitch's tools can sometimes return values double quoted if they start with a number. func unquote(s string) (string, error) { if strings.HasPrefix(s, `"`) { return strconv.Unquote(s) } return s, nil } incus-7.0.0/internal/server/network/ovs/000077500000000000000000000000001517523235500202715ustar00rootroot00000000000000incus-7.0.0/internal/server/network/ovs/errors.go000066400000000000000000000006561517523235500221430ustar00rootroot00000000000000package ovs import ( "errors" ovsdbClient "github.com/ovn-kubernetes/libovsdb/client" ) // ErrExists indicates that a DB record already exists. var ErrExists = errors.New("object already exists") // ErrNotFound indicates that a DB record doesn't exist. var ErrNotFound = ovsdbClient.ErrNotFound // ErrNotManaged indicates that a DB record wasn't created by Incus. var ErrNotManaged = errors.New("object not incus-managed") incus-7.0.0/internal/server/network/ovs/ovs.go000066400000000000000000000031741517523235500214340ustar00rootroot00000000000000package ovs import ( "context" "errors" "runtime" "time" "github.com/cenkalti/backoff/v5" "github.com/go-logr/logr" ovsdbClient "github.com/ovn-kubernetes/libovsdb/client" ovsSwitch "github.com/lxc/incus/v7/internal/server/network/ovs/schema/ovs" ) // VSwitch client. type VSwitch struct { client ovsdbClient.Client cookie ovsdbClient.MonitorCookie rootUUID string } // NewVSwitch initializes a new vSwitch client.. func NewVSwitch(dbAddr string) (*VSwitch, error) { // Prepare the OVSDB client. dbSchema, err := ovsSwitch.FullDatabaseModel() if err != nil { return nil, err } discard := logr.Discard() options := []ovsdbClient.Option{ ovsdbClient.WithLogger(&discard), ovsdbClient.WithEndpoint(dbAddr), ovsdbClient.WithReconnect(5*time.Second, &backoff.ZeroBackOff{}), } // Connect to OVSDB. ovs, err := ovsdbClient.NewOVSDBClient(dbSchema, options...) if err != nil { return nil, err } err = ovs.Connect(context.TODO()) if err != nil { return nil, err } err = ovs.Echo(context.TODO()) if err != nil { return nil, err } monitorCookie, err := ovs.MonitorAll(context.TODO()) if err != nil { return nil, err } // Create the SB struct. client := &VSwitch{ client: ovs, cookie: monitorCookie, } // Set finalizer to stop the monitor. runtime.SetFinalizer(client, func(o *VSwitch) { _ = ovs.MonitorCancel(context.Background(), o.cookie) ovs.Close() }) // Get the root UUID. rows := ovs.Cache().Table("Open_vSwitch").Rows() if len(rows) != 1 { return nil, errors.New("Cannot find the OVS root switch") } for uuid := range rows { client.rootUUID = uuid } return client, nil } incus-7.0.0/internal/server/network/ovs/ovs_actions.go000066400000000000000000000374671517523235500231700ustar00rootroot00000000000000package ovs import ( "context" "errors" "fmt" "net" "slices" "strings" "sync" "time" ovsdbClient "github.com/ovn-kubernetes/libovsdb/client" ovsdbModel "github.com/ovn-kubernetes/libovsdb/model" "github.com/ovn-kubernetes/libovsdb/ovsdb" "github.com/lxc/incus/v7/internal/server/ip" ovsSwitch "github.com/lxc/incus/v7/internal/server/network/ovs/schema/ovs" "github.com/lxc/incus/v7/shared/util" ) // ovnBridgeMappingMutex locks access to read/write external-ids:ovn-bridge-mappings. var ovnBridgeMappingMutex sync.Mutex // GetBridge returns a bridge entry. func (o *VSwitch) GetBridge(ctx context.Context, bridgeName string) (*ovsSwitch.Bridge, error) { bridge := &ovsSwitch.Bridge{Name: bridgeName} err := o.client.Get(ctx, bridge) if err != nil { return nil, err } return bridge, nil } // CreateBridge adds a new bridge. func (o *VSwitch) CreateBridge(ctx context.Context, bridgeName string, mayExist bool, hwaddr net.HardwareAddr, mtu uint32) error { // Create interface. iface := ovsSwitch.Interface{ UUID: "interface", Name: bridgeName, } if mtu > 0 { mtu := int(mtu) iface.MTURequest = &mtu } interfaceOps, err := o.client.Create(&iface) if err != nil { return err } // Create port. port := ovsSwitch.Port{ UUID: "port", Name: bridgeName, Interfaces: []string{iface.UUID}, } portOps, err := o.client.Create(&port) if err != nil { return err } // Create bridge. bridge := ovsSwitch.Bridge{ UUID: "bridge", Name: bridgeName, Ports: []string{port.UUID}, } if hwaddr != nil { bridge.OtherConfig = map[string]string{"hwaddr": hwaddr.String()} } bridgeOps, err := o.client.Create(&bridge) if err != nil { return err } if mayExist { err = o.client.Get(ctx, &bridge) if err != nil && !errors.Is(err, ovsdbClient.ErrNotFound) { return err } if bridge.UUID != "bridge" { // Bridge already exists. return nil } } // Create switch entry. ovsRow := ovsSwitch.OpenvSwitch{ UUID: o.rootUUID, } mutateOps, err := o.client.Where(&ovsRow).Mutate(&ovsRow, ovsdbModel.Mutation{ Field: &ovsRow.Bridges, Mutator: ovsdb.MutateOperationInsert, Value: []string{bridge.UUID}, }) if err != nil { return err } operations := append(interfaceOps, portOps...) operations = append(operations, bridgeOps...) operations = append(operations, mutateOps...) resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } // Wait for kernel interface to appear. for range 50 { time.Sleep(100 * time.Millisecond) if util.PathExists(fmt.Sprintf("/sys/class/net/%s", bridgeName)) { return nil } } return errors.New("Bridge interface failed to appear") } // DeleteBridge deletes a bridge. func (o *VSwitch) DeleteBridge(ctx context.Context, bridgeName string) error { bridge := ovsSwitch.Bridge{ Name: bridgeName, } err := o.client.Get(ctx, &bridge) if err != nil { return err } ovsRow := ovsSwitch.OpenvSwitch{ UUID: o.rootUUID, } operations, err := o.client.Where(&ovsRow).Mutate(&ovsRow, ovsdbModel.Mutation{ Field: &ovsRow.Bridges, Mutator: "delete", Value: []string{bridge.UUID}, }) if err != nil { return err } resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // CreateBridgePort adds a port to the bridge. func (o *VSwitch) CreateBridgePort(ctx context.Context, bridgeName string, portName string, mayExist bool) error { // Get the bridge. bridge := ovsSwitch.Bridge{ Name: bridgeName, } err := o.client.Get(ctx, &bridge) if err != nil { return err } // Create the interface. iface := ovsSwitch.Interface{ UUID: "interface", Name: portName, } interfaceOps, err := o.client.Create(&iface) if err != nil { return err } // Create the port. port := ovsSwitch.Port{ Name: portName, } err = o.client.Get(ctx, &port) if err != nil && !errors.Is(err, ovsdbClient.ErrNotFound) { return err } if port.UUID != "" { if mayExist { // Already exists. return nil } return fmt.Errorf("OVS port %q already exists on %q", portName, bridgeName) } port.UUID = "port" port.Interfaces = []string{iface.UUID} portOps, err := o.client.Create(&port) if err != nil { return err } // Create the bridge port entry. mutateOps, err := o.client.Where(&bridge).Mutate(&bridge, ovsdbModel.Mutation{ Field: &bridge.Ports, Mutator: ovsdb.MutateOperationInsert, Value: []string{port.UUID}, }) if err != nil { return err } operations := append(interfaceOps, portOps...) operations = append(operations, mutateOps...) resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // DeleteBridgePort deletes a port from the bridge (if already detached does nothing). func (o *VSwitch) DeleteBridgePort(ctx context.Context, bridgeName string, portName string) error { operations := []ovsdb.Operation{} // Get the bridge port. bridgePort := ovsSwitch.Port{ Name: string(portName), } err := o.client.Get(ctx, &bridgePort) if err != nil { // Logical switch port is already gone. if errors.Is(err, ErrNotFound) { return nil } return err } // Remove the port from the bridge. bridge := ovsSwitch.Bridge{ Name: string(bridgeName), } updateOps, err := o.client.Where(&bridge).Mutate(&bridge, ovsdbModel.Mutation{ Field: &bridge.Ports, Mutator: ovsdb.MutateOperationDelete, Value: []string{bridgePort.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) // Delete the port itself. deleteOps, err := o.client.Where(&bridgePort).Delete() if err != nil { return err } operations = append(operations, deleteOps...) // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // UpdateBridgePortVLANs sets the VLAN mode and VLAN IDs on the port. func (o *VSwitch) UpdateBridgePortVLANs(ctx context.Context, portName string, mode string, tag int, trunks []int) error { // Get the port. port := &ovsSwitch.Port{ Name: portName, } err := o.client.Get(ctx, port) if err != nil { return err } // Set the options. if mode != "" { port.VLANMode = &mode } else { port.VLANMode = nil } if tag > 0 { port.Tag = &tag } else { port.Tag = nil } port.Trunks = trunks // Update the record. operations, err := o.client.Where(port).Update(port) if err != nil { return err } resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // AssociateInterfaceOVNSwitchPort removes any existing switch ports associated to the specified ovnSwitchPortName // and then associates the specified interfaceName to the OVN switch port. func (o *VSwitch) AssociateInterfaceOVNSwitchPort(ctx context.Context, interfaceName string, ovnSwitchPortName string) error { // Get the interfaces. interfaceList := []ovsSwitch.Interface{} err := o.client.WhereCache(func(iface *ovsSwitch.Interface) bool { return iface.ExternalIDs["iface-id"] == string(ovnSwitchPortName) }).List(ctx, &interfaceList) if err != nil { return err } // Delete the matching ports. for _, iface := range interfaceList { // Get the port. portList := []ovsSwitch.Port{} err := o.client.WhereCache(func(port *ovsSwitch.Port) bool { return slices.Contains(port.Interfaces, iface.UUID) }).List(ctx, &portList) if err != nil { return err } if len(portList) != 1 { return fmt.Errorf("Failed to find port for interface %q", iface.Name) } port := portList[0] // Get the bridge. bridgeList := []ovsSwitch.Bridge{} err = o.client.WhereCache(func(bridge *ovsSwitch.Bridge) bool { return slices.Contains(bridge.Ports, port.UUID) }).List(ctx, &bridgeList) if err != nil { return err } if len(bridgeList) != 1 { return fmt.Errorf("Failed to find bridge for port %q", portList[0].Name) } bridge := bridgeList[0] // Update the records. var operations []ovsdb.Operation // Delete the bridge port. updateOps, err := o.client.Where(&bridge).Mutate(&bridge, ovsdbModel.Mutation{ Field: &bridge.Ports, Mutator: ovsdb.MutateOperationDelete, Value: []string{port.UUID}, }) if err != nil { return err } operations = append(operations, updateOps...) // Delete the port. deleteOps, err := o.client.Where(&port).Delete() if err != nil { return err } operations = append(operations, deleteOps...) // Delete the interface. deleteOps, err = o.client.Where(&iface).Delete() if err != nil { return err } operations = append(operations, deleteOps...) // Apply the changes. resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } // Attempt to remove port, but don't fail if doesn't exist or can't be removed, at least // the switch association has been successfully removed, so the new port being added next // won't fail to work properly. link := &ip.Link{Name: iface.Name} _ = link.Delete() } // Get the new interface. ovsInterface := &ovsSwitch.Interface{ Name: interfaceName, } err = o.client.Get(ctx, ovsInterface) if err != nil { return err } // Update the record. if ovsInterface.ExternalIDs == nil { ovsInterface.ExternalIDs = map[string]string{} } ovsInterface.ExternalIDs["iface-id"] = string(ovnSwitchPortName) operations, err := o.client.Where(ovsInterface).Update(ovsInterface) if err != nil { return err } resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // GetInterfaceAssociatedOVNSwitchPort returns the OVN switch port associated to the interface. func (o *VSwitch) GetInterfaceAssociatedOVNSwitchPort(ctx context.Context, interfaceName string) (string, error) { // Get the OVS interface. ovsInterface := ovsSwitch.Interface{ Name: interfaceName, } err := o.client.Get(ctx, &ovsInterface) if err != nil { return "", err } // Return the iface-id. return ovsInterface.ExternalIDs["iface-id"], nil } // GetChassisID returns the local chassis ID. func (o *VSwitch) GetChassisID(ctx context.Context) (string, error) { // Get the root switch. vSwitch := &ovsSwitch.OpenvSwitch{ UUID: o.rootUUID, } err := o.client.Get(ctx, vSwitch) if err != nil { return "", err } // Return the system-id. return vSwitch.ExternalIDs["system-id"], nil } // GetOVNEncapIP returns the enscapsulation IP used for OVN underlay tunnels. func (o *VSwitch) GetOVNEncapIP(ctx context.Context) (net.IP, error) { // Get the root switch. vSwitch := &ovsSwitch.OpenvSwitch{ UUID: o.rootUUID, } err := o.client.Get(ctx, vSwitch) if err != nil { return nil, err } // Return the system-id. encapIP := net.ParseIP(vSwitch.ExternalIDs["ovn-encap-ip"]) if encapIP == nil { return nil, fmt.Errorf("Invalid ovn-encap-ip address %q", vSwitch.ExternalIDs["ovn-encap-ip"]) } return encapIP, nil } // GetOVNBridgeMappings gets the current OVN bridge mappings. func (o *VSwitch) GetOVNBridgeMappings(ctx context.Context, bridgeName string) ([]string, error) { // Get the root switch. vSwitch := &ovsSwitch.OpenvSwitch{ UUID: o.rootUUID, } err := o.client.Get(ctx, vSwitch) if err != nil { return nil, err } // Return the bridge mappings. val := vSwitch.ExternalIDs["ovn-bridge-mappings"] if val == "" { return []string{}, nil } return strings.SplitN(val, ",", -1), nil } // AddOVNBridgeMapping appends an OVN bridge mapping between a bridge and the logical provider name. func (o *VSwitch) AddOVNBridgeMapping(ctx context.Context, bridgeName string, providerName string) error { // Prevent concurrent changes. ovnBridgeMappingMutex.Lock() defer ovnBridgeMappingMutex.Unlock() // Get the root switch. vSwitch := &ovsSwitch.OpenvSwitch{ UUID: o.rootUUID, } err := o.client.Get(ctx, vSwitch) if err != nil { return err } // Get the current bridge mappings. val := vSwitch.ExternalIDs["ovn-bridge-mappings"] var mappings []string if val != "" { mappings = strings.SplitN(val, ",", -1) } else { mappings = []string{} } // Check if the mapping is already present. newMapping := fmt.Sprintf("%s:%s", providerName, bridgeName) if slices.Contains(mappings, newMapping) { return nil // Mapping is already present, nothing to do. } // Add the new mapping. mappings = append(mappings, newMapping) if vSwitch.ExternalIDs == nil { vSwitch.ExternalIDs = map[string]string{} } vSwitch.ExternalIDs["ovn-bridge-mappings"] = strings.Join(mappings, ",") // Update the record. operations, err := o.client.Where(vSwitch).Update(vSwitch) if err != nil { return err } resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // RemoveOVNBridgeMapping deletes an OVN bridge mapping between a bridge and the logical provider name. func (o *VSwitch) RemoveOVNBridgeMapping(ctx context.Context, bridgeName string, providerName string) error { // Prevent concurrent changes. ovnBridgeMappingMutex.Lock() defer ovnBridgeMappingMutex.Unlock() // Get the root switch. vSwitch := &ovsSwitch.OpenvSwitch{ UUID: o.rootUUID, } err := o.client.Get(ctx, vSwitch) if err != nil { return err } // Get the current bridge mappings. val := vSwitch.ExternalIDs["ovn-bridge-mappings"] mappings := strings.SplitN(val, ",", -1) newMappings := []string{} // Remove the mapping from the list. currentMapping := fmt.Sprintf("%s:%s", providerName, bridgeName) for _, mapping := range mappings { if mapping == currentMapping { continue } newMappings = append(newMappings, mapping) } // If no more mappings, remove the key completely. if len(newMappings) == 0 { delete(vSwitch.ExternalIDs, "ovn-bridge-mappings") } // Update the record. operations, err := o.client.Where(vSwitch).Update(vSwitch) if err != nil { return err } resp, err := o.client.Transact(ctx, operations...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(resp, operations) if err != nil { return err } return nil } // GetBridgePorts returns a list of ports that are connected to the bridge. func (o *VSwitch) GetBridgePorts(ctx context.Context, bridgeName string) ([]string, error) { // Get the bridge. bridge := &ovsSwitch.Bridge{ Name: bridgeName, } err := o.client.Get(ctx, bridge) if err != nil { return nil, err } // Get the ports. portNames := make([]string, 0, len(bridge.Ports)) for _, portUUID := range bridge.Ports { port := &ovsSwitch.Port{ UUID: portUUID, } err = o.client.Get(ctx, port) if err != nil { return nil, err } portNames = append(portNames, port.Name) } return portNames, nil } // GetHardwareOffload returns true if hardware offloading is enabled. func (o *VSwitch) GetHardwareOffload(ctx context.Context) (bool, error) { // Get the root switch. vSwitch := &ovsSwitch.OpenvSwitch{ UUID: o.rootUUID, } err := o.client.Get(ctx, vSwitch) if err != nil { return false, err } // Return the hw-offload state. return vSwitch.OtherConfig["hw-offload"] == "true", nil } // GetOVNSouthboundDBRemoteAddress gets the address of the southbound ovn database. func (o *VSwitch) GetOVNSouthboundDBRemoteAddress(ctx context.Context) (string, error) { vSwitch := &ovsSwitch.OpenvSwitch{ UUID: o.rootUUID, } err := o.client.Get(ctx, vSwitch) if err != nil { return "", err } val := vSwitch.ExternalIDs["ovn-remote"] return val, nil } incus-7.0.0/internal/server/network/ovs/schema/000077500000000000000000000000001517523235500215315ustar00rootroot00000000000000incus-7.0.0/internal/server/network/ovs/schema/ovs/000077500000000000000000000000001517523235500223405ustar00rootroot00000000000000incus-7.0.0/internal/server/network/ovs/schema/ovs/autoattach.go000066400000000000000000000007221517523235500250250ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const AutoAttachTable = "AutoAttach" // AutoAttach defines an object in AutoAttach table type AutoAttach struct { UUID string `ovsdb:"_uuid"` Mappings map[int]int `ovsdb:"mappings" validate:"dive,keys,min=0,max=16777215,endkeys,min=0,max=4095"` SystemDescription string `ovsdb:"system_description"` SystemName string `ovsdb:"system_name"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/bridge.go000066400000000000000000000043741517523235500241330ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const BridgeTable = "Bridge" type ( BridgeFailMode = string BridgeProtocols = string ) var ( BridgeFailModeStandalone BridgeFailMode = "standalone" BridgeFailModeSecure BridgeFailMode = "secure" BridgeProtocolsOpenflow10 BridgeProtocols = "OpenFlow10" BridgeProtocolsOpenflow11 BridgeProtocols = "OpenFlow11" BridgeProtocolsOpenflow12 BridgeProtocols = "OpenFlow12" BridgeProtocolsOpenflow13 BridgeProtocols = "OpenFlow13" BridgeProtocolsOpenflow14 BridgeProtocols = "OpenFlow14" BridgeProtocolsOpenflow15 BridgeProtocols = "OpenFlow15" ) // Bridge defines an object in Bridge table type Bridge struct { UUID string `ovsdb:"_uuid"` AutoAttach *string `ovsdb:"auto_attach"` Controller []string `ovsdb:"controller"` DatapathID *string `ovsdb:"datapath_id"` DatapathType string `ovsdb:"datapath_type"` DatapathVersion string `ovsdb:"datapath_version"` ExternalIDs map[string]string `ovsdb:"external_ids"` FailMode *BridgeFailMode `ovsdb:"fail_mode" validate:"omitempty,oneof='standalone' 'secure'"` FloodVLANs []int `ovsdb:"flood_vlans" validate:"max=4096,dive,min=0,max=4095"` FlowTables map[int]string `ovsdb:"flow_tables" validate:"dive,keys,min=0,max=254"` IPFIX *string `ovsdb:"ipfix"` McastSnoopingEnable bool `ovsdb:"mcast_snooping_enable"` Mirrors []string `ovsdb:"mirrors"` Name string `ovsdb:"name"` Netflow *string `ovsdb:"netflow"` OtherConfig map[string]string `ovsdb:"other_config"` Ports []string `ovsdb:"ports"` Protocols []BridgeProtocols `ovsdb:"protocols" validate:"dive,oneof='OpenFlow10' 'OpenFlow11' 'OpenFlow12' 'OpenFlow13' 'OpenFlow14' 'OpenFlow15'"` RSTPEnable bool `ovsdb:"rstp_enable"` RSTPStatus map[string]string `ovsdb:"rstp_status"` Sflow *string `ovsdb:"sflow"` Status map[string]string `ovsdb:"status"` STPEnable bool `ovsdb:"stp_enable"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/controller.go000066400000000000000000000045101517523235500250520ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ControllerTable = "Controller" type ( ControllerConnectionMode = string ControllerRole = string ControllerType = string ) var ( ControllerConnectionModeInBand ControllerConnectionMode = "in-band" ControllerConnectionModeOutOfBand ControllerConnectionMode = "out-of-band" ControllerRoleOther ControllerRole = "other" ControllerRoleMaster ControllerRole = "master" ControllerRoleSlave ControllerRole = "slave" ControllerTypePrimary ControllerType = "primary" ControllerTypeService ControllerType = "service" ) // Controller defines an object in Controller table type Controller struct { UUID string `ovsdb:"_uuid"` ConnectionMode *ControllerConnectionMode `ovsdb:"connection_mode" validate:"omitempty,oneof='in-band' 'out-of-band'"` ControllerBurstLimit *int `ovsdb:"controller_burst_limit" validate:"omitempty,min=25"` ControllerQueueSize *int `ovsdb:"controller_queue_size" validate:"omitempty,min=1,max=512"` ControllerRateLimit *int `ovsdb:"controller_rate_limit" validate:"omitempty,min=100"` EnableAsyncMessages *bool `ovsdb:"enable_async_messages"` ExternalIDs map[string]string `ovsdb:"external_ids"` InactivityProbe *int `ovsdb:"inactivity_probe"` IsConnected bool `ovsdb:"is_connected"` LocalGateway *string `ovsdb:"local_gateway"` LocalIP *string `ovsdb:"local_ip"` LocalNetmask *string `ovsdb:"local_netmask"` MaxBackoff *int `ovsdb:"max_backoff" validate:"omitempty,min=1000"` OtherConfig map[string]string `ovsdb:"other_config"` Role *ControllerRole `ovsdb:"role" validate:"omitempty,oneof='other' 'master' 'slave'"` Status map[string]string `ovsdb:"status"` Target string `ovsdb:"target"` Type *ControllerType `ovsdb:"type" validate:"omitempty,oneof='primary' 'service'"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/ct_timeout_policy.go000066400000000000000000000037061517523235500264300ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const CTTimeoutPolicyTable = "CT_Timeout_Policy" type ( CTTimeoutPolicyTimeouts = string ) var ( CTTimeoutPolicyTimeoutsTCPSynSent CTTimeoutPolicyTimeouts = "tcp_syn_sent" CTTimeoutPolicyTimeoutsTCPSynRecv CTTimeoutPolicyTimeouts = "tcp_syn_recv" CTTimeoutPolicyTimeoutsTCPEstablished CTTimeoutPolicyTimeouts = "tcp_established" CTTimeoutPolicyTimeoutsTCPFinWait CTTimeoutPolicyTimeouts = "tcp_fin_wait" CTTimeoutPolicyTimeoutsTCPCloseWait CTTimeoutPolicyTimeouts = "tcp_close_wait" CTTimeoutPolicyTimeoutsTCPLastAck CTTimeoutPolicyTimeouts = "tcp_last_ack" CTTimeoutPolicyTimeoutsTCPTimeWait CTTimeoutPolicyTimeouts = "tcp_time_wait" CTTimeoutPolicyTimeoutsTCPClose CTTimeoutPolicyTimeouts = "tcp_close" CTTimeoutPolicyTimeoutsTCPSynSent2 CTTimeoutPolicyTimeouts = "tcp_syn_sent2" CTTimeoutPolicyTimeoutsTCPRetransmit CTTimeoutPolicyTimeouts = "tcp_retransmit" CTTimeoutPolicyTimeoutsTCPUnack CTTimeoutPolicyTimeouts = "tcp_unack" CTTimeoutPolicyTimeoutsUDPFirst CTTimeoutPolicyTimeouts = "udp_first" CTTimeoutPolicyTimeoutsUDPSingle CTTimeoutPolicyTimeouts = "udp_single" CTTimeoutPolicyTimeoutsUDPMultiple CTTimeoutPolicyTimeouts = "udp_multiple" CTTimeoutPolicyTimeoutsICMPFirst CTTimeoutPolicyTimeouts = "icmp_first" CTTimeoutPolicyTimeoutsICMPReply CTTimeoutPolicyTimeouts = "icmp_reply" ) // CTTimeoutPolicy defines an object in CT_Timeout_Policy table type CTTimeoutPolicy struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` Timeouts map[string]int `ovsdb:"timeouts" validate:"dive,keys,oneof='tcp_syn_sent' 'tcp_syn_recv' 'tcp_established' 'tcp_fin_wait' 'tcp_close_wait' 'tcp_last_ack' 'tcp_time_wait' 'tcp_close' 'tcp_syn_sent2' 'tcp_retransmit' 'tcp_unack' 'udp_first' 'udp_single' 'udp_multiple' 'icmp_first' 'icmp_reply',endkeys,min=0,max=4294967295"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/ct_zone.go000066400000000000000000000005221517523235500243270ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const CTZoneTable = "CT_Zone" // CTZone defines an object in CT_Zone table type CTZone struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` TimeoutPolicy *string `ovsdb:"timeout_policy"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/datapath.go000066400000000000000000000007671517523235500244670ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const DatapathTable = "Datapath" // Datapath defines an object in Datapath table type Datapath struct { UUID string `ovsdb:"_uuid"` Capabilities map[string]string `ovsdb:"capabilities"` CTZones map[int]string `ovsdb:"ct_zones" validate:"dive,keys,min=0,max=65535"` DatapathVersion string `ovsdb:"datapath_version"` ExternalIDs map[string]string `ovsdb:"external_ids"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/flow_sample_collector_set.go000066400000000000000000000010231517523235500301140ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const FlowSampleCollectorSetTable = "Flow_Sample_Collector_Set" // FlowSampleCollectorSet defines an object in Flow_Sample_Collector_Set table type FlowSampleCollectorSet struct { UUID string `ovsdb:"_uuid"` Bridge string `ovsdb:"bridge"` ExternalIDs map[string]string `ovsdb:"external_ids"` ID int `ovsdb:"id" validate:"min=0,max=4294967295"` IPFIX *string `ovsdb:"ipfix"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/flow_table.go000066400000000000000000000015661517523235500250150ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const FlowTableTable = "Flow_Table" type ( FlowTableOverflowPolicy = string ) var ( FlowTableOverflowPolicyRefuse FlowTableOverflowPolicy = "refuse" FlowTableOverflowPolicyEvict FlowTableOverflowPolicy = "evict" ) // FlowTable defines an object in Flow_Table table type FlowTable struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` FlowLimit *int `ovsdb:"flow_limit" validate:"omitempty,min=0"` Groups []string `ovsdb:"groups"` Name *string `ovsdb:"name"` OverflowPolicy *FlowTableOverflowPolicy `ovsdb:"overflow_policy" validate:"omitempty,oneof='refuse' 'evict'"` Prefixes []string `ovsdb:"prefixes" validate:"max=3"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/interface.go000066400000000000000000000067701517523235500246410ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const InterfaceTable = "Interface" type ( InterfaceAdminState = string InterfaceCFMRemoteOpstate = string InterfaceDuplex = string InterfaceLinkState = string ) var ( InterfaceAdminStateUp InterfaceAdminState = "up" InterfaceAdminStateDown InterfaceAdminState = "down" InterfaceCFMRemoteOpstateUp InterfaceCFMRemoteOpstate = "up" InterfaceCFMRemoteOpstateDown InterfaceCFMRemoteOpstate = "down" InterfaceDuplexHalf InterfaceDuplex = "half" InterfaceDuplexFull InterfaceDuplex = "full" InterfaceLinkStateUp InterfaceLinkState = "up" InterfaceLinkStateDown InterfaceLinkState = "down" ) // Interface defines an object in Interface table type Interface struct { UUID string `ovsdb:"_uuid"` AdminState *InterfaceAdminState `ovsdb:"admin_state" validate:"omitempty,oneof='up' 'down'"` BFD map[string]string `ovsdb:"bfd"` BFDStatus map[string]string `ovsdb:"bfd_status"` CFMFault *bool `ovsdb:"cfm_fault"` CFMFaultStatus []string `ovsdb:"cfm_fault_status"` CFMFlapCount *int `ovsdb:"cfm_flap_count"` CFMHealth *int `ovsdb:"cfm_health" validate:"omitempty,min=0,max=100"` CFMMpid *int `ovsdb:"cfm_mpid"` CFMRemoteMpids []int `ovsdb:"cfm_remote_mpids"` CFMRemoteOpstate *InterfaceCFMRemoteOpstate `ovsdb:"cfm_remote_opstate" validate:"omitempty,oneof='up' 'down'"` Duplex *InterfaceDuplex `ovsdb:"duplex" validate:"omitempty,oneof='half' 'full'"` Error *string `ovsdb:"error"` ExternalIDs map[string]string `ovsdb:"external_ids"` Ifindex *int `ovsdb:"ifindex" validate:"omitempty,min=0,max=4294967295"` IngressPolicingBurst int `ovsdb:"ingress_policing_burst" validate:"min=0"` IngressPolicingRate int `ovsdb:"ingress_policing_rate" validate:"min=0"` LACPCurrent *bool `ovsdb:"lacp_current"` LinkResets *int `ovsdb:"link_resets"` LinkSpeed *int `ovsdb:"link_speed"` LinkState *InterfaceLinkState `ovsdb:"link_state" validate:"omitempty,oneof='up' 'down'"` LLDP map[string]string `ovsdb:"lldp"` MAC *string `ovsdb:"mac"` MACInUse *string `ovsdb:"mac_in_use"` MTU *int `ovsdb:"mtu"` MTURequest *int `ovsdb:"mtu_request" validate:"omitempty,min=1"` Name string `ovsdb:"name"` Ofport *int `ovsdb:"ofport"` OfportRequest *int `ovsdb:"ofport_request" validate:"omitempty,min=1,max=65279"` Options map[string]string `ovsdb:"options"` OtherConfig map[string]string `ovsdb:"other_config"` Statistics map[string]int `ovsdb:"statistics"` Status map[string]string `ovsdb:"status"` Type string `ovsdb:"type"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/ipfix.go000066400000000000000000000016251517523235500240120ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const IPFIXTable = "IPFIX" // IPFIX defines an object in IPFIX table type IPFIX struct { UUID string `ovsdb:"_uuid"` CacheActiveTimeout *int `ovsdb:"cache_active_timeout" validate:"omitempty,min=0,max=4200"` CacheMaxFlows *int `ovsdb:"cache_max_flows" validate:"omitempty,min=0,max=4294967295"` ExternalIDs map[string]string `ovsdb:"external_ids"` ObsDomainID *int `ovsdb:"obs_domain_id" validate:"omitempty,min=0,max=4294967295"` ObsPointID *int `ovsdb:"obs_point_id" validate:"omitempty,min=0,max=4294967295"` OtherConfig map[string]string `ovsdb:"other_config"` Sampling *int `ovsdb:"sampling" validate:"omitempty,min=1,max=4294967295"` Targets []string `ovsdb:"targets"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/manager.go000066400000000000000000000017501517523235500243040ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const ManagerTable = "Manager" type ( ManagerConnectionMode = string ) var ( ManagerConnectionModeInBand ManagerConnectionMode = "in-band" ManagerConnectionModeOutOfBand ManagerConnectionMode = "out-of-band" ) // Manager defines an object in Manager table type Manager struct { UUID string `ovsdb:"_uuid"` ConnectionMode *ManagerConnectionMode `ovsdb:"connection_mode" validate:"omitempty,oneof='in-band' 'out-of-band'"` ExternalIDs map[string]string `ovsdb:"external_ids"` InactivityProbe *int `ovsdb:"inactivity_probe"` IsConnected bool `ovsdb:"is_connected"` MaxBackoff *int `ovsdb:"max_backoff" validate:"omitempty,min=1000"` OtherConfig map[string]string `ovsdb:"other_config"` Status map[string]string `ovsdb:"status"` Target string `ovsdb:"target"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/mirror.go000066400000000000000000000015621517523235500242050ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const MirrorTable = "Mirror" // Mirror defines an object in Mirror table type Mirror struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` Name string `ovsdb:"name"` OutputPort *string `ovsdb:"output_port"` OutputVLAN *int `ovsdb:"output_vlan" validate:"omitempty,min=1,max=4095"` SelectAll bool `ovsdb:"select_all"` SelectDstPort []string `ovsdb:"select_dst_port"` SelectSrcPort []string `ovsdb:"select_src_port"` SelectVLAN []int `ovsdb:"select_vlan" validate:"max=4096,dive,min=0,max=4095"` Snaplen *int `ovsdb:"snaplen" validate:"omitempty,min=14,max=65535"` Statistics map[string]int `ovsdb:"statistics"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/model.go000066400000000000000000001230521517523235500237720ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel import ( "encoding/json" "github.com/ovn-kubernetes/libovsdb/model" "github.com/ovn-kubernetes/libovsdb/ovsdb" ) // FullDatabaseModel returns the DatabaseModel object to be used in libovsdb func FullDatabaseModel() (model.ClientDBModel, error) { return model.NewClientDBModel("Open_vSwitch", map[string]model.Model{ "AutoAttach": &AutoAttach{}, "Bridge": &Bridge{}, "CT_Timeout_Policy": &CTTimeoutPolicy{}, "CT_Zone": &CTZone{}, "Controller": &Controller{}, "Datapath": &Datapath{}, "Flow_Sample_Collector_Set": &FlowSampleCollectorSet{}, "Flow_Table": &FlowTable{}, "IPFIX": &IPFIX{}, "Interface": &Interface{}, "Manager": &Manager{}, "Mirror": &Mirror{}, "NetFlow": &NetFlow{}, "Open_vSwitch": &OpenvSwitch{}, "Port": &Port{}, "QoS": &QoS{}, "Queue": &Queue{}, "SSL": &SSL{}, "sFlow": &SFlow{}, }) } var schema = `{ "name": "Open_vSwitch", "version": "8.2.0", "tables": { "AutoAttach": { "columns": { "mappings": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 16777215 }, "value": { "type": "integer", "minInteger": 0, "maxInteger": 4095 }, "min": 0, "max": "unlimited" } }, "system_description": { "type": "string" }, "system_name": { "type": "string" } } }, "Bridge": { "columns": { "auto_attach": { "type": { "key": { "type": "uuid", "refTable": "AutoAttach" }, "min": 0, "max": 1 } }, "controller": { "type": { "key": { "type": "uuid", "refTable": "Controller" }, "min": 0, "max": "unlimited" } }, "datapath_id": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 }, "ephemeral": true }, "datapath_type": { "type": "string" }, "datapath_version": { "type": "string" }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "fail_mode": { "type": { "key": { "type": "string", "enum": [ "set", [ "standalone", "secure" ] ] }, "min": 0, "max": 1 } }, "flood_vlans": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4095 }, "min": 0, "max": 4096 } }, "flow_tables": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 254 }, "value": { "type": "uuid", "refTable": "Flow_Table" }, "min": 0, "max": "unlimited" } }, "ipfix": { "type": { "key": { "type": "uuid", "refTable": "IPFIX" }, "min": 0, "max": 1 } }, "mcast_snooping_enable": { "type": "boolean" }, "mirrors": { "type": { "key": { "type": "uuid", "refTable": "Mirror" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string", "mutable": false }, "netflow": { "type": { "key": { "type": "uuid", "refTable": "NetFlow" }, "min": 0, "max": 1 } }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ports": { "type": { "key": { "type": "uuid", "refTable": "Port" }, "min": 0, "max": "unlimited" } }, "protocols": { "type": { "key": { "type": "string", "enum": [ "set", [ "OpenFlow10", "OpenFlow11", "OpenFlow12", "OpenFlow13", "OpenFlow14", "OpenFlow15" ] ] }, "min": 0, "max": "unlimited" } }, "rstp_enable": { "type": "boolean" }, "rstp_status": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "sflow": { "type": { "key": { "type": "uuid", "refTable": "sFlow" }, "min": 0, "max": 1 } }, "status": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "stp_enable": { "type": "boolean" } }, "indexes": [ [ "name" ] ] }, "CT_Timeout_Policy": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "timeouts": { "type": { "key": { "type": "string", "enum": [ "set", [ "tcp_syn_sent", "tcp_syn_recv", "tcp_established", "tcp_fin_wait", "tcp_close_wait", "tcp_last_ack", "tcp_time_wait", "tcp_close", "tcp_syn_sent2", "tcp_retransmit", "tcp_unack", "udp_first", "udp_single", "udp_multiple", "icmp_first", "icmp_reply" ] ] }, "value": { "type": "integer", "minInteger": 0, "maxInteger": 4294967295 }, "min": 0, "max": "unlimited" } } } }, "CT_Zone": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "timeout_policy": { "type": { "key": { "type": "uuid", "refTable": "CT_Timeout_Policy" }, "min": 0, "max": 1 } } } }, "Controller": { "columns": { "connection_mode": { "type": { "key": { "type": "string", "enum": [ "set", [ "in-band", "out-of-band" ] ] }, "min": 0, "max": 1 } }, "controller_burst_limit": { "type": { "key": { "type": "integer", "minInteger": 25 }, "min": 0, "max": 1 } }, "controller_queue_size": { "type": { "key": { "type": "integer", "minInteger": 1, "maxInteger": 512 }, "min": 0, "max": 1 } }, "controller_rate_limit": { "type": { "key": { "type": "integer", "minInteger": 100 }, "min": 0, "max": 1 } }, "enable_async_messages": { "type": { "key": { "type": "boolean" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "inactivity_probe": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "is_connected": { "type": "boolean", "ephemeral": true }, "local_gateway": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "local_ip": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "local_netmask": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "max_backoff": { "type": { "key": { "type": "integer", "minInteger": 1000 }, "min": 0, "max": 1 } }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "role": { "type": { "key": { "type": "string", "enum": [ "set", [ "other", "master", "slave" ] ] }, "min": 0, "max": 1 }, "ephemeral": true }, "status": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "target": { "type": "string" }, "type": { "type": { "key": { "type": "string", "enum": [ "set", [ "primary", "service" ] ] }, "min": 0, "max": 1 } } } }, "Datapath": { "columns": { "capabilities": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ct_zones": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 65535 }, "value": { "type": "uuid", "refTable": "CT_Zone" }, "min": 0, "max": "unlimited" } }, "datapath_version": { "type": "string" }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } } } }, "Flow_Sample_Collector_Set": { "columns": { "bridge": { "type": { "key": { "type": "uuid", "refTable": "Bridge" }, "min": 1, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "id": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4294967295 }, "min": 1, "max": 1 } }, "ipfix": { "type": { "key": { "type": "uuid", "refTable": "IPFIX" }, "min": 0, "max": 1 } } }, "indexes": [ [ "id", "bridge" ] ], "isRoot": true }, "Flow_Table": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "flow_limit": { "type": { "key": { "type": "integer", "minInteger": 0 }, "min": 0, "max": 1 } }, "groups": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } }, "name": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "overflow_policy": { "type": { "key": { "type": "string", "enum": [ "set", [ "refuse", "evict" ] ] }, "min": 0, "max": 1 } }, "prefixes": { "type": { "key": { "type": "string" }, "min": 0, "max": 3 } } } }, "IPFIX": { "columns": { "cache_active_timeout": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4200 }, "min": 0, "max": 1 } }, "cache_max_flows": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4294967295 }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "obs_domain_id": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4294967295 }, "min": 0, "max": 1 } }, "obs_point_id": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4294967295 }, "min": 0, "max": 1 } }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "sampling": { "type": { "key": { "type": "integer", "minInteger": 1, "maxInteger": 4294967295 }, "min": 0, "max": 1 } }, "targets": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } } } }, "Interface": { "columns": { "admin_state": { "type": { "key": { "type": "string", "enum": [ "set", [ "up", "down" ] ] }, "min": 0, "max": 1 }, "ephemeral": true }, "bfd": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "bfd_status": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "cfm_fault": { "type": { "key": { "type": "boolean" }, "min": 0, "max": 1 }, "ephemeral": true }, "cfm_fault_status": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "cfm_flap_count": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "cfm_health": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 100 }, "min": 0, "max": 1 }, "ephemeral": true }, "cfm_mpid": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "cfm_remote_mpids": { "type": { "key": { "type": "integer" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "cfm_remote_opstate": { "type": { "key": { "type": "string", "enum": [ "set", [ "up", "down" ] ] }, "min": 0, "max": 1 }, "ephemeral": true }, "duplex": { "type": { "key": { "type": "string", "enum": [ "set", [ "half", "full" ] ] }, "min": 0, "max": 1 }, "ephemeral": true }, "error": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ifindex": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4294967295 }, "min": 0, "max": 1 }, "ephemeral": true }, "ingress_policing_burst": { "type": { "key": { "type": "integer", "minInteger": 0 } } }, "ingress_policing_rate": { "type": { "key": { "type": "integer", "minInteger": 0 } } }, "lacp_current": { "type": { "key": { "type": "boolean" }, "min": 0, "max": 1 }, "ephemeral": true }, "link_resets": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 }, "ephemeral": true }, "link_speed": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 }, "ephemeral": true }, "link_state": { "type": { "key": { "type": "string", "enum": [ "set", [ "up", "down" ] ] }, "min": 0, "max": 1 }, "ephemeral": true }, "lldp": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "mac": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "mac_in_use": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 }, "ephemeral": true }, "mtu": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 }, "ephemeral": true }, "mtu_request": { "type": { "key": { "type": "integer", "minInteger": 1 }, "min": 0, "max": 1 } }, "name": { "type": "string", "mutable": false }, "ofport": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "ofport_request": { "type": { "key": { "type": "integer", "minInteger": 1, "maxInteger": 65279 }, "min": 0, "max": 1 } }, "options": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "statistics": { "type": { "key": { "type": "string" }, "value": { "type": "integer" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "status": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "type": { "type": "string" } }, "indexes": [ [ "name" ] ] }, "Manager": { "columns": { "connection_mode": { "type": { "key": { "type": "string", "enum": [ "set", [ "in-band", "out-of-band" ] ] }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "inactivity_probe": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "is_connected": { "type": "boolean", "ephemeral": true }, "max_backoff": { "type": { "key": { "type": "integer", "minInteger": 1000 }, "min": 0, "max": 1 } }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "status": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "target": { "type": "string" } }, "indexes": [ [ "target" ] ] }, "Mirror": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "name": { "type": "string" }, "output_port": { "type": { "key": { "type": "uuid", "refTable": "Port", "refType": "weak" }, "min": 0, "max": 1 } }, "output_vlan": { "type": { "key": { "type": "integer", "minInteger": 1, "maxInteger": 4095 }, "min": 0, "max": 1 } }, "select_all": { "type": "boolean" }, "select_dst_port": { "type": { "key": { "type": "uuid", "refTable": "Port", "refType": "weak" }, "min": 0, "max": "unlimited" } }, "select_src_port": { "type": { "key": { "type": "uuid", "refTable": "Port", "refType": "weak" }, "min": 0, "max": "unlimited" } }, "select_vlan": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4095 }, "min": 0, "max": 4096 } }, "snaplen": { "type": { "key": { "type": "integer", "minInteger": 14, "maxInteger": 65535 }, "min": 0, "max": 1 } }, "statistics": { "type": { "key": { "type": "string" }, "value": { "type": "integer" }, "min": 0, "max": "unlimited" }, "ephemeral": true } } }, "NetFlow": { "columns": { "active_timeout": { "type": { "key": { "type": "integer", "minInteger": -1 } } }, "add_id_to_interface": { "type": "boolean" }, "engine_id": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 255 }, "min": 0, "max": 1 } }, "engine_type": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 255 }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "targets": { "type": { "key": { "type": "string" }, "min": 1, "max": "unlimited" } } } }, "Open_vSwitch": { "columns": { "bridges": { "type": { "key": { "type": "uuid", "refTable": "Bridge" }, "min": 0, "max": "unlimited" } }, "cur_cfg": { "type": "integer" }, "datapath_types": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } }, "datapaths": { "type": { "key": { "type": "string" }, "value": { "type": "uuid", "refTable": "Datapath" }, "min": 0, "max": "unlimited" } }, "db_version": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "dpdk_initialized": { "type": "boolean" }, "dpdk_version": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "iface_types": { "type": { "key": { "type": "string" }, "min": 0, "max": "unlimited" } }, "manager_options": { "type": { "key": { "type": "uuid", "refTable": "Manager" }, "min": 0, "max": "unlimited" } }, "next_cfg": { "type": "integer" }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "ovs_version": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "ssl": { "type": { "key": { "type": "uuid", "refTable": "SSL" }, "min": 0, "max": 1 } }, "statistics": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "system_type": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "system_version": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } } }, "isRoot": true }, "Port": { "columns": { "bond_active_slave": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "bond_downdelay": { "type": "integer" }, "bond_fake_iface": { "type": "boolean" }, "bond_mode": { "type": { "key": { "type": "string", "enum": [ "set", [ "balance-tcp", "balance-slb", "active-backup" ] ] }, "min": 0, "max": 1 } }, "bond_updelay": { "type": "integer" }, "cvlans": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4095 }, "min": 0, "max": 4096 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "fake_bridge": { "type": "boolean" }, "interfaces": { "type": { "key": { "type": "uuid", "refTable": "Interface" }, "min": 1, "max": "unlimited" } }, "lacp": { "type": { "key": { "type": "string", "enum": [ "set", [ "active", "passive", "off" ] ] }, "min": 0, "max": 1 } }, "mac": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "name": { "type": "string", "mutable": false }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "protected": { "type": "boolean" }, "qos": { "type": { "key": { "type": "uuid", "refTable": "QoS" }, "min": 0, "max": 1 } }, "rstp_statistics": { "type": { "key": { "type": "string" }, "value": { "type": "integer" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "rstp_status": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "statistics": { "type": { "key": { "type": "string" }, "value": { "type": "integer" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "status": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" }, "ephemeral": true }, "tag": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4095 }, "min": 0, "max": 1 } }, "trunks": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4095 }, "min": 0, "max": 4096 } }, "vlan_mode": { "type": { "key": { "type": "string", "enum": [ "set", [ "trunk", "access", "native-tagged", "native-untagged", "dot1q-tunnel" ] ] }, "min": 0, "max": 1 } } }, "indexes": [ [ "name" ] ] }, "QoS": { "columns": { "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "queues": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 4294967295 }, "value": { "type": "uuid", "refTable": "Queue" }, "min": 0, "max": "unlimited" } }, "type": { "type": "string" } }, "isRoot": true }, "Queue": { "columns": { "dscp": { "type": { "key": { "type": "integer", "minInteger": 0, "maxInteger": 63 }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "other_config": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } } }, "isRoot": true }, "SSL": { "columns": { "bootstrap_ca_cert": { "type": "boolean" }, "ca_cert": { "type": "string" }, "certificate": { "type": "string" }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "private_key": { "type": "string" } } }, "sFlow": { "columns": { "agent": { "type": { "key": { "type": "string" }, "min": 0, "max": 1 } }, "external_ids": { "type": { "key": { "type": "string" }, "value": { "type": "string" }, "min": 0, "max": "unlimited" } }, "header": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "polling": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "sampling": { "type": { "key": { "type": "integer" }, "min": 0, "max": 1 } }, "targets": { "type": { "key": { "type": "string" }, "min": 1, "max": "unlimited" } } } } } }` func Schema() ovsdb.DatabaseSchema { var s ovsdb.DatabaseSchema err := json.Unmarshal([]byte(schema), &s) if err != nil { panic(err) } return s } incus-7.0.0/internal/server/network/ovs/schema/ovs/netflow.go000066400000000000000000000012611517523235500243450ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const NetFlowTable = "NetFlow" // NetFlow defines an object in NetFlow table type NetFlow struct { UUID string `ovsdb:"_uuid"` ActiveTimeout int `ovsdb:"active_timeout" validate:"min=-1"` AddIDToInterface bool `ovsdb:"add_id_to_interface"` EngineID *int `ovsdb:"engine_id" validate:"omitempty,min=0,max=255"` EngineType *int `ovsdb:"engine_type" validate:"omitempty,min=0,max=255"` ExternalIDs map[string]string `ovsdb:"external_ids"` Targets []string `ovsdb:"targets" validate:"min=1"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/open_vswitch.go000066400000000000000000000022771517523235500254070ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const OpenvSwitchTable = "Open_vSwitch" // OpenvSwitch defines an object in Open_vSwitch table type OpenvSwitch struct { UUID string `ovsdb:"_uuid"` Bridges []string `ovsdb:"bridges"` CurCfg int `ovsdb:"cur_cfg"` DatapathTypes []string `ovsdb:"datapath_types"` Datapaths map[string]string `ovsdb:"datapaths"` DbVersion *string `ovsdb:"db_version"` DpdkInitialized bool `ovsdb:"dpdk_initialized"` DpdkVersion *string `ovsdb:"dpdk_version"` ExternalIDs map[string]string `ovsdb:"external_ids"` IfaceTypes []string `ovsdb:"iface_types"` ManagerOptions []string `ovsdb:"manager_options"` NextCfg int `ovsdb:"next_cfg"` OtherConfig map[string]string `ovsdb:"other_config"` OVSVersion *string `ovsdb:"ovs_version"` SSL *string `ovsdb:"ssl"` Statistics map[string]string `ovsdb:"statistics"` SystemType *string `ovsdb:"system_type"` SystemVersion *string `ovsdb:"system_version"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/port.go000066400000000000000000000046501517523235500236600ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const PortTable = "Port" type ( PortBondMode = string PortLACP = string PortVLANMode = string ) var ( PortBondModeBalanceTCP PortBondMode = "balance-tcp" PortBondModeBalanceSLB PortBondMode = "balance-slb" PortBondModeActiveBackup PortBondMode = "active-backup" PortLACPActive PortLACP = "active" PortLACPPassive PortLACP = "passive" PortLACPOff PortLACP = "off" PortVLANModeTrunk PortVLANMode = "trunk" PortVLANModeAccess PortVLANMode = "access" PortVLANModeNativeTagged PortVLANMode = "native-tagged" PortVLANModeNativeUntagged PortVLANMode = "native-untagged" PortVLANModeDot1qTunnel PortVLANMode = "dot1q-tunnel" ) // Port defines an object in Port table type Port struct { UUID string `ovsdb:"_uuid"` BondActiveSlave *string `ovsdb:"bond_active_slave"` BondDowndelay int `ovsdb:"bond_downdelay"` BondFakeIface bool `ovsdb:"bond_fake_iface"` BondMode *PortBondMode `ovsdb:"bond_mode" validate:"omitempty,oneof='balance-tcp' 'balance-slb' 'active-backup'"` BondUpdelay int `ovsdb:"bond_updelay"` CVLANs []int `ovsdb:"cvlans" validate:"max=4096,dive,min=0,max=4095"` ExternalIDs map[string]string `ovsdb:"external_ids"` FakeBridge bool `ovsdb:"fake_bridge"` Interfaces []string `ovsdb:"interfaces" validate:"min=1"` LACP *PortLACP `ovsdb:"lacp" validate:"omitempty,oneof='active' 'passive' 'off'"` MAC *string `ovsdb:"mac"` Name string `ovsdb:"name"` OtherConfig map[string]string `ovsdb:"other_config"` Protected bool `ovsdb:"protected"` QOS *string `ovsdb:"qos"` RSTPStatistics map[string]int `ovsdb:"rstp_statistics"` RSTPStatus map[string]string `ovsdb:"rstp_status"` Statistics map[string]int `ovsdb:"statistics"` Status map[string]string `ovsdb:"status"` Tag *int `ovsdb:"tag" validate:"omitempty,min=0,max=4095"` Trunks []int `ovsdb:"trunks" validate:"max=4096,dive,min=0,max=4095"` VLANMode *PortVLANMode `ovsdb:"vlan_mode" validate:"omitempty,oneof='trunk' 'access' 'native-tagged' 'native-untagged' 'dot1q-tunnel'"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/qos.go000066400000000000000000000007011517523235500234670ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const QoSTable = "QoS" // QoS defines an object in QoS table type QoS struct { UUID string `ovsdb:"_uuid"` ExternalIDs map[string]string `ovsdb:"external_ids"` OtherConfig map[string]string `ovsdb:"other_config"` Queues map[int]string `ovsdb:"queues" validate:"dive,keys,min=0,max=4294967295"` Type string `ovsdb:"type"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/queue.go000066400000000000000000000006231517523235500240140ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const QueueTable = "Queue" // Queue defines an object in Queue table type Queue struct { UUID string `ovsdb:"_uuid"` DSCP *int `ovsdb:"dscp" validate:"omitempty,min=0,max=63"` ExternalIDs map[string]string `ovsdb:"external_ids"` OtherConfig map[string]string `ovsdb:"other_config"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/sflow.go000066400000000000000000000010211517523235500240130ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const SFlowTable = "sFlow" // SFlow defines an object in sFlow table type SFlow struct { UUID string `ovsdb:"_uuid"` Agent *string `ovsdb:"agent"` ExternalIDs map[string]string `ovsdb:"external_ids"` Header *int `ovsdb:"header"` Polling *int `ovsdb:"polling"` Sampling *int `ovsdb:"sampling"` Targets []string `ovsdb:"targets" validate:"min=1"` } incus-7.0.0/internal/server/network/ovs/schema/ovs/ssl.go000066400000000000000000000007611517523235500234740ustar00rootroot00000000000000// Code generated by "libovsdb.modelgen" // DO NOT EDIT. package ovsmodel const SSLTable = "SSL" // SSL defines an object in SSL table type SSL struct { UUID string `ovsdb:"_uuid"` BootstrapCaCert bool `ovsdb:"bootstrap_ca_cert"` CaCert string `ovsdb:"ca_cert"` Certificate string `ovsdb:"certificate"` ExternalIDs map[string]string `ovsdb:"external_ids"` PrivateKey string `ovsdb:"private_key"` } incus-7.0.0/internal/server/network/ovs/utils.go000066400000000000000000000006041517523235500217600ustar00rootroot00000000000000package ovs import ( "strconv" "strings" ) // unquote passes s through strconv.Unquote if the first character is a ", otherwise returns s unmodified. // This is useful as openvswitch's tools can sometimes return values double quoted if they start with a number. func unquote(s string) (string, error) { if strings.HasPrefix(s, `"`) { return strconv.Unquote(s) } return s, nil } incus-7.0.0/internal/server/network/zone/000077500000000000000000000000001517523235500204355ustar00rootroot00000000000000incus-7.0.0/internal/server/network/zone/interface.go000066400000000000000000000020251517523235500227230ustar00rootroot00000000000000package zone import ( "strings" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" ) // NetworkZone represents a Network zone. type NetworkZone interface { // Initialize. init(state *state.State, id int64, projectName string, zoneInfo *api.NetworkZone) // Info. ID() int64 Project() string Info() *api.NetworkZone Etag() []any UsedBy() ([]string, error) Content() (*strings.Builder, error) SOA() (*strings.Builder, error) // Records. AddRecord(req api.NetworkZoneRecordsPost) error GetRecords() ([]api.NetworkZoneRecord, error) GetRecord(name string) (*api.NetworkZoneRecord, error) UpdateRecord(name string, req api.NetworkZoneRecordPut, clientType request.ClientType) error DeleteRecord(name string) error // Internal validation. validateName(name string) error validateConfig(config *api.NetworkZonePut) error // Modifications. Update(config *api.NetworkZonePut, clientType request.ClientType) error Delete() error } incus-7.0.0/internal/server/network/zone/load.go000066400000000000000000000121071517523235500217040ustar00rootroot00000000000000package zone import ( "context" "fmt" "net/http" "strings" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) // LoadByName loads and initializes a Network zone from the database by name. func LoadByName(s *state.State, name string) (NetworkZone, error) { var dbZone []cluster.NetworkZone var zoneInfo *api.NetworkZone err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error filter := cluster.NetworkZoneFilter{Name: &name} dbZone, err = cluster.GetNetworkZones(ctx, tx.Tx(), filter) if err != nil { return err } if len(dbZone) != 1 { return fmt.Errorf("Loading network zone named %s returned an unexpected amount of results: %d", name, len(dbZone)) } zoneInfo, err = dbZone[0].ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return nil, err } var zone NetworkZone = &zone{} zone.init(s, int64(dbZone[0].ID), dbZone[0].Project, zoneInfo) return zone, nil } // LoadByNameAndProject loads and initializes a Network zone from the database by project and name. func LoadByNameAndProject(s *state.State, projectName string, name string) (NetworkZone, error) { var dbZone *cluster.NetworkZone var zoneInfo *api.NetworkZone err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error filter := cluster.NetworkZoneFilter{ Project: &projectName, Name: &name, } zones, err := cluster.GetNetworkZones(ctx, tx.Tx(), filter) if err != nil { return err } if len(zones) == 0 { return api.StatusErrorf(http.StatusNotFound, "Network zone not found") } dbZone = &zones[0] zoneInfo, err = dbZone.ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return nil, err } var zone NetworkZone = &zone{} zone.init(s, int64(dbZone.ID), projectName, zoneInfo) return zone, nil } // Create validates supplied record and creates new Network zone record in the database. func Create(s *state.State, projectName string, zoneInfo *api.NetworkZonesPost) error { var zone NetworkZone = &zone{} zone.init(s, -1, projectName, nil) err := zone.validateName(zoneInfo.Name) if err != nil { return err } err = zone.validateConfig(&zoneInfo.NetworkZonePut) if err != nil { return err } // Load the project. var p *api.Project err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { project, err := cluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return err } p, err = project.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return err } // Validate restrictions. if util.IsTrue(p.Config["restricted"]) { found := false for _, entry := range strings.Split(p.Config["restricted.networks.zones"], ",") { entry = strings.TrimSpace(entry) if zoneInfo.Name == entry || strings.HasSuffix(zoneInfo.Name, "."+entry) { found = true break } } if !found { return api.StatusErrorf(http.StatusForbidden, "Project isn't allowed to use this DNS zone") } } err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbZone := cluster.NetworkZone{ Project: projectName, Name: zoneInfo.Name, Description: zoneInfo.Description, } id, err := cluster.CreateNetworkZone(ctx, tx.Tx(), dbZone) if err != nil { return err } err = cluster.CreateNetworkZoneConfig(ctx, tx.Tx(), id, zoneInfo.Config) if err != nil { return err } return nil }) if err != nil { return err } // Trigger a refresh of the TSIG entries. err = s.DNS.UpdateTSIG() if err != nil { return err } return nil } // Exists checks the zone name(s) provided exists. // If multiple names are provided, also checks that duplicate names aren't specified in the list. func Exists(s *state.State, name ...string) error { checkedzoneNames := make(map[string]struct{}, len(name)) err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { for _, zoneName := range name { filter := cluster.NetworkZoneFilter{Name: &zoneName} dbZone, err := cluster.GetNetworkZones(ctx, tx.Tx(), filter) if err != nil { return err } if len(dbZone) != 1 { return fmt.Errorf("Loading network zone named %s returned an unexpected amount of results: %d", zoneName, len(dbZone)) } status, err := cluster.NetworkZoneExists(ctx, tx.Tx(), dbZone[0].Project, zoneName) if !status { return fmt.Errorf("Network zone %q does not exist", zoneName) } if err != nil { return fmt.Errorf("Error when checking existence of %q network zone in project %q", zoneName, dbZone[0].Project) } _, found := checkedzoneNames[zoneName] if found { return fmt.Errorf("Network zone %q specified multiple times", zoneName) } checkedzoneNames[zoneName] = struct{}{} } return nil }) if err != nil { return err } return nil } incus-7.0.0/internal/server/network/zone/record.go000066400000000000000000000135071517523235500222500ustar00rootroot00000000000000package zone import ( "context" "fmt" "net/http" "slices" "github.com/miekg/dns" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/shared/api" ) func (d *zone) AddRecord(req api.NetworkZoneRecordsPost) error { // Validate. err := d.validateName(req.Name) if err != nil { return err } err = d.validateRecordConfig(req.NetworkZoneRecordPut) if err != nil { return err } // Validate entries. err = d.validateEntries(req.NetworkZoneRecordPut) if err != nil { return err } err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Create the network zone record object. dbRecord := dbCluster.NetworkZoneRecord{ NetworkZoneID: int(d.id), Name: req.Name, Description: req.Description, Entries: req.Entries, } // Add the new record. id, err := dbCluster.CreateNetworkZoneRecord(ctx, tx.Tx(), dbRecord) if err != nil { return err } // Add the config. err = dbCluster.CreateNetworkZoneRecordConfig(ctx, tx.Tx(), id, req.Config) if err != nil { return err } return nil }) if err != nil { return err } return nil } func (d *zone) GetRecords() ([]api.NetworkZoneRecord, error) { s := d.state records := []api.NetworkZoneRecord{} err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { zoneID := int(d.id) filter := dbCluster.NetworkZoneRecordFilter{ NetworkZoneID: &zoneID, } dbRecords, err := dbCluster.GetNetworkZoneRecords(ctx, tx.Tx(), filter) if err != nil { return err } // Convert each record to API format. for _, dbRecord := range dbRecords { apiRecord, err := dbRecord.ToAPI(ctx, tx.Tx()) if err != nil { return err } records = append(records, *apiRecord) } return nil }) if err != nil { return nil, err } return records, nil } func (d *zone) GetRecord(name string) (*api.NetworkZoneRecord, error) { var record *api.NetworkZoneRecord err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { zoneID := int(d.id) filter := dbCluster.NetworkZoneRecordFilter{ NetworkZoneID: &zoneID, Name: &name, } dbRecords, err := dbCluster.GetNetworkZoneRecords(ctx, tx.Tx(), filter) if err != nil { return err } if len(dbRecords) == 0 { return api.StatusErrorf(http.StatusNotFound, "Network zone record not found") } // Convert to API format. apiRecord, err := dbRecords[0].ToAPI(ctx, tx.Tx()) if err != nil { return err } record = apiRecord return nil }) if err != nil { return nil, err } return record, nil } func (d *zone) UpdateRecord(name string, req api.NetworkZoneRecordPut, clientType request.ClientType) error { s := d.state // Validate. err := d.validateRecordConfig(req) if err != nil { return err } // Validate entries. err = d.validateEntries(req) if err != nil { return err } err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { zoneID := int(d.id) filter := dbCluster.NetworkZoneRecordFilter{ NetworkZoneID: &zoneID, Name: &name, } // Get the records matching the filter, not using exist because we need record ID and it would be redundant with Get. dbRecords, err := dbCluster.GetNetworkZoneRecords(ctx, tx.Tx(), filter) if err != nil { return err } if len(dbRecords) == 0 { return api.StatusErrorf(http.StatusNotFound, "Network zone record not found") } // Update the record. dbRecord := dbRecords[0] dbRecord.Description = req.Description dbRecord.Entries = req.Entries err = dbCluster.UpdateNetworkZoneRecord(ctx, tx.Tx(), zoneID, name, dbRecord) if err != nil { return err } // Update the config. err = dbCluster.UpdateNetworkZoneRecordConfig(ctx, tx.Tx(), int64(dbRecord.ID), req.Config) if err != nil { return err } return nil }) if err != nil { return err } return nil } func (d *zone) DeleteRecord(name string) error { s := d.state err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { zoneID := int(d.id) filter := dbCluster.NetworkZoneRecordFilter{ NetworkZoneID: &zoneID, Name: &name, } // Get the records matching the filter, not using exist because we need record ID and it would be redundant with Get. dbRecords, err := dbCluster.GetNetworkZoneRecords(ctx, tx.Tx(), filter) if err != nil { return err } if len(dbRecords) == 0 { return api.StatusErrorf(http.StatusNotFound, "Network zone record not found") } // Delete the record. err = dbCluster.DeleteNetworkZoneRecord(ctx, tx.Tx(), int(d.id), dbRecords[0].ID) if err != nil { return err } return nil }) if err != nil { return err } return nil } // validateRecordConfig checks the config and rules are valid. func (d *zone) validateRecordConfig(info api.NetworkZoneRecordPut) error { rules := map[string]func(value string) error{} err := d.validateConfigMap(info.Config, rules) if err != nil { return err } return nil } // validateEntries checks the validity of the DNS entries. func (d *zone) validateEntries(info api.NetworkZoneRecordPut) error { uniqueEntries := make([]string, 0, len(info.Entries)) for _, entry := range info.Entries { if entry.TTL == 0 { entry.TTL = 300 } _, err := dns.NewRR(fmt.Sprintf("record %d IN %s %s", entry.TTL, entry.Type, entry.Value)) if err != nil { return fmt.Errorf("Bad zone record entry: %w", err) } entryID := entry.Type + "/" + entry.Value if slices.Contains(uniqueEntries, entryID) { return fmt.Errorf("Duplicate record for type %q and value %q", entry.Type, entry.Value) } uniqueEntries = append(uniqueEntries, entryID) } return nil } incus-7.0.0/internal/server/network/zone/zone.go000066400000000000000000000417121517523235500217440ustar00rootroot00000000000000package zone import ( "context" "errors" "fmt" "net" "slices" "strings" "time" incus "github.com/lxc/incus/v7/client" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/network" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // zone represents a Network zone. type zone struct { logger logger.Logger state *state.State id int64 projectName string info *api.NetworkZone } // init initialize internal variables. func (d *zone) init(state *state.State, id int64, projectName string, info *api.NetworkZone) { if info == nil { d.info = &api.NetworkZone{} } else { d.info = info } d.logger = logger.AddContext(logger.Ctx{"project": projectName, "networkzone": d.info.Name}) d.id = id d.projectName = projectName d.state = state if d.info.Config == nil { d.info.Config = make(map[string]string) } } // ID returns the Network zone ID. func (d *zone) ID() int64 { return d.id } // Name returns the project. func (d *zone) Project() string { return d.projectName } // Info returns copy of internal info for the Network zone. func (d *zone) Info() *api.NetworkZone { // Copy internal info to prevent modification externally. info := api.NetworkZone{} info.Name = d.info.Name info.Description = d.info.Description info.Config = localUtil.CopyConfig(d.info.Config) info.UsedBy = nil // To indicate its not populated (use Usedby() function to populate). return &info } // networkUsesZone indicates if the network uses the zone based on its config. func (d *zone) networkUsesZone(netConfig map[string]string) bool { for _, key := range []string{"dns.zone.forward", "dns.zone.reverse.ipv4", "dns.zone.reverse.ipv6"} { zoneNames := util.SplitNTrimSpace(netConfig[key], ",", -1, true) if slices.Contains(zoneNames, d.info.Name) { return true } } return false } // usedBy returns a list of API endpoints referencing this zone. // If firstOnly is true then search stops at first result. func (d *zone) usedBy(firstOnly bool) ([]string, error) { usedBy := []string{} var networkNames []string err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Find networks using the zone. networkNames, err = tx.GetCreatedNetworkNamesByProject(ctx, d.projectName) if err != nil && !response.IsNotFoundError(err) { return fmt.Errorf("Failed loading networks for project %q: %w", d.projectName, err) } for _, networkName := range networkNames { _, network, _, err := tx.GetNetworkInAnyState(ctx, d.projectName, networkName) if err != nil { return fmt.Errorf("Failed to get network config for %q: %w", networkName, err) } // Check if the network is using this zone. if d.networkUsesZone(network.Config) { u := api.NewURL().Path(version.APIVersion, "networks", networkName) usedBy = append(usedBy, u.String()) if firstOnly { return nil } } } return nil }) if err != nil { return nil, err } return usedBy, nil } // UsedBy returns a list of API endpoints referencing this zone. func (d *zone) UsedBy() ([]string, error) { return d.usedBy(false) } // isUsed returns whether or not the zone is in use. func (d *zone) isUsed() (bool, error) { usedBy, err := d.usedBy(true) if err != nil { return false, err } return len(usedBy) > 0, nil } // Etag returns the values used for etag generation. func (d *zone) Etag() []any { return []any{d.info.Name, d.info.Description, d.info.Config} } // validateName checks name is valid. func (d *zone) validateName(name string) error { // Allow root records. if name == "@" { return nil } // Allow wildcards. if strings.HasPrefix(name, "*.") { return nil } return validate.IsAPIName(name, false) } // validateConfig checks the config and rules are valid. func (d *zone) validateConfig(info *api.NetworkZonePut) error { rules := map[string]func(value string) error{} // Regular config keys. // gendoc:generate(entity=network_zone, group=common, key=dns.nameservers) // // --- // type: string set // required: no // shortdesc: Comma-separated list of DNS server FQDNs (for NS records) rules["dns.nameservers"] = validate.IsListOf(validate.IsAny) // gendoc:generate(entity=network_zone, group=common, key=dns.contact) // // --- // type: string // required: no // shortdesc: Admin contact email for DNS server rules["dns.contact"] = validate.Optional(validate.IsAny) // gendoc:generate(entity=network_zone, group=common, key=network.nat) // // --- // type: bool // required: no // defaultdesc: `true` // shortdesc: Whether to generate records for NAT-ed subnets rules["network.nat"] = validate.Optional(validate.IsBool) // Validate peer config. for k := range info.Config { if !strings.HasPrefix(k, "peers.") { continue } // Validate remote name in key. fields := strings.Split(k, ".") if len(fields) != 3 { return fmt.Errorf("Invalid network zone configuration key %q", k) } peerKey := fields[2] // Add the correct validation rule for the dynamic field based on last part of key. switch peerKey { case "address": // gendoc:generate(entity=network_zone, group=common, key=peers.NAME.address) // // --- // type: string // required: no // shortdesc: IP address of a DNS server rules[k] = validate.Optional(validate.IsNetworkAddress) case "key": // gendoc:generate(entity=network_zone, group=common, key=peers.NAME.key) // // --- // type: string // required: no // shortdesc: TSIG key for the server rules[k] = validate.Optional(validate.IsAny) } } // gendoc:generate(entity=network_zone, group=common, key=user.*) // // --- // type: string // required: no // shortdesc: User-provided free-form key/value pairs err := d.validateConfigMap(info.Config, rules) if err != nil { return err } return nil } // validateConfigMap checks zone config map against rules. func (d *zone) validateConfigMap(config map[string]string, rules map[string]func(value string) error) error { checkedFields := map[string]struct{}{} // Run the validator against each field. for k, validator := range rules { checkedFields[k] = struct{}{} // Mark field as checked. err := validator(config[k]) if err != nil { return fmt.Errorf("Invalid value for config option %q: %w", k, err) } } // Look for any unchecked fields, as these are unknown fields and validation should fail. for k := range config { _, checked := checkedFields[k] if checked { continue } // User keys are not validated. if internalInstance.IsUserConfig(k) { continue } return fmt.Errorf("Invalid config option %q", k) } return nil } // Update applies the supplied config to the zone. func (d *zone) Update(config *api.NetworkZonePut, clientType request.ClientType) error { err := d.validateConfig(config) if err != nil { return err } reverter := revert.New() defer reverter.Fail() // Update the database and notify the rest of the cluster. if clientType == request.ClientTypeNormal { oldConfig := d.info.NetworkZonePut // Update database. err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbZone := dbCluster.NetworkZone{ ID: int(d.id), Project: d.projectName, Name: d.info.Name, Description: config.Description, } err := dbCluster.UpdateNetworkZone(ctx, tx.Tx(), dbZone.Project, dbZone.Name, dbZone) if err != nil { return err } err = dbCluster.UpdateNetworkZoneConfig(ctx, tx.Tx(), int64(dbZone.ID), config.Config) if err != nil { return err } return nil }) if err != nil { return err } // Apply changes internally and reinitialize. d.info.NetworkZonePut = *config d.init(d.state, d.id, d.projectName, d.info) reverter.Add(func() { _ = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbZone := dbCluster.NetworkZone{ ID: int(d.id), Project: d.projectName, Name: d.info.Name, Description: oldConfig.Description, } err := dbCluster.UpdateNetworkZone(ctx, tx.Tx(), dbZone.Project, dbZone.Name, dbZone) if err != nil { return err } err = dbCluster.UpdateNetworkZoneConfig(ctx, tx.Tx(), int64(dbZone.ID), oldConfig.Config) if err != nil { return err } return nil }) d.info.NetworkZonePut = oldConfig d.init(d.state, d.id, d.projectName, d.info) }) // Notify all other nodes to update the network zone if no target specified. notifier, err := cluster.NewNotifier(d.state, d.state.Endpoints.NetworkCert(), d.state.ServerCert(), cluster.NotifyAll) if err != nil { return err } err = notifier(func(client incus.InstanceServer) error { return client.UseProject(d.projectName).UpdateNetworkZone(d.info.Name, d.info.NetworkZonePut, "") }) if err != nil { return err } } // Trigger a refresh of the TSIG entries. err = d.state.DNS.UpdateTSIG() if err != nil { return err } reverter.Success() return nil } // Delete deletes the zone. func (d *zone) Delete() error { isUsed, err := d.isUsed() if err != nil { return err } if isUsed { return errors.New("Cannot delete a zone that is in use") } err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Delete the database record. err = dbCluster.DeleteNetworkZone(ctx, tx.Tx(), int(d.id)) return err }) if err != nil { return err } // Trigger a refresh of the TSIG entries. err = d.state.DNS.UpdateTSIG() if err != nil { return err } return nil } // Content returns the DNS zone content. func (d *zone) Content() (*strings.Builder, error) { var err error records := []map[string]string{} // Check if we should include NAT records. includeNAT := util.IsTrueOrEmpty(d.info.Config["network.nat"]) // Get all managed networks across all projects. var projectNetworks map[string]map[int64]api.Network var zoneProjects map[string]string instProjects := []string{d.projectName} err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { projectNetworks, err = tx.GetCreatedNetworks(ctx) if err != nil { return fmt.Errorf("Failed to load all networks: %w", err) } if d.projectName == api.ProjectDefaultName { // Get all projects that don't have the network zone feature enabled. projects, err := dbCluster.GetProjects(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed to load all projects: %w", err) } for _, dbProject := range projects { apiProject, err := dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed to load project %q: %w", dbProject.Name, err) } if !util.IsTrue(apiProject.Config["features.networks.zones"]) { instProjects = append(instProjects, apiProject.Name) } } } // Get a map of zone names to project. zones, err := dbCluster.GetNetworkZones(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed to load all network zones: %w", err) } zoneProjects = make(map[string]string) for _, zone := range zones { zoneProjects[zone.Name] = zone.Project } return nil }) if err != nil { return nil, err } for netProjectName, networks := range projectNetworks { for _, netInfo := range networks { if !d.networkUsesZone(netInfo.Config) { continue } // Load the network. n, err := network.LoadByName(d.state, netProjectName, netInfo.Name) if err != nil { return nil, err } // Check whether what records to include. netConfig := n.Config() includeV4 := includeNAT || util.IsFalseOrEmpty(netConfig["ipv4.nat"]) includeV6 := includeNAT || util.IsFalseOrEmpty(netConfig["ipv6.nat"]) // Check if dealing with a reverse zone. isReverse4 := strings.HasSuffix(d.info.Name, localUtil.IPv4Arpa) isReverse6 := strings.HasSuffix(d.info.Name, localUtil.IPv6Arpa) isReverse := isReverse4 || isReverse6 genRecord := func(name string, ip net.IP) map[string]string { isV4 := ip.To4() != nil // Skip disabled families. if isV4 && !includeV4 { return nil } if !isV4 && !includeV6 { return nil } record := map[string]string{} record["ttl"] = "300" if !isReverse { if isV4 { record["type"] = "A" } else { record["type"] = "AAAA" } record["name"] = name record["value"] = ip.String() } else { // Skip PTR records for wrong family. if isV4 && !isReverse4 { return nil } if !isV4 && !isReverse6 { return nil } // Get the ARPA record. reverseAddr := localUtil.ReverseDNS(ip) if reverseAddr == "" { return nil } record["type"] = "PTR" record["name"] = strings.TrimSuffix(reverseAddr, "."+d.info.Name+".") record["value"] = name + "." } return record } for _, instProjectName := range instProjects { if isReverse { // Load network leases in correct project context for each forward zone referenced. for _, forwardZoneName := range util.SplitNTrimSpace(n.Config()["dns.zone.forward"], ",", -1, true) { if !slices.Contains(instProjects, zoneProjects[forwardZoneName]) { continue } // Load the leases for the forward zone project. leases, err := n.Leases(instProjectName, request.ClientTypeNormal) if err != nil { return nil, err } // Convert leases to usable PTR records. for _, lease := range leases { ip := net.ParseIP(lease.Address) // Get the record. record := genRecord(fmt.Sprintf("%s.%s", lease.Hostname, forwardZoneName), ip) if record == nil { continue } records = append(records, record) } } } else { // Load the leases in the forward zone's project. leases, err := n.Leases(instProjectName, request.ClientTypeNormal) if err != nil { return nil, err } // Convert leases to usable records. for _, lease := range leases { ip := net.ParseIP(lease.Address) // Get the record. record := genRecord(lease.Hostname, ip) if record == nil { continue } records = append(records, record) } } } } } // Add the extra records. extraRecords, err := d.GetRecords() if err != nil { return nil, err } for _, extraRecord := range extraRecords { for _, entry := range extraRecord.Entries { record := map[string]string{} if entry.TTL > 0 { record["ttl"] = fmt.Sprintf("%d", entry.TTL) } else { record["ttl"] = "300" } record["type"] = entry.Type record["name"] = extraRecord.Name record["value"] = entry.Value records = append(records, record) } } // Get the nameservers. nameservers := []string{} for _, entry := range strings.Split(d.info.Config["dns.nameservers"], ",") { entry = strings.TrimSuffix(strings.TrimSpace(entry), ".") if entry == "" { continue } nameservers = append(nameservers, entry) } primary := d.info.Name if len(nameservers) > 0 { primary = nameservers[0] } contact := "hostmaster." + primary if len(d.info.Config["dns.contact"]) > 0 { contact = d.info.Config["dns.contact"] contact = strings.TrimSuffix(strings.TrimSpace(contact), ".") } // Template the zone file. sb := &strings.Builder{} err = zoneTemplate.Execute(sb, map[string]any{ "primary": primary, "contact": contact, "nameservers": nameservers, "zone": d.info.Name, "serial": time.Now().Unix(), "records": records, }) if err != nil { return nil, err } return sb, nil } // SOA returns just the DNS zone SOA record. func (d *zone) SOA() (*strings.Builder, error) { // Get the nameservers. nameservers := []string{} for _, entry := range strings.Split(d.info.Config["dns.nameservers"], ",") { entry = strings.TrimSpace(entry) if entry == "" { continue } nameservers = append(nameservers, entry) } primary := d.info.Name if len(nameservers) > 0 { primary = nameservers[0] } contact := "hostmaster." + primary if len(d.info.Config["dns.contact"]) > 0 { contact = d.info.Config["dns.contact"] contact = strings.TrimSuffix(strings.TrimSpace(contact), ".") } // Template the zone file. sb := &strings.Builder{} err := zoneTemplate.Execute(sb, map[string]any{ "primary": primary, "contact": contact, "nameservers": nameservers, "zone": d.info.Name, "serial": time.Now().Unix(), "records": map[string]string{}, }) if err != nil { return nil, err } return sb, nil } incus-7.0.0/internal/server/network/zone/zone_templates.go000066400000000000000000000007701517523235500240210ustar00rootroot00000000000000package zone import ( "text/template" ) // DNS zone template. var zoneTemplate = template.Must(template.New("zoneTemplate").Parse(` {{.zone}}. 3600 IN SOA {{.primary}}. {{.contact}}. {{.serial}} 120 60 86400 30 {{- range $index, $element := .nameservers}} {{$.zone}}. 300 IN NS {{$element}}. {{- end}} {{- range .records}} {{ if ne .name "@" }}{{.name}}.{{ end }}{{$.zone}}. {{.ttl}} IN {{.type}} {{.value}} {{- end}} {{.zone}}. 3600 IN SOA {{.primary}}. {{.contact}}. {{.serial}} 120 60 86400 30 `)) incus-7.0.0/internal/server/node/000077500000000000000000000000001517523235500167165ustar00rootroot00000000000000incus-7.0.0/internal/server/node/config.go000066400000000000000000000236461517523235500205250ustar00rootroot00000000000000package node import ( "context" "fmt" "maps" "github.com/lxc/incus/v7/internal/ports" "github.com/lxc/incus/v7/internal/server/config" "github.com/lxc/incus/v7/internal/server/db" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/validate" ) // Config holds member-local configuration values. type Config struct { tx *db.NodeTx // DB transaction the values in this config are bound to m config.Map // Low-level map holding the config values. } // ConfigLoad loads a new Config object with the current node-local configuration // values fetched from the database. An optional list of config value triggers // can be passed, each config key must have at most one trigger. func ConfigLoad(ctx context.Context, tx *db.NodeTx) (*Config, error) { // Load current raw values from the database, any error is fatal. values, err := tx.Config(ctx) if err != nil { return nil, fmt.Errorf("Cannot fetch node config from database: %w", err) } m, err := config.SafeLoad(ConfigSchema, values) if err != nil { return nil, fmt.Errorf("Failed to load node config: %w", err) } return &Config{tx: tx, m: m}, nil } // HTTPSAddress returns the address and port this server should expose its // API to, if any. func (c *Config) HTTPSAddress() string { networkAddress := c.m.GetString("core.https_address") if networkAddress != "" { return internalUtil.CanonicalNetworkAddress(networkAddress, ports.HTTPSDefaultPort) } return networkAddress } // BGPAddress returns the address and port to setup the BGP listener on. func (c *Config) BGPAddress() string { return c.m.GetString("core.bgp_address") } // BGPRouterID returns the address to use as a router ID. func (c *Config) BGPRouterID() string { return c.m.GetString("core.bgp_routerid") } // ClusterAddress returns the address and port this cluster member should use for cluster communication. func (c *Config) ClusterAddress() string { clusterAddress := c.m.GetString("cluster.https_address") if clusterAddress != "" { return internalUtil.CanonicalNetworkAddress(clusterAddress, ports.HTTPSDefaultPort) } return clusterAddress } // DebugAddress returns the address and port to setup the pprof listener on. func (c *Config) DebugAddress() string { debugAddress := c.m.GetString("core.debug_address") if debugAddress != "" { return internalUtil.CanonicalNetworkAddress(debugAddress, ports.HTTPDebugDefaultPort) } return debugAddress } // DNSAddress returns the address and port to setup the DNS listener on. func (c *Config) DNSAddress() string { return c.m.GetString("core.dns_address") } // MetricsAddress returns the address and port to setup the metrics listener on. func (c *Config) MetricsAddress() string { metricsAddress := c.m.GetString("core.metrics_address") if metricsAddress != "" { return internalUtil.CanonicalNetworkAddress(metricsAddress, ports.HTTPSMetricsDefaultPort) } return metricsAddress } // NetworkOVSConnection returns the OVS socket path. func (c *Config) NetworkOVSConnection() string { return c.m.GetString("network.ovs.connection") } // StorageBucketsAddress returns the address and port to setup the storage buckets listener on. func (c *Config) StorageBucketsAddress() string { objectAddress := c.m.GetString("core.storage_buckets_address") if objectAddress != "" { return internalUtil.CanonicalNetworkAddress(objectAddress, ports.HTTPSStorageBucketsDefaultPort) } return objectAddress } // StorageBackupsVolume returns the name of the pool/volume to use for storing backup tarballs. func (c *Config) StorageBackupsVolume() string { return c.m.GetString("storage.backups_volume") } // StorageImagesVolume returns the name of the pool/volume to use for storing image tarballs. func (c *Config) StorageImagesVolume() string { return c.m.GetString("storage.images_volume") } // StorageLogsVolume returns the name of the pool/volume to use for storing instance log directories. func (c *Config) StorageLogsVolume() string { return c.m.GetString("storage.logs_volume") } // LinstorSatelliteName returns the LINSTOR satellite name override. func (c *Config) LinstorSatelliteName() string { return c.m.GetString("storage.linstor.satellite.name") } // SyslogSocket returns true if the syslog socket is enabled, otherwise false. func (c *Config) SyslogSocket() bool { return c.m.GetBool("core.syslog_socket") } // Dump current configuration keys and their values. Keys with values matching // their defaults are omitted. func (c *Config) Dump() map[string]string { return c.m.Dump() } // Replace the current configuration with the given values. func (c *Config) Replace(values map[string]string) (map[string]string, error) { return c.update(values) } // Patch changes only the configuration keys in the given map. func (c *Config) Patch(patch map[string]string) (map[string]string, error) { values := c.Dump() // Use current values as defaults maps.Copy(values, patch) return c.update(values) } func (c *Config) update(values map[string]string) (map[string]string, error) { changed, err := c.m.Change(values) if err != nil { return nil, err } err = c.tx.UpdateConfig(changed) if err != nil { return nil, fmt.Errorf("Cannot persist local configuration changes: %w", err) } return changed, nil } // ConfigSchema defines available server configuration keys. var ConfigSchema = config.Schema{ // Network address for this server // gendoc:generate(entity=server, group=core, key=core.https_address) // See {ref}`server-expose`. // --- // type: string // scope: local // shortdesc: Address to bind for the remote API (HTTPS) "core.https_address": {Validator: validate.Optional(validate.IsListenAddress(true, true, false))}, // Network address for cluster communication // gendoc:generate(entity=server, group=cluster, key=cluster.https_address) // See {ref}`cluster-https-address`. // --- // type: string // scope: local // shortdesc: Address to use for clustering traffic "cluster.https_address": {Validator: validate.Optional(validate.IsListenAddress(true, false, false))}, // Network address for the BGP server // gendoc:generate(entity=server, group=core, key=core.bgp_address) // See {ref}`network-bgp`. // --- // type: string // scope: local // shortdesc: Address to bind the BGP server to "core.bgp_address": {Validator: validate.Optional(validate.IsListenAddress(true, true, false))}, // Unique router ID for the BGP server // gendoc:generate(entity=server, group=core, key=core.bgp_routerid) // The identifier must be formatted as an IPv4 address. // --- // type: string // scope: local // shortdesc: A unique identifier for the BGP server "core.bgp_routerid": {Validator: validate.Optional(validate.IsNetworkAddressV4)}, // Network address for the debug server // gendoc:generate(entity=server, group=core, key=core.debug_address) // // --- // type: string // scope: local // shortdesc: Address to bind the `pprof` debug server to (HTTP) "core.debug_address": {Validator: validate.Optional(validate.IsListenAddress(true, true, false))}, // Network address for the DNS server // gendoc:generate(entity=server, group=core, key=core.dns_address) // See {ref}`network-dns-server`. // --- // type: string // scope: local // shortdesc: Address to bind the authoritative DNS server to "core.dns_address": {Validator: validate.Optional(validate.IsListenAddress(true, true, false))}, // Network address for the metrics server // gendoc:generate(entity=server, group=core, key=core.metrics_address) // See {ref}`metrics`. // --- // type: string // scope: local // shortdesc: Address to bind the metrics server to (HTTPS) "core.metrics_address": {Validator: validate.Optional(validate.IsListenAddress(true, true, false))}, // Network address for the storage buckets server // gendoc:generate(entity=server, group=core, key=core.storage_buckets_address) // See {ref}`howto-storage-buckets`. // --- // type: string // scope: local // shortdesc: Address to bind the storage object server to (HTTPS) "core.storage_buckets_address": {Validator: validate.Optional(validate.IsListenAddress(true, true, false))}, // Syslog socket // gendoc:generate(entity=server, group=core, key=core.syslog_socket) // Set this option to `true` to enable the syslog unixgram socket to receive log messages from external processes. // --- // type: bool // scope: local // defaultdesc: `false` // shortdesc: Whether to enable the syslog unixgram socket listener "core.syslog_socket": {Validator: validate.Optional(validate.IsBool), Type: config.Bool}, // gendoc:generate(entity=server, group=miscellaneous, key=network.ovs.connection) // // --- // type: string // scope: global // defaultdesc: `unix:/run/openvswitch/db.sock` // shortdesc: OVS socket path "network.ovs.connection": {Default: "unix:/run/openvswitch/db.sock"}, // Storage volumes to store backups/images/logs on // gendoc:generate(entity=server, group=miscellaneous, key=storage.backups_volume) // Specify the volume using the syntax `POOL/VOLUME`. // --- // type: string // scope: local // shortdesc: Volume to use to store backup tarballs "storage.backups_volume": {}, // gendoc:generate(entity=server, group=miscellaneous, key=storage.images_volume) // Specify the volume using the syntax `POOL/VOLUME`. // --- // type: string // scope: local // shortdesc: Volume to use to store the image tarballs "storage.images_volume": {}, // gendoc:generate(entity=server, group=miscellaneous, key=storage.logs_volume) // Specify the volume using the syntax `POOL/VOLUME`. // --- // type: string // scope: local // shortdesc: Volume to use to store instance log directories "storage.logs_volume": {}, // LINSTOR // gendoc:generate(entity=server, group=miscellaneous, key=storage.linstor.satellite.name) // Set this option to the name of the local LINSTOR satellite node, should it be different from the Incus server name. // --- // type: string // scope: global // shortdesc: LINSTOR satellite node name override "storage.linstor.satellite.name": {}, } incus-7.0.0/internal/server/node/config_test.go000066400000000000000000000114601517523235500215530ustar00rootroot00000000000000package node_test import ( "context" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/node" ) // The server configuration is initially empty. func TestConfigLoad_Initial(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() config, err := node.ConfigLoad(context.Background(), tx) require.NoError(t, err) assert.Equal(t, map[string]string{}, config.Dump()) } // If the database contains invalid keys, they are ignored. func TestConfigLoad_IgnoreInvalidKeys(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() err := tx.UpdateConfig(map[string]string{ "foo": "garbage", "core.https_address": "127.0.0.1:666", }) require.NoError(t, err) config, err := node.ConfigLoad(context.Background(), tx) require.NoError(t, err) values := map[string]string{"core.https_address": "127.0.0.1:666"} assert.Equal(t, values, config.Dump()) } // Triggers can be specified to execute custom code on config key changes. func TestConfigLoad_Triggers(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() config, err := node.ConfigLoad(context.Background(), tx) require.NoError(t, err) assert.Equal(t, map[string]string{}, config.Dump()) } // If some previously set values are missing from the ones passed to Replace(), // they are deleted from the configuration. func TestConfig_ReplaceDeleteValues(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() config, err := node.ConfigLoad(context.Background(), tx) require.NoError(t, err) changed, err := config.Replace(map[string]string{"core.https_address": "127.0.0.1:666"}) assert.NoError(t, err) assert.Equal(t, map[string]string{"core.https_address": "127.0.0.1:666"}, changed) changed, err = config.Replace(map[string]string{}) assert.NoError(t, err) assert.Equal(t, map[string]string{"core.https_address": ""}, changed) assert.Equal(t, "", config.HTTPSAddress()) values, err := tx.Config(context.Background()) require.NoError(t, err) assert.Equal(t, map[string]string{}, values) } // If some previously set values are missing from the ones passed to Patch(), // they are kept as they are. func TestConfig_PatchKeepsValues(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() config, err := node.ConfigLoad(context.Background(), tx) require.NoError(t, err) _, err = config.Replace(map[string]string{"core.https_address": "127.0.0.1:666"}) assert.NoError(t, err) _, err = config.Patch(map[string]string{}) assert.NoError(t, err) assert.Equal(t, "127.0.0.1:666", config.HTTPSAddress()) values, err := tx.Config(context.Background()) require.NoError(t, err) assert.Equal(t, map[string]string{"core.https_address": "127.0.0.1:666"}, values) } // The core.https_address config key is fetched from the db with a new // transaction. func TestHTTPSAddress(t *testing.T) { nodeDB, cleanup := db.NewTestNode(t) defer cleanup() var err error var config *node.Config err = nodeDB.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { config, err = node.ConfigLoad(ctx, tx) return err }) require.NoError(t, err) require.NoError(t, err) assert.Equal(t, "", config.HTTPSAddress()) err = nodeDB.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { config, err = node.ConfigLoad(ctx, tx) if err != nil { return err } _, err = config.Replace(map[string]string{"core.https_address": "127.0.0.1:666"}) return err }) require.NoError(t, err) err = nodeDB.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { config, err = node.ConfigLoad(ctx, tx) return err }) require.NoError(t, err) require.NoError(t, err) assert.Equal(t, "127.0.0.1:666", config.HTTPSAddress()) } // The cluster.https_address config key is fetched from the db with a new // transaction. func TestClusterAddress(t *testing.T) { nodeDB, cleanup := db.NewTestNode(t) defer cleanup() var err error var nodeConfig *node.Config err = nodeDB.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { nodeConfig, err = node.ConfigLoad(ctx, tx) return err }) require.NoError(t, err) assert.Equal(t, "", nodeConfig.ClusterAddress()) err = nodeDB.Transaction(context.Background(), func(ctx context.Context, tx *db.NodeTx) error { nodeConfig, err = node.ConfigLoad(ctx, tx) if err != nil { return err } _, err = nodeConfig.Replace(map[string]string{"cluster.https_address": "127.0.0.1:666"}) return err }) require.NoError(t, err) err = nodeDB.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { nodeConfig, err = node.ConfigLoad(ctx, tx) return err }) require.NoError(t, err) assert.Equal(t, "127.0.0.1:666", nodeConfig.ClusterAddress()) } incus-7.0.0/internal/server/node/raft.go000066400000000000000000000040461517523235500202050ustar00rootroot00000000000000package node import ( "context" "github.com/cowsql/go-cowsql/client" "github.com/lxc/incus/v7/internal/server/db" ) // DetermineRaftNode figures out what raft node ID and address we have, if any. // // This decision is based on the value of the cluster.https_address config key // and on the rows in the raft_nodes table, both stored in the node-level // SQLite database. // // The following rules are applied: // // - If no cluster.https_address config key is set, this is a non-clustered node // and the returned RaftNode will have ID 1 but no address, to signal that // the node should setup an in-memory raft cluster where the node itself // is the only member and leader. // // - If cluster.https_address config key is set, but there is no row in the // raft_nodes table, this is a brand new clustered node that is joining a // cluster, and same behavior as the previous case applies. // // - If cluster.https_address config key is set and there is at least one row // in the raft_nodes table, then this node is considered a raft node if // cluster.https_address matches one of the rows in raft_nodes. In that case, // the matching db.RaftNode row is returned, otherwise nil. func DetermineRaftNode(ctx context.Context, tx *db.NodeTx) (*db.RaftNode, error) { config, err := ConfigLoad(ctx, tx) if err != nil { return nil, err } address := config.ClusterAddress() // If cluster.https_address is an empty string, then this server is // not running in clustering mode. if address == "" { nodeInfo := client.NodeInfo{ID: 1} return &db.RaftNode{NodeInfo: nodeInfo, Name: ""}, nil } nodes, err := tx.GetRaftNodes(ctx) if err != nil { return nil, err } // If cluster.https_address and the raft_nodes table is not populated, // this must be a joining node. if len(nodes) == 0 { nodeInfo := client.NodeInfo{ID: 1} return &db.RaftNode{NodeInfo: nodeInfo, Name: ""}, nil } // Try to find a matching node. for _, node := range nodes { if node.Address == address { return &node, nil } } return nil, nil } incus-7.0.0/internal/server/node/raft_test.go000066400000000000000000000040301517523235500212350ustar00rootroot00000000000000package node_test import ( "context" "testing" "github.com/cowsql/go-cowsql/client" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/node" ) // The raft identity (ID and address) of a node depends on the value of // cluster.https_address and the entries of the raft_nodes table. func TestDetermineRaftNode(t *testing.T) { cases := []struct { title string address string // Value of cluster.https_address addresses []string // Entries in raft_nodes node *db.RaftNode // Expected node value }{ { `no cluster.https_address set`, "", []string{}, &db.RaftNode{NodeInfo: client.NodeInfo{ID: 1}}, }, { `cluster.https_address set and no raft_nodes rows`, "1.2.3.4:8443", []string{}, &db.RaftNode{NodeInfo: client.NodeInfo{ID: 1}}, }, { `cluster.https_address set and matching the one and only raft_nodes row`, "1.2.3.4:8443", []string{"1.2.3.4:8443"}, &db.RaftNode{NodeInfo: client.NodeInfo{ID: 1, Address: "1.2.3.4:8443"}}, }, { `cluster.https_address set and matching one of many raft_nodes rows`, "5.6.7.8:999", []string{"1.2.3.4:666", "5.6.7.8:999"}, &db.RaftNode{NodeInfo: client.NodeInfo{ID: 2, Address: "5.6.7.8:999"}}, }, { `core.cluster set and no matching raft_nodes row`, "1.2.3.4:666", []string{"5.6.7.8:999"}, nil, }, } for _, c := range cases { t.Run(c.title, func(t *testing.T) { tx, cleanup := db.NewTestNodeTx(t) defer cleanup() err := tx.UpdateConfig(map[string]string{"cluster.https_address": c.address}) require.NoError(t, err) for _, address := range c.addresses { _, err := tx.CreateRaftNode(address, "test") require.NoError(t, err) } node, err := node.DetermineRaftNode(context.Background(), tx) require.NoError(t, err) if c.node == nil { assert.Nil(t, node) } else { assert.Equal(t, c.node.ID, node.ID) assert.Equal(t, c.node.Address, node.Address) } }) } } incus-7.0.0/internal/server/operations/000077500000000000000000000000001517523235500201545ustar00rootroot00000000000000incus-7.0.0/internal/server/operations/linux.go000066400000000000000000000027011517523235500216420ustar00rootroot00000000000000//go:build linux && cgo && !agent package operations import ( "context" "fmt" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/shared/api" ) func registerDBOperation(op *Operation, opType operationtype.Type) error { if op.state == nil { return nil } err := op.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { opInfo := cluster.Operation{ UUID: op.id, Type: opType, NodeID: tx.GetNodeID(), } if op.projectName != "" { projectID, err := cluster.GetProjectID(ctx, tx.Tx(), op.projectName) if err != nil { return fmt.Errorf("Fetch project ID: %w", err) } opInfo.ProjectID = &projectID } _, err := cluster.CreateOrReplaceOperation(ctx, tx.Tx(), opInfo) return err }) if err != nil { return fmt.Errorf("failed to add %q Operation %s to database: %w", opType.Description(), op.id, err) } return nil } func removeDBOperation(op *Operation) error { if op.state == nil { return nil } err := op.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return cluster.DeleteOperation(ctx, tx.Tx(), op.id) }) return err } func (op *Operation) sendEvent(eventMessage any) { if op.events == nil { return } _ = op.events.Send(op.projectName, api.EventTypeOperation, eventMessage) } incus-7.0.0/internal/server/operations/notlinux.go000066400000000000000000000012571517523235500223700ustar00rootroot00000000000000//go:build !linux || !cgo || agent package operations import ( "errors" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/shared/api" ) func registerDBOperation(op *Operation, opType operationtype.Type) error { if op.state != nil { return errors.New("registerDBOperation not supported on this platform") } return nil } func removeDBOperation(op *Operation) error { if op.state != nil { return errors.New("registerDBOperation not supported on this platform") } return nil } func (op *Operation) sendEvent(eventMessage any) { if op.events == nil { return } op.events.Send(op.projectName, api.EventTypeOperation, eventMessage) } incus-7.0.0/internal/server/operations/operations.go000066400000000000000000000421141517523235500226700ustar00rootroot00000000000000package operations import ( "context" "errors" "fmt" "maps" "net/http" "sync" "time" "github.com/google/uuid" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/db/operationtype" "github.com/lxc/incus/v7/internal/server/events" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/cancel" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) var debug bool var ( operationsLock sync.Mutex operations = make(map[string]*Operation) ) // OperationClass represents the OperationClass type. type OperationClass int const ( // OperationClassTask represents the Task OperationClass. OperationClassTask OperationClass = 1 // OperationClassWebsocket represents the Websocket OperationClass. OperationClassWebsocket OperationClass = 2 // OperationClassToken represents the Token OperationClass. OperationClassToken OperationClass = 3 ) func (t OperationClass) String() string { return map[OperationClass]string{ OperationClassTask: api.OperationClassTask, OperationClassWebsocket: api.OperationClassWebsocket, OperationClassToken: api.OperationClassToken, }[t] } // Init sets the debug value for the operations package. func Init(d bool) { debug = d } // Lock locks the operations mutex. func Lock() { operationsLock.Lock() } // Unlock unlocks the operations mutex. func Unlock() { operationsLock.Unlock() } // Clone returns a clone of the internal operations map containing references to the actual operations. func Clone() map[string]*Operation { operationsLock.Lock() defer operationsLock.Unlock() return util.CloneMap(operations) } // OperationGetInternal returns the operation with the given id. It returns an // error if it doesn't exist. func OperationGetInternal(id string) (*Operation, error) { operationsLock.Lock() op, ok := operations[id] operationsLock.Unlock() if !ok { return nil, fmt.Errorf("Operation '%s' doesn't exist", id) } return op, nil } // Operation represents an operation. type Operation struct { projectName string id string class OperationClass createdAt time.Time updatedAt time.Time status api.StatusCode url string resources map[string][]api.URL metadata map[string]any err error readonly bool canceler *cancel.HTTPRequestCanceller description string objectType auth.ObjectType entitlement auth.Entitlement dbOpType operationtype.Type requestor *api.EventLifecycleRequestor logger logger.Logger // Those functions are called at various points in the Operation lifecycle onRun func(*Operation) error onCancel func(*Operation) error onConnect func(*Operation, *http.Request, http.ResponseWriter) error // Indicates if operation has finished. finished *cancel.Canceller // Locking for concurrent access to the Operation lock sync.Mutex state *state.State events *events.Server } // OperationCreate creates a new operation and returns it. If it cannot be // created, it returns an error. func OperationCreate(s *state.State, projectName string, opClass OperationClass, opType operationtype.Type, opResources map[string][]api.URL, opMetadata any, onRun func(*Operation) error, onCancel func(*Operation) error, onConnect func(*Operation, *http.Request, http.ResponseWriter) error, r *http.Request) (*Operation, error) { // Don't allow new operations when Incus is shutting down. if s != nil && errors.Is(s.ShutdownCtx.Err(), context.Canceled) { return nil, errors.New("Incus is shutting down") } // Main attributes op := Operation{} op.projectName = projectName op.id = uuid.New().String() op.description = opType.Description() op.objectType, op.entitlement = opType.Permission() op.dbOpType = opType op.class = opClass op.createdAt = time.Now() op.updatedAt = op.createdAt op.status = api.Pending op.url = fmt.Sprintf("/%s/operations/%s", version.APIVersion, op.id) op.resources = opResources op.finished = cancel.New(context.Background()) op.state = s op.logger = logger.AddContext(logger.Ctx{"operation": op.id, "project": op.projectName, "class": op.class.String(), "description": op.description}) if s != nil { op.SetEventServer(s.Events) } // Validate and make a copy of the metadata to avoid concurrent reads/writes. newMetadata, err := parseMetadata(opMetadata) if err != nil { return nil, err } op.metadata = newMetadata // Callback functions op.onRun = onRun op.onCancel = onCancel op.onConnect = onConnect // Quick check. if op.class != OperationClassWebsocket && op.onConnect != nil { return nil, errors.New("Only websocket operations can have a Connect hook") } if op.class == OperationClassWebsocket && op.onConnect == nil { return nil, errors.New("Websocket operations must have a Connect hook") } if op.class == OperationClassToken && op.onRun != nil { return nil, errors.New("Token operations can't have a Run hook") } if op.class == OperationClassToken && op.onCancel != nil { return nil, errors.New("Token operations can't have a Cancel hook") } // Set requestor if request was provided. if r != nil { op.SetRequestor(r) } operationsLock.Lock() operations[op.id] = &op operationsLock.Unlock() err = registerDBOperation(&op, opType) if err != nil { return nil, err } op.logger.Debug("New operation") _, md, _ := op.Render() operationsLock.Lock() op.sendEvent(md) operationsLock.Unlock() return &op, nil } // SetEventServer allows injection of event server. func (op *Operation) SetEventServer(events *events.Server) { op.events = events } // SetRequestor sets a requestor for this operation from an http.Request. func (op *Operation) SetRequestor(r *http.Request) { op.requestor = request.CreateRequestor(r) } // IsSameRequestor compares the current request requestor to that stored (if any). func (op *Operation) IsSameRequestor(r *http.Request) bool { // If no requestor was previously recorded, it's not the same requestor. if op.requestor == nil { return false } // Compare with the recorded requestor. curRequestor := request.CreateRequestor(r) if op.requestor.Username != curRequestor.Username || op.requestor.Protocol != curRequestor.Protocol { return false } return true } // CopyRequestor sets a requestor to match that of another operation. func (op *Operation) CopyRequestor(otherOp *Operation) { op.requestor = otherOp.requestor } // Requestor returns the initial requestor for this operation. func (op *Operation) Requestor() *api.EventLifecycleRequestor { return op.requestor } func (op *Operation) done() { if op.readonly { return } op.lock.Lock() op.readonly = true op.onRun = nil op.onCancel = nil op.onConnect = nil op.finished.Cancel() op.lock.Unlock() go func() { shutdownCtx := context.Background() if op.state != nil { shutdownCtx = op.state.ShutdownCtx } select { case <-shutdownCtx.Done(): return // Expect all operation records to be removed by waitForOperations in one query. case <-time.After(time.Second * 5): // Wait 5s before removing from internal map and database. } operationsLock.Lock() _, ok := operations[op.id] if !ok { operationsLock.Unlock() return } delete(operations, op.id) operationsLock.Unlock() if op.state == nil { return } err := removeDBOperation(op) if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { // Operations can be deleted from the database before the operation clean up go routine has // run in cases where the project that the operation(s) are associated to is deleted first. // So don't log warning if operation not found. op.logger.Warn("Failed to delete operation", logger.Ctx{"status": op.status, "err": err}) } }() } // Start a pending operation. It returns an error if the operation cannot be started. func (op *Operation) Start() error { op.lock.Lock() if op.status != api.Pending { op.lock.Unlock() return errors.New("Only pending operations can be started") } op.status = api.Running if op.onRun != nil { go func(op *Operation) { err := op.onRun(op) if err != nil { op.lock.Lock() op.status = api.Failure op.err = err op.lock.Unlock() op.done() op.logger.Debug("Failure for operation", logger.Ctx{"err": err}) _, md, _ := op.Render() op.lock.Lock() op.sendEvent(md) op.lock.Unlock() return } op.lock.Lock() op.status = api.Success op.lock.Unlock() op.done() op.logger.Debug("Success for operation") _, md, _ := op.Render() op.lock.Lock() op.sendEvent(md) op.lock.Unlock() }(op) } op.lock.Unlock() op.logger.Debug("Started operation") _, md, _ := op.Render() op.lock.Lock() op.sendEvent(md) op.lock.Unlock() return nil } // Cancel cancels a running operation. If the operation cannot be cancelled, it // returns an error. func (op *Operation) Cancel() (chan error, error) { op.lock.Lock() if op.status != api.Running { op.lock.Unlock() return nil, errors.New("Only running operations can be cancelled") } if !op.mayCancel() { op.lock.Unlock() return nil, errors.New("This operation can't be cancelled") } chanCancel := make(chan error, 1) oldStatus := op.status op.status = api.Cancelling op.lock.Unlock() hasOnCancel := op.onCancel != nil if hasOnCancel { go func(op *Operation, oldStatus api.StatusCode, chanCancel chan error) { err := op.onCancel(op) if err != nil { op.lock.Lock() op.status = oldStatus op.lock.Unlock() chanCancel <- err op.logger.Debug("Failed to cancel operation", logger.Ctx{"err": err}) _, md, _ := op.Render() op.lock.Lock() op.sendEvent(md) op.lock.Unlock() return } op.lock.Lock() op.status = api.Cancelled op.lock.Unlock() op.done() chanCancel <- nil op.logger.Debug("Cancelled operation") _, md, _ := op.Render() op.lock.Lock() op.sendEvent(md) op.lock.Unlock() }(op, oldStatus, chanCancel) } op.logger.Debug("Cancelling operation") _, md, _ := op.Render() op.sendEvent(md) if op.canceler != nil { err := op.canceler.Cancel() if err != nil { return nil, err } } if !hasOnCancel { op.lock.Lock() op.status = api.Cancelled op.lock.Unlock() op.done() chanCancel <- nil } op.logger.Debug("Cancelled operation") _, md, _ = op.Render() op.lock.Lock() op.sendEvent(md) op.lock.Unlock() return chanCancel, nil } // Connect connects a websocket operation. If the operation is not a websocket // operation or the operation is not running, it returns an error. func (op *Operation) Connect(r *http.Request, w http.ResponseWriter) (chan error, error) { op.lock.Lock() if op.class != OperationClassWebsocket { op.lock.Unlock() return nil, errors.New("Only websocket operations can be connected") } if op.status != api.Running { op.lock.Unlock() return nil, errors.New("Only running operations can be connected") } chanConnect := make(chan error, 1) go func(op *Operation, chanConnect chan error) { err := op.onConnect(op, r, w) if err != nil { chanConnect <- err op.logger.Debug("Failed to connect to operation", logger.Ctx{"err": err}) return } chanConnect <- nil op.logger.Debug("Connected to operation") }(op, chanConnect) op.lock.Unlock() op.logger.Debug("Connecting to operation") return chanConnect, nil } func (op *Operation) mayCancel() bool { if op.class == OperationClassToken { return true } if op.onCancel != nil { return true } if op.canceler != nil && op.canceler.Cancelable() { return true } return false } // Render renders the operation structure. // Returns URL of operation and operation info. func (op *Operation) Render() (string, *api.Operation, error) { // Setup the resource URLs renderedResources := make(map[string][]string) resources := op.resources if resources != nil { tmpResources := make(map[string][]string) for key, value := range resources { var values []string for _, c := range value { values = append(values, c.Project(op.Project()).String()) } tmpResources[key] = values } renderedResources = tmpResources } op.lock.Lock() // Make a read-only copy of the metadata to avoid concurrent reads/writes. metadata := map[string]any{} for k, v := range op.metadata { metadata[k] = v } // Put together the response struct. retOp := &api.Operation{ ID: op.id, Class: op.class.String(), Description: op.description, CreatedAt: op.createdAt, UpdatedAt: op.updatedAt, Status: op.status.String(), StatusCode: op.status, Resources: renderedResources, Metadata: metadata, MayCancel: op.mayCancel(), } if op.state != nil { retOp.Location = op.state.ServerName } if op.err != nil { retOp.Err = response.SmartError(op.err).String() } op.lock.Unlock() return op.url, retOp, nil } // Wait for the operation to be done. // Returns non-nil error if operation failed or context was cancelled. func (op *Operation) Wait(ctx context.Context) error { select { case <-op.finished.Done(): return op.err case <-ctx.Done(): return ctx.Err() } } // UpdateResources updates the resources of the operation. It returns an error // if the operation is not pending or running, or the operation is read-only. func (op *Operation) UpdateResources(opResources map[string][]api.URL) error { op.lock.Lock() if op.status != api.Pending && op.status != api.Running { op.lock.Unlock() return errors.New("Only pending or running operations can be updated") } if op.readonly { op.lock.Unlock() return errors.New("Read-only operations can't be updated") } op.updatedAt = time.Now() op.resources = opResources op.lock.Unlock() op.logger.Debug("Updated resources for oeration") _, md, _ := op.Render() op.lock.Lock() op.sendEvent(md) op.lock.Unlock() return nil } // UpdateMetadata updates the metadata of the operation. It returns an error // if the operation is not pending or running, or the operation is read-only. func (op *Operation) UpdateMetadata(opMetadata any) error { op.lock.Lock() if op.status != api.Pending && op.status != api.Running { op.lock.Unlock() return errors.New("Only pending or running operations can be updated") } if op.readonly { op.lock.Unlock() return errors.New("Read-only operations can't be updated") } // Validate and make a copy of the metadata to avoid concurrent reads/writes. newMetadata, err := parseMetadata(opMetadata) if err != nil { op.lock.Unlock() return err } op.updatedAt = time.Now() op.metadata = newMetadata op.lock.Unlock() op.logger.Debug("Updated metadata for operation") _, md, _ := op.Render() op.lock.Lock() op.sendEvent(md) op.lock.Unlock() return nil } // ExtendMetadata updates the metadata of the operation with the additional data provided. // It returns an error if the operation is not pending or running, or the operation is read-only. func (op *Operation) ExtendMetadata(metadata any) error { op.lock.Lock() // Quick checks. if op.status != api.Pending && op.status != api.Running { op.lock.Unlock() return errors.New("Only pending or running operations can be updated") } if op.readonly { op.lock.Unlock() return errors.New("Read-only operations can't be updated") } // Parse the new metadata. extraMetadata, err := parseMetadata(metadata) if err != nil { op.lock.Unlock() return err } // Get current metadata. newMetadata := op.metadata // Merge with current one. if op.metadata == nil { newMetadata = extraMetadata } else { maps.Copy(newMetadata, extraMetadata) } // Update the operation. op.updatedAt = time.Now() op.metadata = newMetadata op.lock.Unlock() op.logger.Debug("Updated metadata for operation") _, md, _ := op.Render() op.lock.Lock() op.sendEvent(md) op.lock.Unlock() return nil } // ID returns the operation ID. func (op *Operation) ID() string { return op.id } // Metadata returns a read-only copy of the operation metadata. func (op *Operation) Metadata() map[string]any { op.lock.Lock() defer op.lock.Unlock() // Make a read-only copy of the metadata to avoid concurrent reads/writes. metadata := map[string]any{} for k, v := range op.metadata { metadata[k] = v } return metadata } // URL returns the operation URL. func (op *Operation) URL() string { return op.url } // Resources returns the operation resources. func (op *Operation) Resources() map[string][]api.URL { return op.resources } // SetCanceler sets a canceler. func (op *Operation) SetCanceler(canceler *cancel.HTTPRequestCanceller) { op.canceler = canceler } // Permission returns the operations auth.ObjectType and auth.Entitlement. func (op *Operation) Permission() (auth.ObjectType, auth.Entitlement) { return op.objectType, op.entitlement } // Project returns the operation project. func (op *Operation) Project() string { return op.projectName } // Status returns the operation status. func (op *Operation) Status() api.StatusCode { return op.status } // Class returns the operation class. func (op *Operation) Class() OperationClass { return op.class } // Type returns the db operation type. func (op *Operation) Type() operationtype.Type { return op.dbOpType } incus-7.0.0/internal/server/operations/response.go000066400000000000000000000047761517523235500223570ustar00rootroot00000000000000package operations import ( "fmt" "net/http" "github.com/lxc/incus/v7/internal/server/response" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) // Operation response. type operationResponse struct { op *Operation } // OperationResponse returns an operation response. func OperationResponse(op *Operation) response.Response { return &operationResponse{op} } func (r *operationResponse) Render(w http.ResponseWriter) error { err := r.op.Start() if err != nil { return err } url, md, err := r.op.Render() if err != nil { return err } body := api.ResponseRaw{ Type: api.AsyncResponse, Status: api.OperationCreated.String(), StatusCode: int(api.OperationCreated), Operation: url, Metadata: md, } w.Header().Set("Location", url) w.WriteHeader(http.StatusAccepted) var debugLogger logger.Logger if debug { debugLogger = logger.AddContext(logger.Ctx{"http_code": http.StatusAccepted}) } return localUtil.WriteJSON(w, body, debugLogger) } func (r *operationResponse) String() string { _, md, err := r.op.Render() if err != nil { return fmt.Sprintf("error: %s", err) } return md.ID } // Code returns the HTTP code. func (r *operationResponse) Code() int { return http.StatusAccepted } // Forwarded operation response. // // Returned when the operation has been created on another node. type forwardedOperationResponse struct { op *api.Operation } // ForwardedOperationResponse creates a response that forwards the metadata of // an operation created on another node. func ForwardedOperationResponse(op *api.Operation) response.Response { return &forwardedOperationResponse{ op: op, } } func (r *forwardedOperationResponse) Render(w http.ResponseWriter) error { url := fmt.Sprintf("/%s/operations/%s", version.APIVersion, r.op.ID) body := api.ResponseRaw{ Type: api.AsyncResponse, Status: api.OperationCreated.String(), StatusCode: int(api.OperationCreated), Operation: url, Metadata: r.op, } w.Header().Set("Location", url) w.WriteHeader(http.StatusAccepted) var debugLogger logger.Logger if debug { debugLogger = logger.AddContext(logger.Ctx{"http_code": http.StatusAccepted}) } return localUtil.WriteJSON(w, body, debugLogger) } func (r *forwardedOperationResponse) String() string { return r.op.ID } // Code returns the HTTP code. func (r *forwardedOperationResponse) Code() int { return http.StatusAccepted } incus-7.0.0/internal/server/operations/util.go000066400000000000000000000036221517523235500214630ustar00rootroot00000000000000package operations import ( "errors" "fmt" "reflect" "strconv" "github.com/lxc/incus/v7/shared/units" ) func parseMetadata(metadata any) (map[string]any, error) { newMetadata := make(map[string]any) s := reflect.ValueOf(metadata) if !s.IsValid() { return nil, nil } if s.Kind() == reflect.Map { for _, k := range s.MapKeys() { if k.Kind() != reflect.String { return nil, errors.New("Invalid metadata provided (key isn't a string)") } newMetadata[k.String()] = s.MapIndex(k).Interface() } } else if s.Kind() == reflect.Ptr && !s.Elem().IsValid() { return nil, nil } else { return nil, errors.New("Invalid metadata provided (type isn't a map)") } return newMetadata, nil } // SetProgressMetadata updates an operation metadata map with the provided progress data. func SetProgressMetadata(metadata map[string]any, stage, displayPrefix string, percent, processed, speed int64) { progress := make(map[string]string) // stage, percent, speed sent for API callers. progress["stage"] = stage if processed > 0 { progress["processed"] = strconv.FormatInt(processed, 10) } if percent > 0 { progress["percent"] = strconv.FormatInt(percent, 10) } progress["speed"] = strconv.FormatInt(speed, 10) metadata["progress"] = progress // _progress with formatted text. if percent > 0 { if speed > 0 { metadata[stage+"_progress"] = fmt.Sprintf("%s: %d%% (%s/s)", displayPrefix, percent, units.GetByteSizeString(speed, 2)) } else { metadata[stage+"_progress"] = fmt.Sprintf("%s: %d%%", displayPrefix, percent) } } else if processed > 0 { metadata[stage+"_progress"] = fmt.Sprintf("%s: %s (%s/s)", displayPrefix, units.GetByteSizeString(processed, 2), units.GetByteSizeString(speed, 2)) } else if speed > 0 { metadata[stage+"_progress"] = fmt.Sprintf("%s: %s/s", displayPrefix, units.GetByteSizeString(speed, 2)) } else { metadata[stage+"_progress"] = displayPrefix } } incus-7.0.0/internal/server/operations/websocket.go000066400000000000000000000033771517523235500225030ustar00rootroot00000000000000package operations import ( "fmt" "net/http" "github.com/gorilla/websocket" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/shared/ws" ) type operationWebSocket struct { req *http.Request op *Operation } // OperationWebSocket returns a new websocket operation. func OperationWebSocket(req *http.Request, op *Operation) response.Response { return &operationWebSocket{req, op} } func (r *operationWebSocket) Render(w http.ResponseWriter) error { chanErr, err := r.op.Connect(r.req, w) if err != nil { return err } err = <-chanErr return err } func (r *operationWebSocket) String() string { _, md, err := r.op.Render() if err != nil { return fmt.Sprintf("error: %s", err) } return md.ID } // Code returns the HTTP code. func (r *operationWebSocket) Code() int { return http.StatusOK } type forwardedOperationWebSocket struct { req *http.Request id string source *websocket.Conn // Connection to the node were the operation is running } // ForwardedOperationWebSocket returns a new forwarted websocket operation. func ForwardedOperationWebSocket(req *http.Request, id string, source *websocket.Conn) response.Response { return &forwardedOperationWebSocket{req, id, source} } func (r *forwardedOperationWebSocket) Render(w http.ResponseWriter) error { // Upgrade target connection to websocket. target, err := ws.Upgrader.Upgrade(w, r.req, nil) if err != nil { return err } // Start proxying between sockets. <-ws.Proxy(r.source, target) // Make sure both sides are closed. _ = r.source.Close() _ = target.Close() return nil } func (r *forwardedOperationWebSocket) String() string { return r.id } // Code returns the HTTP code. func (r *forwardedOperationWebSocket) Code() int { return http.StatusOK } incus-7.0.0/internal/server/project/000077500000000000000000000000001517523235500174375ustar00rootroot00000000000000incus-7.0.0/internal/server/project/permission_internal_test.go000066400000000000000000000026061517523235500251150ustar00rootroot00000000000000package project import ( "testing" "github.com/stretchr/testify/assert" "github.com/lxc/incus/v7/shared/idmap" ) func TestParseHostIDMapRange(t *testing.T) { for _, mode := range []string{"uid", "gid", "both"} { var isUID, isGID bool switch mode { case "uid": isUID = true case "gid": isGID = true case "both": isUID = true isGID = true } idmaps, err := parseHostIDMapRange(isUID, isGID, "foo") assert.NotErrorIs(t, err, nil) assert.Nil(t, idmaps) idmaps, err = parseHostIDMapRange(isUID, isGID, "1000") expected := []idmap.Entry{ { IsUID: isUID, IsGID: isGID, HostID: 1000, MapRange: 1, NSID: -1, }, } assert.ErrorIs(t, err, nil) assert.Equal(t, idmaps, expected) idmaps, err = parseHostIDMapRange(isUID, isGID, "1000-1001") expected = []idmap.Entry{ { IsUID: isUID, IsGID: isGID, HostID: 1000, MapRange: 2, NSID: -1, }, } assert.ErrorIs(t, err, nil) assert.Equal(t, idmaps, expected) idmaps, err = parseHostIDMapRange(isUID, isGID, "1000-1001,1002") expected = []idmap.Entry{ { IsUID: isUID, IsGID: isGID, HostID: 1000, MapRange: 2, NSID: -1, }, { IsUID: isUID, IsGID: isGID, HostID: 1002, MapRange: 1, NSID: -1, }, } assert.ErrorIs(t, err, nil) assert.Equal(t, idmaps, expected) } } incus-7.0.0/internal/server/project/permissions.go000066400000000000000000001502521517523235500223460ustar00rootroot00000000000000package project import ( "context" "errors" "fmt" "net/http" "net/url" "path/filepath" "slices" "strconv" "strings" "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" deviceconfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // HiddenStoragePools returns a list of storage pools that should be hidden from users of the project. func HiddenStoragePools(ctx context.Context, tx *db.ClusterTx, projectName string, allPoolNames []string) ([]string, error) { dbProject, err := cluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return nil, fmt.Errorf("Failed getting project: %w", err) } project, err := dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return nil, err } hiddenPools := []string{} for _, poolName := range allPoolNames { if !StoragePoolAllowed(project.Config, poolName) { hiddenPools = append(hiddenPools, poolName) } } return hiddenPools, nil } // AllowImageDownload returns an error if any project-specific restriction is violated when downloading a new image. func AllowImageDownload(tx *db.ClusterTx, projectName string, uri string) error { info, err := fetchProject(tx, projectName, true) if err != nil { return err } if info == nil { return nil } // Check if we have image server restrictions. if util.IsTrue(info.Project.Config["restricted"]) && info.Project.Config["restricted.images.servers"] != "" { u, err := url.Parse(uri) if err != nil { return err } servers := util.SplitNTrimSpace(info.Project.Config["restricted.images.servers"], ",", -1, false) if !slices.Contains(servers, u.Host) { return fmt.Errorf("Image server %q isn't allowed in this project", u.Host) } } return nil } // AllowInstanceCreation returns an error if any project-specific limit or // restriction is violated when creating a new instance. func AllowInstanceCreation(tx *db.ClusterTx, projectName string, req api.InstancesPost) error { info, err := fetchProject(tx, projectName, true) if err != nil { return err } if info == nil { return nil } var instanceType instancetype.Type switch req.Type { case api.InstanceTypeContainer: instanceType = instancetype.Container case api.InstanceTypeVM: instanceType = instancetype.VM default: return fmt.Errorf("Unexpected instance type %q", req.Type) } if req.Profiles == nil { req.Profiles = []string{"default"} } if util.IsTrue(info.Project.Config["restricted"]) { // Restricted projects aren't allowed to use pull migration. if req.Source.Type == "migration" && req.Source.Mode == "pull" { return errors.New("Restricted projects aren't allowed to use pull mode migration") } // Check if we have image server restrictions. if req.Source.Type == "image" && info.Project.Config["restricted.images.servers"] != "" { u, err := url.Parse(req.Source.Server) if err != nil { return err } servers := util.SplitNTrimSpace(info.Project.Config["restricted.images.servers"], ",", -1, false) if !slices.Contains(servers, u.Host) { return fmt.Errorf("Image server %q isn't allowed in this project", u.Host) } } } err = checkInstanceCountLimit(info, instanceType) if err != nil { return err } err = checkTotalInstanceCountLimit(info) if err != nil { return err } // Add the instance being created. info.Instances = append(info.Instances, api.Instance{ Name: req.Name, Project: projectName, InstancePut: req.InstancePut, Type: string(req.Type), }) // Special case restriction checks on volatile.* keys. strip := false // nolint:staticcheck if slices.Contains([]string{"copy", "migration"}, req.Source.Type) { // Allow stripping volatile keys if dealing with a copy or migration. strip = true } err = checkRestrictionsOnVolatileConfig( info.Project, instanceType, req.Name, req.Config, map[string]string{}, strip) if err != nil { return err } err = checkRestrictionsAndAggregateLimits(tx, info) if err != nil { return fmt.Errorf("Failed checking if instance creation allowed: %w", err) } return nil } // Check that we have not exceeded the maximum total allotted number of instances for both containers and vms. func checkTotalInstanceCountLimit(info *projectInfo) error { count, limit, err := getTotalInstanceCountLimit(info) if err != nil { return err } if limit >= 0 && count >= limit { return fmt.Errorf("Reached maximum number of instances in project %q", info.Project.Name) } return nil } func getTotalInstanceCountLimit(info *projectInfo) (int, int, error) { overallValue, ok := info.Project.Config["limits.instances"] if ok { limit, err := strconv.Atoi(overallValue) if err != nil { return -1, -1, err } return len(info.Instances), limit, nil } return len(info.Instances), -1, nil } // Check that we have not reached the maximum number of instances for this type. func checkInstanceCountLimit(info *projectInfo, instanceType instancetype.Type) error { count, limit, err := getInstanceCountLimit(info, instanceType) if err != nil { return err } if limit >= 0 && count >= limit { return fmt.Errorf("Reached maximum number of instances of type %q in project %q", instanceType, info.Project.Name) } return nil } func getInstanceCountLimit(info *projectInfo, instanceType instancetype.Type) (int, int, error) { var key string switch instanceType { case instancetype.Container: key = "limits.containers" case instancetype.VM: key = "limits.virtual-machines" default: return -1, -1, fmt.Errorf("Unexpected instance type %q", instanceType) } instanceCount := 0 for _, inst := range info.Instances { if inst.Type == instanceType.String() { instanceCount++ } } value, ok := info.Project.Config[key] if ok { limit, err := strconv.Atoi(value) if err != nil || limit < 0 { return -1, -1, fmt.Errorf("Unexpected %q value: %q", key, value) } return instanceCount, limit, nil } return instanceCount, -1, nil } // Check restrictions on setting volatile.* keys. func checkRestrictionsOnVolatileConfig(project api.Project, instanceType instancetype.Type, instanceName string, config, currentConfig map[string]string, strip bool) error { if project.Config["restrict"] == "false" { return nil } var restrictedLowLevel string switch instanceType { case instancetype.Container: restrictedLowLevel = "restricted.containers.lowlevel" case instancetype.VM: restrictedLowLevel = "restricted.virtual-machines.lowlevel" } if project.Config[restrictedLowLevel] == "allow" { return nil } // Checker for safe volatile keys. isSafeKey := func(key string) bool { if slices.Contains([]string{"volatile.apply_template", "volatile.base_image", "volatile.last_state.power"}, key) { return true } if strings.HasPrefix(key, instance.ConfigVolatilePrefix) { if strings.HasSuffix(key, ".apply_quota") { return true } if strings.HasSuffix(key, ".hwaddr") { return true } } return false } for key, value := range config { if !strings.HasPrefix(key, instance.ConfigVolatilePrefix) { continue } // Allow given safe volatile keys to be set if isSafeKey(key) { continue } if strip { delete(config, key) continue } currentValue, ok := currentConfig[key] if !ok { return fmt.Errorf("Setting %q on %s %q in project %q is forbidden", key, instanceType, instanceName, project.Name) } if currentValue != value { return fmt.Errorf("Changing %q on %s %q in project %q is forbidden", key, instanceType, instanceName, project.Name) } } return nil } // AllowVolumeCreation returns an error if any project-specific limit or // restriction is violated when creating a new custom volume in a project. func AllowVolumeCreation(tx *db.ClusterTx, projectName string, poolName string, req api.StorageVolumesPost) error { info, err := fetchProject(tx, projectName, true) if err != nil { return err } if info == nil { return nil } // Restricted projects aren't allowed to use pull migration. if util.IsTrue(info.Project.Config["restricted"]) && req.Source.Type == "migration" && req.Source.Mode == "pull" { return errors.New("Restricted projects aren't allowed to use pull mode migration") } // Add the volume being created. info.Volumes = append(info.Volumes, db.StorageVolumeArgs{ Name: req.Name, Config: req.Config, PoolName: poolName, }) err = checkRestrictionsAndAggregateLimits(tx, info) if err != nil { return fmt.Errorf("Failed checking if volume creation allowed: %w", err) } return nil } // GetImageSpaceBudget returns how much disk space is left in the given project // for writing images. // // If no limit is in place, return -1. func GetImageSpaceBudget(tx *db.ClusterTx, projectName string) (int64, error) { info, err := fetchProject(tx, projectName, true) if err != nil { return -1, err } if info == nil { return -1, nil } // If "features.images" is not enabled, the budget is unlimited. if util.IsFalse(info.Project.Config["features.images"]) { return -1, nil } return getSpaceBudget(info) } // GetSpaceBudget returns how much disk space is left in the given project. // // If no limit is in place, return -1. func GetSpaceBudget(tx *db.ClusterTx, projectName string) (int64, error) { info, err := fetchProject(tx, projectName, true) if err != nil { return -1, err } if info == nil { return -1, nil } return getSpaceBudget(info) } func getSpaceBudget(info *projectInfo) (int64, error) { // If "limits.disk" is not set, the budget is unlimited. if info.Project.Config["limits.disk"] == "" { return -1, nil } parser := aggregateLimitConfigValueParsers["limits.disk"] quota, err := parser(info.Project.Config["limits.disk"]) if err != nil { return -1, err } instances, err := expandInstancesConfigAndDevices(info.Instances, info.Profiles) if err != nil { return -1, err } info.Instances = instances totals, err := getTotalsAcrossProjectEntities(info, []string{"limits.disk"}, false) if err != nil { return -1, err } if totals["limits.disk"] < quota { return quota - totals["limits.disk"], nil } return 0, nil } // Check that we would not violate the project limits or restrictions if we // were to commit the given instances and profiles. func checkRestrictionsAndAggregateLimits(tx *db.ClusterTx, info *projectInfo) error { // List of config keys for which we need to check aggregate values // across all project instances. aggregateKeys := []string{} isRestricted := false for key, value := range info.Project.Config { if slices.Contains(allAggregateLimits, key) || strings.HasPrefix(key, projectLimitDiskPool) { aggregateKeys = append(aggregateKeys, key) continue } if key == "restricted" && util.IsTrue(value) { isRestricted = true continue } } if len(aggregateKeys) == 0 && !isRestricted { return nil } // Check pool usage restrictions. if isRestricted && info.Project.Config["restricted.storage-pools.access"] != "" { // Build a list of all the storage pools in use. pools := map[string]int{} for _, profile := range info.Profiles { for _, dev := range profile.Devices { if dev["type"] == "disk" && dev["pool"] != "" { pools[dev["pool"]]++ } } } for _, inst := range info.Instances { for _, dev := range inst.Devices { if dev["type"] == "disk" && dev["pool"] != "" { pools[dev["pool"]]++ } } } for _, vol := range info.Volumes { pools[vol.PoolName]++ } // Check that those pools are allowed. for poolName := range pools { if !StoragePoolAllowed(info.Project.Config, poolName) { return fmt.Errorf("Storage pool %q is not accessible from this project", poolName) } } } instances, err := expandInstancesConfigAndDevices(info.Instances, info.Profiles) if err != nil { return err } info.Instances = instances err = checkAggregateLimits(info, aggregateKeys) if err != nil { return err } if isRestricted { err = checkRestrictions(info.Project, info.Instances, info.Profiles) if err != nil { return err } } return nil } func getAggregateLimits(info *projectInfo, aggregateKeys []string) (map[string]api.ProjectStateResource, error) { result := map[string]api.ProjectStateResource{} if len(aggregateKeys) == 0 { return result, nil } totals, err := getTotalsAcrossProjectEntities(info, aggregateKeys, true) if err != nil { return nil, err } for _, key := range aggregateKeys { limit := int64(-1) limitStr := info.Project.Config[key] if limitStr != "" { keyName := key // Handle pool-specific limits. if strings.HasPrefix(key, projectLimitDiskPool) { keyName = "limits.disk" } parser := aggregateLimitConfigValueParsers[keyName] limit, err = parser(limitStr) if err != nil { return nil, err } } resource := api.ProjectStateResource{ Usage: totals[key], Limit: limit, } result[key] = resource } return result, nil } func checkAggregateLimits(info *projectInfo, aggregateKeys []string) error { if len(aggregateKeys) == 0 { return nil } totals, err := getTotalsAcrossProjectEntities(info, aggregateKeys, false) if err != nil { return fmt.Errorf("Failed getting usage of project entities: %w", err) } for _, key := range aggregateKeys { keyName := key // Handle pool-specific limits. if strings.HasPrefix(key, projectLimitDiskPool) { keyName = "limits.disk" } parser := aggregateLimitConfigValueParsers[keyName] limit, err := parser(info.Project.Config[key]) if err != nil { return err } if totals[key] > limit { return fmt.Errorf("Reached maximum aggregate value %q for %q in project %q", info.Project.Config[key], key, info.Project.Name) } } return nil } // parseHostIDMapRange parse the supplied list of host ID map ranges into a idmap.Entry slice. func parseHostIDMapRange(isUID bool, isGID bool, listValue string) ([]idmap.Entry, error) { var idmaps []idmap.Entry for _, listItem := range util.SplitNTrimSpace(listValue, ",", -1, true) { rangeStart, rangeSize, err := util.ParseUint32Range(listItem) if err != nil { return nil, err } idmaps = append(idmaps, idmap.Entry{ HostID: int64(rangeStart), MapRange: int64(rangeSize), IsUID: isUID, IsGID: isGID, NSID: -1, // We don't have this as we are just parsing host IDs. }) } return idmaps, nil } // Check that the project's restrictions are not violated across the given // instances and profiles. func checkRestrictions(project api.Project, instances []api.Instance, profiles []api.Profile) error { containerConfigChecks := map[string]func(value string) error{} devicesChecks := map[string]func(value map[string]string) error{} allowContainerLowLevel := false allowVMLowLevel := false var allowedIDMapHostUIDs, allowedIDMapHostGIDs []idmap.Entry for i := range allRestrictions { // Check if this particular restriction is defined explicitly in the project config. // If not, use the default value. Assign to local var so it doesn't change to the default value of // another restriction by time check functions run. restrictionKey := i restrictionValue, ok := project.Config[restrictionKey] if !ok { restrictionValue = allRestrictions[restrictionKey] } switch restrictionKey { case "restricted.containers.interception": for _, key := range allowableIntercept { containerConfigChecks[key] = func(instanceValue string) error { disabled := util.IsFalseOrEmpty(instanceValue) if restrictionValue != "allow" && !disabled { return errors.New("Container syscall interception is forbidden") } return nil } } case "restricted.containers.nesting": containerConfigChecks["security.nesting"] = func(instanceValue string) error { if restrictionValue == "block" && util.IsTrue(instanceValue) { return errors.New("Container nesting is forbidden") } return nil } case "restricted.containers.lowlevel": if restrictionValue == "allow" { allowContainerLowLevel = true } case "restricted.containers.privilege": containerConfigChecks["security.privileged"] = func(instanceValue string) error { if restrictionValue != "allow" && util.IsTrue(instanceValue) { return errors.New("Privileged containers are forbidden") } return nil } containerConfigChecks["security.idmap.isolated"] = func(instanceValue string) error { if restrictionValue == "isolated" && util.IsFalseOrEmpty(instanceValue) { return errors.New("Non-isolated containers are forbidden") } return nil } case "restricted.virtual-machines.lowlevel": if restrictionValue == "allow" { allowVMLowLevel = true } case "restricted.devices.unix-char": devicesChecks["unix-char"] = func(device map[string]string) error { if restrictionValue != "allow" { return errors.New("Unix character devices are forbidden") } return nil } case "restricted.devices.unix-block": devicesChecks["unix-block"] = func(device map[string]string) error { if restrictionValue != "allow" { return errors.New("Unix block devices are forbidden") } return nil } case "restricted.devices.unix-hotplug": devicesChecks["unix-hotplug"] = func(device map[string]string) error { if restrictionValue != "allow" { return errors.New("Unix hotplug devices are forbidden") } return nil } case "restricted.devices.infiniband": devicesChecks["infiniband"] = func(device map[string]string) error { if restrictionValue != "allow" { return errors.New("Infiniband devices are forbidden") } return nil } case "restricted.devices.gpu": devicesChecks["gpu"] = func(device map[string]string) error { if restrictionValue != "allow" { return errors.New("GPU devices are forbidden") } return nil } case "restricted.devices.usb": devicesChecks["usb"] = func(device map[string]string) error { if restrictionValue != "allow" { return errors.New("USB devices are forbidden") } return nil } case "restricted.devices.pci": devicesChecks["pci"] = func(device map[string]string) error { if restrictionValue != "allow" { return errors.New("PCI devices are forbidden") } return nil } case "restricted.devices.proxy": devicesChecks["proxy"] = func(device map[string]string) error { if restrictionValue != "allow" { return errors.New("Proxy devices are forbidden") } return nil } case "restricted.devices.nic": devicesChecks["nic"] = func(device map[string]string) error { // Check if the NICs are allowed at all. switch restrictionValue { case "block": return errors.New("Network devices are forbidden") case "managed": if device["network"] == "" { return errors.New("Only managed network devices are allowed") } } // Check if the NIC's parent/network setting is allowed based on the // restricted.devices.nic and restricted.networks.access settings. if device["network"] != "" { if !NetworkAllowed(project.Config, device["network"], true) { return errors.New("Network not allowed in project") } } else if device["parent"] != "" { if !NetworkAllowed(project.Config, device["parent"], false) { return errors.New("Network not allowed in project") } } return nil } case "restricted.devices.disk": devicesChecks["disk"] = func(device map[string]string) error { // The root device is always allowed, but the pool it references // must still be accessible (not equivalent to a size limit of 0). if device["path"] == "/" && device["pool"] != "" { if !StoragePoolAllowed(project.Config, device["pool"]) { return fmt.Errorf("Storage pool %q is not accessible from this project", device["pool"]) } return nil } // Always allow the cloud-init config drive. if device["path"] == "" && device["source"] == "cloud-init:config" { return nil } // Always allow the agent config drive. if device["path"] == "" && device["source"] == "agent:config" { return nil } switch restrictionValue { case "block": return errors.New("Disk devices are forbidden") case "managed": if device["pool"] == "" { return errors.New("Attaching disks not backed by a pool is forbidden") } case "allow": if device["pool"] == "" { allowed, _ := CheckRestrictedDevicesDiskPaths(project.Config, device["source"]) if !allowed { return fmt.Errorf("Disk source path %q not allowed", device["source"]) } } } if device["pool"] != "" && !StoragePoolAllowed(project.Config, device["pool"]) { return fmt.Errorf("Storage pool %q is not accessible from this project", device["pool"]) } return nil } case "restricted.idmap.uid": var err error allowedIDMapHostUIDs, err = parseHostIDMapRange(true, false, restrictionValue) if err != nil { return fmt.Errorf("Failed parsing %q: %w", "restricted.idmap.uid", err) } case "restricted.idmap.gid": var err error allowedIDMapHostGIDs, err = parseHostIDMapRange(false, true, restrictionValue) if err != nil { return fmt.Errorf("Failed parsing %q: %w", "restricted.idmap.uid", err) } } } // Common config check logic between instances and profiles. entityConfigChecker := func(instType instancetype.Type, entityName string, config map[string]string) error { entityTypeLabel := instType.String() if instType == instancetype.Any { entityTypeLabel = "profile" } isContainerOrProfile := instType == instancetype.Container || instType == instancetype.Any isVMOrProfile := instType == instancetype.VM || instType == instancetype.Any for key, value := range config { if ((isContainerOrProfile && !allowContainerLowLevel) || (isVMOrProfile && !allowVMLowLevel)) && key == "raw.idmap" { // If the low-level raw.idmap is used check whether the raw.idmap host IDs // are allowed based on the project's allowed ID map Host UIDs and GIDs. idmaps, err := idmap.NewSetFromIncusIDMap(value) if err != nil { return err } for i, entry := range idmaps.Entries { if !entry.HostIDsCoveredBy(allowedIDMapHostUIDs, allowedIDMapHostGIDs) { return fmt.Errorf(`Use of low-level "raw.idmap" element %d on %s %q of project %q is forbidden`, i, entityTypeLabel, entityName, project.Name) } } // Skip the other checks. continue } if isContainerOrProfile && !allowContainerLowLevel && isContainerLowLevelOptionForbidden(key) { return fmt.Errorf("Use of low-level config %q on %s %q of project %q is forbidden", key, entityTypeLabel, entityName, project.Name) } if isVMOrProfile && !allowVMLowLevel && isVMLowLevelOptionForbidden(key) { return fmt.Errorf("Use of low-level config %q on %s %q of project %q is forbidden", key, entityTypeLabel, entityName, project.Name) } var checker func(value string) error if isContainerOrProfile { checker = containerConfigChecks[key] } if checker == nil { continue } err := checker(value) if err != nil { return fmt.Errorf("Invalid value %q for config %q on %s %q of project %q: %w", value, key, instType, entityName, project.Name, err) } } return nil } // Common devices check logic between instances and profiles. entityDevicesChecker := func(instType instancetype.Type, entityName string, devices map[string]map[string]string) error { entityTypeLabel := instType.String() if instType == instancetype.Any { entityTypeLabel = "profile" } for name, device := range devices { check, ok := devicesChecks[device["type"]] if !ok { continue } err := check(device) if err != nil { return fmt.Errorf("Invalid device %q on %s %q of project %q: %w", name, entityTypeLabel, entityName, project.Name, err) } } return nil } for _, inst := range instances { instType, err := instancetype.New(inst.Type) if err != nil { return err } err = entityConfigChecker(instType, inst.Name, inst.Config) if err != nil { return err } err = entityDevicesChecker(instType, inst.Name, inst.Devices) if err != nil { return err } } for _, profile := range profiles { err := entityConfigChecker(instancetype.Any, profile.Name, profile.Config) if err != nil { return err } err = entityDevicesChecker(instancetype.Any, profile.Name, profile.Devices) if err != nil { return err } } return nil } // CheckRestrictedDevicesDiskPaths checks whether the disk's source path is within the allowed paths specified in // the project's restricted.devices.disk.paths config setting. // If no allowed paths are specified in project, then it allows all paths, and returns true and empty string. // If allowed paths are specified, and one matches, returns true and the matching allowed parent source path. // Otherwise if sourcePath not allowed returns false and empty string. func CheckRestrictedDevicesDiskPaths(projectConfig map[string]string, sourcePath string) (bool, string) { if projectConfig["restricted.devices.disk.paths"] == "" { return true, "" } // Clean, then add trailing slash, to ensure we are prefix matching on whole path. sourcePath = fmt.Sprintf("%s/", filepath.Clean(sourcePath)) for _, parentSourcePath := range strings.Split(projectConfig["restricted.devices.disk.paths"], ",") { // Clean, then add trailing slash, to ensure we are prefix matching on whole path. parentSourcePathTrailing := fmt.Sprintf("%s/", filepath.Clean(parentSourcePath)) if strings.HasPrefix(sourcePath, parentSourcePathTrailing) { return true, parentSourcePath } } return false, "" } var allAggregateLimits = []string{ "limits.cpu", "limits.disk", "limits.memory", "limits.processes", } // allRestrictions lists all available 'restrict.*' config keys along with their default setting. var allRestrictions = map[string]string{ "restricted.backups": "block", "restricted.cluster.groups": "", "restricted.cluster.target": "block", "restricted.containers.nesting": "block", "restricted.containers.interception": "block", "restricted.containers.lowlevel": "block", "restricted.containers.privilege": "unprivileged", "restricted.virtual-machines.lowlevel": "block", "restricted.devices.unix-char": "block", "restricted.devices.unix-block": "block", "restricted.devices.unix-hotplug": "block", "restricted.devices.infiniband": "block", "restricted.devices.gpu": "block", "restricted.devices.usb": "block", "restricted.devices.pci": "block", "restricted.devices.proxy": "block", "restricted.devices.nic": "managed", "restricted.devices.disk": "managed", "restricted.devices.disk.paths": "", "restricted.idmap.uid": "", "restricted.idmap.gid": "", "restricted.images.servers": "", "restricted.networks.access": "", "restricted.snapshots": "block", "restricted.storage-pools.access": "", } // allowableIntercept lists all syscall interception keys which may be allowed. var allowableIntercept = []string{ "security.syscalls.intercept.bpf", "security.syscalls.intercept.bpf.devices", "security.syscalls.intercept.mknod", "security.syscalls.intercept.mount", "security.syscalls.intercept.mount.fuse", "security.syscalls.intercept.setxattr", "security.syscalls.intercept.sysinfo", } // Return true if a low-level container option is forbidden. func isContainerLowLevelOptionForbidden(key string) bool { if strings.HasPrefix(key, "security.syscalls.intercept") && !slices.Contains(allowableIntercept, key) { return true } if strings.HasPrefix(key, "security.bpffs") { return true } if slices.Contains([]string{ "boot.host_shutdown_action", "boot.host_shutdown_timeout", "linux.kernel_modules", "limits.memory.swap", "limits.memory.oom_priority", "raw.apparmor", "raw.idmap", "raw.lxc", "raw.seccomp", "security.guestapi.images", "security.idmap.base", "security.idmap.size", }, key) { return true } return false } // Return true if a low-level VM option is forbidden. func isVMLowLevelOptionForbidden(key string) bool { return slices.Contains([]string{ "boot.host_shutdown_action", "boot.host_shutdown_timeout", "limits.memory.hugepages", "limits.memory.oom_priority", "raw.apparmor", "raw.idmap", "raw.qemu", "raw.qemu.conf", "raw.qemu.qmp.early", "raw.qemu.qmp.post-start", "raw.qemu.qmp.pre-start", "raw.qemu.scriptlet", }, key) } // AllowInstanceUpdate returns an error if any project-specific limit or // restriction is violated when updating an existing instance. func AllowInstanceUpdate(tx *db.ClusterTx, projectName, instanceName string, req api.InstancePut, currentConfig map[string]string) error { var updatedInstance *api.Instance info, err := fetchProject(tx, projectName, true) if err != nil { return err } if info == nil { return nil } // Change the instance being updated. for i, inst := range info.Instances { if inst.Name != instanceName { continue } info.Instances[i].Profiles = req.Profiles info.Instances[i].Config = req.Config info.Instances[i].Devices = req.Devices updatedInstance = &info.Instances[i] } instType, err := instancetype.New(updatedInstance.Type) if err != nil { return err } // Special case restriction checks on volatile.* keys, since we want to // detect if they were changed or added. err = checkRestrictionsOnVolatileConfig( info.Project, instType, updatedInstance.Name, req.Config, currentConfig, false) if err != nil { return err } err = checkRestrictionsAndAggregateLimits(tx, info) if err != nil { return fmt.Errorf("Failed checking if instance update allowed: %w", err) } return nil } // AllowVolumeUpdate returns an error if any project-specific limit or // restriction is violated when updating an existing custom volume. func AllowVolumeUpdate(tx *db.ClusterTx, projectName, volumeName string, req api.StorageVolumePut, currentConfig map[string]string) error { info, err := fetchProject(tx, projectName, true) if err != nil { return err } if info == nil { return nil } // If "limits.disk" is not set, there's nothing to do. if info.Project.Config["limits.disk"] == "" { return nil } // Change the volume being updated. for i, volume := range info.Volumes { if volume.Name != volumeName { continue } info.Volumes[i].Config = req.Config } err = checkRestrictionsAndAggregateLimits(tx, info) if err != nil { return fmt.Errorf("Failed checking if volume update allowed: %w", err) } return nil } // AllowProfileUpdate checks that project limits and restrictions are not // violated when changing a profile. func AllowProfileUpdate(tx *db.ClusterTx, projectName, profileName string, req api.ProfilePut) error { info, err := fetchProject(tx, projectName, true) if err != nil { return err } if info == nil { return nil } // Change the profile being updated. for i, profile := range info.Profiles { if profile.Name != profileName { continue } info.Profiles[i].Config = req.Config info.Profiles[i].Devices = req.Devices } err = checkRestrictionsAndAggregateLimits(tx, info) if err != nil { return fmt.Errorf("Failed checking if profile update allowed: %w", err) } return nil } // AllowProjectUpdate checks the new config to be set on a project is valid. func AllowProjectUpdate(tx *db.ClusterTx, projectName string, config map[string]string, changed []string) error { info, err := fetchProject(tx, projectName, false) if err != nil { return err } info.Instances, err = expandInstancesConfigAndDevices(info.Instances, info.Profiles) if err != nil { return err } // List of keys that need to check aggregate values across all project // instances. aggregateKeys := []string{} for _, key := range changed { if strings.HasPrefix(key, "restricted.") { project := api.Project{ Name: projectName, ProjectPut: api.ProjectPut{ Config: config, }, } err := checkRestrictions(project, info.Instances, info.Profiles) if err != nil { return fmt.Errorf("Conflict detected when changing %q in project %q: %w", key, projectName, err) } continue } switch key { case "limits.instances": err := validateTotalInstanceCountLimit(info.Instances, config[key], projectName) if err != nil { return fmt.Errorf("Can't change limits.instances in project %q: %w", projectName, err) } case "limits.containers": fallthrough case "limits.virtual-machines": err := validateInstanceCountLimit(info.Instances, key, config[key], projectName) if err != nil { return fmt.Errorf("Can't change %q in project %q: %w", key, projectName, err) } case "limits.processes": fallthrough case "limits.cpu": fallthrough case "limits.memory": fallthrough case "limits.disk": aggregateKeys = append(aggregateKeys, key) } } if len(aggregateKeys) > 0 { totals, err := getTotalsAcrossProjectEntities(info, aggregateKeys, false) if err != nil { return err } for _, key := range aggregateKeys { err := validateAggregateLimit(totals, key, config[key]) if err != nil { return err } } } return nil } // Check that limits.instances, i.e. the total limit of containers/virtual machines allocated // to the user is equal to or above the current count. func validateTotalInstanceCountLimit(instances []api.Instance, value, project string) error { if value == "" { return nil } limit, err := strconv.Atoi(value) if err != nil { return err } count := len(instances) if limit < count { return fmt.Errorf(`"limits.instances" is too low: there currently are %d total instances in project %q`, count, project) } return nil } // Check that limits.containers or limits.virtual-machines is equal or above // the current count. func validateInstanceCountLimit(instances []api.Instance, key, value, project string) error { if value == "" { return nil } instanceType := countConfigInstanceType[key] limit, err := strconv.Atoi(value) if err != nil { return err } dbType, err := instancetype.New(string(instanceType)) if err != nil { return err } count := 0 for _, inst := range instances { if inst.Type == dbType.String() { count++ } } if limit < count { return fmt.Errorf(`%q is too low: there currently are %d instances of type %s in project %q`, key, count, instanceType, project) } return nil } var countConfigInstanceType = map[string]api.InstanceType{ "limits.containers": api.InstanceTypeContainer, "limits.virtual-machines": api.InstanceTypeVM, } // Validates an aggregate limit, checking that the new value is not below the // current total amount. func validateAggregateLimit(totals map[string]int64, key, value string) error { if value == "" { return nil } keyName := key // Handle pool-specific limits. if strings.HasPrefix(key, projectLimitDiskPool) { keyName = "limits.disk" } parser := aggregateLimitConfigValueParsers[keyName] limit, err := parser(value) if err != nil { return fmt.Errorf("Invalid value %q for limit %q: %w", value, key, err) } total := totals[key] if limit < total { keyName := key // Handle pool-specific limits. if strings.HasPrefix(key, projectLimitDiskPool) { keyName = "limits.disk" } printer := aggregateLimitConfigValuePrinters[keyName] return fmt.Errorf("%q is too low: current total is %q", key, printer(total)) } return nil } // Return true if the project has some limits or restrictions set. func projectHasLimitsOrRestrictions(project api.Project) bool { for k, v := range project.Config { if strings.HasPrefix(k, "limits.") { return true } if k == "restricted" && util.IsTrue(v) { return true } } return false } // Hold information associated with the project, such as profiles and // instances. type projectInfo struct { Project api.Project Profiles []api.Profile Instances []api.Instance Volumes []db.StorageVolumeArgs } // Fetch the given project from the database along with its profiles, instances // and possibly custom volumes. // // If the skipIfNoLimits flag is true, then profiles, instances and volumes // won't be loaded if the profile has no limits set on it, and nil will be // returned. func fetchProject(tx *db.ClusterTx, projectName string, skipIfNoLimits bool) (*projectInfo, error) { ctx := context.Background() dbProject, err := cluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return nil, fmt.Errorf("Fetch project database object: %w", err) } project, err := dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return nil, err } if skipIfNoLimits && !projectHasLimitsOrRestrictions(*project) { return nil, nil } profilesFilter := cluster.ProfileFilter{} // If the project has the profiles feature enabled, we use its own // profiles to expand the instances configs, otherwise we use the // profiles from the default project. defaultProject := api.ProjectDefaultName if projectName == api.ProjectDefaultName || util.IsTrue(project.Config["features.profiles"]) { profilesFilter.Project = &projectName } else { profilesFilter.Project = &defaultProject } dbProfiles, err := cluster.GetProfiles(ctx, tx.Tx(), profilesFilter) if err != nil { return nil, fmt.Errorf("Fetch profiles from database: %w", err) } dbProfileConfigs, err := cluster.GetAllProfileConfigs(ctx, tx.Tx()) if err != nil { return nil, fmt.Errorf("Fetch profile configs from database: %w", err) } dbProfileDevices, err := cluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return nil, fmt.Errorf("Fetch profile devices from database: %w", err) } profiles := make([]api.Profile, 0, len(dbProfiles)) for _, profile := range dbProfiles { apiProfile, err := profile.ToAPI(ctx, tx.Tx(), dbProfileConfigs, dbProfileDevices) if err != nil { return nil, err } profiles = append(profiles, *apiProfile) } dbInstances, err := cluster.GetInstances(ctx, tx.Tx(), cluster.InstanceFilter{Project: &projectName}) if err != nil { return nil, fmt.Errorf("Fetch project instances from database: %w", err) } dbInstanceDevices, err := cluster.GetAllInstanceDevices(ctx, tx.Tx()) if err != nil { return nil, fmt.Errorf("Fetch instance devices from database: %w", err) } instances := make([]api.Instance, 0, len(dbInstances)) for _, inst := range dbInstances { apiInstance, err := inst.ToAPI(ctx, tx.Tx(), dbInstanceDevices, dbProfileConfigs, dbProfileDevices) if err != nil { return nil, fmt.Errorf("Failed to get API data for instance %q in project %q: %w", inst.Name, inst.Project, err) } instances = append(instances, *apiInstance) } volumes, err := tx.GetCustomVolumesInProject(ctx, projectName) if err != nil { return nil, fmt.Errorf("Fetch project custom volumes from database: %w", err) } info := &projectInfo{ Project: *project, Profiles: profiles, Instances: instances, Volumes: volumes, } return info, nil } // Expand the configuration and devices of the given instances, taking the give // project profiles into account. func expandInstancesConfigAndDevices(instances []api.Instance, profiles []api.Profile) ([]api.Instance, error) { expandedInstances := make([]api.Instance, len(instances)) // Index of all profiles by name. profilesByName := map[string]api.Profile{} for _, profile := range profiles { profilesByName[profile.Name] = profile } for i, inst := range instances { apiProfiles := make([]api.Profile, len(inst.Profiles)) for j, name := range inst.Profiles { profile := profilesByName[name] apiProfiles[j] = profile } expandedInstances[i] = inst expandedInstances[i].Config = db.ExpandInstanceConfig(inst.Config, apiProfiles) expandedInstances[i].Devices = db.ExpandInstanceDevices(deviceconfig.NewDevices(inst.Devices), apiProfiles).CloneNative() } return expandedInstances, nil } // Sum of the effective values for the given limits across all project // entities (instances and custom volumes). func getTotalsAcrossProjectEntities(info *projectInfo, keys []string, skipUnset bool) (map[string]int64, error) { totals := map[string]int64{} for _, key := range keys { totals[key] = 0 if key == "limits.disk" || strings.HasPrefix(key, projectLimitDiskPool) { poolName := "" fields := strings.SplitN(key, projectLimitDiskPool, 2) if len(fields) == 2 { poolName = fields[1] } for _, volume := range info.Volumes { if poolName != "" && volume.PoolName != poolName { continue } value, ok := volume.Config["size"] if !ok { if skipUnset { continue } return nil, fmt.Errorf(`Custom volume %q in project %q has no "size" config set`, volume.Name, info.Project.Name) } limit, err := units.ParseByteSizeString(value) if err != nil { return nil, fmt.Errorf(`Parse "size" for custom volume %q in project %q: %w`, volume.Name, info.Project.Name, err) } totals[key] += limit } } } for _, inst := range info.Instances { limits, err := getInstanceLimits(inst, keys, skipUnset) if err != nil { return nil, err } for _, key := range keys { totals[key] += limits[key] } } return totals, nil } // Return the effective instance-level values for the limits with the given keys. func getInstanceLimits(inst api.Instance, keys []string, skipUnset bool) (map[string]int64, error) { var err error limits := map[string]int64{} for _, key := range keys { var limit int64 keyName := key // Handle pool-specific limits. if strings.HasPrefix(key, projectLimitDiskPool) { keyName = "limits.disk" } parser := aggregateLimitConfigValueParsers[keyName] if key == "limits.disk" || strings.HasPrefix(key, projectLimitDiskPool) { poolName := "" fields := strings.SplitN(key, projectLimitDiskPool, 2) if len(fields) == 2 { poolName = fields[1] } _, device, err := instance.GetRootDiskDevice(inst.Devices) if err != nil { return nil, fmt.Errorf("Failed getting root disk device for instance %q in project %q: %w", inst.Name, inst.Project, err) } if poolName != "" && device["pool"] != poolName { continue } value, ok := device["size"] if !ok || value == "" { if skipUnset { continue } return nil, fmt.Errorf(`Instance %q in project %q has no "size" config set on the root device either directly or via a profile`, inst.Name, inst.Project) } limit, err = parser(value) if err != nil { if skipUnset { continue } return nil, fmt.Errorf("Failed parsing %q for instance %q in project %q", key, inst.Name, inst.Project) } // Add size.state accounting for VM root disks. if inst.Type == instancetype.VM.String() { sizeStateValue, ok := device["size.state"] if !ok { // TODO: In case the VMs storage drivers config drive size isn't the default, // the limits accounting will be incorrect. sizeStateValue = deviceconfig.DefaultVMBlockFilesystemSize } sizeStateLimit, err := parser(sizeStateValue) if err != nil { if skipUnset { continue } return nil, fmt.Errorf("Failed parsing %q for instance %q in project %q", "size.state", inst.Name, inst.Project) } limit += sizeStateLimit } } else { // Skip processing for 'limits.processes' if the instance type is VM, // as this limit is only applicable to containers. if key == "limits.processes" && inst.Type == instancetype.VM.String() { continue } value, ok := inst.Config[key] if !ok || value == "" { if skipUnset { continue } return nil, fmt.Errorf("Instance %q in project %q has no %q config, either directly or via a profile", inst.Name, inst.Project, key) } limit, err = parser(value) if err != nil { if skipUnset { continue } return nil, fmt.Errorf("Failed parsing %q for instance %q in project %q", key, inst.Name, inst.Project) } } limits[key] = limit } return limits, nil } var aggregateLimitConfigValueParsers = map[string]func(string) (int64, error){ "limits.memory": func(value string) (int64, error) { if strings.HasSuffix(value, "%") { return -1, errors.New("Value can't be a percentage") } return units.ParseByteSizeString(value) }, "limits.processes": func(value string) (int64, error) { limit, err := strconv.Atoi(value) if err != nil { return -1, err } return int64(limit), nil }, "limits.cpu": func(value string) (int64, error) { if strings.Contains(value, ",") || strings.Contains(value, "-") { return -1, errors.New("CPUs can't be pinned if project limits are used") } limit, err := strconv.Atoi(value) if err != nil { return -1, err } return int64(limit), nil }, "limits.disk": func(value string) (int64, error) { return units.ParseByteSizeString(value) }, } var aggregateLimitConfigValuePrinters = map[string]func(int64) string{ "limits.memory": func(limit int64) string { return units.GetByteSizeStringIEC(limit, 1) }, "limits.processes": func(limit int64) string { return fmt.Sprintf("%d", limit) }, "limits.cpu": func(limit int64) string { return fmt.Sprintf("%d", limit) }, "limits.disk": func(limit int64) string { return units.GetByteSizeStringIEC(limit, 1) }, } // FilterUsedBy filters a UsedBy list based on project access. func FilterUsedBy(authorizer auth.Authorizer, r *http.Request, entries []string) []string { // Filter the entries. usedBy := []string{} for _, entry := range entries { entityType, projectName, location, pathArgs, err := cluster.URLToEntityType(entry) if err != nil { continue } var object auth.Object switch entityType { case cluster.TypeImage: object = auth.ObjectImage(projectName, pathArgs[0]) case cluster.TypeInstance: object = auth.ObjectInstance(projectName, pathArgs[0]) case cluster.TypeNetwork: object = auth.ObjectNetwork(projectName, pathArgs[0]) case cluster.TypeProfile: object = auth.ObjectProfile(projectName, pathArgs[0]) case cluster.TypeStoragePool: object = auth.ObjectStoragePool(pathArgs[0]) case cluster.TypeStorageVolume: object = auth.ObjectStorageVolume(projectName, pathArgs[0], pathArgs[1], pathArgs[2], location) case cluster.TypeStorageBucket: object = auth.ObjectStorageBucket(projectName, pathArgs[0], pathArgs[1], location) default: continue } err = authorizer.CheckPermission(r.Context(), r, object, auth.EntitlementCanView) if err != nil { continue } usedBy = append(usedBy, entry) } return usedBy } // Return true if particular restriction in project is violated. func projectHasRestriction(project *api.Project, restrictionKey string, blockValue string) bool { if util.IsFalseOrEmpty(project.Config["restricted"]) { return false } restrictionValue, ok := project.Config[restrictionKey] if !ok { restrictionValue = allRestrictions[restrictionKey] } if restrictionValue == blockValue { return true } return false } // CheckClusterTargetRestriction check if user is allowed to use cluster member targeting. func CheckClusterTargetRestriction(authorizer auth.Authorizer, r *http.Request, project *api.Project, targetFlag string) error { if projectHasRestriction(project, "restricted.cluster.target", "block") && targetFlag != "" { // Allow server administrators to move instances around even when restricted (node evacuation, ...) err := authorizer.CheckPermission(r.Context(), r, auth.ObjectServer(), auth.EntitlementCanOverrideClusterTargetRestriction) if err != nil && api.StatusErrorCheck(err, http.StatusForbidden) { return api.StatusErrorf(http.StatusForbidden, "This project doesn't allow cluster member targeting") } else if err != nil { return err } } return nil } // AllowBackupCreation returns an error if any project-specific restriction is violated // when creating a new backup in a project. func AllowBackupCreation(tx *db.ClusterTx, projectName string) error { ctx := context.Background() dbProject, err := cluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return err } project, err := dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return err } if projectHasRestriction(project, "restricted.backups", "block") { return fmt.Errorf("Project %q doesn't allow for backup creation", projectName) } return nil } // AllowSnapshotCreation returns an error if any project-specific restriction is violated // when creating a new snapshot in a project. func AllowSnapshotCreation(p *api.Project) error { if projectHasRestriction(p, "restricted.snapshots", "block") { return fmt.Errorf("Project %q doesn't allow for snapshot creation", p.Name) } return nil } // GetRestrictedClusterGroups returns a slice of restricted cluster groups for the given project. func GetRestrictedClusterGroups(p *api.Project) []string { return util.SplitNTrimSpace(p.Config["restricted.cluster.groups"], ",", -1, true) } // AllowClusterMember returns nil if the given project is allowed to use the cluster member. func AllowClusterMember(p *api.Project, member *db.NodeInfo) error { clusterGroupsAllowed := GetRestrictedClusterGroups(p) if util.IsTrue(p.Config["restricted"]) && len(clusterGroupsAllowed) > 0 { for _, memberGroupName := range member.Groups { if slices.Contains(clusterGroupsAllowed, memberGroupName) { return nil } } return fmt.Errorf("Project isn't allowed to use this cluster member: %q", member.Name) } return nil } // AllowClusterGroup returns nil if the given project is allowed to use the cluster groupName. func AllowClusterGroup(p *api.Project, groupName string) error { clusterGroupsAllowed := GetRestrictedClusterGroups(p) // Skip the check if the project is not restricted if util.IsFalseOrEmpty(p.Config["restricted"]) { return nil } if len(clusterGroupsAllowed) > 0 && !slices.Contains(clusterGroupsAllowed, groupName) { return fmt.Errorf("Project isn't allowed to use this cluster group: %q", groupName) } return nil } // CheckTargetMember checks if the given targetMemberName is present in allMembers // and is allowed for the project. // If the target member is allowed it returns the resolved node information. func CheckTargetMember(p *api.Project, targetMemberName string, allMembers []db.NodeInfo) (*db.NodeInfo, error) { // Find target member. for _, potentialMember := range allMembers { if potentialMember.Name == targetMemberName { // If restricted groups are specified then check member is in at least one of them. err := AllowClusterMember(p, &potentialMember) if err != nil { return nil, api.StatusErrorf(http.StatusForbidden, "%s", err.Error()) } return &potentialMember, nil } } return nil, api.StatusErrorf(http.StatusNotFound, "Cluster member %q not found", targetMemberName) } // CheckTargetGroup checks if the given groupName is allowed for the project. func CheckTargetGroup(ctx context.Context, tx *db.ClusterTx, p *api.Project, groupName string) error { // If restricted groups are specified then check the requested group is in the list. err := AllowClusterGroup(p, groupName) if err != nil { return api.StatusErrorf(http.StatusForbidden, "%s", err.Error()) } // Check if the target group exists. targetGroupExists, err := cluster.ClusterGroupExists(ctx, tx.Tx(), groupName) if err != nil { return err } if !targetGroupExists { return api.StatusErrorf(http.StatusBadRequest, "Cluster group %q doesn't exist", groupName) } return nil } // CheckTarget checks if the given cluster target (member or group) is allowed. // If target is a cluster member and is found in allMembers it returns the resolved node information object. // If target is a cluster group it returns the cluster group name. // In case of error, neither node information nor cluster group name gets returned. func CheckTarget(ctx context.Context, authorizer auth.Authorizer, r *http.Request, tx *db.ClusterTx, p *api.Project, target string, allMembers []db.NodeInfo) (*db.NodeInfo, string, error) { // Extract the target. var targetGroupName string var targetMemberName string after, ok := strings.CutPrefix(target, "@") if ok { targetGroupName = after } else { targetMemberName = target } // Check manual cluster member targeting restrictions. err := CheckClusterTargetRestriction(authorizer, r, p, target) if err != nil { return nil, "", err } if targetMemberName != "" { member, err := CheckTargetMember(p, targetMemberName, allMembers) if err != nil { return nil, "", err } return member, "", nil } else if targetGroupName != "" { err := CheckTargetGroup(ctx, tx, p, targetGroupName) if err != nil { return nil, "", err } return nil, targetGroupName, nil } return nil, "", nil } incus-7.0.0/internal/server/project/permissions_test.go000066400000000000000000000210201517523235500233730ustar00rootroot00000000000000package project_test import ( "context" "crypto/rand" "crypto/rsa" "crypto/x509" "crypto/x509/pkix" "encoding/pem" "math/big" "net/http" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/certificate" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/request" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) // If there's no limit configured on the project, the check passes. func TestAllowInstanceCreation_NotConfigured(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() req := api.InstancesPost{ Name: "c1", Type: api.InstanceTypeContainer, } err := project.AllowInstanceCreation(tx, "default", req) assert.NoError(t, err) } // If a limit is configured and the current number of instances is below it, the check passes. func TestAllowInstanceCreation_Below(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() ctx := context.Background() id, err := cluster.CreateProject(ctx, tx.Tx(), cluster.Project{Name: "p1"}) require.NoError(t, err) err = cluster.CreateProjectConfig(ctx, tx.Tx(), id, map[string]string{"limits.containers": "5"}) require.NoError(t, err) _, err = cluster.CreateInstance(ctx, tx.Tx(), cluster.Instance{ Project: "p1", Name: "c1", Type: instancetype.Container, Architecture: 1, Node: "none", }) require.NoError(t, err) req := api.InstancesPost{ Name: "c2", Type: api.InstanceTypeContainer, } err = project.AllowInstanceCreation(tx, "p1", req) assert.NoError(t, err) } // If a limit is configured and it matches the current number of instances, the // check fails. func TestAllowInstanceCreation_Above(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() ctx := context.Background() id, err := cluster.CreateProject(ctx, tx.Tx(), cluster.Project{Name: "p1"}) require.NoError(t, err) err = cluster.CreateProjectConfig(ctx, tx.Tx(), id, map[string]string{"limits.containers": "1"}) require.NoError(t, err) _, err = cluster.CreateInstance(ctx, tx.Tx(), cluster.Instance{ Project: "p1", Name: "c1", Type: instancetype.Container, Architecture: 1, Node: "none", }) require.NoError(t, err) req := api.InstancesPost{ Name: "c2", Type: api.InstanceTypeContainer, } err = project.AllowInstanceCreation(tx, "p1", req) assert.EqualError(t, err, `Reached maximum number of instances of type "container" in project "p1"`) } // If a limit is configured, but for a different instance type, the check // passes. func TestAllowInstanceCreation_DifferentType(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() ctx := context.Background() id, err := cluster.CreateProject(ctx, tx.Tx(), cluster.Project{Name: "p1"}) require.NoError(t, err) err = cluster.CreateProjectConfig(ctx, tx.Tx(), id, map[string]string{"limits.containers": "1"}) require.NoError(t, err) _, err = cluster.CreateInstance(ctx, tx.Tx(), cluster.Instance{ Project: "p1", Name: "vm1", Type: instancetype.VM, Architecture: 1, Node: "none", }) require.NoError(t, err) req := api.InstancesPost{ Name: "c2", Type: api.InstanceTypeContainer, } err = project.AllowInstanceCreation(tx, "p1", req) assert.NoError(t, err) } // If a limit is configured, but the limit on instances is more // restrictive, the check fails. func TestAllowInstanceCreation_AboveInstances(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() ctx := context.Background() id, err := cluster.CreateProject(ctx, tx.Tx(), cluster.Project{Name: "p1"}) require.NoError(t, err) err = cluster.CreateProjectConfig(ctx, tx.Tx(), id, map[string]string{"limits.containers": "5", "limits.instances": "1"}) require.NoError(t, err) _, err = cluster.CreateInstance(ctx, tx.Tx(), cluster.Instance{ Project: "p1", Name: "c1", Type: instancetype.Container, Architecture: 1, Node: "none", }) require.NoError(t, err) req := api.InstancesPost{ Name: "c2", Type: api.InstanceTypeContainer, } err = project.AllowInstanceCreation(tx, "p1", req) assert.EqualError(t, err, `Reached maximum number of instances in project "p1"`) } // If a direct targeting is blocked, the check fails. func TestCheckClusterTargetRestriction_RestrictedTrue(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() ctx := context.Background() id, err := cluster.CreateProject(ctx, tx.Tx(), cluster.Project{Name: "p1"}) require.NoError(t, err) err = cluster.CreateProjectConfig(ctx, tx.Tx(), id, map[string]string{"restricted": "true", "restricted.cluster.target": "block"}) require.NoError(t, err) dbProject, err := cluster.GetProject(ctx, tx.Tx(), "p1") require.NoError(t, err) p, err := dbProject.ToAPI(ctx, tx.Tx()) require.NoError(t, err) req := &http.Request{} authorizer, err := auth.LoadAuthorizer(context.Background(), auth.DriverTLS, logger.Log, &certificate.Cache{}) require.NoError(t, err) err = project.CheckClusterTargetRestriction(authorizer, req, p, "n1") assert.EqualError(t, err, "This project doesn't allow cluster member targeting") } // If a direct targeting is blocked but the user can override it, the check passes. func TestCheckClusterTargetRestriction_RestrictedTrueWithOverride(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() ctx := context.Background() id, err := cluster.CreateProject(ctx, tx.Tx(), cluster.Project{Name: "p1"}) require.NoError(t, err) err = cluster.CreateProjectConfig(ctx, tx.Tx(), id, map[string]string{"restricted": "true", "restricted.cluster.target": "block"}) require.NoError(t, err) dbProject, err := cluster.GetProject(ctx, tx.Tx(), "p1") require.NoError(t, err) p, err := dbProject.ToAPI(ctx, tx.Tx()) require.NoError(t, err) req := &http.Request{ URL: &api.NewURL().Path("1.0", "instances").WithQuery("target", "node01").URL, } req = req.WithContext(context.WithValue(req.Context(), request.CtxProtocol, "tls")) req = req.WithContext(context.WithValue(req.Context(), request.CtxUsername, "my-certificate-fingerprint")) certificateCache := &certificate.Cache{} privKey, err := rsa.GenerateKey(rand.Reader, 2048) require.NoError(t, err) now := time.Now() template := &x509.Certificate{ SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test"}, NotBefore: now, NotAfter: now.Add(5 * time.Minute), KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, BasicConstraintsValid: true, } der, err := x509.CreateCertificate(rand.Reader, template, template, &privKey.PublicKey, privKey) require.NoError(t, err) // Setting the certificate and not projects means the certificate is not restricted and therefore the user is an // admin that can override the cluster target restriction. certificateCache.SetCertificates([]*api.Certificate{{ Fingerprint: "my-certificate-fingerprint", CertificatePut: api.CertificatePut{ Type: api.CertificateTypeClient, Certificate: string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})), }, }}) authorizer, err := auth.LoadAuthorizer(context.Background(), auth.DriverTLS, logger.Log, certificateCache) require.NoError(t, err) err = project.CheckClusterTargetRestriction(authorizer, req, p, "n1") assert.Nil(t, err) } // If a direct targeting is allowed, the check passes. func TestCheckClusterTargetRestriction_RestrictedFalse(t *testing.T) { tx, cleanup := db.NewTestClusterTx(t) defer cleanup() ctx := context.Background() id, err := cluster.CreateProject(ctx, tx.Tx(), cluster.Project{Name: "p1"}) require.NoError(t, err) err = cluster.CreateProjectConfig(ctx, tx.Tx(), id, map[string]string{"restricted": "false", "restricted.cluster.target": "block"}) require.NoError(t, err) dbProject, err := cluster.GetProject(ctx, tx.Tx(), "p1") require.NoError(t, err) p, err := dbProject.ToAPI(ctx, tx.Tx()) require.NoError(t, err) req := &http.Request{} authorizer, err := auth.LoadAuthorizer(context.Background(), auth.DriverTLS, logger.Log, &certificate.Cache{}) require.NoError(t, err) err = project.CheckClusterTargetRestriction(authorizer, req, p, "n1") assert.NoError(t, err) } incus-7.0.0/internal/server/project/project.go000066400000000000000000000357111517523235500214430ustar00rootroot00000000000000package project import ( "context" "fmt" "slices" "strings" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) // separator is used to delimit the project name from the suffix. const separator = "_" // projectLimitDiskPool is the prefix used for pool-specific disk limits. var projectLimitDiskPool = "limits.disk.pool." // Instance adds the "_" prefix to instance name when the given project name is not "default". func Instance(projectName string, instanceName string) string { if projectName != api.ProjectDefaultName { return fmt.Sprintf("%s%s%s", projectName, separator, instanceName) } return instanceName } // DNS adds "." as a suffix to instance name when the given project name is not "default". func DNS(projectName string, instanceName string) string { if projectName != api.ProjectDefaultName { return fmt.Sprintf("%s.%s", instanceName, projectName) } return instanceName } // InstanceParts takes a project prefixed Instance name string and returns the project and instance name. // If a non-project prefixed Instance name is supplied, then the project is returned as "default" and the instance // name is returned unmodified in the 2nd return value. This is suitable for passing back into Instance(). // Note: This should only be used with Instance names (because they cannot contain the project separator) and this // function relies on this rule as project names can contain the project separator. func InstanceParts(projectInstanceName string) (string, string) { i := strings.LastIndex(projectInstanceName, separator) if i < 0 { // This string is not project prefixed or is part of default project. return api.ProjectDefaultName, projectInstanceName } // As project names can container separator, we effectively split once from the right hand side as // Instance names are not allowed to container the separator value. return projectInstanceName[0:i], projectInstanceName[i+1:] } // StorageVolume adds the "_prefix" to the storage volume name. Even if the project name is "default". func StorageVolume(projectName string, storageVolumeName string) string { return fmt.Sprintf("%s%s%s", projectName, separator, storageVolumeName) } // StorageVolumeParts takes a project prefixed storage volume name and returns the project and storage volume // name as separate variables. func StorageVolumeParts(projectStorageVolumeName string) (string, string) { parts := strings.SplitN(projectStorageVolumeName, "_", 2) // If the given name doesn't contain any project, only return the volume name. if len(parts) == 1 { return "", projectStorageVolumeName } return parts[0], parts[1] } // StorageVolumeProject returns the project name to use to for the volume based on the requested project. // For image volume types the default project is always returned. // For custom volume type, if the project specified has the "features.storage.volumes" flag enabled then the // project name is returned, otherwise the default project name is returned. // For all other volume types the supplied project name is returned. func StorageVolumeProject(c *db.Cluster, projectName string, volumeType int) (string, error) { // Image volumes are effectively a cache and so are always linked to default project. // Optimisation to avoid loading project record. if volumeType == db.StoragePoolVolumeTypeImage { return api.ProjectDefaultName, nil } var project *api.Project err := c.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := cluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return err } project, err = dbProject.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return "", fmt.Errorf("Failed to load project %q: %w", projectName, err) } return StorageVolumeProjectFromRecord(project, volumeType), nil } // StorageVolumeProjectFromRecord returns the project name to use to for the volume based on the supplied project. // For image volume types the default project is always returned. // For custom volume type, if the project supplied has the "features.storage.volumes" flag enabled then the // project name is returned, otherwise the default project name is returned. // For all other volume types the supplied project's name is returned. func StorageVolumeProjectFromRecord(p *api.Project, volumeType int) string { // Image volumes are effectively a cache and so are always linked to default project. if volumeType == db.StoragePoolVolumeTypeImage { return api.ProjectDefaultName } // Non-custom volumes always use the project specified. if volumeType != db.StoragePoolVolumeTypeCustom { return p.Name } // Custom volumes only use the project specified if the project has the features.storage.volumes feature // enabled, otherwise the legacy behaviour of using the default project for custom volumes is used. if util.IsTrue(p.Config["features.storage.volumes"]) { return p.Name } return api.ProjectDefaultName } // StorageBucket adds the "_prefix" to the storage bucket name. Even if the project name is "default". func StorageBucket(projectName string, storageBucketName string) string { return fmt.Sprintf("%s%s%s", projectName, separator, storageBucketName) } // StorageBucketProject returns the effective project name to use to for the bucket based on the requested project. // If the project specified has the "features.storage.buckets" flag enabled then the project name is returned, // otherwise the default project name is returned. func StorageBucketProject(ctx context.Context, c *db.Cluster, projectName string) (string, error) { var p *api.Project err := c.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := cluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return err } p, err = dbProject.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return "", fmt.Errorf("Failed to load project %q: %w", projectName, err) } return StorageBucketProjectFromRecord(p), nil } // StorageBucketProjectFromRecord returns the project name to use to for the bucket based on the supplied project. // If the project supplied has the "features.storage.buckets" flag enabled then the project name is returned, // otherwise the default project name is returned. func StorageBucketProjectFromRecord(p *api.Project) string { // Buckets only use the project specified if the project has the features.storage.buckets feature // enabled, otherwise the default project is used. if util.IsTrue(p.Config["features.storage.buckets"]) { return p.Name } return api.ProjectDefaultName } // StoragePoolAllowed returns whether access is allowed to a particular storage pool based on project limits and restrictions. // A pool is inaccessible if it has an explicit size limit of 0 (limits.disk.pool.POOLNAME=0) or if restricted.storage-pools.access // is set and the pool is not in the allowlist (treated as equivalent to a size limit of 0). func StoragePoolAllowed(reqProjectConfig map[string]string, poolName string) bool { // A pool with an explicit size limit of 0 is never accessible. if reqProjectConfig[projectLimitDiskPool+poolName] == "0" { return false } // If the project isn't restricted, then access to the pool is allowed. if util.IsFalseOrEmpty(reqProjectConfig["restricted"]) { return true } // If restricted.storage-pools.access is not set then allow access to all pools. if reqProjectConfig["restricted.storage-pools.access"] == "" { return true } // Check if requested pool is in list of allowed pools. allowedPools := util.SplitNTrimSpace(reqProjectConfig["restricted.storage-pools.access"], ",", -1, false) return slices.Contains(allowedPools, poolName) } // NetworkProject returns the effective project name to use for the network based on the requested project. // If the requested project has the "features.networks" flag enabled then the requested project's name is returned, // otherwise the default project name is returned. // The second return value is always the requested project's info. func NetworkProject(c *db.Cluster, projectName string) (string, *api.Project, error) { var p *api.Project err := c.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := cluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return err } p, err = dbProject.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return "", nil, fmt.Errorf("Failed to load project %q: %w", projectName, err) } effectiveProjectName := NetworkProjectFromRecord(p) return effectiveProjectName, p, nil } // NetworkProjectFromRecord returns the project name to use for the network based on the supplied project. // If the project supplied has the "features.networks" flag enabled then the project name is returned, // otherwise the default project name is returned. func NetworkProjectFromRecord(p *api.Project) string { // Networks only use the project specified if the project has the features.networks feature enabled, // otherwise the legacy behaviour of using the default project for networks is used. if util.IsTrue(p.Config["features.networks"]) { return p.Name } return api.ProjectDefaultName } // NetworkAllowed returns whether access is allowed to a particular network based on projectConfig. func NetworkAllowed(reqProjectConfig map[string]string, networkName string, isManaged bool) bool { // If project is not restricted, then access to network is allowed. if util.IsFalseOrEmpty(reqProjectConfig["restricted"]) { return true } // If project has no access to NIC devices then also block access to all networks. if reqProjectConfig["restricted.devices.nic"] == "block" { return false } // Don't allow access to unmanaged networks if only managed network access is allowed. if slices.Contains([]string{"managed", ""}, reqProjectConfig["restricted.devices.nic"]) && !isManaged { return false } // If restricted.networks.access is not set then allow access to all networks. if reqProjectConfig["restricted.networks.access"] == "" { return true } // Check if reqquested network is in list of allowed networks. allowedRestrictedNetworks := util.SplitNTrimSpace(reqProjectConfig["restricted.networks.access"], ",", -1, false) return slices.Contains(allowedRestrictedNetworks, networkName) } // NetworkIntegrationAllowed returns whether access is allowed for a particular network integration based on projectConfig. func NetworkIntegrationAllowed(reqProjectConfig map[string]string, integrationName string) bool { // If project is not restricted, then access to network is allowed. if util.IsFalseOrEmpty(reqProjectConfig["restricted"]) { return true } // If restricted.networks.integrations is not set then allow access to all networks. if reqProjectConfig["restricted.networks.integrations"] == "" { return true } // Check if reqquested integration is in list of allowed network integrations. allowedRestrictedIntegrations := util.SplitNTrimSpace(reqProjectConfig["restricted.networks.integrations"], ",", -1, false) return slices.Contains(allowedRestrictedIntegrations, integrationName) } // ImageProjectFromRecord returns the project name to use for the image based on the supplied project. // If the project supplied has the "features.images" flag enabled then the project name is returned, // otherwise the default project name is returned. func ImageProjectFromRecord(p *api.Project) string { // Images only use the project specified if the project has the features.images feature enabled, // otherwise the default project for profiles is used. if util.IsTrue(p.Config["features.images"]) { return p.Name } return api.ProjectDefaultName } // ProfileProject returns the effective project to use for the profile based on the requested project. // If the requested project has the "features.profiles" flag enabled then the requested project's info is returned, // otherwise the default project's info is returned. func ProfileProject(c *db.Cluster, projectName string) (*api.Project, error) { var p *api.Project err := c.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := cluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return fmt.Errorf("Failed loading project %q: %w", projectName, err) } p, err = dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading config for project %q: %w", projectName, err) } effectiveProjectName := ProfileProjectFromRecord(p) if effectiveProjectName == api.ProjectDefaultName { dbProject, err = cluster.GetProject(ctx, tx.Tx(), effectiveProjectName) if err != nil { return fmt.Errorf("Failed loading project %q: %w", effectiveProjectName, err) } } p, err = dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading config for project %q: %w", dbProject.Name, err) } return nil }) if err != nil { return nil, err } return p, nil } // ProfileProjectFromRecord returns the project name to use for the profile based on the supplied project. // If the project supplied has the "features.profiles" flag enabled then the project name is returned, // otherwise the default project name is returned. func ProfileProjectFromRecord(p *api.Project) string { // Profiles only use the project specified if the project has the features.profiles feature enabled, // otherwise the default project for profiles is used. if util.IsTrue(p.Config["features.profiles"]) { return p.Name } return api.ProjectDefaultName } // NetworkZoneProject returns the effective project name to use for network zone based on the requested project. // If the requested project has the "features.networks.zones" flag enabled then the requested project's name is // returned, otherwise the default project name is returned. // The second return value is always the requested project's info. func NetworkZoneProject(c *db.Cluster, projectName string) (string, *api.Project, error) { var p *api.Project err := c.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := cluster.GetProject(ctx, tx.Tx(), projectName) if err != nil { return err } p, err = dbProject.ToAPI(ctx, tx.Tx()) return err }) if err != nil { return "", nil, fmt.Errorf("Failed to load project %q: %w", projectName, err) } effectiveProjectName := NetworkZoneProjectFromRecord(p) return effectiveProjectName, p, nil } // NetworkZoneProjectFromRecord returns the project name to use for the network zone based on the supplied project. // If the project supplied has the "features.networks.zones" flag enabled then the project name is returned, // otherwise the default project name is returned. func NetworkZoneProjectFromRecord(p *api.Project) string { // Network zones only use the project specified if the project has the features.networks.zones feature // enabled, otherwise the legacy behaviour of using the default project for network zones is used. if util.IsTrue(p.Config["features.networks.zones"]) { return p.Name } return api.ProjectDefaultName } incus-7.0.0/internal/server/project/project_test.go000066400000000000000000000022061517523235500224730ustar00rootroot00000000000000package project_test import ( "fmt" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/shared/api" ) func ExampleInstance() { prefixed := project.Instance(api.ProjectDefaultName, "test") fmt.Println(prefixed) prefixed = project.Instance("project_name", "test1") fmt.Println(prefixed) // Output: test // project_name_test1 } func ExampleInstanceParts() { projectName, name := project.InstanceParts("unprefixed") fmt.Println(projectName, name) projectName, name = project.InstanceParts(project.Instance(api.ProjectDefaultName, "test")) fmt.Println(projectName, name) projectName, name = project.InstanceParts("project_name_test") fmt.Println(projectName, name) projectName, name = project.InstanceParts(project.Instance("proj", "test1")) fmt.Println(projectName, name) // Output: default unprefixed // default test // project_name test // proj test1 } func ExampleStorageVolume() { prefixed := project.StorageVolume(api.ProjectDefaultName, "test") fmt.Println(prefixed) prefixed = project.StorageVolume("project_name", "test1") fmt.Println(prefixed) // Output: default_test // project_name_test1 } incus-7.0.0/internal/server/project/state.go000066400000000000000000000052221517523235500211070ustar00rootroot00000000000000package project import ( "context" "fmt" "strconv" "strings" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/shared/api" ) // GetCurrentAllocations returns the current resource utilization for a given project. func GetCurrentAllocations(ctx context.Context, tx *db.ClusterTx, projectName string) (map[string]api.ProjectStateResource, error) { result := map[string]api.ProjectStateResource{} // Get the project. info, err := fetchProject(tx, projectName, false) if err != nil { return nil, err } if info == nil { return nil, fmt.Errorf("Project %q returned empty info struct", projectName) } info.Instances, err = expandInstancesConfigAndDevices(info.Instances, info.Profiles) if err != nil { return nil, err } // Get per-pool limits. poolLimits := []string{} for k := range info.Project.Config { if strings.HasPrefix(k, projectLimitDiskPool) { poolLimits = append(poolLimits, k) } } allAggregateLimits := append(allAggregateLimits, poolLimits...) // Get the instance aggregated values. raw, err := getAggregateLimits(info, allAggregateLimits) if err != nil { return nil, err } result["cpu"] = raw["limits.cpu"] result["disk"] = raw["limits.disk"] result["memory"] = raw["limits.memory"] result["networks"] = raw["limits.networks"] result["processes"] = raw["limits.processes"] // Add the pool-specific disk limits. for k, v := range raw { if strings.HasPrefix(k, projectLimitDiskPool) && v.Limit > 0 { result[fmt.Sprintf("disk.%s", strings.SplitN(k, ".", 4)[3])] = v } } // Get the instance count values. count, limit, err := getTotalInstanceCountLimit(info) if err != nil { return nil, err } result["instances"] = api.ProjectStateResource{ Limit: int64(limit), Usage: int64(count), } count, limit, err = getInstanceCountLimit(info, instancetype.Container) if err != nil { return nil, err } result["containers"] = api.ProjectStateResource{ Limit: int64(limit), Usage: int64(count), } count, limit, err = getInstanceCountLimit(info, instancetype.VM) if err != nil { return nil, err } result["virtual-machines"] = api.ProjectStateResource{ Limit: int64(limit), Usage: int64(count), } // Get the network limit and usage. overallValue, ok := info.Project.Config["limits.networks"] limit = -1 if ok { limit, err = strconv.Atoi(overallValue) if err != nil { return nil, err } } networks, err := tx.GetCreatedNetworks(ctx) if err != nil { return nil, err } result["networks"] = api.ProjectStateResource{ Limit: int64(limit), Usage: int64(len(networks[projectName])), } return result, nil } incus-7.0.0/internal/server/refcount/000077500000000000000000000000001517523235500176165ustar00rootroot00000000000000incus-7.0.0/internal/server/refcount/refcount.go000066400000000000000000000024331517523235500217740ustar00rootroot00000000000000package refcount import ( "fmt" "sync" ) var refCounters = map[string]uint{} // refCounterMutex is used to access refCounters safely. var refCounterMutex sync.Mutex // Get returns the current value of the refCounter. func Get(refCounter string) uint { refCounterMutex.Lock() defer refCounterMutex.Unlock() return refCounters[refCounter] } // Increment increases a refCounter by the value. If the ref counter doesn't exist, a new one is created. // The counter's new value is returned. func Increment(refCounter string, value uint) uint { refCounterMutex.Lock() defer refCounterMutex.Unlock() v := refCounters[refCounter] oldValue := v v = v + value if v < oldValue { panic(fmt.Sprintf("Ref counter %q overflowed from %d to %d", refCounter, oldValue, v)) } refCounters[refCounter] = v return v } // Decrement decreases a refCounter by the value. If the ref counter doesn't exist, a new one is created. // The counter's new value is returned. A counter cannot be decreased below zero. func Decrement(refCounter string, value uint) uint { refCounterMutex.Lock() defer refCounterMutex.Unlock() v := refCounters[refCounter] if v < value { v = 0 } else { v = v - value } if v == 0 { delete(refCounters, refCounter) } else { refCounters[refCounter] = v } return v } incus-7.0.0/internal/server/refcount/refcount_test.go000066400000000000000000000020141517523235500230260ustar00rootroot00000000000000package refcount import ( "fmt" ) func ExampleIncrement() { refCounter1 := "testinc1" fmt.Println(Increment(refCounter1, 1)) fmt.Println(Increment(refCounter1, 1)) fmt.Println(Increment(refCounter1, 2)) refCounter2 := "testinc2" fmt.Println(Increment(refCounter2, 1)) fmt.Println(Increment(refCounter2, 10)) // Test overflow panic. defer func() { r := recover() if r != nil { fmt.Printf("Recovered: %v\n", r) } }() var maxUint uint = ^uint(0) fmt.Println(Increment(refCounter2, maxUint)) // Output: 1 // 2 // 4 // 1 // 11 // Recovered: Ref counter "testinc2" overflowed from 11 to 10 } func ExampleDecrement() { refCounter1 := "testdec1" fmt.Println(Increment(refCounter1, 10)) fmt.Println(Decrement(refCounter1, 1)) fmt.Println(Decrement(refCounter1, 2)) refCounter2 := "testdec2" fmt.Println(Decrement(refCounter2, 1)) fmt.Println(Increment(refCounter2, 1)) fmt.Println(Decrement(refCounter2, 1)) fmt.Println(Decrement(refCounter2, 1)) // Output: 10 // 9 // 7 // 0 // 1 // 0 // 0 } incus-7.0.0/internal/server/request/000077500000000000000000000000001517523235500174615ustar00rootroot00000000000000incus-7.0.0/internal/server/request/const.go000066400000000000000000000025561517523235500211460ustar00rootroot00000000000000package request // CtxKey is the type used for all fields stored in the request context. type CtxKey string // Context keys. const ( // CtxAccess is the access field in request context. CtxAccess CtxKey = "access" // CtxConn is the connection field in the request context. CtxConn CtxKey = "conn" // CtxAddress is the address field in request context. CtxAddress CtxKey = "address" // CtxUsername is the username field in request context. CtxUsername CtxKey = "username" // CtxProtocol is the protocol field in request context. CtxProtocol CtxKey = "protocol" // CtxForwardedAddress is the forwarded address field in request context. CtxForwardedAddress CtxKey = "forwarded_address" // CtxForwardedUsername is the forwarded username field in request context. CtxForwardedUsername CtxKey = "forwarded_username" // CtxForwardedProtocol is the forwarded protocol field in request context. CtxForwardedProtocol CtxKey = "forwarded_protocol" ) // Headers. const ( // HeaderForwardedAddress is the forwarded address field in request header. HeaderForwardedAddress = "X-Incus-forwarded-address" // HeaderForwardedUsername is the forwarded username field in request header. HeaderForwardedUsername = "X-Incus-forwarded-username" // HeaderForwardedProtocol is the forwarded protocol field in request header. HeaderForwardedProtocol = "X-Incus-forwarded-protocol" ) incus-7.0.0/internal/server/request/parse.go000066400000000000000000000016231517523235500211240ustar00rootroot00000000000000package request import ( "net/http" "net/url" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) // ProjectParam returns the project query parameter from the given request or "default" if parameter is not set. func ProjectParam(request *http.Request) string { projectParam := QueryParam(request, "project") if projectParam == "" { projectParam = api.ProjectDefaultName } return projectParam } // QueryParam extracts the given query parameter directly from the URL, never from an // encoded body. func QueryParam(request *http.Request, key string) string { var values url.Values var err error if request.URL != nil { values, err = url.ParseQuery(request.URL.RawQuery) if err != nil { logger.Warnf("Failed to parse query string %q: %v", request.URL.RawQuery, err) return "" } } if values == nil { values = make(url.Values) } return values.Get(key) } incus-7.0.0/internal/server/request/request.go000066400000000000000000000024201517523235500214760ustar00rootroot00000000000000package request import ( "context" "net" "net/http" "github.com/lxc/incus/v7/shared/api" ) // CreateRequestor extracts the lifecycle event requestor data from an http.Request context. func CreateRequestor(r *http.Request) *api.EventLifecycleRequestor { ctx := r.Context() requestor := &api.EventLifecycleRequestor{} // Normal requestor. val, ok := ctx.Value(CtxUsername).(string) if ok { requestor.Username = val } val, ok = ctx.Value(CtxProtocol).(string) if ok { requestor.Protocol = val } requestor.Address = r.RemoteAddr // Forwarded requestor override. val, ok = ctx.Value(CtxForwardedUsername).(string) if ok { requestor.Username = val } val, ok = ctx.Value(CtxForwardedProtocol).(string) if ok { requestor.Protocol = val } val, ok = ctx.Value(CtxForwardedAddress).(string) if ok { requestor.Address = val } // Strip port from address. host, _, err := net.SplitHostPort(requestor.Address) if err == nil { requestor.Address = host } return requestor } // SaveConnectionInContext can be set as the ConnContext field of a http.Server to set the connection // in the request context for later use. func SaveConnectionInContext(ctx context.Context, connection net.Conn) context.Context { return context.WithValue(ctx, CtxConn, connection) } incus-7.0.0/internal/server/response/000077500000000000000000000000001517523235500176275ustar00rootroot00000000000000incus-7.0.0/internal/server/response/response.go000066400000000000000000000437561517523235500220330ustar00rootroot00000000000000package response import ( "bytes" "compress/gzip" "context" "encoding/json" "fmt" "io" "mime/multipart" "net" "net/http" "os" "sync" "time" incus "github.com/lxc/incus/v7/client" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/tcp" "github.com/lxc/incus/v7/shared/util" ) var debug bool // Init sets the debug variable to the provided value. func Init(d bool) { debug = d } // Response represents an API response. type Response interface { Render(w http.ResponseWriter) error String() string Code() int } // DevIncus response. type devIncusResponse struct { content any code int contentType string } func (r *devIncusResponse) Render(w http.ResponseWriter) error { var err error if r.code != http.StatusOK { http.Error(w, fmt.Sprintf("%s", r.content), r.code) } else if r.contentType == "json" { w.Header().Set("Content-Type", "application/json") var debugLogger logger.Logger if debug { debugLogger = logger.Logger(logger.Log) } err = localUtil.WriteJSON(w, r.content, debugLogger) } else if r.contentType != "websocket" { w.Header().Set("Content-Type", "application/octet-stream") _, err = fmt.Fprint(w, r.content.(string)) } if err != nil { return err } return nil } func (r *devIncusResponse) String() string { if r.code == http.StatusOK { return "success" } return "failure" } // Code returns the HTTP code. func (r *devIncusResponse) Code() int { return r.code } // DevIncusErrorResponse returns an error response. If rawResponse is true, a api.ResponseRaw will be sent instead of a minimal devIncusResponse. func DevIncusErrorResponse(err error, rawResponse bool) Response { if rawResponse { return SmartError(err) } code, ok := api.StatusErrorMatch(err) if ok { return &devIncusResponse{content: err.Error(), code: code, contentType: "raw"} } return &devIncusResponse{content: err.Error(), code: http.StatusInternalServerError, contentType: "raw"} } // DevIncusResponse represents a devIncusResponse. If rawResponse is true, a api.ResponseRaw will be sent instead of a minimal devIncusResponse. func DevIncusResponse(code int, content any, contentType string, rawResponse bool) Response { if rawResponse { return SyncResponse(true, content) } return &devIncusResponse{content: content, code: code, contentType: contentType} } // Sync response. type syncResponse struct { success bool etag any metadata any location string code int headers map[string]string plaintext bool compress bool } // EmptySyncResponse represents an empty syncResponse. var EmptySyncResponse = &syncResponse{success: true, metadata: make(map[string]any)} // SyncResponse returns a new syncResponse with the success and metadata fields // set to the provided values. func SyncResponse(success bool, metadata any) Response { return &syncResponse{success: success, metadata: metadata} } // SyncResponseETag returns a new syncResponse with an etag. func SyncResponseETag(success bool, metadata any, etag any) Response { return &syncResponse{success: success, metadata: metadata, etag: etag} } // SyncResponseLocation returns a new syncResponse with a location. func SyncResponseLocation(success bool, metadata any, location string) Response { return &syncResponse{success: success, metadata: metadata, location: location} } // SyncResponseRedirect returns a new syncResponse with a location, indicating // a permanent redirect. func SyncResponseRedirect(address string) Response { return &syncResponse{success: true, location: address, code: http.StatusPermanentRedirect} } // SyncResponseHeaders returns a new syncResponse with headers. func SyncResponseHeaders(success bool, metadata any, headers map[string]string) Response { return &syncResponse{success: success, metadata: metadata, headers: headers} } // SyncResponsePlain return a new syncResponse with plaintext. func SyncResponsePlain(success bool, compress bool, metadata string) Response { return &syncResponse{success: success, metadata: metadata, plaintext: true, compress: compress} } func (r *syncResponse) Render(w http.ResponseWriter) error { // Set an appropriate ETag header if r.etag != nil { etag, err := localUtil.EtagHash(r.etag) if err == nil { w.Header().Set("ETag", fmt.Sprintf("\"%s\"", etag)) } } if r.headers != nil { for h, v := range r.headers { w.Header().Set(h, v) } } if r.location != "" { w.Header().Set("Location", r.location) if r.code == 0 { r.code = 201 } } // Handle plain text headers. if r.plaintext { w.Header().Set("Content-Type", "text/plain") } // Handle compression. if r.compress { w.Header().Set("Content-Encoding", "gzip") } // Write header and status code. if r.code == 0 { r.code = http.StatusOK } if w.Header().Get("Connection") != "keep-alive" { w.WriteHeader(r.code) } // Prepare the JSON response status := api.Success if !r.success { status = api.Failure // If the metadata is an error, consider the response a SmartError // to propagate the data and preserve the status code. err, ok := r.metadata.(error) if ok { return SmartError(err).Render(w) } } // Handle plain text responses. if r.plaintext { if r.metadata != nil { if r.compress { comp := gzip.NewWriter(w) defer comp.Close() _, err := comp.Write([]byte(r.metadata.(string))) if err != nil { return err } } else { _, err := w.Write([]byte(r.metadata.(string))) if err != nil { return err } } } return nil } // Handle JSON responses. resp := api.ResponseRaw{ Type: api.SyncResponse, Status: status.String(), StatusCode: int(status), Metadata: r.metadata, } var debugLogger logger.Logger if debug { debugLogger = logger.AddContext(logger.Ctx{"http_code": r.code}) } return localUtil.WriteJSON(w, resp, debugLogger) } func (r *syncResponse) String() string { if r.success { return "success" } return "failure" } // Code returns the HTTP code. func (r *syncResponse) Code() int { return r.code } // Error response. type errorResponse struct { code int // Code to return in both the HTTP header and Code field of the response body. msg string // Message to return in the Error field of the response body. } // ErrorResponse returns an error response with the given code and msg. func ErrorResponse(code int, msg string) Response { return &errorResponse{code, msg} } // BadRequest returns a bad request response (400) with the given error. func BadRequest(err error) Response { return &errorResponse{http.StatusBadRequest, err.Error()} } // Conflict returns a conflict response (409) with the given error. func Conflict(err error) Response { message := "already exists" if err != nil { message = err.Error() } return &errorResponse{http.StatusConflict, message} } // Forbidden returns a forbidden response (403) with the given error. func Forbidden(err error) Response { message := "not authorized" if err != nil { message = err.Error() } return &errorResponse{http.StatusForbidden, message} } // InternalError returns an internal error response (500) with the given error. func InternalError(err error) Response { return &errorResponse{http.StatusInternalServerError, err.Error()} } // NotFound returns a not found response (404) with the given error. func NotFound(err error) Response { message := "not found" if err != nil { message = err.Error() } return &errorResponse{http.StatusNotFound, message} } // NotImplemented returns a not implemented response (501) with the given error. func NotImplemented(err error) Response { message := "not implemented" if err != nil { message = err.Error() } return &errorResponse{http.StatusNotImplemented, message} } // PreconditionFailed returns a precondition failed response (412) with the // given error. func PreconditionFailed(err error) Response { return &errorResponse{http.StatusPreconditionFailed, err.Error()} } // Unavailable return an unavailable response (503) with the given error. func Unavailable(err error) Response { message := "unavailable" if err != nil { message = err.Error() } return &errorResponse{http.StatusServiceUnavailable, message} } func (r *errorResponse) String() string { return r.msg } // Code returns the HTTP code. func (r *errorResponse) Code() int { return r.code } func (r *errorResponse) Render(w http.ResponseWriter) error { var output io.Writer buf := &bytes.Buffer{} output = buf var captured *bytes.Buffer if debug { captured = &bytes.Buffer{} output = io.MultiWriter(buf, captured) } resp := api.ResponseRaw{ Type: api.ErrorResponse, Error: r.msg, Code: r.code, // Set the error code in the Code field of the response body. } err := json.NewEncoder(output).Encode(resp) if err != nil { return err } if debug { debugLogger := logger.AddContext(logger.Ctx{"http_code": r.code}) localUtil.DebugJSON("Error Response", captured, debugLogger) } w.Header().Set("Content-Type", "application/json") w.Header().Set("X-Content-Type-Options", "nosniff") if w.Header().Get("Connection") != "keep-alive" { w.WriteHeader(r.code) // Set the error code in the HTTP header response. } _, err = fmt.Fprint(w, buf.String()) return err } // FileResponseEntry represents a file response entry. type FileResponseEntry struct { // Required. Identifier string Filename string // Read from a filesystem path. Path string // Read from a file. File io.ReadSeeker FileSize int64 FileModified time.Time // Optional. Cleanup func() } type fileResponse struct { req *http.Request files []FileResponseEntry headers map[string]string } // FileResponse returns a new file response. func FileResponse(r *http.Request, files []FileResponseEntry, headers map[string]string) Response { return &fileResponse{r, files, headers} } func (r *fileResponse) Render(w http.ResponseWriter) error { if r.headers != nil { for k, v := range r.headers { w.Header().Set(k, v) } } // No file, well, it's easy then if len(r.files) == 0 { return nil } // For a single file, return it inline if len(r.files) == 1 { var rs io.ReadSeeker var mt time.Time var sz int64 if r.files[0].Cleanup != nil { defer r.files[0].Cleanup() } if r.files[0].File != nil { rs = r.files[0].File mt = r.files[0].FileModified sz = r.files[0].FileSize } else { f, err := os.Open(r.files[0].Path) if err != nil { return err } defer func() { _ = f.Close() }() fi, err := f.Stat() if err != nil { return err } mt = fi.ModTime() sz = fi.Size() rs = f } // Only set Content-Type header if it is still set to the default or not yet set at all. if w.Header().Get("Content-Type") == "application/json" || w.Header().Get("Content-Type") == "" { w.Header().Set("Content-Type", "application/octet-stream") } w.Header().Set("Content-Length", fmt.Sprintf("%d", sz)) w.Header().Set("Content-Disposition", fmt.Sprintf("inline;filename=%s", r.files[0].Filename)) http.ServeContent(w, r.req, r.files[0].Filename, mt, rs) return nil } // Now the complex multipart answer. mw := multipart.NewWriter(w) defer func() { _ = mw.Close() }() w.Header().Set("Content-Type", mw.FormDataContentType()) w.Header().Set("Transfer-Encoding", "chunked") for _, entry := range r.files { var rd io.Reader if entry.File != nil { rd = entry.File } else { fd, err := os.Open(entry.Path) if err != nil { return err } defer func() { _ = fd.Close() }() rd = fd } fw, err := mw.CreateFormFile(entry.Identifier, entry.Filename) if err != nil { return err } _, err = util.SafeCopy(fw, rd) if err != nil { return err } if entry.Cleanup != nil { entry.Cleanup() } } return mw.Close() } func (r *fileResponse) String() string { return fmt.Sprintf("%d files", len(r.files)) } // Code returns the HTTP code. func (r *fileResponse) Code() int { return http.StatusOK } type forwardedResponse struct { client incus.InstanceServer request *http.Request } // ForwardedResponse takes a request directed to a node and forwards it to // another node, writing back the response it gegs. func ForwardedResponse(client incus.InstanceServer, request *http.Request) Response { return &forwardedResponse{ client: client, request: request, } } func (r *forwardedResponse) Render(w http.ResponseWriter) error { info, err := r.client.GetConnectionInfo() if err != nil { return err } url := fmt.Sprintf("%s%s", info.Addresses[0], r.request.URL.RequestURI()) forwarded, err := http.NewRequest(r.request.Method, url, r.request.Body) if err != nil { return err } for key := range r.request.Header { forwarded.Header.Set(key, r.request.Header.Get(key)) } forwarded.Header.Set("X-Incus-forwarded-host", r.request.Host) httpClient, err := r.client.GetHTTPClient() if err != nil { return err } response, err := httpClient.Do(forwarded) if err != nil { return err } for key := range response.Header { w.Header().Set(key, response.Header.Get(key)) } if w.Header().Get("Connection") != "keep-alive" { w.WriteHeader(response.StatusCode) } _, err = util.SafeCopy(w, response.Body) return err } func (r *forwardedResponse) String() string { return fmt.Sprintf("request to %s", r.request.URL) } // Code returns the HTTP code. func (r *forwardedResponse) Code() int { return http.StatusOK } type manualResponse struct { hook func(w http.ResponseWriter) error } // ManualResponse creates a new manual response responder. func ManualResponse(hook func(w http.ResponseWriter) error) Response { return &manualResponse{hook: hook} } func (r *manualResponse) Render(w http.ResponseWriter) error { return r.hook(w) } func (r *manualResponse) String() string { return "unknown" } // Code returns the HTTP code. func (r *manualResponse) Code() int { return http.StatusNotImplemented } // Unauthorized return an unauthorized response (401) with the given error. func Unauthorized(err error) Response { message := "unauthorized" if err != nil { message = err.Error() } return &errorResponse{http.StatusUnauthorized, message} } // UpgradeResponse upgrades the connection and connects to the backend server. func UpgradeResponse(r *http.Request, conn net.Conn, protocol string, cleanup func()) Response { return &upgradeResponse{req: r, conn: conn, protocol: protocol, cleanup: cleanup} } type upgradeResponse struct { req *http.Request conn net.Conn protocol string cleanup func() } // String returns the response type name. func (r *upgradeResponse) String() string { return r.protocol + " handler" } // Code returns the HTTP code. func (r *upgradeResponse) Code() int { return http.StatusOK } // Render handles the HTTP connection. func (r *upgradeResponse) Render(w http.ResponseWriter) error { if r.cleanup != nil { defer r.cleanup() } defer func() { _ = r.conn.Close() }() hijacker, ok := w.(http.Hijacker) if !ok { return api.StatusErrorf(http.StatusInternalServerError, "Webserver doesn't support hijacking") } remoteConn, _, err := hijacker.Hijack() if err != nil { return api.StatusErrorf(http.StatusInternalServerError, "Failed to hijack connection: %v", err) } defer func() { _ = remoteConn.Close() }() remoteTCP, _ := tcp.ExtractConn(remoteConn) if remoteTCP != nil { // Apply TCP timeouts if remote connection is TCP (rather than Unix). err = tcp.SetTimeouts(remoteTCP, 0) if err != nil { return api.StatusErrorf(http.StatusInternalServerError, "Failed setting TCP timeouts on remote connection: %v", err) } } err = Upgrade(remoteConn, r.protocol) if err != nil { return api.StatusErrorf(http.StatusInternalServerError, "%s", err.Error()) } if r.protocol == "nbd" { // A bit of a hack to handle the fact that NBD is a protocol where // the server speaks first, immediately sending a connection header when // processing a new connection. This immediate bit of raw data can hit the // client prior to it having completed the HTTP part of the upgrade, // causing that header message to get dropped. time.Sleep(250 * time.Millisecond) } ctx, cancel := context.WithCancel(r.req.Context()) l := logger.AddContext(logger.Ctx{ "local": remoteConn.LocalAddr(), "remote": remoteConn.RemoteAddr(), }) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() _, err := util.SafeCopy(remoteConn, r.conn) if err != nil { if ctx.Err() == nil { l.Warn("Failed copying data from local to remote connection", logger.Ctx{"err": err}) } } cancel() // Cancel context first so when remoteConn is closed it doesn't cause a warning. _ = remoteConn.Close() // Trigger the cancellation of the util.SafeCopy reading from remoteConn. }() _, err = util.SafeCopy(r.conn, remoteConn) if err != nil { if ctx.Err() == nil { l.Warn("Failed copying data from remote to local connection", logger.Ctx{"err": err}) } } cancel() // Cancel context first so when conn is closed it doesn't cause a warning. err = r.conn.Close() // Trigger the cancellation of the util.SafeCopy reading from conn. if err != nil { return fmt.Errorf("Failed closing connection to remote server: %w", err) } wg.Wait() // Wait for copy go routine to finish. return nil } type pipeResponse struct { req *http.Request reader *io.PipeReader } // PipeResponse returns a new pipe response. func PipeResponse(r *http.Request, reader *io.PipeReader) Response { return &pipeResponse{r, reader} } // Code returns the HTTP code. func (r *pipeResponse) Code() int { return http.StatusOK } // Render writes the response. func (r *pipeResponse) Render(w http.ResponseWriter) error { defer func() { _ = r.reader.Close() }() w.Header().Set("Content-Type", "application/octet-stream") w.WriteHeader(r.Code()) // We really want to flush the headers now, so that we do not hit a timeout on the receiver side // in the case of slow optimized storage export. flusher, ok := w.(http.Flusher) if ok { flusher.Flush() } _, err := util.SafeCopy(w, r.reader) return err } // String returns a quick description of the response. func (r *pipeResponse) String() string { return "pipe handler" } incus-7.0.0/internal/server/response/smart.go000066400000000000000000000031751517523235500213120ustar00rootroot00000000000000package response import ( "database/sql" "errors" "net/http" "os" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/shared/api" ) var httpResponseErrors = map[int][]error{ http.StatusNotFound: {os.ErrNotExist, sql.ErrNoRows}, http.StatusForbidden: {os.ErrPermission}, http.StatusConflict: {db.ErrAlreadyDefined}, } // SmartError returns the right error message based on err. // It uses the stdlib errors package to unwrap the error and find the cause. func SmartError(err error) Response { if err == nil { return EmptySyncResponse } statusCode, found := api.StatusErrorMatch(err) if found { return &errorResponse{statusCode, err.Error()} } for httpStatusCode, checkErrs := range httpResponseErrors { for _, checkErr := range checkErrs { if errors.Is(err, checkErr) { // This is intended to not be `errors.Is`, so we check if it is a wrapped error. if err != checkErr { // If the error has been wrapped return the top-level error message. return &errorResponse{httpStatusCode, err.Error()} } // If the error hasn't been wrapped, replace the error message with the generic // HTTP status text. return &errorResponse{httpStatusCode, http.StatusText(httpStatusCode)} } } } return &errorResponse{http.StatusInternalServerError, err.Error()} } // IsNotFoundError returns true if the error is considered a Not Found error. func IsNotFoundError(err error) bool { if api.StatusErrorCheck(err, http.StatusNotFound) { return true } for _, checkErr := range httpResponseErrors[http.StatusNotFound] { if errors.Is(err, checkErr) { return true } } return false } incus-7.0.0/internal/server/response/smart_linux.go000066400000000000000000000007741517523235500225330ustar00rootroot00000000000000//go:build linux && cgo && !agent package response import ( "net/http" "github.com/cowsql/go-cowsql/driver" "github.com/mattn/go-sqlite3" ) // Populates error slices with Linux specific error types for use with SmartError(). func init() { httpResponseErrors[http.StatusConflict] = append(httpResponseErrors[http.StatusConflict], sqlite3.ErrConstraintUnique) httpResponseErrors[http.StatusServiceUnavailable] = append(httpResponseErrors[http.StatusServiceUnavailable], driver.ErrNoAvailableLeader) } incus-7.0.0/internal/server/response/swagger.go000066400000000000000000000046361517523235500216260ustar00rootroot00000000000000// Package response contains helpers for rendering HTTP responses. // //nolint:unused package response import ( "github.com/lxc/incus/v7/shared/api" ) // Operation // // swagger:response Operation type swaggerOperation struct { // Empty sync response // in: body Body struct { // Example: async Type string `json:"type"` // Example: Operation created Status string `json:"status"` // Example: 100 StatusCode int `json:"status_code"` // Example: /1.0/operations/66e83638-9dd7-4a26-aef2-5462814869a1 Operation string `json:"operation"` Metadata api.Operation `json:"metadata"` } } // Empty sync response // // swagger:response EmptySyncResponse type swaggerEmptySyncResponse struct { // Empty sync response // in: body Body struct { // Example: sync Type string `json:"type"` // Example: Success Status string `json:"status"` // Example: 200 StatusCode int `json:"status_code"` } } // Bad Request // // swagger:response BadRequest type swaggerBadRequest struct { // Bad Request // in: body Body struct { // Example: error Type string `json:"type"` // Example: bad request Error string `json:"error"` // Example: 400 ErrorCode int `json:"error_code"` } } // Forbidden // // swagger:response Forbidden type swaggerForbidden struct { // Bad Request // in: body Body struct { // Example: error Type string `json:"type"` // Example: not authorized Error string `json:"error"` // Example: 403 ErrorCode int `json:"error_code"` } } // Precondition Failed // // swagger:response PreconditionFailed type swaggerPreconditionFailed struct { // Internal server Error // in: body Body struct { // Example: error Type string `json:"type"` // Example: precondition failed Error string `json:"error"` // Example: 412 ErrorCode int `json:"error_code"` } } // Internal Server Error // // swagger:response InternalServerError type swaggerInternalServerError struct { // Internal server Error // in: body Body struct { // Example: error Type string `json:"type"` // Example: internal server error Error string `json:"error"` // Example: 500 ErrorCode int `json:"error_code"` } } // Not found // // swagger:response NotFound type swaggerNotFound struct { // Not found // in: body Body struct { // Example: error Type string `json:"type"` // Example: not found Error string `json:"error"` // Example: 404 ErrorCode int `json:"error_code"` } } incus-7.0.0/internal/server/response/upgrade.go000066400000000000000000000016141517523235500216070ustar00rootroot00000000000000package response import ( "errors" "fmt" "net" "strings" "time" ) // Upgrade takes a hijacked HTTP connection and sends the HTTP 101 Switching Protocols headers for protocolName. func Upgrade(hijackedConn net.Conn, protocolName string) error { // Write the status line and upgrade header by hand since w.WriteHeader() would fail after Hijack(). sb := strings.Builder{} sb.WriteString("HTTP/1.1 101 Switching Protocols\r\n") sb.WriteString(fmt.Sprintf("Upgrade: %s\r\n", protocolName)) sb.WriteString("Connection: Upgrade\r\n\r\n") _ = hijackedConn.SetWriteDeadline(time.Now().Add(time.Second * 5)) n, err := hijackedConn.Write([]byte(sb.String())) _ = hijackedConn.SetWriteDeadline(time.Time{}) // Cancel deadline. if err != nil { return fmt.Errorf("Failed writing upgrade headers: %w", err) } if n != sb.Len() { return errors.New("Failed writing upgrade headers") } return nil } incus-7.0.0/internal/server/scriptlet/000077500000000000000000000000001517523235500200025ustar00rootroot00000000000000incus-7.0.0/internal/server/scriptlet/auth/000077500000000000000000000000001517523235500207435ustar00rootroot00000000000000incus-7.0.0/internal/server/scriptlet/auth/auth.go000066400000000000000000000112161517523235500222340ustar00rootroot00000000000000package auth import ( "crypto/x509" "errors" "fmt" "go.starlark.net/starlark" "github.com/lxc/incus/v7/internal/server/auth/common" scriptletLoad "github.com/lxc/incus/v7/internal/server/scriptlet/load" "github.com/lxc/incus/v7/internal/server/scriptlet/log" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/scriptlet" ) // AuthorizationRun runs the authorization scriptlet. func AuthorizationRun(l logger.Logger, details *common.RequestDetails, peerCertificates []*x509.Certificate, apiCertificate *api.CertificatePut, object string, entitlement string) (bool, error) { logFunc := log.CreateLogger(l, "Authorization scriptlet") // Remember to match the entries in scriptletLoad.AuthorizationCompile() with this list so Starlark can // perform compile time validation of functions used. env := starlark.StringDict{ "log_info": starlark.NewBuiltin("log_info", logFunc), "log_warn": starlark.NewBuiltin("log_warn", logFunc), "log_error": starlark.NewBuiltin("log_error", logFunc), } prog, thread, err := scriptletLoad.AuthorizationProgram() if err != nil { return false, err } globals, err := prog.Init(thread, env) if err != nil { return false, fmt.Errorf("Failed initializing: %w", err) } globals.Freeze() // Retrieve a global variable from starlark environment. authorizer := globals["authorize"] if authorizer == nil { return false, errors.New("Scriptlet missing authorize function") } extendedDetails := struct { *common.RequestDetails Chain []*x509.Certificate Certificate *api.CertificatePut }{ details, peerCertificates, apiCertificate, } detailsv, err := scriptlet.StarlarkMarshal(extendedDetails) if err != nil { return false, fmt.Errorf("Marshalling details failed: %w", err) } // Call starlark function from Go. v, err := starlark.Call(thread, authorizer, nil, []starlark.Tuple{ { starlark.String("details"), detailsv, }, { starlark.String("object"), starlark.String(object), }, { starlark.String("entitlement"), starlark.String(entitlement), }, }) if err != nil { return false, fmt.Errorf("Failed to run: %w", err) } if v.Type() != "bool" { return false, fmt.Errorf("Failed with unexpected return value: %v", v) } return bool(v.(starlark.Bool)), nil } func getAccess(l logger.Logger, fun string, args []starlark.Tuple) (*api.Access, error) { access := &api.Access{} emptyAccess := &api.Access{} logFunc := log.CreateLogger(l, fmt.Sprintf("Authorization scriptlet (%s)", fun)) // Remember to match the entries in scriptletLoad.AuthorizationCompile() with this list so Starlark can // perform compile time validation of functions used. env := starlark.StringDict{ "log_info": starlark.NewBuiltin("log_info", logFunc), "log_warn": starlark.NewBuiltin("log_warn", logFunc), "log_error": starlark.NewBuiltin("log_error", logFunc), } prog, thread, err := scriptletLoad.AuthorizationProgram() if err != nil { return emptyAccess, err } globals, err := prog.Init(thread, env) if err != nil { return emptyAccess, fmt.Errorf("Failed initializing: %w", err) } globals.Freeze() // Retrieve a global variable from starlark environment. getter := globals[fun] if getter == nil { return emptyAccess, nil } // Call starlark function from Go. v, err := starlark.Call(thread, getter, nil, args) if err != nil { return emptyAccess, fmt.Errorf("Failed to run: %w", err) } value, err := scriptlet.StarlarkUnmarshal(v) if err != nil { return emptyAccess, err } identifiers, ok := value.([]any) if !ok { return emptyAccess, fmt.Errorf("Failed with unexpected return value: %v", v) } for _, id := range identifiers { identifier, ok := id.(string) if !ok { return emptyAccess, fmt.Errorf("Failed with unexpected return value: %v", v) } *access = append(*access, api.AccessEntry{ Identifier: identifier, Role: "unknown", Provider: "scriptlet", }) } return access, nil } // GetInstanceAccessRun runs the optional get_instance_access scriptlet function. func GetInstanceAccessRun(l logger.Logger, projectName string, instanceName string) (*api.Access, error) { return getAccess(l, "get_instance_access", []starlark.Tuple{ { starlark.String("project_name"), starlark.String(projectName), }, { starlark.String("instance_name"), starlark.String(instanceName), }, }) } // GetProjectAccessRun runs the optional get_project_access scriptlet function. func GetProjectAccessRun(l logger.Logger, projectName string) (*api.Access, error) { return getAccess(l, "get_project_access", []starlark.Tuple{ { starlark.String("project_name"), starlark.String(projectName), }, }) } incus-7.0.0/internal/server/scriptlet/instance_placement.go000066400000000000000000000346651517523235500242030ustar00rootroot00000000000000package scriptlet import ( "context" "errors" "fmt" "go.starlark.net/starlark" "github.com/lxc/incus/v7/internal/server/cluster" "github.com/lxc/incus/v7/internal/server/db" dbCluster "github.com/lxc/incus/v7/internal/server/db/cluster" internalInstance "github.com/lxc/incus/v7/internal/server/instance" scriptletLoad "github.com/lxc/incus/v7/internal/server/scriptlet/load" "github.com/lxc/incus/v7/internal/server/scriptlet/log" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" apiScriptlet "github.com/lxc/incus/v7/shared/api/scriptlet" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/resources" "github.com/lxc/incus/v7/shared/scriptlet" ) // InstancePlacementRun runs the instance placement scriptlet and returns the chosen cluster member target. func InstancePlacementRun(ctx context.Context, l logger.Logger, s *state.State, req *apiScriptlet.InstancePlacement, candidateMembers []db.NodeInfo, leaderAddress string) (*db.NodeInfo, error) { ctx, cancel := context.WithCancel(ctx) defer cancel() logFunc := log.CreateLogger(l, "Instance placement scriptlet") var targetMember *db.NodeInfo setTargetFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var memberName string err := starlark.UnpackArgs(b.Name(), args, kwargs, "member_name", &memberName) if err != nil { return nil, err } for i := range candidateMembers { if candidateMembers[i].Name == memberName { targetMember = &candidateMembers[i] break } } if targetMember == nil { l.Error("Instance placement scriptlet set invalid member target", logger.Ctx{"member": memberName}) return nil, fmt.Errorf("Invalid member name: %s", memberName) } l.Info("Instance placement scriptlet set member target", logger.Ctx{"member": targetMember.Name}) return starlark.None, nil } getClusterMemberResourcesFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var memberName string err := starlark.UnpackArgs(b.Name(), args, kwargs, "member_name", &memberName) if err != nil { return nil, err } var res *api.Resources // Get the local resource usage. if memberName == s.ServerName { res, err = resources.GetResources() if err != nil { return nil, err } } else { // Get remote member resource usage. var targetMember *db.NodeInfo for i := range candidateMembers { if candidateMembers[i].Name == memberName { targetMember = &candidateMembers[i] break } } if targetMember == nil { return nil, fmt.Errorf("Invalid member name: %s", memberName) } client, err := cluster.Connect(targetMember.Address, s.Endpoints.NetworkCert(), s.ServerCert(), nil, true) if err != nil { return nil, err } res, err = client.GetServerResources() if err != nil { return nil, err } } rv, err := scriptlet.StarlarkMarshal(res) if err != nil { return nil, fmt.Errorf("Marshalling cluster member resources for %q failed: %w", memberName, err) } return rv, nil } getClusterMemberStateFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var memberName string err := starlark.UnpackArgs(b.Name(), args, kwargs, "member_name", &memberName) if err != nil { return nil, err } var memberState *api.ClusterMemberState // Get the local resource usage. if memberName == s.ServerName { memberState, err = cluster.MemberState(ctx, s, memberName) if err != nil { return nil, err } } else { // Get remote member resource usage. var targetMember *db.NodeInfo for i := range candidateMembers { if candidateMembers[i].Name == memberName { targetMember = &candidateMembers[i] break } } if targetMember == nil { return nil, fmt.Errorf("Invalid member name: %s", memberName) } client, err := cluster.Connect(targetMember.Address, s.Endpoints.NetworkCert(), s.ServerCert(), nil, true) if err != nil { return nil, err } memberState, _, err = client.GetClusterMemberState(memberName) if err != nil { return nil, err } } rv, err := scriptlet.StarlarkMarshal(memberState) if err != nil { return nil, fmt.Errorf("Marshalling cluster member state for %q failed: %w", memberName, err) } return rv, nil } getInstanceResourcesFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var err error var res apiScriptlet.InstanceResources usageCPU, usageMemory, usageDisk, err := internalInstance.ResourceUsage(req.Config, req.Devices, req.Type) if err != nil { return nil, fmt.Errorf("Failed to calculate instance resource usage: %w", err) } res.CPUCores = uint64(usageCPU) res.MemorySize = uint64(usageMemory) res.RootDiskSize = uint64(usageDisk) rv, err := scriptlet.StarlarkMarshal(res) if err != nil { return nil, fmt.Errorf("Marshalling instance resources failed: %w", err) } return rv, nil } getInstancesFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var project string var location string err := starlark.UnpackArgs(b.Name(), args, kwargs, "project??", &project, "location??", &location) if err != nil { return nil, err } instanceList := []api.Instance{} err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { var objects []dbCluster.Instance if project != "" || location != "" { // Prepare a filter. filter := dbCluster.InstanceFilter{} if project != "" { filter.Project = &project } if location != "" { filter.Node = &location } // Get instances based on Project and/or Location filters. objects, err = dbCluster.GetInstances(ctx, tx.Tx(), filter) if err != nil { return err } } else { // Get all instances. objects, err = dbCluster.GetInstances(ctx, tx.Tx()) if err != nil { return err } } objectDevices, err := dbCluster.GetAllInstanceDevices(ctx, tx.Tx()) if err != nil { return err } // Convert the []Instances into []api.Instances. for _, obj := range objects { instance, err := obj.ToAPI(ctx, tx.Tx(), objectDevices, nil, nil) if err != nil { return err } instanceList = append(instanceList, *instance) } return nil }) if err != nil { return nil, err } rv, err := scriptlet.StarlarkMarshal(instanceList) if err != nil { return nil, fmt.Errorf("Marshalling instances failed: %w", err) } return rv, nil } getInstancesCountFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var projectName string var locationName string var includePending bool err := starlark.UnpackArgs(b.Name(), args, kwargs, "project??", &projectName, "location??", &locationName, "pending??", &includePending) if err != nil { return nil, err } var count int err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { count, err = tx.GetInstancesCount(ctx, projectName, locationName, includePending) return err }) if err != nil { return nil, err } rv, err := scriptlet.StarlarkMarshal(count) if err != nil { return nil, fmt.Errorf("Marshalling instance count failed: %w", err) } return rv, nil } getClusterMembersFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var group string var allMembers []db.NodeInfo err := starlark.UnpackArgs(b.Name(), args, kwargs, "group??", &group) if err != nil { return nil, err } err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { allMembers, err = tx.GetNodes(ctx) if err != nil { return err } allMembers, err = tx.GetCandidateMembers(ctx, allMembers, nil, group, nil, s.GlobalConfig.OfflineThreshold()) if err != nil { return err } return nil }) if err != nil { return nil, err } var raftNodes []db.RaftNode err = s.DB.Node.Transaction(ctx, func(ctx context.Context, tx *db.NodeTx) error { raftNodes, err = tx.GetRaftNodes(ctx) if err != nil { return fmt.Errorf("Failed loading RAFT nodes: %w", err) } return nil }) if err != nil { return nil, err } allMembersInfo := make([]*api.ClusterMember, 0, len(allMembers)) err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { failureDomains, err := tx.GetFailureDomainsNames(ctx) if err != nil { return fmt.Errorf("Failed loading failure domains names: %w", err) } memberFailureDomains, err := tx.GetNodesFailureDomains(ctx) if err != nil { return fmt.Errorf("Failed loading member failure domains: %w", err) } maxVersion, err := tx.GetNodeMaxVersion(ctx) if err != nil { return fmt.Errorf("Failed getting max member version: %w", err) } args := db.NodeInfoArgs{ LeaderAddress: leaderAddress, FailureDomains: failureDomains, MemberFailureDomains: memberFailureDomains, OfflineThreshold: s.GlobalConfig.OfflineThreshold(), MaxMemberVersion: maxVersion, RaftNodes: raftNodes, } for i := range allMembers { candidateMemberInfo, err := allMembers[i].ToAPI(ctx, tx, args) if err != nil { return err } allMembersInfo = append(allMembersInfo, candidateMemberInfo) } return nil }) if err != nil { return nil, err } rv, err := scriptlet.StarlarkMarshal(allMembersInfo) if err != nil { return nil, fmt.Errorf("Marshalling cluster members failed: %w", err) } return rv, nil } getProjectFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var name string err := starlark.UnpackArgs(b.Name(), args, kwargs, "name??", &name) if err != nil { return nil, err } var p *api.Project err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { dbProject, err := dbCluster.GetProject(ctx, tx.Tx(), name) if err != nil { return err } p, err = dbProject.ToAPI(ctx, tx.Tx()) if err != nil { return err } return nil }) if err != nil { return nil, err } rv, err := scriptlet.StarlarkMarshal(p) if err != nil { return nil, fmt.Errorf("Marshalling project failed: %w", err) } return rv, nil } var err error var raftNodes []db.RaftNode err = s.DB.Node.Transaction(ctx, func(ctx context.Context, tx *db.NodeTx) error { raftNodes, err = tx.GetRaftNodes(ctx) if err != nil { return fmt.Errorf("Failed loading RAFT nodes: %w", err) } return nil }) if err != nil { return nil, err } candidateMembersInfo := make([]*api.ClusterMember, 0, len(candidateMembers)) err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { failureDomains, err := tx.GetFailureDomainsNames(ctx) if err != nil { return fmt.Errorf("Failed loading failure domains names: %w", err) } memberFailureDomains, err := tx.GetNodesFailureDomains(ctx) if err != nil { return fmt.Errorf("Failed loading member failure domains: %w", err) } maxVersion, err := tx.GetNodeMaxVersion(ctx) if err != nil { return fmt.Errorf("Failed getting max member version: %w", err) } args := db.NodeInfoArgs{ LeaderAddress: leaderAddress, FailureDomains: failureDomains, MemberFailureDomains: memberFailureDomains, OfflineThreshold: s.GlobalConfig.OfflineThreshold(), MaxMemberVersion: maxVersion, RaftNodes: raftNodes, } for i := range candidateMembers { candidateMemberInfo, err := candidateMembers[i].ToAPI(ctx, tx, args) if err != nil { return err } candidateMembersInfo = append(candidateMembersInfo, candidateMemberInfo) } return nil }) if err != nil { return nil, err } // Remember to match the entries in scriptletLoad.InstancePlacementCompile() with this list so Starlark can // perform compile time validation of functions used. env := starlark.StringDict{ "log_info": starlark.NewBuiltin("log_info", logFunc), "log_warn": starlark.NewBuiltin("log_warn", logFunc), "log_error": starlark.NewBuiltin("log_error", logFunc), "set_target": starlark.NewBuiltin("set_target", setTargetFunc), "get_cluster_member_resources": starlark.NewBuiltin("get_cluster_member_resources", getClusterMemberResourcesFunc), "get_cluster_member_state": starlark.NewBuiltin("get_cluster_member_state", getClusterMemberStateFunc), "get_instance_resources": starlark.NewBuiltin("get_instance_resources", getInstanceResourcesFunc), "get_instances": starlark.NewBuiltin("get_instances", getInstancesFunc), "get_instances_count": starlark.NewBuiltin("get_instances_count", getInstancesCountFunc), "get_cluster_members": starlark.NewBuiltin("get_cluster_members", getClusterMembersFunc), "get_project": starlark.NewBuiltin("get_project", getProjectFunc), } prog, thread, err := scriptletLoad.InstancePlacementProgram() if err != nil { return nil, err } go func() { <-ctx.Done() thread.Cancel("Request finished") }() globals, err := prog.Init(thread, env) if err != nil { return nil, fmt.Errorf("Failed initializing: %w", err) } globals.Freeze() // Retrieve a global variable from starlark environment. instancePlacement := globals["instance_placement"] if instancePlacement == nil { return nil, errors.New("Scriptlet missing instance_placement function") } rv, err := scriptlet.StarlarkMarshal(req) if err != nil { return nil, fmt.Errorf("Marshalling request failed: %w", err) } candidateMembersv, err := scriptlet.StarlarkMarshal(candidateMembersInfo) if err != nil { return nil, fmt.Errorf("Marshalling candidate members failed: %w", err) } // Call starlark function from Go. v, err := starlark.Call(thread, instancePlacement, nil, []starlark.Tuple{ { starlark.String("request"), rv, }, { starlark.String("candidate_members"), candidateMembersv, }, }) if err != nil { return nil, fmt.Errorf("Failed to run: %w", err) } if v.Type() != "NoneType" { return nil, fmt.Errorf("Failed with unexpected return value: %v", v) } return targetMember, nil } incus-7.0.0/internal/server/scriptlet/load/000077500000000000000000000000001517523235500207215ustar00rootroot00000000000000incus-7.0.0/internal/server/scriptlet/load/load.go000066400000000000000000000101441517523235500221670ustar00rootroot00000000000000package load import ( "go.starlark.net/starlark" "github.com/lxc/incus/v7/shared/scriptlet" ) // nameInstancePlacement is the name used in Starlark for the instance placement scriptlet. const nameInstancePlacement = "instance_placement" // prefixQEMU is the prefix used in Starlark for the QEMU scriptlet. const prefixQEMU = "qemu" // nameAuthorization is the name used in Starlark for the Authorization scriptlet. const nameAuthorization = "authorization" var loader = scriptlet.NewLoader() // InstancePlacementCompile compiles the instance placement scriptlet. func InstancePlacementCompile(name string, src string) (*starlark.Program, error) { return scriptlet.Compile(name, src, []string{ "log_info", "log_warn", "log_error", "set_target", "get_cluster_member_resources", "get_cluster_member_state", "get_instance_resources", "get_instances", "get_instances_count", "get_cluster_members", "get_project", }) } // InstancePlacementValidate validates the instance placement scriptlet. func InstancePlacementValidate(src string) error { return scriptlet.Validate(InstancePlacementCompile, nameInstancePlacement, src, scriptlet.Declaration{ scriptlet.Required("instance_placement"): {"request", "candidate_members"}, }) } // InstancePlacementSet compiles the instance placement scriptlet into memory for use with InstancePlacementRun. // If empty src is provided the current program is deleted. func InstancePlacementSet(src string) error { return loader.Set(InstancePlacementCompile, nameInstancePlacement, src) } // InstancePlacementProgram returns the precompiled instance placement scriptlet program. func InstancePlacementProgram() (*starlark.Program, *starlark.Thread, error) { return loader.Program("Instance placement", nameInstancePlacement) } // QEMUCompile compiles the QEMU scriptlet. func QEMUCompile(name string, src string) (*starlark.Program, error) { return scriptlet.Compile(name, src, []string{ "log_info", "log_warn", "log_error", "run_qmp", "run_command", "blockdev_add", "blockdev_del", "chardev_add", "chardev_change", "chardev_remove", "device_add", "device_del", "netdev_add", "netdev_del", "object_add", "object_del", "qom_get", "qom_list", "qom_set", "get_qemu_cmdline", "set_qemu_cmdline", "get_qemu_conf", "set_qemu_conf", }) } // QEMUValidate validates the QEMU scriptlet. func QEMUValidate(src string) error { return scriptlet.Validate(QEMUCompile, prefixQEMU, src, scriptlet.Declaration{ scriptlet.Required("qemu_hook"): {"instance", "stage"}, }) } // QEMUSet compiles the QEMU scriptlet into memory for use with QEMURun. // If empty src is provided the current program is deleted. func QEMUSet(src string, instance string) error { return loader.Set(QEMUCompile, prefixQEMU+"/"+instance, src) } // QEMUProgram returns the precompiled QEMU scriptlet program. func QEMUProgram(instance string) (*starlark.Program, *starlark.Thread, error) { return loader.Program("QEMU", prefixQEMU+"/"+instance) } // AuthorizationCompile compiles the authorization scriptlet. func AuthorizationCompile(name string, src string) (*starlark.Program, error) { return scriptlet.Compile(name, src, []string{ "log_info", "log_warn", "log_error", }) } // AuthorizationValidate validates the authorization scriptlet. func AuthorizationValidate(src string) error { return scriptlet.Validate(AuthorizationCompile, nameAuthorization, src, scriptlet.Declaration{ scriptlet.Required("authorize"): {"details", "object", "entitlement"}, scriptlet.Optional("get_instance_access"): {"project_name", "instance_name"}, scriptlet.Optional("get_project_access"): {"project_name"}, }) } // AuthorizationSet compiles the authorization scriptlet into memory for use with AuthorizationRun. // If empty src is provided the current program is deleted. func AuthorizationSet(src string) error { return loader.Set(AuthorizationCompile, nameAuthorization, src) } // AuthorizationProgram returns the precompiled authorization scriptlet program. func AuthorizationProgram() (*starlark.Program, *starlark.Thread, error) { return loader.Program("Authorization", nameAuthorization) } incus-7.0.0/internal/server/scriptlet/log/000077500000000000000000000000001517523235500205635ustar00rootroot00000000000000incus-7.0.0/internal/server/scriptlet/log/logger.go000066400000000000000000000015701517523235500223740ustar00rootroot00000000000000package log import ( "fmt" "strconv" "strings" "go.starlark.net/starlark" "github.com/lxc/incus/v7/shared/logger" ) // createLogger creates a logger for scriptlets. func CreateLogger(l logger.Logger, name string) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var sb strings.Builder for _, arg := range args { s, err := strconv.Unquote(arg.String()) if err != nil { s = arg.String() } sb.WriteString(s) } switch b.Name() { case "log_info": l.Info(fmt.Sprintf("%s: %s", name, sb.String())) case "log_warn": l.Warn(fmt.Sprintf("%s: %s", name, sb.String())) default: l.Error(fmt.Sprintf("%s: %s", name, sb.String())) } return starlark.None, nil } } incus-7.0.0/internal/server/scriptlet/qemu.go000066400000000000000000000260311517523235500213020ustar00rootroot00000000000000package scriptlet import ( "encoding/json" "errors" "fmt" "strings" "go.starlark.net/starlark" "github.com/lxc/incus/v7/internal/server/instance/drivers/cfg" "github.com/lxc/incus/v7/internal/server/instance/drivers/qmp" scriptletLoad "github.com/lxc/incus/v7/internal/server/scriptlet/load" "github.com/lxc/incus/v7/internal/server/scriptlet/log" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/scriptlet" ) // marshalQEMUConf marshals a configuration into a []map[string]any. func marshalQEMUConf(conf any) ([]map[string]any, error) { jsonConf, err := json.Marshal(conf) if err != nil { return nil, err } var newConf []map[string]any err = json.Unmarshal(jsonConf, &newConf) if err != nil { return nil, err } return newConf, nil } // unmarshalQEMUConf unmarshals a configuration into a []cfg.Section. func unmarshalQEMUConf(conf any) ([]cfg.Section, error) { jsonConf, err := json.Marshal(conf) if err != nil { return nil, err } var newConf []cfg.Section err = json.Unmarshal(jsonConf, &newConf) if err != nil { return nil, err } return newConf, nil } // QEMURun runs the QEMU scriptlet. func QEMURun(l logger.Logger, instance *api.Instance, cmdArgs *[]string, conf *[]cfg.Section, m *qmp.Monitor, stage string) error { logFunc := log.CreateLogger(l, "QEMU scriptlet ("+stage+")") // We do not want to handle a qemuCfgSection object within our scriptlet, for simplicity. cfgSections, err := marshalQEMUConf(conf) if err != nil { return err } assertQEMUStarted := func(name string) error { if stage == "config" { return fmt.Errorf("%s cannot be called at config stage", name) } return nil } assertConfigStage := func(name string) error { if stage != "config" { return fmt.Errorf("%s can only be called at config stage", name) } return nil } runQMPFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { err := assertQEMUStarted(b.Name()) if err != nil { return nil, err } var command *starlark.Dict err = starlark.UnpackArgs(b.Name(), args, kwargs, "command", &command) if err != nil { return nil, err } value, err := scriptlet.StarlarkUnmarshal(command) if err != nil { return nil, err } id := uint32(0) req, ok := value.(map[string]any) if ok { id = m.IncreaseID() req["id"] = id value = req } request, err := json.Marshal(value) if err != nil { return nil, err } var resp map[string]any err = m.RunJSON(request, &resp, true, id) if err != nil { return nil, err } rv, err := scriptlet.StarlarkMarshal(resp) if err != nil { return nil, fmt.Errorf("Marshalling QMP response failed: %w", err) } return rv, nil } runCommandFromKwargs := func(funName string, kwargs []starlark.Tuple) (starlark.Value, error) { qmpArgs := make(map[string]any) for _, kwarg := range kwargs { key, err := scriptlet.StarlarkUnmarshal(kwarg.Index(0)) if err != nil { return nil, err } value, err := scriptlet.StarlarkUnmarshal(kwarg.Index(1)) if err != nil { return nil, err } qmpArgs[key.(string)] = value } var resp struct { Return any } err := m.Run(funName, qmpArgs, &resp) if err != nil { return nil, err } // Extract the return value rv, err := scriptlet.StarlarkMarshal(resp.Return) if err != nil { return nil, fmt.Errorf("Marshalling QMP response failed: %w", err) } return rv, nil } runCommandFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { err := assertQEMUStarted(b.Name()) if err != nil { return nil, err } frame := thread.CallFrame(1) errPrefix := fmt.Sprintf("run_command (%d:%d):", frame.Pos.Line, frame.Pos.Col) argsLen := args.Len() if argsLen != 1 { return nil, fmt.Errorf("%s Expected exactly one positional argument, got %d", errPrefix, argsLen) } arg, err := scriptlet.StarlarkUnmarshal(args.Index(0)) if err != nil { return nil, err } funName, ok := arg.(string) if !ok { return nil, fmt.Errorf("%s The positional argument must be a string representing a QMP function", errPrefix) } rv, err := runCommandFromKwargs(funName, kwargs) if err != nil { return nil, err } return rv, nil } makeQOM := func(funName string) *starlark.Builtin { fun := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { err := assertQEMUStarted(b.Name()) if err != nil { return nil, err } frame := thread.CallFrame(1) errPrefix := fmt.Sprintf("%s (%d:%d):", b.Name(), frame.Pos.Line, frame.Pos.Col) argsLen := args.Len() if argsLen != 0 { return nil, fmt.Errorf("%s Expected only keyword arguments, got %d positional argument(s)", errPrefix, argsLen) } rv, err := runCommandFromKwargs(funName, kwargs) if err != nil { return nil, err } return rv, nil } return starlark.NewBuiltin(strings.ReplaceAll(funName, "-", "_"), fun) } getCmdArgsFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { err := starlark.UnpackArgs(b.Name(), args, kwargs) if err != nil { return nil, err } rv, err := scriptlet.StarlarkMarshal(cmdArgs) if err != nil { return nil, fmt.Errorf("Marshalling QEMU command-line arguments failed: %w", err) } return rv, nil } setCmdArgsFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { err := assertConfigStage(b.Name()) if err != nil { return nil, err } var newCmdArgsv *starlark.List err = starlark.UnpackArgs(b.Name(), args, kwargs, "args", &newCmdArgsv) if err != nil { return nil, err } newCmdArgsAny, err := scriptlet.StarlarkUnmarshal(newCmdArgsv) if err != nil { return nil, err } newCmdArgsListAny, ok := newCmdArgsAny.([]any) if !ok { return nil, fmt.Errorf("%s requires a list of strings", b.Name()) } // Check whether -bios or -kernel are in the new arguments, and convert them to string on the go. var newFoundBios, newFoundKernel bool var newCmdArgs []string for _, argAny := range newCmdArgsListAny { arg, ok := argAny.(string) if !ok { return nil, fmt.Errorf("%s requires a list of strings", b.Name()) } newCmdArgs = append(newCmdArgs, arg) if arg == "-bios" { newFoundBios = true } else if arg == "-kernel" { newFoundKernel = true } } // Check whether -bios or -kernel are in the current arguments var foundBios, foundKernel bool for _, arg := range *cmdArgs { if arg == "-bios" { foundBios = true } else if arg == "-kernel" { foundKernel = true } // If we've found both already, we can break early. if foundBios && foundKernel { break } } if foundBios != newFoundBios || foundKernel != newFoundKernel { return nil, errors.New("Addition or deletion of -bios or -kernel is unsupported") } *cmdArgs = newCmdArgs return starlark.None, nil } getConfFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { err := starlark.UnpackArgs(b.Name(), args, kwargs) if err != nil { return nil, err } rv, err := scriptlet.StarlarkMarshal(cfgSections) if err != nil { return nil, fmt.Errorf("Marshalling QEMU configuration failed: %w", err) } return rv, nil } setConfFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { err := assertConfigStage(b.Name()) if err != nil { return nil, err } var newConf *starlark.List err = starlark.UnpackArgs(b.Name(), args, kwargs, "conf", &newConf) if err != nil { return nil, err } confAny, err := scriptlet.StarlarkUnmarshal(newConf) if err != nil { return nil, err } confListAny, ok := confAny.([]any) if !ok { return nil, fmt.Errorf("%s requires a valid configuration", b.Name()) } var newCfgSections []map[string]any for _, section := range confListAny { newSection, ok := section.(map[string]any) if !ok { return nil, fmt.Errorf("%s requires a valid configuration", b.Name()) } newCfgSections = append(newCfgSections, newSection) } // We want to further check the configuration structure, by trying to unmarshal it to a // []cfg.Section. _, err = unmarshalQEMUConf(confAny) if err != nil { return nil, fmt.Errorf("%s requires a valid configuration", b.Name()) } cfgSections = newCfgSections return starlark.None, nil } // Remember to match the entries in scriptletLoad.QEMUCompile() with this list so Starlark can // perform compile time validation of functions used. env := starlark.StringDict{ "log_info": starlark.NewBuiltin("log_info", logFunc), "log_warn": starlark.NewBuiltin("log_warn", logFunc), "log_error": starlark.NewBuiltin("log_error", logFunc), "run_qmp": starlark.NewBuiltin("run_qmp", runQMPFunc), "run_command": starlark.NewBuiltin("run_command", runCommandFunc), "blockdev_add": makeQOM("blockdev-add"), "blockdev_del": makeQOM("blockdev-del"), "chardev_add": makeQOM("chardev-add"), "chardev_change": makeQOM("chardev-change"), "chardev_remove": makeQOM("chardev-remove"), "device_add": makeQOM("device_add"), "device_del": makeQOM("device_del"), "netdev_add": makeQOM("netdev_add"), "netdev_del": makeQOM("netdev_del"), "object_add": makeQOM("object-add"), "object_del": makeQOM("object-del"), "qom_get": makeQOM("qom-get"), "qom_list": makeQOM("qom-list"), "qom_set": makeQOM("qom-set"), "get_qemu_cmdline": starlark.NewBuiltin("get_qemu_cmdline", getCmdArgsFunc), "set_qemu_cmdline": starlark.NewBuiltin("set_qemu_cmdline", setCmdArgsFunc), "get_qemu_conf": starlark.NewBuiltin("get_qemu_conf", getConfFunc), "set_qemu_conf": starlark.NewBuiltin("set_qemu_conf", setConfFunc), } prog, thread, err := scriptletLoad.QEMUProgram(instance.Name) if err != nil { return err } globals, err := prog.Init(thread, env) if err != nil { return fmt.Errorf("Failed initializing: %w", err) } globals.Freeze() // Retrieve a global variable from starlark environment. qemuHook := globals["qemu_hook"] if qemuHook == nil { return errors.New("Scriptlet missing qemu_hook function") } instancev, err := scriptlet.StarlarkMarshal(instance) if err != nil { return fmt.Errorf("Marshalling instance failed: %w", err) } // Call starlark function from Go. v, err := starlark.Call(thread, qemuHook, nil, []starlark.Tuple{ { starlark.String("instance"), instancev, }, { starlark.String("stage"), starlark.String(stage), }, }) if err != nil { return fmt.Errorf("Failed to run: %w", err) } if v.Type() != "NoneType" { return fmt.Errorf("Failed with unexpected return value: %v", v) } // We need to convert the configuration back to a suitable format *conf, err = unmarshalQEMUConf(cfgSections) if err != nil { return err } return nil } incus-7.0.0/internal/server/seccomp/000077500000000000000000000000001517523235500174225ustar00rootroot00000000000000incus-7.0.0/internal/server/seccomp/cgo.go000066400000000000000000000006621517523235500205250ustar00rootroot00000000000000//go:build linux && cgo package seccomp // #cgo CFLAGS: -std=gnu11 -Wvla -Werror -fvisibility=hidden -Winit-self // #cgo CFLAGS: -Wformat=2 -Wshadow -Wendif-labels -fasynchronous-unwind-tables // #cgo CFLAGS: -pipe --param=ssp-buffer-size=4 -g -Wunused // #cgo CFLAGS: -Werror=implicit-function-declaration // #cgo CFLAGS: -Werror=return-type -Wendif-labels -Werror=overflow // #cgo CFLAGS: -Wnested-externs -fexceptions import "C" incus-7.0.0/internal/server/seccomp/seccomp.go000066400000000000000000002022471517523235500214110ustar00rootroot00000000000000//go:build linux && cgo package seccomp /* #ifndef _GNU_SOURCE #define _GNU_SOURCE 1 #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../../../shared/cgo/incus_bpf.h" #include "../../../shared/cgo/incus_seccomp.h" #include "../../../shared/cgo/memory_utils.h" #include "../../../shared/cgo/process_utils.h" #include "../../../shared/cgo/syscall_wrappers.h" struct seccomp_notif_sizes expected_sizes; struct seccomp_notify_proxy_msg { uint64_t __reserved; pid_t monitor_pid; pid_t init_pid; struct seccomp_notif_sizes sizes; uint64_t cookie_len; // followed by: seccomp_notif, seccomp_notif_resp, cookie }; #define INCUS_SCHED_PARAM_SIZE (sizeof(struct sched_param)) #define SECCOMP_PROXY_MSG_SIZE (sizeof(struct seccomp_notify_proxy_msg)) #define SECCOMP_NOTIFY_SIZE (sizeof(struct seccomp_notif)) #define SECCOMP_RESPONSE_SIZE (sizeof(struct seccomp_notif_resp)) #define SECCOMP_MSG_SIZE_MIN (SECCOMP_PROXY_MSG_SIZE + SECCOMP_NOTIFY_SIZE + SECCOMP_RESPONSE_SIZE) #define SECCOMP_COOKIE_SIZE (64 * sizeof(char)) #define SECCOMP_MSG_SIZE_MAX (SECCOMP_MSG_SIZE_MIN + SECCOMP_COOKIE_SIZE) static int seccomp_notify_get_sizes(struct seccomp_notif_sizes *sizes) { if (syscall(SYS_seccomp, SECCOMP_GET_NOTIF_SIZES, 0, sizes) != 0) return -1; if (sizes->seccomp_notif != sizeof(struct seccomp_notif) || sizes->seccomp_notif_resp != sizeof(struct seccomp_notif_resp) || sizes->seccomp_data != sizeof(struct seccomp_data)) return -1; return 0; } static int device_allowed(dev_t dev, mode_t mode) { switch (mode & S_IFMT) { case S_IFCHR: if (dev == makedev(0, 0)) // whiteout return 0; else if (dev == makedev(5, 1)) // /dev/console return 0; else if (dev == makedev(1, 7)) // /dev/full return 0; else if (dev == makedev(1, 3)) // /dev/null return 0; else if (dev == makedev(1, 8)) // /dev/random return 0; else if (dev == makedev(5, 0)) // /dev/tty return 0; else if (dev == makedev(1, 9)) // /dev/urandom return 0; else if (dev == makedev(1, 5)) // /dev/zero return 0; } return -EPERM; } #include struct incus_seccomp_data_arch { int arch; int nr_mknod; int nr_mknodat; int nr_setxattr; int nr_mount; int nr_bpf; int nr_sched_setscheduler; int nr_sysinfo; }; #define INCUS_SECCOMP_NOTIFY_MKNOD 0 #define INCUS_SECCOMP_NOTIFY_MKNODAT 1 #define INCUS_SECCOMP_NOTIFY_SETXATTR 2 #define INCUS_SECCOMP_NOTIFY_MOUNT 3 #define INCUS_SECCOMP_NOTIFY_BPF 4 #define INCUS_SECCOMP_NOTIFY_SCHED_SETSCHEDULER 5 #define INCUS_SECCOMP_NOTIFY_SYSINFO 6 // ordered by likelihood of usage... static const struct incus_seccomp_data_arch seccomp_notify_syscall_table[] = { { -1, INCUS_SECCOMP_NOTIFY_MKNOD, INCUS_SECCOMP_NOTIFY_MKNODAT, INCUS_SECCOMP_NOTIFY_SETXATTR, INCUS_SECCOMP_NOTIFY_MOUNT, INCUS_SECCOMP_NOTIFY_BPF, INCUS_SECCOMP_NOTIFY_SCHED_SETSCHEDULER, INCUS_SECCOMP_NOTIFY_SYSINFO}, #ifdef AUDIT_ARCH_X86_64 { AUDIT_ARCH_X86_64, 133, 259, 188, 165, 321, 144, 99 }, #endif #ifdef AUDIT_ARCH_I386 { AUDIT_ARCH_I386, 14, 297, 226, 21, 357, 156, 116 }, #endif #ifdef AUDIT_ARCH_AARCH64 { AUDIT_ARCH_AARCH64, -1, 33, 5, 40, 280, 119, 179 }, #endif #ifdef AUDIT_ARCH_ARM { AUDIT_ARCH_ARM, 14, 324, 226, 21, 386, 156, 116 }, #endif #ifdef AUDIT_ARCH_ARMEB { AUDIT_ARCH_ARMEB, 14, 324, 226, 21, 386, 156, 116 }, #endif #ifdef AUDIT_ARCH_S390 { AUDIT_ARCH_S390, 14, 290, 224, 21, 351, 156, 116 }, #endif #ifdef AUDIT_ARCH_S390X { AUDIT_ARCH_S390X, 14, 290, 224, 21, 351, 156, 116 }, #endif #ifdef AUDIT_ARCH_PPC { AUDIT_ARCH_PPC, 14, 288, 209, 21, 361, 156, 116 }, #endif #ifdef AUDIT_ARCH_PPC64 { AUDIT_ARCH_PPC64, 14, 288, 209, 21, 361, 156, 116 }, #endif #ifdef AUDIT_ARCH_PPC64LE { AUDIT_ARCH_PPC64LE, 14, 288, 209, 21, 361, 156, 116 }, #endif #ifdef AUDIT_ARCH_RISCV64 { AUDIT_ARCH_RISCV64, -1, 33, 5, 40, 280, 119, 179 }, #endif #ifdef AUDIT_ARCH_SPARC { AUDIT_ARCH_SPARC, 14, 286, 169, 167, 349, 243, 214 }, #endif #ifdef AUDIT_ARCH_SPARC64 { AUDIT_ARCH_SPARC64, 14, 286, 169, 167, 349, 243, 214 }, #endif #ifdef AUDIT_ARCH_MIPS { AUDIT_ARCH_MIPS, 14, 290, 224, 21, -1, 141, 4116 }, #endif #ifdef AUDIT_ARCH_MIPSEL { AUDIT_ARCH_MIPSEL, 14, 290, 224, 21, -1, 141, 4116 }, #endif #ifdef AUDIT_ARCH_MIPS64 { AUDIT_ARCH_MIPS64, 131, 249, 180, 160, -1, 141, 5097 }, #endif #ifdef AUDIT_ARCH_MIPS64N32 { AUDIT_ARCH_MIPS64N32, 131, 253, 180, 160, -1, 141, 4116 }, #endif #ifdef AUDIT_ARCH_MIPSEL64 { AUDIT_ARCH_MIPSEL64, 131, 249, 180, 160, -1, 141, 5097 }, #endif #ifdef AUDIT_ARCH_MIPSEL64N32 { AUDIT_ARCH_MIPSEL64N32, 131, 253, 180, 160, -1, 141, 4116 }, #endif #ifdef AUDIT_ARCH_LOONGARCH64 { AUDIT_ARCH_LOONGARCH64, -1, 33, 5, 40, 280, 119, 179 }, #endif }; static int seccomp_notify_get_syscall(struct seccomp_notif *req, struct seccomp_notif_resp *resp) { resp->id = req->id; resp->flags = req->flags; resp->val = 0; resp->error = 0; for (size_t i = 0; i < (sizeof(seccomp_notify_syscall_table) / sizeof(seccomp_notify_syscall_table[0])); i++) { const struct incus_seccomp_data_arch *entry = &seccomp_notify_syscall_table[i]; if (entry->arch != req->data.arch) continue; if (entry->nr_mknod == req->data.nr) return INCUS_SECCOMP_NOTIFY_MKNOD; if (entry->nr_mknodat == req->data.nr) return INCUS_SECCOMP_NOTIFY_MKNODAT; if (entry->nr_setxattr == req->data.nr) return INCUS_SECCOMP_NOTIFY_SETXATTR; if (entry->nr_mount == req->data.nr) return INCUS_SECCOMP_NOTIFY_MOUNT; if (entry->nr_bpf == req->data.nr) return INCUS_SECCOMP_NOTIFY_BPF; if (entry->nr_sched_setscheduler == req->data.nr) return INCUS_SECCOMP_NOTIFY_SCHED_SETSCHEDULER; if (entry->nr_sysinfo == req->data.nr) return INCUS_SECCOMP_NOTIFY_SYSINFO; break; } errno = EINVAL; return -EINVAL; } static void seccomp_notify_update_response(struct seccomp_notif_resp *resp, int new_neg_errno, uint32_t flags) { resp->error = new_neg_errno; resp->flags |= flags; } static void prepare_seccomp_iovec(struct iovec *iov, struct seccomp_notify_proxy_msg *msg, struct seccomp_notif *notif, struct seccomp_notif_resp *resp, char *cookie) { iov[0].iov_base = msg; iov[0].iov_len = SECCOMP_PROXY_MSG_SIZE; iov[1].iov_base = notif; iov[1].iov_len = SECCOMP_NOTIFY_SIZE; iov[2].iov_base = resp; iov[2].iov_len = SECCOMP_RESPONSE_SIZE; iov[3].iov_base = cookie; iov[3].iov_len = SECCOMP_COOKIE_SIZE; } // We use the BPF_DEVCG_DEV_CHAR macro as a cheap way to detect whether the kernel has // the correct headers available to be compiled for bpf support. Since cgo doesn't have // a good way of letting us probe for structs or enums the alternative would be to vendor // bpf.h similar to what we do for seccomp itself. But that's annoying since bpf.h is quite // large. So users that want bpf interception support should make sure to have the relevant // header available at build time. static inline int pidfd_getfd(int pidfd, int fd, int flags) { return syscall(__NR_pidfd_getfd, pidfd, fd, flags); } #define ptr_to_u64(p) ((__u64)((uintptr_t)(p))) static inline int bpf(int cmd, union bpf_attr *attr, size_t size) { return syscall(__NR_bpf, cmd, attr, size); } static int handle_bpf_syscall(pid_t pid_target, int notify_fd, int mem_fd, int tgid, struct seccomp_notify_proxy_msg *msg, struct seccomp_notif *req, struct seccomp_notif_resp *resp, int *bpf_cmd, int *bpf_prog_type, int *bpf_attach_type, unsigned int flags) { __do_close int pidfd = -EBADF, bpf_target_fd = -EBADF, bpf_attach_fd = -EBADF, bpf_prog_fd = -EBADF; __do_free struct bpf_insn *insn = NULL; char log_buf[4096] = {}; char license[128] = {}; size_t insn_size = 0; union bpf_attr attr = {}, new_attr = {}; unsigned int attr_len = sizeof(attr); struct seccomp_notif_addfd addfd = {}; int ret; int cmd; *bpf_cmd = -EINVAL; *bpf_prog_type = -EINVAL; *bpf_attach_type = -EINVAL; if (attr_len < req->data.args[2]) return -EFBIG; attr_len = req->data.args[2]; *bpf_cmd = req->data.args[0]; switch (req->data.args[0]) { case BPF_PROG_LOAD: cmd = BPF_PROG_LOAD; break; case BPF_PROG_ATTACH: cmd = BPF_PROG_ATTACH; break; case BPF_PROG_DETACH: cmd = BPF_PROG_DETACH; break; default: return -EINVAL; } ret = pread(mem_fd, &attr, attr_len, req->data.args[1]); if (ret < 0) return -errno; *bpf_prog_type = attr.prog_type; if (flags & PIDFD_THREAD) pidfd = incus_pidfd_open(pid_target, PIDFD_THREAD); else pidfd = incus_pidfd_open(tgid, 0); if (pidfd < 0) return -errno; if (ioctl(notify_fd, SECCOMP_IOCTL_NOTIF_ID_VALID, &req->id)) return -errno; switch (cmd) { case BPF_PROG_LOAD: if (attr.prog_type != BPF_PROG_TYPE_CGROUP_DEVICE) return -EINVAL; // bpf is currently limited to 1 million instructions. Don't // allow the container to allocate more than that. if (attr.insn_cnt > 1000000) return -EINVAL; insn_size = sizeof(struct bpf_insn) * attr.insn_cnt; insn = malloc(insn_size); if (!insn) return -ENOMEM; ret = pread(mem_fd, insn, insn_size, attr.insns); if (ret < 0) return -errno; if (ret != insn_size) return -EIO; memcpy(&new_attr, &attr, sizeof(attr)); if (attr.log_size > sizeof(log_buf)) new_attr.log_size = sizeof(log_buf); if (new_attr.log_size > 0) new_attr.log_buf = ptr_to_u64(log_buf); if (attr.license && pread(mem_fd, license, sizeof(license), attr.license) < 0) return -errno; new_attr.insns = ptr_to_u64(insn); new_attr.license = ptr_to_u64(license); bpf_prog_fd = bpf(cmd, &new_attr, sizeof(new_attr)); if (bpf_prog_fd < 0) { int saved_errno = errno; if ((new_attr.log_size) > 0 && (pwrite(mem_fd, log_buf, new_attr.log_size, attr.log_buf) != new_attr.log_size)) errno = saved_errno; return -errno; } addfd.srcfd = bpf_prog_fd; addfd.id = req->id; addfd.flags = 0; ret = ioctl(notify_fd, SECCOMP_IOCTL_NOTIF_ADDFD, &addfd); if (ret < 0) return -errno; resp->val = ret; ret = 0; break; case BPF_PROG_ATTACH: if (attr.attach_type != BPF_CGROUP_DEVICE) return -EINVAL; *bpf_attach_type = attr.attach_type; bpf_target_fd = pidfd_getfd(pidfd, attr.target_fd, 0); if (bpf_target_fd < 0) return -errno; bpf_attach_fd = pidfd_getfd(pidfd, attr.attach_bpf_fd, 0); if (bpf_attach_fd < 0) return -errno; if (!(flags & PIDFD_THREAD) && tgid != pid_target) { // Make sure that the file descriptor table is shared // so we can be sure that we're talking about the same // open files. if (!filetable_shared(tgid, pid_target)) return -EINVAL; // Make sure that the threadgroup leader hasn't been // recycled in the meantime so we're not accidentally // looking at a different threadgroup with the same // open files. if (!process_still_alive(pidfd)) return -EINVAL; } attr.target_fd = bpf_target_fd; attr.attach_bpf_fd = bpf_attach_fd; ret = bpf(cmd, &attr, attr_len); break; case BPF_PROG_DETACH: if (attr.attach_type != BPF_CGROUP_DEVICE) return -EINVAL; *bpf_attach_type = attr.attach_type; bpf_target_fd = pidfd_getfd(pidfd, attr.target_fd, 0); if (bpf_target_fd < 0) return -errno; bpf_attach_fd = pidfd_getfd(pidfd, attr.attach_bpf_fd, 0); if (bpf_attach_fd < 0) return -errno; if (!(flags & PIDFD_THREAD) && tgid != pid_target) { // Make sure that the file descriptor table is shared // so we can be sure that we're talking about the same // open files. if (!filetable_shared(tgid, pid_target)) return -EINVAL; // Make sure that the threadgroup leader hasn't been // recycled in the meantime so we're not accidentally // looking at a different threadgroup with the same // open files. if (!process_still_alive(pidfd)) return -EINVAL; } attr.target_fd = bpf_target_fd; attr.attach_bpf_fd = bpf_attach_fd; ret = bpf(cmd, &attr, attr_len); break; } return ret; } #ifndef MS_LAZYTIME #define MS_LAZYTIME (1<<25) #endif */ import "C" import ( "context" "errors" "fmt" "io" "net" "os" "path/filepath" "regexp" "runtime" "strconv" "strings" "time" "unsafe" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/netutils" "github.com/lxc/incus/v7/internal/server/cgroup" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" _ "github.com/lxc/incus/v7/shared/cgo" // Used by cgo "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) const ( incusSeccompNotifyMknod = C.INCUS_SECCOMP_NOTIFY_MKNOD incusSeccompNotifyMknodat = C.INCUS_SECCOMP_NOTIFY_MKNODAT incusSeccompNotifySetxattr = C.INCUS_SECCOMP_NOTIFY_SETXATTR incusSeccompNotifyMount = C.INCUS_SECCOMP_NOTIFY_MOUNT incusSeccompNotifyBpf = C.INCUS_SECCOMP_NOTIFY_BPF incusSeccompNotifySchedSetscheduler = C.INCUS_SECCOMP_NOTIFY_SCHED_SETSCHEDULER incusSeccompNotifySysinfo = C.INCUS_SECCOMP_NOTIFY_SYSINFO ) const seccompHeader = `2 ` const defaultSeccompPolicy = `reject_force_umount # comment this to allow umount -f; not recommended [all] kexec_load errno 38 open_by_handle_at errno 38 init_module errno 38 finit_module errno 38 delete_module errno 38 ` // 8 == SECCOMP_FILTER_FLAG_NEW_LISTENER // // 2146435072 == SECCOMP_RET_TRACE . const seccompNotifyDisallow = `seccomp errno 22 [1,2146435072,SCMP_CMP_MASKED_EQ,2146435072] seccomp errno 22 [1,8,SCMP_CMP_MASKED_EQ,8] ` const seccompNotifyMknod = `mknod notify [1,8192,SCMP_CMP_MASKED_EQ,61440] mknod notify [1,24576,SCMP_CMP_MASKED_EQ,61440] mknodat notify [2,8192,SCMP_CMP_MASKED_EQ,61440] mknodat notify [2,24576,SCMP_CMP_MASKED_EQ,61440] ` const seccompNotifySetxattr = `setxattr notify [3,1,SCMP_CMP_EQ] ` const seccompNotifySchedSetscheduler = `sched_setscheduler notify ` const seccompNotifySysinfo = `sysinfo notify ` const seccompBlockNewMountAPI = `fsopen errno 38 fsconfig errno 38 ` // We don't want to filter any of the following flag combinations since they do // not cause the creation of a new superblock: // // MS_REMOUNT // MS_BIND // MS_MOVE // MS_UNBINDABLE // MS_PRIVATE // MS_SLAVE // MS_SHARED // MS_KERNMOUNT // MS_I_VERSION // // So define the following mask of allowed flags: // // long unsigned int mask = MS_MGC_VAL | MS_RDONLY | MS_NOSUID | MS_NODEV | // MS_NOEXEC | MS_SYNCHRONOUS | MS_MANDLOCK | // MS_DIRSYNC | MS_NOATIME | MS_NODIRATIME | MS_REC | // MS_VERBOSE | MS_SILENT | MS_POSIXACL | MS_RELATIME | // MS_STRICTATIME | MS_LAZYTIME; // // Now we inverse the flag: // // inverse_mask ~= mask; // // Seccomp will now only intercept these flags if they do not contain any of // the allowed flags, i.e. we only intercept combinations were a new superblock // is created. const seccompNotifyMount = `mount notify [3,0,SCMP_CMP_MASKED_EQ,18446744070422410016] ` // 5 == BPF_PROG_LOAD // 8 == BPF_PROG_ATTACH // 9 == BPF_PROG_DETACH . const seccompNotifyBpf = `bpf notify [0,5,SCMP_CMP_EQ] bpf notify [0,8,SCMP_CMP_EQ] bpf notify [0,9,SCMP_CMP_EQ] ` const compatBlockingPolicy = `[%s] compat_sys_rt_sigaction errno 38 stub_x32_rt_sigreturn errno 38 compat_sys_ioctl errno 38 compat_sys_readv errno 38 compat_sys_writev errno 38 compat_sys_recvfrom errno 38 compat_sys_sendmsg errno 38 compat_sys_recvmsg errno 38 stub_x32_execve errno 38 compat_sys_ptrace errno 38 compat_sys_rt_sigpending errno 38 compat_sys_rt_sigtimedwait errno 38 compat_sys_rt_sigqueueinfo errno 38 compat_sys_sigaltstack errno 38 compat_sys_timer_create errno 38 compat_sys_mq_notify errno 38 compat_sys_kexec_load errno 38 compat_sys_waitid errno 38 compat_sys_set_robust_list errno 38 compat_sys_get_robust_list errno 38 compat_sys_vmsplice errno 38 compat_sys_move_pages errno 38 compat_sys_preadv64 errno 38 compat_sys_pwritev64 errno 38 compat_sys_rt_tgsigqueueinfo errno 38 compat_sys_recvmmsg errno 38 compat_sys_sendmmsg errno 38 compat_sys_process_vm_readv errno 38 compat_sys_process_vm_writev errno 38 compat_sys_setsockopt errno 38 compat_sys_getsockopt errno 38 compat_sys_io_setup errno 38 compat_sys_io_submit errno 38 stub_x32_execveat errno 38 ` // Instance is a seccomp specific instance interface. // This is used rather than instance.Instance to avoid import loops. type Instance interface { Name() string Project() api.Project ExpandedConfig() map[string]string IsPrivileged() bool Architecture() int RootfsPath() string CGroup() (*cgroup.CGroup, error) CurrentIdmap() (*idmap.Set, error) DiskIdmap() (*idmap.Set, error) IdmappedStorage(path string, fstype string) idmap.StorageType InsertSeccompUnixDevice(prefix string, m deviceConfig.Device, pid int) error } var seccompPath = internalUtil.VarPath("security", "seccomp") // ProfilePath returns the seccomp path for the instance. func ProfilePath(c Instance) string { return filepath.Join(seccompPath, project.Instance(c.Project().Name, c.Name())) } // InstanceNeedsPolicy returns whether the instance needs a policy or not. func InstanceNeedsPolicy(c Instance) bool { config := c.ExpandedConfig() // Check for text keys keys := []string{ "raw.seccomp", "security.syscalls.allow", "security.syscalls.deny", "security.syscalls.whitelist", "security.syscalls.blacklist", } for _, k := range keys { _, hasKey := config[k] if hasKey { return true } } // Check for boolean keys that default to false keys = []string{ "security.syscalls.deny_compat", "security.syscalls.blacklist_compat", "security.syscalls.intercept.mknod", "security.syscalls.intercept.sched_setscheduler", "security.syscalls.intercept.setxattr", "security.syscalls.intercept.sysinfo", "security.syscalls.intercept.mount", "security.syscalls.intercept.bpf", } for _, k := range keys { if util.IsTrue(config[k]) { return true } } // Check for boolean keys that default to true value, ok := config["security.syscalls.deny_default"] if !ok { value, ok = config["security.syscalls.blacklist_default"] } if !ok || util.IsTrue(value) { return true } return false } // InstanceNeedsIntercept returns whether instance needs intercept. func InstanceNeedsIntercept(s *state.State, c Instance) (bool, error) { // No need if privileged if c.IsPrivileged() { return false, nil } // If nested, assume the host handles it if s.OS.RunningInUserNS { return false, nil } config := c.ExpandedConfig() keys := []string{ "security.syscalls.intercept.mknod", "security.syscalls.intercept.sched_setscheduler", "security.syscalls.intercept.setxattr", "security.syscalls.intercept.sysinfo", "security.syscalls.intercept.mount", "security.syscalls.intercept.bpf", } for _, key := range keys { if util.IsFalseOrEmpty(config[key]) { continue } return true, nil } return false, nil } // MakePidFd prepares a pidfd to inherit for the init process of the container. func MakePidFd(pid int) (int, *os.File, error) { pidFdFile, err := linux.PidFdOpen(pid, 0) if err != nil { return -1, nil, err } return 3, pidFdFile, nil } func seccompGetPolicyContent(s *state.State, c Instance) (string, error) { config := c.ExpandedConfig() // Full policy override raw := config["raw.seccomp"] if raw != "" { return raw, nil } // Policy header policy := seccompHeader allowlist := config["security.syscalls.allow"] if allowlist == "" { allowlist = config["security.syscalls.whitelist"] } if allowlist != "" { policy += "allowlist\n[all]\n" policy += allowlist } else { policy += "denylist\n[all]\n" defaultFlag, ok := config["security.syscalls.deny_default"] if !ok { defaultFlag, ok = config["security.syscalls.blacklist_default"] } if !ok || util.IsTrue(defaultFlag) { policy += defaultSeccompPolicy } } // Syscall interception ok, err := InstanceNeedsIntercept(s, c) if err != nil { return "", err } if ok { // Prevent the container from overriding our syscall // supervision. policy += seccompNotifyDisallow if util.IsTrue(config["security.syscalls.intercept.mknod"]) { policy += seccompNotifyMknod } if util.IsTrue(config["security.syscalls.intercept.sched_setscheduler"]) { policy += seccompNotifySchedSetscheduler } if util.IsTrue(config["security.syscalls.intercept.setxattr"]) { policy += seccompNotifySetxattr } if util.IsTrue(config["security.syscalls.intercept.sysinfo"]) { policy += seccompNotifySysinfo } if util.IsTrue(config["security.syscalls.intercept.mount"]) { policy += seccompNotifyMount // We block a subset of the new mount api as we can't easily intercept those mounts. // Specifically, we're blocking fsopen and fsconfig which is // enough to get /sbin/mount to fallback to the old mount API. policy += seccompBlockNewMountAPI } if util.IsTrue(config["security.syscalls.intercept.bpf"]) { policy += seccompNotifyBpf } } if allowlist != "" { return policy, nil } // Additional deny entries compat, ok := config["security.syscalls.deny_compat"] if !ok { compat = config["security.syscalls.blacklist_compat"] } if util.IsTrue(compat) { arch, err := osarch.ArchitectureName(c.Architecture()) if err != nil { return "", err } policy += fmt.Sprintf(compatBlockingPolicy, arch) } denylist, ok := config["security.syscalls.deny"] if !ok { denylist = config["security.syscalls.blacklist"] } if denylist != "" { policy += denylist } return policy, nil } // CreateProfile creates a seccomp profile. func CreateProfile(s *state.State, c Instance) error { /* Unlike apparmor, there is no way to "cache" profiles, and profiles * are automatically unloaded when a task dies. Thus, we don't need to * unload them when a container stops, and we don't have to worry about * the mtime on the file for any compiler purpose, so let's just write * out the profile. */ if !InstanceNeedsPolicy(c) { return nil } profile, err := seccompGetPolicyContent(s, c) if err != nil { return err } err = os.MkdirAll(seccompPath, 0o700) if err != nil { return err } return os.WriteFile(ProfilePath(c), []byte(profile), 0o600) } // DeleteProfile removes a seccomp profile. func DeleteProfile(c Instance) { /* similar to AppArmor, if we've never started this container, the * delete can fail and that's ok. */ _ = os.Remove(ProfilePath(c)) } // Server defines a seccomp server. type Server struct { s *state.State path string l net.Listener } // Iovec defines an iovec to move data between kernel and userspace. type Iovec struct { ucred *unix.Ucred memFd int procFd int notifyFd int msg *C.struct_seccomp_notify_proxy_msg req *C.struct_seccomp_notif resp *C.struct_seccomp_notif_resp cookie *C.char iov *C.struct_iovec } // NewSeccompIovec creates a new seccomp iovec. func NewSeccompIovec(ucred *unix.Ucred) *Iovec { msgPtr := C.malloc(C.sizeof_struct_seccomp_notify_proxy_msg) msg := (*C.struct_seccomp_notify_proxy_msg)(msgPtr) C.memset(msgPtr, 0, C.sizeof_struct_seccomp_notify_proxy_msg) regPtr := C.malloc(C.sizeof_struct_seccomp_notif) req := (*C.struct_seccomp_notif)(regPtr) C.memset(regPtr, 0, C.sizeof_struct_seccomp_notif) respPtr := C.malloc(C.sizeof_struct_seccomp_notif_resp) resp := (*C.struct_seccomp_notif_resp)(respPtr) C.memset(respPtr, 0, C.sizeof_struct_seccomp_notif_resp) cookiePtr := C.malloc(64 * C.sizeof_char) cookie := (*C.char)(cookiePtr) C.memset(cookiePtr, 0, 64*C.sizeof_char) iovUnsafePtr := C.malloc(4 * C.sizeof_struct_iovec) iov := (*C.struct_iovec)(iovUnsafePtr) C.memset(iovUnsafePtr, 0, 4*C.sizeof_struct_iovec) C.prepare_seccomp_iovec(iov, msg, req, resp, cookie) return &Iovec{ memFd: -1, procFd: -1, notifyFd: -1, msg: msg, req: req, resp: resp, cookie: cookie, iov: iov, ucred: ucred, } } // PutSeccompIovec puts a seccomp iovec. func (siov *Iovec) PutSeccompIovec() { if siov.memFd >= 0 { _ = unix.Close(siov.memFd) } if siov.procFd >= 0 { _ = unix.Close(siov.procFd) } if siov.notifyFd >= 0 { _ = unix.Close(siov.notifyFd) } C.free(unsafe.Pointer(siov.msg)) C.free(unsafe.Pointer(siov.req)) C.free(unsafe.Pointer(siov.resp)) C.free(unsafe.Pointer(siov.cookie)) C.free(unsafe.Pointer(siov.iov)) } // ReceiveSeccompIovec receives a seccomp iovec. func (siov *Iovec) ReceiveSeccompIovec(fd int) (uint64, error) { bytes, fds, err := netutils.AbstractUnixReceiveFdData(fd, 3, netutils.UnixFdsAcceptLess, unsafe.Pointer(siov.iov), 4) if err != nil { return 0, err } siov.procFd = int(fds[0]) siov.memFd = int(fds[1]) siov.notifyFd = int(fds[2]) logger.Debugf("Syscall handler received fds %d(/proc/), %d(/proc//mem), and %d([seccomp notify])", siov.procFd, siov.memFd, siov.notifyFd) return bytes, nil } // IsValidSeccompIovec checks whether a seccomp iovec is valid. func (siov *Iovec) IsValidSeccompIovec(size uint64) bool { if size < uint64(C.SECCOMP_MSG_SIZE_MIN) { logger.Warnf("Disconnected from seccomp socket after incomplete receive") return false } if siov.msg.__reserved != 0 { logger.Warnf("Disconnected from seccomp socket after client sent non-zero reserved field: pid=%v", siov.ucred.Pid) return false } if siov.msg.sizes.seccomp_notif != C.expected_sizes.seccomp_notif { logger.Warnf("Disconnected from seccomp socket since client uses different seccomp_notif sizes: %d != %d, pid=%v", siov.msg.sizes.seccomp_notif, C.expected_sizes.seccomp_notif, siov.ucred.Pid) return false } if siov.msg.sizes.seccomp_notif_resp != C.expected_sizes.seccomp_notif_resp { logger.Warnf("Disconnected from seccomp socket since client uses different seccomp_notif_resp sizes: %d != %d, pid=%v", siov.msg.sizes.seccomp_notif_resp, C.expected_sizes.seccomp_notif_resp, siov.ucred.Pid) return false } if siov.msg.sizes.seccomp_data != C.expected_sizes.seccomp_data { logger.Warnf("Disconnected from seccomp socket since client uses different seccomp_data sizes: %d != %d, pid=%v", siov.msg.sizes.seccomp_data, C.expected_sizes.seccomp_data, siov.ucred.Pid) return false } return true } // SendSeccompIovec sends seccomp iovec. func (siov *Iovec) SendSeccompIovec(fd int, errno int, flags uint32) error { C.seccomp_notify_update_response(siov.resp, C.int(errno), C.uint32_t(flags)) msghdr := C.struct_msghdr{} msghdr.msg_iov = siov.iov msghdr.msg_iovlen = 4 - 1 // without cookie retry: bytes, err := C.sendmsg(C.int(fd), &msghdr, C.MSG_NOSIGNAL) if bytes < 0 { if err == unix.EINTR { logger.Debugf("Caught EINTR, retrying...") goto retry } logger.Debugf("Disconnected from seccomp socket after failed write for process %v: %s", siov.ucred.Pid, err) return fmt.Errorf("Failed to send response to seccomp client %v", siov.ucred.Pid) } if uint64(bytes) != uint64(C.SECCOMP_MSG_SIZE_MIN) { logger.Debugf("Disconnected from seccomp socket after short write: pid=%v", siov.ucred.Pid) return fmt.Errorf("Failed to send full response to seccomp client %v", siov.ucred.Pid) } logger.Debugf("Send seccomp notification for id(%d)", siov.resp.id) return nil } // NewSeccompServer creates a new seccomp server. func NewSeccompServer(s *state.State, path string, findPID func(pid int32, s *state.State) (Instance, error)) (*Server, error) { ret := C.seccomp_notify_get_sizes(&C.expected_sizes) if ret < 0 { return nil, errors.New("Failed to query kernel for seccomp notifier sizes") } // Cleanup existing sockets if util.PathExists(path) { err := os.Remove(path) if err != nil { return nil, err } } // Bind new socket l, err := net.Listen("unixpacket", path) if err != nil { return nil, err } // Restrict access err = os.Chmod(path, 0o700) if err != nil { return nil, err } // Start the server server := Server{ s: s, path: path, l: l, } go func() { for { c, err := l.Accept() if err != nil { return } go func() { ucred, err := linux.GetUcred(c.(*net.UnixConn)) if err != nil { logger.Errorf("Unable to get ucred from seccomp socket client: %v", err) return } logger.Debugf("Connected to seccomp socket: pid=%v", ucred.Pid) unixFile, err := c.(*net.UnixConn).File() if err != nil { logger.Debugf("Failed to turn unix socket client into file") return } for { siov := NewSeccompIovec(ucred) bytes, err := siov.ReceiveSeccompIovec(int(unixFile.Fd())) if err != nil { logger.Debugf("Disconnected from seccomp socket after failed receive: pid=%v, err=%s", ucred.Pid, err) _ = c.Close() return } if siov.IsValidSeccompIovec(bytes) { go func() { _ = server.HandleValid(int(unixFile.Fd()), siov, findPID) }() } else { go server.HandleInvalid(int(unixFile.Fd()), siov) } } }() } }() return &server, nil } // TaskIDs returns the task IDs for a process. func TaskIDs(pid int) (int64, int64, int64, int64, error) { status, err := os.ReadFile(fmt.Sprintf("/proc/%d/status", pid)) if err != nil { return -1, -1, -1, -1, err } reUID, err := regexp.Compile(`^Uid:\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)`) if err != nil { return -1, -1, -1, -1, err } reGID, err := regexp.Compile(`^Gid:\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)`) if err != nil { return -1, -1, -1, -1, err } var UID int64 = -1 var GID int64 = -1 var fsUID int64 = -1 var fsGID int64 = -1 UIDFound := false GIDFound := false for _, line := range strings.Split(string(status), "\n") { if UIDFound && GIDFound { break } if !UIDFound { m := reUID.FindStringSubmatch(line) if len(m) > 2 { // effective uid result, err := strconv.ParseInt(m[2], 10, 64) if err != nil { return -1, -1, -1, -1, err } UID = result UIDFound = true } if len(m) > 4 { // fsuid result, err := strconv.ParseInt(m[4], 10, 64) if err != nil { return -1, -1, -1, -1, err } fsUID = result } continue } if !GIDFound { m := reGID.FindStringSubmatch(line) if len(m) > 2 { // effective gid result, err := strconv.ParseInt(m[2], 10, 64) if err != nil { return -1, -1, -1, -1, err } GID = result GIDFound = true } if len(m) > 4 { // fsgid result, err := strconv.ParseInt(m[4], 10, 64) if err != nil { return -1, -1, -1, -1, err } fsGID = result } continue } } return UID, GID, fsUID, fsGID, nil } // FindTGID returns the task group leader ID from /proc/ fd. func FindTGID(procFd int) (int, error) { var statusFile *os.File fd, err := unix.Openat(procFd, "status", unix.O_RDONLY|unix.O_CLOEXEC, 0) if err != nil { return -1, err } statusFile = os.NewFile(uintptr(fd), "/proc//status") status, err := io.ReadAll(statusFile) _ = statusFile.Close() if err != nil { return -1, err } reTGID, err := regexp.Compile(`^Tgid:\s+([0-9]+)`) if err != nil { return -1, err } for _, line := range strings.Split(string(status), "\n") { m := reTGID.FindStringSubmatch(line) if len(m) > 1 { result, err := strconv.ParseUint(m[1], 10, 32) if err != nil { return -1, err } return int(result), nil } } return -1, nil } // CallForkmknod executes fork mknod. func CallForkmknod(c Instance, dev deviceConfig.Device, requestPID int, s *state.State) int { uid, gid, fsuid, fsgid, err := TaskIDs(requestPID) if err != nil { return int(-C.EPERM) } pidFdNr, pidFd, err := MakePidFd(requestPID) if err != nil { return int(-C.EPERM) } defer func() { _ = pidFd.Close() }() _, stderr, err := subprocess.RunCommandSplit( context.TODO(), nil, []*os.File{pidFd}, localUtil.GetExecPath(), "forksyscall", "mknod", dev["pid"], fmt.Sprintf("%d", pidFdNr), dev["path"], dev["mode_t"], dev["dev_t"], fmt.Sprintf("%d", uid), fmt.Sprintf("%d", gid), fmt.Sprintf("%d", fsuid), fmt.Sprintf("%d", fsgid)) if err != nil { errno, err := strconv.Atoi(stderr) if err != nil || errno == C.ENOANO { return int(-C.EPERM) } return -errno } return 0 } // HandleInvalid sends a placeholder message to LXC. LXC will notice the short write // and send a default message to the kernel thereby avoiding a 30s block. func (srv *Server) HandleInvalid(fd int, siov *Iovec) { msghdr := C.struct_msghdr{} C.sendmsg(C.int(fd), &msghdr, C.MSG_NOSIGNAL) siov.PutSeccompIovec() } // MknodArgs arguments for mknod. type MknodArgs struct { cMode C.mode_t cDev C.dev_t cPid C.pid_t path string } func (srv *Server) doDeviceSyscall(c Instance, args *MknodArgs, siov *Iovec) int { dev := deviceConfig.Device{} dev["type"] = "unix-char" dev["mode"] = fmt.Sprintf("%#o", args.cMode) dev["major"] = fmt.Sprintf("%d", unix.Major(uint64(args.cDev))) dev["minor"] = fmt.Sprintf("%d", unix.Minor(uint64(args.cDev))) dev["pid"] = fmt.Sprintf("%d", args.cPid) dev["path"] = args.path dev["mode_t"] = fmt.Sprintf("%d", args.cMode) dev["dev_t"] = fmt.Sprintf("%d", args.cDev) // /dev is typically mounted by real root, so even though we can // mknod stuff in it, access will ultimately be denied. // // Instead go straight to the fallback mechanism of using a bind-mount. if !strings.HasPrefix(dev["path"], "/dev") { errno := CallForkmknod(c, dev, int(args.cPid), srv.s) if errno != int(-C.ENOMEDIUM) { return errno } } err := c.InsertSeccompUnixDevice(fmt.Sprintf("forkmknod.unix.%d", int(args.cPid)), dev, int(args.cPid)) if err != nil { return int(-C.EPERM) } return 0 } // HandleMknodSyscall handles a mknod syscall. func (srv *Server) HandleMknodSyscall(c Instance, siov *Iovec) int { ctx := logger.Ctx{ "container": c.Name(), "project": c.Project().Name, "syscall_number": siov.req.data.nr, "audit_architecture": siov.req.data.arch, "seccomp_notify_id": siov.req.id, "seccomp_notify_flags": siov.req.flags, "seccomp_notify_pid": siov.req.pid, "seccomp_notify_fd": siov.notifyFd, "seccomp_notify_mem_fd": siov.memFd, } defer logger.Debug("Handling mknod syscall", ctx) if C.device_allowed(C.dev_t(siov.req.data.args[2]), C.mode_t(siov.req.data.args[1])) < 0 { ctx["err"] = "Device not allowed" ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } cPathBuf := [unix.PathMax]C.char{} _, err := C.pread(C.int(siov.memFd), unsafe.Pointer(&cPathBuf[0]), C.size_t(unix.PathMax), C.off_t(siov.req.data.args[0])) if err != nil { ctx["err"] = fmt.Sprintf("Failed to read memory for mknod syscall: %s", err) ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } args := MknodArgs{ cMode: C.mode_t(siov.req.data.args[1]), cDev: C.dev_t(siov.req.data.args[2]), cPid: C.pid_t(siov.req.pid), path: C.GoString(&cPathBuf[0]), } ctx["syscall_args"] = &args return srv.doDeviceSyscall(c, &args, siov) } // HandleMknodatSyscall handles a mknodat syscall. func (srv *Server) HandleMknodatSyscall(c Instance, siov *Iovec) int { ctx := logger.Ctx{ "container": c.Name(), "project": c.Project().Name, "syscall_number": siov.req.data.nr, "audit_architecture": siov.req.data.arch, "seccomp_notify_id": siov.req.id, "seccomp_notify_flags": siov.req.flags, "seccomp_notify_pid": siov.req.pid, "seccomp_notify_fd": siov.notifyFd, "seccomp_notify_mem_fd": siov.memFd, } defer logger.Debug("Handling mknodat syscall", ctx) // Make sure to handle 64bit kernel, 32bit container/userspace, 64bit daemon. if int32(siov.req.data.args[0]) != int32(C.AT_FDCWD) { ctx["err"] = "Non AT_FDCWD mknodat calls are not allowed" logger.Debug("bla", ctx) ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } siov.resp.error = C.device_allowed(C.dev_t(siov.req.data.args[3]), C.mode_t(siov.req.data.args[2])) if siov.resp.error != 0 { ctx["err"] = "Device not allowed" ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } cPathBuf := [unix.PathMax]C.char{} _, err := C.pread(C.int(siov.memFd), unsafe.Pointer(&cPathBuf[0]), C.size_t(unix.PathMax), C.off_t(siov.req.data.args[1])) if err != nil { ctx["err"] = "Failed to read memory for mknodat syscall: %s" ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } args := MknodArgs{ cMode: C.mode_t(siov.req.data.args[2]), cDev: C.dev_t(siov.req.data.args[3]), cPid: C.pid_t(siov.req.pid), path: C.GoString(&cPathBuf[0]), } ctx["syscall_args"] = &args return srv.doDeviceSyscall(c, &args, siov) } // SetxattrArgs arguments for setxattr. type SetxattrArgs struct { nsuid int64 nsgid int64 nsfsuid int64 nsfsgid int64 size int pid int path string name string value []byte flags C.int } // HandleSetxattrSyscall handles setxattr syscalls. func (srv *Server) HandleSetxattrSyscall(c Instance, siov *Iovec) int { ctx := logger.Ctx{ "container": c.Name(), "project": c.Project().Name, "syscall_number": siov.req.data.nr, "audit_architecture": siov.req.data.arch, "seccomp_notify_id": siov.req.id, "seccomp_notify_flags": siov.req.flags, "seccomp_notify_pid": siov.req.pid, "seccomp_notify_fd": siov.notifyFd, "seccomp_notify_mem_fd": siov.memFd, } defer logger.Debug("Handling setxattr syscall", ctx) args := SetxattrArgs{} args.pid = int(siov.req.pid) pidFdNr, pidFd, err := MakePidFd(args.pid) if err != nil { ctx["err"] = fmt.Sprintf("Failed to open pidfd: %s", err) ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } defer func() { _ = pidFd.Close() }() uid, gid, fsuid, fsgid, err := TaskIDs(args.pid) if err != nil { ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } idmapset, err := c.CurrentIdmap() if err != nil { ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } args.nsuid, args.nsgid = idmapset.ShiftFromNS(uid, gid) args.nsfsuid, args.nsfsgid = idmapset.ShiftFromNS(fsuid, fsgid) // const char *path cBuf := [unix.PathMax]C.char{} _, err = C.pread(C.int(siov.memFd), unsafe.Pointer(&cBuf[0]), C.size_t(unix.PathMax), C.off_t(siov.req.data.args[0])) if err != nil { ctx["err"] = fmt.Sprintf("Failed to read memory for setxattr syscall: %s", err) ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } args.path = C.GoString(&cBuf[0]) // const char *name _, err = C.pread(C.int(siov.memFd), unsafe.Pointer(&cBuf[0]), C.size_t(unix.PathMax), C.off_t(siov.req.data.args[1])) if err != nil { ctx["err"] = fmt.Sprintf("Failed to read memory for setxattr syscall: %s", err) ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } args.name = C.GoString(&cBuf[0]) // size_t size args.size = int(siov.req.data.args[3]) // int flags args.flags = C.int(siov.req.data.args[4]) buf := make([]byte, args.size) _, err = C.pread(C.int(siov.memFd), unsafe.Pointer(&buf[0]), C.size_t(args.size), C.off_t(siov.req.data.args[2])) if err != nil { ctx["err"] = fmt.Sprintf("Failed to read memory for setxattr syscall: %s", err) ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } args.value = buf whiteout := 0 if string(args.name) == "trusted.overlay.opaque" && string(args.value) == "y" { whiteout = 1 } else { ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } _, stderr, err := subprocess.RunCommandSplit( context.TODO(), nil, []*os.File{pidFd}, localUtil.GetExecPath(), "forksyscall", "setxattr", fmt.Sprintf("%d", args.pid), fmt.Sprintf("%d", pidFdNr), fmt.Sprintf("%d", args.nsuid), fmt.Sprintf("%d", args.nsgid), fmt.Sprintf("%d", args.nsfsuid), fmt.Sprintf("%d", args.nsfsgid), args.name, args.path, fmt.Sprintf("%d", args.flags), fmt.Sprintf("%d", whiteout), fmt.Sprintf("%d", args.size), string(args.value)) if err != nil { errno, err := strconv.Atoi(stderr) if err != nil || errno == C.ENOANO { return int(-C.EPERM) } return -errno } return 0 } // SchedSetschedulerArgs arguments for setxattr. type SchedSetschedulerArgs struct { switchPidns int pidCaller int pidTarget int policy C.int schedPriority C.int nsuid int64 nsgid int64 } // HandleSchedSetschedulerSyscall handles sched_setscheduler syscalls. func (srv *Server) HandleSchedSetschedulerSyscall(c Instance, siov *Iovec) int { ctx := logger.Ctx{ "container": c.Name(), "project": c.Project().Name, "syscall_number": siov.req.data.nr, "audit_architecture": siov.req.data.arch, "seccomp_notify_id": siov.req.id, "seccomp_notify_flags": siov.req.flags, "seccomp_notify_pid": siov.req.pid, "seccomp_notify_fd": siov.notifyFd, "seccomp_notify_mem_fd": siov.memFd, } defer logger.Debug("Handling sched_setscheduler syscall", ctx) args := SchedSetschedulerArgs{} args.pidCaller = int(siov.req.pid) pidFdNr, pidFd, err := MakePidFd(args.pidCaller) if err != nil { ctx["err"] = fmt.Sprintf("Failed to open pidfd: %s", err) ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } defer func() { _ = pidFd.Close() }() uid, gid, _, _, err := TaskIDs(args.pidCaller) if err != nil { ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } idmapset, err := c.CurrentIdmap() if err != nil { ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } // Only care about userns root for now. args.nsuid, args.nsgid = idmapset.ShiftFromNS(uid, gid) if args.nsuid != 0 || args.nsgid != 0 { ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } // The target pid is only valid in the container's pid namespace as // we're taking it from the raw system call arguments. args.pidTarget = int(siov.req.data.args[0]) if args.pidTarget < 0 { ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } // If the caller passed zero they want to change their own attributes. if args.pidTarget == 0 { // This pid is relative to our pid namespace so we need to // inform forksyscall to not switch pid namespaces when // emulating the system call. args.pidTarget = args.pidCaller args.switchPidns = 0 } else { // The pid is relative to the container's pid namespace. args.switchPidns = 1 } // error out if policy < 0 if args.policy < 0 { ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } // int policy args.policy = C.int(siov.req.data.args[1]) schedParamArgs := C.struct_sched_param{} _, err = C.pread(C.int(siov.memFd), unsafe.Pointer(&schedParamArgs), C.INCUS_SCHED_PARAM_SIZE, C.off_t(siov.req.data.args[2])) if err != nil { ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } args.schedPriority = schedParamArgs.sched_priority _, stderr, err := subprocess.RunCommandSplit( context.TODO(), nil, []*os.File{pidFd}, localUtil.GetExecPath(), "forksyscall", "sched_setscheduler", fmt.Sprintf("%d", args.pidCaller), fmt.Sprintf("%d", pidFdNr), fmt.Sprintf("%d", args.switchPidns), fmt.Sprintf("%d", args.pidTarget), fmt.Sprintf("%d", args.policy), fmt.Sprintf("%d", args.schedPriority), ) if err != nil { errno, err := strconv.Atoi(stderr) if err != nil || errno == C.ENOANO { return int(-C.EPERM) } return -errno } return 0 } // HandleSysinfoSyscall handles sysinfo syscalls. func (srv *Server) HandleSysinfoSyscall(c Instance, siov *Iovec) int { l := logger.AddContext(logger.Ctx{ "container": c.Name(), "project": c.Project().Name, "syscall_number": siov.req.data.nr, "audit_architecture": siov.req.data.arch, "seccomp_notify_id": siov.req.id, "seccomp_notify_flags": siov.req.flags, "seccomp_notify_pid": siov.req.pid, "seccomp_notify_fd": siov.notifyFd, "seccomp_notify_mem_fd": siov.memFd, }) defer l.Debug("Handling sysinfo syscall") // Pre-fill sysinfo struct with metrics from host system. info := unix.Sysinfo_t{} err := unix.Sysinfo(&info) if err != nil { l.Warn("Failed getting sysinfo", logger.Ctx{"err": err}) C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } instMetrics := Sysinfo{} // Architecture independent place to hold instance metrics. // Handle i386 on x86_64. instMetrics.Unit = 1 if c.Architecture() == osarch.ARCH_64BIT_INTEL_X86 && siov.req.data.arch == C.AUDIT_ARCH_I386 { instMetrics.Unit = 4096 } cg, err := cgroup.NewFileReadWriter(int(siov.msg.init_pid)) if err != nil { l.Warn("Failed loading cgroup", logger.Ctx{"err": err, "pid": siov.msg.init_pid}) C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } // Get instance uptime. pidStat, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", siov.msg.init_pid)) if err != nil { l.Warn("Failed getting init process info", logger.Ctx{"err": err, "pid": siov.msg.init_pid}) C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } fields := strings.Fields(string(pidStat)) tickValue, err := strconv.ParseInt(fields[21], 10, 64) if err != nil { l.Warn("Failed parsing init process info", logger.Ctx{"err": err, "pid": siov.msg.init_pid}) C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } age := float64(tickValue / 100) if age > 0 { instMetrics.Uptime = int64(time.Since(srv.s.OS.BootTime).Seconds() - age) } // Get instance process count. pids, err := cg.GetProcessesUsage() if err != nil { l.Warn("Failed getting process count", logger.Ctx{"err": err}) C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } instMetrics.Procs = uint16(pids) // Get instance memory stats. memStats, err := cg.GetMemoryStats() if err != nil { l.Warn("Failed getting memory stats", logger.Ctx{"err": err}) C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } for k, v := range memStats { switch k { case "shmem": instMetrics.Sharedram = v case "cache": instMetrics.Bufferram = v } } // Get instance memory limit. memoryLimit, err := cg.GetEffectiveMemoryLimit() if err != nil { l.Warn("Failed getting effective memory limit", logger.Ctx{"err": err}) C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } // Get instance memory usage. memoryUsage, err := cg.GetMemoryUsage() if err != nil { l.Warn("Failed to get memory usage", logger.Ctx{"err": err}) C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } instMetrics.Totalram = uint64(memoryLimit) instMetrics.Freeram = instMetrics.Totalram - uint64(memoryUsage) - instMetrics.Bufferram // Get instance swap info. if cgroup.Supports(cgroup.Memory) { swapLimit, err := cg.GetMemorySwapLimit() if err != nil { l.Warn("Failed getting swap limit", logger.Ctx{"err": err}) C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } swapUsage, err := cg.GetMemorySwapUsage() if err != nil { l.Warn("Failed getting swap usage", logger.Ctx{"err": err}) C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } instMetrics.Totalswap = uint64(swapLimit) instMetrics.Freeswap = instMetrics.Totalswap - uint64(swapUsage) } // Write instance metrics to native sysinfo struct. var b []byte if c.Architecture() == osarch.ARCH_64BIT_INTEL_X86 && siov.req.data.arch == C.AUDIT_ARCH_I386 { ret := instMetrics.ToNative32(info) const sz = int(unsafe.Sizeof(ret)) b = (*(*[sz]byte)(unsafe.Pointer(&ret)))[:] } else { instMetrics.ToNative(&info) const sz = int(unsafe.Sizeof(info)) b = (*(*[sz]byte)(unsafe.Pointer(&info)))[:] } // Write sysinfo response into buffer. _, err = unix.Pwrite(siov.memFd, b, int64(siov.req.data.args[0])) if err != nil { l.Warn("Failed writing sysinfo", logger.Ctx{"err": err}) C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } return 0 } // MountArgs arguments for mount. type MountArgs struct { source string target string fstype string flags int data string pid int idmapType idmap.StorageType uid int64 gid int64 fsuid int64 fsgid int64 nsuid int64 nsgid int64 nsfsuid int64 nsfsgid int64 } const knownFlags C.ulong = C.MS_BIND | C.MS_LAZYTIME | C.MS_MANDLOCK | C.MS_NOATIME | C.MS_NODEV | C.MS_NODIRATIME | C.MS_NOEXEC | C.MS_NOSUID | C.MS_REMOUNT | C.MS_RDONLY | C.MS_STRICTATIME | C.MS_SYNCHRONOUS | C.MS_BIND const knownFlagsRecursive C.ulong = knownFlags | C.MS_REC var mountFlagsToOptMap = map[C.ulong]string{ C.MS_BIND: "bind", C.ulong(0): "defaults", C.MS_LAZYTIME: "lazytime", C.MS_MANDLOCK: "mand", C.MS_NOATIME: "noatime", C.MS_NODEV: "nodev", C.MS_NODIRATIME: "nodiratime", C.MS_NOEXEC: "noexec", C.MS_NOSUID: "nosuid", C.MS_REMOUNT: "remount", C.MS_RDONLY: "ro", C.MS_STRICTATIME: "strictatime", C.MS_SYNCHRONOUS: "sync", C.MS_REC | C.MS_BIND: "rbind", } func mountFlagsToOpts(flags C.ulong) string { var currentBit C.ulong opts := "" var msRec C.ulong = (flags & C.MS_REC) flags = (flags &^ C.MS_REC) for currentBit < (4*8 - 1) { var flag C.ulong = (1 << currentBit) currentBit++ if (flags & flag) == 0 { continue } if (flag == C.MS_BIND) && msRec == C.MS_REC { flag |= msRec } optOrArg := mountFlagsToOptMap[flag] if optOrArg == "" { continue } if opts == "" { opts = optOrArg } else { opts = fmt.Sprintf("%s,%s", opts, optOrArg) } } return opts } // mountHandleHugetlbfsArgs adds user namespace root uid and gid to the // hugetlbfs mount options to make it usable in unprivileged containers. func (srv *Server) mountHandleHugetlbfsArgs(c Instance, args *MountArgs, nsuid int64, nsgid int64) error { if args.fstype != "hugetlbfs" { return nil } if args.data == "" { args.data = fmt.Sprintf("uid=%d,gid=%d", nsuid, nsgid) return nil } uidOpt := int64(-1) gidOpt := int64(-1) idmapset, err := c.CurrentIdmap() if err != nil { return err } optStrings := strings.Split(args.data, ",") for i, optString := range optStrings { if strings.HasPrefix(optString, "uid=") { uidFields := strings.Split(optString, "=") if len(uidFields) > 1 { n, err := strconv.ParseInt(uidFields[1], 10, 64) if err != nil { // If the user specified garbage, let the kernel tell em what's what. return nil } uidOpt, _ = idmapset.ShiftIntoNS(n, 0) if uidOpt < 0 { // If the user specified garbage, let the kernel tell em what's what. return nil } optStrings[i] = fmt.Sprintf("uid=%d", uidOpt) } } else if strings.HasPrefix(optString, "gid=") { gidFields := strings.Split(optString, "=") if len(gidFields) > 1 { n, err := strconv.ParseInt(gidFields[1], 10, 64) if err != nil { // If the user specified garbage, let the kernel tell em what's what. return nil } gidOpt, _ = idmapset.ShiftIntoNS(n, 0) if gidOpt < 0 { // If the user specified garbage, let the kernel tell em what's what. return nil } optStrings[i] = fmt.Sprintf("gid=%d", gidOpt) } } } if uidOpt == -1 { optStrings = append(optStrings, fmt.Sprintf("uid=%d", nsuid)) } if gidOpt == -1 { optStrings = append(optStrings, fmt.Sprintf("gid=%d", nsgid)) } args.data = strings.Join(optStrings, ",") args.idmapType = idmap.StorageTypeNone return nil } // HandleMountSyscall handles mount syscalls. func (srv *Server) HandleMountSyscall(c Instance, siov *Iovec) int { ctx := logger.Ctx{ "container": c.Name(), "project": c.Project().Name, "syscall_number": siov.req.data.nr, "audit_architecture": siov.req.data.arch, "seccomp_notify_id": siov.req.id, "seccomp_notify_flags": siov.req.flags, "seccomp_notify_pid": siov.req.pid, "seccomp_notify_fd": siov.notifyFd, "seccomp_notify_mem_fd": siov.memFd, } defer logger.Debug("Handling mount syscall", ctx) args := MountArgs{ pid: int(siov.req.pid), } pidFdNr, pidFd, err := MakePidFd(args.pid) if err != nil { ctx["err"] = fmt.Sprintf("Failed to open pidfd: %s", err) ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } defer func() { _ = pidFd.Close() }() mntSource := [unix.PathMax]C.char{} mntTarget := [unix.PathMax]C.char{} mntFs := [unix.PathMax]C.char{} mntData := [unix.PathMax]C.char{} // const char *source if siov.req.data.args[0] != 0 { _, err := C.pread(C.int(siov.memFd), unsafe.Pointer(&mntSource[0]), C.size_t(unix.PathMax), C.off_t(siov.req.data.args[0])) if err != nil { ctx["err"] = fmt.Sprintf("Failed to read source path for of mount syscall: %s", err) ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } } args.source = C.GoString(&mntSource[0]) ctx["source"] = args.source // const char *target if siov.req.data.args[1] != 0 { _, err := C.pread(C.int(siov.memFd), unsafe.Pointer(&mntTarget[0]), C.size_t(unix.PathMax), C.off_t(siov.req.data.args[1])) if err != nil { ctx["err"] = fmt.Sprintf("Failed to read target path for of mount syscall: %s", err) ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } } args.target = C.GoString(&mntTarget[0]) ctx["target"] = args.target // const char *filesystemtype if siov.req.data.args[1] != 0 { _, err := C.pread(C.int(siov.memFd), unsafe.Pointer(&mntFs[0]), C.size_t(unix.PathMax), C.off_t(siov.req.data.args[2])) if err != nil { ctx["err"] = fmt.Sprintf("Failed to read fstype for of mount syscall: %s", err) ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } } args.fstype = C.GoString(&mntFs[0]) ctx["fstype"] = args.fstype // idmap shift fullSrcPath := filepath.Join(fmt.Sprintf("/proc/%d/root/", args.pid), args.source) if util.PathExists(fullSrcPath) { args.idmapType = srv.MountSyscallShift(c, fullSrcPath, args.fstype) } else { args.idmapType = srv.MountSyscallShift(c, args.source, args.fstype) } // unsigned long mountflags args.flags = int(siov.req.data.args[3]) // const void *data if siov.req.data.args[4] != 0 { _, err := C.pread(C.int(siov.memFd), unsafe.Pointer(&mntData[0]), C.size_t(unix.PathMax), C.off_t(siov.req.data.args[4])) if err != nil { ctx["err"] = fmt.Sprintf("Failed to read mount data for of mount syscall: %s", err) ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } } args.data = C.GoString(&mntData[0]) ctx["data"] = args.data err = linux.PidfdSendSignal(int(pidFd.Fd()), 0, 0) if err != nil { ctx["err"] = fmt.Sprintf("Failed to send signal to target process for of mount syscall: %s", err) ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } ok, fuseBinary := srv.MountSyscallValid(c, &args) if !ok { ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } idmapset, err := c.CurrentIdmap() if err != nil { ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } args.uid, args.gid, args.fsuid, args.fsgid, err = TaskIDs(args.pid) if err != nil { ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } ctx["host_uid"] = args.uid ctx["host_gid"] = args.gid ctx["host_fsuid"] = args.fsuid ctx["host_fsgid"] = args.fsgid args.nsuid, args.nsgid = idmapset.ShiftFromNS(args.uid, args.gid) args.nsfsuid, args.nsfsgid = idmapset.ShiftFromNS(args.fsuid, args.fsgid) ctx["ns_uid"] = args.nsuid ctx["ns_gid"] = args.nsgid ctx["ns_fsuid"] = args.nsfsuid ctx["ns_fsgid"] = args.nsfsgid err = srv.mountHandleHugetlbfsArgs(c, &args, args.uid, args.gid) if err != nil { ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } if fuseBinary != "" { // Record ignored flags for debugging purposes flags := C.ulong(args.flags) ctx["fuse_ignored_flags"] = fmt.Sprintf("%x", (flags &^ (knownFlagsRecursive | C.MS_MGC_MSK))) addOpts := mountFlagsToOpts(flags) fuseSource := fmt.Sprintf("%s#%s", fuseBinary, args.source) fuseOpts := "" if args.data != "" && addOpts != "" { fuseOpts = fmt.Sprintf("%s,%s", args.data, addOpts) } else if args.data != "" { fuseOpts = args.data } else if addOpts != "" { fuseOpts = addOpts } ctx["fuse_source"] = fuseSource ctx["fuse_target"] = args.target ctx["fuse_opts"] = fuseOpts _, _, err = subprocess.RunCommandSplit( context.TODO(), nil, []*os.File{pidFd}, localUtil.GetExecPath(), "forksyscall", "mount", fmt.Sprintf("%d", args.pid), fmt.Sprintf("%d", pidFdNr), fmt.Sprintf("%d", 1), fmt.Sprintf("%d", args.uid), fmt.Sprintf("%d", args.gid), fmt.Sprintf("%d", args.fsuid), fmt.Sprintf("%d", args.fsgid), fuseSource, args.target, fuseOpts) } else { _, _, err = subprocess.RunCommandSplit( context.TODO(), nil, []*os.File{pidFd}, localUtil.GetExecPath(), "forksyscall", "mount", fmt.Sprintf("%d", args.pid), fmt.Sprintf("%d", pidFdNr), fmt.Sprintf("%d", 0), args.source, args.target, args.fstype, fmt.Sprintf("%d", args.flags), string(args.idmapType), fmt.Sprintf("%d", args.uid), fmt.Sprintf("%d", args.gid), fmt.Sprintf("%d", args.fsuid), fmt.Sprintf("%d", args.fsgid), fmt.Sprintf("%d", args.nsuid), fmt.Sprintf("%d", args.nsgid), fmt.Sprintf("%d", args.nsfsuid), fmt.Sprintf("%d", args.nsfsgid), args.data) } if err != nil { ctx["syscall_continue"] = "true" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } return 0 } // HandleBpfSyscall handles mount syscalls. func (srv *Server) HandleBpfSyscall(c Instance, siov *Iovec) int { ctx := logger.Ctx{ "container": c.Name(), "project": c.Project().Name, "syscall_number": siov.req.data.nr, "audit_architecture": siov.req.data.arch, "seccomp_notify_id": siov.req.id, "seccomp_notify_flags": siov.req.flags, "seccomp_notify_pid": siov.req.pid, "seccomp_notify_fd": siov.notifyFd, "seccomp_notify_mem_fd": siov.memFd, } defer logger.Debug("Handling bpf syscall", ctx) var bpfCmd, bpfProgType, bpfAttachType, tgid C.int var flags C.uint if util.IsFalseOrEmpty(c.ExpandedConfig()["security.syscalls.intercept.bpf.devices"]) { ctx["syscall_continue"] = "true" ctx["syscall_handler_reason"] = "No bpf policy specified" C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } flags |= C.PIDFD_THREAD tgid = -1 // Locking to a thread shouldn't be necessary but it still makes me // queezy that Go could just wander off to somewhere. runtime.LockOSThread() ret := C.handle_bpf_syscall( C.pid_t(siov.req.pid), C.int(siov.notifyFd), C.int(siov.memFd), C.int(tgid), siov.msg, siov.req, siov.resp, &bpfCmd, &bpfProgType, &bpfAttachType, flags) runtime.UnlockOSThread() ctx["bpf_cmd"] = fmt.Sprintf("%d", bpfCmd) ctx["bpf_prog_type"] = fmt.Sprintf("%d", bpfProgType) ctx["bpf_attach_type"] = fmt.Sprintf("%d", bpfAttachType) if ret < 0 { ctx["syscall_continue"] = "true" ctx["syscall_handler_error"] = fmt.Sprintf("%s - Failed to handle bpf syscall", unix.Errno(-ret)) C.seccomp_notify_update_response(siov.resp, 0, C.uint32_t(seccompUserNotifFlagContinue)) return 0 } return 0 } func (srv *Server) handleSyscall(c Instance, siov *Iovec) int { switch int(C.seccomp_notify_get_syscall(siov.req, siov.resp)) { case incusSeccompNotifyMknod: return srv.HandleMknodSyscall(c, siov) case incusSeccompNotifyMknodat: return srv.HandleMknodatSyscall(c, siov) case incusSeccompNotifySetxattr: return srv.HandleSetxattrSyscall(c, siov) case incusSeccompNotifyMount: return srv.HandleMountSyscall(c, siov) case incusSeccompNotifyBpf: return srv.HandleBpfSyscall(c, siov) case incusSeccompNotifySchedSetscheduler: return srv.HandleSchedSetschedulerSyscall(c, siov) case incusSeccompNotifySysinfo: return srv.HandleSysinfoSyscall(c, siov) } return int(-C.EINVAL) } const seccompUserNotifFlagContinue uint32 = 0x00000001 // HandleValid handles a valid seccomp notifier message. func (srv *Server) HandleValid(fd int, siov *Iovec, findPID func(pid int32, s *state.State) (Instance, error)) error { defer siov.PutSeccompIovec() c, err := findPID(int32(siov.msg.monitor_pid), srv.s) if err != nil { _ = siov.SendSeccompIovec(fd, 0, seccompUserNotifFlagContinue) logger.Errorf("Failed to find container for monitor %d", siov.msg.monitor_pid) return err } errno := srv.handleSyscall(c, siov) err = siov.SendSeccompIovec(fd, errno, 0) if err != nil { return err } return nil } // Stop stops a seccomp server. func (srv *Server) Stop() error { _ = os.Remove(srv.path) return srv.l.Close() } // MountSyscallFilter creates a mount syscall filter from the config. func MountSyscallFilter(config map[string]string) []string { fs := []string{} if util.IsFalseOrEmpty(config["security.syscalls.intercept.mount"]) { return fs } fsAllowed := strings.Split(config["security.syscalls.intercept.mount.allowed"], ",") if len(fsAllowed) > 0 && fsAllowed[0] != "" { fs = append(fs, fsAllowed...) } return fs } // SyscallInterceptMountFilter creates a new mount syscall interception filter. func SyscallInterceptMountFilter(config map[string]string) (map[string]string, error) { if util.IsFalseOrEmpty(config["security.syscalls.intercept.mount"]) { return map[string]string{}, nil } fsMap := map[string]string{} fsFused := strings.Split(config["security.syscalls.intercept.mount.fuse"], ",") if len(fsFused) > 0 && fsFused[0] != "" { for _, ent := range fsFused { fsfuse := strings.Split(ent, "=") if len(fsfuse) != 2 { return map[string]string{}, fmt.Errorf("security.syscalls.intercept.mount.fuse is not of the form 'filesystem=fuse-binary': %s", ent) } // fsfuse[0] == filesystems that are ok to mount // fsfuse[1] == fuse binary to use to mount filesystemstype fsMap[fsfuse[0]] = fsfuse[1] } } fsAllowed := strings.Split(config["security.syscalls.intercept.mount.allowed"], ",") if len(fsAllowed) > 0 && fsAllowed[0] != "" { for _, allowedfs := range fsAllowed { if fsMap[allowedfs] != "" { return map[string]string{}, fmt.Errorf("Filesystem %s cannot appear in security.syscalls.intercept.mount.allowed and security.syscalls.intercept.mount.fuse", allowedfs) } fsMap[allowedfs] = "" } } return fsMap, nil } // MountSyscallValid checks whether this is a mount syscall we intercept. func (srv *Server) MountSyscallValid(c Instance, args *MountArgs) (bool, string) { fsMap, err := SyscallInterceptMountFilter(c.ExpandedConfig()) if err != nil { return false, "" } fuse, ok := fsMap[args.fstype] if ok { return true, fuse } return false, "" } // MountSyscallShift checks whether this mount syscall needs shifting. func (srv *Server) MountSyscallShift(c Instance, path string, fsType string) idmap.StorageType { if util.IsTrue(c.ExpandedConfig()["security.syscalls.intercept.mount.shift"]) { diskIdmap, err := c.DiskIdmap() if err != nil { return idmap.StorageTypeNone } if diskIdmap == nil { return c.IdmappedStorage(path, fsType) } } return idmap.StorageTypeNone } var pageSize = 4096 func init() { tmp := unix.Getpagesize() if tmp > 0 { pageSize = tmp } } incus-7.0.0/internal/server/seccomp/seccomp_empty.go000066400000000000000000000000671517523235500226230ustar00rootroot00000000000000package seccomp // Placeholder for non-linux systems. incus-7.0.0/internal/server/seccomp/seccomp_test.go000066400000000000000000000011221517523235500224350ustar00rootroot00000000000000//go:build linux && cgo package seccomp import ( "fmt" "testing" ) func TestMountFlagsToOpts(t *testing.T) { opts := mountFlagsToOpts(knownFlags) if opts != "ro,nosuid,nodev,noexec,sync,remount,mand,noatime,nodiratime,bind,strictatime,lazytime" { t.Fatal(fmt.Errorf("Mount options parsing failed with invalid option string: %s", opts)) } opts = mountFlagsToOpts(knownFlagsRecursive) if opts != "ro,nosuid,nodev,noexec,sync,remount,mand,noatime,nodiratime,rbind,strictatime,lazytime" { t.Fatal(fmt.Errorf("Mount options parsing failed with invalid option string: %s", opts)) } } incus-7.0.0/internal/server/seccomp/sysinfo.go000066400000000000000000000003761517523235500214510ustar00rootroot00000000000000package seccomp // Sysinfo architecture independent sysinfo struct. type Sysinfo struct { Uptime int64 Totalram uint64 Freeram uint64 Sharedram uint64 Bufferram uint64 Totalswap uint64 Freeswap uint64 Procs uint16 Unit uint32 } incus-7.0.0/internal/server/seccomp/sysinfo_32.go000066400000000000000000000012421517523235500217460ustar00rootroot00000000000000//go:build 386 || arm || ppc || s390 || mips || mipsle package seccomp import ( "golang.org/x/sys/unix" ) // ToNative fills fields from s into native fields. func (s *Sysinfo) ToNative(n *unix.Sysinfo_t) { n.Bufferram = uint32(s.Bufferram) / s.Unit n.Freeram = uint32(s.Freeram) / s.Unit n.Freeswap = uint32(s.Freeswap) / s.Unit n.Procs = s.Procs n.Sharedram = uint32(s.Sharedram) / s.Unit n.Totalram = uint32(s.Totalram) / s.Unit n.Totalswap = uint32(s.Totalswap) / s.Unit n.Uptime = int32(s.Uptime) n.Unit = uint32(s.Unit) } // ToNative32 should never be called on 32bit arches. func (s *Sysinfo) ToNative32(n unix.Sysinfo_t) *unix.Sysinfo_t { return nil } incus-7.0.0/internal/server/seccomp/sysinfo_64.go000066400000000000000000000030431517523235500217540ustar00rootroot00000000000000//go:build amd64 || ppc64 || ppc64le || arm64 || s390x || mips64 || mips64le || riscv64 || loong64 package seccomp import ( "golang.org/x/sys/unix" ) // ToNative fills fields from s into native fields. func (s *Sysinfo) ToNative(n *unix.Sysinfo_t) { n.Bufferram = s.Bufferram / uint64(s.Unit) n.Freeram = s.Freeram / uint64(s.Unit) n.Freeswap = s.Freeswap / uint64(s.Unit) n.Procs = s.Procs n.Sharedram = s.Sharedram / uint64(s.Unit) n.Totalram = s.Totalram / uint64(s.Unit) n.Totalswap = s.Totalswap / uint64(s.Unit) n.Uptime = s.Uptime n.Unit = s.Unit } type sysinfo32 struct { Uptime int32 Loads [3]uint32 Totalram uint32 Freeram uint32 Sharedram uint32 Bufferram uint32 Totalswap uint32 Freeswap uint32 Procs uint16 Pad uint16 Totalhigh uint32 Freehigh uint32 Unit uint32 _ [8]int8 } // ToNative32 returns the data as an i386 struct. func (s *Sysinfo) ToNative32(n unix.Sysinfo_t) sysinfo32 { return sysinfo32{ Uptime: int32(s.Uptime), Loads: [3]uint32{uint32(n.Loads[0]), uint32(n.Loads[1]), uint32(n.Loads[2])}, Totalram: uint32(s.Totalram / uint64(s.Unit)), Freeram: uint32(s.Freeram / uint64(s.Unit)), Sharedram: uint32(s.Sharedram / uint64(s.Unit)), Bufferram: uint32(s.Bufferram / uint64(s.Unit)), Totalswap: uint32(s.Totalswap / uint64(s.Unit)), Freeswap: uint32(s.Freeswap / uint64(s.Unit)), Procs: uint16(s.Procs), Totalhigh: uint32(n.Totalhigh) * n.Unit / uint32(s.Unit), Freehigh: uint32(n.Freehigh) * n.Unit / uint32(s.Unit), Unit: s.Unit, } } incus-7.0.0/internal/server/state/000077500000000000000000000000001517523235500171115ustar00rootroot00000000000000incus-7.0.0/internal/server/state/notlinux.go000066400000000000000000000004351517523235500213220ustar00rootroot00000000000000//go:build !linux || !cgo || agent package state import ( "context" "github.com/lxc/incus/v7/internal/server/events" ) // State here is just an empty shim to satisfy dependencies. type State struct { Events *events.Server ShutdownCtx context.Context ServerName string } incus-7.0.0/internal/server/state/state.go000066400000000000000000000045601517523235500205650ustar00rootroot00000000000000//go:build linux && cgo && !agent package state import ( "context" "net/http" "net/url" "time" "github.com/lxc/incus/v7/internal/server/auth" "github.com/lxc/incus/v7/internal/server/bgp" clusterConfig "github.com/lxc/incus/v7/internal/server/cluster/config" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/dns" "github.com/lxc/incus/v7/internal/server/endpoints" "github.com/lxc/incus/v7/internal/server/events" "github.com/lxc/incus/v7/internal/server/firewall" "github.com/lxc/incus/v7/internal/server/fsmonitor" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/network/ovn" "github.com/lxc/incus/v7/internal/server/network/ovs" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/internal/server/storage/linstor" "github.com/lxc/incus/v7/internal/server/sys" localtls "github.com/lxc/incus/v7/shared/tls" ) type clusterGateway interface { LeaderAddress() (string, error) } // State is a gateway to the two main stateful components, the database // and the operating system. It's typically used by model entities such as // containers, volumes, etc. in order to perform changes. type State struct { // Shutdown Context ShutdownCtx context.Context // Databases DB *db.DB // BGP server BGP *bgp.Server // Cluster Cluster clusterGateway // DNS server DNS *dns.Server // OS access OS *sys.OS Proxy func(req *http.Request) (*url.URL, error) // REST endpoints Endpoints *endpoints.Endpoints // Event server DevIncusEvents *events.DevIncusServer Events *events.Server // Firewall instance Firewall firewall.Firewall // Server certificate ServerCert func() *localtls.CertInfo UpdateCertificateCache func() // Available instance types based on operational drivers. InstanceTypes map[instancetype.Type]error // Filesystem monitor DevMonitor fsmonitor.FSMonitor // Global configuration GlobalConfig *clusterConfig.Config // Local configuration LocalConfig *node.Config // Local server name. ServerName string // Whether the server is clustered. ServerClustered bool // Local server start time. StartTime time.Time // Authorizer. Authorizer auth.Authorizer // OVN. OVN func() (*ovn.NB, *ovn.SB, error) // OVS. OVS func() (*ovs.VSwitch, error) // Linstor. Linstor func() (*linstor.Client, error) } incus-7.0.0/internal/server/state/testing.go000066400000000000000000000021161517523235500211150ustar00rootroot00000000000000//go:build linux && cgo && !agent package state import ( "context" "testing" clusterConfig "github.com/lxc/incus/v7/internal/server/cluster/config" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/firewall" "github.com/lxc/incus/v7/internal/server/sys" ) // NewTestState returns a State object initialized with testable instances of // the node/cluster databases and of the OS facade. // // Return the newly created State object, along with a function that can be // used for cleaning it up. func NewTestState(t *testing.T) (*State, func()) { node, nodeCleanup := db.NewTestNode(t) cluster, clusterCleanup := db.NewTestCluster(t) os, osCleanup := sys.NewTestOS(t) cleanup := func() { nodeCleanup() clusterCleanup() osCleanup() } state := &State{ ShutdownCtx: context.TODO(), DB: &db.DB{Node: node, Cluster: cluster}, OS: os, Firewall: firewall.New(), UpdateCertificateCache: func() {}, GlobalConfig: &clusterConfig.Config{}, } return state, cleanup } incus-7.0.0/internal/server/storage/000077500000000000000000000000001517523235500174355ustar00rootroot00000000000000incus-7.0.0/internal/server/storage/backend.go000066400000000000000000011572711517523235500213710ustar00rootroot00000000000000package storage import ( "context" "crypto/rand" "database/sql" "encoding/json" "errors" "fmt" "io" "io/fs" "maps" "math/big" "net" "net/http" "net/url" "os" "os/exec" "path/filepath" "slices" "strings" "sync" "syscall" "time" "go.yaml.in/yaml/v4" "golang.org/x/sync/errgroup" incus "github.com/lxc/incus/v7/client" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/instancewriter" internalIO "github.com/lxc/incus/v7/internal/io" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/rsync" "github.com/lxc/incus/v7/internal/server/backup" backupConfig "github.com/lxc/incus/v7/internal/server/backup/config" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/lifecycle" "github.com/lxc/incus/v7/internal/server/locking" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/internal/server/storage/memorypipe" "github.com/lxc/incus/v7/internal/server/storage/s3" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) var ( unavailablePools = make(map[string]struct{}) unavailablePoolsMu = sync.Mutex{} ) // ConnectIfInstanceIsRemote is a reference to cluster.ConnectIfInstanceIsRemote. // //nolint:typecheck var ConnectIfInstanceIsRemote func(s *state.State, projectName string, instName string, r *http.Request) (incus.InstanceServer, error) // instanceDiskVolumeEffectiveFields fields from the instance disks that are applied to the volume's effective // config (but not stored in the disk's volume database record). var instanceDiskVolumeEffectiveFields = []string{ "size", "size.state", } type backend struct { driver drivers.Driver id int64 db api.StoragePool name string state *state.State logger logger.Logger nodes map[int64]db.StoragePoolNode } // ID returns the storage pool ID. func (b *backend) ID() int64 { return b.id } // Name returns the storage pool name. func (b *backend) Name() string { return b.name } // Description returns the storage pool description. func (b *backend) Description() string { return b.db.Description } // Validate storage pool config. func (b *backend) Validate(config map[string]string) error { return b.Driver().Validate(config) } // Status returns the storage pool status. func (b *backend) Status() string { return b.db.Status } // LocalStatus returns storage pool status of the local cluster member. func (b *backend) LocalStatus() string { // Check if pool is unavailable locally and replace status if so. // But don't modify b.db.Status as the status may be recovered later so we don't want to persist it here. if !IsAvailable(b.name) { return api.StoragePoolStatusUnvailable } node, exists := b.nodes[b.state.DB.Cluster.GetNodeID()] if !exists { return api.StoragePoolStatusUnknown } return db.StoragePoolStateToAPIStatus(node.State) } // isStatusReady returns an error if pool is not ready for use on this server. func (b *backend) isStatusReady() error { if b.Status() == api.StoragePoolStatusPending { return errors.New("Specified pool is not fully created") } if b.LocalStatus() == api.StoragePoolStatusUnvailable { return api.StatusErrorf(http.StatusServiceUnavailable, "Storage pool is unavailable on this server") } return nil } // ToAPI returns the storage pool as an API representation. func (b *backend) ToAPI() api.StoragePool { return b.db } // Driver returns the storage pool driver. func (b *backend) Driver() drivers.Driver { return b.driver } // MigrationTypes returns the migration transport method preferred when sending a migration, based // on the migration method requested by the driver's ability. The copySnapshots argument indicates // whether snapshots are migrated as well. clusterMove determines whether the migration is done // within a cluster and storageMove determines whether the storage pool is changed by the migration. // This method is used to determine whether to use optimized migration. func (b *backend) MigrationTypes(contentType drivers.ContentType, refresh bool, copySnapshots bool, clusterMove bool, storageMove bool) []localMigration.Type { return b.driver.MigrationTypes(contentType, refresh, copySnapshots, clusterMove, storageMove) } // Create creates the storage pool layout on the storage device. // localOnly is used for clustering where only a single node should do remote storage setup. func (b *backend) Create(clientType request.ClientType, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"config": b.db.Config, "description": b.db.Description, "clientType": clientType}) l.Debug("Create started") defer l.Debug("Create finished") // Validate config. err := b.driver.Validate(b.db.Config) if err != nil { return err } reverter := revert.New() defer reverter.Fail() path := drivers.GetPoolMountPath(b.name) if internalUtil.IsDir(path) { return fmt.Errorf("Storage pool directory %q already exists", path) } // Create the storage path. err = os.MkdirAll(path, 0o711) if err != nil { return fmt.Errorf("Failed to create storage pool directory %q: %w", path, err) } reverter.Add(func() { _ = os.RemoveAll(path) }) if b.driver.Info().Remote && clientType != request.ClientTypeNormal { if !b.driver.Info().MountedRoot { // Create the directory structure. err = b.createStorageStructure(path) if err != nil { return err } } // Dealing with a remote storage pool, we're done now. reverter.Success() return nil } // Create the storage pool on the storage device. err = b.driver.Create() if err != nil { return err } reverter.Add(func() { _ = b.driver.Delete(op) }) // Mount the storage pool. ourMount, err := b.driver.Mount() if err != nil { return err } // We expect the caller of create to mount the pool if needed, so we should unmount after // storage struct has been created. if ourMount { defer func() { _, _ = b.driver.Unmount() }() } // Create the directory structure. err = b.createStorageStructure(path) if err != nil { return err } reverter.Success() return nil } // GetVolume returns a drivers.Volume containing copies of the supplied volume config and the pools config. func (b *backend) GetVolume(volType drivers.VolumeType, contentType drivers.ContentType, volName string, volConfig map[string]string) drivers.Volume { return drivers.NewVolume(b.driver, b.name, volType, contentType, volName, volConfig, b.db.Config).Clone() } // GetResources returns utilisation information about the pool. func (b *backend) GetResources() (*api.ResourcesStoragePool, error) { l := b.logger.AddContext(nil) l.Debug("GetResources started") defer l.Debug("GetResources finished") if b.Status() == api.StoragePoolStatusPending { return nil, errors.New("The pool is in pending state") } return b.driver.GetResources() } // IsUsed returns whether the storage pool is used by any volumes or profiles (excluding image volumes). func (b *backend) IsUsed() (bool, error) { usedBy, err := UsedBy(context.TODO(), b.state, b, true, true, db.StoragePoolVolumeTypeNameImage) if err != nil { return false, err } return len(usedBy) > 0, nil } // Update updates the pool config. func (b *backend) Update(clientType request.ClientType, newDesc string, newConfig map[string]string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"newDesc": newDesc, "newConfig": newConfig}) l.Debug("Update started") defer l.Debug("Update finished") // Validate config. err := b.driver.Validate(newConfig) if err != nil { return err } // Keep old config. oldConfig := api.ConfigMap{} maps.Copy(oldConfig, b.db.Config) // Diff the configurations. changedConfig, userOnly := b.detectChangedConfig(b.db.Config, newConfig) // Check if the pool source is being changed that the local state is still pending, otherwise prevent it. _, sourceChanged := changedConfig["source"] if sourceChanged && b.LocalStatus() != api.StoragePoolStatusPending { return errors.New("Pool source cannot be changed when not in pending state") } // Prevent shrinking the storage pool. newSize, sizeChanged := changedConfig["size"] if sizeChanged && newSize != "" && newSize != drivers.MaxValue { oldSizeBytes, _ := units.ParseByteSizeString(b.db.Config["size"]) newSizeBytes, _ := units.ParseByteSizeString(newSize) if newSizeBytes < oldSizeBytes { return errors.New("Pool cannot be shrunk") } } // Apply changes to local member if both global pool and node are not pending and non-user config changed. // Otherwise just apply changes to DB (below) ready for the actual global create request to be initiated. if len(changedConfig) > 0 && b.Status() != api.StoragePoolStatusPending && b.LocalStatus() != api.StoragePoolStatusPending && !userOnly { err = b.driver.Update(changedConfig) if err != nil { return err } } // Check if anything was changed by Update. updateChanges, _ := b.detectChangedConfig(oldConfig, b.driver.Config()) maps.Copy(newConfig, updateChanges) // Update the database if something changed and we're in ClientTypeNormal mode. if clientType == request.ClientTypeNormal && (len(changedConfig) > 0 || newDesc != b.db.Description) { err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateStoragePool(ctx, b.name, newDesc, newConfig) }) if err != nil { return err } } return nil } // warningsDelete deletes any persistent warnings for the pool. func (b *backend) warningsDelete() error { err := b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return cluster.DeleteWarnings(ctx, tx.Tx(), cluster.TypeStoragePool, int(b.ID())) }) if err != nil { return fmt.Errorf("Failed deleting persistent warnings: %w", err) } return nil } // Delete removes the pool. func (b *backend) Delete(clientType request.ClientType, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"clientType": clientType}) l.Debug("Delete started") defer l.Debug("Delete finished") // Delete any persistent warnings for pool. err := b.warningsDelete() if err != nil { return err } // If completely gone, just return path := internalUtil.VarPath("storage-pools", b.name) if !util.PathExists(path) { return nil } if clientType != request.ClientTypeNormal && b.driver.Info().Remote { if b.driver.Info().Deactivate || b.driver.Info().MountedRoot { _, err := b.driver.Unmount() if err != nil { return err } } if !b.driver.Info().MountedRoot { // Remote storage may have leftover entries caused by // volumes that were moved or delete while a particular system was offline. err := os.RemoveAll(path) if err != nil { return err } } } else { // Remove any left over image volumes. // This can occur during partial image unpack or if the storage pool has been recovered from an // instance backup file and the image volume DB records were not restored. // If non-image volumes exist, we don't delete the, even if they can then prevent the storage pool // from being deleted, because they should not exist by this point and we don't want to end up // removing an instance or custom volume accidentally. // Errors listing volumes are ignored, as we should still try and delete the storage pool. vols, _ := b.driver.ListVolumes() for _, vol := range vols { if vol.Type() == drivers.VolumeTypeImage { err := b.driver.DeleteVolume(vol, op) if err != nil { return fmt.Errorf("Failed deleting left over image volume %q (%s): %w", vol.Name(), vol.ContentType(), err) } l.Warn("Deleted left over image volume", logger.Ctx{"volName": vol.Name(), "contentType": vol.ContentType()}) } } // Delete the low-level storage. err := b.driver.Delete(op) if err != nil { return err } } // Delete the mountpoint. err = os.Remove(path) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove directory %q: %w", path, err) } unavailablePoolsMu.Lock() delete(unavailablePools, b.Name()) unavailablePoolsMu.Unlock() return nil } // Mount mounts the storage pool. func (b *backend) Mount() (bool, error) { b.logger.Debug("Mount started") defer b.logger.Debug("Mount finished") reverter := revert.New() defer reverter.Fail() reverter.Add(func() { unavailablePoolsMu.Lock() unavailablePools[b.Name()] = struct{}{} unavailablePoolsMu.Unlock() }) path := drivers.GetPoolMountPath(b.name) // Create the storage path if needed. if !internalUtil.IsDir(path) { err := os.MkdirAll(path, 0o711) if err != nil { return false, fmt.Errorf("Failed to create storage pool directory %q: %w", path, err) } } ourMount, err := b.driver.Mount() if err != nil { return false, err } if ourMount { reverter.Add(func() { _, _ = b.Unmount() }) } // Create the directory structure (if needed) after mounted. err = b.createStorageStructure(path) if err != nil { return false, err } reverter.Success() // Ensure pool is marked as available now its mounted. unavailablePoolsMu.Lock() delete(unavailablePools, b.Name()) unavailablePoolsMu.Unlock() return ourMount, nil } // Unmount unmounts the storage pool. func (b *backend) Unmount() (bool, error) { b.logger.Debug("Unmount started") defer b.logger.Debug("Unmount finished") return b.driver.Unmount() } // ApplyPatch runs the requested patch at both backend and driver level. func (b *backend) ApplyPatch(name string) error { b.logger.Info("Applying patch", logger.Ctx{"name": name}) // Run early backend patches. patch, ok := earlyPatches[name] if ok { err := patch(b) if err != nil { return err } } // Run the driver patch itself. err := b.driver.ApplyPatch(name) if err != nil { return err } // Run late backend patches. patch, ok = latePatches[name] if ok { err := patch(b) if err != nil { return err } } return nil } // ensureInstanceSymlink creates a symlink in the instance directory to the instance's mount path // if doesn't exist already. func (b *backend) ensureInstanceSymlink(instanceType instancetype.Type, projectName string, instanceName string, mountPath string) error { if internalInstance.IsSnapshot(instanceName) { return errors.New("Instance must not be snapshot") } symlinkPath := InstancePath(instanceType, projectName, instanceName, false) // Remove any old symlinks left over by previous bugs that may point to a different pool. if util.PathExists(symlinkPath) { err := os.Remove(symlinkPath) if err != nil { return fmt.Errorf("Failed to remove symlink %q: %w", symlinkPath, err) } } // Create new symlink. err := os.Symlink(mountPath, symlinkPath) if err != nil { return fmt.Errorf("Failed to create symlink from %q to %q: %w", mountPath, symlinkPath, err) } return nil } // removeInstanceSymlink removes a symlink in the instance directory to the instance's mount path. func (b *backend) removeInstanceSymlink(instanceType instancetype.Type, projectName string, instanceName string) error { symlinkPath := InstancePath(instanceType, projectName, instanceName, false) if util.PathExists(symlinkPath) { err := os.Remove(symlinkPath) if err != nil { return fmt.Errorf("Failed to remove symlink %q: %w", symlinkPath, err) } } return nil } // ensureInstanceSnapshotSymlink creates a symlink in the snapshot directory to the instance's // snapshot path if doesn't exist already. func (b *backend) ensureInstanceSnapshotSymlink(instanceType instancetype.Type, projectName string, instanceName string) error { // Check we can convert the instance to the volume type needed. volType, err := InstanceTypeToVolumeType(instanceType) if err != nil { return err } parentName, _, _ := api.GetParentAndSnapshotName(instanceName) snapshotSymlink := InstancePath(instanceType, projectName, parentName, true) volStorageName := project.Instance(projectName, parentName) snapshotTargetPath := drivers.GetVolumeSnapshotDir(b.name, volType, volStorageName) // Remove any old symlinks left over by previous bugs that may point to a different pool. if util.PathExists(snapshotSymlink) { err = os.Remove(snapshotSymlink) if err != nil { return fmt.Errorf("Failed to remove symlink %q: %w", snapshotSymlink, err) } } // Create new symlink. err = os.Symlink(snapshotTargetPath, snapshotSymlink) if err != nil { return fmt.Errorf("Failed to create symlink from %q to %q: %w", snapshotTargetPath, snapshotSymlink, err) } return nil } // removeInstanceSnapshotSymlinkIfUnused removes the symlink in the snapshot directory to the // instance's snapshot path if the snapshot path is missing. It is expected that the driver will // remove the instance's snapshot path after the last snapshot is removed or the volume is deleted. func (b *backend) removeInstanceSnapshotSymlinkIfUnused(instanceType instancetype.Type, projectName string, instanceName string) error { // Check we can convert the instance to the volume type needed. volType, err := InstanceTypeToVolumeType(instanceType) if err != nil { return err } parentName, _, _ := api.GetParentAndSnapshotName(instanceName) snapshotSymlink := InstancePath(instanceType, projectName, parentName, true) volStorageName := project.Instance(projectName, parentName) snapshotTargetPath := drivers.GetVolumeSnapshotDir(b.name, volType, volStorageName) // If snapshot parent directory doesn't exist, remove symlink. if !util.PathExists(snapshotTargetPath) { if util.PathExists(snapshotSymlink) { err := os.Remove(snapshotSymlink) if err != nil { return fmt.Errorf("Failed to remove symlink %q: %w", snapshotSymlink, err) } } } return nil } // applyInstanceRootDiskOverrides applies the instance's root disk config to the volume's config. func (b *backend) applyInstanceRootDiskOverrides(inst instance.Instance, vol *drivers.Volume) error { _, rootDiskConf, err := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if err != nil { return err } for _, k := range instanceDiskVolumeEffectiveFields { if rootDiskConf[k] != "" { switch k { case "size": vol.SetConfigSize(rootDiskConf[k]) case "size.state": vol.SetConfigStateSize(rootDiskConf[k]) default: return fmt.Errorf("Unsupported instance disk volume override field %q", k) } } } return nil } // applyInstanceRootDiskInitialValues applies the instance's root disk initial config to the volume's config. func (b *backend) applyInstanceRootDiskInitialValues(inst instance.Instance, volConfig map[string]string) error { _, rootDiskConf, err := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if err != nil { return err } for k, v := range rootDiskConf { prefix, newKey, found := strings.Cut(k, "initial.") if found && prefix == "" { volConfig[newKey] = v } } return nil } // CreateInstance creates an empty instance. func (b *backend) CreateInstance(inst instance.Instance, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) l.Debug("CreateInstance started") defer l.Debug("CreateInstance finished") err := b.isStatusReady() if err != nil { return err } volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentType := InstanceContentType(inst) reverter := revert.New() defer reverter.Fail() volumeConfig := make(map[string]string) err = b.applyInstanceRootDiskInitialValues(inst, volumeConfig) if err != nil { return err } // Validate config and create database entry for new storage volume. err = VolumeDBCreate(b, inst.Project().Name, inst.Name(), "", volType, false, volumeConfig, inst.CreationDate(), time.Time{}, contentType, true, false) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, inst.Project().Name, inst.Name(), volType) }) // Record new volume with authorizer. err = b.state.Authorizer.AddStoragePoolVolume(b.state.ShutdownCtx, inst.Project().Name, b.Name(), volType.Singular(), inst.Name(), "") if err != nil { logger.Error("Failed to add storage volume to authorizer", logger.Ctx{"name": inst.Name(), "type": volType, "pool": b.Name(), "project": inst.Project().Name, "error": err}) } reverter.Add(func() { _ = b.state.Authorizer.DeleteStoragePoolVolume(b.state.ShutdownCtx, inst.Project().Name, b.Name(), volType.Singular(), inst.Name(), "") }) // Generate the effective root device volume for instance. volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(volType, contentType, volStorageName, volumeConfig) err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return err } var filler *drivers.VolumeFiller if inst.Type() == instancetype.Container { filler = &drivers.VolumeFiller{ Fill: func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { // Create an empty rootfs. err := os.Mkdir(filepath.Join(vol.MountPath(), "rootfs"), 0o755) if err != nil && !os.IsExist(err) { return 0, err } return 0, nil }, } } err = b.driver.CreateVolume(vol, filler, op) if err != nil { return err } reverter.Add(func() { _ = b.DeleteInstance(inst, op) }) err = b.ensureInstanceSymlink(inst.Type(), inst.Project().Name, inst.Name(), vol.MountPath()) if err != nil { return err } err = inst.DeferTemplateApply(instance.TemplateTriggerCreate) if err != nil { return err } reverter.Success() return nil } // CreateInstanceFromBackup restores a backup file onto the storage device. Because the backup file // is unpacked and restored onto the storage device before the instance is created in the database // it is necessary to return two functions; a post hook that can be run once the instance has been // created in the database to run any storage layer finalisations, and a revert hook that can be // run if the instance database load process fails that will remove anything created thus far. func (b *backend) CreateInstanceFromBackup(srcBackup backup.Info, srcData io.ReadSeeker, op *operations.Operation) (func(instance.Instance) error, revert.Hook, error) { l := b.logger.AddContext(logger.Ctx{"project": srcBackup.Project, "instance": srcBackup.Name, "snapshots": srcBackup.Snapshots, "optimizedStorage": *srcBackup.OptimizedStorage}) l.Debug("CreateInstanceFromBackup started") defer l.Debug("CreateInstanceFromBackup finished") // Get the volume name on storage. volStorageName := project.Instance(srcBackup.Project, srcBackup.Name) // Get the instance type. instanceType, err := instancetype.New(string(srcBackup.Type)) if err != nil { return nil, nil, err } // Get the volume type. volType, err := InstanceTypeToVolumeType(instanceType) if err != nil { return nil, nil, err } contentType := drivers.ContentTypeFS if volType == drivers.VolumeTypeVM { contentType = drivers.ContentTypeBlock } var volumeConfig map[string]string if srcBackup.Config != nil && srcBackup.Config.Volume != nil { volumeConfig = srcBackup.Config.Volume.Config } // Get instance root size information. if srcBackup.Config != nil && srcBackup.Config.Container != nil { _, rootConfig, err := internalInstance.GetRootDiskDevice(srcBackup.Config.Container.ExpandedDevices) if err == nil && rootConfig["size"] != "" { if volumeConfig == nil { volumeConfig = map[string]string{} } volumeConfig["size"] = rootConfig["size"] } } // Import dependent disks err = b.createDependentVolumesFromBackup(srcBackup, srcData, op) if err != nil { return nil, nil, err } vol := b.GetVolume(volType, contentType, volStorageName, volumeConfig) importRevert := revert.New() defer importRevert.Fail() // Unpack the backup into the new storage volume(s). var volPostHook drivers.VolumePostHook var revertHook revert.Hook if srcBackup.Config.Volume.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { volPostHook, revertHook, err = b.qcow2UnpackVolume(vol, srcBackup.Snapshots, srcData, backup.DefaultBackupPrefix, op) if err != nil { return nil, nil, err } } else { volPostHook, revertHook, err = b.driver.CreateVolumeFromBackup(vol, srcBackup, srcData, backup.DefaultBackupPrefix, op) if err != nil { return nil, nil, err } } if revertHook != nil { importRevert.Add(revertHook) } err = b.ensureInstanceSymlink(instanceType, srcBackup.Project, srcBackup.Name, vol.MountPath()) if err != nil { return nil, nil, err } importRevert.Add(func() { _ = b.removeInstanceSymlink(instanceType, srcBackup.Project, srcBackup.Name) }) if len(srcBackup.Snapshots) > 0 { err = b.ensureInstanceSnapshotSymlink(instanceType, srcBackup.Project, srcBackup.Name) if err != nil { return nil, nil, err } importRevert.Add(func() { _ = b.removeInstanceSnapshotSymlinkIfUnused(instanceType, srcBackup.Project, srcBackup.Name) }) } // Make sure the size isn't part of the instance volume after initial creation. if volumeConfig != nil { delete(volumeConfig, "size") } // Update information in the backup.yaml file. err = vol.MountTask(func(mountPath string, op *operations.Operation) error { return backup.UpdateInstanceConfig(b.state.DB.Cluster, srcBackup, mountPath) }, op) if err != nil { return nil, nil, fmt.Errorf("Error updating backup file: %w", err) } // Create a post hook function that will use the instance (that will be created) to setup a new volume // containing the instance's root disk device's config so that the driver's post hook function can access // that config to perform any post instance creation setup. postHook := func(inst instance.Instance) error { l.Debug("CreateInstanceFromBackup post hook started") defer l.Debug("CreateInstanceFromBackup post hook finished") postHookRevert := revert.New() defer postHookRevert.Fail() // Create database entry for new storage volume. var volumeDescription string var volumeConfig map[string]string volumeCreationDate := inst.CreationDate() if srcBackup.Config != nil && srcBackup.Config.Volume != nil { // If the backup restore interface provides volume config use it, otherwise use // default volume config for the storage pool. volumeDescription = srcBackup.Config.Volume.Description volumeConfig = srcBackup.Config.Volume.Config // Use volume's creation date if available. if !srcBackup.Config.Volume.CreatedAt.IsZero() { volumeCreationDate = srcBackup.Config.Volume.CreatedAt } } // Validate config and create database entry for new storage volume. // Strip unsupported config keys (in case the export was made from a different type of storage pool). err = VolumeDBCreate(b, inst.Project().Name, inst.Name(), volumeDescription, volType, false, volumeConfig, volumeCreationDate, time.Time{}, contentType, true, true) if err != nil { return err } postHookRevert.Add(func() { _ = VolumeDBDelete(b, inst.Project().Name, inst.Name(), volType) }) // Record new volume with authorizer. err = b.state.Authorizer.AddStoragePoolVolume(b.state.ShutdownCtx, inst.Project().Name, b.Name(), volType.Singular(), inst.Name(), "") if err != nil { logger.Error("Failed to add storage volume to authorizer", logger.Ctx{"name": inst.Name(), "type": volType, "pool": b.Name(), "project": inst.Project().Name, "error": err}) } postHookRevert.Add(func() { _ = b.state.Authorizer.DeleteStoragePoolVolume(b.state.ShutdownCtx, inst.Project().Name, b.Name(), volType.Singular(), inst.Name(), "") }) for i, backupFileSnap := range srcBackup.Snapshots { var volumeSnapDescription string var volumeSnapConfig map[string]string var volumeSnapExpiryDate time.Time var volumeSnapCreationDate time.Time // Check if snapshot volume config is available for restore and matches snapshot name. if srcBackup.Config != nil { if len(srcBackup.Config.Snapshots) > i && srcBackup.Config.Snapshots[i] != nil && srcBackup.Config.Snapshots[i].Name == backupFileSnap { // Use instance snapshot's creation date if snap info available. volumeSnapCreationDate = srcBackup.Config.Snapshots[i].CreatedAt } if len(srcBackup.Config.VolumeSnapshots) > i && srcBackup.Config.VolumeSnapshots[i] != nil && srcBackup.Config.VolumeSnapshots[i].Name == backupFileSnap { // If the backup restore interface provides volume snapshot config use it, // otherwise use default volume config for the storage pool. volumeSnapDescription = srcBackup.Config.VolumeSnapshots[i].Description volumeSnapConfig = srcBackup.Config.VolumeSnapshots[i].Config if srcBackup.Config.VolumeSnapshots[i].ExpiresAt != nil { volumeSnapExpiryDate = *srcBackup.Config.VolumeSnapshots[i].ExpiresAt } // Use volume's creation date if available. if !srcBackup.Config.VolumeSnapshots[i].CreatedAt.IsZero() { volumeSnapCreationDate = srcBackup.Config.VolumeSnapshots[i].CreatedAt } } } newSnapshotName := drivers.GetSnapshotVolumeName(inst.Name(), backupFileSnap) // Validate config and create database entry for new storage volume. // Strip unsupported config keys (in case the export was made from a different type of storage pool). err = VolumeDBCreate(b, inst.Project().Name, newSnapshotName, volumeSnapDescription, volType, true, volumeSnapConfig, volumeSnapCreationDate, volumeSnapExpiryDate, contentType, true, true) if err != nil { return err } postHookRevert.Add(func() { _ = VolumeDBDelete(b, inst.Project().Name, newSnapshotName, volType) }) } // Generate the effective root device volume for instance. volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(volType, contentType, volStorageName, volumeConfig) err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return err } // Save any changes that have occurred to the instance's config to the on-disk backup.yaml file. err = b.UpdateInstanceBackupFile(inst, false, op) if err != nil { return fmt.Errorf("Failed updating backup file: %w", err) } // If the driver returned a post hook, run it now. if volPostHook != nil { // Initialize new volume containing root disk config supplied in instance. err = volPostHook(vol) if err != nil { return err } } rootDiskConf := vol.Config() // Apply quota config from root device if its set. Should be done after driver's post hook if set // so that any volume initialisation has been completed first. if rootDiskConf["size"] != "" { size := rootDiskConf["size"] l.Debug("Applying volume quota from root disk config", logger.Ctx{"size": size}) allowUnsafeResize := false if vol.Type() == drivers.VolumeTypeContainer { // Enable allowUnsafeResize for container imports so that filesystem resize // safety checks are avoided in order to allow more imports to succeed when // otherwise the pre-resize estimated checks of resize2fs would prevent // import. If there is truly insufficient size to complete the import the // resize will still fail, but its OK as we will then delete the volume // rather than leaving it in a corrupted state. We don't need to do this // for non-container volumes (nor should we) because block volumes won't // error if we shrink them too much, and custom volumes can be created at // the correct size immediately and don't need a post-import resize step. allowUnsafeResize = true } err = b.driver.SetVolumeQuota(vol, size, allowUnsafeResize, op) if err != nil { // The restored volume can end up being larger than the root disk config's size // property due to the block boundary rounding some storage drivers use. As such // if the restored volume is larger than the config's size and it cannot be shrunk // to the equivalent size on the target storage driver, don't fail as the backup // has still been restored successfully. if errors.Is(err, drivers.ErrCannotBeShrunk) { l.Warn("Could not apply volume quota from root disk config as restored volume cannot be shrunk", logger.Ctx{"size": size}) } else { return fmt.Errorf("Failed applying volume quota to root disk: %w", err) } } // Apply the filesystem volume quota (only when main volume is block). if vol.IsVMBlock() { vmStateSize := rootDiskConf["size.state"] // Apply default VM config filesystem size if main volume size is specified and // no custom vmStateSize is specified. This way if the main volume size is empty // (i.e removing quota) then this will also pass empty quota for the config // filesystem volume as well, allowing a former quota to be removed from both // volumes. if vmStateSize == "" && size != "" { vmStateSize = b.driver.Info().DefaultVMBlockFilesystemSize } l.Debug("Applying filesystem volume quota from root disk config", logger.Ctx{"size.state": vmStateSize}) fsVol := vol.NewVMBlockFilesystemVolume() err := b.driver.SetVolumeQuota(fsVol, vmStateSize, allowUnsafeResize, op) if errors.Is(err, drivers.ErrCannotBeShrunk) { l.Warn("Could not apply VM filesystem volume quota from root disk config as restored volume cannot be shrunk", logger.Ctx{"size": vmStateSize}) } else if err != nil { return fmt.Errorf("Failed applying filesystem volume quota to root disk: %w", err) } } } postHookRevert.Success() return nil } importRevert.Success() return postHook, revertHook, nil } // CreateInstanceFromCopy copies an instance volume and optionally its snapshots to new volume(s). func (b *backend) CreateInstanceFromCopy(inst instance.Instance, src instance.Instance, snapshots bool, allowInconsistent bool, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "src": src.Name(), "snapshots": snapshots}) l.Debug("CreateInstanceFromCopy started") defer l.Debug("CreateInstanceFromCopy finished") err := b.isStatusReady() if err != nil { return err } if inst.Type() != src.Type() { return errors.New("Instance types must match") } volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentType := InstanceContentType(inst) // Get the source storage pool. srcPool, err := LoadByInstance(b.state, src) if err != nil { return err } srcPoolBackend, ok := srcPool.(*backend) if !ok { return errors.New("Source pool is not a backend") } // Check source volume exists, and get its config. srcConfig, err := srcPool.GenerateInstanceBackupConfig(src, snapshots, true, op) if err != nil { return fmt.Errorf("Failed generating instance copy config: %w", err) } // If we are copying snapshots, retrieve a list of snapshots from source volume. var snapshotNames []string if snapshots { snapshotNames = make([]string, 0, len(srcConfig.VolumeSnapshots)) for _, snapshot := range srcConfig.VolumeSnapshots { snapshotNames = append(snapshotNames, snapshot.Name) } } volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(volType, contentType, volStorageName, srcConfig.Volume.Config) volExists, err := b.driver.HasVolume(vol) if err != nil { return err } if volExists { return errors.New("Cannot create volume, already exists on target storage") } // Setup reverter. reverter := revert.New() defer reverter.Fail() // Some driver backing stores require that running instances be frozen during copy. if !src.IsSnapshot() && srcPoolBackend.driver.Info().RunningCopyFreeze && src.IsRunning() && !src.IsFrozen() && !allowInconsistent { b.logger.Info("Freezing instance for consistent copy") err = src.Freeze() if err != nil { return err } defer func() { _ = src.Unfreeze() }() // Attempt to sync the filesystem. _ = linux.SyncFS(src.RootfsPath()) } reverter.Add(func() { _ = b.DeleteInstance(inst, op) }) if b.Name() == srcPool.Name() { l.Debug("CreateInstanceFromCopy same-pool mode detected") // Get the src volume name on storage. srcVolStorageName := project.Instance(src.Project().Name, src.Name()) srcVol := b.GetVolume(volType, contentType, srcVolStorageName, srcConfig.Volume.Config) // Validate config and create database entry for new storage volume. err = VolumeDBCreate(b, inst.Project().Name, inst.Name(), "", vol.Type(), false, vol.Config(), inst.CreationDate(), time.Time{}, contentType, false, true) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, inst.Project().Name, inst.Name(), volType) }) // Record new volume with authorizer. err = b.state.Authorizer.AddStoragePoolVolume(b.state.ShutdownCtx, inst.Project().Name, b.Name(), volType.Singular(), inst.Name(), "") if err != nil { logger.Error("Failed to add storage volume to authorizer", logger.Ctx{"name": inst.Name(), "type": volType, "pool": b.Name(), "project": inst.Project().Name, "error": err}) } reverter.Add(func() { _ = b.state.Authorizer.DeleteStoragePoolVolume(b.state.ShutdownCtx, inst.Project().Name, b.Name(), volType.Singular(), inst.Name(), "") }) // Create database entries for new storage volume snapshots. for i, snapName := range snapshotNames { newSnapshotName := drivers.GetSnapshotVolumeName(inst.Name(), snapName) var volumeSnapExpiryDate time.Time if srcConfig.VolumeSnapshots[i].ExpiresAt != nil { volumeSnapExpiryDate = *srcConfig.VolumeSnapshots[i].ExpiresAt } // Validate config and create database entry for new storage volume. err = VolumeDBCreate(b, inst.Project().Name, newSnapshotName, srcConfig.VolumeSnapshots[i].Description, vol.Type(), true, srcConfig.VolumeSnapshots[i].Config, srcConfig.VolumeSnapshots[i].CreatedAt, volumeSnapExpiryDate, vol.ContentType(), false, true) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, inst.Project().Name, newSnapshotName, vol.Type()) }) } // Generate the effective root device volume for instance. err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return err } err = b.driver.CreateVolumeFromCopy(vol, srcVol, snapshots, allowInconsistent, op) if err != nil { return err } newDevices := inst.LocalDevices() err = src.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error { // Load the pool for the disk. diskPool, err := LoadByName(b.state, newDevices[dev.Name]["pool"]) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } err = diskPool.CreateCustomVolumeFromCopy(inst.Project().Name, src.Project().Name, newDevices[dev.Name]["source"], "", nil, dev.Config["pool"], dev.Config["source"], snapshots, op) if err != nil { return err } return nil }) if err != nil { return err } } else { // We are copying volumes between storage pools so use migration system as it will // be able to negotiate a common transfer method between pool types. l.Debug("CreateInstanceFromCopy cross-pool mode detected") // Negotiate the migration type to use. offeredTypes := srcPool.MigrationTypes(contentType, false, snapshots, false, true) offerHeader := localMigration.TypesToHeader(offeredTypes...) migrationTypes, err := localMigration.MatchTypes(offerHeader, FallbackMigrationType(contentType), b.MigrationTypes(contentType, false, snapshots, false, true)) if err != nil { return fmt.Errorf("Failed to negotiate copy migration type: %w", err) } var srcVolumeSize int64 // For VMs, get source volume size so that target can create the volume the same size. if src.Type() == instancetype.VM { srcVolumeSize, err = InstanceDiskBlockSize(srcPool, src, op) if err != nil { return fmt.Errorf("Failed getting source disk size: %w", err) } } var migrationSnapshots []*migration.Snapshot if snapshots { migrationSnapshots, err = VolumeSnapshotsToMigrationSnapshots(srcConfig.VolumeSnapshots, inst.Project().Name, srcPool, contentType, volType, src.Name()) if err != nil { return err } } newDevices := inst.LocalDevices().CloneNative() dependentVolumesOffer, err := GenerateDependentVolumesOffer(b.state, srcConfig, inst.Project().Name, snapshots, newDevices, false) if err != nil { err := fmt.Errorf("Failed generating instance depending volumes offer: %w", err) return err } volumesWithTypes, err := DependentVolumesMatchMigrationType(b.state, dependentVolumesOffer, snapshots, newDevices, false) if err != nil { err := fmt.Errorf("Failed to negotiate migration types for dependent volumes: %w", err) return err } srcDependentVolumes := []localMigration.DependentVolumeArgs{} dstDependentVolumes := []localMigration.DependentVolumeArgs{} for _, volWithType := range volumesWithTypes { srcDependentVolumes = append(srcDependentVolumes, localMigration.ProtobufToDependentVolume(volWithType.Volume, volWithType.VolumeTypes[0], nil)) vol := localMigration.ProtobufToDependentVolume(volWithType.Volume, volWithType.VolumeTypes[0], newDevices[*volWithType.Volume.DeviceName]) dstDependentVolumes = append(dstDependentVolumes, vol) } ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Run sender and receiver in separate go routines to prevent deadlocks. g, ctx := errgroup.WithContext(ctx) // Use in-memory pipe pair to simulate a connection between the sender and receiver. // Use context from error group so that if either side fails the pipes are closed. aEnd, bEnd := memorypipe.NewPipePair(ctx) // Start each side of the migration concurrently and collect any errors. g.Go(func() error { return srcPool.MigrateInstance(src, aEnd, &localMigration.VolumeSourceArgs{ IndexHeaderVersion: localMigration.IndexHeaderVersion, Name: src.Name(), Snapshots: snapshotNames, MigrationType: migrationTypes[0], TrackProgress: true, // Do use a progress tracker on sender. AllowInconsistent: allowInconsistent, VolumeOnly: !snapshots, Info: &localMigration.Info{Config: srcConfig}, StorageMove: true, DependentVolumes: srcDependentVolumes, }, op) }) g.Go(func() error { return b.CreateInstanceFromMigration(inst, bEnd, localMigration.VolumeTargetArgs{ IndexHeaderVersion: localMigration.IndexHeaderVersion, Name: inst.Name(), Snapshots: migrationSnapshots, MigrationType: migrationTypes[0], VolumeSize: srcVolumeSize, // Block size setting override. TrackProgress: false, // Do not use a progress tracker on receiver. VolumeOnly: !snapshots, StoragePool: srcPool.Name(), DependentVolumes: dstDependentVolumes, }, op) }) err = g.Wait() if err != nil { return fmt.Errorf("Create instance volume from copy failed: %w", err) } } // Setup the symlinks. err = b.ensureInstanceSymlink(inst.Type(), inst.Project().Name, inst.Name(), vol.MountPath()) if err != nil { return err } if len(snapshotNames) > 0 { err = b.ensureInstanceSnapshotSymlink(inst.Type(), inst.Project().Name, inst.Name()) if err != nil { return err } } reverter.Success() return nil } // RefreshCustomVolume refreshes custom volumes (and optionally snapshots) during the custom volume copy operations. // Snapshots that are not present in the source but are in the destination are removed from the // destination if snapshots are included in the synchronization. func (b *backend) RefreshCustomVolume(projectName string, srcProjectName string, volName string, desc string, config map[string]string, srcPoolName, srcVolName string, snapshots bool, excludeOlder bool, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "srcProjectName": srcProjectName, "volName": volName, "desc": desc, "config": config, "srcPoolName": srcPoolName, "srcVolName": srcVolName, "snapshots": snapshots}) l.Debug("RefreshCustomVolume started") defer l.Debug("RefreshCustomVolume finished") err := b.isStatusReady() if err != nil { return err } if srcProjectName == "" { srcProjectName = projectName } // Setup the source pool backend instance. var srcPool Pool if b.name == srcPoolName { srcPool = b // Source and target are in the same pool so share pool var. } else { // Source is in a different pool to target, so load the pool. srcPool, err = LoadByName(b.state, srcPoolName) if err != nil { return err } } // Check source volume exists and is custom type, and get its config. srcConfig, err := srcPool.GenerateCustomVolumeBackupConfig(srcProjectName, srcVolName, snapshots, op) if err != nil { return fmt.Errorf("Failed generating volume refresh config: %w", err) } // Use the source volume's config if not supplied. if config == nil { config = srcConfig.Volume.Config } // Use the source volume's description if not supplied. if desc == "" { desc = srcConfig.Volume.Description } contentDBType, err := VolumeContentTypeNameToContentType(srcConfig.Volume.ContentType) if err != nil { return err } // Get the source volume's content type. contentType, err := VolumeDBContentTypeToContentType(contentDBType) if err != nil { return err } if contentType != drivers.ContentTypeFS && contentType != drivers.ContentTypeBlock { return fmt.Errorf("Volume of content type %q cannot be refreshed", contentType) } storagePoolSupported := slices.Contains(b.Driver().Info().VolumeTypes, drivers.VolumeTypeCustom) if !storagePoolSupported { return errors.New("Storage pool does not support custom volume type") } reverter := revert.New() defer reverter.Fail() // Only send the snapshots that the target needs when refreshing. // There is currently no recorded creation timestamp, so we can only detect changes based on name. var snapshotNames []string if snapshots { // Compare snapshots. sourceSnapshotComparable := make([]ComparableSnapshot, 0, len(srcConfig.VolumeSnapshots)) for _, sourceSnap := range srcConfig.VolumeSnapshots { sourceSnapshotComparable = append(sourceSnapshotComparable, ComparableSnapshot{ Name: sourceSnap.Name, CreationDate: sourceSnap.CreatedAt, }) } targetSnaps, err := VolumeDBSnapshotsGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return err } targetSnapshotsComparable := make([]ComparableSnapshot, 0, len(targetSnaps)) for _, targetSnap := range targetSnaps { _, targetSnapName, _ := api.GetParentAndSnapshotName(targetSnap.Name) targetSnapshotsComparable = append(targetSnapshotsComparable, ComparableSnapshot{ Name: targetSnapName, CreationDate: targetSnap.CreationDate, }) } syncSourceSnapshotIndexes, deleteTargetSnapshotIndexes := CompareSnapshots(sourceSnapshotComparable, targetSnapshotsComparable, excludeOlder) // Delete extra snapshots first. for _, deleteTargetSnapIndex := range deleteTargetSnapshotIndexes { err = b.DeleteCustomVolumeSnapshot(projectName, targetSnaps[deleteTargetSnapIndex].Name, op) if err != nil { return err } } // Ensure that only the requested snapshots are included in the source config. allSnapshots := srcConfig.VolumeSnapshots srcConfig.VolumeSnapshots = make([]*api.StorageVolumeSnapshot, 0, len(syncSourceSnapshotIndexes)) for _, syncSourceSnapIndex := range syncSourceSnapshotIndexes { snapshotNames = append(snapshotNames, allSnapshots[syncSourceSnapIndex].Name) srcConfig.VolumeSnapshots = append(srcConfig.VolumeSnapshots, allSnapshots[syncSourceSnapIndex]) } } volStorageName := project.StorageVolume(projectName, volName) vol := b.GetVolume(drivers.VolumeTypeCustom, contentType, volStorageName, config) // Get the src volume name on storage. srcVolStorageName := project.StorageVolume(srcProjectName, srcVolName) srcVol := srcPool.GetVolume(drivers.VolumeTypeCustom, contentType, srcVolStorageName, srcConfig.Volume.Config) if srcPool == b { l.Debug("RefreshCustomVolume same-pool mode detected") // Only refresh the snapshots that the target needs. srcSnapVols := make([]drivers.Volume, 0, len(srcConfig.VolumeSnapshots)) for _, srcSnap := range srcConfig.VolumeSnapshots { newSnapshotName := drivers.GetSnapshotVolumeName(volName, srcSnap.Name) snapExpiryDate := time.Time{} if srcSnap.ExpiresAt != nil { snapExpiryDate = *srcSnap.ExpiresAt } // Validate config and create database entry for new storage volume from source volume config. err = VolumeDBCreate(b, projectName, newSnapshotName, srcSnap.Description, drivers.VolumeTypeCustom, true, srcSnap.Config, srcSnap.CreatedAt, snapExpiryDate, contentType, false, true) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, projectName, newSnapshotName, vol.Type()) }) // Generate source snapshot volumes list. srcSnapVolumeName := drivers.GetSnapshotVolumeName(srcVolName, srcSnap.Name) srcSnapVolStorageName := project.StorageVolume(projectName, srcSnapVolumeName) srcSnapVol := srcPool.GetVolume(drivers.VolumeTypeCustom, contentType, srcSnapVolStorageName, srcSnap.Config) srcSnapVols = append(srcSnapVols, srcSnapVol) } err = b.driver.RefreshVolume(vol, srcVol, srcSnapVols, false, op) if err != nil { return err } } else { l.Debug("RefreshCustomVolume cross-pool mode detected") // Negotiate the migration type to use. offeredTypes := srcPool.MigrationTypes(contentType, true, snapshots, false, true) offerHeader := localMigration.TypesToHeader(offeredTypes...) migrationTypes, err := localMigration.MatchTypes(offerHeader, FallbackMigrationType(contentType), b.MigrationTypes(contentType, true, snapshots, false, true)) if err != nil { return fmt.Errorf("Failed to negotiate copy migration type: %w", err) } var volSize int64 if contentType == drivers.ContentTypeBlock { err = srcVol.MountTask(func(mountPath string, op *operations.Operation) error { srcPoolBackend, ok := srcPool.(*backend) if !ok { return errors.New("Pool is not a backend") } volDiskPath, err := srcPoolBackend.driver.GetVolumeDiskPath(srcVol) if err != nil { return err } volSize, err = drivers.BlockDiskSizeBytes(volDiskPath) if err != nil { return err } return nil }, nil) if err != nil { return err } } var migrationSnapshots []*migration.Snapshot if snapshots { migrationSnapshots, err = VolumeSnapshotsToMigrationSnapshots(srcConfig.VolumeSnapshots, projectName, srcPool, contentType, drivers.VolumeTypeCustom, srcVolName) if err != nil { return err } } ctx, cancel := context.WithCancel(context.Background()) // Use in-memory pipe pair to simulate a connection between the sender and receiver. aEnd, bEnd := memorypipe.NewPipePair(ctx) // Run sender and receiver in separate go routines to prevent deadlocks. aEndErrCh := make(chan error, 1) bEndErrCh := make(chan error, 1) go func() { err := srcPool.MigrateCustomVolume(srcProjectName, aEnd, &localMigration.VolumeSourceArgs{ IndexHeaderVersion: localMigration.IndexHeaderVersion, Name: srcVolName, Snapshots: snapshotNames, MigrationType: migrationTypes[0], TrackProgress: true, // Do use a progress tracker on sender. ContentType: string(contentType), Info: &localMigration.Info{Config: srcConfig}, StorageMove: true, }, op) if err != nil { cancel() } aEndErrCh <- err }() go func() { err := b.CreateCustomVolumeFromMigration(projectName, bEnd, localMigration.VolumeTargetArgs{ IndexHeaderVersion: localMigration.IndexHeaderVersion, Name: volName, Description: desc, Config: config, Snapshots: migrationSnapshots, MigrationType: migrationTypes[0], TrackProgress: false, // Do not use a progress tracker on receiver. ContentType: string(contentType), VolumeSize: volSize, // Block size setting override. Refresh: true, StoragePool: srcPoolName, }, op) if err != nil { cancel() } bEndErrCh <- err }() // Capture errors from the sender and receiver from their result channels. errs := []error{} aEndErr := <-aEndErrCh if aEndErr != nil { _ = aEnd.Close() errs = append(errs, aEndErr) } bEndErr := <-bEndErrCh if bEndErr != nil { errs = append(errs, bEndErr) } cancel() if len(errs) > 0 { return fmt.Errorf("Refresh custom volume from copy failed: %v", errs) } } reverter.Success() return nil } // RefreshInstance synchronises one instance's volume (and optionally snapshots) over another. // Snapshots that are not present in the source but are in the destination are removed from the // destination if snapshots are included in the synchronisation. An empty srcSnapshots argument // indicates a volume-only refresh. func (b *backend) RefreshInstance(inst instance.Instance, src instance.Instance, srcSnapshots []instance.Instance, allowInconsistent bool, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "src": src.Name(), "srcSnapshots": len(srcSnapshots)}) l.Debug("RefreshInstance started") defer l.Debug("RefreshInstance finished") // This indicates whether or not it's a volume-only refresh. snapshots := len(srcSnapshots) > 0 if inst.Type() != src.Type() { return errors.New("Instance types must match") } volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentType := InstanceContentType(inst) // Load storage volume from database. dbVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return err } // Generate the effective root device volume for instance. volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(volType, contentType, volStorageName, dbVol.Config) err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return err } // Get the source storage pool. srcPool, err := LoadByInstance(b.state, src) if err != nil { return err } srcPoolBackend, ok := srcPool.(*backend) if !ok { return errors.New("Source pool is not a backend") } // Check source volume exists, and get its config. srcConfig, err := srcPool.GenerateInstanceBackupConfig(src, snapshots, true, op) if err != nil { return fmt.Errorf("Failed generating instance refresh config: %w", err) } // Ensure that only the requested snapshots are included in the source config. allSnapshots := srcConfig.VolumeSnapshots srcConfig.VolumeSnapshots = make([]*api.StorageVolumeSnapshot, 0, len(srcSnapshots)) for i := range allSnapshots { found := false for _, srcSnapshot := range srcSnapshots { _, srcSnapshotName, _ := api.GetParentAndSnapshotName(srcSnapshot.Name()) if srcSnapshotName == allSnapshots[i].Name { found = true break } } if found { srcConfig.VolumeSnapshots = append(srcConfig.VolumeSnapshots, allSnapshots[i]) } } // Get source volume construct. srcVolStorageName := project.Instance(src.Project().Name, src.Name()) srcVol := b.GetVolume(volType, contentType, srcVolStorageName, srcConfig.Volume.Config) // Get source snapshot volume constructs. srcSnapVols := make([]drivers.Volume, 0, len(srcConfig.VolumeSnapshots)) snapshotNames := make([]string, 0, len(srcConfig.VolumeSnapshots)) for i := range srcConfig.VolumeSnapshots { newSnapshotName := drivers.GetSnapshotVolumeName(src.Name(), srcConfig.VolumeSnapshots[i].Name) snapVolStorageName := project.Instance(src.Project().Name, newSnapshotName) srcSnapVol := srcPool.GetVolume(volType, contentType, snapVolStorageName, srcConfig.VolumeSnapshots[i].Config) srcSnapVols = append(srcSnapVols, srcSnapVol) snapshotNames = append(snapshotNames, srcConfig.VolumeSnapshots[i].Name) } reverter := revert.New() defer reverter.Fail() // Some driver backing stores require that running instances be frozen during copy. if !src.IsSnapshot() && srcPoolBackend.driver.Info().RunningCopyFreeze && src.IsRunning() && !src.IsFrozen() && !allowInconsistent { b.logger.Info("Freezing instance for consistent refresh") err = src.Freeze() if err != nil { return err } defer func() { _ = src.Unfreeze() }() // Attempt to sync the filesystem. _ = linux.SyncFS(src.RootfsPath()) } if b.Name() == srcPool.Name() { l.Debug("RefreshInstance same-pool mode detected") // Create database entries for new storage volume snapshots. for i := range srcConfig.VolumeSnapshots { newSnapshotName := drivers.GetSnapshotVolumeName(inst.Name(), srcConfig.VolumeSnapshots[i].Name) var volumeSnapExpiryDate time.Time if srcConfig.VolumeSnapshots[i].ExpiresAt != nil { volumeSnapExpiryDate = *srcConfig.VolumeSnapshots[i].ExpiresAt } // Validate config and create database entry for new storage volume. err = VolumeDBCreate(b, inst.Project().Name, newSnapshotName, srcConfig.VolumeSnapshots[i].Description, volType, true, srcConfig.VolumeSnapshots[i].Config, srcConfig.VolumeSnapshots[i].CreatedAt, volumeSnapExpiryDate, contentType, false, true) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, inst.Project().Name, newSnapshotName, volType) }) } err = b.driver.RefreshVolume(vol, srcVol, srcSnapVols, allowInconsistent, op) if err != nil { return err } } else { // We are copying volumes between storage pools so use migration system as it will // be able to negotiate a common transfer method between pool types. l.Debug("RefreshInstance cross-pool mode detected") // Negotiate the migration type to use. offeredTypes := srcPool.MigrationTypes(contentType, true, snapshots, false, true) offerHeader := localMigration.TypesToHeader(offeredTypes...) migrationTypes, err := localMigration.MatchTypes(offerHeader, FallbackMigrationType(contentType), b.MigrationTypes(contentType, true, snapshots, false, true)) if err != nil { return fmt.Errorf("Failed to negotiate copy migration type: %w", err) } var srcVolumeSize int64 // For VMs, get source volume size so that target can create the volume the same size. if src.Type() == instancetype.VM { srcVolumeSize, err = InstanceDiskBlockSize(srcPool, src, op) if err != nil { return fmt.Errorf("Failed getting source disk size: %w", err) } } migrationSnapshots, err := VolumeSnapshotsToMigrationSnapshots(srcConfig.VolumeSnapshots, src.Project().Name, srcPool, contentType, volType, src.Name()) if err != nil { return err } ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Run sender and receiver in separate go routines to prevent deadlocks. g, ctx := errgroup.WithContext(ctx) // Use in-memory pipe pair to simulate a connection between the sender and receiver. // Use context from error group so that if either side fails the pipes are closed. aEnd, bEnd := memorypipe.NewPipePair(ctx) // Start each side of the migration concurrently and collect any errors. g.Go(func() error { return srcPool.MigrateInstance(src, aEnd, &localMigration.VolumeSourceArgs{ IndexHeaderVersion: localMigration.IndexHeaderVersion, Name: src.Name(), Snapshots: snapshotNames, MigrationType: migrationTypes[0], TrackProgress: true, // Do use a progress tracker on sender. AllowInconsistent: allowInconsistent, Refresh: true, // Indicate to sender to use incremental streams. Info: &localMigration.Info{Config: srcConfig}, VolumeOnly: !snapshots, StorageMove: true, }, op) }) g.Go(func() error { return b.CreateInstanceFromMigration(inst, bEnd, localMigration.VolumeTargetArgs{ IndexHeaderVersion: localMigration.IndexHeaderVersion, Name: inst.Name(), Snapshots: migrationSnapshots, MigrationType: migrationTypes[0], Refresh: true, // Indicate to receiver volume should exist. VolumeSize: srcVolumeSize, TrackProgress: false, // Do not use a progress tracker on receiver. VolumeOnly: !snapshots, StoragePool: srcPool.Name(), }, op) }) err = g.Wait() if err != nil { return fmt.Errorf("Create instance volume from copy failed: %w", err) } } err = b.ensureInstanceSymlink(inst.Type(), inst.Project().Name, inst.Name(), vol.MountPath()) if err != nil { return err } err = inst.DeferTemplateApply(instance.TemplateTriggerCopy) if err != nil { return err } reverter.Success() return nil } // imageFiller returns a function that can be used as a filler function with CreateVolume(). // The function returned will unpack the specified image archive into the specified mount path // provided, and for VM images, a raw root block path is required to unpack the qcow2 image into. func (b *backend) imageFiller(fingerprint string, op *operations.Operation) func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { return func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { var tracker *ioprogress.ProgressTracker if op != nil { // Not passed when being done as part of pre-migration setup. metadata := make(map[string]any) tracker = &ioprogress.ProgressTracker{ Handler: func(percent, speed int64) { operations.SetProgressMetadata(metadata, "create_instance_from_image_unpack", "Unpacking image", percent, 0, speed) _ = op.UpdateMetadata(metadata) }, } } imageFile := internalUtil.VarPath("images", fingerprint) return ImageUnpack(imageFile, vol, rootBlockPath, b.state.OS, allowUnsafeResize, targetIsZero, tracker, targetFormat) } } // isoFiller returns a function that can be used as a filler function with CreateVolume(). // The function returned will copy the ISO content into the specified mount path // provided. func (b *backend) isoFiller(data io.Reader) func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { return func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { f, err := os.OpenFile(rootBlockPath, os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { return -1, err } defer func() { _ = f.Close() }() return util.SafeCopy(f, data) } } // CreateInstanceFromImage creates a new volume for an instance populated with the image requested. // On failure caller is expected to call DeleteInstance() to clean up. func (b *backend) CreateInstanceFromImage(inst instance.Instance, fingerprint string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) l.Debug("CreateInstanceFromImage started") defer l.Debug("CreateInstanceFromImage finished") err := b.isStatusReady() if err != nil { return err } volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentType := InstanceContentType(inst) reverter := revert.New() defer reverter.Fail() volumeConfig := make(map[string]string) err = b.applyInstanceRootDiskInitialValues(inst, volumeConfig) if err != nil { return err } // Determine whether an optimized image should be used. useOptimizedImage, err := b.shouldUseOptimizedImage(fingerprint, contentType, volumeConfig, op) if err != nil { return err } // Validate config and create database entry for new storage volume. err = VolumeDBCreate(b, inst.Project().Name, inst.Name(), "", volType, false, volumeConfig, inst.CreationDate(), time.Time{}, contentType, true, false) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, inst.Project().Name, inst.Name(), volType) }) // Record new volume with authorizer. err = b.state.Authorizer.AddStoragePoolVolume(b.state.ShutdownCtx, inst.Project().Name, b.Name(), volType.Singular(), inst.Name(), "") if err != nil { logger.Error("Failed to add storage volume to authorizer", logger.Ctx{"name": inst.Name(), "type": volType, "pool": b.Name(), "project": inst.Project().Name, "error": err}) } reverter.Add(func() { _ = b.state.Authorizer.DeleteStoragePoolVolume(b.state.ShutdownCtx, inst.Project().Name, b.Name(), volType.Singular(), inst.Name(), "") }) // Generate the effective root device volume for instance. volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(volType, contentType, volStorageName, volumeConfig) err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return err } // Leave reverting on failure to caller, they are expected to call DeleteInstance(). // If the driver doesn't support optimized image volumes or the optimized image volume should not be used, // create a new empty volume and populate it with the contents of the image archive. if !useOptimizedImage { volFiller := drivers.VolumeFiller{ Fingerprint: fingerprint, Fill: b.imageFiller(fingerprint, op), } err = b.driver.CreateVolume(vol, &volFiller, op) if err != nil { return err } } else { // If the driver supports optimized images then ensure the optimized image volume has been created // for the images's fingerprint and that it matches the pool's current volume settings, and if not // recreating using the pool's current volume settings. err = b.EnsureImage(fingerprint, op) if err != nil { return err } // Try and load existing volume config on this storage pool so we can compare filesystems if needed. imgDBVol, err := VolumeDBGet(b, api.ProjectDefaultName, fingerprint, drivers.VolumeTypeImage) if err != nil { return err } imgVol := b.GetVolume(drivers.VolumeTypeImage, contentType, fingerprint, imgDBVol.Config) // Derive the volume size to use for a new volume when copying from a source volume. // Where possible (if the source volume has a volatile.rootfs.size property), it checks that the // source volume isn't larger than the volume's "size" and the pool's "volume.size" setting. l.Debug("Checking volume size") newVolSize, err := vol.ConfigSizeFromSource(imgVol) if err != nil { return err } // Set the derived size directly as the "size" property on the new volume so that it is applied. vol.SetConfigSize(newVolSize) l.Debug("Set new volume size", logger.Ctx{"size": newVolSize}) // Proceed to create a new volume by copying the optimized image volume. err = b.driver.CreateVolumeFromCopy(vol, imgVol, false, false, op) // If the driver returns ErrCannotBeShrunk, this means that the cached volume that the new volume // is to be created from is larger than the requested new volume size, and cannot be shrunk. // So we unpack the image directly into a new volume rather than use the optimized snapsot. // This is slower but allows for individual volumes to be created from an image that are smaller // than the pool's volume settings. if errors.Is(err, drivers.ErrCannotBeShrunk) { l.Debug("Cached image volume is larger than new volume and cannot be shrunk, creating non-optimized volume") volFiller := drivers.VolumeFiller{ Fingerprint: fingerprint, Fill: b.imageFiller(fingerprint, op), } err = b.driver.CreateVolume(vol, &volFiller, op) if err != nil { return err } } else if err != nil { return err } } err = b.ensureInstanceSymlink(inst.Type(), inst.Project().Name, inst.Name(), vol.MountPath()) if err != nil { return err } err = inst.DeferTemplateApply(instance.TemplateTriggerCreate) if err != nil { return err } reverter.Success() return nil } // CreateInstanceFromMigration receives an instance being migrated. // The args.Name and args.Config fields are ignored and, instance properties are used instead. func (b *backend) CreateInstanceFromMigration(inst instance.Instance, conn io.ReadWriteCloser, args localMigration.VolumeTargetArgs, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "args": fmt.Sprintf("%+v", args)}) l.Debug("CreateInstanceFromMigration started") defer l.Debug("CreateInstanceFromMigration finished") err := b.isStatusReady() if err != nil { return err } if args.Config != nil { return errors.New("Migration VolumeTargetArgs.Config cannot be set for instances") } volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentType := InstanceContentType(inst) // Receive index header from source if applicable and respond confirming receipt. // This will also communicate the args.Refresh setting back to the source (in case it was changed by the // caller if the instance DB record already exists). srcInfo, err := b.migrationIndexHeaderReceive(l, args.IndexHeaderVersion, conn, args.Refresh) if err != nil { return err } reverter := revert.New() defer reverter.Fail() if !inst.IsSnapshot() && srcInfo.Config != nil && srcInfo.Config.Container != nil { // Create dependent volumes if they exist. cleanupDependentVols, err := b.createDependentVolumesFromMigration(inst, conn, args, srcInfo, op) if err != nil { return err } reverter.Add(func() { cleanupDependentVols() }) } // Now that we got the source details, validate against the instance limits. _, rootDiskConf, err := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if err != nil { return err } if rootDiskConf["size"] != "" { rootDiskConfBytes, err := units.ParseByteSizeString(rootDiskConf["size"]) if err != nil { return err } // Compare volume size with configured root size. // Add a 4MiB allowed extra to account for round to nearest extent (16k on ZFS, 4MiB on LVM). if args.VolumeSize > (rootDiskConfBytes + (4 * 1024 * 1024)) { return errors.New("The configured target instance root disk size is smaller than the migration source") } } var volumeDescription string var volumeConfig map[string]string // Check if the volume exists in database dbVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil && !response.IsNotFoundError(err) { return err } // Prefer using existing volume config (to allow mounting existing volume correctly). if dbVol != nil { volumeConfig = dbVol.Config volumeDescription = dbVol.Description } else if srcInfo != nil && srcInfo.Config != nil && srcInfo.Config.Volume != nil { volumeConfig = srcInfo.Config.Volume.Config volumeDescription = srcInfo.Config.Volume.Description } else { volumeConfig = make(map[string]string) volumeDescription = args.Description } volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(volType, contentType, volStorageName, volumeConfig) // Ensure storage volume settings are honored when doing migration. // This is only done for non-optimized migration because some storage volume settings, // in particular block mode, cannot be honored when doing optimized migration. if args.MigrationType.FSType == migration.MigrationFSType_RSYNC || args.MigrationType.FSType == migration.MigrationFSType_BLOCK_AND_RSYNC { vol.SetHasSource(false) err = b.driver.FillVolumeConfig(vol) if err != nil { return fmt.Errorf("Failed filling volume config: %w", err) } } // Check if the volume exists on storage. volExists, err := b.driver.HasVolume(vol) if err != nil { return err } // Check for inconsistencies between database and storage before continuing. if dbVol == nil && volExists { return errors.New("Volume already exists on storage but not in database") } if dbVol != nil && !volExists { return errors.New("Volume exists in database but not on storage") } // Consistency check for refresh mode. // We expect that the args.Refresh setting will have already been set to false by the caller as part of // detecting if the instance DB record exists or not. If we get here then something has gone wrong. if args.Refresh && !volExists { return errors.New("Cannot refresh volume, doesn't exist on migration target storage") } isRemoteClusterMove := args.ClusterMoveSourceName != "" && b.driver.Info().Remote if !args.Refresh { if volExists { if !isRemoteClusterMove { return errors.New("Cannot create volume, already exists on migration target storage") } } else { // Validate config and create database entry for new storage volume if not refreshing. // Strip unsupported config keys (in case the export was made from a different type of storage pool). err = VolumeDBCreate(b, inst.Project().Name, inst.Name(), volumeDescription, volType, false, vol.Config(), inst.CreationDate(), time.Time{}, contentType, true, true) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, inst.Project().Name, inst.Name(), volType) }) // Record new volume with authorizer. err = b.state.Authorizer.AddStoragePoolVolume(b.state.ShutdownCtx, inst.Project().Name, b.Name(), volType.Singular(), inst.Name(), "") if err != nil { logger.Error("Failed to add storage volume to authorizer", logger.Ctx{"name": inst.Name(), "type": volType, "pool": b.Name(), "project": inst.Project().Name, "error": err}) } reverter.Add(func() { _ = b.state.Authorizer.DeleteStoragePoolVolume(b.state.ShutdownCtx, inst.Project().Name, b.Name(), volType.Singular(), inst.Name(), "") }) } } // Create new volume database records when the storage pool is changed or // when it is not a remote cluster move. if !isRemoteClusterMove || args.StoragePool != "" { for i, snapshot := range args.Snapshots { snapName := snapshot.GetName() newSnapshotName := drivers.GetSnapshotVolumeName(inst.Name(), snapName) snapConfig := vol.Config() // Use parent volume config by default. snapDescription := volumeDescription // Use parent volume description by default. snapExpiryDate := time.Time{} snapCreationDate := time.Time{} // If the source snapshot config is available, use that. if srcInfo != nil && srcInfo.Config != nil { if len(srcInfo.Config.Snapshots) > i && srcInfo.Config.Snapshots[i] != nil && srcInfo.Config.Snapshots[i].Name == snapName { // Use instance snapshot's creation date if snap info available. snapCreationDate = srcInfo.Config.Snapshots[i].CreatedAt } if len(srcInfo.Config.VolumeSnapshots) > i && srcInfo.Config.VolumeSnapshots[i] != nil && srcInfo.Config.VolumeSnapshots[i].Name == snapName { // Check if snapshot volume config is available then use it. snapDescription = srcInfo.Config.VolumeSnapshots[i].Description snapConfig = srcInfo.Config.VolumeSnapshots[i].Config if srcInfo.Config.VolumeSnapshots[i].ExpiresAt != nil { snapExpiryDate = *srcInfo.Config.VolumeSnapshots[i].ExpiresAt } // Use volume's creation date if available. if !srcInfo.Config.VolumeSnapshots[i].CreatedAt.IsZero() { snapCreationDate = srcInfo.Config.VolumeSnapshots[i].CreatedAt } } } // Validate config and create database entry for new storage volume. // Strip unsupported config keys (in case the export was made from a different type of storage pool). err = VolumeDBCreate(b, inst.Project().Name, newSnapshotName, snapDescription, volType, true, snapConfig, snapCreationDate, snapExpiryDate, contentType, true, true) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, inst.Project().Name, newSnapshotName, volType) }) } } // Generate the effective root device volume for instance. err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return err } // Override args.Name and args.Config to ensure volume is created based on instance. args.Config = vol.Config() args.Name = inst.Name() projectName := inst.Project().Name // If migration header supplies a volume size, then use that as block volume size instead of pool default. // This way if the volume being received is larger than the pool default size, the block volume created // will still be able to accommodate it. if args.VolumeSize > 0 && contentType == drivers.ContentTypeBlock { b.logger.Debug("Setting volume size from offer header", logger.Ctx{"size": args.VolumeSize}) args.Config["size"] = fmt.Sprintf("%d", args.VolumeSize) } else if args.Config["size"] != "" { b.logger.Debug("Using volume size from root disk config", logger.Ctx{"size": args.Config["size"]}) } var preFiller drivers.VolumeFiller if !args.Refresh && !isRemoteClusterMove { // If the negotiated migration method is rsync and the instance's base image is // already on the host then setup a pre-filler that will unpack the local image // to try and speed up the rsync of the incoming volume by avoiding the need to // transfer the base image files too. if args.MigrationType.FSType == migration.MigrationFSType_RSYNC { fingerprint := inst.ExpandedConfig()["volatile.base_image"] imageExists := false if fingerprint != "" { err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Confirm that the image is present in the project. _, _, err = tx.GetImage(ctx, fingerprint, cluster.ImageFilter{Project: &projectName}) return err }) if err != nil && !response.IsNotFoundError(err) { return err } // Make sure that the image is available locally too (not guaranteed in clusters). imageExists = err == nil && util.PathExists(internalUtil.VarPath("images", fingerprint)) } if imageExists { l.Debug("Using optimised migration from existing image", logger.Ctx{"fingerprint": fingerprint}) // Populate the volume filler with the fingerprint and image filler // function that can be used by the driver to pre-populate the // volume with the contents of the image. preFiller = drivers.VolumeFiller{ Fingerprint: fingerprint, Fill: b.imageFiller(fingerprint, op), } // Ensure if the image doesn't yet exist on a driver which supports // optimized storage, then it gets created first. err = b.EnsureImage(preFiller.Fingerprint, op) if err != nil { return err } } } } if b.driver.Info().TargetFormat == drivers.BlockVolumeTypeQcow2 && (!b.driver.Info().Remote || args.ClusterMoveSourceName == "" || args.StoragePool != "") { err = b.qcow2CreateVolumeFromMigration(vol, inst.Project().Name, conn, args, &preFiller, op) if err != nil { return err } } else { err = b.driver.CreateVolumeFromMigration(vol, conn, args, &preFiller, op) if err != nil { return err } } if !isRemoteClusterMove { reverter.Add(func() { _ = b.DeleteInstance(inst, op) }) } err = b.ensureInstanceSymlink(inst.Type(), inst.Project().Name, inst.Name(), vol.MountPath()) if err != nil { return err } if len(args.Snapshots) > 0 { err = b.ensureInstanceSnapshotSymlink(inst.Type(), inst.Project().Name, inst.Name()) if err != nil { return err } } reverter.Success() return nil } // RenameInstance renames the instance's root volume and any snapshot volumes. func (b *backend) RenameInstance(inst instance.Instance, newName string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "newName": newName}) l.Debug("RenameInstance started") defer l.Debug("RenameInstance finished") if inst.IsSnapshot() { return errors.New("Instance cannot be a snapshot") } if internalInstance.IsSnapshot(newName) { return errors.New("New name cannot be a snapshot") } // Check we can convert the instance to the volume types needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } volDBType, err := VolumeTypeToDBType(volType) if err != nil { return err } reverter := revert.New() defer reverter.Fail() volume, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil && !response.IsNotFoundError(err) { return err } var snapshots []string err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get any snapshots the instance has in the format /. snapshots, err = tx.GetInstanceSnapshotsNames(ctx, inst.Project().Name, inst.Name()) return err }) if err != nil { return err } if len(snapshots) > 0 { reverter.Add(func() { _ = b.removeInstanceSnapshotSymlinkIfUnused(inst.Type(), inst.Project().Name, newName) _ = b.ensureInstanceSnapshotSymlink(inst.Type(), inst.Project().Name, inst.Name()) }) } volStorageName := project.Instance(inst.Project().Name, inst.Name()) newVolStorageName := project.Instance(inst.Project().Name, newName) contentType := InstanceContentType(inst) vol := b.GetVolume(volType, contentType, volStorageName, volume.Config) // Perform qcow2 renaming before renaming database volumes. if volume.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { err = b.qcow2Rename(vol, newName, inst.Project().Name, op) if err != nil { return err } } // Rename each snapshot DB record to have the new parent volume prefix. for _, srcSnapshot := range snapshots { _, snapName, _ := api.GetParentAndSnapshotName(srcSnapshot) newSnapVolName := drivers.GetSnapshotVolumeName(newName, snapName) err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RenameStoragePoolVolume(ctx, inst.Project().Name, srcSnapshot, newSnapVolName, volDBType, b.ID()) }) if err != nil { return err } reverter.Add(func() { _ = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RenameStoragePoolVolume(ctx, inst.Project().Name, newSnapVolName, srcSnapshot, volDBType, b.ID()) }) }) } err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Rename the parent volume DB record. return tx.RenameStoragePoolVolume(ctx, inst.Project().Name, inst.Name(), newName, volDBType, b.ID()) }) if err != nil { return err } reverter.Add(func() { _ = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RenameStoragePoolVolume(ctx, inst.Project().Name, newName, inst.Name(), volDBType, b.ID()) }) }) // Rename the volume and its snapshots on the storage device. err = b.driver.RenameVolume(vol, newVolStorageName, op) if err != nil { return err } reverter.Add(func() { // There's no need to pass config as it's not needed when renaming a volume. newVol := b.GetVolume(volType, contentType, newVolStorageName, nil) _ = b.driver.RenameVolume(newVol, volStorageName, op) }) // Remove old instance symlink and create new one. err = b.removeInstanceSymlink(inst.Type(), inst.Project().Name, inst.Name()) if err != nil { return err } reverter.Add(func() { _ = b.ensureInstanceSymlink(inst.Type(), inst.Project().Name, inst.Name(), drivers.GetVolumeMountPath(b.name, volType, volStorageName)) }) err = b.ensureInstanceSymlink(inst.Type(), inst.Project().Name, newName, drivers.GetVolumeMountPath(b.name, volType, newVolStorageName)) if err != nil { return err } reverter.Add(func() { _ = b.removeInstanceSymlink(inst.Type(), inst.Project().Name, newName) }) // Remove old instance snapshot symlink and create a new one if needed. err = b.removeInstanceSnapshotSymlinkIfUnused(inst.Type(), inst.Project().Name, inst.Name()) if err != nil { return err } if len(snapshots) > 0 { err = b.ensureInstanceSnapshotSymlink(inst.Type(), inst.Project().Name, newName) if err != nil { return err } } // Record volume rename with authorizer. err = b.state.Authorizer.RenameStoragePoolVolume(b.state.ShutdownCtx, inst.Project().Name, b.Name(), vol.Type().Singular(), inst.Name(), newName, "") if err != nil { logger.Error("Failed to rename storage volume in authorizer", logger.Ctx{"name": inst.Name(), "newName": newName, "type": vol.Type(), "pool": b.Name(), "project": inst.Project().Name, "error": err}) } reverter.Success() return nil } // DeleteInstance removes the instance's root volume (all snapshots need to be removed first). func (b *backend) DeleteInstance(inst instance.Instance, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) l.Debug("DeleteInstance started") defer l.Debug("DeleteInstance finished") if inst.IsSnapshot() { return errors.New("Instance must not be a snapshot") } // Check we can convert the instance to the volume types needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } // Get any snapshot volume DB records that the instance has. dbVolSnaps, err := VolumeDBSnapshotsGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return err } // Check all snapshots are already removed. if len(dbVolSnaps) > 0 { return errors.New("Cannot remove an instance volume that has snapshots") } // Get the volume name on storage. volStorageName := project.Instance(inst.Project().Name, inst.Name()) contentType := InstanceContentType(inst) // There's no need to pass config as it's not needed when deleting a volume. vol := b.GetVolume(volType, contentType, volStorageName, nil) // Delete the volume from the storage device. Must come after snapshots are removed. // Must come before DB VolumeDBDelete so that the volume ID is still available. l.Debug("Deleting instance volume", logger.Ctx{"volName": volStorageName}) volExists, err := b.driver.HasVolume(vol) if err != nil { return err } if volExists { err = b.driver.DeleteVolume(vol, op) if err != nil { return fmt.Errorf("Error deleting storage volume: %w", err) } } // Remove symlinks. err = b.removeInstanceSymlink(inst.Type(), inst.Project().Name, inst.Name()) if err != nil { return err } err = b.removeInstanceSnapshotSymlinkIfUnused(inst.Type(), inst.Project().Name, inst.Name()) if err != nil { return err } // Remove the volume record from the database. err = VolumeDBDelete(b, inst.Project().Name, inst.Name(), vol.Type()) if err != nil { return err } // Record volume deletion with authorizer. err = b.state.Authorizer.DeleteStoragePoolVolume(b.state.ShutdownCtx, inst.Project().Name, b.Name(), vol.Type().Singular(), inst.Name(), "") if err != nil { logger.Error("Failed to remove storage volume from authorizer", logger.Ctx{"name": inst.Name(), "type": vol.Type(), "pool": b.Name(), "project": inst.Project().Name, "error": err}) } return nil } // UpdateInstance updates an instance volume's config. func (b *backend) UpdateInstance(inst instance.Instance, newDesc string, newConfig map[string]string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "newDesc": newDesc, "newConfig": newConfig}) l.Debug("UpdateInstance started") defer l.Debug("UpdateInstance finished") if inst.IsSnapshot() { return errors.New("Instance cannot be a snapshot") } // Check we can convert the instance to the volume types needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } volDBType, err := VolumeTypeToDBType(volType) if err != nil { return err } volStorageName := project.Instance(inst.Project().Name, inst.Name()) contentType := InstanceContentType(inst) // Validate config. newVol := b.GetVolume(volType, contentType, volStorageName, newConfig) err = b.driver.ValidateVolume(newVol, false) if err != nil { return err } // Get current config to compare what has changed. curVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return err } // Apply config changes if there are any. changedConfig, userOnly := b.detectChangedConfig(curVol.Config, newConfig) if len(changedConfig) != 0 { // Check that the volume's size property isn't being changed. if changedConfig["size"] != "" { return errors.New(`Instance volume "size" property cannot be changed`) } // Check that the volume's size.state property isn't being changed. if changedConfig["size.state"] != "" { return errors.New(`Instance volume "size.state" property cannot be changed`) } // Check that the volume's block.filesystem property isn't being changed. if changedConfig["block.filesystem"] != "" { return errors.New(`Instance volume "block.filesystem" property cannot be changed`) } // Load storage volume from database. dbVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return err } // Generate the effective root device volume for instance. volStorageName := project.Instance(inst.Project().Name, inst.Name()) curVol := b.GetVolume(volType, contentType, volStorageName, dbVol.Config) err = b.applyInstanceRootDiskOverrides(inst, &curVol) if err != nil { return err } if !userOnly { err = b.driver.UpdateVolume(curVol, changedConfig) if err != nil { return err } } } // Update the database if something changed. if len(changedConfig) != 0 || newDesc != curVol.Description { err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateStoragePoolVolume(ctx, inst.Project().Name, inst.Name(), volDBType, b.ID(), newDesc, newConfig) }) if err != nil { return err } } b.state.Events.SendLifecycle(inst.Project().Name, lifecycle.StorageVolumeUpdated.Event(newVol, string(newVol.Type()), inst.Project().Name, op, nil)) return nil } // UpdateInstanceSnapshot updates an instance snapshot volume's description. // Volume config is not allowed to be updated and will return an error. func (b *backend) UpdateInstanceSnapshot(inst instance.Instance, newDesc string, newConfig map[string]string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "newDesc": newDesc, "newConfig": newConfig}) l.Debug("UpdateInstanceSnapshot started") defer l.Debug("UpdateInstanceSnapshot finished") if !inst.IsSnapshot() { return errors.New("Instance must be a snapshot") } // Check we can convert the instance to the volume types needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } return b.updateVolumeDescriptionOnly(inst.Project().Name, inst.Name(), volType, newDesc, newConfig, op) } // MigrateInstance sends an instance volume for migration. // The args.Name field is ignored and the name of the instance is used instead. func (b *backend) MigrateInstance(inst instance.Instance, conn io.ReadWriteCloser, args *localMigration.VolumeSourceArgs, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "args": fmt.Sprintf("%+v", args)}) l.Debug("MigrateInstance started") defer l.Debug("MigrateInstance finished") volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentType := InstanceContentType(inst) if len(args.Snapshots) > 0 && args.FinalSync { return errors.New("Snapshots should not be transferred during final sync") } if args.Info == nil { return errors.New("Migration info required") } if args.Info.Config == nil || args.Info.Config.Volume == nil || args.Info.Config.Volume.Config == nil { return errors.New("Volume config is required") } if len(args.Snapshots) != len(args.Info.Config.VolumeSnapshots) { return fmt.Errorf("Requested snapshots count (%d) doesn't match volume snapshot config count (%d)", len(args.Snapshots), len(args.Info.Config.VolumeSnapshots)) } // Load storage volume from database. dbVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return err } // Generate the effective root device volume for instance. volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(volType, contentType, volStorageName, dbVol.Config) err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return err } args.Name = inst.Name() // Override args.Name to ensure instance volume is sent. // Send migration index header frame with volume info and wait for receipt if not doing final sync. if !args.FinalSync { resp, err := b.migrationIndexHeaderSend(l, args.IndexHeaderVersion, conn, args.Info) if err != nil { return err } if resp.Refresh != nil { args.Refresh = *resp.Refresh } } if !inst.IsSnapshot() && args.Info.Config != nil && args.Info.Config.Container != nil { // Migrate dependent volumes if they exist. err = b.migrateDependentVolumes(inst, conn, args, op) if err != nil { return err } } // Detect if source pool driver doesn't support cheap temporary snapshots that allow consistent copy when // running, or if the negotiated protocol is VM non-optimized, meaning a complete raw copy of the active // volume is being sent. // TODO this can be relaxed in the future if the storage drivers that have RunningCopyFreeze=false make // temporary snapshots for block volumes too. But for now this is not the case and we must detect when a // generic migration transfer protocol has been negotiated between source and target pools. runningCopyFreeze := b.driver.Info().RunningCopyFreeze || args.MigrationType.FSType == migration.MigrationFSType_BLOCK_AND_RSYNC // Freeze the instance if not already frozen/stopped, allowInconsistent is not enabled and when its not // possible to make a consistent copy with the instance running. if !inst.IsSnapshot() && runningCopyFreeze && inst.IsRunning() && !inst.IsFrozen() && !args.AllowInconsistent { b.logger.Info("Freezing instance for consistent migration transfer") err = inst.Freeze() if err != nil { return err } defer func() { _ = inst.Unfreeze() }() // Attempt to sync the filesystem. _ = linux.SyncFS(inst.RootfsPath()) } if dbVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 && (!b.driver.Info().Remote || !args.ClusterMove || args.StorageMove) { err = b.qcow2MigrateVolume(b.state, vol, inst.Project().Name, conn, args, op) if err != nil { return err } } else { err = b.driver.MigrateVolume(vol, conn, args, op) if err != nil { return err } } return nil } // CleanupInstancePaths removes any remaining mount paths and symlinks for the instance and its snapshots. func (b *backend) CleanupInstancePaths(inst instance.Instance, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) l.Debug("CleanupInstancePaths started") defer l.Debug("CleanupInstancePaths finished") if inst.IsSnapshot() { return errors.New("Instance must not be a snapshot") } // Check we can convert the instance to the volume types needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } // Get the volume name on storage. volStorageName := project.Instance(inst.Project().Name, inst.Name()) contentType := InstanceContentType(inst) // There's no need to pass config as it's not needed when deleting a volume. vol := b.GetVolume(volType, contentType, volStorageName, nil) // Remove empty snapshot mount paths. snapshotDir := drivers.GetVolumeSnapshotDir(b.Name(), vol.Type(), vol.Name()) ents, err := os.ReadDir(snapshotDir) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed listing instance snapshots directory %q: %w", snapshotDir, err) } for _, ent := range ents { filePath := filepath.Join(snapshotDir, ent.Name()) fileInfo, err := os.Stat(filePath) if err != nil { return err } if !fileInfo.IsDir() { continue } // Remove empty snapshot mount path. err = os.Remove(filePath) if err != nil { return fmt.Errorf("Failed removing instance snapshot mount path %q: %w", filePath, err) } } err = os.Remove(snapshotDir) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed removing instance snapshots directory %q: %w", snapshotDir, err) } // Remove empty mount path. err = os.Remove(vol.MountPath()) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed removing instance mount path %q: %w", vol.MountPath(), err) } // Remove symlinks. err = b.removeInstanceSymlink(inst.Type(), inst.Project().Name, inst.Name()) if err != nil { return fmt.Errorf("Failed removing instance symlink: %w", err) } err = b.removeInstanceSnapshotSymlinkIfUnused(inst.Type(), inst.Project().Name, inst.Name()) if err != nil { return fmt.Errorf("Failed removing instance snapshots symlink: %w", err) } return nil } // BackupInstance creates an instance backup. func (b *backend) BackupInstance(inst instance.Instance, tarWriter *instancewriter.InstanceTarWriter, optimized bool, snapshots bool, dependentVolumes bool, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "optimized": optimized, "snapshots": snapshots}) l.Debug("BackupInstance started") defer l.Debug("BackupInstance finished") volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentType := InstanceContentType(inst) // Load storage volume from database. dbVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return err } // Generate the effective root device volume for instance. volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(volType, contentType, volStorageName, dbVol.Config) err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return err } // Ensure the backup file reflects current config. err = b.UpdateInstanceBackupFile(inst, snapshots, op) if err != nil { return err } defer func() { _ = b.UpdateInstanceBackupFile(inst, true, nil) }() var snapNames []string if snapshots { // Get snapshots in age order, oldest first, and pass names to storage driver. instSnapshots, err := inst.Snapshots() if err != nil { return err } snapNames = make([]string, 0, len(instSnapshots)) for _, instSnapshot := range instSnapshots { _, snapName, _ := api.GetParentAndSnapshotName(instSnapshot.Name()) snapNames = append(snapNames, snapName) } } if dbVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { err = b.qcow2BackupVolume(vol, dbVol, inst.Project().Name, tarWriter, backup.DefaultBackupPrefix, snapNames, op) if err != nil { return err } } else { err = b.driver.BackupVolume(vol, tarWriter, backup.DefaultBackupPrefix, optimized, snapNames, op) if err != nil { return err } } if dependentVolumes { err = inst.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error { // Load the pool for the disk. diskPool, err := LoadByName(b.state, dev.Config["pool"]) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } err = diskPool.BackupCustomVolume(inst.Project().Name, dev.Config["source"], tarWriter, filepath.Join(backup.DefaultBackupPrefix, dev.Name), optimized, snapshots, op) if err != nil { return err } return nil }) if err != nil { return err } } return nil } // GetInstanceUsage returns the disk usage of the instance's root volume. func (b *backend) GetInstanceUsage(inst instance.Instance) (*VolumeUsage, error) { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) l.Debug("GetInstanceUsage started") defer l.Debug("GetInstanceUsage finished") err := b.isStatusReady() if err != nil { return nil, err } volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return nil, err } contentType := InstanceContentType(inst) val := VolumeUsage{} // There's no need to pass config as it's not needed when retrieving the volume usage. volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(volType, contentType, volStorageName, nil) // Get the usage. size, err := b.driver.GetVolumeUsage(vol) if err != nil { return nil, err } val.Used = size // Get the total size. _, rootDiskConf, err := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if err != nil { return nil, err } sizeStr, ok := rootDiskConf["size"] if !ok && volType == drivers.VolumeTypeVM { sizeStr = drivers.DefaultBlockSize } if sizeStr != "" { total, err := units.ParseByteSizeString(sizeStr) if err != nil { return nil, err } if total >= 0 { val.Total = total } } return &val, nil } // SetInstanceQuota sets the quota on the instance's root volume. // Returns ErrInUse if the instance is running and the storage driver doesn't support online resizing. func (b *backend) SetInstanceQuota(inst instance.Instance, size string, vmStateSize string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "size": size, "vm_state_size": vmStateSize}) l.Debug("SetInstanceQuota started") defer l.Debug("SetInstanceQuota finished") // Check we can convert the instance to the volume type needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentVolume := InstanceContentType(inst) volStorageName := project.Instance(inst.Project().Name, inst.Name()) // Load storage volume from database. dbVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return err } // Apply the main volume quota. // There's no need to pass config as it's not needed when setting quotas. vol := b.GetVolume(volType, contentVolume, volStorageName, dbVol.Config) err = b.driver.SetVolumeQuota(vol, size, false, op) if err != nil { return err } // Apply the filesystem volume quota (only when main volume is block). if vol.IsVMBlock() { // Apply default VM config filesystem size if main volume size is specified and no custom // vmStateSize is specified. This way if the main volume size is empty (i.e removing quota) then // this will also pass empty quota for the config filesystem volume as well, allowing a former // quota to be removed from both volumes. if vmStateSize == "" && size != "" { vmStateSize = b.driver.Info().DefaultVMBlockFilesystemSize } fsVol := vol.NewVMBlockFilesystemVolume() err := b.driver.SetVolumeQuota(fsVol, vmStateSize, false, op) if err != nil { return err } } if drivers.IsQcow2Block(vol) { err = b.qcow2Resize(inst, vol, size, op) if err != nil { return err } } return nil } // MountInstance mounts the instance's root volume. func (b *backend) MountInstance(inst instance.Instance, op *operations.Operation) (*MountInfo, error) { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) l.Debug("MountInstance started") defer l.Debug("MountInstance finished") err := b.isStatusReady() if err != nil { return nil, err } reverter := revert.New() defer reverter.Fail() // Check we can convert the instance to the volume type needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return nil, err } contentType := InstanceContentType(inst) // Get the volume. var vol drivers.Volume volStorageName := project.Instance(inst.Project().Name, inst.Name()) if inst.ID() > -1 { // Load storage volume from database. dbVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return nil, err } // Generate the effective root device volume for instance. vol = b.GetVolume(volType, contentType, volStorageName, dbVol.Config) err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return nil, err } } else { contentType := InstanceContentType(inst) vol = b.GetVolume(volType, contentType, volStorageName, nil) } err = b.driver.MountVolume(vol, op) if err != nil { return nil, err } reverter.Add(func() { _, _ = b.driver.UnmountVolume(vol, false, op) }) diskPath, err := b.getInstanceDisk(inst) if err != nil && !errors.Is(err, drivers.ErrNotSupported) { return nil, fmt.Errorf("Failed getting disk path: %w", err) } backingPaths := []string{} if vol.Config()["block.type"] == drivers.BlockVolumeTypeQcow2 { // Get snapshots. volSnaps, err := VolumeDBSnapshotsGet(b, inst.Project().Name, inst.Name(), vol.Type()) if err != nil { return nil, err } for _, snap := range volSnaps { currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(inst.Project().Name, snap.Name), vol.Config()) err = b.driver.MountVolumeSnapshot(currentSnapVol, op) if err != nil { return nil, err } } reverter.Add(func() { for _, snap := range volSnaps { currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(inst.Project().Name, snap.Name), vol.Config()) _, _ = b.driver.UnmountVolumeSnapshot(currentSnapVol, op) } }) // Fetch backing chain for a qcow2 formatted volume. backingPaths, err = b.qcow2BackingPaths(vol, diskPath, inst.Project().Name) if err != nil { return nil, err } } mountInfo := &MountInfo{ DiskPath: diskPath, BackingPath: backingPaths, } reverter.Success() // From here on it is up to caller to call UnmountInstance() when done. // Handle delegation. if b.driver.CanDelegateVolume(vol) { mountInfo.PostHooks = append(mountInfo.PostHooks, func(inst instance.Instance) error { pid := inst.InitPID() // Only apply to running instances. if pid < 1 { return nil } return b.driver.DelegateVolume(vol, pid) }) } return mountInfo, nil } // UnmountInstance unmounts the instance's root volume. func (b *backend) UnmountInstance(inst instance.Instance, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) l.Debug("UnmountInstance started") defer l.Debug("UnmountInstance finished") // Check we can convert the instance to the volume type needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentType := InstanceContentType(inst) // Get the volume. var vol drivers.Volume volStorageName := project.Instance(inst.Project().Name, inst.Name()) if inst.ID() > -1 { // Load storage volume from database. dbVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return err } // Generate the effective root device volume for instance. vol = b.GetVolume(volType, contentType, volStorageName, dbVol.Config) err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return err } } else { vol = b.GetVolume(volType, contentType, volStorageName, nil) } if vol.Config()["block.type"] == drivers.BlockVolumeTypeQcow2 { // Get snapshots. volSnaps, err := VolumeDBSnapshotsGet(b, inst.Project().Name, inst.Name(), vol.Type()) if err != nil { return err } for _, snap := range volSnaps { currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(inst.Project().Name, snap.Name), vol.Config()) _, err = b.driver.UnmountVolumeSnapshot(currentSnapVol, op) if err != nil && !errors.Is(err, drivers.ErrInUse) { return err } } } _, err = b.driver.UnmountVolume(vol, false, op) return err } // getInstanceDisk returns the location of the disk. func (b *backend) getInstanceDisk(inst instance.Instance) (string, error) { if inst.Type() != instancetype.VM { return "", drivers.ErrNotSupported } // Check we can convert the instance to the volume type needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return "", err } contentType := InstanceContentType(inst) volStorageName := project.Instance(inst.Project().Name, inst.Name()) // Get the volume. // There's no need to pass config as it's not needed when getting the // location of the disk block device. vol := b.GetVolume(volType, contentType, volStorageName, nil) // Get the location of the disk block device. diskPath, err := b.driver.GetVolumeDiskPath(vol) if err != nil { return "", err } return diskPath, nil } // CreateInstanceSnapshot creates a snapshot of an instance volume. func (b *backend) CreateInstanceSnapshot(inst instance.Instance, src instance.Instance, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "src": src.Name()}) l.Debug("CreateInstanceSnapshot started") defer l.Debug("CreateInstanceSnapshot finished") if inst.Type() != src.Type() { return errors.New("Instance types must match") } if !inst.IsSnapshot() { return errors.New("Instance must be a snapshot") } if src.IsSnapshot() { return errors.New("Source instance cannot be a snapshot") } // Check we can convert the instance to the volume type needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentType := InstanceContentType(inst) // Load storage volume from database. srcDBVol, err := VolumeDBGet(b, src.Project().Name, src.Name(), volType) if err != nil { return err } reverter := revert.New() defer reverter.Fail() // Validate config and create database entry for new storage volume. err = VolumeDBCreate(b, inst.Project().Name, inst.Name(), srcDBVol.Description, volType, true, srcDBVol.Config, inst.CreationDate(), time.Time{}, contentType, false, true) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, inst.Project().Name, inst.Name(), volType) }) // Some driver backing stores require that running instances be frozen during snapshot. if b.driver.Info().RunningCopyFreeze && src.IsRunning() && !src.IsFrozen() { // Freeze the processes. err = src.Freeze() if err != nil { return err } defer func() { _ = src.Unfreeze() }() // Attempt to sync the filesystem. _ = linux.SyncFS(src.RootfsPath()) } volStorageName := project.Instance(inst.Project().Name, inst.Name()) // Get the volume. vol := b.GetVolume(volType, contentType, volStorageName, srcDBVol.Config) err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return err } // Lock this operation to ensure that the only one snapshot is made at the time. // Other operations will wait for this one to finish. unlock, err := locking.Lock(context.TODO(), drivers.OperationLockName("CreateInstanceSnapshot", b.name, vol.Type(), contentType, src.Name())) if err != nil { return err } defer unlock() err = b.driver.CreateVolumeSnapshot(vol, op) if err != nil { return err } if srcDBVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { // Get the parent volume. volStorageParentName := project.Instance(inst.Project().Name, src.Name()) parentVol := b.GetVolume(volType, contentType, volStorageParentName, srcDBVol.Config) reverter.Add(func() { _ = b.driver.Qcow2DeletionCleanup(vol, volStorageParentName) }) rootDiskName, _, err := internalInstance.GetRootDiskDevice(src.ExpandedDevices().CloneNative()) if err != nil { return err } // parentVol should already be prepared as an overlay by CreateVolumeSnapshot. // vol will be used as the base. err = b.qcow2CreateSnapshot(parentVol, vol, src.Project().Name, src, rootDiskName, inst.IsStateful(), op) if err != nil { return err } reverter.Add(func() { _ = b.qcow2DeleteSnapshot(parentVol, vol, src.Project().Name, src, rootDiskName, nil) }) } err = b.ensureInstanceSnapshotSymlink(inst.Type(), inst.Project().Name, inst.Name()) if err != nil { return err } err = inst.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error { // Load the pool for the disk. diskPool, err := LoadByName(b.state, dev.Config["pool"]) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } _, snapshotName, _ := api.GetParentAndSnapshotName(inst.Name()) err = diskPool.CreateCustomVolumeSnapshot(inst.Project().Name, dev.Config["source"], snapshotName, time.Time{}, inst.IsStateful(), op) if err != nil { return fmt.Errorf("Failed to create device snapshot for volume %q: %w", dev.Config["source"], err) } reverter.Add(func() { _ = diskPool.DeleteCustomVolumeSnapshot(inst.Project().Name, fmt.Sprintf("%s/%s", dev.Config["source"], snapshotName), nil) }) return nil }) if err != nil { return err } reverter.Success() return nil } // RenameInstanceSnapshot renames an instance snapshot. func (b *backend) RenameInstanceSnapshot(inst instance.Instance, newName string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "newName": newName}) l.Debug("RenameInstanceSnapshot started") defer l.Debug("RenameInstanceSnapshot finished") reverter := revert.New() defer reverter.Fail() if !inst.IsSnapshot() { return errors.New("Instance must be a snapshot") } if internalInstance.IsSnapshot(newName) { return errors.New("New name cannot be a snapshot") } // Check we can convert the instance to the volume types needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } volDBType, err := VolumeTypeToDBType(volType) if err != nil { return err } parentName, oldSnapshotName, isSnap := api.GetParentAndSnapshotName(inst.Name()) if !isSnap { return errors.New("Volume name must be a snapshot") } newVolName := drivers.GetSnapshotVolumeName(parentName, newName) // Load storage volume from database. srcDBVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return err } // Load parent storage volume from database. parentDBVol, err := VolumeDBGet(b, inst.Project().Name, parentName, volType) if err != nil { return err } contentType := InstanceContentType(inst) volStorageName := project.Instance(inst.Project().Name, inst.Name()) // Rename storage volume snapshot. snapVol := b.GetVolume(volType, contentType, volStorageName, srcDBVol.Config) if parentDBVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { parentVol := b.GetVolume(volType, contentType, project.Instance(inst.Project().Name, parentName), parentDBVol.Config) err = b.qcow2RenameSnapshot(parentVol, snapVol, newVolName, inst.Project().Name, op) if err != nil { return err } } err = b.driver.RenameVolumeSnapshot(snapVol, newName, op) if err != nil { return err } reverter.Add(func() { // Revert rename. No need to pass config as it's not needed when renaming a volume. newSnapVol := b.GetVolume(volType, contentType, project.Instance(inst.Project().Name, newVolName), nil) _ = b.driver.RenameVolumeSnapshot(newSnapVol, oldSnapshotName, op) }) err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Rename DB volume record. return tx.RenameStoragePoolVolume(ctx, inst.Project().Name, inst.Name(), newVolName, volDBType, b.ID()) }) if err != nil { return err } reverter.Add(func() { _ = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Rename DB volume record back. return tx.RenameStoragePoolVolume(ctx, inst.Project().Name, newVolName, inst.Name(), volDBType, b.ID()) }) }) // Ensure the backup file reflects current config. err = b.UpdateInstanceBackupFile(inst, true, op) if err != nil { return err } reverter.Success() return nil } // DeleteInstanceSnapshot removes the snapshot volume for the supplied snapshot instance. func (b *backend) DeleteInstanceSnapshot(inst instance.Instance, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) l.Debug("DeleteInstanceSnapshot started") defer l.Debug("DeleteInstanceSnapshot finished") parentName, snapName, isSnap := api.GetParentAndSnapshotName(inst.Name()) if !inst.IsSnapshot() || !isSnap { return errors.New("Instance must be a snapshot") } // Check we can convert the instance to the volume types needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentType := InstanceContentType(inst) // Get the parent volume name on storage. parentStorageName := project.Instance(inst.Project().Name, parentName) // Delete the snapshot from the storage device. // Must come before DB VolumeDBDelete so that the volume ID is still available. l.Debug("Deleting instance snapshot volume", logger.Ctx{"volName": parentStorageName, "snapshotName": snapName}) snapVolName := drivers.GetSnapshotVolumeName(parentStorageName, snapName) // Load storage volume from database. srcDBVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return err } vol := b.GetVolume(volType, contentType, snapVolName, srcDBVol.Config) // Load parent storage volume from database. parentDBVol, err := VolumeDBGet(b, inst.Project().Name, parentName, volType) if err != nil { return err } volExists, err := b.driver.HasVolume(vol) if err != nil { return err } // Lock this operation to ensure only one snapshot is deleted at a time. // Other operations will wait until this one completes. unlock, err := locking.Lock(context.TODO(), drivers.OperationLockName("DeleteInstanceSnapshot", b.name, vol.Type(), vol.ContentType(), parentName)) if err != nil { return err } defer unlock() src, err := instance.LoadByProjectAndName(b.state, inst.Project().Name, parentName) if err != nil { return err } if volExists { if parentDBVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { parentVol := b.GetVolume(volType, contentType, parentStorageName, parentDBVol.Config) rootDiskName, _, err := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if err != nil { return err } err = b.qcow2DeleteSnapshot(parentVol, vol, src.Project().Name, src, rootDiskName, op) if err != nil { return err } } else { err = b.driver.DeleteVolumeSnapshot(vol, op) if err != nil { return err } } } // Delete symlink if needed. err = b.removeInstanceSnapshotSymlinkIfUnused(inst.Type(), inst.Project().Name, inst.Name()) if err != nil { return err } // Remove the snapshot volume record from the database if exists. err = VolumeDBDelete(b, inst.Project().Name, inst.Name(), vol.Type()) if err != nil { return err } err = src.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error { // Load the pool for the disk. diskPool, err := LoadByName(b.state, dev.Config["pool"]) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } err = diskPool.DeleteCustomVolumeSnapshot(inst.Project().Name, fmt.Sprintf("%s/%s", dev.Config["source"], snapName), op) if err != nil { return fmt.Errorf("Failed to delete snapshot for volume %q: %w", dev.Config["source"], err) } return nil }) if err != nil { return err } return nil } // CanRestoreInstanceSnapshot checks whether an instance snapshot can be restored. func (b *backend) CanRestoreInstanceSnapshot(inst instance.Instance, src instance.Instance) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "src": src.Name()}) l.Debug("CanRestoreInstanceSnapshot started") defer l.Debug("CanRestoreInstanceSnapshot finished") reverter := revert.New() defer reverter.Fail() if inst.Type() != src.Type() { return errors.New("Instance types must match") } if inst.IsSnapshot() { return errors.New("Instance must not be snapshot") } if !src.IsSnapshot() { return errors.New("Source instance must be a snapshot") } snaps, err := inst.Snapshots() if err != nil { return err } if len(snaps) > 0 && snaps[len(snaps)-1].Name() != src.Name() && inst.HasDependentDisk() { return fmt.Errorf("Snapshot %q cannot be restored due to subsequent snapshot(s).", src.Name()) } // Check we can convert the instance to the volume type needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentType := InstanceContentType(inst) // Load storage volume from database. dbVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return err } // Generate the effective root device volume for instance. volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(volType, contentType, volStorageName, dbVol.Config) _, snapshotName, isSnap := api.GetParentAndSnapshotName(src.Name()) if !isSnap { return errors.New("Volume name must be a snapshot") } srcDBVol, err := VolumeDBGet(b, src.Project().Name, src.Name(), volType) if err != nil { return err } if dbVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { snapVol := b.GetVolume(volType, contentType, project.Instance(inst.Project().Name, src.Name()), srcDBVol.Config) err = b.qcow2CanRestoreSnapshot(vol, snapVol, inst.Project().Name) if err != nil { var snapErr drivers.ErrDeleteSnapshots if errors.As(err, &snapErr) { return nil } return err } return nil } err = b.driver.CanRestoreVolume(vol, snapshotName) if err != nil { var snapErr drivers.ErrDeleteSnapshots if errors.As(err, &snapErr) { return nil } return err } reverter.Success() return nil } // RestoreInstanceSnapshot restores an instance snapshot. func (b *backend) RestoreInstanceSnapshot(inst instance.Instance, src instance.Instance, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "src": src.Name()}) l.Debug("RestoreInstanceSnapshot started") defer l.Debug("RestoreInstanceSnapshot finished") reverter := revert.New() defer reverter.Fail() if inst.Type() != src.Type() { return errors.New("Instance types must match") } if inst.IsSnapshot() { return errors.New("Instance must not be snapshot") } if !src.IsSnapshot() { return errors.New("Source instance must be a snapshot") } // Target instance must not be running. if inst.IsRunning() { return errors.New("Instance must not be running to restore") } snaps, err := inst.Snapshots() if err != nil { return err } if len(snaps) > 0 && snaps[len(snaps)-1].Name() != src.Name() && inst.HasDependentDisk() { return fmt.Errorf("Snapshot %q cannot be restored due to subsequent snapshot(s).", src.Name()) } // Check we can convert the instance to the volume type needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentType := InstanceContentType(inst) // Load storage volume from database. dbVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return err } // Generate the effective root device volume for instance. volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(volType, contentType, volStorageName, dbVol.Config) err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return err } _, snapshotName, isSnap := api.GetParentAndSnapshotName(src.Name()) if !isSnap { return errors.New("Volume name must be a snapshot") } srcDBVol, err := VolumeDBGet(b, src.Project().Name, src.Name(), volType) if err != nil { return err } // Restore snapshot volume config if different. changedConfig, _ := b.detectChangedConfig(dbVol.Config, srcDBVol.Config) if len(changedConfig) != 0 || dbVol.Description != srcDBVol.Description { volDBType, err := VolumeTypeToDBType(volType) if err != nil { return err } err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateStoragePoolVolume(ctx, inst.Project().Name, inst.Name(), volDBType, b.ID(), srcDBVol.Description, srcDBVol.Config) }) if err != nil { return err } reverter.Add(func() { _ = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateStoragePoolVolume(ctx, inst.Project().Name, inst.Name(), volDBType, b.ID(), dbVol.Description, dbVol.Config) }) }) } err = inst.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error { // Load the pool for the disk. diskPool, err := LoadByName(b.state, dev.Config["pool"]) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } err = diskPool.RestoreCustomVolume(inst.Project().Name, dev.Config["source"], snapshotName, op) if err != nil { return err } return nil }) if err != nil { return err } deleteSnapshots := func(snapshots []string, inst instance.Instance) error { // We need to delete some snapshots and try again. snaps, err := inst.Snapshots() if err != nil { return err } // Go through all the snapshots. for _, snap := range snaps { _, snapName, _ := api.GetParentAndSnapshotName(snap.Name()) if !slices.Contains(snapshots, snapName) { continue } // Delete snapshot instance if listed in the error as one that needs removing. err := snap.Delete(true, true) if err != nil { return err } } return nil } if dbVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { snapVol := b.GetVolume(volType, contentType, project.Instance(inst.Project().Name, src.Name()), srcDBVol.Config) err = b.qcow2RestoreSnapshot(vol, snapVol, inst.Project().Name, op) if err != nil { var snapErr drivers.ErrDeleteSnapshots if errors.As(err, &snapErr) { err = deleteSnapshots(snapErr.Snapshots, inst) if err != nil { return err } // Now try restoring again. err = b.qcow2RestoreSnapshot(vol, snapVol, inst.Project().Name, op) if err != nil { return err } return nil } return err } return nil } err = b.driver.RestoreVolume(vol, snapshotName, op) if err != nil { var snapErr drivers.ErrDeleteSnapshots if errors.As(err, &snapErr) { err = deleteSnapshots(snapErr.Snapshots, inst) if err != nil { return err } // Now try restoring again. err = b.driver.RestoreVolume(vol, snapshotName, op) if err != nil { return err } return nil } return err } reverter.Success() return nil } // MountInstanceSnapshot mounts an instance snapshot. It is mounted as read only so that the // snapshot cannot be modified. func (b *backend) MountInstanceSnapshot(inst instance.Instance, op *operations.Operation) (*MountInfo, error) { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) l.Debug("MountInstanceSnapshot started") defer l.Debug("MountInstanceSnapshot finished") if !inst.IsSnapshot() { return nil, errors.New("Instance must be a snapshot") } // Check we can convert the instance to the volume type needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return nil, err } // Load storage volume from database. dbVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return nil, err } contentType := InstanceContentType(inst) // Generate the effective root device volume for instance. volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(volType, contentType, volStorageName, dbVol.Config) err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return nil, err } err = b.driver.MountVolumeSnapshot(vol, op) if err != nil { return nil, err } diskPath, err := b.getInstanceDisk(inst) if err != nil && !errors.Is(err, drivers.ErrNotSupported) { return nil, fmt.Errorf("Failed getting disk path: %w", err) } mountInfo := &MountInfo{ DiskPath: diskPath, } return mountInfo, nil } // UnmountInstanceSnapshot unmounts an instance snapshot. func (b *backend) UnmountInstanceSnapshot(inst instance.Instance, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) l.Debug("UnmountInstanceSnapshot started") defer l.Debug("UnmountInstanceSnapshot finished") if !inst.IsSnapshot() { return errors.New("Instance must be a snapshot") } // Check we can convert the instance to the volume type needed. volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentType := InstanceContentType(inst) // Load storage volume from database. dbVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return err } // Generate the effective root device volume for instance. volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(volType, contentType, volStorageName, dbVol.Config) err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return err } _, err = b.driver.UnmountVolumeSnapshot(vol, op) return err } // EnsureImage creates an optimized volume of the image if supported by the storage pool driver and the volume // doesn't already exist. If the volume already exists then it is checked to ensure it matches the pools current // volume settings ("volume.size" and "block.filesystem" if applicable). If not the optimized volume is removed // and regenerated to apply the pool's current volume settings. func (b *backend) EnsureImage(fingerprint string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"fingerprint": fingerprint}) l.Debug("EnsureImage started") defer l.Debug("EnsureImage finished") err := b.isStatusReady() if err != nil { return err } if !b.driver.Info().OptimizedImages { return nil // Nothing to do for drivers that don't support optimized images volumes. } // We need to lock this operation to ensure that the image is not being created multiple times. // Uses a lock name of "EnsureImage_" to avoid deadlocking with CreateVolume below that also // establishes a lock on the volume type & name if it needs to mount the volume before filling. unlock, err := locking.Lock(context.TODO(), drivers.OperationLockName("EnsureImage", b.name, drivers.VolumeTypeImage, "", fingerprint)) if err != nil { return err } defer unlock() var image *api.Image err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Load image info from database. _, image, err = tx.GetImageFromAnyProject(ctx, fingerprint) return err }) if err != nil { return err } // Derive content type from image type. Image types are not the same as instance types, so don't use // instance type constants for comparison. contentType := drivers.ContentTypeFS if image.Type == "virtual-machine" { contentType = drivers.ContentTypeBlock } // Try and load any existing volume config on this storage pool so we can compare filesystems if needed. imgDBVol, err := VolumeDBGet(b, api.ProjectDefaultName, fingerprint, drivers.VolumeTypeImage) if err != nil && !response.IsNotFoundError(err) { return err } // Create the new image volume. No config for an image volume so set to nil. // Pool config values will be read by the underlying driver if needed. imgVol := b.GetVolume(drivers.VolumeTypeImage, contentType, fingerprint, nil) // If an existing DB row was found, check if filesystem is the same as the current pool's filesystem. // If not we need to delete the existing cached image volume and re-create using new filesystem. // We need to do this for VM block images too, as they create a filesystem based config volume too. if imgDBVol != nil { // Generate a temporary volume instance that represents how a new volume using pool defaults would // be configured. tmpImgVol := imgVol.Clone() err := b.Driver().FillVolumeConfig(tmpImgVol) if err != nil { return err } // Add existing image volume's config to imgVol. imgVol = b.GetVolume(drivers.VolumeTypeImage, contentType, fingerprint, imgDBVol.Config) // Check if the volume's block backed mode differs from the pool's current setting for new volumes. blockModeChanged := tmpImgVol.IsBlockBacked() != imgVol.IsBlockBacked() // Check if the volume is block backed and its filesystem is different from the pool's current // setting for new volumes. blockFSChanged := imgVol.IsBlockBacked() && imgVol.Config()["block.filesystem"] != tmpImgVol.Config()["block.filesystem"] // If the existing image volume no longer matches the pool's settings for new volumes then we need // to delete and re-create it. if blockModeChanged || blockFSChanged { if blockModeChanged { l.Debug("Block mode has changed, regenerating image volume") } else { l.Debug("Block volume filesystem of pool has changed since cached image volume created, regenerating image volume") } err = b.DeleteImage(fingerprint, op) if err != nil { return err } // Reset img volume variables as we just deleted the old one. imgDBVol = nil imgVol = b.GetVolume(drivers.VolumeTypeImage, contentType, fingerprint, nil) } } // Check if we already have a suitable volume on storage device. volExists, err := b.driver.HasVolume(imgVol) if err != nil { return err } if volExists { if imgDBVol != nil { // Work out what size the image volume should be as if we were creating from scratch. // This takes into account the existing volume's "volatile.rootfs.size" setting if set so // as to avoid trying to shrink a larger image volume back to the default size when it is // allowed to be larger than the default as the pool doesn't specify a volume.size. l.Debug("Checking image volume size") newVolSize, err := imgVol.ConfigSizeFromSource(imgVol) if err != nil { return err } imgVol.SetConfigSize(newVolSize) // Try applying the current size policy to the existing volume. If it is the same the // driver should make no changes, and if not then attempt to resize it to the new policy. l.Debug("Setting image volume size", logger.Ctx{"size": imgVol.ConfigSize()}) err = b.driver.SetVolumeQuota(imgVol, imgVol.ConfigSize(), false, op) if errors.Is(err, drivers.ErrCannotBeShrunk) || errors.Is(err, drivers.ErrNotSupported) { // If the driver cannot resize the existing image volume to the new policy size // then delete the image volume and try to recreate using the new policy settings. l.Debug("Volume size of pool has changed since cached image volume created and cached volume cannot be resized, regenerating image volume") err = b.DeleteImage(fingerprint, op) if err != nil { return err } // Reset img volume variables as we just deleted the old one. imgDBVol = nil imgVol = b.GetVolume(drivers.VolumeTypeImage, contentType, fingerprint, nil) } else if err != nil { return err } else { // We already have a valid volume at the correct size, just return. return nil } } else { // We have an unrecorded on-disk volume, assume it's a partial unpack and delete it. // This can occur if Incus process exits unexpectedly during an image unpack or if the // storage pool has been recovered (which would not recreate the image volume DB records). l.Warn("Deleting leftover/partially unpacked image volume") err = b.driver.DeleteVolume(imgVol, op) if err != nil { return fmt.Errorf("Failed deleting leftover/partially unpacked image volume: %w", err) } } } volFiller := drivers.VolumeFiller{ Fingerprint: fingerprint, Fill: b.imageFiller(fingerprint, op), } reverter := revert.New() defer reverter.Fail() // Validate config and create database entry for new storage volume. err = VolumeDBCreate(b, api.ProjectDefaultName, fingerprint, "", drivers.VolumeTypeImage, false, imgVol.Config(), time.Now().UTC(), time.Time{}, contentType, false, false) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, api.ProjectDefaultName, fingerprint, drivers.VolumeTypeImage) }) // Record new volume with authorizer. var location string if b.state.ServerClustered && !b.Driver().Info().Remote { location = b.state.ServerName } // Record new volume with authorizer. err = b.state.Authorizer.AddStoragePoolVolume(b.state.ShutdownCtx, api.ProjectDefaultName, b.Name(), drivers.VolumeTypeImage.Singular(), fingerprint, location) if err != nil { logger.Error("Failed to add storage volume to authorizer", logger.Ctx{"name": fingerprint, "type": drivers.VolumeTypeImage, "pool": b.Name(), "project": api.ProjectDefaultName, "error": err}) } reverter.Add(func() { _ = b.state.Authorizer.DeleteStoragePoolVolume(b.state.ShutdownCtx, api.ProjectDefaultName, b.Name(), drivers.VolumeTypeImage.Singular(), fingerprint, location) }) err = b.driver.CreateVolume(imgVol, &volFiller, op) if err != nil { return err } reverter.Add(func() { _ = b.driver.DeleteVolume(imgVol, op) }) // If the volume filler has recorded the size of the unpacked volume, then store this in the image DB row. if volFiller.Size != 0 { imgVol.Config()["volatile.rootfs.size"] = fmt.Sprintf("%d", volFiller.Size) err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateStoragePoolVolume(ctx, api.ProjectDefaultName, fingerprint, db.StoragePoolVolumeTypeImage, b.id, "", imgVol.Config()) }) if err != nil { return err } } reverter.Success() return nil } // shouldUseOptimizedImage determines if an optimized image should be used based on the provided volume config. // It returns true if the volume config aligns with the pool's default configuration, and an optimized image does // not exist or also matches the pool's default configuration. func (b *backend) shouldUseOptimizedImage(fingerprint string, contentType drivers.ContentType, volConfig map[string]string, op *operations.Operation) (bool, error) { canOptimizeImage := b.driver.Info().OptimizedImages // If the volume config is empty, the default pool configuration is used, making the driver's support // for optimized images the determining factor. However, an optimized image cannot be utilized if the // driver lacks support for it. if !canOptimizeImage || len(volConfig) == 0 { return canOptimizeImage, nil } // Create the image volume with the provided volume config. newImgVol := b.GetVolume(drivers.VolumeTypeImage, contentType, fingerprint, volConfig) err := b.Driver().FillVolumeConfig(newImgVol) if err != nil { return false, err } // Create the image volume with pool's default settings. poolDefaultImgVol := b.GetVolume(drivers.VolumeTypeImage, contentType, fingerprint, nil) err = b.Driver().FillVolumeConfig(poolDefaultImgVol) if err != nil { return false, err } // If the new volume's config doesn't match the pool's default configuration, don't use an optimized image. if !volumeConfigsMatch(newImgVol, poolDefaultImgVol) { return false, nil } // Load existing optimized image, if it exists. imgDBVol, err := VolumeDBGet(b, api.ProjectDefaultName, fingerprint, drivers.VolumeTypeImage) if err != nil && !response.IsNotFoundError(err) { return false, err } if imgDBVol != nil { // Ensure existing optimized image's config matches the pool's default configuration. imgVol := b.GetVolume(drivers.VolumeTypeImage, contentType, fingerprint, imgDBVol.Config) if !volumeConfigsMatch(newImgVol, imgVol) { return false, nil } } return true, nil } // volumeConfigsMatch checks if the block-backed modes of two volumes match, and if they are block-backed, ensures // their filesystem configurations are also identical. func volumeConfigsMatch(vol1, vol2 drivers.Volume) bool { blockModeChanged := vol1.IsBlockBacked() != vol2.IsBlockBacked() blockFSChanged := vol1.IsBlockBacked() && vol1.Config()["block.filesystem"] != vol2.Config()["block.filesystem"] // TODO: Temporary workaround for zfs.blocksize issue: // When zfs.blocksize changes, a new optimized image isn't generated. This ensures we don't use an // optimized image if initial.zfs.blocksize differs from the default pool settings. // // Note: If initial.zfs.blocksize is set to 8KiB and volume.zfs.blocksize is unset (defaults to 8KiB), // they're considered unequal ("" != "8KiB"), preventing the use of a matching optimized image. blockSizeChanged := vol1.IsBlockBacked() && vol1.Config()["zfs.blocksize"] != vol2.Config()["zfs.blocksize"] return !blockModeChanged && !blockFSChanged && !blockSizeChanged } // DeleteImage removes an image from the database and underlying storage device if needed. func (b *backend) DeleteImage(fingerprint string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"fingerprint": fingerprint}) l.Debug("DeleteImage started") defer l.Debug("DeleteImage finished") // We need to lock this operation to ensure that the image is not being deleted multiple times. unlock, err := locking.Lock(context.TODO(), drivers.OperationLockName("DeleteImage", b.name, drivers.VolumeTypeImage, "", fingerprint)) if err != nil { return err } defer unlock() // Load the storage volume in order to get the volume config which is needed for some drivers. imgDBVol, err := VolumeDBGet(b, api.ProjectDefaultName, fingerprint, drivers.VolumeTypeImage) if err != nil { return err } // Get the content type. dbContentType, err := VolumeContentTypeNameToContentType(imgDBVol.ContentType) if err != nil { return err } contentType, err := VolumeDBContentTypeToContentType(dbContentType) if err != nil { return err } vol := b.GetVolume(drivers.VolumeTypeImage, contentType, fingerprint, imgDBVol.Config) volExists, err := b.driver.HasVolume(vol) if err != nil { return err } if volExists { err = b.driver.DeleteVolume(vol, op) if err != nil { return err } } err = VolumeDBDelete(b, api.ProjectDefaultName, fingerprint, vol.Type()) if err != nil { return err } // Record volume deletion with authorizer. var location string if b.state.ServerClustered && !b.Driver().Info().Remote { location = b.state.ServerName } err = b.state.Authorizer.DeleteStoragePoolVolume(b.state.ShutdownCtx, api.ProjectDefaultName, b.Name(), vol.Type().Singular(), fingerprint, location) if err != nil { logger.Error("Failed to remove storage volume from authorizer", logger.Ctx{"name": fingerprint, "type": vol.Type(), "pool": b.Name(), "project": api.ProjectDefaultName, "error": err}) } b.state.Events.SendLifecycle(api.ProjectDefaultName, lifecycle.StorageVolumeDeleted.Event(vol, string(vol.Type()), api.ProjectDefaultName, op, nil)) return nil } // updateVolumeDescriptionOnly is a helper function used when handling update requests for volumes // that only allow their descriptions to be updated. If any config supplied differs from the // current volume's config then an error is returned. func (b *backend) updateVolumeDescriptionOnly(projectName string, volName string, volType drivers.VolumeType, newDesc string, newConfig map[string]string, op *operations.Operation) error { volDBType, err := VolumeTypeToDBType(volType) if err != nil { return err } // Get current config to compare what has changed. curVol, err := VolumeDBGet(b, projectName, volName, volType) if err != nil { return err } if newConfig != nil { changedConfig, _ := b.detectChangedConfig(curVol.Config, newConfig) if len(changedConfig) != 0 { return errors.New("Volume config is not editable") } } // Update the database if description changed. Use current config. if newDesc != curVol.Description { err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateStoragePoolVolume(ctx, projectName, volName, volDBType, b.ID(), newDesc, curVol.Config) }) if err != nil { return err } } // Get content type. dbContentType, err := VolumeContentTypeNameToContentType(curVol.ContentType) if err != nil { return err } contentType, err := VolumeDBContentTypeToContentType(dbContentType) if err != nil { return err } // Validate config. vol := b.GetVolume(drivers.VolumeType(curVol.Type), contentType, volName, newConfig) if !vol.IsSnapshot() { b.state.Events.SendLifecycle(projectName, lifecycle.StorageVolumeUpdated.Event(vol, string(vol.Type()), projectName, op, nil)) } else { b.state.Events.SendLifecycle(projectName, lifecycle.StorageVolumeSnapshotUpdated.Event(vol, string(vol.Type()), projectName, op, nil)) } return nil } // UpdateImage updates image config. func (b *backend) UpdateImage(fingerprint, newDesc string, newConfig map[string]string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"fingerprint": fingerprint, "newDesc": newDesc, "newConfig": newConfig}) l.Debug("UpdateImage started") defer l.Debug("UpdateImage finished") return b.updateVolumeDescriptionOnly(api.ProjectDefaultName, fingerprint, drivers.VolumeTypeImage, newDesc, newConfig, op) } // CreateBucket creates an object bucket. func (b *backend) CreateBucket(projectName string, bucket api.StorageBucketsPost, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "bucketName": bucket.Name, "desc": bucket.Description, "config": bucket.Config}) l.Debug("CreateBucket started") defer l.Debug("CreateBucket finished") err := b.isStatusReady() if err != nil { return err } if !b.Driver().Info().Buckets { return errors.New("Storage pool does not support buckets") } // Validate config and create database entry for new storage bucket. reverter := revert.New() defer reverter.Fail() memberSpecific := !b.Driver().Info().Remote // Member specific if storage pool isn't remote. bucketID, err := BucketDBCreate(context.TODO(), b, projectName, memberSpecific, &bucket) if err != nil { return err } reverter.Add(func() { _ = BucketDBDelete(context.TODO(), b, bucketID) }) bucketVolName := project.StorageVolume(projectName, bucket.Name) bucketVol := b.GetVolume(drivers.VolumeTypeBucket, drivers.ContentTypeFS, bucketVolName, bucket.Config) // Create the bucket on the storage device. if memberSpecific { // Handle common implementation for local storage drivers. err := b.driver.CreateVolume(bucketVol, nil, op) if err != nil { return err } reverter.Add(func() { _ = b.driver.DeleteVolume(bucketVol, op) }) // Initialise the on-disk layout for the in-process S3 handler. err = b.initLocalBucketLayout(projectName, bucket.Name, op) if err != nil { return fmt.Errorf("Failed initialising bucket on storage: %w", err) } } else { // Handle per-driver implementation for remote storage drivers. err = b.driver.CreateBucket(bucketVol, op) if err != nil { return err } } reverter.Success() return nil } func generateLocalBucketKey(accessKey, secretKey string) (*drivers.S3Credentials, error) { const accessKeyAlphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" const secretKeyAlphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" randomString := func(alphabet string, n int) (string, error) { out := make([]byte, n) maxChar := big.NewInt(int64(len(alphabet))) for i := range out { idx, err := rand.Int(rand.Reader, maxChar) if err != nil { return "", err } out[i] = alphabet[idx.Int64()] } return string(out), nil } if accessKey == "" { key, err := randomString(accessKeyAlphabet, 20) if err != nil { return nil, fmt.Errorf("Failed generating access key: %w", err) } accessKey = key } if secretKey == "" { key, err := randomString(secretKeyAlphabet, 40) if err != nil { return nil, fmt.Errorf("Failed generating secret key: %w", err) } secretKey = key } return &drivers.S3Credentials{AccessKey: accessKey, SecretKey: secretKey}, nil } // initLocalBucketLayout mounts the bucket volume and ensures the data/ // directory exists so that the in-process S3 handler can serve writes // against it. func (b *backend) initLocalBucketLayout(projectName, bucketName string, op *operations.Operation) error { mountPath, unmount, err := b.MountLocalBucket(projectName, bucketName, op) if err != nil { return err } defer func() { _ = unmount() }() return os.MkdirAll(filepath.Join(mountPath, "data"), 0o700) } // UpdateBucket updates an object bucket. func (b *backend) UpdateBucket(projectName string, bucketName string, bucket api.StorageBucketPut, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "bucketName": bucketName, "desc": bucket.Description, "config": bucket.Config}) l.Debug("UpdateBucket started") defer l.Debug("UpdateBucket finished") err := b.isStatusReady() if err != nil { return err } if !b.Driver().Info().Buckets { return errors.New("Storage pool does not support buckets") } memberSpecific := !b.Driver().Info().Remote // Member specific if storage pool isn't remote. // Get current config to compare what has changed. var curBucket *db.StorageBucket err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { curBucket, err = tx.GetStoragePoolBucket(ctx, b.id, projectName, memberSpecific, bucketName) return err }) if err != nil { return err } bucketVolName := project.StorageVolume(projectName, curBucket.Name) curBucketVol := b.GetVolume(drivers.VolumeTypeBucket, drivers.ContentTypeFS, bucketVolName, curBucket.Config) // Validate config. newBucketVol := b.GetVolume(drivers.VolumeTypeBucket, drivers.ContentTypeFS, bucketVolName, bucket.Config) err = b.driver.ValidateBucket(newBucketVol) if err != nil { return err } err = b.driver.ValidateVolume(newBucketVol, false) if err != nil { return err } curBucketEtagHash, err := localUtil.EtagHash(curBucket.Etag()) if err != nil { return err } newBucket := api.StorageBucket{ Name: curBucket.Name, StorageBucketPut: bucket, } newBucketEtagHash, err := localUtil.EtagHash(newBucket.Etag()) if err != nil { return err } if curBucketEtagHash == newBucketEtagHash { return nil // Nothing has changed. } changedConfig, userOnly := b.detectChangedConfig(curBucket.Config, bucket.Config) if len(changedConfig) > 0 && !userOnly { if memberSpecific { err = b.driver.UpdateVolume(curBucketVol, changedConfig) if err != nil { return err } } else { // Handle per-driver implementation for remote storage drivers. err = b.driver.UpdateBucket(curBucketVol, changedConfig) if err != nil { return err } } } err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Update the database record. return tx.UpdateStoragePoolBucket(ctx, b.id, curBucket.ID, &bucket) }) if err != nil { return err } return nil } // DeleteBucket deletes an object bucket. func (b *backend) DeleteBucket(projectName string, bucketName string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "bucketName": bucketName}) l.Debug("DeleteBucket started") defer l.Debug("DeleteBucket finished") err := b.isStatusReady() if err != nil { return err } if !b.Driver().Info().Buckets { return errors.New("Storage pool does not support buckets") } memberSpecific := !b.Driver().Info().Remote // Member specific if storage pool isn't remote. var bucket *db.StorageBucket err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { bucket, err = tx.GetStoragePoolBucket(ctx, b.id, projectName, memberSpecific, bucketName) return err }) if err != nil { return err } bucketVolName := project.StorageVolume(projectName, bucket.Name) bucketVol := b.GetVolume(drivers.VolumeTypeBucket, drivers.ContentTypeFS, bucketVolName, bucket.Config) if memberSpecific { // Handle common implementation for local storage drivers. vol := b.GetVolume(drivers.VolumeTypeBucket, drivers.ContentTypeFS, bucketVolName, nil) err = b.driver.DeleteVolume(vol, op) if err != nil { return err } } else { // Handle per-driver implementation for remote storage drivers. err = b.driver.DeleteBucket(bucketVol, op) if err != nil { return err } } _ = BucketDBDelete(context.TODO(), b, bucket.ID) if err != nil { return err } return nil } // ImportBucket takes an existing bucket on the storage backend and ensures that the DB records // are restored as needed to make it operational with Incus. // Used during the recovery import stage. func (b *backend) ImportBucket(projectName string, poolVol *backupConfig.Config, op *operations.Operation) (revert.Hook, error) { if poolVol.Bucket == nil { return nil, errors.New("Invalid pool bucket config supplied") } l := b.logger.AddContext(logger.Ctx{"project": projectName, "bucketName": poolVol.Bucket.Name}) l.Debug("ImportBucket started") defer l.Debug("ImportBucket finished") reverter := revert.New() defer reverter.Fail() // Copy bucket config from backup file if present (so BucketDBCreate can safely modify the copy if needed). bucketConfig := util.CloneMap(poolVol.Bucket.Config) bucket := &api.StorageBucketsPost{ Name: poolVol.Bucket.Name, StorageBucketPut: poolVol.Bucket.StorageBucketPut, } // Validate config and create database entry for restored bucket. bucketID, err := BucketDBCreate(b.state.ShutdownCtx, b, projectName, true, bucket) if err != nil { return nil, err } reverter.Add(func() { _ = BucketDBDelete(b.state.ShutdownCtx, b, bucketID) }) // Get the bucket name on storage. storageBucketName := project.StorageVolume(projectName, bucket.Name) storageBucket := b.GetVolume(drivers.VolumeTypeBucket, drivers.ContentTypeFS, storageBucketName, bucketConfig) err = b.driver.ValidateVolume(storageBucket, false) if err != nil { return nil, err } memberSpecific := !b.Driver().Info().Remote // Member specific if storage pool isn't remote. if !memberSpecific { return nil, errors.New("Importing buckets from a remote storage is not supported") } cleanup := reverter.Clone().Fail reverter.Success() return cleanup, nil } // CreateBucketKey creates an object bucket key. func (b *backend) CreateBucketKey(projectName string, bucketName string, key api.StorageBucketKeysPost, op *operations.Operation) (*api.StorageBucketKey, error) { l := b.logger.AddContext(logger.Ctx{"project": projectName, "bucketName": bucketName, "keyName": key.Name, "desc": key.Description, "role": key.Role}) l.Debug("CreateBucketKey started") defer l.Debug("CreateBucketKey finished") err := b.isStatusReady() if err != nil { return nil, err } if !b.Driver().Info().Buckets { return nil, errors.New("Storage pool does not support buckets") } reverter := revert.New() defer reverter.Fail() memberSpecific := !b.Driver().Info().Remote // Member specific if storage pool isn't remote. var bucket *db.StorageBucket err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { bucket, err = tx.GetStoragePoolBucket(ctx, b.id, projectName, memberSpecific, bucketName) return err }) if err != nil { return nil, err } bucketVolName := project.StorageVolume(projectName, bucket.Name) bucketVol := b.GetVolume(drivers.VolumeTypeBucket, drivers.ContentTypeFS, bucketVolName, bucket.Config) // Create the bucket key on the storage device. creds := drivers.S3Credentials{ AccessKey: key.AccessKey, SecretKey: key.SecretKey, } err = b.driver.ValidateBucketKey(key.Name, creds, key.Role) if err != nil { return nil, err } var newCreds *drivers.S3Credentials if memberSpecific { // For local buckets the credentials are stored solely in the // Incus database; generate any missing fields here. newCreds, err = generateLocalBucketKey(key.AccessKey, key.SecretKey) if err != nil { return nil, err } } else { // Handle per-driver implementation for remote storage drivers. newCreds, err = b.driver.CreateBucketKey(bucketVol, key.Name, creds, key.Role, op) if err != nil { return nil, err } reverter.Add(func() { _ = b.driver.DeleteBucketKey(bucketVol, key.Name, op) }) } key.AccessKey = newCreds.AccessKey key.SecretKey = newCreds.SecretKey newKey := api.StorageBucketKey{ Name: key.Name, StorageBucketKeyPut: api.StorageBucketKeyPut{ Description: key.Description, Role: key.Role, AccessKey: key.AccessKey, SecretKey: key.SecretKey, }, } err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { _, err = tx.CreateStoragePoolBucketKey(ctx, bucket.ID, key) return err }) if err != nil { return nil, err } reverter.Success() return &newKey, err } func (b *backend) UpdateBucketKey(projectName string, bucketName string, keyName string, key api.StorageBucketKeyPut, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "bucketName": bucketName, "keyName": keyName, "desc": key.Description, "role": key.Role}) l.Debug("UpdateBucketKey started") defer l.Debug("UpdateBucketKey finished") err := b.isStatusReady() if err != nil { return err } if !b.Driver().Info().Buckets { return errors.New("Storage pool does not support buckets") } memberSpecific := !b.Driver().Info().Remote // Member specific if storage pool isn't remote. // Get current config to compare what has changed. var bucket *db.StorageBucket var curBucketKey *db.StorageBucketKey err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { bucket, err = tx.GetStoragePoolBucket(ctx, b.id, projectName, memberSpecific, bucketName) if err != nil { return err } curBucketKey, err = tx.GetStoragePoolBucketKey(ctx, bucket.ID, keyName) if err != nil { return err } return nil }) if err != nil { return err } curBucketKeyEtagHash, err := localUtil.EtagHash(curBucketKey.Etag()) if err != nil { return err } newBucketKey := api.StorageBucketKey{ Name: curBucketKey.Name, StorageBucketKeyPut: key, } newBucketKeyEtagHash, err := localUtil.EtagHash(newBucketKey.Etag()) if err != nil { return err } if curBucketKeyEtagHash == newBucketKeyEtagHash { return nil // Nothing has changed. } bucketVolName := project.StorageVolume(projectName, bucket.Name) bucketVol := b.GetVolume(drivers.VolumeTypeBucket, drivers.ContentTypeFS, bucketVolName, bucket.Config) creds := drivers.S3Credentials{ AccessKey: newBucketKey.AccessKey, SecretKey: newBucketKey.SecretKey, } err = b.driver.ValidateBucketKey(keyName, creds, key.Role) if err != nil { return err } if memberSpecific { // For local buckets the key fields are stored only in the DB; // generate any missing values here. newCreds, err := generateLocalBucketKey(creds.AccessKey, creds.SecretKey) if err != nil { return err } key.AccessKey = newCreds.AccessKey key.SecretKey = newCreds.SecretKey } else { // Handle per-driver implementation for remote storage drivers. newCreds, err := b.driver.UpdateBucketKey(bucketVol, keyName, creds, key.Role, op) if err != nil { return err } key.AccessKey = newCreds.AccessKey key.SecretKey = newCreds.SecretKey } err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Update the database record. return tx.UpdateStoragePoolBucketKey(ctx, bucket.ID, curBucketKey.ID, &key) }) if err != nil { return err } return nil } // DeleteBucketKey deletes an object bucket key. func (b *backend) DeleteBucketKey(projectName string, bucketName string, keyName string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "bucketName": bucketName, "keyName": keyName}) l.Debug("DeleteBucketKey started") defer l.Debug("DeleteBucketKey finished") err := b.isStatusReady() if err != nil { return err } if !b.Driver().Info().Buckets { return errors.New("Storage pool does not support buckets") } memberSpecific := !b.Driver().Info().Remote // Member specific if storage pool isn't remote. var bucket *db.StorageBucket var bucketKey *db.StorageBucketKey err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { bucket, err = tx.GetStoragePoolBucket(ctx, b.id, projectName, memberSpecific, bucketName) if err != nil { return err } bucketKey, err = tx.GetStoragePoolBucketKey(ctx, bucket.ID, keyName) if err != nil { return err } return nil }) if err != nil { return err } if !memberSpecific { // Handle per-driver implementation for remote storage drivers. bucketVolName := project.StorageVolume(projectName, bucket.Name) bucketVol := b.GetVolume(drivers.VolumeTypeBucket, drivers.ContentTypeFS, bucketVolName, bucket.Config) // Delete the bucket key from the storage device. err = b.driver.DeleteBucketKey(bucketVol, keyName, op) if err != nil { return err } } err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.DeleteStoragePoolBucketKey(ctx, bucket.ID, bucketKey.ID) }) if err != nil { return fmt.Errorf("Failed deleting bucket key from database: %w", err) } return nil } // MountLocalBucket mounts the local bucket volume and returns its mount path // along with an unmount function that the caller must invoke when finished. func (b *backend) MountLocalBucket(projectName string, bucketName string, op *operations.Operation) (string, func() error, error) { if !b.Driver().Info().Buckets { return "", nil, errors.New("Storage pool does not support buckets") } if b.Driver().Info().Remote { return "", nil, errors.New("Remote buckets cannot be mounted locally") } bucketVolName := project.StorageVolume(projectName, bucketName) bucketVol := b.GetVolume(drivers.VolumeTypeBucket, drivers.ContentTypeFS, bucketVolName, nil) err := b.driver.MountVolume(bucketVol, op) if err != nil { return "", nil, err } unmount := func() error { _, err := b.driver.UnmountVolume(bucketVol, false, op) if err != nil && !errors.Is(err, drivers.ErrInUse) { return err } return nil } return bucketVol.MountPath(), unmount, nil } // GetBucketURL returns S3 URL for bucket. func (b *backend) GetBucketURL(bucketName string) *url.URL { err := b.isStatusReady() if err != nil { return nil } if !b.Driver().Info().Buckets { return nil } memberSpecific := !b.Driver().Info().Remote // Member specific if storage pool isn't remote. if memberSpecific { // Handle common MinIO implementation for local storage drivers. // Check that the storage buckets listener is configured via core.storage_buckets_address. storageBucketsAddress := b.state.Endpoints.StorageBucketsAddress() if storageBucketsAddress == "" { return nil } return &api.NewURL().Scheme("https").Host(storageBucketsAddress).Path(bucketName).URL } // Handle per-driver implementation for remote storage drivers. return b.driver.GetBucketURL(bucketName) } // CreateCustomVolume creates an empty custom volume. func (b *backend) CreateCustomVolume(projectName string, volName string, desc string, config map[string]string, contentType drivers.ContentType, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volName": volName, "desc": desc, "config": config, "contentType": contentType}) l.Debug("CreateCustomVolume started") defer l.Debug("CreateCustomVolume finished") err := b.isStatusReady() if err != nil { return err } // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, volName) vol := b.GetVolume(drivers.VolumeTypeCustom, contentType, volStorageName, config) storagePoolSupported := slices.Contains(b.Driver().Info().VolumeTypes, drivers.VolumeTypeCustom) if !storagePoolSupported { return errors.New("Storage pool does not support custom volume type") } reverter := revert.New() defer reverter.Fail() // Validate config and create database entry for new storage volume. err = VolumeDBCreate(b, projectName, volName, desc, vol.Type(), false, vol.Config(), time.Now().UTC(), time.Time{}, vol.ContentType(), false, false) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, projectName, volName, vol.Type()) }) // Create the empty custom volume on the storage device. err = b.driver.CreateVolume(vol, nil, op) if err != nil { return err } eventCtx := logger.Ctx{"type": vol.Type()} var location string if b.state.ServerClustered && !b.Driver().Info().Remote { eventCtx["location"] = b.state.ServerName location = b.state.ServerName } // Record new volume with authorizer. err = b.state.Authorizer.AddStoragePoolVolume(b.state.ShutdownCtx, projectName, b.Name(), vol.Type().Singular(), volName, location) if err != nil { logger.Error("Failed to add storage volume to authorizer", logger.Ctx{"name": volName, "type": vol.Type(), "pool": b.Name(), "project": projectName, "error": err}) } b.state.Events.SendLifecycle(projectName, lifecycle.StorageVolumeCreated.Event(vol, string(vol.Type()), projectName, op, eventCtx)) reverter.Success() return nil } // CreateCustomVolumeFromCopy creates a custom volume from an existing custom volume. // It copies the snapshots from the source volume by default, but can be disabled if requested. func (b *backend) CreateCustomVolumeFromCopy(projectName string, srcProjectName string, volName string, desc string, config map[string]string, srcPoolName, srcVolName string, snapshots bool, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "srcProjectName": srcProjectName, "volName": volName, "desc": desc, "config": config, "srcPoolName": srcPoolName, "srcVolName": srcVolName, "snapshots": snapshots}) l.Debug("CreateCustomVolumeFromCopy started") defer l.Debug("CreateCustomVolumeFromCopy finished") err := b.isStatusReady() if err != nil { return err } if srcProjectName == "" { srcProjectName = projectName } // Setup the source pool backend instance. var srcPool Pool if b.name == srcPoolName { srcPool = b // Source and target are in the same pool so share pool var. } else { // Source is in a different pool to target, so load the pool. srcPool, err = LoadByName(b.state, srcPoolName) if err != nil { return err } } // Check source volume exists and is custom type, and get its config. srcConfig, err := srcPool.GenerateCustomVolumeBackupConfig(srcProjectName, srcVolName, snapshots, op) if err != nil { return fmt.Errorf("Failed generating volume copy config: %w", err) } // Use the source volume's config if not supplied. if config == nil { config = srcConfig.Volume.Config } // Use the source volume's description if not supplied. if desc == "" { desc = srcConfig.Volume.Description } contentDBType, err := VolumeContentTypeNameToContentType(srcConfig.Volume.ContentType) if err != nil { return err } // Get the source volume's content type. contentType, err := VolumeDBContentTypeToContentType(contentDBType) if err != nil { return err } storagePoolSupported := slices.Contains(b.Driver().Info().VolumeTypes, drivers.VolumeTypeCustom) if !storagePoolSupported { return errors.New("Storage pool does not support custom volume type") } // If we are copying snapshots, retrieve a list of snapshots from source volume. var snapshotNames []string if snapshots { snapshotNames = make([]string, 0, len(srcConfig.VolumeSnapshots)) for _, snapshot := range srcConfig.VolumeSnapshots { snapshotNames = append(snapshotNames, snapshot.Name) } } reverter := revert.New() defer reverter.Fail() // Get the src volume name on storage. srcVolStorageName := project.StorageVolume(srcProjectName, srcVolName) srcVol := srcPool.GetVolume(drivers.VolumeTypeCustom, contentType, srcVolStorageName, srcConfig.Volume.Config) // If the source and target are in the same pool then use CreateVolumeFromCopy rather than // migration system as it will be quicker. if srcPool == b { l.Debug("CreateCustomVolumeFromCopy same-pool mode detected") // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, volName) vol := b.GetVolume(drivers.VolumeTypeCustom, contentType, volStorageName, config) // Validate config and create database entry for new storage volume. err = VolumeDBCreate(b, projectName, volName, desc, vol.Type(), false, vol.Config(), time.Now().UTC(), time.Time{}, vol.ContentType(), false, true) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, projectName, volName, vol.Type()) }) // Create database entries for new storage volume snapshots. for i, snapName := range snapshotNames { newSnapshotName := drivers.GetSnapshotVolumeName(volName, snapName) var volumeSnapExpiryDate time.Time if srcConfig.VolumeSnapshots[i].ExpiresAt != nil { volumeSnapExpiryDate = *srcConfig.VolumeSnapshots[i].ExpiresAt } // Validate config and create database entry for new storage volume. err = VolumeDBCreate(b, projectName, newSnapshotName, srcConfig.VolumeSnapshots[i].Description, vol.Type(), true, srcConfig.VolumeSnapshots[i].Config, srcConfig.VolumeSnapshots[i].CreatedAt, volumeSnapExpiryDate, vol.ContentType(), false, true) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, projectName, newSnapshotName, vol.Type()) }) } err = b.driver.CreateVolumeFromCopy(vol, srcVol, snapshots, false, op) if err != nil { return err } eventCtx := logger.Ctx{"type": vol.Type()} var location string if b.state.ServerClustered && !b.Driver().Info().Remote { eventCtx["location"] = b.state.ServerName location = b.state.ServerName } // Record new volume with authorizer. err = b.state.Authorizer.AddStoragePoolVolume(b.state.ShutdownCtx, projectName, b.Name(), vol.Type().Singular(), volName, location) if err != nil { logger.Error("Failed to add storage volume to authorizer", logger.Ctx{"name": volName, "type": vol.Type(), "pool": b.Name(), "project": projectName, "error": err}) } b.state.Events.SendLifecycle(projectName, lifecycle.StorageVolumeCreated.Event(vol, string(vol.Type()), projectName, op, eventCtx)) reverter.Success() return nil } // We are copying volumes between storage pools so use migration system as it will be able // to negotiate a common transfer method between pool types. l.Debug("CreateCustomVolumeFromCopy cross-pool mode detected") // Negotiate the migration type to use. offeredTypes := srcPool.MigrationTypes(contentType, false, snapshots, false, true) offerHeader := localMigration.TypesToHeader(offeredTypes...) migrationTypes, err := localMigration.MatchTypes(offerHeader, FallbackMigrationType(contentType), b.MigrationTypes(contentType, false, snapshots, false, true)) if err != nil { return fmt.Errorf("Failed to negotiate copy migration type: %w", err) } // If we're copying block volumes, the target block volume needs to be // at least the size of the source volume, otherwise we'll run into // "no space left on device". var volSize int64 if drivers.IsContentBlock(contentType) { err = srcVol.MountTask(func(mountPath string, op *operations.Operation) error { srcPoolBackend, ok := srcPool.(*backend) if !ok { return errors.New("Pool is not a backend") } volDiskPath, err := srcPoolBackend.driver.GetVolumeDiskPath(srcVol) if err != nil { return err } volSize, err = drivers.BlockDiskSizeBytes(volDiskPath) if err != nil { return err } return nil }, nil) if err != nil { return err } } var migrationSnapshots []*migration.Snapshot if snapshots { migrationSnapshots, err = VolumeSnapshotsToMigrationSnapshots(srcConfig.VolumeSnapshots, srcProjectName, srcPool, contentType, drivers.VolumeTypeCustom, srcVolName) if err != nil { return err } } ctx, cancel := context.WithCancel(context.Background()) // Use in-memory pipe pair to simulate a connection between the sender and receiver. aEnd, bEnd := memorypipe.NewPipePair(ctx) // Run sender and receiver in separate go routines to prevent deadlocks. aEndErrCh := make(chan error, 1) bEndErrCh := make(chan error, 1) go func() { err := srcPool.MigrateCustomVolume(srcProjectName, aEnd, &localMigration.VolumeSourceArgs{ IndexHeaderVersion: localMigration.IndexHeaderVersion, Name: srcVolName, Snapshots: snapshotNames, MigrationType: migrationTypes[0], TrackProgress: true, // Do use a progress tracker on sender. ContentType: string(contentType), Info: &localMigration.Info{Config: srcConfig}, VolumeOnly: !snapshots, StorageMove: true, }, op) if err != nil { cancel() } aEndErrCh <- err }() go func() { err := b.CreateCustomVolumeFromMigration(projectName, bEnd, localMigration.VolumeTargetArgs{ IndexHeaderVersion: localMigration.IndexHeaderVersion, Name: volName, Description: desc, Config: config, Snapshots: migrationSnapshots, MigrationType: migrationTypes[0], TrackProgress: false, // Do not use a progress tracker on receiver. ContentType: string(contentType), VolumeSize: volSize, // Block size setting override. VolumeOnly: !snapshots, StoragePool: srcPool.Name(), }, op) if err != nil { cancel() } bEndErrCh <- err }() // Capture errors from the sender and receiver from their result channels. errs := []error{} aEndErr := <-aEndErrCh if aEndErr != nil { _ = aEnd.Close() errs = append(errs, aEndErr) } bEndErr := <-bEndErrCh if bEndErr != nil { errs = append(errs, bEndErr) } cancel() if len(errs) > 0 { return fmt.Errorf("Create custom volume from copy failed: %v", errs) } reverter.Success() return nil } // migrationIndexHeaderSend sends the migration index header to target and waits for confirmation of receipt. func (b *backend) migrationIndexHeaderSend(l logger.Logger, indexHeaderVersion uint32, conn io.ReadWriteCloser, info *localMigration.Info) (*localMigration.InfoResponse, error) { infoResp := localMigration.InfoResponse{} // Send migration index header frame to target if applicable and wait for receipt. if indexHeaderVersion > 0 { headerJSON, err := json.Marshal(info) if err != nil { return nil, fmt.Errorf("Failed encoding migration index header: %w", err) } _, err = conn.Write(headerJSON) if err != nil { return nil, fmt.Errorf("Failed sending migration index header: %w", err) } err = conn.Close() // End the frame. if err != nil { return nil, fmt.Errorf("Failed closing migration index header frame: %w", err) } l.Debug("Sent migration index header, waiting for response", logger.Ctx{"version": indexHeaderVersion}) respBuf, err := io.ReadAll(conn) if err != nil { return nil, fmt.Errorf("Failed reading migration index header: %w", err) } err = json.Unmarshal(respBuf, &infoResp) if err != nil { return nil, fmt.Errorf("Failed decoding migration index header response: %w", err) } if infoResp.Err() != nil { return nil, fmt.Errorf("Failed negotiating migration options: %w", err) } l.Debug("Received migration index header response", logger.Ctx{"response": fmt.Sprintf("%+v", infoResp), "version": indexHeaderVersion}) } return &infoResp, nil } // migrationIndexHeaderReceive receives migration index header from source and sends confirmation of receipt. // Returns the received source index header info. func (b *backend) migrationIndexHeaderReceive(l logger.Logger, indexHeaderVersion uint32, conn io.ReadWriteCloser, refresh bool) (*localMigration.Info, error) { info := localMigration.Info{} // Receive index header from source if applicable and respond confirming receipt. if indexHeaderVersion > 0 { l.Debug("Waiting for migration index header", logger.Ctx{"version": indexHeaderVersion}) buf, err := io.ReadAll(conn) if err != nil { return nil, fmt.Errorf("Failed reading migration index header: %w", err) } err = json.Unmarshal(buf, &info) if err != nil { return nil, fmt.Errorf("Failed decoding migration index header: %w", err) } l.Debug("Received migration index header, sending response", logger.Ctx{"version": indexHeaderVersion}) infoResp := localMigration.InfoResponse{StatusCode: http.StatusOK, Refresh: &refresh} headerJSON, err := json.Marshal(infoResp) if err != nil { return nil, fmt.Errorf("Failed encoding migration index header response: %w", err) } _, err = conn.Write(headerJSON) if err != nil { return nil, fmt.Errorf("Failed sending migration index header response: %w", err) } err = conn.Close() // End the frame. if err != nil { return nil, fmt.Errorf("Failed closing migration index header response frame: %w", err) } l.Debug("Sent migration index header response", logger.Ctx{"version": indexHeaderVersion}) } return &info, nil } // MigrateCustomVolume sends a volume for migration. func (b *backend) MigrateCustomVolume(projectName string, conn io.ReadWriteCloser, args *localMigration.VolumeSourceArgs, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volName": args.Name, "args": fmt.Sprintf("%+v", args)}) l.Debug("MigrateCustomVolume started") defer l.Debug("MigrateCustomVolume finished") // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, args.Name) dbContentType, err := VolumeContentTypeNameToContentType(args.ContentType) if err != nil { return err } contentType, err := VolumeDBContentTypeToContentType(dbContentType) if err != nil { return err } if args.Info == nil { return errors.New("Migration info required") } if args.Info.Config == nil || args.Info.Config.Volume == nil || args.Info.Config.Volume.Config == nil { return errors.New("Volume config is required") } if len(args.Snapshots) != len(args.Info.Config.VolumeSnapshots) { return fmt.Errorf("Requested snapshots count (%d) doesn't match volume snapshot config count (%d)", len(args.Snapshots), len(args.Info.Config.VolumeSnapshots)) } // Send migration index header frame with volume info and wait for receipt. resp, err := b.migrationIndexHeaderSend(l, args.IndexHeaderVersion, conn, args.Info) if err != nil { return err } if resp.Refresh != nil { args.Refresh = *resp.Refresh } volConfig := args.Info.Config.Volume.Config vol := b.GetVolume(drivers.VolumeTypeCustom, contentType, volStorageName, volConfig) if volConfig["block.type"] == drivers.BlockVolumeTypeQcow2 && (!b.driver.Info().Remote || !args.ClusterMove || args.StorageMove) { err = b.qcow2MigrateVolume(b.state, vol, projectName, conn, args, op) if err != nil { return err } } else { err = b.driver.MigrateVolume(vol, conn, args, op) if err != nil { return err } } return nil } // CreateCustomVolumeFromMigration receives a volume being migrated. func (b *backend) CreateCustomVolumeFromMigration(projectName string, conn io.ReadWriteCloser, args localMigration.VolumeTargetArgs, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volName": args.Name, "args": fmt.Sprintf("%+v", args)}) l.Debug("CreateCustomVolumeFromMigration started") defer l.Debug("CreateCustomVolumeFromMigration finished") err := b.isStatusReady() if err != nil { return err } storagePoolSupported := slices.Contains(b.Driver().Info().VolumeTypes, drivers.VolumeTypeCustom) if !storagePoolSupported { return errors.New("Storage pool does not support custom volume type") } var volumeConfig map[string]string // Check if the volume exists in database. dbVol, err := VolumeDBGet(b, projectName, args.Name, drivers.VolumeTypeCustom) if err != nil && !response.IsNotFoundError(err) { return err } // Prefer using existing volume config (to allow mounting existing volume correctly). if dbVol != nil { volumeConfig = dbVol.Config } else { volumeConfig = args.Config } // Check if the volume exists on storage. volStorageName := project.StorageVolume(projectName, args.Name) vol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(args.ContentType), volStorageName, volumeConfig) volExists, err := b.driver.HasVolume(vol) if err != nil { return err } // Check for inconsistencies between database and storage before continuing. if dbVol == nil && volExists { return errors.New("Volume already exists on storage but not in database") } if dbVol != nil && !volExists { return errors.New("Volume exists in database but not on storage") } // Disable refresh mode if volume doesn't exist yet. // Unlike in CreateInstanceFromMigration there is no existing check for if the volume exists, so we must do // it here and disable refresh mode if the volume doesn't exist. if args.Refresh && !volExists { args.Refresh = false } else if !args.Refresh && volExists { return errors.New("Cannot create volume, already exists on migration target storage") } // VolumeSize is set to the actual size of the underlying block device. // The target should use this value if present, otherwise it might get an error like // "no space left on device". if args.VolumeSize > 0 { vol.SetConfigSize(fmt.Sprintf("%d", args.VolumeSize)) } // Receive index header from source if applicable and respond confirming receipt. // This will also let the source know whether to actually perform a refresh, as the target // will set Refresh to false if the volume doesn't exist. srcInfo, err := b.migrationIndexHeaderReceive(l, args.IndexHeaderVersion, conn, args.Refresh) if err != nil { return err } reverter := revert.New() defer reverter.Fail() if !args.Refresh { // Validate config and create database entry for new storage volume. // Strip unsupported config keys (in case the export was made from a different type of storage pool). err = VolumeDBCreate(b, projectName, args.Name, args.Description, vol.Type(), false, vol.Config(), time.Now().UTC(), time.Time{}, vol.ContentType(), true, true) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, projectName, args.Name, vol.Type()) }) } if len(args.Snapshots) > 0 { // Create database entries for new storage volume snapshots. for _, snapshot := range args.Snapshots { snapName := snapshot.GetName() newSnapshotName := drivers.GetSnapshotVolumeName(args.Name, snapName) snapConfig := vol.Config() // Use parent volume config by default. snapDescription := args.Description snapExpiryDate := time.Time{} snapCreationDate := time.Time{} // If the source snapshot config is available, use that. if srcInfo != nil && srcInfo.Config != nil { for _, srcSnap := range srcInfo.Config.VolumeSnapshots { if srcSnap.Name != snapName { continue } snapConfig = srcSnap.Config snapDescription = srcSnap.Description if srcSnap.ExpiresAt != nil { snapExpiryDate = *srcSnap.ExpiresAt } snapCreationDate = srcSnap.CreatedAt break } } // Validate config and create database entry for new storage volume. // Strip unsupported config keys (in case the export was made from a different type of storage pool). err = VolumeDBCreate(b, projectName, newSnapshotName, snapDescription, vol.Type(), true, snapConfig, snapCreationDate, snapExpiryDate, vol.ContentType(), true, true) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, projectName, newSnapshotName, vol.Type()) }) } } if b.driver.Info().TargetFormat == drivers.BlockVolumeTypeQcow2 && (!b.driver.Info().Remote || args.ClusterMoveSourceName == "" || args.StoragePool != "") { err = b.qcow2CreateVolumeFromMigration(vol, projectName, conn, args, nil, op) if err != nil { return err } } else { err = b.driver.CreateVolumeFromMigration(vol, conn, args, nil, op) if err != nil { return err } } eventCtx := logger.Ctx{"type": vol.Type()} var location string if b.state.ServerClustered && !b.Driver().Info().Remote { eventCtx["location"] = b.state.ServerName location = b.state.ServerName } // Record new volume with authorizer. err = b.state.Authorizer.AddStoragePoolVolume(b.state.ShutdownCtx, projectName, b.Name(), vol.Type().Singular(), args.Name, location) if err != nil { logger.Error("Failed to add storage volume to authorizer", logger.Ctx{"name": args.Name, "type": vol.Type(), "pool": b.Name(), "project": projectName, "error": err}) } b.state.Events.SendLifecycle(projectName, lifecycle.StorageVolumeCreated.Event(vol, string(vol.Type()), projectName, op, eventCtx)) reverter.Success() return nil } // RenameCustomVolume renames a custom volume and its snapshots. func (b *backend) RenameCustomVolume(projectName string, volName string, newVolName string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volName": volName, "newVolName": newVolName}) l.Debug("RenameCustomVolume started") defer l.Debug("RenameCustomVolume finished") if internalInstance.IsSnapshot(volName) { return errors.New("Volume name cannot be a snapshot") } if internalInstance.IsSnapshot(newVolName) { return errors.New("New volume name cannot be a snapshot") } reverter := revert.New() defer reverter.Fail() volume, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return err } // Rename each snapshot to have the new parent volume prefix. snapshots, err := VolumeDBSnapshotsGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return err } for _, srcSnapshot := range snapshots { _, snapName, _ := api.GetParentAndSnapshotName(srcSnapshot.Name) newSnapVolName := drivers.GetSnapshotVolumeName(newVolName, snapName) err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RenameStoragePoolVolume(ctx, projectName, srcSnapshot.Name, newSnapVolName, db.StoragePoolVolumeTypeCustom, b.ID()) }) if err != nil { return err } reverter.Add(func() { _ = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RenameStoragePoolVolume(ctx, projectName, newSnapVolName, srcSnapshot.Name, db.StoragePoolVolumeTypeCustom, b.ID()) }) }) } var backups []db.StoragePoolVolumeBackup // Rename each backup to have the new parent volume prefix. err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error backups, err = tx.GetStoragePoolVolumeBackups(ctx, projectName, volName, b.ID()) return err }) if err != nil { return err } for _, br := range backups { backupRow := br // Local var for revert. _, backupName, _ := api.GetParentAndSnapshotName(backupRow.Name) newVolBackupName := drivers.GetSnapshotVolumeName(newVolName, backupName) volBackup := backup.NewVolumeBackup(b.state, projectName, b.name, volName, backupRow.ID, backupRow.Name, backupRow.CreationDate, backupRow.ExpiryDate, backupRow.VolumeOnly, backupRow.OptimizedStorage) err = volBackup.Rename(newVolBackupName) if err != nil { return fmt.Errorf("Failed renaming backup %q to %q: %w", backupRow.Name, newVolBackupName, err) } reverter.Add(func() { _ = volBackup.Rename(backupRow.Name) }) } err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RenameStoragePoolVolume(ctx, projectName, volName, newVolName, db.StoragePoolVolumeTypeCustom, b.ID()) }) if err != nil { return err } reverter.Add(func() { _ = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RenameStoragePoolVolume(ctx, projectName, newVolName, volName, db.StoragePoolVolumeTypeCustom, b.ID()) }) }) // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, volName) newVolStorageName := project.StorageVolume(projectName, newVolName) vol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(volume.ContentType), volStorageName, volume.Config) err = b.driver.RenameVolume(vol, newVolStorageName, op) if err != nil { return err } var location string if b.state.ServerClustered && !b.Driver().Info().Remote { location = b.state.ServerName } err = b.state.Authorizer.RenameStoragePoolVolume(b.state.ShutdownCtx, projectName, b.Name(), vol.Type().Singular(), volName, newVolStorageName, location) if err != nil { logger.Error("Failed to rename storage volume in authorizer", logger.Ctx{"old_name": volName, "new_name": newVolStorageName, "type": vol.Type(), "pool": b.Name(), "project": projectName, "error": err}) } vol = b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(volume.ContentType), newVolStorageName, nil) b.state.Events.SendLifecycle(projectName, lifecycle.StorageVolumeRenamed.Event(vol, string(vol.Type()), projectName, op, logger.Ctx{"old_name": volName})) reverter.Success() return nil } // detectChangedConfig returns the config that has changed between current and new config maps. // Also returns a boolean indicating whether all of the changed keys start with "user.". // Deleted keys will be returned as having an empty string value. func (b *backend) detectChangedConfig(curConfig, newConfig map[string]string) (map[string]string, bool) { // Diff the configurations. changedConfig := make(map[string]string) userOnly := true for key := range curConfig { if curConfig[key] != newConfig[key] { if !strings.HasPrefix(key, "user.") { userOnly = false } changedConfig[key] = newConfig[key] // Will be empty string on deleted keys. } } for key := range newConfig { if curConfig[key] != newConfig[key] { if !strings.HasPrefix(key, "user.") { userOnly = false } changedConfig[key] = newConfig[key] } } return changedConfig, userOnly } // UpdateCustomVolume applies the supplied config to the custom volume. func (b *backend) UpdateCustomVolume(projectName string, volName string, newDesc string, newConfig map[string]string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volName": volName, "newDesc": newDesc, "newConfig": newConfig}) l.Debug("UpdateCustomVolume started") defer l.Debug("UpdateCustomVolume finished") if internalInstance.IsSnapshot(volName) { return errors.New("Volume name cannot be a snapshot") } // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, volName) // Get current config to compare what has changed. curVol, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return err } // Get content type. dbContentType, err := VolumeContentTypeNameToContentType(curVol.ContentType) if err != nil { return err } contentType, err := VolumeDBContentTypeToContentType(dbContentType) if err != nil { return err } // Validate config. newVol := b.GetVolume(drivers.VolumeTypeCustom, contentType, volStorageName, newConfig) err = b.driver.ValidateVolume(newVol, false) if err != nil { return err } // Apply config changes if there are any. changedConfig, userOnly := b.detectChangedConfig(curVol.Config, newConfig) if len(changedConfig) != 0 { // Forbid changing the config for ISO custom volumes as they are read-only. if contentType == drivers.ContentTypeISO { return errors.New("Custom ISO volume config cannot be changed") } // Check that the volume's block.filesystem property isn't being changed. if changedConfig["block.filesystem"] != "" { return errors.New(`Custom volume "block.filesystem" property cannot be changed`) } // Check for config changing that is not allowed when running instances are using it. if changedConfig["security.shifted"] != "" { err = VolumeUsedByInstanceDevices(b.state, b.name, projectName, &curVol.StorageVolume, true, func(dbInst db.InstanceArgs, project api.Project, usedByDevices []string) error { inst, err := instance.Load(b.state, dbInst, project) if err != nil { return err } // Confirm that no running instances are using it when changing shifted state. if inst.IsRunning() && changedConfig["security.shifted"] != "" { return errors.New("Cannot modify shifting with running instances using the volume") } return nil }) if err != nil { return err } } sharedVolume, ok := changedConfig["security.shared"] if ok && util.IsFalseOrEmpty(sharedVolume) { var usedByProfileDevices []api.Profile err = VolumeUsedByProfileDevices(b.state, b.name, projectName, &curVol.StorageVolume, func(profileID int64, profile api.Profile, project api.Project, usedByDevices []string) error { usedByProfileDevices = append(usedByProfileDevices, profile) return nil }) if err != nil { return err } if len(usedByProfileDevices) > 0 { return errors.New("Cannot un-share custom storage block volume if attached to profile") } var usedByInstanceDevices []string err = VolumeUsedByInstanceDevices(b.state, b.name, projectName, &curVol.StorageVolume, true, func(inst db.InstanceArgs, project api.Project, usedByDevices []string) error { usedByInstanceDevices = append(usedByInstanceDevices, inst.Name) return nil }) if err != nil { return err } if len(usedByInstanceDevices) > 1 { return errors.New("Cannot un-share custom storage block volume if attached to more than one instance") } } curVol := b.GetVolume(drivers.VolumeTypeCustom, contentType, volStorageName, curVol.Config) if !userOnly { err = b.driver.UpdateVolume(curVol, changedConfig) if err != nil { return err } } } // Unset idmap keys if volume is unmapped. if util.IsTrue(newConfig["security.unmapped"]) { delete(newConfig, "volatile.idmap.last") delete(newConfig, "volatile.idmap.next") } // Notify instances of disk size changes as needed. newSize, ok := changedConfig["size"] if ok && newSize != "" && contentType == drivers.ContentTypeBlock { // Get the disk size in bytes. size, err := units.ParseByteSizeString(changedConfig["size"]) if err != nil { return err } type instDevice struct { args db.InstanceArgs devices []string } instDevices := []instDevice{} err = VolumeUsedByInstanceDevices(b.state, b.name, projectName, &curVol.StorageVolume, true, func(dbInst db.InstanceArgs, project api.Project, usedByDevices []string) error { if dbInst.Type != instancetype.VM { return nil } instDevices = append(instDevices, instDevice{args: dbInst, devices: usedByDevices}) return nil }) if err != nil { return err } for _, entry := range instDevices { c, err := ConnectIfInstanceIsRemote(b.state, entry.args.Project, entry.args.Name, nil) if err != nil { return err } if c != nil { // Send a remote notification. devs := []string{} for _, devName := range entry.devices { devs = append(devs, fmt.Sprintf("%s:%d", devName, size)) } uri := fmt.Sprintf("/internal/virtual-machines/%d/onresize?devices=%s", entry.args.ID, strings.Join(devs, ",")) _, _, err := c.RawQuery("GET", uri, nil, "") if err != nil { return err } } else { // Update the local instance. inst, err := instance.LoadByProjectAndName(b.state, entry.args.Project, entry.args.Name) if err != nil { return err } if !inst.IsRunning() { continue } for _, devName := range entry.devices { runConf := deviceConfig.RunConfig{} runConf.Mounts = []deviceConfig.MountEntryItem{ { DevName: devName, Size: size, }, } err = inst.DeviceEventHandler(&runConf) if err != nil { return err } } } } } // Update the database if something changed. if len(changedConfig) != 0 || newDesc != curVol.Description { err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateStoragePoolVolume(ctx, projectName, volName, db.StoragePoolVolumeTypeCustom, b.ID(), newDesc, newConfig) }) if err != nil { return err } } b.state.Events.SendLifecycle(projectName, lifecycle.StorageVolumeUpdated.Event(newVol, string(newVol.Type()), projectName, op, nil)) return nil } // UpdateCustomVolumeSnapshot updates the description of a custom volume snapshot. // Volume config is not allowed to be updated and will return an error. func (b *backend) UpdateCustomVolumeSnapshot(projectName string, volName string, newDesc string, newConfig map[string]string, newExpiryDate time.Time, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volName": volName, "newDesc": newDesc, "newConfig": newConfig, "newExpiryDate": newExpiryDate}) l.Debug("UpdateCustomVolumeSnapshot started") defer l.Debug("UpdateCustomVolumeSnapshot finished") if !internalInstance.IsSnapshot(volName) { return errors.New("Volume must be a snapshot") } // Get current config to compare what has changed. curVol, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return err } var curExpiryDate time.Time err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { curExpiryDate, err = tx.GetStorageVolumeSnapshotExpiry(ctx, curVol.ID) return err }) if err != nil { return err } if newConfig != nil { changedConfig, _ := b.detectChangedConfig(curVol.Config, newConfig) if len(changedConfig) != 0 { return errors.New("Volume config is not editable") } } // Update the database if description changed. Use current config. if newDesc != curVol.Description || newExpiryDate != curExpiryDate { err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.UpdateStorageVolumeSnapshot(ctx, projectName, volName, db.StoragePoolVolumeTypeCustom, b.ID(), newDesc, curVol.Config, newExpiryDate) }) if err != nil { return err } } vol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(curVol.ContentType), curVol.Name, curVol.Config) b.state.Events.SendLifecycle(projectName, lifecycle.StorageVolumeSnapshotUpdated.Event(vol, string(vol.Type()), projectName, op, nil)) return nil } // DeleteCustomVolume removes a custom volume and its snapshots. func (b *backend) DeleteCustomVolume(projectName string, volName string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volName": volName}) l.Debug("DeleteCustomVolume started") defer l.Debug("DeleteCustomVolume finished") _, _, isSnap := api.GetParentAndSnapshotName(volName) if isSnap { return errors.New("Volume name cannot be a snapshot") } // Retrieve a list of snapshots. snapshots, err := VolumeDBSnapshotsGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return err } // Remove each snapshot. for _, snapshot := range snapshots { err = b.DeleteCustomVolumeSnapshot(projectName, snapshot.Name, op) if err != nil { return err } } // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, volName) // Get the volume. curVol, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return err } // Get the content type. dbContentType, err := VolumeContentTypeNameToContentType(curVol.ContentType) if err != nil { return err } contentType, err := VolumeDBContentTypeToContentType(dbContentType) if err != nil { return err } // There's no need to pass config as it's not needed when deleting a volume. vol := b.GetVolume(drivers.VolumeTypeCustom, contentType, volStorageName, nil) // Forcefully stop any forkfile process if running. vol.StopForkfile() // Delete the volume from the storage device. Must come after snapshots are removed. volExists, err := b.driver.HasVolume(vol) if err != nil { return err } if volExists { err = b.driver.DeleteVolume(vol, op) if err != nil { return err } } // Remove backups directory for volume. backupsPath := internalUtil.VarPath("backups", "custom", b.name, project.StorageVolume(projectName, volName)) if util.PathExists(backupsPath) { err := os.RemoveAll(backupsPath) if err != nil { return err } } // Finally, remove the volume record from the database. err = VolumeDBDelete(b, projectName, volName, vol.Type()) if err != nil { return err } var location string if b.state.ServerClustered && !b.Driver().Info().Remote { location = b.state.ServerName } // Record volume deletion with authorizer. err = b.state.Authorizer.DeleteStoragePoolVolume(b.state.ShutdownCtx, projectName, b.Name(), vol.Type().Singular(), volName, location) if err != nil { logger.Error("Failed to remove storage volume from authorizer", logger.Ctx{"name": volName, "type": vol.Type(), "pool": b.Name(), "project": projectName, "error": err}) } b.state.Events.SendLifecycle(projectName, lifecycle.StorageVolumeDeleted.Event(vol, string(vol.Type()), projectName, op, nil)) return nil } // GetCustomVolumeDisk returns the location of the disk. func (b *backend) GetCustomVolumeDisk(projectName, volName string) (string, error) { volume, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return "", err } // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, volName) // There's no need to pass config as it's not needed when getting the volume usage. vol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(volume.ContentType), volStorageName, nil) return b.driver.GetVolumeDiskPath(vol) } // GetCustomVolumeUsage returns the disk space used by the custom volume. func (b *backend) GetCustomVolumeUsage(projectName, volName string) (*VolumeUsage, error) { err := b.isStatusReady() if err != nil { return nil, err } volume, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return nil, err } val := VolumeUsage{} // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, volName) // There's no need to pass config as it's not needed when getting the volume usage. vol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(volume.ContentType), volStorageName, nil) // Get the usage. size, err := b.driver.GetVolumeUsage(vol) if err != nil { return nil, err } val.Used = size // Get the total size. sizeStr, ok := vol.Config()["size"] if ok { total, err := units.ParseByteSizeString(sizeStr) if err != nil { return nil, err } if total >= 0 { val.Total = total } } return &val, nil } // MountCustomVolume mounts a custom volume. func (b *backend) MountCustomVolume(projectName, volName string, op *operations.Operation) (*MountInfo, error) { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volName": volName}) l.Debug("MountCustomVolume started") defer l.Debug("MountCustomVolume finished") err := b.isStatusReady() if err != nil { return nil, err } volume, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return nil, err } // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, volName) vol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(volume.ContentType), volStorageName, volume.Config) // Perform the mount. mountInfo := &MountInfo{} err = b.driver.MountVolume(vol, op) if err != nil { return nil, err } backingPaths := []string{} if vol.Config()["block.type"] == drivers.BlockVolumeTypeQcow2 { // Get snapshots. _, volName := project.StorageVolumeParts(vol.Name()) volSnaps, err := VolumeDBSnapshotsGet(b, projectName, volName, vol.Type()) if err != nil { return nil, err } for _, snap := range volSnaps { currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.StorageVolume(projectName, snap.Name), vol.Config()) err = b.driver.MountVolumeSnapshot(currentSnapVol, op) if err != nil { return nil, err } } // Get the location of the disk block device. diskPath, err := b.driver.GetVolumeDiskPath(vol) if err != nil { return nil, err } // Fetch backing chain for a qcow2 formatted volume. backingPaths, err = b.qcow2BackingPaths(vol, diskPath, projectName) if err != nil { return nil, err } } l.Debug("Clustom volume backing paths", logger.Ctx{"backingPaths": backingPaths}) mountInfo.BackingPath = append(mountInfo.BackingPath, backingPaths...) // Handle delegation. if b.driver.CanDelegateVolume(vol) { mountInfo.PostHooks = append(mountInfo.PostHooks, func(inst instance.Instance) error { pid := inst.InitPID() // Only apply to running instances. if pid < 1 { return nil } return b.driver.DelegateVolume(vol, pid) }) } return mountInfo, nil } // UnmountCustomVolume unmounts a custom volume. func (b *backend) UnmountCustomVolume(projectName, volName string, op *operations.Operation) (bool, error) { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volName": volName}) l.Debug("UnmountCustomVolume started") defer l.Debug("UnmountCustomVolume finished") volume, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return false, err } // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, volName) vol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(volume.ContentType), volStorageName, volume.Config) if vol.Config()["block.type"] == drivers.BlockVolumeTypeQcow2 { // Get snapshots. volSnaps, err := VolumeDBSnapshotsGet(b, projectName, volName, vol.Type()) if err != nil { return false, err } for _, snap := range volSnaps { currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.StorageVolume(projectName, snap.Name), vol.Config()) _, err = b.driver.UnmountVolumeSnapshot(currentSnapVol, op) if err != nil && !errors.Is(err, drivers.ErrInUse) { return false, err } } } return b.driver.UnmountVolume(vol, false, op) } // ImportCustomVolume takes an existing custom volume on the storage backend and ensures that the DB records, // volume directories and symlinks are restored as needed to make it operational with Incus. // Used during the recovery import stage. func (b *backend) ImportCustomVolume(projectName string, poolVol *backupConfig.Config, op *operations.Operation) (revert.Hook, error) { if poolVol.Volume == nil { return nil, errors.New("Invalid pool volume config supplied") } l := b.logger.AddContext(logger.Ctx{"project": projectName, "volName": poolVol.Volume.Name}) l.Debug("ImportCustomVolume started") defer l.Debug("ImportCustomVolume finished") reverter := revert.New() defer reverter.Fail() // Copy volume config from backup file if present (so VolumeDBCreate can safely modify the copy if needed). volumeConfig := util.CloneMap(poolVol.Volume.Config) // Validate config and create database entry for restored storage volume. err := VolumeDBCreate(b, projectName, poolVol.Volume.Name, poolVol.Volume.Description, drivers.VolumeTypeCustom, false, volumeConfig, poolVol.Volume.CreatedAt, time.Time{}, drivers.ContentType(poolVol.Volume.ContentType), false, true) if err != nil { return nil, err } reverter.Add(func() { _ = VolumeDBDelete(b, projectName, poolVol.Volume.Name, drivers.VolumeTypeCustom) }) // Create the storage volume snapshot DB records. for _, poolVolSnap := range poolVol.VolumeSnapshots { fullSnapName := drivers.GetSnapshotVolumeName(poolVol.Volume.Name, poolVolSnap.Name) // Copy volume config from backup file if present // (so VolumeDBCreate can safely modify the copy if needed). snapVolumeConfig := util.CloneMap(poolVolSnap.Config) // Validate config and create database entry for restored storage volume. err = VolumeDBCreate(b, projectName, fullSnapName, poolVolSnap.Description, drivers.VolumeTypeCustom, true, snapVolumeConfig, poolVolSnap.CreatedAt, time.Time{}, drivers.ContentType(poolVolSnap.ContentType), false, true) if err != nil { return nil, err } reverter.Add(func() { _ = VolumeDBDelete(b, projectName, fullSnapName, drivers.VolumeTypeCustom) }) } // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, poolVol.Volume.Name) vol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(poolVol.Volume.ContentType), volStorageName, volumeConfig) // Create the mount path if needed. err = vol.EnsureMountPath(false) if err != nil { return nil, err } // Create snapshot mount paths and snapshot parent directory if needed. for _, poolVolSnap := range poolVol.VolumeSnapshots { l.Debug("Ensuring instance snapshot mount path", logger.Ctx{"snapshot": poolVolSnap.Name}) snapVol, err := vol.NewSnapshot(poolVolSnap.Name) if err != nil { return nil, err } err = snapVol.EnsureMountPath(false) if err != nil { return nil, err } } cleanup := reverter.Clone().Fail reverter.Success() return cleanup, err } // CreateCustomVolumeSnapshot creates a snapshot of a custom volume. func (b *backend) CreateCustomVolumeSnapshot(projectName, volName string, newSnapshotName string, newExpiryDate time.Time, instanceStateful bool, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volName": volName, "newSnapshotName": newSnapshotName, "newExpiryDate": newExpiryDate}) l.Debug("CreateCustomVolumeSnapshot started") defer l.Debug("CreateCustomVolumeSnapshot finished") if internalInstance.IsSnapshot(volName) { return errors.New("Volume does not support snapshots") } if internalInstance.IsSnapshot(newSnapshotName) { return errors.New("Snapshot name is not a valid snapshot name") } fullSnapshotName := drivers.GetSnapshotVolumeName(volName, newSnapshotName) // Check snapshot volume doesn't exist already. volume, err := VolumeDBGet(b, projectName, fullSnapshotName, drivers.VolumeTypeCustom) if err != nil && !response.IsNotFoundError(err) { return err } else if volume != nil { return api.StatusErrorf(http.StatusConflict, "Snapshot by that name already exists") } // Load parent volume information and check it exists. parentVol, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { if response.IsNotFoundError(err) { return api.StatusErrorf(http.StatusNotFound, "Parent volume doesn't exist") } return err } volDBContentType, err := VolumeContentTypeNameToContentType(parentVol.ContentType) if err != nil { return err } contentType, err := VolumeDBContentTypeToContentType(volDBContentType) if err != nil { return err } if contentType != drivers.ContentTypeFS && contentType != drivers.ContentTypeBlock { return fmt.Errorf("Volume of content type %q does not support snapshots", contentType) } reverter := revert.New() defer reverter.Fail() // Validate config and create database entry for new storage volume. // Copy volume config from parent. err = VolumeDBCreate(b, projectName, fullSnapshotName, parentVol.Description, drivers.VolumeTypeCustom, true, parentVol.Config, time.Now().UTC(), newExpiryDate, drivers.ContentType(parentVol.ContentType), false, true) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, projectName, fullSnapshotName, drivers.VolumeTypeCustom) }) // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, fullSnapshotName) vol := b.GetVolume(drivers.VolumeTypeCustom, contentType, volStorageName, parentVol.Config) // Lock this operation to ensure that the only one snapshot is made at the time. // Other operations will wait for this one to finish. unlock, err := locking.Lock(context.TODO(), drivers.OperationLockName("CreateCustomVolumeSnapshot", b.name, vol.Type(), contentType, volName)) if err != nil { return err } defer unlock() // Create the snapshot on the storage device. err = b.driver.CreateVolumeSnapshot(vol, op) if err != nil { return err } if parentVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { // Get the parent volume. volStorageParentName := project.StorageVolume(projectName, volName) parentVolume := b.GetVolume(drivers.VolumeTypeCustom, contentType, volStorageParentName, parentVol.Config) reverter.Add(func() { _ = b.driver.Qcow2DeletionCleanup(vol, volStorageParentName) }) inst, devName, err := b.volumeUsedByRunningInstance(parentVol, projectName) if err != nil { return err } // parentVol should already be prepared as an overlay by CreateVolumeSnapshot. // vol will be used as the base. err = b.qcow2CreateSnapshot(parentVolume, vol, projectName, inst, devName, instanceStateful, op) if err != nil { return err } reverter.Add(func() { _ = b.qcow2DeleteSnapshot(parentVolume, vol, projectName, inst, devName, nil) }) } b.state.Events.SendLifecycle(projectName, lifecycle.StorageVolumeSnapshotCreated.Event(vol, string(vol.Type()), projectName, op, logger.Ctx{"type": vol.Type()})) reverter.Success() return nil } // RenameCustomVolumeSnapshot renames a custom volume. func (b *backend) RenameCustomVolumeSnapshot(projectName, volName string, newSnapshotName string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volName": volName, "newSnapshotName": newSnapshotName}) l.Debug("RenameCustomVolumeSnapshot started") defer l.Debug("RenameCustomVolumeSnapshot finished") parentName, oldSnapshotName, isSnap := api.GetParentAndSnapshotName(volName) if !isSnap { return errors.New("Volume name must be a snapshot") } if internalInstance.IsSnapshot(newSnapshotName) { return errors.New("Invalid new snapshot name") } volume, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return err } // Get the parent volume. parentVolume, err := VolumeDBGet(b, projectName, parentName, drivers.VolumeTypeCustom) if err != nil { return err } // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, volName) vol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(volume.ContentType), volStorageName, volume.Config) newVolName := drivers.GetSnapshotVolumeName(parentName, newSnapshotName) if parentVolume.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { parentVol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(parentVolume.ContentType), project.StorageVolume(projectName, parentName), parentVolume.Config) err = b.qcow2RenameSnapshot(parentVol, vol, newVolName, projectName, op) if err != nil { return err } } err = b.driver.RenameVolumeSnapshot(vol, newSnapshotName, op) if err != nil { return err } err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RenameStoragePoolVolume(ctx, projectName, volName, newVolName, db.StoragePoolVolumeTypeCustom, b.ID()) }) if err != nil { // Get the volume name on storage. newVolStorageName := project.StorageVolume(projectName, newVolName) // Revert rename. newVol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(volume.ContentType), newVolStorageName, nil) _ = b.driver.RenameVolumeSnapshot(newVol, oldSnapshotName, op) return err } b.state.Events.SendLifecycle(projectName, lifecycle.StorageVolumeSnapshotRenamed.Event(vol, string(vol.Type()), projectName, op, logger.Ctx{"old_name": oldSnapshotName})) return nil } // DeleteCustomVolumeSnapshot removes a custom volume snapshot. func (b *backend) DeleteCustomVolumeSnapshot(projectName, volName string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volName": volName}) l.Debug("DeleteCustomVolumeSnapshot started") defer l.Debug("DeleteCustomVolumeSnapshot finished") parentName, _, isSnap := api.GetParentAndSnapshotName(volName) if !isSnap { return errors.New("Volume name must be a snapshot") } // Get the volume. volume, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return err } // Get the parent volume. parentVolume, err := VolumeDBGet(b, projectName, parentName, drivers.VolumeTypeCustom) if err != nil { return err } // Get the content type. dbContentType, err := VolumeContentTypeNameToContentType(volume.ContentType) if err != nil { return err } contentType, err := VolumeDBContentTypeToContentType(dbContentType) if err != nil { return err } // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, volName) vol := b.GetVolume(drivers.VolumeTypeCustom, contentType, volStorageName, volume.Config) // Delete the snapshot from the storage device. // Must come before DB VolumeDBDelete so that the volume ID is still available. volExists, err := b.driver.HasVolume(vol) if err != nil { return err } if volExists { if parentVolume.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { inst, devName, err := b.volumeUsedByRunningInstance(parentVolume, projectName) if err != nil { return err } parentVol := b.GetVolume(drivers.VolumeTypeCustom, contentType, project.StorageVolume(projectName, parentName), parentVolume.Config) err = b.qcow2DeleteSnapshot(parentVol, vol, projectName, inst, devName, op) if err != nil { return err } } else { err := b.driver.DeleteVolumeSnapshot(vol, op) if err != nil { return err } } } // Remove the snapshot volume record from the database. err = VolumeDBDelete(b, projectName, volName, vol.Type()) if err != nil { return err } b.state.Events.SendLifecycle(projectName, lifecycle.StorageVolumeSnapshotDeleted.Event(vol, string(vol.Type()), projectName, op, nil)) return nil } // RestoreCustomVolume restores a custom volume from a snapshot. func (b *backend) RestoreCustomVolume(projectName, volName string, snapshotName string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volName": volName, "snapshotName": snapshotName}) l.Debug("RestoreCustomVolume started") defer l.Debug("RestoreCustomVolume finished") // Quick checks. if internalInstance.IsSnapshot(volName) { return errors.New("Volume cannot be snapshot") } if internalInstance.IsSnapshot(snapshotName) { return errors.New("Invalid snapshot name") } // Get current volume. curVol, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return err } // Check that the volume isn't in use by running instances. err = VolumeUsedByInstanceDevices(b.state, b.Name(), projectName, &curVol.StorageVolume, true, func(dbInst db.InstanceArgs, project api.Project, usedByDevices []string) error { inst, err := instance.Load(b.state, dbInst, project) if err != nil { return err } if inst.IsRunning() { return errors.New("Cannot restore custom volume used by running instances") } return nil }) if err != nil { return err } dbContentType, err := VolumeContentTypeNameToContentType(curVol.ContentType) if err != nil { return err } contentType, err := VolumeDBContentTypeToContentType(dbContentType) if err != nil { return err } // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, volName) vol := b.GetVolume(drivers.VolumeTypeCustom, contentType, volStorageName, curVol.Config) deleteSnapshots := func(snapshots []string) error { for _, snapName := range snapshots { err := b.DeleteCustomVolumeSnapshot(projectName, fmt.Sprintf("%s/%s", volName, snapName), op) if err != nil { return err } } return nil } if curVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { fullSnapName := fmt.Sprintf("%s/%s", volName, snapshotName) snapVol := b.GetVolume(drivers.VolumeTypeCustom, contentType, project.StorageVolume(projectName, fullSnapName), curVol.Config) err = b.qcow2RestoreSnapshot(vol, snapVol, projectName, op) if err != nil { var snapErr drivers.ErrDeleteSnapshots if errors.As(err, &snapErr) { err = deleteSnapshots(snapErr.Snapshots) if err != nil { return err } // Now try again. err = b.qcow2RestoreSnapshot(vol, snapVol, projectName, op) if err != nil { return err } } return err } return nil } err = b.driver.RestoreVolume(vol, snapshotName, op) if err != nil { var snapErr drivers.ErrDeleteSnapshots if errors.As(err, &snapErr) { err = deleteSnapshots(snapErr.Snapshots) if err != nil { return err } // Now try again. err = b.driver.RestoreVolume(vol, snapshotName, op) if err != nil { return err } } return err } b.state.Events.SendLifecycle(projectName, lifecycle.StorageVolumeRestored.Event(vol, string(vol.Type()), projectName, op, logger.Ctx{"snapshot": snapshotName})) return nil } func (b *backend) createStorageStructure(path string) error { for _, volType := range b.driver.Info().VolumeTypes { for _, name := range drivers.BaseDirectories[volType].Paths { path := filepath.Join(path, name) err := os.MkdirAll(path, drivers.BaseDirectories[volType].Mode) if err != nil && !os.IsExist(err) { return fmt.Errorf("Failed to create directory %q: %w", path, err) } } } return nil } // GenerateBucketBackupConfig returns the backup config entry for this bucket. func (b *backend) GenerateBucketBackupConfig(projectName string, bucketName string, op *operations.Operation) (*backupConfig.Config, error) { bucket, err := BucketDBGet(b, projectName, bucketName, true) if err != nil { return nil, err } dbBucketKeys, err := BucketKeysDBGet(b, bucket.ID) if err != nil { return nil, err } var bucketKeys []*api.StorageBucketKey for _, key := range dbBucketKeys { bucketKeys = append(bucketKeys, &key.StorageBucketKey) } config := &backupConfig.Config{ Bucket: &bucket.StorageBucket, BucketKeys: bucketKeys, } return config, nil } // GenerateCustomVolumeBackupConfig returns the backup config entry for this volume. func (b *backend) GenerateCustomVolumeBackupConfig(projectName string, volName string, snapshots bool, op *operations.Operation) (*backupConfig.Config, error) { vol, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return nil, err } if vol.Type != db.StoragePoolVolumeTypeNameCustom { return nil, fmt.Errorf("Unsupported volume type %q", vol.Type) } config := &backupConfig.Config{ Volume: &vol.StorageVolume, } if snapshots { dbVolSnaps, err := VolumeDBSnapshotsGet(b, projectName, vol.Name, drivers.VolumeTypeCustom) if err != nil { return nil, err } config.VolumeSnapshots = make([]*api.StorageVolumeSnapshot, 0, len(dbVolSnaps)) for i := range dbVolSnaps { _, snapName, _ := api.GetParentAndSnapshotName(dbVolSnaps[i].Name) snapshot := api.StorageVolumeSnapshot{ StorageVolumeSnapshotPut: api.StorageVolumeSnapshotPut{ Description: dbVolSnaps[i].Description, ExpiresAt: &dbVolSnaps[i].ExpiryDate, }, Name: snapName, // Snapshot only name, not full name. Config: dbVolSnaps[i].Config, ContentType: dbVolSnaps[i].ContentType, CreatedAt: dbVolSnaps[i].CreationDate, } config.VolumeSnapshots = append(config.VolumeSnapshots, &snapshot) } } return config, nil } // GenerateInstanceBackupConfig returns the backup config entry for this instance. // The Container field is only populated for non-snapshot instances. func (b *backend) GenerateInstanceBackupConfig(inst instance.Instance, snapshots bool, dependentVolumes bool, op *operations.Operation) (*backupConfig.Config, error) { // Generate the YAML. ci, _, err := inst.Render() if err != nil { return nil, fmt.Errorf("Failed to render instance metadata: %w", err) } volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return nil, err } volume, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return nil, err } config := &backupConfig.Config{ Pool: &b.db, Volume: &volume.StorageVolume, } // Add profiles from instance. instProfiles := inst.Profiles() config.Profiles = make([]*api.Profile, len(instProfiles)) for i := range instProfiles { config.Profiles[i] = &instProfiles[i] } if dependentVolumes { config.DependentVolumes = []*backupConfig.Config{} err = inst.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error { // Load the pool for the disk. diskPool, err := LoadByName(b.state, dev.Config["pool"]) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } diskConfig, err := diskPool.GenerateCustomVolumeBackupConfig(inst.Project().Name, dev.Config["source"], snapshots, op) if err != nil { return err } poolDB := diskPool.ToAPI() diskConfig.Pool = &poolDB config.DependentVolumes = append(config.DependentVolumes, diskConfig) return nil }) if err != nil { return nil, err } } // Only populate Container field for non-snapshot instances. if !inst.IsSnapshot() { config.Container = ci.(*api.Instance) if snapshots { snapshots, err := inst.Snapshots() if err != nil { return nil, fmt.Errorf("Failed to get snapshots: %w", err) } config.Snapshots = make([]*api.InstanceSnapshot, 0, len(snapshots)) for _, s := range snapshots { si, _, err := s.Render() if err != nil { return nil, err } config.Snapshots = append(config.Snapshots, si.(*api.InstanceSnapshot)) } dbVolSnaps, err := VolumeDBSnapshotsGet(b, inst.Project().Name, inst.Name(), volType) if err != nil { return nil, err } if len(snapshots) != len(dbVolSnaps) { return nil, errors.New("Instance snapshot record count doesn't match instance snapshot volume record count") } config.VolumeSnapshots = make([]*api.StorageVolumeSnapshot, 0, len(dbVolSnaps)) for i := range dbVolSnaps { foundInstanceSnapshot := false for _, snap := range snapshots { if snap.Name() == dbVolSnaps[i].Name { foundInstanceSnapshot = true break } } if !foundInstanceSnapshot { return nil, fmt.Errorf("Instance snapshot record missing for %q", dbVolSnaps[i].Name) } _, snapName, _ := api.GetParentAndSnapshotName(dbVolSnaps[i].Name) config.VolumeSnapshots = append(config.VolumeSnapshots, &api.StorageVolumeSnapshot{ StorageVolumeSnapshotPut: api.StorageVolumeSnapshotPut{ Description: dbVolSnaps[i].Description, ExpiresAt: &dbVolSnaps[i].ExpiryDate, }, Name: snapName, Config: dbVolSnaps[i].Config, ContentType: dbVolSnaps[i].ContentType, CreatedAt: dbVolSnaps[i].CreationDate, }) } } } return config, nil } // UpdateInstanceBackupFile writes the instance's config to the backup.yaml file on the storage device. func (b *backend) UpdateInstanceBackupFile(inst instance.Instance, snapshots bool, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) l.Debug("UpdateInstanceBackupFile started") defer l.Debug("UpdateInstanceBackupFile finished") // We only write backup files out for actual instances. if inst.IsSnapshot() { return nil } config, err := b.GenerateInstanceBackupConfig(inst, snapshots, true, op) if err != nil { return err } data, err := yaml.Dump(config, yaml.V2) if err != nil { return err } // Get the volume name on storage. volStorageName := project.Instance(inst.Project().Name, inst.Name()) volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return err } contentType := InstanceContentType(inst) vol := b.GetVolume(volType, contentType, volStorageName, config.Volume.Config) // Only need to activate and mount the VM's config volume. if inst.Type() == instancetype.VM { vol = vol.NewVMBlockFilesystemVolume() } // Update pool information in the backup.yaml file. err = vol.MountTask(func(mountPath string, op *operations.Operation) error { // Write the YAML path := filepath.Join(inst.Path(), "backup.yaml") f, err := os.Create(path) if err != nil { return fmt.Errorf("Failed to create file %q: %w", path, err) } err = f.Chmod(0o400) if err != nil { return err } err = internalIO.WriteAll(f, data) if err != nil { return err } err = f.Sync() if err != nil { return err } return f.Close() }, op) return err } // CheckInstanceBackupFileSnapshots compares the snapshots on the storage device to those defined in the backup // config supplied and returns an error if they do not match (if deleteMissing argument is false). // If deleteMissing argument is true, then any snapshots that exist on the storage device but not in the backup // config are removed from the storage device, and any snapshots that exist in the backup config but do not exist // on the storage device are ignored. The remaining set of snapshots that exist on both the storage device and the // backup config are returned. They set can be used to re-create the snapshot database entries when importing. func (b *backend) CheckInstanceBackupFileSnapshots(backupConf *backupConfig.Config, projectName string, deleteMissing bool, op *operations.Operation) ([]*api.InstanceSnapshot, error) { l := b.logger.AddContext(logger.Ctx{"project": projectName, "instance": backupConf.Container.Name, "deleteMissing": deleteMissing}) l.Debug("CheckInstanceBackupFileSnapshots started") defer l.Debug("CheckInstanceBackupFileSnapshots finished") instType, err := instancetype.New(string(backupConf.Container.Type)) if err != nil { return nil, err } volType, err := InstanceTypeToVolumeType(instType) if err != nil { return nil, err } // Get the volume name on storage. volStorageName := project.Instance(projectName, backupConf.Container.Name) contentType := drivers.ContentTypeFS if volType == drivers.VolumeTypeVM { contentType = drivers.ContentTypeBlock } // We don't need to use the volume's config for mounting so set to nil. vol := b.GetVolume(volType, contentType, volStorageName, nil) // Get a list of snapshots that exist on storage device. driverSnapshots, err := vol.Snapshots(op) if err != nil { return nil, err } if len(backupConf.Snapshots) != len(driverSnapshots) { if !deleteMissing { return nil, fmt.Errorf("Snapshot count in backup config (%d) and storage device (%d) are different: %w", len(backupConf.Snapshots), len(driverSnapshots), ErrBackupSnapshotsMismatch) } } // Check (and optionally delete) snapshots that do not exist in backup config. for _, driverSnapVol := range driverSnapshots { _, driverSnapOnly, _ := api.GetParentAndSnapshotName(driverSnapVol.Name()) inBackupFile := false for _, backupFileSnap := range backupConf.Snapshots { backupFileSnapOnly := backupFileSnap.Name if driverSnapOnly == backupFileSnapOnly { inBackupFile = true break } } if inBackupFile { continue } if !deleteMissing { return nil, fmt.Errorf("Snapshot %q exists on storage device but not in backup config: %w", driverSnapOnly, ErrBackupSnapshotsMismatch) } err = b.driver.DeleteVolumeSnapshot(driverSnapVol, op) if err != nil { return nil, fmt.Errorf("Failed to delete snapshot %q: %w", driverSnapOnly, err) } l.Warn("Deleted snapshot as not present in backup config", logger.Ctx{"snapshot": driverSnapOnly}) } // Check the snapshots in backup config exist on storage device. existingSnapshots := []*api.InstanceSnapshot{} for _, backupFileSnap := range backupConf.Snapshots { backupFileSnapOnly := backupFileSnap.Name onStorageDevice := false for _, driverSnapVol := range driverSnapshots { _, driverSnapOnly, _ := api.GetParentAndSnapshotName(driverSnapVol.Name()) if driverSnapOnly == backupFileSnapOnly { onStorageDevice = true break } } if !onStorageDevice { if !deleteMissing { return nil, fmt.Errorf("Snapshot %q exists in backup config but not on storage device: %w", backupFileSnapOnly, ErrBackupSnapshotsMismatch) } l.Warn("Skipped snapshot in backup config as not present on storage device", logger.Ctx{"snapshot": backupFileSnap}) continue // Skip snapshots missing on storage device. } existingSnapshots = append(existingSnapshots, backupFileSnap) } return existingSnapshots, nil } // ListUnknownVolumes returns volumes that exist on the storage pool but don't have records in the database. // Returns the unknown volumes parsed/generated backup config in a slice (keyed on project name). func (b *backend) ListUnknownVolumes(op *operations.Operation) (map[string][]*backupConfig.Config, error) { // Get a list of volumes on the storage pool. We only expect to get 1 volume per logical Incus volume. // So for VMs we only expect to get the block volume for a VM and not its filesystem one too. This way we // can operate on the volume using the existing storage pool functions and let the pool then handle the // associated filesystem volume as needed. poolVols, err := b.driver.ListVolumes() if err != nil { return nil, fmt.Errorf("Failed getting pool volumes: %w", err) } projectVols := make(map[string][]*backupConfig.Config) for _, poolVol := range poolVols { volType := poolVol.Type() // If the storage driver has returned a filesystem volume for a VM, this is a break of protocol. if volType == drivers.VolumeTypeVM && poolVol.ContentType() == drivers.ContentTypeFS { return nil, fmt.Errorf("Storage driver returned unexpected VM volume with filesystem content type (%q)", poolVol.Name()) } if volType == drivers.VolumeTypeVM || volType == drivers.VolumeTypeContainer { err = b.detectUnknownInstanceVolume(&poolVol, projectVols, op) if err != nil { return nil, err } } else if volType == drivers.VolumeTypeCustom { err = b.detectUnknownCustomVolume(&poolVol, projectVols, op) if err != nil { return nil, err } } else if volType == drivers.VolumeTypeBucket { err = b.detectUnknownBuckets(&poolVol, projectVols, op) if err != nil { return nil, err } } } return projectVols, nil } // detectUnknownInstanceVolume detects if a volume is unknown and if so attempts to mount the volume and parse the // backup stored on it. It then runs a series of consistency checks that compare the contents of the backup file to // the state of the volume on disk, and if all checks out, it adds the parsed backup file contents to projectVols. func (b *backend) detectUnknownInstanceVolume(vol *drivers.Volume, projectVols map[string][]*backupConfig.Config, op *operations.Operation) error { volType := vol.Type() projectName, instName := project.InstanceParts(vol.Name()) var instID int var instSnapshots []string err := b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Check if an entry for the instance already exists in the DB. instID, err = tx.GetInstanceID(ctx, projectName, instName) if err != nil && !response.IsNotFoundError(err) { return err } instSnapshots, err = tx.GetInstanceSnapshotsNames(ctx, projectName, instName) if err != nil { return err } return nil }) if err != nil { return err } // Check if any entry for the instance volume already exists in the DB. // This will return no record for any temporary pool structs being used (as ID is -1). volume, err := VolumeDBGet(b, projectName, instName, volType) if err != nil && !response.IsNotFoundError(err) { return err } if instID > 0 && volume != nil { return nil // Instance record and storage record already exists in DB, no recovery needed. } else if instID > 0 { return fmt.Errorf("Instance %q in project %q already has instance DB record", instName, projectName) } else if volume != nil { return fmt.Errorf("Instance %q in project %q already has storage DB record", instName, projectName) } backupYamlPath := filepath.Join(vol.MountPath(), "backup.yaml") var backupConf *backupConfig.Config // If the instance is running, it should already be mounted, so check if the backup file // is already accessible, and if so parse it directly, without disturbing the mount count. if util.PathExists(backupYamlPath) { backupConf, err = backup.ParseConfigYamlFile(backupYamlPath) if err != nil { return fmt.Errorf("Failed parsing backup file %q: %w", backupYamlPath, err) } } else { // If backup file not accessible, we take this to mean the instance isn't running // and so we need to mount the volume to access the backup file and then unmount. // This will also create the mount path if needed. err = vol.MountTask(func(_ string, _ *operations.Operation) error { backupConf, err = backup.ParseConfigYamlFile(backupYamlPath) if err != nil { return fmt.Errorf("Failed parsing backup file %q: %w", backupYamlPath, err) } return nil }, op) if err != nil { return err } } // Run some consistency checks on the backup file contents. if backupConf.Pool != nil { if backupConf.Pool.Name != b.name { return fmt.Errorf("Instance %q in project %q has pool name mismatch in its backup file (%q doesn't match's pool's %q)", instName, projectName, backupConf.Pool.Name, b.name) } if backupConf.Pool.Driver != b.Driver().Info().Name { return fmt.Errorf("Instance %q in project %q has pool driver mismatch in its backup file (%q doesn't match's pool's %q)", instName, projectName, backupConf.Pool.Driver, b.Driver().Name()) } } if backupConf.Container == nil { return fmt.Errorf("Instance %q in project %q has no instance information in its backup file", instName, projectName) } if instName != backupConf.Container.Name { return fmt.Errorf("Instance %q in project %q has a different instance name in its backup file (%q)", instName, projectName, backupConf.Container.Name) } apiInstType, err := VolumeTypeToAPIInstanceType(volType) if err != nil { return fmt.Errorf("Failed checking instance type for instance %q in project %q: %w", instName, projectName, err) } if apiInstType != api.InstanceType(backupConf.Container.Type) { return fmt.Errorf("Instance %q in project %q has a different instance type in its backup file (%q)", instName, projectName, backupConf.Container.Type) } if backupConf.Volume == nil { return fmt.Errorf("Instance %q in project %q has no volume information in its backup file", instName, projectName) } if instName != backupConf.Volume.Name { return fmt.Errorf("Instance %q in project %q has a different volume name in its backup file (%q)", instName, projectName, backupConf.Volume.Name) } instVolDBType, err := VolumeTypeNameToDBType(backupConf.Volume.Type) if err != nil { return fmt.Errorf("Failed checking instance volume type for instance %q in project %q: %w", instName, projectName, err) } instVolType, err := VolumeDBTypeToType(instVolDBType) if err != nil { return fmt.Errorf("Failed checking instance volume type for instance %q in project %q: %w", instName, projectName, err) } if volType != instVolType { return fmt.Errorf("Instance %q in project %q has a different volume type in its backup file (%q)", instName, projectName, backupConf.Volume.Type) } // Add to volume to unknown volumes list for the project. if projectVols[projectName] == nil { projectVols[projectName] = []*backupConfig.Config{backupConf} } else { projectVols[projectName] = append(projectVols[projectName], backupConf) } // Check snapshots are consistent between storage layer and backup config file. _, err = b.CheckInstanceBackupFileSnapshots(backupConf, projectName, false, nil) if err != nil { return fmt.Errorf("Instance %q in project %q has snapshot inconsistency: %w", instName, projectName, err) } // Check there are no existing DB records present for snapshots. for _, snapshot := range backupConf.Snapshots { fullSnapshotName := drivers.GetSnapshotVolumeName(instName, snapshot.Name) // Check if an entry for the instance already exists in the DB. if slices.Contains(instSnapshots, fullSnapshotName) { return fmt.Errorf("Instance %q snapshot %q in project %q already has instance DB record", instName, snapshot.Name, projectName) } // Check if any entry for the instance snapshot volume already exists in the DB. // This will return no record for any temporary pool structs being used (as ID is -1). volume, err := VolumeDBGet(b, projectName, fullSnapshotName, volType) if err != nil && !response.IsNotFoundError(err) { return err } else if volume != nil { return fmt.Errorf("Instance %q snapshot %q in project %q already has storage DB record", instName, snapshot.Name, projectName) } } return nil } // detectUnknownCustomVolume detects if a volume is unknown and if so attempts to discover the filesystem of the // volume (for filesystem volumes). It then runs a series of consistency checks, and if all checks out, it adds // generates a simulated backup config for the custom volume and adds it to projectVols. func (b *backend) detectUnknownCustomVolume(vol *drivers.Volume, projectVols map[string][]*backupConfig.Config, op *operations.Operation) error { volType := vol.Type() projectName, volName := project.StorageVolumeParts(vol.Name()) // Check if any entry for the custom volume already exists in the DB. // This will return no record for any temporary pool structs being used (as ID is -1). volume, err := VolumeDBGet(b, projectName, volName, volType) if err != nil && !response.IsNotFoundError(err) { return err } else if volume != nil { return nil // Storage record already exists in DB, no recovery needed. } // Get a list of snapshots that exist on storage device. snapshots, err := b.driver.VolumeSnapshots(*vol, op) if err != nil { return err } contentType := vol.ContentType() var apiContentType string if contentType == drivers.ContentTypeBlock { apiContentType = db.StoragePoolVolumeContentTypeNameBlock } else if contentType == drivers.ContentTypeISO { apiContentType = db.StoragePoolVolumeContentTypeNameISO } else if contentType == drivers.ContentTypeFS { apiContentType = db.StoragePoolVolumeContentTypeNameFS // Detect block volume filesystem (by mounting it (if not already) with filesystem probe mode). if vol.IsBlockBacked() { var blockFS string mountPath := vol.MountPath() if linux.IsMountPoint(mountPath) { blockFS, err = linux.DetectFilesystem(mountPath) if err != nil { return err } } else { err = vol.MountTask(func(mountPath string, op *operations.Operation) error { blockFS, err = linux.DetectFilesystem(mountPath) if err != nil { return err } return nil }, op) if err != nil { return err } } // Record detected filesystem in config. vol.Config()["block.filesystem"] = blockFS } } else { return fmt.Errorf("Unknown custom volume content type %q", contentType) } // This may not always be the correct thing to do, but seeing as we don't know what the volume's config // was lets take a best guess that it was the default config. err = b.driver.FillVolumeConfig(*vol) if err != nil { return fmt.Errorf("Failed filling custom volume default config: %w", err) } // Check the filesystem detected is valid for the storage driver. err = b.driver.ValidateVolume(*vol, false) if err != nil { return fmt.Errorf("Failed custom volume validation: %w", err) } backupConf := &backupConfig.Config{ Volume: &api.StorageVolume{ Name: volName, Type: db.StoragePoolVolumeTypeNameCustom, ContentType: apiContentType, StorageVolumePut: api.StorageVolumePut{ Config: vol.Config(), }, }, } // Populate snapshot volumes. for _, snapOnlyName := range snapshots { backupConf.VolumeSnapshots = append(backupConf.VolumeSnapshots, &api.StorageVolumeSnapshot{ Name: snapOnlyName, // Snapshot only name, not full name. Config: vol.Config(), // Have to assume the snapshot volume config is same as parent. ContentType: apiContentType, }) } // Add to volume to unknown volumes list for the project. if projectVols[projectName] == nil { projectVols[projectName] = []*backupConfig.Config{backupConf} } else { projectVols[projectName] = append(projectVols[projectName], backupConf) } return nil } // detectUnknownBuckets detects if a bucket is unknown and if so attempts to discover the filesystem of the // bucket. It then runs a series of consistency checks, and if all checks out, it generates a simulated backup // config for the bucket and adds it to projectVols. func (b *backend) detectUnknownBuckets(vol *drivers.Volume, projectVols map[string][]*backupConfig.Config, op *operations.Operation) error { projectName, bucketName := project.StorageVolumeParts(vol.Name()) // Check if any entry for the bucket already exists in the DB. bucket, err := BucketDBGet(b, projectName, bucketName, true) if err != nil && !response.IsNotFoundError(err) { return err } else if bucket != nil { return nil // Storage record already exists in DB, no recovery needed. } // This may not always be the correct thing to do, but seeing as we don't know what the volume's config // was lets take a best guess that it was the default config. err = b.driver.FillVolumeConfig(*vol) if err != nil { return fmt.Errorf("Failed filling bucket default config: %w", err) } // Check the detected filesystem is valid for the storage driver. err = b.driver.ValidateVolume(*vol, false) if err != nil { return fmt.Errorf("Failed bucket validation: %w", err) } backupConf := &backupConfig.Config{ Bucket: &api.StorageBucket{ StorageBucketPut: api.StorageBucketPut{ Config: vol.Config(), }, Name: bucketName, }, } // Add the bucket to unknown volumes list for the project. if projectVols[projectName] == nil { projectVols[projectName] = []*backupConfig.Config{backupConf} } else { projectVols[projectName] = append(projectVols[projectName], backupConf) } return nil } // ImportInstance takes an existing instance volume on the storage backend and ensures that the volume directories // and symlinks are restored as needed to make it operational with Incus. Used during the recovery import stage. // If the instance exists on the local cluster member then the local mount status is restored as needed. // If the optional poolVol argument is provided then it is used to create the storage volume database records. func (b *backend) ImportInstance(inst instance.Instance, poolVol *backupConfig.Config, op *operations.Operation) (revert.Hook, error) { l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name()}) l.Debug("ImportInstance started") defer l.Debug("ImportInstance finished") volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return nil, err } var snapshots []string err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Get any snapshots the instance has in the format /. snapshots, err = tx.GetInstanceSnapshotsNames(ctx, inst.Project().Name, inst.Name()) return err }) if err != nil { return nil, err } contentType := InstanceContentType(inst) reverter := revert.New() defer reverter.Fail() var volumeConfig map[string]string // Create storage volume database records if in recover mode. if poolVol != nil { creationDate := inst.CreationDate() // Copy volume config from backup file config if present, // so VolumeDBCreate can safely modify the copy if needed. if poolVol.Volume != nil { volumeConfig = util.CloneMap(poolVol.Volume.Config) if !poolVol.Volume.CreatedAt.IsZero() { creationDate = poolVol.Volume.CreatedAt } } // Validate config and create database entry for recovered storage volume. err = VolumeDBCreate(b, inst.Project().Name, inst.Name(), "", volType, false, volumeConfig, creationDate, time.Time{}, contentType, false, true) if err != nil { return nil, err } reverter.Add(func() { _ = VolumeDBDelete(b, inst.Project().Name, inst.Name(), volType) }) if len(snapshots) > 0 && len(poolVol.VolumeSnapshots) > 0 { // Create storage volume snapshot DB records from the entries in the backup file config. for _, poolVolSnap := range poolVol.VolumeSnapshots { fullSnapName := drivers.GetSnapshotVolumeName(inst.Name(), poolVolSnap.Name) // Copy volume config from backup file if present, // so VolumeDBCreate can safely modify the copy if needed. snapVolumeConfig := util.CloneMap(poolVolSnap.Config) // Validate config and create database entry for recovered storage volume. err = VolumeDBCreate(b, inst.Project().Name, fullSnapName, poolVolSnap.Description, volType, true, snapVolumeConfig, poolVolSnap.CreatedAt, time.Time{}, contentType, false, true) if err != nil { return nil, err } reverter.Add(func() { _ = VolumeDBDelete(b, inst.Project().Name, fullSnapName, volType) }) } } else { b.logger.Warn("Missing volume snapshot info in backup config, using parent volume config") // Create storage volume snapshot DB records based on instance snapshot list, as the // backup config doesn't contain the required info. This is needed because there was a // historical bug that meant that the instance's backup file didn't store the storage // volume snapshot info. for _, i := range snapshots { fullSnapName := i // Local var for revert. // Validate config and create database entry for new storage volume. // Use parent volume config. err = VolumeDBCreate(b, inst.Project().Name, fullSnapName, "", volType, true, volumeConfig, time.Time{}, time.Time{}, contentType, false, true) if err != nil { return nil, err } reverter.Add(func() { _ = VolumeDBDelete(b, inst.Project().Name, fullSnapName, volType) }) } } } // Generate the effective root device volume for instance. volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(volType, contentType, volStorageName, volumeConfig) err = b.applyInstanceRootDiskOverrides(inst, &vol) if err != nil { return nil, err } err = vol.EnsureMountPath(false) if err != nil { return nil, err } // Only attempt to restore mount status on instance's local cluster member. if inst.Location() == b.state.ServerName { l.Debug("Restoring local instance mount status") if inst.IsRunning() { // If the instance is running then this implies the volume is mounted, but if the Incus // daemon has been restarted since the DB records were removed then there will be no mount // reference counter showing the volume is in use. If this is the case then call mount the // volume to increment the reference counter. if !vol.MountInUse() { _, err = b.MountInstance(inst, op) if err != nil { return nil, fmt.Errorf("Failed mounting instance: %w", err) } } } else { // If the instance isn't running then try and unmount it to ensure consistent state after // import. err = b.UnmountInstance(inst, op) if err != nil { return nil, fmt.Errorf("Failed unmounting instance: %w", err) } } } // Create symlink. err = b.ensureInstanceSymlink(inst.Type(), inst.Project().Name, inst.Name(), vol.MountPath()) if err != nil { return nil, err } reverter.Add(func() { // Remove symlinks. _ = b.removeInstanceSymlink(inst.Type(), inst.Project().Name, inst.Name()) _ = b.removeInstanceSnapshotSymlinkIfUnused(inst.Type(), inst.Project().Name, inst.Name()) }) // Create snapshot mount paths and snapshot symlink if needed. if len(snapshots) > 0 { for _, snapName := range snapshots { _, snapOnlyName, _ := api.GetParentAndSnapshotName(snapName) l.Debug("Ensuring instance snapshot mount path", logger.Ctx{"snapshot": snapOnlyName}) snapVol, err := vol.NewSnapshot(snapOnlyName) if err != nil { return nil, err } err = snapVol.EnsureMountPath(false) if err != nil { return nil, err } } err = b.ensureInstanceSnapshotSymlink(inst.Type(), inst.Project().Name, inst.Name()) if err != nil { return nil, err } } cleanup := reverter.Clone().Fail reverter.Success() return cleanup, err } // BackupCustomVolume creates a custom volume backup. func (b *backend) BackupCustomVolume(projectName string, volName string, writer instancewriter.InstanceWriter, basePrefix string, optimized bool, snapshots bool, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volume": volName, "optimized": optimized, "snapshots": snapshots}) l.Debug("BackupCustomVolume started") defer l.Debug("BackupCustomVolume finished") volume, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return err } // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, volume.Name) contentDBType, err := VolumeContentTypeNameToContentType(volume.ContentType) if err != nil { return err } contentType, err := VolumeDBContentTypeToContentType(contentDBType) if err != nil { return err } if contentType != drivers.ContentTypeFS && contentType != drivers.ContentTypeBlock && contentType != drivers.ContentTypeISO { return fmt.Errorf("Volume of content type %q cannot be backed up", contentType) } var snapNames []string if snapshots { // Get snapshots in age order, oldest first, and pass names to storage driver. volSnaps, err := VolumeDBSnapshotsGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return err } snapNames = make([]string, 0, len(volSnaps)) for _, volSnap := range volSnaps { _, snapName, _ := api.GetParentAndSnapshotName(volSnap.Name) snapNames = append(snapNames, snapName) } } vol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(volume.ContentType), volStorageName, volume.Config) if volume.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { err = b.qcow2BackupVolume(vol, volume, projectName, writer, basePrefix, snapNames, op) if err != nil { return err } } else { err = b.driver.BackupVolume(vol, writer, basePrefix, optimized, snapNames, op) if err != nil { return err } } return nil } func (b *backend) CreateCustomVolumeFromISO(projectName string, volName string, srcData io.ReadSeeker, size int64, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "volume": volName}) l.Debug("CreateCustomVolumeFromISO started") defer l.Debug("CreateCustomVolumeFromISO finished") // Check whether we are allowed to create volumes. req := api.StorageVolumesPost{ Name: volName, StorageVolumePut: api.StorageVolumePut{ Config: map[string]string{ "size": fmt.Sprintf("%d", size), }, }, } err := b.state.DB.Cluster.Transaction(b.state.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { return project.AllowVolumeCreation(tx, projectName, b.name, req) }) if err != nil { return fmt.Errorf("Failed checking volume creation allowed: %w", err) } reverter := revert.New() defer reverter.Fail() // Get the volume name on storage. volStorageName := project.StorageVolume(projectName, volName) vol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentTypeISO, volStorageName, req.Config) volExists, err := b.driver.HasVolume(vol) if err != nil { return err } if volExists { return errors.New("Cannot create volume, already exists on target storage") } // Validate config and create database entry for new storage volume. err = VolumeDBCreate(b, projectName, volName, "", vol.Type(), false, vol.Config(), time.Now(), time.Time{}, vol.ContentType(), true, true) if err != nil { return fmt.Errorf("Failed creating database entry for custom volume: %w", err) } reverter.Add(func() { _ = VolumeDBDelete(b, projectName, volName, vol.Type()) }) _, err = srcData.Seek(0, io.SeekStart) if err != nil { return err } volFiller := drivers.VolumeFiller{ Fill: b.isoFiller(srcData), } // Unpack the ISO into the new storage volume(s). err = b.driver.CreateVolume(vol, &volFiller, op) if err != nil { return fmt.Errorf("Failed creating volume: %w", err) } eventCtx := logger.Ctx{"type": vol.Type()} if !b.Driver().Info().Remote { eventCtx["location"] = b.state.ServerName } var location string if b.state.ServerClustered && !b.Driver().Info().Remote { location = b.state.ServerName } // Record new volume with authorizer. err = b.state.Authorizer.AddStoragePoolVolume(b.state.ShutdownCtx, projectName, b.Name(), vol.Type().Singular(), volName, location) if err != nil { logger.Error("Failed to add storage volume to authorizer", logger.Ctx{"name": volName, "type": vol.Type(), "pool": b.Name(), "project": projectName, "error": err}) } b.state.Events.SendLifecycle(projectName, lifecycle.StorageVolumeCreated.Event(vol, string(vol.Type()), projectName, op, eventCtx)) reverter.Success() return nil } // CreateCustomVolumeFromBackup creates a custom volume from a backup. func (b *backend) CreateCustomVolumeFromBackup(srcBackup backup.Info, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": srcBackup.Project, "volume": srcBackup.Name, "snapshots": srcBackup.Snapshots, "optimizedStorage": *srcBackup.OptimizedStorage}) l.Debug("CreateCustomVolumeFromBackup started") defer l.Debug("CreateCustomVolumeFromBackup finished") if srcBackup.Config == nil || srcBackup.Config.Volume == nil { return errors.New("Valid volume config not found in index") } if len(srcBackup.Snapshots) != len(srcBackup.Config.VolumeSnapshots) { return errors.New("Valid volume snapshot config not found in index") } // Check whether we are allowed to create volumes. req := api.StorageVolumesPost{ StorageVolumePut: api.StorageVolumePut{ Config: srcBackup.Config.Volume.Config, }, Name: srcBackup.Name, } err := b.state.DB.Cluster.Transaction(b.state.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { return project.AllowVolumeCreation(tx, srcBackup.Project, b.name, req) }) if err != nil { return fmt.Errorf("Failed checking volume creation allowed: %w", err) } reverter := revert.New() defer reverter.Fail() // Get the volume name on storage. volStorageName := project.StorageVolume(srcBackup.Project, srcBackup.Name) vol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(srcBackup.Config.Volume.ContentType), volStorageName, srcBackup.Config.Volume.Config) // Check if the volume exists in database. dbVol, err := VolumeDBGet(b, srcBackup.Project, srcBackup.Name, vol.Type()) if err != nil && !response.IsNotFoundError(err) { return err } if dbVol != nil { return fmt.Errorf("Volume %q already exists in pool %q", srcBackup.Name, b.name) } // Validate config and create database entry for new storage volume. // Strip unsupported config keys (in case the export was made from a different type of storage pool). err = VolumeDBCreate(b, srcBackup.Project, srcBackup.Name, srcBackup.Config.Volume.Description, vol.Type(), false, vol.Config(), srcBackup.Config.Volume.CreatedAt, time.Time{}, vol.ContentType(), true, true) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, srcBackup.Project, srcBackup.Name, vol.Type()) }) // Create database entries for new storage volume snapshots. for _, s := range srcBackup.Config.VolumeSnapshots { if s == nil { return errors.New("Bad snapshot definition found in index") } snapshot := s // Local var for revert. snapName := snapshot.Name // Due to a historical bug, the volume snapshot names were sometimes written in their full form // (/) rather than the expected snapshot name only form, so we need to handle both. if internalInstance.IsSnapshot(snapshot.Name) { _, snapName, _ = api.GetParentAndSnapshotName(snapshot.Name) } fullSnapName := drivers.GetSnapshotVolumeName(srcBackup.Name, snapName) snapVolStorageName := project.StorageVolume(srcBackup.Project, fullSnapName) snapVol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(srcBackup.Config.Volume.ContentType), snapVolStorageName, snapshot.Config) // Validate config and create database entry for new storage volume. // Strip unsupported config keys (in case the export was made from a different type of storage pool). err = VolumeDBCreate(b, srcBackup.Project, fullSnapName, snapshot.Description, snapVol.Type(), true, snapVol.Config(), snapshot.CreatedAt, *snapshot.ExpiresAt, snapVol.ContentType(), true, true) if err != nil { return err } reverter.Add(func() { _ = VolumeDBDelete(b, srcBackup.Project, fullSnapName, snapVol.Type()) }) } // Unpack the backup into the new storage volume(s). var volPostHook drivers.VolumePostHook var revertHook revert.Hook if srcBackup.Config.Volume.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { volPostHook, revertHook, err = b.qcow2UnpackVolume(vol, srcBackup.Snapshots, srcData, basePrefix, op) if err != nil { return err } } else { volPostHook, revertHook, err = b.driver.CreateVolumeFromBackup(vol, srcBackup, srcData, basePrefix, op) if err != nil { return err } } if revertHook != nil { reverter.Add(revertHook) } // If the driver returned a post hook, return error as custom volumes don't need post hooks and we expect // the storage driver to understand this distinction and ensure that all activities done in the postHook // normally are done in CreateVolumeFromBackup as the DB record is created ahead of time. if volPostHook != nil { return errors.New("Custom volume restore doesn't support post hooks") } eventCtx := logger.Ctx{"type": vol.Type()} if !b.Driver().Info().Remote { eventCtx["location"] = b.state.ServerName } var location string if b.state.ServerClustered && !b.Driver().Info().Remote { location = b.state.ServerName } // Record new volume with authorizer. err = b.state.Authorizer.AddStoragePoolVolume(b.state.ShutdownCtx, srcBackup.Project, b.Name(), vol.Type().Singular(), srcBackup.Name, location) if err != nil { logger.Error("Failed to add storage volume to authorizer", logger.Ctx{"name": srcBackup.Name, "type": vol.Type(), "pool": b.Name(), "project": srcBackup.Project, "error": err}) } b.state.Events.SendLifecycle(srcBackup.Project, lifecycle.StorageVolumeCreated.Event(vol, string(vol.Type()), srcBackup.Project, op, eventCtx)) reverter.Success() return nil } // BackupBucket backups up a bucket to a tarball. func (b *backend) BackupBucket(projectName string, bucketName string, tarWriter *instancewriter.InstanceTarWriter, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": projectName, "bucket": bucketName}) l.Debug("BackupBucket started") defer l.Debug("BackupBucket finished") err := b.isStatusReady() if err != nil { return err } if !b.Driver().Info().Buckets { return errors.New("Storage pool does not support buckets") } memberSpecific := !b.Driver().Info().Remote // Member specific if storage pool isn't remote. var bucket *db.StorageBucket err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { bucket, err = tx.GetStoragePoolBucket(ctx, b.id, projectName, memberSpecific, bucketName) return err }) if err != nil { return err } backupKey, err := b.getFirstReadStorageBucketPoolKey(bucket.ID) if err != nil { return err } bucketURL := b.GetBucketURL(bucket.Name) if bucketURL == nil { return errors.New("The server is lacking a storage buckets listener address") } transferManager := s3.NewTransferManager(bucketURL, backupKey.AccessKey, backupKey.SecretKey) err = transferManager.DownloadAllFiles(bucket.Name, tarWriter) if err != nil { return err } return nil } // CreateBucketFromBackup creates a bucket from a tarball. func (b *backend) CreateBucketFromBackup(srcBackup backup.Info, srcData io.ReadSeeker, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"project": srcBackup.Project, "bucket": srcBackup.Name}) l.Debug("CreateBucketFromBackup started") defer l.Debug("CreateBucketFromBackup finished") err := b.isStatusReady() if err != nil { return err } if !b.Driver().Info().Buckets { return errors.New("Storage pool does not support buckets") } if srcBackup.Config == nil || srcBackup.Config.Bucket == nil { return errors.New("Valid bucket config not found in index") } reverter := revert.New() defer reverter.Fail() bucketRequest := api.StorageBucketsPost{ Name: srcBackup.Name, StorageBucketPut: srcBackup.Config.Bucket.StorageBucketPut, } // Create the bucket to import. err = b.CreateBucket(srcBackup.Project, bucketRequest, op) if err != nil { return err } reverter.Add(func() { _ = b.DeleteBucket(srcBackup.Project, bucketRequest.Name, op) }) // Upload all keys from the backup. for _, bucketKey := range srcBackup.Config.BucketKeys { if bucketKey == nil { return errors.New("Bad bucket key found in index") } bucketKeyRequest := api.StorageBucketKeysPost{ Name: bucketKey.Name, StorageBucketKeyPut: bucketKey.StorageBucketKeyPut, } _, err := b.CreateBucketKey(srcBackup.Project, srcBackup.Name, bucketKeyRequest, op) if err != nil { return err } } // Upload all files from the backup. backupKey, err := b.getFirstAdminStorageBucketPoolKey(srcBackup.Project, srcBackup.Name) if err != nil { return err } bucketURL := b.GetBucketURL(srcBackup.Name) if bucketURL == nil { return errors.New("The server is lacking a storage buckets listener address") } transferManager := s3.NewTransferManager(bucketURL, backupKey.AccessKey, backupKey.SecretKey) err = transferManager.UploadAllFiles(srcBackup.Name, srcData) if err != nil { return err } reverter.Success() return nil } func (b *backend) getFirstReadStorageBucketPoolKey(bucketID int64) (*db.StorageBucketKey, error) { var backupKey *db.StorageBucketKey err := b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { bucketKeys, err := tx.GetStoragePoolBucketKeys(ctx, bucketID) bucketKeysLen := len(bucketKeys) if (err == nil && bucketKeysLen <= 0) || errors.Is(err, sql.ErrNoRows) { return api.StatusErrorf(http.StatusNotFound, "Storage bucket key not found") } else if err != nil { return err } backupKey = bucketKeys[0] return nil }) if err != nil { return nil, err } return backupKey, nil } func (b *backend) getFirstAdminStorageBucketPoolKey(projectName string, bucketName string) (*db.StorageBucketKey, error) { memberSpecific := !b.Driver().Info().Remote // Member specific if storage pool isn't remote. var bucket *db.StorageBucket err := b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error bucket, err = tx.GetStoragePoolBucket(ctx, b.id, projectName, memberSpecific, bucketName) return err }) if err != nil { return nil, err } var bucketKey *db.StorageBucketKey err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { bucketKeys, err := tx.GetStoragePoolBucketKeys(ctx, bucket.ID) bucketKeysLen := len(bucketKeys) if (err == nil && bucketKeysLen <= 0) || errors.Is(err, sql.ErrNoRows) { return api.StatusErrorf(http.StatusNotFound, "Storage bucket key not found") } else if err != nil { return err } for _, key := range bucketKeys { if key.Role == "admin" { bucketKey = key break } } if bucketKey == nil { return api.StatusErrorf(http.StatusNotFound, "No storage bucket admin key found") } return nil }) if err != nil { return nil, err } return bucketKey, nil } // qcow2Rename renames the QCOW2 volume. func (b *backend) qcow2Rename(vol drivers.Volume, newVolName string, projectName string, op *operations.Operation) error { // Get snapshots. _, volName := project.StorageVolumeParts(vol.Name()) volSnaps, err := VolumeDBSnapshotsGet(b, projectName, volName, vol.Type()) if err != nil { return err } err = vol.MountWithSnapshotsTask(func(_ string, _ map[string]string, parentOp *operations.Operation) error { backingPath := "" // Update the metadata of the snapshot which points to a renamed volume. for _, snap := range volSnaps { currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(projectName, snap.Name), nil) currentSnapVolDiskPath, err := b.driver.GetVolumeDiskPath(currentSnapVol) if err != nil { return err } if backingPath != "" { err = drivers.Qcow2Rebase(currentSnapVolDiskPath, backingPath) if err != nil { return err } } _, snapName, _ := api.GetParentAndSnapshotName(snap.Name) newSnapVolName := drivers.GetSnapshotVolumeName(newVolName, snapName) newSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(projectName, newSnapVolName), nil) backingPath, err = b.driver.GetQcow2BackingFilePath(newSnapVol) if err != nil { return err } } // Update the metadata of the main volume if needed. parentDiskPath, err := b.driver.GetVolumeDiskPath(vol) if err != nil { return err } if backingPath != "" { err = drivers.Qcow2Rebase(parentDiskPath, backingPath) if err != nil { return err } } return nil }, op) if err != nil { return err } return nil } // qcow2CreateSnapshot creates the QCOW2 volume snapshot. func (b *backend) qcow2CreateSnapshot(vol drivers.Volume, snapVol drivers.Volume, projectName string, inst instance.Instance, devName string, stateful bool, op *operations.Operation) error { // Return if this is not a qcow2 image. if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { return nil } reverter := revert.New() defer reverter.Fail() if vol.IsVMBlock() { fsParentVol := vol.NewVMBlockFilesystemVolume() fsVol := snapVol.NewVMBlockFilesystemVolume() err := drivers.Qcow2CreateConfigSnapshot(fsParentVol, fsVol, op) if err != nil { return err } reverter.Add(func() { _ = drivers.Qcow2DeleteConfigSnapshot(fsParentVol, fsVol, nil) }) } // For a running instance, mount the snapshot to increase the volume's refCount. // The increased refCount protects against deallocating a volume that is // currently in use by the instance. if inst != nil && inst.IsRunning() { err := b.driver.MountVolumeSnapshot(snapVol, op) if err != nil { return err } reverter.Add(func() { _, _ = b.driver.UnmountVolumeSnapshot(snapVol, nil) }) } snapVolDevPath, err := b.driver.GetQcow2BackingFilePath(snapVol) if err != nil { return err } parentDiskPath, err := b.driver.GetVolumeDiskPath(vol) if err != nil { return err } err = vol.MountWithSnapshotsTask(func(_ string, _ map[string]string, parentOp *operations.Operation) error { if inst != nil && inst.IsRunning() { imgInfo, err := drivers.Qcow2Info(parentDiskPath) if err != nil { return err } err = drivers.Qcow2Create(parentDiskPath, "", int64(imgInfo.VirtualSize)) if err != nil { return err } } else { err = drivers.Qcow2Create(parentDiskPath, snapVolDevPath, 0) if err != nil { return err } } return nil }, op) if err != nil { return err } if inst != nil && inst.IsRunning() { _, snapshotName, isSnap := api.GetParentAndSnapshotName(snapVol.Name()) if !isSnap { return errors.New("Volume name must be a snapshot") } err = inst.CreateQcow2Snapshot(parentDiskPath, devName, snapshotName, snapVolDevPath, stateful) if err != nil { return err } } reverter.Success() return nil } // qcow2CanRestoreSnapshot checks if a qcow2 snapshot can be restored. func (b *backend) qcow2CanRestoreSnapshot(vol drivers.Volume, snapVol drivers.Volume, projectName string) error { if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { return fmt.Errorf("Not a QCOW2 volume type") } snapVolDevPath, err := b.driver.GetQcow2BackingFilePath(snapVol) if err != nil { return err } // Get snapshots. _, volName := project.StorageVolumeParts(vol.Name()) volSnaps, err := VolumeDBSnapshotsGet(b, projectName, volName, vol.Type()) if err != nil { return err } err = vol.MountWithSnapshotsTask(func(_ string, _ map[string]string, op *operations.Operation) error { parentDiskPath, err := b.driver.GetVolumeDiskPath(vol) if err != nil { return err } imgInfo, err := drivers.Qcow2Info(parentDiskPath) if err != nil { return err } // Restoring is allowed only for the most recent snapshot. if imgInfo.BackingFilename != snapVolDevPath { if util.IsFalseOrEmpty(vol.ExpandedConfig("lvmcluster.remove_snapshots")) { return fmt.Errorf("Snapshot %q cannot be restored due to subsequent snapshot(s). Set lvmcluster.remove_snapshots to override", snapVol.Name()) } snapshots := []string{} for i := len(volSnaps); i > 0; i-- { snap := volSnaps[i-1] currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), ProjectVolume(projectName, snap.Name, vol.Type()), nil) currentSnapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(currentSnapVol) if err != nil { return err } _, snapName, _ := api.GetParentAndSnapshotName(snap.Name) snapshots = append(snapshots, snapName) imgInfo, err := drivers.Qcow2Info(currentSnapVolDiskPath) if err != nil { return err } if imgInfo.BackingFilename == snapVolDevPath { break } if imgInfo.BackingFilename == "" { return fmt.Errorf("Snapshot %q not found while restoring", snapVol.Name()) } } // Setup custom error to tell the backend what to delete. err := drivers.ErrDeleteSnapshots{} err.Snapshots = snapshots return err } return nil }, nil) if err != nil { return err } return nil } // qcow2RestoreSnapshot restores the QCOW2 volume snapshot. func (b *backend) qcow2RestoreSnapshot(vol drivers.Volume, snapVol drivers.Volume, projectName string, op *operations.Operation) error { // Return if this is not a qcow2 image. if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { return nil } err := b.qcow2CanRestoreSnapshot(vol, snapVol, projectName) if err != nil { return err } snapVolDevPath, err := b.driver.GetQcow2BackingFilePath(snapVol) if err != nil { return err } err = vol.MountWithSnapshotsTask(func(_ string, _ map[string]string, op *operations.Operation) error { parentDiskPath, err := b.driver.GetVolumeDiskPath(vol) if err != nil { return err } err = drivers.Qcow2Create(parentDiskPath, snapVolDevPath, 0) if err != nil { return err } return nil }, op) if err != nil { return err } if vol.IsVMBlock() { fsParentVol := vol.NewVMBlockFilesystemVolume() fsVol := snapVol.NewVMBlockFilesystemVolume() err = drivers.Qcow2RestoreConfigSnapshot(fsParentVol, fsVol, op) if err != nil { return err } } return nil } // qcow2RenameSnapshot renames the QCOW2 volume snapshot. func (b *backend) qcow2RenameSnapshot(vol drivers.Volume, snapVol drivers.Volume, newVolName string, projectName string, op *operations.Operation) error { // Return if this is not a qcow2 image. if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { return nil } newSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), ProjectVolume(projectName, newVolName, vol.Type()), nil) newSnapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(newSnapVol) if err != nil { return err } snapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(snapVol) if err != nil { return err } // Get snapshots. _, volName := project.StorageVolumeParts(vol.Name()) volSnaps, err := VolumeDBSnapshotsGet(b, projectName, volName, vol.Type()) if err != nil { return err } // Update the metadata of the parent if it points to a renamed volume. err = vol.MountWithSnapshotsTask(func(_ string, _ map[string]string, op *operations.Operation) error { parentDiskPath, err := b.driver.GetVolumeDiskPath(vol) if err != nil { return err } imgInfo, err := drivers.Qcow2Info(parentDiskPath) if err != nil { return err } if imgInfo.BackingFilename == snapVolDiskPath { err = drivers.Qcow2Rebase(parentDiskPath, newSnapVolDiskPath) if err != nil { return err } } // Update the metadata of the snapshot which points to a renamed volume. for _, snap := range volSnaps { currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), ProjectVolume(projectName, snap.Name, vol.Type()), nil) currentSnapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(currentSnapVol) if err != nil { return err } imgInfo, err := drivers.Qcow2Info(currentSnapVolDiskPath) if err != nil { return err } if imgInfo.BackingFilename == snapVolDiskPath { err = drivers.Qcow2Rebase(currentSnapVolDiskPath, newSnapVolDiskPath) if err != nil { return err } } } return nil }, op) if err != nil { return err } if vol.IsVMBlock() { fsParentVol := vol.NewVMBlockFilesystemVolume() fsVol := snapVol.NewVMBlockFilesystemVolume() _, newName, _ := api.GetParentAndSnapshotName(newVolName) err = drivers.Qcow2RenameConfigSnapshot(fsParentVol, fsVol, newName, op) if err != nil { return err } } return nil } // qcow2DeleteSnapshot deletes the QCOW2 volume snapshot. func (b *backend) qcow2DeleteSnapshot(vol drivers.Volume, snapVol drivers.Volume, projectName string, inst instance.Instance, devName string, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"backend": b.name, "instance": vol.Name()}) l.Debug("qcow2DeleteInstanceSnapshot started") defer l.Debug("qcow2DeleteInstanceSnapshot finished") // Return if this is not a qcow2 image. if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { return nil } _, volName := project.StorageVolumeParts(vol.Name()) // Get snapshots. volSnaps, err := VolumeDBSnapshotsGet(b, projectName, volName, vol.Type()) if err != nil { return err } _, snapName, _ := api.GetParentAndSnapshotName(snapVol.Name()) snapIndex := -1 childName := vol.Name() backingFilename := "" // Find snapshot index to delete and its child if exists. for index, snap := range volSnaps { _, currentSnapName, _ := api.GetParentAndSnapshotName(snap.Name) if snapName == currentSnapName { snapIndex = index if index+1 < len(volSnaps) { childName = ProjectVolume(projectName, volSnaps[index+1].Name, vol.Type()) } break } } if inst != nil && inst.IsRunning() { if snapIndex+1 < len(volSnaps) { tmpVol := b.GetVolume(vol.Type(), vol.ContentType(), ProjectVolume(projectName, volSnaps[snapIndex+1].Name, vol.Type()), nil) diskPath, err := b.driver.GetQcow2BackingFilePath(tmpVol) if err != nil { return err } backingFilename = diskPath } err = inst.DeleteQcow2Snapshot(devName, snapIndex, backingFilename) if err != nil { return err } // Use MountRefCountDecrement() instead of UnmountVolume() to avoid deactivating // the entire block device. The device will be reused under a different name // (renamed by Qcow2DeletionCleanup), so we only need to decrement the reference // count for this specific snapshot volume. snapVol.MountRefCountDecrement() fsVol := snapVol.NewVMBlockFilesystemVolume() _, err = b.driver.UnmountVolumeSnapshot(fsVol, op) if err != nil { return err } } else { var destinationVol string snapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(snapVol) if err != nil { return err } // Get parent volume backing file parentVolDiskPath, err := b.driver.GetQcow2BackingFilePath(vol) if err != nil { return err } // Check if the main volume is the parent of the snapshot to be deleted. err = vol.MountWithSnapshotsTask(func(_ string, _ map[string]string, op *operations.Operation) error { parentDiskPath, err := b.driver.GetVolumeDiskPath(vol) if err != nil { return err } imgInfo, err := drivers.Qcow2Info(parentDiskPath) if err != nil { return err } if imgInfo.BackingFilename == snapVolDiskPath { destinationVol = parentVolDiskPath } // Check which snapshot is the parent of the snapshot to be deleted. // Record information about the snapshot that will be deleted. for _, snap := range volSnaps { currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), ProjectVolume(projectName, snap.Name, vol.Type()), nil) currentSnapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(currentSnapVol) if err != nil { return err } imgInfo, err := drivers.Qcow2Info(currentSnapVolDiskPath) if err != nil { return err } if imgInfo.BackingFilename == snapVolDiskPath { destinationVol = currentSnapVolDiskPath } } // Move the data from the deleted snapshot to temporary volume. err = drivers.Qcow2Commit(destinationVol) if err != nil { return err } return nil }, op) if err != nil { return err } } if vol.IsVMBlock() { fsParentVol := vol.NewVMBlockFilesystemVolume() fsVol := snapVol.NewVMBlockFilesystemVolume() err = drivers.Qcow2DeleteConfigSnapshot(fsParentVol, fsVol, op) if err != nil { return err } } // Performs post operation cleanup, including renaming and removing volumes. err = b.driver.Qcow2DeletionCleanup(snapVol, childName) if err != nil { return err } return nil } // qcow2Resize resizes a QCOW2 volume. func (b *backend) qcow2Resize(inst instance.Instance, vol drivers.Volume, size string, op *operations.Operation) error { // Return if this is not a qcow2 image. if !drivers.IsQcow2Block(vol) { return nil } // For a running instance, the QMP command is used. if inst.IsRunning() { return nil } volDiskPath, err := b.driver.GetVolumeDiskPath(vol) if err != nil { return err } imgInfo, err := drivers.Qcow2Info(volDiskPath) if err != nil { return err } sizeBytes, err := units.ParseByteSizeString(size) if err != nil { return err } if sizeBytes <= 0 { return nil } const minSizeBytes = 512 // Round the size to closest minSizeBytes bytes. sizeBytes = drivers.RoundAbove(minSizeBytes, sizeBytes) if sizeBytes <= int64(imgInfo.VirtualSize) { return fmt.Errorf("Qcow2 volume cannot be shrunk") } err = drivers.Qcow2Resize(volDiskPath, sizeBytes) if err != nil { return err } return nil } // qcow2BackingPaths returns information about the backing chain of a qcow2 image. func (b *backend) qcow2BackingPaths(vol drivers.Volume, diskPath string, projectName string) ([]string, error) { if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { return nil, nil } backingPaths := []string{} diskPathSnapshot := map[string]string{} // Get snapshots. _, volName := project.StorageVolumeParts(vol.Name()) volSnaps, err := VolumeDBSnapshotsGet(b, projectName, volName, vol.Type()) if err != nil { return nil, err } for _, snap := range volSnaps { currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), ProjectVolume(projectName, snap.Name, vol.Type()), nil) currentSnapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(currentSnapVol) if err != nil { return nil, err } _, snapName, _ := api.GetParentAndSnapshotName(snap.Name) diskPathSnapshot[currentSnapVolDiskPath] = snapName } b.logger.Debug("Disk path snapshots map:", logger.Ctx{"diskPathSnapshots": diskPathSnapshot}) var chainPaths []string err = vol.MountWithSnapshotsTask(func(_ string, _ map[string]string, op *operations.Operation) error { chainPaths, err = drivers.Qcow2BackingChain(diskPath) if err != nil { return err } for _, p := range chainPaths { snapName := diskPathSnapshot[p] snapVolName := drivers.GetSnapshotVolumeName(vol.Name(), snapName) snapVol := b.GetVolume(vol.Type(), vol.ContentType(), snapVolName, nil) volDiskPath, err := b.driver.GetVolumeDiskPath(snapVol) if err != nil { return err } backingPaths = append(backingPaths, volDiskPath) } return nil }, nil) if err != nil { return nil, err } return backingPaths, nil } // qcow2MigrateVolume migrates QCOW2 volume. func (b *backend) qcow2MigrateVolume(s *state.State, vol drivers.Volume, projectName string, conn io.ReadWriteCloser, volSrcArgs *localMigration.VolumeSourceArgs, op *operations.Operation) error { l := b.logger.AddContext(logger.Ctx{"volName": vol.Name()}) l.Debug("qcow2MigrateVolume started") defer l.Debug("qcow2MigrateVolume finished") bwlimit := b.driver.Config()["rsync.bwlimit"] var rsyncArgs []string // For VM volumes, exclude the generic root disk image file from being transferred via rsync, as it will // be transferred later using a different method. if vol.IsVMBlock() { if volSrcArgs.MigrationType.FSType != migration.MigrationFSType_BLOCK_AND_RSYNC { return drivers.ErrNotSupported } rsyncArgs = []string{"--exclude", "root.img"} } else if vol.ContentType() == drivers.ContentTypeBlock && volSrcArgs.MigrationType.FSType != migration.MigrationFSType_BLOCK_AND_RSYNC || vol.ContentType() == drivers.ContentTypeFS && volSrcArgs.MigrationType.FSType != migration.MigrationFSType_RSYNC { return drivers.ErrNotSupported } var inst instance.Instance var err error var diskName string if vol.IsVMBlock() { _, volName := project.StorageVolumeParts(vol.Name()) inst, err = instance.LoadByProjectAndName(b.state, projectName, volName) if err != nil { return err } diskName, _, err = internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if err != nil { return err } } else if vol.IsCustomBlock() { var instanceArgs *db.InstanceArgs _, volName := project.StorageVolumeParts(vol.Name()) dbVol, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return err } err = VolumeUsedByInstanceDevices(b.state, b.name, projectName, &dbVol.StorageVolume, true, func(dbInst db.InstanceArgs, project api.Project, usedByDevices []string) error { if instanceArgs != nil && instanceArgs.Name != dbInst.Name { return errors.New("Volume is attached to multiple instances") } instanceArgs = &dbInst diskName = usedByDevices[0] return nil }) if err != nil { return err } if instanceArgs != nil { inst, err = instance.LoadByProjectAndName(b.state, projectName, instanceArgs.Name) if err != nil { return err } } } // Define function to send a filesystem volume. sendFSVol := func(vol drivers.Volume, conn io.ReadWriteCloser, mountPath string) error { var wrapper *ioprogress.ProgressTracker if volSrcArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", vol.Name()) } path := internalUtil.AddSlash(mountPath) b.logger.Debug("Sending filesystem volume", logger.Ctx{"volName": vol.Name(), "path": path, "bwlimit": bwlimit, "rsyncArgs": rsyncArgs}) err := rsync.Send(vol.Name(), path, conn, wrapper, volSrcArgs.MigrationType.Features, bwlimit, s.OS.ExecPath, rsyncArgs...) status, _ := linux.ExitStatus(err) if volSrcArgs.AllowInconsistent && status == 24 { return nil } return err } // Define function to send a block volume. sendBlockVol := func(vol drivers.Volume, conn io.ReadWriteCloser, blockIndex int) error { // Close when done to indicate to target side we are finished sending this volume. defer func() { _ = conn.Close() }() var wrapper *ioprogress.ProgressTracker if volSrcArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "block_progress", vol.Name()) } var nbdPath string if inst != nil && inst.IsRunning() { cleanupExport, path, err := inst.ExportQcow2Block(diskName, blockIndex) if err != nil { return err } nbdPath, err = drivers.ConnectQemuNbd(path, "", "", true) if err != nil { return err } defer func() { _ = drivers.DisconnectQemuNbd(nbdPath) cleanupExport() }() } else { path, err := b.driver.GetVolumeDiskPath(vol) if err != nil { return fmt.Errorf("Error getting VM block volume disk path: %w", err) } nbdPath, err = drivers.ConnectQemuNbd(path, "qcow2", "", true) if err != nil { return err } defer func() { _ = drivers.DisconnectQemuNbd(nbdPath) }() } from, err := os.Open(nbdPath) if err != nil { return fmt.Errorf("Error opening file for reading %q: %w", nbdPath, err) } defer func() { _ = from.Close() }() // Setup progress tracker. fromPipe := io.ReadCloser(from) if wrapper != nil { fromPipe = &ioprogress.ProgressReader{ ReadCloser: fromPipe, Tracker: wrapper, } } b.logger.Debug("Sending block volume", logger.Ctx{"volName": vol.Name(), "path": nbdPath}) _, err = util.SafeCopy(conn, fromPipe) if err != nil { return fmt.Errorf("Error copying %q to migration connection: %w", nbdPath, err) } err = from.Close() if err != nil { return fmt.Errorf("Failed to close file %q: %w", nbdPath, err) } return nil } // blockIndex identifies the block device by position rather than name. // QEMU is unaware of snapshot names, but both snapshots and block // devices are ordered from oldest to newest, allowing the index to // reliably select the correct block device to export. blockIndex := 0 // Send all snapshots to target. for _, snapName := range volSrcArgs.Snapshots { snapshot, err := vol.NewSnapshot(snapName) if err != nil { return err } // Send snapshot to target (ensure local snapshot volume is mounted if needed). err = vol.MountWithSnapshotsTask(func(parentMountPath string, snapshotMountPaths map[string]string, op *operations.Operation) error { if vol.ContentType() != drivers.ContentTypeBlock || vol.Type() != drivers.VolumeTypeCustom { err := sendFSVol(snapshot, conn, snapshotMountPaths[snapshot.Name()]) if err != nil { return err } } if vol.IsVMBlock() || (vol.ContentType() == drivers.ContentTypeBlock && vol.Type() == drivers.VolumeTypeCustom) { err = sendBlockVol(snapshot, conn, blockIndex) if err != nil { return err } } return nil }, op) if err != nil { return err } blockIndex += 1 } // Send volume to target (ensure local volume is mounted if needed). return vol.MountWithSnapshotsTask(func(parentMountPath string, _ map[string]string, op *operations.Operation) error { if !drivers.IsContentBlock(vol.ContentType()) || vol.Type() != drivers.VolumeTypeCustom { err := sendFSVol(vol, conn, parentMountPath) if err != nil { return err } } if vol.IsVMBlock() || (drivers.IsContentBlock(vol.ContentType()) && vol.Type() == drivers.VolumeTypeCustom) { err := sendBlockVol(vol, conn, blockIndex) if err != nil { return err } } return nil }, op) } // qcow2CreateVolumeFromMigration creates a QCOW2 volume from a migration. func (b *backend) qcow2CreateVolumeFromMigration(vol drivers.Volume, projectName string, conn io.ReadWriteCloser, volTargetArgs localMigration.VolumeTargetArgs, preFiller *drivers.VolumeFiller, op *operations.Operation) error { l := b.logger.AddContext(nil) l.Debug("qcow2CreateVolumeFromMigration started") defer l.Debug("qcow2CreateVolumeFromMigration finished") // Check migration transport type matches volume type. if drivers.IsContentBlock(vol.ContentType()) { if volTargetArgs.MigrationType.FSType != migration.MigrationFSType_BLOCK_AND_RSYNC { return drivers.ErrNotSupported } } else if volTargetArgs.MigrationType.FSType != migration.MigrationFSType_RSYNC { return drivers.ErrNotSupported } reverter := revert.New() defer reverter.Fail() // Create the main volume if not refreshing. if !volTargetArgs.Refresh { err := b.driver.CreateVolume(vol, preFiller, op) if err != nil { return err } reverter.Add(func() { _ = b.driver.DeleteVolume(vol, op) }) } recvFSVol := func(volName string, conn io.ReadWriteCloser, path string) error { var wrapper *ioprogress.ProgressTracker if volTargetArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", volName) } b.logger.Debug("Receiving filesystem volume started", logger.Ctx{"volName": volName, "path": path, "features": volTargetArgs.MigrationType.Features}) defer b.logger.Debug("Receiving filesystem volume stopped", logger.Ctx{"volName": volName, "path": path}) return rsync.Recv(path, conn, wrapper, volTargetArgs.MigrationType.Features) } recvBlockVol := func(volName string, conn io.ReadWriteCloser, path string) error { var wrapper *ioprogress.ProgressTracker if volTargetArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "block_progress", volName) } nbdPath, err := drivers.ConnectQemuNbd(path, "qcow2", "unmap", false) if err != nil { return err } defer func() { _ = drivers.DisconnectQemuNbd(nbdPath) }() // Reset the disk. err = linux.ClearBlock(nbdPath, 0) if err != nil { return err } to, err := os.OpenFile(nbdPath, os.O_WRONLY|os.O_TRUNC, 0) if err != nil { return fmt.Errorf("Error opening file for writing %q: %w", path, err) } defer func() { _ = to.Close() }() // Setup progress tracker. fromPipe := io.ReadCloser(conn) if wrapper != nil { fromPipe = &ioprogress.ProgressReader{ ReadCloser: fromPipe, Tracker: wrapper, } } b.logger.Debug("Receiving block volume started", logger.Ctx{"volName": volName, "path": path}) defer b.logger.Debug("Receiving block volume stopped", logger.Ctx{"volName": volName, "path": path}) toPipe := io.Writer(to) if !b.driver.Info().ZeroUnpack { toPipe = drivers.NewSparseFileWrapper(to) } _, err = util.SafeCopy(toPipe, fromPipe) if err != nil { return fmt.Errorf("Error copying from migration connection to %q: %w", path, err) } return to.Close() } // Snapshots are sent first by the sender, so create these first. for _, snapshot := range volTargetArgs.Snapshots { snapVol, err := vol.NewSnapshot(snapshot.GetName()) if err != nil { return err } // Ensure the volume is mounted. err = vol.MountWithSnapshotsTask(func(parentMountPath string, snapshotMountPaths map[string]string, op *operations.Operation) error { var err error // Setup paths to the main volume. We will receive each snapshot to these paths and then create // a snapshot of the main volume for each one. path := internalUtil.AddSlash(parentMountPath) pathBlock := "" if vol.IsVMBlock() || (drivers.IsContentBlock(vol.ContentType()) && vol.Type() == drivers.VolumeTypeCustom) { pathBlock, err = b.driver.GetVolumeDiskPath(vol) if err != nil { return fmt.Errorf("Error getting VM block volume disk path: %w", err) } } if snapVol.ContentType() != drivers.ContentTypeBlock || snapVol.Type() != drivers.VolumeTypeCustom { // Receive the filesystem snapshot first (as it is sent first). err = recvFSVol(snapVol.Name(), conn, path) if err != nil { return err } } // Receive the block snapshot next (if needed). if vol.IsVMBlock() || (vol.ContentType() == drivers.ContentTypeBlock && vol.Type() == drivers.VolumeTypeCustom) { err = recvBlockVol(snapVol.Name(), conn, pathBlock) if err != nil { return err } } // Create the snapshot itself. err = b.driver.CreateVolumeSnapshot(snapVol, op) if err != nil { return err } err = b.qcow2CreateSnapshot(vol, snapVol, projectName, nil, "", false, op) if err != nil { return err } // Setup the revert. reverter.Add(func() { _ = b.driver.DeleteVolumeSnapshot(snapVol, op) }) return nil }, op) if err != nil { return err } } err := vol.MountWithSnapshotsTask(func(parentMountPath string, snapshotMountPaths map[string]string, op *operations.Operation) error { var err error // Setup paths to the main volume. We will receive each snapshot to these paths and then create // a snapshot of the main volume for each one. path := internalUtil.AddSlash(parentMountPath) pathBlock := "" if vol.IsVMBlock() || (drivers.IsContentBlock(vol.ContentType()) && vol.Type() == drivers.VolumeTypeCustom) { pathBlock, err = b.driver.GetVolumeDiskPath(vol) if err != nil { return fmt.Errorf("Error getting VM block volume disk path: %w", err) } } if !drivers.IsContentBlock(vol.ContentType()) || vol.Type() != drivers.VolumeTypeCustom { // Receive main volume. err = recvFSVol(vol.Name(), conn, path) if err != nil { return err } } // Receive the final main volume sync if needed. if volTargetArgs.Live && (!drivers.IsContentBlock(vol.ContentType()) || (vol.Type() != drivers.VolumeTypeCustom && vol.Type() != drivers.VolumeTypeVM)) { b.logger.Debug("Starting main volume final sync", logger.Ctx{"volName": vol.Name(), "path": path}) err = recvFSVol(vol.Name(), conn, path) if err != nil { return err } } // Run EnsureMountPath after mounting and syncing to ensure the mounted directory has the // correct permissions set. err = vol.EnsureMountPath(false) if err != nil { return err } // Receive the block volume next (if needed). if vol.IsVMBlock() || (drivers.IsContentBlock(vol.ContentType()) && vol.Type() == drivers.VolumeTypeCustom) { err = recvBlockVol(vol.Name(), conn, pathBlock) if err != nil { return err } } return nil }, op) if err != nil { return err } reverter.Success() return nil } func (b *backend) qcow2BackupVolume(vol drivers.Volume, dbVol *db.StorageVolume, projectName string, writer instancewriter.InstanceWriter, basePrefix string, snapshots []string, op *operations.Operation) error { if len(snapshots) > 0 { // Check requested snapshot match those in storage. err := vol.SnapshotsMatch(snapshots, op) if err != nil { return err } } // Convert the volume type name to our internal integer representation. volumeDBType, err := VolumeTypeNameToDBType(dbVol.Type) if err != nil { return err } inst, deviceName, err := InstanceByVolumeName(b.state, b.name, projectName, vol.Name(), volumeDBType) if err != nil { return err } getDiskPath := func(v drivers.Volume, blockIndex int) (string, func(), error) { blockPath, err := b.driver.GetVolumeDiskPath(v) if err != nil { errMsg := "Error getting VM block volume disk path" if v.Type() == drivers.VolumeTypeCustom { errMsg = "Error getting custom block volume disk path" } return "", nil, fmt.Errorf(errMsg+": %w", err) } if inst != nil && inst.IsRunning() { cleanupExport, path, err := inst.ExportQcow2Block(deviceName, blockIndex) if err != nil { return "", nil, err } nbdPath, err := drivers.ConnectQemuNbd(path, "", "", true) if err != nil { return "", nil, err } cleanup := func() { _ = drivers.DisconnectQemuNbd(nbdPath) cleanupExport() } return nbdPath, cleanup, nil } nbdPath, err := drivers.ConnectQemuNbd(blockPath, "qcow2", "", true) if err != nil { return "", nil, err } cleanup := func() { _ = drivers.DisconnectQemuNbd(nbdPath) } return nbdPath, cleanup, nil } blockIndex := 0 // Handle snapshots. if len(snapshots) > 0 { for _, snapName := range snapshots { prefix := filepath.Join(basePrefix, drivers.BackupSnapshotPrefix(vol), snapName) snapVol, err := vol.NewSnapshot(snapName) if err != nil { return err } err = vol.MountWithSnapshotsTask(func(parentMountPath string, snapsMountPath map[string]string, op *operations.Operation) error { mountPath := snapsMountPath[snapVol.Name()] diskPath, cleanup, err := getDiskPath(snapVol, blockIndex) if err != nil { return err } err = drivers.BackupVolume(b.driver, snapVol, writer, mountPath, diskPath, prefix) if err != nil { cleanup() return err } cleanup() return nil }, op) if err != nil { return err } } } // Copy the main volume itself. err = vol.MountWithSnapshotsTask(func(mountPath string, _ map[string]string, op *operations.Operation) error { diskPath, cleanup, err := getDiskPath(vol, blockIndex) if err != nil { return err } err = drivers.BackupVolume(b.driver, vol, writer, mountPath, diskPath, filepath.Join(basePrefix, drivers.BackupPrefix(vol))) if err != nil { cleanup() return err } cleanup() return nil }, op) if err != nil { return err } return nil } func (b *backend) qcow2UnpackVolume(vol drivers.Volume, snapshots []string, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) (drivers.VolumePostHook, revert.Hook, error) { reverter := revert.New() defer reverter.Fail() // Find the compression algorithm used for backup source data. _, err := srcData.Seek(0, io.SeekStart) if err != nil { return nil, nil, err } tarArgs, _, unpacker, err := archive.DetectCompressionFile(srcData) if err != nil { return nil, nil, err } volExists, err := b.driver.HasVolume(vol) if err != nil { return nil, nil, err } if volExists { return nil, nil, errors.New("Cannot restore volume, already exists on target") } // Create new empty volume. err = b.driver.CreateVolume(vol, nil, nil) if err != nil { return nil, nil, err } reverter.Add(func() { _ = b.driver.DeleteVolume(vol, op) }) getDiskPath := func(v drivers.Volume) (string, func(), error) { blockPath, err := b.driver.GetVolumeDiskPath(v) if err != nil { errMsg := "Error getting VM block volume disk path" if v.Type() == drivers.VolumeTypeCustom { errMsg = "Error getting custom block volume disk path" } return "", nil, fmt.Errorf(errMsg+": %w", err) } nbdPath, err := drivers.ConnectQemuNbd(blockPath, "qcow2", "unmap", false) if err != nil { return "", nil, err } cleanup := func() { _ = drivers.DisconnectQemuNbd(nbdPath) } return nbdPath, cleanup, nil } if len(snapshots) > 0 { // Create new snapshots directory. err := drivers.CreateParentSnapshotDirIfMissing(b.driver.Name(), vol.Type(), vol.Name()) if err != nil { return nil, nil, err } } for _, snapName := range snapshots { err = vol.MountWithSnapshotsTask(func(mountPath string, _ map[string]string, op *operations.Operation) error { backupSnapshotPrefix := filepath.Join(basePrefix, drivers.BackupSnapshotPrefix(vol), snapName) diskPath, cleanup, err := getDiskPath(vol) if err != nil { return err } err = drivers.UnpackVolume(b.driver, vol, srcData, tarArgs, unpacker, backupSnapshotPrefix, mountPath, diskPath) if err != nil { cleanup() return err } cleanup() return nil }, op) if err != nil { return nil, nil, err } snapVol, err := vol.NewSnapshot(snapName) if err != nil { return nil, nil, err } b.logger.Debug("Creating volume snapshot", logger.Ctx{"snapshotName": snapVol.Name()}) err = b.driver.CreateVolumeSnapshot(snapVol, op) if err != nil { return nil, nil, err } err = b.qcow2CreateSnapshot(vol, snapVol, "", nil, "", false, op) if err != nil { return nil, nil, err } reverter.Add(func() { _ = b.driver.DeleteVolumeSnapshot(snapVol, op) }) } err = b.driver.MountVolume(vol, op) if err != nil { return nil, nil, err } reverter.Add(func() { _, _ = b.driver.UnmountVolume(vol, false, op) }) for _, snapName := range snapshots { snapVol, err := vol.NewSnapshot(snapName) if err != nil { return nil, nil, err } err = b.driver.MountVolumeSnapshot(snapVol, op) if err != nil { return nil, nil, err } } reverter.Add(func() { for _, snapName := range snapshots { snapVol, _ := vol.NewSnapshot(snapName) _, _ = b.driver.UnmountVolumeSnapshot(snapVol, op) } }) mountPath := vol.MountPath() diskPath, cleanupNBD, err := getDiskPath(vol) if err != nil { return nil, nil, err } err = drivers.UnpackVolume(b.driver, vol, srcData, tarArgs, unpacker, filepath.Join(basePrefix, drivers.BackupPrefix(vol)), mountPath, diskPath) if err != nil { cleanupNBD() return nil, nil, err } cleanupNBD() // Run EnsureMountPath after mounting and unpacking to ensure the mounted directory has the // correct permissions set. err = vol.EnsureMountPath(false) if err != nil { return nil, nil, err } cleanup := reverter.Clone().Fail // Clone before calling reverter.Success() so we can return the Fail func. reverter.Success() var postHook drivers.VolumePostHook if vol.Type() != drivers.VolumeTypeCustom { // Leave volume mounted (as is needed during backup.yaml generation during latter parts of the // backup restoration process). Create a post hook function that will be called at the end of the // backup restore process to unmount the volume if needed. postHook = func(vol drivers.Volume) error { for _, snapName := range snapshots { snapVol, err := vol.NewSnapshot(snapName) if err != nil { return err } _, err = b.driver.UnmountVolumeSnapshot(snapVol, op) if err != nil { return err } } _, err = b.driver.UnmountVolume(vol, false, op) if err != nil { return err } return nil } } else { // For custom volumes unmount now, there is no post hook as there is no backup.yaml to generate. _, err = b.driver.UnmountVolume(vol, false, op) if err != nil { return nil, nil, err } } return postHook, cleanup, nil } // volumeUsedByRunningInstance returns the name of the running instance and the // device name through which the specified volume is attached. // // If the volume is attached to more than one running instance, an error is // returned, as the caller expects the volume to be in use by at most one // running instance. func (b *backend) volumeUsedByRunningInstance(vol *db.StorageVolume, projectName string) (instance.Instance, string, error) { var inst instance.Instance var devName string runningInstCount := 0 err := VolumeUsedByInstanceDevices(b.state, b.Name(), projectName, &vol.StorageVolume, true, func(dbInst db.InstanceArgs, project api.Project, usedByDevices []string) error { i, err := instance.Load(b.state, dbInst, project) if err != nil { return err } if !i.IsRunning() { return nil } runningInstCount += 1 inst = i if len(usedByDevices) > 0 { devName = usedByDevices[0] } return nil }) if err != nil { return nil, "", err } if runningInstCount > 1 { return nil, "", errors.New("Volume is attached to multiple running instances") } return inst, devName, nil } // createDependentVolumesFromBackup creates dependent volumes from a backup. func (b *backend) createDependentVolumesFromBackup(srcBackup backup.Info, srcData io.ReadSeeker, op *operations.Operation) error { devicesMap := map[string]string{} for devName, dev := range srcBackup.Config.Container.ExpandedDevices { if dev["type"] != "disk" || util.IsFalseOrEmpty(dev["dependent"]) || dev["path"] == "/" || dev["pool"] == "" { continue } devKey := fmt.Sprintf("%s/%s", dev["pool"], dev["source"]) devicesMap[devKey] = devName } for _, disk := range srcBackup.Config.DependentVolumes { if disk == nil { return errors.New("Bad dependent volume definition found in index") } optimizedStorage := srcBackup.OptimizedStorage optimizedHeader := srcBackup.OptimizedHeader snapshots := []string{} for _, snap := range disk.VolumeSnapshots { snapshots = append(snapshots, snap.Name) } bInfo := backup.Info{ Project: disk.Volume.Project, Name: disk.Volume.Name, Backend: disk.Pool.Driver, Pool: disk.Pool.Name, OptimizedStorage: optimizedStorage, OptimizedHeader: optimizedHeader, Snapshots: snapshots, Type: backup.TypeCustom, Config: disk, } b.logger.Debug("Create dependent volume from backup", logger.Ctx{"name": bInfo.Name, "pool": bInfo.Pool}) pool, err := LoadByName(b.state, bInfo.Pool) if err != nil { return err } // Check if the backup is optimized that the source pool driver matches the target pool driver. if *bInfo.OptimizedStorage && pool.Driver().Info().Name != bInfo.Backend { return fmt.Errorf("Optimized backup storage driver %q differs from the target storage pool driver %q", bInfo.Backend, pool.Driver().Info().Name) } // Dump tarball to storage. devKey := fmt.Sprintf("%s/%s", disk.Pool.Name, disk.Volume.Name) devName, ok := devicesMap[devKey] if !ok { return fmt.Errorf("Requested volume %s on pool %s is not attached to the instance", disk.Volume.Name, disk.Pool.Name) } err = pool.CreateCustomVolumeFromBackup(bInfo, srcData, filepath.Join(backup.DefaultBackupPrefix, devName), op) if err != nil { return fmt.Errorf("Create custom volume from backup: %w", err) } } return nil } // migrateDependentVolumes migrates dependent volumes. func (b *backend) migrateDependentVolumes(inst instance.Instance, conn io.ReadWriteCloser, args *localMigration.VolumeSourceArgs, op *operations.Operation) error { for _, dependentVol := range args.DependentVolumes { diskPool, err := LoadByName(b.state, dependentVol.Pool) if err != nil { return fmt.Errorf("Failed loading storage pool: %w", err) } b.logger.Debug("migrateDependentVolumes", logger.Ctx{"name": dependentVol.Name, "pool": dependentVol.Pool, "deviceName": dependentVol.DeviceName, "type": dependentVol.MigrationType}) diskConfig, err := diskPool.GenerateCustomVolumeBackupConfig(inst.Project().Name, dependentVol.Name, !args.VolumeOnly, op) if err != nil { return err } snapshotNames := []string{} if !args.VolumeOnly { for _, snap := range dependentVol.Snapshots { snapshotNames = append(snapshotNames, *snap.Name) } } volumeArgs := &localMigration.VolumeSourceArgs{ IndexHeaderVersion: localMigration.IndexHeaderVersion, Name: dependentVol.Name, MigrationType: dependentVol.MigrationType, TrackProgress: true, ContentType: dependentVol.ContentType, Info: &localMigration.Info{Config: diskConfig}, VolumeOnly: args.VolumeOnly, Snapshots: snapshotNames, } err = diskPool.MigrateCustomVolume(inst.Project().Name, conn, volumeArgs, op) if err != nil { return err } } return nil } // createDependentVolumesFromMigration creates dependent volumes from a migration. func (b *backend) createDependentVolumesFromMigration(inst instance.Instance, conn io.ReadWriteCloser, args localMigration.VolumeTargetArgs, info *localMigration.Info, op *operations.Operation) (func(), error) { reverter := revert.New() defer reverter.Fail() createdVolumes := []localMigration.DependentVolumeArgs{} cleanup := func() { for _, vol := range createdVolumes { diskPool, err := LoadByName(b.state, vol.Pool) if err != nil { continue } _ = diskPool.DeleteCustomVolume(inst.Project().Name, vol.Name, nil) } } reverter.Add(func() { cleanup() }) for idx, dependentVol := range args.DependentVolumes { diskPool, err := LoadByName(b.state, dependentVol.Pool) if err != nil { return nil, fmt.Errorf("Failed loading storage pool: %w", err) } b.logger.Debug("createDependentVolumesFromMigration", logger.Ctx{"name": dependentVol.Name, "type": dependentVol.MigrationType, "size": dependentVol.VolumeSize}) volumeArgs := localMigration.VolumeTargetArgs{ IndexHeaderVersion: localMigration.IndexHeaderVersion, Name: dependentVol.Name, MigrationType: dependentVol.MigrationType, TrackProgress: true, ContentType: dependentVol.ContentType, VolumeOnly: args.VolumeOnly, Config: info.Config.DependentVolumes[idx].Volume.Config, Snapshots: dependentVol.Snapshots, VolumeSize: dependentVol.VolumeSize, } err = diskPool.CreateCustomVolumeFromMigration(inst.Project().Name, conn, volumeArgs, op) if err != nil { return nil, err } createdVolumes = append(createdVolumes, dependentVol) } reverter.Success() return cleanup, nil } // GetInstanceNBD returns an NBD connection to the VM's root disk. func (b *backend) GetInstanceNBD(inst instance.Instance, writable bool) (net.Conn, func(), error) { if writable && inst.IsRunning() { return nil, nil, errors.New("Writable NBD requires the instance be stopped") } instanceDeviceName, _, err := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if err != nil { return nil, nil, err } volStorageName := project.Instance(inst.Project().Name, inst.Name()) vol := b.GetVolume(drivers.VolumeTypeVM, drivers.ContentTypeBlock, volStorageName, nil) if !inst.IsRunning() { b.logger.Debug("NBD connection (offline mode)") return b.connectOfflineNBD(vol) } var volSize int64 var volDiskPath string err = vol.MountTask(func(devPath string, op *operations.Operation) error { volDiskPath, err = b.Driver().GetVolumeDiskPath(vol) if err != nil { return err } volSize, err = drivers.BlockDiskSizeBytes(volDiskPath) if err != nil { return err } return nil }, nil) if err != nil { return nil, nil, err } return inst.ConnectNBD(instanceDeviceName, volSize, writable) } // GetCustomVolumeNBD returns an NBD connection to a VM's additional disk. func (b *backend) GetCustomVolumeNBD(projectName string, volName string, writable bool) (net.Conn, func(), error) { // Get the volume. dbVol, err := VolumeDBGet(b, projectName, volName, drivers.VolumeTypeCustom) if err != nil { return nil, nil, err } volStorageName := project.StorageVolume(projectName, volName) vol := b.GetVolume(drivers.VolumeTypeCustom, drivers.ContentTypeBlock, volStorageName, nil) // Convert the volume type name to our internal integer representation. volumeDbType, err := VolumeTypeNameToDBType(dbVol.Type) if err != nil { return nil, nil, err } inst, instanceDeviceName, err := InstanceByVolumeName(b.state, b.name, projectName, volName, volumeDbType) if err != nil { if errors.Is(err, ErrVolumeNotAttachedToRunningInstance) { b.logger.Debug("NBD connection (offline mode)") return b.connectOfflineNBD(vol) } return nil, nil, err } if !inst.IsRunning() { b.logger.Debug("NBD connection (offline mode)") return b.connectOfflineNBD(vol) } if writable && inst.IsRunning() { return nil, nil, errors.New("Writable NBD requires the instance be stopped") } var volSize int64 var volDiskPath string err = vol.MountTask(func(devPath string, op *operations.Operation) error { volDiskPath, err = b.Driver().GetVolumeDiskPath(vol) if err != nil { return err } volSize, err = drivers.BlockDiskSizeBytes(volDiskPath) if err != nil { return err } return nil }, nil) if err != nil { return nil, nil, err } return inst.ConnectNBD(instanceDeviceName, volSize, writable) } // connectOfflineNBD spawns qemu-nbd for the given volume. func (b *backend) connectOfflineNBD(vol drivers.Volume) (net.Conn, func(), error) { socketPath := filepath.Join(internalUtil.RunPath(fmt.Sprintf("%s-nbd.sock", vol.Name()))) cmd := exec.Command("qemu-nbd", fmt.Sprintf("--socket=%s", socketPath)) errCh := make(chan string) go func() { err := b.Driver().ActivateTask(vol, func(devPath string, op *operations.Operation) error { volDiskPath, err := b.Driver().GetVolumeDiskPath(vol) if err != nil { return err } imgInfo, err := drivers.Qcow2Info(volDiskPath) if err != nil { return err } if imgInfo.Format == "qcow2" && len(imgInfo.FormatSpecific.Data.Bitmaps) > 0 { for _, b := range imgInfo.FormatSpecific.Data.Bitmaps { cmd.Args = append(cmd.Args, fmt.Sprintf("--bitmap=%s", b.Name)) } } cmd.Args = append(cmd.Args, volDiskPath) err = cmd.Run() if err != nil { return fmt.Errorf("Failed to start qemu-nbd: %w", err) } return nil }, nil) if err != nil { b.logger.Error("Failed when running qemu-nbd", logger.Ctx{"err": err}) errCh <- fmt.Sprintf("Failed when running qemu-nbd: %v", err) } }() // Wait for qemu-nbd. timeout := time.After(2 * time.Second) tick := time.Tick(50 * time.Millisecond) isReady := false for { select { case <-timeout: _ = cmd.Process.Kill() return nil, nil, fmt.Errorf("Timeout waiting for qemu-nbd socket") case <-tick: _, err := os.Stat(socketPath) if err == nil { isReady = true } case res := <-errCh: return nil, nil, fmt.Errorf("qemu-nbd failed: %s", res) } if isReady { break } } b.logger.Debug("Dial NBD server (offline mode)", logger.Ctx{"socketPath": socketPath}) nbdConn, err := net.Dial("unix", socketPath) if err != nil { _ = cmd.Process.Kill() return nil, nil, fmt.Errorf("Failed to connect to NBD socket: %w", err) } disconnect := func() { b.logger.Debug("User requested NBD server stopped") _ = nbdConn.Close() _ = cmd.Process.Signal(syscall.SIGTERM) _ = os.Remove(socketPath) } return nbdConn, disconnect, nil } incus-7.0.0/internal/server/storage/backend_mock.go000066400000000000000000000335151517523235500223730ustar00rootroot00000000000000package storage import ( "io" "net" "net/url" "time" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/server/backup" backupConfig "github.com/lxc/incus/v7/internal/server/backup/config" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" ) type mockBackend struct { name string state *state.State logger logger.Logger driver drivers.Driver } func (b *mockBackend) ID() int64 { return 1 // The tests expect the storage pool ID to be 1. } func (b *mockBackend) Name() string { return b.name } func (b *mockBackend) Description() string { return "" } func (b *mockBackend) Validate(config map[string]string) error { return nil } func (b *mockBackend) Status() string { return api.NetworkStatusUnknown } func (b *mockBackend) LocalStatus() string { return api.NetworkStatusUnknown } func (b *mockBackend) ToAPI() api.StoragePool { return api.StoragePool{} } func (b *mockBackend) Driver() drivers.Driver { return b.driver } // MigrationTypes returns the type of transfer methods to be used when doing migrations between pools in preference order. func (b *mockBackend) MigrationTypes(contentType drivers.ContentType, refresh bool, copySnapshots bool, clusterMove bool, storageMove bool) []migration.Type { return []migration.Type{ { FSType: FallbackMigrationType(contentType), Features: []string{"xattrs", "delete", "compress", "bidirectional"}, }, } } func (b *mockBackend) GetResources() (*api.ResourcesStoragePool, error) { return nil, nil } func (b *mockBackend) IsUsed() (bool, error) { return false, nil } func (b *mockBackend) Delete(clientType request.ClientType, op *operations.Operation) error { return nil } func (b *mockBackend) Update(clientType request.ClientType, newDescription string, newConfig map[string]string, op *operations.Operation) error { return nil } func (b *mockBackend) Create(clientType request.ClientType, op *operations.Operation) error { return nil } func (b *mockBackend) Mount() (bool, error) { return true, nil } func (b *mockBackend) Unmount() (bool, error) { return true, nil } func (b *mockBackend) ApplyPatch(name string) error { return nil } func (b *mockBackend) GetVolume(volType drivers.VolumeType, contentType drivers.ContentType, volName string, volConfig map[string]string) drivers.Volume { return drivers.Volume{} } func (b *mockBackend) CreateInstance(inst instance.Instance, op *operations.Operation) error { return nil } func (b *mockBackend) CreateInstanceFromBackup(srcBackup backup.Info, srcData io.ReadSeeker, op *operations.Operation) (func(instance.Instance) error, revert.Hook, error) { return nil, nil, nil } func (b *mockBackend) CreateInstanceFromCopy(inst instance.Instance, src instance.Instance, snapshots bool, allowInconsistent bool, op *operations.Operation) error { return nil } func (b *mockBackend) CreateInstanceFromImage(inst instance.Instance, fingerprint string, op *operations.Operation) error { return nil } func (b *mockBackend) CreateInstanceFromMigration(inst instance.Instance, conn io.ReadWriteCloser, args migration.VolumeTargetArgs, op *operations.Operation) error { return nil } func (b *mockBackend) RenameInstance(inst instance.Instance, newName string, op *operations.Operation) error { return nil } func (b *mockBackend) DeleteInstance(inst instance.Instance, op *operations.Operation) error { return nil } func (b *mockBackend) UpdateInstance(inst instance.Instance, newDesc string, newConfig map[string]string, op *operations.Operation) error { return nil } func (b *mockBackend) GenerateCustomVolumeBackupConfig(projectName string, volName string, snapshots bool, op *operations.Operation) (*backupConfig.Config, error) { return nil, nil } // GenerateInstanceBackupConfig returns the backup config entry for this instance. func (b *mockBackend) GenerateInstanceBackupConfig(inst instance.Instance, snapshots bool, dependentVolumes bool, op *operations.Operation) (*backupConfig.Config, error) { return nil, nil } func (b *mockBackend) UpdateInstanceBackupFile(inst instance.Instance, snapshot bool, op *operations.Operation) error { return nil } func (b *mockBackend) CheckInstanceBackupFileSnapshots(backupConf *backupConfig.Config, projectName string, deleteMissing bool, op *operations.Operation) ([]*api.InstanceSnapshot, error) { return nil, nil } func (b *mockBackend) ListUnknownVolumes(op *operations.Operation) (map[string][]*backupConfig.Config, error) { return nil, nil } func (b *mockBackend) ImportInstance(inst instance.Instance, poolVol *backupConfig.Config, op *operations.Operation) (revert.Hook, error) { return nil, nil } func (b *mockBackend) MigrateInstance(inst instance.Instance, conn io.ReadWriteCloser, args *migration.VolumeSourceArgs, op *operations.Operation) error { return nil } func (b *mockBackend) CleanupInstancePaths(inst instance.Instance, op *operations.Operation) error { return nil } // RefreshCustomVolume refresh a custom volume. func (b *mockBackend) RefreshCustomVolume(projectName string, srcProjectName string, volName string, desc string, config map[string]string, srcPoolName, srcVolName string, srcVolOnly bool, excludeOlder bool, op *operations.Operation) error { return nil } func (b *mockBackend) RefreshInstance(inst instance.Instance, src instance.Instance, srcSnapshots []instance.Instance, allowInconsistent bool, op *operations.Operation) error { return nil } // BackupInstance creates an instance backup. func (b *mockBackend) BackupInstance(inst instance.Instance, tarWriter *instancewriter.InstanceTarWriter, optimized bool, snapshots bool, dependentVolumes bool, op *operations.Operation) error { return nil } func (b *mockBackend) GetInstanceUsage(inst instance.Instance) (*VolumeUsage, error) { return nil, nil } func (b *mockBackend) SetInstanceQuota(inst instance.Instance, size string, vmStateSize string, op *operations.Operation) error { return nil } func (b *mockBackend) MountInstance(inst instance.Instance, op *operations.Operation) (*MountInfo, error) { return &MountInfo{}, nil } func (b *mockBackend) UnmountInstance(inst instance.Instance, op *operations.Operation) error { return nil } func (b *mockBackend) CreateInstanceSnapshot(i instance.Instance, src instance.Instance, op *operations.Operation) error { return nil } func (b *mockBackend) RenameInstanceSnapshot(inst instance.Instance, newName string, op *operations.Operation) error { return nil } func (b *mockBackend) DeleteInstanceSnapshot(inst instance.Instance, op *operations.Operation) error { return nil } // CanRestoreInstanceSnapshot checks if an instance snapshot can be restored. func (b *mockBackend) CanRestoreInstanceSnapshot(inst instance.Instance, src instance.Instance) error { return nil } func (b *mockBackend) RestoreInstanceSnapshot(inst instance.Instance, src instance.Instance, op *operations.Operation) error { return nil } func (b *mockBackend) MountInstanceSnapshot(inst instance.Instance, op *operations.Operation) (*MountInfo, error) { return &MountInfo{}, nil } func (b *mockBackend) UnmountInstanceSnapshot(inst instance.Instance, op *operations.Operation) error { return nil } func (b *mockBackend) UpdateInstanceSnapshot(inst instance.Instance, newDesc string, newConfig map[string]string, op *operations.Operation) error { return nil } func (b *mockBackend) EnsureImage(fingerprint string, op *operations.Operation) error { return nil } func (b *mockBackend) DeleteImage(fingerprint string, op *operations.Operation) error { return nil } func (b *mockBackend) UpdateImage(fingerprint, newDesc string, newConfig map[string]string, op *operations.Operation) error { return nil } func (b *mockBackend) CreateBucket(projectName string, bucket api.StorageBucketsPost, op *operations.Operation) error { return nil } func (b *mockBackend) UpdateBucket(projectName string, bucketName string, bucket api.StorageBucketPut, op *operations.Operation) error { return nil } func (b *mockBackend) DeleteBucket(projectName string, bucketName string, op *operations.Operation) error { return nil } func (b *mockBackend) ImportBucket(projectName string, poolVol *backupConfig.Config, op *operations.Operation) (revert.Hook, error) { return nil, nil } func (b *mockBackend) CreateBucketKey(projectName string, bucketName string, key api.StorageBucketKeysPost, op *operations.Operation) (*api.StorageBucketKey, error) { return nil, nil } func (b *mockBackend) UpdateBucketKey(projectName string, bucketName string, keyName string, key api.StorageBucketKeyPut, op *operations.Operation) error { return nil } func (b *mockBackend) DeleteBucketKey(projectName string, bucketName string, keyName string, op *operations.Operation) error { return nil } // MountLocalBucket mounts the local bucket volume and returns its mount path // along with an unmount function that the caller must invoke when finished. func (b *mockBackend) MountLocalBucket(projectName string, bucketName string, op *operations.Operation) (string, func() error, error) { return "", func() error { return nil }, nil } func (b *mockBackend) GetBucketURL(bucketName string) *url.URL { return nil } func (b *mockBackend) CreateCustomVolume(projectName string, volName string, desc string, config map[string]string, contentType drivers.ContentType, op *operations.Operation) error { return nil } func (b *mockBackend) CreateCustomVolumeFromCopy(projectName string, srcProjectName string, volName string, desc string, config map[string]string, srcPoolName string, srcVolName string, srcVolOnly bool, op *operations.Operation) error { return nil } func (b *mockBackend) RenameCustomVolume(projectName string, volName string, newName string, op *operations.Operation) error { return nil } func (b *mockBackend) UpdateCustomVolume(projectName string, volName string, newDesc string, newConfig map[string]string, op *operations.Operation) error { return nil } func (b *mockBackend) DeleteCustomVolume(projectName string, volName string, op *operations.Operation) error { return nil } func (b *mockBackend) MigrateCustomVolume(projectName string, conn io.ReadWriteCloser, args *migration.VolumeSourceArgs, op *operations.Operation) error { return nil } func (b *mockBackend) CreateCustomVolumeFromMigration(projectName string, conn io.ReadWriteCloser, args migration.VolumeTargetArgs, op *operations.Operation) error { return nil } func (b *mockBackend) GetCustomVolumeDisk(projectName string, volName string) (string, error) { return "", nil } func (b *mockBackend) GetCustomVolumeUsage(projectName string, volName string) (*VolumeUsage, error) { return nil, nil } func (b *mockBackend) MountCustomVolume(projectName string, volName string, op *operations.Operation) (*MountInfo, error) { return nil, nil } func (b *mockBackend) UnmountCustomVolume(projectName string, volName string, op *operations.Operation) (bool, error) { return true, nil } func (b *mockBackend) ImportCustomVolume(projectName string, poolVol *backupConfig.Config, op *operations.Operation) (revert.Hook, error) { return nil, nil } // CreateCustomVolumeSnapshot creates a snapshot of a custom volume. func (b *mockBackend) CreateCustomVolumeSnapshot(projectName string, volName string, newSnapshotName string, expiryDate time.Time, instanceStateful bool, op *operations.Operation) error { return nil } func (b *mockBackend) RenameCustomVolumeSnapshot(projectName string, volName string, newName string, op *operations.Operation) error { return nil } func (b *mockBackend) DeleteCustomVolumeSnapshot(projectName string, volName string, op *operations.Operation) error { return nil } func (b *mockBackend) UpdateCustomVolumeSnapshot(projectName string, volName string, newDesc string, newConfig map[string]string, expiryDate time.Time, op *operations.Operation) error { return nil } func (b *mockBackend) RestoreCustomVolume(projectName string, volName string, snapshotName string, op *operations.Operation) error { return nil } // BackupCustomVolume creates a custom volume backup. func (b *mockBackend) BackupCustomVolume(projectName string, volName string, writer instancewriter.InstanceWriter, basePrefix string, optimized bool, snapshots bool, op *operations.Operation) error { return nil } // CreateCustomVolumeFromBackup creates a custom volume from a backup. func (b *mockBackend) CreateCustomVolumeFromBackup(srcBackup backup.Info, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) error { return nil } func (b *mockBackend) CreateCustomVolumeFromISO(projectName string, volName string, srcData io.ReadSeeker, size int64, op *operations.Operation) error { return nil } // GenerateBucketBackupConfig returns the backup config entry for this bucket. func (b *mockBackend) GenerateBucketBackupConfig(projectName string, bucketName string, op *operations.Operation) (*backupConfig.Config, error) { return nil, nil } // BackupBucket backups up a bucket to a tarball. func (b *mockBackend) BackupBucket(projectName string, bucketName string, tarWriter *instancewriter.InstanceTarWriter, op *operations.Operation) error { return nil } // CreateBucketFromBackup creates a bucket from a tarball. func (b *mockBackend) CreateBucketFromBackup(srcBackup backup.Info, srcData io.ReadSeeker, op *operations.Operation) error { return nil } // GetInstanceNBD returns an NBD connection to the VM's root disk. func (b *mockBackend) GetInstanceNBD(inst instance.Instance, writable bool) (net.Conn, func(), error) { return nil, nil, nil } // GetCustomVolumeNBD returns an NBD connection to a VM's additional disk. func (b *mockBackend) GetCustomVolumeNBD(projectName string, volName string, writable bool) (net.Conn, func(), error) { return nil, nil, nil } incus-7.0.0/internal/server/storage/backend_patches.go000066400000000000000000000152641517523235500230720ustar00rootroot00000000000000package storage import ( "context" "fmt" "net/http" "slices" "time" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) var earlyPatches = map[string]func(b *backend) error{ "storage_missing_snapshot_records": patchMissingSnapshotRecords, "storage_delete_old_snapshot_records": patchDeleteOldSnapshotRecords, "storage_prefix_bucket_names_with_project": patchBucketNames, } var latePatches = map[string]func(b *backend) error{} // Patches start here. // patchMissingSnapshotRecords creates any missing storage volume records for instance volume snapshots. // This is needed because it seems that in 2019 some instance snapshots did not have their associated volume DB // records created. This later caused problems when we started validating that the instance snapshot DB record // count matched the volume snapshot DB record count. func patchMissingSnapshotRecords(b *backend) error { var err error var localNode string err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { localNode, err = tx.GetLocalNodeName(ctx) if err != nil { return fmt.Errorf("Failed to get local member name: %w", err) } return err }) if err != nil { return err } // Get instances on this local server (as the DB helper functions return volumes on local server), also // avoids running the same queries on every cluster member for instances on shared storage. filter := cluster.InstanceFilter{Node: &localNode} err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.InstanceList(ctx, func(inst db.InstanceArgs, p api.Project) error { // Check we can convert the instance to the volume type needed. volType, err := InstanceTypeToVolumeType(inst.Type) if err != nil { return err } contentType := drivers.ContentTypeFS if inst.Type == instancetype.VM { contentType = drivers.ContentTypeBlock } // Get all the instance snapshot DB records. var instPoolName string var snapshots []cluster.Instance err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { instPoolName, err = tx.GetInstancePool(ctx, p.Name, inst.Name) if err != nil { if api.StatusErrorCheck(err, http.StatusNotFound) { // If the instance cannot be associated to a pool its got bigger problems // outside the scope of this patch. Will skip due to empty instPoolName. return nil } return fmt.Errorf("Failed finding pool for instance %q in project %q: %w", inst.Name, p.Name, err) } snapshots, err = tx.GetInstanceSnapshotsWithName(ctx, p.Name, inst.Name) if err != nil { return err } return nil }) if err != nil { return err } if instPoolName != b.Name() { return nil // This instance isn't hosted on this storage pool, skip. } dbVol, err := VolumeDBGet(b, p.Name, inst.Name, volType) if err != nil { return fmt.Errorf("Failed loading storage volume record %q: %w", inst.Name, err) } // Get all the instance volume snapshot DB records. dbVolSnaps, err := VolumeDBSnapshotsGet(b, p.Name, inst.Name, volType) if err != nil { return fmt.Errorf("Failed loading storage volume snapshot records %q: %w", inst.Name, err) } for i := range snapshots { foundVolumeSnapshot := false for _, dbVolSnap := range dbVolSnaps { if dbVolSnap.Name == snapshots[i].Name { foundVolumeSnapshot = true break } } if !foundVolumeSnapshot { b.logger.Info("Creating missing volume snapshot record", logger.Ctx{"project": snapshots[i].Project, "instance": snapshots[i].Name}) err = VolumeDBCreate(b, snapshots[i].Project, snapshots[i].Name, "Auto repaired", volType, true, dbVol.Config, snapshots[i].CreationDate, time.Time{}, contentType, false, true) if err != nil { return err } } } return nil }, filter) }) if err != nil { return err } return nil } // patchDeleteOldSnapshotRecords deletes the remaining snapshot records in storage_volumes // (a previous patch would have already moved them into storage_volume_snapshots). func patchDeleteOldSnapshotRecords(b *backend) error { err := b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { nodeID := tx.GetNodeID() _, err := tx.Tx().Exec(` DELETE FROM storage_volumes WHERE id IN ( SELECT id FROM storage_volumes JOIN ( /* Create a two column intermediary table containing the container name and its snapshot name */ SELECT sv.name inst_name, svs.name snap_name FROM storage_volumes AS sv JOIN storage_volumes_snapshots AS svs ON sv.id = svs.storage_volume_id AND node_id=? AND type=? ) j1 ON name=printf("%s/%s", j1.inst_name, j1.snap_name) /* Only keep the records with a matching 'name' pattern, 'node_id' and 'type' */ ); `, nodeID, db.StoragePoolVolumeTypeContainer) if err != nil { return fmt.Errorf("Failed to delete remaining instance snapshot records in the `storage_volumes` table: %w", err) } return nil }) if err != nil { return err } return nil } // patchBucketNames modifies the naming convention of bucket volumes by adding // the corresponding project name as a prefix. func patchBucketNames(b *backend) error { // Apply patch only for btrfs, dir, lvm, and zfs drivers. if !slices.Contains([]string{"btrfs", "dir", "lvm", "zfs"}, b.driver.Info().Name) { return nil } var buckets map[string]*db.StorageBucket err := b.state.DB.Cluster.Transaction(b.state.ShutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error { // Get local storage buckets. localBuckets, err := tx.GetStoragePoolBuckets(ctx, true) if err != nil { return err } buckets = make(map[string]*db.StorageBucket, len(localBuckets)) for _, bucket := range localBuckets { buckets[bucket.Name] = bucket } return nil }) if err != nil { return err } // Get list of volumes. volumes, err := b.driver.ListVolumes() if err != nil { return err } for _, v := range volumes { // Ensure volume is of type bucket. if v.Type() != drivers.VolumeTypeBucket { continue } // Retrieve the bucket associated with the current volume's name. bucket, ok := buckets[v.Name()] if !ok { continue } newVolumeName := project.StorageVolume(bucket.Project, bucket.Name) // Rename volume. err := b.driver.RenameVolume(v, newVolumeName, nil) if err != nil { return err } } return nil } incus-7.0.0/internal/server/storage/drivers/000077500000000000000000000000001517523235500211135ustar00rootroot00000000000000incus-7.0.0/internal/server/storage/drivers/bucket.go000066400000000000000000000002731517523235500227210ustar00rootroot00000000000000package drivers // S3Credentials represents the credentials to access a bucket. type S3Credentials struct { AccessKey string `json:"access_key"` SecretKey string `json:"secret_key"` } incus-7.0.0/internal/server/storage/drivers/driver_btrfs.go000066400000000000000000000366421517523235500241500ustar00rootroot00000000000000package drivers import ( "errors" "fmt" "io/fs" "os" "os/exec" "path/filepath" "strings" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var ( btrfsVersion string btrfsLoaded bool ) type btrfs struct { common } // load is used to run one-time action per-driver rather than per-pool. func (d *btrfs) load() error { // Register the patches. d.patches = map[string]func() error{ "storage_lvm_skipactivation": nil, "storage_missing_snapshot_records": nil, "storage_delete_old_snapshot_records": nil, "storage_zfs_drop_block_volume_filesystem_extension": nil, "storage_prefix_bucket_names_with_project": nil, } // Done if previously loaded. if btrfsLoaded { return nil } // Validate the required binaries. for _, tool := range []string{"btrfs"} { _, err := exec.LookPath(tool) if err != nil { return fmt.Errorf("Required tool %q is missing", tool) } } // Detect and record the version. if btrfsVersion == "" { out, err := subprocess.RunCommand("btrfs", "version") if err != nil { return err } count, err := fmt.Sscanf(strings.SplitN(out, " ", 2)[1], "v%s\n", &btrfsVersion) if err != nil || count != 1 { return errors.New("The 'btrfs' tool isn't working properly") } } btrfsLoaded = true return nil } // Info returns info about the driver and its environment. func (d *btrfs) Info() Info { return Info{ Name: "btrfs", Version: btrfsVersion, DefaultVMBlockFilesystemSize: deviceConfig.DefaultVMBlockFilesystemSize, OptimizedImages: true, OptimizedBackups: true, OptimizedBackupHeader: true, PreservesInodes: !d.state.OS.RunningInUserNS, Remote: d.isRemote(), VolumeTypes: []VolumeType{VolumeTypeBucket, VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM}, VolumeMultiNode: d.isRemote(), BlockBacking: false, RunningCopyFreeze: false, DirectIO: true, IOUring: true, MountedRoot: true, Buckets: true, } } // FillConfig populates the storage pool's configuration file with the default values. func (d *btrfs) FillConfig() error { loopPath := loopFilePath(d.name) if d.config["source"] == "" || d.config["source"] == loopPath { // Pick a default size of the loop file if not specified. if d.config["size"] == "" { defaultSize, err := loopFileSizeDefault() if err != nil { return err } d.config["size"] = fmt.Sprintf("%dGiB", defaultSize) } } else { // Unset size property since it's irrelevant. d.config["size"] = "" } return nil } // Create is called during pool creation and is effectively using an empty driver struct. // WARNING: The Create() function cannot rely on any of the struct attributes being set. func (d *btrfs) Create() error { // Store the provided source as we are likely to be mangling it. d.config["volatile.initial_source"] = d.config["source"] reverter := revert.New() defer reverter.Fail() err := d.FillConfig() if err != nil { return err } loopPath := loopFilePath(d.name) if d.config["source"] == "" || d.config["source"] == loopPath { // Check for IncusOS. if d.state.OS.IncusOS != nil { return errors.New("Loop backed pools aren't supported on IncusOS") } // Create a loop based pool. d.config["source"] = loopPath // Create the loop file itself. size, err := units.ParseByteSizeString(d.config["size"]) if err != nil { return err } err = ensureSparseFile(d.config["source"], size) if err != nil { return fmt.Errorf("Failed to create the sparse file: %w", err) } reverter.Add(func() { _ = os.Remove(d.config["source"]) }) // Format the file. _, err = makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name}) if err != nil { return fmt.Errorf("Failed to format sparse file: %w", err) } } else if linux.IsBlockdevPath(d.config["source"]) { // Wipe if requested. if util.IsTrue(d.config["source.wipe"]) { err := wipeBlockHeaders(d.config["source"]) if err != nil { return fmt.Errorf("Failed to wipe headers from disk %q: %w", d.config["source"], err) } d.config["source.wipe"] = "" } // Format the block device. _, err := makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name}) if err != nil { return fmt.Errorf("Failed to format block device: %w", err) } // Record the UUID as the source. devUUID, err := fsUUID(d.config["source"]) if err != nil { return err } // Confirm that the symlink is appearing (give it 10s). // In case of timeout it falls back to using the volume's path // instead of its UUID. if tryExists(fmt.Sprintf("/dev/disk/by-uuid/%s", devUUID)) { // Override the config to use the UUID. d.config["source"] = devUUID } } else if d.config["source"] != "" { hostPath := d.config["source"] if d.isSubvolume(hostPath) { // Existing btrfs subvolume. hasSubvolumes, err := d.hasSubvolumes(hostPath) if err != nil { return fmt.Errorf("Could not determine if existing btrfs subvolume is empty: %w", err) } // Check that the provided subvolume is empty. if hasSubvolumes { return errors.New("Requested btrfs subvolume exists but is not empty") } } else { // New btrfs subvolume on existing btrfs filesystem. cleanSource := filepath.Clean(hostPath) daemonDir := internalUtil.VarPath() if util.PathExists(hostPath) { hostPathFS, _ := linux.DetectFilesystem(hostPath) if hostPathFS != "btrfs" { return fmt.Errorf("Provided path does not reside on a btrfs filesystem (detected %s)", hostPathFS) } } if strings.HasPrefix(cleanSource, daemonDir) { if cleanSource != GetPoolMountPath(d.name) { return fmt.Errorf("Only allowed source path under %q is %q", internalUtil.VarPath(), GetPoolMountPath(d.name)) } storagePoolDirFS, _ := linux.DetectFilesystem(internalUtil.VarPath("storage-pools")) if storagePoolDirFS != "btrfs" { return fmt.Errorf("Provided path does not reside on a btrfs filesystem (detected %s)", storagePoolDirFS) } // Delete the current directory to replace by subvolume. err := os.Remove(cleanSource) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove %q: %w", cleanSource, err) } } // Create the subvolume. _, err := subprocess.RunCommand("btrfs", "subvolume", "create", hostPath) if err != nil { return err } } } else { return errors.New(`Invalid "source" property`) } reverter.Success() return nil } // Delete removes the storage pool from the storage device. func (d *btrfs) Delete(op *operations.Operation) error { // If the user completely destroyed it, call it done. if !util.PathExists(GetPoolMountPath(d.name)) { return nil } // Delete potential intermediate btrfs subvolumes. for _, volType := range d.Info().VolumeTypes { for _, dir := range BaseDirectories[volType].Paths { path := filepath.Join(GetPoolMountPath(d.name), dir) if !util.PathExists(path) { continue } if !d.isSubvolume(path) { continue } err := d.deleteSubvolume(path, true) if err != nil { return fmt.Errorf("Failed deleting btrfs subvolume %q", path) } } } // On delete, wipe everything in the directory. mountPath := GetPoolMountPath(d.name) err := wipeDirectory(mountPath) if err != nil { return fmt.Errorf("Failed removing mount path %q: %w", mountPath, err) } // Unmount the path. _, err = d.Unmount() if err != nil { return err } // If the pool path is a subvolume itself, delete it. if d.isSubvolume(mountPath) { err := d.deleteSubvolume(mountPath, false) if err != nil { return err } // And re-create as an empty directory to make the backend happy. err = os.Mkdir(mountPath, 0o700) if err != nil { return fmt.Errorf("Failed creating directory %q: %w", mountPath, err) } } // Delete any loop file we may have used. loopPath := loopFilePath(d.name) err = os.Remove(loopPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed removing loop file %q: %w", loopPath, err) } return nil } // Validate checks that all provide keys are supported and that no conflicting or missing configuration is present. func (d *btrfs) Validate(config map[string]string) error { // gendoc:generate(entity=storage_btrfs, group=common, key=source) // // --- // type: string // scope: local // default: - // shortdesc: Path to an existing block device, loop file or Btrfs subvolume // gendoc:generate(entity=storage_btrfs, group=common, key=source.wipe) // // --- // type: bool // scope: local // default: `false` // shortdesc: Wipe the block device specified in `source` prior to creating the storage pool rules := map[string]func(value string) error{ // gendoc:generate(entity=storage_btrfs, group=common, key=size) // // --- // type: string // scope: local // default: auto (20% of free disk space, >= 5 GiB and <= 30 GiB) // shortdesc: Size of the storage pool when creating loop-based pools (in bytes, suffixes supported, can be increased to grow storage pool) "size": validate.Optional(validate.IsSize), // gendoc:generate(entity=storage_btrfs, group=common, key=btrfs.mount_options) // // --- // type: string // scope: global // default: `user_subvol_rm_allowed` // shortdesc: Mount options for block devices "btrfs.mount_options": validate.IsAny, } return d.validatePool(config, rules, nil) } // Update applies any driver changes required from a configuration change. func (d *btrfs) Update(changedConfig map[string]string) error { // We only care about btrfs.mount_options. val, ok := changedConfig["btrfs.mount_options"] if ok { // Custom mount options don't work inside containers if d.state.OS.RunningInUserNS { return nil } // Trigger a re-mount. d.config["btrfs.mount_options"] = val mntFlags, mntOptions := linux.ResolveMountOptions(strings.Split(d.getMountOptions(), ",")) mntFlags |= unix.MS_REMOUNT err := TryMount("", GetPoolMountPath(d.name), "none", mntFlags, mntOptions) if err != nil { return err } } size, ok := changedConfig["size"] if ok { // Figure out loop path loopPath := loopFilePath(d.name) if d.config["source"] != loopPath { return errors.New("Cannot resize non-loopback pools") } // Resize loop file f, err := os.OpenFile(loopPath, os.O_RDWR, 0o600) if err != nil { return err } defer func() { _ = f.Close() }() sizeBytes, _ := units.ParseByteSizeString(size) err = f.Truncate(sizeBytes) if err != nil { return err } loopDevPath, err := loopDeviceSetup(loopPath) if err != nil { return err } defer func() { _ = loopDeviceAutoDetach(loopDevPath) }() err = loopDeviceSetCapacity(loopDevPath) if err != nil { return err } _, err = subprocess.RunCommand("btrfs", "filesystem", "resize", "max", GetPoolMountPath(d.name)) if err != nil { return err } } return nil } // Mount mounts the storage pool. func (d *btrfs) Mount() (bool, error) { // Check if already mounted. if linux.IsMountPoint(GetPoolMountPath(d.name)) { return false, nil } var err error // Setup mount options. loopPath := loopFilePath(d.name) mntSrc := "" mntDst := GetPoolMountPath(d.name) mntFilesystem := "btrfs" if d.config["source"] == loopPath { mntSrc, err = loopDeviceSetup(d.config["source"]) if err != nil { return false, err } defer func() { _ = loopDeviceAutoDetach(mntSrc) }() } else if filepath.IsAbs(d.config["source"]) { // Bring up an existing device or path. mntSrc = d.config["source"] if !linux.IsBlockdevPath(mntSrc) { mntFilesystem = "none" mntSrcFS, _ := linux.DetectFilesystem(mntSrc) if mntSrcFS != "btrfs" { return false, fmt.Errorf("Source path %q isn't btrfs (detected %s)", mntSrc, mntSrcFS) } } } else { // Mount using UUID. mntSrc = fmt.Sprintf("/dev/disk/by-uuid/%s", d.config["source"]) } // Get the custom mount flags/options. mntFlags, mntOptions := linux.ResolveMountOptions(strings.Split(d.getMountOptions(), ",")) // Handle bind-mounts first. if mntFilesystem == "none" { // Setup the bind-mount itself. err := TryMount(mntSrc, mntDst, mntFilesystem, unix.MS_BIND, "") if err != nil { return false, err } // Custom mount options don't work inside containers if d.state.OS.RunningInUserNS { return true, nil } // Now apply the custom options. mntFlags |= unix.MS_REMOUNT err = TryMount("", mntDst, mntFilesystem, mntFlags, mntOptions) if err != nil { return false, err } return true, nil } // Handle traditional mounts. err = TryMount(mntSrc, mntDst, mntFilesystem, mntFlags, mntOptions) if err != nil { return false, err } return true, nil } // Unmount unmounts the storage pool. func (d *btrfs) Unmount() (bool, error) { // Unmount the pool. ourUnmount, err := forceUnmount(GetPoolMountPath(d.name)) if err != nil { return false, err } return ourUnmount, nil } // GetResources returns the pool resource usage information. func (d *btrfs) GetResources() (*api.ResourcesStoragePool, error) { return genericVFSGetResources(d) } // MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order. func (d *btrfs) MigrationTypes(contentType ContentType, refresh bool, copySnapshots bool, clusterMove bool, storageMove bool) []localMigration.Type { var rsyncFeatures []string btrfsFeatures := []string{migration.BTRFSFeatureMigrationHeader, migration.BTRFSFeatureSubvolumes, migration.BTRFSFeatureSubvolumeUUIDs} // Do not pass compression argument to rsync if the associated // config key, that is rsync.compression, is set to false. if util.IsFalse(d.Config()["rsync.compression"]) { rsyncFeatures = []string{"xattrs", "delete", "bidirectional"} } else { rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"} } // Only offer rsync if running in an unprivileged container. if d.state.OS.RunningInUserNS { var transportType migration.MigrationFSType if IsContentBlock(contentType) { transportType = migration.MigrationFSType_BLOCK_AND_RSYNC } else { transportType = migration.MigrationFSType_RSYNC } return []localMigration.Type{ { FSType: transportType, Features: rsyncFeatures, }, } } if IsContentBlock(contentType) { return []localMigration.Type{ { FSType: migration.MigrationFSType_BTRFS, Features: btrfsFeatures, }, { FSType: migration.MigrationFSType_BLOCK_AND_RSYNC, Features: rsyncFeatures, }, } } if refresh && !copySnapshots { return []localMigration.Type{ { FSType: migration.MigrationFSType_RSYNC, Features: rsyncFeatures, }, } } return []localMigration.Type{ { FSType: migration.MigrationFSType_BTRFS, Features: btrfsFeatures, }, { FSType: migration.MigrationFSType_RSYNC, Features: rsyncFeatures, }, } } incus-7.0.0/internal/server/storage/drivers/driver_btrfs_utils.go000066400000000000000000000434571517523235500253720ustar00rootroot00000000000000package drivers import ( "bufio" "bytes" "context" "errors" "fmt" "io" "io/fs" "os" "os/exec" "path/filepath" "sort" "strconv" "strings" "unsafe" "github.com/google/uuid" "go.yaml.in/yaml/v4" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/backup" localUtil "github.com/lxc/incus/v7/internal/server/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" ) // Errors. var ( errBtrfsNoQuota = errors.New("Quotas disabled on filesystem") errBtrfsNoQGroup = errors.New("Unable to find quota group") ) // btrfsISOVolSuffix suffix used for iso content type volumes. const btrfsISOVolSuffix = ".iso" // setReceivedUUID sets the "Received UUID" field on a subvolume with the given path using ioctl. func setReceivedUUID(path string, UUID string) error { type btrfsIoctlReceivedSubvolArgs struct { uuid [16]byte _ [22]uint64 // padding } f, err := os.Open(path) if err != nil { return fmt.Errorf("Failed opening %s: %w", path, err) } defer func() { _ = f.Close() }() args := btrfsIoctlReceivedSubvolArgs{} strUUID, err := uuid.Parse(UUID) if err != nil { return fmt.Errorf("Failed parsing UUID: %w", err) } binUUID, err := strUUID.MarshalBinary() if err != nil { return fmt.Errorf("Failed converting UUID: %w", err) } copy(args.uuid[:], binUUID) _, _, errno := unix.Syscall(unix.SYS_IOCTL, f.Fd(), linux.IoctlBtrfsSetReceivedSubvol, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed setting received UUID: %w", unix.Errno(errno)) } return nil } func (d *btrfs) getMountOptions() string { // Allow overriding the default options. if d.config["btrfs.mount_options"] != "" { return d.config["btrfs.mount_options"] } return "user_subvol_rm_allowed" } func (d *btrfs) isSubvolume(path string) bool { // Stat the path. fs := unix.Stat_t{} err := unix.Lstat(path, &fs) if err != nil { return false } // Check if BTRFS_FIRST_FREE_OBJECTID is the inode number. if fs.Ino != 256 { return false } return true } func (d *btrfs) hasSubvolumes(path string) (bool, error) { var stdout strings.Builder err := subprocess.RunCommandWithFds(d.state.ShutdownCtx, nil, &stdout, "btrfs", "subvolume", "list", "-o", path) if err != nil { return false, err } return stdout.Len() > 0, nil } func (d *btrfs) getSubvolumes(path string) ([]string, error) { // Make sure the path has a trailing slash. if !strings.HasSuffix(path, "/") { path = path + "/" } poolMountPath := GetPoolMountPath(d.name) if !strings.HasPrefix(path, poolMountPath+"/") { return nil, fmt.Errorf("%q is outside pool mount path %q", path, poolMountPath) } var result []string if d.state.OS.RunningInUserNS { // If using BTRFS in a nested container we cannot use "btrfs subvolume list" due to a permission error. // So instead walk the directory tree testing each directory to see if it is subvolume. err := filepath.Walk(path, func(fpath string, entry fs.FileInfo, err error) error { if err != nil { return err } // Ignore the base path. if strings.TrimRight(fpath, "/") == strings.TrimRight(path, "/") { return nil } // Subvolumes can only be directories. if !entry.IsDir() { return nil } // Check if directory is a subvolume. if d.isSubvolume(fpath) { result = append(result, strings.TrimPrefix(fpath, path)) } return nil }) if err != nil { return nil, err } } else { // If not running inside a nested container we can use "btrfs subvolume list" to get subvolumes which is more // performant than walking the directory tree. var stdout bytes.Buffer err := subprocess.RunCommandWithFds(d.state.ShutdownCtx, nil, &stdout, "btrfs", "subvolume", "list", poolMountPath) if err != nil { return nil, err } path = strings.TrimPrefix(path, poolMountPath+"/") scanner := bufio.NewScanner(&stdout) for scanner.Scan() { fields := strings.Fields(scanner.Text()) if len(fields) != 9 { continue } if !strings.HasPrefix(fields[8], path) { continue } result = append(result, strings.TrimPrefix(fields[8], path)) } } return result, nil } // snapshotSubvolume creates a snapshot of the specified path at the dest supplied. If recursion is true and // sub volumes are found below the path then they are created at the relative location in dest. func (d *btrfs) snapshotSubvolume(path string, dest string, recursion bool) (revert.Hook, error) { reverter := revert.New() defer reverter.Fail() // Single subvolume creation. snapshot := func(path string, dest string) error { _, err := subprocess.RunCommand("btrfs", "subvolume", "snapshot", path, dest) if err != nil { return err } reverter.Add(func() { // Don't delete recursive since there already is a revert hook // for each subvolume that got created. _ = d.deleteSubvolume(dest, false) }) return nil } // First snapshot the root. err := snapshot(path, dest) if err != nil { return nil, err } // Now snapshot all subvolumes of the root. if recursion { // Get the subvolumes list. subSubVols, err := d.getSubvolumes(path) if err != nil { return nil, err } sort.Strings(subSubVols) for _, subSubVol := range subSubVols { subSubVolSnapPath := filepath.Join(dest, subSubVol) // Clear the target for the subvol to use. _ = os.Remove(subSubVolSnapPath) err := snapshot(filepath.Join(path, subSubVol), subSubVolSnapPath) if err != nil { return nil, err } } } cleanup := reverter.Clone().Fail reverter.Success() return cleanup, nil } func (d *btrfs) deleteSubvolume(rootPath string, recursion bool) error { // Single subvolume deletion. destroy := func(path string) error { // Attempt (but don't fail on) to delete any qgroup on the subvolume. qgroup, _, err := d.getQGroup(path) if err == nil { _, _ = subprocess.RunCommand("btrfs", "qgroup", "destroy", qgroup, path) } // Temporarily change ownership & mode to help with nesting. _ = os.Chmod(path, 0o700) _ = os.Chown(path, 0, 0) // Delete the subvolume itself. _, err = subprocess.RunCommand("btrfs", "subvolume", "delete", path) return err } // Try and ensure volume is writable to possibility of destroy failing. err := d.setSubvolumeReadonlyProperty(rootPath, false) if err != nil { d.logger.Warn("Failed setting subvolume writable", logger.Ctx{"path": rootPath, "err": err}) } // Attempt to delete the root subvol itself (short path). err = destroy(rootPath) if err == nil { return nil } else if !recursion { return fmt.Errorf("Failed deleting subvolume %q: %w", rootPath, err) } // Delete subsubvols as recursion enabled. // Get the subvolumes list. subSubVols, err := d.getSubvolumes(rootPath) if err != nil { return err } // Perform a first pass and ensure all sub volumes are writable. sort.Sort(sort.StringSlice(subSubVols)) for _, subSubVol := range subSubVols { subSubVolPath := filepath.Join(rootPath, subSubVol) err = d.setSubvolumeReadonlyProperty(subSubVolPath, false) if err != nil { d.logger.Warn("Failed setting subvolume writable", logger.Ctx{"path": subSubVolPath, "err": err}) } } // Perform a second pass to delete subvolumes. sort.Sort(sort.Reverse(sort.StringSlice(subSubVols))) for _, subSubVol := range subSubVols { subSubVolPath := filepath.Join(rootPath, subSubVol) err := destroy(subSubVolPath) if err != nil { return fmt.Errorf("Failed deleting subvolume %q: %w", subSubVolPath, err) } } // Delete the root subvol itself. err = destroy(rootPath) if err != nil { return fmt.Errorf("Failed deleting subvolume %q: %w", rootPath, err) } return nil } func (d *btrfs) getQGroup(path string) (string, int64, error) { // Try to get the qgroup details. output, err := subprocess.RunCommand("btrfs", "qgroup", "show", "-e", "-f", "--raw", path) if err != nil { return "", -1, errBtrfsNoQuota } // Parse to extract the qgroup identifier. var qgroup string usage := int64(-1) for _, line := range strings.Split(output, "\n") { // Use case-insensitive field title match because BTRFS tooling changed casing between versions. if line == "" || strings.HasPrefix(strings.ToLower(line), "qgroupid") || strings.HasPrefix(line, "-") { continue } fields := strings.Fields(line) // The BTRFS tooling changed the number of columns between versions so we only check for minimum. if len(fields) < 3 { continue } qgroup = fields[0] val, err := strconv.ParseInt(fields[2], 10, 64) if err == nil { usage = val } break } if qgroup == "" { return "", -1, errBtrfsNoQGroup } return qgroup, usage, nil } func (d *btrfs) sendSubvolume(path string, parent string, conn io.ReadWriteCloser, tracker *ioprogress.ProgressTracker) error { defer func() { _ = conn.Close() }() // Assemble btrfs send command. args := []string{"send"} if parent != "" { args = append(args, "-p", parent) } args = append(args, path) cmd := exec.Command("btrfs", args...) stderr, err := cmd.StderrPipe() if err != nil { return err } // Setup progress tracker. var stdout io.WriteCloser = conn if tracker != nil { stdout = &ioprogress.ProgressWriter{ WriteCloser: conn, Tracker: tracker, } } cmd.Stdout = stdout // Run the command. err = cmd.Start() if err != nil { return err } // Read any error. output, err := io.ReadAll(stderr) if err != nil { logger.Errorf("Problem reading btrfs send stderr: %s", err) } err = cmd.Wait() if err != nil { return fmt.Errorf("Btrfs send failed: %w (%s)", err, string(output)) } return nil } // setSubvolumeReadonlyProperty sets the readonly property on the subvolume to true or false. func (d *btrfs) setSubvolumeReadonlyProperty(path string, readonly bool) error { // Silently ignore requests to set subvolume readonly property if running in a user namespace as we won't // be able to change it if it is readonly already, and making it readonly will mean we cannot undo it. if d.state.OS.RunningInUserNS { return nil } args := []string{"property", "set", "-f", "-ts", path, "ro", fmt.Sprintf("%t", readonly)} _, err := subprocess.RunCommand("btrfs", args...) return err } // BTRFSSubVolume is the structure used to store information about a subvolume. // Note: This is used by both migration and backup subsystems so do not modify without considering both! type BTRFSSubVolume struct { Path string `json:"path" yaml:"path"` // Path inside the volume where the subvolume belongs (so / is the top of the volume tree). Snapshot string `json:"snapshot" yaml:"snapshot"` // Snapshot name the subvolume belongs to. Readonly bool `json:"readonly" yaml:"readonly"` // Is the sub volume read only or not. UUID string `json:"uuid" yaml:"uuid"` // The subvolume UUID. } // getSubvolumesMetaData retrieves subvolume meta data with paths relative to the root volume. // The first item in the returned list is the root subvolume itself. func (d *btrfs) getSubvolumesMetaData(vol Volume) ([]BTRFSSubVolume, error) { var subVols []BTRFSSubVolume snapName := "" if vol.IsSnapshot() { _, snapName, _ = api.GetParentAndSnapshotName(vol.name) } // Add main root volume to subvolumes list first. subVols = append(subVols, BTRFSSubVolume{ Snapshot: snapName, Path: string(filepath.Separator), Readonly: BTRFSSubVolumeIsRo(vol.MountPath()), }) // Find any subvolumes in volume. subVolPaths, err := d.getSubvolumes(vol.MountPath()) if err != nil { return nil, err } sort.Strings(subVolPaths) // Add any subvolumes under the root subvolume with relative path to root. for _, subVolPath := range subVolPaths { subVols = append(subVols, BTRFSSubVolume{ Snapshot: snapName, Path: fmt.Sprintf("%s%s", string(filepath.Separator), subVolPath), Readonly: BTRFSSubVolumeIsRo(filepath.Join(vol.MountPath(), subVolPath)), }) } stdout := strings.Builder{} poolMountPath := GetPoolMountPath(vol.pool) if !d.state.OS.RunningInUserNS { // List all subvolumes in the given filesystem with their UUIDs and received UUIDs. err = subprocess.RunCommandWithFds(context.TODO(), nil, &stdout, "btrfs", "subvolume", "list", "-u", "-R", poolMountPath) if err != nil { return nil, err } uuidMap := make(map[string]string) receivedUUIDMap := make(map[string]string) scanner := bufio.NewScanner(strings.NewReader(stdout.String())) for scanner.Scan() { fields := strings.Fields(scanner.Text()) if len(fields) != 13 { continue } uuidMap[filepath.Join(poolMountPath, fields[12])] = fields[10] if fields[8] != "-" { receivedUUIDMap[filepath.Join(poolMountPath, fields[12])] = fields[8] } } for i, subVol := range subVols { subVols[i].UUID = uuidMap[filepath.Join(vol.MountPath(), subVol.Path)] } } return subVols, nil } func (d *btrfs) getSubVolumeReceivedUUID(vol Volume) (string, error) { stdout := strings.Builder{} poolMountPath := GetPoolMountPath(vol.pool) // List all subvolumes in the given filesystem with their UUIDs. err := subprocess.RunCommandWithFds(context.TODO(), nil, &stdout, "btrfs", "subvolume", "list", "-R", poolMountPath) if err != nil { return "", err } scanner := bufio.NewScanner(strings.NewReader(stdout.String())) for scanner.Scan() { fields := strings.Fields(scanner.Text()) if len(fields) != 11 { continue } if vol.MountPath() == filepath.Join(poolMountPath, fields[10]) && fields[8] != "-" { return fields[8], nil } } return "", nil } // BTRFSMetaDataHeader is the meta data header about the volumes being sent/stored. // Note: This is used by both migration and backup subsystems so do not modify without considering both! type BTRFSMetaDataHeader struct { Subvolumes []BTRFSSubVolume `json:"subvolumes" yaml:"subvolumes"` // Sub volumes inside the volume (including the top level ones). } // restorationHeader scans the volume and any specified snapshots, returning a header containing subvolume metadata // for use in restoring a volume and its snapshots onto another system. The metadata returned represents how the // subvolumes should be restored, not necessarily how they are on disk now. Most of the time this is the same, // however in circumstances where the volume being scanned is itself a snapshot, the returned metadata will // not report the volume as readonly or as being a snapshot, as the expectation is that this volume will be // restored on the target system as a normal volume and not a snapshot. func (d *btrfs) restorationHeader(vol Volume, snapshots []string) (*BTRFSMetaDataHeader, error) { var migrationHeader BTRFSMetaDataHeader // Add snapshots to volumes list. for _, snapName := range snapshots { snapVol, _ := vol.NewSnapshot(snapName) // Add snapshot root volume to volumes list. subVols, err := d.getSubvolumesMetaData(snapVol) if err != nil { return nil, err } migrationHeader.Subvolumes = append(migrationHeader.Subvolumes, subVols...) } // Add main root volume to volumes list. subVols, err := d.getSubvolumesMetaData(vol) if err != nil { return nil, err } // If vol is a snapshot itself, we force the volume as writable (even if it isn't on disk) and remove the // snapshot name indicator as the expectation is that this volume is going to be restored on the target // system as a normal (non-snapshot) writable volume. if vol.IsSnapshot() { subVols[0].Readonly = false for i := range subVols { subVols[i].Snapshot = "" } } migrationHeader.Subvolumes = append(migrationHeader.Subvolumes, subVols...) return &migrationHeader, nil } // loadOptimizedBackupHeader extracts optimized backup header from a given ReadSeeker. func (d *btrfs) loadOptimizedBackupHeader(r io.ReadSeeker, mountPath string, basePrefix string) (*BTRFSMetaDataHeader, error) { header := BTRFSMetaDataHeader{} // Extract. tr, cancelFunc, err := backup.TarReader(r, d.state.OS, mountPath) if err != nil { return nil, err } defer cancelFunc() for { hdr, err := tr.Next() if err == io.EOF { break // End of archive. } if err != nil { return nil, fmt.Errorf("Error reading backup file for optimized backup header file: %w", err) } if hdr.Name == filepath.Join(basePrefix, "optimized_header.yaml") { loader, err := yaml.NewLoader(localUtil.MaxBytesReader(tr, 1024*1024)) if err != nil { return nil, fmt.Errorf("Error parsing optimized backup header file: %w", err) } err = loader.Load(&header) if err != nil { return nil, fmt.Errorf("Error parsing optimized backup header file: %w", err) } cancelFunc() return &header, nil } } return nil, errors.New("Optimized backup header file not found") } // receiveSubVolume receives a subvolume from an io.Reader into the receivePath and returns the path to the received subvolume. func (d *btrfs) receiveSubVolume(r io.Reader, receivePath string, tracker *ioprogress.ProgressTracker) (string, error) { files, err := os.ReadDir(receivePath) if err != nil { return "", fmt.Errorf("Failed listing contents of %q: %w", receivePath, err) } // Setup progress tracker. var stdin io.Reader = r if tracker != nil { stdin = &ioprogress.ProgressReader{ Reader: r, Tracker: tracker, } } err = subprocess.RunCommandWithFds(context.TODO(), stdin, nil, "btrfs", "receive", "-e", receivePath) if err != nil { return "", err } // Check contents of target path is expected after receive. newFiles, err := os.ReadDir(receivePath) if err != nil { return "", fmt.Errorf("Failed listing contents of %q: %w", receivePath, err) } filename := "" // Identify the latest received path. for _, a := range newFiles { found := false for _, b := range files { if a.Name() == b.Name() { found = true break } } if !found { filename = a.Name() break } } if filename == "" { return "", errors.New("Failed to determine received subvolume") } subVolPath := filepath.Join(receivePath, filename) return subVolPath, nil } incus-7.0.0/internal/server/storage/drivers/driver_btrfs_volumes.go000066400000000000000000002010751517523235500257140ustar00rootroot00000000000000package drivers import ( "bufio" "bytes" "context" "encoding/json" "errors" "fmt" "io" "os" "os/exec" "path/filepath" "slices" "strings" "time" "github.com/google/uuid" "go.yaml.in/yaml/v4" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/server/backup" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // CreateVolume creates an empty volume and can optionally fill it by executing the supplied filler function. func (d *btrfs) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Operation) error { volPath := vol.MountPath() // Setup revert. reverter := revert.New() defer reverter.Fail() // Create the volume itself. _, err := subprocess.RunCommand("btrfs", "subvolume", "create", volPath) if err != nil { return err } reverter.Add(func() { _ = d.deleteSubvolume(volPath, false) _ = os.Remove(volPath) }) // Create sparse loopback file if volume is block. rootBlockPath := "" if IsContentBlock(vol.contentType) { // We expect the filler to copy the VM image into this path. rootBlockPath, err = d.GetVolumeDiskPath(vol) if err != nil { return err } // Get underlying btrfs mount options. mountinfo, err := linux.GetMountinfo(volPath) if err != nil { return err } mountOptions := strings.Split(d.getMountOptions(), ",") // Enable nodatacow on the parent directory so that when the root disk file is created the setting // is inherited and random writes don't cause fragmentation and old extents to be kept. // BTRFS extents are immutable so when blocks are written they end up in new extents and the old // ones remains until all of its data is dereferenced or rewritten. These old extents are counted // in the quota, and so leaving CoW enabled can cause the BTRFS subvolume quota to be reached even // before the block file itself is full. This setting does not totally prevents CoW from happening // as when a snapshot is taken, writes that happen on the original volume necessarily create a CoW // in order to track the difference between original and snapshot. This will increase the size of // data being referenced. // // An exception is made for when compression is enabled on the underlying storage. if !slices.Contains(mountOptions, "datacow") && !strings.Contains(mountinfo[len(mountinfo)-1], "compress") { _, err = subprocess.RunCommand("chattr", "+C", volPath) if err != nil { return fmt.Errorf("Failed setting nodatacow on %q: %w", volPath, err) } } } err = genericRunFiller(d, vol, rootBlockPath, filler, false) if err != nil { return err } // If we are creating a block volume, resize it to the requested size or the default. // We expect the filler function to have converted the qcow2 image to raw into the rootBlockPath. if IsContentBlock(vol.contentType) { // Convert to bytes. sizeBytes, err := units.ParseByteSizeString(vol.ConfigSize()) if err != nil { return err } // Normally we pass VolumeTypeImage to unsupportedResizeTypes to ensureVolumeBlockFile as block // image volumes cannot be resized because they have a readonly snapshot which the instances are // created from, and that doesn't get updated when the original volumes size is changed. // However during initial volume fill we allow growing of image volumes because the snapshot hasn't // been taken yet. This is why no unsupported volume types are passed to ensureVolumeBlockFile. // This is important, as combined with not enabling allowUnsafeResize it still prevents us from // accidentally shrinking the filled volume if it is larger than vol.ConfigSize(). // In that situation ensureVolumeBlockFile returns ErrCannotBeShrunk, but we ignore it as this just // means the filler run above has needed to increase the volume size beyond the default block // volume size. _, err = ensureVolumeBlockFile(vol, rootBlockPath, sizeBytes, false) if err != nil && !errors.Is(err, ErrCannotBeShrunk) { return err } // Move the GPT alt header to end of disk if needed and if filler specified. if vol.IsVMBlock() && filler != nil && filler.Fill != nil { err = d.moveGPTAltHeader(rootBlockPath) if err != nil { return err } } } else if vol.contentType == ContentTypeFS { // Set initial quota for filesystem volumes. err := d.SetVolumeQuota(vol, vol.ConfigSize(), false, op) if err != nil { return err } } // Tweak any permissions that need tweaking after filling. err = vol.EnsureMountPath(true) if err != nil { return err } // Attempt to mark image read-only. if vol.volType == VolumeTypeImage { err = d.setSubvolumeReadonlyProperty(volPath, true) if err != nil { return err } } reverter.Success() return nil } // CreateVolumeFromBackup restores a backup tarball onto the storage device. func (d *btrfs) CreateVolumeFromBackup(vol Volume, srcBackup backup.Info, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) (VolumePostHook, revert.Hook, error) { // Handle the non-optimized tarballs through the generic unpacker. if !*srcBackup.OptimizedStorage { return genericVFSBackupUnpack(d, d.state.OS, vol, srcBackup.Snapshots, srcData, basePrefix, op) } volExists, err := d.HasVolume(vol) if err != nil { return nil, nil, err } if volExists { return nil, nil, errors.New("Cannot restore volume, already exists on target") } reverter := revert.New() defer reverter.Fail() // Define a revert function that will be used both to revert if an error occurs inside this // function but also return it for use from the calling functions if no error internally. revertHook := func() { for _, snapName := range srcBackup.Snapshots { fullSnapshotName := GetSnapshotVolumeName(vol.name, snapName) snapVol := NewVolume(d, d.name, vol.volType, vol.contentType, fullSnapshotName, vol.config, vol.poolConfig) _ = d.DeleteVolumeSnapshot(snapVol, op) } // And lastly the main volume. _ = d.DeleteVolume(vol, op) } // Only execute the revert function if we have had an error internally. reverter.Add(revertHook) // Find the compression algorithm used for backup source data. _, err = srcData.Seek(0, io.SeekStart) if err != nil { return nil, nil, err } _, _, unpacker, err := archive.DetectCompressionFile(srcData) if err != nil { return nil, nil, err } // Load optimized backup header file if specified. var optimizedHeader *BTRFSMetaDataHeader if *srcBackup.OptimizedHeader { optimizedHeader, err = d.loadOptimizedBackupHeader(srcData, GetVolumeMountPath(d.name, vol.volType, ""), basePrefix) if err != nil { return nil, nil, err } } // Populate optimized header with pseudo data for unified handling when backup doesn't contain the // optimized header file. This approach can only be used to restore root subvolumes (not sub-subvolumes). if optimizedHeader == nil { optimizedHeader = &BTRFSMetaDataHeader{} for _, snapName := range srcBackup.Snapshots { optimizedHeader.Subvolumes = append(optimizedHeader.Subvolumes, BTRFSSubVolume{ Snapshot: snapName, Path: string(filepath.Separator), Readonly: true, // Snapshots are made readonly. }) } optimizedHeader.Subvolumes = append(optimizedHeader.Subvolumes, BTRFSSubVolume{ Snapshot: "", Path: string(filepath.Separator), Readonly: false, }) } // Create a temporary directory to unpack the backup into. tmpUnpackDir, err := os.MkdirTemp(GetVolumeMountPath(d.name, vol.volType, ""), "backup.") if err != nil { return nil, nil, fmt.Errorf("Failed to create temporary directory %q: %w", tmpUnpackDir, err) } defer func() { _ = os.RemoveAll(tmpUnpackDir) }() err = os.Chmod(tmpUnpackDir, 0o100) if err != nil { return nil, nil, fmt.Errorf("Failed to chmod temporary directory %q: %w", tmpUnpackDir, err) } // unpackSubVolume unpacks a subvolume file from a backup tarball file. unpackSubVolume := func(r io.ReadSeeker, unpacker []string, srcFile string, targetPath string) (string, error) { tr, cancelFunc, err := archive.CompressedTarReader(context.Background(), r, unpacker, targetPath) if err != nil { return "", err } defer cancelFunc() for { hdr, err := tr.Next() if err == io.EOF { break // End of archive. } if err != nil { return "", err } if hdr.Name == srcFile { subVolRecvPath, err := d.receiveSubVolume(tr, targetPath, nil) if err != nil { return "", err } cancelFunc() return subVolRecvPath, nil } } return "", fmt.Errorf("Could not find %q", srcFile) } type btrfsCopyOp struct { src string dest string } var copyOps []btrfsCopyOp // unpackVolume unpacks all subvolumes in a volume from a backup tarball file. unpackVolume := func(v Volume, srcFilePrefix string) error { _, snapName, _ := api.GetParentAndSnapshotName(v.name) for _, subVol := range optimizedHeader.Subvolumes { if subVol.Snapshot != snapName { continue // Skip any subvolumes that dont belong to our volume (empty for main). } // Figure out what file we are looking for in the backup file. srcFilePath := filepath.Join(basePrefix, fmt.Sprintf("%s.bin", srcFilePrefix)) if subVol.Path != string(filepath.Separator) { // If subvolume is non-root, then we expect the file to be encoded as its original // path with the leading / removed. srcFilePath = filepath.Join(basePrefix, fmt.Sprintf("%s_%s.bin", srcFilePrefix, linux.PathNameEncode(strings.TrimPrefix(subVol.Path, string(filepath.Separator))))) } // Define where we will move the subvolume after it is unpacked. subVolTargetPath := filepath.Join(v.MountPath(), subVol.Path) tmpUnpackDir := filepath.Join(tmpUnpackDir, snapName) err := os.MkdirAll(tmpUnpackDir, 0o100) if err != nil { return fmt.Errorf("Failed creating directory %q: %w", tmpUnpackDir, err) } d.Logger().Debug("Unpacking optimized volume", logger.Ctx{"name": v.name, "source": srcFilePath, "unpackPath": tmpUnpackDir, "path": subVolTargetPath}) // Unpack the volume into the temporary unpackDir. unpackedSubVolPath, err := unpackSubVolume(srcData, unpacker, srcFilePath, tmpUnpackDir) if err != nil { return err } copyOps = append(copyOps, btrfsCopyOp{ src: unpackedSubVolPath, dest: subVolTargetPath, }) } return nil } if len(srcBackup.Snapshots) > 0 { // Create new snapshots directory. err := CreateParentSnapshotDirIfMissing(d.name, vol.volType, vol.name) if err != nil { return nil, nil, err } // Restore backup snapshots from oldest to newest. for _, snapName := range srcBackup.Snapshots { snapVol, _ := vol.NewSnapshot(snapName) snapDir := "snapshots" srcFilePrefix := snapName if vol.volType == VolumeTypeVM { snapDir = "virtual-machine-snapshots" if vol.contentType == ContentTypeFS { srcFilePrefix = fmt.Sprintf("%s-config", snapName) } } else if vol.volType == VolumeTypeCustom { snapDir = "volume-snapshots" } srcFilePrefix = filepath.Join(snapDir, srcFilePrefix) err = unpackVolume(snapVol, srcFilePrefix) if err != nil { return nil, nil, err } } } // Extract main volume. srcFilePrefix := "container" if vol.volType == VolumeTypeVM { if vol.contentType == ContentTypeFS { srcFilePrefix = "virtual-machine-config" } else { srcFilePrefix = "virtual-machine" } } else if vol.volType == VolumeTypeCustom { srcFilePrefix = "volume" } err = unpackVolume(vol, srcFilePrefix) if err != nil { return nil, nil, err } for _, copyOp := range copyOps { err = d.setSubvolumeReadonlyProperty(copyOp.src, false) if err != nil { return nil, nil, err } // Clear the target for the subvol to use. _ = os.Remove(copyOp.dest) // Move unpacked subvolume into its final location. err = os.Rename(copyOp.src, copyOp.dest) if err != nil { return nil, nil, err } } // Restore readonly property on subvolumes that need it. for _, subVol := range optimizedHeader.Subvolumes { if !subVol.Readonly { continue // All subvolumes are made writable during unpack process so we can skip these. } v := vol if subVol.Snapshot != "" { v, _ = vol.NewSnapshot(subVol.Snapshot) } path := filepath.Join(v.MountPath(), subVol.Path) d.logger.Debug("Setting subvolume readonly", logger.Ctx{"name": v.name, "path": path}) err = d.setSubvolumeReadonlyProperty(path, true) if err != nil { return nil, nil, err } } reverter.Success() return nil, revertHook, nil } // CreateVolumeFromCopy provides same-pool volume copying functionality. func (d *btrfs) CreateVolumeFromCopy(vol Volume, srcVol Volume, copySnapshots bool, allowInconsistent bool, op *operations.Operation) error { reverter := revert.New() defer reverter.Fail() // Scan source for subvolumes (so we can apply the readonly properties on the new volume). subVols, err := d.getSubvolumesMetaData(srcVol) if err != nil { return err } target := vol.MountPath() // Recursively copy the main volume. cleanup, err := d.snapshotSubvolume(srcVol.MountPath(), target, true) if err != nil { return err } if cleanup != nil { reverter.Add(cleanup) } // Restore readonly property on subvolumes in reverse order (except root which should be left writable). subVolCount := len(subVols) for i := range subVols { i = subVolCount - 1 - i subVol := subVols[i] if subVol.Readonly && subVol.Path != string(filepath.Separator) { targetSubVolPath := filepath.Join(target, subVol.Path) err = d.setSubvolumeReadonlyProperty(targetSubVolPath, true) if err != nil { return err } } } // Resize volume to the size specified. Only uses volume "size" property and does not use pool/defaults // to give the caller more control over the size being used. err = d.SetVolumeQuota(vol, vol.config["size"], false, op) if err != nil { return err } // Fixup permissions after snapshot created. err = vol.EnsureMountPath(false) if err != nil { return err } var snapshots []string // Get snapshot list if copying snapshots. if copySnapshots && !srcVol.IsSnapshot() { // Get the list of snapshots. snapshots, err = d.VolumeSnapshots(srcVol, op) if err != nil { return err } } // Copy any snapshots needed. if len(snapshots) > 0 { // Create the parent directory. err = CreateParentSnapshotDirIfMissing(d.name, vol.volType, vol.name) if err != nil { return err } // Copy the snapshots. for _, snapName := range snapshots { srcSnapshot := GetVolumeMountPath(d.name, srcVol.volType, GetSnapshotVolumeName(srcVol.name, snapName)) dstSnapshot := GetVolumeMountPath(d.name, vol.volType, GetSnapshotVolumeName(vol.name, snapName)) cleanup, err := d.snapshotSubvolume(srcSnapshot, dstSnapshot, true) if err != nil { return err } if cleanup != nil { reverter.Add(cleanup) } err = d.setSubvolumeReadonlyProperty(dstSnapshot, true) if err != nil { return err } reverter.Add(func() { _ = d.deleteSubvolume(dstSnapshot, true) }) } } reverter.Success() return nil } // CreateVolumeFromMigration creates a volume being sent via a migration. func (d *btrfs) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, volTargetArgs localMigration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error { // Handle simple rsync and block_and_rsync through generic. if volTargetArgs.MigrationType.FSType == migration.MigrationFSType_RSYNC || volTargetArgs.MigrationType.FSType == migration.MigrationFSType_BLOCK_AND_RSYNC { return genericVFSCreateVolumeFromMigration(d, nil, vol, conn, volTargetArgs, preFiller, op) } else if volTargetArgs.MigrationType.FSType != migration.MigrationFSType_BTRFS { return ErrNotSupported } var migrationHeader BTRFSMetaDataHeader // List of subvolumes to be synced. This is sent back to the source. var syncSubvolumes []BTRFSSubVolume // Inspect negotiated features to see if we are expecting to get a metadata migration header frame. if slices.Contains(volTargetArgs.MigrationType.Features, migration.BTRFSFeatureMigrationHeader) { buf, err := io.ReadAll(conn) if err != nil { return fmt.Errorf("Failed reading BTRFS migration header: %w", err) } err = json.Unmarshal(buf, &migrationHeader) if err != nil { return fmt.Errorf("Failed decoding BTRFS migration header: %w", err) } d.logger.Debug("Received BTRFS migration meta data header", logger.Ctx{"name": vol.name}) } else { // Populate the migrationHeader subvolumes with root volumes only to support older sources. for _, snapshot := range volTargetArgs.Snapshots { migrationHeader.Subvolumes = append(migrationHeader.Subvolumes, BTRFSSubVolume{ Snapshot: snapshot.GetName(), Path: string(filepath.Separator), Readonly: true, // Snapshots are made readonly. }) } migrationHeader.Subvolumes = append(migrationHeader.Subvolumes, BTRFSSubVolume{ Snapshot: "", Path: string(filepath.Separator), Readonly: false, }) } if volTargetArgs.Refresh && slices.Contains(volTargetArgs.MigrationType.Features, migration.BTRFSFeatureSubvolumeUUIDs) { snapshots, err := d.volumeSnapshotsSorted(vol, op) if err != nil { return err } // Reset list of snapshots which are to be received. volTargetArgs.Snapshots = []*migration.Snapshot{} // Map of local subvolumes with their received UUID. localSubvolumes := make(map[string]string) for _, snap := range snapshots { snapVol, _ := vol.NewSnapshot(snap) receivedUUID, err := d.getSubVolumeReceivedUUID(snapVol) if err != nil { return err } localSubvolumes[snap] = receivedUUID } // Figure out which snapshots need to be copied by comparing the UUIDs and received UUIDs from the migration header. for _, migrationSnap := range migrationHeader.Subvolumes { receivedUUID, ok := localSubvolumes[migrationSnap.Snapshot] // Skip this snapshot as it exists on both the source and target, and has the same GUID. if ok && receivedUUID == migrationSnap.UUID { continue } if migrationSnap.Path == "/" && migrationSnap.Snapshot != "" { volTargetArgs.Snapshots = append(volTargetArgs.Snapshots, &migration.Snapshot{Name: &migrationSnap.Snapshot}) } syncSubvolumes = append(syncSubvolumes, BTRFSSubVolume{Path: migrationSnap.Path, Snapshot: migrationSnap.Snapshot, UUID: migrationSnap.UUID}) } migrationHeader = BTRFSMetaDataHeader{Subvolumes: syncSubvolumes} headerJSON, err := json.Marshal(migrationHeader) if err != nil { return fmt.Errorf("Failed encoding BTRFS migration header: %w", err) } _, err = conn.Write(headerJSON) if err != nil { return fmt.Errorf("Failed sending BTRFS migration header: %w", err) } err = conn.Close() // End the frame. if err != nil { return fmt.Errorf("Failed closing BTRFS migration header frame: %w", err) } d.logger.Debug("Sent BTRFS migration meta data header", logger.Ctx{"name": vol.name, "header": migrationHeader}) } else { syncSubvolumes = migrationHeader.Subvolumes } return d.createVolumeFromMigrationOptimized(vol, conn, volTargetArgs, preFiller, syncSubvolumes, op) } func (d *btrfs) createVolumeFromMigrationOptimized(vol Volume, conn io.ReadWriteCloser, volTargetArgs localMigration.VolumeTargetArgs, preFiller *VolumeFiller, subvolumes []BTRFSSubVolume, op *operations.Operation) error { reverter := revert.New() defer reverter.Fail() type btrfsCopyOp struct { src string dest string receivedUUID string } // copyOps represents copy operations which need to take place once *all* subvolumes have been // received. We don't use a map as the order should be kept. copyOps := []btrfsCopyOp{} // receiveVolume receives all subvolumes in a volume from the source. receiveVolume := func(v Volume, receivePath string) error { _, snapName, _ := api.GetParentAndSnapshotName(v.name) // Setup progress tracking. var wrapper *ioprogress.ProgressTracker if volTargetArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", v.name) } for _, subVol := range subvolumes { if subVol.Snapshot != snapName { continue // Skip any subvolumes that dont belong to our volume (empty for main). } receivePath := filepath.Join(receivePath, snapName) err := os.MkdirAll(receivePath, 0o100) if err != nil { return fmt.Errorf("Failed creating %q: %w", receivePath, err) } subVolTargetPath := filepath.Join(v.MountPath(), subVol.Path) d.logger.Debug("Receiving volume", logger.Ctx{"name": v.name, "receivePath": receivePath, "path": subVolTargetPath}) subVolRecvPath, err := d.receiveSubVolume(conn, receivePath, wrapper) if err != nil { return err } receivedVol := Volume{ pool: d.name, mountCustomPath: subVolRecvPath, } UUID, err := d.getSubVolumeReceivedUUID(receivedVol) if err != nil { return fmt.Errorf("Failed getting UUID: %w", err) } // Record the copy operations we need to do after having received all subvolumes. copyOps = append(copyOps, btrfsCopyOp{ src: subVolRecvPath, dest: subVolTargetPath, receivedUUID: UUID, }) } return nil } // Get instances directory (e.g. /var/lib/incus/storage-pools/btrfs/containers). instancesPath := GetVolumeMountPath(d.name, vol.volType, "") // Create a temporary directory which will act as the parent directory of the received ro snapshot. tmpVolumesMountPoint, err := os.MkdirTemp(instancesPath, "migration.") if err != nil { return fmt.Errorf("Failed to create temporary directory under %q: %w", instancesPath, err) } defer func() { _ = os.RemoveAll(tmpVolumesMountPoint) }() err = os.Chmod(tmpVolumesMountPoint, 0o100) if err != nil { return fmt.Errorf("Failed to chmod %q: %w", tmpVolumesMountPoint, err) } // Handle btrfs send/receive migration. if !volTargetArgs.VolumeOnly && len(volTargetArgs.Snapshots) > 0 { // Create the parent directory. err := CreateParentSnapshotDirIfMissing(d.name, vol.volType, vol.name) if err != nil { return err } reverter.Add(func() { _ = deleteParentSnapshotDirIfEmpty(d.name, vol.volType, vol.name) }) // Transfer the snapshots. for _, snapshot := range volTargetArgs.Snapshots { snapVol, _ := vol.NewSnapshot(snapshot.GetName()) err = receiveVolume(snapVol, tmpVolumesMountPoint) if err != nil { return err } } } // Receive main volume. err = receiveVolume(vol, tmpVolumesMountPoint) if err != nil { return err } if volTargetArgs.Refresh { // Delete main volume after receiving it. err = d.deleteSubvolume(vol.MountPath(), true) if err != nil { return err } } // Make all received subvolumes read-write and move them to their final destination for _, op := range copyOps { err = d.setSubvolumeReadonlyProperty(op.src, false) if err != nil { return err } // Clear the target for the subvol to use. _ = os.Remove(op.dest) err = os.Rename(op.src, op.dest) if err != nil { return err } // This sets the "Received UUID" field on the subvolume. // When making the received subvolume read-write before moving it to its final location, // this information is lost (by design). However, this causes issues when performing // incremental streams (error: "cannot find parent subvolume"). // Setting the "Received UUID" field to the value of the received subvolume (before making // it rw) solves this issue. err = setReceivedUUID(op.dest, op.receivedUUID) if err != nil { return fmt.Errorf("Failed setting received UUID: %w", err) } } // Restore readonly property on subvolumes that need it. for _, subVol := range subvolumes { if !subVol.Readonly { continue // All subvolumes are made writable during receive process so we can skip these. } v := vol if subVol.Snapshot != "" { v, _ = vol.NewSnapshot(subVol.Snapshot) } path := filepath.Join(v.MountPath(), subVol.Path) d.logger.Debug("Setting subvolume readonly", logger.Ctx{"name": v.name, "path": path}) err = d.setSubvolumeReadonlyProperty(path, true) if err != nil { return err } } if vol.contentType == ContentTypeFS { // Apply the size limit. err = d.SetVolumeQuota(vol, vol.ConfigSize(), false, op) if err != nil { return err } } reverter.Success() return nil } // RefreshVolume provides same-pool volume and specific snapshots syncing functionality. func (d *btrfs) RefreshVolume(vol Volume, srcVol Volume, srcSnapshots []Volume, allowInconsistent bool, op *operations.Operation) error { // Get target snapshots targetSnapshots, err := d.volumeSnapshotsSorted(vol, op) if err != nil { return fmt.Errorf("Failed to get target snapshots: %w", err) } srcSnapshotsAll, err := d.volumeSnapshotsSorted(srcVol, op) if err != nil { return fmt.Errorf("Failed to get source snapshots: %w", err) } // Optimized refresh only makes sense if the source and target have at least one identical snapshot, // as btrfs can then use an incremental streams instead of just copying the datasets. if len(targetSnapshots) == 0 || len(srcSnapshotsAll) == 0 { d.logger.Debug("Performing generic volume refresh") return genericVFSCopyVolume(d, nil, vol, srcVol, srcSnapshots, true, false, op) } d.logger.Debug("Performing optimized volume refresh") transfer := func(src Volume, target Volume, origin Volume) error { var sender *exec.Cmd srcSubvolPath := filepath.Join(GetPoolMountPath(src.pool), fmt.Sprintf("%s-snapshots/%s", src.volType, src.name)) targetSubvolPath := filepath.Join(GetPoolMountPath(target.pool), fmt.Sprintf("%s-snapshots/%s", target.volType, target.name)) originSubvolPath := filepath.Join(GetPoolMountPath(origin.pool), fmt.Sprintf("%s-snapshots/%s", origin.volType, origin.name)) receiver := exec.Command("btrfs", "receive", targetSubvolPath) sender = exec.Command("btrfs", "send", "-p", originSubvolPath, srcSubvolPath) // Configure the pipes. receiver.Stdin, _ = sender.StdoutPipe() receiver.Stdout = os.Stdout var recvStderr bytes.Buffer receiver.Stderr = &recvStderr var sendStderr bytes.Buffer sender.Stderr = &sendStderr // Run the transfer. err := receiver.Start() if err != nil { return fmt.Errorf("Failed starting BTRFS receive: %w", err) } err = sender.Start() if err != nil { _ = receiver.Process.Kill() return fmt.Errorf("Failed starting BTRFS send: %w", err) } senderErr := make(chan error) go func() { err := sender.Wait() if err != nil { _ = receiver.Process.Kill() // This removes any newlines in the error message. msg := strings.ReplaceAll(strings.TrimSpace(sendStderr.String()), "\n", " ") senderErr <- fmt.Errorf("Failed BTRFS send: %w (%s)", err, msg) return } senderErr <- nil }() err = receiver.Wait() if err != nil { _ = sender.Process.Kill() // This removes any newlines in the error message. msg := strings.ReplaceAll(strings.TrimSpace(recvStderr.String()), "\n", " ") return fmt.Errorf("Failed BTRFS receive: %w (%s)", err, msg) } err = <-senderErr if err != nil { return err } return nil } // Before refreshing a volume, all extra snapshots (those which exist on the target but // not on the source) are removed. Therefore, the last entry in targetSnapshots represents the // most recent identical snapshot of the source volume and target volume. lastIdenticalSnapshot := targetSnapshots[len(targetSnapshots)-1] for i, snap := range srcSnapshots { var srcSnap Volume if i == 0 { srcSnap, err = srcVol.NewSnapshot(lastIdenticalSnapshot) if err != nil { return fmt.Errorf("Failed to create new snapshot volume: %w", err) } } else { srcSnap = srcSnapshots[i-1] } err = transfer(snap, vol, srcSnap) if err != nil { return err } } // Create temporary snapshot of the source volume. snapUUID := uuid.New().String() srcSnap, err := srcVol.NewSnapshot(snapUUID) if err != nil { return err } err = d.CreateVolumeSnapshot(srcSnap, op) if err != nil { return err } // Transfer temporary snapshot to target; this creates a new snapshot for target. parentSnap, err := srcVol.NewSnapshot(srcSnapshotsAll[len(srcSnapshotsAll)-1]) if err != nil { return err } err = transfer(srcSnap, vol, parentSnap) if err != nil { return err } err = d.DeleteVolumeSnapshot(srcSnap, op) if err != nil { return err } err = d.deleteSubvolume(vol.MountPath(), false) if err != nil { return err } targetSnap, err := vol.NewSnapshot(snapUUID) if err != nil { return err } // Set readonly to false on the temporary snapshot, otherwise moving/renaming it won't be // possible. err = d.setSubvolumeReadonlyProperty(targetSnap.MountPath(), false) if err != nil { return err } // Rename temporary target snapshot to the actual target, // e.g. containers-snapshots/c2/ -> containers/c2. err = os.Rename(targetSnap.MountPath(), vol.MountPath()) if err != nil { return err } return nil } // DeleteVolume deletes a volume of the storage device. If any snapshots of the volume remain then // this function will return an error. func (d *btrfs) DeleteVolume(vol Volume, op *operations.Operation) error { // Check that we don't have snapshots. snapshots, err := d.VolumeSnapshots(vol, op) if err != nil { return err } if len(snapshots) > 0 { return errors.New("Cannot remove a volume that has snapshots") } volName := vol.name if vol.volType == VolumeTypeCustom && vol.contentType == ContentTypeISO { volName = fmt.Sprintf("%s%s", vol.name, btrfsISOVolSuffix) } // If the volume doesn't exist, then nothing more to do. volPath := GetVolumeMountPath(d.name, vol.volType, volName) if !util.PathExists(volPath) { return nil } // Delete the volume (and any subvolumes). err = d.deleteSubvolume(volPath, true) if err != nil { return err } // Although the volume snapshot directory should already be removed, lets remove it here // to just in case the top-level directory is left. err = deleteParentSnapshotDirIfEmpty(d.name, vol.volType, volName) if err != nil { return err } return nil } // HasVolume indicates whether a specific volume exists on the storage pool. func (d *btrfs) HasVolume(vol Volume) (bool, error) { return genericVFSHasVolume(vol) } // ValidateVolume validates the supplied volume config. func (d *btrfs) ValidateVolume(vol Volume, removeUnknownKeys bool) error { // gendoc:generate(entity=storage_volume_btrfs, group=common, key=initial.gid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.gid` or `0` // shortdesc: GID of the volume owner in the instance // gendoc:generate(entity=storage_volume_btrfs, group=common, key=initial.mode) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.mode` or `711` // shortdesc: Mode of the volume in the instance // gendoc:generate(entity=storage_volume_btrfs, group=common, key=initial.uid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.uid` or `0` // shortdesc: UID of the volume owner in the instance // gendoc:generate(entity=storage_volume_btrfs, group=common, key=security.shared) // // --- // type: bool // condition: custom block volume // default: same as `volume.security.shared` or `false` // shortdesc: Enable sharing the volume across multiple instances // gendoc:generate(entity=storage_volume_btrfs, group=common, key=security.shifted) // // --- // type: bool // condition: custom volume // default: same as `volume.security.shifted` or `false` // shortdesc: {{enable_ID_shifting}} // gendoc:generate(entity=storage_volume_btrfs, group=common, key=security.unmapped) // // --- // type: bool // condition: custom volume // default: same as `volume.security.unmapped` or `false` // shortdesc: Disable ID mapping for the volume // gendoc:generate(entity=storage_volume_btrfs, group=common, key=size) // // --- // type: string // condition: appropriate driver // default: same as `volume.size` // shortdesc: Size/quota of the storage volume // gendoc:generate(entity=storage_volume_btrfs, group=common, key=snapshots.expiry) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_btrfs, group=common, key=snapshots.expiry.manual) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry.manual` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_btrfs, group=common, key=snapshots.pattern) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.pattern` or `snap%d` // shortdesc: {{snapshot_pattern_format}} [^*] // gendoc:generate(entity=storage_volume_btrfs, group=common, key=snapshots.schedule) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.schedule` // shortdesc: {{snapshot_schedule_format}} // gendoc:generate(entity=storage_bucket_btrfs, group=common, key=size) // // --- // type: string // condition: appropriate driver // default: same as `volume.size` // shortdesc: Size/quota of the storage bucket return d.validateVolume(vol, nil, removeUnknownKeys) } // UpdateVolume applies config changes to the volume. func (d *btrfs) UpdateVolume(vol Volume, changedConfig map[string]string) error { newSize, sizeChanged := changedConfig["size"] if sizeChanged { err := d.SetVolumeQuota(vol, newSize, false, nil) if err != nil { return err } } return d.updateVolume(vol, changedConfig) } // GetVolumeUsage returns the disk space used by the volume. func (d *btrfs) GetVolumeUsage(vol Volume) (int64, error) { // Attempt to get the qgroup information. _, usage, err := d.getQGroup(vol.MountPath()) if err != nil { if errors.Is(err, errBtrfsNoQuota) { return -1, ErrNotSupported } return -1, err } return usage, nil } // SetVolumeQuota applies a size limit on volume. // Does nothing if supplied with an empty/zero size for block volumes, and for filesystem volumes removes quota. func (d *btrfs) SetVolumeQuota(vol Volume, size string, allowUnsafeResize bool, op *operations.Operation) error { // Convert to bytes. sizeBytes, err := units.ParseByteSizeString(size) if err != nil { return err } // For VM block files, resize the file if needed. if vol.contentType == ContentTypeBlock { // Do nothing if size isn't specified. if sizeBytes <= 0 { return nil } rootBlockPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } // Pass VolumeTypeImage as unsupported resize type, as if the image volume doesn't match the // requested size and allowUnsafeResize=false, this needs to be rejected back to caller as // ErrNotSupported so that the caller can take the appropriate action. In the case of optimized // image volumes, this will cause the image volume to be deleted and regenerated with the new size. // In other cases this is probably a bug and the operation should fail anyway. resized, err := ensureVolumeBlockFile(vol, rootBlockPath, sizeBytes, allowUnsafeResize, VolumeTypeImage) if err != nil { return err } // Move the GPT alt header to end of disk if needed and resize has taken place (not needed in // unsafe resize mode as it is expected the caller will do all necessary post resize actions // themselves). if vol.IsVMBlock() && resized && !allowUnsafeResize { err = d.moveGPTAltHeader(rootBlockPath) if err != nil { return err } } return nil } // For non-VM block volumes, set filesystem quota. volPath := vol.MountPath() // Try to locate an existing quota group. qgroup, _, err := d.getQGroup(volPath) if err != nil && !d.state.OS.RunningInUserNS { // If quotas are disabled, attempt to enable them. if errors.Is(err, errBtrfsNoQuota) { if sizeBytes <= 0 { // Nothing to do if the quota is being removed and we don't currently have quota. return nil } path := GetPoolMountPath(d.name) _, err = subprocess.RunCommand("btrfs", "quota", "enable", path) if err != nil { return err } // Try again. qgroup, _, err = d.getQGroup(volPath) } // If there's no qgroup, attempt to create one. if errors.Is(err, errBtrfsNoQGroup) { // Find the volume ID. var output string output, err = subprocess.RunCommand("btrfs", "subvolume", "show", volPath) if err != nil { return fmt.Errorf("Failed to get subvol information: %w", err) } id := "" for _, line := range strings.Split(output, "\n") { line = strings.TrimSpace(line) if strings.HasPrefix(line, "Subvolume ID:") { fields := strings.Split(line, ":") id = strings.TrimSpace(fields[len(fields)-1]) } } if id == "" { return fmt.Errorf("Failed to find subvolume id for %q", volPath) } // Create a qgroup. _, err = subprocess.RunCommand("btrfs", "qgroup", "create", fmt.Sprintf("0/%s", id), volPath) if err != nil { return err } // Try to get the qgroup again. qgroup, _, err = d.getQGroup(volPath) } if err != nil { return err } } // Modify the limit. if sizeBytes > 0 { // Custom handling for filesystem volume associated with a VM. if vol.volType == VolumeTypeVM && util.PathExists(filepath.Join(volPath, genericVolumeDiskFile)) { // Get the size of the VM image. blockSize, err := BlockDiskSizeBytes(filepath.Join(volPath, genericVolumeDiskFile)) if err != nil { return err } // Add that to the requested filesystem size (to ignore it from the quota). sizeBytes += blockSize d.logger.Debug("Accounting for VM image file size", logger.Ctx{"sizeBytes": sizeBytes}) } // Apply the limit to referenced data in qgroup. _, err = subprocess.RunCommand("btrfs", "qgroup", "limit", fmt.Sprintf("%d", sizeBytes), qgroup, volPath) if err != nil { return err } // Remove any former exclusive data limit. _, err = subprocess.RunCommand("btrfs", "qgroup", "limit", "-e", "none", qgroup, volPath) if err != nil { return err } } else if qgroup != "" { // Remove all limits. _, err = subprocess.RunCommand("btrfs", "qgroup", "limit", "none", qgroup, volPath) if err != nil { return err } _, err = subprocess.RunCommand("btrfs", "qgroup", "limit", "-e", "none", qgroup, volPath) if err != nil { return err } } return nil } // GetVolumeDiskPath returns the location and file format of a disk volume. func (d *btrfs) GetVolumeDiskPath(vol Volume) (string, error) { return genericVFSGetVolumeDiskPath(vol) } // ListVolumes returns a list of volumes in storage pool. func (d *btrfs) ListVolumes() ([]Volume, error) { return genericVFSListVolumes(d) } // MountVolume simulates mounting a volume. func (d *btrfs) MountVolume(vol Volume, op *operations.Operation) error { unlock, err := vol.MountLock() if err != nil { return err } defer unlock() // Don't attempt to modify the permission of an existing custom volume root. // A user inside the instance may have modified this and we don't want to reset it on restart. if !util.PathExists(vol.MountPath()) || vol.volType != VolumeTypeCustom { err := vol.EnsureMountPath(false) if err != nil { return err } } vol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolume() when done. return nil } // UnmountVolume simulates unmounting a volume. // As driver doesn't have volumes to unmount it returns false indicating the volume was already unmounted. func (d *btrfs) UnmountVolume(vol Volume, keepBlockDev bool, op *operations.Operation) (bool, error) { unlock, err := vol.MountLock() if err != nil { return false, err } defer unlock() refCount := vol.MountRefCountDecrement() if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": vol.name, "refCount": refCount}) return false, ErrInUse } return false, nil } // RenameVolume renames a volume and its snapshots. func (d *btrfs) RenameVolume(vol Volume, newVolName string, op *operations.Operation) error { return genericVFSRenameVolume(d, vol, newVolName, op) } // readonlySnapshot creates a readonly snapshot. // Returns a revert fail function that can be used to undo this function if a subsequent step fails. func (d *btrfs) readonlySnapshot(vol Volume) (string, revert.Hook, error) { reverter := revert.New() defer reverter.Fail() sourcePath := vol.MountPath() poolPath := GetPoolMountPath(d.name) tmpDir, err := os.MkdirTemp(poolPath, "backup.") if err != nil { return "", nil, err } reverter.Add(func() { _ = os.RemoveAll(tmpDir) }) err = os.Chmod(tmpDir, 0o100) if err != nil { return "", nil, err } mountPath := filepath.Join(tmpDir, vol.name) cleanup, err := d.snapshotSubvolume(sourcePath, mountPath, true) if err != nil { return "", nil, err } if cleanup != nil { reverter.Add(cleanup) } err = d.setSubvolumeReadonlyProperty(mountPath, true) if err != nil { return "", nil, err } d.logger.Debug("Created read-only backup snapshot", logger.Ctx{"sourcePath": sourcePath, "path": mountPath}) cleanup = reverter.Clone().Fail reverter.Success() return mountPath, cleanup, nil } // MigrateVolume sends a volume for migration. func (d *btrfs) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *localMigration.VolumeSourceArgs, op *operations.Operation) error { // Handle simple rsync and block_and_rsync through generic. if volSrcArgs.MigrationType.FSType == migration.MigrationFSType_RSYNC || volSrcArgs.MigrationType.FSType == migration.MigrationFSType_BLOCK_AND_RSYNC { // If volume is filesystem type and is not already a snapshot, create a fast snapshot to ensure migration is consistent. // TODO add support for temporary snapshots of block volumes here. if vol.contentType == ContentTypeFS && !vol.IsSnapshot() { snapshotPath, cleanup, err := d.readonlySnapshot(vol) if err != nil { return err } // Clean up the snapshot. defer cleanup() // Set the path of the volume to the path of the fast snapshot so the migration reads from there instead. vol.mountCustomPath = snapshotPath } return genericVFSMigrateVolume(d, d.state, vol, conn, volSrcArgs, op) } else if volSrcArgs.MigrationType.FSType != migration.MigrationFSType_BTRFS { return ErrNotSupported } // Handle btrfs send/receive migration. if volSrcArgs.MultiSync || volSrcArgs.FinalSync { // This is not needed if the migration is performed using btrfs send/receive. return errors.New("MultiSync should not be used with optimized migration") } var snapshots []string var err error if !volSrcArgs.VolumeOnly { // Generate restoration header, containing info on the subvolumes and how they should be restored. snapshots, err = d.volumeSnapshotsSorted(vol, op) if err != nil { return err } } migrationHeader, err := d.restorationHeader(vol, snapshots) if err != nil { return err } // If we haven't negotiated subvolume support, check if we have any subvolumes in source and fail, // otherwise we would end up not materialising all of the source's files on the target. if !slices.Contains(volSrcArgs.MigrationType.Features, migration.BTRFSFeatureMigrationHeader) || !slices.Contains(volSrcArgs.MigrationType.Features, migration.BTRFSFeatureSubvolumes) { for _, subVol := range migrationHeader.Subvolumes { if subVol.Path != string(filepath.Separator) { return errors.New("Subvolumes detected in source but target does not support receiving subvolumes") } } } // Send metadata migration header frame with subvolume info if we have negotiated that feature. if slices.Contains(volSrcArgs.MigrationType.Features, migration.BTRFSFeatureMigrationHeader) { headerJSON, err := json.Marshal(migrationHeader) if err != nil { return fmt.Errorf("Failed encoding BTRFS migration header: %w", err) } _, err = conn.Write(headerJSON) if err != nil { return fmt.Errorf("Failed sending BTRFS migration header: %w", err) } err = conn.Close() // End the frame. if err != nil { return fmt.Errorf("Failed closing BTRFS migration header frame: %w", err) } d.logger.Debug("Sent migration meta data header", logger.Ctx{"name": vol.name}) } if volSrcArgs.Refresh && slices.Contains(volSrcArgs.MigrationType.Features, migration.BTRFSFeatureSubvolumeUUIDs) { migrationHeader = &BTRFSMetaDataHeader{} buf, err := io.ReadAll(conn) if err != nil { return fmt.Errorf("Failed reading BTRFS migration header: %w", err) } err = json.Unmarshal(buf, &migrationHeader) if err != nil { return fmt.Errorf("Failed decoding BTRFS migration header: %w", err) } d.logger.Debug("Received BTRFS migration meta data header", logger.Ctx{"name": vol.name}) volSrcArgs.Snapshots = []string{} // Override volSrcArgs.Snapshots to only include snapshots which need to be sent. for _, snap := range migrationHeader.Subvolumes { if snap.Path == "/" && snap.Snapshot != "" { volSrcArgs.Snapshots = append(volSrcArgs.Snapshots, snap.Snapshot) } } } return d.migrateVolumeOptimized(vol, conn, volSrcArgs, migrationHeader.Subvolumes, op) } func (d *btrfs) migrateVolumeOptimized(vol Volume, conn io.ReadWriteCloser, volSrcArgs *localMigration.VolumeSourceArgs, subvolumes []BTRFSSubVolume, op *operations.Operation) error { // sendVolume sends a volume and its subvolumes (if negotiated subvolumes feature) to recipient. sendVolume := func(v Volume, sourcePrefix string, parentPrefix string) error { snapName := "" // Default to empty (sending main volume) from migrationHeader.Subvolumes. // Detect if we are sending a snapshot by comparing to main volume name. // We can't only use IsSnapshot() as the main vol may itself be a snapshot. if v.IsSnapshot() && v.name != vol.name { _, snapName, _ = api.GetParentAndSnapshotName(v.name) } // Setup progress tracking. var wrapper *ioprogress.ProgressTracker if volSrcArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", v.name) } sentVols := 0 // Send volume (and any subvolumes if supported) to target. for _, subVolume := range subvolumes { if subVolume.Snapshot != snapName { continue // Only sending subvolumes related to snapshot name (empty for main vol). } if subVolume.Path != string(filepath.Separator) && !slices.Contains(volSrcArgs.MigrationType.Features, migration.BTRFSFeatureSubvolumes) { continue // Skip sending subvolumes of volume if subvolumes feature not negotiated. } // Detect if parent subvolume exists, and if so use it for differential. parentPath := "" if parentPrefix != "" && d.isSubvolume(filepath.Join(parentPrefix, subVolume.Path)) { parentPath = filepath.Join(parentPrefix, subVolume.Path) // Set parent subvolume readonly if needed so we can send the subvolume. if !BTRFSSubVolumeIsRo(parentPath) { err := d.setSubvolumeReadonlyProperty(parentPath, true) if err != nil { return err } defer func() { _ = d.setSubvolumeReadonlyProperty(parentPath, false) }() } } // Set subvolume readonly if needed so we can send it. sourcePath := filepath.Join(sourcePrefix, subVolume.Path) if !BTRFSSubVolumeIsRo(sourcePath) { err := d.setSubvolumeReadonlyProperty(sourcePath, true) if err != nil { return err } defer func() { _ = d.setSubvolumeReadonlyProperty(sourcePath, false) }() } d.logger.Debug("Sending subvolume", logger.Ctx{"name": v.name, "source": sourcePath, "parent": parentPath, "path": subVolume.Path}) err := d.sendSubvolume(sourcePath, parentPath, conn, wrapper) if err != nil { return fmt.Errorf("Failed sending volume %v:%s: %w", v.name, subVolume.Path, err) } sentVols++ } // Ensure we found and sent at least root subvolume of the volume requested. if sentVols < 1 { return fmt.Errorf("No matching subvolume(s) for %q found in subvolumes list", v.name) } return nil } // Transfer the snapshots (and any subvolumes if supported) to target first. lastVolPath := "" // Used as parent for differential transfers. if !vol.IsSnapshot() && !volSrcArgs.VolumeOnly { snapshots, err := vol.Snapshots(op) if err != nil { return err } if volSrcArgs.Refresh { for i, snap := range snapshots { if i == 0 { continue } _, snapName, _ := api.GetParentAndSnapshotName(snap.name) if len(volSrcArgs.Snapshots) > 0 && snapName == volSrcArgs.Snapshots[0] { lastVolPath = snapshots[i-1].MountPath() break } } } for _, snapName := range volSrcArgs.Snapshots { snapVol, _ := vol.NewSnapshot(snapName) err := sendVolume(snapVol, snapVol.MountPath(), lastVolPath) if err != nil { return err } lastVolPath = snapVol.MountPath() } // If no snapshots are to be copied (because they are on the target already), but snapshots // exist on the source, use the latest snapshot as the parent in order to speed up // optimized refresh. if volSrcArgs.Refresh && len(volSrcArgs.Snapshots) == 0 && len(snapshots) > 0 { lastVolPath = snapshots[len(snapshots)-1].MountPath() } } // Get instances directory (e.g. /var/lib/incus/storage-pools/btrfs/containers). instancesPath := GetVolumeMountPath(d.name, vol.volType, "") // Create a temporary directory which will act as the parent directory of the read-only snapshot. tmpVolumesMountPoint, err := os.MkdirTemp(instancesPath, "migration.") if err != nil { return fmt.Errorf("Failed to create temporary directory under %q: %w", instancesPath, err) } defer func() { _ = os.RemoveAll(tmpVolumesMountPoint) }() err = os.Chmod(tmpVolumesMountPoint, 0o100) if err != nil { return fmt.Errorf("Failed to chmod %q: %w", tmpVolumesMountPoint, err) } // Make recursive read-only snapshot of the subvolume as writable subvolumes cannot be sent. migrationSendSnapshotPrefix := filepath.Join(tmpVolumesMountPoint, ".migration-send") _, err = d.snapshotSubvolume(vol.MountPath(), migrationSendSnapshotPrefix, true) if err != nil { return err } defer func() { _ = d.deleteSubvolume(migrationSendSnapshotPrefix, true) }() // Send main volume (and any subvolumes if supported) to target. return sendVolume(vol, migrationSendSnapshotPrefix, lastVolPath) } // BackupVolume copies a volume (and optionally its snapshots) to a specified target path. // This driver does not support optimized backups. func (d *btrfs) BackupVolume(vol Volume, writer instancewriter.InstanceWriter, basePrefix string, optimized bool, snapshots []string, op *operations.Operation) error { // Handle the non-optimized tarballs through the generic packer. if !optimized { // Because the generic backup method will not take a consistent backup if files are being modified // as they are copied to the tarball, as BTRFS allows us to take a quick snapshot without impacting // the parent volume we do so here to ensure the backup taken is consistent. if vol.contentType == ContentTypeFS { snapshotPath, cleanup, err := d.readonlySnapshot(vol) if err != nil { return err } // Clean up the snapshot. defer cleanup() // Set the path of the volume to the path of the fast snapshot so the migration reads from there instead. vol.mountCustomPath = snapshotPath } return genericVFSBackupVolume(d, vol, writer, basePrefix, snapshots, op) } // Optimized backup. if len(snapshots) > 0 { // Check requested snapshot match those in storage. err := vol.SnapshotsMatch(snapshots, op) if err != nil { return err } } // Generate driver restoration header. optimizedHeader, err := d.restorationHeader(vol, snapshots) if err != nil { return err } // Convert to YAML. optimizedHeaderYAML, err := yaml.Dump(&optimizedHeader, yaml.V2) if err != nil { return err } r := bytes.NewReader(optimizedHeaderYAML) indexFileInfo := instancewriter.FileInfo{ FileName: filepath.Join(basePrefix, "optimized_header.yaml"), FileSize: int64(len(optimizedHeaderYAML)), FileMode: 0o644, FileModTime: time.Now(), } // Write to tarball. err = writer.WriteFileFromReader(r, &indexFileInfo) if err != nil { return err } // sendToFile sends a subvolume to backup file. sendToFile := func(path string, parent string, fileName string) error { // Prepare btrfs send arguments. args := []string{"send"} if parent != "" { args = append(args, "-p", parent) } args = append(args, path) // Create temporary file to store output of btrfs send. backupsPath := internalUtil.VarPath("backups") tmpFile, err := os.CreateTemp(backupsPath, fmt.Sprintf("%s_btrfs", backup.WorkingDirPrefix)) if err != nil { return fmt.Errorf("Failed to open temporary file for BTRFS backup: %w", err) } defer func() { _ = tmpFile.Close() }() defer func() { _ = os.Remove(tmpFile.Name()) }() // Write the subvolume to the file. d.logger.Debug("Generating optimized volume file", logger.Ctx{"sourcePath": path, "parent": parent, "file": tmpFile.Name(), "name": fileName}) err = subprocess.RunCommandWithFds(context.TODO(), nil, tmpFile, "btrfs", args...) if err != nil { return err } // Get info (importantly size) of the generated file for tarball header. tmpFileInfo, err := os.Lstat(tmpFile.Name()) if err != nil { return err } err = writer.WriteFile(fileName, tmpFile.Name(), tmpFileInfo, false) if err != nil { return err } return tmpFile.Close() } // addVolume adds a volume and its subvolumes to backup file. addVolume := func(v Volume, sourcePrefix string, parentPrefix string, fileNamePrefix string) error { snapName := "" // Default to empty (sending main volume) from migrationHeader.Subvolumes. // Detect if we are adding a snapshot by comparing to main volume name. // We can't only use IsSnapshot() as the main vol may itself be a snapshot. if v.IsSnapshot() && v.name != vol.name { _, snapName, _ = api.GetParentAndSnapshotName(v.name) } sentVols := 0 // Add volume (and any subvolumes if supported) to backup file. for _, subVolume := range optimizedHeader.Subvolumes { if subVolume.Snapshot != snapName { continue // Only add subvolumes related to snapshot name (empty for main vol). } // Detect if parent subvolume exists, and if so use it for differential. parentPath := "" if parentPrefix != "" && d.isSubvolume(filepath.Join(parentPrefix, subVolume.Path)) { parentPath = filepath.Join(parentPrefix, subVolume.Path) // Set parent subvolume readonly if needed so we can add the subvolume. if !BTRFSSubVolumeIsRo(parentPath) { err = d.setSubvolumeReadonlyProperty(parentPath, true) if err != nil { return err } defer func() { _ = d.setSubvolumeReadonlyProperty(parentPath, false) }() } } // Set subvolume readonly if needed so we can add it. sourcePath := filepath.Join(sourcePrefix, subVolume.Path) if !BTRFSSubVolumeIsRo(sourcePath) { err = d.setSubvolumeReadonlyProperty(sourcePath, true) if err != nil { return err } defer func() { _ = d.setSubvolumeReadonlyProperty(sourcePath, false) }() } // Default to no subvolume name for root subvolume to maintain backwards compatibility // with earlier optimized dump format. Although restoring this backup file on an earlier // system will not restore the subvolumes stored inside the backup. subVolName := "" if subVolume.Path != string(filepath.Separator) { // Encode the path of the subvolume (without the leading /) into the filename so // that we find the file from the optimized header's Path field on restore. subVolName = fmt.Sprintf("_%s", linux.PathNameEncode(strings.TrimPrefix(subVolume.Path, string(filepath.Separator)))) } fileName := fmt.Sprintf("%s%s.bin", fileNamePrefix, subVolName) err = sendToFile(sourcePath, parentPath, filepath.Join(basePrefix, fileName)) if err != nil { return fmt.Errorf("Failed adding volume %v:%s: %w", v.name, subVolume.Path, err) } sentVols++ } // Ensure we found and sent at least root subvolume of the volume requested. if sentVols < 1 { return fmt.Errorf("No matching subvolume(s) for %q found in subvolumes list", v.name) } return nil } // Backup snapshots if populated. lastVolPath := "" // Used as parent for differential exports. for _, snapName := range snapshots { snapVol, _ := vol.NewSnapshot(snapName) // Make a binary btrfs backup. snapDir := "snapshots" fileName := snapName if vol.volType == VolumeTypeVM { snapDir = "virtual-machine-snapshots" if vol.contentType == ContentTypeFS { fileName = fmt.Sprintf("%s-config", snapName) } } else if vol.volType == VolumeTypeCustom { snapDir = "volume-snapshots" } fileNamePrefix := filepath.Join(snapDir, fileName) err := addVolume(snapVol, snapVol.MountPath(), lastVolPath, fileNamePrefix) if err != nil { return err } lastVolPath = snapVol.MountPath() } // Make a temporary copy of the instance. sourceVolume := vol.MountPath() instancesPath := GetVolumeMountPath(d.name, vol.volType, "") tmpInstanceMntPoint, err := os.MkdirTemp(instancesPath, "backup.") if err != nil { return fmt.Errorf("Failed to create temporary directory under %q: %w", instancesPath, err) } defer func() { _ = os.RemoveAll(tmpInstanceMntPoint) }() err = os.Chmod(tmpInstanceMntPoint, 0o100) if err != nil { return fmt.Errorf("Failed to chmod %q: %w", tmpInstanceMntPoint, err) } // Create the read-only snapshot. targetVolume := fmt.Sprintf("%s/.backup", tmpInstanceMntPoint) _, err = d.snapshotSubvolume(sourceVolume, targetVolume, true) if err != nil { return err } defer func() { _ = d.deleteSubvolume(targetVolume, true) }() err = d.setSubvolumeReadonlyProperty(targetVolume, true) if err != nil { return err } // Dump the instance to a file. fileNamePrefix := "container" if vol.volType == VolumeTypeVM { if vol.contentType == ContentTypeFS { fileNamePrefix = "virtual-machine-config" } else { fileNamePrefix = "virtual-machine" } } else if vol.volType == VolumeTypeCustom { fileNamePrefix = "volume" } err = addVolume(vol, targetVolume, lastVolPath, fileNamePrefix) if err != nil { return err } // Ensure snapshot sub volumes are removed. err = d.deleteSubvolume(targetVolume, true) if err != nil { return err } return nil } // CreateVolumeSnapshot creates a snapshot of a volume. func (d *btrfs) CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error { parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) srcPath := GetVolumeMountPath(d.name, snapVol.volType, parentName) snapPath := snapVol.MountPath() // Create the parent directory. err := CreateParentSnapshotDirIfMissing(d.name, snapVol.volType, parentName) if err != nil { return err } reverter := revert.New() defer reverter.Fail() cleanup, err := d.snapshotSubvolume(srcPath, snapPath, true) if err != nil { return err } if cleanup != nil { reverter.Add(cleanup) } err = d.setSubvolumeReadonlyProperty(snapPath, true) if err != nil { return err } // Set any subvolumes that were readonly in the source also readonly in the snapshot. srcVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, snapVol.config, snapVol.poolConfig) subVols, err := d.getSubvolumesMetaData(srcVol) if err != nil { return err } for _, subVol := range subVols { if subVol.Readonly { err = d.setSubvolumeReadonlyProperty(filepath.Join(snapPath, subVol.Path), true) if err != nil { return err } } } reverter.Success() return nil } // DeleteVolumeSnapshot removes a snapshot from the storage device. The volName and snapshotName // must be bare names and should not be in the format "volume/snapshot". func (d *btrfs) DeleteVolumeSnapshot(snapVol Volume, op *operations.Operation) error { snapPath := snapVol.MountPath() // Delete the snapshot. err := d.deleteSubvolume(snapPath, true) if err != nil { return err } // Remove the parent snapshot directory if this is the last snapshot being removed. parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) err = deleteParentSnapshotDirIfEmpty(d.name, snapVol.volType, parentName) if err != nil { return err } return nil } // MountVolumeSnapshot sets up a read-only mount on top of the snapshot to avoid accidental modifications. func (d *btrfs) MountVolumeSnapshot(snapVol Volume, op *operations.Operation) error { unlock, err := snapVol.MountLock() if err != nil { return err } defer unlock() snapPath := snapVol.MountPath() // Don't attempt to modify the permission of an existing custom volume root. // A user inside the instance may have modified this and we don't want to reset it on restart. if !util.PathExists(snapPath) || snapVol.volType != VolumeTypeCustom { err := snapVol.EnsureMountPath(false) if err != nil { return err } } _, err = mountReadOnly(snapPath, snapPath) if err != nil { return err } snapVol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolumeSnapshot() when done. return nil } // UnmountVolumeSnapshot removes the read-only mount placed on top of a snapshot. func (d *btrfs) UnmountVolumeSnapshot(snapVol Volume, op *operations.Operation) (bool, error) { unlock, err := snapVol.MountLock() if err != nil { return false, err } defer unlock() refCount := snapVol.MountRefCountDecrement() if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": snapVol.name, "refCount": refCount}) return false, ErrInUse } snapPath := snapVol.MountPath() return forceUnmount(snapPath) } // VolumeSnapshots returns a list of snapshots for the volume (in no particular order). func (d *btrfs) VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) { return genericVFSVolumeSnapshots(d, vol, op) } // volumeSnapshotsSorted returns a list of snapshots for the volume (ordered by subvolume ID). // Since the subvolume ID is incremental, this also represents the order of creation. func (d *btrfs) volumeSnapshotsSorted(vol Volume, op *operations.Operation) ([]string, error) { stdout := bytes.Buffer{} err := subprocess.RunCommandWithFds(context.TODO(), nil, &stdout, "btrfs", "subvolume", "list", GetPoolMountPath(vol.pool)) if err != nil { return nil, err } var snapshotNames []string snapshotPrefix := fmt.Sprintf("%s-snapshots/%s/", vol.volType, vol.name) scanner := bufio.NewScanner(&stdout) for scanner.Scan() { fields := strings.Fields(scanner.Text()) if len(fields) != 9 { continue } if !strings.HasPrefix(fields[8], snapshotPrefix) { continue } // Exclude subvolumes of snapshots if strings.Contains(strings.TrimPrefix(fields[8], snapshotPrefix), "/") { continue } snapshotNames = append(snapshotNames, filepath.Base(fields[8])) } return snapshotNames, nil } // RestoreVolume restores a volume from a snapshot. func (d *btrfs) RestoreVolume(vol Volume, snapshotName string, op *operations.Operation) error { reverter := revert.New() defer reverter.Fail() srcVol := NewVolume(d, d.name, vol.volType, vol.contentType, GetSnapshotVolumeName(vol.name, snapshotName), vol.config, vol.poolConfig) // Scan source for subvolumes (so we can apply the readonly properties on the restored snapshot). subVols, err := d.getSubvolumesMetaData(srcVol) if err != nil { return err } target := vol.MountPath() // Create a backup so we can revert. backupSubvolume := fmt.Sprintf("%s%s", target, tmpVolSuffix) err = os.Rename(target, backupSubvolume) if err != nil { return fmt.Errorf("Failed to rename %q to %q: %w", target, backupSubvolume, err) } reverter.Add(func() { _ = os.Rename(backupSubvolume, target) }) // Restore the snapshot. cleanup, err := d.snapshotSubvolume(srcVol.MountPath(), target, true) if err != nil { return err } if cleanup != nil { reverter.Add(cleanup) } // Restore readonly property on subvolumes in reverse order (except root which should be left writable). subVolCount := len(subVols) for i := range subVols { i = subVolCount - 1 - i subVol := subVols[i] if subVol.Readonly && subVol.Path != string(filepath.Separator) { targetSubVolPath := filepath.Join(target, subVol.Path) err = d.setSubvolumeReadonlyProperty(targetSubVolPath, true) if err != nil { return err } } } reverter.Success() // Remove the backup subvolume. return d.deleteSubvolume(backupSubvolume, true) } // RenameVolumeSnapshot renames a volume snapshot. func (d *btrfs) RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *operations.Operation) error { return genericVFSRenameVolumeSnapshot(d, snapVol, newSnapshotName, op) } // ActivateTask allows running a function while the volume is active (but not mounted). func (d *btrfs) ActivateTask(vol Volume, task func(devPath string, op *operations.Operation) error, op *operations.Operation) error { // Prevent concurrent mounting actions. unlock, err := vol.MountLock() if err != nil { return err } defer unlock() volDevPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } // Run the task. return task(volDevPath, op) } incus-7.0.0/internal/server/storage/drivers/driver_ceph.go000066400000000000000000000355271517523235500237500ustar00rootroot00000000000000package drivers import ( "bytes" "context" "encoding/json" "errors" "fmt" "os/exec" "strings" "github.com/lxc/incus/v7/internal/migration" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var ( cephVersion string cephLoaded bool ) type ceph struct { common } // load is used to run one-time action per-driver rather than per-pool. func (d *ceph) load() error { // Register the patches. d.patches = map[string]func() error{ "storage_lvm_skipactivation": nil, "storage_missing_snapshot_records": nil, "storage_delete_old_snapshot_records": nil, "storage_zfs_drop_block_volume_filesystem_extension": nil, "storage_prefix_bucket_names_with_project": nil, } // Done if previously loaded. if cephLoaded { return nil } // Handle IncusOS services. if d.state.OS.IncusOS != nil { ok, err := d.state.OS.IncusOS.IsServiceEnabled("ceph") if err != nil { return err } if !ok { return errors.New("IncusOS service \"ceph\" isn't currently enabled") } } // Validate the required binaries. for _, tool := range []string{"ceph", "rbd"} { _, err := exec.LookPath(tool) if err != nil { return fmt.Errorf("Required tool '%s' is missing", tool) } } // Detect and record the version. if cephVersion == "" { out, err := subprocess.RunCommand("rbd", "--version") if err != nil { return err } out = strings.TrimSpace(out) fields := strings.Split(out, " ") if strings.HasPrefix(out, "ceph version ") && len(fields) > 2 { cephVersion = fields[2] } else { cephVersion = out } } cephLoaded = true return nil } // isRemote returns true indicating this driver uses remote storage. func (d *ceph) isRemote() bool { return true } // Info returns info about the driver and its environment. func (d *ceph) Info() Info { return Info{ Name: "ceph", Version: cephVersion, DefaultVMBlockFilesystemSize: deviceConfig.DefaultVMBlockFilesystemSize, OptimizedImages: true, PreservesInodes: false, Remote: d.isRemote(), VolumeTypes: []VolumeType{VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM}, VolumeMultiNode: d.isRemote(), BlockBacking: true, RunningCopyFreeze: true, DirectIO: true, IOUring: true, MountedRoot: false, } } // getPlaceholderVolume returns the volume used to indicate if the pool is in use. func (d *ceph) getPlaceholderVolume() Volume { return NewVolume(d, d.name, VolumeType("incus"), ContentTypeFS, d.config["ceph.osd.pool_name"], nil, nil) } // FillConfig populates the storage pool's configuration file with the default values. func (d *ceph) FillConfig() error { if d.config["ceph.cluster_name"] == "" { d.config["ceph.cluster_name"] = CephDefaultCluster } if d.config["ceph.user.name"] == "" { d.config["ceph.user.name"] = CephDefaultUser } if d.config["ceph.osd.pg_num"] == "" { d.config["ceph.osd.pg_num"] = "32" } return nil } // Create is called during pool creation and is effectively using an empty driver struct. // WARNING: The Create() function cannot rely on any of the struct attributes being set. func (d *ceph) Create() error { reverter := revert.New() defer reverter.Fail() d.config["volatile.initial_source"] = d.config["source"] err := d.FillConfig() if err != nil { return err } // Validate. _, err = units.ParseByteSizeString(d.config["ceph.osd.pg_num"]) if err != nil { return err } // Quick check. if d.config["source"] != "" && d.config["ceph.osd.pool_name"] != "" && d.config["source"] != d.config["ceph.osd.pool_name"] { return errors.New(`The "source" and "ceph.osd.pool_name" property must not differ for Ceph OSD storage pools`) } // Use an existing OSD pool. if d.config["source"] != "" { d.config["ceph.osd.pool_name"] = d.config["source"] } if d.config["ceph.osd.pool_name"] == "" { d.config["ceph.osd.pool_name"] = d.name d.config["source"] = d.name } placeholderVol := d.getPlaceholderVolume() poolExists, err := d.osdPoolExists() if err != nil { return fmt.Errorf("Failed checking the existence of the ceph %q osd pool while attempting to create it because of an internal error: %w", d.config["ceph.osd.pool_name"], err) } if !poolExists { // Create new osd pool. _, err := subprocess.TryRunCommand("ceph", "--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]), "--cluster", d.config["ceph.cluster_name"], "osd", "pool", "create", d.config["ceph.osd.pool_name"], d.config["ceph.osd.pg_num"]) if err != nil { return err } reverter.Add(func() { _ = d.osdDeletePool() }) // Initialize the pool. This is not necessary but allows the pool to be monitored. _, err = subprocess.TryRunCommand("rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "pool", "init", d.config["ceph.osd.pool_name"]) if err != nil { d.logger.Warn("Failed to initialize pool", logger.Ctx{"pool": d.config["ceph.osd.pool_name"], "cluster": d.config["ceph.cluster_name"]}) } // Create placeholder storage volume. Other instances will use this to detect whether this osd // pool is already in use by another instance. err = d.rbdCreateVolume(placeholderVol, "0") if err != nil { return err } d.config["volatile.pool.pristine"] = "true" } else { volExists, err := d.HasVolume(placeholderVol) if err != nil { return err } if volExists { // ceph.osd.force_reuse is deprecated and should not be used. OSD pools are a logical // construct there is no good reason not to create one for dedicated use by the daemon. if util.IsFalseOrEmpty(d.config["ceph.osd.force_reuse"]) { return fmt.Errorf("Pool '%s' in cluster '%s' seems to be in use by another Incus instance. Use 'ceph.osd.force_reuse=true' to force", d.config["ceph.osd.pool_name"], d.config["ceph.cluster_name"]) } d.config["volatile.pool.pristine"] = "false" } else { // Create placeholder storage volume. Other instances will use this to detect whether this osd // pool is already in use by another instance. err := d.rbdCreateVolume(placeholderVol, "0") if err != nil { return err } d.config["volatile.pool.pristine"] = "true" } // Use existing OSD pool. msg, err := subprocess.RunCommand("ceph", "--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]), "--cluster", d.config["ceph.cluster_name"], "osd", "pool", "get", d.config["ceph.osd.pool_name"], "pg_num") if err != nil { return err } idx := strings.Index(msg, "pg_num:") if idx == -1 { return fmt.Errorf("Failed to parse number of placement groups for pool: %s", msg) } msg = msg[(idx + len("pg_num:")):] msg = strings.TrimSpace(msg) // It is ok to update the pool configuration since storage pool // creation via API is implemented such that the storage pool is // checked for a changed config after this function returns and // if so the db for it is updated. d.config["ceph.osd.pg_num"] = msg } reverter.Success() return nil } // Delete removes the storage pool from the storage device. func (d *ceph) Delete(op *operations.Operation) error { // Test if the pool exists. poolExists, err := d.osdPoolExists() if err != nil { return fmt.Errorf("Failed checking the existence of the ceph %q osd pool while attempting to delete it because of an internal error: %w", d.config["ceph.osd.pool_name"], err) } if !poolExists { d.logger.Warn("Pool does not exist", logger.Ctx{"pool": d.config["ceph.osd.pool_name"], "cluster": d.config["ceph.cluster_name"]}) } // Check whether we own the pool and only remove in this case. if util.IsTrue(d.config["volatile.pool.pristine"]) { // Delete the osd pool. if poolExists { err := d.osdDeletePool() if err != nil { return err } } } // If the user completely destroyed it, call it done. if !util.PathExists(GetPoolMountPath(d.name)) { return nil } // On delete, wipe everything in the directory. err = wipeDirectory(GetPoolMountPath(d.name)) if err != nil { return err } return nil } // Validate checks that all provide keys are supported and that no conflicting or missing configuration is present. func (d *ceph) Validate(config map[string]string) error { // gendoc:generate(entity=storage_ceph, group=common, key=source) // // --- // type: string // scope: local // default: - // shortdesc: Existing OSD storage pool to use rules := map[string]func(value string) error{ // gendoc:generate(entity=storage_ceph, group=common, key=ceph.cluster_name) // // --- // type: string // scope: global // default: `ceph` // shortdesc: Name of the Ceph cluster in which to create new storage pools "ceph.cluster_name": validate.IsAny, // gendoc:generate(entity=storage_ceph, group=common, key=ceph.osd.force_reuse) // // --- // type: bool // scope: global // default: - // shortdesc: Deprecated, should not be used. "ceph.osd.force_reuse": validate.Optional(validate.IsBool), // Deprecated, should not be used. // gendoc:generate(entity=storage_ceph, group=common, key=ceph.osd.pg_name) // // --- // type: string // scope: global // default: `32` // shortdesc: Number of placement groups for the OSD storage pool "ceph.osd.pg_num": validate.IsAny, // gendoc:generate(entity=storage_ceph, group=common, key=ceph.osd.pool_name) // // --- // type: string // scope: global // default: name of the pool // shortdesc: Name of the OSD storage pool "ceph.osd.pool_name": validate.IsAny, // gendoc:generate(entity=storage_ceph, group=common, key=ceph.osd.data_pool_name) // // --- // type: string // scope: global // default: - // shortdesc: Name of the OSD data pool "ceph.osd.data_pool_name": validate.IsAny, // gendoc:generate(entity=storage_ceph, group=common, key=ceph.rbd.clone_copy) // // --- // type: bool // scope: global // default: `true` // shortdesc: Whether to use RBD lightweight clones rather than full dataset copies "ceph.rbd.clone_copy": validate.Optional(validate.IsBool), // gendoc:generate(entity=storage_ceph, group=common, key=ceph.rbd.du) // // --- // type: bool // scope: global // default: `true` // shortdesc: Whether to use RBD `du` to obtain disk usage data for stopped instances "ceph.rbd.du": validate.Optional(validate.IsBool), // gendoc:generate(entity=storage_ceph, group=common, key=ceph.rbd.features) // // --- // type: string // scope: global // default: `layering` // shortdesc: Comma-separated list of RBD features to enable on the volumes "ceph.rbd.features": validate.IsAny, // gendoc:generate(entity=storage_ceph, group=common, key=ceph.user.name) // // --- // type: string // scope: global // default: `admin` // shortdesc: The Ceph user to use when creating storage pools and volumes "ceph.user.name": validate.IsAny, // gendoc:generate(entity=storage_ceph, group=common, key=volatile.pool.pristine) // // --- // type: string // scope: global // default: `true` // shortdesc: Whether the pool was empty on creation time "volatile.pool.pristine": validate.IsAny, } return d.validatePool(config, rules, d.commonVolumeRules()) } // Update applies any driver changes required from a configuration change. func (d *ceph) Update(changedConfig map[string]string) error { return nil } // Mount mounts the storage pool. func (d *ceph) Mount() (bool, error) { placeholderVol := d.getPlaceholderVolume() volExists, err := d.HasVolume(placeholderVol) if err != nil { return false, err } if !volExists { return false, errors.New("Placeholder volume does not exist") } return true, nil } // Unmount unmounts the storage pool. func (d *ceph) Unmount() (bool, error) { // Nothing to do here. return true, nil } // GetResources returns the pool resource usage information. func (d *ceph) GetResources() (*api.ResourcesStoragePool, error) { var stdout bytes.Buffer err := subprocess.RunCommandWithFds(context.TODO(), nil, &stdout, "ceph", "--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]), "--cluster", d.config["ceph.cluster_name"], "df", "-f", "json") if err != nil { return nil, err } // Temporary structs for parsing. type cephDfPoolStats struct { BytesUsed int64 `json:"bytes_used"` BytesAvailable int64 `json:"max_avail"` } type cephDfPool struct { Name string `json:"name"` Stats cephDfPoolStats `json:"stats"` } type cephDf struct { Pools []cephDfPool `json:"pools"` } // Parse the JSON output. df := cephDf{} err = json.NewDecoder(&stdout).Decode(&df) if err != nil { return nil, err } var pool *cephDfPool for _, entry := range df.Pools { if entry.Name == d.config["ceph.osd.pool_name"] { pool = &entry break } } if pool == nil { return nil, errors.New("OSD pool missing in df output") } spaceUsed := uint64(pool.Stats.BytesUsed) spaceAvailable := uint64(pool.Stats.BytesAvailable) res := api.ResourcesStoragePool{} res.Space.Total = spaceAvailable + spaceUsed res.Space.Used = spaceUsed return &res, nil } // MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order. func (d *ceph) MigrationTypes(contentType ContentType, refresh bool, copySnapshots bool, clusterMove bool, storageMove bool) []localMigration.Type { var rsyncFeatures []string // Do not pass compression argument to rsync if the associated // config key, that is rsync.compression, is set to false. if util.IsFalse(d.Config()["rsync.compression"]) { rsyncFeatures = []string{"xattrs", "delete", "bidirectional"} } else { rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"} } if refresh { var transportType migration.MigrationFSType if IsContentBlock(contentType) { transportType = migration.MigrationFSType_BLOCK_AND_RSYNC } else { transportType = migration.MigrationFSType_RSYNC } return []localMigration.Type{ { FSType: transportType, Features: rsyncFeatures, }, } } if contentType == ContentTypeBlock { return []localMigration.Type{ { FSType: migration.MigrationFSType_RBD, }, { FSType: migration.MigrationFSType_BLOCK_AND_RSYNC, Features: rsyncFeatures, }, } } return []localMigration.Type{ { FSType: migration.MigrationFSType_RBD, }, { FSType: migration.MigrationFSType_RSYNC, Features: rsyncFeatures, }, } } incus-7.0.0/internal/server/storage/drivers/driver_ceph_utils.go000066400000000000000000001070011517523235500251530ustar00rootroot00000000000000package drivers import ( "encoding/json" "errors" "fmt" "io" "io/fs" "net/http" "os" "os/exec" "slices" "strconv" "strings" "time" "github.com/google/uuid" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // cephBlockVolSuffix suffix used for block content type volumes. const cephBlockVolSuffix = ".block" // cephISOVolSuffix suffix used for iso content type volumes. const cephISOVolSuffix = ".iso" const cephVolumeTypeZombieImage = VolumeType("zombie_image") // CephDefaultCluster represents the default ceph cluster name. const CephDefaultCluster = "ceph" // CephDefaultUser represents the default ceph user name. const CephDefaultUser = "admin" // cephVolTypePrefixes maps volume type to storage volume name prefix. var cephVolTypePrefixes = map[VolumeType]string{ VolumeTypeContainer: db.StoragePoolVolumeTypeNameContainer, VolumeTypeVM: db.StoragePoolVolumeTypeNameVM, VolumeTypeImage: db.StoragePoolVolumeTypeNameImage, VolumeTypeCustom: db.StoragePoolVolumeTypeNameCustom, } // osdPoolExists checks whether a given OSD pool exists. func (d *ceph) osdPoolExists() (bool, error) { _, err := subprocess.RunCommand( "ceph", "--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]), "--cluster", d.config["ceph.cluster_name"], "osd", "pool", "get", d.config["ceph.osd.pool_name"], "size") if err != nil { status, _ := linux.ExitStatus(err) // If the error status code is 2, the pool definitely doesn't exist. if status == 2 { return false, nil } // Else, the error status is not 0 or 2, // so we can't be sure if the pool exists or not // as it might be a network issue, an internal ceph issue, etc. return false, err } return true, nil } // osdDeletePool destroys an OSD pool. // - A call to osdDeletePool will destroy a pool including any storage // volumes that still exist in the pool. // - In case the OSD pool that is supposed to be deleted does not exist this // command will still exit 0. This means that if the caller wants to be sure // that this call actually deleted an OSD pool it needs to check for the // existence of the pool first. func (d *ceph) osdDeletePool() error { _, err := subprocess.RunCommand( "ceph", "--name", fmt.Sprintf("client.%s", d.config["ceph.user.name"]), "--cluster", d.config["ceph.cluster_name"], "osd", "pool", "delete", d.config["ceph.osd.pool_name"], d.config["ceph.osd.pool_name"], "--yes-i-really-really-mean-it") if err != nil { return err } return nil } // rbdCreateVolume creates an RBD storage volume. // Note that the default set of features is intentionally limited // by passing --image-feature explicitly. This is done to ensure that // the chances of a conflict between the features supported by the userspace // library and the kernel module are minimized. Otherwise random panics might // occur. func (d *ceph) rbdCreateVolume(vol Volume, size string) error { sizeBytes, err := units.ParseByteSizeString(size) if err != nil { return err } cmd := []string{ "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], } if d.config["ceph.rbd.features"] != "" { for _, feature := range util.SplitNTrimSpace(d.config["ceph.rbd.features"], ",", -1, true) { cmd = append(cmd, "--image-feature", feature) } } else { cmd = append(cmd, "--image-feature", "layering") } if d.config["ceph.osd.data_pool_name"] != "" { cmd = append(cmd, "--data-pool", d.config["ceph.osd.data_pool_name"]) } cmd = append(cmd, "--size", fmt.Sprintf("%dB", sizeBytes), "create", d.getRBDVolumeName(vol, "", false)) _, err = subprocess.RunCommand("rbd", cmd...) return err } // rbdDeleteVolume deletes an RBD storage volume. // - In case the RBD storage volume that is supposed to be deleted does not // exist this command will still exit 0. This means that if the caller wants // to be sure that this call actually deleted an RBD storage volume it needs // to check for the existence of the pool first. func (d *ceph) rbdDeleteVolume(vol Volume) error { _, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "rm", d.getRBDVolumeName(vol, "", false)) if err != nil { return err } return nil } // rbdMapVolume maps a given RBD storage volume. // This will ensure that the RBD storage volume is accessible as a block device // in the /dev directory and is therefore necessary in order to mount it. func (d *ceph) rbdMapVolume(vol Volume) (string, error) { rbdName := d.getRBDVolumeName(vol, "", false) devPath, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "map", rbdName) if err != nil { return "", err } idx := strings.Index(devPath, "/dev/rbd") if idx < 0 { return "", errors.New("Failed to detect mapped device path") } devPath = strings.TrimSpace(devPath[idx:]) d.logger.Debug("Activated RBD volume", logger.Ctx{"volName": rbdName, "dev": devPath}) return devPath, nil } // rbdUnmapVolume unmaps a given RBD storage volume. // This is a precondition in order to delete an RBD storage volume can. func (d *ceph) rbdUnmapVolume(vol Volume, unmapUntilEINVAL bool) error { busyCount := 0 rbdVol := d.getRBDVolumeName(vol, "", false) ourDeactivate := false again: _, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "unmap", rbdVol) if err != nil { var runError subprocess.RunError if errors.As(err, &runError) { var exitError *exec.ExitError if errors.As(runError.Unwrap(), &exitError) { if exitError.ExitCode() == 22 { // EINVAL (already unmapped). if ourDeactivate { d.logger.Debug("Deactivated RBD volume", logger.Ctx{"volName": rbdVol}) } return nil } if exitError.ExitCode() == 16 { // EBUSY (currently in use). busyCount++ if busyCount == 10 { return err } // Wait a second an try again. time.Sleep(time.Second) goto again } } } return err } if unmapUntilEINVAL { ourDeactivate = true goto again } d.logger.Debug("Deactivated RBD volume", logger.Ctx{"volName": rbdVol}) return nil } // rbdUnmapVolumeSnapshot unmaps a given RBD snapshot. // This is a precondition in order to delete an RBD snapshot can. func (d *ceph) rbdUnmapVolumeSnapshot(vol Volume, snapshotName string, unmapUntilEINVAL bool) error { again: _, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "unmap", d.getRBDVolumeName(vol, snapshotName, false)) if err != nil { var runError subprocess.RunError if errors.As(err, &runError) { var exitError *exec.ExitError if errors.As(runError.Unwrap(), &exitError) { if exitError.ExitCode() == 22 { // EINVAL (already unmapped). return nil } } } return err } if unmapUntilEINVAL { goto again } return nil } // rbdCreateVolumeSnapshot creates a read-write snapshot of a given RBD storage volume. func (d *ceph) rbdCreateVolumeSnapshot(vol Volume, snapshotName string) error { _, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "snap", "create", "--snap", snapshotName, d.getRBDVolumeName(vol, "", false)) if err != nil { return err } return nil } // rbdProtectVolumeSnapshot protects a given snapshot from being deleted. // This is a precondition to be able to create RBD clones from a given snapshot. func (d *ceph) rbdProtectVolumeSnapshot(vol Volume, snapshotName string) error { _, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "snap", "protect", "--snap", snapshotName, d.getRBDVolumeName(vol, "", false)) if err != nil { var runError subprocess.RunError if errors.As(err, &runError) { var exitError *exec.ExitError if errors.As(runError.Unwrap(), &exitError) { if exitError.ExitCode() == 16 { // EBUSY (snapshot already protected). return nil } } } return err } return nil } // rbdUnprotectVolumeSnapshot unprotects a given snapshot. // - This is a precondition to be able to delete an RBD snapshot. // - This command will only succeed if the snapshot does not have any clones. func (d *ceph) rbdUnprotectVolumeSnapshot(vol Volume, snapshotName string) error { _, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "snap", "unprotect", "--snap", snapshotName, d.getRBDVolumeName(vol, "", false)) if err != nil { var runError subprocess.RunError if errors.As(err, &runError) { var exitError *exec.ExitError if errors.As(runError.Unwrap(), &exitError) { if exitError.ExitCode() == 22 { // EBUSY (snapshot already unprotected). return nil } } } return err } return nil } // rbdCreateClone creates a clone from a protected RBD snapshot. func (d *ceph) rbdCreateClone(sourceVol Volume, sourceSnapshotName string, targetVol Volume) error { cmd := []string{ "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], } if d.config["ceph.rbd.features"] != "" { for _, feature := range util.SplitNTrimSpace(d.config["ceph.rbd.features"], ",", -1, true) { cmd = append(cmd, "--image-feature", feature) } } else { cmd = append(cmd, "--image-feature", "layering") } if d.config["ceph.osd.data_pool_name"] != "" { cmd = append(cmd, "--data-pool", d.config["ceph.osd.data_pool_name"]) } cmd = append(cmd, "clone", d.getRBDVolumeName(sourceVol, sourceSnapshotName, true), d.getRBDVolumeName(targetVol, "", true)) _, err := subprocess.RunCommand("rbd", cmd...) if err != nil { return err } return nil } // rbdListSnapshotClones list all clones of an RBD snapshot. func (d *ceph) rbdListSnapshotClones(vol Volume, snapshotName string) ([]string, error) { msg, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "children", "--image", d.getRBDVolumeName(vol, "", false), "--snap", snapshotName) if err != nil { return nil, err } msg = strings.TrimSpace(msg) clones := strings.Fields(msg) if len(clones) == 0 { return nil, api.StatusErrorf(http.StatusNotFound, "Ceph RBD volume snapshot not found") } return clones, nil } // rbdMarkVolumeDeleted marks an RBD storage volume as being in "zombie" state. // An RBD storage volume that is in zombie state is not tracked in the // database anymore but still needs to be kept around for the sake of any // dependent storage entities in the storage pool. This usually happens when an // RBD storage volume has protected snapshots; a scenario most common when // creating a sparse copy of a container or when it updated an image and the // image still has dependent container clones. func (d *ceph) rbdMarkVolumeDeleted(vol Volume, newVolumeName string) error { // Ensure that new volume contains the config from the source volume to maintain filesystem suffix on // new volume name generated in getRBDVolumeName. newVol := NewVolume(d, d.name, vol.volType, vol.contentType, newVolumeName, vol.config, vol.poolConfig) newVol.isDeleted = true deletedName := d.getRBDVolumeName(newVol, "", true) _, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "mv", d.getRBDVolumeName(vol, "", true), deletedName, ) if err != nil { return err } return nil } // rbdRenameVolume renames a given RBD storage volume. // Note that this usually requires that the image be unmapped under its original // name, then renamed, and finally will be remapped again. If it is not unmapped // under its original name and the callers maps it under its new name the image // will be mapped twice. This will prevent it from being deleted. func (d *ceph) rbdRenameVolume(vol Volume, newVolumeName string) error { // Ensure that new volume contains the config from the source volume to maintain filesystem suffix on // new volume name generated in getRBDVolumeName. newVol := NewVolume(d, d.name, vol.volType, vol.contentType, newVolumeName, vol.config, vol.poolConfig) _, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "mv", d.getRBDVolumeName(vol, "", true), d.getRBDVolumeName(newVol, "", true), ) if err != nil { return err } return nil } // rbdRenameVolumeSnapshot renames a given RBD storage volume. // Note that if the snapshot is mapped - which it usually shouldn't be - this // usually requires that the snapshot be unmapped under its original name, then // renamed, and finally will be remapped again. If it is not unmapped under its // original name and the caller maps it under its new name the snapshot will be // mapped twice. This will prevent it from being deleted. func (d *ceph) rbdRenameVolumeSnapshot(vol Volume, oldSnapshotName string, newSnapshotName string) error { _, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "snap", "rename", d.getRBDVolumeName(vol, oldSnapshotName, true), d.getRBDVolumeName(vol, newSnapshotName, true)) if err != nil { return err } return nil } // rbdGetVolumeParent will return the snapshot the RBD clone was created from: // - If the RBD storage volume is not a clone then this function will return // db.NoSuchObjectError. // - The snapshot will be returned as // /@ // The caller will usually want to parse this according to its needs. This // helper library provides two small functions to do this but see below. func (d *ceph) rbdGetVolumeParent(vol Volume) (string, error) { msg, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "info", d.getRBDVolumeName(vol, "", false)) if err != nil { return "", err } idx := strings.Index(msg, "parent: ") if idx == -1 { return "", api.StatusErrorf(http.StatusNotFound, "Ceph RBD volume parent not found") } msg = msg[(idx + len("parent: ")):] msg = strings.TrimSpace(msg) idx = strings.Index(msg, "\n") if idx == -1 { return "", errors.New("Unexpected parsing error") } msg = msg[:idx] msg = strings.TrimSpace(msg) return msg, nil } // rbdDeleteVolumeSnapshot deletes an RBD snapshot. // This requires that the snapshot does not have any clones and is unmapped and // unprotected. func (d *ceph) rbdDeleteVolumeSnapshot(vol Volume, snapshotName string) error { _, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "snap", "rm", d.getRBDVolumeName(vol, snapshotName, false)) if err != nil { return err } return nil } // rbdListVolumeSnapshots retrieves the snapshots of an RBD storage volume. // The format of the snapshot names is simply the part after the @. So given a // valid RBD path relative to a pool // /@ // this will only return // . func (d *ceph) rbdListVolumeSnapshots(vol Volume) ([]string, error) { msg, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "--format", "json", "snap", "ls", d.getRBDVolumeName(vol, "", false)) if err != nil { return []string{}, err } var data []map[string]any err = json.Unmarshal([]byte(msg), &data) if err != nil { return []string{}, err } snapshots := []string{} for _, v := range data { _, ok := v["name"] if !ok { return []string{}, errors.New("No \"name\" property found") } name, ok := v["name"].(string) if !ok { return []string{}, errors.New("\"name\" property did not have string type") } name = strings.TrimSpace(name) snapshots = append(snapshots, name) } if len(snapshots) == 0 { return []string{}, api.StatusErrorf(http.StatusNotFound, "Ceph RBD volume snapshot(s) not found") } return snapshots, nil } // copyWithSnapshots creates a non-sparse copy of a container including its snapshots. // This does not introduce a dependency relation between the source RBD storage // volume and the target RBD storage volume. func (d *ceph) copyWithSnapshots(sourceVolumeName string, targetVolumeName string, sourceParentSnapshot string) error { args := []string{ "export-diff", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], sourceVolumeName, } if sourceParentSnapshot != "" { args = append(args, "--from-snap", sourceParentSnapshot) } // Redirect output to stdout. args = append(args, "-") rbdSendCmd := exec.Command("rbd", args...) rbdRecvCmd := exec.Command( "rbd", "import-diff", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "-", targetVolumeName) rbdRecvCmd.Stdin, _ = rbdSendCmd.StdoutPipe() rbdRecvCmd.Stdout = os.Stdout rbdRecvCmd.Stderr = os.Stderr err := rbdRecvCmd.Start() if err != nil { return err } err = rbdSendCmd.Run() if err != nil { return err } err = rbdRecvCmd.Wait() if err != nil { return err } return nil } // deleteVolume deletes the RBD storage volume of a container including any dependencies. // - This function takes care to delete any RBD storage entities that are marked // as zombie and whose existence is solely dependent on the RBD storage volume // for the container to be deleted. // - This function will mark any storage entities of the container to be deleted // as zombies in case any RBD storage entities in the storage pool have a // dependency relation with it. // - This function uses a C-style convention to return error or success simply // because it is more elegant and simple than the go way. // The function will return // -1 on error // 0 if the RBD storage volume has been deleted // 1 if the RBD storage volume has been marked as a zombie // - deleteVolume in conjunction with deleteVolumeSnapshot // recurses through an OSD storage pool to find and delete any storage // entities that were kept around because of dependency relations but are not // deletable. func (d *ceph) deleteVolume(vol Volume) (int, error) { snaps, err := d.rbdListVolumeSnapshots(vol) if err == nil { var zombies int for _, snap := range snaps { ret, err := d.deleteVolumeSnapshot(vol, snap) if ret < 0 { return -1, err } else if ret == 1 { zombies++ } } if zombies > 0 { // Unmap. err = d.rbdUnmapVolume(vol, true) if err != nil { return -1, err } if vol.isDeleted { return 1, nil } newVolumeName := fmt.Sprintf("%s_%s", vol.name, uuid.New().String()) err := d.rbdMarkVolumeDeleted(vol, newVolumeName) if err != nil { return -1, err } return 1, nil } else if zombies == 0 { // Delete. err = d.rbdDeleteVolume(vol) if err != nil { return -1, err } } } else { if !response.IsNotFoundError(err) { return -1, err } parent, err := d.rbdGetVolumeParent(vol) if err == nil { parentVol, parentSnapshotName, err := d.parseParent(parent) if err != nil { return -1, err } // Unmap. err = d.rbdUnmapVolume(vol, true) if err != nil { return -1, err } // Delete. err = d.rbdDeleteVolume(vol) if err != nil { return -1, err } // Only delete the parent snapshot of the instance if it is a zombie. // This includes both if the parent volume itself is a zombie, or if the just the snapshot // is a zombie. If it is not we know that Incus is still using it. if parentVol.isDeleted || strings.HasPrefix(parentSnapshotName, "zombie_") { ret, err := d.deleteVolumeSnapshot(parentVol, parentSnapshotName) if ret < 0 { return -1, err } } } else { if !response.IsNotFoundError(err) { return -1, err } // Unmap. err = d.rbdUnmapVolume(vol, true) if err != nil { return -1, err } // Delete. err = d.rbdDeleteVolume(vol) if err != nil { return -1, err } } } return 0, nil } // deleteVolumeSnapshot deletes an RBD snapshot of a container including any dependencies. // - This function takes care to delete any RBD storage entities that are marked // as zombie and whose existence is solely dependent on the RBD snapshot for // the container to be deleted. // - This function will mark any storage entities of the container to be deleted // as zombies in case any RBD storage entities in the storage pool have a // dependency relation with it. // - This function uses a C-style convention to return error or success simply // because it is more elegant and simple than the go way. // The function will return // -1 on error // 0 if the RBD storage volume has been deleted // 1 if the RBD storage volume has been marked as a zombie // - deleteVolumeSnapshot in conjunction with deleteVolume // recurses through an OSD storage pool to find and delete any storage // entities that were kept around because of dependency relations but are not // deletable. func (d *ceph) deleteVolumeSnapshot(vol Volume, snapshotName string) (int, error) { clones, err := d.rbdListSnapshotClones(vol, snapshotName) if err != nil { if !response.IsNotFoundError(err) { return -1, err } // Unprotect. err = d.rbdUnprotectVolumeSnapshot(vol, snapshotName) if err != nil { return -1, err } // Unmap. err = d.rbdUnmapVolumeSnapshot(vol, snapshotName, true) if err != nil { return -1, err } // Delete. err = d.rbdDeleteVolumeSnapshot(vol, snapshotName) if err != nil { return -1, err } // Only delete the parent image if it is a zombie. If it is not we know that Incus is still using it. if vol.isDeleted { ret, err := d.deleteVolume(vol) if ret < 0 { return -1, err } } return 0, nil } canDelete := true for _, clone := range clones { _, cloneType, cloneName, isDeleted, err := d.parseClone(clone) if err != nil { return -1, err } if !isDeleted { canDelete = false continue } cloneVol := NewVolume(d, d.name, VolumeType(cloneType), vol.contentType, cloneName, nil, nil) cloneVol.isDeleted = isDeleted ret, err := d.deleteVolume(cloneVol) if ret < 0 { return -1, err } else if ret == 1 { // Only marked as zombie. canDelete = false } } if canDelete { // Unprotect. err = d.rbdUnprotectVolumeSnapshot(vol, snapshotName) if err != nil { return -1, err } // Unmap. err = d.rbdUnmapVolumeSnapshot(vol, snapshotName, true) if err != nil { return -1, err } // Delete. err = d.rbdDeleteVolumeSnapshot(vol, snapshotName) if err != nil { return -1, err } // Only delete the parent image if it is a zombie. If it // is not we know that Incus is still using it. if vol.isDeleted { ret, err := d.deleteVolume(vol) if ret < 0 { return -1, err } } } else { if strings.HasPrefix(snapshotName, "zombie_") { return 1, nil } err := d.rbdUnmapVolumeSnapshot(vol, snapshotName, true) if err != nil { return -1, err } newSnapshotName := fmt.Sprintf("zombie_snapshot_%s", uuid.New().String()) err = d.rbdRenameVolumeSnapshot(vol, snapshotName, newSnapshotName) if err != nil { return -1, err } } return 1, nil } // parseParent splits a string describing a RBD storage entity into its components. // This can be used on strings like: /_@ // and will return a Volume and snapshot name. func (d *ceph) parseParent(parent string) (Volume, string, error) { vol := Volume{} fields := strings.SplitN(parent, "/", 2) if len(fields) != 2 { return vol, "", errors.New("Pool delimiter not found") } parentName := fields[1] vol.pool = fields[0] vol.isDeleted = strings.HasPrefix(parentName, "zombie_") // Handle images. if strings.HasPrefix(parentName, "image_") || strings.HasPrefix(parentName, "zombie_image_") { vol.volType = VolumeTypeImage // Split snapshot name. s := strings.Split(parentName, "@") var name string var snapName string if len(s) > 1 { snapName = s[len(s)-1] name = strings.TrimSuffix(parentName, "@"+snapName) } else { name = parentName } // Remove prefix from name. name = strings.SplitN(name, "image_", 2)[1] // Check for block indicator. if strings.HasSuffix(name, ".block") { name = strings.TrimSuffix(name, ".block") vol.contentType = ContentTypeBlock } else { vol.contentType = ContentTypeFS } // Check for filesystem indicator. if strings.Contains(name, "_") { s := strings.Split(name, "_") filesystem := s[len(s)-1] name = strings.TrimSuffix(name, "_"+filesystem) vol.config = map[string]string{ "block.filesystem": filesystem, } } vol.name = name return vol, snapName, nil } // Handle custom volumes. if strings.HasPrefix(parentName, "custom_") || strings.HasPrefix(parentName, "zombie_custom_") { vol.volType = VolumeTypeCustom // Split snapshot name. s := strings.Split(parentName, "@") var name string var snapName string if len(s) > 1 { snapName = s[len(s)-1] name = strings.TrimSuffix(parentName, "@"+snapName) } else { name = parentName } // Remove prefix from name. name = strings.SplitN(name, "custom_", 2)[1] // Check for block or ISO indicator. if strings.HasSuffix(name, ".block") { name = strings.TrimSuffix(name, ".block") vol.contentType = ContentTypeBlock } else if strings.HasSuffix(name, ".iso") { name = strings.TrimSuffix(name, ".iso") vol.contentType = ContentTypeISO } else { vol.contentType = ContentTypeFS } vol.name = name return vol, snapName, nil } // Handle container volumes. if strings.HasPrefix(parentName, "container_") || strings.HasPrefix(parentName, "zombie_container_") { vol.volType = VolumeTypeContainer // Split snapshot name. s := strings.Split(parentName, "@") var name string var snapName string if len(s) > 1 { snapName = s[len(s)-1] name = strings.TrimSuffix(parentName, "@"+snapName) } else { name = parentName } // Remove prefix from name. name = strings.SplitN(name, "container_", 2)[1] // Check for block indicator. if strings.HasSuffix(name, ".block") { name = strings.TrimSuffix(name, ".block") vol.contentType = ContentTypeBlock } else { vol.contentType = ContentTypeFS } vol.name = name return vol, snapName, nil } // Handle virtual-machines volumes. if strings.HasPrefix(parentName, "virtual-machine_") || strings.HasPrefix(parentName, "zombie_virtual-machine_") { vol.volType = VolumeTypeVM // Split snapshot name. s := strings.Split(parentName, "@") var name string var snapName string if len(s) > 1 { snapName = s[len(s)-1] name = strings.TrimSuffix(parentName, "@"+snapName) } else { name = parentName } // Remove prefix from name. name = strings.SplitN(name, "virtual-machine_", 2)[1] // Check for block indicator. if strings.HasSuffix(name, ".block") { name = strings.TrimSuffix(name, ".block") vol.contentType = ContentTypeBlock } else { vol.contentType = ContentTypeFS } vol.name = name return vol, snapName, nil } return vol, "", fmt.Errorf("Unrecognized parent: %q", parent) } // parseClone splits a strings describing an RBD storage volume. // For example a string like // /_ // will be split into // , , . func (d *ceph) parseClone(clone string) (string, string, string, bool, error) { fields := strings.SplitN(clone, "/", 2) if len(fields) != 2 { return "", "", "", false, errors.New("Pool delimiter not found") } volumeName := fields[1] poolName := fields[0] volumeDeleted := strings.HasPrefix(volumeName, "zombie_") // Strip zombie prefix. volumeName = strings.TrimPrefix(volumeName, "zombie_") f := strings.SplitN(volumeName, "_", 2) if len(f) != 2 { return "", "", "", false, errors.New("Unexpected parsing error") } volumeType := f[0] volumeName = f[1] return poolName, volumeType, volumeName, volumeDeleted, nil } // getRBDMappedDevPath looks at sysfs to retrieve the device path. If it doesn't find it it will map it if told to // do so. Returns bool indicating if map was needed and device path e.g. "/dev/rbd" for an RBD image. func (d *ceph) getRBDMappedDevPath(vol Volume, mapIfMissing bool) (bool, string, error) { // List all RBD devices. files, err := os.ReadDir("/sys/devices/rbd") if err != nil && !errors.Is(err, fs.ErrNotExist) { return false, "", err } // Go through the existing RBD devices. for _, f := range files { fName := f.Name() // Skip if not a directory. if !f.IsDir() { continue } // Skip if not a device directory. idx, err := strconv.ParseUint(fName, 10, 64) if err != nil { continue } // Get the pool for the RBD device. devPoolName, err := os.ReadFile(fmt.Sprintf("/sys/devices/rbd/%s/pool", fName)) if err != nil { // Skip if no pool file. if errors.Is(err, fs.ErrNotExist) { continue } return false, "", err } // Skip if the pools don't match. if strings.TrimSpace(string(devPoolName)) != d.config["ceph.osd.pool_name"] { continue } // Get the volume name for the RBD device. devName, err := os.ReadFile(fmt.Sprintf("/sys/devices/rbd/%s/name", fName)) if err != nil { // Skip if no name file. if errors.Is(err, fs.ErrNotExist) { continue } return false, "", err } rbdName := d.getRBDVolumeName(vol, "", false) // Split RBD name into volume name and snapshot name parts. rbdNameParts := strings.SplitN(rbdName, "@", 2) // Skip if the names don't match (excluding snapshot part of RBD volume name). if strings.TrimSpace(string(devName)) != rbdNameParts[0] { continue } // Get the snapshot name for the RBD device (if exists). devSnap, err := os.ReadFile(fmt.Sprintf("/sys/devices/rbd/%s/current_snap", fName)) if err != nil && !errors.Is(err, fs.ErrNotExist) { return false, "", err } devSnapName := strings.TrimSpace(string(devSnap)) if vol.IsSnapshot() { // Volume is a snapshot, check device's snapshot name matches the volume's snapshot name. if len(rbdNameParts) == 2 && rbdNameParts[1] == devSnapName { return false, fmt.Sprintf("/dev/rbd%d", idx), nil // We found a match. } } else if slices.Contains([]string{"-", ""}, devSnapName) { // Volume is not a snapshot and neither is this device. return false, fmt.Sprintf("/dev/rbd%d", idx), nil // We found a match. } continue } // No device could be found, map it ourselves. if mapIfMissing { devPath, err := d.rbdMapVolume(vol) if err != nil { return false, "", err } return true, devPath, nil } return false, "", fmt.Errorf("Volume %q not mapped to an RBD device", vol.Name()) } // generateUUID regenerates the XFS/btrfs UUID as needed. func (d *ceph) generateUUID(fsType string, devPath string) error { if !renegerateFilesystemUUIDNeeded(fsType) { return nil } // Update the UUID. d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": devPath, "fs": fsType}) err := regenerateFilesystemUUID(fsType, devPath) if err != nil { return err } return nil } func (d *ceph) getRBDVolumeName(vol Volume, snapName string, withPoolName bool) string { out := CephGetRBDImageName(vol, snapName, vol.isDeleted) // If needed, the output will be prefixed with the pool name, e.g. // /_@. if withPoolName { out = fmt.Sprintf("%s/%s", d.config["ceph.osd.pool_name"], out) } return out } // Let's say we want to send the a container "a" including snapshots "snap0" and // "snap1" on storage pool "pool1" from Incus "l1" to Incus "l2" on storage pool // "pool2": // // The pool layout on "l1" would be: // // pool1/container_a // pool1/container_a@snapshot_snap0 // pool1/container_a@snapshot_snap1 // // Then we need to send: // // rbd export-diff pool1/container_a@snapshot_snap0 - | rbd import-diff - pool2/container_a // // (Note that pool2/container_a must have been created by the receiving Incus // instance before.) // // rbd export-diff pool1/container_a@snapshot_snap1 --from-snap snapshot_snap0 - | rbd import-diff - pool2/container_a // rbd export-diff pool1/container_a --from-snap snapshot_snap1 - | rbd import-diff - pool2/container_a func (d *ceph) sendVolume(conn io.ReadWriteCloser, volumeName string, volumeParentName string, tracker *ioprogress.ProgressTracker) error { defer func() { _ = conn.Close() }() args := []string{ "export-diff", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], volumeName, } if volumeParentName != "" { args = append(args, "--from-snap", volumeParentName) } // Redirect output to stdout. args = append(args, "-") cmd := exec.Command("rbd", args...) stderr, err := cmd.StderrPipe() if err != nil { return err } // Setup progress tracker. var stdout io.WriteCloser = conn if tracker != nil { stdout = &ioprogress.ProgressWriter{ WriteCloser: conn, Tracker: tracker, } } cmd.Stdout = stdout err = cmd.Start() if err != nil { return err } output, _ := io.ReadAll(stderr) // Handle errors. err = cmd.Wait() if err != nil { return fmt.Errorf("ceph export-diff failed: %w (%s)", err, string(output)) } return nil } func (d *ceph) receiveVolume(volumeName string, conn io.ReadWriteCloser, writeWrapper func(io.WriteCloser) io.WriteCloser) error { args := []string{ "import-diff", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "-", volumeName, } cmd := exec.Command("rbd", args...) stdin, err := cmd.StdinPipe() if err != nil { return err } stderr, err := cmd.StderrPipe() if err != nil { return err } // Forward input through stdin. chCopyConn := make(chan error, 1) go func() { _, err = util.SafeCopy(stdin, conn) _ = stdin.Close() chCopyConn <- err }() // Run the command. err = cmd.Start() if err != nil { return err } // Read any error. output, _ := io.ReadAll(stderr) // Handle errors. errs := []error{} chCopyConnErr := <-chCopyConn err = cmd.Wait() if err != nil { errs = append(errs, err) if chCopyConnErr != nil { errs = append(errs, chCopyConnErr) } } if len(errs) > 0 { return fmt.Errorf("Problem with ceph import-diff: (%v) %s", errs, string(output)) } return nil } // resizeVolume resizes an RBD volume. This function does not resize any filesystem inside the RBD volume. func (d *ceph) resizeVolume(vol Volume, sizeBytes int64, allowShrink bool) error { args := []string{ "resize", } if allowShrink { args = append(args, "--allow-shrink") } args = append(args, "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "--size", fmt.Sprintf("%dB", sizeBytes), d.getRBDVolumeName(vol, "", false), ) // Resize the block device. _, err := subprocess.TryRunCommand("rbd", args...) return err } incus-7.0.0/internal/server/storage/drivers/driver_ceph_utils_test.go000066400000000000000000000173151517523235500262220ustar00rootroot00000000000000package drivers import ( "fmt" "testing" ) func Test_ceph_getRBDVolumeName(t *testing.T) { type args struct { vol Volume snapName string withPoolName bool } tests := []struct { name string args args want string }{ { "Volume without pool name", args{ vol: NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil), snapName: "", withPoolName: false, }, "container_testvol", }, { "Volume with unknown type and without pool name", args{ vol: NewVolume(nil, "testpool", VolumeType("unknown"), ContentTypeFS, "testvol", nil, nil), snapName: "", withPoolName: false, }, "unknown_testvol", }, { "Volume without pool name in zombie mode", args{ vol: func() Volume { vol := NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil) vol.isDeleted = true return vol }(), snapName: "", withPoolName: false, }, "zombie_container_testvol", }, { "Volume with pool name in zombie mode", args{ vol: func() Volume { vol := NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil) vol.isDeleted = true return vol }(), snapName: "", withPoolName: true, }, "testosdpool/zombie_container_testvol", }, { "Volume snapshot with dedicated snapshot name and without pool name", args{ vol: NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil), snapName: "snapshot_testsnap", withPoolName: false, }, "container_testvol@snapshot_testsnap", }, { "Volume snapshot with dedicated snapshot name and pool name", args{ vol: NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil), snapName: "snapshot_testsnap", withPoolName: true, }, "testosdpool/container_testvol@snapshot_testsnap", }, { "Volume snapshot with pool name", args{ vol: NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol/testsnap", nil, nil), snapName: "", withPoolName: true, }, "testosdpool/container_testvol@snapshot_testsnap", }, { "Volume snapshot with additional dedicated snapshot name and pool name", args{ vol: NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol/testsnap", nil, nil), snapName: "testsnap1", withPoolName: true, }, "testosdpool/container_testvol@testsnap1", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { d := &ceph{ common{ config: map[string]string{ "ceph.osd.pool_name": "testosdpool", }, }, } got := d.getRBDVolumeName(tt.args.vol, tt.args.snapName, tt.args.withPoolName) if got != tt.want { t.Errorf("ceph.getRBDVolumeName() = %v, want %v", got, tt.want) } }) } } func Example_ceph_parseParent() { d := &ceph{} parents := []string{ "pool/zombie_image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4.block@readonly", "pool/zombie_image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4.block", "pool/image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4.block@readonly", "pool/zombie_image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4@readonly", "pool/zombie_image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4", "pool/image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4@readonly", "pool/zombie_image_2cfc5a5567b8d74c0986f3d8a77a2a78e58fe22ea9abd2693112031f85afa1a1_xfs@zombie_snapshot_7f6d679b-ee25-419e-af49-bb805cb32088", "pool/container_bar@zombie_snapshot_ce77e971-6c1b-45c0-b193-dba9ec5e7d82", "pool/container_test-project_c4.block", "pool/zombie_container_test-project_c1_28e7a7ab-740a-490c-8118-7caf7810f83b@zombie_snapshot_1027f4ab-de11-4cee-8015-bd532a1fed76", "pool/custom_default_foo.vol@zombie_snapshot_2e8112b2-91ca-4fc2-a546-c7723395fdbd", } for _, parent := range parents { vol, snapName, err := d.parseParent(parent) fmt.Printf("%s: %v\n", parent, err) if err == nil { fmt.Printf(" pool: %s\n", vol.pool) fmt.Printf(" name: %s\n", vol.name) if snapName != "" { fmt.Printf(" snapName: %s\n", snapName) } fmt.Printf(" volType: %s\n", vol.volType) fmt.Printf(" contentType: %s\n", vol.contentType) fmt.Printf(" config: %+v\n", vol.config) } } // Output: pool/zombie_image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4.block@readonly: // pool: pool // name: 9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb // snapName: readonly // volType: images // contentType: block // config: map[block.filesystem:ext4] // pool/zombie_image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4.block: // pool: pool // name: 9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb // volType: images // contentType: block // config: map[block.filesystem:ext4] // pool/image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4.block@readonly: // pool: pool // name: 9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb // snapName: readonly // volType: images // contentType: block // config: map[block.filesystem:ext4] // pool/zombie_image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4@readonly: // pool: pool // name: 9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb // snapName: readonly // volType: images // contentType: filesystem // config: map[block.filesystem:ext4] // pool/zombie_image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4: // pool: pool // name: 9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb // volType: images // contentType: filesystem // config: map[block.filesystem:ext4] // pool/image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4@readonly: // pool: pool // name: 9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb // snapName: readonly // volType: images // contentType: filesystem // config: map[block.filesystem:ext4] // pool/zombie_image_2cfc5a5567b8d74c0986f3d8a77a2a78e58fe22ea9abd2693112031f85afa1a1_xfs@zombie_snapshot_7f6d679b-ee25-419e-af49-bb805cb32088: // pool: pool // name: 2cfc5a5567b8d74c0986f3d8a77a2a78e58fe22ea9abd2693112031f85afa1a1 // snapName: zombie_snapshot_7f6d679b-ee25-419e-af49-bb805cb32088 // volType: images // contentType: filesystem // config: map[block.filesystem:xfs] // pool/container_bar@zombie_snapshot_ce77e971-6c1b-45c0-b193-dba9ec5e7d82: // pool: pool // name: bar // snapName: zombie_snapshot_ce77e971-6c1b-45c0-b193-dba9ec5e7d82 // volType: containers // contentType: filesystem // config: map[] // pool/container_test-project_c4.block: // pool: pool // name: test-project_c4 // volType: containers // contentType: block // config: map[] // pool/zombie_container_test-project_c1_28e7a7ab-740a-490c-8118-7caf7810f83b@zombie_snapshot_1027f4ab-de11-4cee-8015-bd532a1fed76: // pool: pool // name: test-project_c1_28e7a7ab-740a-490c-8118-7caf7810f83b // snapName: zombie_snapshot_1027f4ab-de11-4cee-8015-bd532a1fed76 // volType: containers // contentType: filesystem // config: map[] // pool/custom_default_foo.vol@zombie_snapshot_2e8112b2-91ca-4fc2-a546-c7723395fdbd: // pool: pool // name: default_foo.vol // snapName: zombie_snapshot_2e8112b2-91ca-4fc2-a546-c7723395fdbd // volType: custom // contentType: filesystem // config: map[] } incus-7.0.0/internal/server/storage/drivers/driver_ceph_volumes.go000066400000000000000000001560201517523235500255120ustar00rootroot00000000000000package drivers import ( "bufio" "context" "encoding/json" "errors" "fmt" "io" "io/fs" "os" "os/exec" "strings" "time" "github.com/google/uuid" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/server/backup" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // CreateVolume creates an empty volume and can optionally fill it by executing the supplied // filler function. func (d *ceph) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Operation) error { // Function to rename an RBD volume. renameVolume := func(oldName string, newName string) error { _, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "mv", oldName, newName, ) return err } // Revert handling. reverter := revert.New() defer reverter.Fail() if vol.contentType == ContentTypeFS { // Create mountpoint. err := vol.EnsureMountPath(true) if err != nil { return err } reverter.Add(func() { _ = os.Remove(vol.MountPath()) }) } // Create a "zombie" deleted volume representation of the specified volume to look for its existence. deletedVol := NewVolume(d, d.name, cephVolumeTypeZombieImage, vol.contentType, vol.name, vol.config, vol.poolConfig) // Check if we have a deleted zombie image. If so, restore it otherwise create a new image volume. if vol.volType == VolumeTypeImage { volExists, err := d.HasVolume(deletedVol) if err != nil { return err } if volExists { canRestore := true // Check if the cached image volume is larger than the current pool volume.size setting // (if so we won't be able to resize the snapshot to that the smaller size later). volSizeBytes, err := d.getVolumeSize(d.getRBDVolumeName(deletedVol, "", true)) if err != nil { return err } poolVolSize := DefaultBlockSize if vol.poolConfig["volume.size"] != "" { poolVolSize = vol.poolConfig["volume.size"] } poolVolSizeBytes, err := units.ParseByteSizeString(poolVolSize) if err != nil { return err } // If the cached volume size is different than the pool volume size, then we can't use the // deleted cached image volume and instead we will rename it to a random UUID so it can't // be restored in the future and a new cached image volume will be created instead. if volSizeBytes != poolVolSizeBytes { d.logger.Debug("Renaming deleted cached image volume so that regeneration is used", logger.Ctx{"fingerprint": vol.Name()}) randomVol := NewVolume(d, d.name, deletedVol.volType, deletedVol.contentType, strings.ReplaceAll(uuid.New().String(), "-", ""), deletedVol.config, deletedVol.poolConfig) err = renameVolume(d.getRBDVolumeName(deletedVol, "", true), d.getRBDVolumeName(randomVol, "", true)) if err != nil { return err } if vol.IsVMBlock() { fsDeletedVol := deletedVol.NewVMBlockFilesystemVolume() fsRandomVol := randomVol.NewVMBlockFilesystemVolume() err = renameVolume(d.getRBDVolumeName(fsDeletedVol, "", true), d.getRBDVolumeName(fsRandomVol, "", true)) if err != nil { return err } } canRestore = false } // Restore the image. if canRestore { d.logger.Debug("Restoring previously deleted cached image volume", logger.Ctx{"fingerprint": vol.Name()}) err = renameVolume(d.getRBDVolumeName(deletedVol, "", true), d.getRBDVolumeName(vol, "", true)) if err != nil { return err } if vol.IsVMBlock() { fsDeletedVol := deletedVol.NewVMBlockFilesystemVolume() fsVol := vol.NewVMBlockFilesystemVolume() err = renameVolume(d.getRBDVolumeName(fsDeletedVol, "", true), d.getRBDVolumeName(fsVol, "", true)) if err != nil { return err } } reverter.Success() return nil } } } // Create volume. err := d.rbdCreateVolume(vol, vol.ConfigSize()) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) devPath, err := d.rbdMapVolume(vol) if err != nil { return err } reverter.Add(func() { _ = d.rbdUnmapVolume(vol, true) }) // Get filesystem. RBDFilesystem := vol.ConfigBlockFilesystem() if vol.contentType == ContentTypeFS { _, err = makeFSType(devPath, RBDFilesystem, nil) if err != nil { return err } } // For VMs, also create the filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.CreateVolume(fsVol, nil, op) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolume(fsVol, op) }) } err = vol.MountTask(func(mountPath string, op *operations.Operation) error { // Run the volume filler function if supplied. if filler != nil && filler.Fill != nil { var err error var devPath string if IsContentBlock(vol.contentType) { // Get the device path. devPath, err = d.GetVolumeDiskPath(vol) if err != nil { return err } } allowUnsafeResize := false if vol.volType == VolumeTypeImage { // Allow filler to resize initial image volume as needed. // Some storage drivers don't normally allow image volumes to be resized due to // them having read-only snapshots that cannot be resized. However when creating // the initial image volume and filling it before the snapshot is taken resizing // can be allowed and is required in order to support unpacking images larger than // the default volume size. The filler function is still expected to obey any // volume size restrictions configured on the pool. // Unsafe resize is also needed to disable filesystem resize safety checks. // This is safe because if for some reason an error occurs the volume will be // discarded rather than leaving a corrupt filesystem. allowUnsafeResize = true } // Run the filler. err = genericRunFiller(d, vol, devPath, filler, allowUnsafeResize) if err != nil { return err } // Move the GPT alt header to end of disk if needed. if vol.IsVMBlock() { err = d.moveGPTAltHeader(devPath) if err != nil { return err } } } if vol.contentType == ContentTypeFS { // Run EnsureMountPath again after mounting and filling to ensure the mount directory has // the correct permissions set. err = vol.EnsureMountPath(true) if err != nil { return err } } return nil }, op) if err != nil { return err } // Create a readonly snapshot of the image volume which will be used a the // clone source for future non-image volumes. if vol.volType == VolumeTypeImage { err = d.rbdUnmapVolume(vol, true) if err != nil { return err } err = d.rbdCreateVolumeSnapshot(vol, "readonly") if err != nil { return err } reverter.Add(func() { _, _ = d.deleteVolumeSnapshot(vol, "readonly") }) err = d.rbdProtectVolumeSnapshot(vol, "readonly") if err != nil { return err } if vol.contentType == ContentTypeBlock { // Re-create the FS config volume's readonly snapshot now that the filler function has run // and unpacked into both config and block volumes. fsVol := NewVolume(d, d.name, vol.volType, ContentTypeFS, vol.name, vol.config, vol.poolConfig) err := d.rbdUnprotectVolumeSnapshot(fsVol, "readonly") if err != nil { return err } _, err = d.deleteVolumeSnapshot(fsVol, "readonly") if err != nil { return err } err = d.rbdCreateVolumeSnapshot(fsVol, "readonly") if err != nil { return err } reverter.Add(func() { _, _ = d.deleteVolumeSnapshot(fsVol, "readonly") }) err = d.rbdProtectVolumeSnapshot(fsVol, "readonly") if err != nil { return err } } } reverter.Success() return nil } // getVolumeSize returns the volume's size in bytes. func (d *ceph) getVolumeSize(volumeName string) (int64, error) { volInfo := struct { Size int64 `json:"size"` }{} jsonInfo, err := subprocess.TryRunCommand( "rbd", "info", "--format", "json", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], volumeName, ) if err != nil { return -1, err } err = json.Unmarshal([]byte(jsonInfo), &volInfo) if err != nil { return -1, err } return volInfo.Size, nil } // CreateVolumeFromBackup re-creates a volume from its exported state. func (d *ceph) CreateVolumeFromBackup(vol Volume, srcBackup backup.Info, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) (VolumePostHook, revert.Hook, error) { return genericVFSBackupUnpack(d, d.state.OS, vol, srcBackup.Snapshots, srcData, basePrefix, op) } // CreateVolumeFromCopy provides same-pool volume copying functionality. func (d *ceph) CreateVolumeFromCopy(vol Volume, srcVol Volume, copySnapshots bool, allowInconsistent bool, op *operations.Operation) error { var err error reverter := revert.New() defer reverter.Fail() // Function to run once the volume is created, which will regenerate the filesystem UUID (if needed), // ensure permissions on mount path inside the volume are correct, and resize the volume to specified size. postCreateTasks := func(v Volume) error { // Map the RBD volume. devPath, err := d.rbdMapVolume(v) if err != nil { return err } defer func() { _ = d.rbdUnmapVolume(v, true) }() if vol.contentType == ContentTypeFS { // Re-generate the UUID. Do this first as ensuring permissions and setting quota can // rely on being able to mount the volume. err = d.generateUUID(v.ConfigBlockFilesystem(), devPath) if err != nil { return err } // Mount the volume and ensure the permissions are set correctly inside the mounted volume. err = v.MountTask(func(_ string, _ *operations.Operation) error { return v.EnsureMountPath(false) }, op) if err != nil { return err } } // Resize volume to the size specified. Only uses volume "size" property and does not use // pool/defaults to give the caller more control over the size being used. err = d.SetVolumeQuota(vol, vol.config["size"], false, op) if err != nil { return err } return nil } // For VMs, also copy the filesystem volume. if vol.IsVMBlock() { srcFSVol := srcVol.NewVMBlockFilesystemVolume() fsVol := vol.NewVMBlockFilesystemVolume() err := d.CreateVolumeFromCopy(fsVol, srcFSVol, copySnapshots, false, op) if err != nil { return err } // Delete on revert. reverter.Add(func() { _ = d.DeleteVolume(fsVol, op) }) } // Retrieve snapshots on the source. snapshots := []string{} if !srcVol.IsSnapshot() && copySnapshots { snapshots, err = d.VolumeSnapshots(srcVol, op) if err != nil { return err } } // Copy without snapshots. if !copySnapshots || len(snapshots) == 0 { // If lightweight clone mode isn't enabled, perform a full copy of the volume. if util.IsFalse(d.config["ceph.rbd.clone_copy"]) { _, err = subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "cp", d.getRBDVolumeName(srcVol, "", true), d.getRBDVolumeName(vol, "", true), ) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) _, err = d.rbdMapVolume(vol) if err != nil { return err } reverter.Add(func() { _ = d.rbdUnmapVolume(vol, true) }) } else { parentVol := srcVol snapshotName := "readonly" if srcVol.volType != VolumeTypeImage { snapshotName = fmt.Sprintf("zombie_snapshot_%s", uuid.New().String()) if srcVol.IsSnapshot() { srcParentName, srcSnapOnlyName, _ := api.GetParentAndSnapshotName(srcVol.name) snapshotName = fmt.Sprintf("snapshot_%s", srcSnapOnlyName) parentVol = NewVolume(d, d.name, srcVol.volType, srcVol.contentType, srcParentName, nil, nil) } else { // Create snapshot. err := d.rbdCreateVolumeSnapshot(srcVol, snapshotName) if err != nil { return err } } // Protect volume so we can create clones of it. err = d.rbdProtectVolumeSnapshot(parentVol, snapshotName) if err != nil { return err } reverter.Add(func() { _ = d.rbdUnprotectVolumeSnapshot(parentVol, snapshotName) }) } err = d.rbdCreateClone(parentVol, snapshotName, vol) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) } err = postCreateTasks(vol) if err != nil { return err } reverter.Success() return nil } // Copy with snapshots. // Create empty placeholder volume err = d.rbdCreateVolume(vol, "0") if err != nil { return err } reverter.Add(func() { _ = d.rbdDeleteVolume(vol) }) // Receive over the placeholder volume we created above. targetVolumeName := d.getRBDVolumeName(vol, "", true) lastSnap := "" if len(snapshots) > 0 { err := CreateParentSnapshotDirIfMissing(d.name, vol.volType, vol.name) if err != nil { return err } } for i, snap := range snapshots { prev := "" if i > 0 { prev = fmt.Sprintf("snapshot_%s", snapshots[i-1]) } lastSnap = fmt.Sprintf("snapshot_%s", snap) sourceVolumeName := d.getRBDVolumeName(srcVol, lastSnap, true) err = d.copyWithSnapshots(sourceVolumeName, targetVolumeName, prev) if err != nil { return err } reverter.Add(func() { _ = d.rbdDeleteVolumeSnapshot(vol, snap) }) snapVol, err := vol.NewSnapshot(snap) if err != nil { return err } err = snapVol.EnsureMountPath(false) if err != nil { return err } } // Copy snapshot. sourceVolumeName := d.getRBDVolumeName(srcVol, "", true) err = d.copyWithSnapshots(sourceVolumeName, targetVolumeName, lastSnap) if err != nil { return err } err = postCreateTasks(vol) if err != nil { return err } reverter.Success() return nil } // CreateVolumeFromMigration creates a volume being sent via a migration. func (d *ceph) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, volTargetArgs localMigration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error { if volTargetArgs.ClusterMoveSourceName != "" && volTargetArgs.StoragePool == "" { err := vol.EnsureMountPath(false) if err != nil { return err } if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.CreateVolumeFromMigration(fsVol, conn, volTargetArgs, preFiller, op) if err != nil { return err } } return nil } // Handle simple rsync and block_and_rsync through generic. if volTargetArgs.MigrationType.FSType == migration.MigrationFSType_RSYNC || volTargetArgs.MigrationType.FSType == migration.MigrationFSType_BLOCK_AND_RSYNC { return genericVFSCreateVolumeFromMigration(d, nil, vol, conn, volTargetArgs, preFiller, op) } else if volTargetArgs.MigrationType.FSType != migration.MigrationFSType_RBD { return ErrNotSupported } if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.CreateVolumeFromMigration(fsVol, conn, volTargetArgs, preFiller, op) if err != nil { return err } } recvName := d.getRBDVolumeName(vol, "", true) volExists, err := d.HasVolume(vol) if err != nil { return err } if !volExists { err := d.rbdCreateVolume(vol, "0") if err != nil { return err } } err = vol.EnsureMountPath(false) if err != nil { return err } // Handle rbd migration. if len(volTargetArgs.Snapshots) > 0 { // Create the parent directory. err := CreateParentSnapshotDirIfMissing(d.name, vol.volType, vol.name) if err != nil { return err } // Transfer the snapshots. for _, snapshot := range volTargetArgs.Snapshots { fullSnapshotName := d.getRBDVolumeName(vol, snapshot.GetName(), true) wrapper := localMigration.ProgressWriter(op, "fs_progress", fullSnapshotName) err = d.receiveVolume(recvName, conn, wrapper) if err != nil { return err } snapVol, err := vol.NewSnapshot(snapshot.GetName()) if err != nil { return err } err = snapVol.EnsureMountPath(false) if err != nil { return err } } } defer func() { // Delete all migration-send-* snapshots. snaps, err := d.rbdListVolumeSnapshots(vol) if err != nil { return } for _, snap := range snaps { if !strings.HasPrefix(snap, "migration-send") { continue } _ = d.rbdDeleteVolumeSnapshot(vol, snap) } }() wrapper := localMigration.ProgressWriter(op, "fs_progress", vol.name) err = d.receiveVolume(recvName, conn, wrapper) if err != nil { return err } // Map the RBD volume. devPath, err := d.rbdMapVolume(vol) if err != nil { return err } defer func() { _ = d.rbdUnmapVolume(vol, true) }() // Re-generate the UUID. err = d.generateUUID(vol.ConfigBlockFilesystem(), devPath) if err != nil { return err } return nil } // RefreshVolume updates an existing volume to match the state of another. func (d *ceph) RefreshVolume(vol Volume, srcVol Volume, srcSnapshots []Volume, allowInconsistent bool, op *operations.Operation) error { return genericVFSCopyVolume(d, nil, vol, srcVol, srcSnapshots, true, allowInconsistent, op) } // DeleteVolume deletes a volume of the storage device. If any snapshots of the volume remain then // this function will return an error. func (d *ceph) DeleteVolume(vol Volume, op *operations.Operation) error { volExists, err := d.HasVolume(vol) if err != nil { return err } if !volExists { return nil } if vol.volType == VolumeTypeImage { // Unmount and unmap. _, err := d.UnmountVolume(vol, false, op) if err != nil { return err } hasReadonlySnapshot, err := d.hasVolume(d.getRBDVolumeName(vol, "readonly", false)) if err != nil { return err } hasDependentSnapshots := false if hasReadonlySnapshot { dependentSnapshots, err := d.rbdListSnapshotClones(vol, "readonly") if err != nil && !response.IsNotFoundError(err) { return err } hasDependentSnapshots = len(dependentSnapshots) > 0 } if hasDependentSnapshots { // If the image has dependent snapshots, then we just mark it as deleted, but don't // actually remove it yet. err = d.rbdMarkVolumeDeleted(vol, vol.name) if err != nil { return err } } else { if hasReadonlySnapshot { // Unprotect snapshot. err := d.rbdUnprotectVolumeSnapshot(vol, "readonly") if err != nil { return err } } // Delete snapshots. _, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "snap", "purge", d.getRBDVolumeName(vol, "", false)) if err != nil { return err } // Delete image. err = d.rbdDeleteVolume(vol) if err != nil { return err } } } else { // Unmount and unmap. _, err := d.UnmountVolume(vol, false, op) if err != nil { return err } _, err = d.deleteVolume(vol) if err != nil { return fmt.Errorf("Failed to delete volume: %w", err) } } if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.DeleteVolume(fsVol, op) if err != nil { return err } } mountPath := vol.MountPath() if vol.contentType == ContentTypeFS && util.PathExists(mountPath) { err := wipeDirectory(mountPath) if err != nil { return err } err = os.Remove(mountPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", mountPath, err) } } return nil } // hasVolume indicates whether a specific RBD volume exists on the storage pool. func (d *ceph) hasVolume(rbdVolumeName string) (bool, error) { ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second) defer cancel() _, err := subprocess.RunCommandContext(ctx, "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "info", rbdVolumeName, ) if err != nil { var runErr subprocess.RunError if errors.As(err, &runErr) { var exitError *exec.ExitError if errors.As(runErr.Unwrap(), &exitError) { if exitError.ExitCode() == 2 { return false, nil } } } return false, err } return true, nil } // HasVolume indicates whether a specific volume exists on the storage pool. func (d *ceph) HasVolume(vol Volume) (bool, error) { return d.hasVolume(d.getRBDVolumeName(vol, "", false)) } // FillVolumeConfig populate volume with default config. func (d *ceph) FillVolumeConfig(vol Volume) error { // Copy volume.* configuration options from pool. // Exclude 'block.filesystem' and 'block.mount_options' // as this ones are handled below in this function and depends from volume type err := d.fillVolumeConfig(&vol, "block.filesystem", "block.mount_options") if err != nil { return err } // Only validate filesystem config keys for filesystem volumes or VM block volumes (which have an // associated filesystem volume). if vol.ContentType() == ContentTypeFS || vol.IsVMBlock() { // Inherit filesystem from pool if not set. if vol.config["block.filesystem"] == "" { vol.config["block.filesystem"] = d.config["volume.block.filesystem"] } // Default filesystem if neither volume nor pool specify an override. if vol.config["block.filesystem"] == "" { // Unchangeable volume property: Set unconditionally. vol.config["block.filesystem"] = DefaultFilesystem } // Inherit filesystem mount options from pool if not set. if vol.config["block.mount_options"] == "" { vol.config["block.mount_options"] = d.config["volume.block.mount_options"] } // Default filesystem mount options if neither volume nor pool specify an override. if vol.config["block.mount_options"] == "" { // Unchangeable volume property: Set unconditionally. vol.config["block.mount_options"] = "discard" } } return nil } // commonVolumeRules returns validation rules which are common for pool and volume. func (d *ceph) commonVolumeRules() map[string]func(value string) error { return map[string]func(value string) error{ // gendoc:generate(entity=storage_volume_ceph, group=common, key=block.filesystem) // // --- // type: string // condition: block-based volume with content type `filesystem` // default: same as `volume.block.filesystem` // shortdesc: {{block_filesystem}} "block.filesystem": validate.Optional(validate.IsOneOf(blockBackedAllowedFilesystems...)), // gendoc:generate(entity=storage_volume_ceph, group=common, key=block.mount_options) // // --- // type: string // condition: block-based volume with content type `filesystem` // default: same as `volume.block.mount_options` // shortdesc: Mount options for block-backed file system volumes "block.mount_options": validate.IsAny, } } // ValidateVolume validates the supplied volume config. func (d *ceph) ValidateVolume(vol Volume, removeUnknownKeys bool) error { // gendoc:generate(entity=storage_volume_ceph, group=common, key=initial.gid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.gid` or `0` // shortdesc: GID of the volume owner in the instance // gendoc:generate(entity=storage_volume_ceph, group=common, key=initial.mode) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.mode` or `711` // shortdesc: Mode of the volume in the instance // gendoc:generate(entity=storage_volume_ceph, group=common, key=initial.uid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.uid` or `0` // shortdesc: UID of the volume owner in the instance // gendoc:generate(entity=storage_volume_ceph, group=common, key=security.shared) // // --- // type: bool // condition: custom block volume // default: same as `volume.security.shared` or `false` // shortdesc: Enable sharing the volume across multiple instances // gendoc:generate(entity=storage_volume_ceph, group=common, key=security.shifted) // // --- // type: bool // condition: custom volume // default: same as `volume.security.shifted` or `false` // shortdesc: {{enable_ID_shifting}} // gendoc:generate(entity=storage_volume_ceph, group=common, key=security.unmapped) // // --- // type: bool // condition: custom volume // default: same as `volume.security.unmapped` or `false` // shortdesc: Disable ID mapping for the volume // gendoc:generate(entity=storage_volume_ceph, group=common, key=size) // // --- // type: string // condition: - // default: same as `volume.size` // shortdesc: Size/quota of the storage volume // gendoc:generate(entity=storage_volume_ceph, group=common, key=snapshots.expiry) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_ceph, group=common, key=snapshots.expiry.manual) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry.manual` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_ceph, group=common, key=snapshots.pattern) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.pattern` or `snap%d` // shortdesc: {{snapshot_pattern_format}} [^*] // gendoc:generate(entity=storage_volume_ceph, group=common, key=snapshots.schedule) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.schedule` // shortdesc: {{snapshot_schedule_format}} commonRules := d.commonVolumeRules() // Disallow block.* settings for regular custom block volumes. These settings only make sense // when using custom filesystem volumes. Incus will create the filesystem // for these volumes, and use the mount options. When attaching a regular block volume to a VM, // these are not mounted by Incus and therefore don't need these config keys. if vol.IsVMBlock() || vol.volType == VolumeTypeCustom && vol.contentType == ContentTypeBlock { delete(commonRules, "block.filesystem") delete(commonRules, "block.mount_options") } return d.validateVolume(vol, commonRules, removeUnknownKeys) } // UpdateVolume applies config changes to the volume. func (d *ceph) UpdateVolume(vol Volume, changedConfig map[string]string) error { newSize, sizeChanged := changedConfig["size"] if sizeChanged { err := d.SetVolumeQuota(vol, newSize, false, nil) if err != nil { return err } } return d.updateVolume(vol, changedConfig) } // GetVolumeUsage returns the disk space used by the volume. func (d *ceph) GetVolumeUsage(vol Volume) (int64, error) { isSnap := vol.IsSnapshot() // If mounted, use the filesystem stats for pretty accurate usage information. if !isSnap && vol.contentType == ContentTypeFS && linux.IsMountPoint(vol.MountPath()) { var stat unix.Statfs_t err := unix.Statfs(vol.MountPath(), &stat) if err != nil { return -1, err } return int64(stat.Blocks-stat.Bfree) * int64(stat.Bsize), nil } // Running rbd du can be resource intensive, so users may want to miss disk usage // data for stopped instances instead of dealing with the performance hit if util.IsFalse(d.config["ceph.rbd.du"]) { return -1, ErrNotSupported } // If not mounted (or not mountable), query the usage from ceph directly. // This is rather inaccurate as there is no way to get the size of a // volume without its snapshots. Instead we need to merge the size // of all snapshots with the delta since last snapshot. This leads to // volumes with lots of changes between snapshots potentially adding up far // more usage than they actually have. type cephDuLine struct { Name string `json:"name"` Snapshot string `json:"snapshot"` ProvisionedSize int64 `json:"provisioned_size"` UsedSize int64 `json:"used_size"` } type cephDuInfo struct { Images []cephDuLine `json:"images"` } ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second) defer cancel() jsonInfo, err := subprocess.RunCommandContext(ctx, "rbd", "du", "--format", "json", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], d.getRBDVolumeName(vol, "", false), ) if err != nil { return -1, err } var usedSize int64 var result cephDuInfo err = json.Unmarshal([]byte(jsonInfo), &result) if err != nil { return -1, err } _, snapName, _ := api.GetParentAndSnapshotName(vol.Name()) snapName = fmt.Sprintf("snapshot_%s", snapName) // rbd du gives the output of all related rbd images, snapshots included. for _, image := range result.Images { if isSnap { // For snapshot volumes we only want to get the specific image used so we can // indicate how much CoW usage that snapshot has. if image.Snapshot == snapName { usedSize = image.UsedSize break } } else { // For non-snapshot volumes, to get the total size of the volume we need to add up // all of the image's usage. usedSize += image.UsedSize } } return usedSize, nil } // SetVolumeQuota applies a size limit on volume. // Does nothing if supplied with an empty/zero size. func (d *ceph) SetVolumeQuota(vol Volume, size string, allowUnsafeResize bool, op *operations.Operation) error { // Convert to bytes. sizeBytes, err := units.ParseByteSizeString(size) if err != nil { return err } // Do nothing if size isn't specified. if sizeBytes <= 0 { return nil } ourMap, devPath, err := d.getRBDMappedDevPath(vol, true) if err != nil { return err } if ourMap { defer func() { _ = d.rbdUnmapVolume(vol, true) }() } oldSizeBytes, err := BlockDiskSizeBytes(devPath) if err != nil { return fmt.Errorf("Error getting current size: %w", err) } // Do nothing if volume is already specified size (+/- 512 bytes). if oldSizeBytes+512 > sizeBytes && oldSizeBytes-512 < sizeBytes { return nil } // Block image volumes cannot be resized because they have a readonly snapshot that doesn't get // updated when the volume's size is changed, and this is what instances are created from. // During initial volume fill allowUnsafeResize is enabled because snapshot hasn't been taken yet. if !allowUnsafeResize && vol.volType == VolumeTypeImage { return ErrNotSupported } inUse := vol.MountInUse() // Resize filesystem if needed. if vol.contentType == ContentTypeFS { fsType := vol.ConfigBlockFilesystem() if sizeBytes < oldSizeBytes { if !filesystemTypeCanBeShrunk(fsType) { return fmt.Errorf("Filesystem %q cannot be shrunk: %w", fsType, ErrCannotBeShrunk) } if inUse { return ErrInUse // We don't allow online shrinking of filesystem volumes. } // Shrink filesystem first. Pass allowUnsafeResize to allow disabling of filesystem // resize safety checks. err = shrinkFileSystem(fsType, devPath, vol, sizeBytes, allowUnsafeResize) if err != nil { return err } // Shrink the block device. err = d.resizeVolume(vol, sizeBytes, true) if err != nil { return err } } else if sizeBytes > oldSizeBytes { // Grow block device first. err = d.resizeVolume(vol, sizeBytes, false) if err != nil { return err } // Grow the filesystem to fill block device. err = growFileSystem(fsType, devPath, vol) if err != nil { return err } } } else { // Only perform pre-resize checks if we are not in "unsafe" mode. // In unsafe mode we expect the caller to know what they are doing and understand the risks. if !allowUnsafeResize { if sizeBytes < oldSizeBytes { return fmt.Errorf("Block volumes cannot be shrunk: %w", ErrCannotBeShrunk) } if inUse { return ErrInUse // We don't allow online resizing of block volumes. } } // Resize block device. err = d.resizeVolume(vol, sizeBytes, allowUnsafeResize) if err != nil { return err } // Move the VM GPT alt header to end of disk if needed (not needed in unsafe resize mode as it is // expected the caller will do all necessary post resize actions themselves). if vol.IsVMBlock() && !allowUnsafeResize { err = d.moveGPTAltHeader(devPath) if err != nil { return err } } } return nil } // GetVolumeDiskPath returns the location of a root disk block device. func (d *ceph) GetVolumeDiskPath(vol Volume) (string, error) { if vol.IsVMBlock() || (vol.volType == VolumeTypeCustom && IsContentBlock(vol.contentType)) { _, devPath, err := d.getRBDMappedDevPath(vol, false) return devPath, err } return "", ErrNotSupported } // ListVolumes returns a list of volumes in storage pool. func (d *ceph) ListVolumes() ([]Volume, error) { vols := make(map[string]Volume) cmd := exec.Command("rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "ls", ) stdout, err := cmd.StdoutPipe() if err != nil { return nil, err } stderr, err := cmd.StderrPipe() if err != nil { return nil, err } err = cmd.Start() if err != nil { return nil, err } scanner := bufio.NewScanner(stdout) for scanner.Scan() { rawName := strings.TrimSpace(scanner.Text()) var volType VolumeType var volName string for _, volumeType := range d.Info().VolumeTypes { prefix := cephVolTypePrefixes[volumeType] if prefix == "" { continue // Unknown volume type. } prefix = fmt.Sprintf("%s_", prefix) if strings.HasPrefix(rawName, prefix) { volType = volumeType volName = strings.TrimPrefix(rawName, prefix) } } if volType == "" { d.logger.Debug("Ignoring unrecognised volume type", logger.Ctx{"name": rawName}) continue // Ignore unrecognised volume. } isBlock := strings.HasSuffix(volName, cephBlockVolSuffix) if volType == VolumeTypeVM && !isBlock { continue // Ignore VM filesystem volumes as we will just return the VM's block volume. } contentType := ContentTypeFS if volType == VolumeTypeCustom && strings.HasSuffix(volName, cephISOVolSuffix) { contentType = ContentTypeISO volName = strings.TrimSuffix(volName, cephISOVolSuffix) } else if volType == VolumeTypeVM || isBlock { contentType = ContentTypeBlock volName = strings.TrimSuffix(volName, cephBlockVolSuffix) } // If a new volume has been found, or the volume will replace an existing image filesystem volume // then proceed to add the volume to the map. We allow image volumes to overwrite existing // filesystem volumes of the same name so that for VM images we only return the block content type // volume (so that only the single "logical" volume is returned). existingVol, foundExisting := vols[volName] if !foundExisting || (existingVol.Type() == VolumeTypeImage && existingVol.ContentType() == ContentTypeFS) { v := NewVolume(d, d.name, volType, contentType, volName, make(map[string]string), d.config) if contentType == ContentTypeFS { v.SetMountFilesystemProbe(true) } vols[volName] = v continue } return nil, fmt.Errorf("Unexpected duplicate volume %q found", volName) } errMsg, err := io.ReadAll(stderr) if err != nil { return nil, err } err = cmd.Wait() if err != nil { return nil, fmt.Errorf("Failed getting volume list: %v: %w", strings.TrimSpace(string(errMsg)), err) } volList := make([]Volume, 0, len(vols)) for _, v := range vols { volList = append(volList, v) } return volList, nil } // MountVolume mounts a volume and increments ref counter. Please call UnmountVolume() when done with the volume. func (d *ceph) MountVolume(vol Volume, op *operations.Operation) error { unlock, err := vol.MountLock() if err != nil { return err } defer unlock() reverter := revert.New() defer reverter.Fail() // Activate RBD volume if needed. activated, volDevPath, err := d.getRBDMappedDevPath(vol, true) if err != nil { return err } if activated { reverter.Add(func() { _ = d.rbdUnmapVolume(vol, true) }) } if vol.contentType == ContentTypeFS { mountPath := vol.MountPath() if !linux.IsMountPoint(mountPath) { err := vol.EnsureMountPath(false) if err != nil { return err } fsType := vol.ConfigBlockFilesystem() if vol.mountFilesystemProbe { fsType, err = fsProbe(volDevPath) if err != nil { return fmt.Errorf("Failed probing filesystem: %w", err) } } mountFlags, mountOptions := linux.ResolveMountOptions(strings.Split(vol.ConfigBlockMountOptions(), ",")) err = TryMount(volDevPath, mountPath, fsType, mountFlags, mountOptions) if err != nil { return err } d.logger.Debug("Mounted RBD volume", logger.Ctx{"volName": vol.name, "dev": volDevPath, "path": mountPath, "options": mountOptions}) } } else if vol.contentType == ContentTypeBlock { // For VMs, mount the filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err = d.MountVolume(fsVol, op) if err != nil { return err } } } vol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolume() when done. reverter.Success() return nil } // UnmountVolume simulates unmounting a volume. // keepBlockDev indicates if backing block device should be not be unmapped if volume is unmounted. func (d *ceph) UnmountVolume(vol Volume, keepBlockDev bool, op *operations.Operation) (bool, error) { unlock, err := vol.MountLock() if err != nil { return false, err } defer unlock() ourUnmount := false mountPath := vol.MountPath() refCount := vol.MountRefCountDecrement() // Attempt to unmount the volume. if vol.contentType == ContentTypeFS && linux.IsMountPoint(mountPath) { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": vol.name, "refCount": refCount}) return false, ErrInUse } err = TryUnmount(mountPath, unix.MNT_DETACH) if err != nil { return false, err } d.logger.Debug("Unmounted RBD volume", logger.Ctx{"volName": vol.name, "path": mountPath, "keepBlockDev": keepBlockDev}) // Attempt to unmap. if !keepBlockDev { err = d.rbdUnmapVolume(vol, true) if err != nil { return false, err } } ourUnmount = true } else if IsContentBlock(vol.contentType) { // For VMs, unmount the filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() ourUnmount, err = d.UnmountVolume(fsVol, false, op) if err != nil { return false, err } } if !keepBlockDev { // Check if device is currently mapped (but don't map if not). _, devPath, _ := d.getRBDMappedDevPath(vol, false) if devPath != "" && util.PathExists(devPath) { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": vol.name, "refCount": refCount}) return false, ErrInUse } // Attempt to unmap. err := d.rbdUnmapVolume(vol, true) if err != nil { return false, err } ourUnmount = true } } } return ourUnmount, nil } // RenameVolume renames a volume and its snapshots. func (d *ceph) RenameVolume(vol Volume, newVolName string, op *operations.Operation) error { return vol.UnmountTask(func(op *operations.Operation) error { reverter := revert.New() defer reverter.Fail() err := d.rbdRenameVolume(vol, newVolName) if err != nil { return err } newVol := NewVolume(d, d.name, vol.volType, vol.contentType, newVolName, nil, nil) reverter.Add(func() { _ = d.rbdRenameVolume(newVol, vol.name) }) // Rename volume dir. if vol.contentType == ContentTypeFS { err = genericVFSRenameVolume(d, vol, newVolName, op) if err != nil { return err } } // For VMs, also rename the filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err = d.RenameVolume(fsVol, newVolName, op) if err != nil { return err } } reverter.Success() return nil }, false, op) } // MigrateVolume sends a volume for migration. func (d *ceph) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *localMigration.VolumeSourceArgs, op *operations.Operation) error { if volSrcArgs.ClusterMove && !volSrcArgs.StorageMove { return nil // When performing a cluster member move don't do anything on the source member. } // Handle simple rsync and block_and_rsync through generic. if volSrcArgs.MigrationType.FSType == migration.MigrationFSType_RSYNC || volSrcArgs.MigrationType.FSType == migration.MigrationFSType_BLOCK_AND_RSYNC { // TODO this should take a temporary snapshot. // Before doing a generic volume migration, we need to ensure volume (or snap volume parent) is // activated to avoid issues activating the snapshot volume device. parent, _, _ := api.GetParentAndSnapshotName(vol.Name()) parentVol := NewVolume(d, d.Name(), vol.volType, vol.contentType, parent, vol.config, vol.poolConfig) err := d.MountVolume(parentVol, op) if err != nil { return err } defer func() { _, _ = d.UnmountVolume(parentVol, false, op) }() return genericVFSMigrateVolume(d, d.state, vol, conn, volSrcArgs, op) } else if volSrcArgs.MigrationType.FSType != migration.MigrationFSType_RBD { return ErrNotSupported } // Handle rbd export-diff/import-diff migration. if volSrcArgs.MultiSync || volSrcArgs.FinalSync { // This is not needed if the migration is performed using rbd export-diff/import-diff. return errors.New("MultiSync should not be used with optimized migration") } if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.MigrateVolume(fsVol, conn, volSrcArgs, op) if err != nil { return err } } if vol.IsSnapshot() { parentName, snapOnlyName, _ := api.GetParentAndSnapshotName(vol.name) sendName := fmt.Sprintf("%s/snapshots_%s_%s_start_clone", d.name, parentName, snapOnlyName) cloneVol := NewVolume(d, d.name, vol.volType, vol.contentType, vol.name, nil, nil) // Mounting the volume snapshot will create the clone "snapshots___start_clone". err := d.MountVolumeSnapshot(cloneVol, op) if err != nil { return err } defer func() { _, _ = d.UnmountVolumeSnapshot(cloneVol, op) }() // Setup progress tracking. var wrapper *ioprogress.ProgressTracker if volSrcArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", vol.name) } err = d.sendVolume(conn, sendName, "", wrapper) if err != nil { return err } return nil } lastSnap := "" for i, snapName := range volSrcArgs.Snapshots { snapshot, _ := vol.NewSnapshot(snapName) prev := "" if i > 0 { prev = fmt.Sprintf("snapshot_%s", volSrcArgs.Snapshots[i-1]) } lastSnap = fmt.Sprintf("snapshot_%s", snapName) sendSnapName := d.getRBDVolumeName(vol, lastSnap, true) // Setup progress tracking. var wrapper *ioprogress.ProgressTracker if volSrcArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", snapshot.name) } err := d.sendVolume(conn, sendSnapName, prev, wrapper) if err != nil { return err } } // Setup progress tracking. var wrapper *ioprogress.ProgressTracker if volSrcArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", vol.name) } runningSnapName := fmt.Sprintf("migration-send-%s", uuid.New().String()) err := d.rbdCreateVolumeSnapshot(vol, runningSnapName) if err != nil { return err } defer func() { _ = d.rbdDeleteVolumeSnapshot(vol, runningSnapName) }() cur := d.getRBDVolumeName(vol, runningSnapName, true) err = d.sendVolume(conn, cur, lastSnap, wrapper) if err != nil { return err } return nil } // BackupVolume creates an exported version of a volume. func (d *ceph) BackupVolume(vol Volume, writer instancewriter.InstanceWriter, basePrefix string, optimized bool, snapshots []string, op *operations.Operation) error { return genericVFSBackupVolume(d, vol, writer, basePrefix, snapshots, op) } // CreateVolumeSnapshot creates a snapshot of a volume. func (d *ceph) CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error { reverter := revert.New() defer reverter.Fail() parentName, snapshotOnlyName, _ := api.GetParentAndSnapshotName(snapVol.name) sourcePath := GetVolumeMountPath(d.name, snapVol.volType, parentName) snapshotName := fmt.Sprintf("snapshot_%s", snapshotOnlyName) if linux.IsMountPoint(sourcePath) { // Attempt to sync and freeze filesystem, but do not error if not able to freeze (as filesystem // could still be busy), as we do not guarantee the consistency of a snapshot. This is costly but // try to ensure that all cached data has been committed to disk. If we don't then the rbd snapshot // of the underlying filesystem can be inconsistent or, in the worst case, empty. unfreezeFS, err := d.filesystemFreeze(sourcePath) if err == nil { defer func() { _ = unfreezeFS() }() } } // Create the parent directory. err := CreateParentSnapshotDirIfMissing(d.name, snapVol.volType, parentName) if err != nil { return err } err = snapVol.EnsureMountPath(false) if err != nil { return err } parentVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, nil, nil) err = d.rbdCreateVolumeSnapshot(parentVol, snapshotName) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolumeSnapshot(snapVol, op) }) // For VM images, create a filesystem volume too. if snapVol.IsVMBlock() { fsVol := snapVol.NewVMBlockFilesystemVolume() err := d.CreateVolumeSnapshot(fsVol, op) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolumeSnapshot(fsVol, op) }) } reverter.Success() return nil } // DeleteVolumeSnapshot removes a snapshot from the storage device. func (d *ceph) DeleteVolumeSnapshot(snapVol Volume, op *operations.Operation) error { // Check if snapshot exists, and return if not. _, err := subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "info", d.getRBDVolumeName(snapVol, "", false)) if err != nil { return nil } parentName, snapshotOnlyName, _ := api.GetParentAndSnapshotName(snapVol.name) snapshotName := fmt.Sprintf("snapshot_%s", snapshotOnlyName) parentVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, nil, nil) _, err = d.deleteVolumeSnapshot(parentVol, snapshotName) if err != nil { return fmt.Errorf("Failed to delete volume snapshot: %w", err) } mountPath := snapVol.MountPath() if snapVol.contentType == ContentTypeFS && util.PathExists(mountPath) { err = wipeDirectory(mountPath) if err != nil { return err } err = os.Remove(mountPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", mountPath, err) } } // Remove the parent snapshot directory if this is the last snapshot being removed. err = deleteParentSnapshotDirIfEmpty(d.name, snapVol.volType, parentName) if err != nil { return err } // For VM images, delete the filesystem volume too. if snapVol.IsVMBlock() { fsVol := snapVol.NewVMBlockFilesystemVolume() err := d.DeleteVolumeSnapshot(fsVol, op) if err != nil { return err } } return nil } // MountVolumeSnapshot simulates mounting a volume snapshot. func (d *ceph) MountVolumeSnapshot(snapVol Volume, op *operations.Operation) error { unlock, err := snapVol.MountLock() if err != nil { return err } defer unlock() reverter := revert.New() defer reverter.Fail() mountPath := snapVol.MountPath() if snapVol.contentType == ContentTypeFS && !linux.IsMountPoint(mountPath) { err := snapVol.EnsureMountPath(false) if err != nil { return err } parentName, snapshotOnlyName, _ := api.GetParentAndSnapshotName(snapVol.name) prefixedSnapOnlyName := fmt.Sprintf("snapshot_%s", snapshotOnlyName) parentVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, nil, nil) // Protect snapshot to prevent data loss. err = d.rbdProtectVolumeSnapshot(parentVol, prefixedSnapOnlyName) if err != nil { return err } reverter.Add(func() { _ = d.rbdUnprotectVolumeSnapshot(parentVol, prefixedSnapOnlyName) }) // Clone snapshot. cloneName := fmt.Sprintf("%s_%s_start_clone", parentName, snapshotOnlyName) cloneVol := NewVolume(d, d.name, VolumeType("snapshots"), ContentTypeFS, cloneName, nil, nil) err = d.rbdCreateClone(parentVol, prefixedSnapOnlyName, cloneVol) if err != nil { return err } reverter.Add(func() { _ = d.rbdDeleteVolume(cloneVol) }) // Map volume. rbdDevPath, err := d.rbdMapVolume(cloneVol) if err != nil { return err } reverter.Add(func() { _ = d.rbdUnmapVolume(cloneVol, true) }) RBDFilesystem := snapVol.ConfigBlockFilesystem() mountFlags, mountOptions := linux.ResolveMountOptions(strings.Split(snapVol.ConfigBlockMountOptions(), ",")) if renegerateFilesystemUUIDNeeded(RBDFilesystem) { if RBDFilesystem == "xfs" { idx := strings.Index(mountOptions, "nouuid") if idx < 0 { mountOptions += ",nouuid" } } else { err = d.generateUUID(RBDFilesystem, rbdDevPath) if err != nil { return err } } } err = TryMount(rbdDevPath, mountPath, RBDFilesystem, mountFlags, mountOptions) if err != nil { return err } d.logger.Debug("Mounted RBD volume snapshot", logger.Ctx{"dev": rbdDevPath, "path": mountPath, "options": mountOptions}) } else if snapVol.contentType == ContentTypeBlock { // Activate RBD volume if needed. _, _, err := d.getRBDMappedDevPath(snapVol, true) if err != nil { return err } // For VMs, mount the filesystem volume. if snapVol.IsVMBlock() { fsVol := snapVol.NewVMBlockFilesystemVolume() err = d.MountVolumeSnapshot(fsVol, op) if err != nil { return err } } } snapVol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolumeSnapshot() when done. reverter.Success() return nil } // UnmountVolumeSnapshot simulates unmounting a volume snapshot. func (d *ceph) UnmountVolumeSnapshot(snapVol Volume, op *operations.Operation) (bool, error) { unlock, err := snapVol.MountLock() if err != nil { return false, err } defer unlock() ourUnmount := false mountPath := snapVol.MountPath() refCount := snapVol.MountRefCountDecrement() if snapVol.contentType == ContentTypeFS && linux.IsMountPoint(mountPath) { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": snapVol.name, "refCount": refCount}) return false, ErrInUse } err = TryUnmount(mountPath, unix.MNT_DETACH) if err != nil { return false, err } d.logger.Debug("Unmounted RBD volume snapshot", logger.Ctx{"path": mountPath}) parentName, snapshotOnlyName, _ := api.GetParentAndSnapshotName(snapVol.name) cloneName := fmt.Sprintf("%s_%s_start_clone", parentName, snapshotOnlyName) cloneVol := NewVolume(d, d.name, VolumeType("snapshots"), ContentTypeFS, cloneName, nil, nil) err = d.rbdUnmapVolume(cloneVol, true) if err != nil { return false, err } volExists, err := d.HasVolume(cloneVol) if err != nil { return false, err } if !volExists { return true, nil } // Delete the temporary RBD volume. err = d.rbdDeleteVolume(cloneVol) if err != nil { return false, err } ourUnmount = true } else if snapVol.contentType == ContentTypeBlock { if snapVol.IsVMBlock() { fsVol := snapVol.NewVMBlockFilesystemVolume() ourUnmount, err = d.UnmountVolumeSnapshot(fsVol, op) if err != nil { return false, err } } // Check if device is currently mapped (but don't map if not). _, devPath, _ := d.getRBDMappedDevPath(snapVol, false) if devPath != "" && util.PathExists(devPath) { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": snapVol.name, "refCount": refCount}) return false, ErrInUse } err := d.rbdUnmapVolume(snapVol, true) if err != nil { return false, err } ourUnmount = true } } return ourUnmount, nil } // VolumeSnapshots returns a list of snapshots for the volume (in no particular order). func (d *ceph) VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) { snapshots, err := d.rbdListVolumeSnapshots(vol) if err != nil { if response.IsNotFoundError(err) { return nil, nil } return nil, err } var ret []string for _, snap := range snapshots { // Ignore zombie snapshots as these are only used internally and // not relevant for users. if strings.HasPrefix(snap, "zombie_") || strings.HasPrefix(snap, "migration-send-") { continue } ret = append(ret, strings.TrimPrefix(snap, "snapshot_")) } return ret, nil } // RestoreVolume restores a volume from a snapshot. func (d *ceph) RestoreVolume(vol Volume, snapshotName string, op *operations.Operation) error { ourUnmount, err := d.UnmountVolume(vol, false, op) if err != nil { return err } if ourUnmount { defer func() { _ = d.MountVolume(vol, op) }() } _, err = subprocess.RunCommand( "rbd", "--id", d.config["ceph.user.name"], "--cluster", d.config["ceph.cluster_name"], "--pool", d.config["ceph.osd.pool_name"], "snap", "rollback", "--snap", fmt.Sprintf("snapshot_%s", snapshotName), d.getRBDVolumeName(vol, "", false)) if err != nil { return err } snapVol, err := vol.NewSnapshot(snapshotName) if err != nil { return err } // Map the RBD volume. devPath, err := d.rbdMapVolume(snapVol) if err != nil { return err } defer func() { _ = d.rbdUnmapVolume(snapVol, true) }() // Re-generate the UUID. err = d.generateUUID(snapVol.ConfigBlockFilesystem(), devPath) if err != nil { return err } return nil } // RenameVolumeSnapshot renames a volume snapshot. func (d *ceph) RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *operations.Operation) error { reverter := revert.New() defer reverter.Fail() parentName, snapshotOnlyName, _ := api.GetParentAndSnapshotName(snapVol.name) oldSnapOnlyName := fmt.Sprintf("snapshot_%s", snapshotOnlyName) newSnapOnlyName := fmt.Sprintf("snapshot_%s", newSnapshotName) parentVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, nil, nil) err := d.rbdRenameVolumeSnapshot(parentVol, oldSnapOnlyName, newSnapOnlyName) if err != nil { return err } reverter.Add(func() { _ = d.rbdRenameVolumeSnapshot(parentVol, newSnapOnlyName, oldSnapOnlyName) }) if snapVol.contentType == ContentTypeFS { err = genericVFSRenameVolumeSnapshot(d, snapVol, newSnapshotName, op) if err != nil { return err } } // For VM images, create a filesystem volume too. if snapVol.IsVMBlock() { fsVol := snapVol.NewVMBlockFilesystemVolume() err := d.RenameVolumeSnapshot(fsVol, newSnapshotName, op) if err != nil { return err } reverter.Add(func() { newFsVol := NewVolume(d, d.name, snapVol.volType, ContentTypeFS, fmt.Sprintf("%s/%s", parentName, newSnapshotName), snapVol.config, snapVol.poolConfig) _ = d.RenameVolumeSnapshot(newFsVol, snapVol.name, op) }) } reverter.Success() return nil } // ActivateTask allows running a function while the volume is active (but not mounted). func (d *ceph) ActivateTask(vol Volume, task func(devPath string, op *operations.Operation) error, op *operations.Operation) error { // Prevent concurrent mounting actions. unlock, err := vol.MountLock() if err != nil { return err } defer unlock() // Activate RBD volume if needed. activated, volDevPath, err := d.getRBDMappedDevPath(vol, true) if err != nil { return err } if !activated { return errors.New("Volume is already active, can't run exclusive activation task") } // Run the task. taskErr := task(volDevPath, op) // Deactivate the volume. err = d.rbdUnmapVolume(vol, true) if err != nil { return err } return taskErr } incus-7.0.0/internal/server/storage/drivers/driver_cephfs.go000066400000000000000000000371041517523235500242720ustar00rootroot00000000000000package drivers import ( "errors" "fmt" "io/fs" "os" "os/exec" "path/filepath" "strings" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var ( cephfsVersion string cephfsLoaded bool ) type cephfs struct { common } // load is used to run one-time action per-driver rather than per-pool. func (d *cephfs) load() error { // Register the patches. d.patches = map[string]func() error{ "storage_lvm_skipactivation": nil, "storage_missing_snapshot_records": nil, "storage_delete_old_snapshot_records": nil, "storage_zfs_drop_block_volume_filesystem_extension": nil, "storage_prefix_bucket_names_with_project": nil, } // Done if previously loaded. if cephfsLoaded { return nil } // Handle IncusOS services. if d.state.OS.IncusOS != nil { ok, err := d.state.OS.IncusOS.IsServiceEnabled("ceph") if err != nil { return err } if !ok { return errors.New("IncusOS service \"ceph\" isn't currently enabled") } } // Validate the required binaries. for _, tool := range []string{"ceph", "rbd"} { _, err := exec.LookPath(tool) if err != nil { return fmt.Errorf("Required tool '%s' is missing", tool) } } // Detect and record the version. if cephfsVersion == "" { out, err := subprocess.RunCommand("rbd", "--version") if err != nil { return err } out = strings.TrimSpace(out) fields := strings.Split(out, " ") if strings.HasPrefix(out, "ceph version ") && len(fields) > 2 { cephfsVersion = fields[2] } else { cephfsVersion = out } } cephfsLoaded = true return nil } // isRemote returns true indicating this driver uses remote storage. func (d *cephfs) isRemote() bool { return true } // Info returns the pool driver information. func (d *cephfs) Info() Info { return Info{ Name: "cephfs", Version: cephfsVersion, DefaultVMBlockFilesystemSize: deviceConfig.DefaultVMBlockFilesystemSize, OptimizedImages: false, PreservesInodes: false, Remote: d.isRemote(), VolumeTypes: []VolumeType{VolumeTypeCustom}, VolumeMultiNode: d.isRemote(), BlockBacking: false, RunningCopyFreeze: false, DirectIO: true, MountedRoot: true, } } // FillConfig populates the storage pool's configuration file with the default values. func (d *cephfs) FillConfig() error { if d.config["cephfs.cluster_name"] == "" { d.config["cephfs.cluster_name"] = CephDefaultCluster } if d.config["cephfs.user.name"] == "" { d.config["cephfs.user.name"] = CephDefaultUser } return nil } // Create is called during pool creation and is effectively using an empty driver struct. // WARNING: The Create() function cannot rely on any of the struct attributes being set. func (d *cephfs) Create() error { reverter := revert.New() defer reverter.Fail() err := d.FillConfig() if err != nil { return err } // Config validation. if d.config["source"] == "" { return errors.New("Missing required source name/path") } if d.config["cephfs.path"] != "" && d.config["cephfs.path"] != d.config["source"] { return errors.New("cephfs.path must match the source") } d.config["cephfs.path"] = d.config["source"] // Parse the namespace / path. fsName, fsPath, _ := strings.Cut(d.config["cephfs.path"], "/") fsPath = "/" + fsPath // If the filesystem already exists, disallow keys associated to creating the filesystem. fsExists, err := d.fsExists(d.config["cephfs.cluster_name"], d.config["cephfs.user.name"], fsName) if err != nil { return fmt.Errorf("Failed to check if %q CephFS exists: %w", fsName, err) } if fsExists { for _, key := range []string{"create_missing", "osd_pg_num", "meta_pool", "data_pool"} { cephfsSourceKey := fmt.Sprintf("cephfs.%s", key) if d.config[cephfsSourceKey] != "" { return fmt.Errorf("Invalid config key %q: CephFS filesystem already exists", cephfsSourceKey) } } } else { createMissing := util.IsTrue(d.config["cephfs.create_missing"]) if !createMissing { return fmt.Errorf("The requested %q CephFS doesn't exist", fsName) } // Set the pg_num to 32 because we need to specify something, but ceph will automatically change it if necessary. pgNum := d.config["cephfs.osd_pg_num"] if pgNum == "" { d.config["cephfs.osd_pg_num"] = "32" } // Create the meta and data pools if necessary. for _, key := range []string{"cephfs.meta_pool", "cephfs.data_pool"} { pool := d.config[key] if pool == "" { return fmt.Errorf("Missing required key %q for creating cephfs osd pool", key) } osdPoolExists, err := d.osdPoolExists(d.config["cephfs.cluster_name"], d.config["cephfs.user.name"], pool) if err != nil { return fmt.Errorf("Failed to check if %q OSD Pool exists: %w", pool, err) } if !osdPoolExists { // Create new osd pool. _, err := subprocess.RunCommand("ceph", "--name", fmt.Sprintf("client.%s", d.config["cephfs.user.name"]), "--cluster", d.config["cephfs.cluster_name"], "osd", "pool", "create", pool, d.config["cephfs.osd_pg_num"], ) if err != nil { return fmt.Errorf("Failed to create ceph OSD pool %q: %w", pool, err) } reverter.Add(func() { // Delete the OSD pool. _, _ = subprocess.RunCommand("ceph", "--name", fmt.Sprintf("client.%s", d.config["cephfs.user.name"]), "--cluster", d.config["cephfs.cluster_name"], "osd", "pool", "delete", pool, pool, "--yes-i-really-really-mean-it", ) }) } } // Create the filesystem. _, err := subprocess.RunCommand("ceph", "--name", fmt.Sprintf("client.%s", d.config["cephfs.user.name"]), "--cluster", d.config["cephfs.cluster_name"], "fs", "new", fsName, d.config["cephfs.meta_pool"], d.config["cephfs.data_pool"], ) if err != nil { return fmt.Errorf("Failed to create CephFS %q: %w", fsName, err) } reverter.Add(func() { // Set the FS to fail so that we can remove it. _, _ = subprocess.RunCommand("ceph", "--name", fmt.Sprintf("client.%s", d.config["cephfs.user.name"]), "--cluster", d.config["cephfs.cluster_name"], "fs", "fail", fsName, ) // Delete the FS. _, _ = subprocess.RunCommand("ceph", "--name", fmt.Sprintf("client.%s", d.config["cephfs.user.name"]), "--cluster", d.config["cephfs.cluster_name"], "fs", "rm", fsName, "--yes-i-really-mean-it", ) }) } // Create a temporary mountpoint. mountPath, err := os.MkdirTemp("", "incus_cephfs_") if err != nil { return fmt.Errorf("Failed to create temporary directory under: %w", err) } defer func() { _ = os.RemoveAll(mountPath) }() err = os.Chmod(mountPath, 0o700) if err != nil { return fmt.Errorf("Failed to chmod '%s': %w", mountPath, err) } mountPoint := filepath.Join(mountPath, "mount") err = os.Mkdir(mountPoint, 0o700) if err != nil { return fmt.Errorf("Failed to create directory '%s': %w", mountPoint, err) } // Collect Ceph information. clusterName := d.config["cephfs.cluster_name"] userName := d.config["cephfs.user.name"] fsid, err := CephFsid(clusterName, userName) if err != nil { return err } monitors, err := CephMonitors(clusterName, userName) if err != nil { return err } key, err := CephKeyring(clusterName, userName) if err != nil { return err } srcPath, options := CephBuildMount( userName, key, fsid, monitors, fsName, "/", ) // Mount the pool. err = TryMount(srcPath, mountPoint, "ceph", 0, strings.Join(options, ",")) if err != nil { return err } defer func() { _, _ = forceUnmount(mountPoint) }() // Create the path if missing. err = os.MkdirAll(filepath.Join(mountPoint, fsPath), 0o755) if err != nil { return fmt.Errorf("Failed to create directory '%s': %w", filepath.Join(mountPoint, fsPath), err) } // Check that the existing path is empty. ok, _ := internalUtil.PathIsEmpty(filepath.Join(mountPoint, fsPath)) if !ok { return errors.New("Only empty CephFS paths can be used as a storage pool") } reverter.Success() return nil } // Delete clears any local and remote data related to this driver instance. func (d *cephfs) Delete(op *operations.Operation) error { // Parse the namespace / path. fsName, fsPath, _ := strings.Cut(d.config["cephfs.path"], "/") fsPath = "/" + fsPath // Create a temporary mountpoint. mountPath, err := os.MkdirTemp("", "incus_cephfs_") if err != nil { return fmt.Errorf("Failed to create temporary directory under: %w", err) } defer func() { _ = os.RemoveAll(mountPath) }() err = os.Chmod(mountPath, 0o700) if err != nil { return fmt.Errorf("Failed to chmod '%s': %w", mountPath, err) } mountPoint := filepath.Join(mountPath, "mount") err = os.Mkdir(mountPoint, 0o700) if err != nil { return fmt.Errorf("Failed to create directory '%s': %w", mountPoint, err) } // Collect Ceph information. clusterName := d.config["cephfs.cluster_name"] userName := d.config["cephfs.user.name"] fsid, err := CephFsid(clusterName, userName) if err != nil { return err } monitors, err := CephMonitors(clusterName, userName) if err != nil { return err } key, err := CephKeyring(clusterName, userName) if err != nil { return err } srcPath, options := CephBuildMount( userName, key, fsid, monitors, fsName, "/", ) // Mount the pool. err = TryMount(srcPath, mountPoint, "ceph", 0, strings.Join(options, ",")) if err != nil { return err } defer func() { _, _ = forceUnmount(mountPoint) }() // On delete, wipe everything in the directory. err = wipeDirectory(GetPoolMountPath(d.name)) if err != nil { return err } // Delete the pool from the parent. if util.PathExists(filepath.Join(mountPoint, fsPath)) { // Delete the path itself. if fsPath != "" && fsPath != "/" { err = os.Remove(filepath.Join(mountPoint, fsPath)) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove directory '%s': %w", filepath.Join(mountPoint, fsPath), err) } } } // Make sure the existing pool is unmounted. _, err = d.Unmount() if err != nil { return err } return nil } // Validate checks that all provide keys are supported and that no conflicting or missing configuration is present. func (d *cephfs) Validate(config map[string]string) error { // gendoc:generate(entity=storage_cephfs, group=common, key=source) // // --- // type: string // scope: local // default: - // shortdesc: Existing CephFS file system or file system path to use rules := map[string]func(value string) error{ // gendoc:generate(entity=storage_cephfs, group=common, key=cephfs.cluster_name) // // --- // type: string // scope: global // default: `ceph` // shortdesc: Name of the Ceph cluster that contains the CephFS file system "cephfs.cluster_name": validate.IsAny, // gendoc:generate(entity=storage_cephfs, group=common, key=cephfs.fscache) // // --- // type: bool // scope: global // default: `false` // shortdesc: Enable use of kernel `fscache` and `cachefilesd` "cephfs.fscache": validate.Optional(validate.IsBool), // gendoc:generate(entity=storage_cephfs, group=common, key=cephfs.path) // // --- // type: string // scope: global // default: `/` // shortdesc: The base path for the CephFS mount "cephfs.path": validate.IsAny, // gendoc:generate(entity=storage_cephfs, group=common, key=cephfs.user.name) // // --- // type: string // scope: global // default: `admin` // shortdesc: The Ceph user to use "cephfs.user.name": validate.IsAny, // gendoc:generate(entity=storage_cephfs, group=common, key=cephfs.create_missing) // // --- // type: bool // scope: global // default: `false` // shortdesc: Create the file system and the missing data and metadata OSD pools "cephfs.create_missing": validate.Optional(validate.IsBool), // gendoc:generate(entity=storage_cephfs, group=common, key=cephfs.osd_pg_num) // // --- // type: string // scope: global // default: - // shortdesc: OSD pool `pg_num` to use when creating missing OSD pools "cephfs.osd_pg_num": validate.Optional(validate.IsInt64), // gendoc:generate(entity=storage_cephfs, group=common, key=cephfs.meta_pool) // // --- // type: string // scope: global // default: - // shortdesc: Metadata OSD pool name to create for the file system "cephfs.meta_pool": validate.IsAny, // gendoc:generate(entity=storage_cephfs, group=common, key=cephfs.data_pool) // // --- // type: string // scope: global // default: - // shortdesc: Data OSD pool name to create for the file system "cephfs.data_pool": validate.IsAny, // gendoc:generate(entity=storage_cephfs, group=common, key=volatile.pool.pristine) // // --- // type: string // scope: global // default: `true` // shortdesc: Whether the CephFS file system was empty on creation time "volatile.pool.pristine": validate.IsAny, } return d.validatePool(config, rules, nil) } // Update applies any driver changes required from a configuration change. func (d *cephfs) Update(changedConfig map[string]string) error { return nil } // Mount brings up the driver and sets it up to be used. func (d *cephfs) Mount() (bool, error) { // Check if already mounted. if linux.IsMountPoint(GetPoolMountPath(d.name)) { return false, nil } // Parse the namespace / path. fsName, fsPath, _ := strings.Cut(d.config["cephfs.path"], "/") fsPath = "/" + fsPath // Collect Ceph information. clusterName := d.config["cephfs.cluster_name"] userName := d.config["cephfs.user.name"] fsid, err := CephFsid(clusterName, userName) if err != nil { return false, err } monitors, err := CephMonitors(clusterName, userName) if err != nil { return false, err } key, err := CephKeyring(clusterName, userName) if err != nil { return false, err } srcPath, options := CephBuildMount( userName, key, fsid, monitors, fsName, fsPath, ) // Mount the pool. err = TryMount(srcPath, GetPoolMountPath(d.name), "ceph", 0, strings.Join(options, ",")) if err != nil { return false, err } return true, nil } // Unmount clears any of the runtime state of the driver. func (d *cephfs) Unmount() (bool, error) { return forceUnmount(GetPoolMountPath(d.name)) } // GetResources returns the pool resource usage information. func (d *cephfs) GetResources() (*api.ResourcesStoragePool, error) { return genericVFSGetResources(d) } // MigrationTypes returns the supported migration types and options supported by the driver. func (d *cephfs) MigrationTypes(contentType ContentType, refresh bool, copySnapshots bool, clusterMove bool, storageMove bool) []localMigration.Type { var rsyncFeatures []string // Do not pass compression argument to rsync if the associated // config key, that is rsync.compression, is set to false. if util.IsFalse(d.Config()["rsync.compression"]) { rsyncFeatures = []string{"delete", "bidirectional"} } else { rsyncFeatures = []string{"delete", "compress", "bidirectional"} } if contentType != ContentTypeFS { return nil } // Do not support xattr transfer on cephfs return []localMigration.Type{ { FSType: migration.MigrationFSType_RSYNC, Features: rsyncFeatures, }, } } incus-7.0.0/internal/server/storage/drivers/driver_cephfs_utils.go000066400000000000000000000027071517523235500255130ustar00rootroot00000000000000package drivers import ( "fmt" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/shared/subprocess" ) // fsExists checks that the Ceph FS instance indeed exists. func (d *cephfs) fsExists(clusterName string, userName string, fsName string) (bool, error) { _, err := subprocess.RunCommand("ceph", "--name", fmt.Sprintf("client.%s", userName), "--cluster", clusterName, "fs", "get", fsName) if err != nil { status, _ := linux.ExitStatus(err) // If the error status code is 2, the fs definitely doesn't exist. if status == 2 { return false, nil } // Else, the error status is not 0 or 2, // so we can't be sure if the fs exists or not // as it might be a network issue, an internal ceph issue, etc. return false, err } return true, nil } // osdPoolExists checks that the Ceph OSD Pool indeed exists. func (d *cephfs) osdPoolExists(clusterName string, userName string, osdPoolName string) (bool, error) { _, err := subprocess.RunCommand("ceph", "--name", fmt.Sprintf("client.%s", userName), "--cluster", clusterName, "osd", "pool", "get", osdPoolName, "size") if err != nil { status, _ := linux.ExitStatus(err) // If the error status code is 2, the pool definitely doesn't exist. if status == 2 { return false, nil } // Else, the error status is not 0 or 2, // so we can't be sure if the pool exists or not // as it might be a network issue, an internal ceph issue, etc. return false, err } return true, nil } incus-7.0.0/internal/server/storage/drivers/driver_cephfs_volumes.go000066400000000000000000000522601517523235500260440ustar00rootroot00000000000000package drivers import ( "errors" "fmt" "io" "io/fs" "os" "path/filepath" "strconv" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/rsync" "github.com/lxc/incus/v7/internal/server/backup" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // CreateVolume creates a new storage volume on disk. func (d *cephfs) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Operation) error { if vol.volType != VolumeTypeCustom { return ErrNotSupported } if vol.contentType != ContentTypeFS { return ErrNotSupported } // Create the main volume path. volPath := vol.MountPath() err := vol.EnsureMountPath(true) if err != nil { return err } // Setup for revert. revertPath := true defer func() { if revertPath { _ = os.RemoveAll(volPath) } }() // Apply the volume quota if specified. err = d.SetVolumeQuota(vol, vol.ConfigSize(), false, op) if err != nil { return err } // Fill the volume. err = genericRunFiller(d, vol, "", filler, false) if err != nil { return err } revertPath = false return nil } // CreateVolumeFromBackup re-creates a volume from its exported state. func (d *cephfs) CreateVolumeFromBackup(vol Volume, srcBackup backup.Info, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) (VolumePostHook, revert.Hook, error) { return genericVFSBackupUnpack(d, d.state.OS, vol, srcBackup.Snapshots, srcData, basePrefix, op) } // CreateVolumeFromCopy copies an existing storage volume (with or without snapshots) into a new volume. func (d *cephfs) CreateVolumeFromCopy(vol Volume, srcVol Volume, copySnapshots bool, allowInconsistent bool, op *operations.Operation) error { bwlimit := d.config["rsync.bwlimit"] // Create the main volume path. volPath := vol.MountPath() err := vol.EnsureMountPath(false) if err != nil { return err } // Create slice of snapshots created if revert needed later. revertSnaps := []string{} defer func() { if revertSnaps == nil { return } // Remove any paths created if we are reverting. for _, snapName := range revertSnaps { fullSnapName := GetSnapshotVolumeName(vol.name, snapName) snapVol := NewVolume(d, d.name, vol.volType, vol.contentType, fullSnapName, vol.config, vol.poolConfig) _ = d.DeleteVolumeSnapshot(snapVol, op) } _ = os.RemoveAll(volPath) }() // Ensure the volume is mounted. err = vol.MountTask(func(mountPath string, op *operations.Operation) error { // If copying snapshots is indicated, check the source isn't itself a snapshot. if copySnapshots && !srcVol.IsSnapshot() { // Get the list of snapshots from the source. srcSnapshots, err := srcVol.Snapshots(op) if err != nil { return err } for _, srcSnapshot := range srcSnapshots { _, snapName, _ := api.GetParentAndSnapshotName(srcSnapshot.name) // Mount the source snapshot. err = srcSnapshot.MountTask(func(srcMountPath string, op *operations.Operation) error { // Copy the snapshot. _, err = rsync.LocalCopy(srcMountPath, mountPath, bwlimit, false) return err }, op) // Create the snapshot itself. err = d.CreateVolumeSnapshot(srcSnapshot, op) if err != nil { return err } // Setup the revert. revertSnaps = append(revertSnaps, snapName) } } // Apply the volume quota if specified. err = d.SetVolumeQuota(vol, vol.ConfigSize(), false, op) if err != nil { return err } // Copy source to destination (mounting each volume if needed). err = srcVol.MountTask(func(srcMountPath string, op *operations.Operation) error { _, err := rsync.LocalCopy(srcMountPath, mountPath, bwlimit, false) return err }, op) if err != nil { return err } // Run EnsureMountPath after mounting and copying to ensure the mounted directory has the // correct permissions set. return vol.EnsureMountPath(false) }, op) if err != nil { return err } revertSnaps = nil // Don't revert. return nil } // CreateVolumeFromMigration creates a new volume (with or without snapshots) from a migration data stream. func (d *cephfs) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, volTargetArgs localMigration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error { if volTargetArgs.MigrationType.FSType != migration.MigrationFSType_RSYNC { return ErrNotSupported } // Create the main volume path. volPath := vol.MountPath() err := vol.EnsureMountPath(false) if err != nil { return err } // Create slice of snapshots created if revert needed later. revertSnaps := []string{} defer func() { if revertSnaps == nil { return } // Remove any paths created if we are reverting. for _, snapName := range revertSnaps { fullSnapName := GetSnapshotVolumeName(vol.name, snapName) snapVol := NewVolume(d, d.name, vol.volType, vol.contentType, fullSnapName, vol.config, vol.poolConfig) _ = d.DeleteVolumeSnapshot(snapVol, op) } _ = os.RemoveAll(volPath) }() // Ensure the volume is mounted. err = vol.MountTask(func(mountPath string, op *operations.Operation) error { path := internalUtil.AddSlash(mountPath) // Snapshots are sent first by the sender, so create these first. for _, snapshot := range volTargetArgs.Snapshots { // Receive the snapshot. var wrapper *ioprogress.ProgressTracker if volTargetArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", snapshot.GetName()) } err = rsync.Recv(path, conn, wrapper, volTargetArgs.MigrationType.Features) if err != nil { return err } fullSnapName := GetSnapshotVolumeName(vol.name, snapshot.GetName()) snapVol := NewVolume(d, d.name, vol.volType, vol.contentType, fullSnapName, vol.config, vol.poolConfig) // Create the snapshot itself. err = d.CreateVolumeSnapshot(snapVol, op) if err != nil { return err } // Setup the revert. revertSnaps = append(revertSnaps, snapshot.GetName()) } if vol.contentType == ContentTypeFS { // Apply the size limit. err = d.SetVolumeQuota(vol, vol.ConfigSize(), false, op) if err != nil { return err } } // Receive the main volume from sender. var wrapper *ioprogress.ProgressTracker if volTargetArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", vol.name) } return rsync.Recv(path, conn, wrapper, volTargetArgs.MigrationType.Features) }, op) if err != nil { return err } revertSnaps = nil return nil } // DeleteVolume destroys the on-disk state of a volume. func (d *cephfs) DeleteVolume(vol Volume, op *operations.Operation) error { snapshots, err := d.VolumeSnapshots(vol, op) if err != nil { return err } if len(snapshots) > 0 { return errors.New("Cannot remove a volume that has snapshots") } volPath := GetVolumeMountPath(d.name, vol.volType, vol.name) // If the volume doesn't exist, then nothing more to do. if !util.PathExists(volPath) { return nil } // Remove the volume from the storage device. err = os.RemoveAll(volPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to delete '%s': %w", volPath, err) } // Although the volume snapshot directory should already be removed, lets remove it here // to just in case the top-level directory is left. snapshotDir := GetVolumeSnapshotDir(d.name, vol.volType, vol.name) err = os.RemoveAll(snapshotDir) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to delete '%s': %w", snapshotDir, err) } return nil } // HasVolume indicates whether a specific volume exists on the storage pool. func (d *cephfs) HasVolume(vol Volume) (bool, error) { return genericVFSHasVolume(vol) } // ValidateVolume validates the supplied volume config. Optionally removes invalid keys from the volume's config. func (d *cephfs) ValidateVolume(vol Volume, removeUnknownKeys bool) error { // gendoc:generate(entity=storage_volume_cephfs, group=common, key=initial.gid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.gid` or `0` // shortdesc: GID of the volume owner in the instance // gendoc:generate(entity=storage_volume_cephfs, group=common, key=initial.mode) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.mode` or `711` // shortdesc: Mode of the volume in the instance // gendoc:generate(entity=storage_volume_cephfs, group=common, key=initial.uid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.uid` or `0` // shortdesc: UID of the volume owner in the instance // gendoc:generate(entity=storage_volume_cephfs, group=common, key=security.shared) // // --- // type: bool // condition: custom block volume // default: same as `volume.security.shared` or `false` // shortdesc: Enable sharing the volume across multiple instances // gendoc:generate(entity=storage_volume_cephfs, group=common, key=security.shifted) // // --- // type: bool // condition: custom volume // default: same as `volume.security.shifted` or `false` // shortdesc: {{enable_ID_shifting}} // gendoc:generate(entity=storage_volume_cephfs, group=common, key=security.unmapped) // // --- // type: bool // condition: custom volume // default: same as `volume.security.unmapped` or `false` // shortdesc: Disable ID mapping for the volume // gendoc:generate(entity=storage_volume_cephfs, group=common, key=size) // // --- // type: string // condition: appropriate driver // default: same as `volume.size` // shortdesc: Size/quota of the storage volume // gendoc:generate(entity=storage_volume_cephfs, group=common, key=snapshots.expiry) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_cephfs, group=common, key=snapshots.expiry.manual) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry.manual` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_cephfs, group=common, key=snapshots.pattern) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.pattern` or `snap%d` // shortdesc: {{snapshot_pattern_format}} [^*] // gendoc:generate(entity=storage_volume_cephfs, group=common, key=snapshots.schedule) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.schedule` // shortdesc: {{snapshot_schedule_format}} return d.validateVolume(vol, nil, removeUnknownKeys) } // UpdateVolume applies the driver specific changes of a volume configuration change. func (d *cephfs) UpdateVolume(vol Volume, changedConfig map[string]string) error { newSize, sizeChanged := changedConfig["size"] if sizeChanged { err := d.SetVolumeQuota(vol, newSize, false, nil) if err != nil { return err } } return d.updateVolume(vol, changedConfig) } // GetVolumeUsage returns the disk space usage of a volume. func (d *cephfs) GetVolumeUsage(vol Volume) (int64, error) { // Snapshot usage not supported for CephFS. if vol.IsSnapshot() { return -1, ErrNotSupported } out, err := subprocess.RunCommand("getfattr", "-n", "ceph.quota.max_bytes", "--only-values", GetVolumeMountPath(d.name, vol.volType, vol.name)) if err != nil { return -1, err } size, err := strconv.ParseInt(out, 10, 64) if err != nil { return -1, err } return size, nil } // SetVolumeQuota applies a size limit on volume. func (d *cephfs) SetVolumeQuota(vol Volume, size string, allowUnsafeResize bool, op *operations.Operation) error { // If size not specified in volume config, then use pool's default volume.size setting. if size == "" || size == "0" { size = d.config["volume.size"] } sizeBytes, err := units.ParseByteSizeString(size) if err != nil { return err } _, err = subprocess.RunCommand("setfattr", "-n", "ceph.quota.max_bytes", "-v", fmt.Sprintf("%d", sizeBytes), GetVolumeMountPath(d.name, vol.volType, vol.name)) return err } // GetVolumeDiskPath returns the location of a root disk block device. func (d *cephfs) GetVolumeDiskPath(vol Volume) (string, error) { return "", ErrNotSupported } // ListVolumes returns a list of volumes in storage pool. func (d *cephfs) ListVolumes() ([]Volume, error) { return genericVFSListVolumes(d) } // MountVolume sets up the volume for use. func (d *cephfs) MountVolume(vol Volume, op *operations.Operation) error { unlock, err := vol.MountLock() if err != nil { return err } defer unlock() vol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolume() when done. return nil } // UnmountVolume clears any runtime state for the volume. // As driver doesn't have volumes to unmount it returns false indicating the volume was already unmounted. func (d *cephfs) UnmountVolume(vol Volume, keepBlockDev bool, op *operations.Operation) (bool, error) { unlock, err := vol.MountLock() if err != nil { return false, err } defer unlock() refCount := vol.MountRefCountDecrement() if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": vol.name, "refCount": refCount}) return false, ErrInUse } return false, nil } // RenameVolume renames the volume and all related filesystem entries. func (d *cephfs) RenameVolume(vol Volume, newVolName string, op *operations.Operation) error { // Create the parent directory. err := CreateParentSnapshotDirIfMissing(d.name, vol.volType, newVolName) if err != nil { return err } type volRevert struct { oldPath string newPath string isSymlink bool } // Create slice to record paths renamed if revert needed later. revertPaths := []volRevert{} defer func() { // Remove any paths rename if we are reverting. for _, vol := range revertPaths { if vol.isSymlink { _ = os.Symlink(vol.oldPath, vol.newPath) } else { _ = os.Rename(vol.newPath, vol.oldPath) } } // Remove the new snapshot directory if we are reverting. if len(revertPaths) > 0 { snapshotDir := GetVolumeSnapshotDir(d.name, vol.volType, newVolName) _ = os.RemoveAll(snapshotDir) } }() // Rename the snapshot directory first. srcSnapshotDir := GetVolumeSnapshotDir(d.name, vol.volType, vol.name) if util.PathExists(srcSnapshotDir) { targetSnapshotDir := GetVolumeSnapshotDir(d.name, vol.volType, newVolName) err = os.Rename(srcSnapshotDir, targetSnapshotDir) if err != nil { return fmt.Errorf("Failed to rename '%s' to '%s': %w", srcSnapshotDir, targetSnapshotDir, err) } revertPaths = append(revertPaths, volRevert{ oldPath: srcSnapshotDir, newPath: targetSnapshotDir, }) } // Rename any snapshots of the volume too. snapshots, err := vol.Snapshots(op) if err != nil { return err } sourcePath := GetVolumeMountPath(d.name, vol.volType, newVolName) targetPath := GetVolumeMountPath(d.name, vol.volType, newVolName) for _, snapshot := range snapshots { // Figure out the snapshot paths. _, snapName, _ := api.GetParentAndSnapshotName(snapshot.name) oldCephSnapPath := filepath.Join(sourcePath, ".snap", snapName) newCephSnapPath := filepath.Join(targetPath, ".snap", snapName) oldPath := GetVolumeMountPath(d.name, vol.volType, GetSnapshotVolumeName(vol.name, snapName)) newPath := GetVolumeMountPath(d.name, vol.volType, GetSnapshotVolumeName(newVolName, snapName)) // Update the symlink. err = os.Symlink(newCephSnapPath, newPath) if err != nil { return fmt.Errorf("Failed to symlink '%s' to '%s': %w", newCephSnapPath, newPath, err) } revertPaths = append(revertPaths, volRevert{ oldPath: oldPath, newPath: oldCephSnapPath, isSymlink: true, }) } oldPath := GetVolumeMountPath(d.name, vol.volType, vol.name) newPath := GetVolumeMountPath(d.name, vol.volType, newVolName) err = os.Rename(oldPath, newPath) if err != nil { return fmt.Errorf("Failed to rename '%s' to '%s': %w", oldPath, newPath, err) } revertPaths = append(revertPaths, volRevert{ oldPath: oldPath, newPath: newPath, }) revertPaths = nil return nil } // MigrateVolume streams the volume (with or without snapshots). func (d *cephfs) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *localMigration.VolumeSourceArgs, op *operations.Operation) error { return genericVFSMigrateVolume(d, d.state, vol, conn, volSrcArgs, op) } // BackupVolume creates an exported version of a volume. func (d *cephfs) BackupVolume(vol Volume, writer instancewriter.InstanceWriter, basePrefix string, optimized bool, snapshots []string, op *operations.Operation) error { return genericVFSBackupVolume(d, vol, writer, basePrefix, snapshots, op) } // CreateVolumeSnapshot creates a new snapshot. func (d *cephfs) CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error { parentName, snapName, _ := api.GetParentAndSnapshotName(snapVol.name) // Create the snapshot. sourcePath := GetVolumeMountPath(d.name, snapVol.volType, parentName) cephSnapPath := filepath.Join(sourcePath, ".snap", snapName) err := os.Mkdir(cephSnapPath, 0o711) if err != nil { return fmt.Errorf("Failed to create directory '%s': %w", cephSnapPath, err) } // Create the parent directory. err = CreateParentSnapshotDirIfMissing(d.name, snapVol.volType, parentName) if err != nil { return err } // Create the symlink. targetPath := snapVol.MountPath() err = os.Symlink(cephSnapPath, targetPath) if err != nil { return fmt.Errorf("Failed to symlink '%s' to '%s': %w", cephSnapPath, targetPath, err) } return nil } // DeleteVolumeSnapshot deletes a snapshot. func (d *cephfs) DeleteVolumeSnapshot(snapVol Volume, op *operations.Operation) error { parentName, snapName, _ := api.GetParentAndSnapshotName(snapVol.name) // Delete the snapshot itself. sourcePath := GetVolumeMountPath(d.name, snapVol.volType, parentName) cephSnapPath := filepath.Join(sourcePath, ".snap", snapName) err := os.Remove(cephSnapPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", cephSnapPath, err) } // Remove the symlink. snapPath := snapVol.MountPath() err = os.Remove(snapPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", snapPath, err) } return nil } // MountVolumeSnapshot makes the snapshot available for use. func (d *cephfs) MountVolumeSnapshot(snapVol Volume, op *operations.Operation) error { unlock, err := snapVol.MountLock() if err != nil { return err } defer unlock() snapVol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolumeSnapshot() when done. return nil } // UnmountVolumeSnapshot clears any runtime state for the snapshot. func (d *cephfs) UnmountVolumeSnapshot(snapVol Volume, op *operations.Operation) (bool, error) { unlock, err := snapVol.MountLock() if err != nil { return false, err } defer unlock() refCount := snapVol.MountRefCountDecrement() if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": snapVol.name, "refCount": refCount}) return false, ErrInUse } return false, nil } // VolumeSnapshots returns a list of snapshots for the volume (in no particular order). func (d *cephfs) VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) { return genericVFSVolumeSnapshots(d, vol, op) } // RestoreVolume resets a volume to its snapshotted state. func (d *cephfs) RestoreVolume(vol Volume, snapshotName string, op *operations.Operation) error { sourcePath := GetVolumeMountPath(d.name, vol.volType, vol.name) cephSnapPath := filepath.Join(sourcePath, ".snap", snapshotName) // Restore using rsync. bwlimit := d.config["rsync.bwlimit"] output, err := rsync.LocalCopy(cephSnapPath, vol.MountPath(), bwlimit, false) if err != nil { return fmt.Errorf("Failed to rsync volume: %s: %w", string(output), err) } return nil } // RenameVolumeSnapshot renames a snapshot. func (d *cephfs) RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *operations.Operation) error { parentName, snapName, _ := api.GetParentAndSnapshotName(snapVol.name) sourcePath := GetVolumeMountPath(d.name, snapVol.volType, parentName) oldCephSnapPath := filepath.Join(sourcePath, ".snap", snapName) newCephSnapPath := filepath.Join(sourcePath, ".snap", newSnapshotName) err := os.Rename(oldCephSnapPath, newCephSnapPath) if err != nil { return fmt.Errorf("Failed to rename '%s' to '%s': %w", oldCephSnapPath, newCephSnapPath, err) } // Re-generate the snapshot symlink. oldPath := snapVol.MountPath() err = os.Remove(oldPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", oldPath, err) } newPath := GetVolumeMountPath(d.name, snapVol.volType, GetSnapshotVolumeName(parentName, newSnapshotName)) err = os.Symlink(newCephSnapPath, newPath) if err != nil { return fmt.Errorf("Failed to symlink '%s' to '%s': %w", newCephSnapPath, newPath, err) } return nil } incus-7.0.0/internal/server/storage/drivers/driver_cephobject.go000066400000000000000000000170741517523235500251340ustar00rootroot00000000000000package drivers import ( "context" "errors" "fmt" "net/http" "os/exec" "strings" "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var ( cephobjectVersion string cephobjectLoaded bool ) // cephobjectRadosgwAdminUser admin user in radosgw. const cephobjectRadosgwAdminUser = "incus-admin" type cephobject struct { common } // load is used to run one-time action per-driver rather than per-pool. func (d *cephobject) load() error { // Register the patches. d.patches = map[string]func() error{ "storage_lvm_skipactivation": nil, "storage_missing_snapshot_records": nil, "storage_delete_old_snapshot_records": nil, "storage_zfs_drop_block_volume_filesystem_extension": nil, "storage_prefix_bucket_names_with_project": nil, } // Done if previously loaded. if cephobjectLoaded { return nil } // Handle IncusOS services. if d.state.OS.IncusOS != nil { ok, err := d.state.OS.IncusOS.IsServiceEnabled("ceph") if err != nil { return err } if !ok { return errors.New("IncusOS service \"ceph\" isn't currently enabled") } } // Validate the required binaries. for _, tool := range []string{"radosgw-admin"} { _, err := exec.LookPath(tool) if err != nil { return fmt.Errorf("Required tool %q is missing", tool) } } // Detect and record the version. if cephobjectVersion == "" { out, err := subprocess.RunCommand("radosgw-admin", "--version") if err != nil { return err } out = strings.TrimSpace(out) fields := strings.Split(out, " ") if strings.HasPrefix(out, "ceph version ") && len(fields) > 2 { cephobjectVersion = fields[2] } else { cephobjectVersion = out } } cephobjectLoaded = true return nil } // isRemote returns true indicating this driver uses remote storage. func (d *cephobject) isRemote() bool { return true } // Info returns the pool driver information. func (d *cephobject) Info() Info { return Info{ Name: "cephobject", Version: cephobjectVersion, OptimizedImages: false, PreservesInodes: false, Remote: d.isRemote(), Buckets: true, VolumeTypes: []VolumeType{}, VolumeMultiNode: false, BlockBacking: false, RunningCopyFreeze: false, DirectIO: false, MountedRoot: false, } } // Validate checks that all provide keys are supported and that no conflicting or missing configuration is present. func (d *cephobject) Validate(config map[string]string) error { rules := map[string]func(value string) error{ // gendoc:generate(entity=storage_cephobject, group=common, key=cephobject.cluster_name) // // --- // type: string // scope: global // default: `ceph` // shortdesc: The Ceph cluster to use "cephobject.cluster_name": validate.IsAny, // gendoc:generate(entity=storage_cephobject, group=common, key=cephobject.user.name) // // --- // type: string // scope: global // default: `admin` // shortdesc: The Ceph user to use "cephobject.user.name": validate.IsAny, // gendoc:generate(entity=storage_cephobject, group=common, key=cephobject.radosgw.endpoint) // // --- // type: string // scope: global // default: - // shortdesc: URL of the `radosgw` gateway process "cephobject.radosgw.endpoint": validate.Optional(validate.IsRequestURL), // gendoc:generate(entity=storage_cephobject, group=common, key=cephobject.radosgw.endpoint_cert_file) // // --- // type: string // scope: global // default: - // shortdesc: Path to the file containing the TLS client certificate to use for endpoint communication "cephobject.radosgw.endpoint_cert_file": validate.Optional(validate.IsAbsFilePath), // gendoc:generate(entity=storage_cephobject, group=common, key=cephobject.bucket_name_prefix) // // --- // type: string // scope: global // default: - // shortdesc: Prefix to add to bucket names in Ceph "cephobject.bucket.name_prefix": validate.Optional(validate.IsAny), // gendoc:generate(entity=storage_cephobject, group=common, key=volatile.pool.pristine) // // --- // type: string // scope: global // default: `true` // shortdesc: Whether the `radosgw` `incus-admin` user existed at creation time "volatile.pool.pristine": validate.Optional(validate.IsBool), } return d.validatePool(config, rules, nil) } // FillConfig populates the storage pool's configuration file with the default values. func (d *cephobject) FillConfig() error { if d.config["cephobject.cluster_name"] == "" { d.config["cephobject.cluster_name"] = CephDefaultCluster } if d.config["cephobject.user.name"] == "" { d.config["cephobject.user.name"] = CephDefaultUser } if d.config["cephobject.radosgw.endpoint"] == "" { return errors.New(`"cephobject.radosgw.endpoint" option is required`) } return nil } // Create is called during pool creation and is effectively using an empty driver struct. // WARNING: The Create() function cannot rely on any of the struct attributes being set. func (d *cephobject) Create() error { err := d.FillConfig() if err != nil { return err } // Check if there is an existing cephobjectRadosgwAdminUser user. adminUserInfo, _, err := d.radosgwadminGetUser(context.TODO(), cephobjectRadosgwAdminUser) if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { return fmt.Errorf("Failed getting admin user %q: %w", cephobjectRadosgwAdminUser, err) } // Create missing cephobjectRadosgwAdminUser user. if adminUserInfo == nil { _, err = d.radosgwadminUserAdd(context.TODO(), cephobjectRadosgwAdminUser, 0) if err != nil { return fmt.Errorf("Failed added admin user %q: %w", cephobjectRadosgwAdminUser, err) } d.config["volatile.pool.pristine"] = "true" // Remove admin user on pool delete. } return nil } // Delete clears any local and remote data related to this driver instance. func (d *cephobject) Delete(op *operations.Operation) error { if util.IsTrue(d.config["volatile.pool.pristine"]) { err := d.radosgwadminUserDelete(context.TODO(), cephobjectRadosgwAdminUser) if err != nil { return fmt.Errorf("Failed deleting admin user %q: %w", cephobjectRadosgwAdminUser, err) } } return nil } // Update applies any driver changes required from a configuration change. func (d *cephobject) Update(changedConfig map[string]string) error { _, prefixChanged := changedConfig["cephobject.bucket.name_prefix"] if prefixChanged { buckets, err := d.radosgwadminBucketList(context.TODO()) if err != nil { return err } for _, bucketName := range buckets { if strings.HasPrefix(bucketName, d.config["cephobject.bucket.name_prefix"]) { return errors.New(`Cannot change "cephobject.bucket.name_prefix" when there are existing buclets`) } } } return nil } // Mount brings up the driver and sets it up to be used. func (d *cephobject) Mount() (bool, error) { return false, nil } // Unmount clears any of the runtime state of the driver. func (d *cephobject) Unmount() (bool, error) { return false, nil } // GetResources returns the pool resource usage information. func (d *cephobject) GetResources() (*api.ResourcesStoragePool, error) { return &api.ResourcesStoragePool{}, nil } // MigrationTypes returns the supported migration types and options supported by the driver. func (d *cephobject) MigrationTypes(contentType ContentType, refresh bool, copySnapshots bool, clusterMove bool, storageMove bool) []migration.Type { return nil } incus-7.0.0/internal/server/storage/drivers/driver_cephobject_buckets.go000066400000000000000000000227201517523235500266460ustar00rootroot00000000000000package drivers import ( "context" "crypto/tls" "crypto/x509" "errors" "fmt" "net/http" "net/url" "os" "path" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/smithy-go" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/storage/s3util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" ) // ValidateVolume validates the supplied volume config. func (d *cephobject) ValidateVolume(vol Volume, removeUnknownKeys bool) error { // gendoc:generate(entity=storage_bucket_cephobject, group=common, key=size) // // --- // type: string // default: - // shortdesc: Quota of the storage bucket return d.validateVolume(vol, nil, removeUnknownKeys) } // s3Client returns a configured S3 client. func (d *cephobject) s3Client(creds S3Credentials) (*s3.Client, error) { u, err := url.ParseRequestURI(d.config["cephobject.radosgw.endpoint"]) if err != nil { return nil, fmt.Errorf("Failed parsing cephobject.radosgw.endpoint: %w", err) } httpClient := &http.Client{} certFilePath := d.config["cephobject.radosgw.endpoint_cert_file"] if u.Scheme == "https" && certFilePath != "" { // Read in the cert file. certs, err := os.ReadFile(certFilePath) if err != nil { return nil, fmt.Errorf("Failed reading %q: %w", certFilePath, err) } rootCAs := x509.NewCertPool() ok := rootCAs.AppendCertsFromPEM(certs) if !ok { return nil, errors.New("Failed adding S3 client certificates") } // Trust the cert pool in our client. httpClient.Transport = &http.Transport{ TLSClientConfig: &tls.Config{ RootCAs: rootCAs, }, } } cfg := aws.Config{ Region: s3util.RegionFromURL(u), Credentials: credentials.NewStaticCredentialsProvider(creds.AccessKey, creds.SecretKey, ""), HTTPClient: httpClient, } endpoint := fmt.Sprintf("%s://%s", u.Scheme, path.Join(u.Host, u.Path)) return s3.NewFromConfig(cfg, func(o *s3.Options) { o.BaseEndpoint = aws.String(endpoint) o.UsePathStyle = true }), nil } // CreateBucket creates a new bucket. func (d *cephobject) CreateBucket(bucket Volume, op *operations.Operation) error { // Check if there is an existing cephobjectRadosgwAdminUser user. adminUserInfo, _, err := d.radosgwadminGetUser(context.TODO(), cephobjectRadosgwAdminUser) if err != nil && !api.StatusErrorCheck(err, http.StatusNotFound) { return fmt.Errorf("Failed getting admin user %q: %w", cephobjectRadosgwAdminUser, err) } // Create missing cephobjectRadosgwAdminUser user. if adminUserInfo == nil { adminUserInfo, err = d.radosgwadminUserAdd(context.TODO(), cephobjectRadosgwAdminUser, 0) if err != nil { return fmt.Errorf("Failed added admin user %q: %w", cephobjectRadosgwAdminUser, err) } } _, bucketName := project.StorageVolumeParts(bucket.name) storageBucketName := d.radosgwBucketName(bucketName) // Must be defined before revert so that its not cancelled by time reverter.Fail runs. ctx, ctxCancel := context.WithTimeout(context.TODO(), time.Duration(time.Second*30)) defer ctxCancel() reverter := revert.New() defer reverter.Fail() s3Client, err := d.s3Client(*adminUserInfo) if err != nil { return err } _, err = s3Client.HeadBucket(ctx, &s3.HeadBucketInput{ Bucket: aws.String(storageBucketName), }) if err == nil { return api.StatusErrorf(http.StatusConflict, "A bucket for that name already exists") } var apiErr smithy.APIError if !errors.As(err, &apiErr) || apiErr.ErrorCode() != "NotFound" { return err } // Create new bucket. _, err = s3Client.CreateBucket(ctx, &s3.CreateBucketInput{ Bucket: aws.String(storageBucketName), }) if err != nil { return fmt.Errorf("Failed creating bucket: %w", err) } reverter.Add(func() { _, _ = s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(storageBucketName)}) }) // Create bucket user. _, err = d.radosgwadminUserAdd(context.TODO(), storageBucketName, -1) if err != nil { return fmt.Errorf("Failed creating bucket user: %w", err) } reverter.Add(func() { _ = d.radosgwadminUserDelete(context.TODO(), storageBucketName) }) // Link bucket to user. err = d.radosgwadminBucketLink(context.TODO(), storageBucketName, storageBucketName) if err != nil { return fmt.Errorf("Failed linking bucket to user: %w", err) } // Set initial quota if specified. if bucket.config["size"] != "" && bucket.config["size"] != "0" { err = d.setBucketQuota(bucket, bucket.config["size"]) if err != nil { return err } } reverter.Success() return nil } // setBucketQuota sets the bucket quota. func (d *cephobject) setBucketQuota(bucket Volume, quotaSize string) error { _, bucketName := project.StorageVolumeParts(bucket.name) storageBucketName := d.radosgwBucketName(bucketName) sizeBytes, err := units.ParseByteSizeString(quotaSize) if err != nil { return fmt.Errorf("Failed parsing bucket quota size: %w", err) } err = d.radosgwadminBucketSetQuota(context.TODO(), storageBucketName, storageBucketName, sizeBytes) if err != nil { return fmt.Errorf("Failed setting bucket quota: %w", err) } return nil } // DeleteBucket deletes an existing bucket. func (d *cephobject) DeleteBucket(bucket Volume, op *operations.Operation) error { _, bucketName := project.StorageVolumeParts(bucket.name) storageBucketName := d.radosgwBucketName(bucketName) err := d.radosgwadminBucketDelete(context.TODO(), storageBucketName) if err != nil { return fmt.Errorf("Failed deleting bucket: %w", err) } err = d.radosgwadminUserDelete(context.TODO(), storageBucketName) if err != nil { return fmt.Errorf("Failed deleting bucket user: %w", err) } return nil } // UpdateBucket updates an existing bucket. func (d *cephobject) UpdateBucket(bucket Volume, changedConfig map[string]string) error { newSize, sizeChanged := changedConfig["size"] if sizeChanged { err := d.setBucketQuota(bucket, newSize) if err != nil { return err } } return nil } // bucketKeyRadosgwAccessRole returns the radosgw access setting for the specified role name. func (d *cephobject) bucketKeyRadosgwAccessRole(roleName string) (string, error) { switch roleName { case "read-only": return "read", nil case "admin": return "full", nil } return "", api.StatusErrorf(http.StatusBadRequest, "Invalid bucket key role") } // CreateBucket creates a new bucket. func (d *cephobject) CreateBucketKey(bucket Volume, keyName string, creds S3Credentials, roleName string, op *operations.Operation) (*S3Credentials, error) { _, bucketName := project.StorageVolumeParts(bucket.name) storageBucketName := d.radosgwBucketName(bucketName) accessRole, err := d.bucketKeyRadosgwAccessRole(roleName) if err != nil { return nil, err } _, bucketSubUsers, err := d.radosgwadminGetUser(context.TODO(), storageBucketName) if err != nil { return nil, fmt.Errorf("Failed getting bucket user: %w", err) } _, subUserExists := bucketSubUsers[keyName] if subUserExists { return nil, api.StatusErrorf(http.StatusConflict, "A bucket key for that name already exists") } // Create a sub user for the key on the bucket user. newCreds, err := d.radosgwadminSubUserAdd(context.TODO(), storageBucketName, keyName, accessRole, creds.AccessKey, creds.SecretKey) if err != nil { return nil, fmt.Errorf("Failed creating bucket user: %w", err) } return newCreds, nil } // UpdateBucketKey updates bucket key. func (d *cephobject) UpdateBucketKey(bucket Volume, keyName string, creds S3Credentials, roleName string, op *operations.Operation) (*S3Credentials, error) { _, bucketName := project.StorageVolumeParts(bucket.name) storageBucketName := d.radosgwBucketName(bucketName) accessRole, err := d.bucketKeyRadosgwAccessRole(roleName) if err != nil { return nil, err } _, bucketSubUsers, err := d.radosgwadminGetUser(context.TODO(), storageBucketName) if err != nil { return nil, fmt.Errorf("Failed getting bucket user: %w", err) } _, subUserExists := bucketSubUsers[keyName] if !subUserExists { return nil, api.StatusErrorf(http.StatusConflict, "A bucket key for that name does not exist") } // We delete and recreate the subuser otherwise if the creds.AccessKey has changed a new access key/secret // will be created, leaving the old one behind still active. err = d.radosgwadminSubUserDelete(context.TODO(), storageBucketName, keyName) if err != nil { return nil, fmt.Errorf("Failed deleting bucket key: %w", err) } // Create a sub user for the key on the bucket user. newCreds, err := d.radosgwadminSubUserAdd(context.TODO(), storageBucketName, keyName, accessRole, creds.AccessKey, creds.SecretKey) if err != nil { return nil, fmt.Errorf("Failed creating bucket user: %w", err) } return newCreds, err } // DeleteBucketKey deletes an existing bucket key. func (d *cephobject) DeleteBucketKey(bucket Volume, keyName string, op *operations.Operation) error { _, bucketName := project.StorageVolumeParts(bucket.name) storageBucketName := d.radosgwBucketName(bucketName) err := d.radosgwadminSubUserDelete(context.TODO(), storageBucketName, keyName) if err != nil { return fmt.Errorf("Failed deleting bucket key: %w", err) } return nil } // GetBucketURL returns the URL of the specified bucket. func (d *cephobject) GetBucketURL(bucketName string) *url.URL { u, err := url.ParseRequestURI(d.config["cephobject.radosgw.endpoint"]) if err != nil { return nil } u.Path = path.Join(u.Path, url.PathEscape(d.radosgwBucketName(bucketName))) return u } incus-7.0.0/internal/server/storage/drivers/driver_cephobject_utils.go000066400000000000000000000162771517523235500263600ustar00rootroot00000000000000package drivers import ( "context" "encoding/json" "fmt" "net/http" "strings" "time" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" ) // radosgwadmin wrapper around radosgw-admin command. func (d *cephobject) radosgwadmin(ctx context.Context, args ...string) (string, error) { _, ok := ctx.Deadline() if !ok { // Set default timeout of 30s if no deadline context provided. var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, time.Duration(30*time.Second)) defer cancel() } cmd := []string{"radosgw-admin", "--cluster", d.config["cephobject.cluster_name"], "--id", d.config["cephobject.user.name"]} cmd = append(cmd, args...) return subprocess.RunCommandContext(ctx, cmd[0], cmd[1:]...) } // radosgwadminGetUser returns credentials for an existing radosgw user (and its sub users). // If no user exists returns api.StatusError with status code set to http.StatusNotFound. func (d *cephobject) radosgwadminGetUser(ctx context.Context, user string) (*S3Credentials, map[string]S3Credentials, error) { out, err := d.radosgwadmin(ctx, "user", "info", "--uid", user) if err != nil { status, _ := linux.ExitStatus(err) if status == 22 { return nil, nil, api.StatusErrorf(http.StatusNotFound, "User not found") } return nil, nil, fmt.Errorf("Failed getting user %q info: %w", user, err) } resp := struct { SubUsers []struct { ID string `json:"id"` } `json:"subusers"` Keys []struct { S3Credentials `yaml:",inline"` User string `json:"user"` } `json:"keys"` }{} err = json.Unmarshal([]byte(out), &resp) if err != nil { return nil, nil, err } // Get list of sub user names and store them without the main user prefix. subUsers := make(map[string]S3Credentials, len(resp.SubUsers)) for _, subUser := range resp.SubUsers { subUserName := strings.TrimPrefix(subUser.ID, fmt.Sprintf("%s:", user)) subUsers[subUserName] = S3Credentials{} } var userKey *S3Credentials // Iterate through the keys extracting the main user key and keys for the known sub users. for _, key := range resp.Keys { if key.User == user { userKey = &S3Credentials{ AccessKey: key.AccessKey, SecretKey: key.SecretKey, } } else { for subUserName := range subUsers { if strings.TrimPrefix(key.User, fmt.Sprintf("%s:", user)) == subUserName { subUser := subUsers[subUserName] subUser.AccessKey = key.AccessKey subUser.SecretKey = key.SecretKey subUsers[subUserName] = subUser } } } } if userKey == nil { return nil, nil, fmt.Errorf("S3 credentials not found for %q user", user) } return userKey, subUsers, nil } // radosgwadminUserAdd creates a radosgw user and return generated credentials. func (d *cephobject) radosgwadminUserAdd(ctx context.Context, user string, maxBuckets int) (*S3Credentials, error) { reverter := revert.New() defer reverter.Fail() out, err := d.radosgwadmin(ctx, "user", "create", "--max-buckets", fmt.Sprintf("%d", maxBuckets), "--display-name", user, "--uid", user) if err != nil { return nil, err } reverter.Add(func() { _ = d.radosgwadminUserDelete(ctx, user) }) creds := struct { Keys []struct { S3Credentials `yaml:",inline"` User string `json:"user"` } `json:"keys"` }{} err = json.Unmarshal([]byte(out), &creds) if err != nil { return nil, err } for _, key := range creds.Keys { if key.User == user { reverter.Success() return &S3Credentials{AccessKey: key.AccessKey, SecretKey: key.SecretKey}, err } } return nil, fmt.Errorf("S3 credentials not found for %q user", user) } // radosgwadminUserDelete deletes radosgw user. func (d *cephobject) radosgwadminUserDelete(ctx context.Context, user string) error { _, err := d.radosgwadmin(ctx, "user", "rm", "--uid", user, "--purge-data") return err } // radosgwadminSubUserAdd adds a radosgw sub user. func (d *cephobject) radosgwadminSubUserAdd(ctx context.Context, user string, subuser string, accessRole string, accessKey string, secretKey string) (*S3Credentials, error) { reverter := revert.New() defer reverter.Fail() args := []string{"subuser", "create", "--uid", user, "--key-type", "s3", "--subuser", subuser, "--access", accessRole} if accessKey == "" { args = append(args, "--gen-access-key") } else { args = append(args, "--access-key", accessKey) } if secretKey == "" { args = append(args, "--gen-secret") } else { args = append(args, "--secret", secretKey) } out, err := d.radosgwadmin(ctx, args...) if err != nil { return nil, err } reverter.Add(func() { _ = d.radosgwadminUserDelete(ctx, user) }) creds := struct { Keys []struct { S3Credentials `yaml:",inline"` User string `json:"user"` } `json:"keys"` }{} err = json.Unmarshal([]byte(out), &creds) if err != nil { return nil, err } keyUser := fmt.Sprintf("%s:%s", user, subuser) for _, key := range creds.Keys { if key.User == keyUser { reverter.Success() return &S3Credentials{AccessKey: key.AccessKey, SecretKey: key.SecretKey}, err } } return nil, fmt.Errorf("S3 credentials not found for %q user", keyUser) } // radosgwadminSubUserDelete deletes radosgw sub-user. func (d *cephobject) radosgwadminSubUserDelete(ctx context.Context, user string, subuser string) error { _, err := d.radosgwadmin(ctx, "subuser", "rm", "--uid", user, "--subuser", subuser) return err } // radosgwadminBucketDelete deletes radosgw bucket. func (d *cephobject) radosgwadminBucketDelete(ctx context.Context, bucket string) error { _, err := d.radosgwadmin(ctx, "bucket", "rm", "--bucket", bucket, "--purge-objects") return err } // radosgwadminBucketLink links a bucket to a user. func (d *cephobject) radosgwadminBucketLink(ctx context.Context, bucket string, user string) error { _, err := d.radosgwadmin(ctx, "bucket", "link", "--bucket", bucket, "--uid", user) return err } // radosgwadminBucketSetQuota sets bucket quota. func (d *cephobject) radosgwadminBucketSetQuota(ctx context.Context, bucket string, user string, size int64) error { if size > 0 { _, err := d.radosgwadmin(ctx, "quota", "enable", "--quota-scope=bucket", "--uid", user) if err != nil { return err } _, err = d.radosgwadmin(ctx, "quota", "set", "--quota-scope=bucket", "--uid", user, "--max-size", fmt.Sprintf("%d", size)) if err != nil { return err } } else { _, err := d.radosgwadmin(ctx, "quota", "disable", "--quota-scope=bucket", "--uid", user) if err != nil { return err } _, err = d.radosgwadmin(ctx, "quota", "set", "--quota-scope=bucket", "--uid", user, "--max-size", "-1") if err != nil { return err } } return nil } // radosgwadminBucketList returns the list of buckets. func (d *cephobject) radosgwadminBucketList(ctx context.Context) ([]string, error) { out, err := d.radosgwadmin(ctx, "bucket", "list") if err != nil { return nil, err } buckets := []string{} err = json.Unmarshal([]byte(out), &buckets) if err != nil { return nil, err } return buckets, nil } // radosgwBucketName returns the bucket name to use for the actual radosgw bucket. func (d *cephobject) radosgwBucketName(bucketName string) string { return fmt.Sprintf("%s%s", d.config["cephobject.bucket.name_prefix"], bucketName) } incus-7.0.0/internal/server/storage/drivers/driver_common.go000066400000000000000000000465361517523235500243230ustar00rootroot00000000000000package drivers import ( "errors" "fmt" "io" "maps" "net/url" "os/exec" "regexp" "slices" "strings" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/server/backup" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) type common struct { name string config map[string]string getVolID func(volType VolumeType, volName string) (int64, error) commonRules *Validators state *state.State logger logger.Logger patches map[string]func() error } func (d *common) init(state *state.State, name string, config map[string]string, logger logger.Logger, volIDFunc func(volType VolumeType, volName string) (int64, error), commonRules *Validators) { d.name = name d.config = config d.getVolID = volIDFunc d.commonRules = commonRules d.state = state d.logger = logger } // isRemote returns false indicating this driver does not use remote storage. func (d *common) isRemote() bool { return false } // validatePool validates a pool config against common rules and optional driver specific rules. func (d *common) validatePool(config map[string]string, driverRules map[string]func(value string) error, volumeRules map[string]func(value string) error) error { checkedFields := map[string]struct{}{} // Get rules common for all drivers. rules := d.commonRules.PoolRules() // Merge driver specific rules into common rules. maps.Copy(rules, driverRules) // Add to pool volume configuration options as volume.* options. // These will be used as default configuration options for volume. for volRule, volValidator := range volumeRules { rules[fmt.Sprintf("volume.%s", volRule)] = volValidator } // Run the validator against each field. for k, validator := range rules { checkedFields[k] = struct{}{} // Mark field as checked. err := validator(config[k]) if err != nil { return fmt.Errorf("Invalid value for option %q: %w", k, err) } } // Look for any unchecked fields, as these are unknown fields and validation should fail. for k := range config { _, checked := checkedFields[k] if checked { continue } // User keys are not validated. if strings.HasPrefix(k, "user.") { continue } return fmt.Errorf("Invalid option %q", k) } return nil } // fillVolumeConfig populates volume config with defaults from pool. // excludeKeys allow exclude some keys from copying to volume config. // Sometimes that can be useful when copying is dependent from specific conditions // and shouldn't be done in generic way. func (d *common) fillVolumeConfig(vol *Volume, excludedKeys ...string) error { for k := range d.config { if !strings.HasPrefix(k, "volume.") { continue } volKey := strings.TrimPrefix(k, "volume.") isExcluded := slices.Contains(excludedKeys, volKey) if isExcluded { continue } // If volume type is not custom or bucket, don't copy "size" property to volume config. if (vol.volType != VolumeTypeCustom && vol.volType != VolumeTypeBucket) && volKey == "size" { continue } // security.shifted and security.unmapped are only relevant for custom filesystem volumes. if (vol.Type() != VolumeTypeCustom || vol.ContentType() != ContentTypeFS) && (volKey == "security.shifted" || volKey == "security.unmapped") { continue } // security.shared is only relevant for custom block volumes. if (vol.Type() != VolumeTypeCustom || vol.ContentType() != ContentTypeBlock) && (volKey == "security.shared") { continue } if vol.config[volKey] == "" { vol.config[volKey] = d.config[k] } } return nil } // FillVolumeConfig populate volume with default config. func (d *common) FillVolumeConfig(vol Volume) error { return d.fillVolumeConfig(&vol) } // validateVolume validates a volume config against common rules and optional driver specific rules. // This functions has a removeUnknownKeys option that if set to true will remove any unknown fields // (excluding those starting with "user.") which can be used when translating a volume config to a // different storage driver that has different options. func (d *common) validateVolume(vol Volume, driverRules map[string]func(value string) error, removeUnknownKeys bool) error { checkedFields := map[string]struct{}{} // Get rules common for all drivers. rules := d.commonRules.VolumeRules(vol) // Merge driver specific rules into common rules. maps.Copy(rules, driverRules) // Run the validator against each field. for k, validator := range rules { checkedFields[k] = struct{}{} // Mark field as checked. err := validator(vol.config[k]) if err != nil { return fmt.Errorf("Invalid value for volume %q option %q: %w", vol.name, k, err) } } // Look for any unchecked fields, as these are unknown fields and validation should fail. for k := range vol.config { _, checked := checkedFields[k] if checked { continue } // User keys are not validated. if strings.HasPrefix(k, "user.") { continue } if removeUnknownKeys { delete(vol.config, k) } else { return fmt.Errorf("Invalid option for volume %q option %q", vol.name, k) } } // If volume type is not custom or bucket, don't allow "size" property. if (vol.volType != VolumeTypeCustom && vol.volType != VolumeTypeBucket) && vol.config["size"] != "" { return fmt.Errorf("Volume %q property is not valid for volume type", "size") } // Check that security.unmapped and security.shifted are not set together. if util.IsTrue(vol.config["security.unmapped"]) && util.IsTrue(vol.config["security.shifted"]) { return errors.New("security.unmapped and security.shifted are mutually exclusive") } if util.IsTrue(vol.config["dependent"]) { err := ValidateDependentConfigKey(vol.config) if err != nil { return err } } return nil } // updateVolume applies the common changes for all drivers of a volume configuration change. func (d *common) updateVolume(vol Volume, changedConfig map[string]string) error { _, changed := changedConfig["dependent"] if changed { return errors.New("dependent cannot be changed") } return nil } // MigrationType returns the type of transfer methods to be used when doing migrations between pools // in preference order. func (d *common) MigrationTypes(contentType ContentType, refresh bool, copySnapshots bool, clusterMove bool, storageMove bool) []localMigration.Type { var transportType migration.MigrationFSType var rsyncFeatures []string // Do not pass compression argument to rsync if the associated // config key, that is rsync.compression, is set to false. if util.IsFalse(d.Config()["rsync.compression"]) { rsyncFeatures = []string{"xattrs", "delete", "bidirectional"} } else { rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"} } if IsContentBlock(contentType) { transportType = migration.MigrationFSType_BLOCK_AND_RSYNC } else { transportType = migration.MigrationFSType_RSYNC } return []localMigration.Type{ { FSType: transportType, Features: rsyncFeatures, }, } } // Name returns the pool name. func (d *common) Name() string { return d.name } // Logger returns the current logger. func (d *common) Logger() logger.Logger { return d.logger } // Config returns the storage pool config (as a copy, so not modifiable). func (d *common) Config() map[string]string { return util.CloneMap(d.config) } // ApplyPatch looks for a suitable patch and runs it. func (d *common) ApplyPatch(name string) error { if d.patches == nil { return fmt.Errorf("The patch mechanism isn't implemented on pool %q", d.name) } // Locate the patch. patch, ok := d.patches[name] if !ok { return fmt.Errorf("Patch %q isn't implemented on pool %q", name, d.name) } // Handle cases where a patch isn't needed. if patch == nil { return nil } return patch() } // moveGPTAltHeader moves the GPT alternative header to the end of the disk device supplied. // If the device supplied is not detected as not being a GPT disk then no action is taken and nil is returned. // If the required sgdisk command is not available a warning is logged, but no error is returned, as really it is // the job of the VM quest to ensure the partitions are resized to the size of the disk (as Incus does not dictate // what partition structure (if any) the disk should have. However we do attempt to move the GPT alternative // header where possible so that the backup header is where it is expected in case of any corruption with the // primary header. func (d *common) moveGPTAltHeader(devPath string) error { path, err := exec.LookPath("sgdisk") if err != nil { d.logger.Warn("Skipped moving GPT alternative header to end of disk as sgdisk command not found", logger.Ctx{"dev": devPath}) return nil } // Our images and VM drives use a 512 bytes sector size. // If the underlying block device uses a different sector size, we // need to fake the correct size through a loop device so sgdisk can // correctly re-locate the partition tables. if linux.IsBlockdevPath(devPath) { blockSize, err := GetPhysicalBlockSize(devPath) if err != nil { return err } if blockSize != 512 { devPath, err = loopDeviceSetupAlign(devPath) if err != nil { return err } defer func() { _ = loopDeviceAutoDetach(devPath) }() } } _, err = subprocess.RunCommand(path, "--move-second-header", devPath) if err == nil { d.logger.Debug("Moved GPT alternative header to end of disk", logger.Ctx{"dev": devPath}) return nil } var runErr subprocess.RunError if errors.As(err, &runErr) { var exitError *exec.ExitError if errors.As(runErr.Unwrap(), &exitError) { // sgdisk manpage says exit status 3 means: // "Non-GPT disk detected and no -g option, but operation requires a write action". if exitError.ExitCode() == 3 { return nil // Non-error as non-GPT disk specified. } } } return err } // CreateVolume creates a new storage volume on disk. func (d *common) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Operation) error { return ErrNotSupported } // CreateVolumeFromBackup re-creates a volume from its exported state. func (d *common) CreateVolumeFromBackup(vol Volume, srcBackup backup.Info, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) (VolumePostHook, revert.Hook, error) { return nil, nil, ErrNotSupported } // CreateVolumeFromCopy copies an existing storage volume (with or without snapshots) into a new volume. func (d *common) CreateVolumeFromCopy(vol Volume, srcVol Volume, copySnapshots bool, allowInconsistent bool, op *operations.Operation) error { return ErrNotSupported } // CreateVolumeFromMigration creates a new volume (with or without snapshots) from a migration data stream. func (d *common) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, volTargetArgs localMigration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error { return ErrNotSupported } // RefreshVolume updates an existing volume to match the state of another. func (d *common) RefreshVolume(vol Volume, srcVol Volume, srcSnapshots []Volume, allowInconsistent bool, op *operations.Operation) error { return ErrNotSupported } // DeleteVolume destroys the on-disk state of a volume. func (d *common) DeleteVolume(vol Volume, op *operations.Operation) error { return ErrNotSupported } // HasVolume indicates whether a specific volume exists on the storage pool. func (d *common) HasVolume(vol Volume) (bool, error) { return false, ErrNotSupported } // ValidateVolume validates the supplied volume config. Optionally removes invalid keys from the volume's config. func (d *common) ValidateVolume(vol Volume, removeUnknownKeys bool) error { return ErrNotSupported } // UpdateVolume applies the driver specific changes of a volume configuration change. func (d *common) UpdateVolume(vol Volume, changedConfig map[string]string) error { return ErrNotSupported } // GetVolumeUsage returns the disk space usage of a volume. func (d *common) GetVolumeUsage(vol Volume) (int64, error) { return -1, ErrNotSupported } // SetVolumeQuota applies a size limit on volume. func (d *common) SetVolumeQuota(vol Volume, size string, allowUnsafeResize bool, op *operations.Operation) error { return ErrNotSupported } // GetVolumeDiskPath returns the location of a root disk block device. func (d *common) GetVolumeDiskPath(vol Volume) (string, error) { return "", ErrNotSupported } // ListVolumes returns a list of volumes in storage pool. func (d *common) ListVolumes() ([]Volume, error) { return nil, ErrNotSupported } // MountVolume sets up the volume for use. func (d *common) MountVolume(vol Volume, op *operations.Operation) error { return ErrNotSupported } // UnmountVolume clears any runtime state for the volume. // As driver doesn't have volumes to unmount it returns false indicating the volume was already unmounted. func (d *common) UnmountVolume(vol Volume, keepBlockDev bool, op *operations.Operation) (bool, error) { return false, ErrNotSupported } // CanDelegateVolume checks whether the volume can be delegated. func (d *common) CanDelegateVolume(vol Volume) bool { return false } // DelegateVolume delegates a volume. func (d *common) DelegateVolume(vol Volume, pid int) error { return nil } // RenameVolume renames the volume and all related filesystem entries. func (d *common) RenameVolume(vol Volume, newVolName string, op *operations.Operation) error { return ErrNotSupported } // MigrateVolume streams the volume (with or without snapshots). func (d *common) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *localMigration.VolumeSourceArgs, op *operations.Operation) error { return ErrNotSupported } // BackupVolume creates an exported version of a volume. func (d *common) BackupVolume(vol Volume, writer instancewriter.InstanceWriter, basePrefix string, optimized bool, snapshots []string, op *operations.Operation) error { return ErrNotSupported } // CreateVolumeSnapshot creates a new snapshot. func (d *common) CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error { return ErrNotSupported } // DeleteVolumeSnapshot deletes a snapshot. func (d *common) DeleteVolumeSnapshot(snapVol Volume, op *operations.Operation) error { return ErrNotSupported } // MountVolumeSnapshot makes the snapshot available for use. func (d *common) MountVolumeSnapshot(snapVol Volume, op *operations.Operation) error { return ErrNotSupported } // UnmountVolumeSnapshot clears any runtime state for the snapshot. func (d *common) UnmountVolumeSnapshot(snapVol Volume, op *operations.Operation) (bool, error) { return false, ErrNotSupported } // VolumeSnapshots returns a list of snapshots for the volume (in no particular order). func (d *common) VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) { return nil, ErrNotSupported } // CanRestoreVolume checks whether a volume snapshot can be restored. func (d *common) CanRestoreVolume(vol Volume, snapshotName string) error { return nil } // RestoreVolume resets a volume to its snapshotted state. func (d *common) RestoreVolume(vol Volume, snapshotName string, op *operations.Operation) error { return ErrNotSupported } // RenameVolumeSnapshot renames a snapshot. func (d *common) RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *operations.Operation) error { return ErrNotSupported } // ValidateBucket validates the supplied bucket name. func (d *common) ValidateBucket(bucket Volume) error { projectName, bucketName := project.StorageVolumeParts(bucket.name) if projectName == "" { return errors.New("Project prefix missing in bucket volume name") } match, err := regexp.MatchString(`^[a-z0-9][\-\.a-z0-9]{2,62}$`, bucketName) if err != nil { return err } if !match { return errors.New("Bucket name must be between 3 and 63 lowercase letters, numbers, periods or hyphens and must start with a letter or number") } return nil } // GetBucketURL returns the URL of the specified bucket. func (d *common) GetBucketURL(bucketName string) *url.URL { return nil } // CreateBucket creates a new bucket. func (d *common) CreateBucket(bucket Volume, op *operations.Operation) error { return ErrNotSupported } // DeleteBucket deletes an existing bucket. func (d *common) DeleteBucket(bucket Volume, op *operations.Operation) error { return ErrNotSupported } // UpdateBucket updates an existing bucket. func (d *common) UpdateBucket(bucket Volume, changedConfig map[string]string) error { return ErrNotSupported } // ValidateBucketKey validates the supplied bucket key config. func (d *common) ValidateBucketKey(keyName string, creds S3Credentials, roleName string) error { if keyName == "" { return errors.New("Key name is required") } validRoles := []string{"admin", "read-only"} if !slices.Contains(validRoles, roleName) { return errors.New("Invalid key role") } return nil } // CreateBucketKey create bucket key. func (d *common) CreateBucketKey(bucket Volume, keyName string, creds S3Credentials, roleName string, op *operations.Operation) (*S3Credentials, error) { return nil, ErrNotSupported } // UpdateBucketKey updates bucket key. func (d *common) UpdateBucketKey(bucket Volume, keyName string, creds S3Credentials, roleName string, op *operations.Operation) (*S3Credentials, error) { return nil, ErrNotSupported } func (d *common) DeleteBucketKey(bucket Volume, keyName string, op *operations.Operation) error { return nil } // roundVolumeBlockSizeBytes returns sizeBytes rounded up to the next multiple // of MinBlockBoundary. func (d *common) roundVolumeBlockSizeBytes(vol Volume, sizeBytes int64) (int64, error) { // QEMU requires image files to be in traditional storage block boundaries. // We use 8k here to ensure our images are compatible with all of our backend drivers. return RoundAbove(MinBlockBoundary, sizeBytes), nil } func (d *common) isBlockBacked(vol Volume) bool { return vol.driver.Info().BlockBacking } // filesystemFreeze syncs and freezes a filesystem and returns an unfreeze function on success. func (d *common) filesystemFreeze(path string) (func() error, error) { err := linux.SyncFS(path) if err != nil { return nil, fmt.Errorf("Failed syncing filesystem %q: %w", path, err) } _, err = subprocess.RunCommand("fsfreeze", "--freeze", path) if err != nil { return nil, fmt.Errorf("Failed freezing filesystem %q: %w", path, err) } d.logger.Info("Filesystem frozen", logger.Ctx{"path": path}) unfreezeFS := func() error { _, err := subprocess.RunCommand("fsfreeze", "--unfreeze", path) if err != nil { return fmt.Errorf("Failed unfreezing filesystem %q: %w", path, err) } d.logger.Info("Filesystem unfrozen", logger.Ctx{"path": path}) return nil } return unfreezeFS, nil } // GetQcow2BackingFilePath generates the backing file path for the specified volume. func (d *common) GetQcow2BackingFilePath(vol Volume) (string, error) { return "", ErrNotSupported } // Qcow2DeletionCleanup performs post block-commit cleanup of qcow2 snapshot artifacts. func (d *common) Qcow2DeletionCleanup(vol Volume, childName string) error { return ErrNotSupported } // ActivateTask allows running a low-level task with the volume active. func (d *common) ActivateTask(vol Volume, task func(devPath string, op *operations.Operation) error, op *operations.Operation) error { return ErrNotSupported } incus-7.0.0/internal/server/storage/drivers/driver_dir.go000066400000000000000000000121771517523235500236030ustar00rootroot00000000000000package drivers import ( "fmt" "path/filepath" "strings" "golang.org/x/sys/unix" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/operations" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) type dir struct { common } // load is used to run one-time action per-driver rather than per-pool. func (d *dir) load() error { // Register the patches. d.patches = map[string]func() error{ "storage_lvm_skipactivation": nil, "storage_missing_snapshot_records": nil, "storage_delete_old_snapshot_records": nil, "storage_zfs_drop_block_volume_filesystem_extension": nil, "storage_prefix_bucket_names_with_project": nil, } return nil } // Info returns info about the driver and its environment. func (d *dir) Info() Info { return Info{ Name: "dir", Version: "1", DefaultVMBlockFilesystemSize: deviceConfig.DefaultVMBlockFilesystemSize, OptimizedImages: false, PreservesInodes: false, Remote: d.isRemote(), VolumeTypes: []VolumeType{VolumeTypeBucket, VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM}, VolumeMultiNode: d.isRemote(), BlockBacking: false, RunningCopyFreeze: true, DirectIO: true, IOUring: true, MountedRoot: true, Buckets: true, } } // FillConfig populates the storage pool's configuration file with the default values. func (d *dir) FillConfig() error { // Set default source if missing. if d.config["source"] == "" { d.config["source"] = GetPoolMountPath(d.name) } return nil } // Create is called during pool creation and is effectively using an empty driver struct. // WARNING: The Create() function cannot rely on any of the struct attributes being set. func (d *dir) Create() error { err := d.FillConfig() if err != nil { return err } sourcePath := d.config["source"] if !util.PathExists(sourcePath) { return fmt.Errorf("Source path '%s' doesn't exist", sourcePath) } // Check that if within INCUS_DIR, we're at our expected spot. cleanSource := filepath.Clean(sourcePath) varPath := strings.TrimRight(internalUtil.VarPath(), "/") + "/" if (cleanSource == internalUtil.VarPath() || strings.HasPrefix(cleanSource, varPath)) && cleanSource != GetPoolMountPath(d.name) { return fmt.Errorf("Source path '%s' is within the Incus directory", cleanSource) } // Check that the path is currently empty. isEmpty, err := internalUtil.PathIsEmpty(sourcePath) if err != nil { return err } if !isEmpty { return fmt.Errorf("Source path '%s' isn't empty", sourcePath) } return nil } // Delete removes the storage pool from the storage device. func (d *dir) Delete(op *operations.Operation) error { // On delete, wipe everything in the directory. err := wipeDirectory(GetPoolMountPath(d.name)) if err != nil { return err } // Unmount the path. _, err = d.Unmount() if err != nil { return err } return nil } // Validate checks that all provide keys are supported and that no conflicting or missing configuration is present. func (d *dir) Validate(config map[string]string) error { // gendoc:generate(entity=storage_dir, group=common, key=rsync.bwlimit) // // --- // type: string // scope: global // default: `0` (no limit) // shortdesc: The upper limit to be placed on the socket I/O when `rsync` must be used to transfer storage entities // gendoc:generate(entity=storage_dir, group=common, key=rsync.compression) // // --- // type: bool // scope: global // default: `true` // shortdesc: Whether to use compression while migrating storage pools // gendoc:generate(entity=storage_dir, group=common, key=source) // // --- // type: string // scope: local // default: - // shortdesc: Path to an existing directory return d.validatePool(config, nil, nil) } // Update applies any driver changes required from a configuration change. func (d *dir) Update(changedConfig map[string]string) error { return nil } // Mount mounts the storage pool. func (d *dir) Mount() (bool, error) { path := GetPoolMountPath(d.name) sourcePath := d.config["source"] // Check if we're dealing with an external mount. if sourcePath == path { return false, nil } // Check if already mounted. if sameMount(sourcePath, path) { return false, nil } // Setup the bind-mount. err := TryMount(sourcePath, path, "none", unix.MS_BIND, "") if err != nil { return false, err } return true, nil } // Unmount unmounts the storage pool. func (d *dir) Unmount() (bool, error) { path := GetPoolMountPath(d.name) // Check if we're dealing with an external mount. if d.config["source"] == path { return false, nil } // Unmount until nothing is left mounted. return forceUnmount(path) } // GetResources returns the pool resource usage information. func (d *dir) GetResources() (*api.ResourcesStoragePool, error) { return genericVFSGetResources(d) } incus-7.0.0/internal/server/storage/drivers/driver_dir_utils.go000066400000000000000000000066751517523235500250310ustar00rootroot00000000000000package drivers import ( "errors" "github.com/lxc/incus/v7/internal/server/storage/quota" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" ) // withoutGetVolID returns a copy of this struct but with a volIDFunc which will cause quotas to be skipped. func (d *dir) withoutGetVolID() Driver { newDriver := &dir{} getVolID := func(volType VolumeType, volName string) (int64, error) { return volIDQuotaSkip, nil } newDriver.init(d.state, d.name, d.config, d.logger, getVolID, d.commonRules) _ = newDriver.load() return newDriver } // setupInitialQuota enables quota on a new volume and sets with an initial quota from config. // Returns a revert fail function that can be used to undo this function if a subsequent step fails. func (d *dir) setupInitialQuota(vol Volume) (revert.Hook, error) { if vol.IsVMBlock() { return nil, nil } volPath := vol.MountPath() // Get the volume ID for the new volume, which is used to set project quota. volID, err := d.getVolID(vol.volType, vol.name) if err != nil { return nil, err } reverter := revert.New() defer reverter.Fail() // Define a function to revert the quota being setup. revertFunc := func() { _ = d.deleteQuota(volPath, volID) } reverter.Add(revertFunc) // Initialize the volume's project using the volume ID and set the quota. sizeBytes, err := units.ParseByteSizeString(vol.ConfigSize()) if err != nil { return nil, err } err = d.setQuota(volPath, volID, sizeBytes) if err != nil { return nil, err } reverter.Success() return revertFunc, nil } // deleteQuota removes the project quota for a volID from a path. func (d *dir) deleteQuota(path string, volID int64) error { if volID == volIDQuotaSkip { // Disabled on purpose, just ignore return nil } if volID == 0 { return errors.New("Missing volume ID") } ok, err := quota.Supported(path) if err != nil || !ok { // Skipping quota as underlying filesystem doesn't support project quotas. return nil } err = quota.DeleteProject(path, d.quotaProjectID(volID)) if err != nil { return err } return nil } // quotaProjectID generates a project quota ID from a volume ID. func (d *dir) quotaProjectID(volID int64) uint32 { if volID == volIDQuotaSkip { // Disabled on purpose, just ignore return 0 } return uint32(volID + 10000) } // setQuota sets the project quota on the path. The volID generates a quota project ID. func (d *dir) setQuota(path string, volID int64, sizeBytes int64) error { if volID == volIDQuotaSkip { // Disabled on purpose, just ignore. return nil } if volID == 0 { return errors.New("Missing volume ID") } ok, err := quota.Supported(path) if err != nil || !ok { if sizeBytes > 0 { // Skipping quota as underlying filesystem doesn't support project quotas. d.logger.Warn("The backing filesystem doesn't support quotas, skipping set quota", logger.Ctx{"path": path, "size": sizeBytes, "volID": volID}) } return nil } projectID := d.quotaProjectID(volID) currentProjectID, err := quota.GetProject(path) if err != nil { return err } // Clear and create new project if desired project ID is different. if currentProjectID != d.quotaProjectID(volID) { err = quota.DeleteProject(path, currentProjectID) if err != nil { return err } err = quota.SetProject(path, projectID) if err != nil { return err } } // Set the project quota size. return quota.SetProjectQuota(path, projectID, sizeBytes) } incus-7.0.0/internal/server/storage/drivers/driver_dir_volumes.go000066400000000000000000000531531517523235500253540ustar00rootroot00000000000000package drivers import ( "errors" "fmt" "io" "io/fs" "os" "path/filepath" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/rsync" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/storage/quota" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // CreateVolume creates an empty volume and can optionally fill it by executing the supplied // filler function. func (d *dir) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Operation) error { volPath := vol.MountPath() reverter := revert.New() defer reverter.Fail() if util.PathExists(vol.MountPath()) { return fmt.Errorf("Volume path %q already exists", vol.MountPath()) } // Create the volume itself. err := vol.EnsureMountPath(true) if err != nil { return err } reverter.Add(func() { _ = os.RemoveAll(volPath) }) // Get path to disk volume if volume is block or iso. rootBlockPath := "" if IsContentBlock(vol.contentType) { // We expect the filler to copy the VM image into this path. rootBlockPath, err = d.GetVolumeDiskPath(vol) if err != nil { return err } } else if vol.volType != VolumeTypeBucket { // Filesystem quotas only used with non-block volume types. revertFunc, err := d.setupInitialQuota(vol) if err != nil { return err } if revertFunc != nil { reverter.Add(revertFunc) } } // Run the volume filler function if supplied. err = genericRunFiller(d, vol, rootBlockPath, filler, false) if err != nil { return err } // If we are creating a block volume, resize it to the requested size or the default. // For block volumes, we expect the filler function to have converted the qcow2 image to raw into the rootBlockPath. // For ISOs the content will just be copied. if IsContentBlock(vol.contentType) { // Convert to bytes. sizeBytes, err := units.ParseByteSizeString(vol.ConfigSize()) if err != nil { return err } // Ignore ErrCannotBeShrunk when setting size this just means the filler run above has needed to // increase the volume size beyond the default block volume size. _, err = ensureVolumeBlockFile(vol, rootBlockPath, sizeBytes, false) if err != nil && !errors.Is(err, ErrCannotBeShrunk) { return err } // Move the GPT alt header to end of disk if needed and if filler specified. if vol.IsVMBlock() && filler != nil && filler.Fill != nil { err = d.moveGPTAltHeader(rootBlockPath) if err != nil { return err } } } reverter.Success() return nil } // CreateVolumeFromBackup restores a backup tarball onto the storage device. func (d *dir) CreateVolumeFromBackup(vol Volume, srcBackup backup.Info, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) (VolumePostHook, revert.Hook, error) { // Run the generic backup unpacker postHook, revertHook, err := genericVFSBackupUnpack(d.withoutGetVolID(), d.state.OS, vol, srcBackup.Snapshots, srcData, basePrefix, op) if err != nil { return nil, nil, err } // genericVFSBackupUnpack returns a nil postHook when volume's type is VolumeTypeCustom which // doesn't need any post hook processing after DB record creation. if postHook != nil { // Define a post hook function that can be run once the backup config has been restored. // This will setup the quota using the restored config. postHookWrapper := func(vol Volume) error { err := postHook(vol) if err != nil { return err } reverter := revert.New() defer reverter.Fail() revertQuota, err := d.setupInitialQuota(vol) if err != nil { return err } reverter.Add(revertQuota) reverter.Success() return nil } return postHookWrapper, revertHook, nil } return nil, revertHook, nil } // CreateVolumeFromCopy provides same-pool volume copying functionality. func (d *dir) CreateVolumeFromCopy(vol Volume, srcVol Volume, copySnapshots bool, allowInconsistent bool, op *operations.Operation) error { var err error var srcSnapshots []Volume if copySnapshots && !srcVol.IsSnapshot() { // Get the list of snapshots from the source. srcSnapshots, err = srcVol.Snapshots(op) if err != nil { return err } } // Run the generic copy. return genericVFSCopyVolume(d, d.setupInitialQuota, vol, srcVol, srcSnapshots, false, allowInconsistent, op) } // CreateVolumeFromMigration creates a volume being sent via a migration. func (d *dir) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, volTargetArgs migration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error { return genericVFSCreateVolumeFromMigration(d, d.setupInitialQuota, vol, conn, volTargetArgs, preFiller, op) } // RefreshVolume provides same-pool volume and specific snapshots syncing functionality. func (d *dir) RefreshVolume(vol Volume, srcVol Volume, srcSnapshots []Volume, allowInconsistent bool, op *operations.Operation) error { return genericVFSCopyVolume(d, d.setupInitialQuota, vol, srcVol, srcSnapshots, true, allowInconsistent, op) } // DeleteVolume deletes a volume of the storage device. If any snapshots of the volume remain then // this function will return an error. func (d *dir) DeleteVolume(vol Volume, op *operations.Operation) error { snapshots, err := d.VolumeSnapshots(vol, op) if err != nil { return err } if len(snapshots) > 0 { return errors.New("Cannot remove a volume that has snapshots") } volPath := vol.MountPath() // If the volume doesn't exist, then nothing more to do. if !util.PathExists(volPath) { return nil } // Get the volume ID for the volume, which is used to remove project quota. if vol.Type() != VolumeTypeBucket { volID, err := d.getVolID(vol.volType, vol.name) if err != nil { return err } // Remove the project quota. err = d.deleteQuota(volPath, volID) if err != nil { return err } } // Remove the volume from the storage device. err = forceRemoveAll(volPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", volPath, err) } // Although the volume snapshot directory should already be removed, lets remove it here // to just in case the top-level directory is left. err = deleteParentSnapshotDirIfEmpty(d.name, vol.volType, vol.name) if err != nil { return err } return nil } // HasVolume indicates whether a specific volume exists on the storage pool. func (d *dir) HasVolume(vol Volume) (bool, error) { return genericVFSHasVolume(vol) } // FillVolumeConfig populate volume with default config. func (d *dir) FillVolumeConfig(vol Volume) error { initialSize := vol.config["size"] err := d.fillVolumeConfig(&vol) if err != nil { return err } // Buckets do not support default volume size. // If size is specified manually, do not remove, so it triggers validation failure and an error to user. if vol.volType == VolumeTypeBucket && initialSize == "" { delete(vol.config, "size") } return nil } // ValidateVolume validates the supplied volume config. Optionally removes invalid keys from the volume's config. func (d *dir) ValidateVolume(vol Volume, removeUnknownKeys bool) error { // gendoc:generate(entity=storage_volume_dir, group=common, key=initial.gid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.gid` or `0` // shortdesc: GID of the volume owner in the instance // gendoc:generate(entity=storage_volume_dir, group=common, key=initial.mode) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.mode` or `711` // shortdesc: Mode of the volume in the instance // gendoc:generate(entity=storage_volume_dir, group=common, key=initial.uid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.uid` or `0` // shortdesc: UID of the volume owner in the instance // gendoc:generate(entity=storage_volume_dir, group=common, key=security.shared) // // --- // type: bool // condition: custom block volume // default: same as `volume.security.shared` or `false` // shortdesc: Enable sharing the volume across multiple instances // gendoc:generate(entity=storage_volume_dir, group=common, key=security.shifted) // // --- // type: bool // condition: custom volume // default: same as `volume.security.shifted` or `false` // shortdesc: {{enable_ID_shifting}} // gendoc:generate(entity=storage_volume_dir, group=common, key=security.unmapped) // // --- // type: bool // condition: custom volume // default: same as `volume.security.unmapped` or `false` // shortdesc: Disable ID mapping for the volume // gendoc:generate(entity=storage_volume_dir, group=common, key=security.size) // // --- // type: string // condition: appropriate driver // default: same as `volume.size` // shortdesc: Size/quota of the storage volume // gendoc:generate(entity=storage_volume_dir, group=common, key=snapshots.expiry) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_dir, group=common, key=snapshots.expiry.manual) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry.manual` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_dir, group=common, key=snapshots.pattern) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.pattern` or `snap%d` // shortdesc: {{snapshot_pattern_format}} [^*] // gendoc:generate(entity=storage_volume_dir, group=common, key=snapshots.schedule) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.schedule` // shortdesc: {{snapshot_schedule_format}} err := d.validateVolume(vol, nil, removeUnknownKeys) if err != nil { return err } if vol.config["size"] != "" && vol.volType == VolumeTypeBucket { return errors.New("Size cannot be specified for buckets") } return nil } // UpdateVolume applies config changes to the volume. func (d *dir) UpdateVolume(vol Volume, changedConfig map[string]string) error { newSize, sizeChanged := changedConfig["size"] if sizeChanged { err := d.SetVolumeQuota(vol, newSize, false, nil) if err != nil { return err } } return d.updateVolume(vol, changedConfig) } // GetVolumeUsage returns the disk space used by the volume. func (d *dir) GetVolumeUsage(vol Volume) (int64, error) { // Snapshot usage not supported for Dir. if vol.IsSnapshot() { return -1, ErrNotSupported } volPath := vol.MountPath() ok, err := quota.Supported(volPath) if err != nil || !ok { return -1, ErrNotSupported } // Get the volume ID for the volume to access quota. volID, err := d.getVolID(vol.volType, vol.name) if err != nil { return -1, err } projectID := d.quotaProjectID(volID) // Get project quota used. size, err := quota.GetProjectUsage(volPath, projectID) if err != nil { return -1, err } return size, nil } // SetVolumeQuota applies a size limit on volume. // Does nothing if supplied with an empty/zero size for block volumes, and for filesystem volumes removes quota. func (d *dir) SetVolumeQuota(vol Volume, size string, allowUnsafeResize bool, op *operations.Operation) error { // Convert to bytes. sizeBytes, err := units.ParseByteSizeString(size) if err != nil { return err } // For VM block files, resize the file if needed. if vol.contentType == ContentTypeBlock { // Do nothing if size isn't specified. if sizeBytes <= 0 { return nil } rootBlockPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } resized, err := ensureVolumeBlockFile(vol, rootBlockPath, sizeBytes, allowUnsafeResize) if err != nil { return err } // Move the GPT alt header to end of disk if needed and resize has taken place (not needed in // unsafe resize mode as it is expected the caller will do all necessary post resize actions // themselves). if vol.IsVMBlock() && resized && !allowUnsafeResize { err = d.moveGPTAltHeader(rootBlockPath) if err != nil { return err } } return nil } else if vol.Type() != VolumeTypeBucket { // For non-VM block volumes, set filesystem quota. volID, err := d.getVolID(vol.volType, vol.name) if err != nil { return err } // Custom handling for filesystem volume associated with a VM. volPath := vol.MountPath() if sizeBytes > 0 && vol.volType == VolumeTypeVM && util.PathExists(filepath.Join(volPath, genericVolumeDiskFile)) { // Get the size of the VM image. blockSize, err := BlockDiskSizeBytes(filepath.Join(volPath, genericVolumeDiskFile)) if err != nil { return err } // Add that to the requested filesystem size (to ignore it from the quota). sizeBytes += blockSize d.logger.Debug("Accounting for VM image file size", logger.Ctx{"sizeBytes": sizeBytes}) } return d.setQuota(vol.MountPath(), volID, sizeBytes) } return nil } // GetVolumeDiskPath returns the location of a disk volume. func (d *dir) GetVolumeDiskPath(vol Volume) (string, error) { return genericVFSGetVolumeDiskPath(vol) } // ListVolumes returns a list of volumes in storage pool. func (d *dir) ListVolumes() ([]Volume, error) { return genericVFSListVolumes(d) } // MountVolume simulates mounting a volume. func (d *dir) MountVolume(vol Volume, op *operations.Operation) error { unlock, err := vol.MountLock() if err != nil { return err } defer unlock() // Don't attempt to modify the permission of an existing custom volume root. // A user inside the instance may have modified this and we don't want to reset it on restart. if !util.PathExists(vol.MountPath()) || vol.volType != VolumeTypeCustom { err := vol.EnsureMountPath(false) if err != nil { return err } } vol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolume() when done. return nil } // UnmountVolume simulates unmounting a volume. // As driver doesn't have volumes to unmount it returns false indicating the volume was already unmounted. func (d *dir) UnmountVolume(vol Volume, keepBlockDev bool, op *operations.Operation) (bool, error) { unlock, err := vol.MountLock() if err != nil { return false, err } defer unlock() refCount := vol.MountRefCountDecrement() if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": vol.name, "refCount": refCount}) return false, ErrInUse } return false, nil } // RenameVolume renames a volume and its snapshots. func (d *dir) RenameVolume(vol Volume, newVolName string, op *operations.Operation) error { return genericVFSRenameVolume(d, vol, newVolName, op) } // MigrateVolume sends a volume for migration. func (d *dir) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *migration.VolumeSourceArgs, op *operations.Operation) error { return genericVFSMigrateVolume(d, d.state, vol, conn, volSrcArgs, op) } // BackupVolume copies a volume (and optionally its snapshots) to a specified target path. // This driver does not support optimized backups. func (d *dir) BackupVolume(vol Volume, writer instancewriter.InstanceWriter, basePrefix string, optimized bool, snapshots []string, op *operations.Operation) error { return genericVFSBackupVolume(d, vol, writer, basePrefix, snapshots, op) } // CreateVolumeSnapshot creates a snapshot of a volume. func (d *dir) CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error { parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) // Create snapshot directory. err := snapVol.EnsureMountPath(false) if err != nil { return err } reverter := revert.New() defer reverter.Fail() snapPath := snapVol.MountPath() reverter.Add(func() { _ = os.RemoveAll(snapPath) }) if snapVol.contentType != ContentTypeBlock || snapVol.volType != VolumeTypeCustom { var rsyncArgs []string if snapVol.IsVMBlock() { rsyncArgs = append(rsyncArgs, "--exclude", genericVolumeDiskFile) } bwlimit := d.config["rsync.bwlimit"] srcPath := GetVolumeMountPath(d.name, snapVol.volType, parentName) d.Logger().Debug("Copying filesystem volume", logger.Ctx{"sourcePath": srcPath, "targetPath": snapPath, "bwlimit": bwlimit, "rsyncArgs": rsyncArgs}) // Copy filesystem volume into snapshot directory. _, err = rsync.LocalCopy(srcPath, snapPath, bwlimit, true, rsyncArgs...) if err != nil { return err } } if snapVol.IsVMBlock() || (snapVol.contentType == ContentTypeBlock && snapVol.volType == VolumeTypeCustom) { parentVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, nil, d.config) srcDevPath, err := d.GetVolumeDiskPath(parentVol) if err != nil { return err } targetDevPath, err := d.GetVolumeDiskPath(snapVol) if err != nil { return err } d.Logger().Debug("Copying block volume", logger.Ctx{"srcDevPath": srcDevPath, "targetPath": targetDevPath}) err = ensureSparseFile(targetDevPath, 0) if err != nil { return err } err = copyDevice(srcDevPath, targetDevPath) if err != nil { return err } } reverter.Success() return nil } // DeleteVolumeSnapshot removes a snapshot from the storage device. The volName and snapshotName // must be bare names and should not be in the format "volume/snapshot". func (d *dir) DeleteVolumeSnapshot(snapVol Volume, op *operations.Operation) error { snapPath := snapVol.MountPath() // Remove the snapshot from the storage device. err := forceRemoveAll(snapPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", snapPath, err) } parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) // Remove the parent snapshot directory if this is the last snapshot being removed. err = deleteParentSnapshotDirIfEmpty(d.name, snapVol.volType, parentName) if err != nil { return err } return nil } // MountVolumeSnapshot sets up a read-only mount on top of the snapshot to avoid accidental modifications. func (d *dir) MountVolumeSnapshot(snapVol Volume, op *operations.Operation) error { unlock, err := snapVol.MountLock() if err != nil { return err } defer unlock() snapPath := snapVol.MountPath() // Don't attempt to modify the permission of an existing custom volume root. // A user inside the instance may have modified this and we don't want to reset it on restart. if !util.PathExists(snapPath) || snapVol.volType != VolumeTypeCustom { err := snapVol.EnsureMountPath(false) if err != nil { return err } } _, err = mountReadOnly(snapPath, snapPath) if err != nil { return err } snapVol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolumeSnapshot() when done. return nil } // UnmountVolumeSnapshot removes the read-only mount placed on top of a snapshot. func (d *dir) UnmountVolumeSnapshot(snapVol Volume, op *operations.Operation) (bool, error) { unlock, err := snapVol.MountLock() if err != nil { return false, err } defer unlock() mountPath := snapVol.MountPath() refCount := snapVol.MountRefCountDecrement() if linux.IsMountPoint(mountPath) { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": snapVol.name, "refCount": refCount}) return false, ErrInUse } snapPath := snapVol.MountPath() return forceUnmount(snapPath) } return false, nil } // VolumeSnapshots returns a list of snapshots for the volume (in no particular order). func (d *dir) VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) { return genericVFSVolumeSnapshots(d, vol, op) } // RestoreVolume restores a volume from a snapshot. func (d *dir) RestoreVolume(vol Volume, snapshotName string, op *operations.Operation) error { snapVol, err := vol.NewSnapshot(snapshotName) if err != nil { return err } srcPath := snapVol.MountPath() if !util.PathExists(srcPath) { return errors.New("Snapshot not found") } volPath := vol.MountPath() // Restore filesystem volume. if vol.contentType != ContentTypeBlock || vol.volType != VolumeTypeCustom { var rsyncArgs []string if vol.IsVMBlock() { rsyncArgs = append(rsyncArgs, "--exclude", genericVolumeDiskFile) } bwlimit := d.config["rsync.bwlimit"] _, err := rsync.LocalCopy(srcPath, volPath, bwlimit, true, rsyncArgs...) if err != nil { return fmt.Errorf("Failed to rsync volume: %w", err) } } // Restore block volume. if vol.IsVMBlock() || (vol.contentType == ContentTypeBlock && vol.volType == VolumeTypeCustom) { srcDevPath, err := d.GetVolumeDiskPath(snapVol) if err != nil { return err } targetDevPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } d.Logger().Debug("Restoring block volume", logger.Ctx{"srcDevPath": srcDevPath, "targetPath": targetDevPath}) err = ensureSparseFile(targetDevPath, 0) if err != nil { return err } err = copyDevice(srcDevPath, targetDevPath) if err != nil { return err } } return nil } // RenameVolumeSnapshot renames a volume snapshot. func (d *dir) RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *operations.Operation) error { return genericVFSRenameVolumeSnapshot(d, snapVol, newSnapshotName, op) } // ActivateTask allows running a function while the volume is active (but not mounted). func (d *dir) ActivateTask(vol Volume, task func(devPath string, op *operations.Operation) error, op *operations.Operation) error { // Prevent concurrent mounting actions. unlock, err := vol.MountLock() if err != nil { return err } defer unlock() volDevPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } // Run the task. return task(volDevPath, op) } incus-7.0.0/internal/server/storage/drivers/driver_linstor.go000066400000000000000000000305741517523235500245200ustar00rootroot00000000000000package drivers import ( "context" "fmt" "strconv" "time" "github.com/lxc/incus/v7/internal/migration" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var ( linstorVersion string linstorLoaded bool ) // linstor represents the Linstor storage driver. type linstor struct { common } func (d *linstor) load() error { // Register the patches. d.patches = map[string]func() error{ "storage_lvm_skipactivation": nil, "storage_missing_snapshot_records": nil, "storage_delete_old_snapshot_records": nil, "storage_zfs_drop_block_volume_filesystem_extension": nil, "storage_prefix_bucket_names_with_project": nil, } // Done if previously loaded. if linstorLoaded { return nil } // Validate the DRBD minimum version. The module should be already loaded by the Linstor satellite service. // // There are two major versions of DRBD, the one from the Linux kernel and the out of tree one used by Linstor. // We need to ensure that the system is using the out of tree version (>= 9.0) as both are available even on modern systems. drbdVer, err := d.drbdVersion() if err != nil { return err } ver, err := version.Parse(drbdVer) if err != nil { return fmt.Errorf("Could not determine DRBD module version: %w", err) } if ver.Major < 9 { return fmt.Errorf("Could not load Linstor driver: Linstor requires DRBD version 9.0 to be loaded, got: %s", ver) } // Get the controller version. controllerVer, err := d.controllerVersion() if err != nil { return err } linstorVersion = controllerVer + " / " + drbdVer linstorLoaded = true return nil } // isRemote returns true indicating this driver uses remote storage. func (d *linstor) isRemote() bool { return true } // Validate checks that all provide keys are supported and that no conflicting or missing configuration is present. func (d *linstor) Validate(config map[string]string) error { rules := map[string]func(value string) error{ // gendoc:generate(entity=storage_linstor, group=common, key=linstor.resource_group.name) // // --- // type: string // scope: global // default: `incus` // shortdesc: Name of the LINSTOR resource group that will be used for the storage pool LinstorResourceGroupNameConfigKey: validate.IsAny, // gendoc:generate(entity=storage_linstor, group=common, key=linstor.resource_group.place_count) // // --- // type: int // scope: global // default: `2` // shortdesc: Number of diskful replicas that should be created for resources in the resource group. Increasing the value of this option on a pool that already has volumes will result in LINSTOR creating new diskful replicas for all existing resources to match the new value LinstorResourceGroupPlaceCountConfigKey: validate.Optional(validate.IsUint32), // gendoc:generate(entity=storage_linstor, group=common, key=linstor.resource_group.storage_pool) // // --- // type: string // scope: global // default: - // shortdesc: The storage pool name in which resources should be placed on satellite nodes LinstorResourceGroupStoragePoolConfigKey: validate.IsAny, // gendoc:generate(entity=storage_linstor, group=common, key=linstor.volume.prefix) // // --- // type: string // scope: global // default: `incus-volume-` // shortdesc: The prefix to use for the internal names of LINSTOR-managed volumes. Cannot be updated after the storage pool is created LinstorVolumePrefixConfigKey: validate.IsShorterThan(24), // gendoc:generate(entity=storage_linstor, group=common, key=volatile.pool.pristine) // // --- // type: string // scope: global // default: `true` // shortdesc: Whether the pool was empty on creation time "volatile.pool.pristine": validate.IsAny, // gendoc:generate(entity=storage_linstor, group=common, key=drbd.on_no_quorum) // // --- // type: string // scope: global // default: - // shortdesc: The DRBD policy to use on resources when quorum is lost (applied to the resource group) DrbdOnNoQuorumConfigKey: validate.Optional(validate.IsOneOf("io-error", "suspend-io")), // gendoc:generate(entity=storage_linstor, group=common, key=drbd.auto_diskful) // // --- // type: string // scope: global // default: - // shortdesc: A duration string describing the time after which a primary diskless resource can be converted to diskful if storage is available on the node (applied to the resource group) DrbdAutoDiskfulConfigKey: validate.Optional(validate.IsMinimumDuration(time.Minute)), // gendoc:generate(entity=storage_linstor, group=common, key=drbd.auto_add_quorum_tiebreaker) // // --- // type: bool // scope: global // default: `true` // shortdesc: Whether to allow LINSTOR to automatically create diskless resources to act as quorum tiebreakers if needed (applied to the resource group) DrbdAutoAddQuorumTiebreakerConfigKey: validate.Optional(validate.IsBool), // gendoc:generate(entity=storage_linstor, group=common, key=source) // // --- // type: string // scope: global // default: `incus` // shortdesc: LINSTOR storage pool name. Alias for `linstor.resource_group.name`. Use either either one or the other or make sure they have the same value. "source": validate.IsAny, } return d.validatePool(config, rules, d.commonVolumeRules()) } // FillConfig populates the storage pool's configuration file with the default values. func (d *linstor) FillConfig() error { if d.config[LinstorResourceGroupPlaceCountConfigKey] == "" { d.config[LinstorResourceGroupPlaceCountConfigKey] = LinstorDefaultResourceGroupPlaceCount } if d.config[LinstorVolumePrefixConfigKey] == "" { d.config[LinstorVolumePrefixConfigKey] = LinstorDefaultVolumePrefix } if d.config[DrbdOnNoQuorumConfigKey] == "" { d.config[DrbdOnNoQuorumConfigKey] = "suspend-io" } if d.config[DrbdAutoAddQuorumTiebreakerConfigKey] == "" { d.config[DrbdAutoAddQuorumTiebreakerConfigKey] = "true" } return nil } // Create is called during storage pool creation. func (d *linstor) Create() error { d.logger.Debug("Creating Linstor storage pool") rev := revert.New() defer rev.Fail() // Track the initial source. d.config["volatile.initial_source"] = d.config["source"] // Fill default config values. err := d.FillConfig() if err != nil { return fmt.Errorf("Could not create Linstor storage pool: %w", err) } // Quick check of conflicting values. if d.config["source"] != "" && d.config[LinstorResourceGroupNameConfigKey] != "" && d.config["source"] != d.config[LinstorResourceGroupNameConfigKey] { return fmt.Errorf(`The "source" and %q property must not differ for LINSTOR storage pools`, LinstorResourceGroupNameConfigKey) } // If a source is provided, use it as the resource group name. if d.config["source"] != "" { d.config[LinstorResourceGroupNameConfigKey] = d.config["source"] } else if d.config[LinstorResourceGroupNameConfigKey] == "" { d.config[LinstorResourceGroupNameConfigKey] = d.name } d.config["source"] = d.config[LinstorResourceGroupNameConfigKey] resourceGroupExists, err := d.resourceGroupExists() if err != nil { return fmt.Errorf("Could not create Linstor storage pool: %w", err) } if !resourceGroupExists { // Create new resource group. d.logger.Debug("Resource group does not exist. Creating one") err := d.createResourceGroup() if err != nil { return fmt.Errorf("Could not create Linstor storage pool: %w", err) } rev.Add(func() { _ = d.deleteResourceGroup() }) d.config["volatile.pool.pristine"] = "true" } else { d.logger.Debug("Resource group already exists. Using an existing one") resourceGroup, err := d.getResourceGroup() if err != nil { return fmt.Errorf("Could not create Linstor storage pool: %w", err) } d.config[LinstorResourceGroupPlaceCountConfigKey] = strconv.Itoa(int(resourceGroup.SelectFilter.PlaceCount)) d.config[LinstorResourceGroupStoragePoolConfigKey] = resourceGroup.SelectFilter.StoragePool } rev.Success() return nil } // Delete removes the storage pool from the storage device. func (d *linstor) Delete(op *operations.Operation) error { d.logger.Debug("Deleting Linstor storage pool") // Test if the resource group exists. resourceGroupExists, err := d.resourceGroupExists() if err != nil { return fmt.Errorf("Could not check if Linstor resource group exists: %w", err) } if !resourceGroupExists { d.logger.Warn("Resource group does not exist") } else { // Check whether we own the resource group and only remove in this case. if util.IsTrue(d.config["volatile.pool.pristine"]) { // Delete the resource group pool. err := d.deleteResourceGroup() if err != nil { return err } d.logger.Debug("Deleted Linstor resource group") } else { d.logger.Debug("Linstor resource group is not owned by Incus, skipping delete") } } // If the user completely destroyed it, call it done. if !util.PathExists(GetPoolMountPath(d.name)) { return nil } // On delete, wipe everything in the directory. err = wipeDirectory(GetPoolMountPath(d.name)) if err != nil { return err } return nil } // Info returns info about the driver and its environment. func (d *linstor) Info() Info { return Info{ Name: "linstor", Version: linstorVersion, VolumeTypes: []VolumeType{VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM}, DefaultVMBlockFilesystemSize: deviceConfig.DefaultVMBlockFilesystemSize, Buckets: false, Remote: d.isRemote(), VolumeMultiNode: false, // DRBD uses an active-passive replication paradigm, so we cannot use the same volume concurrently in multiple nodes. OptimizedImages: true, OptimizedBackups: false, OptimizedBackupHeader: false, PreservesInodes: false, BlockBacking: true, RunningCopyFreeze: true, DirectIO: true, IOUring: true, MountedRoot: false, Deactivate: false, } } // Mount mounts the storage pool. func (d *linstor) Mount() (bool, error) { linstorClient, err := d.state.Linstor() if err != nil { return false, err } satelliteName := d.getSatelliteName() node, err := linstorClient.Client.Nodes.Get(context.TODO(), satelliteName) if err != nil { return false, err } if node.ConnectionStatus != "ONLINE" { return false, fmt.Errorf("Node %s is offline", satelliteName) } return true, nil } // Unmount unmounts the storage pool. func (d *linstor) Unmount() (bool, error) { return true, nil } // Update applies any driver changes required from a configuration change. func (d *linstor) Update(changedConfig map[string]string) error { _, changed := changedConfig[LinstorResourceGroupNameConfigKey] if changed { return fmt.Errorf("%s cannot be changed", LinstorResourceGroupNameConfigKey) } _, changed = changedConfig[LinstorVolumePrefixConfigKey] if changed { return fmt.Errorf("%s cannot be changed", LinstorVolumePrefixConfigKey) } err := d.updateResourceGroup(changedConfig) if err != nil { return err } return nil } // GetResources returns utilisation and space info about the pool. func (d *linstor) GetResources() (*api.ResourcesStoragePool, error) { freeCapacity, totalCapacity, err := d.getResourceGroupSize() if err != nil { return nil, fmt.Errorf("Could not fetch pool space info: %w", err) } // We have no information about inode usage, so we skip that. res := api.ResourcesStoragePool{} res.Space.Total = uint64(totalCapacity) * 1024 res.Space.Used = uint64(totalCapacity-freeCapacity) * 1024 return &res, nil } // MigrationTypes returns the type of transfer methods to be used when doing migrations between pools in preference order. func (d *linstor) MigrationTypes(contentType ContentType, refresh bool, copySnapshots bool, clusterMove bool, storageMove bool) []localMigration.Type { if !clusterMove || storageMove { return []localMigration.Type{d.rsyncMigrationType(contentType)} } return []localMigration.Type{ { FSType: migration.MigrationFSType_LINSTOR, }, d.rsyncMigrationType(contentType), } } incus-7.0.0/internal/server/storage/drivers/driver_linstor_utils.go000066400000000000000000001110421517523235500257260ustar00rootroot00000000000000package drivers import ( "context" "errors" "fmt" "maps" "os" "path/filepath" "slices" "strconv" "strings" "time" linstorClient "github.com/LINBIT/golinstor/client" "github.com/LINBIT/golinstor/clonestatus" "github.com/google/uuid" "github.com/lxc/incus/v7/internal/migration" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) // LinstorSatellitePaths lists the possible FS paths for the Satellite script. var LinstorSatellitePaths = []string{"/usr/share/linstor-server/bin"} // LinstorDefaultResourceGroupPlaceCount represents the default Linstor resource group place count. const LinstorDefaultResourceGroupPlaceCount = "2" // LinstorDefaultVolumePrefix represents the default Linstor volume prefix. const LinstorDefaultVolumePrefix = "incus-volume-" // LinstorResourceGroupNameConfigKey represents the config key that describes the resource group name. const LinstorResourceGroupNameConfigKey = "linstor.resource_group.name" // LinstorResourceGroupPlaceCountConfigKey represents the config key that describes the resource group place count. const LinstorResourceGroupPlaceCountConfigKey = "linstor.resource_group.place_count" // LinstorResourceGroupStoragePoolConfigKey represents the config key that describes the resource group storage pool. const LinstorResourceGroupStoragePoolConfigKey = "linstor.resource_group.storage_pool" // LinstorVolumePrefixConfigKey represents the config key that describes the prefix to add to every volume within a storage pool. const LinstorVolumePrefixConfigKey = "linstor.volume.prefix" // DrbdOnNoQuorumConfigKey represents the config key that describes the DRBD policy when quorum is not reached. const DrbdOnNoQuorumConfigKey = "drbd.on_no_quorum" // DrbdAutoDiskfulConfigKey represents the config key that describes the DRBD timeout for toggling a diskful resource. const DrbdAutoDiskfulConfigKey = "drbd.auto_diskful" // DrbdAutoAddQuorumTiebreakerConfigKey represents the config key that describes whether DRBD will automatically create tiebreaker resources. const DrbdAutoAddQuorumTiebreakerConfigKey = "drbd.auto_add_quorum_tiebreaker" // LinstorRemoveSnapshotsConfigKey represents the config key that describes whether snapshots should be automatically removed with volumes. const LinstorRemoveSnapshotsConfigKey = "linstor.remove_snapshots" // LinstorAuxSnapshotPrefix represents the AuxProp prefix to map Incus and LINSTOR snapshots. const LinstorAuxSnapshotPrefix = "Aux/Incus/snapshot-name/" // LinstorAuxName represents the AuxProp storing the Incus volume name. const LinstorAuxName = "Aux/Incus/name" // LinstorAuxType represents the AuxProp storing the Incus volume type. const LinstorAuxType = "Aux/Incus/type" // LinstorAuxContentType represents the AuxProp storing the Incus volume content type. const LinstorAuxContentType = "Aux/Incus/content-type" // errResourceDefinitionNotFound indicates that a resource definition could not be found in Linstor. var errResourceDefinitionNotFound = errors.New("Resource definition not found") // errSnapshotNotFound indicates that a snapshot could not be found in Linstor. var errSnapshotNotFound = errors.New("Resource definition not found") // drbdPropsMap maps incus config keys to DRBD options. var drbdPropsMap = map[string]string{ DrbdOnNoQuorumConfigKey: "DrbdOptions/Resource/on-no-quorum", DrbdAutoDiskfulConfigKey: "DrbdOptions/auto-diskful", DrbdAutoAddQuorumTiebreakerConfigKey: "DrbdOptions/auto-add-quorum-tiebreaker", } // drbdVersion returns the DRBD version of the currently loaded kernel module. func (d *linstor) drbdVersion() (string, error) { modulePath := "/sys/module/drbd/version" if !util.PathExists(modulePath) { return "", errors.New("Could not determine DRBD module version: Module not loaded") } ver, err := os.ReadFile(modulePath) if err != nil { return "", fmt.Errorf("Could not determine DRBD module version: %w", err) } return strings.TrimSpace(string(ver)), nil } // controllerVersion returns the LINSTOR controller version. func (d *linstor) controllerVersion() (string, error) { var satellitePath string for _, path := range LinstorSatellitePaths { candidate := filepath.Join(path, "Satellite") _, err := os.Stat(candidate) if err == nil { satellitePath = candidate break } } if satellitePath == "" { return "", errors.New("LINSTOR satellite executable not found") } out, err := subprocess.RunCommand(satellitePath, "--version") if err != nil { return "", err } for _, line := range strings.Split(out, "\n") { if strings.HasPrefix(line, "Version:") { fields := strings.Fields(line) if len(fields) < 2 { return "", errors.New("Could not parse LINSTOR satellite version") } return fields[1], nil } } return "", errors.New("Could not parse LINSTOR satellite version") } // resourceGroupExists returns whether the resource group associated with the current storage pool exists. func (d *linstor) resourceGroupExists() (bool, error) { resourceGroup, err := d.getResourceGroup() if err != nil { return false, fmt.Errorf("Could not get resource group: %w", err) } if resourceGroup == nil { return false, nil } return true, nil } // getResourceGroup fetches the resource group for the storage pool. func (d *linstor) getResourceGroup() (*linstorClient.ResourceGroup, error) { d.logger.Debug("Fetching Linstor resource group") // Retrieve the Linstor client. linstor, err := d.state.Linstor() if err != nil { return nil, err } resourceGroupName := d.config[LinstorResourceGroupNameConfigKey] resourceGroup, err := linstor.Client.ResourceGroups.Get(context.TODO(), resourceGroupName) if errors.Is(err, linstorClient.NotFoundError) { return nil, nil } else if err != nil { return nil, fmt.Errorf("Could not get Linstor resource group: %w", err) } return &resourceGroup, nil } // getResourceGroupSize fetches the resource group size info. func (d *linstor) getResourceGroupSize() (int64, int64, error) { // Retrieve the Linstor client. linstor, err := d.state.Linstor() if err != nil { return 0, 0, err } resourceGroupName := d.config[LinstorResourceGroupNameConfigKey] resourceGroup, err := linstor.Client.ResourceGroups.Get(context.TODO(), resourceGroupName) if err != nil { return 0, 0, fmt.Errorf("Could not get Linstor resource group: %w", err) } storagePools, err := linstor.Client.Nodes.GetStoragePoolView(context.TODO(), &linstorClient.ListOpts{ StoragePool: resourceGroup.SelectFilter.StoragePoolList, }) if err != nil { return 0, 0, fmt.Errorf("Could not get Linstor storage pools: %w", err) } freeCapacity := int64(0) totalCapacity := int64(0) for _, storagePool := range storagePools { freeCapacity += storagePool.FreeCapacity totalCapacity += storagePool.TotalCapacity } return freeCapacity, totalCapacity, nil } // createResourceGroup creates a new resource group for the storage pool. func (d *linstor) createResourceGroup() error { d.logger.Debug("Creating Linstor resource group") // Retrieve the Linstor client. linstor, err := d.state.Linstor() if err != nil { return err } placeCount, err := strconv.Atoi(d.config[LinstorResourceGroupPlaceCountConfigKey]) if err != nil { return fmt.Errorf("Could not parse resource group place count property: %w", err) } resourceGroup := linstorClient.ResourceGroup{ Name: d.config[LinstorResourceGroupNameConfigKey], Description: "Resource group managed by Incus", SelectFilter: linstorClient.AutoSelectFilter{ PlaceCount: int32(placeCount), }, } if d.config[LinstorResourceGroupStoragePoolConfigKey] != "" { resourceGroup.SelectFilter.StoragePool = d.config[LinstorResourceGroupStoragePoolConfigKey] } // Create the resource group. err = linstor.Client.ResourceGroups.Create(context.TODO(), resourceGroup) if err != nil { return fmt.Errorf("Could not create Linstor resource group : %w", err) } // Set additional properties based on the config. props, err := d.drbdPropsFromConfig(d.config) if err != nil { return fmt.Errorf("Could parse config into DRBD props: %w", err) } // Some tuning to speed up resync. if props["DrbdOptions/Disk/rs-discard-granularity"] == "" { props["DrbdOptions/Disk/rs-discard-granularity"] = "1048576" } err = linstor.Client.ResourceGroups.Modify(context.TODO(), resourceGroup.Name, linstorClient.ResourceGroupModify{ OverrideProps: props, }) if err != nil { return fmt.Errorf("Could not set properties on Linstor resource group : %w", err) } return nil } // updateResourceGroup updates the resource group for the storage pool. func (d *linstor) updateResourceGroup(changedConfig map[string]string) error { d.logger.Debug("Updating Linstor resource group") // Retrieve the Linstor client. linstor, err := d.state.Linstor() if err != nil { return err } resourceGroupModify := linstorClient.ResourceGroupModify{} placeCount, changed := changedConfig[LinstorResourceGroupPlaceCountConfigKey] if changed { placeCount, err := strconv.Atoi(placeCount) if err != nil { return fmt.Errorf("Could not parse resource group place count property: %w", err) } resourceGroupModify.SelectFilter.PlaceCount = int32(placeCount) } storagePool, changed := changedConfig[LinstorResourceGroupStoragePoolConfigKey] if changed { resourceGroupModify.SelectFilter.StoragePool = storagePool } // Parse and set properties to be overwritten. overrideProps, err := d.drbdPropsFromConfig(changedConfig) if err != nil { return fmt.Errorf("Could parse config into DRBD props: %w", err) } resourceGroupModify.OverrideProps = overrideProps // Parse and set properties to be deleted. deleteProps := []string{} for key, value := range changedConfig { if value != "" { continue } prop, ok := drbdPropsMap[key] if ok { deleteProps = append(deleteProps, prop) } } resourceGroupModify.DeleteProps = deleteProps resourceGroupName := d.config[LinstorResourceGroupNameConfigKey] err = linstor.Client.ResourceGroups.Modify(context.TODO(), resourceGroupName, resourceGroupModify) if err != nil { return fmt.Errorf("Could not update Linstor resource group : %w", err) } return nil } // deleteResourceGroup deleter the resource group for the storage pool. func (d *linstor) deleteResourceGroup() error { // Retrieve the Linstor client. linstor, err := d.state.Linstor() if err != nil { return err } err = linstor.Client.ResourceGroups.Delete(context.TODO(), d.config[LinstorResourceGroupNameConfigKey]) if err != nil { return fmt.Errorf("Could not delete Linstor resource group : %w", err) } return nil } // getResourceDefinition returns the Linstor resource definition for a given volume. func (d *linstor) getResourceDefinition(vol Volume, fetchVolumeDefinitions bool) (linstorClient.ResourceDefinitionWithVolumeDefinition, error) { l := logger.AddContext(logger.Ctx{"vol": vol.name, "volType": vol.volType, "contentType": vol.contentType}) l.Debug("Getting resource definition for volume") linstor, err := d.state.Linstor() if err != nil { return linstorClient.ResourceDefinitionWithVolumeDefinition{}, err } // Query resource definitions that match the desired volume by its name. resourceDefinitions, err := linstor.Client.ResourceDefinitions.GetAll(context.TODO(), linstorClient.RDGetAllRequest{ Props: []string{ LinstorAuxName + "=" + d.config[LinstorVolumePrefixConfigKey] + vol.name, LinstorAuxType + "=" + string(vol.volType), }, WithVolumeDefinitions: fetchVolumeDefinitions, }) if err != nil { return linstorClient.ResourceDefinitionWithVolumeDefinition{}, err } l.Debug("Queried resource definitions", logger.Ctx{"query": LinstorAuxName + "=" + d.config[LinstorVolumePrefixConfigKey] + vol.name, "result": resourceDefinitions}) // Filter resource definitions for the storage pool's resource group. var filteredResourceDefinitions []linstorClient.ResourceDefinitionWithVolumeDefinition for _, rd := range resourceDefinitions { if rd.ResourceGroupName == d.config[LinstorResourceGroupNameConfigKey] { filteredResourceDefinitions = append(filteredResourceDefinitions, rd) } } if len(filteredResourceDefinitions) == 0 { return linstorClient.ResourceDefinitionWithVolumeDefinition{}, errResourceDefinitionNotFound } else if len(filteredResourceDefinitions) > 1 { return linstorClient.ResourceDefinitionWithVolumeDefinition{}, fmt.Errorf("Multiple resource definitions found for volume %s", vol.name) } return filteredResourceDefinitions[0], nil } // getLinstorDevPath return the device path for a given `vol` in the current node. // // If the resource is not available on the current node, it is made available before // fetching its device path. func (d *linstor) getLinstorDevPath(vol Volume) (string, error) { l := logger.AddContext(logger.Ctx{"vol": vol.name, "volType": vol.volType, "contentType": vol.contentType}) l.Debug("Getting device path") linstor, err := d.state.Linstor() if err != nil { return "", err } resourceDefinition, err := d.getResourceDefinition(vol, false) if err != nil { return "", err } // When retrieving the device path, always make sure that the resource is available on the current node. err = linstor.Client.Resources.MakeAvailable(context.TODO(), resourceDefinition.Name, d.getSatelliteName(), linstorClient.ResourceMakeAvailable{ Diskful: false, }) if err != nil { l.Debug("Could not make resource available on node", logger.Ctx{"resourceDefinitionName": resourceDefinition.Name, "nodeName": d.getSatelliteName(), "error": err}) return "", fmt.Errorf("Could not make resource available on node: %w", err) } volumes, err := linstor.Client.Resources.GetVolumes(context.TODO(), resourceDefinition.Name, d.getSatelliteName()) if err != nil { return "", fmt.Errorf("Unable to get Linstor volumes: %w", err) } volumeIndex := 0 if len(volumes) == 2 { // For VM volumes, the associated filesystem volume is a second volume on the same LINSTOR resource. if (vol.volType == VolumeTypeVM || vol.volType == VolumeTypeImage) && vol.contentType == ContentTypeFS { volumeIndex = 1 } } return volumes[volumeIndex].DevicePath, nil } // deleteDisklessResource deletes the diskless resource for the given volume in the current node if one exists. func (d *linstor) deleteDisklessResource(vol Volume) error { l := d.logger.AddContext(logger.Ctx{"volume": vol.Name()}) l.Debug("Checking for diskless resources to delete") linstor, err := d.state.Linstor() if err != nil { return err } resourceDefinition, err := d.getResourceDefinition(vol, false) if err != nil { return err } // Fetch all resources for the given volume in the current node. resources, err := linstor.Client.Resources.GetResourceView(context.TODO(), &linstorClient.ListOpts{ Resource: []string{resourceDefinition.Name}, Node: []string{d.getSatelliteName()}, }) if err != nil { return fmt.Errorf("Unable to get the resources for the resource definition: %w", err) } l.Debug("Got resources for the volume in the current node", logger.Ctx{"resources": resources}) // Delete the DISKLESS resources. for _, r := range resources { if slices.Contains(r.Flags, "DISKLESS") { l.Debug("Deleting diskless resource") err := linstor.Client.Resources.Delete(context.TODO(), r.Name, r.NodeName) if err != nil { return err } l.Debug("Deleted diskless resource") } } return nil } // getVolumeUsage returns the allocated size for a given volume in KiB. func (d *linstor) getVolumeUsage(vol Volume) (int64, error) { l := d.logger.AddContext(logger.Ctx{"volume": vol.Name()}) l.Debug("Getting volume usage") linstor, err := d.state.Linstor() if err != nil { return 0, err } resourceDefinition, err := d.getResourceDefinition(vol, false) if err != nil { return 0, err } // Fetch all resources for the given volume. resources, err := linstor.Client.Resources.GetResourceView(context.TODO(), &linstorClient.ListOpts{ Resource: []string{resourceDefinition.Name}, }) if err != nil { return 0, fmt.Errorf("Unable to get the resources for the resource definition: %w", err) } var resource *linstorClient.ResourceWithVolumes // Find the first non DISKLESS resource. for _, r := range resources { l.Debug("Volume flags", logger.Ctx{"node": r.NodeName, "flags": r.Flags}) if !slices.Contains(r.Flags, "DISKLESS") { resource = &r } } // If no diskful resource is found, usage cannot be determined. if resource == nil { l.Warn("No diskful resource found for volume") return 0, nil } volumes := resource.Volumes volumeIndex := 0 if len(volumes) == 2 { // For VM volumes, the associated filesystem volume is a second volume on the same LINSTOR resource. if (vol.volType == VolumeTypeVM || vol.volType == VolumeTypeImage) && vol.contentType == ContentTypeFS { volumeIndex = 1 } } return volumes[volumeIndex].AllocatedSizeKib, nil } // getSatelliteName returns the local LINSTOR satellite name. // // The logic used to determine the satellite name is documented in the public // driver documentation, as it is relevant for users. Therefore, any changes // to the external behavior of this function should also be reflected in the // public documentation. func (d *linstor) getSatelliteName() string { name := d.state.LocalConfig.LinstorSatelliteName() if name != "" { return name } if d.state.ServerClustered { return d.state.ServerName } return d.state.OS.Hostname } // makeVolumeAvailable makes a volume available on the current node. func (d *linstor) makeVolumeAvailable(vol Volume) error { l := d.logger.AddContext(logger.Ctx{"volume": vol.Name()}) l.Debug("Making volume available on node") linstor, err := d.state.Linstor() if err != nil { return err } resourceDefinition, err := d.getResourceDefinition(vol, false) if err != nil { return err } err = linstor.Client.Resources.MakeAvailable(context.TODO(), resourceDefinition.Name, d.getSatelliteName(), linstorClient.ResourceMakeAvailable{ Diskful: false, }) if err != nil { l.Debug("Could not make resource available on node", logger.Ctx{"resourceDefinitionName": resourceDefinition.Name, "nodeName": d.getSatelliteName(), "error": err}) return fmt.Errorf("Could not make resource available on node: %w", err) } return nil } // createVolumeSnapshot creates a volume snapshot. func (d *linstor) createVolumeSnapshot(parentVol Volume, snapVol Volume) error { l := d.logger.AddContext(logger.Ctx{"parentVol": parentVol.Name(), "snapVol": snapVol.Name()}) l.Debug("Creating volume snapshot") linstor, err := d.state.Linstor() if err != nil { return err } resourceDefinition, err := d.getResourceDefinition(parentVol, false) if err != nil { return err } linstorSnapshotName := d.generateUUIDWithPrefix() err = linstor.Client.Resources.CreateSnapshot(context.TODO(), linstorClient.Snapshot{ Name: linstorSnapshotName, ResourceName: resourceDefinition.Name, }) if err != nil { return fmt.Errorf("Could not create resource snapshot: %w", err) } _, snapshotName, _ := api.GetParentAndSnapshotName(snapVol.Name()) err = linstor.Client.ResourceDefinitions.Modify(context.TODO(), resourceDefinition.Name, linstorClient.GenericPropsModify{ OverrideProps: map[string]string{LinstorAuxSnapshotPrefix + snapshotName: linstorSnapshotName}, }) if err != nil { _ = linstor.Client.Resources.DeleteSnapshot(context.TODO(), resourceDefinition.Name, linstorSnapshotName) return err } return nil } // renameVolumeSnapshot renames a volume snapshot. func (d *linstor) renameVolumeSnapshot(parentVol Volume, snapVol Volume, newSnapshotName string) error { l := d.logger.AddContext(logger.Ctx{"parentVol": parentVol.Name(), "snapVol": snapVol.Name()}) l.Debug("Renaming volume snapshot") linstor, err := d.state.Linstor() if err != nil { return err } resourceDefinition, err := d.getResourceDefinition(parentVol, false) if err != nil { return err } // Get the snapshot name. _, snapshotName, _ := api.GetParentAndSnapshotName(snapVol.Name()) linstorSnapshotName, ok := resourceDefinition.Props[LinstorAuxSnapshotPrefix+snapshotName] if !ok { return fmt.Errorf("Could not find snapshot name mapping for volume %s", parentVol.Name()) } return linstor.Client.ResourceDefinitions.Modify(context.TODO(), resourceDefinition.Name, linstorClient.GenericPropsModify{ DeleteProps: linstorClient.DeleteProps{LinstorAuxSnapshotPrefix + snapshotName}, OverrideProps: map[string]string{LinstorAuxSnapshotPrefix + newSnapshotName: linstorSnapshotName}, }) } // deleteVolumeSnapshot deletes a volume snapshot. func (d *linstor) deleteVolumeSnapshot(parentVol Volume, snapVol Volume) error { l := d.logger.AddContext(logger.Ctx{"parentVol": parentVol.Name(), "snapVol": snapVol.Name()}) l.Debug("Deleting volume snapshot") linstor, err := d.state.Linstor() if err != nil { return err } resourceDefinition, err := d.getResourceDefinition(parentVol, false) if err != nil { return err } // Get the snapshot name. _, snapshotName, _ := api.GetParentAndSnapshotName(snapVol.Name()) linstorSnapshotName, ok := resourceDefinition.Props[LinstorAuxSnapshotPrefix+snapshotName] if !ok { return fmt.Errorf("Could not find snapshot name mapping for volume %s", parentVol.Name()) } err = linstor.Client.Resources.DeleteSnapshot(context.TODO(), resourceDefinition.Name, linstorSnapshotName) if err != nil { return fmt.Errorf("Could not delete resource snapshot: %w", err) } err = linstor.Client.ResourceDefinitions.Modify(context.TODO(), resourceDefinition.Name, linstorClient.GenericPropsModify{ DeleteProps: []string{LinstorAuxSnapshotPrefix + snapshotName}, }) if err != nil { return fmt.Errorf("Could not delete snapshot name mapping aux property: %w", err) } return nil } // getSnapshotMap gets the map from LINSTOR snapshot names to Incus’. func (d *linstor) getSnapshotMap(parentVol Volume) (map[string]string, error) { l := d.logger.AddContext(logger.Ctx{"parentVol": parentVol.Name()}) l.Debug("Getting snapshot map") result := make(map[string]string) resourceDefinition, err := d.getResourceDefinition(parentVol, false) if err != nil { return result, err } for key, value := range resourceDefinition.Props { after, ok := strings.CutPrefix(key, LinstorAuxSnapshotPrefix) if ok { result[value] = after } } return result, nil } // snapshotExists returns whether the given snapshot exists. func (d *linstor) snapshotExists(parentVol Volume, snapVol Volume) (bool, error) { l := d.logger.AddContext(logger.Ctx{"parentVol": parentVol.Name(), "snapVol": snapVol.Name()}) l.Debug("Fetching Linstor snapshot") // Retrieve the Linstor client. linstor, err := d.state.Linstor() if err != nil { return false, err } resourceDefinition, err := d.getResourceDefinition(parentVol, false) if err != nil { if errors.Is(err, errResourceDefinitionNotFound) { return false, nil } return false, err } linstorSnapshotName, err := d.getLinstorSnapshotName(snapVol, resourceDefinition) if err != nil { if errors.Is(err, errSnapshotNotFound) { return false, nil } return false, err } _, err = linstor.Client.Resources.GetSnapshot(context.TODO(), resourceDefinition.Name, linstorSnapshotName) if errors.Is(err, linstorClient.NotFoundError) { l.Debug("Snapshot not found") return false, nil } else if err != nil { return false, fmt.Errorf("Could not get snapshot: %w", err) } l.Debug("Got snapshot") return true, nil } // getLinstorSnapshotName returns the Linstor snapshot name given a snapshot and its parent resource definition. func (d *linstor) getLinstorSnapshotName(snapVol Volume, resourceDefinition linstorClient.ResourceDefinitionWithVolumeDefinition) (string, error) { _, snapshotName, _ := api.GetParentAndSnapshotName(snapVol.Name()) linstorSnapshotName, ok := resourceDefinition.Props[LinstorAuxSnapshotPrefix+snapshotName] if !ok { return "", errSnapshotNotFound } return linstorSnapshotName, nil } // restoreVolume restores a volume state from a snapshot. func (d *linstor) restoreVolume(vol Volume, snapVol Volume) error { l := d.logger.AddContext(logger.Ctx{"vol": vol.Name(), "snapVol": snapVol.Name()}) l.Debug("Restoring volume to snapshot") linstor, err := d.state.Linstor() if err != nil { return err } resourceDefinition, err := d.getResourceDefinition(vol, false) if err != nil { return err } linstorSnapshotName, err := d.getLinstorSnapshotName(snapVol, resourceDefinition) if err != nil { return err } err = linstor.Client.Resources.RollbackSnapshot(context.TODO(), resourceDefinition.Name, linstorSnapshotName) if err != nil { return fmt.Errorf("Could not restore volume to snapshot: %w", err) } return nil } // createResourceDefinitionFromSnapshot creates a new resource definition from a snapshot. func (d *linstor) createResourceDefinitionFromSnapshot(snapVol Volume, vol Volume) error { linstor, err := d.state.Linstor() if err != nil { return err } rev := revert.New() defer rev.Fail() parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) parentVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, nil, nil) parentResourceDefinition, err := d.getResourceDefinition(parentVol, false) if err != nil { return err } linstorSnapshotName, err := d.getLinstorSnapshotName(snapVol, parentResourceDefinition) if err != nil { return err } resourceDefinitionName := d.generateUUIDWithPrefix() err = linstor.Client.ResourceDefinitions.Create(context.TODO(), linstorClient.ResourceDefinitionCreate{ ResourceDefinition: linstorClient.ResourceDefinition{ Name: resourceDefinitionName, ResourceGroupName: d.config[LinstorResourceGroupNameConfigKey], }, }) if err != nil { return fmt.Errorf("Could not create resource definition from snapshot: %w", err) } rev.Add(func() { _ = linstor.Client.ResourceDefinitions.Delete(context.TODO(), resourceDefinitionName) }) err = linstor.Client.Resources.RestoreVolumeDefinitionSnapshot(context.TODO(), parentResourceDefinition.Name, linstorSnapshotName, linstorClient.SnapshotRestore{ ToResource: resourceDefinitionName, }) if err != nil { return fmt.Errorf("Could not restore volume definition from snapshot: %w", err) } err = linstor.Client.Resources.RestoreSnapshot(context.TODO(), parentResourceDefinition.Name, linstorSnapshotName, linstorClient.SnapshotRestore{ ToResource: resourceDefinitionName, }) if err != nil { return fmt.Errorf("Could not restore resource from snapshot: %w", err) } // Set the aux properties on the new resource definition. err = d.setResourceDefinitionProperties(vol, resourceDefinitionName) if err != nil { return err } rev.Success() return nil } // deleteResourceDefinitionFromSnapshot deletes the resource definition created from a snapshot. func (d *linstor) deleteResourceDefinitionFromSnapshot(vol Volume) error { l := d.logger.AddContext(logger.Ctx{"vol": vol.Name()}) l.Debug("Deleting resource definition for snapshot") linstor, err := d.state.Linstor() if err != nil { return err } resourceDefinition, err := d.getResourceDefinition(vol, false) if err != nil { if errors.Is(err, errResourceDefinitionNotFound) { return nil } return err } err = linstor.Client.ResourceDefinitions.Delete(context.TODO(), resourceDefinition.Name) if err != nil { return err } d.logger.Debug("Resource definition for snapshot deleted") return nil } // resizeVolume resizes a volume definition. This function does not resize any filesystem inside the volume. func (d *linstor) resizeVolume(vol Volume, sizeBytes int64) error { linstor, err := d.state.Linstor() if err != nil { return err } resourceDefinition, err := d.getResourceDefinition(vol, false) if err != nil { return err } volumeIndex := 0 // For VM volumes, the associated filesystem volume is a second volume on the same LINSTOR resource. if vol.volType == VolumeTypeVM && vol.contentType == ContentTypeFS { volumeIndex = 1 } // Resize the volume definition. err = linstor.Client.ResourceDefinitions.ModifyVolumeDefinition(context.TODO(), resourceDefinition.Name, volumeIndex, linstorClient.VolumeDefinitionModify{ SizeKib: uint64(sizeBytes) / 1024, }) if err != nil { return fmt.Errorf("Unable to resize volume definition: %w", err) } return nil } func (d *linstor) copyVolume(vol Volume, srcVol Volume) error { l := logger.AddContext(logger.Ctx{"vol": vol.name, "srcVol": srcVol.name}) l.Debug("Copying volume") linstor, err := d.state.Linstor() if err != nil { return err } rev := revert.New() defer rev.Fail() targetResourceDefinitionName := d.generateUUIDWithPrefix() srcResourceDefinition, err := d.getResourceDefinition(srcVol, false) if err != nil { return err } _, err = linstor.Client.ResourceDefinitions.Clone(context.TODO(), srcResourceDefinition.Name, linstorClient.ResourceDefinitionCloneRequest{ Name: targetResourceDefinitionName, ResourceGroup: d.config[LinstorResourceGroupNameConfigKey], }) if err != nil { return fmt.Errorf("Unable to start cloning resource definition: %w", err) } d.logger.Debug("Clone operation started. Will poll for status", logger.Ctx{"srcResourceDefinition": srcResourceDefinition.Name, "targetResourceDefinition": targetResourceDefinitionName}) // Poll the cloning operation status from LINSTOR. The duration of the operation depends on the // underlying storage backend being used. For LVM-thin and ZFS the cloning is optimized and should // be considerably faster than for LVM, which uses `dd`. loop: for { cloneStatus, err := linstor.Client.ResourceDefinitions.CloneStatus(context.TODO(), srcResourceDefinition.Name, targetResourceDefinitionName) if err != nil { return fmt.Errorf("Unable to get clone operation status: %w", err) } d.logger.Debug("Got resource definition clone status", logger.Ctx{"cloneStatus": cloneStatus.Status}) switch cloneStatus.Status { case clonestatus.Complete: break loop case clonestatus.Cloning: time.Sleep(1 * time.Second) case clonestatus.Failed: return errors.New("Clone operation failed") } } rev.Add(func() { _ = linstor.Client.ResourceDefinitions.Delete(context.TODO(), targetResourceDefinitionName) }) // Set the aux properties on the new resource definition. err = d.setResourceDefinitionProperties(vol, targetResourceDefinitionName) if err != nil { return err } rev.Success() return nil } func (d *linstor) getVolumeSize(vol Volume) (int64, error) { resourceDefinition, err := d.getResourceDefinition(vol, true) if err != nil { return 0, err } volumeIndex := 0 if len(resourceDefinition.VolumeDefinitions) == 2 { // For VM volumes, the associated filesystem volume is a second volume on the same LINSTOR resource. if (vol.volType == VolumeTypeVM || vol.volType == VolumeTypeImage) && vol.contentType == ContentTypeFS { volumeIndex = 1 } } return int64(resourceDefinition.VolumeDefinitions[volumeIndex].SizeKib * 1024), nil } // getResourceDefinitions returns all available resource definitions. func (d *linstor) getResourceDefinitions() ([]linstorClient.ResourceDefinitionWithVolumeDefinition, error) { linstor, err := d.state.Linstor() if err != nil { return nil, err } // Get the resource definitions. resourceDefinitions, err := linstor.Client.ResourceDefinitions.GetAll(context.TODO(), linstorClient.RDGetAllRequest{ WithVolumeDefinitions: false, }) if err != nil { return nil, fmt.Errorf("Unable to get resource definitions: %w", err) } return resourceDefinitions, nil } // setResourceDefinitionProperties sets properties on the resource definition based on the volume config. func (d *linstor) setResourceDefinitionProperties(vol Volume, resourceDefinitionName string) error { l := logger.AddContext(logger.Ctx{"volume": vol.Name(), "resourceDefinition": resourceDefinitionName}) l.Debug("Setting resource definition properties", logger.Ctx{"config": vol.config}) linstor, err := d.state.Linstor() if err != nil { return err } // Set the base properties. overrideProps := map[string]string{ LinstorAuxName: d.config[LinstorVolumePrefixConfigKey] + vol.name, LinstorAuxType: string(vol.volType), LinstorAuxContentType: string(vol.contentType), "DrbdOptions/Net/allow-two-primaries": "yes", // Required for mounting volumes simultaneously on two nodes when live migrating } // Parse and set properties derived from config. drbdProps, err := d.drbdPropsFromConfig(vol.config) if err != nil { return fmt.Errorf("Could parse config into DRBD options: %w", err) } maps.Copy(overrideProps, drbdProps) err = linstor.Client.ResourceDefinitions.Modify(context.TODO(), resourceDefinitionName, linstorClient.GenericPropsModify{ OverrideProps: overrideProps, }) if err != nil { return fmt.Errorf("Could not set properties on resource definition: %w", err) } return nil } // updateResourceDefinitionupdates the resource definition with the given changed configs. func (d *linstor) updateResourceDefinition(vol Volume, changedConfig map[string]string) error { l := logger.AddContext(logger.Ctx{"volume": vol.Name()}) l.Debug("Updating resource definition", logger.Ctx{"changedchangedConfig": changedConfig}) linstor, err := d.state.Linstor() if err != nil { return err } resourceDefinition, err := d.getResourceDefinition(vol, false) if err != nil { return err } // Parse and set properties to be overwritten. overrideProps, err := d.drbdPropsFromConfig(changedConfig) if err != nil { return fmt.Errorf("Could parse config into DRBD props: %w", err) } // Parse and set properties to be deleted. deleteProps := []string{} for key, value := range changedConfig { if value != "" { continue } prop, ok := drbdPropsMap[key] if ok { deleteProps = append(deleteProps, prop) } } err = linstor.Client.ResourceDefinitions.Modify(context.TODO(), resourceDefinition.Name, linstorClient.GenericPropsModify{ OverrideProps: overrideProps, DeleteProps: deleteProps, }) if err != nil { return fmt.Errorf("Could not set properties on resource definition: %w", err) } return nil } // drbdPropsFromConfig creates a map of DRBD properties from the given volume or storage pool config. func (d *linstor) drbdPropsFromConfig(config map[string]string) (map[string]string, error) { props := map[string]string{} for key, prop := range drbdPropsMap { value, changed := config[key] if !changed { continue } switch key { case DrbdAutoDiskfulConfigKey: duration, err := time.ParseDuration(value) if err != nil { return nil, err } props[prop] = strconv.Itoa(int(duration.Seconds())) case DrbdAutoAddQuorumTiebreakerConfigKey: if util.IsFalse(value) { props[prop] = "false" } else { props[prop] = "true" } default: props[prop] = value } } return props, nil } // getSnapshots retrieves all snapshots for a given resource definition name. func (d *linstor) getSnapshots(resourceDefinitionName string) ([]linstorClient.Snapshot, error) { linstor, err := d.state.Linstor() if err != nil { return []linstorClient.Snapshot{}, err } snapshots, err := linstor.Client.Resources.GetSnapshots(context.TODO(), resourceDefinitionName) if err != nil { return snapshots, fmt.Errorf("Unable to get snapshots: %w", err) } return snapshots, nil } // rsyncMigrationType returns the migration types to use for a given content type. func (d *linstor) rsyncMigrationType(contentType ContentType) localMigration.Type { var rsyncTransportType migration.MigrationFSType var rsyncFeatures []string // Do not pass compression argument to rsync if the associated // config key, that is rsync.compression, is set to false. if util.IsFalse(d.Config()["rsync.compression"]) { rsyncFeatures = []string{"xattrs", "delete", "bidirectional"} } else { rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"} } if IsContentBlock(contentType) { rsyncTransportType = migration.MigrationFSType_BLOCK_AND_RSYNC } else { rsyncTransportType = migration.MigrationFSType_RSYNC } return localMigration.Type{ FSType: rsyncTransportType, Features: rsyncFeatures, } } // parseVolumeType parses a string into a volume type. func (d *linstor) parseVolumeType(s string) (*VolumeType, bool) { for _, volType := range d.Info().VolumeTypes { if s == string(volType) { return &volType, true } } return nil, false } // parseContentType parses a string into a volume type. func (d *linstor) parseContentType(s string) (*ContentType, bool) { for _, contentType := range []ContentType{ContentTypeFS, ContentTypeBlock, ContentTypeISO} { if s == string(contentType) { return &contentType, true } } return nil, false } // generateUUIDWithPrefix generates a new UUID (without "-") and appends it to the configured volume prefix. func (d *linstor) generateUUIDWithPrefix() string { return d.config[LinstorVolumePrefixConfigKey] + strings.ReplaceAll(uuid.NewString(), "-", "") } incus-7.0.0/internal/server/storage/drivers/driver_linstor_volumes.go000066400000000000000000001241141517523235500262640ustar00rootroot00000000000000package drivers import ( "context" "errors" "fmt" "io" "io/fs" "os" "slices" "strings" "time" linstorClient "github.com/LINBIT/golinstor/client" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/server/backup" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // FillVolumeConfig populate volume with default config. func (d *linstor) FillVolumeConfig(vol Volume) error { // Copy volume.* configuration options from pool. // Exclude 'block.filesystem' and 'block.mount_options' // as this ones are handled below in this function and depends from volume type. err := d.fillVolumeConfig(&vol, "block.filesystem", "block.mount_options") if err != nil { return err } // Only validate filesystem config keys for filesystem volumes or VM block volumes (which have an // associated filesystem volume). if vol.ContentType() == ContentTypeFS || vol.IsVMBlock() { // Inherit filesystem from pool if not set. if vol.config["block.filesystem"] == "" { vol.config["block.filesystem"] = d.config["volume.block.filesystem"] } // Default filesystem if neither volume nor pool specify an override. if vol.config["block.filesystem"] == "" { // Unchangeable volume property: Set unconditionally. vol.config["block.filesystem"] = DefaultFilesystem } // Inherit filesystem mount options from pool if not set. if vol.config["block.mount_options"] == "" { vol.config["block.mount_options"] = d.config["volume.block.mount_options"] } // Default filesystem mount options if neither volume nor pool specify an override. if vol.config["block.mount_options"] == "" { // Unchangeable volume property: Set unconditionally. vol.config["block.mount_options"] = "discard" } } return nil } // commonVolumeRules returns validation rules which are common for pool and volume. func (d *linstor) commonVolumeRules() map[string]func(value string) error { return map[string]func(value string) error{ // gendoc:generate(entity=storage_volume_linstor, group=common, key=block.filesystem) // // --- // type: string // condition: block-based volume with content type `filesystem` // default: same as `volume.block.filesystem` // shortdesc: {{block_filesystem}} "block.filesystem": validate.Optional(validate.IsOneOf(blockBackedAllowedFilesystems...)), // gendoc:generate(entity=storage_volume_linstor, group=common, key=block.mount_options) // // --- // type: string // condition: block-based volume with content type `filesystem` // default: same as `volume.block.mount_options` // shortdesc: Mount options for block-backed file system volumes "block.mount_options": validate.IsAny, // gendoc:generate(entity=storage_volume_linstor, group=common, key=drbd.on_no_quorum) // // --- // type: string // condition: - // default: - // shortdesc: The DRBD policy to use on resources when quorum is lost (applied to the resource definition) DrbdOnNoQuorumConfigKey: validate.Optional(validate.IsOneOf("io-error", "suspend-io")), // gendoc:generate(entity=storage_volume_linstor, group=common, key=drbd.auto_diskful) // // --- // type: string // condition: - // default: - // shortdesc: A duration string describing the time after which a primary diskless resource can be converted to diskful if storage is available on the node (applied to the resource definition) DrbdAutoDiskfulConfigKey: validate.Optional(validate.IsMinimumDuration(time.Minute)), // gendoc:generate(entity=storage_volume_linstor, group=common, key=drbd.auto_add_quorum_tiebreaker) // // --- // type: bool // condition: - // default: `true` // shortdesc: Whether to allow LINSTOR to automatically create diskless resources to act as quorum tiebreakers if needed (applied to the resource definition) DrbdAutoAddQuorumTiebreakerConfigKey: validate.Optional(validate.IsBool), // gendoc:generate(entity=storage_volume_linstor, group=common, key=linstor.remove_snapshots) // // --- // type: bool // condition: - // default: same as `volume.linstor.remove_snapshots` or `false` // shortdesc: Remove snapshots as needed LinstorRemoveSnapshotsConfigKey: validate.Optional(validate.IsBool), } } // ValidateVolume validates the supplied volume config. func (d *linstor) ValidateVolume(vol Volume, removeUnknownKeys bool) error { // gendoc:generate(entity=storage_volume_linstor, group=common, key=initial.gid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.gid` or `0` // shortdesc: GID of the volume owner in the instance // gendoc:generate(entity=storage_volume_linstor, group=common, key=initial.mode) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.mode` or `711` // shortdesc: Mode of the volume in the instance // gendoc:generate(entity=storage_volume_linstor, group=common, key=initial.uid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.uid` or `0` // shortdesc: UID of the volume owner in the instance // gendoc:generate(entity=storage_volume_linstor, group=common, key=security.shared) // // --- // type: bool // condition: custom block volume // default: same as `volume.security.shared` or `false` // shortdesc: Enable sharing the volume across multiple instances // gendoc:generate(entity=storage_volume_linstor, group=common, key=security.shifted) // // --- // type: bool // condition: custom volume // default: same as `volume.security.shifted` or `false` // shortdesc: {{enable_ID_shifting}} // gendoc:generate(entity=storage_volume_linstor, group=common, key=security.unmapped) // // --- // type: bool // condition: custom volume // default: same as `volume.security.unmapped` or `false` // shortdesc: Disable ID mapping for the volume // gendoc:generate(entity=storage_volume_linstor, group=common, key=size) // // --- // type: string // condition: - // default: same as `volume.size` // shortdesc: Size/quota of the storage volume // gendoc:generate(entity=storage_volume_linstor, group=common, key=snapshots.expiry) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_linstor, group=common, key=snapshots.expiry.manual) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry.manual` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_linstor, group=common, key=snapshots.pattern) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.pattern` or `snap%d` // shortdesc: {{snapshot_pattern_format}} [^*] // gendoc:generate(entity=storage_volume_linstor, group=common, key=snapshots.schedule) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.schedule` // shortdesc: {{snapshot_schedule_format}} commonRules := d.commonVolumeRules() // Disallow block.* settings for regular custom block volumes. These settings only make sense // when using custom filesystem volumes. Incus will create the filesystem // for these volumes, and use the mount options. When attaching a regular block volume to a VM, // these are not mounted by Incus and therefore don't need these config keys. if vol.IsVMBlock() || vol.volType == VolumeTypeCustom && vol.contentType == ContentTypeBlock { delete(commonRules, "block.filesystem") delete(commonRules, "block.mount_options") } return d.validateVolume(vol, commonRules, removeUnknownKeys) } // CreateVolume creates an empty volume and can optionally fill it by executing the supplied // filler function. func (d *linstor) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Operation) error { l := d.logger.AddContext(logger.Ctx{"volume": vol.Name()}) l.Debug("Creating a new Linstor volume") rev := revert.New() defer rev.Fail() linstor, err := d.state.Linstor() if err != nil { return err } if vol.contentType == ContentTypeFS { // Create mountpoint. err := vol.EnsureMountPath(true) if err != nil { return err } rev.Add(func() { _ = os.Remove(vol.MountPath()) }) } // Transform byte to KiB. requiredBytes, err := units.ParseByteSizeString(vol.ConfigSize()) if err != nil { return fmt.Errorf("Unable to parse volume size: %w", err) } requiredKiB := requiredBytes / 1024 resourceDefinitionName := d.generateUUIDWithPrefix() volumeSizes := []int64{requiredKiB} // For VMs, create and extra volume on the resource for the filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() // Transform byte to KiB. requiredBytes, err := units.ParseByteSizeString(fsVol.ConfigSize()) if err != nil { return fmt.Errorf("Unable to parse volume size: %w", err) } requiredKiB := requiredBytes / 1024 volumeSizes = append(volumeSizes, requiredKiB) } // Spawn resource. err = linstor.Client.ResourceGroups.Spawn(context.TODO(), d.config[LinstorResourceGroupNameConfigKey], linstorClient.ResourceGroupSpawn{ ResourceDefinitionName: resourceDefinitionName, VolumeSizes: volumeSizes, }) if err != nil { return fmt.Errorf("Unable to spawn from resource group: %w", err) } l.Debug("Spawned a new Linstor resource definition for volume", logger.Ctx{"resourceDefinitionName": resourceDefinitionName}) rev.Add(func() { _ = d.DeleteVolume(vol, op) }) err = d.setResourceDefinitionProperties(vol, resourceDefinitionName) if err != nil { return err } // Setup the filesystem. if vol.contentType == ContentTypeFS { devPath, err := d.getLinstorDevPath(vol) if err != nil { return fmt.Errorf("Could not get device path for filesystem creation: %w", err) } volFilesystem := vol.ConfigBlockFilesystem() _, err = makeFSType(devPath, volFilesystem, nil) if err != nil { return err } } // For VMs, also create the filesystem on the associated filesystem volume. if vol.IsVMBlock() { l.Debug("Creating filesystem on the associated filesystem volume") fsVol := vol.NewVMBlockFilesystemVolume() fsVolDevPath, err := d.getLinstorDevPath(fsVol) if err != nil { return fmt.Errorf("Could not get device path for filesystem creation: %w", err) } fsVolFilesystem := fsVol.ConfigBlockFilesystem() _, err = makeFSType(fsVolDevPath, fsVolFilesystem, nil) l.Debug("Created filesystem on the associated filesystem volume", logger.Ctx{"fsVolDevPath": fsVolDevPath, "fsVolFilesystem": fsVolFilesystem}) if err != nil { return err } } err = vol.MountTask(func(mountPath string, op *operations.Operation) error { // Run the volume filler function if supplied. if filler != nil && filler.Fill != nil { var err error var devPath string if IsContentBlock(vol.contentType) { // Get the device path. devPath, err = d.GetVolumeDiskPath(vol) if err != nil { return err } } allowUnsafeResize := false if vol.volType == VolumeTypeImage { // Allow filler to resize initial image volume as needed. // Some storage drivers don't normally allow image volumes to be resized due to // them having read-only snapshots that cannot be resized. However when creating // the initial image volume and filling it before the snapshot is taken resizing // can be allowed and is required in order to support unpacking images larger than // the default volume size. The filler function is still expected to obey any // volume size restrictions configured on the pool. // Unsafe resize is also needed to disable filesystem resize safety checks. // This is safe because if for some reason an error occurs the volume will be // discarded rather than leaving a corrupt filesystem. allowUnsafeResize = true } // Run the filler. err = genericRunFiller(d, vol, devPath, filler, allowUnsafeResize) if err != nil { return err } // Move the GPT alt header to end of disk if needed. if vol.IsVMBlock() { err = d.moveGPTAltHeader(devPath) if err != nil { return err } } } if vol.contentType == ContentTypeFS { // Run EnsureMountPath again after mounting and filling to ensure the mount directory has // the correct permissions set. err = vol.EnsureMountPath(true) if err != nil { return err } } return nil }, op) if err != nil { return err } rev.Success() return nil } // CreateVolumeFromCopy provides same-pool volume copying functionality. func (d *linstor) CreateVolumeFromCopy(vol Volume, srcVol Volume, copySnapshots bool, allowInconsistent bool, op *operations.Operation) error { l := d.logger.AddContext(logger.Ctx{"vol": vol.Name(), "srcVol": srcVol.Name()}) l.Debug("Creating Linstor volume from copy") rev := revert.New() defer rev.Fail() // For snapshots, restore them into the target volume. if srcVol.IsSnapshot() { l.Debug("Copying snapshot to volume") err := d.createResourceDefinitionFromSnapshot(srcVol, vol) if err != nil { return err } return nil } if copySnapshots { // Get the list of snapshots from the source. srcSnapshots, err := srcVol.Snapshots(op) if err != nil { return err } if len(srcSnapshots) > 0 { return errors.New("Linstor doesn't currently support copying volumes with their snapshots") } } // For VM volumes, the associated filesystem volume is already cloned with the main block // volume, since they share the same resource definition. if vol.volType != VolumeTypeVM || vol.contentType != ContentTypeFS { err := d.copyVolume(vol, srcVol) if err != nil { return err } rev.Add(func() { _ = d.DeleteVolume(vol, op) }) } if vol.contentType == ContentTypeFS { devPath, err := d.getLinstorDevPath(vol) if err != nil { return err } fsType := vol.ConfigBlockFilesystem() // Generate a new filesystem UUID if needed (this is required because some filesystems won't allow // volumes with the same UUID to be mounted at the same time). This should be done before volume // resize as some filesystems will need to mount the filesystem to resize. if renegerateFilesystemUUIDNeeded(fsType) { d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": devPath, "fs": fsType}) err = regenerateFilesystemUUID(fsType, devPath) if err != nil { return err } } // Create mountpoint. err = vol.EnsureMountPath(false) if err != nil { return err } rev.Add(func() { _ = os.Remove(vol.MountPath()) }) } // Resize volume to the size specified. Only uses volume "size" property and does not use // pool/defaults to give the caller more control over the size being used. err := d.SetVolumeQuota(vol, vol.config["size"], false, op) if err != nil { return err } // For VMs, also copy the filesystem volume. if vol.IsVMBlock() { srcFSVol := srcVol.NewVMBlockFilesystemVolume() fsVol := vol.NewVMBlockFilesystemVolume() err = d.CreateVolumeFromCopy(fsVol, srcFSVol, copySnapshots, false, op) if err != nil { return err } } rev.Success() return nil } // RefreshVolume updates an existing volume to match the state of another. func (d *linstor) RefreshVolume(vol Volume, srcVol Volume, srcSnapshots []Volume, allowInconsistent bool, op *operations.Operation) error { return genericVFSCopyVolume(d, nil, vol, srcVol, srcSnapshots, true, allowInconsistent, op) } // DeleteVolume deletes a volume of the storage device. func (d *linstor) DeleteVolume(vol Volume, op *operations.Operation) error { l := d.logger.AddContext(logger.Ctx{"volume": vol.Name()}) l.Debug("Deleting Linstor volume") linstor, err := d.state.Linstor() if err != nil { return err } // Test if the volume exists. volumeExists, err := d.HasVolume(vol) if err != nil { return fmt.Errorf("Unable to check if volume exists: %w", err) } if !volumeExists { l.Warn("Resource definition does not exist") } else { resourceDefinition, err := d.getResourceDefinition(vol, false) if err != nil { return err } err = linstor.Client.ResourceDefinitions.Delete(context.TODO(), resourceDefinition.Name) if err != nil { return fmt.Errorf("Unable to delete the resource definition: %w", err) } } // For VMs, also delete the associated filesystem volume mount path. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() fsVolMountPath := fsVol.MountPath() err := wipeDirectory(fsVolMountPath) if err != nil { return err } err = os.Remove(fsVolMountPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", fsVolMountPath, err) } } mountPath := vol.MountPath() if vol.contentType == ContentTypeFS && util.PathExists(mountPath) { err := wipeDirectory(mountPath) if err != nil { return err } err = os.Remove(mountPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", mountPath, err) } } return nil } // HasVolume indicates whether a specific volume exists on the storage pool. func (d *linstor) HasVolume(vol Volume) (bool, error) { if vol.IsSnapshot() { parentName, _, _ := api.GetParentAndSnapshotName(vol.name) parentVol := NewVolume(d, d.name, vol.volType, vol.contentType, parentName, nil, nil) return d.snapshotExists(parentVol, vol) } _, err := d.getResourceDefinition(vol, false) if err != nil { if errors.Is(err, errResourceDefinitionNotFound) { return false, nil } return false, err } return true, nil } // GetVolumeDiskPath returns the location of a root disk block device. func (d *linstor) GetVolumeDiskPath(vol Volume) (string, error) { if vol.IsVMBlock() || (vol.volType == VolumeTypeCustom && IsContentBlock(vol.contentType)) { devPath, err := d.getLinstorDevPath(vol) return devPath, err } return "", ErrNotSupported } // ListVolumes returns a list of volumes in storage pool. func (d *linstor) ListVolumes() ([]Volume, error) { d.logger.Debug("Listing volumes") resourceDefinitions, err := d.getResourceDefinitions() if err != nil { return nil, err } var volumes []Volume for _, rd := range resourceDefinitions { l := d.logger.AddContext(logger.Ctx{"resourceDefinition": rd.Name}) if rd.ResourceGroupName != d.config[LinstorResourceGroupNameConfigKey] { l.Debug("Ignoring resource definition not linked to the storage pool's resource group") continue } volName, ok := rd.Props[LinstorAuxName] if !ok { l.Debug("Ignoring resource definition with no name aux property set") continue } // Strip the volume prefix. volName = strings.TrimPrefix(volName, d.config[LinstorVolumePrefixConfigKey]) rawVolType, ok := rd.Props[LinstorAuxType] if !ok { l.Debug("Ignoring resource definition with no type aux property set") continue } volType, ok := d.parseVolumeType(rawVolType) if !ok && volType != nil { l.Debug("Ignoring resource definition with invalid type") continue } rawContentType, ok := rd.Props[LinstorAuxContentType] if !ok { l.Debug("Ignoring resource definition with no name contentType property set") continue } contentType, ok := d.parseContentType(rawContentType) if !ok && contentType != nil { l.Debug("Ignoring resource definition with invalid contentType") continue } vol := NewVolume(d, d.name, *volType, *contentType, volName, make(map[string]string), d.config) volumes = append(volumes, vol) l.Debug("Found volume from storage pool", logger.Ctx{"volName": volName, "volType": *volType, "contentType": *contentType}) } return volumes, nil } // ActivateTask allows running a function while the volume is active (but not mounted). func (d *linstor) ActivateTask(vol Volume, task func(devPath string, op *operations.Operation) error, op *operations.Operation) error { // Prevent concurrent mounting actions. unlock, err := vol.MountLock() if err != nil { return err } defer unlock() volDevPath, err := d.getLinstorDevPath(vol) if err != nil { return err } // Run the task. return task(volDevPath, op) } // MountVolume mounts a volume and increments ref counter. Please call UnmountVolume() when done with the volume. func (d *linstor) MountVolume(vol Volume, op *operations.Operation) error { l := d.logger.AddContext(logger.Ctx{"volume": vol.Name()}) l.Debug("Mounting volume") unlock, err := vol.MountLock() if err != nil { return err } defer unlock() rev := revert.New() defer rev.Fail() volDevPath, err := d.getLinstorDevPath(vol) if err != nil { return fmt.Errorf("Could not mount volume: %w", err) } l.Debug("Volume is available on node", logger.Ctx{"volDevPath": volDevPath}) switch vol.contentType { case ContentTypeFS: mountPath := vol.MountPath() l.Debug("Content type FS", logger.Ctx{"mountPath": mountPath}) if !linux.IsMountPoint(mountPath) { err := vol.EnsureMountPath(false) if err != nil { return err } fsType := vol.ConfigBlockFilesystem() if vol.mountFilesystemProbe { fsType, err = fsProbe(volDevPath) if err != nil { return fmt.Errorf("Failed probing filesystem: %w", err) } } mountFlags, mountOptions := linux.ResolveMountOptions(strings.Split(vol.ConfigBlockMountOptions(), ",")) l.Debug("Will try mount", logger.Ctx{"mountFlags": mountFlags, "mountOptions": mountOptions}) err = TryMount(volDevPath, mountPath, fsType, mountFlags, mountOptions) if err != nil { l.Debug("Tried mounting but failed", logger.Ctx{"error": err}) return err } d.logger.Debug("Mounted Linstor volume", logger.Ctx{"volName": vol.name, "dev": volDevPath, "path": mountPath, "options": mountOptions}) } case ContentTypeBlock: l.Debug("Content type Block") // For VMs, mount the filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() l.Debug("Created a new FS volume", logger.Ctx{"fsVol": fsVol}) err = d.MountVolume(fsVol, op) if err != nil { l.Debug("Tried mounting but failed", logger.Ctx{"error": err}) return err } } } vol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolume() when done. rev.Success() l.Debug("Volume mounted") return nil } // UnmountVolume clears any runtime state for the volume. // keepBlockDev indicates if backing block device should be not be deleted if volume is unmounted. func (d *linstor) UnmountVolume(vol Volume, keepBlockDev bool, op *operations.Operation) (bool, error) { unlock, err := vol.MountLock() if err != nil { return false, err } defer unlock() ourUnmount := false mountPath := vol.MountPath() refCount := vol.MountRefCountDecrement() // Attempt to unmount the volume. if vol.contentType == ContentTypeFS && linux.IsMountPoint(mountPath) { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": vol.name, "refCount": refCount}) return false, ErrInUse } err = TryUnmount(mountPath, unix.MNT_DETACH) if err != nil { return false, err } d.logger.Debug("Unmounted Linstor volume", logger.Ctx{"volName": vol.name, "path": mountPath, "keepBlockDev": keepBlockDev}) ourUnmount = true } else if IsContentBlock(vol.contentType) { // For VMs, unmount the filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() return d.UnmountVolume(fsVol, false, op) } } if !keepBlockDev { err = d.deleteDisklessResource(vol) if err != nil { return false, fmt.Errorf("Could not delete diskless resource: %w", err) } } return ourUnmount, nil } // RenameVolume renames a volume. func (d *linstor) RenameVolume(vol Volume, newVolName string, op *operations.Operation) error { l := d.logger.AddContext(logger.Ctx{"volume": vol.Name()}) l.Debug("Renaming Linstor volume") rev := revert.New() defer rev.Fail() linstor, err := d.state.Linstor() if err != nil { return err } resourceDefinition, err := d.getResourceDefinition(vol, false) if err != nil { return err } err = linstor.Client.ResourceDefinitions.Modify(context.TODO(), resourceDefinition.Name, linstorClient.GenericPropsModify{ OverrideProps: map[string]string{ LinstorAuxName: d.config[LinstorVolumePrefixConfigKey] + newVolName, }, }) if err != nil { return fmt.Errorf("Could not set properties on resource definition: %w", err) } return nil } // CreateVolumeSnapshot creates a snapshot of a volume. func (d *linstor) CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error { rev := revert.New() defer rev.Fail() parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) sourcePath := GetVolumeMountPath(d.name, snapVol.volType, parentName) if linux.IsMountPoint(sourcePath) { // Attempt to sync and freeze filesystem, but do not error if not able to freeze (as filesystem // could still be busy), as LINSTOR does not have any notion of the filesystem and therefore can't // guarantee the consistently of the filesystem on a snapshot. This is costly but tries to ensure // that all cached data has been committed to the underlying DRBD device. If we don't then the // LINSTOR snapshot can be inconsistent or, in the worst case, empty. unfreezeFS, err := d.filesystemFreeze(sourcePath) if err == nil { defer func() { _ = unfreezeFS() }() } } // Create the parent directory. err := CreateParentSnapshotDirIfMissing(d.name, snapVol.volType, parentName) if err != nil { return err } err = snapVol.EnsureMountPath(false) if err != nil { return err } parentVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, nil, nil) err = d.createVolumeSnapshot(parentVol, snapVol) if err != nil { return err } rev.Add(func() { _ = d.DeleteVolumeSnapshot(snapVol, op) }) rev.Success() return nil } // DeleteVolumeSnapshot removes a snapshot from the storage device. func (d *linstor) DeleteVolumeSnapshot(snapVol Volume, op *operations.Operation) error { parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) parentVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, nil, nil) snapshotExists, err := d.snapshotExists(parentVol, snapVol) if err != nil { return fmt.Errorf("Failed to delete volume snapshot: %w", err) } // Check if snapshot exists, and return if not. if !snapshotExists { return nil } err = d.deleteVolumeSnapshot(parentVol, snapVol) if err != nil { return fmt.Errorf("Failed to delete volume snapshot: %w", err) } mountPath := snapVol.MountPath() if snapVol.contentType == ContentTypeFS && util.PathExists(mountPath) { err = wipeDirectory(mountPath) if err != nil { return err } err = os.Remove(mountPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", mountPath, err) } } // Remove the parent snapshot directory if this is the last snapshot being removed. err = deleteParentSnapshotDirIfEmpty(d.name, snapVol.volType, parentName) if err != nil { return err } return nil } // CanRestoreVolume checks whether a volume snapshot can be restored. func (d *linstor) CanRestoreVolume(vol Volume, snapshotName string) error { resourceDefinition, err := d.getResourceDefinition(vol, false) if err != nil { return err } // Since LINSTOR only supports rolling back a resource definition to its latest // snapshot, we need to check if the given snapshot is the latest one. existingSnapshots, err := d.getSnapshots(resourceDefinition.Name) if err != nil { return err } // Sort all snapshots by creation date in descending order. slices.SortFunc(existingSnapshots, func(a linstorClient.Snapshot, b linstorClient.Snapshot) int { return a.Snapshots[0].CreateTimestamp.Compare(b.Snapshots[0].CreateTimestamp.Time) * -1 }) linstorSnapshotName, ok := resourceDefinition.Props[LinstorAuxSnapshotPrefix+snapshotName] if !ok { return fmt.Errorf("Could not find snapshot name mapping for volume %s", vol.Name()) } snapshotMap, err := d.getSnapshotMap(vol) if err != nil { return err } // Get all snapshots taken after the one we're trying to restore. snapshots := []string{} for _, s := range existingSnapshots { if s.Name == linstorSnapshotName { break } snapshots = append(snapshots, snapshotMap[s.Name]) } // Check if snapshot removal is allowed. if len(snapshots) > 0 { if util.IsFalseOrEmpty(vol.ExpandedConfig(LinstorRemoveSnapshotsConfigKey)) { return fmt.Errorf("Snapshot %q cannot be restored due to subsequent snapshot(s). Set %s to override", snapshotName, LinstorRemoveSnapshotsConfigKey) } // Setup custom error to tell the backend what to delete. err := ErrDeleteSnapshots{} err.Snapshots = snapshots return err } return nil } // RestoreVolume restores a volume from a snapshot. func (d *linstor) RestoreVolume(vol Volume, snapshotName string, op *operations.Operation) error { ourUnmount, err := d.UnmountVolume(vol, false, op) if err != nil { return err } if ourUnmount { defer func() { _ = d.MountVolume(vol, op) }() } snapVol, err := vol.NewSnapshot(snapshotName) if err != nil { return err } err = d.CanRestoreVolume(vol, snapshotName) if err != nil { return err } // Restore the snapshot. err = d.restoreVolume(vol, snapVol) if err != nil { return err } return nil } // RenameVolumeSnapshot is a no-op. func (d *linstor) RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *operations.Operation) error { parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) parentVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, nil, nil) return d.renameVolumeSnapshot(parentVol, snapVol, newSnapshotName) } // MountVolumeSnapshot mounts a storage volume snapshot. // // The snapshot is restored into a new temporary LINSTOR resource definition that will live for the duration of the mount. func (d *linstor) MountVolumeSnapshot(snapVol Volume, op *operations.Operation) error { l := d.logger.AddContext(logger.Ctx{"volume": snapVol.Name()}) l.Debug("Mounting snapshot volume") unlock, err := snapVol.MountLock() if err != nil { return err } defer unlock() rev := revert.New() defer rev.Fail() // For VMs, mount the filesystem volume. if snapVol.IsVMBlock() { fsVol := snapVol.NewVMBlockFilesystemVolume() l.Debug("Created a new FS volume", logger.Ctx{"fsVol": fsVol}) return d.MountVolumeSnapshot(fsVol, op) } // Create a new temporary resource definition from the snapshot err = d.createResourceDefinitionFromSnapshot(snapVol, snapVol) if err != nil { return err } rev.Add(func() { _ = d.deleteResourceDefinitionFromSnapshot(snapVol) }) volDevPath, err := d.getLinstorDevPath(snapVol) if err != nil { return fmt.Errorf("Could not mount volume: %w", err) } l.Debug("Volume is available on node", logger.Ctx{"volDevPath": volDevPath}) if snapVol.contentType == ContentTypeFS { mountPath := snapVol.MountPath() l.Debug("Content type FS", logger.Ctx{"mountPath": mountPath}) if !linux.IsMountPoint(mountPath) { err := snapVol.EnsureMountPath(false) if err != nil { return err } snapVolFS := snapVol.ConfigBlockFilesystem() if snapVol.mountFilesystemProbe { snapVolFS, err = fsProbe(volDevPath) if err != nil { return fmt.Errorf("Failed probing filesystem: %w", err) } } mountFlags, mountOptions := linux.ResolveMountOptions(strings.Split(snapVol.ConfigBlockMountOptions(), ",")) d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"volDevPath": volDevPath, "fs": snapVolFS}) if renegerateFilesystemUUIDNeeded(snapVolFS) { if snapVolFS == "xfs" { idx := strings.Index(mountOptions, "nouuid") if idx < 0 { mountOptions += ",nouuid" } } else { err = regenerateFilesystemUUID(snapVolFS, volDevPath) if err != nil { return err } } } l.Debug("Will try mount") err = TryMount(volDevPath, mountPath, snapVolFS, mountFlags, mountOptions) if err != nil { l.Debug("Tried mounting but failed", logger.Ctx{"error": err}) return err } d.logger.Debug("Mounted snapshot volume", logger.Ctx{"volName": snapVol.name, "dev": volDevPath, "path": mountPath, "options": mountOptions}) } } snapVol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolumeSnapshot() when done. rev.Success() return nil } // UnmountVolumeSnapshot unmounts a volume snapshot. // // The temporary LINSTOR resource definition created for the mount is deleted. func (d *linstor) UnmountVolumeSnapshot(snapVol Volume, op *operations.Operation) (bool, error) { l := d.logger.AddContext(logger.Ctx{"volume": snapVol.Name()}) l.Debug("Umounting snapshot volume") unlock, err := snapVol.MountLock() if err != nil { return false, err } defer unlock() // For VMs, unmount the filesystem volume. if snapVol.IsVMBlock() { fsVol := snapVol.NewVMBlockFilesystemVolume() return d.UnmountVolumeSnapshot(fsVol, op) } ourUnmount := false mountPath := snapVol.MountPath() refCount := snapVol.MountRefCountDecrement() // Attempt to unmount the filesystem if snapVol.contentType == ContentTypeFS && linux.IsMountPoint(mountPath) { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": snapVol.name, "refCount": refCount}) return false, ErrInUse } ourUnmount, err = forceUnmount(mountPath) if err != nil { return false, err } l.Debug("Unmounted snapshot volume filesystem", logger.Ctx{"path": mountPath}) } l.Debug("Deleting temporary resource definition for snapshot mount") err = d.deleteResourceDefinitionFromSnapshot(snapVol) if err != nil { return false, fmt.Errorf("Could not delete temporary resource definition: %w", err) } d.logger.Debug("Temporary resource definition deleted") return ourUnmount, nil } // UpdateVolume applies config changes to the volume. func (d *linstor) UpdateVolume(vol Volume, changedConfig map[string]string) error { newSize, sizeChanged := changedConfig["size"] if sizeChanged { err := d.SetVolumeQuota(vol, newSize, false, nil) if err != nil { return err } } // If only the size changed, there's nothing left to do. if sizeChanged && len(changedConfig) == 1 { return nil } err := d.updateResourceDefinition(vol, changedConfig) if err != nil { return err } return d.updateVolume(vol, changedConfig) } // GetVolumeUsage returns the disk space used by the volume. func (d *linstor) GetVolumeUsage(vol Volume) (int64, error) { usageInKiB, err := d.getVolumeUsage(vol) if err != nil { return 0, fmt.Errorf("Could not get volume usage: %w", err) } usageInBytes := usageInKiB * 1024 return usageInBytes, nil } // SetVolumeQuota applies a size limit on volume. // Does nothing if supplied with an empty/zero size. func (d *linstor) SetVolumeQuota(vol Volume, size string, allowUnsafeResize bool, op *operations.Operation) error { l := d.logger.AddContext(logger.Ctx{"volume": vol.Name()}) l.Debug("Setting volume quota") // Convert to bytes. sizeBytes, err := units.ParseByteSizeString(size) if err != nil { return err } // Do nothing if size isn't specified. if sizeBytes <= 0 { return nil } // Get the device path. devPath, err := d.getLinstorDevPath(vol) if err != nil { return err } // LVM and ZFS (LINSTOR's backends) round up the final block device size to the next extent. Because of // this, we cannot simply get the volume size from the OS. We must fetch the size from LINSTOR. oldSizeBytes, err := d.getVolumeSize(vol) if err != nil { return fmt.Errorf("Error getting current size: %w", err) } // Do nothing if volume is already specified size (+/- 512 bytes). if oldSizeBytes+512 > sizeBytes && oldSizeBytes-512 < sizeBytes { return nil } inUse := vol.MountInUse() // Resize filesystem if needed. if vol.contentType == ContentTypeFS { fsType := vol.ConfigBlockFilesystem() if sizeBytes < oldSizeBytes { if !filesystemTypeCanBeShrunk(fsType) { return fmt.Errorf("Filesystem %q cannot be shrunk: %w", fsType, ErrCannotBeShrunk) } if inUse { return ErrInUse // We don't allow online shrinking of filesystem volumes. } // Shrink filesystem first. Pass allowUnsafeResize to allow disabling of filesystem // resize safety checks. err = shrinkFileSystem(fsType, devPath, vol, sizeBytes, allowUnsafeResize) if err != nil { return err } // Shrink the block device. err = d.resizeVolume(vol, sizeBytes) if err != nil { return err } } else if sizeBytes > oldSizeBytes { // Grow block device first. err = d.resizeVolume(vol, sizeBytes) if err != nil { return err } // Grow the filesystem to fill block device. err = growFileSystem(fsType, devPath, vol) if err != nil { return err } } } else { // Only perform pre-resize checks if we are not in "unsafe" mode. // In unsafe mode we expect the caller to know what they are doing and understand the risks. if !allowUnsafeResize { if sizeBytes < oldSizeBytes { return fmt.Errorf("Block volumes cannot be shrunk: %w", ErrCannotBeShrunk) } if inUse { return ErrInUse // We don't allow online resizing of block volumes. } } // Resize block device. err = d.resizeVolume(vol, sizeBytes) if err != nil { return err } // Move the VM GPT alt header to end of disk if needed (not needed in unsafe resize mode as it is // expected the caller will do all necessary post resize actions themselves). if vol.IsVMBlock() && !allowUnsafeResize { err = d.moveGPTAltHeader(devPath) if err != nil { return err } } } return nil } // MigrateVolume sends a volume for migration. func (d *linstor) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *localMigration.VolumeSourceArgs, op *operations.Operation) error { d.logger.Debug("Migrating volume", logger.Ctx{"volume": vol.Name(), "volSrcArgs": volSrcArgs}) // Optimized migration is currently only supported for case in which we are migrating // the volume across cluster members on the same storage pool. if volSrcArgs.ClusterMove && !volSrcArgs.StorageMove { d.logger.Debug("Detected migration between cluster members on the same storage pool", logger.Ctx{"volume": vol.Name(), "volSrcArgs": volSrcArgs}) // When migrating between cluster members on the same storage pool, don't do anything on the source member. return nil } // Handle simple rsync and block_and_rsync through generic. if volSrcArgs.MigrationType.FSType == migration.MigrationFSType_RSYNC || volSrcArgs.MigrationType.FSType == migration.MigrationFSType_BLOCK_AND_RSYNC { // TODO: create a fast snapshot to ensure migration consistency when the driver supports snapshots parent, _, _ := api.GetParentAndSnapshotName(vol.Name()) parentVol := NewVolume(d, d.Name(), vol.volType, vol.contentType, parent, vol.config, vol.poolConfig) err := d.MountVolume(parentVol, op) if err != nil { return err } defer func() { _, _ = d.UnmountVolume(parentVol, false, op) }() return genericVFSMigrateVolume(d, d.state, vol, conn, volSrcArgs, op) } else if volSrcArgs.MigrationType.FSType != migration.MigrationFSType_LINSTOR { return ErrNotSupported } return nil } // CreateVolumeFromMigration creates a volume being sent via a migration. func (d *linstor) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, volTargetArgs localMigration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error { l := d.logger.AddContext(logger.Ctx{"volume": vol.Name(), "volTargetArgs": volTargetArgs}) l.Debug("Receiving volume from migration") // Optimized migration is currently only supported for case in which we are migrating // the volume across cluster members on the same storage pool. if volTargetArgs.ClusterMoveSourceName != "" && volTargetArgs.StoragePool == "" { l.Debug("Detected migration between cluster members on the same storage pool") err := d.makeVolumeAvailable(vol) if err != nil { return err } err = vol.EnsureMountPath(false) if err != nil { return err } // For VMs, the associated filesystem volume is contained within the resource, so it // is migrated in the same operation. The only thing left to do is to ensure that its // mount path exists on the host. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err = fsVol.EnsureMountPath(false) if err != nil { return err } } l.Debug("Finished migrating volume") return nil } // Handle simple rsync and block_and_rsync through generic. if volTargetArgs.MigrationType.FSType == migration.MigrationFSType_RSYNC || volTargetArgs.MigrationType.FSType == migration.MigrationFSType_BLOCK_AND_RSYNC { return genericVFSCreateVolumeFromMigration(d, nil, vol, conn, volTargetArgs, preFiller, op) } else if volTargetArgs.MigrationType.FSType != migration.MigrationFSType_LINSTOR { return ErrNotSupported } l.Warn("Unsupported migration type detected") return ErrNotSupported } // VolumeSnapshots returns a list of snapshots for the volume (in no particular order). func (d *linstor) VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) { var snapshots []string resourceDefinition, err := d.getResourceDefinition(vol, false) if err != nil { return snapshots, err } snapshotMap, err := d.getSnapshotMap(vol) if err != nil { return snapshots, err } // Get the snapshots. linstorSnapshots, err := d.getSnapshots(resourceDefinition.Name) if err != nil { return snapshots, err } for _, snapshot := range linstorSnapshots { snapshots = append(snapshots, snapshotMap[snapshot.Name]) } return snapshots, nil } // BackupVolume copies a volume (and optionally its snapshots) to a specified target path. // This driver does not support optimized backups. func (d *linstor) BackupVolume(vol Volume, writer instancewriter.InstanceWriter, basePrefix string, optimized bool, snapshots []string, op *operations.Operation) error { return genericVFSBackupVolume(d, vol, writer, basePrefix, snapshots, op) } // CreateVolumeFromBackup restores a backup tarball onto the storage device. func (d *linstor) CreateVolumeFromBackup(vol Volume, srcBackup backup.Info, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) (VolumePostHook, revert.Hook, error) { return genericVFSBackupUnpack(d, d.state.OS, vol, srcBackup.Snapshots, srcData, basePrefix, op) } incus-7.0.0/internal/server/storage/drivers/driver_lvm.go000066400000000000000000000763451517523235500236320ustar00rootroot00000000000000package drivers import ( "errors" "fmt" "io/fs" "net/http" "os" "os/exec" "path/filepath" "slices" "strconv" "strings" "sync" "time" "github.com/lxc/incus/v7/internal/linux" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) const lvmVgPoolMarker = "incus_pool" // Indicator tag used to mark volume groups as in use. var lvmActivation sync.Mutex var ( lvmExtentSize map[string]int64 lvmExtentSizeMu sync.Mutex ) var ( lvmLoaded bool lvmClusterLoaded bool lvmVersion string ) type lvm struct { common clustered bool } func (d *lvm) load() error { // Register the patches. d.patches = map[string]func() error{ "storage_lvm_skipactivation": d.patchStorageSkipActivation, "storage_missing_snapshot_records": nil, "storage_delete_old_snapshot_records": nil, "storage_zfs_drop_block_volume_filesystem_extension": nil, "storage_prefix_bucket_names_with_project": nil, } // Done if previously loaded. if !d.clustered && lvmLoaded { return nil } if d.clustered && lvmClusterLoaded { return nil } // Validate the required binaries. tools := []string{"lvm"} if d.clustered { tools = append(tools, []string{"lvmlockctl", "sanlock", "btrfs"}...) } for _, tool := range tools { _, err := exec.LookPath(tool) if err != nil { return fmt.Errorf("Required tool %q is missing", tool) } } if d.clustered { err := linux.LoadModule("nbd") if err != nil { return fmt.Errorf("Error loading nbd module: %w", err) } } // Detect and record the version. if lvmVersion == "" { output, err := subprocess.RunCommand("lvm", "version") if err != nil { return fmt.Errorf("Error getting LVM version: %w", err) } lines := strings.Split(output, "\n") for idx, line := range lines { fields := strings.SplitAfterN(line, ":", 2) if len(fields) < 2 { continue } if !strings.Contains(line, "version:") { continue } if idx > 0 { lvmVersion += " / " } lvmVersion += strings.TrimSpace(fields[1]) } } if d.clustered { lvmClusterLoaded = true } else { lvmLoaded = true } return nil } func (d *lvm) init(s *state.State, name string, config map[string]string, log logger.Logger, volIDFunc func(volType VolumeType, volName string) (int64, error), commonRules *Validators) { d.common.init(s, name, config, log, volIDFunc, commonRules) if d.config != nil { _, exists := d.config["lvm.vg_name"] if !exists { sourceType := d.getSourceType(d.config["source"]) if sourceType == lvmSourceTypeDefault || sourceType == lvmSourceTypePhysicalDevice { d.config["lvm.vg_name"] = d.name } else if sourceType == lvmSourceTypeVolumeGroup { d.config["lvm.vg_name"] = d.config["source"] } } } } // isRemote returns true indicating this driver uses remote storage. func (d *lvm) isRemote() bool { return d.clustered } // Info returns info about the driver and its environment. func (d *lvm) Info() Info { name := "lvm" targetFormat := BlockVolumeTypeRaw if d.clustered { name = "lvmcluster" targetFormat = BlockVolumeTypeQcow2 } return Info{ Name: name, Version: lvmVersion, DefaultVMBlockFilesystemSize: deviceConfig.DefaultVMBlockFilesystemSize, OptimizedImages: d.usesThinpool(), // Only thinpool pools support optimized images. PreservesInodes: false, Remote: d.isRemote(), VolumeTypes: []VolumeType{VolumeTypeBucket, VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM}, VolumeMultiNode: d.isRemote(), BlockBacking: true, RunningCopyFreeze: true, SameSource: d.isRemote(), DirectIO: true, IOUring: true, MountedRoot: false, Buckets: !d.isRemote(), Deactivate: d.isRemote(), ZeroUnpack: !d.usesThinpool(), TargetFormat: targetFormat, } } // FillConfig populates the storage pool's configuration file with the default values. func (d *lvm) FillConfig() error { // Set default thin pool name if not specified. if d.usesThinpool() && d.config["lvm.thinpool_name"] == "" { d.config["lvm.thinpool_name"] = lvmThinpoolDefaultName } return nil } // Create creates the storage pool on the storage device. func (d *lvm) Create() error { d.config["volatile.initial_source"] = d.config["source"] var err error var pvExists, vgExists bool var pvName string var vgTags []string reverter := revert.New() defer reverter.Fail() err = d.FillConfig() if err != nil { return err } var usingLoopFile bool sourceType := d.getSourceType(d.config["source"]) // Require the LVM service when on IncusOS. if d.state.OS.IncusOS != nil { ok, err := d.state.OS.IncusOS.IsServiceEnabled("lvm") if err != nil { return err } if !ok { return errors.New("IncusOS service \"lvm\" isn't currently enabled") } } if sourceType == lvmSourceTypeDefault { if d.clustered { return errors.New("lvmcluster requires a shared physical device or a pre-existing shared VG to be used as source") } // Check for IncusOS. if d.state.OS.IncusOS != nil { return errors.New("Loop backed pools aren't supported on IncusOS") } usingLoopFile = true defaultSource := loopFilePath(d.name) // We are using an internal loopback file. d.config["source"] = defaultSource if d.config["lvm.vg_name"] == "" { d.config["lvm.vg_name"] = d.name } // Pick a default size of the loop file if not specified. if d.config["size"] == "" { defaultSize, err := loopFileSizeDefault() if err != nil { return err } d.config["size"] = fmt.Sprintf("%dGiB", defaultSize) } size, err := units.ParseByteSizeString(d.config["size"]) if err != nil { return err } if util.PathExists(d.config["source"]) { return fmt.Errorf("Source file location %q already exists", d.config["source"]) } err = ensureSparseFile(d.config["source"], size) if err != nil { return fmt.Errorf("Failed to create sparse file %q: %w", d.config["source"], err) } reverter.Add(func() { _ = os.Remove(d.config["source"]) }) // Open the loop file. loopDevPath, err := d.openLoopFile(d.config["source"]) if err != nil { return err } defer func() { _ = loopDeviceAutoDetach(loopDevPath) }() // Check if the physical volume already exists. pvName = loopDevPath pvExists, err = d.pysicalVolumeExists(pvName) if err != nil { return err } if pvExists { return fmt.Errorf("A physical volume already exists for %q", pvName) } // Check if the volume group already exists. vgExists, vgTags, err = d.volumeGroupExists(d.config["lvm.vg_name"]) if err != nil { return err } if vgExists { return fmt.Errorf("A volume group already exists called %q", d.config["lvm.vg_name"]) } } else if sourceType == lvmSourceTypePhysicalDevice { // We are using an existing physical device. srcPath := d.config["source"] // Size is invalid as the physical device is already sized. if d.config["size"] != "" && !d.usesThinpool() { return errors.New("Cannot specify size when using an existing physical device for non-thin pool") } if d.config["lvm.vg_name"] == "" { d.config["lvm.vg_name"] = d.name } d.config["source"] = d.config["lvm.vg_name"] if !linux.IsBlockdevPath(srcPath) { return errors.New("Custom loop file locations are not supported") } // Wipe if requested. if util.IsTrue(d.config["source.wipe"]) { err := wipeBlockHeaders(d.config["source"]) if err != nil { return fmt.Errorf("Failed to wipe headers from disk %q: %w", d.config["source"], err) } d.config["source.wipe"] = "" } // Check if the volume group already exists. vgExists, vgTags, err = d.volumeGroupExists(d.config["lvm.vg_name"]) if err != nil { return err } if vgExists { return fmt.Errorf("Volume group already exists, cannot use new physical device at %q", srcPath) } // Check if the physical volume already exists. pvName = srcPath pvExists, err = d.pysicalVolumeExists(pvName) if err != nil { return err } } else if sourceType == lvmSourceTypeVolumeGroup { // We are using an existing volume group, so physical must exist already. pvExists = true // Size is invalid as the volume group is already sized. if d.config["size"] != "" && !d.usesThinpool() { return errors.New("Cannot specify size when using an existing volume group for non-thin pool") } if d.config["lvm.vg_name"] != "" && d.config["lvm.vg_name"] != d.config["source"] { return errors.New("Invalid combination of source and lvm.vg_name properties") } d.config["lvm.vg_name"] = d.config["source"] // Check the volume group already exists. vgExists, vgTags, err = d.volumeGroupExists(d.config["lvm.vg_name"]) if err != nil { return err } if !vgExists { return fmt.Errorf("The requested volume group %q does not exist", d.config["lvm.vg_name"]) } } else { return errors.New("Invalid source property") } // This is an internal error condition which should never be hit. if d.config["lvm.vg_name"] == "" { return errors.New("No name for volume group detected") } // Used to track the result of checking whether the thin pool exists during the existing volume group empty // checks to avoid having to do it twice. thinPoolExists := false if vgExists { // Check that the volume group is empty. Otherwise we will refuse to use it. // The LV count returned includes both normal volumes and thin volumes. lvCount, err := d.countLogicalVolumes(d.config["lvm.vg_name"]) if err != nil { return fmt.Errorf("Failed to determine whether the volume group %q is empty: %w", d.config["lvm.vg_name"], err) } empty := false if lvCount > 0 { if d.usesThinpool() { // Always check if the thin pool exists as we may need to create it later. thinPoolExists, err = d.thinpoolExists(d.config["lvm.vg_name"], d.thinpoolName()) if err != nil { return fmt.Errorf("Failed to determine whether thinpool %q exists in volume group %q: %w", d.config["lvm.vg_name"], d.thinpoolName(), err) } // If the single volume is the storage pool's thin pool LV then we still consider // this an empty volume group. if thinPoolExists && lvCount == 1 { empty = true } } } else { empty = true } // Skip the in use checks if the force reuse option is enabled. This allows a storage pool to be // backed by an existing non-empty volume group. Note: This option should be used with care, as Incus // can then not guarantee that volume name conflicts won't occur with non-Incus created volumes in // the same volume group. This could also potentially lead to Incus deleting a non-Incus volume should // name conflicts occur. if util.IsFalseOrEmpty(d.config["lvm.vg.force_reuse"]) { if !empty { return fmt.Errorf("Volume group %q is not empty", d.config["lvm.vg_name"]) } // Check the tags on the volume group to check it is not already being used. if slices.Contains(vgTags, lvmVgPoolMarker) { return fmt.Errorf("Volume group %q is already used by Incus", d.config["lvm.vg_name"]) } } } else { // Calculate the metadata size (if provided). metadataSizeBytes, err := d.roundedSizeBytesString(d.config["lvm.metadata_size"]) if err != nil { return fmt.Errorf("Invalid lvm.metadata_size: %w", err) } // Create physical volume if doesn't exist. // This is done to provide better errors and revert when possible. if !pvExists && !d.clustered { // This is an internal error condition which should never be hit. if pvName == "" { return errors.New("No name for physical volume detected") } args := []string{} if metadataSizeBytes > 0 { args = append(args, "--metadatasize", fmt.Sprintf("%db", metadataSizeBytes)) } args = append(args, pvName) _, err = subprocess.TryRunCommand("pvcreate", args...) if err != nil { return err } reverter.Add(func() { _, _ = subprocess.TryRunCommand("pvremove", pvName) }) } // Create volume group. args := []string{d.config["lvm.vg_name"], pvName} if d.clustered { args = append(args, "--shared") if metadataSizeBytes > 0 { args = append(args, "--metadatasize", fmt.Sprintf("%db", metadataSizeBytes)) } } _, err = subprocess.TryRunCommand("vgcreate", args...) if err != nil { return err } d.logger.Debug("Volume group created", logger.Ctx{"pv_name": pvName, "vg_name": d.config["lvm.vg_name"]}) reverter.Add(func() { _, _ = subprocess.TryRunCommand("vgremove", d.config["lvm.vg_name"]) }) } if d.clustered { currentSize, err := d.volumeGroupSize(d.config["lvm.vg_name"]) if err != nil { return err } // Record initial size of volume group. d.config["size"] = fmt.Sprintf("%d", currentSize) } // Create thin pool if needed. if d.usesThinpool() { if !thinPoolExists { var thinpoolSizeBytes int64 // If not using loop file then the size setting controls the size of the thinpool volume. if !usingLoopFile { thinpoolSizeBytes, err = d.roundedSizeBytesString(d.config["size"]) if err != nil { return fmt.Errorf("Invalid size: %w", err) } } err = d.createDefaultThinPool(d.Info().Version, d.thinpoolName(), thinpoolSizeBytes) if err != nil { return err } d.logger.Debug("Thin pool created", logger.Ctx{"vg_name": d.config["lvm.vg_name"], "thinpool_name": d.thinpoolName()}) reverter.Add(func() { _ = d.removeLogicalVolume(d.lvmPath(d.config["lvm.vg_name"], "", "", d.thinpoolName())) }) } else if d.config["size"] != "" { return errors.New("Cannot specify size when using an existing thin pool") } } // Mark the volume group with the lvmVgPoolMarker tag to indicate it is now in use by Incus. _, err = subprocess.TryRunCommand("vgchange", "--addtag", lvmVgPoolMarker, d.config["lvm.vg_name"]) if err != nil { return err } d.logger.Debug("Incus marker tag added to volume group", logger.Ctx{"vg_name": d.config["lvm.vg_name"]}) reverter.Success() return nil } // Delete removes the storage pool from the storage device. func (d *lvm) Delete(op *operations.Operation) error { var err error var loopDevPath string // Open the loop file if needed. if filepath.IsAbs(d.config["source"]) && !linux.IsBlockdevPath(d.config["source"]) { loopDevPath, err = d.openLoopFile(d.config["source"]) if err != nil { return err } defer func() { _ = loopDeviceAutoDetach(loopDevPath) }() } vgExists, vgTags, err := d.volumeGroupExists(d.config["lvm.vg_name"]) if err != nil { return err } removeVg := false if vgExists && util.IsFalseOrEmpty(d.config["lvm.vg.force_reuse"]) { // Count normal and thin volumes. lvCount, err := d.countLogicalVolumes(d.config["lvm.vg_name"]) if err != nil { if !api.StatusErrorCheck(err, http.StatusNotFound) { return err } } // Check that volume group is not in use. If it is we need to assume that other users are using // the volume group, so don't remove it. This actually goes against policy since we explicitly // state: our pool, and nothing but our pool, but still, let's not hurt users. if err == nil { if lvCount == 0 { removeVg = true // Volume group is totally empty, safe to remove. } else if d.usesThinpool() && lvCount > 0 { // Lets see if the lv count is just our thin pool, or whether we can only remove // the thin pool itself and not the volume group. thinVolCount, err := d.countThinVolumes(d.config["lvm.vg_name"], d.thinpoolName()) if err != nil { if !api.StatusErrorCheck(err, http.StatusNotFound) { return err } } // Thin pool exists. if err == nil { // If thin pool is empty and the total VG volume count is 1 (our thin pool // volume) then just remove the entire volume group. if thinVolCount == 0 && lvCount == 1 { removeVg = true } else if thinVolCount == 0 && lvCount > 1 { // Otherwise, if the thin pool is empty but the volume group has // other volumes, then just remove the thin pool volume. err = d.removeLogicalVolume(d.lvmPath(d.config["lvm.vg_name"], "", "", d.thinpoolName())) if err != nil { return fmt.Errorf("Failed to delete thin pool %q from volume group %q: %w", d.thinpoolName(), d.config["lvm.vg_name"], err) } d.logger.Debug("Thin pool removed", logger.Ctx{"vg_name": d.config["lvm.vg_name"], "thinpool_name": d.thinpoolName()}) } } } } // Remove volume group if needed. if removeVg { // When deleting a shared VG, it may take more than a minute for the previously released shared locks to clear. _, err := subprocess.TryRunCommandAttemptsDuration(240, 500*time.Millisecond, "vgremove", "-f", d.config["lvm.vg_name"]) if err != nil { return fmt.Errorf("Failed to delete the volume group for the lvm storage pool: %w", err) } d.logger.Debug("Volume group removed", logger.Ctx{"vg_name": d.config["lvm.vg_name"]}) } else { // Otherwise just remove the lvmVgPoolMarker tag to indicate Incus no longer uses this VG. if slices.Contains(vgTags, lvmVgPoolMarker) { _, err = subprocess.TryRunCommand("vgchange", "--deltag", lvmVgPoolMarker, d.config["lvm.vg_name"]) if err != nil { return fmt.Errorf("Failed to remove marker tag on volume group for the lvm storage pool: %w", err) } d.logger.Debug("Incus marker tag removed from volume group", logger.Ctx{"vg_name": d.config["lvm.vg_name"]}) } } } // If we have removed the volume group and this is a loop file, lets clean up the physical volume too. if removeVg && loopDevPath != "" { _, err := subprocess.TryRunCommand("pvremove", "-f", loopDevPath) if err != nil { d.logger.Warn("Failed to destroy the physical volume for the lvm storage pool", logger.Ctx{"err": err}) } d.logger.Debug("Physical volume removed", logger.Ctx{"pv_name": loopDevPath}) err = loopDeviceAutoDetach(loopDevPath) if err != nil { d.logger.Warn("Failed to set LO_FLAGS_AUTOCLEAR on loop device, manual cleanup needed", logger.Ctx{"dev": loopDevPath, "err": err}) } // This is a loop file so deconfigure the associated loop device. err = os.Remove(d.config["source"]) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Error removing LVM pool loop file %q: %w", d.config["source"], err) } d.logger.Debug("Physical loop file removed", logger.Ctx{"file_name": d.config["source"]}) } // Wipe everything in the storage pool directory. err = wipeDirectory(GetPoolMountPath(d.name)) if err != nil { return err } return nil } func (d *lvm) Validate(config map[string]string) error { // gendoc:generate(entity=storage_lvm, group=common, key=source) // // --- // type: string // scope: local // default: - // shortdesc: Path to an existing block device, loop file or LVM volume group. // gendoc:generate(entity=storage_lvm, group=common, key=source.wipe) // // --- // type: bool // scope: local // default: `false` // shortdesc: Wipe the block device specified in `source` prior to creating the storage pool. rules := map[string]func(value string) error{ // gendoc:generate(entity=storage_lvm, group=common, key=size) // // --- // type: string // scope: local // default: auto (20% of free disk space, >= 5 GiB and <= 30 GiB) for `lvm`. // shortdesc: Storage pool size (in bytes, suffixes supported, can be increased to grow storage pool). `lvmcluster` pools: cannot set at creation, but can be updated. "size": validate.Optional(func(value string) error { if d.clustered && value == MaxValue { return nil } return validate.IsSize(value) }), // gendoc:generate(entity=storage_lvm, group=common, key=lvm.vg_name) // // --- // type: string // scope: local // default: name of the pool // shortdesc: Name of the volume group to create. "lvm.vg_name": validate.IsAny, // gendoc:generate(entity=storage_lvm, group=common, key=lvm.metadata_size) // // --- // type: string // scope: global // default: `0` (auto) // shortdesc: The size of the metadata space for the physical volume. "lvm.metadata_size": validate.Optional(validate.IsSize), } if !d.clustered { // gendoc:generate(entity=storage_lvm, group=common, key=lvm.thinpool_name) // // --- // type: string // scope: local // default: `IncusThinPool` // shortdesc: Thin pool where volumes are created. Not usable with `lvmcluster`. rules["lvm.thinpool_name"] = validate.IsAny // gendoc:generate(entity=storage_lvm, group=common, key=lvm.thinpool_metadata_size) // // --- // type: string // scope: global // default: `0` (auto) // shortdesc: The size of the thin pool metadata volume (the default is to let LVM calculate an appropriate size). Not usable with `lvmcluster`. rules["lvm.thinpool_metadata_size"] = validate.Optional(validate.IsSize) // gendoc:generate(entity=storage_lvm, group=common, key=lvm.use_thinpool) // // --- // type: bool // scope: global // default: `true` // shortdesc: Whether the storage pool uses a thin pool for logical volumes. Not usable with `lvmcluster`. rules["lvm.use_thinpool"] = validate.Optional(validate.IsBool) // gendoc:generate(entity=storage_lvm, group=common, key=lvm.vg.force_reuse) // // --- // type: bool // scope: local // default: `false` // shortdesc: Force using an existing non-empty volume group. Not usable with `lvmcluster`. rules["lvm.vg.force_reuse"] = validate.Optional(validate.IsBool) } err := d.validatePool(config, rules, d.commonVolumeRules()) if err != nil { return err } if util.IsFalse(config["lvm.use_thinpool"]) { if config["lvm.thinpool_name"] != "" { return errors.New("The key lvm.use_thinpool cannot be set to false when lvm.thinpool_name is set") } if config["lvm.thinpool_metadata_size"] != "" { return errors.New("The key lvm.use_thinpool cannot be set to false when lvm.thinpool_metadata_size is set") } } return nil } // Update updates the storage pool settings. func (d *lvm) Update(changedConfig map[string]string) error { _, changed := changedConfig["lvm.use_thinpool"] if changed { return errors.New("lvm.use_thinpool cannot be changed") } _, changed = changedConfig["lvm.thinpool_metadata_size"] if changed { return errors.New("lvm.thinpool_metadata_size cannot be changed") } _, changed = changedConfig["lvm.metadata_size"] if changed { return errors.New("lvm.metadata_size cannot be changed") } _, changed = changedConfig["volume.lvm.stripes"] if changed && d.usesThinpool() { return errors.New("volume.lvm.stripes cannot be changed when using thin pool") } _, changed = changedConfig["volume.lvm.stripes.size"] if changed && d.usesThinpool() { return errors.New("volume.lvm.stripes.size cannot be changed when using thin pool") } _, changed = changedConfig["volume.block.type"] if changed { return errors.New("volume.block.type cannot be changed after creation") } if changedConfig["lvm.vg_name"] != "" { _, err := subprocess.TryRunCommand("vgrename", d.config["lvm.vg_name"], changedConfig["lvm.vg_name"]) if err != nil { return fmt.Errorf("Error renaming LVM volume group from %q to %q: %w", d.config["lvm.vg_name"], changedConfig["lvm.vg_name"], err) } d.logger.Debug("Volume group renamed", logger.Ctx{"vg_name": d.config["lvm.vg_name"], "new_vg_name": changedConfig["lvm.vg_name"]}) } if changedConfig["lvm.thinpool_name"] != "" { _, err := subprocess.TryRunCommand("lvrename", d.config["lvm.vg_name"], d.thinpoolName(), changedConfig["lvm.thinpool_name"]) if err != nil { return fmt.Errorf("Error renaming LVM thin pool from %q to %q: %w", d.thinpoolName(), changedConfig["lvm.thinpool_name"], err) } d.logger.Debug("Thin pool volume renamed", logger.Ctx{"vg_name": d.config["lvm.vg_name"], "thinpool": d.thinpoolName(), "new_thinpool": changedConfig["lvm.thinpool_name"]}) } size, ok := changedConfig["size"] if ok && size != "" { if d.clustered { oldSize, err := d.volumeGroupSize(d.config["lvm.vg_name"]) if err != nil { return err } devices, err := d.getPhysicalDevices(d.config["lvm.vg_name"]) if err != nil { return err } sourceType := d.getSourceType(d.config["volatile.initial_source"]) switch sourceType { case lvmSourceTypePhysicalDevice: var sizeBytes int64 if size != MaxValue { sizeBytes, _ = units.ParseByteSizeString(size) if oldSize >= sizeBytes { return fmt.Errorf("The new size must be larger than the current size") } } if len(devices) != 1 { return fmt.Errorf("Physical device source type should have only one device. Current number: %d", len(devices)) } blockdevSize, err := BlockDiskSizeBytes(devices[0]) if err != nil { return err } if sizeBytes > 0 && sizeBytes > blockdevSize { return fmt.Errorf("Specified size is larger than the physical device size") } err = d.resizePhysicalVolume(devices[0], sizeBytes) if err != nil { return err } case lvmSourceTypeVolumeGroup: if size != MaxValue { return fmt.Errorf("Volume group source type supports only the 'max' size value") } for _, dev := range devices { err = d.resizePhysicalVolume(dev, 0) if err != nil { return err } } default: return fmt.Errorf("Resize is only supported when the source is a physical device or a volume group") } currentSize, err := d.volumeGroupSize(d.config["lvm.vg_name"]) if err != nil { return err } if oldSize == currentSize { return fmt.Errorf("The volume group is already at the maximum size") } // Record the new size. d.config["size"] = fmt.Sprintf("%d", currentSize) return nil } // Figure out loop path loopPath := loopFilePath(d.name) if d.config["source"] != loopPath { return errors.New("Cannot resize non-loopback pools") } // Resize loop file f, err := os.OpenFile(loopPath, os.O_RDWR, 0o600) if err != nil { return err } defer func() { _ = f.Close() }() sizeBytes, _ := units.ParseByteSizeString(size) err = f.Truncate(sizeBytes) if err != nil { return err } loopDevPath, err := loopDeviceSetup(loopPath) if err != nil { return err } err = loopDeviceSetCapacity(loopDevPath) if err != nil { return err } // Resize physical volume so that lvresize is able to resize as well. _, err = subprocess.RunCommand("pvresize", "-y", loopDevPath) if err != nil { return err } if d.usesThinpool() { lvPath := d.lvmPath(d.config["lvm.vg_name"], "", "", d.thinpoolName()) // Use the remaining space in the volume group. _, err = subprocess.RunCommand("lvresize", "-f", "-l", "+100%FREE", lvPath) if err != nil { return err } } } return nil } // Mount mounts the storage pool (for loopback image pools this creates a loop device), and checks the volume group // and thin pool volume (if used) exists. func (d *lvm) Mount() (bool, error) { if d.config["lvm.vg_name"] == "" { return false, fmt.Errorf("Cannot mount pool as %q is not specified", "lvm.vg_name") } // Check if VG exists before we do anything, this will indicate if its our mount or not. vgExists, _, _ := d.volumeGroupExists(d.config["lvm.vg_name"]) ourMount := !vgExists waitDuration := time.Second * time.Duration(5) reverter := revert.New() defer reverter.Fail() // If clustered LVM, start lock manager. if d.clustered { _, err := subprocess.RunCommand("vgchange", "--lockstart", d.config["lvm.vg_name"]) if err != nil { return false, fmt.Errorf("Error starting lock manager: %w", err) } } // Open the loop file if the source points to a non-block device file. // This ensures that auto clear isn't enabled on the loop file. if filepath.IsAbs(d.config["source"]) && !linux.IsBlockdevPath(d.config["source"]) { loopDevPath, err := d.openLoopFile(d.config["source"]) if err != nil { return false, err } reverter.Add(func() { _ = loopDeviceAutoDetach(loopDevPath) }) // Wait for volume group to be detected if wasn't detected before. if !vgExists { waitUntil := time.Now().Add(waitDuration) for { vgExists, _, _ = d.volumeGroupExists(d.config["lvm.vg_name"]) if vgExists { break } if time.Now().After(waitUntil) { return false, fmt.Errorf("Volume group %q not found", d.config["lvm.vg_name"]) } time.Sleep(1 * time.Second) } } } else if !vgExists { return false, fmt.Errorf("Volume group %s not found", d.config["lvm.vg_name"]) } // Ensure thinpool exists if needed for storage pool. if d.usesThinpool() { waitUntil := time.Now().Add(waitDuration) for { thinpoolExists, _ := d.thinpoolExists(d.config["lvm.vg_name"], d.thinpoolName()) if thinpoolExists { break } if time.Now().After(waitUntil) { return false, fmt.Errorf("Thin pool not found %q in volume group %q", d.thinpoolName(), d.config["lvm.vg_name"]) } time.Sleep(1 * time.Second) } } reverter.Success() return ourMount, nil } // Unmount unmounts the storage pool (this does nothing). // LVM doesn't currently support unmounting, please see https://github.com/canonical/lxd/issues/9278 func (d *lvm) Unmount() (bool, error) { if d.clustered { _, err := subprocess.RunCommand("vgchange", "--lockstop", d.config["lvm.vg_name"]) if err != nil { return false, fmt.Errorf("Error stopping lock manager: %w", err) } } return false, nil } // GetResources returns utilisation and space info about the pool. func (d *lvm) GetResources() (*api.ResourcesStoragePool, error) { res := api.ResourcesStoragePool{} // Thinpools will always report zero free space on the volume group, so calculate approx // used space using the thinpool logical volume allocated (data and meta) percentages. if d.usesThinpool() { volDevPath := d.lvmPath(d.config["lvm.vg_name"], "", "", d.thinpoolName()) totalSize, usedSize, err := d.thinPoolVolumeUsage(volDevPath) if err != nil { return nil, err } res.Space.Total = totalSize res.Space.Used = usedSize } else { // If thinpools are not in use, calculate used space in volume group. args := []string{ d.config["lvm.vg_name"], "--noheadings", "--units", "b", "--nosuffix", "--separator", ",", "-o", "vg_size,vg_free", } out, err := subprocess.RunCommand("vgs", args...) if err != nil { return nil, err } parts := strings.Split(strings.TrimSpace(out), ",") if len(parts) < 2 { return nil, errors.New("Unexpected output from vgs command") } total, err := strconv.ParseUint(parts[0], 10, 64) if err != nil { return nil, err } res.Space.Total = total free, err := strconv.ParseUint(parts[1], 10, 64) if err != nil { return nil, err } res.Space.Used = total - free } return &res, nil } // roundVolumeBlockSizeBytes returns sizeBytes rounded up to the next multiple // of the volume group extent size. func (d *lvm) roundVolumeBlockSizeBytes(vol Volume, sizeBytes int64) (int64, error) { // Get the volume group's physical extent size, and use that as minimum size. vgExtentSize, err := d.volumeGroupExtentSize(d.config["lvm.vg_name"]) if err != nil { return -1, err } return RoundAbove(vgExtentSize, sizeBytes), nil } incus-7.0.0/internal/server/storage/drivers/driver_lvm_patches.go000066400000000000000000000033361517523235500253270ustar00rootroot00000000000000package drivers import ( "fmt" "strings" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/subprocess" ) // patchStorageSkipActivation set skipactivation=y on all Incus LVM logical volumes (excluding thin pool volumes). func (d *lvm) patchStorageSkipActivation() error { out, err := subprocess.RunCommand("lvs", "--noheadings", "-o", "lv_name,lv_attr", d.config["lvm.vg_name"]) if err != nil { return fmt.Errorf("Error getting LVM logical volume list for storage pool %q: %w", d.config["lvm.vg_name"], err) } for _, line := range strings.Split(out, "\n") { fields := strings.Fields(strings.TrimSpace(line)) if len(fields) != 2 { continue } volName := fields[0] volAttr := fields[1] // Ignore non-Incus prefixes, and thinpool volumes (these should remain auto activated). if !strings.HasPrefix(volName, "images_") && !strings.HasPrefix(volName, "containers_") && !strings.HasPrefix(volName, "virtual-machines_") && !strings.HasPrefix(volName, "custom_") { continue } // Skip volumes that already have k flag set, meaning setactivationskip=y. if strings.HasSuffix(volAttr, "k") { logger.Infof("Skipping volume %q that already has skipactivation=y set in pool %q", volName, d.config["lvm.vg_name"]) continue } // Set the --setactivationskip flag enabled on the volume. _, err = subprocess.RunCommand("lvchange", "--setactivationskip", "y", fmt.Sprintf("%s/%s", d.config["lvm.vg_name"], volName)) if err != nil { return fmt.Errorf("Error setting setactivationskip=y on LVM logical volume %q for storage pool %q: %w", volName, d.config["lvm.vg_name"], err) } logger.Infof("Set setactivationskip=y on volume %q in pool %q", volName, d.config["lvm.vg_name"]) } return nil } incus-7.0.0/internal/server/storage/drivers/driver_lvm_utils.go000066400000000000000000000767301517523235500250500ustar00rootroot00000000000000package drivers import ( "context" "errors" "fmt" "net/http" "os" "os/exec" "path/filepath" "strconv" "strings" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/locking" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // lvmBlockVolSuffix suffix used for block content type volumes. const lvmBlockVolSuffix = ".block" // lvmISOVolSuffix suffix used for iso content type volumes. const lvmISOVolSuffix = ".iso" // lvmSnapshotSeparator separator character used between volume name and snapshot name in logical volume names. const lvmSnapshotSeparator = "-" // lvmEscapedHyphen used to escape hyphens in volume names to avoid conflicts with lvmSnapshotSeparator. const lvmEscapedHyphen = "--" // lvmThinpoolDefaultName is the default name for the thinpool volume. const lvmThinpoolDefaultName = "IncusThinPool" type lvmSourceType int const ( lvmSourceTypeUnknown lvmSourceType = iota lvmSourceTypeDefault lvmSourceTypePhysicalDevice lvmSourceTypeVolumeGroup ) // usesThinpool indicates whether the config specifies to use a thin pool or not. func (d *lvm) usesThinpool() bool { // No thin pool on clustered LVM. if d.clustered { return false } // Default is to use a thinpool. return util.IsTrueOrEmpty(d.config["lvm.use_thinpool"]) } // thinpoolName returns the thinpool volume to use. func (d *lvm) thinpoolName() string { if d.config["lvm.thinpool_name"] != "" { return d.config["lvm.thinpool_name"] } return lvmThinpoolDefaultName } // openLoopFile opens a loop device and returns the device path. func (d *lvm) openLoopFile(source string) (string, error) { if source == "" { return "", errors.New("No source property found for the storage pool") } if filepath.IsAbs(source) && !linux.IsBlockdevPath(source) { unlock, err := locking.Lock(context.TODO(), OperationLockName("openLoopFile", d.name, "", "", "")) if err != nil { return "", err } defer unlock() loopDeviceName, err := loopDeviceSetup(source) if err != nil { return "", err } return loopDeviceName, nil } return "", errors.New("Source is not loop file") } // isLVMNotFoundExitError checks whether the supplied error is an exit error from an LVM command // meaning that the object was not found. Returns true if it is (exit status 5) false if not. func (d *lvm) isLVMNotFoundExitError(err error) bool { var exitError *exec.ExitError if errors.As(err, &exitError) { if exitError.ExitCode() == 5 { return true } } return false } // pysicalVolumeExists checks if an LVM Physical Volume exists. func (d *lvm) pysicalVolumeExists(pvName string) (bool, error) { _, err := subprocess.RunCommand("pvs", "--noheadings", "-o", "pv_name", pvName) if err != nil { if d.isLVMNotFoundExitError(err) { return false, nil } return false, fmt.Errorf("Error checking for LVM physical volume %q: %w", pvName, err) } return true, nil } // volumeGroupExists checks if an LVM Volume Group exists and returns any tags on that volume group. func (d *lvm) volumeGroupExists(vgName string) (bool, []string, error) { output, err := subprocess.RunCommand("vgs", "--noheadings", "-o", "vg_tags", vgName) if err != nil { if d.isLVMNotFoundExitError(err) { return false, nil, nil } return false, nil, fmt.Errorf("Error checking for LVM volume group %q: %w", vgName, err) } output = strings.TrimSpace(output) tags := strings.SplitN(output, ",", -1) return true, tags, nil } // getPhysicalDevices lists PVs backing a volume group. func (d *lvm) getPhysicalDevices(vgName string) ([]string, error) { args := []string{ "--noheadings", "-o", "pv_name", "-S", fmt.Sprintf("vg_name=%s", vgName), } devices, err := subprocess.RunCommand("pvs", args...) if err != nil { return nil, err } result := []string{} for _, line := range strings.Split(strings.TrimSpace(devices), "\n") { result = append(result, strings.TrimSpace(line)) } return result, nil } // volumeGroupExtentSize gets the volume group's physical extent size in bytes. func (d *lvm) volumeGroupExtentSize(vgName string) (int64, error) { // Look for cached value. lvmExtentSizeMu.Lock() defer lvmExtentSizeMu.Unlock() if lvmExtentSize == nil { lvmExtentSize = map[string]int64{} } else if lvmExtentSize[d.name] > 0 { return lvmExtentSize[d.name], nil } output, err := subprocess.TryRunCommand("vgs", "--noheadings", "--nosuffix", "--units", "b", "-o", "vg_extent_size", vgName) if err != nil { if d.isLVMNotFoundExitError(err) { return -1, api.StatusErrorf(http.StatusNotFound, "LVM volume group not found") } return -1, err } output = strings.TrimSpace(output) val, err := strconv.ParseInt(output, 10, 64) if err != nil { return -1, err } lvmExtentSize[d.name] = val return val, nil } // volumeGroupSize gets the volume group's physical size in bytes. func (d *lvm) volumeGroupSize(vgName string) (int64, error) { output, err := subprocess.TryRunCommand("vgs", "--noheadings", "--nosuffix", "--units", "b", "-o", "vg_size", vgName) if err != nil { if d.isLVMNotFoundExitError(err) { return -1, api.StatusErrorf(http.StatusNotFound, "LVM volume group not found") } return -1, err } output = strings.TrimSpace(output) val, err := strconv.ParseInt(output, 10, 64) if err != nil { return -1, err } return val, nil } // resizePhysicalVolume resizes the given PV. func (d *lvm) resizePhysicalVolume(devPath string, size int64) error { args := []string{ "-y", devPath, } if size > 0 { args = append(args, "--setphysicalvolumesize", fmt.Sprintf("%dB", size)) } _, err := subprocess.RunCommand("pvresize", args...) if err != nil { return err } return nil } // countLogicalVolumes gets the count of volumes (both normal and thin) in a volume group. func (d *lvm) countLogicalVolumes(vgName string) (int, error) { output, err := subprocess.TryRunCommand("vgs", "--noheadings", "-o", "lv_count", vgName) if err != nil { if d.isLVMNotFoundExitError(err) { return -1, api.StatusErrorf(http.StatusNotFound, "LVM volume group not found") } return -1, fmt.Errorf("Error counting logical volumes in LVM volume group %q: %w", vgName, err) } output = strings.TrimSpace(output) return strconv.Atoi(output) } // countThinVolumes gets the count of thin volumes in a thin pool. func (d *lvm) countThinVolumes(vgName, poolName string) (int, error) { output, err := subprocess.TryRunCommand("lvs", "--noheadings", "-o", "thin_count", fmt.Sprintf("%s/%s", vgName, poolName)) if err != nil { if d.isLVMNotFoundExitError(err) { return -1, api.StatusErrorf(http.StatusNotFound, "LVM volume group not found") } return -1, fmt.Errorf("Error counting thin volumes in LVM volume group %q: %w", vgName, err) } output = strings.TrimSpace(output) return strconv.Atoi(output) } // thinpoolExists checks whether the specified thinpool exists in a volume group. func (d *lvm) thinpoolExists(vgName string, poolName string) (bool, error) { output, err := subprocess.RunCommand("lvs", "--noheadings", "-o", "lv_attr", fmt.Sprintf("%s/%s", vgName, poolName)) if err != nil { if d.isLVMNotFoundExitError(err) { return false, nil } return false, fmt.Errorf("Error checking for LVM thin pool %q: %w", poolName, err) } // Found LV named poolname, check type: attrs := strings.TrimSpace(string(output[:])) if strings.HasPrefix(attrs, "t") { return true, nil } return false, fmt.Errorf("LVM volume named %q exists but is not a thin pool", poolName) } // logicalVolumeExists checks whether the specified logical volume exists. func (d *lvm) logicalVolumeExists(volDevPath string) (bool, error) { _, err := subprocess.RunCommand("lvs", "--noheadings", "-o", "lv_name", volDevPath) if err != nil { if d.isLVMNotFoundExitError(err) { return false, nil } return false, fmt.Errorf("Error checking for LVM logical volume %q: %w", volDevPath, err) } return true, nil } // createDefaultThinPool creates the default thinpool in the pool's volume group. // If thinpoolSizeBytes >0 will manually set the thinpool volume size. Otherwise it will use 100% of the free space // in the volume group. // If pool lvm.thinpool_metadata_size setting >0 will manually set metadata size for the thinpool, otherwise LVM // will pick an appropriate size. func (d *lvm) createDefaultThinPool(lvmVersion, thinPoolName string, thinpoolSizeBytes int64) error { lvmThinPool := fmt.Sprintf("%s/%s", d.config["lvm.vg_name"], thinPoolName) args := []string{ "--yes", "--wipesignatures", "y", "--thinpool", lvmThinPool, } thinpoolMetadataSizeBytes, err := d.roundedSizeBytesString(d.config["lvm.thinpool_metadata_size"]) if err != nil { return fmt.Errorf("Invalid lvm.thinpool_metadata_size: %w", err) } if thinpoolMetadataSizeBytes > 0 { args = append(args, "--poolmetadatasize", fmt.Sprintf("%db", thinpoolMetadataSizeBytes)) } if thinpoolSizeBytes > 0 { args = append(args, "--size", fmt.Sprintf("%db", thinpoolSizeBytes)) } else { args = append(args, "--extents", "100%FREE") } // Because the thin pool is created as an LVM volume, if the volume stripes option is set we need to apply // it to the thin pool volume, as it cannot be applied to the thin volumes themselves. if d.config["volume.lvm.stripes"] != "" { args = append(args, "--stripes", d.config["volume.lvm.stripes"]) if d.config["volume.lvm.stripes.size"] != "" { stripSizeBytes, err := d.roundedSizeBytesString(d.config["volume.lvm.stripes.size"]) if err != nil { return fmt.Errorf("Invalid volume stripe size %q: %w", d.config["volume.lvm.stripes.size"], err) } args = append(args, "--stripesize", fmt.Sprintf("%db", stripSizeBytes)) } } // Create the thin pool volume. _, err = subprocess.TryRunCommand("lvcreate", args...) if err != nil { return fmt.Errorf("Error creating LVM thin pool named %q: %w", thinPoolName, err) } return nil } // lvmVersionIsAtLeast checks whether the installed version of LVM is at least the specific version. func (d *lvm) lvmVersionIsAtLeast(sTypeVersion string, versionString string) (bool, error) { lvmVersionString := strings.Split(sTypeVersion, "/")[0] lvmVersion, err := version.Parse(lvmVersionString) if err != nil { return false, err } inVersion, err := version.Parse(versionString) if err != nil { return false, err } if lvmVersion.Compare(inVersion) < 0 { return false, nil } return true, nil } // roundedSizeString rounds the size to the nearest multiple of 512 bytes as the LVM tools require this. func (d *lvm) roundedSizeBytesString(size string) (int64, error) { sizeBytes, err := units.ParseByteSizeString(size) if err != nil { return 0, err } if sizeBytes <= 0 { return 0, nil } // LVM tools require sizes in multiples of 512 bytes. const minSizeBytes = 512 if sizeBytes < minSizeBytes { sizeBytes = minSizeBytes } // Round the size to closest minSizeBytes bytes. sizeBytes = int64(sizeBytes/minSizeBytes) * minSizeBytes return sizeBytes, nil } // createLogicalVolume creates a logical volume. func (d *lvm) createLogicalVolume(vgName, thinPoolName string, vol Volume, makeThinLv bool) error { var err error lvSizeBytes, err := d.roundedSizeBytesString(vol.ConfigSize()) if err != nil { return err } lvFullName := d.lvmFullVolumeName(vol.volType, vol.contentType, vol.name) logCtx := logger.Ctx{"vg_name": vgName, "lv_name": lvFullName, "size": fmt.Sprintf("%db", lvSizeBytes)} args := []string{ "--name", lvFullName, "--yes", "--wipesignatures", "y", } if makeThinLv { targetVg := fmt.Sprintf("%s/%s", vgName, thinPoolName) args = append(args, "--thin", "--virtualsize", fmt.Sprintf("%db", lvSizeBytes), targetVg, ) } else { args = append(args, "--size", fmt.Sprintf("%db", lvSizeBytes), vgName, ) // As we are creating a normal logical volume we can apply stripes settings if specified. stripes := vol.ExpandedConfig("lvm.stripes") if stripes != "" { args = append(args, "--stripes", stripes) stripeSize := vol.ExpandedConfig("lvm.stripes.size") if stripeSize != "" { stripSizeBytes, err := d.roundedSizeBytesString(stripeSize) if err != nil { return fmt.Errorf("Invalid volume stripe size %q: %w", stripeSize, err) } args = append(args, "--stripesize", fmt.Sprintf("%db", stripSizeBytes)) } } } _, err = subprocess.TryRunCommand("lvcreate", args...) if err != nil { return fmt.Errorf("Error creating LVM logical volume %q: %w", lvFullName, err) } volPath := d.lvmPath(vgName, vol.volType, vol.contentType, vol.name) volDevPath, err := d.lvmDevPath(volPath) if err != nil { return err } if vol.contentType == ContentTypeFS { _, err = makeFSType(volDevPath, vol.ConfigBlockFilesystem(), nil) if err != nil { return fmt.Errorf("Error making filesystem on LVM logical volume: %w", err) } logCtx["fs"] = vol.ConfigBlockFilesystem() } else if !d.usesThinpool() { // Make sure we get an empty LV. err := linux.ClearBlock(volDevPath, 0) if err != nil { return err } } // Disable auto activation of the volume. // Must be done after volume create so that zeroing and signature wiping can take place. _, err = subprocess.TryRunCommand("lvchange", "--setactivationskip", "y", volPath) if err != nil { return fmt.Errorf("Failed to set activation skip on LVM logical volume %q: %w", volPath, err) } d.logger.Debug("Logical volume created", logCtx) return nil } // createLogicalVolumeSnapshot creates a snapshot of a logical volume. func (d *lvm) createLogicalVolumeSnapshot(vgName string, srcVol Volume, snapVol Volume, readonly bool, makeThinLv bool) (string, error) { srcVolPath := d.lvmPath(vgName, srcVol.volType, srcVol.contentType, srcVol.name) snapLvName := d.lvmFullVolumeName(snapVol.volType, snapVol.contentType, snapVol.name) logCtx := logger.Ctx{"vg_name": vgName, "lv_name": snapLvName, "src_dev": srcVolPath, "thin": makeThinLv} args := []string{"-n", snapLvName, "-s", srcVolPath, "--setactivationskip", "y"} // If the source is not a thin volume the size needs to be specified. // Create snapshot at 100% the size of the origin to allow restoring it to the origin volume without // filling up the CoW snapshot volume and causing it to become invalid. if !makeThinLv { args = append(args, "-l", "100%ORIGIN") } if readonly { args = append(args, "-pr") } else { args = append(args, "-prw") } reverter := revert.New() defer reverter.Fail() // If clustered, we need to acquire exclusive write access for this operation. if d.clustered && !makeThinLv { parent, _, _ := api.GetParentAndSnapshotName(srcVol.Name()) parentVol := NewVolume(d, d.Name(), srcVol.volType, srcVol.contentType, parent, srcVol.config, srcVol.poolConfig) release, err := d.acquireExclusive(parentVol) if err != nil { return "", err } defer release() } _, err := subprocess.TryRunCommand("lvcreate", args...) if err != nil { return "", err } d.logger.Debug("Logical volume snapshot created", logCtx) reverter.Add(func() { _ = d.removeLogicalVolume(d.lvmPath(vgName, snapVol.volType, snapVol.contentType, snapVol.name)) }) targetVolPath := d.lvmPath(vgName, snapVol.volType, snapVol.contentType, snapVol.name) reverter.Success() return targetVolPath, nil } // acquireExclusive switches a volume lock to exclusive mode. func (d *lvm) acquireExclusive(vol Volume) (func(), error) { if !d.clustered { return func() {}, nil } volDevPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) lvmActivation.Lock() defer lvmActivation.Unlock() _, err := subprocess.TryRunCommand("lvchange", "--activate", "ey", "--ignoreactivationskip", volDevPath) if err != nil { return nil, fmt.Errorf("Failed to acquire exclusive lock on LVM logical volume %q: %w", volDevPath, err) } return func() { if vol.ContentType() == ContentTypeBlock && vol.ExpandedConfig("block.type") == BlockVolumeTypeQcow2 { _, _ = subprocess.TryRunCommand("lvchange", "--activate", "sy", "--ignoreactivationskip", volDevPath) } }, nil } // removeLogicalVolume removes a logical volume. func (d *lvm) removeLogicalVolume(volDevPath string) error { _, err := subprocess.TryRunCommand("lvremove", "-f", volDevPath) if err != nil { return err } d.logger.Debug("Logical volume removed", logger.Ctx{"dev": volDevPath}) return nil } // renameLogicalVolume renames a logical volume. func (d *lvm) renameLogicalVolume(volDevPath string, newVolDevPath string) error { _, err := subprocess.TryRunCommand("lvrename", volDevPath, newVolDevPath) if err != nil { return err } d.logger.Debug("Logical volume renamed", logger.Ctx{"dev": volDevPath, "new_dev": newVolDevPath}) return nil } // lvmFullVolumeName returns the logical volume's full name with volume type prefix. It also converts the supplied // volName to a name suitable for use as a logical volume using volNameToLVName(). If an empty volType is passed // then just the volName is returned. If an invalid volType is passed then an empty string is returned. // If a content type of ContentTypeBlock is supplied then the volume name is suffixed with lvmBlockVolSuffix. func (d *lvm) lvmFullVolumeName(volType VolumeType, contentType ContentType, volName string) string { if volType == "" { return volName } contentTypeSuffix := "" if contentType == ContentTypeBlock { contentTypeSuffix = lvmBlockVolSuffix } else if contentType == ContentTypeISO { contentTypeSuffix = lvmISOVolSuffix } // Escape the volume name to a name suitable for using as a logical volume. lvName := strings.ReplaceAll(strings.ReplaceAll(volName, "-", lvmEscapedHyphen), internalInstance.SnapshotDelimiter, lvmSnapshotSeparator) return fmt.Sprintf("%s_%s%s", volType, lvName, contentTypeSuffix) } // lvmPath returns the VG/LV path suitable to pass to LVM commands. func (d *lvm) lvmPath(vgName string, volType VolumeType, contentType ContentType, volName string) string { fullVolName := d.lvmFullVolumeName(volType, contentType, volName) if fullVolName == "" { return "" // Invalid volType supplied. } return fmt.Sprintf("%s/%s", vgName, fullVolName) } // lvmDevPath returns the /dev path for the LV. func (d *lvm) lvmDevPath(pathName string) (string, error) { // Get the block dev. output, err := subprocess.TryRunCommand("lvdisplay", "-c", pathName) if err != nil { return "", err } // Grab the major and minor. fields := strings.Split(output, ":") if len(fields) < 2 { return "", errors.New("Bad lvdisplay output") } major := strings.TrimSpace(fields[len(fields)-2]) minor := strings.TrimSpace(fields[len(fields)-1]) if major == "-1" || minor == "-1" { return "", os.ErrNotExist } target, err := os.Readlink(filepath.Join("/sys/dev/block", major+":"+minor)) if err != nil { return "", err } return filepath.Join("/dev", filepath.Base(target)), nil } // resizeLogicalVolume resizes an LVM logical volume. This function does not resize any filesystem inside the LV. func (d *lvm) resizeLogicalVolume(lvPath string, sizeBytes int64) error { isRecent, err := d.lvmVersionIsAtLeast(lvmVersion, "2.03.17") if err != nil { return fmt.Errorf("Error checking LVM version: %w", err) } args := []string{"-L", fmt.Sprintf("%db", sizeBytes), "-f", lvPath} if isRecent { args = append(args, "--fs=ignore") } _, err = subprocess.TryRunCommand("lvresize", args...) if err != nil { return err } d.logger.Debug("Logical volume resized", logger.Ctx{"dev": lvPath, "size": fmt.Sprintf("%db", sizeBytes)}) return nil } // copyThinpoolVolume makes an optimised copy of a thinpool volume by using thinpool snapshots. func (d *lvm) copyThinpoolVolume(vol, srcVol Volume, srcSnapshots []Volume, refresh bool) error { reverter := revert.New() defer reverter.Fail() removeVols := []string{} // If copying snapshots is indicated, check the source isn't itself a snapshot. if len(srcSnapshots) > 0 && !srcVol.IsSnapshot() { // Create the parent snapshot directory. err := CreateParentSnapshotDirIfMissing(d.name, vol.volType, vol.name) if err != nil { return err } for _, srcSnapshot := range srcSnapshots { _, snapName, _ := api.GetParentAndSnapshotName(srcSnapshot.name) newFullSnapName := GetSnapshotVolumeName(vol.name, snapName) newSnapVol := NewVolume(d, d.Name(), vol.volType, vol.contentType, newFullSnapName, vol.config, vol.poolConfig) volExists, err := d.HasVolume(newSnapVol) if err != nil { return err } if volExists { return fmt.Errorf("LVM snapshot volume already exists %q", newSnapVol.name) } newSnapVolPath := newSnapVol.MountPath() err = newSnapVol.EnsureMountPath(false) if err != nil { return err } reverter.Add(func() { _ = os.RemoveAll(newSnapVolPath) }) // We do not modify the original snapshot so as to avoid damaging if it is corrupted for // some reason. If the filesystem needs to have a unique UUID generated in order to mount // this will be done at restore time to be safe. _, err = d.createLogicalVolumeSnapshot(d.config["lvm.vg_name"], srcSnapshot, newSnapVol, true, d.usesThinpool()) if err != nil { return fmt.Errorf("Error creating LVM logical volume snapshot: %w", err) } reverter.Add(func() { _ = d.removeLogicalVolume(d.lvmPath(d.config["lvm.vg_name"], newSnapVol.volType, newSnapVol.contentType, newSnapVol.name)) }) } } // Handle copying the main volume. volExists, err := d.HasVolume(vol) if err != nil { return err } if volExists { if refresh { newVolPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) tmpVolName := fmt.Sprintf("%s%s", vol.name, tmpVolSuffix) tmpVolPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, tmpVolName) // Rename existing volume to temporary new name so we can revert if needed. err := d.renameLogicalVolume(newVolPath, tmpVolPath) if err != nil { return fmt.Errorf("Error temporarily renaming original LVM logical volume: %w", err) } // Record this volume to be removed at the very end. removeVols = append(removeVols, tmpVolName) reverter.Add(func() { // Rename the original volume back to the original name. _ = d.renameLogicalVolume(tmpVolPath, newVolPath) }) } else { return fmt.Errorf("LVM volume already exists %q", vol.name) } } else { volPath := vol.MountPath() err := vol.EnsureMountPath(false) if err != nil { return err } reverter.Add(func() { _ = os.RemoveAll(volPath) }) } // Create snapshot of source volume as new volume. _, err = d.createLogicalVolumeSnapshot(d.config["lvm.vg_name"], srcVol, vol, false, d.usesThinpool()) if err != nil { return fmt.Errorf("Error creating LVM logical volume snapshot: %w", err) } volPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) reverter.Add(func() { _ = d.removeLogicalVolume(volPath) }) if vol.contentType == ContentTypeFS { // Generate a new filesystem UUID if needed (this is required because some filesystems won't allow // volumes with the same UUID to be mounted at the same time). This should be done before volume // resize as some filesystems will need to mount the filesystem to resize. if renegerateFilesystemUUIDNeeded(vol.ConfigBlockFilesystem()) { _, err = d.activateVolume(vol) if err != nil { return err } volDevPath, err := d.lvmDevPath(volPath) if err != nil { return err } d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": volDevPath, "fs": vol.ConfigBlockFilesystem()}) err = regenerateFilesystemUUID(vol.ConfigBlockFilesystem(), volDevPath) if err != nil { return err } } // Mount the volume and ensure the permissions are set correctly inside the mounted volume. err = vol.MountTask(func(_ string, _ *operations.Operation) error { return vol.EnsureMountPath(false) }, nil) if err != nil { return err } } // Resize volume to the size specified. Only uses volume "size" property and does not use pool/defaults // to give the caller more control over the size being used. err = d.SetVolumeQuota(vol, vol.config["size"], false, nil) if err != nil { return err } // Finally clean up original volumes left that were renamed with a tmpVolSuffix suffix. for _, removeVolName := range removeVols { err := d.removeLogicalVolume(d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, removeVolName)) if err != nil { return fmt.Errorf("Error removing LVM volume %q: %w", vol.name, err) } } reverter.Success() return nil } // logicalVolumeSize gets the size in bytes of a logical volume. func (d *lvm) logicalVolumeSize(volDevPath string) (int64, error) { output, err := subprocess.RunCommand("lvs", "--noheadings", "--nosuffix", "--units", "b", "-o", "lv_size", volDevPath) if err != nil { if d.isLVMNotFoundExitError(err) { return -1, api.StatusErrorf(http.StatusNotFound, "LVM volume not found") } return -1, fmt.Errorf("Error getting size of LVM volume %q: %w", volDevPath, err) } output = strings.TrimSpace(output) return strconv.ParseInt(output, 10, 64) } func (d *lvm) thinPoolVolumeUsage(volDevPath string) (uint64, uint64, error) { args := []string{ volDevPath, "--noheadings", "--units", "b", "--nosuffix", "--separator", ",", "-o", "lv_size,data_percent", } out, err := subprocess.RunCommand("lvs", args...) if err != nil { return 0, 0, err } parts := util.SplitNTrimSpace(out, ",", -1, true) if len(parts) < 2 { return 0, 0, errors.New("Unexpected output from lvs command") } total, err := strconv.ParseUint(parts[0], 10, 64) if err != nil { return 0, 0, fmt.Errorf("Failed parsing thin volume total size (%q): %w", parts[0], err) } totalSize := total // Used percentage is not available if thin volume isn't activated. if parts[1] == "" { return 0, 0, ErrNotSupported } dataPerc, err := strconv.ParseFloat(parts[1], 64) if err != nil { return 0, 0, fmt.Errorf("Failed parsing thin volume used percentage (%q): %w", parts[1], err) } usedSize := uint64(float64(total) * (dataPerc / 100)) return totalSize, usedSize, nil } // parseLogicalVolumeSnapshot parses a raw logical volume name (from lvs command) and checks whether it is a // snapshot of the supplied parent volume. Returns unescaped parsed snapshot name if snapshot volume recognised, // empty string if not. The parent is required due to limitations in the naming scheme that Incus has historically // been used for naming logical volumes meaning that additional context of the parent is required to accurately // recognise snapshot volumes that belong to the parent. func (d *lvm) parseLogicalVolumeSnapshot(parent Volume, lvmVolName string) string { fullVolName := d.lvmFullVolumeName(parent.volType, parent.contentType, parent.name) // If block volume, remove the block suffix ready for comparison with LV list. if parent.IsVMBlock() || (parent.volType == VolumeTypeCustom && parent.contentType == ContentTypeBlock) { if !strings.HasSuffix(lvmVolName, lvmBlockVolSuffix) { return "" } // Remove the block suffix so that snapshot names can be compared and extracted without the suffix. fullVolName = strings.TrimSuffix(fullVolName, lvmBlockVolSuffix) lvmVolName = strings.TrimSuffix(lvmVolName, lvmBlockVolSuffix) } // Prefix we would expect for a snapshot of the parent volume. snapPrefix := fmt.Sprintf("%s%s", fullVolName, lvmSnapshotSeparator) // Prefix used when escaping "-" in volume names. Doesn't indicate a snapshot of parent. badPrefix := fmt.Sprintf("%s%s", fullVolName, lvmEscapedHyphen) // Check the volume matches the snapshot prefix, but doesn't match the prefix that indicates a similarly // named volume that just has escaped "-" characters in it. if strings.HasPrefix(lvmVolName, snapPrefix) && !strings.HasPrefix(lvmVolName, badPrefix) { // Remove volume name prefix (including snapshot delimiter) and unescape snapshot name. return strings.ReplaceAll(strings.TrimPrefix(lvmVolName, snapPrefix), lvmEscapedHyphen, "-") } return "" } // activateVolume activates an LVM logical volume if not already present. Returns true if activated, false if not. func (d *lvm) activateVolume(vol Volume) (bool, error) { var volPath string if d.usesThinpool() || IsQcow2Block(vol) { volPath = d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) } else { // Use parent for non-thinpool vols as activating the parent volume also activates its snapshots. parent, _, _ := api.GetParentAndSnapshotName(vol.Name()) volPath = d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, parent) } _, err := d.lvmDevPath(volPath) if err == nil { // Already active. return false, nil } if !errors.Is(err, os.ErrNotExist) { // Actual failure. return false, err } // Activate the volume. lvmActivation.Lock() defer lvmActivation.Unlock() if d.clustered { if vol.Type() == VolumeTypeVM || vol.ContentType() == ContentTypeBlock { _, err := subprocess.RunCommand("lvchange", "--activate", "sy", "--ignoreactivationskip", volPath) if err != nil { return false, fmt.Errorf("Failed to activate LVM logical volume %q: %w", volPath, err) } } else { _, err := subprocess.RunCommand("lvchange", "--activate", "ey", "--ignoreactivationskip", volPath) if err != nil { return false, fmt.Errorf("Failed to activate LVM logical volume %q: %w", volPath, err) } } } else { _, err := subprocess.RunCommand("lvchange", "--activate", "y", "--ignoreactivationskip", volPath) if err != nil { return false, fmt.Errorf("Failed to activate LVM logical volume %q: %w", volPath, err) } } d.logger.Debug("Activated logical volume", logger.Ctx{"volName": vol.Name(), "dev": volPath}) return true, nil } // deactivateVolume deactivates an LVM logical volume if present. Returns true if deactivated, false if not. func (d *lvm) deactivateVolume(vol Volume) (bool, error) { var volPath string if d.usesThinpool() || IsQcow2Block(vol) { volPath = d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) } else { // Use parent for non-thinpool vols as deactivating the parent volume also activates its snapshots. parent, _, _ := api.GetParentAndSnapshotName(vol.Name()) if vol.IsSnapshot() { parentVol := NewVolume(d, d.name, vol.volType, vol.contentType, parent, nil, d.config) // If parent is in use then skip deactivating non-thinpool snapshot volume as it will fail. if parentVol.MountInUse() || (parentVol.contentType == ContentTypeFS && linux.IsMountPoint(parentVol.MountPath())) { return false, nil } } volPath = d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, parent) } _, err := d.lvmDevPath(volPath) if errors.Is(err, os.ErrNotExist) { // Already deactivated. return false, nil } if err != nil { // Actual failure. return false, err } lvmActivation.Lock() defer lvmActivation.Unlock() // Keep trying to deactivate a few times in case the device is still being flushed. _, err = subprocess.TryRunCommand("lvchange", "--activate", "n", "--ignoreactivationskip", volPath) if err != nil { return false, fmt.Errorf("Failed to deactivate LVM logical volume %q: %w", volPath, err) } d.logger.Debug("Deactivated logical volume", logger.Ctx{"volName": vol.Name(), "dev": volPath}) return true, nil } // getSourceType determines the source type based on the source value. func (d *lvm) getSourceType(source string) lvmSourceType { defaultSource := loopFilePath(d.name) if source == "" || source == defaultSource { return lvmSourceTypeDefault } else if filepath.IsAbs(source) { return lvmSourceTypePhysicalDevice } else if source != "" { return lvmSourceTypeVolumeGroup } return lvmSourceTypeUnknown } incus-7.0.0/internal/server/storage/drivers/driver_lvm_utils_test.go000066400000000000000000000050441517523235500260750ustar00rootroot00000000000000package drivers import ( "fmt" ) func Example_lvm_parseLogicalVolumeName() { d := &lvm{} d.name = "pool" type testVol struct { lvName string parent Volume } parentCT := Volume{ contentType: ContentTypeFS, volType: VolumeTypeContainer, name: "proj_testct-with-hyphens", } parentVM := Volume{ contentType: ContentTypeBlock, volType: VolumeTypeVM, name: "proj_testvm-with-hyphens", } custVol := Volume{ contentType: ContentTypeFS, volType: VolumeTypeCustom, name: "proj_testvol-with-hyphens.block", // .block ending doesn't indicate a block vol. } tests := []testVol{ // Test container snapshots. {parent: parentCT, lvName: "containers_proj_testct--with--hyphens"}, {parent: parentCT, lvName: "containers_proj_testct--with--hyphens-snap1--with--hyphens"}, {parent: parentCT, lvName: "containers_proj_testct--with--hyphens-snap1--with--hyphens.block"}, // Test container with name containing snapshot prefix. {parent: parentCT, lvName: "containers_proj_testct--with--hyphens--snap0"}, // Test virtual machine snapshots. {parent: parentVM, lvName: "virtual-machines_proj_testvm--with--hyphens.block"}, {parent: parentVM, lvName: "virtual-machines_proj_testvm--with--hyphens-snap1--with--hyphens.block"}, {parent: parentVM, lvName: "virtual-machines_proj_testvm--with--hyphens-snap1--with--hyphens.block.block"}, // Test custom volume filesystem snapshots. {parent: custVol, lvName: "custom_proj_testvol--with--hyphens.block"}, {parent: custVol, lvName: "custom_proj_testvol--with--hyphens.block-snap1--with--hyphens.block"}, } for _, test := range tests { snapName := d.parseLogicalVolumeSnapshot(test.parent, test.lvName) if snapName == "" { fmt.Printf("%s: Unrecognised\n", test.lvName) } else { fmt.Printf("%s: %s\n", test.lvName, snapName) } } // Output: containers_proj_testct--with--hyphens: Unrecognised // containers_proj_testct--with--hyphens-snap1--with--hyphens: snap1-with-hyphens // containers_proj_testct--with--hyphens-snap1--with--hyphens.block: snap1-with-hyphens.block // containers_proj_testct--with--hyphens--snap0: Unrecognised // virtual-machines_proj_testvm--with--hyphens.block: Unrecognised // virtual-machines_proj_testvm--with--hyphens-snap1--with--hyphens.block: snap1-with-hyphens // virtual-machines_proj_testvm--with--hyphens-snap1--with--hyphens.block.block: snap1-with-hyphens.block // custom_proj_testvol--with--hyphens.block: Unrecognised // custom_proj_testvol--with--hyphens.block-snap1--with--hyphens.block: snap1-with-hyphens.block } incus-7.0.0/internal/server/storage/drivers/driver_lvm_volumes.go000066400000000000000000001715611517523235500254000ustar00rootroot00000000000000package drivers import ( "bufio" "errors" "fmt" "io" "io/fs" "math" "os" "os/exec" "path/filepath" "strings" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/rsync" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // CreateVolume creates an empty volume and can optionally fill it by executing the supplied filler function. func (d *lvm) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Operation) error { reverter := revert.New() defer reverter.Fail() volPath := vol.MountPath() err := vol.EnsureMountPath(true) if err != nil { return err } reverter.Add(func() { _ = os.RemoveAll(volPath) }) err = d.createLogicalVolume(d.config["lvm.vg_name"], d.thinpoolName(), vol, d.usesThinpool()) if err != nil { return fmt.Errorf("Error creating LVM logical volume: %w", err) } reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) // For VMs, also create the filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.CreateVolume(fsVol, nil, op) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolume(fsVol, op) }) } // Format LV as qcow2 (lvmcluster). if IsQcow2Block(vol) { // Get the device path. devPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } qcow2SizeBytes, err := d.roundedSizeBytesString(vol.ConfigSize()) if err != nil { return err } err = Qcow2Create(devPath, "", qcow2SizeBytes) if err != nil { return err } } else if vol.ContentType() == ContentTypeFS && vol.ExpandedConfig("block.type") == BlockVolumeTypeQcow2 { err = Qcow2CreateConfig(vol, op) if err != nil { return err } } err = vol.MountTask(func(mountPath string, op *operations.Operation) error { // Run the volume filler function if supplied. if filler != nil && filler.Fill != nil { var err error var devPath string if IsContentBlock(vol.contentType) { // Get the device path. devPath, err = d.GetVolumeDiskPath(vol) if err != nil { return err } } allowUnsafeResize := false if vol.volType == VolumeTypeImage || !d.usesThinpool() { // Allow filler to resize initial image and non-thin volumes as needed. // Some storage drivers don't normally allow image volumes to be resized due to // them having read-only snapshots that cannot be resized. However when creating // the initial volume and filling it unsafe resizing can be allowed and is required // in order to support unpacking images larger than the default volume size. // The filler function is still expected to obey any volume size restrictions // configured on the pool. // Unsafe resize is also needed to disable filesystem resize safety checks. // This is safe because if for some reason an error occurs the volume will be // discarded rather than leaving a corrupt filesystem. allowUnsafeResize = true } // Run the filler. err = genericRunFiller(d, vol, devPath, filler, allowUnsafeResize) if err != nil { return err } if IsQcow2Block(vol) { qcow2SizeBytes, err := d.roundedSizeBytesString(vol.ConfigSize()) if err != nil { return err } err = Qcow2Resize(devPath, qcow2SizeBytes) if err != nil { return err } } // Move the GPT alt header to end of disk if needed. if vol.IsVMBlock() { err = d.moveGPTAltHeader(devPath) if err != nil { return err } } } if vol.contentType == ContentTypeFS { // Run EnsureMountPath again after mounting and filling to ensure the mount directory has // the correct permissions set. err = vol.EnsureMountPath(true) if err != nil { return err } } return nil }, op) if err != nil { return err } reverter.Success() return nil } // CreateVolumeFromBackup restores a backup tarball onto the storage device. func (d *lvm) CreateVolumeFromBackup(vol Volume, srcBackup backup.Info, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) (VolumePostHook, revert.Hook, error) { return genericVFSBackupUnpack(d, d.state.OS, vol, srcBackup.Snapshots, srcData, basePrefix, op) } // CreateVolumeFromCopy provides same-pool volume copying functionality. func (d *lvm) CreateVolumeFromCopy(vol Volume, srcVol Volume, copySnapshots bool, allowInconsistent bool, op *operations.Operation) error { var err error var srcSnapshots []Volume if copySnapshots && !srcVol.IsSnapshot() { // Get the list of snapshots from the source. srcSnapshots, err = srcVol.Snapshots(op) if err != nil { return err } } // We can use optimised copying when the pool is backed by an LVM thinpool. if d.usesThinpool() { err = d.copyThinpoolVolume(vol, srcVol, srcSnapshots, false) if err != nil { return err } // For VMs, also copy the filesystem volume. if vol.IsVMBlock() { srcFSVol := srcVol.NewVMBlockFilesystemVolume() fsVol := vol.NewVMBlockFilesystemVolume() return d.copyThinpoolVolume(fsVol, srcFSVol, srcSnapshots, false) } return nil } // Otherwise run the generic copy. return genericVFSCopyVolume(d, nil, vol, srcVol, srcSnapshots, false, allowInconsistent, op) } // CreateVolumeFromMigration creates a volume being sent via a migration. func (d *lvm) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, volTargetArgs migration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error { if d.clustered && volTargetArgs.ClusterMoveSourceName != "" && volTargetArgs.StoragePool == "" { err := vol.EnsureMountPath(false) if err != nil { return err } if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.CreateVolumeFromMigration(fsVol, conn, volTargetArgs, preFiller, op) if err != nil { return err } } return nil } return genericVFSCreateVolumeFromMigration(d, nil, vol, conn, volTargetArgs, preFiller, op) } // RefreshVolume provides same-pool volume and specific snapshots syncing functionality. func (d *lvm) RefreshVolume(vol Volume, srcVol Volume, srcSnapshots []Volume, allowInconsistent bool, op *operations.Operation) error { // We can use optimised copying when the pool is backed by an LVM thinpool. if d.usesThinpool() { return d.copyThinpoolVolume(vol, srcVol, srcSnapshots, true) } // Otherwise run the generic copy. return genericVFSCopyVolume(d, nil, vol, srcVol, srcSnapshots, true, allowInconsistent, op) } // DeleteVolume deletes a volume of the storage device. If any snapshots of the volume remain then this function // will return an error. func (d *lvm) DeleteVolume(vol Volume, op *operations.Operation) error { snapshots, err := d.VolumeSnapshots(vol, op) if err != nil { return err } if len(snapshots) > 0 { return errors.New("Cannot remove a volume that has snapshots") } volPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) lvExists, err := d.logicalVolumeExists(volPath) if err != nil { return err } if lvExists { if vol.contentType == ContentTypeFS { _, err = d.UnmountVolume(vol, false, op) if err != nil { return fmt.Errorf("Error unmounting LVM logical volume: %w", err) } } err = d.removeLogicalVolume(d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name)) if err != nil { return fmt.Errorf("Error removing LVM logical volume: %w", err) } } if vol.contentType == ContentTypeFS { // Remove the volume from the storage device. mountPath := vol.MountPath() err = os.RemoveAll(mountPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Error removing LVM logical volume mount path %q: %w", mountPath, err) } // Although the volume snapshot directory should already be removed, lets remove it here to just in // case the top-level directory is left. err = deleteParentSnapshotDirIfEmpty(d.name, vol.volType, vol.name) if err != nil { return err } } // For VMs, also delete the filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.DeleteVolume(fsVol, op) if err != nil { return err } } return nil } // HasVolume indicates whether a specific volume exists on the storage pool. func (d *lvm) HasVolume(vol Volume) (bool, error) { volPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) return d.logicalVolumeExists(volPath) } // FillVolumeConfig populate volume with default config. func (d *lvm) FillVolumeConfig(vol Volume) error { // Copy volume.* configuration options from pool. // Exclude "block.filesystem" and "block.mount_options" as they depend on volume type (handled below). // Exclude "lvm.stripes", "lvm.stripes.size" as they only work on non-thin storage pools (handled below). err := d.fillVolumeConfig(&vol, "block.filesystem", "block.mount_options", "lvm.stripes", "lvm.stripes.size") if err != nil { return err } // Only validate filesystem config keys for filesystem volumes or VM block volumes (which have an // associated filesystem volume). if vol.ContentType() == ContentTypeFS || vol.IsVMBlock() { // Inherit filesystem from pool if not set. if vol.config["block.filesystem"] == "" { vol.config["block.filesystem"] = d.config["volume.block.filesystem"] } // Default filesystem if neither volume nor pool specify an override. if vol.config["block.filesystem"] == "" { // Unchangeable volume property: Set unconditionally. vol.config["block.filesystem"] = DefaultFilesystem } // Inherit filesystem mount options from pool if not set. if vol.config["block.mount_options"] == "" { vol.config["block.mount_options"] = d.config["volume.block.mount_options"] } // Default filesystem mount options if neither volume nor pool specify an override. if vol.config["block.mount_options"] == "" { // Unchangeable volume property: Set unconditionally. vol.config["block.mount_options"] = "discard" } } if d.clustered && (vol.IsVMBlock() || vol.IsCustomBlock()) { // Set default block type to qcow2. if vol.config["block.type"] == "" { vol.config["block.type"] = BlockVolumeTypeQcow2 } // If on qcow2, the block filesystem is btrfs. if vol.config["block.type"] == BlockVolumeTypeQcow2 && vol.IsVMBlock() { vol.config["block.filesystem"] = "btrfs" } } // Inherit stripe settings from pool if not set and not using thin pool. if !d.usesThinpool() { if vol.config["lvm.stripes"] == "" && d.config["volume.lvm.stripes"] != "" { vol.config["lvm.stripes"] = d.config["volume.lvm.stripes"] } if vol.config["lvm.stripes.size"] == "" && d.config["volume.lvm.stripes.size"] != "" { vol.config["lvm.stripes.size"] = d.config["volume.lvm.stripes.size"] } } return nil } // commonVolumeRules returns validation rules which are common for pool and volume. func (d *lvm) commonVolumeRules() map[string]func(value string) error { rules := map[string]func(value string) error{ // gendoc:generate(entity=storage_volume_lvm, group=common, key=block.mount_options) // // --- // type: string // condition: block-based volume with content type `filesystem` // default: same as `volume.block.mount_options` // shortdesc: Mount options for block-backed file system volumes "block.mount_options": validate.IsAny, // gendoc:generate(entity=storage_volume_lvm, group=common, key=block.filesystem) // // --- // type: string // condition: block-based volume with content type `filesystem` // default: same as `volume.block.filesystem` // shortdesc: {{block_filesystem}} "block.filesystem": validate.Optional(validate.IsOneOf(blockBackedAllowedFilesystems...)), // gendoc:generate(entity=storage_volume_lvm, group=common, key=lvm.stripes) // // --- // type: string // condition: - // default: same as `volume.lvm.stripes` // shortdesc: Number of stripes to use for new volumes (or thin pool volume) "lvm.stripes": validate.Optional(validate.IsUint32), // gendoc:generate(entity=storage_volume_lvm, group=common, key=lvm.stripes.size) // // --- // type: string // condition: - // default: same as `volume.lvm.stripes.size` // shortdesc: Size of stripes to use (at least 4096 bytes and multiple of 512 bytes) "lvm.stripes.size": validate.Optional(validate.IsSize), } if d.clustered { // gendoc:generate(entity=storage_lvm, group=common, key=block.type) // // --- // type:string // condition: block-based volume // default: same as `volume.block.type` // shortdesc: Type of the block volume rules["block.type"] = validate.Optional(validate.IsOneOf(BlockVolumeTypeRaw, BlockVolumeTypeQcow2)) // gendoc:generate(entity=storage_volume_lvm, group=common, key=lvmcluster.remove_snapshots) // // --- // type: bool // condition: - // default: same as `volume.lvmcluster.remove_snapshots` or `false` // shortdesc: Remove snapshots as needed rules["lvmcluster.remove_snapshots"] = validate.Optional(validate.IsBool) } return rules } // ValidateVolume validates the supplied volume config. func (d *lvm) ValidateVolume(vol Volume, removeUnknownKeys bool) error { // gendoc:generate(entity=storage_volume_lvm, group=common, key=initial.gid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.gid` or `0` // shortdesc: GID of the volume owner in the instance // gendoc:generate(entity=storage_volume_lvm, group=common, key=initial.mode) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.mode` or `711` // shortdesc: Mode of the volume in the instance // gendoc:generate(entity=storage_volume_lvm, group=common, key=initial.uid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.uid` or `0` // shortdesc: UID of the volume owner in the instance // gendoc:generate(entity=storage_volume_lvm, group=common, key=security.shared) // // --- // type: bool // condition: custom block volume // default: same as `volume.security.shared` or `false` // shortdesc: Enable sharing the volume across multiple instances // gendoc:generate(entity=storage_volume_lvm, group=common, key=security.shifted) // // --- // type: bool // condition: custom volume // default: same as `volume.security.shifted` or `false` // shortdesc: {{enable_ID_shifting}} // gendoc:generate(entity=storage_volume_lvm, group=common, key=security.unmapped) // // --- // type: bool // condition: custom volume // default: same as `volume.security.unmapped` or `false` // shortdesc: Disable ID mapping for the volume // gendoc:generate(entity=storage_volume_lvm, group=common, key=size) // // --- // type: string // condition: // default: same as `volume.size` // shortdesc: Size/quota of the storage volume // gendoc:generate(entity=storage_volume_lvm, group=common, key=snapshots.expiry) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_lvm, group=common, key=snapshots.expiry.manual) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry.manual` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_lvm, group=common, key=snapshots.pattern) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.pattern` or `snap%d` // shortdesc: {{snapshot_pattern_format}} [^*] // gendoc:generate(entity=storage_volume_lvm, group=common, key=snapshots.schedule) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.schedule` // shortdesc: {{snapshot_schedule_format}} // gendoc:generate(entity=storage_bucket_lvm, group=common, key=size) // // --- // type: string // condition: appropriate driver // default: same as `volume.size` // shortdesc: Size/quota of the storage bucket commonRules := d.commonVolumeRules() // Disallow block.* settings for regular custom block volumes. These settings only make sense // when using custom filesystem volumes. Incus will create the filesystem // for these volumes, and use the mount options. When attaching a regular block volume to a VM, // these are not mounted by Incus and therefore don't need these config keys. if vol.IsVMBlock() || vol.volType == VolumeTypeCustom && vol.contentType == ContentTypeBlock { delete(commonRules, "block.filesystem") delete(commonRules, "block.mount_options") } err := d.validateVolume(vol, commonRules, removeUnknownKeys) if err != nil { return err } if d.usesThinpool() && vol.config["lvm.stripes"] != "" { return errors.New("lvm.stripes cannot be used with thin pool volumes") } if d.usesThinpool() && vol.config["lvm.stripes.size"] != "" { return errors.New("lvm.stripes.size cannot be used with thin pool volumes") } if vol.config["block.type"] == BlockVolumeTypeQcow2 && util.IsTrue(vol.config["security.shared"]) { return errors.New("QCOW2 volume type is incompatible with the 'security.shared' option.") } return nil } // UpdateVolume applies config changes to the volume. func (d *lvm) UpdateVolume(vol Volume, changedConfig map[string]string) error { newSize, sizeChanged := changedConfig["size"] if sizeChanged { err := d.SetVolumeQuota(vol, newSize, false, nil) if err != nil { return err } } _, changed := changedConfig["lvm.stripes"] if changed { return errors.New("lvm.stripes cannot be changed") } _, changed = changedConfig["lvm.stripes.size"] if changed { return errors.New("lvm.stripes.size cannot be changed") } _, changed = changedConfig["block.type"] if changed { return errors.New("block.type cannot be changed after creation") } return d.updateVolume(vol, changedConfig) } // GetVolumeUsage returns the disk space used by the volume (this is not currently supported). func (d *lvm) GetVolumeUsage(vol Volume) (int64, error) { // Snapshot usage not supported for LVM. if vol.IsSnapshot() { return -1, ErrNotSupported } // For non-snapshot filesystem volumes, we only return usage when the volume is mounted. // This is because to get an accurate value we cannot use blocks allocated, as the filesystem will likely // consume blocks and not free them when files are deleted in the volume. This avoids returning different // values depending on whether the volume is mounted or not. if vol.contentType == ContentTypeFS && linux.IsMountPoint(vol.MountPath()) { var stat unix.Statfs_t err := unix.Statfs(vol.MountPath(), &stat) if err != nil { return -1, err } return int64(stat.Blocks-stat.Bfree) * int64(stat.Bsize), nil } else if vol.contentType == ContentTypeBlock && d.usesThinpool() { // For non-snapshot thin pool block volumes we can calculate an approximate usage using the space // allocated to the volume from the thin pool. volPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) _, usedSize, err := d.thinPoolVolumeUsage(volPath) if err != nil { return -1, err } return int64(usedSize), nil } return -1, ErrNotSupported } // SetVolumeQuota applies a size limit on volume. // Does nothing if supplied with an empty/zero size. func (d *lvm) SetVolumeQuota(vol Volume, size string, allowUnsafeResize bool, op *operations.Operation) error { // Do nothing if size isn't specified. if size == "" || size == "0" { return nil } sizeBytes, err := d.roundedSizeBytesString(size) if err != nil { return err } // Read actual size of current volume. volPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) oldSizeBytes, err := d.logicalVolumeSize(volPath) if err != nil { return err } // Get the volume group's physical extent size, as we use this to figure out if the new and old sizes are // going to change beyond 1 extent size, otherwise there is no point in trying to resize as LVM do it. vgExtentSize, err := d.volumeGroupExtentSize(d.config["lvm.vg_name"]) if err != nil { return err } // Round up the number of extents required for new quota size, as this is what the lvresize tool will do. newNumExtents := math.Ceil(float64(sizeBytes) / float64(vgExtentSize)) oldNumExtents := math.Ceil(float64(oldSizeBytes) / float64(vgExtentSize)) extentDiff := int(newNumExtents - oldNumExtents) // If old and new extents required are the same, nothing to do, as LVM won't resize them. if extentDiff == 0 { return nil } l := d.logger.AddContext(logger.Ctx{"dev": volPath, "size": fmt.Sprintf("%db", sizeBytes)}) inUse := vol.MountInUse() // Resize filesystem if needed. if vol.contentType == ContentTypeFS { fsType := vol.ConfigBlockFilesystem() if sizeBytes < oldSizeBytes { if !filesystemTypeCanBeShrunk(fsType) { return fmt.Errorf("Filesystem %q cannot be shrunk: %w", fsType, ErrCannotBeShrunk) } if inUse { return ErrInUse // We don't allow online shrinking of filesystem volumes. } // Activate volume if needed. activated, err := d.activateVolume(vol) if err != nil { return err } if !activated { defer func() { _, _ = d.activateVolume(vol) }() } // Shrink filesystem first. // Pass allowUnsafeResize to allow disabling of filesystem resize safety checks. // We do this as a separate step rather than passing -r to lvresize in resizeLogicalVolume // so that we can have more control over when we trigger unsafe filesystem resize mode, // otherwise by passing -f to lvresize (required for other reasons) this would then pass // -f onto resize2fs as well. volDevPath, err := d.lvmDevPath(volPath) if err != nil { return err } err = shrinkFileSystem(fsType, volDevPath, vol, sizeBytes, allowUnsafeResize) if err != nil { _, _ = d.deactivateVolume(vol) return err } // Deactivate the volume for resizing. _, err = d.deactivateVolume(vol) if err != nil { return err } l.Debug("Logical volume filesystem shrunk") // Shrink the block device. err = d.resizeLogicalVolume(volPath, sizeBytes) if err != nil { return err } } else if sizeBytes > oldSizeBytes { // Get exclusive mode if active. release, err := d.acquireExclusive(vol) if err != nil { return err } defer release() // Grow block device first. err = d.resizeLogicalVolume(volPath, sizeBytes) if err != nil { return err } // Activate the volume for resizing. activated, err := d.activateVolume(vol) if err != nil { return err } if activated { defer func() { _, _ = d.deactivateVolume(vol) }() } // Grow the filesystem to fill block device. volDevPath, err := d.lvmDevPath(volPath) if err != nil { return err } err = growFileSystem(fsType, volDevPath, vol) if err != nil { return err } l.Debug("Logical volume filesystem grown") } } else { // Only perform pre-resize checks if we are not in "unsafe" mode. // In unsafe mode we expect the caller to know what they are doing and understand the risks. if !allowUnsafeResize { if sizeBytes < oldSizeBytes { return fmt.Errorf("Block volumes cannot be shrunk: %w", ErrCannotBeShrunk) } } // Get exclusive mode. release, err := d.acquireExclusive(vol) if err != nil { return err } defer release() // Resize the block device. err = d.resizeLogicalVolume(volPath, sizeBytes) if err != nil { return err } // On thick pools, discard the blocks in the additional space when the volume is grown. if !d.usesThinpool() && oldSizeBytes < sizeBytes { // Activate the volume for discarding. activated, err := d.activateVolume(vol) if err != nil { return err } if activated { defer func() { _, _ = d.deactivateVolume(vol) }() } // Discard the new blocks. volDevPath, err := d.lvmDevPath(volPath) if err != nil { return err } err = linux.ClearBlock(volDevPath, oldSizeBytes) if err != nil { return err } } // Move the VM GPT alt header to end of disk if needed (not needed in unsafe resize mode as it is // expected the caller will do all necessary post resize actions themselves). if vol.IsVMBlock() && !allowUnsafeResize { // Activate the volume for resizing. activated, err := d.activateVolume(vol) if err != nil { return err } if activated { defer func() { _, _ = d.deactivateVolume(vol) }() } // Move the GPT alt header. volDevPath, err := d.lvmDevPath(volPath) if err != nil { return err } err = d.moveGPTAltHeader(volDevPath) if err != nil { return err } } } return nil } // GetVolumeDiskPath returns the location of a disk volume. func (d *lvm) GetVolumeDiskPath(vol Volume) (string, error) { if vol.IsVMBlock() || (vol.volType == VolumeTypeCustom && IsContentBlock(vol.contentType)) { return d.lvmDevPath(d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name)) } return "", ErrNotSupported } // ListVolumes returns a list of volumes in storage pool. func (d *lvm) ListVolumes() ([]Volume, error) { vols := make(map[string]Volume) cmd := exec.Command("lvs", "--noheadings", "-o", "lv_name", d.config["lvm.vg_name"]) stdout, err := cmd.StdoutPipe() if err != nil { return nil, err } stderr, err := cmd.StderrPipe() if err != nil { return nil, err } err = cmd.Start() if err != nil { return nil, err } scanner := bufio.NewScanner(stdout) for scanner.Scan() { rawName := strings.TrimSpace(scanner.Text()) var volType VolumeType var volName string for _, volumeType := range d.Info().VolumeTypes { prefix := fmt.Sprintf("%s_", volumeType) if strings.HasPrefix(rawName, prefix) { volType = volumeType volName = strings.TrimPrefix(rawName, prefix) } } if volType == "" { d.logger.Debug("Ignoring unrecognised volume type", logger.Ctx{"name": rawName}) continue // Ignore unrecognised volume. } lvSnapSepCount := strings.Count(volName, lvmSnapshotSeparator) if lvSnapSepCount%2 != 0 { // If snapshot separator count is odd, then this means we have a lone lvmSnapshotSeparator // that is not part of the lvmEscapedHyphen pair, which means this volume is a snapshot. d.logger.Debug("Ignoring snapshot volume", logger.Ctx{"name": rawName}) continue // Ignore snapshot volumes. } isBlock := strings.HasSuffix(volName, lvmBlockVolSuffix) if volType == VolumeTypeVM && !isBlock { continue // Ignore VM filesystem volumes as we will just return the VM's block volume. } // Unescape raw LVM name to storage volume name. Safe to do now we know we are not dealing // with snapshot volumes. volName = strings.ReplaceAll(volName, lvmEscapedHyphen, "-") contentType := ContentTypeFS if volType == VolumeTypeCustom && strings.HasSuffix(volName, lvmISOVolSuffix) { contentType = ContentTypeISO volName = strings.TrimSuffix(volName, lvmISOVolSuffix) } else if volType == VolumeTypeVM || isBlock { contentType = ContentTypeBlock volName = strings.TrimSuffix(volName, lvmBlockVolSuffix) } // If a new volume has been found, or the volume will replace an existing image filesystem volume // then proceed to add the volume to the map. We allow image volumes to overwrite existing // filesystem volumes of the same name so that for VM images we only return the block content type // volume (so that only the single "logical" volume is returned). existingVol, foundExisting := vols[volName] if !foundExisting || (existingVol.Type() == VolumeTypeImage && existingVol.ContentType() == ContentTypeFS) { v := NewVolume(d, d.name, volType, contentType, volName, make(map[string]string), d.config) if contentType == ContentTypeFS { v.SetMountFilesystemProbe(true) } vols[volName] = v continue } return nil, fmt.Errorf("Unexpected duplicate volume %q found", volName) } errMsg, err := io.ReadAll(stderr) if err != nil { return nil, err } err = cmd.Wait() if err != nil { return nil, fmt.Errorf("Failed getting volume list: %v: %w", strings.TrimSpace(string(errMsg)), err) } volList := make([]Volume, 0, len(vols)) for _, v := range vols { volList = append(volList, v) } return volList, nil } // ActivateTask allows running a function while the volume is active (but not mounted). func (d *lvm) ActivateTask(vol Volume, task func(devPath string, op *operations.Operation) error, op *operations.Operation) error { // Prevent concurrent mounting actions. unlock, err := vol.MountLock() if err != nil { return err } defer unlock() // Setup a reverter. reverter := revert.New() defer reverter.Fail() // Activate the volume. activated, err := d.activateVolume(vol) if err != nil { return err } if !activated { return errors.New("Volume is already active, can't run exclusive activation task") } // Get the device path. volDevPath, err := d.lvmDevPath(d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name)) if err != nil { return err } // Run the task. taskErr := task(volDevPath, op) // Deactivate the volume. _, err = d.deactivateVolume(vol) if err != nil { return err } return taskErr } // MountVolume mounts a volume and increments ref counter. Please call UnmountVolume() when done with the volume. func (d *lvm) MountVolume(vol Volume, op *operations.Operation) error { unlock, err := vol.MountLock() if err != nil { return err } defer unlock() reverter := revert.New() defer reverter.Fail() // Activate LVM volume if needed. activated, err := d.activateVolume(vol) if err != nil { return err } if activated { reverter.Add(func() { _, _ = d.deactivateVolume(vol) }) } if vol.contentType == ContentTypeFS { // Check if already mounted. mountPath := vol.MountPath() if !linux.IsMountPoint(mountPath) { fsType := vol.ConfigBlockFilesystem() volDevPath, err := d.lvmDevPath(d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name)) if err != nil { return err } if vol.mountFilesystemProbe { fsType, err = fsProbe(volDevPath) if err != nil { return fmt.Errorf("Failed probing filesystem: %w", err) } } err = vol.EnsureMountPath(false) if err != nil { return err } mountFlags, mountOptions := linux.ResolveMountOptions(strings.Split(vol.ConfigBlockMountOptions(), ",")) err = TryMount(volDevPath, mountPath, fsType, mountFlags, mountOptions) if err != nil { return fmt.Errorf("Failed to mount LVM logical volume: %w", err) } d.logger.Debug("Mounted logical volume", logger.Ctx{"volName": vol.name, "dev": volDevPath, "path": mountPath, "options": mountOptions}) } } else if vol.contentType == ContentTypeBlock || vol.contentType == ContentTypeISO { // For VMs, mount the filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err = d.MountVolume(fsVol, op) if err != nil { return err } } } vol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolume() when done. reverter.Success() return nil } // UnmountVolume unmounts volume if mounted and not in use. Returns true if this unmounted the volume. // keepBlockDev indicates if backing block device should be not be deactivated when volume is unmounted. func (d *lvm) UnmountVolume(vol Volume, keepBlockDev bool, op *operations.Operation) (bool, error) { unlock, err := vol.MountLock() if err != nil { return false, err } defer unlock() ourUnmount := false mountPath := vol.MountPath() refCount := vol.MountRefCountDecrement() // Check if already mounted. if vol.contentType == ContentTypeFS && linux.IsMountPoint(mountPath) { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": vol.name, "refCount": refCount}) return false, ErrInUse } err = TryUnmount(mountPath, 0) if err != nil { return false, fmt.Errorf("Failed to unmount LVM logical volume: %w", err) } d.logger.Debug("Unmounted logical volume", logger.Ctx{"volName": vol.name, "path": mountPath, "keepBlockDev": keepBlockDev}) // We only deactivate filesystem volumes if an unmount was needed to better align with our // unmount return value indicator. if !keepBlockDev { _, err = d.deactivateVolume(vol) if err != nil { return false, err } } ourUnmount = true } else if IsContentBlock(vol.contentType) { // For VMs and ISOs, unmount the filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() ourUnmount, err = d.UnmountVolume(fsVol, false, op) if err != nil { return false, err } } volDevPath, err := d.lvmDevPath(d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name)) if err != nil && !errors.Is(err, os.ErrNotExist) { return false, err } if !keepBlockDev && volDevPath != "" { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": vol.name, "refCount": refCount}) return false, ErrInUse } _, err = d.deactivateVolume(vol) if err != nil { return false, err } ourUnmount = true } } return ourUnmount, nil } // RenameVolume renames a volume and its snapshots. func (d *lvm) RenameVolume(vol Volume, newVolName string, op *operations.Operation) error { volPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) return vol.UnmountTask(func(op *operations.Operation) error { snapNames, err := d.VolumeSnapshots(vol, op) if err != nil { return err } reverter := revert.New() defer reverter.Fail() // Rename snapshots (change volume prefix to use new parent volume name). for _, snapName := range snapNames { snapVolName := GetSnapshotVolumeName(vol.name, snapName) snapVolPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, snapVolName) newSnapVolName := GetSnapshotVolumeName(newVolName, snapName) newSnapVolPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, newSnapVolName) snapVol, err := vol.NewSnapshot(snapName) if err != nil { return err } releaseSnap, _ := d.acquireExclusive(snapVol) err = d.renameLogicalVolume(snapVolPath, newSnapVolPath) if err != nil { return err } releaseSnap() reverter.Add(func() { _ = d.renameLogicalVolume(newSnapVolPath, snapVolPath) }) } // Rename snapshots dir if present. if vol.contentType == ContentTypeFS { srcSnapshotDir := GetVolumeSnapshotDir(d.name, vol.volType, vol.name) dstSnapshotDir := GetVolumeSnapshotDir(d.name, vol.volType, newVolName) if util.PathExists(srcSnapshotDir) { err = os.Rename(srcSnapshotDir, dstSnapshotDir) if err != nil { return fmt.Errorf("Error renaming LVM logical volume snapshot directory from %q to %q: %w", srcSnapshotDir, dstSnapshotDir, err) } reverter.Add(func() { _ = os.Rename(dstSnapshotDir, srcSnapshotDir) }) } } // Rename actual volume. newVolPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, newVolName) err = d.renameLogicalVolume(volPath, newVolPath) if err != nil { return err } reverter.Add(func() { _ = d.renameLogicalVolume(newVolPath, volPath) }) // Rename volume dir. if vol.contentType == ContentTypeFS { srcVolumePath := GetVolumeMountPath(d.name, vol.volType, vol.name) dstVolumePath := GetVolumeMountPath(d.name, vol.volType, newVolName) err = os.Rename(srcVolumePath, dstVolumePath) if err != nil { return fmt.Errorf("Error renaming LVM logical volume mount path from %q to %q: %w", srcVolumePath, dstVolumePath, err) } reverter.Add(func() { _ = os.Rename(dstVolumePath, srcVolumePath) }) } // For VMs, also rename the filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err = d.RenameVolume(fsVol, newVolName, op) if err != nil { return err } } reverter.Success() return nil }, false, op) } // MigrateVolume sends a volume for migration. func (d *lvm) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *migration.VolumeSourceArgs, op *operations.Operation) error { if d.clustered && volSrcArgs.ClusterMove && !volSrcArgs.StorageMove { // Ensure the volume allows shared access. if vol.volType == VolumeTypeVM || vol.IsCustomBlock() { lvmActivation.Lock() defer lvmActivation.Unlock() // Block volume. volPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.Name()) volDevPath, err := d.lvmDevPath(volPath) if err != nil && !errors.Is(err, os.ErrNotExist) { return err } if volDevPath != "" { if vol.Type() == VolumeTypeVM || vol.ContentType() == ContentTypeBlock || vol.ContentType() == ContentTypeISO { _, err := subprocess.RunCommand("lvchange", "--activate", "sy", "--ignoreactivationskip", volPath) if err != nil { return err } } else { _, err := subprocess.RunCommand("lvchange", "--activate", "ey", "--ignoreactivationskip", volPath) if err != nil { return err } } } // Filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() volPath := d.lvmPath(d.config["lvm.vg_name"], fsVol.volType, fsVol.contentType, fsVol.Name()) volDevPath, err := d.lvmDevPath(volPath) if err != nil && !errors.Is(err, os.ErrNotExist) { return err } if volDevPath != "" { _, err := subprocess.RunCommand("lvchange", "--activate", "sy", "--ignoreactivationskip", volPath) if err != nil { return err } } } } return nil // When performing a cluster member move don't do anything on the source member. } return genericVFSMigrateVolume(d, d.state, vol, conn, volSrcArgs, op) } // BackupVolume copies a volume (and optionally its snapshots) to a specified target path. // This driver does not support optimized backups. func (d *lvm) BackupVolume(vol Volume, writer instancewriter.InstanceWriter, basePrefix string, _ bool, snapshots []string, op *operations.Operation) error { return genericVFSBackupVolume(d, vol, writer, basePrefix, snapshots, op) } // CreateVolumeSnapshot creates a snapshot of a volume. func (d *lvm) CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error { reverter := revert.New() defer reverter.Fail() parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) parentVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, snapVol.config, snapVol.poolConfig) snapPath := snapVol.MountPath() if d.isRemote() && snapVol.ContentType() == ContentTypeBlock { if util.IsTrue(snapVol.ExpandedConfig("security.shared")) { return fmt.Errorf(`Snapshots of shared custom storage volumes aren't supported on "lvmcluster"`) } if snapVol.ExpandedConfig("block.type") != BlockVolumeTypeQcow2 { return fmt.Errorf(`Snapshots of raw block volumes aren't supported on "lvmcluster"`) } parentVolPath := d.lvmPath(d.config["lvm.vg_name"], parentVol.volType, parentVol.contentType, parentName) snapVolPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, snapVol.contentType, snapVol.name) releaseParent, err := d.acquireExclusive(parentVol) if err != nil { return err } defer releaseParent() err = d.renameLogicalVolume(parentVolPath, snapVolPath) if err != nil { return fmt.Errorf("Error temporarily renaming original LVM logical volume: %w", err) } reverter.Add(func() { _, _ = d.acquireExclusive(snapVol) _ = d.renameLogicalVolume(snapVolPath, parentVolPath) // acquireExclusive is called on the parent volume to obtain a release function // that switches the volume back to shared mode. releaseParent, _ := d.acquireExclusive(parentVol) releaseParent() }) releaseSnap, err := d.acquireExclusive(snapVol) if err != nil { return err } defer releaseSnap() err = d.createLogicalVolume(d.config["lvm.vg_name"], d.thinpoolName(), parentVol, d.usesThinpool()) if err != nil { return fmt.Errorf("Error creating LVM logical volume: %w", err) } reverter.Add(func() { releaseParent, _ := d.acquireExclusive(parentVol) _ = d.removeLogicalVolume(parentVolPath) releaseParent() }) reverter.Success() return nil } // Create the parent directory. err := CreateParentSnapshotDirIfMissing(d.name, snapVol.volType, parentName) if err != nil { return err } // Create snapshot directory. err = snapVol.EnsureMountPath(false) if err != nil { return err } reverter.Add(func() { _ = os.RemoveAll(snapPath) }) _, err = d.createLogicalVolumeSnapshot(d.config["lvm.vg_name"], parentVol, snapVol, true, d.usesThinpool()) if err != nil { return fmt.Errorf("Error creating LVM logical volume snapshot: %w", err) } volPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, snapVol.contentType, snapVol.name) reverter.Add(func() { _ = d.removeLogicalVolume(volPath) }) // For VMs, also snapshot the filesystem. if snapVol.IsVMBlock() { parentFSVol := parentVol.NewVMBlockFilesystemVolume() fsVol := snapVol.NewVMBlockFilesystemVolume() _, err = d.createLogicalVolumeSnapshot(d.config["lvm.vg_name"], parentFSVol, fsVol, true, d.usesThinpool()) if err != nil { return fmt.Errorf("Error creating LVM logical volume snapshot: %w", err) } } reverter.Success() return nil } // DeleteVolumeSnapshot removes a snapshot from the storage device. The volName and snapshotName // must be bare names and should not be in the format "volume/snapshot". func (d *lvm) DeleteVolumeSnapshot(snapVol Volume, op *operations.Operation) error { // Remove the snapshot from the storage device. volPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, snapVol.contentType, snapVol.name) lvExists, err := d.logicalVolumeExists(volPath) if err != nil { return err } if lvExists { _, err = d.UnmountVolume(snapVol, false, op) if err != nil { return fmt.Errorf("Error unmounting LVM logical volume: %w", err) } err = d.removeLogicalVolume(d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, snapVol.contentType, snapVol.name)) if err != nil { return fmt.Errorf("Error removing LVM logical volume: %w", err) } } // For VMs, also remove the snapshot filesystem volume. if snapVol.IsVMBlock() { fsVol := snapVol.NewVMBlockFilesystemVolume() err = d.DeleteVolumeSnapshot(fsVol, op) if err != nil { return err } } // Remove the snapshot mount path from the storage device. snapPath := snapVol.MountPath() err = os.RemoveAll(snapPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Error removing LVM snapshot mount path %q: %w", snapPath, err) } // Remove the parent snapshot directory if this is the last snapshot being removed. parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) err = deleteParentSnapshotDirIfEmpty(d.name, snapVol.volType, parentName) if err != nil { return err } return nil } // MountVolumeSnapshot sets up a read-only mount on top of the snapshot to avoid accidental modifications. func (d *lvm) MountVolumeSnapshot(snapVol Volume, op *operations.Operation) error { unlock, err := snapVol.MountLock() if err != nil { return err } defer unlock() reverter := revert.New() defer reverter.Fail() mountPath := snapVol.MountPath() // Check if already mounted. if snapVol.contentType == ContentTypeFS && !linux.IsMountPoint(mountPath) { err = snapVol.EnsureMountPath(false) if err != nil { return err } // Default to mounting the original snapshot directly. This may be changed below if a temporary // snapshot needs to be taken. mountVol := snapVol mountFlags, mountOptions := linux.ResolveMountOptions(strings.Split(mountVol.ConfigBlockMountOptions(), ",")) isQcow2 := snapVol.ExpandedConfig("block.type") == BlockVolumeTypeQcow2 if isQcow2 { parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) mountVol = NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, snapVol.config, snapVol.poolConfig) // Activate volume if needed. _, err = d.activateVolume(mountVol) if err != nil { return err } // Get volume path. volPath := d.lvmPath(d.config["lvm.vg_name"], mountVol.volType, mountVol.contentType, mountVol.name) volDevPath, err := d.lvmDevPath(volPath) if err != nil { return err } // Finally attempt to mount the volume that needs mounting. err = TryMount(volDevPath, mountPath, mountVol.ConfigBlockFilesystem(), mountFlags|unix.MS_RDONLY, mountOptions) if err != nil { return fmt.Errorf("Failed to mount LVM snapshot volume: %w", err) } d.logger.Debug("Mounted logical volume snapshot", logger.Ctx{"dev": volPath, "path": mountPath, "options": mountOptions}) } else { // Regenerate filesystem UUID if needed. This is because some filesystems do not allow mounting // multiple volumes that share the same UUID. As snapshotting a volume will copy its UUID we need // to potentially regenerate the UUID of the snapshot now that we are trying to mount it. // This is done at mount time rather than snapshot time for 2 reasons; firstly snapshots need to be // as fast as possible, and on some filesystems regenerating the UUID is a slow process, secondly // we do not want to modify a snapshot in case it is corrupted for some reason, so at mount time // we take another snapshot of the snapshot, regenerate the temporary snapshot's UUID and then // mount that. regenerateFSUUID := renegerateFilesystemUUIDNeeded(snapVol.ConfigBlockFilesystem()) if regenerateFSUUID { // Instantiate a new volume to be the temporary writable snapshot. tmpVolName := fmt.Sprintf("%s%s", snapVol.name, tmpVolSuffix) tmpVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, tmpVolName, snapVol.config, snapVol.poolConfig) // Create writable snapshot from source snapshot named with a tmpVolSuffix suffix. _, err = d.createLogicalVolumeSnapshot(d.config["lvm.vg_name"], snapVol, tmpVol, false, d.usesThinpool()) if err != nil { return fmt.Errorf("Error creating temporary LVM logical volume snapshot: %w", err) } reverter.Add(func() { _ = d.removeLogicalVolume(d.lvmPath(d.config["lvm.vg_name"], tmpVol.volType, tmpVol.contentType, tmpVol.name)) }) // We are going to mount the temporary volume instead. mountVol = tmpVol } // Activate volume if needed. _, err = d.activateVolume(mountVol) if err != nil { return err } // Get volume path. volPath := d.lvmPath(d.config["lvm.vg_name"], mountVol.volType, mountVol.contentType, mountVol.name) volDevPath, err := d.lvmDevPath(volPath) if err != nil { return err } if regenerateFSUUID { tmpVolFsType := mountVol.ConfigBlockFilesystem() // When mounting XFS filesystems temporarily we can use the nouuid option rather than fully // regenerating the filesystem UUID. if tmpVolFsType == "xfs" { idx := strings.Index(mountOptions, "nouuid") if idx < 0 { mountOptions += ",nouuid" } } else { d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": volPath, "fs": tmpVolFsType}) err = regenerateFilesystemUUID(mountVol.ConfigBlockFilesystem(), volDevPath) if err != nil { return err } } } // Finally attempt to mount the volume that needs mounting. err = TryMount(volDevPath, mountPath, mountVol.ConfigBlockFilesystem(), mountFlags|unix.MS_RDONLY, mountOptions) if err != nil { return fmt.Errorf("Failed to mount LVM snapshot volume: %w", err) } d.logger.Debug("Mounted logical volume snapshot", logger.Ctx{"dev": volPath, "path": mountPath, "options": mountOptions}) } } else if snapVol.contentType == ContentTypeBlock { // Activate volume if needed. _, err = d.activateVolume(snapVol) if err != nil { return err } // For VMs, mount the filesystem volume. if snapVol.IsVMBlock() { fsVol := snapVol.NewVMBlockFilesystemVolume() err = d.MountVolumeSnapshot(fsVol, op) if err != nil { return err } } } snapVol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolumeSnapshot() when done. reverter.Success() return nil } // UnmountVolumeSnapshot removes the read-only mount placed on top of a snapshot. // If a temporary snapshot volume exists then it will attempt to remove it. func (d *lvm) UnmountVolumeSnapshot(snapVol Volume, op *operations.Operation) (bool, error) { unlock, err := snapVol.MountLock() if err != nil { return false, err } defer unlock() ourUnmount := false mountPath := snapVol.MountPath() refCount := snapVol.MountRefCountDecrement() // Check if already mounted. if snapVol.contentType == ContentTypeFS && linux.IsMountPoint(mountPath) { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": snapVol.name, "refCount": refCount}) return false, ErrInUse } err = TryUnmount(mountPath, 0) if err != nil { return false, fmt.Errorf("Failed to unmount LVM snapshot volume: %w", err) } d.logger.Debug("Unmounted logical volume snapshot", logger.Ctx{"path": mountPath}) // Check if a temporary snapshot exists, and if so remove it. tmpVolName := fmt.Sprintf("%s%s", snapVol.name, tmpVolSuffix) tmpVolPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, snapVol.contentType, tmpVolName) exists, err := d.logicalVolumeExists(tmpVolPath) if err != nil { return true, fmt.Errorf("Failed to check existence of temporary LVM snapshot volume %q: %w", tmpVolPath, err) } if exists { err = d.removeLogicalVolume(tmpVolPath) if err != nil { return true, fmt.Errorf("Failed to remove temporary LVM snapshot volume %q: %w", tmpVolPath, err) } } // We only deactivate filesystem volumes if an unmount was needed to better align with our // unmount return value indicator. _, err = d.deactivateVolume(snapVol) if err != nil { return false, err } ourUnmount = true } else if snapVol.contentType == ContentTypeBlock { // For VMs, unmount the filesystem volume. if snapVol.IsVMBlock() { fsVol := snapVol.NewVMBlockFilesystemVolume() ourUnmount, err = d.UnmountVolumeSnapshot(fsVol, op) if err != nil { return false, err } } volPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, snapVol.contentType, snapVol.name) volDevPath, err := d.lvmDevPath(volPath) if err != nil && !errors.Is(err, os.ErrNotExist) { return false, err } if volDevPath != "" { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": snapVol.name, "refCount": refCount}) return false, ErrInUse } _, err = d.deactivateVolume(snapVol) if err != nil { return false, err } ourUnmount = true } } return ourUnmount, nil } // VolumeSnapshots returns a list of snapshots for the volume (in no particular order). func (d *lvm) VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) { // We use the volume list rather than inspecting the logical volumes themselves because the origin // property of an LVM snapshot can be removed/changed when restoring snapshots, such that they are no // marked as origin of the parent volume. Instead we use prefix matching on the volume names to find the // snapshot volumes. cmd := exec.Command("lvs", "--noheadings", "-o", "lv_name", d.config["lvm.vg_name"]) stdout, err := cmd.StdoutPipe() if err != nil { return nil, err } stderr, err := cmd.StderrPipe() if err != nil { return nil, err } err = cmd.Start() if err != nil { return nil, err } snapshots := []string{} scanner := bufio.NewScanner(stdout) for scanner.Scan() { snapName := d.parseLogicalVolumeSnapshot(vol, strings.TrimSpace(scanner.Text())) if snapName == "" { continue // Skip logical volumes that are not recognised as a snapshot of our parent vol. } snapshots = append(snapshots, snapName) } errMsg, err := io.ReadAll(stderr) if err != nil { return nil, err } err = cmd.Wait() if err != nil { return nil, fmt.Errorf("Failed to get snapshot list for volume %q: %v: %w", vol.name, strings.TrimSpace(string(errMsg)), err) } return snapshots, nil } // RestoreVolume restores a volume from a snapshot. func (d *lvm) RestoreVolume(vol Volume, snapshotName string, op *operations.Operation) error { // Instantiate snapshot volume from snapshot name. snapVol, err := vol.NewSnapshot(snapshotName) if err != nil { return err } reverter := revert.New() defer reverter.Fail() // If the pool uses thinpools, then the process for restoring a snapshot is as follows: // 1. Rename the original volume to a temporary name (so we can revert later if needed). // 2. Create a writable snapshot with the original name from the snapshot being restored. // 3. Delete the renamed original volume. if d.usesThinpool() { _, err = d.UnmountVolume(vol, false, op) if err != nil { return fmt.Errorf("Error unmounting LVM logical volume: %w", err) } originalVolPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) tmpVolName := fmt.Sprintf("%s%s", vol.name, tmpVolSuffix) tmpVolPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, tmpVolName) // Rename original logical volume to temporary new name so we can revert if needed. err = d.renameLogicalVolume(originalVolPath, tmpVolPath) if err != nil { return fmt.Errorf("Error temporarily renaming original LVM logical volume: %w", err) } reverter.Add(func() { // Rename the original volume back to the original name. _ = d.renameLogicalVolume(tmpVolPath, originalVolPath) }) // Create writable snapshot from source snapshot named as target volume. _, err = d.createLogicalVolumeSnapshot(d.config["lvm.vg_name"], snapVol, vol, false, true) if err != nil { return fmt.Errorf("Error restoring LVM logical volume snapshot: %w", err) } volPath := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) reverter.Add(func() { _ = d.removeLogicalVolume(volPath) }) // If the volume's filesystem needs to have its UUID regenerated to allow mount then do so now. if vol.contentType == ContentTypeFS && renegerateFilesystemUUIDNeeded(vol.ConfigBlockFilesystem()) { _, err = d.activateVolume(vol) if err != nil { return err } d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": volPath, "fs": vol.ConfigBlockFilesystem()}) volDevPath, err := d.lvmDevPath(volPath) if err != nil { return err } err = regenerateFilesystemUUID(vol.ConfigBlockFilesystem(), volDevPath) if err != nil { return err } } // Finally remove the original logical volume. Should always be the last step to allow revert. err = d.removeLogicalVolume(d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, tmpVolName)) if err != nil { return fmt.Errorf("Error removing original LVM logical volume: %w", err) } reverter.Success() return nil } // If the pool uses classic logical volumes, then the process for restoring a snapshot is as follows: // 1. Ensure snapshot volumes have sufficient CoW capacity to allow restoration. // 2. Mount source and target. // 3. Copy (rsync or dd) source to target. // 4. Unmount source and target. // Ensure that the snapshot volumes have sufficient CoW capacity to allow restoration. // In the past we set snapshot sizes by specifying the same size as the origin volume. Unfortunately due to // the way that LVM extents work, this means that the snapshot CoW capacity can be just a little bit too // small to allow the entire snapshot to be restored to the origin. If this happens then we can end up // invalidating the snapshot meaning it cannot be used anymore! // Nowadays we use the "100%ORIGIN" size when creating snapshots, which lets LVM figure out what the number // of extents is required to restore the whole snapshot, but we need to support resizing older snapshots // taken before this change. So we use lvresize here to grow the snapshot volume to the size of the origin. // The use of "+100%ORIGIN" here rather than just "100%ORIGIN" like we use when taking new snapshots, is // rather counter intuitive. However there seems to be a bug in lvresize/lvextend so that when specifying // "100%ORIGIN", it fails to extend sufficiently, saying that the number of extents in the snapshot matches // that of the origin (which they do). However if we take take that at face value then the restore will // end up invalidating the snapshot. Instead if we specify a much larger value (such as adding 100% of // the origin to the snapshot size) then LVM is able to extend the snapshot a little bit more, and LVM // limits the new size to the maximum CoW size that the snapshot can be (which happens to be the same size // as newer snapshots are taken at using the "100%ORIGIN" size). Confusing isn't it. if snapVol.IsVMBlock() || snapVol.contentType == ContentTypeFS { snapLVPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, ContentTypeFS, snapVol.name) _, err = subprocess.TryRunCommand("lvresize", "-l", "+100%ORIGIN", "-f", snapLVPath) if err != nil { return err } } if snapVol.IsVMBlock() || (snapVol.contentType == ContentTypeBlock && snapVol.volType == VolumeTypeCustom) { snapLVPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, ContentTypeBlock, snapVol.name) _, err = subprocess.TryRunCommand("lvresize", "-l", "+100%ORIGIN", "-f", snapLVPath) if err != nil { return err } } // Mount source and target, copy, then unmount. err = vol.MountTask(func(mountPath string, op *operations.Operation) error { // Copy source to destination (mounting each volume if needed). err = snapVol.MountTask(func(srcMountPath string, op *operations.Operation) error { if snapVol.IsVMBlock() || snapVol.contentType == ContentTypeFS { bwlimit := d.config["rsync.bwlimit"] d.Logger().Debug("Copying filesystem volume", logger.Ctx{"sourcePath": srcMountPath, "targetPath": mountPath, "bwlimit": bwlimit}) _, err := rsync.LocalCopy(srcMountPath, mountPath, bwlimit, true) if err != nil { return err } } if snapVol.IsVMBlock() || (snapVol.contentType == ContentTypeBlock && snapVol.volType == VolumeTypeCustom) { srcDevPath, err := d.GetVolumeDiskPath(snapVol) if err != nil { return err } targetDevPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } d.Logger().Debug("Copying block volume", logger.Ctx{"srcDevPath": srcDevPath, "targetPath": targetDevPath}) err = copyDevice(srcDevPath, targetDevPath) if err != nil { return err } } return nil }, op) if err != nil { return err } // Run EnsureMountPath after mounting and syncing to ensure the mounted directory has the // correct permissions set. err = vol.EnsureMountPath(false) if err != nil { return err } return nil }, op) if err != nil { return fmt.Errorf("Error restoring LVM logical volume snapshot: %w", err) } reverter.Success() return nil } // RenameVolumeSnapshot renames a volume snapshot. func (d *lvm) RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *operations.Operation) error { volPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, snapVol.contentType, snapVol.name) parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) newSnapVolName := GetSnapshotVolumeName(parentName, newSnapshotName) newVolPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, snapVol.contentType, newSnapVolName) release, err := d.acquireExclusive(snapVol) if err != nil { return err } defer release() err = d.renameLogicalVolume(volPath, newVolPath) if err != nil { return fmt.Errorf("Error renaming LVM logical volume: %w", err) } oldPath := snapVol.MountPath() newPath := GetVolumeMountPath(d.name, snapVol.volType, newSnapVolName) if util.PathExists(oldPath) { err = os.Rename(oldPath, newPath) if err != nil { return fmt.Errorf("Error renaming snapshot mount path from %q to %q: %w", oldPath, newPath, err) } } return nil } // GetQcow2BackingFilePath generates the backing file path for the specified volume. func (d *lvm) GetQcow2BackingFilePath(vol Volume) (string, error) { pathName := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) return filepath.Join("/dev", pathName), nil } // Qcow2DeletionCleanup performs post block-commit cleanup of qcow2 snapshot artifacts. func (d *lvm) Qcow2DeletionCleanup(snapVol Volume, childName string) error { childVolPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, snapVol.contentType, childName) snapVolPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, snapVol.contentType, snapVol.name) childVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, childName, snapVol.config, snapVol.poolConfig) // Activate volume if needed. activated, err := d.activateVolume(childVol) if err != nil { return err } defer func() { // Deactivate volume if was not active before operation. if activated { _, _ = d.deactivateVolume(childVol) } }() _, err = d.acquireExclusive(snapVol) if err != nil { return err } _ = d.removeLogicalVolume(childVolPath) err = d.renameLogicalVolume(snapVolPath, childVolPath) if err != nil { return fmt.Errorf("Error temporarily renaming original LVM logical volume: %w", err) } releaseParent, err := d.acquireExclusive(childVol) if err != nil { return err } releaseParent() return nil } incus-7.0.0/internal/server/storage/drivers/driver_mock.go000066400000000000000000000164741517523235500237620ustar00rootroot00000000000000package drivers import ( "io" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/server/backup" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/revert" ) type mock struct { common } // load is used to run one-time action per-driver rather than per-pool. func (d *mock) load() error { return nil } // Info returns info about the driver and its environment. func (d *mock) Info() Info { return Info{ Name: "mock", Version: "1", DefaultVMBlockFilesystemSize: deviceConfig.DefaultVMBlockFilesystemSize, OptimizedImages: false, PreservesInodes: false, Remote: d.isRemote(), VolumeTypes: []VolumeType{VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM}, BlockBacking: false, RunningCopyFreeze: true, DirectIO: true, MountedRoot: true, } } func (d *mock) FillConfig() error { return nil } func (d *mock) Create() error { return nil } func (d *mock) Delete(op *operations.Operation) error { return nil } // Validate checks that all provide keys are supported and that no conflicting or missing configuration is present. func (d *mock) Validate(config map[string]string) error { return d.validatePool(config, nil, nil) } // Update applies any driver changes required from a configuration change. func (d *mock) Update(changedConfig map[string]string) error { return nil } // Mount mounts the storage pool. func (d *mock) Mount() (bool, error) { return true, nil } // Unmount unmounts the storage pool. func (d *mock) Unmount() (bool, error) { return true, nil } // GetResources returns the pool resource usage information. func (d *mock) GetResources() (*api.ResourcesStoragePool, error) { return nil, nil } // CreateVolume creates an empty volume and can optionally fill it by executing the supplied filler function. func (d *mock) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Operation) error { return nil } // CreateVolumeFromBackup restores a backup tarball onto the storage device. func (d *mock) CreateVolumeFromBackup(vol Volume, srcBackup backup.Info, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) (VolumePostHook, revert.Hook, error) { return nil, nil, nil } // CreateVolumeFromCopy provides same-pool volume copying functionality. func (d *mock) CreateVolumeFromCopy(vol Volume, srcVol Volume, copySnapshots bool, allowInconsistent bool, op *operations.Operation) error { return nil } // CreateVolumeFromMigration creates a volume being sent via a migration. func (d *mock) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, volTargetArgs migration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error { return nil } // RefreshVolume provides same-pool volume and specific snapshots syncing functionality. func (d *mock) RefreshVolume(vol Volume, srcVol Volume, srcSnapshots []Volume, allowInconsistent bool, op *operations.Operation) error { return nil } // DeleteVolume deletes a volume of the storage device. If any snapshots of the volume remain then this function // will return an error. func (d *mock) DeleteVolume(vol Volume, op *operations.Operation) error { return nil } // HasVolume indicates whether a specific volume exists on the storage pool. func (d *mock) HasVolume(vol Volume) (bool, error) { return true, nil } // ValidateVolume validates the supplied volume config. Optionally removes invalid keys from the volume's config. func (d *mock) ValidateVolume(vol Volume, removeUnknownKeys bool) error { return nil } // UpdateVolume applies config changes to the volume. func (d *mock) UpdateVolume(vol Volume, changedConfig map[string]string) error { if vol.contentType != ContentTypeFS { return ErrNotSupported } _, changed := changedConfig["size"] if changed { err := d.SetVolumeQuota(vol, changedConfig["size"], false, nil) if err != nil { return err } } return nil } // GetVolumeUsage returns the disk space used by the volume. func (d *mock) GetVolumeUsage(vol Volume) (int64, error) { return 0, nil } // SetVolumeQuota applies a size limit on volume. func (d *mock) SetVolumeQuota(vol Volume, size string, allowUnsafeResize bool, op *operations.Operation) error { return nil } // GetVolumeDiskPath returns the location of a disk volume. func (d *mock) GetVolumeDiskPath(vol Volume) (string, error) { return "", nil } // ListVolumes returns a list of volumes in storage pool. func (d *mock) ListVolumes() ([]Volume, error) { return nil, nil } // MountVolume simulates mounting a volume. func (d *mock) MountVolume(vol Volume, op *operations.Operation) error { return nil } // UnmountVolume simulates unmounting a volume. As dir driver doesn't have volumes to unmount it // returns false indicating the volume was already unmounted. func (d *mock) UnmountVolume(vol Volume, keepBlockDev bool, op *operations.Operation) (bool, error) { return false, nil } // RenameVolume renames a volume and its snapshots. func (d *mock) RenameVolume(vol Volume, newVolName string, op *operations.Operation) error { return nil } // MigrateVolume sends a volume for migration. func (d *mock) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *migration.VolumeSourceArgs, op *operations.Operation) error { return nil } // BackupVolume copies a volume (and optionally its snapshots) to a specified target path. // This driver does not support optimized backups. func (d *mock) BackupVolume(vol Volume, writer instancewriter.InstanceWriter, basePrefix string, optimized bool, snapshots []string, op *operations.Operation) error { return nil } // CreateVolumeSnapshot creates a snapshot of a volume. func (d *mock) CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error { return nil } // DeleteVolumeSnapshot removes a snapshot from the storage device. The volName and snapshotName // must be bare names and should not be in the format "volume/snapshot". func (d *mock) DeleteVolumeSnapshot(snapVol Volume, op *operations.Operation) error { return nil } // MountVolumeSnapshot sets up a read-only mount on top of the snapshot to avoid accidental modifications. func (d *mock) MountVolumeSnapshot(snapVol Volume, op *operations.Operation) error { return nil } // UnmountVolumeSnapshot removes the read-only mount placed on top of a snapshot. func (d *mock) UnmountVolumeSnapshot(snapVol Volume, op *operations.Operation) (bool, error) { return true, nil } // VolumeSnapshots returns a list of snapshots for the volume (in no particular order). func (d *mock) VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) { return nil, nil } // CanRestoreVolume checks whether a volume snapshot can be restored. func (d *mock) CanRestoreVolume(vol Volume, snapshotName string) error { return nil } // RestoreVolume restores a volume from a snapshot. func (d *mock) RestoreVolume(vol Volume, snapshotName string, op *operations.Operation) error { return nil } // RenameVolumeSnapshot renames a volume snapshot. func (d *mock) RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *operations.Operation) error { return nil } incus-7.0.0/internal/server/storage/drivers/driver_truenas.go000066400000000000000000000374441517523235500245120ustar00rootroot00000000000000package drivers import ( "errors" "fmt" "os/exec" "path/filepath" "slices" "strconv" "strings" "github.com/lxc/incus/v7/internal/migration" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) var ( tnVersion string tnLoaded bool ) var tnDefaultSettings = map[string]string{ "atime": "off", "exec": "on", "acltype": "posix", "aclmode": "discard", "comments": "Managed by Incus.TrueNAS", // these are set in createDataset "managedby": "incus.truenas", } type truenas struct { common } func (d *truenas) initVersion() error { if tnVersion != "" { return nil } ver, err := d.version() if err != nil { return err } tnVersion = ver return nil } // load is used to run one-time action per-driver rather than per-pool. func (d *truenas) load() error { // Register the patches. d.patches = map[string]func() error{ "storage_lvm_skipactivation": nil, "storage_missing_snapshot_records": nil, "storage_delete_old_snapshot_records": nil, "storage_zfs_drop_block_volume_filesystem_extension": nil, "storage_prefix_bucket_names_with_project": nil, } // Done if previously loaded. if tnLoaded { return nil } // Handle IncusOS services. if d.state.OS.IncusOS != nil { ok, err := d.state.OS.IncusOS.IsServiceEnabled("iscsi") if err != nil { return err } if !ok { return errors.New("IncusOS service \"iscsi\" isn't currently enabled") } } // Validate the needed tools are present. for _, tool := range []string{tnToolName} { _, err := exec.LookPath(tool) if err != nil { return fmt.Errorf("Required tool '%s' is missing", tool) } } err := d.initVersion() if err != nil { return err } tnLoaded = true return nil } // isRemote returns true indicating this driver uses remote storage. func (d *truenas) isRemote() bool { return true } // Info returns info about the driver and its environment. func (d *truenas) Info() Info { info := Info{ Name: "truenas", Version: tnVersion, DefaultVMBlockFilesystemSize: deviceConfig.DefaultVMBlockFilesystemSize, OptimizedImages: true, OptimizedBackups: false, PreservesInodes: false, Remote: d.isRemote(), VolumeTypes: []VolumeType{VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM}, VolumeMultiNode: false, // can only use the same volume if its read-only.d.isRemote(), BlockBacking: true, RunningCopyFreeze: true, DirectIO: false, IOUring: false, MountedRoot: false, Buckets: false, } return info } // ensureInitialDatasets creates missing initial datasets or configures existing ones with current policy. // Accepts warnOnExistingPolicyApplyError argument, if true will warn rather than fail if applying current policy // to an existing dataset fails. func (d *truenas) ensureInitialDatasets(warnOnExistingPolicyApplyError bool) error { args := make([]string, 0, len(tnDefaultSettings)) for k, v := range tnDefaultSettings { args = append(args, fmt.Sprintf("%s=%s", k, v)) } if d.config["truenas.dataset"] == "" { return nil } err := d.setDatasetProperties(d.config["truenas.dataset"], args...) if err != nil { if !warnOnExistingPolicyApplyError { return fmt.Errorf("Failed applying policy to existing dataset %q: %w", d.config["truenas.dataset"], err) } d.logger.Warn("Failed applying policy to existing dataset", logger.Ctx{"dataset": d.config["truenas.dataset"], "err": err}) } datasets := d.initialDatasets() fullDatasetPaths := make([]string, len(datasets)) for i := 0; i < len(datasets); i++ { fullDatasetPaths[i] = filepath.Join(d.config["truenas.dataset"], datasets[i]) } properties := []string{} shouldCreateMissingDatasets := true return d.updateDatasets(fullDatasetPaths, shouldCreateMissingDatasets, properties...) } // FillConfig populates the storage pool's configuration file with the default values. func (d *truenas) FillConfig() error { // populate source if not already present if d.config["truenas.dataset"] != "" && d.config["source"] == "" { d.config["source"] = d.config["truenas.dataset"] } err := d.parseSource() if err != nil { return err } return nil } func (d *truenas) parseSource() error { // fill config may modify. sourceStr := d.config["source"] var host, path string if strings.HasPrefix(sourceStr, "[") { // IPv6 with brackets endBracket := strings.Index(sourceStr, "]") if endBracket == -1 || endBracket+1 >= len(sourceStr) || sourceStr[endBracket+1] != ':' { // Malformed, treat whole string as path host = "" path = sourceStr } else { host = sourceStr[:endBracket+1] path = sourceStr[endBracket+2:] // skip over "]:" } } else { // Try normal IPv4/hostname h, p, ok := strings.Cut(sourceStr, ":") if ok { host = h path = p } else { // No colon: whole thing is path host = "" path = sourceStr } } if path == "" || filepath.IsAbs(path) { return errors.New(`TrueNAS Driver requires "source" to be specified using the format: [:][[/]...][/]`) } // a pool... means we create a dataset in the root if !strings.Contains(path, "/") { path += "/" } // a trailing slash means use the storage pool name as the dataset if strings.HasSuffix(path, "/") { path += d.name } d.config["truenas.dataset"] = path if host != "" { if d.config["truenas.host"] != "" { host = d.config["truenas.host"] } source := fmt.Sprintf("%s:%s", host, path) d.config["truenas.host"] = host d.config["source"] = source } else { d.config["source"] = path } return nil } // Create is called during pool creation and is effectively using an empty driver struct. // WARNING: The Create() function cannot rely on any of the struct attributes being set. func (d *truenas) Create() error { // Store the provided source as we are likely to be mangling it. d.config["volatile.initial_source"] = d.config["source"] err := d.FillConfig() if err != nil { return err } // create pool dataset exists, err := d.datasetExists(d.config["truenas.dataset"]) if err != nil { return err } if !exists { err = d.createDataset(d.config["truenas.dataset"]) if err != nil { return fmt.Errorf("Failed to create storage pool on TrueNAS host: %s, err: %w", d.config["source"], err) } } else if util.IsFalseOrEmpty(d.config["truenas.force_reuse"]) { // Confirm that the existing pool/dataset is all empty. datasets, err := d.getDatasets(d.config["truenas.dataset"], "all") if err != nil { return err } if len(datasets) > 0 { return fmt.Errorf(`Remote TrueNAS dataset isn't empty: %s`, d.config["truenas.dataset"]) } } // Setup revert in case of problems reverter := revert.New() defer reverter.Fail() reverter.Add(func() { _ = d.Delete(nil) }) err = d.verifyIscsiFunctionality(false) // ensureSetup if err != nil { return fmt.Errorf("Unable to verify TrueNAS iSCSI service: %v", err) } reverter.Success() return nil } // Delete removes the storage pool from the storage device. func (d *truenas) Delete(op *operations.Operation) error { // Check if the dataset/pool is already gone. exists, err := d.datasetExists(d.config["truenas.dataset"]) if err != nil { return err } if exists { // Confirm that nothing's been left behind datasets, err := d.getDatasets(d.config["truenas.dataset"], "all") if err != nil { return err } initialDatasets := d.initialDatasets() for _, dataset := range datasets { dataset = strings.TrimPrefix(dataset, "/") if slices.Contains(initialDatasets, dataset) { continue } fields := strings.Split(dataset, "/") if len(fields) > 1 { return fmt.Errorf("TrueNAS pool has leftover datasets: %s", dataset) } } // Delete the dataset. err = d.deleteDataset(d.config["truenas.dataset"], true) if err != nil { return err } } // On delete, wipe everything in the directory. err = wipeDirectory(GetPoolMountPath(d.name)) if err != nil { return err } return nil } // Validate checks that all provide keys are supported and that no conflicting or missing configuration is present. func (d *truenas) Validate(config map[string]string) error { rules := map[string]func(value string) error{ // only truenas.dataset is required. the tool has default behaviour/connections defined. // gendoc:generate(entity=storage_truenas, group=common, key=source) // // --- // type: string // scope: local // default: - // shortdesc: ZFS dataset to use on the remote TrueNAS host. Format: `[:][/][/]`. If `host` is omitted here, it must be set via `truenas.host`. "source": validate.IsAny, // can be used as a shortcut to specify dataset and optionally host. // gendoc:generate(entity=storage_truenas, group=common, key=truenas.dataset) // // --- // type: string // scope: global // default: - // shortdesc: Remote dataset name. Typically inferred from `source`, but can be overridden. "truenas.dataset": validate.IsAny, // global flags for the tool // gendoc:generate(entity=storage_truenas, group=common, key=truenas.allow_insecure) // // --- // type: bool // scope: global // default: `false` // shortdesc: If set to `true`, allows insecure (non-TLS) connections to the TrueNAS API. "truenas.allow_insecure": validate.Optional(validate.IsBool), // gendoc:generate(entity=storage_truenas, group=common, key=truenas.api_key) // // --- // type: string // scope: global // default: - // shortdesc: API key used to authenticate with the TrueNAS host. "truenas.api_key": validate.IsAny, // gendoc:generate(entity=storage_truenas, group=common, key=truenas.config) // // --- // type: string // scope: global // default: - // shortdesc: Path to a configuration file for the TrueNAS client tool. "truenas.config": validate.IsAny, // gendoc:generate(entity=storage_truenas, group=common, key=truenas.host) // // --- // type: string // scope: global // default: - // shortdesc: Hostname or IP address of the remote TrueNAS system. Optional if included in the `source`, or a configuration is used. "truenas.host": validate.IsAny, // flags for the tool's iscsi commands // gendoc:generate(entity=storage_truenas, group=common, key=truenas.initiator) // // --- // type: string // scope: global // default: - // shortdesc: iSCSI initiator name used during block volume attachment. "truenas.initiator": validate.IsAny, // gendoc:generate(entity=storage_truenas, group=common, key=truenas.portal) // // --- // type: string // scope: global // default: - // shortdesc: iSCSI portal address to use for block volume connections. "truenas.portal": validate.IsAny, // controls behaviour of the driver // gendoc:generate(entity=storage_truenas, group=common, key=truenas.clone_copy) // // --- // type: bool // scope: global // default: `true` // shortdesc: Whether to use lightweight clones rather than full {spellexception}`dataset` copies. "truenas.clone_copy": validate.Optional(validate.IsBool), // gendoc:generate(entity=storage_truenas, group=common, key=truenas.force_reuse) // // --- // type: bool // scope: global // default: `false` // shortdesc: Allow to use an existing non-empty pool. "truenas.force_reuse": validate.Optional(validate.IsBool), } return d.validatePool(config, rules, d.commonVolumeRules()) } // Update applies any driver changes required from a configuration change. func (d *truenas) Update(changedConfig map[string]string) error { _, ok := changedConfig["truenas.dataset"] if ok { return errors.New("truenas.dataset cannot be modified") } // prop changes we want to accept props := []string{ "truenas.allow_insecure", "truenas.api_key", "truenas.config", "truenas.host", "truenas.initiator", "truenas.portal", "truenas.clone_copy", "truenas.force_reuse", } for _, prop := range props { value, ok := changedConfig[prop] if ok { d.config[prop] = value } } return nil } // Mount mounts the storage pool. func (d *truenas) Mount() (bool, error) { // verify pool dataset exists exists, err := d.datasetExists(d.config["truenas.dataset"]) if err != nil { return false, err } if !exists { return false, fmt.Errorf("TrueNAS host is responding, but dataset is missing %s:%s", d.config["truenas.host"], d.config["truenas.dataset"]) } // Apply our default configuration. err = d.ensureInitialDatasets(true) if err != nil { return false, err } // As we have already created the storage pool, and it exists on the host, presumably we already had iscsi setup in the past, so restore it if necessary. err = d.verifyIscsiFunctionality(true) if err != nil { return false, err } return false, nil } // Unmount unmounts the storage pool. func (d *truenas) Unmount() (bool, error) { return true, nil } // GetResources returns the pool resource usage information. func (d *truenas) GetResources() (*api.ResourcesStoragePool, error) { // Get the total amount of space and the used amount of space. props, err := d.getDatasetProperties(d.config["truenas.dataset"], []string{"available", "used"}) if err != nil { return nil, err } // Parse the total amount of space. availableStr := props["available"] available, err := strconv.ParseUint(strings.TrimSpace(availableStr), 10, 64) if err != nil { return nil, err } // Parse the used amount of space. usedStr := props["used"] used, err := strconv.ParseUint(strings.TrimSpace(usedStr), 10, 64) if err != nil { return nil, err } // Build the struct. // Inode allocation is dynamic so no use in reporting them. res := api.ResourcesStoragePool{} res.Space.Total = used + available res.Space.Used = used return &res, nil } // MigrationTypes returns the type of transfer methods to be used when doing migrations between pools in preference order. func (d *truenas) MigrationTypes(contentType ContentType, refresh bool, copySnapshots bool, clusterMove bool, storageMove bool) []localMigration.Type { // TODO: investigate "storageMove" that came from the linstor driver. var rsyncFeatures []string // Do not pass compression argument to rsync if the associated // config key, that is rsync.compression, is set to false. if util.IsFalse(d.Config()["rsync.compression"]) { rsyncFeatures = []string{"xattrs", "delete", "bidirectional"} } else { rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"} } if IsContentBlock(contentType) { return []localMigration.Type{ // TODO: optimized { FSType: migration.MigrationFSType_BLOCK_AND_RSYNC, Features: rsyncFeatures, }, } } if refresh && !copySnapshots { return []localMigration.Type{ { FSType: migration.MigrationFSType_RSYNC, Features: rsyncFeatures, }, } } return []localMigration.Type{ // TODO: optimized { FSType: migration.MigrationFSType_RSYNC, Features: rsyncFeatures, }, } } // roundVolumeBlockSizeBytes returns sizeBytes rounded up to the next multiple // of `vol`'s "truenas.blocksize". func (d *truenas) roundVolumeBlockSizeBytes(vol Volume, sizeBytes int64) (int64, error) { minBlockSize, err := units.ParseByteSizeString(vol.ExpandedConfig("truenas.blocksize")) // minBlockSize will be 0 if truenas.blocksize="" if minBlockSize <= 0 || err != nil { minBlockSize = tnDefaultVolblockSize // 16KiB } return RoundAbove(minBlockSize, sizeBytes), nil } incus-7.0.0/internal/server/storage/drivers/driver_truenas_cache.go000066400000000000000000000124621517523235500256260ustar00rootroot00000000000000package drivers import ( "slices" "strings" "sync" "time" "github.com/lxc/incus/v7/shared/logger" ) // This is a per-pool TrueNAS cache, used to limit the number // of expensive requests to TrueNAS especially during bulk requests like a full // instance list. type truenasCacheEntry struct { Expiry time.Time Value string } var ( truenasCache map[string]map[string]map[string]truenasCacheEntry truenasCacheMu sync.Mutex truenasCachePrefillQueue map[string][]string truenasCachePrefillRunning map[string]bool truenasCachePrefillMu map[string]*sync.RWMutex truenasCacheProperties = []string{"used", "referenced"} ) // truenasCacheEnsurePool initializes the per-pool cache maps if needed. func truenasCacheEnsurePool(pool string) { _, ok := truenasCache[pool] if !ok { truenasCache[pool] = map[string]map[string]truenasCacheEntry{} truenasCachePrefillQueue[pool] = []string{} truenasCachePrefillMu[pool] = &sync.RWMutex{} } } func (d *truenas) prefillCachedProperties(dataset string) { // Define a function to quickly check if a dataset is already cached. isCached := func(dataset string) bool { record, ok := truenasCache[d.name][dataset] if !ok { return false } now := time.Now() for _, propName := range truenasCacheProperties { prop, ok := record[propName] if !ok || prop.Expiry.Before(now) { return false } } return true } // Get the lock. truenasCacheMu.Lock() // Ensure the pool exists in the cache. truenasCacheEnsurePool(d.name) // Check if we already have a valid cache for the dataset. if isCached(dataset) { truenasCacheMu.Unlock() return } // Add the request to the queue. if !slices.Contains(truenasCachePrefillQueue[d.name], dataset) { truenasCachePrefillQueue[d.name] = append(truenasCachePrefillQueue[d.name], dataset) } // Check if a filler is already running. // If not, make a copy of the queue and reset it. var runPrefill bool if !truenasCachePrefillRunning[d.name] { truenasCachePrefillRunning[d.name] = true truenasCachePrefillMu[d.name].Lock() defer func() { truenasCacheMu.Lock() truenasCachePrefillMu[d.name].Unlock() truenasCachePrefillRunning[d.name] = false truenasCacheMu.Unlock() }() runPrefill = true } // Release the lock. truenasCacheMu.Unlock() // Check if we're done. if !runPrefill { // If we got here, the dataset we care about wasn't already in the cache AND there was an existing prefill run ongoing. // Attempt to get a read lock for the prefiller, this will block until the current prefill is done running. // // Depending on timing, the current prefill may or may not have picked us up from the queue. // So we check if we're still in the queue and if we are, we trigger another run which will hopefully pick us up then. truenasCachePrefillMu[d.name].RLock() truenasCachePrefillMu[d.name].RUnlock() //nolint:staticcheck // Check that we made it. truenasCacheMu.Lock() inQueue := slices.Contains(truenasCachePrefillQueue[d.name], dataset) truenasCacheMu.Unlock() if inQueue { // We didn't make it, re-trigger. d.prefillCachedProperties(dataset) return } } // Allow for requests to accumulate. time.Sleep(200 * time.Millisecond) // Copy and clear the queue. truenasCacheMu.Lock() queue := []string{} for _, entry := range truenasCachePrefillQueue[d.name] { if isCached(entry) { continue } queue = append(queue, entry) } truenasCachePrefillQueue[d.name] = []string{} truenasCacheMu.Unlock() // Check that we have something to do. if len(queue) == 0 { return } // Run the filler in batches of 2 datasets (TrueNAS limitation). properties := strings.Join(append([]string{"name"}, truenasCacheProperties...), ",") for i := 0; i < len(queue); i += 2 { batch := queue[i:] if len(batch) > 2 { batch = batch[:2] } args := []string{"list", "--no-headers", "--parsable", "-o", properties, "-r", "-t", "filesystem,volume,snapshot"} args = append(args, batch...) out, err := d.runTool(args...) if err != nil { d.logger.Warn("Couldn't cache TrueNAS properties", logger.Ctx{"err": err}) continue } // Update the cache. truenasCacheMu.Lock() expiry := time.Now().Add(time.Minute) for _, line := range strings.Split(out, "\n") { line = strings.TrimSpace(line) if line == "" { continue } fields := strings.Fields(line) if len(fields) != 3 { continue } record, ok := truenasCache[d.name][fields[0]] if !ok { record = map[string]truenasCacheEntry{} } for i, value := range fields[1:] { key := truenasCacheProperties[i] record[key] = truenasCacheEntry{Expiry: expiry, Value: value} } truenasCache[d.name][fields[0]] = record } truenasCacheMu.Unlock() } } func (d *truenas) getCachedProperty(dataset string, key string) (string, bool) { // Check if this is a cached property. if !slices.Contains(truenasCacheProperties, key) { return "", false } // Update cache if needed. parentDataset := strings.Split(dataset, "@")[0] d.prefillCachedProperties(parentDataset) // Get the value. truenasCacheMu.Lock() defer truenasCacheMu.Unlock() record, ok := truenasCache[d.name][dataset] if !ok { return "", false } value, ok := record[key] if !ok { return "", false } if value.Expiry.Before(time.Now()) { return "", false } return value.Value, true } incus-7.0.0/internal/server/storage/drivers/driver_truenas_utils.go000066400000000000000000000541341517523235500257250ustar00rootroot00000000000000package drivers import ( "context" "encoding/json" "errors" "fmt" "math" "path/filepath" "strings" "time" "github.com/google/uuid" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) const ( tnToolName = "truenas_incus_ctl" tnDefaultVolblockSize = 16 * 1024 ) func (d *truenas) dataset(vol Volume, deleted bool) string { name, snapName, _ := api.GetParentAndSnapshotName(vol.name) if vol.volType == VolumeTypeImage && vol.contentType == ContentTypeFS { name = fmt.Sprintf("%s_%s", name, vol.ConfigBlockFilesystem()) } if (vol.volType == VolumeTypeVM || vol.volType == VolumeTypeImage) && vol.contentType == ContentTypeBlock { name = fmt.Sprintf("%s%s", name, zfsBlockVolSuffix) } else if vol.volType == VolumeTypeCustom && vol.contentType == ContentTypeISO { name = fmt.Sprintf("%s%s", name, zfsISOVolSuffix) } if snapName != "" { if deleted { name = fmt.Sprintf("%s@deleted-%s", name, uuid.New().String()) } else { name = fmt.Sprintf("%s@snapshot-%s", name, snapName) } } else if deleted { if vol.volType != VolumeTypeImage { name = uuid.New().String() } return filepath.Join(d.config["truenas.dataset"], "deleted", string(vol.volType), name) } return filepath.Join(d.config["truenas.dataset"], string(vol.volType), name) } // runTool runs the truenas control tool with the supplied arguments, whilst applying the global flags as appropriate. func (d *truenas) runTool(args ...string) (string, error) { baseArgs := []string{} if util.IsTrue(d.config["truenas.allow_insecure"]) { baseArgs = append(baseArgs, "--allow-insecure") } if d.config["truenas.api_key"] != "" { baseArgs = append(baseArgs, "--api-key", d.config["truenas.api_key"]) } if d.config["truenas.config"] != "" { baseArgs = append(baseArgs, "--config", d.config["truenas.config"]) } if d.config["truenas.host"] != "" { baseArgs = append(baseArgs, "--host", d.config["truenas.host"]) } args = append(baseArgs, args...) out, err := subprocess.RunCommand(tnToolName, args...) if err != nil && strings.Contains(err.Error(), "Post \"http://unix/tnc-daemon\": EOF)") { // this error indicates that the connection to the server was closed when the command was posted. It should be safe to retry the command // the daemon *should've* re-opened the connection, but as of 0.7.2 it doesn't, re-trying should force the connection to be re-opened. d.logger.Error("TrueNAS Tool POST failed with socket EOF, will retry", logger.Ctx{"err": err}) out, err = subprocess.RunCommand(tnToolName, args...) } // will allow us to prepend args return out, err } // runIscsiCmd runs the supplied args against the tools `share iscsi` command whilst applying the appropriate iscsi global flags. func (d *truenas) runIscsiCmd(cmd string, args ...string) (string, error) { baseArgs := []string{"share", "iscsi", cmd} baseArgs = append(baseArgs, "--target-prefix=incus") if d.config["truenas.portal"] != "" { baseArgs = append(baseArgs, "--portal", d.config["truenas.portal"]) } if d.config["truenas.initiator"] != "" { baseArgs = append(baseArgs, "--initiator", d.config["truenas.initiator"]) } args = append(baseArgs, args...) return d.runTool(args...) } func optionsToOptionString(options ...string) string { var builder strings.Builder for i, option := range options { if i > 0 { builder.WriteString(",") } builder.WriteString(option) } optionString := builder.String() return optionString } func (d *truenas) setDatasetProperties(dataset string, options ...string) error { args := []string{"dataset", "update"} // TODO: either move the "--" prepending here, or have the -o syntax work! // optionString := optionsToOptionString(options...) // if optionString != "" { // args = append(args, "-o", optionString) // } for _, option := range options { args = append(args, fmt.Sprintf("--%s", option)) } args = append(args, dataset) out, err := d.runTool(args...) _ = out if err != nil { return err } return nil } // getDatasetOrSnapshotreturns "dataset" or "snapshot" depending on the supplied name // used to disambiguate truenas-admin commands. func (d *truenas) getDatasetOrSnapshot(dataset string) string { if strings.Contains(dataset, "@") { return "snapshot" } return "dataset" } func (d *truenas) datasetExists(dataset string) (bool, error) { out, err := d.runTool(d.getDatasetOrSnapshot(dataset), "list", "--no-headers", "-o", "name", dataset) if err != nil { return false, nil // TODO: need to check if tool returns errors for bad connections, vs not-found. Ie, this occurs when recovering with a bad API key or HOST. } return strings.TrimSpace(out) == dataset, nil } // objectsExist returns a map of existence for a number of objects (snaps, vols, etc). // unused currently but planned to be used in DeleteVolume. func (d *truenas) objectsExist(objects []string, optType string) (map[string]bool, error) { //nolint:unused var t string // unlike zfs, `list` will return nfs and other objects for all. switch optType { case "": t = "filesystem,volume,snapshot" case "dataset": t = "filesystem,volume" default: t = optType } args := []string{"list", "--no-headers", "-o", "name", "-t", t} args = append(args, objects...) out, err := d.runTool(args...) if err != nil { return nil, nil } existsMap := make(map[string]bool) for _, str := range objects { existsMap[str] = false } lines := strings.Split(out, "\n") for _, l := range lines { if l == "" || l == "-" { continue } _, exists := existsMap[l] if exists { existsMap[l] = true } } return existsMap, nil } // initialDatasets returns the list of all expected datasets. func (d *truenas) initialDatasets() []string { entries := []string{"deleted"} // Iterate over the listed supported volume types. for _, volType := range d.Info().VolumeTypes { entries = append(entries, BaseDirectories[volType].Paths[0]) entries = append(entries, filepath.Join("deleted", BaseDirectories[volType].Paths[0])) } return entries } func (d *truenas) getDatasets(dataset string, types string) ([]string, error) { // tool does not support "all", but it also supports "nfs" if types == "all" { types = "filesystem,volume,snapshot" } out, err := d.runTool("list", "--no-headers", "-r", "-o", "name", "-t", types, dataset) if err != nil { return nil, err } children := []string{} for _, line := range strings.Split(out, "\n") { line = strings.TrimSpace(line) if line == dataset || line == "" { continue } line = strings.TrimPrefix(line, dataset) children = append(children, line) } return children, nil } // updateDatasets batch updates or creates one or more datasets with the same options. func (d *truenas) updateDatasets(datasets []string, orCreate bool, options ...string) error { args := []string{"dataset", "update"} // for _, option := range options { // args = append(args, "-o") // args = append(args, option) // } if orCreate { args = append(args, "--create") } optionString := optionsToOptionString(options...) if optionString != "" { args = append(args, "-o", optionString) } args = append(args, "--managedby", tnDefaultSettings["managedby"], "--comments", tnDefaultSettings["comments"]) args = append(args, datasets...) out, err := d.runTool(args...) _ = out if err != nil { return err } return nil } // createDatasets batch creates one or more datasets with the same options. func (d *truenas) createDatasets(datasets []string, options ...string) error { args := []string{"dataset", "create"} // for _, option := range options { // args = append(args, "-o") // args = append(args, option) // } optionString := optionsToOptionString(options...) if optionString != "" { args = append(args, "-o", optionString) } args = append(args, "--managedby", tnDefaultSettings["managedby"], "--comments", tnDefaultSettings["comments"]) args = append(args, datasets...) out, err := d.runTool(args...) _ = out if err != nil { return err } return nil } // cloneSnapshot create a dataset by cloning a snapshot. func (d *truenas) cloneSnapshot(srcSnapshot string, destDataset string) error { args := []string{"snapshot", "clone", srcSnapshot, destDataset} // Clone the snapshot. _, err := d.runTool(args...) if err != nil { return err } return nil } // createSnapshot take a recursive snapshot of dataset@snapname, and optionally delete the old snapshot first. func (d *truenas) createSnapshot(snapName string, deleteFirst bool) error { args := []string{"snapshot", "create", "-r"} if deleteFirst { args = append(args, "--delete") } args = append(args, snapName) // Make the snapshot. out, err := d.runTool(args...) _ = out if err != nil { return err } return nil } func (d *truenas) createDataset(dataset string, options ...string) error { err := d.createDatasets([]string{dataset}, options...) if err != nil { return err } return nil } func (d *truenas) createVolume(dataset string, size int64, options ...string) error { args := []string{"dataset", "create", "-s", "-V", fmt.Sprintf("%d", size)} // for _, option := range options { // args = append(args, "-o") // args = append(args, option) // } for _, option := range options { args = append(args, fmt.Sprintf("--%s", option)) } // optionString := optionsToOptionString(options...) // if optionString != "" { // args = append(args, "-o", optionString) // } args = append(args, "--managedby", tnDefaultSettings["managedby"], "--comments", tnDefaultSettings["comments"]) args = append(args, dataset) out, err := d.runTool(args...) _ = out if err != nil { return err } return nil } func (d *truenas) verifyIscsiFunctionality(ensureSetup bool) error { args := []string{"--parsable"} if ensureSetup { args = append(args, "--setup") } _, err := d.runIscsiCmd("test", args...) if err != nil { return err } return nil } func (d *truenas) createIscsiShare(dataset string, readonly bool) error { args := []string{} if readonly { args = append(args, "--readonly") } args = append(args, dataset) _, err := d.runIscsiCmd("create", args...) if err != nil { if strings.Contains(err.Error(), "UNIQUE constraint failed") { d.logger.Debug(fmt.Sprintf("Detected error while attempting to create iscsi share for: %s, %v", dataset, err)) // there's a race when obtaining an iscsi id in `iscsi create`, lets try sleeping for a bit, and retrying. (NAS-135784) time.Sleep(500 * time.Millisecond) return d.createIscsiShare(dataset, readonly) } return err } return nil } func (d *truenas) deleteIscsiShare(dataset string) error { _, err := d.runIscsiCmd("delete", dataset) // implies `deactivate --wait` if err != nil { return err } return nil } // locateIscsiDataset locates a ZFS volume if already active. Returns devpath if activated, "" if not, or an error. func (d *truenas) locateIscsiDataset(dataset string) (string, error) { reverter := revert.New() defer reverter.Fail() statusPath, err := d.runIscsiCmd("locate", "--parsable", dataset) if err != nil { return "", err } status, volDiskPath, found := strings.Cut(statusPath, "\t") if !found { // early versions of locate returned no status. volDiskPath = status } volDiskPath = strings.TrimSpace(volDiskPath) return volDiskPath, nil } // locateOrActivateIscsiDataset ensures a dataset is activated, and returns the dev path. Will create // the share if necessary and returns a bool to determine if activation was required. func (d *truenas) locateOrActivateIscsiDataset(dataset string) (bool, string, error) { reverter := revert.New() defer reverter.Fail() var statusPath string for range 5 { var err error statusPath, err = d.runIscsiCmd("locate", "--create", "--parsable", dataset) // --create implies activate if err != nil { return false, "", err } if statusPath != "" { break } time.Sleep(time.Second) } reverter.Add(func() { _ = d.deactivateIscsiDataset(dataset) }) status, volDiskPath, _ := strings.Cut(statusPath, "\t") didCreate := false // when `locate --create` has to create a share, it outputs two lines, one for the creation, a second for the activation, we need to discard the first. if status == "created" { d.logger.Debug(fmt.Sprintf("Created iscsi share for TrueNAS volume: %s", volDiskPath)) didCreate = true _, statusPath, _ := strings.Cut(statusPath, "\n") status, volDiskPath, _ = strings.Cut(statusPath, "\t") } didActivate := status == "activated" volDiskPath = strings.TrimSpace(volDiskPath) if volDiskPath != "" { reverter.Success() return didActivate, volDiskPath, nil } if didCreate { return false, "", fmt.Errorf("Successfully created, but was unable to activate TrueNAS volume: %v, perhaps there is an iSCSI communication issue?", dataset) } return false, "", fmt.Errorf("Unable to create, activate or locate TrueNAS volume: %v, ", dataset) } // activateVolume activates a ZFS volume if not already active. Returns devpath if activated, "" if not. func (d *truenas) activateIscsiDataset(dataset string) (string, error) { //nolint:unused reverter := revert.New() defer reverter.Fail() volDiskPath, err := d.runIscsiCmd("activate", "--parsable", dataset) if err != nil { return "", err } reverter.Add(func() { _ = d.deactivateIscsiDataset(dataset) }) volDiskPath = strings.TrimSpace(volDiskPath) if volDiskPath != "" { reverter.Success() return volDiskPath, nil } return "", fmt.Errorf("No path for activated TrueNAS volume: %v", dataset) } // deactivateIscsiDatasetIfActive deactivates a dataset if activated, returns true if deactivated. func (d *truenas) deactivateIscsiDatasetIfActive(dataset string) (bool, error) { statusPath, err := d.runIscsiCmd("locate", "--deactivate", "--parsable", "--wait", dataset) if err != nil { return false, err } status, _, _ := strings.Cut(statusPath, "\t") if status == "failed" || status == "" { return false, nil } if status != "deactivated" { return false, fmt.Errorf("Unexpected status when deactivating TrueNAS volume: %v, '%s'", dataset, statusPath) } return true, nil } // deactivateIscsiDataset deactivates an iscsi share if active. func (d *truenas) deactivateIscsiDataset(dataset string) error { _, err := d.deactivateIscsiDatasetIfActive(dataset) if err != nil { return err } return nil } // refreshIscsiBus refreshes the iscsi bus. func (d *truenas) refreshIscsiBus() error { _, err := d.runTool("share", "iscsi", "refresh") if err != nil { return err } return nil } func (d *truenas) deleteSnapshot(snapshot string, recursive bool, options ...string) error { if strings.Count(snapshot, "@") != 1 { return fmt.Errorf("invalid snapshot name: %s", snapshot) } return d.deleteDataset(snapshot, recursive, options...) } // tryDeleteBusyDataset attempts to delete a dataset, repeating if busy until success, or the context is ended. func (d *truenas) tryDeleteBusyDataset(ctx context.Context, dataset string, recursive bool, options ...string) error { for { if ctx.Err() != nil { return fmt.Errorf("Failed to delete dataset for %q: %w", dataset, ctx.Err()) } // we sometimes we receive a "busy" error when deleting... which I think is a race, although iSCSI should've finished with the zvol by the time // deleteIscsiShare returns, maybe it hasn't yet... so we retry... in general if incus is calling deleteDataset it shouldn't be busy. err := d.deleteDataset(dataset, recursive, options...) if err == nil { return nil } /* Error -32001 Method call error [EBUSY] Failed to delete dataset: cannot destroy '': dataset is busy) */ if !strings.Contains(err.Error(), "[EBUSY]") { return err } d.logger.Warn("Error while trying to delete dataset, will retry", logger.Ctx{"dataset": dataset, "err": err}) // was busy, lets try again. time.Sleep(500 * time.Millisecond) } } func (d *truenas) deleteDataset(dataset string, recursive bool, options ...string) error { args := []string{d.getDatasetOrSnapshot(dataset), "delete"} if recursive { args = append(args, "-r") } for _, option := range options { args = append(args, fmt.Sprintf("--%s", option)) } args = append(args, dataset) _, err := d.runTool(args...) if err != nil { return err } return nil } func (d *truenas) getDatasetProperty(dataset string, key string) (string, error) { output, ok := d.getCachedProperty(dataset, key) if !ok { var err error output, err = d.runTool(d.getDatasetOrSnapshot(dataset), "list", "--no-headers", "--parsable", "-o", key, dataset) if err != nil { return "", err } } return strings.TrimSpace(output), nil } func (d *truenas) getDatasetProperties(dataset string, properties []string) (map[string]string, error) { response, err := d.getDatasetsAndProperties([]string{dataset}, properties) if err != nil { return nil, err } result, exists := response[dataset] if exists { return result, nil } return nil, nil } func (d *truenas) getDatasetsAndProperties(datasets []string, properties []string) (map[string]map[string]string, error) { propsStr := strings.Join(properties, ",") out, err := d.runTool(append([]string{"list", "--json", "--parsable", "-o", propsStr}, datasets...)...) if err != nil { return nil, err } var response any if err = json.Unmarshal([]byte(out), &response); err != nil { return nil, err } var resultsMap map[string]any responseMap, ok := response.(map[string]any) if ok { for _, v := range responseMap { r, ok := v.(map[string]any) if ok { resultsMap = r break } } } if resultsMap == nil { return nil, errors.New("Could not find object inside list --json response") } objectsAsMap := make(map[string]bool) for _, obj := range datasets { objectsAsMap[obj] = true } outMap := make(map[string]map[string]string) for k, result := range resultsMap { _, exists := objectsAsMap[k] if !exists { continue } r, ok := result.(map[string]any) if ok { formattedMap := make(map[string]string) for p, v := range r { var value any vF, ok := v.(float64) if ok && vF == math.Floor(vF) { value = int64(vF) } else { value = v } formattedMap[p] = fmt.Sprint(value) } outMap[k] = formattedMap } } return outMap, nil } // renameSnapshot renames sourceSnapshot to destSnapshot. // sourceSnapshot: @. // destSnapshot: [dataset]@. func (d *truenas) renameSnapshot(sourceSnapshot string, destSnapshot string) error { args := []string{"snapshot", "rename", sourceSnapshot, destSnapshot} _, err := d.runTool(args...) if err != nil { return err } return nil } // renameDatasetwill rename a dataset, or snapshot. updateShares is relatively expensive if there is no possibility of there being a share. func (d *truenas) renameDataset(sourceDataset string, destDataset string, updateShares bool) error { args := []string{d.getDatasetOrSnapshot(sourceDataset), "rename"} if updateShares { _ = d.deleteIscsiShare(sourceDataset) // TODO: remove this when --update-shares supports iscsi args = append(args, "--update-shares") } args = append(args, sourceDataset, destDataset) _, err := d.runTool(args...) if err != nil { return err } return nil } func (d *truenas) deleteDatasetRecursive(dataset string) error { // Locate the origin snapshot (if any). origin, err := d.getDatasetProperty(dataset, "origin") if err != nil { return err } err = d.deleteIscsiShare(dataset) if err != nil { return err } // Try delete the dataset (and any snapshots left), waiting up to 5 seconds if its busy ctx, cancel := context.WithTimeout(d.state.ShutdownCtx, 5*time.Second) defer cancel() err = d.tryDeleteBusyDataset(ctx, dataset, true) if err != nil { return err } // Check if the origin can now be deleted. if origin != "" && origin != "-" { if strings.HasPrefix(origin, filepath.Join(d.config["truenas.dataset"], "deleted")) { // Strip the snapshot name when dealing with a deleted volume. dataset = strings.SplitN(origin, "@", 2)[0] } else if strings.Contains(origin, "@deleted-") || strings.Contains(origin, "@copy-") { // Handle deleted snapshots. dataset = origin } else { // Origin is still active. dataset = "" } if dataset != "" { // Get all clones. clones, err := d.getClones(dataset) if err != nil { return err } if len(clones) == 0 { // Delete the origin. err = d.deleteDatasetRecursive(dataset) if err != nil { return err } } } } return nil } func (d *truenas) version() (string, error) { out, err := subprocess.RunCommand(tnToolName, "version") if err == nil { return strings.TrimSpace(string(out)), nil } return "", errors.New("Could not determine TrueNAS driver version") } // setVolsize sets the volsize property of a zvol, optionally ignoring shrink errors (and warning), requires a zvol. func (d *truenas) setVolsize(dataset string, sizeBytes int64, allowShrink bool) error { ignoreShrinkError := true volsizeProp := fmt.Sprintf("--volsize=%d", sizeBytes) args := []string{"dataset", "update", volsizeProp} if allowShrink { // although the middleware doesn't currently support shrinking, when it does, the tool will support it via this flag. args = append(args, "--allow-shrinking") } args = append(args, dataset) _, err := d.runTool(args...) if err != nil { if !ignoreShrinkError || !strings.Contains(err.Error(), "cannot shrink a zvol") { return err } // middleware currently prevents volume shrinking. d.logger.Warn(fmt.Sprintf("Unable to shrink zvol on TrueNAS server due to middleware restriction, use `zfs set %s %s` to change zvol size manually", volsizeProp, dataset)) } return nil } func (d *truenas) getClones(dataset string) ([]string, error) { out, err := d.runTool("snapshot", "list", "--no-headers", "--parsable", "-r", "-o", "clones", dataset) if err != nil { return nil, err } clones := []string{} for _, line := range strings.Split(out, "\n") { line = strings.TrimSpace(line) if line == dataset || line == "" || line == "-" { continue } line = strings.TrimPrefix(line, fmt.Sprintf("%s/", dataset)) clones = append(clones, line) } return clones, nil } func (d *truenas) randomVolumeName(vol Volume) string { return fmt.Sprintf("%s_%s", vol.name, uuid.New().String()) } incus-7.0.0/internal/server/storage/drivers/driver_truenas_volumes.go000066400000000000000000002003271517523235500262540ustar00rootroot00000000000000package drivers import ( "bufio" "errors" "fmt" "io" "io/fs" "os" "strconv" "strings" "time" "github.com/google/uuid" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/server/backup" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // CreateVolume creates an empty volume and can optionally fill it by executing the supplied // filler function. func (d *truenas) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Operation) error { // Revert handling reverter := revert.New() defer reverter.Fail() if vol.contentType == ContentTypeFS { // Create mountpoint. err := vol.EnsureMountPath(true) if err != nil { return err } reverter.Add(func() { _ = os.Remove(vol.MountPath()) }) } // Look for previously deleted images. (don't look for underlying, or we'll look after we've looked) if vol.volType == VolumeTypeImage { dataset := d.dataset(vol, true) exists, err := d.datasetExists(dataset) if err != nil { return err } if exists { canRestore := true // check if the cached image volume is larger than the current pool volume.size setting (if so we won't be // able to resize the snapshot to that the smaller size later). volSize, err := d.getDatasetProperty(dataset, "volsize") if err != nil { return err } volSizeBytes, err := strconv.ParseInt(volSize, 10, 64) if err != nil { return err } poolVolSize := DefaultBlockSize if vol.poolConfig["volume.size"] != "" { poolVolSize = vol.poolConfig["volume.size"] } poolVolSizeBytes, err := units.ParseByteSizeString(poolVolSize) if err != nil { return err } // Round to block boundary. poolVolSizeBytes, err = d.roundVolumeBlockSizeBytes(vol, poolVolSizeBytes) if err != nil { return err } // If the cached volume size is different than the pool volume size, then we can't use the // deleted cached image volume and instead we will rename it to a random UUID so it can't // be restored in the future and a new cached image volume will be created instead. if volSizeBytes != poolVolSizeBytes { d.logger.Debug("Renaming deleted cached image volume so that regeneration is used", logger.Ctx{"fingerprint": vol.Name()}) randomVol := NewVolume(d, d.name, vol.volType, vol.contentType, d.randomVolumeName(vol), vol.config, vol.poolConfig) _, err := d.runTool("dataset", "rename", dataset, d.dataset(randomVol, true)) if err != nil { return err } if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() randomFsVol := randomVol.NewVMBlockFilesystemVolume() _, err := d.runTool("dataset", "rename", d.dataset(fsVol, true), d.dataset(randomFsVol, true)) if err != nil { return err } } // We have renamed the deleted cached image volume, so we don't want to try and // restore it. canRestore = false } // Restore the image. if canRestore { d.logger.Debug("Restoring previously deleted cached image volume", logger.Ctx{"fingerprint": vol.Name()}) _, err := d.runTool("dataset", "rename", dataset, d.dataset(vol, false)) if err != nil { return err } // After this point we have a restored image, so setup reverter. reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() _, err := d.runTool("dataset", "rename", d.dataset(fsVol, true), d.dataset(fsVol, false)) if err != nil { return err } // no need for reverter.add here as we have succeeded } reverter.Success() return nil } } } var opts []string // Add custom property incus:content_type which allows distinguishing between regular volumes, block_mode enabled volumes, and ISO volumes. if vol.volType == VolumeTypeCustom { opts = append(opts, fmt.Sprintf("user-props=incus:content_type=%s", vol.contentType)) } blockSize := vol.ExpandedConfig("truenas.blocksize") if blockSize != "" { // Convert to bytes. sizeBytes, err := units.ParseByteSizeString(blockSize) if err != nil { return err } // volblocksize maximum value is 128KiB so if the value of truenas.blocksize is bigger set it to 128KiB. if sizeBytes > zfsMaxVolBlocksize { sizeBytes = zfsMaxVolBlocksize } opts = append(opts, fmt.Sprintf("volblocksize=%d", sizeBytes)) } sizeBytes, err := units.ParseByteSizeString(vol.ConfigSize()) if err != nil { return err } sizeBytes, err = d.roundVolumeBlockSizeBytes(vol, sizeBytes) if err != nil { return err } dataset := d.dataset(vol, false) // Create the volume dataset. err = d.createVolume(dataset, sizeBytes, opts...) if err != nil { return err } // After this point we'll have a volume, so setup reverter. reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) err = d.createIscsiShare(dataset, false) if err != nil { return err } if vol.contentType == ContentTypeFS { // activateIscsiDataset does not check if the dataset has been activated. // devPath, err := d.activateIscsiDataset(dataset) _, devPath, err := d.locateOrActivateIscsiDataset(dataset) if err != nil { return err } fsVolFilesystem := vol.ConfigBlockFilesystem() _, err = makeFSType(devPath, fsVolFilesystem, nil) // de-activate even if there is an err err2 := d.deactivateIscsiDataset(dataset) if err != nil { return err } if err2 != nil { return err2 } } // For VM images, create a filesystem volume too. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.CreateVolume(fsVol, nil, op) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolume(fsVol, op) }) } err = vol.MountTask(func(mountPath string, op *operations.Operation) error { // Run the volume filler function if supplied. if filler != nil && filler.Fill != nil { var err error var devPath string if IsContentBlock(vol.contentType) { // Get the device path. devPath, err = d.GetVolumeDiskPath(vol) if err != nil { return err } } allowUnsafeResize := false if vol.volType == VolumeTypeImage { // Allow filler to resize initial image volume as needed. // Some storage drivers don't normally allow image volumes to be resized due to // them having read-only snapshots that cannot be resized. However when creating // the initial image volume and filling it before the snapshot is taken resizing // can be allowed and is required in order to support unpacking images larger than // the default volume size. The filler function is still expected to obey any // volume size restrictions configured on the pool. // Unsafe resize is also needed to disable filesystem resize safety checks. // This is safe because if for some reason an error occurs the volume will be // discarded rather than leaving a corrupt filesystem. allowUnsafeResize = true } // Run the filler. err = genericRunFiller(d, vol, devPath, filler, allowUnsafeResize) if err != nil { return err } // Move the GPT alt header to end of disk if needed. if vol.IsVMBlock() { err = d.moveGPTAltHeader(devPath) if err != nil { return err } } } if vol.contentType == ContentTypeFS { // Run EnsureMountPath again after mounting and filling to ensure the mount directory has // the correct permissions set. err := vol.EnsureMountPath(true) if err != nil { return err } } return nil }, op) if err != nil { return err } // Setup snapshot and unset mountpoint on image. if vol.volType == VolumeTypeImage { // ideally, we don't want to snap the underlying when we create the img, but rather after we've unpacked. // note: we may need to sync the underlying filesystem, it depends if its still mounted, I think it shouldn't be. dataset := d.dataset(vol, false) snapName := fmt.Sprintf("%s@readonly", dataset) // Create snapshot of the main dataset. err := d.createSnapshot(snapName, false) if err != nil { return err } if vol.contentType == ContentTypeBlock { // Re-create the FS config volume's readonly snapshot now that the filler function has run // and unpacked into both config and block volumes. fsVol := vol.NewVMBlockFilesystemVolume() snapName = fmt.Sprintf("%s@readonly", d.dataset(fsVol, false)) err := d.createSnapshot(snapName, true) // delete, then snap. if err != nil { return err } } } // All done. reverter.Success() return nil } // CreateVolumeFromBackup re-creates a volume from its exported state. func (d *truenas) CreateVolumeFromBackup(vol Volume, srcBackup backup.Info, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) (VolumePostHook, revert.Hook, error) { // TODO: optimized version return genericVFSBackupUnpack(d, d.state.OS, vol, srcBackup.Snapshots, srcData, basePrefix, op) } // same as CreateVolumeFromCopy, but will refresh if refresh is true. func (d *truenas) createOrRefeshVolumeFromCopy(vol Volume, srcVol Volume, refresh bool, copySnapshots bool, allowInconsistent bool, op *operations.Operation) error { var err error // Revert handling reverter := revert.New() defer reverter.Fail() if vol.contentType == ContentTypeFS { // Create mountpoint. err = vol.EnsureMountPath(false) if err != nil { return err } reverter.Add(func() { _ = os.Remove(vol.MountPath()) }) } // For VMs, also copy the filesystem dataset. if vol.IsVMBlock() { // For VMs, also copy the filesystem volume. srcFSVol := srcVol.NewVMBlockFilesystemVolume() fsVol := vol.NewVMBlockFilesystemVolume() err = d.createOrRefeshVolumeFromCopy(fsVol, srcFSVol, refresh, copySnapshots, false, op) if err != nil { return err } // Delete on revert. if !refresh { reverter.Add(func() { _ = d.DeleteVolume(fsVol, op) }) } } // Retrieve snapshots on the source. snapshots := []string{} if !srcVol.IsSnapshot() && copySnapshots { snapshots, err = d.VolumeSnapshots(srcVol, op) if err != nil { return err } } // When not allowing inconsistent copies and the volume has a mounted filesystem, we must ensure it is // consistent by syncing and freezing the filesystem to ensure unwritten pages are flushed and that no // further modifications occur while taking the source snapshot. var unfreezeFS func() error sourcePath := srcVol.MountPath() if !allowInconsistent && srcVol.contentType == ContentTypeFS && linux.IsMountPoint(sourcePath) { unfreezeFS, err = d.filesystemFreeze(sourcePath) if err != nil { return err } reverter.Add(func() { _ = unfreezeFS() }) } srcDataset := d.dataset(srcVol, false) var srcSnapshot string if srcVol.volType == VolumeTypeImage { srcSnapshot = fmt.Sprintf("%s@readonly", srcDataset) } else if srcVol.IsSnapshot() { srcSnapshot = srcDataset } else { // Create a new snapshot for copy. srcSnapshot = fmt.Sprintf("%s@copy-%s", srcDataset, uuid.New().String()) err := d.createSnapshot(srcSnapshot, false) if err != nil { return err } // If truenas.clone_copy is disabled delete the snapshot at the end. if util.IsFalse(d.config["truenas.clone_copy"]) || len(snapshots) > 0 { // Delete the snapshot at the end. defer func() { // Delete snapshot (or mark for deferred deletion if cannot be deleted currently). err = d.deleteSnapshot(srcSnapshot, true, "defer") if err != nil { d.logger.Warn("Failed deleting temporary snapshot for copy", logger.Ctx{"snapshot": srcSnapshot, "err": err}) } }() } else { // Delete the snapshot on revert. reverter.Add(func() { // Delete snapshot (or mark for deferred deletion if cannot be deleted currently). err = d.deleteSnapshot(srcSnapshot, true, "defer") if err != nil { d.logger.Warn("Failed deleting temporary snapshot for copy", logger.Ctx{"snapshot": srcSnapshot, "err": err}) } }) } } // Now that source snapshot has been taken we can safely unfreeze the source filesystem. if unfreezeFS != nil { _ = unfreezeFS() } // Delete the volume created on failure. if !refresh { reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) } destDataset := d.dataset(vol, false) // If truenas.clone_copy is disabled or source volume has snapshots, then use full copy mode. if util.IsFalse(d.config["truenas.clone_copy"]) || len(snapshots) > 0 { // Run the replication, snaps + copy- snap. TODO: verify necessary props are replicated. args := []string{"replication", "start", "--recursive", "--readonly-policy=ignore"} if refresh { /* refresh is essentially an optimized form of replace. refresh implies that we may have a dest already, and since the source may be unrelated, we may need to replicate from scratch. The retention policy ensures obsoleted snaps are removed from the dest. */ args = append(args, "--retention-policy=source", "--allow-from-scratch=true") } /* instead of using full replication, and then removing snapshots, we instead take advantage of the replication task's ability to filter snapshots as they are sent. */ snapName := strings.SplitN(srcSnapshot, "@", 2)[1] snapRegex := fmt.Sprintf("(snapshot-.*|%s)", snapName) args = append(args, "--name-regex", snapRegex, srcDataset, destDataset) _, err := d.runTool(args...) if err != nil { return fmt.Errorf("Failed to replicate dataset: %w", err) } // Delete the copy- snapshot on the dest. err = d.deleteSnapshot(fmt.Sprintf("%s@%s", destDataset, snapName), true) if err != nil { return err } } else { // Perform volume clone. err = d.cloneSnapshot(srcSnapshot, destDataset) if err != nil { return err } // Note: user props aren't cloned, so we re-add the content_type if necessary if vol.volType == VolumeTypeCustom { // Add custom property incus:content_type which allows distinguishing between regular volumes, block_mode enabled volumes, and ISO volumes. props := fmt.Sprintf("user-props=incus:content_type=%s", vol.contentType) // TODO: this needs to be better. err = d.setDatasetProperties(destDataset, props) if err != nil { return err } } } // and share the clone/copy. err = d.createIscsiShare(destDataset, false) if err != nil { return err } // Apply the properties. if vol.contentType == ContentTypeFS { if renegerateFilesystemUUIDNeeded(vol.ConfigBlockFilesystem()) { // regen must be done with vol unmounted. _, volPath, err := d.activateVolume(vol) if err != nil { return err } d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": volPath, "fs": vol.ConfigBlockFilesystem()}) err = regenerateFilesystemUUID(vol.ConfigBlockFilesystem(), volPath) if err != nil { return err } } // Mount the volume and ensure the permissions are set correctly inside the mounted volume. err := vol.MountTask(func(_ string, _ *operations.Operation) error { return vol.EnsureMountPath(false) }, op) if err != nil { return err } } // Pass allowUnsafeResize as true when resizing block backed filesystem volumes because we want to allow // the filesystem to be shrunk as small as possible without needing the safety checks that would prevent // leaving the filesystem in an inconsistent state if the resize couldn't be completed. This is because if // the resize fails we will delete the volume anyway so don't have to worry about it being inconsistent. var allowUnsafeResize bool if vol.contentType == ContentTypeFS { allowUnsafeResize = true } // Resize volume to the size specified. Only uses volume "size" property and does not use pool/defaults // to give the caller more control over the size being used. err = d.SetVolumeQuota(vol, vol.config["size"], allowUnsafeResize, op) if err != nil { return err } // All done. reverter.Success() return nil } // CreateVolumeFromCopy provides same-pool volume copying functionality. func (d *truenas) CreateVolumeFromCopy(vol Volume, srcVol Volume, copySnapshots bool, allowInconsistent bool, op *operations.Operation) error { return d.createOrRefeshVolumeFromCopy(vol, srcVol, false, copySnapshots, allowInconsistent, op) // not refreshing. } // CreateVolumeFromMigration creates a volume being sent via a migration. TODO: need to ensure that incus:content_type is copied. func (d *truenas) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, volTargetArgs localMigration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error { if volTargetArgs.ClusterMoveSourceName != "" && volTargetArgs.StoragePool == "" { d.logger.Debug("Detected migration between cluster members on the same storage pool") err := vol.EnsureMountPath(false) if err != nil { return err } if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.CreateVolumeFromMigration(fsVol, conn, volTargetArgs, preFiller, op) if err != nil { return err } } return nil } // Handle simple rsync and block_and_rsync through generic. if volTargetArgs.MigrationType.FSType == migration.MigrationFSType_RSYNC || volTargetArgs.MigrationType.FSType == migration.MigrationFSType_BLOCK_AND_RSYNC { return genericVFSCreateVolumeFromMigration(d, nil, vol, conn, volTargetArgs, preFiller, op) } // TODO: optimized migration return ErrNotSupported } // RefreshVolume updates an existing volume to match the state of another. func (d *truenas) RefreshVolume(vol Volume, srcVol Volume, srcSnapshots []Volume, allowInconsistent bool, op *operations.Operation) error { var err error var targetSnapshots []Volume var srcSnapshotsAll []Volume if !srcVol.IsSnapshot() { // Get target snapshots targetSnapshots, err = vol.Snapshots(op) if err != nil { return fmt.Errorf("Failed to get target snapshots: %w", err) } srcSnapshotsAll, err = srcVol.Snapshots(op) if err != nil { return fmt.Errorf("Failed to get source snapshots: %w", err) } } // If there are no source or target snapshots, perform a simple replacement copy if len(srcSnapshotsAll) == 0 || len(targetSnapshots) == 0 { // this ensures that recursive deletions are performed. err = d.DeleteVolume(vol, op) if err != nil { return err } return d.CreateVolumeFromCopy(vol, srcVol, len(srcSnapshotsAll) == 0, false, op) } // repl task can "refresh" return d.createOrRefeshVolumeFromCopy(vol, srcVol, true, true, false, op) } // DeleteVolume deletes a volume of the storage device. If any snapshots of the volume remain then // this function will return an error. // For image volumes, both filesystem and block volumes will be removed. func (d *truenas) DeleteVolume(vol Volume, op *operations.Operation) error { if vol.volType == VolumeTypeImage && vol.contentType == ContentTypeFS { // deletes all block.filesystem permutations return d.deleteImageFsVolume(vol, op) } return d.deleteVolume(vol, nil, op) } // deleteImageFsVolume efficiently deletes all filesystem variations of an ImageFS (use for vol.volType == VolumeTypeImage && vol.contentType == ContentTypeFS ). func (d *truenas) deleteImageFsVolume(vol Volume, op *operations.Operation) error { if vol.volType != VolumeTypeImage || vol.contentType != ContentTypeFS { return fmt.Errorf("deleteImageFsVolume called on invalid volume: %v", vol) } /* the basic idea is to avoid the iterative existence checks for each filesystem, since we expect all but one not to exist */ // We need to clone vol the otherwise changing `block.filesystem` in tmpVol will also change it in vol. tmpVol := vol.Clone() // form a list of FSs without the actual volume's FS. fsList := []string{} volFs := vol.ConfigBlockFilesystem() for _, filesystem := range blockBackedAllowedFilesystems { if filesystem == volFs { continue } fsList = append(fsList, filesystem) } // generate a list of all the datasets to be existence checked datasets := []string{d.dataset(vol, false)} for _, filesystem := range fsList { tmpVol.config["block.filesystem"] = filesystem datasets = append(datasets, d.dataset(tmpVol, false)) } // returns a map of all the datasets existence, including those that don't exist. existsMap, err := d.objectsExist(datasets, "dataset") if err != nil { return fmt.Errorf("Unable to verify existence of FS Images, Error: %w", err) } // delete all the other file systems for _, filesystem := range fsList { tmpVol.config["block.filesystem"] = filesystem dataset := d.dataset(tmpVol, false) exists, ok := existsMap[dataset] if ok && exists { _ = d.deleteVolume(tmpVol, &exists, op) } } // and finally, delete the actual volume, with whatever its FS is that we specifically skipped earlier. dataset := d.dataset(vol, false) exists, ok := existsMap[dataset] if !ok { return fmt.Errorf("Unable to retrieve existence of FS Image: %s", dataset) } // cleans up mount points etc err = d.deleteVolume(vol, &exists, op) if err != nil { return err } return nil } // deleteVolume deletes the volume if it exists, and cleans up, pass optionalExistance if you know. func (d *truenas) deleteVolume(vol Volume, optionalExistance *bool, op *operations.Operation) error { // Check that we have a dataset to delete. dataset := d.dataset(vol, false) var exists bool // allows performing bulk existence checks. if optionalExistance != nil { exists = *optionalExistance } else { e, err := d.datasetExists(dataset) if err != nil { return err } exists = e // declared and not used: exists } if exists { // Deleted volumes do not need shares _ = d.deleteIscsiShare(dataset) // will implicitly deactivate, if activated. // Handle clones. clones, err := d.getClones(dataset) if err != nil { return err } if len(clones) > 0 { // Move to the deleted path. err := d.renameDataset(dataset, d.dataset(vol, true), false) if err != nil { return err } } else { err := d.deleteDatasetRecursive(dataset) if err != nil { return err } } } if vol.contentType == ContentTypeFS { // Delete the mountpoint if present. err := os.Remove(vol.MountPath()) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", vol.MountPath(), err) } // Delete the snapshot storage. err = os.RemoveAll(GetVolumeSnapshotDir(d.name, vol.volType, vol.name)) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", GetVolumeSnapshotDir(d.name, vol.volType, vol.name), err) } } // For VMs, also delete the filesystem dataset. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.DeleteVolume(fsVol, op) if err != nil { return err } } return nil } // HasVolume indicates whether a specific volume exists on the storage pool. func (d *truenas) HasVolume(vol Volume) (bool, error) { // Check if the dataset exists. dataset := d.dataset(vol, false) return d.datasetExists(dataset) } // ValidateTrueNasVolBlocksize validates blocksize property value on the pool, matches volblocksize. func ValidateTrueNasVolBlocksize(value string) error { /* For volumes, specifies the block size of the volume. The blocksize cannot be changed once the volume has been written, so it should be set at volume creation time. The default blocksize for volumes is 16 KiB. Any power of 2 from 512 bytes to 128 KiB is valid. */ // Convert to bytes. sizeBytes, err := units.ParseByteSizeString(value) if err != nil { return err } if sizeBytes < zfsMinBlocksize || sizeBytes > zfsMaxVolBlocksize || (sizeBytes&(sizeBytes-1)) != 0 { return errors.New("Value should be between 512B and 128KiB, and be power of 2") } return nil } // commonVolumeRules returns validation rules which are common for pool and volume. func (d *truenas) commonVolumeRules() map[string]func(value string) error { return map[string]func(value string) error{ // gendoc:generate(entity=storage_volume_truenas, group=common, key=block.filesystem) // // --- // type: string // condition: - // default: same as `volume.block.filesystem` // shortdesc: {{block_filesystem}} "block.filesystem": validate.Optional(validate.IsOneOf(blockBackedAllowedFilesystems...)), // gendoc:generate(entity=storage_volume_truenas, group=common, key=block.mount_options) // // --- // type: string // condition: - // default: same as `volume.block.mount_options` // shortdesc: Mount options for block-backed file system volumes "block.mount_options": validate.IsAny, // gendoc:generate(entity=storage_volume_truenas, group=common, key=truenas.blocksize) // // --- // type: string // condition: - // default: same as `volume.truenas.blocksize` // shortdesc: Size of the ZFS block in range from 512 bytes to 16 MiB (must be power of 2) - for block volume, a maximum value of 128 KiB will be used even if a higher value is set "truenas.blocksize": validate.Optional(ValidateTrueNasVolBlocksize), // used for volblocksize only. NOTE: zfs.blocksize is hard-coded in backend.shouldUseOptimizedImage... // gendoc:generate(entity=storage_volume_truenas, group=common, key=truenas.remove_snapshots) // // --- // type: bool // condition: - // default: same as `volume.truenas.remove_snapshots` or `false` // shortdesc: Remove snapshots as needed "truenas.remove_snapshots": validate.Optional(validate.IsBool), // gendoc:generate(entity=storage_volume_truenas, group=common, key=truenas.use_refquota) // // --- // type: bool // condition: - // default: same as `volume.truenas.use_refquota` or `false` // shortdesc: Use `refquota` instead of `quota` for space "truenas.use_refquota": validate.Optional(validate.IsBool), } } // ValidateVolume validates the supplied volume config. func (d *truenas) ValidateVolume(vol Volume, removeUnknownKeys bool) error { // gendoc:generate(entity=storage_volume_truenas, group=common, key=initial.gid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.gid` or `0` // shortdesc: GID of the volume owner in the instance // gendoc:generate(entity=storage_volume_truenas, group=common, key=initial.mode) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.mode` or `711` // shortdesc: Mode of the volume in the instance // gendoc:generate(entity=storage_volume_truenas, group=common, key=initial.uid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.uid` or `0` // shortdesc: UID of the volume owner in the instance // gendoc:generate(entity=storage_volume_truenas, group=common, key=security.shared) // // --- // type: bool // condition: custom block volume // default: same as `volume.security.shared` or `false` // shortdesc: Enable sharing the volume across multiple instances // gendoc:generate(entity=storage_volume_truenas, group=common, key=security.shifted) // // --- // type: bool // condition: custom volume // default: same as `volume.security.shifted` or `false` // shortdesc: {{enable_ID_shifting}} // gendoc:generate(entity=storage_volume_truenas, group=common, key=security.unmapped) // // --- // type: bool // condition: custom volume // default: same as `volume.security.unmapped` or `false` // shortdesc: Disable ID mapping for the volume // gendoc:generate(entity=storage_volume_truenas, group=common, key=size) // // --- // type: string // condition: appropriate driver // default: same as `volume.size` // shortdesc: Size/quota of the storage volume // gendoc:generate(entity=storage_volume_truenas, group=common, key=snapshots.expiry) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_truenas, group=common, key=snapshots.expiry.manual) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry.manual` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_truenas, group=common, key=snapshots.pattern) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.pattern` or `snap%d` // shortdesc: {{snapshot_pattern_format}} // gendoc:generate(entity=storage_volume_truenas, group=common, key=snapshots.schedule) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.schedule` // shortdesc: {{snapshot_schedule_format}} commonRules := d.commonVolumeRules() // Disallow block.* settings for regular custom block volumes. These settings only make sense // when using custom filesystem volumes. Incus will create the filesystem // for these volumes, and use the mount options. When attaching a regular block volume to a VM, // these are not mounted by Incus and therefore don't need these config keys. if vol.IsVMBlock() || vol.volType == VolumeTypeCustom && vol.contentType == ContentTypeBlock { delete(commonRules, "block.filesystem") delete(commonRules, "block.mount_options") } return d.validateVolume(vol, commonRules, removeUnknownKeys) } // UpdateVolume applies config changes to the volume. func (d *truenas) UpdateVolume(vol Volume, changedConfig map[string]string) error { // Mangle the current volume to its old values. old := make(map[string]string) for k, v := range changedConfig { if k == "size" || k == "truenas.use_refquota" { old[k] = vol.config[k] vol.config[k] = v } } defer func() { for k, v := range old { vol.config[k] = v } }() // If any of the relevant keys changed, re-apply the quota. if len(old) != 0 { err := d.SetVolumeQuota(vol, vol.ExpandedConfig("size"), false, nil) if err != nil { return err } } return d.updateVolume(vol, changedConfig) } // GetVolumeUsage returns the disk space used by the volume. func (d *truenas) GetVolumeUsage(vol Volume) (int64, error) { // Determine what key to use. key := "used" // If volume isn't snapshot then we can take into account the truenas.use_refquota setting. // Snapshots should also use the "used" ZFS property because the snapshot usage size represents the CoW // usage not the size of the snapshot volume. if !vol.IsSnapshot() { if util.IsTrue(vol.ExpandedConfig("truenas.use_refquota")) { key = "referenced" } // Shortcut for mounted refquota filesystems. if key == "referenced" && vol.contentType == ContentTypeFS && linux.IsMountPoint(vol.MountPath()) { var stat unix.Statfs_t err := unix.Statfs(vol.MountPath(), &stat) if err != nil { return -1, err } return int64(stat.Blocks-stat.Bfree) * int64(stat.Bsize), nil } } // Get the current value. value, err := d.getDatasetProperty(d.dataset(vol, false), key) if err != nil { return -1, err } // Convert to int. valueInt, err := strconv.ParseInt(value, 10, 64) if err != nil { return -1, err } return valueInt, nil } // SetVolumeQuota sets the quota/reservation on the volume. // Does nothing if supplied with an empty/zero size for block volumes. func (d *truenas) SetVolumeQuota(vol Volume, size string, allowUnsafeResize bool, op *operations.Operation) error { // Convert to bytes. sizeBytes, err := units.ParseByteSizeString(size) if err != nil { return err } inUse := vol.MountInUse() dataset := d.dataset(vol, false) // always zvols with blockbacking. // Do nothing if size isn't specified. if sizeBytes <= 0 { return nil } sizeBytes, err = d.roundVolumeBlockSizeBytes(vol, sizeBytes) if err != nil { return err } oldSizeBytesStr, err := d.getDatasetProperty(dataset, "volsize") if err != nil { return err } oldVolSizeBytesInt, err := strconv.ParseInt(oldSizeBytesStr, 10, 64) if err != nil { return err } oldVolSizeBytes := int64(oldVolSizeBytesInt) if oldVolSizeBytes == sizeBytes { return nil } if vol.contentType == ContentTypeFS { if vol.volType == VolumeTypeImage { return fmt.Errorf("Image volumes cannot be resized: %w", ErrCannotBeShrunk) } fsType := vol.ConfigBlockFilesystem() l := d.logger.AddContext(logger.Ctx{"vol": vol, "size": fmt.Sprintf("%db", sizeBytes)}) if sizeBytes < oldVolSizeBytes { if !filesystemTypeCanBeShrunk(fsType) { return fmt.Errorf("Filesystem %q cannot be shrunk: %w", fsType, ErrCannotBeShrunk) } if inUse { return ErrInUse // We don't allow online shrinking of filesystem block volumes. } // Activate volume if needed. activated, volDevPath, err := d.activateVolume(vol) if err != nil { return err } if activated { defer func() { _, _ = d.deactivateVolume(vol) }() } // Shrink filesystem first. // Pass allowUnsafeResize to allow disabling of filesystem resize safety checks. err = shrinkFileSystem(fsType, volDevPath, vol, sizeBytes, allowUnsafeResize) if err != nil { return err } l.Debug("TrueNAS volume filesystem shrunk") // Shrink the block device. err = d.setVolsize(dataset, sizeBytes, true) // allow shrink, shrink errors will be ignored. if err != nil { return err } } else if sizeBytes > oldVolSizeBytes { // Grow block device first, ignoring any shrink errors, which could happen because we've already ignored a shrink error when shrinking. err = d.setVolsize(dataset, sizeBytes, false) if err != nil { return err } // Activate volume after resizing the device. activated, volDevPath, err := d.activateVolume(vol) if err != nil { return err } if activated { defer func() { _, _ = d.deactivateVolume(vol) }() } // Verify the device actually grew. actualSize, err := BlockDiskSizeBytes(volDevPath) if err != nil { return err } if actualSize < sizeBytes { // refresh until it does actually grow for range 20 { // rescan iscsi devices to pickup any size change err = d.refreshIscsiBus() if err != nil { return err } // verify the device actually grew. actualSize, err = BlockDiskSizeBytes(volDevPath) if err != nil { return err } if actualSize >= sizeBytes { break } time.Sleep(100 * time.Millisecond) } } // Verify the device actually grew. if actualSize < sizeBytes { return fmt.Errorf("device %s could not be grown", volDevPath) } // Grow the filesystem to fill block device. err = growFileSystem(fsType, volDevPath, vol) if err != nil { return err } l.Debug("TrueNAS volume filesystem grown") } } else { // Block Volume. // Block image volumes cannot be resized because they have a readonly snapshot that doesn't get // updated when the volume's size is changed, and this is what instances are created from. // During initial volume fill allowUnsafeResize is enabled because snapshot hasn't been taken yet. if !allowUnsafeResize && vol.volType == VolumeTypeImage { return ErrNotSupported } // Only perform pre-resize checks if we are not in "unsafe" mode. // In unsafe mode we expect the caller to know what they are doing and understand the risks. if !allowUnsafeResize { if sizeBytes < oldVolSizeBytes { return fmt.Errorf("Block volumes cannot be shrunk: %w", ErrCannotBeShrunk) } } // Adjust zvol size err = d.setVolsize(dataset, sizeBytes, true) if err != nil { return err } } // Move the VM GPT alt header to end of disk if needed (not needed in unsafe resize mode as // it is expected the caller will do all necessary post resize actions themselves). if vol.IsVMBlock() && !allowUnsafeResize { err = vol.MountTask(func(mountPath string, op *operations.Operation) error { devPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } return d.moveGPTAltHeader(devPath) }, op) if err != nil { return err } } return nil } // getTempSnapshotVolName returns a derived volume name for the server specific clone of the specified snapshot volume. func (d *truenas) getTempSnapshotVolName(vol Volume) string { parent, snapshotOnlyName, _ := api.GetParentAndSnapshotName(vol.Name()) parentVol := NewVolume(d, d.Name(), vol.volType, vol.contentType, parent, vol.config, vol.poolConfig) parentDataset := d.dataset(parentVol, false) // serverName to allow other cluster members to mount the same snapshot at the same time. dataset := fmt.Sprintf("%s_%s_%s-%d%s", parentDataset, snapshotOnlyName, d.state.ServerName, os.Getpid(), tmpVolSuffix) return dataset } // GetVolumeDiskPath returns the location of a root disk block device. func (d *truenas) GetVolumeDiskPath(vol Volume) (string, error) { var dataset string if vol.IsSnapshot() { dataset = d.getTempSnapshotVolName(vol) } else { dataset = d.dataset(vol, false) } return d.locateIscsiDataset(dataset) } // ListVolumes returns a list of volumes in storage pool. func (d *truenas) ListVolumes() ([]Volume, error) { vols := make(map[string]Volume) _ = vols /* from backend.ListUnknownVolumes // Get a list of volumes on the storage pool. We only expect to get 1 volume per logical Incus volume. // So for VMs we only expect to get the block volume for a VM and not its filesystem one too. This way we // can operate on the volume using the existing storage pool functions and let the pool then handle the // associated filesystem volume as needed. */ // Get just filesystem and volume datasets, not snapshots. // The ZFS driver uses two approaches to indicating block volumes; firstly for VM and image volumes it // creates both a filesystem dataset and an associated volume ending in zfsBlockVolSuffix. // However for custom block volumes it does not also end the volume name in zfsBlockVolSuffix (unlike the // LVM and Ceph drivers), so we must also retrieve the dataset type here and look for "volume" types // which also indicate this is a block volume. out, err := d.runTool("list", "--no-headers", "-o", "name,incus:content_type", "-r", "-t", "volume", d.config["truenas.dataset"]) if err != nil { return nil, err } scanner := bufio.NewScanner(strings.NewReader(out)) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) parts := strings.Split(line, "\t") if len(parts) != 2 { return nil, fmt.Errorf("Unexpected volume line %q", line) } zfsVolName := parts[0] incusContentType := parts[1] var volType VolumeType var volName string var volFs string for _, volumeType := range d.Info().VolumeTypes { prefix := fmt.Sprintf("%s/%s/", d.config["truenas.dataset"], volumeType) if strings.HasPrefix(zfsVolName, prefix) { volType = volumeType volName = strings.TrimPrefix(zfsVolName, prefix) } } if volType == "" { d.logger.Debug("Ignoring unrecognised volume type", logger.Ctx{"name": zfsVolName}) continue // Ignore unrecognised volume. } contentType := ContentTypeFS if volType == VolumeTypeVM && !strings.HasSuffix(volName, zfsBlockVolSuffix) { continue // Ignore VM filesystem volumes as we will just return the VM's block volume. } if volType == VolumeTypeCustom && strings.HasSuffix(volName, zfsISOVolSuffix) { contentType = ContentTypeISO volName = strings.TrimSuffix(volName, zfsISOVolSuffix) } else if volType == VolumeTypeVM || (volType == VolumeTypeImage && strings.HasSuffix(volName, zfsBlockVolSuffix)) { contentType = ContentTypeBlock volName = strings.TrimSuffix(volName, zfsBlockVolSuffix) } // FS images have the FS encoded after a _ separator if volType == VolumeTypeImage && strings.Contains(volName, "_") { volName, volFs, _ = strings.Cut(volName, "_") } // If a new volume has been found, or the volume will replace an existing image filesystem volume // then proceed to add the volume to the map. We allow image volumes to overwrite existing // filesystem volumes of the same name so that for VM images we only return the block content type // volume (so that only the single "logical" volume is returned). existingVol, foundExisting := vols[volName] if !foundExisting || (existingVol.Type() == VolumeTypeImage && existingVol.ContentType() == ContentTypeFS) { v := NewVolume(d, d.name, volType, contentType, volName, make(map[string]string), d.config) if volFs != "" { v.config["block.filesystem"] = volFs } // Get correct content type from incus:content_type property. if incusContentType != "-" { v.contentType = ContentType(incusContentType) } /* if its a filesystem, we need to probe it, unless we know the fs, but VMBlock's have an implicit filesystem Volume, and that Volume inherits the probe setting from the block volume. */ if (v.contentType == ContentTypeFS && volFs == "") || v.IsVMBlock() { v.SetMountFilesystemProbe(true) } vols[volName] = v continue } return nil, fmt.Errorf("Unexpected duplicate volume %q found", volName) } volList := make([]Volume, 0, len(vols)) for _, v := range vols { volList = append(volList, v) } return volList, nil } // activateVolume activates a ZFS volume if not already active. Returns true if activated, false if not. func (d *truenas) activateVolume(vol Volume) (bool, string, error) { if !IsContentBlock(vol.contentType) && !vol.IsBlockBacked() { return false, "", nil // Nothing to do for non-block or non-block backed volumes. } dataset := d.dataset(vol, false) // Check if already active. didActivate, devPath, err := d.locateOrActivateIscsiDataset(dataset) if err != nil { return false, "", err } if didActivate { d.logger.Debug("Activated TrueNAS volume", logger.Ctx{"volName": vol.Name(), "dev": dataset}) } return didActivate, devPath, nil } // deactivateVolume deactivates a ZFS volume if activate. Returns true if deactivated, false if not. func (d *truenas) deactivateVolume(vol Volume) (bool, error) { if vol.contentType != ContentTypeBlock && !vol.IsBlockBacked() { return false, nil // Nothing to do for non-block and non-block backed volumes. } dataset := d.dataset(vol, false) // Check if currently active. didDeactivate, err := d.deactivateIscsiDatasetIfActive(dataset) if err != nil { return false, fmt.Errorf("Failed deactivating TrueNAS volume: %w", err) } if didDeactivate { d.logger.Debug("Deactivated TrueNAS volume", logger.Ctx{"volName": vol.name, "dev": dataset}) } return didDeactivate, nil } // ActivateTask allows running a function while the volume is active (but not mounted). func (d *truenas) ActivateTask(vol Volume, task func(devPath string, op *operations.Operation) error, op *operations.Operation) error { // Prevent concurrent mounting actions. unlock, err := vol.MountLock() if err != nil { return err } defer unlock() // Setup a reverter. reverter := revert.New() defer reverter.Fail() // Activate the volume. activated, volDevPath, err := d.activateVolume(vol) if err != nil { return err } if !activated { return errors.New("Volume is already active, can't run exclusive activation task") } // Run the task. taskErr := task(volDevPath, op) // Deactivate the volume. _, err = d.deactivateVolume(vol) if err != nil { return err } return taskErr } // MountVolume mounts a volume and increments ref counter. Please call UnmountVolume() when done with the volume. func (d *truenas) MountVolume(vol Volume, op *operations.Operation) error { unlock, err := vol.MountLock() if err != nil { return err } defer unlock() reverter := revert.New() defer reverter.Fail() // Activate TrueNAS volume if needed. activated, volDevPath, err := d.activateVolume(vol) if err != nil { return err } if activated { reverter.Add(func() { _, _ = d.deactivateVolume(vol) }) } switch vol.contentType { case ContentTypeFS: mountPath := vol.MountPath() if !linux.IsMountPoint(mountPath) { err := vol.EnsureMountPath(false) if err != nil { return err } fsType := vol.ConfigBlockFilesystem() if vol.mountFilesystemProbe { fsType, err = fsProbe(volDevPath) if err != nil { return fmt.Errorf("Failed probing filesystem: %w", err) } } mountFlags, mountOptions := linux.ResolveMountOptions(strings.Split(vol.ConfigBlockMountOptions(), ",")) err = TryMount(volDevPath, mountPath, fsType, mountFlags, mountOptions) if err != nil { return err } d.logger.Debug("Mounted TrueNAS volume", logger.Ctx{"volName": vol.name, "dev": volDevPath, "path": mountPath, "options": mountOptions}) } case ContentTypeBlock: // For VMs, mount the filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err = d.MountVolume(fsVol, op) if err != nil { return err } } } vol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolume() when done. reverter.Success() return nil } // UnmountVolume simulates unmounting a volume. // keepBlockDev indicates if backing block device should be not be unmapped if volume is unmounted. func (d *truenas) UnmountVolume(vol Volume, keepBlockDev bool, op *operations.Operation) (bool, error) { unlock, err := vol.MountLock() if err != nil { return false, err } defer unlock() ourUnmount := false mountPath := vol.MountPath() refCount := vol.MountRefCountDecrement() // Attempt to unmount the volume. if vol.contentType == ContentTypeFS && linux.IsMountPoint(mountPath) { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": vol.name, "refCount": refCount}) return false, ErrInUse } err := linux.SyncFS(mountPath) if err != nil { return false, fmt.Errorf("Failed syncing filesystem %q: %w", mountPath, err) } err = TryUnmount(mountPath, unix.MNT_DETACH) if err != nil { return false, err } d.logger.Debug("Unmounted TrueNAS volume", logger.Ctx{"volName": vol.name, "path": mountPath, "keepBlockDev": keepBlockDev}) // And deactivate. if !keepBlockDev { _, err = d.deactivateVolume(vol) if err != nil { return false, err } } ourUnmount = true } else if IsContentBlock(vol.contentType) { // For VMs, unmount the filesystem volume. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() ourUnmount, err = d.UnmountVolume(fsVol, false, op) if err != nil { return false, err } } if !keepBlockDev { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": vol.name, "refCount": refCount}) return false, ErrInUse } // and de-activate the block device _, err := d.deactivateVolume(vol) if err != nil { return false, err } } } return ourUnmount, nil } // RenameVolume renames a volume and its snapshots. func (d *truenas) RenameVolume(vol Volume, newVolName string, op *operations.Operation) error { newVol := NewVolume(d, d.name, vol.volType, vol.contentType, newVolName, vol.config, vol.poolConfig) // Revert handling. reverter := revert.New() defer reverter.Fail() // First rename the VFS paths. err := genericVFSRenameVolume(d, vol, newVolName, op) if err != nil { return err } reverter.Add(func() { _ = genericVFSRenameVolume(d, newVol, vol.name, op) }) // Rename the ZFS datasets. err = d.renameDataset(d.dataset(vol, false), d.dataset(newVol, false), true) if err != nil { return err } reverter.Add(func() { _ = d.renameDataset(d.dataset(newVol, false), d.dataset(vol, false), true) }) // For VM images, create a filesystem volume too. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.RenameVolume(fsVol, newVolName, op) if err != nil { return err } reverter.Add(func() { newFsVol := NewVolume(d, d.name, newVol.volType, ContentTypeFS, newVol.name, newVol.config, newVol.poolConfig) _ = d.RenameVolume(newFsVol, vol.name, op) }) } // All done. reverter.Success() return nil } // MigrateVolume sends a volume for migration. func (d *truenas) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *localMigration.VolumeSourceArgs, op *operations.Operation) error { if volSrcArgs.ClusterMove && !volSrcArgs.StorageMove { return nil // When performing a cluster member move don't do anything on the source member. } // Handle simple rsync and block_and_rsync through generic. if volSrcArgs.MigrationType.FSType == migration.MigrationFSType_RSYNC || volSrcArgs.MigrationType.FSType == migration.MigrationFSType_BLOCK_AND_RSYNC { // TODO this should take a temporary snapshot. // Before doing a generic volume migration, we need to ensure volume (or snap volume parent) is // activated to avoid issues activating the snapshot volume device. parent, _, _ := api.GetParentAndSnapshotName(vol.Name()) parentVol := NewVolume(d, d.Name(), vol.volType, vol.contentType, parent, vol.config, vol.poolConfig) err := d.MountVolume(parentVol, op) if err != nil { return err } defer func() { _, _ = d.UnmountVolume(parentVol, false, op) }() return genericVFSMigrateVolume(d, d.state, vol, conn, volSrcArgs, op) } // TODO: optimized migration between TrueNAS or ZFS storage pools? return ErrNotSupported } // BackupVolume creates an exported version of a volume. func (d *truenas) BackupVolume(vol Volume, writer instancewriter.InstanceWriter, basePrefix string, optimized bool, snapshots []string, op *operations.Operation) error { // TODO: we should take a snapshot, and backup from the snapshot for consistency. return genericVFSBackupVolume(d, vol, writer, basePrefix, snapshots, op) } // CreateVolumeSnapshot creates a snapshot of a volume. func (d *truenas) CreateVolumeSnapshot(vol Volume, op *operations.Operation) error { parentName, _, _ := api.GetParentAndSnapshotName(vol.name) // Revert handling. reverter := revert.New() defer reverter.Fail() // Create the parent directory. err := CreateParentSnapshotDirIfMissing(d.name, vol.volType, parentName) if err != nil { return err } // Create snapshot directory. err = vol.EnsureMountPath(false) if err != nil { return err } // Sync the filesystem if vol.contentType == ContentTypeFS { /* We want to ensure the current state is flushed to the server before snapping. Although Incus will Freeze Instances and VMs before Snapshot, then perform a SyncFS on the rootfs, that is only when going via CreateInstanceSnapshot, ie a Custom Volume will miss out as that goes via CreateCustomVolumeSnapshot, and there is no SyncFS. We may as well just sync any mounted filesystem, and if its already been synced there shouldn't be too many changes to flush to the server. In theory, a similar problem can exist with raw devices... and we may want to look at using something similar to `blockdev --flushbufs` to flush the block device before the snap. */ volMountPath := GetVolumeMountPath(vol.pool, vol.volType, parentName) if linux.IsMountPoint(volMountPath) { err := linux.SyncFS(volMountPath) if err != nil { return fmt.Errorf("Failed syncing filesystem %q: %w", volMountPath, err) } } } snapDataset := d.dataset(vol, false) // Sync the device. It may not be enough to just sync the mountpoint... because the device may not be mounted. parentDataset, _, ok := strings.Cut(snapDataset, "@") if ok { devPath, err := d.locateIscsiDataset(parentDataset) if err == nil && devPath != "" { err := linux.SyncFS(devPath) if err != nil { return fmt.Errorf("Failed syncing device %q: %w", devPath, err) } } } // Make the snapshot. err = d.createSnapshot(snapDataset, false) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolumeSnapshot(vol, op) }) // For VM images, create a filesystem volume too. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.CreateVolumeSnapshot(fsVol, op) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolumeSnapshot(fsVol, op) }) } // All done. reverter.Success() return nil } // DeleteVolumeSnapshot removes a snapshot from the storage device. func (d *truenas) DeleteVolumeSnapshot(vol Volume, op *operations.Operation) error { // Delete the snapshot, which will fail if there are clones. dataset := d.dataset(vol, false) errDelete := d.deleteSnapshot(dataset, true) if errDelete != nil { // Handle clones. clones, err := d.getClones(dataset) if err != nil { return err } if len(clones) == 0 { return errDelete } // Move to the deleted path. err = d.renameSnapshot(dataset, d.dataset(vol, true)) if err != nil { return err } } // Delete the mountpoint. err := os.Remove(vol.MountPath()) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", vol.MountPath(), err) } // Remove the parent snapshot directory if this is the last snapshot being removed. parentName, _, _ := api.GetParentAndSnapshotName(vol.name) err = deleteParentSnapshotDirIfEmpty(d.name, vol.volType, parentName) if err != nil { return err } // For VM images, create a filesystem volume too. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.DeleteVolumeSnapshot(fsVol, op) if err != nil { return err } } return nil } // MountVolumeSnapshot mounts a storage volume snapshot. // // The snapshot is cloned to a temporary dataset that will live for the duration of the mount. func (d *truenas) MountVolumeSnapshot(snapVol Volume, op *operations.Operation) error { l := d.logger.AddContext(logger.Ctx{"volume": snapVol.Name()}) l.Debug("Mounting snapshot volume") unlock, err := snapVol.MountLock() if err != nil { return err } defer unlock() reverter := revert.New() defer reverter.Fail() // For VMs, mount the filesystem volume. if snapVol.IsVMBlock() { fsVol := snapVol.NewVMBlockFilesystemVolume() l.Debug("Created a new FS volume", logger.Ctx{"fsVol": fsVol}) return d.MountVolumeSnapshot(fsVol, op) } srcSnapshot := d.dataset(snapVol, false) cloneDataset := d.getTempSnapshotVolName(snapVol) // Create a temporary clone from the snapshot. err = d.cloneSnapshot(srcSnapshot, cloneDataset) if err != nil { return err } reverter.Add(func() { _ = d.deleteDatasetRecursive(cloneDataset) }) // and share the clone err = d.createIscsiShare(cloneDataset, snapVol.contentType != ContentTypeFS) // ro if not FS if err != nil { return err } reverter.Add(func() { _ = d.deleteIscsiShare(cloneDataset) }) // and then activate // volDevPath, err := d.activateIscsiDataset(cloneDataset) _, volDevPath, err := d.locateOrActivateIscsiDataset(cloneDataset) if err != nil { return err } reverter.Add(func() { _ = d.deactivateIscsiDataset(cloneDataset) }) if snapVol.contentType == ContentTypeFS { mountPath := snapVol.MountPath() l.Debug("Content type FS", logger.Ctx{"mountPath": mountPath}) if !linux.IsMountPoint(mountPath) { err := snapVol.EnsureMountPath(false) if err != nil { return err } snapVolFS := snapVol.ConfigBlockFilesystem() if snapVol.mountFilesystemProbe { snapVolFS, err = fsProbe(volDevPath) if err != nil { return fmt.Errorf("Failed probing filesystem: %w", err) } } mountFlags, mountOptions := linux.ResolveMountOptions(strings.Split(snapVol.ConfigBlockMountOptions(), ",")) l.Debug("Regenerating filesystem UUID", logger.Ctx{"volDevPath": volDevPath, "fs": snapVolFS}) if renegerateFilesystemUUIDNeeded(snapVolFS) { if snapVolFS == "xfs" { idx := strings.Index(mountOptions, "nouuid") if idx < 0 { mountOptions += ",nouuid" } } else { err = regenerateFilesystemUUID(snapVolFS, volDevPath) if err != nil { return err } } } l.Debug("Will try mount") err = TryMount(volDevPath, mountPath, snapVolFS, mountFlags, mountOptions) if err != nil { l.Debug("Tried mounting but failed", logger.Ctx{"error": err}) return err } l.Debug("Mounted TrueNAS snapshot volume", logger.Ctx{"volName": snapVol.name, "dev": volDevPath, "path": mountPath, "options": mountOptions}) } } snapVol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolumeSnapshot() when done. reverter.Success() return nil } // UnmountVolumeSnapshot unmounts a volume snapshot. // // Will delete the temporary TrueNAS snapshot clone. func (d *truenas) UnmountVolumeSnapshot(snapVol Volume, op *operations.Operation) (bool, error) { l := d.logger.AddContext(logger.Ctx{"volume": snapVol.Name()}) l.Debug("Umounting TrueNAS snapshot volume", logger.Ctx{"vol": snapVol}) unlock, err := snapVol.MountLock() if err != nil { return false, err } defer unlock() // For VMs, unmount the filesystem volume. if snapVol.IsVMBlock() { fsVol := snapVol.NewVMBlockFilesystemVolume() return d.UnmountVolumeSnapshot(fsVol, op) } ourUnmount := false mountPath := snapVol.MountPath() refCount := snapVol.MountRefCountDecrement() // Attempt to unmount the filesystem if snapVol.contentType == ContentTypeFS && linux.IsMountPoint(mountPath) { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": snapVol.name, "refCount": refCount}) return false, ErrInUse } err := linux.SyncFS(mountPath) if err != nil { return false, fmt.Errorf("Failed syncing filesystem %q: %w", mountPath, err) } ourUnmount, err = forceUnmount(mountPath) if err != nil { return false, err } l.Debug("Unmounted TrueNAS snapshot volume filesystem", logger.Ctx{"vol": snapVol, "path": mountPath}) } cloneDataset := d.getTempSnapshotVolName(snapVol) l.Debug("Deleting temporary TrueNAS snapshot volume") // Deactivate & Delete iSCSI share err = d.deleteIscsiShare(cloneDataset) if err != nil { return false, fmt.Errorf("Could not delete iscsi target for temporary snapshot volume: %w", err) } // Destroy clone err = d.deleteDatasetRecursive(cloneDataset) if err != nil { return false, fmt.Errorf("Could not delete temporary snapshot volume: %w", err) } l.Debug("Temporary TrueNAS snapshot volume deleted") return ourUnmount, nil } // VolumeSnapshots returns a list of snapshots for the volume (in no particular order). func (d *truenas) VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) { // Get all children datasets. dataset := d.dataset(vol, false) entries, err := d.getDatasets(dataset, "snapshot") if err != nil { return nil, err } // Filter only the snapshots. snapshots := []string{} for _, entry := range entries { after, ok := strings.CutPrefix(entry, "@snapshot-") if ok { snapshots = append(snapshots, after) } } return snapshots, nil } // CanRestoreVolume checks whether a volume snapshot can be restored. func (d *truenas) CanRestoreVolume(vol Volume, snapshotName string) error { // Get the list of snapshots. dataset := d.dataset(vol, false) entries, err := d.getDatasets(dataset, "snapshot") if err != nil { return err } // Check if more recent snapshots exist. idx := -1 snapshots := []string{} for i, entry := range entries { if entry == fmt.Sprintf("@snapshot-%s", snapshotName) { // Located the current snapshot. idx = i continue } else if idx < 0 { // Skip any previous snapshot. continue } after, ok := strings.CutPrefix(entry, "@snapshot-") if ok { // Located a normal snapshot following ours. snapshots = append(snapshots, after) continue } if strings.HasPrefix(entry, "@") { // Located an internal snapshot. return fmt.Errorf("Snapshot %q cannot be restored due to subsequent internal snapshot(s) (from a copy)", snapshotName) } } // Check if snapshot removal is allowed. if len(snapshots) > 0 { if util.IsFalseOrEmpty(vol.ExpandedConfig("truenas.remove_snapshots")) { return fmt.Errorf("Snapshot %q cannot be restored due to subsequent snapshot(s). Set truenas.remove_snapshots to override", snapshotName) } // Setup custom error to tell the backend what to delete. err := ErrDeleteSnapshots{} err.Snapshots = snapshots return err } return nil } // RestoreVolume restores a volume from a snapshot. func (d *truenas) RestoreVolume(vol Volume, snapshotName string, op *operations.Operation) error { return d.restoreVolume(vol, snapshotName, false, op) } func (d *truenas) restoreVolume(vol Volume, snapshotName string, isMigration bool, op *operations.Operation) error { dataset := d.dataset(vol, false) err := d.CanRestoreVolume(vol, snapshotName) if err != nil { return err } // TODO: this looks like its manually performing the repeated rollback. We should be able to ask middle to do this for us, the trick // is just to verify its good to go, which I think is the case after the above check. ie --recursive // Restore the snapshot. datasets, err := d.getDatasets(dataset, "snapshot") if err != nil { return err } toRollback := make([]string, 0) for _, dataset := range datasets { if !strings.HasSuffix(dataset, fmt.Sprintf("@snapshot-%s", snapshotName)) { continue } toRollback = append(toRollback, fmt.Sprintf("%s%s", d.dataset(vol, false), dataset)) } if len(toRollback) > 0 { snapRbCmd := []string{"snapshot", "rollback"} _, err = d.runTool(append(snapRbCmd, toRollback...)...) if err != nil { return err } } if vol.contentType == ContentTypeFS && renegerateFilesystemUUIDNeeded(vol.ConfigBlockFilesystem()) { _, _, err = d.activateVolume(vol) if err != nil { return err } defer func() { _, _ = d.deactivateVolume(vol) }() volPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": volPath, "fs": vol.ConfigBlockFilesystem()}) err = regenerateFilesystemUUID(vol.ConfigBlockFilesystem(), volPath) if err != nil { return err } } // For VM images, restore the associated filesystem dataset too. if !isMigration && vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.restoreVolume(fsVol, snapshotName, isMigration, op) if err != nil { return err } } return nil } // RenameVolumeSnapshot renames a volume snapshot. func (d *truenas) RenameVolumeSnapshot(vol Volume, newSnapshotName string, op *operations.Operation) error { parentName, _, _ := api.GetParentAndSnapshotName(vol.name) newVol := NewVolume(d, d.name, vol.volType, vol.contentType, fmt.Sprintf("%s/%s", parentName, newSnapshotName), vol.config, vol.poolConfig) // Revert handling. reverter := revert.New() defer reverter.Fail() // First rename the VFS paths. err := genericVFSRenameVolumeSnapshot(d, vol, newSnapshotName, op) if err != nil { return err } reverter.Add(func() { _ = genericVFSRenameVolumeSnapshot(d, newVol, vol.name, op) }) // Rename the ZFS datasets. err = d.renameSnapshot(d.dataset(vol, false), d.dataset(newVol, false)) if err != nil { return err } reverter.Add(func() { _ = d.renameSnapshot(d.dataset(newVol, false), d.dataset(vol, false)) }) // For VM images, create a filesystem volume too. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.RenameVolumeSnapshot(fsVol, newSnapshotName, op) if err != nil { return err } reverter.Add(func() { newFsVol := NewVolume(d, d.name, newVol.volType, ContentTypeFS, newVol.name, newVol.config, newVol.poolConfig) _ = d.RenameVolumeSnapshot(newFsVol, vol.name, op) }) } // All done. reverter.Success() return nil } // FillVolumeConfig populate volume with default config. func (d *truenas) FillVolumeConfig(vol Volume) error { var excludedKeys []string // Copy volume.* configuration options from pool. // If vol has a source, ignore the block mode related config keys from the pool. if vol.hasSource || vol.IsVMBlock() || vol.volType == VolumeTypeCustom && vol.contentType == ContentTypeBlock { excludedKeys = []string{"block.filesystem", "block.mount_options"} } // Copy volume.* configuration options from pool. // Exclude 'block.filesystem' and 'block.mount_options' // as this ones are handled below in this function and depends from volume type err := d.fillVolumeConfig(&vol, excludedKeys...) if err != nil { return err } // Only validate filesystem config keys for filesystem volumes or VM block volumes (which have an // associated filesystem volume). if vol.ContentType() == ContentTypeFS { // Inherit filesystem from pool if not set. if vol.config["block.filesystem"] == "" { vol.config["block.filesystem"] = d.config["volume.block.filesystem"] } // Default filesystem if neither volume nor pool specify an override. if vol.config["block.filesystem"] == "" { // Unchangeable volume property: Set unconditionally. vol.config["block.filesystem"] = DefaultFilesystem } // Inherit filesystem mount options from pool if not set. if vol.config["block.mount_options"] == "" { vol.config["block.mount_options"] = d.config["volume.block.mount_options"] } // Default filesystem mount options if neither volume nor pool specify an override. if vol.config["block.mount_options"] == "" { // Unchangeable volume property: Set unconditionally. vol.config["block.mount_options"] = "discard" } } return nil } incus-7.0.0/internal/server/storage/drivers/driver_types.go000066400000000000000000000046171517523235500241710ustar00rootroot00000000000000package drivers // Info represents information about a storage driver. type Info struct { Name string Version string VolumeTypes []VolumeType // Supported volume types. DefaultVMBlockFilesystemSize string // Default volume size for VM block filesystems. Buckets bool // Buckets supported. Remote bool // Whether the driver uses a remote backing store. VolumeMultiNode bool // Whether volumes can be used on multiple nodes concurrently. OptimizedImages bool // Whether driver stores images as separate volume. OptimizedBackups bool // Whether driver supports optimized volume backups. OptimizedBackupHeader bool // Whether driver generates an optimised backup header file in backup. PreservesInodes bool // Whether driver preserves inodes when volumes are moved hosts. BlockBacking bool // Whether driver uses block devices as backing store. RunningCopyFreeze bool // Whether instance should be frozen during snapshot if running. SameSource bool // Whether the storage pool config from the node that created the pool should be copied to all other cluster nodes. DirectIO bool // Whether the driver supports direct I/O. IOUring bool // Whether the driver supports io_uring. MountedRoot bool // Whether the pool directory itself is a mount. Deactivate bool // Whether an unmount action is required prior to removing the pool. ZeroUnpack bool // Whether to write zeroes (no discard) during unpacking. TargetFormat string // Whether the output image format should be raw or qcow2. } // VolumeFiller provides a struct for filling a volume. type VolumeFiller struct { Fill func(vol Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) // Function to fill the volume. Size int64 // Size of the unpacked volume in bytes. Fingerprint string // If the Filler will unpack an image, it should be this fingerprint. } incus-7.0.0/internal/server/storage/drivers/driver_zfs.go000066400000000000000000000604061517523235500236250ustar00rootroot00000000000000package drivers import ( "errors" "fmt" "io/fs" "os" "os/exec" "path/filepath" "slices" "strconv" "strings" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" deviceConfig "github.com/lxc/incus/v7/internal/server/device/config" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) const zfsDefaultVdevType = "stripe" var zfsSupportedVdevTypes = []string{ zfsDefaultVdevType, "mirror", "raidz1", "raidz2", } var ( zfsVersion string zfsLoaded bool zfsDelegate bool ) var zfsDefaultSettings = map[string]string{ "relatime": "on", "mountpoint": "legacy", "setuid": "on", "exec": "on", "devices": "on", "acltype": "posixacl", "xattr": "sa", } type zfs struct { common } // load is used to run one-time action per-driver rather than per-pool. func (d *zfs) load() error { // Register the patches. d.patches = map[string]func() error{ "storage_lvm_skipactivation": nil, "storage_missing_snapshot_records": nil, "storage_delete_old_snapshot_records": nil, "storage_zfs_drop_block_volume_filesystem_extension": d.patchDropBlockVolumeFilesystemExtension, "storage_prefix_bucket_names_with_project": nil, } // Done if previously loaded. if zfsLoaded { return nil } // Validate the needed tools are present. for _, tool := range []string{"zpool", "zfs"} { _, err := exec.LookPath(tool) if err != nil { return fmt.Errorf("Required tool '%s' is missing", tool) } } // Load the kernel module. err := linux.LoadModule("zfs") if err != nil { return fmt.Errorf("Error loading %q module: %w", "zfs", err) } // Get the version information. if zfsVersion == "" { version, err := d.version() if err != nil { return err } zfsVersion = version } ourVer, err := version.Parse(zfsVersion) if err != nil { return err } // Detect support for ZFS delegation. ver220, err := version.Parse("2.2.0") if err != nil { return err } if ourVer.Compare(ver220) >= 0 { zfsDelegate = true } zfsLoaded = true return nil } // Info returns info about the driver and its environment. func (d *zfs) Info() Info { info := Info{ Name: "zfs", Version: zfsVersion, DefaultVMBlockFilesystemSize: deviceConfig.DefaultVMBlockFilesystemSize, OptimizedImages: true, OptimizedBackups: true, PreservesInodes: true, Remote: d.isRemote(), VolumeTypes: []VolumeType{VolumeTypeBucket, VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM}, VolumeMultiNode: d.isRemote(), BlockBacking: util.IsTrue(d.config["volume.zfs.block_mode"]), RunningCopyFreeze: util.IsTrue(d.config["volume.zfs.block_mode"]), DirectIO: true, MountedRoot: false, Buckets: true, } return info } // ensureInitialDatasets creates missing initial datasets or configures existing ones with current policy. // Accepts warnOnExistingPolicyApplyError argument, if true will warn rather than fail if applying current policy // to an existing dataset fails. func (d *zfs) ensureInitialDatasets(warnOnExistingPolicyApplyError bool) error { // Build the list of datasets to query. datasets := []string{d.config["zfs.pool_name"]} for _, entry := range d.initialDatasets() { datasets = append(datasets, filepath.Join(d.config["zfs.pool_name"], entry)) } // Build the list of properties to check. props := []string{"name", "mountpoint", "volmode"} for k := range zfsDefaultSettings { props = append(props, k) } // Get current state. args := append([]string{"get", "-H", "-p", "-o", "name,property,value", strings.Join(props, ",")}, datasets...) output, _ := subprocess.RunCommand("zfs", args...) currentConfig := map[string]map[string]string{} for _, entry := range strings.Split(output, "\n") { if entry == "" { continue } fields := strings.Fields(entry) if len(fields) != 3 { continue } if currentConfig[fields[0]] == nil { currentConfig[fields[0]] = map[string]string{} } currentConfig[fields[0]][fields[1]] = fields[2] } // Check that the root dataset is correctly configured. args = []string{} for k, v := range zfsDefaultSettings { current := currentConfig[d.config["zfs.pool_name"]][k] if current == v { continue } // Workaround for values having been renamed over time. if k == "acltype" && current == "posix" { continue } if k == "xattr" && current == "on" { continue } args = append(args, fmt.Sprintf("%s=%s", k, v)) } if len(args) > 0 { err := d.setDatasetProperties(d.config["zfs.pool_name"], args...) if err != nil { if !warnOnExistingPolicyApplyError { return fmt.Errorf("Failed applying policy to existing dataset %q: %w", d.config["zfs.pool_name"], err) } d.logger.Warn("Failed applying policy to existing dataset", logger.Ctx{"dataset": d.config["zfs.pool_name"], "err": err}) } } // Check the initial datasets. for _, dataset := range d.initialDatasets() { properties := map[string]string{"mountpoint": "legacy"} if slices.Contains([]string{"virtual-machines", "deleted/virtual-machines"}, dataset) { properties["volmode"] = "none" } datasetPath := filepath.Join(d.config["zfs.pool_name"], dataset) if currentConfig[datasetPath] != nil { args := []string{} for k, v := range properties { if currentConfig[datasetPath][k] == v { continue } args = append(args, fmt.Sprintf("%s=%s", k, v)) } if len(args) > 0 { err := d.setDatasetProperties(datasetPath, args...) if err != nil { if !warnOnExistingPolicyApplyError { return fmt.Errorf("Failed applying policy to existing dataset %q: %w", datasetPath, err) } d.logger.Warn("Failed applying policy to existing dataset", logger.Ctx{"dataset": datasetPath, "err": err}) } } } else { args := []string{} for k, v := range properties { args = append(args, fmt.Sprintf("%s=%s", k, v)) } err := d.createDataset(datasetPath, args...) if err != nil { return fmt.Errorf("Failed creating dataset %q: %w", datasetPath, err) } } } return nil } // FillConfig populates the storage pool's configuration file with the default values. func (d *zfs) FillConfig() error { vdevType, devices := d.parseSource() if !slices.Contains(zfsSupportedVdevTypes, vdevType) { return fmt.Errorf("Unsupported ZFS vdev type %q. Supported types are %v", vdevType, zfsSupportedVdevTypes) } loopPath := loopFilePath(d.name) if len(devices) == 1 && !filepath.IsAbs(devices[0]) { // Handle an existing zpool. if d.config["zfs.pool_name"] == "" { d.config["zfs.pool_name"] = devices[0] } // Unset size property since it's irrelevant. d.config["size"] = "" } else if len(devices) == 0 || (len(devices) == 1 && devices[0] == loopPath) { // Create a loop based pool. d.config["source"] = loopPath // Set default pool_name. if d.config["zfs.pool_name"] == "" { d.config["zfs.pool_name"] = d.name } // Pick a default size of the loop file if not specified. if d.config["size"] == "" { defaultSize, err := loopFileSizeDefault() if err != nil { return err } d.config["size"] = fmt.Sprintf("%dGiB", defaultSize) } } else if sliceAny(devices, func(device string) bool { return !linux.IsBlockdevPath(device) }) { return errors.New("Custom loop file locations are not supported") } else { // Set default pool_name. if d.config["zfs.pool_name"] == "" { d.config["zfs.pool_name"] = d.name } // Unset size property since it's irrelevant. d.config["size"] = "" } return nil } // Create is called during pool creation and is effectively using an empty driver struct. // WARNING: The Create() function cannot rely on any of the struct attributes being set. func (d *zfs) Create() error { // Store the provided source as we are likely to be mangling it. d.config["volatile.initial_source"] = d.config["source"] err := d.FillConfig() if err != nil { return err } vdevType, devices := d.parseSource() loopPath := loopFilePath(d.name) if len(devices) == 1 && !filepath.IsAbs(devices[0]) { // Validate pool_name. if d.config["zfs.pool_name"] != devices[0] { return errors.New("The source must match zfs.pool_name if specified") } if strings.Contains(d.config["zfs.pool_name"], "/") { // Handle a dataset. exists, err := d.datasetExists(d.config["zfs.pool_name"]) if err != nil { return err } if !exists { err := d.createDataset(d.config["zfs.pool_name"], "mountpoint=legacy") if err != nil { return err } if d.state.OS.IncusOS != nil { err := d.setDatasetProperties(d.config["zfs.pool_name"], "incusos:use=incus") if err != nil { return err } } } } else { // Ensure that the pool is available. _, err := d.importPool() if err != nil { return err } } // Confirm that the existing pool/dataset is all empty. datasets, err := d.getDatasets(d.config["zfs.pool_name"], "all") if err != nil { return err } if len(datasets) > 0 { return fmt.Errorf(`Provided ZFS pool (or dataset) isn't empty, run "sudo zfs list -r %s" to see existing entries`, d.config["zfs.pool_name"]) } } else if len(devices) == 1 && devices[0] == loopPath { // Check for IncusOS. if d.state.OS.IncusOS != nil { return errors.New("Loop backed pools aren't supported on IncusOS") } // Validate pool_name. if strings.Contains(d.config["zfs.pool_name"], "/") { return errors.New("zfs.pool_name can't point to a dataset when source isn't set") } // Create the loop file itself. size, err := units.ParseByteSizeString(d.config["size"]) if err != nil { return err } err = ensureSparseFile(loopPath, size) if err != nil { return err } // Create the zpool. createArgs := []string{"create", "-m", "none", "-O", "compression=on", d.config["zfs.pool_name"]} // "zpool create" doesn't have an explicit type for "stripe" vdev type if vdevType != zfsDefaultVdevType { createArgs = append(createArgs, vdevType) } createArgs = append(createArgs, loopPath) _, err = subprocess.RunCommand("zpool", createArgs...) if err != nil { return err } // Apply auto-trim. _, err = subprocess.RunCommand("zpool", "set", "autotrim=on", d.config["zfs.pool_name"]) if err != nil { return err } } else { // At this moment, we have assurance from FillConfig that all devices are existing block devices // Validate pool_name. if strings.Contains(d.config["zfs.pool_name"], "/") { return errors.New("zfs.pool_name can't point to a dataset when source isn't set") } var createArgs []string // Wipe if requested. if util.IsTrue(d.config["source.wipe"]) { for _, device := range devices { err := wipeBlockHeaders(device) if err != nil { return fmt.Errorf("Failed to wipe headers from disk %q: %w", device, err) } } d.config["source.wipe"] = "" createArgs = []string{"create", "-f", "-m", "none", "-O", "compression=on", d.config["zfs.pool_name"]} } else { createArgs = []string{"create", "-m", "none", "-O", "compression=on", d.config["zfs.pool_name"]} } // Create the zpool. // "zpool create" doesn't have an explicit type for "stripe" vdev type if vdevType != zfsDefaultVdevType { createArgs = append(createArgs, vdevType) } createArgs = append(createArgs, devices...) _, err = subprocess.RunCommand("zpool", createArgs...) if err != nil { return err } // Apply auto-trim. _, err = subprocess.RunCommand("zpool", "set", "autotrim=on", d.config["zfs.pool_name"]) if err != nil { return err } // We don't need to keep the original source path around for import. d.config["source"] = d.config["zfs.pool_name"] } // Setup revert in case of problems reverter := revert.New() defer reverter.Fail() reverter.Add(func() { _ = d.Delete(nil) }) // Apply our default configuration. err = d.ensureInitialDatasets(false) if err != nil { return err } reverter.Success() return nil } // Delete removes the storage pool from the storage device. func (d *zfs) Delete(op *operations.Operation) error { // Check if the dataset/pool is already gone. exists, err := d.datasetExists(d.config["zfs.pool_name"]) if err != nil { return err } if exists { // Confirm that nothing's been left behind datasets, err := d.getDatasets(d.config["zfs.pool_name"], "all") if err != nil { return err } initialDatasets := d.initialDatasets() for _, dataset := range datasets { dataset = strings.TrimPrefix(dataset, "/") if slices.Contains(initialDatasets, dataset) { continue } fields := strings.Split(dataset, "/") if len(fields) > 1 { return fmt.Errorf("ZFS pool has leftover datasets: %s", dataset) } } // Delete the pool. if strings.Contains(d.config["zfs.pool_name"], "/") { // Delete the dataset. _, err := subprocess.RunCommand("zfs", "destroy", "-r", d.config["zfs.pool_name"]) if err != nil { return err } } else { // Delete the pool. _, err := subprocess.RunCommand("zpool", "destroy", d.config["zfs.pool_name"]) if err != nil { return err } } } // On delete, wipe everything in the directory. err = wipeDirectory(GetPoolMountPath(d.name)) if err != nil { return err } // Delete any loop file we may have used loopPath := loopFilePath(d.name) err = os.Remove(loopPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", loopPath, err) } return nil } // Validate checks that all provide keys are supported and that no conflicting or missing configuration is present. func (d *zfs) Validate(config map[string]string) error { // gendoc:generate(entity=storage_zfs, group=common, key=source) // // --- // type: string // scope: local // default: - // shortdesc: Path to existing block device(s), loop file or ZFS dataset/pool. Multiple block devices should be separated by `,`. When listing block devices, you can also prefix them with `vdev` type. To specify a `vdev` type, use an `=` sign between the `vdev` type and the block devices (e.g., `mirror=/dev/sda,/dev/sdb`). Only `stripe`, `mirror`, `raidz1` and `raidz2` `vdev` types are supported. // gendoc:generate(entity=storage_zfs, group=common, key=source.wipe) // // --- // type: bool // scope: local // default: `false` // shortdesc: Wipe the block device specified in `source` prior to creating the storage pool rules := map[string]func(value string) error{ // gendoc:generate(entity=storage_zfs, group=common, key=size) // // --- // type: string // scope: local // default: auto (20% of free disk space, >= 5 GiB and <= 30 GiB) // shortdesc: Size of the storage pool when creating loop-based pools (in bytes, suffixes supported, can be increased to grow storage pool) "size": validate.Optional(validate.IsSize), // gendoc:generate(entity=storage_zfs, group=common, key=zfs.pool_name) // // --- // type: string // scope: local // default: name of the pool // shortdesc: Name of the zpool "zfs.pool_name": validate.IsAny, // gendoc:generate(entity=storage_zfs, group=common, key=zfs.clone_copy) // // --- // type: string // scope: global // default: `true` // shortdesc: Whether to use ZFS lightweight clones rather than full {spellexception}`dataset` copies (Boolean), or `rebase` to copy based on the initial image "zfs.clone_copy": validate.Optional(func(value string) error { if value == "rebase" { return nil } return validate.IsBool(value) }), // gendoc:generate(entity=storage_zfs, group=common, key=zfs.export) // // --- // type: bool // scope: global // default: `true` // shortdesc: Disable zpool export while unmount performed "zfs.export": validate.Optional(validate.IsBool), } return d.validatePool(config, rules, d.commonVolumeRules()) } // Update applies any driver changes required from a configuration change. func (d *zfs) Update(changedConfig map[string]string) error { _, ok := changedConfig["zfs.pool_name"] if ok { return errors.New("zfs.pool_name cannot be modified") } size, ok := changedConfig["size"] if ok { // Figure out loop path loopPath := loopFilePath(d.name) _, devices := d.parseSource() if len(devices) != 1 || devices[0] != loopPath { return errors.New("Cannot resize non-loopback pools") } // Resize loop file f, err := os.OpenFile(loopPath, os.O_RDWR, 0o600) if err != nil { return err } defer func() { _ = f.Close() }() sizeBytes, _ := units.ParseByteSizeString(size) err = f.Truncate(sizeBytes) if err != nil { return err } _, err = subprocess.RunCommand("zpool", "online", "-e", d.config["zfs.pool_name"], loopPath) if err != nil { return err } } return nil } // importPool the storage pool. func (d *zfs) importPool() (bool, error) { if d.config["zfs.pool_name"] == "" { return false, fmt.Errorf("Cannot mount pool as %q is not specified", "zfs.pool_name") } // Check if already setup. exists, err := d.datasetExists(d.config["zfs.pool_name"]) if err != nil { return false, err } if exists { return false, nil } // Check if the pool exists. poolName := strings.Split(d.config["zfs.pool_name"], "/")[0] exists, err = d.datasetExists(poolName) if err != nil { return false, err } if exists { return false, errors.New("ZFS zpool exists but dataset is missing") } // Import the pool. if filepath.IsAbs(d.config["source"]) { disksPath := internalUtil.VarPath("disks") _, err := subprocess.RunCommand("zpool", "import", "-f", "-d", disksPath, poolName) if err != nil { return false, err } } else { _, err := subprocess.RunCommand("zpool", "import", poolName) if err != nil { return false, err } } // Check that the dataset now exists. exists, err = d.datasetExists(d.config["zfs.pool_name"]) if err != nil { return false, err } if !exists { return false, errors.New("ZFS zpool exists but dataset is missing") } // We need to explicitly import the keys here so containers can start. This // is always needed because even if the admin has set up auto-import of // keys on the system, because incus manually imports and exports the pools // the keys can get unloaded. // // We could do "zpool import -l" to request the keys during import, but by // doing it separately we know that the key loading specifically failed and // not some other operation. If a user has keylocation=prompt configured, // this command will fail and the pool will fail to load. _, err = subprocess.RunCommand("zfs", "load-key", "-r", d.config["zfs.pool_name"]) if err != nil { _, _ = d.Unmount() return false, fmt.Errorf("Failed to load keys for ZFS dataset %q: %w", d.config["zfs.pool_name"], err) } return true, nil } // Mount mounts the storage pool. func (d *zfs) Mount() (bool, error) { // Import the pool if not already imported. imported, err := d.importPool() if err != nil { return false, err } // Apply our default configuration. err = d.ensureInitialDatasets(true) if err != nil { return false, err } return imported, nil } // Unmount unmounts the storage pool. func (d *zfs) Unmount() (bool, error) { // Skip if zfs.export config is set to false if util.IsFalse(d.config["zfs.export"]) { return false, nil } // Skip if using a dataset and not a full pool. if strings.Contains(d.config["zfs.pool_name"], "/") { return false, nil } // Check if already unmounted. exists, err := d.datasetExists(d.config["zfs.pool_name"]) if err != nil { return false, err } if !exists { return false, nil } // Export the pool. poolName := strings.Split(d.config["zfs.pool_name"], "/")[0] _, err = subprocess.RunCommand("zpool", "export", poolName) if err != nil { return false, err } return true, nil } func (d *zfs) GetResources() (*api.ResourcesStoragePool, error) { // Get the total amount of space. availableStr, err := d.getDatasetProperty(d.config["zfs.pool_name"], "available") if err != nil { return nil, err } available, err := strconv.ParseUint(strings.TrimSpace(availableStr), 10, 64) if err != nil { return nil, err } // Get the used amount of space. usedStr, err := d.getDatasetProperty(d.config["zfs.pool_name"], "used") if err != nil { return nil, err } used, err := strconv.ParseUint(strings.TrimSpace(usedStr), 10, 64) if err != nil { return nil, err } // Build the struct. // Inode allocation is dynamic so no use in reporting them. res := api.ResourcesStoragePool{} res.Space.Total = used + available res.Space.Used = used return &res, nil } // MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order. func (d *zfs) MigrationTypes(contentType ContentType, refresh bool, copySnapshots bool, clusterMove bool, storageMove bool) []localMigration.Type { var rsyncFeatures []string // Do not pass compression argument to rsync if the associated // config key, that is rsync.compression, is set to false. if util.IsFalse(d.Config()["rsync.compression"]) { rsyncFeatures = []string{"xattrs", "delete", "bidirectional"} } else { rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"} } // Detect ZFS features. features := []string{migration.ZFSFeatureMigrationHeader, "compress"} if contentType == ContentTypeFS { features = append(features, migration.ZFSFeatureZvolFilesystems) } if IsContentBlock(contentType) { return []localMigration.Type{ { FSType: migration.MigrationFSType_ZFS, Features: features, }, { FSType: migration.MigrationFSType_BLOCK_AND_RSYNC, Features: rsyncFeatures, }, } } if refresh && !copySnapshots { return []localMigration.Type{ { FSType: migration.MigrationFSType_RSYNC, Features: rsyncFeatures, }, } } return []localMigration.Type{ { FSType: migration.MigrationFSType_ZFS, Features: features, }, { FSType: migration.MigrationFSType_RSYNC, Features: rsyncFeatures, }, } } // patchDropBlockVolumeFilesystemExtension removes the filesystem extension (e.g _ext4) from VM image block volumes. func (d *zfs) patchDropBlockVolumeFilesystemExtension() error { poolName, ok := d.config["zfs.pool_name"] if !ok { poolName = d.name } out, err := subprocess.RunCommand("zfs", "list", "-H", "-r", "-o", "name", "-t", "volume", fmt.Sprintf("%s/images", poolName)) if err != nil { return fmt.Errorf("Failed listing images: %w", err) } for _, volume := range strings.Split(out, "\n") { fields := strings.SplitN(volume, fmt.Sprintf("%s/images/", poolName), 2) if len(fields) != 2 || fields[1] == "" { continue } // Ignore non-block images, and images without filesystem extension if !strings.HasSuffix(fields[1], ".block") || !strings.Contains(fields[1], "_") { continue } // Rename zfs dataset. Snapshots will automatically be renamed. newName := fmt.Sprintf("%s/images/%s.block", poolName, strings.Split(fields[1], "_")[0]) _, err = subprocess.RunCommand("zfs", "rename", volume, newName) if err != nil { return fmt.Errorf("Failed renaming zfs dataset: %w", err) } } return nil } // Returns vdev type and block device(s) from source config. func (d *zfs) parseSource() (string, []string) { sourceParts := strings.Split(d.config["source"], "=") vdevType := zfsDefaultVdevType devices := sourceParts[0] if len(sourceParts) > 1 { vdevType = sourceParts[0] devices = sourceParts[1] } if len(devices) == 0 { return vdevType, make([]string, 0) } return vdevType, strings.Split(devices, ",") } // roundVolumeBlockSizeBytes returns sizeBytes rounded up to the next multiple // of `vol`'s "zfs.blocksize". func (d *zfs) roundVolumeBlockSizeBytes(vol Volume, sizeBytes int64) (int64, error) { minBlockSize, err := units.ParseByteSizeString(vol.ExpandedConfig("zfs.blocksize")) // minBlockSize will be 0 if zfs.blocksize="" if minBlockSize <= 0 || err != nil { // 16KiB is the default volblocksize minBlockSize = 16 * 1024 } return RoundAbove(minBlockSize, sizeBytes), nil } incus-7.0.0/internal/server/storage/drivers/driver_zfs_cache.go000066400000000000000000000106671517523235500247540ustar00rootroot00000000000000package drivers import ( "slices" "strings" "sync" "time" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/subprocess" ) // This is a global ZFS cache for the system, used to limit the number // of expensive requests to ZFS especially during bulk requests like a full // instance list. type zfsCacheEntry struct { Expiry time.Time Value string } var ( zfsCache map[string]map[string]zfsCacheEntry zfsCacheMu sync.Mutex zfsCachePrefillQueue []string zfsCachePrefillRunning bool zfsCachePrefillMu sync.RWMutex zfsCacheProperties = []string{"used", "referenced"} ) func (d *zfs) prefillCachedProperties(dataset string) { // Define a function to quickly check if a dataset is already cached. isCached := func(dataset string) bool { record, ok := zfsCache[dataset] if !ok { return false } now := time.Now() for _, propName := range zfsCacheProperties { prop, ok := record[propName] if !ok || prop.Expiry.Before(now) { return false } } return true } // Get the lock. zfsCacheMu.Lock() // Check if we already have a valid cache for the dataset. if isCached(dataset) { zfsCacheMu.Unlock() return } // Add the request to the queue. if !slices.Contains(zfsCachePrefillQueue, dataset) { zfsCachePrefillQueue = append(zfsCachePrefillQueue, dataset) } // Check if a filler is already running. // If not, make a copy of the queue and reset it. var runPrefill bool if !zfsCachePrefillRunning { zfsCachePrefillRunning = true zfsCachePrefillMu.Lock() defer func() { zfsCacheMu.Lock() zfsCachePrefillMu.Unlock() zfsCachePrefillRunning = false zfsCacheMu.Unlock() }() runPrefill = true } // Release the lock. zfsCacheMu.Unlock() // Check if we're done. if !runPrefill { // If we got here, the dataset we care about wasn't already in the cache AND there was an existing prefill run ongoing. // Attempt to get a read lock for the prefiller, this will block until the current prefill is done running. // // Depending on timing, the current prefill may or may not have picked us up from the queue. // So we check if we're still in the queue and if we are, we trigger another run which will hopefully pick us up then. zfsCachePrefillMu.RLock() zfsCachePrefillMu.RUnlock() //nolint:staticcheck // Check that we made it. zfsCacheMu.Lock() inQueue := slices.Contains(zfsCachePrefillQueue, dataset) zfsCacheMu.Unlock() if inQueue { // We didn't make it, re-trigger. d.prefillCachedProperties(dataset) return } } // Allow for requests to accumulate. time.Sleep(100 * time.Millisecond) // Copy and clear the queue. zfsCacheMu.Lock() queue := []string{} for _, entry := range zfsCachePrefillQueue { if isCached(entry) { continue } queue = append(queue, entry) } zfsCachePrefillQueue = []string{} zfsCacheMu.Unlock() // Check that we have something to do. if len(queue) == 0 { return } // Run the filler. properties := strings.Join(append([]string{"name"}, zfsCacheProperties...), ",") args := []string{"list", "-H", "-p", "-o", properties, "-r", "-t", "filesystem,volume,snapshot"} args = append(args, queue...) out, err := subprocess.RunCommand("zfs", args...) if err != nil { d.logger.Warn("Couldn't cache ZFS properties", logger.Ctx{"err": err}) return } // Update the cache. zfsCacheMu.Lock() defer zfsCacheMu.Unlock() expiry := time.Now().Add(15 * time.Second) for _, line := range strings.Split(out, "\n") { line = strings.TrimSpace(line) if line == "" { continue } fields := strings.Fields(line) if len(fields) != 3 { continue } record, ok := zfsCache[fields[0]] if !ok { record = map[string]zfsCacheEntry{} } for i, value := range fields[1:] { key := zfsCacheProperties[i] record[key] = zfsCacheEntry{Expiry: expiry, Value: value} } zfsCache[fields[0]] = record } } func (d *zfs) getCachedProperty(dataset string, key string) (string, bool) { // Check if this is a cached property. if !slices.Contains(zfsCacheProperties, key) { return "", false } // Update cache if needed. parentDataset := strings.Split(dataset, "@")[0] d.prefillCachedProperties(parentDataset) // Get the value. zfsCacheMu.Lock() defer zfsCacheMu.Unlock() record, ok := zfsCache[dataset] if !ok { return "", false } value, ok := record[key] if !ok { return "", false } if value.Expiry.Before(time.Now()) { return "", false } return value.Value, true } incus-7.0.0/internal/server/storage/drivers/driver_zfs_patches.go000066400000000000000000000000201517523235500253160ustar00rootroot00000000000000package drivers incus-7.0.0/internal/server/storage/drivers/driver_zfs_utils.go000066400000000000000000000274541517523235500250530ustar00rootroot00000000000000package drivers import ( "context" "errors" "fmt" "io" "os" "os/exec" "path/filepath" "slices" "strings" "github.com/google/uuid" "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) const ( // zfsBlockVolSuffix suffix used for block content type volumes. zfsBlockVolSuffix = ".block" // zfsISOVolSuffix suffix used for iso content type volumes. zfsISOVolSuffix = ".iso" // zfsMinBlocksize is a minimum value for recordsize and volblocksize properties. zfsMinBlocksize = 512 // zfsMaxBlocksize is a maximum value for recordsize and volblocksize properties. zfsMaxBlocksize = 16 * 1024 * 1024 // zfsMaxVolBlocksize is a maximum value for volblocksize property. zfsMaxVolBlocksize = 128 * 1024 ) func (d *zfs) dataset(vol Volume, deleted bool) string { name, snapName, _ := api.GetParentAndSnapshotName(vol.name) if vol.volType == VolumeTypeImage && vol.contentType == ContentTypeFS && d.isBlockBacked(vol) { name = fmt.Sprintf("%s_%s", name, vol.ConfigBlockFilesystem()) } if (vol.volType == VolumeTypeVM || vol.volType == VolumeTypeImage) && vol.contentType == ContentTypeBlock { name = fmt.Sprintf("%s%s", name, zfsBlockVolSuffix) } else if vol.volType == VolumeTypeCustom && vol.contentType == ContentTypeISO { name = fmt.Sprintf("%s%s", name, zfsISOVolSuffix) } if snapName != "" { if deleted { name = fmt.Sprintf("%s@deleted-%s", name, uuid.New().String()) } else { name = fmt.Sprintf("%s@snapshot-%s", name, snapName) } } else if deleted { if vol.volType != VolumeTypeImage { name = uuid.New().String() } return filepath.Join(d.config["zfs.pool_name"], "deleted", string(vol.volType), name) } return filepath.Join(d.config["zfs.pool_name"], string(vol.volType), name) } func (d *zfs) createDataset(dataset string, options ...string) error { args := []string{"create"} for _, option := range options { args = append(args, "-o") args = append(args, option) } args = append(args, dataset) _, err := subprocess.RunCommand("zfs", args...) if err != nil { return err } return nil } func (d *zfs) createVolume(dataset string, size int64, options ...string) error { args := []string{"create", "-s", "-V", fmt.Sprintf("%d", size)} for _, option := range options { args = append(args, "-o") args = append(args, option) } args = append(args, dataset) _, err := subprocess.RunCommand("zfs", args...) if err != nil { return err } return nil } func (d *zfs) datasetExists(dataset string) (bool, error) { out, err := subprocess.RunCommand("zfs", "get", "-H", "-o", "name", "name", dataset) if err != nil { return false, nil } return strings.TrimSpace(out) == dataset, nil } func (d *zfs) deleteDatasetRecursive(dataset string) error { // Locate the origin snapshot (if any). origin, err := d.getDatasetProperty(dataset, "origin") if err != nil { return err } // Delete the dataset (and any snapshots left). _, err = subprocess.TryRunCommand("zfs", "destroy", "-r", dataset) if err != nil { return err } // Check if the origin can now be deleted. if origin != "" && origin != "-" { if strings.HasPrefix(origin, filepath.Join(d.config["zfs.pool_name"], "deleted")) { // Strip the snapshot name when dealing with a deleted volume. dataset = strings.SplitN(origin, "@", 2)[0] } else if strings.Contains(origin, "@deleted-") || strings.Contains(origin, "@copy-") { // Handle deleted snapshots. dataset = origin } else { // Origin is still active. dataset = "" } if dataset != "" { // Get all clones. clones, err := d.getClones(dataset) if err != nil { return err } if len(clones) == 0 { // Delete the origin. err = d.deleteDatasetRecursive(dataset) if err != nil { return err } } } } return nil } func (d *zfs) getClones(dataset string) ([]string, error) { out, err := subprocess.RunCommand("zfs", "get", "-H", "-p", "-r", "-o", "value", "clones", dataset) if err != nil { return nil, err } clones := []string{} for _, line := range strings.Split(out, "\n") { line = strings.TrimSpace(line) if line == dataset || line == "" || line == "-" { continue } line = strings.TrimPrefix(line, fmt.Sprintf("%s/", dataset)) clones = append(clones, line) } return clones, nil } func (d *zfs) getDatasets(dataset string, types string) ([]string, error) { out, err := subprocess.RunCommand("zfs", "get", "-H", "-r", "-o", "name", "-t", types, "name", dataset) if err != nil { return nil, err } children := []string{} for _, line := range strings.Split(out, "\n") { line = strings.TrimSpace(line) if line == dataset || line == "" { continue } line = strings.TrimPrefix(line, dataset) children = append(children, line) } return children, nil } func (d *zfs) setDatasetProperties(dataset string, options ...string) error { args := []string{"set"} args = append(args, options...) args = append(args, dataset) _, err := subprocess.RunCommand("zfs", args...) if err != nil { return err } return nil } func (d *zfs) setBlocksizeFromConfig(vol Volume) error { size := vol.ExpandedConfig("zfs.blocksize") if size == "" { return nil } // Convert to bytes. sizeBytes, err := units.ParseByteSizeString(size) if err != nil { return err } return d.setBlocksize(vol, sizeBytes) } func (d *zfs) setBlocksize(vol Volume, size int64) error { if vol.contentType != ContentTypeFS { return nil } err := d.setDatasetProperties(d.dataset(vol, false), fmt.Sprintf("recordsize=%d", size)) if err != nil { return err } return nil } func (d *zfs) getDatasetProperty(dataset string, key string) (string, error) { output, ok := d.getCachedProperty(dataset, key) if !ok { var err error output, err = subprocess.RunCommand("zfs", "get", "-H", "-p", "-o", "value", key, dataset) if err != nil { return "", err } } return strings.TrimSpace(output), nil } func (d *zfs) getDatasetProperties(dataset string, keys ...string) (map[string]string, error) { output, err := subprocess.RunCommand("zfs", "get", "-H", "-p", "-o", "property,value", strings.Join(keys, ","), dataset) if err != nil { return nil, err } props := make(map[string]string, len(keys)) for _, row := range strings.Split(output, "\n") { prop := strings.Split(row, "\t") if len(prop) < 2 { continue } key := prop[0] val := prop[1] props[key] = val } return props, nil } // version returns the ZFS version based on package or kernel module version. func (d *zfs) version() (string, error) { // This function is only really ever relevant on Ubuntu as the only // distro that ships out of sync tools and kernel modules out, err := subprocess.RunCommand("dpkg-query", "--showformat=${Version}", "--show", "zfsutils-linux") if out != "" && err == nil { return strings.TrimSpace(string(out)), nil } // Loaded kernel module version if util.PathExists("/sys/module/zfs/version") { out, err := os.ReadFile("/sys/module/zfs/version") if err == nil { return strings.TrimSpace(string(out)), nil } } // Module information version out, err = subprocess.RunCommand("modinfo", "-F", "version", "zfs") if err == nil { return strings.TrimSpace(string(out)), nil } return "", errors.New("Could not determine ZFS module version") } // initialDatasets returns the list of all expected datasets. func (d *zfs) initialDatasets() []string { entries := []string{"deleted"} // Iterate over the listed supported volume types. for _, volType := range d.Info().VolumeTypes { entries = append(entries, BaseDirectories[volType].Paths[0]) entries = append(entries, filepath.Join("deleted", BaseDirectories[volType].Paths[0])) } return entries } func (d *zfs) needsRecursion(dataset string) bool { // Ignore snapshots for the test. dataset = strings.Split(dataset, "@")[0] entries, err := d.getDatasets(dataset, "filesystem,volume") if err != nil { return false } if len(entries) == 0 { return false } return true } func (d *zfs) sendDataset(dataset string, parent string, volSrcArgs *migration.VolumeSourceArgs, conn io.ReadWriteCloser, tracker *ioprogress.ProgressTracker) error { defer func() { _ = conn.Close() }() // Assemble zfs send command. args := []string{"send"} // Check if nesting is required. // We only want to use recursion (and possibly raw) mode if required as it can interfere with ZFS encryption. if d.needsRecursion(dataset) { args = append(args, "-R", "-w") } if slices.Contains(volSrcArgs.MigrationType.Features, "compress") { args = append(args, "-c") args = append(args, "-L") } if parent != "" { args = append(args, "-i", parent) } args = append(args, dataset) cmd := exec.Command("zfs", args...) stderr, err := cmd.StderrPipe() if err != nil { return err } // Setup progress tracker. var stdout io.WriteCloser = conn if tracker != nil { stdout = &ioprogress.ProgressWriter{ WriteCloser: conn, Tracker: tracker, } } cmd.Stdout = stdout // Run the command. err = cmd.Start() if err != nil { return err } // Read any error. output, _ := io.ReadAll(stderr) // Handle errors. err = cmd.Wait() if err != nil { return fmt.Errorf("zfs send failed: %w (%s)", err, string(output)) } return nil } func (d *zfs) receiveDataset(vol Volume, r io.Reader, tracker *ioprogress.ProgressTracker) error { // Assemble zfs receive command. args := []string{"receive", "-x", "mountpoint", "-F", "-u", d.dataset(vol, false)} if vol.ContentType() == ContentTypeBlock || d.isBlockBacked(vol) { args = []string{"receive", "-F", "-u", d.dataset(vol, false)} } // Setup progress tracker. stdin := r if tracker != nil { stdin = &ioprogress.ProgressReader{ Reader: r, Tracker: tracker, } } err := subprocess.RunCommandWithFds(context.TODO(), stdin, nil, "zfs", args...) if err != nil { return err } return nil } // ValidateZfsBlocksize validates blocksize property value on the pool. func ValidateZfsBlocksize(value string) error { // Convert to bytes. sizeBytes, err := units.ParseByteSizeString(value) if err != nil { return err } if sizeBytes < zfsMinBlocksize || sizeBytes > zfsMaxBlocksize || (sizeBytes&(sizeBytes-1)) != 0 { return errors.New("Value should be between 512B and 16MiB, and be power of 2") } return nil } // ZFSDataset is the structure used to store information about a dataset. type ZFSDataset struct { Name string `json:"name" yaml:"name"` GUID string `json:"guid" yaml:"guid"` } // ZFSMetaDataHeader is the meta data header about the datasets being sent/stored. type ZFSMetaDataHeader struct { SnapshotDatasets []ZFSDataset `json:"snapshot_datasets" yaml:"snapshot_datasets"` } func (d *zfs) datasetHeader(vol Volume, snapshots []string) (*ZFSMetaDataHeader, error) { migrationHeader := ZFSMetaDataHeader{ SnapshotDatasets: make([]ZFSDataset, len(snapshots)), } for i, snapName := range snapshots { snapVol, _ := vol.NewSnapshot(snapName) guid, err := d.getDatasetProperty(d.dataset(snapVol, false), "guid") if err != nil { return nil, err } migrationHeader.SnapshotDatasets[i].Name = snapName migrationHeader.SnapshotDatasets[i].GUID = guid } return &migrationHeader, nil } func (d *zfs) randomVolumeName(vol Volume) string { return fmt.Sprintf("%s_%s", vol.name, uuid.New().String()) } func (d *zfs) delegateDataset(vol Volume, pid int) error { _, err := subprocess.RunCommand("zfs", "zone", fmt.Sprintf("/proc/%d/ns/user", pid), d.dataset(vol, false)) if err != nil { // Detect cases where the same dataset is attached multiple times. if strings.Contains(err.Error(), "dataset already exists") { return nil } return err } return nil } // ZFSSupportsDelegation returns true if the ZFS version on the system supports user namespace delegation. func ZFSSupportsDelegation() bool { return zfsDelegate } incus-7.0.0/internal/server/storage/drivers/driver_zfs_volumes.go000066400000000000000000003304531517523235500254010ustar00rootroot00000000000000package drivers import ( "bufio" "bytes" "context" "encoding/json" "errors" "fmt" "io" "io/fs" "maps" "os" "os/exec" "path/filepath" "slices" "strconv" "strings" "syscall" "time" "unsafe" "github.com/google/uuid" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/server/backup" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // CreateVolume creates an empty volume and can optionally fill it by executing the supplied // filler function. func (d *zfs) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Operation) error { // Revert handling reverter := revert.New() defer reverter.Fail() if vol.contentType == ContentTypeFS { // Create mountpoint. err := vol.EnsureMountPath(true) if err != nil { return err } reverter.Add(func() { _ = os.Remove(vol.MountPath()) }) } // Look for previously deleted images. if vol.volType == VolumeTypeImage { exists, err := d.datasetExists(d.dataset(vol, true)) if err != nil { return err } if exists { canRestore := true if vol.IsBlockBacked() && (vol.contentType == ContentTypeBlock || d.isBlockBacked(vol)) { // For block volumes check if the cached image volume is larger than the current pool volume.size // setting (if so we won't be able to resize the snapshot to that the smaller size later). volSize, err := d.getDatasetProperty(d.dataset(vol, true), "volsize") if err != nil { return err } volSizeBytes, err := strconv.ParseInt(volSize, 10, 64) if err != nil { return err } poolVolSize := DefaultBlockSize if vol.poolConfig["volume.size"] != "" { poolVolSize = vol.poolConfig["volume.size"] } poolVolSizeBytes, err := units.ParseByteSizeString(poolVolSize) if err != nil { return err } // Round to block boundary. poolVolSizeBytes, err = d.roundVolumeBlockSizeBytes(vol, poolVolSizeBytes) if err != nil { return err } // If the cached volume size is different than the pool volume size, then we can't use the // deleted cached image volume and instead we will rename it to a random UUID so it can't // be restored in the future and a new cached image volume will be created instead. if volSizeBytes != poolVolSizeBytes { d.logger.Debug("Renaming deleted cached image volume so that regeneration is used", logger.Ctx{"fingerprint": vol.Name()}) randomVol := NewVolume(d, d.name, vol.volType, vol.contentType, d.randomVolumeName(vol), vol.config, vol.poolConfig) _, err := subprocess.RunCommand("/proc/self/exe", "forkzfs", "--", "rename", d.dataset(vol, true), d.dataset(randomVol, true)) if err != nil { return err } if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() randomFsVol := randomVol.NewVMBlockFilesystemVolume() _, err := subprocess.RunCommand("/proc/self/exe", "forkzfs", "--", "rename", d.dataset(fsVol, true), d.dataset(randomFsVol, true)) if err != nil { return err } } // We have renamed the deleted cached image volume, so we don't want to try and // restore it. canRestore = false } } // Restore the image. if canRestore { d.logger.Debug("Restoring previously deleted cached image volume", logger.Ctx{"fingerprint": vol.Name()}) _, err := subprocess.RunCommand("/proc/self/exe", "forkzfs", "--", "rename", d.dataset(vol, true), d.dataset(vol, false)) if err != nil { return err } // We now have a restored image, so setup revert. reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() _, err := subprocess.RunCommand("/proc/self/exe", "forkzfs", "--", "rename", d.dataset(fsVol, true), d.dataset(fsVol, false)) if err != nil { return err } // No need to revert.add since we have already succeeded. } reverter.Success() return nil } } } if vol.contentType == ContentTypeFS && !d.isBlockBacked(vol) { // Create the filesystem dataset. err := d.createDataset(d.dataset(vol, false), "mountpoint=legacy", "canmount=noauto") if err != nil { return err } // After this point we have a filesystem, so setup revert. reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) // Apply the size limit. err = d.SetVolumeQuota(vol, vol.ConfigSize(), false, op) if err != nil { return err } // Apply the blocksize. err = d.setBlocksizeFromConfig(vol) if err != nil { return err } } else { var opts []string if vol.contentType == ContentTypeFS { // Use volmode=dev so volume is visible as we need to run makeFSType. opts = []string{"volmode=dev"} } else { // Use volmode=none so volume is invisible until mounted. opts = []string{"volmode=none"} } // Add custom property incus:content_type which allows distinguishing between regular volumes, block_mode enabled volumes, and ISO volumes. if vol.volType == VolumeTypeCustom { opts = append(opts, fmt.Sprintf("incus:content_type=%s", vol.contentType)) } // Avoid double caching in the ARC cache and in the guest OS filesystem cache. if vol.volType == VolumeTypeVM { opts = append(opts, "primarycache=metadata", "secondarycache=metadata") } loopPath := loopFilePath(d.name) if d.config["source"] == loopPath { // Create the volume dataset with sync disabled (to avoid kernel lockups when using a disk based pool). opts = append(opts, "sync=disabled") } blockSize := vol.ExpandedConfig("zfs.blocksize") if blockSize != "" { // Convert to bytes. sizeBytes, err := units.ParseByteSizeString(blockSize) if err != nil { return err } // zfs.blocksize can have value in range from 512 to 16MiB because it's used for volblocksize and recordsize // volblocksize maximum value is 128KiB so if the value of zfs.blocksize is bigger set it to 128KiB. if sizeBytes > zfsMaxVolBlocksize { sizeBytes = zfsMaxVolBlocksize } opts = append(opts, fmt.Sprintf("volblocksize=%d", sizeBytes)) } sizeBytes, err := units.ParseByteSizeString(vol.ConfigSize()) if err != nil { return err } sizeBytes, err = d.roundVolumeBlockSizeBytes(vol, sizeBytes) if err != nil { return err } // Create the volume dataset. err = d.createVolume(d.dataset(vol, false), sizeBytes, opts...) if err != nil { return err } // After this point we'll have a volume, so setup revert. reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) if vol.contentType == ContentTypeFS { // Wait up to 30 seconds for the device to appear. ctx, cancel := context.WithTimeout(d.state.ShutdownCtx, 30*time.Second) defer cancel() devPath, err := d.tryGetVolumeDiskPathFromDataset(ctx, d.dataset(vol, false)) if err != nil { return err } zfsFilesystem := vol.ConfigBlockFilesystem() _, err = makeFSType(devPath, zfsFilesystem, nil) if err != nil { return err } err = d.setDatasetProperties(d.dataset(vol, false), "volmode=none") if err != nil { return err } } } // For VM images, create a filesystem volume too. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.CreateVolume(fsVol, nil, op) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolume(fsVol, op) }) } err := vol.MountTask(func(mountPath string, op *operations.Operation) error { // Run the volume filler function if supplied. if filler != nil && filler.Fill != nil { var err error var devPath string if IsContentBlock(vol.contentType) { // Get the device path. devPath, err = d.GetVolumeDiskPath(vol) if err != nil { return err } } allowUnsafeResize := false if vol.volType == VolumeTypeImage { // Allow filler to resize initial image volume as needed. // Some storage drivers don't normally allow image volumes to be resized due to // them having read-only snapshots that cannot be resized. However when creating // the initial image volume and filling it before the snapshot is taken resizing // can be allowed and is required in order to support unpacking images larger than // the default volume size. The filler function is still expected to obey any // volume size restrictions configured on the pool. // Unsafe resize is also needed to disable filesystem resize safety checks. // This is safe because if for some reason an error occurs the volume will be // discarded rather than leaving a corrupt filesystem. allowUnsafeResize = true } // Run the filler. err = genericRunFiller(d, vol, devPath, filler, allowUnsafeResize) if err != nil { return err } // Move the GPT alt header to end of disk if needed. if vol.IsVMBlock() { err = d.moveGPTAltHeader(devPath) if err != nil { return err } } } if vol.contentType == ContentTypeFS { // Run EnsureMountPath again after mounting and filling to ensure the mount directory has // the correct permissions set. err := vol.EnsureMountPath(true) if err != nil { return err } } return nil }, op) if err != nil { return err } // Setup snapshot and unset mountpoint on image. if vol.volType == VolumeTypeImage { // Create snapshot of the main dataset. _, err := subprocess.RunCommand("zfs", "snapshot", "-r", fmt.Sprintf("%s@readonly", d.dataset(vol, false))) if err != nil { return err } if vol.contentType == ContentTypeBlock { // Re-create the FS config volume's readonly snapshot now that the filler function has run // and unpacked into both config and block volumes. fsVol := vol.NewVMBlockFilesystemVolume() _, err := subprocess.RunCommand("zfs", "destroy", "-r", fmt.Sprintf("%s@readonly", d.dataset(fsVol, false))) if err != nil { return err } _, err = subprocess.RunCommand("zfs", "snapshot", "-r", fmt.Sprintf("%s@readonly", d.dataset(fsVol, false))) if err != nil { return err } } } // All done. reverter.Success() return nil } // CreateVolumeFromBackup re-creates a volume from its exported state. func (d *zfs) CreateVolumeFromBackup(vol Volume, srcBackup backup.Info, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) (VolumePostHook, revert.Hook, error) { // Handle the non-optimized tarballs through the generic unpacker. if !*srcBackup.OptimizedStorage { return genericVFSBackupUnpack(d, d.state.OS, vol, srcBackup.Snapshots, srcData, basePrefix, op) } volExists, err := d.HasVolume(vol) if err != nil { return nil, nil, err } if volExists { return nil, nil, errors.New("Cannot restore volume, already exists on target") } reverter := revert.New() defer reverter.Fail() // Define a revert function that will be used both to revert if an error occurs inside this // function but also return it for use from the calling functions if no error internally. revertHook := func() { for _, snapName := range srcBackup.Snapshots { fullSnapshotName := GetSnapshotVolumeName(vol.name, snapName) snapVol := NewVolume(d, d.name, vol.volType, vol.contentType, fullSnapshotName, vol.config, vol.poolConfig) _ = d.DeleteVolumeSnapshot(snapVol, op) } // And lastly the main volume. _ = d.DeleteVolume(vol, op) } // Only execute the revert function if we have had an error internally. reverter.Add(revertHook) // Define function to unpack a volume from a backup tarball file. unpackVolume := func(v Volume, r io.ReadSeeker, unpacker []string, srcFile string, target string) error { d.Logger().Debug("Unpacking optimized volume", logger.Ctx{"source": srcFile, "target": target}) targetPath := fmt.Sprintf("%s/storage-pools/%s", internalUtil.VarPath(""), target) tr, cancelFunc, err := archive.CompressedTarReader(context.Background(), r, unpacker, targetPath) if err != nil { return err } defer cancelFunc() for { hdr, err := tr.Next() if err == io.EOF { break // End of archive. } if err != nil { return err } if hdr.Name == srcFile { // Extract the backup. if v.ContentType() == ContentTypeBlock || d.isBlockBacked(v) { err = subprocess.RunCommandWithFds(context.TODO(), tr, nil, "zfs", "receive", "-F", target) } else { err = subprocess.RunCommandWithFds(context.TODO(), tr, nil, "zfs", "receive", "-x", "mountpoint", "-F", target) } if err != nil { return err } cancelFunc() return nil } } return fmt.Errorf("Could not find %q", srcFile) } var postHook VolumePostHook // Create a list of actual volumes to unpack. var vols []Volume if vol.IsVMBlock() { vols = append(vols, vol.NewVMBlockFilesystemVolume()) } vols = append(vols, vol) for _, v := range vols { // Find the compression algorithm used for backup source data. _, err := srcData.Seek(0, io.SeekStart) if err != nil { return nil, nil, err } _, _, unpacker, err := archive.DetectCompressionFile(srcData) if err != nil { return nil, nil, err } if len(srcBackup.Snapshots) > 0 { // Create new snapshots directory. err := CreateParentSnapshotDirIfMissing(d.name, v.volType, v.name) if err != nil { return nil, nil, err } } // Restore backups from oldest to newest. for _, snapName := range srcBackup.Snapshots { prefix := "snapshots" fileName := fmt.Sprintf("%s.bin", snapName) if v.volType == VolumeTypeVM { prefix = "virtual-machine-snapshots" if v.contentType == ContentTypeFS { fileName = fmt.Sprintf("%s-config.bin", snapName) } } else if v.volType == VolumeTypeCustom { prefix = "volume-snapshots" } srcFile := filepath.Join(basePrefix, prefix, fileName) dstSnapshot := fmt.Sprintf("%s@snapshot-%s", d.dataset(v, false), snapName) err = unpackVolume(v, srcData, unpacker, srcFile, dstSnapshot) if err != nil { return nil, nil, err } } // Extract main volume. fileName := "container.bin" if v.volType == VolumeTypeVM { if v.contentType == ContentTypeFS { fileName = "virtual-machine-config.bin" } else { fileName = "virtual-machine.bin" } } else if v.volType == VolumeTypeCustom { fileName = "volume.bin" } err = unpackVolume(v, srcData, unpacker, filepath.Join(basePrefix, fileName), d.dataset(v, false)) if err != nil { return nil, nil, err } // Strip internal snapshots. entries, err := d.getDatasets(d.dataset(v, false), "snapshot") if err != nil { return nil, nil, err } // Remove only the internal snapshots. for _, entry := range entries { if strings.Contains(entry, "@snapshot-") { continue } if strings.Contains(entry, "@") { _, err := subprocess.RunCommand("zfs", "destroy", fmt.Sprintf("%s%s", d.dataset(v, false), entry)) if err != nil { return nil, nil, err } } } // Re-apply the base mount options. if v.contentType == ContentTypeFS { if zfsDelegate { // Unset the zoned property so the mountpoint property can be updated. err := d.setDatasetProperties(d.dataset(v, false), "zoned=off") if err != nil { return nil, nil, err } } err := d.setDatasetProperties(d.dataset(v, false), "mountpoint=legacy", "canmount=noauto") if err != nil { return nil, nil, err } // Apply the blocksize. err = d.setBlocksizeFromConfig(v) if err != nil { return nil, nil, err } } // Only mount instance filesystem volumes for backup.yaml access. if v.volType != VolumeTypeCustom && v.contentType != ContentTypeBlock { // The import requires a mounted volume, so mount it and have it unmounted as a post hook. err = d.MountVolume(v, op) if err != nil { return nil, nil, err } reverter.Add(func() { _, _ = d.UnmountVolume(v, false, op) }) postHook = func(postVol Volume) error { _, err := d.UnmountVolume(postVol, false, op) return err } } } cleanup := reverter.Clone().Fail // Clone before calling reverter.Success() so we can return the Fail func. reverter.Success() return postHook, cleanup, nil } // CreateVolumeFromCopy provides same-pool volume copying functionality. func (d *zfs) CreateVolumeFromCopy(vol Volume, srcVol Volume, copySnapshots bool, allowInconsistent bool, op *operations.Operation) error { var err error // Revert handling reverter := revert.New() defer reverter.Fail() if vol.contentType == ContentTypeFS { // Create mountpoint. err = vol.EnsureMountPath(false) if err != nil { return err } reverter.Add(func() { _ = os.Remove(vol.MountPath()) }) } // For VMs, also copy the filesystem dataset. if vol.IsVMBlock() { // For VMs, also copy the filesystem volume. srcFSVol := srcVol.NewVMBlockFilesystemVolume() fsVol := vol.NewVMBlockFilesystemVolume() err = d.CreateVolumeFromCopy(fsVol, srcFSVol, copySnapshots, false, op) if err != nil { return err } // Delete on revert. reverter.Add(func() { _ = d.DeleteVolume(fsVol, op) }) } // Retrieve snapshots on the source. snapshots := []string{} if !srcVol.IsSnapshot() && copySnapshots { snapshots, err = d.VolumeSnapshots(srcVol, op) if err != nil { return err } } // When not allowing inconsistent copies and the volume has a mounted filesystem, we must ensure it is // consistent by syncing and freezing the filesystem to ensure unwritten pages are flushed and that no // further modifications occur while taking the source snapshot. var unfreezeFS func() error sourcePath := srcVol.MountPath() if !allowInconsistent && srcVol.contentType == ContentTypeFS && srcVol.IsBlockBacked() && linux.IsMountPoint(sourcePath) { unfreezeFS, err = d.filesystemFreeze(sourcePath) if err != nil { return err } reverter.Add(func() { _ = unfreezeFS() }) } var srcSnapshot string if srcVol.volType == VolumeTypeImage { srcSnapshot = fmt.Sprintf("%s@readonly", d.dataset(srcVol, false)) } else if srcVol.IsSnapshot() { srcSnapshot = d.dataset(srcVol, false) } else { // Create a new snapshot for copy. srcSnapshot = fmt.Sprintf("%s@copy-%s", d.dataset(srcVol, false), uuid.New().String()) _, err := subprocess.RunCommand("zfs", "snapshot", "-r", srcSnapshot) if err != nil { return err } // If zfs.clone_copy is disabled delete the snapshot at the end. if util.IsFalse(d.config["zfs.clone_copy"]) || len(snapshots) > 0 { // Delete the snapshot at the end. defer func() { // Delete snapshot (or mark for deferred deletion if cannot be deleted currently). _, err := subprocess.RunCommand("zfs", "destroy", "-r", "-d", srcSnapshot) if err != nil { d.logger.Warn("Failed deleting temporary snapshot for copy", logger.Ctx{"snapshot": srcSnapshot, "err": err}) } }() } else { // Delete the snapshot on revert. reverter.Add(func() { // Delete snapshot (or mark for deferred deletion if cannot be deleted currently). _, err := subprocess.RunCommand("zfs", "destroy", "-r", "-d", srcSnapshot) if err != nil { d.logger.Warn("Failed deleting temporary snapshot for copy", logger.Ctx{"snapshot": srcSnapshot, "err": err}) } }) } } // Now that source snapshot has been taken we can safely unfreeze the source filesystem. if unfreezeFS != nil { _ = unfreezeFS() } // Delete the volume created on failure. reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) // If zfs.clone_copy is disabled or source volume has snapshots, then use full copy mode. if util.IsFalse(d.config["zfs.clone_copy"]) || len(snapshots) > 0 { snapName := strings.SplitN(srcSnapshot, "@", 2)[1] // Send/receive the snapshot. var sender *exec.Cmd var receiver *exec.Cmd if vol.ContentType() == ContentTypeBlock || d.isBlockBacked(vol) { receiver = exec.Command("zfs", "receive", d.dataset(vol, false)) } else { receiver = exec.Command("zfs", "receive", "-x", "mountpoint", d.dataset(vol, false)) } // Handle transferring snapshots. if len(snapshots) > 0 { // Raw send is required to send/receive encrypted volumes (and enables compression). args := []string{"send", "-R", "-w", srcSnapshot} sender = exec.Command("zfs", args...) } else { args := []string{"send"} // Check if nesting is required. if d.needsRecursion(d.dataset(srcVol, false)) { args = append(args, "-R", "-w") } if d.config["zfs.clone_copy"] == "rebase" { var err error origin := d.dataset(srcVol, false) for { fields := strings.SplitN(origin, "@", 2) // If the origin is a @readonly snapshot under a /images/ path (/images or deleted/images), we're done. if len(fields) > 1 && strings.Contains(fields[0], "/images/") && fields[1] == "readonly" { break } origin, err = d.getDatasetProperty(origin, "origin") if err != nil { return err } if origin == "" || origin == "-" { origin = "" break } } if origin != "" && origin != srcSnapshot { args = append(args, "-i", origin) args = append(args, srcSnapshot) sender = exec.Command("zfs", args...) } else { args = append(args, srcSnapshot) sender = exec.Command("zfs", args...) } } else { args = append(args, srcSnapshot) sender = exec.Command("zfs", args...) } } // Configure the pipes. receiver.Stdin, _ = sender.StdoutPipe() receiver.Stdout = os.Stdout var recvStderr bytes.Buffer receiver.Stderr = &recvStderr var sendStderr bytes.Buffer sender.Stderr = &sendStderr // Run the transfer. err := receiver.Start() if err != nil { return fmt.Errorf("Failed starting ZFS receive: %w", err) } err = sender.Start() if err != nil { _ = receiver.Process.Kill() return fmt.Errorf("Failed starting ZFS send: %w", err) } senderErr := make(chan error) go func() { err := sender.Wait() if err != nil { _ = receiver.Process.Kill() // This removes any newlines in the error message. msg := strings.ReplaceAll(strings.TrimSpace(sendStderr.String()), "\n", " ") senderErr <- fmt.Errorf("Failed ZFS send: %w (%s)", err, msg) return } senderErr <- nil }() err = receiver.Wait() if err != nil { _ = sender.Process.Kill() // This removes any newlines in the error message. msg := strings.ReplaceAll(strings.TrimSpace(recvStderr.String()), "\n", " ") return fmt.Errorf("Failed ZFS receive: %w (%s)", err, msg) } err = <-senderErr if err != nil { return err } // Delete the snapshot. _, err = subprocess.RunCommand("zfs", "destroy", "-r", fmt.Sprintf("%s@%s", d.dataset(vol, false), snapName)) if err != nil { return err } // Cleanup unexpected snapshots. if len(snapshots) > 0 { children, err := d.getDatasets(d.dataset(vol, false), "snapshot") if err != nil { return err } for _, entry := range children { // Check if expected snapshot. if strings.Contains(entry, "@snapshot-") { name := strings.Split(entry, "@snapshot-")[1] if slices.Contains(snapshots, name) { continue } } // Delete the rest. _, err := subprocess.RunCommand("zfs", "destroy", fmt.Sprintf("%s%s", d.dataset(vol, false), entry)) if err != nil { return err } } } } else { // Perform volume clone. args := []string{"clone"} if vol.contentType == ContentTypeBlock { // Use volmode=none so volume is invisible until mounted. args = append(args, "-o", "volmode=none") } if vol.volType == VolumeTypeCustom { // Regenerate incus:content_type, which was lost when cloning args = append(args, "-o", fmt.Sprintf("incus:content_type=%s", vol.contentType)) } args = append(args, srcSnapshot, d.dataset(vol, false)) // Clone the snapshot. _, err := subprocess.RunCommand("zfs", args...) if err != nil { return err } } // Apply the properties. if vol.contentType == ContentTypeFS { if !d.isBlockBacked(srcVol) { err := d.setDatasetProperties(d.dataset(vol, false), "mountpoint=legacy", "canmount=noauto") if err != nil { return err } // Apply the blocksize. err = d.setBlocksizeFromConfig(vol) if err != nil { return err } } if d.isBlockBacked(srcVol) && renegerateFilesystemUUIDNeeded(vol.ConfigBlockFilesystem()) { _, err := d.activateVolume(vol) if err != nil { return err } volPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": volPath, "fs": vol.ConfigBlockFilesystem()}) err = regenerateFilesystemUUID(vol.ConfigBlockFilesystem(), volPath) if err != nil { return err } } // Mount the volume and ensure the permissions are set correctly inside the mounted volume. err := vol.MountTask(func(_ string, _ *operations.Operation) error { return vol.EnsureMountPath(false) }, op) if err != nil { return err } } // Pass allowUnsafeResize as true when resizing block backed filesystem volumes because we want to allow // the filesystem to be shrunk as small as possible without needing the safety checks that would prevent // leaving the filesystem in an inconsistent state if the resize couldn't be completed. This is because if // the resize fails we will delete the volume anyway so don't have to worry about it being inconsistent. var allowUnsafeResize bool if d.isBlockBacked(vol) && vol.contentType == ContentTypeFS { allowUnsafeResize = true } // Resize volume to the size specified. Only uses volume "size" property and does not use pool/defaults // to give the caller more control over the size being used. err = d.SetVolumeQuota(vol, vol.config["size"], allowUnsafeResize, op) if err != nil { return err } // All done. reverter.Success() return nil } // CreateVolumeFromMigration creates a volume being sent via a migration. func (d *zfs) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, volTargetArgs localMigration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error { // Handle simple rsync and block_and_rsync through generic. if volTargetArgs.MigrationType.FSType == migration.MigrationFSType_RSYNC || volTargetArgs.MigrationType.FSType == migration.MigrationFSType_BLOCK_AND_RSYNC { return genericVFSCreateVolumeFromMigration(d, nil, vol, conn, volTargetArgs, preFiller, op) } else if volTargetArgs.MigrationType.FSType != migration.MigrationFSType_ZFS { return ErrNotSupported } var migrationHeader ZFSMetaDataHeader // If no snapshots have been provided it can mean two things: // 1) The target has no snapshots // 2) Snapshots shouldn't be copied (--instance-only flag) volumeOnly := len(volTargetArgs.Snapshots) == 0 if slices.Contains(volTargetArgs.MigrationType.Features, migration.ZFSFeatureMigrationHeader) { // The source will send all of its snapshots with their respective GUID. buf, err := io.ReadAll(conn) if err != nil { return fmt.Errorf("Failed reading ZFS migration header: %w", err) } err = json.Unmarshal(buf, &migrationHeader) if err != nil { return fmt.Errorf("Failed decoding ZFS migration header: %w", err) } } // If we're refreshing, send back all snapshots of the target. if volTargetArgs.Refresh && slices.Contains(volTargetArgs.MigrationType.Features, migration.ZFSFeatureMigrationHeader) { snapshots, err := vol.Snapshots(op) if err != nil { return fmt.Errorf("Failed getting volume snapshots: %w", err) } // If there are no snapshots on the target, there's no point in doing an optimized // refresh. if len(snapshots) == 0 { volTargetArgs.Refresh = false } var respSnapshots []ZFSDataset var syncSnapshots []*migration.Snapshot // Get the GUIDs of all target snapshots. for _, snapVol := range snapshots { guid, err := d.getDatasetProperty(d.dataset(snapVol, false), "guid") if err != nil { return err } _, snapName, _ := api.GetParentAndSnapshotName(snapVol.name) respSnapshots = append(respSnapshots, ZFSDataset{Name: snapName, GUID: guid}) } // Generate list of snapshots which need to be synced, i.e. are available on the source but not on the target. for _, srcSnapshot := range migrationHeader.SnapshotDatasets { found := false for _, dstSnapshot := range respSnapshots { if srcSnapshot.GUID == dstSnapshot.GUID { found = true break } } if !found { syncSnapshots = append(syncSnapshots, &migration.Snapshot{Name: &srcSnapshot.Name}) } } // The following scenario will result in a failure: // - The source has more than one snapshot // - The target has at least one of these snapshot, but not the very first // // It will fail because the source tries sending the first snapshot using `zfs send `. // Since the target does have snapshots, `zfs receive` will fail with: // cannot receive new filesystem stream: destination has snapshots // // We therefore need to check the snapshots, and delete all target snapshots if the above // scenario is true. if !volumeOnly && len(respSnapshots) > 0 && len(migrationHeader.SnapshotDatasets) > 0 && respSnapshots[0].GUID != migrationHeader.SnapshotDatasets[0].GUID { for _, snapVol := range snapshots { // Delete err = d.DeleteVolume(snapVol, op) if err != nil { return err } } // Let the source know that we don't have any snapshots. respSnapshots = []ZFSDataset{} // Let the source know that we need all snapshots. syncSnapshots = []*migration.Snapshot{} for _, dataset := range migrationHeader.SnapshotDatasets { syncSnapshots = append(syncSnapshots, &migration.Snapshot{Name: &dataset.Name}) } } else { // Delete local snapshots which exist on the target but not on the source. for _, snapVol := range snapshots { targetOnlySnapshot := true _, snapName, _ := api.GetParentAndSnapshotName(snapVol.name) for _, migrationSnap := range migrationHeader.SnapshotDatasets { if snapName == migrationSnap.Name { targetOnlySnapshot = false break } } if targetOnlySnapshot { // Delete err = d.DeleteVolume(snapVol, op) if err != nil { return err } } } } migrationHeader = ZFSMetaDataHeader{} migrationHeader.SnapshotDatasets = respSnapshots // Send back all target snapshots with their GUIDs. headerJSON, err := json.Marshal(migrationHeader) if err != nil { return fmt.Errorf("Failed encoding ZFS migration header: %w", err) } _, err = conn.Write(headerJSON) if err != nil { return fmt.Errorf("Failed sending ZFS migration header: %w", err) } err = conn.Close() // End the frame. if err != nil { return fmt.Errorf("Failed closing ZFS migration header frame: %w", err) } // Don't pass the snapshots if it's volume only. if !volumeOnly { volTargetArgs.Snapshots = syncSnapshots } } return d.createVolumeFromMigrationOptimized(vol, conn, volTargetArgs, volumeOnly, preFiller, op) } func (d *zfs) createVolumeFromMigrationOptimized(vol Volume, conn io.ReadWriteCloser, volTargetArgs localMigration.VolumeTargetArgs, volumeOnly bool, preFiller *VolumeFiller, op *operations.Operation) error { if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.createVolumeFromMigrationOptimized(fsVol, conn, volTargetArgs, volumeOnly, preFiller, op) if err != nil { return err } } var snapshots []Volume var err error // Rollback to the latest identical snapshot if performing a refresh. if volTargetArgs.Refresh { snapshots, err = vol.Snapshots(op) if err != nil { return err } if len(snapshots) > 0 { lastIdenticalSnapshot := snapshots[len(snapshots)-1] _, lastIdenticalSnapshotOnlyName, _ := api.GetParentAndSnapshotName(lastIdenticalSnapshot.Name()) err = d.restoreVolume(vol, lastIdenticalSnapshotOnlyName, true, op) if err != nil { return err } } } reverter := revert.New() defer reverter.Fail() // Handle zfs send/receive migration. if len(volTargetArgs.Snapshots) > 0 { // Create the parent directory. err := CreateParentSnapshotDirIfMissing(d.name, vol.volType, vol.name) if err != nil { return err } // Transfer the snapshots. for _, snapshot := range volTargetArgs.Snapshots { snapVol, err := vol.NewSnapshot(snapshot.GetName()) if err != nil { return err } // Setup progress tracking. var wrapper *ioprogress.ProgressTracker if volTargetArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", snapVol.Name()) } err = d.receiveDataset(snapVol, conn, wrapper) if err != nil { _ = d.DeleteVolume(snapVol, op) return fmt.Errorf("Failed receiving snapshot volume %q: %w", snapVol.Name(), err) } reverter.Add(func() { _ = d.DeleteVolumeSnapshot(snapVol, op) }) } } if !volTargetArgs.Refresh { reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) } // Setup progress tracking. var wrapper *ioprogress.ProgressTracker if volTargetArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", vol.name) } // Transfer the main volume. err = d.receiveDataset(vol, conn, wrapper) if err != nil { return fmt.Errorf("Failed receiving volume %q: %w", vol.Name(), err) } // Strip internal snapshots. entries, err := d.getDatasets(d.dataset(vol, false), "snapshot") if err != nil { return err } // keepDataset returns whether to keep the data set or delete it. Data sets that are non-snapshots or // snapshots that match the requested snapshots in volTargetArgs.Snapshots are kept. Any other snapshot // data sets should be removed. keepDataset := func(dataSetName string) bool { // Keep non-snapshot data sets and snapshots that don't have the snapshot prefix indicator. dataSetSnapshotPrefix := "@snapshot-" if !strings.HasPrefix(dataSetName, "@") || !strings.HasPrefix(dataSetName, dataSetSnapshotPrefix) { return false } // Check if snapshot data set matches one of the requested snapshots in volTargetArgs.Snapshots. // If so, then keep it, otherwise request it be removed. entrySnapName := strings.TrimPrefix(dataSetName, dataSetSnapshotPrefix) for _, snapshot := range volTargetArgs.Snapshots { if entrySnapName == snapshot.GetName() { return true // Keep snapshot data set if present in the requested snapshots list. } } return false // Delete any other snapshot data sets that have been transferred. } if volTargetArgs.Refresh { // Only delete the latest migration snapshot. _, err := subprocess.RunCommand("zfs", "destroy", "-r", fmt.Sprintf("%s%s", d.dataset(vol, false), entries[len(entries)-1])) if err != nil { return err } } else { // Remove any snapshots that were transferred but are not needed. for _, entry := range entries { if !keepDataset(entry) { _, err := subprocess.RunCommand("zfs", "destroy", fmt.Sprintf("%s%s", d.dataset(vol, false), entry)) if err != nil { return err } } } } if vol.contentType == ContentTypeFS { // Create mountpoint. err := vol.EnsureMountPath(false) if err != nil { return err } if !d.isBlockBacked(vol) { // Re-apply the base mount options. if zfsDelegate { // Unset the zoned property so the mountpoint property can be updated. err := d.setDatasetProperties(d.dataset(vol, false), "zoned=off") if err != nil { return err } } err = d.setDatasetProperties(d.dataset(vol, false), "mountpoint=legacy", "canmount=noauto") if err != nil { return err } // Apply the size limit. err = d.SetVolumeQuota(vol, vol.ConfigSize(), false, op) if err != nil { return err } // Apply the blocksize. err = d.setBlocksizeFromConfig(vol) if err != nil { return err } } if d.isBlockBacked(vol) && renegerateFilesystemUUIDNeeded(vol.ConfigBlockFilesystem()) { // Activate volume if needed. activated, err := d.activateVolume(vol) if err != nil { return err } if activated { defer func() { _, _ = d.deactivateVolume(vol) }() } volPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": volPath, "fs": vol.ConfigBlockFilesystem()}) err = regenerateFilesystemUUID(vol.ConfigBlockFilesystem(), volPath) if err != nil { return err } } } reverter.Success() return nil } // RefreshVolume updates an existing volume to match the state of another. func (d *zfs) RefreshVolume(vol Volume, srcVol Volume, srcSnapshots []Volume, allowInconsistent bool, op *operations.Operation) error { var err error var targetSnapshots []Volume var srcSnapshotsAll []Volume if !srcVol.IsSnapshot() { // Get target snapshots targetSnapshots, err = vol.Snapshots(op) if err != nil { return fmt.Errorf("Failed to get target snapshots: %w", err) } srcSnapshotsAll, err = srcVol.Snapshots(op) if err != nil { return fmt.Errorf("Failed to get source snapshots: %w", err) } } // If there are no target or source snapshots, perform a simple copy using zfs. // We cannot use generic vfs volume copy here, as zfs will complain if a generic // copy/refresh is followed by an optimized refresh. if len(targetSnapshots) == 0 || len(srcSnapshotsAll) == 0 { err = d.DeleteVolume(vol, op) if err != nil { return err } return d.CreateVolumeFromCopy(vol, srcVol, len(srcSnapshots) > 0, false, op) } transfer := func(src Volume, target Volume, origin Volume) error { var sender *exec.Cmd receiver := exec.Command("zfs", "receive", d.dataset(target, false)) args := []string{"send"} // Check if nesting is required. if d.needsRecursion(d.dataset(src, false)) { args = append(args, "-R", "-w") } if origin.Name() != src.Name() { args = append(args, "-i", d.dataset(origin, false), d.dataset(src, false)) sender = exec.Command("zfs", args...) } else { args = append(args, d.dataset(src, false)) sender = exec.Command("zfs", args...) } // Configure the pipes. receiver.Stdin, _ = sender.StdoutPipe() receiver.Stdout = os.Stdout var recvStderr bytes.Buffer receiver.Stderr = &recvStderr var sendStderr bytes.Buffer sender.Stderr = &sendStderr // Run the transfer. err := receiver.Start() if err != nil { return fmt.Errorf("Failed starting ZFS receive: %w", err) } err = sender.Start() if err != nil { _ = receiver.Process.Kill() return fmt.Errorf("Failed starting ZFS send: %w", err) } senderErr := make(chan error) go func() { err := sender.Wait() if err != nil { _ = receiver.Process.Kill() // This removes any newlines in the error message. msg := strings.ReplaceAll(strings.TrimSpace(sendStderr.String()), "\n", " ") senderErr <- fmt.Errorf("Failed ZFS send: %w (%s)", err, msg) return } senderErr <- nil }() err = receiver.Wait() if err != nil { _ = sender.Process.Kill() // This removes any newlines in the error message. msg := strings.ReplaceAll(strings.TrimSpace(recvStderr.String()), "\n", " ") if strings.Contains(msg, "does not match incremental source") { return ErrSnapshotDoesNotMatchIncrementalSource } return fmt.Errorf("Failed ZFS receive: %w (%s)", err, msg) } err = <-senderErr if err != nil { return err } return nil } // This represents the most recent identical snapshot of the source volume and target volume. lastIdenticalSnapshot := targetSnapshots[len(targetSnapshots)-1] _, lastIdenticalSnapshotOnlyName, _ := api.GetParentAndSnapshotName(lastIdenticalSnapshot.Name()) // Rollback target volume to the latest identical snapshot err = d.RestoreVolume(vol, lastIdenticalSnapshotOnlyName, op) if err != nil { return fmt.Errorf("Failed to restore volume: %w", err) } // Create all missing snapshots on the target using an incremental stream for i, snap := range srcSnapshots { var originSnap Volume if i == 0 { originSnap, err = srcVol.NewSnapshot(lastIdenticalSnapshotOnlyName) if err != nil { return fmt.Errorf("Failed to create new snapshot volume: %w", err) } } else { originSnap = srcSnapshots[i-1] } err = transfer(snap, vol, originSnap) if err != nil { // Don't fail here. If it's not possible to perform an optimized refresh, do a generic // refresh instead. if errors.Is(err, ErrSnapshotDoesNotMatchIncrementalSource) { d.logger.Debug("Unable to perform an optimized refresh, doing a generic refresh", logger.Ctx{"err": err}) return genericVFSCopyVolume(d, nil, vol, srcVol, srcSnapshots, true, allowInconsistent, op) } return fmt.Errorf("Failed to transfer snapshot %q: %w", snap.name, err) } if snap.IsVMBlock() { srcFSVol := snap.NewVMBlockFilesystemVolume() targetFSVol := vol.NewVMBlockFilesystemVolume() originFSVol := originSnap.NewVMBlockFilesystemVolume() err = transfer(srcFSVol, targetFSVol, originFSVol) if err != nil { // Don't fail here. If it's not possible to perform an optimized refresh, do a generic // refresh instead. if errors.Is(err, ErrSnapshotDoesNotMatchIncrementalSource) { d.logger.Debug("Unable to perform an optimized refresh, doing a generic refresh", logger.Ctx{"err": err}) return genericVFSCopyVolume(d, nil, vol, srcVol, srcSnapshots, true, allowInconsistent, op) } return fmt.Errorf("Failed to transfer snapshot %q: %w", snap.name, err) } } } // Create temporary snapshot of the source volume. snapUUID := uuid.New().String() srcSnap, err := srcVol.NewSnapshot(snapUUID) if err != nil { return err } err = d.CreateVolumeSnapshot(srcSnap, op) if err != nil { return err } latestSnapVol := srcSnapshotsAll[len(srcSnapshotsAll)-1] err = transfer(srcSnap, vol, latestSnapVol) if err != nil { // Don't fail here. If it's not possible to perform an optimized refresh, do a generic // refresh instead. if errors.Is(err, ErrSnapshotDoesNotMatchIncrementalSource) { d.logger.Debug("Unable to perform an optimized refresh, doing a generic refresh", logger.Ctx{"err": err}) return genericVFSCopyVolume(d, nil, vol, srcVol, srcSnapshots, true, allowInconsistent, op) } return fmt.Errorf("Failed to transfer main volume: %w", err) } if srcSnap.IsVMBlock() { srcFSVol := srcSnap.NewVMBlockFilesystemVolume() targetFSVol := vol.NewVMBlockFilesystemVolume() originFSVol := latestSnapVol.NewVMBlockFilesystemVolume() err = transfer(srcFSVol, targetFSVol, originFSVol) if err != nil { // Don't fail here. If it's not possible to perform an optimized refresh, do a generic // refresh instead. if errors.Is(err, ErrSnapshotDoesNotMatchIncrementalSource) { d.logger.Debug("Unable to perform an optimized refresh, doing a generic refresh", logger.Ctx{"err": err}) return genericVFSCopyVolume(d, nil, vol, srcVol, srcSnapshots, true, allowInconsistent, op) } return fmt.Errorf("Failed to transfer main volume: %w", err) } } // Restore target volume from main source snapshot. err = d.RestoreVolume(vol, snapUUID, op) if err != nil { return err } // Delete temporary source snapshot. err = d.DeleteVolumeSnapshot(srcSnap, op) if err != nil { return err } // Delete temporary target snapshot. targetSnap, err := vol.NewSnapshot(snapUUID) if err != nil { return err } err = d.DeleteVolumeSnapshot(targetSnap, op) if err != nil { return err } return nil } // DeleteVolume deletes a volume of the storage device. If any snapshots of the volume remain then // this function will return an error. // For image volumes, both filesystem and block volumes will be removed. func (d *zfs) DeleteVolume(vol Volume, op *operations.Operation) error { if vol.volType == VolumeTypeImage { // We need to clone vol the otherwise changing `zfs.block_mode` // in tmpVol will also change it in vol. tmpVol := vol.Clone() for _, filesystem := range blockBackedAllowedFilesystems { tmpVol.config["block.filesystem"] = filesystem err := d.deleteVolume(tmpVol, op) if err != nil { return err } } } return d.deleteVolume(vol, op) } func (d *zfs) deleteVolume(vol Volume, op *operations.Operation) error { // Check that we have a dataset to delete. exists, err := d.datasetExists(d.dataset(vol, false)) if err != nil { return err } if exists { // Handle clones. clones, err := d.getClones(d.dataset(vol, false)) if err != nil { return err } if len(clones) > 0 { // Move to the deleted path. _, err := subprocess.RunCommand("/proc/self/exe", "forkzfs", "--", "rename", d.dataset(vol, false), d.dataset(vol, true)) if err != nil { return err } } else { err := d.deleteDatasetRecursive(d.dataset(vol, false)) if err != nil { return err } } } if vol.contentType == ContentTypeFS { // Delete the mountpoint if present. err := os.Remove(vol.MountPath()) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", vol.MountPath(), err) } // Delete the snapshot storage. err = os.RemoveAll(GetVolumeSnapshotDir(d.name, vol.volType, vol.name)) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", GetVolumeSnapshotDir(d.name, vol.volType, vol.name), err) } } // For VMs, also delete the filesystem dataset. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.DeleteVolume(fsVol, op) if err != nil { return err } } return nil } // HasVolume indicates whether a specific volume exists on the storage pool. func (d *zfs) HasVolume(vol Volume) (bool, error) { // Check if the dataset exists. return d.datasetExists(d.dataset(vol, false)) } // commonVolumeRules returns validation rules which are common for pool and volume. func (d *zfs) commonVolumeRules() map[string]func(value string) error { return map[string]func(value string) error{ // gendoc:generate(entity=storage_volume_zfs, group=common, key=block.filesystem) // // --- // type: string // condition: block-based volume with content type `filesystem` (`zfs.block_mode` enabled) // default: same as `volume.block.filesystem` // shortdesc: {{block_filesystem}} "block.filesystem": validate.Optional(validate.IsOneOf(blockBackedAllowedFilesystems...)), // gendoc:generate(entity=storage_volume_zfs, group=common, key=block.mount_options) // // --- // type: string // condition: block-based volume with content type `filesystem` (`zfs.block_mode` enabled) // default: same as `volume.block.mount_options` // shortdesc: Mount options for block-backed file system volumes "block.mount_options": validate.IsAny, // gendoc:generate(entity=storage_volume_zfs, group=common, key=zfs.blocksize) // // --- // type: string // condition: - // default: same as `volume.zfs.blocksize` // shortdesc: Size of the ZFS block in range from 512 bytes to 16 MiB (must be power of 2) - for block volume, a maximum value of 128 KiB will be used even if a higher value is set "zfs.blocksize": validate.Optional(ValidateZfsBlocksize), // gendoc:generate(entity=storage_volume_zfs, group=common, key=zfs.block_mode) // // --- // type: bool // condition: - // default: same as `volume.zfs.block_mode` // shortdesc: Whether to use a formatted `zvol` rather than a {spellexception}`dataset` (`zfs.block_mode` can be set only for custom storage volumes; use `volume.zfs.block_mode` to enable ZFS block mode for all storage volumes in the pool, including instance volumes) "zfs.block_mode": validate.Optional(validate.IsBool), // gendoc:generate(entity=storage_volume_zfs, group=common, key=zfs.remove_snapshots) // // --- // type: bool // condition: - // default: same as `volume.zfs.remove_snapshots` or `false` // shortdesc: Remove snapshots as needed "zfs.remove_snapshots": validate.Optional(validate.IsBool), // gendoc:generate(entity=storage_volume_zfs, group=common, key=zfs.reserve_space) // // --- // type: bool // condition: - // default: same as `volume.zfs.reserve_space` or `false` // shortdesc: Use `reservation`/`refreservation` along with `quota`/`refquota` "zfs.reserve_space": validate.Optional(validate.IsBool), // gendoc:generate(entity=storage_volume_zfs, group=common, key=zfs.use_refquota) // // --- // type: bool // condition: - // default: same as `volume.zfsuse_refquota` or `false` // shortdesc: Use `refquota` instead of `quota` for space "zfs.use_refquota": validate.Optional(validate.IsBool), // gendoc:generate(entity=storage_volume_zfs, group=common, key=zfs.delegate) // // --- // type: bool // condition: ZFS 2.2 or higher // default: same as `volume.zfs.delegate` // shortdesc: Controls whether to delegate the ZFS dataset and anything underneath it to the container(s) using it. Allows the use of the `zfs` command in the container "zfs.delegate": validate.Optional(validate.IsBool), } } // ValidateVolume validates the supplied volume config. func (d *zfs) ValidateVolume(vol Volume, removeUnknownKeys bool) error { // gendoc:generate(entity=storage_volume_zfs, group=common, key=initial.gid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.gid` or `0` // shortdesc: GID of the volume owner in the instance // gendoc:generate(entity=storage_volume_zfs, group=common, key=initial.mode) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.mode` or `711` // shortdesc: Mode of the volume in the instance // gendoc:generate(entity=storage_volume_zfs, group=common, key=initial.uid) // // --- // type: int // condition: custom volume with content type `filesystem` // default: same as `volume.initial.uid` or `0` // shortdesc: UID of the volume owner in the instance // gendoc:generate(entity=storage_volume_zfs, group=common, key=security.shared) // // --- // type: bool // condition: custom block volume // default: same as `volume.security.shared` or `false` // shortdesc: Enable sharing the volume across multiple instances // gendoc:generate(entity=storage_volume_zfs, group=common, key=security.shifted) // // --- // type: bool // condition: custom volume // default: same as `volume.security.shifted` or `false` // shortdesc: {{enable_ID_shifting}} // gendoc:generate(entity=storage_volume_zfs, group=common, key=security.unmapped) // // --- // type: bool // condition: custom volume // default: same as `volume.security.unmapped` or `false` // shortdesc: Disable ID mapping for the volume // gendoc:generate(entity=storage_volume_zfs, group=common, key=size) // // --- // type: string // condition: - // default: same as `volume.size` // shortdesc: Size/quota of the storage volume // gendoc:generate(entity=storage_volume_zfs, group=common, key=snapshots.expiry) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_zfs, group=common, key=snapshots.expiry.manual) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.expiry.manual` // shortdesc: {{snapshot_expiry_format}} // gendoc:generate(entity=storage_volume_zfs, group=common, key=snapshots.pattern) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.pattern` or `snap%d` // shortdesc: {{snapshot_pattern_format}} [^*] // gendoc:generate(entity=storage_volume_zfs, group=common, key=snapshots.schedule) // // --- // type: string // condition: custom volume // default: same as `volume.snapshot.schedule` // shortdesc: {{snapshot_schedule_format}} // gendoc:generate(entity=storage_bucket_zfs, group=common, key=size) // // --- // type: string // condition: appropriate driver // default: same as `volume.size` // shortdesc: Size/quota of the storage bucket commonRules := d.commonVolumeRules() // Disallow block.* settings for regular custom block volumes. These settings only make sense // when using custom filesystem volumes with block mode enabled. Incus will create the filesystem // for these volumes, and use the mount options. When attaching a regular block volumes to a VM, // these are not mounted by Incus and therefore don't need these config keys. if vol.IsVMBlock() || vol.volType == VolumeTypeCustom && vol.contentType == ContentTypeBlock { delete(commonRules, "zfs.block_mode") delete(commonRules, "block.filesystem") delete(commonRules, "block.mount_options") } else if vol.volType == VolumeTypeCustom && !vol.IsBlockBacked() { delete(commonRules, "block.filesystem") delete(commonRules, "block.mount_options") } return d.validateVolume(vol, commonRules, removeUnknownKeys) } // UpdateVolume applies config changes to the volume. func (d *zfs) UpdateVolume(vol Volume, changedConfig map[string]string) error { // Mangle the current volume to its old values. old := make(map[string]string) for k, v := range changedConfig { if k == "size" || k == "zfs.use_refquota" || k == "zfs.reserve_space" { old[k] = vol.config[k] vol.config[k] = v } if k == "zfs.blocksize" { // Convert to bytes. sizeBytes, err := units.ParseByteSizeString(v) if err != nil { return err } err = d.setBlocksize(vol, sizeBytes) if err != nil { return err } } } defer func() { maps.Copy(vol.config, old) }() // If any of the relevant keys changed, re-apply the quota. if len(old) != 0 { err := d.SetVolumeQuota(vol, vol.ExpandedConfig("size"), false, nil) if err != nil { return err } } return d.updateVolume(vol, changedConfig) } // GetVolumeUsage returns the disk space used by the volume. func (d *zfs) GetVolumeUsage(vol Volume) (int64, error) { // Determine what key to use. key := "used" // If volume isn't snapshot then we can take into account the zfs.use_refquota setting. // Snapshots should also use the "used" ZFS property because the snapshot usage size represents the CoW // usage not the size of the snapshot volume. if !vol.IsSnapshot() { if util.IsTrue(vol.ExpandedConfig("zfs.use_refquota")) { key = "referenced" } // Shortcut for mounted refquota filesystems. if key == "referenced" && vol.contentType == ContentTypeFS && linux.IsMountPoint(vol.MountPath()) { var stat unix.Statfs_t err := unix.Statfs(vol.MountPath(), &stat) if err != nil { return -1, err } return int64(stat.Blocks-stat.Bfree) * int64(stat.Bsize), nil } } // Get the current value. value, err := d.getDatasetProperty(d.dataset(vol, false), key) if err != nil { return -1, err } // Convert to int. valueInt, err := strconv.ParseInt(value, 10, 64) if err != nil { return -1, err } return valueInt, nil } // SetVolumeQuota sets the quota/reservation on the volume. // Does nothing if supplied with an empty/zero size for block volumes. func (d *zfs) SetVolumeQuota(vol Volume, size string, allowUnsafeResize bool, op *operations.Operation) error { // Convert to bytes. sizeBytes, err := units.ParseByteSizeString(size) if err != nil { return err } inUse := vol.MountInUse() // Handle volume datasets. if d.isBlockBacked(vol) && vol.contentType == ContentTypeFS || IsContentBlock(vol.contentType) { // Do nothing if size isn't specified. if sizeBytes <= 0 { return nil } sizeBytes, err = d.roundVolumeBlockSizeBytes(vol, sizeBytes) if err != nil { return err } oldSizeBytesStr, err := d.getDatasetProperty(d.dataset(vol, false), "volsize") if err != nil { return err } oldVolSizeBytesInt, err := strconv.ParseInt(oldSizeBytesStr, 10, 64) if err != nil { return err } oldVolSizeBytes := int64(oldVolSizeBytesInt) if oldVolSizeBytes == sizeBytes { return nil } if vol.contentType == ContentTypeFS { // Activate volume if needed. activated, err := d.activateVolume(vol) if err != nil { return err } if activated { defer func() { _, _ = d.deactivateVolume(vol) }() } if vol.volType == VolumeTypeImage { return fmt.Errorf("Image volumes cannot be resized: %w", ErrCannotBeShrunk) } fsType := vol.ConfigBlockFilesystem() volDevPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } l := d.logger.AddContext(logger.Ctx{"dev": volDevPath, "size": fmt.Sprintf("%db", sizeBytes)}) if sizeBytes < oldVolSizeBytes { if !filesystemTypeCanBeShrunk(fsType) { return fmt.Errorf("Filesystem %q cannot be shrunk: %w", fsType, ErrCannotBeShrunk) } if inUse { return ErrInUse // We don't allow online shrinking of filesystem block volumes. } // Shrink filesystem first. // Pass allowUnsafeResize to allow disabling of filesystem resize safety checks. err = shrinkFileSystem(fsType, volDevPath, vol, sizeBytes, allowUnsafeResize) if err != nil { return err } l.Debug("ZFS volume filesystem shrunk") // Shrink the block device. err = d.setDatasetProperties(d.dataset(vol, false), fmt.Sprintf("volsize=%d", sizeBytes)) if err != nil { return err } } else if sizeBytes > oldVolSizeBytes { // Grow block device first. err = d.setDatasetProperties(d.dataset(vol, false), fmt.Sprintf("volsize=%d", sizeBytes)) if err != nil { return err } // Grow the filesystem to fill block device. err = growFileSystem(fsType, volDevPath, vol) if err != nil { return err } l.Debug("ZFS volume filesystem grown") } } else { // Block image volumes cannot be resized because they have a readonly snapshot that doesn't get // updated when the volume's size is changed, and this is what instances are created from. // During initial volume fill allowUnsafeResize is enabled because snapshot hasn't been taken yet. if !allowUnsafeResize && vol.volType == VolumeTypeImage { return ErrNotSupported } // Only perform pre-resize checks if we are not in "unsafe" mode. // In unsafe mode we expect the caller to know what they are doing and understand the risks. if !allowUnsafeResize { if sizeBytes < oldVolSizeBytes { return fmt.Errorf("Block volumes cannot be shrunk: %w", ErrCannotBeShrunk) } } err = d.setDatasetProperties(d.dataset(vol, false), fmt.Sprintf("volsize=%d", sizeBytes)) if err != nil { return err } } // Move the VM GPT alt header to end of disk if needed (not needed in unsafe resize mode as // it is expected the caller will do all necessary post resize actions themselves). if vol.IsVMBlock() && !allowUnsafeResize { err = vol.MountTask(func(mountPath string, op *operations.Operation) error { devPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } return d.moveGPTAltHeader(devPath) }, op) if err != nil { return err } } return nil } // Clear the existing quota. for _, property := range []string{"quota", "refquota", "reservation", "refreservation"} { err = d.setDatasetProperties(d.dataset(vol, false), fmt.Sprintf("%s=none", property)) if err != nil { return err } } value := fmt.Sprintf("%d", sizeBytes) if sizeBytes == 0 { return nil } // Apply the new quota. quotaKey := "quota" reservationKey := "reservation" if util.IsTrue(vol.ExpandedConfig("zfs.use_refquota")) { quotaKey = "refquota" reservationKey = "refreservation" } err = d.setDatasetProperties(d.dataset(vol, false), fmt.Sprintf("%s=%s", quotaKey, value)) if err != nil { return err } if util.IsTrue(vol.ExpandedConfig("zfs.reserve_space")) { err = d.setDatasetProperties(d.dataset(vol, false), fmt.Sprintf("%s=%s", reservationKey, value)) if err != nil { return err } } return nil } // tryGetVolumeDiskPathFromDataset attempts to find the path of the block device for the given dataset. // It keeps retrying every half a second until the context is canceled or expires. func (d *zfs) tryGetVolumeDiskPathFromDataset(ctx context.Context, dataset string) (string, error) { for { if ctx.Err() != nil { return "", fmt.Errorf("Failed to locate zvol for %q: %w", dataset, ctx.Err()) } diskPath, err := d.getVolumeDiskPathFromDataset(dataset) if err == nil { return diskPath, nil } time.Sleep(500 * time.Millisecond) } } func (d *zfs) getVolumeDiskPathFromDataset(dataset string) (string, error) { // Read all the top-level /dev entries. entries, err := os.ReadDir("/dev/") if err != nil { return "", err } // Filter only the relevant ZFS entries. zfsEntries := make([]os.DirEntry, 0, len(entries)) for _, entry := range entries { // Skip non-ZFS entries. if !strings.HasPrefix(entry.Name(), "zd") { continue } // Skip partitions. if strings.Contains(entry.Name(), "p") { continue } zfsEntries = append(zfsEntries, entry) } // Sort by reverse creation date. slices.SortFunc(zfsEntries, func(a os.DirEntry, b os.DirEntry) int { var ( aCreate time.Time bCreate time.Time ) aInfo, _ := a.Info() if aInfo != nil { stat, ok := aInfo.Sys().(*syscall.Stat_t) if ok { aCreate = time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec)) } } bInfo, _ := b.Info() if bInfo != nil { stat, ok := bInfo.Sys().(*syscall.Stat_t) if ok { bCreate = time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec)) } } if aCreate.Equal(bCreate) { return 0 } if aCreate.Before(bCreate) { return 1 } return -1 }) zfsDataset := func(devPath string) string { // Open the device. r, err := os.OpenFile(devPath, unix.O_RDONLY|unix.O_CLOEXEC, 0) if err != nil { return "" } defer func() { _ = r.Close() }() // Perform the BLKZNAME ioctl. buf := [256]byte{} _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(r.Fd()), linux.IoctlBlkZname, uintptr(unsafe.Pointer(&buf))) if errno != 0 { return "" } return string(bytes.Trim(buf[:], "\x00")) } // Check each entry for a dataset match. for _, entry := range zfsEntries { // Check if it's our dataset. zfsDev := "/dev/" + entry.Name() if zfsDataset(zfsDev) == dataset { return zfsDev, nil } } return "", fmt.Errorf("Could not locate a zvol for %s", dataset) } // GetVolumeDiskPath returns the location of a root disk block device. func (d *zfs) GetVolumeDiskPath(vol Volume) (string, error) { // Wait up to 30 seconds for the device to appear. ctx, cancel := context.WithTimeout(d.state.ShutdownCtx, 30*time.Second) defer cancel() return d.tryGetVolumeDiskPathFromDataset(ctx, d.dataset(vol, false)) } // ListVolumes returns a list of volumes in storage pool. func (d *zfs) ListVolumes() ([]Volume, error) { vols := make(map[string]Volume) // Get just filesystem and volume datasets, not snapshots. // The ZFS driver uses two approaches to indicating block volumes; firstly for VM and image volumes it // creates both a filesystem dataset and an associated volume ending in zfsBlockVolSuffix. // However for custom block volumes it does not also end the volume name in zfsBlockVolSuffix (unlike the // LVM and Ceph drivers), so we must also retrieve the dataset type here and look for "volume" types // which also indicate this is a block volume. cmd := exec.Command("zfs", "list", "-H", "-o", "name,type,incus:content_type", "-r", "-t", "filesystem,volume", d.config["zfs.pool_name"]) stdout, err := cmd.StdoutPipe() if err != nil { return nil, err } stderr, err := cmd.StderrPipe() if err != nil { return nil, err } err = cmd.Start() if err != nil { return nil, err } scanner := bufio.NewScanner(stdout) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) // Splitting fields on tab should be safe as ZFS doesn't appear to allow tabs in dataset names. parts := strings.Split(line, "\t") if len(parts) != 3 { return nil, fmt.Errorf("Unexpected volume line %q", line) } zfsVolName := parts[0] zfsContentType := parts[1] incusContentType := parts[2] var volType VolumeType var volName string for _, volumeType := range d.Info().VolumeTypes { prefix := fmt.Sprintf("%s/%s/", d.config["zfs.pool_name"], volumeType) if strings.HasPrefix(zfsVolName, prefix) { volType = volumeType volName = strings.TrimPrefix(zfsVolName, prefix) } } if volType == "" { d.logger.Debug("Ignoring unrecognised volume type", logger.Ctx{"name": zfsVolName}) continue // Ignore unrecognised volume. } // Detect if a volume is block content type using only the dataset type. isBlock := zfsContentType == "volume" if volType == VolumeTypeVM && !isBlock { continue // Ignore VM filesystem volumes as we will just return the VM's block volume. } contentType := ContentTypeFS if isBlock { contentType = ContentTypeBlock } if volType == VolumeTypeCustom && isBlock && strings.HasSuffix(volName, zfsISOVolSuffix) { contentType = ContentTypeISO volName = strings.TrimSuffix(volName, zfsISOVolSuffix) } else if volType == VolumeTypeVM || isBlock { volName = strings.TrimSuffix(volName, zfsBlockVolSuffix) } // If a new volume has been found, or the volume will replace an existing image filesystem volume // then proceed to add the volume to the map. We allow image volumes to overwrite existing // filesystem volumes of the same name so that for VM images we only return the block content type // volume (so that only the single "logical" volume is returned). existingVol, foundExisting := vols[volName] if !foundExisting || (existingVol.Type() == VolumeTypeImage && existingVol.ContentType() == ContentTypeFS) { v := NewVolume(d, d.name, volType, contentType, volName, make(map[string]string), d.config) if isBlock { // Get correct content type from incus:content_type property. if incusContentType != "-" { v.contentType = ContentType(incusContentType) } if v.contentType == ContentTypeBlock { v.SetMountFilesystemProbe(true) } } vols[volName] = v continue } return nil, fmt.Errorf("Unexpected duplicate volume %q found", volName) } errMsg, err := io.ReadAll(stderr) if err != nil { return nil, err } err = cmd.Wait() if err != nil { return nil, fmt.Errorf("Failed getting volume list: %v: %w", strings.TrimSpace(string(errMsg)), err) } volList := make([]Volume, 0, len(vols)) for _, v := range vols { volList = append(volList, v) } return volList, nil } // activateVolume activates a ZFS volume if not already active. Returns true if activated, false if not. func (d *zfs) activateVolume(vol Volume) (bool, error) { if !IsContentBlock(vol.contentType) && !vol.IsBlockBacked() { return false, nil // Nothing to do for non-block or non-block backed volumes. } reverter := revert.New() defer reverter.Fail() dataset := d.dataset(vol, false) // Check if already active. current, err := d.getDatasetProperty(dataset, "volmode") if err != nil { return false, err } if current != "dev" { // For block backed volumes, we make their associated device appear. err = d.setDatasetProperties(dataset, "volmode=dev") if err != nil { return false, err } reverter.Add(func() { _ = d.setDatasetProperties(dataset, fmt.Sprintf("volmode=%s", current)) }) // Wait up to 30 seconds for the device to appear. ctx, cancel := context.WithTimeout(d.state.ShutdownCtx, 30*time.Second) defer cancel() _, err := d.tryGetVolumeDiskPathFromDataset(ctx, dataset) if err != nil { return false, fmt.Errorf("Failed to activate volume: %v", err) } d.logger.Debug("Activated ZFS volume", logger.Ctx{"volName": vol.Name(), "dev": dataset}) reverter.Success() return true, nil } return false, nil } // deactivateVolume deactivates a ZFS volume if activate. Returns true if deactivated, false if not. func (d *zfs) deactivateVolume(vol Volume) (bool, error) { if vol.contentType != ContentTypeBlock && !vol.IsBlockBacked() { return false, nil // Nothing to do for non-block and non-block backed volumes. } dataset := d.dataset(vol, false) // Check if currently active. current, err := d.getDatasetProperty(dataset, "volmode") if err != nil { return false, err } if current == "dev" { devPath, err := d.GetVolumeDiskPath(vol) if err != nil { return false, fmt.Errorf("Failed locating zvol for deactivation: %w", err) } // We cannot wait longer than the operationlock.TimeoutShutdown to avoid continuing // the unmount process beyond the ongoing request. waitDuration := time.Minute * 5 waitUntil := time.Now().Add(waitDuration) i := 0 for { // Sometimes it takes multiple attempts for ZFS to actually apply this. err = d.setDatasetProperties(dataset, "volmode=none") if err != nil { return false, err } if !util.PathExists(devPath) { d.logger.Debug("Deactivated ZFS volume", logger.Ctx{"volName": vol.name, "dev": dataset}) break } if time.Now().After(waitUntil) { return false, fmt.Errorf("Failed to deactivate zvol after %v", waitDuration) } // Wait for ZFS a chance to flush and udev to remove the device path. d.logger.Debug("Waiting for ZFS volume to deactivate", logger.Ctx{"volName": vol.name, "dev": dataset, "path": devPath, "attempt": i}) if i <= 5 { // Retry more quickly early on. time.Sleep(time.Second * time.Duration(i)) } else { time.Sleep(time.Second * time.Duration(5)) } i++ } return true, nil } return false, nil } // MountVolume mounts a volume and increments ref counter. Please call UnmountVolume() when done with the volume. func (d *zfs) MountVolume(vol Volume, op *operations.Operation) error { unlock, err := vol.MountLock() if err != nil { return err } defer unlock() reverter := revert.New() defer reverter.Fail() dataset := d.dataset(vol, false) mountPath := vol.MountPath() // Check if filesystem volume already mounted. if vol.contentType == ContentTypeFS && !d.isBlockBacked(vol) { if !linux.IsMountPoint(mountPath) { err := d.setDatasetProperties(dataset, "mountpoint=legacy", "canmount=noauto") if err != nil { return err } if zfsDelegate && util.IsTrue(vol.config["zfs.delegate"]) { err = d.setDatasetProperties(dataset, "zoned=on") if err != nil { return err } } err = vol.EnsureMountPath(false) if err != nil { return err } var volOptions []string props, _ := d.getDatasetProperties(dataset, "atime", "relatime") if props["atime"] == "off" { volOptions = append(volOptions, "noatime") } else if props["relatime"] == "off" { volOptions = append(volOptions, "strictatime") } mountFlags, mountOptions := linux.ResolveMountOptions(volOptions) // Mount the dataset. err = TryMount(dataset, mountPath, "zfs", mountFlags, mountOptions) if err != nil { return err } d.logger.Debug("Mounted ZFS dataset", logger.Ctx{"volName": vol.name, "dev": dataset, "path": mountPath}) } } else { // For block devices, we make them appear. activated, err := d.activateVolume(vol) if err != nil { return err } if activated { reverter.Add(func() { _, _ = d.deactivateVolume(vol) }) } if !IsContentBlock(vol.contentType) && d.isBlockBacked(vol) && !linux.IsMountPoint(mountPath) { volPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } err = vol.EnsureMountPath(false) if err != nil { return err } mountFlags, mountOptions := linux.ResolveMountOptions(strings.Split(vol.ConfigBlockMountOptions(), ",")) err = TryMount(volPath, mountPath, vol.ConfigBlockFilesystem(), mountFlags, mountOptions) if err != nil { return err } d.logger.Debug("Mounted ZFS volume", logger.Ctx{"volName": vol.name, "dev": dataset, "path": mountPath}) } if vol.IsVMBlock() { // For VMs, also mount the filesystem dataset. fsVol := vol.NewVMBlockFilesystemVolume() err = d.MountVolume(fsVol, op) if err != nil { return err } } } vol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolume() when done. reverter.Success() return nil } // UnmountVolume unmounts volume if mounted and not in use. Returns true if this unmounted the volume. // keepBlockDev indicates if backing block device should be not be deactivated when volume is unmounted. func (d *zfs) UnmountVolume(vol Volume, keepBlockDev bool, op *operations.Operation) (bool, error) { unlock, err := vol.MountLock() if err != nil { return false, err } defer unlock() ourUnmount := false dataset := d.dataset(vol, false) mountPath := vol.MountPath() refCount := vol.MountRefCountDecrement() if vol.contentType == ContentTypeFS && linux.IsMountPoint(mountPath) { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": vol.name, "refCount": refCount}) return false, ErrInUse } // Unmount the dataset. err = TryUnmount(mountPath, 0) if err != nil { return false, err } blockBacked := d.isBlockBacked(vol) if blockBacked { d.logger.Debug("Unmounted ZFS volume", logger.Ctx{"volName": vol.name, "dev": dataset, "path": mountPath}) } else { d.logger.Debug("Unmounted ZFS dataset", logger.Ctx{"volName": vol.name, "dev": dataset, "path": mountPath}) } if !blockBacked && zfsDelegate && util.IsTrue(vol.config["zfs.delegate"]) { err = d.setDatasetProperties(dataset, "zoned=off") if err != nil { return false, err } } if blockBacked && !keepBlockDev { // For block devices, we make them disappear if active. _, err = d.deactivateVolume(vol) if err != nil { return false, err } } ourUnmount = true } else if IsContentBlock(vol.contentType) { // For VMs, also unmount the filesystem dataset. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() ourUnmount, err = d.UnmountVolume(fsVol, false, op) if err != nil { return false, err } } if !keepBlockDev { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": vol.name, "refCount": refCount}) return false, ErrInUse } // For block devices, we make them disappear if active. ourUnmount, err = d.deactivateVolume(vol) if err != nil { return false, err } } } return ourUnmount, nil } // RenameVolume renames a volume and its snapshots. func (d *zfs) RenameVolume(vol Volume, newVolName string, op *operations.Operation) error { newVol := NewVolume(d, d.name, vol.volType, vol.contentType, newVolName, vol.config, vol.poolConfig) // Revert handling. reverter := revert.New() defer reverter.Fail() // First rename the VFS paths. err := genericVFSRenameVolume(d, vol, newVolName, op) if err != nil { return err } reverter.Add(func() { _ = genericVFSRenameVolume(d, newVol, vol.name, op) }) // Rename the ZFS datasets. _, err = subprocess.RunCommand("zfs", "rename", d.dataset(vol, false), d.dataset(newVol, false)) if err != nil { return err } reverter.Add(func() { _, _ = subprocess.RunCommand("zfs", "rename", d.dataset(newVol, false), d.dataset(vol, false)) }) // Ensure the volume has correct mountpoint settings. if vol.contentType == ContentTypeFS && !d.isBlockBacked(vol) { err = d.setDatasetProperties(d.dataset(newVol, false), "mountpoint=legacy", "canmount=noauto") if err != nil { return err } } // For VM images, create a filesystem volume too. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.RenameVolume(fsVol, newVolName, op) if err != nil { return err } reverter.Add(func() { newFsVol := NewVolume(d, d.name, newVol.volType, ContentTypeFS, newVol.name, newVol.config, newVol.poolConfig) _ = d.RenameVolume(newFsVol, vol.name, op) }) } // All done. reverter.Success() return nil } // CanDelegateVolume checks whether the volume may be delegated. func (d *zfs) CanDelegateVolume(vol Volume) bool { // Not applicable for block backed volumes. if d.isBlockBacked(vol) { return false } // Check that the volume has it enabled. if util.IsFalseOrEmpty(vol.Config()["zfs.delegate"]) { return false } return true } // DelegateVolume allows for the volume to be managed by the instance itself. func (d *zfs) DelegateVolume(vol Volume, pid int) error { if !d.CanDelegateVolume(vol) { return nil } // Check that the current ZFS version supports it. if !zfsDelegate { return errors.New("Local ZFS version doesn't support delegation") } // Set the property. err := d.delegateDataset(vol, pid) if err != nil { return err } return nil } // MigrateVolume sends a volume for migration. func (d *zfs) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *localMigration.VolumeSourceArgs, op *operations.Operation) error { if !volSrcArgs.AllowInconsistent && vol.contentType == ContentTypeFS && vol.IsBlockBacked() { // When migrating using zfs volumes (not datasets), ensure that the filesystem is synced // otherwise the source and target volumes may differ. Tests have shown that only calling // os.SyncFS() doesn't suffice. A freeze and unfreeze is needed. err := vol.MountTask(func(mountPath string, op *operations.Operation) error { unfreezeFS, err := d.filesystemFreeze(mountPath) if err != nil { return err } return unfreezeFS() }, op) if err != nil { return err } } // Handle simple rsync and block_and_rsync through generic. if volSrcArgs.MigrationType.FSType == migration.MigrationFSType_RSYNC || volSrcArgs.MigrationType.FSType == migration.MigrationFSType_BLOCK_AND_RSYNC { // If volume is filesystem type, create a fast snapshot to ensure migration is consistent. // TODO add support for temporary snapshots of block volumes here. if vol.contentType == ContentTypeFS && !vol.IsSnapshot() && linux.IsMountPoint(vol.MountPath()) { snapshotPath, cleanup, err := d.readonlySnapshot(vol) if err != nil { return err } // Clean up the snapshot. defer cleanup() // Set the path of the volume to the path of the fast snapshot so the migration reads from there instead. vol.mountCustomPath = snapshotPath } return genericVFSMigrateVolume(d, d.state, vol, conn, volSrcArgs, op) } else if volSrcArgs.MigrationType.FSType != migration.MigrationFSType_ZFS { return ErrNotSupported } // Handle zfs send/receive migration. if volSrcArgs.MultiSync || volSrcArgs.FinalSync { // This is not needed if the migration is performed using zfs send/receive. return errors.New("MultiSync should not be used with optimized migration") } var srcMigrationHeader *ZFSMetaDataHeader // The target will validate the GUIDs and if successful proceed with the refresh. if slices.Contains(volSrcArgs.MigrationType.Features, migration.ZFSFeatureMigrationHeader) { snapshots, err := d.VolumeSnapshots(vol, op) if err != nil { return err } // Fill the migration header with the snapshot names and dataset GUIDs. srcMigrationHeader, err = d.datasetHeader(vol, snapshots) if err != nil { return err } headerJSON, err := json.Marshal(srcMigrationHeader) if err != nil { return fmt.Errorf("Failed encoding ZFS migration header: %w", err) } // Send the migration header to the target. _, err = conn.Write(headerJSON) if err != nil { return fmt.Errorf("Failed sending ZFS migration header: %w", err) } err = conn.Close() // End the frame. if err != nil { return fmt.Errorf("Failed closing ZFS migration header frame: %w", err) } } // If we haven't negotiated zvol support, ensure volume is not a zvol. if !slices.Contains(volSrcArgs.MigrationType.Features, migration.ZFSFeatureZvolFilesystems) && d.isBlockBacked(vol) { return errors.New("Filesystem zvol detected in source but target does not support receiving zvols") } incrementalStream := true var migrationHeader ZFSMetaDataHeader if volSrcArgs.Refresh && slices.Contains(volSrcArgs.MigrationType.Features, migration.ZFSFeatureMigrationHeader) { buf, err := io.ReadAll(conn) if err != nil { return fmt.Errorf("Failed reading ZFS migration header: %w", err) } err = json.Unmarshal(buf, &migrationHeader) if err != nil { return fmt.Errorf("Failed decoding ZFS migration header: %w", err) } // If the target has no snapshots we cannot use incremental streams and will do a normal copy operation instead. if len(migrationHeader.SnapshotDatasets) == 0 { incrementalStream = false volSrcArgs.Refresh = false } volSrcArgs.Snapshots = []string{} // Override volSrcArgs.Snapshots to only include snapshots which need to be sent. if !volSrcArgs.VolumeOnly { for _, srcDataset := range srcMigrationHeader.SnapshotDatasets { found := false for _, dstDataset := range migrationHeader.SnapshotDatasets { if srcDataset.GUID == dstDataset.GUID { found = true break } } if !found { volSrcArgs.Snapshots = append(volSrcArgs.Snapshots, srcDataset.Name) } } } } return d.migrateVolumeOptimized(vol, conn, volSrcArgs, incrementalStream, op) } func (d *zfs) migrateVolumeOptimized(vol Volume, conn io.ReadWriteCloser, volSrcArgs *localMigration.VolumeSourceArgs, incremental bool, op *operations.Operation) error { if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.migrateVolumeOptimized(fsVol, conn, volSrcArgs, incremental, op) if err != nil { return err } } // Handle zfs send/receive migration. var finalParent string // Transfer the snapshots first. for i, snapName := range volSrcArgs.Snapshots { snapshot, _ := vol.NewSnapshot(snapName) // Figure out parent and current subvolumes. parent := "" if i == 0 && volSrcArgs.Refresh { snapshots, err := vol.Snapshots(op) if err != nil { return err } for k, snap := range snapshots { if k == 0 { continue } if snap.name == fmt.Sprintf("%s/%s", vol.name, snapName) { parent = d.dataset(snapshots[k-1], false) break } } } else if i > 0 { oldSnapshot, _ := vol.NewSnapshot(volSrcArgs.Snapshots[i-1]) parent = d.dataset(oldSnapshot, false) } // Setup progress tracking. var wrapper *ioprogress.ProgressTracker if volSrcArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", snapshot.name) } // Send snapshot to recipient (ensure local snapshot volume is mounted if needed). err := d.sendDataset(d.dataset(snapshot, false), parent, volSrcArgs, conn, wrapper) if err != nil { return err } finalParent = d.dataset(snapshot, false) } // Setup progress tracking. var wrapper *ioprogress.ProgressTracker if volSrcArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", vol.name) } srcSnapshot := d.dataset(vol, false) if !vol.IsSnapshot() { // Create a temporary read-only snapshot. srcSnapshot = fmt.Sprintf("%s@migration-%s", d.dataset(vol, false), uuid.New().String()) _, err := subprocess.RunCommand("zfs", "snapshot", "-r", srcSnapshot) if err != nil { return err } defer func() { // Delete snapshot (or mark for deferred deletion if cannot be deleted currently). _, err := subprocess.RunCommand("zfs", "destroy", "-r", "-d", srcSnapshot) if err != nil { d.logger.Warn("Failed deleting temporary snapshot for migration", logger.Ctx{"snapshot": srcSnapshot, "err": err}) } }() } // Get parent snapshot of the main volume which can then be used to send an incremental stream. if volSrcArgs.Refresh && incremental { localSnapshots, err := vol.Snapshots(op) if err != nil { return err } if len(localSnapshots) > 0 { finalParent = d.dataset(localSnapshots[len(localSnapshots)-1], false) } } // Send the volume itself. err := d.sendDataset(srcSnapshot, finalParent, volSrcArgs, conn, wrapper) if err != nil { return err } return nil } func (d *zfs) readonlySnapshot(vol Volume) (string, revert.Hook, error) { reverter := revert.New() defer reverter.Fail() poolPath := GetPoolMountPath(d.name) tmpDir, err := os.MkdirTemp(poolPath, "backup.") if err != nil { return "", nil, err } reverter.Add(func() { _ = os.RemoveAll(tmpDir) }) err = os.Chmod(tmpDir, 0o100) if err != nil { return "", nil, err } snapshotOnlyName := fmt.Sprintf("temp_ro-%s", uuid.New().String()) snapVol, err := vol.NewSnapshot(snapshotOnlyName) if err != nil { return "", nil, err } snapshotDataset := fmt.Sprintf("%s@%s", d.dataset(vol, false), snapshotOnlyName) // Create a temporary snapshot. _, err = subprocess.RunCommand("zfs", "snapshot", "-r", snapshotDataset) if err != nil { return "", nil, err } reverter.Add(func() { // Delete snapshot (or mark for deferred deletion if cannot be deleted currently). _, err := subprocess.RunCommand("zfs", "destroy", "-r", "-d", snapshotDataset) if err != nil { d.logger.Warn("Failed deleting read-only snapshot", logger.Ctx{"snapshot": snapshotDataset, "err": err}) } }) hook, err := d.mountVolumeSnapshot(snapVol, snapshotDataset, tmpDir, nil) if err != nil { return "", nil, err } reverter.Add(hook) cleanup := reverter.Clone().Fail reverter.Success() return tmpDir, cleanup, nil } // BackupVolume creates an exported version of a volume. func (d *zfs) BackupVolume(vol Volume, writer instancewriter.InstanceWriter, basePrefix string, optimized bool, snapshots []string, op *operations.Operation) error { // Handle the non-optimized tarballs through the generic packer. if !optimized { // Because the generic backup method will not take a consistent backup if files are being modified // as they are copied to the tarball, as ZFS allows us to take a quick snapshot without impacting // the parent volume we do so here to ensure the backup taken is consistent. if vol.contentType == ContentTypeFS && !d.isBlockBacked(vol) { snapshotPath, cleanup, err := d.readonlySnapshot(vol) if err != nil { return err } // Clean up the snapshot. defer cleanup() // Set the path of the volume to the path of the fast snapshot so the migration reads from there instead. vol.mountCustomPath = snapshotPath } return genericVFSBackupVolume(d, vol, writer, basePrefix, snapshots, op) } // Optimized backup. if len(snapshots) > 0 { // Check requested snapshot match those in storage. err := vol.SnapshotsMatch(snapshots, op) if err != nil { return err } } // Backup VM config volumes first. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.BackupVolume(fsVol, writer, basePrefix, optimized, snapshots, op) if err != nil { return err } } // Handle the optimized tarballs. sendToFile := func(path string, parent string, fileName string) error { // Prepare zfs send arguments. args := []string{"send", "-c", "-L"} // Check if nesting is required. if d.needsRecursion(path) { args = append(args, "-R", "-w") } if parent != "" { args = append(args, "-i", parent) } args = append(args, path) // Create temporary file to store output of ZFS send. backupsPath := internalUtil.VarPath("backups") tmpFile, err := os.CreateTemp(backupsPath, fmt.Sprintf("%s_zfs", backup.WorkingDirPrefix)) if err != nil { return fmt.Errorf("Failed to open temporary file for ZFS backup: %w", err) } defer func() { _ = tmpFile.Close() }() defer func() { _ = os.Remove(tmpFile.Name()) }() // Write the subvolume to the file. d.logger.Debug("Generating optimized volume file", logger.Ctx{"sourcePath": path, "file": tmpFile.Name(), "name": fileName}) // Write the subvolume to the file. err = subprocess.RunCommandWithFds(context.TODO(), nil, tmpFile, "zfs", args...) if err != nil { return err } // Get info (importantly size) of the generated file for tarball header. tmpFileInfo, err := os.Lstat(tmpFile.Name()) if err != nil { return err } err = writer.WriteFile(fileName, tmpFile.Name(), tmpFileInfo, false) if err != nil { return err } return tmpFile.Close() } // Handle snapshots. finalParent := "" if len(snapshots) > 0 { for i, snapName := range snapshots { snapshot, _ := vol.NewSnapshot(snapName) // Figure out parent and current subvolumes. parent := "" if i > 0 { oldSnapshot, _ := vol.NewSnapshot(snapshots[i-1]) parent = d.dataset(oldSnapshot, false) } // Make a binary zfs backup. prefix := "snapshots" fileName := fmt.Sprintf("%s.bin", snapName) if vol.volType == VolumeTypeVM { prefix = "virtual-machine-snapshots" if vol.contentType == ContentTypeFS { fileName = fmt.Sprintf("%s-config.bin", snapName) } } else if vol.volType == VolumeTypeCustom { prefix = "volume-snapshots" } target := filepath.Join(basePrefix, prefix, fileName) err := sendToFile(d.dataset(snapshot, false), parent, target) if err != nil { return err } finalParent = d.dataset(snapshot, false) } } // Create a temporary read-only snapshot. srcSnapshot := fmt.Sprintf("%s@backup-%s", d.dataset(vol, false), uuid.New().String()) _, err := subprocess.RunCommand("zfs", "snapshot", "-r", srcSnapshot) if err != nil { return err } defer func() { // Delete snapshot (or mark for deferred deletion if cannot be deleted currently). _, err := subprocess.RunCommand("zfs", "destroy", "-r", "-d", srcSnapshot) if err != nil { d.logger.Warn("Failed deleting temporary snapshot for backup", logger.Ctx{"snapshot": srcSnapshot, "err": err}) } }() // Dump the container to a file. fileName := "container.bin" if vol.volType == VolumeTypeVM { if vol.contentType == ContentTypeFS { fileName = "virtual-machine-config.bin" } else { fileName = "virtual-machine.bin" } } else if vol.volType == VolumeTypeCustom { fileName = "volume.bin" } err = sendToFile(srcSnapshot, finalParent, filepath.Join(basePrefix, fileName)) if err != nil { return err } return nil } // CreateVolumeSnapshot creates a snapshot of a volume. func (d *zfs) CreateVolumeSnapshot(vol Volume, op *operations.Operation) error { parentName, _, _ := api.GetParentAndSnapshotName(vol.name) // Revert handling. reverter := revert.New() defer reverter.Fail() // Create the parent directory. err := CreateParentSnapshotDirIfMissing(d.name, vol.volType, parentName) if err != nil { return err } // Create snapshot directory. err = vol.EnsureMountPath(false) if err != nil { return err } // Make the snapshot. _, err = subprocess.RunCommand("zfs", "snapshot", "-r", d.dataset(vol, false)) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolumeSnapshot(vol, op) }) // For VM images, create a filesystem volume too. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.CreateVolumeSnapshot(fsVol, op) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolumeSnapshot(fsVol, op) }) } // All done. reverter.Success() return nil } // DeleteVolumeSnapshot removes a snapshot from the storage device. func (d *zfs) DeleteVolumeSnapshot(vol Volume, op *operations.Operation) error { parentName, _, _ := api.GetParentAndSnapshotName(vol.name) // Attempt to delete the snapshot. _, errDelete := subprocess.RunCommand("zfs", "destroy", "-r", d.dataset(vol, false)) if errDelete != nil { // Handle clones. clones, err := d.getClones(d.dataset(vol, false)) if err != nil { return err } if len(clones) == 0 { return errDelete } // Move to the deleted path. _, err = subprocess.RunCommand("zfs", "rename", d.dataset(vol, false), d.dataset(vol, true)) if err != nil { return err } } // Delete the mountpoint. err := os.Remove(vol.MountPath()) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", vol.MountPath(), err) } // Remove the parent snapshot directory if this is the last snapshot being removed. err = deleteParentSnapshotDirIfEmpty(d.name, vol.volType, parentName) if err != nil { return err } // For VM images, create a filesystem volume too. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.DeleteVolumeSnapshot(fsVol, op) if err != nil { return err } } return nil } // MountVolumeSnapshot simulates mounting a volume snapshot. func (d *zfs) MountVolumeSnapshot(snapVol Volume, op *operations.Operation) error { unlock, err := snapVol.MountLock() if err != nil { return err } defer unlock() _, err = d.mountVolumeSnapshot(snapVol, d.dataset(snapVol, false), snapVol.MountPath(), op) if err != nil { return err } snapVol.MountRefCountIncrement() // From here on it is up to caller to call UnmountVolumeSnapshot() when done. return nil } func (d *zfs) mountVolumeSnapshot(snapVol Volume, snapshotDataset string, mountPath string, op *operations.Operation) (revert.Hook, error) { reverter := revert.New() defer reverter.Fail() // Check if filesystem volume already mounted. if snapVol.contentType == ContentTypeFS && !d.isBlockBacked(snapVol) { if !linux.IsMountPoint(mountPath) { err := snapVol.EnsureMountPath(false) if err != nil { return nil, err } // Mount the snapshot directly (not possible through tools). err = TryMount(snapshotDataset, mountPath, "zfs", unix.MS_RDONLY, "") if err != nil { return nil, err } d.logger.Debug("Mounted ZFS snapshot dataset", logger.Ctx{"dev": snapshotDataset, "path": mountPath}) } } else { // For block devices, we make them appear by enabling volmode=dev and snapdev=visible on the parent volume. // Ensure snap volume parent is activated to avoid issues activating the snapshot volume device. parent, snapshotOnlyName, _ := api.GetParentAndSnapshotName(snapVol.Name()) parentVol := NewVolume(d, d.Name(), snapVol.volType, snapVol.contentType, parent, snapVol.config, snapVol.poolConfig) err := d.MountVolume(parentVol, op) if err != nil { return nil, err } reverter.Add(func() { _, _ = d.UnmountVolume(parentVol, false, op) }) parentDataset := d.dataset(parentVol, false) // Check if parent already active. parentVolMode, err := d.getDatasetProperty(parentDataset, "volmode") if err != nil { return nil, err } // Order is important here, the parent volmode=dev must be set before snapdev=visible otherwise // it won't take effect. if parentVolMode != "dev" { return nil, errors.New("Parent block volume needs to be mounted first") } // Check if snapdev already set visible. parentSnapdevMode, err := d.getDatasetProperty(parentDataset, "snapdev") if err != nil { return nil, err } if parentSnapdevMode != "visible" { err = d.setDatasetProperties(parentDataset, "snapdev=visible") if err != nil { return nil, err } // Wait half a second to give udev a chance to kick in. time.Sleep(500 * time.Millisecond) d.logger.Debug("Activated ZFS snapshot volume", logger.Ctx{"dev": snapshotDataset}) } if snapVol.contentType != ContentTypeBlock && d.isBlockBacked(snapVol) && !linux.IsMountPoint(mountPath) { err = snapVol.EnsureMountPath(false) if err != nil { return nil, err } mountVol := snapVol mountFlags, mountOptions := linux.ResolveMountOptions(strings.Split(mountVol.ConfigBlockMountOptions(), ",")) dataset := snapshotDataset // Regenerate filesystem UUID if needed. This is because some filesystems do not allow mounting // multiple volumes that share the same UUID. As snapshotting a volume will copy its UUID we need // to potentially regenerate the UUID of the snapshot now that we are trying to mount it. // This is done at mount time rather than snapshot time for 2 reasons; firstly snapshots need to be // as fast as possible, and on some filesystems regenerating the UUID is a slow process, secondly // we do not want to modify a snapshot in case it is corrupted for some reason, so at mount time // we take another snapshot of the snapshot, regenerate the temporary snapshot's UUID and then // mount that. regenerateFSUUID := renegerateFilesystemUUIDNeeded(snapVol.ConfigBlockFilesystem()) if regenerateFSUUID { // Instantiate a new volume to be the temporary writable snapshot. tmpVolName := fmt.Sprintf("%s%s", snapVol.name, tmpVolSuffix) tmpVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, tmpVolName, snapVol.config, snapVol.poolConfig) dataset = fmt.Sprintf("%s_%s%s", parentDataset, snapshotOnlyName, tmpVolSuffix) // Clone snapshot. _, err = subprocess.RunCommand("zfs", "clone", snapshotDataset, dataset) if err != nil { return nil, err } // Delete on revert. reverter.Add(func() { _ = d.deleteDatasetRecursive(dataset) }) err := d.setDatasetProperties(dataset, "volmode=dev") if err != nil { return nil, err } reverter.Add(func() { _ = d.setDatasetProperties(dataset, "volmode=none") }) // Wait half a second to give udev a chance to kick in. time.Sleep(500 * time.Millisecond) d.logger.Debug("Activated ZFS volume", logger.Ctx{"dev": dataset}) // We are going to mount the temporary volume instead. mountVol = tmpVol } volPath, err := d.getVolumeDiskPathFromDataset(dataset) if err != nil { return nil, err } tmpVolFsType := mountVol.ConfigBlockFilesystem() if regenerateFSUUID { // When mounting XFS filesystems temporarily we can use the nouuid option rather than fully // regenerating the filesystem UUID. if tmpVolFsType == "xfs" { idx := strings.Index(mountOptions, "nouuid") if idx < 0 { mountOptions += ",nouuid" } } else { d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": volPath, "fs": tmpVolFsType}) err = regenerateFilesystemUUID(mountVol.ConfigBlockFilesystem(), volPath) if err != nil { return nil, err } } } else { // ext4 will replay the journal if the filesystem is dirty. // To prevent this kind of write access, we mount the ext4 filesystem // with the ro,noload mount options. // The noload option prevents the journal from being loaded on mounting. if tmpVolFsType == "ext4" { idx := strings.Index(mountOptions, "noload") if idx < 0 { mountOptions += ",noload" } } } err = TryMount(volPath, mountPath, mountVol.ConfigBlockFilesystem(), mountFlags|unix.MS_RDONLY, mountOptions) if err != nil { return nil, fmt.Errorf("Failed mounting volume snapshot: %w", err) } } if snapVol.IsVMBlock() { // For VMs, also mount the filesystem dataset. fsVol := snapVol.NewVMBlockFilesystemVolume() err = d.MountVolumeSnapshot(fsVol, op) if err != nil { return nil, err } } } d.logger.Debug("Mounted ZFS snapshot dataset", logger.Ctx{"dev": snapshotDataset, "path": mountPath}) reverter.Add(func() { _, err := forceUnmount(mountPath) if err != nil { return } d.logger.Debug("Unmounted ZFS snapshot dataset", logger.Ctx{"dev": snapshotDataset, "path": mountPath}) }) cleanup := reverter.Clone().Fail reverter.Success() return cleanup, nil } // UnmountVolume simulates unmounting a volume snapshot. func (d *zfs) UnmountVolumeSnapshot(snapVol Volume, op *operations.Operation) (bool, error) { unlock, err := snapVol.MountLock() if err != nil { return false, err } defer unlock() ourUnmount := false mountPath := snapVol.MountPath() snapshotDataset := d.dataset(snapVol, false) refCount := snapVol.MountRefCountDecrement() // For block devices, we make them disappear. if snapVol.contentType == ContentTypeBlock || snapVol.contentType == ContentTypeFS && d.isBlockBacked(snapVol) { // For VMs, also unmount the filesystem dataset. if snapVol.IsVMBlock() { fsSnapVol := snapVol.NewVMBlockFilesystemVolume() ourUnmount, err = d.UnmountVolumeSnapshot(fsSnapVol, op) if err != nil { return false, err } } if snapVol.contentType == ContentTypeFS && d.isBlockBacked(snapVol) && linux.IsMountPoint(mountPath) { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": snapVol.name, "refCount": refCount}) return false, ErrInUse } _, err := forceUnmount(mountPath) if err != nil { return false, err } d.logger.Debug("Unmounted ZFS snapshot dataset", logger.Ctx{"dev": snapshotDataset, "path": mountPath}) ourUnmount = true parent, snapshotOnlyName, _ := api.GetParentAndSnapshotName(snapVol.Name()) parentVol := NewVolume(d, d.Name(), snapVol.volType, snapVol.contentType, parent, snapVol.config, snapVol.poolConfig) parentDataset := d.dataset(parentVol, false) dataset := fmt.Sprintf("%s_%s%s", parentDataset, snapshotOnlyName, tmpVolSuffix) exists, err := d.datasetExists(dataset) if err != nil { return true, fmt.Errorf("Failed to check existence of temporary ZFS snapshot volume %q: %w", dataset, err) } if exists { err = d.deleteDatasetRecursive(dataset) if err != nil { return true, err } } } parent, _, _ := api.GetParentAndSnapshotName(snapVol.Name()) parentVol := NewVolume(d, d.Name(), snapVol.volType, snapVol.contentType, parent, snapVol.config, snapVol.poolConfig) parentDataset := d.dataset(parentVol, false) current, err := d.getDatasetProperty(parentDataset, "snapdev") if err != nil { return false, err } if current == "visible" { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": snapVol.name, "refCount": refCount}) return false, ErrInUse } err := d.setDatasetProperties(parentDataset, "snapdev=hidden") if err != nil { return false, err } d.logger.Debug("Deactivated ZFS snapshot volume", logger.Ctx{"dev": snapshotDataset}) // Ensure snap volume parent is deactivated in case we activated it when mounting snapshot. _, err = d.UnmountVolume(parentVol, false, op) if err != nil { return false, err } ourUnmount = true } } else if snapVol.contentType == ContentTypeFS && linux.IsMountPoint(mountPath) { if refCount > 0 { d.logger.Debug("Skipping unmount as in use", logger.Ctx{"volName": snapVol.name, "refCount": refCount}) return false, ErrInUse } _, err := forceUnmount(mountPath) if err != nil { return false, err } d.logger.Debug("Unmounted ZFS snapshot dataset", logger.Ctx{"dev": snapshotDataset, "path": mountPath}) ourUnmount = true } return ourUnmount, nil } // VolumeSnapshots returns a list of snapshots for the volume (in no particular order). func (d *zfs) VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) { // Get all children datasets. entries, err := d.getDatasets(d.dataset(vol, false), "snapshot") if err != nil { return nil, err } // Filter only the snapshots. snapshots := []string{} for _, entry := range entries { after, ok := strings.CutPrefix(entry, "@snapshot-") if ok { snapshots = append(snapshots, after) } } return snapshots, nil } // CanRestoreVolume restores a volume from a snapshot. func (d *zfs) CanRestoreVolume(vol Volume, snapshotName string) error { // Get the list of snapshots. entries, err := d.getDatasets(d.dataset(vol, false), "snapshot") if err != nil { return err } // Check if more recent snapshots exist. idx := -1 snapshots := []string{} for i, entry := range entries { if entry == fmt.Sprintf("@snapshot-%s", snapshotName) { // Located the current snapshot. idx = i continue } else if idx < 0 { // Skip any previous snapshot. continue } after, ok := strings.CutPrefix(entry, "@snapshot-") if ok { // Located a normal snapshot following ours. snapshots = append(snapshots, after) continue } if strings.HasPrefix(entry, "@") { // Located an internal snapshot. return fmt.Errorf("Snapshot %q cannot be restored due to subsequent internal snapshot(s) (from a copy)", snapshotName) } } // Check if snapshot removal is allowed. if len(snapshots) > 0 { if util.IsFalseOrEmpty(vol.ExpandedConfig("zfs.remove_snapshots")) { return fmt.Errorf("Snapshot %q cannot be restored due to subsequent snapshot(s). Set zfs.remove_snapshots to override", snapshotName) } // Setup custom error to tell the backend what to delete. err := ErrDeleteSnapshots{} err.Snapshots = snapshots return err } return nil } // RestoreVolume restores a volume from a snapshot. func (d *zfs) RestoreVolume(vol Volume, snapshotName string, op *operations.Operation) error { return d.restoreVolume(vol, snapshotName, false, op) } func (d *zfs) restoreVolume(vol Volume, snapshotName string, isMigration bool, op *operations.Operation) error { err := d.CanRestoreVolume(vol, snapshotName) if err != nil { return err } // Restore the snapshot. datasets, err := d.getDatasets(d.dataset(vol, false), "snapshot") if err != nil { return err } for _, dataset := range datasets { if !strings.HasSuffix(dataset, fmt.Sprintf("@snapshot-%s", snapshotName)) { continue } _, err = subprocess.RunCommand("zfs", "rollback", fmt.Sprintf("%s%s", d.dataset(vol, false), dataset)) if err != nil { return err } } if vol.contentType == ContentTypeFS && d.isBlockBacked(vol) && renegerateFilesystemUUIDNeeded(vol.ConfigBlockFilesystem()) { _, err = d.activateVolume(vol) if err != nil { return err } defer func() { _, _ = d.deactivateVolume(vol) }() volPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": volPath, "fs": vol.ConfigBlockFilesystem()}) err = regenerateFilesystemUUID(vol.ConfigBlockFilesystem(), volPath) if err != nil { return err } } // For VM images, restore the associated filesystem dataset too. if !isMigration && vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.restoreVolume(fsVol, snapshotName, isMigration, op) if err != nil { return err } } return nil } // RenameVolumeSnapshot renames a volume snapshot. func (d *zfs) RenameVolumeSnapshot(vol Volume, newSnapshotName string, op *operations.Operation) error { parentName, _, _ := api.GetParentAndSnapshotName(vol.name) newVol := NewVolume(d, d.name, vol.volType, vol.contentType, fmt.Sprintf("%s/%s", parentName, newSnapshotName), vol.config, vol.poolConfig) // Revert handling. reverter := revert.New() defer reverter.Fail() // First rename the VFS paths. err := genericVFSRenameVolumeSnapshot(d, vol, newSnapshotName, op) if err != nil { return err } reverter.Add(func() { _ = genericVFSRenameVolumeSnapshot(d, newVol, vol.name, op) }) // Rename the ZFS datasets. _, err = subprocess.RunCommand("zfs", "rename", d.dataset(vol, false), d.dataset(newVol, false)) if err != nil { return err } reverter.Add(func() { _, _ = subprocess.RunCommand("zfs", "rename", d.dataset(newVol, false), d.dataset(vol, false)) }) // For VM images, create a filesystem volume too. if vol.IsVMBlock() { fsVol := vol.NewVMBlockFilesystemVolume() err := d.RenameVolumeSnapshot(fsVol, newSnapshotName, op) if err != nil { return err } reverter.Add(func() { newFsVol := NewVolume(d, d.name, newVol.volType, ContentTypeFS, newVol.name, newVol.config, newVol.poolConfig) _ = d.RenameVolumeSnapshot(newFsVol, vol.name, op) }) } // All done. reverter.Success() return nil } // FillVolumeConfig populate volume with default config. func (d *zfs) FillVolumeConfig(vol Volume) error { var excludedKeys []string // Copy volume.* configuration options from pool. // If vol has a source, ignore the block mode related config keys from the pool. if vol.hasSource || vol.IsVMBlock() || vol.volType == VolumeTypeCustom && vol.contentType == ContentTypeBlock { excludedKeys = []string{"zfs.block_mode", "block.filesystem", "block.mount_options"} } else if vol.volType == VolumeTypeCustom && !vol.IsBlockBacked() { excludedKeys = []string{"block.filesystem", "block.mount_options"} } err := d.fillVolumeConfig(&vol, excludedKeys...) if err != nil { return err } // Only validate filesystem config keys for filesystem volumes. if d.isBlockBacked(vol) && vol.ContentType() == ContentTypeFS { // Inherit block mode from pool if not set. if vol.config["zfs.block_mode"] == "" { vol.config["zfs.block_mode"] = d.config["volume.zfs.block_mode"] } // Inherit filesystem from pool if not set. if vol.config["block.filesystem"] == "" { vol.config["block.filesystem"] = d.config["volume.block.filesystem"] } // Default filesystem if neither volume nor pool specify an override. if vol.config["block.filesystem"] == "" { // Unchangeable volume property: Set unconditionally. vol.config["block.filesystem"] = DefaultFilesystem } // Inherit filesystem mount options from pool if not set. if vol.config["block.mount_options"] == "" { vol.config["block.mount_options"] = d.config["volume.block.mount_options"] } // Default filesystem mount options if neither volume nor pool specify an override. if vol.config["block.mount_options"] == "" { // Unchangeable volume property: Set unconditionally. vol.config["block.mount_options"] = "discard" } } return nil } func (d *zfs) isBlockBacked(vol Volume) bool { return util.IsTrue(vol.Config()["zfs.block_mode"]) } // ActivateTask allows running a function while the volume is active (but not mounted). func (d *zfs) ActivateTask(vol Volume, task func(devPath string, op *operations.Operation) error, op *operations.Operation) error { // Prevent concurrent mounting actions. unlock, err := vol.MountLock() if err != nil { return err } defer unlock() // Activate the volume. activated, err := d.activateVolume(vol) if err != nil { return err } if !activated { return errors.New("Volume is already active, can't run exclusive activation task") } // Get the device path. volDevPath, err := d.GetVolumeDiskPath(vol) if err != nil { return err } // Run the task. taskErr := task(volDevPath, op) // Deactivate the volume. _, err = d.deactivateVolume(vol) if err != nil { return err } return taskErr } incus-7.0.0/internal/server/storage/drivers/errors.go000066400000000000000000000016741517523235500227660ustar00rootroot00000000000000package drivers import ( "errors" "fmt" ) // ErrUnknownDriver is the "Unknown driver" error. var ErrUnknownDriver = errors.New("Unknown driver") // ErrNotSupported is the "Not supported" error. var ErrNotSupported = errors.New("Not supported") // ErrCannotBeShrunk is the "Cannot be shrunk" error. var ErrCannotBeShrunk = errors.New("Cannot be shrunk") // ErrInUse indicates operation cannot proceed as resource is in use. var ErrInUse = errors.New("In use") // ErrSnapshotDoesNotMatchIncrementalSource in the "Snapshot does not match incremental source" error. var ErrSnapshotDoesNotMatchIncrementalSource = errors.New("Snapshot does not match incremental source") // ErrDeleteSnapshots is a special error used to tell the backend to delete more recent snapshots. type ErrDeleteSnapshots struct { Snapshots []string } func (e ErrDeleteSnapshots) Error() string { return fmt.Sprintf("More recent snapshots must be deleted: %+v", e.Snapshots) } incus-7.0.0/internal/server/storage/drivers/generic_vfs.go000066400000000000000000000710661517523235500237460ustar00rootroot00000000000000package drivers import ( "errors" "fmt" "io" "io/fs" "os" "path/filepath" "strings" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/rsync" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/server/sys" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // genericVolumeBlockExtension extension used for generic block volume disk files. const genericVolumeBlockExtension = "img" // genericVolumeDiskFile used to indicate the file name used for block volume disk files. const genericVolumeDiskFile = "root.img" // genericISOVolumeSuffix suffix used for generic iso content type volumes. const genericISOVolumeSuffix = ".iso" // genericVFSGetResources is a generic GetResources implementation for VFS-only drivers. func genericVFSGetResources(d Driver) (*api.ResourcesStoragePool, error) { // Get the VFS information st, err := linux.StatVFS(GetPoolMountPath(d.Name())) if err != nil { return nil, err } // Fill in the struct res := api.ResourcesStoragePool{} res.Space.Total = st.Blocks * uint64(st.Bsize) res.Space.Used = (st.Blocks - st.Bfree) * uint64(st.Bsize) // Some filesystems don't report inodes since they allocate them // dynamically e.g. btrfs. if st.Files > 0 { res.Inodes.Total = st.Files res.Inodes.Used = st.Files - st.Ffree } return &res, nil } // genericVFSRenameVolume is a generic RenameVolume implementation for VFS-only drivers. func genericVFSRenameVolume(d Driver, vol Volume, newVolName string, op *operations.Operation) error { if vol.IsSnapshot() { return errors.New("Volume must not be a snapshot") } reverter := revert.New() defer reverter.Fail() volName := vol.name // Add a .iso suffix to ISO volumes. if vol.volType == VolumeTypeCustom && vol.contentType == ContentTypeISO { volName = volName + genericISOVolumeSuffix newVolName = newVolName + genericISOVolumeSuffix } // Rename the volume itself. srcVolumePath := GetVolumeMountPath(d.Name(), vol.volType, volName) dstVolumePath := GetVolumeMountPath(d.Name(), vol.volType, newVolName) if util.PathExists(srcVolumePath) { err := os.Rename(srcVolumePath, dstVolumePath) if err != nil { return fmt.Errorf("Failed to rename %q to %q: %w", srcVolumePath, dstVolumePath, err) } reverter.Add(func() { _ = os.Rename(dstVolumePath, srcVolumePath) }) } // And if present, the snapshots too. srcSnapshotDir := GetVolumeSnapshotDir(d.Name(), vol.volType, vol.name) dstSnapshotDir := GetVolumeSnapshotDir(d.Name(), vol.volType, newVolName) if util.PathExists(srcSnapshotDir) { err := os.Rename(srcSnapshotDir, dstSnapshotDir) if err != nil { return fmt.Errorf("Failed to rename %q to %q: %w", srcSnapshotDir, dstSnapshotDir, err) } reverter.Add(func() { _ = os.Rename(dstSnapshotDir, srcSnapshotDir) }) } reverter.Success() return nil } // genericVFSVolumeSnapshots is a generic VolumeSnapshots implementation for VFS-only drivers. func genericVFSVolumeSnapshots(d Driver, vol Volume, op *operations.Operation) ([]string, error) { snapshotDir := GetVolumeSnapshotDir(d.Name(), vol.volType, vol.name) snapshots := []string{} ents, err := os.ReadDir(snapshotDir) if err != nil { // If the snapshots directory doesn't exist, there are no snapshots. if errors.Is(err, fs.ErrNotExist) { return snapshots, nil } return nil, fmt.Errorf("Failed to list directory %q: %w", snapshotDir, err) } for _, ent := range ents { fileInfo, err := os.Stat(filepath.Join(snapshotDir, ent.Name())) if err != nil { return nil, err } if !fileInfo.IsDir() { continue } snapshots = append(snapshots, ent.Name()) } return snapshots, nil } // genericVFSRenameVolumeSnapshot is a generic RenameVolumeSnapshot implementation for VFS-only drivers. func genericVFSRenameVolumeSnapshot(d Driver, snapVol Volume, newSnapshotName string, op *operations.Operation) error { if !snapVol.IsSnapshot() { return errors.New("Volume must be a snapshot") } parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) oldPath := snapVol.MountPath() newPath := GetVolumeMountPath(d.Name(), snapVol.volType, GetSnapshotVolumeName(parentName, newSnapshotName)) if util.PathExists(oldPath) { err := os.Rename(oldPath, newPath) if err != nil { return fmt.Errorf("Failed to rename %q to %q: %w", oldPath, newPath, err) } } return nil } // genericVFSMigrateVolume is a generic MigrateVolume implementation for VFS-only drivers. func genericVFSMigrateVolume(d Driver, s *state.State, vol Volume, conn io.ReadWriteCloser, volSrcArgs *localMigration.VolumeSourceArgs, op *operations.Operation) error { bwlimit := d.Config()["rsync.bwlimit"] var rsyncArgs []string // For VM volumes, exclude the generic root disk image file from being transferred via rsync, as it will // be transferred later using a different method. if vol.IsVMBlock() { if volSrcArgs.MigrationType.FSType != migration.MigrationFSType_BLOCK_AND_RSYNC { return ErrNotSupported } rsyncArgs = []string{"--exclude", genericVolumeDiskFile} } else if vol.contentType == ContentTypeBlock && volSrcArgs.MigrationType.FSType != migration.MigrationFSType_BLOCK_AND_RSYNC || vol.contentType == ContentTypeFS && volSrcArgs.MigrationType.FSType != migration.MigrationFSType_RSYNC { return ErrNotSupported } // Define function to send a filesystem volume. sendFSVol := func(vol Volume, conn io.ReadWriteCloser, mountPath string) error { var wrapper *ioprogress.ProgressTracker if volSrcArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", vol.name) } path := internalUtil.AddSlash(mountPath) d.Logger().Debug("Sending filesystem volume", logger.Ctx{"volName": vol.name, "path": path, "bwlimit": bwlimit, "rsyncArgs": rsyncArgs}) err := rsync.Send(vol.name, path, conn, wrapper, volSrcArgs.MigrationType.Features, bwlimit, s.OS.ExecPath, rsyncArgs...) status, _ := linux.ExitStatus(err) if volSrcArgs.AllowInconsistent && status == 24 { return nil } return err } // Define function to send a block volume. sendBlockVol := func(vol Volume, conn io.ReadWriteCloser) error { // Close when done to indicate to target side we are finished sending this volume. defer func() { _ = conn.Close() }() var wrapper *ioprogress.ProgressTracker if volSrcArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "block_progress", vol.name) } path, err := d.GetVolumeDiskPath(vol) if err != nil { return fmt.Errorf("Error getting VM block volume disk path: %w", err) } from, err := os.Open(path) if err != nil { return fmt.Errorf("Error opening file for reading %q: %w", path, err) } defer func() { _ = from.Close() }() // Setup progress tracker. fromPipe := io.ReadCloser(from) if wrapper != nil { fromPipe = &ioprogress.ProgressReader{ ReadCloser: fromPipe, Tracker: wrapper, } } d.Logger().Debug("Sending block volume", logger.Ctx{"volName": vol.name, "path": path}) _, err = util.SafeCopy(conn, fromPipe) if err != nil { return fmt.Errorf("Error copying %q to migration connection: %w", path, err) } err = from.Close() if err != nil { return fmt.Errorf("Failed to close file %q: %w", path, err) } return nil } // Send all snapshots to target. for _, snapName := range volSrcArgs.Snapshots { snapshot, err := vol.NewSnapshot(snapName) if err != nil { return err } // Send snapshot to target (ensure local snapshot volume is mounted if needed). err = snapshot.MountTask(func(mountPath string, op *operations.Operation) error { if vol.contentType != ContentTypeBlock || vol.volType != VolumeTypeCustom { err := sendFSVol(snapshot, conn, mountPath) if err != nil { return err } } if vol.IsVMBlock() || (vol.contentType == ContentTypeBlock && vol.volType == VolumeTypeCustom) { err = sendBlockVol(snapshot, conn) if err != nil { return err } } return nil }, op) if err != nil { return err } } // Send volume to target (ensure local volume is mounted if needed). return vol.MountTask(func(mountPath string, op *operations.Operation) error { if !IsContentBlock(vol.contentType) || vol.volType != VolumeTypeCustom { err := sendFSVol(vol, conn, mountPath) if err != nil { return err } } if vol.IsVMBlock() || (IsContentBlock(vol.contentType) && vol.volType == VolumeTypeCustom) { err := sendBlockVol(vol, conn) if err != nil { return err } } return nil }, op) } // genericVFSCreateVolumeFromMigration receives a volume and its snapshots over a non-optimized method. // initVolume is run against the main volume (not the snapshots) and is often used for quota initialization. func genericVFSCreateVolumeFromMigration(d Driver, initVolume func(vol Volume) (revert.Hook, error), vol Volume, conn io.ReadWriteCloser, volTargetArgs localMigration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error { // Check migration transport type matches volume type. if IsContentBlock(vol.contentType) { if volTargetArgs.MigrationType.FSType != migration.MigrationFSType_BLOCK_AND_RSYNC { return ErrNotSupported } } else if volTargetArgs.MigrationType.FSType != migration.MigrationFSType_RSYNC { return ErrNotSupported } reverter := revert.New() defer reverter.Fail() // Create the main volume if not refreshing. if !volTargetArgs.Refresh { err := d.CreateVolume(vol, preFiller, op) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) } recvFSVol := func(volName string, conn io.ReadWriteCloser, path string) error { var wrapper *ioprogress.ProgressTracker if volTargetArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "fs_progress", volName) } d.Logger().Debug("Receiving filesystem volume started", logger.Ctx{"volName": volName, "path": path, "features": volTargetArgs.MigrationType.Features}) defer d.Logger().Debug("Receiving filesystem volume stopped", logger.Ctx{"volName": volName, "path": path}) return rsync.Recv(path, conn, wrapper, volTargetArgs.MigrationType.Features) } recvBlockVol := func(volName string, conn io.ReadWriteCloser, path string) error { var wrapper *ioprogress.ProgressTracker if volTargetArgs.TrackProgress { wrapper = localMigration.ProgressTracker(op, "block_progress", volName) } // Reset the disk. err := linux.ClearBlock(path, 0) if err != nil { return err } to, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0) if err != nil { return fmt.Errorf("Error opening file for writing %q: %w", path, err) } defer func() { _ = to.Close() }() // Setup progress tracker. fromPipe := io.ReadCloser(conn) if wrapper != nil { fromPipe = &ioprogress.ProgressReader{ ReadCloser: fromPipe, Tracker: wrapper, } } d.Logger().Debug("Receiving block volume started", logger.Ctx{"volName": volName, "path": path}) defer d.Logger().Debug("Receiving block volume stopped", logger.Ctx{"volName": volName, "path": path}) toPipe := io.Writer(to) if !d.Info().ZeroUnpack { toPipe = NewSparseFileWrapper(to) } _, err = util.SafeCopy(toPipe, fromPipe) if err != nil { return fmt.Errorf("Error copying from migration connection to %q: %w", path, err) } return to.Close() } // Ensure the volume is mounted. err := vol.MountTask(func(mountPath string, op *operations.Operation) error { var err error // Setup paths to the main volume. We will receive each snapshot to these paths and then create // a snapshot of the main volume for each one. path := internalUtil.AddSlash(mountPath) pathBlock := "" if vol.IsVMBlock() || (IsContentBlock(vol.contentType) && vol.volType == VolumeTypeCustom) { pathBlock, err = d.GetVolumeDiskPath(vol) if err != nil { return fmt.Errorf("Error getting VM block volume disk path: %w", err) } } // Snapshots are sent first by the sender, so create these first. for _, snapshot := range volTargetArgs.Snapshots { fullSnapshotName := GetSnapshotVolumeName(vol.name, snapshot.GetName()) snapVol := NewVolume(d, d.Name(), vol.volType, vol.contentType, fullSnapshotName, vol.config, vol.poolConfig) if snapVol.contentType != ContentTypeBlock || snapVol.volType != VolumeTypeCustom { // Receive the filesystem snapshot first (as it is sent first). err = recvFSVol(snapVol.name, conn, path) if err != nil { return err } } // Receive the block snapshot next (if needed). if vol.IsVMBlock() || (vol.contentType == ContentTypeBlock && vol.volType == VolumeTypeCustom) { err = recvBlockVol(snapVol.name, conn, pathBlock) if err != nil { return err } volSize, err := units.ParseByteSizeString(migration.GetSnapshotConfigValue(snapshot, "size")) if err != nil { return err } // During migration (e.g., LVM → dir), the block file may be smaller because // recvBlockVol uses SparseFileWrapper, which omits trailing zero bytes and does not truncate. // enlargeVolumeBlockFile ensures the block file matches the source volume size by applying truncation. if volSize > 0 { err = enlargeVolumeBlockFile(pathBlock, volSize) if err != nil { return err } } } // Create the snapshot itself. d.Logger().Debug("Creating snapshot", logger.Ctx{"volName": snapVol.Name()}) err = d.CreateVolumeSnapshot(snapVol, op) if err != nil { return err } // Setup the revert. reverter.Add(func() { _ = d.DeleteVolumeSnapshot(snapVol, op) }) } // Run volume-specific init logic. if initVolume != nil { _, err := initVolume(vol) if err != nil { return err } } if !IsContentBlock(vol.contentType) || vol.volType != VolumeTypeCustom { // Receive main volume. err = recvFSVol(vol.name, conn, path) if err != nil { return err } } // Receive the final main volume sync if needed. if volTargetArgs.Live && (!IsContentBlock(vol.contentType) || (vol.volType != VolumeTypeCustom && vol.volType != VolumeTypeVM)) { d.Logger().Debug("Starting main volume final sync", logger.Ctx{"volName": vol.name, "path": path}) err = recvFSVol(vol.name, conn, path) if err != nil { return err } } // Run EnsureMountPath after mounting and syncing to ensure the mounted directory has the // correct permissions set. err = vol.EnsureMountPath(false) if err != nil { return err } // Receive the block volume next (if needed). if vol.IsVMBlock() || (IsContentBlock(vol.contentType) && vol.volType == VolumeTypeCustom) { err = recvBlockVol(vol.name, conn, pathBlock) if err != nil { return err } // During migration (e.g., LVM → dir), the block file may be smaller because // recvBlockVol uses SparseFileWrapper, which omits trailing zero bytes and does not truncate. // enlargeVolumeBlockFile ensures the block file matches the source volume size by applying truncation. if volTargetArgs.VolumeSize > 0 { err = enlargeVolumeBlockFile(pathBlock, volTargetArgs.VolumeSize) if err != nil { return err } } } return nil }, op) if err != nil { return err } reverter.Success() return nil } // genericVFSHasVolume is a generic HasVolume implementation for VFS-only drivers. func genericVFSHasVolume(vol Volume) (bool, error) { _, err := os.Lstat(vol.MountPath()) if err != nil { if errors.Is(err, fs.ErrNotExist) { return false, nil } return false, err } return true, nil } // genericVFSGetVolumeDiskPath is a generic GetVolumeDiskPath implementation for VFS-only drivers. func genericVFSGetVolumeDiskPath(vol Volume) (string, error) { if !IsContentBlock(vol.contentType) { return "", ErrNotSupported } return filepath.Join(vol.MountPath(), genericVolumeDiskFile), nil } // genericVFSBackupVolume is a generic BackupVolume implementation for VFS-only drivers. func genericVFSBackupVolume(d Driver, vol Volume, writer instancewriter.InstanceWriter, basePrefix string, snapshots []string, op *operations.Operation) error { if len(snapshots) > 0 { // Check requested snapshot match those in storage. err := vol.SnapshotsMatch(snapshots, op) if err != nil { return err } } getDiskPath := func(v Volume) (string, error) { if v.contentType != ContentTypeBlock && v.contentType != ContentTypeISO { return "", nil } blockPath, err := d.GetVolumeDiskPath(v) if err != nil { errMsg := "Error getting VM block volume disk path" if v.Type() == VolumeTypeCustom { errMsg = "Error getting custom block volume disk path" } return "", fmt.Errorf(errMsg+": %w", err) } return blockPath, nil } // Handle snapshots. if len(snapshots) > 0 { for _, snapName := range snapshots { prefix := filepath.Join(basePrefix, BackupSnapshotPrefix(vol), snapName) snapVol, err := vol.NewSnapshot(snapName) if err != nil { return err } err = snapVol.MountTask(func(mountPath string, op *operations.Operation) error { diskPath, err := getDiskPath(snapVol) if err != nil { return err } err = BackupVolume(d, snapVol, writer, mountPath, diskPath, prefix) if err != nil { return err } return nil }, op) if err != nil { return err } } } // Copy the main volume itself. err := vol.MountTask(func(mountPath string, op *operations.Operation) error { diskPath, err := getDiskPath(vol) if err != nil { return err } err = BackupVolume(d, vol, writer, mountPath, diskPath, filepath.Join(basePrefix, BackupPrefix(vol))) if err != nil { return err } return nil }, op) if err != nil { return err } return nil } // genericVFSBackupUnpack unpacks a non-optimized backup tarball through a storage driver. // Returns a post hook function that should be called once the database entries for the restored backup have been // created and a revert function that can be used to undo the actions this function performs should something // subsequently fail. For VolumeTypeCustom volumes, a nil post hook is returned as it is expected that the DB // record be created before the volume is unpacked due to differences in the archive format that allows this. func genericVFSBackupUnpack(d Driver, sysOS *sys.OS, vol Volume, snapshots []string, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) (VolumePostHook, revert.Hook, error) { reverter := revert.New() defer reverter.Fail() // Find the compression algorithm used for backup source data. _, err := srcData.Seek(0, io.SeekStart) if err != nil { return nil, nil, err } tarArgs, _, unpacker, err := archive.DetectCompressionFile(srcData) if err != nil { return nil, nil, err } volExists, err := d.HasVolume(vol) if err != nil { return nil, nil, err } if volExists { return nil, nil, errors.New("Cannot restore volume, already exists on target") } // Create new empty volume. err = d.CreateVolume(vol, nil, nil) if err != nil { return nil, nil, err } reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) getDiskPath := func(v Volume) (string, error) { if v.contentType != ContentTypeBlock && v.contentType != ContentTypeISO { return "", nil } blockPath, err := d.GetVolumeDiskPath(v) if err != nil { errMsg := "Error getting VM block volume disk path" if v.Type() == VolumeTypeCustom { errMsg = "Error getting custom block volume disk path" } return "", fmt.Errorf(errMsg+": %w", err) } return blockPath, nil } if len(snapshots) > 0 { // Create new snapshots directory. err := CreateParentSnapshotDirIfMissing(d.Name(), vol.volType, vol.name) if err != nil { return nil, nil, err } } for _, snapName := range snapshots { err = vol.MountTask(func(mountPath string, op *operations.Operation) error { backupSnapshotPrefix := filepath.Join(basePrefix, BackupSnapshotPrefix(vol), snapName) diskPath, err := getDiskPath(vol) if err != nil { return err } return UnpackVolume(d, vol, srcData, tarArgs, unpacker, backupSnapshotPrefix, mountPath, diskPath) }, op) if err != nil { return nil, nil, err } snapVol, err := vol.NewSnapshot(snapName) if err != nil { return nil, nil, err } d.Logger().Debug("Creating volume snapshot", logger.Ctx{"snapshotName": snapVol.Name()}) err = d.CreateVolumeSnapshot(snapVol, op) if err != nil { return nil, nil, err } if vol.IsVMBlock() && vol.Config()["block.type"] == BlockVolumeTypeQcow2 { fsParentVol := vol.NewVMBlockFilesystemVolume() fsVol := snapVol.NewVMBlockFilesystemVolume() err := Qcow2CreateConfigSnapshot(fsParentVol, fsVol, op) if err != nil { return nil, nil, err } } reverter.Add(func() { _ = d.DeleteVolumeSnapshot(snapVol, op) }) } err = d.MountVolume(vol, op) if err != nil { return nil, nil, err } reverter.Add(func() { _, _ = d.UnmountVolume(vol, false, op) }) mountPath := vol.MountPath() diskPath, err := getDiskPath(vol) if err != nil { return nil, nil, err } err = UnpackVolume(d, vol, srcData, tarArgs, unpacker, filepath.Join(basePrefix, BackupPrefix(vol)), mountPath, diskPath) if err != nil { return nil, nil, err } // Run EnsureMountPath after mounting and unpacking to ensure the mounted directory has the // correct permissions set. err = vol.EnsureMountPath(false) if err != nil { return nil, nil, err } cleanup := reverter.Clone().Fail // Clone before calling reverter.Success() so we can return the Fail func. reverter.Success() var postHook VolumePostHook if vol.volType != VolumeTypeCustom { // Leave volume mounted (as is needed during backup.yaml generation during latter parts of the // backup restoration process). Create a post hook function that will be called at the end of the // backup restore process to unmount the volume if needed. postHook = func(vol Volume) error { _, err = d.UnmountVolume(vol, false, op) if err != nil { return err } return nil } } else { // For custom volumes unmount now, there is no post hook as there is no backup.yaml to generate. _, err = d.UnmountVolume(vol, false, op) if err != nil { return nil, nil, err } } return postHook, cleanup, nil } // genericVFSCopyVolume copies a volume and its snapshots using a non-optimized method. // initVolume is run against the main volume (not the snapshots) and is often used for quota initialization. func genericVFSCopyVolume(d Driver, initVolume func(vol Volume) (revert.Hook, error), vol Volume, srcVol Volume, srcSnapshots []Volume, refresh bool, allowInconsistent bool, op *operations.Operation) error { if vol.contentType != srcVol.contentType { return errors.New("Content type of source and target must be the same") } bwlimit := d.Config()["rsync.bwlimit"] var rsyncArgs []string if srcVol.IsVMBlock() { rsyncArgs = append(rsyncArgs, "--exclude", genericVolumeDiskFile) } reverter := revert.New() defer reverter.Fail() // Create the main volume if not refreshing. if !refresh { err := d.CreateVolume(vol, nil, op) if err != nil { return err } reverter.Add(func() { _ = d.DeleteVolume(vol, op) }) } // Define function to send a filesystem volume. sendFSVol := func(srcPath string, targetPath string) error { d.Logger().Debug("Copying filesystem volume", logger.Ctx{"sourcePath": srcPath, "targetPath": targetPath, "bwlimit": bwlimit, "rsyncArgs": rsyncArgs}) _, err := rsync.LocalCopy(srcPath, targetPath, bwlimit, true, rsyncArgs...) status, _ := linux.ExitStatus(err) if allowInconsistent && status == 24 { return nil } return err } // Define function to send a block volume. sendBlockVol := func(srcVol Volume, targetVol Volume) error { srcDevPath, err := d.GetVolumeDiskPath(srcVol) if err != nil { return err } targetDevPath, err := d.GetVolumeDiskPath(targetVol) if err != nil { return err } d.Logger().Debug("Copying block volume", logger.Ctx{"srcDevPath": srcDevPath, "targetPath": targetDevPath}) err = copyDevice(srcDevPath, targetDevPath) if err != nil { return err } return nil } // Ensure the volume is mounted. err := vol.MountTask(func(targetMountPath string, op *operations.Operation) error { // If copying snapshots is indicated, check the source isn't itself a snapshot. if len(srcSnapshots) > 0 && !srcVol.IsSnapshot() { for _, srcVol := range srcSnapshots { _, snapName, _ := api.GetParentAndSnapshotName(srcVol.name) // Mount the source snapshot and copy it to the target main volume. // A snapshot will then be taken next so it is stored in the correct volume and // subsequent filesystem rsync transfers benefit from only transferring the files // that changed between snapshots. err := srcVol.MountTask(func(srcMountPath string, op *operations.Operation) error { if srcVol.contentType != ContentTypeBlock || srcVol.volType != VolumeTypeCustom { err := sendFSVol(srcMountPath, targetMountPath) if err != nil { return err } } if srcVol.IsVMBlock() || srcVol.contentType == ContentTypeBlock && srcVol.volType == VolumeTypeCustom { err := sendBlockVol(srcVol, vol) if err != nil { return err } } return nil }, op) if err != nil { return err } fullSnapName := GetSnapshotVolumeName(vol.name, snapName) snapVol := NewVolume(d, d.Name(), vol.volType, vol.contentType, fullSnapName, vol.config, vol.poolConfig) // Create the snapshot itself. d.Logger().Debug("Creating snapshot", logger.Ctx{"volName": snapVol.Name()}) err = d.CreateVolumeSnapshot(snapVol, op) if err != nil { return err } // Setup the revert. reverter.Add(func() { _ = d.DeleteVolumeSnapshot(snapVol, op) }) } } // Run volume-specific init logic. if initVolume != nil { _, err := initVolume(vol) if err != nil { return err } } // Copy source to destination (mounting each volume if needed). err := srcVol.MountTask(func(srcMountPath string, op *operations.Operation) error { if srcVol.contentType != ContentTypeBlock || srcVol.volType != VolumeTypeCustom { err := sendFSVol(srcMountPath, targetMountPath) if err != nil { return err } } if srcVol.IsVMBlock() || srcVol.contentType == ContentTypeBlock && srcVol.volType == VolumeTypeCustom { err := sendBlockVol(srcVol, vol) if err != nil { return err } } return nil }, op) if err != nil { return err } // Run EnsureMountPath after mounting and copying to ensure the mounted directory has the // correct permissions set. err = vol.EnsureMountPath(false) if err != nil { return err } return nil }, op) if err != nil { return err } reverter.Success() return nil } // genericVFSListVolumes returns a list of volumes in storage pool. func genericVFSListVolumes(d Driver) ([]Volume, error) { var vols []Volume poolName := d.Name() poolConfig := d.Config() poolMountPath := GetPoolMountPath(poolName) for _, volType := range d.Info().VolumeTypes { if len(BaseDirectories[volType].Paths) < 1 { return nil, fmt.Errorf("Cannot get base directory name for volume type %q", volType) } volTypePath := filepath.Join(poolMountPath, BaseDirectories[volType].Paths[0]) ents, err := os.ReadDir(volTypePath) if err != nil { return nil, fmt.Errorf("Failed to list directory %q for volume type %q: %w", volTypePath, volType, err) } for _, ent := range ents { volName := ent.Name() contentType := ContentTypeFS if volType == VolumeTypeVM { contentType = ContentTypeBlock } else if volType == VolumeTypeCustom && util.PathExists(filepath.Join(volTypePath, volName, genericVolumeDiskFile)) { if strings.HasSuffix(ent.Name(), genericISOVolumeSuffix) { contentType = ContentTypeISO volName = strings.TrimSuffix(volName, genericISOVolumeSuffix) } else { contentType = ContentTypeBlock } } vols = append(vols, NewVolume(d, poolName, volType, contentType, volName, make(map[string]string), poolConfig)) } } return vols, nil } // genericRunFiller runs the supplied filler, and setting the returned volume size back into filler. func genericRunFiller(d Driver, vol Volume, devPath string, filler *VolumeFiller, allowUnsafeResize bool) error { if filler == nil || filler.Fill == nil { return nil } vol.driver.Logger().Debug("Running filler function", logger.Ctx{"dev": devPath, "path": vol.MountPath()}) volSize, err := filler.Fill(vol, devPath, allowUnsafeResize, !d.Info().ZeroUnpack, d.Info().TargetFormat) if err != nil { return err } filler.Size = volSize return nil } incus-7.0.0/internal/server/storage/drivers/init.go000066400000000000000000000005461517523235500224120ustar00rootroot00000000000000package drivers import ( "sync" ) func init() { zfsCache = map[string]map[string]zfsCacheEntry{} zfsCachePrefillQueue = []string{} truenasCache = map[string]map[string]map[string]truenasCacheEntry{} truenasCachePrefillQueue = map[string][]string{} truenasCachePrefillRunning = map[string]bool{} truenasCachePrefillMu = map[string]*sync.RWMutex{} } incus-7.0.0/internal/server/storage/drivers/interface.go000066400000000000000000000126401517523235500234050ustar00rootroot00000000000000package drivers import ( "io" "net/url" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" ) // driver is the extended internal interface. type driver interface { Driver init(state *state.State, name string, config map[string]string, logger logger.Logger, volIDFunc func(volType VolumeType, volName string) (int64, error), commonRules *Validators) load() error isRemote() bool } // Driver represents a low-level storage driver. type Driver interface { // Internal. Info() Info HasVolume(vol Volume) (bool, error) roundVolumeBlockSizeBytes(vol Volume, sizeBytes int64) (int64, error) isBlockBacked(vol Volume) bool // Export struct details. Name() string Config() map[string]string Logger() logger.Logger // Pool. FillConfig() error Create() error Delete(op *operations.Operation) error // Mount mounts a storage pool if needed, returns true if we caused a new mount, false if already mounted. Mount() (bool, error) // Unmount unmounts a storage pool if needed, returns true if unmounted, false if was not mounted. Unmount() (bool, error) GetResources() (*api.ResourcesStoragePool, error) Validate(config map[string]string) error Update(changedConfig map[string]string) error ApplyPatch(name string) error // Buckets. ValidateBucket(bucket Volume) error GetBucketURL(bucketName string) *url.URL CreateBucket(bucket Volume, op *operations.Operation) error DeleteBucket(bucket Volume, op *operations.Operation) error UpdateBucket(bucket Volume, changedConfig map[string]string) error ValidateBucketKey(keyName string, creds S3Credentials, roleName string) error CreateBucketKey(bucket Volume, keyName string, creds S3Credentials, roleName string, op *operations.Operation) (*S3Credentials, error) UpdateBucketKey(bucket Volume, keyName string, creds S3Credentials, roleName string, op *operations.Operation) (*S3Credentials, error) DeleteBucketKey(bucket Volume, keyName string, op *operations.Operation) error // Volumes. FillVolumeConfig(vol Volume) error ValidateVolume(vol Volume, removeUnknownKeys bool) error CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Operation) error CreateVolumeFromCopy(vol Volume, srcVol Volume, copySnapshots bool, allowInconsistent bool, op *operations.Operation) error RefreshVolume(vol Volume, srcVol Volume, srcSnapshots []Volume, allowInconsistent bool, op *operations.Operation) error DeleteVolume(vol Volume, op *operations.Operation) error RenameVolume(vol Volume, newName string, op *operations.Operation) error UpdateVolume(vol Volume, changedConfig map[string]string) error GetVolumeUsage(vol Volume) (int64, error) SetVolumeQuota(vol Volume, size string, allowUnsafeResize bool, op *operations.Operation) error GetVolumeDiskPath(vol Volume) (string, error) ListVolumes() ([]Volume, error) // ActivateTask is a low-level access function to get to the underlying storage. ActivateTask(vol Volume, task func(devPath string, op *operations.Operation) error, op *operations.Operation) error // MountVolume mounts a storage volume (if not mounted) and increments reference counter. MountVolume(vol Volume, op *operations.Operation) error // MountVolumeSnapshot mounts a storage volume snapshot as readonly. MountVolumeSnapshot(snapVol Volume, op *operations.Operation) error // CanDelegateVolume checks whether the volume can be delegated. CanDelegateVolume(vol Volume) bool // DelegateVolume allows for the volume to be managed by the instance. DelegateVolume(vol Volume, pid int) error // UnmountVolume unmounts a storage volume, returns true if unmounted, false if was not // mounted. UnmountVolume(vol Volume, keepBlockDev bool, op *operations.Operation) (bool, error) // UnmountVolume unmounts a storage volume snapshot, returns true if unmounted, false if was // not mounted. UnmountVolumeSnapshot(snapVol Volume, op *operations.Operation) (bool, error) CanRestoreVolume(vol Volume, snapshotName string) error CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error GetQcow2BackingFilePath(vol Volume) (string, error) DeleteVolumeSnapshot(snapVol Volume, op *operations.Operation) error RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *operations.Operation) error VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) RestoreVolume(vol Volume, snapshotName string, op *operations.Operation) error Qcow2DeletionCleanup(vol Volume, childName string) error // Migration. MigrationTypes(contentType ContentType, refresh bool, copySnapshots bool, clusterMove bool, storageMove bool) []migration.Type MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs *migration.VolumeSourceArgs, op *operations.Operation) error CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, volTargetArgs migration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error // Backup. BackupVolume(vol Volume, writer instancewriter.InstanceWriter, basePrefix string, optimized bool, snapshots []string, op *operations.Operation) error CreateVolumeFromBackup(vol Volume, srcBackup backup.Info, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) (VolumePostHook, revert.Hook, error) } incus-7.0.0/internal/server/storage/drivers/load.go000066400000000000000000000051641517523235500223670ustar00rootroot00000000000000package drivers import ( "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/shared/logger" ) var drivers = map[string]func() driver{ "btrfs": func() driver { return &btrfs{} }, "ceph": func() driver { return &ceph{} }, "cephfs": func() driver { return &cephfs{} }, "cephobject": func() driver { return &cephobject{} }, "dir": func() driver { return &dir{} }, "lvm": func() driver { return &lvm{} }, "lvmcluster": func() driver { return &lvm{clustered: true} }, "truenas": func() driver { return &truenas{} }, "zfs": func() driver { return &zfs{} }, "linstor": func() driver { return &linstor{} }, } // Validators contains functions used for validating a drivers's config. type Validators struct { PoolRules func() map[string]func(string) error VolumeRules func(vol Volume) map[string]func(string) error } // Load returns a Driver for an existing low-level storage pool. func Load(state *state.State, driverName string, name string, config map[string]string, logger logger.Logger, volIDFunc func(volType VolumeType, volName string) (int64, error), commonRules *Validators) (Driver, error) { var driverFunc func() driver // Locate the driver loader. if state.OS.MockMode { driverFunc = func() driver { return &mock{} } } else { df, ok := drivers[driverName] if !ok { return nil, ErrUnknownDriver } driverFunc = df } d := driverFunc() d.init(state, name, config, logger, volIDFunc, commonRules) err := d.load() if err != nil { return nil, err } return d, nil } // SupportedDrivers returns a list of supported storage drivers by loading each storage driver and running its // compatibility inspection process. This can take a long time if a driver is not supported. func SupportedDrivers(s *state.State) []Info { supportedDrivers := make([]Info, 0, len(drivers)) for driverName := range drivers { driver, err := Load(s, driverName, "", nil, nil, nil, nil) if err != nil { continue } supportedDrivers = append(supportedDrivers, driver.Info()) } return supportedDrivers } // AllDriverNames returns a list of all storage driver names. func AllDriverNames() []string { driverNames := make([]string, 0, len(drivers)) for driverName := range drivers { driverNames = append(driverNames, driverName) } return driverNames } // RemoteDriverNames returns a list of remote storage driver names. func RemoteDriverNames() []string { driverNames := make([]string, 0, len(drivers)) for driverName, driverFunc := range drivers { if !driverFunc().isRemote() { continue } driverNames = append(driverNames, driverName) } return driverNames } incus-7.0.0/internal/server/storage/drivers/utils.go000066400000000000000000001112271517523235500226060ustar00rootroot00000000000000package drivers import ( "archive/tar" "context" "errors" "fmt" "io" "io/fs" "os" "os/exec" "path/filepath" "slices" "sort" "strconv" "strings" "time" "unsafe" "golang.org/x/sys/unix" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/operations" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) // MinBlockBoundary minimum block boundary size to use. const MinBlockBoundary = 8192 // MaxValue represents the maximum possible value. const MaxValue = "max" // blockBackedAllowedFilesystems allowed filesystems for block volumes. var blockBackedAllowedFilesystems = []string{"btrfs", "ext4", "xfs"} // wipeDirectory empties the contents of a directory, but leaves it in place. func wipeDirectory(path string) error { // List all entries. entries, err := os.ReadDir(path) if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil } return fmt.Errorf("Failed listing directory %q: %w", path, err) } // Individually wipe all entries. for _, entry := range entries { entryPath := filepath.Join(path, entry.Name()) err := os.RemoveAll(entryPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed removing %q: %w", entryPath, err) } } return nil } // forceRemoveAll wipes a path including any immutable/non-append files. func forceRemoveAll(path string) error { err := os.RemoveAll(path) if err != nil { _, _ = subprocess.RunCommand("chattr", "-ai", "-R", path) err = os.RemoveAll(path) if err != nil { return err } } return nil } // forceUnmount unmounts stacked mounts until no mountpoint remains. func forceUnmount(path string) (bool, error) { unmounted := false for { // Check if already unmounted. if !linux.IsMountPoint(path) { return unmounted, nil } // Try a clean unmount first. err := TryUnmount(path, 0) if err != nil { // Fallback to lazy unmounting. err = unix.Unmount(path, unix.MNT_DETACH) if err != nil { return false, fmt.Errorf("Failed to unmount '%s': %w", path, err) } } unmounted = true } } // mountReadOnly performs a read-only bind-mount. func mountReadOnly(srcPath string, dstPath string) (bool, error) { // Check if already mounted. if linux.IsMountPoint(dstPath) { return false, nil } // Create a mount entry. err := TryMount(srcPath, dstPath, "none", unix.MS_BIND, "") if err != nil { return false, err } // Make it read-only. err = TryMount("", dstPath, "none", unix.MS_BIND|unix.MS_RDONLY|unix.MS_REMOUNT, "") if err != nil { _, _ = forceUnmount(dstPath) return false, err } return true, nil } // sameMount checks if two paths are on the same mountpoint. func sameMount(srcPath string, dstPath string) bool { // Get the source vfs path information var srcFsStat unix.Statfs_t err := unix.Statfs(srcPath, &srcFsStat) if err != nil { return false } // Get the destination vfs path information var dstFsStat unix.Statfs_t err = unix.Statfs(dstPath, &dstFsStat) if err != nil { return false } // Compare statfs if srcFsStat.Type != dstFsStat.Type || srcFsStat.Fsid != dstFsStat.Fsid { return false } // Get the source path information var srcStat unix.Stat_t err = unix.Stat(srcPath, &srcStat) if err != nil { return false } // Get the destination path information var dstStat unix.Stat_t err = unix.Stat(dstPath, &dstStat) if err != nil { return false } // Compare inode if srcStat.Ino != dstStat.Ino { return false } return true } // TryMount tries mounting a filesystem multiple times. This is useful for unreliable backends. func TryMount(src string, dst string, fs string, flags uintptr, options string) error { var err error // Attempt 20 mounts over 10s for range 20 { err = unix.Mount(src, dst, fs, flags, options) if err == nil { break } time.Sleep(500 * time.Millisecond) } if err != nil { return fmt.Errorf("Failed to mount %q on %q using %q: %w", src, dst, fs, err) } return nil } // TryUnmount tries unmounting a filesystem multiple times. This is useful for unreliable backends. func TryUnmount(path string, flags int) error { var err error for i := range 20 { err = unix.Unmount(path, flags) if err == nil { break } logger.Debug("Failed to unmount", logger.Ctx{"path": path, "attempt": i, "err": err}) time.Sleep(500 * time.Millisecond) } if err != nil { return fmt.Errorf("Failed to unmount %q: %w", path, err) } return nil } // tryExists waits up to 10s for a file to exist. func tryExists(path string) bool { // Attempt 20 checks over 10s for range 20 { if util.PathExists(path) { return true } time.Sleep(500 * time.Millisecond) } return false } // fsUUID returns the filesystem UUID for the given block path. func fsUUID(path string) (string, error) { val, err := subprocess.RunCommand("blkid", "-s", "UUID", "-o", "value", path) if err != nil { return "", err } return strings.TrimSpace(val), nil } // fsProbe returns the filesystem type for the given block path. func fsProbe(path string) (string, error) { val, err := subprocess.RunCommand("blkid", "-s", "TYPE", "-o", "value", path) if err != nil { return "", err } return strings.TrimSpace(val), nil } // GetPoolMountPath returns the mountpoint of the given pool. // {INCUS_DIR}/storage-pools/. func GetPoolMountPath(poolName string) string { return internalUtil.VarPath("storage-pools", poolName) } // GetVolumeMountPath returns the mount path for a specific volume based on its pool and type and // whether it is a snapshot or not. For VolumeTypeImage the volName is the image fingerprint. func GetVolumeMountPath(poolName string, volType VolumeType, volName string) string { if internalInstance.IsSnapshot(volName) { return internalUtil.VarPath("storage-pools", poolName, fmt.Sprintf("%s-snapshots", string(volType)), volName) } return internalUtil.VarPath("storage-pools", poolName, string(volType), volName) } // GetVolumeSnapshotDir gets the snapshot mount directory for the parent volume. func GetVolumeSnapshotDir(poolName string, volType VolumeType, volName string) string { parent, _, _ := api.GetParentAndSnapshotName(volName) return internalUtil.VarPath("storage-pools", poolName, fmt.Sprintf("%s-snapshots", string(volType)), parent) } // GetSnapshotVolumeName returns the full volume name for a parent volume and snapshot name. func GetSnapshotVolumeName(parentName, snapshotName string) string { return fmt.Sprintf("%s%s%s", parentName, internalInstance.SnapshotDelimiter, snapshotName) } // CreateParentSnapshotDirIfMissing creates the parent directory for volume snapshots. func CreateParentSnapshotDirIfMissing(poolName string, volType VolumeType, volName string) error { snapshotsPath := GetVolumeSnapshotDir(poolName, volType, volName) // If it's missing, create it. if !util.PathExists(snapshotsPath) { err := os.Mkdir(snapshotsPath, 0o700) if err != nil { return fmt.Errorf("Failed to create parent snapshot directory %q: %w", snapshotsPath, err) } return nil } return nil } // deleteParentSnapshotDirIfEmpty removes the parent snapshot directory if it is empty. // It accepts the pool name, volume type and parent volume name. func deleteParentSnapshotDirIfEmpty(poolName string, volType VolumeType, volName string) error { snapshotsPath := GetVolumeSnapshotDir(poolName, volType, volName) // If it exists, try to delete it. if util.PathExists(snapshotsPath) { isEmpty, err := internalUtil.PathIsEmpty(snapshotsPath) if err != nil { return err } if isEmpty { err := os.Remove(snapshotsPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", snapshotsPath, err) } } } return nil } // ensureSparseFile creates a sparse empty file at specified location with specified size. // If the path already exists, the file is truncated to the requested size. func ensureSparseFile(filePath string, sizeBytes int64) error { f, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE, 0o600) if err != nil { return fmt.Errorf("Failed to open %s: %w", filePath, err) } defer func() { _ = f.Close() }() err = f.Truncate(sizeBytes) if err != nil { return fmt.Errorf("Failed to create sparse file %s: %w", filePath, err) } return f.Close() } // ensureVolumeBlockFile creates new block file or enlarges the raw block file for a volume to the specified size. // Returns true if resize took place, false if not. Requested size is rounded to nearest block size using // roundVolumeBlockSizeBytes() before decision whether to resize is taken. Accepts unsupportedResizeTypes // list that indicates which volume types it should not attempt to resize (when allowUnsafeResize=false) and // instead return ErrNotSupported. func ensureVolumeBlockFile(vol Volume, path string, sizeBytes int64, allowUnsafeResize bool, unsupportedResizeTypes ...VolumeType) (bool, error) { if sizeBytes <= 0 { return false, errors.New("Size cannot be zero") } // Get rounded block size to avoid QEMU boundary issues. var err error sizeBytes, err = vol.driver.roundVolumeBlockSizeBytes(vol, sizeBytes) if err != nil { return false, err } if util.PathExists(path) { fi, err := os.Stat(path) if err != nil { return false, err } oldSizeBytes := fi.Size() if sizeBytes == oldSizeBytes { return false, nil } // Only perform pre-resize checks if we are not in "unsafe" mode. // In unsafe mode we expect the caller to know what they are doing and understand the risks. if !allowUnsafeResize { // Reject if would try and resize a volume type that is not supported. // This needs to come before the ErrCannotBeShrunk check below so that any resize attempt // is blocked with ErrNotSupported error. if slices.Contains(unsupportedResizeTypes, vol.volType) { return false, ErrNotSupported } if sizeBytes < oldSizeBytes { return false, fmt.Errorf("Block volumes cannot be shrunk: %w", ErrCannotBeShrunk) } if vol.MountInUse() { return false, ErrInUse // We don't allow online resizing of block volumes. } } err = ensureSparseFile(path, sizeBytes) if err != nil { return false, fmt.Errorf("Failed resizing disk image %q to size %d: %w", path, sizeBytes, err) } return true, nil } // If path doesn't exist, then there has been no filler function supplied to create it from another source. // So instead create an empty volume (use for PXE booting a VM). err = ensureSparseFile(path, sizeBytes) if err != nil { return false, fmt.Errorf("Failed creating disk image %q as size %d: %w", path, sizeBytes, err) } return false, nil } // enlargeVolumeBlockFile enlarges the raw block file for a volume to the specified size. func enlargeVolumeBlockFile(path string, volSize int64) error { if linux.IsBlockdevPath(path) { return nil } actualSize, err := BlockDiskSizeBytes(path) if err != nil { return err } if volSize < actualSize { return fmt.Errorf("Block volumes cannot be shrunk: %w", ErrCannotBeShrunk) } err = ensureSparseFile(path, volSize) if err != nil { return err } return nil } // mkfsOptions represents options for filesystem creation. type mkfsOptions struct { Label string } // makeFSType creates the provided filesystem. func makeFSType(path string, fsType string, options *mkfsOptions) (string, error) { var err error var msg string fsOptions := options if fsOptions == nil { fsOptions = &mkfsOptions{} } cmd := []string{fmt.Sprintf("mkfs.%s", fsType)} if fsOptions.Label != "" { cmd = append(cmd, "-L", fsOptions.Label) } if fsType == "ext4" { cmd = append(cmd, "-E", "nodiscard,lazy_itable_init=0,lazy_journal_init=0") } // Always add the path to the device as the last argument for wider compatibility with versions of mkfs. cmd = append(cmd, path) msg, err = subprocess.TryRunCommand(cmd[0], cmd[1:]...) if err != nil { return msg, err } return "", nil } // filesystemTypeCanBeShrunk indicates if filesystems of fsType can be shrunk. func filesystemTypeCanBeShrunk(fsType string) bool { if fsType == "" { fsType = DefaultFilesystem } if slices.Contains([]string{"ext4", "btrfs"}, fsType) { return true } return false } // shrinkFileSystem shrinks a filesystem if it is supported. // EXT4 volumes will be unmounted temporarily if needed. // BTRFS volumes will be mounted temporarily if needed. // Accepts a force argument that indicates whether to skip some safety checks when resizing the volume. // This should only be used if the volume will be deleted on resize error. func shrinkFileSystem(fsType string, devPath string, vol Volume, byteSize int64, force bool) error { if fsType == "" { fsType = DefaultFilesystem } if !filesystemTypeCanBeShrunk(fsType) { return ErrCannotBeShrunk } // The smallest unit that resize2fs accepts in byte size (rather than blocks) is kilobytes. strSize := fmt.Sprintf("%dK", byteSize/1024) switch fsType { case "ext4": return vol.UnmountTask(func(op *operations.Operation) error { output, err := subprocess.RunCommand("e2fsck", "-f", "-y", devPath) if err != nil { exitCodeFSModified := false var exitError *exec.ExitError ok := errors.As(err, &exitError) if ok { if exitError.ExitCode() == 1 { exitCodeFSModified = true } } // e2fsck can return non-zero exit code if it has modified the filesystem, but // this isn't an error and we can proceed. if !exitCodeFSModified { // e2fsck provides some context to errors on stdout. return fmt.Errorf("%s: %w", strings.TrimSpace(output), err) } } var args []string if force { // Enable force mode if requested. Should only be done if volume will be deleted // on error as this can result in corrupting the filesystem if fails during resize. // This is useful because sometimes the pre-checks performed by resize2fs are not // accurate and would prevent a successful filesystem shrink. args = append(args, "-f") } args = append(args, devPath, strSize) _, err = subprocess.RunCommand("resize2fs", args...) if err != nil { return err } return nil }, true, nil) case "btrfs": return vol.MountTask(func(mountPath string, op *operations.Operation) error { _, err := subprocess.RunCommand("btrfs", "filesystem", "resize", strSize, mountPath) if err != nil { return err } return nil }, nil) } return fmt.Errorf("Unrecognised filesystem type %q", fsType) } // growFileSystem grows a filesystem if it is supported. The volume will be mounted temporarily if needed. func growFileSystem(fsType string, devPath string, vol Volume) error { if fsType == "" { fsType = DefaultFilesystem } return vol.MountTask(func(mountPath string, op *operations.Operation) error { var err error switch fsType { case "ext4": _, err = subprocess.TryRunCommand("resize2fs", devPath) case "xfs": _, err = subprocess.TryRunCommand("xfs_growfs", mountPath) case "btrfs": _, err = subprocess.TryRunCommand("btrfs", "filesystem", "resize", "max", mountPath) default: return fmt.Errorf("Unrecognised filesystem type %q", fsType) } if err != nil { return fmt.Errorf("Could not grow underlying %q filesystem for %q: %w", fsType, devPath, err) } return nil }, nil) } // renegerateFilesystemUUIDNeeded returns true if fsType requires UUID regeneration, false if not. func renegerateFilesystemUUIDNeeded(fsType string) bool { switch fsType { case "btrfs": return true case "xfs": return true } return false } // regenerateFilesystemUUID changes the filesystem UUID to a new randomly generated one if the fsType requires it. // Otherwise this function does nothing. func regenerateFilesystemUUID(fsType string, devPath string) error { switch fsType { case "btrfs": return regenerateFilesystemBTRFSUUID(devPath) case "xfs": return regenerateFilesystemXFSUUID(devPath) } return errors.New("Filesystem not supported") } // regenerateFilesystemBTRFSUUID changes the BTRFS filesystem UUID to a new randomly generated one. func regenerateFilesystemBTRFSUUID(devPath string) error { // If the snapshot was taken whilst instance was running there may be outstanding transactions that will // cause btrfstune to corrupt superblock, so ensure these are cleared out first. _, err := subprocess.RunCommand("btrfs", "rescue", "zero-log", devPath) if err != nil { return err } _, err = subprocess.RunCommand("btrfstune", "-f", "-u", devPath) if err != nil { return err } return nil } // regenerateFilesystemXFSUUID changes the XFS filesystem UUID to a new randomly generated one. func regenerateFilesystemXFSUUID(devPath string) error { // Attempt to generate a new UUID. msg, err := subprocess.RunCommand("xfs_admin", "-U", "generate", devPath) if err != nil { return err } if msg != "" { // Exit 0 with a msg usually means some log entry getting in the way. _, err = subprocess.RunCommand("xfs_repair", "-o", "force_geometry", "-L", devPath) if err != nil { return err } // Attempt to generate a new UUID again. _, err = subprocess.RunCommand("xfs_admin", "-U", "generate", devPath) if err != nil { return err } } return nil } // copyDevice copies one device path to another using dd running at low priority. // It expects outputPath to exist already, so will not create it. func copyDevice(inputPath string, outputPath string) error { cmd := []string{ "nice", "-n19", // Run dd with low priority to reduce CPU impact on other processes. "dd", fmt.Sprintf("if=%s", inputPath), fmt.Sprintf("of=%s", outputPath), "bs=16M", // Use large buffer to reduce syscalls and speed up copy. "conv=nocreat,sparse", // Don't create output file if missing (expect caller to have created output file), also attempt to make a sparse file. } // Check for Direct I/O support. from, err := os.OpenFile(inputPath, unix.O_DIRECT|unix.O_RDONLY, 0) if err == nil { cmd = append(cmd, "iflag=direct") _ = from.Close() } to, err := os.OpenFile(outputPath, unix.O_DIRECT|unix.O_RDONLY, 0) if err == nil { cmd = append(cmd, "oflag=direct") _ = to.Close() } _, err = subprocess.RunCommand(cmd[0], cmd[1:]...) if err != nil { return err } return nil } // loopFilePath returns the loop file path for a storage pool. func loopFilePath(poolName string) string { return filepath.Join(internalUtil.VarPath("disks"), fmt.Sprintf("%s.img", poolName)) } // ShiftBtrfsRootfs shifts the BTRFS root filesystem. func ShiftBtrfsRootfs(path string, diskIdmap *idmap.Set) error { return shiftBtrfsRootfs(path, diskIdmap, true) } // UnshiftBtrfsRootfs unshifts the BTRFS root filesystem. func UnshiftBtrfsRootfs(path string, diskIdmap *idmap.Set) error { return shiftBtrfsRootfs(path, diskIdmap, false) } // shiftBtrfsRootfs shifts a filesystem that main include read-only subvolumes. func shiftBtrfsRootfs(path string, diskIdmap *idmap.Set, shift bool) error { var err error roSubvols := []string{} subvols, _ := BTRFSSubVolumesGet(path) sort.Strings(subvols) for _, subvol := range subvols { subvol = filepath.Join(path, subvol) if !BTRFSSubVolumeIsRo(subvol) { continue } roSubvols = append(roSubvols, subvol) _ = BTRFSSubVolumeMakeRw(subvol) } if shift { err = diskIdmap.ShiftPath(path, nil) } else { err = diskIdmap.UnshiftPath(path, nil) } for _, subvol := range roSubvols { _ = BTRFSSubVolumeMakeRo(subvol) } return err } // BTRFSSubVolumesGet gets subvolumes. func BTRFSSubVolumesGet(path string) ([]string, error) { result := []string{} if !strings.HasSuffix(path, "/") { path = path + "/" } // Unprivileged users can't get to fs internals. _ = filepath.WalkDir(path, func(fpath string, entry fs.DirEntry, err error) error { // Skip walk errors if err != nil { return nil } // Ignore the base path. if strings.TrimRight(fpath, "/") == strings.TrimRight(path, "/") { return nil } // Subvolumes can only be directories. if !entry.IsDir() { return nil } // Check if a btrfs subvolume. if btrfsIsSubVolume(fpath) { result = append(result, strings.TrimPrefix(fpath, path)) } return nil }) return result, nil } // Deprecated: Use IsSubvolume from the Btrfs driver instead. // btrfsIsSubvolume checks if a given path is a subvolume. func btrfsIsSubVolume(subvolPath string) bool { fs := unix.Stat_t{} err := unix.Lstat(subvolPath, &fs) if err != nil { return false } // Check if BTRFS_FIRST_FREE_OBJECTID if fs.Ino != 256 { return false } return true } // BTRFSSubVolumeIsRo returns if subvolume is read only. func BTRFSSubVolumeIsRo(path string) bool { output, err := subprocess.RunCommand("btrfs", "property", "get", "-ts", path) if err != nil { return false } return strings.HasPrefix(string(output), "ro=true") } // BTRFSSubVolumeMakeRo makes a subvolume read only. Deprecated use btrfs.setSubvolumeReadonlyProperty(). func BTRFSSubVolumeMakeRo(path string) error { _, err := subprocess.RunCommand("btrfs", "property", "set", "-ts", path, "ro", "true") return err } // BTRFSSubVolumeMakeRw makes a sub volume read/write. Deprecated use btrfs.setSubvolumeReadonlyProperty(). func BTRFSSubVolumeMakeRw(path string) error { _, err := subprocess.RunCommand("btrfs", "property", "set", "-ts", path, "ro", "false") return err } // ShiftZFSSkipper indicates which files not to shift for ZFS. func ShiftZFSSkipper(dir string, absPath string, fi os.FileInfo, newuid int64, newgid int64) error { strippedPath := absPath if dir != "" { strippedPath = absPath[len(dir):] } if fi.IsDir() && strippedPath == "/.zfs/snapshot" { return filepath.SkipDir } return nil } // BlockDiskSizeBytes returns the size of a block disk (path can be either block device or raw file). func BlockDiskSizeBytes(blockDiskPath string) (int64, error) { if linux.IsBlockdevPath(blockDiskPath) { // Attempt to open the device path. f, err := os.Open(blockDiskPath) if err != nil { return -1, err } defer func() { _ = f.Close() }() fd := int(f.Fd()) // Retrieve the block device size. res, err := unix.IoctlGetInt(fd, unix.BLKGETSIZE64) if err != nil { return -1, err } return int64(res), nil } // Block device is assumed to be a raw file. fi, err := os.Lstat(blockDiskPath) if err != nil { return -1, err } return fi.Size(), nil } // GetPhysicalBlockSize returns the physical block size for the device. func GetPhysicalBlockSize(blockDiskPath string) (int, error) { // Open the block device. f, err := os.Open(blockDiskPath) if err != nil { return -1, err } defer func() { _ = f.Close() }() // Query the physical block size. var res int32 _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(f.Fd()), unix.BLKPBSZGET, uintptr(unsafe.Pointer(&res))) if errno != 0 { return -1, fmt.Errorf("Failed to BLKPBSZGET: %w", unix.Errno(errno)) } return int(res), nil } // OperationLockName returns the storage specific lock name to use with locking package. func OperationLockName(operationName string, poolName string, volType VolumeType, contentType ContentType, volName string) string { return fmt.Sprintf("%s/%s/%s/%s/%s", operationName, poolName, volType, contentType, volName) } // loopFileSizeDefault returns the size in GiB to use as the default size for a pool loop file. // This is based on the size of the filesystem of daemon's VarPath(). func loopFileSizeDefault() (uint64, error) { st := unix.Statfs_t{} err := unix.Statfs(internalUtil.VarPath(), &st) if err != nil { return 0, fmt.Errorf("Failed getting free space of %q: %w", internalUtil.VarPath(), err) } gibAvailable := uint64(st.Frsize) * st.Bavail / (1024 * 1024 * 1024) if gibAvailable > 30 { return 30, nil // Default to no more than 30GiB. } else if gibAvailable > 5 { return gibAvailable / 5, nil // Use 20% of free space otherwise. } else if gibAvailable == 5 { return gibAvailable, nil // Need at least 5GiB free. } return 0, errors.New("Insufficient free space to create default sized 5GiB pool") } // loopFileSetup sets up a loop device for the provided sourcePath. // It tries to enable direct I/O if supported. func loopDeviceSetup(sourcePath string) (string, error) { out, err := subprocess.RunCommand("losetup", "--find", "--nooverlap", "--direct-io=on", "--show", sourcePath) if err != nil { if strings.Contains(err.Error(), "direct io") || strings.Contains(err.Error(), "Invalid argument") { out, err = subprocess.RunCommand("losetup", "--find", "--nooverlap", "--show", sourcePath) if err != nil { return "", err } } else { return "", err } } return strings.TrimSpace(out), nil } // loopDeviceSetupAlign creates a forced 512-byte aligned loop device. func loopDeviceSetupAlign(sourcePath string) (string, error) { out, err := subprocess.RunCommand("losetup", "-b", "512", "--find", "--nooverlap", "--show", sourcePath) if err != nil { return "", err } return strings.TrimSpace(out), nil } // loopFileAutoDetach enables auto detach mode for a loop device. func loopDeviceAutoDetach(loopDevPath string) error { _, err := subprocess.RunCommand("losetup", "--detach", loopDevPath) return err } // loopDeviceSetCapacity forces the loop driver to reread the size of the file associated with the specified loop device. func loopDeviceSetCapacity(loopDevPath string) error { _, err := subprocess.RunCommand("losetup", "--set-capacity", loopDevPath) return err } // wipeBlockHeaders will wipe the first 4MB of a block device. func wipeBlockHeaders(path string) error { // Open /dev/zero. fdZero, err := os.Open("/dev/zero") if err != nil { return err } defer fdZero.Close() // Open the target disk. fdDisk, err := os.OpenFile(path, os.O_RDWR, 0o600) if err != nil { return err } defer fdDisk.Close() // Wipe the 4MiB header. _, err = io.CopyN(fdDisk, fdZero, 1024*1024*4) if err != nil { return err } return nil } // IsContentBlock returns true if the content type is either block or iso. func IsContentBlock(contentType ContentType) bool { return contentType == ContentTypeBlock || contentType == ContentTypeISO } // NewSparseFileWrapper returns a SparseFileWrapper for the provided io.File. func NewSparseFileWrapper(w *os.File) *SparseFileWrapper { return &SparseFileWrapper{w: w} } // SparseFileWrapper wraps os.File to create sparse Files. type SparseFileWrapper struct { w *os.File } // Write performs the write but skips null bytes. func (sfw *SparseFileWrapper) Write(p []byte) (n int, err error) { // We only support comparing up to 4MB at a time. if len(p) > 4*1024*1024 { return sfw.w.Write(p) } // Check if all zeroes. isZero := true for _, v := range p { if v != 0 { isZero = false break } } // If not all zero, use normal writer. if !isZero { return sfw.w.Write(p) } // Otherwise, poke a hole in the target file. _, err = sfw.w.Seek(int64(len(p)), io.SeekCurrent) if err != nil { return -1, err } return len(p), nil } // sliceAny returns true when any element in a slice satisfy a predicate. func sliceAny[T any](slice []T, predicate func(T) bool) bool { return slices.ContainsFunc(slice, predicate) } // RoundAbove returns the next multiple of `above` greater than `val`. func RoundAbove(above, val int64) int64 { if val < above { val = above } rounded := int64(val/above) * above // Ensure the rounded size is at least x. if rounded < val { rounded += above } return rounded } // ValidateDependentConfigKey validates an enabled dependent configuration key. func ValidateDependentConfigKey(cfg map[string]string) error { if util.IsTrue(cfg["security.shared"]) { return fmt.Errorf("dependent and security.shared are mutually exclusive") } for k := range cfg { if !strings.HasPrefix(k, "snapshots.") { continue } return fmt.Errorf("Dependent disk may not have a %q config", k) } return nil } // BackupPrefix returns backup prefix based on volume type. func BackupPrefix(vol Volume) string { backupPrefix := "container" if vol.IsVMBlock() { backupPrefix = "virtual-machine" } else if vol.volType == VolumeTypeCustom { backupPrefix = "volume" } return backupPrefix } // BackupSnapshotPrefix returns backup snapshot prefix based on volume type. func BackupSnapshotPrefix(vol Volume) string { backupSnapshotsPrefix := "snapshots" if vol.IsVMBlock() { backupSnapshotsPrefix = "virtual-machine-snapshots" } else if vol.volType == VolumeTypeCustom { backupSnapshotsPrefix = "volume-snapshots" } return backupSnapshotsPrefix } // BackupVolume copy a volume into the backup target location. func BackupVolume(d Driver, v Volume, writer instancewriter.InstanceWriter, mountPath string, blockPath string, prefix string) error { // Reset hard link cache as we are copying a new volume (instance or snapshot). writer.ResetHardLinkMap() if v.contentType == ContentTypeBlock || v.contentType == ContentTypeISO { // Get size of disk block device for tarball header. blockDiskSize, err := BlockDiskSizeBytes(blockPath) if err != nil { return fmt.Errorf("Error getting block device size %q: %w", blockPath, err) } var exclude []string // Files to exclude from filesystem volume backup. if !linux.IsBlockdevPath(blockPath) { // Exclude the volume root disk file from the filesystem volume backup. // We will read it as a block device later instead. exclude = append(exclude, blockPath) } if v.IsVMBlock() { logMsg := "Copying virtual machine config volume" d.Logger().Debug(logMsg, logger.Ctx{"sourcePath": mountPath, "prefix": prefix}) err = filepath.Walk(mountPath, func(srcPath string, fi os.FileInfo, err error) error { if err != nil { return err } // Skip any excluded files. if util.StringHasPrefix(srcPath, exclude...) { return nil } name := filepath.Join(prefix, strings.TrimPrefix(srcPath, mountPath)) err = writer.WriteFile(name, srcPath, fi, false) if err != nil { return fmt.Errorf("Error adding %q as %q to tarball: %w", srcPath, name, err) } return nil }) if err != nil { return err } } name := fmt.Sprintf("%s.%s", prefix, genericVolumeBlockExtension) logMsg := "Copying virtual machine block volume" if v.volType == VolumeTypeCustom { logMsg = "Copying custom block volume" } d.Logger().Debug(logMsg, logger.Ctx{"sourcePath": blockPath, "file": name, "size": blockDiskSize}) from, err := os.Open(blockPath) if err != nil { return fmt.Errorf("Error opening file for reading %q: %w", blockPath, err) } defer func() { _ = from.Close() }() var fileSize int64 fileSize, err = strconv.ParseInt(v.config["size"], 10, 64) if err != nil { fileSize = blockDiskSize } fi := instancewriter.FileInfo{ FileName: name, FileSize: fileSize, FileMode: 0o600, FileModTime: time.Now(), } err = writer.WriteFileFromReader(from, &fi) if err != nil { return fmt.Errorf("Error copying %q as %q to tarball: %w", blockPath, name, err) } err = from.Close() if err != nil { return fmt.Errorf("Failed to close file %q: %w", blockPath, err) } return nil } logMsg := "Copying container filesystem volume" if v.volType == VolumeTypeCustom { logMsg = "Copying custom filesystem volume" } d.Logger().Debug(logMsg, logger.Ctx{"sourcePath": mountPath, "prefix": prefix}) // Follow the target if mountPath is a symlink. // Functions like filepath.Walk() won't list any directory content otherwise. target, err := os.Readlink(mountPath) if err == nil { // Make sure the target is valid before return it. _, err = os.Stat(target) if err == nil { mountPath = target } } return filepath.Walk(mountPath, func(srcPath string, fi os.FileInfo, err error) error { if err != nil { if errors.Is(err, fs.ErrNotExist) { logger.Warnf("File vanished during export: %q, skipping", srcPath) return nil } return fmt.Errorf("Error walking file during export: %q: %w", srcPath, err) } name := filepath.Join(prefix, strings.TrimPrefix(srcPath, mountPath)) // Write the file to the tarball with ignoreGrowth enabled so that if the // source file grows during copy we only copy up to the original size. // This means that the file in the tarball may be inconsistent. err = writer.WriteFile(name, srcPath, fi, true) if err != nil { return fmt.Errorf("Error adding %q as %q to tarball: %w", srcPath, name, err) } return nil }) } // UnpackVolume unpack a volume from a backup tarball file. func UnpackVolume(d Driver, vol Volume, r io.ReadSeeker, tarArgs []string, unpacker []string, srcPrefix string, mountPath string, targetPath string) error { volTypeName := "container" if vol.IsVMBlock() { volTypeName = "virtual machine" } else if vol.volType == VolumeTypeCustom { volTypeName = "custom" } // Clear the volume ready for unpack. err := wipeDirectory(mountPath) if err != nil { return fmt.Errorf("Error clearing volume before unpack: %w", err) } // Unpack the filesystem parts of the volume (for containers and custom filesystem volumes that is // the respective root filesystem data or volume itself, and for VMs that is the config volume). // Custom block volumes do not have a filesystem component to their volumes. if !vol.IsCustomBlock() { // Prepare tar arguments. srcParts := strings.Split(srcPrefix, string(os.PathSeparator)) args := append(tarArgs, []string{ "-", "--xattrs-include=*", "--restrict", "--force-local", "--numeric-owner", "-C", mountPath, }...) if vol.Type() == VolumeTypeCustom { // If the volume type is custom, then we need to ensure that we restore the top level // directory's ownership from the backup. We cannot use --strip-components flag because it // removes the top level directory from the unpack list. Instead we use the --transform // flag to remove the prefix path and transform it into the "." current unpack directory. args = append(args, fmt.Sprintf("--transform=s/^%s/./", strings.ReplaceAll(srcPrefix, "/", `\/`))) } else { // For instance volumes, the user created files are stored in the rootfs sub-directory // and so strip-components flag works fine. args = append(args, fmt.Sprintf("--strip-components=%d", len(srcParts))) } // Directory to unpack comes after other options. args = append(args, srcPrefix) // Extract filesystem volume. d.Logger().Debug(fmt.Sprintf("Unpacking %s filesystem volume", volTypeName), logger.Ctx{"source": srcPrefix, "target": mountPath, "args": fmt.Sprintf("%+v", args)}) _, err := r.Seek(0, io.SeekStart) if err != nil { return err } f, err := os.OpenFile(mountPath, os.O_RDONLY, 0) if err != nil { return fmt.Errorf("Error opening directory: %w", err) } defer func() { _ = f.Close() }() allowedCmds := []string{} if len(unpacker) > 0 { allowedCmds = append(allowedCmds, unpacker[0]) } err = archive.ExtractWithFds("tar", args, allowedCmds, io.NopCloser(r), f) if err != nil { return fmt.Errorf("Error starting unpack: %w", err) } } // Extract block file to block volume. if vol.contentType == ContentTypeBlock { srcFile := fmt.Sprintf("%s.%s", srcPrefix, genericVolumeBlockExtension) tr, cancelFunc, err := archive.CompressedTarReader(context.Background(), r, unpacker, mountPath) if err != nil { return err } defer cancelFunc() unpackBlockVolume := func(hdr *tar.Header) error { var allowUnsafeResize bool // Reset the disk. err = linux.ClearBlock(targetPath, 0) if err != nil { return err } // Open block file (use O_CREATE to support drivers that use image files). to, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0o644) if err != nil { return fmt.Errorf("Error opening file for writing %q: %w", targetPath, err) } defer func() { _ = to.Close() }() // Restore original size of volume from raw block backup file size. d.Logger().Debug("Setting volume size from source", logger.Ctx{"source": srcFile, "target": targetPath, "size": hdr.Size}) // Allow potentially destructive resize of volume as we are going to be // overwriting it entirely anyway. This allows shrinking of block volumes. allowUnsafeResize = true err = d.SetVolumeQuota(vol, fmt.Sprintf("%d", hdr.Size), allowUnsafeResize, nil) if err != nil { return err } logMsg := "Unpacking virtual machine block volume" if vol.volType == VolumeTypeCustom { logMsg = "Unpacking custom block volume" } // Copy the data. toPipe := io.Writer(to) if !d.Info().ZeroUnpack { toPipe = NewSparseFileWrapper(to) } d.Logger().Debug(logMsg, logger.Ctx{"source": srcFile, "target": targetPath}) _, err = util.SafeCopy(toPipe, tr) if err != nil { return err } cancelFunc() return to.Close() } for { hdr, err := tr.Next() if err == io.EOF { break // End of archive. } if err != nil { return err } if hdr.Name == srcFile { return unpackBlockVolume(hdr) } } return fmt.Errorf("Could not find %q", srcFile) } return nil } incus-7.0.0/internal/server/storage/drivers/utils_ceph.go000066400000000000000000000225051517523235500236050ustar00rootroot00000000000000package drivers import ( "bufio" "encoding/json" "errors" "fmt" "os" "strings" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/subprocess" "github.com/lxc/incus/v7/shared/util" ) // CephGetRBDImageName returns the RBD image name as it is used in ceph. // Example: // A custom block volume named vol1 in project default will return custom_default_vol1.block. func CephGetRBDImageName(vol Volume, snapName string, zombie bool) string { var out string parentName, snapshotName, isSnapshot := api.GetParentAndSnapshotName(vol.name) // Only use filesystem suffix on filesystem type image volumes (for all content types). if vol.volType == VolumeTypeImage || vol.volType == cephVolumeTypeZombieImage { parentName = fmt.Sprintf("%s_%s", parentName, vol.ConfigBlockFilesystem()) } if vol.contentType == ContentTypeBlock { parentName = fmt.Sprintf("%s%s", parentName, cephBlockVolSuffix) } else if vol.contentType == ContentTypeISO { parentName = fmt.Sprintf("%s%s", parentName, cephISOVolSuffix) } // Use volume's type as storage volume prefix, unless there is an override in cephVolTypePrefixes. volumeTypePrefix := string(vol.volType) volumeTypePrefixOverride, foundOveride := cephVolTypePrefixes[vol.volType] if foundOveride { volumeTypePrefix = volumeTypePrefixOverride } if snapName != "" { // Always use the provided snapshot name if specified. out = fmt.Sprintf("%s_%s@%s", volumeTypePrefix, parentName, snapName) } else { if isSnapshot { // If volumeName is a snapshot (/) and snapName is not set, // assume that it's a normal snapshot (not a zombie) and prefix it with // "snapshot_". out = fmt.Sprintf("%s_%s@snapshot_%s", volumeTypePrefix, parentName, snapshotName) } else { out = fmt.Sprintf("%s_%s", volumeTypePrefix, parentName) } } // If the volume is to be in zombie state (i.e. not tracked in the database), // prefix the output with "zombie_". if zombie { out = fmt.Sprintf("zombie_%s", out) } return out } // CephBuildMount creates a mount string and option list from mount parameters. func CephBuildMount(user string, key string, fsid string, monitors Monitors, fsName string, path string) (source string, options []string) { // Ceph mount paths must begin with a '/', if it doesn't (or is empty). // prefix it now. The leading '/' can be stripped out during option parsing. if !strings.HasPrefix(path, "/") { path = "/" + path } msgrV2 := false monAddrs := monitors.V1 if len(monitors.V2) > 0 { msgrV2 = true monAddrs = monitors.V2 } // Build the source path. source = fmt.Sprintf("%s@%s.%s=%s", user, fsid, fsName, path) // Build the options list. options = []string{ "mon_addr=" + strings.Join(monAddrs, "/"), "name=" + user, } // If key is blank assume cephx is disabled. if key != "" { options = append(options, "secret="+key) } // Pick connection mode. if msgrV2 { options = append(options, "ms_mode=prefer-crc") } else { options = append(options, "ms_mode=legacy") } return source, options } // callCeph makes a call to ceph with the given args. func callCeph(args ...string) (string, error) { out, err := subprocess.RunCommand("ceph", args...) logger.Debug("callCeph", logger.Ctx{ "cmd": "ceph", "args": args, "err": err, "out": out, }) return strings.TrimSpace(out), err } // callCephJSON makes a call to the `ceph` admin tool with the given args then parses the json output into `out`. func callCephJSON(out any, args ...string) error { // Get as JSON format. args = append([]string{"--format", "json"}, args...) // Make the call. jsonOut, err := callCeph(args...) if err != nil { return err } // Parse the JSON. err = json.Unmarshal([]byte(jsonOut), &out) return err } // Monitors holds a list of ceph monitor addresses based on which protocol they expect. type Monitors struct { V1 []string V2 []string } // CephMonitors returns a list of public monitor IP:ports for the given cluster. func CephMonitors(cluster string, client string) (Monitors, error) { // Get the monitor dump, there may be other better ways but this is quick and easy. monitors := struct { Mons []struct { PublicAddrs struct { Addrvec []struct { Type string `json:"type"` Addr string `json:"addr"` } `json:"addrvec"` } `json:"public_addrs"` } `json:"mons"` }{} err := callCephJSON(&monitors, "--cluster", cluster, "--name", EnsureClientPrefix(client), "mon", "dump", ) if err != nil { return Monitors{}, fmt.Errorf("Ceph mon dump for %q failed: %w", cluster, err) } // Loop through monitors then monitor addresses and add them to the list. var ep Monitors for _, mon := range monitors.Mons { for _, addr := range mon.PublicAddrs.Addrvec { if addr.Type == "v1" { ep.V1 = append(ep.V1, addr.Addr) } else if addr.Type == "v2" { ep.V2 = append(ep.V2, addr.Addr) } else { logger.Warnf("Unknown ceph monitor address type: %q:%q", addr.Type, addr.Addr, ) } } } if len(ep.V2) == 0 { if len(ep.V1) == 0 { return Monitors{}, fmt.Errorf("No ceph monitors for %q", cluster) } logger.Warnf("Only found v1 monitors for ceph cluster %q", cluster) } return ep, nil } // CephKeyring retrieves the CephX key for the given entity. func CephKeyring(cluster string, client string) (string, error) { // See if we can't find it from the filesystem directly (short path). value, err := cephKeyringFromFile(cluster, client) if err == nil { return value, nil } // Check that cephx is enabled. authType, err := callCeph("--cluster", cluster, "config", "get", EnsureClientPrefix(client), "auth_service_required") if err != nil { return "", fmt.Errorf("Failed to query ceph config for auth_service_required: %w", err) } if authType == "none" { logger.Infof("Ceph cluster %q has disabled cephx", cluster) return "", nil } // Call ceph auth get. key := struct { Key string `json:"key"` }{} err = callCephJSON(&key, "--cluster", cluster, "auth", "get-key", client) if err != nil { return "", fmt.Errorf("Failed to get keyring for %q on %q: %w", client, cluster, err) } return key.Key, nil } func cephGetKeyFromFile(path string) (string, error) { cephKeyring, err := os.Open(path) if err != nil { return "", fmt.Errorf("Failed to open %q: %w", path, err) } // Locate the keyring entry and its value. var cephSecret string scan := bufio.NewScanner(cephKeyring) for scan.Scan() { line := scan.Text() line = strings.TrimSpace(line) if line == "" { continue } if strings.HasPrefix(line, "key") { fields := strings.SplitN(line, "=", 2) if len(fields) < 2 { continue } cephSecret = strings.TrimSpace(fields[1]) break } } if cephSecret == "" { return "", errors.New("Couldn't find a keyring entry") } return cephSecret, nil } // cephKeyringFromFile gets the key for a particular Ceph cluster and client name. func cephKeyringFromFile(cluster string, client string) (string, error) { var cephSecret string cephConfigPath := fmt.Sprintf("/etc/ceph/%v.conf", cluster) keyringPathFull := fmt.Sprintf("/etc/ceph/%v.client.%v.keyring", cluster, client) keyringPathCluster := fmt.Sprintf("/etc/ceph/%v.keyring", cluster) keyringPathGlobal := "/etc/ceph/keyring" keyringPathGlobalBin := "/etc/ceph/keyring.bin" if util.PathExists(keyringPathFull) { return cephGetKeyFromFile(keyringPathFull) } else if util.PathExists(keyringPathCluster) { return cephGetKeyFromFile(keyringPathCluster) } else if util.PathExists(keyringPathGlobal) { return cephGetKeyFromFile(keyringPathGlobal) } else if util.PathExists(keyringPathGlobalBin) { return cephGetKeyFromFile(keyringPathGlobalBin) } else if util.PathExists(cephConfigPath) { // Open the CEPH config file. cephConfig, err := os.Open(cephConfigPath) if err != nil { return "", fmt.Errorf("Failed to open %q: %w", cephConfigPath, err) } // Locate the keyring entry and its value. scan := bufio.NewScanner(cephConfig) for scan.Scan() { line := scan.Text() line = strings.TrimSpace(line) if line == "" { continue } if strings.HasPrefix(line, "key") { fields := strings.SplitN(line, "=", 2) if len(fields) < 2 { continue } // Check all key related config keys. switch strings.TrimSpace(fields[0]) { case "key": cephSecret = strings.TrimSpace(fields[1]) case "keyfile": key, err := os.ReadFile(fields[1]) if err != nil { return "", err } cephSecret = strings.TrimSpace(string(key)) case "keyring": return cephGetKeyFromFile(strings.TrimSpace(fields[1])) } } if cephSecret != "" { break } } } if cephSecret == "" { return "", errors.New("Couldn't find a keyring entry") } return cephSecret, nil } // CephFsid retrieves the FSID for the given cluster. func CephFsid(cluster string, client string) (string, error) { // Call ceph fsid. fsid := struct { Fsid string `json:"fsid"` }{} err := callCephJSON(&fsid, "--cluster", cluster, "--name", EnsureClientPrefix(client), "fsid") if err != nil { return "", fmt.Errorf("Couldn't get fsid for %q: %w", cluster, err) } return fsid.Fsid, nil } // EnsureClientPrefix returns the given client string with the "client" prefix added, // but only if it does not already start with that prefix. func EnsureClientPrefix(client string) string { if strings.Contains(client, ".") { return client } return "client." + client } incus-7.0.0/internal/server/storage/drivers/utils_qcow2.go000066400000000000000000000272151517523235500237240ustar00rootroot00000000000000package drivers import ( "encoding/json" "errors" "fmt" "io/fs" "os" "path/filepath" "strconv" "strings" "time" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/operations" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/subprocess" ) // Type of the block volume. const ( BlockVolumeTypeRaw = "raw" BlockVolumeTypeQcow2 = "qcow2" ) // Qcow2ConfigVolumeBase represents the base component of a Btrfs subvolume name. const Qcow2ConfigVolumeBase = "instance" // Bitmap represents a dirty bitmap. type Bitmap struct { Name string `json:"name"` } // FormatData contains format specific data. type FormatData struct { Bitmaps []Bitmap `json:"bitmaps"` } // FormatSpecific contains format dependent metadata. type FormatSpecific struct { Type string `json:"type"` Data FormatData `json:"data"` } // ImageInfo contains information about a qcow2 image. type ImageInfo struct { BackingFilename string `json:"backing-filename"` Format string `json:"format"` VirtualSize int `json:"virtual-size"` FormatSpecific FormatSpecific `json:"format-specific"` } // Qcow2Create creates a qcow2-formatted image. func Qcow2Create(path string, backingPath string, size int64) error { args := []string{ "create", "-f", "qcow2", } if backingPath != "" { args = append(args, "-b", backingPath) args = append(args, "-F", "qcow2") } args = append(args, path) if size > 0 { args = append(args, fmt.Sprintf("%db", size)) } _, err := subprocess.RunCommand("qemu-img", args...) if err != nil { return err } return nil } // Qcow2Resize resizes a qcow2-formatted image. func Qcow2Resize(path string, newSize int64) error { args := []string{ "resize", path, fmt.Sprintf("%db", newSize), } _, err := subprocess.RunCommand("qemu-img", args...) if err != nil { return err } return nil } // Qcow2Rebase changes the backing file of a qcow2 image. func Qcow2Rebase(path string, backingPath string) error { _, err := subprocess.RunCommand("qemu-img", "rebase", "-u", "-b", backingPath, "-F", "qcow2", path) if err != nil { return err } return nil } // Qcow2Commit commits changes from a qcow2 image to its immediate backing file. func Qcow2Commit(path string) error { _, err := subprocess.RunCommand("qemu-img", "commit", "-f", "qcow2", path) if err != nil { return err } return nil } // Qcow2Info returns information about a qcow2 image. func Qcow2Info(path string) (*ImageInfo, error) { imgJSON, err := subprocess.RunCommand("qemu-img", "info", "-U", "--output=json", path) if err != nil { return nil, err } imgInfo := ImageInfo{} err = json.Unmarshal([]byte(imgJSON), &imgInfo) if err != nil { return nil, fmt.Errorf("Failed unmarshalling image info %q: %w (%q)", path, err, imgJSON) } return &imgInfo, nil } // Qcow2BackingChain returns information about the backing chain of a qcow2 image. func Qcow2BackingChain(path string) ([]string, error) { result := []string{} imgJSON, err := subprocess.RunCommand("qemu-img", "info", "-U", "--backing-chain", "--output=json", path) if err != nil { return nil, err } imgInfo := []struct { BackingFilename string `json:"backing-filename"` }{} err = json.Unmarshal([]byte(imgJSON), &imgInfo) if err != nil { return nil, fmt.Errorf("Failed unmarshalling image info %q: %w (%q)", path, err, imgJSON) } for _, info := range imgInfo { if info.BackingFilename == "" { break } result = append(result, info.BackingFilename) } return result, nil } // Qcow2MountConfigTask mounts the config filesystem volume with its snapshots and performs the task specified by the parameter. func Qcow2MountConfigTask(vol Volume, op *operations.Operation, task func(mountPath string) error) error { mountPath := fmt.Sprintf("%s%s", vol.MountPath(), tmpVolSuffix) mountVol := NewVolume(vol.driver, vol.driver.Name(), vol.volType, vol.contentType, vol.name, vol.config, vol.poolConfig) mountVol.mountFullFilesystem = true mountVol.mountCustomPath = mountPath wasMounted := linux.IsMountPoint(mountPath) err := mountVol.MountTask(func(mountPath string, op *operations.Operation) error { taskErr := task(mountVol.MountPath()) // Return task error if failed. if taskErr != nil { return taskErr } return nil }, op) if err != nil { return err } // MountTask delegates unmounting to UnmountVolume(), which calculates the // refCount based on the volume name and type. Since a volume can be mounted // at multiple paths, it is only unmounted when the refCount drops to zero. // In this case, we unmount from customPath if the mount is no longer needed. if !wasMounted && linux.IsMountPoint(mountPath) { err = TryUnmount(mountPath, 0) if err != nil { return fmt.Errorf("Failed to unmount logical volume: %w", err) } } // Remove temporary mount path. isEmpty, err := internalUtil.PathIsEmpty(mountPath) if err != nil { return err } if isEmpty { err := os.Remove(mountPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to remove '%s': %w", mountPath, err) } } return nil } // Qcow2CreateConfig creates the btrfs config filesystem associated with the QCOW2 block volume. func Qcow2CreateConfig(vol Volume, op *operations.Operation) error { err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { volPath := filepath.Join(mountPath, Qcow2ConfigVolumeBase) // Create the volume itself. _, err := subprocess.RunCommand("btrfs", "subvolume", "create", volPath) if err != nil { return err } err = syncBtrfs(mountPath) if err != nil { return err } return nil }) if err != nil { return err } return nil } // Qcow2CreateConfigSnapshot creates the btrfs snapshot of the config filesystem associated with the QCOW2 block volume. func Qcow2CreateConfigSnapshot(vol Volume, snapVol Volume, op *operations.Operation) error { err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { _, snapName, _ := api.GetParentAndSnapshotName(snapVol.Name()) dstPath := filepath.Join(mountPath, fmt.Sprintf("%s-%s", Qcow2ConfigVolumeBase, snapName)) srcPath := filepath.Join(mountPath, Qcow2ConfigVolumeBase) _, err := subprocess.RunCommand("btrfs", "subvolume", "snapshot", srcPath, dstPath) if err != nil { return err } err = syncBtrfs(mountPath) if err != nil { return err } return nil }) if err != nil { return err } return nil } // Qcow2RestoreConfigSnapshot restores the btrfs snapshot of the config filesystem associated with the QCOW2 block volume. func Qcow2RestoreConfigSnapshot(vol Volume, snapVol Volume, op *operations.Operation) error { err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { _, snapName, _ := api.GetParentAndSnapshotName(snapVol.Name()) snapPath := fmt.Sprintf("%s-%s", Qcow2ConfigVolumeBase, snapName) // Delete the subvolume itself. _, err := subprocess.RunCommand("btrfs", "subvolume", "delete", filepath.Join(mountPath, Qcow2ConfigVolumeBase)) if err != nil { return err } _, err = subprocess.RunCommand("btrfs", "subvolume", "snapshot", filepath.Join(mountPath, snapPath), filepath.Join(mountPath, Qcow2ConfigVolumeBase)) if err != nil { return err } err = syncBtrfs(mountPath) if err != nil { return err } return nil }) if err != nil { return err } return nil } // Qcow2RenameConfigSnapshot renames the btrfs snapshot of the config filesystem associated with the QCOW2 block volume. func Qcow2RenameConfigSnapshot(vol Volume, snapVol Volume, newName string, op *operations.Operation) error { err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { _, snapName, _ := api.GetParentAndSnapshotName(snapVol.Name()) oldPath := filepath.Join(mountPath, fmt.Sprintf("%s-%s", Qcow2ConfigVolumeBase, snapName)) newPath := filepath.Join(mountPath, fmt.Sprintf("%s-%s", Qcow2ConfigVolumeBase, newName)) err := os.Rename(oldPath, newPath) if err != nil { return fmt.Errorf("Failed to rename %q to %q: %w", oldPath, newPath, err) } err = syncBtrfs(mountPath) if err != nil { return err } return nil }) if err != nil { return err } return nil } // Qcow2DeleteConfigSnapshot deletes the btrfs snapshot of the config filesystem associated with the QCOW2 block volume. func Qcow2DeleteConfigSnapshot(vol Volume, snapVol Volume, op *operations.Operation) error { err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { _, snapName, _ := api.GetParentAndSnapshotName(snapVol.Name()) path := filepath.Join(mountPath, fmt.Sprintf("%s-%s", Qcow2ConfigVolumeBase, snapName)) // Delete the subvolume itself. _, err := subprocess.RunCommand("btrfs", "subvolume", "delete", path) if err != nil { return err } err = syncBtrfs(mountPath) if err != nil { return err } return nil }) if err != nil { return err } // Remove the snapshot mount path from the storage device. snapPath := snapVol.MountPath() err = os.RemoveAll(snapPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Error removing snapshot mount path %q: %w", snapPath, err) } // Remove the parent snapshot directory if this is the last snapshot being removed. parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) err = deleteParentSnapshotDirIfEmpty(snapVol.pool, snapVol.volType, parentName) if err != nil { return err } return nil } func syncBtrfs(mountPath string) error { _, err := subprocess.RunCommand("btrfs", "filesystem", "sync", mountPath) if err != nil { return err } return nil } // IsQcow2Block checks whether a volume is a QCOW2 block device. func IsQcow2Block(vol Volume) bool { return vol.Config()["block.type"] == BlockVolumeTypeQcow2 && vol.ContentType() == ContentTypeBlock } // getFreeNbd returns the first free NBD device. func getFreeNbd() (string, error) { nbdIndex := 0 for { nbdPath := fmt.Sprintf("/sys/class/block/nbd%d", nbdIndex) _, err := os.Stat(nbdPath) if err != nil { return "", fmt.Errorf("No free nbd device found") } data, err := os.ReadFile(fmt.Sprintf("%s/size", nbdPath)) if err != nil { return "", err } size, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64) if err != nil { return "", err } if size == 0 { return fmt.Sprintf("/dev/nbd%d", nbdIndex), nil } nbdIndex += 1 } } // ConnectQemuNbd exports a QCOW2 volume using the NBD protocol via qemu-nbd. func ConnectQemuNbd(devPath string, format string, detectZeroes string, readOnly bool) (string, error) { nbdPath, err := getFreeNbd() if err != nil { return "", err } args := []string{fmt.Sprintf("--connect=%s", nbdPath)} if detectZeroes != "" { if detectZeroes == "unmap" { args = append(args, "--discard=unmap") } args = append(args, fmt.Sprintf("--detect-zeroes=%s", detectZeroes)) } if format != "" { args = append(args, fmt.Sprintf("--format=%s", format)) } if readOnly { args = append(args, "--read-only") } args = append(args, devPath) _, err = subprocess.RunCommand("qemu-nbd", args...) if err != nil { return "", err } // It can take a little while before the /dev/nbdX device is fully mapped. var sz int64 for range 20 { sz, err = BlockDiskSizeBytes(nbdPath) if err != nil { _ = DisconnectQemuNbd(nbdPath) return "", err } if sz > 0 { break } time.Sleep(100 * time.Millisecond) } if sz == 0 { _ = DisconnectQemuNbd(nbdPath) return "", fmt.Errorf("NBD device %q not correctly mapped after 2s", nbdPath) } return nbdPath, nil } // DisconnectQemuNbd disconnects the NBD device at nbdPath. func DisconnectQemuNbd(nbdPath string) error { _, err := subprocess.RunCommand("qemu-nbd", "--disconnect", nbdPath) if err != nil { return err } return nil } incus-7.0.0/internal/server/storage/drivers/utils_test.go000066400000000000000000000022041517523235500236370ustar00rootroot00000000000000package drivers import ( "testing" "github.com/stretchr/testify/assert" ) // Test GetVolumeMountPath. func TestGetVolumeMountPath(t *testing.T) { poolName := "testpool" // Test custom volume. path := GetVolumeMountPath(poolName, VolumeTypeCustom, "testvol") expected := GetPoolMountPath(poolName) + "/custom/testvol" assert.Equal(t, expected, path) // Test custom volume snapshot. path = GetVolumeMountPath(poolName, VolumeTypeCustom, "testvol/snap1") expected = GetPoolMountPath(poolName) + "/custom-snapshots/testvol/snap1" assert.Equal(t, expected, path) // Test image volume. path = GetVolumeMountPath(poolName, VolumeTypeImage, "fingerprint") expected = GetPoolMountPath(poolName) + "/images/fingerprint" assert.Equal(t, expected, path) // Test container volume. path = GetVolumeMountPath(poolName, VolumeTypeContainer, "testvol") expected = GetPoolMountPath(poolName) + "/containers/testvol" assert.Equal(t, expected, path) // Test virtual-machine volume. path = GetVolumeMountPath(poolName, VolumeTypeVM, "testvol") expected = GetPoolMountPath(poolName) + "/virtual-machines/testvol" assert.Equal(t, expected, path) } incus-7.0.0/internal/server/storage/drivers/volume.go000066400000000000000000000725131517523235500227610ustar00rootroot00000000000000package drivers import ( "bytes" "context" "errors" "fmt" "net" "os" "os/exec" "path/filepath" "slices" "strconv" "strings" "syscall" "github.com/pkg/sftp" "golang.org/x/sys/unix" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/server/locking" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/refcount" "github.com/lxc/incus/v7/internal/server/state" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/units" "github.com/lxc/incus/v7/shared/util" ) // tmpVolSuffix Suffix to use for any temporary volumes created by Incus. const tmpVolSuffix = ".incustmp" // isoVolSuffix suffix used for iso content type volumes. const isoVolSuffix = ".iso" // DefaultBlockSize is the default size of block volumes. const DefaultBlockSize = "10GiB" // DefaultFilesystem filesystem to use for block devices by default. const DefaultFilesystem = "ext4" // defaultFilesystemMountOpts mount options to use for filesystem block devices by default. const defaultFilesystemMountOptions = "discard" // volIDQuotaSkip is used to indicate to drivers that quotas should not be setup, used during backup import. const volIDQuotaSkip = int64(-1) // VolumeType represents a storage volume type. type VolumeType string // IsInstance indicates if the VolumeType represents an instance type. func (t VolumeType) IsInstance() bool { if t == VolumeTypeContainer || t == VolumeTypeVM { return true } return false } // Singular returns the singular version of the type name. func (t VolumeType) Singular() string { return strings.TrimSuffix(string(t), "s") } // VolumeTypeBucket represents a bucket storage volume. const VolumeTypeBucket = VolumeType("buckets") // VolumeTypeImage represents an image storage volume. const VolumeTypeImage = VolumeType("images") // VolumeTypeCustom represents a custom storage volume. const VolumeTypeCustom = VolumeType("custom") // VolumeTypeContainer represents a container storage volume. const VolumeTypeContainer = VolumeType("containers") // VolumeTypeVM represents a virtual-machine storage volume. const VolumeTypeVM = VolumeType("virtual-machines") // ContentType indicates the format of the volume. type ContentType string // ContentTypeFS indicates the volume will be populated with a mountable filesystem. const ContentTypeFS = ContentType("filesystem") // ContentTypeBlock indicates the volume will be a block device and its contents and we do not // know which filesystem(s) (if any) are in use. const ContentTypeBlock = ContentType("block") // ContentTypeISO indicates the volume will be an ISO which is read-only, and uses the ISO 9660 filesystem. const ContentTypeISO = ContentType("iso") // VolumePostHook function returned from a storage action that should be run later to complete the action. type VolumePostHook func(vol Volume) error type baseDirectory struct { Paths []string Mode os.FileMode } // BaseDirectories maps volume types to the expected directories. var BaseDirectories = map[VolumeType]baseDirectory{ VolumeTypeBucket: {Paths: []string{"buckets"}, Mode: 0o711}, VolumeTypeContainer: {Paths: []string{"containers", "containers-snapshots"}, Mode: 0o711}, VolumeTypeCustom: {Paths: []string{"custom", "custom-snapshots"}, Mode: 0o700}, VolumeTypeImage: {Paths: []string{"images"}, Mode: 0o700}, VolumeTypeVM: {Paths: []string{"virtual-machines", "virtual-machines-snapshots"}, Mode: 0o700}, } // Volume represents a storage volume, and provides functions to mount and unmount it. type Volume struct { name string pool string poolConfig map[string]string volType VolumeType contentType ContentType config map[string]string driver Driver mountCustomPath string // Mount the filesystem volume at a custom location. mountFilesystemProbe bool // Probe filesystem type when mounting volume (when needed). mountFullFilesystem bool // Whether the whole volume, including snapshots data, should be mounted. It is used by the VM config filesystem. hasSource bool // Whether the volume is created from a source volume. isDeleted bool // Whether we're dealing with a hidden volume (kept until all references are gone). } // NewVolume instantiates a new Volume struct. func NewVolume(driver Driver, poolName string, volType VolumeType, contentType ContentType, volName string, volConfig map[string]string, poolConfig map[string]string) Volume { return Volume{ name: volName, pool: poolName, poolConfig: poolConfig, volType: volType, contentType: contentType, config: volConfig, driver: driver, } } // Name returns volume's name. func (v Volume) Name() string { return v.name } // Pool returns the volume's pool name. func (v Volume) Pool() string { return v.pool } // Config returns the volume's (unexpanded) config. func (v Volume) Config() map[string]string { return v.config } // ExpandedConfig returns either the value of the volume's config key or the pool's config "volume.{key}" value. func (v Volume) ExpandedConfig(key string) string { volVal, ok := v.config[key] if ok { return volVal } return v.poolConfig[fmt.Sprintf("volume.%s", key)] } // NewSnapshot instantiates a new Volume struct representing a snapshot of the parent volume. func (v Volume) NewSnapshot(snapshotName string) (Volume, error) { if v.IsSnapshot() { return Volume{}, errors.New("Cannot create a snapshot volume from a snapshot") } fullSnapName := GetSnapshotVolumeName(v.name, snapshotName) vol := NewVolume(v.driver, v.pool, v.volType, v.contentType, fullSnapName, v.config, v.poolConfig) // Propagate filesystem probe mode of parent volume. vol.SetMountFilesystemProbe(v.mountFilesystemProbe) return vol, nil } // IsSnapshot indicates if volume is a snapshot. func (v Volume) IsSnapshot() bool { return internalInstance.IsSnapshot(v.name) } // MountPath returns the path where the volume will be mounted. func (v Volume) MountPath() string { if v.mountCustomPath != "" { return v.mountCustomPath } volName := v.name if v.volType == VolumeTypeCustom && v.contentType == ContentTypeISO { volName = fmt.Sprintf("%s%s", volName, isoVolSuffix) } return GetVolumeMountPath(v.pool, v.volType, volName) } // mountLockName returns the lock name to use for mount/unmount operations on a volume. func (v Volume) mountLockName() string { return OperationLockName("Mount", v.pool, v.volType, v.contentType, v.name) } // MountLock attempts to lock the mount lock for the volume and returns the UnlockFunc. func (v Volume) MountLock() (locking.UnlockFunc, error) { return locking.Lock(context.TODO(), v.mountLockName()) } // MountRefCountIncrement increments the mount ref counter for the volume and returns the new value. func (v Volume) MountRefCountIncrement() uint { return refcount.Increment(v.mountLockName(), 1) } // MountRefCountDecrement decrements the mount ref counter for the volume and returns the new value. func (v Volume) MountRefCountDecrement() uint { return refcount.Decrement(v.mountLockName(), 1) } // MountInUse returns whether the volume has a mount ref counter >0. func (v Volume) MountInUse() bool { return refcount.Get(v.mountLockName()) > 0 } // EnsureMountPath creates the volume's mount path if missing, then sets the correct permission for the type. // If permission setting fails and the volume is a snapshot then the error is ignored as snapshots are read only. // The boolean flag indicates whether this is being called during volume creation. func (v Volume) EnsureMountPath(creation bool) error { volPath := v.MountPath() reverter := revert.New() defer reverter.Fail() // Create volume's mount path if missing, with any created directories set to 0711. if !util.PathExists(volPath) { if v.IsSnapshot() { // Create the parent directory if needed. parentName, _, _ := api.GetParentAndSnapshotName(v.name) err := CreateParentSnapshotDirIfMissing(v.pool, v.volType, parentName) if err != nil { return err } } err := os.Mkdir(volPath, 0o711) if err != nil { return fmt.Errorf("Failed to create mount directory %q: %w", volPath, err) } reverter.Add(func() { _ = os.Remove(volPath) }) } // If dealing with a custom volume and part of volume creation, apply initial mode and owner. if v.volType == VolumeTypeCustom && v.contentType == ContentTypeFS && creation { initialMode := v.ExpandedConfig("initial.mode") mode := os.FileMode(0o711) if initialMode != "" { m, err := strconv.ParseInt(initialMode, 8, 0) if err != nil { return err } mode = os.FileMode(m) } err := os.Chmod(volPath, mode) if err != nil { return err } uid, gid := 0, 0 initialUID := v.ExpandedConfig("initial.uid") if initialUID != "" { uid, err = strconv.Atoi(initialUID) if err != nil { return err } } initialGID := v.ExpandedConfig("initial.gid") if initialGID != "" { gid, err = strconv.Atoi(initialGID) if err != nil { return err } } // Set the owner of a custom volume if uid or gid have been set. if uid != 0 || gid != 0 { err = os.Chown(volPath, uid, gid) if err != nil { return err } } } // Set very restrictive mode 0100 for non-custom, non-bucket and non-image volumes. if v.volType != VolumeTypeCustom && v.volType != VolumeTypeImage && v.volType != VolumeTypeBucket { mode := os.FileMode(0o100) fInfo, err := os.Lstat(volPath) if err != nil { return fmt.Errorf("Error getting mount directory info %q: %w", volPath, err) } // We expect the mount path to be a directory, so use this for comparison. compareMode := os.ModeDir | mode // Set mode of actual volume's mount path if needed. if fInfo.Mode() != compareMode { err = os.Chmod(volPath, mode) // If the chmod failed, return the error as long as the volume is not a snapshot. // If the volume is a snapshot, we must ignore the error as snapshots are readonly and cannot be // modified after they are taken, such that any permission error is not fixable at mount time. if err != nil && !v.IsSnapshot() { return fmt.Errorf("Failed to chmod mount directory %q (%04o): %w", volPath, mode, err) } } } reverter.Success() return nil } // MountTask runs the supplied task after mounting the volume if needed. If the volume was mounted // for this then it is unmounted when the task finishes. func (v Volume) MountTask(task func(mountPath string, op *operations.Operation) error, op *operations.Operation) error { // If the volume is a snapshot then call the snapshot specific mount/unmount functions as // these will mount the snapshot read only. var err error if v.IsSnapshot() { err = v.driver.MountVolumeSnapshot(v, op) } else { err = v.driver.MountVolume(v, op) } if err != nil { return err } taskErr := task(v.MountPath(), op) // Try and unmount, even on task error. if v.IsSnapshot() { _, err = v.driver.UnmountVolumeSnapshot(v, op) } else { _, err = v.driver.UnmountVolume(v, false, op) } // Return task error if failed. if taskErr != nil { return taskErr } // Return unmount error if failed. if err != nil && !errors.Is(err, ErrInUse) { return err } return nil } // MountWithSnapshotsTask runs the supplied task after mounting the volume with its snapshots if needed. If the volume was mounted // for this then it is unmounted when the task finishes. func (v Volume) MountWithSnapshotsTask(task func(parentMountPath string, snapshotMountPaths map[string]string, op *operations.Operation) error, op *operations.Operation) error { var err error if v.IsSnapshot() { return fmt.Errorf("Volume cannot be snapshot") } err = v.driver.MountVolume(v, op) if err != nil { return err } defer func() { _, _ = v.driver.UnmountVolume(v, false, op) }() snapshots, err := v.Snapshots(op) if err != nil { return err } unmountSnapshots := func(snapshots []Volume) { for _, s := range snapshots { _, _ = v.driver.UnmountVolumeSnapshot(s, op) } } mounted := []Volume{} for _, s := range snapshots { err = v.driver.MountVolumeSnapshot(s, op) if err != nil { unmountSnapshots(mounted) return err } mounted = append(mounted, s) } defer unmountSnapshots(mounted) snapshotMountPaths := map[string]string{} for _, s := range snapshots { snapDiskPath, err := v.driver.GetVolumeDiskPath(s) if err != nil { return err } backingPath, err := v.driver.GetQcow2BackingFilePath(s) if err != nil { return err } // Check whether the backing path and the LVM volume resolve to the same /dev/dm-X device. target, err := filepath.EvalSymlinks(backingPath) if err != nil { return err } if target != snapDiskPath { return fmt.Errorf("/dev symlinks are in an inconsistent state") } snapshotMountPaths[s.Name()] = s.MountPath() } taskErr := task(v.MountPath(), snapshotMountPaths, op) // Return task error if failed. if taskErr != nil { return taskErr } return nil } // UnmountTask runs the supplied task after unmounting the volume if needed. // If the volume was unmounted for this then it is mounted when the task finishes. // keepBlockDev indicates if backing block device should be not be deactivated if volume is unmounted. func (v Volume) UnmountTask(task func(op *operations.Operation) error, keepBlockDev bool, op *operations.Operation) error { // If the volume is a snapshot then call the snapshot specific mount/unmount functions as // these will mount the snapshot read only. if v.IsSnapshot() { ourUnmount, err := v.driver.UnmountVolumeSnapshot(v, op) if err != nil { return err } if ourUnmount { defer func() { _ = v.driver.MountVolumeSnapshot(v, op) }() } } else { ourUnmount, err := v.driver.UnmountVolume(v, keepBlockDev, op) if err != nil { return err } if ourUnmount { defer func() { _ = v.driver.MountVolume(v, op) }() } } return task(op) } // Snapshots returns a list of snapshots for the volume (in no particular order). func (v Volume) Snapshots(op *operations.Operation) ([]Volume, error) { if v.IsSnapshot() { return nil, errors.New("Volume is a snapshot") } snapshots, err := v.driver.VolumeSnapshots(v, op) if err != nil { return nil, err } snapVols := make([]Volume, 0, len(snapshots)) for _, snapName := range snapshots { snapshot, err := v.NewSnapshot(snapName) if err != nil { return nil, err } snapVols = append(snapVols, snapshot) } return snapVols, nil } // SnapshotsMatch checks that the snapshots, according to the storage driver, match those provided (although not // necessarily in the same order). func (v Volume) SnapshotsMatch(snapNames []string, op *operations.Operation) error { if v.IsSnapshot() { return errors.New("Volume is a snapshot") } snapshots, err := v.driver.VolumeSnapshots(v, op) if err != nil { return err } for _, snapName := range snapNames { if !slices.Contains(snapshots, snapName) { return fmt.Errorf("Snapshot %q expected but not in storage", snapName) } } for _, snapshot := range snapshots { if !slices.Contains(snapNames, snapshot) { return fmt.Errorf("Snapshot %q in storage but not expected", snapshot) } } return nil } // IsBlockBacked indicates whether storage device is block backed. func (v Volume) IsBlockBacked() bool { return v.driver.isBlockBacked(v) || v.mountFilesystemProbe } // Type returns the volume type. func (v Volume) Type() VolumeType { return v.volType } // ContentType returns the content type. func (v Volume) ContentType() ContentType { return v.contentType } // IsVMBlock returns true if volume is a block volume for virtual machines or associated images. func (v Volume) IsVMBlock() bool { return (v.volType == VolumeTypeVM || v.volType == VolumeTypeImage) && v.contentType == ContentTypeBlock } // IsCustomBlock returns true if volume is a custom block volume. func (v Volume) IsCustomBlock() bool { return (v.volType == VolumeTypeCustom && v.contentType == ContentTypeBlock) } // NewVMBlockFilesystemVolume returns a copy of the volume with the content type set to ContentTypeFS and the // config "size" property set to "size.state" or DefaultVMBlockFilesystemSize if not set. func (v Volume) NewVMBlockFilesystemVolume() Volume { // Copy volume config so modifications don't affect original volume. newConf := make(map[string]string, len(v.config)) for k, v := range v.config { if k == "zfs.block_mode" { continue // VM filesystem volumes never use ZFS block mode. } newConf[k] = v } if v.config["size.state"] != "" { newConf["size"] = v.config["size.state"] } else { // Fallback to the default VM filesystem size. newConf["size"] = v.driver.Info().DefaultVMBlockFilesystemSize } vol := NewVolume(v.driver, v.pool, v.volType, ContentTypeFS, v.name, newConf, v.poolConfig) // Propagate filesystem probe mode of parent volume. vol.SetMountFilesystemProbe(v.mountFilesystemProbe) return vol } // SetQuota calls SetVolumeQuota on the Volume's driver. func (v Volume) SetQuota(size string, allowUnsafeResize bool, op *operations.Operation) error { return v.driver.SetVolumeQuota(v, size, allowUnsafeResize, op) } // SetConfigSize sets the size config property on the Volume (does not resize volume). func (v Volume) SetConfigSize(size string) { v.config["size"] = size } // SetConfigStateSize sets the size.state config property on the Volume (does not resize volume). func (v Volume) SetConfigStateSize(size string) { v.config["size.state"] = size } // ConfigBlockFilesystem returns the filesystem to use for block volumes. Returns config value "block.filesystem" // if defined in volume or pool's volume config, otherwise the DefaultFilesystem. func (v Volume) ConfigBlockFilesystem() string { blockType := v.ExpandedConfig("block.type") if blockType != "" && blockType == BlockVolumeTypeQcow2 && v.contentType == ContentTypeFS { return "btrfs" } fs := v.ExpandedConfig("block.filesystem") if fs != "" { return fs } return DefaultFilesystem } // ConfigBlockMountOptions returns the filesystem mount options to use for block volumes. Returns config value // "block.mount_options" if defined in volume or pool's volume config, otherwise defaultFilesystemMountOptions. func (v Volume) ConfigBlockMountOptions() string { if v.ExpandedConfig("block.type") == BlockVolumeTypeQcow2 && !v.mountFullFilesystem { _, snapName, isSnap := api.GetParentAndSnapshotName(v.name) subvol := Qcow2ConfigVolumeBase if isSnap { subvol = fmt.Sprintf("%s-%s", Qcow2ConfigVolumeBase, snapName) } return fmt.Sprintf("subvol=%s", subvol) } fs := v.ExpandedConfig("block.mount_options") if fs != "" { return fs } // Use some special options if the filesystem for the volume is BTRFS. if v.ConfigBlockFilesystem() == "btrfs" { return "user_subvol_rm_allowed,discard" } return defaultFilesystemMountOptions } // ConfigSize returns the size to use when creating new a volume. Returns config value "size" if defined in volume // or pool's volume config, otherwise for block volumes and block-backed volumes the defaultBlockSize. For other // volumes an empty string is returned if no size is defined. func (v Volume) ConfigSize() string { size := v.ExpandedConfig("size") // If volume size isn't defined in either volume or pool config, then for block volumes or block-backed // volumes return the defaultBlockSize. if (size == "" || size == "0") && (v.contentType == ContentTypeBlock || v.IsBlockBacked()) { return DefaultBlockSize } // Return defined size or empty string if not defined. return size } // ConfigSizeFromSource derives the volume size to use for a new volume when copying from a source volume. // Where possible (if the source volume has a volatile.rootfs.size property), it checks that the source volume // isn't larger than the volume's "size" setting and the pool's "volume.size" setting. func (v Volume) ConfigSizeFromSource(srcVol Volume) (string, error) { // If source is not an image, then only use volume specified size. This is so the pool volume size isn't // taken into account for non-image volume copies. if srcVol.volType != VolumeTypeImage { return v.config["size"], nil } // VM config filesystem volumes should always have a fixed specified size, so just return volume size. if v.volType == VolumeTypeVM && v.contentType == ContentTypeFS { return v.config["size"], nil } // If the source image doesn't have any size information, then use volume/pool/default size in that order. if srcVol.config["volatile.rootfs.size"] == "" { return v.ConfigSize(), nil } imgSizeBytes, err := units.ParseByteSizeString(srcVol.config["volatile.rootfs.size"]) if err != nil { return "", err } // If volume/pool size is specified (excluding default size), then check it against the image minimum size. volSize := v.ExpandedConfig("size") if volSize != "" && volSize != "0" { volSizeBytes, err := units.ParseByteSizeString(volSize) if err != nil { return volSize, err } // Round the vol size (for comparison only) because some storage drivers round volumes they create, // and so the published images created from those volumes will also be rounded and will not be // directly usable with the same size setting without also rounding for this check. // Because we are not altering the actual size returned to use for the new volume, this will not // affect storage drivers that do not use rounding. volSizeBytes, err = v.driver.roundVolumeBlockSizeBytes(v, volSizeBytes) if err != nil { return volSize, err } // The volume/pool specified size is smaller than image minimum size. We must not continue as // these specified sizes provide protection against unpacking a massive image and filling the pool. if volSizeBytes < imgSizeBytes { return "", fmt.Errorf("Source image size (%d) exceeds specified volume size (%d)", imgSizeBytes, volSizeBytes) } // Use the specified volume size. return volSize, nil } // If volume/pool size not specified above, then fallback to default volume size (if relevant) and compare. volSize = v.ConfigSize() if volSize != "" && volSize != "0" { volSizeBytes, err := units.ParseByteSizeString(volSize) if err != nil { return "", err } // Use image minimum size as volSize if the default volume size is smaller. if volSizeBytes < imgSizeBytes { return srcVol.config["volatile.rootfs.size"], nil } } // Use the default volume size. return volSize, nil } // SetMountFilesystemProbe enables or disables the probing mode when mounting the filesystem volume. func (v *Volume) SetMountFilesystemProbe(probe bool) { v.mountFilesystemProbe = probe } // SetHasSource indicates whether the Volume is created from a source. func (v *Volume) SetHasSource(hasSource bool) { v.hasSource = hasSource } // Clone returns a copy of the volume. func (v Volume) Clone() Volume { // Copy the config map to avoid internal modifications affecting external state. newConfig := util.CloneMap(v.config) // Copy the pool config map to avoid internal modifications affecting external state. newPoolConfig := util.CloneMap(v.poolConfig) return NewVolume(v.driver, v.pool, v.volType, v.contentType, v.name, newConfig, newPoolConfig) } // forkfileLockName returns the forkfile lock name. func (v Volume) forkfileLockName() string { return fmt.Sprintf("forkfile_%s_%s_%s", v.Pool(), v.Type(), v.Name()) } // forkfileRunningLockName returns the forkfile-running lock name. func (v Volume) forkfileRunningLockName() string { return fmt.Sprintf("forkfile-running_%s_%s_%s", v.Pool(), v.Type(), v.Name()) } // forkfileRunPath returns the forkfile running path. func (v Volume) forkfileRunPath() string { name := fmt.Sprintf("%s.%s.%s", v.Pool(), v.Type(), v.Name()) return internalUtil.RunPath(name) } // StopForkfile attempts to send SIGKILL to forkfile then waits for it to exit. func (v Volume) StopForkfile() { // Make sure that when the function exits, no forkfile is running by acquiring the lock (which indicates // that forkfile isn't running and holding the lock) and then releasing it. defer func() { unlock, err := locking.Lock(context.TODO(), v.forkfileRunningLockName()) if err != nil { return } unlock() }() content, err := os.ReadFile(filepath.Join(v.forkfileRunPath(), "forkfile.pid")) if err != nil { return } pid, err := strconv.ParseInt(strings.TrimSpace(string(content)), 10, 64) if err != nil { return } // Forcefully kill the running process. _ = unix.Kill(int(pid), unix.SIGKILL) } // FileSFTPConn returns a connection to the forkfile handler. func (v Volume) FileSFTPConn(s *state.State) (net.Conn, error) { // Lock to avoid concurrent spawning. spawnUnlock, err := locking.Lock(context.TODO(), v.forkfileLockName()) if err != nil { return nil, err } defer spawnUnlock() // Create any missing directories in case the instance has never been started before. err = os.MkdirAll(v.forkfileRunPath(), 0o700) if err != nil { return nil, err } // Trickery to handle paths > 108 chars. dirFile, err := os.Open(v.forkfileRunPath()) if err != nil { return nil, err } defer func() { _ = dirFile.Close() }() forkfileAddr, err := net.ResolveUnixAddr("unix", fmt.Sprintf("/proc/self/fd/%d/forkfile.sock", dirFile.Fd())) if err != nil { return nil, err } // Attempt to connect on existing socket. forkfilePath := filepath.Join(v.forkfileRunPath(), "forkfile.sock") forkfileConn, err := net.DialUnix("unix", nil, forkfileAddr) if err == nil { // Found an existing server. return forkfileConn, nil } // Setup reverter. reverter := revert.New() defer reverter.Fail() // Create the listener. _ = os.Remove(forkfilePath) forkfileListener, err := net.ListenUnix("unix", forkfileAddr) if err != nil { return nil, err } reverter.Add(func() { _ = forkfileListener.Close() _ = os.Remove(forkfilePath) }) // Spawn forkfile in a Go routine. chReady := make(chan error) go func() { // Lock to avoid concurrent running forkfile. runUnlock, err := locking.Lock(context.TODO(), v.forkfileRunningLockName()) if err != nil { chReady <- err return } defer runUnlock() err = v.MountTask(func(mountPath string, _ *operations.Operation) error { // Start building the command. args := []string{ s.OS.ExecPath, "forkfile", "--", } extraFiles := []*os.File{} // Get the listener file. forkfileFile, err := forkfileListener.File() if err != nil { return err } defer func() { _ = forkfileFile.Close() }() args = append(args, "3") extraFiles = append(extraFiles, forkfileFile) // Get the rootfs. rootfsFile, err := os.Open(v.MountPath()) if err != nil { return err } defer func() { _ = rootfsFile.Close() }() args = append(args, "4") extraFiles = append(extraFiles, rootfsFile) // Get the pidfd, omitting it in case of a storage volume. args = append(args, "-1") // Finalize the args. args = append(args, "0") // Prepare sftp server. forkfile := exec.Cmd{ Path: s.OS.ExecPath, Args: args, ExtraFiles: extraFiles, } var stderr bytes.Buffer forkfile.Stderr = &stderr // Get the disk idmap. var idmapset *idmap.Set jsonIdmap, ok := v.config["volatile.idmap.last"] if ok { idmapset, err = idmap.NewSetFromJSON(jsonIdmap) if err != nil { return err } } if idmapset != nil { forkfile.SysProcAttr = &syscall.SysProcAttr{ Cloneflags: syscall.CLONE_NEWUSER, Credential: &syscall.Credential{ Uid: uint32(0), Gid: uint32(0), }, UidMappings: idmapset.ToUIDMappings(), GidMappings: idmapset.ToGIDMappings(), } } // Start the server. err = forkfile.Start() if err != nil { return fmt.Errorf("Failed to run forkfile: %w: %s", err, strings.TrimSpace(stderr.String())) } // Write PID file. pidFile := filepath.Join(v.forkfileRunPath(), "forkfile.pid") err = os.WriteFile(pidFile, fmt.Appendf(nil, "%d\n", forkfile.Process.Pid), 0o600) if err != nil { return fmt.Errorf("Failed to write forkfile PID: %w", err) } // Close the listener and delete the socket immediately after forkfile exits to avoid clients // thinking a listener is available while other deferred calls are being processed. defer func() { _ = forkfileListener.Close() _ = os.Remove(forkfilePath) _ = os.Remove(pidFile) }() // Indicate the process was spawned without error. close(chReady) // Wait for completion. err = forkfile.Wait() if err != nil { logger.Error("SFTP server stopped with error", logger.Ctx{"err": err, "stderr": strings.TrimSpace(stderr.String())}) // Don't return an error as channel is already closed. return nil } return nil }, nil) if err != nil { chReady <- err } }() // Wait for forkfile to have been spawned. err = <-chReady if err != nil { return nil, err } // Connect to the new server. forkfileConn, err = net.DialUnix("unix", nil, forkfileAddr) if err != nil { return nil, err } // All done. reverter.Success() return forkfileConn, nil } // FileSFTP returns an SFTP connection to the forkfile handler. func (v Volume) FileSFTP(s *state.State) (*sftp.Client, error) { // Connect to the forkfile daemon. conn, err := v.FileSFTPConn(s) if err != nil { return nil, err } // Get a SFTP client. client, err := sftp.NewClientPipe(conn, conn) if err != nil { _ = conn.Close() return nil, err } go func() { // Wait for the client to be done before closing the connection. _ = client.Wait() _ = conn.Close() }() return client, nil } incus-7.0.0/internal/server/storage/drivers/volume_test.go000066400000000000000000000111121517523235500240040ustar00rootroot00000000000000package drivers import ( "errors" "testing" "github.com/stretchr/testify/assert" ) // Test Volume_ConfigSizeFromSource. func Test_Volume_ConfigSizeFromSource(t *testing.T) { nonBlockBackedDriver := dir{} blockBackedDriver := lvm{} tests := []struct { vol Volume srcVol Volume err error size string }{ { // Check the volume's size is used when empty non-image source volume used. vol: Volume{driver: &nonBlockBackedDriver, volType: VolumeTypeContainer, config: map[string]string{"size": "1GiB"}}, srcVol: Volume{}, err: nil, size: "1GiB", }, { // Check the volume's pool volume.size isn't used when empty non-image source volume used. vol: Volume{driver: &nonBlockBackedDriver, volType: VolumeTypeContainer, poolConfig: map[string]string{"volume.size": "2GiB"}}, srcVol: Volume{}, err: nil, size: "", }, { // Check the volume's pool volume.size is used when volume size not specified and empty // image source volume used. vol: Volume{driver: &nonBlockBackedDriver, volType: VolumeTypeContainer, poolConfig: map[string]string{"volume.size": "2GiB"}}, srcVol: Volume{volType: VolumeTypeImage}, err: nil, size: "2GiB", }, { // Check the volume's default block disk size is used when volume is a block type and // neither volume or pool volume size is specified and empty image source volume used. vol: Volume{driver: &nonBlockBackedDriver, volType: VolumeTypeVM, contentType: ContentTypeBlock}, srcVol: Volume{volType: VolumeTypeImage}, err: nil, size: DefaultBlockSize, }, { // Check that the volume's smaller size than source image's rootfs size causes error. vol: Volume{driver: &nonBlockBackedDriver, volType: VolumeTypeContainer, config: map[string]string{"size": "1GiB"}}, srcVol: Volume{volType: VolumeTypeImage, config: map[string]string{"volatile.rootfs.size": "15GiB"}}, err: errors.New("Source image size (16106127360) exceeds specified volume size (1073741824)"), size: "", }, { // Check that the volume's larger size than source image's rootfs size overrides. vol: Volume{driver: &nonBlockBackedDriver, volType: VolumeTypeContainer, config: map[string]string{"size": "20GiB"}}, srcVol: Volume{volType: VolumeTypeImage, config: map[string]string{"volatile.rootfs.size": "15GiB"}}, err: nil, size: "20GiB", }, { // Check returned size is empty when the container volume/pool doesn't specify a size and // the pool is not block backed and the volume is container & fs. vol: Volume{driver: &nonBlockBackedDriver, volType: VolumeTypeContainer, contentType: ContentTypeFS, config: map[string]string{}}, srcVol: Volume{volType: VolumeTypeImage, config: map[string]string{"volatile.rootfs.size": "15GiB"}}, err: nil, size: "", }, { // Check returned size is empty when the container volume/pool doesn't specify a size and // the pool is not block backed and the volume is VM & block. vol: Volume{driver: &nonBlockBackedDriver, volType: VolumeTypeVM, contentType: ContentTypeBlock, config: map[string]string{}}, srcVol: Volume{volType: VolumeTypeImage, config: map[string]string{"volatile.rootfs.size": "15GiB"}}, err: nil, size: "15GiB", }, { // Check returned size is source size when the VM volume/pool doesn't specify a size and // the pool is block backed, and the source size is larger than default block disk size. vol: Volume{driver: &blockBackedDriver, volType: VolumeTypeVM, config: map[string]string{}}, srcVol: Volume{volType: VolumeTypeImage, config: map[string]string{"volatile.rootfs.size": "15GiB"}}, err: nil, size: "15GiB", }, { // Check returned size is default block disk size when the VM volume/pool doesn't specify a // size and the pool is block backed, and the source size is smaller than default block // disk size. vol: Volume{driver: &blockBackedDriver, volType: VolumeTypeVM, config: map[string]string{}}, srcVol: Volume{volType: VolumeTypeImage, config: map[string]string{"volatile.rootfs.size": "5GiB"}}, err: nil, size: DefaultBlockSize, }, { // Check volume's size is used when VM filesystem volume is supplied with image source. vol: Volume{driver: &nonBlockBackedDriver, volType: VolumeTypeVM, contentType: ContentTypeFS, config: map[string]string{"size": "50MiB"}}, srcVol: Volume{volType: VolumeTypeImage}, err: nil, size: "50MiB", }, } for _, test := range tests { size, err := test.vol.ConfigSizeFromSource(test.srcVol) assert.Equal(t, test.size, size) assert.Equal(t, test.err, err) } } incus-7.0.0/internal/server/storage/errors.go000066400000000000000000000007501517523235500213020ustar00rootroot00000000000000package storage import ( "errors" ) // ErrNilValue is the "Nil value provided" error. var ErrNilValue = errors.New("Nil value provided") // ErrBackupSnapshotsMismatch is the "Backup snapshots mismatch" error. var ErrBackupSnapshotsMismatch = errors.New("Backup snapshots mismatch") // ErrVolumeNotAttachedToRunningInstance is the "Volume is not attached to running instance" error. var ErrVolumeNotAttachedToRunningInstance = errors.New("Volume is not attached to running instance") incus-7.0.0/internal/server/storage/linstor/000077500000000000000000000000001517523235500211275ustar00rootroot00000000000000incus-7.0.0/internal/server/storage/linstor/linstor.go000066400000000000000000000076071517523235500231620ustar00rootroot00000000000000package linstor import ( "context" "crypto/tls" "crypto/x509" "encoding/pem" "fmt" "net" "net/http" "net/url" linstorClient "github.com/LINBIT/golinstor/client" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) // Client represents an HTTP Linstor client. type Client struct { Client *linstorClient.Client } // linstorLogger wraps the Incus logger to use it with golinstor. type linstorLogger struct{} // wrapLogger wraps a logger for golinstor. func wrapLogger(f func(string, ...logger.Ctx), msg string, args ...any) { f("LINSTOR: " + fmt.Sprintf(msg, args...)) } // Errorf wraps logger.Error for golinstor. func (linstorLogger) Errorf(str string, args ...any) { wrapLogger(logger.Error, str, args...) } // Infof wraps logger.Info for golinstor. func (linstorLogger) Infof(str string, args ...any) { wrapLogger(logger.Info, str, args...) } // Debugf wraps logger.Debug for golinstor. func (linstorLogger) Debugf(str string, args ...any) { wrapLogger(logger.Debug, str, args...) } // Warnf wraps logger.Warn for golinstor. func (linstorLogger) Warnf(str string, args ...any) { wrapLogger(logger.Warn, str, args...) } // NewClient initializes a new Linstor client. func NewClient(controllerConnection, sslCACert, sslClientCert, sslClientKey string) (*Client, error) { logger.Info("Creating new Linstor client", logger.Ctx{"controllerConnection": controllerConnection}) // Configure the client HTTP transport. httpTransport := &http.Transport{} // If a CA cert is provided, use it to validate the server certificates. if sslCACert != "" { rootCAs := x509.NewCertPool() certBlock, _ := pem.Decode([]byte(sslCACert)) caCert, err := x509.ParseCertificate(certBlock.Bytes) if err != nil { return nil, fmt.Errorf("Failed to create Linstor client: %w", err) } rootCAs.AddCert(caCert) httpTransport.TLSClientConfig = &tls.Config{RootCAs: rootCAs} } // If a client certificate and key pair is provided, submit it to the server. if sslClientCert != "" && sslClientKey != "" { clientCert, err := tls.X509KeyPair([]byte(sslClientCert), []byte(sslClientKey)) if err != nil { return nil, fmt.Errorf("Failed to create Linstor client: %w", err) } httpTransport.TLSClientConfig.Certificates = []tls.Certificate{clientCert} } // Setup the Linstor client. httpClient := &http.Client{Transport: httpTransport} parseConnection := func(connection string) (*url.URL, error) { u, err := url.Parse(connection) if err != nil { _, _, err := net.SplitHostPort(connection) if err != nil { // Assume we only got an IP address or hostname. return url.Parse("http://" + net.JoinHostPort(connection, "3370")) } // Assume we got an IP address and port combination. return url.Parse("http://" + connection) } // Handle missing scheme. if u.Scheme == "" { u.Scheme = "http" } // Handle missing path. if u.Host == "" { u.Host = u.Path u.Path = "" } // Add in the port if missing. _, _, err = net.SplitHostPort(u.Host) if err != nil { u.Host = net.JoinHostPort(u.Host, "3370") } return u, nil } controllerURLs := []*url.URL{} for _, connection := range util.SplitNTrimSpace(controllerConnection, ",", -1, true) { u, err := parseConnection(connection) if err != nil { return nil, fmt.Errorf("Bad URL: %w", err) } controllerURLs = append(controllerURLs, u) } c, err := linstorClient.NewClient(linstorClient.BaseURL(controllerURLs...), linstorClient.HTTPClient(httpClient), linstorClient.Log(linstorLogger{})) if err != nil { return nil, fmt.Errorf("Failed to create Linstor client: %w", err) } // Get the controller version to check connection. ctx := context.TODO() version, err := c.Controller.GetVersion(ctx) if err != nil { return nil, fmt.Errorf("Failed to create Linstor client: %w", err) } logger.Info("Connected to Linstor Controller", logger.Ctx{"version": version}) return &Client{Client: c}, nil } incus-7.0.0/internal/server/storage/memorypipe/000077500000000000000000000000001517523235500216235ustar00rootroot00000000000000incus-7.0.0/internal/server/storage/memorypipe/memory_pipe.go000066400000000000000000000063671517523235500245130ustar00rootroot00000000000000package memorypipe import ( "context" "io" ) const bufferSize = 10 // msg represents an internal structure sent between the pipes. type msg struct { data []byte err error } // pipe provides a bidirectional pipe compatible with io.ReadWriteCloser interface. // Note, however, that it does not behave exactly how one would expect an io.ReadWriteCloser to // behave. Specifically the Close() function does not close the pipe, but instead delivers an io.EOF // error to the next reader. After which it can be read again to receive new data. This means the // pipe can be closed multiple times. Each time it indicates that one particular session has ended. // The reason for this is to emulate the WebsocketIO's behaviour by allowing a single persistent // connection to be used for multiple sessions. type pipe struct { ch chan msg ctx context.Context otherEnd *pipe msgRemainder []byte } // Read reads from the pipe into p. Returns number of bytes read and any errors. func (p *pipe) Read(b []byte) (int, error) { if p.msgRemainder != nil { n := copy(b, p.msgRemainder) if len(p.msgRemainder) > n { tmpBuf := make([]byte, len(p.msgRemainder)-n) copy(tmpBuf, p.msgRemainder[n:]) p.msgRemainder = tmpBuf } else { p.msgRemainder = nil } return n, nil } select { case msg := <-p.ch: if msg.err == io.EOF { return 0, msg.err } n := copy(b, msg.data) // Store the remainder of the message for next Read. if len(msg.data) > n { p.msgRemainder = make([]byte, len(msg.data)-n) copy(p.msgRemainder, msg.data[n:]) } else { p.msgRemainder = nil } return n, msg.err case <-p.ctx.Done(): return 0, p.ctx.Err() } } // Write writes to the pipe from p. Returns number of bytes written and any errors. func (p *pipe) Write(b []byte) (int, error) { msg := msg{ data: make([]byte, len(b)), err: nil, } // Create copy of b in case it is modified externally. copy(msg.data, b) select { case p.otherEnd.ch <- msg: // Sent msg to the other side's Read function. return len(msg.data), msg.err case <-p.ctx.Done(): return 0, p.ctx.Err() } } // Close is unusual in that it doesn't actually close the pipe. Instead it sends an io.EOF error // to the other side's Read function. This is so the other side can detect that a session has ended. // Each call to Close will indicate to the other side that a session has ended, whilst allowing the // reuse of a single persistent pipe for multiple sessions. func (p *pipe) Close() error { msg := msg{ data: nil, err: io.EOF, // Indicates to the other side's Read function that session has ended. } select { case p.otherEnd.ch <- msg: // Sent msg to the other side's Read function. return nil case <-p.ctx.Done(): return p.ctx.Err() } } // NewPipePair returns a pair of io.ReadWriterCloser pipes that are connected together such that // writes to one will appear as reads on the other and vice versa. Calling Close() on one end will // indicate to the other end that the session has ended. func NewPipePair(ctx context.Context) (io.ReadWriteCloser, io.ReadWriteCloser) { aEnd := &pipe{ ch: make(chan msg, bufferSize), ctx: ctx, } bEnd := &pipe{ ch: make(chan msg, bufferSize), ctx: ctx, } aEnd.otherEnd = bEnd bEnd.otherEnd = aEnd return aEnd, bEnd } incus-7.0.0/internal/server/storage/memorypipe/memory_pipe_test.go000066400000000000000000000037171517523235500255460ustar00rootroot00000000000000package memorypipe import ( "bytes" "context" "testing" ) // Test memorypipe. func TestMemoryPipe(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() aEnd, bEnd := NewPipePair(ctx) // Create four byte buffer and write it to pipe. sendMsg := []byte{1, 2, 3, 4} n, err := aEnd.Write(sendMsg) if err != nil { t.Errorf("Unexpected write error: %v", err) } if n != len(sendMsg) { t.Errorf("Unexpected write length: %d, expected: %d", n, len(sendMsg)) } // Create two byte buffer and try to read from pipe. We should get half of the sent message. recvMsg := make([]byte, 2) n, err = bEnd.Read(recvMsg) if err != nil { t.Errorf("Unexpected read error: %v", err) } if n != len(recvMsg) { t.Errorf("Unexpected read length: %d, expected: %d", n, len(recvMsg)) } if !bytes.Equal(recvMsg, sendMsg[:2]) { t.Errorf("Unexpected read contents: %v, expected: %v", recvMsg, sendMsg[:2]) } // Now read again into the two byte buffer, we should get the remainder of the send message. n, err = bEnd.Read(recvMsg) if err != nil { t.Errorf("Unexpected read error: %v", err) } if n != len(recvMsg) { t.Errorf("Unexpected read length: %d, expected: %d", n, len(recvMsg)) } if !bytes.Equal(recvMsg, sendMsg[2:]) { t.Errorf("Unexpected read contents: %v, expected: %v", recvMsg, sendMsg[2:]) } // Send a new message. sendMsg = []byte{1, 2, 3, 4, 5} n, err = aEnd.Write(sendMsg) if err != nil { t.Errorf("Unexpected write error: %v", err) } if n != len(sendMsg) { t.Errorf("Unexpected write length: %d, expected: %d", n, len(sendMsg)) } // Read entire message this time. recvMsg = make([]byte, len(sendMsg)) n, err = bEnd.Read(recvMsg) if err != nil { t.Errorf("Unexpected read error: %v", err) } if n != len(recvMsg) { t.Errorf("Unexpected read length: %d, expected: %d", n, len(recvMsg)) } if !bytes.Equal(recvMsg, sendMsg) { t.Errorf("Unexpected read contents: %v, expected: %v", recvMsg, sendMsg) } } incus-7.0.0/internal/server/storage/pool_interface.go000066400000000000000000000235341517523235500227640ustar00rootroot00000000000000package storage import ( "io" "net" "net/url" "time" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/server/backup" backupConfig "github.com/lxc/incus/v7/internal/server/backup/config" "github.com/lxc/incus/v7/internal/server/cluster/request" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/revert" ) // VolumeUsage contains the used and total size of a volume. type VolumeUsage struct { Used int64 Total int64 } // MountInfo represents info about the result of a mount operation. type MountInfo struct { DiskPath string // The location of the block disk (if supported). BackingPath []string // The location of the block disk (backing disk for qcow2). PostHooks []func(inst instance.Instance) error // Hooks to be called following a mount. } // Type represents an Incus storage pool type. type Type interface { Validate(config map[string]string) error } // Pool represents an Incus storage pool. type Pool interface { Type // Pool. ID() int64 Name() string Driver() drivers.Driver Description() string Status() string LocalStatus() string ToAPI() api.StoragePool GetResources() (*api.ResourcesStoragePool, error) IsUsed() (bool, error) Delete(clientType request.ClientType, op *operations.Operation) error Update(clientType request.ClientType, newDesc string, newConfig map[string]string, op *operations.Operation) error Create(clientType request.ClientType, op *operations.Operation) error Mount() (bool, error) Unmount() (bool, error) ApplyPatch(name string) error GetVolume(volumeType drivers.VolumeType, contentType drivers.ContentType, name string, config map[string]string) drivers.Volume // Instances. CreateInstance(inst instance.Instance, op *operations.Operation) error CreateInstanceFromCopy(inst instance.Instance, src instance.Instance, snapshots bool, allowInconsistent bool, op *operations.Operation) error CreateInstanceFromImage(inst instance.Instance, fingerprint string, op *operations.Operation) error CreateInstanceFromMigration(inst instance.Instance, conn io.ReadWriteCloser, args migration.VolumeTargetArgs, op *operations.Operation) error RenameInstance(inst instance.Instance, newName string, op *operations.Operation) error DeleteInstance(inst instance.Instance, op *operations.Operation) error UpdateInstance(inst instance.Instance, newDesc string, newConfig map[string]string, op *operations.Operation) error UpdateInstanceBackupFile(inst instance.Instance, snapshots bool, op *operations.Operation) error GenerateInstanceBackupConfig(inst instance.Instance, snapshots bool, dependentVolumes bool, op *operations.Operation) (*backupConfig.Config, error) CheckInstanceBackupFileSnapshots(backupConf *backupConfig.Config, projectName string, deleteMissing bool, op *operations.Operation) ([]*api.InstanceSnapshot, error) ImportInstance(inst instance.Instance, poolVol *backupConfig.Config, op *operations.Operation) (revert.Hook, error) CleanupInstancePaths(inst instance.Instance, op *operations.Operation) error MigrateInstance(inst instance.Instance, conn io.ReadWriteCloser, args *migration.VolumeSourceArgs, op *operations.Operation) error RefreshInstance(inst instance.Instance, src instance.Instance, srcSnapshots []instance.Instance, allowInconsistent bool, op *operations.Operation) error GetInstanceUsage(inst instance.Instance) (*VolumeUsage, error) SetInstanceQuota(inst instance.Instance, size string, vmStateSize string, op *operations.Operation) error MountInstance(inst instance.Instance, op *operations.Operation) (*MountInfo, error) UnmountInstance(inst instance.Instance, op *operations.Operation) error // Instance snapshots. CanRestoreInstanceSnapshot(inst instance.Instance, src instance.Instance) error CreateInstanceSnapshot(inst instance.Instance, src instance.Instance, op *operations.Operation) error RenameInstanceSnapshot(inst instance.Instance, newName string, op *operations.Operation) error DeleteInstanceSnapshot(inst instance.Instance, op *operations.Operation) error RestoreInstanceSnapshot(inst instance.Instance, src instance.Instance, op *operations.Operation) error MountInstanceSnapshot(inst instance.Instance, op *operations.Operation) (*MountInfo, error) UnmountInstanceSnapshot(inst instance.Instance, op *operations.Operation) error UpdateInstanceSnapshot(inst instance.Instance, newDesc string, newConfig map[string]string, op *operations.Operation) error // Instance backups. BackupInstance(inst instance.Instance, tarWriter *instancewriter.InstanceTarWriter, optimized bool, snapshots bool, dependentVolumes bool, op *operations.Operation) error CreateInstanceFromBackup(srcBackup backup.Info, srcData io.ReadSeeker, op *operations.Operation) (func(instance.Instance) error, revert.Hook, error) GetInstanceNBD(inst instance.Instance, writable bool) (net.Conn, func(), error) // Images. EnsureImage(fingerprint string, op *operations.Operation) error DeleteImage(fingerprint string, op *operations.Operation) error UpdateImage(fingerprint string, newDesc string, newConfig map[string]string, op *operations.Operation) error // Buckets. CreateBucket(projectName string, bucket api.StorageBucketsPost, op *operations.Operation) error UpdateBucket(projectName string, bucketName string, bucket api.StorageBucketPut, op *operations.Operation) error DeleteBucket(projectName string, bucketName string, op *operations.Operation) error ImportBucket(projectName string, poolVol *backupConfig.Config, op *operations.Operation) (revert.Hook, error) CreateBucketKey(projectName string, bucketName string, key api.StorageBucketKeysPost, op *operations.Operation) (*api.StorageBucketKey, error) UpdateBucketKey(projectName string, bucketName string, keyName string, key api.StorageBucketKeyPut, op *operations.Operation) error DeleteBucketKey(projectName string, bucketName string, keyName string, op *operations.Operation) error MountLocalBucket(projectName string, bucketName string, op *operations.Operation) (string, func() error, error) GetBucketURL(bucketName string) *url.URL GenerateBucketBackupConfig(projectName string, bucketName string, op *operations.Operation) (*backupConfig.Config, error) BackupBucket(projectName string, bucketName string, tarWriter *instancewriter.InstanceTarWriter, op *operations.Operation) error CreateBucketFromBackup(srcBackup backup.Info, srcData io.ReadSeeker, op *operations.Operation) error // Custom volumes. CreateCustomVolume(projectName string, volName string, desc string, config map[string]string, contentType drivers.ContentType, op *operations.Operation) error CreateCustomVolumeFromCopy(projectName string, srcProjectName string, volName, desc string, config map[string]string, srcPoolName, srcVolName string, snapshots bool, op *operations.Operation) error UpdateCustomVolume(projectName string, volName string, newDesc string, newConfig map[string]string, op *operations.Operation) error RenameCustomVolume(projectName string, volName string, newVolName string, op *operations.Operation) error DeleteCustomVolume(projectName string, volName string, op *operations.Operation) error GetCustomVolumeDisk(projectName string, volName string) (string, error) GetCustomVolumeUsage(projectName string, volName string) (*VolumeUsage, error) MountCustomVolume(projectName string, volName string, op *operations.Operation) (*MountInfo, error) UnmountCustomVolume(projectName string, volName string, op *operations.Operation) (bool, error) ImportCustomVolume(projectName string, poolVol *backupConfig.Config, op *operations.Operation) (revert.Hook, error) RefreshCustomVolume(projectName string, srcProjectName string, volName, desc string, config map[string]string, srcPoolName, srcVolName string, snapshots bool, excludeOlder bool, op *operations.Operation) error GenerateCustomVolumeBackupConfig(projectName string, volName string, snapshots bool, op *operations.Operation) (*backupConfig.Config, error) CreateCustomVolumeFromISO(projectName string, volName string, srcData io.ReadSeeker, size int64, op *operations.Operation) error // Custom volume snapshots. CreateCustomVolumeSnapshot(projectName string, volName string, newSnapshotName string, newExpiryDate time.Time, instanceStateful bool, op *operations.Operation) error RenameCustomVolumeSnapshot(projectName string, volName string, newSnapshotName string, op *operations.Operation) error DeleteCustomVolumeSnapshot(projectName string, volName string, op *operations.Operation) error UpdateCustomVolumeSnapshot(projectName string, volName string, newDesc string, newConfig map[string]string, newExpiryDate time.Time, op *operations.Operation) error RestoreCustomVolume(projectName string, volName string, snapshotName string, op *operations.Operation) error // Custom volume migration. MigrationTypes(contentType drivers.ContentType, refresh bool, copySnapshots bool, clusterMove bool, storageMove bool) []migration.Type CreateCustomVolumeFromMigration(projectName string, conn io.ReadWriteCloser, args migration.VolumeTargetArgs, op *operations.Operation) error MigrateCustomVolume(projectName string, conn io.ReadWriteCloser, args *migration.VolumeSourceArgs, op *operations.Operation) error // Custom volume backups. BackupCustomVolume(projectName string, volName string, writer instancewriter.InstanceWriter, basePrefix string, optimized bool, snapshots bool, op *operations.Operation) error CreateCustomVolumeFromBackup(srcBackup backup.Info, srcData io.ReadSeeker, basePrefix string, op *operations.Operation) error GetCustomVolumeNBD(projectName string, volName string, writable bool) (net.Conn, func(), error) // Storage volume recovery. ListUnknownVolumes(op *operations.Operation) (map[string][]*backupConfig.Config, error) } incus-7.0.0/internal/server/storage/pool_load.go000066400000000000000000000213521517523235500217370ustar00rootroot00000000000000package storage import ( "context" "fmt" "slices" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" ) // PoolIDTemporary is used to indicate a temporary pool instance that is not in the database. const PoolIDTemporary = -1 // volIDFuncMake returns a function that can be supplied to the underlying storage drivers allowing // them to lookup the volume ID for a specific volume type and volume name. This function is tied // to the Pool ID that it is generated for, meaning the storage drivers do not need to know the ID // of the pool they belong to, or do they need access to the database. func volIDFuncMake(state *state.State, poolID int64) func(volType drivers.VolumeType, volName string) (int64, error) { // Return a function to retrieve a volume ID for a volume Name for use in driver. return func(volType drivers.VolumeType, volName string) (int64, error) { volTypeID, err := VolumeTypeToDBType(volType) if err != nil { return -1, err } // It is possible for the project name to be encoded into the volume name in the // format _. However not all volume types currently use this // encoding format, so if there is no underscore in the volume name then we assume // the project is default. projectName := api.ProjectDefaultName // Currently only Containers, VMs and custom volumes support project level volumes. // This means that other volume types may have underscores in their names that don't // indicate the project name. if volType == drivers.VolumeTypeContainer || volType == drivers.VolumeTypeVM { projectName, volName = project.InstanceParts(volName) } else if volType == drivers.VolumeTypeCustom { projectName, volName = project.StorageVolumeParts(volName) } // Get the storage volume. var dbVolume *db.StorageVolume err = state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, poolID, projectName, volTypeID, volName, true) return err }) if err != nil { return -1, err } return dbVolume.ID, nil } } // commonRules returns a set of common validators. func commonRules() *drivers.Validators { return &drivers.Validators{ PoolRules: validatePoolCommonRules, VolumeRules: validateVolumeCommonRules, } } // NewTemporary instantiates a temporary pool from config supplied and returns a Pool interface. // Not all functionality will be available due to the lack of Pool ID. // If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned. func NewTemporary(state *state.State, info *api.StoragePool) (Pool, error) { // Handle mock requests. if state.OS.MockMode { pool := mockBackend{} pool.name = info.Name pool.state = state pool.logger = logger.AddContext(logger.Ctx{"driver": "mock", "pool": pool.name}) driver, err := drivers.Load(state, "mock", "", nil, pool.logger, nil, nil) if err != nil { return nil, err } pool.driver = driver return &pool, nil } var poolID int64 = PoolIDTemporary // Temporary as not in DB. Not all functionality will be available. // Ensure a config map exists. if info.Config == nil { info.Config = map[string]string{} } logger := logger.AddContext(logger.Ctx{"driver": info.Driver, "pool": info.Name}) // Load the storage driver. driver, err := drivers.Load(state, info.Driver, info.Name, info.Config, logger, volIDFuncMake(state, poolID), commonRules()) if err != nil { return nil, err } // Setup the pool struct. pool := backend{} pool.driver = driver pool.id = poolID pool.db = *info pool.name = info.Name pool.state = state pool.logger = logger pool.nodes = nil // TODO support clustering. return &pool, nil } // LoadByType loads a network by driver type. func LoadByType(state *state.State, driverType string) (Type, error) { logger := logger.AddContext(logger.Ctx{"driver": driverType}) driver, err := drivers.Load(state, driverType, "", nil, logger, nil, commonRules()) if err != nil { return nil, err } // Setup the pool struct. pool := backend{} pool.state = state pool.driver = driver pool.id = PoolIDTemporary pool.logger = logger return &pool, nil } // LoadByRecord instantiates a pool from its record and returns a Pool interface. // If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned. func LoadByRecord(s *state.State, poolID int64, poolInfo api.StoragePool, poolMembers map[int64]db.StoragePoolNode) (Pool, error) { // Ensure a config map exists. if poolInfo.Config == nil { poolInfo.Config = map[string]string{} } logger := logger.AddContext(logger.Ctx{"driver": poolInfo.Driver, "pool": poolInfo.Name}) // Load the storage driver. driver, err := drivers.Load(s, poolInfo.Driver, poolInfo.Name, poolInfo.Config, logger, volIDFuncMake(s, poolID), commonRules()) if err != nil { return nil, err } // Setup the pool struct. pool := backend{} pool.driver = driver pool.id = poolID pool.db = poolInfo pool.name = poolInfo.Name pool.state = s pool.logger = logger pool.nodes = poolMembers return &pool, nil } // LoadByName retrieves the pool from the database by its name and returns a Pool interface. // If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned. func LoadByName(s *state.State, name string) (Pool, error) { // Handle mock requests. if s.OS.MockMode { pool := mockBackend{} pool.name = name pool.state = s pool.logger = logger.AddContext(logger.Ctx{"driver": "mock", "pool": pool.name}) driver, err := drivers.Load(s, "mock", "", nil, pool.logger, nil, nil) if err != nil { return nil, err } pool.driver = driver return &pool, nil } var poolID int64 var dbPool *api.StoragePool var poolNodes map[int64]db.StoragePoolNode err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Load the database record. poolID, dbPool, poolNodes, err = tx.GetStoragePoolInAnyState(ctx, name) return err }) if err != nil { return nil, err } return LoadByRecord(s, poolID, *dbPool, poolNodes) } // LoadByInstance retrieves the pool from the database using the instance's pool. // If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned. If the pool's // driver does not support the instance's type then drivers.ErrNotSupported is returned. func LoadByInstance(s *state.State, inst instance.Instance) (Pool, error) { poolName, err := inst.StoragePool() if err != nil { return nil, fmt.Errorf("Failed getting instance storage pool name: %w", err) } pool, err := LoadByName(s, poolName) if err != nil { return nil, fmt.Errorf("Failed loading storage pool %q: %w", poolName, err) } volType, err := InstanceTypeToVolumeType(inst.Type()) if err != nil { return nil, err } if slices.Contains(pool.Driver().Info().VolumeTypes, volType) { return pool, nil } // Return drivers not supported error for consistency with predefined errors returned by // LoadByName (which can return drivers.ErrUnknownDriver). return nil, drivers.ErrNotSupported } // IsAvailable checks if a pool is available. func IsAvailable(poolName string) bool { unavailablePoolsMu.Lock() defer unavailablePoolsMu.Unlock() _, found := unavailablePools[poolName] return !found } // Patch applies specified patch to all storage pools. // All storage pools must be available locally before any storage pools are patched. func Patch(s *state.State, patchName string) error { unavailablePoolsMu.Lock() if len(unavailablePools) > 0 { unavailablePoolNames := make([]string, 0, len(unavailablePools)) for unavailablePoolName := range unavailablePools { unavailablePoolNames = append(unavailablePoolNames, unavailablePoolName) } unavailablePoolsMu.Unlock() return fmt.Errorf("Unavailable storage pools: %v", unavailablePoolNames) } unavailablePoolsMu.Unlock() var pools []string err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { var err error // Load all the pools. pools, err = tx.GetStoragePoolNames(ctx) return err }) if err != nil { if response.IsNotFoundError(err) { return nil } return fmt.Errorf("Failed loading storage pool names: %w", err) } for _, poolName := range pools { pool, err := LoadByName(s, poolName) if err != nil { return fmt.Errorf("Failed loading storage pool %q: %w", poolName, err) } err = pool.ApplyPatch(patchName) if err != nil { return fmt.Errorf("Failed applying patch to pool %q: %w", poolName, err) } } return nil } incus-7.0.0/internal/server/storage/quota/000077500000000000000000000000001517523235500205665ustar00rootroot00000000000000incus-7.0.0/internal/server/storage/quota/cgo.go000066400000000000000000000006601517523235500216670ustar00rootroot00000000000000//go:build linux && cgo package quota // #cgo CFLAGS: -std=gnu11 -Wvla -Werror -fvisibility=hidden -Winit-self // #cgo CFLAGS: -Wformat=2 -Wshadow -Wendif-labels -fasynchronous-unwind-tables // #cgo CFLAGS: -pipe --param=ssp-buffer-size=4 -g -Wunused // #cgo CFLAGS: -Werror=implicit-function-declaration // #cgo CFLAGS: -Werror=return-type -Wendif-labels -Werror=overflow // #cgo CFLAGS: -Wnested-externs -fexceptions import "C" incus-7.0.0/internal/server/storage/quota/projectquota.go000066400000000000000000000155131517523235500236420ustar00rootroot00000000000000package quota /* #include #include #include #include #include #include #include #include #include #include #include #include #ifndef FS_XFLAG_PROJINHERIT struct fsxattr { __u32 fsx_xflags; __u32 fsx_extsize; __u32 fsx_nextents; __u32 fsx_projid; unsigned char fsx_pad[12]; }; #define FS_XFLAG_PROJINHERIT 0x00000200 #endif #ifndef QIF_DQBLKSIZE_BITS struct if_dqinfo { __u64 dqi_bgrace; __u64 dqi_igrace; __u32 dqi_flags; __u32 dqi_valid; }; struct if_dqblk { __u64 dqb_bhardlimit; __u64 dqb_bsoftlimit; __u64 dqb_curspace; __u64 dqb_ihardlimit; __u64 dqb_isoftlimit; __u64 dqb_curinodes; __u64 dqb_btime; __u64 dqb_itime; __u32 dqb_valid; }; #define QIF_DQBLKSIZE_BITS 10 #endif #ifndef FS_IOC_FSGETXATTR #define FS_IOC_FSGETXATTR _IOR ('X', 31, struct fsxattr) #endif #ifndef FS_IOC_FSSETXATTR #define FS_IOC_FSSETXATTR _IOW ('X', 32, struct fsxattr) #endif #ifndef PRJQUOTA #define PRJQUOTA 2 #endif int quota_supported(char *dev_path) { struct if_dqinfo dqinfo; return quotactl(QCMD(Q_GETINFO, PRJQUOTA), dev_path, 0, (caddr_t)&dqinfo); } int64_t quota_get_usage(char *dev_path, uint32_t id) { struct if_dqblk quota; if (quotactl(QCMD(Q_GETQUOTA, PRJQUOTA), dev_path, id, (caddr_t)"a) < 0) { return -1; } return quota.dqb_curspace; } int quota_set(char *dev_path, uint32_t id, uint64_t hard_bytes) { struct if_dqblk quota; fs_disk_quota_t xfsquota; if (quotactl(QCMD(Q_GETQUOTA, PRJQUOTA), dev_path, id, (caddr_t)"a) < 0) { if (hard_bytes == 0 && errno == ENOENT) return 0; return -1; } quota.dqb_bhardlimit = hard_bytes; if (quotactl(QCMD(Q_SETQUOTA, PRJQUOTA), dev_path, id, (caddr_t)"a) < 0) { xfsquota.d_version = FS_DQUOT_VERSION; xfsquota.d_id = id; xfsquota.d_flags = FS_PROJ_QUOTA; xfsquota.d_fieldmask = FS_DQ_BHARD; xfsquota.d_blk_hardlimit = hard_bytes * 1024 / 512; if (quotactl(QCMD(Q_XSETQLIM, PRJQUOTA), dev_path, id, (caddr_t)&xfsquota) < 0) { return -1; } } return 0; } int quota_set_path(char *path, uint32_t id, bool inherit) { struct fsxattr attr; int fd; int ret; fd = open(path, O_RDONLY | O_CLOEXEC); if (fd < 0) return -1; ret = ioctl(fd, FS_IOC_FSGETXATTR, &attr); if (ret < 0) { close(fd); return -1; } if (inherit) { attr.fsx_xflags |= FS_XFLAG_PROJINHERIT; } attr.fsx_projid = id; ret = ioctl(fd, FS_IOC_FSSETXATTR, &attr); if (ret < 0) { close(fd); return -1; } close(fd); return 0; } int32_t quota_get_path(char *path) { struct fsxattr attr; int fd; int ret; fd = open(path, O_RDONLY | O_CLOEXEC); if (fd < 0) return -1; ret = ioctl(fd, FS_IOC_FSGETXATTR, &attr); if (ret < 0) { close(fd); return -1; } close(fd); return attr.fsx_projid; } */ import "C" import ( "bufio" "errors" "fmt" "io/fs" "os" "path/filepath" "strings" "unsafe" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/shared/util" ) var errNoDevice = errors.New("Couldn't find backing device for mountpoint") func devForPath(path string) (string, error) { // Get major/minor var stat unix.Stat_t err := unix.Lstat(path, &stat) if err != nil { return "", err } devMajor := unix.Major(uint64(stat.Dev)) devMinor := unix.Minor(uint64(stat.Dev)) // Parse mountinfo for it mountinfo, err := os.Open("/proc/self/mountinfo") if err != nil { return "", err } defer func() { _ = mountinfo.Close() }() scanner := bufio.NewScanner(mountinfo) for scanner.Scan() { line := scanner.Text() tokens := strings.Fields(line) if len(tokens) < 5 { continue } if tokens[2] == fmt.Sprintf("%d:%d", devMajor, devMinor) { if util.PathExists(tokens[len(tokens)-2]) { return tokens[len(tokens)-2], nil } } } return "", errNoDevice } // Supported check if the given path supports project quotas. func Supported(path string) (bool, error) { // Get the backing device devPath, err := devForPath(path) if err != nil { return false, err } // Call quotactl through CGo cDevPath := C.CString(devPath) defer C.free(unsafe.Pointer(cDevPath)) return C.quota_supported(cDevPath) == 0, nil } // GetProject returns the project quota ID for the given path. func GetProject(path string) (uint32, error) { // Call ioctl through CGo cPath := C.CString(path) defer C.free(unsafe.Pointer(cPath)) id := C.quota_get_path(cPath) if id < 0 { return 0, fmt.Errorf("Failed to get project from %q", path) } return uint32(id), nil } // SetProject recursively sets the project quota ID (and project inherit flag on directories) for the given path. func SetProject(path string, id uint32) error { err := filepath.Walk(path, func(filePath string, info os.FileInfo, err error) error { if err != nil { if errors.Is(err, fs.ErrNotExist) { return nil } return err } inherit := false if info.IsDir() { inherit = true // Only can set FS_XFLAG_PROJINHERIT on directories. } else if !info.Mode().IsRegular() { // Cannot set project ID on non-regular files after file creation. In fact trying to set // project ID on some file types just blocks forever (such as pipe files). // So skip them as they don't take up disk space anyway. return nil } // Call ioctl through CGo. cPath := C.CString(filePath) defer C.free(unsafe.Pointer(cPath)) if C.quota_set_path(cPath, C.uint32_t(id), C.bool(inherit)) != 0 { return fmt.Errorf(`Failed to set project ID "%d" on %q (inherit %t)`, id, filePath, inherit) } return nil }) return err } // DeleteProject unsets the project id from the path and clears the quota for the project ID. func DeleteProject(path string, id uint32) error { // Unset the project from the path. err := SetProject(path, 0) if err != nil { return err } // Unset the quota on the project. err = SetProjectQuota(path, id, 0) if err != nil { return err } return nil } // GetProjectUsage returns the current consumption. func GetProjectUsage(path string, id uint32) (int64, error) { // Get the backing device. devPath, err := devForPath(path) if err != nil { return -1, err } // Call quotactl through CGo. cDevPath := C.CString(devPath) defer C.free(unsafe.Pointer(cDevPath)) size := C.quota_get_usage(cDevPath, C.uint32_t(id)) if size < 0 { return -1, fmt.Errorf(`Failed to get project consumption for ID "%d" on %q`, id, devPath) } return int64(size), nil } // SetProjectQuota sets the quota on the project ID. func SetProjectQuota(path string, id uint32, bytes int64) error { // Get the backing device devPath, err := devForPath(path) if err != nil { return err } // Call quotactl through CGo cDevPath := C.CString(devPath) defer C.free(unsafe.Pointer(cDevPath)) if C.quota_set(cDevPath, C.uint32_t(id), C.uint64_t(bytes/1024)) != 0 { return fmt.Errorf(`Failed to set project quota for ID "%d" on %q`, id, devPath) } return nil } incus-7.0.0/internal/server/storage/s3/000077500000000000000000000000001517523235500177625ustar00rootroot00000000000000incus-7.0.0/internal/server/storage/s3/headers.go000066400000000000000000000030241517523235500217230ustar00rootroot00000000000000package s3 import ( "strings" ) // AuthorizationHeaderAccessKey attempts to extract the (unverified) access key from the Authorization header. func AuthorizationHeaderAccessKey(authorizationHeader string) string { // Parses an Authorization header as below, trying to extract the access key "PRL470D7Q93X1ZA1L82X". // AWS4-HMAC-SHA256 Credential=PRL470D7Q93X1ZA1L82X/20220825/US/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=d8fdaf67c5072d4ff7ac56e4529e66fb08255aaa79193b212cba4670d058fade after, found := strings.CutPrefix(authorizationHeader, "AWS4-HMAC-SHA256") if found { authHeaderParts := strings.Split(strings.TrimSpace(after), ",") if strings.HasPrefix(authHeaderParts[0], "Credential=") { _, after, found = strings.Cut(authHeaderParts[0], "=") if found { credParts := strings.Split(after, "/") credPartsLen := len(credParts) if credPartsLen >= 5 { // The access key can contain / characters, so perform a reverse range search. return strings.Join(credParts[:credPartsLen-4], "/") } } } return "" } after, found = strings.CutPrefix(authorizationHeader, "AWS") if found { // Parses an older Authorization header as below, to extract the access key "PRL470D7Q93X1ZA1L82X". // AWS PRL470D7Q93X1ZA1L82X:dC5GcyRFCyQIr+y9BdpAwBjkOK0= authHeaderParts := strings.Split(strings.TrimSpace(after), ":") authHeaderPartsLen := len(authHeaderParts) if authHeaderPartsLen > 1 { return strings.Join(authHeaderParts[:authHeaderPartsLen-1], ":") } } return "" } incus-7.0.0/internal/server/storage/s3/local/000077500000000000000000000000001517523235500210545ustar00rootroot00000000000000incus-7.0.0/internal/server/storage/s3/local/auth.go000066400000000000000000000277361517523235500223630ustar00rootroot00000000000000package local import ( "bytes" "crypto/hmac" "crypto/sha256" "encoding/hex" "fmt" "io" "net/http" "net/url" "sort" "strconv" "strings" "time" "github.com/lxc/incus/v7/internal/server/storage/s3" ) // Hardcoded body hash sentinels used by AWS clients. const ( unsignedPayload = "UNSIGNED-PAYLOAD" streamingPayload = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" streamingPayloadTrailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" streamingUnsignedTrailer = "STREAMING-UNSIGNED-PAYLOAD-TRAILER" ) // authenticate verifies the SigV4 signature on the request and returns the // matching credential's role on success, or an *s3.Error response on failure. // // On success, r.Body is replaced with a buffered copy if the body's hash had // to be computed for verification. The caller must use r.Body, not the // original. func (s *Server) authenticate(r *http.Request) (Role, *s3.Error) { authHeader := r.Header.Get("Authorization") if authHeader == "" { return "", &s3.Error{Code: s3.ErrorCodeInvalidAccessKeyID, Message: "Missing Authorization header."} } accessKey := s3.AuthorizationHeaderAccessKey(authHeader) if accessKey == "" { return "", &s3.Error{Code: s3.ErrorCodeInvalidAccessKeyID, Message: "Could not extract access key."} } var secret string var role Role for _, c := range s.creds { if c.AccessKey == accessKey { secret = c.SecretKey role = c.Role break } } if secret == "" { return "", &s3.Error{Code: s3.ErrorCodeInvalidAccessKeyID, Message: "Unknown access key."} } parsed, err := parseAuthorizationHeader(authHeader) if err != nil { return "", &s3.Error{Code: s3.ErrorInvalidRequest, Message: err.Error()} } parsed.amzDate = r.Header.Get("X-Amz-Date") if parsed.amzDate == "" { return "", &s3.Error{Code: s3.ErrorInvalidRequest, Message: "Missing X-Amz-Date header."} } // Resolve the body hash for the canonical request. bodyHash := r.Header.Get("X-Amz-Content-Sha256") streaming := false switch bodyHash { case streamingPayload, streamingPayloadTrailer, streamingUnsignedTrailer: streaming = true case "", unsignedPayload: // Use the header value as-is for the canonical request. default: // A signed body hash was provided. The client claims the body // hashes to bodyHash; we must verify that. Buffer the body, hash // it, compare. if r.Body != nil { buf, readErr := io.ReadAll(r.Body) _ = r.Body.Close() if readErr != nil { return "", &s3.Error{Code: s3.ErrorCodeInternalError, Message: "Failed to read request body."} } actual := sha256Hex(buf) if actual != bodyHash { return "", &s3.Error{Code: s3.ErrorInvalidRequest, Message: "Body hash mismatch."} } r.Body = io.NopCloser(bytes.NewReader(buf)) } } if bodyHash == "" { bodyHash = unsignedPayload } canonical := canonicalRequest(r, parsed.signedHeaders, bodyHash) stringToSign := strings.Join([]string{ "AWS4-HMAC-SHA256", parsed.amzDate, parsed.scope, sha256Hex([]byte(canonical)), }, "\n") signingKey := deriveSigningKey(secret, parsed.scopeDate, parsed.scopeRegion, parsed.scopeService) expected := hmacSHA256Hex(signingKey, stringToSign) if !hmac.Equal([]byte(expected), []byte(parsed.signature)) { return "", &s3.Error{Code: s3.ErrorInvalidRequest, Message: "Signature mismatch."} } if streaming && r.Body != nil && hasAWSChunkedEncoding(r) { err := wrapStreamingBody(r, bodyHash, parsed, signingKey) if err != nil { return "", &s3.Error{Code: s3.ErrorInvalidRequest, Message: err.Error()} } } return role, nil } // hasAWSChunkedEncoding returns true if the request advertises an // aws-chunked Content-Encoding. Several layers of clients (notably // aws-sdk-go-v2 with checksum middleware disabled, or callers that // pre-buffer the body) will set X-Amz-Content-Sha256 to a STREAMING-... // sentinel without actually framing the body in aws-chunked. Use the // content encoding as the authoritative signal for whether to invoke the // chunked decoder. func hasAWSChunkedEncoding(r *http.Request) bool { for _, v := range r.Header.Values("Content-Encoding") { for _, part := range strings.Split(v, ",") { if strings.EqualFold(strings.TrimSpace(part), "aws-chunked") { return true } } } return false } // wrapStreamingBody replaces r.Body with a reader that decodes the // aws-chunked envelope. For the signed payload variants, per-chunk // signatures are verified against the rolling previous-signature seeded // from the request seed signature in parsed.signature. // // The decoded content length is taken from x-amz-decoded-content-length and // reflected in r.ContentLength so downstream handlers see the true size of // the payload they will read. func wrapStreamingBody(r *http.Request, bodyHash string, parsed *parsedAuthorization, signingKey []byte) error { var sign *chunkSigningContext switch bodyHash { case streamingPayload: sign = &chunkSigningContext{ algorithm: "AWS4-HMAC-SHA256-PAYLOAD", signingKey: signingKey, amzDate: parsed.amzDate, scope: parsed.scope, prevSignature: parsed.signature, } case streamingPayloadTrailer: sign = &chunkSigningContext{ algorithm: "AWS4-HMAC-SHA256-PAYLOAD-TRAILER", signingKey: signingKey, amzDate: parsed.amzDate, scope: parsed.scope, prevSignature: parsed.signature, } case streamingUnsignedTrailer: sign = nil default: return fmt.Errorf("unsupported streaming body hash %q", bodyHash) } v := r.Header.Get("X-Amz-Decoded-Content-Length") if v != "" { n, err := strconv.ParseInt(v, 10, 64) if err != nil || n < 0 { return fmt.Errorf("invalid X-Amz-Decoded-Content-Length %q", v) } r.ContentLength = n } else { // Without a decoded length we cannot pre-size the body. Mark // it as unknown so downstream code that streams the body // behaves correctly. r.ContentLength = -1 } body := r.Body r.Body = &readerCloser{ Reader: newChunkedReader(body, sign), Closer: body, } return nil } // readerCloser bundles a Reader and an independent Closer so we can wrap // a request body's bytes through a decoder while still closing the // original underlying body. type readerCloser struct { io.Reader io.Closer } // parsedAuthorization holds the components of an AWS4-HMAC-SHA256 header. type parsedAuthorization struct { accessKey string scope string scopeDate string scopeRegion string scopeService string signedHeaders []string signature string amzDate string } // parseAuthorizationHeader parses an "AWS4-HMAC-SHA256 Credential=..., // SignedHeaders=..., Signature=..." header. func parseAuthorizationHeader(h string) (*parsedAuthorization, error) { const prefix = "AWS4-HMAC-SHA256" rest, ok := strings.CutPrefix(h, prefix) if !ok { return nil, fmt.Errorf("Authorization header is not AWS4-HMAC-SHA256") } rest = strings.TrimSpace(rest) out := &parsedAuthorization{} for _, part := range strings.Split(rest, ",") { part = strings.TrimSpace(part) k, v, ok := strings.Cut(part, "=") if !ok { continue } switch k { case "Credential": // ////aws4_request // // Access keys may contain "/" so do a reverse split. fields := strings.Split(v, "/") if len(fields) < 5 { return nil, fmt.Errorf("malformed Credential field") } out.accessKey = strings.Join(fields[:len(fields)-4], "/") out.scopeDate = fields[len(fields)-4] out.scopeRegion = fields[len(fields)-3] out.scopeService = fields[len(fields)-2] out.scope = strings.Join(fields[len(fields)-4:], "/") case "SignedHeaders": out.signedHeaders = strings.Split(v, ";") sort.Strings(out.signedHeaders) case "Signature": out.signature = v } } if out.accessKey == "" || out.signature == "" || len(out.signedHeaders) == 0 { return nil, fmt.Errorf("incomplete Authorization header") } return out, nil } // canonicalRequest builds the canonical request string defined by SigV4. func canonicalRequest(r *http.Request, signedHeaders []string, bodyHash string) string { var sb strings.Builder sb.WriteString(r.Method) sb.WriteByte('\n') sb.WriteString(canonicalURI(r.URL.Path)) sb.WriteByte('\n') sb.WriteString(canonicalQueryString(r.URL.Query())) sb.WriteByte('\n') for _, name := range signedHeaders { sb.WriteString(name) sb.WriteByte(':') sb.WriteString(canonicalHeaderValue(r, name)) sb.WriteByte('\n') } sb.WriteByte('\n') sb.WriteString(strings.Join(signedHeaders, ";")) sb.WriteByte('\n') sb.WriteString(bodyHash) return sb.String() } // canonicalURI returns the URI with each path segment URI-encoded once. func canonicalURI(path string) string { if path == "" { return "/" } segments := strings.Split(path, "/") for i, seg := range segments { segments[i] = uriEncode(seg, false) } return strings.Join(segments, "/") } // canonicalQueryString returns query parameters sorted by name then value, // each name and value URI-encoded. func canonicalQueryString(values url.Values) string { if len(values) == 0 { return "" } keys := make([]string, 0, len(values)) for k := range values { keys = append(keys, k) } sort.Strings(keys) var sb strings.Builder first := true for _, k := range keys { vals := values[k] sort.Strings(vals) for _, v := range vals { if !first { sb.WriteByte('&') } first = false sb.WriteString(uriEncode(k, true)) sb.WriteByte('=') sb.WriteString(uriEncode(v, true)) } } return sb.String() } // canonicalHeaderValue returns the trimmed value of header name, with // whitespace runs collapsed. func canonicalHeaderValue(r *http.Request, name string) string { if name == "host" { return r.Host } v := r.Header.Get(name) v = strings.TrimSpace(v) for strings.Contains(v, " ") { v = strings.ReplaceAll(v, " ", " ") } return v } // uriEncode percent-encodes s per RFC 3986. Slashes are preserved when // encodeSlash is false. func uriEncode(s string, encodeSlash bool) string { var sb strings.Builder for _, r := range []byte(s) { switch { case r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '_', r == '-', r == '~', r == '.': sb.WriteByte(r) case r == '/' && !encodeSlash: sb.WriteByte(r) default: fmt.Fprintf(&sb, "%%%02X", r) } } return sb.String() } func sha256Hex(b []byte) string { h := sha256.Sum256(b) return hex.EncodeToString(h[:]) } func hmacSHA256(key []byte, msg string) []byte { mac := hmac.New(sha256.New, key) _, _ = mac.Write([]byte(msg)) return mac.Sum(nil) } func hmacSHA256Hex(key []byte, msg string) string { return hex.EncodeToString(hmacSHA256(key, msg)) } func deriveSigningKey(secret, date, region, service string) []byte { kDate := hmacSHA256([]byte("AWS4"+secret), date) kRegion := hmacSHA256(kDate, region) kService := hmacSHA256(kRegion, service) return hmacSHA256(kService, "aws4_request") } // SignRequest signs r in place using SigV4 with the given credential. It is // exposed for tests and not used by the server. func SignRequest(r *http.Request, accessKey, secretKey, region, service string, body []byte, now time.Time) error { amzDate := now.UTC().Format("20060102T150405Z") scopeDate := now.UTC().Format("20060102") r.Header.Set("X-Amz-Date", amzDate) if r.Header.Get("Host") == "" { r.Host = r.URL.Host } bodyHash := unsignedPayload if body != nil { bodyHash = sha256Hex(body) r.Body = io.NopCloser(bytes.NewReader(body)) r.ContentLength = int64(len(body)) } r.Header.Set("X-Amz-Content-Sha256", bodyHash) signed := []string{"host", "x-amz-content-sha256", "x-amz-date"} sort.Strings(signed) canonical := canonicalRequest(r, signed, bodyHash) scope := strings.Join([]string{scopeDate, region, service, "aws4_request"}, "/") stringToSign := strings.Join([]string{ "AWS4-HMAC-SHA256", amzDate, scope, sha256Hex([]byte(canonical)), }, "\n") key := deriveSigningKey(secretKey, scopeDate, region, service) sig := hmacSHA256Hex(key, stringToSign) r.Header.Set("Authorization", fmt.Sprintf( "AWS4-HMAC-SHA256 Credential=%s/%s,SignedHeaders=%s,Signature=%s", accessKey, scope, strings.Join(signed, ";"), sig, )) return nil } incus-7.0.0/internal/server/storage/s3/local/chunked.go000066400000000000000000000157201517523235500230310ustar00rootroot00000000000000package local import ( "bufio" "crypto/hmac" "errors" "fmt" "io" "strconv" "strings" ) // emptyStringSHA256 is the SHA-256 hash of the empty string. It appears in // the chunk and trailer string-to-sign as the canonical-headers slot. var emptyStringSHA256 = sha256Hex(nil) // chunkSigningContext carries the data needed to verify per-chunk SigV4 // signatures for STREAMING-AWS4-HMAC-SHA256-PAYLOAD[-TRAILER] payloads. type chunkSigningContext struct { // algorithm is the chunk-signing algorithm: // "AWS4-HMAC-SHA256-PAYLOAD" or "AWS4-HMAC-SHA256-PAYLOAD-TRAILER". algorithm string // signingKey is the SigV4 derived signing key (kSigning). signingKey []byte // amzDate is the X-Amz-Date header value (YYYYMMDDTHHMMSSZ). amzDate string // scope is "///aws4_request". scope string // prevSignature is the rolling previous-signature, seeded with the // request's seed signature from the Authorization header. It is // updated to the chunk's signature after each chunk is verified. prevSignature string } // chunkedReader decodes an aws-chunked encoded body. Each Read returns // decoded payload bytes only; chunk framing, optional trailers and the // terminating CRLF are consumed transparently. // // When sign is non-nil, every data chunk's "chunk-signature=" parameter is // verified before its bytes are returned. A mismatch surfaces as a Read // error. type chunkedReader struct { br *bufio.Reader sign *chunkSigningContext // chunk holds the current decoded chunk's bytes; pos is how many of // those bytes have been served to callers. chunk []byte pos int // finished is set once the terminating zero-length chunk (and trailer, // if any) has been fully consumed. finished bool } // newChunkedReader wraps r in an aws-chunked decoder. If sign is non-nil, // per-chunk signatures are verified. func newChunkedReader(r io.Reader, sign *chunkSigningContext) *chunkedReader { return &chunkedReader{ br: bufio.NewReader(r), sign: sign, } } // Read returns decoded payload bytes from the wrapped aws-chunked stream. func (c *chunkedReader) Read(p []byte) (int, error) { if c.pos >= len(c.chunk) { if c.finished { return 0, io.EOF } err := c.nextChunk() if err != nil { return 0, err } if c.finished && c.pos >= len(c.chunk) { return 0, io.EOF } } n := copy(p, c.chunk[c.pos:]) c.pos += n return n, nil } // nextChunk reads the next chunk header, optionally verifies its signature, // loads its bytes into c.chunk, and consumes the terminating CRLF. If the // chunk is the terminating zero-length chunk, any trailer is consumed and // the stream is marked finished. func (c *chunkedReader) nextChunk() error { c.chunk = c.chunk[:0] c.pos = 0 header, err := readLine(c.br) if err != nil { return fmt.Errorf("read chunk header: %w", err) } sizeField, sigField := splitChunkHeader(header) size, err := strconv.ParseInt(sizeField, 16, 64) if err != nil || size < 0 { return fmt.Errorf("invalid chunk size %q", sizeField) } if size > 0 { if cap(c.chunk) < int(size) { c.chunk = make([]byte, size) } else { c.chunk = c.chunk[:size] } _, err = io.ReadFull(c.br, c.chunk) if err != nil { return fmt.Errorf("read chunk body: %w", err) } err = consumeCRLF(c.br) if err != nil { return err } err = c.verifyChunkSignature(sigField, c.chunk) if err != nil { return err } return nil } // size == 0: terminating chunk. The chunk body is empty but for // signed payloads still has its own signature over the empty body. err = consumeCRLF(c.br) if err != nil { // AWS streaming with trailer omits the CRLF after the // zero-chunk header (the trailer block follows directly). // Tolerate either layout by allowing consumeCRLF to be // optional here: if the next bytes are not CRLF, treat them // as the trailer block. if !errors.Is(err, errExpectedCRLF) { return err } } err = c.verifyChunkSignature(sigField, nil) if err != nil { return err } err = c.consumeTrailer() if err != nil { return err } c.finished = true return nil } // verifyChunkSignature checks the chunk-signature line for a chunk whose // decoded body is data. For unsigned streams the signature is ignored. func (c *chunkedReader) verifyChunkSignature(sig string, data []byte) error { if c.sign == nil { return nil } if sig == "" { return errors.New("missing chunk signature") } expected := computeChunkSignature(c.sign, sha256Hex(data)) if !hmac.Equal([]byte(expected), []byte(sig)) { return errors.New("chunk signature mismatch") } c.sign.prevSignature = sig return nil } // consumeTrailer reads any trailing headers that appear after the final // zero-length chunk and consumes the closing empty line. The trailing // signature on signed-with-trailer payloads is parsed but not verified. func (c *chunkedReader) consumeTrailer() error { for { line, err := readLine(c.br) if errors.Is(err, io.EOF) { // Streams without an explicit closing CRLF are // tolerated: the body ended at EOF. return nil } if err != nil { return fmt.Errorf("read trailer line: %w", err) } if line == "" { return nil } } } // splitChunkHeader splits a chunk header line into its size field and the // optional chunk-signature value. The header has the form: // // [;chunk-signature=][;...] func splitChunkHeader(header string) (size, signature string) { parts := strings.Split(header, ";") size = strings.TrimSpace(parts[0]) for _, p := range parts[1:] { p = strings.TrimSpace(p) v, ok := strings.CutPrefix(p, "chunk-signature=") if ok { signature = v } } return size, signature } // readLine reads a CRLF-terminated line from r and returns it without the // terminating CRLF. func readLine(r *bufio.Reader) (string, error) { line, err := r.ReadString('\n') if err != nil { return "", err } return strings.TrimRight(line, "\r\n"), nil } // errExpectedCRLF is returned by consumeCRLF when the next two bytes are // not "\r\n". It is sentinel-only so that callers can distinguish the // optional-CRLF case at the end of a stream. var errExpectedCRLF = errors.New("expected CRLF") // consumeCRLF reads exactly "\r\n" from r. func consumeCRLF(r *bufio.Reader) error { buf := make([]byte, 2) _, err := io.ReadFull(r, buf) if err != nil { return fmt.Errorf("read chunk terminator: %w", err) } if buf[0] != '\r' || buf[1] != '\n' { return errExpectedCRLF } return nil } // computeChunkSignature returns the expected hex chunk signature for a chunk // whose decoded SHA-256 hash is chunkHash, given the rolling signing context. // // The string-to-sign is: // // + "\n" + // + "\n" + // + "\n" + // + "\n" + // sha256("") + "\n" + // func computeChunkSignature(ctx *chunkSigningContext, chunkHash string) string { stringToSign := strings.Join([]string{ ctx.algorithm, ctx.amzDate, ctx.scope, ctx.prevSignature, emptyStringSHA256, chunkHash, }, "\n") return hmacSHA256Hex(ctx.signingKey, stringToSign) } incus-7.0.0/internal/server/storage/s3/local/list.go000066400000000000000000000110051517523235500223530ustar00rootroot00000000000000package local import ( "encoding/xml" "errors" "io/fs" "net/http" "path/filepath" "sort" "strconv" "strings" "github.com/lxc/incus/v7/internal/server/storage/s3" ) // listObjectsV2Result is the XML root for ListObjectsV2 responses. type listObjectsV2Result struct { XMLName xml.Name `xml:"ListBucketResult"` Name string `xml:"Name,omitempty"` Prefix string `xml:"Prefix"` Delimiter string `xml:"Delimiter,omitempty"` MaxKeys int `xml:"MaxKeys"` KeyCount int `xml:"KeyCount"` IsTruncated bool `xml:"IsTruncated"` NextContinuationToken string `xml:"NextContinuationToken,omitempty"` StartAfter string `xml:"StartAfter,omitempty"` Contents []listObjectsV2Object `xml:"Contents"` CommonPrefixes []listCommonPrefix `xml:"CommonPrefixes"` } type listObjectsV2Object struct { Key string `xml:"Key"` LastModified string `xml:"LastModified"` ETag string `xml:"ETag"` Size int64 `xml:"Size"` StorageClass string `xml:"StorageClass"` } type listCommonPrefix struct { Prefix string `xml:"Prefix"` } // listObjects implements ListObjectsV2. func (s *Server) listObjects(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() prefix := q.Get("prefix") delimiter := q.Get("delimiter") startAfter := q.Get("start-after") token := q.Get("continuation-token") if token != "" { startAfter = token } maxKeys := 1000 v := q.Get("max-keys") if v != "" { n, err := strconv.Atoi(v) if err == nil && n > 0 && n < 1000 { maxKeys = n } } keys, err := s.collectKeys() if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } sort.Strings(keys) result := &listObjectsV2Result{ Prefix: prefix, Delimiter: delimiter, MaxKeys: maxKeys, StartAfter: q.Get("start-after"), } seenPrefix := map[string]bool{} for _, k := range keys { if startAfter != "" && k <= startAfter { continue } if prefix != "" && !strings.HasPrefix(k, prefix) { continue } if delimiter != "" { rest := strings.TrimPrefix(k, prefix) idx := strings.Index(rest, delimiter) if idx >= 0 { cp := prefix + rest[:idx+len(delimiter)] if !seenPrefix[cp] { seenPrefix[cp] = true if result.KeyCount >= maxKeys { result.IsTruncated = true result.NextContinuationToken = k break } result.CommonPrefixes = append(result.CommonPrefixes, listCommonPrefix{Prefix: cp}) result.KeyCount++ } continue } } if result.KeyCount >= maxKeys { result.IsTruncated = true result.NextContinuationToken = k break } dataPath := filepath.Join(s.dataDir(), k) meta, err := loadOrInferMeta(dataPath) if err != nil { // Data file vanished between walk and stat. continue } result.Contents = append(result.Contents, listObjectsV2Object{ Key: k, LastModified: meta.LastMod.UTC().Format("2006-01-02T15:04:05.000Z"), ETag: `"` + meta.ETag + `"`, Size: meta.Size, StorageClass: "STANDARD", }) result.KeyCount++ } body, err := xml.Marshal(result) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } w.Header().Set("Content-Type", "application/xml") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(``)) _, _ = w.Write(body) } // collectKeys walks the data directory and returns the list of object keys. // Sidecar files, the uploads directory, and temporary files are skipped. func (s *Server) collectKeys() ([]string, error) { root := s.dataDir() keys := []string{} err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { if errors.Is(err, fs.ErrNotExist) && path == root { return filepath.SkipAll } return err } if path == root { return nil } rel, err := filepath.Rel(root, path) if err != nil { return err } if d.IsDir() { if rel == uploadsSubdir { return filepath.SkipDir } return nil } // Skip metadata, in-flight temporary files, and dotfiles. base := filepath.Base(rel) if strings.HasSuffix(base, metaSuffix) || strings.HasSuffix(base, ".tmp") { return nil } keys = append(keys, filepath.ToSlash(rel)) return nil }) if err != nil { return nil, err } return keys, nil } incus-7.0.0/internal/server/storage/s3/local/meta.go000066400000000000000000000037031517523235500223340ustar00rootroot00000000000000package local import ( "crypto/md5" "encoding/hex" "encoding/json" "errors" "io" "io/fs" "os" "time" ) // metaSuffix is appended to the data filename to form the metadata. const metaSuffix = ".meta" // objectMeta is the metadata stored alongside object data files. type objectMeta struct { ContentType string `json:"content_type,omitempty"` ETag string `json:"etag"` Size int64 `json:"size"` LastMod time.Time `json:"last_modified"` UserMeta map[string]string `json:"user_meta,omitempty"` } func readMeta(metaPath string) (*objectMeta, error) { b, err := os.ReadFile(metaPath) if err != nil { return nil, err } m := &objectMeta{} err = json.Unmarshal(b, m) if err != nil { return nil, err } return m, nil } func writeMeta(metaPath string, m *objectMeta) error { b, err := json.Marshal(m) if err != nil { return err } tmp := metaPath + ".tmp" err = os.WriteFile(tmp, b, 0o600) if err != nil { return err } return os.Rename(tmp, metaPath) } func metaPathFor(dataPath string) string { return dataPath + metaSuffix } func removeMeta(metaPath string) error { err := os.Remove(metaPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } return nil } func loadOrInferMeta(dataPath string) (*objectMeta, error) { meta, err := readMeta(metaPathFor(dataPath)) if err == nil { return meta, nil } if !errors.Is(err, fs.ErrNotExist) { return nil, err } st, err := os.Stat(dataPath) if err != nil { return nil, err } if st.IsDir() { return nil, fs.ErrNotExist } f, err := os.Open(dataPath) if err != nil { return nil, err } defer func() { _ = f.Close() }() hasher := md5.New() _, err = io.Copy(hasher, f) if err != nil { return nil, err } meta = &objectMeta{ ETag: hex.EncodeToString(hasher.Sum(nil)), Size: st.Size(), LastMod: st.ModTime().UTC(), } _ = writeMeta(metaPathFor(dataPath), meta) return meta, nil } incus-7.0.0/internal/server/storage/s3/local/migrate.go000066400000000000000000000175131517523235500230420ustar00rootroot00000000000000package local import ( "bytes" "errors" "fmt" "io" "io/fs" "os" "path/filepath" "sort" "strconv" "strings" "sync" ) // minioSubdir is the legacy directory containing data managed by an embedded // minio process. const minioSubdir = "minio" // minioArchivedSubdir is the post-migration name of the legacy directory. const minioArchivedSubdir = ".minio" // minioMetadataDir is minio's per-bucket / global metadata directory. const minioMetadataDir = ".minio.sys" // minioMetaFile is the per-object metadata file written by minio's modern // (xl-storage) backend. Its presence in a directory marks the directory as // an object store, not a path component. const minioMetaFile = "xl.meta" // migrateLocks serialises migrations per bucket directory. var migrateLocks sync.Map // map[string]*sync.Mutex // MigrateMinioBucket converts a bucket directory previously managed by an // embedded minio process to the data/ layout used by Server. // // On success the bucket directory contains: // // data/ object data (one regular file per object key) // .minio/ everything left over from minio (metadata, format files) // // Objects in minio's xl-storage layout each live in a directory named after // the object key. That directory contains an xl.meta file, and either holds // the object data inline (small objects) or alongside a UUID-named // subdirectory containing part files (large / multipart objects). The // migration converts each such object directory into a single regular file // at data/. // // The function is a no-op when minio/ does not exist (already migrated, or // freshly created bucket). func MigrateMinioBucket(bucketDir, bucketName string) error { lockI, _ := migrateLocks.LoadOrStore(bucketDir, &sync.Mutex{}) lock, ok := lockI.(*sync.Mutex) if ok { lock.Lock() defer lock.Unlock() } src := filepath.Join(bucketDir, minioSubdir) _, err := os.Stat(src) if errors.Is(err, fs.ErrNotExist) { return nil } if err != nil { return err } dst := filepath.Join(bucketDir, minioArchivedSubdir) _, err = os.Stat(dst) if err == nil { return errors.New("Both minio/ and .minio/ are present; migration cannot proceed") } if !errors.Is(err, fs.ErrNotExist) { return err } dataDir := filepath.Join(bucketDir, dataSubdir) err = os.MkdirAll(dataDir, 0o700) if err != nil { return err } bucketRoot := filepath.Join(src, bucketName) _, err = os.Stat(bucketRoot) if err == nil { err = walkAndConvert(bucketRoot, dataDir) if err != nil { return fmt.Errorf("Failed migrating minio bucket data: %w", err) } } else if !errors.Is(err, fs.ErrNotExist) { return err } err = os.Rename(src, dst) if err != nil { return fmt.Errorf("Failed archiving minio directory: %w", err) } return nil } // walkAndConvert traverses an xl-storage tree under src, writing each object // it finds as a regular file under dst. minio's metadata directories are // left in place (the source tree is preserved alongside; only data files // are extracted). func walkAndConvert(src, dst string) error { return filepath.WalkDir(src, func(path string, d fs.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if !d.IsDir() { return nil } // Don't descend into metadata dirs. if d.Name() == minioMetadataDir { return filepath.SkipDir } // Object directories contain xl.meta. Convert them and don't // recurse further (the part-data subdirs underneath are the // object's storage, not nested objects). if hasFile(path, minioMetaFile) { rel, err := filepath.Rel(src, path) if err != nil { return err } target := filepath.Join(dst, rel) err = os.MkdirAll(filepath.Dir(target), 0o700) if err != nil { return err } err = extractObjectDir(path, target) if err != nil { return fmt.Errorf("extract %q: %w", rel, err) } return filepath.SkipDir } return nil }) } // extractObjectDir reads the xl.meta inside srcDir and writes the assembled // object data to dst as a regular file. // // Small objects have their data stored inline in xl.meta and are extracted // from there. Larger objects have part files in a UUID-named subdirectory of // srcDir; those are concatenated in part-number order. func extractObjectDir(srcDir, dst string) error { metaBytes, err := os.ReadFile(filepath.Join(srcDir, minioMetaFile)) if err != nil { return err } tmp := dst + ".tmp" out, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) if err != nil { return err } cleanup := func() { _ = out.Close() _ = os.Remove(tmp) } inline, err := xlInlineData(metaBytes) if err == nil && inline != nil { err = stripBitrotHashes(out, bytes.NewReader(inline)) if err != nil { cleanup() return err } } else { // External data: find the part-file subdirectory (the only // non-xl.meta directory inside srcDir) and concatenate its // part.N files. partsDir, err := findPartsDir(srcDir) if err != nil { cleanup() return err } err = concatParts(partsDir, out) if err != nil { cleanup() return err } } err = out.Close() if err != nil { _ = os.Remove(tmp) return err } return os.Rename(tmp, dst) } // findPartsDir returns the path to the single subdirectory under objDir // (other than minio's own metadata directories) that holds the object's // part files. func findPartsDir(objDir string) (string, error) { entries, err := os.ReadDir(objDir) if err != nil { return "", err } for _, e := range entries { if !e.IsDir() { continue } if e.Name() == minioMetadataDir { continue } return filepath.Join(objDir, e.Name()), nil } return "", fmt.Errorf("no parts directory under %q", objDir) } // concatParts writes the concatenation of partsDir/part. files (sorted // numerically by N) to w. func concatParts(partsDir string, w io.Writer) error { entries, err := os.ReadDir(partsDir) if err != nil { return err } type part struct { path string num int } parts := make([]part, 0, len(entries)) for _, e := range entries { if e.IsDir() { continue } name := e.Name() nstr, ok := strings.CutPrefix(name, "part.") if !ok { continue } n, err := strconv.Atoi(nstr) if err != nil { continue } parts = append(parts, part{path: filepath.Join(partsDir, name), num: n}) } if len(parts) == 0 { return fmt.Errorf("no part files in %q", partsDir) } sort.Slice(parts, func(i, j int) bool { return parts[i].num < parts[j].num }) for _, p := range parts { f, err := os.Open(p.path) if err != nil { return err } err = stripBitrotHashes(w, f) _ = f.Close() if err != nil { return err } } return nil } // minio bitrot protection prepends a 32-byte HighwayHash256 hash to every // 1MB block of object data. Both inline data and part files use this // layout. The migration extracts the actual bytes by skipping each hash. const ( bitrotHashSize = 32 bitrotBlockSize = 1 << 20 ) // stripBitrotHashes copies r to w, dropping the leading bitrotHashSize // bytes of every bitrotBlockSize-byte chunk. The final chunk may be // shorter than bitrotBlockSize; its trailing data is still copied. func stripBitrotHashes(w io.Writer, r io.Reader) error { buf := make([]byte, bitrotBlockSize) for { // Read and discard the per-block hash. _, err := io.ReadFull(r, buf[:bitrotHashSize]) if errors.Is(err, io.EOF) { return nil } if err != nil { return err } // Read up to one block worth of data. n, err := io.ReadFull(r, buf[:bitrotBlockSize]) if n > 0 { _, writeErr := w.Write(buf[:n]) if writeErr != nil { return writeErr } } if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { return nil } if err != nil { return err } } } // hasFile reports whether dir contains a regular file with the given name. func hasFile(dir, name string) bool { st, err := os.Stat(filepath.Join(dir, name)) if err != nil { return false } return !st.IsDir() } incus-7.0.0/internal/server/storage/s3/local/multipart.go000066400000000000000000000215501517523235500234270ustar00rootroot00000000000000package local import ( "crypto/md5" "encoding/hex" "encoding/json" "encoding/xml" "errors" "fmt" "io" "io/fs" "net/http" "os" "path/filepath" "sort" "strconv" "time" "github.com/google/uuid" "github.com/lxc/incus/v7/internal/server/storage/s3" ) // uploadInfo is persisted in upload.json under each in-flight upload directory. type uploadInfo struct { Key string `json:"key"` ContentType string `json:"content_type,omitempty"` UserMeta map[string]string `json:"user_meta,omitempty"` Initiated time.Time `json:"initiated"` } func (s *Server) uploadDir(uploadID string) string { return filepath.Join(s.uploadsDir(), uploadID) } func (s *Server) initiateMultipartUpload(w http.ResponseWriter, r *http.Request, key string) { id := uuid.New().String() dir := s.uploadDir(id) err := os.MkdirAll(dir, 0o700) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } info := &uploadInfo{ Key: key, ContentType: r.Header.Get("Content-Type"), UserMeta: extractUserMeta(r.Header), Initiated: time.Now().UTC(), } b, err := json.Marshal(info) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } err = os.WriteFile(filepath.Join(dir, "upload.json"), b, 0o600) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } type initiateResult struct { XMLName xml.Name `xml:"InitiateMultipartUploadResult"` Bucket string `xml:"Bucket,omitempty"` Key string `xml:"Key"` UploadID string `xml:"UploadId"` } resp, err := xml.Marshal(&initiateResult{Key: key, UploadID: id}) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } w.Header().Set("Content-Type", "application/xml") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(``)) _, _ = w.Write(resp) } func (s *Server) uploadPart(w http.ResponseWriter, r *http.Request, key, uploadID string) { dir := s.uploadDir(uploadID) _, err := os.Stat(dir) if err != nil { if errors.Is(err, fs.ErrNotExist) { (&s3.Error{Code: s3.ErrorInvalidRequest, Message: "Upload not found."}).Response(w) return } (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } partStr := r.URL.Query().Get("partNumber") partNumber, err := strconv.Atoi(partStr) if err != nil || partNumber < 1 || partNumber > 10000 { (&s3.Error{Code: s3.ErrorInvalidRequest, Message: "Invalid partNumber."}).Response(w) return } partPath := filepath.Join(dir, fmt.Sprintf("part-%05d", partNumber)) tmp := partPath + ".tmp" f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } hasher := md5.New() _, err = io.Copy(io.MultiWriter(f, hasher), r.Body) closeErr := f.Close() if err != nil || closeErr != nil { _ = os.Remove(tmp) msg := err if msg == nil { msg = closeErr } (&s3.Error{Code: s3.ErrorCodeInternalError, Message: msg.Error()}).Response(w) return } err = os.Rename(tmp, partPath) if err != nil { _ = os.Remove(tmp) (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } etag := hex.EncodeToString(hasher.Sum(nil)) w.Header().Set("ETag", `"`+etag+`"`) w.WriteHeader(http.StatusOK) } // completeRequest models the body of CompleteMultipartUpload. type completeRequest struct { XMLName xml.Name `xml:"CompleteMultipartUpload"` Parts []completePart `xml:"Part"` } type completePart struct { PartNumber int `xml:"PartNumber"` ETag string `xml:"ETag"` } func (s *Server) completeMultipartUpload(w http.ResponseWriter, r *http.Request, key, uploadID string) { dir := s.uploadDir(uploadID) infoBytes, err := os.ReadFile(filepath.Join(dir, "upload.json")) if err != nil { if errors.Is(err, fs.ErrNotExist) { (&s3.Error{Code: s3.ErrorInvalidRequest, Message: "Upload not found."}).Response(w) return } (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } info := &uploadInfo{} err = json.Unmarshal(infoBytes, info) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } body, err := io.ReadAll(r.Body) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } req := &completeRequest{} err = xml.Unmarshal(body, req) if err != nil { (&s3.Error{Code: s3.ErrorInvalidRequest, Message: err.Error()}).Response(w) return } // Sort by part number to assemble in order. sort.Slice(req.Parts, func(i, j int) bool { return req.Parts[i].PartNumber < req.Parts[j].PartNumber }) dataPath, err := s.objectPath(key) if err != nil { (&s3.Error{Code: s3.ErrorInvalidRequest, Message: err.Error()}).Response(w) return } err = os.MkdirAll(filepath.Dir(dataPath), 0o700) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } tmp := dataPath + ".tmp" out, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } combined := md5.New() var size int64 for _, p := range req.Parts { partPath := filepath.Join(dir, fmt.Sprintf("part-%05d", p.PartNumber)) f, err := os.Open(partPath) if err != nil { _ = out.Close() _ = os.Remove(tmp) (&s3.Error{Code: s3.ErrorInvalidRequest, Message: fmt.Sprintf("Missing part %d.", p.PartNumber)}).Response(w) return } n, copyErr := io.Copy(io.MultiWriter(out, combined), f) _ = f.Close() if copyErr != nil { _ = out.Close() _ = os.Remove(tmp) (&s3.Error{Code: s3.ErrorCodeInternalError, Message: copyErr.Error()}).Response(w) return } size += n } err = out.Close() if err != nil { _ = os.Remove(tmp) (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } err = os.Rename(tmp, dataPath) if err != nil { _ = os.Remove(tmp) (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } etag := hex.EncodeToString(combined.Sum(nil)) meta := &objectMeta{ ContentType: info.ContentType, ETag: etag, Size: size, LastMod: time.Now().UTC(), UserMeta: info.UserMeta, } err = writeMeta(metaPathFor(dataPath), meta) if err != nil { _ = os.Remove(dataPath) (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } // Clean up the upload directory. _ = os.RemoveAll(dir) type completeResult struct { XMLName xml.Name `xml:"CompleteMultipartUploadResult"` Key string `xml:"Key"` ETag string `xml:"ETag"` } resp, err := xml.Marshal(&completeResult{Key: key, ETag: `"` + etag + `"`}) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } w.Header().Set("Content-Type", "application/xml") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(``)) _, _ = w.Write(resp) } func (s *Server) abortMultipartUpload(w http.ResponseWriter, key, uploadID string) { dir := s.uploadDir(uploadID) err := os.RemoveAll(dir) if err != nil && !errors.Is(err, fs.ErrNotExist) { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } w.WriteHeader(http.StatusNoContent) } // listMultipartUploads enumerates in-flight uploads. Minimal implementation: // returns all uploads regardless of prefix/delimiter parameters. func (s *Server) listMultipartUploads(w http.ResponseWriter, r *http.Request) { type upload struct { Key string `xml:"Key"` UploadID string `xml:"UploadId"` Initiated string `xml:"Initiated"` } type result struct { XMLName xml.Name `xml:"ListMultipartUploadsResult"` Uploads []upload `xml:"Upload"` } out := &result{} entries, err := os.ReadDir(s.uploadsDir()) if err != nil && !errors.Is(err, fs.ErrNotExist) { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } for _, e := range entries { if !e.IsDir() { continue } b, err := os.ReadFile(filepath.Join(s.uploadsDir(), e.Name(), "upload.json")) if err != nil { continue } info := &uploadInfo{} if json.Unmarshal(b, info) != nil { continue } out.Uploads = append(out.Uploads, upload{ Key: info.Key, UploadID: e.Name(), Initiated: info.Initiated.UTC().Format("2006-01-02T15:04:05.000Z"), }) } body, err := xml.Marshal(out) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } w.Header().Set("Content-Type", "application/xml") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(``)) _, _ = w.Write(body) } incus-7.0.0/internal/server/storage/s3/local/object.go000066400000000000000000000147521517523235500226620ustar00rootroot00000000000000package local import ( "crypto/md5" "encoding/hex" "errors" "fmt" "io" "io/fs" "net/http" "os" "path/filepath" "strconv" "strings" "time" "github.com/lxc/incus/v7/internal/server/storage/s3" ) func (s *Server) objectPath(key string) (string, error) { if key == "" || strings.HasPrefix(key, "/") { return "", errors.New("Invalid object key") } for _, seg := range strings.Split(key, "/") { if seg == ".." || seg == "." { return "", errors.New("Invalid object key") } } first, _, _ := strings.Cut(key, "/") if first == uploadsSubdir || strings.HasSuffix(key, metaSuffix) { return "", errors.New("Reserved object key") } return filepath.Join(s.dataDir(), key), nil } func (s *Server) headObject(w http.ResponseWriter, r *http.Request, key string) { dataPath, err := s.objectPath(key) if err != nil { (&s3.Error{Code: s3.ErrorInvalidRequest, Message: err.Error()}).Response(w) return } meta, err := loadOrInferMeta(dataPath) if err != nil { if errors.Is(err, fs.ErrNotExist) { (&s3.Error{Code: s3.ErrorCodeNoSuchBucket, Message: "Object not found."}).Response(w) return } (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } writeObjectHeaders(w, meta) w.WriteHeader(http.StatusOK) } func (s *Server) getObject(w http.ResponseWriter, r *http.Request, key string) { dataPath, err := s.objectPath(key) if err != nil { (&s3.Error{Code: s3.ErrorInvalidRequest, Message: err.Error()}).Response(w) return } meta, err := loadOrInferMeta(dataPath) if err != nil { if errors.Is(err, fs.ErrNotExist) { (&s3.Error{Code: s3.ErrorCodeNoSuchBucket, Message: "Object not found."}).Response(w) return } (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } f, err := os.Open(dataPath) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } defer func() { _ = f.Close() }() rangeHeader := r.Header.Get("Range") if rangeHeader == "" { writeObjectHeaders(w, meta) w.WriteHeader(http.StatusOK) _, _ = io.Copy(w, f) return } start, end, ok := parseSingleRange(rangeHeader, meta.Size) if !ok { w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", meta.Size)) (&s3.Error{Code: s3.ErrorInvalidRequest, Message: "Invalid Range header."}).Response(w) return } _, err = f.Seek(start, io.SeekStart) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } length := end - start + 1 writeObjectHeaders(w, meta) w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, meta.Size)) w.Header().Set("Content-Length", strconv.FormatInt(length, 10)) w.WriteHeader(http.StatusPartialContent) _, _ = io.CopyN(w, f, length) } func (s *Server) putObject(w http.ResponseWriter, r *http.Request, key string) { dataPath, err := s.objectPath(key) if err != nil { (&s3.Error{Code: s3.ErrorInvalidRequest, Message: err.Error()}).Response(w) return } err = os.MkdirAll(filepath.Dir(dataPath), 0o700) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } tmp := dataPath + ".tmp" f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } hasher := md5.New() written, err := io.Copy(io.MultiWriter(f, hasher), r.Body) closeErr := f.Close() if err != nil || closeErr != nil { _ = os.Remove(tmp) msg := err if msg == nil { msg = closeErr } (&s3.Error{Code: s3.ErrorCodeInternalError, Message: msg.Error()}).Response(w) return } err = os.Rename(tmp, dataPath) if err != nil { _ = os.Remove(tmp) (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } etag := hex.EncodeToString(hasher.Sum(nil)) meta := &objectMeta{ ContentType: r.Header.Get("Content-Type"), ETag: etag, Size: written, LastMod: time.Now().UTC(), UserMeta: extractUserMeta(r.Header), } err = writeMeta(metaPathFor(dataPath), meta) if err != nil { _ = os.Remove(dataPath) (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } w.Header().Set("ETag", `"`+etag+`"`) w.WriteHeader(http.StatusOK) } func (s *Server) deleteObject(w http.ResponseWriter, key string) { dataPath, err := s.objectPath(key) if err != nil { (&s3.Error{Code: s3.ErrorInvalidRequest, Message: err.Error()}).Response(w) return } err = os.Remove(dataPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } err = removeMeta(metaPathFor(dataPath)) if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } w.WriteHeader(http.StatusNoContent) } func writeObjectHeaders(w http.ResponseWriter, meta *objectMeta) { if meta.ContentType != "" { w.Header().Set("Content-Type", meta.ContentType) } w.Header().Set("ETag", `"`+meta.ETag+`"`) w.Header().Set("Content-Length", strconv.FormatInt(meta.Size, 10)) w.Header().Set("Last-Modified", meta.LastMod.UTC().Format(http.TimeFormat)) for k, v := range meta.UserMeta { w.Header().Set("X-Amz-Meta-"+k, v) } } func extractUserMeta(h http.Header) map[string]string { out := map[string]string{} for k, vs := range h { const prefix = "X-Amz-Meta-" if strings.HasPrefix(k, prefix) && len(vs) > 0 { out[strings.TrimPrefix(k, prefix)] = vs[0] } } if len(out) == 0 { return nil } return out } // parseSingleRange parses a single byte-range header (the only form S3 requires). Returns inclusive start and end offsets. func parseSingleRange(h string, size int64) (int64, int64, bool) { rest, ok := strings.CutPrefix(h, "bytes=") if !ok { return 0, 0, false } if strings.Contains(rest, ",") { return 0, 0, false } startStr, endStr, ok := strings.Cut(rest, "-") if !ok { return 0, 0, false } if startStr == "" && endStr != "" { // Suffix range: "-N" means the last N bytes. n, err := strconv.ParseInt(endStr, 10, 64) if err != nil || n <= 0 || n > size { n = size } return size - n, size - 1, true } start, err := strconv.ParseInt(startStr, 10, 64) if err != nil || start < 0 || start >= size { return 0, 0, false } end := size - 1 if endStr != "" { end, err = strconv.ParseInt(endStr, 10, 64) if err != nil || end < start { return 0, 0, false } if end >= size { end = size - 1 } } return start, end, true } incus-7.0.0/internal/server/storage/s3/local/server.go000066400000000000000000000125151517523235500227150ustar00rootroot00000000000000// Package local implements an in-process S3-compatible HTTP handler that // serves objects from a directory on the local filesystem. // // It is used by Incus to expose buckets backed by local storage drivers // (dir, btrfs, zfs) without spawning an external S3 server. // // On-disk layout under the bucket directory: // // data/ object data // data/.meta object metadata (JSON) // data/.uploads// in-flight multipart upload state package local import ( "net/http" "net/url" "path/filepath" "strings" "github.com/lxc/incus/v7/internal/server/storage/s3" ) const ( dataSubdir = "data" uploadsSubdir = ".uploads" ) // Role describes what operations a Credential is permitted to perform. type Role string const ( // RoleAdmin allows all S3 operations on the bucket. RoleAdmin Role = "admin" // RoleReadOnly allows only read operations (GET/HEAD on objects and bucket listing). RoleReadOnly Role = "read-only" ) // Credential is an S3 access-key / secret-key pair authorised against the bucket. type Credential struct { AccessKey string SecretKey string Role Role } // Server serves S3 requests for a single bucket directory. type Server struct { bucketDir string creds []Credential // OnAuthenticated, if set, is invoked once the request has been // authenticated and authorised, before any data on disk is touched. // Errors are returned to the client as an internal-error response and // dispatch is aborted. OnAuthenticated func() error } // NewServer returns a Server rooted at bucketDir. // // bucketDir is the per-bucket directory on the local filesystem. Object data // is read from and written to bucketDir/data/. The directory is created on // demand for object writes. // // creds lists the access-key / secret-key pairs that can authenticate against // the bucket. The first matching access key is used for SigV4 verification. func NewServer(bucketDir string, creds []Credential) *Server { return &Server{ bucketDir: bucketDir, creds: creds, } } func (s *Server) dataDir() string { return filepath.Join(s.bucketDir, dataSubdir) } func (s *Server) uploadsDir() string { return filepath.Join(s.dataDir(), uploadsSubdir) } // ServeHTTP implements http.Handler. // // The bucket name component of the URL is ignored: this handler is scoped to // a single bucket and the caller is expected to have routed by bucket name // already. Routing happens on the remainder of the path. func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Authenticate the request before any I/O. role, authErr := s.authenticate(r) if authErr != nil { authErr.Response(w) return } objectKey := strings.TrimPrefix(r.URL.Path, "/") parts := strings.SplitN(objectKey, "/", 2) if len(parts) == 2 { objectKey = parts[1] } else { objectKey = "" } if !methodAllowedForRole(r.Method, role, objectKey, r.URL.Query()) { (&s3.Error{ Code: s3.ErrorInvalidRequest, Message: "Operation not permitted by credential role.", }).Response(w) return } if s.OnAuthenticated != nil { err := s.OnAuthenticated() if err != nil { (&s3.Error{Code: s3.ErrorCodeInternalError, Message: err.Error()}).Response(w) return } } if objectKey == "" { s.handleBucket(w, r) return } s.handleObject(w, r, objectKey) } func methodAllowedForRole(method string, role Role, objectKey string, q url.Values) bool { // Admin access. if role == RoleAdmin { return true } // Unknown role. if role != RoleReadOnly { return false } // Read-only is limited to GET and HEAD on objects, and GET on the // bucket for listing. Multipart sub-resources are writes. _, ok := q["uploads"] if ok { return false } if q.Get("uploadId") != "" { return false } switch method { case http.MethodGet, http.MethodHead: return true } return false } func (s *Server) handleBucket(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodGet: // ListObjectsV2 (and a few other listings keyed off query parameters). _, ok := r.URL.Query()["uploads"] if ok { s.listMultipartUploads(w, r) return } s.listObjects(w, r) case http.MethodHead: // Bucket exist if we made it this far. w.WriteHeader(http.StatusOK) default: // We don't allow bucket creation/deletion. (&s3.Error{ Code: s3.ErrorInvalidRequest, Message: "Bucket lifecycle is managed by the Incus API.", }).Response(w) } } func (s *Server) handleObject(w http.ResponseWriter, r *http.Request, objectKey string) { q := r.URL.Query() _, ok := q["uploads"] if ok && r.Method == http.MethodPost { s.initiateMultipartUpload(w, r, objectKey) return } uploadID := q.Get("uploadId") if uploadID != "" { switch r.Method { case http.MethodPut: s.uploadPart(w, r, objectKey, uploadID) case http.MethodPost: s.completeMultipartUpload(w, r, objectKey, uploadID) case http.MethodDelete: s.abortMultipartUpload(w, objectKey, uploadID) default: (&s3.Error{Code: s3.ErrorInvalidRequest, Message: "Unsupported method for multipart upload."}).Response(w) } return } switch r.Method { case http.MethodGet: s.getObject(w, r, objectKey) case http.MethodHead: s.headObject(w, r, objectKey) case http.MethodPut: s.putObject(w, r, objectKey) case http.MethodDelete: s.deleteObject(w, objectKey) default: (&s3.Error{Code: s3.ErrorInvalidRequest, Message: "Unsupported method."}).Response(w) } } incus-7.0.0/internal/server/storage/s3/local/xlmeta.go000066400000000000000000000103571517523235500227030ustar00rootroot00000000000000package local import ( "encoding/binary" "errors" "fmt" ) // xlMetaMagic is the four-byte prefix of every xl.meta file written by minio. var xlMetaMagic = [4]byte{'X', 'L', '2', ' '} // xlInlineData parses a minio xl.meta file and returns the inline data blob // for the current object version, if the object is stored inline. // // minio's xl-storage-format wraps the inline data inside a msgpack // structure whose exact shape varies between format revisions. Rather than // decode the entire metadata structure, this function scans the file for // minio's stable inline-section signature: // // byte 0: 0x01 (inline-section format version) // byte 1+: msgpack map[string][]byte (version-id → object bytes) // // The first byte position whose tail parses as a non-empty msgpack map // with a str key and a bin value is taken as the inline data section, // and the value of the first map entry is returned. // // Returns (nil, nil) if the file is well-formed but contains no inline // data section (e.g. the object's data is stored externally as part files). func xlInlineData(b []byte) ([]byte, error) { if len(b) < 12 { return nil, errors.New("xl.meta truncated") } if [4]byte{b[0], b[1], b[2], b[3]} != xlMetaMagic { return nil, errors.New("xl.meta bad magic") } // Search after the 8-byte header for the inline section's signature. // The CRC trailer occupies the last 4 bytes; nothing useful starts // there. for p := 8; p < len(b)-4; p++ { if b[p] != 0x01 { continue } data, ok := tryReadInlineSection(b[p+1:]) if ok { return data, nil } } return nil, nil } // tryReadInlineSection attempts to read a msgpack map[string][]byte at the // start of b and returns the value of the first entry. It returns (nil, // false) on any decoding error. func tryReadInlineSection(b []byte) ([]byte, bool) { r := newMsgpReader(b) count, err := r.readMapLen() if err != nil || count == 0 { return nil, false } _, err = r.readStr() if err != nil { return nil, false } data, err := r.readBin() if err != nil { return nil, false } return data, true } // msgpReader is a minimal msgpack reader supporting only the type families // used by minio xl.meta: bin, str, fixarray, fixmap, array16/32, map16/32. type msgpReader struct { b []byte i int } func newMsgpReader(b []byte) *msgpReader { return &msgpReader{b: b} } func (r *msgpReader) need(n int) error { if r.i+n > len(r.b) { return errors.New("msgpack short read") } return nil } func (r *msgpReader) readByte() (byte, error) { err := r.need(1) if err != nil { return 0, err } c := r.b[r.i] r.i++ return c, nil } func (r *msgpReader) readUint(n int) (uint32, error) { err := r.need(n) if err != nil { return 0, err } var v uint32 switch n { case 1: v = uint32(r.b[r.i]) case 2: v = uint32(binary.BigEndian.Uint16(r.b[r.i:])) case 4: v = binary.BigEndian.Uint32(r.b[r.i:]) } r.i += n return v, nil } func (r *msgpReader) readMapLen() (int, error) { c, err := r.readByte() if err != nil { return 0, err } switch { case c >= 0x80 && c <= 0x8f: return int(c & 0x0f), nil case c == 0xde: v, err := r.readUint(2) return int(v), err case c == 0xdf: v, err := r.readUint(4) return int(v), err } return 0, fmt.Errorf("not a map (0x%02x)", c) } func (r *msgpReader) readBin() ([]byte, error) { c, err := r.readByte() if err != nil { return nil, err } var n uint32 switch c { case 0xc4: n, err = r.readUint(1) case 0xc5: n, err = r.readUint(2) case 0xc6: n, err = r.readUint(4) default: return nil, fmt.Errorf("not a bin (0x%02x)", c) } if err != nil { return nil, err } err = r.need(int(n)) if err != nil { return nil, err } data := r.b[r.i : r.i+int(n)] r.i += int(n) return data, nil } func (r *msgpReader) readStr() (string, error) { c, err := r.readByte() if err != nil { return "", err } var n uint32 switch { case c >= 0xa0 && c <= 0xbf: n = uint32(c & 0x1f) case c == 0xd9: n, err = r.readUint(1) case c == 0xda: n, err = r.readUint(2) case c == 0xdb: n, err = r.readUint(4) default: return "", fmt.Errorf("not a str (0x%02x)", c) } if err != nil { return "", err } err = r.need(int(n)) if err != nil { return "", err } s := string(r.b[r.i : r.i+int(n)]) r.i += int(n) return s, nil } incus-7.0.0/internal/server/storage/s3/policy.go000066400000000000000000000053371517523235500216200ustar00rootroot00000000000000package s3 import ( "encoding/json" "errors" "fmt" "sort" ) const ( roleAdmin = "admin" roleReadOnly = "read-only" ) // Policy defines the S3 policy. type Policy struct { Version string Statement []PolicyStatement } // PolicyStatement defines the S3 policy statement. type PolicyStatement struct { Effect string Action []string Resource []string } // BucketPolicy generates an S3 bucket policy for role. func BucketPolicy(bucketName string, roleName string) (json.RawMessage, error) { switch roleName { case roleAdmin: return fmt.Appendf(nil, `{ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Action": [ "s3:*" ], "Resource": [ "arn:aws:s3:::%s/*" ] }] }`, bucketName), nil case roleReadOnly: return fmt.Appendf(nil, `{ "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Action": [ "s3:ListBucket", "s3:GetBucketLocation", "s3:GetObject", "s3:GetObjectVersion" ], "Resource": [ "arn:aws:s3:::%s/*" ] }] }`, bucketName), nil } return nil, errors.New("Invalid key role") } // BucketPolicyRole compares the given bucket policy with the predefined bucket policies // and returns the role name of the matching policy. func BucketPolicyRole(bucketName string, jsonPolicy string) (string, error) { var policy Policy err := json.Unmarshal([]byte(jsonPolicy), &policy) if err != nil { return "", err } predefinedRoles := []string{roleAdmin, roleReadOnly} for _, role := range predefinedRoles { var rolePolicy Policy jsonRolePolicy, err := BucketPolicy(bucketName, role) if err != nil { return "", err } err = json.Unmarshal([]byte(jsonRolePolicy), &rolePolicy) if err != nil { return "", err } matches := comparePolicy(policy, rolePolicy) if matches { return role, nil } } return "", errors.New("Policy does not match any role") } // comparePolicy checks whether two policies are equal. func comparePolicy(policyA Policy, policyB Policy) bool { if policyA.Version != policyB.Version { return false } if len(policyA.Statement) != len(policyB.Statement) { return false } for i := range policyA.Statement { psA := policyA.Statement[i] psB := policyB.Statement[i] if psA.Effect != psB.Effect { return false } if len(psA.Action) != len(psB.Action) { return false } if len(psA.Resource) != len(psB.Resource) { return false } sort.Strings(psA.Action) sort.Strings(psB.Action) for j := range psA.Action { if psA.Action[j] != psB.Action[j] { return false } } sort.Strings(psB.Resource) sort.Strings(psB.Resource) for j := range psA.Resource { if psA.Resource[j] != psB.Resource[j] { return false } } } return true } incus-7.0.0/internal/server/storage/s3/transfer_manager.go000066400000000000000000000124521517523235500236330ustar00rootroot00000000000000package s3 import ( "context" "errors" "fmt" "io" "net/http" "net/url" "os" "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/lxc/incus/v7/internal/instancewriter" "github.com/lxc/incus/v7/internal/server/backup" "github.com/lxc/incus/v7/internal/server/storage/s3util" "github.com/lxc/incus/v7/shared/logger" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/validate" ) // TransferManager represents a transfer manager. type TransferManager struct { s3URL *url.URL accessKey string secretKey string } // NewTransferManager instantiates a new TransferManager struct. func NewTransferManager(s3URL *url.URL, accessKey string, secretKey string) TransferManager { return TransferManager{ s3URL: s3URL, accessKey: accessKey, secretKey: secretKey, } } // DownloadAllFiles downloads all files from a bucket and writes them to a tar writer. func (t TransferManager) DownloadAllFiles(bucketName string, tarWriter *instancewriter.InstanceTarWriter) error { logger.Debugf("Downloading all files from bucket %s", bucketName) logger.Debugf("Endpoint: %s", t.getEndpoint()) s3Client, err := t.getS3Client() if err != nil { return err } ctx, cancel := context.WithCancel(context.TODO()) defer cancel() paginator := s3.NewListObjectsV2Paginator(s3Client, &s3.ListObjectsV2Input{ Bucket: aws.String(bucketName), }) for paginator.HasMorePages() { page, err := paginator.NextPage(ctx) if err != nil { logger.Errorf("Failed to list objects: %v", err) return err } for _, obj := range page.Contents { key := aws.ToString(obj.Key) // Skip directories because they are part of the key of an actual file if strings.HasSuffix(key, "/") { continue } out, err := s3Client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(bucketName), Key: aws.String(key), }) if err != nil { logger.Errorf("Failed to get object: %v", err) return err } fi := instancewriter.FileInfo{ FileName: fmt.Sprintf("backup/bucket/%s", key), FileSize: aws.ToInt64(obj.Size), FileMode: 0o600, FileModTime: time.Now(), } logger.Debugf("Writing file %s to tar writer", key) logger.Debugf("File size: %d", fi.FileSize) err = tarWriter.WriteFileFromReader(out.Body, &fi) if err != nil { logger.Errorf("Failed to write file to tar writer: %v", err) _ = out.Body.Close() return err } err = out.Body.Close() if err != nil { logger.Errorf("Failed to close object: %v", err) return err } } } return nil } // UploadAllFiles uploads all the provided files to the bucket. func (t TransferManager) UploadAllFiles(bucketName string, srcData io.ReadSeeker) error { logger.Debugf("Uploading all files to bucket %s", bucketName) logger.Debugf("Endpoint: %s", t.getEndpoint()) s3Client, err := t.getS3Client() if err != nil { return err } uploader := transfermanager.New(s3Client) ctx, cancel := context.WithCancel(context.TODO()) defer cancel() // Create temp path and remove it after wards mountPath, err := os.MkdirTemp("", "incus_bucket_import_*") if err != nil { return err } defer func() { _ = os.RemoveAll(mountPath) }() logger.Debugf("Created temp mount path %s", mountPath) tr, cancelFunc, err := backup.TarReader(srcData, nil, mountPath) if err != nil { return err } defer cancelFunc() for { hdr, err := tr.Next() if err != nil { if errors.Is(err, io.EOF) { // End of archive. break } return err } // Skip anything that's not in the bucket itself. if !strings.HasPrefix(hdr.Name, "backup/bucket/") { continue } fileName := strings.TrimPrefix(hdr.Name, "backup/bucket/") _, err = uploader.UploadObject(ctx, &transfermanager.UploadObjectInput{ Bucket: aws.String(bucketName), Key: aws.String(fileName), Body: tr, }) if err != nil { return err } } return nil } func (t TransferManager) getS3Client() (*s3.Client, error) { httpClient := &http.Client{} if t.isSecureEndpoint() { httpClient.Transport = getTransport() } cfg := aws.Config{ Region: s3util.RegionFromURL(t.s3URL), Credentials: credentials.NewStaticCredentialsProvider(t.accessKey, t.secretKey, ""), HTTPClient: httpClient, } scheme := "http" if t.isSecureEndpoint() { scheme = "https" } endpoint := fmt.Sprintf("%s://%s", scheme, t.getEndpoint()) return s3.NewFromConfig(cfg, func(o *s3.Options) { o.BaseEndpoint = aws.String(endpoint) o.UsePathStyle = true }), nil } func (t TransferManager) getEndpoint() string { hostname := t.s3URL.Hostname() if validate.IsNetworkAddressV6(hostname) == nil { hostname = fmt.Sprintf("[%s]", hostname) } return fmt.Sprintf("%s:%s", hostname, t.s3URL.Port()) } func (t TransferManager) isSecureEndpoint() bool { return t.s3URL.Scheme == "https" } func getTransport() *http.Transport { // Get a basic TLS configuration. tlsConfig := localtls.InitTLSConfig() // Skip verification as we're connecting to ourselves on a self-signed certificate. tlsConfig.InsecureSkipVerify = true return &http.Transport{ MaxIdleConns: 10, IdleConnTimeout: 30 * time.Second, DisableCompression: true, TLSClientConfig: tlsConfig, } } incus-7.0.0/internal/server/storage/s3/types.go000066400000000000000000000043361517523235500214630ustar00rootroot00000000000000package s3 import ( "encoding/xml" "net/http" "time" ) // ErrorCodeNoSuchBucket means the specified bucket does not exist. const ErrorCodeNoSuchBucket = "NoSuchBucket" // ErrorCodeInternalError means there was an internal error. const ErrorCodeInternalError = "InternalError" // ErrorCodeInvalidAccessKeyID means there was an invalid access key provided. const ErrorCodeInvalidAccessKeyID = "InvalidAccessKeyId" // ErrorInvalidRequest means there was an invalid request. const ErrorInvalidRequest = "InvalidRequest" var errorHTTPStatusCodes = map[string]int{ ErrorCodeNoSuchBucket: http.StatusNotFound, ErrorCodeInternalError: http.StatusInternalServerError, ErrorCodeInvalidAccessKeyID: http.StatusForbidden, ErrorInvalidRequest: http.StatusBadRequest, } // Error S3 error response. type Error struct { Code string Message string Resource string RequestID string `xml:"RequestId"` BucketName string `xml:"BucketName,omitempty"` HostID string `xml:"HostId"` } // Response writes error as HTTP response. func (r *Error) Response(w http.ResponseWriter) { resp, err := xml.Marshal(r) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/xml") statusCode := errorHTTPStatusCodes[r.Code] if statusCode == 0 { statusCode = http.StatusInternalServerError } w.WriteHeader(statusCode) _, _ = w.Write([]byte(``)) _, _ = w.Write(resp) } // Owner S3 owner. type Owner struct { ID string DisplayName string } // Bucket S3 bucket. type Bucket struct { CreationDate time.Time Name string } // ListAllMyBucketsResult S3 list my buckets. type ListAllMyBucketsResult struct { Owner Owner Buckets []Bucket `xml:"Buckets>Bucket"` } // Response writes error as HTTP response. func (r *ListAllMyBucketsResult) Response(w http.ResponseWriter) { resp, err := xml.Marshal(r) if err != nil { errResult := Error{Code: ErrorCodeInternalError, Message: err.Error()} errResult.Response(w) return } w.Header().Set("Content-Type", "application/xml") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(``)) _, _ = w.Write(resp) } incus-7.0.0/internal/server/storage/s3util/000077500000000000000000000000001517523235500206605ustar00rootroot00000000000000incus-7.0.0/internal/server/storage/s3util/region.go000066400000000000000000000043141517523235500224740ustar00rootroot00000000000000package s3util import ( "net" "net/url" "strings" ) // fallbackRegion is used for S3-compatible endpoints that don't encode an // AWS region in their hostname (Ceph RGW, the Incus in-process handler, // minio-style local servers, etc.). The region is part of the SigV4 // credential scope but isn't otherwise meaningful for these endpoints, // and AWS itself accepts "us-east-1" for legacy global endpoints. const fallbackRegion = "us-east-1" // RegionFromURL returns the AWS region encoded in an S3 endpoint URL, or // fallbackRegion if the URL is not an AWS S3 endpoint. // // Recognised AWS endpoint forms: // // https://s3.amazonaws.com/... -> us-east-1 // https://s3..amazonaws.com/... -> // https://s3-.amazonaws.com/... -> (legacy) // https://.s3.amazonaws.com/... -> us-east-1 // https://.s3..amazonaws.com/... -> // https://.s3-.amazonaws.com/... -> (legacy) func RegionFromURL(u *url.URL) string { if u == nil { return fallbackRegion } // Strip port if present, falling back to the raw host on error // (SplitHostPort errors when there is no port to split). host, _, err := net.SplitHostPort(u.Host) if err != nil { host = u.Host } if !strings.HasSuffix(host, ".amazonaws.com") { return fallbackRegion } prefix := strings.TrimSuffix(host, ".amazonaws.com") parts := strings.Split(prefix, ".") // Find the s3 segment (or s3- legacy form). Anything before // it is a virtual-hosted bucket subdomain we can ignore. s3Idx := -1 for i, p := range parts { if p == "s3" || strings.HasPrefix(p, "s3-") { s3Idx = i break } } if s3Idx == -1 { return fallbackRegion } // Legacy s3- form embeds the region in the segment itself. region := strings.TrimPrefix(parts[s3Idx], "s3-") if region != parts[s3Idx] && region != "" { return region } // Modern s3. form: skip any qualifier segments and return // the next non-qualifier as the region. for _, p := range parts[s3Idx+1:] { switch p { case "dualstack", "fips", "accesspoint": continue } if p != "" { return p } } return fallbackRegion } incus-7.0.0/internal/server/storage/storage.go000066400000000000000000000177471517523235500214500ustar00rootroot00000000000000package storage import ( "context" "fmt" "os" "slices" "sort" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/instance/instancetype" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/state" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/internal/version" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/util" ) // InstancePath returns the directory of an instance or snapshot. func InstancePath(instanceType instancetype.Type, projectName, instanceName string, isSnapshot bool) string { fullName := project.Instance(projectName, instanceName) if instanceType == instancetype.VM { if isSnapshot { return internalUtil.VarPath("virtual-machines-snapshots", fullName) } return internalUtil.VarPath("virtual-machines", fullName) } if isSnapshot { return internalUtil.VarPath("containers-snapshots", fullName) } return internalUtil.VarPath("containers", fullName) } // InstanceImportingFilePath returns the file path used to indicate an instance import is in progress. // This marker file is created when using `incusd import` to import an instance that exists on the storage device // but does not exist in the Incus database. The presence of this file causes the instance not to be removed from // the storage device if the import should fail for some reason. func InstanceImportingFilePath(instanceType instancetype.Type, poolName, projectName, instanceName string) string { fullName := project.Instance(projectName, instanceName) typeDir := "containers" if instanceType == instancetype.VM { typeDir = "virtual-machines" } return internalUtil.VarPath("storage-pools", poolName, typeDir, fullName, ".importing") } // GetStoragePoolMountPoint returns the mountpoint of the given pool. // {INCUS_DIR}/storage-pools/ // Deprecated, use GetPoolMountPath in storage/drivers package. func GetStoragePoolMountPoint(poolName string) string { return internalUtil.VarPath("storage-pools", poolName) } // GetSnapshotMountPoint returns the mountpoint of the given container snapshot. // ${INCUS_DIR}/storage-pools//containers-snapshots/. func GetSnapshotMountPoint(projectName, poolName string, snapshotName string) string { return internalUtil.VarPath("storage-pools", poolName, "containers-snapshots", project.Instance(projectName, snapshotName)) } // GetImageMountPoint returns the mountpoint of the given image. // ${INCUS_DIR}/storage-pools//images/. func GetImageMountPoint(poolName string, fingerprint string) string { return internalUtil.VarPath("storage-pools", poolName, "images", fingerprint) } // GetStoragePoolVolumeSnapshotMountPoint returns the mountpoint of the given pool volume snapshot. // ${INCUS_DIR}/storage-pools//custom-snapshots//. func GetStoragePoolVolumeSnapshotMountPoint(poolName string, snapshotName string) string { return internalUtil.VarPath("storage-pools", poolName, "custom-snapshots", snapshotName) } // CreateContainerMountpoint creates the provided container mountpoint and symlink. func CreateContainerMountpoint(mountPoint string, mountPointSymlink string, privileged bool) error { mntPointSymlinkExist := util.PathExists(mountPointSymlink) mntPointSymlinkTargetExist := util.PathExists(mountPoint) var err error if !mntPointSymlinkTargetExist { err = os.MkdirAll(mountPoint, 0o711) if err != nil { return err } } err = os.Chmod(mountPoint, 0o100) if err != nil { return err } if !mntPointSymlinkExist { err := os.Symlink(mountPoint, mountPointSymlink) if err != nil { return err } } return nil } // CreateSnapshotMountpoint creates the provided container snapshot mountpoint // and symlink. func CreateSnapshotMountpoint(snapshotMountpoint string, snapshotsSymlinkTarget string, snapshotsSymlink string) error { snapshotMntPointExists := util.PathExists(snapshotMountpoint) mntPointSymlinkExist := util.PathExists(snapshotsSymlink) if !snapshotMntPointExists { err := os.MkdirAll(snapshotMountpoint, 0o711) if err != nil { return err } } if !mntPointSymlinkExist { err := os.Symlink(snapshotsSymlinkTarget, snapshotsSymlink) if err != nil { return err } } return nil } // UsedBy returns list of API resources using storage pool. Accepts firstOnly argument to indicate that only the // first resource using network should be returned. This can help to quickly check if the storage pool is in use. // If memberSpecific is true, then the search is restricted to volumes that belong to this member or belong to // all members. The ignoreVolumeType argument can be used to exclude certain volume type(s) from the list. func UsedBy(ctx context.Context, s *state.State, pool Pool, firstOnly bool, memberSpecific bool, ignoreVolumeType ...string) ([]string, error) { var err error var usedBy []string err = s.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Get all the volumes using the storage pool. volumes, err := tx.GetStoragePoolVolumes(ctx, pool.ID(), memberSpecific) if err != nil { return fmt.Errorf("Failed loading storage volumes: %w", err) } for _, vol := range volumes { var u *api.URL if slices.Contains(ignoreVolumeType, vol.Type) { continue } // Generate URL for volume based on types that map to other entities. if vol.Type == db.StoragePoolVolumeTypeNameContainer || vol.Type == db.StoragePoolVolumeTypeNameVM { volName, snapName, isSnap := api.GetParentAndSnapshotName(vol.Name) if isSnap { u = api.NewURL().Path(version.APIVersion, "instances", volName, "snapshots", snapName).Project(vol.Project) } else { u = api.NewURL().Path(version.APIVersion, "instances", volName).Project(vol.Project) } usedBy = append(usedBy, u.String()) } else if vol.Type == db.StoragePoolVolumeTypeNameImage { imgProjectNames, err := tx.GetProjectsUsingImage(ctx, vol.Name) if err != nil { return fmt.Errorf("Failed loading projects using image %q: %w", vol.Name, err) } if len(imgProjectNames) > 0 { for _, imgProjectName := range imgProjectNames { u = api.NewURL().Path(version.APIVersion, "images", vol.Name).Project(imgProjectName).Target(vol.Location) usedBy = append(usedBy, u.String()) } } else { // Handle orphaned image volumes that are not associated to an image. u = vol.URL(version.APIVersion, pool.Name()) usedBy = append(usedBy, u.String()) } } else { u = vol.URL(version.APIVersion, pool.Name()) usedBy = append(usedBy, u.String()) } if firstOnly { return nil } } // Get all buckets using the storage pool. poolID := pool.ID() filters := []db.StorageBucketFilter{{ PoolID: &poolID, }} buckets, err := tx.GetStoragePoolBuckets(ctx, memberSpecific, filters...) if err != nil { return fmt.Errorf("Failed loading storage buckets: %w", err) } for _, bucket := range buckets { u := bucket.URL(version.APIVersion, pool.Name(), bucket.Project) usedBy = append(usedBy, u.String()) if firstOnly { return nil } } // Get all the profiles using the storage pool. profiles, err := cluster.GetProfiles(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading profiles: %w", err) } // Get all the profile devices. profileDevices, err := cluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading profile devices: %w", err) } for _, profile := range profiles { for _, device := range profileDevices[profile.ID] { if device.Type != cluster.TypeDisk { continue } if device.Config["pool"] != pool.Name() { continue } u := api.NewURL().Path(version.APIVersion, "profiles", profile.Name).Project(profile.Project) usedBy = append(usedBy, u.String()) if firstOnly { return nil } break } } return err }) if err != nil { return nil, err } sort.Strings(usedBy) return usedBy, nil } incus-7.0.0/internal/server/storage/utils.go000066400000000000000000001467471517523235500211470ustar00rootroot00000000000000package storage import ( "context" "encoding/json" "errors" "fmt" "io/fs" "os" "path/filepath" "slices" "strings" "time" "golang.org/x/sys/unix" internalInstance "github.com/lxc/incus/v7/internal/instance" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/migration" "github.com/lxc/incus/v7/internal/rsync" "github.com/lxc/incus/v7/internal/server/apparmor" backupConfig "github.com/lxc/incus/v7/internal/server/backup/config" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/instance" "github.com/lxc/incus/v7/internal/server/instance/instancetype" localMigration "github.com/lxc/incus/v7/internal/server/migration" "github.com/lxc/incus/v7/internal/server/node" "github.com/lxc/incus/v7/internal/server/operations" "github.com/lxc/incus/v7/internal/server/project" "github.com/lxc/incus/v7/internal/server/response" "github.com/lxc/incus/v7/internal/server/state" "github.com/lxc/incus/v7/internal/server/storage/drivers" "github.com/lxc/incus/v7/internal/server/sys" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/archive" "github.com/lxc/incus/v7/shared/ioprogress" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" "github.com/lxc/incus/v7/shared/validate" ) // ConfigDiff returns a diff of the provided configs. Additionally, it returns whether or not // only user properties have been changed. func ConfigDiff(oldConfig map[string]string, newConfig map[string]string) ([]string, bool) { changedConfig := []string{} userOnly := true for key := range oldConfig { if oldConfig[key] != newConfig[key] { if !strings.HasPrefix(key, "user.") { userOnly = false } if !slices.Contains(changedConfig, key) { changedConfig = append(changedConfig, key) } } } for key := range newConfig { if oldConfig[key] != newConfig[key] { if !strings.HasPrefix(key, "user.") { userOnly = false } if !slices.Contains(changedConfig, key) { changedConfig = append(changedConfig, key) } } } // Skip on no change if len(changedConfig) == 0 { return nil, false } return changedConfig, userOnly } // VolumeTypeNameToDBType converts a volume type string to internal volume type DB code. func VolumeTypeNameToDBType(volumeTypeName string) (int, error) { switch volumeTypeName { case db.StoragePoolVolumeTypeNameContainer: return db.StoragePoolVolumeTypeContainer, nil case db.StoragePoolVolumeTypeNameVM: return db.StoragePoolVolumeTypeVM, nil case db.StoragePoolVolumeTypeNameImage: return db.StoragePoolVolumeTypeImage, nil case db.StoragePoolVolumeTypeNameCustom: return db.StoragePoolVolumeTypeCustom, nil } return -1, errors.New("Invalid storage volume type name") } // VolumeTypeToDBType converts volume type to internal volume type DB code. func VolumeTypeToDBType(volType drivers.VolumeType) (int, error) { switch volType { case drivers.VolumeTypeContainer: return db.StoragePoolVolumeTypeContainer, nil case drivers.VolumeTypeVM: return db.StoragePoolVolumeTypeVM, nil case drivers.VolumeTypeImage: return db.StoragePoolVolumeTypeImage, nil case drivers.VolumeTypeCustom: return db.StoragePoolVolumeTypeCustom, nil } return -1, fmt.Errorf("Invalid storage volume type: %q", volType) } // VolumeDBTypeToType converts internal volume type DB code to storage driver volume type. func VolumeDBTypeToType(volDBType int) (drivers.VolumeType, error) { switch volDBType { case db.StoragePoolVolumeTypeContainer: return drivers.VolumeTypeContainer, nil case db.StoragePoolVolumeTypeVM: return drivers.VolumeTypeVM, nil case db.StoragePoolVolumeTypeImage: return drivers.VolumeTypeImage, nil case db.StoragePoolVolumeTypeCustom: return drivers.VolumeTypeCustom, nil } return "", fmt.Errorf("Invalid storage volume DB type: %d", volDBType) } // InstanceTypeToVolumeType converts instance type to storage driver volume type. func InstanceTypeToVolumeType(instType instancetype.Type) (drivers.VolumeType, error) { switch instType { case instancetype.Container: return drivers.VolumeTypeContainer, nil case instancetype.VM: return drivers.VolumeTypeVM, nil } return "", errors.New("Invalid instance type") } // VolumeTypeToAPIInstanceType converts storage driver volume type to API instance type type. func VolumeTypeToAPIInstanceType(volType drivers.VolumeType) (api.InstanceType, error) { switch volType { case drivers.VolumeTypeContainer: return api.InstanceTypeContainer, nil case drivers.VolumeTypeVM: return api.InstanceTypeVM, nil } return api.InstanceTypeAny, errors.New("Volume type doesn't have equivalent instance type") } // VolumeContentTypeToDBContentType converts volume type to internal code. func VolumeContentTypeToDBContentType(contentType drivers.ContentType) (int, error) { switch contentType { case drivers.ContentTypeBlock: return db.StoragePoolVolumeContentTypeBlock, nil case drivers.ContentTypeFS: return db.StoragePoolVolumeContentTypeFS, nil case drivers.ContentTypeISO: return db.StoragePoolVolumeContentTypeISO, nil } return -1, errors.New("Invalid volume content type") } // VolumeDBContentTypeToContentType converts internal content type DB code to driver representation. func VolumeDBContentTypeToContentType(volDBType int) (drivers.ContentType, error) { switch volDBType { case db.StoragePoolVolumeContentTypeBlock: return drivers.ContentTypeBlock, nil case db.StoragePoolVolumeContentTypeFS: return drivers.ContentTypeFS, nil case db.StoragePoolVolumeContentTypeISO: return drivers.ContentTypeISO, nil } return "", errors.New("Invalid volume content type") } // VolumeContentTypeNameToContentType converts volume content type string internal code. func VolumeContentTypeNameToContentType(contentTypeName string) (int, error) { switch contentTypeName { case db.StoragePoolVolumeContentTypeNameFS: return db.StoragePoolVolumeContentTypeFS, nil case db.StoragePoolVolumeContentTypeNameBlock: return db.StoragePoolVolumeContentTypeBlock, nil case db.StoragePoolVolumeContentTypeNameISO: return db.StoragePoolVolumeContentTypeISO, nil } return -1, errors.New("Invalid volume content type name") } // VolumeDBGet loads a volume from the database. func VolumeDBGet(pool Pool, projectName string, volumeName string, volumeType drivers.VolumeType) (*db.StorageVolume, error) { p, ok := pool.(*backend) if !ok { return nil, errors.New("Pool is not a backend") } volDBType, err := VolumeTypeToDBType(volumeType) if err != nil { return nil, err } // Get the storage volume. var dbVolume *db.StorageVolume err = p.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { dbVolume, err = tx.GetStoragePoolVolume(ctx, pool.ID(), projectName, volDBType, volumeName, true) if err != nil { if response.IsNotFoundError(err) { return fmt.Errorf("Storage volume %q in project %q of type %q does not exist on pool %q: %w", volumeName, projectName, volumeType, pool.Name(), err) } return err } return nil }) if err != nil { return nil, err } return dbVolume, nil } // VolumeDBCreate creates a volume in the database. // If volumeConfig is supplied, it is modified with any driver level default config options (if not set). // If removeUnknownKeys is true, any unknown config keys are removed from volumeConfig rather than failing. func VolumeDBCreate(pool Pool, projectName string, volumeName string, volumeDescription string, volumeType drivers.VolumeType, snapshot bool, volumeConfig map[string]string, creationDate time.Time, expiryDate time.Time, contentType drivers.ContentType, removeUnknownKeys bool, hasSource bool) error { p, ok := pool.(*backend) if !ok { return errors.New("Pool is not a backend") } // Prevent using this function to create storage volume bucket records. if volumeType == drivers.VolumeTypeBucket { return errors.New("Cannot store volume using bucket type") } // If the volumeType represents an instance type then check that the volumeConfig doesn't contain any of // the instance disk effective override fields (which should not be stored in the database). if volumeType.IsInstance() { for _, k := range instanceDiskVolumeEffectiveFields { _, found := volumeConfig[k] if found { return fmt.Errorf("Instance disk effective override field %q should not be stored in volume config", k) } } } // Convert the volume type to our internal integer representation. volDBType, err := VolumeTypeToDBType(volumeType) if err != nil { return err } volDBContentType, err := VolumeContentTypeToDBContentType(contentType) if err != nil { return err } // Make sure that we don't pass a nil to the next function. if volumeConfig == nil { volumeConfig = map[string]string{} } volType, err := VolumeDBTypeToType(volDBType) if err != nil { return err } vol := drivers.NewVolume(pool.Driver(), pool.Name(), volType, contentType, volumeName, volumeConfig, pool.Driver().Config()) // Set source indicator. vol.SetHasSource(hasSource) // For new volumes, fill default config. if !snapshot { err = pool.Driver().FillVolumeConfig(vol) if err != nil { return err } } // Validate config. err = pool.Driver().ValidateVolume(vol, removeUnknownKeys) if err != nil { return err } err = p.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { // Create the database entry for the storage volume. if snapshot { _, err = tx.CreateStorageVolumeSnapshot(ctx, projectName, volumeName, volumeDescription, volDBType, pool.ID(), vol.Config(), creationDate, expiryDate) } else { _, err = tx.CreateStoragePoolVolume(ctx, projectName, volumeName, volumeDescription, volDBType, pool.ID(), vol.Config(), volDBContentType, creationDate) } return err }) if err != nil { return fmt.Errorf("Error inserting volume %q for project %q in pool %q of type %q into database %q", volumeName, projectName, pool.Name(), volumeType, err) } return nil } // VolumeDBDelete deletes a volume from the database. func VolumeDBDelete(pool Pool, projectName string, volumeName string, volumeType drivers.VolumeType) error { p, ok := pool.(*backend) if !ok { return errors.New("Pool is not a backend") } // Convert the volume type to our internal integer representation. volDBType, err := VolumeTypeToDBType(volumeType) if err != nil { return err } err = p.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.RemoveStoragePoolVolume(ctx, projectName, volumeName, volDBType, pool.ID()) }) if err != nil && !response.IsNotFoundError(err) { return fmt.Errorf("Error deleting storage volume from database: %w", err) } return nil } // VolumeDBSnapshotsGet loads a list of snapshots volumes from the database. func VolumeDBSnapshotsGet(pool Pool, projectName string, volume string, volumeType drivers.VolumeType) ([]db.StorageVolumeArgs, error) { p, ok := pool.(*backend) if !ok { return nil, errors.New("Pool is not a backend") } volDBType, err := VolumeTypeToDBType(volumeType) if err != nil { return nil, err } var snapshots []db.StorageVolumeArgs err = p.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { snapshots, err = tx.GetLocalStoragePoolVolumeSnapshotsWithType(ctx, projectName, volume, volDBType, pool.ID()) return err }) if err != nil { return nil, err } return snapshots, nil } // BucketDBGet loads a bucket from the database. func BucketDBGet(pool Pool, projectName string, bucketName string, memberSpecific bool) (*db.StorageBucket, error) { p, ok := pool.(*backend) if !ok { return nil, errors.New("Pool is not a backend") } var err error var bucket *db.StorageBucket // Get the storage bucket. err = p.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { bucket, err = tx.GetStoragePoolBucket(ctx, pool.ID(), projectName, memberSpecific, bucketName) if err != nil { if response.IsNotFoundError(err) { return fmt.Errorf("Storage bucket %q in project %q does not exist on pool %q: %w", bucketName, projectName, pool.Name(), err) } return err } return nil }) if err != nil { return nil, err } return bucket, nil } // BucketDBCreate creates a bucket in the database. // The supplied bucket's config may be modified with defaults for the storage pool being used. // Returns bucket DB record ID. func BucketDBCreate(ctx context.Context, pool Pool, projectName string, memberSpecific bool, bucket *api.StorageBucketsPost) (int64, error) { p, ok := pool.(*backend) if !ok { return -1, errors.New("Pool is not a backend") } // Make sure that we don't pass a nil to the next function. if bucket.Config == nil { bucket.Config = map[string]string{} } bucketVolName := project.StorageVolume(projectName, bucket.Name) bucketVol := drivers.NewVolume(pool.Driver(), pool.Name(), drivers.VolumeTypeBucket, drivers.ContentTypeFS, bucketVolName, bucket.Config, pool.Driver().Config()) // Fill default config. err := pool.Driver().FillVolumeConfig(bucketVol) if err != nil { return -1, err } // Validate bucket name. err = pool.Driver().ValidateBucket(bucketVol) if err != nil { return -1, err } // Validate bucket volume config. err = pool.Driver().ValidateVolume(bucketVol, false) if err != nil { return -1, err } var bucketID int64 err = p.state.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { // Create the database entry for the storage bucket. bucketID, err = tx.CreateStoragePoolBucket(ctx, p.ID(), projectName, memberSpecific, *bucket) return err }) if err != nil { return -1, fmt.Errorf("Failed inserting storage bucket %q for project %q in pool %q into database: %w", bucket.Name, projectName, pool.Name(), err) } return bucketID, nil } // BucketDBDelete deletes a bucket from the database. func BucketDBDelete(ctx context.Context, pool Pool, bucketID int64) error { p, ok := pool.(*backend) if !ok { return errors.New("Pool is not a backend") } err := p.state.DB.Cluster.Transaction(ctx, func(ctx context.Context, tx *db.ClusterTx) error { return tx.DeleteStoragePoolBucket(ctx, p.ID(), bucketID) }) if err != nil && !response.IsNotFoundError(err) { return fmt.Errorf("Failed deleting storage bucket from database: %w", err) } return nil } // BucketKeysDBGet loads the keys for a bucket from the database. func BucketKeysDBGet(pool Pool, bucketID int64) ([]*db.StorageBucketKey, error) { p, ok := pool.(*backend) if !ok { return nil, errors.New("Pool is not a backend") } var err error var keys []*db.StorageBucketKey // Get the storage bucket keys. err = p.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { keys, err = tx.GetStoragePoolBucketKeys(ctx, bucketID) if err != nil { return err } return nil }) if err != nil { return nil, err } return keys, nil } // poolAndVolumeCommonRules returns a map of pool and volume config common rules common to all drivers. // When vol argument is nil function returns pool specific rules. func poolAndVolumeCommonRules(vol *drivers.Volume) map[string]func(string) error { rules := map[string]func(string) error{ // Note: size should not be modifiable for non-custom volumes and should be checked // in the relevant volume update functions. "size": validate.Optional(validate.IsSize), "snapshots.expiry": func(value string) error { // Validate expression _, err := internalInstance.GetExpiry(time.Time{}, value) return err }, "snapshots.expiry.manual": func(value string) error { // Validate expression _, err := internalInstance.GetExpiry(time.Time{}, value) return err }, "snapshots.schedule": validate.Optional(validate.IsCron([]string{"@hourly", "@daily", "@midnight", "@weekly", "@monthly", "@annually", "@yearly"})), "snapshots.pattern": validate.IsAny, } // Options relevant for custom filesystem volumes. if (vol == nil) || (vol != nil && vol.Type() == drivers.VolumeTypeCustom && vol.ContentType() == drivers.ContentTypeFS) { rules["security.shifted"] = validate.Optional(validate.IsBool) rules["security.unmapped"] = validate.Optional(validate.IsBool) rules["initial.uid"] = validate.Optional(validate.IsInt64) rules["initial.gid"] = validate.Optional(validate.IsInt64) rules["initial.mode"] = validate.Optional(validate.IsInt64) } // security.shared is only relevant for custom block volumes. if (vol == nil) || (vol != nil && vol.Type() == drivers.VolumeTypeCustom && vol.ContentType() == drivers.ContentTypeBlock) { rules["security.shared"] = validate.Optional(validate.IsBool) } return rules } // validatePoolCommonRules returns a map of pool config rules common to all drivers. func validatePoolCommonRules() map[string]func(string) error { rules := map[string]func(string) error{ "source": validate.IsAny, "source.wipe": validate.Optional(validate.IsBool), "volatile.initial_source": validate.IsAny, "rsync.bwlimit": validate.Optional(validate.IsSize), "rsync.compression": validate.Optional(validate.IsBool), } // Add to pool config rules (prefixed with volume.*) which are common for pool and volume. for volRule, volValidator := range poolAndVolumeCommonRules(nil) { rules[fmt.Sprintf("volume.%s", volRule)] = volValidator } return rules } // validateVolumeCommonRules returns a map of volume config rules common to all drivers. func validateVolumeCommonRules(vol drivers.Volume) map[string]func(string) error { rules := poolAndVolumeCommonRules(&vol) // volatile.idmap settings only make sense for filesystem volumes. if vol.ContentType() == drivers.ContentTypeFS { rules["volatile.idmap.last"] = validate.IsAny rules["volatile.idmap.next"] = validate.IsAny } // block.mount_options and block.filesystem settings are only relevant for drivers that are block backed // and when there is a filesystem to actually mount. This includes filesystem volumes and VM Block volumes, // as they have an associated config filesystem volume that shares the config. if vol.IsBlockBacked() && (vol.ContentType() == drivers.ContentTypeFS || vol.IsVMBlock()) { rules["block.mount_options"] = validate.IsAny // Note: block.filesystem should not be modifiable after volume created. // This should be checked in the relevant volume update functions. rules["block.filesystem"] = validate.IsAny } // volatile.rootfs.size is only used for image volumes. if vol.Type() == drivers.VolumeTypeImage { rules["volatile.rootfs.size"] = validate.Optional(validate.IsInt64) } if vol.Type() == drivers.VolumeTypeCustom { rules["dependent"] = validate.Optional(validate.IsBool) } return rules } // ImageUnpack unpacks a filesystem image into the destination path. // There are several formats that images can come in: // Container Format A: Separate metadata tarball and root squashfs file. // - Unpack metadata tarball into mountPath. // - Unpack root squashfs file into mountPath/rootfs. // // Container Format B: Combined tarball containing metadata files and root squashfs. // - Unpack combined tarball into mountPath. // // VM Format A: Separate metadata tarball and root qcow2 file. // - Unpack metadata tarball into mountPath. // - Check rootBlockPath is a file and convert qcow2 file into raw format in rootBlockPath. func ImageUnpack(imageFile string, vol drivers.Volume, destBlockFile string, sysOS *sys.OS, allowUnsafeResize bool, targetIsZero bool, tracker *ioprogress.ProgressTracker, targetFormat string) (int64, error) { l := logger.Log.AddContext(logger.Ctx{"imageFile": imageFile, "volName": vol.Name()}) l.Info("Image unpack started") defer l.Info("Image unpack stopped") // Check available memory. maxMemory, err := linux.DeviceTotalMemory() if err == nil { // Cap the memory to 10%. maxMemory = maxMemory / 10 } else { maxMemory = 0 } // For all formats, first unpack the metadata (or combined) tarball into destPath. imageRootfsFile := imageFile + ".rootfs" destPath := vol.MountPath() // If no destBlockFile supplied then this is a container image unpack. if destBlockFile == "" { rootfsPath := filepath.Join(destPath, "rootfs") // Unpack the main image file. err := archive.Unpack(imageFile, destPath, vol.IsBlockBacked(), maxMemory, tracker) if err != nil { return -1, err } // Check for separate root file. if util.PathExists(imageRootfsFile) { err = os.MkdirAll(rootfsPath, 0o755) if err != nil { return -1, errors.New("Error creating rootfs directory") } err = archive.Unpack(imageRootfsFile, rootfsPath, vol.IsBlockBacked(), maxMemory, tracker) if err != nil { return -1, err } } // Check that the container image unpack has resulted in a rootfs dir. if !util.PathExists(rootfsPath) { return -1, fmt.Errorf("Image is missing a rootfs: %s", imageFile) } // Done with this. return 0, nil } // If a rootBlockPath is supplied then this is a VM image unpack. // Validate the target. fileInfo, err := os.Stat(destBlockFile) if err != nil && !errors.Is(err, fs.ErrNotExist) { return -1, err } if fileInfo != nil && fileInfo.IsDir() { // If the dest block file exists, and it is a directory, fail. return -1, fmt.Errorf("Root block path isn't a file: %s", destBlockFile) } // convertBlockImage converts the qcow2 block image file into a raw block device. If needed it will attempt // to enlarge the destination volume to accommodate the unpacked qcow2 image file. convertBlockImage := func(v drivers.Volume, imgPath string, dstPath string, tracker *ioprogress.ProgressTracker) (int64, error) { // Get info about qcow2 file. Force input format to qcow2 so we don't rely on qemu-img's detection // logic as that has been known to have vulnerabilities and we only support qcow2 images anyway. // Use prlimit because qemu-img can consume considerable RAM & CPU time if fed a maliciously // crafted disk image. Since cloud tenants are not to be trusted, ensure QEMU is limits to 1 GiB // address space and 2 seconds CPU time, which ought to be more than enough for real world images. cmd := []string{"prlimit", "--cpu=2", "--as=1073741824", "qemu-img", "info", "-f", "qcow2", "--output=json", imgPath} imgJSON, err := apparmor.QemuImg(sysOS, cmd, imgPath, dstPath, tracker) if err != nil { return -1, fmt.Errorf("Failed reading image info %q: %w", imgPath, err) } imgInfo := struct { Format string `json:"format"` VirtualSize int64 `json:"virtual-size"` }{} err = json.Unmarshal([]byte(imgJSON), &imgInfo) if err != nil { return -1, fmt.Errorf("Failed unmarshalling image info %q: %w (%q)", imgPath, err, imgJSON) } // Belt and braces qcow2 check. if imgInfo.Format != drivers.BlockVolumeTypeQcow2 { return -1, fmt.Errorf("Unexpected image format %q", imgInfo.Format) } // Check whether image is allowed to be unpacked into pool volume. Create a partial image volume // struct and then use it to check that target volume size can be set as needed. imgVolConfig := map[string]string{ "volatile.rootfs.size": fmt.Sprintf("%d", imgInfo.VirtualSize), } imgVol := drivers.NewVolume(nil, "", drivers.VolumeTypeImage, drivers.ContentTypeBlock, "", imgVolConfig, nil) l.Debug("Checking image unpack size") newVolSize, err := vol.ConfigSizeFromSource(imgVol) if err != nil { return -1, err } if util.PathExists(dstPath) { volSizeBytes, err := drivers.BlockDiskSizeBytes(dstPath) if err != nil { return -1, fmt.Errorf("Error getting current size of %q: %w", dstPath, err) } // If the target volume's size is smaller than the image unpack size, then we need to // increase the target volume's size. if volSizeBytes < imgInfo.VirtualSize { l.Debug("Increasing volume size", logger.Ctx{"imgPath": imgPath, "dstPath": dstPath, "oldSize": volSizeBytes, "newSize": newVolSize, "allowUnsafeResize": allowUnsafeResize}) err = vol.SetQuota(newVolSize, allowUnsafeResize, nil) if err != nil { return -1, fmt.Errorf("Error increasing volume size: %w", err) } } } if targetFormat == drivers.BlockVolumeTypeQcow2 { l.Debug("Writing qcow2 image to disk", logger.Ctx{"imgPath": imgPath, "dstPath": dstPath}) // Attempt to deref all paths. imgFullPath, err := filepath.EvalSymlinks(imgPath) if err == nil { imgPath = imgFullPath } if dstPath != "" { dstFullPath, err := filepath.EvalSymlinks(dstPath) if err == nil { dstPath = dstFullPath } } from, err := os.OpenFile(imgPath, unix.O_RDONLY, 0) if err != nil { return -1, err } defer from.Close() to, err := os.OpenFile(dstPath, unix.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0) if err != nil { return -1, err } defer to.Close() _, err = util.SafeCopy(to, from) if err != nil { return -1, err } return imgInfo.VirtualSize, nil } // Convert the qcow2 format to a raw block device. l.Debug("Converting qcow2 image to raw disk", logger.Ctx{"imgPath": imgPath, "dstPath": dstPath}) cmd = []string{ "nice", "-n19", // Run with low priority to reduce CPU impact on other processes. "qemu-img", "convert", "-p", "-f", "qcow2", "-O", "raw", "-t", "writeback", } // Check for Direct I/O support. from, err := os.OpenFile(imgPath, unix.O_DIRECT|unix.O_RDONLY, 0) if err == nil { cmd = append(cmd, "-T", "none") _ = from.Close() } to, err := os.OpenFile(dstPath, unix.O_DIRECT|unix.O_RDONLY, 0) if err == nil { cmd = append(cmd, "-t", "none") _ = to.Close() } // Extra options when dealing with block devices. if linux.IsBlockdevPath(dstPath) { // Parallel unpacking. cmd = append(cmd, "-W") if targetIsZero { // Our block devices are clean, so skip zeroes. cmd = append(cmd, "-n", "--target-is-zero") } } cmd = append(cmd, imgPath, dstPath) _, err = apparmor.QemuImg(sysOS, cmd, imgPath, dstPath, tracker) if err != nil { return -1, fmt.Errorf("Failed converting image to raw at %q: %w", dstPath, err) } return imgInfo.VirtualSize, nil } var imgSize int64 if util.PathExists(imageRootfsFile) { // Unpack the main image file. err := archive.Unpack(imageFile, destPath, vol.IsBlockBacked(), maxMemory, tracker) if err != nil { return -1, err } // Convert the qcow2 format to a raw block device. imgSize, err = convertBlockImage(vol, imageRootfsFile, destBlockFile, tracker) if err != nil { return -1, err } } else { // Dealing with unified tarballs require an initial unpack to a temporary directory. tempDir, err := os.MkdirTemp(internalUtil.VarPath("images"), "incus_image_unpack_") if err != nil { return -1, err } defer func() { _ = os.RemoveAll(tempDir) }() // Unpack the whole image. err = archive.Unpack(imageFile, tempDir, vol.IsBlockBacked(), maxMemory, tracker) if err != nil { return -1, err } imgPath := filepath.Join(tempDir, "rootfs.img") // Convert the qcow2 format to a raw block device. imgSize, err = convertBlockImage(vol, imgPath, destBlockFile, tracker) if err != nil { return -1, err } // Delete the qcow2. err = os.Remove(imgPath) if err != nil { return -1, fmt.Errorf("Failed to remove %q: %w", imgPath, err) } // Transfer the content excluding the destBlockFile name so that we don't delete the block file // created above if the storage driver stores image files in the same directory as destPath. _, err = rsync.LocalCopy(tempDir, destPath, "", true, "--exclude", filepath.Base(destBlockFile)) if err != nil { return -1, err } } return imgSize, nil } // InstanceContentType returns the instance's content type. func InstanceContentType(inst instance.ConfigReader) drivers.ContentType { contentType := drivers.ContentTypeFS if inst.Type() == instancetype.VM { contentType = drivers.ContentTypeBlock } return contentType } // VolumeUsedByProfileDevices finds profiles using a volume and passes them to profileFunc for evaluation. // The profileFunc is provided with a profile config, project config and a list of device names that are using // the volume. func VolumeUsedByProfileDevices(s *state.State, poolName string, projectName string, vol *api.StorageVolume, profileFunc func(profileID int64, profile api.Profile, project api.Project, usedByDevices []string) error) error { // Convert the volume type name to our internal integer representation. volumeType, err := VolumeTypeNameToDBType(vol.Type) if err != nil { return err } var profiles []api.Profile var profileIDs []int64 var profileProjects []*api.Project // Retrieve required info from the database in single transaction for performance. err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { projects, err := cluster.GetProjects(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading projects: %w", err) } // Index of all projects by name. projectMap := make(map[string]*api.Project, len(projects)) for _, project := range projects { projectMap[project.Name], err = project.ToAPI(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading config for project %q: %w", project.Name, err) } } dbProfiles, err := cluster.GetProfiles(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading profiles: %w", err) } // Get all the profile configs. profileConfigs, err := cluster.GetAllProfileConfigs(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading profile configs: %w", err) } // Get all the profile devices. profileDevices, err := cluster.GetAllProfileDevices(ctx, tx.Tx()) if err != nil { return fmt.Errorf("Failed loading profile devices: %w", err) } for _, profile := range dbProfiles { apiProfile, err := profile.ToAPI(ctx, tx.Tx(), profileConfigs, profileDevices) if err != nil { return fmt.Errorf("Failed getting API Profile %q: %w", profile.Name, err) } profileIDs = append(profileIDs, int64(profile.ID)) profiles = append(profiles, *apiProfile) } profileProjects = make([]*api.Project, len(dbProfiles)) for i, p := range dbProfiles { profileProjects[i] = projectMap[p.Project] } return nil }) if err != nil { return err } // Iterate all profiles, consider only those which belong to a project that has the same effective // storage project as volume. for i, profile := range profiles { profileStorageProject := project.StorageVolumeProjectFromRecord(profileProjects[i], volumeType) if err != nil { return err } // Check profile's storage project is the same as the volume's project. // If not then the volume names mentioned in the profile's config cannot be referring to volumes // in the volume's project we are trying to match, and this profile cannot possibly be using it. if projectName != profileStorageProject { continue } var usedByDevices []string // Iterate through each of the profiles's devices, looking for disks in the same pool as volume. // Then try and match the volume name against the profile device's "source" property. for name, dev := range profile.Devices { if dev["type"] != cluster.TypeDisk.String() { continue } if dev["pool"] != poolName { continue } if strings.Split(dev["source"], "/")[0] == vol.Name { usedByDevices = append(usedByDevices, name) } } if len(usedByDevices) > 0 { err = profileFunc(profileIDs[i], profile, *profileProjects[i], usedByDevices) if err != nil { return err } } } return nil } // VolumeUsedByInstanceDevices finds instances using a volume (either directly or via their expanded profiles if // expandDevices is true) and passes them to instanceFunc for evaluation. If instanceFunc returns an error then it // is returned immediately. The instanceFunc is executed during a DB transaction, so DB queries are not permitted. // The instanceFunc is provided with a instance config, project config, instance's profiles and a list of device // names that are using the volume. func VolumeUsedByInstanceDevices(s *state.State, poolName string, projectName string, vol *api.StorageVolume, expandDevices bool, instanceFunc func(inst db.InstanceArgs, project api.Project, usedByDevices []string) error) error { // Convert the volume type name to our internal integer representation. volumeType, err := VolumeTypeNameToDBType(vol.Type) if err != nil { return err } return s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { return tx.InstanceList(ctx, func(inst db.InstanceArgs, p api.Project) error { // If the volume has a specific cluster member which is different than the instance then skip as // instance cannot be using this volume. if vol.Location != "" && inst.Node != vol.Location { return nil } instStorageProject := project.StorageVolumeProjectFromRecord(&p, volumeType) if err != nil { return err } // Check instance's storage project is the same as the volume's project. // If not then the volume names mentioned in the instance's config cannot be referring to volumes // in the volume's project we are trying to match, and this instance cannot possibly be using it. if projectName != instStorageProject { return nil } // Use local devices for usage check by if expandDevices is false (but don't modify instance). devices := inst.Devices // Expand devices for usage check if expandDevices is true. if expandDevices { devices = db.ExpandInstanceDevices(devices.Clone(), inst.Profiles) } var usedByDevices []string // Iterate through each of the instance's devices, looking for disks in the same pool as volume. // Then try and match the volume name against the instance device's "source" property. for devName, dev := range devices { if dev["type"] != "disk" { continue } if dev["pool"] != poolName { continue } if strings.Split(dev["source"], "/")[0] == vol.Name { usedByDevices = append(usedByDevices, devName) } } if len(usedByDevices) > 0 { err = instanceFunc(inst, p, usedByDevices) if err != nil { return err } } return nil }) }) } // VolumeUsedByExclusiveRemoteInstancesWithProfiles checks if custom volume is exclusively attached to a remote // instance. Returns the remote instance that has the volume exclusively attached. Returns nil if volume available. func VolumeUsedByExclusiveRemoteInstancesWithProfiles(s *state.State, poolName string, projectName string, vol *api.StorageVolume) (*db.InstanceArgs, error) { pool, err := LoadByName(s, poolName) if err != nil { return nil, fmt.Errorf("Failed loading storage pool %q: %w", poolName, err) } info := pool.Driver().Info() // Always return nil if the storage driver supports mounting volumes // on multiple nodes at once and we're not dealing with a filesystem volume // on top of a block device. if info.VolumeMultiNode && !(info.BlockBacking && vol.ContentType == "filesystem") { return nil, nil } // Find if volume is attached to a remote instance. var remoteInstance *db.InstanceArgs err = VolumeUsedByInstanceDevices(s, poolName, projectName, vol, true, func(dbInst db.InstanceArgs, project api.Project, usedByDevices []string) error { if dbInst.Node != s.ServerName { remoteInstance = &dbInst return db.ErrInstanceListStop // Stop the search, this volume is attached to a remote instance. } return nil }) if err != nil && !errors.Is(err, db.ErrInstanceListStop) { return nil, err } return remoteInstance, nil } // VolumeUsedByDaemon indicates whether the volume is used by daemon storage. func VolumeUsedByDaemon(s *state.State, poolName string, volumeName string) (bool, error) { var storageBackups string var storageImages string err := s.DB.Node.Transaction(context.TODO(), func(ctx context.Context, tx *db.NodeTx) error { nodeConfig, err := node.ConfigLoad(ctx, tx) if err != nil { return err } storageBackups = nodeConfig.StorageBackupsVolume() storageImages = nodeConfig.StorageImagesVolume() return nil }) if err != nil { return false, err } fullName := fmt.Sprintf("%s/%s", poolName, volumeName) if storageBackups == fullName || storageImages == fullName { return true, nil } return false, nil } // FallbackMigrationType returns the fallback migration transport to use based on volume content type. func FallbackMigrationType(contentType drivers.ContentType) migration.MigrationFSType { if drivers.IsContentBlock(contentType) { return migration.MigrationFSType_BLOCK_AND_RSYNC } return migration.MigrationFSType_RSYNC } // InstanceMount mounts an instance's storage volume (if not already mounted). // Please call InstanceUnmount when finished. func InstanceMount(pool Pool, inst instance.Instance, op *operations.Operation) (*MountInfo, error) { var err error var mountInfo *MountInfo if inst.IsSnapshot() { mountInfo, err = pool.MountInstanceSnapshot(inst, op) if err != nil { return nil, err } } else { mountInfo, err = pool.MountInstance(inst, op) if err != nil { return nil, err } } return mountInfo, nil } // InstanceUnmount unmounts an instance's storage volume (if not in use). func InstanceUnmount(pool Pool, inst instance.Instance, op *operations.Operation) error { var err error if inst.IsSnapshot() { err = pool.UnmountInstanceSnapshot(inst, op) } else { err = pool.UnmountInstance(inst, op) } return err } // InstanceDiskBlockSize returns the block device size for the instance's disk. // This will mount the instance if not already mounted and will unmount at the end if needed. func InstanceDiskBlockSize(pool Pool, inst instance.Instance, op *operations.Operation) (int64, error) { mountInfo, err := InstanceMount(pool, inst, op) if err != nil { return -1, err } defer func() { _ = InstanceUnmount(pool, inst, op) }() if mountInfo.DiskPath == "" { return -1, errors.New("No disk path available from mount") } blockDiskSize, err := drivers.BlockDiskSizeBytes(mountInfo.DiskPath) if err != nil { return -1, fmt.Errorf("Error getting block disk size %q: %w", mountInfo.DiskPath, err) } return blockDiskSize, nil } // ComparableSnapshot is used when comparing snapshots on different pools to see whether they differ. type ComparableSnapshot struct { // Name of the snapshot (without the parent name). Name string // Identifier of the snapshot (that remains the same when copied between pools). ID string // Creation date time of the snapshot. CreationDate time.Time } // CompareSnapshots returns a list of snapshot indexes (from the associated input slices) to sync from the source // and to delete from the target respectively. // A snapshot will be added to "to sync from source" slice if it either doesn't exist in the target or its ID or // creation date is different to the source. When excludeOlder is true, source snapshots earlier than // latest target snapshot are excluded. // A snapshot will be added to the "to delete from target" slice if it doesn't exist in the source or its ID or // creation date is different to the source. func CompareSnapshots(sourceSnapshots []ComparableSnapshot, targetSnapshots []ComparableSnapshot, excludeOlder bool) ([]int, []int) { // Compare source and target. sourceSnapshotsByName := make(map[string]*ComparableSnapshot, len(sourceSnapshots)) targetSnapshotsByName := make(map[string]*ComparableSnapshot, len(targetSnapshots)) var syncFromSource, deleteFromTarget []int // Generate a list of source snapshots by name. for sourceSnapIndex := range sourceSnapshots { sourceSnapshotsByName[sourceSnapshots[sourceSnapIndex].Name] = &sourceSnapshots[sourceSnapIndex] } // Find the latest creation date among target snapshots. var latestTargetSnapshotTime time.Time // If target snapshot doesn't exist in source, or its creation date or ID differ, // then mark it for deletion on target. for targetSnapIndex := range targetSnapshots { // Generate a list of target snapshots by name for later comparison. targetSnapshotsByName[targetSnapshots[targetSnapIndex].Name] = &targetSnapshots[targetSnapIndex] sourceSnap, sourceSnapExists := sourceSnapshotsByName[targetSnapshots[targetSnapIndex].Name] if !sourceSnapExists || !sourceSnap.CreationDate.Equal(targetSnapshots[targetSnapIndex].CreationDate) || sourceSnap.ID != targetSnapshots[targetSnapIndex].ID { deleteFromTarget = append(deleteFromTarget, targetSnapIndex) } else if targetSnapshots[targetSnapIndex].CreationDate.After(latestTargetSnapshotTime) { latestTargetSnapshotTime = targetSnapshots[targetSnapIndex].CreationDate } } // If source snapshot doesn't exist in target, or its creation date or ID differ, // then mark it for syncing to target. for sourceSnapIndex := range sourceSnapshots { targetSnap, targetSnapExists := targetSnapshotsByName[sourceSnapshots[sourceSnapIndex].Name] if (!targetSnapExists && (!excludeOlder || sourceSnapshots[sourceSnapIndex].CreationDate.After(latestTargetSnapshotTime))) || (targetSnapExists && (!targetSnap.CreationDate.Equal(sourceSnapshots[sourceSnapIndex].CreationDate) || targetSnap.ID != sourceSnapshots[sourceSnapIndex].ID)) { syncFromSource = append(syncFromSource, sourceSnapIndex) } } return syncFromSource, deleteFromTarget } // CalculateVolumeSnapshotSize returns the size of a volume snapshot in bytes. func CalculateVolumeSnapshotSize(projectName string, pool Pool, contentType drivers.ContentType, volumeType drivers.VolumeType, volName string, snapName string) (int64, error) { if contentType != drivers.ContentTypeBlock { return 0, nil } var volSize int64 snapVolumeName := drivers.GetSnapshotVolumeName(volName, snapName) var fullSnapVolName string if volumeType == drivers.VolumeTypeCustom { fullSnapVolName = project.StorageVolume(projectName, snapVolumeName) } else { fullSnapVolName = project.Instance(projectName, snapVolumeName) } dbSnapVol, err := VolumeDBGet(pool, projectName, snapVolumeName, volumeType) if err != nil { return 0, err } snapVol := pool.GetVolume(volumeType, contentType, fullSnapVolName, dbSnapVol.Config) err = snapVol.MountTask(func(mountPath string, op *operations.Operation) error { poolBackend, ok := pool.(*backend) if !ok { return errors.New("Pool is not a backend") } volDiskPath, err := poolBackend.driver.GetVolumeDiskPath(snapVol) if err != nil { return err } volSize, err = drivers.BlockDiskSizeBytes(volDiskPath) if err != nil { return err } return nil }, nil) if err != nil { return 0, err } return volSize, nil } // VolumeSnapshotsToMigrationSnapshots converts a *api.StorageVolumeSnapshot to a *migration.Snapshot. func VolumeSnapshotsToMigrationSnapshots(snapshots []*api.StorageVolumeSnapshot, projectName string, pool Pool, contentType drivers.ContentType, volumeType drivers.VolumeType, volName string) ([]*migration.Snapshot, error) { migrationSnapshots := make([]*migration.Snapshot, 0, len(snapshots)) for _, snap := range snapshots { mSnapshot := &migration.Snapshot{Name: &snap.Name} volSize, err := CalculateVolumeSnapshotSize(projectName, pool, contentType, volumeType, volName, snap.Name) if err != nil { return nil, err } migration.SetSnapshotConfigValue(mSnapshot, "size", fmt.Sprintf("%d", volSize)) migrationSnapshots = append(migrationSnapshots, mSnapshot) } return migrationSnapshots, nil } // ProjectVolume returns a project scoped volume identifier. // It applies the appropriate '_' prefix based on the volume type. func ProjectVolume(projectName string, volName string, volType drivers.VolumeType) string { if volType == drivers.VolumeTypeContainer || volType == drivers.VolumeTypeVM { return project.Instance(projectName, volName) } return project.StorageVolume(projectName, volName) } // DisallowedStorageConfigForCreation returns a list of keys that cannot be specified // during pool creation. func DisallowedStorageConfigForCreation(driverName string) []string { if driverName == "lvmcluster" { return []string{"size"} } return []string{} } // ClusterWideStorageConfig returns a list of keys that are cluster wide. func ClusterWideStorageConfig(driverName string) []string { if driverName == "lvmcluster" { return []string{"size"} } return []string{} } // GenerateDependentVolumesOffer creates an offer header containing // all information required for dependent volume migration. func GenerateDependentVolumesOffer(s *state.State, config *backupConfig.Config, projectName string, snapshots bool, devices api.DevicesMap, clusterMove bool) ([]*migration.DependentVolume, error) { result := make([]*migration.DependentVolume, 0, len(config.DependentVolumes)) if len(config.DependentVolumes) == 0 { return result, nil } devicesMap := DevicesMapFromBackupConfig(config) for _, volConfig := range config.DependentVolumes { poolName := volConfig.Pool.Name contentType := volConfig.Volume.ContentType volName := volConfig.Volume.Name pool, err := LoadByName(s, poolName) if err != nil { return nil, fmt.Errorf("Failed loading pool: %w", err) } deviceName := DeviceByPoolAndVolume(devicesMap, poolName, volName) if deviceName == "" { return nil, fmt.Errorf("Device for volume %s/%s not found", poolName, volName) } shouldMigrate, err := ShouldMigrateDependentVolume(s, poolName, volName, devices[deviceName], clusterMove) if err != nil { return nil, err } if !shouldMigrate { continue } volStorageName := project.StorageVolume(projectName, volName) vol := pool.GetVolume(drivers.VolumeTypeCustom, drivers.ContentType(contentType), volStorageName, volConfig.Volume.Config) poolMigrationTypes := pool.MigrationTypes(drivers.ContentType(contentType), false, snapshots, true, false) if len(poolMigrationTypes) == 0 { return nil, fmt.Errorf("No migration types available") } header := localMigration.TypesToHeader(poolMigrationTypes...) var volSize int64 if drivers.ContentType(contentType) == drivers.ContentTypeBlock { err = vol.MountTask(func(mountPath string, op *operations.Operation) error { volDiskPath, err := pool.Driver().GetVolumeDiskPath(vol) if err != nil { return err } volSize, err = drivers.BlockDiskSizeBytes(volDiskPath) if err != nil { return err } return nil }, nil) if err != nil { return nil, err } } dependentVolume := localMigration.DependentVolumeFromHeader(header, volName, poolName, contentType, volSize, deviceName) dependentVolume.Snapshots = make([]*migration.Snapshot, 0, len(volConfig.VolumeSnapshots)) for _, volSnap := range volConfig.VolumeSnapshots { // Set size for snapshot volume snapSize, err := CalculateVolumeSnapshotSize(projectName, pool, drivers.ContentType(contentType), drivers.VolumeTypeCustom, volName, volSnap.Name) if err != nil { return nil, err } volSnap.Config["size"] = fmt.Sprintf("%d", snapSize) dependentVolume.Snapshots = append(dependentVolume.Snapshots, localMigration.VolumeSnapshotToProtobuf(volSnap)) } result = append(result, dependentVolume) } return result, nil } // DependentVolumeWithType represents a volume and its supported migration types. type DependentVolumeWithType struct { Volume *migration.DependentVolume VolumeTypes []localMigration.Type } // DependentVolumesMatchMigrationType returns the transport type matching the dependent volumes. func DependentVolumesMatchMigrationType(s *state.State, migrationDependentVolumes []*migration.DependentVolume, snapshots bool, overrides api.DevicesMap, source bool) ([]DependentVolumeWithType, error) { dependentVolumes := []DependentVolumeWithType{} for _, vol := range migrationDependentVolumes { contentType := drivers.ContentType(*vol.ContentType) poolName := *vol.Pool if overrides != nil && overrides[*vol.DeviceName] != nil { newPoolName, ok := overrides[*vol.DeviceName]["pool"] if ok { poolName = newPoolName } } pool, err := LoadByName(s, poolName) if err != nil { return nil, fmt.Errorf("Failed loading storage pool: %w", err) } poolMigrationTypes := pool.MigrationTypes(drivers.ContentType(contentType), false, snapshots, true, false) if len(poolMigrationTypes) == 0 { return nil, errors.New("No migration types available") } migrationTypes, err := localMigration.MatchTypes(localMigration.HeaderFromDependentVolume(vol), FallbackMigrationType(contentType), poolMigrationTypes) if err != nil { return nil, fmt.Errorf("Failed to negotiate migration type: %w", err) } // Update header on target. if !source { localMigration.DependentVolumeUpdateHeader(localMigration.TypesToHeader(migrationTypes...), vol) } dependentVolumes = append(dependentVolumes, DependentVolumeWithType{Volume: vol, VolumeTypes: migrationTypes}) } return dependentVolumes, nil } // ShouldMigrateDependentVolume returns true if the dependent volume // needs to be migrated for this instance. Returns false if migration // can be skipped (e.g., on shared storage within the same cluster). func ShouldMigrateDependentVolume(s *state.State, poolName string, volumeName string, overrides map[string]string, clusterMove bool) (bool, error) { if overrides != nil && ((overrides["source"] != "" && volumeName != overrides["source"]) || (overrides["pool"] != "" && poolName != overrides["pool"])) { return true, nil } diskPool, err := LoadByName(s, poolName) if err != nil { return false, fmt.Errorf("Failed loading storage pool: %w", err) } if diskPool.Driver().Info().Remote && clusterMove { return false, nil } return true, nil } // InstanceByVolumeName returns the instance associated with the given volume name. func InstanceByVolumeName(s *state.State, poolName string, projectName string, volumeName string, volumeDBType int) (instance.Instance, string, error) { if volumeDBType == db.StoragePoolVolumeTypeVM { inst, err := instance.LoadByProjectAndName(s, projectName, volumeName) if err != nil { return nil, "", err } instanceDeviceName, _, err := internalInstance.GetRootDiskDevice(inst.ExpandedDevices().CloneNative()) if err != nil { return nil, "", err } return inst, instanceDeviceName, nil } var instanceArgs *db.InstanceArgs var instanceDeviceName string pool, err := LoadByName(s, poolName) if err != nil { return nil, "", err } volumeType, err := VolumeDBTypeToType(volumeDBType) if err != nil { return nil, "", err } // Get the volume. dbVol, err := VolumeDBGet(pool, projectName, volumeName, volumeType) if err != nil { return nil, "", err } // Track down the instance. err = VolumeUsedByInstanceDevices(s, pool.Name(), projectName, &dbVol.StorageVolume, true, func(dbInst db.InstanceArgs, project api.Project, usedByDevices []string) error { if dbInst.Type != instancetype.VM { return fmt.Errorf("Volume is attached to a container") } if instanceArgs != nil && instanceArgs.Name != dbInst.Name { return fmt.Errorf("Volume is attached to multiple instances") } instanceArgs = &dbInst instanceDeviceName = usedByDevices[0] return nil }) if err != nil { return nil, "", err } if instanceArgs == nil { return nil, "", ErrVolumeNotAttachedToRunningInstance } // Load the instance. inst, err := instance.LoadByProjectAndName(s, instanceArgs.Project, instanceArgs.Name) if err != nil { return nil, "", err } return inst, instanceDeviceName, nil } // DeviceByPoolAndVolume returns a device from the devices map for the given pool and volume name. func DeviceByPoolAndVolume(deviceMap map[string]map[string]string, poolName string, volumeName string) string { inner, ok := deviceMap[poolName] if !ok { return "" } v, ok := inner[volumeName] if !ok { return "" } return v } // DevicesMapFromBackupConfig builds a map of instance devices indexed by pool and volume name. func DevicesMapFromBackupConfig(config *backupConfig.Config) map[string]map[string]string { devicesMap := map[string]map[string]string{} for devName, dev := range config.Container.ExpandedDevices { _, hasPool := devicesMap[dev["pool"]] if !hasPool { devicesMap[dev["pool"]] = map[string]string{} } devicesMap[dev["pool"]][dev["source"]] = devName } return devicesMap } incus-7.0.0/internal/server/sys/000077500000000000000000000000001517523235500166075ustar00rootroot00000000000000incus-7.0.0/internal/server/sys/apparmor.go000066400000000000000000000101241517523235500207550ustar00rootroot00000000000000//go:build linux && cgo && !agent package sys import ( "os" "os/exec" "strconv" "strings" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/warningtype" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) // Initialize AppArmor-specific attributes. func (s *OS) initAppArmor() []cluster.Warning { var dbWarnings []cluster.Warning /* Detect AppArmor availability */ _, err := exec.LookPath("apparmor_parser") if util.IsFalse(os.Getenv("INCUS_SECURITY_APPARMOR")) { logger.Warnf("AppArmor support has been manually disabled") dbWarnings = append(dbWarnings, cluster.Warning{ TypeCode: warningtype.AppArmorNotAvailable, LastMessage: "Manually disabled", }) } else if !internalUtil.IsDir("/sys/kernel/security/apparmor") { logger.Warnf("AppArmor support has been disabled because of lack of kernel support") dbWarnings = append(dbWarnings, cluster.Warning{ TypeCode: warningtype.AppArmorNotAvailable, LastMessage: "Disabled because of lack of kernel support", }) } else if err != nil { logger.Warnf("AppArmor support has been disabled because 'apparmor_parser' couldn't be found") dbWarnings = append(dbWarnings, cluster.Warning{ TypeCode: warningtype.AppArmorNotAvailable, LastMessage: "Disabled because 'apparmor_parser' couldn't be found", }) } else { s.AppArmorAvailable = true } /* Detect AppArmor stacking support */ s.AppArmorStacking = appArmorCanStack() /* Detect existing AppArmor stack */ if util.PathExists("/sys/kernel/security/apparmor/.ns_stacked") { contentBytes, err := os.ReadFile("/sys/kernel/security/apparmor/.ns_stacked") if err == nil && string(contentBytes) == "yes\n" { s.AppArmorStacked = true } } /* Detect AppArmor admin support */ if !haveMacAdmin() { if s.AppArmorAvailable { logger.Warnf("Per-container AppArmor profiles are disabled because the mac_admin capability is missing") } } else if s.RunningInUserNS && !s.AppArmorStacked { if s.AppArmorAvailable { logger.Warnf("Per-container AppArmor profiles are disabled because Incus is running in an unprivileged container without stacking") } } else { s.AppArmorAdmin = true } /* Detect AppArmor confinment */ profile := localUtil.AppArmorProfile() if profile != "unconfined" && profile != "" { if s.AppArmorAvailable { logger.Warnf("Per-container AppArmor profiles are disabled because Incus is already protected by AppArmor") } s.AppArmorConfined = true } return dbWarnings } func haveMacAdmin() bool { hdr := unix.CapUserHeader{Pid: 0} // Get hdr version to check. err := unix.Capget(&hdr, nil) if err != nil { return false } // Return false if not version 3. if hdr.Version != unix.LINUX_CAPABILITY_VERSION_3 { return false } var data [2]unix.CapUserData err = unix.Capget(&hdr, &data[0]) if err != nil { return false } idx := unix.CAP_MAC_ADMIN / 32 bit := uint(unix.CAP_MAC_ADMIN % 32) return ((1 << bit) & data[idx].Effective) != 0 } // Returns true if AppArmor stacking support is available. func appArmorCanStack() bool { contentBytes, err := os.ReadFile("/sys/kernel/security/apparmor/features/domain/stack") if err != nil { return false } if string(contentBytes) != "yes\n" { return false } contentBytes, err = os.ReadFile("/sys/kernel/security/apparmor/features/domain/version") if err != nil { return false } content := string(contentBytes) parts := strings.Split(strings.TrimSpace(content), ".") if len(parts) == 0 { logger.Warn("Unknown apparmor domain version", logger.Ctx{"version": content}) return false } major, err := strconv.Atoi(parts[0]) if err != nil { logger.Warn("Unknown apparmor domain version", logger.Ctx{"version": content}) return false } minor := 0 if len(parts) == 2 { minor, err = strconv.Atoi(parts[1]) if err != nil { logger.Warn("Unknown apparmor domain version", logger.Ctx{"version": content}) return false } } return major >= 1 && minor >= 2 } incus-7.0.0/internal/server/sys/apparmor_test.go000066400000000000000000000003661517523235500220230ustar00rootroot00000000000000//go:build linux && cgo && !agent package sys import ( "os" "testing" ) func TestHaveMacAdmin(t *testing.T) { macAdmin := haveMacAdmin() uid := os.Getuid() t.Log(macAdmin, uid) if macAdmin != (uid == 0) { t.Fatal(uid, macAdmin) } } incus-7.0.0/internal/server/sys/empty.go000066400000000000000000000000601517523235500202700ustar00rootroot00000000000000//go:build !linux || !cgo || agent package sys incus-7.0.0/internal/server/sys/fs.go000066400000000000000000000060401517523235500175460ustar00rootroot00000000000000//go:build linux && cgo && !agent package sys import ( "errors" "fmt" "io/fs" "os" "path/filepath" ) // LocalDatabasePath returns the path of the local database file. func (s *OS) LocalDatabasePath() string { return filepath.Join(s.VarDir, "database", "local.db") } // GlobalDatabaseDir returns the path of the global database directory. func (s *OS) GlobalDatabaseDir() string { return filepath.Join(s.VarDir, "database", "global") } // GlobalDatabasePath returns the path of the global database SQLite file // managed by dqlite. func (s *OS) GlobalDatabasePath() string { return filepath.Join(s.GlobalDatabaseDir(), "db.bin") } // initDirs Make sure all our directories are available. func (s *OS) initDirs() error { dirs := []struct { path string mode os.FileMode }{ {s.VarDir, 0o711}, // Instances are 0711 so the runtime can traverse to the data. {filepath.Join(s.VarDir, "containers"), 0o711}, {filepath.Join(s.VarDir, "virtual-machines"), 0o711}, // Snapshots are kept 0700 as the runtime doesn't need access. {filepath.Join(s.VarDir, "containers-snapshots"), 0o700}, {filepath.Join(s.VarDir, "virtual-machines-snapshots"), 0o700}, {filepath.Join(s.VarDir, "backups"), 0o700}, {s.CacheDir, 0o700}, {filepath.Join(s.CacheDir, "resources"), 0o700}, {filepath.Join(s.VarDir, "database"), 0o700}, {filepath.Join(s.VarDir, "devices"), 0o711}, {filepath.Join(s.VarDir, "disks"), 0o700}, {filepath.Join(s.VarDir, "guestapi"), 0o755}, {filepath.Join(s.VarDir, "images"), 0o700}, {s.LogDir, 0o700}, {filepath.Join(s.VarDir, "networks"), 0o711}, {s.RunDir, 0o711}, {filepath.Join(s.VarDir, "security"), 0o700}, {filepath.Join(s.VarDir, "security", "apparmor"), 0o700}, {filepath.Join(s.VarDir, "security", "apparmor", "cache"), 0o700}, {filepath.Join(s.VarDir, "security", "apparmor", "profiles"), 0o700}, {filepath.Join(s.VarDir, "security", "seccomp"), 0o700}, {filepath.Join(s.VarDir, "shmounts"), 0o711}, {filepath.Join(s.VarDir, "storage-pools"), 0o711}, } for _, dir := range dirs { err := os.Mkdir(dir.path, dir.mode) if err != nil { if !os.IsExist(err) { return fmt.Errorf("Failed to init dir %q: %w", dir.path, err) } err = os.Chmod(dir.path, dir.mode) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to chmod dir %q: %w", dir.path, err) } } } return nil } // initStorageDirs make sure all our directories are on the storage layer (after storage is mounted). func (s *OS) initStorageDirs() error { dirs := []struct { path string mode os.FileMode }{ {filepath.Join(s.VarDir, "backups", "custom"), 0o700}, {filepath.Join(s.VarDir, "backups", "instances"), 0o700}, } for _, dir := range dirs { err := os.Mkdir(dir.path, dir.mode) if err != nil { if !os.IsExist(err) { return fmt.Errorf("Failed to init storage dir %q: %w", dir.path, err) } err = os.Chmod(dir.path, dir.mode) if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("Failed to chmod storage dir %q: %w", dir.path, err) } } } return nil } incus-7.0.0/internal/server/sys/os.go000066400000000000000000000167401517523235500175670ustar00rootroot00000000000000//go:build linux && cgo && !agent package sys import ( "errors" "os" "os/user" "path/filepath" "strconv" "strings" "sync" "time" "github.com/lxc/incus/v7/internal/incusos" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/cgroup" "github.com/lxc/incus/v7/internal/server/db/cluster" localUtil "github.com/lxc/incus/v7/internal/server/util" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/idmap" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/osarch" "github.com/lxc/incus/v7/shared/util" ) // InotifyTargetInfo records the inotify information associated with a given // inotify target. type InotifyTargetInfo struct { Mask uint32 Wd int Path string } // InotifyInfo records the inotify information associated with a given // inotify instance. type InotifyInfo struct { Fd int sync.RWMutex Targets map[string]*InotifyTargetInfo } // OS is a high-level facade for accessing operating-system level functionalities. type OS struct { // Directories CacheDir string // Cache directory (e.g. /var/cache/incus/). LogDir string // Log directory (e.g. /var/log/incus/). RunDir string // Runtime directory (e.g. /run/incus/). VarDir string // Data directory (e.g. /var/lib/incus/). // Daemon environment Architectures []int // Cache of detected system architectures BackingFS string // Backing filesystem of $INCUS_DIR/containers ExecPath string // Absolute path to the daemon IdmapSet *idmap.Set // Information about user/group ID mapping InotifyWatch InotifyInfo LxcPath string // Path to the $INCUS_DIR/containers directory MockMode bool // If true some APIs will be mocked (for testing) Nodev bool RunningInUserNS bool Hostname string // Privilege dropping UnprivUser string UnprivUID uint32 UnprivGroup string UnprivGID uint32 // Apparmor features AppArmorAdmin bool AppArmorAvailable bool AppArmorConfined bool AppArmorStacked bool AppArmorStacking bool // SELinux features SELinuxAvailable bool SELinuxContextDaemon string SELinuxContextInstanceLXC string // LXC features LXCFeatures map[string]bool // OS info ReleaseInfo map[string]string Uname *linux.Utsname BootTime time.Time IncusOS *incusos.Client } // DefaultOS returns a fresh uninitialized OS instance with default values. func DefaultOS() *OS { newOS := &OS{ CacheDir: internalUtil.CachePath(), LogDir: internalUtil.LogPath(), RunDir: internalUtil.RunPath(), VarDir: internalUtil.VarPath(), } newOS.InotifyWatch.Fd = -1 newOS.InotifyWatch.Targets = make(map[string]*InotifyTargetInfo) newOS.ReleaseInfo = make(map[string]string) return newOS } // Init our internal data structures. func (s *OS) Init() ([]cluster.Warning, error) { dbWarnings := []cluster.Warning{} err := s.initDirs() if err != nil { return nil, err } s.Architectures, err = localUtil.GetArchitectures() if err != nil { return nil, err } s.LxcPath = filepath.Join(s.VarDir, "containers") s.BackingFS, err = linux.DetectFilesystem(s.LxcPath) if err != nil { logger.Error("Error detecting backing fs", logger.Ctx{"err": err}) } // Detect if it is possible to run daemons as an unprivileged user and group. for _, userName := range []string{"incus", "nobody"} { u, err := user.Lookup(userName) if err != nil { continue } uid, err := strconv.ParseUint(u.Uid, 10, 32) if err != nil { return nil, err } s.UnprivUser = userName s.UnprivUID = uint32(uid) break } for _, groupName := range []string{"incus", "nogroup"} { g, err := user.LookupGroup(groupName) if err != nil { continue } gid, err := strconv.ParseUint(g.Gid, 10, 32) if err != nil { return nil, err } s.UnprivGroup = groupName s.UnprivGID = uint32(gid) break } s.IdmapSet = getIdmapset() s.ExecPath = localUtil.GetExecPath() s.RunningInUserNS = linux.RunningInUserNS() s.Hostname, err = os.Hostname() if err != nil { return nil, err } dbWarnings = append(dbWarnings, s.initAppArmor()...) dbWarnings = append(dbWarnings, s.initSELinux()...) cgroup.Init() // Fill in the OS release info. osInfo, err := osarch.GetOSRelease() if err != nil { return nil, err } s.ReleaseInfo = osInfo uname, err := linux.Uname() if err != nil { return nil, err } s.Uname = uname if util.PathExists("/var/lib/incus-os/") { c, err := incusos.NewClient() if err != nil { return nil, err } s.IncusOS = c } // Fill in the boot time. out, err := os.ReadFile("/proc/stat") if err != nil { return nil, err } btime := int64(0) for _, line := range strings.Split(string(out), "\n") { if !strings.HasPrefix(line, "btime ") { continue } fields := strings.Fields(line) btime, err = strconv.ParseInt(fields[1], 10, 64) if err != nil { return nil, err } break } if btime > 0 { s.BootTime = time.Unix(btime, 0) } return dbWarnings, nil } // InitStorage initializes the storage layer after it has been mounted. func (s *OS) InitStorage() error { return s.initStorageDirs() } // GetUnixSocket returns the full path to the unix.socket file that this daemon is listening on. Used by tests. func (s *OS) GetUnixSocket() string { path := os.Getenv("INCUS_SOCKET") if path != "" { return path } return filepath.Join(s.VarDir, "unix.socket") } func getIdmapset() *idmap.Set { // Try getting the system map. idmapset, err := idmap.NewSetFromSystem("root") if err != nil && !errors.Is(err, idmap.ErrSubidUnsupported) { logger.Error("Unable to parse system idmap", logger.Ctx{"err": err}) return nil } if idmapset != nil { logger.Info("System idmap (root user):") for _, entry := range idmapset.ToLXCString() { logger.Infof(" - %s", entry) } // Only keep the POSIX ranges. submap := idmapset.FilterPOSIX() if submap == nil { logger.Warn("No valid subuid/subgid map, only privileged containers will be functional") return nil } logger.Info("Selected idmap:") for _, entry := range submap.ToLXCString() { logger.Infof(" - %s", entry) } return submap } // Try getting the process map. idmapset, err = idmap.NewSetFromCurrentProcess() if err != nil { logger.Error("Unable to parse process idmap", logger.Ctx{"err": err}) return nil } // Swap HostID for NSID and clear NSID (to turn into a usable map). for i, entry := range idmapset.Entries { idmapset.Entries[i].HostID = entry.NSID idmapset.Entries[i].NSID = 0 } logger.Info("Current process idmap:") for _, entry := range idmapset.ToLXCString() { logger.Infof(" - %s", entry) } // Try splitting a larger chunk from the current map. submap, err := idmapset.Split(65536, 1000000000, 1000000, -1) if err != nil && !errors.Is(err, idmap.ErrNoSuitableSubmap) { logger.Error("Unable to split a submap", logger.Ctx{"err": err}) return nil } if submap != nil { logger.Info("Selected idmap:") for _, entry := range submap.ToLXCString() { logger.Infof(" - %s", entry) } return submap } // Try splitting a smaller chunk from the current map. submap, err = idmapset.Split(65536, 1000000000, 65536, -1) if err != nil { if errors.Is(err, idmap.ErrNoSuitableSubmap) { logger.Warn("Not enough uid/gid available, only privileged containers will be functional") return nil } logger.Error("Unable to split a submap", logger.Ctx{"err": err}) return nil } logger.Info("Selected idmap:") for _, entry := range submap.ToLXCString() { logger.Infof(" - %s", entry) } return submap } incus-7.0.0/internal/server/sys/selinux.go000066400000000000000000000041421517523235500206260ustar00rootroot00000000000000//go:build linux && cgo && !agent package sys import ( "os" "strings" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/warningtype" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/logger" "github.com/lxc/incus/v7/shared/util" ) // Initialize SELinux-specific attributes. func (s *OS) initSELinux() []cluster.Warning { var dbWarnings []cluster.Warning // SELinux support is currently opt-in. if os.Getenv("INCUS_SECURITY_SELINUX") == "" { return dbWarnings } // Detect SELinux availability. if util.IsFalse(os.Getenv("INCUS_SECURITY_SELINUX")) { logger.Warnf("SELinux support has been manually disabled") dbWarnings = append(dbWarnings, cluster.Warning{ TypeCode: warningtype.SELinuxNotAvailable, LastMessage: "Manually disabled", }) return dbWarnings } else if !internalUtil.IsDir("/sys/fs/selinux") { logger.Warnf("SELinux support has been disabled because of lack of kernel support") dbWarnings = append(dbWarnings, cluster.Warning{ TypeCode: warningtype.SELinuxNotAvailable, LastMessage: "Disabled because of lack of kernel support", }) return dbWarnings } s.SELinuxAvailable = true // Read our own context. content, err := os.ReadFile("/proc/self/attr/current") if err != nil { logger.Warnf("SELinux support has been disabled because of unaccessible context data") dbWarnings = append(dbWarnings, cluster.Warning{ TypeCode: warningtype.SELinuxNotAvailable, LastMessage: "Disabled because of unaccessible context data", }) return dbWarnings } s.SELinuxContextDaemon = strings.TrimRight(strings.TrimSpace(string(content)), "\x00") // Handle the various SELinux policy variants here. switch s.SELinuxContextDaemon { case "system_u:system_r:container_runtime_t:s0": logger.Debugf("Detected Fedora-style SELinux setup") s.SELinuxContextInstanceLXC = "system_u:system_r:spc_t:s0" case "system_u:system_r:incusd_t:s0": logger.Debugf("Detected SELinux refpolicy setup") s.SELinuxContextInstanceLXC = "system_u:system_r:container_init_t:s0" } return dbWarnings } incus-7.0.0/internal/server/sys/testing.go000066400000000000000000000027261517523235500206220ustar00rootroot00000000000000//go:build linux && cgo && !agent package sys import ( "os" "path/filepath" "runtime" "testing" "github.com/stretchr/testify/require" ) // NewTestOS returns a new OS instance initialized with test values. func NewTestOS(t *testing.T) (*OS, func()) { dir, err := os.MkdirTemp("", "incus-sys-os-test-") require.NoError(t, err) require.NoError(t, SetupTestCerts(dir)) cleanup := func() { require.NoError(t, os.RemoveAll(dir)) } testOS := &OS{ // FIXME: setting mock mode can be avoided once daemon tasks // are fixed to exit gracefully. See daemon.go. MockMode: true, VarDir: dir, CacheDir: filepath.Join(dir, "cache"), LogDir: filepath.Join(dir, "log"), RunDir: filepath.Join(dir, "run"), } _, err = testOS.Init() require.NoError(t, err) return testOS, cleanup } // SetupTestCerts populates the given test directory with server certificates. // // Since generating certificates is CPU intensive, they will be simply // symlink'ed from the test/deps/ directory. // // FIXME: this function is exported because some tests use it // directly. Eventually we should rework those tests to use NewTestOS // instead. func SetupTestCerts(dir string) error { _, filename, _, _ := runtime.Caller(0) deps := filepath.Join(filepath.Dir(filename), "..", "..", "..", "test", "deps") for _, f := range []string{"server.crt", "server.key"} { err := os.Symlink(filepath.Join(deps, f), filepath.Join(dir, f)) if err != nil { return err } } return nil } incus-7.0.0/internal/server/syslog/000077500000000000000000000000001517523235500173115ustar00rootroot00000000000000incus-7.0.0/internal/server/syslog/listener.go000066400000000000000000000070461517523235500214740ustar00rootroot00000000000000package syslog import ( "context" "fmt" "net" "os" "strings" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/server/events" internalUtil "github.com/lxc/incus/v7/internal/util" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/revert" "github.com/lxc/incus/v7/shared/util" ) // Listen starts the log monitor. func Listen(ctx context.Context, eventServer *events.Server) error { var listenConfig net.ListenConfig sockFile := internalUtil.VarPath("syslog.socket") if util.PathExists(sockFile) { err := os.Remove(sockFile) if err != nil { return fmt.Errorf("Failed deleting stale syslog.socket: %w", err) } } conn, err := listenConfig.ListenPacket(ctx, "unixgram", sockFile) if err != nil { return fmt.Errorf("Failed listening on syslog socket: %w", err) } reverter := revert.New() defer reverter.Fail() reverter.Add(func() { _ = conn.Close() _ = os.Remove(sockFile) }) // Get max size var maxBufSize int uc, ok := conn.(*net.UnixConn) if ok { f, err := uc.File() if err != nil { return fmt.Errorf("Failed getting underlying os.File: %w", err) } maxBufSize, err = unix.GetsockoptInt(int(f.Fd()), unix.SOL_SOCKET, unix.SO_RCVBUF) if err != nil { _ = f.Close() return fmt.Errorf("Failed getting SO_RCVBUF: %w", err) } // This makes the fd non-blocking so that conn.Close() won't block. // See https://github.com/golang/go/issues/29277#issuecomment-447922481 err = unix.SetNonblock(int(f.Fd()), true) if err != nil { _ = f.Close() return fmt.Errorf("Failed setting non-block: %w", err) } _ = f.Close() } // This goroutine waits for the context to be cancelled and then closes the connection causing `ReadFrom` to return an error and exit the goroutine below. go func() { <-ctx.Done() _ = conn.Close() _ = os.Remove(sockFile) }() // This goroutine is used for reading packets, and processing the log message. `ReadFrom` will block until it either receives data, or an error occurs. If the connection is closed, `ReadFrom` will return an error, and the goroutine will terminate. go func() { buf := make([]byte, maxBufSize) // This maps OVN log level names to logrus log levels. logMap := map[string]logrus.Level{ "dbg": logrus.DebugLevel, "info": logrus.InfoLevel, "warn": logrus.WarnLevel, "err": logrus.ErrorLevel, "emer": logrus.ErrorLevel, } for { n, _, err := conn.ReadFrom(buf) if err != nil { return } // Acceptable formats: // - <29> ovs|00017|rconn|INFO|unix:/run/openvswitch/br-int.mgmt: connected" // - <29> ovs|ovn-controller|00017|rconn|INFO|unix:/run/openvswitch/br-int.mgmt: connected" // The first field can be ignored as that information is relevant to syslogd. fields := strings.SplitN(string(buf[:n]), "|", 6) if len(fields) < 5 { continue } applicationName := "" if len(fields) == 6 { applicationName = fields[1] } sequenceNumber := fields[len(fields)-4] moduleName := fields[len(fields)-3] logLevel := strings.ToLower(fields[len(fields)-2]) message := fields[len(fields)-1] if !strings.HasPrefix(moduleName, "acl_log") { continue } event := api.EventLogging{ Level: logMap[logLevel].String(), Message: message, Context: map[string]string{ "sequence": sequenceNumber, }, } if applicationName != "" { event.Context["application"] = applicationName } err = eventServer.Send("", api.EventTypeNetworkACL, event) if err != nil { continue } } }() reverter.Success() return nil } incus-7.0.0/internal/server/task/000077500000000000000000000000001517523235500167335ustar00rootroot00000000000000incus-7.0.0/internal/server/task/func.go000066400000000000000000000003671517523235500202230ustar00rootroot00000000000000package task import ( "context" ) // Func captures the signature of a function executable by a Task. // // When the given context is done, the function must gracefully terminate // whatever logic it's executing. type Func func(context.Context) incus-7.0.0/internal/server/task/group.go000066400000000000000000000053001517523235500204140ustar00rootroot00000000000000package task import ( "context" "fmt" "strconv" "sync" "time" ) // Group of tasks sharing the same lifecycle. // // All tasks in a group will be started and stopped at the same time. type Group struct { cancel context.CancelFunc wg sync.WaitGroup tasks []Task running map[int]bool mu sync.Mutex } // Add a new task to the group, returning its index. func (g *Group) Add(f Func, schedule Schedule) *Task { g.mu.Lock() defer g.mu.Unlock() i := len(g.tasks) g.tasks = append(g.tasks, Task{ f: f, schedule: schedule, reset: make(chan struct{}, 16), // Buffered to not block senders }) return &g.tasks[i] } // Start all the tasks in the group. func (g *Group) Start(ctx context.Context) { // Lock access to the g.running and g.tasks map for the entirety of this function so that // concurrent calls to Start() or Add(0) don't race. This ensures all tasks in this group // are started based on a consistent snapshot of g.running and g.tasks. g.mu.Lock() defer g.mu.Unlock() ctx, g.cancel = context.WithCancel(ctx) if g.running == nil { g.running = make(map[int]bool) } for i := range g.tasks { if g.running[i] { continue } g.running[i] = true task := g.tasks[i] // Local variable for the closure below. g.wg.Add(1) go func(i int) { defer g.wg.Done() task.loop(ctx) // Ensure running map is updated before wait group Done() is called. g.mu.Lock() defer g.mu.Unlock() g.running[i] = false }(i) } } // Stop all tasks in the group. // // This works by sending a cancellation signal to all tasks of the // group and waiting for them to terminate. // // If a task is idle (i.e. not executing its task function) it will terminate // immediately. // // If a task is busy executing its task function, the cancellation signal will // propagate through the context passed to it, and the task will block waiting // for the function to terminate. // // In case the given timeout expires before all tasks complete, this method // exits immediately and returns an error, otherwise it returns nil. func (g *Group) Stop(timeout time.Duration) error { if g.cancel == nil { // We were not even started return nil } g.cancel() graceful := make(chan struct{}, 1) go func() { g.wg.Wait() close(graceful) }() // Wait for graceful termination, but abort if the context expires. ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() select { case <-ctx.Done(): g.mu.Lock() defer g.mu.Unlock() running := []string{} for i, value := range g.running { if value { running = append(running, strconv.Itoa(i)) } } return fmt.Errorf("Task(s) still running: IDs %v", running) case <-graceful: return nil } } incus-7.0.0/internal/server/task/group_test.go000066400000000000000000000020071517523235500214540ustar00rootroot00000000000000package task_test import ( "context" "testing" "time" "github.com/stretchr/testify/assert" "github.com/lxc/incus/v7/internal/server/task" ) func TestGroup_Add(t *testing.T) { group := &task.Group{} ok := make(chan struct{}) f := func(context.Context) { close(ok) } group.Add(f, task.Every(time.Second)) group.Start(context.Background()) assertRecv(t, ok) assert.NoError(t, group.Stop(time.Second)) } func TestGroup_StopUngracefully(t *testing.T) { group := &task.Group{} // Create a task function that blocks. ok := make(chan struct{}) defer close(ok) f := func(context.Context) { ok <- struct{}{} <-ok } group.Add(f, task.Every(time.Second)) group.Start(context.Background()) assertRecv(t, ok) assert.EqualError(t, group.Stop(time.Millisecond), "Task(s) still running: IDs [0]") } // Assert that the given channel receives an object within a second. func assertRecv(t *testing.T, ch chan struct{}) { select { case <-ch: case <-time.After(time.Second): t.Fatal("no object received") } } incus-7.0.0/internal/server/task/schedule.go000066400000000000000000000044321517523235500210610ustar00rootroot00000000000000package task import ( "errors" "time" ) // Schedule captures the signature of a schedule function. // // It should return the amount of time to wait before triggering the next // execution of a task function. // // If it returns zero, the function does not get run at all. // // If it returns a duration greater than zero, the task function gets run once // immediately and then again after the specified amount of time. At that point // the Task re-invokes the schedule function and repeats the same logic. // // If ErrSkip is returned, the immediate execution of the task function gets // skipped, and it will only be possibly executed after the returned interval. // // If any other error is returned, the task won't execute the function, however // if the returned interval is greater than zero it will re-try to run the // schedule function after that amount of time. type Schedule func() (time.Duration, error) // ErrSkip is a special error that may be returned by a Schedule function to // mean to skip a particular execution of the task function, and just wait the // returned interval before re-evaluating. var ErrSkip = errors.New("skip execution of task function") // Every returns a Schedule that always returns the given time interval. func Every(interval time.Duration, options ...EveryOption) Schedule { every := &every{} for _, option := range options { option(every) } first := true return func() (time.Duration, error) { var err error if first && every.skipFirst { err = ErrSkip } first = false return interval, err } } // Daily is a convenience for creating a schedule that runs once a day. func Daily(options ...EveryOption) Schedule { return Every(24*time.Hour, options...) } // Hourly is a convenience for creating a schedule that runs once an hour. func Hourly(options ...EveryOption) Schedule { return Every(time.Hour, options...) } // SkipFirst is an option for the Every schedule that will make the schedule // skip the very first invocation of the task function. var SkipFirst = func(every *every) { every.skipFirst = true } // EveryOption captures a tweak that can be applied to the Every schedule. type EveryOption func(*every) // Captures options for the Every schedule. type every struct { skipFirst bool // If true, return ErrSkip at the very first execution } incus-7.0.0/internal/server/task/start.go000066400000000000000000000012551517523235500204220ustar00rootroot00000000000000package task import ( "context" "time" ) // Start a single task executing the given function with the given schedule. // // This is a convenience around Group and it returns two functions that can be // used to control the task. The first is a "stop" function trying to terminate // the task gracefully within the given timeout and the second is a "reset" // function to reset the task's state. See Group.Stop() and Group.Reset() for // more details. func Start(ctx context.Context, f Func, schedule Schedule) (func(time.Duration) error, func()) { group := Group{} task := group.Add(f, schedule) group.Start(ctx) stop := group.Stop reset := task.Reset return stop, reset } incus-7.0.0/internal/server/task/task.go000066400000000000000000000052161517523235500202300ustar00rootroot00000000000000package task import ( "context" "errors" "time" ) // Task executes a certain function periodically, according to a certain // schedule. type Task struct { f Func // Function to execute. schedule Schedule // Decides if and when to execute f. reset chan struct{} // Resets the schedule and starts over. } // Reset the state of the task as if it had just been started. // // This is handy if the schedule logic has changed, since the schedule function // will be invoked immediately to determine whether and when to run the task // function again. func (t *Task) Reset() { t.reset <- struct{}{} } // Execute the our task function according to our schedule, until the given // context gets cancelled. func (t *Task) loop(ctx context.Context) { // Kick off the task immediately (as long as the schedule is // greater than zero, see below). delay := immediately for { var timer <-chan time.Time schedule, err := t.schedule() switch { case errors.Is(err, ErrSkip): // Reset the delay to be exactly the schedule, so we // rule out the case where it's set to immediately // because it's the first iteration or we got reset. delay = schedule fallthrough // Fall to case nil, to apply normal non-error logic case err == nil: // If the schedule is greater than zero, setup a timer // that will expire after 'delay' seconds (or after the // schedule in case of ErrSkip, to avoid triggering // immediately), otherwise setup a timer that will // never expire (hence the task function won't ever be // run, unless Reset() is called and schedule() starts // returning values greater than zero). if schedule > 0 { timer = time.After(delay) } else { timer = make(chan time.Time) } default: // If the schedule is not greater than zero, abort the // task and return immediately. Otherwise set up the // timer to retry after that amount of time. if schedule <= 0 { return } timer = time.After(schedule) } select { case <-timer: if err == nil { // Execute the task function synchronously. Consumers // are responsible for implementing proper cancellation // of the task function itself using the tomb's context. start := time.Now() t.f(ctx) duration := time.Since(start) delay = schedule - duration if delay < 0 { delay = immediately } } else { // Don't execute the task function, and set the // delay to run it immediately whenever the // schedule function returns a nil error. delay = immediately } case <-ctx.Done(): return case <-t.reset: delay = immediately } } } const immediately = 0 * time.Second incus-7.0.0/internal/server/task/task_test.go000066400000000000000000000101651517523235500212660ustar00rootroot00000000000000package task_test import ( "context" "errors" "testing" "time" "github.com/stretchr/testify/assert" "github.com/lxc/incus/v7/internal/server/task" ) // The given task is executed immediately by the scheduler. func TestTask_ExecuteImmediately(t *testing.T) { f, wait := newFunc(t, 1) defer startTask(t, f, task.Every(time.Second))() //nolint:revive wait(100 * time.Millisecond) } // The given task is executed again after the specified time interval has // elapsed. func TestTask_ExecutePeriodically(t *testing.T) { f, wait := newFunc(t, 2) defer startTask(t, f, task.Every(250*time.Millisecond))() //nolint:revive wait(100 * time.Millisecond) wait(400 * time.Millisecond) } // If the scheduler is reset, the task is re-executed immediately and then // again after the interval. func TestTask_Reset(t *testing.T) { f, wait := newFunc(t, 3) stop, reset := task.Start(context.Background(), f, task.Every(250*time.Millisecond)) defer func() { _ = stop(time.Second) }() wait(50 * time.Millisecond) // First execution, immediately reset() // Trigger a reset wait(50 * time.Millisecond) // Second execution, immediately after reset wait(400 * time.Millisecond) // Third execution, after the timeout } // If the interval is zero, the task function is never run. func TestTask_ZeroInterval(t *testing.T) { f, _ := newFunc(t, 0) defer startTask(t, f, task.Every(0*time.Millisecond))() //nolint:revive // Sleep a little bit to prove that the task function does not get run. time.Sleep(100 * time.Millisecond) } // If the schedule returns an error, the task is aborted. func TestTask_ScheduleError(t *testing.T) { schedule := func() (time.Duration, error) { return 0, errors.New("boom") } f, _ := newFunc(t, 0) defer startTask(t, f, schedule)() //nolint:revive // Sleep a little bit to prove that the task function does not get run. time.Sleep(100 * time.Millisecond) } // If the schedule returns an error, but its interval is positive, the task will // try again to invoke the schedule function after that interval. func TestTask_ScheduleTemporaryError(t *testing.T) { errored := false schedule := func() (time.Duration, error) { if !errored { errored = true return time.Millisecond, errors.New("boom") } return time.Second, nil } f, wait := newFunc(t, 1) defer startTask(t, f, schedule)() //nolint:revive // The task gets executed since the schedule error is temporary and gets // resolved. wait(50 * time.Millisecond) } // If SkipFirst is passed, the given task is only executed at the second round. func TestTask_SkipFirst(t *testing.T) { i := 0 f := func(context.Context) { i++ } defer startTask(t, f, task.Every(250*time.Millisecond, task.SkipFirst))() //nolint:revive time.Sleep(400 * time.Millisecond) assert.Equal(t, 1, i) // The function got executed only once, not twice. } // Create a new task function that sends a notification to a channel every time // it's run. // // Return the task function, along with a "wait" function which will block // until one notification is received through such channel, or fails the test // if no notification is received within the given timeout. // // The n parameter can be used to limit the number of times the task function // is allowed run: when that number is reached the task function will trigger a // test failure (zero means that the task function will make the test fail as // soon as it is invoked). func newFunc(t *testing.T, n int) (task.Func, func(time.Duration)) { i := 0 notifications := make(chan struct{}) f := func(context.Context) { if i == n { t.Errorf("task was supposed to be called at most %d times", n) } notifications <- struct{}{} i++ } wait := func(timeout time.Duration) { select { case <-notifications: case <-time.After(timeout): t.Errorf("no notification received in %s", timeout) } } return f, wait } // Convenience around task.Start which also makes sure that the stop function // of the task actually terminates. func startTask(t *testing.T, f task.Func, schedule task.Schedule) func() { stop, _ := task.Start(context.Background(), f, schedule) return func() { assert.NoError(t, stop(time.Second)) } } incus-7.0.0/internal/server/ucred/000077500000000000000000000000001517523235500170735ustar00rootroot00000000000000incus-7.0.0/internal/server/ucred/ucred.go000066400000000000000000000020101517523235500205150ustar00rootroot00000000000000package ucred import ( "context" "errors" "net" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/internal/server/endpoints/listeners" "github.com/lxc/incus/v7/internal/server/request" ) // ErrNotUnixSocket is returned when the underlying connection isn't a unix socket. var ErrNotUnixSocket = errors.New("Connection isn't a unix socket") // GetConnFromContext extracts the connection from the request context on a HTTP listener. func GetConnFromContext(ctx context.Context) net.Conn { return ctx.Value(request.CtxConn).(net.Conn) } // GetCredFromContext extracts the unix credentials from the request context on a HTTP listener. func GetCredFromContext(ctx context.Context) (*unix.Ucred, error) { conn := GetConnFromContext(ctx) unixConnPtr, ok := conn.(*net.UnixConn) if !ok { bufferedUnixConnPtr, ok := conn.(listeners.BufferedUnixConn) if !ok { return nil, ErrNotUnixSocket } unixConnPtr = bufferedUnixConnPtr.Unix() } return linux.GetUcred(unixConnPtr) } incus-7.0.0/internal/server/util/000077500000000000000000000000001517523235500167465ustar00rootroot00000000000000incus-7.0.0/internal/server/util/apparmor.go000066400000000000000000000004141517523235500211150ustar00rootroot00000000000000package util import ( "os" "strings" ) // AppArmorProfile returns the current apparmor profile. func AppArmorProfile() string { contents, err := os.ReadFile("/proc/self/attr/current") if err == nil { return strings.TrimSpace(string(contents)) } return "" } incus-7.0.0/internal/server/util/config.go000066400000000000000000000017611517523235500205470ustar00rootroot00000000000000package util import ( "fmt" "slices" "sort" "strings" "github.com/lxc/incus/v7/shared/util" ) // CompareConfigs compares two config maps and returns an error if they differ. func CompareConfigs(config1, config2 map[string]string, exclude []string) error { if exclude == nil { exclude = []string{} } delta := []string{} for key, value := range config1 { if slices.Contains(exclude, key) { continue } if config2[key] != value { delta = append(delta, key) } } for key, value := range config2 { if slices.Contains(exclude, key) { continue } if config1[key] != value { present := slices.Contains(delta, key) if !present { delta = append(delta, key) } } } sort.Strings(delta) if len(delta) > 0 { return fmt.Errorf("different values for keys: %s", strings.Join(delta, ", ")) } return nil } // CopyConfig creates a new map with a copy of the given config. func CopyConfig(config map[string]string) map[string]string { return util.CloneMap(config) } incus-7.0.0/internal/server/util/config_test.go000066400000000000000000000016771517523235500216140ustar00rootroot00000000000000package util_test import ( "testing" "github.com/stretchr/testify/assert" localUtil "github.com/lxc/incus/v7/internal/server/util" ) func Test_CompareConfigsMismatch(t *testing.T) { cases := []struct { config1 map[string]string config2 map[string]string error string }{ { map[string]string{"foo": "bar"}, map[string]string{"foo": "egg"}, "different values for keys: foo", }, { map[string]string{"foo": "bar"}, map[string]string{"egg": "buz"}, "different values for keys: egg, foo", }, } for _, c := range cases { t.Run(c.error, func(t *testing.T) { err := localUtil.CompareConfigs(c.config1, c.config2, nil) assert.EqualError(t, err, c.error) }) } } func Test_CompareConfigs(t *testing.T) { config1 := map[string]string{"foo": "bar", "baz": "buz"} config2 := map[string]string{"foo": "egg", "baz": "buz"} err := localUtil.CompareConfigs(config1, config2, []string{"foo"}) assert.NoError(t, err) } incus-7.0.0/internal/server/util/dns.go000066400000000000000000000023741517523235500200670ustar00rootroot00000000000000package util import ( "net" ) // Zone suffixes. const ( // IPv4Arpa represents the IPv4 reverse DNS suffix. IPv4Arpa = ".in-addr.arpa" // IPv6Arpa represents the IPv6 reverse DNS suffix. IPv6Arpa = ".ip6.arpa" ) // ReverseDNS takes an IPv4 or IPv6 address and returns the matching ARPA record. func ReverseDNS(ip net.IP) (arpa string) { if ip == nil { return "" } // Deal with IPv4. if ip.To4() != nil { return uitoa(uint(ip[15])) + "." + uitoa(uint(ip[14])) + "." + uitoa(uint(ip[13])) + "." + uitoa(uint(ip[12])) + IPv4Arpa + "." } // Deal with IPv6. buf := make([]byte, 0, len(ip)*4+len(IPv6Arpa)) // Add it, in reverse, to the buffer. for i := len(ip) - 1; i >= 0; i-- { v := ip[i] buf = append(buf, hexDigit[v&0xF], '.', hexDigit[v>>4], '.') } // Add the suffix. buf = append(buf, IPv6Arpa[1:]+"."...) return string(buf) } // Convert unsigned integer to decimal string. func uitoa(val uint) string { // Avoid string allocation. if val == 0 { return "0" } // Big enough for 64bit value base 10. var buf [20]byte i := len(buf) - 1 for val >= 10 { q := val / 10 buf[i] = byte('0' + val - q*10) i-- val = q } // val < 10 buf[i] = byte('0' + val) return string(buf[i:]) } const hexDigit = "0123456789abcdef" incus-7.0.0/internal/server/util/http.go000066400000000000000000000246241517523235500202640ustar00rootroot00000000000000package util import ( "bytes" "context" "crypto/sha256" "crypto/x509" "encoding/base64" "encoding/json" "encoding/pem" "errors" "fmt" "io" "net" "net/http" "net/url" "strconv" "strings" "time" "github.com/golang-jwt/jwt/v5" "github.com/lxc/incus/v7/internal/ports" "github.com/lxc/incus/v7/shared/api" "github.com/lxc/incus/v7/shared/logger" localtls "github.com/lxc/incus/v7/shared/tls" ) // DebugJSON helper to log JSON. // Accepts a title to prefix the JSON log with, a *bytes.Buffer containing the JSON and a logger to use for // logging the JSON (allowing for custom context to be added to the log). func DebugJSON(title string, r *bytes.Buffer, l logger.Logger) { pretty := &bytes.Buffer{} err := json.Indent(pretty, r.Bytes(), "\t", "\t") if err != nil { l.Debug("Error indenting JSON", logger.Ctx{"err": err}) return } // Print the JSON without the last "\n" str := pretty.String() l.Debug(fmt.Sprintf("%s\n\t%s", title, str[0:len(str)-1])) } // WriteJSON encodes the body as JSON and sends it back to the client // Accepts optional debugLogger that activates debug logging if non-nil. func WriteJSON(w http.ResponseWriter, body any, debugLogger logger.Logger) error { var output io.Writer var captured *bytes.Buffer output = w if debugLogger != nil { captured = &bytes.Buffer{} output = io.MultiWriter(w, captured) } enc := json.NewEncoder(output) enc.SetEscapeHTML(false) err := enc.Encode(body) if captured != nil { DebugJSON("WriteJSON", captured, debugLogger) } return err } // EtagHash hashes the provided data and returns the sha256. func EtagHash(data any) (string, error) { hash256 := sha256.New() err := json.NewEncoder(hash256).Encode(data) if err != nil { return "", err } return fmt.Sprintf("%x", hash256.Sum(nil)), nil } // EtagCheck validates the hash of the current state with the hash // provided by the client. func EtagCheck(r *http.Request, data any) error { match := r.Header.Get("If-Match") if match == "" { return nil } match = strings.Trim(match, "\"") hash, err := EtagHash(data) if err != nil { return err } if hash != match { return api.StatusErrorf(http.StatusPreconditionFailed, "ETag doesn't match: %s vs %s", hash, match) } return nil } // HTTPClient returns an http.Client using the given certificate and proxy. func HTTPClient(certificate string, proxy proxyFunc) (*http.Client, error) { var err error var cert *x509.Certificate if certificate != "" { certBlock, _ := pem.Decode([]byte(certificate)) if certBlock == nil { return nil, errors.New("Invalid certificate") } cert, err = x509.ParseCertificate(certBlock.Bytes) if err != nil { return nil, err } } tlsConfig, err := localtls.GetTLSConfig(cert) if err != nil { return nil, err } tr := &http.Transport{ TLSClientConfig: tlsConfig, DialContext: localtls.RFC3493Dialer, Proxy: proxy, DisableKeepAlives: true, ExpectContinueTimeout: time.Second * 30, ResponseHeaderTimeout: time.Second * 3600, TLSHandshakeTimeout: time.Second * 5, } myhttp := http.Client{ Transport: tr, } // Setup redirect policy myhttp.CheckRedirect = func(req *http.Request, via []*http.Request) error { // Replicate the headers req.Header = via[len(via)-1].Header return nil } return &myhttp, nil } // A function capable of proxing an HTTP request. type proxyFunc func(req *http.Request) (*url.URL, error) // ContextAwareRequest is an interface implemented by http.Request starting // from Go 1.8. It supports graceful cancellation using a context. type ContextAwareRequest interface { WithContext(ctx context.Context) *http.Request } // CheckTrustState checks whether the given client certificate is trusted // (i.e. it has a valid time span and it belongs to the given list of trusted // certificates). // Returns whether or not the certificate is trusted, and the fingerprint of the certificate. func CheckTrustState(cert x509.Certificate, trustedCerts map[string]x509.Certificate, networkCert *localtls.CertInfo, trustCACertificates bool) (bool, string) { // Extra validity check (should have been caught by TLS stack) if time.Now().Before(cert.NotBefore) || time.Now().After(cert.NotAfter) { return false, "" } if networkCert != nil && trustCACertificates { ca := networkCert.CA() if ca != nil && cert.CheckSignatureFrom(ca) == nil { // Check whether the certificate has been revoked. crl := networkCert.CRL() if crl != nil { if crl.CheckSignatureFrom(ca) != nil { return false, "" // CRL not signed by CA } for _, revoked := range crl.RevokedCertificateEntries { if cert.SerialNumber.Cmp(revoked.SerialNumber) == 0 { return false, "" // Certificate is revoked, so not trusted anymore. } } } // Certificate not revoked, so trust it as is signed by CA cert. return true, localtls.CertFingerprint(&cert) } } // Check whether client certificate is in trust store. for fingerprint, v := range trustedCerts { if bytes.Equal(cert.Raw, v.Raw) { logger.Debug("Matched trusted cert", logger.Ctx{"fingerprint": fingerprint, "subject": v.Subject}) return true, fingerprint } } return false, "" } // IsRecursionRequest checks whether the given HTTP request is marked with the // "recursion" flag in its form values. func IsRecursionRequest(r *http.Request) bool { recursionStr := r.FormValue("recursion") recursion, err := strconv.Atoi(recursionStr) if err != nil { return false } return recursion != 0 } // ListenAddresses returns a list of : combinations at which this machine can be reached. // It accepts the configured listen address in the following formats: , : or :. // If a listen port is not specified then then ports.HTTPSDefaultPort is used instead. // If a non-empty and non-wildcard host is passed in then this functions returns a single element list with the // listen address specified. Otherwise if an empty host or wildcard address is specified then all global unicast // addresses actively configured on the host are returned. If an IPv4 wildcard address (0.0.0.0) is specified as // the host then only IPv4 addresses configured on the host are returned. func ListenAddresses(configListenAddress string) ([]string, error) { addresses := make([]string, 0) if configListenAddress == "" { return addresses, nil } // Check if configListenAddress is a bare IP address (wrapped with square brackets or unwrapped) or a // hostname (without port). If so then add the default port to the configListenAddress ready for parsing. unwrappedConfigListenAddress := strings.Trim(configListenAddress, "[]") listenIP := net.ParseIP(unwrappedConfigListenAddress) if listenIP != nil || !strings.Contains(unwrappedConfigListenAddress, ":") { // Use net.JoinHostPort so that IPv6 addresses are correctly wrapped ready for parsing below. configListenAddress = net.JoinHostPort(unwrappedConfigListenAddress, fmt.Sprintf("%d", ports.HTTPSDefaultPort)) } // By this point we should always have the configListenAddress in form :, so lets check that. // This also ensures that any wrapped IPv6 addresses are unwrapped ready for comparison below. localHost, localPort, err := net.SplitHostPort(configListenAddress) if err != nil { return nil, err } if localHost == "" || localHost == "0.0.0.0" || localHost == "::" { ifaces, err := net.Interfaces() if err != nil { return addresses, err } for _, i := range ifaces { addrs, err := i.Addrs() if err != nil { continue } for _, addr := range addrs { var ip net.IP switch v := addr.(type) { case *net.IPNet: ip = v.IP case *net.IPAddr: ip = v.IP } if !ip.IsGlobalUnicast() { continue } if ip.To4() == nil && localHost == "0.0.0.0" { continue } addresses = append(addresses, net.JoinHostPort(ip.String(), localPort)) } } } else { addresses = append(addresses, net.JoinHostPort(localHost, localPort)) } return addresses, nil } // IsJSONRequest returns true if the content type of the HTTP request is JSON. func IsJSONRequest(r *http.Request) bool { for k, vs := range r.Header { if strings.ToLower(k) == "content-type" && len(vs) == 1 && strings.ToLower(vs[0]) == "application/json" { return true } } return false } // CheckJwtToken checks whether the given request has JWT token that is valid and // signed with client certificate from the trusted certificates. // Returns whether or not the token is valid, the fingerprint of the certificate and the certificate. func CheckJwtToken(r *http.Request, trustedCerts map[string]x509.Certificate) (bool, string, *x509.Certificate) { var tokenString string // Try to get token from Authorization header. auth := r.Header.Get("Authorization") if auth != "" { parts := strings.Split(auth, " ") if len(parts) == 2 && strings.ToLower(parts[0]) == "bearer" { tokenString = parts[1] } } // If no token from header, check access_token query parameter. if tokenString == "" { tokenString = r.URL.Query().Get("access_token") } // Return if no token found. if tokenString == "" { return false, "", nil } // Get a new JWT parser. jwtParser := jwt.NewParser() // Parse the token. token, tokenParts, err := jwtParser.ParseUnverified(tokenString, &jwt.RegisteredClaims{}) if err != nil || len(tokenParts) < 2 { return false, "", nil } // Make sure this isn't an OIDC JWT. issuer, err := token.Claims.GetIssuer() if err != nil || issuer != "" { return false, "", nil } // Check if the token is valid (not before / expiration). notBefore, err := token.Claims.GetNotBefore() if err != nil { return false, "", nil } expiresAt, err := token.Claims.GetExpirationTime() if err != nil { return false, "", nil } if (notBefore != nil && time.Now().Before(notBefore.Time)) || (expiresAt != nil && time.Now().After(expiresAt.Time)) { return false, "", nil } // Find the certificate by the token subject. subject, err := token.Claims.GetSubject() if err != nil { return false, "", nil } tokenCert, ok := trustedCerts[subject] if !ok { return false, "", nil } // Verify token signature. tokenSigningString, err := token.SigningString() if err != nil { return false, "", nil } tokenSignature, err := base64.RawURLEncoding.DecodeString(tokenParts[2]) if err != nil { return false, "", nil } err = token.Method.Verify(tokenSigningString, tokenSignature, tokenCert.PublicKey) if err != nil { return false, "", nil } return true, subject, &tokenCert } incus-7.0.0/internal/server/util/http_test.go000066400000000000000000000027451517523235500213230ustar00rootroot00000000000000package util import ( "fmt" ) func ExampleListenAddresses() { listenAddressConfigs := []string{ "", "127.0.0.1:8000", // Valid IPv4 address with port. "127.0.0.1", // Valid IPv4 address without port. "[127.0.0.1]", // Valid wrapped IPv4 address without port. "[::1]:8000", // Valid IPv6 address with port. "::1:8000", // Valid IPv6 address without port (that might look like a port). "::1", // Valid IPv6 address without port. "[::1]", // Valid wrapped IPv6 address without port. "example.com", // Valid hostname without port. "example.com:8000", // Valid hostname with port. "foo:8000:9000", // Invalid host and port combination. ":::8000", // Invalid host and port combination. } for _, listlistenAddressConfig := range listenAddressConfigs { listenAddress, err := ListenAddresses(listlistenAddressConfig) fmt.Printf("%q: %v %v\n", listlistenAddressConfig, listenAddress, err) } // Output: "": [] // "127.0.0.1:8000": [127.0.0.1:8000] // "127.0.0.1": [127.0.0.1:8443] // "[127.0.0.1]": [127.0.0.1:8443] // "[::1]:8000": [[::1]:8000] // "::1:8000": [[::1:8000]:8443] // "::1": [[::1]:8443] // "[::1]": [[::1]:8443] // "example.com": [example.com:8443] // "example.com:8000": [example.com:8000] // "foo:8000:9000": [] address foo:8000:9000: too many colons in address // ":::8000": [] address :::8000: too many colons in address } incus-7.0.0/internal/server/util/kernel.go000066400000000000000000000027261517523235500205640ustar00rootroot00000000000000package util import ( "bufio" "errors" "os" "slices" "strings" ) // SupportsFilesystem checks whether a given filesystem is already supported // by the kernel. Note that if the filesystem is a module, you may need to // load it first. func SupportsFilesystem(filesystem string) bool { file, err := os.Open("/proc/filesystems") if err != nil { return false } defer func() { _ = file.Close() }() scanner := bufio.NewScanner(file) for scanner.Scan() { fields := strings.Fields(strings.TrimSpace(scanner.Text())) entry := fields[len(fields)-1] if entry == filesystem { return true } } return false } // HugepagesPath attempts to locate the mount point of the hugepages filesystem. func HugepagesPath() (string, error) { // Find the source mount of the path file, err := os.Open("/proc/mounts") if err != nil { return "", err } defer func() { _ = file.Close() }() scanner := bufio.NewScanner(file) matches := []string{} for scanner.Scan() { line := scanner.Text() cols := strings.Fields(line) if len(cols) < 3 { continue } if cols[2] == "hugetlbfs" { matches = append(matches, cols[1]) } } if len(matches) == 0 { return "", errors.New("No hugetlbfs mount found, can't use hugepages") } if len(matches) > 1 { if slices.Contains(matches, "/dev/hugepages") { return "/dev/hugepages", nil } return "", errors.New("More than one hugetlbfs instance found and none at standard /dev/hugepages") } return matches[0], nil } incus-7.0.0/internal/server/util/net.go000066400000000000000000000061201517523235500200620ustar00rootroot00000000000000package util import ( "crypto/tls" "crypto/x509" "errors" "fmt" "net" "os" "github.com/lxc/incus/v7/shared/logger" localtls "github.com/lxc/incus/v7/shared/tls" ) // InMemoryNetwork creates a fully in-memory listener and dial function. // // Each time the dial function is invoked a new pair of net.Conn objects will // be created using net.Pipe: the listener's Accept method will unblock and // return one end of the pipe and the other end will be returned by the dial // function. func InMemoryNetwork() (net.Listener, func() net.Conn) { listener := &inMemoryListener{ conns: make(chan net.Conn, 16), closed: make(chan struct{}), } dialer := func() net.Conn { server, client := net.Pipe() listener.conns <- server return client } return listener, dialer } type inMemoryListener struct { conns chan net.Conn closed chan struct{} } // Accept waits for and returns the next connection to the listener. func (l *inMemoryListener) Accept() (net.Conn, error) { select { case conn := <-l.conns: return conn, nil case <-l.closed: return nil, errors.New("closed") } } // Close closes the listener. // Any blocked Accept operations will be unblocked and return errors. func (l *inMemoryListener) Close() error { close(l.closed) return nil } // Addr returns the listener's network address. func (l *inMemoryListener) Addr() net.Addr { return &inMemoryAddr{} } type inMemoryAddr struct{} // Network returns the network name (implements net.Addr). func (a *inMemoryAddr) Network() string { return "memory" } func (a *inMemoryAddr) String() string { return "" } // ServerTLSConfig returns a new server-side tls.Config generated from the give // certificate info. func ServerTLSConfig(cert *localtls.CertInfo) *tls.Config { config := localtls.InitTLSConfig() config.ClientAuth = tls.RequestClientCert config.Certificates = []tls.Certificate{cert.KeyPair()} config.NextProtos = []string{"h2"} // Required by gRPC if cert.CA() != nil { pool := x509.NewCertPool() pool.AddCert(cert.CA()) config.RootCAs = pool config.ClientCAs = pool logger.Infof("Incus is in CA mode, only CA-signed certificates will be allowed") } return config } // SysctlGet retrieves the value of a sysctl file in /proc/sys. func SysctlGet(path string) (string, error) { // Read the current content content, err := os.ReadFile(fmt.Sprintf("/proc/sys/%s", path)) if err != nil { return "", err } return string(content), nil } // SysctlSet writes a value to a sysctl file in /proc/sys. // Requires an even number of arguments as key/value pairs. E.g. SysctlSet("path1", "value1", "path2", "value2"). func SysctlSet(parts ...string) error { partsLen := len(parts) if partsLen%2 != 0 { return errors.New("Requires even number of arguments") } for i := 0; i < partsLen; i = i + 2 { path := parts[i] newValue := parts[i+1] // Get current value. currentValue, err := SysctlGet(path) if err == nil && currentValue == newValue { // Nothing to update. return nil } err = os.WriteFile(fmt.Sprintf("/proc/sys/%s", path), []byte(newValue), 0) if err != nil { return err } } return nil } incus-7.0.0/internal/server/util/net_test.go000066400000000000000000000017201517523235500211220ustar00rootroot00000000000000package util_test import ( "io" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/server/util" ) // The connection returned by the dialer is paired with the one returned by the // Accept() method of the listener. func TestInMemoryNetwork(t *testing.T) { listener, dialer := util.InMemoryNetwork() client := dialer() server, err := listener.Accept() require.NoError(t, err) go func() { _, err := client.Write([]byte("hello")) require.NoError(t, err) }() buffer := make([]byte, 5) n, err := server.Read(buffer) require.NoError(t, err) assert.Equal(t, 5, n) assert.Equal(t, []byte("hello"), buffer) // Closing the server makes all further client reads and // writes fail. err = server.Close() assert.NoError(t, err) _, err = client.Read(buffer) assert.Equal(t, io.EOF, err) _, err = client.Write([]byte("hello")) assert.EqualError(t, err, "io: read/write on closed pipe") } incus-7.0.0/internal/server/util/random.go000066400000000000000000000026511517523235500205610ustar00rootroot00000000000000package util import ( "errors" "fmt" "hash/fnv" "io" "math/rand" ) // GetStableRandomGenerator returns a stable random generator. Uses the FNV-1a hash algorithm to convert the seed // string into an int64 for use as seed to the non-cryptographic random number generator. func GetStableRandomGenerator(seed string) (*rand.Rand, error) { hash := fnv.New64a() _, err := io.WriteString(hash, seed) if err != nil { return nil, err } return rand.New(rand.NewSource(int64(hash.Sum64()))), nil } // GetStableRandomInt64FromList returns a stable random value from a given list. func GetStableRandomInt64FromList(seed int64, list []int64) (int64, error) { if len(list) <= 0 { return 0, errors.New("Cannot get stable random value from empty list") } r, err := GetStableRandomGenerator(fmt.Sprintf("%d", seed)) if err != nil { return 0, fmt.Errorf("Failed to get stable random generator: %w", err) } return list[r.Int63n(int64(len(list)))], nil } // GenerateSequenceInt64 returns a sequence within a given range with given steps. func GenerateSequenceInt64(begin, end, step int) ([]int64, error) { if step == 0 { return []int64{}, errors.New("Step must not be zero") } count := 0 if (end > begin && step > 0) || (end < begin && step < 0) { count = (end-step-begin)/step + 1 } sequence := make([]int64, count) for i := 0; i < count; i, begin = i+1, begin+step { sequence[i] = int64(begin) } return sequence, nil } incus-7.0.0/internal/server/util/reader.go000066400000000000000000000024661517523235500205470ustar00rootroot00000000000000package util import ( "io" ) // MaxBytesReader provides a ReadCloser wrapper which returns an error when reading past a set limit. // This is based on http.MaxBytesReader but adapted for use outside of http. func MaxBytesReader(r io.Reader, n int64) io.Reader { if n < 0 { // Treat negative limits as equivalent to 0. n = 0 } return &maxBytesReader{r: r, i: n, n: n} } // MaxBytesError is returned by [MaxBytesReader] when its read limit is exceeded. type MaxBytesError struct { Limit int64 } func (e *MaxBytesError) Error() string { return "input data too large" } type maxBytesReader struct { r io.Reader // underlying reader i int64 // max bytes initially, for MaxBytesError n int64 // max bytes remaining err error // sticky error } func (l *maxBytesReader) Read(p []byte) (n int, err error) { if l.err != nil { return 0, l.err } if len(p) == 0 { return 0, nil } // If they asked for a 32KB byte read but only 5 bytes are // remaining, no need to read 32KB. 6 bytes will answer the // question of the whether we hit the limit or go past it. // 0 < len(p) < 2^63 if int64(len(p))-1 > l.n { p = p[:l.n+1] } n, err = l.r.Read(p) if int64(n) <= l.n { l.n -= int64(n) l.err = err return n, err } n = int(l.n) l.n = 0 l.err = &MaxBytesError{l.i} return n, l.err } incus-7.0.0/internal/server/util/sys.go000066400000000000000000000026201517523235500201130ustar00rootroot00000000000000//go:build linux && cgo && !agent package util import ( "os" "strings" "golang.org/x/sys/unix" "github.com/lxc/incus/v7/shared/osarch" ) // GetArchitectures returns the list of supported architectures. func GetArchitectures() ([]int, error) { architectures := []int{} architectureName, err := osarch.ArchitectureGetLocal() if err != nil { return nil, err } architecture, err := osarch.ArchitectureID(architectureName) if err != nil { return nil, err } architectures = append(architectures, architecture) personalities, err := osarch.ArchitecturePersonalities(architecture) if err != nil { return nil, err } architectures = append(architectures, personalities...) return architectures, nil } // GetExecPath returns the path to the current binary. func GetExecPath() string { execPath := os.Getenv("INCUS_EXEC_PATH") if execPath != "" { return execPath } execPath, err := os.Readlink("/proc/self/exe") if err != nil { execPath = "bad-exec-path" } // The execPath from /proc/self/exe can end with " (deleted)" if the binary has been removed/changed // since it was first started, strip this so that we only return a valid path. return strings.TrimSuffix(execPath, " (deleted)") } // ReplaceDaemon replaces the daemon by re-execing the binary. func ReplaceDaemon() error { err := unix.Exec(GetExecPath(), os.Args, os.Environ()) if err != nil { return err } return nil } incus-7.0.0/internal/server/util/version.go000066400000000000000000000021101517523235500207540ustar00rootroot00000000000000package util import ( "errors" ) // CompareVersions compares the versions of two cluster members. // // A version consists of the version the member's schema and the number of API // extensions it supports. // // Return 0 if they equal, 1 if the first version is greater than the second // and 2 if the second is greater than the first. // // Return an error if inconsistent versions are detected, for example the first // member's schema is greater than the second's, but the number of extensions is // smaller. func CompareVersions(version1, version2 [2]int, checkExtensions bool) (int, error) { schema1, extensions1 := version1[0], version1[1] schema2, extensions2 := version2[0], version2[1] if !checkExtensions { // Don't compare API extensions. extensions1 = 0 extensions2 = 0 } if schema1 == schema2 && extensions1 == extensions2 { return 0, nil } if schema1 >= schema2 && extensions1 >= extensions2 { return 1, nil } if schema1 <= schema2 && extensions1 <= extensions2 { return 2, nil } return -1, errors.New("Cluster members have inconsistent versions") } incus-7.0.0/internal/server/vsock/000077500000000000000000000000001517523235500171165ustar00rootroot00000000000000incus-7.0.0/internal/server/vsock/vsock.go000066400000000000000000000034751517523235500206030ustar00rootroot00000000000000package vsock import ( "context" "net" "net/http" "strings" "time" "github.com/mdlayher/vsock" localtls "github.com/lxc/incus/v7/shared/tls" ) // Dial connects to a remote vsock. func Dial(cid, port uint32) (net.Conn, error) { return vsock.Dial(cid, port, nil) } // HTTPClient provides an HTTP client for using over vsock. func HTTPClient(vsockID uint32, port int, tlsClientCert string, tlsClientKey string, tlsServerCert string) (*http.Client, error) { client := &http.Client{} // Get the TLS configuration. tlsConfig, err := localtls.GetTLSConfigMem(tlsClientCert, tlsClientKey, "", tlsServerCert, false) if err != nil { return nil, err } client.Transport = &http.Transport{ TLSClientConfig: tlsConfig, // Setup a VM socket dialer. DialContext: func(_ context.Context, network, addr string) (net.Conn, error) { var conn net.Conn var err error // Retry for up to 1s at 100ms interval to handle various failures. for range 10 { conn, err = Dial(vsockID, uint32(port)) if err == nil { break } else { // Handle some fatal errors. msg := err.Error() if strings.Contains(msg, "connection timed out") { // Retry once. conn, err = Dial(vsockID, uint32(port)) break } else if strings.Contains(msg, "connection refused") { break } // Retry the rest. } time.Sleep(100 * time.Millisecond) } if err != nil { return nil, err } return conn, nil }, DisableKeepAlives: true, ExpectContinueTimeout: time.Second * 30, ResponseHeaderTimeout: time.Second * 3600, TLSHandshakeTimeout: time.Second * 5, } // Setup redirect policy. client.CheckRedirect = func(req *http.Request, via []*http.Request) error { // Replicate the headers. req.Header = via[len(via)-1].Header return nil } return client, nil } incus-7.0.0/internal/server/warnings/000077500000000000000000000000001517523235500176215ustar00rootroot00000000000000incus-7.0.0/internal/server/warnings/warnings.go000066400000000000000000000172451517523235500220110ustar00rootroot00000000000000package warnings import ( "context" "errors" "fmt" "time" "github.com/lxc/incus/v7/internal/server/db" "github.com/lxc/incus/v7/internal/server/db/cluster" "github.com/lxc/incus/v7/internal/server/db/warningtype" ) // ResolveWarningsByLocalNodeOlderThan resolves all warnings which are older than the provided time. func ResolveWarningsByLocalNodeOlderThan(dbCluster *db.Cluster, date time.Time) error { var err error var localName string err = dbCluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { localName, err = tx.GetLocalNodeName(ctx) if err != nil { return err } return nil }) if err != nil { return fmt.Errorf("Failed getting local member name: %w", err) } if localName == "" { return errors.New("Local member name not available") } err = dbCluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { warnings, err := cluster.GetWarnings(ctx, tx.Tx()) if err != nil { return err } for _, w := range warnings { if w.Node != localName { continue } if w.LastSeenDate.Before(date) { err = tx.UpdateWarningStatus(w.UUID, warningtype.StatusResolved) if err != nil { return err } } } return nil }) if err != nil { return fmt.Errorf("Failed to resolve warnings: %w", err) } return nil } // ResolveWarningsByLocalNodeAndType resolves warnings with the local member and type code. // Returns error if no local member name. func ResolveWarningsByLocalNodeAndType(dbCluster *db.Cluster, typeCode warningtype.Type) error { var err error var localName string err = dbCluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { localName, err = tx.GetLocalNodeName(ctx) if err != nil { return err } return nil }) if err != nil { return fmt.Errorf("Failed getting local member name: %w", err) } if localName == "" { return errors.New("Local member name not available") } return ResolveWarningsByNodeAndType(dbCluster, localName, typeCode) } // ResolveWarningsByNodeAndType resolves warnings with the given node and type code. func ResolveWarningsByNodeAndType(dbCluster *db.Cluster, nodeName string, typeCode warningtype.Type) error { err := dbCluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { filter := cluster.WarningFilter{ TypeCode: &typeCode, Node: &nodeName, } warnings, err := cluster.GetWarnings(ctx, tx.Tx(), filter) if err != nil { return err } for _, w := range warnings { err = tx.UpdateWarningStatus(w.UUID, warningtype.StatusResolved) if err != nil { return err } } return nil }) if err != nil { return fmt.Errorf("Failed to resolve warnings: %w", err) } return nil } // ResolveWarningsByNodeAndProjectAndType resolves warnings with the given node, project and type code. func ResolveWarningsByNodeAndProjectAndType(dbCluster *db.Cluster, nodeName string, projectName string, typeCode warningtype.Type) error { err := dbCluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { filter := cluster.WarningFilter{ TypeCode: &typeCode, Node: &nodeName, Project: &projectName, } warnings, err := cluster.GetWarnings(ctx, tx.Tx(), filter) if err != nil { return err } for _, w := range warnings { err = tx.UpdateWarningStatus(w.UUID, warningtype.StatusResolved) if err != nil { return err } } return nil }) if err != nil { return fmt.Errorf("Failed to resolve warnings: %w", err) } return nil } // ResolveWarningsByLocalNodeAndProjectAndType resolves warnings with the given project and type code. func ResolveWarningsByLocalNodeAndProjectAndType(dbCluster *db.Cluster, projectName string, typeCode warningtype.Type) error { var err error var localName string err = dbCluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { localName, err = tx.GetLocalNodeName(ctx) if err != nil { return err } return nil }) if err != nil { return fmt.Errorf("Failed getting local member name: %w", err) } if localName == "" { return errors.New("Local member name not available") } return ResolveWarningsByNodeAndProjectAndType(dbCluster, localName, projectName, typeCode) } // ResolveWarningsByNodeAndProjectAndTypeAndEntity resolves warnings with the given node, project, type code, and entity. func ResolveWarningsByNodeAndProjectAndTypeAndEntity(dbCluster *db.Cluster, nodeName string, projectName string, typeCode warningtype.Type, entityTypeCode int, entityID int) error { err := dbCluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { filter := cluster.WarningFilter{ TypeCode: &typeCode, Node: &nodeName, Project: &projectName, EntityTypeCode: &entityTypeCode, EntityID: &entityID, } warnings, err := cluster.GetWarnings(ctx, tx.Tx(), filter) if err != nil { return err } for _, w := range warnings { err = tx.UpdateWarningStatus(w.UUID, warningtype.StatusResolved) if err != nil { return err } } return nil }) if err != nil { return fmt.Errorf("Failed to resolve warnings: %w", err) } return nil } // ResolveWarningsByLocalNodeAndProjectAndTypeAndEntity resolves warnings with the given project, type code, and entity. func ResolveWarningsByLocalNodeAndProjectAndTypeAndEntity(dbCluster *db.Cluster, projectName string, typeCode warningtype.Type, entityTypeCode int, entityID int) error { var err error var localName string err = dbCluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { localName, err = tx.GetLocalNodeName(ctx) if err != nil { return err } return nil }) if err != nil { return fmt.Errorf("Failed getting local member name: %w", err) } if localName == "" { return errors.New("Local member name not available") } return ResolveWarningsByNodeAndProjectAndTypeAndEntity(dbCluster, localName, projectName, typeCode, entityTypeCode, entityID) } // DeleteWarningsByNodeAndProjectAndTypeAndEntity deletes warnings with the given node, project, type code, and entity. func DeleteWarningsByNodeAndProjectAndTypeAndEntity(dbCluster *db.Cluster, nodeName string, projectName string, typeCode warningtype.Type, entityTypeCode int, entityID int) error { err := dbCluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { filter := cluster.WarningFilter{ TypeCode: &typeCode, Node: &nodeName, Project: &projectName, EntityTypeCode: &entityTypeCode, EntityID: &entityID, } warnings, err := cluster.GetWarnings(ctx, tx.Tx(), filter) if err != nil { return err } for _, w := range warnings { err = cluster.DeleteWarning(ctx, tx.Tx(), w.UUID) if err != nil { return err } } return nil }) if err != nil { return fmt.Errorf("Failed to delete warnings: %w", err) } return nil } // DeleteWarningsByLocalNodeAndProjectAndTypeAndEntity resolves warnings with the given project, type code, and entity. func DeleteWarningsByLocalNodeAndProjectAndTypeAndEntity(dbCluster *db.Cluster, projectName string, typeCode warningtype.Type, entityTypeCode int, entityID int) error { var err error var localName string err = dbCluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error { localName, err = tx.GetLocalNodeName(ctx) if err != nil { return err } return nil }) if err != nil { return fmt.Errorf("Failed getting local member name: %w", err) } if localName == "" { return errors.New("Local member name not available") } return DeleteWarningsByNodeAndProjectAndTypeAndEntity(dbCluster, localName, projectName, typeCode, entityTypeCode, entityID) } incus-7.0.0/internal/sql/000077500000000000000000000000001517523235500152625ustar00rootroot00000000000000incus-7.0.0/internal/sql/struct.go000066400000000000000000000012451517523235500171370ustar00rootroot00000000000000package sql // SQLDump represents a full database dump. type SQLDump struct { Text string `json:"text" yaml:"text"` } // SQLQuery represents a DB query. type SQLQuery struct { Database string `json:"database" yaml:"database"` Query string `json:"query" yaml:"query"` } // SQLBatch represents a batch result. type SQLBatch struct { Results []SQLResult } // SQLResult represents a query result. type SQLResult struct { Type string `json:"type" yaml:"type"` Columns []string `json:"columns" yaml:"columns"` Rows [][]any `json:"rows" yaml:"rows"` RowsAffected int64 `json:"rows_affected" yaml:"rows_affected"` } incus-7.0.0/internal/util/000077500000000000000000000000001517523235500154405ustar00rootroot00000000000000incus-7.0.0/internal/util/cert.go000066400000000000000000000037031517523235500167270ustar00rootroot00000000000000package util import ( "fmt" "os" "path/filepath" localtls "github.com/lxc/incus/v7/shared/tls" "github.com/lxc/incus/v7/shared/util" ) // LoadCert reads the server certificate from the given var dir. // // If a cluster certificate is found it will be loaded instead. // If neither a server or cluster certfificate exists, a new server certificate will be generated. func LoadCert(dir string) (*localtls.CertInfo, error) { prefix := "server" if util.PathExists(filepath.Join(dir, "cluster.crt")) { prefix = "cluster" } cert, err := localtls.KeyPairAndCA(dir, prefix, localtls.CertServer, true) if err != nil { return nil, fmt.Errorf("failed to load TLS certificate: %w", err) } return cert, nil } // LoadClusterCert reads the cluster certificate from the given var dir. // // If a cluster certificate doesn't exist, a new one is generated. func LoadClusterCert(dir string) (*localtls.CertInfo, error) { prefix := "cluster" cert, err := localtls.KeyPairAndCA(dir, prefix, localtls.CertServer, true) if err != nil { return nil, fmt.Errorf("failed to load cluster TLS certificate: %w", err) } return cert, nil } // LoadServerCert reads the server certificate from the given var dir. func LoadServerCert(dir string) (*localtls.CertInfo, error) { prefix := "server" cert, err := localtls.KeyPairAndCA(dir, prefix, localtls.CertServer, true) if err != nil { return nil, fmt.Errorf("failed to load TLS certificate: %w", err) } return cert, nil } // WriteCert writes the given material to the appropriate certificate files in // the given directory. func WriteCert(dir, prefix string, cert, key, ca []byte) error { err := os.WriteFile(filepath.Join(dir, prefix+".crt"), cert, 0o644) if err != nil { return err } err = os.WriteFile(filepath.Join(dir, prefix+".key"), key, 0o600) if err != nil { return err } if ca != nil { err = os.WriteFile(filepath.Join(dir, prefix+".ca"), ca, 0o644) if err != nil { return err } } return nil } incus-7.0.0/internal/util/filesystem.go000066400000000000000000000116671517523235500201660ustar00rootroot00000000000000package util import ( "errors" "fmt" "io" "os" "path/filepath" "runtime" internalIO "github.com/lxc/incus/v7/internal/io" "github.com/lxc/incus/v7/shared/util" ) // FileMove tries to move a file by using os.Rename, // if that fails it tries to copy the file and remove the source. func FileMove(oldPath string, newPath string) error { err := os.Rename(oldPath, newPath) if err == nil { return nil } err = FileCopy(oldPath, newPath) if err != nil { return err } _ = os.Remove(oldPath) return nil } // FileCopy copies a file, overwriting the target if it exists. func FileCopy(source string, dest string) error { fi, err := os.Lstat(source) if err != nil { return err } _, uid, gid := internalIO.GetOwnerMode(fi) if fi.Mode()&os.ModeSymlink != 0 { target, err := os.Readlink(source) if err != nil { return err } if util.PathExists(dest) { err = os.Remove(dest) if err != nil { return err } } err = os.Symlink(target, dest) if err != nil { return err } if runtime.GOOS != "windows" { return os.Lchown(dest, uid, gid) } return nil } s, err := os.Open(source) if err != nil { return err } defer func() { _ = s.Close() }() d, err := os.Create(dest) if err != nil { if !os.IsExist(err) { return err } d, err = os.OpenFile(dest, os.O_WRONLY, fi.Mode()) if err != nil { return err } } _, err = util.SafeCopy(d, s) if err != nil { return err } /* chown not supported on windows */ if runtime.GOOS != "windows" { err = d.Chown(uid, gid) if err != nil { return err } } return d.Close() } // DirCopy copies a directory recursively, overwriting the target if it exists. func DirCopy(source string, dest string) error { // Get info about source. info, err := os.Stat(source) if err != nil { return fmt.Errorf("failed to get source directory info: %w", err) } if !info.IsDir() { return errors.New("source is not a directory") } // Remove dest if it already exists. if util.PathExists(dest) { err := os.RemoveAll(dest) if err != nil { return fmt.Errorf("failed to remove destination directory %s: %w", dest, err) } } // Create dest. err = os.MkdirAll(dest, info.Mode()) if err != nil { return fmt.Errorf("failed to create destination directory %s: %w", dest, err) } // Copy all files. entries, err := os.ReadDir(source) if err != nil { return fmt.Errorf("failed to read source directory %s: %w", source, err) } for _, entry := range entries { sourcePath := filepath.Join(source, entry.Name()) destPath := filepath.Join(dest, entry.Name()) if entry.IsDir() { err := DirCopy(sourcePath, destPath) if err != nil { return fmt.Errorf("failed to copy sub-directory from %s to %s: %w", sourcePath, destPath, err) } } else { err := FileCopy(sourcePath, destPath) if err != nil { return fmt.Errorf("failed to copy file from %s to %s: %w", sourcePath, destPath, err) } } } return nil } // AddSlash adds a slash to the end of paths if they don't already have one. // This can be useful for rsyncing things, since rsync has behavior present on // the presence or absence of a trailing slash. func AddSlash(path string) string { if path[len(path)-1] != '/' { return path + "/" } return path } // PathIsEmpty checks if the given path is empty. func PathIsEmpty(path string) (bool, error) { f, err := os.Open(path) if err != nil { return false, err } defer func() { _ = f.Close() }() // read in ONLY one file _, err = f.ReadDir(1) // and if the file is EOF... well, the dir is empty. if err == io.EOF { return true, nil } return false, err } // MkdirAllOwner ensures that directories path exists. // If directories in the path are missing - they will be created with specified permissions and owner. func MkdirAllOwner(path string, perm os.FileMode, uid int, gid int) error { // This function is a slightly modified version of MkdirAll from the Go standard library. // https://golang.org/src/os/path.go?s=488:535#L9 // Fast path: if we can tell whether path is a directory or file, stop with success or error. dir, err := os.Stat(path) if err == nil { if dir.IsDir() { return nil } return errors.New("path exists but isn't a directory") } // Slow path: make sure parent exists and then call Mkdir for path. i := len(path) for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator. i-- } j := i for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element. j-- } if j > 1 { // Create parent err = MkdirAllOwner(path[0:j-1], perm, uid, gid) if err != nil { return err } } // Parent now exists; invoke Mkdir and use its result. err = os.Mkdir(path, perm) errChown := os.Chown(path, uid, gid) if errChown != nil { return errChown } if err != nil { // Handle arguments like "foo/." by // double-checking that directory doesn't exist. dir, err1 := os.Lstat(path) if err1 == nil && dir.IsDir() { return nil } return err } return nil } incus-7.0.0/internal/util/network.go000066400000000000000000000114561517523235500174670ustar00rootroot00000000000000package util import ( "fmt" "net" "slices" "github.com/lxc/incus/v7/internal/ports" ) // CanonicalNetworkAddress parses the given network address and returns a string of the form "host:port", // possibly filling it with the default port if it's missing. It will also wrap a bare IPv6 address with square // brackets if needed. func CanonicalNetworkAddress(address string, defaultPort int) string { host, port, err := net.SplitHostPort(address) if err != nil { ip := net.ParseIP(address) if ip != nil { // If the input address is a bare IP address, then convert it to a proper listen address // using the canonical IP with default port and wrap IPv6 addresses in square brackets. return net.JoinHostPort(ip.String(), fmt.Sprintf("%d", defaultPort)) } // Otherwise assume this is either a host name or a partial address (e.g `[::]`) without // a port number, so append the default port. return fmt.Sprintf("%s:%d", address, defaultPort) } if port == "" && address[len(address)-1] == ':' { // An address that ends with a trailing colon will be parsed as having an empty port. return net.JoinHostPort(host, fmt.Sprintf("%d", defaultPort)) } return address } // CanonicalNetworkAddressFromAddressAndPort returns a network address from separate address and port values. // The address accepts values such as "[::]", "::" and "localhost". func CanonicalNetworkAddressFromAddressAndPort(address string, port int, defaultPort int) string { // Because we accept just the host part of an IPv6 listen address (e.g. `[::]`) don't use net.JoinHostPort. // If a bare IP address is supplied then CanonicalNetworkAddress will use net.JoinHostPort if needed. return CanonicalNetworkAddress(fmt.Sprintf("%s:%d", address, port), defaultPort) } // NetworkInterfaceAddress returns the first global unicast address of any of the system network interfaces. // Return the empty string if none is found. func NetworkInterfaceAddress() string { ifaces, err := net.Interfaces() if err != nil { return "" } for _, iface := range ifaces { addrs, err := iface.Addrs() if err != nil { continue } if len(addrs) == 0 { continue } for _, addr := range addrs { ipNet, ok := addr.(*net.IPNet) if !ok { continue } if !ipNet.IP.IsGlobalUnicast() { continue } return ipNet.IP.String() } } return "" } // IsAddressCovered detects if network address1 is actually covered by // address2, in the sense that they are either the same address or address2 is // specified using a wildcard with the same port of address1. func IsAddressCovered(address1, address2 string) bool { address1 = CanonicalNetworkAddress(address1, ports.HTTPSDefaultPort) address2 = CanonicalNetworkAddress(address2, ports.HTTPSDefaultPort) if address1 == address2 { return true } host1, port1, err := net.SplitHostPort(address1) if err != nil { return false } host2, port2, err := net.SplitHostPort(address2) if err != nil { return false } // If the ports are different, then address1 is clearly not covered by // address2. if port2 != port1 { return false } // If address1 contains a host name, let's try to resolve it, in order // to compare the actual IPs. var addresses1 []net.IP if host1 != "" { ip := net.ParseIP(host1) if ip != nil { addresses1 = append(addresses1, ip) } else { ips, err := net.LookupHost(host1) if err == nil && len(ips) > 0 { for _, ipStr := range ips { ip := net.ParseIP(ipStr) if ip != nil { addresses1 = append(addresses1, ip) } } } } } // If address2 contains a host name, let's try to resolve it, in order // to compare the actual IPs. var addresses2 []net.IP if host2 != "" { ip := net.ParseIP(host2) if ip != nil { addresses2 = append(addresses2, ip) } else { ips, err := net.LookupHost(host2) if err == nil && len(ips) > 0 { for _, ipStr := range ips { ip := net.ParseIP(ipStr) if ip != nil { addresses2 = append(addresses2, ip) } } } } } for _, a1 := range addresses1 { if slices.ContainsFunc(addresses2, a1.Equal) { return true } } // If address2 is using an IPv4 wildcard for the host, then address2 is // only covered if it's an IPv4 address. if host2 == "0.0.0.0" { ip1 := net.ParseIP(host1) if ip1 != nil && ip1.To4() != nil { return true } return false } // If address2 is using an IPv6 wildcard for the host, then address2 is // always covered. if host2 == "::" || host2 == "" { return true } return false } // IsWildCardAddress returns whether the given address is a wildcard. func IsWildCardAddress(address string) bool { address = CanonicalNetworkAddress(address, ports.HTTPSDefaultPort) host, _, err := net.SplitHostPort(address) if err != nil { return false } if host == "0.0.0.0" || host == "::" || host == "" { return true } return false } incus-7.0.0/internal/util/network_test.go000066400000000000000000000056741517523235500205330ustar00rootroot00000000000000package util_test import ( "fmt" "net" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/lxc/incus/v7/internal/ports" internalUtil "github.com/lxc/incus/v7/internal/util" ) func TestCanonicalNetworkAddress(t *testing.T) { cases := map[string]string{ "127.0.0.1": "127.0.0.1:8443", "127.0.0.1:": "127.0.0.1:8443", "foo.bar": "foo.bar:8443", "foo.bar:": "foo.bar:8443", "foo.bar:8444": "foo.bar:8444", "192.168.1.1:443": "192.168.1.1:443", "f921:7358:4510:3fce:ac2e:844:2a35:54e": "[f921:7358:4510:3fce:ac2e:844:2a35:54e]:8443", "[f921:7358:4510:3fce:ac2e:844:2a35:54e]": "[f921:7358:4510:3fce:ac2e:844:2a35:54e]:8443", "[f921:7358:4510:3fce:ac2e:844:2a35:54e]:": "[f921:7358:4510:3fce:ac2e:844:2a35:54e]:8443", "[f921:7358:4510:3fce:ac2e:844:2a35:54e]:8444": "[f921:7358:4510:3fce:ac2e:844:2a35:54e]:8444", } for in, out := range cases { t.Run(in, func(t *testing.T) { assert.Equal(t, out, internalUtil.CanonicalNetworkAddress(in, ports.HTTPSDefaultPort)) }) } } func TestIsAddressCovered(t *testing.T) { type testCase struct { address1 string address2 string covered bool } cases := []testCase{ {"127.0.0.1:8443", "127.0.0.1:8443", true}, {"garbage", "127.0.0.1:8443", false}, {"127.0.0.1:8444", "garbage", false}, {"127.0.0.1:8444", "127.0.0.1:8443", false}, {"127.0.0.1:8443", "0.0.0.0:8443", true}, {"[::1]:8443", "0.0.0.0:8443", false}, {":8443", "0.0.0.0:8443", false}, {"127.0.0.1:8443", "[::]:8443", true}, {"[::1]:8443", "[::]:8443", true}, {"[::1]:8443", ":8443", true}, {":8443", "[::]:8443", true}, {"0.0.0.0:8443", "[::]:8443", true}, {"10.30.0.8:8443", "[::]", true}, {"localhost:8443", "127.0.0.1:8443", true}, {"localhost:8443", "[::]:8443", true}, } // Test some localhost cases too ips, err := net.LookupHost("localhost") if err == nil && len(ips) > 0 && ips[0] == "127.0.0.1" { cases = append(cases, testCase{"127.0.0.1:8443", "localhost:8443", true}) } ips, err = net.LookupHost("ip6-localhost") if err == nil && len(ips) > 0 && ips[0] == "::1" { cases = append(cases, testCase{"[::1]:8443", "ip6-localhost:8443", true}) } for _, c := range cases { t.Run(fmt.Sprintf("%s-%s", c.address1, c.address2), func(t *testing.T) { covered := internalUtil.IsAddressCovered(c.address1, c.address2) if c.covered { assert.True(t, covered) } else { assert.False(t, covered) } }) } } // This is a check against Go's stdlib to make sure that when listening to a port without specifying an address, // then an IPv6 wildcard is assumed. func TestListenImplicitIPv6Wildcard(t *testing.T) { listener, err := net.Listen("tcp", ":9999") require.NoError(t, err) defer func() { _ = listener.Close() }() assert.Equal(t, "[::]:9999", listener.Addr().String()) } incus-7.0.0/internal/util/paths.go000066400000000000000000000044141517523235500171110ustar00rootroot00000000000000package util import ( "os" "path/filepath" "github.com/lxc/incus/v7/shared/util" ) // CachePath returns the directory that Incus should its cache under. If INCUS_DIR is // set, this path is $INCUS_DIR/cache, otherwise it is /var/cache/incus. func CachePath(path ...string) string { varDir := os.Getenv("INCUS_DIR") logDir := "/var/cache/incus" if varDir != "" { logDir = filepath.Join(varDir, "cache") } items := []string{logDir} items = append(items, path...) return filepath.Join(items...) } // LogPath returns the directory that Incus should put logs under. If INCUS_DIR is // set, this path is $INCUS_DIR/logs, otherwise it is either /var/lib/incus/logs (if it exists) or /var/log/incus. func LogPath(path ...string) string { // Default log path is /var/log/incus/. logDir := "/var/log/incus" // If a folder exists under VarPath or if INCUS_DIR was explicitly set, use that. if util.PathExists(VarPath("logs")) || os.Getenv("INCUS_DIR") != "" { logDir = VarPath("logs") } items := []string{logDir} items = append(items, path...) return filepath.Join(items...) } // RunPath returns the directory that Incus should put runtime data under. // If INCUS_DIR is set, this path is $INCUS_DIR/run, otherwise it is /run/incus. func RunPath(path ...string) string { varDir := os.Getenv("INCUS_DIR") runDir := "/run/incus" if varDir != "" { runDir = filepath.Join(varDir, "run") } items := []string{runDir} items = append(items, path...) return filepath.Join(items...) } // VarPath returns the provided path elements joined by a slash and // appended to the end of $INCUS_DIR, which defaults to /var/lib/incus. func VarPath(path ...string) string { varDir := os.Getenv("INCUS_DIR") if varDir == "" { varDir = "/var/lib/incus" } items := []string{varDir} items = append(items, path...) return filepath.Join(items...) } // IsDir returns true if the given path is a directory. func IsDir(name string) bool { stat, err := os.Stat(name) if err != nil { return false } return stat.IsDir() } // IsUnixSocket returns true if the given path is either a Unix socket // or a symbolic link pointing at a Unix socket. func IsUnixSocket(path string) bool { stat, err := os.Stat(path) if err != nil { return false } return (stat.Mode() & os.ModeSocket) == os.ModeSocket } incus-7.0.0/internal/util/random.go000066400000000000000000000006201517523235500172450ustar00rootroot00000000000000package util import ( "crypto/rand" "encoding/hex" "errors" ) // RandomHexString returns a random string of hexadecimal characters. func RandomHexString(length int) (string, error) { buf := make([]byte, length) n, err := rand.Read(buf) if err != nil { return "", err } if n != len(buf) { return "", errors.New("not enough random bytes read") } return hex.EncodeToString(buf), nil } incus-7.0.0/internal/util/storage.go000066400000000000000000000005761517523235500174430ustar00rootroot00000000000000package util // PoolType represents a type of storage pool (local, remote or any). type PoolType string // PoolTypeAny represents any storage pool (local or remote). const PoolTypeAny PoolType = "" // PoolTypeLocal represents local storage pools. const PoolTypeLocal PoolType = "local" // PoolTypeRemote represents remote storage pools. const PoolTypeRemote PoolType = "remote" incus-7.0.0/internal/util/template.go000066400000000000000000000037651517523235500176150ustar00rootroot00000000000000package util import ( "errors" "fmt" "io" "strings" "github.com/flosch/pongo2/v6" ) var bannedTemplateTags = []string{"extends", "import", "include", "ssi"} // RenderTemplate renders a pongo2 template with nesting support. // This supports up to 3 levels of nesting (to avoid loops). func RenderTemplate(template string, ctx pongo2.Context) (string, error) { // Prepare a custom set. custom := pongo2.NewSet("render-template", pongo2.DefaultLoader) // Block the use of some tags. for _, tag := range bannedTemplateTags { err := custom.BanTag(tag) if err != nil { return "", fmt.Errorf("Failed to configure custom pongo2 parser: Failed to block tag tag %q: %w", tag, err) } } // Limit recursion to 3 levels. for range 3 { // Load template from string tpl, err := custom.FromString("{% autoescape off %}" + template + "{% endautoescape %}") if err != nil { return "", err } // Get rendered template ret, err := tpl.Execute(ctx) if err != nil { return ret, err } // Check if another pass is needed. if !strings.Contains(ret, "{{") && !strings.Contains(ret, "{%") { return ret, nil } // Prepare for another run. template = ret } return "", errors.New("Maximum template recursion limit reached") } // RenderTemplateFile renders a pongo2 template to a file. // No nesting is supported in this scenario. func RenderTemplateFile(w io.Writer, template string, ctx pongo2.Context) error { // Prepare a custom set. custom := pongo2.NewSet("render-template", pongo2.DefaultLoader) // Block the use of some tags. for _, tag := range bannedTemplateTags { err := custom.BanTag(tag) if err != nil { return fmt.Errorf("Failed to configure custom pongo2 parser: Failed to block tag tag %q: %w", tag, err) } } // Load template from string tpl, err := custom.FromString("{% autoescape off %}" + template + "{% endautoescape %}") if err != nil { return err } // Get rendered template err = tpl.ExecuteWriter(ctx, w) if err != nil { return err } return nil } incus-7.0.0/internal/util/token.go000066400000000000000000000015161517523235500171120ustar00rootroot00000000000000package util import ( "encoding/base64" "encoding/json" "errors" "github.com/lxc/incus/v7/shared/api" ) // JoinTokenDecode decodes a base64 and JSON encoded join token. func JoinTokenDecode(input string) (*api.ClusterMemberJoinToken, error) { joinTokenJSON, err := base64.StdEncoding.DecodeString(input) if err != nil { return nil, err } var j api.ClusterMemberJoinToken err = json.Unmarshal(joinTokenJSON, &j) if err != nil { return nil, err } if j.ServerName == "" { return nil, errors.New("No server name in join token") } if len(j.Addresses) < 1 { return nil, errors.New("No cluster member addresses in join token") } if j.Secret == "" { return nil, errors.New("No secret in join token") } if j.Fingerprint == "" { return nil, errors.New("No certificate fingerprint in join token") } return &j, nil } incus-7.0.0/internal/version/000077500000000000000000000000001517523235500161505ustar00rootroot00000000000000incus-7.0.0/internal/version/api.go000066400000000000000000000341721517523235500172570ustar00rootroot00000000000000package version import ( "os" "strconv" ) // APIVersion contains the API base version. Only bumped for backward incompatible changes. var APIVersion = "1.0" // APIExtensions is the list of all API extensions in the order they were added. // // The following kind of changes come with a new extensions: // // - New configuration key // - New valid values for a configuration key // - New REST API endpoint // - New argument inside an existing REST API call // - New HTTPs authentication mechanisms or protocols // // This list is used mainly by the server code, but it's in the shared // package as well for reference. var APIExtensions = []string{ "storage_zfs_remove_snapshots", "container_host_shutdown_timeout", "container_stop_priority", "container_syscall_filtering", "auth_pki", "container_last_used_at", "etag", "patch", "usb_devices", "https_allowed_credentials", "image_compression_algorithm", "directory_manipulation", "container_cpu_time", "storage_zfs_use_refquota", "storage_lvm_mount_options", "network", "profile_usedby", "container_push", "container_exec_recording", "certificate_update", "container_exec_signal_handling", "gpu_devices", "container_image_properties", "migration_progress", "id_map", "network_firewall_filtering", "network_routes", "storage", "file_delete", "file_append", "network_dhcp_expiry", "storage_lvm_vg_rename", "storage_lvm_thinpool_rename", "network_vlan", "image_create_aliases", "container_stateless_copy", "container_only_migration", "storage_zfs_clone_copy", "unix_device_rename", "storage_lvm_use_thinpool", "storage_rsync_bwlimit", "network_vxlan_interface", "storage_btrfs_mount_options", "entity_description", "image_force_refresh", "storage_lvm_lv_resizing", "id_map_base", "file_symlinks", "container_push_target", "network_vlan_physical", "storage_images_delete", "container_edit_metadata", "container_snapshot_stateful_migration", "storage_driver_ceph", "storage_ceph_user_name", "resource_limits", "storage_volatile_initial_source", "storage_ceph_force_osd_reuse", "storage_block_filesystem_btrfs", "resources", "kernel_limits", "storage_api_volume_rename", "network_sriov", "console", "restrict_dev_incus", "migration_pre_copy", "infiniband", "dev_incus_events", "proxy", "network_dhcp_gateway", "file_get_symlink", "network_leases", "unix_device_hotplug", "storage_api_local_volume_handling", "operation_description", "clustering", "event_lifecycle", "storage_api_remote_volume_handling", "nvidia_runtime", "container_mount_propagation", "container_backup", "dev_incus_images", "container_local_cross_pool_handling", "proxy_unix", "proxy_udp", "clustering_join", "proxy_tcp_udp_multi_port_handling", "network_state", "proxy_unix_dac_properties", "container_protection_delete", "unix_priv_drop", "pprof_http", "proxy_haproxy_protocol", "network_hwaddr", "proxy_nat", "network_nat_order", "container_full", "backup_compression", "nvidia_runtime_config", "storage_api_volume_snapshots", "storage_unmapped", "projects", "network_vxlan_ttl", "container_incremental_copy", "usb_optional_vendorid", "snapshot_scheduling", "snapshot_schedule_aliases", "container_copy_project", "clustering_server_address", "clustering_image_replication", "container_protection_shift", "snapshot_expiry", "container_backup_override_pool", "snapshot_expiry_creation", "network_leases_location", "resources_cpu_socket", "resources_gpu", "resources_numa", "kernel_features", "id_map_current", "event_location", "storage_api_remote_volume_snapshots", "network_nat_address", "container_nic_routes", "cluster_internal_copy", "seccomp_notify", "lxc_features", "container_nic_ipvlan", "network_vlan_sriov", "storage_cephfs", "container_nic_ipfilter", "resources_v2", "container_exec_user_group_cwd", "container_syscall_intercept", "container_disk_shift", "storage_shifted", "resources_infiniband", "daemon_storage", "instances", "image_types", "resources_disk_sata", "clustering_roles", "images_expiry", "resources_network_firmware", "backup_compression_algorithm", "ceph_data_pool_name", "container_syscall_intercept_mount", "compression_squashfs", "container_raw_mount", "container_nic_routed", "container_syscall_intercept_mount_fuse", "container_disk_ceph", "virtual-machines", "image_profiles", "clustering_architecture", "resources_disk_id", "storage_lvm_stripes", "vm_boot_priority", "unix_hotplug_devices", "api_filtering", "instance_nic_network", "clustering_sizing", "firewall_driver", "projects_limits", "container_syscall_intercept_hugetlbfs", "limits_hugepages", "container_nic_routed_gateway", "projects_restrictions", "custom_volume_snapshot_expiry", "volume_snapshot_scheduling", "trust_ca_certificates", "snapshot_disk_usage", "clustering_edit_roles", "container_nic_routed_host_address", "container_nic_ipvlan_gateway", "resources_usb_pci", "resources_cpu_threads_numa", "resources_cpu_core_die", "api_os", "container_nic_routed_host_table", "container_nic_ipvlan_host_table", "container_nic_ipvlan_mode", "resources_system", "images_push_relay", "network_dns_search", "container_nic_routed_limits", "instance_nic_bridged_vlan", "network_state_bond_bridge", "usedby_consistency", "custom_block_volumes", "clustering_failure_domains", "resources_gpu_mdev", "console_vga_type", "projects_limits_disk", "network_type_macvlan", "network_type_sriov", "container_syscall_intercept_bpf_devices", "network_type_ovn", "projects_networks", "projects_networks_restricted_uplinks", "custom_volume_backup", "backup_override_name", "storage_rsync_compression", "network_type_physical", "network_ovn_external_subnets", "network_ovn_nat", "network_ovn_external_routes_remove", "tpm_device_type", "storage_zfs_clone_copy_rebase", "gpu_mdev", "resources_pci_iommu", "resources_network_usb", "resources_disk_address", "network_physical_ovn_ingress_mode", "network_ovn_dhcp", "network_physical_routes_anycast", "projects_limits_instances", "network_state_vlan", "instance_nic_bridged_port_isolation", "instance_bulk_state_change", "network_gvrp", "instance_pool_move", "gpu_sriov", "pci_device_type", "storage_volume_state", "network_acl", "migration_stateful", "disk_state_quota", "storage_ceph_features", "projects_compression", "projects_images_remote_cache_expiry", "certificate_project", "network_ovn_acl", "projects_images_auto_update", "projects_restricted_cluster_target", "images_default_architecture", "network_ovn_acl_defaults", "gpu_mig", "project_usage", "network_bridge_acl", "warnings", "projects_restricted_backups_and_snapshots", "clustering_join_token", "clustering_description", "server_trusted_proxy", "clustering_update_cert", "storage_api_project", "server_instance_driver_operational", "server_supported_storage_drivers", "event_lifecycle_requestor_address", "resources_gpu_usb", "clustering_evacuation", "network_ovn_nat_address", "network_bgp", "network_forward", "custom_volume_refresh", "network_counters_errors_dropped", "metrics", "image_source_project", "clustering_config", "network_peer", "linux_sysctl", "network_dns", "ovn_nic_acceleration", "certificate_self_renewal", "instance_project_move", "storage_volume_project_move", "cloud_init", "network_dns_nat", "database_leader", "instance_all_projects", "clustering_groups", "ceph_rbd_du", "instance_get_full", "qemu_metrics", "gpu_mig_uuid", "event_project", "clustering_evacuation_live", "instance_allow_inconsistent_copy", "network_state_ovn", "storage_volume_api_filtering", "image_restrictions", "storage_zfs_export", "network_dns_records", "storage_zfs_reserve_space", "network_acl_log", "storage_zfs_blocksize", "metrics_cpu_seconds", "instance_snapshot_never", "certificate_token", "instance_nic_routed_neighbor_probe", "event_hub", "agent_nic_config", "projects_restricted_intercept", "metrics_authentication", "images_target_project", "images_all_projects", "cluster_migration_inconsistent_copy", "cluster_ovn_chassis", "container_syscall_intercept_sched_setscheduler", "storage_lvm_thinpool_metadata_size", "storage_volume_state_total", "instance_file_head", "instances_nic_host_name", "image_copy_profile", "container_syscall_intercept_sysinfo", "clustering_evacuation_mode", "resources_pci_vpd", "qemu_raw_conf", "storage_cephfs_fscache", "network_load_balancer", "vsock_api", "instance_ready_state", "network_bgp_holdtime", "storage_volumes_all_projects", "metrics_memory_oom_total", "storage_buckets", "storage_buckets_create_credentials", "metrics_cpu_effective_total", "projects_networks_restricted_access", "storage_buckets_local", "loki", "acme", "internal_metrics", "cluster_join_token_expiry", "remote_token_expiry", "init_preseed", "storage_volumes_created_at", "cpu_hotplug", "projects_networks_zones", "network_txqueuelen", "cluster_member_state", "instances_placement_scriptlet", "storage_pool_source_wipe", "zfs_block_mode", "instance_generation_id", "disk_io_cache", "amd_sev", "storage_pool_loop_resize", "migration_vm_live", "ovn_nic_nesting", "oidc", "network_ovn_l3only", "ovn_nic_acceleration_vdpa", "cluster_healing", "instances_state_total", "auth_user", "security_csm", "instances_rebuild", "numa_cpu_placement", "custom_volume_iso", "network_allocations", "zfs_delegate", "storage_api_remote_volume_snapshot_copy", "operations_get_query_all_projects", "metadata_configuration", "syslog_socket", "event_lifecycle_name_and_project", "instances_nic_limits_priority", "disk_initial_volume_configuration", "operation_wait", "image_restriction_privileged", "cluster_internal_custom_volume_copy", "disk_io_bus", "storage_cephfs_create_missing", "instance_move_config", "ovn_ssl_config", "certificate_description", "disk_io_bus_virtio_blk", "loki_config_instance", "instance_create_start", "clustering_evacuation_stop_options", "boot_host_shutdown_action", "agent_config_drive", "network_state_ovn_lr", "image_template_permissions", "storage_bucket_backup", "storage_lvm_cluster", "shared_custom_block_volumes", "auth_tls_jwt", "oidc_claim", "device_usb_serial", "numa_cpu_balanced", "image_restriction_nesting", "network_integrations", "instance_memory_swap_bytes", "network_bridge_external_create", "network_zones_all_projects", "storage_zfs_vdev", "container_migration_stateful", "profiles_all_projects", "instances_scriptlet_get_instances", "instances_scriptlet_get_cluster_members", "instances_scriptlet_get_project", "network_acl_stateless", "instance_state_started_at", "networks_all_projects", "network_acls_all_projects", "storage_buckets_all_projects", "resources_load", "instance_access", "project_access", "projects_force_delete", "resources_cpu_flags", "disk_io_bus_cache_filesystem", "instance_oci", "clustering_groups_config", "instances_lxcfs_per_instance", "clustering_groups_vm_cpu_definition", "disk_volume_subpath", "projects_limits_disk_pool", "network_ovn_isolated", "qemu_raw_qmp", "network_load_balancer_health_check", "oidc_scopes", "network_integrations_peer_name", "qemu_scriptlet", "instance_auto_restart", "storage_lvm_metadatasize", "ovn_nic_promiscuous", "ovn_nic_ip_address_none", "instances_state_os_info", "network_load_balancer_state", "instance_nic_macvlan_mode", "storage_lvm_cluster_create", "network_ovn_external_interfaces", "instances_scriptlet_get_instances_count", "cluster_rebalance", "custom_volume_refresh_exclude_older_snapshots", "storage_initial_owner", "storage_live_migration", "instance_console_screenshot", "image_import_alias", "authorization_scriptlet", "console_force", "network_ovn_state_addresses", "network_bridge_acl_devices", "instance_debug_memory", "init_preseed_storage_volumes", "init_preseed_profile_project", `instance_nic_routed_host_address`, "instance_smbios11", "api_filtering_extended", "acme_dns01", "security_iommu", "network_ipv4_dhcp_routes", "network_state_ovn_ls", "network_dns_nameservers", "acme_http01_port", "network_ovn_ipv4_dhcp_expiry", "instance_state_cpu_time", "network_io_bus", "disk_io_bus_usb", "storage_driver_linstor", "instance_oci_entrypoint", "network_address_set", "server_logging", "network_forward_snat", "memory_hotplug", "instance_nic_routed_host_tables", "instance_publish_split", "init_preseed_certificates", "custom_volume_sftp", "network_ovn_external_nic_address", "network_physical_gateway_hwaddr", "backup_s3_upload", "snapshot_manual_expiry", "resources_cpu_address_sizes", "disk_attached", "limits_memory_hotplug", "disk_wwn", "server_logging_webhook", "storage_driver_truenas", "container_disk_tmpfs", "instance_limits_oom", "backup_override_config", "network_ovn_tunnels", "init_preseed_cluster_groups", "usb_attached", "backup_iso", "instance_systemd_credentials", "cluster_group_usedby", "bpf_token_delegation", "file_storage_volume", "network_hwaddr_pattern", "storage_volume_full", "storage_bucket_full", "device_pci_firmware", "resources_serial", "ovn_nic_limits", "storage_lvmcluster_qcow2", "oidc_allowed_subnets", "file_delete_force", "nic_sriov_select_ext", "network_zones_dns_contact", "nic_attached_connected", "nic_sriov_security_trusted", "direct_backup", "instance_snapshot_disk_only_restore", "unix_hotplug_pci", "cluster_evacuating_restoring", "projects_restricted_image_servers", "storage_lvmcluster_size", "authorization_scriptlet_cert", "lvmcluster_remove_snapshots", "daemon_storage_logs", "instances_debug_repair", "network_io_bus_ovn", "dependent", "metrics_project_resources", "storage_volume_nbd", "projects_restricted_storage_pool_access", "server_shutdown_action", "instances_placement_scriptlet_rebalance", } // APIExtensionsCount returns the number of available API extensions. func APIExtensionsCount() int { count := len(APIExtensions) // This environment variable is an internal one to force the code // to believe that we have an API extensions count greater than we // actually have. It's used by integration tests to exercise the // cluster upgrade process. artificialBump := os.Getenv("INCUS_ARTIFICIALLY_BUMP_API_EXTENSIONS") if artificialBump != "" { n, err := strconv.Atoi(artificialBump) if err == nil { count += n } } return count } incus-7.0.0/internal/version/flex.go000066400000000000000000000001171517523235500174340ustar00rootroot00000000000000package version // Version contains the version number. var Version = "7.0.0" incus-7.0.0/internal/version/platform_linux.go000066400000000000000000000012151517523235500215410ustar00rootroot00000000000000//go:build linux package version import ( "strings" "github.com/lxc/incus/v7/internal/linux" "github.com/lxc/incus/v7/shared/osarch" ) func getPlatformVersionStrings() []string { versions := []string{} // Add kernel version. uname, err := linux.Uname() if err != nil { return versions } versions = append(versions, strings.Split(uname.Release, "-")[0]) // Add distribution info. osRelease, err := osarch.GetOSRelease() if err == nil && osRelease["NAME"] != "" { versions = append(versions, osRelease["NAME"]) if osRelease["VERSION_ID"] != "" { versions = append(versions, osRelease["VERSION_ID"]) } } return versions } incus-7.0.0/internal/version/platform_others.go000066400000000000000000000001451517523235500217070ustar00rootroot00000000000000//go:build !linux package version func getPlatformVersionStrings() []string { return []string{} } incus-7.0.0/internal/version/useragent.go000066400000000000000000000027671517523235500205100ustar00rootroot00000000000000package version import ( "fmt" "runtime" "strings" "golang.org/x/text/cases" "golang.org/x/text/language" "github.com/lxc/incus/v7/shared/osarch" ) // UserAgent contains a string suitable as a user-agent. var ( UserAgent = getUserAgent() userAgentStorageBackends []string userAgentFeatures []string ) func getUserAgent() string { archID, err := osarch.ArchitectureID(runtime.GOARCH) if err != nil { panic(err) } arch, err := osarch.ArchitectureName(archID) if err != nil { panic(err) } osTokens := []string{cases.Title(language.English).String(runtime.GOOS), arch} osTokens = append(osTokens, getPlatformVersionStrings()...) // Initial version string agent := fmt.Sprintf("Incus %s", Version) // OS information agent = fmt.Sprintf("%s (%s)", agent, strings.Join(osTokens, "; ")) // Storage information if len(userAgentStorageBackends) > 0 { agent = fmt.Sprintf("%s (%s)", agent, strings.Join(userAgentStorageBackends, "; ")) } // Feature information if len(userAgentFeatures) > 0 { agent = fmt.Sprintf("%s (%s)", agent, strings.Join(userAgentFeatures, "; ")) } return agent } // UserAgentStorageBackends updates the list of storage backends to include in the user-agent. func UserAgentStorageBackends(backends []string) { userAgentStorageBackends = backends UserAgent = getUserAgent() } // UserAgentFeatures updates the list of advertised features. func UserAgentFeatures(features []string) { userAgentFeatures = features UserAgent = getUserAgent() } incus-7.0.0/internal/version/version.go000066400000000000000000000035771517523235500202000ustar00rootroot00000000000000package version import ( "fmt" "regexp" "strconv" "strings" ) // DottedVersion holds element of a version in the maj.min[.patch] format. type DottedVersion struct { Major int Minor int Patch int } // NewDottedVersion returns a new Version. func NewDottedVersion(versionString string) (*DottedVersion, error) { formatError := fmt.Errorf("Invalid version format: %q", versionString) split := strings.Split(versionString, ".") if len(split) < 2 { return nil, formatError } major, err := strconv.Atoi(split[0]) if err != nil { return nil, formatError } minor, err := strconv.Atoi(split[1]) if err != nil { return nil, formatError } patch := -1 if len(split) == 3 { patch, err = strconv.Atoi(split[2]) if err != nil { return nil, formatError } } return &DottedVersion{ Major: major, Minor: minor, Patch: patch, }, nil } // Parse parses a string starting with a dotted version and returns it. func Parse(s string) (*DottedVersion, error) { r, err := regexp.Compile(`^([0-9]+.[0-9]+(?:.[0-9]+)?)`) if err != nil { return nil, err } matches := r.FindStringSubmatch(s) if len(matches) == 0 { return nil, fmt.Errorf("Can't parse a version: %s", s) } return NewDottedVersion(matches[1]) } // String returns version as a string. func (v *DottedVersion) String() string { version := fmt.Sprintf("%d.%d", v.Major, v.Minor) if v.Patch != -1 { version += fmt.Sprintf(".%d", v.Patch) } return version } // Compare returns result of comparison between two versions. func (v *DottedVersion) Compare(other *DottedVersion) int { result := compareInts(v.Major, other.Major) if result != 0 { return result } result = compareInts(v.Minor, other.Minor) if result != 0 { return result } return compareInts(v.Patch, other.Patch) } func compareInts(i1 int, i2 int) int { switch { case i1 < i2: return -1 case i1 > i2: return 1 default: return 0 } } incus-7.0.0/internal/version/version_test.go000066400000000000000000000037161517523235500212320ustar00rootroot00000000000000package version import ( "testing" "github.com/stretchr/testify/suite" ) type versionTestSuite struct { suite.Suite } func TestVersionTestSuite(t *testing.T) { suite.Run(t, &versionTestSuite{}) } func (s *versionTestSuite) TestNewVersion() { v, err := NewDottedVersion("1.2.3") s.Nil(err) s.Equal(1, v.Major) s.Equal(2, v.Minor) s.Equal(3, v.Patch) } func (s *versionTestSuite) TestNewVersionNoPatch() { v, err := NewDottedVersion("1.2") s.Nil(err) s.Equal(-1, v.Patch) } func (s *versionTestSuite) TestNewVersionInvalid() { v, err := NewDottedVersion("1.nope") s.Nil(v) s.NotNil(err) } func (s *versionTestSuite) TestParseDashes() { v, err := Parse("1.2.3-asdf") s.Nil(err) s.Equal(1, v.Major) s.Equal(2, v.Minor) s.Equal(3, v.Patch) } func (s *versionTestSuite) TestParseParentheses() { v, err := Parse("1.2.3(beta1)") s.Nil(err) s.Equal(1, v.Major) s.Equal(2, v.Minor) s.Equal(3, v.Patch) } func (s *versionTestSuite) TestParseFail() { v, err := Parse("asdfaf") s.Nil(v) s.NotNil(err) } func (s *versionTestSuite) TestString() { v, _ := NewDottedVersion("1.2.3") s.Equal("1.2.3", v.String()) } func (s *versionTestSuite) TestCompareEqual() { v1, _ := NewDottedVersion("1.2.3") v2, _ := NewDottedVersion("1.2.3") s.Equal(0, v1.Compare(v2)) s.Equal(0, v2.Compare(v1)) v3, _ := NewDottedVersion("1.2") v4, _ := NewDottedVersion("1.2") s.Equal(0, v3.Compare(v4)) s.Equal(0, v4.Compare(v3)) } func (s *versionTestSuite) TestCompareOlder() { v1, _ := NewDottedVersion("1.2.3") v2, _ := NewDottedVersion("1.2.4") v3, _ := NewDottedVersion("1.3") v4, _ := NewDottedVersion("2.2.3") s.Equal(-1, v1.Compare(v2)) s.Equal(-1, v1.Compare(v3)) s.Equal(-1, v1.Compare(v4)) } func (s *versionTestSuite) TestCompareNewer() { v1, _ := NewDottedVersion("1.2.3") v2, _ := NewDottedVersion("1.2.2") v3, _ := NewDottedVersion("1.1") v4, _ := NewDottedVersion("0.3.3") s.Equal(1, v1.Compare(v2)) s.Equal(1, v1.Compare(v3)) s.Equal(1, v1.Compare(v4)) } incus-7.0.0/po/000077500000000000000000000000001517523235500132655ustar00rootroot00000000000000incus-7.0.0/po/de.po000066400000000000000000013171771517523235500142360ustar00rootroot00000000000000# German translation for LXD # Copyright (C) 2015 - LXD contributors # This file is distributed under the same license as LXD. # Felix Engelmann , 2015. # msgid "" msgstr "" "Project-Id-Version: LXD\n" "Report-Msgid-Bugs-To: lxc-devel@lists.linuxcontainers.org\n" "POT-Creation-Date: 2026-05-01 14:24+0200\n" "PO-Revision-Date: 2026-04-21 03:04+0000\n" "Last-Translator: Dklfajsjfi49wefklsf32 " "\n" "Language-Team: German \n" "Language: de\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=n != 1;\n" "X-Generator: Weblate 5.17.1-dev\n" #: cmd/incus/info.go:437 msgid " Chassis:" msgstr " Gehäuse:" #: cmd/incus/info.go:477 msgid " Firmware:" msgstr " Firmware:" #: cmd/incus/info.go:457 msgid " Motherboard:" msgstr " Mainboard:" #: cmd/incus/storage_bucket.go:260 cmd/incus/storage_bucket.go:1128 #, fuzzy msgid "" "### This is a YAML representation of a storage bucket.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### A storage bucket consists of a set of configuration items.\n" "###\n" "### name: bucket1\n" "### used_by: []\n" "### config:\n" "### size: \"61203283968\"" msgstr "" "### Dies ist eine Darstellung der Eigenschaften eines Abbildes in yaml.\n" "### Jede Zeile die mit '# beginnt wird ignoriert.\n" "###\n" "### Ein Speicher-Volumen besteht aus einer Reihe von " "Konfigurationselementen.\n" "###\n" "### name: vol1\n" "### type: custom\n" "### used_by: []\n" "### config:\n" "### size: \"61203283968\"" #: cmd/incus/storage.go:286 #, fuzzy msgid "" "### This is a YAML representation of a storage pool.\n" "### Any line starting with a '#' will be ignored.\n" "###\n" "### A storage pool consists of a set of configuration items.\n" "###\n" "### An example would look like:\n" "### name: default\n" "### driver: zfs\n" "### used_by: []\n" "### config:\n" "### size: \"61203283968\"\n" "### source: default\n" "### zfs.pool_name: default" msgstr "" "### Dies ist eine YAML-Repräsentation eines Storage-Pools.\n" "### Jede Zeile, die mit einem '#' beginnt wird ignoriert.\n" "###\n" "### Ein Storage-Pool besteht aus einem Satz an Konfigurations-Items.\n" "###\n" "### Ein Beispiel könnte so aussehen:\n" "### name: default\n" "### driver: zfs\n" "### used_by: []\n" "### config:\n" "### size: \"61203283968\"\n" "### source: /home/chb/mnt/lxd_test/default.img\n" "### zfs.pool_name: default" #: cmd/incus/storage_volume.go:924 #, fuzzy msgid "" "### This is a YAML representation of a storage volume.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### A storage volume consists of a set of configuration items.\n" "###\n" "### name: foo\n" "### type: custom\n" "### used_by: []\n" "### config:\n" "### size: \"61203283968\"" msgstr "" "### Dies ist eine Darstellung der Eigenschaften eines Abbildes in yaml.\n" "### Jede Zeile die mit '# beginnt wird ignoriert.\n" "###\n" "### Ein Speicher-Volumen besteht aus einer Reihe von " "Konfigurationselementen.\n" "###\n" "### name: vol1\n" "### type: custom\n" "### used_by: []\n" "### config:\n" "### size: \"61203283968\"" #: cmd/incus/config_trust.go:270 msgid "" "### This is a YAML representation of the certificate.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### Note that the fingerprint is shown but cannot be changed" msgstr "" "### Dies ist eine YAML-Darstellung des Zertifikats.\n" "### Jede Zeile, die mit '#' beginnt, wird ignoriert.\n" "###\n" "### Beachten Sie, dass der Fingerabdruck angezeigt wird, aber nicht geändert " "werden kann." #: cmd/incus/cluster_group.go:398 #, fuzzy msgid "" "### This is a YAML representation of the cluster group.\n" "### Any line starting with a '# will be ignored." msgstr "" "### Dies ist eine yaml-Repräsentation des Cluster-Mitglieds.\n" "### Jede mit '# beginnende Zeile wird ignoriert." #: cmd/incus/cluster.go:875 #, fuzzy msgid "" "### This is a YAML representation of the cluster member.\n" "### Any line starting with a '# will be ignored." msgstr "" "### Dies ist eine yaml-Repräsentation des Cluster-Mitglieds.\n" "### Jede mit '# beginnende Zeile wird ignoriert." #: cmd/incus/config.go:118 #, fuzzy msgid "" "### This is a YAML representation of the configuration.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### A sample configuration looks like:\n" "### name: instance1\n" "### profiles:\n" "### - default\n" "### config:\n" "### volatile.eth0.hwaddr: 10:66:6a:e9:f8:7f\n" "### devices:\n" "### homedir:\n" "### path: /extra\n" "### source: /home/user\n" "### type: disk\n" "### ephemeral: false\n" "###\n" "### Note that the name is shown but cannot be changed" msgstr "" "### Dies ist eine Darstellung der Konfiguration in yaml.\n" "### Jede Zeile die mit '# beginnt wird ignoriert.\n" "###\n" "### Beispiel einer Konfiguration:\n" "### name: container1\n" "### profiles:\n" "### - default\n" "### config:\n" "### volatile.eth0.hwaddr: 00:16:3e:e9:f8:7f\n" "### devices:\n" "### homedir:\n" "### path: /extra\n" "### source: /home/user\n" "### type: disk\n" "### ephemeral: false\n" "###\n" "### Der Name wird zwar angezeigt, lässt sich jedoch nicht ändern." #: cmd/incus/image.go:414 msgid "" "### This is a YAML representation of the image properties.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### Each property is represented by a single line:\n" "### An example would be:\n" "### description: My custom image" msgstr "" "### Dies ist eine Darstellung der Eigenschaften eines Images in yaml.\n" "### Jede Zeile die mit '# beginnt wird ignoriert.\n" "###\n" "### Pro Eigenschaft wird eine Zeile verwendet:\n" "### Zum Beispiel:\n" "### description: Mein eigenes Abbild" #: cmd/incus/config_metadata.go:74 msgid "" "### This is a YAML representation of the instance metadata.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### A sample configuration looks like:\n" "###\n" "### architecture: x86_64\n" "### creation_date: 1477146654\n" "### expiry_date: 0\n" "### properties:\n" "### architecture: x86_64\n" "### description: BusyBox x86_64\n" "### name: busybox-x86_64\n" "### os: BusyBox\n" "### templates:\n" "### /template:\n" "### when:\n" "### - \"\"\n" "### create_only: false\n" "### template: template.tpl\n" "### properties: {}" msgstr "" "### Dies ist eine YAML-Darstellung der Instanz-Metadaten.\n" "###Jede Zeile, die mit '#' beginnt, wird ignoriert.\n" "###\n" "### Eine Beispielkonfiguration sieht folgendermaßen aus:\n" "###\n" "### architecture: x86_64\n" "### creation_date: 1477146654\n" "### expiry_date: 0\n" "### properties:\n" "### architecture: x86_64\n" "### description: BusyBox x86_64\n" "### name: busybox-x86_64\n" "### os: BusyBox\n" "### templates:\n" "### /template:\n" "### when:\n" "### - \"\"\n" "### create_only: false\n" "### template: template.tpl\n" "### properties: {}" #: cmd/incus/network_acl.go:589 #, fuzzy msgid "" "### This is a YAML representation of the network ACL.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### A network ACL consists of a set of rules and configuration items.\n" "###\n" "### An example would look like:\n" "### name: allow-all-inbound\n" "### description: test desc\n" "### egress: []\n" "### ingress:\n" "### - action: allow\n" "### state: enabled\n" "### protocol: \"\"\n" "### source: \"\"\n" "### source_port: \"\"\n" "### destination: \"\"\n" "### destination_port: \"\"\n" "### icmp_type: \"\"\n" "### icmp_code: \"\"\n" "### config:\n" "### user.foo: bah\n" "###\n" "### Note that only the ingress and egress rules, description and " "configuration keys can be changed." msgstr "" "### Dies ist eine Darstellung eines Profils in yaml.\n" "### Jede Zeile die mit '# beginnt wird ignoriert.\n" "###\n" "### Ein Profil besteht aus mehreren Konfigurationselementen gefolgt von\n" "### mehrere Geräten.\n" "###\n" "### Zum Beispiel:\n" "### name: onenic\n" "### config:\n" "### raw.lxc: lxc.aa_profile=unconfined\n" "### devices:\n" "### eth0:\n" "### nictype: bridged\n" "### parent: lxdbr0\n" "### type: nic\n" "###\n" "### Der Name wird zwar angezeigt, lässt sich jedoch nicht ändern.\n" #: cmd/incus/network_address_set.go:460 #, fuzzy msgid "" "### This is a YAML representation of the network address set.\n" "### Any line starting with '#' will be ignored.\n" "###\n" "### For example:\n" "### name: as1\n" "### description: \"Test address set\"\n" "### addresses:\n" "### - 10.0.0.1\n" "### - 2001:db8::1\n" "### external_ids:\n" "### user.foo: bar\n" msgstr "" "### Dies ist eine Darstellung der Eigenschaften eines Abbildes in yaml.\n" "### Jede Zeile die mit '# beginnt wird ignoriert.\n" "###\n" "### Ein Speicher-Volumen besteht aus einer Reihe von " "Konfigurationselementen.\n" "###\n" "### name: vol1\n" "### type: custom\n" "### used_by: []\n" "### config:\n" "### size: \"61203283968\"" #: cmd/incus/network_forward.go:641 #, fuzzy msgid "" "### This is a YAML representation of the network forward.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### A network forward consists of a default target address and optional set " "of port forwards for a listen address.\n" "###\n" "### An example would look like:\n" "### listen_address: 192.0.2.1\n" "### config:\n" "### target_address: 198.51.100.2\n" "### description: test desc\n" "### ports:\n" "### - description: port forward\n" "### protocol: tcp\n" "### listen_port: 80,81,8080-8090\n" "### target_address: 198.51.100.3\n" "### target_port: 80,81,8080-8090\n" "### location: server01\n" "###\n" "### Note that the listen_address and location cannot be changed." msgstr "" "### Dies ist eine Darstellung eines Profils in yaml.\n" "### Jede Zeile die mit '# beginnt wird ignoriert.\n" "###\n" "### Ein Profil besteht aus mehreren Konfigurationselementen gefolgt von\n" "### mehrere Geräten.\n" "###\n" "### Zum Beispiel:\n" "### name: onenic\n" "### config:\n" "### raw.lxc: lxc.aa_profile=unconfined\n" "### devices:\n" "### eth0:\n" "### nictype: bridged\n" "### parent: lxdbr0\n" "### type: nic\n" "###\n" "### Der Name wird zwar angezeigt, lässt sich jedoch nicht ändern.\n" #: cmd/incus/network_integration.go:235 msgid "" "### This is a YAML representation of the network integration.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### Note that the name is shown but cannot be changed" msgstr "" "### Dies ist eine YAML-Darstellung der Netzwerkintegration.\n" "### Jede Zeile, die mit '#' beginnt, wird ignoriert.\n" "###\n" "### Beachten Sie, dass der Name angezeigt wird, aber nicht geändert werden " "kann." #: cmd/incus/network_load_balancer.go:610 #, fuzzy msgid "" "### This is a YAML representation of the network load balancer.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### A network load balancer consists of a set of target backends and port " "forwards for a listen address.\n" "###\n" "### An example would look like:\n" "### listen_address: 192.0.2.1\n" "### config:\n" "### user.foo: bar\n" "### description: test desc\n" "### backends:\n" "### - name: backend1\n" "### description: First backend server\n" "### target_address: 192.0.3.1\n" "### target_port: 80\n" "### - name: backend2\n" "### description: Second backend server\n" "### target_address: 192.0.3.2\n" "### target_port: 80\n" "### ports:\n" "### - description: port forward\n" "### protocol: tcp\n" "### listen_port: 80,81,8080-8090\n" "### target_backend:\n" "### - backend1\n" "### - backend2\n" "### location: server01\n" "###\n" "### Note that the listen_address and location cannot be changed." msgstr "" "### Dies ist eine Darstellung eines Profils in yaml.\n" "### Jede Zeile die mit '# beginnt wird ignoriert.\n" "###\n" "### Ein Profil besteht aus mehreren Konfigurationselementen gefolgt von\n" "### mehrere Geräten.\n" "###\n" "### Zum Beispiel:\n" "### name: onenic\n" "### config:\n" "### raw.lxc: lxc.aa_profile=unconfined\n" "### devices:\n" "### eth0:\n" "### nictype: bridged\n" "### parent: lxdbr0\n" "### type: nic\n" "###\n" "### Der Name wird zwar angezeigt, lässt sich jedoch nicht ändern.\n" #: cmd/incus/network_peer.go:659 msgid "" "### This is a YAML representation of the network peer.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### An example would look like:\n" "### description: A peering to mynet\n" "### config: {}\n" "### name: mypeer\n" "### target_project: default\n" "### target_network: mynet\n" "### status: Pending\n" "###\n" "### Note that the name, target_project, target_network and status fields " "cannot be changed." msgstr "" "### Dies ist eine YAML-Darstellung des Netzwerk-Peers.\n" "### Jede Zeile, die mit '#' beginnt, wird ignoriert.\n" "###\n" "### Ein Beispiel könnte so aussehen:\n" "### description: Eine Peering-Verbindung zu mynet\n" "### config: {}\n" "### name: mypeer\n" "### target_project: default\n" "### target_network: mynet\n" "### status: Pending\n" "###\n" "### Beachten Sie, dass die Felder name, target_project, target_network und " "status nicht geändert werden können." #: cmd/incus/network_zone.go:1269 #, fuzzy msgid "" "### This is a YAML representation of the network zone record.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### A network zone consists of a set of rules and configuration items.\n" "###\n" "### An example would look like:\n" "### name: foo\n" "### description: SPF record\n" "### config:\n" "### user.foo: bah\n" msgstr "" "### Dies ist eine Darstellung der Eigenschaften eines Abbildes in yaml.\n" "### Jede Zeile die mit '# beginnt wird ignoriert.\n" "###\n" "### Ein Speicher-Volumen besteht aus einer Reihe von " "Konfigurationselementen.\n" "###\n" "### name: vol1\n" "### type: custom\n" "### used_by: []\n" "### config:\n" "### size: \"61203283968\"" #: cmd/incus/network_zone.go:603 #, fuzzy msgid "" "### This is a YAML representation of the network zone.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### A network zone consists of a set of rules and configuration items.\n" "###\n" "### An example would look like:\n" "### name: example.net\n" "### description: Internal domain\n" "### config:\n" "### user.foo: bah\n" msgstr "" "### Dies ist eine Darstellung der Eigenschaften eines Abbildes in yaml.\n" "### Jede Zeile die mit '# beginnt wird ignoriert.\n" "###\n" "### Ein Speicher-Volumen besteht aus einer Reihe von " "Konfigurationselementen.\n" "###\n" "### name: vol1\n" "### type: custom\n" "### used_by: []\n" "### config:\n" "### size: \"61203283968\"" #: cmd/incus/network.go:684 #, fuzzy msgid "" "### This is a YAML representation of the network.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### A network consists of a set of configuration items.\n" "###\n" "### An example would look like:\n" "### name: mybr0\n" "### config:\n" "### ipv4.address: 10.62.42.1/24\n" "### ipv4.nat: true\n" "### ipv6.address: fd00:56ad:9f7a:9800::1/64\n" "### ipv6.nat: true\n" "### managed: true\n" "### type: bridge\n" "###\n" "### Note that only the configuration can be changed." msgstr "" "### Dies ist eine Darstellung eines Profils in yaml.\n" "### Jede Zeile die mit '# beginnt wird ignoriert.\n" "###\n" "### Ein Profil besteht aus mehreren Konfigurationselementen gefolgt von\n" "### mehrere Geräten.\n" "###\n" "### Zum Beispiel:\n" "### name: onenic\n" "### config:\n" "### raw.lxc: lxc.aa_profile=unconfined\n" "### devices:\n" "### eth0:\n" "### nictype: bridged\n" "### parent: lxdbr0\n" "### type: nic\n" "###\n" "### Der Name wird zwar angezeigt, lässt sich jedoch nicht ändern.\n" #: cmd/incus/profile.go:497 #, fuzzy msgid "" "### This is a YAML representation of the profile.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### A profile consists of a set of configuration items followed by a set of\n" "### devices.\n" "###\n" "### An example would look like:\n" "### name: onenic\n" "### config:\n" "### raw.lxc: lxc.aa_profile=unconfined\n" "### devices:\n" "### eth0:\n" "### nictype: bridged\n" "### parent: mybr0\n" "### type: nic\n" "###\n" "### Note that the name is shown but cannot be changed" msgstr "" "### Dies ist eine Darstellung eines Profils in yaml.\n" "### Jede Zeile die mit '# beginnt wird ignoriert.\n" "###\n" "### Ein Profil besteht aus mehreren Konfigurationselementen gefolgt von\n" "### mehrere Geräten.\n" "###\n" "### Zum Beispiel:\n" "### name: onenic\n" "### config:\n" "### raw.lxc: lxc.aa_profile=unconfined\n" "### devices:\n" "### eth0:\n" "### nictype: bridged\n" "### parent: lxdbr0\n" "### type: nic\n" "###\n" "### Der Name wird zwar angezeigt, lässt sich jedoch nicht ändern." #: cmd/incus/project.go:322 #, fuzzy msgid "" "### This is a YAML representation of the project.\n" "### Any line starting with a '# will be ignored.\n" "###\n" "### A project consists of a set of features and a description.\n" "###\n" "### An example would look like:\n" "### config:\n" "### features.images: \"true\"\n" "### features.networks: \"true\"\n" "### features.networks.zones: \"true\"\n" "### features.profiles: \"true\"\n" "### features.storage.buckets: \"true\"\n" "### features.storage.volumes: \"true\"\n" "### description: My own project\n" "### name: my-project\n" "###\n" "### Note that the name is shown but cannot be changed" msgstr "" "### Dies ist eine Darstellung eines Profils in yaml.\n" "### Jede Zeile die mit '# beginnt wird ignoriert.\n" "###\n" "### Ein Profil besteht aus mehreren Konfigurationselementen gefolgt von\n" "### mehrere Geräten.\n" "###\n" "### Zum Beispiel:\n" "### name: onenic\n" "### config:\n" "### raw.lxc: lxc.aa_profile=unconfined\n" "### devices:\n" "### eth0:\n" "### nictype: bridged\n" "### parent: lxdbr0\n" "### type: nic\n" "###\n" "### Der Name wird zwar angezeigt, lässt sich jedoch nicht ändern.\n" #: cmd/incus/info.go:333 #, c-format msgid "%d (id: %d, online: %v, NUMA node: %v)" msgstr "%d (ID: %d, Online: %v, NUMA-Knoten: %v)" #: cmd/incus/admin_init_interactive.go:440 #, fuzzy, c-format msgid "%q is not a block device" msgstr "%s ist kein Verzeichnis" #: cmd/incus/admin_init_interactive.go:570 #, fuzzy, c-format msgid "%q is not an IP address" msgstr "%s ist kein Verzeichnis" #: cmd/incus/admin_recover.go:207 #, c-format msgid "%s %q on pool %q in project %q (includes %d snapshots)" msgstr "%s %q im Pool %q im Projekt %q (beinhaltet %d Snapshots)" #: cmd/incus/image.go:1135 #, c-format msgid "%s (%d more)" msgstr "%s (%d mehr)" #: cmd/incus/info.go:175 #, fuzzy, c-format msgid "%s (%s) (%d available)" msgstr "%s (%d mehr)" #: cmd/incus/admin_recover.go:63 #, c-format msgid "%s (backend=%q, source=%q)" msgstr "%s (backend=%q, quelle=%q)" #: cmd/incus/file.go:724 cmd/incus/storage_volume.go:2828 #: cmd/incus/utils_copy.go:96 cmd/incus/utils_sftp.go:391 #, c-format msgid "%s is not a directory" msgstr "%s ist kein Verzeichnis" #: cmd/incus/usage/usage.go:343 #, c-format msgid "%s the %s syntax is deprecated; %s\n" msgstr "" #: cmd/incus/utils_sftp.go:280 #, c-format msgid "'%s' isn't a supported file type" msgstr "'%s' ist kein unterstützter Dateityp" #: cmd/incus/usage/parse.go:238 #, c-format msgid "(skipped: %s)\n" msgstr "" #: cmd/incus/usage/parse.go:240 msgid "(skipped: no value given)" msgstr "" #: cmd/incus/usage/parse.go:35 msgid ", " msgstr "" #: cmd/incus/info.go:323 #, c-format msgid "- Level %d (type: %s): %s" msgstr "- Level %d (Typ: %s): %s" #: cmd/incus/info.go:302 #, c-format msgid "- Partition %d" msgstr "- Partition %d" #: cmd/incus/info.go:211 #, c-format msgid "- Port %d (%s)" msgstr "- Port %d (%s)" #: cmd/incus/warning.go:362 #, fuzzy msgid "--all cannot be used together with other arguments" msgstr "--target kann nicht mit Instanzen verwendet werden" #: cmd/incus/action.go:246 #, fuzzy msgid "--console can't be used while forcing instance shutdown" msgstr "--refresh kann nur mit Containern verwendet werden" #: cmd/incus/action.go:412 #, fuzzy msgid "--console can't be used with --all" msgstr "--refresh kann nur mit Containern verwendet werden" #: cmd/incus/action.go:416 #, fuzzy msgid "--console only works with a single instance" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/config.go:469 cmd/incus/config.go:718 #, fuzzy msgid "--expanded cannot be used with a server" msgstr "--refresh kann nur mit Containern verwendet werden" #: cmd/incus/copy.go:154 msgid "--instance-only can't be passed when the source is a snapshot" msgstr "" "--instance-only kann nicht verwendet werden, wenn die Quelle ein Snapshot ist" #: cmd/incus/admin_init_auto.go:41 #, fuzzy msgid "--network-port can't be used without --network-address" msgstr "--refresh kann nur mit Containern verwendet werden" #: cmd/incus/utils_copy.go:65 #, fuzzy msgid "--no-dereference/-P cannot be used together with stdout as a target" msgstr "--refresh kann nur mit Containern verwendet werden" #: cmd/incus/utils_copy.go:40 msgid "" "--no-dereference/-P, --follow/-H, and --dereference/-L are mutually exclusive" msgstr "" #: cmd/incus/profile.go:227 #, fuzzy msgid "--no-profiles cannot be used together with other arguments" msgstr "--refresh kann nur mit Containern verwendet werden" #: cmd/incus/copy.go:109 #, fuzzy msgid "--no-profiles cannot be used with --refresh" msgstr "--refresh kann nur mit Containern verwendet werden" #: cmd/incus/query.go:77 #, fuzzy msgid "--project cannot be used with the query command" msgstr "--refresh kann nur mit Containern verwendet werden" #: cmd/incus/utils_copy.go:101 cmd/incus/utils_copy.go:146 msgid "--recursive/-r is required when pulling directories" msgstr "" #: cmd/incus/copy.go:165 msgid "--refresh can only be used with instances" msgstr "--refresh kann nur mit Instanzen verwendet werden" #: cmd/incus/image.go:217 msgid "--reuse requires --copy-aliases" msgstr "" #: cmd/incus/move.go:213 msgid "--target can only be used with clusters" msgstr "--target kann nur mit Clustern verwendet werden" #: cmd/incus/config.go:154 cmd/incus/config.go:419 cmd/incus/config.go:557 #: cmd/incus/config.go:744 cmd/incus/info.go:635 msgid "--target cannot be used with instances" msgstr "--target kann nicht mit Instanzen verwendet werden" #: cmd/incus/admin_sql.go:128 #, c-format msgid "=> Query %d:" msgstr "" #: cmd/incus/remote.go:689 msgid "A client certificate is already present" msgstr "Ein Client Zertifikat ist bereits erstellt worden" #: cmd/incus/file.go:762 cmd/incus/storage_volume.go:2855 msgid "" "A target file name must be specified when pushing from stdin; the target is " "a directory" msgstr "" #: cmd/incus/usage/usage.go:808 msgid "ACL" msgstr "" #: cmd/incus/network_allocations.go:81 msgid "ADDRESS" msgstr "ADRESSE" #: cmd/incus/network_address_set.go:156 #, fuzzy msgid "ADDRESSES" msgstr "ADRESSE" #: cmd/incus/alias.go:148 cmd/incus/image.go:1102 cmd/incus/image_alias.go:246 msgid "ALIAS" msgstr "ALIAS" #: cmd/incus/image.go:1103 msgid "ALIASES" msgstr "ALIASES" #: cmd/incus/query.go:32 msgid "API path" msgstr "" #: cmd/incus/list.go:799 msgid "APP" msgstr "APP" #: cmd/incus/cluster.go:198 cmd/incus/image.go:1097 cmd/incus/list.go:473 msgid "ARCHITECTURE" msgstr "ARCHITEKTUR" #: cmd/incus/remote.go:987 #, fuzzy msgid "AUTH TYPE" msgstr "Authentifizierungstyp" #: cmd/incus/remote.go:136 msgid "Accept certificate" msgstr "Akzeptiere das Zertifikat" #: cmd/incus/storage_bucket.go:984 msgid "Access key (auto-generated if empty)" msgstr "" "Zugangsschlüssel (wird automatisch generiert, wenn nichts angegeben wird)" #: cmd/incus/storage_bucket.go:1049 #, c-format msgid "Access key: %s" msgstr "Zugangsschlüssel: %s" #: cmd/incus/config.go:380 msgid "Access the expanded configuration" msgstr "Zugriff auf erweiterte Konfiguration" #: cmd/incus/warning.go:264 cmd/incus/warning.go:265 #, fuzzy msgid "Acknowledge warning" msgstr "Warnung gesehen" #: cmd/incus/query.go:48 #, fuzzy msgid "Action" msgstr "Fingerabdruck: %s\n" #: cmd/incus/query.go:81 #, c-format msgid "Action %q isn't supported by this tool" msgstr "Aktion %q wird von diesem Programm nicht unterstützt" #: cmd/incus/cluster_group.go:740 #, fuzzy msgid "Add a cluster member to a cluster group" msgstr "Cluster Mitglied zu einer Cluster Gruppe hinzufügen:" #: cmd/incus/network_zone.go:1443 #, fuzzy msgid "Add a network zone record entry" msgstr "Füge einen Netzwerk Zonen Eintrag hinzu" #: cmd/incus/network_address_set.go:653 cmd/incus/network_address_set.go:654 #, fuzzy msgid "Add addresses to a network address set" msgstr "Einträge zu einem Netzwerkzonen-Datensatz hinzufügen" #: cmd/incus/network_load_balancer.go:818 msgid "Add backend to a load balancer" msgstr "Backend zu einem Load Balancer hinzufügen" #: cmd/incus/network_load_balancer.go:817 msgid "Add backends to a load balancer" msgstr "Backends zu einem load balancer hinzufügen" #: cmd/incus/network_zone.go:1444 msgid "Add entries to a network zone record" msgstr "Einträge zu einem Netzwerkzonen-Datensatz hinzufügen" #: cmd/incus/config_device.go:92 cmd/incus/config_device.go:93 msgid "Add instance devices" msgstr "Füge Devices zu einer Instanz hinzu" #: cmd/incus/cluster_group.go:739 msgid "Add member to group" msgstr "Mitglied zu einer Gruppe hinzufügen" #: cmd/incus/alias.go:61 cmd/incus/alias.go:62 msgid "Add new aliases" msgstr "Erstelle neue Aliasse" #: cmd/incus/remote.go:125 msgid "Add new remote servers" msgstr "Neue entfernte Server hinzufügen" #: cmd/incus/remote.go:126 msgid "" "Add new remote servers\n" "\n" "URL for remote resources must be HTTPS (https://).\n" "\n" "Basic authentication can be used when combined with the \"simplestreams\" " "protocol:\n" " incus remote add some-name https://LOGIN:PASSWORD@example.com/some/path --" "protocol=simplestreams\n" msgstr "" "Neuen entfernten Server hinzufügen\n" "\n" "Webadressen (URLs) für entfernte Server müssen mit HTTPS (https://) " "beginnen.\n" "\n" "Die Basic Authentifizierungsmethode kann mit dem \"simplestreams\" Protokoll " "genutzt werden:\n" " incus remote add name-ihrer-wahl https://LOGIN:PASSWORT@example.com/" "angegebener/pfad --protocol=simplestreams\n" #: cmd/incus/config_trust.go:93 msgid "Add new trusted client" msgstr "Vertrauenswürdigen Client hinzufügen" #: cmd/incus/config_trust.go:167 #, fuzzy msgid "Add new trusted client certificate" msgstr "Akzeptiere Zertifikat" #: cmd/incus/config_trust.go:168 msgid "" "Add new trusted client certificate\n" "\n" "The following certificate types are supported:\n" "- client (default)\n" "- metrics\n" msgstr "" "Neues vertrauenswürdigen Zertifikat hinzufügen\n" "\n" "Folgende Zertifikat typen aind möglich:\n" "- client (stndard)\n" "- metrics\n" #: cmd/incus/config_trust.go:94 msgid "" "Add new trusted client\n" "\n" "This will issue a trust token to be used by the client to add itself to the " "trust store.\n" msgstr "" "Neuen vertrauenswürdigen Client hinzufügen\n" "\n" "Dies stellt einen Trust-Token aus, der vom Client verwendet wird, um sich " "selbst zum Trust Store hinzuzufügen.\"\n" #: cmd/incus/network_forward.go:838 cmd/incus/network_forward.go:839 msgid "Add ports to a forward" msgstr "Ports zu einem Network Forward hinzufügen" #: cmd/incus/network_load_balancer.go:976 #: cmd/incus/network_load_balancer.go:977 msgid "Add ports to a load balancer" msgstr "Ports zu einem Load Balancer hinzufügen" #: cmd/incus/profile.go:114 cmd/incus/profile.go:115 msgid "Add profiles to instances" msgstr "Profile zu Instanzen hinzufügen" #: cmd/incus/cluster_role.go:52 cmd/incus/cluster_role.go:53 msgid "Add roles to a cluster member" msgstr "Rollen einem Cluster Mitglied hinzufügen" #: cmd/incus/network_acl.go:824 cmd/incus/network_acl.go:825 msgid "Add rules to an ACL" msgstr "Regeln zu einer Zugangskontrollliste (ACL) hinzufügen" #: cmd/incus/color/color.go:44 msgid "Additional Help Topics:" msgstr "" #: cmd/incus/admin_recover.go:126 #, fuzzy msgid "" "Additional storage pool configuration property (KEY=VALUE, empty when done):" msgstr "" "Zusätzliche Storage Pool Konfigurationsparameter hinzufügen (SCHLÜSSEL=WERT, " "leerlassen falls fertig):" #: cmd/incus/admin_init.go:56 msgid "Address to bind to (default: none)" msgstr "IP-Adresse unter der der Server erreichbar sein soll (Standard: keine)" #: cmd/incus/admin_init_interactive.go:576 msgid "Address to bind to (not including port)" msgstr "IP-Adresse unter der der Server erreichbar sein soll (ohne Port)" #: cmd/incus/info.go:215 #, fuzzy, c-format msgid "Address: %s" msgstr "Adresse: %s" #: cmd/incus/info.go:359 #, c-format msgid "Address: %v" msgstr "Adresse: %v" #: cmd/incus/storage_bucket.go:180 #, c-format msgid "Admin access key: %s" msgstr "Administrator Zugangsschlüssel: %s" #: cmd/incus/storage_bucket.go:181 #, c-format msgid "Admin secret key: %s" msgstr "Administrator Geheimer Schlüssel: %s" #: cmd/incus/alias.go:85 cmd/incus/alias.go:198 #, c-format msgid "Alias %s already exists" msgstr "Alias %s existiert bereits" #: cmd/incus/alias.go:192 cmd/incus/alias.go:246 #, c-format msgid "Alias %s doesn't exist" msgstr "Alias %s existiert nicht" #: cmd/incus/image.go:1011 #, c-format msgid "Alias: %s" msgstr "Alias: %s" #: cmd/incus/publish.go:211 #, fuzzy, c-format msgid "Aliases already exists: %s" msgstr "entfernte Instanz %s existiert bereits" #: cmd/incus/image.go:995 cmd/incus/color/color.go:39 msgid "Aliases:" msgstr "Aliase:" #: cmd/incus/cluster.go:1781 msgid "All existing data is lost when joining a cluster, continue?" msgstr "" "Alle existierenden Dateien sind verloren, wenn sie dem Cluster beitreten, " "Vorgang fortsetzen?" #: cmd/incus/profile.go:249 #, fuzzy, c-format msgid "All profiles removed from %s" msgstr "Gerät %s wurde von %s entfernt\n" #: cmd/incus/storage_volume.go:1440 cmd/incus/storage_volume.go:3285 msgid "All projects" msgstr "Alle Projekte" #: cmd/incus/remote.go:218 msgid "All server addresses are unavailable" msgstr "Alle Server Adressen sind nicht erreichbar" #: cmd/incus/config_trust.go:178 #, fuzzy msgid "Alternative certificate name" msgstr "Alternativer Zertifikatsbezeichnung" #: cmd/incus/file.go:450 cmd/incus/file.go:681 cmd/incus/storage_volume.go:2591 #: cmd/incus/storage_volume.go:2789 msgid "Always follow symbolic links in source path" msgstr "" #: cmd/incus/file.go:982 #, fuzzy, c-format msgid "An instance path is required for %s" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/image.go:967 cmd/incus/info.go:501 cmd/incus/info.go:505 #: cmd/incus/info.go:663 #, c-format msgid "Architecture: %s" msgstr "Architektur: %s" #: cmd/incus/info.go:141 #, c-format msgid "Architecture: %v" msgstr "Architektur: %v" #: cmd/incus/cluster.go:1679 msgid "Are you joining an existing cluster?" msgstr "Soll der Server einem bestehenden Cluster beitreten?" #: cmd/incus/cluster.go:1519 #, fuzzy, c-format msgid "Are you sure you want to %s cluster member %q? (yes/no) [default=no]: " msgstr "" "Sind Sie sicher, dass sie das Cluster Mitglied %q %s? (ja/nein) " "[default=nein]: " #: cmd/incus/console.go:405 msgid "As neither could be found, the raw SPICE socket can be found at:" msgstr "" "Da keines der genannten Programme gefunden werden konnte, finden Sie hier " "den SPICE Socket zum manuellen verbinden:" #: cmd/incus/create.go:339 cmd/incus/rebuild.go:99 #, fuzzy msgid "Asked for a VM but image is of type container" msgstr "" "Es soll eine Virtuelle Maschine (VM) genutzt werden, aber das angefragte " "Image ist vom Typ: Container" #: cmd/incus/cluster_group.go:103 cmd/incus/cluster_group.go:104 msgid "Assign sets of groups to cluster members" msgstr "Einem Clustermitglied eine oder mehrere Gruppen zuweisen" #: cmd/incus/profile.go:183 cmd/incus/profile.go:184 msgid "Assign sets of profiles to instances" msgstr "Ein oder mehrere Profile einer Instanz zuweisen" #: cmd/incus/network.go:154 #, fuzzy msgid "Attach network interfaces to instances" msgstr "Netzwerkschnittstellen (network interfaces) zu Instanzen hinzufügen" #: cmd/incus/network.go:244 cmd/incus/network.go:245 msgid "Attach network interfaces to profiles" msgstr "Netzwerkschnittstellen (network interfaces) zu Profilen hinzufügen" #: cmd/incus/storage_volume.go:184 cmd/incus/storage_volume.go:185 #, fuzzy msgid "Attach new custom storage volumes to instances" msgstr "Storage Volumes zu Instanzen hinzufügen" #: cmd/incus/storage_volume.go:255 cmd/incus/storage_volume.go:256 #, fuzzy msgid "Attach new custom storage volumes to profiles" msgstr "Storage Volumes zu Profilen hinzufügen" #: cmd/incus/network.go:155 #, fuzzy msgid "Attach new network interfaces to instances" msgstr "Netzwerkschnittstellen (network interfaces) zu Instanzen hinzufügen" #: cmd/incus/console.go:44 msgid "Attach to instance consoles" msgstr "Konsole der Instanz öffnen" #: cmd/incus/console.go:45 #, fuzzy msgid "" "Attach to instance consoles\n" "\n" "This command allows you to interact with the boot console of an instance\n" "as well as retrieve past log entries from it." msgstr "" "Konsole der Instanz öffnen\n" "\n" "Dieser Befehl ermöglicht ihnen mit der boot Konsole einer Instanz zu " "interagieren,\n" "sowie bereits vorhandene Protokolleinträge (logs) einzusehen." #: cmd/incus/remote.go:597 #, c-format msgid "Authentication type '%s' not supported by server" msgstr "Authentifizierungstyp '%s' wird nicht vom Server unterstützt" #: cmd/incus/info.go:234 #, c-format msgid "Auto negotiation: %v" msgstr "Automatische Aushandlung: %v" #: cmd/incus/image.go:213 #, fuzzy msgid "Auto update is only available in pull mode" msgstr "Die Option \"automatisches Update\" ist nur mit \"pull\" verfügbar" #: cmd/incus/image.go:1005 #, c-format msgid "Auto update: %s" msgstr "automatisches Update: %s" #: cmd/incus/admin_init.go:51 msgid "Automatic (non-interactive) mode" msgstr "Automatischer Modus (nicht-interaktiv)" #: cmd/incus/color/color.go:41 #, fuzzy msgid "Available Commands:" msgstr "Verfügbare Projekte:" #: cmd/incus/remote.go:173 msgid "Available projects:" msgstr "Verfügbare Projekte:" #: cmd/incus/info.go:495 #, c-format msgid "Average: %.2f %.2f %.2f" msgstr "Durchschnitt: %.2f %.2f %.2f" #: cmd/incus/list.go:479 cmd/incus/list.go:480 msgid "BASE IMAGE" msgstr "BASIS ABBILD" #: cmd/incus/network_load_balancer.go:822 msgid "Backend description" msgstr "" #: cmd/incus/network_load_balancer.go:1159 msgid "Backend health:" msgstr "Backend Status:" #: cmd/incus/export.go:103 #, fuzzy, c-format msgid "Backing up instance: %s" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/storage_bucket.go:1352 #, fuzzy, c-format msgid "Backing up storage bucket: %s" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_volume.go:3728 #, fuzzy, c-format msgid "Backing up storage volume: %s" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/export.go:207 cmd/incus/storage_bucket.go:1456 #: cmd/incus/storage_volume.go:3832 msgid "Backup exported successfully!" msgstr "Backup erfolgreich exportiert!" #: cmd/incus/info.go:855 cmd/incus/storage_volume.go:1369 msgid "Backups:" msgstr "Sicherungen:" #: cmd/incus/utils.go:91 #, c-format msgid "Bad device override syntax, expecting ,=: %s" msgstr "" "Ungültige Gerätüberschreibungs-Syntax, erwartet wird ," "=: %s" #: cmd/incus/copy.go:137 cmd/incus/create.go:205 cmd/incus/move.go:250 #: cmd/incus/network_integration.go:139 cmd/incus/project.go:168 #, fuzzy, c-format msgid "Bad key=value pair: %q" msgstr "Alternatives config Verzeichnis." #: cmd/incus/remote.go:143 msgid "Binary helper for retrieving credentials" msgstr "" #: cmd/incus/network.go:941 msgid "Bond:" msgstr "Link-Bündelung:" #: cmd/incus/action.go:353 msgid "Both --all and instance name given" msgstr "Sowohl --all als auch ein Instanzname angegeben" #: cmd/incus/info.go:142 #, fuzzy, c-format msgid "Brand: %v" msgstr "Erstellt: %s" #: cmd/incus/network.go:954 msgid "Bridge:" msgstr "Netzwerk-Brücke:" #: cmd/incus/storage_bucket.go:115 #, fuzzy msgid "Bucket description" msgstr "Fingerabdruck: %s\n" #: cmd/incus/info.go:351 #, c-format msgid "Bus Address: %v" msgstr "Bus Adresse: %v" #: cmd/incus/info.go:778 cmd/incus/network.go:932 msgid "Bytes received" msgstr "Bytes empfangen" #: cmd/incus/info.go:779 cmd/incus/network.go:933 msgid "Bytes sent" msgstr "Bytes gesendet" #: cmd/incus/operation.go:168 msgid "CANCELABLE" msgstr "ABBRECHBAR" #: cmd/incus/config_trust.go:415 msgid "COMMON NAME" msgstr "ALLGEMEINER NAME" #: cmd/incus/storage_volume.go:1570 msgid "CONTENT-TYPE" msgstr "INHALTS-TYP" #: cmd/incus/warning.go:210 msgid "COUNT" msgstr "ANZAHL" #: cmd/incus/top.go:84 #, fuzzy msgid "CPU TIME(s)" msgstr "Prozessorauslastung (%s):" #: cmd/incus/list.go:491 msgid "CPU USAGE" msgstr "CPU NUTZUNG" #: cmd/incus/info.go:719 msgid "CPU usage (in seconds)" msgstr "CPU Nutzung (in Sekunden)" #: cmd/incus/info.go:723 msgid "CPU usage:" msgstr "Prozessorauslastung:" #: cmd/incus/info.go:500 msgid "CPU:" msgstr "CPU:" #: cmd/incus/info.go:504 msgid "CPUs:" msgstr "CPUs:" #: cmd/incus/operation.go:169 #, fuzzy msgid "CREATED" msgstr "ERSTELLT AM" #: cmd/incus/list.go:475 msgid "CREATED AT" msgstr "ERSTELLT AM" #: cmd/incus/info.go:144 #, c-format msgid "CUDA Version: %v" msgstr "CUDA Version: %v" #: cmd/incus/image.go:1004 #, c-format msgid "Cached: %s" msgstr "Zwischengespeichert: %s" #: cmd/incus/info.go:321 msgid "Caches:" msgstr "Zwischenspeicher:" #: cmd/incus/admin_init_interactive.go:601 #, c-format msgid "Can't bind address %q: %w" msgstr "Kann Adresse nicht zuweisen %q: %w" #: cmd/incus/move.go:115 msgid "Can't override configuration or profiles in local rename" msgstr "" #: cmd/incus/move.go:111 #, fuzzy msgid "Can't perform local rename without a new instance name" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/utils.go:333 #, c-format msgid "Can't read from environment file: %w" msgstr "Kann nicht aus der Umgebungsdatei lesen: %w" #: cmd/incus/utils.go:258 cmd/incus/utils.go:278 cmd/incus/utils.go:305 #, c-format msgid "Can't read from stdin: %w" msgstr "Kann nicht von stdin lesen: %w" #: cmd/incus/remote.go:1240 msgid "Can't remove the default remote" msgstr "" #: cmd/incus/list.go:506 msgid "Can't specify --fast with --columns" msgstr "\"Kann --fast nicht zusammen mit --columns angeben.\"" #: cmd/incus/cluster.go:274 cmd/incus/list.go:412 cmd/incus/profile.go:783 msgid "Can't specify --project with --all-projects" msgstr "Kann --project nicht zusammen mit --all-projects angeben" #: cmd/incus/list.go:537 cmd/incus/storage_volume.go:1580 #: cmd/incus/warning.go:225 msgid "Can't specify column L when not clustered" msgstr "" "Kann die Spalte L nicht angeben, wenn keine Clusterbildung vorhanden ist" #: cmd/incus/config.go:618 #, c-format msgid "Can't unset key '%s', it's not currently set" msgstr "" "Kann Schlüssel '%s' nicht zurücksetzen, da er derzeit nicht gesetzt ist" #: cmd/incus/admin_init.go:74 msgid "Can't use --auto and --preseed together" msgstr "" "Die Optionen --auto und --preseed können nicht zusammen verwendet werden" #: cmd/incus/admin_init.go:96 msgid "Can't use --dump with other flags" msgstr "" "Die Option --dump kann nicht zusammen mit anderen Optionen genutzt werden" #: cmd/incus/admin_init.go:82 msgid "Can't use --minimal and --auto together" msgstr "Die Option --minimal und --auto können nicht zusammen genutzt werden" #: cmd/incus/admin_init.go:78 msgid "Can't use --minimal and --preseed together" msgstr "" "Die Option --minimal und --preseed können nicht zusammen genutzt werden" #: cmd/incus/snapshot.go:127 cmd/incus/storage_volume.go:3089 msgid "Can't use both --no-expiry and --expiry" msgstr "" #: cmd/incus/create.go:314 #, c-format msgid "" "Cannot override config for device %q: Device not found in profile devices" msgstr "" "\"Kann Konfiguration für Gerät %q nicht überschreiben: Gerät in den " "Profilgeräten nicht gefunden.\"" #: cmd/incus/utils.go:300 #, fuzzy msgid "Cannot read the stdin twice" msgstr "Kann nicht von stdin lesen: %w" #: cmd/incus/storage_volume.go:425 msgid "" "Cannot set --destination-target when destination server is not clustered" msgstr "" "Kann --destination-target nicht setzen, wenn der Zielserver nicht im Cluster " "ist" #: cmd/incus/storage_volume.go:391 msgid "Cannot set --target when source server is not clustered" msgstr "" "\"Kann --target nicht setzen, wenn der Quellserver nicht im Cluster ist" #: cmd/incus/storage_volume.go:405 msgid "Cannot set --volume-only when copying a snapshot" msgstr "Kann --volume-only nicht setzen, wenn ein Snapshot kopiert wird" #: cmd/incus/network_acl.go:897 #, c-format msgid "Cannot set key: %s" msgstr "Kann Schlüssel nicht setzen: %s" #: cmd/incus/info.go:549 cmd/incus/info.go:561 #, c-format msgid "Card %d:" msgstr "Karte: %d:" #: cmd/incus/info.go:127 #, fuzzy, c-format msgid "Card: %s (%s)" msgstr "" "Benutzung: %s\n" "\n" "Optionen:\n" "\n" #: cmd/incus/config_trust.go:790 #, c-format msgid "Certificate add token for %s deleted" msgstr "\"Zertifikat-Token für %s gelöscht\"" #: cmd/incus/config_trust.go:180 #, fuzzy msgid "Certificate description" msgstr "Fingerprint des Zertifikats: %s" #: cmd/incus/remote.go:261 #, c-format msgid "" "Certificate fingerprint mismatch between certificate token and server %q" msgstr "" #: cmd/incus/cluster.go:1731 cmd/incus/cluster.go:1925 #, c-format msgid "" "Certificate fingerprint mismatch between join token and cluster member %q" msgstr "" #: cmd/incus/remote.go:494 #, c-format msgid "Certificate fingerprint: %s" msgstr "Fingerprint des Zertifikats: %s" #: cmd/incus/network.go:977 msgid "Chassis" msgstr "" #: cmd/incus/admin_waitready.go:78 #, c-format msgid "Checking if the daemon is ready (attempt %d)" msgstr "" #: cmd/incus/cluster.go:1830 #, c-format msgid "Choose %s:" msgstr "" #: cmd/incus/config_trust.go:141 #, fuzzy, c-format msgid "Client %s certificate add token:" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/remote.go:643 msgid "Client certificate now trusted by server:" msgstr "Client-Zertifikat wird nun vom Server als vertrauenswürdig behandelt:" #: cmd/incus/version.go:38 #, c-format msgid "Client version: %s\n" msgstr "Client Version: %s\n" #: cmd/incus/cluster_group.go:234 #, c-format msgid "Cluster group %s created" msgstr "Cluster-Gruppe %s erstellt" #: cmd/incus/cluster_group.go:279 #, c-format msgid "Cluster group %s deleted" msgstr "Cluster-Gruppe %s gelöscht\"" #: cmd/incus/cluster_group.go:600 #, fuzzy, c-format msgid "Cluster group %s isn't currently applied to %s" msgstr "Profil %s wurde auf %s angewandt\n" #: cmd/incus/cluster_group.go:671 #, fuzzy, c-format msgid "Cluster group %s renamed to %s" msgstr "Profil %s wurde auf %s angewandt\n" #: cmd/incus/cluster_group.go:180 #, fuzzy msgid "Cluster group description" msgstr "Cluster-Gruppe %s erstellt" #: cmd/incus/cluster.go:1322 #, fuzzy, c-format msgid "Cluster join token for %s deleted" msgstr "\"Join-Token für Cluster %s:%s gelöscht\"" #: cmd/incus/cluster_group.go:152 #, fuzzy, c-format msgid "Cluster member %s added to cluster groups %v" msgstr "%s als Mitglied zu den Cluster-Gruppen %s hinzugefügt" #: cmd/incus/cluster_group.go:788 #, fuzzy, c-format msgid "Cluster member %s added to group %s" msgstr "Gerät %s wurde von %s entfernt\n" #: cmd/incus/cluster_group.go:777 #, fuzzy, c-format msgid "Cluster member %s is already in group %s" msgstr "Gerät %s wurde von %s entfernt\n" #: cmd/incus/cluster_group.go:620 #, fuzzy, c-format msgid "Cluster member %s removed from group %s" msgstr "Gerät %s wurde von %s entfernt\n" #: cmd/incus/config.go:102 cmd/incus/config.go:382 cmd/incus/config.go:522 #: cmd/incus/config.go:689 cmd/incus/config.go:809 cmd/incus/copy.go:66 #: cmd/incus/create.go:70 cmd/incus/info.go:53 cmd/incus/move.go:67 #: cmd/incus/network.go:348 cmd/incus/network.go:798 cmd/incus/network.go:871 #: cmd/incus/network.go:1047 cmd/incus/network.go:1428 #: cmd/incus/network.go:1512 cmd/incus/network.go:1574 #: cmd/incus/network_forward.go:248 cmd/incus/network_forward.go:319 #: cmd/incus/network_forward.go:484 cmd/incus/network_forward.go:623 #: cmd/incus/network_forward.go:764 cmd/incus/network_forward.go:842 #: cmd/incus/network_forward.go:915 cmd/incus/network_load_balancer.go:251 #: cmd/incus/network_load_balancer.go:322 #: cmd/incus/network_load_balancer.go:470 #: cmd/incus/network_load_balancer.go:592 #: cmd/incus/network_load_balancer.go:744 #: cmd/incus/network_load_balancer.go:821 #: cmd/incus/network_load_balancer.go:886 #: cmd/incus/network_load_balancer.go:980 #: cmd/incus/network_load_balancer.go:1046 cmd/incus/storage.go:117 #: cmd/incus/storage.go:393 cmd/incus/storage.go:469 cmd/incus/storage.go:799 #: cmd/incus/storage.go:892 cmd/incus/storage.go:971 #: cmd/incus/storage_bucket.go:114 cmd/incus/storage_bucket.go:203 #: cmd/incus/storage_bucket.go:253 cmd/incus/storage_bucket.go:371 #: cmd/incus/storage_bucket.go:600 cmd/incus/storage_bucket.go:705 #: cmd/incus/storage_bucket.go:758 cmd/incus/storage_bucket.go:856 #: cmd/incus/storage_bucket.go:982 cmd/incus/storage_bucket.go:1072 #: cmd/incus/storage_bucket.go:1121 cmd/incus/storage_bucket.go:1241 #: cmd/incus/storage_bucket.go:1299 cmd/incus/storage_bucket.go:1478 #: cmd/incus/storage_volume.go:347 cmd/incus/storage_volume.go:554 #: cmd/incus/storage_volume.go:649 cmd/incus/storage_volume.go:905 #: cmd/incus/storage_volume.go:1114 cmd/incus/storage_volume.go:1231 #: cmd/incus/storage_volume.go:1677 cmd/incus/storage_volume.go:1737 #: cmd/incus/storage_volume.go:1819 cmd/incus/storage_volume.go:1968 #: cmd/incus/storage_volume.go:2057 cmd/incus/storage_volume.go:3056 #: cmd/incus/storage_volume.go:3204 cmd/incus/storage_volume.go:3440 #: cmd/incus/storage_volume.go:3509 cmd/incus/storage_volume.go:3573 #: cmd/incus/storage_volume.go:3649 cmd/incus/storage_volume.go:3857 msgid "Cluster member name" msgstr "Cluster-Mitgliedsname" #: cmd/incus/cluster.go:839 msgid "Clustering enabled" msgstr "Clustering aktiviert" #: cmd/incus/cluster.go:170 cmd/incus/cluster.go:1124 #: cmd/incus/cluster_group.go:439 cmd/incus/config_trust.go:399 #: cmd/incus/config_trust.go:585 cmd/incus/image.go:1069 #: cmd/incus/image_alias.go:209 cmd/incus/list.go:140 cmd/incus/network.go:1044 #: cmd/incus/network.go:1242 cmd/incus/network_allocations.go:67 #: cmd/incus/network_forward.go:121 cmd/incus/network_integration.go:412 #: cmd/incus/network_load_balancer.go:128 cmd/incus/network_peer.go:117 #: cmd/incus/network_zone.go:122 cmd/incus/operation.go:149 #: cmd/incus/profile.go:697 cmd/incus/project.go:517 cmd/incus/remote.go:971 #: cmd/incus/snapshot.go:342 cmd/incus/storage.go:660 #: cmd/incus/storage_bucket.go:464 cmd/incus/storage_bucket.go:857 #: cmd/incus/storage_volume.go:1439 cmd/incus/storage_volume.go:3284 #: cmd/incus/top.go:67 cmd/incus/warning.go:98 msgid "Columns" msgstr "Spalten" #: cmd/incus/main.go:142 msgid "Command line client for Incus" msgstr "Kommandozeilen-Client für Incus" #: cmd/incus/main.go:143 msgid "" "Command line client for Incus\n" "\n" "All of Incus's features can be driven through the various commands below.\n" "For help with any of those, simply call them with --help.\n" "\n" "Custom commands can be defined through aliases, use \"incus alias\" to " "control those." msgstr "" "Kommandozeilen-Client für Incus\n" "\n" "Alle Funktionen von Incus können über die unten aufgeführten Befehle " "gesteuert werden.\n" "Für Hilfe zu einem der Befehle, einfach mit --help aufrufen.\n" "\n" "Eigene Befehle können über Aliase definiert werden. Verwenden Sie \"incus " "alias\", um diese zu verwalten." #: cmd/incus/publish.go:44 msgid "Compression algorithm to use (`none` for uncompressed)" msgstr "" #: cmd/incus/export.go:52 msgid "Compression algorithm to use (none for uncompressed)" msgstr "" #: cmd/incus/storage_volume.go:3648 msgid "" "Compression algorithm to use (none for uncompressed, ignored for ISO storage " "volumes)" msgstr "" #: cmd/incus/copy.go:58 cmd/incus/create.go:62 cmd/incus/import.go:40 #, fuzzy msgid "Config key/value to apply to the new instance" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/network_integration.go:96 #, fuzzy msgid "Config key/value to apply to the new network integration" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/project.go:119 #, fuzzy msgid "Config key/value to apply to the new project" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/move.go:59 #, fuzzy msgid "Config key/value to apply to the target instance" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/admin_recover.go:129 msgid "Config option should be in the format KEY=VALUE" msgstr "" #: cmd/incus/cluster.go:934 cmd/incus/cluster_group.go:374 #: cmd/incus/config.go:265 cmd/incus/config.go:340 #: cmd/incus/config_metadata.go:147 cmd/incus/config_trust.go:329 #: cmd/incus/image.go:482 cmd/incus/network.go:759 cmd/incus/network_acl.go:669 #: cmd/incus/network_address_set.go:524 cmd/incus/network_forward.go:726 #: cmd/incus/network_integration.go:294 cmd/incus/network_load_balancer.go:706 #: cmd/incus/network_peer.go:729 cmd/incus/network_zone.go:671 #: cmd/incus/network_zone.go:1338 cmd/incus/profile.go:569 #: cmd/incus/project.go:394 cmd/incus/storage.go:354 #: cmd/incus/storage_bucket.go:333 cmd/incus/storage_bucket.go:1202 #: cmd/incus/storage_volume.go:1031 cmd/incus/storage_volume.go:1063 #, fuzzy, c-format msgid "Config parsing error: %s" msgstr "YAML Analyse Fehler %v\n" #: cmd/incus/admin_init.go:88 msgid "Configuration flags require --auto" msgstr "" #: cmd/incus/admin_init.go:41 cmd/incus/admin_init.go:42 msgid "Configure the daemon" msgstr "" #: cmd/incus/top.go:69 #, fuzzy msgid "Configure the refresh delay in seconds" msgstr "Legen Sie eine Verzögerungszeit angegeben in Sekunden fest:" #: cmd/incus/admin_waitready.go:63 #, c-format msgid "Connecting to the daemon (attempt %d)" msgstr "" #: cmd/incus/storage_volume.go:555 msgid "Content type, block or filesystem" msgstr "" #: cmd/incus/storage_volume.go:1309 #, fuzzy, c-format msgid "Content type: %s" msgstr "Erstellt: %s" #: cmd/incus/info.go:131 #, c-format msgid "Control: %s (%s)" msgstr "" #: cmd/incus/copy.go:64 cmd/incus/move.go:65 msgid "Copy a stateful instance stateless" msgstr "" #: cmd/incus/image.go:174 msgid "Copy aliases from source" msgstr "Kopiere Aliasse von der Quelle" #: cmd/incus/storage_volume.go:343 cmd/incus/storage_volume.go:344 #, fuzzy msgid "Copy custom storage volumes" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/image.go:166 msgid "Copy images between servers" msgstr "" #: cmd/incus/image.go:167 msgid "" "Copy images between servers\n" "\n" "The auto-update flag instructs the server to keep this image up to date.\n" "It requires the source to be an alias and for it to be public." msgstr "" #: cmd/incus/copy.go:45 #, fuzzy msgid "Copy instances within or in between servers" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/copy.go:46 msgid "" "Copy instances within or in between servers\n" "\n" "Transfer modes (--mode):\n" " - pull: Target server pulls the data from the source server (source must " "listen on network)\n" " - push: Source server pushes the data to the target server (target must " "listen on network)\n" " - relay: The CLI connects to both source and server and proxies the data " "(both source and target must listen on network)\n" "\n" "The pull transfer mode is the default as it is compatible with all server " "versions.\n" msgstr "" #: cmd/incus/config_device.go:365 cmd/incus/config_device.go:366 msgid "Copy profile inherited devices and override configuration keys" msgstr "" #: cmd/incus/profile.go:271 cmd/incus/profile.go:272 msgid "Copy profiles" msgstr "" #: cmd/incus/copy.go:63 #, fuzzy msgid "Copy the instance without its snapshots" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/storage_volume.go:349 #, fuzzy msgid "Copy the volume without its snapshots" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/copy.go:67 cmd/incus/image.go:180 cmd/incus/move.go:68 #: cmd/incus/profile.go:273 cmd/incus/storage_volume.go:350 msgid "Copy to a project different from the source" msgstr "" #: cmd/incus/image.go:178 msgid "Copy virtual machine images" msgstr "" #: cmd/incus/image.go:302 #, c-format msgid "Copying the image: %s" msgstr "" #: cmd/incus/storage_volume.go:440 #, fuzzy, c-format msgid "Copying the storage volume: %s" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/info.go:329 #, fuzzy, c-format msgid "Core %d" msgstr "Fehler: %v\n" #: cmd/incus/info.go:327 #, fuzzy msgid "Cores:" msgstr "Fehler: %v\n" #: cmd/incus/remote.go:530 #, fuzzy, c-format msgid "Could not close server cert file %q: %w" msgstr "Kann Verzeichnis für Zertifikate auf dem Server nicht erstellen" #: cmd/incus/remote.go:267 cmd/incus/remote.go:514 msgid "Could not create server cert dir" msgstr "Kann Verzeichnis für Zertifikate auf dem Server nicht erstellen" #: cmd/incus/cluster.go:1391 #, fuzzy, c-format msgid "Could not find certificate file path: %s" msgstr "Fingerabdruck des Zertifikats: % x\n" #: cmd/incus/cluster.go:1395 #, c-format msgid "Could not find certificate key file path: %s" msgstr "" #: cmd/incus/cluster.go:1400 #, fuzzy, c-format msgid "Could not read certificate file: %s with error: %v" msgstr "Fingerabdruck des Zertifikats: % x\n" #: cmd/incus/cluster.go:1405 #, fuzzy, c-format msgid "Could not read certificate key file: %s with error: %v" msgstr "Fingerabdruck des Zertifikats: % x\n" #: cmd/incus/cluster.go:1422 #, c-format msgid "Could not write new remote certificate for remote '%s' with error: %v" msgstr "" #: cmd/incus/remote.go:525 #, fuzzy, c-format msgid "Could not write server cert file %q: %w" msgstr "Kann Verzeichnis für Zertifikate auf dem Server nicht erstellen" #: cmd/incus/network_zone.go:1545 msgid "Couldn't find a matching entry" msgstr "" #: cmd/incus/admin_init_interactive.go:452 #, c-format msgid "Couldn't statfs %s: %w" msgstr "" #: cmd/incus/cluster_group.go:171 cmd/incus/cluster_group.go:172 msgid "Create a cluster group" msgstr "" #: cmd/incus/admin_init_interactive.go:385 #, fuzzy, c-format msgid "Create a new %s pool?" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/create.go:73 #, fuzzy msgid "Create a virtual machine" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/image_alias.go:72 cmd/incus/image_alias.go:73 msgid "Create aliases for existing images" msgstr "" #: cmd/incus/create.go:72 #, fuzzy msgid "Create an empty instance" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/launch.go:24 cmd/incus/launch.go:25 #, fuzzy msgid "Create and start instances from images" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/file.go:108 cmd/incus/file.go:446 cmd/incus/file.go:674 #: cmd/incus/storage_volume.go:2165 cmd/incus/storage_volume.go:2587 #: cmd/incus/storage_volume.go:2782 msgid "Create any directories necessary" msgstr "" #: cmd/incus/storage_volume.go:2155 #, fuzzy msgid "Create files and directories in custom vollume" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/storage_volume.go:2156 #, fuzzy msgid "Create files and directories in custom volume" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/file.go:98 cmd/incus/file.go:99 #, fuzzy msgid "Create files and directories in instances" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/export.go:98 #, fuzzy, c-format msgid "Create instance backup: %w" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/snapshot.go:85 #, fuzzy msgid "Create instance snapshot" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/snapshot.go:86 msgid "" "Create instance snapshots\n" "\n" "When --stateful is used, attempt to checkpoint the instance's\n" "running state, including process memory state, TCP connections, ..." msgstr "" #: cmd/incus/create.go:49 cmd/incus/create.go:50 #, fuzzy msgid "Create instances from images" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/storage_bucket.go:972 cmd/incus/storage_bucket.go:973 #, fuzzy msgid "Create key for a storage bucket" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/network_integration.go:88 cmd/incus/network_integration.go:89 #, fuzzy msgid "Create network integrations" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/storage_bucket.go:106 cmd/incus/storage_bucket.go:107 #, fuzzy msgid "Create new custom storage buckets" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/storage_volume.go:546 cmd/incus/storage_volume.go:547 #, fuzzy msgid "Create new custom storage volumes" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/config_template.go:69 cmd/incus/config_template.go:70 #, fuzzy msgid "Create new instance file templates" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/network_acl.go:356 cmd/incus/network_acl.go:357 msgid "Create new network ACLs" msgstr "" #: cmd/incus/network_address_set.go:231 cmd/incus/network_address_set.go:232 #, fuzzy msgid "Create new network address sets" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/network_forward.go:310 cmd/incus/network_forward.go:311 #, fuzzy msgid "Create new network forwards" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/network_load_balancer.go:312 #: cmd/incus/network_load_balancer.go:313 #, fuzzy msgid "Create new network load balancers" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/network_peer.go:305 cmd/incus/network_peer.go:306 #, fuzzy msgid "Create new network peering" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/network_zone.go:1022 cmd/incus/network_zone.go:1023 #, fuzzy msgid "Create new network zone record" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/network_zone.go:372 cmd/incus/network_zone.go:373 #, fuzzy msgid "Create new network zones" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/network.go:337 cmd/incus/network.go:338 msgid "Create new networks" msgstr "" #: cmd/incus/profile.go:343 cmd/incus/profile.go:344 #, fuzzy msgid "Create profiles" msgstr "Fehlerhafte Profil URL %s" #: cmd/incus/project.go:111 cmd/incus/project.go:112 #, fuzzy msgid "Create projects" msgstr "Fehlerhafte Profil URL %s" #: cmd/incus/storage.go:108 cmd/incus/storage.go:109 #, fuzzy msgid "Create storage pools" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/copy.go:68 cmd/incus/create.go:71 #, fuzzy msgid "Create the instance with no profiles applied" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/image.go:973 cmd/incus/info.go:674 #: cmd/incus/storage_volume.go:1323 #, c-format msgid "Created: %s" msgstr "Erstellt: %s" #: cmd/incus/create.go:135 #, c-format msgid "Creating %s" msgstr "Erstelle %s" #: cmd/incus/file.go:230 cmd/incus/storage_volume.go:2284 #, fuzzy, c-format msgid "Creating %s: %%s" msgstr "Erstelle %s" #: cmd/incus/create.go:137 #, fuzzy msgid "Creating the instance" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/info.go:151 cmd/incus/info.go:260 #, c-format msgid "Current number of VFs: %d" msgstr "" #: cmd/incus/network_forward.go:144 msgid "DEFAULT TARGET ADDRESS" msgstr "" #: cmd/incus/cluster.go:200 cmd/incus/cluster_group.go:465 #: cmd/incus/config_trust.go:417 cmd/incus/image.go:1098 #: cmd/incus/image_alias.go:249 cmd/incus/list.go:476 cmd/incus/network.go:1076 #: cmd/incus/network_acl.go:163 cmd/incus/network_address_set.go:155 #: cmd/incus/network_forward.go:143 cmd/incus/network_integration.go:428 #: cmd/incus/network_load_balancer.go:150 cmd/incus/network_peer.go:139 #: cmd/incus/network_zone.go:145 cmd/incus/network_zone.go:872 #: cmd/incus/operation.go:166 cmd/incus/profile.go:725 cmd/incus/project.go:548 #: cmd/incus/storage.go:686 cmd/incus/storage_bucket.go:481 #: cmd/incus/storage_bucket.go:873 cmd/incus/storage_volume.go:1569 msgid "DESCRIPTION" msgstr "BESCHREIBUNG" #: cmd/incus/top.go:86 msgid "DISK" msgstr "" #: cmd/incus/list.go:477 msgid "DISK USAGE" msgstr "" #: cmd/incus/storage.go:685 msgid "DRIVER" msgstr "" #: cmd/incus/info.go:123 msgid "DRM:" msgstr "" #: cmd/incus/admin_waitready.go:101 #, c-format msgid "Daemon still not running after %ds timeout (%v)" msgstr "" #: cmd/incus/admin_shutdown.go:98 #, c-format msgid "Daemon still running after %ds timeout" msgstr "" #: cmd/incus/info.go:487 #, fuzzy, c-format msgid "Date: %s" msgstr "Erstellt: %s" #: cmd/incus/debug.go:24 msgid "Debug commands" msgstr "" #: cmd/incus/debug.go:25 #, fuzzy msgid "Debug commands for instances" msgstr "Befehle in einer Instanz ausführen" #: cmd/incus/network.go:958 msgid "Default VLAN ID" msgstr "" #: cmd/incus/storage_bucket.go:1298 msgid "Define a compression algorithm: for backup or none" msgstr "" #: cmd/incus/top.go:438 msgid "Delay:" msgstr "" #: cmd/incus/warning.go:345 msgid "Delete all warnings" msgstr "" #: cmd/incus/operation.go:67 cmd/incus/operation.go:68 msgid "Delete background operations (will attempt to cancel)" msgstr "" #: cmd/incus/cluster_group.go:252 cmd/incus/cluster_group.go:253 #, fuzzy msgid "Delete cluster groups" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_volume.go:646 cmd/incus/storage_volume.go:647 #, fuzzy msgid "Delete custom storage volumes" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_volume.go:2329 cmd/incus/storage_volume.go:2330 #, fuzzy msgid "Delete files in custom volume" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/file.go:273 cmd/incus/file.go:274 #, fuzzy msgid "Delete files in instances" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/image_alias.go:132 cmd/incus/image_alias.go:133 msgid "Delete image aliases" msgstr "" #: cmd/incus/image.go:336 cmd/incus/image.go:337 msgid "Delete images" msgstr "" #: cmd/incus/config_template.go:130 cmd/incus/config_template.go:131 #, fuzzy msgid "Delete instance file templates" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/snapshot.go:229 cmd/incus/snapshot.go:230 #, fuzzy msgid "Delete instance snapshots" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/delete.go:36 cmd/incus/delete.go:37 #, fuzzy msgid "Delete instances" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/storage_bucket.go:1068 cmd/incus/storage_bucket.go:1069 #, fuzzy msgid "Delete key from a storage bucket" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_acl.go:753 cmd/incus/network_acl.go:754 msgid "Delete network ACLs" msgstr "" #: cmd/incus/network_address_set.go:607 cmd/incus/network_address_set.go:608 #, fuzzy msgid "Delete network address sets" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_forward.go:760 cmd/incus/network_forward.go:761 msgid "Delete network forwards" msgstr "" #: cmd/incus/network_integration.go:172 cmd/incus/network_integration.go:173 #, fuzzy msgid "Delete network integrations" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_load_balancer.go:740 #: cmd/incus/network_load_balancer.go:741 #, fuzzy msgid "Delete network load balancers" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_peer.go:763 cmd/incus/network_peer.go:764 msgid "Delete network peerings" msgstr "" #: cmd/incus/network_zone.go:1372 cmd/incus/network_zone.go:1373 #, fuzzy msgid "Delete network zone record" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_zone.go:705 cmd/incus/network_zone.go:706 #, fuzzy msgid "Delete network zones" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network.go:445 cmd/incus/network.go:446 msgid "Delete networks" msgstr "" #: cmd/incus/profile.go:422 cmd/incus/profile.go:423 msgid "Delete profiles" msgstr "" #: cmd/incus/project.go:205 cmd/incus/project.go:206 #, fuzzy msgid "Delete projects" msgstr "Fehlerhafte Profil URL %s" #: cmd/incus/storage_bucket.go:200 cmd/incus/storage_bucket.go:201 #, fuzzy msgid "Delete storage buckets" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage.go:211 cmd/incus/storage.go:212 msgid "Delete storage pools" msgstr "" #: cmd/incus/storage_volume.go:3201 cmd/incus/storage_volume.go:3202 #, fuzzy msgid "Delete storage volume snapshots" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/warning.go:342 cmd/incus/warning.go:343 #, fuzzy msgid "Delete warnings" msgstr "Aliasse:\n" #: cmd/incus/storage_volume.go:1355 msgid "Description" msgstr "" #: cmd/incus/color/color.go:36 #, fuzzy msgid "Description:" msgstr "Fingerabdruck: %s\n" #: cmd/incus/info.go:645 cmd/incus/storage_volume.go:1296 #, fuzzy, c-format msgid "Description: %s" msgstr "Fingerabdruck: %s\n" #: cmd/incus/storage_volume.go:348 cmd/incus/storage_volume.go:1678 #, fuzzy msgid "Destination cluster member name" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/storage_volume.go:708 cmd/incus/storage_volume.go:709 #, fuzzy msgid "Detach custom storage volumes from instances" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_volume.go:822 cmd/incus/storage_volume.go:823 #, fuzzy msgid "Detach custom storage volumes from profiles" msgstr "Storage Volumes zu Profilen hinzufügen" #: cmd/incus/network.go:499 cmd/incus/network.go:500 #, fuzzy msgid "Detach network interfaces from instances" msgstr "Netzwerkschnittstellen an Container anbinden" #: cmd/incus/network.go:603 cmd/incus/network.go:604 msgid "Detach network interfaces from profiles" msgstr "" #: cmd/incus/info.go:585 cmd/incus/info.go:597 cmd/incus/info.go:609 #, fuzzy, c-format msgid "Device %d:" msgstr "" "Benutzung: %s\n" "\n" "Optionen:\n" "\n" #: cmd/incus/config_device.go:191 #, fuzzy, c-format msgid "Device %s added to %s" msgstr "Gerät %s wurde zu %s hinzugefügt\n" #: cmd/incus/config_device.go:427 #, fuzzy, c-format msgid "Device %s overridden for %s" msgstr "Gerät %s wurde von %s entfernt\n" #: cmd/incus/config_device.go:546 #, fuzzy, c-format msgid "Device %s removed from %s" msgstr "Gerät %s wurde von %s entfernt\n" #: cmd/incus/info.go:352 #, fuzzy, c-format msgid "Device Address: %v" msgstr "Profil %s erstellt\n" #: cmd/incus/utils.go:47 cmd/incus/utils.go:71 #, fuzzy, c-format msgid "Device already exists: %s" msgstr "entfernte Instanz %s existiert bereits" #: cmd/incus/config_device.go:258 cmd/incus/config_device.go:272 #: cmd/incus/config_device.go:623 cmd/incus/config_device.go:643 #, fuzzy msgid "Device doesn't exist" msgstr "entfernte Instanz %s existiert nicht" #: cmd/incus/config_device.go:646 msgid "" "Device from profile(s) cannot be modified for individual instance. Override " "device or modify profile instead" msgstr "" #: cmd/incus/config_device.go:526 #, c-format msgid "" "Device from profile(s) cannot be removed from individual instance. Override " "device “%s” or modify profile instead" msgstr "" #: cmd/incus/config_device.go:275 msgid "Device from profile(s) cannot be retrieved for individual instance" msgstr "" #: cmd/incus/config_device.go:496 cmd/incus/config_device.go:523 #, fuzzy, c-format msgid "Device “%s” doesn't exist" msgstr "entfernte Instanz %s existiert nicht" #: cmd/incus/info.go:280 cmd/incus/info.go:304 #, fuzzy, c-format msgid "Device: %s" msgstr "" "Benutzung: %s\n" "\n" "Optionen:\n" "\n" #: cmd/incus/info.go:371 #, fuzzy, c-format msgid "Device: %v" msgstr "" "Benutzung: %s\n" "\n" "Optionen:\n" "\n" #: cmd/incus/info.go:372 #, fuzzy, c-format msgid "DeviceID: %v" msgstr "" "Benutzung: %s\n" "\n" "Optionen:\n" "\n" #: cmd/incus/info.go:373 #, fuzzy, c-format msgid "DevicePath: %v" msgstr "" "Benutzung: %s\n" "\n" "Optionen:\n" "\n" #: cmd/incus/create.go:396 #, fuzzy msgid "Didn't get name of new instance from the server" msgstr "" "Falsche Anzahl an Objekten im Abbild, Container oder Sicherungspunkt gelesen." #: cmd/incus/snapshot.go:200 #, fuzzy msgid "Didn't get name of new instance snapshot from the server" msgstr "" "Falsche Anzahl an Objekten im Abbild, Container oder Sicherungspunkt gelesen." #: cmd/incus/storage_volume.go:3172 #, fuzzy msgid "Didn't get name of new volume snapshot from the server" msgstr "" "Falsche Anzahl an Objekten im Abbild, Container oder Sicherungspunkt gelesen." #: cmd/incus/image.go:718 msgid "Directory import is not available on this platform" msgstr "" #: cmd/incus/exec.go:70 msgid "Directory to run the command in (default /root)" msgstr "" #: cmd/incus/file.go:930 cmd/incus/storage_volume.go:2407 msgid "Disable authentication when using SSH SFTP listener" msgstr "" #: cmd/incus/exec.go:66 msgid "Disable pseudo-terminal allocation" msgstr "" #: cmd/incus/exec.go:67 msgid "Disable stdin (reads from /dev/null)" msgstr "" #: cmd/incus/info.go:573 #, c-format msgid "Disk %d:" msgstr "Festplatte %d:" #: cmd/incus/info.go:712 msgid "Disk usage:" msgstr "Festplattennutzung:" #: cmd/incus/info.go:568 msgid "Disk:" msgstr "Festplatte:" #: cmd/incus/info.go:571 msgid "Disks:" msgstr "Festplatten:" #: cmd/incus/cluster.go:172 #, fuzzy msgid "Display clusters from all projects" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/image.go:1071 #, fuzzy msgid "Display images from all projects" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/list.go:143 cmd/incus/top.go:66 #, fuzzy msgid "Display instances from all projects" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/network_zone.go:121 #, fuzzy msgid "Display network zones from all projects" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/profile.go:699 #, fuzzy msgid "Display profiles from all projects" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/top.go:45 msgid "Display resource usage info per instance" msgstr "" #: cmd/incus/storage_bucket.go:463 #, fuzzy msgid "Display storage pool buckets from all projects" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/top.go:46 msgid "" "Displays CPU usage, memory usage, and disk usage per instance\n" "\n" "Default column layout: numD\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which instance attributes to output when displaying in table or compact\n" "format.\n" "\n" "Column arguments are pre-defined shorthand chars (see below).\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Column shorthand chars:\n" " D - disk usage\n" " e - Project name\n" " m - Memory usage\n" " n - Instance name\n" " u - CPU usage (in seconds)" msgstr "" #: cmd/incus/admin_init_interactive.go:222 msgid "Do you want to configure a new local storage pool?" msgstr "" #: cmd/incus/admin_init_interactive.go:234 msgid "Do you want to configure a new remote storage pool?" msgstr "" #: cmd/incus/admin_init_interactive.go:249 msgid "Do you want to configure a new storage pool?" msgstr "" #: cmd/incus/admin_init_interactive.go:518 msgid "Do you want to continue without thin provisioning?" msgstr "" #: cmd/incus/cluster.go:691 msgid "Don't require user confirmation for using --force" msgstr "" #: cmd/incus/main.go:165 msgid "Don't show progress information" msgstr "" #: cmd/incus/network.go:945 msgid "Down delay" msgstr "" #: cmd/incus/info.go:366 cmd/incus/info.go:378 #, fuzzy, c-format msgid "Driver: %v" msgstr "" "Benutzung: %s\n" "\n" "Optionen:\n" "\n" #: cmd/incus/info.go:119 cmd/incus/info.go:205 #, c-format msgid "Driver: %v (%v)" msgstr "" #: cmd/incus/admin_init.go:54 msgid "Dump YAML config to stdout" msgstr "" #: cmd/incus/copy.go:70 msgid "" "During incremental copy, exclude source snapshots earlier than latest target " "snapshot" msgstr "" #: cmd/incus/storage_volume.go:352 msgid "" "During refresh, exclude source snapshots earlier than latest target snapshot" msgstr "" #: cmd/incus/network_zone.go:873 msgid "ENTRIES" msgstr "" #: cmd/incus/list.go:803 msgid "EPHEMERAL" msgstr "" #: cmd/incus/admin_recover.go:151 #, c-format msgid "EXISTING: %q (backend=%q, source=%q)" msgstr "" #: cmd/incus/cluster.go:1145 cmd/incus/config_trust.go:602 #: cmd/incus/snapshot.go:367 cmd/incus/storage_volume.go:3379 #, fuzzy msgid "EXPIRES AT" msgstr "ABLAUFDATUM" #: cmd/incus/config_trust.go:419 msgid "EXPIRY DATE" msgstr "ABLAUFDATUM" #: cmd/incus/cluster_group.go:304 cmd/incus/cluster_group.go:305 msgid "Edit a cluster group" msgstr "" #: cmd/incus/cluster.go:854 cmd/incus/cluster.go:855 #, fuzzy msgid "Edit cluster member configurations as YAML" msgstr "Alternatives config Verzeichnis." #: cmd/incus/file.go:357 cmd/incus/file.go:358 #, fuzzy msgid "Edit files in instances" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/storage_volume.go:2495 cmd/incus/storage_volume.go:2496 #, fuzzy msgid "Edit files in storage volumes" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/image.go:391 cmd/incus/image.go:392 msgid "Edit image properties" msgstr "" #: cmd/incus/config_template.go:176 cmd/incus/config_template.go:177 #, fuzzy msgid "Edit instance file templates" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/config_metadata.go:57 cmd/incus/config_metadata.go:58 #, fuzzy msgid "Edit instance metadata files" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/config.go:95 cmd/incus/config.go:96 #, fuzzy msgid "Edit instance or server configurations as YAML" msgstr "Alternatives config Verzeichnis." #: cmd/incus/network_acl.go:572 cmd/incus/network_acl.go:573 #, fuzzy msgid "Edit network ACL configurations as YAML" msgstr "Alternatives config Verzeichnis." #: cmd/incus/network_address_set.go:443 cmd/incus/network_address_set.go:444 #, fuzzy msgid "Edit network address set configurations as YAML" msgstr "Alternatives config Verzeichnis." #: cmd/incus/network.go:666 cmd/incus/network.go:667 msgid "Edit network configurations as YAML" msgstr "" #: cmd/incus/network_forward.go:619 cmd/incus/network_forward.go:620 #, fuzzy msgid "Edit network forward configurations as YAML" msgstr "Alternatives config Verzeichnis." #: cmd/incus/network_integration.go:222 cmd/incus/network_integration.go:223 #, fuzzy msgid "Edit network integration configurations as YAML" msgstr "Alternatives config Verzeichnis." #: cmd/incus/network_load_balancer.go:588 #: cmd/incus/network_load_balancer.go:589 #, fuzzy msgid "Edit network load balancer configurations as YAML" msgstr "Alternatives config Verzeichnis." #: cmd/incus/network_peer.go:639 cmd/incus/network_peer.go:640 #, fuzzy msgid "Edit network peer configurations as YAML" msgstr "Alternatives config Verzeichnis." #: cmd/incus/network_zone.go:586 cmd/incus/network_zone.go:587 #, fuzzy msgid "Edit network zone configurations as YAML" msgstr "Alternatives config Verzeichnis." #: cmd/incus/network_zone.go:1248 cmd/incus/network_zone.go:1249 #, fuzzy msgid "Edit network zone record configurations as YAML" msgstr "Alternatives config Verzeichnis." #: cmd/incus/profile.go:476 cmd/incus/profile.go:477 msgid "Edit profile configurations as YAML" msgstr "" #: cmd/incus/project.go:301 cmd/incus/project.go:302 msgid "Edit project configurations as YAML" msgstr "" #: cmd/incus/storage_bucket.go:248 cmd/incus/storage_bucket.go:249 #, fuzzy msgid "Edit storage bucket configurations as YAML" msgstr "Alternatives config Verzeichnis." #: cmd/incus/storage_bucket.go:1116 cmd/incus/storage_bucket.go:1117 msgid "Edit storage bucket key as YAML" msgstr "" #: cmd/incus/storage.go:265 cmd/incus/storage.go:266 msgid "Edit storage pool configurations as YAML" msgstr "" #: cmd/incus/storage_volume.go:892 msgid "Edit storage volume configurations as YAML" msgstr "" #: cmd/incus/storage_volume.go:893 #, fuzzy msgid "" "Edit storage volume configurations as YAML\n" "\n" "If the type is not specified, incus assumes the type is \"custom\".\n" "Supported values for type are \"custom\", \"container\" and \"virtual-" "machine\"." msgstr "Profil %s erstellt\n" #: cmd/incus/config_trust.go:261 cmd/incus/config_trust.go:262 #, fuzzy msgid "Edit trust configurations as YAML" msgstr "Alternatives config Verzeichnis." #: cmd/incus/cluster.go:211 cmd/incus/cluster.go:1153 #: cmd/incus/cluster_group.go:473 cmd/incus/config_trust.go:429 #: cmd/incus/config_trust.go:610 cmd/incus/image.go:1116 #: cmd/incus/image_alias.go:258 cmd/incus/list.go:528 cmd/incus/network.go:1091 #: cmd/incus/network.go:1280 cmd/incus/network_allocations.go:92 #: cmd/incus/network_forward.go:157 cmd/incus/network_integration.go:438 #: cmd/incus/network_load_balancer.go:163 cmd/incus/network_peer.go:150 #: cmd/incus/network_zone.go:158 cmd/incus/operation.go:181 #: cmd/incus/profile.go:741 cmd/incus/project.go:558 cmd/incus/remote.go:998 #: cmd/incus/snapshot.go:376 cmd/incus/storage.go:698 #: cmd/incus/storage_bucket.go:498 cmd/incus/storage_bucket.go:882 #: cmd/incus/storage_volume.go:1597 cmd/incus/storage_volume.go:3387 #: cmd/incus/top.go:95 cmd/incus/warning.go:236 #, c-format msgid "Empty column entry (redundant, leading or trailing command) in '%s'" msgstr "" #: cmd/incus/cluster.go:772 msgid "Enable clustering on a single non-clustered server" msgstr "" #: cmd/incus/cluster.go:773 msgid "" "Enable clustering on a single non-clustered server\n" "\n" " This command turns a non-clustered server into the first member of a new\n" " cluster, which will have the given name.\n" "\n" " It's required that the server is already available on the network. You can " "check\n" " that by running 'incus config get core.https_address', and possibly set a " "value\n" " for the address if not yet set." msgstr "" #: cmd/incus/top.go:281 msgid "" "Enter a sorting type ('a' for alphabetical, 'c' for CPU, 'm' for memory, 'd' " "for disk):" msgstr "" "Stellen Sie eine Sortiermethode ein ('a' für Alphabetisch, 'c' für CPU/" "Prozessor, 'm' für Memory/RAM, 'd' für Festplatte (disk):" #: cmd/incus/top.go:262 msgid "Enter new delay in seconds:" msgstr "Legen Sie eine Verzögerungszeit angegeben in Sekunden fest:" #: cmd/incus/network_zone.go:1446 #, fuzzy msgid "Entry TTL" msgstr "Eingangs TTL" #: cmd/incus/exec.go:63 msgid "Environment variable to set (e.g. HOME=/home/foo)" msgstr "" "Festzulegende Umgebungsvariable (Environment variable) (z.B. HOME=/home/foo)" #: cmd/incus/copy.go:61 cmd/incus/create.go:65 #, fuzzy msgid "Ephemeral instance" msgstr "Kurzlebiger Container" #: cmd/incus/cluster.go:1725 cmd/incus/cluster.go:1919 #, c-format msgid "Error connecting to existing cluster member %q: %v" msgstr "Fehler beim Verbinden mit bestehendem Clustermitglied %q: %v" #: cmd/incus/utils_properties.go:223 #, c-format msgid "Error creating decoder: %v" msgstr "Fehler beim Erstellen des Decoders: %v" #: cmd/incus/utils_properties.go:228 #, c-format msgid "Error decoding data: %v" msgstr "Fehler beim Dekodieren der Datei(en): %v" #: cmd/incus/publish.go:202 cmd/incus/utils.go:206 #, c-format msgid "Error retrieving aliases: %w" msgstr "Fehler beim Abrufen des Aliase/Namen: %w" #: cmd/incus/cluster.go:556 cmd/incus/cluster_group.go:923 #: cmd/incus/config.go:578 cmd/incus/config.go:610 cmd/incus/network.go:1479 #: cmd/incus/network_acl.go:497 cmd/incus/network_address_set.go:369 #: cmd/incus/network_forward.go:538 cmd/incus/network_integration.go:605 #: cmd/incus/network_load_balancer.go:524 cmd/incus/network_peer.go:560 #: cmd/incus/network_zone.go:511 cmd/incus/network_zone.go:1169 #: cmd/incus/profile.go:1031 cmd/incus/project.go:809 cmd/incus/storage.go:845 #: cmd/incus/storage_bucket.go:664 cmd/incus/storage_volume.go:1871 #: cmd/incus/storage_volume.go:1914 #, fuzzy, c-format msgid "Error setting properties: %v" msgstr "Fehler beim Festlegen der Parameter: %v" #: cmd/incus/console_windows.go:27 #, fuzzy, c-format msgid "Error setting term size %s" msgstr "Fehler beim setzen der Größe des Terminals/der Konsole %s" #: cmd/incus/config.go:572 cmd/incus/config.go:604 #, c-format msgid "Error unsetting properties: %v" msgstr "Fehler beim Löschen der Parameter: %v" #: cmd/incus/cluster.go:550 cmd/incus/cluster_group.go:917 #: cmd/incus/network.go:1473 cmd/incus/network_acl.go:491 #: cmd/incus/network_forward.go:532 cmd/incus/network_integration.go:599 #: cmd/incus/network_load_balancer.go:518 cmd/incus/network_peer.go:554 #: cmd/incus/network_zone.go:505 cmd/incus/network_zone.go:1163 #: cmd/incus/profile.go:1025 cmd/incus/project.go:803 cmd/incus/storage.go:839 #: cmd/incus/storage_bucket.go:658 cmd/incus/storage_volume.go:1865 #: cmd/incus/storage_volume.go:1908 #, c-format msgid "Error unsetting property: %v" msgstr "Fehler beim Löschen des Parameters: %v" #: cmd/incus/config_template.go:232 #, c-format msgid "Error updating template file: %s" msgstr "Fehler beim Aktualisieren der Template Datei: %s" #: cmd/incus/main.go:413 #, c-format msgid "Error while executing alias expansion: %s\n" msgstr "" #: cmd/incus/color/color.go:34 #, fuzzy msgid "Error:" msgstr "Fehler: %v\n" #: cmd/incus/cluster.go:1456 cmd/incus/cluster.go:1457 #, fuzzy msgid "Evacuate cluster member" msgstr "Evakuieren eines Clustermitglieds" #: cmd/incus/cluster.go:1544 #, fuzzy, c-format msgid "Evacuating cluster member: %s" msgstr "Evakuieren der Clustermitglieder: %s" #: cmd/incus/monitor.go:56 msgid "Event type to listen for" msgstr "Ereignistyp der beobachtet werden soll" #: cmd/incus/color/color.go:40 msgid "Examples:" msgstr "" #: cmd/incus/admin_sql.go:29 msgid "Execute a SQL query against the local or global database" msgstr "Eine SQL Abfrage bei der lokalen oder globalen Datenbank durchführen" #: cmd/incus/admin_sql.go:30 #, fuzzy msgid "" "Execute a SQL query against the local or global database\n" "\n" " The local database is specific to the cluster member you target the\n" " command to, and contains member-specific data (such as the member network\n" " address).\n" "\n" " The global database is common to all members in the cluster, and contains\n" " cluster-specific data (such as profiles, containers, etc).\n" "\n" " Non-clustered servers still have both local and global databases.\n" "\n" " If is the special value \"-\", then the query is read from\n" " standard input.\n" "\n" " If is the special value \".dump\", the command returns a SQL text\n" " dump of the given database.\n" "\n" " If is the special value \".schema\", the command returns the SQL\n" " text schema of the given database.\n" "\n" " If is the special value \".tables\", the command returns the SQL\n" " text tables of the given database.\n" "\n" " This internal command is mostly useful for debugging and disaster\n" " recovery. The development team will occasionally provide hotfixes to users " "as a\n" " set of database queries to fix some data inconsistency." msgstr "" "Eine SQL Abfrage bei der lokalen oder globalen Datenbank durchführen:\n" "\n" " Die lokale Datebank existiert speziell für das Clustermitglied zu dem Sie " "den Befehl senden \n" " und enthält Mitglieds-spezifische Daten (z.b. die Netzwerkadresse des " "Mitglieds). \n" "\n" " Die globale Datenbank teilen sich alle Mitglieder eines Clusters \n" " und sie enthält Cluster-spezifische Daten (z.b. Profile, Container, " "usw.). \n" "\n" " Wenn Sie einen Server ohne Cluster betreiben, gilt dasgleiche, \n" " denn der Server ist dann praktisch ein Cluster mit nur einem Mitglied. \n" "\n" " Wenn dem Befehl \"-\" entspricht, dann wird \n" " die Abfrage vom Standard Input gelesen. \n" "\n" " Wenn dem Befehl \".dump\" entspricht, dann wird ein SQL Text\n" " Auszug der angegebenen Datenbank angezeigt. \n" "\n" " Wenn dem Befehl \".schema\" entspricht, dann wird ein SQL\n" " Text Schema der angegebenen Datenbank angezeigt. \n" "\n" " Dieser interne Befehl ist hauptsächlich nützlich für die Fehlersuche " "(Debugging) \n" " und die Wiederherstellung nach einem schweren Fehler oder Absturz. \n" " Das Entwicklungsteam wird von Zeit zu Zeit hotfixes als Datenbankabfrage-" "Sets \n" " bereitstellen um Dateiunstimmigkeiten zu lösen. \n" "\n" " Dieser Befehl greift standardmäßig auf die globale Datenbank zu (wenn " "nicht anders angegeben) \n" " und funktioniert sowohl lokal als auch im Cluster Modus." #: cmd/incus/exec.go:44 msgid "Execute commands in instances" msgstr "Befehle in einer Instanz ausführen" #: cmd/incus/exec.go:45 #, fuzzy msgid "" "Execute commands in instances\n" "\n" "The command is executed directly using exec, so there is no shell and\n" "shell patterns (variables, file redirects, ...) won't be understood.\n" "If you need a shell environment you need to execute the shell\n" "executable, passing the shell commands as arguments, for example:\n" "\n" " incus exec -- sh -c \"cd /tmp && pwd\"\n" "\n" "Mode defaults to non-interactive, interactive mode is selected if both stdin " "AND stdout are terminals (stderr is ignored)." msgstr "" "Befehle in einer Instanz ausführen\n" "\n" "Der jeweils angegebene Befehl wird standardmäßig direkt ausgeführt mithilfe " "des Befehls \"exec\"; es gibt also normalerweise keine Konsole (Shell) und " "Shell Parameter werden nicht erkannt.\n" "Wenn Sie stattdessen eine Konsole (Shell Umgebung) brauchen oder wollen, " "dann führen Sie den Shell Befehl als Teil des angegebenen Befehls aus und " "übergeben die restlichen Befehle als Shell Argumente, wie im folgenden " "Beispiel:\n" "\n" " incus exec -- sh -c \"cd /tmp && pwd\"\n" "\n" "Der Modus wird standardmäßig nicht-interaktiv ausgeführt, der interaktive " "Modus wird nur aktiviert, wenn sowohl \"stdin\" als auch \"stdout\" " "Terminals sind (stderr wird ignoriert)." #: cmd/incus/utils_properties.go:112 #, c-format msgid "Expected a struct, got a %v" msgstr "" #: cmd/incus/info.go:841 cmd/incus/info.go:892 cmd/incus/storage_volume.go:1356 #: cmd/incus/storage_volume.go:1406 #, fuzzy msgid "Expires at" msgstr "Wird gelöscht am" #: cmd/incus/image.go:979 #, fuzzy, c-format msgid "Expires: %s" msgstr "Wird gelöscht:" #: cmd/incus/image.go:981 #, fuzzy msgid "Expires: never" msgstr "Wird gelöscht: nie" #: cmd/incus/snapshot.go:98 cmd/incus/storage_volume.go:3053 msgid "Expiry date or time span for the new snapshot" msgstr "" #: cmd/incus/debug.go:45 #, fuzzy msgid "Export a virtual machine's memory state" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/image.go:517 msgid "Export and download images" msgstr "Images exportieren und herunterladen" #: cmd/incus/image.go:518 msgid "" "Export and download images\n" "\n" "The output target is optional and defaults to the working directory." msgstr "" "Images exportieren und herunterladen\n" "\n" "Die Angabe eines Ausgabeziels (target) ist optional und wenn nichts " "angegeben wird, wird die resultierende Datei im derzeit aktiven Ordner/Pfad " "gespeichert." #: cmd/incus/storage_volume.go:3643 #, fuzzy msgid "Export custom storage volumes" msgstr "Storage Volume exportieren" #: cmd/incus/storage_volume.go:3644 #, fuzzy msgid "Export custom storage volumes." msgstr "Storage Volume exportieren" #: cmd/incus/export.go:38 msgid "Export instance backups" msgstr "Backups einer Instanz exportieren" #: cmd/incus/export.go:39 #, fuzzy msgid "Export instances as backup tarballs." msgstr "Instanzen als Backup Dateien (.tar) exportieren." #: cmd/incus/storage_bucket.go:1291 msgid "Export storage bucket" msgstr "Storage Buckets exportieren" #: cmd/incus/storage_bucket.go:1292 #, fuzzy msgid "Export storage buckets as tarball." msgstr "Storage Buckets als Dateien (.tar) exportieren." #: cmd/incus/debug.go:46 msgid "" "Export the current memory state of a running virtual machine into a dump " "file.\n" "\t\tThis can be useful for debugging or analysis purposes." msgstr "" #: cmd/incus/storage_volume.go:3646 #, fuzzy msgid "" "Export the volume without its snapshots (ignored for ISO storage volumes)" msgstr "Storage Volume ohne Snapshots exportieren" #: cmd/incus/storage_bucket.go:1416 #, fuzzy, c-format msgid "Exporting backup of storage bucket: %s" msgstr "Backup des Storage Bucket %s wird exportiert" #: cmd/incus/export.go:167 cmd/incus/storage_volume.go:3792 #, fuzzy, c-format msgid "Exporting the backup: %s" msgstr "Backup %s wird exportiert" #: cmd/incus/image.go:589 #, c-format msgid "Exporting the image: %s" msgstr "" #: cmd/incus/cluster.go:199 msgid "FAILURE DOMAIN" msgstr "" #: cmd/incus/config_template.go:314 msgid "FILENAME" msgstr "DATEINAME" #: cmd/incus/config_trust.go:416 cmd/incus/image.go:1100 #: cmd/incus/image.go:1101 cmd/incus/image_alias.go:247 #, fuzzy msgid "FINGERPRINT" msgstr "FINGERABDRUCK" #: cmd/incus/warning.go:211 #, fuzzy msgid "FIRST SEEN" msgstr "ZUERST BEOBACHTET" #: cmd/incus/info.go:693 cmd/incus/remote.go:120 msgid "FQDN" msgstr "" #: cmd/incus/utils.go:687 #, fuzzy, c-format msgid "Failed SSH handshake with client %q: %v" msgstr "SSH handshake mit Client %q gescheitert: %v" #: cmd/incus/utils.go:710 #, c-format msgid "Failed accepting channel client %q: %v" msgstr "" #: cmd/incus/delete.go:92 #, fuzzy, c-format msgid "Failed checking instance %s exists: %w" msgstr "Check ob Instanz existiert fehlgeschlagen \"%s:%s\": %w" #: cmd/incus/utils.go:377 #, fuzzy, c-format msgid "Failed checking instance exists \"%s:%s\": %w" msgstr "Check ob Instanz existiert fehlgeschlagen \"%s:%s\": %w" #: cmd/incus/utils.go:369 #, fuzzy, c-format msgid "Failed checking instance snapshot exists \"%s:%s\": %w" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/utils.go:737 #, c-format msgid "Failed connecting to instance SFTP for client %q: %v" msgstr "" #: cmd/incus/file.go:1000 cmd/incus/storage_volume.go:2464 #: cmd/incus/storage_volume.go:2652 cmd/incus/storage_volume.go:2815 #, fuzzy, c-format msgid "Failed connecting to instance SFTP: %w" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/admin_waitready.go:70 #, fuzzy, c-format msgid "Failed connecting to the daemon (attempt %d): %v" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/config_trust.go:137 #, c-format msgid "Failed converting token operation to certificate add token: %w" msgstr "" #: cmd/incus/cluster.go:1072 cmd/incus/cluster.go:1765 #, fuzzy, c-format msgid "Failed converting token operation to join token: %w" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/delete.go:142 #, fuzzy, c-format msgid "Failed deleting instance %s in project %q: %w" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/storage_volume.go:513 #, fuzzy, c-format msgid "Failed deleting source volume after copy: %w" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/utils.go:644 #, fuzzy, c-format msgid "Failed generating SSH host key: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/cluster.go:1867 #, fuzzy, c-format msgid "Failed generating trust certificate: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/admin_recover.go:58 #, fuzzy, c-format msgid "Failed getting existing storage pools: %w" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/network_peer.go:400 #, c-format msgid "Failed getting peer's status: %w" msgstr "" #: cmd/incus/admin_recover.go:248 #, fuzzy, c-format msgid "Failed import request: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/create.go:152 #, fuzzy, c-format msgid "Failed loading network %q: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/create.go:297 #, c-format msgid "Failed loading profile %q for device override: %w" msgstr "" #: cmd/incus/create.go:215 #, fuzzy, c-format msgid "Failed loading storage pool %q: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/utils.go:649 #, fuzzy, c-format msgid "Failed parsing SSH host key: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/admin_recover.go:194 #, fuzzy, c-format msgid "Failed parsing validation response: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/console.go:383 #, fuzzy, c-format msgid "Failed starting command: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/utils.go:557 #, fuzzy, c-format msgid "Failed starting sshfs: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/storage_volume.go:4042 cmd/incus/utils.go:675 #, fuzzy, c-format msgid "Failed to accept incoming connection: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/remote.go:228 msgid "Failed to add remote" msgstr "" #: cmd/incus/cluster.go:1877 #, fuzzy, c-format msgid "Failed to add server cert to cluster: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/image.go:283 #, fuzzy, c-format msgid "Failed to check for existing aliases: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/admin_waitready.go:85 #, c-format msgid "Failed to check if the daemon is ready (attempt %d): %v" msgstr "" #: cmd/incus/export.go:204 cmd/incus/storage_bucket.go:1453 #: cmd/incus/storage_volume.go:3829 #, fuzzy, c-format msgid "Failed to close export file: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/remote.go:284 #, fuzzy, c-format msgid "Failed to close server cert file %q: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/cluster.go:831 cmd/incus/cluster.go:836 #, fuzzy, c-format msgid "Failed to configure cluster: %w" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/admin_init.go:107 #, fuzzy, c-format msgid "Failed to connect to get server info: %w" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/admin_init.go:102 #, fuzzy, c-format msgid "Failed to connect to local daemon: %w" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/cluster.go:1862 #, fuzzy, c-format msgid "Failed to connect to target cluster node %q: %w" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/remote.go:274 #, c-format msgid "Failed to create %q: %w" msgstr "" #: cmd/incus/utils.go:179 #, fuzzy, c-format msgid "Failed to create alias %s: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/storage_bucket.go:1347 #, fuzzy, c-format msgid "Failed to create backup: %v" msgstr "Akzeptiere Zertifikat" #: cmd/incus/remote.go:299 #, fuzzy, c-format msgid "Failed to create certificate: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/storage_volume.go:3723 #, fuzzy, c-format msgid "Failed to create storage volume backup: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/move.go:197 #, fuzzy, c-format msgid "Failed to delete original instance after copying it: %w" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/debug.go:76 #, fuzzy, c-format msgid "Failed to dump instance memory: %w" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/storage_bucket.go:1430 #, fuzzy, c-format msgid "Failed to fetch storage bucket backup: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/storage_volume.go:3806 #, fuzzy, c-format msgid "Failed to fetch storage volume backup file: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/remote.go:306 #, c-format msgid "Failed to find project: %w" msgstr "" #: cmd/incus/cluster.go:1963 cmd/incus/cluster.go:1968 #, fuzzy, c-format msgid "Failed to join cluster: %w" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/storage_volume.go:4034 cmd/incus/utils.go:660 #, fuzzy, c-format msgid "Failed to listen for connection: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/main.go:133 cmd/incus/main_aliases.go:225 #, c-format msgid "Failed to load configuration: %s" msgstr "" #: cmd/incus/file.go:786 cmd/incus/storage_volume.go:2878 #: cmd/incus/utils_sftp.go:311 #, fuzzy, c-format msgid "Failed to open source file %q: %v" msgstr "Akzeptiere Zertifikat" #: cmd/incus/utils_sftp.go:74 #, fuzzy, c-format msgid "Failed to open target file %q: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/admin_sql.go:103 #, fuzzy, c-format msgid "Failed to parse dump response: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/admin_init_preseed.go:39 #, fuzzy, c-format msgid "Failed to parse the preseed: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/admin_init_preseed.go:30 #, fuzzy, c-format msgid "Failed to read from file: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/admin_init_preseed.go:25 cmd/incus/admin_sql.go:80 #, fuzzy, c-format msgid "Failed to read from stdin: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/copy.go:362 #, fuzzy, c-format msgid "Failed to refresh target instance '%s': %v" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/image.go:289 cmd/incus/utils.go:168 #, fuzzy, c-format msgid "Failed to remove alias %s: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/export.go:198 cmd/incus/storage_bucket.go:1447 #: cmd/incus/storage_volume.go:3823 #, fuzzy, c-format msgid "Failed to rename export file: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/admin_init_interactive.go:88 #, fuzzy, c-format msgid "Failed to render the config: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/admin_sql.go:97 #, fuzzy, c-format msgid "Failed to request dump: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/cluster.go:1826 #, fuzzy, c-format msgid "Failed to retrieve cluster information: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/cluster.go:818 #, fuzzy, c-format msgid "Failed to retrieve current cluster config: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/admin_init_dump.go:18 cmd/incus/admin_init_dump.go:49 #: cmd/incus/admin_init_dump.go:64 cmd/incus/admin_init_dump.go:79 #: cmd/incus/admin_init_dump.go:93 cmd/incus/cluster.go:808 #, fuzzy, c-format msgid "Failed to retrieve current server configuration: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/admin_init_dump.go:28 #, fuzzy, c-format msgid "" "Failed to retrieve current server network configuration for project %q: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/admin_init_auto.go:131 #, fuzzy, c-format msgid "Failed to retrieve list of networks: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/admin_init_auto.go:47 #, fuzzy, c-format msgid "Failed to retrieve list of storage pools: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/cluster.go:1800 #, fuzzy, c-format msgid "Failed to setup trust relationship with cluster: %w" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/cluster.go:1536 #, fuzzy, c-format msgid "Failed to update cluster member state: %w" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/utils_sftp.go:275 #, c-format msgid "Failed to walk path for %s: %s" msgstr "" #: cmd/incus/remote.go:279 #, fuzzy, c-format msgid "Failed to write server cert file %q: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/admin_recover.go:187 #, fuzzy, c-format msgid "Failed validation request: %w" msgstr "Akzeptiere Zertifikat" #: cmd/incus/info.go:416 #, c-format msgid "Family: %v" msgstr "" #: cmd/incus/list.go:142 msgid "Fast mode (same as --columns=nsacPt)" msgstr "" #: cmd/incus/export.go:181 #, fuzzy, c-format msgid "Fetch instance backup file: %w" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/image.go:965 #, fuzzy, c-format msgid "Fingerprint: %s" msgstr "Fingerabdruck: %s\n" #: cmd/incus/color/color.go:42 msgid "Flags:" msgstr "" #: cmd/incus/file.go:449 cmd/incus/file.go:680 cmd/incus/storage_volume.go:2590 #: cmd/incus/storage_volume.go:2788 msgid "Follow command-line symbolic links in source path" msgstr "" #: cmd/incus/cluster.go:1459 msgid "Force a particular evacuation action" msgstr "" #: cmd/incus/cluster.go:1488 msgid "Force a particular restoration action" msgstr "" #: cmd/incus/file.go:109 cmd/incus/storage_volume.go:2166 msgid "Force creating files or directories" msgstr "" #: cmd/incus/project.go:208 msgid "Force delete the project and everything it contains." msgstr "" #: cmd/incus/file.go:276 cmd/incus/storage_volume.go:2332 msgid "Force deleting files, directories, and subdirectories" msgstr "" #: cmd/incus/cluster.go:1504 msgid "Force evacuation without user confirmation" msgstr "" #: cmd/incus/export.go:53 cmd/incus/storage_bucket.go:1300 #: cmd/incus/storage_volume.go:3650 msgid "Force overwriting existing backup file" msgstr "" #: cmd/incus/exec.go:65 msgid "Force pseudo-terminal allocation" msgstr "" #: cmd/incus/cluster.go:690 msgid "Force removing a member, even if degraded" msgstr "" #: cmd/incus/action.go:165 #, fuzzy msgid "Force the instance to stop" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/delete.go:40 msgid "Force the removal of running instances" msgstr "" #: cmd/incus/main.go:161 msgid "Force using the local unix socket" msgstr "" #: cmd/incus/cluster.go:706 #, c-format msgid "" "Forcefully removing a server from the cluster should only be done as a last\n" "resort.\n" "\n" "The removed server will not be functional after this action and will require " "a\n" "full reset, losing any remaining instance, image or storage volume that\n" "the server may have held.\n" "\n" "When possible, a graceful removal should be preferred, this will require you " "to\n" "move any affected instance, image or storage volume to another server prior " "to\n" "the server being cleanly removed from the cluster.\n" "\n" "The --force flag should only be used if the server has died, been " "reinstalled\n" "or is otherwise never expected to come back up.\n" "\n" "Are you really sure you want to force removing %s? (yes/no): " msgstr "" #: cmd/incus/console.go:52 msgid "" "Forces a connection to the console, even if there is already an active " "session" msgstr "" #: cmd/incus/network_address_set.go:100 msgid "Format (csv|json|table|yaml|compact|markdown)" msgstr "" #: cmd/incus/cluster.go:1123 msgid "" "Format (csv|json|table|yaml|compact|markdown), use suffix \",noheader\" to " "disable headers and \",header\" to enable if demanded, e.g. csv,header" msgstr "" #: cmd/incus/admin_sql.go:57 cmd/incus/alias.go:111 cmd/incus/cluster.go:171 #: cmd/incus/cluster_group.go:440 cmd/incus/config_template.go:271 #: cmd/incus/config_trust.go:400 cmd/incus/config_trust.go:584 #: cmd/incus/image.go:1070 cmd/incus/image_alias.go:208 cmd/incus/list.go:141 #: cmd/incus/network.go:1045 cmd/incus/network.go:1241 #: cmd/incus/network_acl.go:104 cmd/incus/network_allocations.go:64 #: cmd/incus/network_forward.go:120 cmd/incus/network_integration.go:411 #: cmd/incus/network_load_balancer.go:127 cmd/incus/network_peer.go:116 #: cmd/incus/network_zone.go:120 cmd/incus/network_zone.go:819 #: cmd/incus/operation.go:147 cmd/incus/profile.go:698 cmd/incus/project.go:518 #: cmd/incus/project.go:993 cmd/incus/remote.go:970 cmd/incus/snapshot.go:341 #: cmd/incus/storage.go:661 cmd/incus/storage_bucket.go:462 #: cmd/incus/storage_bucket.go:855 cmd/incus/storage_volume.go:1466 #: cmd/incus/storage_volume.go:3297 cmd/incus/warning.go:99 msgid "" "Format (csv|json|table|yaml|compact|markdown), use suffix \",noheader\" to " "disable headers and \",header\" to enable it if missing, e.g. csv,header" msgstr "" #: cmd/incus/monitor.go:58 msgid "Format (json|pretty|yaml)" msgstr "" #: cmd/incus/manpage.go:31 msgid "Format (man|md|rest|yaml)" msgstr "" #: cmd/incus/remote.go:752 msgid "Format (pem|pfx)" msgstr "" #: cmd/incus/top.go:68 msgid "Format (table|compact)" msgstr "" #: cmd/incus/debug.go:54 msgid "" "Format of memory dump (e.g. elf, win-dmp, kdump-zlib, kdump-raw-zlib, ...)" msgstr "" #: cmd/incus/network.go:957 msgid "Forward delay" msgstr "" #: cmd/incus/main_aliases.go:151 #, c-format msgid "Found alias %q references an argument outside the given number" msgstr "" #: cmd/incus/info.go:516 cmd/incus/info.go:527 cmd/incus/info.go:532 #: cmd/incus/info.go:538 #, c-format msgid "Free: %v" msgstr "" #: cmd/incus/info.go:330 cmd/incus/info.go:341 #, c-format msgid "Frequency: %vMhz" msgstr "" #: cmd/incus/info.go:339 #, c-format msgid "Frequency: %vMhz (min: %vMhz, max: %vMhz)" msgstr "" #: cmd/incus/remote.go:990 msgid "GLOBAL" msgstr "" #: cmd/incus/info.go:544 msgid "GPU:" msgstr "" #: cmd/incus/info.go:547 msgid "GPUs:" msgstr "" #: cmd/incus/remote.go:858 #, fuzzy msgid "Generate a client token derived from the client certificate" msgstr "Akzeptiere Zertifikat" #: cmd/incus/remote.go:859 msgid "" "Generate a client trust token derived from the existing client certificate " "and private key.\n" "\n" "This is useful for remote authentication workflows where a token is passed " "to another Incus server." msgstr "" #: cmd/incus/image.go:1729 msgid "Generate a metadata tarball" msgstr "" #: cmd/incus/image.go:1731 msgid "" "Generate a metadata tarball\n" "\n" "This command produces an incus.tar.xz tarball for use during import with an " "existing QCOW2 or squashfs disk image.\n" "\n" "This command will prompt for all of the metadata tarball fields:\n" " - Operating system name\n" " - Release\n" " - Variant\n" " - Architecture\n" " - Description\n" msgstr "" #: cmd/incus/manpage.go:27 cmd/incus/manpage.go:28 msgid "Generate manpages for all commands" msgstr "" #: cmd/incus/remote.go:671 #, fuzzy msgid "Generate the client certificate" msgstr "Akzeptiere Zertifikat" #: cmd/incus/remote.go:196 cmd/incus/remote.go:435 cmd/incus/remote.go:694 #: cmd/incus/remote.go:778 cmd/incus/remote.go:883 #, fuzzy msgid "Generating a client certificate. This may take a minute..." msgstr "Generiere Nutzerzertifikat. Dies kann wenige Minuten dauern...\n" #: cmd/incus/project.go:989 cmd/incus/project.go:990 msgid "Get a summary of resource allocations" msgstr "" #: cmd/incus/network_load_balancer.go:1130 #, fuzzy msgid "Get current load balancer status" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_load_balancer.go:1131 #, fuzzy msgid "Get current load-balancer status" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/image.go:1524 cmd/incus/image.go:1525 msgid "Get image properties" msgstr "" #: cmd/incus/network.go:867 cmd/incus/network.go:868 msgid "Get runtime information on networks" msgstr "" #: cmd/incus/cluster_group.go:810 #, fuzzy msgid "Get the key as a cluster group property" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/cluster.go:443 msgid "Get the key as a cluster property" msgstr "" #: cmd/incus/network_acl.go:290 msgid "Get the key as a network ACL property" msgstr "" #: cmd/incus/network_forward.go:404 #, fuzzy msgid "Get the key as a network forward property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_integration.go:339 #, fuzzy msgid "Get the key as a network integration property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_load_balancer.go:408 #, fuzzy msgid "Get the key as a network load balancer property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_peer.go:433 #, fuzzy msgid "Get the key as a network peer property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network.go:799 msgid "Get the key as a network property" msgstr "" #: cmd/incus/network_zone.go:307 #, fuzzy msgid "Get the key as a network zone property" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/network_zone.go:952 #, fuzzy msgid "Get the key as a network zone record property" msgstr "Profil %s erstellt\n" #: cmd/incus/profile.go:610 msgid "Get the key as a profile property" msgstr "" #: cmd/incus/project.go:434 msgid "Get the key as a project property" msgstr "" #: cmd/incus/storage_bucket.go:372 #, fuzzy msgid "Get the key as a storage bucket property" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/storage.go:394 #, fuzzy msgid "Get the key as a storage property" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/storage_volume.go:1115 #, fuzzy msgid "Get the key as a storage volume property" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/config.go:381 msgid "Get the key as an instance property" msgstr "" #: cmd/incus/cluster_group.go:807 #, fuzzy msgid "Get values for cluster group configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/cluster.go:440 #, fuzzy msgid "Get values for cluster member configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/config_device.go:210 cmd/incus/config_device.go:211 #, fuzzy msgid "Get values for device configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/config.go:376 cmd/incus/config.go:377 #, fuzzy msgid "Get values for instance or server configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_acl.go:287 cmd/incus/network_acl.go:288 #, fuzzy msgid "Get values for network ACL configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network.go:794 cmd/incus/network.go:795 msgid "Get values for network configuration keys" msgstr "" #: cmd/incus/network_forward.go:401 cmd/incus/network_forward.go:402 #, fuzzy msgid "Get values for network forward configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_integration.go:334 cmd/incus/network_integration.go:335 #, fuzzy msgid "Get values for network integration configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_load_balancer.go:404 #: cmd/incus/network_load_balancer.go:405 #, fuzzy msgid "Get values for network load balancer configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_peer.go:429 cmd/incus/network_peer.go:430 #, fuzzy msgid "Get values for network peer configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_zone.go:303 cmd/incus/network_zone.go:304 #, fuzzy msgid "Get values for network zone configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_zone.go:948 cmd/incus/network_zone.go:949 #, fuzzy msgid "Get values for network zone record configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/profile.go:604 cmd/incus/profile.go:605 msgid "Get values for profile configuration keys" msgstr "" #: cmd/incus/project.go:429 cmd/incus/project.go:430 #, fuzzy msgid "Get values for project configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_bucket.go:368 cmd/incus/storage_bucket.go:369 #, fuzzy msgid "Get values for storage bucket configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/storage.go:389 cmd/incus/storage.go:390 msgid "Get values for storage pool configuration keys" msgstr "" #: cmd/incus/storage_volume.go:1099 msgid "Get values for storage volume configuration keys" msgstr "" #: cmd/incus/storage_volume.go:1100 #, fuzzy msgid "" "Get values for storage volume configuration keys\n" "\n" "If the type is not specified, incus assumes the type is \"custom\".\n" "Supported values for type are \"custom\", \"container\" and \"virtual-" "machine\".\n" "\n" "For snapshots, add the snapshot name (only if type is one of custom, " "container or virtual-machine)." msgstr "Profil %s erstellt\n" #: cmd/incus/storage_volume.go:3987 msgid "Get write access to the disk" msgstr "" #: cmd/incus/storage_volume.go:413 #, c-format msgid "Given target %q does not match source volume location %q" msgstr "" #: cmd/incus/color/color.go:43 msgid "Global Flags:" msgstr "" #: cmd/incus/exec.go:69 msgid "Group ID to run the command as (default 0)" msgstr "" #: cmd/incus/network.go:1265 msgid "HOSTNAME" msgstr "" #: cmd/incus/info.go:767 #, fuzzy msgid "Host interface" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/info.go:692 #, fuzzy msgid "Hostname" msgstr "Name" #: cmd/incus/info.go:515 cmd/incus/info.go:526 msgid "Hugepages:\n" msgstr "" #: cmd/incus/utils.go:760 #, fuzzy, c-format msgid "I/O copy from SSH to instance failed: %v" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/utils.go:749 #, fuzzy, c-format msgid "I/O copy from instance to SSH failed: %v" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/utils.go:581 #, fuzzy, c-format msgid "I/O copy from instance to sshfs failed: %v" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/utils.go:591 #, fuzzy, c-format msgid "I/O copy from sshfs to instance failed: %v" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/network.go:955 cmd/incus/operation.go:164 msgid "ID" msgstr "" #: cmd/incus/info.go:124 #, c-format msgid "ID: %d" msgstr "" #: cmd/incus/info.go:212 cmd/incus/info.go:279 cmd/incus/info.go:303 #, c-format msgid "ID: %s" msgstr "" #: cmd/incus/project.go:542 msgid "IMAGES" msgstr "" #: cmd/incus/top.go:83 msgid "INSTANCE NAME" msgstr "" #: cmd/incus/info.go:365 #, c-format msgid "IOMMU group: %v" msgstr "" #: cmd/incus/remote.go:120 msgid "IP" msgstr "" #: cmd/incus/network.go:1267 msgid "IP ADDRESS" msgstr "" #: cmd/incus/info.go:783 #, fuzzy msgid "IP addresses" msgstr "Profil %s erstellt\n" #: cmd/incus/network.go:922 #, fuzzy msgid "IP addresses:" msgstr "Profil %s erstellt\n" #: cmd/incus/list.go:471 cmd/incus/network.go:1074 msgid "IPV4" msgstr "" #: cmd/incus/list.go:472 cmd/incus/network.go:1075 msgid "IPV6" msgstr "" #: cmd/incus/network.go:989 #, fuzzy msgid "IPv4 uplink address" msgstr "Profil %s erstellt\n" #: cmd/incus/network.go:993 #, fuzzy msgid "IPv6 uplink address" msgstr "Profil %s erstellt\n" #: cmd/incus/config_trust.go:418 msgid "ISSUE DATE" msgstr "" #: cmd/incus/info.go:370 #, fuzzy, c-format msgid "Id: %v" msgstr "Erstellt: %s" #: cmd/incus/image.go:175 msgid "If an alias already exists, delete and recreate it" msgstr "" #: cmd/incus/rebuild.go:33 msgid "If an instance is running, stop it and then rebuild it" msgstr "" #: cmd/incus/main.go:167 msgid "" "If the command is valid, explain its parsed arguments instead of running it" msgstr "" #: cmd/incus/image.go:696 cmd/incus/publish.go:46 msgid "If the image alias already exists, delete and create a new one" msgstr "" #: cmd/incus/snapshot.go:100 cmd/incus/storage_volume.go:3055 msgid "If the snapshot name already exists, delete and create a new one" msgstr "" #: cmd/incus/main.go:521 msgid "" "If this is your first time running Incus on this machine, you should also " "run: incus admin init" msgstr "" #: cmd/incus/snapshot.go:99 msgid "Ignore any configured auto-expiry for the instance" msgstr "" #: cmd/incus/storage_volume.go:3054 msgid "Ignore any configured auto-expiry for the storage volume" msgstr "" #: cmd/incus/copy.go:71 cmd/incus/move.go:69 msgid "Ignore copy errors for volatile files" msgstr "" #: cmd/incus/action.go:157 #, fuzzy msgid "Ignore the instance state" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/image_alias.go:76 msgid "Image alias description" msgstr "" #: cmd/incus/image.go:1440 msgid "Image already up to date." msgstr "" #: cmd/incus/image.go:319 msgid "Image copied successfully!" msgstr "" #: cmd/incus/publish.go:45 msgid "Image expiration date (format: rfc3339)" msgstr "" #: cmd/incus/image.go:665 msgid "Image exported successfully!" msgstr "" #: cmd/incus/publish.go:47 #, fuzzy msgid "Image format" msgstr "Ungültiges Ziel %s" #: cmd/incus/image.go:859 #, fuzzy, c-format msgid "Image imported with fingerprint: %s" msgstr "Abbild mit Fingerabdruck %s importiert\n" #: cmd/incus/image.go:1438 msgid "Image refreshed successfully!" msgstr "" #: cmd/incus/action.go:161 cmd/incus/launch.go:46 #, fuzzy msgid "Immediately attach to the console" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/import.go:32 msgid "Import backups of instances including their snapshots." msgstr "" #: cmd/incus/storage_bucket.go:1473 #, fuzzy msgid "Import backups of storage buckets." msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/storage_volume.go:3850 #, fuzzy msgid "Import custom storage volumes" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/storage_volume.go:3851 #, fuzzy msgid "Import custom storage volumes." msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/image.go:690 msgid "" "Import image into the image store\n" "\n" "Directory import is only available on Linux and must be performed as root." msgstr "" #: cmd/incus/image.go:689 msgid "Import images into the image store" msgstr "" #: cmd/incus/import.go:31 #, fuzzy msgid "Import instance backups" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/storage_bucket.go:1472 #, fuzzy msgid "Import storage bucket" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_volume.go:3916 msgid "Import type needs to be \"backup\" or \"iso\"" msgstr "" #: cmd/incus/storage_volume.go:3859 msgid "Import type, backup or iso (default \"backup\")" msgstr "" #: cmd/incus/storage_bucket.go:1518 #, fuzzy, c-format msgid "Importing bucket: %s" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/storage_volume.go:3925 #, fuzzy, c-format msgid "Importing custom volume: %s" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/import.go:74 #, fuzzy, c-format msgid "Importing instance: %s" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/create.go:66 msgid "Include environment variables from file" msgstr "" #: cmd/incus/manpage.go:32 msgid "Include less common commands" msgstr "" #: cmd/incus/manpage.go:70 msgid "Incus - Command line client" msgstr "" #: cmd/incus/info.go:241 msgid "Infiniband:" msgstr "" #: cmd/incus/query.go:49 msgid "Input data" msgstr "" #: cmd/incus/info.go:893 msgid "Instance Only" msgstr "" #: cmd/incus/create.go:74 #, fuzzy msgid "Instance description" msgstr "Entferntes Administrator Passwort" #: cmd/incus/utils.go:583 #, fuzzy msgid "Instance disconnected" msgstr "Entferntes Administrator Passwort" #: cmd/incus/utils.go:751 #, c-format msgid "Instance disconnected for client %q" msgstr "" #: cmd/incus/create.go:406 #, c-format msgid "Instance name is: %s" msgstr "" #: cmd/incus/file.go:987 msgid "Instance path cannot be used in SSH SFTP listener mode" msgstr "" #: cmd/incus/publish.go:290 #, fuzzy, c-format msgid "Instance published with fingerprint: %s" msgstr "Abbild mit Fingerabdruck %s importiert\n" #: cmd/incus/snapshot.go:209 #, fuzzy, c-format msgid "Instance snapshot name is: %s" msgstr "'/' ist kein gültiges Zeichen im Namen eines Sicherungspunktes\n" #: cmd/incus/create.go:69 msgid "Instance type" msgstr "" #: cmd/incus/cluster.go:1662 #, fuzzy msgid "Invalid IP address or DNS name" msgstr "Ungültige Quelle %s" #: cmd/incus/export.go:131 cmd/incus/storage_bucket.go:1380 #: cmd/incus/storage_volume.go:3756 #, fuzzy, c-format msgid "Invalid URL %q: %w" msgstr "Ungültiges Ziel %s" #: cmd/incus/remote.go:387 #, c-format msgid "Invalid URL scheme \"%s\" in \"%s\"" msgstr "" #: cmd/incus/main_aliases.go:147 cmd/incus/main_aliases.go:189 #, fuzzy, c-format msgid "Invalid argument %q" msgstr "Ungültiges Ziel %s" #: cmd/incus/export.go:136 cmd/incus/storage_bucket.go:1385 #: cmd/incus/storage_volume.go:3761 #, c-format msgid "Invalid backup name segment in path %q: %w" msgstr "" #: cmd/incus/utils_properties.go:56 #, fuzzy, c-format msgid "Invalid boolean value: %s" msgstr "Ungültige Quelle %s" #: cmd/incus/config_trust.go:516 #, fuzzy msgid "Invalid certificate" msgstr "Akzeptiere Zertifikat" #: cmd/incus/remote.go:768 #, fuzzy, c-format msgid "Invalid certificate format %q" msgstr "Akzeptiere Zertifikat" #: cmd/incus/cluster.go:1903 #, fuzzy, c-format msgid "Invalid cluster join token: %w" msgstr "Profil %s wurde auf %s angewandt\n" #: cmd/incus/list.go:571 #, c-format msgid "Invalid config key '%s' in '%s'" msgstr "" #: cmd/incus/list.go:564 #, c-format msgid "Invalid config key column format (too many fields): '%s'" msgstr "" #: cmd/incus/publish.go:194 #, fuzzy, c-format msgid "Invalid expiration date: %w" msgstr "Ungültige Quelle %s" #: cmd/incus/top.go:156 #, fuzzy, c-format msgid "Invalid format %q" msgstr "Ungültiges Ziel %s" #: cmd/incus/monitor.go:72 #, fuzzy, c-format msgid "Invalid format: %s" msgstr "Ungültiges Ziel %s" #: cmd/incus/top.go:273 msgid "Invalid input, please enter a positive number" msgstr "" #: cmd/incus/cluster.go:1702 #, fuzzy, c-format msgid "Invalid join token: %w" msgstr "Profil %s wurde auf %s angewandt\n" #: cmd/incus/utils.go:272 #, fuzzy, c-format msgid "Invalid key=value configuration: %s" msgstr "Alternatives config Verzeichnis." #: cmd/incus/list.go:592 #, c-format msgid "Invalid max width (must -1, 0 or a positive integer) '%s' in '%s'" msgstr "" #: cmd/incus/list.go:588 #, c-format msgid "Invalid max width (must be an integer) '%s' in '%s'" msgstr "" #: cmd/incus/list.go:578 #, c-format msgid "" "Invalid name in '%s', empty string is only allowed when defining maxWidth" msgstr "" #: cmd/incus/main.go:616 #, fuzzy msgid "Invalid number of arguments" msgstr "ungültiges Argument %s" #: cmd/incus/network_peer.go:350 #, fuzzy msgid "Invalid peer type" msgstr "Ungültiges Ziel %s" #: cmd/incus/remote.go:376 #, fuzzy, c-format msgid "Invalid protocol: %s" msgstr "Ungültiges Ziel %s" #: cmd/incus/top.go:351 #, fuzzy msgid "Invalid sorting type" msgstr "Ungültiges Ziel %s" #: cmd/incus/top.go:302 #, fuzzy msgid "Invalid sorting type provided" msgstr "Ungültiges Ziel %s" #: cmd/incus/file.go:141 cmd/incus/storage_volume.go:2199 #, fuzzy, c-format msgid "Invalid type %q" msgstr "Ungültiges Ziel %s" #: cmd/incus/info.go:244 #, fuzzy, c-format msgid "IsSM: %s (%s)" msgstr "" "Benutzung: %s\n" "\n" "Optionen:\n" "\n" #: cmd/incus/cluster.go:967 cmd/incus/cluster.go:968 #, fuzzy msgid "Join an existing server to a cluster" msgstr "Soll der Server einem bestehenden Cluster beitreten?" #: cmd/incus/cluster.go:1693 msgid "Joining an existing cluster requires root privileges" msgstr "" #: cmd/incus/image.go:176 msgid "Keep the image up to date after initial copy" msgstr "" #: cmd/incus/info.go:691 msgid "Kernel Version" msgstr "" #: cmd/incus/storage_bucket.go:986 #, fuzzy msgid "Key description" msgstr "Fingerabdruck: %s\n" #: cmd/incus/warning.go:212 msgid "LAST SEEN" msgstr "" #: cmd/incus/list.go:481 msgid "LAST USED AT" msgstr "" #: cmd/incus/project.go:1077 msgid "LIMIT" msgstr "" #: cmd/incus/network_forward.go:142 cmd/incus/network_load_balancer.go:149 msgid "LISTEN ADDRESS" msgstr "" #: cmd/incus/list.go:518 cmd/incus/network.go:1269 #: cmd/incus/network_forward.go:146 cmd/incus/network_load_balancer.go:152 #: cmd/incus/operation.go:170 cmd/incus/storage_bucket.go:482 #: cmd/incus/storage_volume.go:1576 cmd/incus/warning.go:221 msgid "LOCATION" msgstr "" #: cmd/incus/info.go:678 #, fuzzy, c-format msgid "Last Used: %s" msgstr "Erstellt: %s" #: cmd/incus/image.go:985 #, c-format msgid "Last used: %s" msgstr "" #: cmd/incus/image.go:987 msgid "Last used: never" msgstr "" #: cmd/incus/create.go:129 #, fuzzy, c-format msgid "Launching %s" msgstr "Erstelle %s" #: cmd/incus/create.go:131 #, fuzzy msgid "Launching the instance" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/info.go:235 #, fuzzy, c-format msgid "Link detected: %v" msgstr "Architektur: %s\n" #: cmd/incus/info.go:237 #, c-format msgid "Link speed: %dMbit/s (%s duplex)" msgstr "" #: cmd/incus/network.go:1219 msgid "List DHCP leases" msgstr "" #: cmd/incus/network.go:1220 msgid "" "List DHCP leases\n" "\n" "Default column layout: hmitL\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which network zone attributes to output when displaying in table or csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " h - Hostname\n" " m - MAC Address\n" " i - IP Address\n" " t - Type\n" " L - Location of the DHCP Lease (e.g. its cluster member)" msgstr "" #: cmd/incus/network_address_set.go:101 #, fuzzy msgid "List address sets across all projects" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/alias.go:109 cmd/incus/alias.go:110 #, fuzzy msgid "List aliases" msgstr "Aliasse:\n" #: cmd/incus/config_trust.go:564 #, fuzzy msgid "List all active certificate add tokens" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/config_trust.go:565 msgid "" "List all active certificate add tokens\n" "\n" "Default column layout: ntE\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which network zone attributes to output when displaying in table or csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " n - Name\n" " t - Token\n" " E - Expires At" msgstr "" #: cmd/incus/cluster.go:1103 #, fuzzy msgid "List all active cluster member join tokens" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/cluster.go:1104 msgid "" "List all active cluster member join tokens\n" "\n" "Default column layout: ntE\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which network zone attributes to output when displaying in table or csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " n - Name\n" " t - Token\n" " E - Expires At" msgstr "" #: cmd/incus/cluster_group.go:418 #, fuzzy msgid "List all the cluster groups" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/cluster_group.go:419 msgid "" "List all the cluster groups\n" "\n" "Default column layout: ndm\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which instance attributes to output when displaying in table or csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " n - Name\n" " d - Description\n" " m - Member" msgstr "" #: cmd/incus/cluster.go:149 msgid "List all the cluster members" msgstr "" #: cmd/incus/cluster.go:150 msgid "" "List all the cluster members\n" "\n" "\tThe -c option takes a (optionally comma-separated) list of arguments\n" "\tthat control which cluster members attributes to output when displaying in " "table\n" "\tor csv format.\n" "\n" "\tDefault column layout is: nurafdsm\n" "\n" "\tColumn shorthand chars:\n" "\n" " n - Server name\n" " u - URL\n" " r - Roles\n" " a - Architecture\n" " f - Failure Domain\n" " d - Description\n" " s - Status\n" " m - Message" msgstr "" #: cmd/incus/warning.go:100 #, fuzzy msgid "List all warnings" msgstr "Aliasse:\n" #: cmd/incus/network_acl.go:101 msgid "List available network ACL" msgstr "" #: cmd/incus/network_acl.go:100 msgid "List available network ACLS" msgstr "" #: cmd/incus/network_address_set.go:96 cmd/incus/network_address_set.go:97 #, fuzzy msgid "List available network address sets" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_forward.go:96 msgid "List available network forwards" msgstr "" #: cmd/incus/network_forward.go:97 msgid "" "List available network forwards\n" "\n" "Default column layout: ldDp\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which instance attributes to output when displaying in table or csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" "l - Listen Address\n" "d - Description\n" "D - Default Target Address\n" "p - Port\n" "L - Location of the network zone (e.g. its cluster member)" msgstr "" #: cmd/incus/network_load_balancer.go:104 #, fuzzy msgid "List available network load balancers" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_load_balancer.go:105 msgid "" "List available network load balancers\n" "\n" "Default column layout: ldp\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which network load balancer attributes to output when displaying\n" "in table or csv format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " l - Listen Address\n" " d - Description\n" " p - Ports\n" " L - Location of the operation (e.g. its cluster member)" msgstr "" #: cmd/incus/network_peer.go:92 msgid "List available network peers" msgstr "" #: cmd/incus/network_peer.go:93 msgid "" "List available network peers\n" "\n" "Default column layout: ndpts\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which network peer attributes to output when displaying in table or csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " n - Name\n" " d - description\n" " p - Peer\n" " t - Type\n" " s - State" msgstr "" #: cmd/incus/network_zone.go:815 cmd/incus/network_zone.go:816 #, fuzzy msgid "List available network zone records" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_zone.go:98 msgid "" "List available network zone\n" "\n" "Default column layout: nDSdus\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which network zone attributes to output when displaying in table or csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " d - Description\n" " e - Project name\n" " n - Name\n" " u - Used by" msgstr "" #: cmd/incus/network_zone.go:97 #, fuzzy msgid "List available network zones" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network.go:1016 msgid "List available networks" msgstr "" #: cmd/incus/network.go:1017 msgid "" "List available networks\n" "\n" "Filters may be of the = form for property based filtering,\n" "or part of the network name. Filters must be delimited by a ','.\n" "\n" "Examples:\n" " - \"foo\" lists all networks that start with the name foo\n" " - \"name=foo\" lists all networks that exactly have the name foo\n" " - \"type=bridge\" lists all networks with the type bridge\n" "\n" "The -c option takes a (optionally comma-separated) list of arguments\n" "that control which image attributes to output when displaying in table\n" "or csv format.\n" "\n" "Default column layout is: ntm46dus\n" "Column shorthand chars:\n" "4 - IPv4 address\n" "6 - IPv6 address\n" "d - Description\n" "e - Project name\n" "m - Managed status\n" "n - Network Interface Name\n" "s - State\n" "t - Interface type\n" "u - Used by (count)" msgstr "" #: cmd/incus/storage.go:637 msgid "List available storage pools" msgstr "" #: cmd/incus/storage.go:638 msgid "" "List available storage pools\n" "\n" "Default column layout: nDdus\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which storage pools attributes to output when displaying in table or csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " n - Name\n" " D - Driver\n" " d - Description\n" " S - Source\n" " u - used by\n" " s - state" msgstr "" #: cmd/incus/operation.go:123 msgid "List background operations" msgstr "" #: cmd/incus/operation.go:124 msgid "" "List background operations\n" "\n" "Default column layout: itdscCl\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which attributes of background operations to output when displaying\n" "in table or csv format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " i - ID\n" " t - Type\n" " d - Description\n" " s - State\n" " c - Cancelable\n" " C - Created\n" " L - Location of the operation (e.g. its cluster member)" msgstr "" #: cmd/incus/image_alias.go:186 msgid "List image aliases" msgstr "" #: cmd/incus/image_alias.go:187 msgid "" "List image aliases\n" "\n" "Filters may be part of the image hash or part of the image alias name.\n" "Default column layout: aftd\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which attributes of image aliases to output when displaying in table or csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " a - Alias\n" " f - Fingerprint\n" " t - Type\n" " d - Description" msgstr "" #: cmd/incus/image.go:1042 msgid "List images" msgstr "" #: cmd/incus/image.go:1043 msgid "" "List images\n" "\n" "Filters may be of the = form for property based filtering,\n" "or part of the image hash or part of the image alias name.\n" "\n" "The -c option takes a (optionally comma-separated) list of arguments\n" "that control which image attributes to output when displaying in table\n" "or csv format.\n" "\n" "Default column layout is: lfpdasu\n" "\n" "Column shorthand chars:\n" "\n" " l - Shortest image alias (and optionally number of other aliases)\n" " L - Newline-separated list of all image aliases\n" " f - Fingerprint (short)\n" " F - Fingerprint (long)\n" " p - Whether image is public\n" " d - Description\n" " e - Project\n" " a - Architecture\n" " s - Size\n" " u - Upload date\n" " t - Type" msgstr "" #: cmd/incus/config_device.go:298 cmd/incus/config_device.go:299 #, fuzzy msgid "List instance devices" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/config_template.go:269 cmd/incus/config_template.go:270 #, fuzzy msgid "List instance file templates" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/snapshot.go:319 #, fuzzy msgid "List instance snapshots" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/snapshot.go:320 msgid "" "List instance snapshots\n" "\n" "Default column layout: nTEs\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which snapshots attributes to output when displaying in table or csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " n - Name\n" " T - Taken At\n" " E - Expires At\n" " s - Stateful" msgstr "" #: cmd/incus/list.go:55 #, fuzzy msgid "List instances" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/list.go:56 #, c-format msgid "" "List instances\n" "\n" "Default column layout: ns46tS\n" "Fast column layout: nsacPt\n" "\n" "A single keyword like \"web\" which will list any instance with a name " "starting by \"web\".\n" "A regular expression on the instance name. (e.g. .*web.*01$).\n" "A key/value pair referring to a configuration item. For those, the\n" "namespace can be abbreviated to the smallest unambiguous identifier.\n" "A key/value pair where the key is a shorthand. Multiple values must be " "delimited by ','. Available shorthands:\n" " - type={instance type}\n" " - status={instance current lifecycle status}\n" " - architecture={instance architecture}\n" " - location={location name}\n" " - ipv4={ip or CIDR}\n" " - ipv6={ip or CIDR}\n" "\n" "Examples:\n" " - \"user.blah=abc\" will list all instances with the \"blah\" user " "property set to \"abc\".\n" " - \"u.blah=abc\" will do the same\n" " - \"security.privileged=true\" will list all privileged instances\n" " - \"s.privileged=true\" will do the same\n" " - \"type=container\" will list all container instances\n" " - \"type=container status=running\" will list all running container " "instances\n" "\n" "A regular expression matching a configuration item or its value. (e.g. " "volatile.eth0.hwaddr=10:66:6a:.*).\n" "\n" "When multiple filters are passed, they are added one on top of the other,\n" "selecting instances which satisfy them all.\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which instance attributes to output when displaying in table or csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " 4 - IPv4 address\n" " 6 - IPv6 address\n" " a - Architecture\n" " b - Storage pool\n" " c - Creation date\n" " d - Description\n" " D - disk usage\n" " e - Project name\n" " l - Last used date\n" " m - Memory usage\n" " M - Memory usage (%)\n" " n - Name\n" " N - Number of Processes\n" " p - PID of the instance's init process\n" " P - Profiles\n" " s - State\n" " S - Number of snapshots\n" " t - Type (persistent or ephemeral)\n" " u - CPU usage (in seconds)\n" " U - Started date\n" " L - Location of the instance (e.g. its cluster member)\n" " f - Base Image Fingerprint (short)\n" " F - Base Image Fingerprint (long)\n" "\n" "Custom columns are defined with \"[config:|devices:]key[:name][:maxWidth]" "\":\n" " KEY: The (extended) config or devices key to display. If [config:|" "devices:] is omitted then it defaults to config key.\n" " NAME: Name to display in the column header.\n" " Defaults to the key if not specified or empty.\n" "\n" " MAXWIDTH: Max width of the column (longer results are truncated).\n" " Defaults to -1 (unlimited). Use 0 to limit to the column header size." msgstr "" #: cmd/incus/network_acl.go:105 #, fuzzy msgid "List network ACLs across all projects" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/network_allocations.go:38 msgid "List network allocations in use" msgstr "" #: cmd/incus/network_allocations.go:39 msgid "" "List network allocations in use\n" "Default column layout: uatnm\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which network allocations attribute attributes to output when\n" "displaying in table or csv format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " u - Used by\n" " a - Address\n" " t - Type\n" " n - NAT\n" " m - Mac Address" msgstr "" #: cmd/incus/network_integration.go:389 #, fuzzy msgid "List network integrations" msgstr "Profil %s erstellt\n" #: cmd/incus/network_integration.go:390 msgid "" "List network integrations\n" "\n" "Default column layout: ndtu\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which network integrations attributes to output when displaying in table or " "csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" "\tn - Name\n" "\td - Description\n" "\tt - Type\n" "\tu - Used by" msgstr "" #: cmd/incus/network.go:1046 #, fuzzy msgid "List networks in all projects" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/config_trust.go:101 cmd/incus/config_trust.go:177 msgid "List of projects to restrict the certificate to" msgstr "" #: cmd/incus/operation.go:148 #, fuzzy msgid "List operations from all projects" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/profile.go:673 msgid "List profiles" msgstr "" #: cmd/incus/profile.go:674 msgid "" "List profiles\n" "\n" "Filters may be of the = form for property based filtering,\n" "or part of the profile name. Filters must be delimited by a ','.\n" "\n" "Examples:\n" " - \"foo\" lists all profiles that start with the name foo\n" " - \"name=foo\" lists all profiles that exactly have the name foo\n" " - \"description=.*bar.*\" lists all profiles with a description that " "contains \"bar\"\n" "\n" "The -c option takes a (optionally comma-separated) list of arguments\n" "that control which profile attributes to output when displaying in table\n" "or csv format.\n" "\n" "Default column layout is: ndu\n" "\n" "Column shorthand chars:\n" "n - Profile Name\n" "d - Description\n" "u - Used By" msgstr "" #: cmd/incus/project.go:497 msgid "List projects" msgstr "" #: cmd/incus/project.go:498 msgid "" "List projects\n" "\n" "The -c option takes a (optionally comma-separated) list of arguments\n" "that control which image attributes to output when displaying in table\n" "or csv format.\n" "Default column layout is: nipvbwzdu\n" "Column shorthand chars:\n" "\n" "n - Project Name\n" "i - Images\n" "p - Profiles\n" "v - Storage Volumes\n" "b - Storage Buckets\n" "w - Networks\n" "z - Network Zones\n" "d - Description\n" "u - Used By" msgstr "" #: cmd/incus/storage_bucket.go:834 #, fuzzy msgid "List storage bucket keys" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_bucket.go:836 msgid "" "List storage bucket keys\n" "\n" "Default column layout: ndr\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which storage bucket keys attributes to output when displaying in table or " "csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " n - Name\n" " d - Description\n" " r - Role" msgstr "" #: cmd/incus/storage_bucket.go:439 #, fuzzy msgid "List storage buckets" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_bucket.go:441 msgid "" "List storage buckets\n" "\n" "Default column layout: ndL\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which storage bucket attributes to output when displaying in table or csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " e - Project name\n" " n - Name\n" " d - Description\n" " L - Location of the storage bucket (e.g. its cluster member)" msgstr "" #: cmd/incus/storage_volume.go:3280 cmd/incus/storage_volume.go:3281 #, fuzzy msgid "List storage volume snapshots" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/storage_volume.go:3286 msgid "" "List storage volume snapshots\n" "\n" "\tThe -c option takes a (optionally comma-separated) list of arguments\n" "\tthat control which image attributes to output when displaying in table\n" "\tor csv format.\n" "\n" "\tColumn shorthand chars:\n" "\t\tn - Name\n" "\t\tT - Taken at\n" "\t\tE - Expiry" msgstr "" #: cmd/incus/storage_volume.go:1436 #, fuzzy msgid "List storage volumes" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_volume.go:1441 msgid "" "List storage volumes\n" "\n" "A single keyword like \"vol\" which will list any storage volume with a name " "starting by \"vol\".\n" "A regular expression on the storage volume name. (e.g. .*vol.*01$).\n" "A key/value pair where the key is a storage volume field name. Multiple " "values must be delimited by ','.\n" "\n" "Examples:\n" " - \"type=custom\" will list all custom storage volumes\n" " - \"type=custom content_type=block\" will list all custom block storage " "volumes\n" "\n" "== Columns ==\n" "The -c option takes a (optionally comma-separated) list of arguments\n" "that control which image attributes to output when displaying in table\n" "or csv format.\n" "\n" "Column shorthand chars:\n" " c - Content type (filesystem or block)\n" " d - Description\n" " e - Project name\n" " L - Location of the instance (e.g. its cluster member)\n" " n - Name\n" " t - Type of volume (custom, image, container or virtual-machine)\n" " u - Number of references (used by)\n" " U - Current disk usage" msgstr "" #: cmd/incus/remote.go:944 msgid "List the available remotes" msgstr "" #: cmd/incus/remote.go:945 msgid "" "List the available remotes\n" "\n" "Default column layout: nupaPsg\n" "\n" "== Columns ==\n" "The -c option takes a comma separated list of arguments that control\n" "which remote attributes to output when displaying in table or csv\n" "format.\n" "\n" "Column arguments are either pre-defined shorthand chars (see below),\n" "or (extended) config keys.\n" "\n" "Commas between consecutive shorthand chars are optional.\n" "\n" "Pre-defined column shorthand chars:\n" " n - Name\n" " u - URL\n" " p - Protocol\n" " a - Auth Type\n" " P - Public\n" " s - Static\n" " g - Global" msgstr "" #: cmd/incus/config_trust.go:377 msgid "List trusted clients" msgstr "" #: cmd/incus/config_trust.go:378 msgid "" "List trusted clients\n" "\n" "The -c option takes a (optionally comma-separated) list of arguments\n" "that control which certificate attributes to output when displaying in " "table\n" "or csv format.\n" "\n" "Default column layout is: ntdfe\n" "\n" "Column shorthand chars:\n" "\n" "\tn - Name\n" "\tt - Type\n" "\tc - Common Name\n" "\tf - Fingerprint\n" "\td - Description\n" "\ti - Issue date\n" "\te - Expiry date\n" "\tr - Whether certificate is restricted\n" "\tp - Newline-separated list of projects" msgstr "" #: cmd/incus/warning.go:76 #, fuzzy msgid "List warnings" msgstr "Aliasse:\n" #: cmd/incus/warning.go:77 msgid "" "List warnings\n" "\n" "The -c option takes a (optionally comma-separated) list of arguments\n" "that control which warning attributes to output when displaying in table\n" "or csv format.\n" "\n" "Default column layout is: utSscpLl\n" "\n" "Column shorthand chars:\n" "\n" " c - Count\n" " l - Last seen\n" " L - Location\n" " f - First seen\n" " p - Project\n" " s - Severity\n" " S - Status\n" " u - UUID\n" " t - Type" msgstr "" #: cmd/incus/operation.go:32 cmd/incus/operation.go:33 msgid "List, show and delete background operations" msgstr "" #: cmd/incus/network_load_balancer.go:323 msgid "Load balancer description" msgstr "" #: cmd/incus/info.go:492 msgid "Load:" msgstr "" #: cmd/incus/info.go:666 cmd/incus/storage_volume.go:1312 #, c-format msgid "Location: %s" msgstr "" #: cmd/incus/info.go:926 #, c-format msgid "Log (%s):" msgstr "" #: cmd/incus/monitor.go:81 msgid "Log level filtering can only be used with pretty formatting" msgstr "" #: cmd/incus/network.go:981 msgid "Logical router" msgstr "" #: cmd/incus/network.go:985 msgid "Logical switch" msgstr "" #: cmd/incus/utils.go:666 #, c-format msgid "Login with username %q and password %q" msgstr "" #: cmd/incus/utils.go:668 msgid "Login without username and password" msgstr "" #: cmd/incus/admin_cluster.go:26 msgid "Low level administration tools for inspecting and recovering clusters." msgstr "" #: cmd/incus/admin_cluster.go:25 msgid "Low-level cluster administration commands" msgstr "" #: cmd/incus/network.go:967 #, fuzzy msgid "Lower device" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/network.go:948 #, fuzzy msgid "Lower devices" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/network.go:1266 cmd/incus/network_allocations.go:84 msgid "MAC ADDRESS" msgstr "" #: cmd/incus/info.go:771 #, fuzzy msgid "MAC address" msgstr "Profil %s erstellt\n" #: cmd/incus/network.go:912 #, c-format msgid "MAC address: %s" msgstr "" #: cmd/incus/info.go:248 #, fuzzy, c-format msgid "MAD: %s (%s)" msgstr "" "Benutzung: %s\n" "\n" "Optionen:\n" "\n" #: cmd/incus/network.go:1073 msgid "MANAGED" msgstr "" #: cmd/incus/cluster_group.go:464 msgid "MEMBERS" msgstr "" #: cmd/incus/top.go:85 msgid "MEMORY" msgstr "" #: cmd/incus/list.go:482 msgid "MEMORY USAGE" msgstr "" #: cmd/incus/list.go:483 #, c-format msgid "MEMORY USAGE%" msgstr "" #: cmd/incus/cluster.go:202 msgid "MESSAGE" msgstr "" #: cmd/incus/network.go:946 msgid "MII Frequency" msgstr "" #: cmd/incus/network.go:947 msgid "MII state" msgstr "" #: cmd/incus/info.go:775 msgid "MTU" msgstr "" #: cmd/incus/network.go:915 #, c-format msgid "MTU: %d" msgstr "" #: cmd/incus/remote.go:142 msgid "Maintain remote connection for faster commands" msgstr "" #: cmd/incus/image.go:173 cmd/incus/image.go:695 msgid "Make image public" msgstr "Veröffentliche Abbild" #: cmd/incus/publish.go:41 #, fuzzy msgid "Make the image public" msgstr "Veröffentliche Abbild" #: cmd/incus/network.go:41 cmd/incus/network.go:42 msgid "Manage and attach instances to networks" msgstr "" #: cmd/incus/cluster_group.go:37 cmd/incus/cluster_group.go:38 #, fuzzy msgid "Manage cluster groups" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/cluster.go:49 cmd/incus/cluster.go:50 msgid "Manage cluster members" msgstr "" #: cmd/incus/cluster_role.go:23 cmd/incus/cluster_role.go:24 #, fuzzy msgid "Manage cluster roles" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/alias.go:23 cmd/incus/alias.go:24 #, fuzzy msgid "Manage command aliases" msgstr "Unbekannter Befehl %s für Abbild" #: cmd/incus/config_device.go:35 cmd/incus/config_device.go:36 #, fuzzy msgid "Manage devices" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/storage_volume.go:2106 cmd/incus/storage_volume.go:2107 #, fuzzy msgid "Manage files in custom volumes" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/file.go:51 cmd/incus/file.go:52 #, fuzzy msgid "Manage files in instances" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/image_alias.go:32 cmd/incus/image_alias.go:33 #, fuzzy msgid "Manage image aliases" msgstr "Veröffentliche Abbild" #: cmd/incus/image.go:51 #, fuzzy msgid "Manage images" msgstr "Veröffentliche Abbild" #: cmd/incus/image.go:52 msgid "" "Manage images\n" "\n" "Instances are created from images. Those images were themselves\n" "either generated from an existing instance or downloaded from an image\n" "server.\n" "\n" "When using remote images, the server will automatically cache images for " "you\n" "and remove them upon expiration.\n" "\n" "The image unique identifier is the hash (sha-256) of its representation\n" "as a compressed tarball (or for split images, the concatenation of the\n" "metadata and rootfs tarballs).\n" "\n" "Images can be referenced by their full hash, shortest unique partial\n" "hash or alias name (if one is set)." msgstr "" #: cmd/incus/admin.go:20 cmd/incus/admin.go:21 cmd/incus/admin_other.go:20 #: cmd/incus/admin_other.go:21 #, fuzzy msgid "Manage incus daemon" msgstr "Veröffentliche Abbild" #: cmd/incus/config.go:31 cmd/incus/config.go:32 #, fuzzy msgid "Manage instance and server configuration options" msgstr "Alternatives config Verzeichnis." #: cmd/incus/config_template.go:27 cmd/incus/config_template.go:28 #, fuzzy msgid "Manage instance file templates" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/config_metadata.go:28 cmd/incus/config_metadata.go:29 #, fuzzy msgid "Manage instance metadata files" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/snapshot.go:35 cmd/incus/snapshot.go:36 #, fuzzy msgid "Manage instance snapshots" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/network_acl.go:806 cmd/incus/network_acl.go:807 #, fuzzy msgid "Manage network ACL rules" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_acl.go:32 cmd/incus/network_acl.go:33 msgid "Manage network ACLs" msgstr "" #: cmd/incus/network_address_set.go:32 cmd/incus/network_address_set.go:33 #, fuzzy msgid "Manage network address sets" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_forward.go:820 cmd/incus/network_forward.go:821 #, fuzzy msgid "Manage network forward ports" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_forward.go:31 cmd/incus/network_forward.go:32 #, fuzzy msgid "Manage network forwards" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_integration.go:30 cmd/incus/network_integration.go:31 #, fuzzy msgid "Manage network integrations" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_load_balancer.go:799 #: cmd/incus/network_load_balancer.go:800 #, fuzzy msgid "Manage network load balancer backends" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_load_balancer.go:958 #: cmd/incus/network_load_balancer.go:959 #, fuzzy msgid "Manage network load balancer ports" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_load_balancer.go:31 cmd/incus/network_load_balancer.go:32 #, fuzzy msgid "Manage network load balancers" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_peer.go:31 cmd/incus/network_peer.go:32 #, fuzzy msgid "Manage network peerings" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_zone.go:1425 cmd/incus/network_zone.go:1426 #, fuzzy msgid "Manage network zone record entries" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_zone.go:756 cmd/incus/network_zone.go:757 #, fuzzy msgid "Manage network zone records" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_zone.go:36 cmd/incus/network_zone.go:37 #, fuzzy msgid "Manage network zones" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/profile.go:38 cmd/incus/profile.go:39 #, fuzzy msgid "Manage profiles" msgstr "Fehlerhafte Profil URL %s" #: cmd/incus/project.go:40 cmd/incus/project.go:41 #, fuzzy msgid "Manage projects" msgstr "Fehlerhafte Profil URL %s" #: cmd/incus/storage_bucket.go:786 #, fuzzy msgid "Manage storage bucket keys" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_bucket.go:787 #, fuzzy msgid "Manage storage bucket keys." msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_bucket.go:39 #, fuzzy msgid "Manage storage buckets" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_bucket.go:40 #, fuzzy msgid "Manage storage buckets." msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage.go:41 cmd/incus/storage.go:42 #, fuzzy msgid "Manage storage pools and volumes" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_volume.go:2992 cmd/incus/storage_volume.go:2993 #, fuzzy msgid "Manage storage volume snapshots" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_volume.go:67 #, fuzzy msgid "Manage storage volumes" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_volume.go:68 msgid "" "Manage storage volumes\n" "\n" "Unless specified through a prefix, all volume operations affect \"custom\" " "(user created) volumes." msgstr "" #: cmd/incus/remote.go:50 cmd/incus/remote.go:51 msgid "Manage the list of remote servers" msgstr "" #: cmd/incus/config_trust.go:37 cmd/incus/config_trust.go:38 msgid "Manage trusted clients" msgstr "" #: cmd/incus/warning.go:32 cmd/incus/warning.go:33 #, fuzzy msgid "Manage warnings" msgstr "Veröffentliche Abbild" #: cmd/incus/remote.go:672 msgid "Manually trigger the generation of a client certificate" msgstr "" #: cmd/incus/info.go:152 cmd/incus/info.go:261 #, c-format msgid "Maximum number of VFs: %d" msgstr "" #: cmd/incus/wait.go:47 msgid "Maximum wait time" msgstr "" #: cmd/incus/info.go:163 #, fuzzy msgid "Mdev profiles:" msgstr "Fehlerhafte Profil URL %s" #: cmd/incus/cluster_role.go:87 #, fuzzy, c-format msgid "Member %q already has role %q" msgstr "Profil %s wurde auf %s angewandt\n" #: cmd/incus/cluster_role.go:148 #, fuzzy, c-format msgid "Member %q does not have role %q" msgstr "Profil %s wurde auf %s angewandt\n" #: cmd/incus/cluster.go:1076 #, fuzzy, c-format msgid "Member %s join token:" msgstr "Profil %s wurde auf %s angewandt\n" #: cmd/incus/cluster.go:755 #, fuzzy, c-format msgid "Member %s removed" msgstr "Gerät %s wurde von %s entfernt\n" #: cmd/incus/cluster.go:665 #, fuzzy, c-format msgid "Member %s renamed to %s" msgstr "Profil %s wurde auf %s angewandt\n" #: cmd/incus/info.go:730 msgid "Memory (current)" msgstr "" #: cmd/incus/info.go:734 msgid "Memory (peak)" msgstr "" #: cmd/incus/info.go:746 msgid "Memory usage:" msgstr "" #: cmd/incus/info.go:513 msgid "Memory:" msgstr "" #: cmd/incus/move.go:284 #, c-format msgid "Migration API failure: %w" msgstr "" #: cmd/incus/move.go:303 #, c-format msgid "Migration operation failure: %w" msgstr "" #: cmd/incus/admin_init.go:52 msgid "Minimal configuration (non-interactive)" msgstr "" #: cmd/incus/monitor.go:57 msgid "" "Minimum level for log messages (only available when using pretty format)" msgstr "" #: cmd/incus/admin_init_interactive.go:470 msgid "Minimum size is 1GiB" msgstr "" #: cmd/incus/file.go:727 msgid "Missing target directory" msgstr "" #: cmd/incus/network.go:942 msgid "Mode" msgstr "" #: cmd/incus/info.go:283 #, c-format msgid "Model: %s" msgstr "" #: cmd/incus/info.go:143 #, c-format msgid "Model: %v" msgstr "" #: cmd/incus/monitor.go:37 #, fuzzy msgid "Monitor a local or remote server" msgstr "Neue entfernte Server hinzufügen" #: cmd/incus/monitor.go:38 msgid "" "Monitor a local or remote server\n" "\n" "By default the monitor will listen to all message types." msgstr "" #: cmd/incus/network.go:545 cmd/incus/storage_volume.go:762 msgid "More than one device matches, specify the device name" msgstr "" #: cmd/incus/file.go:494 msgid "More than one file to download, but target is not a directory" msgstr "" "Mehr als eine Datei herunterzuladen, aber das Ziel ist kein Verzeichnis" #: cmd/incus/storage_volume.go:2396 #, fuzzy msgid "Mount files from custom storage volumes" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/storage_volume.go:2397 msgid "" "Mount files from custom storage volumes.\n" "If no target path is provided, start an SSH SFTP listener instead." msgstr "" #: cmd/incus/file.go:918 #, fuzzy msgid "Mount files from instances" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/file.go:919 msgid "" "Mount files from instances.\n" "If no target path is provided, start an SSH SFTP listener instead." msgstr "" #: cmd/incus/storage_volume.go:1672 cmd/incus/storage_volume.go:1673 #, fuzzy msgid "Move custom storage volumes between pools" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/move.go:40 #, fuzzy msgid "Move instances within or in between servers" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/move.go:41 msgid "" "Move instances within or in between servers\n" "\n" "Transfer modes (--mode):\n" " - pull: Target server pulls the data from the source server (source must " "listen on network)\n" " - push: Source server pushes the data to the target server (target must " "listen on network)\n" " - relay: The CLI connects to both source and server and proxies the data " "(both source and target must listen on network)\n" "\n" "The pull transfer mode is the default as it is compatible with all server " "versions.\n" msgstr "" #: cmd/incus/move.go:63 #, fuzzy msgid "Move the instance without its snapshots" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/storage_volume.go:1679 msgid "Move to a project different from the source" msgstr "" #: cmd/incus/storage_volume.go:444 #, fuzzy, c-format msgid "Moving the storage volume: %s" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_forward.go:975 cmd/incus/network_load_balancer.go:1102 msgid "Multiple ports match. Use --force to remove them all" msgstr "" #: cmd/incus/network_acl.go:1044 msgid "Multiple rules match. Use --force to remove them all" msgstr "" #: cmd/incus/image.go:720 msgid "Must run as root to import from directory" msgstr "" #: cmd/incus/cluster.go:195 cmd/incus/cluster.go:1143 #: cmd/incus/cluster_group.go:463 cmd/incus/config_trust.go:413 #: cmd/incus/config_trust.go:600 cmd/incus/list.go:484 #: cmd/incus/network.go:1071 cmd/incus/network_acl.go:162 #: cmd/incus/network_address_set.go:154 cmd/incus/network_integration.go:427 #: cmd/incus/network_peer.go:138 cmd/incus/network_zone.go:144 #: cmd/incus/network_zone.go:871 cmd/incus/profile.go:723 #: cmd/incus/project.go:541 cmd/incus/project.go:670 cmd/incus/remote.go:984 #: cmd/incus/snapshot.go:365 cmd/incus/storage.go:684 #: cmd/incus/storage_bucket.go:480 cmd/incus/storage_bucket.go:872 #: cmd/incus/storage_volume.go:1568 cmd/incus/storage_volume.go:3377 msgid "NAME" msgstr "" #: cmd/incus/network_allocations.go:83 msgid "NAT" msgstr "" #: cmd/incus/storage_volume.go:3982 #, fuzzy msgid "NBD access to a block storage volume" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_volume.go:3983 #, fuzzy msgid "NBD access to a block storage volume." msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_volume.go:4047 #, fuzzy, c-format msgid "NBD client connected %q" msgstr "Entferntes Administrator Passwort" #: cmd/incus/storage_volume.go:4048 #, fuzzy, c-format msgid "NBD client disconnected %q" msgstr "Entferntes Administrator Passwort" #: cmd/incus/storage_volume.go:4053 #, c-format msgid "NBD connection failed: %v" msgstr "" #: cmd/incus/storage_volume.go:4037 #, c-format msgid "NBD listening on %v" msgstr "" #: cmd/incus/project.go:547 msgid "NETWORK ZONES" msgstr "" #: cmd/incus/project.go:546 msgid "NETWORKS" msgstr "" #: cmd/incus/admin_recover.go:155 #, c-format msgid "NEW: %q (backend=%q, source=%q)" msgstr "" #: cmd/incus/info.go:556 msgid "NIC:" msgstr "" #: cmd/incus/info.go:559 msgid "NICs:" msgstr "" #: cmd/incus/network.go:1120 cmd/incus/operation.go:214 #: cmd/incus/project.go:579 cmd/incus/project.go:588 cmd/incus/project.go:597 #: cmd/incus/project.go:606 cmd/incus/project.go:615 cmd/incus/project.go:624 #: cmd/incus/remote.go:1048 cmd/incus/remote.go:1057 cmd/incus/remote.go:1066 msgid "NO" msgstr "" #: cmd/incus/info.go:104 cmd/incus/info.go:190 cmd/incus/info.go:277 #: cmd/incus/info.go:364 #, c-format msgid "NUMA node: %v" msgstr "" #: cmd/incus/info.go:522 msgid "NUMA nodes:\n" msgstr "" #: cmd/incus/info.go:140 msgid "NVIDIA information:" msgstr "" #: cmd/incus/info.go:145 #, c-format msgid "NVRM Version: %v" msgstr "" #: cmd/incus/info.go:839 cmd/incus/info.go:890 cmd/incus/storage_volume.go:1354 #: cmd/incus/storage_volume.go:1404 msgid "Name" msgstr "" #: cmd/incus/admin_init_interactive.go:419 msgid "Name of the CEPHfs volume:" msgstr "" #: cmd/incus/admin_init_interactive.go:400 #, fuzzy msgid "Name of the OSD storage pool" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/admin_init_interactive.go:501 #, c-format msgid "Name of the existing %s pool or dataset:" msgstr "" #: cmd/incus/admin_init_interactive.go:394 #: cmd/incus/admin_init_interactive.go:488 msgid "Name of the existing CEPH cluster" msgstr "" #: cmd/incus/admin_init_interactive.go:413 msgid "Name of the existing CEPHfs cluster" msgstr "" #: cmd/incus/admin_init_interactive.go:494 msgid "Name of the existing OSD storage pool" msgstr "" #: cmd/incus/admin_init_interactive.go:116 msgid "Name of the existing bridge or host interface:" msgstr "" #: cmd/incus/admin_init_interactive.go:293 #, fuzzy msgid "Name of the new storage pool" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/remote.go:178 msgid "Name of the project to use for this remote:" msgstr "" #: cmd/incus/admin_init_interactive.go:426 #, fuzzy msgid "Name of the shared LVM volume group:" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/admin_recover.go:114 #, fuzzy, c-format msgid "Name of the storage backend (%s):" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/admin_init_interactive.go:331 #, c-format msgid "Name of the storage backend to use (%s)" msgstr "" #: cmd/incus/admin_recover.go:97 #, fuzzy msgid "Name of the storage pool:" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/info.go:644 cmd/incus/network.go:909 #: cmd/incus/storage_volume.go:1294 #, c-format msgid "Name: %s" msgstr "" #: cmd/incus/info.go:317 #, c-format msgid "Name: %v" msgstr "" #: cmd/incus/network.go:426 #, fuzzy, c-format msgid "Network %s created" msgstr "Profil %s erstellt\n" #: cmd/incus/network.go:477 #, fuzzy, c-format msgid "Network %s deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/network.go:424 #, fuzzy, c-format msgid "Network %s pending on member %s" msgstr "Profil %s erstellt\n" #: cmd/incus/network.go:1402 #, fuzzy, c-format msgid "Network %s renamed to %s" msgstr "Profil %s erstellt\n" #: cmd/incus/network_acl.go:430 #, fuzzy, c-format msgid "Network ACL %s created" msgstr "Profil %s erstellt\n" #: cmd/incus/network_acl.go:784 #, fuzzy, c-format msgid "Network ACL %s deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/network_acl.go:735 #, fuzzy, c-format msgid "Network ACL %s renamed to %s" msgstr "Profil %s erstellt\n" #: cmd/incus/network_acl.go:364 #, fuzzy msgid "Network ACL description" msgstr "Profil %s erstellt\n" #: cmd/incus/network_zone.go:736 #, fuzzy, c-format msgid "Network Zone %s deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/network_address_set.go:309 #, fuzzy, c-format msgid "Network address set %s created" msgstr "Profil %s erstellt\n" #: cmd/incus/network_address_set.go:635 #, fuzzy, c-format msgid "Network address set %s deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/network_address_set.go:589 #, fuzzy, c-format msgid "Network address set %s renamed to %s" msgstr "Profil %s erstellt\n" #: cmd/incus/network_address_set.go:239 #, fuzzy msgid "Network address set description" msgstr "Profil %s erstellt\n" #: cmd/incus/network.go:350 #, fuzzy msgid "Network description" msgstr "Profil %s erstellt\n" #: cmd/incus/network_forward.go:382 #, fuzzy, c-format msgid "Network forward %s created" msgstr "Profil %s erstellt\n" #: cmd/incus/network_forward.go:803 #, fuzzy, c-format msgid "Network forward %s deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/network_forward.go:320 #, fuzzy msgid "Network forward description" msgstr "Profil %s erstellt\n" #: cmd/incus/network_integration.go:154 #, fuzzy, c-format msgid "Network integration %s created" msgstr "Profil %s erstellt\n" #: cmd/incus/network_integration.go:200 #, fuzzy, c-format msgid "Network integration %s deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/network_integration.go:547 #, fuzzy, c-format msgid "Network integration %s renamed to %s" msgstr "Profil %s erstellt\n" #: cmd/incus/network_load_balancer.go:385 #, fuzzy, c-format msgid "Network load balancer %s created" msgstr "Profil %s erstellt\n" #: cmd/incus/network_load_balancer.go:783 #, fuzzy, c-format msgid "Network load balancer %s deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/create.go:67 msgid "Network name" msgstr "" #: cmd/incus/network_peer.go:405 #, fuzzy, c-format msgid "Network peer %s created" msgstr "Profil %s erstellt\n" #: cmd/incus/network_peer.go:799 #, fuzzy, c-format msgid "Network peer %s deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/network_peer.go:409 #, c-format msgid "Network peer %s is in unexpected state %q" msgstr "" #: cmd/incus/network_peer.go:407 #, c-format msgid "" "Network peer %s pending (please complete mutual peering on peer network)" msgstr "" #: cmd/incus/network.go:349 #, fuzzy msgid "Network type" msgstr "Profil %s erstellt\n" #: cmd/incus/info.go:796 cmd/incus/network.go:931 #, fuzzy msgid "Network usage:" msgstr "Profil %s erstellt\n" #: cmd/incus/network_zone.go:444 #, fuzzy, c-format msgid "Network zone %s created" msgstr "Profil %s erstellt\n" #: cmd/incus/network_zone.go:1099 #, fuzzy, c-format msgid "Network zone record %s created" msgstr "Profil %s erstellt\n" #: cmd/incus/network_zone.go:1408 #, fuzzy, c-format msgid "Network zone record %s deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/file.go:448 cmd/incus/file.go:679 cmd/incus/storage_volume.go:2589 #: cmd/incus/storage_volume.go:2787 msgid "Never follow symbolic links in source path" msgstr "" #: cmd/incus/publish.go:42 msgid "New alias to define at target" msgstr "" #: cmd/incus/image.go:177 cmd/incus/image.go:697 msgid "New aliases to add to the image" msgstr "" #: cmd/incus/copy.go:59 cmd/incus/create.go:64 cmd/incus/import.go:41 #: cmd/incus/move.go:60 #, fuzzy msgid "New key/value to apply to a specific device" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/admin_init_interactive.go:270 #, c-format msgid "No %s storage backends available" msgstr "" #: cmd/incus/config_trust.go:797 #, c-format msgid "No certificate add token for member %s on remote: %s" msgstr "" #: cmd/incus/cluster.go:1329 #, c-format msgid "No cluster join token for member %s on remote: %s" msgstr "" #: cmd/incus/network.go:554 #, fuzzy msgid "No device found for this network" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_volume.go:771 #, fuzzy msgid "No device found for this storage volume" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_load_balancer.go:1156 msgid "No load-balancer health information available" msgstr "" #: cmd/incus/network_load_balancer.go:938 msgid "No matching backend found" msgstr "" #: cmd/incus/network_forward.go:983 cmd/incus/network_load_balancer.go:1110 msgid "No matching port(s) found" msgstr "" #: cmd/incus/network_acl.go:1055 msgid "No matching rule(s) found" msgstr "" #: cmd/incus/admin_init_interactive.go:267 msgid "No storage backends available" msgstr "" #: cmd/incus/admin_recover.go:213 msgid "No unknown storage pools or volumes found. Nothing to do." msgstr "" #: cmd/incus/info.go:524 #, c-format msgid "Node %d:\n" msgstr "" #: cmd/incus/admin_init_auto.go:31 msgid "" "None of --storage-pool, --storage-create-device or --storage-create-loop may " "be used with the 'dir' backend" msgstr "" #: cmd/incus/admin_init_interactive.go:406 msgid "Number of placement groups" msgstr "" #: cmd/incus/info.go:689 msgid "OS" msgstr "" #: cmd/incus/info.go:690 #, fuzzy msgid "OS Version" msgstr "Fingerabdruck: %s\n" #: cmd/incus/network.go:974 msgid "OVN:" msgstr "" #: cmd/incus/network_address_set.go:747 msgid "One or more provided address isn't currently in the set" msgstr "" #: cmd/incus/remote.go:363 msgid "Only https URLs are supported for oci and simplestreams" msgstr "" #: cmd/incus/image.go:756 msgid "Only https:// is supported for remote image import" msgstr "" #: cmd/incus/network.go:735 cmd/incus/network.go:1464 msgid "Only managed networks can be modified" msgstr "" #: cmd/incus/admin_init_auto.go:35 msgid "" "Only one of --storage-create-device or --storage-create-loop can be specified" msgstr "" #: cmd/incus/webui.go:21 cmd/incus/webui.go:22 #, fuzzy msgid "Open the web interface" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/info.go:688 #, fuzzy msgid "Operating System:" msgstr "Profil %s gelöscht\n" #: cmd/incus/operation.go:96 #, fuzzy, c-format msgid "Operation %s deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/info.go:894 cmd/incus/storage_volume.go:1408 msgid "Optimized Storage" msgstr "" #: cmd/incus/main.go:162 msgid "Override the source project" msgstr "" #: cmd/incus/exec.go:64 msgid "Override the terminal mode (auto, interactive or non-interactive)" msgstr "" #: cmd/incus/info.go:115 cmd/incus/info.go:201 #, c-format msgid "PCI address: %v" msgstr "" #: cmd/incus/info.go:592 #, fuzzy msgid "PCI device:" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/info.go:595 #, fuzzy msgid "PCI devices:" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/network_peer.go:140 msgid "PEER" msgstr "" #: cmd/incus/list.go:486 msgid "PID" msgstr "" #: cmd/incus/info.go:670 #, c-format msgid "PID: %d" msgstr "" #: cmd/incus/network_forward.go:145 cmd/incus/network_load_balancer.go:151 msgid "PORTS" msgstr "" #: cmd/incus/list.go:485 msgid "PROCESSES" msgstr "" #: cmd/incus/list.go:487 cmd/incus/project.go:543 msgid "PROFILES" msgstr "" #: cmd/incus/image.go:1099 cmd/incus/list.go:478 cmd/incus/network.go:1070 #: cmd/incus/network_acl.go:168 cmd/incus/network_address_set.go:161 #: cmd/incus/network_zone.go:143 cmd/incus/profile.go:724 #: cmd/incus/storage_bucket.go:479 cmd/incus/storage_volume.go:1587 #: cmd/incus/top.go:82 cmd/incus/warning.go:213 msgid "PROJECT" msgstr "" #: cmd/incus/config_trust.go:421 msgid "PROJECTS" msgstr "" #: cmd/incus/remote.go:986 msgid "PROTOCOL" msgstr "" #: cmd/incus/image.go:1104 cmd/incus/remote.go:988 msgid "PUBLIC" msgstr "" #: cmd/incus/info.go:780 cmd/incus/network.go:934 msgid "Packets received" msgstr "" #: cmd/incus/info.go:781 cmd/incus/network.go:935 msgid "Packets sent" msgstr "" #: cmd/incus/info.go:300 msgid "Partitions:" msgstr "" #: cmd/incus/main.go:481 cmd/incus/remote.go:817 #, fuzzy, c-format msgid "Password for %s: " msgstr "Administrator Passwort für %s: " #: cmd/incus/utils.go:637 #, fuzzy, c-format msgid "Password rejected for %q" msgstr "Administrator Passwort für %s: " #: cmd/incus/cluster.go:1888 #, fuzzy, c-format msgid "Path %s doesn't exist" msgstr "entfernte Instanz %s existiert nicht" #: cmd/incus/admin_init_interactive.go:438 msgid "Path to the existing block device:" msgstr "" #: cmd/incus/action.go:58 cmd/incus/action.go:59 #, fuzzy msgid "Pause instances" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/network_peer.go:320 #, fuzzy msgid "Peer description" msgstr "Fingerabdruck: %s\n" #: cmd/incus/copy.go:69 msgid "Perform an incremental copy" msgstr "" #: cmd/incus/admin_recover.go:225 msgid "Please create those missing entries and then hit ENTER:" msgstr "" #: cmd/incus/remote.go:219 #, fuzzy msgid "Please provide an alternate server address (empty to abort):" msgstr "Alternatives config Verzeichnis." #: cmd/incus/cluster.go:1709 #, fuzzy msgid "Please provide join token:" msgstr "Profil %s wurde auf %s angewandt\n" #: cmd/incus/remote.go:506 msgid "Please type 'y', 'n' or the fingerprint:" msgstr "" #: cmd/incus/wait.go:46 #, fuzzy msgid "Polling interval (in seconds)" msgstr "CPU Nutzung (in Sekunden)" #: cmd/incus/admin_recover.go:99 msgid "Pool name cannot be empty" msgstr "" #: cmd/incus/network_forward.go:843 cmd/incus/network_load_balancer.go:981 #, fuzzy msgid "Port description" msgstr "Fingerabdruck: %s\n" #: cmd/incus/admin_init_interactive.go:589 msgid "Port to bind to" msgstr "" #: cmd/incus/admin_init.go:57 #, c-format msgid "Port to bind to (default: %d)" msgstr "" #: cmd/incus/info.go:227 #, fuzzy, c-format msgid "Port type: %s" msgstr "Erstellt: %s" #: cmd/incus/info.go:209 msgid "Ports:" msgstr "" #: cmd/incus/admin_init.go:53 msgid "Pre-seed mode, expects YAML config from stdin" msgstr "" #: cmd/incus/top.go:434 msgid "Press 'd' + ENTER to change delay" msgstr "" #: cmd/incus/top.go:435 msgid "Press 's' + ENTER to change sorting method" msgstr "" #: cmd/incus/top.go:436 msgid "Press CTRL-C to exit" msgstr "" #: cmd/incus/utils.go:561 msgid "Press ctrl+c to finish" msgstr "" #: cmd/incus/cluster.go:935 cmd/incus/cluster_group.go:375 #: cmd/incus/config.go:266 cmd/incus/config.go:341 #: cmd/incus/config_metadata.go:148 cmd/incus/config_template.go:233 #: cmd/incus/config_trust.go:330 cmd/incus/image.go:483 #: cmd/incus/network.go:760 cmd/incus/network_acl.go:670 #: cmd/incus/network_address_set.go:525 cmd/incus/network_forward.go:727 #: cmd/incus/network_integration.go:295 cmd/incus/network_load_balancer.go:707 #: cmd/incus/network_peer.go:730 cmd/incus/network_zone.go:672 #: cmd/incus/network_zone.go:1339 cmd/incus/profile.go:570 #: cmd/incus/project.go:395 cmd/incus/storage.go:355 #: cmd/incus/storage_bucket.go:334 cmd/incus/storage_bucket.go:1203 #: cmd/incus/storage_volume.go:1032 cmd/incus/storage_volume.go:1064 msgid "Press enter to open the editor again or ctrl+c to abort change" msgstr "" #: cmd/incus/monitor.go:54 msgid "Pretty rendering (short for --format=pretty)" msgstr "" #: cmd/incus/main.go:160 msgid "Print help" msgstr "" #: cmd/incus/remote.go:749 #, fuzzy msgid "Print or retrieve the client certificate used by this Incus client" msgstr "Ein Client Zertifikat ist bereits erstellt worden" #: cmd/incus/query.go:47 msgid "Print the raw response" msgstr "" #: cmd/incus/main.go:159 msgid "Print version number" msgstr "" #: cmd/incus/info.go:494 cmd/incus/info.go:699 #, fuzzy, c-format msgid "Processes: %d" msgstr "Profil %s erstellt\n" #: cmd/incus/main_aliases.go:239 cmd/incus/main_aliases.go:246 #, fuzzy, c-format msgid "Processing aliases failed: %s" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/info.go:350 cmd/incus/info.go:363 cmd/incus/info.go:377 #, fuzzy, c-format msgid "Product ID: %v" msgstr "Erstellt: %s" #: cmd/incus/info.go:463 #, fuzzy, c-format msgid "Product: %s" msgstr "Ungültiges Ziel %s" #: cmd/incus/info.go:349 cmd/incus/info.go:362 cmd/incus/info.go:376 #: cmd/incus/info.go:412 #, fuzzy, c-format msgid "Product: %v" msgstr "Erstellt: %s" #: cmd/incus/info.go:111 cmd/incus/info.go:197 #, c-format msgid "Product: %v (%v)" msgstr "" #: cmd/incus/profile.go:163 #, fuzzy, c-format msgid "Profile %s added to %s" msgstr "Profil %s wurde auf %s angewandt\n" #: cmd/incus/profile.go:404 #, fuzzy, c-format msgid "Profile %s created" msgstr "Profil %s erstellt\n" #: cmd/incus/profile.go:454 #, fuzzy, c-format msgid "Profile %s deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/profile.go:884 #, fuzzy, c-format msgid "Profile %s isn't currently applied to %s" msgstr "Profil %s wurde auf %s angewandt\n" #: cmd/incus/profile.go:909 #, fuzzy, c-format msgid "Profile %s removed from %s" msgstr "Gerät %s wurde von %s entfernt\n" #: cmd/incus/profile.go:960 #, fuzzy, c-format msgid "Profile %s renamed to %s" msgstr "Profil %s wurde auf %s angewandt\n" #: cmd/incus/profile.go:353 #, fuzzy msgid "Profile description" msgstr "Profil %s erstellt\n" #: cmd/incus/image.go:181 #, fuzzy msgid "Profile to apply to the new image" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/copy.go:60 cmd/incus/create.go:63 #, fuzzy msgid "Profile to apply to the new instance" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/move.go:61 #, fuzzy msgid "Profile to apply to the target instance" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/profile.go:247 #, fuzzy, c-format msgid "Profiles %s applied to %s" msgstr "Profil %s wurde auf %s angewandt\n" #: cmd/incus/image.go:1017 #, fuzzy msgid "Profiles:" msgstr "Profil %s erstellt\n" #: cmd/incus/image.go:1015 #, fuzzy msgid "Profiles: " msgstr "Profil %s erstellt\n" #: cmd/incus/project.go:185 #, fuzzy, c-format msgid "Project %s created" msgstr "Profil %s erstellt\n" #: cmd/incus/project.go:264 #, fuzzy, c-format msgid "Project %s deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/project.go:742 #, fuzzy, c-format msgid "Project %s renamed to %s" msgstr "Profil %s wurde auf %s angewandt\n" #: cmd/incus/project.go:120 #, fuzzy msgid "Project description" msgstr "Profil %s erstellt\n" #: cmd/incus/remote.go:141 msgid "Project to use for the remote" msgstr "" #: cmd/incus/image.go:990 #, fuzzy msgid "Properties:" msgstr "Eigenschaften:\n" #: cmd/incus/image.go:1569 msgid "Property not found" msgstr "" #: cmd/incus/image.go:1010 #, fuzzy, c-format msgid "Protocol: %s" msgstr "Ungültiges Ziel %s" #: cmd/incus/config_trust.go:207 #, fuzzy, c-format msgid "Provided certificate path doesn't exist: %s" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/remote_unix.go:46 msgid "Proxy timeout (exits when no connections)" msgstr "" #: cmd/incus/remote.go:140 msgid "Public image server" msgstr "" #: cmd/incus/image.go:969 #, fuzzy, c-format msgid "Public: %s" msgstr "Öffentlich: %s\n" #: cmd/incus/publish.go:37 cmd/incus/publish.go:38 #, fuzzy msgid "Publish instances as images" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/publish.go:223 #, fuzzy, c-format msgid "Publishing instance: %s" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/storage_volume.go:2578 cmd/incus/storage_volume.go:2579 #, fuzzy msgid "Pull files from custom volumes" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/file.go:437 cmd/incus/file.go:438 #, fuzzy msgid "Pull files from instances" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/file.go:582 cmd/incus/storage_volume.go:2701 #: cmd/incus/utils_sftp.go:210 #, c-format msgid "Pulling %s from %s: %%s" msgstr "" #: cmd/incus/storage_volume.go:2773 cmd/incus/storage_volume.go:2774 #, fuzzy msgid "Push files into custom volumes" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/file.go:665 cmd/incus/file.go:666 #, fuzzy msgid "Push files into instances" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/file.go:853 cmd/incus/storage_volume.go:2945 #: cmd/incus/utils_sftp.go:322 #, c-format msgid "Pushing %s to %s: %%s" msgstr "" #: cmd/incus/query.go:86 msgid "Query path must start with /" msgstr "" #: cmd/incus/image.go:523 cmd/incus/image.go:905 cmd/incus/image.go:1463 msgid "Query virtual machine images" msgstr "" #: cmd/incus/project.go:1076 msgid "RESOURCE" msgstr "" #: cmd/incus/config_trust.go:420 msgid "RESTRICTED" msgstr "" #: cmd/incus/storage_bucket.go:874 msgid "ROLE" msgstr "" #: cmd/incus/cluster.go:197 msgid "ROLES" msgstr "" #: cmd/incus/info.go:296 cmd/incus/info.go:305 #, c-format msgid "Read-Only: %v" msgstr "" #: cmd/incus/rebuild.go:32 #, fuzzy msgid "Rebuild as an empty instance" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/rebuild.go:27 #, fuzzy msgid "Rebuild instances" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/network_zone.go:1032 #, fuzzy msgid "Record description" msgstr "Fingerabdruck: %s\n" #: cmd/incus/admin_recover.go:30 msgid "" "Recover missing instances and volumes from existing and unknown storage pools" msgstr "" #: cmd/incus/admin_recover.go:31 msgid "" "Recover missing instances and volumes from existing and unknown storage " "pools\n" "\n" " This command is mostly used for disaster recovery. It will ask you about " "unknown storage pools and attempt to\n" " access them, along with existing storage pools, and identify any missing " "instances and volumes that exist on the\n" " pools but are not in the database. It will then offer to recreate these " "database records." msgstr "" #: cmd/incus/file.go:447 cmd/incus/file.go:678 cmd/incus/storage_volume.go:2588 #: cmd/incus/storage_volume.go:2786 msgid "Recursively transfer files" msgstr "" #: cmd/incus/storage_volume.go:351 #, fuzzy msgid "Refresh and update the existing storage volume copies" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/image.go:1383 cmd/incus/image.go:1384 msgid "Refresh images" msgstr "" #: cmd/incus/copy.go:384 #, fuzzy, c-format msgid "Refreshing instance: %s" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/image.go:1407 #, c-format msgid "Refreshing the image: %s" msgstr "" #: cmd/incus/remote.go:1157 #, fuzzy, c-format msgid "Remote %s already exists" msgstr "entfernte Instanz %s existiert bereits" #: cmd/incus/remote.go:1148 cmd/incus/remote.go:1228 cmd/incus/remote.go:1292 #: cmd/incus/remote.go:1340 #, fuzzy, c-format msgid "Remote %s doesn't exist" msgstr "entfernte Instanz %s existiert nicht" #: cmd/incus/remote.go:336 #, fuzzy, c-format msgid "Remote %s exists as <%s>" msgstr "entfernte Instanz %s existiert als <%s>" #: cmd/incus/remote.go:1236 #, c-format msgid "Remote %s is global and cannot be removed" msgstr "" #: cmd/incus/remote.go:1152 cmd/incus/remote.go:1232 cmd/incus/remote.go:1344 #, c-format msgid "Remote %s is static and cannot be modified" msgstr "" #: cmd/incus/remote.go:330 msgid "Remote names may not contain colons" msgstr "" #: cmd/incus/remote.go:137 msgid "Remote trust token" msgstr "" #: cmd/incus/info.go:297 #, c-format msgid "Removable: %v" msgstr "" #: cmd/incus/delete.go:52 #, c-format msgid "Remove %s (yes/no): " msgstr "" #: cmd/incus/project.go:221 #, c-format msgid "" "Remove %s and everything it contains (instances, images, volumes, " "networks, ...) (yes/no): " msgstr "" #: cmd/incus/cluster_group.go:563 #, fuzzy msgid "Remove a cluster member from a cluster group" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/cluster.go:686 cmd/incus/cluster.go:687 msgid "Remove a member from the cluster" msgstr "" #: cmd/incus/network_zone.go:1496 #, fuzzy msgid "Remove a network zone record entry" msgstr "Profil %s erstellt\n" #: cmd/incus/network_address_set.go:700 cmd/incus/network_address_set.go:701 #, fuzzy msgid "Remove addresses from a network address set" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/alias.go:221 cmd/incus/alias.go:222 #, fuzzy msgid "Remove aliases" msgstr "Entferntes Administrator Passwort" #: cmd/incus/network_forward.go:912 cmd/incus/network_load_balancer.go:1043 msgid "Remove all ports that match" msgstr "" #: cmd/incus/profile.go:197 #, fuzzy msgid "Remove all profiles from the instance" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/network_acl.go:961 msgid "Remove all rules that match" msgstr "" #: cmd/incus/network_load_balancer.go:883 #, fuzzy msgid "Remove backend from a load balancer" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/network_load_balancer.go:882 #, fuzzy msgid "Remove backends from a load balancer" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/network_zone.go:1497 #, fuzzy msgid "Remove entries from a network zone record" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/config_device.go:447 cmd/incus/config_device.go:448 #, fuzzy msgid "Remove instance devices" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/cluster_group.go:562 msgid "Remove member from group" msgstr "" #: cmd/incus/network_forward.go:910 cmd/incus/network_forward.go:911 #, fuzzy msgid "Remove ports from a forward" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/network_load_balancer.go:1041 #: cmd/incus/network_load_balancer.go:1042 #, fuzzy msgid "Remove ports from a load balancer" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/profile.go:847 cmd/incus/profile.go:848 #, fuzzy msgid "Remove profiles from instances" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/remote.go:1200 cmd/incus/remote.go:1201 msgid "Remove remotes" msgstr "" #: cmd/incus/cluster_role.go:108 cmd/incus/cluster_role.go:109 #, fuzzy msgid "Remove roles from a cluster member" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/network_acl.go:959 cmd/incus/network_acl.go:960 #, fuzzy msgid "Remove rules from an ACL" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/snapshot.go:275 #, fuzzy, c-format msgid "Remove snapshot %s from %s (yes/no): " msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/config_trust.go:711 cmd/incus/config_trust.go:712 msgid "Remove trusted client" msgstr "" #: cmd/incus/cluster_group.go:638 cmd/incus/cluster_group.go:639 msgid "Rename a cluster group" msgstr "" #: cmd/incus/cluster.go:632 cmd/incus/cluster.go:633 msgid "Rename a cluster member" msgstr "" #: cmd/incus/alias.go:167 cmd/incus/alias.go:168 cmd/incus/image_alias.go:361 #: cmd/incus/image_alias.go:362 msgid "Rename aliases" msgstr "" #: cmd/incus/storage_volume.go:1734 cmd/incus/storage_volume.go:1735 #, fuzzy msgid "Rename custom storage volumes" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/snapshot.go:470 cmd/incus/snapshot.go:471 #, fuzzy msgid "Rename instance snapshots" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/rename.go:22 cmd/incus/rename.go:23 #, fuzzy msgid "Rename instances" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/network_acl.go:703 cmd/incus/network_acl.go:704 msgid "Rename network ACLs" msgstr "" #: cmd/incus/network_address_set.go:558 cmd/incus/network_address_set.go:559 #, fuzzy msgid "Rename network address sets" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_integration.go:522 cmd/incus/network_integration.go:523 #, fuzzy msgid "Rename network integrations" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network.go:1369 cmd/incus/network.go:1370 msgid "Rename networks" msgstr "" #: cmd/incus/profile.go:927 cmd/incus/profile.go:928 #, fuzzy msgid "Rename profiles" msgstr "Fehlerhafte Profil URL %s" #: cmd/incus/project.go:704 cmd/incus/project.go:705 #, fuzzy msgid "Rename projects" msgstr "Fehlerhafte Profil URL %s" #: cmd/incus/remote.go:1119 cmd/incus/remote.go:1120 msgid "Rename remotes" msgstr "" #: cmd/incus/storage_volume.go:3437 cmd/incus/storage_volume.go:3438 #, fuzzy msgid "Rename storage volume snapshots" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/storage_volume.go:1769 #, c-format msgid "Renamed storage volume from \"%s\" to \"%s\"" msgstr "" #: cmd/incus/storage_volume.go:3490 #, fuzzy, c-format msgid "Renamed storage volume snapshot from \"%s\" to \"%s\"" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/info.go:135 #, c-format msgid "Render: %s (%s)" msgstr "" #: cmd/incus/cluster.go:1034 cmd/incus/cluster.go:1035 msgid "Request a join token for adding a cluster member" msgstr "" #: cmd/incus/delete.go:41 cmd/incus/snapshot.go:232 msgid "Require user confirmation" msgstr "" #: cmd/incus/info.go:697 msgid "Resources:" msgstr "" #: cmd/incus/action.go:104 cmd/incus/action.go:105 #, fuzzy msgid "Restart instances" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/cluster.go:1485 cmd/incus/cluster.go:1486 #, fuzzy msgid "Restore cluster member" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/snapshot.go:525 msgid "" "Restore instance from snapshots\n" "\n" "If --stateful is passed, then the running state will be restored too.\n" "If --diskonly is passed, then only the disk will be restored." msgstr "" #: cmd/incus/snapshot.go:524 #, fuzzy msgid "Restore instance snapshots" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/storage_volume.go:3507 cmd/incus/storage_volume.go:3508 #, fuzzy msgid "Restore storage volume snapshots" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/cluster.go:1542 #, fuzzy, c-format msgid "Restoring cluster member: %s" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/config_trust.go:100 cmd/incus/config_trust.go:176 msgid "Restrict the certificate to one or more projects" msgstr "" #: cmd/incus/action.go:81 cmd/incus/action.go:82 #, fuzzy msgid "Resume instances" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/console.go:53 #, fuzzy msgid "Retrieve the instance's console log" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/create.go:353 #, c-format msgid "Retrieving image: %s" msgstr "" #: cmd/incus/config_trust.go:744 cmd/incus/config_trust.go:745 #, fuzzy msgid "Revoke certificate add token" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/cluster.go:1258 #, fuzzy msgid "Revoke cluster member join token" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/storage_bucket.go:983 msgid "Role (admin or read-only)" msgstr "" #: cmd/incus/admin_sql.go:137 #, c-format msgid "Rows affected: %d" msgstr "" #: cmd/incus/network_acl.go:827 #, fuzzy msgid "Rule description" msgstr "Fingerabdruck: %s\n" #: cmd/incus/remote_unix.go:40 msgid "Run a local API proxy" msgstr "" #: cmd/incus/remote_unix.go:41 msgid "Run a local API proxy for the remote" msgstr "" #: cmd/incus/network_allocations.go:65 msgid "Run again a specific project" msgstr "" #: cmd/incus/action.go:151 msgid "Run against all instances" msgstr "" #: cmd/incus/network_allocations.go:66 #, fuzzy msgid "Run against all projects" msgstr "Fehlerhafte Profil URL %s" #: cmd/incus/warning.go:214 msgid "SEVERITY" msgstr "" #: cmd/incus/image.go:1105 msgid "SIZE" msgstr "" #: cmd/incus/info.go:424 #, c-format msgid "SKU: %v" msgstr "" #: cmd/incus/list.go:488 msgid "SNAPSHOTS" msgstr "" #: cmd/incus/storage.go:687 msgid "SOURCE" msgstr "" #: cmd/incus/info.go:150 cmd/incus/info.go:259 msgid "SR-IOV information:" msgstr "" #: cmd/incus/utils.go:663 #, c-format msgid "SSH SFTP listening on %v" msgstr "" #: cmd/incus/utils.go:680 #, c-format msgid "SSH client connected %q" msgstr "" #: cmd/incus/utils.go:681 #, fuzzy, c-format msgid "SSH client disconnected %q" msgstr "Entferntes Administrator Passwort" #: cmd/incus/list.go:492 #, fuzzy msgid "STARTED AT" msgstr "ERSTELLT AM" #: cmd/incus/list.go:489 cmd/incus/network.go:1078 #: cmd/incus/network_peer.go:142 cmd/incus/operation.go:167 #: cmd/incus/storage.go:689 cmd/incus/warning.go:215 msgid "STATE" msgstr "" #: cmd/incus/snapshot.go:368 msgid "STATEFUL" msgstr "" #: cmd/incus/remote.go:989 msgid "STATIC" msgstr "" #: cmd/incus/cluster.go:201 msgid "STATUS" msgstr "" #: cmd/incus/project.go:545 msgid "STORAGE BUCKETS" msgstr "" #: cmd/incus/list.go:474 msgid "STORAGE POOL" msgstr "" #: cmd/incus/project.go:544 msgid "STORAGE VOLUMES" msgstr "" #: cmd/incus/network.go:956 msgid "STP" msgstr "" #: cmd/incus/admin_recover.go:167 msgid "Scanning for unknown volumes..." msgstr "" #: cmd/incus/storage_bucket.go:985 msgid "Secret key (auto-generated if empty)" msgstr "" #: cmd/incus/storage_bucket.go:1050 #, fuzzy, c-format msgid "Secret key: %s" msgstr "Erstellt: %s" #: cmd/incus/query.go:37 cmd/incus/query.go:38 msgid "Send a raw query to the server" msgstr "" #: cmd/incus/info.go:354 #, fuzzy, c-format msgid "Serial Number: %v" msgstr "Erstellt: %s" #: cmd/incus/info.go:604 #, fuzzy msgid "Serial device:" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/info.go:607 #, fuzzy msgid "Serial devices:" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/info.go:428 #, fuzzy, c-format msgid "Serial number: %v" msgstr "Erstellt: %s" #: cmd/incus/info.go:451 cmd/incus/info.go:467 #, fuzzy, c-format msgid "Serial: %s" msgstr "" "Benutzung: %s\n" "\n" "Optionen:\n" "\n" #: cmd/incus/remote.go:139 msgid "Server authentication type (tls or oidc)" msgstr "" #: cmd/incus/remote.go:504 msgid "Server certificate NACKed by user" msgstr "Server Zertifikat vom Benutzer nicht akzeptiert" #: cmd/incus/remote.go:639 #, fuzzy msgid "Server doesn't trust us after authentication" msgstr "" "Der Server vertraut uns nicht nachdem er unser Zertifikat hinzugefügt hat" #: cmd/incus/cluster.go:286 cmd/incus/cluster.go:1196 cmd/incus/cluster.go:1291 #: cmd/incus/cluster.go:1387 cmd/incus/cluster_group.go:516 msgid "Server isn't part of a cluster" msgstr "" #: cmd/incus/remote.go:138 msgid "Server protocol (incus, oci or simplestreams)" msgstr "" #: cmd/incus/version.go:49 #, c-format msgid "Server version: %s\n" msgstr "" #: cmd/incus/image.go:1009 #, fuzzy, c-format msgid "Server: %s" msgstr "" "Benutzung: %s\n" "\n" "Optionen:\n" "\n" #: cmd/incus/cluster_group.go:877 #, fuzzy msgid "Set a cluster group's configuration keys" msgstr "Alternatives config Verzeichnis." #: cmd/incus/cluster.go:510 #, fuzzy msgid "Set a cluster member's configuration keys" msgstr "Alternatives config Verzeichnis." #: cmd/incus/file.go:931 cmd/incus/storage_volume.go:2408 msgid "Set authentication user when using SSH SFTP listener" msgstr "" #: cmd/incus/config_device.go:565 #, fuzzy msgid "Set device configuration keys" msgstr "Alternatives config Verzeichnis." #: cmd/incus/config_device.go:567 msgid "" "Set device configuration keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus config device set [:] " msgstr "" #: cmd/incus/config_device.go:573 msgid "" "Set device configuration keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus profile device set [:] " msgstr "" #: cmd/incus/image.go:1587 cmd/incus/image.go:1588 msgid "Set image properties" msgstr "" #: cmd/incus/config.go:506 #, fuzzy msgid "Set instance or server configuration keys" msgstr "Alternatives config Verzeichnis." #: cmd/incus/config.go:507 msgid "" "Set instance or server configuration keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus config set [:][] " msgstr "" #: cmd/incus/network_acl.go:449 #, fuzzy msgid "Set network ACL configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_acl.go:450 msgid "" "Set network ACL configuration keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus network set [:] " msgstr "" #: cmd/incus/network_address_set.go:328 cmd/incus/network_address_set.go:329 #, fuzzy msgid "Set network address set configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network.go:1421 msgid "Set network configuration keys" msgstr "" #: cmd/incus/network.go:1422 msgid "" "Set network configuration keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus network set [:] " msgstr "" #: cmd/incus/network_forward.go:475 #, fuzzy msgid "Set network forward keys" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_forward.go:476 msgid "" "Set network forward keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus network set [:] " msgstr "" #: cmd/incus/network_integration.go:566 #, fuzzy msgid "Set network integration configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_integration.go:567 msgid "" "Set network integration configuration keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus network integration set [:] " "" msgstr "" #: cmd/incus/network_load_balancer.go:461 #, fuzzy msgid "Set network load balancer keys" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_load_balancer.go:462 msgid "" "Set network load balancer keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus network set [:] " msgstr "" #: cmd/incus/network_peer.go:503 #, fuzzy msgid "Set network peer keys" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_peer.go:504 msgid "" "Set network peer keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus network set [:] " msgstr "" #: cmd/incus/network_zone.go:463 #, fuzzy msgid "Set network zone configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_zone.go:464 msgid "" "Set network zone configuration keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus network set [:] " msgstr "" #: cmd/incus/network_zone.go:1118 cmd/incus/network_zone.go:1119 #, fuzzy msgid "Set network zone record configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/profile.go:979 msgid "Set profile configuration keys" msgstr "" #: cmd/incus/profile.go:980 msgid "" "Set profile configuration keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus profile set [:] " msgstr "" #: cmd/incus/project.go:761 #, fuzzy msgid "Set project configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/project.go:762 msgid "" "Set project configuration keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus project set [:] " msgstr "" #: cmd/incus/storage_bucket.go:593 #, fuzzy msgid "Set storage bucket configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_bucket.go:594 msgid "" "Set storage bucket configuration keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus storage bucket set [:] " msgstr "" #: cmd/incus/storage.go:792 #, fuzzy msgid "Set storage pool configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/storage.go:793 msgid "" "Set storage pool configuration keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus storage set [:] " msgstr "" #: cmd/incus/storage_volume.go:1803 #, fuzzy msgid "Set storage volume configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_volume.go:1804 #, fuzzy msgid "" "Set storage volume configuration keys\n" "\n" "For backward compatibility, a single configuration key may still be set " "with:\n" " incus storage volume set [:] [/] " "\n" "\n" "If the type is not specified, Incus assumes the type is \"custom\".\n" "Supported values for type are \"custom\", \"container\" and \"virtual-" "machine\"." msgstr "Profil %s erstellt\n" #: cmd/incus/remote.go:1311 cmd/incus/remote.go:1312 msgid "Set the URL for the remote" msgstr "" #: cmd/incus/file.go:110 cmd/incus/storage_volume.go:2167 #, fuzzy msgid "Set the file's gid on create" msgstr "Setzt die gid der Datei beim Übertragen" #: cmd/incus/storage_volume.go:2784 msgid "Set the file's gid on push" msgstr "Setzt die gid der Datei beim Übertragen" #: cmd/incus/file.go:112 cmd/incus/storage_volume.go:2169 #, fuzzy msgid "Set the file's perms on create" msgstr "Setzt die Dateiberechtigungen beim Übertragen" #: cmd/incus/storage_volume.go:2785 msgid "Set the file's perms on push" msgstr "Setzt die Dateiberechtigungen beim Übertragen" #: cmd/incus/file.go:677 msgid "" "Set the file's perms on push (in recursive mode, sets the target directory's " "permissions if it doesn't exist)" msgstr "" #: cmd/incus/file.go:111 cmd/incus/storage_volume.go:2168 #, fuzzy msgid "Set the file's uid on create" msgstr "Setzt die uid der Datei beim Übertragen" #: cmd/incus/storage_volume.go:2783 msgid "Set the file's uid on push" msgstr "Setzt die uid der Datei beim Übertragen" #: cmd/incus/file.go:676 msgid "" "Set the files' GIDs on push (in recursive mode, only sets the target " "directory's GID if it doesn't exist and -p is used)" msgstr "" #: cmd/incus/file.go:675 msgid "" "Set the files' UIDs on push (in recursive mode, only sets the target " "directory's UID if it doesn't exist and -p is used)" msgstr "" #: cmd/incus/cluster_group.go:880 #, fuzzy msgid "Set the key as a cluster group property" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/cluster.go:513 msgid "Set the key as a cluster property" msgstr "" #: cmd/incus/network_acl.go:456 msgid "Set the key as a network ACL property" msgstr "" #: cmd/incus/network_address_set.go:331 #, fuzzy msgid "Set the key as a network address set property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_forward.go:483 #, fuzzy msgid "Set the key as a network forward property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_integration.go:574 #, fuzzy msgid "Set the key as a network integration property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_load_balancer.go:469 #, fuzzy msgid "Set the key as a network load balancer property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_peer.go:511 #, fuzzy msgid "Set the key as a network peer property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network.go:1429 msgid "Set the key as a network property" msgstr "" #: cmd/incus/network_zone.go:471 #, fuzzy msgid "Set the key as a network zone property" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/network_zone.go:1124 #, fuzzy msgid "Set the key as a network zone record property" msgstr "Profil %s erstellt\n" #: cmd/incus/profile.go:987 msgid "Set the key as a profile property" msgstr "" #: cmd/incus/project.go:769 msgid "Set the key as a project property" msgstr "" #: cmd/incus/storage_bucket.go:601 #, fuzzy msgid "Set the key as a storage bucket property" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/storage.go:800 #, fuzzy msgid "Set the key as a storage property" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/storage_volume.go:1820 #, fuzzy msgid "Set the key as a storage volume property" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/config.go:523 msgid "Set the key as an instance property" msgstr "" #: cmd/incus/file.go:929 cmd/incus/storage_volume.go:2406 msgid "Setup SSH SFTP listener on address:port instead of mounting" msgstr "" #: cmd/incus/admin_init.go:59 msgid "Setup device based storage using DEVICE" msgstr "" #: cmd/incus/admin_init.go:60 msgid "Setup loop based storage with SIZE in GiB" msgstr "" #: cmd/incus/main.go:163 msgid "Show all debug messages" msgstr "" #: cmd/incus/main.go:164 msgid "Show all information messages" msgstr "" #: cmd/incus/cluster_group.go:688 cmd/incus/cluster_group.go:689 #, fuzzy msgid "Show cluster group configurations" msgstr "Profil %s erstellt\n" #: cmd/incus/config_template.go:332 cmd/incus/config_template.go:333 #, fuzzy msgid "Show content of instance file templates" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/cluster.go:333 cmd/incus/cluster.go:334 msgid "Show details of a cluster member" msgstr "" #: cmd/incus/operation.go:289 cmd/incus/operation.go:290 msgid "Show details on a background operation" msgstr "" #: cmd/incus/monitor.go:55 msgid "Show events from all projects" msgstr "" #: cmd/incus/config_device.go:688 cmd/incus/config_device.go:689 #, fuzzy msgid "Show full device configuration" msgstr "Geräte zu Containern oder Profilen hinzufügen" #: cmd/incus/image.go:1460 cmd/incus/image.go:1461 msgid "Show image properties" msgstr "" #: cmd/incus/config_metadata.go:181 cmd/incus/config_metadata.go:182 #, fuzzy msgid "Show instance metadata files" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/config.go:684 cmd/incus/config.go:685 #, fuzzy msgid "Show instance or server configurations" msgstr "Profil %s erstellt\n" #: cmd/incus/info.go:39 cmd/incus/info.go:40 #, fuzzy msgid "Show instance or server information" msgstr "Profil %s erstellt\n" #: cmd/incus/snapshot.go:590 cmd/incus/snapshot.go:591 #, fuzzy msgid "Show instance snapshot configuration" msgstr "Profil %s erstellt\n" #: cmd/incus/main.go:352 cmd/incus/main.go:353 msgid "Show less common commands" msgstr "" #: cmd/incus/version.go:24 cmd/incus/version.go:25 msgid "Show local and remote versions" msgstr "" #: cmd/incus/network_acl.go:185 cmd/incus/network_acl.go:186 #, fuzzy msgid "Show network ACL configurations" msgstr "Profil %s erstellt\n" #: cmd/incus/network_acl.go:238 cmd/incus/network_acl.go:239 #, fuzzy msgid "Show network ACL log" msgstr "Profil %s erstellt\n" #: cmd/incus/network_address_set.go:178 cmd/incus/network_address_set.go:179 #, fuzzy msgid "Show network address set configuration" msgstr "Profil %s erstellt\n" #: cmd/incus/network.go:1509 cmd/incus/network.go:1510 msgid "Show network configurations" msgstr "" #: cmd/incus/network_forward.go:244 cmd/incus/network_forward.go:245 #, fuzzy msgid "Show network forward configurations" msgstr "Profil %s erstellt\n" #: cmd/incus/network_integration.go:668 cmd/incus/network_integration.go:669 #, fuzzy msgid "Show network integration options" msgstr "Profil %s erstellt\n" #: cmd/incus/network_load_balancer.go:247 #: cmd/incus/network_load_balancer.go:248 #, fuzzy msgid "Show network load balancer configurations" msgstr "Profil %s erstellt\n" #: cmd/incus/network_peer.go:245 cmd/incus/network_peer.go:246 #, fuzzy msgid "Show network peer configurations" msgstr "Profil %s erstellt\n" #: cmd/incus/network_zone.go:248 cmd/incus/network_zone.go:249 #, fuzzy msgid "Show network zone configurations" msgstr "Profil %s erstellt\n" #: cmd/incus/network_zone.go:890 #, fuzzy msgid "Show network zone record configuration" msgstr "Profil %s erstellt\n" #: cmd/incus/network_zone.go:891 #, fuzzy msgid "Show network zone record configurations" msgstr "Profil %s erstellt\n" #: cmd/incus/profile.go:1061 cmd/incus/profile.go:1062 msgid "Show profile configurations" msgstr "" #: cmd/incus/project.go:884 cmd/incus/project.go:885 msgid "Show project options" msgstr "" #: cmd/incus/storage_bucket.go:699 cmd/incus/storage_bucket.go:700 #, fuzzy msgid "Show storage bucket configurations" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_bucket.go:1235 cmd/incus/storage_bucket.go:1236 #, fuzzy msgid "Show storage bucket key configurations" msgstr "Profil %s erstellt\n" #: cmd/incus/storage.go:887 cmd/incus/storage.go:888 msgid "Show storage pool configurations and resources" msgstr "" #: cmd/incus/storage_volume.go:1951 #, fuzzy msgid "Show storage volume configurations" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_volume.go:1952 #, fuzzy msgid "" "Show storage volume configurations\n" "\n" "If the type is not specified, Incus assumes the type is \"custom\".\n" "Supported values for type are \"custom\", \"container\" and \"virtual-" "machine\".\n" "\n" "For snapshots, add the snapshot name (only if type is one of custom, " "container or virtual-machine)." msgstr "Profil %s erstellt\n" #: cmd/incus/storage_volume.go:3571 #, fuzzy msgid "Show storage volume snapshhot configurations" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_volume.go:3570 #, fuzzy msgid "Show storage volume snapshot configurations" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_volume.go:1219 #, fuzzy msgid "Show storage volume state information" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_volume.go:1220 #, fuzzy msgid "" "Show storage volume state information\n" "\n" "If the type is not specified, Incus assumes the type is \"custom\".\n" "Supported values for type are \"custom\", \"container\" and \"virtual-" "machine\"." msgstr "Profil %s erstellt\n" #: cmd/incus/project.go:1095 cmd/incus/project.go:1096 msgid "Show the current project" msgstr "" #: cmd/incus/remote.go:716 cmd/incus/remote.go:717 msgid "Show the default remote" msgstr "" #: cmd/incus/config.go:688 msgid "Show the expanded configuration" msgstr "" #: cmd/incus/info.go:50 cmd/incus/project.go:992 #, fuzzy msgid "Show the instance's access list" msgstr "Zeige die letzten 100 Zeilen Protokoll des Containers?" #: cmd/incus/info.go:51 #, fuzzy msgid "Show the instance's recent log entries" msgstr "Zeige die letzten 100 Zeilen Protokoll des Containers?" #: cmd/incus/info.go:52 msgid "Show the resources available to the server" msgstr "" #: cmd/incus/storage.go:891 msgid "Show the resources available to the storage pool" msgstr "" #: cmd/incus/storage.go:468 msgid "Show the used and free space in bytes" msgstr "" #: cmd/incus/config_trust.go:812 cmd/incus/config_trust.go:813 #, fuzzy msgid "Show trust configurations" msgstr "Profil %s erstellt\n" #: cmd/incus/cluster.go:385 cmd/incus/cluster.go:386 #, fuzzy msgid "Show useful information about a cluster member" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/image.go:901 cmd/incus/image.go:902 msgid "Show useful information about images" msgstr "" #: cmd/incus/storage.go:464 cmd/incus/storage.go:465 msgid "Show useful information about storage pools" msgstr "" #: cmd/incus/warning.go:296 cmd/incus/warning.go:297 msgid "Show warning" msgstr "" #: cmd/incus/admin_init_interactive.go:459 msgid "Size in GiB of the new loop device" msgstr "" #: cmd/incus/image.go:966 #, fuzzy, c-format msgid "Size: %.2fMiB" msgstr "Größe: %.2vMB\n" #: cmd/incus/info.go:290 cmd/incus/info.go:306 #, fuzzy, c-format msgid "Size: %s" msgstr "Erstellt: %s" #: cmd/incus/storage_volume.go:3057 msgid "Snapshot description" msgstr "" #: cmd/incus/storage_volume.go:3045 cmd/incus/storage_volume.go:3046 #, fuzzy msgid "Snapshot storage volumes" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/storage_volume.go:1883 msgid "Snapshots are read-only and can't have their configuration changed" msgstr "" #: cmd/incus/info.go:808 cmd/incus/storage_volume.go:1333 msgid "Snapshots:" msgstr "" #: cmd/incus/info.go:507 #, c-format msgid "Socket %d:" msgstr "" #: cmd/incus/action.go:454 #, fuzzy, c-format msgid "Some instances failed to %s" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/top.go:439 msgid "Sorting Method:" msgstr "" #: cmd/incus/admin_recover.go:119 msgid "" "Source of the storage pool (block device, volume group, dataset, path, ... " "as applicable):" msgstr "" #: cmd/incus/image.go:1008 msgid "Source:" msgstr "" #: cmd/incus/storage_volume.go:3986 msgid "Specific address to listen on" msgstr "" #: cmd/incus/action.go:36 cmd/incus/action.go:37 #, fuzzy msgid "Start instances" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/info.go:683 #, fuzzy, c-format msgid "Started: %s" msgstr "Erstellt: %s" #: cmd/incus/launch.go:105 #, c-format msgid "Starting %s" msgstr "" #: cmd/incus/admin_recover.go:237 msgid "Starting recovery..." msgstr "" #: cmd/incus/info.go:765 #, fuzzy msgid "State" msgstr "Erstellt: %s" #: cmd/incus/network.go:916 #, fuzzy, c-format msgid "State: %s" msgstr "Erstellt: %s" #: cmd/incus/info.go:842 msgid "Stateful" msgstr "" #: cmd/incus/info.go:646 #, c-format msgid "Status: %s" msgstr "" #: cmd/incus/action.go:126 cmd/incus/action.go:127 #, fuzzy msgid "Stop instances" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/publish.go:43 msgid "Stop the instance if currently running" msgstr "" #: cmd/incus/publish.go:121 #, fuzzy msgid "Stopping instance failed!" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/delete.go:113 #, fuzzy, c-format msgid "Stopping the instance %s failed: %s" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/admin_init.go:58 msgid "Storage backend to use (btrfs, dir, lvm or zfs, default: dir)" msgstr "" #: cmd/incus/storage_bucket.go:177 #, fuzzy, c-format msgid "Storage bucket %q created" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_bucket.go:231 #, fuzzy, c-format msgid "Storage bucket %q deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/storage_bucket.go:1048 #, fuzzy, c-format msgid "Storage bucket key %q added" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_bucket.go:1099 #, fuzzy, c-format msgid "Storage bucket key %q removed" msgstr "Profil %s erstellt\n" #: cmd/incus/admin_init_auto.go:51 msgid "Storage has already been configured" msgstr "" #: cmd/incus/admin_recover.go:104 #, fuzzy, c-format msgid "Storage pool %q is already on recover list" msgstr "Profil %s erstellt\n" #: cmd/incus/admin_recover.go:200 #, fuzzy, c-format msgid "Storage pool %q of type %q" msgstr "Profil %s erstellt\n" #: cmd/incus/storage.go:192 #, fuzzy, c-format msgid "Storage pool %s created" msgstr "Profil %s erstellt\n" #: cmd/incus/storage.go:243 #, fuzzy, c-format msgid "Storage pool %s deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/storage.go:190 #, fuzzy, c-format msgid "Storage pool %s pending on member %s" msgstr "Profil %s erstellt\n" #: cmd/incus/storage.go:118 #, fuzzy msgid "Storage pool description" msgstr "Profil %s erstellt\n" #: cmd/incus/copy.go:65 cmd/incus/create.go:68 cmd/incus/import.go:39 #: cmd/incus/move.go:66 #, fuzzy msgid "Storage pool name" msgstr "Profilname kann nicht geändert werden" #: cmd/incus/admin_init.go:61 #, fuzzy msgid "Storage pool to use or create" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_volume.go:627 #, fuzzy, c-format msgid "Storage volume %s created" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_volume.go:690 #, fuzzy, c-format msgid "Storage volume %s deleted" msgstr "Profil %s gelöscht\n" #: cmd/incus/storage_volume.go:441 #, fuzzy msgid "Storage volume copied successfully!" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_volume.go:445 #, fuzzy msgid "Storage volume moved successfully!" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_volume.go:3254 #, fuzzy, c-format msgid "Storage volume snapshot %s deleted from %s" msgstr "Profil %s gelöscht\n" #: cmd/incus/action.go:155 #, fuzzy msgid "Store the instance state" msgstr "Herunterfahren des Containers erzwingen." #: cmd/incus/cluster.go:1427 #, fuzzy msgid "Successfully updated cluster certificates" msgstr "Akzeptiere Zertifikat" #: cmd/incus/info.go:219 #, c-format msgid "Supported modes: %s" msgstr "" #: cmd/incus/info.go:223 #, c-format msgid "Supported ports: %s" msgstr "" #: cmd/incus/info.go:738 msgid "Swap (current)" msgstr "" #: cmd/incus/info.go:742 msgid "Swap (peak)" msgstr "" #: cmd/incus/project.go:936 cmd/incus/project.go:937 msgid "Switch the current project" msgstr "" #: cmd/incus/remote.go:1264 cmd/incus/remote.go:1265 msgid "Switch the default remote" msgstr "" #: cmd/incus/file.go:146 cmd/incus/storage_volume.go:2203 msgid "Symlink target path can only be used for type \"symlink\"" msgstr "" #: cmd/incus/info.go:402 msgid "System:" msgstr "" #: cmd/incus/snapshot.go:366 cmd/incus/storage_volume.go:3378 msgid "TAKEN AT" msgstr "" #: cmd/incus/alias.go:149 msgid "TARGET" msgstr "" #: cmd/incus/cluster.go:1144 cmd/incus/config_trust.go:601 msgid "TOKEN" msgstr "" #: cmd/incus/config_trust.go:414 cmd/incus/image.go:1106 #: cmd/incus/image_alias.go:248 cmd/incus/list.go:490 cmd/incus/network.go:1072 #: cmd/incus/network.go:1268 cmd/incus/network_allocations.go:82 #: cmd/incus/network_integration.go:429 cmd/incus/network_peer.go:141 #: cmd/incus/operation.go:165 cmd/incus/storage_volume.go:1567 #: cmd/incus/warning.go:216 msgid "TYPE" msgstr "" #: cmd/incus/info.go:840 cmd/incus/info.go:891 cmd/incus/storage_volume.go:1405 msgid "Taken at" msgstr "" #: cmd/incus/cluster.go:995 msgid "Target isn't a cluster" msgstr "" #: cmd/incus/export.go:74 cmd/incus/storage_bucket.go:1329 #: cmd/incus/storage_volume.go:3703 #, fuzzy, c-format msgid "Target path %q already exists" msgstr "entfernte Instanz %s existiert bereits" #: cmd/incus/file.go:977 cmd/incus/storage_volume.go:2458 #, fuzzy msgid "Target path and --listen flag cannot be used together" msgstr "--refresh kann nur mit Containern verwendet werden" #: cmd/incus/file.go:971 cmd/incus/storage_volume.go:2453 msgid "Target path must be a directory" msgstr "" #: cmd/incus/cluster.go:999 #, fuzzy msgid "Target server is already clustered" msgstr "entfernte Instanz %s existiert bereits" #: cmd/incus/admin_shutdown.go:34 msgid "Tell the daemon to shutdown all instances and exit" msgstr "" #: cmd/incus/admin_shutdown.go:35 msgid "" "Tell the daemon to shutdown all instances and exit\n" "\n" " This will tell the daemon to start a clean shutdown of all instances,\n" " followed by having itself shutdown and exit.\n" "\n" " This can take quite a while as instances can take a long time to\n" " shutdown, especially if a non-standard timeout was configured for them." msgstr "" #: cmd/incus/admin_init_interactive.go:308 #, fuzzy, c-format msgid "The %s storage pool already exists" msgstr "entfernte Instanz %s existiert bereits" #: cmd/incus/console.go:132 msgid "The --show-log flag is only supported for by 'console' output type" msgstr "" #: cmd/incus/admin_init_interactive.go:524 msgid "The LVM thin provisioning tools couldn't be found on the system" msgstr "" #: cmd/incus/admin_init_interactive.go:512 msgid "" "The LVM thin provisioning tools couldn't be found.\n" "LVM can still be used without thin provisioning but this will disable over-" "provisioning,\n" "increase the space requirements and creation time of images, instances and " "snapshots.\n" "\n" "If you wish to use thin provisioning, abort now, install the tools from your " "Linux distribution\n" "and make sure that your user can see and run the \"thin_check\" command " "before running \"init\" again." msgstr "" #: cmd/incus/admin_cluster.go:48 msgid "" "The \"cluster\" subcommand requires access to internal server data.\n" "To do so, it's actually part of the \"incusd\" binary rather than " "\"incus\".\n" "\n" "You can invoke it through \"incusd cluster\"." msgstr "" #: cmd/incus/console.go:404 msgid "" "The client automatically uses either spicy or remote-viewer when present." msgstr "" #: cmd/incus/config_device.go:157 cmd/incus/config_device.go:174 #: cmd/incus/config_device.go:400 #, fuzzy msgid "The device already exists" msgstr "entfernte Instanz %s existiert bereits" #: cmd/incus/admin_recover.go:198 msgid "The following unknown storage pools have been found:" msgstr "" #: cmd/incus/admin_recover.go:205 msgid "The following unknown volumes have been found:" msgstr "" #: cmd/incus/delete.go:97 #, c-format msgid "The instance %s is currently running, stop it first or pass --force" msgstr "" #: cmd/incus/publish.go:90 msgid "" "The instance is currently running. Use --force to have it stopped and " "restarted" msgstr "" #: cmd/incus/create.go:428 msgid "The instance you are starting doesn't have any network attached to it." msgstr "" #: cmd/incus/config.go:590 msgid "The is no config key to set on an instance snapshot." msgstr "" #: cmd/incus/cluster_group.go:857 #, fuzzy, c-format msgid "The key %q does not exist on cluster group %q" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/cluster.go:490 #, fuzzy, c-format msgid "The key %q does not exist on cluster member %q" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/utils.go:467 #, c-format msgid "The local image '%q' couldn't be found, trying '%q:%q' instead." msgstr "" #: cmd/incus/utils.go:463 #, c-format msgid "The local image '%q' couldn't be found, trying '%q:' instead." msgstr "" #: cmd/incus/top.go:160 msgid "The minimum refresh rate is 10s" msgstr "" #: cmd/incus/config_device.go:405 #, fuzzy msgid "The profile device doesn't exist" msgstr "entfernte Instanz %s existiert nicht" #: cmd/incus/cluster_group.go:848 #, fuzzy, c-format msgid "The property %q does not exist on the cluster group %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/cluster.go:481 #, fuzzy, c-format msgid "The property %q does not exist on the cluster member %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/config.go:455 #, fuzzy, c-format msgid "The property %q does not exist on the instance %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/config.go:431 #, fuzzy, c-format msgid "The property %q does not exist on the instance snapshot %s: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/network_load_balancer.go:433 #, fuzzy, c-format msgid "The property %q does not exist on the load balancer %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/network.go:841 #, fuzzy, c-format msgid "The property %q does not exist on the network %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/network_acl.go:327 #, fuzzy, c-format msgid "The property %q does not exist on the network ACL %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/network_forward.go:447 #, fuzzy, c-format msgid "The property %q does not exist on the network forward %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/network_integration.go:363 #, fuzzy, c-format msgid "The property %q does not exist on the network integration %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/network_peer.go:475 #, fuzzy, c-format msgid "The property %q does not exist on the network peer %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/network_zone.go:343 #, fuzzy, c-format msgid "The property %q does not exist on the network zone %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/network_zone.go:993 #, fuzzy, c-format msgid "The property %q does not exist on the network zone record %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/profile.go:647 #, fuzzy, c-format msgid "The property %q does not exist on the profile %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/project.go:471 #, fuzzy, c-format msgid "The property %q does not exist on the project %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/storage_bucket.go:404 #, fuzzy, c-format msgid "The property %q does not exist on the storage bucket %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/storage.go:437 #, fuzzy, c-format msgid "The property %q does not exist on the storage pool %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/storage_volume.go:1193 #, fuzzy, c-format msgid "The property %q does not exist on the storage pool volume %q: %v" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/storage_volume.go:1165 #, c-format msgid "" "The property %q does not exist on the storage pool volume snapshot %s/%s: %v" msgstr "" #: cmd/incus/utils_properties.go:117 #, fuzzy, c-format msgid "The property with tag %q does not exist" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/admin_recover.go:149 msgid "The recovery process will be scanning the following storage pools:" msgstr "" #: cmd/incus/admin_init_auto.go:26 #, c-format msgid "" "The requested backend '%s' isn't available on your system (missing tools)" msgstr "" #: cmd/incus/admin_init_auto.go:22 #, c-format msgid "The requested backend '%s' isn't supported by init" msgstr "" #: cmd/incus/admin_init_interactive.go:122 msgid "The requested interface doesn't exist. Please choose another one." msgstr "" #: cmd/incus/admin_init_interactive.go:159 #, c-format msgid "" "The requested network bridge \"%s\" already exists. Please choose another " "name." msgstr "" #: cmd/incus/admin_init_interactive.go:304 #, c-format msgid "" "The requested storage pool \"%s\" already exists. Please choose another name." msgstr "" #: cmd/incus/webui_unix.go:57 msgid "The server doesn't have a web UI installed" msgstr "" #: cmd/incus/info.go:393 msgid "The server doesn't implement the newer v2 resources API" msgstr "" #: cmd/incus/network.go:533 #, fuzzy, c-format msgid "The specified NIC does not point to the given network (found %s)" msgstr "entfernte Instanz %s existiert nicht" #: cmd/incus/network.go:529 #, fuzzy, c-format msgid "The specified device is not a NIC (%s device)" msgstr "entfernte Instanz %s existiert nicht" #: cmd/incus/storage_volume.go:742 #, fuzzy, c-format msgid "The specified device is not a disk (%s device)" msgstr "entfernte Instanz %s existiert nicht" #: cmd/incus/storage_volume.go:750 #, fuzzy, c-format msgid "" "The specified disk does not point to the given storage volume (found %s)" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/storage_volume.go:746 #, c-format msgid "The specified disk is not in the given pool (found %s)" msgstr "" #: cmd/incus/file.go:113 cmd/incus/storage_volume.go:2170 msgid "The type to create (file, symlink, or directory)" msgstr "" #: cmd/incus/main.go:404 msgid "" "This client hasn't been configured to use a remote server yet.\n" "As your platform can't run native Linux instances, you must connect to a " "remote server.\n" "\n" "If you already added a remote server, make it the default with \"incus " "remote switch NAME\"." msgstr "" #: cmd/incus/webui_windows.go:14 msgid "This command isn't supported on Windows" msgstr "" #: cmd/incus/usage/parse.go:111 msgid "" "This command was called with --explain; its arguments are valid, but no " "further processing is done" msgstr "" #: cmd/incus/admin_recover.go:61 msgid "This server currently has the following storage pools:" msgstr "" #: cmd/incus/cluster.go:822 msgid "This server is already clustered" msgstr "" #: cmd/incus/cluster.go:812 #, fuzzy msgid "This server is not available on the network" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/info.go:331 msgid "Threads:" msgstr "" #: cmd/incus/action.go:166 #, fuzzy msgid "Time to wait for the instance to shutdown cleanly" msgstr "Wartezeit bevor der Container gestoppt wird." #: cmd/incus/image.go:970 #, fuzzy msgid "Timestamps:" msgstr "Zeitstempel:\n" #: cmd/incus/create.go:430 msgid "To attach a network to an instance, use: incus network attach" msgstr "" #: cmd/incus/create.go:429 msgid "To create a new network, use: incus network create" msgstr "" #: cmd/incus/console.go:225 msgid "To detach from the console, press: +a q" msgstr "" #: cmd/incus/main.go:529 #, c-format msgid "" "To start your first container, try: incus launch images:%s\n" "Or for a virtual machine: incus launch images:%s --vm" msgstr "" #: cmd/incus/config.go:290 cmd/incus/config.go:475 cmd/incus/config.go:639 #: cmd/incus/config.go:724 cmd/incus/copy.go:129 cmd/incus/info.go:385 #: cmd/incus/network.go:897 cmd/incus/storage.go:495 msgid "To use --target, the destination remote must be a cluster" msgstr "" #: cmd/incus/storage_volume.go:1318 #, fuzzy, c-format msgid "Total: %s" msgstr "Erstellt: %s" #: cmd/incus/info.go:518 cmd/incus/info.go:529 cmd/incus/info.go:534 #: cmd/incus/info.go:540 #, c-format msgid "Total: %v" msgstr "" #: cmd/incus/info.go:231 #, fuzzy, c-format msgid "Transceiver type: %s" msgstr "unbekannter entfernter Instanz Name: %q" #: cmd/incus/storage_volume.go:1676 msgid "Transfer mode, one of pull (default), push or relay" msgstr "" #: cmd/incus/image.go:179 msgid "Transfer mode. One of pull (default), push or relay" msgstr "" #: cmd/incus/storage_volume.go:346 msgid "Transfer mode. One of pull (default), push or relay." msgstr "" #: cmd/incus/copy.go:62 msgid "Transfer mode. One of pull, push or relay" msgstr "" #: cmd/incus/move.go:64 msgid "Transfer mode. One of pull, push or relay." msgstr "" #: cmd/incus/image.go:768 #, c-format msgid "Transferring image: %s" msgstr "" #: cmd/incus/copy.go:340 cmd/incus/move.go:289 #, fuzzy, c-format msgid "Transferring instance: %s" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/network.go:943 msgid "Transmit policy" msgstr "" #: cmd/incus/remote.go:611 #, fuzzy, c-format msgid "Trust token for %s: " msgstr "Administrator Passwort für %s: " #: cmd/incus/action.go:305 cmd/incus/launch.go:138 #, c-format msgid "Try `incus info --show-log %s%s` for more info" msgstr "" #: cmd/incus/info.go:764 msgid "Type" msgstr "" #: cmd/incus/config_trust.go:179 #, fuzzy msgid "Type of certificate" msgstr "Akzeptiere Zertifikat" #: cmd/incus/console.go:54 msgid "" "Type of connection to establish: 'console' for serial console, 'vga' for " "SPICE graphical output" msgstr "" #: cmd/incus/network_peer.go:319 msgid "Type of peer (local or remote)" msgstr "" #: cmd/incus/image.go:968 cmd/incus/info.go:287 cmd/incus/info.go:432 #: cmd/incus/info.go:443 cmd/incus/info.go:661 cmd/incus/network.go:917 #: cmd/incus/storage_volume.go:1303 #, c-format msgid "Type: %s" msgstr "" #: cmd/incus/project.go:1048 msgid "UNLIMITED" msgstr "" #: cmd/incus/image.go:1107 msgid "UPLOAD DATE" msgstr "" #: cmd/incus/cluster.go:196 cmd/incus/remote.go:120 cmd/incus/remote.go:985 #: cmd/incus/usage/usage.go:859 msgid "URL" msgstr "" #: cmd/incus/project.go:1078 cmd/incus/storage_volume.go:1572 msgid "USAGE" msgstr "" #: cmd/incus/info.go:580 #, fuzzy msgid "USB device:" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/info.go:583 #, fuzzy msgid "USB devices:" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/network.go:1077 cmd/incus/network_acl.go:164 #: cmd/incus/network_address_set.go:157 cmd/incus/network_allocations.go:80 #: cmd/incus/network_integration.go:430 cmd/incus/network_zone.go:146 #: cmd/incus/profile.go:726 cmd/incus/project.go:549 cmd/incus/storage.go:688 #: cmd/incus/storage_volume.go:1571 msgid "USED BY" msgstr "" #: cmd/incus/warning.go:217 msgid "UUID" msgstr "" #: cmd/incus/info.go:146 cmd/incus/info.go:404 #, c-format msgid "UUID: %v" msgstr "" #: cmd/incus/cluster.go:1740 cmd/incus/cluster.go:1935 #, fuzzy msgid "Unable to connect to any of the cluster members specified in join token" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/file.go:390 cmd/incus/storage_volume.go:2529 #, c-format msgid "Unable to create a temporary file: %v" msgstr "" #: cmd/incus/remote.go:256 cmd/incus/remote.go:290 #, fuzzy msgid "Unavailable remote server" msgstr "Neue entfernte Server hinzufügen" #: cmd/incus/config_trust.go:198 #, fuzzy, c-format msgid "Unknown certificate type %q" msgstr "Unbekannter Befehl %s für Abbild" #: cmd/incus/utils.go:703 #, fuzzy, c-format msgid "Unknown channel type for client %q: %s" msgstr "Unbekannter Befehl %s für Abbild" #: cmd/incus/cluster.go:217 cmd/incus/cluster.go:1159 #: cmd/incus/cluster_group.go:479 cmd/incus/config_trust.go:435 #: cmd/incus/config_trust.go:616 cmd/incus/image.go:1122 #: cmd/incus/image_alias.go:264 cmd/incus/list.go:546 cmd/incus/network.go:1097 #: cmd/incus/network.go:1286 cmd/incus/network_allocations.go:98 #: cmd/incus/network_forward.go:163 cmd/incus/network_integration.go:444 #: cmd/incus/network_load_balancer.go:169 cmd/incus/network_peer.go:156 #: cmd/incus/network_zone.go:164 cmd/incus/operation.go:187 #: cmd/incus/profile.go:747 cmd/incus/project.go:564 cmd/incus/remote.go:1004 #: cmd/incus/snapshot.go:382 cmd/incus/storage.go:704 #: cmd/incus/storage_bucket.go:504 cmd/incus/storage_bucket.go:888 #: cmd/incus/storage_volume.go:1603 cmd/incus/storage_volume.go:3393 #: cmd/incus/top.go:101 cmd/incus/warning.go:242 #, c-format msgid "Unknown column shorthand char '%c' in '%s'" msgstr "" #: cmd/incus/console.go:162 #, fuzzy, c-format msgid "Unknown console type %q" msgstr "Unbekannter Befehl %s für Abbild" #: cmd/incus/utils_sftp.go:257 #, fuzzy, c-format msgid "Unknown file type '%s'" msgstr "Unbekannter Befehl %s für Abbild" #: cmd/incus/usage/usage.go:376 #, fuzzy, c-format msgid "Unknown flag --%s" msgstr "Unbekannter Befehl %s für Abbild" #: cmd/incus/network_acl.go:892 cmd/incus/network_acl.go:1011 #, fuzzy, c-format msgid "Unknown key: %s" msgstr "Unbekannter Befehl %s für Abbild" #: cmd/incus/console.go:122 #, fuzzy, c-format msgid "Unknown output type %q" msgstr "Unbekannter Befehl %s für Abbild" #: cmd/incus/cluster_group.go:956 #, fuzzy msgid "Unset a cluster group's configuration keys" msgstr "Alternatives config Verzeichnis." #: cmd/incus/cluster.go:589 #, fuzzy msgid "Unset a cluster member's configuration keys" msgstr "Alternatives config Verzeichnis." #: cmd/incus/move.go:62 #, fuzzy msgid "Unset all profiles on the target instance" msgstr "nicht alle Profile der Quelle sind am Ziel vorhanden." #: cmd/incus/config_device.go:759 cmd/incus/config_device.go:760 #, fuzzy msgid "Unset device configuration keys" msgstr "Alternatives config Verzeichnis." #: cmd/incus/image.go:1651 cmd/incus/image.go:1652 msgid "Unset image properties" msgstr "" #: cmd/incus/config.go:805 cmd/incus/config.go:806 #, fuzzy msgid "Unset instance or server configuration keys" msgstr "Alternatives config Verzeichnis." #: cmd/incus/network_acl.go:530 cmd/incus/network_acl.go:531 #, fuzzy msgid "Unset network ACL configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_address_set.go:401 cmd/incus/network_address_set.go:402 #, fuzzy msgid "Unset network address set configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network.go:1571 cmd/incus/network.go:1572 msgid "Unset network configuration keys" msgstr "" #: cmd/incus/network_forward.go:573 #, fuzzy msgid "Unset network forward configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_forward.go:574 #, fuzzy msgid "Unset network forward keys" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_integration.go:638 cmd/incus/network_integration.go:639 #, fuzzy msgid "Unset network integration configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_load_balancer.go:559 #, fuzzy msgid "Unset network load balancer configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_load_balancer.go:560 #, fuzzy msgid "Unset network load balancer keys" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_peer.go:593 #, fuzzy msgid "Unset network peer configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_peer.go:594 #, fuzzy msgid "Unset network peer keys" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_zone.go:544 cmd/incus/network_zone.go:545 #, fuzzy msgid "Unset network zone configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/network_zone.go:1202 cmd/incus/network_zone.go:1203 #, fuzzy msgid "Unset network zone record configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/profile.go:1116 cmd/incus/profile.go:1117 msgid "Unset profile configuration keys" msgstr "" #: cmd/incus/project.go:842 cmd/incus/project.go:843 #, fuzzy msgid "Unset project configuration keys" msgstr "Profil %s erstellt\n" #: cmd/incus/storage_bucket.go:755 cmd/incus/storage_bucket.go:756 #, fuzzy msgid "Unset storage bucket configuration keys" msgstr "Alternatives config Verzeichnis." #: cmd/incus/storage.go:967 cmd/incus/storage.go:968 msgid "Unset storage pool configuration keys" msgstr "" #: cmd/incus/storage_volume.go:2045 #, fuzzy msgid "Unset storage volume configuration keys" msgstr "Alternatives config Verzeichnis." #: cmd/incus/storage_volume.go:2046 #, fuzzy msgid "" "Unset storage volume configuration keys\n" "\n" "If the type is not specified, Incus assumes the type is \"custom\".\n" "Supported values for type are \"custom\", \"container\" and \"virtual-" "machine\"." msgstr "Profil %s erstellt\n" #: cmd/incus/cluster_group.go:959 #, fuzzy msgid "Unset the key as a cluster group property" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/cluster.go:592 msgid "Unset the key as a cluster property" msgstr "" #: cmd/incus/network_acl.go:534 msgid "Unset the key as a network ACL property" msgstr "" #: cmd/incus/network_address_set.go:405 #, fuzzy msgid "Unset the key as a network address set property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_forward.go:577 #, fuzzy msgid "Unset the key as a network forward property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_integration.go:643 #, fuzzy msgid "Unset the key as a network integration property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_load_balancer.go:563 #, fuzzy msgid "Unset the key as a network load balancer property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_peer.go:597 #, fuzzy msgid "Unset the key as a network peer property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network.go:1575 #, fuzzy msgid "Unset the key as a network property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/network_zone.go:548 #, fuzzy msgid "Unset the key as a network zone property" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/network_zone.go:1206 #, fuzzy msgid "Unset the key as a network zone record property" msgstr "Profil %s erstellt\n" #: cmd/incus/profile.go:1120 msgid "Unset the key as a profile property" msgstr "" #: cmd/incus/project.go:846 msgid "Unset the key as a project property" msgstr "" #: cmd/incus/storage_bucket.go:759 #, fuzzy msgid "Unset the key as a storage bucket property" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/storage.go:972 #, fuzzy msgid "Unset the key as a storage property" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/storage_volume.go:2058 #, fuzzy msgid "Unset the key as a storage volume property" msgstr "Kein Zertifikat für diese Verbindung" #: cmd/incus/config.go:810 msgid "Unset the key as an instance property" msgstr "" #: cmd/incus/info.go:912 #, c-format msgid "Unsupported instance type: %s" msgstr "" #: cmd/incus/network.go:944 msgid "Up delay" msgstr "" #: cmd/incus/cluster.go:1344 #, fuzzy msgid "Update cluster certificate" msgstr "Akzeptiere Zertifikat" #: cmd/incus/cluster.go:1346 msgid "" "Update cluster certificate with PEM certificate and key read from input " "files." msgstr "" #: cmd/incus/profile.go:274 msgid "Update the target profile from the source if it already exists" msgstr "" #: cmd/incus/top.go:238 #, c-format msgid "Updated interval to %v" msgstr "" #: cmd/incus/image.go:976 #, c-format msgid "Uploaded: %s" msgstr "" #: cmd/incus/network.go:960 #, fuzzy msgid "Upper devices" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/color/color.go:37 #, fuzzy msgid "Usage:" msgstr "Erstellt: %s" #: cmd/incus/storage_volume.go:1316 #, fuzzy, c-format msgid "Usage: %s" msgstr "Erstellt: %s" #: cmd/incus/main.go:50 msgid "" "Use \"{{.CommandPath}} [] --help\" for more information about a " "command." msgstr "" #: cmd/incus/export.go:51 msgid "" "Use storage driver optimized format (can only be restored on a similar pool)" msgstr "" #: cmd/incus/storage_volume.go:3647 msgid "" "Use storage driver optimized format (can only be restored on a similar pool, " "ignored for ISO storage volumes)" msgstr "" #: cmd/incus/main.go:166 msgid "Use with help or --help to view sub-commands" msgstr "" #: cmd/incus/info.go:517 cmd/incus/info.go:528 cmd/incus/info.go:533 #: cmd/incus/info.go:539 #, c-format msgid "Used: %v" msgstr "" #: cmd/incus/exec.go:68 msgid "User ID to run the command as (default 0)" msgstr "" #: cmd/incus/cluster.go:1787 #, fuzzy msgid "User aborted configuration" msgstr "Profil %s erstellt\n" #: cmd/incus/cluster.go:725 cmd/incus/delete.go:57 cmd/incus/project.go:226 #: cmd/incus/snapshot.go:280 msgid "User aborted delete operation" msgstr "" #: cmd/incus/info.go:154 cmd/incus/info.go:263 #, c-format msgid "VFs: %d" msgstr "" #: cmd/incus/network.go:968 msgid "VLAN ID" msgstr "" #: cmd/incus/network.go:959 msgid "VLAN filtering" msgstr "" #: cmd/incus/network.go:966 msgid "VLAN:" msgstr "" #: cmd/incus/info.go:348 cmd/incus/info.go:361 cmd/incus/info.go:375 #, fuzzy, c-format msgid "Vendor ID: %v" msgstr "Fehler: %v\n" #: cmd/incus/info.go:439 cmd/incus/info.go:459 cmd/incus/info.go:479 #, fuzzy, c-format msgid "Vendor: %s" msgstr "Fehler: %v\n" #: cmd/incus/info.go:313 cmd/incus/info.go:347 cmd/incus/info.go:360 #: cmd/incus/info.go:374 cmd/incus/info.go:408 #, fuzzy, c-format msgid "Vendor: %v" msgstr "Fehler: %v\n" #: cmd/incus/info.go:107 cmd/incus/info.go:193 #, c-format msgid "Vendor: %v (%v)" msgstr "" #: cmd/incus/info.go:252 #, fuzzy, c-format msgid "Verb: %s (%s)" msgstr "" "Benutzung: %s\n" "\n" "Optionen:\n" "\n" #: cmd/incus/info.go:447 cmd/incus/info.go:471 cmd/incus/info.go:483 #, fuzzy, c-format msgid "Version: %s" msgstr "Fingerabdruck: %s\n" #: cmd/incus/info.go:420 #, fuzzy, c-format msgid "Version: %v" msgstr "Fehler: %v\n" #: cmd/incus/storage_volume.go:1407 msgid "Volume Only" msgstr "" #: cmd/incus/storage_volume.go:556 #, fuzzy msgid "Volume description" msgstr "Fingerabdruck: %s\n" #: cmd/incus/storage_volume.go:3181 #, fuzzy, c-format msgid "Volume snapshot name is: %s" msgstr "'/' ist kein gültiges Zeichen im Namen eines Sicherungspunktes\n" #: cmd/incus/info.go:293 #, c-format msgid "WWN: %s" msgstr "" #: cmd/incus/wait.go:30 msgid "Wait for an instance to satisfy a condition" msgstr "" #: cmd/incus/wait.go:31 msgid "" "Wait for an instance to satisfy a condition\n" "\n" "Supported Conditions:\n" "\n" " agent Wait for the VM agent to be running\n" " ip Wait for any globally routable IP address\n" " ipv4 Wait for a globally routable IPv4 address\n" " ipv6 Wait for a globally routable IPv6 address\n" " status=STATUS Wait for the instance status to become STATUS" msgstr "" #: cmd/incus/admin_waitready.go:30 msgid "Wait for the daemon to be ready to process requests" msgstr "" #: cmd/incus/admin_waitready.go:31 msgid "" "Wait for the daemon to be ready to process requests\n" "\n" " This command will block until the daemon is reachable over its REST API " "and\n" " is done with early start tasks like re-starting previously started\n" " containers." msgstr "" #: cmd/incus/query.go:46 msgid "Wait for the operation to complete" msgstr "" #: cmd/incus/color/color.go:35 msgid "Warning:" msgstr "" #: cmd/incus/admin_init_interactive.go:541 msgid "" "We detected that you are running inside an unprivileged container.\n" "This means that unless you manually configured your host otherwise,\n" "you will not have enough uids and gids to allocate to your containers.\n" "\n" "Your container's own allocation can be reused to avoid the problem.\n" "Doing so makes your nested containers slightly less safe as they could\n" "in theory attack their parent container and gain more privileges than\n" "they otherwise would." msgstr "" #: cmd/incus/webui_unix.go:105 #, c-format msgid "Web server running at: %s" msgstr "" #: cmd/incus/cluster.go:1668 msgid "What IP address or DNS name should be used to reach this server?" msgstr "" #: cmd/incus/admin_init_interactive.go:171 msgid "What IPv4 address should be used?" msgstr "" #: cmd/incus/admin_init_interactive.go:192 msgid "What IPv6 address should be used?" msgstr "" #: cmd/incus/cluster.go:1638 msgid "What member name should be used to identify this server in the cluster?" msgstr "" #: cmd/incus/admin_init_interactive.go:152 msgid "What should the new bridge be called?" msgstr "" #: cmd/incus/admin_init_interactive.go:341 #, fuzzy msgid "Where should this storage pool store its data?" msgstr "entfernte Instanz %s existiert bereits" #: cmd/incus/export.go:50 #, fuzzy msgid "Whether or not to only backup the instance (without dependent volumes)" msgstr "Zustand des laufenden Containers sichern oder nicht" #: cmd/incus/export.go:49 #, fuzzy msgid "Whether or not to only backup the instance (without snapshots)" msgstr "Zustand des laufenden Containers sichern oder nicht" #: cmd/incus/snapshot.go:535 #, fuzzy msgid "Whether or not to restore the instance's disk only" msgstr "Zustand des laufenden Containers sichern oder nicht" #: cmd/incus/snapshot.go:534 #, fuzzy msgid "" "Whether or not to restore the instance's running state from snapshot (if " "available)" msgstr "" "Laufenden Zustand des Containers aus dem Sicherungspunkt (falls vorhanden) " "wiederherstellen oder nicht" #: cmd/incus/snapshot.go:97 #, fuzzy msgid "Whether or not to snapshot the instance's running state" msgstr "Zustand des laufenden Containers sichern oder nicht" #: cmd/incus/rebuild.go:28 msgid "" "Wipe the instance root disk and re-initialize with a new image (or empty " "volume)." msgstr "" #: cmd/incus/admin_init_interactive.go:68 msgid "Would you like a YAML \"init\" preseed to be printed?" msgstr "" #: cmd/incus/admin_init_interactive.go:616 msgid "Would you like stale cached images to be updated automatically?" msgstr "" #: cmd/incus/admin_init_interactive.go:562 #, fuzzy msgid "Would you like the server to be available over the network?" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/admin_recover.go:228 msgid "Would you like those to be recovered?" msgstr "" #: cmd/incus/admin_init_interactive.go:183 msgid "Would you like to NAT IPv4 traffic on your bridge?" msgstr "" #: cmd/incus/admin_init_interactive.go:204 msgid "Would you like to NAT IPv6 traffic on your bridge?" msgstr "" #: cmd/incus/admin_recover.go:158 msgid "Would you like to continue with scanning for lost volumes?" msgstr "" #: cmd/incus/admin_init_interactive.go:356 #, c-format msgid "Would you like to create a new btrfs subvolume under %s?" msgstr "" #: cmd/incus/admin_init_interactive.go:102 msgid "Would you like to create a new local network bridge?" msgstr "" #: cmd/incus/admin_init_interactive.go:372 msgid "Would you like to create a new zfs dataset under rpool/incus?" msgstr "" #: cmd/incus/admin_init_interactive.go:550 msgid "Would you like to have your containers share their parent's allocation?" msgstr "" #: cmd/incus/admin_recover.go:75 msgid "Would you like to recover another storage pool?" msgstr "" #: cmd/incus/admin_init_interactive.go:109 msgid "Would you like to use an existing bridge or host interface?" msgstr "" #: cmd/incus/admin_init_interactive.go:432 msgid "" "Would you like to use an existing empty block device (e.g. a disk or " "partition)?" msgstr "" #: cmd/incus/admin_init_interactive.go:34 msgid "Would you like to use clustering?" msgstr "" #: cmd/incus/network.go:1117 cmd/incus/operation.go:217 #: cmd/incus/project.go:581 cmd/incus/project.go:590 cmd/incus/project.go:599 #: cmd/incus/project.go:608 cmd/incus/project.go:617 cmd/incus/project.go:626 #: cmd/incus/remote.go:1050 cmd/incus/remote.go:1059 cmd/incus/remote.go:1068 msgid "YES" msgstr "" #: cmd/incus/admin_recover.go:220 msgid "You are currently missing the following:" msgstr "" #: cmd/incus/exec.go:112 msgid "You can't pass -t and -T at the same time" msgstr "" #: cmd/incus/exec.go:116 #, fuzzy msgid "You can't pass -t or -T at the same time as --mode" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/copy.go:116 #, fuzzy msgid "You must specify a destination instance name" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/network_zone.go:382 #, fuzzy msgid "Zone description" msgstr "Fingerabdruck: %s\n" #: cmd/incus/usage/usage.go:809 #, fuzzy msgid "address" msgstr "Profil %s erstellt\n" #: cmd/incus/usage/usage.go:810 #, fuzzy msgid "address set" msgstr "Adresse: %s" #: cmd/incus/usage/usage.go:811 #, fuzzy msgid "alias" msgstr "" #: cmd/incus/info.go:654 #, fuzzy msgid "application" msgstr "Eigenschaften:\n" #: cmd/incus/usage/usage.go:812 msgid "backend" msgstr "" #: cmd/incus/usage/usage.go:813 msgid "backup file" msgstr "" #: cmd/incus/usage/usage.go:814 msgid "bucket" msgstr "" #: cmd/incus/usage/parse.go:77 #, c-format msgid "cannot parse this argument; unexpected %s" msgstr "" #: cmd/incus/usage/parse.go:80 #, c-format msgid "cannot parse this argument; unexpected %s in %s" msgstr "" #: cmd/incus/cluster.go:1338 cmd/incus/config_trust.go:162 msgid "cert.crt" msgstr "" #: cmd/incus/cluster.go:1338 msgid "cert.key" msgstr "" #: cmd/incus/usage/usage.go:815 msgid "client" msgstr "" #: cmd/incus/cluster.go:962 msgid "cluster" msgstr "" #: cmd/incus/alias.go:55 msgid "command" msgstr "" #: cmd/incus/usage/usage.go:816 #, fuzzy msgid "command-line argument" msgstr "Unbekannter Befehl %s für Abbild" #: cmd/incus/wait.go:25 #, fuzzy msgid "condition" msgstr "Fingerabdruck: %s\n" #: cmd/incus/project.go:672 cmd/incus/remote.go:1019 msgid "current" msgstr "" #: cmd/incus/storage.go:521 msgid "description" msgstr "" #: cmd/incus/usage/usage.go:817 #, fuzzy msgid "device" msgstr "kann nicht zum selben Container Namen kopieren" #: cmd/incus/usage/usage.go:819 #, fuzzy msgid "directory" msgstr "%s ist kein Verzeichnis" #: cmd/incus/image.go:955 msgid "disabled" msgstr "" #: cmd/incus/storage.go:520 cmd/incus/usage/usage.go:820 msgid "driver" msgstr "" #: cmd/incus/image.go:957 msgid "enabled" msgstr "" #: cmd/incus/info.go:658 msgid "ephemeral" msgstr "" #: cmd/incus/action.go:446 #, fuzzy, c-format msgid "error: %v" msgstr "Fehler: %v\n" #: cmd/incus/usage/parse.go:95 #, c-format msgid "expected %s" msgstr "" #: cmd/incus/usage/usage.go:822 msgid "expiry" msgstr "" #: cmd/incus/usage/usage.go:823 #, fuzzy msgid "file" msgstr "Profil %s erstellt\n" #: cmd/incus/usage/usage.go:824 msgid "filter" msgstr "" #: cmd/incus/usage/usage.go:825 #, fuzzy msgid "fingerprint" msgstr "Fingerabdruck: %s\n" #: cmd/incus/usage/usage.go:826 msgid "group" msgstr "" #: cmd/incus/usage/usage.go:827 msgid "image" msgstr "" #: cmd/incus/alias.go:63 msgid "" "incus alias add list \"list -c ns46S\"\n" " Overwrite the \"list\" command to pass -c ns46S." msgstr "" #: cmd/incus/alias.go:223 msgid "" "incus alias remove my-list\n" " Remove the \"my-list\" alias." msgstr "" #: cmd/incus/alias.go:169 msgid "" "incus alias rename list my-list\n" " Rename existing alias \"list\" to \"my-list\"." msgstr "" #: cmd/incus/cluster.go:857 msgid "" "incus cluster edit < member.yaml\n" " Update a cluster member using the content of member.yaml" msgstr "" #: cmd/incus/cluster_group.go:106 msgid "" "incus cluster group assign foo default,bar\n" " Set the groups for \"foo\" to \"default\" and \"bar\".\n" "\n" "incus cluster group assign foo default\n" " Reset \"foo\" to only using the \"default\" cluster group." msgstr "" "incus cluster group assign foo default,bar\n" " Weist dem Clustermitglied \"foo\" die Gruppen \"default\" und \"bar\" " "zu.\n" "\n" "incus cluster group assign foo default\n" " Setzt die Gruppen des Clustermitglied zurück und weist nur die Gruppe " "\"default\" zu." #: cmd/incus/cluster_group.go:174 #, fuzzy msgid "" "incus cluster group create g1\n" " Create a cluster group named g1\n" "\n" "incus cluster group create g1 < config.yaml\n" " Create a cluster group named g1 with configuration from config.yaml" msgstr "" "incus profile create p1\n" "\n" "incus profile create p1 < config.yaml\n" " Erstellt ein Profil namens \"p1\" unter Nutzung der Konfigurationsdatei " "\"config.yaml\"" #: cmd/incus/config_device.go:95 msgid "" "incus config device add [:]instance1 disk source=/" "share/c1 path=/opt\n" " Will mount the host's /share/c1 onto /opt in the instance.\n" "\n" "incus config device add [:]instance1 disk pool=some-" "pool source=some-volume path=/opt\n" " Will mount the some-volume volume on some-pool onto /opt in the instance." msgstr "" #: cmd/incus/config.go:98 msgid "" "incus config edit < instance.yaml\n" " Update the instance configuration from config.yaml." msgstr "" #: cmd/incus/config.go:512 msgid "" "incus config set [:] limits.cpu=2\n" " Will set a CPU limit of \"2\" for the instance.\n" "\n" "incus config set my-instance cloud-init.user-data - < cloud-init.yaml\n" " Sets the cloud-init user-data for instance \"my-instance\" by reading " "\"cloud-init.yaml\" through stdin.\n" "\n" "incus config set core.https_address=[::]:8443\n" " Will have the server listen on IPv4 and IPv6 port 8443." msgstr "" #: cmd/incus/config_template.go:72 #, fuzzy msgid "" "incus config template create u1 t1\n" " Create template t1 for instance u1\n" "\n" "incus config template create u1 t1 < config.tpl\n" " Create template t1 for instance u1 from config.tpl" msgstr "" "incus profile create p1\n" "\n" "incus profile create p1 < config.yaml\n" " Erstellt ein Profil namens \"p1\" unter Nutzung der Konfigurationsdatei " "\"config.yaml\"" #: cmd/incus/create.go:51 msgid "" "incus create images:debian/12 u1\n" " Create the instance u1\n" "\n" "incus create images:debian/12 u1 < config.yaml\n" " Create the instance with configuration from config.yaml\n" "\n" "incus launch images:debian/12 v2 --vm -d root,size=50GiB -d " "root,io.bus=nvme\n" " Create and start a virtual machine, overriding the disk size and bus" msgstr "" #: cmd/incus/storage_volume.go:2580 msgid "" "incus custom volume file pull local v1/foo/etc/hosts .\n" " To pull /etc/hosts from the custom volume and write it to the current " "directory.\n" "\n" "incus file pull local v1 foo/etc/hosts -\n" " To pull /etc/hosts from the custom volume and write its output to " "standard output." msgstr "" #: cmd/incus/debug.go:49 msgid "" "incus debug dump-memory vm1 memory-dump.elf --format=elf\n" " Creates an ELF format memory dump of the vm1 instance." msgstr "" #: cmd/incus/exec.go:56 msgid "" "incus exec c1 bash\n" "\tRun the \"bash\" command in instance \"c1\"\n" "\n" "incus exec c1 -- ls -lh /\n" "\tRun the \"ls -lh /\" command in instance \"c1\"" msgstr "" #: cmd/incus/export.go:41 #, fuzzy msgid "" "incus export u1 backup0.tar.gz\n" "\tDownload a backup tarball of the u1 instance.\n" "\n" "incus export u1 -\n" "\tDownload a backup tarball with it written to the standard output." msgstr "" "incus storage bucket export default b1\n" " Erstellen einer Sicherungsdatei (backup) vom storage bucket namens " "\"b1\" im pool \"default\"." #: cmd/incus/file.go:101 msgid "" "incus file create foo/bar\n" " To create a file /bar in the foo instance.\n" "\n" "incus file create --type=symlink foo/bar baz\n" " To create a symlink /bar in instance foo whose target is baz." msgstr "" #: cmd/incus/file.go:922 msgid "" "incus file mount foo/root fooroot\n" " To mount /root from the instance foo onto the local fooroot directory.\n" "\n" "incus file mount foo\n" " To start an SSH SFTP listener for the root filesystem of instance foo." msgstr "" #: cmd/incus/file.go:439 msgid "" "incus file pull foo/etc/hosts .\n" " To pull /etc/hosts from the instance and write it to the current " "directory.\n" "\n" "incus file pull foo/etc/hosts -\n" " To pull /etc/hosts from the instance and write its output to standard " "output." msgstr "" #: cmd/incus/file.go:667 msgid "" "incus file push /etc/hosts foo/etc/hosts\n" " To push /etc/hosts into the instance \"foo\".\n" "\n" "echo \"Hello world\" | incus file push - foo/root/test\n" " To read \"Hello world\" from standard input and write it into /root/test " "in instance \"foo\"." msgstr "" #: cmd/incus/image.go:393 msgid "" "incus image edit \n" " Launch a text editor to edit the properties\n" "\n" "incus image edit < image.yaml\n" " Load the image properties from a YAML file" msgstr "" #: cmd/incus/import.go:34 msgid "" "incus import backup0.tar.gz\n" " Create a new instance using backup0.tar.gz as the source." msgstr "" #: cmd/incus/info.go:42 msgid "" "incus info [:] [--show-log]\n" " For instance information.\n" "\n" "incus info [:] [--resources]\n" " For server information." msgstr "" #: cmd/incus/launch.go:27 msgid "" "incus launch images:debian/12 u1\n" " Create and start a container named u1\n" "\n" "incus launch images:debian/12 u1 < config.yaml\n" " Create and start a container with configuration from config.yaml\n" "\n" "incus launch images:debian/12 u2 -t aws:t2.micro\n" " Create and start a container using the same size as an AWS t2.micro (1 " "vCPU, 1GiB of RAM)\n" "\n" "incus launch images:debian/12 v1 --vm -c limits.cpu=4 -c limits.memory=4GiB\n" " Create and start a virtual machine with 4 vCPUs and 4GiB of RAM\n" "\n" "incus launch images:debian/12 v2 --vm -d root,size=50GiB -d " "root,io.bus=nvme\n" " Create and start a virtual machine, overriding the disk size and bus" msgstr "" #: cmd/incus/list.go:130 msgid "" "incus list -c " "nFs46,volatile.eth0.hwaddr:MAC,config:image.os,devices:eth0.parent:ETHP\n" " Show instances using the \"NAME\", \"BASE IMAGE\", \"STATE\", \"IPV4\", " "\"IPV6\" and \"MAC\" columns.\n" " \"BASE IMAGE\", \"MAC\" and \"IMAGE OS\" are custom columns generated from " "instance configuration keys.\n" " \"ETHP\" is a custom column generated from a device key.\n" "\n" "incus list -c ns,user.comment:comment\n" " List instances with their running state and user comment." msgstr "" #: cmd/incus/monitor.go:42 msgid "" "incus monitor --type=logging\n" " Only show log messages.\n" "\n" "incus monitor --pretty --type=logging --loglevel=info\n" " Show a pretty log of messages with info level or higher.\n" "\n" "incus monitor --type=lifecycle\n" " Only show lifecycle events." msgstr "" #: cmd/incus/move.go:51 #, fuzzy msgid "" "incus move [:] [:][] " "[--instance-only]\n" " Move an instance between two hosts, renaming it if destination name " "differs.\n" "\n" "incus move [--instance-only]\n" " Rename a local instance." msgstr "" "Verschiebt Container innerhalb einer oder zwischen lxd Instanzen\n" "\n" "lxc move \n" #: cmd/incus/network_acl.go:358 #, fuzzy msgid "" "incus network acl create a1\n" " Create network acl a1\n" "\n" "incus network acl create a1 < config.yaml\n" " Create network acl with configuration from config.yaml" msgstr "" "incus project create p1\n" " Erstellt ein Projekt namens \"p1\"\n" "\n" "incus project create p1 < config.yaml\n" " Erstellt ein Projekt namens \"p1\" mit der Konfigurationsdatei " "\"config.yaml\"" #: cmd/incus/network_address_set.go:233 #, fuzzy msgid "" "incus network address-set create as1\n" " Create network address set as1\n" "\n" "incus network address-set create as1 < config.yaml\n" " Create network address set with configuration from config.yaml" msgstr "" "incus storage volume create p1 v1\n" "\n" "incus storage volume create p1 v1 < config.yaml\n" "\tErstellt ein storage volume mit dem Namen \"v1\" im pool \"p1\" mit den " "Konfigurationsdetails aus der Datei \"config.yaml\"." #: cmd/incus/network.go:339 msgid "" "incus network create foo\n" " Create a new network called foo\n" "\n" "incus network create foo < config.yaml\n" " Create a new network called foo using the content of config.yaml.\n" "\n" "incus network create bar network=baz --type ovn\n" " Create a new OVN network called bar using baz as its uplink network" msgstr "" #: cmd/incus/network_forward.go:312 msgid "" "incus network forward create n1 127.0.0.1\n" "\n" "incus network forward create n1 127.0.0.1 < config.yaml\n" " Create a new network forward for network n1 from config.yaml" msgstr "" #: cmd/incus/network_integration.go:90 #, fuzzy msgid "" "incus network integration create o1 ovn\n" " Create network integration o1 of type ovn\n" "\n" "incus network integration create o1 ovn < config.yaml\n" " Create network integration o1 of type ovn with configuration from " "config.yaml" msgstr "" "incus project create p1\n" " Erstellt ein Projekt namens \"p1\"\n" "\n" "incus project create p1 < config.yaml\n" " Erstellt ein Projekt namens \"p1\" mit der Konfigurationsdatei " "\"config.yaml\"" #: cmd/incus/network_integration.go:225 msgid "" "incus network integration edit < network-" "integration.yaml\n" " Update a network integration using the content of network-" "integration.yaml" msgstr "" #: cmd/incus/network_load_balancer.go:314 #, fuzzy msgid "" "incus network load-balancer create n1 127.0.0.1\n" " Create network load-balancer for network n1\n" "\n" "incus network load-balancer create n1 127.0.0.1 < config.yaml\n" " Create network load-balancer for network n1 with configuration from " "config.yaml" msgstr "" "incus storage volume create p1 v1\n" "\n" "incus storage volume create p1 v1 < config.yaml\n" "\tErstellt ein storage volume mit dem Namen \"v1\" im pool \"p1\" mit den " "Konfigurationsdetails aus der Datei \"config.yaml\"." #: cmd/incus/network_peer.go:307 msgid "" "incus network peer create default peer1 web/default\n" " Create a new peering between network \"default\" in the current project " "and network \"default\" in the \"web\" project\n" "\n" "incus network peer create default peer2 ovn-ic --type=remote\n" " Create a new peering between network \"default\" in the current project " "and other remote networks through the \"ovn-ic\" integration\n" "\n" "incus network peer create default peer3 web/default < config.yaml\n" "\tCreate a new peering between network default in the current project and " "network default in the web project using the configuration\n" "\tin the file config.yaml" msgstr "" #: cmd/incus/network_zone.go:374 #, fuzzy msgid "" "incus network zone create z1\n" " Create network zone z1\n" "\n" "incus network zone create z1 < config.yaml\n" " Create network zone z1 with configuration from config.yaml" msgstr "" "incus project create p1\n" " Erstellt ein Projekt namens \"p1\"\n" "\n" "incus project create p1 < config.yaml\n" " Erstellt ein Projekt namens \"p1\" mit der Konfigurationsdatei " "\"config.yaml\"" #: cmd/incus/network_zone.go:1024 #, fuzzy msgid "" "incus network zone record create z1 r1\n" " Create record r1 for zone z1\n" "\n" "incus network zone record create z1 r1 < config.yaml\n" " Create record r1 for zone z1 with configuration from config.yaml" msgstr "" "incus project create p1\n" " Erstellt ein Projekt namens \"p1\"\n" "\n" "incus project create p1 < config.yaml\n" " Erstellt ein Projekt namens \"p1\" mit der Konfigurationsdatei " "\"config.yaml\"" #: cmd/incus/operation.go:292 #, fuzzy msgid "" "incus operation show 344a79e4-d88a-45bf-9c39-c72c26f6ab8a\n" " Show details on that operation UUID" msgstr "" "incus operation show 344a79e4-d88a-45bf-9c39-c72c26f6ab8a\n" " Zeigt Details der genannten operation UUID" #: cmd/incus/profile.go:186 #, fuzzy msgid "" "incus profile assign foo default,bar\n" " Set the profiles for \"foo\" to \"default\" and \"bar\".\n" "\n" "incus profile assign foo default\n" " Reset \"foo\" to only using the \"default\" profile.\n" "\n" "incus profile assign foo --no-profiles\n" " Remove all profile assigned to \"foo\"" msgstr "" "incus profile assign foo default,bar\n" " Der Instanz \"foo\" die Profile \"default\" und \"bar\" zuweisen.\n" "\n" "incus profile assign foo default\n" " Die Profile der Instanz \"foo\" zurücksetzen und nur das Profil " "\"default\" zuweisen.\n" "\n" "incus profile assign foo ''\n" " Die Profile der Instanz \"foo\" zurücksetzen und kein Profil zuweisen." #: cmd/incus/profile.go:345 #, fuzzy msgid "" "incus profile create p1\n" " Create a profile named p1\n" "\n" "incus profile create p1 < config.yaml\n" " Create a profile named p1 with configuration from config.yaml" msgstr "" "incus profile create p1\n" "\n" "incus profile create p1 < config.yaml\n" " Erstellt ein Profil namens \"p1\" unter Nutzung der Konfigurationsdatei " "\"config.yaml\"" #: cmd/incus/config_device.go:102 msgid "" "incus profile device add [:]profile1 disk source=/" "share/c1 path=/opt\n" " Will mount the host's /share/c1 onto /opt in the instance.\n" "\n" "incus profile device add [:]profile1 disk pool=some-" "pool source=some-volume path=/opt\n" " Will mount the some-volume volume on some-pool onto /opt in the instance." msgstr "" "incus profile device add [:]profile1 disk source=/" "share/c1 path=/opt\n" " Mounted den Dateiordner \"/share/c1\" vom Host Computer in den Dateipfad " "\"/opt\" innerhalb der Instanz.\n" "\n" "incus profile device add [:]profile1 disk pool=some-" "pool source=some-volume path=/opt\n" " Mounted das Storage Volume \"some-volume\" im Storage pool \"some-pool\" " "in den Dateipfad \"/opt\" innerhalb der Instanz." #: cmd/incus/profile.go:479 msgid "" "incus profile edit < profile.yaml\n" " Update a profile using the content of profile.yaml" msgstr "" "incus profile edit < profile.yaml\n" " Automatisches Editieren eines Profils mithilfe der Konfigurationsdatei " "\"profile.yaml\"" #: cmd/incus/project.go:113 #, fuzzy msgid "" "incus project create p1\n" " Create a project named p1\n" "\n" "incus project create p1 < config.yaml\n" " Create a project named p1 with configuration from config.yaml" msgstr "" "incus project create p1\n" " Erstellt ein Projekt namens \"p1\"\n" "\n" "incus project create p1 < config.yaml\n" " Erstellt ein Projekt namens \"p1\" mit der Konfigurationsdatei " "\"config.yaml\"" #: cmd/incus/project.go:304 msgid "" "incus project edit < project.yaml\n" " Update a project using the content of project.yaml" msgstr "" "incus project edit < project.yaml\n" " Automatisches Editieren der Konfiguration eines Projekts mit der " "Konfigurationsdatei \"project.yaml\"" #: cmd/incus/query.go:40 msgid "" "incus query -X DELETE --wait /1.0/instances/c1\n" " Delete local instance \"c1\"." msgstr "" "incus query -X DELETE --wait /1.0/instances/c1\n" " Löscht die lokale Instanz \"c1\"." #: cmd/incus/snapshot.go:91 msgid "" "incus snapshot create u1 snap0\n" "\tCreate a snapshot of \"u1\" called \"snap0\".\n" "\n" "incus snapshot create u1 snap0 < config.yaml\n" "\tCreate a snapshot of \"u1\" called \"snap0\" with the configuration from " "\"config.yaml\"." msgstr "" "incus snapshot create u1 snap0\n" "\tErstellt einen snapshot \"snap0\" von der Instanz \"u1\".\n" "\n" "incus snapshot create u1 snap0 < config.yaml\n" "\tErstellt einen snapshot \"snap0\" von der Instanz \"u1\" mit der " "Konfigurationsdatei \"config.yaml\"." #: cmd/incus/snapshot.go:530 #, fuzzy msgid "" "incus snapshot restore u1 snap0\n" " Restore instance u1 to snapshot snap0" msgstr "" "incus snapshot create u1 snap0\n" "Erstellt einen snapshot \"snap0\" von der Instanz \"u1\".\n" "\n" "incus snapshot restore u1 snap0\n" "Stellt den Zustand vom snapshot \"snap0\" für die Instanz \"u1\" wieder her." #: cmd/incus/storage_bucket.go:108 #, fuzzy msgid "" "incus storage bucket create p1 b01\n" "\tCreate a new storage bucket named b01 in storage pool p1\n" "\n" "incus storage bucket create p1 b01 < config.yaml\n" "\tCreate a new storage bucket named b01 in storage pool p1 using the content " "of config.yaml" msgstr "" "incus storage bucket create p1 b01\n" "\tErstellt einen neuen storage bucket genannt \"b01\" im storage pool " "\"p1\"\n" "\n" "incus storage bucket create p1 b01 < config.yaml\n" "\tErstellt einen neuen storage bucket genannt \"b01\" im storage pool \"p1\" " "unter Nutzung der Konfigurationsdatei \"config.yaml\"" #: cmd/incus/storage_bucket.go:250 msgid "" "incus storage bucket edit [:] < bucket.yaml\n" " Update a storage bucket using the content of bucket.yaml." msgstr "" "incus storage bucket edit [:] < bucket.yaml\n" " Automatisches Editieren eines storage bucket mithilfe der " "Konfigurationsdetails aus der Datei \"bucket.yaml\"." #: cmd/incus/storage_bucket.go:1118 msgid "" "incus storage bucket edit [:] < key.yaml\n" " Update a storage bucket key using the content of key.yaml." msgstr "" "incus storage bucket edit [:] < key.yaml\n" " Automatisches Editieren eines storage bucket Schlüssels mithilfe der " "Konfigurationsdetails aus der Datei \"key.yaml\"." #: cmd/incus/storage_bucket.go:1294 #, fuzzy msgid "" "incus storage bucket export default b1\n" " Download a backup tarball of the b1 storage bucket from the default pool." msgstr "" "incus storage bucket export default b1\n" " Erstellen einer Sicherungsdatei (backup) vom storage bucket namens " "\"b1\" im pool \"default\"." #: cmd/incus/storage_bucket.go:1475 msgid "" "incus storage bucket import default backup0.tar.gz\n" "\t\tCreate a new storage bucket using backup0.tar.gz as the source." msgstr "" "incus storage bucket import default backup0.tar.gz\n" "\t\tErstellt einen neuen storage bucket aus der Datei \"backup0.tar.gz\"." #: cmd/incus/storage_bucket.go:974 msgid "" "incus storage bucket key create p1 b01 k1\n" "\tCreate a key called k1 for the bucket b01 in the pool p1.\n" "\n" "incus storage bucket key create p1 b01 k1 < config.yaml\n" "\tCreate a key called k1 for the bucket b01 in the pool p1 using the content " "of config.yaml." msgstr "" "incus storage bucket key create p1 b01 k1\n" "\tErzeugt einen Schlüssel genannt \"k1\" für einen storage bucket \"b01\" im " "pool \"p1\".\n" "\n" "incus storage bucket key create p1 b01 k1 < config.yaml\n" "\tErzeugt einen Schlüssel genannt \"k1\" für einen storage bucket \"b01\" im " "pool \"p1\" unter Nutzung der Konfigurationsdetails aus der Datei " "\"config.yaml\"." #: cmd/incus/storage_bucket.go:1237 msgid "" "incus storage bucket key show default data foo\n" " Will show the properties of a bucket key called \"foo\" for a bucket " "called \"data\" in the \"default\" pool." msgstr "" "incus storage bucket key show default data foo\n" " Zeigt die Eigenschaften eines storage bucket Schlüssels genannt " "\"foo\" für einen storage bucket namens \"data\" im pool \"default\"." #: cmd/incus/storage_bucket.go:701 msgid "" "incus storage bucket show default data\n" " Will show the properties of a bucket called \"data\" in the \"default\" " "pool." msgstr "" "incus storage bucket show default data\n" " Zeigt die Eigenschaften eines storage bucket namens \"data\" im pool " "\"default\"." #: cmd/incus/storage.go:110 #, fuzzy msgid "" "incus storage create s1 dir\n" " Create a storage pool s1\n" "\n" "incus storage create s1 dir < config.yaml\n" " Create a storage pool s1 using the content of config.yaml\n" "\t" msgstr "" "incus storage volume create p1 v1\n" "\n" "incus storage volume create p1 v1 < config.yaml\n" "\tErstellt ein storage volume mit dem Namen \"v1\" im pool \"p1\" mit den " "Konfigurationsdetails aus der Datei \"config.yaml\"." #: cmd/incus/storage.go:268 msgid "" "incus storage edit [:] < pool.yaml\n" " Update a storage pool using the content of pool.yaml." msgstr "" "incus storage edit [:] < pool.yaml\n" " Automatisches Editieren der Konfiguration des storage pool mit den " "Konfigurationsdetails aus der Datei \"pool.yaml\"." #: cmd/incus/storage_volume.go:548 #, fuzzy msgid "" "incus storage volume create default foo\n" " Create custom storage volume \"foo\" in pool \"default\"\n" "\n" "incus storage volume create default foo < config.yaml\n" " Create custom storage volume \"foo\" in pool \"default\" with " "configuration from config.yaml" msgstr "" "incus storage volume snapshot create default v1 snap0\n" " Erstellt einen Snapshot von \"v1\" im pool \"default\" mit dem Namen " "\"snap0\".\n" "\n" "incus storage volume snapshot create default v1 snap0 < config.yaml\n" " Erstellt einen Snapshot von \"v1\" im pool \"default\" mit dem Namen " "\"snap0\" mit den Konfigurationsdetails aus der angegebenen Datei " "\"config.yaml\"." #: cmd/incus/storage_volume.go:898 #, fuzzy msgid "" "incus storage volume edit default container/c1\n" " Edit container storage volume \"c1\" in pool \"default\"\n" "\n" "incus storage volume edit default foo < volume.yaml\n" " Edit custom storage volume \"foo\" in pool \"default\" using the content " "of volume.yaml" msgstr "" "incus storage volume snapshot create default v1 snap0\n" " Erstellt einen Snapshot von \"v1\" im pool \"default\" mit dem Namen " "\"snap0\".\n" "\n" "incus storage volume snapshot create default v1 snap0 < config.yaml\n" " Erstellt einen Snapshot von \"v1\" im pool \"default\" mit dem Namen " "\"snap0\" mit den Konfigurationsdetails aus der angegebenen Datei " "\"config.yaml\"." #: cmd/incus/storage_volume.go:2158 msgid "" "incus storage volume file create foo bar/baz\n" " To create a file baz in the bar volume on the foo pool.\n" "\n" "incus file create --type=symlink foo bar/baz qux\n" " To create a symlink qux in bar storage volume on the foo pool whose " "target is baz." msgstr "" #: cmd/incus/storage_volume.go:2400 msgid "" "incus storage volume file mount mypool myvolume localdir\n" " To mount the storage volume myvolume from pool mypool onto the local " "directory localdir.\n" "\n" "incus storage volume file mount mypool myvolume\n" " To start an SSH SFTP listener for the storage volume myvolume from pool " "mypool." msgstr "" #: cmd/incus/storage_volume.go:2775 msgid "" "incus storage volume file push /etc/hosts local v1/etc/hosts\n" " To push /etc/hosts into the custom volume \"v1\".\n" "\n" "echo \"Hello world\" | incus storage volume file push - local v1 test\n" " To read \"Hello world\" from standard input and write it into test in " "volume \"v1\"." msgstr "" #: cmd/incus/storage_volume.go:1107 msgid "" "incus storage volume get default data size\n" " Returns the size of a custom volume \"data\" in pool \"default\"\n" "\n" "incus storage volume get default virtual-machine/data snapshots.expiry\n" " Returns the snapshot expiration period for a virtual machine \"data\" in " "pool \"default\"" msgstr "" #: cmd/incus/storage_volume.go:3852 msgid "" "incus storage volume import default backup0.tar.gz\n" " Create a new custom volume using backup0.tar.gz as the source\n" "\n" "incus storage volume import default some-installer.iso installer --type=iso\n" " Create a new custom volume storing some-installer.iso for use as a CD-" "ROM image" msgstr "" #: cmd/incus/storage_volume.go:1225 msgid "" "incus storage volume info default foo\n" " Returns state information for a custom volume \"foo\" in pool " "\"default\"\n" "\n" "incus storage volume info default virtual-machine/v1\n" " Returns state information for virtual machine \"v1\" in pool \"default\"" msgstr "" #: cmd/incus/storage_volume.go:1812 msgid "" "incus storage volume set default data size=1GiB\n" " Sets the size of a custom volume \"data\" in pool \"default\" to 1 GiB\n" "\n" "incus storage volume set default virtual-machine/data snapshots.expiry=7d\n" " Sets the snapshot expiration period for a virtual machine \"data\" in " "pool \"default\" to seven days" msgstr "" #: cmd/incus/storage_volume.go:1959 msgid "" "incus storage volume show default foo\n" " Will show the properties of custom volume \"foo\" in pool \"default\"\n" "\n" "incus storage volume show default virtual-machine/v1\n" " Will show the properties of the virtual-machine volume \"v1\" in pool " "\"default\"\n" "\n" "incus storage volume show default container/c1\n" " Will show the properties of the container volume \"c1\" in pool " "\"default\"" msgstr "" #: cmd/incus/storage_volume.go:3047 #, fuzzy msgid "" "incus storage volume snapshot create default foo snap0\n" " Create a snapshot of \"foo\" in pool \"default\" called \"snap0\"\n" "\n" "incus storage volume snapshot create default vol1 snap0 < config.yaml\n" " Create a snapshot of \"foo\" in pool \"default\" called \"snap0\" with " "the configuration from \"config.yaml\"" msgstr "" "incus storage volume snapshot create default v1 snap0\n" " Erstellt einen Snapshot von \"v1\" im pool \"default\" mit dem Namen " "\"snap0\".\n" "\n" "incus storage volume snapshot create default v1 snap0 < config.yaml\n" " Erstellt einen Snapshot von \"v1\" im pool \"default\" mit dem Namen " "\"snap0\" mit den Konfigurationsdetails aus der angegebenen Datei " "\"config.yaml\"." #: cmd/incus/storage_volume.go:2051 msgid "" "incus storage volume unset default foo size\n" " Removes the size/quota of custom volume \"foo\" in pool \"default\"\n" "\n" "incus storage volume unset default virtual-machine/v1 snapshots.expiry\n" " Removes the snapshot expiration period of virtual machine volume \"v1\" " "in pool \"default\"" msgstr "" #: cmd/incus/wait.go:41 msgid "" "incus wait v1 agent\n" "\tWait for VM instance v1 to have a functional agent." msgstr "" #: cmd/incus/storage.go:518 msgid "info" msgstr "Info" #: cmd/incus/usage/usage.go:828 #, fuzzy msgid "instance" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/usage/usage.go:829 #, fuzzy msgid "interface" msgstr "Anhalten des Containers fehlgeschlagen!" #: cmd/incus/usage/usage.go:832 msgid "key" msgstr "" #: cmd/incus/usage/usage.go:830 #, fuzzy msgid "listen address" msgstr "der Name des Ursprung Containers muss angegeben werden" #: cmd/incus/usage/usage.go:831 msgid "listen port" msgstr "" #: cmd/incus/usage/usage.go:834 msgid "member" msgstr "" #: cmd/incus/remote.go:503 msgid "n" msgstr "n" #: cmd/incus/storage.go:519 msgid "name" msgstr "Name" #: cmd/incus/usage/usage.go:835 #, fuzzy msgid "network" msgstr "Profil %s erstellt\n" #: cmd/incus/usage/usage.go:836 #, fuzzy msgid "network integration" msgstr "Profil %s erstellt\n" #: cmd/incus/network_peer.go:299 #, fuzzy msgid "network or integration" msgstr "Profil %s erstellt\n" #: cmd/incus/usage/usage.go:894 #, fuzzy, c-format msgid "new %s name" msgstr "Name" #: cmd/incus/config_trust.go:478 cmd/incus/image.go:945 cmd/incus/image.go:950 #: cmd/incus/image.go:1164 msgid "no" msgstr "nein" #: cmd/incus/usage/parse.go:50 #, c-format msgid "not enough arguments; expected a value for %s" msgstr "" #: cmd/incus/remote.go:495 msgid "ok (y/n/[fingerprint])?" msgstr "ok (j/n/[fingerprint])?" #: cmd/incus/usage/parse.go:35 #, c-format msgid "one of %s or %s" msgstr "" #: cmd/incus/usage/usage.go:837 #, fuzzy msgid "operation" msgstr "Eigenschaften:\n" #: cmd/incus/usage/usage.go:838 msgid "path" msgstr "" #: cmd/incus/usage/usage.go:839 msgid "peer" msgstr "" #: cmd/incus/usage/legacysupport.go:8 msgid "please switch to the “=” syntax" msgstr "" #: cmd/incus/config.go:55 msgid "please use `incus profile`" msgstr "bitte nutzen Sie ìncus profile`" #: cmd/incus/usage/usage.go:840 msgid "pool" msgstr "" #: cmd/incus/usage/usage.go:841 msgid "port" msgstr "" #: cmd/incus/admin_init.go:36 msgid "preseed.yaml" msgstr "" #: cmd/incus/usage/usage.go:842 #, fuzzy msgid "profile" msgstr "Profil %s erstellt\n" #: cmd/incus/usage/usage.go:843 #, fuzzy msgid "project" msgstr "Alle Projekte" #: cmd/incus/usage/usage.go:844 #, fuzzy msgid "protocol" msgstr "Ungültiges Ziel %s" #: cmd/incus/usage/usage.go:845 msgid "query" msgstr "" #: cmd/incus/usage/usage.go:846 msgid "record" msgstr "" #: cmd/incus/usage/usage.go:847 #, fuzzy msgid "remote" msgstr "" "Ändert den Laufzustand eines Containers in %s.\n" "\n" "lxd %s \n" #: cmd/incus/usage/usage.go:851 msgid "role" msgstr "" #: cmd/incus/image.go:684 msgid "rootfs tarball" msgstr "" #: cmd/incus/usage/usage.go:852 msgid "snapshot" msgstr "" #: cmd/incus/remote_unix.go:35 msgid "socket file" msgstr "" #: cmd/incus/storage.go:523 msgid "space used" msgstr "Speicherplatz in Benutzung" #: cmd/incus/utils.go:601 msgid "sshfs has stopped" msgstr "sshfs wurde gestoppt" #: cmd/incus/utils.go:560 #, c-format msgid "sshfs mounting %q on %q" msgstr "sshfs mounted %q auf %q" #: cmd/incus/file.go:758 #, fuzzy msgid "stdin can only be used once, with no other source arguments" msgstr "--target kann nicht mit Instanzen verwendet werden" #: cmd/incus/usage/usage.go:854 msgid "symlink target path" msgstr "" #: cmd/incus/usage/usage.go:855 msgid "tarball" msgstr "" #: cmd/incus/usage/usage.go:900 #, fuzzy, c-format msgid "target %s" msgstr "" #: cmd/incus/usage/usage.go:856 msgid "template" msgstr "" #: cmd/incus/remote.go:120 cmd/incus/usage/usage.go:857 msgid "token" msgstr "" #: cmd/incus/usage/parse.go:67 #, c-format msgid "too many arguments; unexpected %s" msgstr "" #: cmd/incus/storage.go:522 msgid "total space" msgstr "Gesamter Speicherplatz" #: cmd/incus/usage/usage.go:853 cmd/incus/usage/usage.go:858 msgid "type" msgstr "" #: cmd/incus/usage/parse.go:91 #, fuzzy, c-format msgid "unexpected %s" msgstr "Architektur: %s\n" #: cmd/incus/usage/parse.go:98 #, c-format msgid "unexpected %s; expected %s" msgstr "" #: cmd/incus/usage/parse.go:47 msgid "unexpected end of argument; did you forget a suffix?" msgstr "" #: cmd/incus/version.go:39 msgid "unreachable" msgstr "nicht erreichbar" #: cmd/incus/storage.go:517 msgid "used by" msgstr "wird benutzt von" #: cmd/incus/usage/usage.go:860 msgid "value" msgstr "" #: cmd/incus/usage/usage.go:861 #, fuzzy msgid "volume" msgstr "Spalten" #: cmd/incus/usage/usage.go:862 msgid "warning UUID" msgstr "" #: cmd/incus/remote.go:505 msgid "y" msgstr "j" #: cmd/incus/cluster.go:724 cmd/incus/config_trust.go:475 #: cmd/incus/delete.go:56 cmd/incus/image.go:947 cmd/incus/image.go:952 #: cmd/incus/image.go:1161 cmd/incus/project.go:225 cmd/incus/snapshot.go:279 msgid "yes" msgstr "ja" #: cmd/incus/usage/usage.go:863 msgid "zone" msgstr "" #: cmd/incus/usage/utils.go:52 #, c-format msgid "“%s”" msgstr "" #~ msgid "Action (defaults to GET)" #~ msgstr "Aktion (Standard ist: GET)" #~ msgid "Can't supply uid/gid/mode in recursive mode" #~ msgstr "Kann uid/gid/mode im rekursiven Modus nicht bereitstellen" #, fuzzy, c-format #~ msgid "Failed connecting to instance SFTP: %s %w" #~ msgstr "kann nicht zum selben Container Namen kopieren" #, fuzzy #~ msgid "Target is not a directory" #~ msgstr "%s ist kein Verzeichnis" #, fuzzy, c-format #~ msgid "Serial: %v" #~ msgstr "Erstellt: %s" #, fuzzy, c-format #~ msgid "Failed to retrieve current server config: %w" #~ msgstr "Akzeptiere Zertifikat" #~ msgid "Can't pull a directory without --recursive" #~ msgstr "Verzeichnis kann ohne --recursive nicht abgerufen werde" #~ msgid "(none)" #~ msgstr "(kein Wert)" #~ msgid "--empty cannot be combined with an image name" #~ msgstr "--empty kann nicht mit einem image namen kombiniert werden" #~ msgid "A client name must be provided" #~ msgstr "Ein Name für den Client muss angegeben werden" #, fuzzy #~ msgid "A cluster member name must be provided" #~ msgstr "der Name des Ursprung Containers muss angegeben werden" #~ msgid "Alias name missing" #~ msgstr "Aliasbezeichnung fehlt" #, c-format #~ msgid "Bad key/value pair: %s" #~ msgstr "Ungültiges key/value Paar: %s" #, c-format #~ msgid "Bad key=value pair: %s" #~ msgstr "Ungültiges key=value Paar: %s" #, c-format #~ msgid "Bad property: %s" #~ msgstr "Ungültige Eigenschaft: %s" #~ msgid "Can't use an image with --empty" #~ msgstr "Kann kein Image mit --empty verwenden" #, fuzzy, c-format #~ msgid "Failed connecting to instance SFTP brrr: %s %w" #~ msgstr "kann nicht zum selben Container Namen kopieren" #, fuzzy, c-format #~ msgid "Failed to connect to cluster member: %w" #~ msgstr "kann nicht zum selben Container Namen kopieren" #, fuzzy, c-format #~ msgid "Failed to parse servers: %w" #~ msgstr "Akzeptiere Zertifikat" #, fuzzy #~ msgid "Filtering isn't supported yet" #~ msgstr "" #~ "Anzeigen von Informationen über entfernte Instanzen wird noch nicht " #~ "unterstützt\n" #, fuzzy, c-format #~ msgid "Image identifier missing: %s" #~ msgstr "Abbild mit Fingerabdruck %s importiert\n" #, fuzzy #~ msgid "Invalid database type" #~ msgstr "Ungültiges Ziel %s" #, fuzzy, c-format #~ msgid "Invalid instance name: %s" #~ msgstr "Ungültige Quelle %s" #, fuzzy, c-format #~ msgid "Invalid instance path: %q" #~ msgstr "Ungültige Quelle %s" #, fuzzy, c-format #~ msgid "Invalid path %s" #~ msgstr "Ungültiges Ziel %s" #, fuzzy #~ msgid "Invalid snapshot name" #~ msgstr "Ungültige Quelle %s" #, c-format #~ msgid "Invalid source %s" #~ msgstr "Ungültige Quelle %s" #, c-format #~ msgid "Invalid target %s" #~ msgstr "Ungültiges Ziel %s" #, fuzzy, c-format #~ msgid "Invalid volume %s" #~ msgstr "Ungültige Quelle %s" #, fuzzy #~ msgid "Missing bucket name" #~ msgstr "Fehlende Zusammenfassung." #, fuzzy #~ msgid "Missing certificate fingerprint" #~ msgstr "Fingerabdruck des Zertifikats: % x\n" #, fuzzy #~ msgid "Missing cluster group name" #~ msgstr "der Name des Ursprung Containers muss angegeben werden" #, fuzzy #~ msgid "Missing cluster member name" #~ msgstr "der Name des Ursprung Containers muss angegeben werden" #, fuzzy #~ msgid "Missing instance name" #~ msgstr "der Name des Ursprung Containers muss angegeben werden" #, fuzzy #~ msgid "Missing key name" #~ msgstr "Fehlende Zusammenfassung." #, fuzzy #~ msgid "Missing name" #~ msgstr "Fehlende Zusammenfassung." #, fuzzy #~ msgid "Missing network ACL name" #~ msgstr "Profilname kann nicht geändert werden" #, fuzzy #~ msgid "Missing network address set name" #~ msgstr "Profilname kann nicht geändert werden" #, fuzzy #~ msgid "Missing network integration name" #~ msgstr "Profilname kann nicht geändert werden" #, fuzzy #~ msgid "Missing network zone name" #~ msgstr "Profilname kann nicht geändert werden" #, fuzzy #~ msgid "Missing network zone record name" #~ msgstr "Profilname kann nicht geändert werden" #, fuzzy #~ msgid "Missing operation name" #~ msgstr "Profilname kann nicht geändert werden" #, fuzzy #~ msgid "Missing peer name" #~ msgstr "Fehlende Zusammenfassung." #, fuzzy #~ msgid "Missing pool name" #~ msgstr "Profilname kann nicht geändert werden" #, fuzzy #~ msgid "Missing project name" #~ msgstr "Profilname kann nicht geändert werden" #, fuzzy #~ msgid "Missing source volume name" #~ msgstr "Kein Zertifikat für diese Verbindung" #, fuzzy #~ msgid "Missing storage pool name" #~ msgstr "Profilname kann nicht geändert werden" #, fuzzy #~ msgid "Missing target network or integration" #~ msgstr "Profilname kann nicht geändert werden" #, fuzzy #~ msgid "Must supply instance name for: " #~ msgstr "der Name des Ursprung Containers muss angegeben werden" #, fuzzy, c-format #~ msgid "No value found in %q" #~ msgstr "kein Wert in %q gefunden\n" #, fuzzy #~ msgid "You must specify a source instance name" #~ msgstr "der Name des Ursprung Containers muss angegeben werden" #, fuzzy #~ msgid "You need to specify an image name or use --empty" #~ msgstr "der Name des Ursprung Containers muss angegeben werden" #~ msgid " " #~ msgstr " " #, fuzzy #~ msgid ": [:]" #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #~ msgid " " #~ msgstr " " #, fuzzy #~ msgid ": " #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid " [:] /" #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "... [:]/" #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[] []" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[] [] []" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] []" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:] [...]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] [...]" #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] =..." #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] =..." #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] [[:]...]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] [key=value...]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:]" #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] =..." #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] [[:]...]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] [key=value...]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:]" #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:]
..." #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] =..." #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] [[:]...]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] [key=value...]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] [[:]...]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:]" #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] =..." #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] [[:]...]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:]" #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] :" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] [:]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] [:][]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] []" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] [[:]...]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:]" #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] =..." #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] [key=value...]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] [key=value...]" #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] ..." #~ msgstr "" #~ "Ändert den Laufzustand eines Containers in %s.\n" #~ "\n" #~ "lxd %s \n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:] " #~ msgstr "" #~ "Löscht einen Container oder Container Sicherungspunkt.\n" #~ "\n" #~ "Entfernt einen Container (oder Sicherungspunkt) und alle dazugehörigen\n" #~ "Daten (Konfiguration, Sicherungspunkte, ...).\n" #, fuzzy #~ msgid "[:]